diff --git a/Documentation/ABI/testing/configfs-stp-policy-p_ost b/Documentation/ABI/testing/configfs-stp-policy-p_ost new file mode 100644 index 0000000000000..498739b49da06 --- /dev/null +++ b/Documentation/ABI/testing/configfs-stp-policy-p_ost @@ -0,0 +1,6 @@ +What: /config/stp-policy/:p_ost.//entity +Date: Oct 2025 +KernelVersion: 6.18 +Description: + Set the entity which is to identify the source, RW. + diff --git a/Documentation/ABI/testing/sysfs-bus-amba-devices-tgu b/Documentation/ABI/testing/sysfs-bus-amba-devices-tgu new file mode 100644 index 0000000000000..a6b6019c8ef17 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-amba-devices-tgu @@ -0,0 +1,51 @@ +What: /sys/bus/amba/devices//enable_tgu +Date: April 2026 +KernelVersion: 7.1 +Contact: Jinlong Mao , Songwei Chai +Description: + (RW) Set/Get the enable/disable status of TGU + Accepts only one of the 2 values - 0 or 1. + 0 : disable TGU. + 1 : enable TGU. + +What: /sys/bus/amba/devices//step[0:7]_priority[0:3]/reg[0:17] +Date: April 2026 +KernelVersion: 7.1 +Contact: Jinlong Mao , Songwei Chai +Description: + (RW) Set/Get the sensed signal with specific step and priority for TGU. + +What: /sys/bus/amba/devices//step[0:7]_condition_decode/reg[0:3] +Date: April 2026 +KernelVersion: 7.1 +Contact: Jinlong Mao , Songwei Chai +Description: + (RW) Set/Get the decode mode with specific step for TGU. + +What: /sys/bus/amba/devices//step[0:7]_condition_select/reg[0:3] +Date: April 2026 +KernelVersion: 7.1 +Contact: Jinlong Mao , Songwei Chai +Description: + (RW) Set/Get the next action with specific step for TGU. + +What: /sys/bus/amba/devices//step[0:7]_timer/reg[0:1] +Date: April 2026 +KernelVersion: 7.1 +Contact: Jinlong Mao , Songwei Chai +Description: + (RW) Set/Get the timer value with specific step for TGU. + +What: /sys/bus/amba/devices//step[0:7]_counter/reg[0:1] +Date: April 2026 +KernelVersion: 7.1 +Contact: Jinlong Mao , Songwei Chai +Description: + (RW) Set/Get the counter value with specific step for TGU. + +What: /sys/bus/amba/devices//reset_tgu +Date: April 2026 +KernelVersion: 7.1 +Contact: Jinlong Mao , Songwei Chai +Description: + (Write) Write 1 to reset the dataset for TGU. diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-ctcu b/Documentation/ABI/testing/sysfs-bus-coresight-devices-ctcu new file mode 100644 index 0000000000000..fa64ea8e6e69b --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-ctcu @@ -0,0 +1,6 @@ +What: /sys/bus/coresight/devices//irq_threshold +Date: Oct. 2025 +KernelVersion: 6.19 +Contact: Tingwei Zhang ; Jinlong Mao ; Jie Gan +Description: + (RW) Configure the IRQ value for byte-cntr register. diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpda b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpda new file mode 100644 index 0000000000000..2cf2dcfc13c81 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpda @@ -0,0 +1,50 @@ +What: /sys/bus/coresight/devices//trig_async_enable +Date: August 2025 +KernelVersion: 6.17 +Contact: Jinlong Mao , Tao Zhang , Jie Gan +Description: + (RW) Enable/disable cross trigger synchronization sequence interface. + +What: /sys/bus/coresight/devices//trig_flag_ts_enable +Date: August 2025 +KernelVersion: 6.17 +Contact: Jinlong Mao , Tao Zhang , Jie Gan +Description: + (RW) Enable/disable cross trigger FLAG packet request interface. + +What: /sys/bus/coresight/devices//trig_freq_enable +Date: August 2025 +KernelVersion: 6.17 +Contact: Jinlong Mao , Tao Zhang , Jie Gan +Description: + (RW) Enable/disable cross trigger FREQ packet request interface. + +What: /sys/bus/coresight/devices//freq_ts_enable +Date: August 2025 +KernelVersion: 6.17 +Contact: Jinlong Mao , Tao Zhang , Jie Gan +Description: + (RW) Enable/disable the timestamp for all FREQ packets. + +What: /sys/bus/coresight/devices//global_flush_req +Date: August 2025 +KernelVersion: 6.17 +Contact: Jinlong Mao , Tao Zhang , Jie Gan +Description: + (RW) Set global (all ports) flush request bit. The bit remains set until a + global flush request sequence completes. + +What: /sys/bus/coresight/devices//cmbchan_mode +Date: August 2025 +KernelVersion: 6.17 +Contact: Jinlong Mao , Tao Zhang , Jie Gan +Description: + (RW) Configure the CMB/MCMB channel mode for all enabled ports. + Value 0 means raw channel mapping mode. Value 1 means channel pair marking mode. + +What: /sys/bus/coresight/devices//port_flush_req +Date: August 2025 +KernelVersion: 6.17 +Contact: Jinlong Mao , Tao Zhang , Jie Gan +Description: + (RW) Configure the bit i to requests a flush operation of port i on the TPDA. diff --git a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes new file mode 100644 index 0000000000000..6a3fc379afae3 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes @@ -0,0 +1,39 @@ +What: /sys/class/reboot-mode//reboot_modes +Date: August 2025 +KernelVersion: 6.17.0-rc1 +Contact: linux-pm@vger.kernel.org + Description: + This interface exposes the reboot-mode arguments + registered with the reboot-mode framework. It is + a read-only interface and provides a space + separated list of reboot-mode arguments supported + on the current platform. + Example: + recovery fastboot bootloader + + The exact sysfs path may vary depending on the + name of the driver that registers the arguments. + Example: + /sys/class/reboot-mode/nvmem-reboot-mode/reboot_modes + /sys/class/reboot-mode/syscon-reboot-mode/reboot_modes + /sys/class/reboot-mode/qcom-pon/reboot_modes + + The supported arguments can be used by userspace + to invoke device reset using the reboot() system + call, with the "argument" as string to "*arg" + parameter along with LINUX_REBOOT_CMD_RESTART2. + Example: + reboot(LINUX_REBOOT_MAGIC1, LINUX_REBOOT_MAGIC2, + LINUX_REBOOT_CMD_RESTART2, "bootloader"); + + A driver can expose the supported arguments by + registering them with the reboot-mode framework + using the property names that follow the + mode- format. + Example: + mode-bootloader, mode-recovery. + + This attribute is useful for scripts or initramfs + logic that need to programmatically determine + which reboot-mode arguments are valid before + triggering a reboot. diff --git a/Documentation/ABI/testing/sysfs-driver-eud b/Documentation/ABI/testing/sysfs-driver-eud index 2bab0db2d2f0f..a2a4fc5a9915e 100644 --- a/Documentation/ABI/testing/sysfs-driver-eud +++ b/Documentation/ABI/testing/sysfs-driver-eud @@ -7,3 +7,19 @@ Description: EUD based on a 1 or a 0 value. By enabling EUD, the user is able to activate the mini-usb hub of EUD for debug and trace capabilities. + +What: /sys/bus/platform/drivers/qcom_eud/.../port +Date: January 2026 +Contact: Elson Serrao +Description: + Selects which USB port the Embedded USB Debugger (EUD) + is mapped to on platforms providing multiple High-Speed + USB ports. + + Valid values: + primary - Primary USB port + secondary - Secondary USB port + + The attribute is writable only while EUD is disabled. + Reading the attribute returns the currently selected + USB port. diff --git a/Documentation/PCI/endpoint/pci-vntb-howto.rst b/Documentation/PCI/endpoint/pci-vntb-howto.rst index 9a7a2f0a68498..3679f5c302549 100644 --- a/Documentation/PCI/endpoint/pci-vntb-howto.rst +++ b/Documentation/PCI/endpoint/pci-vntb-howto.rst @@ -52,14 +52,14 @@ pci-epf-vntb device, the following commands can be used:: # cd /sys/kernel/config/pci_ep/ # mkdir functions/pci_epf_vntb/func1 -The "mkdir func1" above creates the pci-epf-ntb function device that will +The "mkdir func1" above creates the pci-epf-vntb function device that will be probed by pci_epf_vntb driver. The PCI endpoint framework populates the directory with the following configurable fields:: - # ls functions/pci_epf_ntb/func1 - baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + # ls functions/pci_epf_vntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-vntb.0 progif_code secondary subsys_id vendorid cache_line_size interrupt_pin msix_interrupts primary revid subclass_code subsys_vendor_id @@ -111,13 +111,13 @@ A sample configuration for virtual NTB driver for virtual PCI bus:: # echo 0x080A > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vntb_pid # echo 0x10 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vbus_number -Binding pci-epf-ntb Device to EP Controller +Binding pci-epf-vntb Device to EP Controller -------------------------------------------- NTB function device should be attached to PCI endpoint controllers connected to the host. - # ln -s controllers/5f010000.pcie_ep functions/pci-epf-ntb/func1/primary + # ln -s controllers/5f010000.pcie_ep functions/pci_epf_vntb/func1/primary Once the above step is completed, the PCI endpoint controllers are ready to establish a link with the host. @@ -139,7 +139,7 @@ lspci Output at Host side ------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 00:00.0 PCI bridge: Freescale Semiconductor Inc Device 0000 (rev 01) @@ -152,7 +152,7 @@ lspci Output at EP Side / Virtual PCI bus ----------------------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 10:00.0 Unassigned class [ffff]: Dawicontrol Computersysteme GmbH Device 1234 (rev ff) diff --git a/Documentation/admin-guide/LSM/Smack.rst b/Documentation/admin-guide/LSM/Smack.rst index 6d44f4fdbf59f..c5ed775f2d107 100644 --- a/Documentation/admin-guide/LSM/Smack.rst +++ b/Documentation/admin-guide/LSM/Smack.rst @@ -601,10 +601,15 @@ specification. Task Attribute ~~~~~~~~~~~~~~ -The Smack label of a process can be read from /proc//attr/current. A -process can read its own Smack label from /proc/self/attr/current. A +The Smack label of a process can be read from ``/proc//attr/current``. A +process can read its own Smack label from ``/proc/self/attr/current``. A privileged process can change its own Smack label by writing to -/proc/self/attr/current but not the label of another process. +``/proc/self/attr/current`` but not the label of another process. + +Format of writing is : only the label or the label followed by one of the +3 trailers: ``\n`` (by common agreement for ``/proc/...`` interfaces), +``\0`` (because some applications incorrectly include it), +``\n\0`` (because we think some applications may incorrectly include it). File Attribute ~~~~~~~~~~~~~~ @@ -696,6 +701,11 @@ sockets. A privileged program may set this to match the label of another task with which it hopes to communicate. +UNIX domain socket (UDS) with a BSD address functions both as a file in a +filesystem and as a socket. As a file, it carries the SMACK64 attribute. This +attribute is not involved in Smack security enforcement and is immutably +assigned the label "*". + Smack Netlabel Exceptions ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 0e6c67ac585a0..edcd125d6992f 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2538,10 +2538,10 @@ Cpuset Interface Files Users can manually set it to a value that is different from "cpuset.cpus". One constraint in setting it is that the list of CPUs must be exclusive with respect to "cpuset.cpus.exclusive" - of its sibling. If "cpuset.cpus.exclusive" of a sibling cgroup - isn't set, its "cpuset.cpus" value, if set, cannot be a subset - of it to leave at least one CPU available when the exclusive - CPUs are taken away. + and "cpuset.cpus.exclusive.effective" of its siblings. Another + constraint is that it cannot be a superset of "cpuset.cpus" + of its sibling in order to leave at least one CPU available to + that sibling when the exclusive CPUs are taken away. For a parent cgroup, any one of its exclusive CPUs can only be distributed to at most one of its child cgroups. Having an diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6c42061ca20e5..ab4e03f91e744 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7925,6 +7925,9 @@ p = USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT (Reduce timeout of the SET_ADDRESS request from 5000 ms to 500 ms); + q = USB_QUIRK_FORCE_ONE_CONFIG (Device + claims zero configurations, + forcing to 1); Example: quirks=0781:5580:bk,0a5c:5834:gij usbhid.mousepoll= diff --git a/Documentation/admin-guide/mm/damon/reclaim.rst b/Documentation/admin-guide/mm/damon/reclaim.rst index af05ae6170184..26aa86a654de6 100644 --- a/Documentation/admin-guide/mm/damon/reclaim.rst +++ b/Documentation/admin-guide/mm/damon/reclaim.rst @@ -71,6 +71,10 @@ of parametrs except ``enabled`` again. Once the re-reading is done, this parameter is set as ``N``. If invalid parameters are found while the re-reading, DAMON_RECLAIM will be disabled. +Once ``Y`` is written to this parameter, the user must not write to any +parameters until reading ``commit_inputs`` again returns ``N``. If users +violate this rule, the kernel may exhibit undefined behavior. + min_age ------- diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index a7ec57060f64f..93cdf16937159 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -202,6 +202,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| | | | #562869,1047329 | | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml index 2a91670ccb8cd..16b30ecb9e4ab 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml @@ -84,7 +84,9 @@ properties: - const: arm,coresight-cti - const: arm,primecell - items: - - const: arm,coresight-cti-v8-arch + - enum: + - arm,coresight-cti-v8-arch + - qcom,coresight-cti-extended - const: arm,coresight-cti - const: arm,primecell diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml index b74db15e5f8af..8f32d4e3bbb75 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml @@ -28,19 +28,32 @@ select: properties: compatible: contains: - const: arm,coresight-dynamic-funnel + enum: + - arm,coresight-dynamic-funnel + - arm,coresight-cpu-funnel required: - compatible allOf: - $ref: /schemas/arm/primecell.yaml# + - if: + properties: + compatible: + contains: + const: arm,coresight-cpu-funnel + then: + required: + - power-domains + properties: compatible: - items: - - const: arm,coresight-dynamic-funnel - - const: arm,primecell - + oneOf: + - items: + - const: arm,coresight-dynamic-funnel + - const: arm,primecell + - items: + - const: arm,coresight-cpu-funnel reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml index 17ea936b796fd..5ce30c4e9c415 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml @@ -28,18 +28,32 @@ select: properties: compatible: contains: - const: arm,coresight-dynamic-replicator + enum: + - arm,coresight-dynamic-replicator + - arm,coresight-cpu-replicator required: - compatible allOf: - $ref: /schemas/arm/primecell.yaml# + - if: + properties: + compatible: + contains: + const: arm,coresight-cpu-replicator + then: + required: + - power-domains + properties: compatible: - items: - - const: arm,coresight-dynamic-replicator - - const: arm,primecell + oneOf: + - items: + - const: arm,coresight-dynamic-replicator + - const: arm,primecell + - items: + - const: arm,coresight-cpu-replicator reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml index 96dd5b5f771a3..d7c0b618fe98a 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml @@ -29,18 +29,32 @@ select: properties: compatible: contains: - const: arm,coresight-tmc + enum: + - arm,coresight-tmc + - arm,coresight-cpu-tmc required: - compatible allOf: - $ref: /schemas/arm/primecell.yaml# + - if: + properties: + compatible: + contains: + const: arm,coresight-cpu-tmc + then: + required: + - power-domains + properties: compatible: - items: - - const: arm,coresight-tmc - - const: arm,primecell + oneOf: + - items: + - const: arm,coresight-tmc + - const: arm,primecell + - items: + - const: arm,coresight-cpu-tmc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/arm/psci.yaml b/Documentation/devicetree/bindings/arm/psci.yaml index 7360a2849b5bd..eca38f8747d32 100644 --- a/Documentation/devicetree/bindings/arm/psci.yaml +++ b/Documentation/devicetree/bindings/arm/psci.yaml @@ -98,6 +98,27 @@ properties: [1] Kernel documentation - ARM idle states bindings Documentation/devicetree/bindings/cpu/idle-states.yaml + reboot-mode: + type: object + $ref: /schemas/power/reset/reboot-mode.yaml# + unevaluatedProperties: false + properties: + # "mode-normal" is just SYSTEM_RESET + mode-normal: false + patternProperties: + "^mode-.*$": + minItems: 1 + maxItems: 2 + description: | + Describes a vendor-specific reset type. The string after "mode-" + maps a reboot mode to the parameters in the PSCI SYSTEM_RESET2 call. + + Parameters are named mode-xxx = , where xxx + is the name of the magic reboot mode, type is the lower 31 bits + of the reset_type, and, optionally, the cookie value. If the cookie + is not provided, it is defaulted to zero. + The 31st bit (vendor-resets) will be implicitly set by the driver. + patternProperties: "^power-domain-": $ref: /schemas/power/power-domain.yaml# @@ -137,6 +158,15 @@ allOf: required: - cpu_off - cpu_on + - if: + not: + properties: + compatible: + contains: + const: arm,psci-1.0 + then: + properties: + reboot-mode: false additionalProperties: false @@ -261,4 +291,17 @@ examples: domain-idle-states = <&cluster_ret>, <&cluster_pwrdn>; }; }; + + - |+ + + // Case 5: SYSTEM_RESET2 vendor resets + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + + reboot-mode { + mode-edl = <0>; + mode-bootloader = <1 2>; + }; + }; ... diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml index c969c16c21ef7..02797e5f3b586 100644 --- a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml +++ b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml @@ -39,6 +39,16 @@ properties: items: - const: apb + interrupts: + items: + - description: Byte cntr interrupt for etr0 + - description: Byte cntr interrupt for etr1 + + interrupt-names: + items: + - const: etr0 + - const: etr1 + label: description: Description of a coresight device. @@ -60,6 +70,8 @@ additionalProperties: false examples: - | + #include + ctcu@1001000 { compatible = "qcom,sa8775p-ctcu"; reg = <0x1001000 0x1000>; @@ -67,6 +79,11 @@ examples: clocks = <&aoss_qmp>; clock-names = "apb"; + interrupts = , + ; + interrupt-names = "etr0", + "etr1"; + in-ports { #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml index 4edc47483851f..c349306f0d520 100644 --- a/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml +++ b/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml @@ -36,9 +36,12 @@ properties: $nodename: pattern: "^tpdm(@[0-9a-f]+)$" compatible: - items: - - const: qcom,coresight-tpdm - - const: arm,primecell + oneOf: + - items: + - const: qcom,coresight-static-tpdm + - items: + - const: qcom,coresight-tpdm + - const: arm,primecell reg: maxItems: 1 @@ -147,4 +150,18 @@ examples: }; }; }; + + turing-llm-tpdm { + compatible = "qcom,coresight-static-tpdm"; + + qcom,cmb-element-bits = <32>; + + out-ports { + port { + turing_llm_tpdm_out: endpoint { + remote-endpoint = <&turing0_funnel_in1>; + }; + }; + }; + }; ... diff --git a/Documentation/devicetree/bindings/arm/qcom,tgu.yaml b/Documentation/devicetree/bindings/arm/qcom,tgu.yaml new file mode 100644 index 0000000000000..76440f2497b97 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/qcom,tgu.yaml @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/arm/qcom,tgu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Trigger Generation Unit - TGU + +description: | + The Trigger Generation Unit (TGU) is a Data Engine which can be utilized + to sense a plurality of signals and create a trigger into the CTI or + generate interrupts to processors. The TGU is like the trigger circuit + of a Logic Analyzer. The corresponding trigger logic can be realized by + configuring the conditions for each step after sensing the signal. + Once setup and enabled, it will observe sense inputs and based upon + the activity of those inputs, even over clock cycles, may detect a + preprogrammed state/sequence and then produce a trigger or interrupt. + + The primary use case of the TGU is to detect patterns or sequences on a + given set of signals within some region to identify the issue in time + once there is abnormal behavior in the subsystem. + +maintainers: + - Mao Jinlong + - Songwei Chai + +# Need a custom select here or 'arm,primecell' will match on lots of nodes +select: + properties: + compatible: + contains: + enum: + - qcom,tgu + required: + - compatible + +properties: + compatible: + items: + - const: qcom,tgu + - const: arm,primecell + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: apb_pclk + +required: + - compatible + - reg + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + tgu@10b0e000 { + compatible = "qcom,tgu", "arm,primecell"; + reg = <0x10b0e000 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; +... diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml index 18b5ed044f9fc..e8f461b987f12 100644 --- a/Documentation/devicetree/bindings/arm/qcom.yaml +++ b/Documentation/devicetree/bindings/arm/qcom.yaml @@ -61,6 +61,11 @@ properties: - qcom,apq8084-sbc - const: qcom,apq8084 + - items: + - enum: + - qcom,glymur-crd + - const: qcom,glymur + - items: - enum: - microsoft,dempsey @@ -321,6 +326,12 @@ properties: - qcom,ipq9574-ap-al02-c9 - const: qcom,ipq9574 + - items: + - enum: + - qcom,kaanapali-mtp + - qcom,kaanapali-qrd + - const: qcom,kaanapali + - description: Sierra Wireless MangOH Green with WP8548 Module items: - const: swir,mangoh-green-wp8548 @@ -856,6 +867,7 @@ properties: - items: - enum: + - qcom,monaco-ac-evk - qcom,monaco-evk - qcom,qcs8300-ride - const: qcom,qcs8300 @@ -863,6 +875,7 @@ properties: - items: - enum: - qcom,qcs615-ride + - qcom,talos-evk - const: qcom,qcs615 - const: qcom,sm6150 @@ -1034,6 +1047,7 @@ properties: - items: - enum: + - ayaneo,pocket-s2 - qcom,sm8650-hdk - qcom,sm8650-mtp - qcom,sm8650-qrd @@ -1083,7 +1097,19 @@ properties: - items: - enum: - - asus,zenbook-a14-ux3407qa + - qcom,purwa-iot-evk + - const: qcom,purwa-iot-som + - const: qcom,x1p42100 + + - items: + - enum: + - asus,zenbook-a14-ux3407qa-lcd + - asus,zenbook-a14-ux3407qa-oled + - const: asus,zenbook-a14-ux3407qa + - const: qcom,x1p42100 + + - items: + - enum: - hp,omnibook-x14-fe1 - lenovo,thinkbook-16 - qcom,x1p42100-crd diff --git a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml index 37e3ebd554874..a620a2ff5c568 100644 --- a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml +++ b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml @@ -21,6 +21,7 @@ properties: compatible: enum: - qcom,ipq5424-llcc + - qcom,kaanapali-llcc - qcom,qcs615-llcc - qcom,qcs8300-llcc - qcom,qdu1000-llcc @@ -272,6 +273,7 @@ allOf: compatible: contains: enum: + - qcom,kaanapali-llcc - qcom,sm8450-llcc - qcom,sm8550-llcc - qcom,sm8650-llcc diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml index 78fa057266857..3f5f1336262ee 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml @@ -18,6 +18,7 @@ properties: compatible: enum: - qcom,glymur-rpmh-clk + - qcom,kaanapali-rpmh-clk - qcom,milos-rpmh-clk - qcom,qcs615-rpmh-clk - qcom,qdu1000-rpmh-clk diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml index fcd2727dae467..6fee5c4f01b03 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml @@ -17,6 +17,7 @@ description: | See also: include/dt-bindings/clock/qcom,sm8450-videocc.h include/dt-bindings/clock/qcom,sm8650-videocc.h + include/dt-bindings/clock/qcom,x1p42100-videocc.h properties: compatible: @@ -26,6 +27,7 @@ properties: - qcom,sm8550-videocc - qcom,sm8650-videocc - qcom,x1e80100-videocc + - qcom,x1p42100-videocc clocks: items: @@ -61,6 +63,7 @@ allOf: enum: - qcom,sm8450-videocc - qcom,sm8550-videocc + - qcom,x1p42100-videocc then: required: - required-opps diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml index 2c992b3437f29..784fef8306812 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml @@ -25,6 +25,7 @@ properties: items: - enum: - qcom,glymur-tcsr + - qcom,kaanapali-tcsr - qcom,milos-tcsr - qcom,sar2130p-tcsr - qcom,sm8550-tcsr diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml index aab7039fd28db..0114d347b26ff 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml @@ -13,11 +13,15 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on SM8750 - See also: include/dt-bindings/clock/qcom,sm8750-gcc.h + See also: + include/dt-bindings/clock/qcom,kaanapali-gcc.h + include/dt-bindings/clock/qcom,sm8750-gcc.h properties: compatible: - const: qcom,sm8750-gcc + enum: + - qcom,kaanapali-gcc + - qcom,sm8750-gcc clocks: items: diff --git a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml index 938a2f1ff3fca..b28614186cc09 100644 --- a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml @@ -23,6 +23,7 @@ properties: compatible: enum: - qcom,x1e80100-camcc + - qcom,x1p42100-camcc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/clock/qcom,x1e80100-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,x1e80100-gcc.yaml index 68dde0720c711..1b15b50709545 100644 --- a/Documentation/devicetree/bindings/clock/qcom,x1e80100-gcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,x1e80100-gcc.yaml @@ -32,9 +32,36 @@ properties: - description: PCIe 5 pipe clock - description: PCIe 6a pipe clock - description: PCIe 6b pipe clock - - description: USB QMP Phy 0 clock source - - description: USB QMP Phy 1 clock source - - description: USB QMP Phy 2 clock source + - description: USB4_0 QMPPHY clock source + - description: USB4_1 QMPPHY clock source + - description: USB4_2 QMPPHY clock source + - description: USB4_0 PHY DP0 GMUX clock source + - description: USB4_0 PHY DP1 GMUX clock source + - description: USB4_0 PHY PCIE PIPEGMUX clock source + - description: USB4_0 PHY PIPEGMUX clock source + - description: USB4_0 PHY SYS PCIE PIPEGMUX clock source + - description: USB4_1 PHY DP0 GMUX 2 clock source + - description: USB4_1 PHY DP1 GMUX 2 clock source + - description: USB4_1 PHY PCIE PIPEGMUX clock source + - description: USB4_1 PHY PIPEGMUX clock source + - description: USB4_1 PHY SYS PCIE PIPEGMUX clock source + - description: USB4_2 PHY DP0 GMUX 2 clock source + - description: USB4_2 PHY DP1 GMUX 2 clock source + - description: USB4_2 PHY PCIE PIPEGMUX clock source + - description: USB4_2 PHY PIPEGMUX clock source + - description: USB4_2 PHY SYS PCIE PIPEGMUX clock source + - description: USB4_0 PHY RX 0 clock source + - description: USB4_0 PHY RX 1 clock source + - description: USB4_1 PHY RX 0 clock source + - description: USB4_1 PHY RX 1 clock source + - description: USB4_2 PHY RX 0 clock source + - description: USB4_2 PHY RX 1 clock source + - description: USB4_0 PHY PCIE PIPE clock source + - description: USB4_0 PHY max PIPE clock source + - description: USB4_1 PHY PCIE PIPE clock source + - description: USB4_1 PHY max PIPE clock source + - description: USB4_2 PHY PCIE PIPE clock source + - description: USB4_2 PHY max PIPE clock source power-domains: description: @@ -67,7 +94,34 @@ examples: <&pcie6b_phy>, <&usb_1_ss0_qmpphy 0>, <&usb_1_ss1_qmpphy 1>, - <&usb_1_ss2_qmpphy 2>; + <&usb_1_ss2_qmpphy 2>, + <&usb4_0_phy_dp0_gmux_clk>, + <&usb4_0_phy_dp1_gmux_clk>, + <&usb4_0_phy_pcie_pipegmux_clk>, + <&usb4_0_phy_pipegmux_clk>, + <&usb4_0_phy_sys_pcie_pipegmux_clk>, + <&usb4_1_phy_dp0_gmux_2_clk>, + <&usb4_1_phy_dp1_gmux_2_clk>, + <&usb4_1_phy_pcie_pipegmux_clk>, + <&usb4_1_phy_pipegmux_clk>, + <&usb4_1_phy_sys_pcie_pipegmux_clk>, + <&usb4_2_phy_dp0_gmux_2_clk>, + <&usb4_2_phy_dp1_gmux_2_clk>, + <&usb4_2_phy_pcie_pipegmux_clk>, + <&usb4_2_phy_pipegmux_clk>, + <&usb4_2_phy_sys_pcie_pipegmux_clk>, + <&usb4_0_phy_rx_0_clk>, + <&usb4_0_phy_rx_1_clk>, + <&usb4_1_phy_rx_0_clk>, + <&usb4_1_phy_rx_1_clk>, + <&usb4_2_phy_rx_0_clk>, + <&usb4_2_phy_rx_1_clk>, + <&usb4_0_phy_pcie_pipe_clk>, + <&usb4_0_phy_max_pipe_clk>, + <&usb4_1_phy_pcie_pipe_clk>, + <&usb4_1_phy_max_pipe_clk>, + <&usb4_2_phy_pcie_pipe_clk>, + <&usb4_2_phy_max_pipe_clk>; power-domains = <&rpmhpd RPMHPD_CX>; #clock-cells = <1>; #reset-cells = <1>; diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml index 08fe6a707a371..4c0acd36fca11 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml @@ -13,6 +13,7 @@ properties: compatible: items: - enum: + - qcom,kaanapali-inline-crypto-engine - qcom,qcs8300-inline-crypto-engine - qcom,sa8775p-inline-crypto-engine - qcom,sc7180-inline-crypto-engine @@ -27,8 +28,23 @@ properties: maxItems: 1 clocks: + minItems: 1 + maxItems: 2 + + clock-names: + minItems: 1 + items: + - const: core + - const: iface + + power-domains: maxItems: 1 + operating-points-v2: true + + opp-table: + type: object + required: - compatible - reg @@ -36,6 +52,25 @@ required: additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,eliza-inline-crypto-engine + - qcom,milos-inline-crypto-engine + + then: + required: + - power-domains + - clock-names + properties: + clocks: + minItems: 2 + clock-names: + minItems: 2 + examples: - | #include @@ -44,6 +79,31 @@ examples: compatible = "qcom,sm8550-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0x01d88000 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; + + operating-points-v2 = <&ice_opp_table>; + + ice_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-201500000 { + opp-hz = /bits/ 64 <201500000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-403000000 { + opp-hz = /bits/ 64 <403000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; }; ... diff --git a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml index ed7e16bd11d33..aa3c097a6acdb 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml @@ -29,6 +29,7 @@ properties: - qcom,sm8550-trng - qcom,sm8650-trng - qcom,sm8750-trng + - qcom,x1e80100-trng - const: qcom,trng reg: diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt8713sx.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt8713sx.yaml new file mode 100644 index 0000000000000..0a6dc56e337cb --- /dev/null +++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt8713sx.yaml @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/bridge/lontium,lt8713sx.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Lontium LT8713SX Type-C/DP1.4 to Type-C/DP1.4/HDMI2.0/DP++ bridge-hub + +maintainers: + - Tony + +description: + The Lontium LT8713SX is a Type-C/DP1.4 to Type-C/DP1.4/HDMI2.0 converter + that integrates one DP input and up to three configurable output interfaces + (DP1.4 / HDMI2.0 / DP++), with SST/MST functionality and audio support. + +properties: + compatible: + enum: + - lontium,lt8713sx + + reg: + maxItems: 1 + + vcc-supply: + description: Regulator for 3.3V vcc. + + vdd-supply: + description: Regulator for 1.1V vdd. + + reset-gpios: + description: GPIO connected to active low RESET pin. + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + properties: + port@0: + $ref: /schemas/graph.yaml#/properties/port + description: + DP port for DP input from soc to bridge chip + + port@1: + $ref: /schemas/graph.yaml#/properties/port + description: + DP port for DP output from bridge + + port@2: + $ref: /schemas/graph.yaml#/properties/port + description: + Additional DP port for DP output from bridge + + required: + - port@0 + +required: + - compatible + - reg + - ports + +additionalProperties: false + +examples: + - | + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + bridge@4f { + compatible = "lontium,lt8713sx"; + reg = <0x4f>; + reset-gpios = <&tlmm 6 GPIO_ACTIVE_LOW>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + lt8713sx_dp_in: endpoint { + remote-endpoint = <&mdss_dp0_out>; + }; + }; + + port@1 { + reg = <1>; + lt8713sx_dp0_out: endpoint { + remote-endpoint = <&dp0_connector_in>; + }; + }; + + port@2 { + reg = <2>; + lt8713sx_dp1_out: endpoint { + remote-endpoint = <&dp1_connector_in>; + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml index 9a6e9b25d14a9..98079df61cbb3 100644 --- a/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml +++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml @@ -4,19 +4,20 @@ $id: http://devicetree.org/schemas/display/bridge/lontium,lt9211.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Lontium LT9211 DSI/LVDS/DPI to DSI/LVDS/DPI bridge. +title: Lontium LT9211/LT9211C DSI/LVDS/DPI to DSI/LVDS/DPI bridge. maintainers: - Marek Vasut description: | - The LT9211 are bridge devices which convert Single/Dual-Link DSI/LVDS + The LT9211 and LT9211C are bridge devices which convert Single/Dual-Link DSI/LVDS or Single DPI to Single/Dual-Link DSI/LVDS or Single DPI. properties: compatible: enum: - lontium,lt9211 + - lontium,lt9211c reg: maxItems: 1 @@ -91,6 +92,43 @@ examples: vccio-supply = <<9211_1v8>; + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + endpoint { + remote-endpoint = <&dsi0_out>; + }; + }; + + port@2 { + reg = <2>; + + endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; + }; + }; + - | + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + lvds-bridge@3b { + compatible = "lontium,lt9211c"; + reg = <0x3b>; + + reset-gpios = <&tlmm 128 GPIO_ACTIVE_HIGH>; + + vccio-supply = <<9211c_1v8>; + ports { #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml index aeb4e4f36044a..d26e51db7b6bb 100644 --- a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml +++ b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml @@ -31,6 +31,11 @@ properties: - qcom,sm8650-dp - qcom,x1e80100-dp + - items: + - enum: + - qcom,qcs8300-dp + - const: qcom,sa8775p-dp + - items: - enum: - qcom,sm6350-dp @@ -53,6 +58,12 @@ properties: - qcom,sm8550-dp - const: qcom,sm8350-dp + - items: + - enum: + - qcom,sm6150-dp + - const: qcom,sm8150-dp + - const: qcom,sm8350-dp + - items: - enum: - qcom,sm8750-dp diff --git a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml index 4400d4cce0722..6276350e582f7 100644 --- a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml +++ b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml @@ -45,6 +45,11 @@ properties: - qcom,sm8650-dsi-ctrl - qcom,sm8750-dsi-ctrl - const: qcom,mdss-dsi-ctrl + - items: + - enum: + - qcom,qcs8300-dsi-ctrl + - const: qcom,sa8775p-dsi-ctrl + - const: qcom,mdss-dsi-ctrl - enum: - qcom,dsi-ctrl-6g-qcm2290 - qcom,mdss-dsi-ctrl # This should always come with an SoC-specific compatible diff --git a/Documentation/devicetree/bindings/display/msm/dsi-phy-7nm.yaml b/Documentation/devicetree/bindings/display/msm/dsi-phy-7nm.yaml index 1ca820a500b72..7a83387502da1 100644 --- a/Documentation/devicetree/bindings/display/msm/dsi-phy-7nm.yaml +++ b/Documentation/devicetree/bindings/display/msm/dsi-phy-7nm.yaml @@ -14,18 +14,24 @@ allOf: properties: compatible: - enum: - - qcom,dsi-phy-7nm - - qcom,dsi-phy-7nm-8150 - - qcom,sa8775p-dsi-phy-5nm - - qcom,sar2130p-dsi-phy-5nm - - qcom,sc7280-dsi-phy-7nm - - qcom,sm6375-dsi-phy-7nm - - qcom,sm8350-dsi-phy-5nm - - qcom,sm8450-dsi-phy-5nm - - qcom,sm8550-dsi-phy-4nm - - qcom,sm8650-dsi-phy-4nm - - qcom,sm8750-dsi-phy-3nm + oneOf: + - items: + - enum: + - qcom,dsi-phy-7nm + - qcom,dsi-phy-7nm-8150 + - qcom,sa8775p-dsi-phy-5nm + - qcom,sar2130p-dsi-phy-5nm + - qcom,sc7280-dsi-phy-7nm + - qcom,sm6375-dsi-phy-7nm + - qcom,sm8350-dsi-phy-5nm + - qcom,sm8450-dsi-phy-5nm + - qcom,sm8550-dsi-phy-4nm + - qcom,sm8650-dsi-phy-4nm + - qcom,sm8750-dsi-phy-3nm + - items: + - enum: + - qcom,qcs8300-dsi-phy-5nm + - const: qcom,sa8775p-dsi-phy-5nm reg: items: diff --git a/Documentation/devicetree/bindings/display/msm/gpu.yaml b/Documentation/devicetree/bindings/display/msm/gpu.yaml index 3696b083e3530..52294cfad7410 100644 --- a/Documentation/devicetree/bindings/display/msm/gpu.yaml +++ b/Documentation/devicetree/bindings/display/msm/gpu.yaml @@ -45,11 +45,11 @@ properties: - const: amd,imageon clocks: - minItems: 2 + minItems: 1 maxItems: 7 clock-names: - minItems: 2 + minItems: 1 maxItems: 7 reg: @@ -388,26 +388,68 @@ allOf: required: - clocks - clock-names - else: - if: - properties: - compatible: - contains: - oneOf: - - pattern: '^qcom,adreno-[67][0-9][0-9]\.[0-9]+$' - - pattern: '^qcom,adreno-[0-9a-f]{8}$' - - then: # Starting with A6xx, the clocks are usually defined in the GMU node - properties: - clocks: false - clock-names: false - - reg-names: - minItems: 1 - items: - - const: kgsl_3d0_reg_memory - - const: cx_mem - - const: cx_dbgc + + - if: + properties: + compatible: + contains: + const: qcom,adreno-612.0 + then: + properties: + clocks: + items: + - description: GPU Core clock + + clock-names: + items: + - const: core + + reg-names: + minItems: 1 + items: + - const: kgsl_3d0_reg_memory + - const: cx_mem + - const: cx_dbgc + + required: + - clocks + - clock-names + + - if: + properties: + compatible: + contains: + enum: + - qcom,adreno-615.0 + - qcom,adreno-618.0 + - qcom,adreno-619.0 + - qcom,adreno-621.0 + - qcom,adreno-623.0 + - qcom,adreno-630.2 + - qcom,adreno-635.0 + - qcom,adreno-640.1 + - qcom,adreno-650.2 + - qcom,adreno-660.1 + - qcom,adreno-663.0 + - qcom,adreno-680.1 + - qcom,adreno-690.0 + - qcom,adreno-730.1 + - qcom,adreno-43030c00 + - qcom,adreno-43050a01 + - qcom,adreno-43050c01 + - qcom,adreno-43051401 + + then: # Starting with A6xx, the clocks are usually defined in the GMU node + properties: + clocks: false + clock-names: false + + reg-names: + minItems: 1 + items: + - const: kgsl_3d0_reg_memory + - const: cx_mem + - const: cx_dbgc examples: - | diff --git a/Documentation/devicetree/bindings/display/msm/qcom,adreno-rgmu.yaml b/Documentation/devicetree/bindings/display/msm/qcom,adreno-rgmu.yaml new file mode 100644 index 0000000000000..bacc5b32e6d7b --- /dev/null +++ b/Documentation/devicetree/bindings/display/msm/qcom,adreno-rgmu.yaml @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +%YAML 1.2 +--- + +$id: http://devicetree.org/schemas/display/msm/qcom,adreno-rgmu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: RGMU attached to certain Adreno GPUs + +maintainers: + - Rob Clark + +description: + RGMU (Reduced Graphics Management Unit) IP is present in some GPUs that + belong to Adreno A6xx family. It is a small state machine that helps to + toggle the GX GDSC (connected to CX rail) to implement IFPC feature and save + power. + +properties: + compatible: + items: + - const: qcom,adreno-rgmu-612.0 + - const: qcom,adreno-rgmu + + reg: + items: + - description: Core RGMU registers + + clocks: + items: + - description: GMU clock + - description: GPU CX clock + - description: GPU AXI clock + - description: GPU MEMNOC clock + - description: GPU SMMU vote clock + + clock-names: + items: + - const: gmu + - const: cxo + - const: axi + - const: memnoc + - const: smmu_vote + + power-domains: + items: + - description: CX GDSC power domain + - description: GX GDSC power domain + + power-domain-names: + items: + - const: cx + - const: gx + + interrupts: + items: + - description: GMU OOB interrupt + - description: GMU interrupt + + interrupt-names: + items: + - const: oob + - const: gmu + + operating-points-v2: true + opp-table: + type: object + +required: + - compatible + - reg + - clocks + - clock-names + - power-domains + - power-domain-names + - interrupts + - interrupt-names + - operating-points-v2 + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + + gmu@506a000 { + compatible = "qcom,adreno-rgmu-612.0", "qcom,adreno-rgmu"; + + reg = <0x05000000 0x90000>; + + clocks = <&gpucc GPU_CC_CX_GMU_CLK>, + <&gpucc GPU_CC_CXO_CLK>, + <&gcc GCC_DDRSS_GPU_AXI_CLK>, + <&gcc GCC_GPU_MEMNOC_GFX_CLK>, + <&gpucc GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK>; + clock-names = "gmu", + "cxo", + "axi", + "memnoc", + "smmu_vote"; + + power-domains = <&gpucc CX_GDSC>, + <&gpucc GX_GDSC>; + power-domain-names = "cx", + "gx"; + + interrupts = , + ; + interrupt-names = "oob", + "gmu"; + + operating-points-v2 = <&gmu_opp_table>; + + gmu_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/display/msm/qcom,qcs8300-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,qcs8300-mdss.yaml new file mode 100644 index 0000000000000..c41a86203e78a --- /dev/null +++ b/Documentation/devicetree/bindings/display/msm/qcom,qcs8300-mdss.yaml @@ -0,0 +1,386 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/msm/qcom,qcs8300-mdss.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. QCS8300 Display MDSS + +maintainers: + - Yongxing Mou + +description: + QCS8300 MSM Mobile Display Subsystem(MDSS), which encapsulates sub-blocks like + DPU display controller, DP interfaces and EDP etc. + +$ref: /schemas/display/msm/mdss-common.yaml# + +properties: + compatible: + const: qcom,qcs8300-mdss + + clocks: + items: + - description: Display AHB + - description: Display hf AXI + - description: Display core + + iommus: + maxItems: 1 + + interconnects: + maxItems: 3 + + interconnect-names: + maxItems: 3 + +patternProperties: + "^display-controller@[0-9a-f]+$": + type: object + additionalProperties: true + + properties: + compatible: + contains: + const: qcom,qcs8300-dpu + + "^displayport-controller@[0-9a-f]+$": + type: object + additionalProperties: true + + properties: + compatible: + contains: + const: qcom,qcs8300-dp + + "^dsi@[0-9a-f]+$": + type: object + additionalProperties: true + properties: + compatible: + contains: + const: qcom,qcs8300-dsi-ctrl + + "^phy@[0-9a-f]+$": + type: object + additionalProperties: true + properties: + compatible: + contains: + enum: + - qcom,qcs8300-dsi-phy-5nm + - qcom,qcs8300-edp-phy + +required: + - compatible + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + #include + #include + #include + #include + #include + + mdss: display-subsystem@ae00000 { + compatible = "qcom,qcs8300-mdss"; + reg = <0x0ae00000 0x1000>; + reg-names = "mdss"; + + interconnects = <&mmss_noc MASTER_MDP0 QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_MDP1 QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "mdp0-mem", + "mdp1-mem", + "cpu-cfg"; + + resets = <&dispcc_core_bcr>; + power-domains = <&dispcc_gdsc>; + + clocks = <&dispcc_ahb_clk>, + <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc_mdp_clk>; + + interrupts = ; + interrupt-controller; + #interrupt-cells = <1>; + + iommus = <&apps_smmu 0x1000 0x402>; + + #address-cells = <1>; + #size-cells = <1>; + ranges; + + display-controller@ae01000 { + compatible = "qcom,qcs8300-dpu", "qcom,sa8775p-dpu"; + reg = <0x0ae01000 0x8f000>, + <0x0aeb0000 0x2008>; + reg-names = "mdp", "vbif"; + + clocks = <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_MDP_LUT_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_MDP_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_VSYNC_CLK>; + clock-names = "nrt_bus", + "iface", + "lut", + "core", + "vsync"; + + assigned-clocks = <&dispcc0 MDSS_DISP_CC_MDSS_VSYNC_CLK>; + assigned-clock-rates = <19200000>; + operating-points-v2 = <&mdp_opp_table>; + power-domains = <&rpmhpd RPMHPD_MMCX>; + + interrupt-parent = <&mdss>; + interrupts = <0>; + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + + dpu_intf0_out: endpoint { + remote-endpoint = <&mdss_dp0_in>; + }; + }; + + port@1 { + reg = <1>; + dpu_intf1_out: endpoint { + remote-endpoint = <&mdss_dsi0_in>; + }; + }; + }; + + mdp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-375000000 { + opp-hz = /bits/ 64 <375000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-575000000 { + opp-hz = /bits/ 64 <575000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + + opp-650000000 { + opp-hz = /bits/ 64 <650000000>; + required-opps = <&rpmhpd_opp_turbo_l1>; + }; + }; + }; + + dsi@ae94000 { + compatible = "qcom,qcs8300-dsi-ctrl", + "qcom,sa8775p-dsi-ctrl", + "qcom,mdss-dsi-ctrl"; + reg = <0x0ae94000 0x400>; + reg-names = "dsi_ctrl"; + + interrupt-parent = <&mdss>; + interrupts = <4>; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_BYTE0_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_BYTE0_INTF_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_PCLK0_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_ESC0_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP_HF_AXI_CLK>; + clock-names = "byte", + "byte_intf", + "pixel", + "core", + "iface", + "bus"; + assigned-clocks = <&dispcc MDSS_DISP_CC_MDSS_BYTE0_CLK_SRC>, + <&dispcc MDSS_DISP_CC_MDSS_PCLK0_CLK_SRC>; + assigned-clock-parents = <&mdss_dsi0_phy 0>, <&mdss_dsi0_phy 1>; + phys = <&mdss_dsi0_phy>; + + operating-points-v2 = <&dsi0_opp_table>; + power-domains = <&rpmhpd RPMHPD_MMCX>; + + vdda-supply = <&vreg_l5a>; + + #address-cells = <1>; + #size-cells = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + mdss0_dsi0_in: endpoint { + remote-endpoint = <&dpu_intf1_out>; + }; + }; + + port@1 { + reg = <1>; + mdss0_dsi0_out: endpoint { }; + }; + }; + + dsi0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-358000000 { + opp-hz = /bits/ 64 <358000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + + mdss_dsi0_phy: phy@ae94400 { + compatible = "qcom,qcs8300-dsi-phy-5nm", + "qcom,sa8775p-dsi-phy-5nm"; + reg = <0x0ae94400 0x200>, + <0x0ae94600 0x280>, + <0x0ae94900 0x27c>; + reg-names = "dsi_phy", + "dsi_phy_lane", + "dsi_pll"; + + #clock-cells = <1>; + #phy-cells = <0>; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>, + <&rpmhcc RPMH_CXO_CLK>; + clock-names = "iface", "ref"; + + vdds-supply = <&vreg_l4a>; + }; + + mdss_dp0_phy: phy@aec2a00 { + compatible = "qcom,qcs8300-edp-phy", "qcom,sa8775p-edp-phy"; + + reg = <0x0aec2a00 0x200>, + <0x0aec2200 0xd0>, + <0x0aec2600 0xd0>, + <0x0aec2000 0x1c8>; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>; + clock-names = "aux", + "cfg_ahb"; + + #clock-cells = <1>; + #phy-cells = <0>; + + vdda-phy-supply = <&vreg_l1c>; + vdda-pll-supply = <&vreg_l4a>; + }; + + displayport-controller@af54000 { + compatible = "qcom,qcs8300-dp", "qcom,sa8775p-dp"; + + pinctrl-0 = <&dp_hot_plug_det>; + pinctrl-names = "default"; + + reg = <0xaf54000 0x104>, + <0xaf54200 0x0c0>, + <0xaf55000 0x770>, + <0xaf56000 0x09c>, + <0xaf57000 0x09c>, + <0xaf58000 0x09c>, + <0xaf59000 0x09c>, + <0xaf5a000 0x23c>, + <0xaf5b000 0x23c>; + + interrupt-parent = <&mdss>; + interrupts = <12>; + clocks = <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_LINK_INTF_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK>; + clock-names = "core_iface", + "core_aux", + "ctrl_link", + "ctrl_link_iface", + "stream_pixel", + "stream_1_pixel", + "stream_2_pixel", + "stream_3_pixel"; + assigned-clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK_SRC>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK_SRC>, + <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK_SRC>; + assigned-clock-parents = <&mdss_dp0_phy 0>, + <&mdss_dp0_phy 1>, + <&mdss_dp0_phy 1>, + <&mdss_dp0_phy 1>; + phys = <&mdss_dp0_phy>; + phy-names = "dp"; + operating-points-v2 = <&dp_opp_table>; + power-domains = <&rpmhpd RPMHPD_MMCX>; + + #sound-dai-cells = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mdss_dp0_in: endpoint { + remote-endpoint = <&dpu_intf0_out>; + }; + }; + + port@1 { + reg = <1>; + + mdss_dp_out: endpoint { }; + }; + }; + + dp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-270000000 { + opp-hz = /bits/ 64 <270000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-810000000 { + opp-hz = /bits/ 64 <810000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml index e2730a2f25cfb..6c827cf9692b9 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml @@ -200,9 +200,11 @@ examples: <0x0aec2000 0x1c8>; clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, - <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>; + <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; #clock-cells = <1>; #phy-cells = <0>; diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm6150-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm6150-mdss.yaml index 9ac24f99d3ada..46e9335f849fe 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,sm6150-mdss.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,sm6150-mdss.yaml @@ -51,6 +51,14 @@ patternProperties: compatible: const: qcom,sm6150-dpu + "^displayport-controller@[0-9a-f]+$": + type: object + additionalProperties: true + properties: + compatible: + contains: + const: qcom,sm6150-dp + "^dsi@[0-9a-f]+$": type: object additionalProperties: true @@ -130,35 +138,37 @@ examples: #size-cells = <0>; port@0 { - reg = <0>; - dpu_intf0_out: endpoint { - }; + reg = <0>; + + dpu_intf0_out: endpoint { + }; }; port@1 { - reg = <1>; - dpu_intf1_out: endpoint { - remote-endpoint = <&mdss_dsi0_in>; - }; + reg = <1>; + + dpu_intf1_out: endpoint { + remote-endpoint = <&mdss_dsi0_in>; + }; }; }; mdp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-19200000 { - opp-hz = /bits/ 64 <19200000>; - required-opps = <&rpmhpd_opp_low_svs>; + opp-192000000 { + opp-hz = /bits/ 64 <192000000>; + required-opps = <&rpmhpd_opp_low_svs>; }; - opp-25600000 { - opp-hz = /bits/ 64 <25600000>; - required-opps = <&rpmhpd_opp_svs>; + opp-256000000 { + opp-hz = /bits/ 64 <256000000>; + required-opps = <&rpmhpd_opp_svs>; }; opp-307200000 { - opp-hz = /bits/ 64 <307200000>; - required-opps = <&rpmhpd_opp_nom>; + opp-hz = /bits/ 64 <307200000>; + required-opps = <&rpmhpd_opp_nom>; }; }; }; diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml index 0a46120dd8680..d9b980a897229 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml @@ -13,11 +13,16 @@ $ref: /schemas/display/msm/dpu-common.yaml# properties: compatible: - enum: - - qcom,sa8775p-dpu - - qcom,sm8650-dpu - - qcom,sm8750-dpu - - qcom,x1e80100-dpu + oneOf: + - enum: + - qcom,sa8775p-dpu + - qcom,sm8650-dpu + - qcom,sm8750-dpu + - qcom,x1e80100-dpu + - items: + - enum: + - qcom,qcs8300-dpu + - const: qcom,sa8775p-dpu reg: items: diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml index 4151f475f3bc3..14942dfbdb09b 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml @@ -10,7 +10,7 @@ maintainers: - Krzysztof Kozlowski description: - SM8650 MSM Mobile Display Subsystem(MDSS), which encapsulates sub-blocks like + SM8750 MSM Mobile Display Subsystem(MDSS), which encapsulates sub-blocks like DPU display controller, DSI and DP interfaces etc. $ref: /schemas/display/msm/mdss-common.yaml# diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index bbe4da2a11054..4cd867854a5fd 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -24,6 +24,8 @@ properties: - qcom,sm6350-gpi-dma - items: - enum: + - qcom,glymur-gpi-dma + - qcom,kaanapali-gpi-dma - qcom,milos-gpi-dma - qcom,qcm2290-gpi-dma - qcom,qcs8300-gpi-dma diff --git a/Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml b/Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml new file mode 100644 index 0000000000000..ac5a08f8f76df --- /dev/null +++ b/Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/embedded-controller/qcom,hamoa-crd-ec.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Hamoa Embedded Controller + +maintainers: + - Sibi Sankar + - Anvesh Jain P + +description: + Qualcomm Snapdragon based Hamoa/Purwa and Glymur reference devices have an + EC running on different MCU chips. The EC handles things like fan control, + temperature sensors, access to EC internal state changes. + +properties: + compatible: + oneOf: + - items: + - enum: + - qcom,glymur-crd-ec + - qcom,hamoa-iot-evk-ec + - const: qcom,hamoa-crd-ec + - enum: + - qcom,hamoa-crd-ec + + reg: + const: 0x76 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + embedded-controller@76 { + compatible = "qcom,hamoa-crd-ec"; + reg = <0x76>; + + interrupts-extended = <&tlmm 66 IRQ_TYPE_LEVEL_HIGH>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml index ef97faac7e47c..d66459f1d84e4 100644 --- a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml +++ b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml @@ -23,6 +23,7 @@ properties: - enum: - qcom,scm-apq8064 - qcom,scm-apq8084 + - qcom,scm-glymur - qcom,scm-ipq4019 - qcom,scm-ipq5018 - qcom,scm-ipq5332 @@ -31,6 +32,7 @@ properties: - qcom,scm-ipq806x - qcom,scm-ipq8074 - qcom,scm-ipq9574 + - qcom,scm-kaanapali - qcom,scm-mdm9607 - qcom,scm-milos - qcom,scm-msm8226 @@ -202,6 +204,7 @@ allOf: compatible: contains: enum: + - qcom,scm-kaanapali - qcom,scm-milos - qcom,scm-sm8450 - qcom,scm-sm8550 diff --git a/Documentation/devicetree/bindings/hwmon/qcom,bcl-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/qcom,bcl-hwmon.yaml new file mode 100644 index 0000000000000..6ffadc4d28bcf --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/qcom,bcl-hwmon.yaml @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/qcom,bcl-hwmon.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm SPMI PMIC Battery Current Limiting (BCL) Hardware Monitor + +maintainers: + - Manaf Meethalavalappu Pallikunhi + +description: | + SPMI PMIC Battery Current Limiting (BCL) hardware provides monitoring and + alarm functionality for battery overcurrent and battery or system under + voltage conditions. It monitors battery voltage and current, and + can trigger interrupts when configurable thresholds are exceeded. + +properties: + compatible: + oneOf: + - description: v1 based BCL + items: + - enum: + - qcom,pm7250b-bcl + - qcom,pm8250b-bcl + - const: qcom,bcl-v1 + + - description: v2 based BCL + items: + - enum: + - qcom,pm8350b-bcl + - qcom,pm8350c-bcl + - const: qcom,bcl-v2 + + - description: v3 bmx based BCL + items: + - enum: + - qcom,pm8550b-bcl + - qcom,pm7550ba-bcl + - const: qcom,bcl-v3-bmx + + - description: v3 core based BCL + items: + - enum: + - qcom,pm8550-bc0l + - qcom,pm7550-bcl + - const: qcom,bcl-v3-core + + - description: v3 wb based BCL + items: + - enum: + - qcom,pmw5100-bcl + - const: qcom,bcl-v3-wb + + - description: v4 bmx based BCL + items: + - enum: + - qcom,pmih010-bcl + - const: qcom,bcl-v4-bmx + + - description: v4 bmx with different scale based BCL + items: + - enum: + - qcom,pmv010-bcl + - const: qcom,bcl-v4-pmv010 + + - description: v4 core based BCL + items: + - enum: + - qcom,pmh010-bcl + - const: qcom,bcl-v4-core + + - description: v4 wb based BCL + items: + - enum: + - qcom,pmw6100-bcl + - const: qcom,bcl-v4-wb + + reg: + maxItems: 1 + description: BCL base address in the SPMI PMIC register map + + interrupts: + minItems: 2 + maxItems: 2 + description: + BCL alarm interrupts for different threshold levels + + interrupt-names: + items: + - const: bcl-max-min + - const: bcl-critical + + overcurrent-thresholds-milliamp: + description: + Current thresholds in milliamperes for the two configurable current + alarm levels (max and critical). These values are used to override + default thresholds if a platform has different battery ocp specification. + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 2 + maxItems: 2 + +required: + - compatible + - reg + - interrupts + - interrupt-names + +unevaluatedProperties: false + +examples: + - | + #include + + pmic { + #address-cells = <1>; + #size-cells = <0>; + + sensor@1d00 { + compatible = "qcom,pm7250b-bcl", "qcom,bcl-v1"; + reg = <0x1d00>; + interrupts = <0x2 0x1d 0x0 IRQ_TYPE_EDGE_RISING>, + <0x2 0x1d 0x1 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "bcl-max-min", + "bcl-critical"; + overcurrent-thresholds-milliamp = <5500 6000>; + }; + }; diff --git a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml index 9bc99d736343f..35d3a0685ac44 100644 --- a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml +++ b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml @@ -15,6 +15,7 @@ properties: oneOf: - enum: - qcom,msm8226-cci + - qcom,msm8953-cci - qcom,msm8974-cci - qcom,msm8996-cci @@ -25,17 +26,21 @@ properties: - items: - enum: + - qcom,kaanapali-cci - qcom,qcm2290-cci + - qcom,qcs8300-cci - qcom,sa8775p-cci - qcom,sc7280-cci - qcom,sc8280xp-cci - qcom,sdm670-cci - qcom,sdm845-cci + - qcom,sm6150-cci - qcom,sm6350-cci - qcom,sm8250-cci - qcom,sm8450-cci - qcom,sm8550-cci - qcom,sm8650-cci + - qcom,sm8750-cci - qcom,x1e80100-cci - const: qcom,msm8996-cci # CCI v2 @@ -128,7 +133,10 @@ allOf: compatible: contains: enum: + - qcom,kaanapali-cci - qcom,qcm2290-cci + - qcom,qcs8300-cci + - qcom,sm8750-cci then: properties: clocks: @@ -146,6 +154,7 @@ allOf: - contains: enum: - qcom,msm8916-cci + - qcom,msm8953-cci - const: qcom,msm8996-cci then: @@ -243,6 +252,7 @@ allOf: contains: enum: - qcom,sa8775p-cci + - qcom,sm6150-cci - qcom,sm8550-cci - qcom,sm8650-cci - qcom,x1e80100-cci diff --git a/Documentation/devicetree/bindings/i2c/qcom,i2c-geni-qcom.yaml b/Documentation/devicetree/bindings/i2c/qcom,i2c-geni-qcom.yaml index 51534953a69cf..9401dc2d50522 100644 --- a/Documentation/devicetree/bindings/i2c/qcom,i2c-geni-qcom.yaml +++ b/Documentation/devicetree/bindings/i2c/qcom,i2c-geni-qcom.yaml @@ -60,6 +60,13 @@ properties: power-domains: maxItems: 1 + qcom,qup-multi-owner: + type: boolean + description: + Indicates that the QUP-based controller is shared with one or more + other system processors and must not be assumed to have exclusive + ownership by the operating system. + reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/interconnect/qcom,kaanapali-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,kaanapali-rpmh.yaml new file mode 100644 index 0000000000000..2808acc0191f3 --- /dev/null +++ b/Documentation/devicetree/bindings/interconnect/qcom,kaanapali-rpmh.yaml @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interconnect/qcom,kaanapali-rpmh.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm RPMh Network-On-Chip Interconnect on Kaanapali SoC + +maintainers: + - Raviteja Laggyshetty + +description: | + RPMh interconnect providers support system bandwidth requirements through + RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is + able to communicate with the BCM through the Resource State Coordinator (RSC) + associated with each execution environment. Provider nodes must point to at + least one RPMh device child node pertaining to their RSC and each provider + can map to multiple RPMh resources. + + See also: include/dt-bindings/interconnect/qcom,kaanapali-rpmh.h + +properties: + compatible: + enum: + - qcom,kaanapali-aggre-noc + - qcom,kaanapali-clk-virt + - qcom,kaanapali-cnoc-main + - qcom,kaanapali-cnoc-cfg + - qcom,kaanapali-gem-noc + - qcom,kaanapali-lpass-ag-noc + - qcom,kaanapali-lpass-lpiaon-noc + - qcom,kaanapali-lpass-lpicx-noc + - qcom,kaanapali-mc-virt + - qcom,kaanapali-mmss-noc + - qcom,kaanapali-nsp-noc + - qcom,kaanapali-pcie-anoc + - qcom,kaanapali-system-noc + + reg: + maxItems: 1 + + clocks: + minItems: 2 + maxItems: 3 + +required: + - compatible + +allOf: + - $ref: qcom,rpmh-common.yaml# + - if: + properties: + compatible: + contains: + enum: + - qcom,kaanapali-clk-virt + - qcom,kaanapali-mc-virt + then: + properties: + reg: false + else: + required: + - reg + + - if: + properties: + compatible: + contains: + enum: + - qcom,kaanapali-pcie-anoc + then: + properties: + clocks: + items: + - description: aggre-NOC PCIe AXI clock + - description: cfg-NOC PCIe a-NOC AHB clock + + - if: + properties: + compatible: + contains: + enum: + - qcom,kaanapali-aggre-noc + then: + properties: + clocks: + items: + - description: aggre UFS PHY AXI clock + - description: aggre USB3 PRIM AXI clock + - description: RPMH CC IPA clock + + - if: + properties: + compatible: + contains: + enum: + - qcom,kaanapali-aggre-noc + - qcom,kaanapali-pcie-anoc + then: + required: + - clocks + else: + properties: + clocks: false + +unevaluatedProperties: false + +examples: + - | + clk_virt: interconnect-0 { + compatible = "qcom,kaanapali-clk-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + aggre_noc: interconnect@16e0000 { + compatible = "qcom,kaanapali-aggre-noc"; + reg = <0x016e0000 0x42400>; + #interconnect-cells = <2>; + clocks = <&gcc_aggre_ufs_phy_axi_clk>, + <&gcc_aggre_usb3_prim_axi_clk>, + <&rpmhcc_ipa_clk>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml index 256de140c03dc..0d1a268db921d 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml @@ -25,6 +25,7 @@ properties: - const: qcom,msm8998-bwmon # BWMON v4 - items: - enum: + - qcom,kaanapali-cpu-bwmon - qcom,qcm2290-cpu-bwmon - qcom,qcs615-cpu-bwmon - qcom,qcs8300-cpu-bwmon diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml index 38d0c2d57dd6d..e92c3eeb19b3b 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml @@ -27,6 +27,7 @@ properties: items: - enum: - qcom,glymur-pdc + - qcom,kaanapali-pdc - qcom,qcs615-pdc - qcom,qcs8300-pdc - qcom,qdu1000-pdc @@ -52,6 +53,7 @@ properties: - qcom,sm8650-pdc - qcom,sm8750-pdc - qcom,x1e80100-pdc + - qcom,x1p42100-pdc - const: qcom,pdc reg: @@ -59,6 +61,7 @@ properties: items: - description: PDC base register region - description: Edge or Level config register for SPI interrupts + - description: PDC config for pass through or secondary IRQ mode for GPIOs '#interrupt-cells': const: 2 @@ -79,6 +82,10 @@ properties: The tuples indicates the valid mapping of valid PDC ports and their hwirq mapping. + qcom,qmp: + $ref: /schemas/types.yaml#/definitions/phandle + description: Reference to the AOSS side-channel message RAM. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 89495f094d52b..2ffc48a276dfb 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -35,6 +35,8 @@ properties: - description: Qcom SoCs implementing "qcom,smmu-500" and "arm,mmu-500" items: - enum: + - qcom,glymur-smmu-500 + - qcom,kaanapali-smmu-500 - qcom,milos-smmu-500 - qcom,qcm2290-smmu-500 - qcom,qcs615-smmu-500 diff --git a/Documentation/devicetree/bindings/leds/leds-qcom-lpg.yaml b/Documentation/devicetree/bindings/leds/leds-qcom-lpg.yaml index 841a0229c472a..1278152e8eed1 100644 --- a/Documentation/devicetree/bindings/leds/leds-qcom-lpg.yaml +++ b/Documentation/devicetree/bindings/leds/leds-qcom-lpg.yaml @@ -38,6 +38,7 @@ properties: - items: - enum: - qcom,pm8550-pwm + - qcom,pmh0101-pwm - const: qcom,pm8350c-pwm - items: - enum: diff --git a/Documentation/devicetree/bindings/leds/qcom,spmi-flash-led.yaml b/Documentation/devicetree/bindings/leds/qcom,spmi-flash-led.yaml index bcf0ad4ea57ea..0df3e460a792b 100644 --- a/Documentation/devicetree/bindings/leds/qcom,spmi-flash-led.yaml +++ b/Documentation/devicetree/bindings/leds/qcom,spmi-flash-led.yaml @@ -28,6 +28,7 @@ properties: - qcom,pm8150l-flash-led - qcom,pm8350c-flash-led - qcom,pm8550-flash-led + - qcom,pmh0101-flash-led - qcom,pmi8998-flash-led - const: qcom,spmi-flash-led diff --git a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml index 9122c3d2dc30f..9e91247c521a1 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml @@ -19,6 +19,7 @@ properties: - items: - enum: - qcom,glymur-cpucp-mbox + - qcom,kaanapali-cpucp-mbox - const: qcom,x1e80100-cpucp-mbox - enum: - qcom,x1e80100-cpucp-mbox diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml index e5c423130db67..7c4d6170491db 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml @@ -24,6 +24,8 @@ properties: compatible: items: - enum: + - qcom,glymur-ipcc + - qcom,kaanapali-ipcc - qcom,milos-ipcc - qcom,qcs8300-ipcc - qcom,qdu1000-ipcc diff --git a/Documentation/devicetree/bindings/media/qcom,msm8939-camss.yaml b/Documentation/devicetree/bindings/media/qcom,msm8939-camss.yaml new file mode 100644 index 0000000000000..77b389d76a437 --- /dev/null +++ b/Documentation/devicetree/bindings/media/qcom,msm8939-camss.yaml @@ -0,0 +1,254 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/qcom,msm8939-camss.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm MSM8939 Camera Subsystem (CAMSS) + +maintainers: + - Vincent Knecht + +description: + The CAMSS IP is a CSI decoder and ISP present on Qualcomm platforms + +properties: + compatible: + const: qcom,msm8939-camss + + reg: + maxItems: 11 + + reg-names: + items: + - const: csiphy0 + - const: csiphy0_clk_mux + - const: csiphy1 + - const: csiphy1_clk_mux + - const: csid0 + - const: csid1 + - const: ispif + - const: csi_clk_mux + - const: vfe0 + - const: csid2 + - const: vfe0_vbif + + clocks: + maxItems: 24 + + clock-names: + items: + - const: top_ahb + - const: ispif_ahb + - const: csiphy0_timer + - const: csiphy1_timer + - const: csi0_ahb + - const: csi0 + - const: csi0_phy + - const: csi0_pix + - const: csi0_rdi + - const: csi1_ahb + - const: csi1 + - const: csi1_phy + - const: csi1_pix + - const: csi1_rdi + - const: ahb + - const: vfe0 + - const: csi_vfe0 + - const: vfe_ahb + - const: vfe_axi + - const: csi2_ahb + - const: csi2 + - const: csi2_phy + - const: csi2_pix + - const: csi2_rdi + + interrupts: + maxItems: 7 + + interrupt-names: + items: + - const: csiphy0 + - const: csiphy1 + - const: csid0 + - const: csid1 + - const: ispif + - const: vfe0 + - const: csid2 + + iommus: + maxItems: 1 + + power-domains: + items: + - description: VFE GDSC - Video Front End, Global Distributed Switch + Controller. + + vdda-supply: + description: + Definition of the regulator used as 1.2V analog power supply. + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + description: + CSI input ports. + + patternProperties: + "^port@[0-1]$": + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false + + description: + Input port for receiving CSI data. + + properties: + endpoint: + $ref: video-interfaces.yaml# + unevaluatedProperties: false + + properties: + data-lanes: + minItems: 1 + maxItems: 4 + + bus-type: + enum: + - 4 # MEDIA_BUS_TYPE_CSI2_DPHY + + required: + - data-lanes + +required: + - compatible + - reg + - reg-names + - clocks + - clock-names + - interrupts + - interrupt-names + - iommus + - power-domains + - vdda-supply + - ports + +additionalProperties: false + +examples: + - | + #include + #include + + isp@1b0ac00 { + compatible = "qcom,msm8939-camss"; + + reg = <0x01b0ac00 0x200>, + <0x01b00030 0x4>, + <0x01b0b000 0x200>, + <0x01b00038 0x4>, + <0x01b08000 0x100>, + <0x01b08400 0x100>, + <0x01b0a000 0x500>, + <0x01b00020 0x10>, + <0x01b10000 0x1000>, + <0x01b08800 0x100>, + <0x01b40000 0x200>; + + reg-names = "csiphy0", + "csiphy0_clk_mux", + "csiphy1", + "csiphy1_clk_mux", + "csid0", + "csid1", + "ispif", + "csi_clk_mux", + "vfe0", + "csid2", + "vfe0_vbif"; + + clocks = <&gcc GCC_CAMSS_TOP_AHB_CLK>, + <&gcc GCC_CAMSS_ISPIF_AHB_CLK>, + <&gcc GCC_CAMSS_CSI0PHYTIMER_CLK>, + <&gcc GCC_CAMSS_CSI1PHYTIMER_CLK>, + <&gcc GCC_CAMSS_CSI0_AHB_CLK>, + <&gcc GCC_CAMSS_CSI0_CLK>, + <&gcc GCC_CAMSS_CSI0PHY_CLK>, + <&gcc GCC_CAMSS_CSI0PIX_CLK>, + <&gcc GCC_CAMSS_CSI0RDI_CLK>, + <&gcc GCC_CAMSS_CSI1_AHB_CLK>, + <&gcc GCC_CAMSS_CSI1_CLK>, + <&gcc GCC_CAMSS_CSI1PHY_CLK>, + <&gcc GCC_CAMSS_CSI1PIX_CLK>, + <&gcc GCC_CAMSS_CSI1RDI_CLK>, + <&gcc GCC_CAMSS_AHB_CLK>, + <&gcc GCC_CAMSS_VFE0_CLK>, + <&gcc GCC_CAMSS_CSI_VFE0_CLK>, + <&gcc GCC_CAMSS_VFE_AHB_CLK>, + <&gcc GCC_CAMSS_VFE_AXI_CLK>, + <&gcc GCC_CAMSS_CSI2_AHB_CLK>, + <&gcc GCC_CAMSS_CSI2_CLK>, + <&gcc GCC_CAMSS_CSI2PHY_CLK>, + <&gcc GCC_CAMSS_CSI2PIX_CLK>, + <&gcc GCC_CAMSS_CSI2RDI_CLK>; + + clock-names = "top_ahb", + "ispif_ahb", + "csiphy0_timer", + "csiphy1_timer", + "csi0_ahb", + "csi0", + "csi0_phy", + "csi0_pix", + "csi0_rdi", + "csi1_ahb", + "csi1", + "csi1_phy", + "csi1_pix", + "csi1_rdi", + "ahb", + "vfe0", + "csi_vfe0", + "vfe_ahb", + "vfe_axi", + "csi2_ahb", + "csi2", + "csi2_phy", + "csi2_pix", + "csi2_rdi"; + + interrupts = , + , + , + , + , + , + ; + + interrupt-names = "csiphy0", + "csiphy1", + "csid0", + "csid1", + "ispif", + "vfe0", + "csid2"; + + iommus = <&apps_iommu 3>; + + power-domains = <&gcc VFE_GDSC>; + + vdda-supply = <®_1v2>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + csiphy1_ep: endpoint { + data-lanes = <0 2>; + remote-endpoint = <&sensor_ep>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml index 80a4540a22dc2..e5f170aa4d9ee 100644 --- a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml +++ b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml @@ -120,6 +120,14 @@ properties: items: - const: top + vdda-phy-supply: + description: + Phandle to a 0.88V regulator supply to CSI PHYs. + + vdda-pll-supply: + description: + Phandle to 1.2V regulator supply to CSI PHYs pll block. + ports: $ref: /schemas/graph.yaml#/properties/ports @@ -160,6 +168,8 @@ required: - power-domains - power-domain-names - ports + - vdda-phy-supply + - vdda-pll-supply additionalProperties: false @@ -328,6 +338,9 @@ examples: power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; power-domain-names = "top"; + vdda-phy-supply = <&vreg_l4a_0p88>; + vdda-pll-supply = <&vreg_l1c_1p2>; + ports { #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml index bfd8b1ad47312..5de4445a486d7 100644 --- a/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml @@ -50,6 +50,9 @@ properties: iommus: maxItems: 1 + iommu-map: + maxItems: 1 + memory-region: maxItems: 1 @@ -101,6 +104,7 @@ unevaluatedProperties: false examples: - | #include + #include #include venus: video-codec@aa00000 { @@ -118,5 +122,6 @@ examples: clock-names = "core", "iface", "bus", "vcodec0_core", "vcodec0_bus"; iommus = <&apps_smmu 0x0c00 0x60>; + iommu-map = ; memory-region = <&venus_mem>; }; diff --git a/Documentation/devicetree/bindings/media/qcom,sm6150-camss.yaml b/Documentation/devicetree/bindings/media/qcom,sm6150-camss.yaml new file mode 100644 index 0000000000000..ba7b0acb9128b --- /dev/null +++ b/Documentation/devicetree/bindings/media/qcom,sm6150-camss.yaml @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/qcom,sm6150-camss.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm SM6150 Camera Subsystem (CAMSS) + +maintainers: + - Wenmeng Liu + +description: + This binding describes the camera subsystem hardware found on SM6150 + Qualcomm SoCs. It includes submodules such as CSIPHY (CSI Physical layer) + and CSID (CSI Decoder), which comply with the MIPI CSI2 protocol. + + The subsystem also integrates a set of real-time image processing engines + and their associated configuration modules, as well as non-real-time engines. + +properties: + compatible: + const: qcom,sm6150-camss + + reg: + items: + - description: Registers for CSID 0 + - description: Registers for CSID 1 + - description: Registers for CSID Lite + - description: Registers for CSIPHY 0 + - description: Registers for CSIPHY 1 + - description: Registers for CSIPHY 2 + - description: Registers for VFE 0 + - description: Registers for VFE 1 + - description: Registers for VFE Lite + - description: Registers for BPS (Bayer Processing Segment) + - description: Registers for CAMNOC + - description: Registers for CPAS CDM + - description: Registers for CPAS TOP + - description: Registers for ICP (Imaging Control Processor) CSR (Control and Status Registers) + - description: Registers for ICP QGIC (Qualcomm Generic Interrupt Controller) + - description: Registers for ICP SIERRA ((A5 subsystem communication)) + - description: Registers for IPE (Image Postprocessing Engine) 0 + - description: Registers for JPEG DMA + - description: Registers for JPEG ENC + - description: Registers for LRME (Low Resolution Motion Estimation) + + reg-names: + items: + - const: csid0 + - const: csid1 + - const: csid_lite + - const: csiphy0 + - const: csiphy1 + - const: csiphy2 + - const: vfe0 + - const: vfe1 + - const: vfe_lite + - const: bps + - const: camnoc + - const: cpas_cdm + - const: cpas_top + - const: icp_csr + - const: icp_qgic + - const: icp_sierra + - const: ipe0 + - const: jpeg_dma + - const: jpeg_enc + - const: lrme + + clocks: + maxItems: 33 + + clock-names: + items: + - const: gcc_ahb + - const: gcc_axi_hf + - const: camnoc_axi + - const: cpas_ahb + - const: csiphy0 + - const: csiphy0_timer + - const: csiphy1 + - const: csiphy1_timer + - const: csiphy2 + - const: csiphy2_timer + - const: soc_ahb + - const: vfe0 + - const: vfe0_axi + - const: vfe0_cphy_rx + - const: vfe0_csid + - const: vfe1 + - const: vfe1_axi + - const: vfe1_cphy_rx + - const: vfe1_csid + - const: vfe_lite + - const: vfe_lite_cphy_rx + - const: vfe_lite_csid + - const: bps + - const: bps_ahb + - const: bps_axi + - const: bps_areg + - const: icp + - const: ipe0 + - const: ipe0_ahb + - const: ipe0_areg + - const: ipe0_axi + - const: jpeg + - const: lrme + + interrupts: + maxItems: 15 + + interrupt-names: + items: + - const: csid0 + - const: csid1 + - const: csid_lite + - const: csiphy0 + - const: csiphy1 + - const: csiphy2 + - const: vfe0 + - const: vfe1 + - const: vfe_lite + - const: camnoc + - const: cdm + - const: icp + - const: jpeg_dma + - const: jpeg_enc + - const: lrme + + interconnects: + maxItems: 4 + + interconnect-names: + items: + - const: ahb + - const: hf_0 + - const: hf_1 + - const: sf_mnoc + + iommus: + items: + - description: Camera IFE 0 non-protected stream + - description: Camera IFE 1 non-protected stream + - description: Camera IFE 3 non-protected stream + - description: Camera CDM non-protected stream + - description: Camera LRME read non-protected stream + - description: Camera IPE 0 read non-protected stream + - description: Camera BPS read non-protected stream + - description: Camera IPE 0 write non-protected stream + - description: Camera BPS write non-protected stream + - description: Camera LRME write non-protected stream + - description: Camera JPEG read non-protected stream + - description: Camera JPEG write non-protected stream + - description: Camera ICP stream + + power-domains: + items: + - description: + IFE0 GDSC - Image Front End, Global Distributed Switch Controller. + - description: + IFE1 GDSC - Image Front End, Global Distributed Switch Controller. + - description: + Titan GDSC - Titan ISP Block, Global Distributed Switch Controller. + - description: + Titan BPS - Bayer Processing Segment, Global Distributed Switch Controller. + - description: + IPE GDSC - Image Postprocessing Engine, Global Distributed Switch Controller. + + power-domain-names: + items: + - const: ife0 + - const: ife1 + - const: top + - const: bps + - const: ipe + + vdd-csiphy-1p2-supply: + description: + Phandle to a 1.2V regulator supply to CSI PHYs. + + vdd-csiphy-1p8-supply: + description: + Phandle to 1.8V regulator supply to CSI PHYs pll block. + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + description: + CSI input ports. + + patternProperties: + "^port@[0-2]$": + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false + + description: + Input port for receiving CSI data from a CSIPHY. + + properties: + endpoint: + $ref: video-interfaces.yaml# + unevaluatedProperties: false + + properties: + data-lanes: + minItems: 1 + maxItems: 4 + + required: + - data-lanes + +required: + - compatible + - reg + - reg-names + - clocks + - clock-names + - interrupts + - interrupt-names + - interconnects + - interconnect-names + - iommus + - power-domains + - power-domain-names + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + #include + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + camss: isp@acb3000 { + compatible = "qcom,sm6150-camss"; + + reg = <0x0 0x0acb3000 0x0 0x1000>, + <0x0 0x0acba000 0x0 0x1000>, + <0x0 0x0acc8000 0x0 0x1000>, + <0x0 0x0ac65000 0x0 0x1000>, + <0x0 0x0ac66000 0x0 0x1000>, + <0x0 0x0ac67000 0x0 0x1000>, + <0x0 0x0acaf000 0x0 0x4000>, + <0x0 0x0acb6000 0x0 0x4000>, + <0x0 0x0acc4000 0x0 0x4000>, + <0x0 0x0ac6f000 0x0 0x3000>, + <0x0 0x0ac42000 0x0 0x5000>, + <0x0 0x0ac48000 0x0 0x1000>, + <0x0 0x0ac40000 0x0 0x1000>, + <0x0 0x0ac18000 0x0 0x3000>, + <0x0 0x0ac00000 0x0 0x6000>, + <0x0 0x0ac10000 0x0 0x8000>, + <0x0 0x0ac87000 0x0 0x3000>, + <0x0 0x0ac52000 0x0 0x4000>, + <0x0 0x0ac4e000 0x0 0x4000>, + <0x0 0x0ac6b000 0x0 0x0a00>; + reg-names = "csid0", + "csid1", + "csid_lite", + "csiphy0", + "csiphy1", + "csiphy2", + "vfe0", + "vfe1", + "vfe_lite", + "bps", + "camnoc", + "cpas_cdm", + "cpas_top", + "icp_csr", + "icp_qgic", + "icp_sierra", + "ipe0", + "jpeg_dma", + "jpeg_enc", + "lrme"; + + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_AXI_CLK>, + <&camcc CAM_CC_IFE_0_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_0_CSID_CLK>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_AXI_CLK>, + <&camcc CAM_CC_IFE_1_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_1_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_BPS_CLK>, + <&camcc CAM_CC_BPS_AHB_CLK>, + <&camcc CAM_CC_BPS_AXI_CLK>, + <&camcc CAM_CC_BPS_AREG_CLK>, + <&camcc CAM_CC_ICP_CLK>, + <&camcc CAM_CC_IPE_0_CLK>, + <&camcc CAM_CC_IPE_0_AHB_CLK>, + <&camcc CAM_CC_IPE_0_AREG_CLK>, + <&camcc CAM_CC_IPE_0_AXI_CLK>, + <&camcc CAM_CC_JPEG_CLK>, + <&camcc CAM_CC_LRME_CLK>; + + clock-names = "gcc_ahb", + "gcc_axi_hf", + "camnoc_axi", + "cpas_ahb", + "csiphy0", + "csiphy0_timer", + "csiphy1", + "csiphy1_timer", + "csiphy2", + "csiphy2_timer", + "soc_ahb", + "vfe0", + "vfe0_axi", + "vfe0_cphy_rx", + "vfe0_csid", + "vfe1", + "vfe1_axi", + "vfe1_cphy_rx", + "vfe1_csid", + "vfe_lite", + "vfe_lite_cphy_rx", + "vfe_lite_csid", + "bps", + "bps_ahb", + "bps_axi", + "bps_areg", + "icp", + "ipe0", + "ipe0_ahb", + "ipe0_areg", + "ipe0_axi", + "jpeg", + "lrme"; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_CAMNOC_HF0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_HF1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "ahb", + "hf_0", + "hf_1", + "sf_mnoc"; + + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "csid0", + "csid1", + "csid_lite", + "csiphy0", + "csiphy1", + "csiphy2", + "vfe0", + "vfe1", + "vfe_lite", + "camnoc", + "cdm", + "icp", + "jpeg_dma", + "jpeg_enc", + "lrme"; + + iommus = <&apps_smmu 0x0820 0x40>, + <&apps_smmu 0x0840 0x00>, + <&apps_smmu 0x0860 0x40>, + <&apps_smmu 0x0c00 0x00>, + <&apps_smmu 0x0cc0 0x00>, + <&apps_smmu 0x0c80 0x00>, + <&apps_smmu 0x0ca0 0x00>, + <&apps_smmu 0x0d00 0x00>, + <&apps_smmu 0x0d20 0x00>, + <&apps_smmu 0x0d40 0x00>, + <&apps_smmu 0x0d80 0x20>, + <&apps_smmu 0x0da0 0x20>, + <&apps_smmu 0x0de2 0x00>; + + power-domains = <&camcc IFE_0_GDSC>, + <&camcc IFE_1_GDSC>, + <&camcc TITAN_TOP_GDSC>, + <&camcc BPS_GDSC>, + <&camcc IPE_0_GDSC>; + power-domain-names = "ife0", + "ife1", + "top", + "bps", + "ipe"; + + vdd-csiphy-1p2-supply = <&vreg_l11a_1p2>; + vdd-csiphy-1p8-supply = <&vreg_l12a_1p8>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + csiphy_ep0: endpoint { + data-lanes = <0 1>; + remote-endpoint = <&sensor_ep>; + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/media/qcom,sm8550-iris.yaml b/Documentation/devicetree/bindings/media/qcom,sm8550-iris.yaml index 9c4b760508b50..0400ca1bff05d 100644 --- a/Documentation/devicetree/bindings/media/qcom,sm8550-iris.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sm8550-iris.yaml @@ -26,6 +26,7 @@ properties: - qcom,qcs8300-iris - qcom,sm8550-iris - qcom,sm8650-iris + - qcom,x1p42100-iris reg: maxItems: 1 @@ -41,13 +42,16 @@ properties: - const: mmcx clocks: - maxItems: 3 + minItems: 3 + maxItems: 4 clock-names: + minItems: 3 items: - const: iface - const: core - const: vcodec0_core + - const: vcodec0_bse firmware-name: maxItems: 1 @@ -115,6 +119,23 @@ allOf: maxItems: 1 reset-names: maxItems: 1 + - if: + properties: + compatible: + enum: + - qcom,x1p42100-iris + then: + properties: + clocks: + minItems: 4 + clock-names: + minItems: 4 + else: + properties: + clocks: + maxItems: 3 + clock-names: + maxItems: 3 unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/media/qcom,sm8650-camss.yaml b/Documentation/devicetree/bindings/media/qcom,sm8650-camss.yaml new file mode 100644 index 0000000000000..9c8de722601ec --- /dev/null +++ b/Documentation/devicetree/bindings/media/qcom,sm8650-camss.yaml @@ -0,0 +1,375 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/qcom,sm8650-camss.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm SM8650 Camera Subsystem (CAMSS) + +maintainers: + - Vladimir Zapolskiy + +description: + The CAMSS IP is a CSI decoder and ISP present on Qualcomm platforms. + +properties: + compatible: + const: qcom,sm8650-camss + + reg: + maxItems: 17 + + reg-names: + items: + - const: csid_wrapper + - const: csid0 + - const: csid1 + - const: csid2 + - const: csid_lite0 + - const: csid_lite1 + - const: csiphy0 + - const: csiphy1 + - const: csiphy2 + - const: csiphy3 + - const: csiphy4 + - const: csiphy5 + - const: vfe0 + - const: vfe1 + - const: vfe2 + - const: vfe_lite0 + - const: vfe_lite1 + + clocks: + maxItems: 33 + + clock-names: + items: + - const: camnoc_axi + - const: cpas_ahb + - const: cpas_fast_ahb + - const: cpas_vfe0 + - const: cpas_vfe1 + - const: cpas_vfe2 + - const: cpas_vfe_lite + - const: csid + - const: csiphy0 + - const: csiphy0_timer + - const: csiphy1 + - const: csiphy1_timer + - const: csiphy2 + - const: csiphy2_timer + - const: csiphy3 + - const: csiphy3_timer + - const: csiphy4 + - const: csiphy4_timer + - const: csiphy5 + - const: csiphy5_timer + - const: csiphy_rx + - const: gcc_axi_hf + - const: qdss_debug_xo + - const: vfe0 + - const: vfe0_fast_ahb + - const: vfe1 + - const: vfe1_fast_ahb + - const: vfe2 + - const: vfe2_fast_ahb + - const: vfe_lite + - const: vfe_lite_ahb + - const: vfe_lite_cphy_rx + - const: vfe_lite_csid + + interrupts: + maxItems: 16 + + interrupt-names: + items: + - const: csid0 + - const: csid1 + - const: csid2 + - const: csid_lite0 + - const: csid_lite1 + - const: csiphy0 + - const: csiphy1 + - const: csiphy2 + - const: csiphy3 + - const: csiphy4 + - const: csiphy5 + - const: vfe0 + - const: vfe1 + - const: vfe2 + - const: vfe_lite0 + - const: vfe_lite1 + + interconnects: + maxItems: 2 + + interconnect-names: + items: + - const: ahb + - const: hf_mnoc + + iommus: + maxItems: 3 + + power-domains: + items: + - description: IFE0 GDSC - Image Front End, Global Distributed Switch Controller. + - description: IFE1 GDSC - Image Front End, Global Distributed Switch Controller. + - description: IFE2 GDSC - Image Front End, Global Distributed Switch Controller. + - description: Titan GDSC - Titan ISP Block, Global Distributed Switch Controller. + + power-domain-names: + items: + - const: ife0 + - const: ife1 + - const: ife2 + - const: top + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + description: + CSI input ports. + + patternProperties: + "^port@[0-5]$": + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false + + description: + Input port for receiving CSI data from a CSIPHY. + + properties: + endpoint: + $ref: video-interfaces.yaml# + unevaluatedProperties: false + + properties: + data-lanes: + minItems: 1 + maxItems: 4 + + bus-type: + enum: + - 1 # MEDIA_BUS_TYPE_CSI2_CPHY + - 4 # MEDIA_BUS_TYPE_CSI2_DPHY + + required: + - data-lanes + + vdd-csiphy01-0p9-supply: + description: + Phandle to a 0.9V regulator supply to CSIPHY0 and CSIPHY1 IP blocks. + + vdd-csiphy01-1p2-supply: + description: + Phandle to a 1.2V regulator supply to CSIPHY0 and CSIPHY1 IP blocks. + + vdd-csiphy24-0p9-supply: + description: + Phandle to a 0.9V regulator supply to CSIPHY2 and CSIPHY4 IP blocks. + + vdd-csiphy24-1p2-supply: + description: + Phandle to a 1.2V regulator supply to CSIPHY2 and CSIPHY4 IP blocks. + + vdd-csiphy35-0p9-supply: + description: + Phandle to a 0.9V regulator supply to CSIPHY3 and CSIPHY5 IP blocks. + + vdd-csiphy35-1p2-supply: + description: + Phandle to a 1.2V regulator supply to CSIPHY3 and CSIPHY5 IP blocks. + +required: + - compatible + - reg + - reg-names + - clocks + - clock-names + - interconnects + - interconnect-names + - interrupts + - interrupt-names + - iommus + - power-domains + - power-domain-names + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@acb6000 { + compatible = "qcom,sm8650-camss"; + reg = <0 0x0acb6000 0 0x1000>, + <0 0x0acb8000 0 0x1000>, + <0 0x0acba000 0 0x1000>, + <0 0x0acbc000 0 0x1000>, + <0 0x0accb000 0 0x1000>, + <0 0x0acd0000 0 0x1000>, + <0 0x0ace4000 0 0x2000>, + <0 0x0ace6000 0 0x2000>, + <0 0x0ace8000 0 0x2000>, + <0 0x0acea000 0 0x2000>, + <0 0x0acec000 0 0x2000>, + <0 0x0acee000 0 0x2000>, + <0 0x0ac62000 0 0xf000>, + <0 0x0ac71000 0 0xf000>, + <0 0x0ac80000 0 0xf000>, + <0 0x0accc000 0 0x2000>, + <0 0x0acd1000 0 0x2000>; + reg-names = "csid_wrapper", + "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csiphy0", + "csiphy1", + "csiphy2", + "csiphy3", + "csiphy4", + "csiphy5", + "vfe0", + "vfe1", + "vfe2", + "vfe_lite0", + "vfe_lite1"; + clocks = <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CPAS_IFE_0_CLK>, + <&camcc CAM_CC_CPAS_IFE_1_CLK>, + <&camcc CAM_CC_CPAS_IFE_2_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI3PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY3_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI5PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY5_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_QDSS_DEBUG_XO_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_2_CLK>, + <&camcc CAM_CC_IFE_2_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>; + clock-names = "camnoc_axi", + "cpas_ahb", + "cpas_fast_ahb", + "cpas_vfe0", + "cpas_vfe1", + "cpas_vfe2", + "cpas_vfe_lite", + "csid", + "csiphy0", + "csiphy0_timer", + "csiphy1", + "csiphy1_timer", + "csiphy2", + "csiphy2_timer", + "csiphy3", + "csiphy3_timer", + "csiphy4", + "csiphy4_timer", + "csiphy5", + "csiphy5_timer", + "csiphy_rx", + "gcc_axi_hf", + "qdss_debug_xo", + "vfe0", + "vfe0_fast_ahb", + "vfe1", + "vfe1_fast_ahb", + "vfe2", + "vfe2_fast_ahb", + "vfe_lite", + "vfe_lite_ahb", + "vfe_lite_cphy_rx", + "vfe_lite_csid"; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csiphy0", + "csiphy1", + "csiphy2", + "csiphy3", + "csiphy4", + "csiphy5", + "vfe0", + "vfe1", + "vfe2", + "vfe_lite0", + "vfe_lite1"; + interconnects = <&gem_noc MASTER_APPSS_PROC 0 + &config_noc SLAVE_CAMERA_CFG 0>, + <&mmss_noc MASTER_CAMNOC_HF 0 + &mc_virt SLAVE_EBI1 0>; + interconnect-names = "ahb", "hf_mnoc"; + iommus = <&apps_smmu 0x800 0x20>, + <&apps_smmu 0x18a0 0x40>, + <&apps_smmu 0x1860 0x00>; + power-domains = <&camcc CAM_CC_IFE_0_GDSC>, + <&camcc CAM_CC_IFE_1_GDSC>, + <&camcc CAM_CC_IFE_2_GDSC>, + <&camcc CAM_CC_TITAN_TOP_GDSC>; + power-domain-names = "ife0", "ife1", "ife2", "top"; + vdd-csiphy01-0p9-supply = <&vreg_0p9>; + vdd-csiphy01-1p2-supply = <&vreg_1p2>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + csiphy1_ep: endpoint { + data-lanes = <0 1>; + remote-endpoint = <&camera_sensor>; + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml b/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml index b075341caafc1..94a9fac227dd8 100644 --- a/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml +++ b/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml @@ -39,6 +39,14 @@ properties: - const: vfe_lite0 - const: vfe_lite1 + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + + ranges: true + clocks: maxItems: 29 @@ -104,7 +112,22 @@ properties: - const: sf_icp_mnoc iommus: - maxItems: 8 + oneOf: + - items: + - description: S1 HLOS IFE and IFE_LITE non-protected read + - description: S1 HLOS IFE and IFE_LITE non-protected write + - description: S1 HLOS SFE non-protected read + - description: S1 HLOS SFE non-protected write + - description: S1 HLOS CDM IFE non-protected + - description: Legacy slot 0 - do not use + - description: Legacy slot 1 - do not use + - description: Legacy slot 2 - do not use + - items: + - description: S1 HLOS IFE and IFE_LITE non-protected read + - description: S1 HLOS IFE and IFE_LITE non-protected write + - description: S1 HLOS SFE non-protected read + - description: S1 HLOS SFE non-protected write + - description: S1 HLOS CDM IFE non-protected power-domains: items: @@ -126,11 +149,22 @@ properties: description: Phandle to 1.8V regulator supply to a PHY. + phys: + maxItems: 4 + + phy-names: + items: + - const: csiphy0 + - const: csiphy1 + - const: csiphy2 + - const: csiphy4 + ports: $ref: /schemas/graph.yaml#/properties/ports description: - CSI input ports. + CSI input ports. Supports either standard single sensor mode or + Qualcomm's combo mode with one sensor in 2x1 + 1x1 data-lane, clock-lane mode. patternProperties: "^port@[0-3]$": @@ -138,26 +172,94 @@ properties: unevaluatedProperties: false description: - Input port for receiving CSI data from a CSIPHY. + Input port for receiving CSI data. properties: - endpoint: + endpoint@0: $ref: video-interfaces.yaml# unevaluatedProperties: false + description: + Endpoint for receiving a single sensor input (or first leg of combo). + properties: data-lanes: minItems: 1 - maxItems: 4 + maxItems: 4 # Base max allows 4 (for D-PHY) + + clock-lanes: + maxItems: 1 bus-type: enum: - 1 # MEDIA_BUS_TYPE_CSI2_CPHY - 4 # MEDIA_BUS_TYPE_CSI2_DPHY + endpoint@1: + $ref: video-interfaces.yaml# + unevaluatedProperties: false + + description: + Endpoint for receiving the second leg of a combo sensor input. + + properties: + data-lanes: + maxItems: 1 + + clock-lanes: + maxItems: 1 + + bus-type: + const: 4 # Combo is D-PHY specific + required: - data-lanes + allOf: + # Case 1: Combo Mode (endpoint@1 is present) + # If endpoint@1 exists, we restrict endpoint@0 to 2 lanes (D-PHY split) + - if: + required: + - endpoint@1 + then: + properties: + endpoint@0: + properties: + data-lanes: + minItems: 2 + maxItems: 2 + bus-type: + const: 4 + endpoint@1: + properties: + data-lanes: + minItems: 1 + maxItems: 1 + bus-type: + const: 4 + + # Case 2: Single Mode (endpoint@1 is missing) + # We explicitly allow up to 4 lanes here to cover the D-PHY use case. + - if: + not: + required: + - endpoint@1 + then: + properties: + endpoint@0: + properties: + data-lanes: + minItems: 1 + maxItems: 4 + +patternProperties: + "^phy@[0-9a-f]+$": + $ref: /schemas/phy/qcom,x1e80100-csi2-phy.yaml + unevaluatedProperties: false + + "^opp-table(-.*)?$": + type: object + required: - compatible - reg @@ -171,8 +273,6 @@ required: - iommus - power-domains - power-domain-names - - vdd-csiphy-0p8-supply - - vdd-csiphy-1p2-supply - ports additionalProperties: false @@ -184,6 +284,7 @@ examples: #include #include #include + #include #include soc { @@ -229,6 +330,10 @@ examples: "vfe_lite0", "vfe_lite1"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + clocks = <&camcc CAM_CC_CAMNOC_AXI_NRT_CLK>, <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, <&camcc CAM_CC_CORE_AHB_CLK>, @@ -332,13 +437,10 @@ examples: "sf_icp_mnoc"; iommus = <&apps_smmu 0x800 0x60>, + <&apps_smmu 0x820 0x60>, + <&apps_smmu 0x840 0x60>, <&apps_smmu 0x860 0x60>, - <&apps_smmu 0x1800 0x60>, - <&apps_smmu 0x1860 0x60>, - <&apps_smmu 0x18e0 0x00>, - <&apps_smmu 0x1980 0x20>, - <&apps_smmu 0x1900 0x00>, - <&apps_smmu 0x19a0 0x20>; + <&apps_smmu 0x18a0 0x0>; power-domains = <&camcc CAM_CC_IFE_0_GDSC>, <&camcc CAM_CC_IFE_1_GDSC>, diff --git a/Documentation/devicetree/bindings/media/qcom,x1p42100-camss.yaml b/Documentation/devicetree/bindings/media/qcom,x1p42100-camss.yaml new file mode 100644 index 0000000000000..8bfa7e616c3b6 --- /dev/null +++ b/Documentation/devicetree/bindings/media/qcom,x1p42100-camss.yaml @@ -0,0 +1,424 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/qcom,x1p42100-camss.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm X1P42100 Camera Subsystem (CAMSS) + +maintainers: + - Wenmeng Liu + +description: + The CAMSS IP is a CSI decoder and ISP present on Qualcomm platforms. + +properties: + compatible: + const: qcom,x1p42100-camss + + reg: + maxItems: 14 + + reg-names: + items: + - const: csid0 + - const: csid1 + - const: csid2 + - const: csid_lite0 + - const: csid_lite1 + - const: csid_wrapper + - const: csiphy0 + - const: csiphy4 + - const: csitpg0 + - const: csitpg1 + - const: csitpg2 + - const: vfe0 + - const: vfe_lite0 + - const: vfe_lite1 + + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + + ranges: true + + clocks: + maxItems: 22 + + clock-names: + items: + - const: camnoc_nrt_axi + - const: camnoc_rt_axi + - const: core_ahb + - const: cpas_ahb + - const: cpas_fast_ahb + - const: cpas_vfe0 + - const: cpas_vfe_lite + - const: cphy_rx_clk_src + - const: csid + - const: csid_csiphy_rx + - const: csiphy0 + - const: csiphy0_timer + - const: csiphy4 + - const: csiphy4_timer + - const: gcc_axi_hf + - const: gcc_axi_sf + - const: vfe0 + - const: vfe0_fast_ahb + - const: vfe_lite + - const: vfe_lite_ahb + - const: vfe_lite_cphy_rx + - const: vfe_lite_csid + + interrupts: + maxItems: 10 + + interrupt-names: + items: + - const: csid0 + - const: csid1 + - const: csid2 + - const: csid_lite0 + - const: csid_lite1 + - const: csiphy0 + - const: csiphy4 + - const: vfe0 + - const: vfe_lite0 + - const: vfe_lite1 + + interconnects: + maxItems: 4 + + interconnect-names: + items: + - const: ahb + - const: hf_mnoc + - const: sf_mnoc + - const: sf_icp_mnoc + + iommus: + oneOf: + - items: + - description: S1 HLOS IFE and IFE_LITE non-protected read + - description: S1 HLOS IFE and IFE_LITE non-protected write + - description: S1 HLOS SFE non-protected read + - description: S1 HLOS SFE non-protected write + - description: S1 HLOS CDM IFE non-protected + - description: Legacy slot 0 - do not use + - description: Legacy slot 1 - do not use + - description: Legacy slot 2 - do not use + - items: + - description: S1 HLOS IFE and IFE_LITE non-protected read + - description: S1 HLOS IFE and IFE_LITE non-protected write + - description: S1 HLOS SFE non-protected read + - description: S1 HLOS SFE non-protected write + - description: S1 HLOS CDM IFE non-protected + + power-domains: + items: + - description: IFE0 GDSC - Image Front End, Global Distributed Switch Controller. + - description: Titan Top GDSC - Titan ISP Block, Global Distributed Switch Controller. + + power-domain-names: + items: + - const: ife0 + - const: top + + vdd-csiphy-0p8-supply: + description: + 0.8V supply to a PHY. + + vdd-csiphy-1p2-supply: + description: + 1.2V supply to a PHY. + + phys: + maxItems: 2 + + phy-names: + items: + - const: csiphy0 + - const: csiphy4 + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + description: + CSI input ports. Supports either standard single sensor mode or + Qualcomm's combo mode with one sensor in 2x1 + 1x1 data-lane, clock-lane mode. + + patternProperties: + "^port@[0-3]$": + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false + + description: + Input port for receiving CSI data. + + properties: + endpoint@0: + $ref: video-interfaces.yaml# + unevaluatedProperties: false + + description: + Endpoint for receiving a single sensor input (or first leg of combo). + + properties: + data-lanes: + minItems: 1 + maxItems: 4 # Base max allows 4 (for D-PHY) + + clock-lanes: + maxItems: 1 + + bus-type: + enum: + - 1 # MEDIA_BUS_TYPE_CSI2_CPHY + - 4 # MEDIA_BUS_TYPE_CSI2_DPHY + + endpoint@1: + $ref: video-interfaces.yaml# + unevaluatedProperties: false + + description: + Endpoint for receiving the second leg of a combo sensor input. + + properties: + data-lanes: + maxItems: 1 + + clock-lanes: + maxItems: 1 + + bus-type: + const: 4 # Combo is D-PHY specific + + required: + - data-lanes + + allOf: + # Case 1: Combo Mode (endpoint@1 is present) + # If endpoint@1 exists, we restrict endpoint@0 to 2 lanes (D-PHY split) + - if: + required: + - endpoint@1 + then: + properties: + endpoint@0: + properties: + data-lanes: + minItems: 2 + maxItems: 2 + bus-type: + const: 4 + endpoint@1: + properties: + data-lanes: + minItems: 1 + maxItems: 1 + bus-type: + const: 4 + + # Case 2: Single Mode (endpoint@1 is missing) + # We explicitly allow up to 4 lanes here to cover the D-PHY use case. + - if: + not: + required: + - endpoint@1 + then: + properties: + endpoint@0: + properties: + data-lanes: + minItems: 1 + maxItems: 4 + +patternProperties: + "^phy@[0-9a-f]+$": + $ref: /schemas/phy/qcom,x1e80100-csi2-phy.yaml + unevaluatedProperties: false + + "^opp-table(-.*)?$": + type: object + +required: + - compatible + - reg + - reg-names + - clocks + - clock-names + - interrupts + - interrupt-names + - interconnects + - interconnect-names + - iommus + - power-domains + - power-domain-names + - ports + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + #include + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + camss: isp@acb7000 { + compatible = "qcom,x1p42100-camss"; + + reg = <0 0x0acb7000 0 0x2000>, + <0 0x0acb9000 0 0x2000>, + <0 0x0acbb000 0 0x2000>, + <0 0x0acc6000 0 0x1000>, + <0 0x0acca000 0 0x1000>, + <0 0x0acb6000 0 0x1000>, + <0 0x0ace4000 0 0x1000>, + <0 0x0acec000 0 0x4000>, + <0 0x0acf6000 0 0x1000>, + <0 0x0acf7000 0 0x1000>, + <0 0x0acf8000 0 0x1000>, + <0 0x0ac62000 0 0x4000>, + <0 0x0acc7000 0 0x2000>, + <0 0x0accb000 0 0x2000>; + + reg-names = "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csid_wrapper", + "csiphy0", + "csiphy4", + "csitpg0", + "csitpg1", + "csitpg2", + "vfe0", + "vfe_lite0", + "vfe_lite1"; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + clocks = <&camcc CAM_CC_CAMNOC_AXI_NRT_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CPAS_IFE_0_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>; + + clock-names = "camnoc_nrt_axi", + "camnoc_rt_axi", + "core_ahb", + "cpas_ahb", + "cpas_fast_ahb", + "cpas_vfe0", + "cpas_vfe_lite", + "cphy_rx_clk_src", + "csid", + "csid_csiphy_rx", + "csiphy0", + "csiphy0_timer", + "csiphy4", + "csiphy4_timer", + "gcc_axi_hf", + "gcc_axi_sf", + "vfe0", + "vfe0_fast_ahb", + "vfe_lite", + "vfe_lite_ahb", + "vfe_lite_cphy_rx", + "vfe_lite_csid"; + + interrupts = , + , + , + , + , + , + , + , + , + ; + + interrupt-names = "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csiphy0", + "csiphy4", + "vfe0", + "vfe_lite0", + "vfe_lite1"; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_ICP QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + + interconnect-names = "ahb", + "hf_mnoc", + "sf_mnoc", + "sf_icp_mnoc"; + + iommus = <&apps_smmu 0x800 0x60>, + <&apps_smmu 0x820 0x60>, + <&apps_smmu 0x840 0x60>, + <&apps_smmu 0x860 0x60>, + <&apps_smmu 0x18a0 0x0>; + + power-domains = <&camcc CAM_CC_IFE_0_GDSC>, + <&camcc CAM_CC_TITAN_TOP_GDSC>; + + power-domain-names = "ife0", + "top"; + + vdd-csiphy-0p8-supply = <&csiphy_0p8_supply>; + vdd-csiphy-1p2-supply = <&csiphy_1p2_supply>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + csiphy_ep0: endpoint { + data-lanes = <0 1>; + remote-endpoint = <&sensor_ep>; + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml index 078a6886f8b1e..9da1ab1502089 100644 --- a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml +++ b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml @@ -78,6 +78,9 @@ properties: - qcom,pmc8380 - qcom,pmd8028 - qcom,pmd9635 + - qcom,pmh0101 + - qcom,pmh0104 + - qcom,pmh0110 - qcom,pmi632 - qcom,pmi8950 - qcom,pmi8962 @@ -87,6 +90,7 @@ properties: - qcom,pmk8002 - qcom,pmk8350 - qcom,pmk8550 + - qcom,pmk8850 - qcom,pmm8155au - qcom,pmm8654au - qcom,pmp8074 @@ -180,6 +184,10 @@ patternProperties: type: object $ref: /schemas/rtc/qcom-pm8xxx-rtc.yaml# + "^sensor@[0-9a-f]+$": + type: object + $ref: /schemas/hwmon/qcom,bcl-hwmon.yaml# + "^temp-alarm@[0-9a-f]+$": type: object $ref: /schemas/thermal/qcom,spmi-temp-alarm.yaml# diff --git a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml index 14ae3f00ef7e0..ae55b0a70766a 100644 --- a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml +++ b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml @@ -48,6 +48,7 @@ properties: - qcom,tcsr-ipq8064 - qcom,tcsr-ipq8074 - qcom,tcsr-ipq9574 + - qcom,tcsr-kaanapali - qcom,tcsr-mdm9615 - qcom,tcsr-msm8226 - qcom,tcsr-msm8660 diff --git a/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml b/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml index 9fce8cd7b0b62..d24950ccea952 100644 --- a/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml +++ b/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml @@ -41,7 +41,7 @@ properties: patternProperties: "^sdhci@[0-9a-f]+$": type: object - $ref: mmc-controller.yaml + $ref: sdhci-common.yaml unevaluatedProperties: false properties: diff --git a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml index 594bd174ff211..aad9f0b86b541 100644 --- a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml +++ b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml @@ -42,6 +42,7 @@ properties: - qcom,ipq5424-sdhci - qcom,ipq6018-sdhci - qcom,ipq9574-sdhci + - qcom,kaanapali-sdhci - qcom,milos-sdhci - qcom,qcm2290-sdhci - qcom,qcs404-sdhci @@ -138,6 +139,11 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 description: platform specific settings for DLL_CONFIG reg. + qcom,ice: + $ref: /schemas/types.yaml#/definitions/phandle + description: + phandle to the Inline Crypto Engine (ICE) hardware block for this controller. + iommus: minItems: 1 maxItems: 8 @@ -191,35 +197,69 @@ allOf: enum: - qcom,sdhci-msm-v4 then: - properties: - reg: - minItems: 2 - items: - - description: Host controller register map - - description: SD Core register map - - description: CQE register map - - description: Inline Crypto Engine register map - reg-names: - minItems: 2 - items: - - const: hc - - const: core - - const: cqhci - - const: ice + if: + required: + - qcom,ice + then: + properties: + reg: + minItems: 2 + items: + - description: Host controller register map + - description: SD Core register map + - description: CQE register map + reg-names: + minItems: 2 + items: + - const: hc + - const: core + - const: cqhci + else: + properties: + reg: + minItems: 2 + items: + - description: Host controller register map + - description: SD Core register map + - description: CQE register map + - description: Inline Crypto Engine register map + reg-names: + minItems: 2 + items: + - const: hc + - const: core + - const: cqhci + - const: ice else: - properties: - reg: - minItems: 1 - items: - - description: Host controller register map - - description: CQE register map - - description: Inline Crypto Engine register map - reg-names: - minItems: 1 - items: - - const: hc - - const: cqhci - - const: ice + if: + required: + - qcom,ice + then: + properties: + reg: + minItems: 1 + items: + - description: Host controller register map + - description: CQE register map + reg-names: + minItems: 1 + items: + - const: hc + - const: cqhci + else: + properties: + reg: + minItems: 1 + items: + - description: Host controller register map + - description: CQE register map + - description: Inline Crypto Engine register map + reg-names: + minItems: 1 + items: + - const: hc + - const: cqhci + - const: ice unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/net/can/microchip,mcp251xfd.yaml b/Documentation/devicetree/bindings/net/can/microchip,mcp251xfd.yaml index c155c9c6db392..49d6f6bbe1939 100644 --- a/Documentation/devicetree/bindings/net/can/microchip,mcp251xfd.yaml +++ b/Documentation/devicetree/bindings/net/can/microchip,mcp251xfd.yaml @@ -49,6 +49,17 @@ properties: Must be half or less of "clocks" frequency. maximum: 20000000 + gpio-controller: true + + "#gpio-cells": + const: 2 + +patternProperties: + "^.+-hog(-[0-9]+)?$": + type: object + required: + - gpio-hog + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml index 3f6dc6a3a9f1a..7d1612acca48d 100644 --- a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml +++ b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml @@ -39,6 +39,7 @@ properties: - qcom,qcs404-qfprom - qcom,qcs615-qfprom - qcom,qcs8300-qfprom + - qcom,sa8775p-qfprom - qcom,sar2130p-qfprom - qcom,sc7180-qfprom - qcom,sc7280-qfprom diff --git a/Documentation/devicetree/bindings/pci/amlogic,axg-pcie.yaml b/Documentation/devicetree/bindings/pci/amlogic,axg-pcie.yaml index 79a21ba0f9fd6..c8258ef403283 100644 --- a/Documentation/devicetree/bindings/pci/amlogic,axg-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/amlogic,axg-pcie.yaml @@ -36,13 +36,13 @@ properties: reg: items: - - description: External local bus interface registers + - description: Data Bus Interface registers - description: Meson designed configuration registers - description: PCIe configuration space reg-names: items: - - const: elbi + - const: dbi - const: cfg - const: config @@ -113,7 +113,7 @@ examples: pcie: pcie@f9800000 { compatible = "amlogic,axg-pcie", "snps,dw-pcie"; reg = <0xf9800000 0x400000>, <0xff646000 0x2000>, <0xf9f00000 0x100000>; - reg-names = "elbi", "cfg", "config"; + reg-names = "dbi", "cfg", "config"; interrupts = ; clocks = <&pclk>, <&clk_port>, <&clk_phy>; clock-names = "pclk", "port", "general"; diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml index 4d0a915566030..f760807b5feb6 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml @@ -76,6 +76,11 @@ properties: items: - const: pci +required: + - power-domains + - resets + - reset-names + allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml index 15ba2385eb73c..b6fb0e37cec59 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml @@ -61,6 +61,9 @@ properties: required: - interconnects - interconnect-names + - power-domains + - resets + - reset-names allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8150.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8150.yaml index 26b247a41785f..17c0d58af37c6 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8150.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8150.yaml @@ -74,6 +74,11 @@ properties: items: - const: pci +required: + - power-domains + - resets + - reset-names + allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8250.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8250.yaml index af4dae68d5087..591f1a1eddde1 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8250.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8250.yaml @@ -83,6 +83,11 @@ properties: items: - const: pci +required: + - power-domains + - resets + - reset-names + allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml index dde3079adbb33..299473af51d19 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml @@ -73,6 +73,11 @@ properties: items: - const: pci +required: + - power-domains + - resets + - reset-names + allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml index 6e0a6d8f0ed07..6a17d753122c6 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml @@ -77,6 +77,11 @@ properties: items: - const: pci +required: + - power-domains + - resets + - reset-names + allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml index 38b561e23c1fd..29d7df7aca60b 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml @@ -83,6 +83,11 @@ properties: - const: pci # PCIe core reset - const: link_down # PCIe link down reset +required: + - power-domains + - resets + - reset-names + allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml index 61581ffbfb248..59be6c66b39b5 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml @@ -16,7 +16,12 @@ description: properties: compatible: - const: qcom,pcie-x1e80100 + oneOf: + - const: qcom,pcie-x1e80100 + - items: + - enum: + - qcom,glymur-pcie + - const: qcom,pcie-x1e80100 reg: minItems: 6 diff --git a/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml b/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml new file mode 100644 index 0000000000000..fae4660647809 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pci/toshiba,tc9563.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Toshiba TC9563 PCIe switch + +maintainers: + - Krishna Chaitanya Chundru + +description: | + Toshiba TC9563 PCIe switch has one upstream and three downstream ports. + The 3rd downstream port has integrated endpoint device of Ethernet MAC. + Other two downstream ports are supposed to connect to external device. + + The TC9563 PCIe switch can be configured through I2C interface before + PCIe link is established to change FTS, ASPM related entry delays, + tx amplitude etc for better power efficiency and functionality. + +properties: + compatible: + enum: + - pci1179,0623 + + reg: + maxItems: 1 + + resx-gpios: + maxItems: 1 + description: + GPIO controlling the RESX# pin. + + vdd18-supply: true + + vdd09-supply: true + + vddc-supply: true + + vddio1-supply: true + + vddio2-supply: true + + vddio18-supply: true + + i2c-parent: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + A phandle to the parent I2C node and the slave address of the device + used to configure tc9563 to change FTS, tx amplitude etc. + items: + - description: Phandle to the I2C controller node + - description: I2C slave address + +patternProperties: + "^pcie@[1-3],0$": + description: + child nodes describing the internal downstream ports of + the tc9563 switch. + type: object + allOf: + - $ref: "#/$defs/tc9563-node" + - $ref: /schemas/pci/pci-pci-bridge.yaml# + unevaluatedProperties: false + +$defs: + tc9563-node: + type: object + + properties: + toshiba,tx-amplitude-microvolt: + description: + Change Tx Margin setting for low power consumption. + + toshiba,no-dfe-support: + type: boolean + description: + Disable DFE (Decision Feedback Equalizer), which mitigates + intersymbol interference and some reflections caused by + impedance mismatches. + +required: + - resx-gpios + - vdd18-supply + - vdd09-supply + - vddc-supply + - vddio1-supply + - vddio2-supply + - vddio18-supply + - i2c-parent + +allOf: + - $ref: "#/$defs/tc9563-node" + - $ref: /schemas/pci/pci-bus-common.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + + pcie { + #address-cells = <3>; + #size-cells = <2>; + + pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + bus-range = <0x01 0xff>; + + pcie@0,0 { + compatible = "pci1179,0623"; + + reg = <0x10000 0x0 0x0 0x0 0x0>; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + ranges; + bus-range = <0x02 0xff>; + + i2c-parent = <&qup_i2c 0x77>; + + vdd18-supply = <&vdd>; + vdd09-supply = <&vdd>; + vddc-supply = <&vdd>; + vddio1-supply = <&vdd>; + vddio2-supply = <&vdd>; + vddio18-supply = <&vdd>; + + resx-gpios = <&gpio 1 GPIO_ACTIVE_LOW>; + + pcie@1,0 { + compatible = "pciclass,0604"; + reg = <0x20800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x03 0xff>; + + toshiba,no-dfe-support; + }; + + pcie@2,0 { + compatible = "pciclass,0604"; + reg = <0x21000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x04 0xff>; + }; + + pcie@3,0 { + compatible = "pciclass,0604"; + reg = <0x21800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x05 0xff>; + + toshiba,tx-amplitude-microvolt = <10>; + + ethernet@0,0 { + reg = <0x50000 0x0 0x0 0x0 0x0>; + }; + + ethernet@0,1 { + reg = <0x50100 0x0 0x0 0x0 0x0>; + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml index eb97181cbb957..ba757b08b9b1b 100644 --- a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml @@ -37,12 +37,15 @@ properties: - description: PLL register block clocks: - maxItems: 2 + minItems: 2 + maxItems: 3 clock-names: + minItems: 2 items: - const: aux - const: cfg_ahb + - const: ref "#clock-cells": const: 1 @@ -64,6 +67,30 @@ required: - "#clock-cells" - "#phy-cells" +allOf: + - if: + properties: + compatible: + enum: + - qcom,sa8775p-edp-phy + - qcom,x1e80100-dp-phy + then: + properties: + clocks: + minItems: 3 + maxItems: 3 + clock-names: + minItems: 3 + maxItems: 3 + else: + properties: + clocks: + minItems: 2 + maxItems: 2 + clock-names: + minItems: 2 + maxItems: 2 + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/phy/qcom,qcs615-qmp-usb3dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qcs615-qmp-usb3dp-phy.yaml new file mode 100644 index 0000000000000..efb465c71c1b5 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/qcom,qcs615-qmp-usb3dp-phy.yaml @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/qcom,qcs615-qmp-usb3dp-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm QMP USB3-DP PHY controller (DP, QCS615) + +maintainers: + - Xiangxu Yin + +description: + The QMP PHY controller supports physical layer functionality for both USB3 + and DisplayPort over USB-C. While it enables mode switching between USB3 and + DisplayPort, but does not support combo mode. + +properties: + compatible: + enum: + - qcom,qcs615-qmp-usb3-dp-phy + + reg: + maxItems: 1 + + clocks: + maxItems: 4 + + clock-names: + items: + - const: aux + - const: ref + - const: cfg_ahb + - const: pipe + + resets: + maxItems: 2 + + reset-names: + items: + - const: phy_phy + - const: dp_phy + + vdda-phy-supply: true + + vdda-pll-supply: true + + "#clock-cells": + const: 1 + description: + See include/dt-bindings/phy/phy-qcom-qmp.h + + "#phy-cells": + const: 1 + description: + See include/dt-bindings/phy/phy-qcom-qmp.h + + qcom,tcsr-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: phandle to TCSR hardware block + - description: offset of the VLS CLAMP register + - description: offset of the PHY mode register + description: Clamp and PHY mode register present in the TCSR + +required: + - compatible + - reg + - clocks + - clock-names + - resets + - reset-names + - vdda-phy-supply + - vdda-pll-supply + - "#clock-cells" + - "#phy-cells" + - qcom,tcsr-reg + +additionalProperties: false + +examples: + - | + #include + #include + + phy@88e8000 { + compatible = "qcom,qcs615-qmp-usb3-dp-phy"; + reg = <0x88e8000 0x2000>; + + clocks = <&gcc GCC_USB2_SEC_PHY_AUX_CLK>, + <&gcc GCC_USB3_SEC_CLKREF_CLK>, + <&gcc GCC_AHB2PHY_WEST_CLK>, + <&gcc GCC_USB2_SEC_PHY_PIPE_CLK>; + clock-names = "aux", + "ref", + "cfg_ahb", + "pipe"; + + resets = <&gcc GCC_USB3PHY_PHY_SEC_BCR>, + <&gcc GCC_USB3_DP_PHY_SEC_BCR>; + reset-names = "phy_phy", + "dp_phy"; + + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l12a>; + + #clock-cells = <1>; + #phy-cells = <1>; + + qcom,tcsr-reg = <&tcsr 0xbff0 0xb24c>; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,sa8775p-dwmac-sgmii-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sa8775p-dwmac-sgmii-phy.yaml index 90fc8c039219c..0a1330b9118d6 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sa8775p-dwmac-sgmii-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sa8775p-dwmac-sgmii-phy.yaml @@ -36,6 +36,10 @@ properties: description: Phandle to a regulator that provides power to the PHY. + vdda-0p9-supply: + description: + Phandle to a 0.9V regulator supply to the PHY. + "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml index 119b4ff36dbd6..fae9360cc3954 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml @@ -16,6 +16,8 @@ description: properties: compatible: enum: + - qcom,glymur-qmp-gen4x2-pcie-phy + - qcom,glymur-qmp-gen5x4-pcie-phy - qcom,qcs615-qmp-gen3x1-pcie-phy - qcom,qcs8300-qmp-gen4x2-pcie-phy - qcom,sa8775p-qmp-gen4x2-pcie-phy @@ -55,7 +57,7 @@ properties: clocks: minItems: 5 - maxItems: 7 + maxItems: 6 clock-names: minItems: 5 @@ -66,7 +68,6 @@ properties: - enum: [rchng, refgen] - const: pipe - const: pipediv2 - - const: phy_aux power-domains: maxItems: 1 @@ -178,6 +179,9 @@ allOf: compatible: contains: enum: + - qcom,glymur-qmp-gen4x2-pcie-phy + - qcom,glymur-qmp-gen5x4-pcie-phy + - qcom,qcs8300-qmp-gen4x2-pcie-phy - qcom,sa8775p-qmp-gen4x2-pcie-phy - qcom,sa8775p-qmp-gen4x4-pcie-phy - qcom,sc8280xp-qmp-gen3x1-pcie-phy @@ -200,19 +204,8 @@ allOf: compatible: contains: enum: - - qcom,qcs8300-qmp-gen4x2-pcie-phy - then: - properties: - clocks: - minItems: 7 - clock-names: - minItems: 7 - - - if: - properties: - compatible: - contains: - enum: + - qcom,glymur-qmp-gen4x2-pcie-phy + - qcom,glymur-qmp-gen5x4-pcie-phy - qcom,sm8550-qmp-gen4x2-pcie-phy - qcom,sm8650-qmp-gen4x2-pcie-phy - qcom,x1e80100-qmp-gen4x2-pcie-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml index fba7b2549ddee..166e3787db588 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml @@ -20,6 +20,10 @@ properties: - enum: - qcom,qcs615-qmp-ufs-phy - const: qcom,sm6115-qmp-ufs-phy + - items: + - enum: + - qcom,x1e80100-qmp-ufs-phy + - const: qcom,sm8550-qmp-ufs-phy - items: - enum: - qcom,qcs8300-qmp-ufs-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,x1e80100-csi2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,x1e80100-csi2-phy.yaml new file mode 100644 index 0000000000000..63114151104b4 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/qcom,x1e80100-csi2-phy.yaml @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/qcom,x1e80100-csi2-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm CSI2 PHY + +maintainers: + - Bryan O'Donoghue + +description: + Qualcomm MIPI CSI2 C-PHY/D-PHY combination PHY. Connects MIPI CSI2 sensors + to Qualcomm's Camera CSI Decoder. The PHY supports both C-PHY and D-PHY + modes. + +properties: + compatible: + const: qcom,x1e80100-csi2-phy + + reg: + maxItems: 1 + + "#phy-cells": + const: 1 + description: + The single cell specifies the PHY operating mode. + See include/dt-bindings/phy/phy-qcom-mipi-csi2.h for valid values. + + clocks: + maxItems: 2 + + clock-names: + items: + - const: core + - const: timer + + interrupts: + maxItems: 1 + + operating-points-v2: + maxItems: 1 + + power-domains: + items: + - description: MXC or MXA voltage rail + - description: MMCX voltage rail + + power-domain-names: + items: + - const: mx + - const: mmcx + + vdda-0p9-supply: + description: Phandle to a 0.9V regulator supply to a PHY. + + vdda-1p2-supply: + description: Phandle to 1.2V regulator supply to a PHY. + +required: + - compatible + - reg + - "#phy-cells" + - clocks + - clock-names + - interrupts + - operating-points-v2 + - power-domains + - power-domain-names + - vdda-0p9-supply + - vdda-1p2-supply + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + #include + + csiphy4: csiphy@ace4000 { + compatible = "qcom,x1e80100-csi2-phy"; + reg = <0x0ace4000 0x2000>; + #phy-cells = <1>; + + clocks = <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "core", + "timer"; + + operating-points-v2 = <&csiphy_opp_table>; + + interrupts = ; + + power-domains = <&rpmhpd RPMHPD_MX>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "mx", + "mmcx"; + + vdda-0p9-supply = <&vreg_l2c_0p8>; + vdda-1p2-supply = <&vreg_l1c_1p2>; + }; + + csiphy_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>, + <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>, + <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_low_svs>, + <&rpmhpd_opp_low_svs>; + }; + }; + + isp@acb7000 { + phys = <&csiphy4 PHY_QCOM_CSI2_MODE_DPHY>; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,kaanapali-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,kaanapali-tlmm.yaml new file mode 100644 index 0000000000000..53534a07a1f0e --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,kaanapali-tlmm.yaml @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,kaanapali-tlmm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Kaanapali TLMM block + +maintainers: + - Jingyi Wang + +description: + Top Level Mode Multiplexer pin controller in Qualcomm Kaanapali SoC. + +allOf: + - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml# + +properties: + compatible: + const: qcom,kaanapali-tlmm + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + gpio-reserved-ranges: + minItems: 1 + maxItems: 109 + + gpio-line-names: + maxItems: 217 + +patternProperties: + "-state$": + oneOf: + - $ref: "#/$defs/qcom-kaanapali-tlmm-state" + - patternProperties: + "-pins$": + $ref: "#/$defs/qcom-kaanapali-tlmm-state" + additionalProperties: false + +$defs: + qcom-kaanapali-tlmm-state: + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state + unevaluatedProperties: false + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + oneOf: + - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-9]|21[0-6])$" + - enum: [ ufs_reset, sdc2_clk, sdc2_cmd, sdc2_data ] + minItems: 1 + maxItems: 36 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + enum: [ gpio, aoss_cti, atest_char, atest_usb, audio_ext_mclk0, + audio_ext_mclk1, audio_ref_clk, cam_asc_mclk2, cam_asc_mclk4, + cam_mclk, cci_async_in, cci_i2c_scl, cci_i2c_sda, cci_timer, + cmu_rng, coex_uart1_rx, coex_uart1_tx, coex_uart2_rx, + coex_uart2_tx, dbg_out_clk, ddr_bist_complete, ddr_bist_fail, + ddr_bist_start, ddr_bist_stop, ddr_pxi0, ddr_pxi1, ddr_pxi2, + ddr_pxi3, dp_hot, egpio, gcc_gp1, gcc_gp2, gcc_gp3, gnss_adc0, + gnss_adc1, i2chub0_se0, i2chub0_se1, i2chub0_se2, i2chub0_se3, + i2chub0_se4, i2s0_data0, i2s0_data1, i2s0_sck, i2s0_ws, + i2s1_data0, i2s1_data1, i2s1_sck, i2s1_ws, ibi_i3c, jitter_bist, + mdp_esync0_out, mdp_esync1_out, mdp_vsync, mdp_vsync0_out, + mdp_vsync1_out, mdp_vsync2_out, mdp_vsync3_out, mdp_vsync5_out, + mdp_vsync_e, nav_gpio0, nav_gpio1, nav_gpio2, nav_gpio3, + pcie0_clk_req_n, phase_flag, pll_bist_sync, pll_clk_aux, + prng_rosc0, prng_rosc1, prng_rosc2, prng_rosc3, qdss_cti, + qdss_gpio_traceclk, qdss_gpio_tracectl, qdss_gpio_tracedata, + qlink_big_enable, qlink_big_request, qlink_little_enable, + qlink_little_request, qlink_wmss, qspi0, qspi1, qspi2, qspi3, + qspi_clk, qspi_cs, qup1_se0, qup1_se1, qup1_se2, qup1_se3, + qup1_se4, qup1_se5, qup1_se6, qup1_se7, qup2_se0, qup2_se1, + qup2_se2, qup2_se3, qup2_se4, qup3_se0, qup3_se1, qup3_se2, + qup3_se3, qup3_se4, qup3_se5, qup4_se0, qup4_se1, qup4_se2, + qup4_se3, qup4_se4, sd_write_protect, sdc40, sdc41, sdc42, sdc43, + sdc4_clk, sdc4_cmd, sys_throttle, tb_trig_sdc2, tb_trig_sdc4, + tmess_prng0, tmess_prng1, tmess_prng2, tmess_prng3, tsense_pwm1, + tsense_pwm2, tsense_pwm3, tsense_pwm4, tsense_pwm5, tsense_pwm6, + tsense_pwm7, uim0_clk, uim0_data, uim0_present, uim0_reset, uim1_clk, + uim1_data, uim1_present, uim1_reset, usb0_hs, usb_phy, vfr_0, vfr_1, + vsense_trigger_mirnat, wcn_sw, wcn_sw_ctrl ] + + required: + - pins + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + #include + + tlmm: pinctrl@f100000 { + compatible = "qcom,kaanapali-tlmm"; + reg = <0x0f100000 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&tlmm 0 0 218>; + interrupt-controller; + #interrupt-cells = <2>; + + qup-uart7-state { + pins = "gpio62", "gpio63"; + function = "qup1_se7"; + }; + }; +... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml index 5e6dfcc3fe9b3..8ae4489637f3d 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml @@ -59,7 +59,11 @@ properties: - qcom,pmc8180-gpio - qcom,pmc8180c-gpio - qcom,pmc8380-gpio + - qcom,pmcx0102-gpio - qcom,pmd8028-gpio + - qcom,pmh0101-gpio + - qcom,pmh0104-gpio + - qcom,pmh0110-gpio - qcom,pmi632-gpio - qcom,pmi8950-gpio - qcom,pmi8994-gpio @@ -68,6 +72,7 @@ properties: - qcom,pmiv0104-gpio - qcom,pmk8350-gpio - qcom,pmk8550-gpio + - qcom,pmk8850-gpio - qcom,pmm8155au-gpio - qcom,pmm8654au-gpio - qcom,pmp8074-gpio @@ -191,6 +196,8 @@ allOf: - qcom,pm8950-gpio - qcom,pm8953-gpio - qcom,pmi632-gpio + - qcom,pmh0104-gpio + - qcom,pmk8850-gpio then: properties: gpio-line-names: @@ -303,6 +310,8 @@ allOf: compatible: contains: enum: + - qcom,pmcx0102-gpio + - qcom,pmh0110-gpio - qcom,pmi8998-gpio then: properties: @@ -318,6 +327,7 @@ allOf: compatible: contains: enum: + - qcom,pmh0101-gpio - qcom,pmih0108-gpio then: properties: @@ -481,13 +491,18 @@ $defs: - gpio1-gpio22 for pm8994 - gpio1-gpio26 for pm8998 - gpio1-gpio22 for pma8084 + - gpio1-gpio14 for pmcx0102 - gpio1-gpio4 for pmd8028 + - gpio1-gpio18 for pmh0101 + - gpio1-gpio8 for pmh0104 + - gpio1-gpio14 for pmh0110 - gpio1-gpio8 for pmi632 - gpio1-gpio2 for pmi8950 - gpio1-gpio10 for pmi8994 - gpio1-gpio18 for pmih0108 - gpio1-gpio4 for pmk8350 - gpio1-gpio6 for pmk8550 + - gpio1-gpio8 for pmk8850 - gpio1-gpio10 for pmm8155au - gpio1-gpio12 for pmm8654au - gpio1-gpio12 for pmp8074 (holes on gpio1 and gpio12) diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml index af5fef872529b..27af5b8aa1340 100644 --- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml +++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml @@ -18,6 +18,7 @@ properties: oneOf: - enum: - qcom,glymur-rpmhpd + - qcom,kaanapali-rpmhpd - qcom,mdm9607-rpmpd - qcom,milos-rpmhpd - qcom,msm8226-rpmpd diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml index 4c5b0629aa3e6..4669095039c8c 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml @@ -51,10 +51,15 @@ description: | For PM8450, smps1 - smps6, ldo1 - ldo4 For PM8550, smps1 - smps6, ldo1 - ldo17, bob1 - bob2 For PM8998, smps1 - smps13, ldo1 - ldo28, lvs1 - lvs2 + For PMH0101, ldo1 - ldo18, bob1 - bob2 + For PMH0104, smps1 - smps4 + For PMH0110, smps1 - smps10, ldo1 - ldo4 For PMI8998, bob For PMC8380, smps1 - smps8, ldo1 - lodo3 + For PMCX0102, smps1 - smps10, ldo1 - ldo4 For PMR735A, smps1 - smps3, ldo1 - ldo7 For PMR735B, ldo1 - ldo12 + For PMR735D, ldo1 - ldo7 For PMX55, smps1 - smps7, ldo1 - ldo16 For PMX65, smps1 - smps8, ldo1 - ldo21 For PMX75, smps1 - smps10, ldo1 - ldo21 @@ -85,12 +90,17 @@ properties: - qcom,pmc8180-rpmh-regulators - qcom,pmc8180c-rpmh-regulators - qcom,pmc8380-rpmh-regulators + - qcom,pmcx0102-rpmh-regulators - qcom,pmg1110-rpmh-regulators + - qcom,pmh0101-rpmh-regulators + - qcom,pmh0104-rpmh-regulators + - qcom,pmh0110-rpmh-regulators - qcom,pmi8998-rpmh-regulators - qcom,pmm8155au-rpmh-regulators - qcom,pmm8654au-rpmh-regulators - qcom,pmr735a-rpmh-regulators - qcom,pmr735b-rpmh-regulators + - qcom,pmr735d-rpmh-regulators - qcom,pmx55-rpmh-regulators - qcom,pmx65-rpmh-regulators - qcom,pmx75-rpmh-regulators @@ -100,7 +110,7 @@ properties: RPMh resource name suffix used for the regulators found on this PMIC. $ref: /schemas/types.yaml#/definitions/string - enum: [a, b, c, d, e, f, g, h, i, j, k, l, m, n] + pattern: "^[a-n]|[A-N]_E[0-3]+$" qcom,always-wait-for-ack: description: | @@ -246,6 +256,7 @@ allOf: compatible: enum: - qcom,pm8005-rpmh-regulators + - qcom,pmh0104-rpmh-regulators then: patternProperties: "^vdd-s[1-4]-supply$": true @@ -422,6 +433,34 @@ allOf: properties: vdd-s1-supply: true + - if: + properties: + compatible: + enum: + - qcom,pmh0101-rpmh-regulators + then: + properties: + vdd-l1-l4-l10-supply: true + vdd-l2-l13-l14-supply: true + vdd-l3-l11-supply: true + vdd-l5-l16-supply: true + vdd-l6-l7-supply: true + vdd-l8-l9-supply: true + patternProperties: + "^vdd-l(1[2578])-supply$": true + "^vdd-bob[1-2]-supply$": true + + - if: + properties: + compatible: + enum: + - qcom,pmcx0102-rpmh-regulators + - qcom,pmh0110-rpmh-regulators + then: + patternProperties: + "^vdd-l[1-4]-supply$": true + "^vdd-s([1-9]|10)-supply$": true + - if: properties: compatible: @@ -459,6 +498,18 @@ allOf: patternProperties: "^vdd-l([3-6]|9|1[0-2])-supply$": true + - if: + properties: + compatible: + enum: + - qcom,pmr735d-rpmh-regulators + then: + properties: + vdd-l1-l2-l5-supply: true + vdd-l3-l4-supply: true + patternProperties: + "^vdd-l[6-7]-supply$": true + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml index 63a82e7a8bf88..6a736161d5aed 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml @@ -44,6 +44,9 @@ properties: - const: stop-ack - const: shutdown-ack + iommus: + maxItems: 1 + power-domains: minItems: 1 maxItems: 3 @@ -77,6 +80,12 @@ properties: and devices related to the ADSP. unevaluatedProperties: false + cooling: + $ref: /schemas/thermal/qcom,qmi-cooling.yaml# + description: + Cooling subnode which represents the cooling devices exposed by the Modem. + unevaluatedProperties: false + required: - clocks - clock-names diff --git a/Documentation/devicetree/bindings/serial/renesas,rsci.yaml b/Documentation/devicetree/bindings/serial/renesas,rsci.yaml index f50d8e02f4764..6b1f827a335b3 100644 --- a/Documentation/devicetree/bindings/serial/renesas,rsci.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,rsci.yaml @@ -54,8 +54,6 @@ properties: power-domains: maxItems: 1 - uart-has-rtscts: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/slimbus/slimbus.yaml b/Documentation/devicetree/bindings/slimbus/slimbus.yaml index 89017d9cda109..5a941610ce4ed 100644 --- a/Documentation/devicetree/bindings/slimbus/slimbus.yaml +++ b/Documentation/devicetree/bindings/slimbus/slimbus.yaml @@ -75,16 +75,22 @@ examples: #size-cells = <1>; ranges; - slim@28080000 { + controller@28080000 { compatible = "qcom,slim-ngd-v1.5.0"; reg = <0x091c0000 0x2c000>; interrupts = ; - #address-cells = <2>; + dmas = <&slimbam 3>, <&slimbam 4>; + dma-names = "rx", "tx"; + #address-cells = <1>; #size-cells = <0>; - - audio-codec@1,0 { + slim@1 { + reg = <1>; + #address-cells = <2>; + #size-cells = <0>; + codec@1,0 { compatible = "slim217,1a0"; reg = <1 0>; + }; }; + }; }; - }; diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml index 851a1260f8dc8..c5c1bac2db013 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml @@ -25,6 +25,8 @@ properties: compatible: items: - enum: + - qcom,glymur-aoss-qmp + - qcom,kaanapali-aoss-qmp - qcom,milos-aoss-qmp - qcom,qcs615-aoss-qmp - qcom,qcs8300-aoss-qmp diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,eud.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,eud.yaml index 84218636c0d8d..21f75038a81c6 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,eud.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,eud.yaml @@ -45,10 +45,63 @@ properties: $ref: /schemas/graph.yaml#/properties/port description: This port is to be attached to the type C connector. + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + +patternProperties: + "^eud-path@[0-1]$": + type: object + description: + Represents one High-Speed UTMI path that EUD intercepts. Use eud-path nodes + to associate role-switching behavior with specific port connections, allowing + EUD to manage role transitions independently for each UTMI path. + + properties: + reg: + maxItems: 1 + description: Path number + + usb-role-switch: + type: boolean + description: + Indicates that EUD should act as a role switch for this path. + In device role, debug mode inserts the EUD hub into the UTMI path. In + host role, the EUD hub is bypassed and UTMI traffic flows directly + between the PHY and the USB controller. + + ports: + $ref: /schemas/graph.yaml#/properties/ports + description: + These ports are to be attached to the endpoint of the USB controller node + and USB connector node. + + properties: + port@0: + $ref: /schemas/graph.yaml#/properties/port + description: This port is to be attached to the USB controller. + + port@1: + $ref: /schemas/graph.yaml#/properties/port + description: This port is to be attached to the USB connector. + + required: + - reg + - ports + + additionalProperties: false + required: - compatible - reg - - ports + +oneOf: + - required: + - ports + - required: + - eud-path@0 additionalProperties: false diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml index 7085bf88afaba..c570221094197 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml @@ -37,12 +37,19 @@ properties: - const: qcom,pmic-glink - items: - enum: + - qcom,kaanapali-pmic-glink - qcom,milos-pmic-glink - qcom,sm8650-pmic-glink - qcom,sm8750-pmic-glink - qcom,x1e80100-pmic-glink - const: qcom,sm8550-pmic-glink - const: qcom,pmic-glink + - items: + - enum: + - qcom,glymur-pmic-glink + - const: qcom,x1e80100-pmic-glink + - const: qcom,sm8550-pmic-glink + - const: qcom,pmic-glink '#address-cells': const: 1 diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml index 1fdbeecc5eff9..3a3313ea0890a 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml @@ -21,10 +21,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: Analog power supply - dvdd-supply: + DVDD-supply: description: Digital power supply reset-gpios: @@ -60,7 +60,7 @@ allOf: properties: dsd-path: false -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml index d3d494ae8abfe..dc8f85f266bf3 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml @@ -19,10 +19,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: A 1.8V supply that powers up the AVDD pin. - dvdd-supply: + DVDD-supply: description: A 1.2V supply that powers up the DVDD pin. reset-gpios: diff --git a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml index 4a7129d0b1574..551edf39e7663 100644 --- a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml +++ b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml @@ -164,7 +164,7 @@ allOf: properties: compatible: contains: - const: st,stm32mph7-sai + const: st,stm32h7-sai then: properties: clocks: diff --git a/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml b/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml index 1696ac46a660e..d9aac33b695bf 100644 --- a/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml +++ b/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml @@ -20,6 +20,7 @@ properties: compatible: items: - enum: + - qcom,qcs615-qspi - qcom,sc7180-qspi - qcom,sc7280-qspi - qcom,sdm845-qspi diff --git a/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml b/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml new file mode 100644 index 0000000000000..3b5005b96c6d5 --- /dev/null +++ b/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spmi/qcom,glymur-spmi-pmic-arb.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Glymur SPMI Controller (PMIC Arbiter v8) + +maintainers: + - David Collins + +description: | + The Glymur SPMI PMIC Arbiter implements HW version 8 and it's an SPMI + controller with wrapping arbitration logic to allow for multiple on-chip + devices to control up to 4 SPMI separate buses. + + The PMIC Arbiter can also act as an interrupt controller, providing interrupts + to slave devices. + +allOf: + - $ref: /schemas/spmi/qcom,spmi-pmic-arb-common.yaml + +properties: + compatible: + oneOf: + - items: + - enum: + - qcom,kaanapali-spmi-pmic-arb + - const: qcom,glymur-spmi-pmic-arb + - enum: + - qcom,glymur-spmi-pmic-arb + + reg: + items: + - description: core registers + - description: tx-channel per virtual slave registers + - description: rx-channel (called observer) per virtual slave registers + - description: channel to PMIC peripheral mapping registers + + reg-names: + items: + - const: core + - const: chnls + - const: obsrvr + - const: chnl_map + + ranges: true + + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + +patternProperties: + "^spmi@[a-f0-9]+$": + type: object + $ref: /schemas/spmi/spmi.yaml + unevaluatedProperties: false + + properties: + reg: + items: + - description: configuration registers + - description: interrupt controller registers + - description: channel owner EE mapping registers + + reg-names: + items: + - const: cnfg + - const: intr + - const: chnl_owner + + interrupts: + maxItems: 1 + + interrupt-names: + const: periph_irq + + interrupt-controller: true + + '#interrupt-cells': + const: 4 + description: | + cell 1: slave ID for the requested interrupt (0-15) + cell 2: peripheral ID for requested interrupt (0-255) + cell 3: the requested peripheral interrupt (0-7) + cell 4: interrupt flags indicating level-sense information, + as defined in dt-bindings/interrupt-controller/irq.h + +required: + - compatible + - reg-names + +unevaluatedProperties: false + +examples: + - | + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + arbiter@c400000 { + compatible = "qcom,glymur-spmi-pmic-arb"; + reg = <0x0 0xc400000 0x0 0x3000>, + <0x0 0xc900000 0x0 0x400000>, + <0x0 0xc4c0000 0x0 0x400000>, + <0x0 0xc403000 0x0 0x8000>; + reg-names = "core", "chnls", "obsrvr", "chnl_map"; + + qcom,ee = <0>; + qcom,channel = <0>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + spmi@c426000 { + reg = <0x0 0xc426000 0x0 0x4000>, + <0x0 0xc8c0000 0x0 0x10000>, + <0x0 0xc42a000 0x0 0x8000>; + reg-names = "cnfg", "intr", "chnl_owner"; + + interrupts-extended = <&pdc 1 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "periph_irq"; + interrupt-controller; + #interrupt-cells = <4>; + + #address-cells = <2>; + #size-cells = <0>; + }; + + spmi@c437000 { + reg = <0x0 0xc437000 0x0 0x4000>, + <0x0 0xc8d0000 0x0 0x10000>, + <0x0 0xc43b000 0x0 0x8000>; + reg-names = "cnfg", "intr", "chnl_owner"; + + interrupts-extended = <&pdc 3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "periph_irq"; + interrupt-controller; + #interrupt-cells = <4>; + + #address-cells = <2>; + #size-cells = <0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/spmi/qcom,spmi-pmic-arb-common.yaml b/Documentation/devicetree/bindings/spmi/qcom,spmi-pmic-arb-common.yaml new file mode 100644 index 0000000000000..8c38ed145e745 --- /dev/null +++ b/Documentation/devicetree/bindings/spmi/qcom,spmi-pmic-arb-common.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spmi/qcom,spmi-pmic-arb-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. SPMI Controller (common) + +maintainers: + - David Collins + +description: | + This defines some common properties used to define Qualcomm SPMI controllers + for PMIC arbiter. + +properties: + qcom,ee: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 5 + description: + indicates the active Execution Environment identifier + + qcom,channel: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 5 + description: + which of the PMIC Arb provided channels to use for accesses + +required: + - qcom,ee + - qcom,channel + +additionalProperties: true diff --git a/Documentation/devicetree/bindings/spmi/qcom,spmi-pmic-arb.yaml b/Documentation/devicetree/bindings/spmi/qcom,spmi-pmic-arb.yaml index 51daf1b847a9b..d0c683dd52841 100644 --- a/Documentation/devicetree/bindings/spmi/qcom,spmi-pmic-arb.yaml +++ b/Documentation/devicetree/bindings/spmi/qcom,spmi-pmic-arb.yaml @@ -19,6 +19,7 @@ description: | allOf: - $ref: spmi.yaml + - $ref: qcom,spmi-pmic-arb-common.yaml properties: compatible: @@ -71,20 +72,6 @@ properties: '#size-cells': true - qcom,ee: - $ref: /schemas/types.yaml#/definitions/uint32 - minimum: 0 - maximum: 5 - description: > - indicates the active Execution Environment identifier - - qcom,channel: - $ref: /schemas/types.yaml#/definitions/uint32 - minimum: 0 - maximum: 5 - description: > - which of the PMIC Arb provided channels to use for accesses - qcom,bus-id: $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 @@ -97,8 +84,6 @@ properties: required: - compatible - reg-names - - qcom,ee - - qcom,channel unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/spmi/qcom,x1e80100-spmi-pmic-arb.yaml b/Documentation/devicetree/bindings/spmi/qcom,x1e80100-spmi-pmic-arb.yaml index 7c3cc20a80d6c..08369fdd21612 100644 --- a/Documentation/devicetree/bindings/spmi/qcom,x1e80100-spmi-pmic-arb.yaml +++ b/Documentation/devicetree/bindings/spmi/qcom,x1e80100-spmi-pmic-arb.yaml @@ -17,6 +17,9 @@ description: | The PMIC Arbiter can also act as an interrupt controller, providing interrupts to slave devices. +allOf: + - $ref: qcom,spmi-pmic-arb-common.yaml + properties: compatible: oneOf: @@ -45,20 +48,6 @@ properties: '#size-cells': const: 2 - qcom,ee: - $ref: /schemas/types.yaml#/definitions/uint32 - minimum: 0 - maximum: 5 - description: > - indicates the active Execution Environment identifier - - qcom,channel: - $ref: /schemas/types.yaml#/definitions/uint32 - minimum: 0 - maximum: 5 - description: > - which of the PMIC Arb provided channels to use for accesses - patternProperties: "^spmi@[a-f0-9]+$": type: object @@ -96,10 +85,8 @@ patternProperties: required: - compatible - reg-names - - qcom,ee - - qcom,channel -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/sram/qcom,imem.yaml b/Documentation/devicetree/bindings/sram/qcom,imem.yaml index 6a627c57ae2fe..1e29a8ff287f3 100644 --- a/Documentation/devicetree/bindings/sram/qcom,imem.yaml +++ b/Documentation/devicetree/bindings/sram/qcom,imem.yaml @@ -19,6 +19,7 @@ properties: - enum: - qcom,apq8064-imem - qcom,ipq5424-imem + - qcom,kaanapali-imem - qcom,msm8226-imem - qcom,msm8974-imem - qcom,msm8976-imem diff --git a/Documentation/devicetree/bindings/thermal/qcom,qmi-cooling.yaml b/Documentation/devicetree/bindings/thermal/qcom,qmi-cooling.yaml new file mode 100644 index 0000000000000..90b46712d2412 --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/qcom,qmi-cooling.yaml @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright 2023 (c), Linaro Limited + +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/qcom,qmi-cooling.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm QMI based thermal mitigation (TMD) cooling devices. + +maintainers: + - Caleb Connolly + - Gaurav Kohli + +description: + Qualcomm QMI-based TMD cooling devices are used to mitigate thermal conditions + across multiple remote subsystems. These devices operate based on junction temperature + sensors (TSENS) associated with thermal zones for each subsystem. + + Each subnode corresponds to a control interface for a single instance of the TMD + service running on a remote subsystem. + +definitions: + tmd: + type: object + description: | + A single Thermal Mitigation Device exposed by a remote subsystem. + properties: + label: + maxItems: 1 + "#cooling-cells": + $ref: /schemas/thermal/thermal-cooling-devices.yaml#/properties/#cooling-cells + phandle: true + + required: + - label + - "#cooling-cells" + + additionalProperties: false + +properties: + compatible: + enum: + - qcom,qmi-cooling-modem + - qcom,qmi-cooling-cdsp + + vdd: + $ref: "#/definitions/tmd" + description: + Modem processor temperature TMD + properties: + label: + const: modem + +required: + - compatible + +allOf: + - if: + properties: + compatible: + contains: + const: qcom,qmi-cooling-cdsp + then: + properties: + cdsp_sw: + $ref: "#/definitions/tmd" + description: + CDSP software TMD + properties: + label: + const: cdsp_sw + +unevaluatedProperties: false + +examples: + - | + remoteproc-cdsp { + cooling { + compatible = "qcom,qmi-cooling-cdsp"; + + cdsp_sw0: cdsp_sw { + label = "cdsp_sw"; + #cooling-cells = <2>; + }; + }; + }; + + remoteproc-cdsp1 { + cooling { + compatible = "qcom,qmi-cooling-cdsp1"; + + cdsp_sw1: cdsp_sw { + label = "cdsp_sw"; + #cooling-cells = <2>; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml index 78e2f6573b96f..6e9d7996b7b81 100644 --- a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml +++ b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml @@ -50,12 +50,14 @@ properties: items: - enum: - qcom,glymur-tsens + - qcom,kaanapali-tsens - qcom,milos-tsens - qcom,msm8953-tsens - qcom,msm8996-tsens - qcom,msm8998-tsens - qcom,qcm2290-tsens - qcom,qcs615-tsens + - qcom,qcs8300-tsens - qcom,sa8255p-tsens - qcom,sa8775p-tsens - qcom,sar2130p-tsens @@ -74,6 +76,7 @@ properties: - qcom,sm8450-tsens - qcom,sm8550-tsens - qcom,sm8650-tsens + - qcom,sm8750-tsens - qcom,x1e80100-tsens - const: qcom,tsens-v2 diff --git a/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml b/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml index d94ef4e6b85a4..fe18e41ebac7a 100644 --- a/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml +++ b/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml @@ -31,21 +31,27 @@ select: properties: compatible: - items: - - enum: - - qcom,msm8998-ufshc - - qcom,qcs8300-ufshc - - qcom,sa8775p-ufshc - - qcom,sc7180-ufshc - - qcom,sc7280-ufshc - - qcom,sc8180x-ufshc - - qcom,sc8280xp-ufshc - - qcom,sm8250-ufshc - - qcom,sm8350-ufshc - - qcom,sm8450-ufshc - - qcom,sm8550-ufshc - - const: qcom,ufshc - - const: jedec,ufs-2.0 + oneOf: + - items: + - enum: + - qcom,x1e80100-ufshc + - const: qcom,sm8550-ufshc + - const: qcom,ufshc + - items: + - enum: + - qcom,msm8998-ufshc + - qcom,qcs8300-ufshc + - qcom,sa8775p-ufshc + - qcom,sc7180-ufshc + - qcom,sc7280-ufshc + - qcom,sc8180x-ufshc + - qcom,sc8280xp-ufshc + - qcom,sm8250-ufshc + - qcom,sm8350-ufshc + - qcom,sm8450-ufshc + - qcom,sm8550-ufshc + - const: qcom,ufshc + - const: jedec,ufs-2.0 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml b/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml index 9a94b2a74a1eb..6ab13785e8328 100644 --- a/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml +++ b/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml @@ -15,6 +15,7 @@ properties: - usb5e3,608 - usb5e3,610 - usb5e3,620 + - usb5e3,625 - usb5e3,626 reg: true @@ -26,6 +27,10 @@ properties: description: The regulator that provides 3.3V or 5.0V core power to the hub. + vdd12-supply: + description: + The regulator that provides 1.2V power to the hub. + peer-hub: true ports: @@ -56,6 +61,7 @@ allOf: properties: peer-hub: false vdd-supply: false + vdd12-supply: false - if: properties: @@ -68,6 +74,19 @@ allOf: properties: peer-hub: true vdd-supply: true + vdd12-supply: false + + - if: + properties: + compatible: + contains: + enum: + - usb5e3,625 + then: + properties: + peer-hub: true + vdd-supply: true + vdd12-supply: true unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml index d49a58d5478ff..301e873684ae9 100644 --- a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml @@ -67,6 +67,7 @@ properties: - qcom,sm8450-dwc3 - qcom,sm8550-dwc3 - qcom,sm8650-dwc3 + - qcom,sm8750-dwc3 - qcom,x1e80100-dwc3 - qcom,x1e80100-dwc3-mp - const: qcom,snps-dwc3 @@ -213,6 +214,7 @@ allOf: - qcom,sdx65-dwc3 - qcom,sdx75-dwc3 - qcom,sm6350-dwc3 + - qcom,sm8750-dwc3 then: properties: clocks: @@ -501,6 +503,7 @@ allOf: - qcom,sm8450-dwc3 - qcom,sm8550-dwc3 - qcom,sm8650-dwc3 + - qcom,sm8750-dwc3 then: properties: interrupts: diff --git a/Documentation/devicetree/bindings/usb/renesas,upd720201-pci.yaml b/Documentation/devicetree/bindings/usb/renesas,upd720201-pci.yaml new file mode 100644 index 0000000000000..34acee62cdd21 --- /dev/null +++ b/Documentation/devicetree/bindings/usb/renesas,upd720201-pci.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/usb/renesas,upd720201-pci.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: UPD720201/UPD720202 USB 3.0 xHCI Host Controller (PCIe) + +maintainers: + - Neil Armstrong + +description: + UPD720201 USB 3.0 xHCI Host Controller via PCIe x1 Gen2 interface. + The UPD720202 supports up to two downstream ports, while UPD720201 + supports up to four downstream USB 3.0 rev1.0 ports. + +properties: + compatible: + const: pci1912,0014 + + reg: + maxItems: 1 + + avdd33-supply: + description: +3.3 V power supply for analog circuit + + vdd10-supply: + description: +1.05 V power supply + + vdd33-supply: + description: +3.3 V power supply + +required: + - compatible + - reg + - avdd33-supply + - vdd10-supply + - vdd33-supply + +allOf: + - $ref: usb-xhci.yaml + +additionalProperties: false + +examples: + - | + pcie@0 { + reg = <0x0 0x1000>; + ranges = <0x02000000 0x0 0x100000 0x10000000 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + + usb-controller@0 { + compatible = "pci1912,0014"; + reg = <0x0 0x0 0x0 0x0 0x0>; + avdd33-supply = <&avdd33_reg>; + vdd10-supply = <&vdd10_reg>; + vdd33-supply = <&vdd33_reg>; + }; + }; diff --git a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml index bec1c8047bc05..06099e93c6c30 100644 --- a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml +++ b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml @@ -25,6 +25,14 @@ properties: interrupts: maxItems: 1 + id-gpios: + description: + An input gpio for USB ID pin. Upon detecting a UFP device, HD3SS3220 + will keep ID pin high if VBUS is not at VSafe0V. Once VBUS is at VSafe0V, + the HD3SS3220 will assert ID pin low. This is done to enforce Type-C + requirement that VBUS must be at VSafe0V before re-enabling VBUS. + maxItems: 1 + ports: $ref: /schemas/graph.yaml#/properties/ports description: OF graph bindings (specified in bindings/graph.txt) that model diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index f1d1882009ba9..7fa32190761b6 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -211,6 +211,8 @@ patternProperties: description: Axiado Corporation "^axis,.*": description: Axis Communications AB + "^ayaneo,.*": + description: Anyun Intelligent Technology (Hong Kong) Co., Ltd "^azoteq,.*": description: Azoteq (Pty) Ltd "^azw,.*": diff --git a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml index 49e2b807db0bc..f2c4bc900e5ff 100644 --- a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml @@ -22,6 +22,7 @@ properties: - qcom,apss-wdt-ipq5332 - qcom,apss-wdt-ipq5424 - qcom,apss-wdt-ipq9574 + - qcom,apss-wdt-kaanapali - qcom,apss-wdt-msm8226 - qcom,apss-wdt-msm8974 - qcom,apss-wdt-msm8994 @@ -42,6 +43,7 @@ properties: - qcom,apss-wdt-sm6350 - qcom,apss-wdt-sm8150 - qcom,apss-wdt-sm8250 + - qcom,apss-wdt-x1e80100 - const: qcom,kpss-wdt - const: qcom,kpss-wdt deprecated: true diff --git a/Documentation/driver-api/dmaengine/provider.rst b/Documentation/driver-api/dmaengine/provider.rst index 1594598b33178..6428211405472 100644 --- a/Documentation/driver-api/dmaengine/provider.rst +++ b/Documentation/driver-api/dmaengine/provider.rst @@ -630,6 +630,15 @@ DMA_CTRL_REUSE - This flag is only supported if the channel reports the DMA_LOAD_EOT capability. +- DMA_PREP_LOCK + + - If set, the DMA controller will be locked for the duration of the current + transaction. + +- DMA_PREP_UNLOCK + + - If set, DMA will release he controller lock. + General Design Notes ==================== diff --git a/Documentation/driver-api/dpll.rst b/Documentation/driver-api/dpll.rst index be1fc643b645e..83118c728ed90 100644 --- a/Documentation/driver-api/dpll.rst +++ b/Documentation/driver-api/dpll.rst @@ -198,26 +198,28 @@ be requested with the same attribute with ``DPLL_CMD_DEVICE_SET`` command. ================================== ====================================== Device may also provide ability to adjust a signal phase on a pin. -If pin phase adjustment is supported, minimal and maximal values that pin -handle shall be provide to the user on ``DPLL_CMD_PIN_GET`` respond -with ``DPLL_A_PIN_PHASE_ADJUST_MIN`` and ``DPLL_A_PIN_PHASE_ADJUST_MAX`` +If pin phase adjustment is supported, minimal and maximal values and +granularity that pin handle shall be provided to the user on +``DPLL_CMD_PIN_GET`` respond with ``DPLL_A_PIN_PHASE_ADJUST_MIN``, +``DPLL_A_PIN_PHASE_ADJUST_MAX`` and ``DPLL_A_PIN_PHASE_ADJUST_GRAN`` attributes. Configured phase adjust value is provided with ``DPLL_A_PIN_PHASE_ADJUST`` attribute of a pin, and value change can be requested with the same attribute with ``DPLL_CMD_PIN_SET`` command. - =============================== ====================================== - ``DPLL_A_PIN_ID`` configured pin id - ``DPLL_A_PIN_PHASE_ADJUST_MIN`` attr minimum value of phase adjustment - ``DPLL_A_PIN_PHASE_ADJUST_MAX`` attr maximum value of phase adjustment - ``DPLL_A_PIN_PHASE_ADJUST`` attr configured value of phase - adjustment on parent dpll device - ``DPLL_A_PIN_PARENT_DEVICE`` nested attribute for requesting - configuration on given parent dpll - device - ``DPLL_A_PIN_PARENT_ID`` parent dpll device id - ``DPLL_A_PIN_PHASE_OFFSET`` attr measured phase difference - between a pin and parent dpll device - =============================== ====================================== + ================================ ========================================== + ``DPLL_A_PIN_ID`` configured pin id + ``DPLL_A_PIN_PHASE_ADJUST_GRAN`` attr granularity of phase adjustment value + ``DPLL_A_PIN_PHASE_ADJUST_MIN`` attr minimum value of phase adjustment + ``DPLL_A_PIN_PHASE_ADJUST_MAX`` attr maximum value of phase adjustment + ``DPLL_A_PIN_PHASE_ADJUST`` attr configured value of phase + adjustment on parent dpll device + ``DPLL_A_PIN_PARENT_DEVICE`` nested attribute for requesting + configuration on given parent dpll + device + ``DPLL_A_PIN_PARENT_ID`` parent dpll device id + ``DPLL_A_PIN_PHASE_OFFSET`` attr measured phase difference + between a pin and parent dpll device + ================================ ========================================== All phase related values are provided in pico seconds, which represents time difference between signals phase. The negative value means that @@ -384,6 +386,8 @@ according to attribute purpose. frequencies ``DPLL_A_PIN_ANY_FREQUENCY_MIN`` attr minimum value of frequency ``DPLL_A_PIN_ANY_FREQUENCY_MAX`` attr maximum value of frequency + ``DPLL_A_PIN_PHASE_ADJUST_GRAN`` attr granularity of phase + adjustment value ``DPLL_A_PIN_PHASE_ADJUST_MIN`` attr minimum value of phase adjustment ``DPLL_A_PIN_PHASE_ADJUST_MAX`` attr maximum value of phase diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index ab989807a2cb6..c493fcbe0df4a 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -783,6 +783,56 @@ controlled by the "uuid" mount option, which supports these values: mounted with "uuid=on". +Durability and copy up +---------------------- + +The fsync(2) system call ensures that the data and metadata of a file +are safely written to the backing storage, which is expected to +guarantee the existence of the information post system crash. + +Without an fsync(2) call, there is no guarantee that the observed +data after a system crash will be either the old or the new data, but +in practice, the observed data after crash is often the old or new data +or a mix of both. + +When an overlayfs file is modified for the first time, copy up will +create a copy of the lower file and its parent directories in the upper +layer. Since the Linux filesystem API does not enforce any particular +ordering on storing changes without explicit fsync(2) calls, in case +of a system crash, the upper file could end up with no data at all +(i.e. zeros), which would be an unusual outcome. To avoid this +experience, overlayfs calls fsync(2) on the upper file before completing +data copy up with rename(2) or link(2) to make the copy up "atomic". + +By default, overlayfs does not explicitly call fsync(2) on copied up +directories or on metadata-only copy up, so it provides no guarantee to +persist the user's modification unless the user calls fsync(2). +The fsync during copy up only guarantees that if a copy up is observed +after a crash, the observed data is not zeroes or intermediate values +from the copy up staging area. + +On traditional local filesystems with a single journal (e.g. ext4, xfs), +fsync on a file also persists the parent directory changes, because they +are usually modified in the same transaction, so metadata durability during +data copy up effectively comes for free. Overlayfs further limits risk by +disallowing network filesystems as upper layer. + +Overlayfs can be tuned to prefer performance or durability when storing +to the underlying upper layer. This is controlled by the "fsync" mount +option, which supports these values: + +- "auto": (default) + Call fsync(2) on upper file before completion of data copy up. + No explicit fsync(2) on directory or metadata-only copy up. +- "strict": + Call fsync(2) on upper file and directories before completion of any + copy up. +- "volatile": [*] + Prefer performance over durability (see `Volatile mount`_) + +[*] The mount option "volatile" is an alias to "fsync=volatile". + + Volatile mount -------------- diff --git a/Documentation/hwmon/adm1177.rst b/Documentation/hwmon/adm1177.rst index 1c85a2af92bf7..375f6d6e03a7d 100644 --- a/Documentation/hwmon/adm1177.rst +++ b/Documentation/hwmon/adm1177.rst @@ -27,10 +27,10 @@ for details. Sysfs entries ------------- -The following attributes are supported. Current maxim attribute +The following attributes are supported. Current maximum attribute is read-write, all other attributes are read-only. -in0_input Measured voltage in microvolts. +in0_input Measured voltage in millivolts. -curr1_input Measured current in microamperes. -curr1_max_alarm Overcurrent alarm in microamperes. +curr1_input Measured current in milliamperes. +curr1_max Overcurrent shutdown threshold in milliamperes. diff --git a/Documentation/hwmon/aht10.rst b/Documentation/hwmon/aht10.rst index 213644b4ecba6..7903b6434326d 100644 --- a/Documentation/hwmon/aht10.rst +++ b/Documentation/hwmon/aht10.rst @@ -20,6 +20,14 @@ Supported chips: English: http://www.aosong.com/userfiles/files/media/Data%20Sheet%20AHT20.pdf + * Aosong DHT20 + + Prefix: 'dht20' + + Addresses scanned: None + + Datasheet: https://www.digikey.co.nz/en/htmldatasheets/production/9184855/0/0/1/101020932 + Author: Johannes Cornelis Draaijer @@ -33,7 +41,7 @@ The address of this i2c device may only be 0x38 Special Features ---------------- -AHT20 has additional CRC8 support which is sent as the last byte of the sensor +AHT20, DHT20 has additional CRC8 support which is sent as the last byte of the sensor values. Usage Notes diff --git a/Documentation/hwmon/g762.rst b/Documentation/hwmon/g762.rst index 0371b3365c48c..f224552a2d3cc 100644 --- a/Documentation/hwmon/g762.rst +++ b/Documentation/hwmon/g762.rst @@ -17,7 +17,7 @@ done via a userland daemon like fancontrol. Note that those entries do not provide ways to setup the specific hardware characteristics of the system (reference clock, pulses per fan revolution, ...); Those can be modified via devicetree bindings -documented in Documentation/devicetree/bindings/hwmon/g762.txt or +documented in Documentation/devicetree/bindings/hwmon/gmt,g762.yaml or using a specific platform_data structure in board initialization file (see include/linux/platform_data/g762.h). diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst index 3e549ba95a15a..45eec9dd349aa 100644 --- a/Documentation/hwmon/nct6683.rst +++ b/Documentation/hwmon/nct6683.rst @@ -65,6 +65,7 @@ AMD BC-250 NCT6686D EC firmware version 1.0 build 07/28/21 ASRock X570 NCT6683D EC firmware version 1.0 build 06/28/19 ASRock X670E NCT6686D EC firmware version 1.0 build 05/19/22 ASRock B650 Steel Legend WiFi NCT6686D EC firmware version 1.0 build 11/09/23 +ASRock Z590 Taichi NCT6686D EC firmware version 1.0 build 01/25/21 MSI B550 NCT6687D EC firmware version 1.0 build 05/07/20 MSI X670-P NCT6687D EC firmware version 0.0 build 09/27/22 MSI X870E NCT6687D EC firmware version 0.0 build 11/13/24 diff --git a/Documentation/hwmon/peci-cputemp.rst b/Documentation/hwmon/peci-cputemp.rst index fe0422248dc5e..266b62a46f49c 100644 --- a/Documentation/hwmon/peci-cputemp.rst +++ b/Documentation/hwmon/peci-cputemp.rst @@ -51,8 +51,9 @@ temp1_max Provides thermal control temperature of the CPU package temp1_crit Provides shutdown temperature of the CPU package which is also known as the maximum processor junction temperature, Tjmax or Tprochot. -temp1_crit_hyst Provides the hysteresis value from Tcontrol to Tjmax of - the CPU package. +temp1_crit_hyst Provides the hysteresis temperature of the CPU + package. Returns Tcontrol, the temperature at which + the critical condition clears. temp2_label "DTS" temp2_input Provides current temperature of the CPU package scaled @@ -62,8 +63,9 @@ temp2_max Provides thermal control temperature of the CPU package temp2_crit Provides shutdown temperature of the CPU package which is also known as the maximum processor junction temperature, Tjmax or Tprochot. -temp2_crit_hyst Provides the hysteresis value from Tcontrol to Tjmax of - the CPU package. +temp2_crit_hyst Provides the hysteresis temperature of the CPU + package. Returns Tcontrol, the temperature at which + the critical condition clears. temp3_label "Tcontrol" temp3_input Provides current Tcontrol temperature of the CPU diff --git a/Documentation/netlink/specs/dpll.yaml b/Documentation/netlink/specs/dpll.yaml index 80728f6f9bc87..78d0724d7e12c 100644 --- a/Documentation/netlink/specs/dpll.yaml +++ b/Documentation/netlink/specs/dpll.yaml @@ -440,6 +440,12 @@ attribute-sets: doc: | Capable pin provides list of pins that can be bound to create a reference-sync pin pair. + - + name: phase-adjust-gran + type: u32 + doc: | + Granularity of phase adjustment, in picoseconds. The value of + phase adjustment must be a multiple of this granularity. - name: pin-parent-device @@ -616,6 +622,7 @@ operations: - capabilities - parent-device - parent-pin + - phase-adjust-gran - phase-adjust-min - phase-adjust-max - phase-adjust diff --git a/Documentation/netlink/specs/fou.yaml b/Documentation/netlink/specs/fou.yaml index 8e7974ec453fc..331f1b342b3ad 100644 --- a/Documentation/netlink/specs/fou.yaml +++ b/Documentation/netlink/specs/fou.yaml @@ -39,6 +39,8 @@ attribute-sets: - name: ipproto type: u8 + checks: + min: 1 - name: type type: u8 diff --git a/Documentation/netlink/specs/net_shaper.yaml b/Documentation/netlink/specs/net_shaper.yaml index 0b1b54be48f92..3f2ad772b64b1 100644 --- a/Documentation/netlink/specs/net_shaper.yaml +++ b/Documentation/netlink/specs/net_shaper.yaml @@ -247,8 +247,8 @@ operations: flags: [admin-perm] do: - pre: net-shaper-nl-pre-doit - post: net-shaper-nl-post-doit + pre: net-shaper-nl-pre-doit-write + post: net-shaper-nl-post-doit-write request: attributes: - ifindex @@ -278,8 +278,8 @@ operations: flags: [admin-perm] do: - pre: net-shaper-nl-pre-doit - post: net-shaper-nl-post-doit + pre: net-shaper-nl-pre-doit-write + post: net-shaper-nl-post-doit-write request: attributes: *ns-binding @@ -309,8 +309,8 @@ operations: flags: [admin-perm] do: - pre: net-shaper-nl-pre-doit - post: net-shaper-nl-post-doit + pre: net-shaper-nl-pre-doit-write + post: net-shaper-nl-post-doit-write request: attributes: - ifindex diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index a06cb99d66dcd..7a637e87005fe 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -3195,12 +3195,13 @@ enhanced_dad - BOOLEAN =========== ratelimit - INTEGER - Limit the maximal rates for sending ICMPv6 messages. + Limit the maximal rates for sending ICMPv6 messages to a particular + peer. 0 to disable any limiting, - otherwise the minimal space between responses in milliseconds. + otherwise the space between responses in milliseconds. - Default: 1000 + Default: 100 ratemask - list of comma separated ranges For ICMPv6 message types matching the ranges in the ratemask, limit diff --git a/Documentation/networking/net_cachelines/inet_sock.rst b/Documentation/networking/net_cachelines/inet_sock.rst index b11bf48fa2b36..4c72a28a7012e 100644 --- a/Documentation/networking/net_cachelines/inet_sock.rst +++ b/Documentation/networking/net_cachelines/inet_sock.rst @@ -5,42 +5,43 @@ inet_sock struct fast path usage breakdown ========================================== -======================= ===================== =================== =================== ====================================================================================================== -Type Name fastpath_tx_access fastpath_rx_access comment -======================= ===================== =================== =================== ====================================================================================================== -struct sock sk read_mostly read_mostly tcp_init_buffer_space,tcp_init_transfer,tcp_finish_connect,tcp_connect,tcp_send_rcvq,tcp_send_syn_data -struct ipv6_pinfo* pinet6 -be16 inet_sport read_mostly __tcp_transmit_skb -be32 inet_daddr read_mostly ip_select_ident_segs -be32 inet_rcv_saddr -be16 inet_dport read_mostly __tcp_transmit_skb -u16 inet_num -be32 inet_saddr -s16 uc_ttl read_mostly __ip_queue_xmit/ip_select_ttl -u16 cmsg_flags -struct ip_options_rcu* inet_opt read_mostly __ip_queue_xmit -u16 inet_id read_mostly ip_select_ident_segs -u8 tos read_mostly ip_queue_xmit -u8 min_ttl -u8 mc_ttl -u8 pmtudisc -u8:1 recverr -u8:1 is_icsk -u8:1 freebind -u8:1 hdrincl -u8:1 mc_loop -u8:1 transparent -u8:1 mc_all -u8:1 nodefrag -u8:1 bind_address_no_port -u8:1 recverr_rfc4884 -u8:1 defer_connect read_mostly tcp_sendmsg_fastopen -u8 rcv_tos -u8 convert_csum -int uc_index -int mc_index -be32 mc_addr -struct ip_mc_socklist* mc_list -struct inet_cork_full cork read_mostly __tcp_transmit_skb -struct local_port_range -======================= ===================== =================== =================== ====================================================================================================== +======================== ===================== =================== =================== ====================================================================================================== +Type Name fastpath_tx_access fastpath_rx_access comment +======================== ===================== =================== =================== ====================================================================================================== +struct sock sk read_mostly read_mostly tcp_init_buffer_space,tcp_init_transfer,tcp_finish_connect,tcp_connect,tcp_send_rcvq,tcp_send_syn_data +struct ipv6_pinfo* pinet6 +struct ipv6_fl_socklist* ipv6_fl_list read_mostly tcp_v6_connect,__ip6_datagram_connect,udpv6_sendmsg,rawv6_sendmsg +be16 inet_sport read_mostly __tcp_transmit_skb +be32 inet_daddr read_mostly ip_select_ident_segs +be32 inet_rcv_saddr +be16 inet_dport read_mostly __tcp_transmit_skb +u16 inet_num +be32 inet_saddr +s16 uc_ttl read_mostly __ip_queue_xmit/ip_select_ttl +u16 cmsg_flags +struct ip_options_rcu* inet_opt read_mostly __ip_queue_xmit +u16 inet_id read_mostly ip_select_ident_segs +u8 tos read_mostly ip_queue_xmit +u8 min_ttl +u8 mc_ttl +u8 pmtudisc +u8:1 recverr +u8:1 is_icsk +u8:1 freebind +u8:1 hdrincl +u8:1 mc_loop +u8:1 transparent +u8:1 mc_all +u8:1 nodefrag +u8:1 bind_address_no_port +u8:1 recverr_rfc4884 +u8:1 defer_connect read_mostly tcp_sendmsg_fastopen +u8 rcv_tos +u8 convert_csum +int uc_index +int mc_index +be32 mc_addr +struct ip_mc_socklist* mc_list +struct inet_cork_full cork read_mostly __tcp_transmit_skb +struct local_port_range +======================== ===================== =================== =================== ====================================================================================================== diff --git a/Documentation/process/2.Process.rst b/Documentation/process/2.Process.rst index ef3b116492df0..f4fc0da8999d9 100644 --- a/Documentation/process/2.Process.rst +++ b/Documentation/process/2.Process.rst @@ -104,8 +104,10 @@ kernels go out with a handful of known regressions though, hopefully, none of them are serious. Once a stable release is made, its ongoing maintenance is passed off to the -"stable team," currently Greg Kroah-Hartman. The stable team will release -occasional updates to the stable release using the 5.x.y numbering scheme. +"stable team," currently consists of Greg Kroah-Hartman and Sasha Levin. The +stable team will release occasional updates to the stable release using the +5.x.y numbering scheme. + To be considered for an update release, a patch must (1) fix a significant bug, and (2) already be merged into the mainline for the next development kernel. Kernels will typically receive stable updates for a little more diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst index 0a4eaa7d66ddd..55b845d382368 100644 --- a/Documentation/sound/alsa-configuration.rst +++ b/Documentation/sound/alsa-configuration.rst @@ -2372,6 +2372,10 @@ quirk_flags audible volume * bit 25: ``mixer_capture_min_mute`` Similar to bit 24 but for capture streams + * bit 26: ``skip_iface_setup`` + Skip the probe-time interface setup (usb_set_interface, + init_pitch, init_sample_rate); redundant with + snd_usb_endpoint_prepare() at stream-open time This module supports multiple devices, autoprobe and hotplugging. diff --git a/Documentation/tools/rtla/common_appendix.rst b/Documentation/tools/rtla/common_appendix.txt similarity index 100% rename from Documentation/tools/rtla/common_appendix.rst rename to Documentation/tools/rtla/common_appendix.txt diff --git a/Documentation/tools/rtla/common_hist_options.rst b/Documentation/tools/rtla/common_hist_options.txt similarity index 100% rename from Documentation/tools/rtla/common_hist_options.rst rename to Documentation/tools/rtla/common_hist_options.txt diff --git a/Documentation/tools/rtla/common_options.rst b/Documentation/tools/rtla/common_options.txt similarity index 100% rename from Documentation/tools/rtla/common_options.rst rename to Documentation/tools/rtla/common_options.txt diff --git a/Documentation/tools/rtla/common_osnoise_description.rst b/Documentation/tools/rtla/common_osnoise_description.txt similarity index 100% rename from Documentation/tools/rtla/common_osnoise_description.rst rename to Documentation/tools/rtla/common_osnoise_description.txt diff --git a/Documentation/tools/rtla/common_osnoise_options.rst b/Documentation/tools/rtla/common_osnoise_options.txt similarity index 100% rename from Documentation/tools/rtla/common_osnoise_options.rst rename to Documentation/tools/rtla/common_osnoise_options.txt diff --git a/Documentation/tools/rtla/common_timerlat_aa.rst b/Documentation/tools/rtla/common_timerlat_aa.txt similarity index 100% rename from Documentation/tools/rtla/common_timerlat_aa.rst rename to Documentation/tools/rtla/common_timerlat_aa.txt diff --git a/Documentation/tools/rtla/common_timerlat_description.rst b/Documentation/tools/rtla/common_timerlat_description.txt similarity index 100% rename from Documentation/tools/rtla/common_timerlat_description.rst rename to Documentation/tools/rtla/common_timerlat_description.txt diff --git a/Documentation/tools/rtla/common_timerlat_options.rst b/Documentation/tools/rtla/common_timerlat_options.txt similarity index 100% rename from Documentation/tools/rtla/common_timerlat_options.rst rename to Documentation/tools/rtla/common_timerlat_options.txt diff --git a/Documentation/tools/rtla/common_top_options.rst b/Documentation/tools/rtla/common_top_options.txt similarity index 100% rename from Documentation/tools/rtla/common_top_options.rst rename to Documentation/tools/rtla/common_top_options.txt diff --git a/Documentation/tools/rtla/rtla-hwnoise.rst b/Documentation/tools/rtla/rtla-hwnoise.rst index 3a7163c02ac8e..26512b15fe7ba 100644 --- a/Documentation/tools/rtla/rtla-hwnoise.rst +++ b/Documentation/tools/rtla/rtla-hwnoise.rst @@ -29,11 +29,11 @@ collection of the tracer output. OPTIONS ======= -.. include:: common_osnoise_options.rst +.. include:: common_osnoise_options.txt -.. include:: common_top_options.rst +.. include:: common_top_options.txt -.. include:: common_options.rst +.. include:: common_options.txt EXAMPLE ======= @@ -106,4 +106,4 @@ AUTHOR ====== Written by Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/tools/rtla/rtla-osnoise-hist.rst b/Documentation/tools/rtla/rtla-osnoise-hist.rst index 1fc60ef261067..007521c865d97 100644 --- a/Documentation/tools/rtla/rtla-osnoise-hist.rst +++ b/Documentation/tools/rtla/rtla-osnoise-hist.rst @@ -15,7 +15,7 @@ SYNOPSIS DESCRIPTION =========== -.. include:: common_osnoise_description.rst +.. include:: common_osnoise_description.txt The **rtla osnoise hist** tool collects all **osnoise:sample_threshold** occurrence in a histogram, displaying the results in a user-friendly way. @@ -24,11 +24,11 @@ collection of the tracer output. OPTIONS ======= -.. include:: common_osnoise_options.rst +.. include:: common_osnoise_options.txt -.. include:: common_hist_options.rst +.. include:: common_hist_options.txt -.. include:: common_options.rst +.. include:: common_options.txt EXAMPLE ======= @@ -65,4 +65,4 @@ AUTHOR ====== Written by Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/tools/rtla/rtla-osnoise-top.rst b/Documentation/tools/rtla/rtla-osnoise-top.rst index b1cbd7bcd4aed..6ccadae389457 100644 --- a/Documentation/tools/rtla/rtla-osnoise-top.rst +++ b/Documentation/tools/rtla/rtla-osnoise-top.rst @@ -15,7 +15,7 @@ SYNOPSIS DESCRIPTION =========== -.. include:: common_osnoise_description.rst +.. include:: common_osnoise_description.txt **rtla osnoise top** collects the periodic summary from the *osnoise* tracer, including the counters of the occurrence of the interference source, @@ -26,11 +26,11 @@ collection of the tracer output. OPTIONS ======= -.. include:: common_osnoise_options.rst +.. include:: common_osnoise_options.txt -.. include:: common_top_options.rst +.. include:: common_top_options.txt -.. include:: common_options.rst +.. include:: common_options.txt EXAMPLE ======= @@ -60,4 +60,4 @@ AUTHOR ====== Written by Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/tools/rtla/rtla-osnoise.rst b/Documentation/tools/rtla/rtla-osnoise.rst index c129b206ce348..540d2bf6c1524 100644 --- a/Documentation/tools/rtla/rtla-osnoise.rst +++ b/Documentation/tools/rtla/rtla-osnoise.rst @@ -14,7 +14,7 @@ SYNOPSIS DESCRIPTION =========== -.. include:: common_osnoise_description.rst +.. include:: common_osnoise_description.txt The *osnoise* tracer outputs information in two ways. It periodically prints a summary of the noise of the operating system, including the counters of @@ -56,4 +56,4 @@ AUTHOR ====== Written by Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/tools/rtla/rtla-timerlat-hist.rst b/Documentation/tools/rtla/rtla-timerlat-hist.rst index 4923a362129bb..f56fe546411bd 100644 --- a/Documentation/tools/rtla/rtla-timerlat-hist.rst +++ b/Documentation/tools/rtla/rtla-timerlat-hist.rst @@ -16,7 +16,7 @@ SYNOPSIS DESCRIPTION =========== -.. include:: common_timerlat_description.rst +.. include:: common_timerlat_description.txt The **rtla timerlat hist** displays a histogram of each tracer event occurrence. This tool uses the periodic information, and the @@ -25,13 +25,13 @@ occurrence. This tool uses the periodic information, and the OPTIONS ======= -.. include:: common_timerlat_options.rst +.. include:: common_timerlat_options.txt -.. include:: common_hist_options.rst +.. include:: common_hist_options.txt -.. include:: common_options.rst +.. include:: common_options.txt -.. include:: common_timerlat_aa.rst +.. include:: common_timerlat_aa.txt EXAMPLE ======= @@ -110,4 +110,4 @@ AUTHOR ====== Written by Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/tools/rtla/rtla-timerlat-top.rst b/Documentation/tools/rtla/rtla-timerlat-top.rst index 50968cdd2095a..7dbe625d0c424 100644 --- a/Documentation/tools/rtla/rtla-timerlat-top.rst +++ b/Documentation/tools/rtla/rtla-timerlat-top.rst @@ -16,7 +16,7 @@ SYNOPSIS DESCRIPTION =========== -.. include:: common_timerlat_description.rst +.. include:: common_timerlat_description.txt The **rtla timerlat top** displays a summary of the periodic output from the *timerlat* tracer. It also provides information for each @@ -26,13 +26,13 @@ seem with the option **-T**. OPTIONS ======= -.. include:: common_timerlat_options.rst +.. include:: common_timerlat_options.txt -.. include:: common_top_options.rst +.. include:: common_top_options.txt -.. include:: common_options.rst +.. include:: common_options.txt -.. include:: common_timerlat_aa.rst +.. include:: common_timerlat_aa.txt **--aa-only** *us* @@ -133,4 +133,4 @@ AUTHOR ------ Written by Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/tools/rtla/rtla-timerlat.rst b/Documentation/tools/rtla/rtla-timerlat.rst index 20e2d259467fd..ce9f57e038c37 100644 --- a/Documentation/tools/rtla/rtla-timerlat.rst +++ b/Documentation/tools/rtla/rtla-timerlat.rst @@ -14,7 +14,7 @@ SYNOPSIS DESCRIPTION =========== -.. include:: common_timerlat_description.rst +.. include:: common_timerlat_description.txt The **rtla timerlat top** mode displays a summary of the periodic output from the *timerlat* tracer. The **rtla timerlat hist** mode displays @@ -51,4 +51,4 @@ AUTHOR ====== Written by Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/tools/rtla/rtla.rst b/Documentation/tools/rtla/rtla.rst index fc0d233efcd5d..2a5fb7004ad44 100644 --- a/Documentation/tools/rtla/rtla.rst +++ b/Documentation/tools/rtla/rtla.rst @@ -45,4 +45,4 @@ AUTHOR ====== Daniel Bristot de Oliveira -.. include:: common_appendix.rst +.. include:: common_appendix.txt diff --git a/Documentation/trace/events-pci.rst b/Documentation/trace/events-pci.rst new file mode 100644 index 0000000000000..03ff4ad30ddfa --- /dev/null +++ b/Documentation/trace/events-pci.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================== +Subsystem Trace Points: PCI +=========================== + +Overview +======== +The PCI tracing system provides tracepoints to monitor critical hardware events +that can impact system performance and reliability. These events normally show +up here: + + /sys/kernel/tracing/events/pci + +Cf. include/trace/events/pci.h for the events definitions. + +Available Tracepoints +===================== + +pci_hp_event +------------ + +Monitors PCI hotplug events including card insertion/removal and link +state changes. +:: + + pci_hp_event "%s slot:%s, event:%s\n" + +**Event Types**: + +* ``LINK_UP`` - PCIe link established +* ``LINK_DOWN`` - PCIe link lost +* ``CARD_PRESENT`` - Card detected in slot +* ``CARD_NOT_PRESENT`` - Card removed from slot + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pci_hp_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 1311.177459: pci_hp_event: 0000:00:02.0 slot:10, event:CARD_PRESENT + + irq/51-pciehp-88 [001] ..... 1311.177566: pci_hp_event: 0000:00:02.0 slot:10, event:LINK_UP + +pcie_link_event +--------------- + +Monitors PCIe link speed changes and provides detailed link status information. +:: + + pcie_link_event "%s type:%d, reason:%d, cur_bus_speed:%d, max_bus_speed:%d, width:%u, flit_mode:%u, status:%s\n" + +**Parameters**: + +* ``type`` - PCIe device type (4=Root Port, etc.) +* ``reason`` - Reason for link change: + + - ``0`` - Link retrain + - ``1`` - Bus enumeration + - ``2`` - Bandwidth notification enable + - ``3`` - Bandwidth notification IRQ + - ``4`` - Hotplug event + + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pcie_link_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 381.545386: pcie_link_event: 0000:00:02.0 type:4, reason:4, cur_bus_speed:20, max_bus_speed:23, width:1, flit_mode:0, status:DLLLA diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index b4a429dc4f7ad..0a40bfabcf19b 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -54,6 +54,7 @@ applications. events-power events-nmi events-msr + events-pci boottime-trace histogram histogram-design diff --git a/Documentation/trace/p_ost.rst b/Documentation/trace/p_ost.rst new file mode 100644 index 0000000000000..df93b889eb4ce --- /dev/null +++ b/Documentation/trace/p_ost.rst @@ -0,0 +1,36 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================== +MIPI OST over STP +=================== + +The OST(Open System Trace) driver is used with STM class devices to +generate standardized trace stream. Trace sources can be identified +by different entity ids. + +CONFIG_STM_PROTO_OST is for p_ost driver enablement. Once this config +is enabled, you can select the p_ost protocol by command below: + +# mkdir /sys/kernel/config/stp-policy/stm0:p_ost.policy + +The policy name format is extended like this: + :. + +With coresight-stm device, it will be look like "stm0:p_ost.policy". + +With MIPI OST protocol driver, the attributes for each protocol node is: +# mkdir /sys/kernel/config/stp-policy/stm0:p_ost.policy/default +# ls /sys/kernel/config/stp-policy/stm0:p_ost.policy/default +channels entity masters + +The entity here is the set the entity that p_ost supports. Currently +p_ost supports ftrace, console and diag entity. + +Set entity: +# echo 'ftrace' > /sys/kernel/config/stp-policy/stm0:p_ost.policy/default/entity + +Get available and currently selected (shown in square brackets) entity that p_ost supports: +# cat /sys/kernel/config/stp-policy/stm0:p_ost.policy/default/entity +[ftrace] console diag + +See Documentation/ABI/testing/configfs-stp-policy-p_ost for more details. diff --git a/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst b/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst index 806ed73ac474c..bcbd0382242fb 100644 --- a/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst +++ b/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst @@ -274,6 +274,14 @@ Compressed Formats of macroblocks to decode a full corresponding frame to the matching capture buffer. + * .. _V4L2-PIX-FMT-AV1: + + - ``V4L2_PIX_FMT_AV1`` + - 'AV01' + - AV1 compressed video frame. This format is adapted for implementing AV1 + pipeline. The decoder implements stateful video decoder and expects one + temporal unit per buffer in OBU stream format. + The encoder generates one Temporal Unit per buffer. .. raw:: latex \normalsize diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 57061fa29e6a0..9f835f68b4fb9 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -7800,8 +7800,10 @@ Will return -EBUSY if a VCPU has already been created. Valid feature flags in args[0] are:: - #define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) - #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) + #define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) + #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) + #define KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST (1ULL << 2) + #define KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST (1ULL << 3) Enabling KVM_X2APIC_API_USE_32BIT_IDS changes the behavior of KVM_SET_GSI_ROUTING, KVM_SIGNAL_MSI, KVM_SET_LAPIC, and KVM_GET_LAPIC, @@ -7814,6 +7816,28 @@ as a broadcast even in x2APIC mode in order to support physical x2APIC without interrupt remapping. This is undesirable in logical mode, where 0xff represents CPUs 0-7 in cluster 0. +Setting KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST instructs KVM to enable +Suppress EOI Broadcasts. KVM will advertise support for Suppress EOI +Broadcast to the guest and suppress LAPIC EOI broadcasts when the guest +sets the Suppress EOI Broadcast bit in the SPIV register. This flag is +supported only when using a split IRQCHIP. + +Setting KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST disables support for +Suppress EOI Broadcasts entirely, i.e. instructs KVM to NOT advertise +support to the guest. + +Modern VMMs should either enable KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST +or KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST. If not, legacy quirky +behavior will be used by KVM: in split IRQCHIP mode, KVM will advertise +support for Suppress EOI Broadcasts but not actually suppress EOI +broadcasts; for in-kernel IRQCHIP mode, KVM will not advertise support for +Suppress EOI Broadcasts. + +Setting both KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST and +KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST will fail with an EINVAL error, +as will setting KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST without a split +IRCHIP. + 7.8 KVM_CAP_S390_USER_INSTR0 ---------------------------- @@ -8379,6 +8403,14 @@ KVM_X86_QUIRK_IGNORE_GUEST_PAT By default, on Intel platforms, KVM ignores guest software, for example if it does not expose a bochs graphics device (which is known to have had a buggy driver). + +KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM By default, KVM relaxes the consistency + check for GUEST_IA32_DEBUGCTL in vmcs12 + to allow FREEZE_IN_SMM to be set. When + this quirk is disabled, KVM requires this + bit to be cleared. Note that the vmcs02 + bit is still completely controlled by the + host, regardless of the quirk setting. =================================== ============================================ 7.32 KVM_CAP_MAX_VCPU_ID diff --git a/MAINTAINERS b/MAINTAINERS index e8f06145fb54c..9e4444ba3ea36 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3216,6 +3216,7 @@ F: arch/arm64/boot/dts/qcom/ F: drivers/bus/qcom* F: drivers/firmware/qcom/ F: drivers/soc/qcom/ +F: drivers/watchdog/gunyah_wdt.c F: include/dt-bindings/arm/qcom,ids.h F: include/dt-bindings/firmware/qcom,scm.h F: include/dt-bindings/soc/qcom* @@ -7883,6 +7884,7 @@ S: Maintained B: https://gitlab.freedesktop.org/drm/msm/-/issues T: git https://gitlab.freedesktop.org/drm/msm.git F: Documentation/devicetree/bindings/display/msm/gpu.yaml +F: Documentation/devicetree/bindings/display/msm/qcom,adreno-rgmu.yaml F: Documentation/devicetree/bindings/opp/opp-v2-qcom-adreno.yaml F: drivers/gpu/drm/msm/adreno/ F: drivers/gpu/drm/msm/msm_gpu.* @@ -21054,6 +21056,15 @@ S: Maintained F: Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml F: drivers/net/wwan/qcom_bam_dmux.c +QUALCOMM BCL HARDWARE MONITOR DRIVER +M: Manaf Meethalavalappu Pallikunhi +L: linux-hwmon@vger.kernel.org +L: linux-arm-msm@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/hwmon/qcom,bcl-hwmon.yaml +F: drivers/hwmon/qcom-bcl-hwmon.c +F: drivers/hwmon/qcom-bcl-hwmon.h + QUALCOMM BLUETOOTH DRIVER L: linux-arm-msm@vger.kernel.org S: Maintained @@ -21149,6 +21160,14 @@ F: Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml F: drivers/misc/fastrpc.c F: include/uapi/misc/fastrpc.h +QUALCOMM HAMOA EMBEDDED CONTROLLER DRIVER +M: Anvesh Jain P +M: Sibi Sankar +L: linux-arm-msm@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml +F: drivers/platform/arm64/qcom-hamoa-ec.c + QUALCOMM HEXAGON ARCHITECTURE M: Brian Cain L: linux-hexagon@vger.kernel.org @@ -21241,6 +21260,17 @@ S: Maintained F: Documentation/devicetree/bindings/media/qcom,*-iris.yaml F: drivers/media/platform/qcom/iris/ +QUALCOMM MIPI CSI2 PHY DRIVER +M: Bryan O'Donoghue +L: linux-phy@lists.infradead.org +L: linux-media@vger.kernel.org +L: linux-arm-msm@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/phy/qcom,*-csi2-phy.yaml +F: drivers/phy/qualcomm/phy-qcom-mipi-csi2*.c +F: drivers/phy/qualcomm/phy-qcom-mipi-csi2*.h +F: include/dt-bindings/phy/phy-qcom-mipi-csi2* + QUALCOMM NAND CONTROLLER DRIVER M: Manivannan Sadhasivam L: linux-mtd@lists.infradead.org @@ -25538,6 +25568,14 @@ F: drivers/thermal/cpufreq_cooling.c F: drivers/thermal/cpuidle_cooling.c F: include/linux/cpu_cooling.h +THERMAL/REMOTEPROC_COOLING +M: Gaurav Kohli +L: linux-pm@vger.kernel.org +S: Supported +F: drivers/thermal/remoteproc_cooling.c +F: include/linux/remoteproc_cooling.h + + THERMAL/POWER_ALLOCATOR M: Lukasz Luba L: linux-pm@vger.kernel.org diff --git a/Makefile b/Makefile index a082a1d7c7d9b..c8343ec96a09a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 PATCHLEVEL = 18 -SUBLEVEL = 0 +SUBLEVEL = 25 EXTRAVERSION = NAME = Baby Opossum Posse @@ -473,6 +473,7 @@ KBUILD_USERLDFLAGS := $(USERLDFLAGS) export rust_common_flags := --edition=2021 \ -Zbinary_dep_depinfo=y \ -Astable_features \ + -Aunused_features \ -Dnon_ascii_idents \ -Dunsafe_op_in_unsafe_fn \ -Wmissing_docs \ @@ -1440,6 +1441,15 @@ ifneq ($(wildcard $(resolve_btfids_O)),) $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean endif +PHONY += objtool_clean objtool_mrproper + +objtool_O = $(abspath $(objtree))/tools/objtool + +objtool_clean objtool_mrproper: +ifneq ($(wildcard $(objtool_O)),) + $(Q)$(MAKE) -sC $(abs_srctree)/tools/objtool O=$(objtool_O) srctree=$(abs_srctree) $(patsubst objtool_%,%,$@) +endif + tools/: FORCE $(Q)mkdir -p $(objtree)/tools $(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ @@ -1578,7 +1588,7 @@ CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c \ - .builtin-dtbs-list .builtin-dtb.S + .builtin-dtbs-list .builtin-dtbs.S # Directories & files removed with 'make mrproper' MRPROPER_FILES += include/config include/generated \ @@ -1590,7 +1600,8 @@ MRPROPER_FILES += include/config include/generated \ certs/x509.genkey \ vmlinux-gdb.py \ rpmbuild \ - rust/libmacros.so rust/libmacros.dylib + rust/libmacros.so rust/libmacros.dylib \ + rust/libpin_init_internal.so rust/libpin_init_internal.dylib # clean - Delete most, but leave enough to build external modules # @@ -1602,7 +1613,7 @@ vmlinuxclean: $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean) -clean: archclean vmlinuxclean resolve_btfids_clean +clean: archclean vmlinuxclean resolve_btfids_clean objtool_clean # mrproper - Delete all generated files, including .config # @@ -1613,7 +1624,7 @@ PHONY += $(mrproper-dirs) mrproper $(mrproper-dirs): $(Q)$(MAKE) $(clean)=$(patsubst _mrproper_%,%,$@) -mrproper: clean $(mrproper-dirs) +mrproper: clean objtool_mrproper $(mrproper-dirs) $(call cmd,rmfiles) @find . $(RCS_FIND_IGNORE) \ \( -name '*.rmeta' \) \ diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 90e7a95391022..c9508ec37efc4 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -17,6 +17,7 @@ #include /* For TASK_SIZE */ #include #include +#include struct mm_struct; struct vm_area_struct; @@ -183,6 +184,9 @@ extern inline void pud_set(pud_t * pudp, pmd_t * pmdp) { pud_val(*pudp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } +extern void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr); + extern inline unsigned long pmd_page_vaddr(pmd_t pmd) { @@ -202,7 +206,7 @@ extern inline int pte_none(pte_t pte) { return !pte_val(pte); } extern inline int pte_present(pte_t pte) { return pte_val(pte) & _PAGE_VALID; } extern inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_val(*ptep) = 0; + WRITE_ONCE(pte_val(*ptep), 0); } extern inline int pmd_none(pmd_t pmd) { return !pmd_val(pmd); } @@ -264,6 +268,33 @@ extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address) extern pgd_t swapper_pg_dir[1024]; +#ifdef CONFIG_COMPACTION +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR + +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, + pte_t *ptep) +{ + pte_t pte = READ_ONCE(*ptep); + + pte_clear(mm, address, ptep); + return pte; +} + +#define __HAVE_ARCH_PTEP_CLEAR_FLUSH + +static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = ptep_get_and_clear(mm, addr, ptep); + + page_table_check_pte_clear(mm, pte); + migrate_flush_tlb_page(vma, addr); + return pte; +} + +#endif /* * The Alpha doesn't have any external MMU info: the kernel page * tables contain all the necessary information. diff --git a/arch/alpha/include/asm/tlbflush.h b/arch/alpha/include/asm/tlbflush.h index ba4b359d6c395..0c8529997f54e 100644 --- a/arch/alpha/include/asm/tlbflush.h +++ b/arch/alpha/include/asm/tlbflush.h @@ -58,7 +58,9 @@ flush_tlb_other(struct mm_struct *mm) unsigned long *mmc = &mm->context[smp_processor_id()]; /* Check it's not zero first to avoid cacheline ping pong when possible. */ - if (*mmc) *mmc = 0; + + if (READ_ONCE(*mmc)) + WRITE_ONCE(*mmc, 0); } #ifndef CONFIG_SMP diff --git a/arch/alpha/include/uapi/asm/ioctls.h b/arch/alpha/include/uapi/asm/ioctls.h index 971311605288f..a09d04b49cc65 100644 --- a/arch/alpha/include/uapi/asm/ioctls.h +++ b/arch/alpha/include/uapi/asm/ioctls.h @@ -23,10 +23,10 @@ #define TCSETSW _IOW('t', 21, struct termios) #define TCSETSF _IOW('t', 22, struct termios) -#define TCGETA _IOR('t', 23, struct termio) -#define TCSETA _IOW('t', 24, struct termio) -#define TCSETAW _IOW('t', 25, struct termio) -#define TCSETAF _IOW('t', 28, struct termio) +#define TCGETA 0x40127417 +#define TCSETA 0x80127418 +#define TCSETAW 0x80127419 +#define TCSETAF 0x8012741c #define TCSBRK _IO('t', 29) #define TCXONC _IO('t', 30) diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S index 2efa7dfc798a9..2d136c63db161 100644 --- a/arch/alpha/kernel/vmlinux.lds.S +++ b/arch/alpha/kernel/vmlinux.lds.S @@ -71,6 +71,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/alpha/mm/Makefile b/arch/alpha/mm/Makefile index 101dbd06b4ceb..2d05664058f64 100644 --- a/arch/alpha/mm/Makefile +++ b/arch/alpha/mm/Makefile @@ -3,4 +3,4 @@ # Makefile for the linux alpha-specific parts of the memory manager. # -obj-y := init.o fault.o +obj-y := init.o fault.o tlbflush.o diff --git a/arch/alpha/mm/tlbflush.c b/arch/alpha/mm/tlbflush.c new file mode 100644 index 0000000000000..ccbc317b9a348 --- /dev/null +++ b/arch/alpha/mm/tlbflush.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Alpha TLB shootdown helpers + * + * Copyright (C) 2025 Magnus Lindholm + * + * Alpha-specific TLB flush helpers that cannot be expressed purely + * as inline functions. + * + * These helpers provide combined MM context handling (ASN rollover) + * and immediate TLB invalidation for page migration and memory + * compaction paths, where lazy shootdowns are insufficient. + */ + +#include +#include +#include +#include +#include +#include + +#define asn_locked() (cpu_data[smp_processor_id()].asn_lock) + +/* + * Migration/compaction helper: combine mm context (ASN) handling with an + * immediate per-page TLB invalidate and (for exec) an instruction barrier. + * + * This mirrors the SMP combined IPI handler semantics, but runs locally on UP. + */ +#ifndef CONFIG_SMP +void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + int tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2; + + /* + * First do the mm-context side: + * If we're currently running this mm, reload a fresh context ASN. + * Otherwise, mark context invalid. + * + * On UP, this is mostly about matching the SMP semantics and ensuring + * exec/i-cache tagging assumptions hold when compaction migrates pages. + */ + if (mm == current->active_mm) + flush_tlb_current(mm); + else + flush_tlb_other(mm); + + /* + * Then do the immediate translation kill for this VA. + * For exec mappings, order instruction fetch after invalidation. + */ + tbi(tbi_type, addr); +} + +#else +struct tlb_mm_and_addr { + struct mm_struct *mm; + unsigned long addr; + int tbi_type; /* 2 = DTB, 3 = ITB+DTB */ +}; + +static void ipi_flush_mm_and_page(void *x) +{ + struct tlb_mm_and_addr *d = x; + + /* Part 1: mm context side (Alpha uses ASN/context as a key mechanism). */ + if (d->mm == current->active_mm && !asn_locked()) + __load_new_mm_context(d->mm); + else + flush_tlb_other(d->mm); + + /* Part 2: immediate per-VA invalidation on this CPU. */ + tbi(d->tbi_type, d->addr); +} + +void migrate_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + struct tlb_mm_and_addr d = { + .mm = mm, + .addr = addr, + .tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2, + }; + + /* + * One synchronous rendezvous: every CPU runs ipi_flush_mm_and_page(). + * This is the "combined" version of flush_tlb_mm + per-page invalidate. + */ + preempt_disable(); + on_each_cpu(ipi_flush_mm_and_page, &d, 1); + + /* + * mimic flush_tlb_mm()'s mm_users<=1 optimization. + */ + if (atomic_read(&mm->mm_users) <= 1) { + + int cpu, this_cpu; + this_cpu = smp_processor_id(); + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + if (!cpu_online(cpu) || cpu == this_cpu) + continue; + if (READ_ONCE(mm->context[cpu])) + WRITE_ONCE(mm->context[cpu], 0); + } + } + preempt_enable(); +} + +#endif diff --git a/arch/arc/kernel/vmlinux.lds.S b/arch/arc/kernel/vmlinux.lds.S index 61a1b2b96e1d8..6af63084ff285 100644 --- a/arch/arc/kernel/vmlinux.lds.S +++ b/arch/arc/kernel/vmlinux.lds.S @@ -123,6 +123,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 4fb985b76e97f..70cd3b5b5a059 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1215,7 +1215,7 @@ config HIGHMEM config HIGHPTE bool "Allocate 2nd-level pagetables from highmem" if EXPERT - depends on HIGHMEM + depends on HIGHMEM && !PREEMPT_RT default y help The VM uses one page of physical memory for each page table. diff --git a/arch/arm/boot/compressed/vmlinux.lds.S b/arch/arm/boot/compressed/vmlinux.lds.S index d411abd4310ea..2d916647df03c 100644 --- a/arch/arm/boot/compressed/vmlinux.lds.S +++ b/arch/arm/boot/compressed/vmlinux.lds.S @@ -21,6 +21,7 @@ SECTIONS COMMON_DISCARDS *(.ARM.exidx*) *(.ARM.extab*) + *(.modinfo) *(.note.*) *(.rel.*) *(.printk_index) diff --git a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts index be486d28d04fa..428cab5a0e906 100644 --- a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts +++ b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts @@ -102,6 +102,7 @@ /* The P66 uses a different EINT then the reference design */ interrupts = <6 9 IRQ_TYPE_EDGE_FALLING>; /* EINT9 (PG9) */ /* The icn8318 binding expects wake-gpios instead of power-gpios */ + /delete-property/ power-gpios; wake-gpios = <&pio 1 3 GPIO_ACTIVE_HIGH>; /* PB3 */ touchscreen-size-x = <800>; touchscreen-size-y = <480>; diff --git a/arch/arm/boot/dts/microchip/sam9x7.dtsi b/arch/arm/boot/dts/microchip/sam9x7.dtsi index 46dacbbd201dd..d242d7a934d0f 100644 --- a/arch/arm/boot/dts/microchip/sam9x7.dtsi +++ b/arch/arm/boot/dts/microchip/sam9x7.dtsi @@ -1226,7 +1226,7 @@ interrupt-controller; #gpio-cells = <2>; gpio-controller; - #gpio-lines = <26>; + #gpio-lines = <27>; clocks = <&pmc PMC_TYPE_PERIPHERAL 3>; }; diff --git a/arch/arm/boot/dts/microchip/sama5d2.dtsi b/arch/arm/boot/dts/microchip/sama5d2.dtsi index 17430d7f20555..fde890f18d201 100644 --- a/arch/arm/boot/dts/microchip/sama5d2.dtsi +++ b/arch/arm/boot/dts/microchip/sama5d2.dtsi @@ -571,7 +571,7 @@ AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(12))>; dma-names = "tx", "rx"; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; @@ -642,7 +642,7 @@ AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(14))>; dma-names = "tx", "rx"; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; @@ -854,7 +854,7 @@ AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(16))>; dma-names = "tx", "rx"; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; @@ -925,7 +925,7 @@ AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(18))>; dma-names = "tx", "rx"; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; @@ -997,7 +997,7 @@ AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(20))>; dma-names = "tx", "rx"; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/microchip/sama7d65.dtsi b/arch/arm/boot/dts/microchip/sama7d65.dtsi index e53e2dd6d530c..868045c650a73 100644 --- a/arch/arm/boot/dts/microchip/sama7d65.dtsi +++ b/arch/arm/boot/dts/microchip/sama7d65.dtsi @@ -527,7 +527,7 @@ interrupts = ; clocks = <&pmc PMC_TYPE_PERIPHERAL 37>; #address-cells = <1>; - #size-cells = <1>; + #size-cells = <0>; dmas = <&dma0 AT91_XDMAC_DT_PERID(12)>, <&dma0 AT91_XDMAC_DT_PERID(11)>; dma-names = "tx", "rx"; @@ -557,7 +557,7 @@ dma-names = "tx", "rx"; atmel,use-dma-rx; atmel,use-dma-tx; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; atmel,usart-mode = ; status = "disabled"; }; @@ -618,7 +618,7 @@ clocks = <&pmc PMC_TYPE_PERIPHERAL 40>; clock-names = "usart"; atmel,usart-mode = ; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; }; @@ -643,7 +643,7 @@ dma-names = "tx", "rx"; atmel,use-dma-rx; atmel,use-dma-tx; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; atmel,usart-mode = ; status = "disabled"; }; @@ -676,7 +676,7 @@ flx9: flexcom@e2820000 { compatible = "microchip,sama7d65-flexcom", "atmel,sama5d2-flexcom"; reg = <0xe2820000 0x200>; - ranges = <0x0 0xe281c000 0x800>; + ranges = <0x0 0xe2820000 0x800>; clocks = <&pmc PMC_TYPE_PERIPHERAL 43>; #address-cells = <1>; #size-cells = <1>; diff --git a/arch/arm/boot/dts/microchip/sama7g5.dtsi b/arch/arm/boot/dts/microchip/sama7g5.dtsi index 381cbcfcb34a1..03ef3d9aaeec6 100644 --- a/arch/arm/boot/dts/microchip/sama7g5.dtsi +++ b/arch/arm/boot/dts/microchip/sama7g5.dtsi @@ -824,7 +824,7 @@ dma-names = "tx", "rx"; atmel,use-dma-rx; atmel,use-dma-tx; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; }; @@ -850,7 +850,7 @@ dma-names = "tx", "rx"; atmel,use-dma-rx; atmel,use-dma-tx; - atmel,fifo-size = <16>; + atmel,fifo-size = <32>; status = "disabled"; }; }; diff --git a/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts b/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts index acc44010d5106..3ad9db4b14425 100644 --- a/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts +++ b/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts @@ -1,47 +1,10 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT) /* * USB armory MkI device tree file * https://inversepath.com/usbarmory * * Copyright (C) 2015, Inverse Path * Andrej Rosano - * - * This file is dual-licensed: you can use it either under the terms - * of the GPL or the X11 license, at your option. Note that this dual - * licensing only applies to this file, and not this project as a - * whole. - * - * a) This file is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or (at your option) any later version. - * - * This file is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Or, alternatively, - * - * b) Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ /dts-v1/; diff --git a/arch/arm/boot/dts/nxp/imx/imx6q-ba16.dtsi b/arch/arm/boot/dts/nxp/imx/imx6q-ba16.dtsi index 53013b12c2ecb..02d66523668d2 100644 --- a/arch/arm/boot/dts/nxp/imx/imx6q-ba16.dtsi +++ b/arch/arm/boot/dts/nxp/imx/imx6q-ba16.dtsi @@ -337,7 +337,7 @@ pinctrl-0 = <&pinctrl_rtc>; reg = <0x32>; interrupt-parent = <&gpio4>; - interrupts = <10 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <10 IRQ_TYPE_LEVEL_LOW>; }; }; diff --git a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi index 2236901a00313..8e9ed93da129e 100644 --- a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi +++ b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi @@ -302,6 +302,7 @@ mpwm: pwm@400e8000 { compatible = "nxp,lpc3220-motor-pwm"; reg = <0x400e8000 0x78>; + clocks = <&clk LPC32XX_CLK_MCPWM>; #pwm-cells = <3>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/renesas/r8a7793-gose.dts b/arch/arm/boot/dts/renesas/r8a7793-gose.dts index 45b267ec26794..5c6928c941aca 100644 --- a/arch/arm/boot/dts/renesas/r8a7793-gose.dts +++ b/arch/arm/boot/dts/renesas/r8a7793-gose.dts @@ -373,7 +373,6 @@ port@3 { reg = <3>; adv7180_out: endpoint { - bus-width = <8>; remote-endpoint = <&vin1ep>; }; }; diff --git a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts index 3258b2e274346..4a72aa7663f25 100644 --- a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts +++ b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts @@ -308,8 +308,6 @@ &switch { status = "okay"; - #address-cells = <1>; - #size-cells = <0>; pinctrl-names = "default"; pinctrl-0 = <&pins_eth3>, <&pins_eth4>, <&pins_mdio1>; diff --git a/arch/arm/boot/dts/samsung/exynos4210-i9100.dts b/arch/arm/boot/dts/samsung/exynos4210-i9100.dts index df229fb8a16be..8a635bee59fa9 100644 --- a/arch/arm/boot/dts/samsung/exynos4210-i9100.dts +++ b/arch/arm/boot/dts/samsung/exynos4210-i9100.dts @@ -853,6 +853,7 @@ #size-cells = <0>; non-removable; + cap-power-off-card; bus-width = <4>; mmc-pwrseq = <&wlan_pwrseq>; vmmc-supply = <&vtf_reg>; diff --git a/arch/arm/boot/dts/samsung/exynos4210-trats.dts b/arch/arm/boot/dts/samsung/exynos4210-trats.dts index 95e0e01b6ff6b..6bd902cb8f4ad 100644 --- a/arch/arm/boot/dts/samsung/exynos4210-trats.dts +++ b/arch/arm/boot/dts/samsung/exynos4210-trats.dts @@ -518,6 +518,7 @@ #size-cells = <0>; non-removable; + cap-power-off-card; bus-width = <4>; mmc-pwrseq = <&wlan_pwrseq>; vmmc-supply = <&tflash_reg>; diff --git a/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts b/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts index bdc30f8cf748f..91490693432b6 100644 --- a/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts +++ b/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts @@ -610,6 +610,7 @@ #size-cells = <0>; non-removable; + cap-power-off-card; bus-width = <4>; mmc-pwrseq = <&wlan_pwrseq>; vmmc-supply = <&ldo5_reg>; diff --git a/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi b/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi index 05ddddb565ee3..48245b1665a69 100644 --- a/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi +++ b/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi @@ -1440,6 +1440,7 @@ #address-cells = <1>; #size-cells = <0>; non-removable; + cap-power-off-card; bus-width = <4>; mmc-pwrseq = <&wlan_pwrseq>; diff --git a/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp15-som.dtsi b/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp15-som.dtsi index bf0c32027baf7..370b2afbf15bf 100644 --- a/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp15-som.dtsi +++ b/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp15-som.dtsi @@ -185,13 +185,13 @@ interrupt-parent = <&gpioi>; vio-supply = <&v3v3>; vcc-supply = <&v3v3>; + st,sample-time = <4>; + st,mod-12b = <1>; + st,ref-sel = <0>; + st,adc-freq = <1>; touchscreen { compatible = "st,stmpe-ts"; - st,sample-time = <4>; - st,mod-12b = <1>; - st,ref-sel = <0>; - st,adc-freq = <1>; st,ave-ctrl = <1>; st,touch-det-delay = <2>; st,settling = <2>; diff --git a/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi b/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi index ad1e60a9b6fde..b75dabfa56ae7 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi +++ b/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi @@ -16,7 +16,7 @@ }; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; leds { diff --git a/arch/arm/boot/dts/ti/omap/am335x-boneblue.dts b/arch/arm/boot/dts/ti/omap/am335x-boneblue.dts index f579df4c2c540..d430f0bef1653 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-boneblue.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-boneblue.dts @@ -13,7 +13,7 @@ compatible = "ti,am335x-bone-blue", "ti,am33xx"; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; leds { diff --git a/arch/arm/boot/dts/ti/omap/am335x-chiliboard.dts b/arch/arm/boot/dts/ti/omap/am335x-chiliboard.dts index 648e97fe1dfd5..ae5bc58984972 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-chiliboard.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-chiliboard.dts @@ -12,7 +12,7 @@ "ti,am33xx"; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; leds { diff --git a/arch/arm/boot/dts/ti/omap/am335x-evm.dts b/arch/arm/boot/dts/ti/omap/am335x-evm.dts index 20222f82f21bf..856fa1191ed24 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-evm.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-evm.dts @@ -23,7 +23,7 @@ }; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; vbat: fixedregulator0 { diff --git a/arch/arm/boot/dts/ti/omap/am335x-evmsk.dts b/arch/arm/boot/dts/ti/omap/am335x-evmsk.dts index eba888dcd60e7..d8baccdf8bc46 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-evmsk.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-evmsk.dts @@ -30,7 +30,7 @@ }; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; vbat: fixedregulator0 { diff --git a/arch/arm/boot/dts/ti/omap/am335x-guardian.dts b/arch/arm/boot/dts/ti/omap/am335x-guardian.dts index 4b070e634b281..6ce3a2d029eed 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-guardian.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-guardian.dts @@ -14,7 +14,7 @@ compatible = "bosch,am335x-guardian", "ti,am33xx"; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; tick-timer = &timer2; }; diff --git a/arch/arm/boot/dts/ti/omap/am335x-icev2.dts b/arch/arm/boot/dts/ti/omap/am335x-icev2.dts index 6f0f4fba043b9..ba488bba6925d 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-icev2.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-icev2.dts @@ -22,7 +22,7 @@ }; chosen { - stdout-path = &uart3; + stdout-path = "serial3:115200n8"; }; vbat: fixedregulator0 { diff --git a/arch/arm/boot/dts/ti/omap/am335x-myirtech-myd.dts b/arch/arm/boot/dts/ti/omap/am335x-myirtech-myd.dts index 06a352f98b220..476a6bdaf43f3 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-myirtech-myd.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-myirtech-myd.dts @@ -15,7 +15,7 @@ compatible = "myir,myd-am335x", "myir,myc-am335x", "ti,am33xx"; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; clk12m: clk12m { diff --git a/arch/arm/boot/dts/ti/omap/am335x-netcom-plus-2xx.dts b/arch/arm/boot/dts/ti/omap/am335x-netcom-plus-2xx.dts index f66d57bb685ee..f0519ab301416 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-netcom-plus-2xx.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-netcom-plus-2xx.dts @@ -222,10 +222,10 @@ "ModeA1", "ModeA2", "ModeA3", - "NC", - "NC", - "NC", - "NC", + "ModeB0", + "ModeB1", + "ModeB2", + "ModeB3", "NC", "NC", "NC", diff --git a/arch/arm/boot/dts/ti/omap/am335x-osd3358-sm-red.dts b/arch/arm/boot/dts/ti/omap/am335x-osd3358-sm-red.dts index d28d397288476..23caaaabf3513 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-osd3358-sm-red.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-osd3358-sm-red.dts @@ -147,7 +147,7 @@ }; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; leds { diff --git a/arch/arm/boot/dts/ti/omap/am335x-pdu001.dts b/arch/arm/boot/dts/ti/omap/am335x-pdu001.dts index c9ccb9de21ad7..9f611debc2090 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-pdu001.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-pdu001.dts @@ -21,7 +21,7 @@ compatible = "ti,am33xx"; chosen { - stdout-path = &uart3; + stdout-path = "serial3:115200n8"; }; cpus { diff --git a/arch/arm/boot/dts/ti/omap/am335x-pocketbeagle.dts b/arch/arm/boot/dts/ti/omap/am335x-pocketbeagle.dts index 78ce860e59b3d..24d9f90fad01f 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-pocketbeagle.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-pocketbeagle.dts @@ -15,7 +15,7 @@ compatible = "ti,am335x-pocketbeagle", "ti,am335x-bone", "ti,am33xx"; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; leds { diff --git a/arch/arm/boot/dts/ti/omap/am335x-sl50.dts b/arch/arm/boot/dts/ti/omap/am335x-sl50.dts index f3524e5ee43e2..1dc4e344efd63 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-sl50.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-sl50.dts @@ -25,7 +25,7 @@ }; chosen { - stdout-path = &uart0; + stdout-path = "serial0:115200n8"; }; leds { diff --git a/arch/arm/boot/dts/ti/omap/omap3-beagle-xm.dts b/arch/arm/boot/dts/ti/omap/omap3-beagle-xm.dts index 08ee0f8ea68fd..71b39a923d37c 100644 --- a/arch/arm/boot/dts/ti/omap/omap3-beagle-xm.dts +++ b/arch/arm/boot/dts/ti/omap/omap3-beagle-xm.dts @@ -291,7 +291,7 @@ }; twl_power: power { - compatible = "ti,twl4030-power-beagleboard-xm", "ti,twl4030-power-idle-osc-off"; + compatible = "ti,twl4030-power-idle-osc-off"; ti,use_poweroff; }; }; diff --git a/arch/arm/boot/dts/ti/omap/omap3-n900.dts b/arch/arm/boot/dts/ti/omap/omap3-n900.dts index c50ca572d1b9b..7db73d9bed9e4 100644 --- a/arch/arm/boot/dts/ti/omap/omap3-n900.dts +++ b/arch/arm/boot/dts/ti/omap/omap3-n900.dts @@ -508,7 +508,7 @@ }; twl_power: power { - compatible = "ti,twl4030-power-n900", "ti,twl4030-power-idle-osc-off"; + compatible = "ti,twl4030-power-idle-osc-off"; ti,use_poweroff; }; }; diff --git a/arch/arm/include/asm/string.h b/arch/arm/include/asm/string.h index 6c607c68f3ad7..96fc6cf460ecb 100644 --- a/arch/arm/include/asm/string.h +++ b/arch/arm/include/asm/string.h @@ -39,10 +39,17 @@ static inline void *memset32(uint32_t *p, uint32_t v, __kernel_size_t n) } #define __HAVE_ARCH_MEMSET64 -extern void *__memset64(uint64_t *, uint32_t low, __kernel_size_t, uint32_t hi); +extern void *__memset64(uint64_t *, uint32_t first, __kernel_size_t, uint32_t second); static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n) { - return __memset64(p, v, n * 8, v >> 32); + union { + uint64_t val; + struct { + uint32_t first, second; + }; + } word = { .val = v }; + + return __memset64(p, word.first, n * 8, word.second); } /* diff --git a/arch/arm/include/asm/word-at-a-time.h b/arch/arm/include/asm/word-at-a-time.h index f9a3897b06e7f..5023f98d8293d 100644 --- a/arch/arm/include/asm/word-at-a-time.h +++ b/arch/arm/include/asm/word-at-a-time.h @@ -67,7 +67,7 @@ static inline unsigned long find_zero(unsigned long mask) */ static inline unsigned long load_unaligned_zeropad(const void *addr) { - unsigned long ret, offset; + unsigned long ret, tmp; /* Load word from unaligned pointer addr */ asm( @@ -75,9 +75,9 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) "2:\n" " .pushsection .text.fixup,\"ax\"\n" " .align 2\n" - "3: and %1, %2, #0x3\n" - " bic %2, %2, #0x3\n" - " ldr %0, [%2]\n" + "3: bic %1, %2, #0x3\n" + " ldr %0, [%1]\n" + " and %1, %2, #0x3\n" " lsl %1, %1, #0x3\n" #ifndef __ARMEB__ " lsr %0, %0, %1\n" @@ -90,7 +90,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) " .align 3\n" " .long 1b, 3b\n" " .popsection" - : "=&r" (ret), "=&r" (offset) + : "=&r" (ret), "=&r" (tmp) : "r" (addr), "Qo" (*(unsigned long *)addr)); return ret; diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index e38a30477f3d7..566c40f0f7c77 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -161,6 +161,7 @@ static void __init patch_vdso(void *ehdr) vdso_nullpatch_one(&einfo, "__vdso_gettimeofday"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64"); + vdso_nullpatch_one(&einfo, "__vdso_clock_getres"); } } diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S index f2e8d4fac0687..5afb725998ec0 100644 --- a/arch/arm/kernel/vmlinux-xip.lds.S +++ b/arch/arm/kernel/vmlinux-xip.lds.S @@ -154,6 +154,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ARM_DETAILS ARM_ASSERTS diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index d592a203f9c6b..c07843c3c53d3 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -153,6 +153,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ARM_DETAILS ARM_ASSERTS diff --git a/arch/arm/mach-omap2/control.c b/arch/arm/mach-omap2/control.c index 79860b23030de..eb6fc7c61b6e0 100644 --- a/arch/arm/mach-omap2/control.c +++ b/arch/arm/mach-omap2/control.c @@ -732,7 +732,7 @@ int __init omap2_control_base_init(void) */ int __init omap_control_init(void) { - struct device_node *np, *scm_conf; + struct device_node *np, *scm_conf, *clocks_node; const struct of_device_id *match; const struct omap_prcm_init_data *data; int ret; @@ -753,16 +753,19 @@ int __init omap_control_init(void) if (IS_ERR(syscon)) { ret = PTR_ERR(syscon); - goto of_node_put; + goto err_put_scm_conf; } - if (of_get_child_by_name(scm_conf, "clocks")) { + clocks_node = of_get_child_by_name(scm_conf, "clocks"); + if (clocks_node) { + of_node_put(clocks_node); ret = omap2_clk_provider_init(scm_conf, data->index, syscon, NULL); if (ret) - goto of_node_put; + goto err_put_scm_conf; } + of_node_put(scm_conf); } else { /* No scm_conf found, direct access */ ret = omap2_clk_provider_init(np, data->index, NULL, @@ -780,6 +783,9 @@ int __init omap_control_init(void) return 0; +err_put_scm_conf: + if (scm_conf) + of_node_put(scm_conf); of_node_put: of_node_put(np); return ret; diff --git a/arch/arm/mm/physaddr.c b/arch/arm/mm/physaddr.c index 3f263c840ebc4..1a37ebfacbba9 100644 --- a/arch/arm/mm/physaddr.c +++ b/arch/arm/mm/physaddr.c @@ -38,7 +38,7 @@ static inline bool __virt_addr_valid(unsigned long x) phys_addr_t __virt_to_phys(unsigned long x) { WARN(!__virt_addr_valid(x), - "virt_to_phys used for non-linear address: %pK (%pS)\n", + "virt_to_phys used for non-linear address: %px (%pS)\n", (void *)x, (void *)x); return __virt_to_phys_nodebug(x); diff --git a/arch/arm64/Kbuild b/arch/arm64/Kbuild index 5bfbf7d79c99b..d876bc0e54211 100644 --- a/arch/arm64/Kbuild +++ b/arch/arm64/Kbuild @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only + +# Branch profiling isn't noinstr-safe +subdir-ccflags-$(CONFIG_TRACE_BRANCH_PROFILING) += -DDISABLE_BRANCH_PROFILING + obj-y += kernel/ mm/ net/ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_XEN) += xen/ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6663ffd23f252..840a945cb4acd 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1154,6 +1154,18 @@ config ARM64_ERRATUM_3194386 If unsure, say Y. +config ARM64_ERRATUM_4193714 + bool "C1-Pro: 4193714: SME DVMSync early acknowledgement" + depends on ARM64_SME + default y + help + Enable workaround for C1-Pro acknowledging the DVMSync before + the SME memory accesses are complete. This will cause TLB + maintenance for processes using SME to also issue an IPI to + the affected CPUs. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi index 07aaaf71ea9ae..f226df3ce153b 100644 --- a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi +++ b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi @@ -969,6 +969,10 @@ no-sd; resets = <&reset RESET_SD_EMMC_A>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; + }; sd: mmc@8a000 { @@ -984,6 +988,9 @@ no-sdio; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; nand: nand-controller@8d000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index 04fb130ac7c6a..bbf94a1f92a10 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -1960,6 +1960,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@7000 { @@ -1972,6 +1975,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; nfc: nand-controller@7800 { diff --git a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi index dcc927a9da802..8d8ab775404d9 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi @@ -2431,6 +2431,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_b: mmc@ffe05000 { @@ -2443,6 +2446,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@ffe07000 { @@ -2455,6 +2461,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; usb: usb@ffe09000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12b.dtsi index f04efa8282561..23358d94844c9 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12b.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12b.dtsi @@ -87,7 +87,7 @@ i-cache-line-size = <32>; i-cache-size = <0x8000>; i-cache-sets = <32>; - next-level-cache = <&l2_cache_l>; + next-level-cache = <&l2_cache_b>; #cooling-cells = <2>; }; @@ -103,7 +103,7 @@ i-cache-line-size = <32>; i-cache-size = <0x8000>; i-cache-sets = <32>; - next-level-cache = <&l2_cache_l>; + next-level-cache = <&l2_cache_b>; #cooling-cells = <2>; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi index f69923da07feb..a9c830a570cc6 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi @@ -824,6 +824,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -832,6 +835,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -840,6 +846,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi index ba535010a3c91..e202d84f06720 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi @@ -894,6 +894,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -902,6 +905,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -910,6 +916,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi index 9d99ed2994dfa..dfc0a30a6e61b 100644 --- a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi @@ -819,13 +819,16 @@ reg = <0x0 0xfe088000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_SDEMMC_A>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_A>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; cap-sdio-irq; keep-power-in-suspend; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; }; sd: mmc@fe08a000 { @@ -838,6 +841,9 @@ clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; emmc: mmc@fe08c000 { @@ -845,13 +851,16 @@ reg = <0x0 0xfe08c000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_NAND>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_C>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_NAND_EMMC>; no-sdio; no-sd; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_C>; + assigned-clock-rates = <24000000>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 06fe257f08be4..4ae1ce919dafc 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -21,6 +21,25 @@ }; }; +/* + * Keep the power-domains used for the HDMI port on. + */ +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +/* + * The M2 Mac mini uses dispext for the HDMI output so it's not necessary to + * keep disp0 power-domains always-on. + */ +&ps_disp0_sys { + /delete-property/ apple,always-on; +}; + +&ps_disp0_fe { + /delete-property/ apple,always-on; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/exynos/google/gs101.dtsi b/arch/arm64/boot/dts/exynos/google/gs101.dtsi index 31c99526470d0..c01ff4e1b0d5d 100644 --- a/arch/arm64/boot/dts/exynos/google/gs101.dtsi +++ b/arch/arm64/boot/dts/exynos/google/gs101.dtsi @@ -288,7 +288,7 @@ cmu_misc: clock-controller@10010000 { compatible = "google,gs101-cmu-misc"; - reg = <0x10010000 0x8000>; + reg = <0x10010000 0x10000>; #clock-cells = <1>; clocks = <&cmu_top CLK_DOUT_CMU_MISC_BUS>, <&cmu_top CLK_DOUT_CMU_MISC_SSS>; @@ -365,7 +365,7 @@ cmu_peric0: clock-controller@10800000 { compatible = "google,gs101-cmu-peric0"; - reg = <0x10800000 0x4000>; + reg = <0x10800000 0x10000>; #clock-cells = <1>; clocks = <&ext_24_5m>, <&cmu_top CLK_DOUT_CMU_PERIC0_BUS>, @@ -911,7 +911,7 @@ cmu_peric1: clock-controller@10c00000 { compatible = "google,gs101-cmu-peric1"; - reg = <0x10c00000 0x4000>; + reg = <0x10c00000 0x10000>; #clock-cells = <1>; clocks = <&ext_24_5m>, <&cmu_top CLK_DOUT_CMU_PERIC1_BUS>, @@ -1265,7 +1265,7 @@ cmu_hsi0: clock-controller@11000000 { compatible = "google,gs101-cmu-hsi0"; - reg = <0x11000000 0x4000>; + reg = <0x11000000 0x10000>; #clock-cells = <1>; clocks = <&ext_24_5m>, @@ -1332,7 +1332,7 @@ cmu_hsi2: clock-controller@14400000 { compatible = "google,gs101-cmu-hsi2"; - reg = <0x14400000 0x4000>; + reg = <0x14400000 0x10000>; #clock-cells = <1>; clocks = <&ext_24_5m>, <&cmu_top CLK_DOUT_CMU_HSI2_BUS>, @@ -1395,16 +1395,16 @@ cmu_apm: clock-controller@17400000 { compatible = "google,gs101-cmu-apm"; - reg = <0x17400000 0x8000>; + reg = <0x17400000 0x10000>; #clock-cells = <1>; clocks = <&ext_24_5m>; clock-names = "oscclk"; }; - sysreg_apm: syscon@174204e0 { + sysreg_apm: syscon@17420000 { compatible = "google,gs101-apm-sysreg", "syscon"; - reg = <0x174204e0 0x1000>; + reg = <0x17420000 0x10000>; }; pmu_system_controller: system-controller@17460000 { @@ -1497,7 +1497,7 @@ cmu_top: clock-controller@1e080000 { compatible = "google,gs101-cmu-top"; - reg = <0x1e080000 0x8000>; + reg = <0x1e080000 0x10000>; #clock-cells = <1>; clocks = <&ext_24_5m>; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi index 752caa38eb03b..266038fbbef97 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi @@ -351,17 +351,6 @@ >; }; - pinctrl_usdhc1: usdhc1grp { - fsl,pins = < - MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK 0x190 - MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD 0x1d0 - MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0 0x1d0 - MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1 0x1d0 - MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2 0x1d0 - MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3 0x1d0 - >; - }; - pinctrl_usdhc2: usdhc2grp { fsl,pins = < MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x190 diff --git a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts index d7f7f9aafb7d1..0d009f4be804e 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts +++ b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx.dts @@ -69,6 +69,10 @@ samsung,esc-clock-frequency = <20000000>; }; +®_usdhc2_vqmmc { + status = "okay"; +}; + &sai3 { assigned-clocks = <&clk IMX8MN_CLK_SAI3>; assigned-clock-parents = <&clk IMX8MN_AUDIO_PLL1_OUT>; @@ -216,8 +220,7 @@ , , , - , - ; + ; }; pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp { @@ -226,8 +229,7 @@ , , , - , - ; + ; }; pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp { @@ -236,8 +238,7 @@ , , , - , - ; + ; }; pinctrl_usdhc2_gpio: usdhc2-gpiogrp { diff --git a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi index 1d23814e11cd3..e2ccebf6ee13f 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi @@ -30,6 +30,20 @@ regulator-max-microvolt = <3300000>; }; + reg_usdhc2_vqmmc: regulator-usdhc2-vqmmc { + compatible = "regulator-gpio"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_usdhc2_vqmmc>; + regulator-name = "V_SD2"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + states = <1800000 0x1>, + <3300000 0x0>; + vin-supply = <&ldo5_reg>; + status = "disabled"; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -233,6 +247,10 @@ vddio-supply = <&ldo3_reg>; }; +&usdhc2 { + vqmmc-supply = <®_usdhc2_vqmmc>; +}; + &usdhc3 { pinctrl-names = "default", "state_100mhz", "state_200mhz"; pinctrl-0 = <&pinctrl_usdhc3>; @@ -287,6 +305,10 @@ fsl,pins = ; }; + pinctrl_reg_usdhc2_vqmmc: regusdhc2vqmmcgrp { + fsl,pins = ; + }; + pinctrl_usdhc3: usdhc3grp { fsl,pins = , , diff --git a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi index 68c2e0156a5c8..f8303b7e2bd22 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi @@ -113,6 +113,7 @@ ethphy0f: ethernet-phy@1 { /* SMSC LAN8740Ai */ compatible = "ethernet-phy-id0007.c110", "ethernet-phy-ieee802.3-c22"; + clocks = <&clk IMX8MP_CLK_ENET_QOS>; interrupt-parent = <&gpio3>; interrupts = <19 IRQ_TYPE_LEVEL_LOW>; pinctrl-0 = <&pinctrl_ethphy0>; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts index f7346b3d35fe5..a122f2ed5f531 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts @@ -704,7 +704,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_gpt1: gpt1grp { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts index 4eedd00d83b9f..ac05c05193c5b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts @@ -860,7 +860,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_hoggpio2: hoggpio2grp { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts index 88ad422c27603..399230144ce39 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts @@ -9,7 +9,7 @@ #include "imx8mp-tx8p-ml81.dtsi" / { - compatible = "gocontroll,moduline-display", "fsl,imx8mp"; + compatible = "gocontroll,moduline-display-106", "karo,tx8p-ml81", "fsl,imx8mp"; chassis-type = "embedded"; hardware = "Moduline Display V1.06"; model = "GOcontroll Moduline Display baseboard"; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi index fe8ba16eb40e7..761ee046eb72e 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi @@ -47,6 +47,7 @@ <&clk IMX8MP_SYS_PLL2_100M>, <&clk IMX8MP_SYS_PLL2_50M>; assigned-clock-rates = <266000000>, <100000000>, <50000000>; + nvmem-cells = <ð_mac1>; phy-handle = <ðphy0>; phy-mode = "rmii"; pinctrl-0 = <&pinctrl_eqos>; @@ -75,6 +76,10 @@ }; }; +&fec { + nvmem-cells = <ð_mac2>; +}; + &gpio1 { gpio-line-names = "SODIMM_152", "SODIMM_42", diff --git a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw702x.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw702x.dtsi index cbf0c9a740faa..fb159199b39de 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw702x.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw702x.dtsi @@ -395,13 +395,6 @@ status = "okay"; }; -/* off-board header */ -&uart1 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_uart1>; - status = "okay"; -}; - /* console */ &uart2 { pinctrl-names = "default"; @@ -409,25 +402,6 @@ status = "okay"; }; -/* off-board header */ -&uart3 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_uart3>; - status = "okay"; -}; - -/* off-board */ -&usdhc1 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_usdhc1>; - bus-width = <4>; - non-removable; - status = "okay"; - bus-width = <4>; - non-removable; - status = "okay"; -}; - /* eMMC */ &usdhc3 { pinctrl-names = "default", "state_100mhz", "state_200mhz"; @@ -523,13 +497,6 @@ >; }; - pinctrl_uart1: uart1grp { - fsl,pins = < - MX8MP_IOMUXC_UART1_RXD__UART1_DCE_RX 0x140 - MX8MP_IOMUXC_UART1_TXD__UART1_DCE_TX 0x140 - >; - }; - pinctrl_uart2: uart2grp { fsl,pins = < MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX 0x140 @@ -537,24 +504,6 @@ >; }; - pinctrl_uart3: uart3grp { - fsl,pins = < - MX8MP_IOMUXC_UART3_RXD__UART3_DCE_RX 0x140 - MX8MP_IOMUXC_UART3_TXD__UART3_DCE_TX 0x140 - >; - }; - - pinctrl_usdhc1: usdhc1grp { - fsl,pins = < - MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK 0x190 - MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD 0x1d0 - MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0 0x1d0 - MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1 0x1d0 - MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2 0x1d0 - MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3 0x1d0 - >; - }; - pinctrl_usdhc3: usdhc3grp { fsl,pins = < MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK 0x190 diff --git a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw72xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw72xx.dtsi index cf747ec6fa16e..76020ef89bf3e 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw72xx.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw72xx.dtsi @@ -365,17 +365,6 @@ >; }; - pinctrl_usdhc1: usdhc1grp { - fsl,pins = < - MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK 0x190 - MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD 0x1d0 - MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0 0x1d0 - MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1 0x1d0 - MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2 0x1d0 - MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3 0x1d0 - >; - }; - pinctrl_usdhc2: usdhc2grp { fsl,pins = < MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK 0x190 diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts index 077c5cd2586f7..4533a84fb0b95 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts @@ -7,7 +7,7 @@ &a53_opp_table { opp-1000000000 { - opp-microvolt = <950000>; + opp-microvolt = <1000000>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi index 9e0e2d7271efb..e5f228517b6db 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi @@ -844,9 +844,9 @@ regulator-max-microvolt = <1300000>; regulator-boot-on; regulator-ramp-delay = <1250>; - rohm,dvs-run-voltage = <880000>; - rohm,dvs-idle-voltage = <820000>; - rohm,dvs-suspend-voltage = <810000>; + rohm,dvs-run-voltage = <900000>; + rohm,dvs-idle-voltage = <850000>; + rohm,dvs-suspend-voltage = <850000>; regulator-always-on; }; @@ -856,8 +856,8 @@ regulator-max-microvolt = <1300000>; regulator-boot-on; regulator-ramp-delay = <1250>; - rohm,dvs-run-voltage = <950000>; - rohm,dvs-idle-voltage = <850000>; + rohm,dvs-run-voltage = <1000000>; + rohm,dvs-idle-voltage = <900000>; regulator-always-on; }; @@ -866,14 +866,14 @@ regulator-min-microvolt = <700000>; regulator-max-microvolt = <1300000>; regulator-boot-on; - rohm,dvs-run-voltage = <850000>; + rohm,dvs-run-voltage = <900000>; }; buck4_reg: BUCK4 { regulator-name = "buck4"; regulator-min-microvolt = <700000>; regulator-max-microvolt = <1300000>; - rohm,dvs-run-voltage = <930000>; + rohm,dvs-run-voltage = <1000000>; }; buck5_reg: BUCK5 { @@ -1407,13 +1407,3 @@ fsl,ext-reset-output; status = "okay"; }; - -&a53_opp_table { - opp-1000000000 { - opp-microvolt = <850000>; - }; - - opp-1500000000 { - opp-microvolt = <950000>; - }; -}; diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi index 607962f807beb..6a25e219832ce 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi @@ -1632,7 +1632,7 @@ <&clk IMX8MQ_GPU_PLL_OUT>, <&clk IMX8MQ_GPU_PLL>; assigned-clock-rates = <800000000>, <800000000>, - <800000000>, <800000000>, <0>; + <800000000>, <400000000>, <0>; power-domains = <&pgc_gpu>; }; diff --git a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts index 9c0b6b8d6459d..df99fe88cf4ac 100644 --- a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts +++ b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts @@ -256,6 +256,7 @@ regulator-max-microvolt = <3000000>; gpio = <&lsio_gpio4 7 GPIO_ACTIVE_HIGH>; enable-active-high; + off-on-delay-us = <4800>; }; reg_audio: regulator-audio { @@ -560,7 +561,7 @@ compatible = "isil,isl29023"; reg = <0x44>; interrupt-parent = <&lsio_gpio4>; - interrupts = <11 IRQ_TYPE_EDGE_FALLING>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; }; pressure-sensor@60 { diff --git a/arch/arm64/boot/dts/freescale/imx8qm-ss-dma.dtsi b/arch/arm64/boot/dts/freescale/imx8qm-ss-dma.dtsi index d4856b8590e0c..e186c31bfd482 100644 --- a/arch/arm64/boot/dts/freescale/imx8qm-ss-dma.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8qm-ss-dma.dtsi @@ -171,25 +171,25 @@ &lpuart0 { compatible = "fsl,imx8qm-lpuart", "fsl,imx8qxp-lpuart"; - dmas = <&edma2 13 0 0>, <&edma2 12 0 1>; + dmas = <&edma2 12 0 FSL_EDMA_RX>, <&edma2 13 0 0>; dma-names = "rx","tx"; }; &lpuart1 { compatible = "fsl,imx8qm-lpuart", "fsl,imx8qxp-lpuart"; - dmas = <&edma2 15 0 0>, <&edma2 14 0 1>; + dmas = <&edma2 14 0 FSL_EDMA_RX>, <&edma2 15 0 0>; dma-names = "rx","tx"; }; &lpuart2 { compatible = "fsl,imx8qm-lpuart", "fsl,imx8qxp-lpuart"; - dmas = <&edma2 17 0 0>, <&edma2 16 0 1>; + dmas = <&edma2 16 0 FSL_EDMA_RX>, <&edma2 17 0 0>; dma-names = "rx","tx"; }; &lpuart3 { compatible = "fsl,imx8qm-lpuart", "fsl,imx8qxp-lpuart"; - dmas = <&edma2 19 0 0>, <&edma2 18 0 1>; + dmas = <&edma2 18 0 FSL_EDMA_RX>, <&edma2 19 0 0>; dma-names = "rx","tx"; }; diff --git a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi index 5792952b7a8e1..c99d7bc168483 100644 --- a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi +++ b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi @@ -272,20 +272,20 @@ /* enable SION for data and cmd pad due to ERR052021 */ pinctrl_usdhc1: usdhc1grp { fsl,pins = /* PD | FSEL 3 | DSE X5 */ - , + , /* HYS | FSEL 0 | no drive */ , /* HYS | FSEL 3 | X5 */ - , + , /* HYS | FSEL 3 | X4 */ - , - , - , - , - , - , - , - ; + , + , + , + , + , + , + , + ; }; pinctrl_wdog: wdoggrp { diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts index 0852067eab2cb..197c8f8b7f669 100644 --- a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts +++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts @@ -507,6 +507,7 @@ pinctrl-2 = <&pinctrl_usdhc1_200mhz>; bus-width = <8>; non-removable; + fsl,tuning-step = <1>; status = "okay"; }; @@ -519,6 +520,7 @@ vmmc-supply = <®_usdhc2_vmmc>; bus-width = <4>; no-mmc; + fsl,tuning-step = <1>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi b/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi index 82914ca148d3a..c095d7f115c21 100644 --- a/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi @@ -270,21 +270,21 @@ /* enable SION for data and cmd pad due to ERR052021 */ pinctrl_usdhc1: usdhc1grp { fsl,pins = < - /* PD | FSEL 3 | DSE X5 */ - MX93_PAD_SD1_CLK__USDHC1_CLK 0x5be + /* PD | FSEL 3 | DSE X4 */ + MX93_PAD_SD1_CLK__USDHC1_CLK 0x59e /* HYS | FSEL 0 | no drive */ MX93_PAD_SD1_STROBE__USDHC1_STROBE 0x1000 - /* HYS | FSEL 3 | X5 */ - MX93_PAD_SD1_CMD__USDHC1_CMD 0x400011be - /* HYS | FSEL 3 | X4 */ - MX93_PAD_SD1_DATA0__USDHC1_DATA0 0x4000119e - MX93_PAD_SD1_DATA1__USDHC1_DATA1 0x4000119e - MX93_PAD_SD1_DATA2__USDHC1_DATA2 0x4000119e - MX93_PAD_SD1_DATA3__USDHC1_DATA3 0x4000119e - MX93_PAD_SD1_DATA4__USDHC1_DATA4 0x4000119e - MX93_PAD_SD1_DATA5__USDHC1_DATA5 0x4000119e - MX93_PAD_SD1_DATA6__USDHC1_DATA6 0x4000119e - MX93_PAD_SD1_DATA7__USDHC1_DATA7 0x4000119e + /* HYS | PU | FSEL 3 | DSE X4 */ + MX93_PAD_SD1_CMD__USDHC1_CMD 0x4000139e + /* HYS | PU | FSEL 3 | DSE X4 */ + MX93_PAD_SD1_DATA0__USDHC1_DATA0 0x4000139e + MX93_PAD_SD1_DATA1__USDHC1_DATA1 0x4000139e + MX93_PAD_SD1_DATA2__USDHC1_DATA2 0x4000139e + MX93_PAD_SD1_DATA3__USDHC1_DATA3 0x4000139e + MX93_PAD_SD1_DATA4__USDHC1_DATA4 0x4000139e + MX93_PAD_SD1_DATA5__USDHC1_DATA5 0x4000139e + MX93_PAD_SD1_DATA6__USDHC1_DATA6 0x4000139e + MX93_PAD_SD1_DATA7__USDHC1_DATA7 0x4000139e >; }; diff --git a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts index 148243470dd4a..0953c25ef5576 100644 --- a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts @@ -61,6 +61,7 @@ fan0: pwm-fan { compatible = "pwm-fan"; + fan-supply = <®_vcc_12v>; #cooling-cells = <2>; cooling-levels = <64 128 192 255>; pwms = <&tpm6 0 4000000 PWM_POLARITY_INVERTED>; diff --git a/arch/arm64/boot/dts/freescale/imx95-tqma9596sa.dtsi b/arch/arm64/boot/dts/freescale/imx95-tqma9596sa.dtsi index 180124cc5bce1..16c40d11d3b5d 100644 --- a/arch/arm64/boot/dts/freescale/imx95-tqma9596sa.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95-tqma9596sa.dtsi @@ -115,7 +115,7 @@ flash0: flash@0 { compatible = "jedec,spi-nor"; reg = <0>; - spi-max-frequency = <80000000>; + spi-max-frequency = <66000000>; spi-tx-bus-width = <4>; spi-rx-bus-width = <4>; vcc-supply = <®_1v8>; @@ -617,7 +617,7 @@ fsl,pins = ; }; - pinctrl_tpm5: tpm4grp { + pinctrl_tpm5: tpm5grp { fsl,pins = ; }; diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi index 6da961eb3fe5c..583c5f7a84b54 100644 --- a/arch/arm64/boot/dts/freescale/imx95.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi @@ -806,7 +806,7 @@ interrupts = ; #address-cells = <3>; #size-cells = <0>; - clocks = <&scmi_clk IMX95_CLK_BUSAON>, + clocks = <&scmi_clk IMX95_CLK_BUSWAKEUP>, <&scmi_clk IMX95_CLK_I3C2SLOW>; clock-names = "pclk", "fast_clk"; status = "disabled"; diff --git a/arch/arm64/boot/dts/freescale/mba8mx.dtsi b/arch/arm64/boot/dts/freescale/mba8mx.dtsi index 79daba930ad64..3e41da4d6122b 100644 --- a/arch/arm64/boot/dts/freescale/mba8mx.dtsi +++ b/arch/arm64/boot/dts/freescale/mba8mx.dtsi @@ -185,7 +185,7 @@ reset-assert-us = <500000>; reset-deassert-us = <500>; interrupt-parent = <&expander2>; - interrupts = <6 IRQ_TYPE_EDGE_FALLING>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; }; }; }; diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts index 7d370dac4c857..579d55daa7d04 100644 --- a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts +++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts @@ -179,7 +179,7 @@ }; &pcie { - reset-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio4 4 GPIO_ACTIVE_LOW>; vpcie-supply = <®_pcie>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi index f6bc001c38326..2f4ad5da5e33c 100644 --- a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi @@ -122,6 +122,7 @@ #address-cells = <1>; #size-cells = <1>; ranges = <0x0 0x0 0xf0000000 0x10000000>; + dma-ranges = <0x0 0x0 0x0 0x40000000>; crg: clock-reset-controller@8a22000 { compatible = "hisilicon,hi3798cv200-crg", "syscon", "simple-mfd"; diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile index a4df4c21399ee..b50799b2a65fb 100644 --- a/arch/arm64/boot/dts/mediatek/Makefile +++ b/arch/arm64/boot/dts/mediatek/Makefile @@ -104,6 +104,8 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += mt8390-genio-700-evk.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8395-kontron-3-5-sbc-i1200.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8395-radxa-nio-12l.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8395-radxa-nio-12l-8-hd-panel.dtbo +mt8395-radxa-nio-12l-8-hd-panel-dtbs := mt8395-radxa-nio-12l.dtb mt8395-radxa-nio-12l-8-hd-panel.dtbo +dtb-$(CONFIG_ARCH_MEDIATEK) += mt8395-radxa-nio-12l-8-hd-panel.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8516-pumpkin.dtb # Device tree overlays support diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts index cce326aec1aa5..40af5656d6f15 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts @@ -91,7 +91,7 @@ &pio { bt_pins_wakeup: bt-pins-wakeup { - piins-bt-wakeup { + pins-bt-wakeup { pinmux = ; input-enable; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi index 960d8955d018c..27ce4b31cc99d 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi @@ -1812,15 +1812,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - ovl_2l1_in: endpoint { + + ovl_2l1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&mmsys_ep_ext>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - ovl_2l1_out: endpoint { + + ovl_2l1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_in>; }; }; @@ -1872,15 +1880,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - rdma1_in: endpoint { + + rdma1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&ovl_2l1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - rdma1_out: endpoint { + + rdma1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&dpi_in>; }; }; @@ -2076,15 +2092,24 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - dpi_in: endpoint { + + dpi_in: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - dpi_out: endpoint { }; + + dpi_out: endpoint@1 { + reg = <1>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8195.dtsi b/arch/arm64/boot/dts/mediatek/mt8195.dtsi index ec452d6570314..c7adafaa83288 100644 --- a/arch/arm64/boot/dts/mediatek/mt8195.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8195.dtsi @@ -3067,7 +3067,7 @@ jpgdec@0,10000 { compatible = "mediatek,mt8195-jpgdec-hw"; - reg = <0 0 0x10000 0x10000>;/* JPGDEC_C1 */ + reg = <0 0x10000 0 0x10000>;/* JPGDEC_C1 */ iommus = <&iommu_vdo M4U_PORT_L19_JPGDEC_WDMA0>, <&iommu_vdo M4U_PORT_L19_JPGDEC_BSDMA0>, <&iommu_vdo M4U_PORT_L19_JPGDEC_WDMA1>, diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts index 5aa6afd56cbc6..dfbd1c72388c1 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts @@ -1809,6 +1809,8 @@ status = "okay"; vbus-supply = <&usbc_vbus>; mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; }; usb3-0 { diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi index df034dbb82853..5657045c53d90 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi @@ -9,6 +9,7 @@ #include #include #include +#include / { compatible = "nvidia,tegra234"; @@ -127,6 +128,56 @@ pinmux: pinmux@2430000 { compatible = "nvidia,tegra234-pinmux"; reg = <0x0 0x2430000 0x0 0x19100>; + + pex_rst_c4_in_state: pinmux-pex-rst-c4-in { + pex_rst { + nvidia,pins = "pex_l4_rst_n_pl1"; + nvidia,function = "rsvd1"; + nvidia,pull = ; + nvidia,tristate = ; + nvidia,enable-input = ; + }; + }; + + pex_rst_c5_in_state: pinmux-pex-rst-c5-in { + pex_rst { + nvidia,pins = "pex_l5_rst_n_paf1"; + nvidia,function = "rsvd1"; + nvidia,pull = ; + nvidia,tristate = ; + nvidia,enable-input = ; + }; + }; + + pex_rst_c6_in_state: pinmux-pex-rst-c6-in { + pex_rst { + nvidia,pins = "pex_l6_rst_n_paf3"; + nvidia,function = "rsvd1"; + nvidia,pull = ; + nvidia,tristate = ; + nvidia,enable-input = ; + }; + }; + + pex_rst_c7_in_state: pinmux-pex-rst-c7-in { + pex_rst { + nvidia,pins = "pex_l7_rst_n_pag1"; + nvidia,function = "rsvd1"; + nvidia,pull = ; + nvidia,tristate = ; + nvidia,enable-input = ; + }; + }; + + pex_rst_c10_in_state: pinmux-pex-rst-c10-in { + pex_rst { + nvidia,pins = "pex_l10_rst_n_pag7"; + nvidia,function = "rsvd1"; + nvidia,pull = ; + nvidia,tristate = ; + nvidia,enable-input = ; + }; + }; }; gpcdma: dma-controller@2600000 { @@ -4630,6 +4681,8 @@ <&bpmp TEGRA234_RESET_PEX2_CORE_10>; reset-names = "apb", "core"; + pinctrl-names = "default"; + pinctrl-0 = <&pex_rst_c10_in_state>; interrupts = ; /* controller interrupt */ interrupt-names = "intr"; @@ -4881,6 +4934,8 @@ <&bpmp TEGRA234_RESET_PEX0_CORE_4>; reset-names = "apb", "core"; + pinctrl-names = "default"; + pinctrl-0 = <&pex_rst_c4_in_state>; interrupts = ; /* controller interrupt */ interrupt-names = "intr"; nvidia,bpmp = <&bpmp 4>; @@ -5023,6 +5078,8 @@ <&bpmp TEGRA234_RESET_PEX1_CORE_5>; reset-names = "apb", "core"; + pinctrl-names = "default"; + pinctrl-0 = <&pex_rst_c5_in_state>; interrupts = ; /* controller interrupt */ interrupt-names = "intr"; @@ -5115,6 +5172,8 @@ <&bpmp TEGRA234_RESET_PEX1_CORE_6>; reset-names = "apb", "core"; + pinctrl-names = "default"; + pinctrl-0 = <&pex_rst_c6_in_state>; interrupts = ; /* controller interrupt */ interrupt-names = "intr"; @@ -5207,6 +5266,8 @@ <&bpmp TEGRA234_RESET_PEX2_CORE_7>; reset-names = "apb", "core"; + pinctrl-names = "default"; + pinctrl-0 = <&pex_rst_c7_in_state>; interrupts = ; /* controller interrupt */ interrupt-names = "intr"; diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile index 296688f7cb265..4c830c54f955e 100644 --- a/arch/arm64/boot/dts/qcom/Makefile +++ b/arch/arm64/boot/dts/qcom/Makefile @@ -13,7 +13,17 @@ dtb-$(CONFIG_ARCH_QCOM) += apq8039-t2.dtb dtb-$(CONFIG_ARCH_QCOM) += apq8094-sony-xperia-kitakami-karin_windy.dtb dtb-$(CONFIG_ARCH_QCOM) += apq8096-db820c.dtb dtb-$(CONFIG_ARCH_QCOM) += apq8096-ifc6640.dtb +dtb-$(CONFIG_ARCH_QCOM) += glymur-crd.dtb dtb-$(CONFIG_ARCH_QCOM) += hamoa-iot-evk.dtb + +hamoa-iot-evk-el2-dtbs := hamoa-iot-evk.dtb x1-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += hamoa-iot-evk-el2.dtb +dtb-$(CONFIG_ARCH_QCOM) += hamoa-iot-evk-camera-imx577.dtbo + +hamoa-iot-evk-camera-imx577-dtbs := hamoa-iot-evk.dtb hamoa-iot-evk-camera-imx577.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += hamoa-iot-evk-camera-imx577.dtb dtb-$(CONFIG_ARCH_QCOM) += ipq5018-rdp432-c2.dtb dtb-$(CONFIG_ARCH_QCOM) += ipq5018-tplink-archer-ax55-v1.dtb dtb-$(CONFIG_ARCH_QCOM) += ipq5332-rdp441.dtb @@ -30,12 +40,47 @@ dtb-$(CONFIG_ARCH_QCOM) += ipq9574-rdp433.dtb dtb-$(CONFIG_ARCH_QCOM) += ipq9574-rdp449.dtb dtb-$(CONFIG_ARCH_QCOM) += ipq9574-rdp453.dtb dtb-$(CONFIG_ARCH_QCOM) += ipq9574-rdp454.dtb +dtb-$(CONFIG_ARCH_QCOM) += kaanapali-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += lemans-evk.dtb +lemans-evk-emmc-dtbs := lemans-evk.dtb lemans-evk-emmc.dtbo +dtb-$(CONFIG_ARCH_QCOM) += lemans-evk-emmc.dtb + +lemans-evk-sd-card-dtbs := lemans-evk.dtb lemans-evk-sd-card.dtbo +dtb-$(CONFIG_ARCH_QCOM) += lemans-evk-sd-card.dtb + lemans-evk-camera-csi1-imx577-dtbs := lemans-evk.dtb lemans-evk-camera-csi1-imx577.dtbo +lemans-evk-el2-dtbs := lemans-evk.dtb lemans-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += lemans-evk-camera-csi1-imx577.dtb +dtb-$(CONFIG_ARCH_QCOM) += lemans-evk-el2.dtb + +lemans-evk-ifp-mezzanine-dtbs := lemans-evk.dtb lemans-evk-ifp-mezzanine.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += lemans-evk-ifp-mezzanine.dtb +dtb-$(CONFIG_ARCH_QCOM) += monaco-ac-evk.dtb dtb-$(CONFIG_ARCH_QCOM) += monaco-evk.dtb + +monaco-evk-el2-dtbs := monaco-evk.dtb monaco-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += monaco-evk-el2.dtb + +monaco-evk-camera-imx577-dtbs := monaco-evk.dtb monaco-evk-camera-imx577.dtbo +dtb-$(CONFIG_ARCH_QCOM) += monaco-evk-camera-imx577.dtb + +monaco-ac-evk-ifp-mezzanine-dtbs := monaco-ac-evk.dtb monaco-evk-ifp-mezzanine.dtbo +dtb-$(CONFIG_ARCH_QCOM) += monaco-ac-evk-ifp-mezzanine.dtb + +monaco-evk-ifp-mezzanine-dtbs := monaco-evk.dtb monaco-evk-ifp-mezzanine.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += monaco-evk-ifp-mezzanine.dtb + +monaco-evk-sd-card-dtbs := monaco-evk.dtb monaco-evk-sd-card.dtbo +dtb-$(CONFIG_ARCH_QCOM) += monaco-evk-sd-card.dtb + +monaco-evk-emmc-dtbs := monaco-evk.dtb monaco-evk-emmc.dtbo +dtb-$(CONFIG_ARCH_QCOM) += monaco-evk-emmc.dtb + dtb-$(CONFIG_ARCH_QCOM) += msm8216-samsung-fortuna3g.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-acer-a1-724.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-alcatel-idol347.dtb @@ -119,24 +164,59 @@ dtb-$(CONFIG_ARCH_QCOM) += msm8998-sony-xperia-yoshino-lilac.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-sony-xperia-yoshino-maple.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-sony-xperia-yoshino-poplar.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-xiaomi-sagit.dtb +dtb-$(CONFIG_ARCH_QCOM) += purwa-iot-evk.dtb + +purwa-iot-evk-el2-dtbs := purwa-iot-evk.dtb x1-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += purwa-iot-evk-el2.dtb dtb-$(CONFIG_ARCH_QCOM) += qcm6490-fairphone-fp5.dtb + dtb-$(CONFIG_ARCH_QCOM) += qcm6490-idp.dtb +qcm6490-idp-el2-dtbs := qcm6490-idp.dtb kodiak-el2.dtbo +dtb-$(CONFIG_ARCH_QCOM) += qcm6490-idp-el2.dtb + dtb-$(CONFIG_ARCH_QCOM) += qcm6490-particle-tachyon.dtb dtb-$(CONFIG_ARCH_QCOM) += qcm6490-shift-otter.dtb dtb-$(CONFIG_ARCH_QCOM) += qcs404-evb-1000.dtb dtb-$(CONFIG_ARCH_QCOM) += qcs404-evb-4000.dtb dtb-$(CONFIG_ARCH_QCOM) += qcs615-ride.dtb + +qcs615-ride-el2-dtbs := qcs615-ride.dtb talos-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs615-ride-el2.dtb + dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2.dtb +qcs6490-rb3gen2-el2-dtbs := qcs6490-rb3gen2.dtb kodiak-el2.dtbo +dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-el2.dtb qcs6490-rb3gen2-vision-mezzanine-dtbs := qcs6490-rb3gen2.dtb qcs6490-rb3gen2-vision-mezzanine.dtbo qcs6490-rb3gen2-industrial-mezzanine-dtbs := qcs6490-rb3gen2.dtb qcs6490-rb3gen2-industrial-mezzanine.dtbo dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-industrial-mezzanine.dtb +qcs6490-rb3gen2-industrial-mezzanine-el2-dtbs := qcs6490-rb3gen2-industrial-mezzanine.dtb kodiak-el2.dtbo +dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-industrial-mezzanine-el2.dtb + dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-vision-mezzanine.dtb +qcs6490-rb3gen2-vision-mezzanine-el2-dtbs := qcs6490-rb3gen2-vision-mezzanine.dtb kodiak-el2.dtbo +dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-vision-mezzanine-el2.dtb + dtb-$(CONFIG_ARCH_QCOM) += qcs8300-ride.dtb + +qcs8300-ride-el2-dtbs := qcs8300-ride.dtb monaco-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs8300-ride-el2.dtb + dtb-$(CONFIG_ARCH_QCOM) += qcs8550-aim300-aiot.dtb dtb-$(CONFIG_ARCH_QCOM) += qcs9100-ride.dtb dtb-$(CONFIG_ARCH_QCOM) += qcs9100-ride-r3.dtb + +qcs9100-ride-el2-dtbs := qcs9100-ride.dtb lemans-el2.dtbo +qcs9100-ride-r3-el2-dtbs := qcs9100-ride-r3.dtb lemans-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs9100-ride-r3.dtb + +dtb-$(CONFIG_ARCH_QCOM) += qcs9100-ride-el2.dtb +dtb-$(CONFIG_ARCH_QCOM) += qcs9100-ride-r3-el2.dtb dtb-$(CONFIG_ARCH_QCOM) += qdu1000-idp.dtb dtb-$(CONFIG_ARCH_QCOM) += qrb2210-rb1.dtb dtb-$(CONFIG_ARCH_QCOM) += qrb4210-rb2.dtb @@ -150,7 +230,9 @@ dtb-$(CONFIG_ARCH_QCOM) += sa8155p-adp.dtb dtb-$(CONFIG_ARCH_QCOM) += sa8295p-adp.dtb dtb-$(CONFIG_ARCH_QCOM) += sa8540p-ride.dtb dtb-$(CONFIG_ARCH_QCOM) += sa8775p-ride.dtb + dtb-$(CONFIG_ARCH_QCOM) += sa8775p-ride-r3.dtb + sc7180-acer-aspire1-el2-dtbs := sc7180-acer-aspire1.dtb sc7180-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += sc7180-acer-aspire1.dtb sc7180-acer-aspire1-el2.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-idp.dtb @@ -303,6 +385,7 @@ dtb-$(CONFIG_ARCH_QCOM) += sm8550-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8550-qrd.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8550-samsung-q5q.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8550-sony-xperia-yodo-pdx234.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm8650-ayaneo-pocket-s2.dtb sm8650-hdk-display-card-dtbs := sm8650-hdk.dtb sm8650-hdk-display-card.dtbo @@ -312,6 +395,13 @@ dtb-$(CONFIG_ARCH_QCOM) += sm8650-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8650-qrd.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8750-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8750-qrd.dtb +dtb-$(CONFIG_ARCH_QCOM) += talos-evk.dtb +dtb-$(CONFIG_ARCH_QCOM) += talos-evk-camera-imx577.dtbo +talos-evk-camera-imx577-dtbs := talos-evk.dtb talos-evk-camera-imx577.dtbo +talos-evk-lvds-auo,g133han01-dtbs := \ + talos-evk.dtb talos-evk-lvds-auo,g133han01.dtbo +dtb-$(CONFIG_ARCH_QCOM) += talos-evk-camera-imx577.dtb +dtb-$(CONFIG_ARCH_QCOM) += talos-evk-lvds-auo,g133han01.dtb x1e001de-devkit-el2-dtbs := x1e001de-devkit.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1e001de-devkit.dtb x1e001de-devkit-el2.dtb x1e78100-lenovo-thinkpad-t14s-el2-dtbs := x1e78100-lenovo-thinkpad-t14s.dtb x1-el2.dtbo @@ -344,9 +434,88 @@ x1e80100-qcp-el2-dtbs := x1e80100-qcp.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1e80100-qcp.dtb x1e80100-qcp-el2.dtb x1p42100-asus-zenbook-a14-el2-dtbs := x1p42100-asus-zenbook-a14.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1p42100-asus-zenbook-a14.dtb x1p42100-asus-zenbook-a14-el2.dtb +x1p42100-asus-zenbook-a14-lcd-el2-dtbs := x1p42100-asus-zenbook-a14-lcd.dtb x1-el2.dtbo +dtb-$(CONFIG_ARCH_QCOM) += x1p42100-asus-zenbook-a14-lcd.dtb x1p42100-asus-zenbook-a14-lcd-el2.dtb x1p42100-crd-el2-dtbs := x1p42100-crd.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1p42100-crd.dtb x1p42100-crd-el2.dtb x1p42100-hp-omnibook-x14-el2-dtbs := x1p42100-hp-omnibook-x14.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1p42100-hp-omnibook-x14.dtb x1p42100-hp-omnibook-x14-el2.dtb x1p42100-lenovo-thinkbook-16-el2-dtbs := x1p42100-lenovo-thinkbook-16.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1p42100-lenovo-thinkbook-16.dtb x1p42100-lenovo-thinkbook-16-el2.dtb + +hamoa-evk-camx-dtbs := hamoa-iot-evk.dtb hamoa-evk-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += hamoa-evk-camx.dtb + +lemans-evk-camx-dtbs := lemans-evk.dtb lemans-evk-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += lemans-evk-camx.dtb + +lemans-camx-el2-dtbs := lemans-evk-el2.dtb lemans-evk-camx.dtbo lemans-camx-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += lemans-camx-el2.dtb + +dtb-$(CONFIG_ARCH_QCOM) += lemans-evk-staging.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += lemans-staging.dtbo + +monaco-evk-camx-dtbs := monaco-evk.dtb monaco-evk-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += monaco-evk-camx.dtb + +monaco-camx-el2-dtbs := monaco-evk-el2.dtb monaco-evk-camx.dtbo monaco-camx-el2.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += monaco-camx-el2.dtb + +dtb-$(CONFIG_ARCH_QCOM) += monaco-evk-staging.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += monaco-staging.dtbo + +purwa-evk-camx-dtbs := purwa-iot-evk.dtb purwa-evk-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += purwa-evk-camx.dtb + +qcs5430-fps-camx-dtbs := qcs6490-rb3gen2.dtb qcs5430-fps-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs5430-fps-camx.dtb + +qcs615-ride-camx-dtbs := qcs615-ride.dtb qcs615-ride-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs615-ride-camx.dtb + +qcs6490-rb3gen2-vision-mezzanine-camx-dtbs := qcs6490-rb3gen2-vision-mezzanine.dtb \ + qcs6490-rb3gen2-vision-mezzanine-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-vision-mezzanine-camx.dtb + +dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-staging.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs6490-rb3gen2-industrial-mezzanine-staging.dtbo + +qcs8300-ride-camx-dtbs:= qcs8300-ride.dtb qcs8300-ride-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs8300-ride-camx.dtb + +qcs9100-ride-camx-dtbs:= qcs9100-ride.dtb sa8775p-ride-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs9100-ride-camx.dtb + +qcs9100-ride-r3-camx-dtbs:= qcs9100-ride-r3.dtb sa8775p-ride-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += qcs9100-ride-r3-camx.dtb + +sa8775p-ride-camx-dtbs:= sa8775p-ride.dtb sa8775p-ride-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += sa8775p-ride-camx.dtb + +sa8775p-ride-r3-camx-dtbs:= sa8775p-ride-r3.dtb sa8775p-ride-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += sa8775p-ride-r3-camx.dtb + +talos-evk-camx-dtbs := talos-evk.dtb talos-evk-camx.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += talos-evk-camx.dtb + +dtb-$(CONFIG_ARCH_QCOM) += talos-staging.dtbo + +dtb-$(CONFIG_ARCH_QCOM) += kodiak-staging.dtbo diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts new file mode 100644 index 0000000000000..a5113b7b85983 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; + +#include "glymur.dtsi" +#include "glymur-pmics.dtsi" +#include + +/ { + model = "Qualcomm Technologies, Inc. Glymur CRD"; + compatible = "qcom,glymur-crd", "qcom,glymur"; + + aliases { + serial0 = &uart21; + serial1 = &uart14; + i2c0 = &i2c16; + i2c1 = &i2c17; + i2c2 = &i2c18; + i2c3 = &i2c19; + i2c4 = &i2c20; + i2c5 = &i2c22; + i2c6 = &i2c23; + i2c7 = &i2c8; + i2c8 = &i2c9; + i2c9 = &i2c10; + i2c10 = &i2c11; + i2c11 = &i2c12; + i2c12 = &i2c13; + i2c13 = &i2c15; + i2c14 = &i2c0; + i2c15 = &i2c1; + i2c16 = &i2c2; + i2c17 = &i2c3; + i2c18 = &i2c4; + i2c19 = &i2c5; + i2c20 = &i2c6; + spi0 = &spi16; + spi1 = &spi17; + spi2 = &spi18; + spi3 = &spi19; + spi4 = &spi20; + spi5 = &spi22; + spi6 = &spi23; + spi7 = &spi8; + spi8 = &spi9; + spi9 = &spi10; + spi10 = &spi11; + spi11 = &spi12; + spi12 = &spi13; + spi13 = &spi15; + spi14 = &spi0; + spi15 = &spi1; + spi16 = &spi2; + spi17 = &spi3; + spi18 = &spi4; + spi19 = &spi5; + spi20 = &spi6; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + gpio-keys { + compatible = "gpio-keys"; + + pinctrl-0 = <&key_vol_up_default>; + pinctrl-names = "default"; + + key-volume-up { + label = "Volume Up"; + linux,code = ; + gpios = <&pmh0101_gpios 6 GPIO_ACTIVE_LOW>; + debounce-interval = <15>; + linux,can-disable; + wakeup-source; + }; + }; + + vreg_nvme: regulator-nvme { + compatible = "regulator-fixed"; + + regulator-name = "VREG_NVME_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmh0101_gpios 14 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&nvme_reg_en>; + pinctrl-names = "default"; + }; +}; + +&tlmm { + gpio-reserved-ranges = <4 4>, <10 2>, <44 4>; /*Security SPI (TPM)*/ +}; + +&pmh0101_gpios { + key_vol_up_default: key-vol-up-default-state { + pins = "gpio6"; + function = "gpio"; + output-disable; + bias-pull-up; + }; +}; + +&pon_resin { + linux,code = ; + status = "okay"; +}; + +&apps_rsc { + + vph_pwr: vph-pwr-regulator { + compatible = "regulator-fixed"; + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + }; + + regulators-0 { + compatible = "qcom,pmh0101-rpmh-regulators"; + qcom,pmic-id = "B_E0"; + + vdd-bob1-supply = <&vph_pwr>; + vdd-bob2-supply = <&vph_pwr>; + vdd-l1-l10-l15-supply = <&vreg_s9f_e0_1p9>; + vdd-l2-l7-l8-l9-l16-supply = <&vreg_bob1_e0>; + vdd-l11-l12-l18-supply = <&vreg_s7f_e0_1p32>; + vdd-l17-supply = <&vreg_bob2_e0>; + + vreg_bob1_e0: bob1 { + regulator-name = "vreg_bob1_e0"; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <4224000>; + regulator-initial-mode = ; + }; + + vreg_bob2_e0: bob2 { + regulator-name = "vreg_bob2_e0"; + regulator-min-microvolt = <2540000>; + regulator-max-microvolt = <3600000>; + regulator-initial-mode = ; + }; + + vreg_l1b_e0_1p8: ldo1 { + regulator-name = "vreg_l1b_e0_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l2b_e0_2p9: ldo2 { + regulator-name = "vreg_l2b_e0_2p9"; + regulator-min-microvolt = <2900000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = ; + }; + + vreg_l7b_e0_2p79: ldo7 { + regulator-name = "vreg_l7b_e0_2p79"; + regulator-min-microvolt = <2790000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = ; + }; + + vreg_l8b_e0_1p50: ldo8 { + regulator-name = "vreg_l8b_e0_1p50"; + regulator-min-microvolt = <1504000>; + regulator-max-microvolt = <3544000>; + regulator-initial-mode = ; + }; + + vreg_l9b_e0_2p7: ldo9 { + regulator-name = "vreg_l9b_e0_2p7"; + regulator-min-microvolt = <2700000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = ; + }; + + vreg_l10b_e0_1p8: ldo10 { + regulator-name = "vreg_l10b_e0_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l11b_e0_1p2: ldo11 { + regulator-name = "vreg_l11b_e0_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_l12b_e0_1p14: ldo12 { + regulator-name = "vreg_l12b_e0_1p14"; + regulator-min-microvolt = <1140000>; + regulator-max-microvolt = <1260000>; + regulator-initial-mode = ; + }; + + vreg_l15b_e0_1p8: ldo15 { + regulator-name = "vreg_l15b_e0_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l17b_e0_2p4: ldo17 { + regulator-name = "vreg_l17b_e0_2p4"; + regulator-min-microvolt = <2400000>; + regulator-max-microvolt = <2700000>; + regulator-initial-mode = ; + }; + + vreg_l18b_e0_1p2: ldo18 { + regulator-name = "vreg_l18b_e0_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + }; + + regulators-1 { + compatible = "qcom,pmcx0102-rpmh-regulators"; + qcom,pmic-id = "C_E0"; + vdd-s1-supply = <&vph_pwr>; + vdd-s8-supply = <&vph_pwr>; + + vreg_s1c_e0_0p3: smps1 { + regulator-name = "vreg_s1c_e0_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_s8c_e0_0p3: smps8 { + regulator-name = "vreg_s8c_e0_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + }; + + regulators-2 { + compatible = "qcom,pmcx0102-rpmh-regulators"; + qcom,pmic-id = "C_E1"; + + vdd-l2-supply = <&vreg_s7f_e0_1p32>; + vdd-l1-l3-l4-supply = <&vreg_s8f_e0_0p95>; + + vreg_l1c_e1_0p82: ldo1 { + regulator-name = "vreg_l1c_e1_0p82"; + regulator-min-microvolt = <825000>; + regulator-max-microvolt = <958000>; + regulator-initial-mode = ; + }; + + vreg_l2c_e1_1p14: ldo2 { + regulator-name = "vreg_l2c_e1_1p14"; + regulator-min-microvolt = <1140000>; + regulator-max-microvolt = <1300000>; + regulator-initial-mode = ; + }; + + vreg_l3c_e1_0p89: ldo3 { + regulator-name = "vreg_l3c_e1_0p89"; + regulator-min-microvolt = <890000>; + regulator-max-microvolt = <980000>; + regulator-initial-mode = ; + }; + + vreg_l4c_e1_0p72: ldo4 { + regulator-name = "vreg_l4c_e1_0p72"; + regulator-min-microvolt = <720000>; + regulator-max-microvolt = <980000>; + regulator-initial-mode = ; + }; + }; + + regulators-3 { + compatible = "qcom,pmh0110-rpmh-regulators"; + qcom,pmic-id = "F_E0"; + vdd-s7-supply = <&vph_pwr>; + vdd-s8-supply = <&vph_pwr>; + vdd-s9-supply = <&vph_pwr>; + vdd-s10-supply = <&vph_pwr>; + vdd-l2-supply = <&vreg_s8f_e0_0p95>; + vdd-l3-supply = <&vreg_s8f_e0_0p95>; + vdd-l4-supply = <&vreg_s8f_e0_0p95>; + + vreg_s7f_e0_1p32: smps7 { + regulator-name = "vreg_s7f_e0_1p32"; + regulator-min-microvolt = <1320000>; + regulator-max-microvolt = <1352000>; + regulator-initial-mode = ; + }; + + vreg_s8f_e0_0p95: smps8 { + regulator-name = "vreg_s8f_e0_0p95"; + regulator-min-microvolt = <952000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_s9f_e0_1p9: smps9 { + regulator-name = "vreg_s9f_e0_1p9"; + regulator-min-microvolt = <1900000>; + regulator-max-microvolt = <2000000>; + regulator-initial-mode = ; + }; + + vreg_s10f_e0_0p3: smps10 { + regulator-name = "vreg_s10f_e0_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_l2f_e0_0p82: ldo2 { + regulator-name = "vreg_l2f_e0_0p82"; + regulator-min-microvolt = <825000>; + regulator-max-microvolt = <980000>; + regulator-initial-mode = ; + }; + + vreg_l3f_e0_0p72: ldo3 { + regulator-name = "vreg_l3f_e0_0p72"; + regulator-min-microvolt = <720000>; + regulator-max-microvolt = <980000>; + regulator-initial-mode = ; + }; + + vreg_l4f_e0_0p3: ldo4 { + regulator-name = "vreg_l4f_e0_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + }; + + regulators-4 { + compatible = "qcom,pmh0110-rpmh-regulators"; + qcom,pmic-id = "F_E1"; + vdd-s1-supply = <&vph_pwr>; + vdd-s3-supply = <&vph_pwr>; + vdd-s5-supply = <&vph_pwr>; + vdd-s6-supply = <&vph_pwr>; + vdd-s7-supply = <&vph_pwr>; + vdd-l1-supply = <&vreg_s8f_e0_0p95>; + vdd-l2-supply = <&vreg_s8f_e0_0p95>; + vdd-l4-supply = <&vreg_s8f_e0_0p95>; + + vreg_s1f_e1_0p3: smps1 { + regulator-name = "vreg_s1f_e1_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_s5f_e1_0p3: smps5 { + regulator-name = "vreg_s5f_e1_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_s6f_e1_0p3: smps6 { + regulator-name = "vreg_s6f_e1_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_s7f_e1_0p3: smps7 { + regulator-name = "vreg_s7f_e1_0p3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_l1f_e1_0p82: ldo1 { + regulator-name = "vreg_l1f_e1_0p82"; + regulator-min-microvolt = <825000>; + regulator-max-microvolt = <950000>; + regulator-initial-mode = ; + }; + + vreg_l2f_e1_0p83: ldo2 { + regulator-name = "vreg_l2f_e1_0p83"; + regulator-min-microvolt = <830000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_l4f_e1_1p08: ldo4 { + regulator-name = "vreg_l4f_e1_1p08"; + regulator-min-microvolt = <1080000>; + regulator-max-microvolt = <1320000>; + regulator-initial-mode = ; + }; + }; + + regulators-5 { + compatible = "qcom,pmh0110-rpmh-regulators"; + qcom,pmic-id = "H_E0"; + + vdd-l1-supply = <&vreg_s8f_e0_0p95>; + vdd-l2-supply = <&vreg_s8f_e0_0p95>; + vdd-l3-supply = <&vreg_s9f_e0_1p9>; + vdd-l4-supply = <&vreg_s7f_e0_1p32>; + + vreg_l1h_e0_0p89: ldo1 { + regulator-name = "vreg_l1h_e0_0p89"; + regulator-min-microvolt = <825000>; + regulator-max-microvolt = <950000>; + regulator-initial-mode = ; + }; + + vreg_l2h_e0_0p72: ldo2 { + regulator-name = "vreg_l2h_e0_0p72"; + regulator-min-microvolt = <830000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_l3h_e0_0p32: ldo3 { + regulator-name = "vreg_l3h_e0_0p32"; + regulator-min-microvolt = <320000>; + regulator-max-microvolt = <2000000>; + regulator-initial-mode = ; + }; + + vreg_l4h_e0_1p2: ldo4 { + regulator-name = "vreg_l4h_e0_1p2"; + regulator-min-microvolt = <1080000>; + regulator-max-microvolt = <1320000>; + regulator-initial-mode = ; + }; + }; +}; + +&pmk8850_rtc { + no-alarm; +}; + +&pmh0101_gpios { + nvme_reg_en: nvme-reg-en-state { + pins = "gpio14"; + function = "normal"; + bias-disable; + }; +}; + +&tlmm { + pcie5_default: pcie5-default-state { + clkreq-n-pins { + pins = "gpio153"; + function = "pcie5_clk_req_n"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-n-pins { + pins = "gpio152"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-n-pins { + pins = "gpio154"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; +}; + +&pcie5 { + vddpe-3v3-supply = <&vreg_nvme>; + + pinctrl-0 = <&pcie5_default>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie5_phy { + vdda-phy-supply = <&vreg_l2f_e0_0p82>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + + status = "okay"; +}; + +&pcie5port0 { + perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; diff --git a/arch/arm64/boot/dts/qcom/glymur-ipcc.h b/arch/arm64/boot/dts/qcom/glymur-ipcc.h new file mode 100644 index 0000000000000..700cd7114909f --- /dev/null +++ b/arch/arm64/boot/dts/qcom/glymur-ipcc.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __DTS_GLYMUR_MAILBOX_IPCC_H +#define __DTS_GLYMUR_MAILBOX_IPCC_H + +/* Glymur physical client IDs */ +#define IPCC_MPROC_AOP 0 +#define IPCC_MPROC_TZ 1 +#define IPCC_MPROC_MPSS 2 +#define IPCC_MPROC_LPASS 3 +#define IPCC_MPROC_SLPI 4 +#define IPCC_MPROC_SDC 5 +#define IPCC_MPROC_CDSP 6 +#define IPCC_MPROC_NPU 7 +#define IPCC_MPROC_APSS 8 +#define IPCC_MPROC_GPU 9 +#define IPCC_MPROC_ICP 11 +#define IPCC_MPROC_VPU 12 +#define IPCC_MPROC_PCIE0 13 +#define IPCC_MPROC_PCIE1 14 +#define IPCC_MPROC_PCIE2 15 +#define IPCC_MPROC_SPSS 16 +#define IPCC_MPROC_PCIE3 19 +#define IPCC_MPROC_PCIE4 20 +#define IPCC_MPROC_PCIE5 21 +#define IPCC_MPROC_PCIE6 22 +#define IPCC_MPROC_TME 23 +#define IPCC_MPROC_WPSS 24 +#define IPCC_MPROC_PCIE7 44 +#define IPCC_MPROC_SOCCP 46 + +#define IPCC_COMPUTE_L0_LPASS 0 +#define IPCC_COMPUTE_L0_CDSP 1 +#define IPCC_COMPUTE_L0_APSS 2 +#define IPCC_COMPUTE_L0_GPU 3 +#define IPCC_COMPUTE_L0_CVP 6 +#define IPCC_COMPUTE_L0_ICP 7 +#define IPCC_COMPUTE_L0_VPU 8 +#define IPCC_COMPUTE_L0_DPU 9 +#define IPCC_COMPUTE_L0_SOCCP 11 + +#define IPCC_COMPUTE_L1_LPASS 0 +#define IPCC_COMPUTE_L1_CDSP 1 +#define IPCC_COMPUTE_L1_APSS 2 +#define IPCC_COMPUTE_L1_GPU 3 +#define IPCC_COMPUTE_L1_CVP 6 +#define IPCC_COMPUTE_L1_ICP 7 +#define IPCC_COMPUTE_L1_VPU 8 +#define IPCC_COMPUTE_L1_DPU 9 +#define IPCC_COMPUTE_L1_SOCCP 11 + +#define IPCC_PERIPH_LPASS 0 +#define IPCC_PERIPH_APSS 1 +#define IPCC_PERIPH_PCIE0 2 +#define IPCC_PERIPH_PCIE1 3 +#define IPCC_PERIPH_PCIE2 6 +#define IPCC_PERIPH_PCIE3 7 +#define IPCC_PERIPH_PCIE4 8 +#define IPCC_PERIPH_PCIE5 9 +#define IPCC_PERIPH_PCIE6 10 +#define IPCC_PERIPH_PCIE7 11 +#define IPCC_PERIPH_SOCCP 13 +#define IPCC_PERIPH_WPSS 16 + +#endif diff --git a/arch/arm64/boot/dts/qcom/glymur-pmics.dtsi b/arch/arm64/boot/dts/qcom/glymur-pmics.dtsi new file mode 100644 index 0000000000000..7f9080b2a86ce --- /dev/null +++ b/arch/arm64/boot/dts/qcom/glymur-pmics.dtsi @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#define PMH0110_F_E0 0x5 +#define PMH0110_H_E0 0x7 +#define PMH0104_I_E0 0x8 +#define PMH0104_J_E0 0x9 + +#define PMH0110_F_E1 0x5 +#define PMH0104_L_E1 0xb + +#include "pmk8850.dtsi" /* SPMI0: SID-0*/ +#include "pmh0101.dtsi" /* SPMI0: SID-1*/ +#include "pmcx0102.dtsi" /* SPMI0: SID-2/3/4/6 SPMI1: SID-2/3/4 */ +#include "pmh0110.dtsi" /* SPMI0: SID-5/7 SPMI1: SID-5 */ +#include "pmh0104.dtsi" /* SPMI0: SID-8/9 SPMI1: SID-11 */ diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi new file mode 100644 index 0000000000000..fbdf479f753a9 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/glymur.dtsi @@ -0,0 +1,4591 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/ { + interrupt-parent = <&intc>; + #address-cells = <2>; + #size-cells = <2>; + + chosen { }; + + clocks { + xo_board: xo-board { + compatible = "fixed-clock"; + clock-frequency = <38400000>; + #clock-cells = <0>; + }; + + sleep_clk: sleep-clk { + compatible = "fixed-clock"; + clock-frequency = <32000>; + #clock-cells = <0>; + }; + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu0: cpu0@0 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x0>; + enable-method = "psci"; + power-domains = <&CPU_PD0>, <&scmi_perf 0>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER0_C4>; + next-level-cache = <&l2_0>; + + l2_0: l2-cache { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + }; + + cpu1: cpu1@100 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x100>; + enable-method = "psci"; + power-domains = <&CPU_PD1>, <&scmi_perf 0>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER0_C4>; + next-level-cache = <&l2_0>; + }; + + cpu2: cpu2@200 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x200>; + enable-method = "psci"; + power-domains = <&CPU_PD2>, <&scmi_perf 0>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER0_C4>; + next-level-cache = <&l2_0>; + }; + + cpu3: cpu3@300 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x300>; + enable-method = "psci"; + power-domains = <&CPU_PD3>, <&scmi_perf 0>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER0_C4>; + next-level-cache = <&l2_0>; + }; + + cpu4: cpu4@400 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x400>; + enable-method = "psci"; + power-domains = <&CPU_PD4>, <&scmi_perf 0>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER0_C4>; + next-level-cache = <&l2_0>; + }; + + cpu5: cpu5@500 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x500>; + enable-method = "psci"; + power-domains = <&CPU_PD5>, <&scmi_perf 0>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER0_C4>; + next-level-cache = <&l2_0>; + }; + + cpu6: cpu6@10000 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10000>; + enable-method = "psci"; + power-domains = <&CPU_PD6>, <&scmi_perf 1>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER1_C4>; + next-level-cache = <&l2_1>; + + l2_1: l2-cache { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + }; + + cpu7: cpu7@10100 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10100>; + enable-method = "psci"; + power-domains = <&CPU_PD7>, <&scmi_perf 1>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER1_C4>; + next-level-cache = <&l2_1>; + }; + + cpu8: cpu8@10200 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10200>; + enable-method = "psci"; + power-domains = <&CPU_PD8>, <&scmi_perf 1>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER1_C4>; + next-level-cache = <&l2_1>; + }; + + cpu9: cpu9@10300 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10300>; + enable-method = "psci"; + power-domains = <&CPU_PD9>, <&scmi_perf 1>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER1_C4>; + next-level-cache = <&l2_1>; + }; + + cpu10: cpu10@10400 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10400>; + enable-method = "psci"; + power-domains = <&CPU_PD10>, <&scmi_perf 1>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER1_C4>; + next-level-cache = <&l2_1>; + }; + + cpu11: cpu11@10500 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10500>; + enable-method = "psci"; + power-domains = <&CPU_PD11>, <&scmi_perf 1>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER1_C4>; + next-level-cache = <&l2_1>; + }; + + cpu12: cpu12@20000 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x20000>; + enable-method = "psci"; + power-domains = <&CPU_PD12>, <&scmi_perf 2>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER2_C4>; + next-level-cache = <&l2_2>; + + l2_2: l2-cache { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + }; + + cpu13: cpu13@20100 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x20100>; + enable-method = "psci"; + power-domains = <&CPU_PD13>, <&scmi_perf 2>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER2_C4>; + next-level-cache = <&l2_2>; + }; + + cpu14: cpu14@20200 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x20200>; + enable-method = "psci"; + power-domains = <&CPU_PD14>, <&scmi_perf 2>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER2_C4>; + next-level-cache = <&l2_2>; + }; + + cpu15: cpu15@20300 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x20300>; + enable-method = "psci"; + power-domains = <&CPU_PD15>, <&scmi_perf 2>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER2_C4>; + next-level-cache = <&l2_2>; + }; + + cpu16: cpu16@20400 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x20400>; + enable-method = "psci"; + power-domains = <&CPU_PD16>, <&scmi_perf 2>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER2_C4>; + next-level-cache = <&l2_2>; + }; + + cpu17: cpu17@20500 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x20500>; + enable-method = "psci"; + power-domains = <&CPU_PD17>, <&scmi_perf 2>; + power-domain-names = "psci", "perf"; + cpu-idle-states = <&CLUSTER2_C4>; + next-level-cache = <&l2_2>; + }; + + idle-states { + entry-method = "psci"; + + CLUSTER0_C4: cpu-sleep-0 { + compatible = "arm,idle-state"; + idle-state-name = "ret"; + arm,psci-suspend-param = <0x00000004>; + entry-latency-us = <180>; + exit-latency-us = <320>; + min-residency-us = <1000>; + }; + + CLUSTER1_C4: cpu-sleep-1 { + compatible = "arm,idle-state"; + idle-state-name = "ret"; + arm,psci-suspend-param = <0x00000004>; + entry-latency-us = <180>; + exit-latency-us = <320>; + min-residency-us = <1000>; + }; + + CLUSTER2_C4: cpu-sleep-2 { + compatible = "arm,idle-state"; + idle-state-name = "ret"; + arm,psci-suspend-param = <0x00000004>; + entry-latency-us = <180>; + exit-latency-us = <320>; + min-residency-us = <1000>; + }; + + cluster0_cl5: cluster-sleep-0 { + compatible = "domain-idle-state"; + idle-state-name = "ret"; + arm,psci-suspend-param = <0x01000054>; + entry-latency-us = <2000>; + exit-latency-us = <2000>; + min-residency-us = <9000>; + }; + + cluster1_cl5: cluster-sleep-1 { + compatible = "domain-idle-state"; + idle-state-name = "ret"; + arm,psci-suspend-param = <0x01000054>; + entry-latency-us = <2000>; + exit-latency-us = <2000>; + min-residency-us = <9000>; + }; + + cluster2_cl5: cluster-sleep-2 { + compatible = "domain-idle-state"; + idle-state-name = "ret"; + arm,psci-suspend-param = <0x01000054>; + entry-latency-us = <2000>; + exit-latency-us = <2000>; + min-residency-us = <9000>; + }; + + APSS_OFF: cluster-ss3 { + compatible = "domain-idle-state"; + idle-state-name = "apps-pc"; + entry-latency-us = <2800>; + exit-latency-us = <4400>; + min-residency-us = <10150>; + arm,psci-suspend-param = <0x0200C354>; + }; + }; + }; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + core1 { + cpu = <&cpu1>; + }; + core2 { + cpu = <&cpu2>; + }; + core3 { + cpu = <&cpu3>; + }; + core4 { + cpu = <&cpu4>; + }; + core5 { + cpu = <&cpu5>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu6>; + }; + core1 { + cpu = <&cpu7>; + }; + core2 { + cpu = <&cpu8>; + }; + core3 { + cpu = <&cpu9>; + }; + core4 { + cpu = <&cpu10>; + }; + core5 { + cpu = <&cpu11>; + }; + }; + + cluster2 { + core0 { + cpu = <&cpu12>; + }; + core1 { + cpu = <&cpu13>; + }; + core2 { + cpu = <&cpu14>; + }; + core3 { + cpu = <&cpu15>; + }; + core4 { + cpu = <&cpu16>; + }; + core5 { + cpu = <&cpu17>; + }; + }; + }; + + firmware { + scm: scm { + compatible = "qcom,scm-glymur", "qcom,scm"; + qcom,dload-mode = <&tcsr 0x3000>; + interconnects = <&aggre2_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + }; + + scmi { + compatible = "arm,scmi"; + mboxes = <&pdp0_mbox 0>, <&pdp0_mbox 1>; + mbox-names = "tx", "rx"; + shmem = <&cpu_scp_lpri1>, <&cpu_scp_lpri0>; + + #address-cells = <1>; + #size-cells = <0>; + + scmi_perf: protocol@13 { + reg = <0x13>; + #power-domain-cells = <1>; + }; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + llcc_lpi_mem: llcc-lpi-region@80000000 { + reg = <0x0 0x80000000 0x0 0x600000>; + no-map; + }; + + llcc_lpi_reserved_mem: llcc-lpi-reserved-region@80900000 { + reg = <0x0 0x80900000 0x0 0x500000>; + no-map; + }; + + cpucp_log_mem: cpucp-log-region@80e00000 { + reg = <0x0 0x80e00000 0x0 0x40000>; + no-map; + }; + + cpucp_fw_mem: cpucp-fw-region@80e40000 { + reg = <0x0 0x80e40000 0x0 0x5c0000>; + no-map; + }; + + pdp_mem: pdp-region@81400000 { + reg = <0x0 0x81400000 0x0 0x100000>; + no-map; + }; + + tags_mem: tags-region@81500000 { + reg = <0x0 0x81500000 0x0 0x360000>; + no-map; + }; + + xbl_dtlog_mem: xbl-dtlog-region@81a00000 { + reg = <0x0 0x81a00000 0x0 0x40000>; + no-map; + }; + + xbl_ramdump_mem: xbl-ramdump-region@81a40000 { + reg = <0x0 0x81a40000 0x0 0x1c0000>; + no-map; + }; + + aop_image_mem: aop-image-region@81c00000 { + reg = <0x0 0x81c00000 0x0 0x60000>; + no-map; + }; + + aop_cmd_db_mem: aop-cmd-db-region@81c60000 { + compatible = "qcom,cmd-db"; + reg = <0x0 0x81c60000 0x0 0x20000>; + no-map; + }; + + aop_config_mem: aop-config-region@81c80000 { + reg = <0x0 0x81c80000 0x0 0x20000>; + no-map; + }; + + tme_crash_dump_mem: tme-crash-dump-region@81ca0000 { + reg = <0x0 0x81ca0000 0x0 0x40000>; + no-map; + }; + + tme_log_mem: tme-log-region@81ce0000 { + reg = <0x0 0x81ce0000 0x0 0x10000>; + no-map; + }; + + gpu_prr_mem: gpu-prr-region@81d00000 { + reg = <0x0 0x81d00000 0x0 0x10000>; + no-map; + }; + + tpm_control_mem: tpm-control-region@81d10000 { + reg = <0x0 0x81d10000 0x0 0x10000>; + no-map; + }; + + usb_ucsi_shared_mem: usb-ucsi-shared-region@81d20000 { + reg = <0x0 0x81d20000 0x0 0x10000>; + no-map; + }; + + pld_pep_mem: pld-pep-region@81d30000 { + reg = <0x0 0x81d30000 0x0 0x10000>; + no-map; + }; + + pld_gmu_mem: pld-gmu-region@81d40000 { + reg = <0x0 0x81d40000 0x0 0x10000>; + no-map; + }; + + pld_pdp_mem: pld-pdp-region@81d50000 { + reg = <0x0 0x81d50000 0x0 0x10000>; + no-map; + }; + + secdata_apss_mem: secdata-apss-region@81d60000 { + reg = <0x0 0x81d60000 0x0 0x10000>; + no-map; + }; + + qcskext_mem: qcskext-region@81d70000 { + reg = <0x0 0x81d70000 0x0 0x20000>; + no-map; + }; + + qup_fw_mem: qup-fw-region@81d90000 { + reg = <0x0 0x81d90000 0x0 0x18000>; + no-map; + }; + + softsku_mem: softsku-region@81da8000 { + reg = <0x0 0x81da8000 0x0 0x9000>; + no-map; + }; + + pdp_ns_shared_mem: pdp-ns-shared-region@81e00000 { + reg = <0x0 0x81e00000 0x0 0x200000>; + no-map; + }; + + acpi_ta_smem_mem: acpi-ta-smem-region@82000000 { + reg = <0x0 0x82000000 0x0 0x200000>; + no-map; + }; + + soccp_sdi_mem: soccp-sdi-region@823a0000 { + reg = <0x0 0x823a0000 0x0 0x40000>; + no-map; + }; + + lpm_violatordata_smem_mem: lpm-violatordata-smem-region@823e0000 { + reg = <0x0 0x823e0000 0x0 0x100000>; + no-map; + }; + + tz_stat_mem: tz-stat-region@82700000 { + reg = <0x0 0x82700000 0x0 0x100000>; + no-map; + }; + + no_reclaim_xbl_scratch_buffer2_mem: no-reclaim-xbl-scratch-buffer2-region@82800000 { + reg = <0x0 0x82800000 0x0 0x1c0000>; + no-map; + }; + + reclaimable_xbl_scratch_buffer_mem: reclaimable-xbl-scratch-buffer-region@829c0000 { + reg = <0x0 0x829c0000 0x0 0x1540000>; + no-map; + }; + + no_reclaim_xbl_scratch_buffer1_mem: no-reclaim-xbl-scratch-buffer1-region@83f00000 { + reg = <0x0 0x83f00000 0x0 0x100000>; + no-map; + }; + + ncc_mem: ncc-region@84000000 { + reg = <0x0 0x84000000 0x0 0x400000>; + no-map; + }; + + spu_secure_shared_memory_mem: spu-secure-shared-memory-region@84900000 { + reg = <0x0 0x84900000 0x0 0x100000>; + no-map; + }; + + adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap-region@84a00000 { + reg = <0x0 0x84a00000 0x0 0x900000>; + no-map; + }; + + smmu_pt_mem: smmu-pt-region@85370000 { + reg = <0x0 0x85370000 0x0 0xa20000>; + no-map; + }; + + oobdaretag_mem: oobdaretag-region@86e10000 { + reg = <0x0 0x86e10000 0x0 0x360000>; + no-map; + }; + + oob_secure_mem: oob-secure-region@87170000 { + reg = <0x0 0x87170000 0x0 0xbc0000>; + no-map; + }; + + oobdtbqc_mem: oobdtbqc-region@87d30000 { + reg = <0x0 0x87d30000 0x0 0x20000>; + no-map; + }; + + oobdtboem_mem: oobdtboem-region@87d50000 { + reg = <0x0 0x87d50000 0x0 0x20000>; + no-map; + }; + + oob_nonsecure_mem: oob-nonsecure-region@87e00000 { + reg = <0x0 0x87e00000 0x0 0xc00000>; + no-map; + }; + + spss_region_mem: spss-region-region@88a00000 { + reg = <0x0 0x88a00000 0x0 0x400000>; + no-map; + }; + + soccp_boot_mem: soccp-boot-region@88e00000 { + reg = <0x0 0x88e00000 0x0 0x400000>; + no-map; + }; + + soccp_boot_dtb_mem: soccp-boot-dtb-region@89200000 { + reg = <0x0 0x89200000 0x0 0x20000>; + no-map; + }; + + soccpdtb_mem: soccpdtb-region@892e0000 { + reg = <0x0 0x892e0000 0x0 0x20000>; + no-map; + }; + + soccp_mem: soccp-region@89300000 { + reg = <0x0 0x89300000 0x0 0x400000>; + no-map; + }; + + cvp_mem: cvp-region@89700000 { + reg = <0x0 0x89700000 0x0 0x700000>; + no-map; + }; + + adspslpi_mem: adspslpi-region@89e00000 { + reg = <0x0 0x89e00000 0x0 0x3a00000>; + no-map; + }; + + q6_adsp_dtb_mem: q6-adsp-dtb-region@8d800000 { + reg = <0x0 0x8d800000 0x0 0x80000>; + no-map; + }; + + cdsp_mem: cdsp-region@8d900000 { + reg = <0x0 0x8d900000 0x0 0x4000000>; + no-map; + }; + + q6_cdsp_dtb_mem: q6-cdsp-dtb-region@91900000 { + reg = <0x0 0x91900000 0x0 0x80000>; + no-map; + }; + + gpu_microcode_mem: gpu-microcode-region@919fe000 { + reg = <0x0 0x919fe000 0x0 0x2000>; + no-map; + }; + + camera_mem: camera-region@91a00000 { + reg = <0x0 0x91a00000 0x0 0x800000>; + no-map; + }; + + av1_encoder_mem: av1-encoder-region@92200000 { + reg = <0x0 0x92200000 0x0 0x700000>; + no-map; + }; + + video_mem: video-region@92900000 { + reg = <0x0 0x92900000 0x0 0xc00000>; + no-map; + }; + + reserved_mem: reserved-region@93500000 { + reg = <0x0 0x93500000 0x0 0xa00000>; + no-map; + }; + + oob_cached_heap_standalone_mem: oob-cached-heap-standalone-region@a2000000 { + reg = <0x0 0xa2000000 0x0 0x5000000>; + no-map; + }; + + bert_mem: bert-region@d6f00000 { + reg = <0x0 0xd6f00000 0x0 0x100000>; + no-map; + }; + + hyp_ac_config_mem: hyp-ac-config-region@d7000000 { + reg = <0x0 0xd7000000 0x0 0x19000>; + no-map; + }; + + xbl_sc_mem: xbl-sc-region@d7019000 { + reg = <0x0 0xd7019000 0x0 0x60000>; + no-map; + }; + + xbl_sc_reserved_mem: xbl-sc-reserved-region@d7079000 { + reg = <0x0 0xd7079000 0x0 0x107000>; + no-map; + }; + + qtee_mem: qtee-region@d7180000 { + reg = <0x0 0xd7180000 0x0 0x680000>; + no-map; + }; + + ta_mem: ta-region@d7800000 { + reg = <0x0 0xd7800000 0x0 0x1c00000>; + no-map; + }; + + tags_mem_2: tags-region-2@d9400000 { + reg = <0x0 0xd9400000 0x0 0x9c0000>; + no-map; + }; + + display_buffer_mem: display-buffer-region@f9a20000 { + reg = <0x0 0xf9a20000 0x0 0x2300000>; + no-map; + }; + + oob_nc_mdm_standalone_mode_mem: oob-nc-mdm-standalone-mode-region@fc000000 { + reg = <0x0 0xfc000000 0x0 0x2000000>; + no-map; + }; + + oob_nc_wlan_standalone_mode_mem: oob-nc-wlan-standalone-mode-region@fe000000 { + reg = <0x0 0xfe000000 0x0 0xc00000>; + no-map; + }; + + oob_nc_glink_always_mem: oob-nc-glink-always-region@fec00000 { + reg = <0x0 0xfec00000 0x0 0x180000>; + no-map; + }; + + oob_nc_glink_assisted_mode_mem: oob-nc-glink-assisted-mode-region@fed80000 { + reg = <0x0 0xfed80000 0x0 0x140000>; + no-map; + }; + + oob_nc_mdm_assisted_mode_mem: oob-nc-mdm-assisted-mode-region@feec0000 { + reg = <0x0 0xfeec0000 0x0 0x10000>; + no-map; + }; + + oob_nc_wlan_assisted_mode_mem: oob-nc-wlan-assisted-mode-region@feed0000 { + reg = <0x0 0xfeed0000 0x0 0x10000>; + no-map; + }; + + smem_mem: smem-region@ffe00000 { + compatible = "qcom,smem"; + reg = <0x0 0xffe00000 0x0 0x200000>; + hwlocks = <&tcsr_mutex 3>; + no-map; + }; + + }; + + clk_virt: interconnect-0 { + compatible = "qcom,glymur-clk-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + mc_virt: interconnect-1 { + compatible = "qcom,glymur-mc-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + memory { + device_type = "memory"; + /* We expect the bootloader to fill in the size */ + reg = <0 0 0 0>; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = ; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + + CPU_PD0: power-domain-cpu0 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER0_PD>; + }; + + CPU_PD1: power-domain-cpu1 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER0_PD>; + }; + + CPU_PD2: power-domain-cpu2 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER0_PD>; + }; + + CPU_PD3: power-domain-cpu3 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER0_PD>; + }; + + CPU_PD4: power-domain-cpu4 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER0_PD>; + }; + + CPU_PD5: power-domain-cpu5 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER0_PD>; + }; + + CPU_PD6: power-domain-cpu6 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER1_PD>; + }; + + CPU_PD7: power-domain-cpu7 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER1_PD>; + }; + + CPU_PD8: power-domain-cpu8 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER1_PD>; + }; + + CPU_PD9: power-domain-cpu9 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER1_PD>; + }; + + CPU_PD10: power-domain-cpu10 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER1_PD>; + }; + + CPU_PD11: power-domain-cpu11 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER1_PD>; + }; + + CPU_PD12: power-domain-cpu12 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER2_PD>; + }; + + CPU_PD13: power-domain-cpu13 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER2_PD>; + }; + + CPU_PD14: power-domain-cpu14 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER2_PD>; + }; + + CPU_PD15: power-domain-cpu15 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER2_PD>; + }; + + CPU_PD16: power-domain-cpu16 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER2_PD>; + }; + + CPU_PD17: power-domain-cpu17 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER2_PD>; + }; + + CLUSTER0_PD: power-domain-cpu-cluster0 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER3_PD>; + domain-idle-states = <&cluster0_cl5>; + }; + + CLUSTER1_PD: power-domain-cpu-cluster1 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER3_PD>; + domain-idle-states = <&cluster1_cl5>; + }; + + CLUSTER2_PD: power-domain-cpu-cluster2 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER3_PD>; + domain-idle-states = <&cluster2_cl5>; + }; + + CLUSTER3_PD: power-domain-cpu-cluster3 { + #power-domain-cells = <0>; + domain-idle-states = <&APSS_OFF>; + }; + }; + + soc: soc@0 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges = <0 0 0 0 0x10 0>; + dma-ranges = <0 0 0 0 0x10 0>; + + gcc: clock-controller@100000 { + compatible = "qcom,glymur-gcc"; + reg = <0 0x100000 0 0x1f9000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&sleep_clk>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <&pcie5_phy>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; + + gpi_dma2: dma-controller@800000 { + compatible = "qcom,glymur-gpi-dma", "qcom,sm6350-gpi-dma"; + reg = <0 0x00800000 0 0x60000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + dma-channels = <16>; + dma-channel-mask = <0x3f>; + #dma-cells = <3>; + iommus = <&apps_smmu 0xd76 0x0>; + status = "ok"; + }; + + qupv3_2: geniqup@8c0000 { + compatible = "qcom,geni-se-qup"; + reg = <0x0 0x008c0000 0x0 0x3000>; + clocks = <&gcc GCC_QUPV3_WRAP_2_M_AHB_CLK>, + <&gcc GCC_QUPV3_WRAP_2_S_AHB_CLK>; + clock-names = "m-ahb", + "s-ahb"; + iommus = <&apps_smmu 0xd63 0x0>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "ok"; + + i2c16: i2c@880000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00880000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 0 QCOM_GPI_I2C>, + <&gpi_dma2 1 0 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c16_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi16: spi@880000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00880000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 0 QCOM_GPI_SPI>, + <&gpi_dma2 1 0 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi16_data_clk>, <&qup_spi16_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c17: i2c@884000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00884000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 1 QCOM_GPI_I2C>, + <&gpi_dma2 1 1 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c17_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi17: spi@884000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00884000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 1 QCOM_GPI_SPI>, + <&gpi_dma2 1 1 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi17_data_clk>, <&qup_spi17_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c18: i2c@888000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00888000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 2 QCOM_GPI_I2C>, + <&gpi_dma2 1 2 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c18_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi18: spi@888000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00888000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 2 QCOM_GPI_SPI>, + <&gpi_dma2 1 2 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi18_data_clk>, <&qup_spi18_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c19: i2c@88c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x0088c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 3 QCOM_GPI_I2C>, + <&gpi_dma2 1 3 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c19_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi19: spi@88c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x0088c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 3 QCOM_GPI_SPI>, + <&gpi_dma2 1 3 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi19_data_clk>, <&qup_spi19_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart19: serial@88c000 { + compatible = "qcom,geni-uart"; + reg = <0 0x0088c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + pinctrl-0 = <&qup_uart19_default>; + pinctrl-names = "default"; + status = "disabled"; + }; + + i2c20: i2c@890000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00890000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 4 QCOM_GPI_I2C>, + <&gpi_dma2 1 4 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c20_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi20: spi@890000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00890000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 4 QCOM_GPI_SPI>, + <&gpi_dma2 1 4 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi20_data_clk>, <&qup_spi20_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c21: i2c@894000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00894000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 5 QCOM_GPI_I2C>, + <&gpi_dma2 1 5 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c21_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi21: spi@894000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00894000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 5 QCOM_GPI_SPI>, + <&gpi_dma2 1 5 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi21_data_clk>, <&qup_spi21_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart21: serial@894000 { + compatible = "qcom,geni-debug-uart"; + reg = <0x0 0x00894000 0x0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config"; + pinctrl-0 = <&qup_uart21_default>; + pinctrl-names = "default"; + }; + + i2c22: i2c@898000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00898000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 6 QCOM_GPI_I2C>, + <&gpi_dma2 1 6 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c22_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi22: spi@898000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00898000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 6 QCOM_GPI_SPI>, + <&gpi_dma2 1 6 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi22_data_clk>, <&qup_spi22_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart22: serial@898000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00898000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config"; + pinctrl-0 = <&qup_uart22_default>; + pinctrl-names = "default"; + status = "disabled"; + }; + + i2c23: i2c@89c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x0089c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S7_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 7 QCOM_GPI_I2C>, + <&gpi_dma2 1 7 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c23_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi23: spi@89c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x0089c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP2_S7_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma2 0 7 QCOM_GPI_SPI>, + <&gpi_dma2 1 7 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi23_data_clk>, <&qup_spi23_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + }; + + gpi_dma1: dma-controller@a00000 { + compatible = "qcom,glymur-gpi-dma", "qcom,sm6350-gpi-dma"; + reg = <0 0x00a00000 0 0x60000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + dma-channels = <16>; + dma-channel-mask = <0x3f>; + #dma-cells = <3>; + iommus = <&apps_smmu 0xcb6 0x0>; + status = "okay"; + }; + + qupv3_1: geniqup@ac0000 { + compatible = "qcom,geni-se-qup"; + reg = <0 0x00ac0000 0 0x3000>; + clocks = <&gcc GCC_QUPV3_WRAP_1_M_AHB_CLK>, + <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>; + clock-names = "m-ahb", + "s-ahb"; + iommus = <&apps_smmu 0xca3 0x0>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "ok"; + + i2c8: i2c@a80000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a80000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>, + <&gpi_dma1 1 0 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c8_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi8: spi@a80000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a80000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 0 QCOM_GPI_SPI>, + <&gpi_dma1 1 0 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi8_data_clk>, <&qup_spi8_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c9: i2c@a84000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a84000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>, + <&gpi_dma1 1 1 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c9_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi9: spi@a84000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a84000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 1 QCOM_GPI_SPI>, + <&gpi_dma1 1 1 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi9_data_clk>, <&qup_spi9_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c10: i2c@a88000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a88000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>, + <&gpi_dma1 1 2 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c10_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi10: spi@a88000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a88000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 2 QCOM_GPI_SPI>, + <&gpi_dma1 1 2 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi10_data_clk>, <&qup_spi10_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c11: i2c@a8c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a8c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 3 QCOM_GPI_I2C>, + <&gpi_dma1 1 3 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c11_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi11: spi@a8c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a8c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 3 QCOM_GPI_SPI>, + <&gpi_dma1 1 3 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi11_data_clk>, <&qup_spi11_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c12: i2c@a90000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a90000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>, + <&gpi_dma1 1 4 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c12_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi12: spi@a90000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a90000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 4 QCOM_GPI_SPI>, + <&gpi_dma1 1 4 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi12_data_clk>, <&qup_spi12_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c13: i2c@a94000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a94000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 5 QCOM_GPI_I2C>, + <&gpi_dma1 1 5 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c13_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi13: spi@a94000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a94000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 5 QCOM_GPI_SPI>, + <&gpi_dma1 1 5 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi13_data_clk>, <&qup_spi13_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c14: i2c@a98000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a98000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 6 QCOM_GPI_I2C>, + <&gpi_dma1 1 6 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c14_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi14: spi@a98000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a98000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 6 QCOM_GPI_SPI>, + <&gpi_dma1 1 6 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi14_data_clk>, <&qup_spi14_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart14: serial@a98000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a98000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + pinctrl-0 = <&qup_uart14_default>; + pinctrl-names = "default"; + status = "disabled"; + }; + + i2c15: i2c@a9c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a9c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 7 QCOM_GPI_I2C>, + <&gpi_dma1 1 7 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c15_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi15: spi@a9c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a9c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma1 0 7 QCOM_GPI_SPI>, + <&gpi_dma1 1 7 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi15_data_clk>, <&qup_spi15_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + }; + + gpi_dma0: dma-controller@b00000 { + compatible = "qcom,glymur-gpi-dma", "qcom,sm6350-gpi-dma"; + reg = <0 0x00b00000 0 0x60000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + dma-channels = <16>; + dma-channel-mask = <0x3f>; + #dma-cells = <3>; + iommus = <&apps_smmu 0xd36 0x0>; + status = "ok"; + }; + + qupv3_0: geniqup@bc0000 { + compatible = "qcom,geni-se-qup"; + reg = <0 0x00bc0000 0 0x3000>; + clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>, + <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>; + clock-names = "m-ahb", + "s-ahb"; + iommus = <&apps_smmu 0xd23 0x0>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "ok"; + + i2c0: i2c@b80000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b80000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>, + <&gpi_dma0 1 0 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c0_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi0: spi@b80000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b80000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 0 QCOM_GPI_SPI>, + <&gpi_dma0 1 0 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi0_data_clk>, <&qup_spi0_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@b84000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b84000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 1 QCOM_GPI_I2C>, + <&gpi_dma0 1 1 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c1_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi1: spi@b84000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b84000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 1 QCOM_GPI_SPI>, + <&gpi_dma0 1 1 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi1_data_clk>, <&qup_spi1_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@b88000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b88000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 2 QCOM_GPI_I2C>, + <&gpi_dma0 1 2 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c2_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi2: spi@b88000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b88000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 2 QCOM_GPI_SPI>, + <&gpi_dma0 1 2 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi2_data_clk>, <&qup_spi2_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart2: serial@b88000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00b88000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config"; + pinctrl-0 = <&qup_uart2_default>; + pinctrl-names = "default"; + status = "disabled"; + }; + + i2c3: i2c@b8c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b8c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 3 QCOM_GPI_I2C>, + <&gpi_dma0 1 3 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c3_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi3: spi@b8c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b8c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 3 QCOM_GPI_SPI>, + <&gpi_dma0 1 3 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi3_data_clk>, <&qup_spi3_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@b90000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b90000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 4 QCOM_GPI_I2C>, + <&gpi_dma0 1 4 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c4_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi4: spi@b90000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b90000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 4 QCOM_GPI_SPI>, + <&gpi_dma0 1 4 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi4_data_clk>, <&qup_spi4_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c5: i2c@b94000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b94000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 5 QCOM_GPI_I2C>, + <&gpi_dma0 1 5 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c5_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi5: spi@b94000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b94000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 5 QCOM_GPI_SPI>, + <&gpi_dma0 1 5 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi5_data_clk>, <&qup_spi5_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c6: i2c@b98000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b98000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 6 QCOM_GPI_I2C>, + <&gpi_dma0 1 6 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c6_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi6: spi@b98000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b98000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 6 QCOM_GPI_SPI>, + <&gpi_dma0 1 6 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi6_data_clk>, <&qup_spi6_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c7: i2c@b9c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00b9c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 7 QCOM_GPI_I2C>, + <&gpi_dma0 1 7 QCOM_GPI_I2C>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_i2c7_data_clk>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi7: spi@b9c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00b9c000 0 0x4000>; + interrupts = ; + clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; + clock-names = "se"; + interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>, + <&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qup-core", + "qup-config", + "qup-memory"; + dmas = <&gpi_dma0 0 7 QCOM_GPI_SPI>, + <&gpi_dma0 1 7 QCOM_GPI_SPI>; + dma-names = "tx", + "rx"; + pinctrl-0 = <&qup_spi7_data_clk>, <&qup_spi7_cs>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + }; + + cnoc_main: interconnect@1500000 { + compatible = "qcom,glymur-cnoc-main"; + reg = <0x0 0x01500000 0x0 0x17080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + config_noc: interconnect@1600000 { + compatible = "qcom,glymur-cnoc-cfg"; + reg = <0x0 0x01600000 0x0 0x6600>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + system_noc: interconnect@1680000 { + compatible = "qcom,glymur-system-noc"; + reg = <0x0 0x01680000 0x0 0x1c080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + pcie_west_anoc: interconnect@16c0000 { + compatible = "qcom,glymur-pcie-west-anoc"; + reg = <0x0 0x016c0000 0x0 0xf580>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + clocks = <&gcc GCC_AGGRE_NOC_PCIE_3A_WEST_SF_AXI_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_3B_WEST_SF_AXI_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_4_WEST_SF_AXI_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_6_WEST_SF_AXI_CLK>; + }; + + pcie_east_anoc: interconnect@16d0000 { + compatible = "qcom,glymur-pcie-east-anoc"; + reg = <0x0 0x016d0000 0x0 0xf300>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + clocks = <&gcc GCC_AGGRE_NOC_PCIE_5_EAST_SF_AXI_CLK>; + }; + + aggre1_noc: interconnect@16e0000 { + compatible = "qcom,glymur-aggre1-noc"; + reg = <0x0 0x016e0000 0x0 0x14400>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + aggre2_noc: interconnect@1720000 { + compatible = "qcom,glymur-aggre2-noc"; + reg = <0x0 0x01720000 0x0 0x14400>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + clocks = <&gcc GCC_AGGRE_USB3_TERT_AXI_CLK>, + <&gcc GCC_AGGRE_USB4_2_AXI_CLK>, + <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>; + }; + + aggre3_noc: interconnect@1700000 { + compatible = "qcom,glymur-aggre3-noc"; + reg = <0x0 0x01700000 0x0 0x1d400>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + aggre4_noc: interconnect@1740000 { + compatible = "qcom,glymur-aggre4-noc"; + reg = <0x0 0x01740000 0x0 0x14400>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + clocks = <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>, + <&gcc GCC_AGGRE_USB3_SEC_AXI_CLK>, + <&gcc GCC_AGGRE_USB4_0_AXI_CLK>, + <&gcc GCC_AGGRE_USB4_1_AXI_CLK>; + }; + + mmss_noc: interconnect@1780000 { + compatible = "qcom,glymur-mmss-noc"; + reg = <0x0 0x01780000 0x0 0x5b800>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + pcie_east_slv_noc: interconnect@1900000 { + compatible = "qcom,glymur-pcie-east-slv-noc"; + reg = <0x0 0x01900000 0x0 0xe080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + pcie_west_slv_noc: interconnect@1920000 { + compatible = "qcom,glymur-pcie-west-slv-noc"; + reg = <0x0 0x01920000 0x0 0xf180>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + pcie5: pci@1b40000 { + device_type = "pci"; + compatible = "qcom,glymur-pcie", "qcom,pcie-x1e80100"; + reg = <0x0 0x01b40000 0x0 0x3000>, + <0x7 0xa0000000 0x0 0xf20>, + <0x7 0xa0000f40 0x0 0xa8>, + <0x7 0xb0000000 0x0 0x4000>, + <0x7 0xa0100000 0x0 0x100000>, + <0x0 0x01b43000 0x0 0x1000>; + reg-names = "parf", + "dbi", + "elbi", + "atu", + "config", + "mhi"; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x02000000 0 0x7a000000 0 0x7a000000 0 0x4000000>; + bus-range = <0 0xff>; + + dma-coherent; + + linux,pci-domain = <5>; + num-lanes = <4>; + max-link-speed = <5>; + + operating-points-v2 = <&pcie5_opp_table>; + + msi-map = <0x0 &gic_its 0xd0000 0x10000>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc 0 0 0 526 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc 0 0 0 428 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc 0 0 0 429 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc 0 0 0 435 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_5_AUX_CLK>, + <&gcc GCC_PCIE_5_CFG_AHB_CLK>, + <&gcc GCC_PCIE_5_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_5_SLV_AXI_CLK>, + <&gcc GCC_PCIE_5_SLV_Q2A_AXI_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_5_EAST_SF_AXI_CLK>; + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a", + "noc_aggr"; + + assigned-clocks = <&gcc GCC_PCIE_5_AUX_CLK>; + assigned-clock-rates = <19200000>; + + interconnects = <&pcie_east_anoc MASTER_PCIE_5 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &pcie_east_slv_noc SLAVE_PCIE_5 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "pcie-mem", + "cpu-pcie"; + + resets = <&gcc GCC_PCIE_5_BCR>, + <&gcc GCC_PCIE_5_LINK_DOWN_BCR>; + reset-names = "pci", + "link_down"; + + power-domains = <&gcc GCC_PCIE_5_GDSC>; + + eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>; + eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>; + eq-presets-32gts = /bits/ 8 <0x55 0x55 0x55 0x55>; + + status = "disabled"; + + pcie5_opp_table: opp-table { + compatible = "operating-points-v2"; + + /* GEN 1 x1 */ + opp-2500000 { + opp-hz = /bits/ 64 <2500000>; + required-opps = <&rpmhpd_opp_low_svs>; + opp-peak-kBps = <250000 1>; + }; + + /* GEN 1 x2 and GEN 2 x1 */ + opp-5000000 { + opp-hz = /bits/ 64 <5000000>; + required-opps = <&rpmhpd_opp_low_svs>; + opp-peak-kBps = <500000 1>; + }; + + /* GEN 1 x4 and GEN 2 x2 */ + opp-10000000 { + opp-hz = /bits/ 64 <10000000>; + required-opps = <&rpmhpd_opp_low_svs>; + opp-peak-kBps = <1000000 1>; + }; + + /* GEN 2 x4 */ + opp-20000000 { + opp-hz = /bits/ 64 <20000000>; + required-opps = <&rpmhpd_opp_low_svs>; + opp-peak-kBps = <2000000 1>; + }; + + /* GEN 3 x1 */ + opp-8000000 { + opp-hz = /bits/ 64 <8000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <984500 1>; + }; + + /* GEN 3 x2 and GEN 4 x1 */ + opp-16000000 { + opp-hz = /bits/ 64 <16000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <1969000 1>; + }; + + /* GEN 3 x4, GEN 4 x2 and GEN5 x1*/ + opp-32000000 { + opp-hz = /bits/ 64 <32000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <3938000 1>; + }; + + /* GEN 4 x4 and GEN 5 x2 */ + opp-64000000 { + opp-hz = /bits/ 64 <64000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <7876000 1>; + }; + + /* GEN 5 x4 */ + opp-128000000 { + opp-hz = /bits/ 64 <128000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <15753000 1>; + }; + }; + + pcie5port0: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + phys = <&pcie5_phy>; + }; + }; + + pcie5_phy: phy@1b50000 { + compatible = "qcom,glymur-qmp-gen5x4-pcie-phy"; + reg = <0x0 0x01b50000 0x0 0x10000>; + + clocks = <&gcc GCC_PCIE_PHY_5_AUX_CLK>, + <&gcc GCC_PCIE_5_CFG_AHB_CLK>, + <&tcsrcc TCSR_PCIE_1_CLKREF_EN>, + <&gcc GCC_PCIE_5_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_5_PIPE_CLK>, + <&gcc GCC_PCIE_5_PIPE_DIV2_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe", + "pipediv2"; + + resets = <&gcc GCC_PCIE_5_PHY_BCR>, + <&gcc GCC_PCIE_5_NOCSR_COM_PHY_BCR>; + reset-names = "phy", + "phy_nocsr"; + + assigned-clocks = <&gcc GCC_PCIE_5_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + power-domains = <&gcc GCC_PCIE_5_PHY_GDSC>; + + #clock-cells = <0>; + clock-output-names = "pcie5_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + + tcsr_mutex: hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0x0 0x01f40000 0x0 0x20000>; + + #hwlock-cells = <1>; + }; + + tcsrcc: clock-controller@1fd5044 { + compatible = "qcom,glymur-tcsr"; + reg = <0x0 0x1fd5044 0x0 0x48>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + tcsr: syscon@1fd6000 { + compatible = "syscon"; + reg = <0x0 0x1fd6000 0x0 0x20000>; + }; + + hsc_noc: interconnect@2000000 { + compatible = "qcom,glymur-hscnoc"; + reg = <0x0 0x02000000 0x0 0x93a080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + lpass_lpiaon_noc: interconnect@7400000 { + compatible = "qcom,glymur-lpass-lpiaon-noc"; + reg = <0x0 0x07400000 0x0 0x19080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + lpass_lpicx_noc: interconnect@7420000 { + compatible = "qcom,glymur-lpass-lpicx-noc"; + reg = <0x0 0x07420000 0x0 0x44080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + lpass_ag_noc: interconnect@7e40000 { + compatible = "qcom,glymur-lpass-ag-noc"; + reg = <0x0 0x07e40000 0x0 0xe080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + pdc: interrupt-controller@b220000 { + compatible = "qcom,glymur-pdc", "qcom,pdc"; + reg = <0x0 0x0b220000 0x0 0x10000>; + qcom,pdc-ranges = <0 745 51>, + <51 527 47>, + <98 609 32>, + <130 717 12>, + <142 251 5>, + <147 796 16>, + <171 4104 36>; + #interrupt-cells = <2>; + interrupt-parent = <&intc>; + interrupt-controller; + }; + + sram@c30f000 { + compatible = "qcom,rpmh-stats"; + reg = <0x0 0x0c30f000 0x0 0x400>; + }; + + pmic_arbiter: arbiter@c400000 { + compatible = "qcom,glymur-spmi-pmic-arb"; + reg = <0x0 0x0c400000 0x0 0x00003000>, + <0x0 0x0c900000 0x0 0x00400000>, + <0x0 0x0c4c0000 0x0 0x00400000>, + <0x0 0x0c403000 0x0 0x00008000>; + reg-names = "core", + "chnls", + "obsrvr", + "chnl_map"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + qcom,channel = <0>; + qcom,ee = <0>; + + spmi_bus0: spmi@c426000 { + reg = <0x0 0x0c426000 0x0 0x00004000>, + <0x0 0x0c8c0000 0x0 0x00010000>, + <0x0 0x0c42a000 0x0 0x00008000>; + reg-names = "cnfg", + "intr", + "chnl_owner"; + interrupts-extended = <&pdc 1 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "periph_irq"; + interrupt-controller; + #interrupt-cells = <4>; + #address-cells = <2>; + #size-cells = <0>; + }; + + spmi_bus1: spmi@c437000 { + reg = <0x0 0x0c437000 0x0 0x00004000>, + <0x0 0x0c8d0000 0x0 0x00010000>, + <0x0 0x0c43b000 0x0 0x00008000>; + reg-names = "cnfg", + "intr", + "chnl_owner"; + interrupts-extended = <&pdc 3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "periph_irq"; + interrupt-controller; + #interrupt-cells = <4>; + #address-cells = <2>; + #size-cells = <0>; + }; + + spmi_bus2: spmi@c48000 { + reg = <0x0 0x0c448000 0x0 0x00004000>, + <0x0 0x0c8e0000 0x0 0x00010000>, + <0x0 0x0c44c000 0x0 0x00008000>; + reg-names = "cnfg", + "intr", + "chnl_owner"; + interrupts-extended = <&pdc 72 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "periph_irq"; + interrupt-controller; + #interrupt-cells = <4>; + #address-cells = <2>; + #size-cells = <0>; + }; + }; + + tlmm: pinctrl@f100000 { + compatible = "qcom,glymur-tlmm"; + reg = <0x0 0x0f100000 0x0 0xf00000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 249>; + wakeup-parent = <&pdc>; + + qup_i2c0_data_clk: qup-i2c0-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio0"; + function = "qup0_se0"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio1"; + function = "qup0_se0"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c1_data_clk: qup-i2c1-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio4"; + function = "qup0_se1"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio5"; + function = "qup0_se1"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c2_data_clk: qup-i2c2-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio8"; + function = "qup0_se2"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio9"; + function = "qup0_se2"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c3_data_clk: qup-i2c3-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio12"; + function = "qup0_se3"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio13"; + function = "qup0_se3"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c4_data_clk: qup-i2c4-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio16"; + function = "qup0_se4"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio17"; + function = "qup0_se4"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c5_data_clk: qup-i2c5-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio20"; + function = "qup0_se5"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio21"; + function = "qup0_se5"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c6_data_clk: qup-i2c6-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio6"; + function = "qup0_se6"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio7"; + function = "qup0_se6"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c7_data_clk: qup-i2c7-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio14"; + function = "qup0_se7"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio15"; + function = "qup0_se7"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c8_data_clk: qup-i2c8-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio32"; + function = "qup1_se0"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio33"; + function = "qup1_se0"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c9_data_clk: qup-i2c9-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio36"; + function = "qup1_se1"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio37"; + function = "qup1_se1"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c10_data_clk: qup-i2c10-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio40"; + function = "qup1_se2"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio41"; + function = "qup1_se2"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c11_data_clk: qup-i2c11-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio44"; + function = "qup1_se3"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio45"; + function = "qup1_se3"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c12_data_clk: qup-i2c12-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio48"; + function = "qup1_se4"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio49"; + function = "qup1_se4"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c13_data_clk: qup-i2c13-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio52"; + function = "qup1_se5"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio53"; + function = "qup1_se5"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c14_data_clk: qup-i2c14-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio56"; + function = "qup1_se6"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio57"; + function = "qup1_se6"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c15_data_clk: qup-i2c15-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio54"; + function = "qup1_se7"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio55"; + function = "qup1_se7"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c16_data_clk: qup-i2c16-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio64"; + function = "qup2_se0"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio65"; + function = "qup2_se0"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c17_data_clk: qup-i2c17-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio68"; + function = "qup2_se1"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio69"; + function = "qup2_se1"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c18_data_clk: qup-i2c18-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio72"; + function = "qup2_se2"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio73"; + function = "qup2_se2"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c19_data_clk: qup-i2c19-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio76"; + function = "qup2_se3"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio77"; + function = "qup2_se3"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c20_data_clk: qup-i2c20-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio80"; + function = "qup2_se4"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio81"; + function = "qup2_se4"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c21_data_clk: qup-i2c21-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio84"; + function = "qup2_se5"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio85"; + function = "qup2_se5"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c22_data_clk: qup-i2c22-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio88"; + function = "qup2_se6"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio89"; + function = "qup2_se6"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_i2c23_data_clk: qup-i2c23-data-clk-state { + sda_pins { + /* SDA */ + pins = "gpio80"; + function = "qup2_se7"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + /* SCL */ + pins = "gpio81"; + function = "qup2_se7"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + qup_spi0_cs: qup-spi0-cs-state { + pins = "gpio3"; + function = "qup0_se0"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi0_data_clk: qup-spi0-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio0"; + function = "qup0_se0"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio1"; + function = "qup0_se0"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio2"; + function = "qup0_se0"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi1_cs: qup-spi1-cs-state { + pins = "gpio7"; + function = "qup0_se1"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi1_data_clk: qup-spi1-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio4"; + function = "qup0_se1"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio5"; + function = "qup0_se1"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio6"; + function = "qup0_se1"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi2_cs: qup-spi2-cs-state { + pins = "gpio11"; + function = "qup0_se2"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi2_data_clk: qup-spi2-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio8"; + function = "qup0_se2"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio9"; + function = "qup0_se2"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio10"; + function = "qup0_se2"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi3_cs: qup-spi3-cs-state { + pins = "gpio15"; + function = "qup0_se3"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi3_data_clk: qup-spi3-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio12"; + function = "qup0_se3"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio13"; + function = "qup0_se3"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio14"; + function = "qup0_se3"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi4_cs: qup-spi4-cs-state { + pins = "gpio19"; + function = "qup0_se4"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi4_data_clk: qup-spi4-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio16"; + function = "qup0_se4"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio17"; + function = "qup0_se4"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio18"; + function = "qup0_se4"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi5_cs: qup-spi5-cs-state { + pins = "gpio23"; + function = "qup0_se5"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi5_data_clk: qup-spi5-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio20"; + function = "qup0_se5"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio21"; + function = "qup0_se5"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio22"; + function = "qup0_se5"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi6_cs: qup-spi6-cs-state { + pins = "gpio5"; + function = "qup0_se6"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi6_data_clk: qup-spi6-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio6"; + function = "qup0_se6"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio7"; + function = "qup0_se6"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio4"; + function = "qup0_se6"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi7_cs: qup-spi7-cs-state { + pins = "gpio13"; + function = "qup0_se7"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi7_data_clk: qup-spi7-data-clk-state { + /* MISO, MOSI, CLK */ + miso-pins { + /* MISO */ + pins = "gpio14"; + function = "qup0_se7"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio15"; + function = "qup0_se7"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio12"; + function = "qup0_se7"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi8_cs: qup-spi8-cs-state { + pins = "gpio35"; + function = "qup1_se0"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi8_data_clk: qup-spi8-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio32"; + function = "qup1_se0"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio33"; + function = "qup1_se0"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio34"; + function = "qup1_se0"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi9_cs: qup-spi9-cs-state { + pins = "gpio39"; + function = "qup1_se1"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi9_data_clk: qup-spi9-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio36"; + function = "qup1_se1"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio37"; + function = "qup1_se1"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio38"; + function = "qup1_se1"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi10_cs: qup-spi10-cs-state { + pins = "gpio43"; + function = "qup1_se2"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi10_data_clk: qup-spi10-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio40"; + function = "qup1_se2"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio41"; + function = "qup1_se2"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio42"; + function = "qup1_se2"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi11_cs: qup-spi11-cs-state { + pins = "gpio47"; + function = "qup1_se3"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi11_data_clk: qup-spi11-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio44"; + function = "qup1_se3"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio45"; + function = "qup1_se3"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio46"; + function = "qup1_se3"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi12_cs: qup-spi12-cs-state { + pins = "gpio51"; + function = "qup1_se4"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi12_data_clk: qup-spi12-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio48"; + function = "qup1_se4"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio49"; + function = "qup1_se4"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio50"; + function = "qup1_se4"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi13_cs: qup-spi13-cs-state { + pins = "gpio55"; + function = "qup1_se5"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi13_data_clk: qup-spi13-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio52"; + function = "qup1_se5"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio53"; + function = "qup1_se5"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio54"; + function = "qup1_se5"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi14_cs: qup-spi14-cs-state { + pins = "gpio59"; + function = "qup1_se6"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi14_data_clk: qup-spi14-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio56"; + function = "qup1_se6"; + drive-strength = <6>; + bias-disable; + }; + mosi-pins { + /* MOSI */ + pins = "gpio57"; + function = "qup1_se6"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio58"; + function = "qup1_se6"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi15_cs: qup-spi15-cs-state { + pins = "gpio53"; + function = "qup1_se7"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi15_data_clk: qup-spi15-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio54"; + function = "qup1_se7"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio55"; + function = "qup1_se7"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio52"; + function = "qup1_se7"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi16_cs: qup-spi16-cs-state { + pins = "gpio67"; + function = "qup2_se0"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi16_data_clk: qup-spi16-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio64"; + function = "qup2_se0"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio65"; + function = "qup2_se0"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio66"; + function = "qup2_se0"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi17_cs: qup-spi17-cs-state { + pins = "gpio71"; + function = "qup2_se1"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi17_data_clk: qup-spi17-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio68"; + function = "qup2_se1"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio69"; + function = "qup2_se1"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio70"; + function = "qup2_se1"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi18_cs: qup-spi18-cs-state { + pins = "gpio75"; + function = "qup2_se2"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi18_data_clk: qup-spi18-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio72"; + function = "qup2_se2"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio73"; + function = "qup2_se2"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio74"; + function = "qup2_se2"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi19_cs: qup-spi19-cs-state { + pins = "gpio79"; + function = "qup2_se3"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi19_data_clk: qup-spi19-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio76"; + function = "qup2_se3"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio77"; + function = "qup2_se3"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio78"; + function = "qup2_se3"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi20_cs: qup-spi20-cs-state { + pins = "gpio83"; + function = "qup2_se4"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi20_data_clk: qup-spi20-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio80"; + function = "qup2_se4"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio81"; + function = "qup2_se4"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio82"; + function = "qup2_se4"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi21_cs: qup-spi21-cs-state { + pins = "gpio87"; + function = "qup2_se5"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi21_data_clk: qup-spi21-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio84"; + function = "qup2_se5"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio85"; + function = "qup2_se5"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio86"; + function = "qup2_se5"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi22_cs: qup-spi22-cs-state { + pins = "gpio91"; + function = "qup2_se6"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi22_data_clk: qup-spi22-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio88"; + function = "qup2_se6"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio89"; + function = "qup2_se6"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio90"; + function = "qup2_se6"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_spi23_cs: qup-spi23-cs-state { + pins = "gpio83"; + function = "qup2_se7"; + drive-strength = <6>; + bias-disable; + }; + + qup_spi23_data_clk: qup-spi23-data-clk-state { + miso-pins { + /* MISO */ + pins = "gpio80"; + function = "qup2_se7"; + drive-strength = <6>; + bias-disable; + }; + + mosi-pins { + /* MOSI */ + pins = "gpio81"; + function = "qup2_se7"; + drive-strength = <6>; + bias-disable; + }; + + clk-pins { + /* CLK */ + pins = "gpio82"; + function = "qup2_se7"; + drive-strength = <6>; + bias-disable; + }; + }; + + qup_uart2_default: qup-uart2-default-state { + tx-pins { + pins = "gpio10"; + function = "qup0_se2"; + drive-strength = <2>; + bias-disable; + }; + + rx-pins { + pins = "gpio11"; + function = "qup0_se2"; + drive-strength = <2>; + bias-disable; + }; + }; + + qup_uart14_default: qup-uart14-default-state { + cts-pins { + pins = "gpio56"; + function = "qup1_se6"; + drive-strength = <2>; + bias-disable; + }; + + rts-pins { + pins = "gpio57"; + function = "qup1_se6"; + drive-strength = <2>; + bias-disable; + }; + + tx-pins { + pins = "gpio58"; + function = "qup1_se6"; + drive-strength = <2>; + bias-disable; + }; + + rx-pins { + pins = "gpio59"; + function = "qup1_se6"; + drive-strength = <2>; + bias-disable; + }; + }; + + qup_uart19_default: qup-uart19-default-state { + cts-pins { + pins = "gpio76"; + function = "qup2_se3"; + drive-strength = <2>; + bias-disable; + }; + + rts-pins { + pins = "gpio77"; + function = "qup2_se3"; + drive-strength = <2>; + bias-disable; + }; + + tx-pins { + pins = "gpio78"; + function = "qup2_se3"; + drive-strength = <2>; + bias-disable; + }; + + rx-pins { + pins = "gpio79"; + function = "qup2_se3"; + drive-strength = <2>; + bias-disable; + }; + }; + + qup_uart21_default: qup-uart21-default-state { + tx-pins { + pins = "gpio86"; + function = "qup2_se5"; + drive-strength = <2>; + bias-disable; + }; + + rx-pins { + pins = "gpio87"; + function = "qup2_se5"; + drive-strength = <2>; + bias-disable; + }; + }; + + qup_uart22_default: qup-uart22-default-state { + tx-pins { + pins = "gpio90"; + function = "qup2_se6"; + drive-strength = <2>; + bias-disable; + }; + + rx-pins { + pins = "gpio91"; + function = "qup2_se6"; + drive-strength = <2>; + bias-disable; + }; + }; + }; + + apps_smmu: iommu@15000000 { + compatible = "qcom,glymur-smmu-500", + "qcom,smmu-500", + "arm,mmu-500"; + reg = <0x0 0x15000000 0x0 0x100000>; + + #iommu-cells = <2>; + #global-interrupts = <1>; + + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + + dma-coherent; + }; + + intc: interrupt-controller@17000000 { + compatible = "arm,gic-v3"; + reg = <0x0 0x17000000 0x0 0x10000>, + <0x0 0x17080000 0x0 0x480000>; + + interrupts = ; + + #interrupt-cells = <3>; + interrupt-controller; + + #redistributor-regions = <1>; + redistributor-stride = <0x0 0x40000>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + gic_its: gic-its@17040000 { + compatible = "arm,gic-v3-its"; + reg = <0x0 0x17040000 0x0 0x40000>; + + msi-controller; + #msi-cells = <1>; + }; + }; + + watchdog@17600000 { + compatible = "qcom,kpss-wdt"; + reg = <0x0 0x17600000 0x0 0x1000>; + clocks = <&sleep_clk>; + interrupts = ; + }; + + pdp0_mbox: mailbox@17610000 { + compatible = "qcom,glymur-cpucp-mbox", "qcom,x1e80100-cpucp-mbox"; + reg = <0 0x17610000 0 0x8000>, <0 0x19980000 0 0x8000>; + interrupts = ; + #mbox-cells = <1>; + qcom,rx-chans = <0x7>; + }; + + timer@17810000 { + compatible = "arm,armv7-timer-mem"; + reg = <0x0 0x17810000 0x0 0x1000>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + frame@17811000 { + reg = <0x0 0x17811000 0x0 0x1000>, + <0x0 0x17812000 0x0 0x1000>; + + interrupts = , + ; + + frame-number = <0>; + }; + + frame@17813000 { + reg = <0x0 0x17813000 0x0 0x1000>; + + interrupts = ; + + frame-number = <1>; + + status = "disabled"; + }; + + frame@17815000 { + reg = <0x0 0x17815000 0x0 0x1000>; + + interrupts = ; + + frame-number = <2>; + + status = "disabled"; + }; + + frame@17817000 { + reg = <0x0 0x17817000 0x0 0x1000>; + + interrupts = ; + + frame-number = <3>; + + status = "disabled"; + }; + + frame@17819000 { + reg = <0x0 0x17819000 0x0 0x1000>; + + interrupts = ; + + frame-number = <4>; + + status = "disabled"; + }; + + frame@1781b000 { + reg = <0x0 0x1781b000 0x0 0x1000>; + + interrupts = ; + + frame-number = <5>; + + status = "disabled"; + }; + + frame@1781d000 { + reg = <0x0 0x1781d000 0x0 0x1000>; + + interrupts = ; + + frame-number = <6>; + + status = "disabled"; + }; + }; + + apps_rsc: rsc@18900000 { + compatible = "qcom,rpmh-rsc"; + label = "apps_rsc"; + reg = <0x0 0x18900000 0x0 0x10000>, + <0x0 0x18910000 0x0 0x10000>, + <0x0 0x18920000 0x0 0x10000>; + reg-names = "drv-0", + "drv-1", + "drv-2"; + interrupts = , + , + ; + qcom,tcs-offset = <0xd00>; + qcom,drv-id = <2>; + qcom,tcs-config = , + , + , + ; + power-domains = <&CLUSTER3_PD>; + + apps_bcm_voter: bcm-voter { + compatible = "qcom,bcm-voter"; + }; + + rpmhcc: clock-controller { + compatible = "qcom,glymur-rpmh-clk"; + + clocks = <&xo_board>; + clock-names = "xo"; + + #clock-cells = <1>; + }; + + rpmhpd: power-controller { + compatible = "qcom,glymur-rpmhpd"; + + operating-points-v2 = <&rpmhpd_opp_table>; + + #power-domain-cells = <1>; + + rpmhpd_opp_table: opp-table { + compatible = "operating-points-v2"; + + rpmhpd_opp_ret: opp-16 { + opp-level = ; + }; + + rpmhpd_opp_min_svs: opp-48 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d2: opp-52 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d1: opp-56 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d0: opp-60 { + opp-level = ; + }; + + rpmhpd_opp_low_svs: opp-64 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_l1: opp-80 { + opp-level = ; + }; + + rpmhpd_opp_svs: opp-128 { + opp-level = ; + }; + + rpmhpd_opp_svs_l0: opp-144 { + opp-level = ; + }; + + rpmhpd_opp_svs_l1: opp-192 { + opp-level = ; + }; + + rpmhpd_opp_nom: opp-256 { + opp-level = ; + }; + + rpmhpd_opp_nom_l1: opp-320 { + opp-level = ; + }; + + rpmhpd_opp_nom_l2: opp-336 { + opp-level = ; + }; + + rpmhpd_opp_turbo: opp-384 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l1: opp-416 { + opp-level = ; + }; + }; + }; + }; + + nsi_noc: interconnect@1d600000 { + compatible = "qcom,glymur-nsinoc"; + reg = <0x0 0x1d600000 0x0 0x14080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + oobm_ss_noc: interconnect@1f300000 { + compatible = "qcom,glymur-oobm-ss-noc"; + reg = <0x0 0x1f300000 0x0 0x49a00>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + nsp_noc: interconnect@320c0000 { + compatible = "qcom,glymur-nsp-noc"; + reg = <0x0 0x320c0000 0x0 0x21280>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + sram: sram@81e08000 { + compatible = "mmio-sram"; + reg = <0x0 0x81e08600 0x0 0x300>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x0 0x81e08600 0x300>; + + cpu_scp_lpri0: scp-sram-section@0 { + compatible = "arm,scmi-shmem"; + reg = <0x0 0x180>; + }; + + cpu_scp_lpri1: scp-sram-section@180 { + compatible = "arm,scmi-shmem"; + reg = <0x180 0x180>; + }; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/hamoa-camera-sensor.dtsi b/arch/arm64/boot/dts/qcom/hamoa-camera-sensor.dtsi new file mode 100644 index 0000000000000..7ebfcf1ec3912 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/hamoa-camera-sensor.dtsi @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&cam_cci0 { + /*cam0-cmk_imx577*/ + qcom,cam-sensor0 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l4m_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-names = "cam_default", "cam_suspend"; + pinctrl-0 = <&cam_sensor_mclk1_active &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend &cam_sensor_suspend_rst1>; + gpios = <&tlmm 97 0>, + <&tlmm 110 0>, + <&tlmm 19 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK1", + "CAMIF_RESET1", + "CAM_CUSTOM1"; + cci-master = <1>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <0>; + status = "okay"; + }; +}; + +&soc { + qcom,cam-res-mgr { + compatible = "qcom,cam-res-mgr"; + status = "okay"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/hamoa-camera.dtsi b/arch/arm64/boot/dts/qcom/hamoa-camera.dtsi new file mode 100644 index 0000000000000..94dd42b380f4e --- /dev/null +++ b/arch/arm64/boot/dts/qcom/hamoa-camera.dtsi @@ -0,0 +1,2473 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +&soc { + cam_icp: qcom,icp@ac01000 { + compatible = "qcom,cam-icp_v2"; + icp-version = <0x0200>; + reg = <0x0 0xac01000 0x0 0x400>, + <0x0 0xac01800 0x0 0x400>, + <0x0 0x0ac04000 0x0 0x1000>; + reg-names = "icp_csr", "icp_cirq", "icp_wd0"; + reg-cam-base = <0x1000 0x1800 0x4000>; + interrupts = ; + interrupt-names = "icp"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + memory-region = <&camera_mem>; + clocks = <&camcc CAM_CC_ICP_AHB_CLK>, + <&camcc CAM_CC_ICP_CLK_SRC>, + <&camcc CAM_CC_ICP_CLK>; + clock-names = "icp_ahb_clk", + "icp_clk_src", + "icp_clk"; + clock-rates = <0 300000000 0>, + <0 400000000 0>, + <0 480000000 0>, + <0 600000000 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "nominal"; + nrt-device; + src-clock-name = "icp_clk_src"; + operating-points-v2 = <&icp_opp_table>; + clock-control-debugfs = "true"; + fw_name = "qcom/x1e80100/CAMERA_ICP"; + ubwc-ipe-fetch-cfg = <0x707b 0x7083>; + ubwc-ipe-write-cfg = <0x161ef 0x1620f>; + ubwc-bps-fetch-cfg = <0x707b 0x7083>; + ubwc-bps-write-cfg = <0x161ef 0x1620f>; + qos-val = <0x00000A0A>; + cam_hw_pid = <3>; + cell-index = <0>; + status = "okay"; + + icp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_cpas: qcom,cam-cpas@ac13000 { + compatible = "qcom,cam-cpas"; + label = "cpas"; + arch-compat = "cpas_top"; + reg = <0x0 0xac13000 0x0 0x1000>, + <0x0 0xac19000 0x0 0xC000>, + <0x0 0xbbf0000 0x0 0x1F00>; + reg-names = "cam_cpas_top", "cam_camnoc", "cam_rpmh"; + reg-cam-base = <0x13000 0x19000 0x0bbf0000>; + interrupts = ; + interrupt-names = "cpas_camnoc"; + camnoc-axi-min-ib-bw = <3000000000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_RT_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_NRT_CLK>, + <&camcc CAM_CC_QDSS_DEBUG_XO_CLK>; + clock-names = "gcc_ahb_clk", + "gcc_axi_hf_clk", + "gcc_axi_sf_clk", + "cam_cc_slow_ahb_clk_src", + "cpas_ahb_clk", + "cpas_core_ahb_clk", + "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_fast_ahb_clk", + "camnoc_axi_clk_src", + "camnoc_axi_clk", + "camnoc_axi_nrt_clk", + "camcc_debug_clk"; + clock-rates = < 0 0 0 0 0 0 0 0 0 0 0 0>, + < 0 0 0 64000000 0 0 80000000 0 240000000 0 0 0>, + < 0 0 0 80000000 0 0 100000000 0 300000000 0 0 0>, + < 0 0 0 80000000 0 0 400000000 0 400000000 0 0 0>; + clock-cntl-level = "suspend", "lowsvsd1", "lowsvs", "nominal"; + clock-names-option = "cam_icp_clk"; + clocks-option = <&camcc CAM_CC_ICP_CLK>; + clock-rates-option = <400000000>; + src-clock-name = "camnoc_axi_clk_src"; + operating-points-v2 = <&cpas_opp_table>; + control-camnoc-axi-clk; + camnoc-bus-width = <32>; + cam-icc-path-names = "cam_ahb"; + camnoc-axi-clk-bw-margin-perc = <20>; + interconnects =<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_CAMERA_CFG 0>, + <&mmss_noc MASTER_CAMNOC_HF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_SF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_ICP 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "cam_ahb", + "cam_hf_0", + "cam_sf_0", + "cam_sf_icp"; + rpmh-bcm-info = <12 0x4 0x800 0 4>; + cam-ahb-num-cases = <8>; + cam-ahb-bw-KBps = <0 0>, <0 76800>, <0 76800>, <0 150000>, + <0 150000>, <0 300000>, <0 300000>, <0 300000>; + vdd-corners = ; + vdd-corner-ahb-mapping = "suspend", "lowsvs", + "lowsvs", "svs", "svs_l1", + "nominal", "nominal", "nominal", + "turbo", "turbo"; + client-id-based; + client-names = "csiphy0", "csiphy1", "csiphy2", "csiphy4", + "cci0", "cci1", "csid0", "csid1", "csid2", + "csid3", "ife0", "ife1", "ife2", "ife3", "sfe0", + "ipe0", "rt-cdm0", "rt-cdm1", "rt-cdm2", + "rt-cdm3", "cam-cdm-intf0", "bps0", "icp0", + "jpeg-dma0", "jpeg-enc0", "tpg13", "tpg14", + "tpg15"; + cell-index = <0>; + status = "okay"; + + camera-bus-nodes { + level0-nodes { + level-index = <0>; + + bps0_all_rd: bps0-all-rd { + cell-index = <0>; + node-name = "bps0-all-rd"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_rd>; + }; + + bps0_all_wr: bps0-all-wr { + cell-index = <1>; + node-name = "bps0-all-wr"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_wr>; + }; + + icp0_all_rd: icp0-all-rd { + cell-index = <2>; + node-name = "icp0-all-rd"; + client-name = "icp0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt1_rd>; + }; + + ife0_linear_stats_wr: ife0-linear-stats-wr { + cell-index = <3>; + node-name = "ife0-linear-stats-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr3>; + }; + + ife0_pdaf_wr: ife0-pdaf-wr { + cell-index = <4>; + node-name = "ife0-pdaf-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_rt0_wr2>; + }; + + ife0_rdi_pixel_raw_wr: ife0-rdi-pixel-raw-wr { + cell-index = <5>; + node-name = "ife0-rdi-pixel-raw-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr1>; + }; + + ife0_ubwc_wr: ife0-ubwc-wr { + cell-index = <6>; + node-name = "ife0-ubwc-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr0>; + }; + + ife1_linear_stats_wr: ife1-linear-stats-wr { + cell-index = <7>; + node-name = "ife1-linear-stats-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr3>; + }; + + ife1_pdaf_wr: ife1-pdaf-wr { + cell-index = <8>; + node-name = "ife1-pdaf-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_rt0_wr2>; + }; + + ife1_rdi_pixel_raw_wr: ife1-rdi-pixel-raw-wr { + cell-index = <9>; + node-name = "ife1-rdi-pixel-raw-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr1>; + }; + + ife1_ubwc_wr: ife1-ubwc-wr { + cell-index = <10>; + node-name = "ife1-ubwc-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr0>; + }; + + ife2_rdi_stats_pixel_raw_wr: ife2-rdi-stats-pixel-raw-wr { + cell-index = <11>; + node-name = "ife2-rdi-stats-pixel-raw-wr"; + client-name = "ife2"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr4>; + }; + + ife3_rdi_stats_pixel_raw_wr: ife3-rdi-stats-pixel-raw-wr { + cell-index = <12>; + node-name = "ife3-rdi-stats-pixel-raw-wr"; + client-name = "ife3"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr4>; + }; + + ipe0_all_wr: ipe0-all-wr { + cell-index = <13>; + node-name = "ipe0-all-wr"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level2_nrt0_wr>; + }; + + ipe0_in_rd: ipe0-in-rd { + cell-index = <14>; + node-name = "ipe0-in-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_rd>; + }; + + ipe0_ref_rd: ipe0-ref-rd { + cell-index = <15>; + node-name = "ipe0-ref-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_rd>; + }; + + jpeg_dma0_all_rd: jpeg-dma0-all-rd { + cell-index = <16>; + node-name = "jpeg-dma0-all-rd"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd0>; + }; + + jpeg_dma0_all_wr: jpeg-dma0-all-wr { + cell-index = <17>; + node-name = "jpeg-dma0-all-wr"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_wr0>; + }; + + jpeg_enc0_all_rd: jpeg-enc0-all-rd { + cell-index = <18>; + node-name = "jpeg-enc0-all-rd"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd0>; + }; + + jpeg_enc0_all_wr: jpeg-enc0-all-wr { + cell-index = <19>; + node-name = "jpeg-enc0-all-wr"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_wr0>; + }; + + rt_cdm0_all_rd: rt-cdm0-all-rd { + cell-index = <20>; + node-name = "rt-cdm0-all-rd"; + client-name = "rt-cdm0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd1>; + }; + + rt_cdm1_all_rd: rt-cdm1-all-rd { + cell-index = <21>; + node-name = "rt-cdm1-all-rd"; + client-name = "rt-cdm1"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd1>; + }; + + /* IFE Lite 0 */ + rt_cdm2_all_rd: rt-cdm2-all-rd { + cell-index = <22>; + node-name = "rt-cdm2-all-rd"; + client-name = "rt-cdm2"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd1>; + }; + + /* IFE Lite 1 */ + rt_cdm3_all_rd: rt-cdm3-all-rd { + cell-index = <23>; + node-name = "rt-cdm3-all-rd"; + client-name = "rt-cdm3"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd1>; + }; + + sfe0_all_rd: sfe0-all-rd { + cell-index = <24>; + node-name = "sfe0-all-rd"; + client-name = "sfe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level2_rt0_rd>; + }; + + sfe0_rdi_stats_nrdi_wr: sfe0-rdi-stats-nrdi-wr { + cell-index = <25>; + node-name = "sfe0-rdi-stats-nrdi-wr"; + client-name = "sfe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr1>; + }; + }; + + level1-nodes { + level-index = <1>; + camnoc-max-needed; + + level1_nrt0_rd0: level1-nrt0-rd0 { + cell-index = <26>; + node-name = "level1-nrt0-rd0"; + parent-node = <&level2_nrt0_rd>; + traffic-merge-type = ; + }; + + level1_nrt0_rd1: level1-nrt0-rd1 { + cell-index = <27>; + node-name = "level1-nrt0-rd1"; + parent-node = <&level2_nrt0_rd>; + traffic-merge-type = ; + }; + + level1_nrt0_wr0: level1-nrt0-wr0 { + cell-index = <28>; + node-name = "level1-nrt0-wr0"; + parent-node = <&level2_nrt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_rd0: level1-rt0-rd0 { + cell-index = <29>; + node-name = "level1-sfe-rd"; + parent-node = <&level2_rt0_rd>; + traffic-merge-type = ; + }; + + level1_rt0_wr0: level1-rt0-wr0 { + cell-index = <30>; + node-name = "level1-ife-ubwc-wr"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr1: level1-rt0-wr1 { + cell-index = <31>; + node-name = "level1-ife-rdi-wr"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr2: level1-rt0-wr2 { + cell-index = <32>; + node-name = "level1-ife-pdaf"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr3: level1-rt0-wr3 { + cell-index = <33>; + node-name = "level1-ife01-linear-stats"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr4: level1-rt0-wr4 { + cell-index = <34>; + node-name = "level1-ifelite"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + }; + + level2-nodes { + level-index = <2>; + camnoc-max-needed; + + level2_nrt0_rd: level2-nrt0-rd { + cell-index = <35>; + node-name = "level2-nrt0-rd"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + level2_nrt0_wr: level2-nrt0-wr { + cell-index = <36>; + node-name = "level2-nrt0-wr"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + level2_nrt1_rd: level2-nrt1-rd { + cell-index = <37>; + node-name = "level2-nrt1-rd"; + parent-node = <&level3_nrt1_rd_wr_sum>; + traffic-merge-type = ; + bus-width-factor = <4>; + }; + + level2_rt0_rd: level2-rt0-rd { + cell-index = <38>; + node-name = "level2-rt0-rd"; + parent-node = <&level3_rt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + level2_rt0_wr: level2-rt0-wr { + cell-index = <39>; + node-name = "level2-rt0-wr"; + parent-node = <&level3_rt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + }; + + level3-nodes { + level-index = <3>; + + level3_nrt0_rd_wr_sum: level3-nrt0-rd-wr-sum { + cell-index = <40>; + node-name = "level3-nrt0-rd-wr-sum"; + traffic-merge-type = ; + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_0"; + }; + }; + + level3_nrt1_rd_wr_sum: level3-nrt1-rd-wr-sum { + cell-index = <41>; + node-name = "level3-nrt1-rd-wr-sum"; + traffic-merge-type = ; + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_icp"; + }; + }; + + level3_rt0_rd_wr_sum: level3-rt0-rd-wr-sum { + cell-index = <42>; + node-name = "level3-rt0-rd-wr-sum"; + traffic-merge-type = ; + ib-bw-voting-needed; + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_hf_0"; + }; + }; + }; + }; + + cpas_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_cci0: qcom,cci0@ac15000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac15000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x15000>; + interrupts = ; + interrupt-names = "cci0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_0_CLK_SRC>, + <&camcc CAM_CC_CCI_0_CLK>; + clock-names = "cci_0_clk_src", + "cci_0_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_0_clk_src"; + operating-points-v2 = <&cci0_opp_table>; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + pinctrl-0 = <&cci0_active>; + pinctrl-1 = <&cci0_suspend>; + pinctrl-2 = <&cci1_active>; + pinctrl-3 = <&cci1_suspend>; + cell-index = <0>; + status = "okay"; + + cci0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci0: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_400Khz_cci0: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_1Mhz_cci0: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <0>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_100Khz_cci0: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + }; + + cam_cci1: qcom,cci1@ac16000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac16000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x16000>; + interrupts = ; + interrupt-names = "cci1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_1_CLK_SRC>, + <&camcc CAM_CC_CCI_1_CLK>; + clock-names = "cci_1_clk_src", + "cci_1_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_1_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + pinctrl-0 = <&cci2_active>; + pinctrl-1 = <&cci2_suspend>; + pinctrl-2 = <&cci3_active>; + pinctrl-3 = <&cci3_suspend>; + operating-points-v2 = <&cci1_opp_table>; + cell-index = <1>; + status = "okay"; + + cci1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci1: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_400Khz_cci1: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_1Mhz_cci1: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <0>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_100Khz_cci1: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + }; + + qcom,rt-cdm0@ac25000 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac25000 0x0 0x400>; + reg-names = "rt-cdm0"; + reg-cam-base = <0x25000>; + interrupts = ; + interrupt-names = "rt-cdm0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <400000000 0>; + clock-cntl-level = "nominal"; + operating-points-v2 = <&cdm_cpas_opp_table0>; + nrt-device; + cdm-client-names = "ife0", "dualife0"; + config-fifo; + fifo-depths = <64 0 0 0>; + cam_hw_pid = <25>; + cam-hw-mid = <0>; + single-context-cdm; + cell-index = <0>; + status = "okay"; + + cdm_cpas_opp_table0: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,rt-cdm1@ac26000 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac26000 0x0 0x400>; + reg-names = "rt-cdm1"; + reg-cam-base = <0x26000>; + interrupts = ; + interrupt-names = "rt-cdm1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + clock-rates = <400000000 0>; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-cntl-level = "nominal"; + operating-points-v2 = <&cdm_cpas_opp_table1>; + nrt-device; + cdm-client-names = "ife1", "dualife1"; + config-fifo; + fifo-depths = <64 0 0 0>; + cam_hw_pid = <26>; + cam-hw-mid = <0>; + single-context-cdm; + cell-index = <1>; + status = "okay"; + + cdm_cpas_opp_table1: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_jpeg_enc: qcom,jpegenc@ac2a000 { + compatible = "qcom,cam_jpeg_enc_680"; + reg = <0x0 0xac2a000 0x0 0x1000>, + <0x0 0x0ac19000 0x0 0xc000>; + reg-names = "jpegenc_hw", "cam_camnoc"; + reg-cam-base = <0x2a000 0x19000>; + interrupts = ; + interrupt-names = "jpeg"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "jpegenc_clk_src", + "jpegenc_clk"; + clock-rates = <160000000 0>, + <200000000 0>, + <400000000 0>, + <480000000 0>, + <600000000 0>; + src-clock-name = "jpegenc_clk_src"; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + operating-points-v2 = <&jpeg_enc_opp_table>; + nrt-device; + cam_hw_pid = <12 14>; + cam_hw_rd_mid = <0>; + cam_hw_wr_mid = <1>; + cell-index = <0>; + status = "okay"; + + jpeg_enc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-48000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_jpeg_dma: qcom,jpegdma@ac2b000 { + compatible = "qcom,cam_jpeg_dma_680"; + reg = <0x0 0xac2b000 0x0 0x1000>, + <0x0 0x0ac19000 0x0 0xc000>; + reg-names = "jpegdma_hw", "cam_camnoc"; + reg-cam-base = <0x2b000 0x19000>; + interrupts = ; + interrupt-names = "jpegdma"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "jpegdma_clk_src", + "jpegdma_clk"; + clock-rates = <160000000 0>, + <200000000 0>, + <400000000 0>, + <480000000 0>, + <600000000 0>; + src-clock-name = "jpegdma_clk_src"; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + operating-points-v2 = <&jpeg_dma_opp_table>; + nrt-device; + cam_hw_pid = <13 15>; + cam_hw_rd_mid = <0>; + cam_hw_wr_mid = <1>; + cell-index = <0>; + status = "okay"; + + jpeg_dma_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-48000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_bps: qcom,bps@ac2c000 { + compatible = "qcom,cam-bps680"; + reg = <0x0 0xac2c000 0x0 0xb000>; + reg-names = "bps_top"; + reg-cam-base = <0x2c000>; + power-domains = <&camcc CAM_CC_BPS_GDSC>; + clocks = <&camcc CAM_CC_BPS_AHB_CLK>, + <&camcc CAM_CC_BPS_FAST_AHB_CLK>, + <&camcc CAM_CC_BPS_CLK_SRC>, + <&camcc CAM_CC_BPS_CLK>, + <&camcc CAM_CC_CPAS_BPS_CLK>; + clock-names = "bps_ahb_clk", + "bps_fast_ahb_clk", + "bps_clk_src", + "bps_clk", + "cam_cc_cpas_bps_clk"; + clock-rates = <0 0 160000000 0 0>, + <0 0 200000000 0 0>, + <0 0 400000000 0 0>, + <0 0 600000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "nominal"; + nrt-device; + src-clock-name = "bps_clk_src"; + operating-points-v2 = <&bps_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <6 7>; + cell-index = <0>; + status = "okay"; + + bps_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_ipe0: qcom,ipe0@ac42000 { + compatible = "qcom,cam-ipe680"; + reg = <0x0 0xac42000 0x0 0x18000>; + reg-names = "ipe0_top"; + reg-cam-base = <0x42000>; + power-domains = <&camcc CAM_CC_IPE_0_GDSC>; + clocks = <&camcc CAM_CC_IPE_NPS_AHB_CLK>, + <&camcc CAM_CC_IPE_NPS_FAST_AHB_CLK>, + <&camcc CAM_CC_IPE_PPS_FAST_AHB_CLK>, + <&camcc CAM_CC_IPE_NPS_CLK_SRC>, + <&camcc CAM_CC_IPE_NPS_CLK>, + <&camcc CAM_CC_IPE_PPS_CLK>, + <&camcc CAM_CC_CPAS_IPE_NPS_CLK>; + clock-names = "ipe_nps_ahb_clk", + "ipe_nps_fast_ahb_clk", + "ipe_pps_fast_ahb_clk", + "ipe_nps_clk_src", + "ipe_nps_clk", + "ipe_pps_clk", + "cam_cc_cpas_ipe_nps_clk"; + clock-rates = <0 0 0 304000000 0 0 0>, + <0 0 0 364000000 0 0 0>, + <0 0 0 500000000 0 0 0>, + <0 0 0 600000000 0 0 0>, + <0 0 0 700000000 0 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + nrt-device; + src-clock-name = "ipe_nps_clk_src"; + operating-points-v2 = <&ipe0_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <22 23 30>; + cell-index = <0>; + status = "okay"; + + ipe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-304000000 { + opp-hz = /bits/ 64 <304000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-364000000 { + opp-hz = /bits/ 64 <364000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-700000000 { + opp-hz = /bits/ 64 <700000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe0: qcom,ife0@ac62000 { + compatible = "qcom,vfe680"; + reg = <0x0 0xac62000 0x0 0xf000>, + <0x0 0xac19000 0x0 0xc000>; + reg-names = "ife", "cam_camnoc"; + reg-cam-base = <0x62000 0x19000>; + rt-wrapper-base = <0x62000>; + interrupts = ; + interrupt-names = "ife0"; + power-domains = <&camcc CAM_CC_IFE_0_GDSC>; + clocks = <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_CPAS_IFE_0_CLK>; + clock-names = "ife_0_fast_ahb", + "ife_0_clk_src", + "ife_0_clk", + "cam_cc_cpas_ife_0_clk"; + clock-rates = <0 345600000 0 0>, + <0 432000000 0 0>, + <0 594000000 0 0>, + <0 675000000 0 0>, + <0 727000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + src-clock-name = "ife_0_clk_src"; + operating-points-v2 = <&vfe0_opp_table>; + clock-control-debugfs = "true"; + clock-names-option = "ife_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_0_DSP_CLK>; + clock-rates-option = <594000000>; + ubwc-static-cfg = <0x1026 0x1036>; + cam_hw_pid = <16 4 20 8>; + cell-index = <0>; + status = "okay"; + + vfe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-345600000 { + opp-hz = /bits/ 64 <345600000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-675000000 { + opp-hz = /bits/ 64 <675000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-727000000 { + opp-hz = /bits/ 64 <727000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe1: qcom,ife1@ac71000 { + compatible = "qcom,vfe680"; + reg = <0x0 0xac71000 0x0 0xf000>, + <0x0 0xac19000 0x0 0xc000>; + reg-names = "ife", "cam_camnoc"; + reg-cam-base = <0x71000 0x19000>; + rt-wrapper-base = <0x62000>; + interrupts = ; + interrupt-names = "ife1"; + power-domains = <&camcc CAM_CC_IFE_1_GDSC>; + clocks = <&camcc CAM_CC_IFE_1_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_1_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_CPAS_IFE_1_CLK>; + clock-names = "ife_1_fast_ahb", + "ife_1_clk_src", + "ife_1_clk", + "cam_cc_cpas_ife_1_clk"; + clock-rates = <0 345600000 0 0>, + <0 432000000 0 0>, + <0 594000000 0 0>, + <0 675000000 0 0>, + <0 727000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + src-clock-name = "ife_1_clk_src"; + operating-points-v2 = <&vfe1_opp_table>; + clock-control-debugfs = "true"; + clock-names-option = "ife_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_1_DSP_CLK>; + clock-rates-option = <594000000>; + ubwc-static-cfg = <0x1026 0x1036>; + cam_hw_pid = <17 5 21 9>; + cell-index = <1>; + status = "okay"; + + vfe1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-345600000 { + opp-hz = /bits/ 64 <345600000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-675000000 { + opp-hz = /bits/ 64 <675000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-727000000 { + opp-hz = /bits/ 64 <727000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_sfe0: qcom,sfe0@ac9e000 { + compatible = "qcom,sfe680"; + reg = <0x0 0xac9e000 0x0 0x8000>; + reg-names = "sfe0"; + reg-cam-base = <0x9e000>; + rt-wrapper-base = <0x62000>; + interrupts = ; + interrupt-names = "sfe"; + power-domains = <&camcc CAM_CC_SFE_0_GDSC>; + clocks = <&camcc CAM_CC_SFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_SFE_0_CLK_SRC>, + <&camcc CAM_CC_SFE_0_CLK>, + <&camcc CAM_CC_CPAS_SFE_0_CLK>; + clock-names = "sfe_0_fast_ahb", + "sfe_0_clk_src", + "sfe_0_clk", + "cam_cc_cpas_sfe_0_clk"; + clock-rates = <0 345600000 0 0>, + <0 432000000 0 0>, + <0 594000000 0 0>, + <0 675000000 0 0>, + <0 727000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + src-clock-name = "sfe_0_clk_src"; + operating-points-v2 = <&sfe0_opp_table>; + cam_hw_pid = <10 2>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "okay"; + + sfe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-345600000 { + opp-hz = /bits/ 64 <345600000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-675000000 { + opp-hz = /bits/ 64 <675000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-727000000 { + opp-hz = /bits/ 64 <727000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid0: qcom,csid0@acb7000 { + compatible = "qcom,csid695"; + reg = <0x0 0xacb7000 0x0 0xd00>, + <0x0 0xacb6000 0x0 0x1000>; + reg-names = "csid", "csid_top"; + reg-cam-base = <0xb7000 0xb6000>; + rt-wrapper-base = <0x62000>; + interrupts = ; + interrupt-names = "csid0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <1 0 0>; + clocks = <&camcc CAM_CC_CSID_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "csid_clk_src", + "csid_clk", + "csiphy_rx_clk"; + clock-rates = <300000000 0 0>, + <400000000 0 0>, + <480000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "csid_clk_src"; + operating-points-v2 = <&csid0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "okay"; + + csid0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid1: qcom,csid1@acb9000 { + compatible = "qcom,csid695"; + reg = <0x0 0xacb9000 0x0 0xd00>, + <0x0 0xacb6000 0x0 0x1000>; + reg-names = "csid", "csid_top"; + reg-cam-base = <0xb9000 0xb6000>; + rt-wrapper-base = <0x62000>; + interrupts = ; + interrupt-names = "csid1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <1 0 0>; + clocks = <&camcc CAM_CC_CSID_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "csid_clk_src", + "csid_clk", + "csiphy_rx_clk"; + clock-rates = <300000000 0 0>, + <400000000 0 0>, + <480000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "csid_clk_src"; + operating-points-v2 = <&csid1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "okay"; + + csid1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite0: qcom,csid-lite0@acc6000 { + compatible = "qcom,csid-lite680"; + reg = <0x0 0xacc6000 0x0 0xa00>; + reg-names = "csid-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xc6000>; + interrupts = ; + interrupt-names = "csid-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 1 0 0 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb_clk", + "ife_lite_csid_clk_src", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 266666667 0 0 0 0>, + <0 400000000 0 0 0 0>, + <0 480000000 0 0 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <2>; + status = "okay"; + + csid_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite0: qcom,ife-lite0@acc6000 { + compatible = "qcom,vfe-lite695"; + reg = <0x0 0xacc6000 0x0 0x4000>; + reg-names = "ife-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xc6000>; + interrupts = ; + interrupt-names = "ife-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 0 0 1 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb_clk", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk_src", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 0 0 266666667 0 0>, + <0 0 0 400000000 0 0>, + <0 0 0 480000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite0_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <0>; + cell-index = <2>; + status = "okay"; + + vfe_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite1: qcom,csid-lite1@acca000 { + compatible = "qcom,csid-lite680"; + reg = <0x0 0xacca000 0x0 0xa00>; + reg-names = "csid-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xca000>; + interrupts = ; + interrupt-names = "csid-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 1 0 0 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb_clk", + "ife_lite_csid_clk_src", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 266666667 0 0 0 0>, + <0 400000000 0 0 0 0>, + <0 480000000 0 0 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <3>; + status = "okay"; + + csid_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite1: qcom,ife-lite1@acca000 { + compatible = "qcom,vfe-lite695"; + reg = <0x0 0xacca000 0x0 0x4000>; + reg-names = "ife-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xca000>; + interrupts = ; + interrupt-names = "ife-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 0 0 1 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk_src", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 0 0 266666667 0 0>, + <0 0 0 400000000 0 0>, + <0 0 0 480000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite1_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <1>; + cell-index = <3>; + status = "okay"; + + vfe_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy0: qcom,csiphy0@ace4000 { + compatible = "qcom,csiphy-v2.1.2", "qcom,csiphy"; + reg = <0x0 0xace4000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe4000>; + interrupts = ; + interrupt-names = "CSIPHY0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + csi-vdd-1p2-supply = <&vreg_l1c_1p2>; + csi-vdd-0p9-supply = <&vreg_l2c_0p8>; + rgltr-cntrl-support; + rgltr-enable-sync = <1>; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 920000>; + rgltr-load-current = <58900 147000>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy0_clk", + "csi0phytimer_clk_src", + "csi0phytimer_clk"; + src-clock-name = "csi0phytimer_clk_src"; + clock-cntl-level = "nominal"; + clock-rates = <480000000 0 400000000 0>; + operating-points-v2 = <&csiphy0_opp_table>; + cell-index = <0>; + status = "okay"; + + csiphy0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy1: qcom,csiphy1@ace6000 { + compatible = "qcom,csiphy-v2.1.2", "qcom,csiphy"; + reg = <0x0 0xace6000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe6000>; + interrupts = ; + interrupt-names = "CSIPHY1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + csi-vdd-1p2-supply = <&vreg_l1c_1p2>; + csi-vdd-0p9-supply = <&vreg_l2c_0p8>; + rgltr-cntrl-support; + rgltr-enable-sync = <1>; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 920000>; + rgltr-load-current = <58900 147000>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy1_clk", + "csi1phytimer_clk_src", + "csi1phytimer_clk"; + src-clock-name = "csi1phytimer_clk_src"; + clock-cntl-level = "nominal"; + clock-rates = <480000000 0 400000000 0>; + operating-points-v2 = <&csiphy1_opp_table>; + cell-index = <1>; + status = "okay"; + + csiphy1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy2: qcom,csiphy2@ace8000 { + compatible = "qcom,csiphy-v2.1.2", "qcom,csiphy"; + reg = <0x0 0xace8000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe8000>; + interrupts = ; + interrupt-names = "CSIPHY2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + csi-vdd-1p2-supply = <&vreg_l1c_1p2>; + csi-vdd-0p9-supply = <&vreg_l2c_0p8>; + rgltr-cntrl-support; + rgltr-enable-sync = <1>; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 920000>; + rgltr-load-current = <58900 147000>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy2_clk", + "csi2phytimer_clk_src", + "csi2phytimer_clk"; + src-clock-name = "csi2phytimer_clk_src"; + clock-cntl-level = "nominal"; + clock-rates = <480000000 0 400000000 0>; + operating-points-v2 = <&csiphy2_opp_table>; + cell-index = <2>; + status = "okay"; + + csiphy2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy4: qcom,csiphy4@acec000 { + compatible = "qcom,csiphy-v2.1.2", "qcom,csiphy"; + reg = <0x0 0xacec000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xec000>; + interrupts = ; + interrupt-names = "CSIPHY4"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + csi-vdd-1p2-supply = <&vreg_l1c_1p2>; + csi-vdd-0p9-supply = <&vreg_l2c_0p8>; + rgltr-cntrl-support; + rgltr-enable-sync = <1>; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 920000>; + rgltr-load-current = <58900 147000>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy4_clk", + "csi4phytimer_clk_src", + "csi4phytimer_clk"; + src-clock-name = "csi4phytimer_clk_src"; + clock-cntl-level = "nominal"; + clock-rates = <480000000 0 400000000 0>; + operating-points-v2 = <&csiphy4_opp_table>; + cell-index = <4>; + status = "okay"; + + csiphy4_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy_tpg13: qcom,tpg13@acf6000 { + compatible = "qcom,cam-tpg103"; + reg = <0x0 0xacf6000 0x0 0x400>, + <0x0 0xac13000 0x0 0x1000>; + reg-names = "tpg0", "cam_cpas_top"; + reg-cam-base = <0xf6000 0x13000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg0"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>, + <480000000 0>; + clock-cntl-level = "lowsvs", "nominal"; + src-clock-name = "cphy_rx_clk_src"; + operating-points-v2 = <&csiphy_tpg0_opp_table>; + cell-index = <13>; + phy-id = <0>; + status = "okay"; + + csiphy_tpg0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy_tpg14: qcom,tpg14@acf7000 { + compatible = "qcom,cam-tpg103"; + reg = <0x0 0xacf7000 0x0 0x400>, + <0x0 0xac13000 0x0 0x1000>; + reg-names = "tpg1", "cam_cpas_top"; + reg-cam-base = <0xf7000 0x13000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg1"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>, + <480000000 0>; + clock-cntl-level = "lowsvs", "nominal"; + src-clock-name = "cphy_rx_clk_src"; + operating-points-v2 = <&csiphy_tpg1_opp_table>; + cell-index = <14>; + phy-id = <1>; + status = "okay"; + + csiphy_tpg1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy_tpg15: qcom,tpg15@acf8000 { + compatible = "qcom,cam-tpg103"; + reg = <0x0 0xacf8000 0x0 0x400>, + <0x0 0xac13000 0x0 0x1000>; + reg-names = "tpg2", "cam_cpas_top"; + reg-cam-base = <0xf8000 0x13000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg2"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>, + <480000000 0>; + clock-cntl-level = "lowsvs", "nominal"; + src-clock-name = "cphy_rx_clk_src"; + operating-points-v2 = <&csiphy_tpg2_opp_table>; + cell-index = <15>; + phy-id = <2>; + status = "okay"; + + csiphy_tpg2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,rt-cdm2@acf9000 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xacf9000 0x0 0x400>; + reg-names = "rt-cdm2"; + reg-cam-base = <0xf9000>; + interrupts = ; + interrupt-names = "rt-cdm2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_lite_ahb"; + clock-rates = <80000000 0>; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-cntl-level = "nominal"; + operating-points-v2 = <&cdm_cpas_opp_table2>; + nrt-device; + cdm-client-names = "ife2"; + config-fifo; + fifo-depths = <64 0 0 0>; + cam_hw_pid = <24>; + cam-hw-mid = <0>; + single-context-cdm; + cell-index = <2>; + status = "okay"; + + cdm_cpas_opp_table2: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,rt-cdm3@acfa000 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xacfa000 0x0 0x400>; + reg-names = "rt-cdm3"; + reg-cam-base = <0xfa000>; + interrupts = ; + interrupt-names = "rt-cdm3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_lite_ahb"; + clock-rates = <80000000 0>; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-cntl-level = "nominal"; + operating-points-v2 = <&cdm_cpas_opp_table3>; + nrt-device; + cdm-client-names = "ife3"; + config-fifo; + fifo-depths = <64 0 0 0>; + cam_hw_pid = <27>; + cam-hw-mid = <0>; + single-context-cdm; + cell-index = <3>; + status = "okay"; + + cdm_cpas_opp_table3: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,cam-cdm-intf { + compatible = "qcom,cam-cdm-intf"; + label = "cam-cdm-intf"; + num-hw-cdm = <1>; + cdm-client-names = "vfe", + "jpegdma", + "jpegenc"; + cell-index = <0>; + status = "okay"; + }; + + cam_icp_firmware: qcom,cam-icp { + compatible = "qcom,cam-icp"; + compat-hw-name = "qcom,icp", + "qcom,ipe0", + "qcom,bps"; + num-icp = <1>; + num-ipe = <1>; + num-bps = <1>; + status = "okay"; + icp_pc_en; + icp_use_pil; + }; + + qcom,cam-isp { + compatible = "qcom,cam-isp"; + arch-compat = "ife"; + status = "okay"; + }; + + qcom,cam-jpeg { + compatible = "qcom,cam-jpeg"; + compat-hw-name = "qcom,jpegenc", + "qcom,jpegdma"; + num-jpeg-enc = <1>; + num-jpeg-dma = <1>; + status = "okay"; + }; + + qcom,cam-req-mgr { + compatible = "qcom,cam-req-mgr"; + status = "okay"; + }; + + cam_smmu: qcom,cam-smmu { + compatible = "qcom,msm-cam-smmu", "simple-bus"; + status = "okay"; + force_cache_allocs; + need_shared_buffer_padding; + + msm-cam-smmu-cpas-cdm { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x18A0 0x0000>; + cam-smmu-label = "rt-cdm"; + dma-coherent; + multiple-client-devices; + + cpas_cdm_iova_mem_map: iova-mem-map { + iova-mem-region-io { + /* IO region is approximately 4.0 GB */ + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-icp { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x1800 0x0060>, + <&apps_smmu 0x1820 0x0060>, + <&apps_smmu 0x1840 0x0060>, + <&apps_smmu 0x1860 0x0060>, + <&apps_smmu 0x1900 0x0000>, + <&apps_smmu 0x1980 0x0020>, + <&apps_smmu 0x19A0 0x0020>; + cam-smmu-label = "icp"; + iova-region-discard = <0xe0000000 0x800000>; + dma-coherent; + + icp_iova_mem_map: iova-mem-map { + iova-mem-qdss-region { + /* QDSS region is appropriate 1MB */ + iova-region-name = "qdss"; + iova-region-start = <0x10b00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x5>; + qdss-phy-addr = <0x16790000>; + status = "okay"; + }; + + iova-mem-region-fwuncached-region { + /* FW uncached region is 7MB long */ + iova-region-name = "fw_uncached"; + iova-region-start = <0x10400000>; + iova-region-len = <0x700000>; + iova-region-id = <0x6>; + status = "okay"; + }; + + iova-mem-region-io { + /* IO region is approximately 3.8 GB */ + iova-region-name = "io"; + iova-region-start = <0x10c00000>; + iova-region-len = <0xee300000>; + iova-region-id = <0x3>; + iova-region-discard = <0xe0000000 0x800000>; + status = "okay"; + }; + + iova-mem-region-shared { + /* Shared region is ~250MB long */ + iova-region-name = "shared"; + iova-region-start = <0x800000>; + iova-region-len = <0xFC00000>; + iova-region-id = <0x1>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-ife { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0800 0x0060>, + <&apps_smmu 0x0820 0x0060>, + <&apps_smmu 0x0840 0x0060>, + <&apps_smmu 0x0860 0x0060>; + dma-coherent; + cam-smmu-label = "ife", "sfe"; + multiple-client-devices; + + ife_iova_mem_map: iova-mem-map { + /* IO region is approximately 4.0 GB */ + iova-mem-region-io { + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-jpeg { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x18E0 0x0000>; + cam-smmu-label = "jpeg"; + dma-coherent; + + jpeg_iova_mem_map: iova-mem-map { + /* IO region is approximately 4.0 GB */ + iova-mem-region-io { + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-secure { + compatible = "qcom,msm-cam-smmu-cb"; + cam-smmu-label = "cam-secure"; + qcom,secure-cb; + }; + }; + + qcom,cam-sync { + compatible = "qcom,cam-sync"; + status = "okay"; + }; + + qcom,camera-main { + compatible = "qcom,camera_x1e80100"; + status = "okay"; + }; +}; + +&tlmm { + cam_sensor_active_int: cam-sensor-active-int { + /* CUSTOM */ + mux { + pins = "gpio19"; + function = "cam_pwe"; + }; + + config { + pins = "gpio19"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst0: cam-sensor-active-rst0 { + /* RESET REAR */ + config { + pins = "gpio109"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + pins = "gpio109"; + function = "gpio"; + }; + }; + + cam_sensor_active_rst1: cam-sensor-active-rst1 { + /* RESET */ + mux { + pins = "gpio110"; + function = "gpio"; + }; + + config { + pins = "gpio110"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst4: cam-sensor-active-rst4 { + /* RESET REAR */ + config { + pins = "gpio237"; + bias-disable; /* No PULL */ + drive-strength = <12>; /* 12 MA */ + }; + + mux { + pins = "gpio237"; + function = "gpio"; + }; + }; + + cam_sensor_mclk0_active: cam-sensor-mclk0-active { + /* mclk0 */ + config { + pins = "gpio96"; + bias-disable; /* No PULL */ + drive-strength = <6>; /* 6 MA */ + }; + + mux { + pins = "gpio96"; + function = "cam_mclk"; + }; + }; + + cam_sensor_mclk0_suspend: cam-sensor-mclk0-suspend { + /* mclk0 */ + config { + pins = "gpio96"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <6>; /* 6 MA */ + }; + + mux { + pins = "gpio96"; + function = "cam_mclk"; + }; + }; + + cam_sensor_mclk1_active: cam-sensor-mclk1-active { + /* MCLK1 */ + mux { + pins = "gpio97"; + function = "cam_mclk"; + }; + + config { + pins = "gpio97"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_suspend: cam-sensor-mclk1-suspend { + /* MCLK1 */ + mux { + pins = "gpio97"; + function = "cam_mclk"; + }; + + config { + pins = "gpio97"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk4_active: cam-sensor-mclk4-active { + /* mclk4 */ + config { + pins = "gpio100"; + bias-disable; /* No PULL */ + drive-strength = <12>; /* 12 MA */ + }; + + mux { + pins = "gpio100"; + function = "cam_aon"; + }; + }; + + cam_sensor_mclk4_suspend: cam-sensor-mclk4-suspend { + /* mclk4 */ + config { + pins = "gpio100"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <12>; /* 12 MA */ + }; + + mux { + pins = "gpio100"; + function = "cam_aon"; + }; + }; + + cam_sensor_suspend_int: cam-sensor-suspend-int { + /* CUSTOM */ + mux { + pins = "gpio19"; + function = "cam_pwe"; + }; + + config { + pins = "gpio19"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_suspend_rst0: cam-sensor-suspend-rst0 { + /* RESET REAR */ + config { + pins = "gpio109"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + + mux { + pins = "gpio109"; + function = "gpio"; + }; + }; + + cam_sensor_suspend_rst1: cam-sensor-suspend-rst1 { + /* RESET */ + mux { + pins = "gpio110"; + function = "gpio"; + }; + + config { + pins = "gpio110"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst4: cam-sensor-suspend-rst4 { + /* RESET REAR */ + config { + pins = "gpio237"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <12>; /* 12 MA */ + output-low; + }; + + mux { + pins = "gpio237"; + function = "gpio"; + }; + }; + + cci0_active: cci0-active { + config { + /* DATA, CLK */ + pins = "gpio101","gpio102"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio101","gpio102"; // Only 2 + function = "cci_i2c"; + }; + }; + + cci0_suspend: cci0-suspend { + config { + /* DATA, CLK */ + pins = "gpio101","gpio102"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio101","gpio102"; + function = "cci_i2c"; + }; + }; + + cci1_active: cci1-active { + config { + /* DATA, CLK */ + pins = "gpio103","gpio104"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio103","gpio104"; // Only 2 + function = "cci_i2c"; + }; + }; + + cci1_suspend: cci1-suspend { + config { + /* DATA, CLK */ + pins = "gpio103","gpio104"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio103","gpio104"; + function = "cci_i2c"; + }; + }; + + cci2_active: cci2-active { + config { + /* DATA, CLK */ + pins = "gpio105","gpio106"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio105","gpio106"; // Only 2 + function = "cci_i2c"; + }; + }; + + cci2_suspend: cci2-suspend { + config { + /* DATA, CLK */ + pins = "gpio105","gpio106"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio105","gpio106"; + function = "cci_i2c"; + }; + }; + + cci3_active: cci3-active { + config { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + function = "aon_cci"; + }; + }; + + cci3_suspend: cci3-suspend { + config { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + function = "aon_cci"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/hamoa-evk-camx.dtso b/arch/arm64/boot/dts/qcom/hamoa-evk-camx.dtso new file mode 100644 index 0000000000000..d63daf990f2a1 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/hamoa-evk-camx.dtso @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hamoa-camera.dtsi" +#include "hamoa-camera-sensor.dtsi" + +&camss { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-evk-camera-imx577.dtso b/arch/arm64/boot/dts/qcom/hamoa-iot-evk-camera-imx577.dtso new file mode 100644 index 0000000000000..f45a7fbd14b1f --- /dev/null +++ b/arch/arm64/boot/dts/qcom/hamoa-iot-evk-camera-imx577.dtso @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include + +&{/} { + vreg_cam1_1p8: regulator-cam1 { + compatible = "regulator-fixed"; + regulator-name = "vreg_cam1"; + startup-delay-us = <1000>; + enable-active-high; + gpio = <&tlmm 19 GPIO_ACTIVE_HIGH>; + }; +}; + +&camss { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + #address-cells = <1>; + #size-cells = <0>; + + csiphy1_ep: endpoint@0 { + clock-lanes = <7>; + data-lanes = <0 1 2 3>; + remote-endpoint = <&imx577_ep>; + }; + }; + }; +}; + +&cci0 { + status = "okay"; +}; + +&cci0_i2c1 { + #address-cells = <1>; + #size-cells = <0>; + + camera@1a { + compatible = "sony,imx577"; + reg = <0x1a>; + + reset-gpios = <&tlmm 110 GPIO_ACTIVE_LOW>; + pinctrl-0 = <&cam1_default>; + pinctrl-names = "default"; + + clocks = <&camcc CAM_CC_MCLK1_CLK>; + assigned-clocks = <&camcc CAM_CC_MCLK1_CLK>; + assigned-clock-rates = <24000000>; + + dvdd-supply = <&vreg_cam1_1p8>; + dovdd-supply = <&vreg_l4m_1p8>; + + port { + imx577_ep: endpoint { + link-frequencies = /bits/ 64 <600000000>; + data-lanes = <0 1 2 3>; + remote-endpoint = <&csiphy1_ep>; + }; + }; + }; +}; + +&csiphy1 { + vdda-0p8-supply = <&vreg_l2c_0p8>; + vdda-1p2-supply = <&vreg_l1c_1p2>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts index df8d6e5c1f45e..c99fcf0d4a122 100644 --- a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts +++ b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts @@ -5,7 +5,9 @@ /dts-v1/; +#include #include "hamoa-iot-som.dtsi" +#include / { model = "Qualcomm Technologies, Inc. Hamoa IoT EVK"; @@ -17,6 +19,16 @@ serial1 = &uart14; }; + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pmk8550_pwm 0 5000000>; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; + power-supply = <&vreg_edp_bl>; + + pinctrl-0 = <&edp_bl_en>, <&edp_bl_pwm>; + pinctrl-names = "default"; + }; + wcd938x: audio-codec { compatible = "qcom,wcd9385-codec"; @@ -48,6 +60,32 @@ stdout-path = "serial0:115200n8"; }; + connector3 { + compatible = "usb-a-connector"; + label = "USB-3-Type-A"; + power-role = "source"; + + vbus-supply = <®ulator_usb3_vbus>; + + port { + connector_3_in: endpoint { + }; + }; + }; + + connector6 { + compatible = "usb-a-connector"; + label = "USB-6-Type-A"; + power-role = "source"; + + vbus-supply = <®ulator_usb6_vbus>; + + port { + connector_4_in: endpoint { + }; + }; + }; + pmic-glink { compatible = "qcom,x1e80100-pmic-glink", "qcom,sm8550-pmic-glink", @@ -183,6 +221,22 @@ regulator-boot-on; }; + vreg_edp_bl: regulator-edp-bl { + compatible = "regulator-fixed"; + + regulator-name = "VBL9"; + regulator-min-microvolt = <3600000>; + regulator-max-microvolt = <3600000>; + + gpio = <&pmc8380_3_gpios 10 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&edp_bl_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + vreg_nvme: regulator-nvme { compatible = "regulator-fixed"; @@ -199,6 +253,48 @@ regulator-boot-on; }; + vreg_pcie_12v: regulator-pcie-12v { + compatible = "regulator-fixed"; + + regulator-name = "VREG_PCIE_12V"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + + gpio = <&pm8550ve_8_gpios 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&pcie_x8_12v>; + pinctrl-names = "default"; + }; + + vreg_pcie_3v3: regulator-pcie-3v3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_PCIE_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8380_3_gpios 6 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&pm_sde7_main_3p3_en>; + pinctrl-names = "default"; + }; + + vreg_pcie_3v3_aux: regulator-pcie-3v3-aux { + compatible = "regulator-fixed"; + + regulator-name = "VREG_PCIE_3P3_AUX"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8380_3_gpios 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&pm_sde7_aux_3p3_en>; + pinctrl-names = "default"; + }; + /* Left unused as the retimer is not used on this board. */ vreg_rtmr0_1p15: regulator-rtmr0-1p15 { compatible = "regulator-fixed"; @@ -344,6 +440,26 @@ regulator-boot-on; }; + regulator_usb3_vbus: regulator-usb3-vbus { + compatible = "regulator-fixed"; + regulator-name = "USB3_VBUS"; + gpio = <&pm8550ve_9_gpios 4 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&usb3_en>; + pinctrl-names = "default"; + enable-active-high; + regulator-always-on; + }; + + regulator_usb6_vbus: regulator-usb6-vbus { + compatible = "regulator-fixed"; + regulator-name = "USB6_VBUS"; + gpio = <&pm8550ve_9_gpios 5 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&usb6_en>; + pinctrl-names = "default"; + enable-active-high; + regulator-always-on; + }; + vph_pwr: regulator-vph-pwr { compatible = "regulator-fixed"; @@ -381,6 +497,23 @@ vin-supply = <&vreg_wcn_3p3>; }; + vreg_wcn_bt_en: regulator-wcn-bt-en { + compatible = "regulator-fixed"; + + regulator-name = "VREG_WCN_BT_EN"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + gpio = <&tlmm 116 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&wcn_bt_en>; + pinctrl-names = "default"; + + regulator-always-on; + regulator-boot-on; + }; + vreg_wcn_3p3: regulator-wcn-3p3 { compatible = "regulator-fixed"; @@ -531,10 +664,9 @@ vddrfa1p2-supply = <&vreg_wcn_1p9>; vddrfa1p8-supply = <&vreg_wcn_1p9>; - bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>; wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>; - pinctrl-0 = <&wcn_bt_en>; + pinctrl-0 = <&wcn_wlan_en>; pinctrl-names = "default"; regulators { @@ -683,6 +815,16 @@ pinctrl-0 = <&eusb6_reset_n>; pinctrl-names = "default"; }; + + embedded-controller@76 { + compatible = "qcom,hamoa-iot-evk-ec", "qcom,hamoa-crd-ec"; + reg = <0x76>; + + interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>; + + pinctrl-0 = <&ec_int_n_default>; + pinctrl-names = "default"; + }; }; &i2c7 { @@ -743,20 +885,32 @@ }; &lpass_tlmm { - spkr_01_sd_n_active: spkr-01-sd-n-active-state { + spkr_0_sd_n_active: spkr-0-sd-n-active-state { pins = "gpio12"; function = "gpio"; drive-strength = <16>; bias-disable; - output-low; }; - spkr_23_sd_n_active: spkr-23-sd-n-active-state { + spkr_1_sd_n_active: spkr-1-sd-n-active-state { pins = "gpio13"; function = "gpio"; drive-strength = <16>; bias-disable; - output-low; + }; + + spkr_2_sd_n_active: spkr-2-sd-n-active-state { + pins = "gpio17"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + spkr_3_sd_n_active: spkr-3-sd-n-active-state { + pins = "gpio18"; + function = "gpio"; + drive-strength = <16>; + bias-disable; }; }; @@ -807,6 +961,8 @@ aux-bus { panel { compatible = "edp-panel"; + + backlight = <&backlight>; power-supply = <&vreg_edp_3p3>; port { @@ -832,10 +988,53 @@ status = "okay"; }; +&pcie3_port0 { + vpcie12v-supply = <&vreg_pcie_12v>; + vpcie3v3-supply = <&vreg_pcie_3v3>; + vpcie3v3aux-supply = <&vreg_pcie_3v3_aux>; + + reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>; +}; + +&pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + + wifi@0 { + compatible = "pci17cb,1107"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + }; +}; + +&pcie5 { + vddpe-3v3-supply = <&vreg_wwan>; +}; + +&pcie5_port0 { + reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; +}; + &pcie6a { vddpe-3v3-supply = <&vreg_nvme>; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { rtmr0_default: rtmr0-reset-n-active-state { pins = "gpio10"; @@ -856,6 +1055,17 @@ }; }; +&pm8550ve_8_gpios { + pcie_x8_12v: pcie-12v-default-state { + pins = "gpio8"; + function = "normal"; + output-enable; + output-high; + bias-pull-down; + power-source = <0>; + }; +}; + &pm8550ve_9_gpios { usb0_1p8_reg_en: usb0-1p8-reg-en-state { pins = "gpio8"; @@ -865,6 +1075,77 @@ input-disable; output-enable; }; + + usb3_en: usb3-en-state { + pins = "gpio4"; + function = "normal"; + qcom,drive-strength = ; + output-enable; + power-source = <0>; + }; + + usb6_en: usb6-en-state { + pins = "gpio5"; + function = "normal"; + qcom,drive-strength = ; + output-enable; + power-source = <0>; + }; +}; + +&pm8550_pwm { + status = "okay"; + + multi-led { + color = ; + function = LED_FUNCTION_STATUS; + + #address-cells = <1>; + #size-cells = <0>; + + led@1 { + reg = <1>; + color = ; + }; + + led@2 { + reg = <2>; + color = ; + }; + }; +}; + +&pmc8380_3_gpios { + edp_bl_en: edp-bl-en-state { + pins = "gpio4"; + function = "normal"; + power-source = <1>; + input-disable; + output-enable; + }; + + edp_bl_reg_en: edp-bl-reg-en-state { + pins = "gpio10"; + function = "normal"; + }; +}; + +&pmc8380_3_gpios { + pm_sde7_aux_3p3_en: pcie-aux-3p3-default-state { + pins = "gpio8"; + function = "normal"; + output-enable; + bias-pull-down; + power-source = <0>; + }; + + pm_sde7_main_3p3_en: pcie-main-3p3-default-state { + pins = "gpio6"; + function = "normal"; + output-enable; + bias-pull-down; + power-source = <0>; + }; }; &pmc8380_5_gpios { @@ -878,6 +1159,17 @@ }; }; +&pmk8550_gpios { + edp_bl_pwm: edp-bl-pwm-state { + pins = "gpio5"; + function = "func3"; + }; +}; + +&pmk8550_pwm { + status = "okay"; +}; + &smb2360_0 { status = "okay"; }; @@ -905,15 +1197,27 @@ vdd3-supply = <&vreg_l8b_3p0>; }; +&spi11 { + status = "okay"; + + tpm@0 { + compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi"; + reg = <0>; + spi-max-frequency = <20000000>; + }; +}; + &swr0 { status = "okay"; - pinctrl-0 = <&wsa_swr_active>, <&spkr_01_sd_n_active>; + pinctrl-0 = <&wsa_swr_active>; pinctrl-names = "default"; /* WSA8845, Left Woofer */ left_woofer: speaker@0,0 { compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_0_sd_n_active>; + pinctrl-names = "default"; reg = <0 0>; reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>; #sound-dai-cells = <0>; @@ -926,8 +1230,10 @@ /* WSA8845, Left Tweeter */ left_tweeter: speaker@0,1 { compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_1_sd_n_active>; + pinctrl-names = "default"; reg = <0 1>; - reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>; + reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>; #sound-dai-cells = <0>; sound-name-prefix = "TweeterLeft"; vdd-1p8-supply = <&vreg_l15b_1p8>; @@ -961,14 +1267,16 @@ &swr3 { status = "okay"; - pinctrl-0 = <&wsa2_swr_active>, <&spkr_23_sd_n_active>; + pinctrl-0 = <&wsa2_swr_active>; pinctrl-names = "default"; /* WSA8845, Right Woofer */ right_woofer: speaker@0,0 { compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_2_sd_n_active>; + pinctrl-names = "default"; reg = <0 0>; - reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>; + reset-gpios = <&lpass_tlmm 17 GPIO_ACTIVE_LOW>; #sound-dai-cells = <0>; sound-name-prefix = "WooferRight"; vdd-1p8-supply = <&vreg_l15b_1p8>; @@ -979,8 +1287,10 @@ /* WSA8845, Right Tweeter */ right_tweeter: speaker@0,1 { compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_3_sd_n_active>; + pinctrl-names = "default"; reg = <0 1>; - reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>; + reset-gpios = <&lpass_tlmm 18 GPIO_ACTIVE_LOW>; #sound-dai-cells = <0>; sound-name-prefix = "TweeterRight"; vdd-1p8-supply = <&vreg_l15b_1p8>; @@ -990,6 +1300,28 @@ }; &tlmm { + ec_int_n_default: ec-int-n-state { + pins = "gpio66"; + function = "gpio"; + bias-disable; + }; + + cam1_default: cam1-default-state { + mclk-pins { + pins = "gpio97"; + function = "cam_mclk"; + drive-strength = <2>; + bias-disable; + }; + + rst-pins { + pins = "gpio110"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + }; + edp_reg_en: edp-reg-en-state { pins = "gpio70"; function = "gpio"; @@ -1123,6 +1455,13 @@ bias-disable; }; + wcn_wlan_en: wcn-wlan-en-state { + pins = "gpio117"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + wwan_sw_en: wwan-sw-en-state { pins = "gpio221"; function = "gpio"; @@ -1154,13 +1493,13 @@ compatible = "qcom,wcn7850-bt"; max-speed = <3200000>; - vddaon-supply = <&vreg_pmu_aon_0p59>; - vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; - vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; - vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; - vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; - vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; - vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; + vddrfacmn-supply = <&vreg_wcn_3p3>; + vddaon-supply = <&vreg_wcn_3p3>; + vddwlcx-supply = <&vreg_wcn_3p3>; + vddwlmx-supply = <&vreg_wcn_3p3>; + vddrfa0p8-supply = <&vreg_wcn_3p3>; + vddrfa1p2-supply = <&vreg_wcn_3p3>; + vddrfa1p8-supply = <&vreg_wcn_3p3>; }; }; @@ -1170,6 +1509,24 @@ status = "okay"; }; +&ufs_mem_phy { + vdda-phy-supply = <&vreg_l3i_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&ufs_mem_hc { + reset-gpios = <&tlmm 238 GPIO_ACTIVE_LOW>; + + vcc-supply = <&vreg_l17b_2p5>; + vcc-max-microamp = <1300000>; + vccq-supply = <&vreg_l2i_1p2>; + vccq-max-microamp = <1200000>; + + status = "okay"; +}; + &usb_1_ss0_dwc3_hs { remote-endpoint = <&pmic_glink_ss0_hs_in>; }; @@ -1220,3 +1577,69 @@ &usb_mp_hsphy1 { phys = <&eusb6_repeater>; }; + +&thermal_zones { + gpuss-0-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; + + gpuss-1-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; + + gpuss-2-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; + + gpuss-3-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; + + gpuss-4-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; + + gpuss-5-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; + + gpuss-6-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; + + gpuss-7-thermal { + trips { + trip-point0 { + temperature = <105000>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi index 1aead50b8920b..5d9af325c931e 100644 --- a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi @@ -3,8 +3,8 @@ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" #include #include @@ -376,6 +376,21 @@ regulator-initial-mode = ; }; }; + + /* PM8010_M */ + regulators-8 { + compatible = "qcom,pm8010-rpmh-regulators"; + qcom,pmic-id = "m"; + + vdd-l3-l4-supply = <&vreg_s4c_1p8>; + + vreg_l4m_1p8: ldo4 { + regulator-name = "vrer_l4m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1808000>; + regulator-initial-mode = ; + }; + }; }; &iris { @@ -390,10 +405,21 @@ firmware-name = "qcom/x1e80100/gen70500_zap.mbn"; }; -&pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; +&pcie3 { + pinctrl-0 = <&pcie3_default>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie3_phy { + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; +&pcie4 { pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -407,10 +433,21 @@ status = "okay"; }; -&pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +&pcie5 { + pinctrl-0 = <&pcie5_default>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie5_phy { + vdda-phy-supply = <&vreg_l3i_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; +&pcie6a { pinctrl-0 = <&pcie6a_default>; pinctrl-names = "default"; @@ -451,8 +488,30 @@ }; &tlmm { - gpio-reserved-ranges = <34 2>, /* TPM LP & INT */ - <44 4>; /* SPI (TPM) */ + gpio-reserved-ranges = <34 2>; /* TPM LP & INT */ + + pcie3_default: pcie3-default-state { + clkreq-n-pins { + pins = "gpio144"; + function = "pcie3_clk"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-n-pins { + pins = "gpio143"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-n-pins { + pins = "gpio145"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; pcie4_default: pcie4-default-state { clkreq-n-pins { @@ -477,6 +536,29 @@ }; }; + pcie5_default: pcie5-default-state { + clkreq-n-pins { + pins = "gpio150"; + function = "pcie5_clk"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-n-pins { + pins = "gpio149"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-n-pins { + pins = "gpio151"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; + pcie6a_default: pcie6a-default-state { clkreq-n-pins { pins = "gpio153"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-pmics.dtsi b/arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi similarity index 100% rename from arch/arm64/boot/dts/qcom/x1e80100-pmics.dtsi rename to arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi diff --git a/arch/arm64/boot/dts/qcom/x1e80100.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi similarity index 85% rename from arch/arm64/boot/dts/qcom/x1e80100.dtsi rename to arch/arm64/boot/dts/qcom/hamoa.dtsi index 51576d9c935de..309a16e9c5540 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -76,6 +80,7 @@ power-domains = <&cpu_pd0>, <&scmi_dvfs 0>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; l2_0: l2-cache { compatible = "cache"; @@ -93,6 +98,7 @@ power-domains = <&cpu_pd1>, <&scmi_dvfs 0>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu2: cpu@200 { @@ -104,6 +110,7 @@ power-domains = <&cpu_pd2>, <&scmi_dvfs 0>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu3: cpu@300 { @@ -115,6 +122,7 @@ power-domains = <&cpu_pd3>, <&scmi_dvfs 0>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu4: cpu@10000 { @@ -126,6 +134,7 @@ power-domains = <&cpu_pd4>, <&scmi_dvfs 1>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; l2_1: l2-cache { compatible = "cache"; @@ -143,6 +152,7 @@ power-domains = <&cpu_pd5>, <&scmi_dvfs 1>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu6: cpu@10200 { @@ -154,6 +164,7 @@ power-domains = <&cpu_pd6>, <&scmi_dvfs 1>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu7: cpu@10300 { @@ -165,6 +176,7 @@ power-domains = <&cpu_pd7>, <&scmi_dvfs 1>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu8: cpu@20000 { @@ -176,6 +188,7 @@ power-domains = <&cpu_pd8>, <&scmi_dvfs 2>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; l2_2: l2-cache { compatible = "cache"; @@ -193,6 +206,7 @@ power-domains = <&cpu_pd9>, <&scmi_dvfs 2>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu10: cpu@20200 { @@ -204,6 +218,7 @@ power-domains = <&cpu_pd10>, <&scmi_dvfs 2>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu11: cpu@20300 { @@ -215,6 +230,7 @@ power-domains = <&cpu_pd11>, <&scmi_dvfs 2>; power-domain-names = "psci", "perf"; cpu-idle-states = <&cluster_c4>; + #cooling-cells = <2>; }; cpu-map { @@ -281,7 +297,7 @@ idle-state-name = "ret"; arm,psci-suspend-param = <0x00000004>; entry-latency-us = <180>; - exit-latency-us = <500>; + exit-latency-us = <320>; min-residency-us = <600>; }; }; @@ -302,6 +318,14 @@ exit-latency-us = <4000>; min-residency-us = <7000>; }; + + domain_ss3: domain-sleep-0 { + compatible = "domain-idle-state"; + arm,psci-suspend-param = <0x0200c354>; + entry-latency-us = <2800>; + exit-latency-us = <4400>; + min-residency-us = <9000>; + }; }; }; @@ -320,8 +344,7 @@ firmware { scm: scm { compatible = "qcom,scm-x1e80100", "qcom,scm"; - interconnects = <&aggre2_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS - &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + /* TODO: add interconnects */ qcom,dload-mode = <&tcsr 0x19000>; }; @@ -448,7 +471,12 @@ system_pd: power-domain-system { #power-domain-cells = <0>; - /* TODO: system-wide idle states */ + domain-idle-states = <&domain_ss3>; + }; + + reboot-mode { + mode-bootloader = <0x80010001 0x2>; + mode-edl = <0x80000000 0x1>; }; }; @@ -791,8 +819,8 @@ #address-cells = <2>; #size-cells = <2>; - dma-ranges = <0 0 0 0 0x10 0>; - ranges = <0 0 0 0 0x10 0>; + dma-ranges = <0 0 0 0 0x100 0>; + ranges = <0 0 0 0 0x100 0>; gcc: clock-controller@100000 { compatible = "qcom,x1e80100-gcc"; @@ -807,7 +835,34 @@ <0>, <&usb_1_ss0_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, <&usb_1_ss1_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, - <&usb_1_ss2_qmpphy QMP_USB43DP_USB3_PIPE_CLK>; + <&usb_1_ss2_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <&ufs_mem_phy 0>, + <&ufs_mem_phy 1>, + <&ufs_mem_phy 2>; power-domains = <&rpmhpd RPMHPD_CX>; #clock-cells = <1>; @@ -2910,7 +2965,7 @@ reg = <0 0x00fda000 0 0x4000>; clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_1_CLKREF_EN>, <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; clock-names = "aux", @@ -2981,7 +3036,7 @@ reg = <0 0x00fdf000 0 0x4000>; clocks = <&gcc GCC_USB3_TERT_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_2_CLKREF_EN>, <&gcc GCC_USB3_TERT_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>; clock-names = "aux", @@ -3033,6 +3088,35 @@ }; }; + cryptobam: dma-controller@1dc4000 { + compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0"; + reg = <0x0 0x01dc4000 0x0 0x28000>; + interrupts = ; + #dma-cells = <1>; + iommus = <&apps_smmu 0x480 0x0000>, + <&apps_smmu 0x481 0x0000>; + qcom,ee = <0>; + qcom,controlled-remotely; + num-channels = <20>; + qcom,num-ees = <4>; + }; + + crypto: crypto@1dfa000 { + compatible = "qcom,x1e80100-qce", "qcom,sm8150-qce", "qcom,qce"; + reg = <0x0 0x01dfa000 0x0 0x6000>; + dmas = <&cryptobam 4>, <&cryptobam 5>; + dma-names = "rx", "tx"; + iommus = <&apps_smmu 0x480 0x0000>, + <&apps_smmu 0x481 0x0000>; + interconnects = <&aggre2_noc MASTER_CRYPTO 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "memory"; + }; + + rng: rng@10c3000 { + compatible = "qcom,x1e80100-trng", "qcom,trng"; + reg = <0x0 0x010c3000 0x0 0x1000>; + }; + cnoc_main: interconnect@1500000 { compatible = "qcom,x1e80100-cnoc-main"; reg = <0 0x01500000 0 0x14400>; @@ -3226,9 +3310,6 @@ power-domains = <&gcc GCC_PCIE_3_GDSC>; - phys = <&pcie3_phy>; - phy-names = "pciephy"; - eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555 0x5555 0x5555 0x5555 0x5555>; eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55 0x55 0x55 0x55 0x55>; @@ -3311,12 +3392,14 @@ }; }; - pcie3_port: pcie@0 { + pcie3_port0: pcie@0 { device_type = "pci"; compatible = "pciclass,0604"; reg = <0x0 0x0 0x0 0x0 0x0>; bus-range = <0x01 0xff>; + phys = <&pcie3_phy>; + #address-cells = <3>; #size-cells = <2>; ranges; @@ -3445,13 +3528,22 @@ power-domains = <&gcc GCC_PCIE_6A_GDSC>; required-opps = <&rpmhpd_opp_nom>; - phys = <&pcie6a_phy>; - phy-names = "pciephy"; - eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>; eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>; status = "disabled"; + + pcie6a_port0: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + phys = <&pcie6a_phy>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; }; pcie6a_phy: phy@1bfc000 { @@ -3577,12 +3669,21 @@ power-domains = <&gcc GCC_PCIE_5_GDSC>; required-opps = <&rpmhpd_opp_nom>; - phys = <&pcie5_phy>; - phy-names = "pciephy"; - eq-presets-8gts = /bits/ 16 <0x5555 0x5555>; status = "disabled"; + + pcie5_port0: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + phys = <&pcie5_phy>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; }; pcie5_phy: phy@1c06000 { @@ -3707,9 +3808,6 @@ power-domains = <&gcc GCC_PCIE_4_GDSC>; required-opps = <&rpmhpd_opp_nom>; - phys = <&pcie4_phy>; - phy-names = "pciephy"; - eq-presets-8gts = /bits/ 16 <0x5555 0x5555>; status = "disabled"; @@ -3719,6 +3817,8 @@ reg = <0x0 0x0 0x0 0x0 0x0>; bus-range = <0x01 0xff>; + phys = <&pcie4_phy>; + #address-cells = <3>; #size-cells = <2>; ranges; @@ -3760,6 +3860,122 @@ status = "disabled"; }; + ufs_mem_phy: phy@1d80000 { + compatible = "qcom,x1e80100-qmp-ufs-phy", + "qcom,sm8550-qmp-ufs-phy"; + reg = <0x0 0x01d80000 0x0 0x2000>; + + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&gcc GCC_UFS_PHY_PHY_AUX_CLK>, + <&tcsr TCSR_UFS_PHY_CLKREF_EN>; + + clock-names = "ref", + "ref_aux", + "qref"; + resets = <&ufs_mem_hc 0>; + reset-names = "ufsphy"; + + power-domains = <&gcc GCC_UFS_MEM_PHY_GDSC>; + + #clock-cells = <1>; + #phy-cells = <0>; + + status = "disabled"; + }; + + ufs_mem_hc: ufshc@1d84000 { + compatible = "qcom,x1e80100-ufshc", + "qcom,sm8550-ufshc", + "qcom,ufshc"; + reg = <0x0 0x01d84000 0x0 0x3000>; + + interrupts = ; + + clocks = <&gcc GCC_UFS_PHY_AXI_CLK>, + <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>, + <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>, + <&rpmhcc RPMH_LN_BB_CLK3>, + <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>, + <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>, + <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>; + clock-names = "core_clk", + "bus_aggr_clk", + "iface_clk", + "core_clk_unipro", + "ref_clk", + "tx_lane0_sync_clk", + "rx_lane0_sync_clk", + "rx_lane1_sync_clk"; + + operating-points-v2 = <&ufs_opp_table>; + + resets = <&gcc GCC_UFS_PHY_BCR>; + reset-names = "rst"; + + interconnects = <&aggre1_noc MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "ufs-ddr", + "cpu-ufs"; + + power-domains = <&gcc GCC_UFS_PHY_GDSC>; + required-opps = <&rpmhpd_opp_nom>; + + iommus = <&apps_smmu 0x1a0 0>; + dma-coherent; + + lanes-per-direction = <2>; + + phys = <&ufs_mem_phy>; + phy-names = "ufsphy"; + + #reset-cells = <1>; + + status = "disabled"; + + ufs_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-75000000 { + opp-hz = /bits/ 64 <75000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <75000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <150000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <300000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + tcsr_mutex: hwlock@1f40000 { compatible = "qcom,tcsr-mutex"; reg = <0 0x01f40000 0 0x20000>; @@ -4922,6 +5138,7 @@ interconnect-names = "usb-ddr", "apps-usb"; + qcom,select-utmi-as-pipe-clk; wakeup-source; status = "disabled"; @@ -4939,15 +5156,8 @@ dma-coherent; - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - usb_2_dwc3_hs: endpoint { - }; + port { + usb_2_dwc3_hs: endpoint { }; }; }; @@ -5269,8 +5479,8 @@ resets = <&gcc GCC_VIDEO_AXI0_CLK_ARES>; reset-names = "bus"; - iommus = <&apps_smmu 0x1940 0>, - <&apps_smmu 0x1947 0>; + iommu-map = , + ; dma-coherent; /* @@ -5335,119 +5545,559 @@ #power-domain-cells = <1>; }; - mdss: display-subsystem@ae00000 { - compatible = "qcom,x1e80100-mdss"; - reg = <0 0x0ae00000 0 0x1000>; - reg-names = "mdss"; + cci0: cci@ac15000 { + compatible = "qcom,x1e80100-cci", "qcom,msm8996-cci"; + reg = <0 0x0ac15000 0 0x1000>; - interrupts = ; + interrupts = ; - clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, - <&gcc GCC_DISP_HF_AXI_CLK>, - <&dispcc DISP_CC_MDSS_MDP_CLK>; + clocks = <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CCI_0_CLK>; + clock-names = "camnoc_axi", + "cpas_ahb", + "cci"; - resets = <&dispcc DISP_CC_MDSS_CORE_BCR>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; - interconnects = <&mmss_noc MASTER_MDP QCOM_ICC_TAG_ALWAYS - &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ALWAYS>, - <&mc_virt MASTER_LLCC QCOM_ICC_TAG_ALWAYS - &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, - <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY - &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; - interconnect-names = "mdp0-mem", - "mdp1-mem", - "cpu-cfg"; + pinctrl-0 = <&cci0_default>; + pinctrl-1 = <&cci0_sleep>; + pinctrl-names = "default", "sleep"; - power-domains = <&dispcc MDSS_GDSC>; + #address-cells = <1>; + #size-cells = <0>; - iommus = <&apps_smmu 0x1c00 0x2>; + status = "disabled"; - interrupt-controller; - #interrupt-cells = <1>; + cci0_i2c0: i2c-bus@0 { + reg = <0>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; - #address-cells = <2>; - #size-cells = <2>; - ranges; + cci0_i2c1: i2c-bus@1 { + reg = <1>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; - status = "disabled"; + cci1: cci@ac16000 { + compatible = "qcom,x1e80100-cci", "qcom,msm8996-cci"; + reg = <0 0x0ac16000 0 0x1000>; - mdss_mdp: display-controller@ae01000 { - compatible = "qcom,x1e80100-dpu"; - reg = <0 0x0ae01000 0 0x8f000>, - <0 0x0aeb0000 0 0x2008>; - reg-names = "mdp", - "vbif"; + interrupts = ; - interrupts-extended = <&mdss 0>; + clocks = <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CCI_1_CLK>; + clock-names = "camnoc_axi", + "cpas_ahb", + "cci"; - clocks = <&gcc GCC_DISP_HF_AXI_CLK>, - <&dispcc DISP_CC_MDSS_AHB_CLK>, - <&dispcc DISP_CC_MDSS_MDP_LUT_CLK>, - <&dispcc DISP_CC_MDSS_MDP_CLK>, - <&dispcc DISP_CC_MDSS_VSYNC_CLK>; - clock-names = "nrt_bus", - "iface", - "lut", - "core", - "vsync"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; - operating-points-v2 = <&mdp_opp_table>; + pinctrl-0 = <&cci1_default>; + pinctrl-1 = <&cci1_sleep>; + pinctrl-names = "default", "sleep"; - power-domains = <&rpmhpd RPMHPD_MMCX>; + #address-cells = <1>; + #size-cells = <0>; - ports { - #address-cells = <1>; - #size-cells = <0>; + status = "disabled"; - port@0 { - reg = <0>; + cci1_i2c0: i2c-bus@0 { + reg = <0>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; - mdss_intf0_out: endpoint { - remote-endpoint = <&mdss_dp0_in>; - }; - }; + cci1_i2c1: i2c-bus@1 { + reg = <1>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; - port@4 { - reg = <4>; + camss: isp@acb7000 { + compatible = "qcom,x1e80100-camss"; + + reg = <0 0x0acb7000 0 0x2000>, + <0 0x0acb9000 0 0x2000>, + <0 0x0acbb000 0 0x2000>, + <0 0x0acc6000 0 0x1000>, + <0 0x0acca000 0 0x1000>, + <0 0x0acb6000 0 0x1000>, + <0 0x0ace4000 0 0x1000>, + <0 0x0ace6000 0 0x1000>, + <0 0x0ace8000 0 0x1000>, + <0 0x0acec000 0 0x4000>, + <0 0x0acf6000 0 0x1000>, + <0 0x0acf7000 0 0x1000>, + <0 0x0acf8000 0 0x1000>, + <0 0x0ac62000 0 0xf000>, + <0 0x0ac71000 0 0xf000>, + <0 0x0acc7000 0 0x2000>, + <0 0x0accb000 0 0x2000>; + + reg-names = "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csid_wrapper", + "csiphy0", + "csiphy1", + "csiphy2", + "csiphy4", + "csitpg0", + "csitpg1", + "csitpg2", + "vfe0", + "vfe1", + "vfe_lite0", + "vfe_lite1"; + + clocks = <&camcc CAM_CC_CAMNOC_AXI_NRT_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CPAS_IFE_0_CLK>, + <&camcc CAM_CC_CPAS_IFE_1_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>; + + clock-names = "camnoc_nrt_axi", + "camnoc_rt_axi", + "core_ahb", + "cpas_ahb", + "cpas_fast_ahb", + "cpas_vfe0", + "cpas_vfe1", + "cpas_vfe_lite", + "cphy_rx_clk_src", + "csid", + "csid_csiphy_rx", + "csiphy0", + "csiphy0_timer", + "csiphy1", + "csiphy1_timer", + "csiphy2", + "csiphy2_timer", + "csiphy4", + "csiphy4_timer", + "gcc_axi_hf", + "gcc_axi_sf", + "vfe0", + "vfe0_fast_ahb", + "vfe1", + "vfe1_fast_ahb", + "vfe_lite", + "vfe_lite_ahb", + "vfe_lite_cphy_rx", + "vfe_lite_csid"; + + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + ; + + interrupt-names = "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csiphy0", + "csiphy1", + "csiphy2", + "csiphy4", + "vfe0", + "vfe1", + "vfe_lite0", + "vfe_lite1"; - mdss_intf4_out: endpoint { - remote-endpoint = <&mdss_dp1_in>; - }; - }; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_ICP QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "ahb", + "hf_mnoc", + "sf_mnoc", + "sf_icp_mnoc"; - port@5 { - reg = <5>; + iommus = <&apps_smmu 0x800 0x60>, + <&apps_smmu 0x820 0x60>, + <&apps_smmu 0x840 0x60>, + <&apps_smmu 0x860 0x60>, + <&apps_smmu 0x18a0 0x0>; - mdss_intf5_out: endpoint { - remote-endpoint = <&mdss_dp3_in>; - }; - }; + #address-cells = <2>; + #size-cells = <2>; + ranges; - port@6 { - reg = <6>; + phys = <&csiphy0 PHY_QCOM_CSI2_MODE_DPHY>, + <&csiphy1 PHY_QCOM_CSI2_MODE_DPHY>, + <&csiphy2 PHY_QCOM_CSI2_MODE_DPHY>, + <&csiphy4 PHY_QCOM_CSI2_MODE_DPHY>; + phy-names = "csiphy0", + "csiphy1", + "csiphy2", + "csiphy4"; + + power-domains = <&camcc CAM_CC_IFE_0_GDSC>, + <&camcc CAM_CC_IFE_1_GDSC>, + <&camcc CAM_CC_TITAN_TOP_GDSC>; + power-domain-names = "ife0", + "ife1", + "top"; - mdss_intf6_out: endpoint { - remote-endpoint = <&mdss_dp2_in>; - }; - }; - }; + status = "disabled"; - mdp_opp_table: opp-table { - compatible = "operating-points-v2"; + ports { + #address-cells = <1>; + #size-cells = <0>; - opp-200000000 { - opp-hz = /bits/ 64 <200000000>; - required-opps = <&rpmhpd_opp_low_svs>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + camss_csiphy0_inep0: endpoint@0 { + reg = <0>; }; + }; - opp-325000000 { - opp-hz = /bits/ 64 <325000000>; - required-opps = <&rpmhpd_opp_svs>; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + camss_csiphy1_inep0: endpoint@0 { + reg = <0>; }; + }; - opp-375000000 { - opp-hz = /bits/ 64 <375000000>; - required-opps = <&rpmhpd_opp_svs_l1>; + port@2 { + reg = <2>; + #address-cells = <1>; + #size-cells = <0>; + camss_csiphy2_inep0: endpoint@0 { + reg = <0>; + }; + }; + + port@3 { + reg = <3>; + #address-cells = <1>; + #size-cells = <0>; + camss_csiphy4_inep0: endpoint@0 { + reg = <0>; + }; + }; + }; + + csiphy0: phy@ace4000 { + compatible = "qcom,x1e80100-csi2-phy"; + reg = <0 0x0ace4000 0 0x2000>; + + clocks = <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "core", + "timer"; + + operating-points-v2 = <&csiphy_mxc_opp_table>; + + interrupts = ; + + power-domains = <&rpmhpd RPMHPD_MXC>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "mx", + "mmcx"; + + #phy-cells = <1>; + + status = "disabled"; + }; + + csiphy1: phy@ace6000 { + compatible = "qcom,x1e80100-csi2-phy"; + reg = <0 0x0ace6000 0 0x2000>; + + clocks = <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>; + clock-names = "core", + "timer"; + + operating-points-v2 = <&csiphy_mxc_opp_table>; + + interrupts = ; + + power-domains = <&rpmhpd RPMHPD_MXC>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "mx", + "mmcx"; + + #phy-cells = <1>; + + status = "disabled"; + }; + + csiphy2: phy@ace8000 { + compatible = "qcom,x1e80100-csi2-phy"; + reg = <0 0x0ace8000 0 0x2000>; + + clocks = <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>; + clock-names = "core", + "timer"; + + operating-points-v2 = <&csiphy_mxc_opp_table>; + + interrupts = ; + + power-domains = <&rpmhpd RPMHPD_MXC>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "mx", + "mmcx"; + + #phy-cells = <1>; + + status = "disabled"; + }; + + csiphy4: phy@acec000 { + compatible = "qcom,x1e80100-csi2-phy"; + reg = <0 0x0acec000 0 0x2000>; + + clocks = <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>; + clock-names = "core", + "timer"; + + operating-points-v2 = <&csiphy_mxa_opp_table>; + + interrupts = ; + + power-domains = <&rpmhpd RPMHPD_MX>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "mx", + "mmcx"; + + #phy-cells = <1>; + + status = "disabled"; + }; + + csiphy_mxc_opp_table: opp-table-mxc { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>, + <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>, + <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_low_svs>, + <&rpmhpd_opp_low_svs>; + }; + }; + + csiphy_mxa_opp_table: opp-table-mxa { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>, + <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>, + <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_low_svs>, + <&rpmhpd_opp_low_svs>; + }; + }; + }; + + camcc: clock-controller@ade0000 { + compatible = "qcom,x1e80100-camcc"; + reg = <0 0x0ade0000 0 0x20000>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&bi_tcxo_div2>, + <&bi_tcxo_ao_div2>, + <&sleep_clk>; + power-domains = <&rpmhpd RPMHPD_MXC>, + <&rpmhpd RPMHPD_MMCX>; + required-opps = <&rpmhpd_opp_low_svs>, + <&rpmhpd_opp_low_svs>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; + + mdss: display-subsystem@ae00000 { + compatible = "qcom,x1e80100-mdss"; + reg = <0 0x0ae00000 0 0x1000>; + reg-names = "mdss"; + + interrupts = ; + + clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc DISP_CC_MDSS_MDP_CLK>; + + resets = <&dispcc DISP_CC_MDSS_CORE_BCR>; + + interconnects = <&mmss_noc MASTER_MDP QCOM_ICC_TAG_ALWAYS + &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ALWAYS>, + <&mc_virt MASTER_LLCC QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "mdp0-mem", + "mdp1-mem", + "cpu-cfg"; + + power-domains = <&dispcc MDSS_GDSC>; + + iommus = <&apps_smmu 0x1c00 0x2>; + + interrupt-controller; + #interrupt-cells = <1>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + status = "disabled"; + + mdss_mdp: display-controller@ae01000 { + compatible = "qcom,x1e80100-dpu"; + reg = <0 0x0ae01000 0 0x8f000>, + <0 0x0aeb0000 0 0x2008>; + reg-names = "mdp", + "vbif"; + + interrupts-extended = <&mdss 0>; + + clocks = <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&dispcc DISP_CC_MDSS_MDP_LUT_CLK>, + <&dispcc DISP_CC_MDSS_MDP_CLK>, + <&dispcc DISP_CC_MDSS_VSYNC_CLK>; + clock-names = "nrt_bus", + "iface", + "lut", + "core", + "vsync"; + + operating-points-v2 = <&mdp_opp_table>; + + power-domains = <&rpmhpd RPMHPD_MMCX>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mdss_intf0_out: endpoint { + remote-endpoint = <&mdss_dp0_in>; + }; + }; + + port@4 { + reg = <4>; + + mdss_intf4_out: endpoint { + remote-endpoint = <&mdss_dp1_in>; + }; + }; + + port@5 { + reg = <5>; + + mdss_intf5_out: endpoint { + remote-endpoint = <&mdss_dp3_in>; + }; + }; + + port@6 { + reg = <6>; + + mdss_intf6_out: endpoint { + remote-endpoint = <&mdss_dp2_in>; + }; + }; + }; + + mdp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-325000000 { + opp-hz = /bits/ 64 <325000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-375000000 { + opp-hz = /bits/ 64 <375000000>; + required-opps = <&rpmhpd_opp_svs_l1>; }; opp-514000000 { @@ -5817,9 +6467,11 @@ <0 0x0aec2000 0 0x1c8>; clocks = <&dispcc DISP_CC_MDSS_DPTX2_AUX_CLK>, - <&dispcc DISP_CC_MDSS_AHB_CLK>; + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&tcsr TCSR_EDP_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; power-domains = <&rpmhpd RPMHPD_MX>; @@ -5837,9 +6489,11 @@ <0 0x0aec5000 0 0x1c8>; clocks = <&dispcc DISP_CC_MDSS_DPTX3_AUX_CLK>, - <&dispcc DISP_CC_MDSS_AHB_CLK>; + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&tcsr TCSR_EDP_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; power-domains = <&rpmhpd RPMHPD_MX>; @@ -5877,8 +6531,10 @@ pdc: interrupt-controller@b220000 { compatible = "qcom,x1e80100-pdc", "qcom,pdc"; - reg = <0 0x0b220000 0 0x30000>, <0 0x174000f0 0 0x64>; - + reg = <0 0x0b220000 0 0x30000>, + <0 0x174000f0 0 0x64>, + <0 0x0b2045e8 0 0x4>; + qcom,qmp = <&aoss_qmp>; qcom,pdc-ranges = <0 480 42>, <42 251 5>, <47 522 52>, <99 609 32>, <131 717 12>, <143 816 19>; @@ -5961,6 +6617,78 @@ gpio-ranges = <&tlmm 0 0 239>; wakeup-parent = <&pdc>; + cci0_default: cci0-default-state { + cci0_i2c0_default: cci0-i2c0-default-pins { + /* cci_i2c_sda0, cci_i2c_scl0 */ + pins = "gpio101", "gpio102"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-up; + }; + + cci0_i2c1_default: cci0-i2c1-default-pins { + /* cci_i2c_sda1, cci_i2c_scl1 */ + pins = "gpio103", "gpio104"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + cci0_sleep: cci0-sleep-state { + cci0_i2c0_sleep: cci0-i2c0-sleep-pins { + /* cci_i2c_sda0, cci_i2c_scl0 */ + pins = "gpio101", "gpio102"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-down; + }; + + cci0_i2c1_sleep: cci0-i2c1-sleep-pins { + /* cci_i2c_sda1, cci_i2c_scl1 */ + pins = "gpio103", "gpio104"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + cci1_default: cci1-default-state { + cci1_i2c0_default: cci1-i2c0-default-pins { + /* cci_i2c_sda2, cci_i2c_scl2 */ + pins = "gpio105", "gpio106"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-up; + }; + + cci1_i2c1_default: cci1-i2c1-default-pins { + /* aon_cci_i2c_sda3, aon_cci_i2c_scl3 */ + pins = "gpio235", "gpio236"; + function = "aon_cci"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + cci1_sleep: cci1-sleep-state { + cci1_i2c0_sleep: cci1-i2c0-sleep-pins { + /* cci_i2c_sda2, cci_i2c_scl2 */ + pins = "gpio105", "gpio106"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-down; + }; + + cci1_i2c1_sleep: cci1-i2c1-sleep-pins { + /* aon_cci_i2c_sda3, aon_cci_i2c_scl3 */ + pins = "gpio235", "gpio236"; + function = "aon_cci"; + drive-strength = <2>; + bias-pull-down; + }; + }; + edp0_hpd_default: edp0-hpd-default-state { pins = "gpio119"; function = "edp0_hot"; @@ -6634,24 +7362,53 @@ }; }; - stm@10002000 { - compatible = "arm,coresight-stm", "arm,primecell"; - reg = <0x0 0x10002000 0x0 0x1000>, - <0x0 0x16280000 0x0 0x180000>; - reg-names = "stm-base", - "stm-stimulus-base"; + ctcu@10001000 { + compatible = "qcom,hamoa-ctcu", "qcom,sa8775p-ctcu"; + reg = <0x0 0x10001000 0x0 0x1000>; clocks = <&aoss_qmp>; - clock-names = "apb_pclk"; + clock-names = "apb"; - out-ports { - port { - stm_out: endpoint { - remote-endpoint = <&funnel0_in7>; + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + ctcu_in0: endpoint { + remote-endpoint = <&etr0_out>; }; }; - }; - }; + + port@1 { + reg = <1>; + + ctcu_in1: endpoint { + remote-endpoint = <&etr1_out>; + }; + }; + }; + }; + + stm@10002000 { + compatible = "arm,coresight-stm", "arm,primecell"; + reg = <0x0 0x10002000 0x0 0x1000>, + <0x0 0x16280000 0x0 0x180000>; + reg-names = "stm-base", + "stm-stimulus-base"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + out-ports { + port { + stm_out: endpoint { + remote-endpoint = <&funnel0_in7>; + }; + }; + }; + }; tpdm@10003000 { compatible = "qcom,coresight-tpdm", "arm,primecell"; @@ -6785,6 +7542,14 @@ }; }; + port@4 { + reg = <4>; + + funnel1_in4: endpoint { + remote-endpoint = <&apss_funnel_out>; + }; + }; + port@5 { reg = <5>; @@ -6848,6 +7613,122 @@ }; }; + replicator@10046000 { + compatible = "arm,coresight-dynamic-replicator", "arm,primecell"; + reg = <0x0 0x10046000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + port { + qdss_rep_in: endpoint { + remote-endpoint = <&swao_rep_out0>; + }; + }; + }; + + out-ports { + port { + qdss_rep_out0: endpoint { + remote-endpoint = <&etr_rep_in>; + }; + }; + }; + }; + + tmc_etr: tmc@10048000 { + compatible = "arm,coresight-tmc", "arm,primecell"; + reg = <0x0 0x10048000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + iommus = <&apps_smmu 0x04e0 0x0>; + + arm,scatter-gather; + + in-ports { + port { + etr0_in: endpoint { + remote-endpoint = <&etr_rep_out0>; + }; + }; + }; + + out-ports { + port { + etr0_out: endpoint { + remote-endpoint = <&ctcu_in0>; + }; + }; + }; + }; + + replicator@1004e000 { + compatible = "arm,coresight-dynamic-replicator", "arm,primecell"; + reg = <0x0 0x1004e000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + port { + etr_rep_in: endpoint { + remote-endpoint = <&qdss_rep_out0>; + }; + }; + }; + + out-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + etr_rep_out0: endpoint { + remote-endpoint = <&etr0_in>; + }; + }; + + port@1 { + reg = <1>; + + etr_rep_out1: endpoint { + remote-endpoint = <&etr1_in>; + }; + }; + }; + }; + + tmc_etr1: tmc@1004f000 { + compatible = "arm,coresight-tmc", "arm,primecell"; + reg = <0x0 0x1004f000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + iommus = <&apps_smmu 0x0500 0x0>; + + arm,scatter-gather; + arm,buffer-size = <0x400000>; + + in-ports { + port { + etr1_in: endpoint { + remote-endpoint = <&etr_rep_out1>; + }; + }; + }; + + out-ports { + port { + etr1_out: endpoint { + remote-endpoint = <&ctcu_in1>; + }; + }; + }; + }; + tpdm@10800000 { compatible = "qcom,coresight-tpdm", "arm,primecell"; reg = <0x0 0x10800000 0x0 0x1000>; @@ -7161,7 +8042,20 @@ }; out-ports { - port { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + swao_rep_out0: endpoint { + remote-endpoint = <&qdss_rep_in>; + }; + }; + + port@1 { + reg = <1>; + swao_rep_out1: endpoint { remote-endpoint = <&eud_in>; }; @@ -8075,127 +8969,1004 @@ }; }; - apps_smmu: iommu@15000000 { - compatible = "qcom,x1e80100-smmu-500", "qcom,smmu-500", "arm,mmu-500"; - reg = <0 0x15000000 0 0x100000>; + apss_funnel: funnel@12080000 { + compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; + reg = <0x0 0x12080000 0x0 0x1000>; - interrupts = , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - #iommu-cells = <2>; - #global-interrupts = <1>; + in-ports { + #address-cells = <1>; + #size-cells = <0>; - dma-coherent; - }; + port@0 { + reg = <0>; - pcie_smmu: iommu@15400000 { - compatible = "arm,smmu-v3"; - reg = <0 0x15400000 0 0x80000>; - #iommu-cells = <1>; - interrupts = , - , - ; - interrupt-names = "eventq", - "gerror", - "cmdq-sync"; - dma-coherent; - status = "reserved"; /* Controlled by Gunyah. */ - }; + apss_funnel_in0: endpoint { + remote-endpoint = <&ncc0_etf_out>; + }; + }; + + port@1 { + reg = <1>; + + apss_funnel_in1: endpoint { + remote-endpoint = <&ncc1_etf_out>; + }; + }; + + port@2 { + reg = <2>; + + apss_funnel_in2: endpoint { + remote-endpoint = <&ncc2_etf_out>; + }; + }; + }; + + out-ports { + port { + apss_funnel_out: endpoint { + remote-endpoint = + <&funnel1_in4>; + }; + }; + }; + }; + + etm@13021000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu0>; + qcom,skip-power-up; + + out-ports { + port { + etm0_out: endpoint { + remote-endpoint = <&ncc0_0_rep_in>; + }; + }; + }; + }; + + etm@13121000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu1>; + qcom,skip-power-up; + + out-ports { + port { + etm1_out: endpoint { + remote-endpoint = <&ncc0_1_rep_in>; + }; + }; + }; + }; + + etm@13221000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu2>; + qcom,skip-power-up; + + out-ports { + port { + etm2_out: endpoint { + remote-endpoint = <&ncc0_2_rep_in>; + }; + }; + }; + }; + + etm@13321000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu3>; + qcom,skip-power-up; + + out-ports { + port { + etm3_out: endpoint { + remote-endpoint = <&ncc0_3_rep_in>; + }; + }; + }; + }; + + funnel@13401000 { + compatible = "arm,coresight-cpu-funnel"; + reg = <0x0 0x13401000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd0>; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + reg = <2>; + + ncc0_2_funnel_in2: endpoint { + remote-endpoint = <&ncc0_1_funnel_out>; + }; + }; + }; + + out-ports { + port { + ncc0_2_funnel_out: endpoint { + remote-endpoint = <&ncc0_etf_in>; + }; + }; + }; + }; + + tmc@13409000 { + compatible = "arm,coresight-cpu-tmc"; + reg = <0x0 0x13409000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd0>; + + in-ports { + port { + ncc0_etf_in: endpoint { + remote-endpoint = <&ncc0_2_funnel_out>; + }; + }; + }; + + out-ports { + port { + ncc0_etf_out: endpoint { + remote-endpoint = <&apss_funnel_in0>; + }; + }; + }; + }; + + replicator@13490000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x13490000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd0>; + + in-ports { + port { + ncc0_0_rep_in: endpoint { + remote-endpoint = <&etm0_out>; + }; + }; + }; + + out-ports { + port { + ncc0_0_rep_out: endpoint { + remote-endpoint = <&ncc0_1_funnel_in0>; + }; + }; + }; + }; + + replicator@134a0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x134a0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd0>; + + in-ports { + port { + ncc0_1_rep_in: endpoint { + remote-endpoint = <&etm1_out>; + }; + }; + }; + + out-ports { + port { + ncc0_1_rep_out: endpoint { + remote-endpoint = <&ncc0_1_funnel_in1>; + }; + }; + }; + }; + + replicator@134b0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x134b0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd0>; + + in-ports { + port { + ncc0_2_rep_in: endpoint { + remote-endpoint = <&etm2_out>; + }; + }; + }; + + out-ports { + port { + ncc0_2_rep_out: endpoint { + remote-endpoint = <&ncc0_1_funnel_in2>; + }; + }; + }; + }; + + replicator@134c0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x134c0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd0>; + + in-ports { + port { + ncc0_3_rep_in: endpoint { + remote-endpoint = <&etm3_out>; + }; + }; + }; + + out-ports { + port { + ncc0_3_rep_out: endpoint { + remote-endpoint = <&ncc0_1_funnel_in3>; + }; + }; + }; + }; + + funnel@134d0000 { + compatible = "arm,coresight-cpu-funnel"; + reg = <0x0 0x134d0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd0>; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + ncc0_1_funnel_in0: endpoint { + remote-endpoint = <&ncc0_0_rep_out>; + }; + }; + + port@1 { + reg = <1>; + + ncc0_1_funnel_in1: endpoint { + remote-endpoint = <&ncc0_1_rep_out>; + }; + }; + + port@2 { + reg = <2>; + + ncc0_1_funnel_in2: endpoint { + remote-endpoint = <&ncc0_2_rep_out>; + }; + }; + + port@3 { + reg = <3>; + + ncc0_1_funnel_in3: endpoint { + remote-endpoint = <&ncc0_3_rep_out>; + }; + }; + }; + + out-ports { + port { + ncc0_1_funnel_out: endpoint { + remote-endpoint = <&ncc0_2_funnel_in2>; + }; + }; + }; + }; + + etm@13521000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu4>; + qcom,skip-power-up; + + out-ports { + port { + etm4_out: endpoint { + remote-endpoint = <&ncc1_0_rep_in>; + }; + }; + }; + }; + + etm@13621000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu5>; + qcom,skip-power-up; + + out-ports { + port { + etm5_out: endpoint { + remote-endpoint = <&ncc1_1_rep_in>; + }; + }; + }; + }; + + etm@13721000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu6>; + qcom,skip-power-up; + + out-ports { + port { + etm6_out: endpoint { + remote-endpoint = <&ncc1_2_rep_in>; + }; + }; + }; + }; + + etm@13821000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu7>; + qcom,skip-power-up; + + out-ports { + port { + etm7_out: endpoint { + remote-endpoint = <&ncc1_3_rep_in>; + }; + }; + }; + }; + + funnel@13901000 { + compatible = "arm,coresight-cpu-funnel"; + reg = <0x0 0x13901000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd1>; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + reg = <2>; + + ncc1_2_funnel_in2: endpoint { + remote-endpoint = <&ncc1_1_funnel_out>; + }; + }; + }; + + out-ports { + port { + ncc1_2_funnel_out: endpoint { + remote-endpoint = <&ncc1_etf_in>; + }; + }; + }; + }; + + tmc@13909000 { + compatible = "arm,coresight-cpu-tmc"; + reg = <0x0 0x13909000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd1>; + + in-ports { + port { + ncc1_etf_in: endpoint { + remote-endpoint = <&ncc1_2_funnel_out>; + }; + }; + }; + + out-ports { + port { + ncc1_etf_out: endpoint { + remote-endpoint = <&apss_funnel_in1>; + }; + }; + }; + }; + + replicator@13990000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x13990000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd1>; + + in-ports { + port { + ncc1_0_rep_in: endpoint { + remote-endpoint = <&etm4_out>; + }; + }; + }; + + out-ports { + port { + ncc1_0_rep_out: endpoint { + remote-endpoint = <&ncc1_1_funnel_in0>; + }; + }; + }; + }; + + replicator@139a0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x139a0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd1>; + + in-ports { + port { + ncc1_1_rep_in: endpoint { + remote-endpoint = <&etm5_out>; + }; + }; + }; + + out-ports { + port { + ncc1_1_rep_out: endpoint { + remote-endpoint = <&ncc1_1_funnel_in1>; + }; + }; + }; + }; + + replicator@139b0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x139b0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd1>; + + in-ports { + port { + ncc1_2_rep_in: endpoint { + remote-endpoint = <&etm6_out>; + }; + }; + }; + + out-ports { + port { + ncc1_2_rep_out: endpoint { + remote-endpoint = <&ncc1_1_funnel_in2>; + }; + }; + }; + }; + + replicator@139c0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x139c0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd1>; + + in-ports { + port { + ncc1_3_rep_in: endpoint { + remote-endpoint = <&etm7_out>; + }; + }; + }; + + out-ports { + port { + ncc1_3_rep_out: endpoint { + remote-endpoint = <&ncc1_1_funnel_in3>; + }; + }; + }; + }; + + funnel@139d0000 { + compatible = "arm,coresight-cpu-funnel"; + reg = <0x0 0x139d0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd1>; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + ncc1_1_funnel_in0: endpoint { + remote-endpoint = <&ncc1_0_rep_out>; + }; + }; + + port@1 { + reg = <1>; + + ncc1_1_funnel_in1: endpoint { + remote-endpoint = <&ncc1_1_rep_out>; + }; + }; + + port@2 { + reg = <2>; + + ncc1_1_funnel_in2: endpoint { + remote-endpoint = <&ncc1_2_rep_out>; + }; + }; + + port@3 { + reg = <3>; + + ncc1_1_funnel_in3: endpoint { + remote-endpoint = <&ncc1_3_rep_out>; + }; + }; + }; + + out-ports { + port { + ncc1_1_funnel_out: endpoint { + remote-endpoint = <&ncc1_2_funnel_in2>; + }; + }; + }; + }; + + etm8: etm@13a21000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu8>; + qcom,skip-power-up; + + out-ports { + port { + etm8_out: endpoint { + remote-endpoint = <&ncc2_0_rep_in>; + }; + }; + }; + }; + + etm9: etm@13b21000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu9>; + qcom,skip-power-up; + + out-ports { + port { + etm9_out: endpoint { + remote-endpoint = <&ncc2_1_rep_in>; + }; + }; + }; + }; + + etm10: etm@13c21000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu10>; + qcom,skip-power-up; + + out-ports { + port { + etm10_out: endpoint { + remote-endpoint = <&ncc2_2_rep_in>; + }; + }; + }; + }; + + etm11: etm@13d21000 { + compatible = "arm,coresight-etm4x-sysreg"; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + cpu = <&cpu11>; + qcom,skip-power-up; + + out-ports { + port { + etm11_out: endpoint { + remote-endpoint = <&ncc2_3_rep_in>; + }; + }; + }; + }; + + cluster2_funnel_l2: funnel@13e01000 { + compatible = "arm,coresight-cpu-funnel"; + reg = <0x0 0x13e01000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd2>; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + reg = <2>; + + ncc2_2_funnel_in2: endpoint { + remote-endpoint = <&ncc2_1_funnel_out>; + }; + }; + }; + + out-ports { + port { + ncc2_2_funnel_out: endpoint { + remote-endpoint = <&ncc2_etf_in>; + }; + }; + }; + }; + + cluster2_etf: tmc@13e09000 { + compatible = "arm,coresight-cpu-tmc"; + reg = <0x0 0x13e09000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd2>; + + in-ports { + port { + ncc2_etf_in: endpoint { + remote-endpoint = <&ncc2_2_funnel_out>; + }; + }; + }; + + out-ports { + port { + ncc2_etf_out: endpoint { + remote-endpoint = <&apss_funnel_in2>; + }; + }; + }; + }; + + cluster2_rep_2_0: replicator@13e90000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x13e90000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd2>; + + in-ports { + port { + ncc2_0_rep_in: endpoint { + remote-endpoint = <&etm8_out>; + }; + }; + }; + + out-ports { + port { + ncc2_0_rep_out: endpoint { + remote-endpoint = <&ncc2_1_funnel_in0>; + }; + }; + }; + }; + + cluster2_rep_2_1: replicator@13ea0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x13ea0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd2>; + + in-ports { + port { + ncc2_1_rep_in: endpoint { + remote-endpoint = <&etm9_out>; + }; + }; + }; + + out-ports { + port { + ncc2_1_rep_out: endpoint { + remote-endpoint = <&ncc2_1_funnel_in1>; + }; + }; + }; + }; + + cluster2_rep_2_2: replicator@13eb0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x13eb0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd2>; + + in-ports { + port { + ncc2_2_rep_in: endpoint { + remote-endpoint = <&etm10_out>; + }; + }; + }; + + out-ports { + port { + ncc2_2_rep_out: endpoint { + remote-endpoint = <&ncc2_1_funnel_in2>; + }; + }; + }; + }; + + cluster2_rep_2_3: replicator@13ec0000 { + compatible = "arm,coresight-cpu-replicator"; + reg = <0x0 0x13ec0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd2>; + + in-ports { + port { + ncc2_3_rep_in: endpoint { + remote-endpoint = <&etm11_out>; + }; + }; + }; + + out-ports { + port { + ncc2_3_rep_out: endpoint { + remote-endpoint = <&ncc2_1_funnel_in3>; + }; + }; + }; + }; + + cluster2_funnel_l1: funnel@13ed0000 { + compatible = "arm,coresight-cpu-funnel"; + reg = <0x0 0x13ed0000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + power-domains = <&cluster_pd2>; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + ncc2_1_funnel_in0: endpoint { + remote-endpoint = <&ncc2_0_rep_out>; + }; + }; + + port@1 { + reg = <1>; + + ncc2_1_funnel_in1: endpoint { + remote-endpoint = <&ncc2_1_rep_out>; + }; + }; + + port@2 { + reg = <2>; + + ncc2_1_funnel_in2: endpoint { + remote-endpoint = <&ncc2_2_rep_out>; + }; + }; + + port@3 { + reg = <3>; + + ncc2_1_funnel_in3: endpoint { + remote-endpoint = <&ncc2_3_rep_out>; + }; + }; + }; + + out-ports { + port { + ncc2_1_funnel_out: endpoint { + remote-endpoint = <&ncc2_2_funnel_in2>; + }; + }; + }; + }; + + apps_smmu: iommu@15000000 { + compatible = "qcom,x1e80100-smmu-500", "qcom,smmu-500", "arm,mmu-500"; + reg = <0 0x15000000 0 0x100000>; + + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + + #iommu-cells = <2>; + #global-interrupts = <1>; + + dma-coherent; + }; + + pcie_smmu: iommu@15400000 { + compatible = "arm,smmu-v3"; + reg = <0 0x15400000 0 0x80000>; + #iommu-cells = <1>; + interrupts = , + , + ; + interrupt-names = "eventq", + "gerror", + "cmdq-sync"; + dma-coherent; + status = "reserved"; /* Controlled by Gunyah. */ + }; intc: interrupt-controller@17000000 { compatible = "arm,gic-v3"; @@ -8223,6 +9994,14 @@ }; }; + apss_watchdog: watchdog@17410000 { + compatible = "qcom,apss-wdt-x1e80100", "qcom,kpss-wdt"; + reg = <0x0 0x17410000 0x0 0x1000>; + clocks = <&sleep_clk>; + interrupts = ; + status = "reserved"; /* Reserved by Gunyah */ + }; + cpucp_mbox: mailbox@17430000 { compatible = "qcom,x1e80100-cpucp-mbox"; reg = <0 0x17430000 0 0x10000>, <0 0x18830000 0 0x10000>; diff --git a/arch/arm64/boot/dts/qcom/ipq5424.dtsi b/arch/arm64/boot/dts/qcom/ipq5424.dtsi index ef2b52f3597d9..227d5ce297515 100644 --- a/arch/arm64/boot/dts/qcom/ipq5424.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq5424.dtsi @@ -213,7 +213,7 @@ }; tfa@8a832000 { - reg = <0x0 0x8a832000 0x0 0x7d000>; + reg = <0x0 0x8a832000 0x0 0x80000>; no-map; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/qcom/kaanapali-ipcc.h b/arch/arm64/boot/dts/qcom/kaanapali-ipcc.h new file mode 100644 index 0000000000000..125375a4aac0a --- /dev/null +++ b/arch/arm64/boot/dts/qcom/kaanapali-ipcc.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __DTS_KAANAPALI_MAILBOX_IPCC_H +#define __DTS_KAANAPALI_MAILBOX_IPCC_H + +/* Physical client IDs */ +#define IPCC_MPROC_AOP 0 +#define IPCC_MPROC_TZ 1 +#define IPCC_MPROC_MPSS 2 +#define IPCC_MPROC_LPASS 3 +#define IPCC_MPROC_SDC 4 +#define IPCC_MPROC_CDSP 5 +#define IPCC_MPROC_APSS 6 +#define IPCC_MPROC_SOCCP 13 +#define IPCC_MPROC_DCP 14 +#define IPCC_MPROC_SPSS 15 +#define IPCC_MPROC_TME 16 +#define IPCC_MPROC_WPSS 17 + +#define IPCC_COMPUTE_L0_CDSP 2 +#define IPCC_COMPUTE_L0_APSS 3 +#define IPCC_COMPUTE_L0_GPU 4 +#define IPCC_COMPUTE_L0_CVP 8 +#define IPCC_COMPUTE_L0_CAM 9 +#define IPCC_COMPUTE_L0_CAM1 10 +#define IPCC_COMPUTE_L0_DCP 11 +#define IPCC_COMPUTE_L0_VPU 12 +#define IPCC_COMPUTE_L0_SOCCP 16 + +#define IPCC_COMPUTE_L1_CDSP 2 +#define IPCC_COMPUTE_L1_APSS 3 +#define IPCC_COMPUTE_L1_GPU 4 +#define IPCC_COMPUTE_L1_CVP 8 +#define IPCC_COMPUTE_L1_CAM 9 +#define IPCC_COMPUTE_L1_CAM1 10 +#define IPCC_COMPUTE_L1_DCP 11 +#define IPCC_COMPUTE_L1_VPU 12 +#define IPCC_COMPUTE_L1_SOCCP 16 + +#define IPCC_PERIPH_CDSP 2 +#define IPCC_PERIPH_APSS 3 +#define IPCC_PERIPH_PCIE0 4 +#define IPCC_PERIPH_PCIE1 5 + +#define IPCC_FENCE_CDSP 2 +#define IPCC_FENCE_APSS 3 +#define IPCC_FENCE_GPU 4 +#define IPCC_FENCE_CVP 8 +#define IPCC_FENCE_CAM 8 +#define IPCC_FENCE_CAM1 10 +#define IPCC_FENCE_DCP 11 +#define IPCC_FENCE_VPU 20 +#define IPCC_FENCE_SOCCP 24 + +#endif diff --git a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts new file mode 100644 index 0000000000000..9cf3158e27124 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts @@ -0,0 +1,704 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; + +#include +#include "kaanapali.dtsi" + +/ { + model = "Qualcomm Technologies, Inc. Kaanapali MTP"; + compatible = "qcom,kaanapali-mtp", "qcom,kaanapali"; + chassis-type = "handset"; + + aliases { + serial0 = &uart7; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + clocks { + xo_board: xo-board { + compatible = "fixed-clock"; + clock-frequency = <76800000>; + #clock-cells = <0>; + }; + + sleep_clk: sleep-clk { + compatible = "fixed-clock"; + clock-frequency = <32764>; + #clock-cells = <0>; + }; + + bi_tcxo_div2: bi-tcxo-div2-clk { + compatible = "fixed-factor-clock"; + #clock-cells = <0>; + + clocks = <&rpmhcc RPMH_CXO_CLK>; + clock-mult = <1>; + clock-div = <2>; + }; + + bi_tcxo_ao_div2: bi-tcxo-ao-div2-clk { + compatible = "fixed-factor-clock"; + #clock-cells = <0>; + + clocks = <&rpmhcc RPMH_CXO_CLK_A>; + clock-mult = <1>; + clock-div = <2>; + }; + }; +}; + +&apps_rsc { + regulators-0 { + compatible = "qcom,pmh0101-rpmh-regulators"; + + qcom,pmic-id = "B_E0"; + + vreg_bob1: bob1 { + regulator-name = "vreg_bob1"; + regulator-min-microvolt = <3008000>; + regulator-max-microvolt = <4000000>; + regulator-initial-mode = ; + }; + + vreg_bob2: bob2 { + regulator-name = "vreg_bob2"; + regulator-min-microvolt = <2704000>; + regulator-max-microvolt = <3552000>; + regulator-initial-mode = ; + }; + + vreg_l1b_1p8: ldo1 { + regulator-name = "vreg_l1b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2b_3p0: ldo2 { + regulator-name = "vreg_l2b_3p0"; + regulator-min-microvolt = <3008000>; + regulator-max-microvolt = <3048000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l4b_1p8: ldo4 { + regulator-name = "vreg_l4b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l5b_3p1: ldo5 { + regulator-name = "vreg_l5b_3p1"; + regulator-min-microvolt = <3100000>; + regulator-max-microvolt = <3148000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l6b_1p8: ldo6 { + regulator-name = "vreg_l6b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l7b_1p8: ldo7 { + regulator-name = "vreg_l7b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l8b_1p8: ldo8 { + regulator-name = "vreg_l8b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l9b_2p9: ldo9 { + regulator-name = "vreg_l9b_2p9"; + regulator-min-microvolt = <2960000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l10b_1p8: ldo10 { + regulator-name = "vreg_l10b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l11b_1p0: ldo11 { + regulator-name = "vreg_l11b_1p0"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1292000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l12b_1p8: ldo12 { + regulator-name = "vreg_l12b_1p8"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l13b_3p0: ldo13 { + regulator-name = "vreg_l13b_3p0"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l14b_3p2: ldo14 { + regulator-name = "vreg_l14b_3p2"; + regulator-min-microvolt = <3200000>; + regulator-max-microvolt = <3200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l15b_1p8: ldo15 { + regulator-name = "vreg_l15b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l17b_2p5: ldo17 { + regulator-name = "vreg_l17b_2p5"; + regulator-min-microvolt = <2504000>; + regulator-max-microvolt = <2504000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l18b_1p2: ldo18 { + regulator-name = "vreg_l18b_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-1 { + compatible = "qcom,pmh0110-rpmh-regulators"; + + qcom,pmic-id = "D_E0"; + + vreg_s10d_1p0: smps10 { + regulator-name = "vreg_s10d_1p0"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + }; + + vreg_l1d_1p2: ldo1 { + regulator-name = "vreg_l1d_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1256000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2d_0p9: ldo2 { + regulator-name = "vreg_l2d_0p9"; + regulator-min-microvolt = <912000>; + regulator-max-microvolt = <958000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3d_0p8: ldo3 { + regulator-name = "vreg_l3d_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <880000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l4d_1p2: ldo4 { + regulator-name = "vreg_l4d_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-2 { + compatible = "qcom,pmh0110-rpmh-regulators"; + + qcom,pmic-id = "F_E0"; + + vreg_s6f_0p5: smps6 { + regulator-name = "vreg_s6f_0p5"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <570000>; + regulator-initial-mode = ; + }; + + vreg_s7f_1p2: smps7 { + regulator-name = "vreg_s7f_1p2"; + regulator-min-microvolt = <1224000>; + regulator-max-microvolt = <1372000>; + regulator-initial-mode = ; + }; + + vreg_s8f_1p8: smps8 { + regulator-name = "vreg_s8f_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2000000>; + regulator-initial-mode = ; + }; + + vreg_l1f_1p2: ldo1 { + regulator-name = "vreg_l1f_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2f_1p2: ldo2 { + regulator-name = "vreg_l2f_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3f_0p8: ldo3 { + regulator-name = "vreg_l3f_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <936000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l4f_0p8: ldo4 { + regulator-name = "vreg_l4f_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-3 { + compatible = "qcom,pmh0110-rpmh-regulators"; + + qcom,pmic-id = "G_E0"; + + vreg_s7g_0p9: smps7 { + regulator-name = "vreg_s7g_0p9"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + }; + + vreg_s9g_1p0: smps9 { + regulator-name = "vreg_s9g_1p0"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + }; + + vreg_l1g_1p2: ldo1 { + regulator-name = "vreg_l1g_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2g_1p8: ldo2 { + regulator-name = "vreg_l2g_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3g_1p2: ldo3 { + regulator-name = "vreg_l3g_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l4g_0p9: ldo4 { + regulator-name = "vreg_l4g_0p9"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-4 { + compatible = "qcom,pmh0110-rpmh-regulators"; + + qcom,pmic-id = "I_E0"; + + vreg_s7i_0p9: smps7 { + regulator-name = "vreg_s7i_0p9"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <972000>; + regulator-initial-mode = ; + }; + + vreg_l2i_1p2: ldo2 { + regulator-name = "vreg_l2i_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3i_0p8: ldo3 { + regulator-name = "vreg_l3i_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-5 { + compatible = "qcom,pmh0104-rpmh-regulators"; + + qcom,pmic-id = "J_E1"; + + vreg_s1j_0p8: smps1 { + regulator-name = "vreg_s1j_0p8"; + regulator-min-microvolt = <400000>; + regulator-max-microvolt = <1000000>; + regulator-initial-mode = ; + }; + + vreg_s2j_0p8: smps2 { + regulator-name = "vreg_s2j_0p8"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + }; + + vreg_s3j_1p2: smps3 { + regulator-name = "vreg_s3j_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1300000>; + regulator-initial-mode = ; + }; + + vreg_s4j_0p7: smps4 { + regulator-name = "vreg_s4j_0p7"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1000000>; + regulator-initial-mode = ; + }; + }; + + regulators-6 { + compatible = "qcom,pmr735d-rpmh-regulators"; + + qcom,pmic-id = "K_E1"; + + vreg_l1k_0p8: ldo1 { + regulator-name = "vreg_l1k_0p8"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2k_0p7: ldo2 { + regulator-name = "vreg_l2k_0p7"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1000000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3k_1p2: ldo3 { + regulator-name = "vreg_l3k_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l4k_1p0: ldo4 { + regulator-name = "vreg_l4k_1p0"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l5k_0p7: ldo5 { + regulator-name = "vreg_l5k_0p7"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1000000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l6k_1p7: ldo6 { + regulator-name = "vreg_l6k_1p7"; + regulator-min-microvolt = <1700000>; + regulator-max-microvolt = <2000000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l7k_0p7: ldo7 { + regulator-name = "vreg_l7k_0p7"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <848000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-7 { + compatible = "qcom,pm8010-rpmh-regulators"; + + qcom,pmic-id = "M_E1"; + + vreg_l1m_1p0: ldo1 { + regulator-name = "vreg_l1m_1p0"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2m_1p0: ldo2 { + regulator-name = "vreg_l2m_1p0"; + regulator-min-microvolt = <1096000>; + regulator-max-microvolt = <1104000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3m_2p8: ldo3 { + regulator-name = "vreg_l3m_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2900000>; + regulator-initial-mode = ; + }; + + vreg_l4m_2p2: ldo4 { + regulator-name = "vreg_l4m_2p2"; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-initial-mode = ; + }; + + vreg_l6m_2p8: ldo6 { + regulator-name = "vreg_l6m_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + }; + + vreg_l7m_2p8: ldo7 { + regulator-name = "vreg_l7m_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + }; + }; + + regulators-8 { + compatible = "qcom,pm8010-rpmh-regulators"; + + qcom,pmic-id = "N_E1"; + + vreg_l1n_1p1: ldo1 { + regulator-name = "vreg_l1n_1p1"; + regulator-min-microvolt = <1096000>; + regulator-max-microvolt = <1104000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2n_1p2: ldo2 { + regulator-name = "vreg_l2n_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3n_1p8: ldo3 { + regulator-name = "vreg_l3n_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l4n_1p8: ldo4 { + regulator-name = "vreg_l4n_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l5n_2p8: ldo5 { + regulator-name = "vreg_l5n_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + }; + + vreg_l6n_2p8: ldo6 { + regulator-name = "vreg_l6n_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + }; + + vreg_l7n_3p3: ldo7 { + regulator-name = "vreg_l7n_3p3"; + regulator-min-microvolt = <3304000>; + regulator-max-microvolt = <3304000>; + regulator-initial-mode = ; + }; + }; +}; + +&tlmm { + gpio-reserved-ranges = <36 4>, /* NFC eSE SPI */ + <74 1>, /* eSE */ + <119 2>, /* SoCCP */ + <144 4>; /* CXM UART */ +}; + +&uart7 { + status = "okay"; +}; + +&ufs_mem_hc { + reset-gpios = <&tlmm 217 GPIO_ACTIVE_LOW>; + + vcc-supply = <&vreg_l17b_2p5>; + vcc-max-microamp = <1200000>; + vccq-supply = <&vreg_l4d_1p2>; + vccq-max-microamp = <1200000>; + + status = "okay"; +}; + +&ufs_mem_phy { + vdda-phy-supply = <&vreg_l4g_0p9>; + vdda-pll-supply = <&vreg_l1d_1p2>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi new file mode 100644 index 0000000000000..12ee29fe64507 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi @@ -0,0 +1,1324 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/ { + interrupt-parent = <&intc>; + + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x0>; + enable-method = "psci"; + next-level-cache = <&l2_0>; + power-domains = <&cpu_pd0>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 0>; + + l2_0: l2-cache { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + }; + + cpu1: cpu@100 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x100>; + enable-method = "psci"; + next-level-cache = <&l2_0>; + power-domains = <&cpu_pd1>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 0>; + }; + + cpu2: cpu@200 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x200>; + enable-method = "psci"; + next-level-cache = <&l2_0>; + power-domains = <&cpu_pd2>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 0>; + }; + + cpu3: cpu@300 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x300>; + enable-method = "psci"; + next-level-cache = <&l2_0>; + power-domains = <&cpu_pd3>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 0>; + }; + + cpu4: cpu@400 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x400>; + enable-method = "psci"; + next-level-cache = <&l2_0>; + power-domains = <&cpu_pd4>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 0>; + }; + + cpu5: cpu@500 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x500>; + enable-method = "psci"; + next-level-cache = <&l2_0>; + power-domains = <&cpu_pd5>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 0>; + }; + + cpu6: cpu@10000 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10000>; + enable-method = "psci"; + next-level-cache = <&l2_1>; + power-domains = <&cpu_pd6>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 1>; + + l2_1: l2-cache { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + }; + + cpu7: cpu@10100 { + device_type = "cpu"; + compatible = "qcom,oryon"; + reg = <0x0 0x10100>; + enable-method = "psci"; + next-level-cache = <&l2_1>; + power-domains = <&cpu_pd7>; + power-domain-names = "psci"; + clocks = <&pdp_scmi_perf 1>; + }; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + + core1 { + cpu = <&cpu1>; + }; + + core2 { + cpu = <&cpu2>; + }; + + core3 { + cpu = <&cpu3>; + }; + + core4 { + cpu = <&cpu4>; + }; + + core5 { + cpu = <&cpu5>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu6>; + }; + + core1 { + cpu = <&cpu7>; + }; + }; + }; + + idle-states { + entry-method = "psci"; + + cluster0_c4: cpu-sleep-0 { + compatible = "arm,idle-state"; + idle-state-name = "retention"; + arm,psci-suspend-param = <0x00000004>; + entry-latency-us = <93>; + exit-latency-us = <129>; + min-residency-us = <560>; + }; + + cluster1_c4: cpu-sleep-1 { + compatible = "arm,idle-state"; + idle-state-name = "retention"; + arm,psci-suspend-param = <0x00000004>; + entry-latency-us = <172>; + exit-latency-us = <130>; + min-residency-us = <686>; + }; + }; + + domain-idle-states { + cluster_cl5: cluster-sleep-0 { + compatible = "domain-idle-state"; + arm,psci-suspend-param = <0x01000054>; + entry-latency-us = <2150>; + exit-latency-us = <1983>; + min-residency-us = <9144>; + }; + + domain_ss3: domain-sleep-0 { + compatible = "domain-idle-state"; + arm,psci-suspend-param = <0x0200c354>; + entry-latency-us = <2800>; + exit-latency-us = <4400>; + min-residency-us = <10150>; + }; + }; + }; + + firmware { + scm: scm { + compatible = "qcom,scm-kaanapali", "qcom,scm"; + qcom,dload-mode = <&tcsr 0x19000>; + }; + + scmi: scmi { + compatible = "arm,scmi"; + mboxes = <&pdp0_mbox 0>, <&pdp0_mbox 1>; + mbox-names = "tx", "rx"; + shmem = <&pdp_tx>, <&pdp_rx>; + + #address-cells = <1>; + #size-cells = <0>; + + pdp_scmi_perf: protocol@13 { + reg = <0x13>; + #clock-cells = <1>; + }; + }; + }; + + clk_virt: interconnect-0 { + compatible = "qcom,kaanapali-clk-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + mc_virt: interconnect-1 { + compatible = "qcom,kaanapali-mc-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + memory@a0000000 { + device_type = "memory"; + /* We expect the bootloader to fill in the size */ + reg = <0x0 0xa0000000 0x0 0x0>; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = ; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + + cpu_pd0: power-domain-cpu0 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster0_c4>; + }; + + cpu_pd1: power-domain-cpu1 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster0_c4>; + }; + + cpu_pd2: power-domain-cpu2 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster0_c4>; + }; + + cpu_pd3: power-domain-cpu3 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster0_c4>; + }; + + cpu_pd4: power-domain-cpu4 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster0_c4>; + }; + + cpu_pd5: power-domain-cpu5 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster0_c4>; + }; + + cpu_pd6: power-domain-cpu6 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster1_c4>; + }; + + cpu_pd7: power-domain-cpu7 { + #power-domain-cells = <0>; + power-domains = <&cluster_pd>; + domain-idle-states = <&cluster1_c4>; + }; + + cluster_pd: power-domain-cluster { + #power-domain-cells = <0>; + domain-idle-states = <&cluster_cl5>; + power-domains = <&system_pd>; + }; + + system_pd: power-domain-system { + #power-domain-cells = <0>; + domain-idle-states = <&domain_ss3>; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + pdp_mem: pdp_region@81300000 { + reg = <0x0 0x81300000 0x0 0x100000>; + no-map; + }; + + aop_cmd_db_mem: aop-cmd-db@81c60000 { + compatible = "qcom,cmd-db"; + reg = <0x0 0x81c60000 0x0 0x20000>; + no-map; + }; + + smem_mem: smem@81d00000 { + compatible = "qcom,smem"; + reg = <0x0 0x81d00000 0x0 0x200000>; + hwlocks = <&tcsr_mutex 3>; + no-map; + }; + + pdp_ns_shared_mem: pdp_ns_shared_region@81f00000 { + reg = <0x0 0x81f00000 0x0 0x100000>; + no-map; + }; + + dsm_partition_1_mem: dsm-partition-1@84a00000 { + reg = <0x0 0x84a00000 0x0 0x5500000>; + no-map; + }; + + dsm_partition_2_mem: dsm-partition-2@89f00000 { + reg = <0x0 0x89f00000 0x0 0xa80000>; + no-map; + }; + + mpss_mem: mpss@8aa00000 { + reg = <0x0 0x8aa00000 0x0 0xeb00000>; + no-map; + }; + + q6_mpss_dtb_mem: q6-mpss-dtb@99500000 { + reg = <0x0 0x99500000 0x0 0x80000>; + no-map; + }; + + ipa_fw_mem: ipa-fw@99580000 { + reg = <0x0 0x99580000 0x0 0x10000>; + no-map; + }; + + ipa_gsi_mem: ipa-gsi@99590000 { + reg = <0x0 0x99590000 0x0 0xa000>; + no-map; + }; + + gpu_microcode_mem: gpu-microcode@9959a000 { + reg = <0x0 0x9959a000 0x0 0x2000>; + no-map; + }; + + camera_mem: camera@99600000 { + reg = <0x0 0x99600000 0x0 0x800000>; + no-map; + }; + + camera_2_mem: camera-2@99e00000 { + reg = <0x0 0x99e00000 0x0 0x800000>; + no-map; + }; + + video_mem: video@9a600000 { + reg = <0x0 0x9a600000 0x0 0x800000>; + no-map; + }; + + cvp_mem: cvp@9ae00000 { + reg = <0x0 0x9ae00000 0x0 0x700000>; + no-map; + }; + + cdsp_mem: cdsp@9b500000 { + reg = <0x0 0x9b500000 0x0 0x1900000>; + no-map; + }; + + q6_cdsp_dtb_mem: q6-cdsp-dtb@9ce00000 { + reg = <0x0 0x9ce00000 0x0 0x80000>; + no-map; + }; + + soccp_mem: soccp@a03d0000 { + reg = <0x0 0xa03d0000 0x0 0x500000>; + no-map; + }; + + soccp_dtb_mem: soccp-dtb@a08d0000 { + reg = <0x0 0xa08d0000 0x0 0x40000>; + no-map; + }; + + q6_adsp_dtb_mem: q6-adsp-dtb@a1380000 { + reg = <0x0 0xa1380000 0x0 0x80000>; + no-map; + }; + + adspslpi_mem: adspslpi@a1400000 { + reg = <0x0 0xa1400000 0x0 0x4c00000>; + no-map; + }; + + rmtfs_mem: rmtfs@d7c00000 { + compatible = "qcom,rmtfs-mem"; + reg = <0 0xd7c00000 0 0x400000>; + no-map; + + qcom,client-id = <1>; + qcom,vmid = ; + }; + }; + + soc: soc@0 { + compatible = "simple-bus"; + + #address-cells = <2>; + #size-cells = <2>; + dma-ranges = <0 0 0 0 0x10 0>; + ranges = <0 0 0 0 0x10 0>; + + gcc: clock-controller@100000 { + compatible = "qcom,kaanapali-gcc"; + reg = <0x0 0x00100000 0x0 0x1f4200>; + + clocks = <&bi_tcxo_div2>, + <0>, + <&sleep_clk>, + <0>, + <0>, + <0>, + <0>, + <0>; + + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; + + qupv3_1: geniqup@ac0000 { + compatible = "qcom,geni-se-qup"; + reg = <0x0 0x00ac0000 0x0 0x2000>; + + clocks = <&gcc GCC_QUPV3_WRAP_1_M_AXI_CLK>, + <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>; + clock-names = "m-ahb", + "s-ahb"; + + iommus = <&apps_smmu 0xa3 0x0>; + + dma-coherent; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uart7: serial@a9c000 { + compatible = "qcom,geni-debug-uart"; + reg = <0x0 0x00a9c000 0x0 0x4000>; + + interrupts = ; + + clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>; + clock-names = "se"; + + interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS + &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "qup-core", + "qup-config"; + + pinctrl-0 = <&qup_uart7_default>; + pinctrl-names = "default"; + + status = "disabled"; + }; + }; + + ipcc: mailbox@1106000 { + compatible = "qcom,kaanapali-ipcc", "qcom,ipcc"; + reg = <0x0 0x01106000 0x0 0x1000>; + + interrupts = ; + interrupt-controller; + #interrupt-cells = <3>; + + #mbox-cells = <2>; + }; + + cnoc_main: interconnect@1500000 { + compatible = "qcom,kaanapali-cnoc-main"; + reg = <0x0 0x01500000 0x0 0x1a080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + config_noc: interconnect@1600000 { + compatible = "qcom,kaanapali-cnoc-cfg"; + reg = <0x0 0x01600000 0x0 0x6200>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + system_noc: interconnect@1680000 { + compatible = "qcom,kaanapali-system-noc"; + reg = <0x0 0x01680000 0x0 0x1f080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + pcie_noc: interconnect@16c0000 { + compatible = "qcom,kaanapali-pcie-anoc"; + reg = <0x0 0x016c0000 0x0 0x11400>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + clocks = <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>, + <&gcc GCC_CFG_NOC_PCIE_ANOC_AHB_CLK>; + }; + + aggre_noc: interconnect@16e0000 { + compatible = "qcom,kaanapali-aggre-noc"; + reg = <0x0 0x016e0000 0x0 0x42400>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, + <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>, + <&rpmhcc RPMH_IPA_CLK>; + }; + + mmss_noc: interconnect@1780000 { + compatible = "qcom,kaanapali-mmss-noc"; + reg = <0x0 0x01780000 0x0 0x5b800>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + ufs_mem_phy: phy@1d80000 { + compatible = "qcom,kaanapali-qmp-ufs-phy", "qcom,sm8750-qmp-ufs-phy"; + reg = <0x0 0x01d80000 0x0 0x2000>; + + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&gcc GCC_UFS_PHY_PHY_AUX_CLK>, + <&tcsrcc TCSR_UFS_CLKREF_EN>; + + clock-names = "ref", + "ref_aux", + "qref"; + + resets = <&ufs_mem_hc 0>; + reset-names = "ufsphy"; + + power-domains = <&gcc GCC_UFS_MEM_PHY_GDSC>; + + #clock-cells = <1>; + #phy-cells = <0>; + + status = "disabled"; + }; + + ufs_mem_hc: ufs@1d84000 { + compatible = "qcom,kaanapali-ufshc", + "qcom,ufshc", + "jedec,ufs-2.0"; + reg = <0x0 0x01d84000 0x0 0x3000>; + + interrupts = ; + + clocks = <&gcc GCC_UFS_PHY_AXI_CLK>, + <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>, + <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>, + <&rpmhcc RPMH_LN_BB_CLK3>, + <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>, + <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>, + <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>; + clock-names = "core_clk", + "bus_aggr_clk", + "iface_clk", + "core_clk_unipro", + "ref_clk", + "tx_lane0_sync_clk", + "rx_lane0_sync_clk", + "rx_lane1_sync_clk"; + + operating-points-v2 = <&ufs_opp_table>; + + resets = <&gcc GCC_UFS_PHY_BCR>; + reset-names = "rst"; + + interconnects = <&aggre_noc MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "ufs-ddr", + "cpu-ufs"; + + power-domains = <&gcc GCC_UFS_PHY_GDSC>; + required-opps = <&rpmhpd_opp_nom>; + + iommus = <&apps_smmu 0x60 0x0>; + dma-coherent; + + lanes-per-direction = <2>; + qcom,ice = <&ice>; + + phys = <&ufs_mem_phy>; + phy-names = "ufsphy"; + + #reset-cells = <1>; + + status = "disabled"; + + ufs_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-75000000 { + opp-hz = /bits/ 64 <75000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <75000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <100000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-403000000 { + opp-hz = /bits/ 64 <403000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <403000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + ice: crypto@1d88000 { + compatible = "qcom,kaanapali-inline-crypto-engine", + "qcom,inline-crypto-engine"; + reg = <0x0 0x01d88000 0x0 0x18000>; + + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; + }; + + tcsr_mutex: hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0x0 0x01f40000 0x0 0x20000>; + #hwlock-cells = <1>; + }; + + tcsr: syscon@1fc0000 { + compatible = "qcom,tcsr-kaanapali", "syscon"; + reg = <0x0 0x1fc0000 0x0 0x30000>; + }; + + tcsrcc: clock-controller@1fd5044 { + compatible = "qcom,kaanapali-tcsr", "syscon"; + reg = <0x0 0x01fd5044 0x0 0x1c>; + + clocks = <&rpmhcc RPMH_CXO_CLK>; + + #clock-cells = <1>; + #reset-cells = <1>; + }; + + lpass_lpiaon_noc: interconnect@7400000 { + compatible = "qcom,kaanapali-lpass-lpiaon-noc"; + reg = <0x0 0x07400000 0x0 0x19080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + lpass_lpicx_noc: interconnect@7420000 { + compatible = "qcom,kaanapali-lpass-lpicx-noc"; + reg = <0x0 0x07420000 0x0 0x44080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + lpass_ag_noc: interconnect@7f40000 { + compatible = "qcom,kaanapali-lpass-ag-noc"; + reg = <0x0 0x07f40000 0x0 0xe080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + pdc: interrupt-controller@b220000 { + compatible = "qcom,kaanapali-pdc", "qcom,pdc"; + reg = <0x0 0x0b220000 0x0 0x10000>, + <0x0 0x179600f0 0x0 0xf4>; + + qcom,pdc-ranges = <0 745 38>, + <40 785 11>, + <51 527 4>, + <58 534 2>, + <61 537 20>, + <84 559 14>, + <98 609 32>, + <130 717 12>, + <142 251 5>, + <147 796 16>, + <163 783 2>, + <165 531 2>, + <167 536 1>, + <168 557 2>, + <170 415 1>, + <171 438 1>, + <172 579 1>, + <173 703 1>, + <174 708 1>, + <175 714 1>, + <176 68 1>, + <177 86 1>, + <178 96 1>, + <179 249 1>; + #interrupt-cells = <2>; + interrupt-parent = <&intc>; + interrupt-controller; + }; + + aoss_qmp: power-management@c300000 { + compatible = "qcom,kaanapali-aoss-qmp", "qcom,aoss-qmp"; + reg = <0x0 0x0c300000 0x0 0x400>; + + interrupts-extended = <&ipcc IPCC_MPROC_AOP + IPCC_MPROC_SIGNAL_GLINK_QMP + IRQ_TYPE_EDGE_RISING>; + + mboxes = <&ipcc IPCC_MPROC_AOP + IPCC_MPROC_SIGNAL_GLINK_QMP>; + + #clock-cells = <0>; + }; + + tlmm: pinctrl@f100000 { + compatible = "qcom,kaanapali-tlmm"; + reg = <0x0 0x0f100000 0x0 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&tlmm 0 0 218>; + interrupt-controller; + #interrupt-cells = <2>; + wakeup-parent = <&pdc>; + + qup_uart7_default: qup-uart7-state { + /* TX, RX */ + pins = "gpio62", "gpio63"; + function = "qup1_se7"; + drive-strength = <2>; + bias-disable; + }; + }; + + sram@14680000 { + compatible = "qcom,kaanapali-imem", "syscon", "simple-mfd"; + reg = <0x0 0x14680000 0x0 0x1000>; + ranges = <0 0 0x14680000 0x1000>; + + #address-cells = <1>; + #size-cells = <1>; + + pil-reloc@94c { + compatible = "qcom,pil-reloc-info"; + reg = <0x94c 0xc8>; + }; + }; + + apps_smmu: iommu@15000000 { + compatible = "qcom,kaanapali-smmu-500", "qcom,smmu-500", "arm,mmu-500"; + reg = <0x0 0x15000000 0x0 0x100000>; + + interrupts =, + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + + #iommu-cells = <2>; + #global-interrupts = <1>; + + dma-coherent; + }; + + intc: interrupt-controller@17000000 { + compatible = "arm,gic-v3"; + reg = <0x0 0x17000000 0x0 0x10000>, + <0x0 0x17080000 0x0 0x200000>; + + interrupts = ; + + #interrupt-cells = <3>; + interrupt-controller; + + #redistributor-regions = <1>; + redistributor-stride = <0x0 0x40000>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + gic_its: msi-controller@17040000 { + compatible = "arm,gic-v3-its"; + reg = <0x0 0x17040000 0x0 0x20000>; + + msi-controller; + #msi-cells = <1>; + }; + }; + + watchdog@17600000 { + compatible = "qcom,apss-wdt-kaanapali", "qcom,kpss-wdt"; + reg = <0x0 0x17600000 0x0 0x1000>; + clocks = <&sleep_clk>; + interrupts = ; + }; + + pdp0_mbox: mailbox@17610000 { + compatible = "qcom,kaanapali-cpucp-mbox", "qcom,x1e80100-cpucp-mbox"; + reg = <0 0x17610000 0 0x8000>, <0 0x19980000 0 0x8000>; + interrupts = ; + #mbox-cells = <1>; + }; + + timer@17810000 { + compatible = "arm,armv7-timer-mem"; + reg = <0x0 0x17810000 0x0 0x1000>; + + #address-cells = <2>; + #size-cells = <1>; + ranges = <0 0 0 0 0x20000000>; + + frame@17811000 { + reg = <0x0 0x17811000 0x1000>, + <0x0 0x17812000 0x1000>; + frame-number = <0>; + interrupts = , + ; + }; + + frame@17813000 { + reg = <0x0 0x17813000 0x1000>; + frame-number = <1>; + interrupts = ; + status = "disabled"; + }; + + frame@17815000 { + reg = <0x0 0x17815000 0x1000>; + frame-number = <2>; + interrupts = ; + status = "disabled"; + }; + + frame@17817000 { + reg = <0x0 0x17817000 0x1000>; + frame-number = <3>; + interrupts = ; + status = "disabled"; + }; + + frame@17819000 { + reg = <0x0 0x17819000 0x1000>; + frame-number = <4>; + interrupts = ; + status = "disabled"; + }; + + frame@1781b000 { + reg = <0x0 0x1781b000 0x1000>; + frame-number = <5>; + interrupts = ; + status = "disabled"; + }; + + frame@1781d000 { + reg = <0x0 0x1781d000 0x1000>; + frame-number = <6>; + interrupts = ; + status = "disabled"; + }; + }; + + apps_rsc: rsc@18900000 { + compatible = "qcom,rpmh-rsc"; + reg = <0x0 0x18900000 0x0 0x10000>, + <0x0 0x18910000 0x0 0x10000>, + <0x0 0x18920000 0x0 0x10000>; + reg-names = "drv-0", + "drv-1", + "drv-2"; + interrupts = , + , + ; + + power-domains = <&system_pd>; + label = "apps_rsc"; + + qcom,tcs-offset = <0xd00>; + qcom,drv-id = <2>; + qcom,tcs-config = , + , + , + ; + + apps_bcm_voter: bcm-voter { + compatible = "qcom,bcm-voter"; + }; + + rpmhcc: clock-controller { + compatible = "qcom,kaanapali-rpmh-clk"; + #clock-cells = <1>; + clocks = <&xo_board>; + clock-names = "xo"; + }; + + rpmhpd: power-controller { + compatible = "qcom,kaanapali-rpmhpd"; + + operating-points-v2 = <&rpmhpd_opp_table>; + + #power-domain-cells = <1>; + + rpmhpd_opp_table: opp-table { + compatible = "operating-points-v2"; + + rpmhpd_opp_ret: opp-16 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d3: opp-50 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d2_1: opp-51 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d2: opp-52 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d1_1: opp-54 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d1: opp-56 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d0: opp-60 { + opp-level = ; + }; + + rpmhpd_opp_low_svs: opp-64 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_l0: opp-76 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_l1: opp-80 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_l2: opp-96 { + opp-level = ; + }; + + rpmhpd_opp_svs: opp-128 { + opp-level = ; + }; + + rpmhpd_opp_svs_l0: opp-144 { + opp-level = ; + }; + + rpmhpd_opp_svs_l1: opp-192 { + opp-level = ; + }; + + rpmhpd_opp_svs_l2: opp-224 { + opp-level = ; + }; + + rpmhpd_opp_nom: opp-256 { + opp-level = ; + }; + + rpmhpd_opp_nom_l1: opp-320 { + opp-level = ; + }; + + rpmhpd_opp_nom_l2: opp-336 { + opp-level = ; + }; + + rpmhpd_opp_turbo: opp-384 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l0: opp-400 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l1: opp-416 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l2: opp-432 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l3: opp-448 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l4: opp-452 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l5: opp-456 { + opp-level = ; + }; + + rpmhpd_opp_super_turbo_no_cpr: opp-480 { + opp-level = ; + }; + }; + }; + }; + + nsp_noc: interconnect@260c0000 { + compatible = "qcom,kaanapali-nsp-noc"; + reg = <0x0 0x260c0000 0x0 0x21280>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + /* Cluster 0 */ + pmu@310b3400 { + compatible = "qcom,kaanapali-cpu-bwmon", "qcom,sdm845-bwmon"; + reg = <0x0 0x310b3400 0x0 0x600>; + + interrupts = ; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>; + + operating-points-v2 = <&cpu_bwmon_opp_table>; + + cpu_bwmon_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-0 { + opp-peak-kBps = <2188000>; + }; + + opp-1 { + opp-peak-kBps = <5412000>; + }; + + opp-2 { + opp-peak-kBps = <6220000>; + }; + + opp-3 { + opp-peak-kBps = <6832000>; + }; + + opp-4 { + opp-peak-kBps = <8368000>; + }; + + opp-5 { + opp-peak-kBps = <10944000>; + }; + + opp-6 { + opp-peak-kBps = <12748000>; + }; + + opp-7 { + opp-peak-kBps = <14744000>; + }; + + opp-8 { + opp-peak-kBps = <16896000>; + }; + + opp-9 { + opp-peak-kBps = <19120000>; + }; + + opp-10 { + opp-peak-kBps = <21332000>; + }; + }; + }; + + /* Cluster 1 */ + pmu@310b7400 { + compatible = "qcom,kaanapali-cpu-bwmon", "qcom,sdm845-bwmon"; + reg = <0x0 0x310b7400 0x0 0x600>; + + interrupts = ; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>; + + operating-points-v2 = <&cpu_bwmon_opp_table>; + }; + + gem_noc: interconnect@31100000 { + compatible = "qcom,kaanapali-gem-noc"; + reg = <0x0 0x31100000 0x0 0x153080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + system-cache-controller@31800000 { + compatible = "qcom,kaanapali-llcc"; + reg = <0x0 0x31800000 0x0 0x200000>, + <0x0 0x32800000 0x0 0x200000>, + <0x0 0x31c00000 0x0 0x200000>, + <0x0 0x32c00000 0x0 0x200000>, + <0x0 0x34800000 0x0 0x200000>, + <0x0 0x34c00000 0x0 0x200000>; + reg-names = "llcc0_base", + "llcc1_base", + "llcc2_base", + "llcc3_base", + "llcc_broadcast_base", + "llcc_broadcast_and_base"; + + interrupts = ; + }; + + sram: sram@81f08000 { + compatible = "mmio-sram"; + reg = <0x0 0x81f08000 0x0 0x200>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x0 0x81f08000 0x200>; + + pdp_rx: scp-sram-section@0 { + compatible = "arm,scmi-shmem"; + reg = <0x0 0x80>; + }; + + pdp_tx: scp-sram-section@100 { + compatible = "arm,scmi-shmem"; + reg = <0x100 0x80>; + }; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + + interrupts = , + , + , + ; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/kodiak-el2.dtso b/arch/arm64/boot/dts/qcom/kodiak-el2.dtso new file mode 100644 index 0000000000000..96501dd3033a1 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/kodiak-el2.dtso @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Kodiak specific modifications required to boot in EL2. + */ + + +/dts-v1/; +/plugin/; + +&gpu_zap_shader { + status = "disabled"; +}; + +&remoteproc_adsp { + iommus = <&apps_smmu 0x1800 0x0>; +}; + +&remoteproc_cdsp { + iommus = <&apps_smmu 0x11a0 0x0400>; +}; + +&remoteproc_wpss { + iommus = <&apps_smmu 0x1c03 0x1>, + <&apps_smmu 0x1c83 0x1>; +}; + +&venus { + status = "okay"; + video-firmware { + iommus = <&apps_smmu 0x21A2 0x0000>; + }; +}; + +&watchdog { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/kodiak-staging.dtso b/arch/arm64/boot/dts/qcom/kodiak-staging.dtso new file mode 100644 index 0000000000000..1d4c6d7d39dd0 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/kodiak-staging.dtso @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Kodiak staging overlay - add staging-specific device tree modifications here. + */ + +/dts-v1/; +/plugin/; + +&soc { + tgu@6b0e000 { + compatible = "qcom,tgu", "arm,primecell"; + reg = <0x0 0x6b0e000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-auto.dtsi b/arch/arm64/boot/dts/qcom/lemans-auto.dtsi index 8db958d60fd1d..07fa9cb148ae6 100644 --- a/arch/arm64/boot/dts/qcom/lemans-auto.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans-auto.dtsi @@ -102,3 +102,7 @@ }; }; }; + +&iris { + firmware-name = "qcom/vpu/vpu30_p4_s6.mbn"; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-camera-sensor.dtsi b/arch/arm64/boot/dts/qcom/lemans-camera-sensor.dtsi new file mode 100644 index 0000000000000..d531f89dda08e --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-camera-sensor.dtsi @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&cam_cci0 { + /* GMSL deserializer 0 */ + qcom,cam-gmsl-deserializer0 { + cell-index = <0>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active>; + pinctrl-1 = <&cam_sensor_mclk0_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 7 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET0"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser0_port0: endpoint { + remote-endpoint = <&gmsl_sensor0_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser0_port1: endpoint { + remote-endpoint = <&gmsl_sensor1_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser0_port2: endpoint { + remote-endpoint = <&gmsl_sensor2_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser0_port3: endpoint { + remote-endpoint = <&gmsl_sensor3_ep>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 0 */ + qcom,cam-gmsl-sensor0 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <0>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor0_ep: endpoint { + remote-endpoint = <&deser0_port0>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 1 */ + qcom,cam-gmsl-sensor1 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <1>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor1_ep: endpoint { + remote-endpoint = <&deser0_port1>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 2 */ + qcom,cam-gmsl-sensor2 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <2>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor2_ep: endpoint { + remote-endpoint = <&deser0_port2>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 3 */ + qcom,cam-gmsl-sensor3 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <3>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor3_ep: endpoint { + remote-endpoint = <&deser0_port3>; + }; + }; + }; +}; + +&cam_cci1 { + /* GMSL deserializer 1 */ + qcom,cam-gmsl-deserializer1 { + cell-index = <1>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active>; + pinctrl-1 = <&cam_sensor_mclk1_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 8 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser1_port0: endpoint { + remote-endpoint = <&gmsl_sensor4_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser1_port1: endpoint { + remote-endpoint = <&gmsl_sensor5_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser1_port2: endpoint { + remote-endpoint = <&gmsl_sensor6_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser1_port3: endpoint { + remote-endpoint = <&gmsl_sensor7_ep>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 0 */ + qcom,cam-gmsl-sensor4 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <4>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor4_ep: endpoint { + remote-endpoint = <&deser1_port0>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 1 */ + qcom,cam-gmsl-sensor5 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <5>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor5_ep: endpoint { + remote-endpoint = <&deser1_port1>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 2 */ + qcom,cam-gmsl-sensor6 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <6>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor6_ep: endpoint { + remote-endpoint = <&deser1_port2>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 3 */ + qcom,cam-gmsl-sensor7 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <7>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor7_ep: endpoint { + remote-endpoint = <&deser1_port3>; + }; + }; + }; +}; + +&cam_cci2 { + /* GMSL deserializer 2 */ + qcom,cam-gmsl-deserializer2 { + cell-index = <2>; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active>; + pinctrl-1 = <&cam_sensor_mclk2_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 9 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET2"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser2_port0: endpoint { + remote-endpoint = <&gmsl_sensor8_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser2_port1: endpoint { + remote-endpoint = <&gmsl_sensor9_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser2_port2: endpoint { + remote-endpoint = <&gmsl_sensor10_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser2_port3: endpoint { + remote-endpoint = <&gmsl_sensor11_ep>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 0 */ + qcom,cam-gmsl-sensor8 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <8>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor8_ep: endpoint { + remote-endpoint = <&deser2_port0>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 1 */ + qcom,cam-gmsl-sensor9 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <9>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor9_ep: endpoint { + remote-endpoint = <&deser2_port1>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 2 */ + qcom,cam-gmsl-sensor10 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <10>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor10_ep: endpoint { + remote-endpoint = <&deser2_port2>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 3 */ + qcom,cam-gmsl-sensor11 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <11>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor11_ep: endpoint { + remote-endpoint = <&deser2_port3>; + }; + }; + }; +}; + +&cam_cci3 { + /* GMSL deserializer 3 */ + qcom,cam-gmsl-deserializer3 { + cell-index = <3>; + csiphy-sd-index = <3>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active>; + pinctrl-1 = <&cam_sensor_mclk3_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 10 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET3"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK3_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser3_port0: endpoint { + remote-endpoint = <&gmsl_sensor12_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser3_port1: endpoint { + remote-endpoint = <&gmsl_sensor13_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser3_port2: endpoint { + remote-endpoint = <&gmsl_sensor14_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser3_port3: endpoint { + remote-endpoint = <&gmsl_sensor15_ep>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 0 */ + qcom,cam-gmsl-sensor12 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <12>; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor12_ep: endpoint { + remote-endpoint = <&deser3_port0>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 1 */ + qcom,cam-gmsl-sensor13 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <13>; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor13_ep: endpoint { + remote-endpoint = <&deser3_port1>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 2 */ + qcom,cam-gmsl-sensor14 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <14>; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor14_ep: endpoint { + remote-endpoint = <&deser3_port2>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 3 */ + qcom,cam-gmsl-sensor15 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <15>; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor15_ep: endpoint { + remote-endpoint = <&deser3_port3>; + }; + }; + }; +}; + +&soc { + qcom,cam-res-mgr { + compatible = "qcom,cam-res-mgr"; + gpios-shared = <518 519 520 521>; + status = "ok"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-camera.dtsi b/arch/arm64/boot/dts/qcom/lemans-camera.dtsi new file mode 100644 index 0000000000000..d6cce86922c26 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-camera.dtsi @@ -0,0 +1,2552 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&soc { + cam_cci0: qcom,cci0@ac13000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac13000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x13000>; + interrupts = ; + interrupt-names = "CCI0"; + operating-points-v2 = <&cci0_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_0_CLK_SRC>, + <&camcc CAM_CC_CCI_0_CLK>; + clock-names = "cci_0_clk_src", + "cci_0_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_0_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci0_active>; + pinctrl-1 = <&cci0_suspend>; + pinctrl-2 = <&cci1_active>; + pinctrl-3 = <&cci1_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <0>; + status = "ok"; + + cci0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci0: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci0: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci0: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci0: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_cci1: qcom,cci1@ac14000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac14000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x14000>; + interrupts = ; + interrupt-names = "CCI1"; + operating-points-v2 = <&cci1_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_1_CLK_SRC>, + <&camcc CAM_CC_CCI_1_CLK>; + clock-names = "cci_1_clk_src", + "cci_1_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_1_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci2_active>; + pinctrl-1 = <&cci2_suspend>; + pinctrl-2 = <&cci3_active>; + pinctrl-3 = <&cci3_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <1>; + status = "ok"; + + cci1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci1: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci1: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci1: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci1: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_cci2: qcom,cci2@ac15000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac15000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x15000>; + interrupts = ; + interrupt-names = "CCI2"; + operating-points-v2 = <&cci2_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_2_CLK_SRC>, + <&camcc CAM_CC_CCI_2_CLK>; + clock-names = "cci_2_clk_src", + "cci_2_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_2_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci4_active>; + pinctrl-1 = <&cci4_suspend>; + pinctrl-2 = <&cci5_active>; + pinctrl-3 = <&cci5_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <2>; + status = "ok"; + + cci2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci2: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci2: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci2: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci2: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_cci3: qcom,cci3@ac16000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac16000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x16000>; + interrupts = ; + interrupt-names = "CCI3"; + operating-points-v2 = <&cci3_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_3_CLK_SRC>, + <&camcc CAM_CC_CCI_3_CLK>; + clock-names = "cci_3_clk_src", + "cci_3_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_3_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci6_active>; + pinctrl-1 = <&cci6_suspend>; + pinctrl-2 = <&cci7_active>; + pinctrl-3 = <&cci7_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <3>; + status = "ok"; + + cci3_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci3: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci3: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci3: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci3: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_csiphy0: qcom,csiphy0@ac9c000 { + compatible = "qcom,csiphy-v1.3.0", "qcom,csiphy"; + reg = <0x0 0x0ac9c000 0x0 0x2000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy0_opp_table>; + reg-cam-base = <0x9c000>; + interrupts = ; + interrupt-names = "CSIPHY0"; + csi-vdd-1p2-supply = <&vreg_l1c>; + csi-vdd-0p9-supply = <&vreg_l4a>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1140000 831000>; + rgltr-max-voltage = <1260000 920000>; + rgltr-load-current = <8900 15900>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy0_clk", + "csi0phytimer_clk_src", + "csi0phytimer_clk"; + src-clock-name = "cphy_rx_clk_src"; + clock-cntl-level = "lowsvs"; + clock-rates = <400000000 0 400000000 0>; + aggregator-rx; + cell-index = <0>; + status = "ok"; + + csiphy0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy1: qcom,csiphy1@ac9e000 { + compatible = "qcom,csiphy-v1.3.0", "qcom,csiphy"; + reg = <0x0 0xac9e000 0x0 0x2000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy1_opp_table>; + reg-cam-base = <0x9e000>; + interrupts = ; + interrupt-names = "CSIPHY1"; + csi-vdd-1p2-supply = <&vreg_l1c>; + csi-vdd-0p9-supply = <&vreg_l4a>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1140000 831000>; + rgltr-max-voltage = <1260000 920000>; + rgltr-load-current = <8900 15900>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy1_clk", + "csi1phytimer_clk_src", + "csi1phytimer_clk"; + src-clock-name = "cphy_rx_clk_src"; + clock-cntl-level = "lowsvs"; + clock-rates = <400000000 0 400000000 0>; + cell-index = <1>; + aggregator-rx; + status = "ok"; + + csiphy1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy2: qcom,csiphy2@aca0000 { + compatible = "qcom,csiphy-v1.3.0", "qcom,csiphy"; + reg = <0x0 0xaca0000 0x0 0x2000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy2_opp_table>; + reg-cam-base = <0xa0000>; + interrupts = ; + interrupt-names = "CSIPHY2"; + csi-vdd-1p2-supply = <&vreg_l1c>; + csi-vdd-0p9-supply = <&vreg_l4a>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1140000 831000>; + rgltr-max-voltage = <1260000 920000>; + rgltr-load-current = <8900 15900>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy2_clk", + "csi2phytimer_clk_src", + "csi2phytimer_clk"; + src-clock-name = "cphy_rx_clk_src"; + clock-cntl-level = "lowsvs"; + clock-rates = <400000000 0 400000000 0>; + cell-index = <2>; + aggregator-rx; + status = "ok"; + + csiphy2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy3: qcom,csiphy3@aca2000 { + compatible = "qcom,csiphy-v1.3.0", "qcom,csiphy"; + reg = <0x0 0xaca2000 0x0 0x2000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy3_opp_table>; + reg-cam-base = <0xa2000>; + interrupts = ; + interrupt-names = "CSIPHY3"; + csi-vdd-1p2-supply = <&vreg_l1c>; + csi-vdd-0p9-supply = <&vreg_l4a>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1140000 831000>; + rgltr-max-voltage = <1260000 920000>; + rgltr-load-current = <8900 15900>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY3_CLK>, + <&camcc CAM_CC_CSI3PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI3PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy3_clk", + "csi3phytimer_clk_src", + "csi3phytimer_clk"; + src-clock-name = "cphy_rx_clk_src"; + clock-cntl-level = "lowsvs"; + clock-rates = <400000000 0 400000000 0>; + cell-index = <3>; + aggregator-rx; + status = "ok"; + + csiphy3_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy_tpg17: qcom,tpg17@acac000 { + compatible = "qcom,cam-tpg1031"; + reg = <0x0 0xacac000 0x0 0x400>, + <0x0 0xac11000 0x0 0x1000>; + reg-names = "tpg0", "cam_cpas_top"; + reg-cam-base = <0xac000 0x11000>; + operating-points-v2 = <&csiphy_tpg0_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg0"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cphy_rx_clk_src"; + cell-index = <17>; + phy-id = <0>; + status = "ok"; + + csiphy_tpg0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy_tpg18: qcom,tpg18@acad000 { + compatible = "qcom,cam-tpg1031"; + reg = <0x0 0xacad000 0x0 0x400>, + <0x0 0xac11000 0x0 0x1000>; + reg-names = "tpg1", "cam_cpas_top"; + reg-cam-base = <0xad000 0x11000>; + operating-points-v2 = <&csiphy_tpg1_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg1"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cphy_rx_clk_src"; + cell-index = <18>; + phy-id = <1>; + status = "ok"; + + csiphy_tpg1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy_tpg19: qcom,tpg19@acae000 { + compatible = "qcom,cam-tpg1031"; + reg = <0x0 0xacae000 0x0 0x400>, + <0x0 0xac11000 0x0 0x1000>; + reg-names = "tpg2", "cam_cpas_top"; + reg-cam-base = <0xae000 0x11000>; + operating-points-v2 = <&csiphy_tpg2_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg2"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cphy_rx_clk_src"; + cell-index = <19>; + phy-id = <2>; + status = "ok"; + + csiphy_tpg2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,cam-cdm-intf { + compatible = "qcom,cam-cdm-intf"; + cell-index = <0>; + label = "cam-cdm-intf"; + num-hw-cdm = <1>; + cdm-client-names = "vfe"; + status = "ok"; + }; + + cam_cpas: qcom,cam-cpas { + compatible = "qcom,cam-cpas"; + label = "cpas"; + arch-compat = "cpas_top"; + reg = <0x0 0xac11000 0x0 0x1000>, + <0x0 0xac1A000 0x0 0x9400>, + <0x0 0xbbf0000 0x0 0x1f00>; + reg-names = "cam_cpas_top", "cam_camnoc", "cam_rpmh"; + reg-cam-base = <0x11000 0x1A000 0x0bbf0000>; + interrupts = ; + interrupt-names = "cpas_camnoc"; + camnoc-axi-min-ib-bw = <3000000000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_QDSS_DEBUG_XO_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "gcc_camera_sf_axi_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk", + "cam_cc_core_ahb_clk", + "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_fast_ahb_clk", + "cam_cc_camnoc_axi_clk_src", + "cam_cc_camnoc_axi_clk", + "cam_cc_qdss_debug_xo_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <0 0 0 0 0 0 0 0 0 0 0>, + <0 0 0 80000000 0 0 300000000 0 400000000 0 0>, + <0 0 0 80000000 0 0 400000000 0 400000000 0 0>; + clock-cntl-level = "suspend", "svs_l1", "nominal"; + clock-names-option = "cam_icp_clk"; + clocks-option = <&camcc CAM_CC_ICP_CLK>; + clock-rates-option = <480000000>; + operating-points-v2 = <&cpas_opp_table>; + control-camnoc-axi-clk; + camnoc-bus-width = <32>; + camnoc-axi-clk-bw-margin-perc = <20>; + cam-icc-path-names = "cam_ahb"; + interconnects = <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_CAMERA_CFG 0>, + <&mmss_noc MASTER_CAMNOC_HF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_SF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_ICP 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "cam_ahb", "cam_hf_0", "cam_sf_0", "cam_sf_icp"; + cam-ahb-num-cases = <7>; + cam-ahb-bw-KBps = <0 0>, <0 76800>, <0 150000>, <0 150000>, + <0 300000>, <0 300000>, <0 300000>; + vdd-corners = ; + vdd-corner-ahb-mapping = "suspend", "lowsvs", "lowsvs", "svs", "svs_l1", + "nominal", "nominal", "nominal", "turbo", "turbo"; + client-id-based; + client-names = "csiphy0", "csiphy1", "csiphy2", "csiphy3", + "cci0", "cci1", "cci2", "cci3","csid0", "csid1", + "csid2", "csid3", "csid4", "csid5", "csid6", + "ife0", "ife1", "ife2", "ife3", "ife4", + "ife5", "ife6", "ipe0", "sfe0", "sfe1", + "cam-cdm-intf0", "rt-cdm0", "rt-cdm1", "rt-cdm2", + "rt-cdm3", "icp0", "tpg17", "tpg18", "tpg19"; + enable-secure-qos-update = <1>; + cell-index = <0>; + status = "ok"; + + cpas_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + + camera-bus-nodes { + level0-nodes { + level-index = <0>; + + icp_all_rd: icp-all-rd { + cell-index = <0>; + node-name = "icp-all-rd"; + client-name = "icp0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_icp_rd>; + }; + + ife_0_wr_0: ife-0-wr-0 { + cell-index = <1>; + node-name = "ife-0-wr-0"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_0_wr_1: ife-0-wr-1 { + cell-index = <2>; + node-name = "ife-0-wr-1"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr1>; + }; + + ife_1_wr_0: ife-1-wr-0 { + cell-index = <3>; + node-name = "ife-1-wr-0"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_1_wr_1: ife-1-wr-1 { + cell-index = <4>; + node-name = "ife-1-wr-1"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr1>; + }; + + ife_lite_0_wr_0: ife-lite-0-wr-0 { + cell-index = <5>; + node-name = "ife-lite-0-wr-0"; + client-name = "ife2"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_1_wr_0: ife-lite-1-wr-0 { + cell-index = <6>; + node-name = "ife-lite-1-wr-0"; + client-name = "ife3"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_2_wr_0: ife-lite-2-wr-0 { + cell-index = <7>; + node-name = "ife-lite-2-wr-0"; + client-name = "ife4"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_3_wr_0: ife-lite-3-wr-0 { + cell-index = <8>; + node-name = "ife-lite-3-wr-0"; + client-name = "ife5"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_4_wr_0: ife-lite-4-wr-0 { + cell-index = <9>; + node-name = "ife-lite-4-wr-0"; + client-name = "ife6"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ipe_0_rd_all: ipe-0-rd-all { + cell-index = <10>; + node-name = "ipe-0-rd-all"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level2_nrt_rd1>; + }; + + ipe_0_wr_2: ipe-0-wr-2 { + cell-index = <11>; + node-name = "ipe-0-wr-2"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level2_nrt_wr2>; + }; + + ipe_cdm0_all_rd: ipe-cdm0-all-rd { + cell-index = <12>; + node-name = "ipe-cdm0-all-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm0_all_rd_2: rt-cdm0-all-rd-2 { + cell-index = <13>; + node-name = "rt-cdm0-all-rd-2"; + client-name = "rt-cdm0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm1_all_rd_2: rt-cdm1-all-rd-2 { + cell-index = <14>; + node-name = "rt-cdm1-all-rd-2"; + client-name = "rt-cdm1"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm2_all_rd_2: rt-cdm2-all-rd-2 { + cell-index = <15>; + node-name = "rt-cdm2-all-rd-2"; + client-name = "rt-cdm2"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm3_all_rd_2: rt-cdm3-all-rd-2 { + cell-index = <16>; + node-name = "rt-cdm3-all-rd-2"; + client-name = "rt-cdm3"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + sfe_0_rd_0: sfe-0-rd-0 { + cell-index = <17>; + node-name = "sfe-0-rd-0"; + client-name = "sfe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_rd0>; + }; + + sfe_1_rd_0: sfe-1-rd-0 { + cell-index = <18>; + node-name = "sfe-1-rd-0"; + client-name = "sfe1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_rd0>; + }; + }; + + level1-nodes { + level-index = <1>; + camnoc-max-needed; + + level1_nrt_rd2: level1-nrt-rd2 { + cell-index = <19>; + node-name = "level1-nrt-rd2"; + parent-node = <&level2_nrt_rd2>; + traffic-merge-type = ; + }; + + level1_rt_rd0: level1-rt-read0 { + cell-index = <20>; + node-name = "level1-rt-rd0"; + parent-node = <&level2_rt_rd0>; + traffic-merge-type = ; + }; + + level1_rt_wr0: level1-rt-wr0 { + cell-index = <21>; + node-name = "level1-rt-wr0"; + parent-node = <&level2_rt_wr0>; + traffic-merge-type = ; + }; + + level1_rt_wr1: level1-rt-wr1 { + cell-index = <22>; + node-name = "level1-rt-wr1"; + parent-node = <&level2_rt_wr1>; + traffic-merge-type = ; + }; + }; + + level2-nodes { + level-index = <2>; + camnoc-max-needed; + + level2_icp_rd: level2-icp-rd { + cell-index = <23>; + node-name = "level2-icp-rd"; + parent-node = <&level3_nrt1_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt_rd1: level2-nrt-rd1 { + cell-index = <24>; + node-name = "level2-nrt-rd1"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt_rd2: level2-nrt-rd2 { + cell-index = <25>; + node-name = "level2-nrt-rd2"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt_wr2: level2-nrt-wr2 { + cell-index = <26>; + node-name = "level2-nrt-wr2"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_rt_rd0: level2-rt-read0 { + cell-index = <27>; + node-name = "level2-rt-rd0"; + parent-node = <&level3_rt_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_rt_wr0: level2-rt-wr0 { + cell-index = <28>; + node-name = "level2-rt-wr0"; + parent-node = <&level3_rt_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_rt_wr1: level2-rt-wr1 { + cell-index = <29>; + node-name = "level2-rt-wr1"; + parent-node = <&level3_rt_rd_wr_sum>; + traffic-merge-type = ; + }; + }; + + level3-nodes { + level-index = <3>; + + level3_nrt0_rd_wr_sum: level3-nrt0-rd-wr-sum { + cell-index = <30>; + node-name = "level3-nrt0-rd-wr-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_0"; + }; + }; + + level3_nrt1_rd_wr_sum: level3-nrt1-rd-wr-sum { + cell-index = <31>; + node-name = "level3-nrt1-rd-wr-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_icp"; + }; + }; + + level3_rt_rd_wr_sum: level3-rt-rd-wr-sum { + cell-index = <32>; + node-name = "level3-rt-rd-wr-sum"; + traffic-merge-type = ; + ib-bw-voting-needed; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_hf_0"; + }; + }; + }; + }; + }; + + cam_icp_firmware: qcom,cam-icp { + compatible = "qcom,cam-icp"; + compat-hw-name = "qcom,icp", "qcom,ipe0"; + num-icp = <1>; + num-ipe = <1>; + icp_use_pil; + status = "ok"; + }; + + qcom,cam-i3c-id-table { + i3c-sensor-id-table = <0x1B0 0x0766>; + i3c-eeprom-id-table = <>; + i3c-actuator-id-table = <>; + i3c-ois-id-table = <>; + status = "disabled"; + }; + + qcom,cam-isp { + compatible = "qcom,cam-isp"; + arch-compat = "ife"; + status = "ok"; + }; + + qcom,cam-req-mgr { + compatible = "qcom,cam-req-mgr"; + status = "ok"; + }; + + qcom,camera-main { + compatible = "qcom,camera"; + status = "ok"; + }; + + qcom,cam-smmu { + compatible = "qcom,msm-cam-smmu", "simple-bus"; + force_cache_allocs; + status = "ok"; + + msm-cam-smmu-cdm { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0860 0x400>, + <&apps_smmu 0x0C60 0x400>; + cam-smmu-label = "rt-cdm"; + dma-coherent; + multiple-client-devices; + + rt_cdm_iova_mem_map: iova-mem-map { + iova-mem-region-io { + /* IO region is approximately 4.0 GB */ + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-icp { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0800 0x400>, + <&apps_smmu 0x0840 0x480>, + <&apps_smmu 0x08C0 0x480>, + <&apps_smmu 0x0C00 0x400>, + <&apps_smmu 0x0C40 0x480>, + <&apps_smmu 0x0CC0 0x480>; + cam-smmu-label = "icp"; + qcom,iommu-faults = "stall-disable", "non-fatal"; + iova-region-discard = <0xe0000000 0x800000>; + dma-coherent; + + icp_iova_mem_map: iova-mem-map { + iova-mem-region-fwuncached-region { + /* FW uncached region is 7MB long */ + iova-region-name = "fw_uncached"; + iova-region-start = <0x10400000>; + iova-region-len = <0x700000>; + iova-region-id = <0x6>; + subregion_support; + status = "ok"; + + /* Used for HFI queues/sec heap */ + iova-mem-region-generic-region { + iova-region-name = "icp_hfi"; + iova-region-start = <0x10400000>; + iova-region-len = <0x200000>; + iova-region-id = <0x0>; + }; + }; + + iova-mem-region-io { + /* IO region is approximately 3.8 GB */ + iova-region-name = "io"; + iova-region-start = <0x10c00000>; + iova-region-len = <0xee300000>; + iova-region-id = <0x3>; + iova-region-discard = <0xe0000000 0x800000>; + status = "ok"; + }; + + iova-mem-region-qdss { + /* QDSS region is appropriate 1MB */ + iova-region-name = "qdss"; + iova-region-start = <0x10b00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x5>; + qdss-phy-addr = <0x16790000>; + status = "ok"; + }; + + iova-mem-region-shared { + /* Shared region is ~250MB long */ + iova-region-name = "shared"; + iova-region-start = <0x800000>; + iova-region-len = <0xfc00000>; + iova-region-id = <0x1>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-ife { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x3400 0x0020>, + <&apps_smmu 0x3420 0x0020>; + cam-smmu-label = "ife", "sfe"; + multiple-client-devices; + dma-coherent; + + ife_iova_mem_map: iova-mem-map { + /* IO region is approximately 64 GB */ + iova-mem-region-io { + iova-region-name = "io"; + /* start address: 0x100000 */ + /* leaving 1 MB pad at start */ + iova-region-start = <0x100000>; + /* Length: 0xfffe00000 */ + /* leaving 1 MB pad at end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-secure { + compatible = "qcom,msm-cam-smmu-cb"; + cam-smmu-label = "cam-secure"; + qcom,secure-cb; + }; + }; + + qcom,cam-sync { + compatible = "qcom,cam-sync"; + status = "ok"; + }; + + cam_csid0: qcom,csid0 { + compatible = "qcom,csid690"; + reg = <0x0 0xac7a000 0x0 0xf01>, + <0x0 0xac78000 0x0 0x1000>; + reg-names = "csid0", "csid_top"; + reg-cam-base = <0x7a000 0x78000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "csid0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CSID_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cam_cc_csid_clk_src", + "cam_cc_csid_clk", + "cam_cc_csid_csiphy_rx_clk"; + clock-rates = <400000000 0 0>; + clock-cntl-level = "svs_l1"; + src-clock-name = "cam_cc_csid_clk_src"; + operating-points-v2 = <&csid0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "ok"; + + csid0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + + cam_csid1: qcom,csid1 { + compatible = "qcom,csid690"; + reg = <0x0 0xac7c000 0x0 0xf01>, + <0x0 0xac78000 0x0 0x1000>; + reg-names = "csid1", "csid_top"; + reg-cam-base = <0x7c000 0x78000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "csid0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CSID_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cam_cc_csid_clk_src", + "cam_cc_csid_clk", + "cam_cc_csid_csiphy_rx_clk"; + clock-rates = <400000000 0 0>; + clock-cntl-level = "svs_l1"; + src-clock-name = "cam_cc_csid_clk_src"; + operating-points-v2 = <&csid1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "ok"; + + csid1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + + cam_csid_lite0: qcom,csid-lite0 { + compatible = "qcom,csid-lite690"; + reg = <0x0 0xac84000 0x0 0xf01>; + reg-names = "csid-lite0"; + reg-cam-base = <0x84000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <2>; + status = "ok"; + + csid_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite1: qcom,csid-lite1 { + compatible = "qcom,csid-lite690"; + reg = <0x0 0xac88000 0x0 0xf01>; + reg-names = "csid-lite1"; + reg-cam-base = <0x88000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <3>; + status = "ok"; + + csid_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite2: qcom,csid-lite2 { + compatible = "qcom,csid-lite690"; + reg = <0x0 0xac8c000 0x0 0xf01>; + reg-names = "csid-lite2"; + reg-cam-base = <0x8c000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite2_opp_table>; + clock-control-debugfs = "true"; + cell-index = <4>; + status = "ok"; + + csid_lite2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite3: qcom,csid-lite3 { + compatible = "qcom,csid-lite690"; + reg = <0x0 0xac90000 0x0 0xf01>; + reg-names = "csid-lite3"; + reg-cam-base = <0x90000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite3_opp_table>; + clock-control-debugfs = "true"; + cell-index = <5>; + status = "ok"; + + csid_lite3_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite4: qcom,csid-lite4 { + compatible = "qcom,csid-lite690"; + reg = <0x0 0xac94000 0x0 0xf01>; + reg-names = "csid-lite4"; + reg-cam-base = <0x94000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite4"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite4_opp_table>; + clock-control-debugfs = "true"; + cell-index = <6>; + status = "ok"; + + csid_lite4_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_icp: qcom,icp { + compatible = "qcom,cam-icp_v3_0"; + reg = <0x0 0xac01000 0x0 0x400>, + <0x0 0xac01800 0x0 0x400>, + <0x0 0xac04000 0x0 0x1000>; + reg-names ="icp_csr", "icp_cirq", "icp_wd0"; + reg-cam-base = <0x1000 0x1800 0x4000>; + interrupts = ; + interrupt-names = "icp"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + memory-region = <&pil_camera_mem>; + src-clock-name = "cam_cc_icp_clk_src"; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_ICP_AHB_CLK>, + <&camcc CAM_CC_ICP_CLK_SRC>, + <&camcc CAM_CC_ICP_CLK>; + clock-names = "cam_cc_soc_fast_ahb_clk_src", + "cam_cc_icp_ahb_clk", + "cam_cc_icp_clk_src", + "cam_cc_icp_clk"; + clock-rates = <300000000 0 480000000 0>, + <400000000 0 600000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + nrt-device; + fw_name = "qcom/sa8775p/CAMERA_ICP"; + ubwc-ipe-fetch-cfg = <0x707b 0x7083>; + ubwc-ipe-write-cfg = <0x161ef 0x1620f>; + qos-val = <0xa0a>; + operating-points-v2 = <&icp_opp_table>; + icp-version = <0x0300>; + cam_hw_pid = <12>; + cell-index = <0>; + status = "ok"; + + icp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe0: qcom,ife0 { + compatible = "qcom,vfe690"; + reg = <0x0 0xac4d000 0x0 0xd000>, + <0x0 0xac1a000 0x0 0x9400>; + reg-names = "ife0", "cam_camnoc"; + reg-cam-base = <0x4d000 0x1a000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "ife0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>; + clock-names = "cam_cc_cpas_ife_0_clk", + "cam_cc_ife_0_clk", + "cam_cc_ife_0_clk_src", + "cam_cc_ife_0_fast_ahb_clk"; + clock-rates = <0 0 480000000 0>, + <0 0 600000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_0_clk_src"; + operating-points-v2 = <&vfe0_opp_table>; + clock-control-debugfs = "true"; + ubwc-static-cfg = <0x1026 0x1036>; + cam_hw_pid = <8 16>; + cell-index = <0>; + status = "ok"; + + vfe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe1: qcom,ife1 { + compatible = "qcom,vfe690"; + reg = <0x0 0xac5a000 0x0 0xd000>, + <0x0 0xac1a000 0x0 0x9400>; + reg-names = "ife1", "cam_camnoc"; + reg-cam-base = <0x5a000 0x1a000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "ife1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_CLK_SRC>, + <&camcc CAM_CC_IFE_1_FAST_AHB_CLK>; + clock-names = "cam_cc_cpas_ife_1_clk", + "cam_cc_ife_1_clk", + "cam_cc_ife_1_clk_src", + "cam_cc_ife_1_fast_ahb_clk"; + clock-rates = <0 0 480000000 0>, + <0 0 600000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_1_clk_src"; + operating-points-v2 = <&vfe1_opp_table>; + clock-control-debugfs = "true"; + ubwc-static-cfg = <0x1026 0x1036>; + cam_hw_pid = <9 17>; + cell-index = <1>; + status = "ok"; + + vfe1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite0: qcom,ife-lite0 { + compatible = "qcom,vfe-lite690"; + reg = <0x0 0xac84000 0x0 0x1d00>; + reg-names = "ife-lite0"; + reg-cam-base = <0x84000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite0_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <2>; + status = "ok"; + + vfe_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite1: qcom,ife-lite1 { + compatible = "qcom,vfe-lite690"; + reg = <0x0 0xac88000 0x0 0x1d00>; + reg-names = "ife-lite1"; + reg-cam-base = <0x88000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite1_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <3>; + status = "ok"; + + vfe_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite2: qcom,ife-lite2 { + compatible = "qcom,vfe-lite690"; + reg = <0x0 0xac8c000 0x0 0x1d00>; + reg-names = "ife-lite2"; + reg-cam-base = <0x8c000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite2_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <4>; + status = "ok"; + + vfe_lite2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite3: qcom,ife-lite3 { + compatible = "qcom,vfe-lite690"; + reg = <0x0 0xac90000 0x0 0x1d00>; + reg-names = "ife-lite3"; + reg-cam-base = <0x90000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite3_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <5>; + status = "ok"; + + vfe_lite3_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite4: qcom,ife-lite4 { + compatible = "qcom,vfe-lite690"; + reg = <0x0 0xac94000 0x0 0x1d00>; + reg-names = "ife-lite4"; + reg-cam-base = <0x94000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite4"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite4_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <6>; + status = "ok"; + + vfe_lite4_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_ipe0: qcom,ipe0 { + compatible = "qcom,cam-ipe"; + reg = <0x0 0xac2d000 0x0 0x18000>; + reg-names = "ipe0_top"; + reg-cam-base = <0x2d000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + src-clock-name = "cam_cc_ipe_clk_src"; + clocks = <&camcc CAM_CC_CPAS_IPE_CLK>, + <&camcc CAM_CC_IPE_AHB_CLK>, + <&camcc CAM_CC_IPE_CLK>, + <&camcc CAM_CC_IPE_CLK_SRC>, + <&camcc CAM_CC_IPE_FAST_AHB_CLK>, + <&camcc CAM_CC_FAST_AHB_CLK_SRC>; + clock-names = "cam_cc_cpas_ipe_clk", + "cam_cc_ipe_ahb_clk", + "cam_cc_ipe_clk", + "cam_cc_ipe_clk_src", + "cam_cc_ipe_fast_ahb_clk", + "cam_cc_fast_ahb_clk_src"; + clock-rates = <0 0 0 480000000 0 300000000>, + <0 0 0 600000000 0 400000000>; + clock-cntl-level = "svs_l1", "nominal"; + operating-points-v2 = <&ipe0_opp_table>; + nrt-device; + cam_hw_pid = <14 15>; + cell-index = <0>; + status = "ok"; + + ipe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,rt-cdm0 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac26000 0x0 0x1000>; + reg-names = "rt-cdm0"; + reg-cam-base = <0x26000>; + interrupts = ; + interrupt-names = "rt-cdm0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <300000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table0>; + clock-cntl-level = "svs"; + cdm-client-names = "ife0", "dualife0"; + single-context-cdm; + cell-index = <0>; + status = "ok"; + + cdm_cpas_opp_table0: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,rt-cdm1 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac27000 0x0 0x1000>; + reg-names = "rt-cdm1"; + reg-cam-base = <0x27000>; + interrupts = ; + interrupt-names = "rt-cdm1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <300000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table1>; + clock-cntl-level = "svs_l1"; + cdm-client-names = "ife1", "dualife1"; + single-context-cdm; + cell-index = <1>; + status = "ok"; + + cdm_cpas_opp_table1: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,rt-cdm2 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac28000 0x0 0x1000>; + reg-names = "rt-cdm2"; + reg-cam-base = <0x28000>; + interrupts = ; + interrupt-names = "rt-cdm2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-rates = <80000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table2>; + clock-cntl-level = "svs_l1"; + cdm-client-names = "ife2", "ife3", "ife4", "ife5", "ife6"; + single-context-cdm; + cell-index = <2>; + status = "ok"; + + cdm_cpas_opp_table2: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,rt-cdm3 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac29000 0x0 0x1000>; + reg-names = "rt-cdm3"; + reg-cam-base = <0x29000>; + interrupts = ; + interrupt-names = "rt-cdm3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-rates = <80000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table3>; + clock-cntl-level = "svs"; + cdm-client-names = "ife2", "ife3", "ife4", "ife5", "ife6"; + single-context-cdm; + cell-index = <3>; + status = "ok"; + + cdm_cpas_opp_table3: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_sfe_lite0: qcom,sfe-lite0 { + compatible = "qcom,sfe-lite690"; + reg = <0x0 0xac74000 0x0 0x1000>; + reg-names = "sfe-lite0"; + reg-cam-base = <0x74000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "sfe-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SFE_LITE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_SFE_LITE_0_CLK>, + <&camcc CAM_CC_CPAS_SFE_LITE_0_CLK>; + clock-names = "cam_cc_sfe_lite_0_fast_ahb_clk", + "cam_cc_sfe_lite_0_clk", + "cam_cc_cpas_sfe_lite_0_clk"; + clock-rates = <0 480000000 300000000>, + <0 600000000 400000000>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_sfe_lite_0_clk"; + operating-points-v2 = <&cam_sfe_lite_opp_table0>; + cam_hw_pid = <4>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "ok"; + + cam_sfe_lite_opp_table0: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_sfe_lite1: qcom,sfe-lite1 { + compatible = "qcom,sfe-lite690"; + reg = <0x0 0xac75000 0x0 0x1000>; + reg-names = "sfe-lite1"; + reg-cam-base = <0x75000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "sfe-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SFE_LITE_1_FAST_AHB_CLK>, + <&camcc CAM_CC_SFE_LITE_1_CLK>, + <&camcc CAM_CC_CPAS_SFE_LITE_1_CLK>; + clock-names = "cam_cc_sfe_lite_1_fast_ahb_clk", + "cam_cc_sfe_lite_1_clk", + "cam_cc_cpas_sfe_1_clk"; + clock-rates = <0 480000000 300000000>, + <0 600000000 400000000>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_sfe_lite_1_clk"; + operating-points-v2 = <&cam_sfe_lite_opp_table1>; + cam_hw_pid = <5>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "ok"; + + cam_sfe_lite_opp_table1: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; +}; + +&tlmm { + cam_sensor_mclk0_active: cam-sensor-mclk0-active { + /* MCLK0 */ + mux { + pins = "gpio72"; + function = "cam_mclk"; + }; + + config { + pins = "gpio72"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk0_suspend: cam-sensor-mclk0-suspend { + /* MCLK0 */ + mux { + pins = "gpio72"; + function = "cam_mclk"; + }; + + config { + pins = "gpio72"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_active: cam-sensor-mclk1-active { + /* MCLK1 */ + mux { + pins = "gpio73"; + function = "cam_mclk"; + }; + + config { + pins = "gpio73"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_suspend: cam-sensor-mclk1-suspend { + /* MCLK1 */ + mux { + pins = "gpio73"; + function = "cam_mclk"; + }; + + config { + pins = "gpio73"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_active: cam-sensor-mclk2-active { + /* MCLK2 */ + mux { + pins = "gpio74"; + function = "cam_mclk"; + }; + + config { + pins = "gpio74"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_suspend: cam-sensor-mclk2-suspend { + /* MCLK2 */ + mux { + pins = "gpio74"; + function = "cam_mclk"; + }; + + config { + pins = "gpio74"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk3_active: cam-sensor-mclk3-active { + /* MCLK3 */ + mux { + pins = "gpio75"; + function = "cam_mclk"; + }; + + config { + pins = "gpio75"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk3_suspend: cam-sensor-mclk3-suspend { + /* MCLK3 */ + mux { + pins = "gpio75"; + function = "cam_mclk"; + }; + + config { + pins = "gpio75"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci0_active: cci0-active { + mux { + /* CLK, DATA */ + pins = "gpio61","gpio60"; + function = "cci_i2c"; + }; + + config { + pins = "gpio61","gpio60"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci0_suspend: cci0-suspend { + mux { + /* CLK, DATA */ + pins = "gpio61","gpio60"; + function = "cci_i2c"; + }; + + config { + pins = "gpio61","gpio60"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci1_active: cci1-active { + mux { + /* CLK, DATA */ + pins = "gpio53","gpio52"; + function = "cci_i2c"; + }; + + config { + pins = "gpio53","gpio52"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci1_suspend: cci1-suspend { + mux { + /* CLK, DATA */ + pins = "gpio53","gpio52"; + function = "cci_i2c"; + }; + + config { + pins = "gpio53","gpio52"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci2_active: cci2-active { + mux { + /* CLK, DATA */ + pins = "gpio63","gpio62"; + function = "cci_i2c"; + }; + + config { + pins = "gpio63","gpio62"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci2_suspend: cci2-suspend { + mux { + /* CLK, DATA */ + pins = "gpio63","gpio62"; + function = "cci_i2c"; + }; + + config { + pins = "gpio63","gpio62"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci3_active: cci3-active { + mux { + /* CLK, DATA */ + pins = "gpio55","gpio54"; + function = "cci_i2c"; + }; + + config { + pins = "gpio55","gpio54"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci3_suspend: cci3-suspend { + mux { + /* CLK, DATA */ + pins = "gpio55","gpio54"; + function = "cci_i2c"; + }; + + config { + pins = "gpio55","gpio54"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci4_active: cci4-active { + mux { + /* CLK, DATA */ + pins = "gpio65","gpio64"; + function = "cci_i2c"; + }; + + config { + pins = "gpio65","gpio64"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci4_suspend: cci4-suspend { + mux { + /* CLK, DATA */ + pins = "gpio65","gpio64"; + function = "cci_i2c"; + }; + + config { + pins = "gpio65","gpio64"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci5_active: cci5-active { + mux { + /* CLK, DATA */ + pins = "gpio57","gpio56"; + function = "cci_i2c"; + }; + + config { + pins = "gpio57","gpio56"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci5_suspend: cci5-suspend { + mux { + /* CLK, DATA */ + pins = "gpio57","gpio56"; + function = "cci_i2c"; + }; + + config { + pins = "gpio57","gpio56"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci6_active: cci6-active { + mux { + /* CLK, DATA */ + pins = "gpio67","gpio66"; + function = "cci_i2c"; + }; + + config { + pins = "gpio67","gpio66"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci6_suspend: cci6-suspend { + mux { + /* CLK, DATA */ + pins = "gpio67","gpio66"; + function = "cci_i2c"; + }; + + config { + pins = "gpio67","gpio66"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci7_active: cci7-active { + mux { + /* CLK, DATA */ + pins = "gpio59","gpio58"; + function = "cci_i2c"; + }; + + config { + pins = "gpio59","gpio58"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; /* I2C PULL */ + }; + }; + + cci7_suspend: cci7-suspend { + mux { + /* CLK, DATA */ + pins = "gpio59","gpio58"; + function = "cci_i2c"; + }; + + config { + pins = "gpio59","gpio58"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-camx-el2.dtso b/arch/arm64/boot/dts/qcom/lemans-camx-el2.dtso new file mode 100644 index 0000000000000..c7ddbc01a0070 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-camx-el2.dtso @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +/ { + fragment@0 { + target-path = "/soc@0/qcom,cam-cpas"; + __overlay__ { + enable-secure-qos-update = <0>; + }; + }; + + fragment@1 { + target-path = "/soc@0/qcom,cam-icp"; + __overlay__ { + camera-firmware { + iommus = <&apps_smmu 0x08c1 0x0400>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-el2.dtso b/arch/arm64/boot/dts/qcom/lemans-el2.dtso new file mode 100644 index 0000000000000..98153a06b4bbf --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-el2.dtso @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/* + * Lemans specific modifications required to boot in EL2. + */ + +/dts-v1/; +/plugin/; + +&gpu_zap_shader { + status = "disabled"; +}; + +&iris { + video-firmware { + iommus = <&apps_smmu 0x0882 0x0400>; + }; +}; + +/* + * When running under Gunyah, remote processor firmware IOMMU streams is + * controlled by the Gunyah however when we take ownership of it in EL2, + * we need to configure it properly to use remote processor. + */ +&remoteproc_adsp { + iommus = <&apps_smmu 0x3000 0x0>; +}; + +&remoteproc_cdsp0 { + iommus = <&apps_smmu 0x21c0 0x0400>; +}; + +&remoteproc_cdsp1 { + iommus = <&apps_smmu 0x29c0 0x0400>; +}; + +&remoteproc_gpdsp0 { + iommus = <&apps_smmu 0x38a0 0x0>; +}; + +&remoteproc_gpdsp1 { + iommus = <&apps_smmu 0x38c0 0x0>; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-camera-sensor.dtsi b/arch/arm64/boot/dts/qcom/lemans-evk-camera-sensor.dtsi new file mode 100644 index 0000000000000..39b847cd20228 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-evk-camera-sensor.dtsi @@ -0,0 +1,1081 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&cam_cci0 { + /* GMSL deserializer 0 */ + qcom,cam-gmsl-deserializer0 { + cell-index = <0>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active>; + pinctrl-1 = <&cam_sensor_mclk0_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 7 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET0"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser0_port0: endpoint { + remote-endpoint = <&gmsl_sensor0_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser0_port1: endpoint { + remote-endpoint = <&gmsl_sensor1_ep>; + }; + }; + + port@2 { + reg = <0>; + deser0_port2: endpoint { + remote-endpoint = <&gmsl_sensor2_ep>; + }; + }; + + port@3 { + reg = <1>; + deser0_port3: endpoint { + remote-endpoint = <&gmsl_sensor3_ep>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 0 */ + qcom,cam-gmsl-sensor0 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <0>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor0_ep: endpoint { + remote-endpoint = <&deser0_port0>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 1 */ + qcom,cam-gmsl-sensor1 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <1>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor1_ep: endpoint { + remote-endpoint = <&deser0_port1>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 2 */ + qcom,cam-gmsl-sensor2 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <2>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor2_ep: endpoint { + remote-endpoint = <&deser0_port2>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 3 */ + qcom,cam-gmsl-sensor3 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <3>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor3_ep: endpoint { + remote-endpoint = <&deser0_port3>; + }; + }; + }; + + /*cam0a-ov9282*/ + qcom,cam-sensor1 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 72 0>, + <&tlmm 132 0>, + <&pmm8654au_0_gpios 7 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <20>; + status = "ok"; + }; + + /*cam24-ov13858*/ + qcom,cam-sensor24 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam24>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 72 0>, + <&tlmm 132 0>, + <&pmm8654au_0_gpios 7 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <24>; + status = "ok"; + }; + + /*cam0-cmk_imx577*/ + qcom,cam-sensor27 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam27>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 72 0>, + <&tlmm 132 0>, + <&pmm8654au_0_gpios 7 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <27>; + status = "ok"; + }; + + eeprom_cam24: qcom,eeprom24 { + compatible = "qcom,eeprom"; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 72 0>, + <&tlmm 132 0>, + <&pmm8654au_0_gpios 7 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <24>; + status = "ok"; + }; + + eeprom_cam27: qcom,eeprom27 { + compatible = "qcom,eeprom"; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + gpios = <&tlmm 72 0>, + <&tlmm 132 0>, + <&pmm8654au_0_gpios 7 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <27>; + status = "ok"; + }; +}; + +&cam_cci1 { + /* GMSL deserializer 1 */ + qcom,cam-gmsl-deserializer1 { + cell-index = <1>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active>; + pinctrl-1 = <&cam_sensor_mclk1_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 8 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser1_port0: endpoint { + remote-endpoint = <&gmsl_sensor4_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser1_port1: endpoint { + remote-endpoint = <&gmsl_sensor5_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser1_port2: endpoint { + remote-endpoint = <&gmsl_sensor6_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser1_port3: endpoint { + remote-endpoint = <&gmsl_sensor7_ep>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 0 */ + qcom,cam-gmsl-sensor4 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <4>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor4_ep: endpoint { + remote-endpoint = <&deser1_port0>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 1 */ + qcom,cam-gmsl-sensor5 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <5>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor5_ep: endpoint { + remote-endpoint = <&deser1_port1>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 2 */ + qcom,cam-gmsl-sensor6 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <6>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor6_ep: endpoint { + remote-endpoint = <&deser1_port2>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 3 */ + qcom,cam-gmsl-sensor7 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <7>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor7_ep: endpoint { + remote-endpoint = <&deser1_port3>; + }; + }; + }; + + /*cam1-ov9282*/ + qcom,cam-sensor21 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 73 0>, + <&tlmm 133 0>, + <&pmm8654au_0_gpios 8 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK1", + "CAMIF_RESET1", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <21>; + status = "ok"; + }; + + /*cam1-cmk_imx577*/ + qcom,cam-sensor28 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam28>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 73 0>, + <&tlmm 133 0>, + <&pmm8654au_0_gpios 8 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK1", + "CAMIF_RESET1", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <28>; + status = "ok"; + }; + + /*cam1-ov9282*/ + qcom,cam-sensor31 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + gpios = <&tlmm 73 0>, + <&expander2 2 0>, + <&pmm8654au_0_gpios 8 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK1", + "CAMIF_RESET1", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <31>; + status = "ok"; + }; + + eeprom_cam28: qcom,eeprom28 { + compatible = "qcom,eeprom"; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 73 0>, + <&tlmm 133 0>, + <&pmm8654au_0_gpios 8 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK1", + "CAM_RESET1", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <28>; + status = "ok"; + }; +}; + +&cam_cci2 { + /* GMSL deserializer 2 */ + qcom,cam-gmsl-deserializer2 { + cell-index = <2>; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active>; + pinctrl-1 = <&cam_sensor_mclk2_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 9 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET2"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser2_port0: endpoint { + remote-endpoint = <&gmsl_sensor8_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser2_port1: endpoint { + remote-endpoint = <&gmsl_sensor9_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser2_port2: endpoint { + remote-endpoint = <&gmsl_sensor10_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser2_port3: endpoint { + remote-endpoint = <&gmsl_sensor11_ep>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 0 */ + qcom,cam-gmsl-sensor8 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <8>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor8_ep: endpoint { + remote-endpoint = <&deser2_port0>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 1 */ + qcom,cam-gmsl-sensor9 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <9>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor9_ep: endpoint { + remote-endpoint = <&deser2_port1>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 2 */ + qcom,cam-gmsl-sensor10 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <10>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor10_ep: endpoint { + remote-endpoint = <&deser2_port2>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 3 */ + qcom,cam-gmsl-sensor11 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <11>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor11_ep: endpoint { + remote-endpoint = <&deser2_port3>; + }; + }; + }; + + /*cam2-ov9282*/ + qcom,cam-sensor22 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 74 0>, + <&tlmm 134 0>, + <&pmm8654au_0_gpios 9 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK2", + "CAMIF_RESET2", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <22>; + status = "ok"; + }; + + /*cam2-cmk_imx577*/ + qcom,cam-sensor29 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam29>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 74 0>, + <&tlmm 134 0>, + <&pmm8654au_0_gpios 9 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK2", + "CAMIF_RESET2", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <29>; + status = "ok"; + }; + + eeprom_cam29: qcom,eeprom29 { + compatible = "qcom,eeprom"; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 74 0>, + <&tlmm 134 0>, + <&pmm8654au_0_gpios 9 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK2", + "CAM_RESET2", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <29>; + status = "ok"; + }; +}; + +&cam_cci3 { + /* GMSL deserializer 3 */ + qcom,cam-gmsl-deserializer3 { + cell-index = <3>; + csiphy-sd-index = <3>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active>; + pinctrl-1 = <&cam_sensor_mclk3_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&pmm8654au_0_gpios 10 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET3"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK3_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser3_port0: endpoint { + remote-endpoint = <&gmsl_sensor12_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser3_port1: endpoint { + remote-endpoint = <&gmsl_sensor13_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser3_port2: endpoint { + remote-endpoint = <&gmsl_sensor14_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser3_port3: endpoint { + remote-endpoint = <&gmsl_sensor15_ep>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 0 */ + qcom,cam-gmsl-sensor12 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <12>; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor12_ep: endpoint { + remote-endpoint = <&deser3_port0>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 1 */ + qcom,cam-gmsl-sensor13 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <13>; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor13_ep: endpoint { + remote-endpoint = <&deser3_port1>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 2 */ + qcom,cam-gmsl-sensor14 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <14>; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor14_ep: endpoint { + remote-endpoint = <&deser3_port2>; + }; + }; + }; + + /* GMSL deserializer 3 sensor 3 */ + qcom,cam-gmsl-sensor15 { + cell-index = <15>; + compatible = "qcom,cam-gmsl-sensor"; + csiphy-sd-index = <3>; + status = "ok"; + + port { + gmsl_sensor15_ep: endpoint { + remote-endpoint = <&deser3_port3>; + }; + }; + }; + + /*cam3-ov9282*/ + qcom,cam-sensor23 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <3>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active + &cam_sensor_active_rst3>; + pinctrl-1 = <&cam_sensor_mclk3_suspend + &cam_sensor_suspend_rst3>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 75 0>, + <&tlmm 135 0>, + <&pmm8654au_0_gpios 10 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK3", + "CAMIF_RESET3", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK3_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <23>; + status = "ok"; + }; + + /*cam3-cmk_imx577*/ + qcom,cam-sensor30 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <3>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam30>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-names = "cam_default", "cam_suspend"; + pinctrl-0 = <&cam_sensor_mclk3_active + &cam_sensor_active_rst3>; + pinctrl-1 = <&cam_sensor_mclk3_suspend + &cam_sensor_suspend_rst3>; + gpios = <&tlmm 75 0>, + <&tlmm 135 0>, + <&pmm8654au_0_gpios 10 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK3", + "CAMIF_RESET3", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK3_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <30>; + status = "ok"; + }; + + eeprom_cam30: qcom,eeprom30 { + compatible = "qcom,eeprom"; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active + &cam_sensor_active_rst3>; + pinctrl-1 = <&cam_sensor_mclk3_suspend + &cam_sensor_suspend_rst3>; + gpios = <&tlmm 75 0>, + <&tlmm 135 0>, + <&pmm8654au_0_gpios 10 0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK3", + "CAM_RESET3", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK3_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <30>; + status = "ok"; + }; +}; + +&soc { + qcom,cam-res-mgr { + compatible = "qcom,cam-res-mgr"; + gpios-shared = <518 519 520 521>; + gpios-shared-pinctrl = <633>; + shared-pctrl-gpio-names = "mclk1"; + pinctrl-0 = <&cam_sensor_mclk1_active>; + pinctrl-1 = <&cam_sensor_mclk1_suspend>; + pinctrl-names = "mclk1_active", "mclk1_suspend"; + status = "ok"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-camx.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-camx.dtso new file mode 100644 index 0000000000000..520fb14660142 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-evk-camx.dtso @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "lemans-camera.dtsi" +#include "lemans-evk-camera-sensor.dtsi" + +&camss { + status = "disabled"; +}; + +&tlmm { + cam_sensor_active_rst0: cam-sensor-active-rst0 { + /* RESET */ + mux { + pins = "gpio132"; + function = "gpio"; + }; + + config { + pins = "gpio132"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst1: cam-sensor-active-rst1 { + /* RESET */ + mux { + pins = "gpio133"; + function = "gpio"; + }; + + config { + pins = "gpio133"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst2: cam-sensor-active-rst2 { + /* RESET */ + mux { + pins = "gpio134"; + function = "gpio"; + }; + + config { + pins = "gpio134"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst3: cam-sensor-active-rst3 { + /* RESET */ + mux { + pins = "gpio135"; + function = "gpio"; + }; + + config { + pins = "gpio135"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_suspend_rst0: cam-sensor-suspend-rst0 { + /* RESET */ + mux { + pins = "gpio132"; + function = "gpio"; + }; + + config { + pins = "gpio132"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst1: cam-sensor-suspend-rst1 { + /* RESET */ + mux { + pins = "gpio133"; + function = "gpio"; + }; + + config { + pins = "gpio133"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst2: cam-sensor-suspend-rst2 { + /* RESET */ + mux { + pins = "gpio134"; + function = "gpio"; + }; + + config { + pins = "gpio134"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst3: cam-sensor-suspend-rst3 { + /* RESET */ + mux { + pins = "gpio135"; + function = "gpio"; + }; + + config { + pins = "gpio135"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-emmc.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-emmc.dtso new file mode 100644 index 0000000000000..52739d39a1ff8 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-evk-emmc.dtso @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include + +/ { + vmmc_sdc1: regulator-l8c { + compatible = "regulator-fixed"; + regulator-name = "vreg-sdc1"; + + regulator-min-microvolt = <2960000>; + regulator-max-microvolt = <2960000>; + }; + + vqmmc_sdc1: regulator-s4a { + compatible = "regulator-fixed"; + regulator-name = "vqmmc-sdc1"; + + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; +}; + +&sdhc { + vmmc-supply = <&vmmc_sdc1>; + vqmmc-supply = <&vqmmc_sdc1>; + + pinctrl-0 = <&sdc_default>, <&sdc_rclk>; + pinctrl-1 = <&sdc_sleep>, <&sdc_rclk_sleep>; + + pinctrl-names = "default", "sleep"; + + supports-cqe; + non-removable; + + qcom,dll-config = <0x000F64EC>; + max-frequency = <50000000>; + + bus-width = <8>; + no-sd; + no-sdio; + + status = "okay"; +}; + +&tlmm { + sdc_rclk: sdc1-rclk-state { + pins = "sdc1_rclk"; + bias-pull-down; + }; + + sdc_rclk_sleep: sdc1-rclk-sleep-state { + pins = "sdc1_rclk"; + drive-strength = <2>; + bias-bus-hold; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso new file mode 100644 index 0000000000000..44bd9b1a17652 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include + +&{/} { + model = "Qualcomm Technologies, Inc. Lemans-evk IFP Mezzanine"; + + dp2-connector { + compatible = "dp-connector"; + label = "eDP2"; + type = "full-size"; + + port { + dp2_connector_in: endpoint { + remote-endpoint = <&mdss1_dp0_out>; + }; + }; + }; + + dp3-connector { + compatible = "dp-connector"; + label = "eDP3"; + type = "full-size"; + + port { + dp3_connector_in: endpoint { + remote-endpoint = <&mdss1_dp1_out>; + }; + }; + }; + + vreg_0p9: regulator-0v9 { + compatible = "regulator-fixed"; + regulator-name = "VREG_0P9"; + + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + }; + + vreg_1p8: regulator-1v8 { + compatible = "regulator-fixed"; + regulator-name = "VREG_1P8"; + + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; +}; + +ðernet1 { + phy-handle = <&hsgmii_phy1>; + phy-mode = "2500base-x"; + + pinctrl-0 = <ðernet1_default>; + pinctrl-names = "default"; + + snps,mtl-rx-config = <&mtl_rx_setup1>; + snps,mtl-tx-config = <&mtl_tx_setup1>; + + nvmem-cells = <&mac_addr1>; + nvmem-cell-names = "mac-address"; + + status = "okay"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + hsgmii_phy1: ethernet-phy@18 { + compatible = "ethernet-phy-id004d.d101"; + reg = <0x18>; + reset-gpios = <&pmm8654au_2_gpios 9 GPIO_ACTIVE_LOW>; + reset-assert-us = <11000>; + reset-deassert-us = <70000>; + }; + }; + + mtl_rx_setup1: rx-queues-config { + snps,rx-queues-to-use = <4>; + snps,rx-sched-sp; + + queue0 { + snps,dcb-algorithm; + snps,map-to-dma-channel = <0x0>; + snps,route-up; + snps,priority = <0x1>; + }; + + queue1 { + snps,dcb-algorithm; + snps,map-to-dma-channel = <0x1>; + snps,route-ptp; + }; + + queue2 { + snps,avb-algorithm; + snps,map-to-dma-channel = <0x2>; + snps,route-avcp; + }; + + queue3 { + snps,avb-algorithm; + snps,map-to-dma-channel = <0x3>; + snps,priority = <0xc>; + }; + }; + + mtl_tx_setup1: tx-queues-config { + snps,tx-queues-to-use = <4>; + + queue0 { + snps,dcb-algorithm; + }; + + queue1 { + snps,dcb-algorithm; + }; + + queue2 { + snps,avb-algorithm; + snps,send_slope = <0x1000>; + snps,idle_slope = <0x1000>; + snps,high_credit = <0x3e800>; + snps,low_credit = <0xffc18000>; + }; + + queue3 { + snps,avb-algorithm; + snps,send_slope = <0x1000>; + snps,idle_slope = <0x1000>; + snps,high_credit = <0x3e800>; + snps,low_credit = <0xffc18000>; + }; + }; +}; + +&i2c18 { + #address-cells = <1>; + #size-cells = <0>; + + eeprom@52 { + compatible = "giantec,gt24c256c", "atmel,24c256"; + reg = <0x52>; + pagesize = <64>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + mac_addr1: mac-addr@0 { + reg = <0x0 0x6>; + }; + }; + }; +}; + +&mdss1 { + status = "okay"; +}; + +&mdss1_dp0 { + pinctrl-0 = <&dp2_hot_plug_det>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&mdss1_dp1 { + pinctrl-0 = <&dp3_hot_plug_det>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&mdss1_dp0_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + remote-endpoint = <&dp2_connector_in>; +}; + +&mdss1_dp1_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + remote-endpoint = <&dp3_connector_in>; +}; + +&mdss1_dp0_phy { + status = "okay"; +}; + +&mdss1_dp1_phy { + status = "okay"; +}; + +&pcie0 { + iommu-map = <0x0 &pcie_smmu 0x0 0x1>, + <0x100 &pcie_smmu 0x1 0x1>, + <0x208 &pcie_smmu 0x2 0x1>, + <0x210 &pcie_smmu 0x3 0x1>, + <0x218 &pcie_smmu 0x4 0x1>, + <0x300 &pcie_smmu 0x5 0x1>, + <0x400 &pcie_smmu 0x6 0x1>, + <0x500 &pcie_smmu 0x7 0x1>, + <0x501 &pcie_smmu 0x8 0x1>; +}; + +&pcieport0 { + #address-cells = <3>; + #size-cells = <2>; + + pcie@0,0 { + compatible = "pci1179,0623"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x2 0xff>; + + vddc-supply = <&vreg_0p9>; + vdd18-supply = <&vreg_1p8>; + vdd09-supply = <&vreg_0p9>; + vddio1-supply = <&vreg_1p8>; + vddio2-supply = <&vreg_1p8>; + vddio18-supply = <&vreg_1p8>; + + i2c-parent = <&i2c18 0x77>; + + resx-gpios = <&tlmm 140 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&tc9563_resx_n>; + pinctrl-names = "default"; + + pcie@1,0 { + reg = <0x20800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x3 0xff>; + }; + + pcie@2,0 { + reg = <0x21000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x4 0xff>; + }; + + pcie@3,0 { + reg = <0x21800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x5 0xff>; + + pci@0,0 { + reg = <0x50000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + + pci@0,1 { + reg = <0x50100 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + }; + }; +}; + +&serdes1 { + phy-supply = <&vreg_l5a>; + + status = "okay"; +}; + +&tlmm { + dp2_hot_plug_det: dp2-hot-plug-det-state { + pins = "gpio104"; + function = "edp2_hot"; + bias-disable; + }; + + dp3_hot_plug_det: dp3-hot-plug-det-state { + pins = "gpio103"; + function = "edp3_hot"; + bias-disable; + }; + + ethernet1_default: ethernet1-default-state { + ethernet1-mdc-pins { + pins = "gpio20"; + function = "emac1_mdc"; + drive-strength = <16>; + bias-pull-up; + }; + + ethernet1-mdio-pins { + pins = "gpio21"; + function = "emac1_mdio"; + drive-strength = <16>; + bias-pull-up; + }; + }; + + tc9563_resx_n: tc9563-resx-state { + pins = "gpio140"; + function = "gpio"; + bias-disable; + /* Reset pin of tc9563 is active low hence set default + * state of this pin to output-high. + */ + output-high; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-sd-card.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-sd-card.dtso new file mode 100644 index 0000000000000..007ab63473495 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-evk-sd-card.dtso @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include + +&sdhc { + vmmc-supply = <&vmmc_sdc>; + vqmmc-supply = <&vreg_sdc>; + + pinctrl-0 = <&sdc_default>, <&sd_cd>; + pinctrl-1 = <&sdc_sleep>, <&sd_cd>; + pinctrl-names = "default", "sleep"; + + bus-width = <4>; + cd-gpios = <&tlmm 36 GPIO_ACTIVE_LOW>; + no-mmc; + no-sdio; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-staging.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-staging.dtso new file mode 100644 index 0000000000000..cf345ec5d06c9 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-evk-staging.dtso @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include + +&i2c18 { + eeprom@52 { + nvmem-layout { + mac_addr2: mac-addr@6 { + reg = <0x6 0x6>; + }; + mac_addr3: mac-addr@c { + reg = <0xc 0x6>; + }; + }; + }; +}; + +&pcieport0 { + pcie@0,0 { + pcie@3,0 { + pci@0,0 { + interrupts-extended = <&tlmm 56 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + nvmem-cells = <&mac_addr2>; + nvmem-cell-names = "mac-address"; + pinctrl-names = "default"; + pinctrl-0 = <&aqr_intn_wol_sig>; + phy-reset-gpios = <&tlmm 76 GPIO_ACTIVE_HIGH>; + reset-deassert-us = <221000>; + }; + + pci@0,1 { + interrupts-extended = <&tlmm 57 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + nvmem-cells = <&mac_addr3>; + nvmem-cell-names = "mac-address"; + pinctrl-names = "default"; + pinctrl-0 = <&napa_intn_wol_sig>; + phy-reset-gpios = <&tlmm 77 GPIO_ACTIVE_HIGH>; + reset-deassert-us = <20000>; + }; + }; + }; +}; + +&tlmm { + qps615_intn_wol { + aqr_intn_wol_sig: aqr-intn-wol-sig { + pins = "gpio56"; + function = "gpio"; + input-enable; + bias-disable; + }; + napa_intn_wol_sig: napa-intn-wol-sig { + pins = "gpio57"; + function = "gpio"; + input-enable; + bias-disable; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk.dts b/arch/arm64/boot/dts/qcom/lemans-evk.dts index c7dc9b8f44578..4589f76ddc433 100644 --- a/arch/arm64/boot/dts/qcom/lemans-evk.dts +++ b/arch/arm64/boot/dts/qcom/lemans-evk.dts @@ -6,6 +6,7 @@ /dts-v1/; #include +#include #include #include @@ -20,6 +21,8 @@ ethernet0 = ðernet0; mmc1 = &sdhc; serial0 = &uart10; + serial1 = &uart17; + serial2 = &uart0; }; dmic: audio-codec-0 { @@ -37,6 +40,93 @@ stdout-path = "serial0:115200n8"; }; + connector-0 { + compatible = "usb-c-connector"; + label = "USB0-Type-C"; + data-role = "dual"; + power-role = "dual"; + + vbus-supply = <&vbus_supply_regulator_0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb0_con_hs_ep: endpoint { + remote-endpoint = <&usb_0_dwc3_hs>; + }; + }; + port@1 { + reg = <1>; + + usb0_con_ss_ep: endpoint { + remote-endpoint = <&hd3ss3220_0_in_ep>; + }; + }; + }; + }; + + connector-1 { + compatible = "usb-c-connector"; + label = "USB1-Type-C"; + data-role = "host"; + power-role = "source"; + + vbus-supply = <&vbus_supply_regulator_1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb1_con_ss_ep: endpoint { + remote-endpoint = <&hd3ss3220_1_in_ep>; + }; + }; + + port@1 { + reg = <1>; + + usb1_hs_in: endpoint { + remote-endpoint = <&usb_hub_2_1>; + }; + + }; + + port@2 { + reg = <2>; + + usb1_ss_in: endpoint { + remote-endpoint = <&usb_hub_3_1>; + }; + }; + }; + }; + + connector-2 { + compatible = "gpio-usb-b-connector", "usb-b-connector"; + label = "micro-USB"; + type = "micro"; + + id-gpios = <&pmm8654au_2_gpios 11 GPIO_ACTIVE_HIGH>; + vbus-gpios = <&expander3 3 GPIO_ACTIVE_HIGH>; + vbus-supply = <&vbus_supply_regulator_2>; + + pinctrl-names = "default"; + pinctrl-0 = <&usb2_id>; + + port { + usb2_con_hs_ep: endpoint { + remote-endpoint = <&usb_2_dwc3_hs>; + }; + }; + }; + edp0-connector { compatible = "dp-connector"; label = "EDP0"; @@ -99,6 +189,50 @@ sound-dai = <&q6apm>; }; }; + + dp0-dai-link { + link-name = "DisplayPort0 Playback"; + + cpu { + sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>; + }; + + codec { + sound-dai = <&mdss0_dp0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + }; + + vbus_supply_regulator_0: regulator-vbus-supply-0 { + compatible = "regulator-fixed"; + regulator-name = "vbus_supply_0"; + gpio = <&expander1 2 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + vbus_supply_regulator_1: regulator-vbus-supply-1 { + compatible = "regulator-fixed"; + regulator-name = "vbus_supply_1"; + gpio = <&expander1 3 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-boot-on; + enable-active-high; + }; + + vbus_supply_regulator_2: vbus-supply-regulator-2 { + compatible = "regulator-fixed"; + regulator-name = "vbus_supply_2"; + gpio = <&pmm8654au_1_gpios 9 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; }; vmmc_sdc: regulator-vmmc-sdc { @@ -109,6 +243,17 @@ regulator-max-microvolt = <2950000>; }; + vreg_dcin_12v: regulator-dcin-12v { + compatible = "regulator-fixed"; + + regulator-name = "VREG_DCIN_12V"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + + regulator-boot-on; + regulator-always-on; + }; + vreg_sdc: regulator-vreg-sdc { compatible = "regulator-gpio"; @@ -122,6 +267,18 @@ startup-delay-us = <100>; }; + + vreg_wcn_3p3: regulator-wcn-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_WCN_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + vin-supply = <&vreg_dcin_12v>; + + regulator-boot-on; + }; }; &apps_rsc { @@ -453,6 +610,86 @@ status = "okay"; }; +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/sa8775p/a663_zap.mbn"; +}; + +&i2c11 { + status = "okay"; + + usb-typec@67 { + compatible = "ti,hd3ss3220"; + reg = <0x67>; + + interrupts-extended = <&pmm8654au_2_gpios 5 IRQ_TYPE_EDGE_FALLING>; + + id-gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&usb_id>, <&usb0_intr_state>; + pinctrl-names = "default"; + + wakeup-source; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hd3ss3220_0_in_ep: endpoint { + remote-endpoint = <&usb0_con_ss_ep>; + }; + }; + + port@1 { + reg = <1>; + + hd3ss3220_0_out_ep: endpoint { + remote-endpoint = <&usb_0_dwc3_ss>; + }; + }; + }; + }; + + usb-typec@47 { + compatible = "ti,hd3ss3220"; + reg = <0x47>; + + interrupts-extended = <&pmm8654au_2_gpios 6 IRQ_TYPE_EDGE_FALLING>; + + id-gpios = <&tlmm 51 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&usb1_id>, <&usb1_intr>; + pinctrl-names = "default"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hd3ss3220_1_in_ep: endpoint { + remote-endpoint = <&usb1_con_ss_ep>; + }; + }; + + port@1 { + reg = <1>; + + hd3ss3220_1_out_ep: endpoint { + }; + }; + }; + }; + +}; + &i2c18 { status = "okay"; @@ -461,6 +698,11 @@ reg = <0x38>; #gpio-cells = <2>; gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 138 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander0_int>; + pinctrl-names = "default"; }; expander1: gpio@39 { @@ -468,6 +710,11 @@ reg = <0x39>; #gpio-cells = <2>; gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 19 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander1_int>; + pinctrl-names = "default"; }; expander2: gpio@3a { @@ -475,6 +722,11 @@ reg = <0x3a>; #gpio-cells = <2>; gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 139 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander2_int>; + pinctrl-names = "default"; }; expander3: gpio@3b { @@ -482,6 +734,11 @@ reg = <0x3b>; #gpio-cells = <2>; gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 39 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander3_int>; + pinctrl-names = "default"; }; eeprom@50 { @@ -501,6 +758,20 @@ }; }; +&i2c19 { + status = "okay"; + + fan_controller: fan@18 { + compatible = "ti,amc6821"; + reg = <0x18>; + #pwm-cells = <2>; + + fan { + pwms = <&fan_controller 40000 PWM_POLARITY_INVERTED>; + }; + }; +}; + &iris { firmware-name = "qcom/vpu/vpu30_p4_s6_16mb.mbn"; @@ -587,15 +858,56 @@ status = "okay"; }; +&psci { + reboot-mode { + mode-bootloader = <0x10001 0x2>; + mode-edl = <0 0x1>; + }; +}; + +&pmm8654au_2_gpios { + usb0_intr_state: usb0-intr-state { + pins = "gpio5"; + function = "normal"; + input-enable; + bias-pull-up; + power-source = <0>; + }; + + usb1_intr: usb1-intr-state { + pins = "gpio6"; + function = "normal"; + input-enable; + bias-pull-up; + power-source = <0>; + }; + + usb2_id: usb2-id-state { + pins = "gpio11"; + function = "normal"; + input-enable; + bias-pull-up; + power-source = <0>; + }; +}; + +&qup_i2c19_default { + drive-strength = <2>; + bias-pull-up; +}; + &qupv3_id_0 { + firmware-name = "qcom/sa8775p/qupv3fw.elf"; status = "okay"; }; &qupv3_id_1 { + firmware-name = "qcom/sa8775p/qupv3fw.elf"; status = "okay"; }; &qupv3_id_2 { + firmware-name = "qcom/sa8775p/qupv3fw.elf"; status = "okay"; }; @@ -629,24 +941,19 @@ status = "okay"; }; -&sdhc { - vmmc-supply = <&vmmc_sdc>; - vqmmc-supply = <&vreg_sdc>; - - pinctrl-0 = <&sdc_default>, <&sd_cd>; - pinctrl-1 = <&sdc_sleep>, <&sd_cd>; - pinctrl-names = "default", "sleep"; - - bus-width = <4>; - cd-gpios = <&tlmm 36 GPIO_ACTIVE_LOW>; - no-mmc; - no-sdio; - +&spi16 { status = "okay"; + + tpm@0 { + compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi"; + reg = <0>; + spi-max-frequency = <20000000>; + }; }; &serdes0 { phy-supply = <&vreg_l5a>; + vdda-0p9-supply = <&vreg_l4a>; status = "okay"; }; @@ -672,6 +979,30 @@ }; }; + expander0_int: expander0-int-state { + pins = "gpio138"; + function = "gpio"; + bias-pull-up; + }; + + expander1_int: expander1-int-state { + pins = "gpio19"; + function = "gpio"; + bias-pull-up; + }; + + expander2_int: expander2-int-state { + pins = "gpio139"; + function = "gpio"; + bias-pull-up; + }; + + expander3_int: expander3-int-state { + pins = "gpio39"; + function = "gpio"; + bias-pull-up; + }; + pcie0_default_state: pcie0-default-state { clkreq-pins { pins = "gpio1"; @@ -718,11 +1049,34 @@ }; }; + qup_i2c11_default: qup-i2c11-state { + pins = "gpio48", "gpio49"; + function = "qup1_se4"; + drive-strength = <2>; + bias-pull-up; + }; + sd_cd: sd-cd-state { pins = "gpio36"; function = "gpio"; bias-pull-up; }; + + usb_id: usb-id-state { + pins = "gpio50"; + function = "gpio"; + bias-pull-up; + }; + + usb1_id: usb1-id-state { + pins = "gpio51"; + function = "gpio"; + bias-pull-up; + }; +}; + +&uart0 { + status = "okay"; }; &uart10 { @@ -733,6 +1087,24 @@ status = "okay"; }; +&uart17 { + status = "okay"; + + bluetooth: bluetooth { + compatible = "qcom,wcn6855-bt"; + max-speed = <3200000>; + + vddrfacmn-supply = <&vreg_wcn_3p3>; + vddaon-supply = <&vreg_wcn_3p3>; + vddwlcx-supply = <&vreg_wcn_3p3>; + vddwlmx-supply = <&vreg_wcn_3p3>; + vddbtcmx-supply = <&vreg_wcn_3p3>; + vddrfa0p8-supply = <&vreg_wcn_3p3>; + vddrfa1p2-supply = <&vreg_wcn_3p3>; + vddrfa1p8-supply = <&vreg_wcn_3p3>; + }; +}; + &ufs_mem_hc { reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; vcc-supply = <&vreg_l8a>; @@ -751,11 +1123,17 @@ }; &usb_0 { - dr_mode = "peripheral"; - status = "okay"; }; +&usb_0_dwc3_hs { + remote-endpoint = <&usb0_con_hs_ep>; +}; + +&usb_0_dwc3_ss { + remote-endpoint = <&hd3ss3220_0_out_ep>; +}; + &usb_0_hsphy { vdda-pll-supply = <&vreg_l7a>; vdda18-supply = <&vreg_l6c>; @@ -771,6 +1149,134 @@ status = "okay"; }; +&usb_1 { + dr_mode = "host"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "okay"; + + usb_hub_2_x: hub@1 { + compatible = "usb5e3,610"; + reg = <1>; + + peer-hub = <&usb_hub_3_x>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + usb_hub_2_1: endpoint { + remote-endpoint = <&usb1_hs_in>; + }; + }; + + /* + * Port-2 and port-3 are not connected to anything on corekit. + */ + port@2 { + reg = <2>; + + usb_hub_2_2: endpoint { + }; + }; + + port@3 { + reg = <3>; + + usb_hub_2_3: endpoint { + }; + }; + + /* + * Port-4 is connected to M.2 E key connector on corekit. + */ + port@4 { + reg = <4>; + + usb_hub_2_4: endpoint { + }; + }; + }; + }; + + usb_hub_3_x: hub@2 { + compatible = "usb5e3,625"; + reg = <2>; + + peer-hub = <&usb_hub_2_x>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + usb_hub_3_1: endpoint { + remote-endpoint = <&usb1_ss_in>; + }; + }; + + port@2 { + reg = <2>; + + usb_hub_3_2: endpoint { + }; + }; + + port@3 { + reg = <3>; + + usb_hub_3_3: endpoint { + }; + }; + + port@4 { + reg = <4>; + + usb_hub_3_4: endpoint { + }; + }; + }; + }; +}; + +&usb_1_hsphy { + vdda-pll-supply = <&vreg_l7a>; + vdda18-supply = <&vreg_l6c>; + vdda33-supply = <&vreg_l9a>; + + status = "okay"; +}; + +&usb_1_qmpphy { + vdda-phy-supply = <&vreg_l1c>; + vdda-pll-supply = <&vreg_l7a>; + + status = "okay"; +}; + +&usb_2 { + status = "okay"; +}; + +&usb_2_dwc3_hs { + remote-endpoint = <&usb2_con_hs_ep>; +}; + +&usb_2_hsphy { + vdda-pll-supply = <&vreg_l7a>; + vdda18-supply = <&vreg_l6c>; + vdda33-supply = <&vreg_l9a>; + + status = "okay"; +}; + &xo_board_clk { clock-frequency = <38400000>; }; diff --git a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi index c69aa2f41ce29..d66b33a1812ca 100644 --- a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi @@ -180,6 +180,30 @@ }; }; + dp2-connector { + compatible = "dp-connector"; + label = "eDP2"; + type = "full-size"; + + port { + dp2_connector_in: endpoint { + remote-endpoint = <&mdss1_dp0_out>; + }; + }; + }; + + dp3-connector { + compatible = "dp-connector"; + label = "eDP3"; + type = "full-size"; + + port { + dp3_connector_in: endpoint { + remote-endpoint = <&mdss1_dp1_out>; + }; + }; + }; + dp-dsi0-connector { compatible = "dp-connector"; label = "DSI0"; @@ -436,6 +460,14 @@ }; }; +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/sa8775p/a663_zap.mbn"; +}; + &i2c11 { clock-frequency = <400000>; status = "okay"; @@ -548,8 +580,6 @@ }; &iris { - firmware-name = "qcom/vpu/vpu30_p4_s6.mbn"; - status = "okay"; }; @@ -631,6 +661,50 @@ status = "okay"; }; +&mdss1 { + status = "okay"; +}; + +&mdss1_dp0 { + pinctrl-0 = <&dp2_hot_plug_det>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&mdss1_dp0_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + remote-endpoint = <&dp2_connector_in>; +}; + +&mdss1_dp0_phy { + vdda-phy-supply = <&vreg_l1c>; + vdda-pll-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&mdss1_dp1 { + pinctrl-0 = <&dp3_hot_plug_det>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&mdss1_dp1_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + remote-endpoint = <&dp3_connector_in>; +}; + +&mdss1_dp1_phy { + vdda-phy-supply = <&vreg_l1c>; + vdda-pll-supply = <&vreg_l4a>; + + status = "okay"; +}; + &pmm8654au_0_gpios { gpio-line-names = "DS_EN", "POFF_COMPLETE", @@ -722,6 +796,13 @@ "GNSS_BOOT_MODE"; }; +&psci { + reboot-mode { + mode-bootloader = <0x10001 0x2>; + mode-edl = <0 0x1>; + }; +}; + &qupv3_id_1 { status = "okay"; }; @@ -779,11 +860,15 @@ &serdes0 { phy-supply = <&vreg_l5a>; + vdda-0p9-supply = <&vreg_l4a>; + status = "okay"; }; &serdes1 { phy-supply = <&vreg_l5a>; + vdda-0p9-supply = <&vreg_l4a>; + status = "okay"; }; @@ -808,6 +893,18 @@ bias-disable; }; + dp2_hot_plug_det: dp2-hot-plug-det-state { + pins = "gpio104"; + function = "edp2_hot"; + bias-disable; + }; + + dp3_hot_plug_det: dp3-hot-plug-det-state { + pins = "gpio103"; + function = "edp3_hot"; + bias-disable; + }; + io_expander_intr_active: io-expander-intr-active-state { pins = "gpio98"; function = "gpio"; diff --git a/arch/arm64/boot/dts/qcom/lemans-staging.dtso b/arch/arm64/boot/dts/qcom/lemans-staging.dtso new file mode 100644 index 0000000000000..ad919bfb6b19a --- /dev/null +++ b/arch/arm64/boot/dts/qcom/lemans-staging.dtso @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Lemans staging overlay - add staging-specific device tree modifications here. + */ + +/dts-v1/; +/plugin/; + +&soc { + tgu@4b0e000 { + compatible = "qcom,tgu", "arm,primecell"; + reg = <0x0 0x4b0e000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + + tgu@4b0f000 { + compatible = "qcom,tgu", "arm,primecell"; + reg = <0x0 0x4b0f000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi index cf685cb186edc..61a89cac64dcc 100644 --- a/arch/arm64/boot/dts/qcom/lemans.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans.dtsi @@ -17,10 +17,12 @@ #include #include #include +#include #include #include #include #include +#include / { interrupt-parent = <&intc>; @@ -54,6 +56,7 @@ qcom,freq-domain = <&cpufreq_hw 0>; next-level-cache = <&l2_0>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu0_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -83,6 +86,7 @@ qcom,freq-domain = <&cpufreq_hw 0>; next-level-cache = <&l2_1>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu0_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -107,6 +111,7 @@ qcom,freq-domain = <&cpufreq_hw 0>; next-level-cache = <&l2_2>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu0_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -131,6 +136,7 @@ qcom,freq-domain = <&cpufreq_hw 0>; next-level-cache = <&l2_3>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu0_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -155,6 +161,7 @@ qcom,freq-domain = <&cpufreq_hw 1>; next-level-cache = <&l2_4>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu4_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -185,6 +192,7 @@ qcom,freq-domain = <&cpufreq_hw 1>; next-level-cache = <&l2_5>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu4_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -209,6 +217,7 @@ qcom,freq-domain = <&cpufreq_hw 1>; next-level-cache = <&l2_6>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu4_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -233,6 +242,7 @@ qcom,freq-domain = <&cpufreq_hw 1>; next-level-cache = <&l2_7>; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; operating-points-v2 = <&cpu4_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -498,6 +508,30 @@ }; }; + dp_opp_table: opp-table-dp { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-270000000 { + opp-hz = /bits/ 64 <270000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-810000000 { + opp-hz = /bits/ 64 <810000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + dummy-sink { compatible = "arm,coresight-dummy-sink"; @@ -602,6 +636,30 @@ qcom,bcm-voters = <&apps_bcm_voter>; }; + mdss_mdp_opp_table: opp-table-mdp { + compatible = "operating-points-v2"; + + opp-375000000 { + opp-hz = /bits/ 64 <375000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-575000000 { + opp-hz = /bits/ 64 <575000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + + opp-650000000 { + opp-hz = /bits/ 64 <650000000>; + required-opps = <&rpmhpd_opp_turbo_l1>; + }; + }; + /* Will be updated by the bootloader. */ memory@80000000 { device_type = "memory"; @@ -622,7 +680,7 @@ interrupts = ; }; - psci { + psci: psci { compatible = "arm,psci-1.0"; method = "smc"; @@ -1098,6 +1156,18 @@ #mbox-cells = <2>; }; + qfprom: efuse@784000 { + compatible = "qcom,sa8775p-qfprom", "qcom,qfprom"; + reg = <0x0 0x00784000 0x0 0x3000>; + #address-cells = <1>; + #size-cells = <1>; + + gpu_speed_bin: gpu_speed_bin@240c { + reg = <0x240c 0x1>; + bits = <0 8>; + }; + }; + gpi_dma2: dma-controller@800000 { compatible = "qcom,sa8775p-gpi-dma", "qcom,sm6350-gpi-dma"; reg = <0x0 0x00800000 0x0 0x60000>; @@ -2753,7 +2823,11 @@ compatible = "qcom,sa8775p-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0x0 0x01d88000 0x0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { @@ -2769,6 +2843,18 @@ <&apps_smmu 0x481 0x00>; }; + crypto: crypto@1dfa000 { + compatible = "qcom,sa8775p-qce", "qcom,sm8150-qce", "qcom,qce"; + reg = <0x0 0x01dfa000 0x0 0x6000>; + dmas = <&cryptobam 4>, <&cryptobam 5>; + dma-names = "rx", "tx"; + iommus = <&apps_smmu 0x480 0x0>, + <&apps_smmu 0x481 0x0>; + interconnects = <&aggre2_noc MASTER_CRYPTO_CORE0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "memory"; + }; + ctcu@4001000 { compatible = "qcom,sa8775p-ctcu"; reg = <0x0 0x04001000 0x0 0x1000>; @@ -3838,7 +3924,9 @@ sdhc: mmc@87c4000 { compatible = "qcom,sa8775p-sdhci", "qcom,sdhci-msm-v5"; - reg = <0x0 0x087c4000 0x0 0x1000>; + reg = <0x0 0x87C4000 0x0 0x1000>, + <0x0 0x87C5000 0x0 0x1000>; + reg-names = "hc", "cqhci"; interrupts = , ; @@ -3846,9 +3934,11 @@ "pwr_irq"; clocks = <&gcc GCC_SDCC1_AHB_CLK>, - <&gcc GCC_SDCC1_APPS_CLK>; + <&gcc GCC_SDCC1_APPS_CLK>, + <&rpmhcc RPMH_CXO_CLK>; clock-names = "iface", - "core"; + "core", + "xo"; interconnects = <&aggre1_noc MASTER_SDC QCOM_ICC_TAG_ALWAYS &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, @@ -3970,7 +4060,27 @@ snps,dis-u1-entry-quirk; snps,dis-u2-entry-quirk; + usb-role-switch; status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb_0_dwc3_hs: endpoint { + }; + }; + + port@1 { + reg = <1>; + + usb_0_dwc3_ss: endpoint { + }; + }; + }; }; usb_1_hsphy: phy@88e6000 { @@ -4106,6 +4216,7 @@ <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_USB2 0>; interconnect-names = "usb-ddr", "apps-usb"; + qcom,select-utmi-as-pipe-clk; wakeup-source; iommus = <&apps_smmu 0x020 0x0>; @@ -4114,7 +4225,14 @@ snps,dis-u1-entry-quirk; snps,dis-u2-entry-quirk; + usb-role-switch; + status = "disabled"; + + port { + usb_2_dwc3_hs: endpoint { + }; + }; }; tcsr_mutex: hwlock@1f40000 { @@ -4128,6 +4246,113 @@ reg = <0x0 0x1fc0000 0x0 0x30000>; }; + gpu: gpu@3d00000 { + compatible = "qcom,adreno-663.0", "qcom,adreno"; + reg = <0x0 0x03d00000 0x0 0x40000>, + <0x0 0x03d9e000 0x0 0x1000>, + <0x0 0x03d61000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; + interrupts = ; + iommus = <&adreno_smmu 0 0xc00>, + <&adreno_smmu 1 0xc00>; + operating-points-v2 = <&gpu_opp_table>; + qcom,gmu = <&gmu>; + interconnects = <&gem_noc MASTER_GFX3D QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "gfx-mem"; + #cooling-cells = <2>; + + nvmem-cells = <&gpu_speed_bin>; + nvmem-cell-names = "speed_bin"; + + status = "disabled"; + + gpu_zap_shader: zap-shader { + memory-region = <&pil_gpu_mem>; + }; + + gpu_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-405000000 { + opp-hz = /bits/ 64 <405000000>; + opp-level = ; + opp-peak-kBps = <5285156>; + opp-supported-hw = <0x3>; + }; + + opp-530000000 { + opp-hz = /bits/ 64 <530000000>; + opp-level = ; + opp-peak-kBps = <12484375>; + opp-supported-hw = <0x2>; + }; + + opp-676000000 { + opp-hz = /bits/ 64 <676000000>; + opp-level = ; + opp-peak-kBps = <8171875>; + opp-supported-hw = <0x1>; + }; + + opp-778000000 { + opp-hz = /bits/ 64 <778000000>; + opp-level = ; + opp-peak-kBps = <10687500>; + opp-supported-hw = <0x1>; + }; + + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-level = ; + opp-peak-kBps = <12484375>; + opp-supported-hw = <0x1>; + }; + }; + }; + + gmu: gmu@3d6a000 { + compatible = "qcom,adreno-gmu-663.0", "qcom,adreno-gmu"; + reg = <0x0 0x03d6a000 0x0 0x34000>, + <0x0 0x03de0000 0x0 0x10000>, + <0x0 0x0b290000 0x0 0x10000>; + reg-names = "gmu", "rscc", "gmu_pdc"; + interrupts = , + ; + interrupt-names = "hfi", "gmu"; + clocks = <&gpucc GPU_CC_CX_GMU_CLK>, + <&gpucc GPU_CC_CXO_CLK>, + <&gcc GCC_DDRSS_GPU_AXI_CLK>, + <&gcc GCC_GPU_MEMNOC_GFX_CLK>, + <&gpucc GPU_CC_AHB_CLK>, + <&gpucc GPU_CC_HUB_CX_INT_CLK>, + <&gpucc GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK>; + clock-names = "gmu", + "cxo", + "axi", + "memnoc", + "ahb", + "hub", + "smmu_vote"; + power-domains = <&gpucc GPU_CC_CX_GDSC>, + <&gpucc GPU_CC_GX_GDSC>; + power-domain-names = "cx", + "gx"; + iommus = <&adreno_smmu 5 0xc00>; + operating-points-v2 = <&gmu_opp_table>; + + gmu_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + opp-level = ; + }; + }; + }; + gpucc: clock-controller@3d90000 { compatible = "qcom,sa8775p-gpucc"; reg = <0x0 0x03d90000 0x0 0xa000>; @@ -4342,11 +4567,13 @@ memory-region = <&pil_video_mem>; + firmware-name = "qcom/vpu/vpu30_p4_s6_16mb.mbn"; + resets = <&gcc GCC_VIDEO_AXI0_CLK_ARES>; reset-names = "bus"; - iommus = <&apps_smmu 0x0880 0x0400>, - <&apps_smmu 0x0887 0x0400>; + iommu-map = , + ; dma-coherent; status = "disabled"; @@ -4802,7 +5029,7 @@ assigned-clocks = <&dispcc0 MDSS_DISP_CC_MDSS_VSYNC_CLK>; assigned-clock-rates = <19200000>; - operating-points-v2 = <&mdss0_mdp_opp_table>; + operating-points-v2 = <&mdss_mdp_opp_table>; power-domains = <&rpmhpd SA8775P_MMCX>; interrupt-parent = <&mdss0>; @@ -4844,30 +5071,6 @@ }; }; }; - - mdss0_mdp_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-375000000 { - opp-hz = /bits/ 64 <375000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - - opp-500000000 { - opp-hz = /bits/ 64 <500000000>; - required-opps = <&rpmhpd_opp_nom>; - }; - - opp-575000000 { - opp-hz = /bits/ 64 <575000000>; - required-opps = <&rpmhpd_opp_turbo>; - }; - - opp-650000000 { - opp-hz = /bits/ 64 <650000000>; - required-opps = <&rpmhpd_opp_turbo_l1>; - }; - }; }; mdss0_dsi0: dsi@ae94000 { @@ -5034,9 +5237,11 @@ <0x0 0x0aec2000 0x0 0x1c8>; clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, - <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>; + <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; #clock-cells = <1>; #phy-cells = <0>; @@ -5053,9 +5258,11 @@ <0x0 0x0aec5000 0x0 0x1c8>; clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK>, - <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>; + <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; #clock-cells = <1>; #phy-cells = <0>; @@ -5133,30 +5340,6 @@ mdss0_dp0_out: endpoint { }; }; }; - - dp_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - - opp-270000000 { - opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; - }; - - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - - opp-810000000 { - opp-hz = /bits/ 64 <810000000>; - required-opps = <&rpmhpd_opp_nom>; - }; - }; }; mdss0_dp1: displayport-controller@af5c000 { @@ -5196,7 +5379,7 @@ phys = <&mdss0_dp1_phy>; phy-names = "dp"; - operating-points-v2 = <&dp1_opp_table>; + operating-points-v2 = <&dp_opp_table>; power-domains = <&rpmhpd SA8775P_MMCX>; #sound-dai-cells = <0>; @@ -5221,30 +5404,6 @@ mdss0_dp1_out: endpoint { }; }; }; - - dp1_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - - opp-270000000 { - opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; - }; - - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - - opp-810000000 { - opp-hz = /bits/ 64 <810000000>; - required-opps = <&rpmhpd_opp_nom>; - }; - }; }; }; @@ -6777,6 +6936,259 @@ }; }; + mdss1: display-subsystem@22000000 { + compatible = "qcom,sa8775p-mdss"; + reg = <0x0 0x22000000 0x0 0x1000>; + reg-names = "mdss"; + + interconnects = <&mmss_noc MASTER_MDP_CORE1_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_MDP_CORE1_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "mdp0-mem", + "mdp1-mem", + "cpu-cfg"; + + resets = <&dispcc1 MDSS_DISP_CC_MDSS_CORE_BCR>; + + power-domains = <&dispcc1 MDSS_DISP_CC_MDSS_CORE_GDSC>; + + clocks = <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP1_HF_AXI_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_MDP_CLK>; + + interrupts = ; + interrupt-controller; + #interrupt-cells = <1>; + + iommus = <&apps_smmu 0x1800 0x402>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + status = "disabled"; + + display-controller@22001000 { + compatible = "qcom,sa8775p-dpu"; + reg = <0x0 0x22001000 0x0 0x8f000>, + <0x0 0x220b0000 0x0 0x3000>; + reg-names = "mdp", "vbif"; + + clocks = <&gcc GCC_DISP1_HF_AXI_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_MDP_LUT_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_MDP_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_VSYNC_CLK>; + clock-names = "nrt_bus", + "iface", + "lut", + "core", + "vsync"; + + assigned-clocks = <&dispcc1 MDSS_DISP_CC_MDSS_VSYNC_CLK>; + assigned-clock-rates = <19200000>; + + operating-points-v2 = <&mdss_mdp_opp_table>; + power-domains = <&rpmhpd SA8775P_MMCX>; + + interrupt-parent = <&mdss1>; + interrupts = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dpu1_intf0_out: endpoint { + remote-endpoint = <&mdss1_dp0_in>; + }; + }; + + port@1 { + reg = <1>; + dpu1_intf4_out: endpoint { + remote-endpoint = <&mdss1_dp1_in>; + }; + }; + }; + }; + + mdss1_dp0_phy: phy@220c2a00 { + compatible = "qcom,sa8775p-edp-phy"; + reg = <0x0 0x220c2a00 0x0 0x200>, + <0x0 0x220c2200 0x0 0xd0>, + <0x0 0x220c2600 0x0 0xd0>, + <0x0 0x220c2000 0x0 0x1c8>; + + clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; + clock-names = "aux", + "cfg_ahb", + "ref"; + + #clock-cells = <1>; + #phy-cells = <0>; + + status = "disabled"; + }; + + mdss1_dp1_phy: phy@220c5a00 { + compatible = "qcom,sa8775p-edp-phy"; + reg = <0x0 0x220c5a00 0x0 0x200>, + <0x0 0x220c5200 0x0 0xd0>, + <0x0 0x220c5600 0x0 0xd0>, + <0x0 0x220c5000 0x0 0x1c8>; + + clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; + clock-names = "aux", + "cfg_ahb", + "ref"; + + #clock-cells = <1>; + #phy-cells = <0>; + + status = "disabled"; + }; + + mdss1_dp0: displayport-controller@22154000 { + compatible = "qcom,sa8775p-dp"; + reg = <0x0 0x22154000 0x0 0x104>, + <0x0 0x22154200 0x0 0x0c0>, + <0x0 0x22155000 0x0 0x770>, + <0x0 0x22156000 0x0 0x09c>, + <0x0 0x22157000 0x0 0x09c>, + <0x0 0x22158000 0x0 0x09c>, + <0x0 0x22159000 0x0 0x09c>, + <0x0 0x2215a000 0x0 0x23c>, + <0x0 0x2215b000 0x0 0x23c>; + + interrupt-parent = <&mdss1>; + interrupts = <12>; + + clocks = <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_LINK_INTF_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK>; + clock-names = "core_iface", + "core_aux", + "ctrl_link", + "ctrl_link_iface", + "stream_pixel", + "stream_1_pixel", + "stream_2_pixel", + "stream_3_pixel"; + assigned-clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK_SRC>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK_SRC>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK_SRC>; + assigned-clock-parents = <&mdss1_dp0_phy 0>, + <&mdss1_dp0_phy 1>, + <&mdss1_dp0_phy 1>, + <&mdss1_dp0_phy 1>, + <&mdss1_dp0_phy 1>; + phys = <&mdss1_dp0_phy>; + phy-names = "dp"; + + operating-points-v2 = <&dp_opp_table>; + power-domains = <&rpmhpd SA8775P_MMCX>; + + #sound-dai-cells = <0>; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + mdss1_dp0_in: endpoint { + remote-endpoint = <&dpu1_intf0_out>; + }; + }; + + port@1 { + reg = <1>; + mdss1_dp0_out: endpoint { }; + }; + }; + }; + + mdss1_dp1: displayport-controller@2215c000 { + compatible = "qcom,sa8775p-dp"; + reg = <0x0 0x2215c000 0x0 0x104>, + <0x0 0x2215c200 0x0 0x0c0>, + <0x0 0x2215d000 0x0 0x770>, + <0x0 0x2215e000 0x0 0x09c>, + <0x0 0x2215f000 0x0 0x09c>, + <0x0 0x22160000 0x0 0x09c>, + <0x0 0x22161000 0x0 0x09c>, + <0x0 0x22162000 0x0 0x23c>, + <0x0 0x22163000 0x0 0x23c>; + + interrupt-parent = <&mdss1>; + interrupts = <13>; + + clocks = <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_LINK_INTF_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK>; + clock-names = "core_iface", + "core_aux", + "ctrl_link", + "ctrl_link_iface", + "stream_pixel", + "stream_1_pixel"; + assigned-clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK_SRC>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC>, + <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC>; + assigned-clock-parents = <&mdss1_dp1_phy 0>, + <&mdss1_dp1_phy 1>, + <&mdss1_dp1_phy 1>; + phys = <&mdss1_dp1_phy>; + phy-names = "dp"; + + operating-points-v2 = <&dp_opp_table>; + power-domains = <&rpmhpd SA8775P_MMCX>; + + #sound-dai-cells = <0>; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + mdss1_dp1_in: endpoint { + remote-endpoint = <&dpu1_intf4_out>; + }; + }; + + port@1 { + reg = <1>; + mdss1_dp1_out: endpoint { }; + }; + }; + + }; + }; + dispcc1: clock-controller@22100000 { compatible = "qcom,sa8775p-dispcc1"; reg = <0x0 0x22100000 0x0 0x20000>; @@ -6784,13 +7196,13 @@ <&rpmhcc RPMH_CXO_CLK>, <&rpmhcc RPMH_CXO_CLK_A>, <&sleep_clk>, - <0>, <0>, <0>, <0>, + <&mdss1_dp0_phy 0>, <&mdss1_dp0_phy 1>, + <&mdss1_dp1_phy 0>, <&mdss1_dp1_phy 1>, <0>, <0>, <0>, <0>; power-domains = <&rpmhpd SA8775P_MMCX>; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; - status = "disabled"; }; ethernet1: ethernet@23000000 { @@ -7005,6 +7417,14 @@ }; }; }; + + cooling { + compatible = "qcom,qmi-cooling-cdsp"; + cdsp_sw0: cdsp_sw { + label = "cdsp_sw"; + #cooling-cells = <2>; + }; + }; }; remoteproc_cdsp1: remoteproc@2a300000 { @@ -7161,6 +7581,15 @@ }; }; }; + + + cooling { + compatible = "qcom,qmi-cooling-cdsp"; + cdsp_sw1: cdsp_sw { + label = "cdsp_sw"; + #cooling-cells = <2>; + }; + }; }; remoteproc_adsp: remoteproc@30000000 { @@ -7382,8 +7811,15 @@ thermal-sensors = <&tsens0 5>; + cooling-maps { + map0 { + trip = <&gpuss0_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + trips { - trip-point0 { + gpuss0_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7402,8 +7838,15 @@ thermal-sensors = <&tsens0 6>; + cooling-maps { + map0 { + trip = <&gpuss1_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + trips { - trip-point0 { + gpuss1_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7422,8 +7865,15 @@ thermal-sensors = <&tsens0 7>; + cooling-maps { + map0 { + trip = <&gpuss2_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + trips { - trip-point0 { + gpuss2_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7612,8 +8062,15 @@ thermal-sensors = <&tsens1 5>; + cooling-maps { + map0 { + trip = <&gpuss3_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + trips { - trip-point0 { + gpuss3_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7632,8 +8089,15 @@ thermal-sensors = <&tsens1 6>; + cooling-maps { + map0 { + trip = <&gpuss4_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + trips { - trip-point0 { + gpuss4_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7652,8 +8116,15 @@ thermal-sensors = <&tsens1 7>; + cooling-maps { + map0 { + trip = <&gpuss5_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + trips { - trip-point0 { + gpuss5_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7843,7 +8314,7 @@ thermal-sensors = <&tsens2 5>; trips { - trip-point0 { + nsp_0_0_0_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7855,6 +8326,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_0_0_0_alert0>; + cooling-device = <&cdsp_sw0 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-0-1-0-thermal { @@ -7863,7 +8342,7 @@ thermal-sensors = <&tsens2 6>; trips { - trip-point0 { + nsp_0_1_0_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7875,6 +8354,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_0_1_0_alert0>; + cooling-device = <&cdsp_sw0 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-0-2-0-thermal { @@ -7883,7 +8370,7 @@ thermal-sensors = <&tsens2 7>; trips { - trip-point0 { + nsp_0_2_0_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7895,6 +8382,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_0_2_0_alert0>; + cooling-device = <&cdsp_sw0 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-1-0-0-thermal { @@ -7903,7 +8398,7 @@ thermal-sensors = <&tsens2 8>; trips { - trip-point0 { + nsp_1_0_0_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7915,6 +8410,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_1_0_0_alert0>; + cooling-device = <&cdsp_sw1 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-1-1-0-thermal { @@ -7923,7 +8426,7 @@ thermal-sensors = <&tsens2 9>; trips { - trip-point0 { + nsp_1_1_0_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7935,6 +8438,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_1_1_0_alert0>; + cooling-device = <&cdsp_sw1 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-1-2-0-thermal { @@ -7943,7 +8454,7 @@ thermal-sensors = <&tsens2 10>; trips { - trip-point0 { + nsp_1_2_0_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -7955,6 +8466,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_1_2_0_alert0>; + cooling-device = <&cdsp_sw1 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; ddrss-0-thermal { @@ -8097,7 +8616,7 @@ thermal-sensors = <&tsens3 5>; trips { - trip-point0 { + nsp_0_0_1_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -8109,6 +8628,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_0_0_1_alert0>; + cooling-device = <&cdsp_sw0 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-0-1-1-thermal { @@ -8117,7 +8644,7 @@ thermal-sensors = <&tsens3 6>; trips { - trip-point0 { + nsp_0_1_1_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -8129,6 +8656,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_0_1_1_alert0>; + cooling-device = <&cdsp_sw0 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-0-2-1-thermal { @@ -8137,7 +8672,7 @@ thermal-sensors = <&tsens3 7>; trips { - trip-point0 { + nsp_0_2_1_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -8149,6 +8684,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_0_2_1_alert0>; + cooling-device = <&cdsp_sw0 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-1-0-1-thermal { @@ -8157,7 +8700,7 @@ thermal-sensors = <&tsens3 8>; trips { - trip-point0 { + nsp_1_0_1_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -8169,6 +8712,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_1_0_1_alert0>; + cooling-device = <&cdsp_sw1 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-1-1-1-thermal { @@ -8177,7 +8728,7 @@ thermal-sensors = <&tsens3 9>; trips { - trip-point0 { + nsp_1_1_1_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -8189,6 +8740,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_1_1_1_alert0>; + cooling-device = <&cdsp_sw1 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nsp-1-2-1-thermal { @@ -8197,7 +8756,7 @@ thermal-sensors = <&tsens3 10>; trips { - trip-point0 { + nsp_1_2_1_alert0: trip-point0 { temperature = <105000>; hysteresis = <5000>; type = "passive"; @@ -8209,6 +8768,14 @@ type = "passive"; }; }; + + cooling-maps { + map0 { + trip = <&nsp_1_2_1_alert0>; + cooling-device = <&cdsp_sw1 + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; ddrss-1-thermal { diff --git a/arch/arm64/boot/dts/qcom/monaco-ac-evk.dts b/arch/arm64/boot/dts/qcom/monaco-ac-evk.dts new file mode 100644 index 0000000000000..6405d1e1939b1 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-ac-evk.dts @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; + +#include "monaco-evk-common.dtsi" + +/ { + model = "Qualcomm Technologies, Inc. Monaco-ac EVK"; + compatible = "qcom,monaco-ac-evk", "qcom,qcs8300"; +}; + +&apps_rsc { + regulators-0 { + vreg_s4a: smps4 { + regulator-name = "vreg_s4a"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_s9a: smps9 { + regulator-name = "vreg_s9a"; + regulator-min-microvolt = <1352000>; + regulator-max-microvolt = <1352000>; + regulator-initial-mode = ; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-camera-sensor.dtsi b/arch/arm64/boot/dts/qcom/monaco-camera-sensor.dtsi new file mode 100644 index 0000000000000..a2a2cd0a27a3a --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-camera-sensor.dtsi @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&cam_cci0 { + /* GMSL deserializer 0 */ + qcom,cam-gmsl-deserializer0 { + cell-index = <0>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active>; + pinctrl-1 = <&cam_sensor_mclk0_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 73 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET0"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser0_port0: endpoint { + remote-endpoint = <&gmsl_sensor0_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser0_port1: endpoint { + remote-endpoint = <&gmsl_sensor1_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser0_port2: endpoint { + remote-endpoint = <&gmsl_sensor2_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser0_port3: endpoint { + remote-endpoint = <&gmsl_sensor3_ep>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 0 */ + qcom,cam-gmsl-sensor0 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <0>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor0_ep: endpoint { + remote-endpoint = <&deser0_port0>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 1 */ + qcom,cam-gmsl-sensor1 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <1>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor1_ep: endpoint { + remote-endpoint = <&deser0_port1>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 2 */ + qcom,cam-gmsl-sensor2 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <2>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor2_ep: endpoint { + remote-endpoint = <&deser0_port2>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 3 */ + qcom,cam-gmsl-sensor3 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <3>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor3_ep: endpoint { + remote-endpoint = <&deser0_port3>; + }; + }; + }; +}; + +&cam_cci1 { + /* GMSL deserializer 1 */ + qcom,cam-gmsl-deserializer1 { + cell-index = <1>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active>; + pinctrl-1 = <&cam_sensor_mclk1_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 74 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser1_port0: endpoint { + remote-endpoint = <&gmsl_sensor4_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser1_port1: endpoint { + remote-endpoint = <&gmsl_sensor5_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser1_port2: endpoint { + remote-endpoint = <&gmsl_sensor6_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser1_port3: endpoint { + remote-endpoint = <&gmsl_sensor7_ep>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 0 */ + qcom,cam-gmsl-sensor4 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <4>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor4_ep: endpoint { + remote-endpoint = <&deser1_port0>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 1 */ + qcom,cam-gmsl-sensor5 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <5>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor5_ep: endpoint { + remote-endpoint = <&deser1_port1>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 2 */ + qcom,cam-gmsl-sensor6 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <6>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor6_ep: endpoint { + remote-endpoint = <&deser1_port2>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 3 */ + qcom,cam-gmsl-sensor7 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <7>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor7_ep: endpoint { + remote-endpoint = <&deser1_port3>; + }; + }; + }; +}; + +&cam_cci2 { + /* GMSL deserializer 2 */ + qcom,cam-gmsl-deserializer2 { + cell-index = <2>; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active>; + pinctrl-1 = <&cam_sensor_mclk2_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 75 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET2"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser2_port0: endpoint { + remote-endpoint = <&gmsl_sensor8_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser2_port1: endpoint { + remote-endpoint = <&gmsl_sensor9_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser2_port2: endpoint { + remote-endpoint = <&gmsl_sensor10_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser2_port3: endpoint { + remote-endpoint = <&gmsl_sensor11_ep>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 0 */ + qcom,cam-gmsl-sensor8 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <8>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor8_ep: endpoint { + remote-endpoint = <&deser2_port0>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 1 */ + qcom,cam-gmsl-sensor9 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <9>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor9_ep: endpoint { + remote-endpoint = <&deser2_port1>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 2 */ + qcom,cam-gmsl-sensor10 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <10>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor10_ep: endpoint { + remote-endpoint = <&deser2_port2>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 3 */ + qcom,cam-gmsl-sensor11 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <11>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor11_ep: endpoint { + remote-endpoint = <&deser2_port3>; + }; + }; + }; +}; + +&soc { + qcom,cam-res-mgr { + compatible = "qcom,cam-res-mgr"; + gpios-shared = <609 610 611>; + status = "ok"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-camera.dtsi b/arch/arm64/boot/dts/qcom/monaco-camera.dtsi new file mode 100644 index 0000000000000..8250f66c34526 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-camera.dtsi @@ -0,0 +1,2499 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&soc { + cam_cci0: qcom,cci0@ac13000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac13000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x13000>; + interrupts = ; + interrupt-names = "CCI0"; + operating-points-v2 = <&cci0_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_0_CLK_SRC>, + <&camcc CAM_CC_CCI_0_CLK>; + clock-names = "cci_0_clk_src", + "cci_0_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_0_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci_i2c_scl0_active &cci_i2c_sda0_active>; + pinctrl-1 = <&cci_i2c_scl0_suspend &cci_i2c_sda0_suspend>; + pinctrl-2 = <&cci_i2c_scl1_active &cci_i2c_sda1_active>; + pinctrl-3 = <&cci_i2c_scl1_suspend &cci_i2c_sda1_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <0>; + status = "ok"; + + cci0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci0: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci0: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci0: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci0: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_cci1: qcom,cci1@ac14000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac14000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x14000>; + interrupts = ; + interrupt-names = "CCI1"; + operating-points-v2 = <&cci1_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_1_CLK_SRC>, + <&camcc CAM_CC_CCI_1_CLK>; + clock-names = "cci_1_clk_src", + "cci_1_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_1_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci_i2c_scl2_active &cci_i2c_sda2_active>; + pinctrl-1 = <&cci_i2c_scl2_suspend &cci_i2c_sda2_suspend>; + pinctrl-2 = <&cci_i2c_scl3_active &cci_i2c_sda3_active>; + pinctrl-3 = <&cci_i2c_scl3_suspend &cci_i2c_sda3_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <1>; + status = "ok"; + + cci1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci1: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci1: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci1: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci1: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_cci2: qcom,cci2@ac15000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac15000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x15000>; + interrupts = ; + interrupt-names = "CCI2"; + operating-points-v2 = <&cci2_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_2_CLK_SRC>, + <&camcc CAM_CC_CCI_2_CLK>; + clock-names = "cci_2_clk_src", + "cci_2_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_2_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci_i2c_scl4_active &cci_i2c_sda4_active>; + pinctrl-1 = <&cci_i2c_scl4_suspend &cci_i2c_sda4_suspend>; + pinctrl-2 = <&cci_i2c_scl5_active &cci_i2c_sda5_active>; + pinctrl-3 = <&cci_i2c_scl5_suspend &cci_i2c_sda5_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <2>; + status = "ok"; + + cci2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci2: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci2: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci2: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci2: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_csiphy0: qcom,csiphy0@ac9c000 { + compatible = "qcom,csiphy-v1.3.0", "qcom,csiphy"; + reg = <0x0 0x0ac9c000 0x0 0x2000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy0_opp_table>; + reg-cam-base = <0x9c000>; + interrupts = ; + interrupt-names = "CSIPHY0"; + csi-vdd-1p2-supply = <&vreg_l5a>; + csi-vdd-0p9-supply = <&vreg_l4a>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 912000>; + rgltr-load-current = <8900 15900>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy0_clk", + "csi0phytimer_clk_src", + "csi0phytimer_clk"; + src-clock-name = "cphy_rx_clk_src"; + clock-cntl-level = "lowsvs"; + clock-rates = <400000000 0 400000000 0>; + cell-index = <0>; + aggregator-rx; + status = "ok"; + + csiphy0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy1: qcom,csiphy1@ac9e000 { + compatible = "qcom,csiphy-v1.3.0", "qcom,csiphy"; + reg = <0x0 0xac9e000 0x0 0x2000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy1_opp_table>; + reg-cam-base = <0x9e000>; + interrupts = ; + interrupt-names = "CSIPHY1"; + csi-vdd-1p2-supply = <&vreg_l5a>; + csi-vdd-0p9-supply = <&vreg_l4a>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 912000>; + rgltr-load-current = <8900 15900>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy1_clk", + "csi1phytimer_clk_src", + "csi1phytimer_clk"; + src-clock-name = "cphy_rx_clk_src"; + clock-cntl-level = "lowsvs"; + clock-rates = <400000000 0 400000000 0>; + cell-index = <1>; + aggregator-rx; + status = "ok"; + + csiphy1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy2: qcom,csiphy2@aca0000 { + compatible = "qcom,csiphy-v1.3.0", "qcom,csiphy"; + reg = <0x0 0xaca0000 0x0 0x2000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy2_opp_table>; + reg-cam-base = <0xa0000>; + interrupts = ; + interrupt-names = "CSIPHY2"; + csi-vdd-1p2-supply = <&vreg_l5a>; + csi-vdd-0p9-supply = <&vreg_l4a>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 912000>; + rgltr-load-current = <8900 15900>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy2_clk", + "csi2phytimer_clk_src", + "csi2phytimer_clk"; + src-clock-name = "cphy_rx_clk_src"; + clock-cntl-level = "lowsvs"; + clock-rates = <400000000 0 400000000 0>; + cell-index = <2>; + aggregator-rx; + status = "ok"; + + csiphy2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy_tpg13: qcom,tpg13@acac000 { + compatible = "qcom,cam-tpg1031"; + reg = <0x0 0xacac000 0x0 0x400>, + <0x0 0xac11000 0x0 0x1000>; + reg-names = "tpg0", "cam_cpas_top"; + reg-cam-base = <0xac000 0x11000>; + operating-points-v2 = <&csiphy_tpg0_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg0"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cphy_rx_clk_src"; + cell-index = <13>; + phy-id = <0>; + status = "ok"; + + csiphy_tpg0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy_tpg14: qcom,tpg14@acad000 { + compatible = "qcom,cam-tpg1031"; + reg = <0x0 0xacad000 0x0 0x400>, + <0x0 0xac11000 0x0 0x1000>; + reg-names = "tpg1", "cam_cpas_top"; + reg-cam-base = <0xad000 0x11000>; + operating-points-v2 = <&csiphy_tpg1_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg1"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cphy_rx_clk_src"; + cell-index = <14>; + phy-id = <1>; + status = "ok"; + + csiphy_tpg1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csiphy_tpg15: qcom,tpg15@acae000 { + compatible = "qcom,cam-tpg1031"; + reg = <0x0 0xacae000 0x0 0x400>, + <0x0 0xac11000 0x0 0x1000>; + reg-names = "tpg2", "cam_cpas_top"; + reg-cam-base = <0xae000 0x11000>; + operating-points-v2 = <&csiphy_tpg2_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg2"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cphy_rx_clk_src"; + cell-index = <15>; + phy-id = <2>; + status = "ok"; + + csiphy_tpg2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,cam-cdm-intf { + compatible = "qcom,cam-cdm-intf"; + cell-index = <0>; + label = "cam-cdm-intf"; + num-hw-cdm = <1>; + cdm-client-names = "vfe"; + status = "ok"; + }; + + cam_cpas: qcom,cam-cpas { + compatible = "qcom,cam-cpas"; + label = "cpas"; + arch-compat = "cpas_top"; + reg = <0x0 0xac11000 0x0 0x1000>, + <0x0 0xac1A000 0x0 0x9400>, + <0x0 0xbbf0000 0x0 0x1f00>; + reg-names = "cam_cpas_top", "cam_camnoc", "cam_rpmh"; + reg-cam-base = <0x11000 0x1A000 0x0bbf0000>; + interrupts = ; + interrupt-names = "cpas_camnoc"; + camnoc-axi-min-ib-bw = <3000000000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_QDSS_DEBUG_XO_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "gcc_camera_sf_axi_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk", + "cam_cc_core_ahb_clk", + "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_fast_ahb_clk", + "cam_cc_camnoc_axi_clk_src", + "cam_cc_camnoc_axi_clk", + "cam_cc_qdss_debug_xo_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <0 0 0 0 0 0 0 0 0 0 0>, + <0 0 0 80000000 0 0 300000000 0 400000000 0 0>, + <0 0 0 80000000 0 0 400000000 0 400000000 0 0>; + clock-cntl-level = "suspend", "svs_l1", "nominal"; + clock-names-option = "cam_icp_clk"; + clocks-option = <&camcc CAM_CC_ICP_CLK>; + clock-rates-option = <480000000>; + operating-points-v2 = <&cpas_opp_table>; + control-camnoc-axi-clk; + camnoc-bus-width = <32>; + camnoc-axi-clk-bw-margin-perc = <20>; + cam-icc-path-names = "cam_ahb"; + interconnects = <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_CAMERA_CFG 0>, + <&mmss_noc MASTER_CAMNOC_HF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_SF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_ICP 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "cam_ahb", "cam_hf_0", "cam_sf_0", "cam_sf_icp"; + cam-ahb-num-cases = <7>; + cam-ahb-bw-KBps = <0 0>, <0 76800>, <0 150000>, <0 150000>, + <0 300000>, <0 300000>, <0 300000>; + vdd-corners = ; + vdd-corner-ahb-mapping = "suspend", "lowsvs", "lowsvs", "svs", "svs_l1", + "nominal", "nominal", "nominal", "turbo", "turbo"; + client-id-based; + client-names = "csiphy0", "csiphy1", "csiphy2", + "cci0", "cci1", "cci2", + "csid0", "csid1", "csid2", "csid3", + "csid4", "csid5", "csid6", "ife0", + "ife1", "ife2", "ife3", "ife4", + "ife5", "ife6", "ipe0", "sfe0", "sfe1", + "cam-cdm-intf0", "rt-cdm0", "rt-cdm1", "rt-cdm2", + "rt-cdm3", "icp0", "tpg13", "tpg14", "tpg15"; + enable-secure-qos-update = <1>; + cell-index = <0>; + status = "ok"; + + cpas_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + + camera-bus-nodes { + level0-nodes { + level-index = <0>; + + icp_all_rd: icp-all-rd { + cell-index = <0>; + node-name = "icp-all-rd"; + client-name = "icp0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_icp_rd>; + }; + + ife_0_wr_0: ife-0-wr-0 { + cell-index = <1>; + node-name = "ife-0-wr-0"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_0_wr_1: ife-0-wr-1 { + cell-index = <2>; + node-name = "ife-0-wr-1"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr1>; + }; + + ife_1_wr_0: ife-1-wr-0 { + cell-index = <3>; + node-name = "ife-1-wr-0"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_1_wr_1: ife-1-wr-1 { + cell-index = <4>; + node-name = "ife-1-wr-1"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr1>; + }; + + ife_lite_0_wr_0: ife-lite-0-wr-0 { + cell-index = <5>; + node-name = "ife-lite-0-wr-0"; + client-name = "ife2"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_1_wr_0: ife-lite-1-wr-0 { + cell-index = <6>; + node-name = "ife-lite-1-wr-0"; + client-name = "ife3"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_2_wr_0: ife-lite-2-wr-0 { + cell-index = <7>; + node-name = "ife-lite-2-wr-0"; + client-name = "ife4"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_3_wr_0: ife-lite-3-wr-0 { + cell-index = <8>; + node-name = "ife-lite-3-wr-0"; + client-name = "ife5"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ife_lite_4_wr_0: ife-lite-4-wr-0 { + cell-index = <9>; + node-name = "ife-lite-4-wr-0"; + client-name = "ife6"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_wr0>; + }; + + ipe_0_rd_all: ipe-0-rd-all { + cell-index = <10>; + node-name = "ipe-0-rd-all"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level2_nrt_rd1>; + }; + + ipe_0_wr_2: ipe-0-wr-2 { + cell-index = <11>; + node-name = "ipe-0-wr-2"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level2_nrt_wr2>; + }; + + ipe_cdm0_all_rd: ipe-cdm0-all-rd { + cell-index = <12>; + node-name = "ipe-cdm0-all-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm0_all_rd_2: rt-cdm0-all-rd-2 { + cell-index = <13>; + node-name = "rt-cdm0-all-rd-2"; + client-name = "rt-cdm0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm1_all_rd_2: rt-cdm1-all-rd-2 { + cell-index = <14>; + node-name = "rt-cdm1-all-rd-2"; + client-name = "rt-cdm1"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm2_all_rd_2: rt-cdm2-all-rd-2 { + cell-index = <15>; + node-name = "rt-cdm2-all-rd-2"; + client-name = "rt-cdm2"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + rt_cdm3_all_rd_2: rt-cdm3-all-rd-2 { + cell-index = <16>; + node-name = "rt-cdm3-all-rd-2"; + client-name = "rt-cdm3"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt_rd2>; + }; + + sfe_0_rd_0: sfe-0-rd-0 { + cell-index = <17>; + node-name = "sfe-0-rd-0"; + client-name = "sfe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_rd0>; + }; + + sfe_1_rd_0: sfe-1-rd-0 { + cell-index = <18>; + node-name = "sfe-1-rd-0"; + client-name = "sfe1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt_rd0>; + }; + }; + + level1-nodes { + level-index = <1>; + camnoc-max-needed; + + level1_nrt_rd2: level1-nrt-rd2 { + cell-index = <19>; + node-name = "level1-nrt-rd2"; + parent-node = <&level2_nrt_rd2>; + traffic-merge-type = ; + }; + + level1_rt_rd0: level1-rt-read0 { + cell-index = <20>; + node-name = "level1-rt-rd0"; + parent-node = <&level2_rt_rd0>; + traffic-merge-type = ; + }; + + level1_rt_wr0: level1-rt-wr0 { + cell-index = <21>; + node-name = "level1-rt-wr0"; + parent-node = <&level2_rt_wr0>; + traffic-merge-type = ; + }; + + level1_rt_wr1: level1-rt-wr1 { + cell-index = <22>; + node-name = "level1-rt-wr1"; + parent-node = <&level2_rt_wr1>; + traffic-merge-type = ; + }; + }; + + level2-nodes { + level-index = <2>; + camnoc-max-needed; + + level2_icp_rd: level2-icp-rd { + cell-index = <23>; + node-name = "level2-icp-rd"; + parent-node = <&level3_nrt1_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt_rd1: level2-nrt-rd1 { + cell-index = <24>; + node-name = "level2-nrt-rd1"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt_rd2: level2-nrt-rd2 { + cell-index = <25>; + node-name = "level2-nrt-rd2"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt_wr2: level2-nrt-wr2 { + cell-index = <26>; + node-name = "level2-nrt-wr2"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_rt_rd0: level2-rt-read0 { + cell-index = <27>; + node-name = "level2-rt-rd0"; + parent-node = <&level3_rt_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_rt_wr0: level2-rt-wr0 { + cell-index = <28>; + node-name = "level2-rt-wr0"; + parent-node = <&level3_rt_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_rt_wr1: level2-rt-wr1 { + cell-index = <29>; + node-name = "level2-rt-wr1"; + parent-node = <&level3_rt_rd_wr_sum>; + traffic-merge-type = ; + }; + }; + + level3-nodes { + level-index = <3>; + + level3_nrt0_rd_wr_sum: level3-nrt0-rd-wr-sum { + cell-index = <30>; + node-name = "level3-nrt0-rd-wr-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_0"; + }; + }; + + level3_nrt1_rd_wr_sum: level3-nrt1-rd-wr-sum { + cell-index = <31>; + node-name = "level3-nrt1-rd-wr-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_icp"; + }; + }; + + level3_rt_rd_wr_sum: level3-rt-rd-wr-sum { + cell-index = <32>; + node-name = "level3-rt-rd-wr-sum"; + traffic-merge-type = ; + ib-bw-voting-needed; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_hf_0"; + }; + }; + }; + }; + }; + + cam_icp_firmware: qcom,cam-icp { + compatible = "qcom,cam-icp"; + compat-hw-name = "qcom,icp", "qcom,ipe0"; + num-icp = <1>; + num-ipe = <1>; + icp_use_pil; + status = "ok"; + }; + + qcom,cam-i3c-id-table { + compatible = "qcom,cam-i3c-id-table"; + i3c-sensor-id-table = <0x1B0 0x0766>; + i3c-eeprom-id-table = <>; + i3c-actuator-id-table = <>; + i3c-ois-id-table = <>; + status = "disabled"; + }; + + qcom,cam-isp { + compatible = "qcom,cam-isp"; + arch-compat = "ife"; + status = "ok"; + }; + + qcom,cam-req-mgr { + compatible = "qcom,cam-req-mgr"; + status = "ok"; + }; + + qcom,camera-main { + compatible = "qcom,camera"; + status = "ok"; + }; + + qcom,cam-smmu { + compatible = "qcom,msm-cam-smmu", "simple-bus"; + force_cache_allocs; + status = "ok"; + + msm-cam-smmu-cdm { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0860 0x400>, + <&apps_smmu 0x0C60 0x400>; + cam-smmu-label = "rt-cdm"; + dma-coherent; + multiple-client-devices; + + rt_cdm_iova_mem_map: iova-mem-map { + iova-mem-region-io { + /* IO region is approximately 4.0 GB */ + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-icp { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0800 0x400>, + <&apps_smmu 0x0840 0x480>, + <&apps_smmu 0x08C0 0x480>, + <&apps_smmu 0x0C00 0x400>, + <&apps_smmu 0x0C40 0x480>, + <&apps_smmu 0x0CC0 0x480>; + cam-smmu-label = "icp"; + qcom,iommu-faults = "stall-disable", "non-fatal"; + iova-region-discard = <0xe0000000 0x800000>; + dma-coherent; + + icp_iova_mem_map: iova-mem-map { + iova-mem-region-fwuncached-region { + /* FW uncached region is 7MB long */ + iova-region-name = "fw_uncached"; + iova-region-start = <0x10400000>; + iova-region-len = <0x700000>; + iova-region-id = <0x6>; + subregion_support; + status = "ok"; + + /* Used for HFI queues/sec heap */ + iova-mem-region-generic-region { + iova-region-name = "icp_hfi"; + iova-region-start = <0x10400000>; + iova-region-len = <0x200000>; + iova-region-id = <0x0>; + }; + }; + + iova-mem-region-io { + /* IO region is approximately 3.8 GB */ + iova-region-name = "io"; + iova-region-start = <0x10c00000>; + iova-region-len = <0xee300000>; + iova-region-id = <0x3>; + iova-region-discard = <0xe0000000 0x800000>; + status = "ok"; + }; + + iova-mem-region-qdss { + /* QDSS region is appropriate 1MB */ + iova-region-name = "qdss"; + iova-region-start = <0x10b00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x5>; + qdss-phy-addr = <0x16790000>; + status = "ok"; + }; + + iova-mem-region-shared { + /* Shared region is ~250MB long */ + iova-region-name = "shared"; + iova-region-start = <0x800000>; + iova-region-len = <0xfc00000>; + iova-region-id = <0x1>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-ife { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x2400 0x0020>, + <&apps_smmu 0x2420 0x0020>; + cam-smmu-label = "ife", "sfe"; + multiple-client-devices; + dma-coherent; + + ife_iova_mem_map: iova-mem-map { + /* IO region is approximately 64 GB */ + iova-mem-region-io { + iova-region-name = "io"; + /* start address: 0x100000 */ + /* leaving 1 MB pad at start */ + iova-region-start = <0x100000>; + /* Length: 0xfffe00000 */ + /* leaving 1 MB pad at end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-secure { + compatible = "qcom,msm-cam-smmu-cb"; + cam-smmu-label = "cam-secure"; + qcom,secure-cb; + }; + }; + + qcom,cam-sync { + compatible = "qcom,cam-sync"; + status = "ok"; + }; + + cam_csid0: qcom,csid0 { + compatible = "qcom,csid692"; + reg = <0x0 0xac7a000 0x0 0xf01>, + <0x0 0xac78000 0x0 0x1000>; + reg-names = "csid0", "csid_top"; + reg-cam-base = <0x7a000 0x78000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "csid0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CSID_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cam_cc_csid_clk_src", + "cam_cc_csid_clk", + "cam_cc_csid_csiphy_rx_clk"; + clock-rates = <400000000 0 0>; + clock-cntl-level = "svs_l1"; + src-clock-name = "cam_cc_csid_clk_src"; + operating-points-v2 = <&csid0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "ok"; + + csid0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + + cam_csid1: qcom,csid1 { + compatible = "qcom,csid692"; + reg = <0x0 0xac7c000 0x0 0xf01>, + <0x0 0xac78000 0x0 0x1000>; + reg-names = "csid1", "csid_top"; + reg-cam-base = <0x7c000 0x78000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "csid1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CSID_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cam_cc_csid_clk_src", + "cam_cc_csid_clk", + "cam_cc_csid_csiphy_rx_clk"; + clock-rates = <400000000 0 0>; + clock-cntl-level = "svs_l1"; + src-clock-name = "cam_cc_csid_clk_src"; + operating-points-v2 = <&csid1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "ok"; + + csid1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + + cam_csid_lite0: qcom,csid-lite0 { + compatible = "qcom,csid-lite692"; + reg = <0x0 0xac84000 0x0 0xf01>; + reg-names = "csid-lite0"; + reg-cam-base = <0x84000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <2>; + status = "ok"; + + csid_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite1: qcom,csid-lite1 { + compatible = "qcom,csid-lite692"; + reg = <0x0 0xac88000 0x0 0xf01>; + reg-names = "csid-lite1"; + reg-cam-base = <0x88000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <3>; + status = "ok"; + + csid_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite2: qcom,csid-lite2 { + compatible = "qcom,csid-lite692"; + reg = <0x0 0xac8c000 0x0 0xf01>; + reg-names = "csid-lite2"; + reg-cam-base = <0x8c000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite2_opp_table>; + clock-control-debugfs = "true"; + cell-index = <4>; + status = "ok"; + + csid_lite2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite3: qcom,csid-lite3 { + compatible = "qcom,csid-lite692"; + reg = <0x0 0xac90000 0x0 0xf01>; + reg-names = "csid-lite3"; + reg-cam-base = <0x90000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite3_opp_table>; + clock-control-debugfs = "true"; + cell-index = <5>; + status = "ok"; + + csid_lite3_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite4: qcom,csid-lite4 { + compatible = "qcom,csid-lite692"; + reg = <0x0 0xac94000 0x0 0xf01>; + reg-names = "csid-lite4"; + reg-cam-base = <0x94000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "csid-lite4"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite4_opp_table>; + clock-control-debugfs = "true"; + cell-index = <6>; + status = "ok"; + + csid_lite4_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_icp: qcom,icp { + compatible = "qcom,cam-icp_v3_0"; + reg = <0x0 0xac01000 0x0 0x400>, + <0x0 0xac01800 0x0 0x400>, + <0x0 0xac04000 0x0 0x1000>; + reg-names ="icp_csr", "icp_cirq", "icp_wd0"; + reg-cam-base = <0x1000 0x1800 0x4000>; + interrupts = ; + interrupt-names = "icp"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + memory-region = <&camera_mem>; + src-clock-name = "cam_cc_icp_clk_src"; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_ICP_AHB_CLK>, + <&camcc CAM_CC_ICP_CLK_SRC>, + <&camcc CAM_CC_ICP_CLK>; + clock-names = "cam_cc_soc_fast_ahb_clk_src", + "cam_cc_icp_ahb_clk", + "cam_cc_icp_clk_src", + "cam_cc_icp_clk"; + clock-rates = <300000000 0 480000000 0>, + <400000000 0 600000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + nrt-device; + fw_name = "qcom/qcs8300/CAMERA_ICP"; + ubwc-ipe-fetch-cfg = <0x707b 0x7083>; + ubwc-ipe-write-cfg = <0x161ef 0x1620f>; + qos-val = <0xa0a>; + operating-points-v2 = <&icp_opp_table>; + icp-version = <0x0300>; + cam_hw_pid = <12>; + cell-index = <0>; + status = "ok"; + + icp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe0: qcom,ife0 { + compatible = "qcom,vfe692"; + reg = <0x0 0xac4d000 0x0 0xf000>, + <0x0 0xac1a000 0x0 0x9400>; + reg-names = "ife0", "cam_camnoc"; + reg-cam-base = <0x4d000 0x1a000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "ife0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>; + clock-names = "cam_cc_cpas_ife_0_clk", + "cam_cc_ife_0_clk", + "cam_cc_ife_0_clk_src", + "cam_cc_ife_0_fast_ahb_clk"; + clock-rates = <0 0 480000000 0>, + <0 0 600000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_0_clk_src"; + operating-points-v2 = <&vfe0_opp_table>; + clock-control-debugfs = "true"; + ubwc-static-cfg = <0x1026 0x1036>; + cam_hw_pid = <8 16>; + cell-index = <0>; + status = "ok"; + + vfe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe1: qcom,ife1 { + compatible = "qcom,vfe692"; + reg = <0x0 0xac60000 0x0 0xf000>, + <0x0 0xac1a000 0x0 0x9400>; + reg-names = "ife1", "cam_camnoc"; + reg-cam-base = <0x60000 0x1a000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "ife1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_CLK_SRC>, + <&camcc CAM_CC_IFE_1_FAST_AHB_CLK>; + clock-names = "cam_cc_cpas_ife_1_clk", + "cam_cc_ife_1_clk", + "cam_cc_ife_1_clk_src", + "cam_cc_ife_1_fast_ahb_clk"; + clock-rates = <0 0 480000000 0>, + <0 0 600000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_1_clk_src"; + operating-points-v2 = <&vfe1_opp_table>; + clock-control-debugfs = "true"; + ubwc-static-cfg = <0x1026 0x1036>; + cam_hw_pid = <9 17>; + cell-index = <1>; + status = "ok"; + + vfe1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite0: qcom,ife-lite0 { + compatible = "qcom,vfe-lite692"; + reg = <0x0 0xac84000 0x0 0x1d00>; + reg-names = "ife-lite0"; + reg-cam-base = <0x84000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite0_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <2>; + status = "ok"; + + vfe_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite1: qcom,ife-lite1 { + compatible = "qcom,vfe-lite692"; + reg = <0x0 0xac88000 0x0 0x1d00>; + reg-names = "ife-lite1"; + reg-cam-base = <0x88000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite1_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <3>; + status = "ok"; + + vfe_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite2: qcom,ife-lite2 { + compatible = "qcom,vfe-lite692"; + reg = <0x0 0xac8c000 0x0 0x1d00>; + reg-names = "ife-lite2"; + reg-cam-base = <0x8c000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite2_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <4>; + status = "ok"; + + vfe_lite2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite3: qcom,ife-lite3 { + compatible = "qcom,vfe-lite692"; + reg = <0x0 0xac90000 0x0 0x1d00>; + reg-names = "ife-lite3"; + reg-cam-base = <0x90000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite3_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <5>; + status = "ok"; + + vfe_lite3_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite4: qcom,ife-lite4 { + compatible = "qcom,vfe-lite692"; + reg = <0x0 0xac94000 0x0 0x1d00>; + reg-names = "ife-lite4"; + reg-cam-base = <0x94000>; + rt-wrapper-base = <0x83000>; + interrupts = ; + interrupt-names = "ife-lite4"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>; + clock-names = "cam_cc_cpas_ife_lite_clk", + "cam_cc_ife_lite_ahb_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_ife_lite_clk"; + clock-rates = <0 0 400000000 0 0 400000000 0>, + <0 0 400000000 0 0 480000000 0>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite4_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <27>; + cell-index = <6>; + status = "ok"; + + vfe_lite4_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_ipe0: qcom,ipe0 { + compatible = "qcom,cam-ipe"; + reg = <0x0 0xac2d000 0x0 0x18000>; + reg-names = "ipe0_top"; + reg-cam-base = <0x2d000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + src-clock-name = "cam_cc_ipe_clk_src"; + clocks = <&camcc CAM_CC_CPAS_IPE_CLK>, + <&camcc CAM_CC_IPE_AHB_CLK>, + <&camcc CAM_CC_IPE_CLK>, + <&camcc CAM_CC_IPE_CLK_SRC>, + <&camcc CAM_CC_IPE_FAST_AHB_CLK>, + <&camcc CAM_CC_FAST_AHB_CLK_SRC>; + clock-names = "cam_cc_cpas_ipe_clk", + "cam_cc_ipe_ahb_clk", + "cam_cc_ipe_clk", + "cam_cc_ipe_clk_src", + "cam_cc_ipe_fast_ahb_clk", + "cam_cc_fast_ahb_clk_src"; + clock-rates = <0 0 0 480000000 0 300000000>, + <0 0 0 600000000 0 400000000>; + clock-cntl-level = "svs_l1", "nominal"; + operating-points-v2 = <&ipe0_opp_table>; + nrt-device; + cam_hw_pid = <14 15>; + cell-index = <0>; + status = "ok"; + + ipe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,rt-cdm0 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac26000 0x0 0x1000>; + reg-names = "rt-cdm0"; + reg-cam-base = <0x26000>; + interrupts = ; + interrupt-names = "rt-cdm0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <300000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table0>; + clock-cntl-level = "svs"; + cdm-client-names = "ife0", "dualife0"; + single-context-cdm; + cell-index = <0>; + status = "ok"; + + cdm_cpas_opp_table0: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,rt-cdm1 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac27000 0x0 0x1000>; + reg-names = "rt-cdm1"; + reg-cam-base = <0x27000>; + interrupts = ; + interrupt-names = "rt-cdm1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <300000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table1>; + clock-cntl-level = "svs_l1"; + cdm-client-names = "ife1", "dualife1"; + single-context-cdm; + cell-index = <1>; + status = "ok"; + + cdm_cpas_opp_table1: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,rt-cdm2 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac28000 0x0 0x1000>; + reg-names = "rt-cdm2"; + reg-cam-base = <0x28000>; + interrupts = ; + interrupt-names = "rt-cdm2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-rates = <80000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table2>; + clock-cntl-level = "svs_l1"; + cdm-client-names = "ife2", "ife3", "ife4", "ife5", "ife6"; + single-context-cdm; + cell-index = <2>; + status = "ok"; + + cdm_cpas_opp_table2: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + qcom,rt-cdm3 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac29000 0x0 0x1000>; + reg-names = "rt-cdm3"; + reg-cam-base = <0x29000>; + interrupts = ; + interrupt-names = "rt-cdm3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-rates = <80000000 0>; + operating-points-v2 = <&cdm_cpas_opp_table3>; + clock-cntl-level = "svs"; + cdm-client-names = "ife2", "ife3", "ife4", "ife5", "ife6"; + single-context-cdm; + cell-index = <3>; + status = "ok"; + + cdm_cpas_opp_table3: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_sfe_lite0: qcom,sfe-lite0 { + compatible = "qcom,sfe-lite692"; + reg = <0x0 0xac74000 0x0 0x1000>; + reg-names = "sfe-lite0"; + reg-cam-base = <0x74000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "sfe-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SFE_LITE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_SFE_LITE_0_CLK>, + <&camcc CAM_CC_CPAS_SFE_LITE_0_CLK>; + clock-names = "cam_cc_sfe_lite_0_fast_ahb_clk", + "cam_cc_sfe_lite_0_clk", + "cam_cc_cpas_sfe_lite_0_clk"; + clock-rates = <0 480000000 300000000>, + <0 600000000 400000000>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_sfe_lite_0_clk"; + operating-points-v2 = <&cam_sfe_lite_opp_table0>; + cam_hw_pid = <4>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "ok"; + + cam_sfe_lite_opp_table0: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_sfe_lite1: qcom,sfe-lite1 { + compatible = "qcom,sfe-lite692"; + reg = <0x0 0xac75000 0x0 0x1000>; + reg-names = "sfe-lite1"; + reg-cam-base = <0x75000>; + rt-wrapper-base = <0x4d000>; + interrupts = ; + interrupt-names = "sfe-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SFE_LITE_1_FAST_AHB_CLK>, + <&camcc CAM_CC_SFE_LITE_1_CLK>, + <&camcc CAM_CC_CPAS_SFE_LITE_1_CLK>; + clock-names = "cam_cc_sfe_lite_1_fast_ahb_clk", + "cam_cc_sfe_lite_1_clk", + "cam_cc_cpas_sfe_1_clk"; + clock-rates = <0 480000000 300000000>, + <0 600000000 400000000>; + clock-cntl-level = "svs_l1", "nominal"; + src-clock-name = "cam_cc_sfe_lite_1_clk"; + operating-points-v2 = <&cam_sfe_lite_opp_table1>; + cam_hw_pid = <5>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "ok"; + + cam_sfe_lite_opp_table1: opp-table { + compatible = "operating-points-v2"; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; +}; + +&tlmm { + cam_sensor_mclk0_active: cam-sensor-mclk0-active { + /* MCLK0 */ + mux { + pins = "gpio67"; + function = "cam_mclk"; + }; + + config { + pins = "gpio67"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk0_suspend: cam-sensor-mclk0-suspend { + /* MCLK0 */ + mux { + pins = "gpio67"; + function = "cam_mclk"; + }; + + config { + pins = "gpio67"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_active: cam-sensor-mclk1-active { + /* MCLK1 */ + mux { + pins = "gpio68"; + function = "cam_mclk"; + }; + + config { + pins = "gpio68"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_suspend: cam-sensor-mclk1-suspend { + /* MCLK1 */ + mux { + pins = "gpio68"; + function = "cam_mclk"; + }; + + config { + pins = "gpio68"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_active: cam-sensor-mclk2-active { + /* MCLK2 */ + mux { + pins = "gpio69"; + function = "cam_mclk"; + }; + + config { + pins = "gpio69"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_suspend: cam-sensor-mclk2-suspend { + /* MCLK2 */ + mux { + pins = "gpio69"; + function = "cam_mclk"; + }; + + config { + pins = "gpio69"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl0_active: cci-i2c-scl0-active { + mux { + /* CLK, DATA */ + pins = "gpio58"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio58"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_scl0_suspend: cci-i2c-scl0-suspend { + mux { + /* CLK, DATA */ + pins = "gpio58"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio58"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl1_active: cci-i2c-scl1-active { + mux { + /* CLK, DATA */ + pins = "gpio30"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio30"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_scl1_suspend: cci-i2c-scl1-suspend { + mux { + /* CLK, DATA */ + pins = "gpio30"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio30"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl2_active: cci-i2c-scl2-active { + mux { + /* CLK, DATA */ + pins = "gpio60"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio60"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_scl2_suspend: cci-i2c-scl2-suspend { + mux { + /* CLK, DATA */ + pins = "gpio60"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio60"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl3_active: cci-i2c-scl3-active { + mux { + pins = "gpio32"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio32"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_scl3_suspend: cci-i2c-scl3-suspend { + mux { + pins = "gpio32"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio32"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl4_active: cci-i2c-scl4-active { + mux { + /* CLK, DATA */ + pins = "gpio62"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio62"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_scl4_suspend: cci-i2c-scl4-suspend { + mux { + /* CLK, DATA */ + pins = "gpio62"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio62"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl5_active: cci-i2c-scl5-active { + mux { + /* CLK, DATA */ + pins = "gpio55"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio55"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_scl5_suspend: cci-i2c-scl5-suspend { + mux { + /* CLK, DATA */ + pins = "gpio55"; + function = "cci_i2c_scl"; + }; + + config { + pins = "gpio55"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda0_active: cci-i2c-sda0-active { + mux { + /* CLK, DATA */ + pins = "gpio57"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio57"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_sda0_suspend: cci-i2c-sda0-suspend { + mux { + /* CLK, DATA */ + pins = "gpio57"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio57"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda1_active: cci-i2c-sda1-active { + mux { + /* CLK, DATA */ + pins = "gpio29"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio29"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_sda1_suspend: cci-i2c-sda1-suspend { + mux { + /* CLK, DATA */ + pins = "gpio29"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio29"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda2_active: cci-i2c-sda2-active { + mux { + /* CLK, DATA */ + pins = "gpio59"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio59"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_sda2_suspend: cci-i2c-sda2-suspend { + mux { + /* CLK, DATA */ + pins = "gpio59"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio59"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda3_active: cci-i2c-sda3-active { + mux { + pins = "gpio31"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio31"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_sda3_suspend: cci-i2c-sda3-suspend { + mux { + pins = "gpio31"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio31"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda4_active: cci-i2c-sda4-active { + mux { + /* CLK, DATA */ + pins = "gpio61"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio61"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_sda4_suspend: cci-i2c-sda4-suspend { + mux { + /* CLK, DATA */ + pins = "gpio61"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio61"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda5_active: cci-i2c-sda5-active { + mux { + /* CLK, DATA */ + pins = "gpio54"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio54"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + qcom,i2c_pull; + }; + }; + + cci_i2c_sda5_suspend: cci-i2c-sda5-suspend { + mux { + /* CLK, DATA */ + pins = "gpio54"; + function = "cci_i2c_sda"; + }; + + config { + pins = "gpio54"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-camx-el2.dtso b/arch/arm64/boot/dts/qcom/monaco-camx-el2.dtso new file mode 100644 index 0000000000000..c7ddbc01a0070 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-camx-el2.dtso @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +/ { + fragment@0 { + target-path = "/soc@0/qcom,cam-cpas"; + __overlay__ { + enable-secure-qos-update = <0>; + }; + }; + + fragment@1 { + target-path = "/soc@0/qcom,cam-icp"; + __overlay__ { + camera-firmware { + iommus = <&apps_smmu 0x08c1 0x0400>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-el2.dtso b/arch/arm64/boot/dts/qcom/monaco-el2.dtso new file mode 100644 index 0000000000000..7cae65ae499c3 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-el2.dtso @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Monaco specific modifications required to boot in EL2. + */ + +/dts-v1/; +/plugin/; + +&gpu_zap_shader { + status = "disabled"; +}; + +&iris { + status = "okay"; + video-firmware { + iommus = <&apps_smmu 0x0882 0x0400>; + }; +}; + +&remoteproc_adsp { + iommus = <&apps_smmu 0x2000 0x0>; +}; + +&remoteproc_cdsp { + iommus = <&apps_smmu 0x19c0 0x0400>; +}; + +&remoteproc_gpdsp { + iommus = <&apps_smmu 0x28a0 0x0>; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-camera-imx577.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-camera-imx577.dtso new file mode 100644 index 0000000000000..2237f0fc4a141 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-camera-imx577.dtso @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include + +&{/} { + vreg_cam1_2p8: vreg-cam1-2p8 { + compatible = "regulator-fixed"; + regulator-name = "vreg_cam1_2p8"; + startup-delay-us = <10000>; + enable-active-high; + gpio = <&tlmm 74 GPIO_ACTIVE_HIGH>; + }; +}; + +&camss { + vdda-phy-supply = <&vreg_l4a>; + vdda-pll-supply = <&vreg_l5a>; + + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + csiphy1_ep: endpoint { + clock-lanes = <7>; + data-lanes = <0 1 2 3>; + remote-endpoint = <&imx577_ep1>; + }; + }; + }; +}; + +&cci1 { + pinctrl-0 = <&cci1_i2c0_default>; + pinctrl-1 = <&cci1_i2c0_sleep>; + + status = "okay"; +}; + +&cci1_i2c0 { + #address-cells = <1>; + #size-cells = <0>; + + camera@1a { + compatible = "sony,imx577"; + reg = <0x1a>; + + reset-gpios = <&expander2 1 GPIO_ACTIVE_LOW>; + pinctrl-0 = <&cam1_default>; + pinctrl-names = "default"; + + clocks = <&camcc CAM_CC_MCLK1_CLK>; + assigned-clocks = <&camcc CAM_CC_MCLK1_CLK>; + assigned-clock-rates = <24000000>; + + avdd-supply = <&vreg_cam1_2p8>; + + port { + imx577_ep1: endpoint { + clock-lanes = <7>; + link-frequencies = /bits/ 64 <600000000>; + data-lanes = <0 1 2 3>; + remote-endpoint = <&csiphy1_ep>; + }; + }; + }; +}; + +&tlmm { + cam1_default: cam1-default-state { + mclk-pins { + pins = "gpio68"; + function = "cam_mclk"; + drive-strength = <2>; + bias-disable; + }; + + ldo-avdd-pins { + pins = "gpio74"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-camera-sensor.dtsi b/arch/arm64/boot/dts/qcom/monaco-evk-camera-sensor.dtsi new file mode 100644 index 0000000000000..8bdf5a555cfe5 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-camera-sensor.dtsi @@ -0,0 +1,719 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&cam_cci0 { + /* GMSL deserializer 0 */ + qcom,cam-gmsl-deserializer0 { + cell-index = <0>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active>; + pinctrl-1 = <&cam_sensor_mclk0_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 73 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET0"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser0_port0: endpoint { + remote-endpoint = <&gmsl_sensor0_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser0_port1: endpoint { + remote-endpoint = <&gmsl_sensor1_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser0_port2: endpoint { + remote-endpoint = <&gmsl_sensor2_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser0_port3: endpoint { + remote-endpoint = <&gmsl_sensor3_ep>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 0 */ + qcom,cam-gmsl-sensor0 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <0>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor0_ep: endpoint { + remote-endpoint = <&deser0_port0>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 1 */ + qcom,cam-gmsl-sensor1 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <1>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor1_ep: endpoint { + remote-endpoint = <&deser0_port1>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 2 */ + qcom,cam-gmsl-sensor2 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <2>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor2_ep: endpoint { + remote-endpoint = <&deser0_port2>; + }; + }; + }; + + /* GMSL deserializer 0 sensor 3 */ + qcom,cam-gmsl-sensor3 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <3>; + csiphy-sd-index = <0>; + status = "ok"; + + port { + gmsl_sensor3_ep: endpoint { + remote-endpoint = <&deser0_port3>; + }; + }; + }; + + /*cam0-ov9282*/ + rb4_slot0: qcom,cam-sensor16 { + compatible = "qcom,cam-sensor"; + cell-index = <16>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 67 0>, + <&tlmm 73 0>, + <&expander2 0 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam0-imx577*/ + qcom,cam-sensor20 { + compatible = "qcom,cam-sensor"; + cell-index = <20>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam20>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 67 0>, + <&tlmm 73 0>, + <&expander2 0 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + eeprom_cam20: qcom,eeprom20 { + cell-index = <20>; + compatible = "qcom,eeprom"; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 67 0>, + <&tlmm 73 0>, + <&expander2 0 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK0", + "CAMIF_RESET0", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + status = "ok"; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + }; +}; + +&cam_cci1 { + /* GMSL deserializer 1 */ + qcom,cam-gmsl-deserializer1 { + cell-index = <1>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active>; + pinctrl-1 = <&cam_sensor_mclk1_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 74 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser1_port0: endpoint { + remote-endpoint = <&gmsl_sensor4_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser1_port1: endpoint { + remote-endpoint = <&gmsl_sensor5_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser1_port2: endpoint { + remote-endpoint = <&gmsl_sensor6_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser1_port3: endpoint { + remote-endpoint = <&gmsl_sensor7_ep>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 0 */ + qcom,cam-gmsl-sensor4 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <4>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor4_ep: endpoint { + remote-endpoint = <&deser1_port0>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 1 */ + qcom,cam-gmsl-sensor5 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <5>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor5_ep: endpoint { + remote-endpoint = <&deser1_port1>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 2 */ + qcom,cam-gmsl-sensor6 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <6>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor6_ep: endpoint { + remote-endpoint = <&deser1_port2>; + }; + }; + }; + + /* GMSL deserializer 1 sensor 3 */ + qcom,cam-gmsl-sensor7 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <7>; + csiphy-sd-index = <1>; + status = "ok"; + + port { + gmsl_sensor7_ep: endpoint { + remote-endpoint = <&deser1_port3>; + }; + }; + }; + + /*cam1-ov9282*/ + rb4_slot1: qcom,cam-sensor17 { + compatible = "qcom,cam-sensor"; + cell-index = <17>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 68 0>, + <&tlmm 74 0>, + <&expander2 1 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK1", + "CAMIF_RESET1", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam0-imx577*/ + qcom,cam-sensor21 { + compatible = "qcom,cam-sensor"; + cell-index = <21>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam21>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 68 0>, + <&tlmm 74 0>, + <&expander2 1 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK1", + "CAMIF_RESET1", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + eeprom_cam21: qcom,eeprom21 { + compatible = "qcom,eeprom"; + cell-index = <21>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 68 0>, + <&tlmm 74 0>, + <&expander2 1 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK1", + "CAMIF_RESET1", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; +}; + +&cam_cci2 { + /* GMSL deserializer 2 */ + qcom,cam-gmsl-deserializer2 { + cell-index = <2>; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active>; + pinctrl-1 = <&cam_sensor_mclk2_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 75 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_RESET2"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + + port@0 { + reg = <0>; + + deser2_port0: endpoint { + remote-endpoint = <&gmsl_sensor8_ep>; + }; + }; + + port@1 { + reg = <1>; + + deser2_port1: endpoint { + remote-endpoint = <&gmsl_sensor9_ep>; + }; + }; + + port@2 { + reg = <0>; + + deser2_port2: endpoint { + remote-endpoint = <&gmsl_sensor10_ep>; + }; + }; + + port@3 { + reg = <1>; + + deser2_port3: endpoint { + remote-endpoint = <&gmsl_sensor11_ep>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 0 */ + qcom,cam-gmsl-sensor8 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <8>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor8_ep: endpoint { + remote-endpoint = <&deser2_port0>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 1 */ + qcom,cam-gmsl-sensor9 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <9>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor9_ep: endpoint { + remote-endpoint = <&deser2_port1>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 2 */ + qcom,cam-gmsl-sensor10 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <10>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor10_ep: endpoint { + remote-endpoint = <&deser2_port2>; + }; + }; + }; + + /* GMSL deserializer 2 sensor 3 */ + qcom,cam-gmsl-sensor11 { + compatible = "qcom,cam-gmsl-sensor"; + cell-index = <11>; + csiphy-sd-index = <2>; + status = "ok"; + + port { + gmsl_sensor11_ep: endpoint { + remote-endpoint = <&deser2_port3>; + }; + }; + }; + + /*cam2-ov9282*/ + rb4_slot2: qcom,cam-sensor18 { + compatible = "qcom,cam-sensor"; + cell-index = <18>; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 69 0>, + <&tlmm 75 0>, + <&expander2 2 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK2", + "CAMIF_RESET2", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam2-imx577*/ + qcom,cam-sensor22 { + compatible = "qcom,cam-sensor"; + cell-index = <22>; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam22>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 69 0>, + <&tlmm 75 0>, + <&expander2 2 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK2", + "CAMIF_RESET2", + "CAM_CUSTOM1"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + eeprom_cam22: qcom,eeprom22 { + compatible = "qcom,eeprom"; + cell-index = <22>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 69 0>, + <&tlmm 75 0>, + <&expander2 2 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAM_MCLK2", + "CAMIF_RESET2", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; +}; + +&soc { + qcom,cam-res-mgr { + compatible = "qcom,cam-res-mgr"; + status = "ok"; + gpios-shared = <609 610 611>; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-camx.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-camx.dtso new file mode 100644 index 0000000000000..3945c77528ccb --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-camx.dtso @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "monaco-camera.dtsi" +#include "monaco-evk-camera-sensor.dtsi" + +&camss { + status = "disabled"; +}; + +&tlmm { + cam_sensor_active_rst0: cam-sensor-active-rst0 { + /* RESET */ + mux { + pins = "gpio73"; + function = "gpio"; + }; + + config { + pins = "gpio73"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst1: cam-sensor-active-rst1 { + /* RESET */ + mux { + pins = "gpio74"; + function = "gpio"; + }; + + config { + pins = "gpio74"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst2: cam-sensor-active-rst2 { + /* RESET */ + mux { + pins = "gpio75"; + function = "gpio"; + }; + + config { + pins = "gpio75"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_suspend_rst0: cam-sensor-suspend-rst0 { + /* RESET */ + mux { + pins = "gpio73"; + function = "gpio"; + }; + + config { + pins = "gpio73"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst1: cam-sensor-suspend-rst1 { + /* RESET */ + mux { + pins = "gpio74"; + function = "gpio"; + }; + config { + pins = "gpio74"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst2: cam-sensor-suspend-rst2 { + /* RESET */ + mux { + pins = "gpio75"; + function = "gpio"; + }; + config { + pins = "gpio75"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-common.dtsi b/arch/arm64/boot/dts/qcom/monaco-evk-common.dtsi new file mode 100644 index 0000000000000..2434d1f9f58ba --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-common.dtsi @@ -0,0 +1,1108 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include + +#include "monaco.dtsi" +#include "monaco-pmics.dtsi" + +/ { + aliases { + ethernet0 = ðernet0; + i2c1 = &i2c1; + serial0 = &uart7; + serial1 = &uart2; + serial2 = &uart6; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + connector-1 { + compatible = "usb-c-connector"; + label = "USB1-Type-C"; + data-role = "host"; + power-role = "source"; + + vbus-supply = <&usb1_vbus>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb1_con_hs_ep: endpoint { + remote-endpoint = <&usb_hub_2_1>; + }; + }; + + port@1 { + reg = <1>; + + usb1_con_ss_ep: endpoint { + remote-endpoint = <&hd3ss3220_1_in_ep>; + }; + + }; + }; + }; + + connector-2 { + compatible = "gpio-usb-b-connector", "usb-b-connector"; + label = "micro-USB"; + type = "micro"; + + id-gpios = <&pmm8620au_0_gpios 9 GPIO_ACTIVE_HIGH>; + vbus-gpios = <&expander6 7 GPIO_ACTIVE_HIGH>; + vbus-supply = <&usb2_vbus>; + + pinctrl-0 = <&usb2_id>; + pinctrl-names = "default"; + + port { + usb2_con_hs_ep: endpoint { + remote-endpoint = <&usb_2_dwc3_hs>; + }; + }; + }; + + dmic: audio-codec-0 { + compatible = "dmic-codec"; + #sound-dai-cells = <0>; + num-channels = <1>; + }; + + max98357a: audio-codec-1 { + compatible = "maxim,max98357a"; + #sound-dai-cells = <0>; + }; + + dp-connector-0 { + compatible = "dp-connector"; + label = "DP0"; + type = "mini"; + + port { + dp0_connector_in: endpoint { + remote-endpoint = <<8713sx_dp0_out>; + }; + }; + }; + + dp-connector-1 { + compatible = "dp-connector"; + label = "DP1"; + type = "mini"; + + port { + dp1_connector_in: endpoint { + remote-endpoint = <<8713sx_dp1_out>; + }; + }; + }; + + usb1_vbus: regulator-usb1-vbus { + compatible = "regulator-fixed"; + regulator-name = "usb1_vbus"; + gpio = <&expander1 3 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + usb2_vbus: regulator-usb2-vbus { + compatible = "regulator-fixed"; + regulator-name = "usb2_vbus"; + gpio = <&pmm8650au_1_gpios 7 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + sound { + compatible = "qcom,qcs8275-sndcard"; + model = "MONACO-EVK"; + + pinctrl-0 = <&hs0_mi2s_active>, <&mi2s1_active>; + pinctrl-names = "default"; + + hs0-mi2s-playback-dai-link { + link-name = "HS0 MI2S Playback"; + + codec { + sound-dai = <&max98357a>; + }; + + cpu { + sound-dai = <&q6apmbedai PRIMARY_MI2S_RX>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + sec-mi2s-capture-dai-link { + link-name = "Secondary MI2S Capture"; + + codec { + sound-dai = <&dmic>; + }; + + cpu { + sound-dai = <&q6apmbedai SECONDARY_MI2S_TX>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + }; + + vreg_cam0_2p8: vreg-cam0-2p8 { + compatible = "regulator-fixed"; + regulator-name = "vreg_cam0_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + startup-delay-us = <10000>; + + gpio = <&tlmm 73 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&cam0_avdd_2v8_en_default>; + pinctrl-names = "default"; + }; + + vreg_cam1_2p8: vreg-cam1-2p8 { + compatible = "regulator-fixed"; + regulator-name = "vreg_cam1_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + startup-delay-us = <10000>; + + gpio = <&tlmm 74 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&cam1_avdd_2v8_en_default>; + pinctrl-names = "default"; + }; + + vreg_cam2_2p8: vreg-cam2-2p8 { + compatible = "regulator-fixed"; + regulator-name = "vreg_cam2_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + startup-delay-us = <10000>; + + gpio = <&tlmm 75 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&cam2_avdd_2v8_en_default>; + pinctrl-names = "default"; + }; + + vreg_dcin_12v: regulator-dcin-12v { + compatible = "regulator-fixed"; + + regulator-name = "VREG_DCIN_12V"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + + regulator-boot-on; + regulator-always-on; + }; + + vreg_wcn_3p3: regulator-wcn-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_WCN_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + vin-supply = <&vreg_dcin_12v>; + + regulator-boot-on; + }; +}; + +&apps_rsc { + regulators-0 { + compatible = "qcom,pmm8654au-rpmh-regulators"; + qcom,pmic-id = "a"; + + vreg_l3a: ldo3 { + regulator-name = "vreg_l3a"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l4a: ldo4 { + regulator-name = "vreg_l4a"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l5a: ldo5 { + regulator-name = "vreg_l5a"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l6a: ldo6 { + regulator-name = "vreg_l6a"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l7a: ldo7 { + regulator-name = "vreg_l7a"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l8a: ldo8 { + regulator-name = "vreg_l8a"; + regulator-min-microvolt = <2504000>; + regulator-max-microvolt = <2960000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l9a: ldo9 { + regulator-name = "vreg_l9a"; + regulator-min-microvolt = <2970000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-1 { + compatible = "qcom,pmm8654au-rpmh-regulators"; + qcom,pmic-id = "c"; + + vreg_s5c: smps5 { + regulator-name = "vreg_s5c"; + regulator-min-microvolt = <1104000>; + regulator-max-microvolt = <1104000>; + regulator-initial-mode = ; + }; + + vreg_l1c: ldo1 { + regulator-name = "vreg_l1c"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <512000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2c: ldo2 { + regulator-name = "vreg_l2c"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <904000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l4c: ldo4 { + regulator-name = "vreg_l4c"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l7c: ldo7 { + regulator-name = "vreg_l7c"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l8c: ldo8 { + regulator-name = "vreg_l8c"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l9c: ldo9 { + regulator-name = "vreg_l9c"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; +}; + +ðernet0 { + phy-mode = "2500base-x"; + phy-handle = <&hsgmii_phy0>; + + pinctrl-0 = <ðernet0_default>; + pinctrl-names = "default"; + + snps,mtl-rx-config = <&mtl_rx_setup>; + snps,mtl-tx-config = <&mtl_tx_setup>; + nvmem-cells = <&mac_addr0>; + nvmem-cell-names = "mac-address"; + + status = "okay"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + hsgmii_phy0: ethernet-phy@1c { + compatible = "ethernet-phy-id004d.d101"; + reg = <0x1c>; + reset-gpios = <&tlmm 31 GPIO_ACTIVE_LOW>; + reset-assert-us = <11000>; + reset-deassert-us = <70000>; + }; + }; + + mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <4>; + snps,rx-sched-sp; + + queue0 { + snps,dcb-algorithm; + snps,map-to-dma-channel = <0x0>; + snps,route-up; + snps,priority = <0x1>; + }; + + queue1 { + snps,dcb-algorithm; + snps,map-to-dma-channel = <0x1>; + snps,route-ptp; + }; + + queue2 { + snps,avb-algorithm; + snps,map-to-dma-channel = <0x2>; + snps,route-avcp; + }; + + queue3 { + snps,avb-algorithm; + snps,map-to-dma-channel = <0x3>; + snps,priority = <0xc>; + }; + }; + + mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <4>; + + queue0 { + snps,dcb-algorithm; + }; + + queue1 { + snps,dcb-algorithm; + }; + + queue2 { + snps,avb-algorithm; + snps,send_slope = <0x1000>; + snps,idle_slope = <0x1000>; + snps,high_credit = <0x3e800>; + snps,low_credit = <0xffc18000>; + }; + + queue3 { + snps,avb-algorithm; + snps,send_slope = <0x1000>; + snps,idle_slope = <0x1000>; + snps,high_credit = <0x3e800>; + snps,low_credit = <0xffc18000>; + }; + }; +}; + +&gpi_dma0 { + status = "okay"; +}; + +&gpi_dma1 { + status = "okay"; +}; + +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/qcs8300/a623_zap.mbn"; +}; + +&i2c0 { + status = "okay"; + + bridge@4f { + compatible = "lontium,lt8713sx"; + reg = <0x4f>; + reset-gpios = <&expander5 6 GPIO_ACTIVE_LOW>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + lt8713sx_dp_in: endpoint { + remote-endpoint = <&mdss_dp0_out>; + }; + }; + + port@1 { + reg = <1>; + + lt8713sx_dp0_out: endpoint { + remote-endpoint = <&dp0_connector_in>; + }; + }; + + port@2 { + reg = <2>; + + lt8713sx_dp1_out: endpoint { + remote-endpoint = <&dp1_connector_in>; + }; + }; + }; + }; + + usb-typec@47 { + compatible = "ti,hd3ss3220"; + reg = <0x47>; + + interrupts-extended = <&tlmm 45 IRQ_TYPE_EDGE_FALLING>; + + id-gpios = <&tlmm 13 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&usb1_id>, <&usb1_intr>; + pinctrl-names = "default"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hd3ss3220_1_in_ep: endpoint { + remote-endpoint = <&usb1_con_ss_ep>; + }; + }; + + port@1 { + reg = <1>; + + hd3ss3220_1_out_ep: endpoint { + remote-endpoint = <&usb_hub_3_1>; + }; + }; + }; + }; +}; + +&i2c1 { + pinctrl-0 = <&qup_i2c1_default>; + pinctrl-names = "default"; + + status = "okay"; + + fan_controller: fan@18 { + compatible = "ti,amc6821"; + reg = <0x18>; + #pwm-cells = <2>; + + fan { + pwms = <&fan_controller 40000 PWM_POLARITY_INVERTED>; + }; + }; + + eeprom0: eeprom@50 { + compatible = "atmel,24c256"; + reg = <0x50>; + pagesize = <64>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + mac_addr0: mac-addr@0 { + reg = <0x0 0x6>; + }; + }; + }; +}; + +&i2c15 { + pinctrl-0 = <&qup_i2c15_default>; + pinctrl-names = "default"; + + status = "okay"; + + expander0: gpio@38 { + compatible = "ti,tca9538"; + reg = <0x38>; + #gpio-cells = <2>; + gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 56 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander0_int>; + pinctrl-names = "default"; + }; + + expander1: gpio@39 { + compatible = "ti,tca9538"; + reg = <0x39>; + #gpio-cells = <2>; + gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 16 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander1_int>; + pinctrl-names = "default"; + }; + + expander2: gpio@3a { + compatible = "ti,tca9538"; + reg = <0x3a>; + #gpio-cells = <2>; + gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 95 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander2_int>; + pinctrl-names = "default"; + }; + + expander3: gpio@3b { + compatible = "ti,tca9538"; + reg = <0x3b>; + #gpio-cells = <2>; + gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 24 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander3_int>; + pinctrl-names = "default"; + }; + + expander4: gpio@3c { + compatible = "ti,tca9538"; + reg = <0x3c>; + #gpio-cells = <2>; + gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 96 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander4_int>; + pinctrl-names = "default"; + }; + + expander5: gpio@3d { + compatible = "ti,tca9538"; + reg = <0x3d>; + #gpio-cells = <2>; + gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander5_int>; + pinctrl-names = "default"; + + usb1_ss_sel_switch_hog: usb1-ss-sel-switch-hog { + gpio-hog; + gpios = <5 GPIO_ACTIVE_HIGH>; + output-high; + }; + }; + + expander6: gpio@3e { + compatible = "ti,tca9538"; + reg = <0x3e>; + #gpio-cells = <2>; + gpio-controller; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&tlmm 52 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&expander6_int>; + pinctrl-names = "default"; + }; +}; + +&iris { + status = "okay"; +}; + +&mdss { + status = "okay"; +}; + +&mdss_dp0 { + pinctrl-0 = <&dp_hot_plug_det>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&mdss_dp0_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + remote-endpoint = <<8713sx_dp_in>; +}; + +&mdss_dp0_phy { + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&pcie0 { + pinctrl-0 = <&pcie0_default_state>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie0_phy { + vdda-phy-supply = <&vreg_l6a>; + vdda-pll-supply = <&vreg_l5a>; + vdda-qref-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&pcie1 { + pinctrl-0 = <&pcie1_default_state>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie1_phy { + vdda-phy-supply = <&vreg_l6a>; + vdda-pll-supply = <&vreg_l5a>; + vdda-qref-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&pcieport0 { + reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>; +}; + +&pcieport1 { + reset-gpios = <&tlmm 23 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>; +}; + +&pmm8620au_0_gpios { + usb2_id: usb2-id-state { + pins = "gpio9"; + function = "normal"; + input-enable; + bias-pull-up; + power-source = <0>; + }; +}; + +&qup_i2c0_data_clk { + drive-strength = <2>; + bias-pull-up; +}; + +&qupv3_id_0 { + firmware-name = "qcom/qcs8300/qupv3fw.elf"; + status = "okay"; +}; + +&qupv3_id_1 { + firmware-name = "qcom/qcs8300/qupv3fw.elf"; + status = "okay"; +}; + +&remoteproc_adsp { + firmware-name = "qcom/qcs8300/adsp.mbn"; + + status = "okay"; +}; + +&remoteproc_cdsp { + firmware-name = "qcom/qcs8300/cdsp0.mbn"; + + status = "okay"; +}; + +&remoteproc_gpdsp { + firmware-name = "qcom/qcs8300/gpdsp0.mbn"; + + status = "okay"; +}; + +&serdes0 { + phy-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&spi10 { + status = "okay"; + + tpm@0 { + compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi"; + reg = <0>; + spi-max-frequency = <20000000>; + }; +}; + +&tlmm { + pcie0_default_state: pcie0-default-state { + wake-pins { + pins = "gpio0"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + clkreq-pins { + pins = "gpio1"; + function = "pcie0_clkreq"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-pins { + pins = "gpio2"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + ethernet0_default: ethernet0-default-state { + ethernet0_mdc: ethernet0-mdc-pins { + pins = "gpio5"; + function = "emac0_mdc"; + drive-strength = <16>; + bias-pull-up; + }; + + ethernet0_mdio: ethernet0-mdio-pins { + pins = "gpio6"; + function = "emac0_mdio"; + drive-strength = <16>; + bias-pull-up; + }; + }; + + expander5_int: expander5-int-state { + pins = "gpio3"; + function = "gpio"; + bias-pull-up; + }; + + usb1_hub_reset_hog: usb1-hub-reset-hog { + gpio-hog; + gpios = <7 GPIO_ACTIVE_HIGH>; + output-high; + }; + + usb1_id: usb1-id-state { + pins = "gpio13"; + function = "gpio"; + bias-pull-up; + }; + + usb1_hs_sel_switch_hog: usb1-hs-sel-switch-hog { + gpio-hog; + gpios = <14 GPIO_ACTIVE_HIGH>; + output-high; + }; + + expander1_int: expander1-int-state { + pins = "gpio16"; + function = "gpio"; + bias-pull-up; + }; + + qup_i2c1_default: qup-i2c1-state { + pins = "gpio19", "gpio20"; + function = "qup0_se1"; + drive-strength = <2>; + bias-pull-up; + }; + + pcie1_default_state: pcie1-default-state { + wake-pins { + pins = "gpio21"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + clkreq-pins { + pins = "gpio22"; + function = "pcie1_clkreq"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-pins { + pins = "gpio23"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + expander3_int: expander3-int-state { + pins = "gpio24"; + function = "gpio"; + bias-pull-up; + }; + + usb1_intr: usb1-intr-state { + pins = "gpio45"; + function = "gpio"; + bias-pull-up; + drive-strength = <2>; + }; + + expander6_int: expander6-int-state { + pins = "gpio52"; + function = "gpio"; + bias-pull-up; + }; + + expander0_int: expander0-int-state { + pins = "gpio56"; + function = "gpio"; + bias-pull-up; + }; + + cam0_avdd_2v8_en_default: cam0-avdd-2v8-en-state { + pins = "gpio73"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + cam1_avdd_2v8_en_default: cam1-avdd-2v8-en-state { + pins = "gpio74"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + cam2_avdd_2v8_en_default: cam2-avdd-2v8-en-state { + pins = "gpio75"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + qup_i2c15_default: qup-i2c15-state { + pins = "gpio91", "gpio92"; + function = "qup1_se7"; + drive-strength = <2>; + bias-pull-up; + }; + + expander2_int: expander2-int-state { + pins = "gpio95"; + function = "gpio"; + bias-pull-up; + }; + + expander4_int: expander4-int-state { + pins = "gpio96"; + function = "gpio"; + bias-pull-up; + }; +}; + +&uart2 { + status = "okay"; + + bluetooth: bluetooth { + compatible = "qcom,wcn6855-bt"; + max-speed = <3200000>; + + vddrfacmn-supply = <&vreg_wcn_3p3>; + vddaon-supply = <&vreg_wcn_3p3>; + vddwlcx-supply = <&vreg_wcn_3p3>; + vddwlmx-supply = <&vreg_wcn_3p3>; + vddbtcmx-supply = <&vreg_wcn_3p3>; + vddrfa0p8-supply = <&vreg_wcn_3p3>; + vddrfa1p2-supply = <&vreg_wcn_3p3>; + vddrfa1p8-supply = <&vreg_wcn_3p3>; + }; +}; + +&uart6 { + status = "okay"; +}; + +&uart7 { + status = "okay"; +}; + +&ufs_mem_hc { + reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>; + vcc-supply = <&vreg_l8a>; + vcc-max-microamp = <1100000>; + vccq-supply = <&vreg_l4c>; + vccq-max-microamp = <1200000>; + + status = "okay"; +}; + +&ufs_mem_phy { + vdda-phy-supply = <&vreg_l4a>; + vdda-pll-supply = <&vreg_l5a>; + + status = "okay"; +}; + +&usb_1 { + dr_mode = "host"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "okay"; + + usb_hub_2_x: hub@1 { + compatible = "usb5e3,610"; + reg = <1>; + + peer-hub = <&usb_hub_3_x>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + usb_hub_2_1: endpoint { + remote-endpoint = <&usb1_con_hs_ep>; + }; + }; + + /* + * Port-4 is connected to M.2 E key connector on corekit. + */ + port@4 { + reg = <4>; + + usb_hub_2_4: endpoint { + }; + }; + }; + }; + + usb_hub_3_x: hub@2 { + compatible = "usb5e3,625"; + reg = <2>; + + peer-hub = <&usb_hub_2_x>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + usb_hub_3_1: endpoint { + remote-endpoint = <&hd3ss3220_1_out_ep>; + }; + }; + + port@4 { + reg = <4>; + + usb_hub_3_4: endpoint { + }; + }; + }; + }; +}; + +&usb_1_hsphy { + vdda-pll-supply = <&vreg_l7a>; + vdda18-supply = <&vreg_l7c>; + vdda33-supply = <&vreg_l9a>; + + status = "okay"; +}; + +&usb_qmpphy { + vdda-phy-supply = <&vreg_l7a>; + vdda-pll-supply = <&vreg_l5a>; + + status = "okay"; +}; + +&usb_2 { + status = "okay"; +}; + +&usb_2_dwc3_hs { + remote-endpoint = <&usb2_con_hs_ep>; +}; + +&usb_2_hsphy { + vdda-pll-supply = <&vreg_l7a>; + vdda18-supply = <&vreg_l7c>; + vdda33-supply = <&vreg_l9a>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-emmc.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-emmc.dtso new file mode 100644 index 0000000000000..70edbb1248a5a --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-emmc.dtso @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +/ { + vreg_s2s: regulator-vreg-s2s { + compatible = "regulator-fixed"; + regulator-name = "regulator-s2s"; + + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; +}; + +&sdhc_1 { + vmmc-supply = <&vreg_l8a>; + vqmmc-supply = <&vreg_s2s>; + + supports-cqe; + + pinctrl-0 = <&sdc1_state_on>; + pinctrl-1 = <&sdc1_state_off>; + + pinctrl-names = "default", "sleep"; + + non-removable; + + bus-width = <8>; + no-sd; + no-sdio; + + status = "okay"; +}; + +&ufs_mem_hc { + status = "disabled"; +}; + +&vreg_l8a { + regulator-min-microvolt = <2960000>; + regulator-max-microvolt = <2960000>; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso new file mode 100644 index 0000000000000..e6beb4393430b --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include + +&{/} { + model = "Qualcomm Technologies, Inc. Monaco-EVK IFP Mezzanine"; + + vreg_0p9: regulator-0v9 { + compatible = "regulator-fixed"; + regulator-name = "VREG_0P9"; + + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + }; + + vreg_1p8: regulator-1v8 { + compatible = "regulator-fixed"; + regulator-name = "VREG_1P8"; + + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; +}; + +&i2c15 { + #address-cells = <1>; + #size-cells = <0>; + + eeprom1: eeprom@52 { + compatible = "giantec,gt24c256c", "atmel,24c256"; + reg = <0x52>; + pagesize = <64>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + }; + }; +}; + +&pcie0 { + iommu-map = <0x0 &pcie_smmu 0x0 0x1>, + <0x100 &pcie_smmu 0x1 0x1>, + <0x208 &pcie_smmu 0x2 0x1>, + <0x210 &pcie_smmu 0x3 0x1>, + <0x218 &pcie_smmu 0x4 0x1>, + <0x300 &pcie_smmu 0x5 0x1>, + <0x400 &pcie_smmu 0x6 0x1>, + <0x500 &pcie_smmu 0x7 0x1>, + <0x501 &pcie_smmu 0x8 0x1>; +}; + +&pcieport0 { + #address-cells = <3>; + #size-cells = <2>; + + pcie@0,0 { + compatible = "pci1179,0623"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x2 0xff>; + + vddc-supply = <&vreg_0p9>; + vdd18-supply = <&vreg_1p8>; + vdd09-supply = <&vreg_0p9>; + vddio1-supply = <&vreg_1p8>; + vddio2-supply = <&vreg_1p8>; + vddio18-supply = <&vreg_1p8>; + + i2c-parent = <&i2c15 0x77>; + + resx-gpios = <&tlmm 124 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&tc9563_resx_n>; + pinctrl-names = "default"; + + pcie@1,0 { + reg = <0x20800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x3 0xff>; + }; + + pcie@2,0 { + reg = <0x21000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x4 0xff>; + }; + + pcie@3,0 { + reg = <0x21800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x5 0xff>; + + pci@0,0 { + reg = <0x50000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + + pci@0,1 { + reg = <0x50100 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + }; + }; +}; + +&tlmm { + tc9563_resx_n: tc9563-resx-state { + pins = "gpio124"; + function = "gpio"; + bias-disable; + /* Reset pin of tc9563 is active low hence set default + * state of this pin to output-high. + */ + output-high; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-sd-card.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-sd-card.dtso new file mode 100644 index 0000000000000..606d0c5001cf5 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-sd-card.dtso @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include + +/ { + vmmc_sdc: regulator-dummy { + compatible = "regulator-fixed"; + + regulator-name = "vmmc_sdc"; + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <2950000>; + }; + + vreg_sdc: regulator-sdc { + compatible = "regulator-gpio"; + + regulator-name = "vreg_sdc"; + regulator-type = "voltage"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2950000>; + + gpios = <&expander1 7 GPIO_ACTIVE_HIGH>; + states = <1800000 1>, <2950000 0>; + + startup-delay-us = <100>; + }; +}; + +&sdhc_1 { + vmmc-supply = <&vmmc_sdc>; + vqmmc-supply = <&vreg_sdc>; + + pinctrl-0 = <&sdc1_state_on>, <&sd_cd>; + pinctrl-1 = <&sdc1_state_off>, <&sd_cd>; + pinctrl-names = "default", "sleep"; + + cap-sd-highspeed; + no-1-8-v; + + bus-width = <4>; + cd-gpios = <&tlmm 11 GPIO_ACTIVE_LOW>; + no-mmc; + no-sdio; + + status = "okay"; +}; + +&sdhc1_opp_table { + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-202000000 { + opp-hz = /bits/ 64 <202000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; +}; + +&tlmm { + sd_cd: sd-cd-state { + pins = "gpio11"; + function = "gpio"; + bias-pull-up; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-staging.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-staging.dtso new file mode 100644 index 0000000000000..75cf03300c27b --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-evk-staging.dtso @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include + +&i2c15 { + eeprom1: eeprom@52 { + nvmem-layout { + mac_addr1: mac-addr@0 { + reg = <0x0 0x6>; + }; + + mac_addr2: mac-addr@6 { + reg = <0x6 0x6>; + }; + }; + }; +}; + +&pcieport0 { + pcie@0,0 { + pcie@3,0 { + pci@0,0 { + interrupts-extended = <&tlmm 40 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + nvmem-cells = <&mac_addr1>; + nvmem-cell-names = "mac-address"; + pinctrl-names = "default"; + pinctrl-0 = <&aqr_intn_wol_sig>; + phy-reset-gpios = <&tlmm 10 GPIO_ACTIVE_HIGH>; + reset-deassert-us = <221000>; + }; + + pci@0,1 { + interrupts-extended = <&tlmm 39 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + nvmem-cells = <&mac_addr2>; + nvmem-cell-names = "mac-address"; + pinctrl-names = "default"; + pinctrl-0 = <&napa_intn_wol_sig>; + phy-reset-gpios = <&expander5 0 GPIO_ACTIVE_LOW>; + reset-deassert-us = <20000>; + }; + }; + }; +}; + +&tlmm { + qps615_intn_wol { + aqr_intn_wol_sig: aqr-intn-wol-sig { + pins = "gpio40"; + function = "gpio"; + input-enable; + bias-disable; + }; + + napa_intn_wol_sig: napa-intn-wol-sig { + pins = "gpio39"; + function = "gpio"; + input-enable; + bias-disable; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk.dts b/arch/arm64/boot/dts/qcom/monaco-evk.dts index e72cf6725a52c..56422ab42b27a 100644 --- a/arch/arm64/boot/dts/qcom/monaco-evk.dts +++ b/arch/arm64/boot/dts/qcom/monaco-evk.dts @@ -5,503 +5,30 @@ /dts-v1/; -#include -#include -#include - -#include "qcs8300.dtsi" -#include "qcs8300-pmics.dtsi" +#include "monaco-evk-common.dtsi" / { model = "Qualcomm Technologies, Inc. Monaco EVK"; compatible = "qcom,monaco-evk", "qcom,qcs8300"; - aliases { - ethernet0 = ðernet0; - i2c1 = &i2c1; - serial0 = &uart7; - }; - - chosen { - stdout-path = "serial0:115200n8"; - }; - - dmic: audio-codec-0 { - compatible = "dmic-codec"; - #sound-dai-cells = <0>; - num-channels = <1>; - }; - - max98357a: audio-codec-1 { - compatible = "maxim,max98357a"; - #sound-dai-cells = <0>; - }; - - sound { - compatible = "qcom,qcs8275-sndcard"; - model = "MONACO-EVK"; - - pinctrl-0 = <&hs0_mi2s_active>, <&mi2s1_active>; - pinctrl-names = "default"; - - hs0-mi2s-playback-dai-link { - link-name = "HS0 MI2S Playback"; - - codec { - sound-dai = <&max98357a>; - }; - - cpu { - sound-dai = <&q6apmbedai PRIMARY_MI2S_RX>; - }; - - platform { - sound-dai = <&q6apm>; - }; - }; - - sec-mi2s-capture-dai-link { - link-name = "Secondary MI2S Capture"; - - codec { - sound-dai = <&dmic>; - }; - - cpu { - sound-dai = <&q6apmbedai SECONDARY_MI2S_TX>; - }; - - platform { - sound-dai = <&q6apm>; - }; - }; - }; -}; - -&apps_rsc { - regulators-0 { - compatible = "qcom,pmm8654au-rpmh-regulators"; - qcom,pmic-id = "a"; - - vreg_l3a: ldo3 { - regulator-name = "vreg_l3a"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <1200000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l4a: ldo4 { - regulator-name = "vreg_l4a"; - regulator-min-microvolt = <880000>; - regulator-max-microvolt = <912000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l5a: ldo5 { - regulator-name = "vreg_l5a"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <1200000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l6a: ldo6 { - regulator-name = "vreg_l6a"; - regulator-min-microvolt = <880000>; - regulator-max-microvolt = <912000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l7a: ldo7 { - regulator-name = "vreg_l7a"; - regulator-min-microvolt = <880000>; - regulator-max-microvolt = <912000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l8a: ldo8 { - regulator-name = "vreg_l8a"; - regulator-min-microvolt = <2504000>; - regulator-max-microvolt = <2960000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l9a: ldo9 { - regulator-name = "vreg_l9a"; - regulator-min-microvolt = <2970000>; - regulator-max-microvolt = <3072000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - }; - - regulators-1 { - compatible = "qcom,pmm8654au-rpmh-regulators"; - qcom,pmic-id = "c"; - - vreg_s5c: smps5 { - regulator-name = "vreg_s5c"; - regulator-min-microvolt = <1104000>; - regulator-max-microvolt = <1104000>; - regulator-initial-mode = ; - }; - - vreg_l1c: ldo1 { - regulator-name = "vreg_l1c"; - regulator-min-microvolt = <300000>; - regulator-max-microvolt = <512000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l2c: ldo2 { - regulator-name = "vreg_l2c"; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <904000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l4c: ldo4 { - regulator-name = "vreg_l4c"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <1200000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l7c: ldo7 { - regulator-name = "vreg_l7c"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l8c: ldo8 { - regulator-name = "vreg_l8c"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - - vreg_l9c: ldo9 { - regulator-name = "vreg_l9c"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-initial-mode = ; - regulator-allow-set-load; - regulator-allowed-modes = ; - }; - }; -}; - -ðernet0 { - phy-mode = "2500base-x"; - phy-handle = <&hsgmii_phy0>; - - pinctrl-0 = <ðernet0_default>; - pinctrl-names = "default"; - - snps,mtl-rx-config = <&mtl_rx_setup>; - snps,mtl-tx-config = <&mtl_tx_setup>; - nvmem-cells = <&mac_addr0>; - nvmem-cell-names = "mac-address"; - - status = "okay"; - - mdio { - compatible = "snps,dwmac-mdio"; - #address-cells = <1>; - #size-cells = <0>; - - hsgmii_phy0: ethernet-phy@1c { - compatible = "ethernet-phy-id004d.d101"; - reg = <0x1c>; - reset-gpios = <&tlmm 31 GPIO_ACTIVE_LOW>; - reset-assert-us = <11000>; - reset-deassert-us = <70000>; - }; - }; - - mtl_rx_setup: rx-queues-config { - snps,rx-queues-to-use = <4>; - snps,rx-sched-sp; - - queue0 { - snps,dcb-algorithm; - snps,map-to-dma-channel = <0x0>; - snps,route-up; - snps,priority = <0x1>; - }; - - queue1 { - snps,dcb-algorithm; - snps,map-to-dma-channel = <0x1>; - snps,route-ptp; - }; - - queue2 { - snps,avb-algorithm; - snps,map-to-dma-channel = <0x2>; - snps,route-avcp; - }; - - queue3 { - snps,avb-algorithm; - snps,map-to-dma-channel = <0x3>; - snps,priority = <0xc>; - }; - }; - - mtl_tx_setup: tx-queues-config { - snps,tx-queues-to-use = <4>; - - queue0 { - snps,dcb-algorithm; - }; - - queue1 { - snps,dcb-algorithm; - }; - - queue2 { - snps,avb-algorithm; - snps,send_slope = <0x1000>; - snps,idle_slope = <0x1000>; - snps,high_credit = <0x3e800>; - snps,low_credit = <0xffc18000>; - }; - - queue3 { - snps,avb-algorithm; - snps,send_slope = <0x1000>; - snps,idle_slope = <0x1000>; - snps,high_credit = <0x3e800>; - snps,low_credit = <0xffc18000>; - }; - }; -}; - -&gpi_dma0 { - status = "okay"; -}; - -&gpi_dma1 { - status = "okay"; -}; - -&gpu { - status = "okay"; -}; - -&gpu_zap_shader { - firmware-name = "qcom/qcs8300/a623_zap.mbn"; -}; - -&i2c1 { - pinctrl-0 = <&qup_i2c1_default>; - pinctrl-names = "default"; + /* This comes from a PMIC handled within the SAIL domain */ + vreg_s2s: vreg-s2s { + compatible = "regulator-fixed"; + regulator-name = "vreg_s2s"; - status = "okay"; - - eeprom0: eeprom@50 { - compatible = "atmel,24c256"; - reg = <0x50>; - pagesize = <64>; - - nvmem-layout { - compatible = "fixed-layout"; - #address-cells = <1>; - #size-cells = <1>; - - mac_addr0: mac-addr@0 { - reg = <0x0 0x6>; - }; - }; - }; -}; - -&i2c15 { - pinctrl-0 = <&qup_i2c15_default>; - pinctrl-names = "default"; - - status = "okay"; - - expander0: gpio@38 { - compatible = "ti,tca9538"; - reg = <0x38>; - #gpio-cells = <2>; - gpio-controller; - }; - - expander1: gpio@39 { - compatible = "ti,tca9538"; - reg = <0x39>; - #gpio-cells = <2>; - gpio-controller; - }; - - expander2: gpio@3a { - compatible = "ti,tca9538"; - reg = <0x3a>; - #gpio-cells = <2>; - gpio-controller; - }; - - expander3: gpio@3b { - compatible = "ti,tca9538"; - reg = <0x3b>; - #gpio-cells = <2>; - gpio-controller; - }; - - expander4: gpio@3c { - compatible = "ti,tca9538"; - reg = <0x3c>; - #gpio-cells = <2>; - gpio-controller; - }; - - expander5: gpio@3d { - compatible = "ti,tca9538"; - reg = <0x3d>; - #gpio-cells = <2>; - gpio-controller; - }; - - expander6: gpio@3e { - compatible = "ti,tca9538"; - reg = <0x3e>; - #gpio-cells = <2>; - gpio-controller; - }; -}; - -&iris { - status = "okay"; -}; - -&qupv3_id_0 { - status = "okay"; -}; - -&qupv3_id_1 { - status = "okay"; -}; - -&remoteproc_adsp { - firmware-name = "qcom/qcs8300/adsp.mbn"; - - status = "okay"; -}; - -&remoteproc_cdsp { - firmware-name = "qcom/qcs8300/cdsp0.mbn"; - - status = "okay"; -}; - -&remoteproc_gpdsp { - firmware-name = "qcom/qcs8300/gpdsp0.mbn"; - - status = "okay"; -}; - -&serdes0 { - phy-supply = <&vreg_l4a>; - - status = "okay"; -}; - -&tlmm { - ethernet0_default: ethernet0-default-state { - ethernet0_mdc: ethernet0-mdc-pins { - pins = "gpio5"; - function = "emac0_mdc"; - drive-strength = <16>; - bias-pull-up; - }; - - ethernet0_mdio: ethernet0-mdio-pins { - pins = "gpio6"; - function = "emac0_mdio"; - drive-strength = <16>; - bias-pull-up; - }; - }; - - qup_i2c1_default: qup-i2c1-state { - pins = "gpio19", "gpio20"; - function = "qup0_se1"; - drive-strength = <2>; - bias-pull-up; - }; - - qup_i2c15_default: qup-i2c15-state { - pins = "gpio91", "gpio92"; - function = "qup1_se7"; - drive-strength = <2>; - bias-pull-up; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; }; }; -&uart7 { - status = "okay"; -}; - -&ufs_mem_hc { - reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>; - vcc-supply = <&vreg_l8a>; - vcc-max-microamp = <1100000>; - vccq-supply = <&vreg_l4c>; - vccq-max-microamp = <1200000>; - - status = "okay"; -}; - -&ufs_mem_phy { - vdda-phy-supply = <&vreg_l4a>; - vdda-pll-supply = <&vreg_l5a>; - - status = "okay"; -}; - -&usb_1 { - dr_mode = "peripheral"; +&sdhc_1 { + vmmc-supply = <&vreg_l8a>; + vqmmc-supply = <&vreg_s2s>; - status = "okay"; -}; - -&usb_1_hsphy { - vdda-pll-supply = <&vreg_l7a>; - vdda18-supply = <&vreg_l7c>; - vdda33-supply = <&vreg_l9a>; + no-sd; + no-sdio; + non-removable; status = "okay"; }; -&usb_qmpphy { - vdda-phy-supply = <&vreg_l7a>; - vdda-pll-supply = <&vreg_l5a>; - - status = "okay"; -}; diff --git a/arch/arm64/boot/dts/qcom/qcs8300-pmics.dtsi b/arch/arm64/boot/dts/qcom/monaco-pmics.dtsi similarity index 100% rename from arch/arm64/boot/dts/qcom/qcs8300-pmics.dtsi rename to arch/arm64/boot/dts/qcom/monaco-pmics.dtsi diff --git a/arch/arm64/boot/dts/qcom/monaco-staging.dtso b/arch/arm64/boot/dts/qcom/monaco-staging.dtso new file mode 100644 index 0000000000000..fc44741b8caab --- /dev/null +++ b/arch/arm64/boot/dts/qcom/monaco-staging.dtso @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Monaco staging overlay - add staging-specific device tree modifications here. + */ + +/dts-v1/; +/plugin/; + +&soc { + tgu@4b0e000 { + compatible = "qcom,tgu", "arm,primecell"; + reg = <0x0 0x4b0e000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + + tgu@4b0f000 { + compatible = "qcom,tgu", "arm,primecell"; + reg = <0x0 0x4b0f000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs8300.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi similarity index 79% rename from arch/arm64/boot/dts/qcom/qcs8300.dtsi rename to arch/arm64/boot/dts/qcom/monaco.dtsi index 8d78ccac411e4..e243884a2de9a 100644 --- a/arch/arm64/boot/dts/qcom/qcs8300.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco.dtsi @@ -3,6 +3,7 @@ * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. */ +#include #include #include #include @@ -20,6 +21,7 @@ #include #include #include +#include / { interrupt-parent = <&intc>; @@ -54,6 +56,7 @@ power-domain-names = "psci"; capacity-dmips-mhz = <1946>; dynamic-power-coefficient = <472>; + #cooling-cells = <2>; qcom,freq-domain = <&cpufreq_hw 0>; operating-points-v2 = <&cpu0_opp_table>; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY @@ -78,6 +81,7 @@ power-domains = <&cpu_pd1>; power-domain-names = "psci"; capacity-dmips-mhz = <1946>; + #cooling-cells = <2>; dynamic-power-coefficient = <472>; qcom,freq-domain = <&cpufreq_hw 0>; operating-points-v2 = <&cpu0_opp_table>; @@ -103,6 +107,7 @@ power-domains = <&cpu_pd2>; power-domain-names = "psci"; capacity-dmips-mhz = <1946>; + #cooling-cells = <2>; dynamic-power-coefficient = <507>; qcom,freq-domain = <&cpufreq_hw 2>; operating-points-v2 = <&cpu2_opp_table>; @@ -128,6 +133,7 @@ power-domains = <&cpu_pd3>; power-domain-names = "psci"; capacity-dmips-mhz = <1946>; + #cooling-cells = <2>; dynamic-power-coefficient = <507>; qcom,freq-domain = <&cpufreq_hw 2>; operating-points-v2 = <&cpu2_opp_table>; @@ -153,6 +159,7 @@ power-domains = <&cpu_pd4>; power-domain-names = "psci"; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; qcom,freq-domain = <&cpufreq_hw 1>; operating-points-v2 = <&cpu4_opp_table>; @@ -178,6 +185,7 @@ power-domains = <&cpu_pd5>; power-domain-names = "psci"; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; qcom,freq-domain = <&cpufreq_hw 1>; operating-points-v2 = <&cpu4_opp_table>; @@ -203,6 +211,7 @@ power-domains = <&cpu_pd6>; power-domain-names = "psci"; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; qcom,freq-domain = <&cpufreq_hw 1>; operating-points-v2 = <&cpu4_opp_table>; @@ -228,6 +237,7 @@ power-domains = <&cpu_pd7>; power-domain-names = "psci"; capacity-dmips-mhz = <1024>; + #cooling-cells = <2>; dynamic-power-coefficient = <100>; qcom,freq-domain = <&cpufreq_hw 1>; operating-points-v2 = <&cpu4_opp_table>; @@ -663,7 +673,7 @@ interrupts = ; }; - psci { + psci: psci { compatible = "arm,psci-1.0"; method = "smc"; @@ -756,6 +766,11 @@ hwlocks = <&tcsr_mutex 3>; }; + gunyah_md_mem: gunyah-md-region@91a80000 { + reg = <0x0 0x91a80000 0x0 0x80000>; + no-map; + }; + lpass_machine_learning_mem: lpass-machine-learning-region@93b00000 { reg = <0x0 0x93b00000 0x0 0xf00000>; no-map; @@ -767,7 +782,7 @@ }; camera_mem: camera-region@95200000 { - reg = <0x0 0x95200000 0x0 0x500000>; + reg = <0x0 0x95200000 0x0 0x700000>; no-map; }; @@ -903,8 +918,8 @@ #power-domain-cells = <1>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&sleep_clk>, - <0>, - <0>, + <&pcie0_phy>, + <&pcie1_phy>, <0>, <0>, <0>, @@ -2255,6 +2270,312 @@ qcom,bcm-voters = <&apps_bcm_voter>; }; + pcie0: pci@1c00000 { + device_type = "pci"; + compatible = "qcom,pcie-qcs8300", "qcom,pcie-sa8775p"; + reg = <0x0 0x01c00000 0x0 0x3000>, + <0x0 0x40000000 0x0 0xf20>, + <0x0 0x40000f20 0x0 0xa8>, + <0x0 0x40001000 0x0 0x4000>, + <0x0 0x40100000 0x0 0x100000>, + <0x0 0x01c03000 0x0 0x1000>; + reg-names = "parf", + "dbi", + "elbi", + "atu", + "config", + "mhi"; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, + <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>; + bus-range = <0x00 0xff>; + + dma-coherent; + + linux,pci-domain = <0>; + num-lanes = <2>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_0_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>; + assigned-clock-rates = <19200000>; + + interconnects = <&pcie_anoc MASTER_PCIE_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_PCIE_0 QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "pcie-mem", + "cpu-pcie"; + + iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, + <0x100 &pcie_smmu 0x0001 0x1>; + + resets = <&gcc GCC_PCIE_0_BCR>, + <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; + reset-names = "pci", + "link_down"; + + power-domains = <&gcc GCC_PCIE_0_GDSC>; + + eq-presets-8gts = /bits/ 16 <0x5555 0x5555>; + eq-presets-16gts = /bits/ 8 <0x55 0x55>; + + status = "disabled"; + + pcie3_opp_table: opp-table { + compatible = "operating-points-v2"; + + /* GEN 1 x1 */ + opp-2500000 { + opp-hz = /bits/ 64 <2500000>; + required-opps = <&rpmhpd_opp_svs_l1>; + opp-peak-kBps = <250000 1>; + }; + + /* GEN 1 x2 and GEN 2 x1 */ + opp-5000000 { + opp-hz = /bits/ 64 <5000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + opp-peak-kBps = <500000 1>; + }; + + /* GEN 2 x2 */ + opp-10000000 { + opp-hz = /bits/ 64 <10000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + opp-peak-kBps = <1000000 1>; + }; + + /* GEN 3 x1 */ + opp-8000000 { + opp-hz = /bits/ 64 <8000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + opp-peak-kBps = <984500 1>; + }; + + /* GEN 3 x2 and GEN 4 x1 */ + opp-16000000 { + opp-hz = /bits/ 64 <16000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <1969000 1>; + }; + + /* GEN 4 x2 */ + opp-32000000 { + opp-hz = /bits/ 64 <32000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <3938000 1>; + }; + }; + + pcieport0: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + phys = <&pcie0_phy>; + }; + }; + + pcie0_phy: phy@1c04000 { + compatible = "qcom,qcs8300-qmp-gen4x2-pcie-phy"; + reg = <0x0 0x01c04000 0x0 0x2000>; + + clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_EN>, + <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_0_PIPE_CLK>, + <&gcc GCC_PCIE_0_PIPEDIV2_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe", + "pipediv2"; + + resets = <&gcc GCC_PCIE_0_PHY_BCR>; + reset-names = "phy"; + + assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + #clock-cells = <0>; + clock-output-names = "pcie_0_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + + pcie1: pci@1c10000 { + device_type = "pci"; + compatible = "qcom,pcie-qcs8300", "qcom,pcie-sa8775p"; + reg = <0x0 0x01c10000 0x0 0x3000>, + <0x0 0x60000000 0x0 0xf20>, + <0x0 0x60000f20 0x0 0xa8>, + <0x0 0x60001000 0x0 0x4000>, + <0x0 0x60100000 0x0 0x100000>, + <0x0 0x01c13000 0x0 0x1000>; + reg-names = "parf", + "dbi", + "elbi", + "atu", + "config", + "mhi"; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>, + <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x1fd00000>; + bus-range = <0x00 0xff>; + + dma-coherent; + + linux,pci-domain = <1>; + num-lanes = <4>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_1_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; + assigned-clock-rates = <19200000>; + + interconnects = <&pcie_anoc MASTER_PCIE_1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_PCIE_1 QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, + <0x100 &pcie_smmu 0x0081 0x1>; + + resets = <&gcc GCC_PCIE_1_BCR>, + <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; + reset-names = "pci", + "link_down"; + + power-domains = <&gcc GCC_PCIE_1_GDSC>; + + eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>; + eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>; + + status = "disabled"; + + pcieport1: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + phys = <&pcie1_phy>; + }; + }; + + pcie1_phy: phy@1c14000 { + compatible = "qcom,sa8775p-qmp-gen4x4-pcie-phy"; + reg = <0x0 0x01c14000 0x0 0x4000>; + + clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_EN>, + <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_1_PIPE_CLK>, + <&gcc GCC_PCIE_1_PIPEDIV2_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe", + "pipediv2"; + + assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + resets = <&gcc GCC_PCIE_1_PHY_BCR>; + reset-names = "phy"; + + #clock-cells = <0>; + clock-output-names = "pcie_1_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + ufs_mem_hc: ufs@1d84000 { compatible = "qcom,qcs8300-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; reg = <0x0 0x01d84000 0x0 0x3000>; @@ -2346,7 +2667,39 @@ compatible = "qcom,qcs8300-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0x0 0x01d88000 0x0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; + + operating-points-v2 = <&ice_opp_table>; + + ice_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-201600000 { + opp-hz = /bits/ 64 <201600000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-403200000 { + opp-hz = /bits/ 64 <403200000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + crypto: crypto@1dfa000 { + compatible = "qcom,qcs8300-qce", "qcom,sm8150-qce", "qcom,qce"; + reg = <0x0 0x01dfa000 0x0 0x6000>; + dmas = <&cryptobam 4>, <&cryptobam 5>; + dma-names = "rx", "tx"; + iommus = <&apps_smmu 0x480 0x0>, + <&apps_smmu 0x481 0x0>; + interconnects = <&aggre2_noc MASTER_CRYPTO_CORE0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "memory"; }; tcsr_mutex: hwlock@1f40000 { @@ -4198,9 +4551,10 @@ qcom,dll-config = <0x000f64ee>; qcom,ddr-config = <0x80040868>; - supports-cqe; dma-coherent; + qcom,ice = <&sdhc_ice>; + status = "disabled"; sdhc1_opp_table: opp-table { @@ -4228,6 +4582,33 @@ }; }; + sdhc_ice: crypto@87c8000 { + compatible = "qcom,qcs8300-inline-crypto-engine", + "qcom,inline-crypto-engine"; + reg = <0x0 0x087c8000 0x0 0x18000>; + clocks = <&gcc GCC_SDCC1_ICE_CORE_CLK>, + <&gcc GCC_SDCC1_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&rpmhpd RPMHPD_CX>; + + operating-points-v2 = <&ice_mmc_opp_table>; + + ice_mmc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + usb_1_hsphy: phy@8904000 { compatible = "qcom,qcs8300-usb-hs-phy", "qcom,usb-snps-hs-7nm-phy"; @@ -4682,7 +5063,14 @@ qcom,select-utmi-as-pipe-clk; wakeup-source; + usb-role-switch; + status = "disabled"; + + port { + usb_2_dwc3_hs: endpoint { + }; + }; }; iris: video-codec@aa00000 { @@ -4769,6 +5157,288 @@ #power-domain-cells = <1>; }; + cci0: cci@ac13000 { + compatible = "qcom,qcs8300-cci", "qcom,msm8996-cci"; + reg = <0x0 0x0ac13000 0x0 0x1000>; + + interrupts = ; + + clocks = <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CCI_0_CLK>; + clock-names = "cpas_ahb", + "cci"; + + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + + pinctrl-0 = <&cci0_i2c0_default &cci0_i2c1_default>; + pinctrl-1 = <&cci0_i2c0_sleep &cci0_i2c1_sleep>; + pinctrl-names = "default", "sleep"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + + cci0_i2c0: i2c-bus@0 { + reg = <0>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + + cci0_i2c1: i2c-bus@1 { + reg = <1>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + cci1: cci@ac14000 { + compatible = "qcom,qcs8300-cci", "qcom,msm8996-cci"; + reg = <0x0 0x0ac14000 0x0 0x1000>; + + interrupts = ; + + clocks = <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CCI_1_CLK>; + clock-names = "cpas_ahb", + "cci"; + + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + + pinctrl-0 = <&cci1_i2c0_default &cci1_i2c1_default>; + pinctrl-1 = <&cci1_i2c0_sleep &cci1_i2c1_sleep>; + pinctrl-names = "default", "sleep"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + + cci1_i2c0: i2c-bus@0 { + reg = <0>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + + cci1_i2c1: i2c-bus@1 { + reg = <1>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + cci2: cci@ac15000 { + compatible = "qcom,qcs8300-cci", "qcom,msm8996-cci"; + reg = <0x0 0x0ac15000 0x0 0x1000>; + + interrupts = ; + + clocks = <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CCI_2_CLK>; + clock-names = "cpas_ahb", + "cci"; + + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + + pinctrl-0 = <&cci2_i2c0_default &cci2_i2c1_default>; + pinctrl-1 = <&cci2_i2c0_sleep &cci2_i2c1_sleep>; + pinctrl-names = "default", "sleep"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + + cci2_i2c0: i2c-bus@0 { + reg = <0>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + + cci2_i2c1: i2c-bus@1 { + reg = <1>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + camss: isp@ac78000 { + compatible = "qcom,qcs8300-camss"; + + reg = <0x0 0xac78000 0x0 0x1000>, + <0x0 0xac7a000 0x0 0xf00>, + <0x0 0xac7c000 0x0 0xf00>, + <0x0 0xac84000 0x0 0xf00>, + <0x0 0xac88000 0x0 0xf00>, + <0x0 0xac8c000 0x0 0xf00>, + <0x0 0xac90000 0x0 0xf00>, + <0x0 0xac94000 0x0 0xf00>, + <0x0 0xac9c000 0x0 0x2000>, + <0x0 0xac9e000 0x0 0x2000>, + <0x0 0xaca0000 0x0 0x2000>, + <0x0 0xacac000 0x0 0x400>, + <0x0 0xacad000 0x0 0x400>, + <0x0 0xacae000 0x0 0x400>, + <0x0 0xac4d000 0x0 0xf000>, + <0x0 0xac60000 0x0 0xf000>, + <0x0 0xac85000 0x0 0xd00>, + <0x0 0xac89000 0x0 0xd00>, + <0x0 0xac8d000 0x0 0xd00>, + <0x0 0xac91000 0x0 0xd00>, + <0x0 0xac95000 0x0 0xd00>; + reg-names = "csid_wrapper", + "csid0", + "csid1", + "csid_lite0", + "csid_lite1", + "csid_lite2", + "csid_lite3", + "csid_lite4", + "csiphy0", + "csiphy1", + "csiphy2", + "tpg0", + "tpg1", + "tpg2", + "vfe0", + "vfe1", + "vfe_lite0", + "vfe_lite1", + "vfe_lite2", + "vfe_lite3", + "vfe_lite4"; + + clocks = <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_0_CLK>, + <&camcc CAM_CC_CPAS_IFE_1_CLK>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_ICP_AHB_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>; + clock-names = "camnoc_axi", + "core_ahb", + "cpas_ahb", + "cpas_fast_ahb_clk", + "cpas_vfe_lite", + "cpas_vfe0", + "cpas_vfe1", + "csid", + "csiphy0", + "csiphy0_timer", + "csiphy1", + "csiphy1_timer", + "csiphy2", + "csiphy2_timer", + "csiphy_rx", + "gcc_axi_hf", + "gcc_axi_sf", + "icp_ahb", + "vfe0", + "vfe0_fast_ahb", + "vfe1", + "vfe1_fast_ahb", + "vfe_lite", + "vfe_lite_ahb", + "vfe_lite_cphy_rx", + "vfe_lite_csid"; + + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "csid0", + "csid1", + "csid_lite0", + "csid_lite1", + "csid_lite2", + "csid_lite3", + "csid_lite4", + "csiphy0", + "csiphy1", + "csiphy2", + "tpg0", + "tpg1", + "tpg2", + "vfe0", + "vfe1", + "vfe_lite0", + "vfe_lite1", + "vfe_lite2", + "vfe_lite3", + "vfe_lite4"; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "ahb", + "hf_0"; + + iommus = <&apps_smmu 0x2400 0x20>; + + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + power-domain-names = "top"; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + }; + + port@2 { + reg = <2>; + }; + }; + }; + camcc: clock-controller@ade0000 { compatible = "qcom,qcs8300-camcc"; reg = <0x0 0x0ade0000 0x0 0x20000>; @@ -4782,41 +5452,354 @@ #power-domain-cells = <1>; }; - dispcc: clock-controller@af00000 { - compatible = "qcom,sa8775p-dispcc0"; - reg = <0x0 0x0af00000 0x0 0x20000>; - clocks = <&gcc GCC_DISP_AHB_CLK>, - <&rpmhcc RPMH_CXO_CLK>, - <&rpmhcc RPMH_CXO_CLK_A>, - <&sleep_clk>, - <0>, <0>, <0>, <0>, - <0>, <0>, <0>, <0>; - power-domains = <&rpmhpd RPMHPD_MMCX>; - #clock-cells = <1>; - #reset-cells = <1>; - #power-domain-cells = <1>; - }; + mdss: display-subsystem@ae00000 { + compatible = "qcom,qcs8300-mdss"; + reg = <0x0 0x0ae00000 0x0 0x1000>; + reg-names = "mdss"; + + interrupts = ; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_MDP_CLK>; + + resets = <&dispcc MDSS_DISP_CC_MDSS_CORE_BCR>; + + interconnects = <&mmss_noc MASTER_MDP0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_MDP1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "mdp0-mem", + "mdp1-mem", + "cpu-cfg"; + + power-domains = <&dispcc MDSS_DISP_CC_MDSS_CORE_GDSC>; + + iommus = <&apps_smmu 0x1000 0x402>; - pdc: interrupt-controller@b220000 { - compatible = "qcom,qcs8300-pdc", "qcom,pdc"; - reg = <0x0 0xb220000 0x0 0x30000>, - <0x0 0x17c000f0 0x0 0x64>; - interrupt-parent = <&intc>; - #interrupt-cells = <2>; interrupt-controller; - qcom,pdc-ranges = <0 480 40>, - <40 140 14>, - <54 263 1>, - <55 306 4>, - <59 312 3>, - <62 374 2>, - <64 434 2>, - <66 438 2>, - <70 520 1>, - <73 523 1>, - <118 568 6>, - <124 609 3>, - <159 638 1>, + #interrupt-cells = <1>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + status = "disabled"; + + mdss_mdp: display-controller@ae01000 { + compatible = "qcom,qcs8300-dpu", "qcom,sa8775p-dpu"; + reg = <0x0 0x0ae01000 0x0 0x8f000>, + <0x0 0x0aeb0000 0x0 0x2008>; + reg-names = "mdp", "vbif"; + + interrupts-extended = <&mdss 0>; + + clocks = <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_MDP_LUT_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_MDP_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_VSYNC_CLK>; + clock-names = "nrt_bus", + "iface", + "lut", + "core", + "vsync"; + + assigned-clocks = <&dispcc MDSS_DISP_CC_MDSS_VSYNC_CLK>; + assigned-clock-rates = <19200000>; + + operating-points-v2 = <&mdp_opp_table>; + power-domains = <&rpmhpd RPMHPD_MMCX>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dpu_intf0_out: endpoint { + remote-endpoint = <&mdss_dp0_in>; + }; + }; + + port@1 { + reg = <1>; + dpu_intf1_out: endpoint { + remote-endpoint = <&mdss_dsi0_in>; + }; + }; + }; + + mdp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-375000000 { + opp-hz = /bits/ 64 <375000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-575000000 { + opp-hz = /bits/ 64 <575000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + + opp-650000000 { + opp-hz = /bits/ 64 <650000000>; + required-opps = <&rpmhpd_opp_turbo_l1>; + }; + }; + }; + + mdss_dsi0: dsi@ae94000 { + compatible = "qcom,qcs8300-dsi-ctrl", + "qcom,sa8775p-dsi-ctrl", + "qcom,mdss-dsi-ctrl"; + reg = <0x0 0x0ae94000 0x0 0x400>; + reg-names = "dsi_ctrl"; + + interrupt-parent = <&mdss>; + interrupts = <4>; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_BYTE0_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_BYTE0_INTF_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_PCLK0_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_ESC0_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP_HF_AXI_CLK>; + clock-names = "byte", + "byte_intf", + "pixel", + "core", + "iface", + "bus"; + + assigned-clocks = <&dispcc MDSS_DISP_CC_MDSS_BYTE0_CLK_SRC>, + <&dispcc MDSS_DISP_CC_MDSS_PCLK0_CLK_SRC>; + assigned-clock-parents = <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>, + <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>; + + phys = <&mdss_dsi0_phy>; + + operating-points-v2 = <&mdss_dsi_opp_table>; + power-domains = <&rpmhpd RPMHPD_MMCX>; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + mdss_dsi0_in: endpoint { + remote-endpoint = <&dpu_intf1_out>; + }; + }; + + port@1 { + reg = <1>; + mdss_dsi0_out: endpoint { + }; + }; + }; + + mdss_dsi_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-358000000 { + opp-hz = /bits/ 64 <358000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + + mdss_dsi0_phy: phy@ae94400 { + compatible = "qcom,qcs8300-dsi-phy-5nm", + "qcom,sa8775p-dsi-phy-5nm"; + reg = <0x0 0x0ae94400 0x0 0x200>, + <0x0 0x0ae94600 0x0 0x280>, + <0x0 0x0ae94900 0x0 0x27c>; + reg-names = "dsi_phy", + "dsi_phy_lane", + "dsi_pll"; + + #clock-cells = <1>; + #phy-cells = <0>; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>, + <&rpmhcc RPMH_CXO_CLK>; + clock-names = "iface", "ref"; + + status = "disabled"; + }; + + mdss_dp0_phy: phy@aec2a00 { + compatible = "qcom,qcs8300-edp-phy", "qcom,sa8775p-edp-phy"; + + reg = <0x0 0x0aec2a00 0x0 0x19c>, + <0x0 0x0aec2200 0x0 0xec>, + <0x0 0x0aec2600 0x0 0xec>, + <0x0 0x0aec2000 0x0 0x1c8>; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>; + clock-names = "aux", + "cfg_ahb"; + + power-domains = <&rpmhpd RPMHPD_MX>; + + #clock-cells = <1>; + #phy-cells = <0>; + + status = "disabled"; + }; + + mdss_dp0: displayport-controller@af54000 { + compatible = "qcom,qcs8300-dp", "qcom,sa8775p-dp"; + + reg = <0x0 0x0af54000 0x0 0x200>, + <0x0 0x0af54200 0x0 0x200>, + <0x0 0x0af55000 0x0 0xc00>, + <0x0 0x0af56000 0x0 0x09c>, + <0x0 0x0af57000 0x0 0x09c>, + <0x0 0x0af58000 0x0 0x09c>, + <0x0 0x0af59000 0x0 0x09c>, + <0x0 0x0af5a000 0x0 0x23c>, + <0x0 0x0af5b000 0x0 0x23c>; + + interrupts-extended = <&mdss 12>; + + clocks = <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_LINK_INTF_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK>; + clock-names = "core_iface", + "core_aux", + "ctrl_link", + "ctrl_link_iface", + "stream_pixel", + "stream_1_pixel", + "stream_2_pixel", + "stream_3_pixel"; + assigned-clocks = <&dispcc MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK_SRC>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK_SRC>, + <&dispcc MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK_SRC>; + assigned-clock-parents = <&mdss_dp0_phy 0>, + <&mdss_dp0_phy 1>, + <&mdss_dp0_phy 1>, + <&mdss_dp0_phy 1>, + <&mdss_dp0_phy 1>; + phys = <&mdss_dp0_phy>; + phy-names = "dp"; + + operating-points-v2 = <&dp_opp_table>; + power-domains = <&rpmhpd RPMHPD_MMCX>; + + #sound-dai-cells = <0>; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mdss_dp0_in: endpoint { + remote-endpoint = <&dpu_intf0_out>; + }; + }; + + port@1 { + reg = <1>; + + mdss_dp0_out: endpoint { }; + }; + }; + + dp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-270000000 { + opp-hz = /bits/ 64 <270000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-810000000 { + opp-hz = /bits/ 64 <810000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + }; + + dispcc: clock-controller@af00000 { + compatible = "qcom,sa8775p-dispcc0"; + reg = <0x0 0x0af00000 0x0 0x20000>; + clocks = <&gcc GCC_DISP_AHB_CLK>, + <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&sleep_clk>, + <&mdss_dp0_phy 0>, + <&mdss_dp0_phy 1>, + <0>, <0>, + <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>, + <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>, + <0>, <0>; + power-domains = <&rpmhpd RPMHPD_MMCX>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; + + pdc: interrupt-controller@b220000 { + compatible = "qcom,qcs8300-pdc", "qcom,pdc"; + reg = <0x0 0xb220000 0x0 0x30000>, + <0x0 0x17c000f0 0x0 0x64>; + interrupt-parent = <&intc>; + #interrupt-cells = <2>; + interrupt-controller; + qcom,pdc-ranges = <0 480 40>, + <40 140 14>, + <54 263 1>, + <55 306 4>, + <59 312 3>, + <62 374 2>, + <64 434 2>, + <66 438 2>, + <70 520 1>, + <73 523 1>, + <118 568 6>, + <124 609 3>, + <159 638 1>, <160 720 3>, <169 728 30>, <199 416 2>, @@ -4892,6 +5875,204 @@ #interrupt-cells = <2>; wakeup-parent = <&pdc>; + cci0_i2c0_default: cci0-0-default-state { + sda-pins { + pins = "gpio57"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + pins = "gpio58"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + cci0_i2c0_sleep: cci0-0-sleep-state { + sda-pins { + pins = "gpio57"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-down; + }; + + scl-pins { + pins = "gpio58"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + cci0_i2c1_default: cci0-1-default-state { + sda-pins { + pins = "gpio29"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + pins = "gpio30"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + cci0_i2c1_sleep: cci0-1-sleep-state { + sda-pins { + pins = "gpio29"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-down; + }; + + scl-pins { + pins = "gpio30"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + cci1_i2c0_default: cci1-0-default-state { + sda-pins { + pins = "gpio59"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + pins = "gpio60"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + cci1_i2c0_sleep: cci1-0-sleep-state { + sda-pins { + pins = "gpio59"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-down; + }; + + scl-pins { + pins = "gpio60"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + cci1_i2c1_default: cci1-1-default-state { + sda-pins { + pins = "gpio31"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + pins = "gpio32"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + cci1_i2c1_sleep: cci1-1-sleep-state { + sda-pins { + pins = "gpio31"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-down; + }; + + scl-pins { + pins = "gpio32"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + cci2_i2c0_default: cci2-0-default-state { + sda-pins { + pins = "gpio61"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + pins = "gpio62"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + cci2_i2c0_sleep: cci2-0-sleep-state { + sda-pins { + pins = "gpio61"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-down; + }; + + scl-pins { + pins = "gpio62"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + cci2_i2c1_default: cci2-1-default-state { + sda-pins { + pins = "gpio54"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + + scl-pins { + pins = "gpio55"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-up = <2200>; + }; + }; + + cci2_i2c1_sleep: cci2-1-sleep-state { + sda-pins { + pins = "gpio54"; + function = "cci_i2c_sda"; + drive-strength = <2>; + bias-pull-down; + }; + + scl-pins { + pins = "gpio55"; + function = "cci_i2c_scl"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + dp_hot_plug_det: dp-hot-plug-det-state { + pins = "gpio94"; + function = "edp0_hot"; + bias-disable; + }; + hs0_mi2s_active: hs0-mi2s-active-state { pins = "gpio106", "gpio107", "gpio108", "gpio109"; function = "hs0_mi2s"; @@ -5430,12 +6611,12 @@ }; qup_uart10_rts: qup-uart10-rts-state { - pins = "gpio84"; + pins = "gpio85"; function = "qup1_se2"; }; qup_uart10_tx: qup-uart10-tx-state { - pins = "gpio85"; + pins = "gpio86"; function = "qup1_se2"; }; @@ -6078,16 +7259,45 @@ label = "gpdsp"; qcom,remote-pid = <17>; - }; - }; - ethernet0: ethernet@23040000 { - compatible = "qcom,qcs8300-ethqos", "qcom,sa8775p-ethqos"; - reg = <0x0 0x23040000 0x0 0x00010000>, - <0x0 0x23056000 0x0 0x00000100>; - reg-names = "stmmaceth", "rgmii"; + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "gdsp0"; + #address-cells = <1>; + #size-cells = <0>; - interrupts = , + compute-cb@1 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <1>; + iommus = <&apps_smmu 0x28a1 0x0>; + dma-coherent; + }; + + compute-cb@2 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <2>; + iommus = <&apps_smmu 0x28a2 0x0>; + dma-coherent; + }; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x28a3 0x0>; + dma-coherent; + }; + }; + }; + }; + + ethernet0: ethernet@23040000 { + compatible = "qcom,qcs8300-ethqos", "qcom,sa8775p-ethqos"; + reg = <0x0 0x23040000 0x0 0x00010000>, + <0x0 0x23056000 0x0 0x00000100>; + reg-names = "stmmaceth", "rgmii"; + + interrupts = , ; interrupt-names = "macirq", "sfty"; @@ -6210,6 +7420,650 @@ }; }; }; + + cooling { + compatible = "qcom,qmi-cooling-cdsp"; + cdsp_sw: cdsp_sw { + label = "cdsp_sw"; + #cooling-cells = <2>; + }; + }; + }; + + tsens2: thermal-sensor@c251000 { + compatible = "qcom,qcs8300-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c251000 0x0 0x1000>, + <0x0 0x0c224000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", "critical"; + #qcom,sensors = <10>; + #thermal-sensor-cells = <1>; + }; + + tsens3: thermal-sensor@c252000 { + compatible = "qcom,qcs8300-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c252000 0x0 0x1000>, + <0x0 0x0c225000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", "critical"; + #qcom,sensors = <10>; + #thermal-sensor-cells = <1>; + }; + + tsens0: thermal-sensor@c263000 { + compatible = "qcom,qcs8300-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c263000 0x0 0x1000>, + <0x0 0x0c222000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", "critical"; + #qcom,sensors = <10>; + #thermal-sensor-cells = <1>; + }; + + tsens1: thermal-sensor@c265000 { + compatible = "qcom,qcs8300-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c265000 0x0 0x1000>, + <0x0 0x0c223000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", "critical"; + #qcom,sensors = <10>; + #thermal-sensor-cells = <1>; + }; + }; + + thermal_zones: thermal-zones { + aoss-0-thermal { + thermal-sensors = <&tsens0 0>; + + trips { + aoss0-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-0-0-thermal { + thermal-sensors = <&tsens0 1>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-1-0-thermal { + thermal-sensors = <&tsens0 2>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-2-0-thermal { + thermal-sensors = <&tsens0 3>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-3-0-thermal { + thermal-sensors = <&tsens0 4>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + gpuss-0-thermal { + thermal-sensors = <&tsens0 5>; + + trips { + gpuss0_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + gpuss0-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&gpuss0_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + audio-thermal { + thermal-sensors = <&tsens0 6>; + + trips { + audio-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + camss-0-thermal { + thermal-sensors = <&tsens0 7>; + + trips { + camss-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + pcie-0-thermal { + thermal-sensors = <&tsens0 8>; + + trips { + pcie-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpuss-0-0-thermal { + thermal-sensors = <&tsens0 9>; + + trips { + cpuss0-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + aoss-1-thermal { + thermal-sensors = <&tsens1 0>; + + trips { + aoss1-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-0-1-thermal { + thermal-sensors = <&tsens1 1>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-1-1-thermal { + thermal-sensors = <&tsens1 2>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-2-1-thermal { + thermal-sensors = <&tsens1 3>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-0-3-1-thermal { + thermal-sensors = <&tsens1 4>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + gpuss-1-thermal { + thermal-sensors = <&tsens1 5>; + + trips { + gpuss1_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + gpuss1-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&gpuss1_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + video-thermal { + thermal-sensors = <&tsens1 6>; + + trips { + video-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + camss-1-thermal { + thermal-sensors = <&tsens1 7>; + + trips { + camss1-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + pcie-1-thermal { + thermal-sensors = <&tsens1 8>; + + trips { + pcie-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpuss-0-1-thermal { + thermal-sensors = <&tsens1 9>; + + trips { + cpuss0-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + aoss-2-thermal { + thermal-sensors = <&tsens2 0>; + + trips { + aoss2-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-0-0-thermal { + thermal-sensors = <&tsens2 1>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-1-0-thermal { + thermal-sensors = <&tsens2 2>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-2-0-thermal { + thermal-sensors = <&tsens2 3>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-3-0-thermal { + thermal-sensors = <&tsens2 4>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + nsp-0-0-0-thermal { + thermal-sensors = <&tsens2 5>; + + trips { + nsp_0_0_0_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + nsp-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&nsp_0_0_0_alert0>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + nsp-0-1-0-thermal { + thermal-sensors = <&tsens2 6>; + + trips { + nsp_0_1_0_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + nsp-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&nsp_0_1_0_alert0>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + nsp-0-2-0-thermal { + thermal-sensors = <&tsens2 7>; + + trips { + nsp_0_2_0_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + nsp-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&nsp_0_2_0_alert0>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + ddrss-0-thermal { + thermal-sensors = <&tsens2 8>; + + trips { + ddrss-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpuss-1-0-thermal { + thermal-sensors = <&tsens2 9>; + + trips { + cpuss1-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + aoss-3-thermal { + thermal-sensors = <&tsens3 0>; + + trips { + aoss3-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-0-1-thermal { + thermal-sensors = <&tsens3 1>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-1-1-thermal { + thermal-sensors = <&tsens3 2>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-2-1-thermal { + thermal-sensors = <&tsens3 3>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpu-1-3-1-thermal { + thermal-sensors = <&tsens3 4>; + + trips { + cpu-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + nsp-0-0-1-thermal { + thermal-sensors = <&tsens3 5>; + + trips { + nsp_0_0_1_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + nsp-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&nsp_0_0_1_alert0>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + nsp-0-1-1-thermal { + thermal-sensors = <&tsens3 6>; + + trips { + nsp_0_1_1_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + nsp-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&nsp_0_1_1_alert0>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + nsp-0-2-1-thermal { + thermal-sensors = <&tsens3 7>; + + trips { + nsp_0_2_1_alert0: trip-point0 { + temperature = <115000>; + hysteresis = <5000>; + type = "passive"; + }; + + nsp-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&nsp_0_2_1_alert0>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + ddrss-1-thermal { + thermal-sensors = <&tsens3 8>; + + trips { + ddrss-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + }; + + cpuss-1-1-thermal { + thermal-sensors = <&tsens3 9>; + + trips { + cpuss1-critical { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi index 4c983b10dd925..7ace3540ef0a0 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi @@ -378,7 +378,7 @@ status = "okay"; sideinteraction: touch@2c { - compatible = "ad,ad7147_captouch"; + compatible = "adi,ad7147_captouch"; reg = <0x2c>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi index c75b522f6eba6..33608b1d7d060 100644 --- a/arch/arm64/boot/dts/qcom/msm8996.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi @@ -3496,6 +3496,9 @@ <&gcc GCC_USB20_MASTER_CLK>; assigned-clock-rates = <19200000>, <60000000>; + interconnects = <&pnoc MASTER_USB_HS &bimc SLAVE_EBI_CH0>, + <&bimc MASTER_AMPSS_M0 &pnoc SLAVE_USB_HS>; + interconnect-names = "usb-ddr", "apps-usb"; power-domains = <&gcc USB30_GDSC>; qcom,select-utmi-as-pipe-clk; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/pm7250b.dtsi b/arch/arm64/boot/dts/qcom/pm7250b.dtsi index 0761e6b5fd8d1..dffd84dd87a85 100644 --- a/arch/arm64/boot/dts/qcom/pm7250b.dtsi +++ b/arch/arm64/boot/dts/qcom/pm7250b.dtsi @@ -202,6 +202,16 @@ interrupt-controller; #interrupt-cells = <2>; }; + + sensor@1d00 { + compatible = "qcom,pm7250b-bcl", "qcom,bcl-v1"; + reg = <0x1d00>; + interrupts = , + ; + interrupt-names = "bcl-max-min", + "bcl-critical"; + overcurrent-thresholds-milliamp = <5500 6000>; + }; }; pmic@PM7250B_SID1 { diff --git a/arch/arm64/boot/dts/qcom/pm8350c.dtsi b/arch/arm64/boot/dts/qcom/pm8350c.dtsi index 1a24e6439e36d..f0cf55a7fc9e5 100644 --- a/arch/arm64/boot/dts/qcom/pm8350c.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8350c.dtsi @@ -41,6 +41,15 @@ #pwm-cells = <2>; status = "disabled"; }; + + sensor@4700 { + compatible = "qcom,pm8350c-bcl", "qcom,bcl-v2"; + reg = <0x4700>; + interrupts = <0x2 0x47 0x0 IRQ_TYPE_EDGE_RISING>, + <0x2 0x47 0x1 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "bcl-max-min", + "bcl-critical"; + }; }; }; diff --git a/arch/arm64/boot/dts/qcom/pmcx0102.dtsi b/arch/arm64/boot/dts/qcom/pmcx0102.dtsi new file mode 100644 index 0000000000000..839738dda6cd5 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/pmcx0102.dtsi @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +&spmi_bus0 { + pmcx0102_c_e0: pmic@2 { + compatible = "qcom,pmcx0102", "qcom,spmi-pmic"; + reg = <0x2 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmcx0102_c_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x2 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmcx0102_c_e0_gpios: gpio@8800 { + compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmcx0102_c_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; + + pmcx0102_d_e0: pmic@3 { + compatible = "qcom,pmcx0102", "qcom,spmi-pmic"; + reg = <0x3 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmcx0102_d_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x3 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmcx0102_d_e0_gpios: gpio@8800 { + compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmcx0102_d_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; + + pmcx0102_e_e0: pmic@4 { + compatible = "qcom,pmcx0102", "qcom,spmi-pmic"; + reg = <0x4 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmcx0102_e_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x4 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmcx0102_e_e0_gpios: gpio@8800 { + compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmcx0102_e_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; + + pmcx0102_g_e0: pmic@5 { + compatible = "qcom,pmcx0102", "qcom,spmi-pmic"; + reg = <0x6 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmcx0102_g_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x6 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmcx0102_g_e0_gpios: gpio@8800 { + compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmcx0102_g_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +}; + +&spmi_bus1 { + pmcx0102_c_e1: pmic@2 { + compatible = "qcom,pmcx0102", "qcom,spmi-pmic"; + reg = <0x2 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmcx0102_c_e1_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x2 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmcx0102_c_e1_gpios: gpio@8800 { + compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmcx0102_c_e1_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; + + pmcx0102_d_e1: pmic@3 { + compatible = "qcom,pmcx0102", "qcom,spmi-pmic"; + reg = <0x3 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmcx0102_d_e1_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x3 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmcx0102_d_e1_gpios: gpio@8800 { + compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmcx0102_d_e1_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; + + pmcx0102_e_e1: pmic@4 { + compatible = "qcom,pmcx0102", "qcom,spmi-pmic"; + reg = <0x4 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmcx0102_e_e1_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x4 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmcx0102_e_e1_gpios: gpio@8800 { + compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmcx0102_e_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/pmh0101.dtsi b/arch/arm64/boot/dts/qcom/pmh0101.dtsi new file mode 100644 index 0000000000000..831c79305f7a8 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/pmh0101.dtsi @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +&spmi_bus0 { + pmic@1 { + compatible = "qcom,pmh0101", "qcom,spmi-pmic"; + reg = <0x1 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmh0101_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = <0x1 0xa 0x0 IRQ_TYPE_EDGE_BOTH>; + #thermal-sensor-cells = <0>; + }; + + pmh0101_gpios: gpio@8800 { + compatible = "qcom,pmh0101-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0101_gpios 0 0 18>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + pmh0101_flash: led-controller@ee00 { + compatible = "qcom,pmh0101-flash-led", "qcom,spmi-flash-led"; + reg = <0xee00>; + status = "disabled"; + }; + + pmh0101_pwm: pwm { + compatible = "qcom,pmh0101-pwm", "qcom,pm8350c-pwm"; + #pwm-cells = <2>; + status = "disabled"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/pmh0104.dtsi b/arch/arm64/boot/dts/qcom/pmh0104.dtsi new file mode 100644 index 0000000000000..d3ea7486d842e --- /dev/null +++ b/arch/arm64/boot/dts/qcom/pmh0104.dtsi @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +&spmi_bus0 { +#ifdef PMH0104_I_E0_SID + pmh0104_i_e0: pmic@PMH0104_I_E0_SID { + compatible = "qcom,pmh0104", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pmh0104_i_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0104_i_e0_gpios: gpio@8800 { + compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0104_i_e0_gpios 0 0 8>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif + +#ifdef PMH0104_J_E0_SID + pmh0104_j_e0: pmic@PMH0104_J_E0_SID { + compatible = "qcom,pmh0104", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + + pmh0104_j_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0104_j_e0_gpios: gpio@8800 { + compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0104_j_e0_gpios 0 0 8>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif +}; + +&spmi_bus1 { +#ifdef PMH0104_J_E1_SID + pmh0104_j_e1: pmic@PMH0104_J_E1_SID { + compatible = "qcom,pmh0104", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + + pmh0104_j_e1_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0104_j_e1_gpios: gpio@8800 { + compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0104_j_e1_gpios 0 0 8>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif + +#ifdef PMH0104_L_E1_SID + pmh0104_l_e1: pmic@PMH0104_L_E1_SID { + compatible = "qcom,pmh0104", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pmh0104_l_e1_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0104_l_e1_gpios: gpio@8800 { + compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0104_l_e1_gpios 0 0 8>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif +}; diff --git a/arch/arm64/boot/dts/qcom/pmh0110.dtsi b/arch/arm64/boot/dts/qcom/pmh0110.dtsi new file mode 100644 index 0000000000000..4a5c66e5c9fbc --- /dev/null +++ b/arch/arm64/boot/dts/qcom/pmh0110.dtsi @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +&spmi_bus0 { + +#ifdef PMH0110_D_E0_SID + pmh0110_d_e0: pmic@PMH0110_D_E0_SID { + compatible = "qcom,pmh0110", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pmh0110_d_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0110_d_e0_gpios: gpio@8800 { + compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0110_d_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif + +#ifdef PMH0110_F_E0_SID + pmh0110_f_e0: pmic@PMH0110_F_E0_SID { + compatible = "qcom,pmh0110", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + + pmh0110_f_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0110_f_e0_gpios: gpio@8800 { + compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0110_f_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif + +#ifdef PMH0110_G_E0_SID + pmh0110_g_e0: pmic@PMH0110_G_E0_SID { + compatible = "qcom,pmh0110", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pmh0110_g_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0110_g_e0_gpios: gpio@8800 { + compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0110_g_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif + +#ifdef PMH0110_H_E0_SID + pmh0110_h_e0: pmic@PMH0110_H_E0_SID { + compatible = "qcom,pmh0110", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pmh0110_h_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0110_h_e0_gpios: gpio@8800 { + compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0110_h_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif + +#ifdef PMH0110_I_E0_SID + pmh0110_i_e0: pmic@PMH0110_I_E0_SID { + compatible = "qcom,pmh0110", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pmh0110_i_e0_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0110_i_e0_gpios: gpio@8800 { + compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0110_i_e0_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif +}; + +&spmi_bus1 { +#ifdef PMH0110_F_E1_SID + pmh0110_f_e1: pmic@PMH0110_F_E1_SID { + compatible = "qcom,pmh0110", "qcom,spmi-pmic"; + reg = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pmh0110_f_e1_temp_alarm: temp-alarm@a00 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0xa00>; + interrupts = ; + #thermal-sensor-cells = <0>; + }; + + pmh0110_f_e1_gpios: gpio@8800 { + compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio"; + reg = <0x8800>; + gpio-controller; + gpio-ranges = <&pmh0110_f_e1_gpios 0 0 14>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +#endif +}; diff --git a/arch/arm64/boot/dts/qcom/pmk8850.dtsi b/arch/arm64/boot/dts/qcom/pmk8850.dtsi new file mode 100644 index 0000000000000..c19a98ca984b2 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/pmk8850.dtsi @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include + +&spmi_bus0 { + pmic@0 { + compatible = "qcom,pmk8850", "qcom,spmi-pmic"; + reg = <0x0 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pmk8850_pon: pon@1300 { + compatible = "qcom,pmk8350-pon"; + reg = <0x1300>, <0x800>; + reg-names = "hlos", "pbs"; + + pon_pwrkey: pwrkey { + compatible = "qcom,pmk8350-pwrkey"; + interrupts = <0x0 0x13 0x7 IRQ_TYPE_EDGE_BOTH>; + linux,code = ; + }; + + pon_resin: resin { + compatible = "qcom,pmk8350-resin"; + interrupts = <0x0 0x13 0x6 IRQ_TYPE_EDGE_BOTH>; + status = "disabled"; + }; + }; + + pmk8850_gpios: gpio@b800 { + compatible = "qcom,pmk8850-gpio", "qcom,spmi-gpio"; + reg = <0xb800>; + gpio-controller; + gpio-ranges = <&pmk8850_gpios 0 0 8>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + pmk8850_rtc: rtc@6100 { + compatible = "qcom,pmk8350-rtc"; + reg = <0x6100>, <0x6200>; + reg-names = "rtc", "alarm"; + interrupts = <0x0 0x62 0x1 IRQ_TYPE_EDGE_RISING>; + }; + + pmk8850_sdam_2: nvram@7100 { + compatible = "qcom,spmi-sdam"; + reg = <0x7100>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x7100 0x100>; + + reboot_reason: reboot-reason@48 { + reg = <0x48 0x1>; + bits = <1 7>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/purwa-camera.dtsi b/arch/arm64/boot/dts/qcom/purwa-camera.dtsi new file mode 100644 index 0000000000000..5896e5759e22f --- /dev/null +++ b/arch/arm64/boot/dts/qcom/purwa-camera.dtsi @@ -0,0 +1,2056 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +&soc { + cam_icp: qcom,icp@ac01000 { + compatible = "qcom,cam-icp_v2"; + icp-version = <0x0200>; + reg = <0x0 0xac01000 0x0 0x400>, + <0x0 0xac01800 0x0 0x400>, + <0x0 0x0ac04000 0x0 0x1000>; + reg-names = "icp_csr", "icp_cirq", "icp_wd0"; + reg-cam-base = <0x1000 0x1800 0x4000>; + interrupts = ; + interrupt-names = "icp"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + memory-region = <&camera_mem>; + clocks = <&camcc CAM_CC_ICP_AHB_CLK>, + <&camcc CAM_CC_ICP_CLK_SRC>, + <&camcc CAM_CC_ICP_CLK>; + clock-names = "icp_ahb_clk", + "icp_clk_src", + "icp_clk"; + clock-rates = <0 300000000 0>, + <0 400000000 0>, + <0 480000000 0>, + <0 600000000 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "nominal"; + nrt-device; + src-clock-name = "icp_clk_src"; + operating-points-v2 = <&icp_opp_table>; + clock-control-debugfs = "true"; + fw_name = "qcom/x1p4x100/CAMERA_ICP"; + ubwc-ipe-fetch-cfg = <0x707b 0x7083>; + ubwc-ipe-write-cfg = <0x161ef 0x1620f>; + ubwc-bps-fetch-cfg = <0x707b 0x7083>; + ubwc-bps-write-cfg = <0x161ef 0x1620f>; + qos-val = <0x00000A0A>; + cam_hw_pid = <3>; + cell-index = <0>; + status = "okay"; + + icp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_cpas: qcom,cam-cpas@ac13000 { + compatible = "qcom,cam-cpas"; + label = "cpas"; + arch-compat = "cpas_top"; + reg = <0x0 0xac13000 0x0 0x1000>, + <0x0 0xac19000 0x0 0xC000>, + <0x0 0xbbf0000 0x0 0x1F00>; + reg-names = "cam_cpas_top", "cam_camnoc", "cam_rpmh"; + reg-cam-base = <0x13000 0x19000 0x0bbf0000>; + interrupts = ; + interrupt-names = "cpas_camnoc"; + camnoc-axi-min-ib-bw = <3000000000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_RT_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_NRT_CLK>, + <&camcc CAM_CC_QDSS_DEBUG_XO_CLK>; + clock-names = "gcc_ahb_clk", + "gcc_axi_hf_clk", + "gcc_axi_sf_clk", + "cam_cc_slow_ahb_clk_src", + "cpas_ahb_clk", + "cpas_core_ahb_clk", + "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_fast_ahb_clk", + "camnoc_axi_clk_src", + "camnoc_axi_clk", + "camnoc_axi_nrt_clk", + "camcc_debug_clk"; + clock-rates = < 0 0 0 0 0 0 0 0 0 0 0 0>, + < 0 0 0 64000000 0 0 80000000 0 240000000 0 0 0>, + < 0 0 0 80000000 0 0 100000000 0 300000000 0 0 0>, + < 0 0 0 80000000 0 0 400000000 0 400000000 0 0 0>; + clock-cntl-level = "suspend", "lowsvsd1", "lowsvs", "nominal"; + clock-names-option = "cam_icp_clk"; + clocks-option = <&camcc CAM_CC_ICP_CLK>; + clock-rates-option = <400000000>; + src-clock-name = "camnoc_axi_clk_src"; + operating-points-v2 = <&cpas_opp_table>; + control-camnoc-axi-clk; + camnoc-bus-width = <32>; + cam-icc-path-names = "cam_ahb"; + camnoc-axi-clk-bw-margin-perc = <20>; + interconnects =<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_CAMERA_CFG 0>, + <&mmss_noc MASTER_CAMNOC_HF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_SF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_ICP 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "cam_ahb", + "cam_hf_0", + "cam_sf_0", + "cam_sf_icp"; + rpmh-bcm-info = <12 0x4 0x800 0 4>; + cam-ahb-num-cases = <8>; + cam-ahb-bw-KBps = <0 0>, <0 76800>, <0 76800>, <0 150000>, + <0 150000>, <0 300000>, <0 300000>, <0 300000>; + vdd-corners = ; + vdd-corner-ahb-mapping = "suspend", "lowsvs", + "lowsvs", "svs", "svs_l1", + "nominal", "nominal", "nominal", + "turbo", "turbo"; + client-id-based; + client-names = "csiphy0", "csiphy4", "cci0", "cci1", "csid0", + "csid1", "csid2", "ife0", "ife1", "ife2", "ipe0", + "rt-cdm0", "rt-cdm1", "rt-cdm2", "rt-cdm3", + "cam-cdm-intf0", "bps0", "icp0", "jpeg-dma0", + "jpeg-enc0", "tpg13", "tpg14", "tpg15"; + cell-index = <0>; + status = "okay"; + + camera-bus-nodes { + level0-nodes { + level-index = <0>; + + bps0_all_rd: bps0-all-rd { + cell-index = <0>; + node-name = "bps0-all-rd"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_rd>; + }; + + bps0_all_wr: bps0-all-wr { + cell-index = <1>; + node-name = "bps0-all-wr"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_wr>; + }; + + icp0_all_rd: icp0-all-rd { + cell-index = <2>; + node-name = "icp0-all-rd"; + client-name = "icp0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt1_rd>; + }; + + ife0_linear_stats_wr: ife0-linear-stats-wr { + cell-index = <3>; + node-name = "ife0-linear-stats-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr3>; + }; + + ife0_pdaf_wr: ife0-pdaf-wr { + cell-index = <4>; + node-name = "ife0-pdaf-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_rt0_wr2>; + }; + + ife0_rdi_pixel_raw_wr: ife0-rdi-pixel-raw-wr { + cell-index = <5>; + node-name = "ife0-rdi-pixel-raw-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr1>; + }; + + ife0_ubwc_wr: ife0-ubwc-wr { + cell-index = <6>; + node-name = "ife0-ubwc-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_wr0>; + }; + + ipe0_all_wr: ipe0-all-wr { + cell-index = <7>; + node-name = "ipe0-all-wr"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level2_nrt0_wr>; + }; + + ipe0_in_rd: ipe0-in-rd { + cell-index = <8>; + node-name = "ipe0-in-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_rd>; + }; + + ipe0_ref_rd: ipe0-ref-rd { + cell-index = <9>; + node-name = "ipe0-ref-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_rd>; + }; + + jpeg_dma0_all_rd: jpeg-dma0-all-rd { + cell-index = <10>; + node-name = "jpeg-dma0-all-rd"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd0>; + }; + + jpeg_dma0_all_wr: jpeg-dma0-all-wr { + cell-index = <11>; + node-name = "jpeg-dma0-all-wr"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_wr0>; + }; + + jpeg_enc0_all_rd: jpeg-enc0-all-rd { + cell-index = <12>; + node-name = "jpeg-enc0-all-rd"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd0>; + }; + + jpeg_enc0_all_wr: jpeg-enc0-all-wr { + cell-index = <13>; + node-name = "jpeg-enc0-all-wr"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_wr0>; + }; + + rt_cdm0_all_rd: rt-cdm0-all-rd { + cell-index = <14>; + node-name = "rt-cdm0-all-rd"; + client-name = "rt-cdm0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd1>; + }; + + /* IFE Lite 0 */ + rt_cdm2_all_rd: rt-cdm2-all-rd { + cell-index = <15>; + node-name = "rt-cdm2-all-rd"; + client-name = "rt-cdm2"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd1>; + }; + + /* IFE Lite 1 */ + rt_cdm3_all_rd: rt-cdm3-all-rd { + cell-index = <16>; + node-name = "rt-cdm3-all-rd"; + client-name = "rt-cdm3"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_rd1>; + }; + }; + + level1-nodes { + level-index = <1>; + camnoc-max-needed; + + level1_nrt0_rd0: level1-nrt0-rd0 { + cell-index = <17>; + node-name = "level1-nrt0-rd0"; + parent-node = <&level2_nrt0_rd>; + traffic-merge-type = ; + }; + + level1_nrt0_rd1: level1-nrt0-rd1 { + cell-index = <18>; + node-name = "level1-nrt0-rd1"; + parent-node = <&level2_nrt0_rd>; + traffic-merge-type = ; + }; + + level1_nrt0_wr0: level1-nrt0-wr0 { + cell-index = <19>; + node-name = "level1-nrt0-wr0"; + parent-node = <&level2_nrt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr0: level1-rt0-wr0 { + cell-index = <20>; + node-name = "level1-ife-ubwc-wr"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr1: level1-rt0-wr1 { + cell-index = <21>; + node-name = "level1-ife-rdi-wr"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr2: level1-rt0-wr2 { + cell-index = <22>; + node-name = "level1-ife-pdaf"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr3: level1-rt0-wr3 { + cell-index = <23>; + node-name = "level1-ife01-linear-stats"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + + level1_rt0_wr4: level1-rt0-wr4 { + cell-index = <24>; + node-name = "level1-ifelite"; + parent-node = <&level2_rt0_wr>; + traffic-merge-type = ; + }; + }; + + level2-nodes { + level-index = <2>; + camnoc-max-needed; + + level2_nrt0_rd: level2-nrt0-rd { + cell-index = <25>; + node-name = "level2-nrt0-rd"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + level2_nrt0_wr: level2-nrt0-wr { + cell-index = <26>; + node-name = "level2-nrt0-wr"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + level2_nrt1_rd: level2-nrt1-rd { + cell-index = <27>; + node-name = "level2-nrt1-rd"; + parent-node = <&level3_nrt1_rd_wr_sum>; + traffic-merge-type = ; + bus-width-factor = <4>; + }; + + level2_rt0_rd: level2-rt0-rd { + cell-index = <28>; + node-name = "level2-rt0-rd"; + parent-node = <&level3_rt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + level2_rt0_wr: level2-rt0-wr { + cell-index = <29>; + node-name = "level2-rt0-wr"; + parent-node = <&level3_rt0_rd_wr_sum>; + traffic-merge-type = + ; + }; + + }; + + level3-nodes { + level-index = <3>; + + level3_nrt0_rd_wr_sum: level3-nrt0-rd-wr-sum { + cell-index = <30>; + node-name = "level3-nrt0-rd-wr-sum"; + traffic-merge-type = ; + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_0"; + }; + }; + + level3_nrt1_rd_wr_sum: level3-nrt1-rd-wr-sum { + cell-index = <31>; + node-name = "level3-nrt1-rd-wr-sum"; + traffic-merge-type = ; + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_icp"; + }; + }; + + level3_rt0_rd_wr_sum: level3-rt0-rd-wr-sum { + cell-index = <32>; + node-name = "level3-rt0-rd-wr-sum"; + traffic-merge-type = ; + ib-bw-voting-needed; + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_hf_0"; + }; + }; + }; + }; + + cpas_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_cci0: qcom,cci0@ac15000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac15000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x15000>; + interrupts = ; + interrupt-names = "cci0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_0_CLK_SRC>, + <&camcc CAM_CC_CCI_0_CLK>; + clock-names = "cci_0_clk_src", + "cci_0_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_0_clk_src"; + operating-points-v2 = <&cci0_opp_table>; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + pinctrl-0 = <&cci0_active>; + pinctrl-1 = <&cci0_suspend>; + pinctrl-2 = <&cci1_active>; + pinctrl-3 = <&cci1_suspend>; + cell-index = <0>; + status = "okay"; + + cci0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci0: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_400Khz_cci0: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_1Mhz_cci0: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <0>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_100Khz_cci0: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + }; + + cam_cci1: qcom,cci1@ac16000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac16000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x16000>; + interrupts = ; + interrupt-names = "cci1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_1_CLK_SRC>, + <&camcc CAM_CC_CCI_1_CLK>; + clock-names = "cci_1_clk_src", + "cci_1_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_1_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + pinctrl-0 = <&cci2_active>; + pinctrl-1 = <&cci2_suspend>; + pinctrl-2 = <&cci3_active>; + pinctrl-3 = <&cci3_suspend>; + operating-points-v2 = <&cci1_opp_table>; + cell-index = <1>; + status = "okay"; + + cci1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci1: qcom,i2c-custom-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <1>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_400Khz_cci1: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_1Mhz_cci1: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <0>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_100Khz_cci1: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + }; + + qcom,rt-cdm0@ac25000 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xac25000 0x0 0x400>; + reg-names = "rt-cdm0"; + reg-cam-base = <0x25000>; + interrupts = ; + interrupt-names = "rt-cdm0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_fast_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + src-clock-name = "cam_cc_fast_ahb_clk_src"; + clock-rates = <400000000 0>; + clock-cntl-level = "nominal"; + operating-points-v2 = <&cdm_cpas_opp_table0>; + nrt-device; + cdm-client-names = "ife0", "dualife0"; + config-fifo; + fifo-depths = <64 0 0 0>; + cam_hw_pid = <25>; + cam-hw-mid = <0>; + single-context-cdm; + cell-index = <0>; + status = "okay"; + + cdm_cpas_opp_table0: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_jpeg_enc: qcom,jpegenc@ac2a000 { + compatible = "qcom,cam_jpeg_enc_680"; + reg = <0x0 0xac2a000 0x0 0x1000>, + <0x0 0x0ac19000 0x0 0xc000>; + reg-names = "jpegenc_hw", "cam_camnoc"; + reg-cam-base = <0x2a000 0x19000>; + interrupts = ; + interrupt-names = "jpeg"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "jpegenc_clk_src", + "jpegenc_clk"; + clock-rates = <160000000 0>, + <200000000 0>, + <400000000 0>, + <480000000 0>, + <600000000 0>; + src-clock-name = "jpegenc_clk_src"; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + operating-points-v2 = <&jpeg_enc_opp_table>; + nrt-device; + cam_hw_pid = <12 14>; + cam_hw_rd_mid = <0>; + cam_hw_wr_mid = <1>; + cell-index = <0>; + status = "okay"; + + jpeg_enc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-48000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_jpeg_dma: qcom,jpegdma@ac2b000 { + compatible = "qcom,cam_jpeg_dma_680"; + reg = <0x0 0xac2b000 0x0 0x1000>, + <0x0 0x0ac19000 0x0 0xc000>; + reg-names = "jpegdma_hw", "cam_camnoc"; + reg-cam-base = <0x2b000 0x19000>; + interrupts = ; + interrupt-names = "jpegdma"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "jpegdma_clk_src", + "jpegdma_clk"; + clock-rates = <160000000 0>, + <200000000 0>, + <400000000 0>, + <480000000 0>, + <600000000 0>; + src-clock-name = "jpegdma_clk_src"; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + operating-points-v2 = <&jpeg_dma_opp_table>; + nrt-device; + cam_hw_pid = <13 15>; + cam_hw_rd_mid = <0>; + cam_hw_wr_mid = <1>; + cell-index = <0>; + status = "okay"; + + jpeg_dma_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-48000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_bps: qcom,bps@ac2c000 { + compatible = "qcom,cam-bps680"; + reg = <0x0 0xac2c000 0x0 0xb000>; + reg-names = "bps_top"; + reg-cam-base = <0x2c000>; + power-domains = <&camcc CAM_CC_BPS_GDSC>; + clocks = <&camcc CAM_CC_BPS_AHB_CLK>, + <&camcc CAM_CC_BPS_FAST_AHB_CLK>, + <&camcc CAM_CC_BPS_CLK_SRC>, + <&camcc CAM_CC_BPS_CLK>, + <&camcc CAM_CC_CPAS_BPS_CLK>; + clock-names = "bps_ahb_clk", + "bps_fast_ahb_clk", + "bps_clk_src", + "bps_clk", + "cam_cc_cpas_bps_clk"; + clock-rates = <0 0 160000000 0 0>, + <0 0 200000000 0 0>, + <0 0 400000000 0 0>, + <0 0 600000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "nominal"; + nrt-device; + src-clock-name = "bps_clk_src"; + operating-points-v2 = <&bps_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <6 7>; + cell-index = <0>; + status = "okay"; + + bps_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_ipe0: qcom,ipe0@ac42000 { + compatible = "qcom,cam-ipe680"; + reg = <0x0 0xac42000 0x0 0x18000>; + reg-names = "ipe0_top"; + reg-cam-base = <0x42000>; + power-domains = <&camcc CAM_CC_IPE_0_GDSC>; + clocks = <&camcc CAM_CC_IPE_NPS_AHB_CLK>, + <&camcc CAM_CC_IPE_NPS_FAST_AHB_CLK>, + <&camcc CAM_CC_IPE_PPS_FAST_AHB_CLK>, + <&camcc CAM_CC_IPE_NPS_CLK_SRC>, + <&camcc CAM_CC_IPE_NPS_CLK>, + <&camcc CAM_CC_IPE_PPS_CLK>, + <&camcc CAM_CC_CPAS_IPE_NPS_CLK>; + clock-names = "ipe_nps_ahb_clk", + "ipe_nps_fast_ahb_clk", + "ipe_pps_fast_ahb_clk", + "ipe_nps_clk_src", + "ipe_nps_clk", + "ipe_pps_clk", + "cam_cc_cpas_ipe_nps_clk"; + clock-rates = <0 0 0 304000000 0 0 0>, + <0 0 0 364000000 0 0 0>, + <0 0 0 500000000 0 0 0>, + <0 0 0 600000000 0 0 0>, + <0 0 0 700000000 0 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + nrt-device; + src-clock-name = "ipe_nps_clk_src"; + operating-points-v2 = <&ipe0_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <22 23 30>; + cell-index = <0>; + status = "okay"; + + ipe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-304000000 { + opp-hz = /bits/ 64 <304000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-364000000 { + opp-hz = /bits/ 64 <364000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-700000000 { + opp-hz = /bits/ 64 <700000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe0: qcom,ife0@ac62000 { + compatible = "qcom,vfe680"; + reg = <0x0 0xac62000 0x0 0xf000>, + <0x0 0xac19000 0x0 0xc000>; + reg-names = "ife", "cam_camnoc"; + reg-cam-base = <0x62000 0x19000>; + rt-wrapper-base = <0x62000>; + interrupts = ; + interrupt-names = "ife0"; + power-domains = <&camcc CAM_CC_IFE_0_GDSC>; + clocks = <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_CPAS_IFE_0_CLK>; + clock-names = "ife_0_fast_ahb", + "ife_0_clk_src", + "ife_0_clk", + "cam_cc_cpas_ife_0_clk"; + clock-rates = <0 345600000 0 0>, + <0 432000000 0 0>, + <0 594000000 0 0>, + <0 675000000 0 0>, + <0 727000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "svs", "svs_l1", "nominal"; + src-clock-name = "ife_0_clk_src"; + operating-points-v2 = <&vfe0_opp_table>; + clock-control-debugfs = "true"; + clock-names-option = "ife_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_0_DSP_CLK>; + clock-rates-option = <594000000>; + ubwc-static-cfg = <0x1026 0x1036>; + cam_hw_pid = <16 4 20 8>; + cell-index = <0>; + status = "okay"; + + vfe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-345600000 { + opp-hz = /bits/ 64 <345600000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-675000000 { + opp-hz = /bits/ 64 <675000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-727000000 { + opp-hz = /bits/ 64 <727000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid0: qcom,csid0@acb7000 { + compatible = "qcom,csid695"; + reg = <0x0 0xacb7000 0x0 0xd00>, + <0x0 0xacb6000 0x0 0x1000>; + reg-names = "csid", "csid_top"; + reg-cam-base = <0xb7000 0xb6000>; + rt-wrapper-base = <0x62000>; + interrupts = ; + interrupt-names = "csid0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <1 0 0>; + clocks = <&camcc CAM_CC_CSID_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "csid_clk_src", + "csid_clk", + "csiphy_rx_clk"; + clock-rates = <300000000 0 0>, + <400000000 0 0>, + <480000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "csid_clk_src"; + operating-points-v2 = <&csid0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "okay"; + + csid0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite0: qcom,csid-lite0@acc6000 { + compatible = "qcom,csid-lite680"; + reg = <0x0 0xacc6000 0x0 0xa00>; + reg-names = "csid-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xc6000>; + interrupts = ; + interrupt-names = "csid-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 1 0 0 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb_clk", + "ife_lite_csid_clk_src", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 266666667 0 0 0 0>, + <0 400000000 0 0 0 0>, + <0 480000000 0 0 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "okay"; + + csid_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite0: qcom,ife-lite0@acc6000 { + compatible = "qcom,vfe-lite695"; + reg = <0x0 0xacc6000 0x0 0x4000>; + reg-names = "ife-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xc6000>; + interrupts = ; + interrupt-names = "ife-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 0 0 1 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb_clk", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk_src", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 0 0 266666667 0 0>, + <0 0 0 400000000 0 0>, + <0 0 0 480000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite0_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <0>; + cell-index = <1>; + status = "okay"; + + vfe_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite1: qcom,csid-lite1@acca000 { + compatible = "qcom,csid-lite680"; + reg = <0x0 0xacca000 0x0 0xa00>; + reg-names = "csid-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xca000>; + interrupts = ; + interrupt-names = "csid-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 1 0 0 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb_clk", + "ife_lite_csid_clk_src", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 266666667 0 0 0 0>, + <0 400000000 0 0 0 0>, + <0 480000000 0 0 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <2>; + status = "okay"; + + csid_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite1: qcom,ife-lite1@acca000 { + compatible = "qcom,vfe-lite695"; + reg = <0x0 0xacca000 0x0 0x4000>; + reg-names = "ife-lite"; + rt-wrapper-base = <0xc6000>; + reg-cam-base = <0xca000>; + interrupts = ; + interrupt-names = "ife-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + shared-clks = <0 0 0 1 0 0>; + clocks = <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>; + clock-names = "ife_lite_ahb", + "ife_lite_csid_clk", + "ife_lite_cphy_rx_clk", + "ife_lite_clk_src", + "ife_lite_clk", + "cam_cc_cpas_ife_lite_clk"; + clock-rates = <0 0 0 266666667 0 0>, + <0 0 0 400000000 0 0>, + <0 0 0 480000000 0 0>; + clock-cntl-level = "lowsvsd1", "lowsvs", "nominal"; + src-clock-name = "ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite1_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <1>; + cell-index = <2>; + status = "okay"; + + vfe_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-266666667 { + opp-hz = /bits/ 64 <266666667>; + required-opps = <&rpmhpd_opp_low_svs_d1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy0: qcom,csiphy0@ace4000 { + compatible = "qcom,csiphy-v2.1.2", "qcom,csiphy"; + reg = <0x0 0xace4000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe4000>; + interrupts = ; + interrupt-names = "CSIPHY0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + csi-vdd-1p2-supply = <&vreg_l1c_1p2>; + csi-vdd-0p9-supply = <&vreg_l2c_0p8>; + rgltr-cntrl-support; + rgltr-enable-sync = <1>; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 920000>; + rgltr-load-current = <58900 147000>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy0_clk", + "csi0phytimer_clk_src", + "csi0phytimer_clk"; + src-clock-name = "csi0phytimer_clk_src"; + clock-cntl-level = "nominal"; + clock-rates = <480000000 0 400000000 0>; + operating-points-v2 = <&csiphy0_opp_table>; + cell-index = <0>; + status = "okay"; + + csiphy0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy4: qcom,csiphy4@acec000 { + compatible = "qcom,csiphy-v2.1.2", "qcom,csiphy"; + reg = <0x0 0xacec000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xec000>; + interrupts = ; + interrupt-names = "CSIPHY4"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + csi-vdd-1p2-supply = <&vreg_l1c_1p2>; + csi-vdd-0p9-supply = <&vreg_l2c_0p8>; + rgltr-cntrl-support; + rgltr-enable-sync = <1>; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1200000 920000>; + rgltr-load-current = <58900 147000>; + shared-clks = <1 0 0 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy4_clk", + "csi4phytimer_clk_src", + "csi4phytimer_clk"; + src-clock-name = "csi4phytimer_clk_src"; + clock-cntl-level = "nominal"; + clock-rates = <480000000 0 400000000 0>; + operating-points-v2 = <&csiphy4_opp_table>; + cell-index = <4>; + status = "okay"; + + csiphy4_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy_tpg13: qcom,tpg13@acf6000 { + compatible = "qcom,cam-tpg103"; + reg = <0x0 0xacf6000 0x0 0x400>, + <0x0 0xac13000 0x0 0x1000>; + reg-names = "tpg0", "cam_cpas_top"; + reg-cam-base = <0xf6000 0x13000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg0"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>, + <480000000 0>; + clock-cntl-level = "lowsvs", "nominal"; + src-clock-name = "cphy_rx_clk_src"; + operating-points-v2 = <&csiphy_tpg0_opp_table>; + cell-index = <13>; + phy-id = <0>; + status = "okay"; + + csiphy_tpg0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy_tpg14: qcom,tpg14@acf7000 { + compatible = "qcom,cam-tpg103"; + reg = <0x0 0xacf7000 0x0 0x400>, + <0x0 0xac13000 0x0 0x1000>; + reg-names = "tpg1", "cam_cpas_top"; + reg-cam-base = <0xf7000 0x13000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg1"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>, + <480000000 0>; + clock-cntl-level = "lowsvs", "nominal"; + src-clock-name = "cphy_rx_clk_src"; + operating-points-v2 = <&csiphy_tpg1_opp_table>; + cell-index = <14>; + phy-id = <1>; + status = "okay"; + + csiphy_tpg1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy_tpg15: qcom,tpg15@acf8000 { + compatible = "qcom,cam-tpg103"; + reg = <0x0 0xacf8000 0x0 0x400>, + <0x0 0xac13000 0x0 0x1000>; + reg-names = "tpg2", "cam_cpas_top"; + reg-cam-base = <0xf8000 0x13000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + interrupts = ; + interrupt-names = "tpg2"; + shared-clks = <1 0>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>; + clock-names = "cphy_rx_clk_src", + "csid_csiphy_rx_clk"; + clock-rates = <400000000 0>, + <480000000 0>; + clock-cntl-level = "lowsvs", "nominal"; + src-clock-name = "cphy_rx_clk_src"; + operating-points-v2 = <&csiphy_tpg2_opp_table>; + cell-index = <15>; + phy-id = <2>; + status = "okay"; + + csiphy_tpg2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,rt-cdm2@acf9000 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xacf9000 0x0 0x400>; + reg-names = "rt-cdm2"; + reg-cam-base = <0xf9000>; + interrupts = ; + interrupt-names = "rt-cdm2"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_lite_ahb"; + clock-rates = <80000000 0>; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-cntl-level = "nominal"; + operating-points-v2 = <&cdm_cpas_opp_table2>; + nrt-device; + cdm-client-names = "ife2"; + config-fifo; + fifo-depths = <64 0 0 0>; + cam_hw_pid = <24>; + cam-hw-mid = <0>; + single-context-cdm; + cell-index = <2>; + status = "okay"; + + cdm_cpas_opp_table2: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,rt-cdm3@acfa000 { + compatible = "qcom,cam-rt-cdm2_1"; + label = "rt-cdm"; + reg = <0x0 0xacfa000 0x0 0x400>; + reg-names = "rt-cdm3"; + reg-cam-base = <0xfa000>; + interrupts = ; + interrupt-names = "rt-cdm3"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_lite_ahb"; + clock-rates = <80000000 0>; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + clock-cntl-level = "nominal"; + operating-points-v2 = <&cdm_cpas_opp_table3>; + nrt-device; + cdm-client-names = "ife3"; + config-fifo; + fifo-depths = <64 0 0 0>; + cam_hw_pid = <27>; + cam-hw-mid = <0>; + single-context-cdm; + cell-index = <3>; + status = "okay"; + + cdm_cpas_opp_table3: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,cam-cdm-intf { + compatible = "qcom,cam-cdm-intf"; + label = "cam-cdm-intf"; + num-hw-cdm = <1>; + cdm-client-names = "vfe", + "jpegdma", + "jpegenc"; + cell-index = <0>; + status = "okay"; + }; + + cam_icp_firmware: qcom,cam-icp { + compatible = "qcom,cam-icp"; + compat-hw-name = "qcom,icp", + "qcom,ipe0", + "qcom,bps"; + num-icp = <1>; + num-ipe = <1>; + num-bps = <1>; + status = "okay"; + icp_pc_en; + icp_use_pil; + }; + + qcom,cam-isp { + compatible = "qcom,cam-isp"; + arch-compat = "ife"; + status = "okay"; + }; + + qcom,cam-jpeg { + compatible = "qcom,cam-jpeg"; + compat-hw-name = "qcom,jpegenc", + "qcom,jpegdma"; + num-jpeg-enc = <1>; + num-jpeg-dma = <1>; + status = "okay"; + }; + + qcom,cam-req-mgr { + compatible = "qcom,cam-req-mgr"; + status = "okay"; + }; + + cam_smmu: qcom,cam-smmu { + compatible = "qcom,msm-cam-smmu", "simple-bus"; + status = "okay"; + force_cache_allocs; + need_shared_buffer_padding; + + msm-cam-smmu-cpas-cdm { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x18A0 0x0000>; + cam-smmu-label = "rt-cdm"; + dma-coherent; + multiple-client-devices; + + cpas_cdm_iova_mem_map: iova-mem-map { + iova-mem-region-io { + /* IO region is approximately 4.0 GB */ + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-icp { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x1800 0x0060>, + <&apps_smmu 0x1820 0x0060>, + <&apps_smmu 0x1840 0x0060>, + <&apps_smmu 0x1860 0x0060>, + <&apps_smmu 0x1900 0x0000>, + <&apps_smmu 0x1980 0x0020>, + <&apps_smmu 0x19A0 0x0020>; + cam-smmu-label = "icp"; + iova-region-discard = <0xe0000000 0x800000>; + dma-coherent; + + icp_iova_mem_map: iova-mem-map { + iova-mem-qdss-region { + /* QDSS region is appropriate 1MB */ + iova-region-name = "qdss"; + iova-region-start = <0x10b00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x5>; + qdss-phy-addr = <0x16790000>; + status = "okay"; + }; + + iova-mem-region-fwuncached-region { + /* FW uncached region is 7MB long */ + iova-region-name = "fw_uncached"; + iova-region-start = <0x10400000>; + iova-region-len = <0x700000>; + iova-region-id = <0x6>; + status = "okay"; + }; + + iova-mem-region-io { + /* IO region is approximately 3.8 GB */ + iova-region-name = "io"; + iova-region-start = <0x10c00000>; + iova-region-len = <0xee300000>; + iova-region-id = <0x3>; + iova-region-discard = <0xe0000000 0x800000>; + status = "okay"; + }; + + iova-mem-region-shared { + /* Shared region is ~250MB long */ + iova-region-name = "shared"; + iova-region-start = <0x800000>; + iova-region-len = <0xFC00000>; + iova-region-id = <0x1>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-ife { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0800 0x0060>, + <&apps_smmu 0x0820 0x0060>, + <&apps_smmu 0x0840 0x0060>, + <&apps_smmu 0x0860 0x0060>; + dma-coherent; + cam-smmu-label = "ife"; + multiple-client-devices; + + ife_iova_mem_map: iova-mem-map { + /* IO region is approximately 4.0 GB */ + iova-mem-region-io { + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-jpeg { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x18E0 0x0000>; + cam-smmu-label = "jpeg"; + dma-coherent; + + jpeg_iova_mem_map: iova-mem-map { + /* IO region is approximately 4.0 GB */ + iova-mem-region-io { + iova-region-name = "io"; + /* 1 MB pad for start */ + iova-region-start = <0x100000>; + /* 1 MB pad for end */ + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-secure { + compatible = "qcom,msm-cam-smmu-cb"; + cam-smmu-label = "cam-secure"; + qcom,secure-cb; + }; + }; + + qcom,cam-sync { + compatible = "qcom,cam-sync"; + status = "okay"; + }; + + qcom,camera-main { + compatible = "qcom,camera_x1e80100"; + status = "okay"; + }; +}; + +&tlmm { + cam_sensor_active_int: cam-sensor-active-int { + /* CUSTOM */ + mux { + pins = "gpio19"; + function = "cam_pwe"; + }; + + config { + pins = "gpio19"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst0: cam-sensor-active-rst0 { + /* RESET REAR */ + config { + pins = "gpio109"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + pins = "gpio109"; + function = "gpio"; + }; + }; + + cam_sensor_active_rst1: cam-sensor-active-rst1 { + /* RESET */ + mux { + pins = "gpio110"; + function = "gpio"; + }; + + config { + pins = "gpio110"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst4: cam-sensor-active-rst4 { + /* RESET REAR */ + config { + pins = "gpio237"; + bias-disable; /* No PULL */ + drive-strength = <12>; /* 12 MA */ + }; + + mux { + pins = "gpio237"; + function = "gpio"; + }; + }; + + cam_sensor_mclk0_active: cam-sensor-mclk0-active { + /* mclk0 */ + config { + pins = "gpio96"; + bias-disable; /* No PULL */ + drive-strength = <6>; /* 6 MA */ + }; + + mux { + pins = "gpio96"; + function = "cam_mclk"; + }; + }; + + cam_sensor_mclk0_suspend: cam-sensor-mclk0-suspend { + /* mclk0 */ + config { + pins = "gpio96"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <6>; /* 6 MA */ + }; + + mux { + pins = "gpio96"; + function = "cam_mclk"; + }; + }; + + cam_sensor_mclk1_active: cam-sensor-mclk1-active { + /* MCLK1 */ + mux { + pins = "gpio97"; + function = "cam_mclk"; + }; + + config { + pins = "gpio97"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_suspend: cam-sensor-mclk1-suspend { + /* MCLK1 */ + mux { + pins = "gpio97"; + function = "cam_mclk"; + }; + + config { + pins = "gpio97"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk4_active: cam-sensor-mclk4-active { + /* mclk4 */ + config { + pins = "gpio100"; + bias-disable; /* No PULL */ + drive-strength = <12>; /* 12 MA */ + }; + + mux { + pins = "gpio100"; + function = "cam_aon"; + }; + }; + + cam_sensor_mclk4_suspend: cam-sensor-mclk4-suspend { + /* mclk4 */ + config { + pins = "gpio100"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <12>; /* 12 MA */ + }; + + mux { + pins = "gpio100"; + function = "cam_aon"; + }; + }; + + cam_sensor_suspend_int: cam-sensor-suspend-int { + /* CUSTOM */ + mux { + pins = "gpio19"; + function = "cam_pwe"; + }; + + config { + pins = "gpio19"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_suspend_rst0: cam-sensor-suspend-rst0 { + /* RESET REAR */ + config { + pins = "gpio109"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + + mux { + pins = "gpio109"; + function = "gpio"; + }; + }; + + cam_sensor_suspend_rst1: cam-sensor-suspend-rst1 { + /* RESET */ + mux { + pins = "gpio110"; + function = "gpio"; + }; + + config { + pins = "gpio110"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst4: cam-sensor-suspend-rst4 { + /* RESET REAR */ + config { + pins = "gpio237"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <12>; /* 12 MA */ + output-low; + }; + + mux { + pins = "gpio237"; + function = "gpio"; + }; + }; + + cci0_active: cci0-active { + config { + /* DATA, CLK */ + pins = "gpio101","gpio102"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio101","gpio102"; // Only 2 + function = "cci_i2c"; + }; + }; + + cci0_suspend: cci0-suspend { + config { + /* DATA, CLK */ + pins = "gpio101","gpio102"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio101","gpio102"; + function = "cci_i2c"; + }; + }; + + cci1_active: cci1-active { + config { + /* DATA, CLK */ + pins = "gpio103","gpio104"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio103","gpio104"; // Only 2 + function = "cci_i2c"; + }; + }; + + cci1_suspend: cci1-suspend { + config { + /* DATA, CLK */ + pins = "gpio103","gpio104"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio103","gpio104"; + function = "cci_i2c"; + }; + }; + + cci2_active: cci2-active { + config { + /* DATA, CLK */ + pins = "gpio105","gpio106"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio105","gpio106"; // Only 2 + function = "cci_i2c"; + }; + }; + + cci2_suspend: cci2-suspend { + config { + /* DATA, CLK */ + pins = "gpio105","gpio106"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio105","gpio106"; + function = "cci_i2c"; + }; + }; + + cci3_active: cci3-active { + config { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + function = "aon_cci"; + }; + }; + + cci3_suspend: cci3-suspend { + config { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + mux { + /* DATA, CLK */ + pins = "gpio235","gpio236"; + function = "aon_cci"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/purwa-evk-camx.dtso b/arch/arm64/boot/dts/qcom/purwa-evk-camx.dtso new file mode 100644 index 0000000000000..26603aed4eafc --- /dev/null +++ b/arch/arm64/boot/dts/qcom/purwa-evk-camx.dtso @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "purwa-camera.dtsi" + +&camss { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts b/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts new file mode 100644 index 0000000000000..ac99ec0f95fcb --- /dev/null +++ b/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts @@ -0,0 +1,1587 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; + +#include +#include "purwa-iot-som.dtsi" +#include + +/ { + model = "Qualcomm Technologies, Inc. Purwa IoT EVK"; + compatible = "qcom,purwa-iot-evk", "qcom,purwa-iot-som", "qcom,x1p42100"; + chassis-type = "embedded"; + + aliases { + serial0 = &uart21; + serial1 = &uart14; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pmk8550_pwm 0 5000000>; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; + power-supply = <&vreg_edp_bl>; + + pinctrl-0 = <&edp_bl_en>, <&edp_bl_pwm>; + pinctrl-names = "default"; + }; + + wcd938x: audio-codec { + compatible = "qcom,wcd9385-codec"; + + pinctrl-0 = <&wcd_default>; + pinctrl-names = "default"; + + reset-gpios = <&tlmm 191 GPIO_ACTIVE_LOW>; + + qcom,micbias1-microvolt = <1800000>; + qcom,micbias2-microvolt = <1800000>; + qcom,micbias3-microvolt = <1800000>; + qcom,micbias4-microvolt = <1800000>; + qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 + 500000 500000 500000 500000>; + qcom,mbhc-headset-vthreshold-microvolt = <1700000>; + qcom,mbhc-headphone-vthreshold-microvolt = <50000>; + qcom,rx-device = <&wcd_rx>; + qcom,tx-device = <&wcd_tx>; + + vdd-buck-supply = <&vreg_l15b_1p8>; + vdd-rxtx-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l15b_1p8>; + vdd-mic-bias-supply = <&vreg_bob1>; + + #sound-dai-cells = <1>; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + connector3 { + compatible = "usb-a-connector"; + label = "USB-3-Type-A"; + power-role = "source"; + + vbus-supply = <®ulator_usb3_vbus>; + + port { + connector_3_in: endpoint { + }; + }; + }; + + connector6 { + compatible = "usb-a-connector"; + label = "USB-6-Type-A"; + power-role = "source"; + + vbus-supply = <®ulator_usb6_vbus>; + + port { + connector_4_in: endpoint { + }; + }; + }; + + pmic-glink { + compatible = "qcom,x1e80100-pmic-glink", + "qcom,sm8550-pmic-glink", + "qcom,pmic-glink"; + #address-cells = <1>; + #size-cells = <0>; + orientation-gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>, + <&tlmm 123 GPIO_ACTIVE_HIGH>, + <&tlmm 125 GPIO_ACTIVE_HIGH>; + + connector@0 { + compatible = "usb-c-connector"; + reg = <0>; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_ss0_hs_in: endpoint { + remote-endpoint = <&usb_1_ss0_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss0_ss_in: endpoint { + remote-endpoint = <&retimer_ss0_ss_out>; + }; + }; + + port@2 { + reg = <2>; + + pmic_glink_ss0_con_sbu_in: endpoint { + remote-endpoint = <&retimer_ss0_con_sbu_out>; + }; + }; + }; + }; + + connector@1 { + compatible = "usb-c-connector"; + reg = <1>; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_ss1_hs_in: endpoint { + remote-endpoint = <&usb_1_ss1_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss1_ss_in: endpoint { + remote-endpoint = <&retimer_ss1_ss_out>; + }; + }; + + port@2 { + reg = <2>; + + pmic_glink_ss1_con_sbu_in: endpoint { + remote-endpoint = <&retimer_ss1_con_sbu_out>; + }; + }; + }; + }; + + connector@2 { + compatible = "usb-c-connector"; + reg = <2>; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_ss2_hs_in: endpoint { + remote-endpoint = <&usb_1_ss2_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss2_ss_in: endpoint { + remote-endpoint = <&retimer_ss2_ss_out>; + }; + }; + + port@2 { + reg = <2>; + + pmic_glink_ss2_con_sbu_in: endpoint { + remote-endpoint = <&retimer_ss2_con_sbu_out>; + }; + }; + }; + }; + }; + + vreg_edp_3p3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_EDP_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&edp_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_edp_bl: regulator-edp-bl { + compatible = "regulator-fixed"; + + regulator-name = "VBL9"; + regulator-min-microvolt = <3600000>; + regulator-max-microvolt = <3600000>; + + gpio = <&pmc8380_3_gpios 10 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&edp_bl_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_nvme: regulator-nvme { + compatible = "regulator-fixed"; + + regulator-name = "VREG_NVME_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 18 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&nvme_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_pcie_12v: regulator-pcie-12v { + compatible = "regulator-fixed"; + + regulator-name = "VREG_PCIE_12V"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + + gpio = <&pm8550ve_8_gpios 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&pcie_x8_12v>; + pinctrl-names = "default"; + }; + + vreg_pcie_3v3: regulator-pcie-3v3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_PCIE_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8380_3_gpios 6 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&pm_sde7_main_3p3_en>; + pinctrl-names = "default"; + }; + + vreg_pcie_3v3_aux: regulator-pcie-3v3-aux { + compatible = "regulator-fixed"; + + regulator-name = "VREG_PCIE_3P3_AUX"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8380_3_gpios 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&pm_sde7_aux_3p3_en>; + pinctrl-names = "default"; + }; + + /* Left unused as the retimer is not used on this board. */ + vreg_rtmr0_1p15: regulator-rtmr0-1p15 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR0_1P15"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + + gpio = <&pmc8380_5_gpios 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb0_pwr_1p15_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr0_1p8: regulator-rtmr0-1p8 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR0_1P8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + gpio = <&pm8550ve_9_gpios 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb0_1p8_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr0_3p3: regulator-rtmr0-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR0_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pm8550_gpios 11 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb0_3p3_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr1_1p15: regulator-rtmr1-1p15 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR1_1P15"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + + gpio = <&tlmm 188 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb1_pwr_1p15_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr1_1p8: regulator-rtmr1-1p8 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR1_1P8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + gpio = <&tlmm 175 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb1_pwr_1p8_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr1_3p3: regulator-rtmr1-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR1_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 186 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb1_pwr_3p3_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr2_1p15: regulator-rtmr2-1p15 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR2_1P15"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + + gpio = <&tlmm 189 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb2_pwr_1p15_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr2_1p8: regulator-rtmr2-1p8 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR2_1P8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + gpio = <&tlmm 126 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb2_pwr_1p8_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_rtmr2_3p3: regulator-rtmr2-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_RTMR2_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 187 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&usb2_pwr_3p3_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + regulator_usb3_vbus: regulator-usb3-vbus { + compatible = "regulator-fixed"; + regulator-name = "USB3_VBUS"; + gpio = <&pm8550ve_9_gpios 4 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&usb3_en>; + pinctrl-names = "default"; + enable-active-high; + regulator-always-on; + }; + + regulator_usb6_vbus: regulator-usb6-vbus { + compatible = "regulator-fixed"; + regulator-name = "USB6_VBUS"; + gpio = <&pm8550ve_9_gpios 5 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&usb6_en>; + pinctrl-names = "default"; + enable-active-high; + regulator-always-on; + }; + + vph_pwr: regulator-vph-pwr { + compatible = "regulator-fixed"; + + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + + regulator-always-on; + regulator-boot-on; + }; + + /* + * TODO: These two regulators are actually part of the removable M.2 + * card and not the EVK mainboard. Need to describe this differently. + * Functionally it works correctly, because all we need to do is to + * turn on the actual 3.3V supply above. + */ + vreg_wcn_0p95: regulator-wcn-0p95 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_WCN_0P95"; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <950000>; + + vin-supply = <&vreg_wcn_3p3>; + }; + + vreg_wcn_1p9: regulator-wcn-1p9 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_WCN_1P9"; + regulator-min-microvolt = <1900000>; + regulator-max-microvolt = <1900000>; + + vin-supply = <&vreg_wcn_3p3>; + }; + + vreg_wcn_3p3: regulator-wcn-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_WCN_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 214 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&wcn_sw_en>; + pinctrl-names = "default"; + + regulator-always-on; + regulator-boot-on; + }; + + vreg_wcn_bt_en: regulator-wcn-bt-en { + compatible = "regulator-fixed"; + + regulator-name = "VREG_WCN_BT_EN"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + gpio = <&tlmm 116 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&wcn_bt_en>; + pinctrl-names = "default"; + + regulator-always-on; + regulator-boot-on; + }; + + vreg_wwan: regulator-wwan { + compatible = "regulator-fixed"; + + regulator-name = "SDX_VPH_PWR"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 221 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&wwan_sw_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + sound { + compatible = "qcom,x1e80100-sndcard"; + model = "X1E80100-EVK"; + audio-routing = "WooferLeft IN", "WSA WSA_SPK1 OUT", + "TweeterLeft IN", "WSA WSA_SPK2 OUT", + "WooferRight IN", "WSA2 WSA_SPK2 OUT", + "TweeterRight IN", "WSA2 WSA_SPK2 OUT", + "IN1_HPHL", "HPHL_OUT", + "IN2_HPHR", "HPHR_OUT", + "AMIC2", "MIC BIAS2", + "VA DMIC0", "MIC BIAS3", + "VA DMIC1", "MIC BIAS3", + "VA DMIC2", "MIC BIAS1", + "VA DMIC3", "MIC BIAS1", + "TX SWR_INPUT1", "ADC2_OUTPUT"; + + wcd-playback-dai-link { + link-name = "WCD Playback"; + + codec { + sound-dai = <&wcd938x 0>, <&swr1 0>, <&lpass_rxmacro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai RX_CODEC_DMA_RX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + wcd-capture-dai-link { + link-name = "WCD Capture"; + + codec { + sound-dai = <&wcd938x 1>, <&swr2 1>, <&lpass_txmacro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai TX_CODEC_DMA_TX_3>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + wsa-dai-link { + link-name = "WSA Playback"; + + codec { + sound-dai = <&left_woofer>, + <&left_tweeter>, + <&swr0 0>, + <&lpass_wsamacro 0>, + <&right_woofer>, + <&right_tweeter>, + <&swr3 0>, + <&lpass_wsa2macro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + va-dai-link { + link-name = "VA Capture"; + + codec { + sound-dai = <&lpass_vamacro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + }; + + wcn7850-pmu { + compatible = "qcom,wcn7850-pmu"; + + vdd-supply = <&vreg_wcn_0p95>; + vddio-supply = <&vreg_l15b_1p8>; + vddaon-supply = <&vreg_wcn_0p95>; + vdddig-supply = <&vreg_wcn_0p95>; + vddrfa1p2-supply = <&vreg_wcn_1p9>; + vddrfa1p8-supply = <&vreg_wcn_1p9>; + + wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&wcn_wlan_en>; + pinctrl-names = "default"; + + regulators { + vreg_pmu_rfa_cmn: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn"; + }; + + vreg_pmu_aon_0p59: ldo1 { + regulator-name = "vreg_pmu_aon_0p59"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p85: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p85"; + }; + + vreg_pmu_btcmx_0p85: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p85"; + }; + + vreg_pmu_rfa_0p8: ldo5 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo6 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p8: ldo7 { + regulator-name = "vreg_pmu_rfa_1p8"; + }; + + vreg_pmu_pcie_0p9: ldo8 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_pcie_1p8: ldo9 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + }; + }; +}; + +&i2c1 { + clock-frequency = <400000>; + status = "okay"; + + typec-mux@8 { + compatible = "parade,ps8830"; + reg = <0x08>; + + clocks = <&rpmhcc RPMH_RF_CLK5>; + + vdd-supply = <&vreg_rtmr2_1p15>; + vdd33-supply = <&vreg_rtmr2_3p3>; + vdd33-cap-supply = <&vreg_rtmr2_3p3>; + vddar-supply = <&vreg_rtmr2_1p15>; + vddat-supply = <&vreg_rtmr2_1p15>; + vddio-supply = <&vreg_rtmr2_1p8>; + + reset-gpios = <&tlmm 185 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&rtmr2_default>; + pinctrl-names = "default"; + + orientation-switch; + retimer-switch; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + retimer_ss2_ss_out: endpoint { + remote-endpoint = <&pmic_glink_ss2_ss_in>; + }; + }; + + port@1 { + reg = <1>; + + retimer_ss2_ss_in: endpoint { + remote-endpoint = <&usb_1_ss2_qmpphy_out>; + }; + }; + + port@2 { + reg = <2>; + + retimer_ss2_con_sbu_out: endpoint { + remote-endpoint = <&pmic_glink_ss2_con_sbu_in>; + }; + }; + }; + }; +}; + +&camss { + status = "okay"; +}; + +&i2c3 { + clock-frequency = <400000>; + + status = "okay"; + + typec-mux@8 { + compatible = "parade,ps8830"; + reg = <0x8>; + + clocks = <&rpmhcc RPMH_RF_CLK4>; + + vdd-supply = <&vreg_rtmr0_1p15>; + vdd33-supply = <&vreg_rtmr0_3p3>; + vdd33-cap-supply = <&vreg_rtmr0_3p3>; + vddar-supply = <&vreg_rtmr0_1p15>; + vddat-supply = <&vreg_rtmr0_1p15>; + vddio-supply = <&vreg_rtmr0_1p8>; + + reset-gpios = <&pm8550_gpios 10 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&rtmr0_default>; + pinctrl-names = "default"; + + retimer-switch; + orientation-switch; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + retimer_ss0_ss_out: endpoint { + remote-endpoint = <&pmic_glink_ss0_ss_in>; + }; + }; + + port@1 { + reg = <1>; + + retimer_ss0_ss_in: endpoint { + remote-endpoint = <&usb_1_ss0_qmpphy_out>; + }; + }; + + port@2 { + reg = <2>; + + retimer_ss0_con_sbu_out: endpoint { + remote-endpoint = <&pmic_glink_ss0_con_sbu_in>; + }; + }; + }; + }; +}; + +&i2c5 { + clock-frequency = <400000>; + + status = "okay"; + + eusb3_repeater: redriver@47 { + compatible = "nxp,ptn3222"; + reg = <0x47>; + #phy-cells = <0>; + + vdd3v3-supply = <&vreg_l13b_3p0>; + vdd1v8-supply = <&vreg_l4b_1p8>; + + reset-gpios = <&tlmm 6 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&eusb3_reset_n>; + pinctrl-names = "default"; + }; + + eusb5_repeater: redriver@43 { + compatible = "nxp,ptn3222"; + reg = <0x43>; + #phy-cells = <0>; + + vdd3v3-supply = <&vreg_l13b_3p0>; + vdd1v8-supply = <&vreg_l4b_1p8>; + + reset-gpios = <&tlmm 7 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&eusb5_reset_n>; + pinctrl-names = "default"; + }; + + eusb6_repeater: redriver@4f { + compatible = "nxp,ptn3222"; + reg = <0x4f>; + #phy-cells = <0>; + + vdd3v3-supply = <&vreg_l13b_3p0>; + vdd1v8-supply = <&vreg_l4b_1p8>; + + reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&eusb6_reset_n>; + pinctrl-names = "default"; + }; +}; + +&i2c7 { + clock-frequency = <400000>; + + status = "okay"; + + typec-mux@8 { + compatible = "parade,ps8830"; + reg = <0x8>; + + clocks = <&rpmhcc RPMH_RF_CLK4>; + + vdd-supply = <&vreg_rtmr1_1p15>; + vdd33-supply = <&vreg_rtmr1_3p3>; + vdd33-cap-supply = <&vreg_rtmr1_3p3>; + vddar-supply = <&vreg_rtmr1_1p15>; + vddat-supply = <&vreg_rtmr1_1p15>; + vddio-supply = <&vreg_rtmr1_1p8>; + + reset-gpios = <&tlmm 176 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&rtmr1_default>; + pinctrl-names = "default"; + + retimer-switch; + orientation-switch; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + retimer_ss1_ss_out: endpoint { + remote-endpoint = <&pmic_glink_ss1_ss_in>; + }; + }; + + port@1 { + reg = <1>; + + retimer_ss1_ss_in: endpoint { + remote-endpoint = <&usb_1_ss1_qmpphy_out>; + }; + }; + + port@2 { + reg = <2>; + + retimer_ss1_con_sbu_out: endpoint { + remote-endpoint = <&pmic_glink_ss1_con_sbu_in>; + }; + }; + }; + }; +}; + +&lpass_tlmm { + spkr_0_sd_n_active: spkr-0-sd-n-active-state { + pins = "gpio12"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + spkr_1_sd_n_active: spkr-1-sd-n-active-state { + pins = "gpio13"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + spkr_2_sd_n_active: spkr-2-sd-n-active-state { + pins = "gpio17"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + spkr_3_sd_n_active: spkr-3-sd-n-active-state { + pins = "gpio18"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; +}; + +&lpass_vamacro { + pinctrl-0 = <&dmic01_default>, <&dmic23_default>; + pinctrl-names = "default"; + + vdd-micb-supply = <&vreg_l1b_1p8>; + qcom,dmic-sample-rate = <4800000>; +}; + +&mdss { + status = "okay"; +}; + +&mdss_dp0 { + status = "okay"; +}; + +&mdss_dp0_out { + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; +}; + +&mdss_dp1 { + status = "okay"; +}; + +&mdss_dp1_out { + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; +}; + +&mdss_dp2 { + status = "okay"; +}; + +&mdss_dp2_out { + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; +}; + +&mdss_dp3 { + /delete-property/ #sound-dai-cells; + + pinctrl-0 = <&edp0_hpd_default>; + pinctrl-names = "default"; + + status = "okay"; + + aux-bus { + panel { + compatible = "edp-panel"; + + backlight = <&backlight>; + power-supply = <&vreg_edp_3p3>; + + port { + edp_panel_in: endpoint { + remote-endpoint = <&mdss_dp3_out>; + }; + }; + }; + }; +}; + +&mdss_dp3_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + + remote-endpoint = <&edp_panel_in>; +}; + +&mdss_dp3_phy { + vdda-phy-supply = <&vreg_l3j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; + + status = "okay"; +}; + +&pcie3_port0 { + vpcie12v-supply = <&vreg_pcie_12v>; + vpcie3v3-supply = <&vreg_pcie_3v3>; + vpcie3v3aux-supply = <&vreg_pcie_3v3_aux>; + + reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>; +}; + +&pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + + wifi@0 { + compatible = "pci17cb,1107"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + }; +}; + +&pcie5 { + vddpe-3v3-supply = <&vreg_wwan>; +}; + +&pcie5_port0 { + reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; +}; + +&pcie6a { + vddpe-3v3-supply = <&vreg_nvme>; +}; + +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + +&pm8550_gpios { + rtmr0_default: rtmr0-reset-n-active-state { + pins = "gpio10"; + function = "normal"; + power-source = <1>; /* 1.8V */ + bias-disable; + input-disable; + output-enable; + }; + + usb0_3p3_reg_en: usb0-3p3-reg-en-state { + pins = "gpio11"; + function = "normal"; + power-source = <1>; /* 1.8V */ + bias-disable; + input-disable; + output-enable; + }; +}; + +&pm8550ve_8_gpios { + pcie_x8_12v: pcie-12v-default-state { + pins = "gpio8"; + function = "normal"; + output-enable; + output-high; + bias-pull-down; + power-source = <0>; + }; +}; + +&pm8550ve_9_gpios { + usb0_1p8_reg_en: usb0-1p8-reg-en-state { + pins = "gpio8"; + function = "normal"; + power-source = <1>; /* 1.8V */ + bias-disable; + input-disable; + output-enable; + }; + + usb3_en: usb3-en-state { + pins = "gpio4"; + function = "normal"; + qcom,drive-strength = ; + output-enable; + power-source = <0>; + }; + + usb6_en: usb6-en-state { + pins = "gpio5"; + function = "normal"; + qcom,drive-strength = ; + output-enable; + power-source = <0>; + }; +}; + +&pm8550_pwm { + status = "okay"; + + multi-led { + color = ; + function = LED_FUNCTION_STATUS; + + #address-cells = <1>; + #size-cells = <0>; + + led@1 { + reg = <1>; + color = ; + }; + + led@2 { + reg = <2>; + color = ; + }; + }; +}; + +&pmc8380_3_gpios { + edp_bl_en: edp-bl-en-state { + pins = "gpio4"; + function = "normal"; + power-source = <1>; + input-disable; + output-enable; + }; + + edp_bl_reg_en: edp-bl-reg-en-state { + pins = "gpio10"; + function = "normal"; + }; + + pm_sde7_aux_3p3_en: pcie-aux-3p3-default-state { + pins = "gpio8"; + function = "normal"; + output-enable; + bias-pull-down; + power-source = <0>; + }; + + pm_sde7_main_3p3_en: pcie-main-3p3-default-state { + pins = "gpio6"; + function = "normal"; + output-enable; + bias-pull-down; + power-source = <0>; + }; +}; + +&pmc8380_5_gpios { + usb0_pwr_1p15_reg_en: usb0-pwr-1p15-reg-en-state { + pins = "gpio8"; + function = "normal"; + power-source = <1>; /* 1.8V */ + bias-disable; + input-disable; + output-enable; + }; +}; + +&pmk8550_gpios { + edp_bl_pwm: edp-bl-pwm-state { + pins = "gpio5"; + function = "func3"; + }; +}; + +&pmk8550_pwm { + status = "okay"; +}; + +&smb2360_0 { + status = "okay"; +}; + +&smb2360_0_eusb2_repeater { + vdd18-supply = <&vreg_l3d_1p8>; + vdd3-supply = <&vreg_l2b_3p0>; +}; + +&smb2360_1 { + status = "okay"; +}; + +&smb2360_1_eusb2_repeater { + vdd18-supply = <&vreg_l3d_1p8>; + vdd3-supply = <&vreg_l14b_3p0>; +}; + +&smb2360_2 { + status = "okay"; +}; + +&smb2360_2_eusb2_repeater { + vdd18-supply = <&vreg_l3d_1p8>; + vdd3-supply = <&vreg_l8b_3p0>; +}; + +&spi11 { + status = "okay"; + + tpm@0 { + compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi"; + reg = <0>; + spi-max-frequency = <20000000>; + }; +}; + +&swr0 { + status = "okay"; + + pinctrl-0 = <&wsa_swr_active>; + pinctrl-names = "default"; + + /* WSA8845, Left Woofer */ + left_woofer: speaker@0,0 { + compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_0_sd_n_active>; + pinctrl-names = "default"; + reg = <0 0>; + reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "WooferLeft"; + vdd-1p8-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l12b_1p2>; + qcom,port-mapping = <1 2 3 7 10 13>; + }; + + /* WSA8845, Left Tweeter */ + left_tweeter: speaker@0,1 { + compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_1_sd_n_active>; + pinctrl-names = "default"; + reg = <0 1>; + reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "TweeterLeft"; + vdd-1p8-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l12b_1p2>; + qcom,port-mapping = <4 5 6 7 11 13>; + }; +}; + +&swr1 { + status = "okay"; + + /* WCD9385 RX */ + wcd_rx: codec@0,4 { + compatible = "sdw20217010d00"; + reg = <0 4>; + qcom,rx-port-mapping = <1 2 3 4 5>; + }; +}; + +&swr2 { + status = "okay"; + + /* WCD9385 TX */ + wcd_tx: codec@0,3 { + compatible = "sdw20217010d00"; + reg = <0 3>; + qcom,tx-port-mapping = <2 2 3 4>; + }; +}; + +&swr3 { + status = "okay"; + + pinctrl-0 = <&wsa2_swr_active>; + pinctrl-names = "default"; + + /* WSA8845, Right Woofer */ + right_woofer: speaker@0,0 { + compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_2_sd_n_active>; + pinctrl-names = "default"; + reg = <0 0>; + reset-gpios = <&lpass_tlmm 17 GPIO_ACTIVE_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "WooferRight"; + vdd-1p8-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l12b_1p2>; + qcom,port-mapping = <1 2 3 7 10 13>; + }; + + /* WSA8845, Right Tweeter */ + right_tweeter: speaker@0,1 { + compatible = "sdw20217020400"; + pinctrl-0 = <&spkr_3_sd_n_active>; + pinctrl-names = "default"; + reg = <0 1>; + reset-gpios = <&lpass_tlmm 18 GPIO_ACTIVE_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "TweeterRight"; + vdd-1p8-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l12b_1p2>; + qcom,port-mapping = <4 5 6 7 11 13>; + }; +}; + +&tlmm { + edp_reg_en: edp-reg-en-state { + pins = "gpio70"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + eusb3_reset_n: eusb3-reset-n-state { + pins = "gpio6"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + output-low; + }; + + eusb5_reset_n: eusb5-reset-n-state { + pins = "gpio7"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + output-low; + }; + + eusb6_reset_n: eusb6-reset-n-state { + pins = "gpio184"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + output-low; + }; + + nvme_reg_en: nvme-reg-en-state { + pins = "gpio18"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + rtmr1_default: rtmr1-reset-n-active-state { + pins = "gpio176"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + rtmr2_default: rtmr2-reset-n-active-state { + pins = "gpio185"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + usb1_pwr_1p15_reg_en: usb1-pwr-1p15-reg-en-state { + pins = "gpio188"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + usb1_pwr_1p8_reg_en: usb1-pwr-1p8-reg-en-state { + pins = "gpio175"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + usb1_pwr_3p3_reg_en: usb1-pwr-3p3-reg-en-state { + pins = "gpio186"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + usb2_pwr_1p15_reg_en: usb2-pwr-1p15-reg-en-state { + pins = "gpio189"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + usb2_pwr_1p8_reg_en: usb2-pwr-1p8-reg-en-state { + pins = "gpio126"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + usb2_pwr_3p3_reg_en: usb2-pwr-3p3-reg-en-state { + pins = "gpio187"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + usb_1_ss0_sbu_default: usb-1-ss0-sbu-state { + mode-pins { + pins = "gpio166"; + function = "gpio"; + bias-disable; + drive-strength = <2>; + output-high; + }; + + oe-n-pins { + pins = "gpio168"; + function = "gpio"; + bias-disable; + drive-strength = <2>; + }; + + sel-pins { + pins = "gpio167"; + function = "gpio"; + bias-disable; + drive-strength = <2>; + }; + }; + + wcd_default: wcd-reset-n-active-state { + pins = "gpio191"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + output-low; + }; + + wcn_bt_en: wcn-bt-en-state { + pins = "gpio116"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wcn_wlan_en: wcn-wlan-en-state { + pins = "gpio117"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wwan_sw_en: wwan-sw-en-state { + pins = "gpio221"; + function = "gpio"; + drive-strength = <4>; + bias-disable; + }; + + wcn_sw_en: wcn-sw-en-state { + pins = "gpio214"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + /* Switches USB signal routing between the USB connector and the Wi-Fi card. */ + wcn_usb_sw_n: wcn-usb-sw-n-state { + pins = "gpio225"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + output-high; + }; +}; + +&uart14 { + status = "okay"; + + bluetooth { + compatible = "qcom,wcn7850-bt"; + max-speed = <3200000>; + + vddaon-supply = <&vreg_wcn_3p3>; + vddwlcx-supply = <&vreg_wcn_3p3>; + vddwlmx-supply = <&vreg_wcn_3p3>; + vddrfacmn-supply = <&vreg_wcn_3p3>; + vddrfa0p8-supply = <&vreg_wcn_3p3>; + vddrfa1p2-supply = <&vreg_wcn_3p3>; + vddrfa1p8-supply = <&vreg_wcn_3p3>; + }; +}; + +&uart21 { + compatible = "qcom,geni-debug-uart"; + + status = "okay"; +}; + +&ufs_mem_hc { + reset-gpios = <&tlmm 238 GPIO_ACTIVE_LOW>; + + vcc-supply = <&vreg_l17b_2p5>; + vcc-max-microamp = <1300000>; + vccq-supply = <&vreg_l2i_1p2>; + vccq-max-microamp = <1200000>; + + status = "okay"; +}; + +&ufs_mem_phy { + vdda-phy-supply = <&vreg_l3i_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&usb_1_ss0_dwc3_hs { + remote-endpoint = <&pmic_glink_ss0_hs_in>; +}; + +&usb_1_ss0_hsphy { + phys = <&smb2360_0_eusb2_repeater>; +}; + +&usb_1_ss0_qmpphy_out { + remote-endpoint = <&retimer_ss0_ss_in>; +}; + +&usb_1_ss1_dwc3_hs { + remote-endpoint = <&pmic_glink_ss1_hs_in>; +}; + +&usb_1_ss1_hsphy { + phys = <&smb2360_1_eusb2_repeater>; +}; + +&usb_1_ss1_qmpphy_out { + remote-endpoint = <&retimer_ss1_ss_in>; +}; + +&usb_1_ss2_dwc3_hs { + remote-endpoint = <&pmic_glink_ss2_hs_in>; +}; + +&usb_1_ss2_hsphy { + phys = <&smb2360_2_eusb2_repeater>; +}; + +&usb_1_ss2_qmpphy_out { + remote-endpoint = <&retimer_ss2_ss_in>; +}; + +&usb_2_hsphy { + phys = <&eusb5_repeater>; + + pinctrl-0 = <&wcn_usb_sw_n>; + pinctrl-names = "default"; +}; + +&usb_mp_hsphy0 { + phys = <&eusb3_repeater>; +}; + +&usb_mp_hsphy1 { + phys = <&eusb6_repeater>; +}; diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi new file mode 100644 index 0000000000000..339e5242283d9 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi @@ -0,0 +1,704 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "purwa.dtsi" +#include "hamoa-pmics.dtsi" +#include +#include + +/delete-node/ &pmc8380_6; +/delete-node/ &pmc8380_6_thermal; + +/ { + reserved-memory { + linux,cma { + compatible = "shared-dma-pool"; + size = <0x0 0x8000000>; + reusable; + linux,cma-default; + }; + }; +}; + +&apps_rsc { + /* PMC8380C_B */ + regulators-0 { + compatible = "qcom,pm8550-rpmh-regulators"; + qcom,pmic-id = "b"; + + vdd-bob1-supply = <&vph_pwr>; + vdd-bob2-supply = <&vph_pwr>; + vdd-l1-l4-l10-supply = <&vreg_s4c_1p8>; + vdd-l2-l13-l14-supply = <&vreg_bob1>; + vdd-l5-l16-supply = <&vreg_bob1>; + vdd-l6-l7-supply = <&vreg_bob2>; + vdd-l8-l9-supply = <&vreg_bob1>; + vdd-l12-supply = <&vreg_s5j_1p2>; + vdd-l15-supply = <&vreg_s4c_1p8>; + vdd-l17-supply = <&vreg_bob2>; + + vreg_bob1: bob1 { + regulator-name = "vreg_bob1"; + regulator-min-microvolt = <3008000>; + regulator-max-microvolt = <3960000>; + regulator-initial-mode = ; + }; + + vreg_bob2: bob2 { + regulator-name = "vreg_bob2"; + regulator-min-microvolt = <2504000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + }; + + vreg_l1b_1p8: ldo1 { + regulator-name = "vreg_l1b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l2b_3p0: ldo2 { + regulator-name = "vreg_l2b_3p0"; + regulator-min-microvolt = <3072000>; + regulator-max-microvolt = <3100000>; + regulator-initial-mode = ; + }; + + vreg_l4b_1p8: ldo4 { + regulator-name = "vreg_l4b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l5b_3p0: ldo5 { + regulator-name = "vreg_l5b_3p0"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = ; + }; + + vreg_l6b_1p8: ldo6 { + regulator-name = "vreg_l6b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2960000>; + regulator-initial-mode = ; + }; + + vreg_l7b_2p8: ldo7 { + regulator-name = "vreg_l7b_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + }; + + vreg_l8b_3p0: ldo8 { + regulator-name = "vreg_l8b_3p0"; + regulator-min-microvolt = <3072000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + }; + + vreg_l9b_2p9: ldo9 { + regulator-name = "vreg_l9b_2p9"; + regulator-min-microvolt = <2960000>; + regulator-max-microvolt = <2960000>; + regulator-initial-mode = ; + }; + + vreg_l10b_1p8: ldo10 { + regulator-name = "vreg_l10b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l12b_1p2: ldo12 { + regulator-name = "vreg_l12b_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-always-on; + }; + + vreg_l13b_3p0: ldo13 { + regulator-name = "vreg_l13b_3p0"; + regulator-min-microvolt = <3072000>; + regulator-max-microvolt = <3100000>; + regulator-initial-mode = ; + }; + + vreg_l14b_3p0: ldo14 { + regulator-name = "vreg_l14b_3p0"; + regulator-min-microvolt = <3072000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + }; + + vreg_l15b_1p8: ldo15 { + regulator-name = "vreg_l15b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-always-on; + }; + + vreg_l16b_2p9: ldo16 { + regulator-name = "vreg_l16b_2p9"; + regulator-min-microvolt = <2912000>; + regulator-max-microvolt = <2912000>; + regulator-initial-mode = ; + }; + + vreg_l17b_2p5: ldo17 { + regulator-name = "vreg_l17b_2p5"; + regulator-min-microvolt = <2504000>; + regulator-max-microvolt = <2504000>; + regulator-initial-mode = ; + }; + }; + + /* PMC8380VE_C */ + regulators-1 { + compatible = "qcom,pm8550ve-rpmh-regulators"; + qcom,pmic-id = "c"; + + vdd-l1-supply = <&vreg_s5j_1p2>; + vdd-l2-supply = <&vreg_s1f_0p7>; + vdd-l3-supply = <&vreg_s1f_0p7>; + vdd-s4-supply = <&vph_pwr>; + + vreg_s4c_1p8: smps4 { + regulator-name = "vreg_s4c_1p8"; + regulator-min-microvolt = <1856000>; + regulator-max-microvolt = <2000000>; + regulator-initial-mode = ; + }; + + vreg_l1c_1p2: ldo1 { + regulator-name = "vreg_l1c_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_l2c_0p8: ldo2 { + regulator-name = "vreg_l2c_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_l3c_0p8: ldo3 { + regulator-name = "vreg_l3c_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + }; + + /* PMC8380_D */ + regulators-2 { + compatible = "qcom,pmc8380-rpmh-regulators"; + qcom,pmic-id = "d"; + + vdd-l1-supply = <&vreg_s1f_0p7>; + vdd-l2-supply = <&vreg_s1f_0p7>; + vdd-l3-supply = <&vreg_s4c_1p8>; + vdd-s1-supply = <&vph_pwr>; + + vreg_l1d_0p8: ldo1 { + regulator-name = "vreg_l1d_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_l2d_0p9: ldo2 { + regulator-name = "vreg_l2d_0p9"; + regulator-min-microvolt = <912000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_l3d_1p8: ldo3 { + regulator-name = "vreg_l3d_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + }; + + /* PMC8380_E */ + regulators-3 { + compatible = "qcom,pmc8380-rpmh-regulators"; + qcom,pmic-id = "e"; + + vdd-l2-supply = <&vreg_s1f_0p7>; + vdd-l3-supply = <&vreg_s5j_1p2>; + + vreg_l2e_0p8: ldo2 { + regulator-name = "vreg_l2e_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_l3e_1p2: ldo3 { + regulator-name = "vreg_l3e_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + }; + + /* PMC8380_F */ + regulators-4 { + compatible = "qcom,pmc8380-rpmh-regulators"; + qcom,pmic-id = "f"; + + vdd-l1-supply = <&vreg_s5j_1p2>; + vdd-l2-supply = <&vreg_s5j_1p2>; + vdd-l3-supply = <&vreg_s5j_1p2>; + vdd-s1-supply = <&vph_pwr>; + + vreg_s1f_0p7: smps1 { + regulator-name = "vreg_s1f_0p7"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + }; + + vreg_l1f_1p0: ldo1 { + regulator-name = "vreg_l1f_1p0"; + regulator-min-microvolt = <1024000>; + regulator-max-microvolt = <1024000>; + regulator-initial-mode = ; + }; + + vreg_l2f_1p0: ldo2 { + regulator-name = "vreg_l2f_1p0"; + regulator-min-microvolt = <1024000>; + regulator-max-microvolt = <1024000>; + regulator-initial-mode = ; + }; + + vreg_l3f_1p0: ldo3 { + regulator-name = "vreg_l3f_1p0"; + regulator-min-microvolt = <1024000>; + regulator-max-microvolt = <1024000>; + regulator-initial-mode = ; + }; + }; + + /* PMC8380VE_I */ + regulators-6 { + compatible = "qcom,pm8550ve-rpmh-regulators"; + qcom,pmic-id = "i"; + + vdd-l1-supply = <&vreg_s4c_1p8>; + vdd-l2-supply = <&vreg_s5j_1p2>; + vdd-l3-supply = <&vreg_s1f_0p7>; + vdd-s1-supply = <&vph_pwr>; + vdd-s2-supply = <&vph_pwr>; + + vreg_s1i_0p9: smps1 { + regulator-name = "vreg_s1i_0p9"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_s2i_1p0: smps2 { + regulator-name = "vreg_s2i_1p0"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1100000>; + regulator-initial-mode = ; + }; + + vreg_l1i_1p8: ldo1 { + regulator-name = "vreg_l1i_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + vreg_l2i_1p2: ldo2 { + regulator-name = "vreg_l2i_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + vreg_l3i_0p8: ldo3 { + regulator-name = "vreg_l3i_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + }; + + /* PMC8380VE_J */ + regulators-7 { + compatible = "qcom,pm8550ve-rpmh-regulators"; + qcom,pmic-id = "j"; + + vdd-l1-supply = <&vreg_s1f_0p7>; + vdd-l2-supply = <&vreg_s5j_1p2>; + vdd-l3-supply = <&vreg_s1f_0p7>; + vdd-s5-supply = <&vph_pwr>; + + vreg_s5j_1p2: smps5 { + regulator-name = "vreg_s5j_1p2"; + regulator-min-microvolt = <1256000>; + regulator-max-microvolt = <1304000>; + regulator-initial-mode = ; + }; + + vreg_l1j_0p8: ldo1 { + regulator-name = "vreg_l1j_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + vreg_l2j_1p2: ldo2 { + regulator-name = "vreg_l2j_1p2"; + regulator-min-microvolt = <1256000>; + regulator-max-microvolt = <1256000>; + regulator-initial-mode = ; + }; + + vreg_l3j_0p8: ldo3 { + regulator-name = "vreg_l3j_0p8"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + }; + + /* PM8010_M */ + regulators-8 { + compatible = "qcom,pm8010-rpmh-regulators"; + qcom,pmic-id = "m"; + + vdd-l3-l4-supply = <&vreg_s4c_1p8>; + + vreg_l4m_1p8: ldo4 { + regulator-name = "vrer_l4m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1808000>; + regulator-initial-mode = ; + }; + }; +}; + +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/x1p42100/gen71500_zap.mbn"; +}; + +&iris { + status = "okay"; +}; + +&pcie3 { + pinctrl-0 = <&pcie3_default>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie3_phy { + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&pcie4 { + pinctrl-0 = <&pcie4_default>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie4_phy { + vdda-phy-supply = <&vreg_l3i_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&pcie5 { + pinctrl-0 = <&pcie5_default>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie5_phy { + vdda-phy-supply = <&vreg_l3i_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&pcie6a { + pinctrl-0 = <&pcie6a_default>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie6a_phy { + vdda-phy-supply = <&vreg_l1d_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; + + status = "okay"; +}; + +&qupv3_0 { + status = "okay"; +}; + +&qupv3_1 { + status = "okay"; +}; + +&qupv3_2 { + status = "okay"; +}; + +&remoteproc_adsp { + firmware-name = "qcom/x1e80100/adsp.mbn", + "qcom/x1e80100/adsp_dtb.mbn"; + + status = "okay"; +}; + +&remoteproc_cdsp { + firmware-name = "qcom/x1e80100/cdsp.mbn", + "qcom/x1e80100/cdsp_dtb.mbn"; + + status = "okay"; +}; + +&tlmm { + gpio-reserved-ranges = <34 2>; /* TPM LP & INT */ + + pcie3_default: pcie3-default-state { + clkreq-n-pins { + pins = "gpio144"; + function = "pcie3_clk"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-n-pins { + pins = "gpio143"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-n-pins { + pins = "gpio145"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + pcie4_default: pcie4-default-state { + clkreq-n-pins { + pins = "gpio147"; + function = "pcie4_clk"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-n-pins { + pins = "gpio146"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-n-pins { + pins = "gpio148"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + pcie5_default: pcie5-default-state { + clkreq-n-pins { + pins = "gpio150"; + function = "pcie5_clk"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-n-pins { + pins = "gpio149"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-n-pins { + pins = "gpio151"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + pcie6a_default: pcie6a-default-state { + clkreq-n-pins { + pins = "gpio153"; + function = "pcie6a_clk"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-n-pins { + pins = "gpio152"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-n-pins { + pins = "gpio154"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + + }; + }; +}; + +&usb_1_ss0 { + status = "okay"; +}; + +&usb_1_ss0_dwc3 { + dr_mode = "otg"; + usb-role-switch; +}; + +&usb_1_ss0_hsphy { + vdd-supply = <&vreg_l3j_0p8>; + vdda12-supply = <&vreg_l2j_1p2>; + + status = "okay"; +}; + +&usb_1_ss0_qmpphy { + vdda-phy-supply = <&vreg_l2j_1p2>; + vdda-pll-supply = <&vreg_l1j_0p8>; + + status = "okay"; +}; + +&usb_1_ss1 { + status = "okay"; +}; + +&usb_1_ss1_dwc3 { + dr_mode = "otg"; + usb-role-switch; +}; + +&usb_1_ss1_hsphy { + vdd-supply = <&vreg_l3j_0p8>; + vdda12-supply = <&vreg_l2j_1p2>; + + status = "okay"; +}; + +&usb_1_ss1_qmpphy { + vdda-phy-supply = <&vreg_l2j_1p2>; + vdda-pll-supply = <&vreg_l2d_0p9>; + + status = "okay"; +}; + +&usb_1_ss2 { + status = "okay"; +}; + +&usb_1_ss2_dwc3 { + dr_mode = "otg"; + usb-role-switch; +}; + +&usb_1_ss2_hsphy { + vdd-supply = <&vreg_l3j_0p8>; + vdda12-supply = <&vreg_l2j_1p2>; + + status = "okay"; +}; + +&usb_1_ss2_qmpphy { + vdda-phy-supply = <&vreg_l2j_1p2>; + vdda-pll-supply = <&vreg_l2d_0p9>; + + status = "okay"; +}; + +&usb_2 { + status = "okay"; +}; + +&usb_2_dwc3 { + dr_mode = "host"; +}; + +&usb_2_hsphy { + vdd-supply = <&vreg_l2e_0p8>; + vdda12-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&usb_mp { + status = "okay"; +}; + +&usb_mp_hsphy0 { + vdd-supply = <&vreg_l2e_0p8>; + vdda12-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&usb_mp_hsphy1 { + vdd-supply = <&vreg_l2e_0p8>; + vdda12-supply = <&vreg_l3e_1p2>; + + status = "okay"; +}; + +&usb_mp_qmpphy0 { + vdda-phy-supply = <&vreg_l3e_1p2>; + vdda-pll-supply = <&vreg_l3c_0p8>; + + status = "okay"; +}; + +&usb_mp_qmpphy1 { + vdda-phy-supply = <&vreg_l3e_1p2>; + vdda-pll-supply = <&vreg_l3c_0p8>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/x1p42100.dtsi b/arch/arm64/boot/dts/qcom/purwa.dtsi similarity index 69% rename from arch/arm64/boot/dts/qcom/x1p42100.dtsi rename to arch/arm64/boot/dts/qcom/purwa.dtsi index 10d26958d3c66..10d2207ce81e9 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100.dtsi +++ b/arch/arm64/boot/dts/qcom/purwa.dtsi @@ -3,8 +3,10 @@ * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. */ -/* X1P42100 is heavily based on X1E80100, with some meaningful differences */ -#include "x1e80100.dtsi" +/* X1P42100 is heavily based on hamoa, with some meaningful differences */ +#include "hamoa.dtsi" + +#include /delete-node/ &bwmon_cluster0; /delete-node/ &cluster_pd2; @@ -17,15 +19,194 @@ /delete-node/ &cpu_pd9; /delete-node/ &cpu_pd10; /delete-node/ &cpu_pd11; +/delete-node/ &csiphy1; +/delete-node/ &csiphy2; /delete-node/ &gpu_opp_table; /delete-node/ &gpu_speed_bin; +/delete-node/ &iris_opp_table; /delete-node/ &pcie3_phy; /delete-node/ &thermal_zones; +/delete-node/ &etm8; +/delete-node/ &etm9; +/delete-node/ &etm10; +/delete-node/ &etm11; +/delete-node/ &cluster2_funnel_l1; +/delete-node/ &cluster2_funnel_l2; +/delete-node/ &cluster2_etf; +/delete-node/ &cluster2_rep_2_0; +/delete-node/ &cluster2_rep_2_1; +/delete-node/ &cluster2_rep_2_2; +/delete-node/ &cluster2_rep_2_3; +/delete-node/ &apss_funnel_in2; + +&camss { + compatible = "qcom,x1p42100-camss"; + + reg = <0 0x0acb7000 0 0x2000>, + <0 0x0acb9000 0 0x2000>, + <0 0x0acbb000 0 0x2000>, + <0 0x0acc6000 0 0x1000>, + <0 0x0acca000 0 0x1000>, + <0 0x0acb6000 0 0x1000>, + <0 0x0ace4000 0 0x1000>, + <0 0x0acec000 0 0x4000>, + <0 0x0acf6000 0 0x1000>, + <0 0x0acf7000 0 0x1000>, + <0 0x0acf8000 0 0x1000>, + <0 0x0ac62000 0 0x4000>, + <0 0x0acc7000 0 0x2000>, + <0 0x0accb000 0 0x2000>; + + reg-names = "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csid_wrapper", + "csiphy0", + "csiphy4", + "csitpg0", + "csitpg1", + "csitpg2", + "vfe0", + "vfe_lite0", + "vfe_lite1"; + + interrupts = , + , + , + , + , + , + , + , + , + ; + + interrupt-names = "csid0", + "csid1", + "csid2", + "csid_lite0", + "csid_lite1", + "csiphy0", + "csiphy4", + "vfe0", + "vfe_lite0", + "vfe_lite1"; + + clocks = <&camcc CAM_CC_CAMNOC_AXI_NRT_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_RT_CLK>, + <&camcc CAM_CC_CORE_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPAS_FAST_AHB_CLK>, + <&camcc CAM_CC_CPAS_IFE_0_CLK>, + <&camcc CAM_CC_CPAS_IFE_LITE_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSID_CLK>, + <&camcc CAM_CC_CSID_CSIPHY_RX_CLK>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_FAST_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_AHB_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>; + + clock-names = "camnoc_nrt_axi", + "camnoc_rt_axi", + "core_ahb", + "cpas_ahb", + "cpas_fast_ahb", + "cpas_vfe0", + "cpas_vfe_lite", + "cphy_rx_clk_src", + "csid", + "csid_csiphy_rx", + "csiphy0", + "csiphy0_timer", + "csiphy4", + "csiphy4_timer", + "gcc_axi_hf", + "gcc_axi_sf", + "vfe0", + "vfe0_fast_ahb", + "vfe_lite", + "vfe_lite_ahb", + "vfe_lite_cphy_rx", + "vfe_lite_csid"; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_ICP QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "ahb", + "hf_mnoc", + "sf_mnoc", + "sf_icp_mnoc"; + + iommus = <&apps_smmu 0x800 0x60>, + <&apps_smmu 0x860 0x60>, + <&apps_smmu 0x1860 0x60>, + <&apps_smmu 0x18e0 0x00>, + <&apps_smmu 0x19a0 0x20>; + + power-domains = <&camcc CAM_CC_IFE_0_GDSC>, + <&camcc CAM_CC_TITAN_TOP_GDSC>; + power-domain-names = "ife0", + "top"; + + phys = <&csiphy0 PHY_TYPE_DPHY>, + <&csiphy4 PHY_TYPE_DPHY>; + phy-names = "csiphy0", + "csiphy4"; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + camss_csiphy0_inep0: endpoint@0 { + reg = <0>; + }; + }; + + port@3 { + reg = <3>; + #address-cells = <1>; + #size-cells = <0>; + camss_csiphy4_inep0: endpoint@0 { + reg = <0>; + }; + }; + }; +}; + +&camcc { + compatible = "qcom,x1p42100-camcc"; +}; &gcc { compatible = "qcom,x1p42100-gcc", "qcom,x1e80100-gcc"; }; +&videocc { + compatible = "qcom,x1p42100-videocc"; +}; + &gmu { compatible = "qcom,adreno-gmu-x145.0", "qcom,adreno-gmu"; }; @@ -33,6 +214,8 @@ &gpu { compatible = "qcom,adreno-43030c00", "qcom,adreno"; + iommus = <&adreno_smmu 0 0x0>; + nvmem-cells = <&gpu_speed_bin>; nvmem-cell-names = "speed_bin"; @@ -143,6 +326,55 @@ compatible = "qcom,x1p42100-gpucc"; }; +&iris { + compatible = "qcom,x1p42100-iris"; + + clocks = <&gcc GCC_VIDEO_AXI0_CLK>, + <&videocc VIDEO_CC_MVS0C_CLK>, + <&videocc VIDEO_CC_MVS0_CLK>, + <&videocc VIDEO_CC_MVS0_BSE_CLK>; + clock-names = "iface", + "core", + "vcodec0_core", + "vcodec0_bse"; + + operating-points-v2 = <&iris_opp_table_x1p42100>; + + iris_opp_table_x1p42100: opp-table { + compatible = "operating-points-v2"; + + opp-210000000 { + opp-hz = /bits/ 64 <210000000 105000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>, + <&rpmhpd_opp_low_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000 150000000>; + required-opps = <&rpmhpd_opp_low_svs_d1>, + <&rpmhpd_opp_svs>; + }; + + opp-335000000 { + opp-hz = /bits/ 64 <335000000 167500000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_svs_l1>; + }; + + opp-424000000 { + opp-hz = /bits/ 64 <424000000 212000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_nom>; + }; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000 250000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_turbo>; + }; + }; +}; + /* PCIe3 has half the lanes compared to X1E80100 */ &pcie3 { num-lanes = <4>; @@ -152,6 +384,10 @@ compatible = "qcom,x1p42100-qmp-gen4x4-pcie-phy"; }; +&pdc { + compatible = "qcom,x1p42100-pdc", "qcom,pdc"; +}; + &qfprom { gpu_speed_bin: gpu-speed-bin@119 { reg = <0x119 0x2>; diff --git a/arch/arm64/boot/dts/qcom/qcm2290.dtsi b/arch/arm64/boot/dts/qcom/qcm2290.dtsi index 08141b41de246..e0e400fdd2497 100644 --- a/arch/arm64/boot/dts/qcom/qcm2290.dtsi +++ b/arch/arm64/boot/dts/qcom/qcm2290.dtsi @@ -1503,8 +1503,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-07000200", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; interrupts = ; @@ -1685,25 +1689,25 @@ }; }; - camss: camss@5c6e000 { + camss: camss@5c11000 { compatible = "qcom,qcm2290-camss"; - reg = <0x0 0x5c6e000 0x0 0x1000>, + reg = <0x0 0x5c11000 0x0 0x1000>, + <0x0 0x5c6e000 0x0 0x1000>, <0x0 0x5c75000 0x0 0x1000>, <0x0 0x5c52000 0x0 0x1000>, <0x0 0x5c53000 0x0 0x1000>, <0x0 0x5c66000 0x0 0x400>, <0x0 0x5c68000 0x0 0x400>, - <0x0 0x5c11000 0x0 0x1000>, <0x0 0x5c6f000 0x0 0x4000>, <0x0 0x5c76000 0x0 0x4000>; - reg-names = "csid0", + reg-names = "top", + "csid0", "csid1", "csiphy0", "csiphy1", "csitpg0", "csitpg1", - "top", "vfe0", "vfe1"; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts index 519e458e1a890..50d38f902433f 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts @@ -47,6 +47,8 @@ stride = <(1224 * 4)>; format = "a8r8g8b8"; clocks = <&gcc GCC_DISP_HF_AXI_CLK>; + vci-supply = <&vreg_oled_vci>; + dvdd-supply = <&vreg_oled_dvdd>; }; }; @@ -95,7 +97,7 @@ reg = <0>; pmic_glink_hs_in: endpoint { - remote-endpoint = <&usb_1_dwc3_hs>; + remote-endpoint = <&eud_con>; }; }; @@ -652,6 +654,10 @@ }; }; +&eud_con { + remote-endpoint = <&pmic_glink_hs_in>; +}; + &gcc { protected-clocks = , , @@ -1386,10 +1392,6 @@ status = "okay"; }; -&usb_1_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in>; -}; - &usb_1_hsphy { vdda-pll-supply = <&vreg_l10c>; vdda18-supply = <&vreg_l1c>; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts index 73fce639370cd..4b591a3b39951 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts @@ -36,6 +36,7 @@ aliases { serial0 = &uart5; + serial1 = &uart7; }; pm8350c_pwm_backlight: backlight { @@ -177,7 +178,7 @@ pinctrl-0 = <&wcd_default>; pinctrl-names = "default"; - reset-gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>; + reset-gpios = <&tlmm 83 GPIO_ACTIVE_LOW>; vdd-buck-supply = <&vreg_l17b_1p7>; vdd-rxtx-supply = <&vreg_l18b_1p8>; @@ -194,6 +195,63 @@ #sound-dai-cells = <1>; }; + + wcn6750-pmu { + compatible = "qcom,wcn6750-pmu"; + pinctrl-0 = <&bt_en>; + pinctrl-names = "default"; + vddaon-supply = <&vreg_s7b_0p972>; + vddasd-supply = <&vreg_l11c_2p8>; + vddpmu-supply = <&vreg_s7b_0p972>; + vddrfa0p8-supply = <&vreg_s7b_0p972>; + vddrfa1p2-supply = <&vreg_s8b_1p272>; + vddrfa1p7-supply = <&vreg_s1b_1p872>; + vddrfa2p2-supply = <&vreg_s1c_2p19>; + + bt-enable-gpios = <&tlmm 85 GPIO_ACTIVE_HIGH>; + + regulators { + vreg_pmu_rfa_cmn: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn"; + }; + + vreg_pmu_aon_0p59: ldo1 { + regulator-name = "vreg_pmu_aon_0p59"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p85: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p85"; + }; + + vreg_pmu_btcmx_0p85: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p85"; + }; + + vreg_pmu_rfa_0p8: ldo5 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo6 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p7: ldo7 { + regulator-name = "vreg_pmu_rfa_1p7"; + }; + + vreg_pmu_pcie_0p9: ldo8 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_pcie_1p8: ldo9 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + }; + }; }; &apps_rsc { @@ -533,12 +591,7 @@ }; &gcc { - protected-clocks = ,, - , , - , , - , , - , , - , , + protected-clocks = , , ,, , , , @@ -615,6 +668,22 @@ status = "okay"; }; +&pcie1 { + perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>, <&pcie1_clkreq_n>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie1_phy { + vdda-phy-supply = <&vreg_l10c_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; + + status = "okay"; +}; + &pm7250b_gpios { lcd_disp_bias_en: lcd-disp-bias-en-state { pins = "gpio2"; @@ -695,6 +764,46 @@ status = "okay"; }; +&psci { + reboot-mode { + mode-bootloader = <0x10001 0x2>; + mode-edl = <0 0x1>; + }; +}; + +&qup_uart7_cts { + /* + * Configure a bias-bus-hold on CTS to lower power + * usage when Bluetooth is turned off. Bus hold will + * maintain a low power state regardless of whether + * the Bluetooth module drives the pin in either + * direction or leaves the pin fully unpowered. + */ + bias-bus-hold; +}; + +&qup_uart7_rts { + /* We'll drive RTS, so no pull */ + drive-strength = <2>; + bias-disable; +}; + +&qup_uart7_rx { + /* + * Configure a pull-up on RX. This is needed to avoid + * garbage data when the TX pin of the Bluetooth module is + * in tri-state (module powered off or not driving the + * signal yet). + */ + bias-pull-up; +}; + +&qup_uart7_tx { + /* We'll drive TX, so no pull */ + drive-strength = <2>; + bias-disable; +}; + &qupv3_id_0 { status = "okay"; }; @@ -920,6 +1029,75 @@ gpio-reserved-ranges = <32 2>, /* ADSP */ <48 4>; /* NFC */ + bt_en: bt-en-state { + pins = "gpio85"; + function = "gpio"; + output-low; + bias-disable; + }; + + qup_uart7_sleep_cts: qup-uart7-sleep-cts-state { + pins = "gpio28"; + function = "gpio"; + /* + * Configure a bias-bus-hold on CTS to lower power + * usage when Bluetooth is turned off. Bus hold will + * maintain a low power state regardless of whether + * the Bluetooth module drives the pin in either + * direction or leaves the pin fully unpowered. + */ + bias-bus-hold; + }; + + qup_uart7_sleep_rts: qup-uart7-sleep-rts-state { + pins = "gpio29"; + function = "gpio"; + /* + * Configure pull-down on RTS. As RTS is active low + * signal, pull it low to indicate the BT SoC that it + * can wakeup the system anytime from suspend state by + * pulling RX low (by sending wakeup bytes). + */ + bias-pull-down; + }; + + qup_uart7_sleep_rx: qup-uart7-sleep-rx-state { + pins = "gpio31"; + function = "gpio"; + /* + * Configure a pull-up on RX. This is needed to avoid + * garbage data when the TX pin of the Bluetooth module + * is floating which may cause spurious wakeups. + */ + bias-pull-up; + }; + + qup_uart7_sleep_tx: qup-uart7-sleep-tx-state { + pins = "gpio30"; + function = "gpio"; + /* + * Configure pull-up on TX when it isn't actively driven + * to prevent BT SoC from receiving garbage during sleep. + */ + bias-pull-up; + }; + + pcie1_reset_n: pcie1-reset-n-state { + pins = "gpio2"; + function = "gpio"; + drive-strength = <16>; + output-low; + bias-disable; + }; + + pcie1_wake_n: pcie1-wake-n-state { + pins = "gpio3"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + sd_cd: sd-cd-state { pins = "gpio91"; function = "gpio"; @@ -938,6 +1116,31 @@ status = "okay"; }; +&uart7 { + /delete-property/ interrupts; + interrupts-extended = <&intc GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>, + <&tlmm 31 IRQ_TYPE_EDGE_FALLING>; + pinctrl-1 = <&qup_uart7_sleep_cts>, + <&qup_uart7_sleep_rts>, + <&qup_uart7_sleep_tx>, + <&qup_uart7_sleep_rx>; + pinctrl-names = "default", + "sleep"; + + status = "okay"; + + bluetooth: bluetooth { + compatible = "qcom,wcn6750-bt"; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + max-speed = <3200000>; + }; +}; + &ufs_mem_hc { reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>; vcc-supply = <&vreg_l7b_2p952>; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts index 251e72f114287..c8494a9254fcd 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts @@ -74,7 +74,7 @@ reg = <0>; pmic_glink_hs_in: endpoint { - remote-endpoint = <&usb_1_dwc3_hs>; + remote-endpoint = <&eud_con>; }; }; @@ -524,6 +524,10 @@ ; }; +&eud_con { + remote-endpoint = <&pmic_glink_hs_in>; +}; + &gpu { status = "okay"; }; @@ -826,10 +830,6 @@ status = "okay"; }; -&usb_1_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in>; -}; - &usb_1_hsphy { vdda-pll-supply = <&vreg_l10c_0p88>; vdda33-supply = <&vreg_l2b_3p072>; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts index eb8efba1b9dda..9885e1631a87c 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts @@ -84,7 +84,7 @@ reg = <0>; pmic_glink_hs_in: endpoint { - remote-endpoint = <&usb_1_dwc3_hs>; + remote-endpoint = <&eud_con>; }; }; @@ -118,6 +118,11 @@ no-map; }; + removed_mem: removed@c0000000 { + reg = <0x0 0xc0000000 0x0 0x5100000>; + no-map; + }; + rmtfs_mem: rmtfs@f8500000 { compatible = "qcom,rmtfs-mem"; reg = <0x0 0xf8500000 0x0 0x600000>; @@ -485,6 +490,10 @@ }; }; +&eud_con { + remote-endpoint = <&pmic_glink_hs_in>; +}; + &gcc { protected-clocks = , , @@ -916,10 +925,6 @@ status = "okay"; }; -&usb_1_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in>; -}; - &usb_dp_qmpphy_out { remote-endpoint = <&pmic_glink_ss_in>; }; diff --git a/arch/arm64/boot/dts/qcom/qcs5430-fps-camx.dtso b/arch/arm64/boot/dts/qcom/qcs5430-fps-camx.dtso new file mode 100644 index 0000000000000..4255147f00ef9 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs5430-fps-camx.dtso @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* +* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +*/ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "qcs6490-camera.dtsi" +#include "qcs6490-rb3gen2-camera-sensor.dtsi" + +&cam_csid2 { + status = "disabled"; +}; + +&cam_vfe2 { + status = "disabled"; +}; + +&camss { + status = "disabled"; +}; + +&cci1 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs615-ride-camx.dtso b/arch/arm64/boot/dts/qcom/qcs615-ride-camx.dtso new file mode 100644 index 0000000000000..a40d8817a7a20 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs615-ride-camx.dtso @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "talos-camera.dtsi" +#include "talos-camera-sensor.dtsi" diff --git a/arch/arm64/boot/dts/qcom/qcs615-ride.dts b/arch/arm64/boot/dts/qcom/qcs615-ride.dts index 705ea71b07a10..7fdcbc129fce2 100644 --- a/arch/arm64/boot/dts/qcom/qcs615-ride.dts +++ b/arch/arm64/boot/dts/qcom/qcs615-ride.dts @@ -7,10 +7,10 @@ #include #include #include -#include "sm6150.dtsi" +#include "talos.dtsi" #include "pm8150.dtsi" / { - model = "Qualcomm Technologies, Inc. QCS615 Ride"; + model = "Qualcomm Technologies, Inc. QCS615 Ride (IQ-615 Beta EVK)"; compatible = "qcom,qcs615-ride", "qcom,qcs615", "qcom,sm6150"; chassis-type = "embedded"; @@ -39,6 +39,32 @@ }; }; + dp0-connector { + compatible = "dp-connector"; + label = "DP0"; + type = "mini"; + + hpd-gpios = <&io_expander 8 GPIO_ACTIVE_HIGH>; + + port { + dp0_connector_in: endpoint { + remote-endpoint = <&mdss_dp0_out>; + }; + }; + }; + + dp-dsi0-connector { + compatible = "dp-connector"; + label = "DSI0"; + type = "mini"; + + port { + dp_dsi0_connector_in: endpoint { + remote-endpoint = <&dsi2dp_bridge_out>; + }; + }; + }; + vreg_conn_1p8: regulator-conn-1p8 { compatible = "regulator-fixed"; regulator-name = "vreg_conn_1p8"; @@ -65,6 +91,64 @@ regulator-always-on; }; + vreg_12p0: regulator-vreg-12p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_12P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + vreg_1p0: regulator-vreg-1p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_1P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + vin-supply = <&vreg_1p8>; + }; + + vreg_1p8: regulator-vreg-1p8 { + compatible = "regulator-fixed"; + regulator-name = "VREG_1P8"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + vin-supply = <&vreg_5p0>; + }; + + vreg_3p0: regulator-vreg-3p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_3P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + vin-supply = <&vreg_12p0>; + }; + + vreg_5p0: regulator-vreg-5p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_5P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + + vin-supply = <&vreg_12p0>; + }; + wcn6855-pmu { compatible = "qcom,wcn6855-pmu"; @@ -288,6 +372,156 @@ }; }; +ðernet { + pinctrl-0 = <ðernet_defaults>; + pinctrl-names = "default"; + + phy-handle = <&rgmii_phy>; + phy-mode = "rgmii-id"; + + snps,mtl-rx-config = <&mtl_rx_setup>; + snps,mtl-tx-config = <&mtl_tx_setup>; + + status = "okay"; + + mdio: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + rgmii_phy: phy@7 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x7>; + + interrupts-extended = <&tlmm 121 IRQ_TYPE_EDGE_FALLING>; + device_type = "ethernet-phy"; + reset-gpios = <&tlmm 104 GPIO_ACTIVE_LOW>; + reset-assert-us = <11000>; + reset-deassert-us = <70000>; + }; + }; + + mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <1>; + snps,rx-sched-sp; + + queue0 { + snps,dcb-algorithm; + snps,map-to-dma-channel = <0x0>; + snps,route-up; + snps,priority = <0x1>; + }; + }; + + mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <1>; + snps,tx-sched-wrr; + + queue0 { + snps,weight = <0x10>; + snps,dcb-algorithm; + snps,priority = <0x0>; + }; + }; +}; + +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/qcs615/a612_zap.mbn"; +}; + +&i2c2 { + clock-frequency = <400000>; + status = "okay"; + + io_expander: pinctrl@3e { + compatible = "semtech,sx1509q"; + reg = <0x3e>; + interrupts-extended = <&tlmm 58 IRQ_TYPE_EDGE_FALLING>; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + semtech,probe-reset; + }; + + i2c-mux@77 { + compatible = "nxp,pca9542"; + reg = <0x77>; + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + bridge@58 { + compatible = "analogix,anx7625"; + reg = <0x58>; + interrupts-extended = <&io_expander 0 IRQ_TYPE_EDGE_FALLING>; + enable-gpios = <&tlmm 4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&tlmm 5 GPIO_ACTIVE_HIGH>; + vdd10-supply = <&vreg_1p0>; + vdd18-supply = <&vreg_1p8>; + vdd33-supply = <&vreg_3p0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dsi2dp_bridge_in: endpoint { + remote-endpoint = <&mdss_dsi0_out>; + }; + }; + + port@1 { + reg = <1>; + + dsi2dp_bridge_out: endpoint { + remote-endpoint = <&dp_dsi0_connector_in>; + }; + }; + }; + }; + }; + }; +}; + +&mdss { + status = "okay"; +}; + +&mdss_dp0 { + status = "okay"; +}; + +&mdss_dp0_out { + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000>; + remote-endpoint = <&dp0_connector_in>; +}; + +&mdss_dsi0 { + vdda-supply = <&vreg_l11a>; + status = "okay"; +}; + +&mdss_dsi0_out { + remote-endpoint = <&dsi2dp_bridge_in>; + data-lanes = <0 1 2 3>; +}; + +&mdss_dsi0_phy { + vcca-supply = <&vreg_l5a>; + status = "okay"; +}; + &pcie { perst-gpios = <&tlmm 101 GPIO_ACTIVE_LOW>; wake-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>; @@ -343,6 +577,25 @@ status = "okay"; }; +&psci { + reboot-mode { + mode-bootloader = <0x10001 0x2>; + mode-edl = <0 0x1>; + }; +}; + +&qspi { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <25000000>; + spi-tx-bus-width = <2>; + spi-rx-bus-width = <2>; + }; +}; + &qupv3_id_0 { status = "okay"; }; @@ -364,6 +617,55 @@ }; &tlmm { + ethernet_defaults: ethernet-defaults-state { + mdc-pins { + pins = "gpio113"; + function = "rgmii"; + bias-pull-up; + }; + + mdio-pins { + pins = "gpio114"; + function = "rgmii"; + bias-pull-up; + }; + + rgmii-rx-pins { + pins = "gpio81", "gpio82", "gpio83", "gpio102", "gpio103", "gpio112"; + function = "rgmii"; + bias-disable; + drive-strength = <2>; + }; + + rgmii-tx-pins { + pins = "gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio97"; + function = "rgmii"; + bias-pull-up; + drive-strength = <16>; + }; + + phy-intr-pins { + pins = "gpio121"; + function = "gpio"; + bias-disable; + drive-strength = <8>; + }; + + pps-pins { + pins = "gpio91"; + function = "rgmii"; + bias-disable; + drive-strength = <8>; + }; + + phy-reset-pins { + pins = "gpio104"; + function = "gpio"; + bias-pull-up; + drive-strength = <16>; + }; + }; + bt_en_state: bt-en-state { pins = "gpio85"; function = "gpio"; @@ -397,6 +699,7 @@ wlan_en_state: wlan-en-state { pins = "gpio98"; function = "gpio"; + drive-strength = <16>; bias-pull-down; output-low; }; @@ -458,7 +761,7 @@ }; }; -&usb_1_hsphy { +&usb_hsphy_1 { vdd-supply = <&vreg_l5a>; vdda-pll-supply = <&vreg_l12a>; vdda-phy-dpdm-supply = <&vreg_l13a>; @@ -473,12 +776,17 @@ status = "okay"; }; -&usb_1 { +&usb_qmpphy_2 { + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l12a>; + status = "okay"; }; -&usb_1_dwc3 { +&usb_1 { dr_mode = "peripheral"; + + status = "okay"; }; &usb_hsphy_2 { @@ -490,11 +798,9 @@ }; &usb_2 { - status = "okay"; -}; - -&usb_2_dwc3 { dr_mode = "host"; + + status = "okay"; }; &ufs_mem_hc { diff --git a/arch/arm64/boot/dts/qcom/qcs6490-camera.dtsi b/arch/arm64/boot/dts/qcom/qcs6490-camera.dtsi new file mode 100644 index 0000000000000..2fadfb9de1fee --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs6490-camera.dtsi @@ -0,0 +1,2471 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&soc { + cam_a5: qcom,a5 { + compatible = "qcom,cam-a5"; + cell-index = <0>; + reg = <0x0 0xac00000 0x0 0x6000>, + <0x0 0xac10000 0x0 0x8000>, + <0x0 0xac18000 0x0 0x3000>; + reg-names = "a5_qgic", "a5_sierra", "a5_csr"; + reg-cam-base = <0x00000 0x10000 0x18000>; + interrupts = ; + interrupt-names = "a5"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_ICP_AHB_CLK>, + <&camcc CAM_CC_ICP_CLK_SRC>, + <&camcc CAM_CC_ICP_CLK>; + clock-names = "soc_fast_ahb", + "icp_ahb_clk", + "icp_clk_src", + "icp_clk"; + clock-rates = <100000000 0 400000000 0>, + <200000000 0 400000000 0>, + <300000000 0 600000000 0>, + <400000000 0 600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal"; + src-clock-name = "soc_fast_ahb"; + operating-points-v2 = <&a5_opp_table>; + fw_name = "qcom/qcm6490/CAMERA_ICP_170.elf"; + ubwc-ipe-fetch-cfg = <0x7073 0x707b>; + ubwc-ipe-write-cfg = <0x161cf 0x161ef>; + ubwc-bps-fetch-cfg = <0x7073 0x707b>; + ubwc-bps-write-cfg = <0x161cf 0x161ef>; + qos-val = <0x00000A0A>; + status = "ok"; + + a5_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_bps: qcom,bps { + compatible = "qcom,cam-bps"; + cell-index = <0>; + reg = <0x0 0xac6f000 0x0 0x8000>; + reg-names = "bps_top"; + reg-cam-base = <0x6f000>; + power-domains = <&camcc CAM_CC_BPS_GDSC>; + clocks = <&camcc CAM_CC_BPS_AHB_CLK>, + <&camcc CAM_CC_BPS_AREG_CLK>, + <&camcc CAM_CC_BPS_AXI_CLK>, + <&camcc CAM_CC_BPS_CLK_SRC>, + <&camcc CAM_CC_BPS_CLK>; + clock-names = "bps_ahb_clk", + "bps_areg_clk", + "bps_axi_clk", + "bps_clk_src", + "bps_clk"; + clock-rates = <0 0 0 200000000 0>, + <0 0 0 400000000 0>, + <0 0 0 480000000 0>, + <0 0 0 600000000 0>, + <0 0 0 600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", + "nominal", "turbo"; + src-clock-name = "bps_clk_src"; + operating-points-v2 = <&bps_opp_table>; + clock-control-debugfs = "true"; + status = "ok"; + + bps_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + qcom,cam-cdm-intf { + compatible = "qcom,cam-cdm-intf"; + cell-index = <0>; + label = "cam-cdm-intf"; + num-hw-cdm = <1>; + cdm-client-names = "vfe", + "jpegdma", + "jpegenc", + "lrmecdm"; + status = "ok"; + }; + + qcom,cam-cpas { + compatible = "qcom,cam-cpas"; + cell-index = <0>; + label = "cpas"; + arch-compat = "cpas_top"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0 0x0ac40000 0x0 0x1000>, + <0x0 0x0ac9f000 0x0 0x10000>; + reg-names = "cam_cpas_top", "cam_camnoc"; + reg-cam-base = <0x40000 0x9f000>; + cam_hw_fuse = , + , + , + , + , + ; + interrupts = ; + interrupt-names = "cpas_camnoc"; + camnoc-axi-min-ib-bw = <3000000000>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&gcc GCC_CAMERA_SF_AXI_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_ICP_AHB_CLK>, + <&camcc CAM_CC_ICP_CLK>; + clock-names = "gcc_camera_hf_axi_clk", + "gcc_camera_sf_axi_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk", + "cam_cc_camnoc_axi_clk_src", + "cam_cc_camnoc_axi_clk", + "cam_cc_icp_ahb_clk", + "cam_cc_icp_clk"; + src-clock-name = "cam_cc_camnoc_axi_clk_src"; + clock-rates = <0 0 0 0 0 0 0 0>, + <0 0 80000000 0 150000000 0 0 0>, + <0 0 80000000 0 240000000 0 0 0>, + <0 0 80000000 0 320000000 0 0 0>, + <0 0 80000000 0 400000000 0 0 0>, + <0 0 80000000 0 480000000 0 0 0>; + clock-cntl-level = "suspend", "lowsvs", "svs", + "svs_l1", "nominal", "turbo"; + operating-points-v2 = <&cpas_opp_table>; + control-camnoc-axi-clk; + camnoc-bus-width = <32>; + camnoc-axi-clk-bw-margin-perc = <20>; + interconnects = <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_CAMERA_CFG 0>, + <&mmss_noc MASTER_CAMNOC_HF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_SF 0 &mc_virt SLAVE_EBI1 0>, + <&mmss_noc MASTER_CAMNOC_ICP 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "cam_ahb", "cam_hf_0", "cam_sf_0", "cam_sf_icp"; + cam-ahb-num-cases = <7>; + cam-ahb-bw-KBps = <0 0>, <0 76800>, <0 150000>, <0 150000>, + <0 300000>, <0 300000>, <0 300000>; + vdd-corners = ; + vdd-corner-ahb-mapping = "suspend", "lowsvs", "lowsvs", "svs", "svs_l1", + "nominal", "nominal", "nominal", "turbo", "turbo"; + client-id-based; + client-names = "csiphy0", "csiphy1", "csiphy2", "csiphy3", + "csiphy4", "cci0", "cci1", "csid0", "csid1", + "csid2", "csid3", "csid4", "ife0", "ife1", + "ife2", "ife3", "ife4", "ipe0", "cam-cdm-intf0", + "cpas-cdm0", "bps0", "icp0", "jpeg-dma0", "jpeg-enc0", + "lrmecpas0"; + status = "ok"; + + cpas_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + + camera-bus-nodes { + level0-nodes { + level-index = <0>; + + bps0_all_rd: bps0-all-rd { + cell-index = <0>; + node-name = "bps0-all-rd"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_read0>; + }; + + bps0_all_wr: bps0-all-wr { + cell-index = <1>; + node-name = "bps0-all-wr"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write0>; + }; + + cpas_cdm0_all_rd: cpas-cdm0-all-rd { + cell-index = <2>; + node-name = "cpas-cdm0-all-rd"; + client-name = "cpas-cdm0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt0_read0>; + }; + + icp0_all_rd: icp0-all-rd { + cell-index = <3>; + node-name = "icp0-all-rd"; + client-name = "icp0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level2_nrt1_read0>; + }; + + ife0_pixelall_wr: ife0-pixelall-wr { + cell-index = <4>; + node-name = "ife0-pixelall-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt1_write0>; + }; + + ife0_rdi_wr: ife0-rdi-wr { + cell-index = <5>; + node-name = "ife0-rdi-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ife1_pixelall_wr: ife1-pixelall-wr { + cell-index = <6>; + node-name = "ife1-pixelall-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt1_write0>; + }; + + ife1_rdi_wr: ife1-rdi-wr { + cell-index = <7>; + node-name = "ife1-rdi-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ife2_pixelall_wr: ife2-pixelall-wr { + cell-index = <8>; + node-name = "ife2-pixelall-wr"; + client-name = "ife2"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt1_write1>; + }; + + ife2_rdi_wr: ife2-rdi-wr { + cell-index = <9>; + node-name = "ife2-rdi-wr"; + client-name = "ife2"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ife3_rdi_wr: ife3-rdi-wr { + cell-index = <10>; + node-name = "ife3-rdi-wr"; + client-name = "ife3"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ife4_rdi_wr: ife4-rdi-wr { + cell-index = <11>; + node-name = "ife4-rdi-wr"; + client-name = "ife4"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ipe0_all_rd: ipe0-all-rd { + cell-index = <12>; + node-name = "ipe0-all-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_nrt0_read0>; + }; + + ipe0_ref_wr: ipe0-ref-wr { + cell-index = <13>; + node-name = "ipe0-ref-wr"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write0>; + }; + + ipe0_viddisp_wr: ipe0-viddisp-wr { + cell-index = <14>; + node-name = "ipe0-viddisp-wr"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_nrt0_write0>; + }; + + jpeg_dma0_all_rd: jpeg-dma0-all-rd { + cell-index = <15>; + node-name = "jpeg-dma0-all-rd"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt1_read0>; + }; + + jpeg_dma0_all_wr: jpeg-dma0-all-wr { + cell-index = <16>; + node-name = "jpeg-dma0-all-wr"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt1_write0>; + }; + + jpeg_enc0_all_rd: jpeg-enc0-all-rd { + cell-index = <17>; + node-name = "jpeg-enc0-all-rd"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt1_read0>; + }; + + jpeg_enc0_all_wr: jpeg-enc0-all-wr { + cell-index = <18>; + node-name = "jpeg-enc0-all-wr"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt1_write0>; + }; + + lrme0_all_rd: lrme0-all-rd { + cell-index = <19>; + node-name = "lrme0-all-rd"; + client-name = "lrmecpas0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_read0>; + }; + + lrme0_all_wr: lrme0-all-wr { + cell-index = <20>; + node-name = "lrme0-all-wr"; + client-name = "lrmecpas0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write0>; + }; + }; + + level1-nodes { + level-index = <1>; + camnoc-max-needed; + + level1_nrt0_write0: level1-nrt0-write0 { + cell-index = <21>; + node-name = "level1-nrt0-write0"; + parent-node = <&level2_nrt0_write0>; + traffic-merge-type = ; + }; + + level1_nrt0_read0: level1-nrt0-read0 { + cell-index = <22>; + node-name = "level1-nrt0-read0"; + parent-node = <&level2_nrt0_read0>; + traffic-merge-type = ; + }; + + level1_nrt1_write0: level1-nrt1-write0 { + cell-index = <23>; + node-name = "level1-nrt1-write0"; + parent-node = <&level2_nrt0_write0>; + traffic-merge-type = ; + }; + + level1_nrt1_read0: level1-nrt1-read0 { + cell-index = <24>; + node-name = "level1-nrt1-read0"; + parent-node = <&level2_nrt0_read0>; + traffic-merge-type = ; + }; + + level1_rt0_write0: level1-rt0-write0 { + cell-index = <25>; + node-name = "level1-rt0-write0"; + parent-node = <&level2_rt0_write0>; + traffic-merge-type = ; + }; + + level1_rt1_write0: level1-rt1-write0 { + cell-index = <26>; + node-name = "level1-rt1-write0"; + parent-node = <&level2_rt1_write0>; + traffic-merge-type = ; + }; + + level1_rt1_write1: level1-rt1-write1 { + cell-index = <27>; + node-name = "level1-rt1-write1"; + parent-node = <&level2_rt1_write0>; + traffic-merge-type = ; + }; + }; + + level2-nodes { + level-index = <3>; + camnoc-max-needed; + + level2_nrt0_write0: level2-nrt0-write0 { + cell-index = <28>; + node-name = "level2-nrt0-write0"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt0_read0: level2-nrt0-read0 { + cell-index = <29>; + node-name = "level2-nrt0-read0"; + parent-node = <&level3_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level2_nrt1_read0: level2-nrt1-read0 { + cell-index = <30>; + node-name = "level2-nrt1-read0"; + parent-node = <&level3_nrt1_rd_sum>; + traffic-merge-type = + ; + bus-width-factor = <4>; + }; + + level2_rt0_write0: level2-rt0-write0 { + cell-index = <31>; + node-name = "level2-rt0-write0"; + parent-node = <&level3_rt0_wr_sum>; + traffic-merge-type = ; + }; + + level2_rt1_write0: level2-rt1-write0 { + cell-index = <32>; + node-name = "level2-rt1-write0"; + parent-node = <&level3_rt0_wr_sum>; + traffic-merge-type = ; + }; + }; + + level3-nodes { + level-index = <3>; + + level3_nrt0_rd_wr_sum: level3-nrt0-rd-wr-sum { + cell-index = <33>; + node-name = "level3-nrt0-rd-wr-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + interconnect-names = "cam_sf_0"; + interconnects = + <&mmss_noc MASTER_CAMNOC_SF 0 + &mc_virt SLAVE_EBI1 0>; + }; + }; + + level3_nrt1_rd_sum: level3-nrt1-rd-sum { + cell-index = <34>; + node-name = "level3-nrt1-rd-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + interconnect-names = "cam_sf_icp"; + interconnects = + <&mmss_noc MASTER_CAMNOC_ICP 0 + &mc_virt SLAVE_EBI1 0>; + }; + }; + + level3_rt0_wr_sum: level3-rt0-wr-sum { + cell-index = <35>; + node-name = "level3-rt0-wr-sum"; + traffic-merge-type = ; + ib-bw-voting-needed; + + qcom,axi-port-mnoc { + interconnect-names = "cam_hf_0"; + interconnects = + <&mmss_noc MASTER_CAMNOC_HF 0 + &mc_virt SLAVE_EBI1 0>; + }; + }; + }; + }; + }; + + qcom,cam-icp { + compatible = "qcom,cam-icp"; + compat-hw-name = "qcom,a5", + "qcom,ipe0", + "qcom,bps"; + num-a5 = <1>; + num-ipe = <1>; + num-bps = <1>; + icp_pc_en; + status = "ok"; + }; + + qcom,cam-isp { + compatible = "qcom,cam-isp"; + arch-compat = "ife"; + status = "ok"; + }; + + qcom,cam-req-mgr { + compatible = "qcom,cam-req-mgr"; + status = "ok"; + }; + + qcom,cam-jpeg { + compatible = "qcom,cam-jpeg"; + compat-hw-name = "qcom,jpegenc", + "qcom,jpegdma"; + num-jpeg-enc = <1>; + num-jpeg-dma = <1>; + status = "ok"; + }; + + qcom,cam-lrme { + compatible = "qcom,cam-lrme"; + arch-compat = "lrme"; + status = "ok"; + }; + + qcom,camera-main { + compatible = "qcom,camera_kt"; + status = "ok"; + }; + + qcom,cam-smmu { + compatible = "qcom,msm-cam-smmu", "simple-bus"; + force_cache_allocs; + status = "ok"; + + msm-cam-icp-fw { + compatible = "qcom,msm-cam-smmu-fw-dev"; + label="icp"; + memory-region = <&camera_mem>; + }; + + msm-cam-smmu-cpas-cdm { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x2040 0x0>; + cam-smmu-label = "cpas-cdm"; + dma-coherent; + + cpas_cdm_iova_mem_map: iova-mem-map { + iova-mem-region-io { + /* IO region is approximately 3.4 GB */ + iova-region-name = "io"; + iova-region-start = <0x100000>; + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm_cam_smmu_icp { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x2000 0x20>, + <&apps_smmu 0x2020 0x20>, + <&apps_smmu 0x2062 0x0>, + <&apps_smmu 0x2080 0x20>, + <&apps_smmu 0x20A0 0x20>, + <&apps_smmu 0x2140 0x0>; + cam-smmu-label = "icp"; + dma-coherent; + + icp_iova_mem_map: iova-mem-map { + iova-mem-region-firmware { + /* Firmware region is 5MB */ + iova-region-name = "firmware"; + iova-region-start = <0x0>; + iova-region-len = <0x500000>; + iova-region-id = <0x0>; + status = "ok"; + }; + + iova-mem-region-io { + /* IO region is approximately 3.7 GB */ + iova-region-name = "io"; + iova-region-start = <0x10c00000>; + iova-region-len = <0xcf300000>; + iova-region-id = <0x3>; + status = "ok"; + }; + + iova-mem-region-qdss { + /* QDSS region is appropriate 1MB */ + iova-region-name = "qdss"; + iova-region-start = <0x10b00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x5>; + qdss-phy-addr = <0x16790000>; + status = "ok"; + }; + + iova-mem-region-secondary-heap { + /* Secondary heap region is 1MB long */ + iova-region-name = "secheap"; + iova-region-start = <0x10a00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x4>; + status = "ok"; + }; + + iova-mem-region-shared { + /* Shared region is 150MB long */ + iova-region-name = "shared"; + iova-region-start = <0x7400000>; + iova-region-len = <0x9600000>; + iova-region-id = <0x1>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-ife { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x800 0x4E0>, + <&apps_smmu 0x820 0x4E0>, + <&apps_smmu 0x840 0x4E0>, + <&apps_smmu 0x860 0x4E0>, + <&apps_smmu 0x880 0x4E0>, + <&apps_smmu 0x8A0 0x4E0>, + <&apps_smmu 0x8C0 0x4E0>, + <&apps_smmu 0x8E0 0x4E0>, + <&apps_smmu 0xC00 0x4E0>, + <&apps_smmu 0xC20 0x4E0>, + <&apps_smmu 0xC40 0x4E0>, + <&apps_smmu 0xC60 0x4E0>, + <&apps_smmu 0xC80 0x4E0>, + <&apps_smmu 0xCA0 0x4E0>, + <&apps_smmu 0xCC0 0x4E0>, + <&apps_smmu 0xCE0 0x4E0>; + cam-smmu-label = "ife"; + multiple-client-devices; + dma-coherent; + + ife_iova_mem_map: iova-mem-map { + /* IO region is approximately 4 GB */ + iova-mem-region-io { + iova-region-name = "io"; + iova-region-start = <0x100000>; + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-jpeg { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x20C0 0x20>, + <&apps_smmu 0x20E0 0x20>; + cam-smmu-label = "jpeg"; + dma-coherent; + + jpeg_iova_mem_map: iova-mem-map { + /* IO region is approximately 3.4 GB */ + iova-mem-region-io { + iova-region-name = "io"; + iova-region-start = <0x100000>; + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-lrme { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x2100 0x20>, + <&apps_smmu 0x2120 0x20>; + cam-smmu-label = "lrme"; + dma-coherent; + + lrme_iova_mem_map: iova-mem-map { + iova-mem-region-shared { + /* Shared region is 100MB long */ + iova-region-name = "shared"; + iova-region-start = <0x7400000>; + iova-region-len = <0x6400000>; + iova-region-id = <0x1>; + status = "ok"; + }; + + /* IO region is approximately 3.3 GB */ + iova-mem-region-io { + iova-region-name = "io"; + iova-region-start = <0x100000>; + iova-region-len = <0xffe00000>; + iova-region-id = <0x3>; + status = "ok"; + }; + }; + }; + + msm-cam-smmu-secure { + compatible = "qcom,msm-cam-smmu-cb"; + cam-smmu-label = "cam-secure"; + qcom,secure-cb; + }; + }; + + qcom,cam-sync { + compatible = "qcom,cam-sync"; + status = "ok"; + }; + + cam_cci0: qcom,cci0 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac4a000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x4a000>; + interrupts = ; + interrupt-names = "cci"; + operating-points-v2 = <&cci0_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_0_CLK_SRC>, + <&camcc CAM_CC_CCI_0_CLK>; + clock-names = "cci_0_clk_src", + "cci_0_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_0_clk_src"; + pinctrl-0 = <&cci0_active &cci1_active>; + pinctrl-1 = <&cci0_suspend &cci1_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 69 0>, + <&tlmm 70 0>, + <&tlmm 71 0>, + <&tlmm 72 0>; + gpio-req-tbl-num = <0 1 2 3>; + gpio-req-tbl-flags = <1 1 1 1>; + gpio-req-tbl-label = "CCI_I2C_DATA0", + "CCI_I2C_CLK0", + "CCI_I2C_DATA1", + "CCI_I2C_CLK1"; + cell-index = <0>; + status = "ok"; + + cci0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci0: qcom,i2c-custom-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci0: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci0: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <0>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci0: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + cam_cci1: qcom,cci1 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac4b000 0x0 0x1000>; + reg-names = "cci"; + reg-cam-base = <0x4b000>; + interrupts = ; + interrupt-names = "cci"; + operating-points-v2 = <&cci1_opp_table>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_1_CLK_SRC>, + <&camcc CAM_CC_CCI_1_CLK>; + clock-names = "cci_1_clk_src", + "cci_1_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_1_clk_src"; + pinctrl-0 = <&cci2_active &cci3_active>; + pinctrl-1 = <&cci2_suspend &cci3_suspend>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 73 0>, + <&tlmm 74 0>, + <&tlmm 75 0>, + <&tlmm 76 0>; + gpio-req-tbl-num = <0 1 2 3>; + gpio-req-tbl-flags = <1 1 1 1>; + gpio-req-tbl-label = "CCI_I2C_DATA2", + "CCI_I2C_CLK2", + "CCI_I2C_DATA3", + "CCI_I2C_CLK3"; + cell-index = <1>; + status = "ok"; + + cci1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci1: qcom,i2c-custom-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_400Khz_cci1: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_1Mhz_cci1: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <0>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + + i2c_freq_100Khz_cci1: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "ok"; + }; + }; + + qcom,cpas-cdm0 { + compatible = "qcom,cam170-cpas-cdm0"; + cell-index = <0>; + label = "cpas-cdm"; + reg = <0x0 0xac48000 0x0 0x1000>; + reg-names = "cpas-cdm"; + reg-cam-base = <0x48000>; + interrupts = ; + interrupt-names = "cpas-cdm"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>; + clock-names = "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk"; + clock-rates = <80000000 0>; + clock-cntl-level = "svs"; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + operating-points-v2 = <&cdm_cpas_opp_table>; + cdm-client-names = "ife0", "ife1", "ife2", + "ife3", "ife4", "dualife"; + status = "ok"; + + cdm_cpas_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_csid0: qcom,csid0 { + compatible = "qcom,csid165_204"; + cell-index = <0>; + reg = <0x0 0xacb3000 0x0 0x1000>; + reg-names = "csid"; + reg-cam-base = <0xb3000>; + interrupts = ; + interrupt-names = "csid0"; + power-domains = <&camcc CAM_CC_IFE_0_GDSC>; + clocks = <&camcc CAM_CC_IFE_0_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CSID_CLK>, + <&camcc CAM_CC_IFE_0_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_AXI_CLK>; + clock-names = "ife_csid_clk_src", + "ife_csid_clk", + "ife_cphy_rx_clk", + "ife_clk_src", + "ife_clk", + "ife_axi_clk"; + clock-rates = <300000000 0 0 380000000 0 0>, + <400000000 0 0 510000000 0 0>, + <400000000 0 0 637000000 0 0>, + <400000000 0 0 760000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_csid_clk_src"; + operating-points-v2 = <&csid0_opp_table>; + clock-control-debugfs = "true"; + status = "ok"; + + csid0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-380000000 { + opp-hz = /bits/ 64 <380000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-510000000 { + opp-hz = /bits/ 64 <510000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-637000000 { + opp-hz = /bits/ 64 <637000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-760000000 { + opp-hz = /bits/ 64 <760000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid1: qcom,csid1 { + compatible = "qcom,csid165_204"; + cell-index = <1>; + reg = <0x0 0xacba000 0x0 0x1000>; + reg-names = "csid"; + reg-cam-base = <0xba000>; + interrupts = ; + interrupt-names = "csid1"; + power-domains = <&camcc CAM_CC_IFE_1_GDSC>; + clocks = <&camcc CAM_CC_IFE_1_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CSID_CLK>, + <&camcc CAM_CC_IFE_1_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_1_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_AXI_CLK>; + clock-names = "ife_csid_clk_src", + "ife_csid_clk", + "ife_cphy_rx_clk", + "ife_clk_src", + "ife_clk", + "ife_axi_clk"; + clock-rates = <300000000 0 0 380000000 0 0>, + <400000000 0 0 510000000 0 0>, + <400000000 0 0 637000000 0 0>, + <400000000 0 0 760000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_csid_clk_src"; + operating-points-v2 = <&csid1_opp_table>; + clock-control-debugfs = "true"; + status = "ok"; + + csid1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-380000000 { + opp-hz = /bits/ 64 <380000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-510000000 { + opp-hz = /bits/ 64 <510000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-637000000 { + opp-hz = /bits/ 64 <637000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-760000000 { + opp-hz = /bits/ 64 <760000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid2: qcom,csid2 { + compatible = "qcom,csid165_204"; + cell-index = <2>; + reg = <0x0 0x0acc1000 0x0 0x1000>; + reg-names = "csid"; + reg-cam-base = <0xc1000>; + interrupts = ; + interrupt-names = "csid2"; + power-domains = <&camcc CAM_CC_IFE_2_GDSC>; + clocks = <&camcc CAM_CC_IFE_2_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_2_CSID_CLK>, + <&camcc CAM_CC_IFE_2_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_2_CLK_SRC>, + <&camcc CAM_CC_IFE_2_CLK>, + <&camcc CAM_CC_IFE_2_AXI_CLK>; + clock-names = "ife_csid_clk_src", + "ife_csid_clk", + "ife_cphy_rx_clk", + "ife_clk_src", + "ife_clk", + "ife_axi_clk"; + clock-rates = <300000000 0 0 380000000 0 0>, + <400000000 0 0 510000000 0 0>, + <400000000 0 0 637000000 0 0>, + <400000000 0 0 760000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_csid_clk_src"; + operating-points-v2 = <&csid2_opp_table>; + clock-control-debugfs = "true"; + status = "ok"; + + csid2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-380000000 { + opp-hz = /bits/ 64 <380000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-510000000 { + opp-hz = /bits/ 64 <510000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-637000000 { + opp-hz = /bits/ 64 <637000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-760000000 { + opp-hz = /bits/ 64 <760000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite0: qcom,csid-lite0 { + compatible = "qcom,csid-lite165"; + cell-index = <3>; + reg = <0x0 0xacc8000 0x0 0x1000>; + reg-names = "csid-lite"; + reg-cam-base = <0xc8000>; + interrupts = ; + interrupt-names = "csid-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_IFE_LITE_0_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_0_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_0_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_0_CLK>; + clock-names = "ife_csid_clk_src", + "ife_csid_clk", + "ife_cphy_rx_clk", + "ife_clk_src", + "ife_clk"; + clock-rates = <300000000 0 0 320000000 0>, + <400000000 0 0 400000000 0>, + <400000000 0 0 480000000 0>, + <400000000 0 0 600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_csid_clk_src"; + operating-points-v2 = <&csid_lite0_opp_table>; + clock-control-debugfs = "true"; + status = "ok"; + + csid_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csid_lite1: qcom,csid-lite1 { + compatible = "qcom,csid-lite165"; + cell-index = <4>; + reg = <0x0 0x0accf000 0x0 0x1000>; + reg-names = "csid-lite"; + reg-cam-base = <0xcf000>; + interrupts = ; + interrupt-names = "csid-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_IFE_LITE_1_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_1_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_1_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_1_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_1_CLK>; + clock-names = "ife_csid_clk_src", + "ife_csid_clk", + "ife_cphy_rx_clk", + "ife_clk_src", + "ife_clk"; + clock-rates = <300000000 0 0 320000000 0>, + <400000000 0 0 400000000 0>, + <400000000 0 0 480000000 0>, + <400000000 0 0 600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_csid_clk_src"; + operating-points-v2 = <&csid_lite1_opp_table>; + clock-control-debugfs = "true"; + status = "ok"; + + csid_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_csiphy0: qcom,csiphy0 { + compatible = "qcom,csiphy-v1.2.1", "qcom,csiphy"; + cell-index = <0>; + reg = <0x0 0x0ace0000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe0000>; + interrupts = ; + interrupt-names = "csiphy0"; + operating-points-v2 = <&csiphy0_opp_table>; + csi-vdd-1p2-supply = <&vreg_l6b_1p2>; + csi-vdd-0p9-supply = <&vreg_l10c_0p88>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1260000 1050000>; + rgltr-load-current = <54000 96400>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy0_clk", + "csi0phytimer_clk_src", + "csi0phytimer_clk"; + clock-rates = <300000000 0 300000000 0>, + <400000000 0 300000000 0>; + clock-cntl-level = "lowsvs", "svs"; + src-clock-name = "csi0phytimer_clk_src"; + status = "ok"; + + csiphy0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + }; + }; + + cam_csiphy1: qcom,csiphy1 { + cell-index = <1>; + compatible = "qcom,csiphy-v1.2.1", "qcom,csiphy"; + reg = <0x0 0xace2000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe2000>; + interrupts = ; + interrupt-names = "csiphy1"; + operating-points-v2 = <&csiphy1_opp_table>; + csi-vdd-1p2-supply = <&vreg_l6b_1p2>; + csi-vdd-0p9-supply = <&vreg_l10c_0p88>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1260000 1050000>; + rgltr-load-current = <54000 96400>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy1_clk", + "csi1phytimer_clk_src", + "csi1phytimer_clk"; + src-clock-name = "csi1phytimer_clk_src"; + clock-cntl-level = "lowsvs", "svs"; + clock-rates = <300000000 0 300000000 0>, + <400000000 0 300000000 0>; + status = "ok"; + + csiphy1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + }; + }; + + cam_csiphy2: qcom,csiphy2 { + cell-index = <2>; + compatible = "qcom,csiphy-v1.2.1", "qcom,csiphy"; + reg = <0x0 0xace4000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe4000>; + interrupts = ; + interrupt-names = "csiphy2"; + operating-points-v2 = <&csiphy2_opp_table>; + csi-vdd-1p2-supply = <&vreg_l6b_1p2>; + csi-vdd-0p9-supply = <&vreg_l10c_0p88>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1260000 1050000>; + rgltr-load-current = <54000 96400>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy2_clk", + "csi2phytimer_clk_src", + "csi2phytimer_clk"; + src-clock-name = "csi2phytimer_clk_src"; + clock-cntl-level = "lowsvs", "svs"; + clock-rates = <300000000 0 300000000 0>, + <400000000 0 300000000 0>; + status = "ok"; + + csiphy2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + }; + }; + + cam_csiphy3: qcom,csiphy3 { + cell-index = <3>; + compatible = "qcom,csiphy-v1.2.1", "qcom,csiphy"; + reg = <0x0 0xace6000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe6000>; + interrupts = ; + interrupt-names = "csiphy3"; + operating-points-v2 = <&csiphy3_opp_table>; + csi-vdd-1p2-supply = <&vreg_l6b_1p2>; + csi-vdd-0p9-supply = <&vreg_l10c_0p88>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1260000 1050000>; + rgltr-load-current = <54000 96400>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY3_CLK>, + <&camcc CAM_CC_CSI3PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI3PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy3_clk", + "csi3phytimer_clk_src", + "csi3phytimer_clk"; + src-clock-name = "csi3phytimer_clk_src"; + clock-cntl-level = "lowsvs", "svs"; + clock-rates = <300000000 0 300000000 0>, + <400000000 0 300000000 0>; + status = "ok"; + + csiphy3_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + }; + }; + + cam_csiphy4: qcom,csiphy4 { + cell-index = <4>; + compatible = "qcom,csiphy-v1.2.1", "qcom,csiphy"; + reg = <0x0 0xace8000 0x0 0x2000>; + reg-names = "csiphy"; + reg-cam-base = <0xe8000>; + interrupts = ; + interrupt-names = "csiphy4"; + operating-points-v2 = <&csiphy4_opp_table>; + csi-vdd-1p2-supply = <&vreg_l6b_1p2>; + csi-vdd-0p9-supply = <&vreg_l10c_0p88>; + regulator-names = "csi-vdd-1p2", "csi-vdd-0p9"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1200000 880000>; + rgltr-max-voltage = <1260000 1050000>; + rgltr-load-current = <54000 96400>; + clocks = <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY4_CLK>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI4PHYTIMER_CLK>; + clock-names = "cphy_rx_clk_src", + "csiphy4_clk", + "csi4phytimer_clk_src", + "csi4phytimer_clk"; + src-clock-name = "csi4phytimer_clk_src"; + clock-cntl-level = "lowsvs", "svs"; + clock-rates = <300000000 0 300000000 0>, + <400000000 0 300000000 0>; + status = "ok"; + + csiphy4_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + }; + }; + + cam_ipe0: qcom,ipe0 { + compatible = "qcom,cam-ipe"; + cell-index = <0>; + reg = <0x0 0xac87000 0x0 0xa000>; + reg-names = "ipe0_top"; + reg-cam-base = <0x87000>; + power-domains = <&camcc CAM_CC_IPE_0_GDSC>; + clocks = <&camcc CAM_CC_IPE_0_AHB_CLK>, + <&camcc CAM_CC_IPE_0_AREG_CLK>, + <&camcc CAM_CC_IPE_0_AXI_CLK>, + <&camcc CAM_CC_IPE_0_CLK_SRC>, + <&camcc CAM_CC_IPE_0_CLK>; + clock-names = "ipe_0_ahb_clk", + "ipe_0_areg_clk", + "ipe_0_axi_clk", + "ipe_0_clk_src", + "ipe_0_clk"; + clock-rates = <0 0 0 300000000 0>, + <0 0 0 430000000 0>, + <0 0 0 520000000 0>, + <0 0 0 600000000 0>, + <0 0 0 600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", + "nominal", "turbo"; + src-clock-name = "ipe_0_clk_src"; + operating-points-v2 = <&ipe0_opp_table>; + clock-control-debugfs = "true"; + status = "ok"; + + ipe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-430000000 { + opp-hz = /bits/ 64 <430000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-520000000 { + opp-hz = /bits/ 64 <520000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_jpeg_dma: qcom,jpegdma { + compatible = "qcom,cam_jpeg_dma_165"; + cell-index = <0>; + reg = <0x0 0xac52000 0x0 0x4000>, + <0x0 0x0ac9f000 0x0 0x10000>; + reg-names = "jpegdma_hw", "cam_camnoc"; + reg-cam-base = <0x52000 0x9f000>; + interrupts = ; + interrupt-names = "jpegdma"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "jpegdma_clk_src", + "jpegdma_clk"; + clock-rates = <600000000 0>; + clock-cntl-level = "nominal"; + src-clock-name = "jpegdma_clk_src"; + operating-points-v2 = <&jpeg_dma_opp_table>; + cam_hw_pid = <20 21>; + cam_hw_rd_mid = <0>; + cam_hw_wr_mid = <2>; + status = "ok"; + + jpeg_dma_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_jpeg_enc: qcom,jpegenc { + compatible = "qcom,cam_jpeg_enc_165"; + cell-index = <0>; + reg = <0x0 0xac4e000 0x0 0x4000>, + <0x0 0x0ac9f000 0x0 0x10000>; + reg-names = "jpege_hw", "cam_camnoc"; + reg-cam-base = <0x4e000 0x9f000>; + interrupts = ; + interrupt-names = "jpeg"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "jpegenc_clk_src", + "jpegenc_clk"; + clock-rates = <600000000 0>; + clock-cntl-level = "nominal"; + src-clock-name = "jpegenc_clk_src"; + operating-points-v2 = <&jpeg_enc_opp_table>; + cam_hw_pid = <22 23>; + cam_hw_rd_mid = <0>; + cam_hw_wr_mid = <2>; + status = "ok"; + + jpeg_enc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_lrme: qcom,lrme { + compatible = "qcom,lrme"; + cell-index = <0>; + reg = <0x0 0xac6b000 0x0 0x1000>; + reg-names = "lrme"; + reg-cam-base = <0x6b000>; + interrupts = ; + interrupt-names = "lrme"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_LRME_CLK_SRC>, + <&camcc CAM_CC_LRME_CLK>; + clock-names = "lrme_clk_src", + "lrme_clk"; + clock-rates = <240000000 0>, + <300000000 0>, + <320000000 0>, + <400000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal"; + src-clock-name = "lrme_clk_src"; + operating-points-v2 = <&lrme_opp_table>; + status = "ok"; + + lrme_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe0: qcom,vfe0 { + compatible = "qcom,vfe165_160"; + reg = <0x0 0xacaf000 0x0 0x5200>; + reg-names = "ife"; + reg-cam-base = <0xaf000>; + interrupts = ; + interrupt-names = "ife0"; + power-domains = <&camcc CAM_CC_IFE_0_GDSC>; + clocks = <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_AXI_CLK>; + clock-names = "ife_clk_src", + "ife_clk", + "ife_axi_clk"; + clock-rates = <380000000 0 0>, + <510000000 0 0>, + <637000000 0 0>, + <760000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_clk_src"; + operating-points-v2 = <&vfe0_opp_table>; + clock-control-debugfs = "true"; + clock-names-option = "ife_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_0_DSP_CLK>; + clock-rates-option = <760000000>; + cam_hw_pid = <24 8>; + cell-index = <0>; + status = "ok"; + + vfe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-380000000 { + opp-hz = /bits/ 64 <380000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-510000000 { + opp-hz = /bits/ 64 <510000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-637000000 { + opp-hz = /bits/ 64 <637000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-760000000 { + opp-hz = /bits/ 64 <760000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe1: qcom,vfe1 { + compatible = "qcom,vfe165_160"; + reg = <0x0 0xacb6000 0x0 0x5200>; + reg-names = "ife"; + reg-cam-base = <0xb6000>; + interrupts = ; + interrupt-names = "ife1"; + power-domains = <&camcc CAM_CC_IFE_1_GDSC>; + clocks = <&camcc CAM_CC_IFE_1_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_AXI_CLK>; + clock-names = "ife_clk_src", + "ife_clk", + "ife_axi_clk"; + clock-rates = <380000000 0 0>, + <510000000 0 0>, + <637000000 0 0>, + <760000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_clk_src"; + operating-points-v2 = <&vfe1_opp_table>; + clock-control-debugfs = "true"; + clock-names-option = "ife_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_1_DSP_CLK>; + clock-rates-option = <760000000>; + cam_hw_pid = <25 9>; + cell-index = <1>; + status = "ok"; + + vfe1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-380000000 { + opp-hz = /bits/ 64 <380000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-510000000 { + opp-hz = /bits/ 64 <510000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-637000000 { + opp-hz = /bits/ 64 <637000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-760000000 { + opp-hz = /bits/ 64 <760000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + + cam_vfe2: qcom,vfe2 { + compatible = "qcom,vfe165_160"; + reg = <0x0 0x0acbd000 0x0 0x5200>; + reg-names = "ife"; + reg-cam-base = <0xbd000>; + interrupts = ; + interrupt-names = "ife2"; + power-domains = <&camcc CAM_CC_IFE_2_GDSC>; + clocks = <&camcc CAM_CC_IFE_2_CLK_SRC>, + <&camcc CAM_CC_IFE_2_CLK>, + <&camcc CAM_CC_IFE_2_AXI_CLK>; + clock-names = "ife_clk_src", + "ife_clk", + "ife_axi_clk"; + clock-rates = <380000000 0 0>, + <510000000 0 0>, + <637000000 0 0>, + <760000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_clk_src"; + operating-points-v2 = <&vfe2_opp_table>; + clock-control-debugfs = "true"; + clock-names-option = "ife_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_2_DSP_CLK>; + clock-rates-option = <760000000>; + cam_hw_pid = <3 10>; + cell-index = <2>; + status = "ok"; + + vfe2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-380000000 { + opp-hz = /bits/ 64 <380000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-510000000 { + opp-hz = /bits/ 64 <510000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-637000000 { + opp-hz = /bits/ 64 <637000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-760000000 { + opp-hz = /bits/ 64 <760000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite0: qcom,vfe-lite0 { + compatible = "qcom,vfe-lite165"; + cell-index = <3>; + reg = <0x0 0xacc4000 0x0 0x5000>; + reg-names = "ife-lite"; + reg-cam-base = <0xc4000>; + interrupts = ; + interrupt-names = "ife-lite0"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_IFE_LITE_0_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_0_CLK>; + clock-names = "ife_clk_src", + "ife_clk"; + clock-rates = <320000000 0>, + <400000000 0>, + <480000000 0>, + <600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_clk_src"; + operating-points-v2 = <&vfe_lite0_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <11>; + status = "ok"; + + vfe_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_vfe_lite1: qcom,vfe-lite1 { + compatible = "qcom,vfe-lite165"; + reg = <0x0 0x0accb000 0x0 0x5000>; + reg-names = "ife-lite"; + reg-cam-base = <0xcb000>; + interrupts = ; + interrupt-names = "ife-lite1"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_IFE_LITE_1_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_1_CLK>; + clock-names = "ife_clk_src", + "ife_clk"; + clock-rates = <320000000 0>, + <400000000 0>, + <480000000 0>, + <600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "turbo"; + src-clock-name = "ife_clk_src"; + operating-points-v2 = <&vfe_lite1_opp_table>; + clock-control-debugfs = "true"; + cam_hw_pid = <12>; + cell-index = <4>; + status = "ok"; + + vfe_lite1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; +}; + +&tlmm { + cam_sensor_active_mux: cam-sensor-active-mux { + /* GC630 bypass mux */ + mux { + pins = "gpio1"; + function = "gpio"; + }; + + config { + pins = "gpio1"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst0: cam-sensor-active-rst0 { + /* RESET REAR */ + mux { + pins = "gpio20"; + function = "gpio"; + }; + + config { + pins = "gpio20"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst1: cam-sensor-active-rst1 { + /* RESET REARAUX */ + mux { + pins = "gpio21"; + function = "gpio"; + }; + + config { + pins = "gpio21"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst2: cam-sensor-active-rst2 { + /* RESET 2 */ + mux { + pins = "gpio77"; + function = "gpio"; + }; + + config { + pins = "gpio77"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst3: cam-sensor-active-rst3 { + /* RESET 3 */ + mux { + pins = "gpio78"; + function = "gpio"; + }; + + config { + pins = "gpio78"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst4: cam-sensor-active-rst4 { + /* RESET 4 */ + mux { + pins = "gpio79"; + function = "gpio"; + }; + + config { + pins = "gpio79"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk0_active: cam_sensor-mclk0-active { + /* MCLK0 */ + mux { + pins = "gpio64"; + function = "cam_mclk"; + }; + + config { + pins = "gpio64"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk0_suspend: cam-sensor-mclk0-suspend { + /* MCLK0 */ + mux { + pins = "gpio64"; + function = "cam_mclk"; + }; + + config { + pins = "gpio64"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_active: cam-sensor-mclk1-active { + /* MCLK1 */ + mux { + pins = "gpio65"; + function = "cam_mclk"; + }; + + config { + pins = "gpio65"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk1_suspend: cam-sensor-mclk1-suspend { + /* MCLK1 */ + mux { + pins = "gpio65"; + function = "cam_mclk"; + }; + + config { + pins = "gpio65"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_active: cam-sensor-mclk2-active { + /* MCLK2 */ + mux { + pins = "gpio66"; + function = "cam_mclk"; + }; + + config { + pins = "gpio66"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_suspend: cam-sensor-mclk2-suspend { + /* MCLK2 */ + mux { + pins = "gpio66"; + function = "cam_mclk"; + }; + + config { + pins = "gpio66"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk3_active: cam-sensor-mclk3-active { + /* MCLK3 */ + mux { + pins = "gpio67"; + function = "cam_mclk"; + }; + + config { + pins = "gpio67"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk3_suspend: cam-sensor-mclk3-suspend { + /* MCLK3 */ + mux { + pins = "gpio67"; + function = "cam_mclk"; + }; + + config { + pins = "gpio67"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk4_active: cam-sensor-mclk4-active { + /* MCLK4 */ + mux { + pins = "gpio68"; + function = "cam_mclk"; + }; + + config { + pins = "gpio68"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk4_suspend: cam-sensor-mclk4-suspend { + /* MCLK4 */ + mux { + pins = "gpio68"; + function = "cam_mclk"; + }; + + config { + pins = "gpio68"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk5_active: cam-sensor-mclk5-active { + /* MCLK5 */ + mux { + pins = "gpio98"; + function = "cam_mclk"; + }; + + config { + pins = "gpio98"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk5_suspend: cam-sensor-mclk5-suspend { + /* MCLK5 */ + mux { + pins = "gpio98"; + function = "cam_mclk"; + }; + + config { + pins = "gpio98"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_suspend_mux: cam-sensor-suspend-mux { + /* GC630 bypass mux */ + mux { + pins = "gpio1"; + function = "gpio"; + }; + + config { + pins = "gpio1"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst0: cam_sensor-suspend-rst0 { + /* RESET REAR */ + mux { + pins = "gpio20"; + function = "gpio"; + }; + + config { + pins = "gpio20"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst1: cam-sensor-suspend-rst1 { + /* RESET REARAUX */ + mux { + pins = "gpio21"; + function = "gpio"; + }; + + config { + pins = "gpio21"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst2: cam-sensor-suspend-rst2 { + /* RESET 2 */ + mux { + pins = "gpio77"; + function = "gpio"; + }; + + config { + pins = "gpio77"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst3: cam-sensor-suspend-rst3 { + /* RESET 3 */ + mux { + pins = "gpio78"; + function = "gpio"; + }; + + config { + pins = "gpio78"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst4: cam-sensor-suspend-rst4 { + /* RESET 4 */ + mux { + pins = "gpio79"; + function = "gpio"; + }; + + config { + pins = "gpio79"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cci0_active: cci0-active { + mux { + /* DATA, CLK */ + pins = "gpio69","gpio70"; // Only 2 + function = "cci_i2c"; + }; + + config { + pins = "gpio69","gpio70"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci0_suspend: cci0-suspend { + mux { + /* DATA, CLK */ + pins = "gpio69","gpio70"; + function = "cci_i2c"; + }; + + config { + pins = "gpio69","gpio70"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci1_active: cci1-active { + mux { + /* DATA, CLK */ + pins = "gpio71","gpio72"; + function = "cci_i2c"; + }; + + config { + pins = "gpio71","gpio72"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci1_suspend: cci1-suspend { + mux { + /* DATA, CLK */ + pins = "gpio71","gpio72"; + function = "cci_i2c"; + }; + + config { + pins = "gpio71","gpio72"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci2_active: cci2-active { + mux { + /* DATA, CLK */ + pins = "gpio73","gpio74"; + function = "cci_i2c"; + }; + + config { + pins = "gpio73","gpio74"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci2_suspend: cci2-suspend { + mux { + /* DATA, CLK */ + pins = "gpio73","gpio74"; + function = "cci_i2c"; + }; + + config { + pins = "gpio73","gpio74"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci3_active: cci3-active { + mux { + /* DATA, CLK */ + pins = "gpio75","gpio76"; + function = "cci_i2c"; + }; + + config { + pins = "gpio75","gpio76"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci3_suspend: cci3-suspend { + mux { + /* DATA, CLK */ + pins = "gpio75","gpio76"; + function = "cci_i2c"; + }; + + config { + pins = "gpio75","gpio76"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + + }; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-camera-sensor.dtsi b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-camera-sensor.dtsi new file mode 100644 index 0000000000000..8b75b1a441db4 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-camera-sensor.dtsi @@ -0,0 +1,551 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&cam_cci0 { + //cam1 interpose on dk csi0 + actuator_rear_cam11: qcom,actuator11 { + compatible = "qcom,actuator"; + cell-index = <11>; + cci-master = <1>; + regulator-names = "cam_vaf"; + rgltr-cntrl-support; + rgltr-min-voltage = <2856000>; + rgltr-max-voltage = <3104000>; + rgltr-load-current = <100000>; + }; + + eeprom_rear_cam11: qcom,eeprom11 { + compatible = "qcom,eeprom"; + cell-index = <11>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio", "cam_clk"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000 0>; + rgltr-max-voltage = <1800000 0>; + rgltr-load-current = <120000 0>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 65 0>, + <&tlmm 21 0>, + <&pm7250b_gpios 3 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK1", + "CAM_RESET1", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <1>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam0a-ov9282*/ + qcom,cam-sensor1 { + compatible = "qcom,cam-sensor"; + cell-index = <1>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk0_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk0_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 64 0>, + <&tlmm 20 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAMIF_MCLK0", + "CAM_RESET0"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam1-ov9282*/ + qcom,cam-sensor6 { + compatible = "qcom,cam-sensor"; + cell-index = <6>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 65 0>, + <&tlmm 21 0>, + <&pm7250b_gpios 3 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK1", + "CAM_RESET1", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <1>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam1-imx586/imx686*/ + qcom,cam-sensor11 { + compatible = "qcom,cam-sensor"; + cell-index = <11>; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + actuator-src = <&actuator_rear_cam11>; + eeprom-src = <&eeprom_rear_cam11>; + cam_vio-supply = <&vreg_l18b_1p8>; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + regulator-names = "cam_vio", "cam_clk"; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000 0>; + rgltr-max-voltage = <1800000 0>; + rgltr-load-current = <120000 0>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk1_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk1_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 65 0>, + <&tlmm 21 0>, + <&pm7250b_gpios 3 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK1", + "CAM_RESET1", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <1>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + +}; + +&cam_cci1 { + eeprom_cam2: qcom,eeprom0 { + compatible = "qcom,eeprom"; + cell-index = <8>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 66 0>, + <&tlmm 77 0>, + <&pm7250b_gpios 7 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK2", + "CAM_RESET2", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + eeprom_cam3: qcom,eeprom3 { + compatible = "qcom,eeprom"; + cell-index = <0>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active + &cam_sensor_active_rst3>; + pinctrl-1 = <&cam_sensor_mclk3_suspend + &cam_sensor_suspend_rst3>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 67 0>, + <&tlmm 78 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAMIF_MCLK3", + "CAM_RESET3"; + sensor-position = <1>; + sensor-mode = <0>; + cci-master = <1>; + qcom,cam-power-seq-type ="cam_reset","cam_vio", + "cam_clk","cam_reset"; + qcom,cam-power-seq-val = "cam_reset","cam_vio", + "cam_mclk","cam_reset"; + qcom,cam-power-seq-cfg-val = <0 1 24000000 1>; + qcom,cam-power-seq-delay = <1 0 1 18>; + clocks = <&camcc CAM_CC_MCLK3_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam3-imx577*/ + qcom,cam-sensor0 { + compatible = "qcom,cam-sensor"; + cell-index = <0>; + csiphy-sd-index = <3>; + eeprom-src = <&eeprom_cam3>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active + &cam_sensor_active_rst3>; + pinctrl-1 = <&cam_sensor_mclk3_suspend + &cam_sensor_suspend_rst3>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 67 0>, + <&tlmm 78 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAMIF_MCLK3", + "CAM_RESET3"; + sensor-mode = <0>; + cci-master = <1>; + clocks = <&camcc CAM_CC_MCLK3_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /* GMSL 1 */ + qcom,cam-sensor4 { + compatible = "qcom,cam-sensor"; + cell-index = <4>; + csiphy-sd-index = <4>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + gpios = <&tlmm 93 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_4"; + sensor-mode = <0>; + cci-master = <1>; + status = "ok"; + }; + + /* GMSL 2 */ + qcom,cam-sensor5 { + compatible = "qcom,cam-sensor"; + cell-index = <5>; + csiphy-sd-index = <3>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + gpios = <&tlmm 93 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_5"; + sensor-mode = <0>; + cci-master = <1>; + status = "ok"; + }; + + /*cam0b-ov9282*/ + qcom,cam-sensor7 { + compatible = "qcom,cam-sensor"; + cell-index = <7>; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk4_active + &cam_sensor_active_rst5>; + pinctrl-1 = <&cam_sensor_mclk4_suspend + &cam_sensor_suspend_rst5>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 68 0>, + <&tlmm 149 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAMIF_MCLK4", + "CAM_RESET5"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK4_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /*cam2-imx577*/ + qcom,cam-sensor8 { + compatible = "qcom,cam-sensor"; + cell-index = <8>; + csiphy-sd-index = <2>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam2>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst2>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst2>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 66 0>, + <&tlmm 77 0>, + <&pm7250b_gpios 7 0>; + gpio-reset = <1>; + gpio-custom1 = <2>; + gpio-req-tbl-num = <0 1 2>; + gpio-req-tbl-flags = <1 0 0>; + gpio-req-tbl-label = "CAMIF_MCLK2", + "CAM_RESET2", + "CAM_CUSTOM1"; + sensor-mode = <0>; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK2_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + status = "ok"; + }; + + /* GMSL 3 */ + qcom,cam-sensor9 { + compatible = "qcom,cam-sensor"; + cell-index = <9>; + csiphy-sd-index = <4>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + gpios = <&tlmm 93 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_9"; + sensor-mode = <0>; + cci-master = <1>; + status = "ok"; + }; + + /* GMSL 4 */ + qcom,cam-sensor10 { + compatible = "qcom,cam-sensor"; + cell-index = <10>; + csiphy-sd-index = <4>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + cam_vio-supply = <&vreg_l18b_1p8>; + regulator-names = "cam_vio"; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + gpios = <&tlmm 93 0>; + gpio-reset = <0>; + gpio-req-tbl-num = <0>; + gpio-req-tbl-flags = <0>; + gpio-req-tbl-label = "CAM_10"; + sensor-mode = <0>; + cci-master = <1>; + status = "ok"; + }; +}; + +&soc { + qcom,cam-res-mgr { + compatible = "qcom,cam-res-mgr"; + gpios-shared-pinctrl = <640>; + shared-pctrl-gpio-names = "gmsl"; + pinctrl-0 = <&cam_sensor_active_gmsl>; + pinctrl-1 = <&cam_sensor_suspend_gmsl>; + pinctrl-names = "gmsl_active", "gmsl_suspend"; + status = "ok"; + }; +}; + +&tlmm { + cam_sensor_active_gmsl: cam-sensor-active-gmsl { + /* GMSL_ENABLE */ + mux { + pins = "gpio93"; + function = "gpio"; + }; + + config { + pins = "gpio93"; + bias-pull-up; + drive-strength = <2>; /* 2 MA */ + output-high; + }; + }; + + cam_sensor_active_rst5: cam-sensor-active-rst5 { + /* RESET 5 */ + mux { + pins = "gpio115"; + function = "gpio"; + }; + + config { + pins = "gpio115"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_suspend_gmsl: cam-sensor-suspend-gmsl { + /* GMSL_ENABLE */ + mux { + pins = "gpio93"; + function = "gpio"; + }; + + config { + pins = "gpio93"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst5: cam-sensor-suspend-rst5 { + /* RESET 5 */ + mux { + pins = "gpio115"; + function = "gpio"; + }; + + config { + pins = "gpio115"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine-staging.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine-staging.dtso new file mode 100644 index 0000000000000..edb284558b5a6 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine-staging.dtso @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include + +&{/} { + qep_vreg: qep_vreg { + compatible = "regulator-fixed"; + regulator-name = "qep_vreg"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + gpio = <&pm7325_gpios 8 0>; + enable-active-high; + }; + + aqr_vreg: aqr_vreg { + compatible = "regulator-fixed"; + regulator-name = "aqr_vreg"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + gpio = <&pm7250b_gpios 4 0>; + enable-active-high; + }; +}; + +&pcie1_port0 { + pcie@0,0 { + pcie@3,0 { + qps615: pci@0,0 { + compatible = "pci1179,0220"; + interrupts-extended = <&tlmm 141 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + pinctrl-names = "default"; + pinctrl-0 = <&aqr_intn_wol_sig>; + phy-reset-gpios = <&qps615 0 GPIO_ACTIVE_LOW>; + phy-supply = <&aqr_vreg>; + reset-deassert-us = <221000>; + + gpio-controller; + #gpio-cells = <2>; + }; + + pci@0,1 { + compatible = "pci1179,0220"; + interrupts-extended = <&tlmm 101 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + pinctrl-names = "default"; + pinctrl-0 = <&napa_intn_wol_sig>; + phy-reset-gpios = <&qps615 1 GPIO_ACTIVE_LOW>; + phy-supply = <&qep_vreg>; + reset-deassert-us = <20000>; + }; + }; + }; +}; + +&pcie0_port { + pcie@0,0 { + pcie@3,0 { + pci@0,0 { + interrupts-extended = <&tlmm 136 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + pinctrl-names = "default"; + pinctrl-0 = <&rtl_rc0_intn_wol_sig>; + phy-mode = "sgmii"; + phy-reset-gpios = <&tlmm 90 GPIO_ACTIVE_LOW>; + reset-deassert-us = <75000>; + }; + + pci@0,1 { + interrupts-extended = <&tlmm 142 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + pinctrl-names = "default"; + pinctrl-0 = <&napa_rc0_intn_wol_sig>; + phy-reset-gpios = <&tlmm 25 GPIO_ACTIVE_LOW>; + phy-supply = <&qep_vreg>; + reset-deassert-us = <20000>; + }; + }; + }; +}; + +&tlmm { + qps615_intn_wol { + aqr_intn_wol_sig: aqr_intn_wol_sig { + pins = "gpio141"; + function = "gpio"; + input-enable; + bias-disable; + }; + + napa_intn_wol_sig: napa_intn_wol_sig { + pins = "gpio101"; + function = "gpio"; + input-enable; + bias-disable; + }; + + rtl_rc0_intn_wol_sig: rtl_rc0_intn_wol_sig { + pins = "gpio136"; + function = "gpio"; + input-enable; + bias-disable; + }; + + napa_rc0_intn_wol_sig: napa_rc0_intn_wol_sig { + pins = "gpio142"; + function = "gpio"; + input-enable; + bias-disable; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso index 619a42b5ef48d..c367ded70ec7c 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso @@ -5,9 +5,37 @@ /dts-v1/; /plugin/; +#include #include #include +&{/} { + + vreg_0p9: regulator-vreg-0p9 { + compatible = "regulator-fixed"; + regulator-name = "VREG_0P9"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-always-on; + regulator-boot-on; + }; + + vreg_1p8: regulator-vreg-1p8 { + compatible = "regulator-fixed"; + regulator-name = "VREG_1P8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-always-on; + regulator-boot-on; + }; +}; + +&remoteproc_wpss { + status = "disabled"; +}; + &spi11 { #address-cells = <1>; #size-cells = <0>; @@ -19,3 +47,262 @@ spi-max-frequency = <20000000>; }; }; + +&pcie0 { + perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&pcie0_reset_n>, <&pcie0_wake_n>, <&pcie0_clkreq_n>; + pinctrl-names = "default"; + + iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, + <0x100 &apps_smmu 0x1c01 0x1>, + <0x208 &apps_smmu 0x1c04 0x1>, + <0x210 &apps_smmu 0x1c05 0x1>, + <0x218 &apps_smmu 0x1c06 0x1>, + <0x300 &apps_smmu 0x1c07 0x1>, + <0x400 &apps_smmu 0x1c08 0x1>, + <0x500 &apps_smmu 0x1c09 0x1>, + <0x501 &apps_smmu 0x1c10 0x1>; + + status = "okay"; +}; + +&pcie0_phy { + vdda-phy-supply = <&vreg_l10c_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; + + status = "okay"; +}; + +&pcie0_port { + #address-cells = <3>; + #size-cells = <2>; + + pcie@0,0 { + compatible = "pci1179,0623"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x2 0xff>; + + vddc-supply = <&vreg_0p9>; + vdd18-supply = <&vreg_1p8>; + vdd09-supply = <&vreg_0p9>; + vddio1-supply = <&vreg_1p8>; + vddio2-supply = <&vreg_1p8>; + vddio18-supply = <&vreg_1p8>; + + i2c-parent = <&i2c1 0x33>; + + resx-gpios = <&tlmm 78 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&pcie0_tc9563_resx_n>; + pinctrl-names = "default"; + + aspm-no-l0s; + aspm-no-l1; + + pcie@1,0 { + reg = <0x20800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x3 0xff>; + + aspm-no-l0s; + aspm-no-l1; + }; + + pcie@2,0 { + reg = <0x21000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x4 0xff>; + + aspm-no-l0s; + aspm-no-l1; + }; + + pcie@3,0 { + reg = <0x21800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x5 0xff>; + + pci@0,0 { + reg = <0x50000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + + pci@0,1 { + reg = <0x50100 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + }; + + }; +}; + +&pcie1 { + iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, + <0x100 &apps_smmu 0x1c81 0x1>, + <0x208 &apps_smmu 0x1c84 0x1>, + <0x210 &apps_smmu 0x1c85 0x1>, + <0x218 &apps_smmu 0x1c86 0x1>, + <0x300 &apps_smmu 0x1c87 0x1>, + <0x408 &apps_smmu 0x1c90 0x1>, + <0x410 &apps_smmu 0x1c91 0x1>, + <0x418 &apps_smmu 0x1c92 0x1>, + <0x500 &apps_smmu 0x1c93 0x1>, + <0x600 &apps_smmu 0x1c94 0x1>, + <0x700 &apps_smmu 0x1c95 0x1>, + <0x701 &apps_smmu 0x1c96 0x1>, + <0x800 &apps_smmu 0x1c97 0x1>, + <0x900 &apps_smmu 0x1c98 0x1>, + <0x901 &apps_smmu 0x1c99 0x1>; +}; + +&pcie1_switch0_dsp1 { + #address-cells = <3>; + #size-cells = <2>; + + pcie@0,0 { + compatible = "pci1179,0623"; + reg = <0x30000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x2 0xff>; + + vddc-supply = <&vdd_ntn_0p9>; + vdd18-supply = <&vdd_ntn_1p8>; + vdd09-supply = <&vdd_ntn_0p9>; + vddio1-supply = <&vdd_ntn_1p8>; + vddio2-supply = <&vdd_ntn_1p8>; + vddio18-supply = <&vdd_ntn_1p8>; + + i2c-parent = <&i2c1 0x77>; + + resx-gpios = <&tlmm 124 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&pcie1_tc9563_resx_n>; + pinctrl-names = "default"; + + aspm-no-l0s; + aspm-no-l1; + + pcie@1,0 { + reg = <0x40800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x3 0xff>; + + aspm-no-l0s; + aspm-no-l1; + }; + + pcie@2,0 { + reg = <0x41000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x4 0xff>; + + aspm-no-l0s; + aspm-no-l1; + }; + + pcie@3,0 { + reg = <0x41800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x5 0xff>; + + pci@0,0 { + reg = <0x50000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + + pci@0,1 { + reg = <0x50100 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + }; + }; +}; + +&tlmm { + pcie0_tc9563_resx_n: pcie0-tc9563-resx-state { + pins = "gpio78"; + function = "gpio"; + bias-disable; + input-disable; + output-enable; + }; + + pcie0_reset_n: pcie0-reset-n-state { + pins = "gpio87"; + function = "gpio"; + drive-strength = <16>; + output-low; + bias-disable; + }; + + pcie0_clkreq_n: pcie0-clkreq-n-state { + pins = "gpio88"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + pcie0_wake_n: pcie0-wake-n-state { + pins = "gpio89"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + pcie1_tc9563_resx_n: pcie1-tc9563-resx-state { + pins = "gpio124"; + function = "gpio"; + bias-disable; + input-disable; + output-enable; + }; + +}; + +&wifi { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-staging.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-staging.dtso new file mode 100644 index 0000000000000..6c112224f0ce9 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-staging.dtso @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include + +&{/} { + qep_vreg: qep_vreg { + compatible = "regulator-fixed"; + regulator-name = "qep_vreg"; + gpio = <&pm7325_gpios 8 0>; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + enable-active-high; + }; + + aqr_vreg: aqr_vreg { + compatible = "regulator-fixed"; + regulator-name = "aqr_vreg"; + gpio = <&pm7250b_gpios 4 0>; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + enable-active-high; + }; +}; + +&pcie1_port0 { + pcie@0,0 { + pcie@3,0 { + /* + * PF0: also acts as the QPS615 GPIO controller. + * gpio-controller / #gpio-cells expose the TC956X + * internal GPIO lines (hardware numbers 0-13) so that + * phy-reset-gpios can reference them. + */ + qps615: pci@0,0 { + interrupts-extended = <&tlmm 141 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + phy-supply = <&aqr_vreg>; + pinctrl-names = "default"; + pinctrl-0 = <&aqr_intn_wol_sig>; + phy-reset-gpios = <&qps615 0 GPIO_ACTIVE_LOW>; + reset-deassert-us = <221000>; + + gpio-controller; + #gpio-cells = <2>; + }; + + pci@0,1 { + interrupts-extended = <&tlmm 101 IRQ_TYPE_EDGE_FALLING>; + interrupt-names = "wol_irq"; + phy-supply = <&qep_vreg>; + pinctrl-names = "default"; + pinctrl-0 = <&napa_intn_wol_sig>; + phy-reset-gpios = <&qps615 1 GPIO_ACTIVE_LOW>; + reset-deassert-us = <20000>; + }; + }; + }; +}; + +&tlmm { + qps615_intn_wol { + aqr_intn_wol_sig: aqr_intn_wol_sig { + mux { + pins = "gpio141"; + function = "gpio"; + }; + + config { + pins = "gpio141"; + input-enable; + bias-disable; + }; + }; + + napa_intn_wol_sig: napa_intn_wol_sig { + mux { + pins = "gpio101"; + function = "gpio"; + }; + + config { + pins = "gpio101"; + input-enable; + bias-disable; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-vision-mezzanine-camx.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-vision-mezzanine-camx.dtso new file mode 100644 index 0000000000000..eb183d7dbd3d5 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-vision-mezzanine-camx.dtso @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "qcs6490-camera.dtsi" +#include "qcs6490-rb3gen2-camera-sensor.dtsi" + +&camss { + status = "disabled"; +}; + +&cci1 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts index 18cea88120014..7174d1fcdab35 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts @@ -44,13 +44,19 @@ stdout-path = "serial0:115200n8"; }; + clocks { + mcp2518fd_osc: can-clk { + compatible = "fixed-clock"; + clock-frequency = <40000000>; + #clock-cells = <0>; + }; + }; + dp-connector { compatible = "dp-connector"; label = "DP"; type = "mini"; - hpd-gpios = <&tlmm 60 GPIO_ACTIVE_HIGH>; - port { dp_connector_in: endpoint { remote-endpoint = <&mdss_edp_out>; @@ -194,7 +200,7 @@ reg = <0>; pmic_glink_hs_in: endpoint { - remote-endpoint = <&usb_1_dwc3_hs>; + remote-endpoint = <&eud_con>; }; }; @@ -255,6 +261,16 @@ }; }; + vdd_micro_usb_vbus: regulator-micro-usb-vbus { + compatible = "regulator-fixed"; + regulator-name = "MICRO_USB_VBUS"; + gpio = <&tlmm 63 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-boot-on; + enable-active-high; + }; + vph_pwr: vph-pwr-regulator { compatible = "regulator-fixed"; regulator-name = "vph_pwr"; @@ -262,6 +278,77 @@ regulator-max-microvolt = <3700000>; }; + vreg_pcie0_1p05: regulator-pcie0-1p05v { + compatible = "regulator-fixed"; + regulator-name = "PCIE0_1.05V"; + gpio = <&pm7250b_gpios 4 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + enable-active-high; + pinctrl-0 = <&upd_pwr_en2_state>; + pinctrl-names = "default"; + }; + + vreg_pcie0_3p3: regulator-pcie0-3p3v-dual { + compatible = "regulator-fixed"; + regulator-name = "PCIE0_3.3V_Dual"; + gpio = <&pm7250b_gpios 1 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + enable-active-high; + pinctrl-0 = <&upd_pwr_en1_state>; + pinctrl-names = "default"; + }; + + vdd_ntn_0p9: regulator-vdd-ntn-0p9 { + compatible = "regulator-fixed"; + regulator-name = "VDD_NTN_0P9"; + gpio = <&pm8350c_gpios 2 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <899400>; + regulator-max-microvolt = <899400>; + enable-active-high; + pinctrl-0 = <&ntn_0p9_en>; + pinctrl-names = "default"; + regulator-enable-ramp-delay = <4300>; + }; + + vdd_ntn_1p8: regulator-vdd-ntn-1p8 { + compatible = "regulator-fixed"; + regulator-name = "VDD_NTN_1P8"; + gpio = <&pm8350c_gpios 3 GPIO_ACTIVE_HIGH>; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + enable-active-high; + pinctrl-0 = <&ntn_1p8_en>; + pinctrl-names = "default"; + regulator-enable-ramp-delay = <10000>; + }; + + usb2-connector { + compatible = "gpio-usb-b-connector", + "usb-b-connector"; + label = "micro-USB"; + type = "micro"; + id-gpios = <&tlmm 61 GPIO_ACTIVE_HIGH>; + vbus-gpios = <&pm7325_gpios 9 GPIO_ACTIVE_HIGH>; + vbus-supply = <&vdd_micro_usb_vbus>; + pinctrl-0 = <&usb2_id_detect>; + pinctrl-names = "default"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb2_connector_ep: endpoint { + remote-endpoint = <&usb2_controller_ep>; + }; + }; + }; + }; + wcn6750-pmu { compatible = "qcom,wcn6750-pmu"; pinctrl-0 = <&bt_en>; @@ -648,6 +735,10 @@ }; }; +&eud_con { + remote-endpoint = <&pmic_glink_hs_in>; +}; + &gcc { protected-clocks = , , @@ -666,7 +757,7 @@ }; &gpi_dma0 { - status = "okay"; + status = "disabled"; }; &gpi_dma1 { @@ -688,6 +779,7 @@ lt9611_codec: hdmi-bridge@2b { compatible = "lontium,lt9611uxc"; reg = <0x2b>; + #sound-dai-cells = <1>; interrupts-extended = <&tlmm 24 IRQ_TYPE_EDGE_FALLING>; reset-gpios = <&pm7250b_gpios 2 GPIO_ACTIVE_HIGH>; @@ -764,6 +856,72 @@ }; }; }; + + i2c-mux@71 { + compatible = "nxp,pca9847"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x71>; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <1>; + + usb-hub@2d { + compatible = "smsc,usb4604"; + reg = <0x2d>; + }; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + usb-hub@2d { + compatible = "smsc,usb4604"; + reg = <0x2d>; + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <3>; + }; + + i2c@4 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <4>; + }; + + i2c@5 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <5>; + }; + + i2c@6 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <6>; + }; + + i2c@7 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <7>; + }; + }; }; &lpass_va_macro { @@ -783,7 +941,7 @@ }; &mdss_dp_out { - data-lanes = <0 1>; + data-lanes = <0 1 2 3>; remote-endpoint = <&usb_dp_qmpphy_dp_in>; }; @@ -843,6 +1001,123 @@ status = "okay"; }; +&pcie1_port0 { + pcie@0,0 { + compatible = "pci1179,0623"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x2 0xff>; + + vddc-supply = <&vdd_ntn_0p9>; + vdd18-supply = <&vdd_ntn_1p8>; + vdd09-supply = <&vdd_ntn_0p9>; + vddio1-supply = <&vdd_ntn_1p8>; + vddio2-supply = <&vdd_ntn_1p8>; + vddio18-supply = <&vdd_ntn_1p8>; + + i2c-parent = <&i2c0 0x77>; + + resx-gpios = <&pm8350c_gpios 1 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&tc9563_resx_n>; + pinctrl-names = "default"; + + aspm-no-l0s; + aspm-no-l1; + + pcie1_switch0_dsp1: pcie@1,0 { + reg = <0x20800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x3 0xff>; + + aspm-no-l0s; + aspm-no-l1; + }; + + pcie@2,0 { + reg = <0x21000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + + device_type = "pci"; + ranges; + bus-range = <0x4 0xff>; + + aspm-no-l0s; + aspm-no-l1; + + /* Renesas μPD720201 PCIe USB3.0 Host Controller */ + usb-controller@0,0 { + compatible = "pci1912,0014"; + reg = <0x40000 0x0 0x0 0x0 0x0>; + + avdd33-supply = <&vreg_pcie0_3p3>; + vdd10-supply = <&vreg_pcie0_1p05>; + vdd33-supply = <&vreg_pcie0_3p3>; + + pinctrl-0 = <&upd_hub_rst_state>; + pinctrl-names = "default"; + + #address-cells = <1>; + #size-cells = <0>; + + /* Genesys Logic GL3590 USB Hub Controller */ + gl3590_2_0: hub@1 { + compatible = "usb5e3,610"; + reg = <1>; + + reset-gpios = <&tlmm 162 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&usb_hub_reset_state>; + pinctrl-names = "default"; + + peer-hub = <&gl3590_3_0>; + }; + + gl3590_3_0: hub@2 { + compatible = "usb5e3,625"; + reg = <2>; + + peer-hub = <&gl3590_2_0>; + }; + }; + }; + + pcie@3,0 { + reg = <0x21800 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + bus-range = <0x5 0xff>; + + pci@0,0 { + reg = <0x50000 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + + pci@0,1 { + reg = <0x50100 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + ranges; + }; + }; + }; +}; + &pm7325_gpios { kypd_vol_up_n: kypd-vol-up-n-state { pins = "gpio6"; @@ -975,6 +1250,13 @@ status = "okay"; }; +&psci { + reboot-mode { + mode-bootloader = <0x10001 0x2>; + mode-edl = <0 0x1>; + }; +}; + &qup_uart7_cts { /* * Configure a bias-bus-hold on CTS to lower power @@ -1009,10 +1291,12 @@ }; &qupv3_id_0 { + firmware-name = "qcom/qcs6490/qupv3fw.elf"; status = "okay"; }; &qupv3_id_1 { + firmware-name = "qcom/qcs6490/qupv3fw.elf"; status = "okay"; }; @@ -1052,6 +1336,9 @@ compatible = "qcom,qcs6490-rb3gen2-sndcard"; model = "QCS6490-RB3Gen2"; + pinctrl-0 = <&mi2s1_data0>, <&mi2s1_mclk>, <&mi2s1_sclk>, <&mi2s1_ws>; + pinctrl-names = "default"; + audio-routing = "SpkrLeft IN", "WSA_SPK1 OUT", "SpkrRight IN", "WSA_SPK2 OUT", "VA DMIC0", "vdd-micb", @@ -1091,6 +1378,60 @@ sound-dai = <&q6apm>; }; }; + + mi2s1-playback-dai-link { + link-name = "Secondary MI2S Playback"; + + codec { + sound-dai = <<9611_codec 0>; + }; + + cpu { + sound-dai = <&q6apmbedai SECONDARY_MI2S_RX>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + dp-dai-link { + link-name = "DisplayPort0 Playback"; + + codec { + sound-dai = <&mdss_dp>; + }; + + cpu { + sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; +}; + +&spi3 { + status = "okay"; + + can@0 { + compatible = "microchip,mcp2518fd"; + reg = <0>; + interrupts-extended = <&tlmm 7 IRQ_TYPE_LEVEL_LOW>; + clocks = <&mcp2518fd_osc>; + spi-max-frequency = <10000000>; + vdd-supply = <&vreg_l11c_2p8>; + gpio-controller; + #gpio-cells = <2>; + + gpio0-hog { + gpio-hog; + gpios = <0 GPIO_ACTIVE_LOW>; + output-high; + line-name = "mcp251xfd-gpio0"; + }; + }; }; &swr2 { @@ -1119,6 +1460,49 @@ }; }; +&pm8350c_gpios { + ntn_0p9_en: ntn-0p9-en-state { + pins = "gpio2"; + function = "normal"; + + bias-disable; + input-disable; + output-enable; + power-source = <0>; + }; + + ntn_1p8_en: ntn-1p8-en-state { + pins = "gpio3"; + function = "normal"; + + bias-disable; + input-disable; + output-enable; + power-source = <0>; + }; + + upd_hub_rst_state: upd-hub-rst-state { + pins = "gpio4"; + function = "normal"; + + bias-disable; + input-disable; + output-enable; + output-high; + power-source = <0>; + }; + + tc9563_resx_n: tc9563-resx-state { + pins = "gpio1"; + function = "normal"; + + bias-disable; + input-disable; + output-enable; + power-source = <0>; + }; +}; + &tlmm { gpio-reserved-ranges = <32 2>, /* ADSP */ <48 4>; /* NFC */ @@ -1175,6 +1559,12 @@ */ bias-pull-up; }; + + usb2_id_detect: usb2-id-detect-state { + pins = "gpio61"; + function = "gpio"; + bias-pull-up; + }; }; &uart5 { @@ -1213,10 +1603,6 @@ status = "okay"; }; -&usb_1_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in>; -}; - &usb_1_dwc3_ss { remote-endpoint = <&usb_dp_qmpphy_usb_ss_in>; }; @@ -1233,11 +1619,37 @@ vdda-phy-supply = <&vreg_l6b_1p2>; vdda-pll-supply = <&vreg_l1b_0p912>; + mode-switch; orientation-switch; status = "okay"; }; +&usb_2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb2_controller_ep: endpoint { + remote-endpoint = <&usb2_connector_ep>; + }; + }; + }; +}; + +&usb_2_hsphy { + vdda-pll-supply = <&vreg_l10c_0p88>; + vdda18-supply = <&vreg_l1c_1p8>; + vdda33-supply = <&vreg_l2b_3p072>; + + status = "okay"; +}; + &usb_dp_qmpphy_out { remote-endpoint = <&redriver_phy_con_ss>; }; @@ -1283,11 +1695,19 @@ /* PINCTRL - ADDITIONS TO NODES IN PARENT DEVICE TREE FILES */ &edp_hot_plug_det { - function = "gpio"; bias-disable; }; &pm7250b_gpios { + upd_pwr_en1_state: upd-pwr-en1-state { + pins = "gpio1"; + function = "normal"; + + output-enable; + input-disable; + power-source = <0>; + }; + lt9611_rst_pin: lt9611-rst-state { pins = "gpio2"; function = "normal"; @@ -1296,6 +1716,15 @@ input-disable; power-source = <0>; }; + + upd_pwr_en2_state: upd-pwr-en2-state { + pins = "gpio4"; + function = "normal"; + + output-enable; + input-disable; + power-source = <0>; + }; }; &sdc2_clk { @@ -1341,9 +1770,40 @@ function = "gpio"; bias-pull-up; }; + + usb_hub_reset_state: usb-hub-reset-state { + pins = "gpio162"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; }; &lpass_audiocc { compatible = "qcom,qcm6490-lpassaudiocc"; /delete-property/ power-domains; }; + +&mi2s1_data0 { + drive-strength = <8>; + bias-disable; +}; + +&mi2s1_mclk { + drive-strength = <8>; + bias-disable; + output-high; +}; + +&mi2s1_sclk { + drive-strength = <8>; + bias-disable; + output-high; +}; + +&mi2s1_ws { + drive-strength = <8>; + bias-disable; + output-high; +}; diff --git a/arch/arm64/boot/dts/qcom/qcs8300-ride-camx.dtso b/arch/arm64/boot/dts/qcom/qcs8300-ride-camx.dtso new file mode 100644 index 0000000000000..8f26e6dc070b0 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/qcs8300-ride-camx.dtso @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "monaco-camera.dtsi" +#include "monaco-camera-sensor.dtsi" diff --git a/arch/arm64/boot/dts/qcom/qcs8300-ride.dts b/arch/arm64/boot/dts/qcom/qcs8300-ride.dts index cabb3f508704b..4e2fb94a147c5 100644 --- a/arch/arm64/boot/dts/qcom/qcs8300-ride.dts +++ b/arch/arm64/boot/dts/qcom/qcs8300-ride.dts @@ -8,8 +8,8 @@ #include #include -#include "qcs8300.dtsi" -#include "qcs8300-pmics.dtsi" +#include "monaco.dtsi" +#include "monaco-pmics.dtsi" / { model = "Qualcomm Technologies, Inc. QCS8300 Ride"; compatible = "qcom,qcs8300-ride", "qcom,qcs8300"; @@ -18,12 +18,37 @@ aliases { serial0 = &uart7; mmc0 = &sdhc_1; + serial1 = &uart2; }; chosen { stdout-path = "serial0:115200n8"; }; + dp0-connector { + compatible = "dp-connector"; + label = "DP0"; + type = "full-size"; + + port { + dp0_connector_in: endpoint { + remote-endpoint = <&mdss_dp0_out>; + }; + }; + }; + + dp-dsi0-connector { + compatible = "dp-connector"; + label = "DSI0"; + type = "full-size"; + + port { + dp_dsi0_connector_in: endpoint { + remote-endpoint = <&dsi2dp_bridge_out>; + }; + }; + }; + regulator-usb2-vbus { compatible = "regulator-fixed"; regulator-name = "USB2_VBUS"; @@ -33,6 +58,147 @@ enable-active-high; regulator-always-on; }; + + vreg_12p0: regulator-vreg-12p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_12P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + vreg_5p0: regulator-vreg-5p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_5P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + + vin-supply = <&vreg_12p0>; + }; + + vreg_1p8: regulator-vreg-1p8 { + compatible = "regulator-fixed"; + regulator-name = "VREG_1P8"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + vin-supply = <&vreg_5p0>; + }; + + vreg_1p0: regulator-vreg-1p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_1P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + vin-supply = <&vreg_1p8>; + }; + + vreg_3p0: regulator-vreg-3p0 { + compatible = "regulator-fixed"; + regulator-name = "VREG_3P0"; + + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + vin-supply = <&vreg_12p0>; + }; + + vreg_conn_1p8: vreg-conn-1p8 { + compatible = "regulator-fixed"; + regulator-name = "vreg_conn_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us = <4000>; + enable-active-high; + gpio = <&pmm8650au_1_gpios 4 GPIO_ACTIVE_HIGH>; + }; + + vreg_conn_pa: vreg-conn-pa { + compatible = "regulator-fixed"; + regulator-name = "vreg_conn_pa"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us = <4000>; + enable-active-high; + gpio = <&pmm8650au_1_gpios 6 GPIO_ACTIVE_HIGH>; + }; + + wcn6855-pmu { + compatible = "qcom,wcn6855-pmu"; + + pinctrl-0 = <&wlan_en_state>; + pinctrl-names = "default"; + + vddio-supply = <&vreg_conn_pa>; + vddaon-supply = <&vreg_conn_1p8>; + vddpmu-supply = <&vreg_conn_pa>; + vddpmumx-supply = <&vreg_conn_1p8>; + vddpmucx-supply = <&vreg_conn_pa>; + vddrfa0p95-supply = <&vreg_conn_1p8>; + vddrfa1p3-supply = <&vreg_conn_pa>; + vddrfa1p9-supply = <&vreg_conn_1p8>; + vddpcie1p3-supply = <&vreg_conn_pa>; + vddpcie1p9-supply = <&vreg_conn_1p8>; + + bt-enable-gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>; + wlan-enable-gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>; + + regulators { + vreg_pmu_rfa_cmn: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn"; + }; + + vreg_pmu_aon_0p59: ldo1 { + regulator-name = "vreg_pmu_aon_0p59"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p85: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p85"; + }; + + vreg_pmu_btcmx_0p85: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p85"; + }; + + vreg_pmu_rfa_0p8: ldo5 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo6 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p7: ldo7 { + regulator-name = "vreg_pmu_rfa_1p7"; + }; + + vreg_pmu_pcie_0p9: ldo8 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_pcie_1p8: ldo9 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + }; + }; }; &apps_rsc { @@ -304,6 +470,70 @@ firmware-name = "qcom/qcs8300/a623_zap.mbn"; }; +&i2c8 { + clock-frequency = <400000>; + status = "okay"; + + io_expander: gpio@74 { + compatible = "ti,tca9539"; + reg = <0x74>; + interrupts-extended = <&tlmm 93 IRQ_TYPE_EDGE_BOTH>; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + reset-gpios = <&tlmm 66 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&io_expander_intr_active>, + <&io_expander_reset_active>; + pinctrl-names = "default"; + }; + + i2c-mux@70 { + compatible = "nxp,pca9543"; + reg = <0x70>; + + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + bridge@58 { + compatible = "analogix,anx7625"; + reg = <0x58>; + interrupts-extended = <&io_expander 2 IRQ_TYPE_EDGE_FALLING>; + enable-gpios = <&io_expander 1 GPIO_ACTIVE_HIGH>; + reset-gpios = <&io_expander 0 GPIO_ACTIVE_HIGH>; + vdd10-supply = <&vreg_1p0>; + vdd18-supply = <&vreg_1p8>; + vdd33-supply = <&vreg_3p0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dsi2dp_bridge_in: endpoint { + remote-endpoint = <&mdss_dsi0_out>; + }; + }; + + port@1 { + reg = <1>; + dsi2dp_bridge_out: endpoint { + remote-endpoint = <&dp_dsi0_connector_in>; + }; + }; + }; + }; + }; + }; +}; + &pmm8650au_1_gpios { usb2_en: usb2-en-state { pins = "gpio7"; @@ -317,10 +547,121 @@ status = "okay"; }; +&mdss { + status = "okay"; +}; + +&mdss_dp0 { + pinctrl-0 = <&dp_hot_plug_det>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&mdss_dp0_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + remote-endpoint = <&dp0_connector_in>; +}; + +&mdss_dp0_phy { + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&mdss_dsi0 { + vdda-supply = <&vreg_l5a>; + + status = "okay"; +}; + +&mdss_dsi0_phy { + vdds-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&mdss_dsi0_out { + data-lanes = <0 1 2 3>; + remote-endpoint = <&dsi2dp_bridge_in>; +}; + +&pcie0 { + pinctrl-0 = <&pcie0_default_state>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcieport0 { + reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>; +}; + +&pcie0_phy { + vdda-phy-supply = <&vreg_l6a>; + vdda-pll-supply = <&vreg_l5a>; + vdda-qref-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&pcie1 { + pinctrl-0 = <&pcie1_default_state>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcieport1 { + reset-gpios = <&tlmm 23 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>; +}; + +&pcie1_phy { + vdda-phy-supply = <&vreg_l6a>; + vdda-pll-supply = <&vreg_l5a>; + vdda-qref-supply = <&vreg_l4a>; + + status = "okay"; +}; + +&pcieport0 { + wifi@0 { + compatible = "pci17cb,1103"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + + qcom,calibration-variant = "QC_QCS8300_Ride"; + }; +}; + +&psci { + reboot-mode { + mode-bootloader = <0x10001 0x2>; + mode-edl = <0 0x1>; + }; +}; + &qupv3_id_0 { status = "okay"; }; +&qupv3_id_1 { + status = "okay"; +}; + &remoteproc_adsp { firmware-name = "qcom/qcs8300/adsp.mbn"; status = "okay"; @@ -338,6 +679,8 @@ &serdes0 { phy-supply = <&vreg_l5a>; + vdda-0p9-supply = <&vreg_l4a>; + status = "okay"; }; @@ -354,6 +697,7 @@ vmmc-supply = <&vreg_l8a>; vqmmc-supply = <&vreg_s4a>; + supports-cqe; non-removable; no-sd; no-sdio; @@ -362,6 +706,13 @@ }; &tlmm { + bt_en_state: bt-en-state { + pins = "gpio55"; + function = "gpio"; + bias-pull-down; + output-low; + }; + ethernet0_default: ethernet0-default-state { ethernet0_mdc: ethernet0-mdc-pins { pins = "gpio5"; @@ -377,6 +728,99 @@ bias-pull-up; }; }; + + dp_hot_plug_det: dp-hot-plug-det-state { + pins = "gpio94"; + function = "edp0_hot"; + bias-disable; + }; + + io_expander_intr_active: io-expander-intr-active-state { + pins = "gpio93"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + io_expander_reset_active: io-expander-reset-active-state { + pins = "gpio66"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + output-high; + }; + + pcie0_default_state: pcie0-default-state { + wake-pins { + pins = "gpio0"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + clkreq-pins { + pins = "gpio1"; + function = "pcie0_clkreq"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-pins { + pins = "gpio2"; + function = "gpio"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + pcie1_default_state: pcie1-default-state { + wake-pins { + pins = "gpio21"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + clkreq-pins { + pins = "gpio22"; + function = "pcie1_clkreq"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-pins { + pins = "gpio23"; + function = "gpio"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + wlan_en_state: wlan-en-state { + pins = "gpio54"; + function = "gpio"; + bias-pull-up; + output-low; + }; +}; + +&uart2 { + status = "okay"; + + bluetooth: bluetooth { + compatible = "qcom,wcn6855-bt"; + firmware-name = "QCA6698/hpnv21", "QCA6698/hpbtfw21.tlv"; + max-speed = <3200000>; + + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + }; }; &uart7 { diff --git a/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts b/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts index 67ba508e92ba1..3eedfb2cf4799 100644 --- a/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts +++ b/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts @@ -649,7 +649,7 @@ &uart3 { /delete-property/ interrupts; interrupts-extended = <&intc GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>, - <&tlmm 11 IRQ_TYPE_LEVEL_HIGH>; + <&tlmm 11 IRQ_TYPE_EDGE_FALLING>; pinctrl-0 = <&uart3_default>; pinctrl-1 = <&uart3_sleep>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts index bdf2d66e40c62..44ca3e61c33d4 100644 --- a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts +++ b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts @@ -694,7 +694,7 @@ &uart3 { interrupts-extended = <&intc GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>, - <&tlmm 11 IRQ_TYPE_LEVEL_HIGH>; + <&tlmm 11 IRQ_TYPE_EDGE_FALLING>; pinctrl-0 = <&uart3_default>; pinctrl-1 = <&uart3_sleep>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/sa8775p-ride-camx.dtso b/arch/arm64/boot/dts/qcom/sa8775p-ride-camx.dtso new file mode 100644 index 0000000000000..f81fa6565f281 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sa8775p-ride-camx.dtso @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "lemans-camera.dtsi" +#include "lemans-camera-sensor.dtsi" diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi index a0df10a97c7f8..892b3d2f1bf99 100644 --- a/arch/arm64/boot/dts/qcom/sc7180.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi @@ -1599,7 +1599,11 @@ compatible = "qcom,sc7180-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0 0x01d90000 0 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; ipa: ipa@1e40000 { diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi index 4b04dea57ec8c..b2a785662b815 100644 --- a/arch/arm64/boot/dts/qcom/sc7280.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi @@ -190,6 +190,14 @@ qcom,client-id = <1>; qcom,vmid = ; }; + + adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap { + compatible = "shared-dma-pool"; + alloc-ranges = <0x0 0x00000000 0x0 0xffffffff>; + reusable; + alignment = <0x0 0x400000>; + size = <0x0 0x800000>; + }; }; cpus { @@ -858,7 +866,7 @@ interrupts = ; }; - psci { + psci: psci { compatible = "arm,psci-1.0"; method = "smc"; @@ -1045,6 +1053,8 @@ qcom,dll-config = <0x0007642c>; qcom,ddr-config = <0x80040868>; + qcom,ice = <&sdhc_ice>; + mmc-ddr-1_8v; mmc-hs200-1_8v; mmc-hs400-1_8v; @@ -1071,6 +1081,38 @@ }; }; + sdhc_ice: crypto@7c8000 { + compatible = "qcom,sc7280-inline-crypto-engine", + "qcom,inline-crypto-engine"; + reg = <0x0 0x007c8000 0x0 0x18000>; + clocks = <&gcc GCC_SDCC1_ICE_CORE_CLK>, + <&gcc GCC_SDCC1_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&rpmhpd SC7280_CX>; + + operating-points-v2 = <&ice_mmc_opp_table>; + + ice_mmc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + gpi_dma0: dma-controller@900000 { #dma-cells = <3>; compatible = "qcom,sc7280-gpi-dma", "qcom,sm6350-gpi-dma"; @@ -2424,7 +2466,7 @@ status = "disabled"; - pcie@0 { + pcie1_port0: pcie@0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; bus-range = <0x01 0xff>; @@ -2440,7 +2482,7 @@ reg = <0 0x01c0e000 0 0x1000>; clocks = <&gcc GCC_PCIE_1_AUX_CLK>, <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_CLKREF_EN>, + <&rpmhcc RPMH_CXO_CLK>, <&gcc GCC_PCIE1_PHY_RCHNG_CLK>, <&gcc GCC_PCIE_1_PIPE_CLK>; clock-names = "aux", @@ -2573,8 +2615,33 @@ ice: crypto@1d88000 { compatible = "qcom,sc7280-inline-crypto-engine", "qcom,inline-crypto-engine"; - reg = <0 0x01d88000 0 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + reg = <0 0x01d88000 0 0x18000>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; + + operating-points-v2 = <&ice_opp_table>; + + ice_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-75000000 { + opp-hz = /bits/ 64 <75000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; }; cryptobam: dma-controller@1dc4000 { @@ -3338,6 +3405,85 @@ }; }; + tpda@6004000 { + compatible = "qcom,coresight-tpda", "arm,primecell"; + reg = <0x0 0x06004000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1c { + reg = <28>; + qdss_tpda_in28: endpoint { + remote-endpoint = <&spdm_tpdm_out>; + }; + }; + }; + + out-ports { + port { + qdss_tpda_out: endpoint { + remote-endpoint = <&qdss_dl_funnel_in0>; + }; + }; + }; + }; + + funnel@6005000 { + compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; + reg = <0x0 0x06005000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + port { + qdss_dl_funnel_in0: endpoint { + remote-endpoint = <&qdss_tpda_out>; + }; + }; + }; + + out-ports { + port { + qdss_dl_funnel_out: endpoint { + remote-endpoint = <&funnel0_in6>; + }; + }; + }; + }; + + tpdm@600f000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x0600f000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <32>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + spdm_tpdm_out: endpoint { + remote-endpoint = <&qdss_tpda_in28>; + }; + }; + }; + }; + + cti@6010000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x06010000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + funnel@6041000 { compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; reg = <0 0x06041000 0 0x1000>; @@ -3357,6 +3503,14 @@ #address-cells = <1>; #size-cells = <0>; + port@6 { + reg = <6>; + + funnel0_in6: endpoint { + remote-endpoint = <&qdss_dl_funnel_out>; + }; + }; + port@7 { reg = <7>; funnel0_in7: endpoint { @@ -3471,6 +3625,38 @@ }; }; + cti@6b00000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x06b00000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + + cti@6b01000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x06b01000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + + cti@6b02000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x06b02000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + + cti@6b03000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x06b03000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + funnel@6b04000 { compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; reg = <0 0x06b04000 0 0x1000>; @@ -3490,6 +3676,13 @@ #address-cells = <1>; #size-cells = <0>; + port@6 { + reg = <6>; + swao_funnel_in6: endpoint { + remote-endpoint = <&aoss_tpda_out>; + }; + }; + port@7 { reg = <7>; swao_funnel_in: endpoint { @@ -3548,6 +3741,170 @@ }; }; + tpda@6b08000 { + compatible = "qcom,coresight-tpda", "arm,primecell"; + reg = <0x0 0x06b08000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + aoss_tpda_in0: endpoint { + remote-endpoint = <&swao_prio0_tpdm_out>; + }; + }; + + port@1 { + reg = <1>; + + aoss_tpda_in1: endpoint { + remote-endpoint = <&swao_prio1_tpdm_out>; + }; + }; + + port@2 { + reg = <2>; + + aoss_tpda_in2: endpoint { + remote-endpoint = <&swao_prio2_tpdm_out>; + }; + }; + + port@3 { + reg = <3>; + + aoss_tpda_in3: endpoint { + remote-endpoint = <&swao_prio3_tpdm_out>; + }; + }; + + port@4 { + reg = <4>; + + aoss_tpda_in4: endpoint { + remote-endpoint = <&swao_tpdm_out>; + }; + }; + }; + + out-ports { + port { + aoss_tpda_out: endpoint { + remote-endpoint = <&swao_funnel_in6>; + }; + }; + }; + }; + + tpdm@6b09000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x06b09000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + swao_prio0_tpdm_out: endpoint { + remote-endpoint = <&aoss_tpda_in0>; + }; + }; + }; + }; + + tpdm@6b0a000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x06b0a000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + swao_prio1_tpdm_out: endpoint { + remote-endpoint = <&aoss_tpda_in1>; + }; + }; + }; + }; + + tpdm@6b0b000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x06b0b000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + swao_prio2_tpdm_out: endpoint { + remote-endpoint = <&aoss_tpda_in2>; + }; + }; + }; + }; + + tpdm@6b0c000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x06b0c000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + swao_prio3_tpdm_out: endpoint { + remote-endpoint = <&aoss_tpda_in3>; + }; + }; + }; + }; + + tpdm@6b0d000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x06b0d000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,dsb-element-bits = <32>; + qcom,dsb-msrs-num = <32>; + + out-ports { + port { + swao_tpdm_out: endpoint { + remote-endpoint = <&aoss_tpda_in4>; + }; + }; + }; + }; + + cti@6b11000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x06b11000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; + etm@7040000 { compatible = "arm,coresight-etm4x", "arm,primecell"; reg = <0 0x07040000 0 0x1000>; @@ -3986,12 +4343,6 @@ phy-names = "usb2-phy"; maximum-speed = "high-speed"; usb-role-switch; - - port { - usb2_role_switch: endpoint { - remote-endpoint = <&eud_ep>; - }; - }; }; qspi: spi@88dc000 { @@ -4133,6 +4484,9 @@ qcom,glink-channels = "fastrpcglink-apps-dsp"; label = "adsp"; qcom,non-secure-domain; + memory-region = <&adsp_rpc_remote_heap_mem>; + qcom,vmids = ; #address-cells = <1>; #size-cells = <0>; @@ -4154,6 +4508,7 @@ compatible = "qcom,fastrpc-compute-cb"; reg = <5>; iommus = <&apps_smmu 0x1805 0x0>; + qcom,nsessions = <5>; dma-coherent; }; }; @@ -4307,25 +4662,37 @@ }; eud: eud@88e0000 { - compatible = "qcom,sc7280-eud", "qcom,eud"; - reg = <0 0x88e0000 0 0x2000>, - <0 0x88e2000 0 0x1000>; - interrupts-extended = <&pdc 11 IRQ_TYPE_LEVEL_HIGH>; - - status = "disabled"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - eud_ep: endpoint { - remote-endpoint = <&usb2_role_switch>; - }; - }; - }; - }; + compatible = "qcom,sc7280-eud", "qcom,eud"; + reg = <0 0x88e0000 0 0x2000>, + <0 0x88e2000 0 0x1000>; + interrupts-extended = <&pdc 11 IRQ_TYPE_LEVEL_HIGH>; + + #address-cells = <1>; + #size-cells = <0>; + + eud0: eud-path@0 { + reg = <0>; + usb-role-switch; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + eud_usb: endpoint { + remote-endpoint = <&usb_1_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + eud_con: endpoint { + }; + }; + }; + }; + }; nsp_noc: interconnect@a0c0000 { reg = <0 0x0a0c0000 0 0x10000>; @@ -4482,6 +4849,14 @@ }; }; }; + + cooling { + compatible = "qcom,qmi-cooling-cdsp"; + cdsp_sw: cdsp_sw { + label = "cdsp_sw"; + #cooling-cells = <2>; + }; + }; }; usb_1: usb@a600000 { @@ -4547,6 +4922,7 @@ reg = <0>; usb_1_dwc3_hs: endpoint { + remote-endpoint = <&eud_usb>; }; }; @@ -5514,6 +5890,11 @@ function = "mi2s1_ws"; }; + mi2s1_mclk: mi2s1-mclk-state { + pins = "gpio105"; + function = "sec_mi2s"; + }; + pcie0_clkreq_n: pcie0-clkreq-n-state { pins = "gpio88"; function = "pcie0_clkreqn"; @@ -7287,12 +7668,26 @@ type = "hot"; }; + nspss0_alert1: trip-point1 { + temperature = <100000>; + hysteresis = <5000>; + type = "passive"; + }; + nspss0_crit: nspss0-crit { temperature = <110000>; hysteresis = <0>; type = "critical"; }; }; + + cooling-maps { + map0 { + trip = <&nspss0_alert1>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nspss1-thermal { @@ -7305,12 +7700,26 @@ type = "hot"; }; + nspss1_alert1: trip-point1 { + temperature = <100000>; + hysteresis = <5000>; + type = "passive"; + }; + nspss1_crit: nspss1-crit { temperature = <110000>; hysteresis = <0>; type = "critical"; }; }; + + cooling-maps { + map0 { + trip = <&nspss1_alert1>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; video-thermal { diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index 279e5e6beae20..d89938e17e093 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -967,8 +967,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 6 QCOM_GPI_SPI>, - <&gpi_dma2 1 6 QCOM_GPI_SPI>; + dmas = <&gpi_dma2 0 0 QCOM_GPI_I2C>, + <&gpi_dma2 1 0 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -989,8 +989,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 0 QCOM_GPI_I2C>, - <&gpi_dma2 1 0 QCOM_GPI_I2C>; + dmas = <&gpi_dma2 0 0 QCOM_GPI_SPI>, + <&gpi_dma2 1 0 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1011,8 +1011,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 0 QCOM_GPI_SPI>, - <&gpi_dma2 1 0 QCOM_GPI_SPI>; + dmas = <&gpi_dma2 0 1 QCOM_GPI_I2C>, + <&gpi_dma2 1 1 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1033,8 +1033,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 1 QCOM_GPI_I2C>, - <&gpi_dma2 1 1 QCOM_GPI_I2C>; + dmas = <&gpi_dma2 0 1 QCOM_GPI_SPI>, + <&gpi_dma2 1 1 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1069,8 +1069,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 1 QCOM_GPI_SPI>, - <&gpi_dma2 1 1 QCOM_GPI_SPI>; + dmas = <&gpi_dma2 0 2 QCOM_GPI_I2C>, + <&gpi_dma2 1 2 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1091,8 +1091,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 2 QCOM_GPI_I2C>, - <&gpi_dma2 1 2 QCOM_GPI_I2C>; + dmas = <&gpi_dma2 0 2 QCOM_GPI_SPI>, + <&gpi_dma2 1 2 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1131,8 +1131,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 2 QCOM_GPI_SPI>, - <&gpi_dma2 1 2 QCOM_GPI_SPI>; + dmas = <&gpi_dma2 0 3 QCOM_GPI_I2C>, + <&gpi_dma2 1 3 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1153,8 +1153,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 3 QCOM_GPI_I2C>, - <&gpi_dma2 1 3 QCOM_GPI_I2C>; + dmas = <&gpi_dma2 0 3 QCOM_GPI_SPI>, + <&gpi_dma2 1 3 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1175,8 +1175,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 3 QCOM_GPI_SPI>, - <&gpi_dma2 1 3 QCOM_GPI_SPI>; + dmas = <&gpi_dma2 0 4 QCOM_GPI_I2C>, + <&gpi_dma2 1 4 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1197,8 +1197,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 4 QCOM_GPI_I2C>, - <&gpi_dma2 1 4 QCOM_GPI_I2C>; + dmas = <&gpi_dma2 0 4 QCOM_GPI_SPI>, + <&gpi_dma2 1 4 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1241,8 +1241,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 4 QCOM_GPI_SPI>, - <&gpi_dma2 1 4 QCOM_GPI_SPI>; + dmas = <&gpi_dma2 0 5 QCOM_GPI_SPI>, + <&gpi_dma2 1 5 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1285,8 +1285,8 @@ <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma2 0 5 QCOM_GPI_SPI>, - <&gpi_dma2 1 5 QCOM_GPI_SPI>; + dmas = <&gpi_dma2 0 6 QCOM_GPI_SPI>, + <&gpi_dma2 1 6 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1338,7 +1338,7 @@ }; }; - gpi_dma0: dma-controller@900000 { + gpi_dma0: dma-controller@900000 { compatible = "qcom,sc8280xp-gpi-dma", "qcom,sm6350-gpi-dma"; reg = <0 0x00900000 0 0x60000>; @@ -1393,8 +1393,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 7 QCOM_GPI_I2C>, - <&gpi_dma0 1 7 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>, + <&gpi_dma0 1 0 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1415,8 +1415,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>, - <&gpi_dma0 1 0 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 0 QCOM_GPI_SPI>, + <&gpi_dma0 1 0 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1437,8 +1437,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 0 QCOM_GPI_SPI>, - <&gpi_dma0 1 0 QCOM_GPI_SPI>; + dmas = <&gpi_dma0 0 1 QCOM_GPI_I2C>, + <&gpi_dma0 1 1 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1459,8 +1459,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 1 QCOM_GPI_I2C>, - <&gpi_dma0 1 1 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 1 QCOM_GPI_SPI>, + <&gpi_dma0 1 1 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1481,8 +1481,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 1 QCOM_GPI_SPI>, - <&gpi_dma0 1 1 QCOM_GPI_SPI>; + dmas = <&gpi_dma0 0 2 QCOM_GPI_I2C>, + <&gpi_dma0 1 2 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1503,8 +1503,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 2 QCOM_GPI_I2C>, - <&gpi_dma0 1 2 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 2 QCOM_GPI_SPI>, + <&gpi_dma0 1 2 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1539,8 +1539,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 2 QCOM_GPI_SPI>, - <&gpi_dma0 1 2 QCOM_GPI_SPI>; + dmas = <&gpi_dma0 0 3 QCOM_GPI_I2C>, + <&gpi_dma0 1 3 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1561,8 +1561,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 3 QCOM_GPI_I2C>, - <&gpi_dma0 1 3 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 3 QCOM_GPI_SPI>, + <&gpi_dma0 1 3 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1583,8 +1583,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 3 QCOM_GPI_SPI>, - <&gpi_dma0 1 3 QCOM_GPI_SPI>; + dmas = <&gpi_dma0 0 4 QCOM_GPI_I2C>, + <&gpi_dma0 1 4 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1605,8 +1605,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 4 QCOM_GPI_I2C>, - <&gpi_dma0 1 4 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 4 QCOM_GPI_SPI>, + <&gpi_dma0 1 4 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1627,8 +1627,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 4 QCOM_GPI_SPI>, - <&gpi_dma0 1 4 QCOM_GPI_SPI>; + dmas = <&gpi_dma0 0 5 QCOM_GPI_I2C>, + <&gpi_dma0 1 5 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1649,8 +1649,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 5 QCOM_GPI_I2C>, - <&gpi_dma0 1 5 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 5 QCOM_GPI_SPI>, + <&gpi_dma0 1 5 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1671,8 +1671,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 5 QCOM_GPI_SPI>, - <&gpi_dma0 1 5 QCOM_GPI_SPI>; + dmas = <&gpi_dma0 0 6 QCOM_GPI_I2C>, + <&gpi_dma0 1 6 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1693,8 +1693,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 6 QCOM_GPI_I2C>, - <&gpi_dma0 1 6 QCOM_GPI_I2C>; + dmas = <&gpi_dma0 0 6 QCOM_GPI_SPI>, + <&gpi_dma0 1 6 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1715,8 +1715,8 @@ <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma0 0 6 QCOM_GPI_SPI>, - <&gpi_dma0 1 6 QCOM_GPI_SPI>; + dmas = <&gpi_dma0 0 7 QCOM_GPI_I2C>, + <&gpi_dma0 1 7 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1800,8 +1800,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 7 QCOM_GPI_I2C>, - <&gpi_dma1 1 7 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>, + <&gpi_dma1 1 0 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1822,8 +1822,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>, - <&gpi_dma1 1 0 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 0 QCOM_GPI_SPI>, + <&gpi_dma1 1 0 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1844,8 +1844,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 0 QCOM_GPI_SPI>, - <&gpi_dma1 1 0 QCOM_GPI_SPI>; + dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>, + <&gpi_dma1 1 1 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1866,8 +1866,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>, - <&gpi_dma1 1 1 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 1 QCOM_GPI_SPI>, + <&gpi_dma1 1 1 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1888,8 +1888,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 1 QCOM_GPI_SPI>, - <&gpi_dma1 1 1 QCOM_GPI_SPI>; + dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>, + <&gpi_dma1 1 2 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1910,8 +1910,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>, - <&gpi_dma1 1 2 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 2 QCOM_GPI_SPI>, + <&gpi_dma1 1 2 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1932,8 +1932,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 2 QCOM_GPI_SPI>, - <&gpi_dma1 1 2 QCOM_GPI_SPI>; + dmas = <&gpi_dma1 0 3 QCOM_GPI_I2C>, + <&gpi_dma1 1 3 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1954,8 +1954,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 3 QCOM_GPI_I2C>, - <&gpi_dma1 1 3 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 3 QCOM_GPI_SPI>, + <&gpi_dma1 1 3 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -1976,8 +1976,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 3 QCOM_GPI_SPI>, - <&gpi_dma1 1 3 QCOM_GPI_SPI>; + dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>, + <&gpi_dma1 1 4 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -1998,8 +1998,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>, - <&gpi_dma1 1 4 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 4 QCOM_GPI_SPI>, + <&gpi_dma1 1 4 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -2020,8 +2020,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 4 QCOM_GPI_SPI>, - <&gpi_dma1 1 4 QCOM_GPI_SPI>; + dmas = <&gpi_dma1 0 5 QCOM_GPI_I2C>, + <&gpi_dma1 1 5 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -2042,8 +2042,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 5 QCOM_GPI_I2C>, - <&gpi_dma1 1 5 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 5 QCOM_GPI_SPI>, + <&gpi_dma1 1 5 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -2064,8 +2064,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 5 QCOM_GPI_SPI>, - <&gpi_dma1 1 5 QCOM_GPI_SPI>; + dmas = <&gpi_dma1 0 6 QCOM_GPI_I2C>, + <&gpi_dma1 1 6 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -2086,8 +2086,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 6 QCOM_GPI_I2C>, - <&gpi_dma1 1 6 QCOM_GPI_I2C>; + dmas = <&gpi_dma1 0 6 QCOM_GPI_SPI>, + <&gpi_dma1 1 6 QCOM_GPI_SPI>; dma-names = "tx", "rx"; @@ -2108,8 +2108,8 @@ <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "qup-core", "qup-config", "qup-memory"; - dmas = <&gpi_dma1 0 6 QCOM_GPI_SPI>, - <&gpi_dma1 1 6 QCOM_GPI_SPI>; + dmas = <&gpi_dma1 0 7 QCOM_GPI_I2C>, + <&gpi_dma1 1 7 QCOM_GPI_I2C>; dma-names = "tx", "rx"; @@ -5773,8 +5773,12 @@ clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&rpmhpd SC8280XP_NSP>; - power-domain-names = "nsp"; + power-domains = <&rpmhpd SC8280XP_NSP>, + <&rpmhpd SC8280XP_CX>, + <&rpmhpd SC8280XP_MXC>; + power-domain-names = "nsp", + "cx", + "mxc"; memory-region = <&pil_nsp0_mem>; @@ -5904,8 +5908,12 @@ clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&rpmhpd SC8280XP_NSP>; - power-domain-names = "nsp"; + power-domains = <&rpmhpd SC8280XP_NSP>, + <&rpmhpd SC8280XP_CX>, + <&rpmhpd SC8280XP_MXC>; + power-domain-names = "nsp", + "cx", + "mxc"; memory-region = <&pil_nsp1_mem>; diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi index 8b1a45a4e56ed..876a6871745cf 100644 --- a/arch/arm64/boot/dts/qcom/sdm630.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi @@ -598,8 +598,8 @@ }; gpu_speed_bin: gpu-speed-bin@41a0 { - reg = <0x41a2 0x1>; - bits = <5 7>; + reg = <0x41a2 0x2>; + bits = <5 8>; }; }; @@ -1563,6 +1563,7 @@ reg-names = "mdss_phys", "vbif_phys"; power-domains = <&mmcc MDSS_GDSC>; + resets = <&mmcc MDSS_BCR>; clocks = <&mmcc MDSS_AHB_CLK>, <&mmcc MDSS_AXI_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index 8abf3e909502f..5147d6d3cc26b 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -379,6 +379,12 @@ regulator-initial-mode = ; }; + vreg_l23a_3p3: ldo23 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + }; + vreg_l24a_3p075: ldo24 { regulator-min-microvolt = <3088000>; regulator-max-microvolt = <3088000>; @@ -850,7 +856,6 @@ status = "okay"; pinctrl-names = "default"; pinctrl-0 = <&qup_spi0_default>; - cs-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>; can@0 { compatible = "microchip,mcp2517fd"; @@ -1156,6 +1161,7 @@ vdd-1.8-xo-supply = <&vreg_l7a_1p8>; vdd-1.3-rfa-supply = <&vreg_l17a_1p3>; vdd-3.3-ch0-supply = <&vreg_l25a_3p3>; + vdd-3.3-ch1-supply = <&vreg_l23a_3p3>; qcom,snoc-host-cap-8bit-quirk; qcom,calibration-variant = "Thundercomm_DB845C"; diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi index dcfffb271fcf3..1036305231b20 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi @@ -160,7 +160,6 @@ gpio = <&tlmm 88 0>; enable-active-high; - regulator-boot-on; }; }; @@ -257,7 +256,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-always-on; + regulator-boot-on; }; vreg_l17a_1p3: ldo17 { @@ -803,8 +802,8 @@ bias-disable; }; - tri_state_key_default: tri-state-key-default-state { - pins = "gpio40", "gpio42", "gpio26"; + alert_slider_default: alert-slider-default-state { + pins = "gpio126", "gpio52", "gpio24"; function = "gpio"; drive-strength = <2>; bias-disable; diff --git a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts index 75a53f0bbebd0..493c69e991746 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts @@ -158,7 +158,7 @@ }; }; - i2c21 { + i2c-21 { compatible = "i2c-gpio"; sda-gpios = <&tlmm 127 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; scl-gpios = <&tlmm 128 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; @@ -599,15 +599,15 @@ &i2c14 { status = "okay"; - pmic@66 { + max77705: pmic@66 { compatible = "maxim,max77705"; reg = <0x66>; + #interrupt-cells = <1>; interrupt-parent = <&pm8998_gpios>; interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; pinctrl-0 = <&pmic_int_default>; pinctrl-names = "default"; - #address-cells = <1>; - #size-cells = <0>; leds { compatible = "maxim,max77705-rgb"; @@ -646,8 +646,8 @@ reg = <0x69>; compatible = "maxim,max77705-charger"; monitored-battery = <&battery>; - interrupt-parent = <&pm8998_gpios>; - interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + interrupt-parent = <&max77705>; + interrupts = <0>; }; fuel-gauge@36 { @@ -655,8 +655,8 @@ compatible = "maxim,max77705-battery"; power-supplies = <&max77705_charger>; maxim,rsns-microohm = <5000>; - interrupt-parent = <&pm8998_gpios>; - interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + interrupt-parent = <&max77705>; + interrupts = <2>; }; }; diff --git a/arch/arm64/boot/dts/qcom/sm6115.dtsi b/arch/arm64/boot/dts/qcom/sm6115.dtsi index 91fc36b59abf9..8b8395f6a2dfc 100644 --- a/arch/arm64/boot/dts/qcom/sm6115.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6115.dtsi @@ -1715,8 +1715,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-610.0", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; /* There's no (real) GMU, so we have to handle quite a bunch of clocks! */ clocks = <&gpucc GPU_CC_GX_GFX3D_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi index 8459b27cacc72..19a7b9f9ea8bc 100644 --- a/arch/arm64/boot/dts/qcom/sm6350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi @@ -1180,11 +1180,11 @@ <0 0>, <0 0>, <37500000 150000000>, - <75000000 300000000>, <0 0>, <0 0>, <0 0>, - <0 0>; + <0 0>, + <75000000 300000000>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts b/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts index f16b47b6a74c5..4e097ffa18baa 100644 --- a/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts +++ b/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts @@ -99,7 +99,7 @@ reg = <0>; pmic_glink_hs_in: endpoint { - remote-endpoint = <&usb_1_dwc3_hs>; + remote-endpoint = <&eud_con>; }; }; @@ -884,6 +884,10 @@ }; }; +&eud_con { + remote-endpoint = <&pmic_glink_hs_in>; +}; + &gcc { protected-clocks = , , @@ -1435,10 +1439,6 @@ status = "okay"; }; -&usb_1_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in>; -}; - &usb_1_hsphy { vdda-pll-supply = <&vdd_a_usbhs_core>; vdda18-supply = <&vdd_a_usbhs_1p8>; diff --git a/arch/arm64/boot/dts/qcom/sm8250-samsung-common.dtsi b/arch/arm64/boot/dts/qcom/sm8250-samsung-common.dtsi index cf3d917addd82..ef7ea4f72bf99 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-samsung-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250-samsung-common.dtsi @@ -159,7 +159,8 @@ }; &tlmm { - gpio-reserved-ranges = <40 4>; /* I2C (Unused) */ + gpio-reserved-ranges = <20 4>, /* SPI (fingerprint scanner) */ + <40 4>; /* Unused */ }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 23420e6924728..cdaff13325ccb 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -5348,8 +5348,12 @@ ice: crypto@1d88000 { compatible = "qcom,sm8450-inline-crypto-engine", "qcom,inline-crypto-engine"; - reg = <0 0x01d88000 0 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + reg = <0 0x01d88000 0 0x18000>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi index 7724dba75db79..7a56d2625014c 100644 --- a/arch/arm64/boot/dts/qcom/sm8550.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi @@ -2414,7 +2414,11 @@ "qcom,inline-crypto-engine"; reg = <0 0x01d88000 0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; tcsr_mutex: hwlock@1f40000 { @@ -4097,8 +4101,6 @@ usb_1: usb@a600000 { compatible = "qcom,sm8550-dwc3", "qcom,snps-dwc3"; reg = <0x0 0x0a600000 0x0 0xfc100>; - #address-cells = <1>; - #size-cells = <0>; clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>, <&gcc GCC_USB30_PRIM_MASTER_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts new file mode 100644 index 0000000000000..0dc994f4e48d9 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts @@ -0,0 +1,1551 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2023, Linaro Limited + * Copyright (c) 2025, Kancy Joe + */ + +/dts-v1/; + +#include +#include +#include +#include "sm8650.dtsi" +#include "pm8550.dtsi" +#include "pm8550b.dtsi" +#define PMK8550VE_SID 8 +#include "pm8550ve.dtsi" +#include "pm8550vs.dtsi" +#include "pmk8550.dtsi" + +/delete-node/ &rmtfs_mem; +/delete-node/ &hwfence_shbuf; + +/ { + model = "AYANEO Pocket S2 (Pro)"; + compatible = "ayaneo,pocket-s2", "qcom,sm8650"; + chassis-type = "handset"; + + aliases { + serial0 = &uart15; + serial1 = &uart14; + }; + + wcd939x: audio-codec { + compatible = "qcom,wcd9395-codec", "qcom,wcd9390-codec"; + + pinctrl-0 = <&wcd_default>; + pinctrl-names = "default"; + + qcom,micbias1-microvolt = <1800000>; + qcom,micbias2-microvolt = <1800000>; + qcom,micbias3-microvolt = <1800000>; + qcom,micbias4-microvolt = <1800000>; + qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000 500000 500000 500000>; + qcom,mbhc-headset-vthreshold-microvolt = <1700000>; + qcom,mbhc-headphone-vthreshold-microvolt = <50000>; + qcom,rx-device = <&wcd_rx>; + qcom,tx-device = <&wcd_tx>; + + reset-gpios = <&tlmm 107 GPIO_ACTIVE_LOW>; + + vdd-buck-supply = <&vreg_l15b_1p8>; + vdd-rxtx-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l15b_1p8>; + vdd-mic-bias-supply = <&vreg_bob1>; + + #sound-dai-cells = <1>; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + fan: fan { + compatible = "pwm-fan"; + + interrupts-extended = <&tlmm 14 IRQ_TYPE_EDGE_FALLING>; + + pwms = <&pm8550_pwm 3 50000>; + + fan-supply = <&fan_pwr>; + + #cooling-cells = <2>; + cooling-levels = <0 16 32 45 60 80 105 130 155 180 205 230 255>; + + pinctrl-0 = <&fan_int>, <&pwm_fan_ctrl_active>; + pinctrl-1 = <&pwm_fan_ctrl_sleep>; + pinctrl-names = "default", "sleep"; + }; + + fan_pwr: fan-pwr-regulator { + compatible = "regulator-fixed"; + + regulator-name = "fan_pwr"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + + gpios = <&tlmm 125 GPIO_ACTIVE_HIGH>; + enable-active-high; + + vin-supply = <&fan_vdd>; + + pinctrl-0 = <&fan_pwr_pins>; + pinctrl-names = "default"; + }; + + fan_vdd: fan-vdd-regulator { + compatible = "regulator-fixed"; + + regulator-name = "fan_vdd"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + + gpios = <&tlmm 124 GPIO_ACTIVE_HIGH>; + enable-active-high; + + vin-supply = <&vph_pwr>; + + pinctrl-0 = <&fan_vdd_pins>; + pinctrl-names = "default"; + }; + + gpio-keys { + compatible = "gpio-keys"; + + pinctrl-0 = <&volume_up_n>; + pinctrl-names = "default"; + + key-volume-up { + label = "Volume Up"; + linux,code = ; + gpios = <&pm8550_gpios 6 GPIO_ACTIVE_LOW>; + debounce-interval = <15>; + linux,can-disable; + wakeup-source; + }; + }; + + pmic-glink { + compatible = "qcom,sm8650-pmic-glink", + "qcom,sm8550-pmic-glink", + "qcom,pmic-glink"; + #address-cells = <1>; + #size-cells = <0>; + + orientation-gpios = <&tlmm 29 GPIO_ACTIVE_HIGH>; + + connector@0 { + compatible = "usb-c-connector"; + reg = <0>; + + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_hs_in: endpoint { + remote-endpoint = <&usb_1_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss_in: endpoint { + remote-endpoint = <&redriver_ss_out>; + }; + }; + + port@2 { + reg = <2>; + + pmic_glink_sbu: endpoint { + remote-endpoint = <&wcd_usbss_sbu_mux>; + }; + }; + }; + }; + }; + + upd720201_avdd33_reg: upd720201-avdd33-regulator { + compatible = "regulator-fixed"; + + regulator-name = "upd720201_avdd33"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpios = <&tlmm 123 GPIO_ACTIVE_HIGH>; + enable-active-high; + + vin-supply = <&vph_pwr>; + + pinctrl-0 = <&upd720201_avdd33>; + pinctrl-names = "default"; + }; + + upd720201_vdd10_reg: upd720201-vdd10-regulator { + compatible = "regulator-fixed"; + + regulator-name = "upd720201_vdd10"; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + + gpios = <&tlmm 122 GPIO_ACTIVE_HIGH>; + enable-active-high; + + vin-supply = <&vph_pwr>; + + pinctrl-0 = <&upd720201_vdd10>; + pinctrl-names = "default"; + }; + + upd720201_vdd33_reg: upd720201-vdd33-regulator { + compatible = "regulator-fixed"; + + regulator-name = "upd720201_vdd33"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>; + enable-active-high; + + vin-supply = <&vph_pwr>; + + pinctrl-0 = <&upd720201_vdd33>; + pinctrl-names = "default"; + }; + + sound { + compatible = "qcom,sm8650-sndcard", "qcom,sm8450-sndcard"; + model = "SM8650-APS2"; + audio-routing = "SpkrLeft IN", "WSA_SPK1 OUT", + "SpkrRight IN", "WSA_SPK2 OUT", + "IN1_HPHL", "HPHL_OUT", + "IN2_HPHR", "HPHR_OUT", + "AMIC1", "MIC BIAS1", + "AMIC2", "MIC BIAS2", + "AMIC3", "MIC BIAS3", + "AMIC4", "MIC BIAS3", + "AMIC5", "MIC BIAS4", + "TX SWR_INPUT0", "ADC1_OUTPUT", + "TX SWR_INPUT1", "ADC2_OUTPUT", + "TX SWR_INPUT7", "DMIC1_OUTPUT", + "TX SWR_INPUT8", "DMIC2_OUTPUT"; + + wcd-playback-dai-link { + link-name = "WCD Playback"; + + codec { + sound-dai = <&wcd939x 0>, + <&swr1 0>, + <&lpass_rxmacro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai RX_CODEC_DMA_RX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + wcd-capture-dai-link { + link-name = "WCD Capture"; + codec { + sound-dai = <&wcd939x 1>, + <&swr2 0>, + <&lpass_txmacro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai TX_CODEC_DMA_TX_3>; + }; + + + platform { + sound-dai = <&q6apm>; + }; + }; + + wsa-dai-link { + link-name = "WSA Playback"; + + codec { + sound-dai = <&right_spkr>, + <&left_spkr>, + <&swr3 0>, + <&lpass_wsa2macro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + va-dai-link { + link-name = "VA Capture"; + + codec { + sound-dai = <&lpass_vamacro 0>; + }; + + cpu { + sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + dp-dai-link { + link-name = "DisplayPort Playback"; + + codec { + sound-dai = <&mdss_dp0>; + }; + + cpu { + sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + }; + + vph_pwr: vph-pwr-regulator { + compatible = "regulator-fixed"; + + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + + regulator-always-on; + regulator-boot-on; + }; + + wcn7850-pmu { + compatible = "qcom,wcn7850-pmu"; + + pinctrl-names = "default"; + pinctrl-0 = <&wlan_en>, <&bt_default>; + + wlan-enable-gpios = <&tlmm 16 GPIO_ACTIVE_HIGH>; + bt-enable-gpios = <&tlmm 17 GPIO_ACTIVE_HIGH>; + + vdd-supply = <&vreg_s4i_0p85>; + vddio-supply = <&vreg_l15b_1p8>; + vddio1p2-supply = <&vreg_l3c_1p2>; + vddaon-supply = <&vreg_s2c_0p8>; + vdddig-supply = <&vreg_s3c_0p9>; + vddrfa1p2-supply = <&vreg_s1c_1p2>; + vddrfa1p8-supply = <&vreg_s6c_1p8>; + + clocks = <&rpmhcc RPMH_RF_CLK1>; + + regulators { + vreg_pmu_rfa_cmn: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn"; + }; + + vreg_pmu_aon_0p59: ldo1 { + regulator-name = "vreg_pmu_aon_0p59"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p85: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p85"; + }; + + vreg_pmu_btcmx_0p85: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p85"; + }; + + vreg_pmu_rfa_0p8: ldo5 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo6 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p8: ldo7 { + regulator-name = "vreg_pmu_rfa_1p8"; + }; + + vreg_pmu_pcie_0p9: ldo8 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_pcie_1p8: ldo9 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + }; + }; +}; + +&apps_rsc { + regulators-0 { + compatible = "qcom,pm8550-rpmh-regulators"; + + vdd-bob1-supply = <&vph_pwr>; + vdd-bob2-supply = <&vph_pwr>; + vdd-l2-l13-l14-supply = <&vreg_bob1>; + vdd-l3-supply = <&vreg_s1c_1p2>; + vdd-l5-l16-supply = <&vreg_bob1>; + vdd-l6-l7-supply = <&vreg_bob1>; + vdd-l8-l9-supply = <&vreg_bob1>; + vdd-l11-supply = <&vreg_s1c_1p2>; + vdd-l12-supply = <&vreg_s6c_1p8>; + vdd-l15-supply = <&vreg_s6c_1p8>; + vdd-l17-supply = <&vreg_bob2>; + + qcom,pmic-id = "b"; + + vreg_bob1: bob1 { + regulator-name = "vreg_bob1"; + regulator-min-microvolt = <3296000>; + regulator-max-microvolt = <3960000>; + regulator-initial-mode = ; + }; + + vreg_bob2: bob2 { + regulator-name = "vreg_bob2"; + regulator-min-microvolt = <2720000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + }; + + vreg_l2b_3p0: ldo2 { + regulator-name = "vreg_l2b_3p0"; + regulator-min-microvolt = <3008000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l5b_3p1: ldo5 { + regulator-name = "vreg_l5b_3p1"; + regulator-min-microvolt = <3104000>; + regulator-max-microvolt = <3104000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l6b_1p8: ldo6 { + regulator-name = "vreg_l6b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l7b_1p8: ldo7 { + regulator-name = "vreg_l7b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + }; + + vreg_l8b_1p8: ldo8 { + regulator-name = "vreg_l8b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l9b_2p9: ldo9 { + regulator-name = "vreg_l9b_2p9"; + regulator-min-microvolt = <2960000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l11b_1p2: ldo11 { + regulator-name = "vreg_l11b_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1504000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l12b_1p8: ldo12 { + regulator-name = "vreg_l12b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l13b_3p0: ldo13 { + regulator-name = "vreg_l13b_3p0"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l14b_3p2: ldo14 { + regulator-name = "vreg_l14b_3p2"; + regulator-min-microvolt = <3200000>; + regulator-max-microvolt = <3200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l15b_1p8: ldo15 { + regulator-name = "vreg_l15b_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l16b_2p8: ldo16 { + regulator-name = "vreg_l16b_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + regulator-always-on; + regulator-boot-on; + }; + + vreg_l17b_2p5: ldo17 { + regulator-name = "vreg_l17b_2p5"; + regulator-min-microvolt = <2504000>; + regulator-max-microvolt = <2504000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-1 { + compatible = "qcom,pm8550vs-rpmh-regulators"; + + vdd-l1-supply = <&vreg_s1c_1p2>; + vdd-l2-supply = <&vreg_s1c_1p2>; + vdd-l3-supply = <&vreg_s1c_1p2>; + vdd-s1-supply = <&vph_pwr>; + vdd-s2-supply = <&vph_pwr>; + vdd-s3-supply = <&vph_pwr>; + vdd-s4-supply = <&vph_pwr>; + vdd-s5-supply = <&vph_pwr>; + vdd-s6-supply = <&vph_pwr>; + + qcom,pmic-id = "c"; + + vreg_s1c_1p2: smps1 { + regulator-name = "vreg_s1c_1p2"; + regulator-min-microvolt = <1256000>; + regulator-max-microvolt = <1348000>; + regulator-initial-mode = ; + }; + + vreg_s2c_0p8: smps2 { + regulator-name = "vreg_s2c_0p8"; + regulator-min-microvolt = <852000>; + regulator-max-microvolt = <1036000>; + regulator-initial-mode = ; + }; + + vreg_s3c_0p9: smps3 { + regulator-name = "vreg_s3c_0p9"; + regulator-min-microvolt = <976000>; + regulator-max-microvolt = <1064000>; + regulator-initial-mode = ; + }; + + vreg_s4c_1p2: smps4 { + regulator-name = "vreg_s4c_1p2"; + regulator-min-microvolt = <1224000>; + regulator-max-microvolt = <1280000>; + regulator-initial-mode = ; + }; + + vreg_s5c_0p7: smps5 { + regulator-name = "vreg_s5c_0p7"; + regulator-min-microvolt = <752000>; + regulator-max-microvolt = <900000>; + regulator-initial-mode = ; + }; + + vreg_s6c_1p8: smps6 { + regulator-name = "vreg_s6c_1p8"; + regulator-min-microvolt = <1856000>; + regulator-max-microvolt = <2000000>; + regulator-initial-mode = ; + }; + + vreg_l1c_1p2: ldo1 { + regulator-name = "vreg_l1c_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3c_1p2: ldo3 { + regulator-name = "vreg_l3c_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + regulator-always-on; + regulator-boot-on; + }; + }; + + regulators-2 { + compatible = "qcom,pm8550vs-rpmh-regulators"; + + vdd-l1-supply = <&vreg_s3c_0p9>; + + qcom,pmic-id = "d"; + + vreg_l1d_0p88: ldo1 { + regulator-name = "vreg_l1d_0p88"; + regulator-min-microvolt = <912000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-3 { + compatible = "qcom,pm8550vs-rpmh-regulators"; + + vdd-l3-supply = <&vreg_s3c_0p9>; + + qcom,pmic-id = "e"; + + vreg_l3e_0p9: ldo3 { + regulator-name = "vreg_l3e_0p9"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-4 { + compatible = "qcom,pm8550vs-rpmh-regulators"; + + vdd-l1-supply = <&vreg_s3c_0p9>; + vdd-l3-supply = <&vreg_s3c_0p9>; + + qcom,pmic-id = "g"; + + vreg_l1g_0p91: ldo1 { + regulator-name = "vreg_l1g_0p91"; + regulator-min-microvolt = <912000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3g_0p91: ldo3 { + regulator-name = "vreg_l3g_0p91"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; + + regulators-5 { + compatible = "qcom,pm8550ve-rpmh-regulators"; + + vdd-l1-supply = <&vreg_s3c_0p9>; + vdd-l2-supply = <&vreg_s3c_0p9>; + vdd-l3-supply = <&vreg_s1c_1p2>; + vdd-s4-supply = <&vph_pwr>; + + qcom,pmic-id = "i"; + + vreg_s4i_0p85: smps4 { + regulator-name = "vreg_s4i_0p85"; + regulator-min-microvolt = <852000>; + regulator-max-microvolt = <1004000>; + regulator-initial-mode = ; + }; + + vreg_l1i_0p88: ldo1 { + regulator-name = "vreg_l1i_0p88"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2i_0p88: ldo2 { + regulator-name = "vreg_l2i_0p88"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3i_1p2: ldo3 { + regulator-name = "vreg_l3i_0p91"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + }; +}; + +&cpu2_top_thermal { + trips { + cpu2_active: cpu2-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; + }; + + cooling-maps { + map { + trip = <&cpu2_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; +}; + +&cpu3_top_thermal { + trips { + cpu3_active: cpu3-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; + }; + + cooling-maps { + map { + trip = <&cpu3_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; +}; + +&cpu4_top_thermal { + trips { + cpu4_active: cpu4-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; + }; + + cooling-maps { + map { + trip = <&cpu4_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; +}; + +&cpu5_top_thermal { + trips { + cpu5_active: cpu5-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; + }; + + cooling-maps { + map { + trip = <&cpu5_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; +}; + +&cpu6_top_thermal { + trips { + cpu6_active: cpu6-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; + }; + + cooling-maps { + map { + trip = <&cpu6_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; +}; + +&cpu7_top_thermal { + trips { + cpu7_active: cpu7-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; + }; + + cooling-maps { + map { + trip = <&cpu7_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; +}; + +&gpi_dma1 { + status = "okay"; +}; + +&gpi_dma2 { + status = "okay"; +}; + +&gpu0_cooling_maps { + map1 { + trip = <&gpu0_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu1_cooling_maps { + map1 { + trip = <&gpu1_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu2_cooling_maps { + map1 { + trip = <&gpu2_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu3_cooling_maps { + map1 { + trip = <&gpu3_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu4_cooling_maps { + map1 { + trip = <&gpu4_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu5_cooling_maps { + map1 { + trip = <&gpu5_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu6_cooling_maps { + map1 { + trip = <&gpu6_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu7_cooling_maps { + map1 { + trip = <&gpu7_active>; + cooling-device = <&fan THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; +}; + +&gpu0_trips { + gpu0_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; +}; + +&gpu1_trips { + gpu1_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; +}; + +&gpu2_trips { + gpu2_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; +}; + +&gpu3_trips { + gpu3_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; +}; + +&gpu4_trips { + gpu4_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; +}; + +&gpu5_trips { + gpu5_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; +}; + +&gpu6_trips { + gpu6_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; + +}; + +&gpu7_trips { + gpu7_active: trip-active { + temperature = <38000>; + hysteresis = <2000>; + type = "active"; + }; +}; + +&i2c3 { + clock-frequency = <100000>; + + status = "okay"; + + wcd_usbss: typec-mux@e { + compatible = "qcom,wcd9395-usbss", "qcom,wcd9390-usbss"; + reg = <0xe>; + + vdd-supply = <&vreg_l15b_1p8>; + reset-gpios = <&tlmm 152 GPIO_ACTIVE_HIGH>; + + mode-switch; + orientation-switch; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + wcd_usbss_sbu_mux: endpoint { + remote-endpoint = <&pmic_glink_sbu>; + }; + }; + }; + }; +}; + +&i2c6 { + clock-frequency = <100000>; + + status = "okay"; + + typec-mux@1c { + compatible = "onnn,nb7vpq904m"; + reg = <0x1c>; + + vcc-supply = <&vreg_l15b_1p8>; + + retimer-switch; + orientation-switch; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + redriver_ss_out: endpoint { + remote-endpoint = <&pmic_glink_ss_in>; + }; + }; + + port@1 { + reg = <1>; + + redriver_ss_in: endpoint { + remote-endpoint = <&usb_dp_qmpphy_out>; + }; + }; + }; + }; +}; + +&iris { + status = "okay"; +}; + +&lpass_wsa2macro { + status = "okay"; +}; + +&mdss { + status = "okay"; +}; + +&mdss_dp0 { + status = "okay"; +}; + +&mdss_dp0_out { + status = "okay"; +}; + +&pcie0 { + wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>; + perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&pcie0_default_state>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcieport0 { + wifi@0 { + compatible = "pci17cb,1107"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + }; +}; + +&pcie0_phy { + vdda-phy-supply = <&vreg_l1i_0p88>; + vdda-pll-supply = <&vreg_l3i_1p2>; + + status = "okay"; +}; + +&pcie1 { + wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>; + perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&pcie1_default_state>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie1_port0 { + /* Renesas μPD720201 PCIe USB3.0 HOST CONTROLLER */ + usb-controller@0 { + compatible = "pci1912,0014"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + avdd33-supply = <&upd720201_avdd33_reg>; + vdd10-supply = <&upd720201_vdd10_reg>; + vdd33-supply = <&upd720201_vdd33_reg>; + + pinctrl-0 = <&gamepad_pwr_en>; + pinctrl-names = "default"; + }; +}; + +&pcie1_phy { + vdda-phy-supply = <&vreg_l3e_0p9>; + vdda-pll-supply = <&vreg_l3i_1p2>; + vdda-qref-supply = <&vreg_l1i_0p88>; + + status = "okay"; +}; + +&pon_pwrkey { + status = "okay"; +}; + +&pon_resin { + linux,code = ; + + status = "okay"; +}; + +&pm8550_gpios { + volume_up_n: volume-up-n-state { + pins = "gpio6"; + function = "normal"; + bias-pull-up; + input-enable; + power-source = <1>; + }; + + pwm_fan_ctrl_active: pwm-fan-ctrl-active-state { + pins = "gpio9"; + function = "func1"; + bias-disable; + power-source = <0>; + qcom,drive-strength = ; + }; + + pwm_fan_ctrl_sleep: pwm-fan-ctrl-sleep-state { + pins = "gpio9"; + function = "normal"; + output-high; + bias-disable; + power-source = <0>; + qcom,drive-strength = ; + }; + + sdc2_card_det_n: sdc2-card-det-state { + pins = "gpio12"; + function = "normal"; + bias-pull-up; + input-enable; + output-disable; + power-source = <1>; /* 1.8 V */ + }; +}; + +&pm8550_pwm { + status = "okay"; + + multi-led { + color = ; + function = LED_FUNCTION_STATUS; + label = "Power Status"; + + #address-cells = <1>; + #size-cells = <0>; + + led@1 { + reg = <1>; + color = ; + }; + + led@2 { + reg = <2>; + color = ; + }; + + led@3 { + reg = <3>; + color = ; + }; + }; +}; + +&pm8550b_eusb2_repeater { + vdd18-supply = <&vreg_l15b_1p8>; + vdd3-supply = <&vreg_l5b_3p1>; +}; + +&qup_i2c3_data_clk { + /* Use internal I2C pull-up */ + bias-pull-up = <2200>; +}; + +&qupv3_id_0 { + status = "okay"; +}; + +&qupv3_id_1 { + status = "okay"; +}; + +&remoteproc_adsp { + firmware-name = "qcom/sm8650/ayaneo/ps2/adsp.mbn", + "qcom/sm8650/ayaneo/ps2/adsp_dtb.mbn"; + + status = "okay"; +}; + +&remoteproc_cdsp { + firmware-name = "qcom/sm8650/ayaneo/ps2/cdsp.mbn", + "qcom/sm8650/ayaneo/ps2/cdsp_dtb.mbn"; + + status = "okay"; +}; + +&reserved_memory { + lost_reg_mem: lost-reg-mem { + reg = <0 0x9b09c000 0 0x4000>; + no-map; + }; + + hwfence_shbuf: hwfence-shbuf@d4e23000 { + reg = <0 0xd4e23000 0 0x2dd000>; + no-map; + }; + + splash_region: splash-region { + label = "cont_splash_region"; + reg = <0 0xd5100000 0 0x2b00000>; + no-map; + }; +}; + +&sdhc_2 { + cd-gpios = <&pm8550_gpios 12 GPIO_ACTIVE_LOW>; + + vmmc-supply = <&vreg_l9b_2p9>; + vqmmc-supply = <&vreg_l8b_1p8>; + bus-width = <4>; + no-sdio; + no-mmc; + + pinctrl-0 = <&sdc2_default>, <&sdc2_card_det_n>; + pinctrl-1 = <&sdc2_sleep>, <&sdc2_card_det_n>; + pinctrl-names = "default", "sleep"; + + status = "okay"; +}; + +&sleep_clk { + clock-frequency = <32764>; +}; + +&swr1 { + status = "okay"; + + /* WCD9395 RX */ + wcd_rx: codec@0,4 { + compatible = "sdw20217010e00"; + reg = <0 4>; + + /* + * WCD9395 RX Port 1 (HPH_L/R) <=> SWR1 Port 1 (HPH_L/R) + * WCD9395 RX Port 2 (CLSH) <=> SWR1 Port 2 (CLSH) + * WCD9395 RX Port 3 (COMP_L/R) <=> SWR1 Port 3 (COMP_L/R) + * WCD9395 RX Port 4 (LO) <=> SWR1 Port 4 (LO) + * WCD9395 RX Port 5 (DSD_L/R) <=> SWR1 Port 5 (DSD_L/R) + * WCD9395 RX Port 6 (HIFI_PCM_L/R) <=> SWR1 Port 9 (HIFI_PCM_L/R) + */ + qcom,rx-port-mapping = <1 2 3 4 5 9>; + }; +}; + +&swr2 { + status = "okay"; + + /* WCD9395 TX */ + wcd_tx: codec@0,3 { + compatible = "sdw20217010e00"; + reg = <0 3>; + + /* + * WCD9395 TX Port 1 (ADC1,2,3,4) <=> SWR2 Port 2 (TX SWR_INPUT 0,1,2,3) + * WCD9395 TX Port 2 (ADC3,4 & DMIC0,1) <=> SWR2 Port 2 (TX SWR_INPUT 0,1,2,3) + * WCD9395 TX Port 3 (DMIC0,1,2,3 & MBHC) <=> SWR2 Port 3 (TX SWR_INPUT 4,5,6,7) + * WCD9395 TX Port 4 (DMIC4,5,6,7) <=> SWR2 Port 4 (TX SWR_INPUT 8,9,10,11) + */ + qcom,tx-port-mapping = <2 2 3 4>; + }; +}; + +&swr3 { + status = "okay"; + + pinctrl-0 = <&wsa2_swr_active>, <&spkr_23_sd_n_active>; + pinctrl-names = "default"; + + /* WSA8845, Speaker Left */ + left_spkr: speaker@0,0 { + compatible = "sdw20217020400"; + reg = <0 0>; + #sound-dai-cells = <0>; + reset-gpios = <&tlmm 77 GPIO_ACTIVE_LOW>; + sound-name-prefix = "SpkrLeft"; + vdd-1p8-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l3c_1p2>; + + /* + * WSA8845 Port 1 (DAC) <=> SWR3 Port 1 (SPKR_L) + * WSA8845 Port 2 (COMP) <=> SWR3 Port 2 (SPKR_L_COMP) + * WSA8845 Port 3 (BOOST) <=> SWR3 Port 3 (SPKR_L_BOOST) + * WSA8845 Port 4 (PBR) <=> SWR3 Port 7 (PBR) + * WSA8845 Port 5 (VISENSE) <=> SWR3 Port 10 (SPKR_L_VI) + * WSA8845 Port 6 (CPS) <=> SWR3 Port 13 (CPS) + */ + qcom,port-mapping = <1 2 3 7 10 13>; + }; + + /* WSA8845, Speaker Right */ + right_spkr: speaker@0,1 { + compatible = "sdw20217020400"; + reg = <0 1>; + #sound-dai-cells = <0>; + reset-gpios = <&tlmm 77 GPIO_ACTIVE_LOW>; + sound-name-prefix = "SpkrRight"; + vdd-1p8-supply = <&vreg_l15b_1p8>; + vdd-io-supply = <&vreg_l3c_1p2>; + + /* + * WSA8845 Port 1 (DAC) <=> SWR3 Port 4 (SPKR_R) + * WSA8845 Port 2 (COMP) <=> SWR3 Port 5 (SPKR_R_COMP) + * WSA8845 Port 3 (BOOST) <=> SWR3 Port 6 (SPKR_R_BOOST) + * WSA8845 Port 4 (PBR) <=> SWR3 Port 7 (PBR) + * WSA8845 Port 5 (VISENSE) <=> SWR3 Port 11 (SPKR_R_VI) + * WSA8845 Port 6 (CPS) <=> SWR3 Port 13 (CPS) + */ + qcom,port-mapping = <4 5 6 7 11 13>; + }; +}; + +&tlmm { + /* Reserved I/Os for NFC */ + gpio-reserved-ranges = <32 4>, <36 1>, <38 6>, <74 1>; + + bt_default: bt-default-state { + bt-en-pins { + pins = "gpio17"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + sw-ctrl-pins { + pins = "gpio18"; + function = "gpio"; + bias-pull-down; + }; + }; + + fan_pwr_pins: fan-pwr-state { + pins = "gpio125"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + fan_vdd_pins: fan-vdd-state { + pins = "gpio124"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + fan_int: fan-int-state { + pins = "gpio14"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + + upd720201_avdd33: upd720201-avdd33-state { + pins = "gpio123"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + upd720201_vdd10: pd720201-vdd10-state { + pins = "gpio122"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + upd720201_vdd33: upd720201-vdd33-state { + pins = "gpio121"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + gamepad_pwr_en: gamepad-pwr-en-active-state { + pins = "gpio28"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + output-high; + }; + + spkr_23_sd_n_active: spkr-23-sd-n-active-state { + pins = "gpio77"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + spkr_01_sd_n_active: spkr-01-sd-n-active-state { + pins = "gpio21"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + wcd_default: wcd-reset-n-active-state { + pins = "gpio107"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + wlan_en: wlan-en-state { + pins = "gpio16"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; +}; + +&uart14 { + status = "okay"; + + bluetooth { + compatible = "qcom,wcn7850-bt"; + + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; + + max-speed = <3200000>; + }; +}; + +&uart15 { + status = "okay"; +}; + +&ufs_mem_hc { + reset-gpios = <&tlmm 210 GPIO_ACTIVE_LOW>; + + vcc-supply = <&vreg_l17b_2p5>; + vcc-max-microamp = <1300000>; + vccq-supply = <&vreg_l1c_1p2>; + vccq-max-microamp = <1200000>; + + status = "okay"; +}; + +&ufs_mem_phy { + vdda-phy-supply = <&vreg_l1d_0p88>; + vdda-pll-supply = <&vreg_l3i_1p2>; + + status = "okay"; +}; + +/* + * DPAUX -> WCD9395 -> USB_SBU -> USB-C + * eUSB2 DP/DM -> PM85550HS -> eUSB2 DP/DM -> WCD9395 -> USB-C + * USB SS -> NB7VPQ904MMUTWG -> USB-C + */ + +&usb_1 { + dr_mode = "otg"; + usb-role-switch; + + status = "okay"; +}; + +&usb_1_dwc3_hs { + remote-endpoint = <&pmic_glink_hs_in>; +}; + +&usb_1_hsphy { + vdd-supply = <&vreg_l1i_0p88>; + vdda12-supply = <&vreg_l3i_1p2>; + + phys = <&pm8550b_eusb2_repeater>; + + status = "okay"; +}; + +&usb_dp_qmpphy { + vdda-phy-supply = <&vreg_l3i_1p2>; + vdda-pll-supply = <&vreg_l3g_0p91>; + + status = "okay"; +}; + +&usb_dp_qmpphy_out { + remote-endpoint = <&redriver_ss_in>; +}; + +&xo_board { + clock-frequency = <76800000>; +}; diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi index ebf1971b1bfbe..842396cf75d95 100644 --- a/arch/arm64/boot/dts/qcom/sm8650.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi @@ -3882,7 +3882,7 @@ }; }; - pcie@0 { + pcie1_port0: pcie@0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; bus-range = <0x01 0xff>; @@ -3988,6 +3988,8 @@ iommus = <&apps_smmu 0x60 0>; + dma-coherent; + lanes-per-direction = <2>; qcom,ice = <&ice>; @@ -4044,7 +4046,11 @@ "qcom,inline-crypto-engine"; reg = <0 0x01d88000 0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { @@ -5113,9 +5119,6 @@ dma-coherent; - #address-cells = <1>; - #size-cells = <0>; - status = "disabled"; ports { @@ -5684,6 +5687,7 @@ phy-names = "dp"; #sound-dai-cells = <0>; + sound-name-prefix = "DisplayPort0"; status = "disabled"; @@ -7513,7 +7517,7 @@ }; }; - cpu2-top-thermal { + cpu2_top_thermal: cpu2-top-thermal { thermal-sensors = <&tsens0 5>; trips { @@ -7537,7 +7541,7 @@ }; }; - cpu3-top-thermal { + cpu3_top_thermal: cpu3-top-thermal { thermal-sensors = <&tsens0 7>; trips { @@ -7561,7 +7565,7 @@ }; }; - cpu4-top-thermal { + cpu4_top_thermal: cpu4-top-thermal { thermal-sensors = <&tsens0 9>; trips { @@ -7585,7 +7589,7 @@ }; }; - cpu5-top-thermal { + cpu5_top_thermal: cpu5-top-thermal { thermal-sensors = <&tsens0 11>; trips { @@ -7609,7 +7613,7 @@ }; }; - cpu6-top-thermal { + cpu6_top_thermal: cpu6-top-thermal { thermal-sensors = <&tsens0 13>; trips { @@ -7651,7 +7655,7 @@ }; }; - cpu7-top-thermal { + cpu7_top_thermal: cpu7-top-thermal { thermal-sensors = <&tsens1 1>; trips { @@ -7914,14 +7918,14 @@ thermal-sensors = <&tsens2 1>; - cooling-maps { + gpu0_cooling_maps: cooling-maps { map0 { trip = <&gpu0_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu0_trips: trips { gpu0_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; @@ -7947,14 +7951,14 @@ thermal-sensors = <&tsens2 2>; - cooling-maps { + gpu1_cooling_maps: cooling-maps { map0 { trip = <&gpu1_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu1_trips: trips { gpu1_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; @@ -7980,14 +7984,14 @@ thermal-sensors = <&tsens2 3>; - cooling-maps { + gpu2_cooling_maps: cooling-maps { map0 { trip = <&gpu2_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu2_trips: trips { gpu2_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; @@ -8013,14 +8017,14 @@ thermal-sensors = <&tsens2 4>; - cooling-maps { + gpu3_cooling_maps: cooling-maps { map0 { trip = <&gpu3_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu3_trips: trips { gpu3_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; @@ -8046,14 +8050,14 @@ thermal-sensors = <&tsens2 5>; - cooling-maps { + gpu4_cooling_maps: cooling-maps { map0 { trip = <&gpu4_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu4_trips: trips { gpu4_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; @@ -8079,14 +8083,14 @@ thermal-sensors = <&tsens2 6>; - cooling-maps { + gpu5_cooling_maps: cooling-maps { map0 { trip = <&gpu5_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu5_trips: trips { gpu5_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; @@ -8112,14 +8116,14 @@ thermal-sensors = <&tsens2 7>; - cooling-maps { + gpu6_cooling_maps: cooling-maps { map0 { trip = <&gpu6_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu6_trips: trips { gpu6_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; @@ -8145,14 +8149,14 @@ thermal-sensors = <&tsens2 8>; - cooling-maps { + gpu7_cooling_maps: cooling-maps { map0 { trip = <&gpu7_alert0>; cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; }; }; - trips { + gpu7_trips: trips { gpu7_alert0: trip-point0 { temperature = <95000>; hysteresis = <1000>; diff --git a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts index 3bbb53b7c71f3..9b8c453b6fe2b 100644 --- a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts @@ -960,9 +960,6 @@ }; &pcie0 { - wake-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>; - perst-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie0_default_state>; pinctrl-names = "default"; @@ -977,6 +974,9 @@ }; &pcieport0 { + wake-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>; + reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -1200,3 +1200,25 @@ status = "okay"; }; + +&usb_1 { + dr_mode = "peripheral"; + + status = "okay"; +}; + +&usb_1_hsphy { + vdd-supply = <&vreg_l2d_0p88>; + vdda12-supply = <&vreg_l3g_1p2>; + + phys = <&pmih0108_eusb2_repeater>; + + status = "okay"; +}; + +&usb_dp_qmpphy { + vdda-phy-supply = <&vreg_l3g_1p2>; + vdda-pll-supply = <&vreg_l2d_0p88>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts index 13c7b9664c89c..fc5d12bb41a53 100644 --- a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts @@ -1054,3 +1054,25 @@ status = "okay"; }; + +&usb_1 { + dr_mode = "peripheral"; + + status = "okay"; +}; + +&usb_1_hsphy { + vdd-supply = <&vreg_l2d_0p88>; + vdda12-supply = <&vreg_l3g_1p2>; + + phys = <&pmih0108_eusb2_repeater>; + + status = "okay"; +}; + +&usb_dp_qmpphy { + vdda-phy-supply = <&vreg_l3g_1p2>; + vdda-pll-supply = <&vreg_l2d_0p88>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index a82d9867c7cb6..4e03cd78c72e8 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -12,11 +12,13 @@ #include #include #include +#include #include #include #include #include #include +#include / { interrupt-parent = <&intc>; @@ -36,6 +38,7 @@ next-level-cache = <&l2_0>; power-domains = <&cpu_pd0>; power-domain-names = "psci"; + #cooling-cells = <2>; l2_0: l2-cache { compatible = "cache"; @@ -52,6 +55,7 @@ next-level-cache = <&l2_0>; power-domains = <&cpu_pd1>; power-domain-names = "psci"; + #cooling-cells = <2>; }; cpu2: cpu@200 { @@ -62,6 +66,7 @@ next-level-cache = <&l2_0>; power-domains = <&cpu_pd2>; power-domain-names = "psci"; + #cooling-cells = <2>; }; cpu3: cpu@300 { @@ -72,6 +77,7 @@ next-level-cache = <&l2_0>; power-domains = <&cpu_pd3>; power-domain-names = "psci"; + #cooling-cells = <2>; }; cpu4: cpu@400 { @@ -82,6 +88,7 @@ next-level-cache = <&l2_0>; power-domains = <&cpu_pd4>; power-domain-names = "psci"; + #cooling-cells = <2>; }; cpu5: cpu@500 { @@ -92,6 +99,7 @@ next-level-cache = <&l2_0>; power-domains = <&cpu_pd5>; power-domain-names = "psci"; + #cooling-cells = <2>; }; cpu6: cpu@10000 { @@ -102,6 +110,7 @@ next-level-cache = <&l2_1>; power-domains = <&cpu_pd6>; power-domain-names = "psci"; + #cooling-cells = <2>; l2_1: l2-cache { compatible = "cache"; @@ -118,6 +127,7 @@ next-level-cache = <&l2_1>; power-domains = <&cpu_pd7>; power-domain-names = "psci"; + #cooling-cells = <2>; }; cpu-map { @@ -2057,7 +2067,11 @@ "qcom,inline-crypto-engine"; reg = <0x0 0x01d88000 0x0 0x18000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc GCC_UFS_PHY_GDSC>; }; cryptobam: dma-controller@1dc4000 { @@ -2072,6 +2086,8 @@ <&apps_smmu 0x481 0>; qcom,ee = <0>; + qcom,num-ees = <4>; + num-channels = <20>; qcom,controlled-remotely; }; @@ -2581,6 +2597,163 @@ }; }; + usb_1_hsphy: phy@88e3000 { + compatible = "qcom,sm8750-m31-eusb2-phy"; + reg = <0x0 0x88e3000 0x0 0x29c>; + + clocks = <&tcsrcc TCSR_USB2_CLKREF_EN>; + clock-names = "ref"; + + resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>; + + #phy-cells = <0>; + + status = "disabled"; + }; + + usb_dp_qmpphy: phy@88e8000 { + compatible = "qcom,sm8750-qmp-usb3-dp-phy"; + reg = <0x0 0x088e8000 0x0 0x4000>; + + clocks = <&gcc GCC_USB3_PRIM_PHY_AUX_CLK>, + <&tcsrcc TCSR_USB3_CLKREF_EN>, + <&gcc GCC_USB3_PRIM_PHY_COM_AUX_CLK>, + <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>; + clock-names = "aux", + "ref", + "com_aux", + "usb3_pipe"; + + resets = <&gcc GCC_USB3_PHY_PRIM_BCR>, + <&gcc GCC_USB3_DP_PHY_PRIM_BCR>; + reset-names = "phy", + "common"; + + power-domains = <&gcc GCC_USB3_PHY_GDSC>; + + #clock-cells = <1>; + #phy-cells = <1>; + + orientation-switch; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb_dp_qmpphy_out: endpoint { + }; + }; + + port@1 { + reg = <1>; + + usb_dp_qmpphy_usb_ss_in: endpoint { + remote-endpoint = <&usb_1_dwc3_ss>; + }; + }; + + port@2 { + reg = <2>; + + usb_dp_qmpphy_dp_in: endpoint { + }; + }; + }; + }; + + usb_1: usb@a600000 { + compatible = "qcom,sm8750-dwc3", "qcom,snps-dwc3"; + reg = <0x0 0x0a600000 0x0 0xfc100>; + + clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>, + <&gcc GCC_USB30_PRIM_MASTER_CLK>, + <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>, + <&gcc GCC_USB30_PRIM_SLEEP_CLK>, + <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>; + clock-names = "cfg_noc", + "core", + "iface", + "sleep", + "mock_utmi"; + + assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>, + <&gcc GCC_USB30_PRIM_MASTER_CLK>; + assigned-clock-rates = <19200000>, + <200000000>; + + interrupts-extended = <&intc GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH>, + <&intc GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>, + <&intc GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>, + <&pdc 14 IRQ_TYPE_EDGE_BOTH>, + <&pdc 15 IRQ_TYPE_EDGE_BOTH>, + <&pdc 17 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "dwc_usb3", + "pwr_event", + "hs_phy_irq", + "dp_hs_phy_irq", + "dm_hs_phy_irq", + "ss_phy_irq"; + + power-domains = <&gcc GCC_USB30_PRIM_GDSC>; + required-opps = <&rpmhpd_opp_nom>; + + resets = <&gcc GCC_USB30_PRIM_BCR>; + + interconnects = <&aggre1_noc MASTER_USB3_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_USB3_0 QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "usb-ddr", "apps-usb"; + + iommus = <&apps_smmu 0x40 0x0>; + + phys = <&usb_1_hsphy>, + <&usb_dp_qmpphy QMP_USB43DP_USB3_PHY>; + phy-names = "usb2-phy", + "usb3-phy"; + + snps,hird-threshold = /bits/ 8 <0x0>; + snps,usb2-gadget-lpm-disable; + snps,dis_u2_susphy_quirk; + snps,dis_enblslpm_quirk; + snps,dis-u1-entry-quirk; + snps,dis-u2-entry-quirk; + snps,is-utmi-l1-suspend; + snps,usb3_lpm_capable; + snps,usb2-lpm-disable; + snps,has-lpm-erratum; + tx-fifo-resize; + + dma-coherent; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usb_1_dwc3_hs: endpoint { + }; + }; + + port@1 { + reg = <1>; + + usb_1_dwc3_ss: endpoint { + remote-endpoint = <&usb_dp_qmpphy_usb_ss_in>; + }; + }; + }; + }; + pdc: interrupt-controller@b220000 { compatible = "qcom,sm8750-pdc", "qcom,pdc"; reg = <0x0 0x0b220000 0x0 0x10000>, <0x0 0x164400f0 0x0 0x64>; @@ -3154,930 +3327,2807 @@ #reset-cells = <1>; }; - apps_smmu: iommu@15000000 { - compatible = "qcom,sm8750-smmu-500", "qcom,smmu-500", "arm,mmu-500"; - reg = <0x0 0x15000000 0x0 0x100000>; - - interrupts = , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - ; + stm@10002000 { + compatible = "arm,coresight-stm", "arm,primecell"; + reg = <0x0 0x10002000 0x0 0x1000>, + <0x0 0x37280000 0x0 0x180000>; + reg-names = "stm-base", + "stm-stimulus-base"; - #iommu-cells = <2>; - #global-interrupts = <1>; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - dma-coherent; + out-ports { + port { + stm_out: endpoint { + remote-endpoint = <&funnel_in0_in7>; + }; + }; + }; }; - intc: interrupt-controller@16000000 { - compatible = "arm,gic-v3"; - reg = <0x0 0x16000000 0x0 0x10000>, - <0x0 0x16080000 0x0 0x200000>; + tpda@10004000 { + compatible = "qcom,coresight-tpda", "arm,primecell"; + reg = <0x0 0x10004000 0x0 0x1000>; - interrupts = ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - #interrupt-cells = <3>; - interrupt-controller; + in-ports { + #address-cells = <1>; + #size-cells = <0>; - #redistributor-regions = <1>; - redistributor-stride = <0x0 0x40000>; + port@1 { + reg = <1>; - #address-cells = <2>; - #size-cells = <2>; - ranges; + tpda_qdss_in1: endpoint { + remote-endpoint = <&tpdm_spdm_out>; + }; + }; - gic_its: msi-controller@16040000 { - compatible = "arm,gic-v3-its"; - reg = <0x0 0x16040000 0x0 0x20000>; + }; - msi-controller; - #msi-cells = <1>; + out-ports { + port { + tpda_qdss_out: endpoint { + remote-endpoint = <&funnel_in0_in6>; + }; + }; }; }; - pcie0: pcie@1c00000 { - device_type = "pci"; - compatible = "qcom,pcie-sm8750", "qcom,pcie-sm8550"; - reg = <0x0 0x01c00000 0x0 0x3000>, - <0x0 0x40000000 0x0 0xf1d>, - <0x0 0x40000f20 0x0 0xa8>, - <0x0 0x40001000 0x0 0x1000>, - <0x0 0x40100000 0x0 0x100000>, - <0x0 0x01C03000 0x0 0x1000>; - reg-names = "parf", - "dbi", - "elbi", - "atu", - "config", - "mhi"; + tpdm@1000f000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x1000f000 0x0 0x1000>; - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, - <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x23d00000>, - <0x03000000 0x4 0x00000000 0x4 0x00000000 0x3 0x00000000>; - bus-range = <0x00 0xff>; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - dma-coherent; + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; - linux,pci-domain = <0>; + out-ports { + port { + tpdm_spdm_out: endpoint { + remote-endpoint = <&tpda_qdss_in1>; + }; + }; + }; + }; - msi-map = <0x0 &gic_its 0x1400 0x1>, - <0x100 &gic_its 0x1401 0x1>; - msi-map-mask = <0xff00>; + funnel@10041000 { + compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; + reg = <0x0 0x10041000 0x0 0x1000>; - num-lanes = <2>; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - interrupts = , - , - , - , - , - , - , - , - ; - interrupt-names = "msi0", - "msi1", - "msi2", - "msi3", - "msi4", - "msi5", - "msi6", - "msi7", - "global"; + in-ports { + #address-cells = <1>; + #size-cells = <0>; - #interrupt-cells = <1>; - interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc 0 0 0 150 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc 0 0 0 151 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc 0 0 0 152 IRQ_TYPE_LEVEL_HIGH>; + port@0 { + reg = <0>; - clocks = <&gcc GCC_PCIE_0_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>, - <&gcc GCC_DDRSS_PCIE_SF_QTB_CLK>, - <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>, - <&gcc GCC_CNOC_PCIE_SF_AXI_CLK>; - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a", - "ddrss_sf_tbu", - "noc_aggr", - "cnoc_sf_axi"; + funnel_in0_in0: endpoint { + remote-endpoint = + <&tn_ag_out>; + }; + }; - interconnects = <&pcie_noc MASTER_PCIE_0 QCOM_ICC_TAG_ALWAYS - &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, - <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS - &cnoc_main SLAVE_PCIE_0 QCOM_ICC_TAG_ALWAYS>; - interconnect-names = "pcie-mem", - "cpu-pcie"; + port@6 { + reg = <6>; - iommu-map = <0x0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + funnel_in0_in6: endpoint { + remote-endpoint = <&tpda_qdss_out>; + }; + }; - resets = <&gcc GCC_PCIE_0_BCR>; - reset-names = "pci"; + port@7 { + reg = <7>; - power-domains = <&gcc GCC_PCIE_0_GDSC>; + funnel_in0_in7: endpoint { + remote-endpoint = <&stm_out>; + }; + }; + }; - operating-points-v2 = <&pcie0_opp_table>; + out-ports { + port { + funnel_in0_out: endpoint { + remote-endpoint = <&funnel_aoss_in7>; + }; + }; + }; + }; - status = "disabled"; + tpdm@10800000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10800000 0x0 0x1000>; - pcie0_opp_table: opp-table { - compatible = "operating-points-v2"; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - /* GEN 1 x1 */ - opp-2500000 { - opp-hz = /bits/ 64 <2500000>; - required-opps = <&rpmhpd_opp_low_svs>; - opp-peak-kBps = <250000 1>; + qcom,dsb-element-bits = <32>; + qcom,dsb-msrs-num = <32>; + + out-ports { + port { + tpdm_modem0_out: endpoint { + remote-endpoint = <&tpda_modem_in0>; + }; }; + }; + }; - /* GEN 1 x2 and GEN 2 x1 */ - opp-5000000 { - opp-hz = /bits/ 64 <5000000>; - required-opps = <&rpmhpd_opp_low_svs>; - opp-peak-kBps = <500000 1>; + tpda@10803000 { + compatible = "qcom,coresight-tpda", "arm,primecell"; + reg = <0x0 0x10803000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + tpda_modem_in0: endpoint { + remote-endpoint = <&tpdm_modem0_out>; + }; }; - /* GEN 2 x2 */ - opp-10000000 { - opp-hz = /bits/ 64 <10000000>; - required-opps = <&rpmhpd_opp_low_svs>; - opp-peak-kBps = <1000000 1>; + port@1 { + reg = <1>; + + tpda_modem_in1: endpoint { + remote-endpoint = <&tpdm_modem1_out>; + }; }; + }; - /* GEN 3 x1 */ - opp-8000000 { - opp-hz = /bits/ 64 <8000000>; - required-opps = <&rpmhpd_opp_nom>; - opp-peak-kBps = <984500 1>; + out-ports { + port { + tpda_modem_out: endpoint { + remote-endpoint = <&funnel_modem_dl_in0>; + }; }; + }; + }; - /* GEN 3 x2 */ - opp-16000000 { - opp-hz = /bits/ 64 <16000000>; - required-opps = <&rpmhpd_opp_nom>; - opp-peak-kBps = <1969000 1>; + funnel@10804000 { + compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; + reg = <0x0 0x10804000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + port { + funnel_modem_dl_in0: endpoint { + remote-endpoint = <&tpda_modem_out>; + }; }; + }; + out-ports { + port { + funnel_modem_dl_out: endpoint { + remote-endpoint = <&tn_ag_in13>; + }; + }; }; + }; - pcieport0: pcie@0 { - device_type = "pci"; - reg = <0x0 0x0 0x0 0x0 0x0>; - bus-range = <0x01 0xff>; + cti@1080b000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x1080b000 0x0 0x1000>; - #address-cells = <3>; - #size-cells = <2>; - ranges; - phys = <&pcie0_phy>; - }; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; }; - pcie0_phy: phy@1c06000 { - compatible = "qcom,sm8750-qmp-gen3x2-pcie-phy"; - reg = <0 0x01c06000 0 0x2000>; + tpdm@1082c000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x1082c000 0x0 0x1000>; - clocks = <&gcc GCC_PCIE_0_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&tcsrcc TCSR_PCIE_0_CLKREF_EN>, - <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>, - <&gcc GCC_PCIE_0_PIPE_CLK>; - clock-names = "aux", - "cfg_ahb", - "ref", - "rchng", - "pipe"; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>; - assigned-clock-rates = <100000000>; + qcom,dsb-msrs-num = <32>; - resets = <&gcc GCC_PCIE_0_PHY_BCR>; - reset-names = "phy"; + out-ports { + port { + tpdm_gcc_out: endpoint { + remote-endpoint = <&tn_ag_in17>; + }; + }; + }; + }; - power-domains = <&gcc GCC_PCIE_0_PHY_GDSC>; + tpdm@10841000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10841000 0x0 0x1000>; - #clock-cells = <0>; - clock-output-names = "pcie0_pipe_clk"; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - #phy-cells = <0>; + qcom,cmb-msrs-num = <32>; - status = "disabled"; + out-ports { + port { + tpdm_prng_out: endpoint { + remote-endpoint = <&tn_ag_in18>; + }; + }; + }; }; - ufs_mem_phy: phy@1d80000 { - compatible = "qcom,sm8750-qmp-ufs-phy"; - reg = <0x0 0x01d80000 0x0 0x2000>; + tpdm@1084e000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x1084e000 0x0 0x1000>; - clocks = <&rpmhcc RPMH_CXO_CLK>, - <&gcc GCC_UFS_PHY_PHY_AUX_CLK>, - <&tcsrcc TCSR_UFS_CLKREF_EN>; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - clock-names = "ref", - "ref_aux", - "qref"; + qcom,cmb-element-bits = <32>; + qcom,cmb-msrs-num = <32>; - resets = <&ufs_mem_hc 0>; - reset-names = "ufsphy"; + out-ports { + port { + tpdm_mm_bcv_out: endpoint { + remote-endpoint = <&tpda_mm_in0>; + }; + }; + }; + }; - power-domains = <&gcc GCC_UFS_MEM_PHY_GDSC>; + tpdm@1084f000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x1084f000 0x0 0x1000>; - #clock-cells = <1>; - #phy-cells = <0>; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - status = "disabled"; - }; + qcom,cmb-element-bits = <32>; + qcom,cmb-msrs-num = <32>; - ufs_mem_hc: ufs@1d84000 { - compatible = "qcom,sm8750-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; - reg = <0x0 0x01d84000 0x0 0x3000>; + out-ports { + port { + tpdm_mm_lmh_out: endpoint { + remote-endpoint = <&tpda_mm_in1>; + }; + }; + }; + }; - interrupts = ; + tpdm@10850000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10850000 0x0 0x1000>; - clocks = <&gcc GCC_UFS_PHY_AXI_CLK>, - <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, - <&gcc GCC_UFS_PHY_AHB_CLK>, - <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>, - <&rpmhcc RPMH_LN_BB_CLK3>, - <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>, - <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>, - <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>; - clock-names = "core_clk", - "bus_aggr_clk", - "iface_clk", - "core_clk_unipro", - "ref_clk", - "tx_lane0_sync_clk", - "rx_lane0_sync_clk", - "rx_lane1_sync_clk"; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - operating-points-v2 = <&ufs_opp_table>; + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; - resets = <&gcc GCC_UFS_PHY_BCR>; - reset-names = "rst"; + out-ports { + port { + tpdm_mm_dpm_out: endpoint { + remote-endpoint = <&tpda_mm_in2>; + }; + }; + }; + }; - interconnects = <&aggre1_noc MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS - &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, - <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY - &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; - interconnect-names = "ufs-ddr", - "cpu-ufs"; + tpda@10851000 { + compatible = "qcom,coresight-tpda", "arm,primecell"; + reg = <0x0 0x10851000 0x0 0x1000>; - power-domains = <&gcc GCC_UFS_PHY_GDSC>; - required-opps = <&rpmhpd_opp_nom>; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - iommus = <&apps_smmu 0x60 0>; - dma-coherent; + in-ports { + #address-cells = <1>; + #size-cells = <0>; - lanes-per-direction = <2>; + port@0 { + reg = <0>; - phys = <&ufs_mem_phy>; - phy-names = "ufsphy"; + tpda_mm_in0: endpoint { + remote-endpoint = <&tpdm_mm_bcv_out>; + }; + }; - #reset-cells = <1>; + port@1 { + reg = <1>; - status = "disabled"; + tpda_mm_in1: endpoint { + remote-endpoint = <&tpdm_mm_lmh_out>; + }; + }; - ufs_opp_table: opp-table { - compatible = "operating-points-v2"; + port@2 { + reg = <2>; - opp-100000000 { - opp-hz = /bits/ 64 <100000000>, - /bits/ 64 <0>, - /bits/ 64 <0>, - /bits/ 64 <100000000>, - /bits/ 64 <0>, - /bits/ 64 <0>, - /bits/ 64 <0>, - /bits/ 64 <0>; - required-opps = <&rpmhpd_opp_low_svs>; + tpda_mm_in2: endpoint { + remote-endpoint = <&tpdm_mm_dpm_out>; + }; }; + }; - opp-403000000 { - opp-hz = /bits/ 64 <403000000>, - /bits/ 64 <0>, - /bits/ 64 <0>, - /bits/ 64 <403000000>, - /bits/ 64 <0>, - /bits/ 64 <0>, - /bits/ 64 <0>, - /bits/ 64 <0>; - required-opps = <&rpmhpd_opp_nom>; + out-ports { + port { + tpda_mm_out: endpoint { + remote-endpoint = <&tn_ag_in4>; + }; }; }; }; - apps_rsc: rsc@16500000 { - compatible = "qcom,rpmh-rsc"; - reg = <0x0 0x16500000 0x0 0x10000>, - <0x0 0x16510000 0x0 0x10000>, - <0x0 0x16520000 0x0 0x10000>; - reg-names = "drv-0", - "drv-1", - "drv-2"; - - interrupts = , - , - ; - qcom,tcs-offset = <0xd00>; - qcom,drv-id = <2>; - qcom,tcs-config = , , - , ; + tpdm@10980000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10980000 0x0 0x1000>; - label = "apps_rsc"; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - power-domains = <&system_pd>; + qcom,dsb-element-bits = <32>; + qcom,dsb-msrs-num = <32>; - apps_bcm_voter: bcm-voter { - compatible = "qcom,bcm-voter"; + out-ports { + port { + tpdm_turing_out: endpoint { + remote-endpoint = <&tpda_turing_in0>; + }; + }; }; + }; - rpmhcc: clock-controller { - compatible = "qcom,sm8750-rpmh-clk"; - - clocks = <&xo_board>; - clock-names = "xo"; + tpda@10986000 { + compatible = "qcom,coresight-tpda", "arm,primecell"; + reg = <0x0 0x10986000 0x0 0x1000>; - #clock-cells = <1>; - }; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - rpmhpd: power-controller { - compatible = "qcom,sm8750-rpmhpd"; + in-ports { + #address-cells = <1>; + #size-cells = <0>; - operating-points-v2 = <&rpmhpd_opp_table>; + port@0 { + reg = <0>; - #power-domain-cells = <1>; + tpda_turing_in0: endpoint { + remote-endpoint = <&tpdm_turing_out>; + }; + }; - rpmhpd_opp_table: opp-table { - compatible = "operating-points-v2"; + port@1 { + reg = <1>; - rpmhpd_opp_ret: opp-16 { - opp-level = ; + tpda_turing_in1: endpoint { + remote-endpoint = <&tpdm_turing_llm_out>; }; + }; - rpmhpd_opp_min_svs: opp-48 { - opp-level = ; - }; + port@2 { + reg = <2>; - rpmhpd_opp_low_svs_d3: opp-50 { - opp-level = ; + tpda_turing_in2: endpoint { + remote-endpoint = <&tpdm_turing_llm2_out>; }; + }; + }; - rpmhpd_opp_low_svs_d2: opp-52 { - opp-level = ; + out-ports { + port { + tpda_turing_out: endpoint { + remote-endpoint = <&funnel_turing_in0>; }; + }; + }; + }; - rpmhpd_opp_low_svs_d1: opp-56 { - opp-level = ; - }; + funnel@10987000 { + compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; + reg = <0x0 0x10987000 0x0 0x1000>; - rpmhpd_opp_low_svs_d0: opp-60 { - opp-level = ; - }; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - rpmhpd_opp_low_svs: opp-64 { - opp-level = ; + in-ports { + port { + funnel_turing_in0: endpoint { + remote-endpoint = <&tpda_turing_out>; }; + }; + }; - rpmhpd_opp_low_svs_l1: opp-80 { - opp-level = ; + out-ports { + port { + funnel_turing_out: endpoint { + remote-endpoint = <&tn_ag_in16>; }; + }; + }; + }; - rpmhpd_opp_svs: opp-128 { - opp-level = ; - }; + cti@1098b000 { + compatible = "arm,coresight-cti", "arm,primecell"; + reg = <0x0 0x1098b000 0x0 0x1000>; - rpmhpd_opp_svs_l0: opp-144 { - opp-level = ; - }; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; - rpmhpd_opp_svs_l1: opp-192 { - opp-level = ; - }; + tpdm@109a3000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109a3000 0x0 0x1000>; - rpmhpd_opp_svs_l2: opp-224 { - opp-level = ; - }; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - rpmhpd_opp_nom: opp-256 { - opp-level = ; - }; + qcom,cmb-msrs-num = <32>; + qcom,dsb-msrs-num = <32>; - rpmhpd_opp_nom_l1: opp-320 { - opp-level = ; + out-ports { + port { + tpdm_pmu_out: endpoint { + remote-endpoint = <&tn_ag_in29>; }; + }; + }; + }; - rpmhpd_opp_nom_l2: opp-336 { - opp-level = ; - }; + tpdm@109a4000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109a4000 0x0 0x1000>; - rpmhpd_opp_turbo: opp-384 { - opp-level = ; - }; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - rpmhpd_opp_turbo_l1: opp-416 { - opp-level = ; - }; + qcom,cmb-msrs-num = <32>; - rpmhpd_opp_turbo_l2: opp-432 { - opp-level = ; + out-ports { + port { + tpdm_ipcc_cmb_out: endpoint { + remote-endpoint = <&tn_ag_in28>; }; + }; + }; + }; - rpmhpd_opp_turbo_l3: opp-448 { - opp-level = ; - }; + tpdm@109a5000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109a5000 0x0 0x1000>; - rpmhpd_opp_turbo_l4: opp-452 { - opp-level = ; - }; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - rpmhpd_opp_super_turbo_no_cpr: opp-480 { - opp-level = - ; + qcom,dsb-msrs-num = <32>; + + out-ports { + port { + tpdm_dl_mm_out: endpoint { + remote-endpoint = <&tn_ag_in25>; }; }; }; }; - timer@16800000 { - compatible = "arm,armv7-timer-mem"; - reg = <0x0 0x16800000 0x0 0x1000>; - - #address-cells = <2>; - #size-cells = <1>; - ranges = <0 0 0 0 0x20000000>; + tpdm@109a6000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109a6000 0x0 0x1000>; - frame@16801000 { - reg = <0x0 0x16801000 0x1000>, - <0x0 0x16802000 0x1000>; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - interrupts = , - ; + qcom,dsb-msrs-num = <32>; - frame-number = <0>; + out-ports { + port { + tpdm_north_dsb_out: endpoint { + remote-endpoint = <&tn_ag_in26>; + }; + }; }; + }; - frame@16803000 { - reg = <0x0 0x16803000 0x1000>; + tpdm@109a7000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109a7000 0x0 0x1000>; - interrupts = ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - frame-number = <1>; + qcom,dsb-msrs-num = <32>; - status = "disabled"; + out-ports { + port { + tpdm_south_dsb_out: endpoint { + remote-endpoint = <&tn_ag_in27>; + }; + }; }; + }; - frame@16805000 { - reg = <0x0 0x16805000 0x1000>; + tpdm@109a8000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109a8000 0x0 0x1000>; - interrupts = ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - frame-number = <2>; + qcom,cmb-msrs-num = <32>; - status = "disabled"; + out-ports { + port { + tpdm_rdpm_cmb0_out: endpoint { + remote-endpoint = <&tn_ag_in30>; + }; + }; }; + }; - frame@16807000 { - reg = <0x0 0x16807000 0x1000>; + tpdm@109a9000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109a9000 0x0 0x1000>; - interrupts = ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - frame-number = <3>; + qcom,cmb-msrs-num = <32>; - status = "disabled"; + out-ports { + port { + tpdm_rdpm_cmb1_out: endpoint { + remote-endpoint = <&tn_ag_in31>; + }; + }; }; + }; - frame@16809000 { - reg = <0x0 0x16809000 0x1000>; + tpdm@109aa000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109aa000 0x0 0x1000>; - interrupts = ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - frame-number = <4>; + qcom,cmb-msrs-num = <32>; - status = "disabled"; + out-ports { + port { + tpdm_rdpm_cmb2_out: endpoint { + remote-endpoint = <&tn_ag_in32>; + }; + }; }; + }; - frame@1680b000 { - reg = <0x0 0x1680b000 0x1000>; + tn@109ab000 { + compatible = "qcom,coresight-tnoc", "arm,primecell"; + reg = <0x0 0x109ab000 0x0 0x4200>; - interrupts = ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - frame-number = <5>; + in-ports { + #address-cells = <1>; + #size-cells = <0>; - status = "disabled"; - }; + port@4 { + reg = <4>; - frame@1680d000 { - reg = <0x0 0x1680d000 0x1000>; + tn_ag_in4: endpoint { + remote-endpoint = <&tpda_mm_out>; + }; + }; - interrupts = ; + port@d { + reg = <0xd>; - frame-number = <6>; + tn_ag_in13: endpoint { + remote-endpoint = <&funnel_modem_dl_out>; + }; + }; - status = "disabled"; - }; - }; + port@10 { + reg = <0x10>; - /* cluster0 */ - pmu@240b3400 { - compatible = "qcom,sm8750-cpu-bwmon", "qcom,sdm845-bwmon"; - reg = <0x0 0x240b3400 0x0 0x600>; + tn_ag_in16: endpoint { + remote-endpoint = <&funnel_turing_out>; + }; + }; - interrupts = ; + port@11 { + reg = <0x11>; - interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY - &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>; + tn_ag_in17: endpoint { + remote-endpoint = <&tpdm_gcc_out>; + }; + }; - operating-points-v2 = <&cpu_bwmon_opp_table>; + port@12 { + reg = <0x12>; - nonposted-mmio; + tn_ag_in18: endpoint { + remote-endpoint = <&tpdm_prng_out>; + }; + }; - cpu_bwmon_opp_table: opp-table { - compatible = "operating-points-v2"; + port@13 { + reg = <0x13>; - opp-0 { - opp-peak-kBps = <800000>; + tn_ag_in19: endpoint { + remote-endpoint = <&tpdm_qm_out>; + }; }; - opp-1 { - opp-peak-kBps = <2188000>; - }; + port@19 { + reg = <0x19>; - opp-2 { - opp-peak-kBps = <5414400>; + tn_ag_in25: endpoint { + remote-endpoint = <&tpdm_dl_mm_out>; + }; }; - opp-3 { - opp-peak-kBps = <6220800>; + port@1a { + reg = <0x1a>; + + tn_ag_in26: endpoint { + remote-endpoint = <&tpdm_north_dsb_out>; + }; }; - opp-4 { - opp-peak-kBps = <6835200>; + port@1b { + reg = <0x1b>; + + tn_ag_in27: endpoint { + remote-endpoint = <&tpdm_south_dsb_out>; + }; }; - opp-5 { - opp-peak-kBps = <8371200>; + port@1c { + reg = <0x1c>; + + tn_ag_in28: endpoint { + remote-endpoint = <&tpdm_ipcc_cmb_out>; + }; }; - opp-6 { - opp-peak-kBps = <10944000>; + port@1d { + reg = <0x1d>; + + tn_ag_in29: endpoint { + remote-endpoint = <&tpdm_pmu_out>; + }; }; - opp-7 { - opp-peak-kBps = <12748800>; + port@1e { + reg = <0x1e>; + + tn_ag_in30: endpoint { + remote-endpoint = <&tpdm_rdpm_cmb0_out>; + }; }; - opp-8 { - opp-peak-kBps = <14745600>; + port@1f { + reg = <0x1f>; + + tn_ag_in31: endpoint { + remote-endpoint = <&tpdm_rdpm_cmb1_out>; + }; }; - opp-9 { - opp-peak-kBps = <16896000>; + port@20 { + reg = <0x20>; + + tn_ag_in32: endpoint { + remote-endpoint = <&tpdm_rdpm_cmb2_out>; + }; }; + }; - opp-10 { - opp-peak-kBps = <19046400>; + out-ports { + port { + tn_ag_out: endpoint { + remote-endpoint = <&funnel_in0_in0>; + }; }; }; }; - /* cluster1 */ - pmu@240b7400 { - compatible = "qcom,sm8750-cpu-bwmon", "qcom,sdm845-bwmon"; - reg = <0x0 0x240b7400 0x0 0x600>; + tpdm@109d0000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x109d0000 0x0 0x1000>; - interrupts = ; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY - &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>; + qcom,dsb-msrs-num = <32>; - operating-points-v2 = <&cpu_bwmon_opp_table>; + out-ports { + port { + tpdm_qm_out: endpoint { + remote-endpoint = <&tn_ag_in19>; + }; + }; + }; }; - gem_noc: interconnect@24100000 { - compatible = "qcom,sm8750-gem-noc"; - reg = <0x0 0x24100000 0x0 0x14b080>; - qcom,bcm-voters = <&apps_bcm_voter>; - #interconnect-cells = <2>; - }; + funnel@10b04000 { + compatible = "arm,coresight-dynamic-funnel", "arm,primecell"; + reg = <0x0 0x10b04000 0x0 0x1000>; - system-cache-controller@24800000 { - compatible = "qcom,sm8750-llcc"; - reg = <0x0 0x24800000 0x0 0x200000>, - <0x0 0x25800000 0x0 0x200000>, - <0x0 0x24c00000 0x0 0x200000>, - <0x0 0x25c00000 0x0 0x200000>, - <0x0 0x26800000 0x0 0x200000>, - <0x0 0x26c00000 0x0 0x200000>; - reg-names = "llcc0_base", - "llcc1_base", - "llcc2_base", - "llcc3_base", - "llcc_broadcast_base", - "llcc_broadcast_and_base"; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - interrupts = ; - }; + in-ports { + #address-cells = <1>; + #size-cells = <0>; - nsp_noc: interconnect@320c0000 { - compatible = "qcom,sm8750-nsp-noc"; - reg = <0x0 0x320c0000 0x0 0x13080>; - qcom,bcm-voters = <&apps_bcm_voter>; - #interconnect-cells = <2>; - }; + port@6 { + reg = <6>; - remoteproc_cdsp: remoteproc@32300000 { - compatible = "qcom,sm8750-cdsp-pas", "qcom,sm8650-cdsp-pas"; - reg = <0x0 0x32300000 0x0 0x10000>; + funnel_aoss_in6: endpoint { + remote-endpoint = <&tpda_aoss_out>; + }; + }; - interrupts-extended = <&intc GIC_SPI 578 IRQ_TYPE_EDGE_RISING>, - <&smp2p_cdsp_in 0 IRQ_TYPE_EDGE_RISING>, - <&smp2p_cdsp_in 1 IRQ_TYPE_EDGE_RISING>, - <&smp2p_cdsp_in 2 IRQ_TYPE_EDGE_RISING>, - <&smp2p_cdsp_in 3 IRQ_TYPE_EDGE_RISING>, - <&smp2p_cdsp_in 7 IRQ_TYPE_EDGE_RISING>; - interrupt-names = "wdog", - "fatal", - "ready", - "handover", - "stop-ack", - "shutdown-ack"; + port@7 { + reg = <7>; - clocks = <&rpmhcc RPMH_CXO_CLK>; - clock-names = "xo"; + funnel_aoss_in7: endpoint { + remote-endpoint = <&funnel_in0_out>; + }; + }; - interconnects = <&nsp_noc MASTER_CDSP_PROC QCOM_ICC_TAG_ALWAYS - &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + }; - power-domains = <&rpmhpd RPMHPD_CX>, - <&rpmhpd RPMHPD_MXC>, - <&rpmhpd RPMHPD_NSP>; - power-domain-names = "cx", - "mxc", - "nsp"; + out-ports { + port { + funnel_aoss_out: endpoint { + remote-endpoint = <&tmc_etf_in>; + }; + }; + }; + }; - memory-region = <&cdsp_mem>, <&q6_cdsp_dtb_mem>, <&global_sync_mem>; - qcom,qmp = <&aoss_qmp>; - qcom,smem-states = <&smp2p_cdsp_out 0>; - qcom,smem-state-names = "stop"; + tmc@10b05000 { + compatible = "arm,coresight-tmc", "arm,primecell"; + reg = <0x0 0x10b05000 0x0 0x1000>; - status = "disabled"; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; - glink-edge { - interrupts-extended = <&ipcc IPCC_CLIENT_CDSP - IPCC_MPROC_SIGNAL_GLINK_QMP - IRQ_TYPE_EDGE_RISING>; - mboxes = <&ipcc IPCC_CLIENT_CDSP - IPCC_MPROC_SIGNAL_GLINK_QMP>; - qcom,remote-pid = <5>; - label = "cdsp"; + in-ports { + port { + tmc_etf_in: endpoint { + remote-endpoint = <&funnel_aoss_out>; + }; + }; + }; + }; - fastrpc { - compatible = "qcom,fastrpc"; - qcom,glink-channels = "fastrpcglink-apps-dsp"; - label = "cdsp"; - qcom,non-secure-domain; - #address-cells = <1>; - #size-cells = <0>; + tpda@10b08000 { + compatible = "qcom,coresight-tpda", "arm,primecell"; + reg = <0x0 0x10b08000 0x0 0x1000>; - compute-cb@1 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <1>; - iommus = <&apps_smmu 0x19c1 0x0>, - <&apps_smmu 0x0c21 0x0>, - <&apps_smmu 0x0c01 0x40>; - dma-coherent; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + in-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + tpda_aoss_in0: endpoint { + remote-endpoint = <&tpdm_swao_prio0_out>; }; + }; - compute-cb@2 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <2>; - iommus = <&apps_smmu 0x1962 0x0>, - <&apps_smmu 0x0c02 0x20>, - <&apps_smmu 0x0c42 0x0>, - <&apps_smmu 0x19c2 0x0>; - dma-coherent; + port@1 { + reg = <1>; + + tpda_aoss_in1: endpoint { + remote-endpoint = <&tpdm_swao_prio1_out>; }; + }; - compute-cb@3 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <3>; - iommus = <&apps_smmu 0x1963 0x0>, - <&apps_smmu 0x0c23 0x0>, - <&apps_smmu 0x0c03 0x40>, - <&apps_smmu 0x19c3 0x0>; - dma-coherent; + port@2 { + reg = <2>; + + tpda_aoss_in2: endpoint { + remote-endpoint = <&tpdm_swao_prio2_out>; }; + }; - compute-cb@4 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <4>; - iommus = <&apps_smmu 0x1964 0x0>, - <&apps_smmu 0x0c24 0x0>, - <&apps_smmu 0x0c04 0x40>, - <&apps_smmu 0x19c4 0x0>; - dma-coherent; + port@3 { + reg = <3>; + + tpda_aoss_in3: endpoint { + remote-endpoint = <&tpdm_swao_prio3_out>; }; + }; - compute-cb@5 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <5>; - iommus = <&apps_smmu 0x1965 0x0>, - <&apps_smmu 0x0c25 0x0>, - <&apps_smmu 0x0c05 0x40>, - <&apps_smmu 0x19c5 0x0>; - dma-coherent; + port@4 { + reg = <4>; + + tpda_aoss_in4: endpoint { + remote-endpoint =<&tpdm_swao_out>; }; + }; + }; - compute-cb@6 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <6>; - iommus = <&apps_smmu 0x1966 0x0>, - <&apps_smmu 0x0c06 0x20>, - <&apps_smmu 0x0c46 0x0>, - <&apps_smmu 0x19c6 0x0>; - dma-coherent; + out-ports { + port { + tpda_aoss_out: endpoint { + remote-endpoint = <&funnel_aoss_in6>; }; + }; + }; + }; - compute-cb@7 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <7>; - iommus = <&apps_smmu 0x1967 0x0>, - <&apps_smmu 0x0c27 0x0>, - <&apps_smmu 0x0c07 0x40>, - <&apps_smmu 0x19c7 0x0>; - dma-coherent; + tpdm@10b09000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10b09000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + tpdm_swao_prio0_out: endpoint { + remote-endpoint = <&tpda_aoss_in0>; }; + }; + }; + }; - compute-cb@8 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <8>; - iommus = <&apps_smmu 0x1968 0x0>, - <&apps_smmu 0x0c08 0x20>, - <&apps_smmu 0x0c48 0x0>, - <&apps_smmu 0x19c8 0x0>; - dma-coherent; + tpdm@10b0a000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10b0a000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + tpdm_swao_prio1_out: endpoint { + remote-endpoint = <&tpda_aoss_in1>; }; + }; + }; + }; - /* note: secure cb9 in downstream */ + tpdm@10b0b000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10b0b000 0x0 0x1000>; - compute-cb@12 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <12>; - iommus = <&apps_smmu 0x196c 0x0>, - <&apps_smmu 0x0c2c 0x20>, - <&apps_smmu 0x0c0c 0x40>, - <&apps_smmu 0x19cc 0x0>; - dma-coherent; + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + tpdm_swao_prio2_out: endpoint { + remote-endpoint = <&tpda_aoss_in2>; }; + }; + }; + }; - compute-cb@13 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <13>; - iommus = <&apps_smmu 0x196d 0x0>, - <&apps_smmu 0x0c0d 0x20>, - <&apps_smmu 0x0c2e 0x0>, - <&apps_smmu 0x0c4d 0x0>, - <&apps_smmu 0x19cd 0x0>; - dma-coherent; + tpdm@10b0c000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10b0c000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,cmb-element-bits = <64>; + qcom,cmb-msrs-num = <32>; + + out-ports { + port { + tpdm_swao_prio3_out: endpoint { + remote-endpoint = <&tpda_aoss_in3>; }; + }; + }; + }; - compute-cb@14 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <14>; - iommus = <&apps_smmu 0x196e 0x0>, - <&apps_smmu 0x0c0e 0x20>, - <&apps_smmu 0x19ce 0x0>; - dma-coherent; + tpdm@10b0d000 { + compatible = "qcom,coresight-tpdm", "arm,primecell"; + reg = <0x0 0x10b0d000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + qcom,dsb-element-bits = <32>; + qcom,dsb-msrs-num = <32>; + + out-ports { + port { + tpdm_swao_out: endpoint { + remote-endpoint = <&tpda_aoss_in4>; }; }; }; }; - }; - timer { - compatible = "arm,armv8-timer"; + apps_smmu: iommu@15000000 { + compatible = "qcom,sm8750-smmu-500", "qcom,smmu-500", "arm,mmu-500"; + reg = <0x0 0x15000000 0x0 0x100000>; - interrupts = , - , - , - ; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + + #iommu-cells = <2>; + #global-interrupts = <1>; + + dma-coherent; + }; + + intc: interrupt-controller@16000000 { + compatible = "arm,gic-v3"; + reg = <0x0 0x16000000 0x0 0x10000>, + <0x0 0x16080000 0x0 0x200000>; + + interrupts = ; + + #interrupt-cells = <3>; + interrupt-controller; + + #redistributor-regions = <1>; + redistributor-stride = <0x0 0x40000>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + gic_its: msi-controller@16040000 { + compatible = "arm,gic-v3-its"; + reg = <0x0 0x16040000 0x0 0x20000>; + + msi-controller; + #msi-cells = <1>; + }; + }; + + pcie0: pcie@1c00000 { + device_type = "pci"; + compatible = "qcom,pcie-sm8750", "qcom,pcie-sm8550"; + reg = <0x0 0x01c00000 0x0 0x3000>, + <0x0 0x40000000 0x0 0xf1d>, + <0x0 0x40000f20 0x0 0xa8>, + <0x0 0x40001000 0x0 0x1000>, + <0x0 0x40100000 0x0 0x100000>, + <0x0 0x01C03000 0x0 0x1000>; + reg-names = "parf", + "dbi", + "elbi", + "atu", + "config", + "mhi"; + + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, + <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x23d00000>, + <0x03000000 0x4 0x00000000 0x4 0x00000000 0x3 0x00000000>; + bus-range = <0x00 0xff>; + + dma-coherent; + + linux,pci-domain = <0>; + + msi-map = <0x0 &gic_its 0x1400 0x1>, + <0x100 &gic_its 0x1401 0x1>; + msi-map-mask = <0xff00>; + + num-lanes = <2>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc 0 0 0 150 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc 0 0 0 151 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc 0 0 0 152 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_0_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>, + <&gcc GCC_DDRSS_PCIE_SF_QTB_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>, + <&gcc GCC_CNOC_PCIE_SF_AXI_CLK>; + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a", + "ddrss_sf_tbu", + "noc_aggr", + "cnoc_sf_axi"; + + interconnects = <&pcie_noc MASTER_PCIE_0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &cnoc_main SLAVE_PCIE_0 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "pcie-mem", + "cpu-pcie"; + + iommu-map = <0x0 &apps_smmu 0x1400 0x1>, + <0x100 &apps_smmu 0x1401 0x1>; + + resets = <&gcc GCC_PCIE_0_BCR>; + reset-names = "pci"; + + power-domains = <&gcc GCC_PCIE_0_GDSC>; + + operating-points-v2 = <&pcie0_opp_table>; + + status = "disabled"; + + pcie0_opp_table: opp-table { + compatible = "operating-points-v2"; + + /* GEN 1 x1 */ + opp-2500000 { + opp-hz = /bits/ 64 <2500000>; + required-opps = <&rpmhpd_opp_low_svs>; + opp-peak-kBps = <250000 1>; + }; + + /* GEN 1 x2 and GEN 2 x1 */ + opp-5000000 { + opp-hz = /bits/ 64 <5000000>; + required-opps = <&rpmhpd_opp_low_svs>; + opp-peak-kBps = <500000 1>; + }; + + /* GEN 2 x2 */ + opp-10000000 { + opp-hz = /bits/ 64 <10000000>; + required-opps = <&rpmhpd_opp_low_svs>; + opp-peak-kBps = <1000000 1>; + }; + + /* GEN 3 x1 */ + opp-8000000 { + opp-hz = /bits/ 64 <8000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <984500 1>; + }; + + /* GEN 3 x2 */ + opp-16000000 { + opp-hz = /bits/ 64 <16000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <1969000 1>; + }; + + }; + + pcieport0: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + phys = <&pcie0_phy>; + }; + }; + + pcie0_phy: phy@1c06000 { + compatible = "qcom,sm8750-qmp-gen3x2-pcie-phy"; + reg = <0 0x01c06000 0 0x2000>; + + clocks = <&gcc GCC_PCIE_0_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&tcsrcc TCSR_PCIE_0_CLKREF_EN>, + <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_0_PIPE_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe"; + + assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + resets = <&gcc GCC_PCIE_0_PHY_BCR>; + reset-names = "phy"; + + power-domains = <&gcc GCC_PCIE_0_PHY_GDSC>; + + #clock-cells = <0>; + clock-output-names = "pcie0_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + + ufs_mem_phy: phy@1d80000 { + compatible = "qcom,sm8750-qmp-ufs-phy"; + reg = <0x0 0x01d80000 0x0 0x2000>; + + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&gcc GCC_UFS_PHY_PHY_AUX_CLK>, + <&tcsrcc TCSR_UFS_CLKREF_EN>; + + clock-names = "ref", + "ref_aux", + "qref"; + + resets = <&ufs_mem_hc 0>; + reset-names = "ufsphy"; + + power-domains = <&gcc GCC_UFS_MEM_PHY_GDSC>; + + #clock-cells = <1>; + #phy-cells = <0>; + + status = "disabled"; + }; + + ufs_mem_hc: ufs@1d84000 { + compatible = "qcom,sm8750-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; + reg = <0x0 0x01d84000 0x0 0x3000>; + + interrupts = ; + + clocks = <&gcc GCC_UFS_PHY_AXI_CLK>, + <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>, + <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>, + <&rpmhcc RPMH_LN_BB_CLK3>, + <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>, + <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>, + <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>; + clock-names = "core_clk", + "bus_aggr_clk", + "iface_clk", + "core_clk_unipro", + "ref_clk", + "tx_lane0_sync_clk", + "rx_lane0_sync_clk", + "rx_lane1_sync_clk"; + + operating-points-v2 = <&ufs_opp_table>; + + resets = <&gcc GCC_UFS_PHY_BCR>; + reset-names = "rst"; + + interconnects = <&aggre1_noc MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "ufs-ddr", + "cpu-ufs"; + + power-domains = <&gcc GCC_UFS_PHY_GDSC>; + required-opps = <&rpmhpd_opp_nom>; + + iommus = <&apps_smmu 0x60 0>; + dma-coherent; + + lanes-per-direction = <2>; + + phys = <&ufs_mem_phy>; + phy-names = "ufsphy"; + + #reset-cells = <1>; + + status = "disabled"; + + ufs_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <100000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-403000000 { + opp-hz = /bits/ 64 <403000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <403000000>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>, + /bits/ 64 <0>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + apps_rsc: rsc@16500000 { + compatible = "qcom,rpmh-rsc"; + reg = <0x0 0x16500000 0x0 0x10000>, + <0x0 0x16510000 0x0 0x10000>, + <0x0 0x16520000 0x0 0x10000>; + reg-names = "drv-0", + "drv-1", + "drv-2"; + + interrupts = , + , + ; + qcom,tcs-offset = <0xd00>; + qcom,drv-id = <2>; + qcom,tcs-config = , , + , ; + + label = "apps_rsc"; + + power-domains = <&system_pd>; + + apps_bcm_voter: bcm-voter { + compatible = "qcom,bcm-voter"; + }; + + rpmhcc: clock-controller { + compatible = "qcom,sm8750-rpmh-clk"; + + clocks = <&xo_board>; + clock-names = "xo"; + + #clock-cells = <1>; + }; + + rpmhpd: power-controller { + compatible = "qcom,sm8750-rpmhpd"; + + operating-points-v2 = <&rpmhpd_opp_table>; + + #power-domain-cells = <1>; + + rpmhpd_opp_table: opp-table { + compatible = "operating-points-v2"; + + rpmhpd_opp_ret: opp-16 { + opp-level = ; + }; + + rpmhpd_opp_min_svs: opp-48 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d3: opp-50 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d2: opp-52 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d1: opp-56 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_d0: opp-60 { + opp-level = ; + }; + + rpmhpd_opp_low_svs: opp-64 { + opp-level = ; + }; + + rpmhpd_opp_low_svs_l1: opp-80 { + opp-level = ; + }; + + rpmhpd_opp_svs: opp-128 { + opp-level = ; + }; + + rpmhpd_opp_svs_l0: opp-144 { + opp-level = ; + }; + + rpmhpd_opp_svs_l1: opp-192 { + opp-level = ; + }; + + rpmhpd_opp_svs_l2: opp-224 { + opp-level = ; + }; + + rpmhpd_opp_nom: opp-256 { + opp-level = ; + }; + + rpmhpd_opp_nom_l1: opp-320 { + opp-level = ; + }; + + rpmhpd_opp_nom_l2: opp-336 { + opp-level = ; + }; + + rpmhpd_opp_turbo: opp-384 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l1: opp-416 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l2: opp-432 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l3: opp-448 { + opp-level = ; + }; + + rpmhpd_opp_turbo_l4: opp-452 { + opp-level = ; + }; + + rpmhpd_opp_super_turbo_no_cpr: opp-480 { + opp-level = + ; + }; + }; + }; + }; + + timer@16800000 { + compatible = "arm,armv7-timer-mem"; + reg = <0x0 0x16800000 0x0 0x1000>; + + #address-cells = <2>; + #size-cells = <1>; + ranges = <0 0 0 0 0x20000000>; + + frame@16801000 { + reg = <0x0 0x16801000 0x1000>, + <0x0 0x16802000 0x1000>; + + interrupts = , + ; + + frame-number = <0>; + }; + + frame@16803000 { + reg = <0x0 0x16803000 0x1000>; + + interrupts = ; + + frame-number = <1>; + + status = "disabled"; + }; + + frame@16805000 { + reg = <0x0 0x16805000 0x1000>; + + interrupts = ; + + frame-number = <2>; + + status = "disabled"; + }; + + frame@16807000 { + reg = <0x0 0x16807000 0x1000>; + + interrupts = ; + + frame-number = <3>; + + status = "disabled"; + }; + + frame@16809000 { + reg = <0x0 0x16809000 0x1000>; + + interrupts = ; + + frame-number = <4>; + + status = "disabled"; + }; + + frame@1680b000 { + reg = <0x0 0x1680b000 0x1000>; + + interrupts = ; + + frame-number = <5>; + + status = "disabled"; + }; + + frame@1680d000 { + reg = <0x0 0x1680d000 0x1000>; + + interrupts = ; + + frame-number = <6>; + + status = "disabled"; + }; + }; + + /* cluster0 */ + pmu@240b3400 { + compatible = "qcom,sm8750-cpu-bwmon", "qcom,sdm845-bwmon"; + reg = <0x0 0x240b3400 0x0 0x600>; + + interrupts = ; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>; + + operating-points-v2 = <&cpu_bwmon_opp_table>; + + nonposted-mmio; + + cpu_bwmon_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-0 { + opp-peak-kBps = <800000>; + }; + + opp-1 { + opp-peak-kBps = <2188000>; + }; + + opp-2 { + opp-peak-kBps = <5414400>; + }; + + opp-3 { + opp-peak-kBps = <6220800>; + }; + + opp-4 { + opp-peak-kBps = <6835200>; + }; + + opp-5 { + opp-peak-kBps = <8371200>; + }; + + opp-6 { + opp-peak-kBps = <10944000>; + }; + + opp-7 { + opp-peak-kBps = <12748800>; + }; + + opp-8 { + opp-peak-kBps = <14745600>; + }; + + opp-9 { + opp-peak-kBps = <16896000>; + }; + + opp-10 { + opp-peak-kBps = <19046400>; + }; + }; + }; + + /* cluster1 */ + pmu@240b7400 { + compatible = "qcom,sm8750-cpu-bwmon", "qcom,sdm845-bwmon"; + reg = <0x0 0x240b7400 0x0 0x600>; + + interrupts = ; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>; + + operating-points-v2 = <&cpu_bwmon_opp_table>; + }; + + gem_noc: interconnect@24100000 { + compatible = "qcom,sm8750-gem-noc"; + reg = <0x0 0x24100000 0x0 0x14b080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + system-cache-controller@24800000 { + compatible = "qcom,sm8750-llcc"; + reg = <0x0 0x24800000 0x0 0x200000>, + <0x0 0x25800000 0x0 0x200000>, + <0x0 0x24c00000 0x0 0x200000>, + <0x0 0x25c00000 0x0 0x200000>, + <0x0 0x26800000 0x0 0x200000>, + <0x0 0x26c00000 0x0 0x200000>; + reg-names = "llcc0_base", + "llcc1_base", + "llcc2_base", + "llcc3_base", + "llcc_broadcast_base", + "llcc_broadcast_and_base"; + + interrupts = ; + }; + + nsp_noc: interconnect@320c0000 { + compatible = "qcom,sm8750-nsp-noc"; + reg = <0x0 0x320c0000 0x0 0x13080>; + qcom,bcm-voters = <&apps_bcm_voter>; + #interconnect-cells = <2>; + }; + + remoteproc_cdsp: remoteproc@32300000 { + compatible = "qcom,sm8750-cdsp-pas", "qcom,sm8650-cdsp-pas"; + reg = <0x0 0x32300000 0x0 0x10000>; + + interrupts-extended = <&intc GIC_SPI 578 IRQ_TYPE_EDGE_RISING>, + <&smp2p_cdsp_in 0 IRQ_TYPE_EDGE_RISING>, + <&smp2p_cdsp_in 1 IRQ_TYPE_EDGE_RISING>, + <&smp2p_cdsp_in 2 IRQ_TYPE_EDGE_RISING>, + <&smp2p_cdsp_in 3 IRQ_TYPE_EDGE_RISING>, + <&smp2p_cdsp_in 7 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "wdog", + "fatal", + "ready", + "handover", + "stop-ack", + "shutdown-ack"; + + clocks = <&rpmhcc RPMH_CXO_CLK>; + clock-names = "xo"; + + interconnects = <&nsp_noc MASTER_CDSP_PROC QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + + power-domains = <&rpmhpd RPMHPD_CX>, + <&rpmhpd RPMHPD_MXC>, + <&rpmhpd RPMHPD_NSP>; + power-domain-names = "cx", + "mxc", + "nsp"; + + memory-region = <&cdsp_mem>, <&q6_cdsp_dtb_mem>, <&global_sync_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_cdsp_out 0>; + qcom,smem-state-names = "stop"; + + status = "disabled"; + + glink-edge { + interrupts-extended = <&ipcc IPCC_CLIENT_CDSP + IPCC_MPROC_SIGNAL_GLINK_QMP + IRQ_TYPE_EDGE_RISING>; + mboxes = <&ipcc IPCC_CLIENT_CDSP + IPCC_MPROC_SIGNAL_GLINK_QMP>; + qcom,remote-pid = <5>; + label = "cdsp"; + + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "cdsp"; + qcom,non-secure-domain; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@1 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <1>; + iommus = <&apps_smmu 0x19c1 0x0>, + <&apps_smmu 0x0c21 0x0>, + <&apps_smmu 0x0c01 0x40>; + dma-coherent; + }; + + compute-cb@2 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <2>; + iommus = <&apps_smmu 0x1962 0x0>, + <&apps_smmu 0x0c02 0x20>, + <&apps_smmu 0x0c42 0x0>, + <&apps_smmu 0x19c2 0x0>; + dma-coherent; + }; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x1963 0x0>, + <&apps_smmu 0x0c23 0x0>, + <&apps_smmu 0x0c03 0x40>, + <&apps_smmu 0x19c3 0x0>; + dma-coherent; + }; + + compute-cb@4 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <4>; + iommus = <&apps_smmu 0x1964 0x0>, + <&apps_smmu 0x0c24 0x0>, + <&apps_smmu 0x0c04 0x40>, + <&apps_smmu 0x19c4 0x0>; + dma-coherent; + }; + + compute-cb@5 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <5>; + iommus = <&apps_smmu 0x1965 0x0>, + <&apps_smmu 0x0c25 0x0>, + <&apps_smmu 0x0c05 0x40>, + <&apps_smmu 0x19c5 0x0>; + dma-coherent; + }; + + compute-cb@6 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <6>; + iommus = <&apps_smmu 0x1966 0x0>, + <&apps_smmu 0x0c06 0x20>, + <&apps_smmu 0x0c46 0x0>, + <&apps_smmu 0x19c6 0x0>; + dma-coherent; + }; + + compute-cb@7 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <7>; + iommus = <&apps_smmu 0x1967 0x0>, + <&apps_smmu 0x0c27 0x0>, + <&apps_smmu 0x0c07 0x40>, + <&apps_smmu 0x19c7 0x0>; + dma-coherent; + }; + + compute-cb@8 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <8>; + iommus = <&apps_smmu 0x1968 0x0>, + <&apps_smmu 0x0c08 0x20>, + <&apps_smmu 0x0c48 0x0>, + <&apps_smmu 0x19c8 0x0>; + dma-coherent; + }; + + /* note: secure cb9 in downstream */ + + compute-cb@12 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <12>; + iommus = <&apps_smmu 0x196c 0x0>, + <&apps_smmu 0x0c2c 0x20>, + <&apps_smmu 0x0c0c 0x40>, + <&apps_smmu 0x19cc 0x0>; + dma-coherent; + }; + + compute-cb@13 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <13>; + iommus = <&apps_smmu 0x196d 0x0>, + <&apps_smmu 0x0c0d 0x20>, + <&apps_smmu 0x0c2e 0x0>, + <&apps_smmu 0x0c4d 0x0>, + <&apps_smmu 0x19cd 0x0>; + dma-coherent; + }; + + compute-cb@14 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <14>; + iommus = <&apps_smmu 0x196e 0x0>, + <&apps_smmu 0x0c0e 0x20>, + <&apps_smmu 0x19ce 0x0>; + dma-coherent; + }; + }; + }; + }; + + tsens0: thermal-sensor@c228000 { + compatible = "qcom,sm8750-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c228000 0x0 0x1000>, + <0x0 0x0c222000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", + "critical"; + #qcom,sensors = <15>; + #thermal-sensor-cells = <1>; + }; + + tsens1: thermal-sensor@c229000 { + compatible = "qcom,sm8750-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c229000 0x0 0x1000>, + <0x0 0x0c223000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", + "critical"; + #qcom,sensors = <7>; + #thermal-sensor-cells = <1>; + }; + + tsens2: thermal-sensor@c22a000 { + compatible = "qcom,sm8750-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c22a000 0x0 0x1000>, + <0x0 0x0c224000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", + "critical"; + #qcom,sensors = <16>; + #thermal-sensor-cells = <1>; + }; + + tsens3: thermal-sensor@c22b000 { + compatible = "qcom,sm8750-tsens", "qcom,tsens-v2"; + reg = <0x0 0x0c22b000 0x0 0x1000>, + <0x0 0x0c225000 0x0 0x1000>; + interrupts = , + ; + interrupt-names = "uplow", + "critical"; + #qcom,sensors = <9>; + #thermal-sensor-cells = <1>; + }; + }; + + thermal-zones { + aoss0-thermal { + thermal-sensors = <&tsens0 0>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + aoss0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-0-0-thermal { + thermal-sensors = <&tsens0 1>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-0-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-0-1-thermal { + thermal-sensors = <&tsens0 2>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-0-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-1-0-thermal { + thermal-sensors = <&tsens0 3>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-1-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-1-1-thermal { + thermal-sensors = <&tsens0 4>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-1-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-2-0-thermal { + thermal-sensors = <&tsens0 5>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-2-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-2-1-thermal { + thermal-sensors = <&tsens0 6>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-2-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-3-0-thermal { + thermal-sensors = <&tsens0 7>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-3-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-3-1-thermal { + thermal-sensors = <&tsens0 8>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-3-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-4-0-thermal { + thermal-sensors = <&tsens0 9>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-4-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-4-1-thermal { + thermal-sensors = <&tsens0 10>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-4-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-5-0-thermal { + thermal-sensors = <&tsens0 11>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-5-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-0-5-1-thermal { + thermal-sensors = <&tsens0 12>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-0-5-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpuss-0-0-thermal { + thermal-sensors = <&tsens0 13>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpuss-0-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpuss-0-1-thermal { + thermal-sensors = <&tsens0 14>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpuss-0-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + aoss1-thermal { + thermal-sensors = <&tsens1 0>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + aoss1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-1-0-0-thermal { + thermal-sensors = <&tsens1 1>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-1-0-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-1-0-1-thermal { + thermal-sensors = <&tsens1 2>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-1-0-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-1-1-0-thermal { + thermal-sensors = <&tsens1 3>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-1-1-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpu-1-1-1-thermal { + thermal-sensors = <&tsens1 4>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpu-1-1-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpuss-1-0-thermal { + thermal-sensors = <&tsens1 5>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpuss-1-0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cpuss-1-1-thermal { + thermal-sensors = <&tsens1 6>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + cpuss-1-1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + aoss2-thermal { + thermal-sensors = <&tsens2 0>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + aoss2-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss0-thermal { + thermal-sensors = <&tsens2 1>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss1-thermal { + thermal-sensors = <&tsens2 2>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss2-thermal { + thermal-sensors = <&tsens2 3>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss2-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss3-thermal { + thermal-sensors = <&tsens2 4>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss3-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss4-thermal { + thermal-sensors = <&tsens2 5>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss4-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss5-thermal { + thermal-sensors = <&tsens2 6>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss5-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss6-thermal { + thermal-sensors = <&tsens2 7>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss6-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpuss7-thermal { + thermal-sensors = <&tsens2 8>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + gpuss7-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + modem0-thermal { + thermal-sensors = <&tsens2 9>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + modem0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + modem1-thermal { + thermal-sensors = <&tsens2 10>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + modem1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + modem2-thermal { + thermal-sensors = <&tsens2 11>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + modem2-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + modem3-thermal { + thermal-sensors = <&tsens2 12>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + modem3-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + camera0-thermal { + thermal-sensors = <&tsens2 13>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + camera0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + camera1-thermal { + thermal-sensors = <&tsens2 14>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + camera1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + video-thermal { + thermal-sensors = <&tsens2 15>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + video-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + aoss3-thermal { + thermal-sensors = <&tsens3 0>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + aoss3-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + nsphvx0-thermal { + thermal-sensors = <&tsens3 1>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + nsphvx0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + nsphvx1-thermal { + thermal-sensors = <&tsens3 2>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + nsphvx1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + nsphvx2-thermal { + thermal-sensors = <&tsens3 3>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + nsphvx2-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + nsphmx0-thermal { + thermal-sensors = <&tsens3 4>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + nsphmx0-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + nsphmx1-thermal { + thermal-sensors = <&tsens3 5>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + nsphmx1-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + nsphmx2-thermal { + thermal-sensors = <&tsens3 6>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + nsphmx2-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + nsphmx3-thermal { + thermal-sensors = <&tsens3 7>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <5000>; + type = "hot"; + }; + + nsphmx3-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + ddr-thermal { + thermal-sensors = <&tsens3 8>; + + trips { + trip-point0 { + temperature = <120000>; + hysteresis = <2000>; + type = "hot"; + }; + + ddr-critical { + temperature = <125000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + + interrupts = , + , + , + ; + }; + + tpdm_llm { + compatible = "qcom,coresight-static-tpdm"; + qcom,cmb-element-bits = <32>; + + out-ports { + port { + tpdm_turing_llm_out: endpoint { + remote-endpoint = <&tpda_turing_in1>; + }; + }; + }; + }; + + tpdm_llm2 { + compatible = "qcom,coresight-static-tpdm"; + qcom,cmb-element-bits = <32>; + + out-ports { + port { + tpdm_turing_llm2_out: endpoint { + remote-endpoint = <&tpda_turing_in2>; + }; + }; + }; + }; + + tpdm_modem1 { + compatible = "qcom,coresight-static-tpdm"; + qcom,dsb-element-bits = <32>; + + out-ports { + port { + tpdm_modem1_out: endpoint { + remote-endpoint = <&tpda_modem_in1>; + }; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/qcom/talos-camera-sensor.dtsi b/arch/arm64/boot/dts/qcom/talos-camera-sensor.dtsi new file mode 100644 index 0000000000000..006b345cc0023 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-camera-sensor.dtsi @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +&cam_cci { + /*cam0-imx577*/ + tl_slot0: qcom,cam-sensor0 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <0>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam0>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk3_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 28 0>, + <&tlmm 75 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAM_MCLK0", "CAMIF_RESET0"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <0>; + status = "okay"; + }; + + /*cam1-imx577*/ + t1_slot1: qcom,cam-sensor1 { + compatible = "qcom,cam-sensor"; + csiphy-sd-index = <1>; + sensor-position-roll = <0>; + sensor-position-pitch = <0>; + sensor-position-yaw = <180>; + eeprom-src = <&eeprom_cam1>; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst1>; + gpios = <&tlmm 30 0>, + <&tlmm 29 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAM_MCLK1", "CAMIF_RESET1"; + cci-master = <1>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <1>; + status = "okay"; + }; + + /*cam0-imx577-eeprom*/ + eeprom_cam0: qcom,eeprom0 { + compatible = "qcom,eeprom"; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk3_active + &cam_sensor_active_rst0>; + pinctrl-1 = <&cam_sensor_mclk3_suspend + &cam_sensor_suspend_rst0>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 28 0>, + <&tlmm 75 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAM_MCLK0", "CAMIF_RESET0"; + cci-master = <0>; + clocks = <&camcc CAM_CC_MCLK0_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <0>; + status = "okay"; + }; + + /*cam1-imx577-eeprom*/ + eeprom_cam1: qcom,eeprom1 { + compatible = "qcom,eeprom"; + cam_vio-supply = <&vreg_s4a>; + regulator-names = "cam_vio"; + power-domains = <&camcc TITAN_TOP_GDSC>; + rgltr-cntrl-support; + pwm-switch; + rgltr-min-voltage = <1800000>; + rgltr-max-voltage = <1800000>; + rgltr-load-current = <120000>; + gpio-no-mux = <0>; + pinctrl-0 = <&cam_sensor_mclk2_active + &cam_sensor_active_rst1>; + pinctrl-1 = <&cam_sensor_mclk2_suspend + &cam_sensor_suspend_rst1>; + pinctrl-names = "cam_default", "cam_suspend"; + gpios = <&tlmm 30 0>, + <&tlmm 29 0>; + gpio-reset = <1>; + gpio-req-tbl-num = <0 1>; + gpio-req-tbl-flags = <1 0>; + gpio-req-tbl-label = "CAM_MCLK1", "CAMIF_RESET1"; + cci-master = <1>; + clocks = <&camcc CAM_CC_MCLK1_CLK>; + clock-names = "cam_clk"; + clock-cntl-level = "nominal"; + clock-rates = <24000000>; + cell-index = <1>; + status = "okay"; + }; +}; + +&soc { + qcom,cam-res-mgr { + compatible = "qcom,cam-res-mgr"; + status = "ok"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/talos-camera.dtsi b/arch/arm64/boot/dts/qcom/talos-camera.dtsi new file mode 100644 index 0000000000000..9a869b4546499 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-camera.dtsi @@ -0,0 +1,1985 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +&soc { + cam_icp: qcom,icp@ac00000 { + compatible = "qcom,cam-icp_v1"; + icp-version = <0x0100>; + reg = <0x0 0xac00000 0x0 0x6000>, + <0x0 0xac10000 0x0 0x8000>, + <0x0 0xac18000 0x0 0x3000>; + reg-names = "icp_qgic", "icp_sierra", "icp_csr"; + reg-cam-base = <0x00000 0x10000 0x18000>; + interrupts = ; + interrupt-names = "a5"; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_FAST_AHB_CLK_SRC>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_ICP_CLK>, + <&camcc CAM_CC_ICP_CLK_SRC>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_fast_ahb_clk_src", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_camnoc_axi_clk", + "cam_cc_icp_clk", + "cam_cc_icp_clk_src"; + clock-rates = <0 0 100000000 0 0 0 0 240000000>, + <0 0 200000000 0 0 0 0 360000000>, + <0 0 300000000 0 0 0 0 432000000>, + <0 0 404000000 0 0 0 0 480000000>, + <0 0 404000000 0 0 0 0 540000000>, + <0 0 404000000 0 0 0 0 600000000>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal", "nominal_l1", "turbo"; + src-clock-name = "cam_cc_icp_clk_src"; + operating-points-v2 = <&a5_opp_table>; + fw_name = "qcom/qcs615/CAMERA_ICP.elf"; + ubwc-cfg = <0x73 0x1CF>; + cam_hw_pid = <15>; + cell-index = <0>; + status = "okay"; + + a5_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-360000000 { + opp-hz = /bits/ 64 <360000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_nom_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + qcom,cam-cpas@ac40000 { + compatible = "qcom,cam-cpas"; + label = "cpas"; + arch-compat = "cpas_top"; + reg = <0x0 0xac40000 0x0 0x1000>, + <0x0 0xac42000 0x0 0x5000>; + reg-names = "cam_cpas_top", "cam_camnoc"; + reg-cam-base = <0x40000 0x42000>; + interrupts = ; + interrupt-names = "cpas_camnoc"; + camnoc-axi-min-ib-bw = <3000000000>; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_cpas_ahb_clk", + "cam_cc_camnoc_axi_clk"; + clock-rates = <0 0 0 0 0 0>, + <0 0 0 80000000 0 150000000>, + <0 0 0 80000000 0 240000000>, + <0 0 0 80000000 0 265000000>, + <0 0 0 80000000 0 355000000>, + <0 0 0 80000000 0 426000000>; + clock-cntl-level = "suspend", "lowsvs", "svs", + "svs_l1", "nominal", "turbo"; + src-clock-name = "cam_cc_camnoc_axi_clk"; + operating-points-v2 = <&cpas_opp_table>; + control-camnoc-axi-clk; + camnoc-bus-width = <32>; + camnoc-axi-clk-bw-margin-perc = <20>; + cam-icc-path-names = "cam_ahb"; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_CAMNOC_HF0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_HF1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "cam_ahb", + "cam_hf_0_mnoc", + "cam_hf_1_mnoc", + "cam_sf_0_mnoc"; + cam-ahb-num-cases = <9>; + cam-ahb-bw-KBps = <0 0>, <0 76800>, <0 150000>, <0 150000>, + <0 300000>, <0 300000>, <0 300000>, + <0 300000>, <0 300000>; + vdd-corners = ; + vdd-corner-ahb-mapping = "suspend", "lowsvs", "lowsvs", "svs", + "svs_l1", "nominal", "nominal", + "nominal", "turbo", "turbo"; + client-id-based; + client-names = "csiphy0", "csiphy1", "csiphy2", "cci0", + "csid0", "csid1", "csid2","ife0", + "ife1", "ife2", "ipe0", "cam-cdm-intf0", + "cpas-cdm0", "bps0", "icp0", "jpeg-dma0", + "jpeg-enc0", "lrmecpas0"; + cell-index = <0>; + status = "okay"; + + cpas_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-265000000 { + opp-hz = /bits/ 64 <265000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-355000000 { + opp-hz = /bits/ 64 <355000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-426000000 { + opp-hz = /bits/ 64 <426000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + + camera-bus-nodes { + level0-nodes { + level-index = <0>; + + bps0_all_rd: bps0-all-rd { + cell-index = <0>; + node-name = "bps0-all-rd"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_read0>; + }; + + bps0_all_wr: bps0-all-wr { + cell-index = <1>; + node-name = "bps0-all-wr"; + client-name = "bps0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write0>; + }; + + cpas_cdm0_all_rd: cpas-cdm0-all-rd { + cell-index = <2>; + node-name = "cpas-cdm0-all-rd"; + client-name = "cpas-cdm0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_read2>; + }; + + icp0_all_rd: icp0-all-rd { + cell-index = <3>; + node-name = "icp0-all-rd"; + client-name = "icp0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_read3>; + }; + + ife0_linear_pdaf_wr: ife0-linear-pdaf-wr { + cell-index = <4>; + node-name = "ife0-linear-pdaf-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ife0_rdi_pixel_raw_wr: ife0-rdi-pixel-raw-wr { + cell-index = <5>; + node-name = "ife0-rdi-pixel-raw-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ife0_ubwc_stats_wr: ife0-ubwc-stats-wr { + cell-index = <6>; + node-name = "ife0-ubwc-stats-wr"; + client-name = "ife0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ife1_linear_pdaf_wr: ife1-linear-pdaf-wr { + cell-index = <7>; + node-name = "ife1-linear-pdaf-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt1_write0>; + }; + + ife1_rdi_pixel_raw_wr: ife1-rdi-pixel-raw-wr { + cell-index = <8>; + node-name = "ife1-rdi-pixel-raw-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt1_write0>; + }; + + ife1_ubwc_stats_wr: ife1-ubwc-stats-wr { + cell-index = <9>; + node-name = "ife1-ubwc-stats-wr"; + client-name = "ife1"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt1_write0>; + }; + + ife2_rdi_wr: ife2-rdi-wr { + cell-index = <10>; + node-name = "ife2-rdi-wr"; + client-name = "ife2"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_rt0_write0>; + }; + + ipe0_all_rd: ipe0-all-rd { + cell-index = <11>; + node-name = "ipe0-all-rd"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_nrt0_read0>; + }; + + ipe0_ref_wr: ipe0-ref-wr { + cell-index = <12>; + node-name = "ipe0-ref-wr"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write0>; + }; + + ipe0_viddisp_wr: ipe0-viddisp-wr { + cell-index = <13>; + node-name = "ipe0-viddisp-wr"; + client-name = "ipe0"; + traffic-data = ; + traffic-transaction-type = ; + constituent-paths = ; + parent-node = <&level1_nrt0_write0>; + }; + + jpeg_dma0_all_rd: jpeg-dma0-all-rd { + cell-index = <14>; + node-name = "jpeg-dma0-all-rd"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = + ; + parent-node = <&level1_nrt0_read1>; + }; + + jpeg_dma0_all_wr: jpeg-dma0-all-wr { + cell-index = <15>; + node-name = "jpeg-dma0-all-wr"; + client-name = "jpeg-dma0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write1>; + }; + + jpeg_enc0_all_rd: jpeg-enc0-all-rd { + cell-index = <16>; + node-name = "jpeg-enc0-all-rd"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_read1>; + }; + + jpeg_enc0_all_wr: jpeg-enc0-all-wr { + cell-index = <17>; + node-name = "jpeg-enc0-all-wr"; + client-name = "jpeg-enc0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write1>; + }; + + lrme0_all_rd: lrme0-all-rd { + cell-index = <18>; + node-name = "lrme0-all-rd"; + client-name = "lrmecpas0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_read0>; + }; + + lrme0_all_wr: lrme0-all-wr { + cell-index = <19>; + node-name = "lrme0-all-wr"; + client-name = "lrmecpas0"; + traffic-data = ; + traffic-transaction-type = ; + parent-node = <&level1_nrt0_write0>; + }; + }; + + level1-nodes { + level-index = <1>; + camnoc-max-needed; + + level1_nrt0_read0: level1-nrt0-read0 { + cell-index = <20>; + node-name = "level1-nrt0-read0"; + parent-node = <&level2_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level1_nrt0_read1: level1-nrt0-read1 { + cell-index = <21>; + node-name = "level1-nrt0-read1"; + parent-node = <&level2_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level1_nrt0_read2: level1-nrt0-read2 { + cell-index = <22>; + node-name = "level1-nrt0-read2"; + parent-node = <&level2_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level1_nrt0_read3: level1-nrt0-read3 { + cell-index = <23>; + node-name = "level1-nrt0-read3"; + parent-node = <&level2_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level1_nrt0_write0: level1-nrt0-write0 { + cell-index = <24>; + node-name = "level1-nrt0-write0"; + parent-node = <&level2_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level1_nrt0_write1: level1-nrt0-write1 { + cell-index = <25>; + node-name = "level1-nrt0-write1"; + parent-node = <&level2_nrt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level1_rt0_write0: level1-rt0-write0 { + cell-index = <26>; + node-name = "level1-rt0-write0"; + parent-node = <&level2_rt0_rd_wr_sum>; + traffic-merge-type = ; + }; + + level1_rt1_write0: level1-rt1-write0 { + cell-index = <27>; + node-name = "level1-rt1-write0"; + parent-node = <&level2_rt1_rd_wr_sum>; + traffic-merge-type = ; + }; + }; + + level2-nodes { + level-index = <2>; + + level2_nrt0_rd_wr_sum: level2-nrt0-rd-wr-sum { + cell-index = <28>; + node-name = "level2-nrt0-rd-wr-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_sf_0_mnoc"; + }; + }; + + level2_rt0_rd_wr_sum: level2-rt0-rd-wr-sum { + cell-index = <29>; + node-name = "level2-rt0-rd-wr-sum"; + traffic-merge-type = ; + ib-bw-voting-needed; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_hf_0_mnoc"; + }; + }; + + level2_rt1_rd_wr_sum: level2-rt1-rd-wr-sum { + cell-index = <30>; + node-name = "level3-rt1-rd-wr-sum"; + traffic-merge-type = ; + + qcom,axi-port-mnoc { + cam-icc-path-names = "cam_hf_1_mnoc"; + }; + }; + }; + }; + }; + + qcom,cpas-cdm0@ac48000 { + compatible = "qcom,cam170-cpas-cdm0"; + label = "cpas-cdm"; + reg = <0x0 0xac48000 0x0 0x1000>; + reg-names = "cpas-cdm"; + reg-cam-base = <0x48000>; + interrupts = ; + interrupt-names = "cpas-cdm"; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_camnoc_axi_clk", + "cam_cc_slow_ahb_clk_src"; + clock-rates = <0 0 0 0 0 80000000>; + src-clock-name = "cam_cc_slow_ahb_clk_src"; + operating-points-v2 = <&cpas_cdm_opp_table>; + clock-cntl-level = "svs"; + cdm-client-names = "ife0", "ife1", "ife2", + "ife3", "dualife"; + single-context-cdm; + cell-index = <0>; + status = "okay"; + + cpas_cdm_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-80000000 { + opp-hz = /bits/ 64 <80000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + + cam_cci: qcom,cci@ac4a000 { + compatible = "qcom,cci", "simple-bus"; + reg = <0x0 0xac4a000 0x0 0x4000>; + reg-names = "cci"; + reg-cam-base = <0x4a000>; + interrupts = ; + interrupt-names = "CCI"; + operating-points-v2 = <&cci_opp_table>; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CCI_CLK_SRC>, + <&camcc CAM_CC_CCI_CLK>; + clock-names = "cci_clk_src", + "cci_clk"; + clock-rates = <37500000 0>; + clock-cntl-level = "lowsvs"; + src-clock-name = "cci_clk_src"; + pctrl-idx-mapping = ; + pctrl-map-names = "m0", "m1"; + pinctrl-0 = <&cci_i2c_scl0_active &cci_i2c_sda0_active>; + pinctrl-1 = <&cci_i2c_scl0_suspend &cci_i2c_sda0_suspend>; + pinctrl-2 = <&cci_i2c_scl1_active &cci_i2c_sda1_active>; + pinctrl-3 = <&cci_i2c_scl1_suspend &cci_i2c_sda1_suspend>; + pinctrl-names = "m0_active", "m0_suspend", + "m1_active", "m1_suspend"; + cell-index = <0>; + status = "okay"; + + cci_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-37500000 { + opp-hz = /bits/ 64 <37500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + i2c_freq_custom_cci: qcom,i2c-custom-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <1>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_400Khz_cci: qcom,i2c-fast-mode { + hw-thigh = <38>; + hw-tlow = <56>; + hw-tsu-sto = <40>; + hw-tsu-sta = <40>; + hw-thd-dat = <22>; + hw-thd-sta = <35>; + hw-tbuf = <62>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_1Mhz_cci: qcom,i2c-fast-plus-mode { + hw-thigh = <16>; + hw-tlow = <22>; + hw-tsu-sto = <17>; + hw-tsu-sta = <18>; + hw-thd-dat = <16>; + hw-thd-sta = <15>; + hw-tbuf = <24>; + hw-scl-stretch-en = <0>; + hw-trdhld = <3>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + + i2c_freq_100Khz_cci: qcom,i2c-standard-mode { + hw-thigh = <201>; + hw-tlow = <174>; + hw-tsu-sto = <204>; + hw-tsu-sta = <231>; + hw-thd-dat = <22>; + hw-thd-sta = <162>; + hw-tbuf = <227>; + hw-scl-stretch-en = <0>; + hw-trdhld = <6>; + hw-tsp = <3>; + cci-clk-src = <37500000>; + status = "okay"; + }; + }; + + cam_jpeg_enc: qcom,jpegenc@ac4e000 { + compatible = "qcom,cam_jpeg_enc_170"; + reg = <0x0 0xac4e000 0x0 0x4000>; + reg-names = "jpege_hw"; + reg-cam-base = <0x4e000>; + interrupts = ; + interrupt-names = "jpeg"; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_camnoc_axi_clk", + "cam_cc_jpeg_clk_src", + "cam_cc_jpeg_clk"; + clock-rates = <0 0 0 0 0 600000000 0>; + src-clock-name = "cam_cc_jpeg_clk_src"; + operating-points-v2 = <&jpeg_enc_opp_table>; + clock-cntl-level = "turbo"; + cell-index = <0>; + status = "okay"; + + jpeg_enc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_jpeg_dma: qcom,jpegdma@ac52000{ + compatible = "qcom,cam_jpeg_dma_170"; + reg = <0x0 0xac52000 0x0 0x4000>; + reg-names = "jpegdma_hw"; + reg-cam-base = <0x52000>; + interrupts = ; + interrupt-names = "jpegdma"; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_JPEG_CLK_SRC>, + <&camcc CAM_CC_JPEG_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_camnoc_axi_clk", + "cam_cc_jpeg_clk_src", + "cam_cc_jpeg_clk"; + clock-rates = <0 0 0 0 0 600000000 0>; + src-clock-name = "cam_cc_jpeg_clk_src"; + operating-points-v2 = <&jpeg_dma_opp_table>; + clock-cntl-level = "turbo"; + cell-index = <0>; + status = "okay"; + + jpeg_dma_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_csiphy0: qcom,csiphy@ac65000 { + compatible = "qcom,csiphy-v2.0.0", "qcom,csiphy"; + reg = <0x0 0x0ac65000 0x0 0x1000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy0_opp_table>; + reg-cam-base = <0x65000>; + interrupts = ; + interrupt-names = "CSIPHY0"; + csi-vdd-voltage = <1200000>; + mipi-csi-vdd-supply = <&vreg_l11a>; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>; + clock-names = "camnoc_axi_clk", + "soc_ahb_clk", + "slow_ahb_src_clk", + "cpas_ahb_clk", + "cphy_rx_clk_src", + "csiphy0_clk", + "csi0phytimer_clk_src", + "csi0phytimer_clk"; + src-clock-name = "csi0phytimer_clk_src"; + clock-cntl-level = "svs_l1", "turbo"; + clock-rates = <0 0 0 0 269333333 0 269333333 0>, + <0 0 0 0 384000000 0 269333333 0>; + cell-index = <0>; + status = "okay"; + + csiphy0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-269333333 { + opp-hz = /bits/ 64 <269333333>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-384000000 { + opp-hz = /bits/ 64 <384000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_csiphy1: qcom,csiphy@ac66000 { + compatible = "qcom,csiphy-v2.0.0", "qcom,csiphy"; + reg = <0x0 0xac66000 0x0 0x1000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy1_opp_table>; + reg-cam-base = <0x66000>; + interrupts = ; + interrupt-names = "CSIPHY1"; + csi-vdd-voltage = <1200000>; + mipi-csi-vdd-supply = <&vreg_l11a>; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>; + clock-names = "camnoc_axi_clk", + "soc_ahb_clk", + "slow_ahb_src_clk", + "cpas_ahb_clk", + "cphy_rx_clk_src", + "csiphy1_clk", + "csi1phytimer_clk_src", + "csi1phytimer_clk"; + src-clock-name = "csi1phytimer_clk_src"; + clock-cntl-level = "svs_l1", "turbo"; + clock-rates = <0 0 0 0 269333333 0 269333333 0>, + <0 0 0 0 384000000 0 269333333 0>; + cell-index = <1>; + status = "okay"; + + csiphy1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-269333333 { + opp-hz = /bits/ 64 <269333333>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-384000000 { + opp-hz = /bits/ 64 <384000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_csiphy2: qcom,csiphy@ac67000 { + compatible = "qcom,csiphy-v2.0.0", "qcom,csiphy"; + reg = <0x0 0xac67000 0x0 0x1000>; + reg-names = "csiphy"; + operating-points-v2 = <&csiphy2_opp_table>; + reg-cam-base = <0x67000>; + interrupts = ; + interrupt-names = "CSIPHY2"; + csi-vdd-voltage = <1200000>; + mipi-csi-vdd-supply = <&vreg_l11a>; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK_SRC>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>; + clock-names = "camnoc_axi_clk", + "soc_ahb_clk", + "slow_ahb_src_clk", + "cpas_ahb_clk", + "cphy_rx_clk_src", + "csiphy2_clk", + "csi2phytimer_clk_src", + "csi2phytimer_clk"; + src-clock-name = "csi2phytimer_clk_src"; + clock-cntl-level = "svs_l1", "turbo"; + clock-rates = <0 0 0 0 269333333 0 269333333 0>, + <0 0 0 0 384000000 0 269333333 0>; + cell-index = <2>; + status = "okay"; + + csiphy2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-269333333 { + opp-hz = /bits/ 64 <269333333>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-384000000 { + opp-hz = /bits/ 64 <384000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_lrme: qcom,lrme@ac6b000 { + compatible = "qcom,lrme"; + reg = <0x0 0xac6b000 0x0 0xa00>; + reg-names = "lrme"; + reg-cam-base = <0x6b000>; + interrupts = ; + interrupt-names = "lrme"; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_LRME_CLK_SRC>, + <&camcc CAM_CC_LRME_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_camnoc_axi_clk", + "cam_cc_lrme_clk_src", + "cam_cc_lrme_clk"; + clock-rates = <0 0 0 0 0 200000000 0>, + <0 0 0 0 0 216000000 0>, + <0 0 0 0 0 300000000 0>, + <0 0 0 0 0 404000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal"; + src-clock-name = "cam_cc_lrme_clk_src"; + operating-points-v2 = <&lrme_opp_table>; + cam_hw_pid = <6 10>; + cell-index = <0>; + status = "okay"; + + lrme_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-216000000 { + opp-hz = /bits/ 64 <216000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-404000000 { + opp-hz = /bits/ 64 <404000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + cam_bps: qcom,bps@0x0ac6f000 { + compatible = "qcom,cam-bps"; + reg = <0x0 0xac6f000 0x0 0x3000>; + reg-names = "bps_top"; + reg-cam-base = <0x6f000>; + power-domains = <&camcc BPS_GDSC>; + clocks = <&camcc CAM_CC_BPS_AHB_CLK>, + <&camcc CAM_CC_BPS_AREG_CLK>, + <&camcc CAM_CC_BPS_AXI_CLK>, + <&camcc CAM_CC_BPS_CLK>, + <&camcc CAM_CC_BPS_CLK_SRC>; + clock-names = "cam_cc_bps_ahb_clk", + "cam_cc_bps_areg_clk", + "cam_cc_bps_axi_clk", + "cam_cc_bps_clk", + "cam_cc_bps_clk_src"; + clock-rates = <0 0 0 0 200000000>, + <0 0 0 0 360000000>, + <0 0 0 0 432000000>, + <0 0 0 0 480000000>, + <0 0 0 0 540000000>, + <0 0 0 0 600000000>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", + "nominal", "nominal_l1", "turbo"; + src-clock-name = "cam_cc_bps_clk_src"; + operating-points-v2 = <&bps_opp_table>; + cam_hw_pid = <5 9>; + cell-index = <0>; + status = "okay"; + + bps_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-360000000 { + opp-hz = /bits/ 64 <360000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_nom_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_ipe0: qcom,ipe0@0x0ac87000 { + compatible = "qcom,cam-ipe"; + reg = <0x0 0xac87000 0x0 0x3000>; + reg-names = "ipe0_top"; + reg-cam-base = <0x87000>; + power-domains = <&camcc IPE_0_GDSC>; + clock-names = "cam_cc_ipe_0_ahb_clk", + "cam_cc_ipe_0_areg_clk", + "cam_cc_ipe_0_axi_clk", + "cam_cc_ipe_0_clk", + "cam_cc_ipe_0_clk_src"; + clocks = <&camcc CAM_CC_IPE_0_AHB_CLK>, + <&camcc CAM_CC_IPE_0_AREG_CLK>, + <&camcc CAM_CC_IPE_0_AXI_CLK>, + <&camcc CAM_CC_IPE_0_CLK>, + <&camcc CAM_CC_IPE_0_CLK_SRC>; + clock-rates = <0 0 0 0 240000000>, + <0 0 0 0 360000000>, + <0 0 0 0 432000000>, + <0 0 0 0 480000000>, + <0 0 0 0 540000000>, + <0 0 0 0 600000000>; + clock-cntl-level = "lowsvs","svs", "svs_l1", + "nominal", "nominal_l1", "turbo"; + src-clock-name = "cam_cc_ipe_0_clk_src"; + operating-points-v2 = <&ipe0_opp_table>; + cam_hw_pid = <4 8>; + cell-index = <0>; + status = "okay"; + + ipe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-360000000 { + opp-hz = /bits/ 64 <360000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_nom_l1>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_vfe0: qcom,vfe0@acaf000 { + compatible = "qcom,vfe170"; + reg = <0x0 0xacaf000 0x0 0x4000>, + <0x0 0xac42000 0x0 0x5000>; + reg-names = "ife0", "cam_camnoc"; + reg-cam-base = <0xaf000 0x42000>; + interrupts = ; + interrupt-names = "ife0"; + power-domains = <&camcc IFE_0_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_IFE_0_AXI_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_0_clk", + "cam_cc_ife_0_clk_src", + "cam_cc_camnoc_axi_clk", + "cam_cc_ife_0_axi_clk"; + clock-rates = <0 0 0 0 0 0 240000000 0 0>, + <0 0 0 0 0 0 360000000 0 0>, + <0 0 0 0 0 0 432000000 0 0>, + <0 0 0 0 0 0 540000000 0 0>, + <0 0 0 0 0 0 600000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal", "turbo"; + src-clock-name = "cam_cc_ife_0_clk_src"; + operating-points-v2 = <&vfe0_opp_table>; + clock-names-option = "cam_cc_ife_0_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_0_DSP_CLK>; + clock-rates-option = <600000000>; + cam_hw_pid = <1>; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "okay"; + + vfe0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-360000000 { + opp-hz = /bits/ 64 <360000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_csid0: qcom,csid0@acb3000 { + compatible = "qcom,csid170"; + reg = <0x0 0xacb3000 0x0 0x1000>; + reg-names = "csid0"; + reg-cam-base = <0xb3000>; + interrupts = ; + interrupt-names = "csid0"; + power-domains = <&camcc IFE_0_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CSID_CLK>, + <&camcc CAM_CC_IFE_0_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CPHY_RX_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_IFE_0_AXI_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_0_csid_clk", + "cam_cc_ife_0_csid_clk_src", + "cam_cc_ife_0_cphy_rx_clk", + "cam_cc_cphy_rx_clk_src", + "cam_cc_ife_0_clk", + "cam_cc_ife_0_clk_src", + "cam_cc_camnoc_axi_clk", + "cam_cc_ife_0_axi_clk"; + clock-rates = <0 0 0 0 0 0 100000000 0 0 0 240000000 0 0>, + <0 0 0 0 0 0 200000000 0 0 0 360000000 0 0>, + <0 0 0 0 0 0 320000000 0 0 0 432000000 0 0>, + <0 0 0 0 0 0 404000000 0 0 0 540000000 0 0>, + <0 0 0 0 0 0 480000000 0 0 0 540000000 0 0>, + <0 0 0 0 0 0 540000000 0 0 0 600000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal", "nominal_l1", "turbo"; + operating-points-v2 = <&csid0_opp_table>; + src-clock-name = "cam_cc_ife_0_csid_clk_src"; + clock-control-debugfs = "true"; + cell-index = <0>; + status = "okay"; + + csid0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-404000000 { + opp-hz = /bits/ 64 <404000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom_l1>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_vfe1: qcom,vfe1@acb6000 { + compatible = "qcom,vfe170"; + reg = <0x0 0xacb6000 0x0 0x4000>, + <0x0 0xac42000 0x0 0x5000>; + reg-names = "ife1", "cam_camnoc"; + reg-cam-base = <0xb6000 0x42000>; + interrupts = ; + interrupt-names = "ife1"; + power-domains = <&camcc IFE_1_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_IFE_1_AXI_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_1_clk", + "cam_cc_ife_1_clk_src", + "cam_cc_camnoc_axi_clk", + "cam_cc_ife_1_axi_clk"; + clock-rates = <0 0 0 0 0 0 240000000 0 0>, + <0 0 0 0 0 0 360000000 0 0>, + <0 0 0 0 0 0 432000000 0 0>, + <0 0 0 0 0 0 540000000 0 0>, + <0 0 0 0 0 0 600000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal", "turbo"; + src-clock-name = "cam_cc_ife_1_clk_src"; + operating-points-v2 = <&vfe1_opp_table>; + clock-names-option = "cam_cc_ife_1_dsp_clk"; + clocks-option = <&camcc CAM_CC_IFE_1_DSP_CLK>; + clock-rates-option = <600000000>; + cam_hw_pid = <2>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "okay"; + + vfe1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-360000000 { + opp-hz = /bits/ 64 <360000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_csid1: qcom,csid1@acba000 { + compatible = "qcom,csid170"; + reg = <0x0 0xacba000 0x0 0x1000>; + reg-names = "csid1"; + reg-cam-base = <0xba000>; + interrupts = ; + interrupt-names = "csid1"; + power-domains = <&camcc IFE_1_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CSID_CLK>, + <&camcc CAM_CC_IFE_1_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CPHY_RX_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_IFE_1_AXI_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_1_csid_clk", + "cam_cc_ife_1_csid_clk_src", + "cam_cc_ife_1_cphy_rx_clk", + "cam_cc_cphy_rx_clk_src", + "cam_cc_ife_1_clk", + "cam_cc_ife_1_clk_src", + "cam_cc_camnoc_axi_clk", + "cam_cc_ife_1_axi_clk"; + clock-rates = <0 0 0 0 0 0 100000000 0 0 0 240000000 0 0>, + <0 0 0 0 0 0 200000000 0 0 0 360000000 0 0>, + <0 0 0 0 0 0 320000000 0 0 0 432000000 0 0>, + <0 0 0 0 0 0 404000000 0 0 0 540000000 0 0>, + <0 0 0 0 0 0 480000000 0 0 0 540000000 0 0>, + <0 0 0 0 0 0 540000000 0 0 0 600000000 0 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal", "nominal_l1", "turbo"; + src-clock-name = "cam_cc_ife_1_csid_clk_src"; + operating-points-v2 = <&csid1_opp_table>; + clock-control-debugfs = "true"; + cell-index = <1>; + status = "okay"; + + csid1_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-404000000 { + opp-hz = /bits/ 64 <404000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom_l1>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_vfe_lite: qcom,vfe-lite@acc4000 { + compatible = "qcom,vfe-lite170"; + reg = <0x0 0xacc4000 0x0 0x4000>; + reg-names = "ife-lite0"; + reg-cam-base = <0xc4000>; + interrupts = ; + interrupt-names = "ife-lite0"; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_lite_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_camnoc_axi_clk"; + clock-rates = <0 0 0 0 0 0 240000000 0>, + <0 0 0 0 0 0 360000000 0>, + <0 0 0 0 0 0 432000000 0>, + <0 0 0 0 0 0 540000000 0>, + <0 0 0 0 0 0 600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal", "turbo"; + src-clock-name = "cam_cc_ife_lite_clk_src"; + operating-points-v2 = <&vfe_lite0_opp_table>; + cam_hw_pid = <3>; + clock-control-debugfs = "true"; + cell-index = <2>; + status = "okay"; + + vfe_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-360000000 { + opp-hz = /bits/ 64 <360000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + cam_csid_lite: qcom,csid-lite@acc8000 { + compatible = "qcom,csid-lite170"; + reg = <0x0 0xacc8000 0x0 0x1000>; + reg-names = "csid-lite0"; + reg-cam-base = <0xc8000>; + interrupts = ; + interrupt-names = "csid-lite0"; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_SLOW_AHB_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_CPHY_RX_CLK_SRC>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK_SRC>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>; + clock-names = "gcc_camera_ahb_clk", + "gcc_camera_hf_axi_clk", + "cam_cc_soc_ahb_clk", + "cam_cc_cpas_ahb_clk", + "cam_cc_slow_ahb_clk_src", + "cam_cc_ife_lite_csid_clk", + "cam_cc_ife_lite_csid_clk_src", + "cam_cc_ife_lite_cphy_rx_clk", + "cam_cc_cphy_rx_clk_src", + "cam_cc_ife_lite_clk", + "cam_cc_ife_lite_clk_src", + "cam_cc_camnoc_axi_clk"; + clock-rates = <0 0 0 0 0 0 100000000 0 0 0 240000000 0>, + <0 0 0 0 0 0 200000000 0 0 0 360000000 0>, + <0 0 0 0 0 0 320000000 0 0 0 432000000 0>, + <0 0 0 0 0 0 404000000 0 0 0 540000000 0>, + <0 0 0 0 0 0 480000000 0 0 0 540000000 0>, + <0 0 0 0 0 0 540000000 0 0 0 600000000 0>; + clock-cntl-level = "lowsvs", "svs", "svs_l1", "nominal", "nominal_l1", "turbo"; + src-clock-name = "cam_cc_ife_lite_csid_clk_src"; + operating-points-v2 = <&csid_lite0_opp_table>; + clock-control-debugfs = "true"; + cell-index = <2>; + status = "okay"; + + csid_lite0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-320000000 { + opp-hz = /bits/ 64 <320000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-404000000 { + opp-hz = /bits/ 64 <404000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-480000000 { + opp-hz = /bits/ 64 <480000000>; + required-opps = <&rpmhpd_opp_nom_l1>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + }; + + qcom,cam-cdm-intf { + compatible = "qcom,cam-cdm-intf"; + cell-index = <0>; + label = "cam-cdm-intf"; + num-hw-cdm = <1>; + cdm-client-names = "vfe", "jpegdma", + "jpegenc", "lrmecdm"; + status = "okay"; + }; + + qcom,cam-icp { + compatible = "qcom,cam-icp"; + compat-hw-name = "qcom,icp", + "qcom,ipe0", + "qcom,bps"; + num-icp = <1>; + num-ipe = <1>; + num-bps = <1>; + ipe0-mask = <0x1000>; + icp_pc_en; + status = "okay"; + }; + + cam_isp_mgr: qcom,cam-isp { + compatible = "qcom,cam-isp"; + arch-compat = "ife"; + status = "okay"; + }; + + cam_jpeg_mgr: qcom,cam-jpeg { + compatible = "qcom,cam-jpeg"; + compat-hw-name = "qcom,jpegenc", + "qcom,jpegdma"; + num-jpeg-enc = <1>; + num-jpeg-dma = <1>; + status = "okay"; + }; + + cam_lrme_mgr: qcom,cam-lrme { + compatible = "qcom,cam-lrme"; + arch-compat = "lrme"; + status = "okay"; + }; + + qcom,camera-main { + compatible = "qcom,camera_qcs615"; + status = "okay"; + }; + + qcom,cam-req-mgr { + compatible = "qcom,cam-req-mgr"; + status = "okay"; + }; + + cam_smmu: qcom,cam-smmu { + compatible = "qcom,msm-cam-smmu", "simple-bus"; + status = "okay"; + + msm-cam-icp-fw { + compatible = "qcom,msm-cam-smmu-fw-dev"; + label="icp"; + memory-region = <&pil_camera_mem>; + }; + + msm-cam-smmu-cpas-cdm { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0c00 0x0>; + cam-smmu-label = "cpas-cdm"; + + cpas_cdm_iova_mem_map: iova-mem-map { + iova-mem-region-io { + /* IO region is approximately 3.4 GB */ + iova-region-name = "io"; + iova-region-start = <0x7400000>; + iova-region-len = <0xd8c00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-icp { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0de2 0x0>, + <&apps_smmu 0x0c80 0x0>, + <&apps_smmu 0x0ca0 0x0>, + <&apps_smmu 0x0d00 0x0>, + <&apps_smmu 0x0d20 0x0>; + cam-smmu-label = "icp"; + + icp_iova_mem_map: iova-mem-map { + iova-mem-region-firmware { + /* Firmware region is 8MB */ + iova-region-name = "firmware"; + iova-region-start = <0x0>; + iova-region-len = <0x800000>; + iova-region-id = <0x0>; + status = "okay"; + }; + + iova-mem-region-io { + /* IO region is approximately 3.3 GB */ + iova-region-name = "io"; + iova-region-start = <0x10C00000>; + iova-region-len = <0xCF300000>; + iova-region-id = <0x3>; + status = "okay"; + }; + + iova-mem-qdss-region { + /* qdss region is approximately 1MB */ + iova-region-name = "qdss"; + iova-region-start = <0x10B00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x5>; + qdss-phy-addr = <0x16790000>; + status = "okay"; + }; + + iova-mem-region-secondary-heap { + /* Secondary heap region is 1MB long */ + iova-region-name = "secheap"; + iova-region-start = <0x10A00000>; + iova-region-len = <0x100000>; + iova-region-id = <0x4>; + status = "okay"; + }; + + iova-mem-region-shared { + /* Shared region is 150MB long */ + iova-region-name = "shared"; + iova-region-start = <0x7400000>; + iova-region-len = <0x9600000>; + iova-region-id = <0x1>; + iova-granularity = <0x15>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-ife { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0820 0x40>, + <&apps_smmu 0x0840 0x00>, + <&apps_smmu 0x0860 0x40>; + cam-smmu-label = "ife"; + multiple-client-devices; + com,iommu-faults = "stall-disable", "non-fatal"; + + ife_iova_mem_map: iova-mem-map { + /* IO region is approximately 3.4 GB */ + iova-mem-region-io { + iova-region-name = "io"; + iova-region-start = <0x7400000>; + iova-region-len = <0xd8c00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-jpeg { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0xd80 0x20>, + <&apps_smmu 0xda0 0x20>; + cam-smmu-label = "jpeg"; + + jpeg_iova_mem_map: iova-mem-map { + /* IO region is approximately 3.4 GB */ + iova-mem-region-io { + iova-region-name = "io"; + iova-region-start = <0x7400000>; + iova-region-len = <0xd8c00000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-lrme { + compatible = "qcom,msm-cam-smmu-cb"; + iommus = <&apps_smmu 0x0cc0 0x0>, + <&apps_smmu 0x0d40 0x0>; + cam-smmu-label = "lrme"; + + lrme_iova_mem_map: iova-mem-map { + iova-mem-region-shared { + /* Shared region is 100MB long */ + iova-region-name = "shared"; + iova-region-start = <0x7400000>; + iova-region-len = <0x6400000>; + iova-region-id = <0x1>; + status = "okay"; + }; + + /* IO region is approximately 3.3 GB */ + iova-mem-region-io { + iova-region-name = "io"; + iova-region-start = <0xd800000>; + iova-region-len = <0xd2800000>; + iova-region-id = <0x3>; + status = "okay"; + }; + }; + }; + + msm-cam-smmu-secure { + compatible = "qcom,msm-cam-smmu-cb"; + cam-smmu-label = "cam-secure"; + qcom,secure-cb; + }; + }; + + qcom,cam-sync { + compatible = "qcom,cam-sync"; + status = "okay"; + }; +}; + +&tlmm { + cam_sensor_active_rst0: cam-sensor-active-rst0 { + /* RESET */ + mux { + pins = "gpio75"; + function = "gpio"; + }; + + config { + pins = "gpio75"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst1: cam-sensor-active-rst1 { + /* RESET */ + mux { + pins = "gpio29"; + function = "gpio"; + }; + + config { + pins = "gpio29"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_active_rst2: cam-sensor-active-rst2 { + /* RESET */ + mux { + pins = "gpio37"; + function = "gpio"; + }; + + config { + pins = "gpio37"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_active: cam-sensor-mclk2-active { + /* MCLK2 */ + mux { + pins = "gpio30"; + function = "cam_mclk"; + }; + + config { + pins = "gpio30"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk2_suspend: cam-sensor-mclk2-suspend { + /* MCLK2 */ + mux { + pins = "gpio30"; + function = "cam_mclk"; + }; + + config { + pins = "gpio30"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk3_active: cam-sensor-mclk3-active { + /* MCLK3 */ + mux { + pins = "gpio28"; + function = "cam_mclk"; + }; + + config { + pins = "gpio28"; + bias-disable; /* No PULL */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_mclk3_suspend: cam-sensor-mclk3-suspend { + /* MCLK3 */ + mux { + pins = "gpio28"; + function = "cam_mclk"; + }; + + config { + pins = "gpio28"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cam_sensor_suspend_rst0: cam-sensor-suspend-rst0 { + /* RESET */ + mux { + pins = "gpio75"; + function = "gpio"; + }; + + config { + pins = "gpio75"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst1: cam-sensor-suspend-rst1 { + /* RESET */ + mux { + pins = "gpio29"; + function = "gpio"; + }; + + config { + pins = "gpio29"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cam_sensor_suspend_rst2: cam-sensor-suspend-rst2 { + /* RESET */ + mux { + pins = "gpio37"; + function = "gpio"; + }; + + config { + pins = "gpio37"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + output-low; + }; + }; + + cci_i2c_scl0_active: cci-i2c-scl0-active { + mux { + /* CLK, DATA */ + pins = "gpio33"; + function = "cci_i2c"; + }; + + config { + pins = "gpio33"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl0_suspend: cci-i2c-scl0-suspend { + mux { + /* CLK, DATA */ + pins = "gpio33"; + function = "cci_i2c"; + }; + + config { + pins = "gpio33"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl1_active: cci-i2c-scl1-active { + mux { + /* CLK, DATA */ + pins = "gpio35"; + function = "cci_i2c"; + }; + + config { + pins = "gpio35"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_scl1_suspend: cci-i2c-scl1-suspend { + mux { + /* CLK, DATA */ + pins = "gpio35"; + function = "cci_i2c"; + }; + + config { + pins = "gpio35"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + + cci_i2c_sda0_active: cci-i2c-sda0-active { + mux { + /* CLK, DATA */ + pins = "gpio32"; + function = "cci_i2c"; + }; + + config { + pins = "gpio32"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda0_suspend: cci-i2c-sda0-suspend { + mux { + /* CLK, DATA */ + pins = "gpio32"; + function = "cci_i2c"; + }; + + config { + pins = "gpio32"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda1_active: cci-i2c-sda1-active { + mux { + /* CLK, DATA */ + pins = "gpio34"; + function = "cci_i2c"; + }; + + config { + pins = "gpio34"; + bias-pull-up; /* PULL UP*/ + drive-strength = <2>; /* 2 MA */ + }; + }; + + cci_i2c_sda1_suspend: cci-i2c-sda1-suspend { + mux { + /* CLK, DATA */ + pins = "gpio34"; + function = "cci_i2c"; + }; + + config { + pins = "gpio34"; + bias-pull-down; /* PULL DOWN */ + drive-strength = <2>; /* 2 MA */ + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/talos-el2.dtso b/arch/arm64/boot/dts/qcom/talos-el2.dtso new file mode 100644 index 0000000000000..08b5c9a537fb6 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-el2.dtso @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + + * Talos specific modifications required to boot in EL2. + */ + +#include + +/dts-v1/; +/plugin/; + +&gpu_zap_shader { + status = "disabled"; +}; + +&remoteproc_adsp { + iommus = <&apps_smmu 0x1720 0x0>; +}; + +&remoteproc_cdsp { + iommus = <&apps_smmu 0x1080 0x0>; +}; + +&venus { + iommu-map = ; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/talos-evk-camera-imx577.dtso b/arch/arm64/boot/dts/qcom/talos-evk-camera-imx577.dtso new file mode 100644 index 0000000000000..53006a861878f --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-evk-camera-imx577.dtso @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include + +&camss { + vdd-csiphy-1p2-supply = <&vreg_l11a>; + vdd-csiphy-1p8-supply = <&vreg_l12a>; + + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + csiphy1_ep: endpoint { + data-lanes = <0 1 2 3>; + remote-endpoint = <&imx577_ep1>; + }; + }; + }; +}; + +&cci { + status = "okay"; +}; + +&cci_i2c1 { + #address-cells = <1>; + #size-cells = <0>; + + camera@1a { + compatible = "sony,imx577"; + reg = <0x1a>; + + reset-gpios = <&tlmm 29 GPIO_ACTIVE_LOW>; + pinctrl-0 = <&cam2_default>; + pinctrl-names = "default"; + + clocks = <&camcc CAM_CC_MCLK2_CLK>; + assigned-clocks = <&camcc CAM_CC_MCLK2_CLK>; + assigned-clock-rates = <24000000>; + + avdd-supply = <&vreg_s4a>; + + port { + imx577_ep1: endpoint { + link-frequencies = /bits/ 64 <600000000>; + data-lanes = <1 2 3 4>; + remote-endpoint = <&csiphy1_ep>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/talos-evk-camx.dtso b/arch/arm64/boot/dts/qcom/talos-evk-camx.dtso new file mode 100644 index 0000000000000..a40d8817a7a20 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-evk-camx.dtso @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/dts-v1/; +/plugin/; + +#include +#include +#include +#include +#include + +#include "talos-camera.dtsi" +#include "talos-camera-sensor.dtsi" diff --git a/arch/arm64/boot/dts/qcom/talos-evk-lvds-auo,g133han01.dtso b/arch/arm64/boot/dts/qcom/talos-evk-lvds-auo,g133han01.dtso new file mode 100644 index 0000000000000..fc2296e346b83 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-evk-lvds-auo,g133han01.dtso @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +/dts-v1/; +/plugin/; + +#include + +&{/} { + backlight: backlight { + compatible = "gpio-backlight"; + gpios = <&tlmm 59 GPIO_ACTIVE_HIGH>, + <&tlmm 115 GPIO_ACTIVE_HIGH>; + default-on; + }; + + panel-lvds { + compatible = "auo,g133han01"; + power-supply = <&vreg_v3p3>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* LVDS A (Odd pixels) */ + port@0 { + reg = <0>; + dual-lvds-odd-pixels; + + lvds_panel_out_a: endpoint { + remote-endpoint = <&sn65dsi84_out_a>; + }; + }; + + /* LVDS B (Even pixels) */ + port@1 { + reg = <1>; + dual-lvds-even-pixels; + + lvds_panel_out_b: endpoint { + remote-endpoint = <&sn65dsi84_out_b>; + }; + }; + }; + }; + + vreg_v3p3: regulator-v3p3 { + compatible = "regulator-fixed"; + regulator-name = "vdd-3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&hdmi_connector { + status = "disabled"; +}; + +&i2c1 { + clock-frequency = <400000>; + + status = "okay"; + + hdmi_bridge: bridge@3d { + status = "disabled"; + }; + + lvds_bridge: bridge@2c { + compatible = "ti,sn65dsi84"; + reg = <0x2c>; + enable-gpios = <&tlmm 42 GPIO_ACTIVE_HIGH>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sn65dsi84_in: endpoint { + data-lanes = <1 2 3 4>; + remote-endpoint = <&mdss_dsi0_out>; + }; + }; + + port@2 { + reg = <2>; + + sn65dsi84_out_a: endpoint { + data-lanes = <1 2 3 4>; + remote-endpoint = <&lvds_panel_out_a>; + }; + }; + + port@3 { + reg = <3>; + + sn65dsi84_out_b: endpoint { + data-lanes = <1 2 3 4>; + remote-endpoint = <&lvds_panel_out_b>; + }; + }; + }; + }; +}; + +&mdss_dsi0 { + vdda-supply = <&vreg_l11a>; + + status = "okay"; +}; + +&mdss_dsi0_out { + remote-endpoint = <&sn65dsi84_in>; + data-lanes = <0 1 2 3>; +}; + +&tlmm { + lcd_bklt_en: lcd-bklt-en-state { + pins = "gpio115"; + function = "gpio"; + bias-disable; + }; + + lcd_bklt_pwm: lcd-bklt-pwm-state { + pins = "gpio59"; + function = "gpio"; + bias-disable; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi new file mode 100644 index 0000000000000..23b9c646d6d9b --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +/dts-v1/; + +#include +#include +#include "talos.dtsi" +#include "pm8150.dtsi" +/ { + aliases { + mmc0 = &sdhc_1; + serial0 = &uart0; + serial1 = &uart7; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + clocks { + can_osc: can-oscillator { + compatible = "fixed-clock"; + clock-frequency = <20000000>; + #clock-cells = <0>; + }; + + sleep_clk: sleep-clk { + compatible = "fixed-clock"; + clock-frequency = <32764>; + #clock-cells = <0>; + }; + + xo_board_clk: xo-board-clk { + compatible = "fixed-clock"; + clock-frequency = <38400000>; + #clock-cells = <0>; + }; + }; + + regulator-usb2-vbus { + compatible = "regulator-fixed"; + regulator-name = "USB2_VBUS"; + gpio = <&pm8150_gpios 10 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&usb2_en>; + pinctrl-names = "default"; + enable-active-high; + regulator-always-on; + }; + + vreg_conn_1p8: regulator-conn-1p8 { + compatible = "regulator-fixed"; + regulator-name = "vreg_conn_1p8"; + startup-delay-us = <4000>; + enable-active-high; + gpio = <&pm8150_gpios 1 GPIO_ACTIVE_HIGH>; + }; + + vreg_conn_pa: regulator-conn-pa { + compatible = "regulator-fixed"; + regulator-name = "vreg_conn_pa"; + startup-delay-us = <4000>; + enable-active-high; + gpio = <&pm8150_gpios 6 GPIO_ACTIVE_HIGH>; + }; + + vreg_v3p3_can: regulator-v3p3-can { + compatible = "regulator-fixed"; + regulator-name = "vreg-v3p3-can"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + vreg_v5p0_can: regulator-v5p0-can { + compatible = "regulator-fixed"; + regulator-name = "vreg-v5p0-can"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-boot-on; + regulator-always-on; + }; + + wcn6855-pmu { + compatible = "qcom,wcn6855-pmu"; + + pinctrl-0 = <&bt_en_state>, <&wlan_en_state>; + pinctrl-names = "default"; + + bt-enable-gpios = <&tlmm 85 GPIO_ACTIVE_HIGH>; + wlan-enable-gpios = <&tlmm 84 GPIO_ACTIVE_HIGH>; + + vddio-supply = <&vreg_conn_pa>; + vddaon-supply = <&vreg_s5a>; + vddpmu-supply = <&vreg_conn_1p8>; + vddpmumx-supply = <&vreg_conn_1p8>; + vddpmucx-supply = <&vreg_conn_pa>; + vddrfa0p95-supply = <&vreg_s5a>; + vddrfa1p3-supply = <&vreg_s6a>; + vddrfa1p9-supply = <&vreg_l15a>; + vddpcie1p3-supply = <&vreg_s6a>; + vddpcie1p9-supply = <&vreg_l15a>; + + regulators { + vreg_pmu_rfa_cmn: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn"; + }; + + vreg_pmu_aon_0p59: ldo1 { + regulator-name = "vreg_pmu_aon_0p59"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p85: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p85"; + }; + + vreg_pmu_btcmx_0p85: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p85"; + }; + + vreg_pmu_rfa_0p8: ldo5 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo6 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p7: ldo7 { + regulator-name = "vreg_pmu_rfa_1p7"; + }; + + vreg_pmu_pcie_0p9: ldo8 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_pcie_1p8: ldo9 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + }; + }; + + wifi_1p8v: regulator-wifi-1p8v { + compatible = "regulator-fixed"; + regulator-name = "wifi_1p8v"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + gpio = <&tlmm 91 GPIO_ACTIVE_HIGH>; + enable-active-high; + pinctrl-0 = <&wifi_reg_en_pins_state>; + pinctrl-names = "default"; + regulator-boot-on; + regulator-always-on; + }; + + wifi_3p85v: regulator-wifi-3p85v { + compatible = "regulator-fixed"; + regulator-name = "wifi_3p85v"; + regulator-min-microvolt = <3850000>; + regulator-max-microvolt = <3850000>; + gpio = <&tlmm 91 GPIO_ACTIVE_HIGH>; + enable-active-high; + pinctrl-0 = <&wifi_reg_en_pins_state>; + pinctrl-names = "default"; + regulator-boot-on; + regulator-always-on; + }; +}; + +&apps_rsc { + regulators-0 { + compatible = "qcom,pm8150-rpmh-regulators"; + qcom,pmic-id = "a"; + + vreg_s3a: smps3 { + regulator-name = "vreg_s3a"; + regulator-min-microvolt = <600000>; + regulator-max-microvolt = <650000>; + regulator-initial-mode = ; + }; + + vreg_s4a: smps4 { + regulator-name = "vreg_s4a"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1829000>; + regulator-initial-mode = ; + }; + + vreg_s5a: smps5 { + regulator-name = "vreg_s5a"; + regulator-min-microvolt = <1896000>; + regulator-max-microvolt = <2040000>; + regulator-initial-mode = ; + }; + + vreg_s6a: smps6 { + regulator-name = "vreg_s6a"; + regulator-min-microvolt = <1304000>; + regulator-max-microvolt = <1404000>; + regulator-initial-mode = ; + }; + + vreg_l1a: ldo1 { + regulator-name = "vreg_l1a"; + regulator-min-microvolt = <488000>; + regulator-max-microvolt = <852000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l2a: ldo2 { + regulator-name = "vreg_l2a"; + regulator-min-microvolt = <1650000>; + regulator-max-microvolt = <3100000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l3a: ldo3 { + regulator-name = "vreg_l3a"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1248000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l5a: ldo5 { + regulator-name = "vreg_l5a"; + regulator-min-microvolt = <875000>; + regulator-max-microvolt = <975000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l7a: ldo7 { + regulator-name = "vreg_l7a"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1900000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l8a: ldo8 { + regulator-name = "vreg_l8a"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l10a: ldo10 { + regulator-name = "vreg_l10a"; + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l11a: ldo11 { + regulator-name = "vreg_l11a"; + regulator-min-microvolt = <1232000>; + regulator-max-microvolt = <1260000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l12a: ldo12 { + regulator-name = "vreg_l12a"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1890000>; + regulator-initial-mode = ; + }; + + vreg_l13a: ldo13 { + regulator-name = "vreg_l13a"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3230000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l15a: ldo15 { + regulator-name = "vreg_l15a"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1904000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l16a: ldo16 { + regulator-name = "vreg_l16a"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + regulator-allow-set-load; + regulator-allowed-modes = ; + }; + + vreg_l17a: ldo17 { + regulator-name = "vreg_l17a"; + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + }; + }; +}; + +&gpi_dma0 { + status = "okay"; +}; + +&gpi_dma1 { + status = "okay"; +}; + +ðernet { + pinctrl-0 = <ðernet_defaults>; + pinctrl-names = "default"; + + phy-handle = <&rgmii_phy>; + phy-mode = "rgmii-id"; + + snps,mtl-rx-config = <&mtl_rx_setup>; + snps,mtl-tx-config = <&mtl_tx_setup>; + + status = "okay"; + + mdio: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + rgmii_phy: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x1>; + + interrupts-extended = <&tlmm 121 IRQ_TYPE_EDGE_FALLING>; + device_type = "ethernet-phy"; + reset-gpios = <&tlmm 105 GPIO_ACTIVE_LOW>; + reset-assert-us = <11000>; + reset-deassert-us = <70000>; + }; + }; + + mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <1>; + snps,rx-sched-sp; + + queue0 { + snps,dcb-algorithm; + snps,map-to-dma-channel = <0x0>; + snps,route-up; + snps,priority = <0x1>; + }; + }; + + mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <1>; + snps,tx-sched-wrr; + + queue0 { + snps,weight = <0x10>; + snps,dcb-algorithm; + snps,priority = <0x0>; + }; + }; +}; + +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/qcs615/a612_zap.mbn"; +}; + +&i2c5 { + clock-frequency = <400000>; + status = "okay"; + + eeprom@57 { + compatible = "atmel,24c02"; + reg = <0x57>; + pagesize = <16>; + }; + + eeprom@5f { + compatible = "atmel,24mac602"; + reg = <0x5f>; + pagesize = <16>; + }; +}; + +&mdss { + status = "okay"; +}; + +&mdss_dp0 { + status = "okay"; +}; + +&mdss_dp0_out { + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000>; + remote-endpoint = <&dp0_connector_in>; +}; + +&mdss_dsi0 { + vdda-supply = <&vreg_l11a>; + status = "okay"; +}; + +&mdss_dsi0_phy { + vcca-supply = <&vreg_l5a>; + status = "okay"; +}; + +&pcie { + perst-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&pcie_default_state>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&pcie_phy { + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l12a>; + + status = "okay"; +}; + +&pcie_port0 { + wifi@0 { + compatible = "pci17cb,1103"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + qcom,calibration-variant = "QC_QCS615_Ride"; + + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + }; +}; + +&pm8150_gpios { + usb2_en: usb2-en-state { + pins = "gpio10"; + function = "normal"; + output-enable; + power-source = <0>; + }; +}; + +&qupv3_id_0 { + status = "okay"; +}; + +&qupv3_id_1 { + status = "okay"; +}; + +&remoteproc_adsp { + firmware-name = "qcom/qcs615/adsp.mbn"; + + status = "okay"; +}; + +&remoteproc_cdsp { + firmware-name = "qcom/qcs615/cdsp.mbn"; + + status = "okay"; +}; + +&sdhc_1 { + pinctrl-0 = <&sdc1_state_on>; + pinctrl-1 = <&sdc1_state_off>; + pinctrl-names = "default", "sleep"; + + bus-width = <8>; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + vmmc-supply = <&vreg_l17a>; + vqmmc-supply = <&vreg_s4a>; + + non-removable; + no-sd; + no-sdio; + + status = "okay"; +}; + +&spi6 { + status = "okay"; + + can@0 { + compatible = "microchip,mcp2515"; + reg = <0>; + clocks = <&can_osc>; + interrupts-extended = <&tlmm 87 IRQ_TYPE_LEVEL_LOW>; + spi-max-frequency = <10000000>; + vdd-supply = <&vreg_v3p3_can>; + xceiver-supply = <&vreg_v5p0_can>; + }; +}; + +&tlmm { + bt_en_state: bt-en-state { + pins = "gpio85"; + function = "gpio"; + bias-pull-down; + }; + + ethernet_defaults: ethernet-defaults-state { + mdc-pins { + pins = "gpio113"; + function = "rgmii"; + bias-pull-up; + }; + + mdio-pins { + pins = "gpio114"; + function = "rgmii"; + bias-pull-up; + }; + + rgmii-rx-pins { + pins = "gpio81", "gpio82", "gpio83", "gpio102", "gpio103", "gpio112"; + function = "rgmii"; + bias-disable; + drive-strength = <2>; + }; + + rgmii-tx-pins { + pins = "gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio97"; + function = "rgmii"; + bias-pull-up; + drive-strength = <16>; + }; + + phy-intr-pins { + pins = "gpio121"; + function = "gpio"; + bias-disable; + drive-strength = <8>; + }; + + phy-reset-pins { + pins = "gpio105"; + function = "gpio"; + bias-pull-up; + drive-strength = <16>; + }; + }; + + pcie_default_state: pcie-default-state { + clkreq-pins { + pins = "gpio90"; + function = "pcie_clk_req"; + drive-strength = <2>; + bias-pull-up; + }; + + perst-pins { + pins = "gpio89"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + + wake-pins { + pins = "gpio100"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + wifi_reg_en_pins_state: wifi-reg-en-pins-state { + pins = "gpio91"; + function = "gpio"; + drive-strength = <8>; + output-high; + bias-pull-up; + }; + + wlan_en_state: wlan-en-state { + pins = "gpio84"; + function = "gpio"; + drive-strength = <16>; + bias-pull-down; + output-low; + }; +}; + +&uart0 { + status = "okay"; +}; + +&uart7 { + status = "okay"; + + bluetooth { + compatible = "qcom,wcn6855-bt"; + firmware-name = "QCA6698/hpnv21", "QCA6698/hpbtfw21.tlv"; + + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + }; +}; + +&usb_1 { + dr_mode = "host"; + + status = "okay"; +}; + +&usb_hsphy_1 { + vdd-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l12a>; + vdda-phy-dpdm-supply = <&vreg_l13a>; + + status = "okay"; +}; + +&usb_2 { + dr_mode = "host"; + + status = "okay"; +}; + +&usb_hsphy_2 { + vdd-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l12a>; + vdda-phy-dpdm-supply = <&vreg_l13a>; + + status = "okay"; +}; + +&usb_qmpphy { + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l12a>; + + status = "okay"; +}; + +&usb_qmpphy_2 { + vdda-phy-supply = <&vreg_l11a>; + vdda-pll-supply = <&vreg_l5a>; + + status = "okay"; +}; + +&ufs_mem_hc { + reset-gpios = <&tlmm 123 GPIO_ACTIVE_LOW>; + vcc-supply = <&vreg_l17a>; + vcc-max-microamp = <600000>; + vccq2-supply = <&vreg_s4a>; + vccq2-max-microamp = <600000>; + + status = "okay"; +}; + +&ufs_mem_phy { + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l12a>; + + status = "okay"; +}; + +&venus { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/talos-evk.dts b/arch/arm64/boot/dts/qcom/talos-evk.dts new file mode 100644 index 0000000000000..b7f514fbc7b2d --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-evk.dts @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +/dts-v1/; + +#include "talos-evk-som.dtsi" +#include + +/ { + model = "Qualcomm QCS615 IQ 615 EVK"; + compatible = "qcom,talos-evk", "qcom,qcs615", "qcom,sm6150"; + chassis-type = "embedded"; + + aliases { + mmc1 = &sdhc_2; + }; + + dp0-connector { + compatible = "dp-connector"; + label = "DP0"; + type = "full-size"; + + hpd-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>; + + port { + dp0_connector_in: endpoint { + remote-endpoint = <&mdss_dp0_out>; + }; + }; + }; + + hdmi_connector: hdmi-out { + compatible = "hdmi-connector"; + type = "d"; + + port { + hdmi_con_out: endpoint { + remote-endpoint = <&adv7535_out>; + }; + }; + }; + + sound { + compatible = "qcom,qcs615-sndcard"; + model = "TALOS-EVK"; + + pinctrl-0 = <&mi2s1_pins>, <&mi2s_mclk>; + pinctrl-names = "default"; + + pri-mi2s-capture-dai-link { + link-name = "Primary MI2S Capture"; + + codec { + sound-dai = <&codec_da7212>; + }; + + cpu { + sound-dai = <&q6apmbedai PRIMARY_MI2S_TX>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + + pri-mi2s-playback-dai-link { + link-name = "Primary MI2S Playback"; + + codec { + sound-dai = <&codec_da7212>; + }; + + cpu { + sound-dai = <&q6apmbedai PRIMARY_MI2S_RX>; + }; + + platform { + sound-dai = <&q6apm>; + }; + }; + }; + + vreg_v1p8_out: regulator-v1p8-out { + compatible = "regulator-fixed"; + regulator-name = "vreg-v1p8-out"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vreg_v5p0_out>; + regulator-boot-on; + regulator-always-on; + }; + + vreg_v3p3_out: regulator-v3p3-out { + compatible = "regulator-fixed"; + regulator-name = "vreg-v3p3-out"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vreg_v5p0_out>; + regulator-boot-on; + regulator-always-on; + }; + + vreg_v5p0_out: regulator-v5p0-out { + compatible = "regulator-fixed"; + regulator-name = "vreg-v5p0-out"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-boot-on; + regulator-always-on; + /* Powered by system 20V rail (USBC_VBUS_IN) */ + }; +}; + +&i2c1 { + clock-frequency = <400000>; + status = "okay"; + + hdmi_bridge: bridge@3d { + compatible = "adi,adv7535"; + reg = <0x3d>; + avdd-supply = <&vreg_v1p8_out>; + dvdd-supply = <&vreg_v1p8_out>; + pvdd-supply = <&vreg_v1p8_out>; + a2vdd-supply = <&vreg_v1p8_out>; + v3p3-supply = <&vreg_v3p3_out>; + interrupts-extended = <&tlmm 26 IRQ_TYPE_LEVEL_LOW>; + adi,dsi-lanes = <4>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adv7535_in: endpoint { + remote-endpoint = <&mdss_dsi0_out>; + }; + }; + + port@1 { + reg = <1>; + + adv7535_out: endpoint { + remote-endpoint = <&hdmi_con_out>; + }; + }; + }; + }; +}; + +&i2c5 { + status = "okay"; + + codec_da7212: codec@1a { + compatible = "dlg,da7212"; + reg = <0x1a>; + #sound-dai-cells = <0>; + clocks = <&q6prmcc LPASS_CLK_ID_MCLK_2 LPASS_CLK_ATTRIBUTE_COUPLE_NO>; + clock-names = "mclk"; + VDDA-supply = <&vreg_v1p8_out>; + VDDIO-supply = <&vreg_v1p8_out>; + VDDMIC-supply = <&vreg_v3p3_out>; + }; +}; + +&mdss_dsi0_out { + remote-endpoint = <&adv7535_in>; + data-lanes = <0 1 2 3>; +}; + +&pon_pwrkey { + status = "okay"; +}; + +&pon_resin { + linux,code = ; + + status = "okay"; +}; + +&sdhc_2 { + pinctrl-0 = <&sdc2_state_on>; + pinctrl-1 = <&sdc2_state_off>; + pinctrl-names = "default", "sleep"; + + bus-width = <4>; + cd-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>; + + vmmc-supply = <&vreg_l10a>; + vqmmc-supply = <&vreg_s4a>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/talos-staging.dtso b/arch/arm64/boot/dts/qcom/talos-staging.dtso new file mode 100644 index 0000000000000..04602073418d4 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/talos-staging.dtso @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Talos staging overlay - add staging-specific device tree modifications here. + */ + +/dts-v1/; +/plugin/; + +&soc { + tgu@6b0c000 { + compatible = "qcom,tgu", "arm,primecell"; + reg = <0x0 0x6b0c000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/sm6150.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi similarity index 72% rename from arch/arm64/boot/dts/qcom/sm6150.dtsi rename to arch/arm64/boot/dts/qcom/talos.dtsi index 3d2a1cb02b628..dcbacbde54a3d 100644 --- a/arch/arm64/boot/dts/qcom/sm6150.dtsi +++ b/arch/arm64/boot/dts/qcom/talos.dtsi @@ -3,6 +3,7 @@ * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved. */ +#include #include #include #include @@ -11,11 +12,15 @@ #include #include #include +#include #include #include +#include #include #include +#include #include +#include / { interrupt-parent = <&intc>; @@ -39,6 +44,10 @@ clocks = <&cpufreq_hw 0>; qcom,freq-domain = <&cpufreq_hw 0>; #cooling-cells = <2>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_0: l2-cache { compatible = "cache"; @@ -60,6 +69,10 @@ next-level-cache = <&l2_100>; clocks = <&cpufreq_hw 0>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_100: l2-cache { compatible = "cache"; @@ -81,6 +94,10 @@ next-level-cache = <&l2_200>; clocks = <&cpufreq_hw 0>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_200: l2-cache { compatible = "cache"; @@ -102,6 +119,10 @@ next-level-cache = <&l2_300>; clocks = <&cpufreq_hw 0>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_300: l2-cache { compatible = "cache"; @@ -123,6 +144,10 @@ next-level-cache = <&l2_400>; clocks = <&cpufreq_hw 0>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_400: l2-cache { compatible = "cache"; @@ -144,6 +169,10 @@ next-level-cache = <&l2_500>; clocks = <&cpufreq_hw 0>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_500: l2-cache { compatible = "cache"; @@ -166,6 +195,10 @@ clocks = <&cpufreq_hw 1>; qcom,freq-domain = <&cpufreq_hw 1>; #cooling-cells = <2>; + operating-points-v2 = <&cpu6_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_600: l2-cache { compatible = "cache"; @@ -187,6 +220,10 @@ next-level-cache = <&l2_700>; clocks = <&cpufreq_hw 1>; qcom,freq-domain = <&cpufreq_hw 1>; + operating-points-v2 = <&cpu6_opp_table>; + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>, + <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>; l2_700: l2-cache { compatible = "cache"; @@ -239,6 +276,111 @@ }; }; + cpu0_opp_table: opp-table-cpu0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-peak-kBps = <(300000 * 4) (300000 * 16)>; + }; + + opp-576000000 { + opp-hz = /bits/ 64 <576000000>; + opp-peak-kBps = <(300000 * 4) (576000 * 16)>; + }; + + opp-748800000 { + opp-hz = /bits/ 64 <748800000>; + opp-peak-kBps = <(300000 * 4) (576000 * 16)>; + }; + + opp-998400000 { + opp-hz = /bits/ 64 <998400000>; + opp-peak-kBps = <(451000 * 4) (806400 * 16)>; + }; + + opp-1209600000 { + opp-hz = /bits/ 64 <1209600000>; + opp-peak-kBps = <(547000 * 4) (1017600 * 16)>; + }; + + opp-1363200000 { + opp-hz = /bits/ 64 <1363200000>; + opp-peak-kBps = <(768000 * 4) (1209600 * 16)>; + }; + + opp-1516800000 { + opp-hz = /bits/ 64 <1516800000>; + opp-peak-kBps = <(768000 * 4) (1209600 * 16)>; + }; + + opp-1593600000 { + opp-hz = /bits/ 64 <1593600000>; + opp-peak-kBps = <(1017000 * 4) (1363200 * 16)>; + }; + }; + + cpu6_opp_table: opp-table-cpu6 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-peak-kBps = <(451000 * 4) (300000 * 16)>; + }; + + opp-652800000 { + opp-hz = /bits/ 64 <652800000>; + opp-peak-kBps = <(451000 * 4) (576000 * 16)>; + }; + + opp-768000000 { + opp-hz = /bits/ 64 <768000000>; + opp-peak-kBps = <(451000 * 4) (576000 * 16)>; + }; + + opp-979200000 { + opp-hz = /bits/ 64 <979200000>; + opp-peak-kBps = <(547000 * 4) (806400 * 16)>; + }; + + opp-1017600000 { + opp-hz = /bits/ 64 <1017600000>; + opp-peak-kBps = <(547000 * 4) (806400 * 16)>; + }; + + opp-1094400000 { + opp-hz = /bits/ 64 <109440000>; + opp-peak-kBps = <(1017600 * 4) (940800 * 16)>; + }; + + opp-1209600000 { + opp-hz = /bits/ 64 <1209600000>; + opp-peak-kBps = <(1017600 * 4) (1017600 * 16)>; + }; + + opp-1363200000 { + opp-hz = /bits/ 64 <1363200000>; + opp-peak-kBps = <(1555000 * 4) (1209600 * 16)>; + }; + + opp-1516800000 { + opp-hz = /bits/ 64 <1516800000>; + opp-peak-kBps = <(1555000 * 4) (1209600 * 16)>; + }; + + opp-1708800000 { + opp-hz = /bits/ 64 <1708800000>; + opp-peak-kBps = <(1555000 * 4) (1363200 * 16)>; + }; + + opp-1900800000 { + opp-hz = /bits/ 64 <1900800000>; + opp-peak-kBps = <(1555000 * 4) (1363200 * 16)>; + }; + }; + dummy_eud: dummy-sink { compatible = "arm,coresight-dummy-sink"; @@ -355,7 +497,7 @@ smp2p-adsp { compatible = "qcom,smp2p"; qcom,smem = <443>, <429>; - interrupts = ; + interrupts = ; /* On this platform, bit 26 (normally SLPI) is repurposed for ADSP */ mboxes = <&apss_shared 26>; @@ -377,7 +519,7 @@ smp2p-cdsp { compatible = "qcom,smp2p"; qcom,smem = <94>, <432>; - interrupts = ; + interrupts = ; mboxes = <&apss_shared 6>; qcom,local-pid = <0>; @@ -396,9 +538,27 @@ }; + qspi_opp_table: opp-table-qspi { + compatible = "operating-points-v2"; + + opp-60000000 { + opp-hz = /bits/ 64 <60000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-133250000 { + opp-hz = /bits/ 64 <133250000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-266500000 { + opp-hz = /bits/ 64 <266500000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + qup_opp_table: opp-table-qup { compatible = "operating-points-v2"; - opp-shared; opp-75000000 { opp-hz = /bits/ 64 <75000000>; @@ -416,7 +576,17 @@ }; }; - psci { + pmu-a55 { + compatible = "arm,cortex-a55-pmu"; + interrupts = ; + }; + + pmu-a76 { + compatible = "arm,cortex-a76-pmu"; + interrupts = ; + }; + + psci: psci { compatible = "arm,psci-1.0"; method = "smc"; @@ -474,6 +644,11 @@ &cluster_sleep_1 &cluster_sleep_2>; }; + + reboot-mode { + mode-bootloader = <0x80010001 0x2>; + mode-edl = <0x80000000 0x1>; + }; }; reserved-memory { @@ -508,6 +683,16 @@ reg = <0x0 0x95900000 0x0 0x1e00000>; no-map; }; + + pil_gpu_mem: pil-gpu@97715000 { + reg = <0x0 0x97715000 0x0 0x2000>; + no-map; + }; + + pil_camera_mem: pil-camera-region@97717000 { + reg = <0x0 0x97717000 0x0 0x800000>; + no-map; + }; }; soc: soc@0 { @@ -517,6 +702,40 @@ #address-cells = <2>; #size-cells = <2>; + ethernet: ethernet@20000 { + compatible = "qcom,qcs615-ethqos", "qcom,qcs404-ethqos"; + reg = <0x0 0x00020000 0x0 0x10000>, + <0x0 0x00036000 0x0 0x100>; + reg-names = "stmmaceth", + "rgmii"; + + clocks = <&gcc GCC_EMAC_AXI_CLK>, + <&gcc GCC_EMAC_SLV_AHB_CLK>, + <&gcc GCC_EMAC_PTP_CLK>, + <&gcc GCC_EMAC_RGMII_CLK>; + clock-names = "stmmaceth", + "pclk", + "ptp_ref", + "rgmii"; + + interrupts = , + ; + interrupt-names = "macirq", + "eth_lpi"; + + power-domains = <&gcc EMAC_GDSC>; + resets = <&gcc GCC_EMAC_BCR>; + + iommus = <&apps_smmu 0x1c0 0x0>; + + snps,tso; + snps,pbl = <32>; + rx-fifo-depth = <16384>; + tx-fifo-depth = <20480>; + + status = "disabled"; + }; + gcc: clock-controller@100000 { compatible = "qcom,qcs615-gcc"; reg = <0 0x00100000 0 0x1f0000>; @@ -555,8 +774,8 @@ "cqhci", "ice"; - interrupts = , - ; + interrupts = , + ; interrupt-names = "hc_irq", "pwr_irq"; @@ -617,14 +836,14 @@ compatible = "qcom,qcs615-gpi-dma", "qcom,sdm845-gpi-dma"; reg = <0x0 0x800000 0x0 0x60000>; #dma-cells = <3>; - interrupts = , - , - , - , - , - , - , - ; + interrupts = , + , + , + , + , + , + , + ; dma-channels = <8>; dma-channel-mask = <0xf>; iommus = <&apps_smmu 0xd6 0x0>; @@ -651,7 +870,7 @@ clock-names = "se"; pinctrl-0 = <&qup_uart0_tx>, <&qup_uart0_rx>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; interconnects = <&aggre1_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS @@ -668,7 +887,7 @@ reg = <0x0 0x884000 0x0 0x4000>; #address-cells = <1>; #size-cells = <0>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; clock-names = "se"; pinctrl-0 = <&qup_i2c1_data_clk>; @@ -696,7 +915,7 @@ reg = <0x0 0x888000 0x0 0x4000>; #address-cells = <1>; #size-cells = <0>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; pinctrl-0 = <&qup_i2c2_data_clk>; @@ -722,7 +941,7 @@ spi2: spi@888000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00888000 0x0 0x4000>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; pinctrl-0 = <&qup_spi2_data_clk>, <&qup_spi2_cs>; @@ -747,7 +966,7 @@ uart2: serial@888000 { compatible = "qcom,geni-uart"; reg = <0x0 0x00888000 0x0 0x4000>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; pinctrl-0 = <&qup_uart2_cts>, <&qup_uart2_rts>, @@ -769,7 +988,7 @@ reg = <0x0 0x88c000 0x0 0x4000>; #address-cells = <1>; #size-cells = <0>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; clock-names = "se"; pinctrl-0 = <&qup_i2c3_data_clk>; @@ -797,14 +1016,14 @@ compatible = "qcom,qcs615-gpi-dma", "qcom,sdm845-gpi-dma"; reg = <0x0 0xa00000 0x0 0x60000>; #dma-cells = <3>; - interrupts = , - , - , - , - , - , - , - ; + interrupts = , + , + , + , + , + , + , + ; dma-channels = <8>; dma-channel-mask = <0xf>; iommus = <&apps_smmu 0x376 0x0>; @@ -831,7 +1050,7 @@ clock-names = "se"; pinctrl-0 = <&qup_i2c4_data_clk>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; #address-cells = <1>; #size-cells = <0>; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS @@ -859,7 +1078,7 @@ clock-names = "se"; pinctrl-0 = <&qup_spi4_data_clk>, <&qup_spi4_cs>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; #address-cells = <1>; #size-cells = <0>; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS @@ -885,7 +1104,7 @@ pinctrl-0 = <&qup_uart4_cts>, <&qup_uart4_rts>, <&qup_uart4_tx>, <&qup_uart4_rx>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS @@ -904,7 +1123,7 @@ clock-names = "se"; pinctrl-0 = <&qup_i2c5_data_clk>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; #address-cells = <1>; #size-cells = <0>; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS @@ -932,7 +1151,7 @@ clock-names = "se"; pinctrl-0 = <&qup_i2c6_data_clk>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; #address-cells = <1>; #size-cells = <0>; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS @@ -960,7 +1179,7 @@ clock-names = "se"; pinctrl-0 = <&qup_spi6_data_clk>, <&qup_spi6_cs>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; #address-cells = <1>; #size-cells = <0>; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS @@ -986,7 +1205,7 @@ pinctrl-0 = <&qup_uart6_cts>, <&qup_uart6_rts>, <&qup_uart6_tx>, <&qup_uart6_rx>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS @@ -1005,7 +1224,7 @@ clock-names = "se"; pinctrl-0 = <&qup_i2c7_data_clk>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; #address-cells = <1>; #size-cells = <0>; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS @@ -1033,7 +1252,7 @@ clock-names = "se"; pinctrl-0 = <&qup_spi7_data_clk>, <&qup_spi7_cs>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; #address-cells = <1>; #size-cells = <0>; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS @@ -1059,7 +1278,7 @@ pinctrl-0 = <&qup_uart7_cts>, <&qup_uart7_rts>, <&qup_uart7_tx>, <&qup_uart7_rx>; pinctrl-names = "default"; - interrupts = ; + interrupts = ; interconnects = <&aggre1_noc MASTER_BLSP_1 QCOM_ICC_TAG_ALWAYS &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS @@ -1126,15 +1345,15 @@ linux,pci-domain = <0>; num-lanes = <1>; - interrupts = , - , - , - , - , - , - , - , - ; + interrupts = , + , + , + , + , + , + , + , + ; interrupt-names = "msi0", "msi1", "msi2", @@ -1147,10 +1366,10 @@ #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map = <0 0 0 1 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH 0>, + <0 0 0 2 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH 0>, + <0 0 0 3 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH 0>, + <0 0 0 4 &intc GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH 0>; clocks = <&gcc GCC_PCIE_0_PIPE_CLK>, <&gcc GCC_PCIE_0_AUX_CLK>, @@ -1254,16 +1473,16 @@ reg-names = "std", "ice"; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_UFS_PHY_AXI_CLK>, <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, <&gcc GCC_UFS_PHY_AHB_CLK>, <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>, - <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, <&rpmhcc RPMH_CXO_CLK>, <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>, - <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>; + <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>, + <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; clock-names = "core_clk", "bus_aggr_clk", "iface_clk", @@ -1363,7 +1582,7 @@ cryptobam: dma-controller@1dc4000 { compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0"; reg = <0x0 0x01dc4000 0x0 0x24000>; - interrupts = ; + interrupts = ; #dma-cells = <1>; qcom,ee = <0>; qcom,controlled-remotely; @@ -1402,7 +1621,7 @@ reg-names = "east", "west", "south"; - interrupts = ; + interrupts = ; gpio-ranges = <&tlmm 0 0 124>; gpio-controller; #gpio-cells = <2>; @@ -1410,6 +1629,92 @@ #interrupt-cells = <2>; wakeup-parent = <&pdc>; + cam0_default: cam0-default-state { + pins = "gpio28"; + function = "cam_mclk"; + drive-strength = <2>; + bias-disable; + }; + + cam1_default: cam1-default-state { + pins = "gpio29"; + function = "cam_mclk"; + drive-strength = <2>; + bias-disable; + }; + + cam2_default: cam2-default-state { + pins = "gpio30"; + function = "cam_mclk"; + drive-strength = <2>; + bias-disable; + }; + + cam3_default: cam3-default-state { + pins = "gpio31"; + function = "cam_mclk"; + drive-strength = <2>; + bias-disable; + }; + + cci_i2c0_default: cci-i2c0-default-state { + /* SDA, SCL */ + pins = "gpio32", "gpio33"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-up; + }; + + cci_i2c1_default: cci-i2c1-default-state { + /* SDA, SCL */ + pins = "gpio34", "gpio35"; + function = "cci_i2c"; + drive-strength = <2>; + bias-pull-up; + }; + + mi2s1_pins: mi2s1-state { + pins = "gpio108", "gpio109", "gpio110", "gpio111"; + function = "mi2s_1"; + drive-strength = <8>; + bias-disable; + }; + + mi2s_mclk: mi2s-mclk-state { + pins = "gpio122"; + function = "mclk2"; + drive-strength = <8>; + bias-disable; + }; + + qspi_cs0: qspi-cs0-state { + pins = "gpio44"; + function = "qspi"; + bias-disable; + drive-strength = <6>; + }; + + qspi_data0123: qspi-data0123-state { + pins = "gpio45", "gpio46", "gpio47", "gpio49"; + function = "qspi"; + bias-pull-down; + drive-strength = <6>; + }; + + qspi_clk: qspi-clk-state { + pins = "gpio48"; + function = "qspi"; + bias-pull-down; + drive-strength = <6>; + }; + + qspi_cs1: qspi-cs1-state { + pins = "gpio50"; + function = "qspi"; + bias-pull-down; + drive-strength = <6>; + }; + qup_i2c1_data_clk: qup-i2c1-data-clk-state { pins = "gpio4", "gpio5"; function = "qup0"; @@ -1687,6 +1992,111 @@ }; }; + gpu: gpu@5000000 { + compatible = "qcom,adreno-612.0", "qcom,adreno"; + reg = <0x0 0x05000000 0x0 0x40000>, + <0x0 0x0509e000 0x0 0x1000>, + <0x0 0x05061000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; + + clocks = <&gpucc GPU_CC_GX_GFX3D_CLK>; + clock-names = "core"; + + interrupts = ; + + interconnects = <&gem_noc MASTER_GFX3D QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "gfx-mem"; + + iommus = <&adreno_smmu 0x0 0x401>; + + operating-points-v2 = <&gpu_opp_table>; + power-domains = <&rpmhpd RPMHPD_CX>; + + qcom,gmu = <&gmu>; + + #cooling-cells = <2>; + + status = "disabled"; + + gpu_zap_shader: zap-shader { + memory-region = <&pil_gpu_mem>; + }; + + gpu_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-845000000 { + opp-hz = /bits/ 64 <845000000>; + required-opps = <&rpmhpd_opp_turbo>; + opp-peak-kBps = <7050000>; + }; + + opp-745000000 { + opp-hz = /bits/ 64 <745000000>; + required-opps = <&rpmhpd_opp_nom_l1>; + opp-peak-kBps = <6075000>; + }; + + opp-650000000 { + opp-hz = /bits/ 64 <650000000>; + required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <5287500>; + }; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + opp-peak-kBps = <3975000>; + }; + + opp-435000000 { + opp-hz = /bits/ 64 <435000000>; + required-opps = <&rpmhpd_opp_svs>; + opp-peak-kBps = <3000000>; + }; + }; + }; + + gmu: gmu@506a000 { + compatible = "qcom,adreno-rgmu-612.0", "qcom,adreno-rgmu"; + reg = <0x0 0x0506d000 0x0 0x2c000>; + + clocks = <&gpucc GPU_CC_CX_GMU_CLK>, + <&gpucc GPU_CC_CXO_CLK>, + <&gcc GCC_DDRSS_GPU_AXI_CLK>, + <&gcc GCC_GPU_MEMNOC_GFX_CLK>, + <&gpucc GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK>; + clock-names = "gmu", + "cxo", + "axi", + "memnoc", + "smmu_vote"; + + power-domains = <&gpucc CX_GDSC>, + <&gpucc GX_GDSC>; + power-domain-names = "cx", + "gx"; + + interrupts = , + ; + interrupt-names = "oob", + "gmu"; + + operating-points-v2 = <&gmu_opp_table>; + + gmu_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + }; + gpucc: clock-controller@5090000 { compatible = "qcom,qcs615-gpucc"; reg = <0 0x05090000 0 0x9000>; @@ -1700,6 +2110,31 @@ #power-domain-cells = <1>; }; + adreno_smmu: iommu@50a0000 { + compatible = "qcom,qcs615-smmu-500", "qcom,adreno-smmu", + "qcom,smmu-500", "arm,mmu-500"; + reg = <0x0 0x050a0000 0x0 0x40000>; + #iommu-cells = <2>; + #global-interrupts = <1>; + interrupts = , + , + , + , + , + , + , + , + ; + clocks = <&gcc GCC_GPU_MEMNOC_GFX_CLK>, + <&gpucc GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK>, + <&gcc GCC_GPU_SNOC_DVM_GFX_CLK>; + clock-names = "mem", + "hlos", + "iface"; + power-domains = <&gpucc CX_GDSC>; + dma-coherent; + }; + stm@6002000 { compatible = "arm,coresight-stm", "arm,primecell"; reg = <0x0 0x06002000 0x0 0x1000>, @@ -2110,6 +2545,14 @@ #address-cells = <1>; #size-cells = <0>; + port@0 { + reg = <0>; + + replicator0_out0: endpoint { + remote-endpoint = <&tmc_etr_in>; + }; + }; + port@1 { reg = <1>; @@ -2144,6 +2587,25 @@ }; }; + tmc_etr: tmc@6048000 { + compatible = "arm,coresight-tmc", "arm,primecell"; + reg = <0x0 0x06048000 0x0 0x1000>; + + clocks = <&aoss_qmp>; + clock-names = "apb_pclk"; + + iommus = <&apps_smmu 0x01e0 0x0>; + arm,scatter-gather; + + in-ports { + port { + tmc_etr_in: endpoint { + remote-endpoint = <&replicator0_out0>; + }; + }; + }; + }; + replicator@604a000 { compatible = "arm,coresight-dynamic-replicator", "arm,primecell"; reg = <0x0 0x0604a000 0x0 0x1000>; @@ -3329,7 +3791,7 @@ compatible = "qcom,qcs615-cdsp-pas", "qcom,sm8150-cdsp-pas"; reg = <0x0 0x08300000 0x0 0x4040>; - interrupts-extended = <&intc GIC_SPI 578 IRQ_TYPE_EDGE_RISING>, + interrupts-extended = <&intc GIC_SPI 578 IRQ_TYPE_EDGE_RISING 0>, <&cdsp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>, <&cdsp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>, <&cdsp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>, @@ -3356,7 +3818,7 @@ status = "disabled"; glink-edge { - interrupts = ; + interrupts = ; mboxes = <&apss_shared 4>; label = "cdsp"; qcom,remote-pid = <5>; @@ -3411,12 +3873,20 @@ }; }; }; + + cooling { + compatible = "qcom,qmi-cooling-cdsp"; + cdsp_sw: cdsp_sw { + label = "cdsp_sw"; + #cooling-cells = <2>; + }; + }; }; pmu@90b6300 { compatible = "qcom,qcs615-cpu-bwmon", "qcom,sdm845-bwmon"; reg = <0x0 0x090b6300 0x0 0x600>; - interrupts = ; + interrupts = ; interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>; @@ -3438,7 +3908,7 @@ pmu@90cd000 { compatible = "qcom,qcs615-llcc-bwmon", "qcom,sc7280-llcc-bwmon"; reg = <0x0 0x090cd000 0x0 0x1000>; - interrupts = ; + interrupts = ; interconnects = <&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>; @@ -3490,8 +3960,8 @@ reg = <0x0 0x08804000 0x0 0x1000>; reg-names = "hc"; - interrupts = , - ; + interrupts = , + ; interrupt-names = "hc_irq", "pwr_irq"; @@ -3539,6 +4009,39 @@ }; }; + qspi: spi@88df000 { + compatible = "qcom,qcs615-qspi", + "qcom,qspi-v1"; + reg = <0x0 0x088df000 0x0 0x1000>; + + interrupts = ; + + clocks = <&gcc GCC_QSPI_CNOC_PERIPH_AHB_CLK>, + <&gcc GCC_QSPI_CORE_CLK>; + clock-names = "iface", + "core"; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS + &config_noc SLAVE_QSPI QCOM_ICC_TAG_ALWAYS>, + <&aggre1_noc MASTER_QSPI QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "qspi-config", + "qspi-memory"; + + power-domains = <&rpmhpd RPMHPD_CX>; + operating-points-v2 = <&qspi_opp_table>; + + iommus = <&apps_smmu 0x160 0x0>; + + pinctrl-0 = <&qspi_clk>, <&qspi_cs0>, <&qspi_data0123>; + pinctrl-names = "default"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + }; + dc_noc: interconnect@9160000 { reg = <0x0 0x09160000 0x0 0x3200>; compatible = "qcom,qcs615-dc-noc"; @@ -3564,7 +4067,7 @@ venus: video-codec@aa00000 { compatible = "qcom,qcs615-venus", "qcom,sc7180-venus"; reg = <0x0 0x0aa00000 0x0 0x100000>; - interrupts = ; + interrupts = ; clocks = <&videocc VIDEO_CC_VENUS_CTL_CORE_CLK>, <&videocc VIDEO_CC_VENUS_AHB_CLK>, @@ -3646,6 +4149,241 @@ #power-domain-cells = <1>; }; + cci: cci@ac4a000 { + compatible = "qcom,sm6150-cci", "qcom,msm8996-cci"; + + reg = <0x0 0x0ac4a000 0x0 0x4000>; + interrupts = ; + power-domains = <&camcc TITAN_TOP_GDSC>; + clocks = <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CCI_CLK>; + clock-names = "camnoc_axi", + "cpas_ahb", + "cci"; + pinctrl-0 = <&cci_i2c0_default &cci_i2c1_default>; + pinctrl-names = "default"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + + cci_i2c0: i2c-bus@0 { + reg = <0>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + + cci_i2c1: i2c-bus@1 { + reg = <1>; + clock-frequency = <1000000>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + camss: isp@acb3000 { + compatible = "qcom,sm6150-camss"; + + reg = <0x0 0x0acb3000 0x0 0x1000>, + <0x0 0x0acba000 0x0 0x1000>, + <0x0 0x0acc8000 0x0 0x1000>, + <0x0 0x0ac65000 0x0 0x1000>, + <0x0 0x0ac66000 0x0 0x1000>, + <0x0 0x0ac67000 0x0 0x1000>, + <0x0 0x0acaf000 0x0 0x4000>, + <0x0 0x0acb6000 0x0 0x4000>, + <0x0 0x0acc4000 0x0 0x4000>, + <0x0 0x0ac6f000 0x0 0x3000>, + <0x0 0x0ac42000 0x0 0x5000>, + <0x0 0x0ac48000 0x0 0x1000>, + <0x0 0x0ac40000 0x0 0x1000>, + <0x0 0x0ac18000 0x0 0x3000>, + <0x0 0x0ac00000 0x0 0x6000>, + <0x0 0x0ac10000 0x0 0x8000>, + <0x0 0x0ac87000 0x0 0x3000>, + <0x0 0x0ac52000 0x0 0x4000>, + <0x0 0x0ac4e000 0x0 0x4000>, + <0x0 0x0ac6b000 0x0 0x0a00>; + reg-names = "csid0", + "csid1", + "csid_lite", + "csiphy0", + "csiphy1", + "csiphy2", + "vfe0", + "vfe1", + "vfe_lite", + "bps", + "camnoc", + "cpas_cdm", + "cpas_top", + "icp_csr", + "icp_qgic", + "icp_sierra", + "ipe0", + "jpeg_dma", + "jpeg_enc", + "lrme"; + + clocks = <&gcc GCC_CAMERA_AHB_CLK>, + <&gcc GCC_CAMERA_HF_AXI_CLK>, + <&camcc CAM_CC_CAMNOC_AXI_CLK>, + <&camcc CAM_CC_CPAS_AHB_CLK>, + <&camcc CAM_CC_CSIPHY0_CLK>, + <&camcc CAM_CC_CSI0PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY1_CLK>, + <&camcc CAM_CC_CSI1PHYTIMER_CLK>, + <&camcc CAM_CC_CSIPHY2_CLK>, + <&camcc CAM_CC_CSI2PHYTIMER_CLK>, + <&camcc CAM_CC_SOC_AHB_CLK>, + <&camcc CAM_CC_IFE_0_CLK>, + <&camcc CAM_CC_IFE_0_AXI_CLK>, + <&camcc CAM_CC_IFE_0_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_0_CSID_CLK>, + <&camcc CAM_CC_IFE_1_CLK>, + <&camcc CAM_CC_IFE_1_AXI_CLK>, + <&camcc CAM_CC_IFE_1_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_1_CSID_CLK>, + <&camcc CAM_CC_IFE_LITE_CLK>, + <&camcc CAM_CC_IFE_LITE_CPHY_RX_CLK>, + <&camcc CAM_CC_IFE_LITE_CSID_CLK>, + <&camcc CAM_CC_BPS_CLK>, + <&camcc CAM_CC_BPS_AHB_CLK>, + <&camcc CAM_CC_BPS_AXI_CLK>, + <&camcc CAM_CC_BPS_AREG_CLK>, + <&camcc CAM_CC_ICP_CLK>, + <&camcc CAM_CC_IPE_0_CLK>, + <&camcc CAM_CC_IPE_0_AHB_CLK>, + <&camcc CAM_CC_IPE_0_AREG_CLK>, + <&camcc CAM_CC_IPE_0_AXI_CLK>, + <&camcc CAM_CC_JPEG_CLK>, + <&camcc CAM_CC_LRME_CLK>; + clock-names = "gcc_ahb", + "gcc_axi_hf", + "camnoc_axi", + "cpas_ahb", + "csiphy0", + "csiphy0_timer", + "csiphy1", + "csiphy1_timer", + "csiphy2", + "csiphy2_timer", + "soc_ahb", + "vfe0", + "vfe0_axi", + "vfe0_cphy_rx", + "vfe0_csid", + "vfe1", + "vfe1_axi", + "vfe1_cphy_rx", + "vfe1_csid", + "vfe_lite", + "vfe_lite_cphy_rx", + "vfe_lite_csid", + "bps", + "bps_ahb", + "bps_axi", + "bps_areg", + "icp", + "ipe0", + "ipe0_ahb", + "ipe0_areg", + "ipe0_axi", + "jpeg", + "lrme"; + + interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>, + <&mmss_noc MASTER_CAMNOC_HF0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_HF1 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "ahb", + "hf_0", + "hf_1", + "sf_mnoc"; + + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "csid0", + "csid1", + "csid_lite", + "csiphy0", + "csiphy1", + "csiphy2", + "vfe0", + "vfe1", + "vfe_lite", + "camnoc", + "cdm", + "icp", + "jpeg_dma", + "jpeg_enc", + "lrme"; + + iommus = <&apps_smmu 0x0820 0x40>, + <&apps_smmu 0x0840 0x00>, + <&apps_smmu 0x0860 0x40>, + <&apps_smmu 0x0c00 0x00>, + <&apps_smmu 0x0cc0 0x00>, + <&apps_smmu 0x0c80 0x00>, + <&apps_smmu 0x0ca0 0x00>, + <&apps_smmu 0x0d00 0x00>, + <&apps_smmu 0x0d20 0x00>, + <&apps_smmu 0x0d40 0x00>, + <&apps_smmu 0x0d80 0x20>, + <&apps_smmu 0x0da0 0x20>, + <&apps_smmu 0x0de2 0x00>; + + power-domains = <&camcc IFE_0_GDSC>, + <&camcc IFE_1_GDSC>, + <&camcc TITAN_TOP_GDSC>, + <&camcc BPS_GDSC>, + <&camcc IPE_0_GDSC>; + power-domain-names = "ife0", + "ife1", + "top", + "bps", + "ipe"; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + }; + + port@2 { + reg = <2>; + }; + }; + }; + camcc: clock-controller@ad00000 { compatible = "qcom,qcs615-camcc"; reg = <0 0x0ad00000 0 0x10000>; @@ -3657,17 +4395,278 @@ #power-domain-cells = <1>; }; + mdss: display-subsystem@ae00000 { + compatible = "qcom,sm6150-mdss"; + reg = <0x0 0x0ae00000 0x0 0x1000>; + reg-names = "mdss"; + + interconnects = <&mmss_noc MASTER_MDP0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "mdp0-mem", + "cpu-cfg"; + + power-domains = <&dispcc MDSS_CORE_GDSC>; + + clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc DISP_CC_MDSS_MDP_CLK>; + + interrupts = ; + interrupt-controller; + #interrupt-cells = <1>; + + iommus = <&apps_smmu 0x800 0x0>; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + status = "disabled"; + + mdss_mdp: display-controller@ae01000 { + compatible = "qcom,sm6150-dpu"; + reg = <0x0 0x0ae01000 0x0 0x8f000>, + <0x0 0x0aeb0000 0x0 0x2008>; + reg-names = "mdp", + "vbif"; + + clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP_HF_AXI_CLK>, + <&dispcc DISP_CC_MDSS_MDP_CLK>, + <&dispcc DISP_CC_MDSS_VSYNC_CLK>; + clock-names = "iface", + "bus", + "core", + "vsync"; + + operating-points-v2 = <&mdp_opp_table>; + power-domains = <&rpmhpd RPMHPD_CX>; + + interrupts-extended = <&mdss 0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dpu_intf0_out: endpoint { + remote-endpoint = <&mdss_dp0_in>; + }; + }; + + port@1 { + reg = <1>; + + dpu_intf1_out: endpoint { + remote-endpoint = <&mdss_dsi0_in>; + }; + }; + }; + + mdp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-192000000 { + opp-hz = /bits/ 64 <192000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-256000000 { + opp-hz = /bits/ 64 <256000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-307200000 { + opp-hz = /bits/ 64 <307200000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + }; + + mdss_dp0: displayport-controller@ae90000 { + compatible = "qcom,sm6150-dp", "qcom,sm8150-dp", "qcom,sm8350-dp"; + + reg = <0x0 0x0ae90000 0x0 0x200>, + <0x0 0x0ae90200 0x0 0x200>, + <0x0 0x0ae90400 0x0 0x600>, + <0x0 0x0ae90a00 0x0 0x600>, + <0x0 0x0ae91000 0x0 0x600>; + + interrupt-parent = <&mdss>; + interrupts = <12>; + + clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&dispcc DISP_CC_MDSS_DP_AUX_CLK>, + <&dispcc DISP_CC_MDSS_DP_LINK_CLK>, + <&dispcc DISP_CC_MDSS_DP_LINK_INTF_CLK>, + <&dispcc DISP_CC_MDSS_DP_PIXEL_CLK>, + <&dispcc DISP_CC_MDSS_DP_PIXEL1_CLK>; + clock-names = "core_iface", + "core_aux", + "ctrl_link", + "ctrl_link_iface", + "stream_pixel", + "stream_1_pixel"; + + assigned-clocks = <&dispcc DISP_CC_MDSS_DP_LINK_CLK_SRC>, + <&dispcc DISP_CC_MDSS_DP_PIXEL_CLK_SRC>, + <&dispcc DISP_CC_MDSS_DP_PIXEL1_CLK_SRC>; + assigned-clock-parents = <&usb_qmpphy_2 QMP_USB43DP_DP_LINK_CLK>, + <&usb_qmpphy_2 QMP_USB43DP_DP_VCO_DIV_CLK>, + <&usb_qmpphy_2 QMP_USB43DP_DP_VCO_DIV_CLK>; + + phys = <&usb_qmpphy_2 QMP_USB43DP_DP_PHY>; + phy-names = "dp"; + + operating-points-v2 = <&dp_opp_table>; + power-domains = <&rpmhpd RPMHPD_CX>; + + #sound-dai-cells = <0>; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mdss_dp0_in: endpoint { + remote-endpoint = <&dpu_intf0_out>; + }; + }; + + port@1 { + reg = <1>; + + mdss_dp0_out: endpoint { + data-lanes = <3 2 0 1>; + }; + }; + }; + + dp_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-160000000 { + opp-hz = /bits/ 64 <160000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-270000000 { + opp-hz = /bits/ 64 <270000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + + mdss_dsi0: dsi@ae94000 { + compatible = "qcom,sm6150-dsi-ctrl", "qcom,mdss-dsi-ctrl"; + reg = <0x0 0x0ae94000 0x0 0x400>; + reg-names = "dsi_ctrl"; + + interrupts-extended = <&mdss 4>; + + clocks = <&dispcc DISP_CC_MDSS_BYTE0_CLK>, + <&dispcc DISP_CC_MDSS_BYTE0_INTF_CLK>, + <&dispcc DISP_CC_MDSS_PCLK0_CLK>, + <&dispcc DISP_CC_MDSS_ESC0_CLK>, + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_DISP_HF_AXI_CLK>; + clock-names = "byte", + "byte_intf", + "pixel", + "core", + "iface", + "bus"; + + assigned-clocks = <&dispcc DISP_CC_MDSS_BYTE0_CLK_SRC>, + <&dispcc DISP_CC_MDSS_PCLK0_CLK_SRC>; + assigned-clock-parents = <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>, + <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>; + + operating-points-v2 = <&dsi0_opp_table>; + power-domains = <&rpmhpd RPMHPD_CX>; + + phys = <&mdss_dsi0_phy>; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + + dsi0_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-164000000 { + opp-hz = /bits/ 64 <164000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mdss_dsi0_in: endpoint { + remote-endpoint = <&dpu_intf1_out>; + }; + }; + + port@1 { + reg = <1>; + + mdss_dsi0_out: endpoint { + }; + }; + }; + }; + + mdss_dsi0_phy: phy@ae94400 { + compatible = "qcom,sm6150-dsi-phy-14nm"; + reg = <0x0 0x0ae94400 0x0 0x100>, + <0x0 0x0ae94500 0x0 0x300>, + <0x0 0x0ae94800 0x0 0x124>; + reg-names = "dsi_phy", + "dsi_phy_lane", + "dsi_pll"; + + #clock-cells = <1>; + #phy-cells = <0>; + + clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&rpmhcc RPMH_CXO_CLK>; + clock-names = "iface", + "ref"; + + status = "disabled"; + }; + }; + dispcc: clock-controller@af00000 { compatible = "qcom,qcs615-dispcc"; reg = <0 0x0af00000 0 0x20000>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&gcc GCC_DISP_GPLL0_DIV_CLK_SRC>, + <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>, + <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>, <0>, - <0>, - <0>, - <0>, - <0>; + <&usb_qmpphy_2 QMP_USB43DP_DP_LINK_CLK>, + <&usb_qmpphy_2 QMP_USB43DP_DP_VCO_DIV_CLK>; #clock-cells = <1>; #reset-cells = <1>; @@ -3687,7 +4686,7 @@ aoss_qmp: power-management@c300000 { compatible = "qcom,qcs615-aoss-qmp", "qcom,aoss-qmp"; reg = <0x0 0x0c300000 0x0 0x400>; - interrupts = ; + interrupts = ; mboxes = <&apss_shared 0>; #clock-cells = <0>; @@ -3719,71 +4718,71 @@ #global-interrupts = <1>; dma-coherent; - interrupts = , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - ; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; }; spmi_bus: spmi@c440000 { @@ -3812,12 +4811,22 @@ compatible = "arm,gic-v3"; reg = <0x0 0x17a00000 0x0 0x10000>, /* GICD */ <0x0 0x17a60000 0x0 0x100000>; /* GICR * 8 */ - interrupts = ; + interrupts = ; #address-cells = <0>; - #interrupt-cells = <3>; + #interrupt-cells = <4>; interrupt-controller; #redistributor-regions = <1>; redistributor-stride = <0x0 0x20000>; + + ppi-partitions { + ppi_cluster0: interrupt-partition-0 { + affinity = <&cpu0 &cpu1 &cpu2 &cpu3 &cpu4 &cpu5>; + }; + + ppi_cluster1: interrupt-partition-1 { + affinity = <&cpu6 &cpu7>; + }; + }; }; apss_shared: mailbox@17c00000 { @@ -3830,7 +4839,7 @@ watchdog: watchdog@17c10000 { compatible = "qcom,apss-wdt-qcs615", "qcom,kpss-wdt"; reg = <0x0 0x17c10000 0x0 0x1000>; - interrupts = ; + interrupts = ; clocks = <&sleep_clk>; }; @@ -3845,49 +4854,49 @@ reg = <0x17c21000 0x1000>, <0x17c22000 0x1000>; frame-number = <0>; - interrupts = , - ; + interrupts = , + ; }; frame@17c23000 { reg = <0x17c23000 0x1000>; frame-number = <1>; - interrupts = ; + interrupts = ; status = "disabled"; }; frame@17c25000 { reg = <0x17c25000 0x1000>; frame-number = <2>; - interrupts = ; + interrupts = ; status = "disabled"; }; frame@17c27000 { reg = <0x17c27000 0x1000>; frame-number = <3>; - interrupts = ; + interrupts = ; status = "disabled"; }; frame@17c29000 { reg = <0x17c29000 0x1000>; frame-number = <4>; - interrupts = ; + interrupts = ; status = "disabled"; }; frame@17c2b000 { reg = <0x17c2b000 0x1000>; frame-number = <5>; - interrupts = ; + interrupts = ; status = "disabled"; }; frame@17c2d000 { reg = <0x17c2d000 0x1000>; frame-number = <6>; - interrupts = ; + interrupts = ; status = "disabled"; }; }; @@ -3901,9 +4910,9 @@ "drv-1", "drv-2"; - interrupts = , - , - ; + interrupts = , + , + ; qcom,drv-id = <2>; qcom,tcs-offset = <0xd00>; @@ -3978,7 +4987,17 @@ }; }; - usb_1_hsphy: phy@88e2000 { + osm_l3: interconnect@18321000 { + compatible = "qcom,qcs615-osm-l3", "qcom,sm8150-osm-l3", "qcom,osm-l3"; + reg = <0x0 0x18321000 0x0 0x1400>; + + clocks = <&rpmhcc RPMH_CXO_CLK>, <&gcc GPLL0>; + clock-names = "xo", "alternate"; + + #interconnect-cells = <1>; + }; + + usb_hsphy_1: phy@88e2000 { compatible = "qcom,qcs615-qusb2-phy"; reg = <0x0 0x88e2000 0x0 0x180>; @@ -4036,9 +5055,35 @@ status = "disabled"; }; - usb_1: usb@a6f8800 { - compatible = "qcom,qcs615-dwc3", "qcom,dwc3"; - reg = <0x0 0x0a6f8800 0x0 0x400>; + usb_qmpphy_2: phy@88e8000 { + compatible = "qcom,qcs615-qmp-usb3-dp-phy"; + reg = <0x0 0x088e8000 0x0 0x2000>; + + clocks = <&gcc GCC_USB2_SEC_PHY_AUX_CLK>, + <&gcc GCC_USB3_SEC_CLKREF_CLK>, + <&gcc GCC_AHB2PHY_WEST_CLK>, + <&gcc GCC_USB2_SEC_PHY_PIPE_CLK>; + clock-names = "aux", + "ref", + "cfg_ahb", + "pipe"; + + resets = <&gcc GCC_USB3PHY_PHY_SEC_BCR >, + <&gcc GCC_USB3_DP_PHY_SEC_BCR>; + reset-names = "phy_phy", + "dp_phy"; + + #clock-cells = <1>; + #phy-cells = <1>; + + qcom,tcsr-reg = <&tcsr 0xbff0 0xb24c>; + + status = "disabled"; + }; + + usb_1: usb@a600000 { + compatible = "qcom,qcs615-dwc3", "qcom,snps-dwc3"; + reg = <0x0 0x0a600000 0x0 0xfc100>; clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>, <&gcc GCC_USB30_PRIM_MASTER_CLK>, @@ -4057,52 +5102,49 @@ <&gcc GCC_USB30_PRIM_MASTER_CLK>; assigned-clock-rates = <19200000>, <200000000>; - interrupts-extended = <&intc GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>, - <&intc GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>, + interrupts-extended = <&intc GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH 0>, + <&intc GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH 0>, + <&intc GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH 0>, <&pdc 9 IRQ_TYPE_EDGE_BOTH>, <&pdc 8 IRQ_TYPE_EDGE_BOTH>, <&pdc 6 IRQ_TYPE_LEVEL_HIGH>; - interrupt-names = "pwr_event", + interrupt-names = "dwc_usb3", + "pwr_event", "hs_phy_irq", "dp_hs_phy_irq", "dm_hs_phy_irq", "ss_phy_irq"; + iommus = <&apps_smmu 0x140 0x0>; power-domains = <&gcc USB30_PRIM_GDSC>; required-opps = <&rpmhpd_opp_nom>; + phys = <&usb_hsphy_1>, <&usb_qmpphy>; + phy-names = "usb2-phy", "usb3-phy"; + resets = <&gcc GCC_USB30_PRIM_BCR>; #address-cells = <2>; #size-cells = <2>; ranges; - status = "disabled"; - - usb_1_dwc3: usb@a600000 { - compatible = "snps,dwc3"; - reg = <0x0 0x0a600000 0x0 0xcd00>; - - iommus = <&apps_smmu 0x140 0x0>; - interrupts = ; + snps,dis-u1-entry-quirk; + snps,dis-u2-entry-quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,dis_enblslpm_quirk; + snps,has-lpm-erratum; + snps,hird-threshold = /bits/ 8 <0x10>; + snps,usb3_lpm_capable; - phys = <&usb_1_hsphy>, <&usb_qmpphy>; - phy-names = "usb2-phy", "usb3-phy"; + wakeup-source; - snps,dis-u1-entry-quirk; - snps,dis-u2-entry-quirk; - snps,dis_u2_susphy_quirk; - snps,dis_u3_susphy_quirk; - snps,dis_enblslpm_quirk; - snps,has-lpm-erratum; - snps,hird-threshold = /bits/ 8 <0x10>; - snps,usb3_lpm_capable; - }; + status = "disabled"; }; - usb_2: usb@a8f8800 { - compatible = "qcom,qcs615-dwc3", "qcom,dwc3"; - reg = <0x0 0x0a8f8800 0x0 0x400>; + usb_2: usb@a800000 { + compatible = "qcom,qcs615-dwc3", "qcom,snps-dwc3"; + reg = <0x0 0x0a800000 0x0 0xfc100>; clocks = <&gcc GCC_CFG_NOC_USB2_SEC_AXI_CLK>, <&gcc GCC_USB20_SEC_MASTER_CLK>, @@ -4121,18 +5163,24 @@ <&gcc GCC_USB20_SEC_MASTER_CLK>; assigned-clock-rates = <19200000>, <200000000>; - interrupts-extended = <&intc GIC_SPI 663 IRQ_TYPE_LEVEL_HIGH>, - <&intc GIC_SPI 662 IRQ_TYPE_LEVEL_HIGH>, + interrupts-extended = <&intc GIC_SPI 664 IRQ_TYPE_LEVEL_HIGH 0>, + <&intc GIC_SPI 663 IRQ_TYPE_LEVEL_HIGH 0>, + <&intc GIC_SPI 662 IRQ_TYPE_LEVEL_HIGH 0>, <&pdc 11 IRQ_TYPE_EDGE_BOTH>, <&pdc 10 IRQ_TYPE_EDGE_BOTH>; - interrupt-names = "pwr_event", + interrupt-names = "dwc_usb3", + "pwr_event", "hs_phy_irq", "dp_hs_phy_irq", "dm_hs_phy_irq"; + iommus = <&apps_smmu 0xe0 0x0>; power-domains = <&gcc USB20_SEC_GDSC>; required-opps = <&rpmhpd_opp_nom>; + phys = <&usb_hsphy_2>; + phy-names = "usb2-phy"; + resets = <&gcc GCC_USB20_SEC_BCR>; qcom,select-utmi-as-pipe-clk; @@ -4141,34 +5189,24 @@ #size-cells = <2>; ranges; - status = "disabled"; - - usb_2_dwc3: usb@a800000 { - compatible = "snps,dwc3"; - reg = <0x0 0x0a800000 0x0 0xcd00>; - - iommus = <&apps_smmu 0xe0 0x0>; - interrupts = ; - - phys = <&usb_hsphy_2>; - phy-names = "usb2-phy"; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,dis_enblslpm_quirk; + snps,has-lpm-erratum; + snps,hird-threshold = /bits/ 8 <0x10>; + maximum-speed = "high-speed"; - snps,dis_u2_susphy_quirk; - snps,dis_u3_susphy_quirk; - snps,dis_enblslpm_quirk; - snps,has-lpm-erratum; - snps,hird-threshold = /bits/ 8 <0x10>; + wakeup-source; - maximum-speed = "high-speed"; - }; + status = "disabled"; }; tsens0: thermal-sensor@c263000 { compatible = "qcom,qcs615-tsens", "qcom,tsens-v2"; reg = <0x0 0x0c263000 0x0 0x1000>, <0x0 0x0c222000 0x0 0x1000>; - interrupts = , - ; + interrupts = , + ; interrupt-names = "uplow", "critical"; #qcom,sensors = <16>; #thermal-sensor-cells = <1>; @@ -4178,7 +5216,7 @@ compatible = "qcom,qcs615-adsp-pas", "qcom,sm8150-adsp-pas"; reg = <0x0 0x62400000 0x0 0x4040>; - interrupts-extended = <&intc GIC_SPI 162 IRQ_TYPE_EDGE_RISING>, + interrupts-extended = <&intc GIC_SPI 162 IRQ_TYPE_EDGE_RISING 0>, <&adsp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>, <&adsp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>, <&adsp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>, @@ -4205,7 +5243,7 @@ status = "disabled"; glink_edge: glink-edge { - interrupts = ; + interrupts = ; mboxes = <&apss_shared 24>; label = "lpass"; qcom,remote-pid = <2>; @@ -4246,6 +5284,45 @@ dma-coherent; }; }; + + gpr: gpr { + compatible = "qcom,gpr"; + qcom,glink-channels = "adsp_apps"; + qcom,domain = ; + qcom,intents = <512 20>; + #address-cells = <1>; + #size-cells = <0>; + + q6apm: service@1 { + compatible = "qcom,q6apm"; + reg = ; + #sound-dai-cells = <0>; + qcom,protection-domain = "avs/audio", + "msm/adsp/audio_pd"; + + q6apmbedai: bedais { + compatible = "qcom,q6apm-lpass-dais"; + #sound-dai-cells = <1>; + }; + + q6apmdai: dais { + compatible = "qcom,q6apm-dais"; + iommus = <&apps_smmu 0x1721 0x0>; + }; + }; + + q6prm: service@2 { + compatible = "qcom,q6prm"; + reg = ; + qcom,protection-domain = "avs/audio", + "msm/adsp/audio_pd"; + + q6prmcc: clock-controller { + compatible = "qcom,q6prm-lpass-clocks"; + #clock-cells = <2>; + }; + }; + }; }; }; @@ -4264,10 +5341,10 @@ arch_timer: timer { compatible = "arm,armv8-timer"; - interrupts = , - , - , - ; + interrupts = , + , + , + ; }; thermal-zones { @@ -4383,24 +5460,51 @@ thermal-sensors = <&tsens0 9>; trips { + gpu_alert0: trip-point0 { + temperature = <105000>; + hysteresis = <5000>; + type = "passive"; + }; + gpu-critical { temperature = <115000>; hysteresis = <1000>; type = "critical"; }; }; + + cooling-maps { + map0 { + trip = <&gpu_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; q6-hvx-thermal { thermal-sensors = <&tsens0 10>; trips { + q6_hvx_alert0: trip-point0 { + temperature = <105000>; + hysteresis = <5000>; + type = "passive"; + }; + q6-hvx-critical { temperature = <115000>; hysteresis = <1000>; type = "critical"; }; }; + + cooling-maps { + map0 { + trip = <&q6_hvx_alert0>; + cooling-device = <&cdsp_sw + THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; mdm-core-thermal { diff --git a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi index ee3c8c5e2c50c..0a382cc9e6431 100644 --- a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi @@ -11,10 +11,9 @@ #include #include -#include "x1e80100-pmics.dtsi" +#include "hamoa-pmics.dtsi" / { - model = "ASUS Zenbook A14"; chassis-type = "laptop"; aliases { @@ -1005,14 +1004,10 @@ status = "okay"; aux-bus { - panel { + panel: panel { compatible = "edp-panel"; - enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; power-supply = <&vreg_edp_3p3>; - pinctrl-0 = <&edp_bl_en>; - pinctrl-names = "default"; - port { edp_panel_in: endpoint { remote-endpoint = <&mdss_dp3_out>; @@ -1037,9 +1032,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1053,10 +1045,12 @@ status = "okay"; }; -&pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +&pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; +}; +&pcie6a { vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -1072,6 +1066,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { rtmr0_default: rtmr0-reset-n-active-state { pins = "gpio10"; diff --git a/arch/arm64/boot/dts/qcom/x1-crd.dtsi b/arch/arm64/boot/dts/qcom/x1-crd.dtsi index 3c9455fede5c0..f72b30b276781 100644 --- a/arch/arm64/boot/dts/qcom/x1-crd.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-crd.dtsi @@ -6,10 +6,11 @@ #include #include #include +#include #include #include -#include "x1e80100-pmics.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Qualcomm Technologies, Inc. X1E80100 CRD"; @@ -858,12 +859,101 @@ regulator-initial-mode = ; }; }; + + regulators-8 { + compatible = "qcom,pm8010-rpmh-regulators"; + qcom,pmic-id = "m"; + + vdd-l1-l2-supply = <&vreg_s5j_1p2>; + vdd-l3-l4-supply = <&vreg_s4c_1p8>; + vdd-l7-supply = <&vreg_bob1>; + + vreg_l3m_1p8: ldo3 { + regulator-name = "vreg_l3m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1808000>; + regulator-initial-mode = ; + }; + + vreg_l4m_1p8: ldo4 { + regulator-name = "vreg_l4m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1808000>; + regulator-initial-mode = ; + }; + + vreg_l7m_2p9: ldo7 { + regulator-name = "vreg_l7m_2p9"; + regulator-min-microvolt = <2912000>; + regulator-max-microvolt = <2912000>; + regulator-initial-mode = ; + }; + }; }; &gpu { status = "okay"; }; +&camss { + status = "okay"; + + ports { + /* + * port0 => csiphy0 + * port1 => csiphy1 + * port2 => csiphy2 + * port3 => csiphy4 + */ + port@3 { + camss_csiphy4_inep0: endpoint@0 { + clock-lanes = <7>; + data-lanes = <0 1 2 3>; + remote-endpoint = <&ov08x40_ep>; + }; + }; + }; +}; + +&cci1 { + status = "okay"; +}; + +&cci1_i2c1 { + camera@36 { + compatible = "ovti,ov08x40"; + reg = <0x36>; + + reset-gpios = <&tlmm 237 GPIO_ACTIVE_LOW>; + pinctrl-0 = <&cam_rgb_default>; + pinctrl-names = "default"; + + clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clock-rates = <19200000>; + + orientation = <0>; /* front facing */ + + avdd-supply = <&vreg_l7b_2p8>; + dovdd-supply = <&vreg_l3m_1p8>; + + port { + ov08x40_ep: endpoint { + data-lanes = <1 2 3 4>; + link-frequencies = /bits/ 64 <400000000>; + remote-endpoint = <&camss_csiphy4_inep0>; + }; + }; + }; +}; + +&csiphy4 { + vdda-0p9-supply = <&vreg_l2c_0p8>; + vdda-1p2-supply = <&vreg_l1c_1p2>; + + status = "okay"; +}; + &i2c0 { clock-frequency = <400000>; @@ -1035,6 +1125,16 @@ #phy-cells = <0>; }; + + embedded-controller@76 { + compatible = "qcom,hamoa-crd-ec"; + reg = <0x76>; + + interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>; + + pinctrl-0 = <&ec_int_n_default>; + pinctrl-names = "default"; + }; }; &i2c7 { @@ -1209,15 +1309,17 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; status = "okay"; }; +&pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; +}; + &pcie4_phy { vdda-phy-supply = <&vreg_l3i_0p8>; vdda-pll-supply = <&vreg_l3e_1p2>; @@ -1226,9 +1328,6 @@ }; &pcie5 { - perst-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_wwan>; pinctrl-0 = <&pcie5_default>; @@ -1244,10 +1343,12 @@ status = "okay"; }; -&pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +&pcie5_port0 { + reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; +}; +&pcie6a { vddpe-3v3-supply = <&vreg_nvme>; pinctrl-names = "default"; @@ -1263,6 +1364,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { kypd_vol_up_n: kypd-vol-up-n-state { pins = "gpio6"; @@ -1478,6 +1584,28 @@ <44 4>, /* SPI (TPM) */ <238 1>; /* UFS Reset */ + ec_int_n_default: ec-int-n-state { + pins = "gpio66"; + function = "gpio"; + bias-disable; + }; + + cam_rgb_default: cam-rgb-default-state { + mclk-pins { + pins = "gpio100"; + function = "cam_aon"; + drive-strength = <16>; + bias-disable; + }; + + reset-n-pins { + pins = "gpio237"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + }; + edp_reg_en: edp-reg-en-state { pins = "gpio70"; function = "gpio"; diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi index cc64558ed5e6f..ebe53046750ca 100644 --- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi @@ -10,9 +10,10 @@ #include #include #include +#include #include -#include "x1e80100-pmics.dtsi" +#include "hamoa-pmics.dtsi" / { chassis-type = "laptop"; @@ -792,6 +793,66 @@ }; }; +&camss { + status = "okay"; + + ports { + /* + * port0 => csiphy0 + * port1 => csiphy1 + * port2 => csiphy2 + * port3 => csiphy4 + */ + port@3 { + camss_csiphy4_inep0: endpoint@0 { + clock-lanes = <7>; + data-lanes = <0 1>; + remote-endpoint = <&ov02e10_ep>; + }; + }; + }; +}; + +&cci1 { + status = "okay"; +}; + +&cci1_i2c1 { + camera@10 { + compatible = "ovti,ov02e10"; + reg = <0x10>; + + reset-gpios = <&tlmm 237 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_rgb_default>; + + clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clock-rates = <19200000>; + + orientation = <0>; /* front facing */ + + avdd-supply = <&vreg_l7b_2p8>; + dvdd-supply = <&vreg_l7b_2p8>; + dovdd-supply = <&vreg_cam_1p8>; + + port { + ov02e10_ep: endpoint { + data-lanes = <1 2>; + link-frequencies = /bits/ 64 <360000000>; + remote-endpoint = <&camss_csiphy4_inep0>; + }; + }; + }; +}; + +&csiphy4 { + vdda-0p9-supply = <&vreg_l2c_0p8>; + vdda-1p2-supply = <&vreg_l1c_1p2>; + + status = "okay"; +}; + &i2c0 { clock-frequency = <400000>; @@ -1023,7 +1084,6 @@ }; &mdss_dp0_out { - data-lanes = <0 1>; link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; }; @@ -1032,13 +1092,15 @@ }; &mdss_dp1_out { - data-lanes = <0 1>; link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; }; &mdss_dp3 { /delete-property/ #sound-dai-cells; + pinctrl-0 = <&edp0_hpd_default>; + pinctrl-names = "default"; + status = "okay"; aux-bus { @@ -1080,9 +1142,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1097,6 +1156,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -1114,9 +1176,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -1125,6 +1184,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pcie6a_phy { vdda-phy-supply = <&vreg_l1d_0p8>; vdda-pll-supply = <&vreg_l2j_1p2>; diff --git a/arch/arm64/boot/dts/qcom/x1-el2.dtso b/arch/arm64/boot/dts/qcom/x1-el2.dtso index 2d1c9151cf1b4..ee006742d6f3b 100644 --- a/arch/arm64/boot/dts/qcom/x1-el2.dtso +++ b/arch/arm64/boot/dts/qcom/x1-el2.dtso @@ -7,6 +7,10 @@ /dts-v1/; /plugin/; +&apss_watchdog { + status = "okay"; +}; + /* We can't and don't need to use zap shader in EL2 as linux can zap the gpu on it's own. */ &gpu_zap_shader { status = "disabled"; @@ -48,6 +52,14 @@ status = "okay"; }; +&remoteproc_adsp { + iommus = <&apps_smmu 0x1000 0x80>; +}; + +&remoteproc_cdsp { + iommus = <&apps_smmu 0x0c00 0x0>; +}; + /* * The "SBSA watchdog" is implemented in software in Gunyah * and can't be used when running in EL2. diff --git a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi index a4075434162aa..41063948c583d 100644 --- a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi @@ -1065,9 +1065,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1082,6 +1079,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -1099,9 +1099,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -1110,6 +1107,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pcie6a_phy { vdda-phy-supply = <&vreg_l1d_0p8>; vdda-pll-supply = <&vreg_l2j_1p2>; diff --git a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts index bfc649d4b643c..457dd0c654af8 100644 --- a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts +++ b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts @@ -8,8 +8,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Qualcomm Technologies, Inc. X1E001DE Snapdragon Devkit for Windows"; @@ -1003,9 +1003,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1019,10 +1016,12 @@ status = "okay"; }; -&pcie5 { - perst-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; +&pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; +}; +&pcie5 { vddpe-3v3-supply = <&vreg_wwan>; pinctrl-0 = <&pcie5_default>; @@ -1038,10 +1037,12 @@ status = "okay"; }; -&pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +&pcie5_port0 { + reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; +}; +&pcie6a { vddpe-3v3-supply = <&vreg_nvme>; pinctrl-names = "default"; @@ -1057,6 +1058,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { rtmr0_default: rtmr0-reset-n-active-state { pins = "gpio10"; diff --git a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi index 654cbce9d6ecb..24e3fbcfbb7bf 100644 --- a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi @@ -9,11 +9,12 @@ #include #include #include +#include #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Lenovo ThinkPad T14s Gen 6"; @@ -475,6 +476,13 @@ regulator-initial-mode = ; }; + vreg_l7b_2p8: ldo7 { + regulator-name = "vreg_l7b_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + }; + vreg_l8b_3p0: ldo8 { regulator-name = "vreg_l8b_3p0"; regulator-min-microvolt = <3072000>; @@ -718,6 +726,118 @@ regulator-initial-mode = ; }; }; + + regulators-8 { + compatible = "qcom,pm8010-rpmh-regulators"; + qcom,pmic-id = "m"; + + vdd-l1-l2-supply = <&vreg_s5j_1p2>; + vdd-l3-l4-supply = <&vreg_s4c_1p8>; + vdd-l7-supply = <&vreg_bob1>; + + vreg_l1m_1p2: ldo1 { + regulator-name = "vreg_l1m_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1260000>; + regulator-initial-mode = ; + }; + + vreg_l2m_1p2: ldo2 { + regulator-name = "vreg_l2m_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1260000>; + regulator-initial-mode = ; + }; + + vreg_l3m_1p8: ldo3 { + regulator-name = "vreg_l3m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1900000>; + regulator-initial-mode = ; + }; + + vreg_l4m_1p8: ldo4 { + regulator-name = "vreg_l4m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1900000>; + regulator-initial-mode = ; + }; + + vreg_l5m_2p8: ldo5 { + regulator-name = "vreg_l5m_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + }; + + vreg_l7m_2p8: ldo7 { + regulator-name = "vreg_l7m_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + }; + }; + +}; + +&camss { + status = "okay"; + + ports { + /* + * port0 => csiphy0 + * port1 => csiphy1 + * port2 => csiphy2 + * port3 => csiphy4 + */ + port@3 { + camss_csiphy4_inep0: endpoint@0 { + clock-lanes = <7>; + data-lanes = <0 1>; + remote-endpoint = <&ov02c10_ep>; + }; + }; + }; +}; + +&cci1 { + status = "okay"; +}; + +&cci1_i2c1 { + camera@36 { + compatible = "ovti,ov02c10"; + reg = <0x36>; + + reset-gpios = <&tlmm 237 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_rgb_default>; + + clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clock-rates = <19200000>; + + orientation = <0>; /* front facing */ + + avdd-supply = <&vreg_l7m_2p8>; + dvdd-supply = <&vreg_l2m_1p2>; + dovdd-supply = <&vreg_l4m_1p8>; + + port { + ov02c10_ep: endpoint { + data-lanes = <1 2>; + link-frequencies = /bits/ 64 <400000000>; + remote-endpoint = <&camss_csiphy4_inep0>; + }; + }; + }; +}; + +&csiphy4 { + vdda-0p9-supply = <&vreg_l2c_0p8>; + vdda-1p2-supply = <&vreg_l1c_1p2>; + + status = "okay"; }; &gpu { @@ -1065,9 +1185,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1081,10 +1198,12 @@ status = "okay"; }; -&pcie5 { - perst-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; +&pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; +}; +&pcie5 { vddpe-3v3-supply = <&vreg_wwan>; pinctrl-0 = <&pcie5_default>; @@ -1100,10 +1219,12 @@ status = "okay"; }; -&pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +&pcie5_port0 { + reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; +}; +&pcie6a { vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -1119,6 +1240,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { rtmr0_default: rtmr0-reset-n-active-state { pins = "gpio10"; @@ -1287,6 +1413,22 @@ <72 2>, /* Secure EC I2C connection (?) */ <238 1>; /* UFS Reset */ + cam_rgb_default: cam-rgb-default-state { + mclk-pins { + pins = "gpio100"; + function = "cam_aon"; + drive-strength = <16>; + bias-disable; + }; + + reset-n-pins { + pins = "gpio237"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + }; + ec_int_n_default: ec-int-n-state { pins = "gpio66"; function = "gpio"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts b/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts index 0113d856b3ad4..aebc7d96f03f9 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts @@ -11,8 +11,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "ASUS Vivobook S 15"; @@ -631,9 +631,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -648,6 +645,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -665,9 +665,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -683,6 +680,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pmc8380_3_gpios { edp_bl_en: edp-bl-en-state { pins = "gpio4"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts b/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts index 0d0bcc50207d7..f95398205224a 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts @@ -6,12 +6,71 @@ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-asus-zenbook-a14.dtsi" / { model = "ASUS Zenbook A14 (UX3407RA)"; compatible = "asus,zenbook-a14-ux3407ra", "qcom,x1e80100"; + + wcn7850-pmu { + compatible = "qcom,wcn7850-pmu"; + + vdd-supply = <&vreg_wcn_0p95>; + vddio-supply = <&vreg_l15b_1p8>; + vddaon-supply = <&vreg_wcn_0p95>; + vdddig-supply = <&vreg_wcn_0p95>; + vddrfa1p2-supply = <&vreg_wcn_1p9>; + vddrfa1p8-supply = <&vreg_wcn_1p9>; + + bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>; + wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>; + pinctrl-names = "default"; + + regulators { + vreg_pmu_rfa_cmn: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn"; + }; + + vreg_pmu_aon_0p59: ldo1 { + regulator-name = "vreg_pmu_aon_0p59"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p85: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p85"; + }; + + vreg_pmu_btcmx_0p85: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p85"; + }; + + vreg_pmu_rfa_0p8: ldo5 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo6 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p8: ldo7 { + regulator-name = "vreg_pmu_rfa_1p8"; + }; + + vreg_pmu_pcie_0p9: ldo8 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_pcie_1p8: ldo9 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + }; + }; }; &gpu { @@ -22,6 +81,14 @@ firmware-name = "qcom/x1e80100/ASUSTeK/zenbook-a14/qcdxkmsuc8380.mbn"; }; +&panel { + compatible = "samsung,atna40cu11", "samsung,atna33xc20"; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&edp_bl_en>; + pinctrl-names = "default"; +}; + &remoteproc_adsp { firmware-name = "qcom/x1e80100/ASUSTeK/zenbook-a14/qcadsp8380.mbn", "qcom/x1e80100/ASUSTeK/zenbook-a14/adsp_dtbs.elf"; @@ -35,3 +102,41 @@ status = "okay"; }; + +&pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + + wifi@0 { + compatible = "pci17cb,1107"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + }; +}; + +&uart14 { + status = "okay"; + + bluetooth { + compatible = "qcom,wcn7850-bt"; + + vddaon-supply = <&vreg_pmu_aon_0p59>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p85>; + + max-speed = <3000000>; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-crd.dts b/arch/arm64/boot/dts/qcom/x1e80100-crd.dts index dfc378e1a056a..429deffcf3e9d 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-crd.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-crd.dts @@ -5,7 +5,7 @@ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-crd.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts index cf2a7c2628881..75e10d97c3867 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-dell-thena.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts index 32ad9679550ef..a8ff7ef258a13 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-dell-thena.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts index 58f8caaa72580..6943c305f0690 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts @@ -12,8 +12,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Dell XPS 13 9345"; @@ -941,9 +941,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -958,6 +955,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -975,9 +975,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -993,6 +990,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { rtmr0_default: rtmr0-reset-n-active-state { pins = "gpio10"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts b/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts index e5a839d458402..b79e59e1c413a 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts @@ -6,8 +6,8 @@ /dts-v1/; -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" #include "x1-hp-omnibook-x14.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts index e0642fe8343f6..e4e6eed9f02ed 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts @@ -7,10 +7,11 @@ #include #include +#include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Lenovo Yoga Slim 7x"; @@ -552,6 +553,13 @@ regulator-initial-mode = ; }; + vreg_l7b_2p8: ldo7 { + regulator-name = "vreg_l7b_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-initial-mode = ; + }; + vreg_l8b_3p0: ldo8 { regulator-name = "vreg_l8b_3p0"; regulator-min-microvolt = <3072000>; @@ -795,6 +803,57 @@ regulator-initial-mode = ; }; }; + + regulators-8 { + compatible = "qcom,pm8010-rpmh-regulators"; + qcom,pmic-id = "m"; + + vdd-l1-l2-supply = <&vreg_s5j_1p2>; + vdd-l3-l4-supply = <&vreg_s4c_1p8>; + vdd-l7-supply = <&vreg_bob1>; + + vreg_l1m_1p2: ldo1 { + regulator-name = "vreg_l1m_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1260000>; + regulator-initial-mode = ; + }; + + vreg_l2m_1p2: ldo2 { + regulator-name = "vreg_l2m_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1260000>; + regulator-initial-mode = ; + }; + + vreg_l3m_1p8: ldo3 { + regulator-name = "vreg_l3m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1900000>; + regulator-initial-mode = ; + }; + + vreg_l4m_1p8: ldo4 { + regulator-name = "vreg_l4m_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1900000>; + regulator-initial-mode = ; + }; + + vreg_l5m_2p8: ldo5 { + regulator-name = "vreg_l5m_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + }; + + vreg_l7m_2p8: ldo7 { + regulator-name = "vreg_l7m_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + }; + }; }; &gpu { @@ -805,6 +864,66 @@ }; }; +&camss { + status = "okay"; + + ports { + /* + * port0 => csiphy0 + * port1 => csiphy1 + * port2 => csiphy2 + * port3 => csiphy4 + */ + port@3 { + camss_csiphy4_inep0: endpoint@0 { + clock-lanes = <7>; + data-lanes = <0 1>; + remote-endpoint = <&ov02c10_ep>; + }; + }; + }; +}; + +&cci1 { + status = "okay"; +}; + +&cci1_i2c1 { + camera@36 { + compatible = "ovti,ov02c10"; + reg = <0x36>; + + reset-gpios = <&tlmm 237 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_rgb_default>; + + clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clocks = <&camcc CAM_CC_MCLK4_CLK>; + assigned-clock-rates = <19200000>; + + orientation = <0>; /* front facing */ + + avdd-supply = <&vreg_l7b_2p8>; + dvdd-supply = <&vreg_l1m_1p2>; + dovdd-supply = <&vreg_l3m_1p8>; + + port { + ov02c10_ep: endpoint { + data-lanes = <1 2>; + link-frequencies = /bits/ 64 <400000000>; + remote-endpoint = <&camss_csiphy4_inep0>; + }; + }; + }; +}; + +&csiphy4 { + vdda-0p9-supply = <&vreg_l2c_0p8>; + vdda-1p2-supply = <&vreg_l1c_1p2>; + + status = "okay"; +}; + &i2c0 { clock-frequency = <400000>; @@ -1126,9 +1245,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1143,6 +1259,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -1160,9 +1279,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -1178,6 +1294,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { rtmr0_default: rtmr0-reset-n-active-state { pins = "gpio10"; @@ -1350,6 +1471,22 @@ <44 4>, /* SPI (TPM) */ <238 1>; /* UFS Reset */ + cam_rgb_default: cam-rgb-default-state { + mclk-pins { + pins = "gpio100"; + function = "cam_aon"; + drive-strength = <16>; + bias-disable; + }; + + reset-n-pins { + pins = "gpio237"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + }; + edp_reg_en: edp-reg-en-state { pins = "gpio70"; function = "gpio"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi index ed468b93ba50e..27c8cf520c098 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi @@ -9,8 +9,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { aliases { @@ -1094,9 +1094,6 @@ }; &pcie3 { - perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 145 GPIO_ACTIVE_HIGH>; - pinctrl-0 = <&pcie3_default>; pinctrl-names = "default"; @@ -1112,6 +1109,11 @@ status = "okay"; }; +&pcie3_port0 { + reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>; +}; + &pcie4 { status = "okay"; }; @@ -1124,6 +1126,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -1141,9 +1146,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -1159,6 +1161,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_gpios { rtmr0_default: rtmr0-reset-n-active-state { pins = "gpio10"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts index 4a9b6d791e7f4..a13a1aed8dc55 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts @@ -8,8 +8,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Qualcomm Technologies, Inc. X1E80100 QCP"; @@ -979,8 +979,6 @@ &pcie3 { pinctrl-names = "default"; pinctrl-0 = <&pcie3_default>; - perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>; status = "okay"; }; @@ -992,16 +990,16 @@ status = "okay"; }; -&pcie3_port { +&pcie3_port0 { vpcie12v-supply = <&vreg_pcie_12v>; vpcie3v3-supply = <&vreg_pcie_3v3>; vpcie3v3aux-supply = <&vreg_pcie_3v3_aux>; + + reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>; }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1016,6 +1014,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -1033,9 +1034,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-names = "default"; @@ -1051,6 +1049,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &qupv3_0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14-lcd.dts b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14-lcd.dts new file mode 100644 index 0000000000000..be756069131d7 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14-lcd.dts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2025 Aleksandrs Vinarskis + */ + +/dts-v1/; + +#include "x1p42100-asus-zenbook-a14.dtsi" + +/ { + model = "ASUS Zenbook A14 (UX3407QA, LCD)"; + compatible = "asus,zenbook-a14-ux3407qa-lcd", "asus,zenbook-a14-ux3407qa", "qcom,x1p42100"; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pmk8550_pwm 0 416667>; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; + power-supply = <&vreg_edp_bl>; + + pinctrl-0 = <&edp_bl_en>, <&edp_bl_pwm>; + pinctrl-names = "default"; + }; + + vreg_edp_bl: regulator-edp-bl { + compatible = "regulator-fixed"; + + regulator-name = "VBL9"; + regulator-min-microvolt = <3600000>; + regulator-max-microvolt = <3600000>; + + gpio = <&pmc8380_3_gpios 10 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&edp_bl_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; +}; + +&panel { + backlight = <&backlight>; +}; + +&pmc8380_3_gpios { + edp_bl_reg_en: edp-bl-reg-en-state { + pins = "gpio10"; + function = "normal"; + }; +}; + +&pmk8550_gpios { + edp_bl_pwm: edp-bl-pwm-state { + pins = "gpio5"; + function = "func3"; + }; +}; + +&pmk8550_pwm { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts index bd75ff898601a..68cd318d69073 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts @@ -6,136 +6,17 @@ /dts-v1/; -#include "x1p42100.dtsi" -#include "x1-asus-zenbook-a14.dtsi" - -/delete-node/ &pmc8380_6; -/delete-node/ &pmc8380_6_thermal; +#include "x1p42100-asus-zenbook-a14.dtsi" / { model = "ASUS Zenbook A14 (UX3407QA)"; - compatible = "asus,zenbook-a14-ux3407qa", "qcom,x1p42100"; - - wcn6855-pmu { - compatible = "qcom,wcn6855-pmu"; - - vddaon-supply = <&vreg_wcn_0p95>; - vddio-supply = <&vreg_wcn_1p9>; - vddpcie1p3-supply = <&vreg_wcn_1p9>; - vddpcie1p9-supply = <&vreg_wcn_1p9>; - vddpmu-supply = <&vreg_wcn_0p95>; - vddpmucx-supply = <&vreg_wcn_0p95>; - vddpmumx-supply = <&vreg_wcn_0p95>; - vddrfa0p95-supply = <&vreg_wcn_0p95>; - vddrfa1p3-supply = <&vreg_wcn_1p9>; - vddrfa1p9-supply = <&vreg_wcn_1p9>; - - bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>; - wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>; - - pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>; - pinctrl-names = "default"; - - regulators { - vreg_pmu_rfa_cmn_0p8: ldo0 { - regulator-name = "vreg_pmu_rfa_cmn_0p8"; - }; - - vreg_pmu_aon_0p8: ldo1 { - regulator-name = "vreg_pmu_aon_0p8"; - }; - - vreg_pmu_wlcx_0p8: ldo2 { - regulator-name = "vreg_pmu_wlcx_0p8"; - }; - - vreg_pmu_wlmx_0p8: ldo3 { - regulator-name = "vreg_pmu_wlmx_0p8"; - }; - - vreg_pmu_btcmx_0p8: ldo4 { - regulator-name = "vreg_pmu_btcmx_0p8"; - }; - - vreg_pmu_pcie_1p8: ldo5 { - regulator-name = "vreg_pmu_pcie_1p8"; - }; - - vreg_pmu_pcie_0p9: ldo6 { - regulator-name = "vreg_pmu_pcie_0p9"; - }; - - vreg_pmu_rfa_0p8: ldo7 { - regulator-name = "vreg_pmu_rfa_0p8"; - }; - - vreg_pmu_rfa_1p2: ldo8 { - regulator-name = "vreg_pmu_rfa_1p2"; - }; - - vreg_pmu_rfa_1p7: ldo9 { - regulator-name = "vreg_pmu_rfa_1p7"; - }; - }; - }; + compatible = "asus,zenbook-a14-ux3407qa-oled", "asus,zenbook-a14-ux3407qa", "qcom,x1p42100"; }; -&gpu { - status = "okay"; -}; - -&gpu_zap_shader { - firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcdxkmsucpurwa.mbn"; -}; - -&pcie4_port0 { - wifi@0 { - compatible = "pci17cb,1103"; - reg = <0x10000 0x0 0x0 0x0 0x0>; - - vddaon-supply = <&vreg_pmu_aon_0p8>; - vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; - vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; - vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; - vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; - vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; - vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; - vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; - vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; - - qcom,calibration-variant = "UX3407Q"; - }; -}; - -&remoteproc_adsp { - firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcadsp8380.mbn", - "qcom/x1p42100/ASUSTeK/zenbook-a14/adsp_dtbs.elf"; - - status = "okay"; -}; - -&remoteproc_cdsp { - firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qccdsp8380.mbn", - "qcom/x1p42100/ASUSTeK/zenbook-a14/cdsp_dtbs.elf"; - - status = "okay"; -}; - -&uart14 { - status = "okay"; - - bluetooth { - compatible = "qcom,wcn6855-bt"; - - vddaon-supply = <&vreg_pmu_aon_0p8>; - vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>; - vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; - vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; - vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; - vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; - vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; - vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; +&panel { + compatible = "samsung,atna40ct06", "samsung,atna33xc20"; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; - max-speed = <3000000>; - }; + pinctrl-0 = <&edp_bl_en>; + pinctrl-names = "default"; }; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi new file mode 100644 index 0000000000000..22470a97e1e39 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2025 Aleksandrs Vinarskis + */ + +/dts-v1/; + +#include "purwa.dtsi" +#include "x1-asus-zenbook-a14.dtsi" + +/delete-node/ &pmc8380_6; +/delete-node/ &pmc8380_6_thermal; + +/ { + wcn6855-pmu { + compatible = "qcom,wcn6855-pmu"; + + vddaon-supply = <&vreg_wcn_0p95>; + vddio-supply = <&vreg_wcn_1p9>; + vddpcie1p3-supply = <&vreg_wcn_1p9>; + vddpcie1p9-supply = <&vreg_wcn_1p9>; + vddpmu-supply = <&vreg_wcn_0p95>; + vddpmucx-supply = <&vreg_wcn_0p95>; + vddpmumx-supply = <&vreg_wcn_0p95>; + vddrfa0p95-supply = <&vreg_wcn_0p95>; + vddrfa1p3-supply = <&vreg_wcn_1p9>; + vddrfa1p9-supply = <&vreg_wcn_1p9>; + + bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>; + wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>; + pinctrl-names = "default"; + + regulators { + vreg_pmu_rfa_cmn_0p8: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn_0p8"; + }; + + vreg_pmu_aon_0p8: ldo1 { + regulator-name = "vreg_pmu_aon_0p8"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p8: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p8"; + }; + + vreg_pmu_btcmx_0p8: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p8"; + }; + + vreg_pmu_pcie_1p8: ldo5 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + + vreg_pmu_pcie_0p9: ldo6 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_rfa_0p8: ldo7 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo8 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p7: ldo9 { + regulator-name = "vreg_pmu_rfa_1p7"; + }; + }; + }; +}; + +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcdxkmsucpurwa.mbn"; +}; + +&pcie4_port0 { + wifi@0 { + compatible = "pci17cb,1103"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + vddaon-supply = <&vreg_pmu_aon_0p8>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; + + qcom,calibration-variant = "UX3407Q"; + }; +}; + +&remoteproc_adsp { + firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcadsp8380.mbn", + "qcom/x1p42100/ASUSTeK/zenbook-a14/adsp_dtbs.elf"; + + status = "okay"; +}; + +&remoteproc_cdsp { + firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qccdsp8380.mbn", + "qcom/x1p42100/ASUSTeK/zenbook-a14/cdsp_dtbs.elf"; + + status = "okay"; +}; + +&uart14 { + status = "okay"; + + bluetooth { + compatible = "qcom,wcn6855-bt"; + + vddaon-supply = <&vreg_pmu_aon_0p8>; + vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; + + max-speed = <3000000>; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-crd.dts b/arch/arm64/boot/dts/qcom/x1p42100-crd.dts index cf999c2cf8d4e..7ed4116b95904 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-crd.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-crd.dts @@ -5,7 +5,7 @@ /dts-v1/; -#include "x1p42100.dtsi" +#include "purwa.dtsi" #include "x1-crd.dtsi" /delete-node/ &pmc8380_6; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts b/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts index 6696cab2de3ec..0f338e457abda 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts @@ -2,8 +2,8 @@ /dts-v1/; -#include "x1p42100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "purwa.dtsi" +#include "hamoa-pmics.dtsi" #include "x1-hp-omnibook-x14.dtsi" /delete-node/ &pmc8380_6; /delete-node/ &pmc8380_6_thermal; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts index 1ac46cdc43861..c5e77d32e5957 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts @@ -13,8 +13,8 @@ #include #include -#include "x1p42100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "purwa.dtsi" +#include "hamoa-pmics.dtsi" /delete-node/ &pmc8380_6; /delete-node/ &pmc8380_6_thermal; @@ -1082,9 +1082,6 @@ }; &pcie4 { - perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; - pinctrl-0 = <&pcie4_default>; pinctrl-names = "default"; @@ -1099,6 +1096,9 @@ }; &pcie4_port0 { + reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>; + wifi@0 { compatible = "pci17cb,1107"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -1116,9 +1116,6 @@ }; &pcie6a { - perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; - wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; - vddpe-3v3-supply = <&vreg_nvme>; pinctrl-0 = <&pcie6a_default>; @@ -1134,6 +1131,11 @@ status = "okay"; }; +&pcie6a_port0 { + reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>; +}; + &pm8550_pwm { status = "okay"; }; diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts index 1da8e476b2193..812b133cf29ed 100644 --- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts +++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts @@ -118,14 +118,25 @@ reg = <0x6 0x00000000 0x1 0x00000000>; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + tfa@40000000 { + reg = <0x0 0x40000000 0x0 0x8000000>; + no-map; + }; + }; + /* Page 27 / DSI to Display */ - mini-dp-con { + dp-con { compatible = "dp-connector"; label = "CN6"; type = "full-size"; port { - mini_dp_con_in: endpoint { + dp_con_in: endpoint { remote-endpoint = <&sn65dsi86_out>; }; }; @@ -407,7 +418,7 @@ port@1 { reg = <1>; sn65dsi86_out: endpoint { - remote-endpoint = <&mini_dp_con_in>; + remote-endpoint = <&dp_con_in>; }; }; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts index 08e814c03fa85..ed6fcdc337a0b 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts @@ -8,7 +8,6 @@ /dts-v1/; /* Switch selection settings */ -#define SW_LCD_EN 0 #define SW_GPIO8_CAN0_STB 0 #define SW_GPIO9_CAN1_STB 0 #define SW_LCD_EN 0 diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi index 630f7a98df386..100d5cab9b12f 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi @@ -546,16 +546,6 @@ status = "disabled"; }; - wdt0: watchdog@11c00400 { - compatible = "renesas,r9a09g057-wdt"; - reg = <0 0x11c00400 0 0x400>; - clocks = <&cpg CPG_MOD 0x4b>, <&cpg CPG_MOD 0x4c>; - clock-names = "pclk", "oscclk"; - resets = <&cpg 0x75>; - power-domains = <&cpg>; - status = "disabled"; - }; - wdt1: watchdog@14400000 { compatible = "renesas,r9a09g057-wdt"; reg = <0 0x14400000 0 0x400>; @@ -566,23 +556,18 @@ status = "disabled"; }; - wdt2: watchdog@13000000 { - compatible = "renesas,r9a09g057-wdt"; - reg = <0 0x13000000 0 0x400>; - clocks = <&cpg CPG_MOD 0x4f>, <&cpg CPG_MOD 0x50>; - clock-names = "pclk", "oscclk"; - resets = <&cpg 0x77>; - power-domains = <&cpg>; - status = "disabled"; - }; - - wdt3: watchdog@13000400 { - compatible = "renesas,r9a09g057-wdt"; - reg = <0 0x13000400 0 0x400>; - clocks = <&cpg CPG_MOD 0x51>, <&cpg CPG_MOD 0x52>; - clock-names = "pclk", "oscclk"; - resets = <&cpg 0x78>; + rtc: rtc@11c00800 { + compatible = "renesas,r9a09g057-rtca3", "renesas,rz-rtca3"; + reg = <0 0x11c00800 0 0x400>; + interrupts = , + , + ; + interrupt-names = "alarm", "period", "carry"; + clocks = <&cpg CPG_MOD 0x53>, <&rtxin_clk>; + clock-names = "bus", "counter"; power-domains = <&cpg>; + resets = <&cpg 0x79>, <&cpg 0x7a>; + reset-names = "rtc", "rtest"; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi index 7f1aca218c9fb..06aae2c635676 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi @@ -267,8 +267,8 @@ cpg: clock-controller@80280000 { compatible = "renesas,r9a09g077-cpg-mssr"; - reg = <0 0x80280000 0 0x1000>, - <0 0x81280000 0 0x9000>; + reg = <0 0x80280000 0 0x10000>, + <0 0x81280000 0 0x10000>; clocks = <&extal_clk>; clock-names = "extal"; #clock-cells = <2>; diff --git a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi index f06c19c73adb8..6dd80fa2755e8 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi @@ -267,8 +267,8 @@ cpg: clock-controller@80280000 { compatible = "renesas,r9a09g087-cpg-mssr"; - reg = <0 0x80280000 0 0x1000>, - <0 0x81280000 0 0x9000>; + reg = <0 0x80280000 0 0x10000>, + <0 0x81280000 0 0x10000>; clocks = <&extal_clk>; clock-names = "extal"; #clock-cells = <2>; diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi index 39845faec8943..a5d4d70e83c90 100644 --- a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi +++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi @@ -166,7 +166,7 @@ <100000000>; renesas,settings = [ 80 00 11 19 4c 42 dc 2f 06 7d 20 1a 5f 1e f2 27 - 00 40 00 00 00 00 00 00 06 0c 19 02 3f f0 90 86 + 00 40 00 00 00 00 00 00 06 0c 19 02 3b f0 90 86 a0 80 30 30 9c ]; }; diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi index 5c91002c99c48..9c6f712a62eff 100644 --- a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi +++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi @@ -49,6 +49,7 @@ regulator-max-microvolt = <3300000>; gpios-states = <0>; states = <3300000 0>, <1800000 1>; + regulator-ramp-delay = <60>; }; #endif @@ -154,8 +155,7 @@ ctrl-pins { pinmux = , /* SD0_CLK */ , /* SD0_CMD */ - , /* SD0_CD */ - ; /* SD0_WP */ + ; /* SD0_CD */ }; }; @@ -212,6 +212,7 @@ pinctrl-names = "default", "state_uhs"; vmmc-supply = <®_3p3v>; vqmmc-supply = <&vqmmc_sdhi0>; + wp-gpios = <&pinctrl RZT2H_GPIO(22, 6) GPIO_ACTIVE_HIGH>; bus-width = <4>; sd-uhs-sdr50; sd-uhs-sdr104; diff --git a/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso b/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso index 0af1e0a6c7f48..fc53c1aae3b52 100644 --- a/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso +++ b/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso @@ -25,6 +25,7 @@ regulator-max-microvolt = <3300000>; gpios-states = <0>; states = <3300000 0>, <1800000 1>; + regulator-ramp-delay = <60>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts b/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts index e7d4a2f9a95ea..78a7775c3b226 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts @@ -424,7 +424,6 @@ &pcie0 { ep-gpios = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; - max-link-speed = <2>; num-lanes = <2>; pinctrl-names = "default"; status = "okay"; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s.dtsi index 8d94d9f91a5c6..3a9a10f531bdb 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s.dtsi @@ -71,7 +71,6 @@ }; &pcie0 { - max-link-speed = <1>; num-lanes = <1>; vpcie3v3-supply = <&vcc3v3_sys>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts index eaaca08a76018..54abe0585ec51 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts @@ -421,10 +421,6 @@ status = "okay"; }; -&hdmi_sound { - status = "okay"; -}; - &i2c0 { clock-frequency = <400000>; i2c-scl-falling-time-ns = <4>; @@ -883,12 +879,6 @@ }; }; - wifi { - wifi_host_wake_l: wifi-host-wake-l { - rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - wireless-bluetooth { bt_wake_pin: bt-wake-pin { rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; @@ -946,19 +936,7 @@ pinctrl-names = "default"; pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; sd-uhs-sdr104; - #address-cells = <1>; - #size-cells = <0>; status = "okay"; - - brcmf: wifi@1 { - compatible = "brcm,bcm4329-fmac"; - reg = <1>; - interrupt-parent = <&gpio0>; - interrupts = ; - interrupt-names = "host-wake"; - pinctrl-names = "default"; - pinctrl-0 = <&wifi_host_wake_l>; - }; }; &sdhci { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts index 2dca1dca20b8b..5de964d369b09 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts @@ -40,13 +40,13 @@ button-up { label = "Volume Up"; linux,code = ; - press-threshold-microvolt = <100000>; + press-threshold-microvolt = <2000>; }; button-down { label = "Volume Down"; linux,code = ; - press-threshold-microvolt = <600000>; + press-threshold-microvolt = <300000>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rock-3c.dts b/arch/arm64/boot/dts/rockchip/rk3566-rock-3c.dts index 6224d72813e59..80ac40555e023 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-rock-3c.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-rock-3c.dts @@ -466,6 +466,7 @@ compatible = "belling,bl24c16a", "atmel,24c16"; reg = <0x50>; pagesize = <16>; + vcc-supply = <&vcca1v8_pmu>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi index e719a3df126c5..658097ed69714 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi @@ -185,7 +185,7 @@ <0x0 0xf2000000 0x0 0x00100000>; ranges = <0x01000000 0x0 0xf2100000 0x0 0xf2100000 0x0 0x00100000>, <0x02000000 0x0 0xf2200000 0x0 0xf2200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x40000000 0x0 0x40000000>; + <0x03000000 0x3 0x40000000 0x3 0x40000000 0x0 0x40000000>; reg-names = "dbi", "apb", "config"; resets = <&cru SRST_PCIE30X1_POWERUP>; reset-names = "pipe"; @@ -238,7 +238,7 @@ <0x0 0xf0000000 0x0 0x00100000>; ranges = <0x01000000 0x0 0xf0100000 0x0 0xf0100000 0x0 0x00100000>, <0x02000000 0x0 0xf0200000 0x0 0xf0200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x80000000 0x0 0x40000000>; + <0x03000000 0x3 0x80000000 0x3 0x80000000 0x0 0x40000000>; reg-names = "dbi", "apb", "config"; resets = <&cru SRST_PCIE30X2_POWERUP>; reset-names = "pipe"; diff --git a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi index fd2214b6fad40..d654f98460ec0 100644 --- a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi @@ -975,7 +975,7 @@ power-domains = <&power RK3568_PD_PIPE>; ranges = <0x01000000 0x0 0xf4100000 0x0 0xf4100000 0x0 0x00100000>, <0x02000000 0x0 0xf4200000 0x0 0xf4200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x00000000 0x0 0x40000000>; + <0x03000000 0x3 0x00000000 0x3 0x00000000 0x0 0x40000000>; resets = <&cru SRST_PCIE20_POWERUP>; reset-names = "pipe"; #address-cells = <3>; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts index db8fef7a4f1b9..ffe55f970f461 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts @@ -223,6 +223,18 @@ vin-supply = <&vcc_3v3_s3>; }; + vcc3v3_sd: regulator-vcc-3v3-sd { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_pwren>; + regulator-name = "vcc3v3_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s0>; + }; + vcc_ufs_s0: regulator-vcc-ufs-s0 { compatible = "regulator-fixed"; regulator-name = "vcc_ufs_s0"; @@ -810,6 +822,12 @@ }; }; + sdmmc { + sdmmc_pwren: sdmmc-pwren { + rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + usb { usb_host_pwren: usb-host-pwren { rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; @@ -851,11 +869,15 @@ bus-width = <4>; cap-mmc-highspeed; cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>; disable-wp; max-frequency = <200000000>; no-sdio; no-mmc; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_det &sdmmc0_bus4>; sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; vqmmc-supply = <&vccio_sd_s0>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts index cce34c541f7c5..bb2cc2814b83f 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts +++ b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts @@ -201,6 +201,7 @@ pinctrl-names = "default"; pinctrl-0 = <&hp_det_l>; + simple-audio-card,bitclock-master = <&masterdai>; simple-audio-card,format = "i2s"; simple-audio-card,hp-det-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_LOW>; simple-audio-card,mclk-fs = <256>; @@ -211,15 +212,16 @@ "Headphones", "HPOR", "IN1P", "Microphone Jack"; simple-audio-card,widgets = - "Headphone", "Headphone Jack", + "Headphone", "Headphones", "Microphone", "Microphone Jack"; simple-audio-card,codec { sound-dai = <&rt5616>; }; - simple-audio-card,cpu { + masterdai: simple-audio-card,cpu { sound-dai = <&sai2>; + system-clock-frequency = <12288000>; }; }; }; @@ -727,10 +729,12 @@ rt5616: audio-codec@1b { compatible = "realtek,rt5616"; reg = <0x1b>; - assigned-clocks = <&cru CLK_SAI2_MCLKOUT>; + assigned-clocks = <&cru CLK_SAI2_MCLKOUT_TO_IO>; assigned-clock-rates = <12288000>; - clocks = <&cru CLK_SAI2_MCLKOUT>; + clocks = <&cru CLK_SAI2_MCLKOUT_TO_IO>; clock-names = "mclk"; + pinctrl-0 = <&sai2m0_mclk>; + pinctrl-names = "default"; #sound-dai-cells = <0>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi index 0b0851a7e4ea9..98c9f8013158c 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi @@ -5228,6 +5228,13 @@ /* ufs_rstn */ <4 RK_PD0 1 &pcfg_pull_none>; }; + + /omit-if-no-ref/ + ufs_rstgpio: ufs-rstgpio { + rockchip,pins = + /* ufs_rstn */ + <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_down>; + }; }; ufs_testdata0 { diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi index a86fc6b4e8c45..70e67d4dccb8a 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi @@ -1261,7 +1261,7 @@ gpu: gpu@27800000 { compatible = "rockchip,rk3576-mali", "arm,mali-bifrost"; - reg = <0x0 0x27800000 0x0 0x200000>; + reg = <0x0 0x27800000 0x0 0x20000>; assigned-clocks = <&scmi_clk SCMI_CLK_GPU>; assigned-clock-rates = <198000000>; clocks = <&cru CLK_GPU>; @@ -1826,7 +1826,7 @@ assigned-clock-parents = <&cru CLK_REF_MPHY_26M>; interrupts = ; power-domains = <&power RK3576_PD_USB>; - pinctrl-0 = <&ufs_refclk>; + pinctrl-0 = <&ufs_refclk &ufs_rstgpio>; pinctrl-names = "default"; resets = <&cru SRST_A_UFS_BIU>, <&cru SRST_A_UFS_SYS>, <&cru SRST_A_UFS>, <&cru SRST_P_UFS_GRF>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index e2500e31c434a..7e74e04057cfd 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1200,7 +1200,7 @@ status = "disabled"; }; - rknn_mmu_1: iommu@fdac9000 { + rknn_mmu_1: iommu@fdaca000 { compatible = "rockchip,rk3588-iommu", "rockchip,rk3568-iommu"; reg = <0x0 0xfdaca000 0x0 0x100>; interrupts = ; @@ -1230,7 +1230,7 @@ status = "disabled"; }; - rknn_mmu_2: iommu@fdad9000 { + rknn_mmu_2: iommu@fdada000 { compatible = "rockchip,rk3588-iommu", "rockchip,rk3568-iommu"; reg = <0x0 0xfdada000 0x0 0x100>; interrupts = ; @@ -1955,7 +1955,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf3100000 0x0 0xf3100000 0x0 0x00100000>, <0x02000000 0x0 0xf3200000 0x0 0xf3200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0xc0000000 0x0 0x40000000>; + <0x03000000 0x9 0xc0000000 0x9 0xc0000000 0x0 0x40000000>; reg = <0xa 0x40c00000 0x0 0x00400000>, <0x0 0xfe180000 0x0 0x00010000>, <0x0 0xf3000000 0x0 0x00100000>; @@ -2007,7 +2007,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf4100000 0x0 0xf4100000 0x0 0x00100000>, <0x02000000 0x0 0xf4200000 0x0 0xf4200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0xa 0x00000000 0x0 0x40000000>; + <0x03000000 0xa 0x00000000 0xa 0x00000000 0x0 0x40000000>; reg = <0xa 0x41000000 0x0 0x00400000>, <0x0 0xfe190000 0x0 0x00010000>, <0x0 0xf4000000 0x0 0x00100000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi index 6e5a58428bbab..a2640014ee042 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi @@ -375,7 +375,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf0100000 0x0 0xf0100000 0x0 0x00100000>, <0x02000000 0x0 0xf0200000 0x0 0xf0200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x00000000 0x0 0x40000000>; + <0x03000000 0x9 0x00000000 0x9 0x00000000 0x0 0x40000000>; reg = <0xa 0x40000000 0x0 0x00400000>, <0x0 0xfe150000 0x0 0x00010000>, <0x0 0xf0000000 0x0 0x00100000>; @@ -462,7 +462,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf1100000 0x0 0xf1100000 0x0 0x00100000>, <0x02000000 0x0 0xf1200000 0x0 0xf1200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x40000000 0x0 0x40000000>; + <0x03000000 0x9 0x40000000 0x9 0x40000000 0x0 0x40000000>; reg = <0xa 0x40400000 0x0 0x00400000>, <0x0 0xfe160000 0x0 0x00010000>, <0x0 0xf1000000 0x0 0x00100000>; @@ -512,7 +512,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf2100000 0x0 0xf2100000 0x0 0x00100000>, <0x02000000 0x0 0xf2200000 0x0 0xf2200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x80000000 0x0 0x40000000>; + <0x03000000 0x9 0x80000000 0x9 0x80000000 0x0 0x40000000>; reg = <0xa 0x40800000 0x0 0x00400000>, <0x0 0xfe170000 0x0 0x00010000>, <0x0 0xf2000000 0x0 0x00100000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi index 3bbe78810ec6f..7aac77dfc5f16 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi @@ -331,12 +331,12 @@ data-role = "dual"; /* fusb302 supports PD Rev 2.0 Ver 1.2 */ pd-revision = /bits/ 8 <0x2 0x0 0x1 0x2>; - power-role = "sink"; - try-power-role = "sink"; op-sink-microwatt = <1000000>; sink-pdos = , ; + source-pdos = + ; altmodes { displayport { diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus.dts b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus.dts index 5e984a44120e4..07a840d9b3859 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus.dts @@ -110,6 +110,11 @@ }; }; +&usb_con { + power-role = "dual"; + try-power-role = "sink"; +}; + &usbdp_phy0 { pinctrl-names = "default"; pinctrl-0 = <&usbc_sbu_dc>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b.dts b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b.dts index 8ef01010d985b..da13dafcbc823 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b.dts @@ -49,6 +49,10 @@ }; }; +&usb_con { + power-role = "sink"; +}; + &usbdp_phy0 { pinctrl-names = "default"; pinctrl-0 = <&usbc_sbu_dc>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5t.dts b/arch/arm64/boot/dts/rockchip/rk3588-rock-5t.dts index c1763835f53d4..0dd90c744380b 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5t.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5t.dts @@ -130,6 +130,10 @@ }; }; +&usb_con { + power-role = "source"; +}; + &usbdp_phy0 { pinctrl-names = "default"; pinctrl-0 = <&usbc_sbu_dc>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts b/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts index 19a08f7794e67..041a0fff22ccb 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts @@ -228,6 +228,13 @@ regulator-off-in-suspend; }; }; + + eeprom: eeprom@50 { + compatible = "belling,bl24c16a", "atmel,24c16"; + reg = <0x50>; + pagesize = <16>; + vcc-supply = <&vcc_3v3_pmu>; + }; }; &i2c2 { @@ -249,12 +256,6 @@ regulator-off-in-suspend; }; }; - - eeprom: eeprom@50 { - compatible = "belling,bl24c16a", "atmel,24c16"; - reg = <0x50>; - pagesize = <16>; - }; }; &i2c3 { @@ -600,7 +601,7 @@ }; }; - vcc_3v3_s3: dcdc-reg8 { + vcc_3v3_pmu: vcc_3v3_s3: dcdc-reg8 { regulator-name = "vcc_3v3_s3"; regulator-always-on; regulator-boot-on; diff --git a/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts b/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts index 6e165073f7329..bb6d6393d2e46 100644 --- a/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts +++ b/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts @@ -266,6 +266,7 @@ &ommanager { memory-region = <&mm_ospi1>; + memory-region-names = "ospi1"; pinctrl-0 = <&ospi_port1_clk_pins_a &ospi_port1_io03_pins_a &ospi_port1_cs0_pins_a>; diff --git a/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso b/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso index 173ac60723b64..b4daa674eaa1e 100644 --- a/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso +++ b/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso @@ -14,7 +14,7 @@ }; &main_pmx0 { - gpmc0_pins_default: gpmc0-pins-default { + gpmc0_pins_default: gpmc0-default-pins { pinctrl-single,pins = < AM62X_IOPAD(0x003c, PIN_INPUT, 0) /* (K19) GPMC0_AD0 */ AM62X_IOPAD(0x0040, PIN_INPUT, 0) /* (L19) GPMC0_AD1 */ diff --git a/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts b/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts index 83af889e790ae..9a74df221f2ad 100644 --- a/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts +++ b/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts @@ -146,6 +146,7 @@ regulator-name = "vdd_mmc1"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_sys>; regulator-boot-on; enable-active-high; gpio = <&exp1 3 GPIO_ACTIVE_HIGH>; @@ -165,14 +166,16 @@ }; vddshv_sdio: regulator-6 { + /* output of TLV7103318QDSERQ1 */ compatible = "regulator-gpio"; regulator-name = "vddshv_sdio"; pinctrl-names = "default"; pinctrl-0 = <&vddshv_sdio_pins_default>; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_5v0>; regulator-boot-on; - gpios = <&main_gpio1 31 GPIO_ACTIVE_HIGH>; + gpios = <&main_gpio0 59 GPIO_ACTIVE_HIGH>; states = <1800000 0x0>, <3300000 0x1>; bootph-all; @@ -198,7 +201,7 @@ pmic_irq_pins_default: pmic-irq-default-pins { pinctrl-single,pins = < - AM62DX_MCU_IOPAD(0x000, PIN_INPUT, 7) /* (E11) MCU_GPIO0_0 */ + AM62DX_IOPAD(0x01f4, PIN_INPUT, 7) /* (F17) EXTINTn.GPIO1_31 */ >; }; @@ -334,7 +337,7 @@ vddshv_sdio_pins_default: vddshv-sdio-default-pins { pinctrl-single,pins = < - AM62DX_IOPAD(0x1f4, PIN_OUTPUT, 7) /* (M19) GPMC0_CLK.GPIO1_31 */ + AM62DX_IOPAD(0x00f0, PIN_INPUT, 7) /* (Y21) GPIO0_59 */ >; bootph-all; }; diff --git a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi index 99810047614e3..b7d559c61f3f8 100644 --- a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi @@ -112,7 +112,7 @@ regulator-max-microvolt = <3300000>; regulator-min-microvolt = <3300000>; regulator-name = "+V3.3_SD"; - startup-delay-us = <2000>; + startup-delay-us = <20000>; }; reg_sd1_vqmmc: regulator-sdhci1-vqmmc { diff --git a/arch/arm64/boot/dts/ti/k3-am62p.dtsi b/arch/arm64/boot/dts/ti/k3-am62p.dtsi index 75a15c368c11b..dd24c40c7965d 100644 --- a/arch/arm64/boot/dts/ti/k3-am62p.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62p.dtsi @@ -59,7 +59,7 @@ <0x00 0x01000000 0x00 0x01000000 0x00 0x01b28400>, /* First peripheral window */ <0x00 0x08000000 0x00 0x08000000 0x00 0x00200000>, /* Main CPSW */ <0x00 0x0e000000 0x00 0x0e000000 0x00 0x01d20000>, /* Second peripheral window */ - <0x00 0x0fd00000 0x00 0x0fd00000 0x00 0x00020000>, /* GPU */ + <0x00 0x0fd80000 0x00 0x0fd80000 0x00 0x00080000>, /* GPU */ <0x00 0x20000000 0x00 0x20000000 0x00 0x0a008000>, /* Third peripheral window */ <0x00 0x30040000 0x00 0x30040000 0x00 0x00080000>, /* PRUSS-M */ <0x00 0x30101000 0x00 0x30101000 0x00 0x00010100>, /* CSI window */ diff --git a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso index 7fc73cfacadb8..1176a52d560b7 100644 --- a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso +++ b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso @@ -30,13 +30,10 @@ <&main_pktdma 0xc206 15>, /* egress slice 1 */ <&main_pktdma 0xc207 15>, /* egress slice 1 */ <&main_pktdma 0x4200 15>, /* ingress slice 0 */ - <&main_pktdma 0x4201 15>, /* ingress slice 1 */ - <&main_pktdma 0x4202 0>, /* mgmnt rsp slice 0 */ - <&main_pktdma 0x4203 0>; /* mgmnt rsp slice 1 */ + <&main_pktdma 0x4201 15>; /* ingress slice 1 */ dma-names = "tx0-0", "tx0-1", "tx0-2", "tx0-3", "tx1-0", "tx1-1", "tx1-2", "tx1-3", - "rx0", "rx1", - "rxmgm0", "rxmgm1"; + "rx0", "rx1"; firmware-name = "ti-pruss/am65x-sr2-pru0-prueth-fw.elf", "ti-pruss/am65x-sr2-rtu0-prueth-fw.elf", diff --git a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso index 996c42ec4253e..bea8efa3e9094 100644 --- a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso +++ b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso @@ -20,13 +20,13 @@ }; &main_pmx0 { - main_gpio1_exp_header_gpio_pins_default: main-gpio1-exp-header-gpio-pins-default { + main_gpio1_exp_header_gpio_pins_default: main-gpio1-exp-header-gpio-default-pins { pinctrl-single,pins = < AM64X_IOPAD(0x0220, PIN_INPUT, 7) /* (D14) SPI1_CS1.GPIO1_48 */ >; }; - main_spi1_pins_default: main-spi1-pins-default { + main_spi1_pins_default: main-spi1-default-pins { pinctrl-single,pins = < AM64X_IOPAD(0x0224, PIN_INPUT, 0) /* (C14) SPI1_CLK */ AM64X_IOPAD(0x021C, PIN_OUTPUT, 0) /* (B14) SPI1_CS0 */ @@ -35,7 +35,7 @@ >; }; - main_uart3_pins_default: main-uart3-pins-default { + main_uart3_pins_default: main-uart3-default-pins { pinctrl-single,pins = < AM64X_IOPAD(0x0048, PIN_INPUT, 2) /* (U20) GPMC0_AD3.UART3_RXD */ AM64X_IOPAD(0x004c, PIN_OUTPUT, 2) /* (U18) GPMC0_AD4.UART3_TXD */ @@ -52,7 +52,7 @@ &main_spi1 { pinctrl-names = "default"; pinctrl-0 = <&main_spi1_pins_default>; - ti,pindir-d0-out-d1-in = <1>; + ti,pindir-d0-out-d1-in; status = "okay"; }; diff --git a/arch/arm64/boot/dts/ti/k3-am69-sk.dts b/arch/arm64/boot/dts/ti/k3-am69-sk.dts index 5896e57b5b9ed..0e2d12cb051da 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am69-sk.dts @@ -236,8 +236,8 @@ main_i2c0_pins_default: main-i2c0-default-pins { pinctrl-single,pins = < - J784S4_IOPAD(0x0e0, PIN_INPUT_PULLUP, 0) /* (AN36) I2C0_SCL */ - J784S4_IOPAD(0x0e4, PIN_INPUT_PULLUP, 0) /* (AP37) I2C0_SDA */ + J784S4_IOPAD(0x0e0, PIN_INPUT, 0) /* (AN36) I2C0_SCL */ + J784S4_IOPAD(0x0e4, PIN_INPUT, 0) /* (AP37) I2C0_SDA */ >; }; @@ -416,8 +416,8 @@ mcu_i2c0_pins_default: mcu-i2c0-default-pins { pinctrl-single,pins = < - J784S4_WKUP_IOPAD(0x0a0, PIN_INPUT_PULLUP, 0) /* (M35) MCU_I2C0_SCL */ - J784S4_WKUP_IOPAD(0x0a4, PIN_INPUT_PULLUP, 0) /* (G34) MCU_I2C0_SDA */ + J784S4_WKUP_IOPAD(0x0a0, PIN_INPUT, 0) /* (M35) MCU_I2C0_SCL */ + J784S4_WKUP_IOPAD(0x0a4, PIN_INPUT, 0) /* (G34) MCU_I2C0_SDA */ >; }; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-sk.dts b/arch/arm64/boot/dts/ti/k3-j721e-sk.dts index 5e5784ef6f858..77dcc160eda3f 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-j721e-sk.dts @@ -474,6 +474,12 @@ J721E_IOPAD(0x234, PIN_INPUT, 7) /* (U3) EXT_REFCLK1.GPIO1_12 */ >; }; + + vdd_sd_dv_pins_default: vdd-sd-dv-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x1dc, PIN_OUTPUT, 7) /* (Y1) SPI1_CLK.GPIO0_118 */ + >; + }; }; &wkup_pmx0 { @@ -536,12 +542,6 @@ >; }; - vdd_sd_dv_pins_default: vdd-sd-dv-default-pins { - pinctrl-single,pins = < - J721E_IOPAD(0x1dc, PIN_OUTPUT, 7) /* (Y1) SPI1_CLK.GPIO0_118 */ - >; - }; - wkup_uart0_pins_default: wkup-uart0-default-pins { pinctrl-single,pins = < J721E_WKUP_IOPAD(0xa0, PIN_INPUT, 0) /* (J29) WKUP_UART0_RXD */ diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi index 419c1a70e028d..2834f0a8bbee0 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi @@ -270,8 +270,8 @@ main_i2c0_pins_default: main-i2c0-default-pins { pinctrl-single,pins = < - J784S4_IOPAD(0x0e0, PIN_INPUT_PULLUP, 0) /* (AN36) I2C0_SCL */ - J784S4_IOPAD(0x0e4, PIN_INPUT_PULLUP, 0) /* (AP37) I2C0_SDA */ + J784S4_IOPAD(0x0e0, PIN_INPUT, 0) /* (AN36) I2C0_SCL */ + J784S4_IOPAD(0x0e4, PIN_INPUT, 0) /* (AP37) I2C0_SDA */ >; }; diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi index 9cc0901d58fbf..c2636e624f18b 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi @@ -2378,42 +2378,6 @@ assigned-clock-parents = <&k3_clks 351 4>; }; - watchdog4: watchdog@2240000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2240000 0x00 0x100>; - clocks = <&k3_clks 352 0>; - power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 352 0>; - assigned-clock-parents = <&k3_clks 352 4>; - }; - - watchdog5: watchdog@2250000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2250000 0x00 0x100>; - clocks = <&k3_clks 353 0>; - power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 353 0>; - assigned-clock-parents = <&k3_clks 353 4>; - }; - - watchdog6: watchdog@2260000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2260000 0x00 0x100>; - clocks = <&k3_clks 354 0>; - power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 354 0>; - assigned-clock-parents = <&k3_clks 354 4>; - }; - - watchdog7: watchdog@2270000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2270000 0x00 0x100>; - clocks = <&k3_clks 355 0>; - power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 355 0>; - assigned-clock-parents = <&k3_clks 355 4>; - }; - /* * The following RTI instances are coupled with MCU R5Fs, c7x and * GPU so keeping them reserved as these will be used by their diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi index 0160fe0da9838..78fcd0c40abcf 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi @@ -6,17 +6,40 @@ */ &cbass_main { - c71_3: dsp@67800000 { - compatible = "ti,j721s2-c71-dsp"; - reg = <0x00 0x67800000 0x00 0x00080000>, - <0x00 0x67e00000 0x00 0x0000c000>; - reg-names = "l2sram", "l1dram"; - resets = <&k3_reset 40 1>; - firmware-name = "j784s4-c71_3-fw"; - ti,sci = <&sms>; - ti,sci-dev-id = <40>; - ti,sci-proc-ids = <0x33 0xff>; - status = "disabled"; + watchdog4: watchdog@2240000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2240000 0x00 0x100>; + clocks = <&k3_clks 352 0>; + power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 352 0>; + assigned-clock-parents = <&k3_clks 352 4>; + }; + + watchdog5: watchdog@2250000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2250000 0x00 0x100>; + clocks = <&k3_clks 353 0>; + power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 353 0>; + assigned-clock-parents = <&k3_clks 353 4>; + }; + + watchdog6: watchdog@2260000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2260000 0x00 0x100>; + clocks = <&k3_clks 354 0>; + power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 354 0>; + assigned-clock-parents = <&k3_clks 354 4>; + }; + + watchdog7: watchdog@2270000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2270000 0x00 0x100>; + clocks = <&k3_clks 355 0>; + power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 355 0>; + assigned-clock-parents = <&k3_clks 355 4>; }; pcie2_rc: pcie@2920000 { @@ -113,6 +136,19 @@ status = "disabled"; }; }; + + c71_3: dsp@67800000 { + compatible = "ti,j721s2-c71-dsp"; + reg = <0x00 0x67800000 0x00 0x00080000>, + <0x00 0x67e00000 0x00 0x0000c000>; + reg-names = "l2sram", "l1dram"; + resets = <&k3_reset 40 1>; + firmware-name = "j784s4-c71_3-fw"; + ti,sci = <&sms>; + ti,sci-dev-id = <40>; + ti,sci-proc-ids = <0x33 0xff>; + status = "disabled"; + }; }; &scm_conf { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi index 938b014ca9231..b55c6b2e8e0e1 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi @@ -192,11 +192,6 @@ }; firmware { - optee: optee { - compatible = "linaro,optee-tz"; - method = "smc"; - }; - zynqmp_firmware: zynqmp-firmware { compatible = "xlnx,zynqmp-firmware"; #power-domain-cells = <1>; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 1a48faad2473a..1faafc9a19462 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -211,6 +211,8 @@ CONFIG_NET_9P=y CONFIG_NET_9P_VIRTIO=y CONFIG_NFC=m CONFIG_NFC_NCI=m +CONFIG_NFC_NXP_NCI=m +CONFIG_NFC_NXP_NCI_I2C=m CONFIG_NFC_S3FWRN5_I2C=m CONFIG_PCI=y CONFIG_PCIEPORTBUS=y @@ -249,6 +251,7 @@ CONFIG_PCIE_LAYERSCAPE_GEN4=y CONFIG_PCI_ENDPOINT=y CONFIG_PCI_ENDPOINT_CONFIGFS=y CONFIG_PCI_EPF_TEST=m +CONFIG_PCI_PWRCTRL_GENERIC=m CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_FW_LOADER_USER_HELPER=y @@ -465,7 +468,9 @@ CONFIG_TOUCHSCREEN_GOODIX=m CONFIG_TOUCHSCREEN_GOODIX_BERLIN_SPI=m CONFIG_TOUCHSCREEN_ELAN=m CONFIG_TOUCHSCREEN_EDT_FT5X06=m +CONFIG_TOUCHSCREEN_HIMAX_HX83112B=m CONFIG_INPUT_MISC=y +CONFIG_INPUT_AW86927=m CONFIG_INPUT_BBNSM_PWRKEY=m CONFIG_INPUT_PM8941_PWRKEY=y CONFIG_INPUT_PM8XXX_VIBRATOR=m @@ -602,6 +607,7 @@ CONFIG_PINCTRL_DA9062=m CONFIG_PINCTRL_MAX77620=y CONFIG_PINCTRL_RK805=m CONFIG_PINCTRL_SINGLE=y +CONFIG_PINCTRL_SX150X=m CONFIG_PINCTRL_OWL=y CONFIG_PINCTRL_S700=y CONFIG_PINCTRL_S900=y @@ -616,12 +622,14 @@ CONFIG_PINCTRL_IMX8ULP=y CONFIG_PINCTRL_IMX91=y CONFIG_PINCTRL_IMX93=y CONFIG_PINCTRL_MSM=y +CONFIG_PINCTRL_GLYMUR=y CONFIG_PINCTRL_IPQ5018=y CONFIG_PINCTRL_IPQ5332=y CONFIG_PINCTRL_IPQ5424=y CONFIG_PINCTRL_IPQ8074=y CONFIG_PINCTRL_IPQ6018=y CONFIG_PINCTRL_IPQ9574=y +CONFIG_PINCTRL_KAANAPALI=y CONFIG_PINCTRL_MSM8916=y CONFIG_PINCTRL_MSM8953=y CONFIG_PINCTRL_MSM8976=y @@ -662,7 +670,6 @@ CONFIG_PINCTRL_LPASS_LPI=m CONFIG_PINCTRL_SC7280_LPASS_LPI=m CONFIG_PINCTRL_SM6115_LPASS_LPI=m CONFIG_PINCTRL_SM8250_LPASS_LPI=m -CONFIG_PINCTRL_SM8350_LPASS_LPI=m CONFIG_PINCTRL_SM8450_LPASS_LPI=m CONFIG_PINCTRL_SC8280XP_LPASS_LPI=m CONFIG_PINCTRL_SM8550_LPASS_LPI=m @@ -803,6 +810,7 @@ CONFIG_MFD_STM32_LPTIMER=m CONFIG_MFD_STM32_TIMERS=m CONFIG_MFD_WCD934X=m CONFIG_MFD_KHADAS_MCU=m +CONFIG_MFD_QCOM_PM8008=m CONFIG_REGULATOR_FIXED_VOLTAGE=y CONFIG_REGULATOR_AXP20X=y CONFIG_REGULATOR_BD718XX=y @@ -828,6 +836,7 @@ CONFIG_REGULATOR_PCA9450=y CONFIG_REGULATOR_PF8X00=y CONFIG_REGULATOR_PFUZE100=y CONFIG_REGULATOR_PWM=y +CONFIG_REGULATOR_QCOM_PM8008=m CONFIG_REGULATOR_QCOM_REFGEN=m CONFIG_REGULATOR_QCOM_RPMH=y CONFIG_REGULATOR_QCOM_SMD_RPM=y @@ -934,11 +943,16 @@ CONFIG_DRM_PANEL_LVDS=m CONFIG_DRM_PANEL_SIMPLE=m CONFIG_DRM_PANEL_EDP=m CONFIG_DRM_PANEL_HIMAX_HX8279=m +CONFIG_DRM_PANEL_HIMAX_HX83112A=m +CONFIG_DRM_PANEL_HIMAX_HX83112B=m CONFIG_DRM_PANEL_ILITEK_ILI9882T=m CONFIG_DRM_PANEL_KHADAS_TS050=m CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m +CONFIG_DRM_PANEL_NOVATEK_NT36672A=m CONFIG_DRM_PANEL_NOVATEK_NT36672E=m +CONFIG_DRM_PANEL_NOVATEK_NT37801=m CONFIG_DRM_PANEL_RAYDIUM_RM67191=m +CONFIG_DRM_PANEL_RAYDIUM_RM692E5=m CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20=m CONFIG_DRM_PANEL_SITRONIX_ST7703=m CONFIG_DRM_PANEL_STARTEK_KD070FHFID015=m @@ -947,7 +961,9 @@ CONFIG_DRM_PANEL_VISIONOX_VTDR6130=m CONFIG_DRM_DISPLAY_CONNECTOR=m CONFIG_DRM_FSL_LDB=m CONFIG_DRM_ITE_IT6263=m +CONFIG_DRM_LONTIUM_LT8713SX=m CONFIG_DRM_LONTIUM_LT8912B=m +CONFIG_DRM_LONTIUM_LT9211=m CONFIG_DRM_LONTIUM_LT9611=m CONFIG_DRM_LONTIUM_LT9611UXC=m CONFIG_DRM_ITE_IT66121=m @@ -992,6 +1008,7 @@ CONFIG_FB=y CONFIG_FB_EFI=y CONFIG_FB_MODE_HELPERS=y CONFIG_BACKLIGHT_PWM=m +CONFIG_BACKLIGHT_QCOM_WLED=m CONFIG_BACKLIGHT_LP855X=m CONFIG_LOGO=y # CONFIG_LOGO_LINUX_MONO is not set @@ -1001,7 +1018,10 @@ CONFIG_SND=m CONFIG_SND_ALOOP=m CONFIG_SND_HDA_TEGRA=m CONFIG_SND_HDA_CODEC_HDMI=m +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_USB_AUDIO_QMI=m CONFIG_SND_SOC=m +CONFIG_SND_SOC_USB=m CONFIG_SND_BCM2835_SOC_I2S=m CONFIG_SND_SOC_FSL_ASRC=m CONFIG_SND_SOC_FSL_MICFIL=m @@ -1025,6 +1045,7 @@ CONFIG_SND_MESON_AXG_SOUND_CARD=m CONFIG_SND_MESON_GX_SOUND_CARD=m CONFIG_SND_SOC_QCOM=m CONFIG_SND_SOC_APQ8016_SBC=m +CONFIG_SND_SOC_QDSP6_USB=m CONFIG_SND_SOC_MSM8996=m CONFIG_SND_SOC_SDM845=m CONFIG_SND_SOC_SM8250=m @@ -1071,6 +1092,7 @@ CONFIG_SND_SOC_J721E_EVM=m CONFIG_SND_SOC_AK4613=m CONFIG_SND_SOC_AK4619=m CONFIG_SND_SOC_DA7213=m +CONFIG_SND_SOC_DMIC=m CONFIG_SND_SOC_ES7134=m CONFIG_SND_SOC_ES7241=m CONFIG_SND_SOC_ES8316=m @@ -1091,6 +1113,7 @@ CONFIG_SND_SOC_TLV320AIC32X4_I2C=m CONFIG_SND_SOC_TLV320AIC3X_I2C=m CONFIG_SND_SOC_WCD9335=m CONFIG_SND_SOC_WCD934X=m +CONFIG_SND_SOC_WCD937X_SDW=m CONFIG_SND_SOC_WCD939X_SDW=m CONFIG_SND_SOC_WM8524=m CONFIG_SND_SOC_WM8904=m @@ -1117,6 +1140,7 @@ CONFIG_USB_OTG=y CONFIG_USB_XHCI_HCD=y CONFIG_USB_XHCI_PCI_RENESAS=m CONFIG_USB_XHCI_RZV2M=y +CONFIG_USB_XHCI_SIDEBAND=y CONFIG_USB_XHCI_TEGRA=y CONFIG_USB_BRCMSTB=m CONFIG_USB_EHCI_HCD=y @@ -1188,6 +1212,7 @@ CONFIG_TYPEC_MUX_GPIO_SBU=m CONFIG_TYPEC_MUX_IT5205=m CONFIG_TYPEC_MUX_NB7VPQ904M=m CONFIG_TYPEC_MUX_PS883X=m +CONFIG_TYPEC_MUX_PTN36502=m CONFIG_TYPEC_MUX_WCD939X_USBSS=m CONFIG_TYPEC_DP_ALTMODE=m CONFIG_MMC=y @@ -1232,12 +1257,14 @@ CONFIG_SCSI_UFS_EXYNOS=y CONFIG_SCSI_UFS_ROCKCHIP=y CONFIG_NEW_LEDS=y CONFIG_LEDS_CLASS=y +CONFIG_LEDS_CLASS_FLASH=m CONFIG_LEDS_CLASS_MULTICOLOR=m CONFIG_LEDS_LM3692X=m CONFIG_LEDS_PCA9532=m CONFIG_LEDS_GPIO=y CONFIG_LEDS_PWM=y CONFIG_LEDS_SYSCON=y +CONFIG_LEDS_QCOM_FLASH=m CONFIG_LEDS_QCOM_LPG=m CONFIG_LEDS_TRIGGER_TIMER=y CONFIG_LEDS_TRIGGER_DISK=y @@ -1329,6 +1356,7 @@ CONFIG_CROS_EC_RPMSG=m CONFIG_CROS_EC_SPI=y CONFIG_CROS_KBD_LED_BACKLIGHT=m CONFIG_CROS_EC_CHARDEV=m +CONFIG_EC_QCOM_HAMOA=m CONFIG_COMMON_CLK_RK808=y CONFIG_COMMON_CLK_SCMI=y CONFIG_COMMON_CLK_SCPI=y @@ -1363,12 +1391,18 @@ CONFIG_COMMON_CLK_MT8192_SCP_ADSP=y CONFIG_COMMON_CLK_MT8192_VDECSYS=y CONFIG_COMMON_CLK_MT8192_VENCSYS=y CONFIG_COMMON_CLK_QCOM=y +CONFIG_CLK_GLYMUR_DISPCC=y +CONFIG_CLK_GLYMUR_GCC=y +CONFIG_CLK_GLYMUR_TCSRCC=y +CONFIG_CLK_KAANAPALI_GCC=y CONFIG_CLK_X1E80100_CAMCC=m CONFIG_CLK_X1E80100_DISPCC=m CONFIG_CLK_X1E80100_GCC=y CONFIG_CLK_X1E80100_GPUCC=m CONFIG_CLK_X1E80100_TCSRCC=y +CONFIG_CLK_X1P42100_CAMCC=m CONFIG_CLK_X1P42100_GPUCC=m +CONFIG_CLK_X1P42100_VIDEOCC=m CONFIG_CLK_QCM2290_GPUCC=m CONFIG_QCOM_A53PLL=y CONFIG_QCOM_CLK_APCS_MSM8916=y @@ -1386,6 +1420,7 @@ CONFIG_IPQ_GCC_8074=y CONFIG_IPQ_GCC_9574=y CONFIG_IPQ_NSSCC_9574=m CONFIG_MSM_GCC_8916=y +CONFIG_MSM_GCC_8953=y CONFIG_MSM_MMCC_8994=m CONFIG_MSM_GCC_8994=y CONFIG_MSM_GCC_8996=y @@ -1425,22 +1460,26 @@ CONFIG_SDM_VIDEOCC_845=y CONFIG_SDM_DISPCC_845=y CONFIG_SDM_LPASSCC_845=m CONFIG_SDX_GCC_75=y +CONFIG_SM_CAMCC_6350=m CONFIG_SM_CAMCC_8250=m CONFIG_SM_CAMCC_8550=m CONFIG_SM_CAMCC_8650=m CONFIG_SM_DISPCC_6115=m CONFIG_SM_DISPCC_8250=y +CONFIG_SM_DISPCC_6350=m CONFIG_SM_DISPCC_8450=m CONFIG_SM_DISPCC_8550=m CONFIG_SM_DISPCC_8750=m CONFIG_SM_GCC_4450=y CONFIG_SM_GCC_6115=y +CONFIG_SM_GCC_6350=y CONFIG_SM_GCC_8350=y CONFIG_SM_GCC_8450=y CONFIG_SM_GCC_8550=y CONFIG_SM_GCC_8650=y CONFIG_SM_GCC_8750=y CONFIG_SM_GPUCC_6115=m +CONFIG_SM_GPUCC_6350=m CONFIG_SM_GPUCC_8150=y CONFIG_SM_GPUCC_8250=y CONFIG_SM_GPUCC_8350=m @@ -1451,6 +1490,7 @@ CONFIG_SM_TCSRCC_8550=y CONFIG_SM_TCSRCC_8650=y CONFIG_SM_TCSRCC_8750=m CONFIG_SA_VIDEOCC_8775P=m +CONFIG_SM_VIDEOCC_6350=m CONFIG_SM_VIDEOCC_8250=y CONFIG_SM_VIDEOCC_8550=m CONFIG_QCOM_HFPLL=y @@ -1641,6 +1681,7 @@ CONFIG_PHY_QCOM_QMP=m CONFIG_PHY_QCOM_QUSB2=m CONFIG_PHY_QCOM_EUSB2_REPEATER=m CONFIG_PHY_QCOM_M31_USB=m +CONFIG_PHY_QCOM_M31_EUSB=m CONFIG_PHY_QCOM_USB_HS=m CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2=m CONFIG_PHY_QCOM_USB_HS_28NM=m @@ -1706,6 +1747,7 @@ CONFIG_OF_FPGA_REGION=m CONFIG_OF_OVERLAY=y CONFIG_TEE=y CONFIG_OPTEE=y +CONFIG_QCOMTEE=y CONFIG_MUX_GPIO=m CONFIG_MUX_MMIO=y CONFIG_SLIMBUS=m @@ -1718,7 +1760,10 @@ CONFIG_INTERCONNECT_IMX8MN=m CONFIG_INTERCONNECT_IMX8MQ=m CONFIG_INTERCONNECT_IMX8MP=y CONFIG_INTERCONNECT_QCOM=y +CONFIG_INTERCONNECT_QCOM_GLYMUR=y +CONFIG_INTERCONNECT_QCOM_KAANAPALI=y CONFIG_INTERCONNECT_QCOM_MSM8916=m +CONFIG_INTERCONNECT_QCOM_MSM8953=y CONFIG_INTERCONNECT_QCOM_MSM8996=y CONFIG_INTERCONNECT_QCOM_OSM_L3=m CONFIG_INTERCONNECT_QCOM_QCM2290=y @@ -1734,6 +1779,7 @@ CONFIG_INTERCONNECT_QCOM_SC8280XP=y CONFIG_INTERCONNECT_QCOM_SDM845=y CONFIG_INTERCONNECT_QCOM_SDX75=y CONFIG_INTERCONNECT_QCOM_SM6115=y +CONFIG_INTERCONNECT_QCOM_SM6350=y CONFIG_INTERCONNECT_QCOM_SM8150=y CONFIG_INTERCONNECT_QCOM_SM8250=y CONFIG_INTERCONNECT_QCOM_SM8350=y diff --git a/arch/arm64/configs/prune.config b/arch/arm64/configs/prune.config new file mode 100644 index 0000000000000..7f505d6bcab3b --- /dev/null +++ b/arch/arm64/configs/prune.config @@ -0,0 +1,323 @@ +# CONFIG_ARCH_ACTIONS is not set +# CONFIG_ARCH_AIROHA is not set +# CONFIG_ARCH_SUNXI is not set +# CONFIG_ARCH_ALPINE is not set +# CONFIG_ARCH_APPLE is not set +# CONFIG_ARCH_BCM is not set +# CONFIG_ARCH_BCM2835 is not set +# CONFIG_ARCH_BCM_IPROC is not set +# CONFIG_ARCH_BCMBCA is not set +# CONFIG_ARCH_BRCMSTB is not set +# CONFIG_ARCH_BERLIN is not set +# CONFIG_ARCH_BLAIZE is not set +# CONFIG_ARCH_EXYNOS is not set +# CONFIG_ARCH_SPARX5 is not set +# CONFIG_ARCH_K3 is not set +# CONFIG_ARCH_LG1K is not set +# CONFIG_ARCH_HISI is not set +# CONFIG_ARCH_KEEMBAY is not set +# CONFIG_ARCH_MEDIATEK is not set +# CONFIG_ARCH_MESON is not set +# CONFIG_ARCH_MVEBU is not set +# CONFIG_ARCH_NXP is not set +# CONFIG_ARCH_MA35 is not set +# CONFIG_ARCH_NPCM is not set +# CONFIG_ARCH_REALTEK is not set +# CONFIG_ARCH_RENESAS is not set +# CONFIG_ARCH_ROCKCHIP is not set +# CONFIG_ARCH_SEATTLE is not set +# CONFIG_ARCH_INTEL_SOCFPGA is not set +# CONFIG_ARCH_STM32 is not set +# CONFIG_ARCH_SYNQUACER is not set +# CONFIG_ARCH_TEGRA is not set +# CONFIG_ARCH_TESLA_FSD is not set +# CONFIG_ARCH_SPRD is not set +# CONFIG_ARCH_THUNDER is not set +# CONFIG_ARCH_THUNDER2 is not set +# CONFIG_ARCH_UNIPHIER is not set +# CONFIG_ARCH_VEXPRESS is not set +# CONFIG_ARCH_VISCONTI is not set +# CONFIG_ARCH_XGENE is not set +# CONFIG_ARCH_ZYNQMP is not set +# CONFIG_MOUSE_PS2 is not set +# CONFIG_RODATA_FULL_DEFAULT_ENABLED is not set +# CONFIG_NET_DSA_TAG_OCELOT is not set +# CONFIG_NET_DSA_TAG_OCELOT_8021Q is not set +# CONFIG_BT_HCIBTUSB_MTK is not set +# CONFIG_BT_HCIUART_BCM is not set +# CONFIG_BT_HCIUART_MRVL is not set +# CONFIG_BT_MRVL is not set +# CONFIG_BT_MRVL_SDIO is not set +# CONFIG_PCIE_ALTERA is not set +# CONFIG_PCIE_ALTERA_MSI is not set +# CONFIG_PCI_HOST_THUNDER_PEM is not set +# CONFIG_PCI_HOST_THUNDER_ECAM is not set +# CONFIG_PCI_XGENE is not set +# CONFIG_PCI_MESON is not set +# CONFIG_PCI_HISI is not set +# CONFIG_PCIE_KIRIN is not set +# CONFIG_BRCMSTB_GISB_ARB is not set +# CONFIG_VEXPRESS_CONFIG is not set +# CONFIG_GNSS_MTK_SERIAL is not set +# CONFIG_MTD_CFI_INTELEXT is not set +# CONFIG_MTD_CFI_AMDSTD is not set +# CONFIG_MTD_CFI_STAA is not set +# CONFIG_MTD_NAND_DENALI_DT is not set +# CONFIG_MTD_NAND_BRCMNAND is not set +# CONFIG_MTD_NAND_BRCMNAND_BCMBCA is not set +# CONFIG_MTD_NAND_BRCMNAND_BRCMSTB is not set +# CONFIG_MTD_NAND_BRCMNAND_IPROC is not set +# CONFIG_B53_SRAB_DRIVER is not set +# CONFIG_NET_DSA_BCM_SF2 is not set +# CONFIG_AMD_XGBE is not set +# CONFIG_BCMGENET is not set +# CONFIG_BNX2X is not set +# CONFIG_SYSTEMPORT is not set +# CONFIG_MACB is not set +# CONFIG_THUNDER_NIC_PF is not set +# CONFIG_HIX5HD2_GMAC is not set +# CONFIG_HNS_DSAF is not set +# CONFIG_HNS_ENET is not set +# CONFIG_HNS3 is not set +# CONFIG_HNS3_HCLGE is not set +# CONFIG_HNS3_ENET is not set +# CONFIG_MVMDIO is not set +# CONFIG_SKY2 is not set +# CONFIG_MLX4_EN is not set +# CONFIG_MLX5_CORE is not set +# CONFIG_MLX5_CORE_EN is not set +# CONFIG_R8169 is not set +# CONFIG_SMC91X is not set +# CONFIG_SMSC911X is not set +# CONFIG_BROADCOM_PHY is not set +# CONFIG_BCM54140_PHY is not set +# CONFIG_MICROSEMI_PHY is not set +# CONFIG_ROCKCHIP_PHY is not set +# CONFIG_DP83867_PHY is not set +# CONFIG_DP83TD510_PHY is not set +# CONFIG_VITESSE_PHY is not set +# CONFIG_CAN_FLEXCAN is not set +# CONFIG_BRCMFMAC is not set +# CONFIG_MWIFIEX is not set +# CONFIG_MWIFIEX_SDIO is not set +# CONFIG_MWIFIEX_PCIE is not set +# CONFIG_MT7921E is not set +# CONFIG_WL18XX is not set +# CONFIG_WLCORE_SDIO is not set +# CONFIG_KEYBOARD_CROS_EC is not set +# CONFIG_KEYBOARD_MTK_PMIC is not set +# CONFIG_MOUSE_ELAN_I2C is not set +# CONFIG_TOUCHSCREEN_ATMEL_MXT is not set +# CONFIG_TOUCHSCREEN_GOODIX is not set +# CONFIG_TOUCHSCREEN_ELAN is not set +# CONFIG_TOUCHSCREEN_EDT_FT5X06 is not set +# CONFIG_INPUT_TPS65219_PWRBUTTON is not set +# CONFIG_SERIAL_XILINX_PS_UART is not set +# CONFIG_SERIAL_XILINX_PS_UART_CONSOLE is not set +# CONFIG_SERIAL_FSL_LPUART is not set +# CONFIG_SERIAL_FSL_LPUART_CONSOLE is not set +# CONFIG_SERIAL_FSL_LINFLEXUART is not set +# CONFIG_SERIAL_FSL_LINFLEXUART_CONSOLE is not set +# CONFIG_TCG_TIS_SPI_CR50 is not set +# CONFIG_TCG_TIS_I2C_CR50 is not set +# CONFIG_TCG_TIS_I2C_INFINEON is not set +# CONFIG_I2C_CADENCE is not set +# CONFIG_I2C_DESIGNWARE_PLATFORM is not set +# CONFIG_I2C_RK3X is not set +# CONFIG_I2C_CROS_EC_TUNNEL is not set +# CONFIG_SPI_CADENCE_QUADSPI is not set +# CONFIG_SPI_DESIGNWARE is not set +# CONFIG_SPI_DW_DMA is not set +# CONFIG_SPI_DW_MMIO is not set +# CONFIG_PINCTRL_MAX77620 is not set +# CONFIG_CHARGER_MT6360 is not set +# CONFIG_CHARGER_BQ25890 is not set +# CONFIG_CHARGER_BQ25980 is not set +# CONFIG_SENSORS_JC42 is not set +# CONFIG_SENSORS_LM75 is not set +# CONFIG_SENSORS_LM90 is not set +# CONFIG_SENSORS_INA2XX is not set +# CONFIG_SENSORS_INA3221 is not set +# CONFIG_ARM_SP805_WATCHDOG is not set +# CONFIG_DW_WATCHDOG is not set +# CONFIG_PM8916_WATCHDOG is not set +# CONFIG_MFD_BD9571MWV is not set +# CONFIG_MFD_AXP20X_I2C is not set +# CONFIG_MFD_HI6421_PMIC is not set +# CONFIG_MFD_MAX77620 is not set +# CONFIG_MFD_MT6360 is not set +# CONFIG_MFD_MT6397 is not set +# CONFIG_MFD_RK8XX_I2C is not set +# CONFIG_MFD_RK8XX_SPI is not set +# CONFIG_MFD_SEC_CORE is not set +# CONFIG_MFD_TI_AM335X_TSCADC is not set +# CONFIG_MFD_TPS65219 is not set +# CONFIG_MFD_WM8994 is not set +# CONFIG_MFD_ROHM_BD718XX is not set +# CONFIG_REGULATOR_AXP20X is not set +# CONFIG_REGULATOR_BD718XX is not set +# CONFIG_REGULATOR_BD9571MWV is not set +# CONFIG_REGULATOR_CROS_EC is not set +# CONFIG_REGULATOR_FAN53555 is not set +# CONFIG_REGULATOR_HI6421V530 is not set +# CONFIG_REGULATOR_MAX77620 is not set +# CONFIG_REGULATOR_MAX8973 is not set +# CONFIG_REGULATOR_MP8859 is not set +# CONFIG_REGULATOR_MT6315 is not set +# CONFIG_REGULATOR_MT6357 is not set +# CONFIG_REGULATOR_MT6358 is not set +# CONFIG_REGULATOR_MT6359 is not set +# CONFIG_REGULATOR_MT6360 is not set +# CONFIG_REGULATOR_MT6397 is not set +# CONFIG_REGULATOR_PFUZE100 is not set +# CONFIG_REGULATOR_RK808 is not set +# CONFIG_REGULATOR_S2MPS11 is not set +# CONFIG_REGULATOR_TPS65132 is not set +# CONFIG_REGULATOR_TPS65219 is not set +# CONFIG_REGULATOR_RK808 is not set +# CONFIG_REGULATOR_S2MPS11 is not set +# CONFIG_REGULATOR_TPS65132 is not set +# CONFIG_REGULATOR_TPS65219 is not set +# CONFIG_MEDIA_ANALOG_TV_SUPPORT is not set +# CONFIG_MEDIA_DIGITAL_TV_SUPPORT is not set +# CONFIG_VIDEO_IMX219 is not set +# CONFIG_VIDEO_OV5640 is not set +# CONFIG_VIDEO_OV5645 is not set +# CONFIG_DRM_I2C_NXP_TDA998X is not set +# CONFIG_DRM_MALI_DISPLAY is not set +# CONFIG_DRM_KOMEDA is not set +# CONFIG_DRM_PANEL_BOE_TV101WUM_NL6 is not set +# CONFIG_DRM_PANEL_MANTIX_MLAF057WE51 is not set +# CONFIG_DRM_PANEL_RAYDIUM_RM67191 is not set +# CONFIG_DRM_PANEL_SITRONIX_ST7703 is not set +# CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA is not set +# CONFIG_DRM_PANEL_VISIONOX_VTDR6130 is not set +# CONFIG_DRM_LONTIUM_LT8912B is not set +# CONFIG_DRM_NWL_MIPI_DSI is not set +# CONFIG_DRM_PARADE_PS8640 is not set +# CONFIG_DRM_SAMSUNG_DSIM is not set +# CONFIG_DRM_SII902X is not set +# CONFIG_DRM_SIMPLE_BRIDGE is not set +# CONFIG_DRM_THINE_THC63LVD1024 is not set +# CONFIG_DRM_TOSHIBA_TC358768 is not set +# CONFIG_DRM_TI_TFP410 is not set +# CONFIG_DRM_TI_SN65DSI83 is not set +# CONFIG_DRM_TI_SN65DSI86 is not set +# CONFIG_DRM_I2C_ADV7511_AUDIO is not set +# CONFIG_DRM_CDNS_MHDP8546 is not set +# CONFIG_DRM_ETNAVIV is not set +# CONFIG_DRM_HISI_HIBMC is not set +# CONFIG_DRM_HISI_KIRIN is not set +# CONFIG_DRM_PL111 is not set +# CONFIG_DRM_LIMA is not set +# CONFIG_DRM_PANFROST is not set +# CONFIG_DRM_TIDSS is not set +# CONFIG_BACKLIGHT_LP855X is not set +# CONFIG_SND_SOC_FSL_ASRC is not set +# CONFIG_SND_SOC_FSL_SAI is not set +# CONFIG_SND_SOC_FSL_AUDMIX is not set +# CONFIG_SND_SOC_FSL_SSI is not set +# CONFIG_SND_SOC_FSL_SPDIF is not set +# CONFIG_SND_SOC_FSL_ESAI is not set +# CONFIG_SND_SOC_FSL_MICFIL is not set +# CONFIG_SND_SOC_FSL_EASRC is not set +# CONFIG_SND_SOC_IMX_AUDMUX is not set +# CONFIG_SND_SOC_AK4613 is not set +# CONFIG_SND_SOC_ES7134 is not set +# CONFIG_SND_SOC_ES7241 is not set +# CONFIG_SND_SOC_ES8316 is not set +# CONFIG_SND_SOC_GTM601 is not set +# CONFIG_SND_SOC_PCM3168A_I2C is not set +# CONFIG_SND_SOC_RT5640 is not set +# CONFIG_SND_SOC_RT5659 is not set +# CONFIG_SND_SOC_TAS2552 is not set +# CONFIG_SND_SOC_TAS571X is not set +# CONFIG_SND_SOC_TLV320AIC31XX is not set +# CONFIG_SND_SOC_TLV320AIC32X4_I2C is not set +# CONFIG_SND_SOC_TLV320AIC3X_I2C is not set +# CONFIG_SND_SOC_TS3A227E is not set +# CONFIG_SND_SOC_WM8524 is not set +# CONFIG_SND_SOC_WM8904 is not set +# CONFIG_SND_SOC_WM8960 is not set +# CONFIG_SND_SOC_WM8962 is not set +# CONFIG_SND_SOC_WM8978 is not set +# CONFIG_SND_SOC_MT6358 is not set +# CONFIG_SND_SOC_NAU8822 is not set +# CONFIG_MMC_SDHCI_OF_ARASAN is not set +# CONFIG_MMC_SDHCI_OF_DWCMSHC is not set +# CONFIG_MMC_SDHCI_CADENCE is not set +# CONFIG_MMC_SDHCI_F_SDH30 is not set +# CONFIG_MMC_DW_EXYNOS is not set +# CONFIG_MMC_DW_HI3798CV200 is not set +# CONFIG_MMC_DW_K3 is not set +# CONFIG_MMC_MTK is not set +# CONFIG_MMC_SDHCI_XENON is not set +# CONFIG_MMC_SDHCI_AM654 is not set +# CONFIG_RTC_DRV_DS1307 is not set +# CONFIG_RTC_DRV_HYM8563 is not set +# CONFIG_RTC_DRV_MAX77686 is not set +# CONFIG_RTC_DRV_RK808 is not set +# CONFIG_RTC_DRV_PCF85063 is not set +# CONFIG_RTC_DRV_PCF85363 is not set +# CONFIG_RTC_DRV_M41T80 is not set +# CONFIG_RTC_DRV_BQ32K is not set +# CONFIG_RTC_DRV_RX8581 is not set +# CONFIG_RTC_DRV_RV3028 is not set +# CONFIG_RTC_DRV_RV8803 is not set +# CONFIG_RTC_DRV_S5M is not set +# CONFIG_RTC_DRV_DS3232 is not set +# CONFIG_RTC_DRV_PCF2127 is not set +# CONFIG_RTC_DRV_CROS_EC is not set +# CONFIG_RTC_DRV_MT6397 is not set +# CONFIG_BCM_SBA_RAID is not set +# CONFIG_FSL_EDMA is not set +# CONFIG_MV_XOR_V2 is not set +# CONFIG_CHROME_PLATFORMS is not set +# CONFIG_CROS_EC is not set +# CONFIG_CROS_EC_I2C is not set +# CONFIG_CROS_EC_RPMSG is not set +# CONFIG_CROS_EC_SPI is not set +# CONFIG_CROS_EC_CHARDEV is not set +# CONFIG_CLK_VEXPRESS_OSC is not set +# CONFIG_COMMON_CLK_RK808 is not set +# CONFIG_COMMON_CLK_CS2000_CP is not set +# CONFIG_COMMON_CLK_S2MPS11 is not set +# CONFIG_COMMON_CLK_XGENE is not set +# CONFIG_COMMON_CLK_RS9_PCIE is not set +# CONFIG_COMMON_CLK_VC5 is not set +# CONFIG_COMMON_CLK_BD718XX is not set +# CONFIG_SOC_TI is not set +# CONFIG_TI_ADS1015 is not set +# CONFIG_TI_AM335X_ADC is not set +# CONFIG_IIO_CROS_EC_SENSORS_CORE is not set +# CONFIG_IIO_CROS_EC_SENSORS is not set +# CONFIG_IIO_ST_LSM6DSX is not set +# CONFIG_IIO_CROS_EC_LIGHT_PROX is not set +# CONFIG_SENSORS_ISL29018 is not set +# CONFIG_VCNL4000 is not set +# CONFIG_IIO_ST_MAGN_3AXIS is not set +# CONFIG_IIO_CROS_EC_BARO is not set +# CONFIG_MPL3115 is not set +# CONFIG_PWM_CROS_EC is not set +# CONFIG_PHY_CADENCE_TORRENT is not set +# CONFIG_PHY_CADENCE_SIERRA is not set +# CONFIG_CRYPTO_DEV_HISI_SEC2 is not set +# CONFIG_CRYPTO_DEV_HISI_ZIP is not set +# CONFIG_CRYPTO_DEV_HISI_HPRE is not set +# CONFIG_CRYPTO_DEV_HISI_TRNG is not set +# CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set +# CONFIG_DEVMEM is not set +# CONFIG_STRICT_DEVMEM is not set +# CONFIG_ACORN_PARTITION is not set +# CONFIG_AIX_PARTITION is not set +# CONFIG_AMIGA_PARTITION is not set +# CONFIG_ATARI_PARTITION is not set +# CONFIG_CMDLINE_PARTITION is not set +# CONFIG_KARMA_PARTITION is not set +# CONFIG_LDM_PARTITION is not set +# CONFIG_MAC_PARTITION is not set +# CONFIG_OSF_PARTITION is not set +# CONFIG_SGI_PARTITION is not set +# CONFIG_SUN_PARTITION is not set +# CONFIG_SYSV68_PARTITION is not set +# CONFIG_ULTRIX_PARTITION is not set diff --git a/arch/arm64/configs/qcom.config b/arch/arm64/configs/qcom.config new file mode 100644 index 0000000000000..0687934d8f9e6 --- /dev/null +++ b/arch/arm64/configs/qcom.config @@ -0,0 +1,113 @@ +# qcom.config for Qualcomm-specific kernel configuration +# +# $ make ARCH=arm64 defconfig qcom.config +# +# Keep alphabetically sorted +CONFIG_ARM64_SW_TTBR0_PAN=y +CONFIG_ATH_REG_DYNAMIC_USER_REG_HINTS=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_CFG80211_CERTIFICATION_ONUS=y +CONFIG_CMA=y +CONFIG_CONNECTOR=y +CONFIG_CORESIGHT_CORESIGHT_TNOC=m +CONFIG_CORESIGHT_CTCU=m +CONFIG_CORESIGHT_DUMMY=m +CONFIG_CORESIGHT_SOURCE_ETM4X=m +CONFIG_QCOM_TGU=m +CONFIG_CORESIGHT_TPDM=m +CONFIG_CPU_IDLE_THERMAL=y +# CONFIG_DEBUG_INFO_REDUCED is not set +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_CMA=y +CONFIG_DMABUF_HEAPS_SYSTEM=y +CONFIG_EDAC_QCOM=m +CONFIG_EXPERT=y +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_FW_LOADER_COMPRESS_ZSTD=y +CONFIG_GUNYAH_WATCHDOG=y +CONFIG_I6300ESB_WDT=y +CONFIG_IDLE_INJECT=y +CONFIG_INPUT_UINPUT=y +CONFIG_KPROBES=y +CONFIG_MACVLAN=y +CONFIG_MACVTAP=y +CONFIG_PCI_PWRCTRL_TC9563=y +CONFIG_PM_SLEEP=y +CONFIG_PM_SLEEP_SMP=y +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_WAKELOCKS_LIMIT=0 +# CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM=y +CONFIG_POWERCAP=y +CONFIG_PROC_EVENTS=y +CONFIG_QCA808X_PHY=m +CONFIG_QCOM_QMI_COOLING=y +CONFIG_REMOTEPROC_THERMAL=y +CONFIG_SCHED_DEBUG=y +CONFIG_SCHEDSTATS=y +CONFIG_STM_PROTO_BASIC=m +CONFIG_STM_PROTO_SYS_T=m +CONFIG_STM_SOURCE_CONSOLE=m +CONFIG_STM_SOURCE_FTRACE=m +CONFIG_STM_SOURCE_HEARTBEAT=m +CONFIG_SENSORS_AMC6821=y +CONFIG_SENSORS_EMC2305=y +CONFIG_SENSORS_QCOM_BCL=y +CONFIG_TRACE_MMIO_ACCESS=y +CONFIG_UCLAMP_TASK_GROUP=y +CONFIG_UCLAMP_TASK=y +CONFIG_UHID=m +CONFIG_USB_CONN_GPIO=m +CONFIG_USB_HSIC_USB4604=m +CONFIG_VCPU_STALL_DETECTOR=y +CONFIG_VHOST_NET=y +CONFIG_VHOST_VSOCK=y +CONFIG_VSOCKETS=y +# CONFIG_VSOCKETS_DIAG is not set +# CONFIG_VSOCKETS_LOOPBACK is not set +# CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_NOOP is not set +CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_PANIC=y +CONFIG_WATCHDOG_PRETIMEOUT_GOV_NOOP=y +CONFIG_WATCHDOG_PRETIMEOUT_GOV_PANIC=y +CONFIG_WATCHDOG_PRETIMEOUT_GOV_SEL=m +CONFIG_WATCHDOG_PRETIMEOUT_GOV=y +CONFIG_WATCHDOG_SYSFS=y +CONFIG_ZRAM=y +# Support CRYPTO APIs +CONFIG_CRYPTO_USER_API=y +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_USER_API_SKCIPHER=y +CONFIG_CRYPTO_USER_API_AEAD=y +# Support ICE/UFS crypto driver +CONFIG_BLK_INLINE_ENCRYPTION=y +CONFIG_QCOM_INLINE_CRYPTO_ENGINE=m +CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y +CONFIG_SCSI_UFS_CRYPTO=y +CONFIG_FS_ENCRYPTION=y +CONFIG_MMC_CRYPTO=y +# +# SCSI support type (disk, tape, CD-ROM) +# +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_SG=m +CONFIG_BLK_DEV_BSG=y +# +# SCSI Transports +# +CONFIG_SCSI_AACRAID=m +CONFIG_SCSI_SMARTPQI=m +# Support performance profiling +CONFIG_FTRACE=y +# Kubernetes support +CONFIG_CFS_BANDWIDTH=y +CONFIG_CGROUP_FAVOR_DYNMODS=y +CONFIG_IPC_NS=y +CONFIG_NAMESPACES=y +CONFIG_NET_NS=y +CONFIG_PID_NS=y +CONFIG_UTS_NS=y diff --git a/arch/arm64/configs/qcom_debug.config b/arch/arm64/configs/qcom_debug.config new file mode 100644 index 0000000000000..1399972795b68 --- /dev/null +++ b/arch/arm64/configs/qcom_debug.config @@ -0,0 +1,4 @@ +CONFIG_QCOM_DCC=y +CONFIG_QCOM_DCC_DEV=y +CONFIG_QCOM_MEMORY_DUMP_V2=y +CONFIG_QCOM_MEMORY_DUMP_DEV=y diff --git a/arch/arm64/configs/rt.config b/arch/arm64/configs/rt.config new file mode 100644 index 0000000000000..ca92615d38a17 --- /dev/null +++ b/arch/arm64/configs/rt.config @@ -0,0 +1,3 @@ +# rt.config for Real time kernel features +CONFIG_EXPERT=y +CONFIG_PREEMPT_RT=y diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 4995b6e223350..9625570dc3535 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -133,7 +133,7 @@ static int ghash_finup(struct shash_desc *desc, const u8 *src, u8 buf[GHASH_BLOCK_SIZE] = {}; memcpy(buf, src, len); - ghash_do_simd_update(1, ctx->digest, src, key, NULL, + ghash_do_simd_update(1, ctx->digest, buf, key, NULL, pmull_ghash_update_p8); memzero_explicit(buf, sizeof(buf)); } diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 9d769291a3067..121210b7ffd0b 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -66,6 +66,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); case ARM64_WORKAROUND_SPECULATIVE_SSBS: return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386); + case ARM64_WORKAROUND_4193714: + return IS_ENABLED(CONFIG_ARM64_ERRATUM_4193714); case ARM64_MPAM: /* * KVM MPAM support doesn't rely on the host kernel supporting MPAM. diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 9b00b75acbf29..18f98fb7ee783 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -98,6 +98,7 @@ #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 #define ARM_CPU_PART_NEOVERSE_N3 0xD8E +#define ARM_CPU_PART_C1_PRO 0xD8B #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -189,6 +190,7 @@ #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) +#define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index b8cf0ea43cc05..0fa8d1d5722e0 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -428,6 +428,24 @@ static inline size_t sme_state_size(struct task_struct const *task) return __sme_state_size(task_get_sme_vl(task)); } +void sme_enable_dvmsync(void); +void sme_set_active(void); +void sme_clear_active(void); + +static inline void sme_enter_from_user_mode(void) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) && + test_thread_flag(TIF_SME)) + sme_clear_active(); +} + +static inline void sme_exit_to_user_mode(void) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) && + test_thread_flag(TIF_SME)) + sme_set_active(); +} + #else static inline void sme_user_disable(void) { BUILD_BUG(); } @@ -456,6 +474,9 @@ static inline size_t sme_state_size(struct task_struct const *task) return 0; } +static inline void sme_enter_from_user_mode(void) { } +static inline void sme_exit_to_user_mode(void) { } + #endif /* ! CONFIG_ARM64_SME */ /* For use by EFI runtime services calls only */ diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 83e03abbb2ca9..8cbd1e96fd50b 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -264,19 +264,33 @@ __iowrite64_copy(void __iomem *to, const void *from, size_t count) typedef int (*ioremap_prot_hook_t)(phys_addr_t phys_addr, size_t size, pgprot_t *prot); int arm64_ioremap_prot_hook_register(const ioremap_prot_hook_t hook); +void __iomem *__ioremap_prot(phys_addr_t phys, size_t size, pgprot_t prot); -#define ioremap_prot ioremap_prot +static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, + pgprot_t user_prot) +{ + pgprot_t prot; + ptdesc_t user_prot_val = pgprot_val(user_prot); + + if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) + return NULL; -#define _PAGE_IOREMAP PROT_DEVICE_nGnRE + prot = __pgprot_modify(PAGE_KERNEL, PTE_ATTRINDX_MASK, + user_prot_val & PTE_ATTRINDX_MASK); + return __ioremap_prot(phys, size, prot); +} +#define ioremap_prot ioremap_prot +#define ioremap(addr, size) \ + __ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) #define ioremap_wc(addr, size) \ - ioremap_prot((addr), (size), __pgprot(PROT_NORMAL_NC)) + __ioremap_prot((addr), (size), __pgprot(PROT_NORMAL_NC)) #define ioremap_np(addr, size) \ - ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRnE)) + __ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRnE)) #define ioremap_encrypted(addr, size) \ - ioremap_prot((addr), (size), PAGE_KERNEL) + __ioremap_prot((addr), (size), PAGE_KERNEL) /* * io{read,write}{16,32,64}be() macros @@ -297,7 +311,7 @@ static inline void __iomem *ioremap_cache(phys_addr_t addr, size_t size) if (pfn_is_map_memory(__phys_to_pfn(addr))) return (void __iomem *)__phys_to_virt(addr); - return ioremap_prot(addr, size, __pgprot(PROT_NORMAL)); + return __ioremap_prot(addr, size, __pgprot(PROT_NORMAL)); } /* diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 85dceb1c66f45..4d22bf22d2274 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -25,6 +25,8 @@ */ #define PTE_PRESENT_INVALID (PTE_NG) /* only when !PTE_VALID */ +#define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG) + #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP #define PTE_UFFD_WP (_AT(pteval_t, 1) << 58) /* uffd-wp tracking */ #define PTE_SWP_UFFD_WP (_AT(pteval_t, 1) << 3) /* only for swp ptes */ @@ -50,11 +52,11 @@ #define _PAGE_DEFAULT (_PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL)) -#define _PAGE_KERNEL (PROT_NORMAL) -#define _PAGE_KERNEL_RO ((PROT_NORMAL & ~PTE_WRITE) | PTE_RDONLY) -#define _PAGE_KERNEL_ROX ((PROT_NORMAL & ~(PTE_WRITE | PTE_PXN)) | PTE_RDONLY) -#define _PAGE_KERNEL_EXEC (PROT_NORMAL & ~PTE_PXN) -#define _PAGE_KERNEL_EXEC_CONT ((PROT_NORMAL & ~PTE_PXN) | PTE_CONT) +#define _PAGE_KERNEL (PROT_NORMAL | PTE_DIRTY) +#define _PAGE_KERNEL_RO ((PROT_NORMAL & ~PTE_WRITE) | PTE_RDONLY | PTE_DIRTY) +#define _PAGE_KERNEL_ROX ((PROT_NORMAL & ~(PTE_WRITE | PTE_PXN)) | PTE_RDONLY | PTE_DIRTY) +#define _PAGE_KERNEL_EXEC ((PROT_NORMAL & ~PTE_PXN) | PTE_DIRTY) +#define _PAGE_KERNEL_EXEC_CONT ((PROT_NORMAL & ~PTE_PXN) | PTE_CONT | PTE_DIRTY) #define _PAGE_SHARED (_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE) #define _PAGE_SHARED_EXEC (_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_WRITE) @@ -164,9 +166,6 @@ static inline bool __pure lpa2_is_enabled(void) #define _PAGE_GCS (_PAGE_DEFAULT | PTE_NG | PTE_UXN | PTE_WRITE | PTE_USER) #define _PAGE_GCS_RO (_PAGE_DEFAULT | PTE_NG | PTE_UXN | PTE_USER) -#define PAGE_GCS __pgprot(_PAGE_GCS) -#define PAGE_GCS_RO __pgprot(_PAGE_GCS_RO) - #define PIE_E0 ( \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS), PIE_GCS) | \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS_RO), PIE_R) | \ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 0944e296dd4a4..1765e484fb9f0 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -175,8 +175,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) #define pte_none(pte) (!pte_val(pte)) -#define __pte_clear(mm, addr, ptep) \ - __set_pte(ptep, __pte(0)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) /* @@ -355,9 +353,11 @@ static inline pte_t pte_mknoncont(pte_t pte) return clear_pte_bit(pte, __pgprot(PTE_CONT)); } -static inline pte_t pte_mkvalid(pte_t pte) +static inline pte_t pte_mkvalid_k(pte_t pte) { - return set_pte_bit(pte, __pgprot(PTE_VALID)); + pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); + pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL)); + return pte; } static inline pte_t pte_mkinvalid(pte_t pte) @@ -627,6 +627,7 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) +#define pmd_mkvalid_k(pmd) pte_pmd(pte_mkvalid_k(pmd_pte(pmd))) #define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd))) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP #define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd)) @@ -668,6 +669,8 @@ static inline pmd_t pmd_mkspecial(pmd_t pmd) #define pud_young(pud) pte_young(pud_pte(pud)) #define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud))) +#define pud_mkwrite_novma(pud) pte_pud(pte_mkwrite_novma(pud_pte(pud))) +#define pud_mkvalid_k(pud) pte_pud(pte_mkvalid_k(pud_pte(pud))) #define pud_write(pud) pte_write(pud_pte(pud)) static inline pud_t pud_mkhuge(pud_t pud) @@ -1316,6 +1319,13 @@ static inline bool pud_user_accessible_page(pud_t pud) /* * Atomic pte/pmd modifications. */ + +static inline void __pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + __set_pte(ptep, __pte(0)); +} + static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h index 97d9256d33c97..ac370a4a01ee9 100644 --- a/arch/arm64/include/asm/rwonce.h +++ b/arch/arm64/include/asm/rwonce.h @@ -58,7 +58,7 @@ default: \ atomic = 0; \ } \ - atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ + atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(*__x) *)__x);\ }) #endif /* !BUILD_VDSO */ diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index 0cde2f473971e..eb60c97355530 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -2,7 +2,7 @@ #ifndef __ASM_SUSPEND_H #define __ASM_SUSPEND_H -#define NR_CTX_REGS 13 +#define NR_CTX_REGS 14 #define NR_CALLEE_SAVED_REGS 12 /* diff --git a/arch/arm64/include/asm/tlbbatch.h b/arch/arm64/include/asm/tlbbatch.h index fedb0b87b8db4..6297631532e59 100644 --- a/arch/arm64/include/asm/tlbbatch.h +++ b/arch/arm64/include/asm/tlbbatch.h @@ -2,11 +2,17 @@ #ifndef _ARCH_ARM64_TLBBATCH_H #define _ARCH_ARM64_TLBBATCH_H +#include + struct arch_tlbflush_unmap_batch { +#ifdef CONFIG_ARM64_ERRATUM_4193714 /* - * For arm64, HW can do tlb shootdown, so we don't - * need to record cpumask for sending IPI + * Track CPUs that need SME DVMSync on completion of this batch. + * Otherwise, the arm64 HW can do tlb shootdown, so we don't need to + * record cpumask for sending IPI */ + cpumask_var_t cpumask; +#endif }; #endif /* _ARCH_ARM64_TLBBATCH_H */ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 18a5dc0c9a540..f53ab3ba0c48b 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -31,19 +31,11 @@ */ #define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \ "tlbi " #op "\n" \ - ALTERNATIVE("nop\n nop", \ - "dsb ish\n tlbi " #op, \ - ARM64_WORKAROUND_REPEAT_TLBI, \ - CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ : : ) #define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \ - "tlbi " #op ", %0\n" \ - ALTERNATIVE("nop\n nop", \ - "dsb ish\n tlbi " #op ", %0", \ - ARM64_WORKAROUND_REPEAT_TLBI, \ - CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ - : : "r" (arg)) + "tlbi " #op ", %x0\n" \ + : : "rZ" (arg)) #define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg) @@ -88,6 +80,71 @@ static inline unsigned long get_trans_granule(void) } } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +void sme_do_dvmsync(const struct cpumask *mask); + +static inline void sme_dvmsync(struct mm_struct *mm) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + sme_do_dvmsync(mm_cpumask(mm)); +} + +static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, + struct mm_struct *mm) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + /* + * Order the mm_cpumask() read after the hardware DVMSync. + */ + dsb(ish); + if (cpumask_empty(mm_cpumask(mm))) + return; + + /* + * Allocate the batch cpumask on first use. Fall back to an immediate + * IPI for this mm in case of failure. + */ + if (!cpumask_available(batch->cpumask) && + !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { + sme_do_dvmsync(mm_cpumask(mm)); + return; + } + + cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); +} + +static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + if (!cpumask_available(batch->cpumask)) + return; + + sme_do_dvmsync(batch->cpumask); + cpumask_clear(batch->cpumask); +} + +#else + +static inline void sme_dvmsync(struct mm_struct *mm) +{ +} +static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, + struct mm_struct *mm) +{ +} +static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +{ +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + /* * Level-based TLBI operations. * @@ -181,6 +238,48 @@ static inline unsigned long get_trans_granule(void) (__pages >> (5 * (scale) + 1)) - 1; \ }) +#define __repeat_tlbi_sync(op, arg...) \ +do { \ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI)) \ + break; \ + __tlbi(op, ##arg); \ + dsb(ish); \ +} while (0) + +/* + * Complete broadcast TLB maintenance issued by the host which invalidates + * stage 1 information in the host's own translation regime. + */ +static inline void __tlbi_sync_s1ish(struct mm_struct *mm) +{ + dsb(ish); + __repeat_tlbi_sync(vale1is, 0); + sme_dvmsync(mm); +} + +static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) +{ + dsb(ish); + __repeat_tlbi_sync(vale1is, 0); + sme_dvmsync_batch(batch); +} + +static inline void __tlbi_sync_s1ish_kernel(void) +{ + dsb(ish); + __repeat_tlbi_sync(vale1is, 0); +} + +/* + * Complete broadcast TLB maintenance issued by hyp code which invalidates + * stage 1 translation information in any translation regime. + */ +static inline void __tlbi_sync_s1ish_hyp(void) +{ + dsb(ish); + __repeat_tlbi_sync(vale2is, 0); +} + /* * TLB Invalidation * ================ @@ -266,7 +365,7 @@ static inline void flush_tlb_all(void) { dsb(ishst); __tlbi(vmalle1is); - dsb(ish); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -278,7 +377,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm) asid = __TLBI_VADDR(0, ASID(mm)); __tlbi(aside1is, asid); __tlbi_user(aside1is, asid); - dsb(ish); + __tlbi_sync_s1ish(mm); mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); } @@ -305,20 +404,11 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { flush_tlb_page_nosync(vma, uaddr); - dsb(ish); + __tlbi_sync_s1ish(vma->vm_mm); } static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) { - /* - * TLB flush deferral is not required on systems which are affected by - * ARM64_WORKAROUND_REPEAT_TLBI, as __tlbi()/__tlbi_user() implementation - * will have two consecutive TLBI instructions with a dsb(ish) in between - * defeating the purpose (i.e save overall 'dsb ish' cost). - */ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI)) - return false; - return true; } @@ -334,7 +424,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - dsb(ish); + __tlbi_sync_s1ish_batch(batch); } /* @@ -469,7 +559,7 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma, { __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, last_level, tlb_level); - dsb(ish); + __tlbi_sync_s1ish(vma->vm_mm); } static inline void flush_tlb_range(struct vm_area_struct *vma, @@ -501,7 +591,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end dsb(ishst); __flush_tlb_range_op(vaale1is, start, pages, stride, 0, TLBI_TTL_UNKNOWN, false, lpa2_is_enabled()); - dsb(ish); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -515,7 +605,7 @@ static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) dsb(ishst); __tlbi(vaae1is, addr); - dsb(ish); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -523,6 +613,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b struct mm_struct *mm, unsigned long start, unsigned long end) { __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); + sme_dvmsync_add_pending(batch, mm); } #endif diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 1aa4ecb73429f..93d184b4da2fe 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -480,7 +480,7 @@ extern __must_check long strnlen_user(const char __user *str, long n); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); -static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) +static inline size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size) { kasan_check_write(dst, size); return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size); diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index f1cb2447afc9c..b285e174f4f51 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -377,7 +377,7 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) prot = __acpi_get_writethrough_mem_attribute(); } } - return ioremap_prot(phys, size, prot); + return __ioremap_prot(phys, size, prot); } /* diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 8cb3b575a0316..6c8c4301d9c6d 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -551,6 +552,23 @@ static const struct midr_range erratum_spec_ssbs_list[] = { }; #endif +#ifdef CONFIG_ARM64_ERRATUM_4193714 +static bool has_sme_dvmsync_erratum(const struct arm64_cpu_capabilities *entry, + int scope) +{ + if (!id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) + return false; + + return is_affected_midr_range(entry, scope); +} + +static void cpu_enable_sme_dvmsync(const struct arm64_cpu_capabilities *__unused) +{ + if (this_cpu_has_cap(ARM64_WORKAROUND_4193714)) + sme_enable_dvmsync(); +} +#endif + #ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38 static const struct midr_range erratum_ac03_cpu_38_list[] = { MIDR_ALL_VERSIONS(MIDR_AMPERE1), @@ -870,6 +888,18 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_RANGE_LIST(erratum_spec_ssbs_list), }, #endif +#ifdef CONFIG_ARM64_ERRATUM_4193714 + { + .desc = "C1-Pro SME DVMSync early acknowledgement", + .capability = ARM64_WORKAROUND_4193714, + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, + .matches = has_sme_dvmsync_erratum, + .cpu_enable = cpu_enable_sme_dvmsync, + /* C1-Pro r0p0 - r1p2 (the latter only when REVIDR_EL1[0]==0) */ + .midr_range = MIDR_RANGE(MIDR_C1_PRO, 0, 0, 1, 2), + MIDR_FIXED(MIDR_CPU_VAR_REV(1, 2), BIT(0)), + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD { .desc = "ARM errata 2966298, 3117295", diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index a9c81715ce59e..5b97dfcf796d9 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -84,6 +85,7 @@ static __always_inline void __enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); mte_disable_tco_entry(current); + sme_enter_from_user_mode(); } static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) @@ -102,6 +104,7 @@ static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) local_irq_disable(); exit_to_user_mode_prepare(regs); local_daif_mask(); + sme_exit_to_user_mode(); mte_check_tfsr_exit(); exit_to_user_mode(); } diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index e3f8f51748bc9..ca18214ce2abe 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1384,6 +1386,83 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) put_cpu_fpsimd_context(); } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +/* + * SME/CME erratum handling. + */ +static cpumask_t sme_dvmsync_cpus; + +/* + * These helpers are only called from non-preemptible contexts, so + * smp_processor_id() is safe here. + */ +void sme_set_active(void) +{ + unsigned int cpu = smp_processor_id(); + + if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus)) + return; + + cpumask_set_cpu(cpu, mm_cpumask(current->mm)); + + /* + * A subsequent (post ERET) SME access may use a stale address + * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for + * the completion of cpumask_set_cpu() above as it appears in program + * order before the SME access. The post-TLBI+DSB read of mm_cpumask() + * will lead to the IPI being issued. + * + * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 + */ +} + +void sme_clear_active(void) +{ + unsigned int cpu = smp_processor_id(); + + if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus)) + return; + + /* + * With SCTLR_EL1.IESB enabled, the SME memory transactions are + * completed on entering EL1. + */ + cpumask_clear_cpu(cpu, mm_cpumask(current->mm)); +} + +static void sme_dvmsync_ipi(void *unused) +{ + /* + * With SCTLR_EL1.IESB on, taking an exception is sufficient to ensure + * the completion of the SME memory accesses, so no need for an + * explicit DSB. + */ +} + +void sme_do_dvmsync(const struct cpumask *mask) +{ + /* + * This is called from the TLB maintenance functions after the DSB ISH + * to send the hardware DVMSync message. If this CPU sees the mask as + * empty, the remote CPU executing sme_set_active() would have seen + * the DVMSync and no IPI required. + */ + if (cpumask_empty(mask)) + return; + + preempt_disable(); + smp_call_function_many(mask, sme_dvmsync_ipi, NULL, true); + preempt_enable(); +} + +void sme_enable_dvmsync(void) +{ + cpumask_set_cpu(smp_processor_id(), &sme_dvmsync_cpus); +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + /* * Trapped SME access * diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 18749e9a6c2da..9717568518ba7 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -402,7 +402,7 @@ int swsusp_arch_suspend(void) * Memory allocated by get_safe_page() will be dealt with by the hibernate code, * we don't need to free it here. */ -int swsusp_arch_resume(void) +int __nocfi swsusp_arch_resume(void) { int rc; void *zero_page; diff --git a/arch/arm64/kernel/kexec_image.c b/arch/arm64/kernel/kexec_image.c index 532d72ea42ee8..b70f4df15a1ae 100644 --- a/arch/arm64/kernel/kexec_image.c +++ b/arch/arm64/kernel/kexec_image.c @@ -41,7 +41,7 @@ static void *image_load(struct kimage *image, struct arm64_image_header *h; u64 flags, value; bool be_image, be_kernel; - struct kexec_buf kbuf; + struct kexec_buf kbuf = {}; unsigned long text_offset, kernel_segment_number; struct kexec_segment *kernel_segment; int ret; diff --git a/arch/arm64/kernel/pi/patch-scs.c b/arch/arm64/kernel/pi/patch-scs.c index bbe7d30ed12b3..dac568e4a54f2 100644 --- a/arch/arm64/kernel/pi/patch-scs.c +++ b/arch/arm64/kernel/pi/patch-scs.c @@ -192,6 +192,14 @@ static int scs_handle_fde_frame(const struct eh_frame *frame, size -= 2; break; + case DW_CFA_advance_loc4: + loc += *opcode++ * code_alignment_factor; + loc += (*opcode++ << 8) * code_alignment_factor; + loc += (*opcode++ << 16) * code_alignment_factor; + loc += (*opcode++ << 24) * code_alignment_factor; + size -= 4; + break; + case DW_CFA_def_cfa: case DW_CFA_offset_extended: size = skip_xleb128(&opcode, size); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index fba7ca102a8c4..4c328b7c79ba3 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -292,6 +293,7 @@ static void flush_gcs(void) current->thread.gcs_base = 0; current->thread.gcs_size = 0; current->thread.gcs_el0_mode = 0; + current->thread.gcs_el0_locked = 0; write_sysreg_s(GCSCRE0_EL1_nTR, SYS_GCSCRE0_EL1); write_sysreg_s(0, SYS_GCSPR_EL0); } @@ -338,8 +340,41 @@ void flush_thread(void) flush_gcs(); } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +static void arch_dup_tlbbatch_mask(struct task_struct *dst) +{ + /* + * Clear the inherited cpumask with memset() to cover both cases where + * cpumask_var_t is a pointer or an array. It will be allocated lazily + * in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y. + */ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + memset(&dst->tlb_ubc.arch.cpumask, 0, + sizeof(dst->tlb_ubc.arch.cpumask)); +} + +static void arch_release_tlbbatch_mask(struct task_struct *tsk) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + free_cpumask_var(tsk->tlb_ubc.arch.cpumask); +} + +#else + +static void arch_dup_tlbbatch_mask(struct task_struct *dst) +{ +} + +static void arch_release_tlbbatch_mask(struct task_struct *tsk) +{ +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + void arch_release_task_struct(struct task_struct *tsk) { + arch_release_tlbbatch_mask(tsk); fpsimd_release_task(tsk); } @@ -355,6 +390,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) *dst = *src; + arch_dup_tlbbatch_mask(dst); + /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 80a580e019c50..b3801f532b10b 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -887,6 +887,7 @@ static u8 spectre_bhb_loop_affected(void) MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), {}, }; static const struct midr_range spectre_bhb_k24_list[] = { diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 4b001121c72db..2e9ce5a45ed2d 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -942,20 +942,18 @@ static int sve_set_common(struct task_struct *target, vq = sve_vq_from_vl(task_get_vl(target, type)); /* Enter/exit streaming mode */ - if (system_supports_sme()) { - switch (type) { - case ARM64_VEC_SVE: - target->thread.svcr &= ~SVCR_SM_MASK; - set_tsk_thread_flag(target, TIF_SVE); - break; - case ARM64_VEC_SME: - target->thread.svcr |= SVCR_SM_MASK; - set_tsk_thread_flag(target, TIF_SME); - break; - default: - WARN_ON_ONCE(1); - return -EINVAL; - } + switch (type) { + case ARM64_VEC_SVE: + target->thread.svcr &= ~SVCR_SM_MASK; + set_tsk_thread_flag(target, TIF_SVE); + break; + case ARM64_VEC_SME: + target->thread.svcr |= SVCR_SM_MASK; + set_tsk_thread_flag(target, TIF_SME); + break; + default: + WARN_ON_ONCE(1); + return -EINVAL; } /* Always zero V regs, FPSR, and FPCR */ @@ -1460,6 +1458,9 @@ static int poe_get(struct task_struct *target, if (!system_supports_poe()) return -EINVAL; + if (target == current) + current->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); + return membuf_write(&to, &target->thread.por_el0, sizeof(target->thread.por_el0)); } diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c index c64a06f58c0bc..9e846ce4ef9ca 100644 --- a/arch/arm64/kernel/rsi.c +++ b/arch/arm64/kernel/rsi.c @@ -12,6 +12,7 @@ #include #include +#include #include static struct realm_config config; @@ -146,7 +147,7 @@ void __init arm64_rsi_init(void) return; if (WARN_ON(rsi_get_realm_config(&config))) return; - prot_ns_shared = BIT(config.ipa_bits - 1); + prot_ns_shared = __phys_to_pte_val(BIT(config.ipa_bits - 1)); if (arm64_ioremap_prot_hook_register(realm_ioremap_hook)) return; diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 1110eeb21f572..08ffc5a5aea4c 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -449,12 +449,28 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) if (user->sve_size < SVE_SIG_CONTEXT_SIZE(vq)) return -EINVAL; + if (sm) { + sme_alloc(current, false); + if (!current->thread.sme_state) + return -ENOMEM; + } + sve_alloc(current, true); if (!current->thread.sve_state) { clear_thread_flag(TIF_SVE); return -ENOMEM; } + if (sm) { + current->thread.svcr |= SVCR_SM_MASK; + set_thread_flag(TIF_SME); + } else { + current->thread.svcr &= ~SVCR_SM_MASK; + set_thread_flag(TIF_SVE); + } + + current->thread.fp_type = FP_STATE_SVE; + err = __copy_from_user(current->thread.sve_state, (char __user const *)user->sve + SVE_SIG_REGS_OFFSET, @@ -462,12 +478,6 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) if (err) return -EFAULT; - if (flags & SVE_SIG_FLAG_SM) - current->thread.svcr |= SVCR_SM_MASK; - else - set_thread_flag(TIF_SVE); - current->thread.fp_type = FP_STATE_SVE; - err = read_fpsimd_context(&fpsimd, user); if (err) return err; @@ -576,6 +586,10 @@ static int restore_za_context(struct user_ctxs *user) if (user->za_size < ZA_SIG_CONTEXT_SIZE(vq)) return -EINVAL; + sve_alloc(current, false); + if (!current->thread.sve_state) + return -ENOMEM; + sme_alloc(current, true); if (!current->thread.sme_state) { current->thread.svcr &= ~SVCR_ZA_MASK; diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 4a609e9b65de0..03fde2677d5be 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -37,7 +37,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end) * We pick the reserved-ASID to minimise the impact. */ __tlbi(aside1is, __TLBI_VADDR(0, 0)); - dsb(ish); + __tlbi_sync_s1ish(current->mm); } ret = caches_clean_inval_user_pou(start, start + chunk); diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index ad6133b89e7a4..2964aad0362e4 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,6 +349,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS HEAD_SYMBOLS diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 834f13fb1fb7d..2d04fb56746ea 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2428,7 +2428,7 @@ static u64 kvm_get_sysreg_res0(struct kvm *kvm, enum vcpu_sysreg sr) masks = kvm->arch.sysreg_masks; - return masks->mask[sr - __VNCR_START__].res0; + return masks->mask[sr - __SANITISED_REG_START__].res0; } static bool check_fgt_bit(struct kvm_vcpu *vcpu, enum vcpu_sysreg sr, diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 1c87699fd886e..332c453b87cf8 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -29,7 +29,7 @@ #include "trace.h" -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; @@ -42,7 +42,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, hvc_exit_stat), STATS_DESC_COUNTER(VCPU, wfe_exit_stat), diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 49db32f3ddf71..ece04bb10ab09 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -516,7 +516,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) granule = kvm_granule_size(level); cur.start = ALIGN_DOWN(addr, granule); cur.end = cur.start + granule; - if (!range_included(&cur, range)) + if (!range_included(&cur, range) && level < KVM_PGTABLE_LAST_LEVEL) continue; *range = cur; return 0; diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index ae8391baebc30..218976287d3fe 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -271,7 +271,7 @@ static void fixmap_clear_slot(struct hyp_fixmap_slot *slot) */ dsb(ishst); __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level); - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); } diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 43bde061b65de..d866f6ba19b5f 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -343,6 +343,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc /* No restrictions for non-protected VMs. */ if (!kvm_vm_is_protected(kvm)) { hyp_vm->kvm.arch.flags = host_arch_flags; + hyp_vm->kvm.arch.flags &= ~BIT_ULL(KVM_ARCH_FLAG_ID_REGS_INITIALIZED); bitmap_copy(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, @@ -469,6 +470,35 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h return ret; } +static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + const struct kvm *host_kvm = hyp_vm->host_kvm; + struct kvm *kvm = &hyp_vm->kvm; + + if (!test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &host_kvm->arch.flags)) + return -EINVAL; + + if (test_and_set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags)) + return 0; + + memcpy(kvm->arch.id_regs, host_kvm->arch.id_regs, sizeof(kvm->arch.id_regs)); + + return 0; +} + +static int pkvm_vcpu_init_sysregs(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + int ret = 0; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); + else + ret = vm_copy_id_regs(hyp_vcpu); + + return ret; +} + static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct pkvm_hyp_vm *hyp_vm, struct kvm_vcpu *host_vcpu) @@ -488,8 +518,9 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; - if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); + ret = pkvm_vcpu_init_sysregs(hyp_vcpu); + if (ret) + goto done; ret = pkvm_vcpu_init_traps(hyp_vcpu); if (ret) diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c index 48da9ca9763f6..3dc1ce0d27fe6 100644 --- a/arch/arm64/kvm/hyp/nvhe/tlb.c +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c @@ -169,7 +169,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, */ dsb(ish); __tlbi(vmalle1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); exit_vmid_context(&cxt); @@ -226,7 +226,7 @@ void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, dsb(ish); __tlbi(vmalle1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); exit_vmid_context(&cxt); @@ -240,7 +240,7 @@ void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) enter_vmid_context(mmu, &cxt, false); __tlbi(vmalls12e1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); exit_vmid_context(&cxt); @@ -266,5 +266,5 @@ void __kvm_flush_vm_context(void) /* Same remark as in enter_vmid_context() */ dsb(ish); __tlbi(alle1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); } diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index c351b4abd5dbf..cbf8cd2e16735 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -483,7 +483,7 @@ static int hyp_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx, *unmapped += granule; } - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); mm_ops->put_page(ctx->ptep); diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c index ec25698186297..35855dadfb1b3 100644 --- a/arch/arm64/kvm/hyp/vhe/tlb.c +++ b/arch/arm64/kvm/hyp/vhe/tlb.c @@ -115,7 +115,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, */ dsb(ish); __tlbi(vmalle1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); exit_vmid_context(&cxt); @@ -176,7 +176,7 @@ void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, dsb(ish); __tlbi(vmalle1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); exit_vmid_context(&cxt); @@ -192,7 +192,7 @@ void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) enter_vmid_context(mmu, &cxt); __tlbi(vmalls12e1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); exit_vmid_context(&cxt); @@ -217,7 +217,7 @@ void __kvm_flush_vm_context(void) { dsb(ishst); __tlbi(alle1is); - dsb(ish); + __tlbi_sync_s1ish_hyp(); } /* @@ -358,7 +358,7 @@ int __kvm_tlbi_s1e2(struct kvm_s2_mmu *mmu, u64 va, u64 sys_encoding) default: ret = -EINVAL; } - dsb(ish); + __tlbi_sync_s1ish_hyp(); isb(); if (mmu) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 7cc964af8d305..0d38dc72dfc62 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1712,14 +1712,12 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, } /* - * Both the canonical IPA and fault IPA must be hugepage-aligned to - * ensure we find the right PFN and lay down the mapping in the right - * place. + * Both the canonical IPA and fault IPA must be aligned to the + * mapping size to ensure we find the right PFN and lay down the + * mapping in the right place. */ - if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE) { - fault_ipa &= ~(vma_pagesize - 1); - ipa &= ~(vma_pagesize - 1); - } + fault_ipa = ALIGN_DOWN(fault_ipa, vma_pagesize); + ipa = ALIGN_DOWN(ipa, vma_pagesize); gfn = ipa >> PAGE_SHIFT; mte_allowed = kvm_vma_mte_allowed(vma); diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 959532422d3a3..b963fd975aaca 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -247,6 +247,20 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) kvm_vcpu_set_be(vcpu); *vcpu_pc(vcpu) = target_pc; + + /* + * We may come from a state where either a PC update was + * pending (SMC call resulting in PC being increpented to + * skip the SMC) or a pending exception. Make sure we get + * rid of all that, as this cannot be valid out of reset. + * + * Note that clearing the exception mask also clears PC + * updates, but that's an implementation detail, and we + * really want to make it explicit. + */ + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); vcpu_set_reg(vcpu, 0, reset_state.r0); } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index ec3fbe0b8d525..7b7f3c932dcd5 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1801,6 +1801,9 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu, ID_AA64MMFR3_EL1_SCTLRX | ID_AA64MMFR3_EL1_S1POE | ID_AA64MMFR3_EL1_S1PIE; + + if (!system_supports_poe()) + val &= ~ID_AA64MMFR3_EL1_S1POE; break; case SYS_ID_MMFR4_EL1: val &= ~ID_MMFR4_EL1_CCIDX; diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index da62edbc1205a..30fa88e49be49 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -140,26 +140,9 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } - kvm_for_each_vcpu(i, vcpu, kvm) { - ret = vgic_allocate_private_irqs_locked(vcpu, type); - if (ret) - break; - } - - if (ret) { - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - kfree(vgic_cpu->private_irqs); - vgic_cpu->private_irqs = NULL; - } - - goto out_unlock; - } - kvm->arch.vgic.in_kernel = true; kvm->arch.vgic.vgic_model = type; kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST; - kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF; aa64pfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC; @@ -176,6 +159,23 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, aa64pfr0); kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, pfr1); + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = vgic_allocate_private_irqs_locked(vcpu, type); + if (ret) + break; + } + + if (ret) { + kvm_for_each_vcpu(i, vcpu, kvm) { + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + kfree(vgic_cpu->private_irqs); + vgic_cpu->private_irqs = NULL; + } + + kvm->arch.vgic.vgic_model = 0; + goto out_unlock; + } + if (type == KVM_DEV_TYPE_ARM_VGIC_V3) kvm->arch.vgic.nassgicap = system_supports_direct_sgis(); diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index cb2062e7e2340..e278e060e78a9 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -23,9 +23,24 @@ static inline unsigned long xloops_to_cycles(unsigned long xloops) return (xloops * loops_per_jiffy * HZ) >> 32; } +/* + * Force the use of CNTVCT_EL0 in order to have the same base as WFxT. + * This avoids some annoying issues when CNTVOFF_EL2 is not reset 0 on a + * KVM host running at EL1 until we do a vcpu_put() on the vcpu. When + * running at EL2, the effective offset is always 0. + * + * Note that userspace cannot change the offset behind our back either, + * as the vcpu mutex is held as long as KVM_RUN is in progress. + */ +static cycles_t notrace __delay_cycles(void) +{ + guard(preempt_notrace)(); + return __arch_counter_get_cntvct_stable(); +} + void __delay(unsigned long cycles) { - cycles_t start = get_cycles(); + cycles_t start = __delay_cycles(); if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; @@ -35,17 +50,17 @@ void __delay(unsigned long cycles) * early, use a WFET loop to complete the delay. */ wfit(end); - while ((get_cycles() - start) < cycles) + while ((__delay_cycles() - start) < cycles) wfet(end); } else if (arch_timer_evtstrm_available()) { const cycles_t timer_evt_period = USECS_TO_CYCLES(ARCH_TIMER_EVT_STREAM_PERIOD_US); - while ((get_cycles() - start + timer_evt_period) < cycles) + while ((__delay_cycles() - start + timer_evt_period) < cycles) wfe(); } - while ((get_cycles() - start) < cycles) + while ((__delay_cycles() - start) < cycles) cpu_relax(); } EXPORT_SYMBOL(__delay); diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index c0557945939ce..29024e20e876d 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -581,6 +581,27 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, } EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); +static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) +{ + pte_t *cont_ptep = contpte_align_down(ptep); + /* + * PFNs differ per sub-PTE. Match only bits consumed by + * __ptep_set_access_flags(): AF, DIRTY and write permission. + */ + const pteval_t cmp_mask = PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; + pteval_t entry_cmp = pte_val(entry) & cmp_mask; + int i; + + for (i = 0; i < CONT_PTES; i++) { + pteval_t pte_cmp = pte_val(__ptep_get(cont_ptep + i)) & cmp_mask; + + if (pte_cmp != entry_cmp) + return false; + } + + return true; +} + int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) @@ -590,13 +611,37 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, int i; /* - * Gather the access/dirty bits for the contiguous range. If nothing has - * changed, its a noop. + * Check whether all sub-PTEs in the CONT block already match the + * requested access flags/write permission, using raw per-PTE values + * rather than the gathered ptep_get() view. + * + * __ptep_set_access_flags() can update AF, dirty and write + * permission, but only to make the mapping more permissive. + * + * ptep_get() gathers AF/dirty state across the whole CONT block, + * which is correct for a CPU with FEAT_HAFDBS. But page-table + * walkers that evaluate each descriptor individually (e.g. a CPU + * without DBM support, or an SMMU without HTTU, or with HA/HD + * disabled in CD.TCR) can keep faulting on the target sub-PTE if + * only a sibling has been updated. Gathering can therefore cause + * false no-ops when only a sibling has been updated: + * - write faults: target still has PTE_RDONLY (needs PTE_RDONLY cleared) + * - read faults: target still lacks PTE_AF + * + * Per Arm ARM (DDI 0487) D8.7.1, any sub-PTE in a CONT range may + * become the effective cached translation, so all entries must have + * consistent attributes. Check the full CONT block before returning + * no-op, and when any sub-PTE mismatches, proceed to update the whole + * range. */ - orig_pte = pte_mknoncont(ptep_get(ptep)); - if (pte_val(orig_pte) == pte_val(entry)) + if (contpte_all_subptes_match_access_flags(ptep, entry)) return 0; + /* + * Use raw target pte (not gathered) for write-bit unfold decision. + */ + orig_pte = pte_mknoncont(__ptep_get(ptep)); + /* * We can fix up access/dirty bits without having to unfold the contig * range. But if the write bit is changing, we must unfold. diff --git a/arch/arm64/mm/gcs.c b/arch/arm64/mm/gcs.c index 6e93f78de79b1..04a23a497f205 100644 --- a/arch/arm64/mm/gcs.c +++ b/arch/arm64/mm/gcs.c @@ -199,8 +199,8 @@ int arch_set_shadow_stack_status(struct task_struct *task, unsigned long arg) size = gcs_size(0); gcs = alloc_gcs(0, size); - if (!gcs) - return -ENOMEM; + if (IS_ERR_VALUE(gcs)) + return gcs; task->thread.gcspr_el0 = gcs + size - sizeof(u64); task->thread.gcs_base = gcs; diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 524d34a0e9219..c1742eb63ddde 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -48,6 +48,7 @@ #include #include #include +#include /* * We need to be able to catch inadvertent references to memstart_addr @@ -331,6 +332,9 @@ void __init bootmem_init(void) * reserved, so do it here. */ arch_reserve_crashkernel(); +#if defined(CONFIG_QCOM_MEMORY_DUMP_V2) + reserve_memdump_cma(); +#endif memblock_dump_all(); } diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c index 10e246f112710..1e4794a2af7d6 100644 --- a/arch/arm64/mm/ioremap.c +++ b/arch/arm64/mm/ioremap.c @@ -14,8 +14,8 @@ int arm64_ioremap_prot_hook_register(ioremap_prot_hook_t hook) return 0; } -void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, - pgprot_t pgprot) +void __iomem *__ioremap_prot(phys_addr_t phys_addr, size_t size, + pgprot_t pgprot) { unsigned long last_addr = phys_addr + size - 1; @@ -38,7 +38,7 @@ void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, return generic_ioremap_prot(phys_addr, size, pgprot); } -EXPORT_SYMBOL(ioremap_prot); +EXPORT_SYMBOL(__ioremap_prot); /* * Must be called after early_fixmap_init diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 08ee177432c2f..92b2f5097a96c 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -34,6 +34,8 @@ static pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; +static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; + /* * You really shouldn't be using read() or write() on /dev/mem. This might go * away in the future. @@ -73,9 +75,11 @@ static int __init adjust_protection_map(void) protection_map[VM_EXEC | VM_SHARED] = PAGE_EXECONLY; } - if (lpa2_is_enabled()) + if (lpa2_is_enabled()) { for (int i = 0; i < ARRAY_SIZE(protection_map); i++) pgprot_val(protection_map[i]) &= ~PTE_SHARED; + gcs_page_prot &= ~PTE_SHARED; + } return 0; } @@ -87,7 +91,11 @@ pgprot_t vm_get_page_prot(vm_flags_t vm_flags) /* Short circuit GCS to avoid bloating the table. */ if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) { - prot = _PAGE_GCS_RO; + /* Honour mprotect(PROT_NONE) on shadow stack mappings */ + if (vm_flags & VM_ACCESS_FLAGS) + prot = gcs_page_prot; + else + prot = pgprot_val(protection_map[VM_NONE]); } else { prot = pgprot_val(protection_map[vm_flags & (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 2ba01dc8ef822..ed04c42a826d8 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -49,6 +49,8 @@ #define NO_CONT_MAPPINGS BIT(1) #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ +#define INVALID_PHYS_ADDR (-1ULL) + DEFINE_STATIC_KEY_FALSE(arm64_ptdump_lock_key); u64 kimage_voffset __ro_after_init; @@ -194,11 +196,11 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end, } while (ptep++, addr += PAGE_SIZE, addr != end); } -static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, - unsigned long end, phys_addr_t phys, - pgprot_t prot, - phys_addr_t (*pgtable_alloc)(enum pgtable_type), - int flags) +static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, + unsigned long end, phys_addr_t phys, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), + int flags) { unsigned long next; pmd_t pmd = READ_ONCE(*pmdp); @@ -213,6 +215,8 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, pmdval |= PMD_TABLE_PXN; BUG_ON(!pgtable_alloc); pte_phys = pgtable_alloc(TABLE_PTE); + if (pte_phys == INVALID_PHYS_ADDR) + return -ENOMEM; ptep = pte_set_fixmap(pte_phys); init_clear_pgtable(ptep); ptep += pte_index(addr); @@ -244,11 +248,13 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, * walker. */ pte_clear_fixmap(); + + return 0; } -static void init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, - phys_addr_t phys, pgprot_t prot, - phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags) +static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags) { unsigned long next; @@ -269,22 +275,29 @@ static void init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), READ_ONCE(pmd_val(*pmdp)))); } else { - alloc_init_cont_pte(pmdp, addr, next, phys, prot, - pgtable_alloc, flags); + int ret; + + ret = alloc_init_cont_pte(pmdp, addr, next, phys, prot, + pgtable_alloc, flags); + if (ret) + return ret; BUG_ON(pmd_val(old_pmd) != 0 && pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); } phys += next - addr; } while (pmdp++, addr = next, addr != end); + + return 0; } -static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, - unsigned long end, phys_addr_t phys, - pgprot_t prot, - phys_addr_t (*pgtable_alloc)(enum pgtable_type), - int flags) +static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, + unsigned long end, phys_addr_t phys, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), + int flags) { + int ret; unsigned long next; pud_t pud = READ_ONCE(*pudp); pmd_t *pmdp; @@ -301,6 +314,8 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, pudval |= PUD_TABLE_PXN; BUG_ON(!pgtable_alloc); pmd_phys = pgtable_alloc(TABLE_PMD); + if (pmd_phys == INVALID_PHYS_ADDR) + return -ENOMEM; pmdp = pmd_set_fixmap(pmd_phys); init_clear_pgtable(pmdp); pmdp += pmd_index(addr); @@ -320,20 +335,26 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, (flags & NO_CONT_MAPPINGS) == 0) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); - init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags); + ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags); + if (ret) + goto out; pmdp += pmd_index(next) - pmd_index(addr); phys += next - addr; } while (addr = next, addr != end); +out: pmd_clear_fixmap(); + + return ret; } -static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, - phys_addr_t phys, pgprot_t prot, - phys_addr_t (*pgtable_alloc)(enum pgtable_type), - int flags) +static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), + int flags) { + int ret = 0; unsigned long next; p4d_t p4d = READ_ONCE(*p4dp); pud_t *pudp; @@ -346,6 +367,8 @@ static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, p4dval |= P4D_TABLE_PXN; BUG_ON(!pgtable_alloc); pud_phys = pgtable_alloc(TABLE_PUD); + if (pud_phys == INVALID_PHYS_ADDR) + return -ENOMEM; pudp = pud_set_fixmap(pud_phys); init_clear_pgtable(pudp); pudp += pud_index(addr); @@ -375,8 +398,10 @@ static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, BUG_ON(!pgattr_change_is_safe(pud_val(old_pud), READ_ONCE(pud_val(*pudp)))); } else { - alloc_init_cont_pmd(pudp, addr, next, phys, prot, - pgtable_alloc, flags); + ret = alloc_init_cont_pmd(pudp, addr, next, phys, prot, + pgtable_alloc, flags); + if (ret) + goto out; BUG_ON(pud_val(old_pud) != 0 && pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); @@ -384,14 +409,18 @@ static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, phys += next - addr; } while (pudp++, addr = next, addr != end); +out: pud_clear_fixmap(); + + return ret; } -static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, - phys_addr_t phys, pgprot_t prot, - phys_addr_t (*pgtable_alloc)(enum pgtable_type), - int flags) +static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), + int flags) { + int ret; unsigned long next; pgd_t pgd = READ_ONCE(*pgdp); p4d_t *p4dp; @@ -404,6 +433,8 @@ static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, pgdval |= PGD_TABLE_PXN; BUG_ON(!pgtable_alloc); p4d_phys = pgtable_alloc(TABLE_P4D); + if (p4d_phys == INVALID_PHYS_ADDR) + return -ENOMEM; p4dp = p4d_set_fixmap(p4d_phys); init_clear_pgtable(p4dp); p4dp += p4d_index(addr); @@ -418,8 +449,10 @@ static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, next = p4d_addr_end(addr, end); - alloc_init_pud(p4dp, addr, next, phys, prot, - pgtable_alloc, flags); + ret = alloc_init_pud(p4dp, addr, next, phys, prot, + pgtable_alloc, flags); + if (ret) + goto out; BUG_ON(p4d_val(old_p4d) != 0 && p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); @@ -427,15 +460,19 @@ static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, phys += next - addr; } while (p4dp++, addr = next, addr != end); +out: p4d_clear_fixmap(); + + return ret; } -static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - pgprot_t prot, - phys_addr_t (*pgtable_alloc)(enum pgtable_type), - int flags) +static int __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), + int flags) { + int ret; unsigned long addr, end, next; pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); @@ -444,7 +481,7 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, * within a page, we cannot map the region as the caller expects. */ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) - return; + return -EINVAL; phys &= PAGE_MASK; addr = virt & PAGE_MASK; @@ -452,25 +489,45 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, do { next = pgd_addr_end(addr, end); - alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc, - flags); + ret = alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc, + flags); + if (ret) + return ret; phys += next - addr; } while (pgdp++, addr = next, addr != end); + + return 0; } -static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - pgprot_t prot, - phys_addr_t (*pgtable_alloc)(enum pgtable_type), - int flags) +static int __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), + int flags) { + int ret; + mutex_lock(&fixmap_lock); - __create_pgd_mapping_locked(pgdir, phys, virt, size, prot, - pgtable_alloc, flags); + ret = __create_pgd_mapping_locked(pgdir, phys, virt, size, prot, + pgtable_alloc, flags); mutex_unlock(&fixmap_lock); + + return ret; } -#define INVALID_PHYS_ADDR (-1ULL) +static void early_create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(enum pgtable_type), + int flags) +{ + int ret; + + ret = __create_pgd_mapping(pgdir, phys, virt, size, prot, pgtable_alloc, + flags); + if (ret) + panic("Failed to create page tables\n"); +} static phys_addr_t __pgd_pgtable_alloc(struct mm_struct *mm, gfp_t gfp, enum pgtable_type pgtable_type) @@ -511,21 +568,13 @@ try_pgd_pgtable_alloc_init_mm(enum pgtable_type pgtable_type, gfp_t gfp) static phys_addr_t __maybe_unused pgd_pgtable_alloc_init_mm(enum pgtable_type pgtable_type) { - phys_addr_t pa; - - pa = __pgd_pgtable_alloc(&init_mm, GFP_PGTABLE_KERNEL, pgtable_type); - BUG_ON(pa == INVALID_PHYS_ADDR); - return pa; + return __pgd_pgtable_alloc(&init_mm, GFP_PGTABLE_KERNEL, pgtable_type); } static phys_addr_t pgd_pgtable_alloc_special_mm(enum pgtable_type pgtable_type) { - phys_addr_t pa; - - pa = __pgd_pgtable_alloc(NULL, GFP_PGTABLE_KERNEL, pgtable_type); - BUG_ON(pa == INVALID_PHYS_ADDR); - return pa; + return __pgd_pgtable_alloc(NULL, GFP_PGTABLE_KERNEL, pgtable_type); } static void split_contpte(pte_t *ptep) @@ -555,6 +604,8 @@ static int split_pmd(pmd_t *pmdp, pmd_t pmd, gfp_t gfp, bool to_cont) tableprot |= PMD_TABLE_PXN; prot = __pgprot((pgprot_val(prot) & ~PTE_TYPE_MASK) | PTE_TYPE_PAGE); + if (!pmd_valid(pmd)) + prot = pte_pgprot(pte_mkinvalid(pfn_pte(0, prot))); prot = __pgprot(pgprot_val(prot) & ~PTE_CONT); if (to_cont) prot = __pgprot(pgprot_val(prot) | PTE_CONT); @@ -600,6 +651,8 @@ static int split_pud(pud_t *pudp, pud_t pud, gfp_t gfp, bool to_cont) tableprot |= PUD_TABLE_PXN; prot = __pgprot((pgprot_val(prot) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT); + if (!pud_valid(pud)) + prot = pmd_pgprot(pmd_mkinvalid(pfn_pmd(0, prot))); prot = __pgprot(pgprot_val(prot) & ~PTE_CONT); if (to_cont) prot = __pgprot(pgprot_val(prot) | PTE_CONT); @@ -935,8 +988,8 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } - __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, + NO_CONT_MAPPINGS); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, @@ -950,8 +1003,8 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, if (page_mappings_only) flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; - __create_pgd_mapping(mm->pgd, phys, virt, size, prot, - pgd_pgtable_alloc_special_mm, flags); + early_create_pgd_mapping(mm->pgd, phys, virt, size, prot, + pgd_pgtable_alloc_special_mm, flags); } static void update_mapping_prot(phys_addr_t phys, unsigned long virt, @@ -963,8 +1016,8 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, return; } - __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, + NO_CONT_MAPPINGS); /* flush the TLBs after updating live kernel mappings */ flush_tlb_kernel_range(virt, virt + size); @@ -973,8 +1026,8 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, phys_addr_t end, pgprot_t prot, int flags) { - __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, - prot, early_pgtable_alloc, flags); + early_create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, + prot, early_pgtable_alloc, flags); } void __init mark_linear_text_alias_ro(void) @@ -1207,6 +1260,8 @@ static int __init __kpti_install_ng_mappings(void *__unused) remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); if (!cpu) { + int ret; + alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE); kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd); @@ -1227,9 +1282,11 @@ static int __init __kpti_install_ng_mappings(void *__unused) // covers the PTE[] page itself, the remaining entries are free // to be used as a ad-hoc fixmap. // - __create_pgd_mapping_locked(kpti_ng_temp_pgd, __pa(alloc), - KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL, - kpti_ng_pgd_alloc, 0); + ret = __create_pgd_mapping_locked(kpti_ng_temp_pgd, __pa(alloc), + KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL, + kpti_ng_pgd_alloc, 0); + if (ret) + panic("Failed to create page tables\n"); } cpu_install_idmap(); @@ -1282,9 +1339,9 @@ static int __init map_entry_trampoline(void) /* Map only the text into the trampoline page table */ memset(tramp_pg_dir, 0, PGD_SIZE); - __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, - entry_tramp_text_size(), prot, - pgd_pgtable_alloc_init_mm, NO_BLOCK_MAPPINGS); + early_create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, + entry_tramp_text_size(), prot, + pgd_pgtable_alloc_init_mm, NO_BLOCK_MAPPINGS); /* Map both the text and data into the kernel page table */ for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) @@ -1926,23 +1983,28 @@ int arch_add_memory(int nid, u64 start, u64 size, if (force_pte_mapping()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; - __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), - size, params->pgprot, pgd_pgtable_alloc_init_mm, - flags); + ret = __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), + size, params->pgprot, pgd_pgtable_alloc_init_mm, + flags); + if (ret) + goto err; memblock_clear_nomap(start, size); ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT, params); if (ret) - __remove_pgd_mapping(swapper_pg_dir, - __phys_to_virt(start), size); - else { - /* Address of hotplugged memory can be smaller */ - max_pfn = max(max_pfn, PFN_UP(start + size)); - max_low_pfn = max_pfn; - } + goto err; + + /* Address of hotplugged memory can be smaller */ + max_pfn = max(max_pfn, PFN_UP(start + size)); + max_low_pfn = max_pfn; + + return 0; +err: + __remove_pgd_mapping(swapper_pg_dir, + __phys_to_virt(start), size); return ret; } diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 5135f2d66958d..02849728e6601 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -25,6 +25,11 @@ static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk) { struct page_change_data *masks = walk->private; + /* + * Some users clear and set bits which alias each other (e.g. PTE_NG and + * PTE_PRESENT_INVALID). It is therefore important that we always clear + * first then set. + */ val &= ~(pgprot_val(masks->clear_mask)); val |= (pgprot_val(masks->set_mask)); @@ -36,7 +41,7 @@ static int pageattr_pud_entry(pud_t *pud, unsigned long addr, { pud_t val = pudp_get(pud); - if (pud_sect(val)) { + if (pud_leaf(val)) { if (WARN_ON_ONCE((next - addr) != PUD_SIZE)) return -EINVAL; val = __pud(set_pageattr_masks(pud_val(val), walk)); @@ -52,7 +57,7 @@ static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, { pmd_t val = pmdp_get(pmd); - if (pmd_sect(val)) { + if (pmd_leaf(val)) { if (WARN_ON_ONCE((next - addr) != PMD_SIZE)) return -EINVAL; val = __pmd(set_pageattr_masks(pmd_val(val), walk)); @@ -132,11 +137,12 @@ static int __change_memory_common(unsigned long start, unsigned long size, ret = update_range_prot(start, size, set_mask, clear_mask); /* - * If the memory is being made valid without changing any other bits - * then a TLBI isn't required as a non-valid entry cannot be cached in - * the TLB. + * If the memory is being switched from present-invalid to valid without + * changing any other bits then a TLBI isn't required as a non-valid + * entry cannot be cached in the TLB. */ - if (pgprot_val(set_mask) != PTE_VALID || pgprot_val(clear_mask)) + if (pgprot_val(set_mask) != PTE_PRESENT_VALID_KERNEL || + pgprot_val(clear_mask) != PTE_PRESENT_INVALID) flush_tlb_kernel_range(start, start + size); return ret; } @@ -148,6 +154,7 @@ static int change_memory_common(unsigned long addr, int numpages, unsigned long size = PAGE_SIZE * numpages; unsigned long end = start + size; struct vm_struct *area; + int ret; int i; if (!PAGE_ALIGNED(addr)) { @@ -185,8 +192,10 @@ static int change_memory_common(unsigned long addr, int numpages, if (rodata_full && (pgprot_val(set_mask) == PTE_RDONLY || pgprot_val(clear_mask) == PTE_RDONLY)) { for (i = 0; i < area->nr_pages; i++) { - __change_memory_common((u64)page_address(area->pages[i]), + ret = __change_memory_common((u64)page_address(area->pages[i]), PAGE_SIZE, set_mask, clear_mask); + if (ret) + return ret; } } @@ -231,18 +240,18 @@ int set_memory_valid(unsigned long addr, int numpages, int enable) { if (enable) return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(PTE_VALID), - __pgprot(0)); + __pgprot(PTE_PRESENT_VALID_KERNEL), + __pgprot(PTE_PRESENT_INVALID)); else return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(0), - __pgprot(PTE_VALID)); + __pgprot(PTE_PRESENT_INVALID), + __pgprot(PTE_PRESENT_VALID_KERNEL)); } int set_direct_map_invalid_noflush(struct page *page) { - pgprot_t clear_mask = __pgprot(PTE_VALID); - pgprot_t set_mask = __pgprot(0); + pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL); + pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID); if (!can_set_direct_map()) return 0; @@ -253,8 +262,8 @@ int set_direct_map_invalid_noflush(struct page *page) int set_direct_map_default_noflush(struct page *page) { - pgprot_t set_mask = __pgprot(PTE_VALID | PTE_WRITE); - pgprot_t clear_mask = __pgprot(PTE_RDONLY); + pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE); + pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY); if (!can_set_direct_map()) return 0; @@ -290,8 +299,8 @@ static int __set_memory_enc_dec(unsigned long addr, * entries or Synchronous External Aborts caused by RIPAS_EMPTY */ ret = __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(set_prot), - __pgprot(clear_prot | PTE_VALID)); + __pgprot(set_prot | PTE_PRESENT_INVALID), + __pgprot(clear_prot | PTE_PRESENT_VALID_KERNEL)); if (ret) return ret; @@ -305,8 +314,8 @@ static int __set_memory_enc_dec(unsigned long addr, return ret; return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(PTE_VALID), - __pgprot(0)); + __pgprot(PTE_PRESENT_VALID_KERNEL), + __pgprot(PTE_PRESENT_INVALID)); } static int realm_set_memory_encrypted(unsigned long addr, int numpages) @@ -398,15 +407,15 @@ bool kernel_page_present(struct page *page) pud = READ_ONCE(*pudp); if (pud_none(pud)) return false; - if (pud_sect(pud)) - return true; + if (pud_leaf(pud)) + return pud_valid(pud); pmdp = pmd_offset(pudp, addr); pmd = READ_ONCE(*pmdp); if (pmd_none(pmd)) return false; - if (pmd_sect(pmd)) - return true; + if (pmd_leaf(pmd)) + return pmd_valid(pmd); ptep = pte_offset_kernel(pmdp, addr); return pte_valid(__ptep_get(ptep)); diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 86818511962b6..17df19647556f 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -100,6 +100,10 @@ SYM_FUNC_START(cpu_do_suspend) * call stack. */ str x18, [x0, #96] +alternative_if ARM64_HAS_TCR2 + mrs x2, REG_TCR2_EL1 + str x2, [x0, #104] +alternative_else_nop_endif ret SYM_FUNC_END(cpu_do_suspend) @@ -134,6 +138,10 @@ SYM_FUNC_START(cpu_do_resume) msr tcr_el1, x8 msr vbar_el1, x9 msr mdscr_el1, x10 +alternative_if ARM64_HAS_TCR2 + ldr x2, [x0, #104] + msr REG_TCR2_EL1, x2 +alternative_else_nop_endif msr sctlr_el1, x12 set_this_cpu_offset x13 diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c index 18543b603c77b..cca9706a875c3 100644 --- a/arch/arm64/mm/trans_pgd.c +++ b/arch/arm64/mm/trans_pgd.c @@ -31,36 +31,6 @@ static void *trans_alloc(struct trans_pgd_info *info) return info->trans_alloc_page(info->trans_alloc_arg); } -static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) -{ - pte_t pte = __ptep_get(src_ptep); - - if (pte_valid(pte)) { - /* - * Resume will overwrite areas that may be marked - * read only (code, rodata). Clear the RDONLY bit from - * the temporary mappings we use during restore. - */ - __set_pte(dst_ptep, pte_mkwrite_novma(pte)); - } else if (!pte_none(pte)) { - /* - * debug_pagealloc will removed the PTE_VALID bit if - * the page isn't in use by the resume kernel. It may have - * been in use by the original kernel, in which case we need - * to put it back in our copy to do the restore. - * - * Other cases include kfence / vmalloc / memfd_secret which - * may call `set_direct_map_invalid_noflush()`. - * - * Before marking this entry valid, check the pfn should - * be mapped. - */ - BUG_ON(!pfn_valid(pte_pfn(pte))); - - __set_pte(dst_ptep, pte_mkvalid(pte_mkwrite_novma(pte))); - } -} - static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, unsigned long end) { @@ -76,7 +46,11 @@ static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, src_ptep = pte_offset_kernel(src_pmdp, start); do { - _copy_pte(dst_ptep, src_ptep, addr); + pte_t pte = __ptep_get(src_ptep); + + if (pte_none(pte)) + continue; + __set_pte(dst_ptep, pte_mkvalid_k(pte_mkwrite_novma(pte))); } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); return 0; @@ -109,8 +83,7 @@ static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp, if (copy_pte(info, dst_pmdp, src_pmdp, addr, next)) return -ENOMEM; } else { - set_pmd(dst_pmdp, - __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); + set_pmd(dst_pmdp, pmd_mkvalid_k(pmd_mkwrite_novma(pmd))); } } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); @@ -145,8 +118,7 @@ static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp, if (copy_pmd(info, dst_pudp, src_pudp, addr, next)) return -ENOMEM; } else { - set_pud(dst_pudp, - __pud(pud_val(pud) & ~PUD_SECT_RDONLY)); + set_pud(dst_pudp, pud_mkvalid_k(pud_mkwrite_novma(pud))); } } while (dst_pudp++, src_pudp++, addr = next, addr != end); diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0c9a50a1e73e7..107eb71b533a0 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1004,7 +1004,7 @@ static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx) arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) return; - if (capable(CAP_SYS_ADMIN)) + if (ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) return; if (supports_clearbhb(SCOPE_SYSTEM)) { @@ -2122,7 +2122,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) extable_offset = round_up(prog_size + PLT_TARGET_SIZE, extable_align); image_size = extable_offset + extable_size; ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, - sizeof(u32), &header, &image_ptr, + sizeof(u64), &header, &image_ptr, jit_fill_hole); if (!ro_header) { prog = orig_prog; @@ -2939,7 +2939,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, u64 plt_target = 0ULL; bool poking_bpf_entry; - if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) /* Only poking bpf text is supported. Since kernel function * entry is set up by ftrace, we reply on ftrace to poke kernel * functions. diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 1b32c1232d28d..16d123088ddd4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -101,6 +101,7 @@ WORKAROUND_2077057 WORKAROUND_2457168 WORKAROUND_2645198 WORKAROUND_2658417 +WORKAROUND_4193714 WORKAROUND_AMPERE_AC03_CPU_38 WORKAROUND_AMPERE_AC04_CPU_23 WORKAROUND_TRBE_OVERWRITE_FILL_MODE diff --git a/arch/csky/kernel/vmlinux.lds.S b/arch/csky/kernel/vmlinux.lds.S index d718961786d24..81943981b3af4 100644 --- a/arch/csky/kernel/vmlinux.lds.S +++ b/arch/csky/kernel/vmlinux.lds.S @@ -109,6 +109,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index a6ca7dff42153..7ff4011089850 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -45,8 +45,8 @@ static inline void csky_cmpxchg_fixup(struct pt_regs *regs) if (trap_no(regs) != VEC_TLBMODIFIED) return; - if (instruction_pointer(regs) == csky_cmpxchg_stw) - instruction_pointer_set(regs, csky_cmpxchg_ldw); + if (instruction_pointer(regs) == (unsigned long)&csky_cmpxchg_stw) + instruction_pointer_set(regs, (unsigned long)&csky_cmpxchg_ldw); return; } #endif diff --git a/arch/hexagon/kernel/vmlinux.lds.S b/arch/hexagon/kernel/vmlinux.lds.S index 1150b77fa281c..aae22283b5e00 100644 --- a/arch/hexagon/kernel/vmlinux.lds.S +++ b/arch/hexagon/kernel/vmlinux.lds.S @@ -62,6 +62,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .hexagon.attributes 0 : { *(.hexagon.attributes) } diff --git a/arch/loongarch/boot/dts/loongson-2k0500.dtsi b/arch/loongarch/boot/dts/loongson-2k0500.dtsi index 588ebc3bded40..51d990890803c 100644 --- a/arch/loongarch/boot/dts/loongson-2k0500.dtsi +++ b/arch/loongarch/boot/dts/loongson-2k0500.dtsi @@ -131,6 +131,7 @@ reg-names = "main", "isr0"; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <2>; interrupt-parent = <&cpuintc>; interrupts = <2>; @@ -149,6 +150,7 @@ reg-names = "main", "isr0"; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <2>; interrupt-parent = <&cpuintc>; interrupts = <4>; @@ -164,6 +166,7 @@ compatible = "loongson,ls2k0500-eiointc"; reg = <0x0 0x1fe11600 0x0 0xea00>; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <1>; interrupt-parent = <&cpuintc>; interrupts = <3>; diff --git a/arch/loongarch/boot/dts/loongson-2k1000.dtsi b/arch/loongarch/boot/dts/loongson-2k1000.dtsi index d8e01e2534dde..830815a5da863 100644 --- a/arch/loongarch/boot/dts/loongson-2k1000.dtsi +++ b/arch/loongarch/boot/dts/loongson-2k1000.dtsi @@ -46,7 +46,7 @@ }; /* i2c of the dvi eeprom edid */ - i2c-gpio-0 { + i2c-0 { compatible = "i2c-gpio"; scl-gpios = <&gpio0 0 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; sda-gpios = <&gpio0 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; @@ -57,7 +57,7 @@ }; /* i2c of the eeprom edid */ - i2c-gpio-1 { + i2c-1 { compatible = "i2c-gpio"; scl-gpios = <&gpio0 33 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; sda-gpios = <&gpio0 32 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; @@ -114,6 +114,7 @@ <0x0 0x1fe01140 0x0 0x8>; reg-names = "main", "isr0", "isr1"; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <2>; interrupt-parent = <&cpuintc>; interrupts = <2>; @@ -131,6 +132,7 @@ <0x0 0x1fe01148 0x0 0x8>; reg-names = "main", "isr0", "isr1"; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <2>; interrupt-parent = <&cpuintc>; interrupts = <3>; @@ -437,54 +439,47 @@ gmac0: ethernet@3,0 { reg = <0x1800 0x0 0x0 0x0 0x0>; - interrupt-parent = <&liointc0>; - interrupts = <12 IRQ_TYPE_LEVEL_HIGH>, - <13 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&liointc0 12 IRQ_TYPE_LEVEL_HIGH>, + <&liointc0 13 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq", "eth_lpi"; status = "disabled"; }; gmac1: ethernet@3,1 { reg = <0x1900 0x0 0x0 0x0 0x0>; - interrupt-parent = <&liointc0>; - interrupts = <14 IRQ_TYPE_LEVEL_HIGH>, - <15 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&liointc0 14 IRQ_TYPE_LEVEL_HIGH>, + <&liointc0 15 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq", "eth_lpi"; status = "disabled"; }; ehci0: usb@4,1 { reg = <0x2100 0x0 0x0 0x0 0x0>; - interrupt-parent = <&liointc1>; - interrupts = <18 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&liointc1 18 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; ohci0: usb@4,2 { reg = <0x2200 0x0 0x0 0x0 0x0>; - interrupt-parent = <&liointc1>; - interrupts = <19 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&liointc1 19 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; display@6,0 { reg = <0x3000 0x0 0x0 0x0 0x0>; - interrupt-parent = <&liointc0>; - interrupts = <28 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&liointc0 28 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; hda@7,0 { reg = <0x3800 0x0 0x0 0x0 0x0>; - interrupt-parent = <&liointc0>; - interrupts = <4 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&liointc0 4 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; sata: sata@8,0 { reg = <0x4000 0x0 0x0 0x0 0x0>; - interrupt-parent = <&liointc0>; - interrupts = <19 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&liointc0 19 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; diff --git a/arch/loongarch/boot/dts/loongson-2k2000.dtsi b/arch/loongarch/boot/dts/loongson-2k2000.dtsi index 00cc485b753b1..ff094e60af1c9 100644 --- a/arch/loongarch/boot/dts/loongson-2k2000.dtsi +++ b/arch/loongarch/boot/dts/loongson-2k2000.dtsi @@ -126,6 +126,7 @@ reg = <0x0 0x1fe01400 0x0 0x64>; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <2>; interrupt-parent = <&cpuintc>; interrupts = <2>; @@ -140,6 +141,7 @@ compatible = "loongson,ls2k2000-eiointc"; reg = <0x0 0x1fe01600 0x0 0xea00>; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <1>; interrupt-parent = <&cpuintc>; interrupts = <3>; @@ -149,6 +151,7 @@ compatible = "loongson,pch-pic-1.0"; reg = <0x0 0x10000000 0x0 0x400>; interrupt-controller; + #address-cells = <0>; #interrupt-cells = <2>; loongson,pic-base-vec = <0>; interrupt-parent = <&eiointc>; @@ -291,65 +294,57 @@ gmac0: ethernet@3,0 { reg = <0x1800 0x0 0x0 0x0 0x0>; - interrupts = <12 IRQ_TYPE_LEVEL_HIGH>, - <13 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&pic 12 IRQ_TYPE_LEVEL_HIGH>, + <&pic 13 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq", "eth_lpi"; - interrupt-parent = <&pic>; status = "disabled"; }; gmac1: ethernet@3,1 { reg = <0x1900 0x0 0x0 0x0 0x0>; - interrupts = <14 IRQ_TYPE_LEVEL_HIGH>, - <15 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&pic 14 IRQ_TYPE_LEVEL_HIGH>, + <&pic 15 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq", "eth_lpi"; - interrupt-parent = <&pic>; status = "disabled"; }; gmac2: ethernet@3,2 { reg = <0x1a00 0x0 0x0 0x0 0x0>; - interrupts = <17 IRQ_TYPE_LEVEL_HIGH>, - <18 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&pic 17 IRQ_TYPE_LEVEL_HIGH>, + <&pic 18 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq", "eth_lpi"; - interrupt-parent = <&pic>; status = "disabled"; }; xhci0: usb@4,0 { reg = <0x2000 0x0 0x0 0x0 0x0>; - interrupts = <48 IRQ_TYPE_LEVEL_HIGH>; - interrupt-parent = <&pic>; + interrupts-extended = <&pic 48 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; xhci1: usb@19,0 { reg = <0xc800 0x0 0x0 0x0 0x0>; - interrupts = <22 IRQ_TYPE_LEVEL_HIGH>; - interrupt-parent = <&pic>; + interrupts-extended = <&pic 22 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; display@6,1 { reg = <0x3100 0x0 0x0 0x0 0x0>; - interrupts = <28 IRQ_TYPE_LEVEL_HIGH>; - interrupt-parent = <&pic>; + interrupts-extended = <&pic 28 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; i2s@7,0 { reg = <0x3800 0x0 0x0 0x0 0x0>; - interrupts = <78 IRQ_TYPE_LEVEL_HIGH>, - <79 IRQ_TYPE_LEVEL_HIGH>; + interrupts-extended = <&pic 78 IRQ_TYPE_LEVEL_HIGH>, + <&pic 79 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "tx", "rx"; - interrupt-parent = <&pic>; status = "disabled"; }; sata: sata@8,0 { reg = <0x4000 0x0 0x0 0x0 0x0>; - interrupts = <16 IRQ_TYPE_LEVEL_HIGH>; - interrupt-parent = <&pic>; + interrupts-extended = <&pic 16 IRQ_TYPE_LEVEL_HIGH>; status = "disabled"; }; diff --git a/arch/loongarch/include/asm/linkage.h b/arch/loongarch/include/asm/linkage.h index e2eca1a25b4ef..a1bd6a3ee03a1 100644 --- a/arch/loongarch/include/asm/linkage.h +++ b/arch/loongarch/include/asm/linkage.h @@ -41,4 +41,40 @@ .cfi_endproc; \ SYM_END(name, SYM_T_NONE) +/* + * This is for the signal handler trampoline, which is used as the return + * address of the signal handlers in userspace instead of called normally. + * The long standing libgcc bug https://gcc.gnu.org/PR124050 requires a + * nop between .cfi_startproc and the actual address of the trampoline, so + * we cannot simply use SYM_FUNC_START. + * + * This wrapper also contains all the .cfi_* directives for recovering + * the content of the GPRs and the "return address" (where the rt_sigreturn + * syscall will jump to), assuming there is a struct rt_sigframe (where + * a struct sigcontext containing those information we need to recover) at + * $sp. The "DWARF for the LoongArch(TM) Architecture" manual states + * column 0 is for $zero, but it does not make too much sense to + * save/restore the hardware zero register. Repurpose this column here + * for the return address (here it's not the content of $ra we cannot use + * the default column 3). + */ +#define SYM_SIGFUNC_START(name) \ + .cfi_startproc; \ + .cfi_signal_frame; \ + .cfi_def_cfa 3, RT_SIGFRAME_SC; \ + .cfi_return_column 0; \ + .cfi_offset 0, SC_PC; \ + \ + .irp num, 1, 2, 3, 4, 5, 6, 7, 8, \ + 9, 10, 11, 12, 13, 14, 15, 16, \ + 17, 18, 19, 20, 21, 22, 23, 24, \ + 25, 26, 27, 28, 29, 30, 31; \ + .cfi_offset \num, SC_REGS + \num * SZREG; \ + .endr; \ + \ + nop; \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) + +#define SYM_SIGFUNC_END(name) SYM_FUNC_END(name) + #endif diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 03fb60432fde7..9a7029285fd51 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -297,9 +297,9 @@ static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) #define __swp_offset(x) ((x).val >> 24) #define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) -#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) +#define __swp_entry_to_pte(x) __pte((x).val) #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) -#define __swp_entry_to_pmd(x) ((pmd_t) { (x).val | _PAGE_HUGE }) +#define __swp_entry_to_pmd(x) __pmd((x).val | _PAGE_HUGE) static inline bool pte_swp_exclusive(pte_t pte) { diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 3c2fb16b11b64..f81375e5e89c0 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -7,6 +7,7 @@ #define _LOONGARCH_SETUP_H #include +#include #include #include @@ -14,6 +15,8 @@ extern unsigned long eentry; extern unsigned long tlbrentry; +extern unsigned long pcpu_handlers[NR_CPUS]; +extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; extern char init_command_line[COMMAND_LINE_SIZE]; extern void tlb_init(int cpu); extern void cpu_cache_init(void); diff --git a/arch/loongarch/include/asm/sigframe.h b/arch/loongarch/include/asm/sigframe.h new file mode 100644 index 0000000000000..109298b8d7e0b --- /dev/null +++ b/arch/loongarch/include/asm/sigframe.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#include +#include + +struct rt_sigframe { + struct siginfo rs_info; + struct ucontext rs_uctx; +}; diff --git a/arch/loongarch/include/asm/topology.h b/arch/loongarch/include/asm/topology.h index f06e7ff25bb7c..6b79d6183085a 100644 --- a/arch/loongarch/include/asm/topology.h +++ b/arch/loongarch/include/asm/topology.h @@ -12,7 +12,7 @@ extern cpumask_t cpus_on_node[]; -#define cpumask_of_node(node) (&cpus_on_node[node]) +#define cpumask_of_node(node) ((node) == NUMA_NO_NODE ? cpu_all_mask : &cpus_on_node[node]) struct pci_bus; extern int pcibus_to_node(struct pci_bus *); diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h index 0d22991ae430d..a9b40d2313436 100644 --- a/arch/loongarch/include/asm/uaccess.h +++ b/arch/loongarch/include/asm/uaccess.h @@ -196,8 +196,13 @@ do { \ \ __get_kernel_common(*((type *)(dst)), sizeof(type), \ (__force type *)(src)); \ - if (unlikely(__gu_err)) \ + if (unlikely(__gu_err)) { \ + pr_info("%s: memory access failed, ecode 0x%x\n", \ + __func__, read_csr_excode()); \ + pr_info("%s: the caller is %pS\n", \ + __func__, __builtin_return_address(0)); \ goto err_label; \ + } \ } while (0) #define __put_kernel_nofault(dst, src, type, err_label) \ @@ -207,8 +212,13 @@ do { \ \ __pu_val = *(__force type *)(src); \ __put_kernel_common(((type *)(dst)), sizeof(type)); \ - if (unlikely(__pu_err)) \ + if (unlikely(__pu_err)) { \ + pr_info("%s: memory access failed, ecode 0x%x\n", \ + __func__, read_csr_excode()); \ + pr_info("%s: the caller is %pS\n", \ + __func__, __builtin_return_address(0)); \ goto err_label; \ + } \ } while (0) extern unsigned long __copy_user(void *to, const void *from, __kernel_size_t n); diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 3017c71576009..2cc953f113ac0 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static void __used output_ptreg_defines(void) @@ -220,6 +221,7 @@ static void __used output_sc_defines(void) COMMENT("Linux sigcontext offsets."); OFFSET(SC_REGS, sigcontext, sc_regs); OFFSET(SC_PC, sigcontext, sc_pc); + OFFSET(RT_SIGFRAME_SC, rt_sigframe, rs_uctx.uc_mcontext); BLANK(); } diff --git a/arch/loongarch/kernel/env.c b/arch/loongarch/kernel/env.c index 23bd5ae2212c2..b28554f702d25 100644 --- a/arch/loongarch/kernel/env.c +++ b/arch/loongarch/kernel/env.c @@ -42,16 +42,15 @@ static int __init init_cpu_fullname(void) int cpu, ret; char *cpuname; const char *model; - struct device_node *root; /* Parsing cpuname from DTS model property */ - root = of_find_node_by_path("/"); - ret = of_property_read_string(root, "model", &model); + ret = of_property_read_string(of_root, "model", &model); if (ret == 0) { cpuname = kstrdup(model, GFP_KERNEL); + if (!cpuname) + return -ENOMEM; loongson_sysconf.cpuname = strsep(&cpuname, " "); } - of_node_put(root); if (loongson_sysconf.cpuname && !strncmp(loongson_sysconf.cpuname, "Loongson", 8)) { for (cpu = 0; cpu < NR_CPUS; cpu++) diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index bf037f0c6b26c..9a9c34ea24b60 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -246,18 +246,21 @@ static int text_copy_cb(void *data) if (smp_processor_id() == copy->cpu) { ret = copy_to_kernel_nofault(copy->dst, copy->src, copy->len); - if (ret) + if (ret) { pr_err("%s: operation failed\n", __func__); + return ret; + } } flush_icache_range((unsigned long)copy->dst, (unsigned long)copy->dst + copy->len); - return ret; + return 0; } int larch_insn_text_copy(void *dst, void *src, size_t len) { int ret = 0; + int err = 0; size_t start, end; struct insn_copy copy = { .dst = dst, @@ -269,9 +272,19 @@ int larch_insn_text_copy(void *dst, void *src, size_t len) start = round_down((size_t)dst, PAGE_SIZE); end = round_up((size_t)dst + len, PAGE_SIZE); - set_memory_rw(start, (end - start) / PAGE_SIZE); + err = set_memory_rw(start, (end - start) / PAGE_SIZE); + if (err) { + pr_info("%s: set_memory_rw() failed\n", __func__); + return err; + } + ret = stop_machine(text_copy_cb, ©, cpu_online_mask); - set_memory_rox(start, (end - start) / PAGE_SIZE); + + err = set_memory_rox(start, (end - start) / PAGE_SIZE); + if (err) { + pr_info("%s: set_memory_rox() failed\n", __func__); + return err; + } return ret; } diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S index d6b474ad1d5e5..5729c20e5b8b0 100644 --- a/arch/loongarch/kernel/mcount_dyn.S +++ b/arch/loongarch/kernel/mcount_dyn.S @@ -94,7 +94,6 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) * at the callsite, so there is no need to restore the T series regs. */ ftrace_common_return: - PTR_L ra, sp, PT_R1 PTR_L a0, sp, PT_R4 PTR_L a1, sp, PT_R5 PTR_L a2, sp, PT_R6 @@ -104,12 +103,17 @@ ftrace_common_return: PTR_L a6, sp, PT_R10 PTR_L a7, sp, PT_R11 PTR_L fp, sp, PT_R22 - PTR_L t0, sp, PT_ERA PTR_L t1, sp, PT_R13 - PTR_ADDI sp, sp, PT_SIZE bnez t1, .Ldirect + + PTR_L ra, sp, PT_R1 + PTR_L t0, sp, PT_ERA + PTR_ADDI sp, sp, PT_SIZE jr t0 .Ldirect: + PTR_L t0, sp, PT_R1 + PTR_L ra, sp, PT_ERA + PTR_ADDI sp, sp, PT_SIZE jr t1 SYM_CODE_END(ftrace_common) @@ -161,6 +165,8 @@ SYM_CODE_END(return_to_handler) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS SYM_CODE_START(ftrace_stub_direct_tramp) UNWIND_HINT_UNDEFINED - jr t0 + move t1, ra + move ra, t0 + jr t1 SYM_CODE_END(ftrace_stub_direct_tramp) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ diff --git a/arch/loongarch/kernel/perf_event.c b/arch/loongarch/kernel/perf_event.c index 9d257c8519c90..e34a6fb33e11c 100644 --- a/arch/loongarch/kernel/perf_event.c +++ b/arch/loongarch/kernel/perf_event.c @@ -626,6 +626,18 @@ static const struct loongarch_perf_event *loongarch_pmu_map_cache_event(u64 conf return pev; } +static inline bool loongarch_pmu_event_requires_counter(const struct perf_event *event) +{ + switch (event->attr.type) { + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + case PERF_TYPE_RAW: + return true; + default: + return false; + } +} + static int validate_group(struct perf_event *event) { struct cpu_hw_events fake_cpuc; @@ -633,15 +645,18 @@ static int validate_group(struct perf_event *event) memset(&fake_cpuc, 0, sizeof(fake_cpuc)); - if (loongarch_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0) + if (loongarch_pmu_event_requires_counter(leader) && + loongarch_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0) return -EINVAL; for_each_sibling_event(sibling, leader) { - if (loongarch_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0) + if (loongarch_pmu_event_requires_counter(sibling) && + loongarch_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0) return -EINVAL; } - if (loongarch_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0) + if (loongarch_pmu_event_requires_counter(event) && + loongarch_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0) return -EINVAL; return 0; diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index efd9edf65603c..d1e04f9e0f795 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -130,6 +130,11 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) preempt_enable(); + if (IS_ENABLED(CONFIG_RANDSTRUCT)) { + memcpy(dst, src, sizeof(struct task_struct)); + return 0; + } + if (!used_math()) memcpy(dst, src, offsetof(struct task_struct, thread.fpu.fpr)); else diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c index b5e2312a2fca5..76abbb8d29316 100644 --- a/arch/loongarch/kernel/relocate.c +++ b/arch/loongarch/kernel/relocate.c @@ -183,7 +183,7 @@ static inline void __init *determine_relocation_address(void) if (kaslr_disabled()) return destination; - kernel_length = (long)_end - (long)_text; + kernel_length = (unsigned long)_end - (unsigned long)_text; random_offset = get_random_boot() << 16; random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); @@ -232,7 +232,7 @@ unsigned long __init relocate_kernel(void) early_memunmap(cmdline, COMMAND_LINE_SIZE); if (random_offset) { - kernel_length = (long)(_end) - (long)(_text); + kernel_length = (unsigned long)(_end) - (unsigned long)(_text); /* Copy the kernel to it's new location */ memcpy(location_new, _text, kernel_length); diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 25a87378e48e4..2b260d15b2e25 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -56,6 +56,7 @@ #define SMBIOS_FREQLOW_MASK 0xFF #define SMBIOS_CORE_PACKAGE_OFFSET 0x23 #define SMBIOS_THREAD_PACKAGE_OFFSET 0x25 +#define SMBIOS_THREAD_PACKAGE_2_OFFSET 0x2E #define LOONGSON_EFI_ENABLE (1 << 3) unsigned long fw_arg0, fw_arg1, fw_arg2; @@ -126,7 +127,12 @@ static void __init parse_cpu_table(const struct dmi_header *dm) cpu_clock_freq = freq_temp * 1000000; loongson_sysconf.cpuname = (void *)dmi_string_parse(dm, dmi_data[16]); - loongson_sysconf.cores_per_package = *(dmi_data + SMBIOS_THREAD_PACKAGE_OFFSET); + loongson_sysconf.cores_per_package = *(u8 *)(dmi_data + SMBIOS_THREAD_PACKAGE_OFFSET); + if (dm->length >= 0x30 && loongson_sysconf.cores_per_package == 0xff) { + /* SMBIOS 3.0+ has ThreadCount2 for more than 255 threads */ + loongson_sysconf.cores_per_package = + *(u16 *)(dmi_data + SMBIOS_THREAD_PACKAGE_2_OFFSET); + } pr_info("CpuClock = %llu\n", cpu_clock_freq); } @@ -415,6 +421,7 @@ static void __init arch_mem_init(char **cmdline_p) PFN_UP(__pa_symbol(&__nosave_end))); memblock_dump_all(); + memblock_set_bottom_up(false); early_memtest(PFN_PHYS(ARCH_PFN_OFFSET), PFN_PHYS(max_low_pfn)); } diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c index c9f7ca778364e..d4151d2fb82ee 100644 --- a/arch/loongarch/kernel/signal.c +++ b/arch/loongarch/kernel/signal.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -51,11 +52,6 @@ #define lock_lbt_owner() ({ preempt_disable(); pagefault_disable(); }) #define unlock_lbt_owner() ({ pagefault_enable(); preempt_enable(); }) -struct rt_sigframe { - struct siginfo rs_info; - struct ucontext rs_uctx; -}; - struct _ctx_layout { struct sctx_info *addr; unsigned int size; diff --git a/arch/loongarch/kernel/switch.S b/arch/loongarch/kernel/switch.S index 9c23cb7e432f1..3007e909e0d80 100644 --- a/arch/loongarch/kernel/switch.S +++ b/arch/loongarch/kernel/switch.S @@ -25,8 +25,8 @@ SYM_FUNC_START(__switch_to) stptr.d a4, a0, THREAD_SCHED_CFA #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) la t7, __stack_chk_guard - LONG_L t8, a1, TASK_STACK_CANARY - LONG_S t8, t7, 0 + ldptr.d t8, a1, TASK_STACK_CANARY + stptr.d t8, t7, 0 #endif move tp, a2 cpu_restore_nonscratch a1 diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index da5926fead4af..8e51ce004572c 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -535,10 +535,15 @@ asmlinkage void noinstr do_fpe(struct pt_regs *regs, unsigned long fcsr) asmlinkage void noinstr do_ade(struct pt_regs *regs) { irqentry_state_t state = irqentry_enter(regs); + unsigned int esubcode = FIELD_GET(CSR_ESTAT_ESUBCODE, regs->csr_estat); + + if ((esubcode == EXSUBCODE_ADEM) && fixup_exception(regs)) + goto out; die_if_kernel("Kernel ade access", regs); force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)regs->csr_badvaddr); +out: irqentry_exit(regs, state); } diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c index 0d5fa64a22252..85c2fcb76930c 100644 --- a/arch/loongarch/kernel/unwind_orc.c +++ b/arch/loongarch/kernel/unwind_orc.c @@ -358,13 +358,21 @@ static bool is_entry_func(unsigned long addr) static inline unsigned long bt_address(unsigned long ra) { - extern unsigned long eentry; +#if defined(CONFIG_NUMA) && !defined(CONFIG_PREEMPT_RT) + int cpu; + int vec_sz = sizeof(exception_handlers); - if (__kernel_text_address(ra)) - return ra; + for_each_possible_cpu(cpu) { + if (!pcpu_handlers[cpu]) + continue; - if (__module_text_address(ra)) - return ra; + if (ra >= pcpu_handlers[cpu] && + ra < pcpu_handlers[cpu] + vec_sz) { + ra = ra + eentry - pcpu_handlers[cpu]; + break; + } + } +#endif if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) { unsigned long func; @@ -383,10 +391,13 @@ static inline unsigned long bt_address(unsigned long ra) break; } - return func + offset; + ra = func + offset; } - return ra; + if (__kernel_text_address(ra)) + return ra; + + return 0; } bool unwind_next_frame(struct unwind_state *state) @@ -508,13 +519,10 @@ bool unwind_next_frame(struct unwind_state *state) state->pc = bt_address(pc); if (!state->pc) { - pr_err("cannot find unwind pc at %p\n", (void *)pc); + pr_err("cannot find unwind pc at %px\n", (void *)pc); goto err; } - if (!__kernel_text_address(state->pc)) - goto err; - return true; err: diff --git a/arch/loongarch/kernel/unwind_prologue.c b/arch/loongarch/kernel/unwind_prologue.c index 729e775bd40dd..da07acad7973a 100644 --- a/arch/loongarch/kernel/unwind_prologue.c +++ b/arch/loongarch/kernel/unwind_prologue.c @@ -23,10 +23,6 @@ extern const int unwind_hint_lasx; extern const int unwind_hint_lbt; extern const int unwind_hint_ri; extern const int unwind_hint_watch; -extern unsigned long eentry; -#ifdef CONFIG_NUMA -extern unsigned long pcpu_handlers[NR_CPUS]; -#endif static inline bool scan_handlers(unsigned long entry_offset) { @@ -65,7 +61,7 @@ static inline bool scan_handlers(unsigned long entry_offset) static inline bool fix_exception(unsigned long pc) { -#ifdef CONFIG_NUMA +#if defined(CONFIG_NUMA) && !defined(CONFIG_PREEMPT_RT) int cpu; for_each_possible_cpu(cpu) { diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index 08ea921cdec16..d0e1377a041d6 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -147,6 +147,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS #ifdef CONFIG_EFI_STUB diff --git a/arch/loongarch/kvm/intc/eiointc.c b/arch/loongarch/kvm/intc/eiointc.c index a1cc116b4dace..78510fce1a47b 100644 --- a/arch/loongarch/kvm/intc/eiointc.c +++ b/arch/loongarch/kvm/intc/eiointc.c @@ -83,7 +83,7 @@ static inline void eiointc_update_sw_coremap(struct loongarch_eiointc *s, if (!(s->status & BIT(EIOINTC_ENABLE_CPU_ENCODE))) { cpuid = ffs(cpuid) - 1; - cpuid = (cpuid >= 4) ? 0 : cpuid; + cpuid = ((cpuid < 0) || (cpuid >= 4)) ? 0 : cpuid; } vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid); @@ -679,6 +679,7 @@ static void kvm_eiointc_destroy(struct kvm_device *dev) kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device); kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device_vext); kfree(eiointc); + kfree(dev); } static struct kvm_device_ops kvm_eiointc_dev_ops = { diff --git a/arch/loongarch/kvm/intc/ipi.c b/arch/loongarch/kvm/intc/ipi.c index 05cefd29282e8..1058c13dba7f4 100644 --- a/arch/loongarch/kvm/intc/ipi.c +++ b/arch/loongarch/kvm/intc/ipi.c @@ -459,6 +459,7 @@ static void kvm_ipi_destroy(struct kvm_device *dev) ipi = kvm->arch.ipi; kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device); kfree(ipi); + kfree(dev); } static struct kvm_device_ops kvm_ipi_dev_ops = { diff --git a/arch/loongarch/kvm/intc/pch_pic.c b/arch/loongarch/kvm/intc/pch_pic.c index a698a73de399b..4addb34bf432b 100644 --- a/arch/loongarch/kvm/intc/pch_pic.c +++ b/arch/loongarch/kvm/intc/pch_pic.c @@ -475,6 +475,7 @@ static void kvm_pch_pic_destroy(struct kvm_device *dev) /* unregister pch pic device and free it's memory */ kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &s->device); kfree(s); + kfree(dev); } static struct kvm_device_ops kvm_pch_pic_dev_ops = { diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 1245a6b358966..14bd36597ab33 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -13,7 +13,7 @@ #define CREATE_TRACE_POINTS #include "trace.h" -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, int_exits), STATS_DESC_COUNTER(VCPU, idle_exits), @@ -561,6 +561,9 @@ struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid) { struct kvm_phyid_map *map; + if (cpuid < 0) + return NULL; + if (cpuid >= KVM_MAX_PHYID) return NULL; diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index a49b1c1a3dd1f..85246a70d95d3 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -9,7 +9,7 @@ #include #include -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_ICOUNTER(VM, pages), STATS_DESC_ICOUNTER(VM, hugepages), diff --git a/arch/loongarch/mm/cache.c b/arch/loongarch/mm/cache.c index 6be04d36ca076..496916845ff76 100644 --- a/arch/loongarch/mm/cache.c +++ b/arch/loongarch/mm/cache.c @@ -160,8 +160,8 @@ void cpu_cache_init(void) static const pgprot_t protection_map[16] = { [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER | - _PAGE_PROTNONE | _PAGE_NO_EXEC | - _PAGE_NO_READ), + _PAGE_NO_EXEC | _PAGE_NO_READ | + (_PAGE_PROTNONE ? : _PAGE_PRESENT)), [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC), @@ -180,8 +180,8 @@ static const pgprot_t protection_map[16] = { [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT), [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER | - _PAGE_PROTNONE | _PAGE_NO_EXEC | - _PAGE_NO_READ), + _PAGE_NO_EXEC | _PAGE_NO_READ | + (_PAGE_PROTNONE ? : _PAGE_PRESENT)), [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC), diff --git a/arch/loongarch/mm/kasan_init.c b/arch/loongarch/mm/kasan_init.c index 170da98ad4f55..0fc02ca064573 100644 --- a/arch/loongarch/mm/kasan_init.c +++ b/arch/loongarch/mm/kasan_init.c @@ -40,39 +40,43 @@ static pgd_t kasan_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); #define __pte_none(early, pte) (early ? pte_none(pte) : \ ((pte_val(pte) & _PFN_MASK) == (unsigned long)__pa(kasan_early_shadow_page))) -void *kasan_mem_to_shadow(const void *addr) +static void *mem_to_shadow(const void *addr) { - if (!kasan_enabled()) { + unsigned long offset = 0; + unsigned long maddr = (unsigned long)addr; + unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; + + if (maddr >= FIXADDR_START) return (void *)(kasan_early_shadow_page); - } else { - unsigned long maddr = (unsigned long)addr; - unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; - unsigned long offset = 0; - - if (maddr >= FIXADDR_START) - return (void *)(kasan_early_shadow_page); - - maddr &= XRANGE_SHADOW_MASK; - switch (xrange) { - case XKPRANGE_CC_SEG: - offset = XKPRANGE_CC_SHADOW_OFFSET; - break; - case XKPRANGE_UC_SEG: - offset = XKPRANGE_UC_SHADOW_OFFSET; - break; - case XKPRANGE_WC_SEG: - offset = XKPRANGE_WC_SHADOW_OFFSET; - break; - case XKVRANGE_VC_SEG: - offset = XKVRANGE_VC_SHADOW_OFFSET; - break; - default: - WARN_ON(1); - return NULL; - } - return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); + maddr &= XRANGE_SHADOW_MASK; + switch (xrange) { + case XKPRANGE_CC_SEG: + offset = XKPRANGE_CC_SHADOW_OFFSET; + break; + case XKPRANGE_UC_SEG: + offset = XKPRANGE_UC_SHADOW_OFFSET; + break; + case XKPRANGE_WC_SEG: + offset = XKPRANGE_WC_SHADOW_OFFSET; + break; + case XKVRANGE_VC_SEG: + offset = XKVRANGE_VC_SHADOW_OFFSET; + break; + default: + WARN_ON(1); + return NULL; } + + return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); +} + +void *kasan_mem_to_shadow(const void *addr) +{ + if (kasan_enabled()) + return mem_to_shadow(addr); + else + return (void *)(kasan_early_shadow_page); } const void *kasan_shadow_to_mem(const void *shadow_addr) @@ -293,11 +297,8 @@ void __init kasan_init(void) /* Maps everything to a single page of zeroes */ kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, true); - kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START), - kasan_mem_to_shadow((void *)KFENCE_AREA_END)); - - /* Enable KASAN here before kasan_mem_to_shadow(). */ - kasan_init_generic(); + kasan_populate_early_shadow(mem_to_shadow((void *)VMALLOC_START), + mem_to_shadow((void *)KFENCE_AREA_END)); /* Populate the linear mapping */ for_each_mem_range(i, &pa_start, &pa_end) { @@ -307,13 +308,13 @@ void __init kasan_init(void) if (start >= end) break; - kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), - (unsigned long)kasan_mem_to_shadow(end), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow(start), + (unsigned long)mem_to_shadow(end), NUMA_NO_NODE); } /* Populate modules mapping */ - kasan_map_populate((unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR), - (unsigned long)kasan_mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow((void *)MODULES_VADDR), + (unsigned long)mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); /* * KAsan may reuse the contents of kasan_early_shadow_pte directly, so we * should make sure that it maps the zero page read-only. @@ -328,4 +329,5 @@ void __init kasan_init(void) /* At this point kasan is fully initialized. Enable error messages */ init_task.kasan_depth = 0; + kasan_init_generic(); } diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c index 3b427b319db21..24add95ecb65e 100644 --- a/arch/loongarch/mm/tlb.c +++ b/arch/loongarch/mm/tlb.c @@ -202,7 +202,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep local_irq_restore(flags); } -static void setup_ptwalker(void) +static void __no_sanitize_address setup_ptwalker(void) { unsigned long pwctl0, pwctl1; unsigned long pgd_i = 0, pgd_w = 0; @@ -260,7 +260,6 @@ static void output_pgtable_bits_defines(void) #ifdef CONFIG_NUMA unsigned long pcpu_handlers[NR_CPUS]; #endif -extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; static void setup_tlb_handler(int cpu) { diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index f97dc99364017..e9d666508ae28 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -139,6 +139,7 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; + move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); /* Reserve space for the move_imm + jirl instruction */ for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) emit_insn(ctx, nop); @@ -238,7 +239,7 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) * Call the next bpf prog and skip the first instruction * of TCC initialization. */ - emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 6); + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 7); } } @@ -280,6 +281,8 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) * goto out; */ tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; + emit_zext_32(ctx, a2, true); + off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ @@ -950,6 +953,22 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); } + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { + const struct btf_func_model *m; + int i; + + m = bpf_jit_find_kfunc_model(ctx->prog, insn); + if (!m) + return -EINVAL; + + for (i = 0; i < m->nr_args; i++) { + u8 reg = regmap[BPF_REG_1 + i]; + bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG; + + emit_abi_ext(ctx, reg, m->arg_size[i], sign); + } + } + move_addr(ctx, t1, func_addr); emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); @@ -1265,7 +1284,7 @@ static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) return 0; } - return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, (u64)target); + return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target); } static int emit_call(struct jit_ctx *ctx, u64 addr) @@ -1288,6 +1307,10 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, void *old_addr, void *new_addr) { int ret; + unsigned long size = 0; + unsigned long offset = 0; + void *image = NULL; + char namebuf[KSYM_NAME_LEN]; bool is_call = (poke_type == BPF_MOD_CALL); u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; @@ -1295,9 +1318,20 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, /* Only poking bpf text is supported. Since kernel function entry * is set up by ftrace, we rely on ftrace to poke kernel functions. */ - if (!is_bpf_text_address((unsigned long)ip)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) return -ENOTSUPP; + image = ip - offset; + + /* zero offset means we're poking bpf prog entry */ + if (offset == 0) { + /* skip to the nop instruction in bpf prog entry: + * move t0, ra + * nop + */ + ip = image + LOONGARCH_INSN_SIZE; + } + ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call); if (ret) return ret; @@ -1619,14 +1653,12 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i /* To traced function */ /* Ftrace jump skips 2 NOP instructions */ - if (is_kernel_text((unsigned long)orig_call)) + if (is_kernel_text((unsigned long)orig_call) || + is_module_text_address((unsigned long)orig_call)) orig_call += LOONGARCH_FENTRY_NBYTES; /* Direct jump skips 5 NOP instructions */ else if (is_bpf_text_address((unsigned long)orig_call)) orig_call += LOONGARCH_BPF_FENTRY_NBYTES; - /* Module tracing not supported - cause kernel lockups */ - else if (is_module_text_address((unsigned long)orig_call)) - return -ENOTSUPP; if (flags & BPF_TRAMP_F_CALL_ORIG) { move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); @@ -1719,12 +1751,16 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); - if (flags & BPF_TRAMP_F_SKIP_FRAME) + if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* return to parent function */ - emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); - else - /* return to traced function */ + move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); + } else { + /* return to traced function */ + move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA); + move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0); + } } ret = ctx->idx; diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h index 5697158fd1645..75b6330030a9d 100644 --- a/arch/loongarch/net/bpf_jit.h +++ b/arch/loongarch/net/bpf_jit.h @@ -88,6 +88,32 @@ static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, boo emit_insn(ctx, addiw, reg, reg, 0); } +/* Emit proper extension according to ABI requirements. + * Note that it requires a value of size `size` already resides in register `reg`. + */ +static inline void emit_abi_ext(struct jit_ctx *ctx, int reg, u8 size, bool sign) +{ + /* ABI requires unsigned char/short to be zero-extended */ + if (!sign && (size == 1 || size == 2)) + return; + + switch (size) { + case 1: + emit_insn(ctx, extwb, reg, reg); + break; + case 2: + emit_insn(ctx, extwh, reg, reg); + break; + case 4: + emit_insn(ctx, addiw, reg, reg, 0); + break; + case 8: + break; + default: + pr_warn("bpf_jit: invalid size %d for extension\n", size); + } +} + static inline void move_addr(struct jit_ctx *ctx, enum loongarch_gpr rd, u64 addr) { u64 imm_11_0, imm_31_12, imm_51_32, imm_63_52; diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c index d9fc5d520b377..d233ea2218fe0 100644 --- a/arch/loongarch/pci/pci.c +++ b/arch/loongarch/pci/pci.c @@ -5,15 +5,21 @@ #include #include #include +#include #include #include #include +#include #include #include #define PCI_DEVICE_ID_LOONGSON_HOST 0x7a00 #define PCI_DEVICE_ID_LOONGSON_DC1 0x7a06 #define PCI_DEVICE_ID_LOONGSON_DC2 0x7a36 +#define PCI_DEVICE_ID_LOONGSON_DC3 0x7a46 +#define PCI_DEVICE_ID_LOONGSON_GPU1 0x7a15 +#define PCI_DEVICE_ID_LOONGSON_GPU2 0x7a25 +#define PCI_DEVICE_ID_LOONGSON_GPU3 0x7a35 int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn, int reg, int len, u32 *val) @@ -97,3 +103,79 @@ static void pci_fixup_vgadev(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC1, pci_fixup_vgadev); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC2, pci_fixup_vgadev); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC3, pci_fixup_vgadev); + +#define CRTC_NUM_MAX 2 +#define CRTC_OUTPUT_ENABLE 0x100 + +static void loongson_gpu_fixup_dma_hang(struct pci_dev *pdev, bool on) +{ + u32 i, val, count, crtc_offset, device; + void __iomem *crtc_reg, *base, *regbase; + static u32 crtc_status[CRTC_NUM_MAX] = { 0 }; + + base = pdev->bus->ops->map_bus(pdev->bus, pdev->devfn + 1, 0); + device = readw(base + PCI_DEVICE_ID); + + regbase = ioremap(readq(base + PCI_BASE_ADDRESS_0) & ~0xffull, SZ_64K); + if (!regbase) { + pci_err(pdev, "Failed to ioremap()\n"); + return; + } + + switch (device) { + case PCI_DEVICE_ID_LOONGSON_DC2: + crtc_reg = regbase + 0x1240; + crtc_offset = 0x10; + break; + case PCI_DEVICE_ID_LOONGSON_DC3: + crtc_reg = regbase; + crtc_offset = 0x400; + break; + } + + for (i = 0; i < CRTC_NUM_MAX; i++, crtc_reg += crtc_offset) { + val = readl(crtc_reg); + + if (!on) + crtc_status[i] = val; + + /* No need to fixup if the status is off at startup. */ + if (!(crtc_status[i] & CRTC_OUTPUT_ENABLE)) + continue; + + if (on) + val |= CRTC_OUTPUT_ENABLE; + else + val &= ~CRTC_OUTPUT_ENABLE; + + mb(); + writel(val, crtc_reg); + + for (count = 0; count < 40; count++) { + val = readl(crtc_reg) & CRTC_OUTPUT_ENABLE; + if ((on && val) || (!on && !val)) + break; + udelay(1000); + } + + pci_info(pdev, "DMA hang fixup at reg[0x%lx]: 0x%x\n", + (unsigned long)crtc_reg & 0xffff, readl(crtc_reg)); + } + + iounmap(regbase); +} + +static void pci_fixup_dma_hang_early(struct pci_dev *pdev) +{ + loongson_gpu_fixup_dma_hang(pdev, false); +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_early); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_early); + +static void pci_fixup_dma_hang_final(struct pci_dev *pdev) +{ + loongson_gpu_fixup_dma_hang(pdev, true); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_final); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_final); diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index d8316f9934824..f4d076ce18436 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -21,7 +21,7 @@ cflags-vdso := $(ccflags-vdso) \ $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ -std=gnu11 -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \ -fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \ - $(call cc-option, -fno-asynchronous-unwind-tables) \ + $(call cc-option, -fasynchronous-unwind-tables) \ $(call cc-option, -fno-stack-protector) aflags-vdso := $(ccflags-vdso) \ -D__ASSEMBLY__ -Wa,-gdwarf-2 @@ -36,7 +36,7 @@ endif # VDSO linker flags. ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ - $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id -T + $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id --eh-frame-hdr -T # # Shared build commands. diff --git a/arch/loongarch/vdso/sigreturn.S b/arch/loongarch/vdso/sigreturn.S index 9cb3c58fad03b..59f940d928de7 100644 --- a/arch/loongarch/vdso/sigreturn.S +++ b/arch/loongarch/vdso/sigreturn.S @@ -12,13 +12,13 @@ #include #include +#include .section .text - .cfi_sections .debug_frame -SYM_FUNC_START(__vdso_rt_sigreturn) +SYM_SIGFUNC_START(__vdso_rt_sigreturn) li.w a7, __NR_rt_sigreturn syscall 0 -SYM_FUNC_END(__vdso_rt_sigreturn) +SYM_SIGFUNC_END(__vdso_rt_sigreturn) diff --git a/arch/m68k/kernel/vmlinux-nommu.lds b/arch/m68k/kernel/vmlinux-nommu.lds index 2624fc18c131f..45d7f4b0177b4 100644 --- a/arch/m68k/kernel/vmlinux-nommu.lds +++ b/arch/m68k/kernel/vmlinux-nommu.lds @@ -85,6 +85,7 @@ SECTIONS { _end = .; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/kernel/vmlinux-std.lds b/arch/m68k/kernel/vmlinux-std.lds index 1ccdd04ae4624..7326586afe15f 100644 --- a/arch/m68k/kernel/vmlinux-std.lds +++ b/arch/m68k/kernel/vmlinux-std.lds @@ -58,6 +58,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/kernel/vmlinux-sun3.lds b/arch/m68k/kernel/vmlinux-sun3.lds index f13ddcc2af5c2..1b19fef201fba 100644 --- a/arch/m68k/kernel/vmlinux-sun3.lds +++ b/arch/m68k/kernel/vmlinux-sun3.lds @@ -51,6 +51,7 @@ __init_begin = .; _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/lib/memmove.c b/arch/m68k/lib/memmove.c index 6519f7f349f66..e33f00b02e4c0 100644 --- a/arch/m68k/lib/memmove.c +++ b/arch/m68k/lib/memmove.c @@ -24,6 +24,15 @@ void *memmove(void *dest, const void *src, size_t n) src = csrc; n--; } +#if defined(CONFIG_M68000) + if ((long)src & 1) { + char *cdest = dest; + const char *csrc = src; + for (; n; n--) + *cdest++ = *csrc++; + return xdest; + } +#endif if (n > 2 && (long)dest & 2) { short *sdest = dest; const short *ssrc = src; @@ -66,6 +75,15 @@ void *memmove(void *dest, const void *src, size_t n) src = csrc; n--; } +#if defined(CONFIG_M68000) + if ((long)src & 1) { + char *cdest = dest; + const char *csrc = src; + for (; n; n--) + *--cdest = *--csrc; + return xdest; + } +#endif if (n > 2 && (long)dest & 2) { short *sdest = dest; const short *ssrc = src; diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index e8683f58fd3e2..83a6b68d8a39a 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1408,7 +1408,6 @@ config CPU_LOONGSON32 select CPU_MIPS32 select CPU_MIPSR2 select CPU_HAS_PREFETCH - select CPU_HAS_LOAD_STORE_LR select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_HIGHMEM select CPU_SUPPORTS_CPUFREQ diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index 404390bb87eaf..3f11e5218e6c6 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -484,7 +484,6 @@ # endif # ifndef cpu_vmbits # define cpu_vmbits cpu_data[0].vmbits -# define __NEED_VMBITS_PROBE # endif #endif diff --git a/arch/mips/include/asm/cpu-info.h b/arch/mips/include/asm/cpu-info.h index fd60837ce50bc..211b578af6aa0 100644 --- a/arch/mips/include/asm/cpu-info.h +++ b/arch/mips/include/asm/cpu-info.h @@ -80,9 +80,7 @@ struct cpuinfo_mips { int srsets; /* Shadow register sets */ int package;/* physical package number */ unsigned int globalnumber; -#ifdef CONFIG_64BIT int vmbits; /* Virtual memory size in bits */ -#endif void *data; /* Additional data */ unsigned int watch_reg_count; /* Number that exist */ unsigned int watch_reg_use_cnt; /* Usable by ptrace */ diff --git a/arch/mips/include/asm/mach-loongson2ef/loongson.h b/arch/mips/include/asm/mach-loongson2ef/loongson.h index 4a098fb102325..0e586787eb87a 100644 --- a/arch/mips/include/asm/mach-loongson2ef/loongson.h +++ b/arch/mips/include/asm/mach-loongson2ef/loongson.h @@ -324,4 +324,10 @@ extern unsigned long _loongson_addrwincfg_base; #endif /* ! CONFIG_CPU_SUPPORTS_ADDRWINCFG */ +#ifdef CONFIG_PCI +void loongson2ef_pcibios_init(void); +#else +static inline void loongson2ef_pcibios_init(void) { } +#endif + #endif /* __ASM_MACH_LOONGSON2EF_LOONGSON_H */ diff --git a/arch/mips/include/asm/mach-loongson64/topology.h b/arch/mips/include/asm/mach-loongson64/topology.h index 3414a1fd17835..89bb4deab98a6 100644 --- a/arch/mips/include/asm/mach-loongson64/topology.h +++ b/arch/mips/include/asm/mach-loongson64/topology.h @@ -7,7 +7,7 @@ #define cpu_to_node(cpu) (cpu_logical_map(cpu) >> 2) extern cpumask_t __node_cpumask[]; -#define cpumask_of_node(node) (&__node_cpumask[node]) +#define cpumask_of_node(node) ((node) == NUMA_NO_NODE ? cpu_all_mask : &__node_cpumask[node]) struct pci_bus; extern int pcibus_to_node(struct pci_bus *); diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index f799c0d723dac..12a095dbf9e2a 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -1871,6 +1871,8 @@ do { \ #define read_c0_entryhi() __read_ulong_c0_register($10, 0) #define write_c0_entryhi(val) __write_ulong_c0_register($10, 0, val) +#define read_c0_entryhi_64() __read_64bit_c0_register($10, 0) +#define write_c0_entryhi_64(val) __write_64bit_c0_register($10, 0, val) #define read_c0_guestctl1() __read_32bit_c0_register($10, 4) #define write_c0_guestctl1(val) __write_32bit_c0_register($10, 4, val) diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 1e49e05ac8b1c..489612ed9d498 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -210,11 +210,14 @@ static inline void set_elf_base_platform(const char *plat) static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) { -#ifdef __NEED_VMBITS_PROBE - write_c0_entryhi(0x3fffffffffffe000ULL); - back_to_back_c0_hazard(); - c->vmbits = fls64(read_c0_entryhi() & 0x3fffffffffffe000ULL); -#endif + int vmbits = 31; + + if (cpu_has_64bits) { + write_c0_entryhi_64(0x3fffffffffffe000ULL); + back_to_back_c0_hazard(); + vmbits = fls64(read_c0_entryhi_64() & 0x3fffffffffffe000ULL); + } + c->vmbits = vmbits; } static void set_isa(struct cpuinfo_mips *c, unsigned int isa) diff --git a/arch/mips/kernel/cpu-r3k-probe.c b/arch/mips/kernel/cpu-r3k-probe.c index 0c826f729f752..edcf04de0a6fb 100644 --- a/arch/mips/kernel/cpu-r3k-probe.c +++ b/arch/mips/kernel/cpu-r3k-probe.c @@ -137,6 +137,8 @@ void cpu_probe(void) else cpu_set_nofpu_opts(c); + c->vmbits = 31; + reserve_exception_space(0, 0x400); } diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c index f39e85fd58fa9..b15615b285690 100644 --- a/arch/mips/kernel/ftrace.c +++ b/arch/mips/kernel/ftrace.c @@ -54,10 +54,20 @@ static inline void ftrace_dyn_arch_init_insns(void) u32 *buf; unsigned int v1; - /* la v1, _mcount */ - v1 = 3; - buf = (u32 *)&insn_la_mcount[0]; - UASM_i_LA(&buf, v1, MCOUNT_ADDR); + /* If we are not in compat space, the number of generated + * instructions will exceed the maximum expected limit of 2. + * To prevent buffer overflow, we avoid generating them. + * insn_la_mcount will not be used later in ftrace_make_call. + */ + if (uasm_in_compat_space_p(MCOUNT_ADDR)) { + /* la v1, _mcount */ + v1 = 3; + buf = (u32 *)&insn_la_mcount[0]; + UASM_i_LA(&buf, v1, MCOUNT_ADDR); + } else { + pr_warn("ftrace: mcount address beyond 32 bits is not supported (%lX)\n", + MCOUNT_ADDR); + } /* jal (ftrace_caller + 8), jump over the first two instruction */ buf = (u32 *)&insn_jal_ftrace_caller; @@ -189,6 +199,13 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) unsigned int new; unsigned long ip = rec->ip; + /* When the code to patch does not belong to the kernel code + * space, we must use insn_la_mcount. However, if MCOUNT_ADDR + * is not in compat space, insn_la_mcount is not usable. + */ + if (!core_kernel_text(ip) && !uasm_in_compat_space_p(MCOUNT_ADDR)) + return -EFAULT; + new = core_kernel_text(ip) ? insn_jal_ftrace_caller : insn_la_mcount[0]; #ifdef CONFIG_64BIT diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c index 7f1c136ad8506..59833210542ff 100644 --- a/arch/mips/kernel/relocate.c +++ b/arch/mips/kernel/relocate.c @@ -420,7 +420,20 @@ void *__init relocate_kernel(void) goto out; /* The current thread is now within the relocated image */ +#ifndef CONFIG_CC_IS_CLANG __current_thread_info = RELOCATED(&init_thread_union); +#else + /* + * LLVM may wrongly restore $gp ($28) in epilog even if it's + * intentionally modified. Work around this by using inline + * assembly to assign $gp. $gp couldn't be listed as output or + * clobber, or LLVM will still restore its original value. + * See also LLVM upstream issue + * https://github.com/llvm/llvm-project/issues/176546 + */ + asm volatile("move $28, %0" : : + "r" (RELOCATED(&init_thread_union))); +#endif /* Return the new kernel's entry point */ kernel_entry = RELOCATED(start_kernel); diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S index 2b708fac8d2c1..579b2cc1995ae 100644 --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -217,6 +217,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* These must appear regardless of . */ diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index a75587018f447..e0311ca3f2e18 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -38,7 +38,7 @@ #define VECTORSPACING 0x100 /* for EI/VI mode */ #endif -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; @@ -51,7 +51,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, wait_exits), STATS_DESC_COUNTER(VCPU, cache_exits), diff --git a/arch/mips/lib/multi3.c b/arch/mips/lib/multi3.c index 4c2483f410c26..92b3778bb56fe 100644 --- a/arch/mips/lib/multi3.c +++ b/arch/mips/lib/multi3.c @@ -4,12 +4,12 @@ #include "libgcc.h" /* - * GCC 7 & older can suboptimally generate __multi3 calls for mips64r6, so for + * GCC 9 & older can suboptimally generate __multi3 calls for mips64r6, so for * that specific case only we implement that intrinsic here. * * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82981 */ -#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) && (__GNUC__ < 8) +#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) && (__GNUC__ < 10) /* multiply 64-bit values, low 64-bits returned */ static inline long long notrace dmulu(long long a, long long b) @@ -51,4 +51,4 @@ ti_type notrace __multi3(ti_type a, ti_type b) } EXPORT_SYMBOL(__multi3); -#endif /* 64BIT && CPU_MIPSR6 && GCC7 */ +#endif /* 64BIT && CPU_MIPSR6 && GCC9 */ diff --git a/arch/mips/loongson2ef/common/pci.c b/arch/mips/loongson2ef/common/pci.c index 7d9ea51e8c01e..0f11392104bfd 100644 --- a/arch/mips/loongson2ef/common/pci.c +++ b/arch/mips/loongson2ef/common/pci.c @@ -17,7 +17,7 @@ static struct resource loongson_pci_mem_resource = { static struct resource loongson_pci_io_resource = { .name = "pci io space", - .start = LOONGSON_PCI_IO_START, + .start = 0x00000000UL, /* See loongson2ef_pcibios_init(). */ .end = IO_SPACE_LIMIT, .flags = IORESOURCE_IO, }; @@ -73,15 +73,19 @@ static void __init setup_pcimap(void) #endif } -static int __init pcibios_init(void) +void __init loongson2ef_pcibios_init(void) { setup_pcimap(); + /* + * ISA-mode only IDE controllers have a hard dependency on ISA IO ports. + * + * Claim them by setting PCI IO space to start at 0x00000000, and set + * PCIBIOS_MIN_IO to prevent non-legacy PCI devices from touching + * reserved regions. + */ + PCIBIOS_MIN_IO = LOONGSON_PCI_IO_START; + loongson_pci_controller.io_map_base = mips_io_port_base; register_pci_controller(&loongson_pci_controller); - - - return 0; } - -arch_initcall(pcibios_init); diff --git a/arch/mips/loongson2ef/common/setup.c b/arch/mips/loongson2ef/common/setup.c index 4fd27f4f90edb..a639e35acce59 100644 --- a/arch/mips/loongson2ef/common/setup.c +++ b/arch/mips/loongson2ef/common/setup.c @@ -27,4 +27,5 @@ EXPORT_SYMBOL(__wbflush); void __init plat_mem_setup(void) { + loongson2ef_pcibios_init(); } diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index e3b4224c9a406..ad9b0430a28e2 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -207,7 +207,8 @@ void cpu_cache_init(void) { if (IS_ENABLED(CONFIG_CPU_R3000) && cpu_has_3k_cache) r3k_cache_init(); - if (IS_ENABLED(CONFIG_CPU_R4K_CACHE_TLB) && cpu_has_4k_cache) + if ((IS_ENABLED(CONFIG_CPU_R4K_CACHE_TLB) || + IS_ENABLED(CONFIG_CPU_SB1)) && cpu_has_4k_cache) r4k_cache_init(); if (IS_ENABLED(CONFIG_CPU_CAVIUM_OCTEON) && cpu_has_octeon_cache) diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index a673d3d68254b..8986048f9b110 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -425,6 +425,28 @@ void __init paging_init(void) static struct kcore_list kcore_kseg0; #endif +static inline void __init highmem_init(void) +{ +#ifdef CONFIG_HIGHMEM + unsigned long tmp; + + /* + * If CPU cannot support HIGHMEM discard the memory above highstart_pfn + */ + if (cpu_has_dc_aliases) { + memblock_remove(PFN_PHYS(highstart_pfn), -1); + return; + } + + for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) { + struct page *page = pfn_to_page(tmp); + + if (!memblock_is_memory(PFN_PHYS(tmp))) + SetPageReserved(page); + } +#endif +} + void __init arch_mm_preinit(void) { /* @@ -435,6 +457,7 @@ void __init arch_mm_preinit(void) maar_init(); setup_zero_pages(); /* Setup zeroed pages. */ + highmem_init(); #ifdef CONFIG_64BIT if ((unsigned long) &_text > (unsigned long) CKSEG0) diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 44a662536148e..24fe85fa169d1 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -511,87 +513,259 @@ static int __init set_ntlb(char *str) __setup("ntlb=", set_ntlb); -/* Comparison function for EntryHi VPN fields. */ -static int r4k_vpn_cmp(const void *a, const void *b) +/* The start bit position of VPN2 and Mask in EntryHi/PageMask registers. */ +#define VPN2_SHIFT 13 + +/* Read full EntryHi even with CONFIG_32BIT. */ +static inline unsigned long long read_c0_entryhi_native(void) +{ + return cpu_has_64bits ? read_c0_entryhi_64() : read_c0_entryhi(); +} + +/* Write full EntryHi even with CONFIG_32BIT. */ +static inline void write_c0_entryhi_native(unsigned long long v) { - long v = *(unsigned long *)a - *(unsigned long *)b; - int s = sizeof(long) > sizeof(int) ? sizeof(long) * 8 - 1: 0; - return s ? (v != 0) | v >> s : v; + if (cpu_has_64bits) + write_c0_entryhi_64(v); + else + write_c0_entryhi(v); } +/* TLB entry state for uniquification. */ +struct tlbent { + unsigned long long wired:1; + unsigned long long global:1; + unsigned long long asid:10; + unsigned long long vpn:51; + unsigned long long pagesz:5; + unsigned long long index:14; +}; + /* - * Initialise all TLB entries with unique values that do not clash with - * what we have been handed over and what we'll be using ourselves. + * Comparison function for TLB entry sorting. Place wired entries first, + * then global entries, then order by the increasing VPN/ASID and the + * decreasing page size. This lets us avoid clashes with wired entries + * easily and get entries for larger pages out of the way first. + * + * We could group bits so as to reduce the number of comparisons, but this + * is seldom executed and not performance-critical, so prefer legibility. */ -static void __ref r4k_tlb_uniquify(void) +static int r4k_entry_cmp(const void *a, const void *b) { - int tlbsize = current_cpu_data.tlbsize; - bool use_slab = slab_is_available(); - int start = num_wired_entries(); - phys_addr_t tlb_vpn_size; - unsigned long *tlb_vpns; - unsigned long vpn_mask; - int cnt, ent, idx, i; - - vpn_mask = GENMASK(cpu_vmbits - 1, 13); - vpn_mask |= IS_ENABLED(CONFIG_64BIT) ? 3ULL << 62 : 1 << 31; + struct tlbent ea = *(struct tlbent *)a, eb = *(struct tlbent *)b; + + if (ea.wired > eb.wired) + return -1; + else if (ea.wired < eb.wired) + return 1; + else if (ea.global > eb.global) + return -1; + else if (ea.global < eb.global) + return 1; + else if (ea.vpn < eb.vpn) + return -1; + else if (ea.vpn > eb.vpn) + return 1; + else if (ea.asid < eb.asid) + return -1; + else if (ea.asid > eb.asid) + return 1; + else if (ea.pagesz > eb.pagesz) + return -1; + else if (ea.pagesz < eb.pagesz) + return 1; + else + return 0; +} - tlb_vpn_size = tlbsize * sizeof(*tlb_vpns); - tlb_vpns = (use_slab ? - kmalloc(tlb_vpn_size, GFP_KERNEL) : - memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns))); - if (WARN_ON(!tlb_vpns)) - return; /* Pray local_flush_tlb_all() is good enough. */ +/* + * Fetch all the TLB entries. Mask individual VPN values retrieved with + * the corresponding page mask and ignoring any 1KiB extension as we'll + * be using 4KiB pages for uniquification. + */ +static void __ref r4k_tlb_uniquify_read(struct tlbent *tlb_vpns, int tlbsize) +{ + int start = num_wired_entries(); + unsigned long long vpn_mask; + bool global; + int i; - htw_stop(); + vpn_mask = GENMASK(current_cpu_data.vmbits - 1, VPN2_SHIFT); + vpn_mask |= cpu_has_64bits ? 3ULL << 62 : 1 << 31; - for (i = start, cnt = 0; i < tlbsize; i++, cnt++) { - unsigned long vpn; + for (i = 0; i < tlbsize; i++) { + unsigned long long entryhi, vpn, mask, asid; + unsigned int pagesz; write_c0_index(i); mtc0_tlbr_hazard(); tlb_read(); tlb_read_hazard(); - vpn = read_c0_entryhi(); - vpn &= vpn_mask & PAGE_MASK; - tlb_vpns[cnt] = vpn; - /* Prevent any large pages from overlapping regular ones. */ - write_c0_pagemask(read_c0_pagemask() & PM_DEFAULT_MASK); - mtc0_tlbw_hazard(); - tlb_write_indexed(); - tlbw_use_hazard(); + global = !!(read_c0_entrylo0() & ENTRYLO_G); + entryhi = read_c0_entryhi_native(); + mask = read_c0_pagemask(); + + asid = entryhi & cpu_asid_mask(¤t_cpu_data); + vpn = (entryhi & vpn_mask & ~mask) >> VPN2_SHIFT; + pagesz = ilog2((mask >> VPN2_SHIFT) + 1); + + tlb_vpns[i].global = global; + tlb_vpns[i].asid = global ? 0 : asid; + tlb_vpns[i].vpn = vpn; + tlb_vpns[i].pagesz = pagesz; + tlb_vpns[i].wired = i < start; + tlb_vpns[i].index = i; } +} + +/* + * Write unique values to all but the wired TLB entries each, using + * the 4KiB page size. This size might not be supported with R6, but + * EHINV is mandatory for R6, so we won't ever be called in that case. + * + * A sorted table is supplied with any wired entries at the beginning, + * followed by any global entries, and then finally regular entries. + * We start at the VPN and ASID values of zero and only assign user + * addresses, therefore guaranteeing no clash with addresses produced + * by UNIQUE_ENTRYHI. We avoid any VPN values used by wired or global + * entries, by increasing the VPN value beyond the span of such entry. + * + * When a VPN/ASID clash is found with a regular entry we increment the + * ASID instead until no VPN/ASID clash has been found or the ASID space + * has been exhausted, in which case we increase the VPN value beyond + * the span of the largest clashing entry. + * + * We do not need to be concerned about FTLB or MMID configurations as + * those are required to implement the EHINV feature. + */ +static void __ref r4k_tlb_uniquify_write(struct tlbent *tlb_vpns, int tlbsize) +{ + unsigned long long asid, vpn, vpn_size, pagesz; + int widx, gidx, idx, sidx, lidx, i; - sort(tlb_vpns, cnt, sizeof(tlb_vpns[0]), r4k_vpn_cmp, NULL); + vpn_size = 1ULL << (current_cpu_data.vmbits - VPN2_SHIFT); + pagesz = ilog2((PM_4K >> VPN2_SHIFT) + 1); - write_c0_pagemask(PM_DEFAULT_MASK); + write_c0_pagemask(PM_4K); write_c0_entrylo0(0); write_c0_entrylo1(0); - idx = 0; - ent = tlbsize; - for (i = start; i < tlbsize; i++) - while (1) { - unsigned long entryhi, vpn; + asid = 0; + vpn = 0; + widx = 0; + gidx = 0; + for (sidx = 0; sidx < tlbsize && tlb_vpns[sidx].wired; sidx++) + ; + for (lidx = sidx; lidx < tlbsize && tlb_vpns[lidx].global; lidx++) + ; + idx = gidx = sidx + 1; + for (i = sidx; i < tlbsize; i++) { + unsigned long long entryhi, vpn_pagesz = 0; - entryhi = UNIQUE_ENTRYHI(ent); - vpn = entryhi & vpn_mask & PAGE_MASK; + while (1) { + if (WARN_ON(vpn >= vpn_size)) { + dump_tlb_all(); + /* Pray local_flush_tlb_all() will cope. */ + return; + } - if (idx >= cnt || vpn < tlb_vpns[idx]) { - write_c0_entryhi(entryhi); - write_c0_index(i); - mtc0_tlbw_hazard(); - tlb_write_indexed(); - ent++; - break; - } else if (vpn == tlb_vpns[idx]) { - ent++; - } else { + /* VPN must be below the next wired entry. */ + if (widx < sidx && vpn >= tlb_vpns[widx].vpn) { + vpn = max(vpn, + (tlb_vpns[widx].vpn + + (1ULL << tlb_vpns[widx].pagesz))); + asid = 0; + widx++; + continue; + } + /* VPN must be below the next global entry. */ + if (gidx < lidx && vpn >= tlb_vpns[gidx].vpn) { + vpn = max(vpn, + (tlb_vpns[gidx].vpn + + (1ULL << tlb_vpns[gidx].pagesz))); + asid = 0; + gidx++; + continue; + } + /* Try to find a free ASID so as to conserve VPNs. */ + if (idx < tlbsize && vpn == tlb_vpns[idx].vpn && + asid == tlb_vpns[idx].asid) { + unsigned long long idx_pagesz; + + idx_pagesz = tlb_vpns[idx].pagesz; + vpn_pagesz = max(vpn_pagesz, idx_pagesz); + do + idx++; + while (idx < tlbsize && + vpn == tlb_vpns[idx].vpn && + asid == tlb_vpns[idx].asid); + asid++; + if (asid > cpu_asid_mask(¤t_cpu_data)) { + vpn += vpn_pagesz; + asid = 0; + vpn_pagesz = 0; + } + continue; + } + /* VPN mustn't be above the next regular entry. */ + if (idx < tlbsize && vpn > tlb_vpns[idx].vpn) { + vpn = max(vpn, + (tlb_vpns[idx].vpn + + (1ULL << tlb_vpns[idx].pagesz))); + asid = 0; idx++; + continue; } + break; } + entryhi = (vpn << VPN2_SHIFT) | asid; + write_c0_entryhi_native(entryhi); + write_c0_index(tlb_vpns[i].index); + mtc0_tlbw_hazard(); + tlb_write_indexed(); + + tlb_vpns[i].asid = asid; + tlb_vpns[i].vpn = vpn; + tlb_vpns[i].pagesz = pagesz; + + asid++; + if (asid > cpu_asid_mask(¤t_cpu_data)) { + vpn += 1ULL << pagesz; + asid = 0; + } + } +} + +/* + * Initialise all TLB entries with unique values that do not clash with + * what we have been handed over and what we'll be using ourselves. + */ +static void __ref r4k_tlb_uniquify(void) +{ + int tlbsize = current_cpu_data.tlbsize; + bool use_slab = slab_is_available(); + phys_addr_t tlb_vpn_size; + struct tlbent *tlb_vpns; + + tlb_vpn_size = tlbsize * sizeof(*tlb_vpns); + tlb_vpns = (use_slab ? + kmalloc(tlb_vpn_size, GFP_ATOMIC) : + memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns))); + if (WARN_ON(!tlb_vpns)) + return; /* Pray local_flush_tlb_all() is good enough. */ + + htw_stop(); + + r4k_tlb_uniquify_read(tlb_vpns, tlbsize); + + sort(tlb_vpns, tlbsize, sizeof(*tlb_vpns), r4k_entry_cmp, NULL); + + r4k_tlb_uniquify_write(tlb_vpns, tlbsize); + + write_c0_pagemask(PM_DEFAULT_MASK); + tlbw_use_hazard(); htw_start(); flush_micro_tlb(); @@ -640,7 +814,8 @@ static void r4k_tlb_configure(void) temp_tlb_entry = current_cpu_data.tlbsize - 1; /* From this point on the ARC firmware is dead. */ - r4k_tlb_uniquify(); + if (!cpu_has_tlbinv) + r4k_tlb_uniquify(); local_flush_tlb_all(); /* Did I tell you that ARC SUCKS? */ diff --git a/arch/mips/ralink/clk.c b/arch/mips/ralink/clk.c index 9db73fcac522e..5c1eb46ef5d07 100644 --- a/arch/mips/ralink/clk.c +++ b/arch/mips/ralink/clk.c @@ -21,16 +21,16 @@ static const char *clk_cpu(int *idx) { switch (ralink_soc) { case RT2880_SOC: - *idx = 0; + *idx = 1; return "ralink,rt2880-sysc"; case RT3883_SOC: - *idx = 0; + *idx = 1; return "ralink,rt3883-sysc"; case RT305X_SOC_RT3050: - *idx = 0; + *idx = 1; return "ralink,rt3050-sysc"; case RT305X_SOC_RT3052: - *idx = 0; + *idx = 1; return "ralink,rt3052-sysc"; case RT305X_SOC_RT3350: *idx = 1; diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c index b7f6f782d9a13..ffa4d38ca95df 100644 --- a/arch/mips/rb532/devices.c +++ b/arch/mips/rb532/devices.c @@ -212,11 +212,12 @@ static struct platform_device rb532_wdt = { static struct plat_serial8250_port rb532_uart_res[] = { { .type = PORT_16550A, - .membase = (char *)KSEG1ADDR(REGBASE + UART0BASE), + .mapbase = REGBASE + UART0BASE, + .mapsize = 0x1000, .irq = UART0_IRQ, .regshift = 2, .iotype = UPIO_MEM, - .flags = UPF_BOOT_AUTOCONF, + .flags = UPF_BOOT_AUTOCONF | UPF_IOREMAP, }, { .flags = 0, diff --git a/arch/mips/sgi-ip22/ip22-gio.c b/arch/mips/sgi-ip22/ip22-gio.c index 5893ea4e382ca..19b70928d6dc3 100644 --- a/arch/mips/sgi-ip22/ip22-gio.c +++ b/arch/mips/sgi-ip22/ip22-gio.c @@ -372,7 +372,8 @@ static void ip22_check_gio(int slotno, unsigned long addr, int irq) gio_dev->resource.flags = IORESOURCE_MEM; gio_dev->irq = irq; dev_set_name(&gio_dev->dev, "%d", slotno); - gio_device_register(gio_dev); + if (gio_device_register(gio_dev)) + gio_dev_put(gio_dev); } else printk(KERN_INFO "GIO: slot %d : Empty\n", slotno); } diff --git a/arch/nios2/kernel/vmlinux.lds.S b/arch/nios2/kernel/vmlinux.lds.S index 37b9580550646..206f92445bfad 100644 --- a/arch/nios2/kernel/vmlinux.lds.S +++ b/arch/nios2/kernel/vmlinux.lds.S @@ -57,6 +57,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/openrisc/include/asm/barrier.h b/arch/openrisc/include/asm/barrier.h index 7538294721bed..8e592c9909023 100644 --- a/arch/openrisc/include/asm/barrier.h +++ b/arch/openrisc/include/asm/barrier.h @@ -4,6 +4,8 @@ #define mb() asm volatile ("l.msync" ::: "memory") +#define nop() asm volatile ("l.nop") + #include #endif /* __ASM_BARRIER_H */ diff --git a/arch/openrisc/kernel/vmlinux.lds.S b/arch/openrisc/kernel/vmlinux.lds.S index 049bff45f6126..9b29c3211774c 100644 --- a/arch/openrisc/kernel/vmlinux.lds.S +++ b/arch/openrisc/kernel/vmlinux.lds.S @@ -101,6 +101,7 @@ SECTIONS /* Throw in the debugging sections */ STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded -- must be last */ diff --git a/arch/parisc/boot/compressed/vmlinux.lds.S b/arch/parisc/boot/compressed/vmlinux.lds.S index ab7b439908578..87d24cc824b66 100644 --- a/arch/parisc/boot/compressed/vmlinux.lds.S +++ b/arch/parisc/boot/compressed/vmlinux.lds.S @@ -90,6 +90,7 @@ SECTIONS /* Sections to be discarded */ DISCARDS /DISCARD/ : { + *(.modinfo) #ifdef CONFIG_64BIT /* temporary hack until binutils is fixed to not emit these * for static binaries diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 2c139a4dbf4b8..17afe7a59edfd 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -85,7 +85,7 @@ extern void __update_cache(pte_t pte); printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, (unsigned long)pgd_val(e)) /* This is the size of the initially mapped kernel memory */ -#if defined(CONFIG_64BIT) +#if defined(CONFIG_64BIT) || defined(CONFIG_KALLSYMS) #define KERNEL_INITIAL_ORDER 26 /* 1<<26 = 64MB */ #else #define KERNEL_INITIAL_ORDER 25 /* 1<<25 = 32MB */ diff --git a/arch/parisc/kernel/asm-offsets.c b/arch/parisc/kernel/asm-offsets.c index 9abfe65492c65..3de4b5933b107 100644 --- a/arch/parisc/kernel/asm-offsets.c +++ b/arch/parisc/kernel/asm-offsets.c @@ -258,6 +258,8 @@ int main(void) BLANK(); DEFINE(TIF_BLOCKSTEP_PA_BIT, 31-TIF_BLOCKSTEP); DEFINE(TIF_SINGLESTEP_PA_BIT, 31-TIF_SINGLESTEP); + DEFINE(TIF_32BIT_PA_BIT, 31-TIF_32BIT); + BLANK(); DEFINE(ASM_PMD_SHIFT, PMD_SHIFT); DEFINE(ASM_PGDIR_SHIFT, PGDIR_SHIFT); diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 4c5240d3a3c7c..b189265785dc7 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -953,7 +953,7 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, #else "1: cmpb,<<,n %0,%2,1b\n" #endif - " fic,m %3(%4,%0)\n" + " fdc,m %3(%4,%0)\n" "2: sync\n" ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1") : "+r" (start), "+r" (error) @@ -968,7 +968,7 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, #else "1: cmpb,<<,n %0,%2,1b\n" #endif - " fdc,m %3(%4,%0)\n" + " fic,m %3(%4,%0)\n" "2: sync\n" ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1") : "+r" (start), "+r" (error) diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c index 1f8936fc22928..427e1334f8352 100644 --- a/arch/parisc/kernel/drivers.c +++ b/arch/parisc/kernel/drivers.c @@ -435,7 +435,7 @@ static struct parisc_device * __init create_tree_node(char id, dev->dev.dma_mask = &dev->dma_mask; dev->dev.coherent_dma_mask = dev->dma_mask; if (device_register(&dev->dev)) { - kfree(dev); + put_device(&dev->dev); return NULL; } diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index f4bf61a34701e..e04c5d806c105 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -1059,8 +1059,6 @@ ENTRY_CFI(intr_save) /* for os_hpmc */ STREG %r17, PT_IOR(%r29) #if defined(CONFIG_64BIT) - b,n intr_save2 - skip_save_ior: /* We have a itlb miss, and when executing code above 4 Gb on ILP64, we * need to adjust iasq/iaoq here in the same way we adjusted isr/ior @@ -1069,10 +1067,17 @@ skip_save_ior: bb,COND(>=),n %r8,PSW_W_BIT,intr_save2 LDREG PT_IASQ0(%r29), %r16 LDREG PT_IAOQ0(%r29), %r17 - /* adjust iasq/iaoq */ + /* adjust iasq0/iaoq0 */ space_adjust %r16,%r17,%r1 STREG %r16, PT_IASQ0(%r29) STREG %r17, PT_IAOQ0(%r29) + + LDREG PT_IASQ1(%r29), %r16 + LDREG PT_IAOQ1(%r29), %r17 + /* adjust iasq1/iaoq1 */ + space_adjust %r16,%r17,%r1 + STREG %r16, PT_IASQ1(%r29) + STREG %r17, PT_IAOQ1(%r29) #else skip_save_ior: #endif @@ -1841,6 +1846,10 @@ syscall_restore_rfi: extru,= %r19,TIF_BLOCKSTEP_PA_BIT,1,%r0 depi -1,7,1,%r20 /* T bit */ +#ifdef CONFIG_64BIT + extru,<> %r19,TIF_32BIT_PA_BIT,1,%r0 + depi -1,4,1,%r20 /* W bit */ +#endif STREG %r20,TASK_PT_PSW(%r1) /* Always store space registers, since sr3 can be changed (e.g. fork) */ @@ -1854,7 +1863,6 @@ syscall_restore_rfi: STREG %r25,TASK_PT_IASQ0(%r1) STREG %r25,TASK_PT_IASQ1(%r1) - /* XXX W bit??? */ /* Now if old D bit is clear, it means we didn't save all registers * on syscall entry, so do that now. This only happens on TRACEME * calls, or if someone attached to us while we were on a syscall. diff --git a/arch/parisc/kernel/head.S b/arch/parisc/kernel/head.S index 96e0264ac9616..9188c8d874370 100644 --- a/arch/parisc/kernel/head.S +++ b/arch/parisc/kernel/head.S @@ -56,6 +56,7 @@ ENTRY(parisc_kernel_start) .import __bss_start,data .import __bss_stop,data + .import __end,data load32 PA(__bss_start),%r3 load32 PA(__bss_stop),%r4 @@ -149,7 +150,11 @@ $cpu_ok: * everything ... it will get remapped correctly later */ ldo 0+_PAGE_KERNEL_RWX(%r0),%r3 /* Hardwired 0 phys addr start */ load32 (1<<(KERNEL_INITIAL_ORDER-PAGE_SHIFT)),%r11 /* PFN count */ - load32 PA(pg0),%r1 + load32 PA(_end),%r1 + SHRREG %r1,PAGE_SHIFT,%r1 /* %r1 is PFN count for _end symbol */ + cmpb,<<,n %r11,%r1,1f + copy %r1,%r11 /* %r1 PFN count smaller than %r11 */ +1: load32 PA(pg0),%r1 $pgt_fill_loop: STREGM %r3,ASM_PTE_ENTRY_SIZE(%r1) diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index e64ab5d2a40d6..703644e5bfc4a 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -85,6 +85,9 @@ void machine_restart(char *cmd) #endif /* set up a new led state on systems shipped with a LED State panel */ pdc_chassis_send_status(PDC_CHASSIS_DIRECT_SHUTDOWN); + + /* prevent interrupts during reboot */ + set_eiem(0); /* "Normal" system reset */ pdc_do_reset(); diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index ace483b6f19ad..d3e17a7a89016 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -120,14 +120,6 @@ void __init setup_arch(char **cmdline_p) #endif printk(KERN_CONT ".\n"); - /* - * Check if initial kernel page mappings are sufficient. - * panic early if not, else we may access kernel functions - * and variables which can't be reached. - */ - if (__pa((unsigned long) &_end) >= KERNEL_INITIAL_SIZE) - panic("KERNEL_INITIAL_ORDER too small!"); - #ifdef CONFIG_64BIT if(parisc_narrow_firmware) { printk(KERN_INFO "Kernel is using PDC in 32-bit mode.\n"); @@ -279,6 +271,18 @@ void __init start_parisc(void) int ret, cpunum; struct pdc_coproc_cfg coproc_cfg; + /* + * Check if initial kernel page mapping is sufficient. + * Print warning if not, because we may access kernel functions and + * variables which can't be reached yet through the initial mappings. + * Note that the panic() and printk() functions are not functional + * yet, so we need to use direct iodc() firmware calls instead. + */ + const char warn1[] = "CRITICAL: Kernel may crash because " + "KERNEL_INITIAL_ORDER is too small.\n"; + if (__pa((unsigned long) &_end) >= KERNEL_INITIAL_SIZE) + pdc_iodc_print(warn1, sizeof(warn1) - 1); + /* check QEMU/SeaBIOS marker in PAGE0 */ running_on_qemu = (memcmp(&PAGE0->pad0, "SeaBIOS", 8) == 0); diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S index b445e47903cfd..0ca93d6d72354 100644 --- a/arch/parisc/kernel/vmlinux.lds.S +++ b/arch/parisc/kernel/vmlinux.lds.S @@ -165,6 +165,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS .note 0 : { *(.note) } diff --git a/arch/powerpc/boot/addnote.c b/arch/powerpc/boot/addnote.c index 53b3b2621457d..78704927453aa 100644 --- a/arch/powerpc/boot/addnote.c +++ b/arch/powerpc/boot/addnote.c @@ -68,8 +68,8 @@ static int e_class = ELFCLASS32; #define PUT_16BE(off, v)(buf[off] = ((v) >> 8) & 0xff, \ buf[(off) + 1] = (v) & 0xff) #define PUT_32BE(off, v)(PUT_16BE((off), (v) >> 16L), PUT_16BE((off) + 2, (v))) -#define PUT_64BE(off, v)((PUT_32BE((off), (v) >> 32L), \ - PUT_32BE((off) + 4, (v)))) +#define PUT_64BE(off, v)((PUT_32BE((off), (unsigned long long)(v) >> 32L), \ + PUT_32BE((off) + 4, (unsigned long long)(v)))) #define GET_16LE(off) ((buf[off]) + (buf[(off)+1] << 8)) #define GET_32LE(off) (GET_16LE(off) + (GET_16LE((off)+2U) << 16U)) @@ -78,7 +78,8 @@ static int e_class = ELFCLASS32; #define PUT_16LE(off, v) (buf[off] = (v) & 0xff, \ buf[(off) + 1] = ((v) >> 8) & 0xff) #define PUT_32LE(off, v) (PUT_16LE((off), (v)), PUT_16LE((off) + 2, (v) >> 16L)) -#define PUT_64LE(off, v) (PUT_32LE((off), (v)), PUT_32LE((off) + 4, (v) >> 32L)) +#define PUT_64LE(off, v) (PUT_32LE((off), (unsigned long long)(v)), \ + PUT_32LE((off) + 4, (unsigned long long)(v) >> 32L)) #define GET_16(off) (e_data == ELFDATA2MSB ? GET_16BE(off) : GET_16LE(off)) #define GET_32(off) (e_data == ELFDATA2MSB ? GET_32BE(off) : GET_32LE(off)) diff --git a/arch/powerpc/include/asm/book3s/32/tlbflush.h b/arch/powerpc/include/asm/book3s/32/tlbflush.h index e43534da5207a..4be2200a3c7e1 100644 --- a/arch/powerpc/include/asm/book3s/32/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/32/tlbflush.h @@ -11,6 +11,7 @@ void hash__flush_tlb_mm(struct mm_struct *mm); void hash__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); void hash__flush_range(struct mm_struct *mm, unsigned long start, unsigned long end); +void hash__flush_gather(struct mmu_gather *tlb); #ifdef CONFIG_SMP void _tlbie(unsigned long address); @@ -29,7 +30,9 @@ void _tlbia(void); static inline void tlb_flush(struct mmu_gather *tlb) { /* 603 needs to flush the whole TLB here since it doesn't use a hash table. */ - if (!mmu_has_feature(MMU_FTR_HPTE_TABLE)) + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) + hash__flush_gather(tlb); + else _tlbia(); } diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index 3463514232071..af12e2ba8eb85 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -524,7 +524,6 @@ void slb_save_contents(struct slb_entry *slb_ptr); void slb_dump_contents(struct slb_entry *slb_ptr); extern void slb_vmalloc_update(void); -void preload_new_slb_context(unsigned long start, unsigned long sp); #ifdef CONFIG_PPC_64S_HASH_MMU void slb_set_size(u16 size); diff --git a/arch/powerpc/include/asm/crash_reserve.h b/arch/powerpc/include/asm/crash_reserve.h index 6467ce29b1fab..d1b570ddbf98b 100644 --- a/arch/powerpc/include/asm/crash_reserve.h +++ b/arch/powerpc/include/asm/crash_reserve.h @@ -5,4 +5,12 @@ /* crash kernel regions are Page size agliged */ #define CRASH_ALIGN PAGE_SIZE +#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +static inline bool arch_add_crash_res_to_iomem(void) +{ + return false; +} +#define arch_add_crash_res_to_iomem arch_add_crash_res_to_iomem +#endif + #endif /* _ASM_POWERPC_CRASH_RESERVE_H */ diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 5e34611de9ef4..b7ebb4ac2c710 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -289,6 +289,8 @@ void eeh_pe_dev_traverse(struct eeh_pe *root, void eeh_pe_restore_bars(struct eeh_pe *pe); const char *eeh_pe_loc_get(struct eeh_pe *pe); struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe); +const char *eeh_pe_loc_get_bus(struct pci_bus *bus); +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe); void eeh_show_enabled(void); int __init eeh_init(struct eeh_ops *ops); diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index dab63b82a8d4f..f2009d7c8cfa7 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -134,7 +134,6 @@ static __always_inline void kuap_assert_locked(void) static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(NULL, from, size, KUAP_READ); } @@ -146,7 +145,6 @@ static __always_inline void allow_write_to_user(void __user *to, unsigned long s static __always_inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(to, from, size, KUAP_READ_WRITE); } diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 4f5a46a77fa2b..5ef59b1f6d6d5 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -253,7 +253,7 @@ __gus_failed: \ ".section .fixup,\"ax\"\n" \ "4: li %0,%3\n" \ " li %1,0\n" \ - " li %1+1,0\n" \ + " li %L1,0\n" \ " b 3b\n" \ ".previous\n" \ EX_TABLE(1b, 4b) \ @@ -301,6 +301,7 @@ do { \ __typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr)); \ \ might_fault(); \ + barrier_nospec(); \ allow_read_from_user(__gu_addr, __gu_size); \ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ prevent_read_from_user(__gu_addr, __gu_size); \ @@ -329,6 +330,7 @@ raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) { unsigned long ret; + barrier_nospec(); allow_read_write_user(to, from, n); ret = __copy_tofrom_user(to, from, n); prevent_read_write_user(to, from, n); @@ -405,8 +407,7 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n) } #endif -extern long __copy_from_user_flushcache(void *dst, const void __user *src, - unsigned size); +extern size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size); static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len) { @@ -415,6 +416,7 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt might_fault(); + barrier_nospec(); allow_read_write_user((void __user *)ptr, ptr, len); return true; } @@ -431,6 +433,7 @@ user_read_access_begin(const void __user *ptr, size_t len) might_fault(); + barrier_nospec(); allow_read_from_user(ptr, len); return true; } diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c index 7f63f1cdc6c39..ca00c4824e313 100644 --- a/arch/powerpc/kernel/btext.c +++ b/arch/powerpc/kernel/btext.c @@ -20,6 +20,7 @@ #include #include #include +#include #define NO_SCROLL @@ -463,7 +464,7 @@ static noinline void draw_byte(unsigned char c, long locX, long locY) { unsigned char *base = calc_base(locX << 3, locY << 4); unsigned int font_index = c * 16; - const unsigned char *font = font_sun_8x16.data + font_index; + const unsigned char *font = PTRRELOC(font_sun_8x16.data) + font_index; int rb = dispDeviceRowBytes; rmci_maybe_on(); diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index ef78ff77cf8f2..028f691585323 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -846,7 +846,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) pci_lock_rescan_remove(); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (!bus) { pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n", __func__, pe->phb->global_number, pe->addr); @@ -886,14 +886,15 @@ void eeh_handle_normal_event(struct eeh_pe *pe) /* Log the event */ if (pe->type & EEH_PE_PHB) { pr_err("EEH: Recovering PHB#%x, location: %s\n", - pe->phb->global_number, eeh_pe_loc_get(pe)); + pe->phb->global_number, eeh_pe_loc_get_bus(bus)); } else { struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb); pr_err("EEH: Recovering PHB#%x-PE#%x\n", pe->phb->global_number, pe->addr); pr_err("EEH: PE location: %s, PHB location: %s\n", - eeh_pe_loc_get(pe), eeh_pe_loc_get(phb_pe)); + eeh_pe_loc_get_bus(bus), + eeh_pe_loc_get_bus(eeh_pe_bus_get_nolock(phb_pe))); } #ifdef CONFIG_STACKTRACE @@ -1098,7 +1099,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (bus) pci_hp_remove_devices(bus); else @@ -1222,7 +1223,7 @@ void eeh_handle_special_event(void) (phb_pe->state & EEH_PE_RECOVERING)) continue; - bus = eeh_pe_bus_get(phb_pe); + bus = eeh_pe_bus_get_nolock(phb_pe); if (!bus) { pr_err("%s: Cannot find PCI bus for " "PHB#%x-PE#%x\n", diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index e740101fadf3b..040e8f69a4aa8 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -812,6 +812,24 @@ void eeh_pe_restore_bars(struct eeh_pe *pe) const char *eeh_pe_loc_get(struct eeh_pe *pe) { struct pci_bus *bus = eeh_pe_bus_get(pe); + return eeh_pe_loc_get_bus(bus); +} + +/** + * eeh_pe_loc_get_bus - Retrieve location code binding to the given PCI bus + * @bus: PCI bus + * + * Retrieve the location code associated with the given PCI bus. If the bus + * is a root bus, the location code is fetched from the PHB device tree node + * or root port. Otherwise, the location code is obtained from the device + * tree node of the upstream bridge of the bus. The function walks up the + * bus hierarchy if necessary, checking each node for the appropriate + * location code property ("ibm,io-base-loc-code" for root buses, + * "ibm,slot-location-code" for others). If no location code is found, + * returns "N/A". + */ +const char *eeh_pe_loc_get_bus(struct pci_bus *bus) +{ struct device_node *dn; const char *loc = NULL; @@ -838,8 +856,9 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) } /** - * eeh_pe_bus_get - Retrieve PCI bus according to the given PE + * _eeh_pe_bus_get - Retrieve PCI bus according to the given PE * @pe: EEH PE + * @do_lock: Is the caller already held the pci_lock_rescan_remove? * * Retrieve the PCI bus according to the given PE. Basically, * there're 3 types of PEs: PHB/Bus/Device. For PHB PE, the @@ -847,7 +866,7 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) * returned for BUS PE. However, we don't have associated PCI * bus for DEVICE PE. */ -struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +static struct pci_bus *_eeh_pe_bus_get(struct eeh_pe *pe, bool do_lock) { struct eeh_dev *edev; struct pci_dev *pdev; @@ -862,11 +881,58 @@ struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) /* Retrieve the parent PCI bus of first (top) PCI device */ edev = list_first_entry_or_null(&pe->edevs, struct eeh_dev, entry); - pci_lock_rescan_remove(); + if (do_lock) + pci_lock_rescan_remove(); pdev = eeh_dev_to_pci_dev(edev); if (pdev) bus = pdev->bus; - pci_unlock_rescan_remove(); + if (do_lock) + pci_unlock_rescan_remove(); return bus; } + +/** + * eeh_pe_bus_get - Retrieve PCI bus associated with the given EEH PE, locking + * if needed + * @pe: Pointer to the EEH PE + * + * This function is a wrapper around _eeh_pe_bus_get(), which retrieves the PCI + * bus associated with the provided EEH PE structure. It acquires the PCI + * rescans lock to ensure safe access to shared data during the retrieval + * process. This function should be used when the caller requires the PCI bus + * while holding the rescan/remove lock, typically during operations that modify + * or inspect PCIe device state in a safe manner. + * + * RETURNS: + * A pointer to the PCI bus associated with the EEH PE, or NULL if none found. + */ + +struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, true); +} + +/** + * eeh_pe_bus_get_nolock - Retrieve PCI bus associated with the given EEH PE + * without locking + * @pe: Pointer to the EEH PE + * + * This function is a variant of _eeh_pe_bus_get() that retrieves the PCI bus + * associated with the specified EEH PE without acquiring the + * pci_lock_rescan_remove lock. It should only be used when the caller can + * guarantee safe access to PE structures without the need for that lock, + * typically in contexts where the lock is already held locking is otherwise + * managed. + * + * RETURNS: + * pointer to the PCI bus associated with the EEH PE, or NULL if none is found. + * + * NOTE: + * Use this function carefully to avoid race conditions and data corruption. + */ + +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, false); +} diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index f4a8c98772491..1beb578c64114 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -263,10 +263,9 @@ interrupt_return: mtspr SPRN_SRR1,r12 BEGIN_FTR_SECTION + lwarx r0,0,r1 +END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) stwcx. r0,0,r1 /* to clear the reservation */ -FTR_SECTION_ELSE - lwarx r0,0,r1 -ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) lwz r3,_CCR(r1) lwz r4,_LINK(r1) @@ -306,10 +305,9 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) mtspr SPRN_SRR1,r12 BEGIN_FTR_SECTION + lwarx r0,0,r1 +END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) stwcx. r0,0,r1 /* to clear the reservation */ -FTR_SECTION_ELSE - lwarx r0,0,r1 -ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) lwz r3,_LINK(r1) lwz r4,_CTR(r1) diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index eb23966ac0a9f..a45fe147868bc 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1897,8 +1897,6 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) return 0; } -void preload_new_slb_context(unsigned long start, unsigned long sp); - /* * Set up a thread for executing a new program */ @@ -1906,9 +1904,6 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) { #ifdef CONFIG_PPC64 unsigned long load_addr = regs->gpr[2]; /* saved by ELF_PLAT_INIT */ - - if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !radix_enabled()) - preload_new_slb_context(start, sp); #endif #ifdef CONFIG_PPC_TRANSACTIONAL_MEM diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 68edb66c2964b..0cd9c0c21af3e 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -822,6 +822,8 @@ static int parse_thread_groups(struct device_node *dn, count = of_property_count_u32_elems(dn, "ibm,thread-groups"); thread_group_array = kcalloc(count, sizeof(u32), GFP_KERNEL); + if (!thread_group_array) + return -ENOMEM; ret = of_property_read_u32_array(dn, "ibm,thread-groups", thread_group_array, count); if (ret) diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index de6ee7d35cff0..5589af3e40844 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -398,6 +398,7 @@ SECTIONS _end = . ; DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index d1a2d755381ca..f86a6fc11e91c 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -22,6 +22,9 @@ #include #include +#define cpu_to_be_ulong __PASTE(cpu_to_be, BITS_PER_LONG) +#define __be_word __PASTE(__be, BITS_PER_LONG) + #ifdef CONFIG_CRASH_DUMP void machine_crash_shutdown(struct pt_regs *regs) { @@ -136,37 +139,28 @@ int __init overlaps_crashkernel(unsigned long start, unsigned long size) } /* Values we need to export to the second kernel via the device tree. */ -static phys_addr_t kernel_end; -static phys_addr_t crashk_base; -static phys_addr_t crashk_size; -static unsigned long long mem_limit; - -static struct property kernel_end_prop = { - .name = "linux,kernel-end", - .length = sizeof(phys_addr_t), - .value = &kernel_end, -}; +static __be_word crashk_base; +static __be_word crashk_size; +static __be_word mem_limit; static struct property crashk_base_prop = { .name = "linux,crashkernel-base", - .length = sizeof(phys_addr_t), + .length = sizeof(__be_word), .value = &crashk_base }; static struct property crashk_size_prop = { .name = "linux,crashkernel-size", - .length = sizeof(phys_addr_t), + .length = sizeof(__be_word), .value = &crashk_size, }; static struct property memory_limit_prop = { .name = "linux,memory-limit", - .length = sizeof(unsigned long long), + .length = sizeof(__be_word), .value = &mem_limit, }; -#define cpu_to_be_ulong __PASTE(cpu_to_be, BITS_PER_LONG) - static void __init export_crashk_values(struct device_node *node) { /* There might be existing crash kernel properties, but we can't @@ -190,6 +184,15 @@ static void __init export_crashk_values(struct device_node *node) mem_limit = cpu_to_be_ulong(memory_limit); of_update_property(node, &memory_limit_prop); } +#endif /* CONFIG_CRASH_RESERVE */ + +static __be_word kernel_end; + +static struct property kernel_end_prop = { + .name = "linux,kernel-end", + .length = sizeof(__be_word), + .value = &kernel_end, +}; static int __init kexec_setup(void) { @@ -200,16 +203,17 @@ static int __init kexec_setup(void) return -ENOENT; /* remove any stale properties so ours can be found */ - of_remove_property(node, of_find_property(node, kernel_end_prop.name, NULL)); + of_remove_property(node, of_find_property(node, kernel_end_prop.name, + NULL)); /* information needed by userspace when using default_machine_kexec */ kernel_end = cpu_to_be_ulong(__pa(_end)); of_add_property(node, &kernel_end_prop); +#ifdef CONFIG_CRASH_RESERVE export_crashk_values(node); - +#endif of_node_put(node); return 0; } late_initcall(kexec_setup); -#endif /* CONFIG_CRASH_RESERVE */ diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c index 222aa326dacee..825ab8a88f18e 100644 --- a/arch/powerpc/kexec/core_64.c +++ b/arch/powerpc/kexec/core_64.c @@ -202,6 +202,23 @@ static void kexec_prepare_cpus_wait(int wait_state) mb(); } + +/* + * The add_cpu() call in wake_offline_cpus() can fail as cpu_bootable() + * returns false for CPUs that fail the cpu_smt_thread_allowed() check + * or non primary threads if SMT is disabled. Re-enable SMT and set the + * number of SMT threads to threads per core. + */ +static void kexec_smt_reenable(void) +{ +#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) + lock_device_hotplug(); + cpu_smt_num_threads = threads_per_core; + cpu_smt_control = CPU_SMT_ENABLED; + unlock_device_hotplug(); +#endif +} + /* * We need to make sure each present CPU is online. The next kernel will scan * the device tree and assume primary threads are online and query secondary @@ -216,6 +233,8 @@ static void wake_offline_cpus(void) { int cpu = 0; + kexec_smt_reenable(); + for_each_present_cpu(cpu) { if (!cpu_online(cpu)) { printk(KERN_INFO "kexec: Waking offline cpu %d.\n", diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index e7ef8b2a25546..5f6d50e4c3d45 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -450,6 +450,11 @@ static int load_elfcorehdr_segment(struct kimage *image, struct kexec_buf *kbuf) kbuf->buffer = headers; kbuf->mem = KEXEC_BUF_MEM_UNKNOWN; kbuf->bufsz = headers_sz; + + /* + * Account for extra space required to accommodate additional memory + * ranges in elfcorehdr due to memory hotplug events. + */ kbuf->memsz = headers_sz + kdump_extra_elfcorehdr_size(cmem); kbuf->top_down = false; @@ -460,7 +465,14 @@ static int load_elfcorehdr_segment(struct kimage *image, struct kexec_buf *kbuf) } image->elf_load_addr = kbuf->mem; - image->elf_headers_sz = headers_sz; + + /* + * If CONFIG_CRASH_HOTPLUG is enabled, the elfcorehdr kexec segment + * memsz can be larger than bufsz. Always initialize elf_headers_sz + * with memsz. This ensures the correct size is reserved for elfcorehdr + * memory in the FDT prepared for kdump. + */ + image->elf_headers_sz = kbuf->memsz; image->elf_headers = headers; out: kfree(cmem); diff --git a/arch/powerpc/kexec/ranges.c b/arch/powerpc/kexec/ranges.c index 3702b0bdab141..426bdca4667e7 100644 --- a/arch/powerpc/kexec/ranges.c +++ b/arch/powerpc/kexec/ranges.c @@ -697,8 +697,8 @@ int remove_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size) * two half. */ else { + size = mem_rngs->ranges[i].end - end + 1; mem_rngs->ranges[i].end = base - 1; - size = mem_rngs->ranges[i].end - end; ret = add_mem_range(mem_ranges, end + 1, size); } } diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index d79c5d1098c05..2efbe05caed76 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -38,7 +38,7 @@ /* #define EXIT_DEBUG */ -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_ICOUNTER(VM, num_2M_pages), STATS_DESC_ICOUNTER(VM, num_1G_pages) @@ -53,7 +53,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, sum_exits), STATS_DESC_COUNTER(VCPU, mmio_exits), diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 3401b96be475e..f3ddb24ece749 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -36,7 +36,7 @@ unsigned long kvmppc_booke_handlers; -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_ICOUNTER(VM, num_2M_pages), STATS_DESC_ICOUNTER(VM, num_1G_pages) @@ -51,7 +51,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, sum_exits), STATS_DESC_COUNTER(VCPU, mmio_exits), diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c index 4e724c4c01add..0f0f2d851ac67 100644 --- a/arch/powerpc/lib/pmem.c +++ b/arch/powerpc/lib/pmem.c @@ -66,15 +66,16 @@ EXPORT_SYMBOL_GPL(arch_invalidate_pmem); /* * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols */ -long __copy_from_user_flushcache(void *dest, const void __user *src, - unsigned size) +size_t copy_from_user_flushcache(void *dest, const void __user *src, + size_t size) { - unsigned long copied, start = (unsigned long) dest; + unsigned long not_copied, start = (unsigned long) dest; - copied = __copy_from_user(dest, src, size); + src = mask_user_address(src); + not_copied = __copy_from_user(dest, src, size); clean_pmem_range(start, start + size); - return copied; + return not_copied; } void memcpy_flushcache(void *dest, const void *src, size_t size) diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c index 9ad6b56bfec96..e54a7b0112322 100644 --- a/arch/powerpc/mm/book3s32/tlb.c +++ b/arch/powerpc/mm/book3s32/tlb.c @@ -105,3 +105,12 @@ void hash__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) flush_hash_pages(mm->context.id, vmaddr, pmd_val(*pmd), 1); } EXPORT_SYMBOL(hash__flush_tlb_page); + +void hash__flush_gather(struct mmu_gather *tlb) +{ + if (tlb->fullmm || tlb->need_flush_all) + hash__flush_tlb_mm(tlb->mm); + else + hash__flush_range(tlb->mm, tlb->start, tlb->end); +} +EXPORT_SYMBOL(hash__flush_gather); diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index 3aee3af614af8..c99be1286d517 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -1302,11 +1302,14 @@ static void __init htab_initialize(void) unsigned long table; unsigned long pteg_count; unsigned long prot; - phys_addr_t base = 0, size = 0, end; + phys_addr_t base = 0, size = 0, end, limit = MEMBLOCK_ALLOC_ANYWHERE; u64 i; DBG(" -> htab_initialize()\n"); + if (firmware_has_feature(FW_FEATURE_LPAR)) + limit = ppc64_rma_size; + if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) { mmu_kernel_ssize = MMU_SEGSIZE_1T; mmu_highuser_ssize = MMU_SEGSIZE_1T; @@ -1322,7 +1325,7 @@ static void __init htab_initialize(void) // Too early to use nr_cpu_ids, so use NR_CPUS tmp = memblock_phys_alloc_range(sizeof(struct stress_hpt_struct) * NR_CPUS, __alignof__(struct stress_hpt_struct), - 0, MEMBLOCK_ALLOC_ANYWHERE); + MEMBLOCK_LOW_LIMIT, limit); memset((void *)tmp, 0xff, sizeof(struct stress_hpt_struct) * NR_CPUS); stress_hpt_struct = __va(tmp); @@ -1356,11 +1359,10 @@ static void __init htab_initialize(void) mmu_hash_ops.hpte_clear_all(); #endif } else { - unsigned long limit = MEMBLOCK_ALLOC_ANYWHERE; table = memblock_phys_alloc_range(htab_size_bytes, htab_size_bytes, - 0, limit); + MEMBLOCK_LOW_LIMIT, limit); if (!table) panic("ERROR: Failed to allocate %pa bytes below %pa\n", &htab_size_bytes, &limit); diff --git a/arch/powerpc/mm/book3s64/internal.h b/arch/powerpc/mm/book3s64/internal.h index a57a25f06a215..c26a6f0c90fcd 100644 --- a/arch/powerpc/mm/book3s64/internal.h +++ b/arch/powerpc/mm/book3s64/internal.h @@ -24,8 +24,6 @@ static inline bool stress_hpt(void) void hpt_do_stress(unsigned long ea, unsigned long hpte_group); -void slb_setup_new_exec(void); - void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush); #endif /* ARCH_POWERPC_MM_BOOK3S64_INTERNAL_H */ diff --git a/arch/powerpc/mm/book3s64/mmu_context.c b/arch/powerpc/mm/book3s64/mmu_context.c index 4e1e45420bd49..fb9dcf9ca5998 100644 --- a/arch/powerpc/mm/book3s64/mmu_context.c +++ b/arch/powerpc/mm/book3s64/mmu_context.c @@ -150,8 +150,6 @@ static int hash__init_new_context(struct mm_struct *mm) void hash__setup_new_exec(void) { slice_setup_new_exec(); - - slb_setup_new_exec(); } #else static inline int hash__init_new_context(struct mm_struct *mm) diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c index 6b783552403c6..7e053c561a09d 100644 --- a/arch/powerpc/mm/book3s64/slb.c +++ b/arch/powerpc/mm/book3s64/slb.c @@ -328,94 +328,6 @@ static void preload_age(struct thread_info *ti) ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR; } -void slb_setup_new_exec(void) -{ - struct thread_info *ti = current_thread_info(); - struct mm_struct *mm = current->mm; - unsigned long exec = 0x10000000; - - WARN_ON(irqs_disabled()); - - /* - * preload cache can only be used to determine whether a SLB - * entry exists if it does not start to overflow. - */ - if (ti->slb_preload_nr + 2 > SLB_PRELOAD_NR) - return; - - hard_irq_disable(); - - /* - * We have no good place to clear the slb preload cache on exec, - * flush_thread is about the earliest arch hook but that happens - * after we switch to the mm and have already preloaded the SLBEs. - * - * For the most part that's probably okay to use entries from the - * previous exec, they will age out if unused. It may turn out to - * be an advantage to clear the cache before switching to it, - * however. - */ - - /* - * preload some userspace segments into the SLB. - * Almost all 32 and 64bit PowerPC executables are linked at - * 0x10000000 so it makes sense to preload this segment. - */ - if (!is_kernel_addr(exec)) { - if (preload_add(ti, exec)) - slb_allocate_user(mm, exec); - } - - /* Libraries and mmaps. */ - if (!is_kernel_addr(mm->mmap_base)) { - if (preload_add(ti, mm->mmap_base)) - slb_allocate_user(mm, mm->mmap_base); - } - - /* see switch_slb */ - asm volatile("isync" : : : "memory"); - - local_irq_enable(); -} - -void preload_new_slb_context(unsigned long start, unsigned long sp) -{ - struct thread_info *ti = current_thread_info(); - struct mm_struct *mm = current->mm; - unsigned long heap = mm->start_brk; - - WARN_ON(irqs_disabled()); - - /* see above */ - if (ti->slb_preload_nr + 3 > SLB_PRELOAD_NR) - return; - - hard_irq_disable(); - - /* Userspace entry address. */ - if (!is_kernel_addr(start)) { - if (preload_add(ti, start)) - slb_allocate_user(mm, start); - } - - /* Top of stack, grows down. */ - if (!is_kernel_addr(sp)) { - if (preload_add(ti, sp)) - slb_allocate_user(mm, sp); - } - - /* Bottom of heap, grows up. */ - if (heap && !is_kernel_addr(heap)) { - if (preload_add(ti, heap)) - slb_allocate_user(mm, heap); - } - - /* see switch_slb */ - asm volatile("isync" : : : "memory"); - - local_irq_enable(); -} - static void slb_cache_slbie_kernel(unsigned int index) { unsigned long slbie_data = get_paca()->slb_cache[index]; diff --git a/arch/powerpc/mm/ptdump/hashpagetable.c b/arch/powerpc/mm/ptdump/hashpagetable.c index a6baa6166d940..671d0dc00c6d0 100644 --- a/arch/powerpc/mm/ptdump/hashpagetable.c +++ b/arch/powerpc/mm/ptdump/hashpagetable.c @@ -216,6 +216,8 @@ static int native_find(unsigned long ea, int psize, bool primary, u64 *v, u64 vpn = hpt_vpn(ea, vsid, ssize); hash = hpt_hash(vpn, shift, ssize); want_v = hpte_encode_avpn(vpn, psize, ssize); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + want_v = hpte_old_to_new_v(want_v); /* to check in the secondary hash table, we invert the hash */ if (!primary) @@ -229,6 +231,10 @@ static int native_find(unsigned long ea, int psize, bool primary, u64 *v, u64 /* HPTE matches */ *v = be64_to_cpu(hptep->v); *r = be64_to_cpu(hptep->r); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + *v = hpte_new_to_old_v(*v, *r); + *r = hpte_new_to_old_r(*r); + } return 0; } ++hpte_group; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 88ad5ba7b87fd..189ef7b72081d 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -437,7 +437,7 @@ void bpf_jit_free(struct bpf_prog *fp) bool bpf_jit_supports_kfunc_call(void) { - return true; + return IS_ENABLED(CONFIG_PPC64); } bool bpf_jit_supports_arena(void) @@ -722,9 +722,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * retval_off [ return value ] * [ reg argN ] * [ ... ] - * regs_off [ reg_arg1 ] prog ctx context - * nregs_off [ args count ] - * ip_off [ traced function ] + * regs_off [ reg_arg1 ] prog_ctx + * nregs_off [ args count ] ((u64 *)prog_ctx)[-1] + * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] * [ ... ] * run_ctx_off [ bpf_tramp_run_ctx ] * [ reg argN ] @@ -824,7 +824,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im bpf_trampoline_save_args(image, ctx, func_frame_offset, nr_regs, regs_off); - /* Save our return address */ + /* Save our LR/return address */ EMIT(PPC_RAW_MFLR(_R3)); if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off)); @@ -832,24 +832,34 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF)); /* - * Save ip address of the traced function. - * We could recover this from LR, but we will need to address for OOL trampoline, - * and optional GEP area. + * Derive IP address of the traced function. + * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction + * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and + * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively. + * Relevant stub sequence: + * + * bl + * LR (R3) => mtlr r0 + * b + * + * Recover kernel function/bpf program address from the unconditional + * branch instruction at the end of OOL stub. */ if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) { EMIT(PPC_RAW_LWZ(_R4, _R3, 4)); EMIT(PPC_RAW_SLWI(_R4, _R4, 6)); EMIT(PPC_RAW_SRAWI(_R4, _R4, 6)); EMIT(PPC_RAW_ADD(_R3, _R3, _R4)); - EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); } if (flags & BPF_TRAMP_F_IP_ARG) EMIT(PPC_RAW_STL(_R3, _R1, ip_off)); - if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) + if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { /* Fake our LR for unwind */ + EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF)); + } /* Save function arg count -- see bpf_get_func_arg_cnt() */ EMIT(PPC_RAW_LI(_R3, nr_regs)); @@ -1122,7 +1132,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, branch_flags = poke_type == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; /* We currently only support poking bpf programs */ - if (!__bpf_address_lookup(bpf_func, &size, &offset, name)) { + if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); return -EOPNOTSUPP; } diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 1fe37128c8764..94f3f031d039b 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -319,6 +319,83 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context * return 0; } +static int zero_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size) +{ + switch (size) { + case 1: + /* zero-extend 8 bits into 64 bits */ + EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 56)); + return 0; + case 2: + /* zero-extend 16 bits into 64 bits */ + EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 48)); + return 0; + case 4: + /* zero-extend 32 bits into 64 bits */ + EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 32)); + fallthrough; + case 8: + /* Nothing to do */ + return 0; + default: + return -1; + } +} + +static int sign_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size) +{ + switch (size) { + case 1: + /* sign-extend 8 bits into 64 bits */ + EMIT(PPC_RAW_EXTSB(dst_reg, src_reg)); + return 0; + case 2: + /* sign-extend 16 bits into 64 bits */ + EMIT(PPC_RAW_EXTSH(dst_reg, src_reg)); + return 0; + case 4: + /* sign-extend 32 bits into 64 bits */ + EMIT(PPC_RAW_EXTSW(dst_reg, src_reg)); + fallthrough; + case 8: + /* Nothing to do */ + return 0; + default: + return -1; + } +} + +/* + * Handle powerpc ABI expectations from caller: + * - Unsigned arguments are zero-extended. + * - Signed arguments are sign-extended. + */ +static int prepare_for_kfunc_call(const struct bpf_prog *fp, u32 *image, + struct codegen_context *ctx, + const struct bpf_insn *insn) +{ + const struct btf_func_model *m = bpf_jit_find_kfunc_model(fp, insn); + int i; + + if (!m) + return -1; + + for (i = 0; i < m->nr_args; i++) { + /* Note that BPF ABI only allows up to 5 args for kfuncs */ + u32 reg = bpf_to_ppc(BPF_REG_1 + i), size = m->arg_size[i]; + + if (!(m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG)) { + if (zero_extend(image, ctx, reg, reg, size)) + return -1; + } else { + if (sign_extend(image, ctx, reg, reg, size)) + return -1; + } + } + + return 0; +} + static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) { /* @@ -353,27 +430,32 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o /* * tail_call_cnt++; + * Writeback this updated value only if tailcall succeeds. */ EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 1)); - EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx))); /* prog = array->ptrs[index]; */ - EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_1), b2p_index, 8)); - EMIT(PPC_RAW_ADD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), b2p_bpf_array)); - EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), offsetof(struct bpf_array, ptrs))); + EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_2), b2p_index, 8)); + EMIT(PPC_RAW_ADD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), b2p_bpf_array)); + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), + offsetof(struct bpf_array, ptrs))); /* * if (prog == NULL) * goto out; */ - EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_1), 0)); + EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_2), 0)); PPC_BCC_SHORT(COND_EQ, out); /* goto *(prog->bpf_func + prologue_size); */ - EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), offsetof(struct bpf_prog, bpf_func))); - EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), - FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); - EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_1))); + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), + offsetof(struct bpf_prog, bpf_func))); + EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), + FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); + EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_2))); + + /* Writeback updated tailcall count */ + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx))); /* tear down stack, restore NVRs, ... */ bpf_jit_emit_common_epilogue(image, ctx); @@ -931,14 +1013,16 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code /* special mov32 for zext */ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31)); break; - } else if (off == 8) { - EMIT(PPC_RAW_EXTSB(dst_reg, src_reg)); - } else if (off == 16) { - EMIT(PPC_RAW_EXTSH(dst_reg, src_reg)); - } else if (off == 32) { - EMIT(PPC_RAW_EXTSW(dst_reg, src_reg)); - } else if (dst_reg != src_reg) - EMIT(PPC_RAW_MR(dst_reg, src_reg)); + } + if (off == 0) { + /* MOV */ + if (dst_reg != src_reg) + EMIT(PPC_RAW_MR(dst_reg, src_reg)); + } else { + /* MOVSX: dst = (s8,s16,s32)src (off = 8,16,32) */ + if (sign_extend(image, ctx, src_reg, dst_reg, off / 8)) + return -1; + } goto bpf_alu32_trunc; case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ @@ -1395,6 +1479,12 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code if (ret < 0) return ret; + /* Take care of powerpc ABI requirements before kfunc call */ + if (insn[i].src_reg == BPF_PSEUDO_KFUNC_CALL) { + if (prepare_for_kfunc_call(fp, image, ctx, &insn[i])) + return -1; + } + ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr); if (ret) return ret; diff --git a/arch/powerpc/perf/callchain.c b/arch/powerpc/perf/callchain.c index 26aa26482c9ac..992cc5c982144 100644 --- a/arch/powerpc/perf/callchain.c +++ b/arch/powerpc/perf/callchain.c @@ -103,6 +103,11 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + perf_callchain_store(entry, perf_arch_instruction_pointer(regs)); + + if (!current->mm) + return; + if (!is_32bit_task()) perf_callchain_user_64(entry, regs); else diff --git a/arch/powerpc/perf/callchain_32.c b/arch/powerpc/perf/callchain_32.c index ddcc2d8aa64a5..0de21c5d272c2 100644 --- a/arch/powerpc/perf/callchain_32.c +++ b/arch/powerpc/perf/callchain_32.c @@ -142,7 +142,6 @@ void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry, next_ip = perf_arch_instruction_pointer(regs); lr = regs->link; sp = regs->gpr[1]; - perf_callchain_store(entry, next_ip); while (entry->nr < entry->max_stack) { fp = (unsigned int __user *) (unsigned long) sp; diff --git a/arch/powerpc/perf/callchain_64.c b/arch/powerpc/perf/callchain_64.c index 115d1c105e8a8..30fb61c5f0cb0 100644 --- a/arch/powerpc/perf/callchain_64.c +++ b/arch/powerpc/perf/callchain_64.c @@ -77,7 +77,6 @@ void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry, next_ip = perf_arch_instruction_pointer(regs); lr = regs->link; sp = regs->gpr[1]; - perf_callchain_store(entry, next_ip); while (entry->nr < entry->max_stack) { fp = (unsigned long __user *) sp; diff --git a/arch/powerpc/platforms/83xx/km83xx.c b/arch/powerpc/platforms/83xx/km83xx.c index 2b5d187d9b62d..9ef8fb39dd1b1 100644 --- a/arch/powerpc/platforms/83xx/km83xx.c +++ b/arch/powerpc/platforms/83xx/km83xx.c @@ -155,8 +155,8 @@ machine_device_initcall(mpc83xx_km, mpc83xx_declare_of_platform_devices); /* list of the supported boards */ static char *board[] __initdata = { - "Keymile,KMETER1", - "Keymile,kmpbec8321", + "keymile,KMETER1", + "keymile,kmpbec8321", NULL }; diff --git a/arch/powerpc/platforms/pseries/cmm.c b/arch/powerpc/platforms/pseries/cmm.c index 0823fa2da1516..310dab4bc8679 100644 --- a/arch/powerpc/platforms/pseries/cmm.c +++ b/arch/powerpc/platforms/pseries/cmm.c @@ -532,6 +532,7 @@ static int cmm_migratepage(struct balloon_dev_info *b_dev_info, spin_lock_irqsave(&b_dev_info->pages_lock, flags); balloon_page_insert(b_dev_info, newpage); + __count_vm_event(BALLOON_MIGRATE); b_dev_info->isolated_pages--; spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); @@ -550,7 +551,6 @@ static int cmm_migratepage(struct balloon_dev_info *b_dev_info, static void cmm_balloon_compaction_init(void) { - balloon_devinfo_init(&b_dev_info); b_dev_info.migratepage = cmm_migratepage; } #else /* CONFIG_BALLOON_COMPACTION */ @@ -572,6 +572,7 @@ static int cmm_init(void) if (!firmware_has_feature(FW_FEATURE_CMO) && !simulate) return -EOPNOTSUPP; + balloon_devinfo_init(&b_dev_info); cmm_balloon_compaction_init(); rc = register_oom_notifier(&cmm_oom_nb); diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index a82aaa786e9e0..56f17296545ac 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -19,6 +19,11 @@ #include "pseries.h" +struct pseries_msi_device { + unsigned int msi_quota; + unsigned int msi_used; +}; + static int query_token, change_token; #define RTAS_QUERY_FN 0 @@ -433,8 +438,28 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev struct msi_domain_info *info = domain->host_data; struct pci_dev *pdev = to_pci_dev(dev); int type = (info->flags & MSI_FLAG_PCI_MSIX) ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; + int ret; + + struct pseries_msi_device *pseries_dev __free(kfree) + = kmalloc(sizeof(*pseries_dev), GFP_KERNEL); + if (!pseries_dev) + return -ENOMEM; + + while (1) { + ret = rtas_prepare_msi_irqs(pdev, nvec, type, arg); + if (!ret) + break; + else if (ret > 0) + nvec = ret; + else + return ret; + } - return rtas_prepare_msi_irqs(pdev, nvec, type, arg); + pseries_dev->msi_quota = nvec; + pseries_dev->msi_used = 0; + + arg->scratchpad[0].ptr = no_free_ptr(pseries_dev); + return 0; } /* @@ -443,9 +468,13 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev */ static void pseries_msi_ops_teardown(struct irq_domain *domain, msi_alloc_info_t *arg) { + struct pseries_msi_device *pseries_dev = arg->scratchpad[0].ptr; struct pci_dev *pdev = to_pci_dev(domain->dev); rtas_disable_msi(pdev); + + WARN_ON(pseries_dev->msi_used); + kfree(pseries_dev); } static void pseries_msi_shutdown(struct irq_data *d) @@ -546,12 +575,18 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs, void *arg) { struct pci_controller *phb = domain->host_data; + struct pseries_msi_device *pseries_dev; msi_alloc_info_t *info = arg; struct msi_desc *desc = info->desc; struct pci_dev *pdev = msi_desc_to_pci_dev(desc); int hwirq; int i, ret; + pseries_dev = info->scratchpad[0].ptr; + + if (pseries_dev->msi_used + nr_irqs > pseries_dev->msi_quota) + return -ENOSPC; + hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_index); if (hwirq < 0) { dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); @@ -567,9 +602,10 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq goto out; irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, - &pseries_msi_irq_chip, domain->host_data); + &pseries_msi_irq_chip, pseries_dev); } + pseries_dev->msi_used += nr_irqs; return 0; out: @@ -582,9 +618,11 @@ static void pseries_irq_domain_free(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); - struct pci_controller *phb = irq_data_get_irq_chip_data(d); + struct pseries_msi_device *pseries_dev = irq_data_get_irq_chip_data(d); + struct pci_controller *phb = domain->host_data; pr_debug("%s bridge %pOF %d #%d\n", __func__, phb->dn, virq, nr_irqs); + pseries_dev->msi_used -= nr_irqs; irq_domain_free_irqs_parent(domain, virq, nr_irqs); } diff --git a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh index bac186bdf64a7..9218d43aeb548 100755 --- a/arch/powerpc/tools/ftrace-gen-ool-stubs.sh +++ b/arch/powerpc/tools/ftrace-gen-ool-stubs.sh @@ -15,9 +15,9 @@ if [ -z "$is_64bit" ]; then RELOCATION=R_PPC_ADDR32 fi -num_ool_stubs_total=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | +num_ool_stubs_total=$($objdump -r -j __patchable_function_entries -d "$vmlinux_o" | grep -c "$RELOCATION") -num_ool_stubs_inittext=$($objdump -r -j __patchable_function_entries "$vmlinux_o" | +num_ool_stubs_inittext=$($objdump -r -j __patchable_function_entries -d "$vmlinux_o" | grep -e ".init.text" -e ".text.startup" | grep -c "$RELOCATION") num_ool_stubs_text=$((num_ool_stubs_total - num_ool_stubs_inittext)) diff --git a/arch/powerpc/tools/gcc-check-fpatchable-function-entry.sh b/arch/powerpc/tools/gcc-check-fpatchable-function-entry.sh index 06706903503b6..baed467a016b3 100755 --- a/arch/powerpc/tools/gcc-check-fpatchable-function-entry.sh +++ b/arch/powerpc/tools/gcc-check-fpatchable-function-entry.sh @@ -2,7 +2,6 @@ # SPDX-License-Identifier: GPL-2.0 set -e -set -o pipefail # To debug, uncomment the following line # set -x diff --git a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh index 73e331e7660ef..6193b0ed0c775 100755 --- a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh +++ b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh @@ -2,7 +2,6 @@ # SPDX-License-Identifier: GPL-2.0 set -e -set -o pipefail # To debug, uncomment the following line # set -x diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index bfc3d0b75b9b2..5301adf5f3f5d 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -31,11 +31,7 @@ $(obj)/xipImage: vmlinux FORCE endif -ifdef CONFIG_RELOCATABLE -$(obj)/Image: vmlinux.unstripped FORCE -else $(obj)/Image: vmlinux FORCE -endif $(call if_changed,objcopy) $(obj)/Image.gz: $(obj)/Image FORCE diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index a75d6325607b4..14c5acb935e94 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -4,7 +4,8 @@ menu "Accelerated Cryptographic Algorithms for CPU (riscv)" config CRYPTO_AES_RISCV64 tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS select CRYPTO_ALGAPI select CRYPTO_LIB_AES select CRYPTO_SKCIPHER @@ -20,7 +21,8 @@ config CRYPTO_AES_RISCV64 config CRYPTO_GHASH_RISCV64 tristate "Hash functions: GHASH" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS select CRYPTO_GCM help GCM GHASH function (NIST SP 800-38D) @@ -30,7 +32,8 @@ config CRYPTO_GHASH_RISCV64 config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS select CRYPTO_HASH select CRYPTO_LIB_SM3 help @@ -42,7 +45,8 @@ config CRYPTO_SM3_RISCV64 config CRYPTO_SM4_RISCV64 tristate "Ciphers: SM4 (ShangMi 4)" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS select CRYPTO_ALGAPI select CRYPTO_SM4 help diff --git a/arch/riscv/include/asm/compat.h b/arch/riscv/include/asm/compat.h index 6081327e55f5b..28e115eed2180 100644 --- a/arch/riscv/include/asm/compat.h +++ b/arch/riscv/include/asm/compat.h @@ -2,7 +2,7 @@ #ifndef __ASM_COMPAT_H #define __ASM_COMPAT_H -#define COMPAT_UTS_MACHINE "riscv\0\0" +#define COMPAT_UTS_MACHINE "riscv32\0\0" /* * Architecture specific compatibility types diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 5a08eb5fe99fc..30d1ea93dde34 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -124,10 +124,6 @@ #ifdef CONFIG_64BIT #include -#define VA_USER_SV39 (UL(1) << (VA_BITS_SV39 - 1)) -#define VA_USER_SV48 (UL(1) << (VA_BITS_SV48 - 1)) -#define VA_USER_SV57 (UL(1) << (VA_BITS_SV57 - 1)) - #define MMAP_VA_BITS_64 ((VA_BITS >= VA_BITS_SV48) ? VA_BITS_SV48 : VA_BITS) #define MMAP_MIN_VA_BITS_64 (VA_BITS_SV39) #define MMAP_VA_BITS (is_compat_task() ? VA_BITS_SV32 : MMAP_VA_BITS_64) diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index f5f4f7f85543f..6aef591a6bfcb 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -97,13 +97,23 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, unsigne */ #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT +/* + * Use a temporary variable for the output of the asm goto to avoid a + * triggering an LLVM assertion due to sign extending the output when + * it is used in later function calls: + * https://github.com/llvm/llvm-project/issues/143795 + */ #define __get_user_asm(insn, x, ptr, label) \ +do { \ + u64 __tmp; \ asm_goto_output( \ "1:\n" \ " " insn " %0, %1\n" \ _ASM_EXTABLE_UACCESS_ERR(1b, %l2, %0) \ - : "=&r" (x) \ - : "m" (*(ptr)) : : label) + : "=&r" (__tmp) \ + : "m" (*(ptr)) : : label); \ + (x) = (__typeof__(x))(unsigned long)__tmp; \ +} while (0) #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ #define __get_user_asm(insn, x, ptr, label) \ do { \ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index f60fce69b7259..a01f6439d62b1 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -3,12 +3,6 @@ # Makefile for the RISC-V Linux kernel # -ifdef CONFIG_FTRACE -CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) -endif CFLAGS_syscall_table.o += $(call cc-disable-warning, override-init) CFLAGS_compat_syscall_table.o += $(call cc-disable-warning, override-init) @@ -24,7 +18,6 @@ CFLAGS_sbi_ecall.o := -mcmodel=medany ifdef CONFIG_FTRACE CFLAGS_REMOVE_alternative.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_cpufeature.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_sbi_ecall.o = $(CC_FLAGS_FTRACE) endif ifdef CONFIG_RELOCATABLE CFLAGS_alternative.o += -fno-pie @@ -43,6 +36,14 @@ CFLAGS_sbi_ecall.o += -D__NO_FORTIFY endif endif +ifdef CONFIG_FTRACE +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sbi_ecall.o = $(CC_FLAGS_FTRACE) +endif + always-$(KBUILD_BUILTIN) += vmlinux.lds obj-y += head.o diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 72ca768f4e919..2367e9755524a 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -279,23 +279,22 @@ static const unsigned int riscv_a_exts[] = { RISCV_ISA_EXT_ZALRSC, }; +#define RISCV_ISA_EXT_ZKN \ + RISCV_ISA_EXT_ZBKB, \ + RISCV_ISA_EXT_ZBKC, \ + RISCV_ISA_EXT_ZBKX, \ + RISCV_ISA_EXT_ZKND, \ + RISCV_ISA_EXT_ZKNE, \ + RISCV_ISA_EXT_ZKNH + static const unsigned int riscv_zk_bundled_exts[] = { - RISCV_ISA_EXT_ZBKB, - RISCV_ISA_EXT_ZBKC, - RISCV_ISA_EXT_ZBKX, - RISCV_ISA_EXT_ZKND, - RISCV_ISA_EXT_ZKNE, + RISCV_ISA_EXT_ZKN, RISCV_ISA_EXT_ZKR, - RISCV_ISA_EXT_ZKT, + RISCV_ISA_EXT_ZKT }; static const unsigned int riscv_zkn_bundled_exts[] = { - RISCV_ISA_EXT_ZBKB, - RISCV_ISA_EXT_ZBKC, - RISCV_ISA_EXT_ZBKX, - RISCV_ISA_EXT_ZKND, - RISCV_ISA_EXT_ZKNE, - RISCV_ISA_EXT_ZKNH, + RISCV_ISA_EXT_ZKN }; static const unsigned int riscv_zks_bundled_exts[] = { diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 15fec5d1e6dec..0bf629204c76a 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -175,7 +175,7 @@ struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { {DBG_REG_T1, GDB_SIZEOF_REG, offsetof(struct pt_regs, t1)}, {DBG_REG_T2, GDB_SIZEOF_REG, offsetof(struct pt_regs, t2)}, {DBG_REG_FP, GDB_SIZEOF_REG, offsetof(struct pt_regs, s0)}, - {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, s1)}, {DBG_REG_A0, GDB_SIZEOF_REG, offsetof(struct pt_regs, a0)}, {DBG_REG_A1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, {DBG_REG_A2, GDB_SIZEOF_REG, offsetof(struct pt_regs, a2)}, @@ -244,8 +244,9 @@ sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) gdb_regs[DBG_REG_S6_OFF] = task->thread.s[6]; gdb_regs[DBG_REG_S7_OFF] = task->thread.s[7]; gdb_regs[DBG_REG_S8_OFF] = task->thread.s[8]; - gdb_regs[DBG_REG_S9_OFF] = task->thread.s[10]; - gdb_regs[DBG_REG_S10_OFF] = task->thread.s[11]; + gdb_regs[DBG_REG_S9_OFF] = task->thread.s[9]; + gdb_regs[DBG_REG_S10_OFF] = task->thread.s[10]; + gdb_regs[DBG_REG_S11_OFF] = task->thread.s[11]; gdb_regs[DBG_REG_EPC_OFF] = task->thread.ra; } diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 31a392993cb45..b5188dc74727d 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -324,8 +324,10 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen)) return -EINVAL; - if (!(arg & PR_TAGGED_ADDR_ENABLE)) + if (!(arg & PR_TAGGED_ADDR_ENABLE)) { pmlen = PMLEN_0; + pmm = ENVCFG_PMM_PMLEN_0; + } if (mmap_write_lock_killable(mm)) return -EINTR; diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index 24b3f57d467f8..aff93090c4efc 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -51,10 +51,11 @@ void suspend_restore_csrs(struct suspend_context *context) #ifdef CONFIG_MMU if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) { - csr_write(CSR_STIMECMP, context->stimecmp); #if __riscv_xlen < 64 + csr_write(CSR_STIMECMP, ULONG_MAX); csr_write(CSR_STIMECMPH, context->stimecmph); #endif + csr_write(CSR_STIMECMP, context->stimecmp); } csr_write(CSR_SATP, context->satp); diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 80230de167def..47afea4ff1a8d 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -339,8 +339,10 @@ void do_trap_ecall_u(struct pt_regs *regs) add_random_kstack_offset(); - if (syscall >= 0 && syscall < NR_syscalls) + if (syscall >= 0 && syscall < NR_syscalls) { + syscall = array_index_nospec(syscall, NR_syscalls); syscall_handler(regs, syscall); + } /* * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(), diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c index 901e67adf5760..34048c4c26dcd 100644 --- a/arch/riscv/kernel/vector.c +++ b/arch/riscv/kernel/vector.c @@ -109,8 +109,8 @@ bool insn_is_vector(u32 insn_buf) return false; } -static int riscv_v_thread_zalloc(struct kmem_cache *cache, - struct __riscv_v_ext_state *ctx) +static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache, + struct __riscv_v_ext_state *ctx) { void *datap; @@ -120,13 +120,15 @@ static int riscv_v_thread_zalloc(struct kmem_cache *cache, ctx->datap = datap; memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); + ctx->vlenb = riscv_v_vsize / 32; + return 0; } void riscv_v_thread_alloc(struct task_struct *tsk) { #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE - riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); + riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); #endif } @@ -212,12 +214,14 @@ bool riscv_v_first_use_handler(struct pt_regs *regs) * context where VS has been off. So, try to allocate the user's V * context and resume execution. */ - if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) { + if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, ¤t->thread.vstate)) { force_sig(SIGBUS); return true; } + riscv_v_vstate_on(regs); riscv_v_vstate_set_restore(current, regs); + return true; } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 61bd5ba6680a7..997f9eb3b22b1 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -170,6 +170,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .riscv.attributes 0 : { *(.riscv.attributes) } diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 5ce35aba60696..d26c4967c20e2 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -24,7 +24,7 @@ #define CREATE_TRACE_POINTS #include "trace.h" -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, ecall_exit_stat), STATS_DESC_COUNTER(VCPU, wfi_exit_stat), diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index de1f96ea62251..4d89b94128aea 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -298,6 +298,22 @@ static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, return (rc <= 0) ? rc : 1; } +static bool is_load_guest_page_fault(unsigned long scause) +{ + /** + * If a g-stage page fault occurs, the direct approach + * is to let the g-stage page fault handler handle it + * naturally, however, calling the g-stage page fault + * handler here seems rather strange. + * Considering this is a corner case, we can directly + * return to the guest and re-execute the same PC, this + * will trigger a g-stage page fault again and then the + * regular g-stage page fault handler will populate + * g-stage page table. + */ + return (scause == EXC_LOAD_GUEST_PAGE_FAULT); +} + /** * kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap * @@ -323,6 +339,8 @@ int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ct->sepc, &utrap); if (utrap.scause) { + if (is_load_guest_page_fault(utrap.scause)) + return 1; utrap.sepc = ct->sepc; kvm_riscv_vcpu_trap_redirect(vcpu, &utrap); return 1; @@ -378,6 +396,8 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc, &utrap); if (utrap.scause) { + if (is_load_guest_page_fault(utrap.scause)) + return 1; /* Redirect trap if we failed to read instruction */ utrap.sepc = ct->sepc; kvm_riscv_vcpu_trap_redirect(vcpu, &utrap); @@ -504,6 +524,8 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc, &utrap); if (utrap.scause) { + if (is_load_guest_page_fault(utrap.scause)) + return 1; /* Redirect trap if we failed to read instruction */ utrap.sepc = ct->sepc; kvm_riscv_vcpu_trap_redirect(vcpu, &utrap); diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index 66d91ae6e9b2a..715a06ae8c131 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -13,7 +13,7 @@ #include #include -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index df22b10d91415..e60d2b823e091 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -270,6 +270,7 @@ config S390 select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE + select SYSTEM_DATA_VERIFICATION if KEXEC_SIG select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select TTY @@ -296,7 +297,7 @@ config ARCH_SUPPORTS_KEXEC_FILE def_bool y config ARCH_SUPPORTS_KEXEC_SIG - def_bool MODULE_SIG_FORMAT + def_bool y config ARCH_SUPPORTS_KEXEC_PURGATORY def_bool y diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 02f2cf0827487..1768424a1824f 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -21,6 +21,7 @@ KBUILD_AFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_AFLAGS_DECOMPRESSOR)) KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_CFLAGS_DECOMPRESSOR)) KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 50988022f9ea3..070bc18babd0e 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -137,6 +137,15 @@ SECTIONS } _end = .; + /* Sections to be discarded */ + /DISCARD/ : { + COMMON_DISCARDS + *(.eh_frame) + *(*__ksymtab*) + *(___kcrctab*) + *(.modinfo) + } + DWARF_DEBUG ELF_DETAILS @@ -161,12 +170,4 @@ SECTIONS *(.rela.*) *(.rela_*) } ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!") - - /* Sections to be discarded */ - /DISCARD/ : { - COMMON_DISCARDS - *(.eh_frame) - *(*__ksymtab*) - *(___kcrctab*) - } } diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index f3184073e754f..dad02f5b3c8d3 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -62,8 +62,8 @@ do { \ * @size: number of elements in array */ #define array_index_mask_nospec array_index_mask_nospec -static inline unsigned long array_index_mask_nospec(unsigned long index, - unsigned long size) +static __always_inline unsigned long array_index_mask_nospec(unsigned long index, + unsigned long size) { unsigned long mask; diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index e99f8bca8e08c..96727f3bd0dce 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -12,6 +12,7 @@ #ifndef __ASSEMBLER__ #include +#include #include asm(".include \"asm/fpu-insn-asm.h\"\n"); @@ -393,6 +394,7 @@ static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) : [vxr] "=Q" (*(u8 *)vxr) : [index] "d" (index), [v1] "I" (v1) : "memory"); + kmsan_unpoison_memory(vxr, size); } #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */ @@ -409,6 +411,7 @@ static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) : [vxr] "=R" (*(u8 *)vxr) : [index] "d" (index), [v1] "I" (v1) : "memory", "1"); + kmsan_unpoison_memory(vxr, size); } #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */ diff --git a/arch/s390/include/asm/idle.h b/arch/s390/include/asm/idle.h index 09f763b9eb40a..133059d9a949c 100644 --- a/arch/s390/include/asm/idle.h +++ b/arch/s390/include/asm/idle.h @@ -23,5 +23,6 @@ extern struct device_attribute dev_attr_idle_count; extern struct device_attribute dev_attr_idle_time_us; void psw_idle(struct s390_idle_data *data, unsigned long psw_mask); +void update_timer_idle(void); #endif /* _S390_IDLE_H */ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 93e1034485d78..70010bba27e79 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -164,7 +164,7 @@ static __always_inline void __stackleak_poison(unsigned long erase_low, " j 4f\n" "3: mvc 8(1,%[addr]),0(%[addr])\n" "4:" - : [addr] "+&a" (erase_low), [count] "+&d" (count), [tmp] "=&a" (tmp) + : [addr] "+&a" (erase_low), [count] "+&a" (count), [tmp] "=&a" (tmp) : [poison] "d" (poison) : "memory", "cc" ); diff --git a/arch/s390/include/uapi/asm/ipl.h b/arch/s390/include/uapi/asm/ipl.h index 2cd28af50dd43..3d64a22516994 100644 --- a/arch/s390/include/uapi/asm/ipl.h +++ b/arch/s390/include/uapi/asm/ipl.h @@ -15,6 +15,7 @@ struct ipl_pl_hdr { #define IPL_PL_FLAG_IPLPS 0x80 #define IPL_PL_FLAG_SIPL 0x40 #define IPL_PL_FLAG_IPLSR 0x20 +#define IPL_PL_FLAG_SBP 0x10 /* IPL Parameter Block header */ struct ipl_pb_hdr { diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 75b0fbb236d05..68e770e3a7dce 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -254,6 +254,7 @@ SYM_CODE_START(system_call) xgr %r9,%r9 xgr %r10,%r10 xgr %r11,%r11 + xgr %r12,%r12 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13) MBEAR %r2,%r13 @@ -390,6 +391,7 @@ SYM_CODE_START(\name) xgr %r6,%r6 xgr %r7,%r7 xgr %r10,%r10 + xgr %r12,%r12 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) MBEAR %r11,%r13 @@ -479,6 +481,7 @@ SYM_CODE_START(mcck_int_handler) xgr %r6,%r6 xgr %r7,%r7 xgr %r10,%r10 + xgr %r12,%r12 stmg %r8,%r9,__PT_PSW(%r11) xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 39cb8d0ae3480..0f9e53f0a0686 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -21,11 +21,10 @@ static DEFINE_PER_CPU(struct s390_idle_data, s390_idle); -void account_idle_time_irq(void) +void update_timer_idle(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); struct lowcore *lc = get_lowcore(); - unsigned long idle_time; u64 cycles_new[8]; int i; @@ -35,13 +34,19 @@ void account_idle_time_irq(void) this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); } - idle_time = lc->int_clock - idle->clock_idle_enter; - lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock; lc->last_update_clock = lc->int_clock; lc->system_timer += lc->last_update_timer - idle->timer_idle_enter; lc->last_update_timer = lc->sys_enter_timer; +} + +void account_idle_time_irq(void) +{ + struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); + unsigned long idle_time; + + idle_time = get_lowcore()->int_clock - idle->clock_idle_enter; /* Account time spent with enabled wait psw loaded as idle time. */ WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 961a3d60a4ddd..dcdc7e2748486 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -262,6 +262,24 @@ static struct kobj_attribute sys_##_prefix##_##_name##_attr = \ sys_##_prefix##_##_name##_show, \ sys_##_prefix##_##_name##_store) +#define DEFINE_IPL_ATTR_BOOTPROG_RW(_prefix, _name, _fmt_out, _fmt_in, _hdr, _value) \ + IPL_ATTR_SHOW_FN(_prefix, _name, _fmt_out, (unsigned long long) _value) \ +static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t len) \ +{ \ + unsigned long long value; \ + if (sscanf(buf, _fmt_in, &value) != 1) \ + return -EINVAL; \ + (_value) = value; \ + (_hdr).flags &= ~IPL_PL_FLAG_SBP; \ + return len; \ +} \ +static struct kobj_attribute sys_##_prefix##_##_name##_attr = \ + __ATTR(_name, 0644, \ + sys_##_prefix##_##_name##_show, \ + sys_##_prefix##_##_name##_store) + #define DEFINE_IPL_ATTR_STR_RW(_prefix, _name, _fmt_out, _fmt_in, _value)\ IPL_ATTR_SHOW_FN(_prefix, _name, _fmt_out, _value) \ static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj, \ @@ -818,12 +836,13 @@ DEFINE_IPL_ATTR_RW(reipl_fcp, wwpn, "0x%016llx\n", "%llx\n", reipl_block_fcp->fcp.wwpn); DEFINE_IPL_ATTR_RW(reipl_fcp, lun, "0x%016llx\n", "%llx\n", reipl_block_fcp->fcp.lun); -DEFINE_IPL_ATTR_RW(reipl_fcp, bootprog, "%lld\n", "%lld\n", - reipl_block_fcp->fcp.bootprog); DEFINE_IPL_ATTR_RW(reipl_fcp, br_lba, "%lld\n", "%lld\n", reipl_block_fcp->fcp.br_lba); DEFINE_IPL_ATTR_RW(reipl_fcp, device, "0.0.%04llx\n", "0.0.%llx\n", reipl_block_fcp->fcp.devno); +DEFINE_IPL_ATTR_BOOTPROG_RW(reipl_fcp, bootprog, "%lld\n", "%lld\n", + reipl_block_fcp->hdr, + reipl_block_fcp->fcp.bootprog); static void reipl_get_ascii_loadparm(char *loadparm, struct ipl_parameter_block *ibp) @@ -942,10 +961,11 @@ DEFINE_IPL_ATTR_RW(reipl_nvme, fid, "0x%08llx\n", "%llx\n", reipl_block_nvme->nvme.fid); DEFINE_IPL_ATTR_RW(reipl_nvme, nsid, "0x%08llx\n", "%llx\n", reipl_block_nvme->nvme.nsid); -DEFINE_IPL_ATTR_RW(reipl_nvme, bootprog, "%lld\n", "%lld\n", - reipl_block_nvme->nvme.bootprog); DEFINE_IPL_ATTR_RW(reipl_nvme, br_lba, "%lld\n", "%lld\n", reipl_block_nvme->nvme.br_lba); +DEFINE_IPL_ATTR_BOOTPROG_RW(reipl_nvme, bootprog, "%lld\n", "%lld\n", + reipl_block_nvme->hdr, + reipl_block_nvme->nvme.bootprog); static struct attribute *reipl_nvme_attrs[] = { &sys_reipl_nvme_fid_attr.attr, @@ -1038,8 +1058,9 @@ static const struct bin_attribute *const reipl_eckd_bin_attrs[] = { }; DEFINE_IPL_CCW_ATTR_RW(reipl_eckd, device, reipl_block_eckd->eckd); -DEFINE_IPL_ATTR_RW(reipl_eckd, bootprog, "%lld\n", "%lld\n", - reipl_block_eckd->eckd.bootprog); +DEFINE_IPL_ATTR_BOOTPROG_RW(reipl_eckd, bootprog, "%lld\n", "%lld\n", + reipl_block_eckd->hdr, + reipl_block_eckd->eckd.bootprog); static struct attribute *reipl_eckd_attrs[] = { &sys_reipl_eckd_device_attr.attr, @@ -1567,12 +1588,13 @@ DEFINE_IPL_ATTR_RW(dump_fcp, wwpn, "0x%016llx\n", "%llx\n", dump_block_fcp->fcp.wwpn); DEFINE_IPL_ATTR_RW(dump_fcp, lun, "0x%016llx\n", "%llx\n", dump_block_fcp->fcp.lun); -DEFINE_IPL_ATTR_RW(dump_fcp, bootprog, "%lld\n", "%lld\n", - dump_block_fcp->fcp.bootprog); DEFINE_IPL_ATTR_RW(dump_fcp, br_lba, "%lld\n", "%lld\n", dump_block_fcp->fcp.br_lba); DEFINE_IPL_ATTR_RW(dump_fcp, device, "0.0.%04llx\n", "0.0.%llx\n", dump_block_fcp->fcp.devno); +DEFINE_IPL_ATTR_BOOTPROG_RW(dump_fcp, bootprog, "%lld\n", "%lld\n", + dump_block_fcp->hdr, + dump_block_fcp->fcp.bootprog); DEFINE_IPL_ATTR_SCP_DATA_RW(dump_fcp, dump_block_fcp->hdr, dump_block_fcp->fcp, @@ -1604,10 +1626,11 @@ DEFINE_IPL_ATTR_RW(dump_nvme, fid, "0x%08llx\n", "%llx\n", dump_block_nvme->nvme.fid); DEFINE_IPL_ATTR_RW(dump_nvme, nsid, "0x%08llx\n", "%llx\n", dump_block_nvme->nvme.nsid); -DEFINE_IPL_ATTR_RW(dump_nvme, bootprog, "%lld\n", "%llx\n", - dump_block_nvme->nvme.bootprog); DEFINE_IPL_ATTR_RW(dump_nvme, br_lba, "%lld\n", "%llx\n", dump_block_nvme->nvme.br_lba); +DEFINE_IPL_ATTR_BOOTPROG_RW(dump_nvme, bootprog, "%lld\n", "%llx\n", + dump_block_nvme->hdr, + dump_block_nvme->nvme.bootprog); DEFINE_IPL_ATTR_SCP_DATA_RW(dump_nvme, dump_block_nvme->hdr, dump_block_nvme->nvme, @@ -1635,8 +1658,9 @@ static const struct attribute_group dump_nvme_attr_group = { /* ECKD dump device attributes */ DEFINE_IPL_CCW_ATTR_RW(dump_eckd, device, dump_block_eckd->eckd); -DEFINE_IPL_ATTR_RW(dump_eckd, bootprog, "%lld\n", "%llx\n", - dump_block_eckd->eckd.bootprog); +DEFINE_IPL_ATTR_BOOTPROG_RW(dump_eckd, bootprog, "%lld\n", "%llx\n", + dump_block_eckd->hdr, + dump_block_eckd->eckd.bootprog); IPL_ATTR_BR_CHR_SHOW_FN(dump, dump_block_eckd->eckd); IPL_ATTR_BR_CHR_STORE_FN(dump, dump_block_eckd->eckd); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index bdf9c7cb5685b..080e9285b3379 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -146,6 +146,10 @@ void noinstr do_io_irq(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); bool from_idle; + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); + irq_enter_rcu(); if (user_mode(regs)) { @@ -154,7 +158,6 @@ void noinstr do_io_irq(struct pt_regs *regs) current->thread.last_break = regs->last_break; } - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); @@ -182,6 +185,10 @@ void noinstr do_ext_irq(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); bool from_idle; + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); + irq_enter_rcu(); if (user_mode(regs)) { @@ -194,7 +201,6 @@ void noinstr do_ext_irq(struct pt_regs *regs) regs->int_parm = get_lowcore()->ext_params; regs->int_parm_long = get_lowcore()->ext_params2; - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index f432869f89213..4ddb09c4a0d64 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -842,7 +842,7 @@ static bool is_callchain_event(struct perf_event *event) u64 sample_type = event->attr.sample_type; return sample_type & (PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER | - PERF_SAMPLE_STACK_USER); + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_STACK_USER); } static int cpumsf_pmu_event_init(struct perf_event *event) @@ -1169,6 +1169,7 @@ static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt, static void hw_perf_event_update(struct perf_event *event, int flush_all) { unsigned long long event_overflow, sampl_overflow, num_sdb; + struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); struct hw_perf_event *hwc = &event->hw; union hws_trailer_header prev, new; struct hws_trailer_entry *te; @@ -1248,8 +1249,11 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) * are dropped. * Slightly increase the interval to avoid hitting this limit. */ - if (event_overflow) + if (event_overflow) { SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10); + if (SAMPL_RATE(hwc) > cpuhw->qsi.max_sampl_rate) + SAMPL_RATE(hwc) = cpuhw->qsi.max_sampl_rate; + } } static inline unsigned long aux_sdb_index(struct aux_buffer *aux, diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index da84c0dc6b7e0..70df4ca5d4436 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -697,6 +697,7 @@ static void __ref smp_get_core_info(struct sclp_core_info *info, int early) continue; info->core[info->configured].core_id = address >> smp_cpu_mt_shift; + info->core[info->configured].type = boot_core_type; info->configured++; } info->combined = info->configured; diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c index 4fee74553ca2d..bd7c02b3700d1 100644 --- a/arch/s390/kernel/syscall.c +++ b/arch/s390/kernel/syscall.c @@ -13,6 +13,7 @@ */ #include +#include #include #include #include @@ -121,8 +122,10 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET))) goto out; regs->gprs[2] = -ENOSYS; - if (likely(nr < NR_syscalls)) + if (likely(nr < NR_syscalls)) { + nr = array_index_nospec(nr, NR_syscalls); regs->gprs[2] = current->thread.sys_call_table[nr](regs); + } out: syscall_exit_to_user_mode(regs); } diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index d74d4c52ccd05..9289e9e535c70 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -212,6 +212,7 @@ SECTIONS /* Debugging sections. */ STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 234a0ba305108..122d30b104401 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -225,10 +225,6 @@ static u64 vtime_delta(void) return timer - lc->last_update_timer; } -/* - * Update process times based on virtual cpu times stored by entry.S - * to the lowcore fields user_timer, system_timer & steal_clock. - */ void vtime_account_kernel(struct task_struct *tsk) { struct lowcore *lc = get_lowcore(); @@ -238,27 +234,17 @@ void vtime_account_kernel(struct task_struct *tsk) lc->guest_timer += delta; else lc->system_timer += delta; - - virt_timer_forward(delta); } EXPORT_SYMBOL_GPL(vtime_account_kernel); void vtime_account_softirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->softirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->softirq_timer += vtime_delta(); } void vtime_account_hardirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->hardirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->hardirq_timer += vtime_delta(); } /* diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 16ba04062854a..a0162d03e16b9 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -64,7 +64,7 @@ #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ (KVM_MAX_VCPUS + LOCAL_IRQS)) -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_COUNTER(VM, inject_io), STATS_DESC_COUNTER(VM, inject_float_mchk), @@ -90,7 +90,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, exit_userspace), STATS_DESC_COUNTER(VCPU, exit_null), diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c index 1721b73b78036..81c0235c04666 100644 --- a/arch/s390/lib/xor.c +++ b/arch/s390/lib/xor.c @@ -28,8 +28,8 @@ static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1, " j 3f\n" "2: xc 0(1,%1),0(%2)\n" "3:" - : : "d" (bytes), "a" (p1), "a" (p2) - : "0", "cc", "memory"); + : "+d" (bytes), "+a" (p1), "+a" (p2) + : : "0", "cc", "memory"); } static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1, @@ -96,7 +96,6 @@ static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p5) { asm volatile( - " larl 1,2f\n" " aghi %0,-1\n" " jm 6f\n" " srlg 0,%0,8\n" diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e1ad05bfd28ad..d1f165048055b 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -436,10 +436,17 @@ void do_secure_storage_access(struct pt_regs *regs) folio = phys_to_folio(addr); if (unlikely(!folio_try_get(folio))) return; - rc = arch_make_folio_accessible(folio); + rc = uv_convert_from_secure(folio_to_phys(folio)); + if (!rc) + clear_bit(PG_arch_1, &folio->flags.f); folio_put(folio); + /* + * There are some valid fixup types for kernel + * accesses to donated secure memory. zeropad is one + * of them. + */ if (rc) - BUG(); + return handle_fault_error_nolock(regs, 0); } else { if (faulthandler_disabled()) return handle_fault_error_nolock(regs, 0); diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c index d4c3c36855e26..38a2d82cd88a7 100644 --- a/arch/s390/mm/gmap_helpers.c +++ b/arch/s390/mm/gmap_helpers.c @@ -47,6 +47,7 @@ static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr) { struct vm_area_struct *vma; + unsigned long pgstev; spinlock_t *ptl; pgste_t pgste; pte_t *ptep; @@ -65,9 +66,13 @@ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr) if (pte_swap(*ptep)) { preempt_disable(); pgste = pgste_get_lock(ptep); + pgstev = pgste_val(pgste); - ptep_zap_swap_entry(mm, pte_to_swp_entry(*ptep)); - pte_clear(mm, vmaddr, ptep); + if ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || + (pgstev & _PGSTE_GPS_ZERO)) { + ptep_zap_swap_entry(mm, pte_to_swp_entry(*ptep)); + pte_clear(mm, vmaddr, ptep); + } pgste_set_unlock(ptep, pgste); preempt_enable(); diff --git a/arch/s390/mm/pfault.c b/arch/s390/mm/pfault.c index e6175d75e4b07..ea2a224459b6b 100644 --- a/arch/s390/mm/pfault.c +++ b/arch/s390/mm/pfault.c @@ -62,7 +62,7 @@ int __pfault_init(void) "0: nopr %%r7\n" EX_TABLE(0b, 0b) : [rc] "+d" (rc) - : [refbk] "a" (&pfault_init_refbk), "m" (pfault_init_refbk) + : [refbk] "a" (virt_to_phys(&pfault_init_refbk)), "m" (pfault_init_refbk) : "cc"); return rc; } @@ -84,7 +84,7 @@ void __pfault_fini(void) "0: nopr %%r7\n" EX_TABLE(0b, 0b) : - : [refbk] "a" (&pfault_fini_refbk), "m" (pfault_fini_refbk) + : [refbk] "a" (virt_to_phys(&pfault_fini_refbk)), "m" (pfault_fini_refbk) : "cc"); } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index c82c577db2bcd..c541019d91356 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -232,24 +232,33 @@ int zpci_fmb_disable_device(struct zpci_dev *zdev) static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data; - int rc; + + if (!zdev_enabled(zdev)) + goto out_err; rc = __zpci_load(&data, req, offset); - if (!rc) { - data = le64_to_cpu((__force __le64) data); - data >>= (8 - len) * 8; - *val = (u32) data; - } else - *val = 0xffffffff; + if (rc) + goto out_err; + data = le64_to_cpu((__force __le64)data); + data >>= (8 - len) * 8; + *val = (u32)data; + return 0; + +out_err: + PCI_SET_ERROR_RESPONSE(val); return rc; } static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data = val; - int rc; + + if (!zdev_enabled(zdev)) + return rc; data <<= (8 - len) * 8; data = (__force u64) cpu_to_le64(data); diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index bd39b36e7bd68..cfb73bbcdfe85 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -22,6 +22,7 @@ KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING KBUILD_CFLAGS += -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_CFLAGS += $(call cc-option,-fno-PIE) +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) KBUILD_AFLAGS += -D__DISABLE_EXPORTS diff --git a/arch/sh/drivers/platform_early.c b/arch/sh/drivers/platform_early.c index 143747c45206f..48ddbc547bd9a 100644 --- a/arch/sh/drivers/platform_early.c +++ b/arch/sh/drivers/platform_early.c @@ -26,10 +26,6 @@ static int platform_match(struct device *dev, struct device_driver *drv) struct platform_device *pdev = to_platform_device(dev); struct platform_driver *pdrv = to_platform_driver(drv); - /* When driver_override is set, only bind to the matching driver */ - if (pdev->driver_override) - return !strcmp(pdev->driver_override, drv->name); - /* Then try to match against the id table */ if (pdrv->id_table) return platform_match_id(pdrv->id_table, pdev) != NULL; diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S index 008c30289eaa6..169c63fb3c1dc 100644 --- a/arch/sh/kernel/vmlinux.lds.S +++ b/arch/sh/kernel/vmlinux.lds.S @@ -89,6 +89,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/sparc/include/uapi/asm/ioctls.h b/arch/sparc/include/uapi/asm/ioctls.h index 7fd2f5873c9e7..a8bbdf9877a41 100644 --- a/arch/sparc/include/uapi/asm/ioctls.h +++ b/arch/sparc/include/uapi/asm/ioctls.h @@ -5,10 +5,10 @@ #include /* Big T */ -#define TCGETA _IOR('T', 1, struct termio) -#define TCSETA _IOW('T', 2, struct termio) -#define TCSETAW _IOW('T', 3, struct termio) -#define TCSETAF _IOW('T', 4, struct termio) +#define TCGETA 0x40125401 /* _IOR('T', 1, struct termio) */ +#define TCSETA 0x80125402 /* _IOW('T', 2, struct termio) */ +#define TCSETAW 0x80125403 /* _IOW('T', 3, struct termio) */ +#define TCSETAF 0x80125404 /* _IOW('T', 4, struct termio) */ #define TCSBRK _IO('T', 5) #define TCXONC _IO('T', 6) #define TCFLSH _IO('T', 7) diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index a9448088e762e..b290107170e94 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -181,6 +181,28 @@ static int __init ofpci_debug(char *str) __setup("ofpci_debug=", ofpci_debug); +static void of_fixup_pci_pref(struct pci_dev *dev, int index, + struct resource *res) +{ + struct pci_bus_region region; + + if (!(res->flags & IORESOURCE_MEM_64)) + return; + + if (!resource_size(res)) + return; + + pcibios_resource_to_bus(dev->bus, ®ion, res); + if (region.end <= ~((u32)0)) + return; + + if (!(res->flags & IORESOURCE_PREFETCH)) { + res->flags |= IORESOURCE_PREFETCH; + pci_info(dev, "reg 0x%x: fixup: pref added to 64-bit resource\n", + index); + } +} + static unsigned long pci_parse_of_flags(u32 addr0) { unsigned long flags = 0; @@ -244,6 +266,7 @@ static void pci_parse_of_addrs(struct platform_device *op, res->end = op_res->end; res->flags = flags; res->name = pci_name(dev); + of_fixup_pci_pref(dev, i, res); pci_info(dev, "reg 0x%x: %pR\n", i, res); } diff --git a/arch/sparc/kernel/process.c b/arch/sparc/kernel/process.c index 0442ab00518d3..7d69877511fac 100644 --- a/arch/sparc/kernel/process.c +++ b/arch/sparc/kernel/process.c @@ -17,14 +17,18 @@ asmlinkage long sparc_fork(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; + unsigned long orig_i1; long ret; struct kernel_clone_args args = { .exit_signal = SIGCHLD, - /* Reuse the parent's stack for the child. */ - .stack = regs->u_regs[UREG_FP], }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + ret = kernel_clone(&args); /* If we get an error and potentially restart the system @@ -40,16 +44,19 @@ asmlinkage long sparc_fork(struct pt_regs *regs) asmlinkage long sparc_vfork(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; + unsigned long orig_i1; long ret; - struct kernel_clone_args args = { .flags = CLONE_VFORK | CLONE_VM, .exit_signal = SIGCHLD, - /* Reuse the parent's stack for the child. */ - .stack = regs->u_regs[UREG_FP], }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + ret = kernel_clone(&args); /* If we get an error and potentially restart the system @@ -65,15 +72,18 @@ asmlinkage long sparc_vfork(struct pt_regs *regs) asmlinkage long sparc_clone(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; - unsigned int flags = lower_32_bits(regs->u_regs[UREG_I0]); + unsigned long orig_i1; + unsigned int flags; long ret; + struct kernel_clone_args args = {0}; - struct kernel_clone_args args = { - .flags = (flags & ~CSIGNAL), - .exit_signal = (flags & CSIGNAL), - .tls = regs->u_regs[UREG_I3], - }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + flags = lower_32_bits(regs->u_regs[UREG_I0]); + args.flags = (flags & ~CSIGNAL); + args.exit_signal = (flags & CSIGNAL); + args.tls = regs->u_regs[UREG_I3]; #ifdef CONFIG_COMPAT if (test_thread_flag(TIF_32BIT)) { diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index f1b86eb303404..7ea510d9b42f2 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -191,6 +191,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/Kconfig b/arch/um/Kconfig index 49781bee79058..93ed850d508ed 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -5,6 +5,7 @@ menu "UML-specific options" config UML bool default y + select ARCH_DISABLE_KASAN_INLINE if STATIC_LINK select ARCH_NEEDS_DEFER_KASAN if STATIC_LINK select ARCH_WANTS_DYNAMIC_TASK_STRUCT select ARCH_HAS_CACHE_LINE_SIZE diff --git a/arch/um/Makefile b/arch/um/Makefile index 7be0143b5ba35..721b652ffb658 100644 --- a/arch/um/Makefile +++ b/arch/um/Makefile @@ -46,19 +46,17 @@ ARCH_INCLUDE := -I$(srctree)/$(SHARED_HEADERS) ARCH_INCLUDE += -I$(srctree)/$(HOST_DIR)/um/shared KBUILD_CPPFLAGS += -I$(srctree)/$(HOST_DIR)/um -# -Dvmap=kernel_vmap prevents anything from referencing the libpcap.o symbol so -# named - it's a common symbol in libpcap, so we get a binary which crashes. -# -# Same things for in6addr_loopback and mktime - found in libc. For these two we -# only get link-time error, luckily. +# -Dstrrchr=kernel_strrchr (as well as the various in6addr symbols) prevents +# anything from referencing +# libc symbols with the same name, which can cause a linker error. # # -Dlongjmp=kernel_longjmp prevents anything from referencing the libpthread.a # embedded copy of longjmp, same thing for setjmp. # -# These apply to USER_CFLAGS to. +# These apply to USER_CFLAGS too. KBUILD_CFLAGS += $(CFLAGS) $(CFLAGS-y) -D__arch_um__ \ - $(ARCH_INCLUDE) $(MODE_INCLUDE) -Dvmap=kernel_vmap \ + $(ARCH_INCLUDE) $(MODE_INCLUDE) \ -Dlongjmp=kernel_longjmp -Dsetjmp=kernel_setjmp \ -Din6addr_loopback=kernel_in6addr_loopback \ -Din6addr_any=kernel_in6addr_any -Dstrrchr=kernel_strrchr \ diff --git a/arch/um/include/asm/kasan.h b/arch/um/include/asm/kasan.h index b54a4e937fd12..81bcdc0f962e6 100644 --- a/arch/um/include/asm/kasan.h +++ b/arch/um/include/asm/kasan.h @@ -24,10 +24,6 @@ #ifdef CONFIG_KASAN void kasan_init(void); - -#if defined(CONFIG_STATIC_LINK) && defined(CONFIG_KASAN_INLINE) -#error UML does not work in KASAN_INLINE mode with STATIC_LINK enabled! -#endif #else static inline void kasan_init(void) { } #endif /* CONFIG_KASAN */ diff --git a/arch/um/kernel/dyn.lds.S b/arch/um/kernel/dyn.lds.S index a36b7918a011a..ad3cefeff2acb 100644 --- a/arch/um/kernel/dyn.lds.S +++ b/arch/um/kernel/dyn.lds.S @@ -172,6 +172,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 9c9c66dc45f05..13d461712c997 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -43,7 +43,9 @@ * cares about its entry, so it's OK if another processor is modifying its * entry. */ -struct task_struct *cpu_tasks[NR_CPUS]; +struct task_struct *cpu_tasks[NR_CPUS] = { + [0 ... NR_CPUS - 1] = &init_task, +}; EXPORT_SYMBOL(cpu_tasks); void free_stack(unsigned long stack, int order) diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index cfbbbf8500c34..ed2f67848a50e 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -239,8 +239,6 @@ static struct notifier_block panic_exit_notifier = { void uml_finishsetup(void) { - cpu_tasks[0] = &init_task; - atomic_notifier_chain_register(&panic_notifier_list, &panic_exit_notifier); diff --git a/arch/um/kernel/uml.lds.S b/arch/um/kernel/uml.lds.S index a409d4b66114f..30aa24348d60c 100644 --- a/arch/um/kernel/uml.lds.S +++ b/arch/um/kernel/uml.lds.S @@ -113,6 +113,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/os-Linux/signal.c b/arch/um/os-Linux/signal.c index 11f07f4982700..bebeb6b4bee8b 100644 --- a/arch/um/os-Linux/signal.c +++ b/arch/um/os-Linux/signal.c @@ -35,7 +35,6 @@ void (*sig_info[NSIG])(int, struct siginfo *, struct uml_pt_regs *, void *mc) = static void sig_handler_common(int sig, struct siginfo *si, mcontext_t *mc) { struct uml_pt_regs r; - int save_errno = errno; r.is_user = 0; if (sig == SIGSEGV) { @@ -49,8 +48,6 @@ static void sig_handler_common(int sig, struct siginfo *si, mcontext_t *mc) unblock_signals_trace(); (*sig_info[sig])(sig, si, &r, mc); - - errno = save_errno; } /* @@ -201,8 +198,11 @@ static void hard_handler(int sig, siginfo_t *si, void *p) { ucontext_t *uc = p; mcontext_t *mc = &uc->uc_mcontext; + int save_errno = errno; (*handlers[sig])(sig, (struct siginfo *)si, mc); + + errno = save_errno; } void set_handler(int sig) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index fa3b616af03a2..ee41af778a9dc 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -279,6 +279,7 @@ config X86 select HAVE_PCI select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select ASYNC_KERNEL_PGTABLE_FREE if IOMMU_SVA select MMU_GATHER_RCU_TABLE_FREE select MMU_GATHER_MERGE_VMAS select HAVE_POSIX_CPU_TIMERS_TASK_WORK @@ -297,6 +298,7 @@ config X86 select HAVE_SYSCALL_TRACEPOINTS select HAVE_UACCESS_VALIDATION if HAVE_OBJTOOL select HAVE_UNSTABLE_SCHED_CLOCK + select HAVE_UNWIND_USER_FP if X86_64 select HAVE_USER_RETURN_NOTIFIER select HAVE_GENERIC_VDSO select VDSO_GETRANDOM if X86_64 diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 74657589264df..2013840d6318f 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -110,6 +110,7 @@ vmlinux-objs-$(CONFIG_EFI_SBAT) += $(obj)/sbat.o ifdef CONFIG_EFI_SBAT $(obj)/sbat.o: $(CONFIG_EFI_SBAT_FILE) +AFLAGS_sbat.o += -I $(srctree) endif $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index bdd26050dff77..0e89e197e1126 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "../string.h" #include "efi.h" @@ -168,9 +169,10 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) * For 4- to 5-level paging transition, set up current CR3 as * the first and the only entry in a new top-level page table. */ - *trampoline_32bit = __native_read_cr3() | _PAGE_TABLE_NOENC; + *trampoline_32bit = native_read_cr3_pa() | _PAGE_TABLE_NOENC; } else { - unsigned long src; + u64 *new_cr3; + pgd_t *pgdp; /* * For 5- to 4-level paging transition, copy page table pointed @@ -180,8 +182,9 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) * We cannot just point to the page table from trampoline as it * may be above 4G. */ - src = *(unsigned long *)__native_read_cr3() & PAGE_MASK; - memcpy(trampoline_32bit, (void *)src, PAGE_SIZE); + pgdp = (pgd_t *)native_read_cr3_pa(); + new_cr3 = (u64 *)(native_pgd_val(pgdp[0]) & PTE_PFN_MASK); + memcpy(trampoline_32bit, new_cr3, PAGE_SIZE); } toggle_la57(trampoline_32bit); diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 6e5c32a53d034..b688303666052 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -27,17 +27,17 @@ #include "sev.h" static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); -struct ghcb *boot_ghcb; +struct ghcb *boot_ghcb __section(".data"); #undef __init #define __init #define __BOOT_COMPRESSED -u8 snp_vmpl; -u16 ghcb_version; +u8 snp_vmpl __section(".data"); +u16 ghcb_version __section(".data"); -u64 boot_svsm_caa_pa; +u64 boot_svsm_caa_pa __section(".data"); /* Include code for early handlers */ #include "../../boot/startup/sev-shared.c" @@ -187,6 +187,7 @@ bool sev_es_check_ghcb_fault(unsigned long address) MSR_AMD64_SNP_RESERVED_BIT13 | \ MSR_AMD64_SNP_RESERVED_BIT15 | \ MSR_AMD64_SNP_SECURE_AVIC | \ + MSR_AMD64_SNP_RESERVED_BITS19_22 | \ MSR_AMD64_SNP_RESERVED_MASK) #ifdef CONFIG_AMD_SECURE_AVIC diff --git a/arch/x86/boot/compressed/vmlinux.lds.S b/arch/x86/boot/compressed/vmlinux.lds.S index 587ce3e7c5048..e0b152715d9c6 100644 --- a/arch/x86/boot/compressed/vmlinux.lds.S +++ b/arch/x86/boot/compressed/vmlinux.lds.S @@ -88,7 +88,7 @@ SECTIONS /DISCARD/ : { *(.dynamic) *(.dynsym) *(.dynstr) *(.dynbss) *(.hash) *(.gnu.hash) - *(.note.*) + *(.note.*) *(.modinfo) } .got.plt (INFO) : { diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c index 4e22ffd735168..e52e886ad313d 100644 --- a/arch/x86/boot/startup/sev-shared.c +++ b/arch/x86/boot/startup/sev-shared.c @@ -31,7 +31,7 @@ static u32 cpuid_std_range_max __ro_after_init; static u32 cpuid_hyp_range_max __ro_after_init; static u32 cpuid_ext_range_max __ro_after_init; -bool sev_snp_needs_sfw; +bool sev_snp_needs_sfw __section(".data"); void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile index 3b8ae214a6a64..b2e9ec2f69014 100644 --- a/arch/x86/coco/sev/Makefile +++ b/arch/x86/coco/sev/Makefile @@ -8,3 +8,5 @@ UBSAN_SANITIZE_noinstr.o := n # GCC may fail to respect __no_sanitize_address or __no_kcsan when inlining KASAN_SANITIZE_noinstr.o := n KCSAN_SANITIZE_noinstr.o := n + +GCOV_PROFILE_noinstr.o := n diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 9ae3b11754e65..d20e9cc065a87 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -122,6 +122,7 @@ static const char * const sev_status_feat_names[] = { [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt", [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC", + [MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT] = "IBPBOnEntry", }; /* @@ -2008,8 +2009,7 @@ void snp_msg_free(struct snp_msg_desc *mdesc) free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); iounmap((__force void __iomem *)mdesc->secrets); - memset(mdesc, 0, sizeof(*mdesc)); - kfree(mdesc); + kfree_sensitive(mdesc); } EXPORT_SYMBOL_GPL(snp_msg_free); diff --git a/arch/x86/coco/sev/noinstr.c b/arch/x86/coco/sev/noinstr.c index b527eafb63123..422ce72f51d58 100644 --- a/arch/x86/coco/sev/noinstr.c +++ b/arch/x86/coco/sev/noinstr.c @@ -120,6 +120,9 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) WARN_ON(!irqs_disabled()); + if (!sev_cfg.ghcbs_initialized) + return boot_ghcb; + data = this_cpu_read(runtime_data); ghcb = &data->ghcb_page; @@ -163,6 +166,9 @@ noinstr void __sev_put_ghcb(struct ghcb_state *state) WARN_ON(!irqs_disabled()); + if (!sev_cfg.ghcbs_initialized) + return; + data = this_cpu_read(runtime_data); ghcb = &data->ghcb_page; diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index f004a4dc74c2d..9f50f0c1c00f5 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c @@ -159,8 +159,6 @@ void __init fred_complete_exception_setup(void) static noinstr void fred_extint(struct pt_regs *regs) { unsigned int vector = regs->fred_ss.vector; - unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR, - NR_SYSTEM_VECTORS); if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) return; @@ -169,7 +167,8 @@ static noinstr void fred_extint(struct pt_regs *regs) irqentry_state_t state = irqentry_enter(regs); instrumentation_begin(); - sysvec_table[index](regs); + sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR, + NR_SYSTEM_VECTORS)](regs); instrumentation_end(); irqentry_exit(regs, state); } else { @@ -177,6 +176,16 @@ static noinstr void fred_extint(struct pt_regs *regs) } } +#ifdef CONFIG_AMD_MEM_ENCRYPT +noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) +{ + if (user_mode(regs)) + return user_exc_vmm_communication(regs, error_code); + else + return kernel_exc_vmm_communication(regs, error_code); +} +#endif + static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) { /* Optimize for #PF. That's the only exception which matters performance wise */ @@ -207,6 +216,10 @@ static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) #ifdef CONFIG_X86_CET case X86_TRAP_CP: return exc_control_protection(regs, error_code); #endif +#ifdef CONFIG_AMD_MEM_ENCRYPT + case X86_TRAP_VC: return exc_vmm_communication(regs, error_code); +#endif + default: return fred_bad_type(regs, error_code); } diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index b20661b8621d1..8868f5f5379ba 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -763,7 +763,12 @@ static void amd_pmu_enable_all(int added) if (!test_bit(idx, cpuc->active_mask)) continue; - amd_pmu_enable_event(cpuc->events[idx]); + /* + * FIXME: cpuc->events[idx] can become NULL in a subtle race + * condition with NMI->throttle->x86_pmu_stop(). + */ + if (cpuc->events[idx]) + amd_pmu_enable_event(cpuc->events[idx]); } } diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index e8b6af199c738..9293ce50574da 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -656,14 +656,11 @@ static int amd_uncore_df_event_init(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; int ret = amd_uncore_event_init(event); - if (ret || pmu_version < 2) - return ret; - hwc->config = event->attr.config & (pmu_version >= 2 ? AMD64_PERFMON_V2_RAW_EVENT_MASK_NB : AMD64_RAW_EVENT_MASK_NB); - return 0; + return ret; } static int amd_uncore_df_add(struct perf_event *event, int flags) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index fa6c47b509897..8a0cd2ebb60db 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1344,6 +1344,7 @@ static void x86_pmu_enable(struct pmu *pmu) hwc->state |= PERF_HES_ARCH; x86_pmu_stop(event, PERF_EF_UPDATE); + cpuc->events[hwc->idx] = NULL; } /* @@ -1358,8 +1359,12 @@ static void x86_pmu_enable(struct pmu *pmu) else if (i < n_running) continue; - if (hwc->state & PERF_HES_ARCH) + cpuc->events[hwc->idx] = event; + + if (hwc->state & PERF_HES_ARCH) { + static_call(x86_pmu_set_period)(event); continue; + } /* * if cpuc->enabled = 0, then no wrmsr as @@ -1531,7 +1536,6 @@ static void x86_pmu_start(struct perf_event *event, int flags) event->hw.state = 0; - cpuc->events[idx] = event; __set_bit(idx, cpuc->active_mask); static_call(x86_pmu_enable)(event); perf_event_update_userpage(event); @@ -1610,7 +1614,6 @@ void x86_pmu_stop(struct perf_event *event, int flags) if (test_bit(hwc->idx, cpuc->active_mask)) { static_call(x86_pmu_disable)(event); __clear_bit(hwc->idx, cpuc->active_mask); - cpuc->events[hwc->idx] = NULL; WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); hwc->state |= PERF_HES_STOPPED; } @@ -1648,6 +1651,7 @@ static void x86_pmu_del(struct perf_event *event, int flags) * Not a TXN, therefore cleanup properly. */ x86_pmu_stop(event, PERF_EF_UPDATE); + cpuc->events[event->hw.idx] = NULL; for (i = 0; i < cpuc->n_events; i++) { if (event == cpuc->event_list[i]) @@ -2845,46 +2849,6 @@ static unsigned long get_segment_base(unsigned int segment) return get_desc_base(desc); } -#ifdef CONFIG_UPROBES -/* - * Heuristic-based check if uprobe is installed at the function entry. - * - * Under assumption of user code being compiled with frame pointers, - * `push %rbp/%ebp` is a good indicator that we indeed are. - * - * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern. - * If we get this wrong, captured stack trace might have one extra bogus - * entry, but the rest of stack trace will still be meaningful. - */ -static bool is_uprobe_at_func_entry(struct pt_regs *regs) -{ - struct arch_uprobe *auprobe; - - if (!current->utask) - return false; - - auprobe = current->utask->auprobe; - if (!auprobe) - return false; - - /* push %rbp/%ebp */ - if (auprobe->insn[0] == 0x55) - return true; - - /* endbr64 (64-bit only) */ - if (user_64bit_mode(regs) && is_endbr((u32 *)auprobe->insn)) - return true; - - return false; -} - -#else -static bool is_uprobe_at_func_entry(struct pt_regs *regs) -{ - return false; -} -#endif /* CONFIG_UPROBES */ - #ifdef CONFIG_IA32_EMULATION #include @@ -3100,8 +3064,8 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) cap->version = x86_pmu.version; cap->num_counters_gp = x86_pmu_num_counters(NULL); cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); - cap->bit_width_gp = x86_pmu.cntval_bits; - cap->bit_width_fixed = x86_pmu.cntval_bits; + cap->bit_width_gp = cap->num_counters_gp ? x86_pmu.cntval_bits : 0; + cap->bit_width_fixed = cap->num_counters_fixed ? x86_pmu.cntval_bits : 0; cap->events_mask = (unsigned int)x86_pmu.events_maskl; cap->events_mask_len = x86_pmu.events_mask_len; cap->pebs_ept = x86_pmu.pebs_ept; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index fe65be0b9d9c4..3046058e7e238 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3249,6 +3249,9 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) if (!test_bit(bit, cpuc->active_mask)) continue; + /* Event may have already been cleared: */ + if (!event) + continue; /* * There may be unprocessed PEBS records in the PEBS buffer, @@ -4029,7 +4032,9 @@ static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event) if (!event->attr.exclude_kernel) flags &= ~PERF_SAMPLE_REGS_USER; if (event->attr.sample_regs_user & ~PEBS_GP_REGS) - flags &= ~(PERF_SAMPLE_REGS_USER | PERF_SAMPLE_REGS_INTR); + flags &= ~PERF_SAMPLE_REGS_USER; + if (event->attr.sample_regs_intr & ~PEBS_GP_REGS) + flags &= ~PERF_SAMPLE_REGS_INTR; return flags; } @@ -4219,6 +4224,19 @@ static inline void intel_pmu_set_acr_caused_constr(struct perf_event *event, event->hw.dyn_constraint &= hybrid(event->pmu, acr_cause_mask64); } +static inline int intel_set_branch_counter_constr(struct perf_event *event, + int *num) +{ + if (branch_sample_call_stack(event)) + return -EINVAL; + if (branch_sample_counters(event)) { + (*num)++; + event->hw.dyn_constraint &= x86_pmu.lbr_counters; + } + + return 0; +} + static int intel_pmu_hw_config(struct perf_event *event) { int ret = x86_pmu_hw_config(event); @@ -4278,21 +4296,19 @@ static int intel_pmu_hw_config(struct perf_event *event) * group, which requires the extra space to store the counters. */ leader = event->group_leader; - if (branch_sample_call_stack(leader)) + if (intel_set_branch_counter_constr(leader, &num)) return -EINVAL; - if (branch_sample_counters(leader)) { - num++; - leader->hw.dyn_constraint &= x86_pmu.lbr_counters; - } leader->hw.flags |= PERF_X86_EVENT_BRANCH_COUNTERS; for_each_sibling_event(sibling, leader) { - if (branch_sample_call_stack(sibling)) + if (intel_set_branch_counter_constr(sibling, &num)) + return -EINVAL; + } + + /* event isn't installed as a sibling yet. */ + if (event != leader) { + if (intel_set_branch_counter_constr(event, &num)) return -EINVAL; - if (branch_sample_counters(sibling)) { - num++; - sibling->hw.dyn_constraint &= x86_pmu.lbr_counters; - } } if (num > fls(x86_pmu.lbr_counters)) @@ -4425,8 +4441,10 @@ static int intel_pmu_hw_config(struct perf_event *event) intel_pmu_set_acr_caused_constr(leader, idx++, cause_mask); if (leader->nr_siblings) { - for_each_sibling_event(sibling, leader) - intel_pmu_set_acr_caused_constr(sibling, idx++, cause_mask); + for_each_sibling_event(sibling, leader) { + if (is_x86_event(sibling)) + intel_pmu_set_acr_caused_constr(sibling, idx++, cause_mask); + } } if (leader != event) @@ -7024,6 +7042,7 @@ __init int intel_pmu_init(void) case INTEL_ATOM_SILVERMONT_D: case INTEL_ATOM_SILVERMONT_MID: case INTEL_ATOM_AIRMONT: + case INTEL_ATOM_AIRMONT_NP: case INTEL_ATOM_SILVERMONT_MID2: memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index ec753e39b0077..15db12647b2f8 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -70,7 +70,7 @@ * perf code: 0x01 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL, * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL, - * ADL,RPL,MTL,ARL,LNL + * ADL,RPL,MTL,ARL * Scope: Package (physical package) * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. * perf code: 0x02 @@ -522,7 +522,6 @@ static const struct cstate_model lnl_cstates __initconst = { BIT(PERF_CSTATE_CORE_C7_RES), .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | - BIT(PERF_CSTATE_PKG_C3_RES) | BIT(PERF_CSTATE_PKG_C6_RES) | BIT(PERF_CSTATE_PKG_C10_RES), }; @@ -599,6 +598,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &slm_cstates), X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &slm_cstates), X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &slm_cstates), + X86_MATCH_VFM(INTEL_ATOM_AIRMONT_NP, &slm_cstates), X86_MATCH_VFM(INTEL_BROADWELL, &snb_cstates), X86_MATCH_VFM(INTEL_BROADWELL_D, &snb_cstates), diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 7d57ce706feb1..c5adbe4409047 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -383,7 +383,7 @@ static bool intel_uncore_has_discovery_tables_pci(int *ignore) (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP; die = get_device_die_id(dev); - if (die < 0) + if ((die < 0) || (die >= uncore_max_dies())) continue; parse_discovery_table(dev, die, bar_offset, &parsed, ignore); diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index e1f370b8d065f..a338ee01bb242 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -6610,6 +6610,32 @@ static struct intel_uncore_type gnr_uncore_ubox = { .attr_update = uncore_alias_groups, }; +static struct uncore_event_desc gnr_uncore_imc_events[] = { + INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x01,umask=0x00"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0, "event=0x05,umask=0xcf"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1, "event=0x06,umask=0xcf"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0, "event=0x05,umask=0xf0"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1, "event=0x06,umask=0xf0"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1.unit, "MiB"), + { /* end: all zeroes */ }, +}; + +static struct intel_uncore_type gnr_uncore_imc = { + SPR_UNCORE_MMIO_COMMON_FORMAT(), + .name = "imc", + .fixed_ctr_bits = 48, + .fixed_ctr = SNR_IMC_MMIO_PMON_FIXED_CTR, + .fixed_ctl = SNR_IMC_MMIO_PMON_FIXED_CTL, + .event_descs = gnr_uncore_imc_events, +}; + static struct intel_uncore_type gnr_uncore_pciex8 = { SPR_UNCORE_PCI_COMMON_FORMAT(), .name = "pciex8", @@ -6657,7 +6683,7 @@ static struct intel_uncore_type *gnr_uncores[UNCORE_GNR_NUM_UNCORE_TYPES] = { NULL, &spr_uncore_pcu, &gnr_uncore_ubox, - &spr_uncore_imc, + &gnr_uncore_imc, NULL, &gnr_uncore_upi, NULL, diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c index 7f5007a4752a1..8052596b85036 100644 --- a/arch/x86/events/msr.c +++ b/arch/x86/events/msr.c @@ -78,6 +78,7 @@ static bool test_intel(int idx, void *data) case INTEL_ATOM_SILVERMONT: case INTEL_ATOM_SILVERMONT_D: case INTEL_ATOM_AIRMONT: + case INTEL_ATOM_AIRMONT_NP: case INTEL_ATOM_GOLDMONT: case INTEL_ATOM_GOLDMONT_D: diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 2b969386dcddc..493e6ba51e06d 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1558,13 +1558,22 @@ static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period struct hw_perf_event *hwc = &event->hw; unsigned int hw_event, bts_event; - if (event->attr.freq) + /* + * Only use BTS for fixed rate period==1 events. + */ + if (event->attr.freq || period != 1) + return false; + + /* + * BTS doesn't virtualize. + */ + if (event->attr.exclude_host) return false; hw_event = hwc->config & INTEL_ARCH_EVENT_MASK; bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS); - return hw_event == bts_event && period == 1; + return hw_event == bts_event; } static inline bool intel_pmu_has_bts(struct perf_event *event) diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c index 042e8712d8dea..8aafccf7a52c7 100644 --- a/arch/x86/hyperv/hv_vtl.c +++ b/arch/x86/hyperv/hv_vtl.c @@ -105,7 +105,7 @@ static void hv_vtl_ap_entry(void) static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) { - u64 status; + u64 status, rsp, rip; int ret = 0; struct hv_enable_vp_vtl *input; unsigned long irq_flags; @@ -118,9 +118,11 @@ static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) struct desc_struct *gdt; struct task_struct *idle = idle_thread_get(cpu); - u64 rsp = (unsigned long)idle->thread.sp; + if (IS_ERR(idle)) + return PTR_ERR(idle); - u64 rip = (u64)&hv_vtl_ap_entry; + rsp = (unsigned long)idle->thread.sp; + rip = (u64)&hv_vtl_ap_entry; native_store_gdt(&gdt_ptr); store_idt(&idt_ptr); diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index 880ca15073ed1..99017bb1031e9 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -10,7 +10,7 @@ /* * Despite that some emulators terminate on UD2, we use it for WARN(). */ -#define ASM_UD2 _ASM_BYTES(0x0f, 0x0b) +#define ASM_UD2 __ASM_FORM(ud2) #define INSN_UD2 0x0b0f #define LEN_UD2 2 diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index c40b9ebc1fb40..ab3fbbd947ed9 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -111,6 +111,12 @@ extern bhi_thunk __bhi_args_end[]; struct pt_regs; +#ifdef CONFIG_CALL_PADDING +#define CFI_OFFSET (CONFIG_FUNCTION_PADDING_CFI+5) +#else +#define CFI_OFFSET 5 +#endif + #ifdef CONFIG_CFI enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); #define __bpfcall @@ -119,11 +125,9 @@ static inline int cfi_get_offset(void) { switch (cfi_mode) { case CFI_FINEIBT: - return 16; + return /* fineibt_prefix_size */ 16; case CFI_KCFI: - if (IS_ENABLED(CONFIG_CALL_PADDING)) - return 16; - return 5; + return CFI_OFFSET; default: return 0; } diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index f227a70ac91f0..51b4cdbea061a 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -138,7 +138,7 @@ extern void __init efi_apply_memmap_quirks(void); extern int __init efi_reuse_config(u64 tables, int nr_tables); extern void efi_delete_dummy_variable(void); extern void efi_crash_gracefully_on_page_fault(unsigned long phys_addr); -extern void efi_free_boot_services(void); +extern void efi_unmap_boot_services(void); void arch_efi_call_virt_setup(void); void arch_efi_call_virt_teardown(void); diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index b08c95872eed9..c56e1e63b8932 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -57,7 +57,7 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs) } #define arch_ftrace_partial_regs(regs) do { \ - regs->flags &= ~X86_EFLAGS_FIXED; \ + regs->flags |= X86_EFLAGS_FIXED; \ regs->cs = __KERNEL_CS; \ } while (0) diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h index 5a0d42464d442..4e55d17558465 100644 --- a/arch/x86/include/asm/irq_remapping.h +++ b/arch/x86/include/asm/irq_remapping.h @@ -87,4 +87,11 @@ static inline void panic_if_irq_remap(const char *msg) } #endif /* CONFIG_IRQ_REMAP */ + +#ifdef CONFIG_X86_POSTED_MSI +void intel_ack_posted_msi_irq(struct irq_data *irqd); +#else +#define intel_ack_posted_msi_irq NULL +#endif + #endif /* __X86_IRQ_REMAPPING_H */ diff --git a/arch/x86/include/asm/kfence.h b/arch/x86/include/asm/kfence.h index ff5c7134a37aa..dfd5c74ba41a2 100644 --- a/arch/x86/include/asm/kfence.h +++ b/arch/x86/include/asm/kfence.h @@ -42,10 +42,35 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) { unsigned int level; pte_t *pte = lookup_address(addr, &level); + pteval_t val, new; if (WARN_ON(!pte || level != PG_LEVEL_4K)) return false; + val = pte_val(*pte); + + /* + * protect requires making the page not-present. If the PTE is + * already in the right state, there's nothing to do. + */ + if (protect != !!(val & _PAGE_PRESENT)) + return true; + + /* + * Otherwise, flip the Present bit, taking care to avoid writing an + * L1TF-vulnerable PTE (not present, without the high address bits + * set). + */ + new = val ^ _PAGE_PRESENT; + set_pte(pte, __pte(flip_protnone_guard(val, new, PTE_PFN_MASK))); + + /* + * If the page was protected (non-present) and we're making it + * present, there is no need to flush the TLB at all. + */ + if (!protect) + return true; + /* * We need to avoid IPIs, as we may get KFENCE allocations or faults * with interrupts disabled. Therefore, the below is best-effort, and @@ -53,11 +78,6 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) * lazy fault handling takes care of faults after the page is PRESENT. */ - if (protect) - set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT)); - else - set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT)); - /* * Flush this CPU's TLB, assuming whoever did the allocation/free is * likely to continue running on this CPU. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 48598d017d6f3..03866683ac336 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1229,6 +1229,12 @@ enum kvm_irqchip_mode { KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ }; +enum kvm_suppress_eoi_broadcast_mode { + KVM_SUPPRESS_EOI_BROADCAST_QUIRKED, /* Legacy behavior */ + KVM_SUPPRESS_EOI_BROADCAST_ENABLED, /* Enable Suppress EOI broadcast */ + KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */ +}; + struct kvm_x86_msr_filter { u8 count; bool default_allow:1; @@ -1480,6 +1486,7 @@ struct kvm_arch { bool x2apic_format; bool x2apic_broadcast_quirk_disabled; + enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode; bool has_mapped_host_mmio; bool guest_can_read_msr_platform_info; @@ -2143,6 +2150,11 @@ u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu); * the gfn, i.e. retrying the instruction will hit a * !PRESENT fault, which results in a new shadow page * and sends KVM back to square one. + * + * EMULTYPE_SKIP_SOFT_INT - Set in combination with EMULTYPE_SKIP to only skip + * an instruction if it could generate a given software + * interrupt, which must be encoded via + * EMULTYPE_SET_SOFT_INT_VECTOR(). */ #define EMULTYPE_NO_DECODE (1 << 0) #define EMULTYPE_TRAP_UD (1 << 1) @@ -2153,6 +2165,10 @@ u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu); #define EMULTYPE_PF (1 << 6) #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) #define EMULTYPE_WRITE_PF_TO_SP (1 << 8) +#define EMULTYPE_SKIP_SOFT_INT (1 << 9) + +#define EMULTYPE_SET_SOFT_INT_VECTOR(v) ((u32)((v) & 0xff) << 16) +#define EMULTYPE_GET_SOFT_INT_VECTOR(e) (((e) >> 16) & 0xff) static inline bool kvm_can_emulate_event_vectoring(int emul_type) { @@ -2378,7 +2394,6 @@ int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, int kvm_add_user_return_msr(u32 msr); int kvm_find_user_return_msr(u32 msr); int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask); -void kvm_user_return_msr_update_cache(unsigned int index, u64 val); u64 kvm_get_user_return_msr(unsigned int slot); static inline bool kvm_is_supported_user_return_msr(u32 msr) @@ -2458,7 +2473,8 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages); KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS | \ KVM_X86_QUIRK_SLOT_ZAP_ALL | \ KVM_X86_QUIRK_STUFF_FEATURE_MSRS | \ - KVM_X86_QUIRK_IGNORE_GUEST_PAT) + KVM_X86_QUIRK_IGNORE_GUEST_PAT | \ + KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM) #define KVM_X86_CONDITIONAL_QUIRKS \ (KVM_X86_QUIRK_CD_NW_CLEARED | \ diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index 9d38ae744a2e4..a7294656ad908 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -68,7 +68,7 @@ * Depending on -fpatchable-function-entry=N,N usage (CONFIG_CALL_PADDING) the * CFI symbol layout changes. * - * Without CALL_THUNKS: + * Without CALL_PADDING: * * .align FUNCTION_ALIGNMENT * __cfi_##name: @@ -77,7 +77,7 @@ * .long __kcfi_typeid_##name * name: * - * With CALL_THUNKS: + * With CALL_PADDING: * * .align FUNCTION_ALIGNMENT * __cfi_##name: diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 9e1720d73244f..66ddea295c266 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -645,6 +645,9 @@ #define MSR_AMD64_DC_CFG 0xc0011022 #define MSR_AMD64_TW_CFG 0xc0011023 +#define MSR_AMD64_FP_CFG 0xc0011028 +#define MSR_AMD64_FP_CFG_ZEN1_DENORM_FIX_BIT 9 + #define MSR_AMD64_DE_CFG 0xc0011029 #define MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT 1 #define MSR_AMD64_DE_CFG_LFENCE_SERIALIZE BIT_ULL(MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT) @@ -711,7 +714,10 @@ #define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT) #define MSR_AMD64_SNP_SECURE_AVIC_BIT 18 #define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT) -#define MSR_AMD64_SNP_RESV_BIT 19 +#define MSR_AMD64_SNP_RESERVED_BITS19_22 GENMASK_ULL(22, 19) +#define MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT 23 +#define MSR_AMD64_SNP_IBPB_ON_ENTRY BIT_ULL(MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT) +#define MSR_AMD64_SNP_RESV_BIT 24 #define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT) #define MSR_AMD64_SAVIC_CONTROL 0xc0010138 #define MSR_AMD64_SAVIC_EN_BIT 0 diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h index 50f75467f73d0..b5dec859bc75a 100644 --- a/arch/x86/include/asm/ptrace.h +++ b/arch/x86/include/asm/ptrace.h @@ -187,12 +187,12 @@ convert_ip_to_linear(struct task_struct *child, struct pt_regs *regs); extern void send_sigtrap(struct pt_regs *regs, int error_code, int si_code); -static inline unsigned long regs_return_value(struct pt_regs *regs) +static __always_inline unsigned long regs_return_value(struct pt_regs *regs) { return regs->ax; } -static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) +static __always_inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) { regs->ax = rc; } @@ -277,34 +277,34 @@ static __always_inline bool ip_within_syscall_gap(struct pt_regs *regs) } #endif -static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +static __always_inline unsigned long kernel_stack_pointer(struct pt_regs *regs) { return regs->sp; } -static inline unsigned long instruction_pointer(struct pt_regs *regs) +static __always_inline unsigned long instruction_pointer(struct pt_regs *regs) { return regs->ip; } -static inline void instruction_pointer_set(struct pt_regs *regs, - unsigned long val) +static __always_inline +void instruction_pointer_set(struct pt_regs *regs, unsigned long val) { regs->ip = val; } -static inline unsigned long frame_pointer(struct pt_regs *regs) +static __always_inline unsigned long frame_pointer(struct pt_regs *regs) { return regs->bp; } -static inline unsigned long user_stack_pointer(struct pt_regs *regs) +static __always_inline unsigned long user_stack_pointer(struct pt_regs *regs) { return regs->sp; } -static inline void user_stack_pointer_set(struct pt_regs *regs, - unsigned long val) +static __always_inline +void user_stack_pointer_set(struct pt_regs *regs, unsigned long val) { regs->sp = val; } diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 91a3fb8ae7ff4..269879b9323bd 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -507,7 +507,7 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif -#define ARCH_HAS_NOCACHE_UACCESS 1 +#define ARCH_HAS_NONTEMPORAL_UACCESS 1 /* * The "unsafe" user accesses aren't really "unsafe", but the naming diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index 40379a1adbb84..fff19e73ccb33 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h @@ -26,13 +26,7 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n) return __copy_user_ll(to, (__force const void *)from, n); } -static __always_inline unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, - unsigned long n) -{ - return __copy_from_user_ll_nocache_nozero(to, from, n); -} - +unsigned long __must_check copy_from_user_inatomic_nontemporal(void *, const void __user *, unsigned long n); unsigned long __must_check clear_user(void __user *mem, unsigned long len); unsigned long __must_check __clear_user(void __user *mem, unsigned long len); diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 641f45c22f9da..d7daec6dd1684 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -147,26 +147,28 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) return copy_user_generic((__force void *)dst, src, size); } -extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size); -extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size); +#define copy_to_nontemporal copy_to_nontemporal +extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size); +extern size_t copy_user_flushcache(void *dst, const void __user *src, size_t size); static inline int -__copy_from_user_inatomic_nocache(void *dst, const void __user *src, +copy_from_user_inatomic_nontemporal(void *dst, const void __user *src, unsigned size) { long ret; kasan_check_write(dst, size); + src = mask_user_address(src); stac(); - ret = __copy_user_nocache(dst, src, size); + ret = copy_to_nontemporal(dst, (__force const void *)src, size); clac(); return ret; } -static inline int -__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) +static inline size_t +copy_from_user_flushcache(void *dst, const void __user *src, size_t size) { kasan_check_write(dst, size); - return __copy_user_flushcache(dst, src, size); + return copy_user_flushcache(dst, src, size); } /* diff --git a/arch/x86/include/asm/unwind_user.h b/arch/x86/include/asm/unwind_user.h new file mode 100644 index 0000000000000..12064284bc4e5 --- /dev/null +++ b/arch/x86/include/asm/unwind_user.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_UNWIND_USER_H +#define _ASM_X86_UNWIND_USER_H + +#ifdef CONFIG_HAVE_UNWIND_USER_FP + +#include +#include + +#define ARCH_INIT_USER_FP_FRAME(ws) \ + .cfa_off = 2*(ws), \ + .ra_off = -1*(ws), \ + .fp_off = -2*(ws), \ + .use_fp = true, + +#define ARCH_INIT_USER_FP_ENTRY_FRAME(ws) \ + .cfa_off = 1*(ws), \ + .ra_off = -1*(ws), \ + .fp_off = 0, \ + .use_fp = false, + +static inline int unwind_user_word_size(struct pt_regs *regs) +{ + /* We can't unwind VM86 stacks */ + if (regs->flags & X86_VM_MASK) + return 0; +#ifdef CONFIG_X86_64 + if (!user_64bit_mode(regs)) + return sizeof(int); +#endif + return sizeof(long); +} + +static inline bool unwind_user_at_function_start(struct pt_regs *regs) +{ + return is_uprobe_at_func_entry(regs); +} + +#endif /* CONFIG_HAVE_UNWIND_USER_FP */ + +#endif /* _ASM_X86_UNWIND_USER_H */ diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h index 1ee2e5115955c..362210c799987 100644 --- a/arch/x86/include/asm/uprobes.h +++ b/arch/x86/include/asm/uprobes.h @@ -62,4 +62,13 @@ struct arch_uprobe_task { unsigned int saved_tf; }; +#ifdef CONFIG_UPROBES +extern bool is_uprobe_at_func_entry(struct pt_regs *regs); +#else +static bool is_uprobe_at_func_entry(struct pt_regs *regs) +{ + return false; +} +#endif /* CONFIG_UPROBES */ + #endif /* _ASM_UPROBES_H */ diff --git a/arch/x86/include/asm/vmware.h b/arch/x86/include/asm/vmware.h index c9cf43d5ef238..4220dae14a2d9 100644 --- a/arch/x86/include/asm/vmware.h +++ b/arch/x86/include/asm/vmware.h @@ -140,7 +140,7 @@ unsigned long vmware_hypercall3(unsigned long cmd, unsigned long in1, "b" (in1), "c" (cmd), "d" (0) - : "cc", "memory"); + : "di", "si", "cc", "memory"); return out0; } @@ -165,7 +165,7 @@ unsigned long vmware_hypercall4(unsigned long cmd, unsigned long in1, "b" (in1), "c" (cmd), "d" (0) - : "cc", "memory"); + : "di", "si", "cc", "memory"); return out0; } diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index d420c9c066d48..67b3bea44c674 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -197,13 +197,13 @@ struct kvm_msrs { __u32 nmsrs; /* number of msrs in entries */ __u32 pad; - struct kvm_msr_entry entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_msr_entry, entries); }; /* for KVM_GET_MSR_INDEX_LIST */ struct kvm_msr_list { __u32 nmsrs; /* number of msrs in entries */ - __u32 indices[]; + __DECLARE_FLEX_ARRAY(__u32, indices); }; /* Maximum size of any access bitmap in bytes */ @@ -245,7 +245,7 @@ struct kvm_cpuid_entry { struct kvm_cpuid { __u32 nent; __u32 padding; - struct kvm_cpuid_entry entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_cpuid_entry, entries); }; struct kvm_cpuid_entry2 { @@ -267,7 +267,7 @@ struct kvm_cpuid_entry2 { struct kvm_cpuid2 { __u32 nent; __u32 padding; - struct kvm_cpuid_entry2 entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_cpuid_entry2, entries); }; /* for KVM_GET_PIT and KVM_SET_PIT */ @@ -398,7 +398,7 @@ struct kvm_xsave { * the contents of CPUID leaf 0xD on the host. */ __u32 region[1024]; - __u32 extra[]; + __DECLARE_FLEX_ARRAY(__u32, extra); }; #define KVM_MAX_XCRS 16 @@ -476,6 +476,7 @@ struct kvm_sync_regs { #define KVM_X86_QUIRK_SLOT_ZAP_ALL (1 << 7) #define KVM_X86_QUIRK_STUFF_FEATURE_MSRS (1 << 8) #define KVM_X86_QUIRK_IGNORE_GUEST_PAT (1 << 9) +#define KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM (1 << 10) #define KVM_STATE_NESTED_FORMAT_VMX 0 #define KVM_STATE_NESTED_FORMAT_SVM 1 @@ -563,7 +564,7 @@ struct kvm_pmu_event_filter { __u32 fixed_counter_bitmap; __u32 flags; __u32 pad[4]; - __u64 events[]; + __DECLARE_FLEX_ARRAY(__u64, events); }; #define KVM_PMU_EVENT_ALLOW 0 @@ -913,8 +914,10 @@ struct kvm_sev_snp_launch_finish { __u64 pad1[4]; }; -#define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) -#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) +#define KVM_X2APIC_API_USE_32BIT_IDS _BITULL(0) +#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK _BITULL(1) +#define KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST _BITULL(2) +#define KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST _BITULL(3) struct kvm_hyperv_eventfd { __u32 conn_id; diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index bc184dd38d993..558b96d53e002 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -44,6 +44,20 @@ KCOV_INSTRUMENT_unwind_orc.o := n KCOV_INSTRUMENT_unwind_frame.o := n KCOV_INSTRUMENT_unwind_guess.o := n +# Disable KCOV to prevent crashes during kexec: load_segments() invalidates +# the GS base, which KCOV relies on for per-CPU data. +# +# As KCOV and KEXEC compatibility should be preserved (e.g. syzkaller is +# using it to collect crash dumps during kernel fuzzing), disabling +# KCOV for KEXEC kernels is not an option. Selectively disabling KCOV +# instrumentation for individual affected functions can be fragile, while +# adding more checks to KCOV would slow it down. +# +# As a compromise solution, disable KCOV instrumentation for the whole +# source code file. If its coverage is ever needed, other approaches +# should be considered. +KCOV_INSTRUMENT_machine_kexec_64.o := n + CFLAGS_head32.o := -fno-stack-protector CFLAGS_head64.o := -fno-stack-protector CFLAGS_irq.o := -I $(src)/../include/asm/trace diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 9fa321a95eb33..d6138b2b633a3 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "sleep.h" /* To include x86_acpi_suspend_lowlevel */ static int __initdata acpi_force = 0; @@ -164,11 +165,14 @@ static bool __init acpi_is_processor_usable(u32 lapic_flags) if (lapic_flags & ACPI_MADT_ENABLED) return true; - if (!acpi_support_online_capable || - (lapic_flags & ACPI_MADT_ONLINE_CAPABLE)) - return true; + if (acpi_support_online_capable) + return lapic_flags & ACPI_MADT_ONLINE_CAPABLE; - return false; + /* + * QEMU expects legacy "Enabled=0" LAPIC entries to be counted as usable + * in order to support CPU hotplug in guests. + */ + return !hypervisor_is_type(X86_HYPER_NATIVE); } static int __init diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 8ee5ff547357a..bd16e9f40d51a 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1168,7 +1168,7 @@ void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) poison_endbr(addr); if (IS_ENABLED(CONFIG_FINEIBT)) - poison_cfi(addr - 16); + poison_cfi(addr - CFI_OFFSET); } } @@ -1375,6 +1375,8 @@ extern u8 fineibt_preamble_end[]; #define fineibt_preamble_ud 0x13 #define fineibt_preamble_hash 5 +#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) + /* * : * 0: b8 78 56 34 12 mov $0x12345678, %eax @@ -1620,7 +1622,7 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end) * have determined there are no indirect calls to it and we * don't need no CFI either. */ - if (!is_endbr(addr + 16)) + if (!is_endbr(addr + CFI_OFFSET)) continue; hash = decode_preamble_hash(addr, &arity); @@ -1628,6 +1630,15 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end) addr, addr, 5, addr)) return -EINVAL; + /* + * FineIBT relies on being at func-16, so if the preamble is + * actually larger than that, place it the tail end. + * + * NOTE: this is possible with things like DEBUG_CALL_THUNKS + * and DEBUG_FORCE_FUNCTION_ALIGN_64B. + */ + addr += CFI_OFFSET - fineibt_prefix_size; + text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size); WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678); text_poke_early(addr + fineibt_preamble_hash, &hash, 4); @@ -1650,10 +1661,10 @@ static void cfi_rewrite_endbr(s32 *start, s32 *end) for (s = start; s < end; s++) { void *addr = (void *)s + *s; - if (!exact_endbr(addr + 16)) + if (!exact_endbr(addr + CFI_OFFSET)) continue; - poison_endbr(addr + 16); + poison_endbr(addr + CFI_OFFSET); } } @@ -1758,7 +1769,8 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, if (FINEIBT_WARN(fineibt_preamble_size, 20) || FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) || FINEIBT_WARN(fineibt_caller_size, 14) || - FINEIBT_WARN(fineibt_paranoid_size, 20)) + FINEIBT_WARN(fineibt_paranoid_size, 20) || + WARN_ON_ONCE(CFI_OFFSET < fineibt_prefix_size)) return; if (cfi_mode == CFI_AUTO) { @@ -1871,6 +1883,11 @@ static void poison_cfi(void *addr) */ switch (cfi_mode) { case CFI_FINEIBT: + /* + * FineIBT preamble is at func-16. + */ + addr += CFI_OFFSET - fineibt_prefix_size; + /* * FineIBT prefix should start with an ENDBR. */ @@ -1909,8 +1926,6 @@ static void poison_cfi(void *addr) } } -#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) - /* * When regs->ip points to a 0xD6 byte in the FineIBT preamble, * return true and fill out target and type. diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 680d305589a3a..aa1b0ef5e9314 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1890,6 +1890,7 @@ void __init check_x2apic(void) static inline void try_to_enable_x2apic(int remap_mode) { } static inline void __x2apic_enable(void) { } +static inline void __x2apic_disable(void) { } #endif /* !CONFIG_X86_X2APIC */ void __init enable_IR_x2apic(void) @@ -2452,6 +2453,11 @@ static void lapic_resume(void) if (x2apic_mode) { __x2apic_enable(); } else { + if (x2apic_enabled()) { + pr_warn_once("x2apic: re-enabled by firmware during resume. Disabling\n"); + __x2apic_disable(); + } + /* * Make sure the APICBASE points to the right address * diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 15209f220e1fd..42568ceec4816 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -1708,8 +1708,22 @@ static void __init uv_system_init_hub(void) struct uv_hub_info_s *new_hub; /* Allocate & fill new per hub info list */ - new_hub = (bid == 0) ? &uv_hub_info_node0 - : kzalloc_node(bytes, GFP_KERNEL, uv_blade_to_node(bid)); + if (bid == 0) { + new_hub = &uv_hub_info_node0; + } else { + int nid; + + /* + * Deconfigured sockets are mapped to SOCK_EMPTY. Use + * NUMA_NO_NODE to allocate on a valid node. + */ + nid = uv_blade_to_node(bid); + if (nid == SOCK_EMPTY) + nid = NUMA_NO_NODE; + + new_hub = kzalloc_node(bytes, GFP_KERNEL, nid); + } + if (WARN_ON_ONCE(!new_hub)) { /* do not kfree() bid 0, which is statically allocated */ while (--bid > 0) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 5d46709c58d0b..ec965de4abec9 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -949,29 +949,20 @@ static void init_amd_zen1(struct cpuinfo_x86 *c) msr_clear_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT); clear_cpu_cap(c, X86_FEATURE_IRPERF); } -} - -static bool cpu_has_zenbleed_microcode(void) -{ - u32 good_rev = 0; - switch (boot_cpu_data.x86_model) { - case 0x30 ... 0x3f: good_rev = 0x0830107b; break; - case 0x60 ... 0x67: good_rev = 0x0860010c; break; - case 0x68 ... 0x6f: good_rev = 0x08608107; break; - case 0x70 ... 0x7f: good_rev = 0x08701033; break; - case 0xa0 ... 0xaf: good_rev = 0x08a00009; break; - - default: - return false; - } - - if (boot_cpu_data.microcode < good_rev) - return false; - - return true; + pr_notice_once("AMD Zen1 FPDSS bug detected, enabling mitigation.\n"); + msr_set_bit(MSR_AMD64_FP_CFG, MSR_AMD64_FP_CFG_ZEN1_DENORM_FIX_BIT); } +static const struct x86_cpu_id amd_zenbleed_microcode[] = { + ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x0830107b), + ZEN_MODEL_STEP_UCODE(0x17, 0x60, 0x1, 0x0860010c), + ZEN_MODEL_STEP_UCODE(0x17, 0x68, 0x1, 0x08608107), + ZEN_MODEL_STEP_UCODE(0x17, 0x71, 0x0, 0x08701033), + ZEN_MODEL_STEP_UCODE(0x17, 0xa0, 0x0, 0x08a00009), + {} +}; + static void zen2_zenbleed_check(struct cpuinfo_x86 *c) { if (cpu_has(c, X86_FEATURE_HYPERVISOR)) @@ -980,7 +971,7 @@ static void zen2_zenbleed_check(struct cpuinfo_x86 *c) if (!cpu_has(c, X86_FEATURE_AVX)) return; - if (!cpu_has_zenbleed_microcode()) { + if (!x86_match_min_microcode_rev(amd_zenbleed_microcode)) { pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n"); msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); } else { diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 02d97834a1d4d..68053f4f2c9be 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -407,7 +407,7 @@ static __always_inline void setup_umip(struct cpuinfo_x86 *c) /* These bits should not change their value after CPU init is finished. */ static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | - X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED; + X86_CR4_FSGSBASE | X86_CR4_CET; static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); static unsigned long cr4_pinned_bits __ro_after_init; @@ -2020,12 +2020,6 @@ static void identify_cpu(struct cpuinfo_x86 *c) setup_smap(c); setup_umip(c); - /* Enable FSGSBASE instructions if available. */ - if (cpu_has(c, X86_FEATURE_FSGSBASE)) { - cr4_set_bits(X86_CR4_FSGSBASE); - elf_hwcap2 |= HWCAP2_FSGSBASE; - } - /* * The vendor-specific functions might have changed features. * Now we do "generic changes." @@ -2386,6 +2380,18 @@ void cpu_init_exception_handling(bool boot_cpu) /* GHCB needs to be setup to handle #VC. */ setup_ghcb(); + /* + * On CPUs with FSGSBASE support, paranoid_entry() uses + * ALTERNATIVE-patched RDGSBASE/WRGSBASE instructions. Secondary CPUs + * boot after alternatives are patched globally, so early exceptions + * execute patched code that depends on FSGSBASE. Enable the feature + * before any exceptions occur. + */ + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { + cr4_set_bits(X86_CR4_FSGSBASE); + elf_hwcap2 |= HWCAP2_FSGSBASE; + } + if (cpu_feature_enabled(X86_FEATURE_FRED)) { /* The boot CPU has enabled FRED during early boot */ if (!boot_cpu) diff --git a/arch/x86/kernel/cpu/mce/threshold.c b/arch/x86/kernel/cpu/mce/threshold.c index f4a007616468e..61eaa17749311 100644 --- a/arch/x86/kernel/cpu/mce/threshold.c +++ b/arch/x86/kernel/cpu/mce/threshold.c @@ -85,7 +85,8 @@ void cmci_storm_end(unsigned int bank) { struct mca_storm_desc *storm = this_cpu_ptr(&storm_desc); - __clear_bit(bank, this_cpu_ptr(mce_poll_banks)); + if (!mce_flags.amd_threshold) + __clear_bit(bank, this_cpu_ptr(mce_poll_banks)); storm->banks[bank].history = 0; storm->banks[bank].in_storm_mode = false; diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index a881bf4c20114..46673530bc6f0 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -186,50 +186,61 @@ static u32 cpuid_to_ucode_rev(unsigned int val) return p.ucode_rev; } +static u32 get_cutoff_revision(u32 rev) +{ + switch (rev >> 8) { + case 0x80012: return 0x8001277; break; + case 0x80082: return 0x800820f; break; + case 0x83010: return 0x830107c; break; + case 0x86001: return 0x860010e; break; + case 0x86081: return 0x8608108; break; + case 0x87010: return 0x8701034; break; + case 0x8a000: return 0x8a0000a; break; + case 0xa0010: return 0xa00107a; break; + case 0xa0011: return 0xa0011da; break; + case 0xa0012: return 0xa001243; break; + case 0xa0082: return 0xa00820e; break; + case 0xa1011: return 0xa101153; break; + case 0xa1012: return 0xa10124e; break; + case 0xa1081: return 0xa108109; break; + case 0xa2010: return 0xa20102f; break; + case 0xa2012: return 0xa201212; break; + case 0xa4041: return 0xa404109; break; + case 0xa5000: return 0xa500013; break; + case 0xa6012: return 0xa60120a; break; + case 0xa7041: return 0xa704109; break; + case 0xa7052: return 0xa705208; break; + case 0xa7080: return 0xa708009; break; + case 0xa70c0: return 0xa70C009; break; + case 0xaa001: return 0xaa00116; break; + case 0xaa002: return 0xaa00218; break; + case 0xb0021: return 0xb002146; break; + case 0xb0081: return 0xb008111; break; + case 0xb1010: return 0xb101046; break; + case 0xb2040: return 0xb204031; break; + case 0xb4040: return 0xb404031; break; + case 0xb4041: return 0xb404101; break; + case 0xb6000: return 0xb600031; break; + case 0xb6080: return 0xb608031; break; + case 0xb7000: return 0xb700031; break; + default: break; + + } + return 0; +} + static bool need_sha_check(u32 cur_rev) { + u32 cutoff; + if (!cur_rev) { cur_rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax); pr_info_once("No current revision, generating the lowest one: 0x%x\n", cur_rev); } - switch (cur_rev >> 8) { - case 0x80012: return cur_rev <= 0x8001277; break; - case 0x80082: return cur_rev <= 0x800820f; break; - case 0x83010: return cur_rev <= 0x830107c; break; - case 0x86001: return cur_rev <= 0x860010e; break; - case 0x86081: return cur_rev <= 0x8608108; break; - case 0x87010: return cur_rev <= 0x8701034; break; - case 0x8a000: return cur_rev <= 0x8a0000a; break; - case 0xa0010: return cur_rev <= 0xa00107a; break; - case 0xa0011: return cur_rev <= 0xa0011da; break; - case 0xa0012: return cur_rev <= 0xa001243; break; - case 0xa0082: return cur_rev <= 0xa00820e; break; - case 0xa1011: return cur_rev <= 0xa101153; break; - case 0xa1012: return cur_rev <= 0xa10124e; break; - case 0xa1081: return cur_rev <= 0xa108109; break; - case 0xa2010: return cur_rev <= 0xa20102f; break; - case 0xa2012: return cur_rev <= 0xa201212; break; - case 0xa4041: return cur_rev <= 0xa404109; break; - case 0xa5000: return cur_rev <= 0xa500013; break; - case 0xa6012: return cur_rev <= 0xa60120a; break; - case 0xa7041: return cur_rev <= 0xa704109; break; - case 0xa7052: return cur_rev <= 0xa705208; break; - case 0xa7080: return cur_rev <= 0xa708009; break; - case 0xa70c0: return cur_rev <= 0xa70C009; break; - case 0xaa001: return cur_rev <= 0xaa00116; break; - case 0xaa002: return cur_rev <= 0xaa00218; break; - case 0xb0021: return cur_rev <= 0xb002146; break; - case 0xb0081: return cur_rev <= 0xb008111; break; - case 0xb1010: return cur_rev <= 0xb101046; break; - case 0xb2040: return cur_rev <= 0xb204031; break; - case 0xb4040: return cur_rev <= 0xb404031; break; - case 0xb4041: return cur_rev <= 0xb404101; break; - case 0xb6000: return cur_rev <= 0xb600031; break; - case 0xb6080: return cur_rev <= 0xb608031; break; - case 0xb7000: return cur_rev <= 0xb700031; break; - default: break; - } + cutoff = get_cutoff_revision(cur_rev); + if (cutoff) + return cur_rev <= cutoff; pr_info("You should not be seeing this. Please send the following couple of lines to x86--kernel.org\n"); pr_info("CPUID(1).EAX: 0x%x, current revision: 0x%x\n", bsp_cpuid_1_eax, cur_rev); @@ -247,7 +258,7 @@ static bool cpu_has_entrysign(void) if (fam == 0x1a) { if (model <= 0x2f || (0x40 <= model && model <= 0x4f) || - (0x60 <= model && model <= 0x6f)) + (0x60 <= model && model <= 0x7f)) return true; } @@ -494,6 +505,7 @@ static int verify_patch(const u8 *buf, size_t buf_size, u32 *patch_size) { u8 family = x86_family(bsp_cpuid_1_eax); struct microcode_header_amd *mc_hdr; + u32 cur_rev, cutoff, patch_rev; u32 sh_psize; u16 proc_id; u8 patch_fam; @@ -533,11 +545,32 @@ static int verify_patch(const u8 *buf, size_t buf_size, u32 *patch_size) proc_id = mc_hdr->processor_rev_id; patch_fam = 0xf + (proc_id >> 12); - ucode_dbg("Patch-ID 0x%08x: family: 0x%x\n", mc_hdr->patch_id, patch_fam); - if (patch_fam != family) return 1; + cur_rev = get_patch_level(); + + /* No cutoff revision means old/unaffected by signing algorithm weakness => matches */ + cutoff = get_cutoff_revision(cur_rev); + if (!cutoff) + goto ok; + + patch_rev = mc_hdr->patch_id; + + ucode_dbg("cur_rev: 0x%x, cutoff: 0x%x, patch_rev: 0x%x\n", + cur_rev, cutoff, patch_rev); + + if (cur_rev <= cutoff && patch_rev <= cutoff) + goto ok; + + if (cur_rev > cutoff && patch_rev > cutoff) + goto ok; + + return 1; + +ok: + ucode_dbg("Patch-ID 0x%08x: family: 0x%x\n", mc_hdr->patch_id, patch_fam); + return 0; } @@ -606,8 +639,6 @@ static size_t parse_container(u8 *ucode, size_t size, struct cont_desc *desc) mc = (struct microcode_amd *)(buf + SECTION_HDR_SIZE); - ucode_dbg("patch_id: 0x%x\n", mc->hdr.patch_id); - if (mc_patch_matches(mc, eq_id)) { desc->psize = patch_size; desc->mc = mc; diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index f75c140906d00..539edd6d6dc8c 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -136,7 +136,7 @@ bool __init microcode_loader_disabled(void) return dis_ucode_ldr; } -static void early_parse_cmdline(void) +static void __init early_parse_cmdline(void) { char cmd_buf[64] = {}; char *s, *p = cmd_buf; diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 06ca5a30140c2..79a1f2d5f5019 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -818,7 +818,8 @@ static __init bool get_mem_config(void) if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) return __get_mem_config_intel(&hw_res->r_resctrl); - else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) + else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || + boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) return __rdt_get_mem_config_amd(&hw_res->r_resctrl); return false; @@ -978,7 +979,8 @@ static __init void rdt_init_res_defs(void) { if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) rdt_init_res_defs_intel(); - else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) + else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || + boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) rdt_init_res_defs_amd(); } @@ -1010,8 +1012,19 @@ void resctrl_cpu_detect(struct cpuinfo_x86 *c) c->x86_cache_occ_scale = ebx; c->x86_cache_mbm_width_offset = eax & 0xff; - if (c->x86_vendor == X86_VENDOR_AMD && !c->x86_cache_mbm_width_offset) - c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_AMD; + if (!c->x86_cache_mbm_width_offset) { + switch (c->x86_vendor) { + case X86_VENDOR_AMD: + c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_AMD; + break; + case X86_VENDOR_HYGON: + c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_HYGON; + break; + default: + /* Leave c->x86_cache_mbm_width_offset as 0 */ + break; + } + } } } diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 9f4c2f0aaf5c8..6da9bd1a188b9 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -14,6 +14,9 @@ #define MBM_CNTR_WIDTH_OFFSET_AMD 20 +/* Hygon MBM counter width as an offset from MBM_CNTR_WIDTH_BASE */ +#define MBM_CNTR_WIDTH_OFFSET_HYGON 8 + #define RMID_VAL_ERROR BIT_ULL(63) #define RMID_VAL_UNAVAIL BIT_ULL(62) diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index 6073a16628f9e..425404e7b7b42 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -27,7 +27,6 @@ #include #include -#include #include #include #include @@ -240,20 +239,6 @@ static __init void topo_register_apic(u32 apic_id, u32 acpi_id, bool present) cpuid_to_apicid[cpu] = apic_id; topo_set_cpuids(cpu, apic_id, acpi_id); } else { - u32 pkgid = topo_apicid(apic_id, TOPO_PKG_DOMAIN); - - /* - * Check for present APICs in the same package when running - * on bare metal. Allow the bogosity in a guest. - */ - if (hypervisor_is_type(X86_HYPER_NATIVE) && - topo_unit_count(pkgid, TOPO_PKG_DOMAIN, phys_cpu_present_map)) { - pr_info_once("Ignoring hot-pluggable APIC ID %x in present package.\n", - apic_id); - topo_info.nr_rejected_cpus++; - return; - } - topo_info.nr_disabled_cpus++; } diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 71ee20102a8af..b10684dedc589 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -181,8 +181,8 @@ static void show_regs_if_on_stack(struct stack_info *info, struct pt_regs *regs, * in false positive reports. Disable instrumentation to avoid those. */ __no_kmsan_checks -static void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, - unsigned long *stack, const char *log_lvl) +static void __show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, + unsigned long *stack, const char *log_lvl) { struct unwind_state state; struct stack_info stack_info = {0}; @@ -303,6 +303,25 @@ static void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, } } +static void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, + unsigned long *stack, const char *log_lvl) +{ + /* + * Disable KASAN to avoid false positives during walking another + * task's stacks, as values on these stacks may change concurrently + * with task execution. + */ + bool disable_kasan = task && task != current; + + if (disable_kasan) + kasan_disable_current(); + + __show_trace_log_lvl(task, regs, stack, log_lvl); + + if (disable_kasan) + kasan_enable_current(); +} + void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) { diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index e88eacb1b5bbf..9badb21e5a5bc 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -318,10 +318,29 @@ EXPORT_SYMBOL_GPL(fpu_enable_guest_xfd_features); #ifdef CONFIG_X86_64 void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) { + struct fpstate *fpstate = guest_fpu->fpstate; + fpregs_lock(); - guest_fpu->fpstate->xfd = xfd; - if (guest_fpu->fpstate->in_use) - xfd_update_state(guest_fpu->fpstate); + + /* + * KVM's guest ABI is that setting XFD[i]=1 *can* immediately revert the + * save state to its initial configuration. Likewise, KVM_GET_XSAVE does + * the same as XSAVE and returns XSTATE_BV[i]=0 whenever XFD[i]=1. + * + * If the guest's FPU state is in hardware, just update XFD: the XSAVE + * in fpu_swap_kvm_fpstate will clear XSTATE_BV[i] whenever XFD[i]=1. + * + * If however the guest's FPU state is NOT resident in hardware, clear + * disabled components in XSTATE_BV now, or a subsequent XRSTOR will + * attempt to load disabled components and generate #NM _in the host_. + */ + if (xfd && test_thread_flag(TIF_NEED_FPU_LOAD)) + fpstate->regs.xsave.header.xfeatures &= ~xfd; + + fpstate->xfd = xfd; + if (fpstate->in_use) + xfd_update_state(fpstate); + fpregs_unlock(); } EXPORT_SYMBOL_GPL(fpu_update_guest_xfd); @@ -429,6 +448,13 @@ int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, if (ustate->xsave.header.xfeatures & ~xcr0) return -EINVAL; + /* + * Disabled features must be in their initial state, otherwise XRSTOR + * causes an exception. + */ + if (WARN_ON_ONCE(ustate->xsave.header.xfeatures & kstate->xfd)) + return -EINVAL; + /* * Nullify @vpkru to preserve its current value if PKRU's bit isn't set * in the header. KVM's odd ABI is to leave PKRU untouched in this diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 28e4fd65c9da7..5f54a207ace46 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1945,7 +1945,7 @@ static int dump_xsave_layout_desc(struct coredump_params *cprm) }; if (!dump_emit(cprm, &xc, sizeof(xc))) - return 0; + return -1; num_records++; } @@ -1983,7 +1983,7 @@ int elf_coredump_extra_notes_write(struct coredump_params *cprm) return 1; num_records = dump_xsave_layout_desc(cprm); - if (!num_records) + if (num_records < 0) return 1; /* Total size should be equal to the number of records */ diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 823dbdd0eb410..ae94f7b0fdf16 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -354,6 +354,9 @@ SYM_CODE_START(return_to_handler) UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR + /* Store original rsp for pt_regs.sp value. */ + movq %rsp, %rdi + /* Restore return_to_handler value that got eaten by previous ret instruction. */ subq $8, %rsp UNWIND_HINT_FUNC @@ -364,7 +367,7 @@ SYM_CODE_START(return_to_handler) movq %rax, RAX(%rsp) movq %rdx, RDX(%rsp) movq %rbp, RBP(%rsp) - movq %rsp, RSP(%rsp) + movq %rdi, RSP(%rsp) movq %rsp, %rdi call ftrace_return_to_handler diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 10721a1252269..59d38411a1d99 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -396,6 +396,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi) /* Posted Interrupt Descriptors for coalesced MSIs to be posted */ DEFINE_PER_CPU_ALIGNED(struct pi_desc, posted_msi_pi_desc); +static DEFINE_PER_CPU_CACHE_HOT(bool, posted_msi_handler_active); void intel_posted_msi_init(void) { @@ -413,6 +414,25 @@ void intel_posted_msi_init(void) this_cpu_write(posted_msi_pi_desc.ndst, destination); } +void intel_ack_posted_msi_irq(struct irq_data *irqd) +{ + irq_move_irq(irqd); + + /* + * Handle the rare case that irq_retrigger() raised the actual + * assigned vector on the target CPU, which means that it was not + * invoked via the posted MSI handler below. In that case APIC EOI + * is required as otherwise the ISR entry becomes stale and lower + * priority interrupts are never going to be delivered after that. + * + * If the posted handler invoked the device interrupt handler then + * the EOI would be premature because it would acknowledge the + * posted vector. + */ + if (unlikely(!__this_cpu_read(posted_msi_handler_active))) + apic_eoi(); +} + static __always_inline bool handle_pending_pir(unsigned long *pir, struct pt_regs *regs) { unsigned long pir_copy[NR_PIR_WORDS]; @@ -445,6 +465,8 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification) pid = this_cpu_ptr(&posted_msi_pi_desc); + /* Mark the handler active for intel_ack_posted_msi_irq() */ + __this_cpu_write(posted_msi_handler_active, true); inc_irq_stat(posted_msi_notification_count); irq_enter(); @@ -473,6 +495,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification) apic_eoi(); irq_exit(); + __this_cpu_write(posted_msi_handler_active, false); set_irq_regs(old_regs); } #endif /* X86_POSTED_MSI */ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index c3244ac680d14..f3b451eb49be1 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -192,6 +192,13 @@ setup_efi_state(struct boot_params *params, unsigned long params_load_addr, struct efi_info *current_ei = &boot_params.efi_info; struct efi_info *ei = ¶ms->efi_info; + if (!params->acpi_rsdp_addr) { + if (efi.acpi20 != EFI_INVALID_TABLE_ADDR) + params->acpi_rsdp_addr = efi.acpi20; + else if (efi.acpi != EFI_INVALID_TABLE_ADDR) + params->acpi_rsdp_addr = efi.acpi; + } + if (!efi_enabled(EFI_RUNTIME_SERVICES)) return 0; diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 1b2edd07a3e17..383d4a4784f5b 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -439,9 +439,15 @@ int __init ima_free_kexec_buffer(void) int __init ima_get_kexec_buffer(void **addr, size_t *size) { + int ret; + if (!ima_kexec_buffer_size) return -ENOENT; + ret = ima_validate_range(ima_kexec_buffer_phys, ima_kexec_buffer_size); + if (ret) + return ret; + *addr = __va(ima_kexec_buffer_phys); *size = ima_kexec_buffer_size; diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c index 978232b6d48d7..ff8edea8511b4 100644 --- a/arch/x86/kernel/shstk.c +++ b/arch/x86/kernel/shstk.c @@ -351,7 +351,8 @@ static int shstk_pop_sigframe(unsigned long *ssp) need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp; if (need_to_check_vma) - mmap_read_lock_killable(current->mm); + if (mmap_read_lock_killable(current->mm)) + return -EINTR; err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp); if (unlikely(err)) diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 845aeaf36b8d2..46aed82243964 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -1819,3 +1819,59 @@ bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, else return regs->sp <= ret->stack; } + +/* + * Heuristic-based check if uprobe is installed at the function entry. + * + * Under assumption of user code being compiled with frame pointers, + * `push %rbp/%ebp` is a good indicator that we indeed are. + * + * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern. + * If we get this wrong, captured stack trace might have one extra bogus + * entry, but the rest of stack trace will still be meaningful. + */ +bool is_uprobe_at_func_entry(struct pt_regs *regs) +{ + struct arch_uprobe *auprobe; + + if (!current->utask) + return false; + + auprobe = current->utask->auprobe; + if (!auprobe) + return false; + + /* push %rbp/%ebp */ + if (auprobe->insn[0] == 0x55) + return true; + + /* endbr64 (64-bit only) */ + if (user_64bit_mode(regs) && is_endbr((u32 *)auprobe->insn)) + return true; + + return false; +} + +#ifdef CONFIG_IA32_EMULATION +unsigned long arch_uprobe_get_xol_area(void) +{ + struct thread_info *ti = current_thread_info(); + unsigned long vaddr; + + /* + * HACK: we are not in a syscall, but x86 get_unmapped_area() paths + * ignore TIF_ADDR32 and rely on in_32bit_syscall() to calculate + * vm_unmapped_area_info.high_limit. + * + * The #ifdef above doesn't cover the CONFIG_X86_X32_ABI=y case, + * but in this case in_32bit_syscall() -> in_x32_syscall() always + * (falsely) returns true because ->orig_ax == -1. + */ + if (test_thread_flag(TIF_ADDR32)) + ti->status |= TS_COMPAT; + vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + ti->status &= ~TS_COMPAT; + + return vaddr; +} +#endif diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index d7af4a64c211b..4ed82b1fe173b 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -424,6 +424,7 @@ SECTIONS .llvm_bb_addr_map : { *(.llvm_bb_addr_map) } #endif + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 52524e0ca97f7..913ffb995279a 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -509,11 +509,18 @@ static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2, u32 vcpu_caps[NR_KVM_CPU_CAPS]; int r; + /* + * Apply pending runtime CPUID updates to the current CPUID entries to + * avoid false positives due to mismatches on KVM-owned feature flags. + */ + if (vcpu->arch.cpuid_dynamic_bits_dirty) + kvm_update_cpuid_runtime(vcpu); + /* * Swap the existing (old) entries with the incoming (new) entries in * order to massage the new entries, e.g. to account for dynamic bits - * that KVM controls, without clobbering the current guest CPUID, which - * KVM needs to preserve in order to unwind on failure. + * that KVM controls, without losing the current guest CPUID, which KVM + * needs to preserve in order to unwind on failure. * * Similarly, save the vCPU's current cpu_caps so that the capabilities * can be updated alongside the CPUID entries when performing runtime diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c index 2c2783296aedb..a26fa4222f292 100644 --- a/arch/x86/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -561,7 +561,7 @@ static void kvm_ioapic_update_eoi_one(struct kvm_vcpu *vcpu, spin_lock(&ioapic->lock); if (trigger_mode != IOAPIC_LEVEL_TRIG || - kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) + kvm_lapic_suppress_eoi_broadcast(apic)) return; ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 0ae7f913d7826..a9845a1a9cd5d 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -105,6 +105,63 @@ bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) apic_test_vector(vector, apic->regs + APIC_IRR); } +static bool kvm_lapic_advertise_suppress_eoi_broadcast(struct kvm *kvm) +{ + switch (kvm->arch.suppress_eoi_broadcast_mode) { + case KVM_SUPPRESS_EOI_BROADCAST_ENABLED: + return true; + case KVM_SUPPRESS_EOI_BROADCAST_DISABLED: + return false; + case KVM_SUPPRESS_EOI_BROADCAST_QUIRKED: + /* + * The default in-kernel I/O APIC emulates the 82093AA and does not + * implement an EOI register. Some guests (e.g. Windows with the + * Hyper-V role enabled) disable LAPIC EOI broadcast without + * checking the I/O APIC version, which can cause level-triggered + * interrupts to never be EOI'd. + * + * To avoid this, KVM doesn't advertise Suppress EOI Broadcast + * support when using the default in-kernel I/O APIC. + * + * Historically, in split IRQCHIP mode, KVM always advertised + * Suppress EOI Broadcast support but did not actually suppress + * EOIs, resulting in quirky behavior. + */ + return !ioapic_in_kernel(kvm); + default: + WARN_ON_ONCE(1); + return false; + } +} + +bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic) +{ + struct kvm *kvm = apic->vcpu->kvm; + + if (!(kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)) + return false; + + switch (kvm->arch.suppress_eoi_broadcast_mode) { + case KVM_SUPPRESS_EOI_BROADCAST_ENABLED: + return true; + case KVM_SUPPRESS_EOI_BROADCAST_DISABLED: + return false; + case KVM_SUPPRESS_EOI_BROADCAST_QUIRKED: + /* + * Historically, in split IRQCHIP mode, KVM ignored the suppress + * EOI broadcast bit set by the guest and broadcasts EOIs to the + * userspace I/O APIC. For In-kernel I/O APIC, the support itself + * is not advertised, can only be enabled via KVM_SET_APIC_STATE, + * and KVM's I/O APIC doesn't emulate Directed EOIs; but if the + * feature is enabled, it is respected (with odd behavior). + */ + return ioapic_in_kernel(kvm); + default: + WARN_ON_ONCE(1); + return false; + } +} + __read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_has_noapic_vcpu); @@ -554,15 +611,9 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu) v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); - /* - * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) - * which doesn't have EOI register; Some buggy OSes (e.g. Windows with - * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC - * version first and level-triggered interrupts never get EOIed in - * IOAPIC. - */ + if (guest_cpu_cap_has(vcpu, X86_FEATURE_X2APIC) && - !ioapic_in_kernel(vcpu->kvm)) + kvm_lapic_advertise_suppress_eoi_broadcast(vcpu->kvm)) v |= APIC_LVR_DIRECTED_EOI; kvm_lapic_set_reg(apic, APIC_LVR, v); } @@ -1517,6 +1568,15 @@ static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) /* Request a KVM exit to inform the userspace IOAPIC. */ if (irqchip_split(apic->vcpu->kvm)) { + /* + * Don't exit to userspace if the guest has enabled Directed + * EOI, a.k.a. Suppress EOI Broadcasts, in which case the local + * APIC doesn't broadcast EOIs (the guest must EOI the target + * I/O APIC(s) directly). + */ + if (kvm_lapic_suppress_eoi_broadcast(apic)) + return; + apic->vcpu->arch.pending_ioapic_eoi = vector; kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); return; @@ -2131,15 +2191,33 @@ static void advance_periodic_target_expiration(struct kvm_lapic *apic) ktime_t delta; /* - * Synchronize both deadlines to the same time source or - * differences in the periods (caused by differences in the - * underlying clocks or numerical approximation errors) will - * cause the two to drift apart over time as the errors - * accumulate. + * Use kernel time as the time source for both the hrtimer deadline and + * TSC-based deadline so that they stay synchronized. Computing each + * deadline independently will cause the two deadlines to drift apart + * over time as differences in the periods accumulate, e.g. due to + * differences in the underlying clocks or numerical approximation errors. */ apic->lapic_timer.target_expiration = ktime_add_ns(apic->lapic_timer.target_expiration, apic->lapic_timer.period); + + /* + * If the new expiration is in the past, e.g. because userspace stopped + * running the VM for an extended duration, then force the expiration + * to "now" and don't try to play catch-up with the missed events. KVM + * will only deliver a single interrupt regardless of how many events + * are pending, i.e. restarting the timer with an expiration in the + * past will do nothing more than waste host cycles, and can even lead + * to a hard lockup in extreme cases. + */ + if (ktime_before(apic->lapic_timer.target_expiration, now)) + apic->lapic_timer.target_expiration = now; + + /* + * Note, ensuring the expiration isn't in the past also prevents delta + * from going negative, which could cause the TSC deadline to become + * excessively large due to it an unsigned value. + */ delta = ktime_sub(apic->lapic_timer.target_expiration, now); apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + nsec_to_cycles(apic->vcpu, delta); @@ -2970,9 +3048,9 @@ static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) apic_timer_expired(apic, true); - if (lapic_is_periodic(apic)) { + if (lapic_is_periodic(apic) && !WARN_ON_ONCE(!apic->lapic_timer.period)) { advance_periodic_target_expiration(apic); - hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); + hrtimer_set_expires(&ktimer->timer, ktimer->target_expiration); return HRTIMER_RESTART; } else return HRTIMER_NORESTART; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 282b9b7da98cd..e5f5a222eced0 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -231,6 +231,8 @@ static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu) bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector); +bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic); + void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu); void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 667d66cf76d5e..dad7abb1112b7 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3044,12 +3044,6 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, bool prefetch = !fault || fault->prefetch; bool write_fault = fault && fault->write; - if (unlikely(is_noslot_pfn(pfn))) { - vcpu->stat.pf_mmio_spte_created++; - mark_mmio_spte(vcpu, sptep, gfn, pte_access); - return RET_PF_EMULATE; - } - if (is_shadow_present_pte(*sptep)) { if (prefetch && is_last_spte(*sptep, level) && pfn == spte_to_pfn(*sptep)) @@ -3066,13 +3060,22 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, child = spte_to_child_sp(pte); drop_parent_pte(vcpu->kvm, child, sptep); flush = true; - } else if (WARN_ON_ONCE(pfn != spte_to_pfn(*sptep))) { + } else if (pfn != spte_to_pfn(*sptep)) { + WARN_ON_ONCE(vcpu->arch.mmu->root_role.direct); drop_spte(vcpu->kvm, sptep); flush = true; } else was_rmapped = 1; } + if (unlikely(is_noslot_pfn(pfn))) { + vcpu->stat.pf_mmio_spte_created++; + mark_mmio_spte(vcpu, sptep, gfn, pte_access); + if (flush) + kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level); + return RET_PF_EMULATE; + } + wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch, false, host_writable, &spte); diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index fef00546c8856..ba6e9485e8247 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -158,15 +158,34 @@ static void avic_set_x2apic_msr_interception(struct vcpu_svm *svm, svm->x2avic_msrs_intercepted = intercept; } +static u32 avic_get_max_physical_id(struct kvm_vcpu *vcpu) +{ + u32 arch_max; + + if (x2avic_enabled && apic_x2apic_mode(vcpu->arch.apic)) + arch_max = X2AVIC_MAX_PHYSICAL_ID; + else + arch_max = AVIC_MAX_PHYSICAL_ID; + + /* + * Despite its name, KVM_CAP_MAX_VCPU_ID represents the maximum APIC ID + * plus one, so the max possible APIC ID is one less than that. + */ + return min(vcpu->kvm->arch.max_vcpu_ids - 1, arch_max); +} + static void avic_activate_vmcb(struct vcpu_svm *svm) { struct vmcb *vmcb = svm->vmcb01.ptr; + struct kvm_vcpu *vcpu = &svm->vcpu; vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; - + vmcb->control.avic_physical_id |= avic_get_max_physical_id(vcpu); vmcb->control.int_ctl |= AVIC_ENABLE_MASK; + svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); + /* * Note: KVM supports hybrid-AVIC mode, where KVM emulates x2APIC MSR * accesses, while interrupt injection to a running vCPU can be @@ -176,7 +195,7 @@ static void avic_activate_vmcb(struct vcpu_svm *svm) */ if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) { vmcb->control.int_ctl |= X2APIC_MODE_MASK; - vmcb->control.avic_physical_id |= X2AVIC_MAX_PHYSICAL_ID; + /* Disabling MSR intercept for x2APIC registers */ avic_set_x2apic_msr_interception(svm, false); } else { @@ -186,8 +205,6 @@ static void avic_activate_vmcb(struct vcpu_svm *svm) */ kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu); - /* For xAVIC and hybrid-xAVIC modes */ - vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID; /* Enabling MSR intercept for x2APIC registers */ avic_set_x2apic_msr_interception(svm, true); } @@ -200,6 +217,9 @@ static void avic_deactivate_vmcb(struct vcpu_svm *svm) vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; + if (!sev_es_guest(svm->vcpu.kvm)) + svm_set_intercept(svm, INTERCEPT_CR8_WRITE); + /* * If running nested and the guest uses its own MSR bitmap, there * is no need to update L0's msr bitmap @@ -321,7 +341,7 @@ void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb) vmcb->control.avic_physical_id = __sme_set(__pa(kvm_svm->avic_physical_id_table)); vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE; - if (kvm_apicv_activated(svm->vcpu.kvm)) + if (kvm_vcpu_apicv_active(&svm->vcpu)) avic_activate_vmcb(svm); else avic_deactivate_vmcb(svm); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index da6e80b3ac353..db35033999a8e 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -613,6 +613,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 struct kvm_vcpu *vcpu = &svm->vcpu; nested_vmcb02_compute_g_pat(svm); + vmcb_mark_dirty(vmcb02, VMCB_NPT); /* Load the nested guest state */ if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { @@ -751,6 +752,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm, vmcb02->control.nested_ctl = vmcb01->control.nested_ctl; vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa; vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; + vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); /* * Stash vmcb02's counter if the guest hasn't moved past the guilty @@ -983,7 +985,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) if (!nested_vmcb_check_save(vcpu) || !nested_vmcb_check_controls(vcpu)) { vmcb12->control.exit_code = SVM_EXIT_ERR; - vmcb12->control.exit_code_hi = 0; + vmcb12->control.exit_code_hi = -1u; vmcb12->control.exit_info_1 = 0; vmcb12->control.exit_info_2 = 0; goto out; @@ -1016,7 +1018,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) svm->soft_int_injected = false; svm->vmcb->control.exit_code = SVM_EXIT_ERR; - svm->vmcb->control.exit_code_hi = 0; + svm->vmcb->control.exit_code_hi = -1u; svm->vmcb->control.exit_info_1 = 0; svm->vmcb->control.exit_info_2 = 0; @@ -1878,10 +1880,9 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, * thus MMU might not be initialized correctly. * Set it again to fix this. */ - ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, nested_npt_enabled(svm), false); - if (WARN_ON_ONCE(ret)) + if (ret) goto out_free; svm->nested.force_msr_bitmap_recalc = true; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0835c664fbfdb..4b778e71b4c36 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -679,10 +679,16 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, if (ulen == 0 || uaddr + ulen < uaddr) return ERR_PTR(-EINVAL); - /* Calculate number of pages. */ + /* + * Calculate the number of pages that need to be pinned to cover the + * entire range. Note! This isn't simply ulen >> PAGE_SHIFT, as KVM + * doesn't require the incoming address+size to be page aligned! + */ first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT; npages = (last - first + 1); + if (npages > INT_MAX) + return ERR_PTR(-EINVAL); locked = sev->pages_locked + npages; lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; @@ -691,9 +697,6 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, return ERR_PTR(-ENOMEM); } - if (WARN_ON_ONCE(npages > INT_MAX)) - return ERR_PTR(-EINVAL); - /* Avoid using vmalloc for smaller buffers. */ size = npages * sizeof(struct page *); if (size > PAGE_SIZE) @@ -871,6 +874,11 @@ static int sev_es_sync_vmsa(struct vcpu_svm *svm) u8 *d; int i; + lockdep_assert_held(&vcpu->mutex); + + if (vcpu->arch.guest_state_protected) + return -EINVAL; + /* Check some debug related fields before encrypting the VMSA */ if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1)) return -EINVAL; @@ -1016,6 +1024,9 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!sev_es_guest(kvm)) return -ENOTTY; + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + kvm_for_each_vcpu(i, vcpu, kvm) { ret = mutex_lock_killable(&vcpu->mutex); if (ret) @@ -2036,8 +2047,8 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) struct kvm_vcpu *src_vcpu; unsigned long i; - if (src->created_vcpus != atomic_read(&src->online_vcpus) || - dst->created_vcpus != atomic_read(&dst->online_vcpus)) + if (kvm_is_vcpu_creation_in_progress(src) || + kvm_is_vcpu_creation_in_progress(dst)) return -EBUSY; if (!sev_es_guest(src)) @@ -2443,6 +2454,13 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) unsigned long i; int ret; + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + + ret = kvm_lock_all_vcpus(kvm); + if (ret) + return ret; + data.gctx_paddr = __psp_pa(sev->snp_context); data.page_type = SNP_PAGE_TYPE_VMSA; @@ -2452,12 +2470,12 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) ret = sev_es_sync_vmsa(svm); if (ret) - return ret; + goto out; /* Transition the VMSA page to a firmware state. */ ret = rmp_make_private(pfn, INITIAL_VMSA_GPA, PG_LEVEL_4K, sev->asid, true); if (ret) - return ret; + goto out; /* Issue the SNP command to encrypt the VMSA */ data.address = __sme_pa(svm->sev_es.vmsa); @@ -2466,7 +2484,7 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (ret) { snp_page_reclaim(kvm, pfn); - return ret; + goto out; } svm->vcpu.arch.guest_state_protected = true; @@ -2480,7 +2498,9 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) svm_enable_lbrv(vcpu); } - return 0; +out: + kvm_unlock_all_vcpus(kvm); + return ret; } static int snp_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) @@ -2684,6 +2704,8 @@ int sev_mem_enc_register_region(struct kvm *kvm, struct enc_region *region; int ret = 0; + guard(mutex)(&kvm->lock); + if (!sev_guest(kvm)) return -ENOTTY; @@ -2698,12 +2720,10 @@ int sev_mem_enc_register_region(struct kvm *kvm, if (!region) return -ENOMEM; - mutex_lock(&kvm->lock); region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, FOLL_WRITE | FOLL_LONGTERM); if (IS_ERR(region->pages)) { ret = PTR_ERR(region->pages); - mutex_unlock(&kvm->lock); goto e_free; } @@ -2721,8 +2741,6 @@ int sev_mem_enc_register_region(struct kvm *kvm, region->size = range->size; list_add_tail(®ion->list, &sev->regions_list); - mutex_unlock(&kvm->lock); - return ret; e_free: diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 9d29b2e7e855d..939b944185548 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -272,6 +272,7 @@ static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) } static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu, + int emul_type, bool commit_side_effects) { struct vcpu_svm *svm = to_svm(vcpu); @@ -293,7 +294,7 @@ static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu, if (unlikely(!commit_side_effects)) old_rflags = svm->vmcb->save.rflags; - if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) + if (!kvm_emulate_instruction(vcpu, emul_type)) return 0; if (unlikely(!commit_side_effects)) @@ -311,11 +312,13 @@ static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu, static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu) { - return __svm_skip_emulated_instruction(vcpu, true); + return __svm_skip_emulated_instruction(vcpu, EMULTYPE_SKIP, true); } -static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu) +static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu, u8 vector) { + const int emul_type = EMULTYPE_SKIP | EMULTYPE_SKIP_SOFT_INT | + EMULTYPE_SET_SOFT_INT_VECTOR(vector); unsigned long rip, old_rip = kvm_rip_read(vcpu); struct vcpu_svm *svm = to_svm(vcpu); @@ -331,7 +334,7 @@ static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu) * in use, the skip must not commit any side effects such as clearing * the interrupt shadow or RFLAGS.RF. */ - if (!__svm_skip_emulated_instruction(vcpu, !nrips)) + if (!__svm_skip_emulated_instruction(vcpu, emul_type, !nrips)) return -EIO; rip = kvm_rip_read(vcpu); @@ -367,7 +370,7 @@ static void svm_inject_exception(struct kvm_vcpu *vcpu) kvm_deliver_exception_payload(vcpu, ex); if (kvm_exception_is_soft(ex->vector) && - svm_update_soft_interrupt_rip(vcpu)) + svm_update_soft_interrupt_rip(vcpu, ex->vector)) return; svm->vmcb->control.event_inj = ex->vector @@ -1029,8 +1032,7 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) svm_set_intercept(svm, INTERCEPT_CR0_WRITE); svm_set_intercept(svm, INTERCEPT_CR3_WRITE); svm_set_intercept(svm, INTERCEPT_CR4_WRITE); - if (!kvm_vcpu_apicv_active(vcpu)) - svm_set_intercept(svm, INTERCEPT_CR8_WRITE); + svm_set_intercept(svm, INTERCEPT_CR8_WRITE); set_dr_intercepts(svm); @@ -1138,7 +1140,7 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) svm_clr_intercept(svm, INTERCEPT_PAUSE); } - if (kvm_vcpu_apicv_active(vcpu)) + if (enable_apicv && irqchip_in_kernel(vcpu->kvm)) avic_init_vmcb(svm, vmcb); if (vnmi) @@ -2091,12 +2093,13 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) ret = kvm_skip_emulated_instruction(vcpu); + /* KVM always performs VMLOAD/VMSAVE on VMCB01 (see __svm_vcpu_run()) */ if (vmload) { - svm_copy_vmloadsave_state(svm->vmcb, vmcb12); + svm_copy_vmloadsave_state(svm->vmcb01.ptr, vmcb12); svm->sysenter_eip_hi = 0; svm->sysenter_esp_hi = 0; } else { - svm_copy_vmloadsave_state(vmcb12, svm->vmcb); + svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr); } kvm_vcpu_unmap(vcpu, &map); @@ -2435,6 +2438,7 @@ static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu, if (cr0 ^ val) { svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; + svm->vmcb->control.exit_code_hi = 0; ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE); } @@ -2593,9 +2597,11 @@ static int dr_interception(struct kvm_vcpu *vcpu) static int cr8_write_interception(struct kvm_vcpu *vcpu) { + u8 cr8_prev = kvm_get_cr8(vcpu); int r; - u8 cr8_prev = kvm_get_cr8(vcpu); + WARN_ON_ONCE(kvm_vcpu_apicv_active(vcpu)); + /* instruction emulation calls kvm_set_cr8() */ r = cr_interception(vcpu); if (lapic_in_kernel(vcpu)) @@ -3633,11 +3639,12 @@ static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu) static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) { + struct kvm_queued_interrupt *intr = &vcpu->arch.interrupt; struct vcpu_svm *svm = to_svm(vcpu); u32 type; - if (vcpu->arch.interrupt.soft) { - if (svm_update_soft_interrupt_rip(vcpu)) + if (intr->soft) { + if (svm_update_soft_interrupt_rip(vcpu, intr->nr)) return; type = SVM_EVTINJ_TYPE_SOFT; @@ -3645,12 +3652,10 @@ static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) type = SVM_EVTINJ_TYPE_INTR; } - trace_kvm_inj_virq(vcpu->arch.interrupt.nr, - vcpu->arch.interrupt.soft, reinjected); + trace_kvm_inj_virq(intr->nr, intr->soft, reinjected); ++vcpu->stat.irq_injections; - svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | - SVM_EVTINJ_VALID | type; + svm->vmcb->control.event_inj = intr->nr | SVM_EVTINJ_VALID | type; } void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, @@ -4526,31 +4531,45 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu, case SVM_EXIT_WRITE_CR0: { unsigned long cr0, val; - if (info->intercept == x86_intercept_cr_write) + /* + * Adjust the exit code accordingly if a CR other than CR0 is + * being written, and skip straight to the common handling as + * only CR0 has an additional selective intercept. + */ + if (info->intercept == x86_intercept_cr_write && info->modrm_reg) { icpt_info.exit_code += info->modrm_reg; - - if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 || - info->intercept == x86_intercept_clts) break; + } - if (!(vmcb12_is_intercept(&svm->nested.ctl, - INTERCEPT_SELECTIVE_CR0))) + /* + * Convert the exit_code to SVM_EXIT_CR0_SEL_WRITE if a + * selective CR0 intercept is triggered (the common logic will + * treat the selective intercept as being enabled). Note, the + * unconditional intercept has higher priority, i.e. this is + * only relevant if *only* the selective intercept is enabled. + */ + if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_CR0_WRITE) || + !(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))) break; - cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; - val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; + /* CLTS never triggers INTERCEPT_SELECTIVE_CR0 */ + if (info->intercept == x86_intercept_clts) + break; + /* LMSW always triggers INTERCEPT_SELECTIVE_CR0 */ if (info->intercept == x86_intercept_lmsw) { - cr0 &= 0xfUL; - val &= 0xfUL; - /* lmsw can't clear PE - catch this here */ - if (cr0 & X86_CR0_PE) - val |= X86_CR0_PE; + icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; + break; } + /* + * MOV-to-CR0 only triggers INTERCEPT_SELECTIVE_CR0 if any bit + * other than SVM_CR0_SELECTIVE_MASK is changed. + */ + cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; + val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; if (cr0 ^ val) icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; - break; } case SVM_EXIT_READ_DR0: @@ -4611,6 +4630,7 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu, if (static_cpu_has(X86_FEATURE_NRIPS)) vmcb->control.next_rip = info->next_rip; vmcb->control.exit_code = icpt_info.exit_code; + vmcb->control.exit_code_hi = 0; vmexit = nested_svm_exit_handled(svm); ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED @@ -5267,6 +5287,8 @@ static __init void svm_set_cpu_caps(void) */ kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); + + kvm_setup_xss_caps(); } static __init int svm_hardware_setup(void) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index dd78e64023450..e66a16e59b1a5 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -764,9 +764,10 @@ int nested_svm_vmexit(struct vcpu_svm *svm); static inline int nested_svm_simple_vmexit(struct vcpu_svm *svm, u32 exit_code) { - svm->vmcb->control.exit_code = exit_code; - svm->vmcb->control.exit_info_1 = 0; - svm->vmcb->control.exit_info_2 = 0; + svm->vmcb->control.exit_code = exit_code; + svm->vmcb->control.exit_code_hi = 0; + svm->vmcb->control.exit_info_1 = 0; + svm->vmcb->control.exit_info_2 = 0; return nested_svm_vmexit(svm); } diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index bcea087b642fd..36574b6b637d7 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -19,6 +19,7 @@ #include "trace.h" #include "vmx.h" #include "smm.h" +#include "x86_ops.h" static bool __read_mostly enable_shadow_vmcs = 1; module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); @@ -3261,10 +3262,24 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, if (CC(vmcs12->guest_cr4 & X86_CR4_CET && !(vmcs12->guest_cr0 & X86_CR0_WP))) return -EINVAL; - if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && - (CC(!kvm_dr7_valid(vmcs12->guest_dr7)) || - CC(!vmx_is_valid_debugctl(vcpu, vmcs12->guest_ia32_debugctl, false)))) - return -EINVAL; + if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { + u64 debugctl = vmcs12->guest_ia32_debugctl; + + /* + * FREEZE_IN_SMM is not virtualized, but allow L1 to set it in + * vmcs12's DEBUGCTL under a quirk for backwards compatibility. + * Note that the quirk only relaxes the consistency check. The + * vmcc02 bit is still under the control of the host. In + * particular, if a host administrator decides to clear the bit, + * then L1 has no say in the matter. + */ + if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM)) + debugctl &= ~DEBUGCTLMSR_FREEZE_IN_SMM; + + if (CC(!kvm_dr7_valid(vmcs12->guest_dr7)) || + CC(!vmx_is_valid_debugctl(vcpu, debugctl, false))) + return -EINVAL; + } if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) @@ -5216,7 +5231,7 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, if (vmx->nested.update_vmcs01_apicv_status) { vmx->nested.update_vmcs01_apicv_status = false; - kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); + vmx_refresh_apicv_exec_ctrl(vcpu); } if (vmx->nested.update_vmcs01_hwapic_isr) { diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 0a49c863c811b..d9ffc16dfefe6 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -763,25 +763,6 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) return tdx_vcpu_state_details_intr_pending(vcpu_state_details); } -/* - * Compared to vmx_prepare_switch_to_guest(), there is not much to do - * as SEAMCALL/SEAMRET calls take care of most of save and restore. - */ -void tdx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (vt->guest_state_loaded) - return; - - if (likely(is_64bit_mm(current->mm))) - vt->msr_host_kernel_gs_base = current->thread.gsbase; - else - vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); - - vt->guest_state_loaded = true; -} - struct tdx_uret_msr { u32 msr; unsigned int slot; @@ -795,19 +776,38 @@ static struct tdx_uret_msr tdx_uret_msrs[] = { {.msr = MSR_TSC_AUX,}, }; -static void tdx_user_return_msr_update_cache(void) +void tdx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) { + struct vcpu_vt *vt = to_vt(vcpu); int i; + if (vt->guest_state_loaded) + return; + + if (likely(is_64bit_mm(current->mm))) + vt->msr_host_kernel_gs_base = current->thread.gsbase; + else + vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); + + vt->guest_state_loaded = true; + + /* + * Explicitly set user-return MSRs that are clobbered by the TDX-Module + * if VP.ENTER succeeds, i.e. on TD-Exit, with the values that would be + * written by the TDX-Module. Don't rely on the TDX-Module to actually + * clobber the MSRs, as the contract is poorly defined and not upheld. + * E.g. the TDX-Module will synthesize an EPT Violation without doing + * VM-Enter if it suspects a zero-step attack, and never "restore" VMM + * state. + */ for (i = 0; i < ARRAY_SIZE(tdx_uret_msrs); i++) - kvm_user_return_msr_update_cache(tdx_uret_msrs[i].slot, - tdx_uret_msrs[i].defval); + kvm_set_user_return_msr(tdx_uret_msrs[i].slot, + tdx_uret_msrs[i].defval, -1ull); } static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) { struct vcpu_vt *vt = to_vt(vcpu); - struct vcpu_tdx *tdx = to_tdx(vcpu); if (!vt->guest_state_loaded) return; @@ -815,11 +815,6 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) ++vcpu->stat.host_state_reload; wrmsrl(MSR_KERNEL_GS_BASE, vt->msr_host_kernel_gs_base); - if (tdx->guest_entered) { - tdx_user_return_msr_update_cache(); - tdx->guest_entered = false; - } - vt->guest_state_loaded = false; } @@ -1059,7 +1054,6 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) update_debugctlmsr(vcpu->arch.host_debugctl); tdx_load_host_xsave_state(vcpu); - tdx->guest_entered = true; vcpu->arch.regs_avail &= TDX_REGS_AVAIL_SET; @@ -3447,10 +3441,6 @@ static int __init __tdx_bringup(void) /* * Check if MSRs (tdx_uret_msrs) can be saved/restored * before returning to user space. - * - * this_cpu_ptr(user_return_msrs)->registered isn't checked - * because the registration is done at vcpu runtime by - * tdx_user_return_msr_update_cache(). */ tdx_uret_msrs[i].slot = kvm_find_user_return_msr(tdx_uret_msrs[i].msr); if (tdx_uret_msrs[i].slot == -1) { diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h index ca39a9391db1e..7f258870dc410 100644 --- a/arch/x86/kvm/vmx/tdx.h +++ b/arch/x86/kvm/vmx/tdx.h @@ -67,7 +67,6 @@ struct vcpu_tdx { u64 vp_enter_ret; enum vcpu_tdx_state state; - bool guest_entered; u64 map_gpa_next; u64 map_gpa_end; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 91b6f2f3edc2a..c084f48e2b0b9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8021,6 +8021,8 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_clear(X86_FEATURE_SHSTK); kvm_cpu_cap_clear(X86_FEATURE_IBT); } + + kvm_setup_xss_caps(); } static bool vmx_is_io_intercepted(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c9c2aa6f4705e..c71869e545908 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -121,8 +121,10 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE -#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ - KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) +#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ + KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \ + KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ + KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) static void update_cr8_intercept(struct kvm_vcpu *vcpu); static void process_nmi(struct kvm_vcpu *vcpu); @@ -240,7 +242,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_ipiv); bool __read_mostly enable_device_posted_irqs = true; EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_device_posted_irqs); -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_COUNTER(VM, mmu_shadow_zapped), STATS_DESC_COUNTER(VM, mmu_pte_write), @@ -266,7 +268,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, pf_taken), STATS_DESC_COUNTER(VCPU, pf_fixed), @@ -681,15 +683,6 @@ int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr); -void kvm_user_return_msr_update_cache(unsigned int slot, u64 value) -{ - struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs); - - msrs->values[slot].curr = value; - kvm_user_return_register_notifier(msrs); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_user_return_msr_update_cache); - u64 kvm_get_user_return_msr(unsigned int slot) { return this_cpu_ptr(user_return_msrs)->values[slot].curr; @@ -1045,6 +1038,13 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr); +static bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) +{ + u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; + + return (vcpu->arch.apf.msr_en_val & mask) == mask; +} + static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) { return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); @@ -1137,15 +1137,20 @@ void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned lon } if ((cr0 ^ old_cr0) & X86_CR0_PG) { - kvm_clear_async_pf_completion_queue(vcpu); - kvm_async_pf_hash_reset(vcpu); - /* * Clearing CR0.PG is defined to flush the TLB from the guest's * perspective. */ if (!(cr0 & X86_CR0_PG)) kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + /* + * Check for async #PF completion events when enabling paging, + * as the vCPU may have previously encountered async #PFs (it's + * entirely legal for the guest to toggle paging on/off without + * waiting for the async #PF queue to drain). + */ + else if (kvm_pv_async_pf_enabled(vcpu)) + kvm_make_request(KVM_REQ_APF_READY, vcpu); } if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) @@ -3650,13 +3655,6 @@ static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 0; } -static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) -{ - u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; - - return (vcpu->arch.apf.msr_en_val & mask) == mask; -} - static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) { gpa_t gpa = data & ~0x3f; @@ -4140,47 +4138,47 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_WALL_CLOCK_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; vcpu->kvm->arch.wall_clock = data; kvm_write_wall_clock(vcpu->kvm, data, 0); break; case MSR_KVM_WALL_CLOCK: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; vcpu->kvm->arch.wall_clock = data; kvm_write_wall_clock(vcpu->kvm, data, 0); break; case MSR_KVM_SYSTEM_TIME_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); break; case MSR_KVM_SYSTEM_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); break; case MSR_KVM_ASYNC_PF_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_pv_enable_async_pf(vcpu, data)) return 1; break; case MSR_KVM_ASYNC_PF_INT: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_pv_enable_async_pf_int(vcpu, data)) return 1; break; case MSR_KVM_ASYNC_PF_ACK: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (data & 0x1) { vcpu->arch.apf.pageready_pending = false; kvm_check_async_pf_completion(vcpu); @@ -4188,7 +4186,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_STEAL_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (unlikely(!sched_info_on())) return 1; @@ -4206,7 +4204,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_PV_EOI_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) return 1; @@ -4214,7 +4212,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_KVM_POLL_CONTROL: if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; /* only enable bit supported */ if (data & (-1ULL << 1)) @@ -4515,61 +4513,61 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_WALL_CLOCK: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->kvm->arch.wall_clock; break; case MSR_KVM_WALL_CLOCK_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->kvm->arch.wall_clock; break; case MSR_KVM_SYSTEM_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.time; break; case MSR_KVM_SYSTEM_TIME_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.time; break; case MSR_KVM_ASYNC_PF_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.apf.msr_en_val; break; case MSR_KVM_ASYNC_PF_INT: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.apf.msr_int_val; break; case MSR_KVM_ASYNC_PF_ACK: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = 0; break; case MSR_KVM_STEAL_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.st.msr_val; break; case MSR_KVM_PV_EOI_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.pv_eoi.msr_val; break; case MSR_KVM_POLL_CONTROL: if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.msr_kvm_poll_control; break; @@ -4970,6 +4968,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; case KVM_CAP_X2APIC_API: r = KVM_X2APIC_API_VALID_FLAGS; + if (kvm && !irqchip_split(kvm)) + r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; break; case KVM_CAP_NESTED_STATE: r = kvm_x86_ops.nested_ops->get_state ? @@ -5846,9 +5846,18 @@ static int kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, struct kvm_xsave *guest_xsave) { + union fpregs_state *xstate = (union fpregs_state *)guest_xsave->region; + if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; + /* + * For backwards compatibility, do not expect disabled features to be in + * their initial state. XSTATE_BV[i] must still be cleared whenever + * XFD[i]=1, or XRSTOR would cause a #NM. + */ + xstate->xsave.header.xfeatures &= ~vcpu->arch.guest_fpu.fpstate->xfd; + return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu, guest_xsave->region, kvm_caps.supported_xcr0, @@ -6778,11 +6787,24 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS) break; + if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && + (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)) + break; + + if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && + !irqchip_split(kvm)) + break; + if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS) kvm->arch.x2apic_format = true; if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) kvm->arch.x2apic_broadcast_quirk_disabled = true; + if (cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) + kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED; + if (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) + kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED; + r = 0; break; case KVM_CAP_X86_DISABLE_EXITS: @@ -8214,7 +8236,13 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; frag->gpa = gpa; - frag->data = val; + if (write && bytes <= 8u) { + frag->val = 0; + frag->data = &frag->val; + memcpy(&frag->val, val, bytes); + } else { + frag->data = val; + } frag->len = bytes; return X86EMUL_CONTINUE; } @@ -8229,6 +8257,9 @@ static int emulator_read_write(struct x86_emulate_ctxt *ctxt, gpa_t gpa; int rc; + if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val))) + return X86EMUL_UNHANDLEABLE; + if (ops->read_write_prepare && ops->read_write_prepare(vcpu, val, bytes)) return X86EMUL_CONTINUE; @@ -9337,6 +9368,23 @@ static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) return false; } +static bool is_soft_int_instruction(struct x86_emulate_ctxt *ctxt, + int emulation_type) +{ + u8 vector = EMULTYPE_GET_SOFT_INT_VECTOR(emulation_type); + + switch (ctxt->b) { + case 0xcc: + return vector == BP_VECTOR; + case 0xcd: + return vector == ctxt->src.val; + case 0xce: + return vector == OF_VECTOR; + default: + return false; + } +} + /* * Decode an instruction for emulation. The caller is responsible for handling * code breakpoints. Note, manually detecting code breakpoints is unnecessary @@ -9447,6 +9495,10 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * injecting single-step #DBs. */ if (emulation_type & EMULTYPE_SKIP) { + if (emulation_type & EMULTYPE_SKIP_SOFT_INT && + !is_soft_int_instruction(ctxt, emulation_type)) + return 0; + if (ctxt->mode != X86EMUL_MODE_PROT64) ctxt->eip = (u32)ctxt->_eip; else @@ -9928,6 +9980,23 @@ static struct notifier_block pvclock_gtod_notifier = { }; #endif +void kvm_setup_xss_caps(void) +{ + if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) + kvm_caps.supported_xss = 0; + + if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && + !kvm_cpu_cap_has(X86_FEATURE_IBT)) + kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; + + if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) { + kvm_cpu_cap_clear(X86_FEATURE_SHSTK); + kvm_cpu_cap_clear(X86_FEATURE_IBT); + kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps); + static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) { memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); @@ -10106,19 +10175,6 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) if (!tdp_enabled) kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; - if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) - kvm_caps.supported_xss = 0; - - if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && - !kvm_cpu_cap_has(X86_FEATURE_IBT)) - kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; - - if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) { - kvm_cpu_cap_clear(X86_FEATURE_SHSTK); - kvm_cpu_cap_clear(X86_FEATURE_IBT); - kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; - } - if (kvm_caps.has_tsc_control) { /* * Make sure the user can only configure tsc_khz values that @@ -11568,8 +11624,7 @@ static inline int vcpu_block(struct kvm_vcpu *vcpu) if (is_guest_mode(vcpu)) { int r = kvm_check_nested_events(vcpu); - WARN_ON_ONCE(r == -EBUSY); - if (r < 0) + if (r < 0 && r != -EBUSY) return 0; } @@ -11777,6 +11832,9 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu) frag++; vcpu->mmio_cur_fragment++; } else { + if (WARN_ON_ONCE(frag->data == &frag->val)) + return -EIO; + /* Go forward to the next mmio piece. */ frag->data += len; frag->gpa += len; @@ -12117,9 +12175,11 @@ static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) return; if (is_pae_paging(vcpu)) { + kvm_vcpu_srcu_read_lock(vcpu); for (i = 0 ; i < 4 ; i++) sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + kvm_vcpu_srcu_read_unlock(vcpu); } } diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index f3dc77f006f90..c8a561c17e9ad 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -457,6 +457,8 @@ extern struct kvm_host_values kvm_host; extern bool enable_pmu; +void kvm_setup_xss_caps(void); + /* * Get a filtered version of KVM's supported XCR0 that strips out dynamic * features for which the current process doesn't (yet) have permission to use. diff --git a/arch/x86/lib/copy_user_uncached_64.S b/arch/x86/lib/copy_user_uncached_64.S index 18350b343c2ab..8ed0ce3ad2271 100644 --- a/arch/x86/lib/copy_user_uncached_64.S +++ b/arch/x86/lib/copy_user_uncached_64.S @@ -27,7 +27,7 @@ * Output: * rax uncopied bytes or 0 if successful. */ -SYM_FUNC_START(__copy_user_nocache) +SYM_FUNC_START(copy_to_nontemporal) ANNOTATE_NOENDBR /* If destination is not 7-byte aligned, we'll have to align it */ testb $7,%dil @@ -240,5 +240,5 @@ _ASM_EXTABLE_UA(95b, .Ldone) _ASM_EXTABLE_UA(52b, .Ldone0) _ASM_EXTABLE_UA(53b, .Ldone0) -SYM_FUNC_END(__copy_user_nocache) -EXPORT_SYMBOL(__copy_user_nocache) +SYM_FUNC_END(copy_to_nontemporal) +EXPORT_SYMBOL(copy_to_nontemporal) diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index f6f436f1d5737..ac27e39fc993d 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -322,10 +322,11 @@ unsigned long __copy_user_ll(void *to, const void *from, unsigned long n) } EXPORT_SYMBOL(__copy_user_ll); -unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, +unsigned long copy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n) { - __uaccess_begin_nospec(); + if (!user_access_begin(from, n)) + return n; #ifdef CONFIG_X86_INTEL_USERCOPY if (n > 64 && static_cpu_has(X86_FEATURE_XMM2)) n = __copy_user_intel_nocache(to, from, n); @@ -334,7 +335,7 @@ unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *fr #else __copy_user(to, from, n); #endif - __uaccess_end(); + user_access_end(); return n; } -EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); +EXPORT_SYMBOL(copy_from_user_inatomic_nontemporal); diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 654280aaa3e9e..c47d8cd0e243b 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -43,17 +43,17 @@ void arch_wb_cache_pmem(void *addr, size_t size) } EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); -long __copy_user_flushcache(void *dst, const void __user *src, unsigned size) +size_t copy_user_flushcache(void *dst, const void __user *src, size_t size) { unsigned long flushed, dest = (unsigned long) dst; - long rc; + unsigned long rc; - stac(); - rc = __copy_user_nocache(dst, src, size); - clac(); + src = masked_user_access_begin(src); + rc = copy_to_nontemporal(dst, (__force const void *)src, size); + user_access_end(); /* - * __copy_user_nocache() uses non-temporal stores for the bulk + * copy_to_nontemporal() uses non-temporal stores for the bulk * of the transfer, but we need to manually flush if the * transfer is unaligned. A cached memory copy is used when * destination or size is not naturally aligned. That is: diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 5b9908f13dcfd..3a5364853eab8 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -4,6 +4,8 @@ KCOV_INSTRUMENT_tlb.o := n KCOV_INSTRUMENT_mem_encrypt.o := n KCOV_INSTRUMENT_mem_encrypt_amd.o := n KCOV_INSTRUMENT_pgprot.o := n +# See the "Disable KCOV" comment in arch/x86/kernel/Makefile. +KCOV_INSTRUMENT_physaddr.o := n KASAN_SANITIZE_mem_encrypt.o := n KASAN_SANITIZE_mem_encrypt_amd.o := n diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 998bd807fc7ba..b83a06739b511 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -821,8 +821,6 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, force_sig_pkuerr((void __user *)address, pkey); else force_sig_fault(SIGSEGV, si_code, (void __user *)address); - - local_irq_disable(); } static noinline void @@ -1474,15 +1472,12 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code, do_kern_addr_fault(regs, error_code, address); } else { do_user_addr_fault(regs, error_code, address); - /* - * User address page fault handling might have reenabled - * interrupts. Fixing up all potential exit points of - * do_user_addr_fault() and its leaf functions is just not - * doable w/o creating an unholy mess or turning the code - * upside down. - */ - local_irq_disable(); } + /* + * page fault handling might have reenabled interrupts, + * make sure to disable them again. + */ + local_irq_disable(); } DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 0e4270e20fadb..3d9a5e4ccaa43 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1031,7 +1031,7 @@ static void __meminit free_pagetable(struct page *page, int order) free_reserved_pages(page, nr_pages); #endif } else { - __free_pages(page, order); + pagetable_free(page_ptdesc(page)); } } diff --git a/arch/x86/mm/kaslr.c b/arch/x86/mm/kaslr.c index 3c306de52fd4d..834641c6049a5 100644 --- a/arch/x86/mm/kaslr.c +++ b/arch/x86/mm/kaslr.c @@ -115,12 +115,12 @@ void __init kernel_randomize_memory(void) /* * Adapt physical memory region size based on available memory, - * except when CONFIG_PCI_P2PDMA is enabled. P2PDMA exposes the - * device BAR space assuming the direct map space is large enough - * for creating a ZONE_DEVICE mapping in the direct map corresponding - * to the physical BAR address. + * except when CONFIG_ZONE_DEVICE is enabled. ZONE_DEVICE wants to map + * any physical address into the direct-map. KASLR wants to reliably + * steal some physical address bits. Those design choices are in direct + * conflict. */ - if (!IS_ENABLED(CONFIG_PCI_P2PDMA) && (memory_tb < kaslr_regions[0].size_tb)) + if (!IS_ENABLED(CONFIG_ZONE_DEVICE) && (memory_tb < kaslr_regions[0].size_tb)) kaslr_regions[0].size_tb = memory_tb; /* diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 970981893c9b8..fffb6ef1997d2 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -429,7 +429,7 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) { list_del(&ptdesc->pt_list); - __free_page(ptdesc_page(ptdesc)); + pagetable_free(ptdesc); } } diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index ddf248c3ee7dc..2e5ecfdce73c3 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -729,7 +729,7 @@ int pmd_clear_huge(pmd_t *pmd) int pud_free_pmd_page(pud_t *pud, unsigned long addr) { pmd_t *pmd, *pmd_sv; - pte_t *pte; + struct ptdesc *pt; int i; pmd = pud_pgtable(*pud); @@ -750,8 +750,8 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) for (i = 0; i < PTRS_PER_PMD; i++) { if (!pmd_none(pmd_sv[i])) { - pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); - pte_free_kernel(&init_mm, pte); + pt = page_ptdesc(pmd_page(pmd_sv[i])); + pagetable_dtor_free(pt); } } @@ -772,15 +772,15 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) */ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) { - pte_t *pte; + struct ptdesc *pt; - pte = (pte_t *)pmd_page_vaddr(*pmd); + pt = page_ptdesc(pmd_page(*pmd)); pmd_clear(pmd); /* INVLPG to clear all paging-structure caches */ flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); - pte_free_kernel(&init_mm, pte); + pagetable_dtor_free(pt); return 1; } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index de5083cb1d374..788671a32d8ee 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -438,17 +438,8 @@ static void emit_kcfi(u8 **pprog, u32 hash) EMIT1_off32(0xb8, hash); /* movl $hash, %eax */ #ifdef CONFIG_CALL_PADDING - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); + for (int i = 0; i < CONFIG_FUNCTION_PADDING_CFI; i++) + EMIT1(0x90); #endif EMIT_ENDBR(); diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 463b784499a8f..791c52c8393f4 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -837,7 +837,7 @@ static void __init __efi_enter_virtual_mode(void) } efi_check_for_embedded_firmwares(); - efi_free_boot_services(); + efi_unmap_boot_services(); if (!efi_is_mixed()) efi_native_runtime_setup(); diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 553f330198f2f..79f0818131e83 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -341,7 +341,7 @@ void __init efi_reserve_boot_services(void) /* * Because the following memblock_reserve() is paired - * with memblock_free_late() for this region in + * with free_reserved_area() for this region in * efi_free_boot_services(), we must be extremely * careful not to reserve, and subsequently free, * critical regions of memory (like the kernel image) or @@ -404,17 +404,33 @@ static void __init efi_unmap_pages(efi_memory_desc_t *md) pr_err("Failed to unmap VA mapping for 0x%llx\n", va); } -void __init efi_free_boot_services(void) +struct efi_freeable_range { + u64 start; + u64 end; +}; + +static struct efi_freeable_range *ranges_to_free; + +void __init efi_unmap_boot_services(void) { struct efi_memory_map_data data = { 0 }; efi_memory_desc_t *md; int num_entries = 0; + int idx = 0; + size_t sz; void *new, *new_md; /* Keep all regions for /sys/kernel/debug/efi */ if (efi_enabled(EFI_DBG)) return; + sz = sizeof(*ranges_to_free) * (efi.memmap.nr_map + 1); + ranges_to_free = kzalloc(sz, GFP_KERNEL); + if (!ranges_to_free) { + pr_err("Failed to allocate storage for freeable EFI regions\n"); + return; + } + for_each_efi_memory_desc(md) { unsigned long long start = md->phys_addr; unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; @@ -471,7 +487,15 @@ void __init efi_free_boot_services(void) start = SZ_1M; } - memblock_free_late(start, size); + /* + * With CONFIG_DEFERRED_STRUCT_PAGE_INIT parts of the memory + * map are still not initialized and we can't reliably free + * memory here. + * Queue the ranges to free at a later point. + */ + ranges_to_free[idx].start = start; + ranges_to_free[idx].end = start + size; + idx++; } if (!num_entries) @@ -512,6 +536,31 @@ void __init efi_free_boot_services(void) } } +static int __init efi_free_boot_services(void) +{ + struct efi_freeable_range *range = ranges_to_free; + unsigned long freed = 0; + + if (!ranges_to_free) + return 0; + + while (range->start) { + void *start = phys_to_virt(range->start); + void *end = phys_to_virt(range->end); + + free_reserved_area(start, end, -1, NULL); + freed += (end - start); + range++; + } + kfree(ranges_to_free); + + if (freed) + pr_info("Freeing EFI boot services memory: %ldK\n", freed / SZ_1K); + + return 0; +} +arch_initcall(efi_free_boot_services); + /* * A number of config table entries get remapped to virtual addresses * after entering EFI virtual mode. However, the kexec kernel requires diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 344030c1a81d4..53ee2d53fcf8e 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -91,10 +91,12 @@ SYM_CODE_START(pvh_start_xen) leal rva(early_stack_end)(%ebp), %esp +#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) /* Enable PAE mode. */ mov %cr4, %eax orl $X86_CR4_PAE, %eax mov %eax, %cr4 +#endif #ifdef CONFIG_X86_64 /* Enable Long mode. */ diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 53282dc7d5ac5..23b91bf9b6630 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -470,7 +470,7 @@ int __init arch_xen_unpopulated_init(struct resource **res) * driver to know how much of the physmap is unpopulated and * set an accurate initial memory target. */ - xen_released_pages += xen_extra_mem[i].n_pfns; + xen_unpopulated_pages += xen_extra_mem[i].n_pfns; /* Zero so region is not also added to the balloon driver. */ xen_extra_mem[i].n_pfns = 0; } diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 4806cc28d7ca7..b74ff8bc7f2a8 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -108,7 +108,7 @@ static int xen_cpu_dead_pv(unsigned int cpu); * calls. */ DEFINE_PER_CPU(bool, xen_in_preemptible_hcall); -EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall); +EXPORT_PER_CPU_SYMBOL_GPL(xen_in_preemptible_hcall); /* * In case of scheduling the flag must be cleared and restored after diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 9fb9f35331502..6a75fe1c7a5c0 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -380,7 +380,7 @@ static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from) blkg_rwstat_add_aux(&to->merged, &from->merged); blkg_rwstat_add_aux(&to->service_time, &from->service_time); blkg_rwstat_add_aux(&to->wait_time, &from->wait_time); - bfq_stat_add_aux(&from->time, &from->time); + bfq_stat_add_aux(&to->time, &from->time); bfq_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum); bfq_stat_add_aux(&to->avg_queue_size_samples, &from->avg_queue_size_samples); diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 4a8d3d96bfe49..9e9d081e86bb2 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -697,7 +697,7 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) unsigned int limit, act_idx; /* Sync reads have full depth available */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) limit = data->q->nr_requests; else limit = bfqd->async_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; @@ -7181,7 +7181,7 @@ static void bfq_exit_queue(struct elevator_queue *e) blk_stat_disable_accounting(bfqd->queue); blk_queue_flag_clear(QUEUE_FLAG_DISABLE_WBT_DEF, bfqd->queue); - set_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT, &e->flags); + wbt_enable_default(bfqd->queue->disk); kfree(bfqd); } diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index 687952f63bbbf..b8b7587be9679 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -142,7 +142,7 @@ bool bio_integrity_prep(struct bio *bio) return true; set_flags = false; gfp |= __GFP_ZERO; - } else if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE) + } else if (bi->metadata_size > bi->pi_tuple_size) gfp |= __GFP_ZERO; break; default: diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 9b27963680dc3..964eebbee14d0 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -140,14 +140,21 @@ EXPORT_SYMBOL_GPL(blk_rq_integrity_map_user); bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, struct request *next) { + struct bio_integrity_payload *bip, *bip_next; + if (blk_integrity_rq(req) == 0 && blk_integrity_rq(next) == 0) return true; if (blk_integrity_rq(req) == 0 || blk_integrity_rq(next) == 0) return false; - if (bio_integrity(req->bio)->bip_flags != - bio_integrity(next->bio)->bip_flags) + bip = bio_integrity(req->bio); + bip_next = bio_integrity(next->bio); + if (bip->bip_flags != bip_next->bip_flags) + return false; + + if (bip->bip_flags & BIP_CHECK_APPTAG && + bip->app_tag != bip_next->app_tag) return false; if (req->nr_integrity_segments + next->nr_integrity_segments > @@ -163,15 +170,21 @@ bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, bool blk_integrity_merge_bio(struct request_queue *q, struct request *req, struct bio *bio) { + struct bio_integrity_payload *bip, *bip_bio = bio_integrity(bio); int nr_integrity_segs; - if (blk_integrity_rq(req) == 0 && bio_integrity(bio) == NULL) + if (blk_integrity_rq(req) == 0 && bip_bio == NULL) return true; - if (blk_integrity_rq(req) == 0 || bio_integrity(bio) == NULL) + if (blk_integrity_rq(req) == 0 || bip_bio == NULL) + return false; + + bip = bio_integrity(req->bio); + if (bip->bip_flags != bip_bio->bip_flags) return false; - if (bio_integrity(req->bio)->bip_flags != bio_integrity(bio)->bip_flags) + if (bip->bip_flags & BIP_CHECK_APPTAG && + bip->app_tag != bip_bio->app_tag) return false; nr_integrity_segs = blk_rq_count_integrity_sg(q, bio); diff --git a/block/blk-lib.c b/block/blk-lib.c index 3030a772d3aa0..352e3c0f8a7d7 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -202,13 +202,13 @@ static void __blkdev_issue_zero_pages(struct block_device *bdev, unsigned int nr_vecs = __blkdev_sectors_to_bio_pages(nr_sects); struct bio *bio; - bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask); - bio->bi_iter.bi_sector = sector; - if ((flags & BLKDEV_ZERO_KILLABLE) && fatal_signal_pending(current)) break; + bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask); + bio->bi_iter.bi_sector = sector; + do { unsigned int len; diff --git a/block/blk-merge.c b/block/blk-merge.c index 37864c5d287ef..03b61923cf109 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -158,8 +158,9 @@ static struct bio *bio_submit_split(struct bio *bio, int split_sectors) return bio; } -struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, - unsigned *nsegs) +static struct bio *__bio_split_discard(struct bio *bio, + const struct queue_limits *lim, unsigned *nsegs, + unsigned int max_sectors) { unsigned int max_discard_sectors, granularity; sector_t tmp; @@ -169,8 +170,7 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, granularity = max(lim->discard_granularity >> 9, 1U); - max_discard_sectors = - min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); + max_discard_sectors = min(max_sectors, bio_allowed_max_sectors(lim)); max_discard_sectors -= max_discard_sectors % granularity; if (unlikely(!max_discard_sectors)) return bio; @@ -194,6 +194,19 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, return bio_submit_split(bio, split_sectors); } +struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, + unsigned *nsegs) +{ + unsigned int max_sectors; + + if (bio_op(bio) == REQ_OP_SECURE_ERASE) + max_sectors = lim->max_secure_erase_sectors; + else + max_sectors = lim->max_discard_sectors; + + return __bio_split_discard(bio, lim, nsegs, max_sectors); +} + static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, bool is_atomic) { diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 4896525b1c054..553d93b88e194 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -686,8 +686,10 @@ void blk_mq_debugfs_register_hctxs(struct request_queue *q) struct blk_mq_hw_ctx *hctx; unsigned long i; + mutex_lock(&q->debugfs_mutex); queue_for_each_hw_ctx(q, hctx, i) blk_mq_debugfs_register_hctx(q, hctx); + mutex_unlock(&q->debugfs_mutex); } void blk_mq_debugfs_unregister_hctxs(struct request_queue *q) diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index e0bed16485c34..128f2be9d4207 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -427,11 +427,25 @@ void blk_mq_free_sched_tags(struct elevator_tags *et, kfree(et); } -void blk_mq_free_sched_tags_batch(struct xarray *et_table, +void blk_mq_free_sched_res(struct elevator_resources *res, + struct elevator_type *type, + struct blk_mq_tag_set *set) +{ + if (res->et) { + blk_mq_free_sched_tags(res->et, set); + res->et = NULL; + } + if (res->data) { + blk_mq_free_sched_data(type, res->data); + res->data = NULL; + } +} + +void blk_mq_free_sched_res_batch(struct xarray *elv_tbl, struct blk_mq_tag_set *set) { struct request_queue *q; - struct elevator_tags *et; + struct elv_change_ctx *ctx; lockdep_assert_held_write(&set->update_nr_hwq_lock); @@ -444,13 +458,46 @@ void blk_mq_free_sched_tags_batch(struct xarray *et_table, * concurrently. */ if (q->elevator) { - et = xa_load(et_table, q->id); - if (unlikely(!et)) + ctx = xa_load(elv_tbl, q->id); + if (!ctx) { WARN_ON_ONCE(1); - else - blk_mq_free_sched_tags(et, set); + continue; + } + blk_mq_free_sched_res(&ctx->res, ctx->type, set); + } + } +} + +void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl) +{ + unsigned long i; + struct elv_change_ctx *ctx; + + xa_for_each(elv_tbl, i, ctx) { + xa_erase(elv_tbl, i); + kfree(ctx); + } +} + +int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl, + struct blk_mq_tag_set *set) +{ + struct request_queue *q; + struct elv_change_ctx *ctx; + + lockdep_assert_held_write(&set->update_nr_hwq_lock); + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + ctx = kzalloc(sizeof(struct elv_change_ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + if (xa_insert(elv_tbl, q->id, ctx, GFP_KERNEL)) { + kfree(ctx); + return -ENOMEM; } } + return 0; } struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, @@ -498,12 +545,33 @@ struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, return NULL; } -int blk_mq_alloc_sched_tags_batch(struct xarray *et_table, +int blk_mq_alloc_sched_res(struct request_queue *q, + struct elevator_type *type, + struct elevator_resources *res, + unsigned int nr_hw_queues) +{ + struct blk_mq_tag_set *set = q->tag_set; + + res->et = blk_mq_alloc_sched_tags(set, nr_hw_queues, + blk_mq_default_nr_requests(set)); + if (!res->et) + return -ENOMEM; + + res->data = blk_mq_alloc_sched_data(q, type); + if (IS_ERR(res->data)) { + blk_mq_free_sched_tags(res->et, set); + return -ENOMEM; + } + + return 0; +} + +int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl, struct blk_mq_tag_set *set, unsigned int nr_hw_queues) { + struct elv_change_ctx *ctx; struct request_queue *q; - struct elevator_tags *et; - gfp_t gfp = GFP_NOIO | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY; + int ret = -ENOMEM; lockdep_assert_held_write(&set->update_nr_hwq_lock); @@ -516,39 +584,44 @@ int blk_mq_alloc_sched_tags_batch(struct xarray *et_table, * concurrently. */ if (q->elevator) { - et = blk_mq_alloc_sched_tags(set, nr_hw_queues, - blk_mq_default_nr_requests(set)); - if (!et) + ctx = xa_load(elv_tbl, q->id); + if (WARN_ON_ONCE(!ctx)) { + ret = -ENOENT; + goto out_unwind; + } + + ret = blk_mq_alloc_sched_res(q, q->elevator->type, + &ctx->res, nr_hw_queues); + if (ret) goto out_unwind; - if (xa_insert(et_table, q->id, et, gfp)) - goto out_free_tags; } } return 0; -out_free_tags: - blk_mq_free_sched_tags(et, set); + out_unwind: list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) { if (q->elevator) { - et = xa_load(et_table, q->id); - if (et) - blk_mq_free_sched_tags(et, set); + ctx = xa_load(elv_tbl, q->id); + if (ctx) + blk_mq_free_sched_res(&ctx->res, + ctx->type, set); } } - return -ENOMEM; + return ret; } /* caller must have a reference to @e, will grab another one if successful */ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e, - struct elevator_tags *et) + struct elevator_resources *res) { unsigned int flags = q->tag_set->flags; + struct elevator_tags *et = res->et; struct blk_mq_hw_ctx *hctx; struct elevator_queue *eq; unsigned long i; int ret; - eq = elevator_alloc(q, e, et); + eq = elevator_alloc(q, e, res); if (!eq) return -ENOMEM; diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 8e21a6b1415d9..5678e15bd33c4 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -19,18 +19,52 @@ void __blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx); void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx); int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e, - struct elevator_tags *et); + struct elevator_resources *res); void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e); void blk_mq_sched_free_rqs(struct request_queue *q); struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, unsigned int nr_hw_queues, unsigned int nr_requests); -int blk_mq_alloc_sched_tags_batch(struct xarray *et_table, +int blk_mq_alloc_sched_res(struct request_queue *q, + struct elevator_type *type, + struct elevator_resources *res, + unsigned int nr_hw_queues); +int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl, struct blk_mq_tag_set *set, unsigned int nr_hw_queues); +int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl, + struct blk_mq_tag_set *set); +void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl); void blk_mq_free_sched_tags(struct elevator_tags *et, struct blk_mq_tag_set *set); -void blk_mq_free_sched_tags_batch(struct xarray *et_table, +void blk_mq_free_sched_res(struct elevator_resources *res, + struct elevator_type *type, + struct blk_mq_tag_set *set); +void blk_mq_free_sched_res_batch(struct xarray *et_table, struct blk_mq_tag_set *set); +/* + * blk_mq_alloc_sched_data() - Allocates scheduler specific data + * Returns: + * - Pointer to allocated data on success + * - NULL if no allocation needed + * - ERR_PTR(-ENOMEM) in case of failure + */ +static inline void *blk_mq_alloc_sched_data(struct request_queue *q, + struct elevator_type *e) +{ + void *sched_data; + + if (!e || !e->ops.alloc_sched_data) + return NULL; + + sched_data = e->ops.alloc_sched_data(q); + return (sched_data) ?: ERR_PTR(-ENOMEM); +} + +static inline void blk_mq_free_sched_data(struct elevator_type *e, void *data) +{ + if (e && e->ops.free_sched_data) + e->ops.free_sched_data(data); +} static inline void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx) { @@ -103,4 +137,9 @@ static inline void blk_mq_set_min_shallow_depth(struct request_queue *q, depth); } +static inline bool blk_mq_is_sync_read(blk_opf_t opf) +{ + return op_is_sync(opf) && !op_is_write(opf); +} + #endif diff --git a/block/blk-mq.c b/block/blk-mq.c index d626d32f6e576..4ebb92014eae1 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -335,12 +336,12 @@ void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set) { struct request_queue *q; - mutex_lock(&set->tag_list_lock); - list_for_each_entry(q, &set->tag_list, tag_set_list) { + rcu_read_lock(); + list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) { if (!blk_queue_skip_tagset_quiesce(q)) blk_mq_quiesce_queue_nowait(q); } - mutex_unlock(&set->tag_list_lock); + rcu_read_unlock(); blk_mq_wait_quiesce_done(set); } @@ -350,12 +351,12 @@ void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set) { struct request_queue *q; - mutex_lock(&set->tag_list_lock); - list_for_each_entry(q, &set->tag_list, tag_set_list) { + rcu_read_lock(); + list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) { if (!blk_queue_skip_tagset_quiesce(q)) blk_mq_unquiesce_queue(q); } - mutex_unlock(&set->tag_list_lock); + rcu_read_unlock(); } EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset); @@ -3707,8 +3708,9 @@ static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) { struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_online); + int ret = 0; - if (blk_mq_hctx_has_online_cpu(hctx, cpu)) + if (!hctx->nr_ctx || blk_mq_hctx_has_online_cpu(hctx, cpu)) return 0; /* @@ -3727,12 +3729,24 @@ static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) * frozen and there are no requests. */ if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) { - while (blk_mq_hctx_has_requests(hctx)) + while (blk_mq_hctx_has_requests(hctx)) { + /* + * The wakeup capable IRQ handler of block device is + * not called during suspend. Skip the loop by checking + * pm_wakeup_pending to prevent the deadlock and improve + * suspend latency. + */ + if (pm_wakeup_pending()) { + clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); + ret = -EBUSY; + break; + } msleep(5); + } percpu_ref_put(&hctx->queue->q_usage_counter); } - return 0; + return ret; } /* @@ -4294,7 +4308,7 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q) struct blk_mq_tag_set *set = q->tag_set; mutex_lock(&set->tag_list_lock); - list_del(&q->tag_set_list); + list_del_rcu(&q->tag_set_list); if (list_is_singular(&set->tag_list)) { /* just transitioned to unshared */ set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; @@ -4302,7 +4316,6 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q) blk_mq_update_tag_set_shared(set, false); } mutex_unlock(&set->tag_list_lock); - INIT_LIST_HEAD(&q->tag_set_list); } static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, @@ -4321,7 +4334,7 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, } if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED) queue_set_hctx_shared(q, true); - list_add_tail(&q->tag_set_list, &set->tag_list); + list_add_tail_rcu(&q->tag_set_list, &set->tag_list); mutex_unlock(&set->tag_list_lock); } @@ -4734,38 +4747,45 @@ static void blk_mq_update_queue_map(struct blk_mq_tag_set *set) } } -static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set, - int new_nr_hw_queues) +static struct blk_mq_tags **blk_mq_prealloc_tag_set_tags( + struct blk_mq_tag_set *set, + int new_nr_hw_queues) { struct blk_mq_tags **new_tags; int i; if (set->nr_hw_queues >= new_nr_hw_queues) - goto done; + return NULL; new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *), GFP_KERNEL, set->numa_node); if (!new_tags) - return -ENOMEM; + return ERR_PTR(-ENOMEM); if (set->tags) memcpy(new_tags, set->tags, set->nr_hw_queues * sizeof(*set->tags)); - kfree(set->tags); - set->tags = new_tags; for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) { - if (!__blk_mq_alloc_map_and_rqs(set, i)) { - while (--i >= set->nr_hw_queues) - __blk_mq_free_map_and_rqs(set, i); - return -ENOMEM; + if (blk_mq_is_shared_tags(set->flags)) { + new_tags[i] = set->shared_tags; + } else { + new_tags[i] = blk_mq_alloc_map_and_rqs(set, i, + set->queue_depth); + if (!new_tags[i]) + goto out_unwind; } cond_resched(); } -done: - set->nr_hw_queues = new_nr_hw_queues; - return 0; + return new_tags; +out_unwind: + while (--i >= set->nr_hw_queues) { + if (!blk_mq_is_shared_tags(set->flags)) + blk_mq_free_map_and_rqs(set, new_tags[i], i); + } + kfree(new_tags); + return ERR_PTR(-ENOMEM); } /* @@ -4983,27 +5003,28 @@ struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q, * Switch back to the elevator type stored in the xarray. */ static void blk_mq_elv_switch_back(struct request_queue *q, - struct xarray *elv_tbl, struct xarray *et_tbl) + struct xarray *elv_tbl) { - struct elevator_type *e = xa_load(elv_tbl, q->id); - struct elevator_tags *t = xa_load(et_tbl, q->id); + struct elv_change_ctx *ctx = xa_load(elv_tbl, q->id); + + if (WARN_ON_ONCE(!ctx)) + return; /* The elv_update_nr_hw_queues unfreezes the queue. */ - elv_update_nr_hw_queues(q, e, t); + elv_update_nr_hw_queues(q, ctx); /* Drop the reference acquired in blk_mq_elv_switch_none. */ - if (e) - elevator_put(e); + if (ctx->type) + elevator_put(ctx->type); } /* - * Stores elevator type in xarray and set current elevator to none. It uses - * q->id as an index to store the elevator type into the xarray. + * Stores elevator name and type in ctx and set current elevator to none. */ static int blk_mq_elv_switch_none(struct request_queue *q, struct xarray *elv_tbl) { - int ret = 0; + struct elv_change_ctx *ctx; lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock); @@ -5015,10 +5036,11 @@ static int blk_mq_elv_switch_none(struct request_queue *q, * can't run concurrently. */ if (q->elevator) { + ctx = xa_load(elv_tbl, q->id); + if (WARN_ON_ONCE(!ctx)) + return -ENOENT; - ret = xa_insert(elv_tbl, q->id, q->elevator->type, GFP_KERNEL); - if (WARN_ON_ONCE(ret)) - return ret; + ctx->name = q->elevator->type->elevator_name; /* * Before we switch elevator to 'none', take a reference to @@ -5029,9 +5051,14 @@ static int blk_mq_elv_switch_none(struct request_queue *q, */ __elevator_get(q->elevator->type); + /* + * Store elevator type so that we can release the reference + * taken above later. + */ + ctx->type = q->elevator->type; elevator_set_none(q); } - return ret; + return 0; } static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, @@ -5041,7 +5068,8 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int prev_nr_hw_queues = set->nr_hw_queues; unsigned int memflags; int i; - struct xarray elv_tbl, et_tbl; + struct xarray elv_tbl; + struct blk_mq_tags **new_tags; bool queues_frozen = false; lockdep_assert_held(&set->tag_list_lock); @@ -5055,11 +5083,12 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, memflags = memalloc_noio_save(); - xa_init(&et_tbl); - if (blk_mq_alloc_sched_tags_batch(&et_tbl, set, nr_hw_queues) < 0) - goto out_memalloc_restore; - xa_init(&elv_tbl); + if (blk_mq_alloc_sched_ctx_batch(&elv_tbl, set) < 0) + goto out_free_ctx; + + if (blk_mq_alloc_sched_res_batch(&elv_tbl, set, nr_hw_queues) < 0) + goto out_free_ctx; list_for_each_entry(q, &set->tag_list, tag_set_list) { blk_mq_debugfs_unregister_hctxs(q); @@ -5075,11 +5104,18 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, if (blk_mq_elv_switch_none(q, &elv_tbl)) goto switch_back; + new_tags = blk_mq_prealloc_tag_set_tags(set, nr_hw_queues); + if (IS_ERR(new_tags)) + goto switch_back; + list_for_each_entry(q, &set->tag_list, tag_set_list) blk_mq_freeze_queue_nomemsave(q); queues_frozen = true; - if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0) - goto switch_back; + if (new_tags) { + kfree(set->tags); + set->tags = new_tags; + } + set->nr_hw_queues = nr_hw_queues; fallback: blk_mq_update_queue_map(set); @@ -5105,7 +5141,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, /* switch_back expects queue to be frozen */ if (!queues_frozen) blk_mq_freeze_queue_nomemsave(q); - blk_mq_elv_switch_back(q, &elv_tbl, &et_tbl); + blk_mq_elv_switch_back(q, &elv_tbl); } list_for_each_entry(q, &set->tag_list, tag_set_list) { @@ -5116,9 +5152,9 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, blk_mq_add_hw_queues_cpuhp(q); } +out_free_ctx: + blk_mq_free_sched_ctx_batch(&elv_tbl); xa_destroy(&elv_tbl); - xa_destroy(&et_tbl); -out_memalloc_restore: memalloc_noio_restore(memflags); /* Free the excess tags when nr_hw_queues shrink. */ diff --git a/block/blk-settings.c b/block/blk-settings.c index d74b13ec8e548..f2c1940fe6f1a 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -148,10 +148,9 @@ static int blk_validate_integrity_limits(struct queue_limits *lim) return -EINVAL; } - if (bi->pi_tuple_size > bi->metadata_size) { - pr_warn("pi_tuple_size (%u) exceeds metadata_size (%u)\n", - bi->pi_tuple_size, - bi->metadata_size); + if (bi->pi_offset + bi->pi_tuple_size > bi->metadata_size) { + pr_warn("pi_offset (%u) + pi_tuple_size (%u) exceeds metadata_size (%u)\n", + bi->pi_offset, bi->pi_tuple_size, bi->metadata_size); return -EINVAL; } diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 76c47fe9b8d68..af12526d866a9 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -78,8 +78,14 @@ queue_requests_store(struct gendisk *disk, const char *page, size_t count) /* * Serialize updating nr_requests with concurrent queue_requests_store() * and switching elevator. + * + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) */ - down_write(&set->update_nr_hwq_lock); + if (!down_write_trylock(&set->update_nr_hwq_lock)) + return -EBUSY; if (nr == q->nr_requests) goto unlock; @@ -143,21 +149,22 @@ queue_ra_store(struct gendisk *disk, const char *page, size_t count) { unsigned long ra_kb; ssize_t ret; - unsigned int memflags; struct request_queue *q = disk->queue; ret = queue_var_store(&ra_kb, page, count); if (ret < 0) return ret; /* - * ->ra_pages is protected by ->limits_lock because it is usually - * calculated from the queue limits by queue_limits_commit_update. + * The ->ra_pages change below is protected by ->limits_lock because it + * is usually calculated from the queue limits by + * queue_limits_commit_update(). + * + * bdi->ra_pages reads are not serialized against bdi->ra_pages writes. + * Use WRITE_ONCE() to write bdi->ra_pages once. */ mutex_lock(&q->limits_lock); - memflags = blk_mq_freeze_queue(q); - disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10); + WRITE_ONCE(disk->bdi->ra_pages, ra_kb >> (PAGE_SHIFT - 10)); mutex_unlock(&q->limits_lock); - blk_mq_unfreeze_queue(q, memflags); return ret; } @@ -375,21 +382,18 @@ static ssize_t queue_nomerges_store(struct gendisk *disk, const char *page, size_t count) { unsigned long nm; - unsigned int memflags; struct request_queue *q = disk->queue; ssize_t ret = queue_var_store(&nm, page, count); if (ret < 0) return ret; - memflags = blk_mq_freeze_queue(q); blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q); blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q); if (nm == 2) blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q); else if (nm) blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q); - blk_mq_unfreeze_queue(q, memflags); return ret; } @@ -409,7 +413,6 @@ queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count) #ifdef CONFIG_SMP struct request_queue *q = disk->queue; unsigned long val; - unsigned int memflags; ret = queue_var_store(&val, page, count); if (ret < 0) @@ -421,7 +424,6 @@ queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count) * are accessed individually using atomic test_bit operation. So we * don't grab any lock while updating these flags. */ - memflags = blk_mq_freeze_queue(q); if (val == 2) { blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q); blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q); @@ -432,7 +434,6 @@ queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count) blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q); blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q); } - blk_mq_unfreeze_queue(q, memflags); #endif return ret; } @@ -446,11 +447,9 @@ static ssize_t queue_poll_delay_store(struct gendisk *disk, const char *page, static ssize_t queue_poll_store(struct gendisk *disk, const char *page, size_t count) { - unsigned int memflags; ssize_t ret = count; struct request_queue *q = disk->queue; - memflags = blk_mq_freeze_queue(q); if (!(q->limits.features & BLK_FEAT_POLL)) { ret = -EINVAL; goto out; @@ -459,7 +458,6 @@ static ssize_t queue_poll_store(struct gendisk *disk, const char *page, pr_info_ratelimited("writes to the poll attribute are ignored.\n"); pr_info_ratelimited("please use driver specific parameters instead.\n"); out: - blk_mq_unfreeze_queue(q, memflags); return ret; } @@ -472,7 +470,7 @@ static ssize_t queue_io_timeout_show(struct gendisk *disk, char *page) static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page, size_t count) { - unsigned int val, memflags; + unsigned int val; int err; struct request_queue *q = disk->queue; @@ -480,9 +478,7 @@ static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page, if (err || val == 0) return -EINVAL; - memflags = blk_mq_freeze_queue(q); blk_queue_rq_timeout(q, msecs_to_jiffies(val)); - blk_mq_unfreeze_queue(q, memflags); return count; } @@ -942,7 +938,7 @@ int blk_register_queue(struct gendisk *disk) elevator_set_default(q); blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q); - wbt_enable_default(disk); + wbt_init_enable_default(disk); /* Now everything is ready and send out KOBJ_ADD uevent */ kobject_uevent(&disk->queue_kobj, KOBJ_ADD); diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 2c5b64b1a724a..c19d052a8f2f1 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -22,9 +22,7 @@ #define THROTL_QUANTUM 32 /* Throttling is performed over a slice and after that slice is renewed */ -#define DFL_THROTL_SLICE_HD (HZ / 10) -#define DFL_THROTL_SLICE_SSD (HZ / 50) -#define MAX_THROTL_SLICE (HZ) +#define DFL_THROTL_SLICE (HZ / 10) /* A workqueue to queue throttle related work */ static struct workqueue_struct *kthrotld_workqueue; @@ -1341,10 +1339,7 @@ static int blk_throtl_init(struct gendisk *disk) goto out; } - if (blk_queue_nonrot(q)) - td->throtl_slice = DFL_THROTL_SLICE_SSD; - else - td->throtl_slice = DFL_THROTL_SLICE_HD; + td->throtl_slice = DFL_THROTL_SLICE; td->track_bio_latency = !queue_is_mq(q); if (!td->track_bio_latency) blk_stat_enable_accounting(q); diff --git a/block/blk-wbt.c b/block/blk-wbt.c index eb8037bae0bda..0974875f77bda 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -699,7 +699,7 @@ static void wbt_requeue(struct rq_qos *rqos, struct request *rq) /* * Enable wbt if defaults are configured that way */ -void wbt_enable_default(struct gendisk *disk) +static bool __wbt_enable_default(struct gendisk *disk) { struct request_queue *q = disk->queue; struct rq_qos *rqos; @@ -716,19 +716,31 @@ void wbt_enable_default(struct gendisk *disk) if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT) RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT; mutex_unlock(&disk->rqos_state_mutex); - return; + return false; } mutex_unlock(&disk->rqos_state_mutex); /* Queue not registered? Maybe shutting down... */ if (!blk_queue_registered(q)) - return; + return false; if (queue_is_mq(q) && enable) - wbt_init(disk); + return true; + return false; +} + +void wbt_enable_default(struct gendisk *disk) +{ + __wbt_enable_default(disk); } EXPORT_SYMBOL_GPL(wbt_enable_default); +void wbt_init_enable_default(struct gendisk *disk) +{ + if (__wbt_enable_default(disk)) + WARN_ON_ONCE(wbt_init(disk)); +} + u64 wbt_default_latency_nsec(struct request_queue *q) { /* diff --git a/block/blk-wbt.h b/block/blk-wbt.h index e5fc653b9b76f..925f224757383 100644 --- a/block/blk-wbt.h +++ b/block/blk-wbt.h @@ -5,6 +5,7 @@ #ifdef CONFIG_BLK_WBT int wbt_init(struct gendisk *disk); +void wbt_init_enable_default(struct gendisk *disk); void wbt_disable_default(struct gendisk *disk); void wbt_enable_default(struct gendisk *disk); @@ -16,6 +17,10 @@ u64 wbt_default_latency_nsec(struct request_queue *); #else +static inline void wbt_init_enable_default(struct gendisk *disk) +{ +} + static inline void wbt_disable_default(struct gendisk *disk) { } diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 5e2a5788dc3b2..0c812f3bd7df3 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -71,6 +71,11 @@ struct blk_zone_wplug { struct gendisk *disk; }; +static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk) +{ + return 1U << disk->zone_wplugs_hash_bits; +} + /* * Zone write plug flags bits: * - BLK_ZONE_WPLUG_PLUGGED: Indicates that the zone write plug is plugged, @@ -611,6 +616,8 @@ static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug) { struct bio *bio; + lockdep_assert_held(&zwplug->lock); + if (bio_list_empty(&zwplug->bio_list)) return; @@ -618,6 +625,8 @@ static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug) zwplug->disk->disk_name, zwplug->zone_no); while ((bio = bio_list_pop(&zwplug->bio_list))) blk_zone_wplug_bio_io_error(zwplug, bio); + + zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED; } /* @@ -698,71 +707,94 @@ static int disk_zone_sync_wp_offset(struct gendisk *disk, sector_t sector) disk_report_zones_cb, &args); } -static bool blk_zone_wplug_handle_reset_or_finish(struct bio *bio, - unsigned int wp_offset) +static void blk_zone_reset_bio_endio(struct bio *bio) { struct gendisk *disk = bio->bi_bdev->bd_disk; - sector_t sector = bio->bi_iter.bi_sector; struct blk_zone_wplug *zwplug; - unsigned long flags; - - /* Conventional zones cannot be reset nor finished. */ - if (!bdev_zone_is_seq(bio->bi_bdev, sector)) { - bio_io_error(bio); - return true; - } /* - * No-wait reset or finish BIOs do not make much sense as the callers - * issue these as blocking operations in most cases. To avoid issues - * the BIO execution potentially failing with BLK_STS_AGAIN, warn about - * REQ_NOWAIT being set and ignore that flag. - */ - if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) - bio->bi_opf &= ~REQ_NOWAIT; - - /* - * If we have a zone write plug, set its write pointer offset to 0 - * (reset case) or to the zone size (finish case). This will abort all - * BIOs plugged for the target zone. It is fine as resetting or - * finishing zones while writes are still in-flight will result in the + * If we have a zone write plug, set its write pointer offset to 0. + * This will abort all BIOs plugged for the target zone. It is fine as + * resetting zones while writes are still in-flight will result in the * writes failing anyway. */ - zwplug = disk_get_zone_wplug(disk, sector); + zwplug = disk_get_zone_wplug(disk, bio->bi_iter.bi_sector); if (zwplug) { + unsigned long flags; + spin_lock_irqsave(&zwplug->lock, flags); - disk_zone_wplug_set_wp_offset(disk, zwplug, wp_offset); + disk_zone_wplug_set_wp_offset(disk, zwplug, 0); spin_unlock_irqrestore(&zwplug->lock, flags); disk_put_zone_wplug(zwplug); } - - return false; } -static bool blk_zone_wplug_handle_reset_all(struct bio *bio) +static void blk_zone_reset_all_bio_endio(struct bio *bio) { struct gendisk *disk = bio->bi_bdev->bd_disk; struct blk_zone_wplug *zwplug; unsigned long flags; - sector_t sector; + unsigned int i; + + if (atomic_read(&disk->nr_zone_wplugs)) { + /* Update the condition of all zone write plugs. */ + rcu_read_lock(); + for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++) { + hlist_for_each_entry_rcu(zwplug, + &disk->zone_wplugs_hash[i], + node) { + spin_lock_irqsave(&zwplug->lock, flags); + disk_zone_wplug_set_wp_offset(disk, zwplug, 0); + spin_unlock_irqrestore(&zwplug->lock, flags); + } + } + rcu_read_unlock(); + } +} + +static void blk_zone_finish_bio_endio(struct bio *bio) +{ + struct block_device *bdev = bio->bi_bdev; + struct gendisk *disk = bdev->bd_disk; + struct blk_zone_wplug *zwplug; /* - * Set the write pointer offset of all zone write plugs to 0. This will - * abort all plugged BIOs. It is fine as resetting zones while writes - * are still in-flight will result in the writes failing anyway. + * If we have a zone write plug, set its write pointer offset to the + * zone size. This will abort all BIOs plugged for the target zone. It + * is fine as resetting zones while writes are still in-flight will + * result in the writes failing anyway. */ - for (sector = 0; sector < get_capacity(disk); - sector += disk->queue->limits.chunk_sectors) { - zwplug = disk_get_zone_wplug(disk, sector); - if (zwplug) { - spin_lock_irqsave(&zwplug->lock, flags); - disk_zone_wplug_set_wp_offset(disk, zwplug, 0); - spin_unlock_irqrestore(&zwplug->lock, flags); - disk_put_zone_wplug(zwplug); - } + zwplug = disk_get_zone_wplug(disk, bio->bi_iter.bi_sector); + if (zwplug) { + unsigned long flags; + + spin_lock_irqsave(&zwplug->lock, flags); + disk_zone_wplug_set_wp_offset(disk, zwplug, + bdev_zone_sectors(bdev)); + spin_unlock_irqrestore(&zwplug->lock, flags); + disk_put_zone_wplug(zwplug); } +} - return false; +void blk_zone_mgmt_bio_endio(struct bio *bio) +{ + /* If the BIO failed, we have nothing to do. */ + if (bio->bi_status != BLK_STS_OK) + return; + + switch (bio_op(bio)) { + case REQ_OP_ZONE_RESET: + blk_zone_reset_bio_endio(bio); + return; + case REQ_OP_ZONE_RESET_ALL: + blk_zone_reset_all_bio_endio(bio); + return; + case REQ_OP_ZONE_FINISH: + blk_zone_finish_bio_endio(bio); + return; + default: + return; + } } static void disk_zone_wplug_schedule_bio_work(struct gendisk *disk, @@ -1106,6 +1138,30 @@ static void blk_zone_wplug_handle_native_zone_append(struct bio *bio) disk_put_zone_wplug(zwplug); } +static bool blk_zone_wplug_handle_zone_mgmt(struct bio *bio) +{ + if (bio_op(bio) != REQ_OP_ZONE_RESET_ALL && + !bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) { + /* + * Zone reset and zone finish operations do not apply to + * conventional zones. + */ + bio_io_error(bio); + return true; + } + + /* + * No-wait zone management BIOs do not make much sense as the callers + * issue these as blocking operations in most cases. To avoid issues + * with the BIO execution potentially failing with BLK_STS_AGAIN, warn + * about REQ_NOWAIT being set and ignore that flag. + */ + if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) + bio->bi_opf &= ~REQ_NOWAIT; + + return false; +} + /** * blk_zone_plug_bio - Handle a zone write BIO with zone write plugging * @bio: The BIO being submitted @@ -1153,12 +1209,9 @@ bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) case REQ_OP_WRITE_ZEROES: return blk_zone_wplug_handle_write(bio, nr_segs); case REQ_OP_ZONE_RESET: - return blk_zone_wplug_handle_reset_or_finish(bio, 0); case REQ_OP_ZONE_FINISH: - return blk_zone_wplug_handle_reset_or_finish(bio, - bdev_zone_sectors(bdev)); case REQ_OP_ZONE_RESET_ALL: - return blk_zone_wplug_handle_reset_all(bio); + return blk_zone_wplug_handle_zone_mgmt(bio); default: return false; } @@ -1332,11 +1385,6 @@ static void blk_zone_wplug_bio_work(struct work_struct *work) disk_put_zone_wplug(zwplug); } -static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk) -{ - return 1U << disk->zone_wplugs_hash_bits; -} - void disk_init_zone_resources(struct gendisk *disk) { spin_lock_init(&disk->zone_wplugs_lock); @@ -1516,8 +1564,13 @@ static int disk_update_zone_resources(struct gendisk *disk, { struct request_queue *q = disk->queue; unsigned int nr_seq_zones, nr_conv_zones; - unsigned int pool_size; + unsigned int pool_size, memflags; struct queue_limits lim; + int ret = 0; + + lim = queue_limits_start_update(q); + + memflags = blk_mq_freeze_queue(q); disk->nr_zones = args->nr_zones; disk->zone_capacity = args->zone_capacity; @@ -1527,11 +1580,10 @@ static int disk_update_zone_resources(struct gendisk *disk, if (nr_conv_zones >= disk->nr_zones) { pr_warn("%s: Invalid number of conventional zones %u / %u\n", disk->disk_name, nr_conv_zones, disk->nr_zones); - return -ENODEV; + ret = -ENODEV; + goto unfreeze; } - lim = queue_limits_start_update(q); - /* * Some devices can advertize zone resource limits that are larger than * the number of sequential zones of the zoned block device, e.g. a @@ -1568,7 +1620,15 @@ static int disk_update_zone_resources(struct gendisk *disk, } commit: - return queue_limits_commit_update_frozen(q, &lim); + ret = queue_limits_commit_update(q, &lim); + +unfreeze: + if (ret) + disk_free_zone_resources(disk); + + blk_mq_unfreeze_queue(q, memflags); + + return ret; } static int blk_revalidate_conv_zone(struct blk_zone *zone, unsigned int idx, @@ -1733,7 +1793,7 @@ int blk_revalidate_disk_zones(struct gendisk *disk) sector_t zone_sectors = q->limits.chunk_sectors; sector_t capacity = get_capacity(disk); struct blk_revalidate_zone_args args = { }; - unsigned int noio_flag; + unsigned int memflags, noio_flag; int ret = -ENOMEM; if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) @@ -1783,20 +1843,14 @@ int blk_revalidate_disk_zones(struct gendisk *disk) ret = -ENODEV; } - /* - * Set the new disk zone parameters only once the queue is frozen and - * all I/Os are completed. - */ if (ret > 0) - ret = disk_update_zone_resources(disk, &args); - else - pr_warn("%s: failed to revalidate zones\n", disk->disk_name); - if (ret) { - unsigned int memflags = blk_mq_freeze_queue(q); + return disk_update_zone_resources(disk, &args); - disk_free_zone_resources(disk); - blk_mq_unfreeze_queue(q, memflags); - } + pr_warn("%s: failed to revalidate zones\n", disk->disk_name); + + memflags = blk_mq_freeze_queue(q); + disk_free_zone_resources(disk); + blk_mq_unfreeze_queue(q, memflags); return ret; } diff --git a/block/blk.h b/block/blk.h index 170794632135d..06dfb5b670179 100644 --- a/block/blk.h +++ b/block/blk.h @@ -11,8 +11,7 @@ #include #include "blk-crypto-internal.h" -struct elevator_type; -struct elevator_tags; +struct elv_change_ctx; /* * Default upper limit for the software max_sectors limit used for regular I/Os. @@ -209,10 +208,14 @@ static inline unsigned int blk_queue_get_max_sectors(struct request *rq) struct request_queue *q = rq->q; enum req_op op = req_op(rq); - if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) + if (unlikely(op == REQ_OP_DISCARD)) return min(q->limits.max_discard_sectors, UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_SECURE_ERASE)) + return min(q->limits.max_secure_erase_sectors, + UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_WRITE_ZEROES)) return q->limits.max_write_zeroes_sectors; @@ -333,8 +336,8 @@ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, bool blk_insert_flush(struct request *rq); -void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e, - struct elevator_tags *t); +void elv_update_nr_hw_queues(struct request_queue *q, + struct elv_change_ctx *ctx); void elevator_set_default(struct request_queue *q); void elevator_set_none(struct request_queue *q); @@ -489,9 +492,23 @@ static inline bool blk_req_bio_is_zone_append(struct request *rq, void blk_zone_write_plug_bio_merged(struct bio *bio); void blk_zone_write_plug_init_request(struct request *rq); void blk_zone_append_update_request_bio(struct request *rq, struct bio *bio); +void blk_zone_mgmt_bio_endio(struct bio *bio); void blk_zone_write_plug_bio_endio(struct bio *bio); static inline void blk_zone_bio_endio(struct bio *bio) { + /* + * Zone management BIOs may impact zone write plugs (e.g. a zone reset + * changes a zone write plug zone write pointer offset), but these + * operation do not go through zone write plugging as they may operate + * on zones that do not have a zone write + * plug. blk_zone_mgmt_bio_endio() handles the potential changes to zone + * write plugs that are present. + */ + if (op_is_zone_mgmt(bio_op(bio))) { + blk_zone_mgmt_bio_endio(bio); + return; + } + /* * For write BIOs to zoned devices, signal the completion of the BIO so * that the next write BIO can be submitted by zone write plugging. diff --git a/block/elevator.c b/block/elevator.c index e2ebfbf107b3a..7a97998cd8bd7 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -45,19 +45,6 @@ #include "blk-wbt.h" #include "blk-cgroup.h" -/* Holding context data for changing elevator */ -struct elv_change_ctx { - const char *name; - bool no_uevent; - - /* for unregistering old elevator */ - struct elevator_queue *old; - /* for registering new elevator */ - struct elevator_queue *new; - /* holds sched tags data */ - struct elevator_tags *et; -}; - static DEFINE_SPINLOCK(elv_list_lock); static LIST_HEAD(elv_list); @@ -134,7 +121,7 @@ static struct elevator_type *elevator_find_get(const char *name) static const struct kobj_type elv_ktype; struct elevator_queue *elevator_alloc(struct request_queue *q, - struct elevator_type *e, struct elevator_tags *et) + struct elevator_type *e, struct elevator_resources *res) { struct elevator_queue *eq; @@ -147,7 +134,8 @@ struct elevator_queue *elevator_alloc(struct request_queue *q, kobject_init(&eq->kobj, &elv_ktype); mutex_init(&eq->sysfs_lock); hash_init(eq->hash); - eq->et = et; + eq->et = res->et; + eq->elevator_data = res->data; return eq; } @@ -593,7 +581,7 @@ static int elevator_switch(struct request_queue *q, struct elv_change_ctx *ctx) } if (new_e) { - ret = blk_mq_init_sched(q, new_e, ctx->et); + ret = blk_mq_init_sched(q, new_e, &ctx->res); if (ret) goto out_unfreeze; ctx->new = q->elevator; @@ -617,7 +605,8 @@ static int elevator_switch(struct request_queue *q, struct elv_change_ctx *ctx) return ret; } -static void elv_exit_and_release(struct request_queue *q) +static void elv_exit_and_release(struct elv_change_ctx *ctx, + struct request_queue *q) { struct elevator_queue *e; unsigned memflags; @@ -629,7 +618,7 @@ static void elv_exit_and_release(struct request_queue *q) mutex_unlock(&q->elevator_lock); blk_mq_unfreeze_queue(q, memflags); if (e) { - blk_mq_free_sched_tags(e->et, q->tag_set); + blk_mq_free_sched_res(&ctx->res, ctx->type, q->tag_set); kobject_put(&e->kobj); } } @@ -640,19 +629,19 @@ static int elevator_change_done(struct request_queue *q, int ret = 0; if (ctx->old) { - bool enable_wbt = test_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT, - &ctx->old->flags); + struct elevator_resources res = { + .et = ctx->old->et, + .data = ctx->old->elevator_data + }; elv_unregister_queue(q, ctx->old); - blk_mq_free_sched_tags(ctx->old->et, q->tag_set); + blk_mq_free_sched_res(&res, ctx->old->type, q->tag_set); kobject_put(&ctx->old->kobj); - if (enable_wbt) - wbt_enable_default(q->disk); } if (ctx->new) { ret = elv_register_queue(q, ctx->new, !ctx->no_uevent); if (ret) - elv_exit_and_release(q); + elv_exit_and_release(ctx, q); } return ret; } @@ -669,10 +658,10 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx) lockdep_assert_held(&set->update_nr_hwq_lock); if (strncmp(ctx->name, "none", 4)) { - ctx->et = blk_mq_alloc_sched_tags(set, set->nr_hw_queues, - blk_mq_default_nr_requests(set)); - if (!ctx->et) - return -ENOMEM; + ret = blk_mq_alloc_sched_res(q, ctx->type, &ctx->res, + set->nr_hw_queues); + if (ret) + return ret; } memflags = blk_mq_freeze_queue(q); @@ -693,11 +682,12 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx) blk_mq_unfreeze_queue(q, memflags); if (!ret) ret = elevator_change_done(q, ctx); + /* - * Free sched tags if it's allocated but we couldn't switch elevator. + * Free sched resource if it's allocated but we couldn't switch elevator. */ - if (ctx->et && !ctx->new) - blk_mq_free_sched_tags(ctx->et, set); + if (!ctx->new) + blk_mq_free_sched_res(&ctx->res, ctx->type, set); return ret; } @@ -706,32 +696,29 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx) * The I/O scheduler depends on the number of hardware queues, this forces a * reattachment when nr_hw_queues changes. */ -void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e, - struct elevator_tags *t) +void elv_update_nr_hw_queues(struct request_queue *q, + struct elv_change_ctx *ctx) { struct blk_mq_tag_set *set = q->tag_set; - struct elv_change_ctx ctx = {}; int ret = -ENODEV; WARN_ON_ONCE(q->mq_freeze_depth == 0); - if (e && !blk_queue_dying(q) && blk_queue_registered(q)) { - ctx.name = e->elevator_name; - ctx.et = t; - + if (ctx->type && !blk_queue_dying(q) && blk_queue_registered(q)) { mutex_lock(&q->elevator_lock); /* force to reattach elevator after nr_hw_queue is updated */ - ret = elevator_switch(q, &ctx); + ret = elevator_switch(q, ctx); mutex_unlock(&q->elevator_lock); } blk_mq_unfreeze_queue_nomemrestore(q); if (!ret) - WARN_ON_ONCE(elevator_change_done(q, &ctx)); + WARN_ON_ONCE(elevator_change_done(q, ctx)); + /* - * Free sched tags if it's allocated but we couldn't switch elevator. + * Free sched resource if it's allocated but we couldn't switch elevator. */ - if (t && !ctx.new) - blk_mq_free_sched_tags(t, set); + if (!ctx->new) + blk_mq_free_sched_res(&ctx->res, ctx->type, set); } /* @@ -745,7 +732,6 @@ void elevator_set_default(struct request_queue *q) .no_uevent = true, }; int err; - struct elevator_type *e; /* now we allow to switch elevator */ blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q); @@ -758,8 +744,8 @@ void elevator_set_default(struct request_queue *q) * have multiple queues or mq-deadline is not available, default * to "none". */ - e = elevator_find_get(ctx.name); - if (!e) + ctx.type = elevator_find_get(ctx.name); + if (!ctx.type) return; if ((q->nr_hw_queues == 1 || @@ -769,7 +755,7 @@ void elevator_set_default(struct request_queue *q) pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n", ctx.name, err); } - elevator_put(e); + elevator_put(ctx.type); } void elevator_set_none(struct request_queue *q) @@ -818,8 +804,18 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, ctx.name = strstrip(elevator_name); elv_iosched_load_module(ctx.name); + ctx.type = elevator_find_get(ctx.name); - down_read(&set->update_nr_hwq_lock); + /* + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) + */ + if (!down_read_trylock(&set->update_nr_hwq_lock)) { + ret = -EBUSY; + goto out; + } if (!blk_queue_no_elv_switch(q)) { ret = elevator_change(q, &ctx); if (!ret) @@ -828,6 +824,10 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, ret = -ENOENT; } up_read(&set->update_nr_hwq_lock); + +out: + if (ctx.type) + elevator_put(ctx.type); return ret; } diff --git a/block/elevator.h b/block/elevator.h index c4d20155065e8..0217263760423 100644 --- a/block/elevator.h +++ b/block/elevator.h @@ -32,12 +32,36 @@ struct elevator_tags { struct blk_mq_tags *tags[]; }; +struct elevator_resources { + /* holds elevator data */ + void *data; + /* holds elevator tags */ + struct elevator_tags *et; +}; + +/* Holding context data for changing elevator */ +struct elv_change_ctx { + const char *name; + bool no_uevent; + + /* for unregistering old elevator */ + struct elevator_queue *old; + /* for registering new elevator */ + struct elevator_queue *new; + /* store elevator type */ + struct elevator_type *type; + /* store elevator resources */ + struct elevator_resources res; +}; + struct elevator_mq_ops { int (*init_sched)(struct request_queue *, struct elevator_queue *); void (*exit_sched)(struct elevator_queue *); int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); void (*depth_updated)(struct request_queue *); + void *(*alloc_sched_data)(struct request_queue *); + void (*free_sched_data)(void *); bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); @@ -132,7 +156,6 @@ struct elevator_queue #define ELEVATOR_FLAG_REGISTERED 0 #define ELEVATOR_FLAG_DYING 1 -#define ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT 2 /* * block elevator interface @@ -163,7 +186,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *page, size_t count); extern bool elv_bio_merge_ok(struct request *, struct bio *); struct elevator_queue *elevator_alloc(struct request_queue *, - struct elevator_type *, struct elevator_tags *); + struct elevator_type *, struct elevator_resources *); /* * Helper functions. diff --git a/block/genhd.c b/block/genhd.c index 9bbc38d127926..bd3a6841e5b59 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -90,7 +90,7 @@ bool set_capacity_and_notify(struct gendisk *disk, sector_t size) (disk->flags & GENHD_FL_HIDDEN)) return false; - pr_info("%s: detected capacity change from %lld to %lld\n", + pr_info_ratelimited("%s: detected capacity change from %lld to %lld\n", disk->disk_name, capacity, size); /* diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 18efd6ef2a2b9..e3eaeea62e24d 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -544,7 +544,7 @@ static void kyber_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) * We use the scheduler tags as per-hardware queue queueing tokens. * Async requests can be limited at this stage. */ - if (!op_is_sync(opf)) { + if (!blk_mq_is_sync_read(opf)) { struct kyber_queue_data *kqd = data->q->elevator->elevator_data; data->shallow_depth = kqd->async_depth; diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 3e741d33142d3..29d00221fbea6 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -71,7 +71,6 @@ struct io_stats_per_prio { * present on both sort_list[] and fifo_list[]. */ struct dd_per_prio { - struct list_head dispatch; struct rb_root sort_list[DD_DIR_COUNT]; struct list_head fifo_list[DD_DIR_COUNT]; /* Position of the most recently dispatched request. */ @@ -84,6 +83,7 @@ struct deadline_data { * run time data */ + struct list_head dispatch; struct dd_per_prio per_prio[DD_PRIO_COUNT]; /* Data direction of latest dispatched request. */ @@ -306,6 +306,19 @@ static bool started_after(struct deadline_data *dd, struct request *rq, return time_after(start_time, latest_start); } +static struct request *dd_start_request(struct deadline_data *dd, + enum dd_data_dir data_dir, + struct request *rq) +{ + u8 ioprio_class = dd_rq_ioclass(rq); + enum dd_prio prio = ioprio_class_to_prio[ioprio_class]; + + dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq); + dd->per_prio[prio].stats.dispatched++; + rq->rq_flags |= RQF_STARTED; + return rq; +} + /* * deadline_dispatch_requests selects the best request according to * read/write expire, fifo_batch, etc and with a start time <= @latest_start. @@ -316,21 +329,9 @@ static struct request *__dd_dispatch_request(struct deadline_data *dd, { struct request *rq, *next_rq; enum dd_data_dir data_dir; - enum dd_prio prio; - u8 ioprio_class; lockdep_assert_held(&dd->lock); - if (!list_empty(&per_prio->dispatch)) { - rq = list_first_entry(&per_prio->dispatch, struct request, - queuelist); - if (started_after(dd, rq, latest_start)) - return NULL; - list_del_init(&rq->queuelist); - data_dir = rq_data_dir(rq); - goto done; - } - /* * batches are currently reads XOR writes */ @@ -410,13 +411,7 @@ static struct request *__dd_dispatch_request(struct deadline_data *dd, */ dd->batching++; deadline_move_request(dd, per_prio, rq); -done: - ioprio_class = dd_rq_ioclass(rq); - prio = ioprio_class_to_prio[ioprio_class]; - dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq); - dd->per_prio[prio].stats.dispatched++; - rq->rq_flags |= RQF_STARTED; - return rq; + return dd_start_request(dd, data_dir, rq); } /* @@ -463,6 +458,14 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) enum dd_prio prio; spin_lock(&dd->lock); + + if (!list_empty(&dd->dispatch)) { + rq = list_first_entry(&dd->dispatch, struct request, queuelist); + list_del_init(&rq->queuelist); + dd_start_request(dd, rq_data_dir(rq), rq); + goto unlock; + } + rq = dd_dispatch_prio_aged_requests(dd, now); if (rq) goto unlock; @@ -492,7 +495,7 @@ static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) struct deadline_data *dd = data->q->elevator->elevator_data; /* Do not throttle synchronous reads. */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) return; /* @@ -551,10 +554,10 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) eq->elevator_data = dd; + INIT_LIST_HEAD(&dd->dispatch); for (prio = 0; prio <= DD_PRIO_MAX; prio++) { struct dd_per_prio *per_prio = &dd->per_prio[prio]; - INIT_LIST_HEAD(&per_prio->dispatch); INIT_LIST_HEAD(&per_prio->fifo_list[DD_READ]); INIT_LIST_HEAD(&per_prio->fifo_list[DD_WRITE]); per_prio->sort_list[DD_READ] = RB_ROOT; @@ -658,7 +661,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, trace_block_rq_insert(rq); if (flags & BLK_MQ_INSERT_AT_HEAD) { - list_add(&rq->queuelist, &per_prio->dispatch); + list_add(&rq->queuelist, &dd->dispatch); rq->fifo_time = jiffies; } else { deadline_add_rq_rb(per_prio, rq); @@ -725,8 +728,7 @@ static void dd_finish_request(struct request *rq) static bool dd_has_work_for_prio(struct dd_per_prio *per_prio) { - return !list_empty_careful(&per_prio->dispatch) || - !list_empty_careful(&per_prio->fifo_list[DD_READ]) || + return !list_empty_careful(&per_prio->fifo_list[DD_READ]) || !list_empty_careful(&per_prio->fifo_list[DD_WRITE]); } @@ -735,6 +737,9 @@ static bool dd_has_work(struct blk_mq_hw_ctx *hctx) struct deadline_data *dd = hctx->queue->elevator->elevator_data; enum dd_prio prio; + if (!list_empty_careful(&dd->dispatch)) + return true; + for (prio = 0; prio <= DD_PRIO_MAX; prio++) if (dd_has_work_for_prio(&dd->per_prio[prio])) return true; @@ -943,49 +948,39 @@ static int dd_owned_by_driver_show(void *data, struct seq_file *m) return 0; } -#define DEADLINE_DISPATCH_ATTR(prio) \ -static void *deadline_dispatch##prio##_start(struct seq_file *m, \ - loff_t *pos) \ - __acquires(&dd->lock) \ -{ \ - struct request_queue *q = m->private; \ - struct deadline_data *dd = q->elevator->elevator_data; \ - struct dd_per_prio *per_prio = &dd->per_prio[prio]; \ - \ - spin_lock(&dd->lock); \ - return seq_list_start(&per_prio->dispatch, *pos); \ -} \ - \ -static void *deadline_dispatch##prio##_next(struct seq_file *m, \ - void *v, loff_t *pos) \ -{ \ - struct request_queue *q = m->private; \ - struct deadline_data *dd = q->elevator->elevator_data; \ - struct dd_per_prio *per_prio = &dd->per_prio[prio]; \ - \ - return seq_list_next(v, &per_prio->dispatch, pos); \ -} \ - \ -static void deadline_dispatch##prio##_stop(struct seq_file *m, void *v) \ - __releases(&dd->lock) \ -{ \ - struct request_queue *q = m->private; \ - struct deadline_data *dd = q->elevator->elevator_data; \ - \ - spin_unlock(&dd->lock); \ -} \ - \ -static const struct seq_operations deadline_dispatch##prio##_seq_ops = { \ - .start = deadline_dispatch##prio##_start, \ - .next = deadline_dispatch##prio##_next, \ - .stop = deadline_dispatch##prio##_stop, \ - .show = blk_mq_debugfs_rq_show, \ +static void *deadline_dispatch_start(struct seq_file *m, loff_t *pos) + __acquires(&dd->lock) +{ + struct request_queue *q = m->private; + struct deadline_data *dd = q->elevator->elevator_data; + + spin_lock(&dd->lock); + return seq_list_start(&dd->dispatch, *pos); +} + +static void *deadline_dispatch_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct request_queue *q = m->private; + struct deadline_data *dd = q->elevator->elevator_data; + + return seq_list_next(v, &dd->dispatch, pos); +} + +static void deadline_dispatch_stop(struct seq_file *m, void *v) + __releases(&dd->lock) +{ + struct request_queue *q = m->private; + struct deadline_data *dd = q->elevator->elevator_data; + + spin_unlock(&dd->lock); } -DEADLINE_DISPATCH_ATTR(0); -DEADLINE_DISPATCH_ATTR(1); -DEADLINE_DISPATCH_ATTR(2); -#undef DEADLINE_DISPATCH_ATTR +static const struct seq_operations deadline_dispatch_seq_ops = { + .start = deadline_dispatch_start, + .next = deadline_dispatch_next, + .stop = deadline_dispatch_stop, + .show = blk_mq_debugfs_rq_show, +}; #define DEADLINE_QUEUE_DDIR_ATTRS(name) \ {#name "_fifo_list", 0400, \ @@ -1008,9 +1003,7 @@ static const struct blk_mq_debugfs_attr deadline_queue_debugfs_attrs[] = { {"batching", 0400, deadline_batching_show}, {"starved", 0400, deadline_starved_show}, {"async_depth", 0400, dd_async_depth_show}, - {"dispatch0", 0400, .seq_ops = &deadline_dispatch0_seq_ops}, - {"dispatch1", 0400, .seq_ops = &deadline_dispatch1_seq_ops}, - {"dispatch2", 0400, .seq_ops = &deadline_dispatch2_seq_ops}, + {"dispatch", 0400, .seq_ops = &deadline_dispatch_seq_ops}, {"owned_by_driver", 0400, dd_owned_by_driver_show}, {"queued", 0400, dd_queued_show}, {}, diff --git a/crypto/aead.c b/crypto/aead.c index 5d14b775036ee..51ab3af691af2 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -120,6 +120,7 @@ static int crypto_aead_init_tfm(struct crypto_tfm *tfm) struct aead_alg *alg = crypto_aead_alg(aead); crypto_aead_set_flags(aead, CRYPTO_TFM_NEED_KEY); + crypto_aead_set_reqsize(aead, crypto_tfm_alg_reqsize(tfm)); aead->authsize = alg->maxauthsize; diff --git a/crypto/af_alg.c b/crypto/af_alg.c index ca6fdcc6c54ac..b61c3ba126ed1 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -623,8 +623,10 @@ static int af_alg_alloc_tsgl(struct sock *sk) sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); sgl->cur = 0; - if (sg) + if (sg) { + sg_unmark_end(sg + MAX_SGL_ENTS - 1); sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); + } list_add_tail(&sgl->list, &ctx->tsgl_list); } @@ -635,15 +637,13 @@ static int af_alg_alloc_tsgl(struct sock *sk) /** * af_alg_count_tsgl - Count number of TX SG entries * - * The counting starts from the beginning of the SGL to @bytes. If - * an @offset is provided, the counting of the SG entries starts at the @offset. + * The counting starts from the beginning of the SGL to @bytes. * * @sk: socket of connection to user space * @bytes: Count the number of SG entries holding given number of bytes. - * @offset: Start the counting of SG entries from the given offset. * Return: Number of TX SG entries found given the constraints */ -unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) +unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes) { const struct alg_sock *ask = alg_sk(sk); const struct af_alg_ctx *ctx = ask->private; @@ -658,25 +658,11 @@ unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) const struct scatterlist *sg = sgl->sg; for (i = 0; i < sgl->cur; i++) { - size_t bytes_count; - - /* Skip offset */ - if (offset >= sg[i].length) { - offset -= sg[i].length; - bytes -= sg[i].length; - continue; - } - - bytes_count = sg[i].length - offset; - - offset = 0; sgl_count++; - - /* If we have seen requested number of bytes, stop */ - if (bytes_count >= bytes) + if (sg[i].length >= bytes) return sgl_count; - bytes -= bytes_count; + bytes -= sg[i].length; } } @@ -688,19 +674,14 @@ EXPORT_SYMBOL_GPL(af_alg_count_tsgl); * af_alg_pull_tsgl - Release the specified buffers from TX SGL * * If @dst is non-null, reassign the pages to @dst. The caller must release - * the pages. If @dst_offset is given only reassign the pages to @dst starting - * at the @dst_offset (byte). The caller must ensure that @dst is large - * enough (e.g. by using af_alg_count_tsgl with the same offset). + * the pages. * * @sk: socket of connection to user space * @used: Number of bytes to pull from TX SGL * @dst: If non-NULL, buffer is reassigned to dst SGL instead of releasing. The * caller must release the buffers in dst. - * @dst_offset: Reassign the TX SGL from given offset. All buffers before - * reaching the offset is released. */ -void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, - size_t dst_offset) +void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst) { struct alg_sock *ask = alg_sk(sk); struct af_alg_ctx *ctx = ask->private; @@ -724,19 +705,11 @@ void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, * Assumption: caller created af_alg_count_tsgl(len) * SG entries in dst. */ - if (dst) { - if (dst_offset >= plen) { - /* discard page before offset */ - dst_offset -= plen; - } else { - /* reassign page to dst after offset */ - get_page(page); - sg_set_page(dst + j, page, - plen - dst_offset, - sg[i].offset + dst_offset); - dst_offset = 0; - j++; - } + if (dst && plen) { + /* reassign page to dst */ + get_page(page); + sg_set_page(dst + j, page, plen, sg[i].offset); + j++; } sg[i].length -= plen; @@ -1212,15 +1185,14 @@ struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, if (unlikely(!areq)) return ERR_PTR(-ENOMEM); + memset(areq, 0, areqlen); + ctx->inflight = true; areq->areqlen = areqlen; areq->sk = sk; areq->first_rsgl.sgl.sgt.sgl = areq->first_rsgl.sgl.sgl; - areq->last_rsgl = NULL; INIT_LIST_HEAD(&areq->rsgl_list); - areq->tsgl = NULL; - areq->tsgl_entries = 0; return areq; } @@ -1257,6 +1229,8 @@ int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, seglen = min_t(size_t, (maxsize - len), msg_data_left(msg)); + /* Never pin more pages than the remaining RX accounting budget. */ + seglen = min_t(size_t, seglen, af_alg_rcvbuf(sk)); if (list_empty(&areq->rsgl_list)) { rsgl = &areq->first_rsgl; diff --git a/crypto/ahash.c b/crypto/ahash.c index dfb4f5476428f..66492ae75fcfb 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -423,7 +423,11 @@ static int ahash_update_finish(struct ahash_request *req, int err) req->nbytes += nonzero - blen; - blen = err < 0 ? 0 : err + nonzero; + blen = 0; + if (err >= 0) { + blen = err + nonzero; + err = 0; + } if (ahash_request_isvirt(req)) memcpy(buf, req->svirt + req->nbytes - blen, blen); else @@ -661,6 +665,12 @@ int crypto_ahash_import_core(struct ahash_request *req, const void *in) in); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; + if (crypto_ahash_block_only(tfm)) { + unsigned int reqsize = crypto_ahash_reqsize(tfm); + u8 *buf = ahash_request_ctx(req); + + buf[reqsize - 1] = 0; + } return crypto_ahash_alg(tfm)->import_core(req, in); } EXPORT_SYMBOL_GPL(crypto_ahash_import_core); @@ -674,10 +684,14 @@ int crypto_ahash_import(struct ahash_request *req, const void *in) if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; if (crypto_ahash_block_only(tfm)) { + unsigned int plen = crypto_ahash_blocksize(tfm) + 1; unsigned int reqsize = crypto_ahash_reqsize(tfm); + unsigned int ss = crypto_ahash_statesize(tfm); u8 *buf = ahash_request_ctx(req); - buf[reqsize - 1] = 0; + memcpy(buf + reqsize - plen, in + ss - plen, plen); + if (buf[reqsize - 1] >= plen) + return -EOVERFLOW; } return crypto_ahash_alg(tfm)->import(req, in); } diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 79b016a899a1e..f8bd45f7dc839 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -72,9 +71,8 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, struct alg_sock *pask = alg_sk(psk); struct af_alg_ctx *ctx = ask->private; struct crypto_aead *tfm = pask->private; - unsigned int i, as = crypto_aead_authsize(tfm); + unsigned int as = crypto_aead_authsize(tfm); struct af_alg_async_req *areq; - struct af_alg_tsgl *tsgl, *tmp; struct scatterlist *rsgl_src, *tsgl_src = NULL; int err = 0; size_t used = 0; /* [in] TX bufs to be en/decrypted */ @@ -146,7 +144,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, if (usedpages < outlen) { size_t less = outlen - usedpages; - if (used < less) { + if (used < less + (ctx->enc ? 0 : as)) { err = -EINVAL; goto free; } @@ -154,23 +152,24 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, outlen -= less; } + /* + * Create a per request TX SGL for this request which tracks the + * SG entries from the global TX SGL. + */ processed = used + ctx->aead_assoclen; - list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) { - for (i = 0; i < tsgl->cur; i++) { - struct scatterlist *process_sg = tsgl->sg + i; - - if (!(process_sg->length) || !sg_page(process_sg)) - continue; - tsgl_src = process_sg; - break; - } - if (tsgl_src) - break; - } - if (processed && !tsgl_src) { - err = -EFAULT; + areq->tsgl_entries = af_alg_count_tsgl(sk, processed); + if (!areq->tsgl_entries) + areq->tsgl_entries = 1; + areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), + areq->tsgl_entries), + GFP_KERNEL); + if (!areq->tsgl) { + err = -ENOMEM; goto free; } + sg_init_table(areq->tsgl, areq->tsgl_entries); + af_alg_pull_tsgl(sk, processed, areq->tsgl); + tsgl_src = areq->tsgl; /* * Copy of AAD from source to destination @@ -179,76 +178,15 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, * when user space uses an in-place cipher operation, the kernel * will copy the data as it does not see whether such in-place operation * is initiated. - * - * To ensure efficiency, the following implementation ensure that the - * ciphers are invoked to perform a crypto operation in-place. This - * is achieved by memory management specified as follows. */ /* Use the RX SGL as source (and destination) for crypto op. */ rsgl_src = areq->first_rsgl.sgl.sgt.sgl; - if (ctx->enc) { - /* - * Encryption operation - The in-place cipher operation is - * achieved by the following operation: - * - * TX SGL: AAD || PT - * | | - * | copy | - * v v - * RX SGL: AAD || PT || Tag - */ - memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, - processed); - af_alg_pull_tsgl(sk, processed, NULL, 0); - } else { - /* - * Decryption operation - To achieve an in-place cipher - * operation, the following SGL structure is used: - * - * TX SGL: AAD || CT || Tag - * | | ^ - * | copy | | Create SGL link. - * v v | - * RX SGL: AAD || CT ----+ - */ - - /* Copy AAD || CT to RX SGL buffer for in-place operation. */ - memcpy_sglist(areq->first_rsgl.sgl.sgt.sgl, tsgl_src, outlen); - - /* Create TX SGL for tag and chain it to RX SGL. */ - areq->tsgl_entries = af_alg_count_tsgl(sk, processed, - processed - as); - if (!areq->tsgl_entries) - areq->tsgl_entries = 1; - areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), - areq->tsgl_entries), - GFP_KERNEL); - if (!areq->tsgl) { - err = -ENOMEM; - goto free; - } - sg_init_table(areq->tsgl, areq->tsgl_entries); - - /* Release TX SGL, except for tag data and reassign tag data. */ - af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as); - - /* chain the areq TX SGL holding the tag with RX SGL */ - if (usedpages) { - /* RX SGL present */ - struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl; - struct scatterlist *sg = sgl_prev->sgt.sgl; - - sg_unmark_end(sg + sgl_prev->sgt.nents - 1); - sg_chain(sg, sgl_prev->sgt.nents + 1, areq->tsgl); - } else - /* no RX SGL present (e.g. authentication only) */ - rsgl_src = areq->tsgl; - } + memcpy_sglist(rsgl_src, tsgl_src, ctx->aead_assoclen); /* Initialize the crypto operation */ - aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src, + aead_request_set_crypt(&areq->cra_u.aead_req, tsgl_src, areq->first_rsgl.sgl.sgt.sgl, used, ctx->iv); aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen); aead_request_set_tfm(&areq->cra_u.aead_req, tfm); @@ -450,7 +388,7 @@ static void aead_sock_destruct(struct sock *sk) struct crypto_aead *tfm = pask->private; unsigned int ivlen = crypto_aead_ivsize(tfm); - af_alg_pull_tsgl(sk, ctx->used, NULL, 0); + af_alg_pull_tsgl(sk, ctx->used, NULL); sock_kzfree_s(sk, ctx->iv, ivlen); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c index e3f1a4852737b..4d3dfc60a16a6 100644 --- a/crypto/algif_hash.c +++ b/crypto/algif_hash.c @@ -416,9 +416,8 @@ static int hash_accept_parent_nokey(void *private, struct sock *sk) if (!ctx) return -ENOMEM; - ctx->result = NULL; + memset(ctx, 0, len); ctx->len = len; - ctx->more = false; crypto_init_wait(&ctx->wait); ask->private = ctx; diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c index 10c41adac3b1f..1a86e40c8372e 100644 --- a/crypto/algif_rng.c +++ b/crypto/algif_rng.c @@ -248,9 +248,8 @@ static int rng_accept_parent(void *private, struct sock *sk) if (!ctx) return -ENOMEM; + memset(ctx, 0, len); ctx->len = len; - ctx->addtl = NULL; - ctx->addtl_len = 0; /* * No seeding done at that point -- if multiple accepts are diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 125d395c5e009..ba0a17fd95aca 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -130,6 +130,11 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, * full block size buffers. */ if (ctx->more || len < ctx->used) { + if (len < bs) { + err = -EINVAL; + goto free; + } + len -= len % bs; cflags |= CRYPTO_SKCIPHER_REQ_NOTFINAL; } @@ -138,7 +143,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, * Create a per request TX SGL for this request which tracks the * SG entries from the global TX SGL. */ - areq->tsgl_entries = af_alg_count_tsgl(sk, len, 0); + areq->tsgl_entries = af_alg_count_tsgl(sk, len); if (!areq->tsgl_entries) areq->tsgl_entries = 1; areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl), @@ -149,7 +154,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, goto free; } sg_init_table(areq->tsgl, areq->tsgl_entries); - af_alg_pull_tsgl(sk, len, areq->tsgl, 0); + af_alg_pull_tsgl(sk, len, areq->tsgl); /* Initialize the crypto operation */ skcipher_request_set_tfm(&areq->cra_u.skcipher_req, tfm); @@ -363,7 +368,7 @@ static void skcipher_sock_destruct(struct sock *sk) struct alg_sock *pask = alg_sk(psk); struct crypto_skcipher *tfm = pask->private; - af_alg_pull_tsgl(sk, ctx->used, NULL, 0); + af_alg_pull_tsgl(sk, ctx->used, NULL); sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); if (ctx->state) sock_kzfree_s(sk, ctx->state, crypto_skcipher_statesize(tfm)); diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c index ba2d9d1ea235a..348966ea2175c 100644 --- a/crypto/asymmetric_keys/asymmetric_type.c +++ b/crypto/asymmetric_keys/asymmetric_type.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -141,12 +142,17 @@ struct asymmetric_key_id *asymmetric_key_generate_id(const void *val_1, size_t len_2) { struct asymmetric_key_id *kid; - - kid = kmalloc(sizeof(struct asymmetric_key_id) + len_1 + len_2, - GFP_KERNEL); + size_t kid_sz; + size_t len; + + if (check_add_overflow(len_1, len_2, &len)) + return ERR_PTR(-EOVERFLOW); + if (check_add_overflow(sizeof(struct asymmetric_key_id), len, &kid_sz)) + return ERR_PTR(-EOVERFLOW); + kid = kmalloc(kid_sz, GFP_KERNEL); if (!kid) return ERR_PTR(-ENOMEM); - kid->len = len_1 + len_2; + kid->len = len; memcpy(kid->data, val_1, len_1); memcpy(kid->data + len_1, val_2, len_2); return kid; diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index 8df3fa60a44f8..b56b0d3c443bf 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -584,10 +584,10 @@ int x509_process_extension(void *context, size_t hdrlen, * 0x04 is where keyCertSign lands in this bit string * 0x80 is where digitalSignature lands in this bit string */ - if (v[0] != ASN1_BTS) - return -EBADMSG; if (vlen < 4) return -EBADMSG; + if (v[0] != ASN1_BTS) + return -EBADMSG; if (v[2] >= 8) return -EBADMSG; if (v[3] & 0x80) @@ -620,10 +620,10 @@ int x509_process_extension(void *context, size_t hdrlen, * (Expect 0xFF if the CA is TRUE) * vlen should match the entire extension size */ - if (v[0] != (ASN1_CONS_BIT | ASN1_SEQ)) - return -EBADMSG; if (vlen < 2) return -EBADMSG; + if (v[0] != (ASN1_CONS_BIT | ASN1_SEQ)) + return -EBADMSG; if (v[1] != vlen - 2) return -EBADMSG; /* Empty SEQUENCE means CA:FALSE (default value omitted per DER) */ diff --git a/crypto/authenc.c b/crypto/authenc.c index a723769c87779..ac679ce2cb953 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -37,7 +37,7 @@ struct authenc_request_ctx { static void authenc_request_complete(struct aead_request *req, int err) { - if (err != -EINPROGRESS) + if (err != -EINPROGRESS && err != -EBUSY) aead_request_complete(req, err); } @@ -107,27 +107,42 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, return err; } -static void authenc_geniv_ahash_done(void *data, int err) +static void authenc_geniv_ahash_finish(struct aead_request *req) { - struct aead_request *req = data; struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct aead_instance *inst = aead_alg_instance(authenc); struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); - if (err) - goto out; - scatterwalk_map_and_copy(ahreq->result, req->dst, req->assoclen + req->cryptlen, crypto_aead_authsize(authenc), 1); +} -out: +static void authenc_geniv_ahash_done(void *data, int err) +{ + struct aead_request *req = data; + + if (!err) + authenc_geniv_ahash_finish(req); aead_request_complete(req, err); } -static int crypto_authenc_genicv(struct aead_request *req, unsigned int flags) +/* + * Used when the ahash request was invoked in the async callback context + * of the previous skcipher request. Eat any EINPROGRESS notifications. + */ +static void authenc_geniv_ahash_done2(void *data, int err) +{ + struct aead_request *req = data; + + if (!err) + authenc_geniv_ahash_finish(req); + authenc_request_complete(req, err); +} + +static int crypto_authenc_genicv(struct aead_request *req, unsigned int mask) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct aead_instance *inst = aead_alg_instance(authenc); @@ -136,6 +151,7 @@ static int crypto_authenc_genicv(struct aead_request *req, unsigned int flags) struct crypto_ahash *auth = ctx->auth; struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); + unsigned int flags = aead_request_flags(req) & ~mask; u8 *hash = areq_ctx->tail; int err; @@ -143,7 +159,8 @@ static int crypto_authenc_genicv(struct aead_request *req, unsigned int flags) ahash_request_set_crypt(ahreq, req->dst, hash, req->assoclen + req->cryptlen); ahash_request_set_callback(ahreq, flags, - authenc_geniv_ahash_done, req); + mask ? authenc_geniv_ahash_done2 : + authenc_geniv_ahash_done, req); err = crypto_ahash_digest(ahreq); if (err) @@ -159,12 +176,11 @@ static void crypto_authenc_encrypt_done(void *data, int err) { struct aead_request *areq = data; - if (err) - goto out; - - err = crypto_authenc_genicv(areq, 0); - -out: + if (err) { + aead_request_complete(areq, err); + return; + } + err = crypto_authenc_genicv(areq, CRYPTO_TFM_REQ_MAY_SLEEP); authenc_request_complete(areq, err); } @@ -199,11 +215,18 @@ static int crypto_authenc_encrypt(struct aead_request *req) if (err) return err; - return crypto_authenc_genicv(req, aead_request_flags(req)); + return crypto_authenc_genicv(req, 0); +} + +static void authenc_decrypt_tail_done(void *data, int err) +{ + struct aead_request *req = data; + + authenc_request_complete(req, err); } static int crypto_authenc_decrypt_tail(struct aead_request *req, - unsigned int flags) + unsigned int mask) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct aead_instance *inst = aead_alg_instance(authenc); @@ -214,6 +237,7 @@ static int crypto_authenc_decrypt_tail(struct aead_request *req, struct skcipher_request *skreq = (void *)(areq_ctx->tail + ictx->reqoff); unsigned int authsize = crypto_aead_authsize(authenc); + unsigned int flags = aead_request_flags(req) & ~mask; u8 *ihash = ahreq->result + authsize; struct scatterlist *src, *dst; @@ -230,7 +254,9 @@ static int crypto_authenc_decrypt_tail(struct aead_request *req, skcipher_request_set_tfm(skreq, ctx->enc); skcipher_request_set_callback(skreq, flags, - req->base.complete, req->base.data); + mask ? authenc_decrypt_tail_done : + req->base.complete, + mask ? req : req->base.data); skcipher_request_set_crypt(skreq, src, dst, req->cryptlen - authsize, req->iv); @@ -241,12 +267,11 @@ static void authenc_verify_ahash_done(void *data, int err) { struct aead_request *req = data; - if (err) - goto out; - - err = crypto_authenc_decrypt_tail(req, 0); - -out: + if (err) { + aead_request_complete(req, err); + return; + } + err = crypto_authenc_decrypt_tail(req, CRYPTO_TFM_REQ_MAY_SLEEP); authenc_request_complete(req, err); } @@ -273,7 +298,7 @@ static int crypto_authenc_decrypt(struct aead_request *req) if (err) return err; - return crypto_authenc_decrypt_tail(req, aead_request_flags(req)); + return crypto_authenc_decrypt_tail(req, 0); } static int crypto_authenc_init_tfm(struct crypto_aead *tfm) diff --git a/crypto/authencesn.c b/crypto/authencesn.c index d1bf0fda3f2ef..af3d584e584fb 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -169,6 +169,9 @@ static int crypto_authenc_esn_encrypt(struct aead_request *req) struct scatterlist *src, *dst; int err; + if (assoclen < 8) + return -EINVAL; + sg_init_table(areq_ctx->src, 2); src = scatterwalk_ffwd(areq_ctx->src, req->src, assoclen); dst = src; @@ -204,6 +207,7 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, u8 *ohash = areq_ctx->tail; unsigned int cryptlen = req->cryptlen - authsize; unsigned int assoclen = req->assoclen; + struct scatterlist *src = req->src; struct scatterlist *dst = req->dst; u8 *ihash = ohash + crypto_ahash_digestsize(auth); u32 tmp[2]; @@ -211,23 +215,29 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, if (!authsize) goto decrypt; - /* Move high-order bits of sequence number back. */ - scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); - scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); + if (src == dst) { + /* Move high-order bits of sequence number back. */ + scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); + scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); + } else + memcpy_sglist(dst, src, assoclen); if (crypto_memneq(ihash, ohash, authsize)) return -EBADMSG; decrypt: - sg_init_table(areq_ctx->dst, 2); dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen); + if (req->src == req->dst) + src = dst; + else + src = scatterwalk_ffwd(areq_ctx->src, src, assoclen); skcipher_request_set_tfm(skreq, ctx->enc); skcipher_request_set_callback(skreq, flags, req->base.complete, req->base.data); - skcipher_request_set_crypt(skreq, dst, dst, cryptlen, req->iv); + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); return crypto_skcipher_decrypt(skreq); } @@ -252,28 +262,36 @@ static int crypto_authenc_esn_decrypt(struct aead_request *req) unsigned int assoclen = req->assoclen; unsigned int cryptlen = req->cryptlen; u8 *ihash = ohash + crypto_ahash_digestsize(auth); + struct scatterlist *src = req->src; struct scatterlist *dst = req->dst; u32 tmp[2]; int err; - cryptlen -= authsize; + if (assoclen < 8) + return -EINVAL; - if (req->src != dst) - memcpy_sglist(dst, req->src, assoclen + cryptlen); + if (!authsize) + goto tail; + cryptlen -= authsize; scatterwalk_map_and_copy(ihash, req->src, assoclen + cryptlen, authsize, 0); - if (!authsize) - goto tail; - /* Move high-order bits of sequence number to the end. */ - scatterwalk_map_and_copy(tmp, dst, 0, 8, 0); - scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); - - sg_init_table(areq_ctx->dst, 2); - dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); + scatterwalk_map_and_copy(tmp, src, 0, 8, 0); + if (src == dst) { + scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); + dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); + } else { + scatterwalk_map_and_copy(tmp, dst, 0, 4, 1); + scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen - 4, 4, 1); + + src = scatterwalk_ffwd(areq_ctx->src, src, 8); + dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); + memcpy_sglist(dst, src, assoclen + cryptlen - 8); + dst = req->dst; + } ahash_request_set_tfm(ahreq, auth); ahash_request_set_crypt(ahreq, dst, ohash, assoclen + cryptlen); diff --git a/crypto/deflate.c b/crypto/deflate.c index 21404515dc77e..fd388dad5d60b 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -163,18 +163,21 @@ static int deflate_decompress_one(struct acomp_req *req, do { unsigned int dcur; + unsigned long avail_in; dcur = acomp_walk_next_dst(&walk); - if (!dcur) { - out_of_space = true; - break; - } stream->avail_out = dcur; stream->next_out = walk.dst.virt.addr; + avail_in = stream->avail_in; ret = zlib_inflate(stream, Z_NO_FLUSH); + if (!dcur && avail_in == stream->avail_in) { + out_of_space = true; + break; + } + dcur -= stream->avail_out; acomp_walk_done_dst(&walk, dcur); } while (ret == Z_OK && stream->avail_in); diff --git a/crypto/krb5enc.c b/crypto/krb5enc.c index a1de55994d922..fefa8d2c75322 100644 --- a/crypto/krb5enc.c +++ b/crypto/krb5enc.c @@ -39,12 +39,6 @@ struct krb5enc_request_ctx { char tail[]; }; -static void krb5enc_request_complete(struct aead_request *req, int err) -{ - if (err != -EINPROGRESS) - aead_request_complete(req, err); -} - /** * crypto_krb5enc_extractkeys - Extract Ke and Ki keys from the key blob. * @keys: Where to put the key sizes and pointers @@ -127,7 +121,7 @@ static void krb5enc_encrypt_done(void *data, int err) { struct aead_request *req = data; - krb5enc_request_complete(req, err); + aead_request_complete(req, err); } /* @@ -154,7 +148,7 @@ static int krb5enc_dispatch_encrypt(struct aead_request *req, dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); skcipher_request_set_tfm(skreq, enc); - skcipher_request_set_callback(skreq, aead_request_flags(req), + skcipher_request_set_callback(skreq, flags, krb5enc_encrypt_done, req); skcipher_request_set_crypt(skreq, src, dst, req->cryptlen, req->iv); @@ -188,13 +182,16 @@ static void krb5enc_encrypt_ahash_done(void *data, int err) struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); if (err) - return krb5enc_request_complete(req, err); + goto out; krb5enc_insert_checksum(req, ahreq->result); err = krb5enc_dispatch_encrypt(req, 0); - if (err != -EINPROGRESS) - aead_request_complete(req, err); + if (err == -EINPROGRESS) + return; + +out: + aead_request_complete(req, err); } /* @@ -264,17 +261,16 @@ static void krb5enc_decrypt_hash_done(void *data, int err) { struct aead_request *req = data; - if (err) - return krb5enc_request_complete(req, err); - - err = krb5enc_verify_hash(req); - krb5enc_request_complete(req, err); + if (!err) + err = krb5enc_verify_hash(req); + aead_request_complete(req, err); } /* * Dispatch the hashing of the plaintext after we've done the decryption. */ -static int krb5enc_dispatch_decrypt_hash(struct aead_request *req) +static int krb5enc_dispatch_decrypt_hash(struct aead_request *req, + unsigned int flags) { struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); struct aead_instance *inst = aead_alg_instance(krb5enc); @@ -290,7 +286,7 @@ static int krb5enc_dispatch_decrypt_hash(struct aead_request *req) ahash_request_set_tfm(ahreq, auth); ahash_request_set_crypt(ahreq, req->dst, hash, req->assoclen + req->cryptlen - authsize); - ahash_request_set_callback(ahreq, aead_request_flags(req), + ahash_request_set_callback(ahreq, flags, krb5enc_decrypt_hash_done, req); err = crypto_ahash_digest(ahreq); @@ -300,6 +296,21 @@ static int krb5enc_dispatch_decrypt_hash(struct aead_request *req) return krb5enc_verify_hash(req); } +static void krb5enc_decrypt_done(void *data, int err) +{ + struct aead_request *req = data; + + if (err) + goto out; + + err = krb5enc_dispatch_decrypt_hash(req, 0); + if (err == -EINPROGRESS) + return; + +out: + aead_request_complete(req, err); +} + /* * Dispatch the decryption of the ciphertext. */ @@ -323,7 +334,7 @@ static int krb5enc_dispatch_decrypt(struct aead_request *req) skcipher_request_set_tfm(skreq, ctx->enc); skcipher_request_set_callback(skreq, aead_request_flags(req), - req->base.complete, req->base.data); + krb5enc_decrypt_done, req); skcipher_request_set_crypt(skreq, src, dst, req->cryptlen - authsize, req->iv); @@ -338,7 +349,7 @@ static int krb5enc_decrypt(struct aead_request *req) if (err < 0) return err; - return krb5enc_dispatch_decrypt_hash(req); + return krb5enc_dispatch_decrypt_hash(req, aead_request_flags(req)); } static int krb5enc_init_tfm(struct crypto_aead *tfm) diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 1d010e2a1b1a2..b95e5974e327b 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -101,26 +101,97 @@ void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, } EXPORT_SYMBOL_GPL(memcpy_to_sglist); +/** + * memcpy_sglist() - Copy data from one scatterlist to another + * @dst: The destination scatterlist. Can be NULL if @nbytes == 0. + * @src: The source scatterlist. Can be NULL if @nbytes == 0. + * @nbytes: Number of bytes to copy + * + * The scatterlists can describe exactly the same memory, in which case this + * function is a no-op. No other overlaps are supported. + * + * Context: Any context + */ void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct skcipher_walk walk = {}; + unsigned int src_offset, dst_offset; - if (unlikely(nbytes == 0)) /* in case sg == NULL */ + if (unlikely(nbytes == 0)) /* in case src and/or dst is NULL */ return; - walk.total = nbytes; - - scatterwalk_start(&walk.in, src); - scatterwalk_start(&walk.out, dst); + src_offset = src->offset; + dst_offset = dst->offset; + for (;;) { + /* Compute the length to copy this step. */ + unsigned int len = min3(src->offset + src->length - src_offset, + dst->offset + dst->length - dst_offset, + nbytes); + struct page *src_page = sg_page(src); + struct page *dst_page = sg_page(dst); + const void *src_virt; + void *dst_virt; + + if (IS_ENABLED(CONFIG_HIGHMEM)) { + /* HIGHMEM: we may have to actually map the pages. */ + const unsigned int src_oip = offset_in_page(src_offset); + const unsigned int dst_oip = offset_in_page(dst_offset); + const unsigned int limit = PAGE_SIZE; + + /* Further limit len to not cross a page boundary. */ + len = min3(len, limit - src_oip, limit - dst_oip); + + /* Compute the source and destination pages. */ + src_page += src_offset / PAGE_SIZE; + dst_page += dst_offset / PAGE_SIZE; + + if (src_page != dst_page) { + /* Copy between different pages. */ + memcpy_page(dst_page, dst_oip, + src_page, src_oip, len); + flush_dcache_page(dst_page); + } else if (src_oip != dst_oip) { + /* Copy between different parts of same page. */ + dst_virt = kmap_local_page(dst_page); + memcpy(dst_virt + dst_oip, dst_virt + src_oip, + len); + kunmap_local(dst_virt); + flush_dcache_page(dst_page); + } /* Else, it's the same memory. No action needed. */ + } else { + /* + * !HIGHMEM: no mapping needed. Just work in the linear + * buffer of each sg entry. Note that we can cross page + * boundaries, as they are not significant in this case. + */ + src_virt = page_address(src_page) + src_offset; + dst_virt = page_address(dst_page) + dst_offset; + if (src_virt != dst_virt) { + memcpy(dst_virt, src_virt, len); + if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) + __scatterwalk_flush_dcache_pages( + dst_page, dst_offset, len); + } /* Else, it's the same memory. No action needed. */ + } + nbytes -= len; + if (nbytes == 0) /* No more to copy? */ + break; - skcipher_walk_first(&walk, true); - do { - if (walk.src.virt.addr != walk.dst.virt.addr) - memcpy(walk.dst.virt.addr, walk.src.virt.addr, - walk.nbytes); - skcipher_walk_done(&walk, 0); - } while (walk.nbytes); + /* + * There's more to copy. Advance the offsets by the length + * copied this step, and advance the sg entries as needed. + */ + src_offset += len; + if (src_offset >= src->offset + src->length) { + src = sg_next(src); + src_offset = src->offset; + } + dst_offset += len; + if (dst_offset >= dst->offset + dst->length) { + dst = sg_next(dst); + dst_offset = dst->offset; + } + } } EXPORT_SYMBOL_GPL(memcpy_sglist); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 2bae99e335268..678bb4145d783 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -50,6 +50,7 @@ static int seqiv_aead_encrypt(struct aead_request *req) struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); struct aead_request *subreq = aead_request_ctx(req); crypto_completion_t compl; + bool unaligned_info; void *data; u8 *info; unsigned int ivsize = 8; @@ -68,8 +69,9 @@ static int seqiv_aead_encrypt(struct aead_request *req) memcpy_sglist(req->dst, req->src, req->assoclen + req->cryptlen); - if (unlikely(!IS_ALIGNED((unsigned long)info, - crypto_aead_alignmask(geniv) + 1))) { + unaligned_info = !IS_ALIGNED((unsigned long)info, + crypto_aead_alignmask(geniv) + 1); + if (unlikely(unaligned_info)) { info = kmemdup(req->iv, ivsize, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC); @@ -89,7 +91,7 @@ static int seqiv_aead_encrypt(struct aead_request *req) scatterwalk_map_and_copy(info, req->dst, req->assoclen, ivsize, 1); err = crypto_aead_encrypt(subreq); - if (unlikely(info != req->iv)) + if (unlikely(unaligned_info)) seqiv_aead_encrypt_complete2(req, err); return err; } diff --git a/crypto/zstd.c b/crypto/zstd.c index ac318d333b684..32a339b74f34c 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -75,11 +75,6 @@ static int zstd_init(struct crypto_acomp *acomp_tfm) return ret; } -static void zstd_exit(struct crypto_acomp *acomp_tfm) -{ - crypto_acomp_free_streams(&zstd_streams); -} - static int zstd_compress_one(struct acomp_req *req, struct zstd_ctx *ctx, const void *src, void *dst, unsigned int *dlen) { @@ -297,7 +292,6 @@ static struct acomp_alg zstd_acomp = { .cra_module = THIS_MODULE, }, .init = zstd_init, - .exit = zstd_exit, .compress = zstd_compress, .decompress = zstd_decompress, }; @@ -310,6 +304,7 @@ static int __init zstd_mod_init(void) static void __exit zstd_mod_fini(void) { crypto_unregister_acomp(&zstd_acomp); + crypto_acomp_free_streams(&zstd_streams); } module_init(zstd_mod_init); diff --git a/drivers/Makefile b/drivers/Makefile index 8e1ffa4358d5f..acade03f59425 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -176,6 +176,7 @@ obj-$(CONFIG_RAS) += ras/ obj-$(CONFIG_USB4) += thunderbolt/ obj-$(CONFIG_CORESIGHT) += hwtracing/coresight/ obj-y += hwtracing/intel_th/ +obj-y += hwtracing/qcom/ obj-$(CONFIG_STM) += hwtracing/stm/ obj-$(CONFIG_HISI_PTT) += hwtracing/ptt/ obj-y += android/ diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index e9f9b1fa5dc1b..4610f491f0881 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -189,7 +189,6 @@ aie2_sched_notify(struct amdxdna_sched_job *job) up(&job->hwctx->priv->job_sem); job->job_done = true; - dma_fence_put(fence); mmput_async(job->mm); aie2_job_put(job); } @@ -204,10 +203,13 @@ aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) cmd_abo = job->cmd_bo; - if (unlikely(!data)) + if (unlikely(job->job_timeout)) { + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); + ret = -EINVAL; goto out; + } - if (unlikely(size != sizeof(u32))) { + if (unlikely(!data) || unlikely(size != sizeof(u32))) { amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); ret = -EINVAL; goto out; @@ -260,6 +262,13 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) int ret = 0; cmd_abo = job->cmd_bo; + + if (unlikely(job->job_timeout)) { + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); + ret = -EINVAL; + goto out; + } + if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); ret = -EINVAL; @@ -285,7 +294,7 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) ret = -EINVAL; goto out; } - amdxdna_cmd_set_state(cmd_abo, fail_cmd_status); + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR); if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) { struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL); @@ -308,6 +317,9 @@ aie2_sched_job_run(struct drm_sched_job *sched_job) struct dma_fence *fence; int ret; + if (hwctx->status != HWCTX_STAT_READY) + return NULL; + if (!mmget_not_zero(job->mm)) return ERR_PTR(-ESRCH); @@ -362,6 +374,7 @@ aie2_sched_job_timedout(struct drm_sched_job *sched_job) xdna = hwctx->client->xdna; trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq); + job->job_timeout = true; mutex_lock(&xdna->dev_lock); aie2_hwctx_stop(xdna, hwctx, sched_job); @@ -671,17 +684,22 @@ void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) ndev->hwctx_num--; XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); - drm_sched_entity_destroy(&hwctx->priv->entity); - aie2_hwctx_wait_for_idle(hwctx); /* Request fw to destroy hwctx and cancel the rest pending requests */ + drm_sched_stop(&hwctx->priv->sched, NULL); aie2_release_resource(hwctx); + hwctx->status = HWCTX_STAT_STOP; + drm_sched_start(&hwctx->priv->sched, 0); + + mutex_unlock(&xdna->dev_lock); + drm_sched_entity_destroy(&hwctx->priv->entity); /* Wait for all submitted jobs to be completed or canceled */ wait_event(hwctx->priv->job_free_wq, atomic64_read(&hwctx->job_submit_cnt) == atomic64_read(&hwctx->job_free_cnt)); + mutex_lock(&xdna->dev_lock); drm_sched_fini(&hwctx->priv->sched); aie2_ctx_syncobj_destroy(hwctx); diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c index 87c425e3d2b99..6e07793bbeacf 100644 --- a/drivers/accel/amdxdna/aie2_pci.c +++ b/drivers/accel/amdxdna/aie2_pci.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "aie2_msg_priv.h" #include "aie2_pci.h" @@ -486,6 +487,11 @@ static int aie2_init(struct amdxdna_dev *xdna) unsigned long bars = 0; int i, nvec, ret; + if (!hypervisor_is_type(X86_HYPER_NATIVE)) { + XDNA_ERR(xdna, "Running under hypervisor not supported"); + return -EINVAL; + } + ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL); if (!ndev) return -ENOMEM; @@ -845,7 +851,7 @@ static int aie2_get_hwctx_status(struct amdxdna_client *client, } args->buffer_size -= (u32)(array_args.buffer - args->buffer); - return ret; + return 0; } static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args) @@ -898,6 +904,12 @@ static int aie2_query_ctx_status_array(struct amdxdna_client *client, drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + if (args->element_size > SZ_4K || args->num_element > SZ_1K) { + XDNA_DBG(xdna, "Invalid element size %d or number of element %d", + args->element_size, args->num_element); + return -EINVAL; + } + array_args.element_size = min(args->element_size, sizeof(struct amdxdna_drm_hwctx_entry)); array_args.buffer = args->buffer; @@ -914,7 +926,7 @@ static int aie2_query_ctx_status_array(struct amdxdna_client *client, args->num_element = (u32)((array_args.buffer - args->buffer) / args->element_size); - return ret; + return 0; } static int aie2_get_array(struct amdxdna_client *client, diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c index 4bfe4ef20550f..cfee89681ff3c 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.c +++ b/drivers/accel/amdxdna/amdxdna_ctx.c @@ -104,7 +104,10 @@ void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) if (size) { count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); - if (unlikely(count <= num_masks)) { + if (unlikely(count <= num_masks || + count * sizeof(u32) + + offsetof(struct amdxdna_cmd, data[0]) > + abo->mem.size)) { *size = 0; return NULL; } @@ -389,6 +392,7 @@ void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release"); amdxdna_arg_bos_put(job); amdxdna_gem_put_obj(job->cmd_bo); + dma_fence_put(job->fence); } int amdxdna_cmd_submit(struct amdxdna_client *client, diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h index 7cd7a55936f09..8c1d181df6e79 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.h +++ b/drivers/accel/amdxdna/amdxdna_ctx.h @@ -105,6 +105,7 @@ struct amdxdna_sched_job { /* user can wait on this fence */ struct dma_fence *out_fence; bool job_done; + bool job_timeout; u64 seq; struct amdxdna_gem_obj *cmd_bo; size_t bo_cnt; diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c index d407a36eb4126..a1c9cc4a2b9d8 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -20,8 +20,6 @@ #include "amdxdna_pci_drv.h" #include "amdxdna_ubuf.h" -#define XDNA_MAX_CMD_BO_SIZE SZ_32K - MODULE_IMPORT_NS("DMA_BUF"); static int @@ -392,35 +390,33 @@ static const struct dma_buf_ops amdxdna_dmabuf_ops = { .vunmap = drm_gem_dmabuf_vunmap, }; -static int amdxdna_gem_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map) +static int amdxdna_gem_obj_vmap(struct amdxdna_gem_obj *abo, void **vaddr) { - struct amdxdna_gem_obj *abo = to_xdna_obj(obj); - - iosys_map_clear(map); - - dma_resv_assert_held(obj->resv); + struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); + int ret; if (is_import_bo(abo)) - dma_buf_vmap(abo->dma_buf, map); + ret = dma_buf_vmap_unlocked(abo->dma_buf, &map); else - drm_gem_shmem_object_vmap(obj, map); + ret = drm_gem_vmap(to_gobj(abo), &map); - if (!map->vaddr) - return -ENOMEM; - - return 0; + *vaddr = map.vaddr; + return ret; } -static void amdxdna_gem_obj_vunmap(struct drm_gem_object *obj, struct iosys_map *map) +static void amdxdna_gem_obj_vunmap(struct amdxdna_gem_obj *abo) { - struct amdxdna_gem_obj *abo = to_xdna_obj(obj); + struct iosys_map map; + + if (!abo->mem.kva) + return; - dma_resv_assert_held(obj->resv); + iosys_map_set_vaddr(&map, abo->mem.kva); if (is_import_bo(abo)) - dma_buf_vunmap(abo->dma_buf, map); + dma_buf_vunmap_unlocked(abo->dma_buf, &map); else - drm_gem_shmem_object_vunmap(obj, map); + drm_gem_vunmap(to_gobj(abo), &map); } static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int flags) @@ -455,7 +451,6 @@ static void amdxdna_gem_obj_free(struct drm_gem_object *gobj) { struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); - struct iosys_map map = IOSYS_MAP_INIT_VADDR(abo->mem.kva); XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr); @@ -468,7 +463,7 @@ static void amdxdna_gem_obj_free(struct drm_gem_object *gobj) if (abo->type == AMDXDNA_BO_DEV_HEAP) drm_mm_takedown(&abo->mm); - drm_gem_vunmap(gobj, &map); + amdxdna_gem_obj_vunmap(abo); mutex_destroy(&abo->lock); if (is_import_bo(abo)) { @@ -489,8 +484,8 @@ static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = { .pin = drm_gem_shmem_object_pin, .unpin = drm_gem_shmem_object_unpin, .get_sg_table = drm_gem_shmem_object_get_sg_table, - .vmap = amdxdna_gem_obj_vmap, - .vunmap = amdxdna_gem_obj_vunmap, + .vmap = drm_gem_shmem_object_vmap, + .vunmap = drm_gem_shmem_object_vunmap, .mmap = amdxdna_gem_obj_mmap, .vm_ops = &drm_gem_shmem_vm_ops, .export = amdxdna_gem_prime_export, @@ -663,7 +658,6 @@ amdxdna_drm_create_dev_heap(struct drm_device *dev, struct drm_file *filp) { struct amdxdna_client *client = filp->driver_priv; - struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); struct amdxdna_dev *xdna = to_xdna_dev(dev); struct amdxdna_gem_obj *abo; int ret; @@ -692,12 +686,11 @@ amdxdna_drm_create_dev_heap(struct drm_device *dev, abo->mem.dev_addr = client->xdna->dev_info->dev_mem_base; drm_mm_init(&abo->mm, abo->mem.dev_addr, abo->mem.size); - ret = drm_gem_vmap(to_gobj(abo), &map); + ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva); if (ret) { XDNA_ERR(xdna, "Vmap heap bo failed, ret %d", ret); goto release_obj; } - abo->mem.kva = map.vaddr; client->dev_heap = abo; drm_gem_object_get(to_gobj(abo)); @@ -748,15 +741,8 @@ amdxdna_drm_create_cmd_bo(struct drm_device *dev, struct amdxdna_drm_create_bo *args, struct drm_file *filp) { - struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); struct amdxdna_dev *xdna = to_xdna_dev(dev); struct amdxdna_gem_obj *abo; - int ret; - - if (args->size > XDNA_MAX_CMD_BO_SIZE) { - XDNA_ERR(xdna, "Command bo size 0x%llx too large", args->size); - return ERR_PTR(-EINVAL); - } if (args->size < sizeof(struct amdxdna_cmd)) { XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size); @@ -770,18 +756,7 @@ amdxdna_drm_create_cmd_bo(struct drm_device *dev, abo->type = AMDXDNA_BO_CMD; abo->client = filp->driver_priv; - ret = drm_gem_vmap(to_gobj(abo), &map); - if (ret) { - XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); - goto release_obj; - } - abo->mem.kva = map.vaddr; - return abo; - -release_obj: - drm_gem_object_put(to_gobj(abo)); - return ERR_PTR(ret); } int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) @@ -878,6 +853,7 @@ struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, struct amdxdna_dev *xdna = client->xdna; struct amdxdna_gem_obj *abo; struct drm_gem_object *gobj; + int ret; gobj = drm_gem_object_lookup(client->filp, bo_hdl); if (!gobj) { @@ -886,9 +862,26 @@ struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, } abo = to_xdna_obj(gobj); - if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type) + if (bo_type != AMDXDNA_BO_INVALID && abo->type != bo_type) + goto put_obj; + + if (bo_type != AMDXDNA_BO_CMD || abo->mem.kva) return abo; + if (abo->mem.size > SZ_32K) { + XDNA_ERR(xdna, "Cmd bo is too big %ld", abo->mem.size); + goto put_obj; + } + + ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva); + if (ret) { + XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); + goto put_obj; + } + + return abo; + +put_obj: drm_gem_object_put(gobj); return NULL; } diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c index da1ac89bb78f1..2bacb89cd80c9 100644 --- a/drivers/accel/amdxdna/amdxdna_mailbox.c +++ b/drivers/accel/amdxdna/amdxdna_mailbox.c @@ -112,22 +112,6 @@ static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg) return readl(ringbuf_addr); } -static int mailbox_reg_read_non_zero(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 *val) -{ - struct xdna_mailbox_res *mb_res = &mb_chann->mb->res; - void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg; - int ret, value; - - /* Poll till value is not zero */ - ret = readx_poll_timeout(readl, ringbuf_addr, value, - value, 1 /* us */, 100); - if (ret < 0) - return ret; - - *val = value; - return 0; -} - static inline void mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val) { @@ -206,26 +190,34 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) u32 head, tail; u32 start_addr; u32 tmp_tail; + int ret; head = mailbox_get_headptr(mb_chann, CHAN_RES_X2I); tail = mb_chann->x2i_tail; - ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I); + ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I) - sizeof(u32); start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr; tmp_tail = tail + mb_msg->pkg_size; - if (tail < head && tmp_tail >= head) - goto no_space; - - if (tail >= head && (tmp_tail > ringbuf_size - sizeof(u32) && - mb_msg->pkg_size >= head)) - goto no_space; - if (tail >= head && tmp_tail > ringbuf_size - sizeof(u32)) { +check_again: + if (tail >= head && tmp_tail > ringbuf_size) { write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; writel(TOMBSTONE, write_addr); /* tombstone is set. Write from the start of the ringbuf */ tail = 0; + tmp_tail = tail + mb_msg->pkg_size; + } + + if (tail < head && tmp_tail >= head) { + ret = read_poll_timeout(mailbox_get_headptr, head, + tmp_tail < head || tail >= head, + 1, 100, false, mb_chann, CHAN_RES_X2I); + if (ret) + return ret; + + if (tail >= head) + goto check_again; } write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; @@ -237,9 +229,6 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) mb_msg->pkg.header.id); return 0; - -no_space: - return -ENOSPC; } static int @@ -283,8 +272,7 @@ static int mailbox_get_msg(struct mailbox_channel *mb_chann) u32 start_addr; int ret; - if (mailbox_reg_read_non_zero(mb_chann, mb_chann->res[CHAN_RES_I2X].mb_tail_ptr_reg, &tail)) - return -EINVAL; + tail = mailbox_get_tailptr(mb_chann, CHAN_RES_I2X); head = mb_chann->i2x_head; ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_I2X); start_addr = mb_chann->res[CHAN_RES_I2X].rb_start_addr; @@ -513,6 +501,7 @@ xdna_mailbox_create_channel(struct mailbox *mb, } mb_chann->bad_state = false; + mailbox_reg_write(mb_chann, mb_chann->iohub_int_addr, 0); MB_DBG(mb_chann, "Mailbox channel created (irq: %d)", mb_chann->msix_irq); return mb_chann; diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c index 569cd703729d3..88643e28af84b 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.c +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c @@ -88,6 +88,8 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) ret = -ENODEV; goto unbind_sva; } + client->mm = current->mm; + mmgrab(client->mm); init_srcu_struct(&client->hwctx_srcu); xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); mutex_init(&client->mm_lock); @@ -127,6 +129,7 @@ static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) drm_gem_object_put(to_gobj(client->dev_heap)); iommu_sva_unbind_device(client->sva); + mmdrop(client->mm); XDNA_DBG(xdna, "pid %d closed", client->pid); kfree(client); @@ -292,7 +295,7 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) fs_reclaim_release(GFP_KERNEL); } - xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", 0); + xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM); if (!xdna->notifier_wq) return -ENOMEM; diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h index 72d6696d49da8..64009ca249827 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.h +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -128,6 +128,7 @@ struct amdxdna_client { struct iommu_sva *sva; int pasid; + struct mm_struct *mm; }; #define amdxdna_for_each_hwctx(client, hwctx_id, entry) \ diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c index 077b2261cf2a0..62a478f6b45fb 100644 --- a/drivers/accel/amdxdna/amdxdna_ubuf.c +++ b/drivers/accel/amdxdna/amdxdna_ubuf.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -34,15 +35,21 @@ static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach, ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0, ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL); if (ret) - return ERR_PTR(ret); + goto err_free_sg; if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA) { ret = dma_map_sgtable(attach->dev, sg, direction, 0); if (ret) - return ERR_PTR(ret); + goto err_free_table; } return sg; + +err_free_table: + sg_free_table(sg); +err_free_sg: + kfree(sg); + return ERR_PTR(ret); } static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach, @@ -170,7 +177,10 @@ struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, goto free_ent; } - exp_info.size += va_ent[i].len; + if (check_add_overflow(exp_info.size, va_ent[i].len, &exp_info.size)) { + ret = -EINVAL; + goto free_ent; + } } ubuf->nr_pages = exp_info.size >> PAGE_SHIFT; diff --git a/drivers/accel/ivpu/ivpu_gem.c b/drivers/accel/ivpu/ivpu_gem.c index 59cfcf3eaded9..fda0a18e6d639 100644 --- a/drivers/accel/ivpu/ivpu_gem.c +++ b/drivers/accel/ivpu/ivpu_gem.c @@ -27,8 +27,8 @@ static const struct drm_gem_object_funcs ivpu_gem_funcs; static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) { ivpu_dbg(vdev, BO, - "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n", - action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx_id, + "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n", + action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr, (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, (bool)drm_gem_is_imported(&bo->base.base)); } @@ -43,22 +43,47 @@ static inline void ivpu_bo_unlock(struct ivpu_bo *bo) dma_resv_unlock(bo->base.base.resv); } +static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo) +{ + struct sg_table *sgt; + + drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach); + + ivpu_bo_lock(bo); + + sgt = bo->base.sgt; + if (!sgt) { + sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) + ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt)); + else + bo->base.sgt = sgt; + } + + ivpu_bo_unlock(bo); + + return sgt; +} + /* - * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. + * ivpu_bo_bind() - pin the backing physical pages and map them to VPU. * * This function pins physical memory pages, then maps the physical pages * to IOMMU address space and finally updates the VPU MMU page tables * to allow the VPU to translate VPU address to IOMMU address. */ -int __must_check ivpu_bo_pin(struct ivpu_bo *bo) +int __must_check ivpu_bo_bind(struct ivpu_bo *bo) { struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); struct sg_table *sgt; int ret = 0; - ivpu_dbg_bo(vdev, bo, "pin"); + ivpu_dbg_bo(vdev, bo, "bind"); - sgt = drm_gem_shmem_get_pages_sgt(&bo->base); + if (bo->base.base.import_attach) + sgt = ivpu_bo_map_attachment(vdev, bo); + else + sgt = drm_gem_shmem_get_pages_sgt(&bo->base); if (IS_ERR(sgt)) { ret = PTR_ERR(sgt); ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); @@ -99,7 +124,9 @@ ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); if (!ret) { bo->ctx = ctx; + bo->ctx_id = ctx->id; bo->vpu_addr = bo->mm_node.start; + ivpu_dbg_bo(vdev, bo, "vaddr"); } else { ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret); } @@ -115,7 +142,7 @@ static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) { struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); - lockdep_assert(dma_resv_held(bo->base.base.resv) || !kref_read(&bo->base.base.refcount)); + dma_resv_assert_held(bo->base.base.resv); if (bo->mmu_mapped) { drm_WARN_ON(&vdev->drm, !bo->ctx); @@ -130,13 +157,15 @@ static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) bo->ctx = NULL; } - if (drm_gem_is_imported(&bo->base.base)) - return; - if (bo->base.sgt) { - dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); - sg_free_table(bo->base.sgt); - kfree(bo->base.sgt); + if (bo->base.base.import_attach) { + dma_buf_unmap_attachment(bo->base.base.import_attach, + bo->base.sgt, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); + sg_free_table(bo->base.sgt); + kfree(bo->base.sgt); + } bo->base.sgt = NULL; } } @@ -182,10 +211,11 @@ struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t siz struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) { + struct ivpu_device *vdev = to_ivpu_device(dev); struct device *attach_dev = dev->dev; struct dma_buf_attachment *attach; - struct sg_table *sgt; struct drm_gem_object *obj; + struct ivpu_bo *bo; int ret; attach = dma_buf_attach(dma_buf, attach_dev); @@ -194,25 +224,25 @@ struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, get_dma_buf(dma_buf); - sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL); - if (IS_ERR(sgt)) { - ret = PTR_ERR(sgt); - goto fail_detach; - } - - obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); + obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL); if (IS_ERR(obj)) { ret = PTR_ERR(obj); - goto fail_unmap; + goto fail_detach; } obj->import_attach = attach; obj->resv = dma_buf->resv; + bo = to_ivpu_bo(obj); + + mutex_lock(&vdev->bo_list_lock); + list_add_tail(&bo->bo_list_node, &vdev->bo_list); + mutex_unlock(&vdev->bo_list_lock); + + ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo)); + return obj; -fail_unmap: - dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL); fail_detach: dma_buf_detach(dma_buf, attach); dma_buf_put(dma_buf); @@ -220,7 +250,7 @@ struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, return ERR_PTR(ret); } -static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags, u32 ctx_id) +static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags) { struct drm_gem_shmem_object *shmem; struct ivpu_bo *bo; @@ -238,7 +268,6 @@ static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 fla return ERR_CAST(shmem); bo = to_ivpu_bo(&shmem->base); - bo->ctx_id = ctx_id; bo->base.map_wc = flags & DRM_IVPU_BO_WC; bo->flags = flags; @@ -246,7 +275,7 @@ static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 fla list_add_tail(&bo->bo_list_node, &vdev->bo_list); mutex_unlock(&vdev->bo_list_lock); - ivpu_dbg_bo(vdev, bo, "alloc"); + ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size); return bo; } @@ -281,16 +310,22 @@ static void ivpu_gem_bo_free(struct drm_gem_object *obj) ivpu_dbg_bo(vdev, bo, "free"); + drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node)); + mutex_lock(&vdev->bo_list_lock); list_del(&bo->bo_list_node); - mutex_unlock(&vdev->bo_list_lock); drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) && !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0); drm_WARN_ON(&vdev->drm, bo->base.vaddr); + ivpu_bo_lock(bo); ivpu_bo_unbind_locked(bo); + ivpu_bo_unlock(bo); + + mutex_unlock(&vdev->bo_list_lock); + drm_WARN_ON(&vdev->drm, bo->mmu_mapped); drm_WARN_ON(&vdev->drm, bo->ctx); @@ -326,7 +361,7 @@ int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *fi if (size == 0) return -EINVAL; - bo = ivpu_bo_alloc(vdev, size, args->flags, file_priv->ctx.id); + bo = ivpu_bo_alloc(vdev, size, args->flags); if (IS_ERR(bo)) { ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)", bo, file_priv->ctx.id, args->size, args->flags); @@ -360,7 +395,7 @@ ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); - bo = ivpu_bo_alloc(vdev, size, flags, IVPU_GLOBAL_CONTEXT_MMU_SSID); + bo = ivpu_bo_alloc(vdev, size, flags); if (IS_ERR(bo)) { ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)", bo, range->start, size, flags); @@ -371,7 +406,7 @@ ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, if (ret) goto err_put; - ret = ivpu_bo_pin(bo); + ret = ivpu_bo_bind(bo); if (ret) goto err_put; diff --git a/drivers/accel/ivpu/ivpu_gem.h b/drivers/accel/ivpu/ivpu_gem.h index aa8ff14f7aae1..ade0d127453ff 100644 --- a/drivers/accel/ivpu/ivpu_gem.h +++ b/drivers/accel/ivpu/ivpu_gem.h @@ -24,7 +24,7 @@ struct ivpu_bo { bool mmu_mapped; }; -int ivpu_bo_pin(struct ivpu_bo *bo); +int ivpu_bo_bind(struct ivpu_bo *bo); void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx); struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size); diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c index afdb3b2aa72a7..aa33f562d29c1 100644 --- a/drivers/accel/ivpu/ivpu_hw_btrs.c +++ b/drivers/accel/ivpu/ivpu_hw_btrs.c @@ -752,7 +752,7 @@ int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable) } } -void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 active_percent) +void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u8 active_percent) { u32 val = 0; u32 cmd = enable ? DCT_ENABLE : DCT_DISABLE; diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.h b/drivers/accel/ivpu/ivpu_hw_btrs.h index 032c384ac3d4d..c4c10e22f30f3 100644 --- a/drivers/accel/ivpu/ivpu_hw_btrs.h +++ b/drivers/accel/ivpu/ivpu_hw_btrs.h @@ -36,7 +36,7 @@ u32 ivpu_hw_btrs_dpu_freq_get(struct ivpu_device *vdev); bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq); bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq); int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable); -void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 active_percent); +void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u8 active_percent); u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev); u32 ivpu_hw_btrs_telemetry_size_get(struct ivpu_device *vdev); u32 ivpu_hw_btrs_telemetry_enable_get(struct ivpu_device *vdev); diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c index 060f1fc031d34..1e4caf5726474 100644 --- a/drivers/accel/ivpu/ivpu_job.c +++ b/drivers/accel/ivpu/ivpu_job.c @@ -732,7 +732,7 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 job->bos[i] = to_ivpu_bo(obj); - ret = ivpu_bo_pin(job->bos[i]); + ret = ivpu_bo_bind(job->bos[i]); if (ret) return ret; } @@ -1012,7 +1012,7 @@ void ivpu_context_abort_work_fn(struct work_struct *work) if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) if (ivpu_jsm_reset_engine(vdev, 0)) - return; + goto runtime_put; mutex_lock(&vdev->context_list_lock); xa_for_each(&vdev->context_xa, ctx_id, file_priv) { @@ -1036,7 +1036,7 @@ void ivpu_context_abort_work_fn(struct work_struct *work) goto runtime_put; if (ivpu_jsm_hws_resume_engine(vdev, 0)) - return; + goto runtime_put; /* * In hardware scheduling mode NPU already has stopped processing jobs * and won't send us any further notifications, thus we have to free job related resources diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c index 475ddc94f1cfe..457ccf09df545 100644 --- a/drivers/accel/ivpu/ivpu_pm.c +++ b/drivers/accel/ivpu/ivpu_pm.c @@ -502,6 +502,11 @@ void ivpu_pm_irq_dct_work_fn(struct work_struct *work) else ret = ivpu_pm_dct_disable(vdev); - if (!ret) - ivpu_hw_btrs_dct_set_status(vdev, enable, vdev->pm->dct_active_percent); + if (!ret) { + /* Convert percent to U1.7 format */ + u8 val = DIV_ROUND_CLOSEST(vdev->pm->dct_active_percent * 128, 100); + + ivpu_hw_btrs_dct_set_status(vdev, enable, val); + } + } diff --git a/drivers/accel/qaic/mhi_controller.c b/drivers/accel/qaic/mhi_controller.c index 13a14c6c61689..4d787f77ce419 100644 --- a/drivers/accel/qaic/mhi_controller.c +++ b/drivers/accel/qaic/mhi_controller.c @@ -39,7 +39,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -55,7 +54,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -71,7 +69,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -87,7 +84,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -103,7 +99,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -119,7 +114,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -135,7 +129,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -151,7 +144,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -167,7 +159,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -183,7 +174,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -199,7 +189,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -215,7 +204,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -231,7 +219,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -247,7 +234,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -263,7 +249,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -279,7 +264,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -295,7 +279,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -311,7 +294,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -327,7 +309,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -343,7 +324,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -359,7 +339,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -375,7 +354,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -391,7 +369,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -407,7 +384,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -423,7 +399,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -439,7 +414,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, .wake_capable = false, }, }; @@ -458,7 +432,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -474,7 +447,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -490,7 +462,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -506,7 +477,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -522,7 +492,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -538,7 +507,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -554,7 +522,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -570,7 +537,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -586,7 +552,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -602,7 +567,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -618,7 +582,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -634,7 +597,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -650,7 +612,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -666,7 +627,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -682,7 +642,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -698,7 +657,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -714,7 +672,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -730,7 +687,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, .wake_capable = false, }, }; diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c index b86a8e48e731b..8eae30fe14f98 100644 --- a/drivers/accel/qaic/qaic_control.c +++ b/drivers/accel/qaic/qaic_control.c @@ -910,7 +910,7 @@ static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len */ return -ENODEV; - if (status) { + if (usr && status) { /* * Releasing resources failed on the device side, which puts * us in a bind since they may still be in use, so enable the @@ -1105,6 +1105,9 @@ static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u mutex_lock(&qdev->cntl_mutex); if (!list_empty(&elem.list)) list_del(&elem.list); + /* resp_worker() processed the response but the wait was interrupted */ + else if (ret == -ERESTARTSYS) + ret = 0; if (!ret && !elem.buf) ret = -ETIMEDOUT; else if (ret > 0 && !elem.buf) @@ -1415,9 +1418,49 @@ static void resp_worker(struct work_struct *work) } mutex_unlock(&qdev->cntl_mutex); - if (!found) + if (!found) { + /* + * The user might have gone away at this point without waiting + * for QAIC_TRANS_DEACTIVATE_FROM_DEV transaction coming from + * the device. If this is not handled correctly, the host will + * not know that the DBC[n] has been freed on the device. + * Due to this failure in synchronization between the device and + * the host, if another user requests to activate a network, and + * the device assigns DBC[n] again, save_dbc_buf() will hang, + * waiting for dbc[n]->in_use to be set to false, which will not + * happen unless the qaic_dev_reset_clean_local_state() gets + * called by resetting the device (or re-inserting the module). + * + * As a solution, we look for QAIC_TRANS_DEACTIVATE_FROM_DEV + * transactions in the message before disposing of it, then + * handle releasing the DBC resources. + * + * Since the user has gone away, if the device could not + * deactivate the network (status != 0), there is no way to + * enable and reassign the DBC to the user. We can put trust in + * the device that it will release all the active DBCs in + * response to the QAIC_TRANS_TERMINATE_TO_DEV transaction, + * otherwise, the user can issue an soc_reset to the device. + */ + u32 msg_count = le32_to_cpu(msg->hdr.count); + u32 msg_len = le32_to_cpu(msg->hdr.len); + u32 len = 0; + int j; + + for (j = 0; j < msg_count && len < msg_len; ++j) { + struct wire_trans_hdr *trans_hdr; + + trans_hdr = (struct wire_trans_hdr *)(msg->data + len); + if (le32_to_cpu(trans_hdr->type) == QAIC_TRANS_DEACTIVATE_FROM_DEV) { + if (decode_deactivate(qdev, trans_hdr, &len, NULL)) + len += le32_to_cpu(trans_hdr->len); + } else { + len += le32_to_cpu(trans_hdr->len); + } + } /* request must have timed out, drop packet */ kfree(msg); + } kfree(resp); } diff --git a/drivers/accel/rocket/rocket_core.c b/drivers/accel/rocket/rocket_core.c index abe7719c1db46..b3b2fa9ba645a 100644 --- a/drivers/accel/rocket/rocket_core.c +++ b/drivers/accel/rocket/rocket_core.c @@ -59,8 +59,11 @@ int rocket_core_init(struct rocket_core *core) core->iommu_group = iommu_group_get(dev); err = rocket_job_init(core); - if (err) + if (err) { + iommu_group_put(core->iommu_group); + core->iommu_group = NULL; return err; + } pm_runtime_use_autosuspend(dev); @@ -76,7 +79,7 @@ int rocket_core_init(struct rocket_core *core) err = pm_runtime_resume_and_get(dev); if (err) { - rocket_job_fini(core); + rocket_core_fini(core); return err; } diff --git a/drivers/accel/rocket/rocket_drv.c b/drivers/accel/rocket/rocket_drv.c index 5c0b63f0a8f00..f6ef4c7aeef11 100644 --- a/drivers/accel/rocket/rocket_drv.c +++ b/drivers/accel/rocket/rocket_drv.c @@ -13,6 +13,7 @@ #include #include +#include "rocket_device.h" #include "rocket_drv.h" #include "rocket_gem.h" #include "rocket_job.h" @@ -158,6 +159,8 @@ static const struct drm_driver rocket_drm_driver = { static int rocket_probe(struct platform_device *pdev) { + int ret; + if (rdev == NULL) { /* First core probing, initialize DRM device. */ rdev = rocket_device_init(drm_dev, &rocket_drm_driver); @@ -177,7 +180,17 @@ static int rocket_probe(struct platform_device *pdev) rdev->num_cores++; - return rocket_core_init(&rdev->cores[core]); + ret = rocket_core_init(&rdev->cores[core]); + if (ret) { + rdev->num_cores--; + + if (rdev->num_cores == 0) { + rocket_device_fini(rdev); + rdev = NULL; + } + } + + return ret; } static void rocket_remove(struct platform_device *pdev) diff --git a/drivers/acpi/acpi_pcc.c b/drivers/acpi/acpi_pcc.c index 97064e943768a..e3f302b9dee5f 100644 --- a/drivers/acpi/acpi_pcc.c +++ b/drivers/acpi/acpi_pcc.c @@ -52,7 +52,7 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function, struct pcc_data *data; struct acpi_pcc_info *ctx = handler_context; struct pcc_mbox_chan *pcc_chan; - static acpi_status ret; + acpi_status ret; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 7ec1dc04fd11b..5a562e27d3a80 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -50,6 +50,7 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) { u8 value1 = 0; u8 value2 = 0; + struct pci_dev *ide_dev = NULL, *isa_dev = NULL; if (!dev) @@ -107,12 +108,16 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) * each IDE controller's DMA status to make sure we catch all * DMA activity. */ - dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, + ide_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB, PCI_ANY_ID, PCI_ANY_ID, NULL); - if (dev) { - errata.piix4.bmisx = pci_resource_start(dev, 4); - pci_dev_put(dev); + if (ide_dev) { + errata.piix4.bmisx = pci_resource_start(ide_dev, 4); + if (errata.piix4.bmisx) + dev_dbg(&ide_dev->dev, + "Bus master activity detection (BM-IDE) erratum enabled\n"); + + pci_dev_put(ide_dev); } /* @@ -124,25 +129,23 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) * disable C3 support if this is enabled, as some legacy * devices won't operate well if fast DMA is disabled. */ - dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, + isa_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_0, PCI_ANY_ID, PCI_ANY_ID, NULL); - if (dev) { - pci_read_config_byte(dev, 0x76, &value1); - pci_read_config_byte(dev, 0x77, &value2); - if ((value1 & 0x80) || (value2 & 0x80)) + if (isa_dev) { + pci_read_config_byte(isa_dev, 0x76, &value1); + pci_read_config_byte(isa_dev, 0x77, &value2); + if ((value1 & 0x80) || (value2 & 0x80)) { errata.piix4.fdma = 1; - pci_dev_put(dev); + dev_dbg(&isa_dev->dev, + "Type-F DMA livelock erratum (C3 disabled)\n"); + } + pci_dev_put(isa_dev); } break; } - if (errata.piix4.bmisx) - dev_dbg(&dev->dev, "Bus master activity detection (BM-IDE) erratum enabled\n"); - if (errata.piix4.fdma) - dev_dbg(&dev->dev, "Type-F DMA livelock erratum (C3 disabled)\n"); - return 0; } diff --git a/drivers/acpi/acpica/acpredef.h b/drivers/acpi/acpica/acpredef.h index da2c45880cc7e..c9e65c6a20690 100644 --- a/drivers/acpi/acpica/acpredef.h +++ b/drivers/acpi/acpica/acpredef.h @@ -450,7 +450,7 @@ const union acpi_predefined_info acpi_gbl_predefined_methods[] = { {{"_DSM", METHOD_4ARGS(ACPI_TYPE_BUFFER, ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER, - ACPI_TYPE_ANY | ACPI_TYPE_PACKAGE) | + ACPI_TYPE_PACKAGE | ACPI_TYPE_ANY) | ARG_COUNT_IS_MINIMUM, METHOD_RETURNS(ACPI_RTYPE_ALL)}}, /* Must return a value, but it can be of any type */ diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index fa3475da7ea9b..b6198f73c81df 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -163,7 +163,9 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, return_ACPI_STATUS(AE_NOT_EXIST); } - if (region_obj->region.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) { + if (field_obj + && region_obj->region.space_id == + ACPI_ADR_SPACE_PLATFORM_COMM) { struct acpi_pcc_info *ctx = handler_desc->address_space.context; diff --git a/drivers/acpi/acpica/exoparg3.c b/drivers/acpi/acpica/exoparg3.c index bf08110ed6d25..c8c8c4e49563e 100644 --- a/drivers/acpi/acpica/exoparg3.c +++ b/drivers/acpi/acpica/exoparg3.c @@ -10,6 +10,7 @@ #include #include "accommon.h" #include "acinterp.h" +#include #include "acparser.h" #include "amlcode.h" @@ -51,8 +52,7 @@ ACPI_MODULE_NAME("exoparg3") acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) { union acpi_operand_object **operand = &walk_state->operands[0]; - struct acpi_signal_fatal_info *fatal; - acpi_status status = AE_OK; + struct acpi_signal_fatal_info fatal; ACPI_FUNCTION_TRACE_STR(ex_opcode_3A_0T_0R, acpi_ps_get_opcode_name(walk_state->opcode)); @@ -60,28 +60,23 @@ acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) switch (walk_state->opcode) { case AML_FATAL_OP: /* Fatal (fatal_type fatal_code fatal_arg) */ - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "FatalOp: Type %X Code %X Arg %X " - "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n", - (u32)operand[0]->integer.value, - (u32)operand[1]->integer.value, - (u32)operand[2]->integer.value)); - - fatal = ACPI_ALLOCATE(sizeof(struct acpi_signal_fatal_info)); - if (fatal) { - fatal->type = (u32) operand[0]->integer.value; - fatal->code = (u32) operand[1]->integer.value; - fatal->argument = (u32) operand[2]->integer.value; - } + fatal.type = (u32)operand[0]->integer.value; + fatal.code = (u32)operand[1]->integer.value; + fatal.argument = (u32)operand[2]->integer.value; - /* Always signal the OS! */ + ACPI_BIOS_ERROR((AE_INFO, + "Fatal ACPI BIOS error (Type 0x%X Code 0x%X Arg 0x%X)\n", + fatal.type, fatal.code, fatal.argument)); - status = acpi_os_signal(ACPI_SIGNAL_FATAL, fatal); + /* Always signal the OS! */ - /* Might return while OS is shutting down, just continue */ + acpi_os_signal(ACPI_SIGNAL_FATAL, &fatal); - ACPI_FREE(fatal); - goto cleanup; + /* + * Might return while OS is shutting down, so abort the AML execution + * by returning an error. + */ + return_ACPI_STATUS(AE_ERROR); case AML_EXTERNAL_OP: /* @@ -93,21 +88,16 @@ acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) * wrong if an external opcode ever gets here. */ ACPI_ERROR((AE_INFO, "Executed External Op")); - status = AE_OK; - goto cleanup; + + return_ACPI_STATUS(AE_OK); default: ACPI_ERROR((AE_INFO, "Unknown AML opcode 0x%X", walk_state->opcode)); - status = AE_AML_BAD_OPCODE; - goto cleanup; + return_ACPI_STATUS(AE_AML_BAD_OPCODE); } - -cleanup: - - return_ACPI_STATUS(status); } /******************************************************************************* diff --git a/drivers/acpi/acpica/nswalk.c b/drivers/acpi/acpica/nswalk.c index a2ac06a26e921..5670ff5a43cd4 100644 --- a/drivers/acpi/acpica/nswalk.c +++ b/drivers/acpi/acpica/nswalk.c @@ -169,9 +169,12 @@ acpi_ns_walk_namespace(acpi_object_type type, if (start_node == ACPI_ROOT_OBJECT) { start_node = acpi_gbl_root_node; - if (!start_node) { - return_ACPI_STATUS(AE_NO_NAMESPACE); - } + } + + /* Avoid walking the namespace if the StartNode is NULL */ + + if (!start_node) { + return_ACPI_STATUS(AE_NO_NAMESPACE); } /* Null child means "get first node" */ diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 2c474e6477e12..1a0b85923cd42 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -1,6 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACPI_APEI) += apei.o obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o +# clang versions prior to 18 may blow out the stack with KASAN +ifeq ($(CONFIG_COMPILE_TEST)_$(CONFIG_CC_IS_CLANG)_$(call clang-min-version, 180000),y_y_) +KASAN_SANITIZE_ghes.o := n +endif +obj-$(CONFIG_ACPI_APEI_PCIEAER) += ghes_helpers.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o einj-y := einj-core.o einj-$(CONFIG_ACPI_APEI_EINJ_CXL) += einj-cxl.o diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 97ee19f2cae06..fc96a5e234f06 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -22,12 +22,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -292,6 +294,7 @@ static struct ghes *ghes_new(struct acpi_hest_generic *generic) error_block_length = GHES_ESTATUS_MAX_SIZE; } ghes->estatus = kmalloc(error_block_length, GFP_KERNEL); + ghes->estatus_length = error_block_length; if (!ghes->estatus) { rc = -ENOMEM; goto err_unmap_status_addr; @@ -363,13 +366,15 @@ static int __ghes_check_estatus(struct ghes *ghes, struct acpi_hest_generic_status *estatus) { u32 len = cper_estatus_len(estatus); + u32 max_len = min(ghes->generic->error_block_length, + ghes->estatus_length); if (len < sizeof(*estatus)) { pr_warn_ratelimited(FW_WARN GHES_PFX "Truncated error status block!\n"); return -EIO; } - if (len > ghes->generic->error_block_length) { + if (!len || len > max_len) { pr_warn_ratelimited(FW_WARN GHES_PFX "Invalid error status block length!\n"); return -EIO; } @@ -552,26 +557,49 @@ static bool ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, } static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, - int sev, bool sync) + int sev, bool sync) { struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata); int flags = sync ? MF_ACTION_REQUIRED : 0; + int length = gdata->error_data_length; + char error_type[120]; bool queued = false; int sec_sev, i; char *p; - log_arm_hw_error(err); - sec_sev = ghes_severity(gdata->error_severity); + if (length >= sizeof(*err)) { + log_arm_hw_error(err, sec_sev); + } else { + pr_warn(FW_BUG "arm error length: %d\n", length); + pr_warn(FW_BUG "length is too small\n"); + pr_warn(FW_BUG "firmware-generated error record is incorrect\n"); + return false; + } + if (sev != GHES_SEV_RECOVERABLE || sec_sev != GHES_SEV_RECOVERABLE) return false; p = (char *)(err + 1); + length -= sizeof(err); + for (i = 0; i < err->err_info_num; i++) { - struct cper_arm_err_info *err_info = (struct cper_arm_err_info *)p; - bool is_cache = (err_info->type == CPER_ARM_CACHE_ERROR); - bool has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR); - const char *error_type = "unknown error"; + struct cper_arm_err_info *err_info; + bool is_cache, has_pa; + + /* Ensure we have enough data for the error info header */ + if (length < sizeof(*err_info)) + break; + + err_info = (struct cper_arm_err_info *)p; + + /* Validate the claimed length before using it */ + length -= err_info->length; + if (length < 0) + break; + + is_cache = err_info->type & CPER_ARM_CACHE_ERROR; + has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR); /* * The field (err_info->error_info & BIT(26)) is fixed to set to @@ -585,12 +613,15 @@ static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, continue; } - if (err_info->type < ARRAY_SIZE(cper_proc_error_type_strs)) - error_type = cper_proc_error_type_strs[err_info->type]; + cper_bits_to_str(error_type, sizeof(error_type), + FIELD_GET(CPER_ARM_ERR_TYPE_MASK, err_info->type), + cper_proc_error_type_strs, + ARRAY_SIZE(cper_proc_error_type_strs)); pr_warn_ratelimited(FW_WARN GHES_PFX - "Unhandled processor error type: %s\n", - error_type); + "Unhandled processor error type 0x%02x: %s%s\n", + err_info->type, error_type, + (err_info->type & ~CPER_ARM_ERR_TYPE_MASK) ? " with reserved bit(s)" : ""); p += err_info->length; } @@ -715,24 +746,8 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, struct cxl_cper_prot_err_work_data wd; u8 *dvsec_start, *cap_start; - if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) { - pr_err_ratelimited("CXL CPER invalid agent type\n"); - return; - } - - if (!(prot_err->valid_bits & PROT_ERR_VALID_ERROR_LOG)) { - pr_err_ratelimited("CXL CPER invalid protocol error log\n"); + if (cxl_cper_sec_prot_err_valid(prot_err)) return; - } - - if (prot_err->err_len != sizeof(struct cxl_ras_capability_regs)) { - pr_err_ratelimited("CXL CPER invalid RAS Cap size (%u)\n", - prot_err->err_len); - return; - } - - if (!(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) - pr_warn(FW_WARN "CXL CPER no device serial number\n"); guard(spinlock_irqsave)(&cxl_cper_prot_err_work_lock); @@ -895,11 +910,9 @@ static void ghes_do_proc(struct ghes *ghes, arch_apei_report_mem_error(sev, mem_err); queued = ghes_handle_memory_failure(gdata, sev, sync); - } - else if (guid_equal(sec_type, &CPER_SEC_PCIE)) { + } else if (guid_equal(sec_type, &CPER_SEC_PCIE)) { ghes_handle_aer(gdata); - } - else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { + } else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { queued = ghes_handle_arm_hw_error(gdata, sev, sync); } else if (guid_equal(sec_type, &CPER_SEC_CXL_PROT_ERR)) { struct cxl_cper_sec_prot_err *prot_err = acpi_hest_get_payload(gdata); diff --git a/drivers/acpi/apei/ghes_helpers.c b/drivers/acpi/apei/ghes_helpers.c new file mode 100644 index 0000000000000..f3d162139a974 --- /dev/null +++ b/drivers/acpi/apei/ghes_helpers.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright(c) 2025 Intel Corporation. All rights reserved + +#include +#include + +int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err) +{ + if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) { + pr_err_ratelimited("CXL CPER invalid agent type\n"); + return -EINVAL; + } + + if (!(prot_err->valid_bits & PROT_ERR_VALID_ERROR_LOG)) { + pr_err_ratelimited("CXL CPER invalid protocol error log\n"); + return -EINVAL; + } + + if (prot_err->err_len != sizeof(struct cxl_ras_capability_regs)) { + pr_err_ratelimited("CXL CPER invalid RAS Cap size (%u)\n", + prot_err->err_len); + return -EINVAL; + } + + if ((prot_err->agent_type == RCD || prot_err->agent_type == DEVICE || + prot_err->agent_type == LD || prot_err->agent_type == FMLD) && + !(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) + pr_warn_ratelimited(FW_WARN + "CXL CPER no device serial number\n"); + + return 0; +} +EXPORT_SYMBOL_GPL(cxl_cper_sec_prot_err_valid); diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 67b76492c839c..8196c17b5a970 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -212,7 +212,14 @@ static int acpi_battery_get_property(struct power_supply *psy, if (battery->state & ACPI_BATTERY_STATE_DISCHARGING) val->intval = acpi_battery_handle_discharging(battery); else if (battery->state & ACPI_BATTERY_STATE_CHARGING) - val->intval = POWER_SUPPLY_STATUS_CHARGING; + /* Validate the status by checking the current. */ + if (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && + battery->rate_now == 0) { + /* On charge but no current (0W/0mA). */ + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + } else { + val->intval = POWER_SUPPLY_STATUS_CHARGING; + } else if (battery->state & ACPI_BATTERY_STATE_CHARGE_LIMITING) val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; else if (acpi_battery_is_charged(battery)) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 3bdeeee3414e6..b59b0100d03c5 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -362,7 +362,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) end: if (cmd == CMD_WRITE) { if (unlikely(ret)) { - for_each_possible_cpu(i) { + for_each_online_cpu(i) { struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i); if (!desc) @@ -524,7 +524,7 @@ int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data) else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY) cpu_data->shared_type = CPUFREQ_SHARED_TYPE_ANY; - for_each_possible_cpu(i) { + for_each_online_cpu(i) { if (i == cpu) continue; @@ -1366,7 +1366,8 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) /* Are any of the regs PCC ?*/ if (CPC_IN_PCC(highest_reg) || CPC_IN_PCC(lowest_reg) || CPC_IN_PCC(lowest_non_linear_reg) || CPC_IN_PCC(nominal_reg) || - CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg)) { + CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg) || + CPC_IN_PCC(guaranteed_reg)) { if (pcc_ss_id < 0) { pr_debug("Invalid pcc_ss_id\n"); return -ENODEV; diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 7855bbf752b1e..1f4fc78a124fa 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1655,6 +1655,8 @@ static int acpi_ec_setup(struct acpi_ec *ec, struct acpi_device *device, bool ca ret = ec_install_handlers(ec, device, call_reg); if (ret) { + ec_remove_handlers(ec); + if (ec == first_ec) first_ec = NULL; diff --git a/drivers/acpi/fan.h b/drivers/acpi/fan.h index bedbab0e8e4e9..0d73433c38892 100644 --- a/drivers/acpi/fan.h +++ b/drivers/acpi/fan.h @@ -11,6 +11,7 @@ #define _ACPI_FAN_H_ #include +#include #define ACPI_FAN_DEVICE_IDS \ {"INT3404", }, /* Fan */ \ @@ -60,6 +61,38 @@ struct acpi_fan { struct device_attribute fine_grain_control; }; +/** + * acpi_fan_speed_valid - Check if fan speed value is valid + * @speeed: Speed value returned by the ACPI firmware + * + * Check if the fan speed value returned by the ACPI firmware is valid. This function is + * necessary as ACPI firmware implementations can return 0xFFFFFFFF to signal that the + * ACPI fan does not support speed reporting. Additionally, some buggy ACPI firmware + * implementations return a value larger than the 32-bit integer value defined by + * the ACPI specification when using placeholder values. Such invalid values are also + * detected by this function. + * + * Returns: True if the fan speed value is valid, false otherwise. + */ +static inline bool acpi_fan_speed_valid(u64 speed) +{ + return speed < U32_MAX; +} + +/** + * acpi_fan_power_valid - Check if fan power value is valid + * @power: Power value returned by the ACPI firmware + * + * Check if the fan power value returned by the ACPI firmware is valid. + * See acpi_fan_speed_valid() for details. + * + * Returns: True if the fan power value is valid, false otherwise. + */ +static inline bool acpi_fan_power_valid(u64 power) +{ + return power < U32_MAX; +} + int acpi_fan_get_fst(acpi_handle handle, struct acpi_fan_fst *fst); int acpi_fan_create_attributes(struct acpi_device *device); void acpi_fan_delete_attributes(struct acpi_device *device); diff --git a/drivers/acpi/fan_hwmon.c b/drivers/acpi/fan_hwmon.c index 4b2c2007f2d7f..47a02ef5a6067 100644 --- a/drivers/acpi/fan_hwmon.c +++ b/drivers/acpi/fan_hwmon.c @@ -15,10 +15,6 @@ #include "fan.h" -/* Returned when the ACPI fan does not support speed reporting */ -#define FAN_SPEED_UNAVAILABLE U32_MAX -#define FAN_POWER_UNAVAILABLE U32_MAX - static struct acpi_fan_fps *acpi_fan_get_current_fps(struct acpi_fan *fan, u64 control) { unsigned int i; @@ -77,7 +73,7 @@ static umode_t acpi_fan_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_ * when the associated attribute should not be created. */ for (i = 0; i < fan->fps_count; i++) { - if (fan->fps[i].power != FAN_POWER_UNAVAILABLE) + if (acpi_fan_power_valid(fan->fps[i].power)) return 0444; } @@ -106,7 +102,7 @@ static int acpi_fan_hwmon_read(struct device *dev, enum hwmon_sensor_types type, case hwmon_fan: switch (attr) { case hwmon_fan_input: - if (fst.speed == FAN_SPEED_UNAVAILABLE) + if (!acpi_fan_speed_valid(fst.speed)) return -ENODEV; if (fst.speed > LONG_MAX) @@ -134,7 +130,7 @@ static int acpi_fan_hwmon_read(struct device *dev, enum hwmon_sensor_types type, if (!fps) return -EIO; - if (fps->power == FAN_POWER_UNAVAILABLE) + if (!acpi_fan_power_valid(fps->power)) return -ENODEV; if (fps->power > LONG_MAX / MICROWATT_PER_MILLIWATT) diff --git a/drivers/acpi/osi.c b/drivers/acpi/osi.c index f2c943b934be0..9470f1830ff50 100644 --- a/drivers/acpi/osi.c +++ b/drivers/acpi/osi.c @@ -389,6 +389,19 @@ static const struct dmi_system_id acpi_osi_dmi_table[] __initconst = { }, }, + /* + * The screen backlight turns off during udev device creation + * when returning true for _OSI("Windows 2009") + */ + { + .callback = dmi_disable_osi_win7, + .ident = "Acer Aspire One D255", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "AOD255"), + }, + }, + /* * The wireless hotkey does not work on those machines when * returning true for _OSI("Windows 2012") diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index 5ff343096ece0..fd3ac84b596fa 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -1681,7 +1681,7 @@ acpi_status __init acpi_os_initialize(void) * Use acpi_os_map_generic_address to pre-map the reset * register if it's in system memory. */ - void *rv; + void __iomem *rv; rv = acpi_os_map_generic_address(&acpi_gbl_FADT.reset_register); pr_debug("%s: Reset register mapping %s\n", __func__, diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 361a7721a6a87..7da5ae5594a72 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -1113,6 +1113,19 @@ static const struct dmi_system_id dmi_leave_unused_power_resources_on[] = { DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE Click Mini L9W-B"), }, }, + { + /* + * THUNDEROBOT ZERO laptop: Due to its SSDT table bug, power + * resource 'PXP' will be shut down on initialization, making + * the NVMe #2 and the NVIDIA dGPU both unavailable (they're + * both controlled by 'PXP'). + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "THUNDEROBOT"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZERO"), + } + + }, {} }; diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index 9b6b71a2ffb54..a4498357bd165 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -54,7 +54,7 @@ static int map_x2apic_id(struct acpi_subtable_header *entry, if (!(apic->lapic_flags & ACPI_MADT_ENABLED)) return -ENODEV; - if (device_declaration && (apic->uid == acpi_id)) { + if (apic->uid == acpi_id && (device_declaration || acpi_id < 255)) { *apic_id = apic->local_apic_id; return 0; } diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index 43d5e457814e1..19737f9e1e16f 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -1472,7 +1472,7 @@ static struct fwnode_handle *acpi_graph_get_next_endpoint( if (!prev) { do { - port = fwnode_get_next_child_node(fwnode, port); + port = acpi_get_next_subnode(fwnode, port); /* * The names of the port nodes begin with "port@" * followed by the number of the port node and they also @@ -1490,13 +1490,13 @@ static struct fwnode_handle *acpi_graph_get_next_endpoint( if (!port) return NULL; - endpoint = fwnode_get_next_child_node(port, prev); + endpoint = acpi_get_next_subnode(port, prev); while (!endpoint) { - port = fwnode_get_next_child_node(fwnode, port); + port = acpi_get_next_subnode(fwnode, port); if (!port) break; if (is_acpi_graph_node(port, "port")) - endpoint = fwnode_get_next_child_node(port, NULL); + endpoint = acpi_get_next_subnode(port, NULL); } /* @@ -1714,6 +1714,7 @@ static int acpi_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode, if (fwnode_property_read_u32(fwnode, "reg", &endpoint->id)) fwnode_property_read_u32(fwnode, "endpoint", &endpoint->id); + fwnode_handle_put(port_fwnode); return 0; } diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index d16906f46484d..bc8050d8a6f51 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -532,6 +532,12 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "16T90SP"), }, }, + { + /* JWIPC JVC9100 */ + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "JVC9100"), + }, + }, { } }; @@ -706,6 +712,8 @@ struct irq_override_cmp { static const struct irq_override_cmp override_table[] = { { irq1_level_low_skip_override, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, false }, + { irq1_level_low_skip_override, 10, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 1, false }, + { irq1_level_low_skip_override, 11, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 1, false }, { irq1_edge_low_force_override, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, }; diff --git a/drivers/acpi/riscv/rimt.c b/drivers/acpi/riscv/rimt.c index 7f423405e5ef0..8eaa8731bddd6 100644 --- a/drivers/acpi/riscv/rimt.c +++ b/drivers/acpi/riscv/rimt.c @@ -263,6 +263,13 @@ static int rimt_iommu_xlate(struct device *dev, struct acpi_rimt_node *node, u32 if (!rimt_fwnode) return -EPROBE_DEFER; + /* + * EPROBE_DEFER ensures IOMMU is probed before the devices that + * depend on them. During shutdown, however, the IOMMU may be removed + * first, leading to issues. To avoid this, a device link is added + * which enforces the correct removal order. + */ + device_link_add(dev, rimt_fwnode->dev, DL_FLAG_AUTOREMOVE_CONSUMER); return acpi_iommu_fwspec_init(dev, deviceid, rimt_fwnode); } diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index c8ee8e42b0f64..0b7fa4a8c379c 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -386,6 +386,14 @@ static const struct dmi_system_id acpisleep_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "80E1"), }, }, + { + .callback = init_nvs_save_s3, + .ident = "Lenovo G70-35", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "80Q5"), + }, + }, /* * ThinkPad X1 Tablet(2016) cannot do suspend-to-idle using * the Low Power S0 Idle firmware interface (see diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index dd0b40b9bbe8b..377a268867c21 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -45,6 +45,7 @@ static const struct acpi_device_id lps0_device_ids[] = { #define ACPI_LPS0_EXIT 6 #define ACPI_LPS0_MS_ENTRY 7 #define ACPI_LPS0_MS_EXIT 8 +#define ACPI_MS_TURN_ON_DISPLAY 9 /* AMD */ #define ACPI_LPS0_DSM_UUID_AMD "e3f32452-febc-43ce-9039-932122d37721" @@ -373,6 +374,8 @@ static const char *acpi_sleep_dsm_state_to_str(unsigned int state) return "lps0 ms entry"; case ACPI_LPS0_MS_EXIT: return "lps0 ms exit"; + case ACPI_MS_TURN_ON_DISPLAY: + return "lps0 ms turn on display"; } } else { switch (state) { @@ -619,6 +622,9 @@ void acpi_s2idle_restore_early(void) if (lps0_dsm_func_mask_microsoft > 0) { acpi_sleep_run_lps0_dsm(ACPI_LPS0_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); + /* Intent to turn on display */ + acpi_sleep_run_lps0_dsm(ACPI_MS_TURN_ON_DISPLAY, + lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); /* Modern Standby exit */ acpi_sleep_run_lps0_dsm(ACPI_LPS0_MS_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index 4ee30c2897a2b..418951639f511 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -81,6 +81,18 @@ static const struct override_status_id override_status_ids[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Mipad2"), }), + /* + * Lenovo Yoga Book uses PWM2 for touch keyboard backlight control. + * It needs to be enabled only for the Android device version (YB1-X90* + * aka YETI-11); the Windows version (YB1-X91*) uses ACPI control + * methods. + */ + PRESENT_ENTRY_HID("80862289", "2", INTEL_ATOM_AIRMONT, { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "CHERRYVIEW D1 PLATFORM"), + DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "YETI-11"), + }), + /* * The INT0002 device is necessary to clear wakeup interrupt sources * on Cherry Trail devices, without it we get nobody cared IRQ msgs. diff --git a/drivers/amba/tegra-ahb.c b/drivers/amba/tegra-ahb.c index c0e8b765522dc..f23c3ed018101 100644 --- a/drivers/amba/tegra-ahb.c +++ b/drivers/amba/tegra-ahb.c @@ -144,6 +144,7 @@ int tegra_ahb_enable_smmu(struct device_node *dn) if (!dev) return -EPROBE_DEFER; ahb = dev_get_drvdata(dev); + put_device(dev); val = gizmo_readl(ahb, AHB_ARBITRATION_XBAR_CTRL); val |= AHB_ARBITRATION_XBAR_CTRL_SMMU_INIT_DONE; gizmo_writel(ahb, val, AHB_ARBITRATION_XBAR_CTRL); diff --git a/drivers/android/binder.c b/drivers/android/binder.c index a3a1b5c33ba36..a6b3bfe0d9b81 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2991,6 +2991,10 @@ static void binder_set_txn_from_error(struct binder_transaction *t, int id, * @t: the binder transaction that failed * @data_size: the user provided data size for the transaction * @error: enum binder_driver_return_protocol returned to sender + * + * Note that t->buffer is not safe to access here, as it may have been + * released (or not yet allocated). Callers should guarantee all the + * transaction items used here are safe to access. */ static void binder_netlink_report(struct binder_proc *proc, struct binder_transaction *t, @@ -3780,6 +3784,14 @@ static void binder_transaction(struct binder_proc *proc, goto err_dead_proc_or_thread; } } else { + /* + * Make a transaction copy. It is not safe to access 't' after + * binder_proc_transaction() reported a pending frozen. The + * target could thaw and consume the transaction at any point. + * Instead, use a safe 't_copy' for binder_netlink_report(). + */ + struct binder_transaction t_copy = *t; + BUG_ON(target_node == NULL); BUG_ON(t->buffer->async_transaction != 1); return_error = binder_proc_transaction(t, target_proc, NULL); @@ -3790,7 +3802,7 @@ static void binder_transaction(struct binder_proc *proc, */ if (return_error == BR_TRANSACTION_PENDING_FROZEN) { tcomplete->type = BINDER_WORK_TRANSACTION_PENDING; - binder_netlink_report(proc, t, tr->data_size, + binder_netlink_report(proc, &t_copy, tr->data_size, return_error); } binder_enqueue_thread_work(thread, tcomplete); @@ -3812,8 +3824,9 @@ static void binder_transaction(struct binder_proc *proc, return; err_dead_proc_or_thread: - binder_txn_error("%d:%d dead process or thread\n", - thread->pid, proc->pid); + binder_txn_error("%d:%d %s process or thread\n", + proc->pid, thread->pid, + return_error == BR_FROZEN_REPLY ? "frozen" : "dead"); return_error_line = __LINE__; binder_dequeue_work(proc, tcomplete); err_translate_failed: @@ -4510,7 +4523,7 @@ static int binder_thread_write(struct binder_proc *proc, } } binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%d:%d BC_DEAD_BINDER_DONE %016llx found %pK\n", + "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n", proc->pid, thread->pid, (u64)cookie, death); if (death == NULL) { diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs index 08d362deaf61e..c26d113ede966 100644 --- a/drivers/android/binder/node.rs +++ b/drivers/android/binder/node.rs @@ -541,10 +541,10 @@ impl Node { guard = self.owner.inner.lock(); } - let death_list = core::mem::take(&mut self.inner.access_mut(&mut guard).death_list); - drop(guard); - for death in death_list { + while let Some(death) = self.inner.access_mut(&mut guard).death_list.pop_front() { + drop(guard); death.into_arc().set_dead(); + guard = self.owner.inner.lock(); } } diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs index 9379038f61f51..3d5bfaeda4a35 100644 --- a/drivers/android/binder/page_range.rs +++ b/drivers/android/binder/page_range.rs @@ -13,6 +13,8 @@ // // The shrinker will use trylock methods because it locks them in a different order. +use crate::AssertSync; + use core::{ marker::PhantomPinned, mem::{size_of, size_of_val, MaybeUninit}, @@ -142,6 +144,30 @@ pub(crate) struct ShrinkablePageRange { _pin: PhantomPinned, } +// We do not define any ops. For now, used only to check identity of vmas. +static BINDER_VM_OPS: AssertSync = AssertSync(pin_init::zeroed()); + +// To ensure that we do not accidentally install pages into or zap pages from the wrong vma, we +// check its vm_ops and private data before using it. +fn check_vma(vma: &virt::VmaRef, owner: *const ShrinkablePageRange) -> Option<&virt::VmaMixedMap> { + // SAFETY: Just reading the vm_ops pointer of any active vma is safe. + let vm_ops = unsafe { (*vma.as_ptr()).vm_ops }; + if !ptr::eq(vm_ops, &BINDER_VM_OPS.0) { + return None; + } + + // SAFETY: Reading the vm_private_data pointer of a binder-owned vma is safe. + let vm_private_data = unsafe { (*vma.as_ptr()).vm_private_data }; + // The ShrinkablePageRange is only dropped when the Process is dropped, which only happens once + // the file's ->release handler is invoked, which means the ShrinkablePageRange outlives any + // VMA associated with it, so there can't be any false positives due to pointer reuse here. + if !ptr::eq(vm_private_data, owner.cast()) { + return None; + } + + vma.as_mixedmap_vma() +} + struct Inner { /// Array of pages. /// @@ -308,6 +334,18 @@ impl ShrinkablePageRange { inner.size = num_pages; inner.vma_addr = vma.start(); + // This pointer is only used for comparison - it's not dereferenced. + // + // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on + // `vm_private_data`. + unsafe { + (*vma.as_ptr()).vm_private_data = ptr::from_ref(self).cast_mut().cast::() + }; + + // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on + // `vm_ops`. + unsafe { (*vma.as_ptr()).vm_ops = &BINDER_VM_OPS.0 }; + Ok(num_pages) } @@ -399,22 +437,24 @@ impl ShrinkablePageRange { // // Using `mmput_async` avoids this, because then the `mm` cleanup is instead queued to a // workqueue. - MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?) - .mmap_read_lock() - .vma_lookup(vma_addr) - .ok_or(ESRCH)? - .as_mixedmap_vma() - .ok_or(ESRCH)? - .vm_insert_page(user_page_addr, &new_page) - .inspect_err(|err| { - pr_warn!( - "Failed to vm_insert_page({}): vma_addr:{} i:{} err:{:?}", - user_page_addr, - vma_addr, - i, - err - ) - })?; + check_vma( + MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?) + .mmap_read_lock() + .vma_lookup(vma_addr) + .ok_or(ESRCH)?, + self, + ) + .ok_or(ESRCH)? + .vm_insert_page(user_page_addr, &new_page) + .inspect_err(|err| { + pr_warn!( + "Failed to vm_insert_page({}): vma_addr:{} i:{} err:{:?}", + user_page_addr, + vma_addr, + i, + err + ) + })?; let inner = self.lock.lock(); @@ -667,12 +707,15 @@ unsafe extern "C" fn rust_shrink_free_page( let mmap_read; let mm_mutex; let vma_addr; + let range_ptr; { // CAST: The `list_head` field is first in `PageInfo`. let info = item as *mut PageInfo; // SAFETY: The `range` field of `PageInfo` is immutable. - let range = unsafe { &*((*info).range) }; + range_ptr = unsafe { (*info).range }; + // SAFETY: The `range` outlives its `PageInfo` values. + let range = unsafe { &*range_ptr }; mm = match range.mm.mmget_not_zero() { Some(mm) => MmWithUser::into_mmput_async(mm), @@ -717,9 +760,11 @@ unsafe extern "C" fn rust_shrink_free_page( // SAFETY: The lru lock is locked when this method is called. unsafe { bindings::spin_unlock(&raw mut (*lru).lock) }; - if let Some(vma) = mmap_read.vma_lookup(vma_addr) { - let user_page_addr = vma_addr + (page_index << PAGE_SHIFT); - vma.zap_page_range_single(user_page_addr, PAGE_SIZE); + if let Some(unchecked_vma) = mmap_read.vma_lookup(vma_addr) { + if let Some(vma) = check_vma(unchecked_vma, range_ptr) { + let user_page_addr = vma_addr + (page_index << PAGE_SHIFT); + vma.zap_page_range_single(user_page_addr, PAGE_SIZE); + } } drop(mmap_read); @@ -727,8 +772,5 @@ unsafe extern "C" fn rust_shrink_free_page( drop(mm); drop(page); - // SAFETY: We just unlocked the lru lock, but it should be locked when we return. - unsafe { bindings::spin_lock(&raw mut (*lru).lock) }; - LRU_REMOVED_ENTRY } diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index 7607353a5e924..7d66d8cba932d 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -1259,7 +1259,8 @@ impl Process { } pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) { - if let Some(death) = self.inner.lock().pull_delivered_death(cookie) { + let death = self.inner.lock().pull_delivered_death(cookie); + if let Some(death) = death { death.set_notification_done(thread); } } @@ -1362,8 +1363,12 @@ impl Process { work.into_arc().cancel(); } - let delivered_deaths = take(&mut self.inner.lock().delivered_deaths); - drop(delivered_deaths); + // Clear delivered_deaths list. + // + // Scope ensures that MutexGuard is dropped while executing the body. + while let Some(delivered_death) = { self.inner.lock().delivered_deaths.pop_front() } { + drop(delivered_death); + } // Free any resources kept alive by allocated buffers. let omapping = self.inner.lock().mapping.take(); diff --git a/drivers/android/binder/range_alloc/array.rs b/drivers/android/binder/range_alloc/array.rs index 07e1dec2ce630..ada1d1b4302e5 100644 --- a/drivers/android/binder/range_alloc/array.rs +++ b/drivers/android/binder/range_alloc/array.rs @@ -118,7 +118,7 @@ impl ArrayRangeAllocator { size: usize, is_oneway: bool, pid: Pid, - ) -> Result { + ) -> Result<(usize, bool)> { // Compute new value of free_oneway_space, which is set only on success. let new_oneway_space = if is_oneway { match self.free_oneway_space.checked_sub(size) { @@ -146,7 +146,38 @@ impl ArrayRangeAllocator { .ok() .unwrap(); - Ok(insert_at_offset) + // Start detecting spammers once we have less than 20% + // of async space left (which is less than 10% of total + // buffer size). + // + // (This will short-circuit, so `low_oneway_space` is + // only called when necessary.) + let oneway_spam_detected = + is_oneway && new_oneway_space < self.size / 10 && self.low_oneway_space(pid); + + Ok((insert_at_offset, oneway_spam_detected)) + } + + /// Find the amount and size of buffers allocated by the current caller. + /// + /// The idea is that once we cross the threshold, whoever is responsible + /// for the low async space is likely to try to send another async transaction, + /// and at some point we'll catch them in the act. This is more efficient + /// than keeping a map per pid. + fn low_oneway_space(&self, calling_pid: Pid) -> bool { + let mut total_alloc_size = 0; + let mut num_buffers = 0; + + // Warn if this pid has more than 50 transactions, or more than 50% of + // async space (which is 25% of total buffer size). Oneway spam is only + // detected when the threshold is exceeded. + for range in &self.ranges { + if range.state.is_oneway() && range.state.pid() == calling_pid { + total_alloc_size += range.size; + num_buffers += 1; + } + } + num_buffers > 50 || total_alloc_size > self.size / 4 } pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result { diff --git a/drivers/android/binder/range_alloc/mod.rs b/drivers/android/binder/range_alloc/mod.rs index 2301e2bc1a1fc..1f4734468ff11 100644 --- a/drivers/android/binder/range_alloc/mod.rs +++ b/drivers/android/binder/range_alloc/mod.rs @@ -188,11 +188,11 @@ impl RangeAllocator { self.reserve_new(args) } Impl::Array(array) => { - let offset = + let (offset, oneway_spam_detected) = array.reserve_new(args.debug_id, args.size, args.is_oneway, args.pid)?; Ok(ReserveNew::Success(ReserveNewSuccess { offset, - oneway_spam_detected: false, + oneway_spam_detected, _empty_array_alloc: args.empty_array_alloc, _new_tree_alloc: args.new_tree_alloc, _tree_alloc: args.tree_alloc, diff --git a/drivers/android/binder/range_alloc/tree.rs b/drivers/android/binder/range_alloc/tree.rs index 7b1a248fcb026..27c451f73614a 100644 --- a/drivers/android/binder/range_alloc/tree.rs +++ b/drivers/android/binder/range_alloc/tree.rs @@ -164,15 +164,6 @@ impl TreeRangeAllocator { self.free_oneway_space }; - // Start detecting spammers once we have less than 20% - // of async space left (which is less than 10% of total - // buffer size). - // - // (This will short-circut, so `low_oneway_space` is - // only called when necessary.) - let oneway_spam_detected = - is_oneway && new_oneway_space < self.size / 10 && self.low_oneway_space(pid); - let (found_size, found_off, tree_node, free_tree_node) = match self.find_best_match(size) { None => { pr_warn!("ENOSPC from range_alloc.reserve_new - size: {}", size); @@ -203,6 +194,15 @@ impl TreeRangeAllocator { self.free_tree.insert(free_tree_node); } + // Start detecting spammers once we have less than 20% + // of async space left (which is less than 10% of total + // buffer size). + // + // (This will short-circuit, so `low_oneway_space` is + // only called when necessary.) + let oneway_spam_detected = + is_oneway && new_oneway_space < self.size / 10 && self.low_oneway_space(pid); + Ok((found_off, oneway_spam_detected)) } diff --git a/drivers/android/binder/rust_binder_main.rs b/drivers/android/binder/rust_binder_main.rs index 6773b7c273ec9..2b04e83835f6c 100644 --- a/drivers/android/binder/rust_binder_main.rs +++ b/drivers/android/binder/rust_binder_main.rs @@ -300,7 +300,7 @@ impl kernel::Module for BinderModule { /// Makes the inner type Sync. #[repr(transparent)] pub struct AssertSync(T); -// SAFETY: Used only to insert `file_operations` into a global, which is safe. +// SAFETY: Used only to insert C bindings types into globals, which is safe. unsafe impl Sync for AssertSync {} /// File operations that rust_binderfs.c can use. diff --git a/drivers/android/binder/rust_binderfs.c b/drivers/android/binder/rust_binderfs.c index 6b497146b698b..c7e26753f1df5 100644 --- a/drivers/android/binder/rust_binderfs.c +++ b/drivers/android/binder/rust_binderfs.c @@ -132,8 +132,8 @@ static int binderfs_binder_device_create(struct inode *ref_inode, mutex_lock(&binderfs_minors_mutex); if (++info->device_count <= info->mount_opts.max) minor = ida_alloc_max(&binderfs_minors, - use_reserve ? BINDERFS_MAX_MINOR : - BINDERFS_MAX_MINOR_CAPPED, + use_reserve ? BINDERFS_MAX_MINOR - 1 : + BINDERFS_MAX_MINOR_CAPPED - 1, GFP_KERNEL); else minor = -ENOSPC; @@ -416,8 +416,8 @@ static int binderfs_binder_ctl_create(struct super_block *sb) /* Reserve a new minor number for the new device. */ mutex_lock(&binderfs_minors_mutex); minor = ida_alloc_max(&binderfs_minors, - use_reserve ? BINDERFS_MAX_MINOR : - BINDERFS_MAX_MINOR_CAPPED, + use_reserve ? BINDERFS_MAX_MINOR - 1 : + BINDERFS_MAX_MINOR_CAPPED - 1, GFP_KERNEL); mutex_unlock(&binderfs_minors_mutex); if (minor < 0) { diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs index 7e34ccd394f80..6a2ddf0039c14 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -39,6 +39,10 @@ use core::{ sync::atomic::{AtomicU32, Ordering}, }; +fn is_aligned(value: usize, to: usize) -> bool { + value % to == 0 +} + /// Stores the layout of the scatter-gather entries. This is used during the `translate_objects` /// call and is discarded when it returns. struct ScatterGatherState { @@ -69,17 +73,24 @@ struct ScatterGatherEntry { } /// This entry specifies that a fixup should happen at `target_offset` of the -/// buffer. If `skip` is nonzero, then the fixup is a `binder_fd_array_object` -/// and is applied later. Otherwise if `skip` is zero, then the size of the -/// fixup is `sizeof::()` and `pointer_value` is written to the buffer. -struct PointerFixupEntry { - /// The number of bytes to skip, or zero for a `binder_buffer_object` fixup. - skip: usize, - /// The translated pointer to write when `skip` is zero. - pointer_value: u64, - /// The offset at which the value should be written. The offset is relative - /// to the original buffer. - target_offset: usize, +/// buffer. +enum PointerFixupEntry { + /// A fixup for a `binder_buffer_object`. + Fixup { + /// The translated pointer to write. + pointer_value: u64, + /// The offset at which the value should be written. The offset is relative + /// to the original buffer. + target_offset: usize, + }, + /// A skip for a `binder_fd_array_object`. + Skip { + /// The number of bytes to skip. + skip: usize, + /// The offset at which the skip should happen. The offset is relative + /// to the original buffer. + target_offset: usize, + }, } /// Return type of `apply_and_validate_fixup_in_parent`. @@ -762,8 +773,7 @@ impl Thread { parent_entry.fixup_min_offset = info.new_min_offset; parent_entry.pointer_fixups.push( - PointerFixupEntry { - skip: 0, + PointerFixupEntry::Fixup { pointer_value: buffer_ptr_in_user_space, target_offset: info.target_offset, }, @@ -789,6 +799,10 @@ impl Thread { let num_fds = usize::try_from(obj.num_fds).map_err(|_| EINVAL)?; let fds_len = num_fds.checked_mul(size_of::()).ok_or(EINVAL)?; + if !is_aligned(parent_offset, size_of::()) { + return Err(EINVAL.into()); + } + let info = sg_state.validate_parent_fixup(parent_index, parent_offset, fds_len)?; view.alloc.info_add_fd_reserve(num_fds)?; @@ -803,13 +817,16 @@ impl Thread { } }; + if !is_aligned(parent_entry.sender_uaddr, size_of::()) { + return Err(EINVAL.into()); + } + parent_entry.fixup_min_offset = info.new_min_offset; parent_entry .pointer_fixups .push( - PointerFixupEntry { + PointerFixupEntry::Skip { skip: fds_len, - pointer_value: 0, target_offset: info.target_offset, }, GFP_KERNEL, @@ -820,6 +837,7 @@ impl Thread { .sender_uaddr .checked_add(parent_offset) .ok_or(EINVAL)?; + let mut fda_bytes = KVec::new(); UserSlice::new(UserPtr::from_addr(fda_uaddr as _), fds_len) .read_all(&mut fda_bytes, GFP_KERNEL)?; @@ -871,17 +889,21 @@ impl Thread { let mut reader = UserSlice::new(UserPtr::from_addr(sg_entry.sender_uaddr), sg_entry.length).reader(); for fixup in &mut sg_entry.pointer_fixups { - let fixup_len = if fixup.skip == 0 { - size_of::() - } else { - fixup.skip + let (fixup_len, fixup_offset) = match fixup { + PointerFixupEntry::Fixup { target_offset, .. } => { + (size_of::(), *target_offset) + } + PointerFixupEntry::Skip { + skip, + target_offset, + } => (*skip, *target_offset), }; - let target_offset_end = fixup.target_offset.checked_add(fixup_len).ok_or(EINVAL)?; - if fixup.target_offset < end_of_previous_fixup || offset_end < target_offset_end { + let target_offset_end = fixup_offset.checked_add(fixup_len).ok_or(EINVAL)?; + if fixup_offset < end_of_previous_fixup || offset_end < target_offset_end { pr_warn!( "Fixups oob {} {} {} {}", - fixup.target_offset, + fixup_offset, end_of_previous_fixup, offset_end, target_offset_end @@ -890,13 +912,13 @@ impl Thread { } let copy_off = end_of_previous_fixup; - let copy_len = fixup.target_offset - end_of_previous_fixup; + let copy_len = fixup_offset - end_of_previous_fixup; if let Err(err) = alloc.copy_into(&mut reader, copy_off, copy_len) { pr_warn!("Failed copying into alloc: {:?}", err); return Err(err.into()); } - if fixup.skip == 0 { - let res = alloc.write::(fixup.target_offset, &fixup.pointer_value); + if let PointerFixupEntry::Fixup { pointer_value, .. } = fixup { + let res = alloc.write::(fixup_offset, pointer_value); if let Err(err) = res { pr_warn!("Failed copying ptr into alloc: {:?}", err); return Err(err.into()); @@ -949,25 +971,30 @@ impl Thread { let data_size = trd.data_size.try_into().map_err(|_| EINVAL)?; let aligned_data_size = ptr_align(data_size).ok_or(EINVAL)?; - let offsets_size = trd.offsets_size.try_into().map_err(|_| EINVAL)?; - let aligned_offsets_size = ptr_align(offsets_size).ok_or(EINVAL)?; - let buffers_size = tr.buffers_size.try_into().map_err(|_| EINVAL)?; - let aligned_buffers_size = ptr_align(buffers_size).ok_or(EINVAL)?; + let offsets_size: usize = trd.offsets_size.try_into().map_err(|_| EINVAL)?; + let buffers_size: usize = tr.buffers_size.try_into().map_err(|_| EINVAL)?; let aligned_secctx_size = match secctx.as_ref() { Some((_offset, ctx)) => ptr_align(ctx.len()).ok_or(EINVAL)?, None => 0, }; + if !is_aligned(offsets_size, size_of::()) { + return Err(EINVAL.into()); + } + if !is_aligned(buffers_size, size_of::()) { + return Err(EINVAL.into()); + } + // This guarantees that at least `sizeof(usize)` bytes will be allocated. let len = usize::max( aligned_data_size - .checked_add(aligned_offsets_size) - .and_then(|sum| sum.checked_add(aligned_buffers_size)) + .checked_add(offsets_size) + .and_then(|sum| sum.checked_add(buffers_size)) .and_then(|sum| sum.checked_add(aligned_secctx_size)) .ok_or(ENOMEM)?, - size_of::(), + size_of::(), ); - let secctx_off = aligned_data_size + aligned_offsets_size + aligned_buffers_size; + let secctx_off = aligned_data_size + offsets_size + buffers_size; let mut alloc = match to_process.buffer_alloc(debug_id, len, is_oneway, self.process.task.pid()) { Ok(alloc) => alloc, @@ -991,21 +1018,18 @@ impl Thread { // Copy offsets if there are any. if offsets_size > 0 { - { - let mut reader = - UserSlice::new(UserPtr::from_addr(trd_data_ptr.offsets as _), offsets_size) - .reader(); - alloc.copy_into(&mut reader, aligned_data_size, offsets_size)?; - } + let mut offsets_reader = + UserSlice::new(UserPtr::from_addr(trd_data_ptr.offsets as _), offsets_size) + .reader(); let offsets_start = aligned_data_size; - let offsets_end = aligned_data_size + aligned_offsets_size; + let offsets_end = aligned_data_size + offsets_size; // This state is used for BINDER_TYPE_PTR objects. let sg_state = sg_state.insert(ScatterGatherState { unused_buffer_space: UnusedBufferSpace { offset: offsets_end, - limit: len, + limit: offsets_end + buffers_size, }, sg_entries: KVec::new(), ancestors: KVec::new(), @@ -1014,12 +1038,14 @@ impl Thread { // Traverse the objects specified. let mut view = AllocationView::new(&mut alloc, data_size); for (index, index_offset) in (offsets_start..offsets_end) - .step_by(size_of::()) + .step_by(size_of::()) .enumerate() { - let offset = view.alloc.read(index_offset)?; + let offset = offsets_reader.read::()?; + view.alloc.write(index_offset, &offset)?; + let offset: usize = offset.try_into().map_err(|_| EINVAL)?; - if offset < end_of_previous_object { + if offset < end_of_previous_object || !is_aligned(offset, size_of::()) { pr_warn!("Got transaction with invalid offset."); return Err(EINVAL.into()); } @@ -1051,7 +1077,7 @@ impl Thread { } // Update the indexes containing objects to clean up. - let offset_after_object = index_offset + size_of::(); + let offset_after_object = index_offset + size_of::(); view.alloc .set_info_offsets(offsets_start..offset_after_object); } diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 979c96b74cad3..d5ed64543bbf4 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -81,7 +81,7 @@ static void binder_insert_free_buffer(struct binder_alloc *alloc, new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: add free buffer, size %zd, at %pK\n", + "%d: add free buffer, size %zd, at %p\n", alloc->pid, new_buffer_size, new_buffer); while (*p) { @@ -572,7 +572,7 @@ static struct binder_buffer *binder_alloc_new_buf_locked( } binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n", + "%d: binder_alloc_buf size %zd got buffer %p size %zd\n", alloc->pid, size, buffer, buffer_size); /* @@ -748,7 +748,7 @@ static void binder_free_buf_locked(struct binder_alloc *alloc, ALIGN(buffer->extra_buffers_size, sizeof(void *)); binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_free_buf %pK size %zd buffer_size %zd\n", + "%d: binder_free_buf %p size %zd buffer_size %zd\n", alloc->pid, buffer, size, buffer_size); BUG_ON(buffer->free); diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c index be8e64eb39ec5..6c6f52e1c032e 100644 --- a/drivers/android/binderfs.c +++ b/drivers/android/binderfs.c @@ -132,8 +132,8 @@ static int binderfs_binder_device_create(struct inode *ref_inode, mutex_lock(&binderfs_minors_mutex); if (++info->device_count <= info->mount_opts.max) minor = ida_alloc_max(&binderfs_minors, - use_reserve ? BINDERFS_MAX_MINOR : - BINDERFS_MAX_MINOR_CAPPED, + use_reserve ? BINDERFS_MAX_MINOR - 1 : + BINDERFS_MAX_MINOR_CAPPED - 1, GFP_KERNEL); else minor = -ENOSPC; @@ -424,8 +424,8 @@ static int binderfs_binder_ctl_create(struct super_block *sb) /* Reserve a new minor number for the new device. */ mutex_lock(&binderfs_minors_mutex); minor = ida_alloc_max(&binderfs_minors, - use_reserve ? BINDERFS_MAX_MINOR : - BINDERFS_MAX_MINOR_CAPPED, + use_reserve ? BINDERFS_MAX_MINOR - 1 : + BINDERFS_MAX_MINOR_CAPPED - 1, GFP_KERNEL); mutex_unlock(&binderfs_minors_mutex); if (minor < 0) { diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 7a7f88b3fa2b1..1d73a53370cf3 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -68,6 +68,7 @@ enum board_ids { /* board IDs for specific chipsets in alphabetical order */ board_ahci_al, board_ahci_avn, + board_ahci_jmb585, board_ahci_mcp65, board_ahci_mcp77, board_ahci_mcp89, @@ -212,6 +213,15 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_avn_ops, }, + /* JMicron JMB582/585: 64-bit DMA is broken, force 32-bit */ + [board_ahci_jmb585] = { + AHCI_HFLAGS (AHCI_HFLAG_IGN_IRQ_IF_ERR | + AHCI_HFLAG_32BIT_ONLY), + .flags = AHCI_FLAG_COMMON, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_ops, + }, [board_ahci_mcp65] = { AHCI_HFLAGS (AHCI_HFLAG_NO_FPDMA_AA | AHCI_HFLAG_NO_PMP | AHCI_HFLAG_YES_NCQ), @@ -439,6 +449,10 @@ static const struct pci_device_id ahci_pci_tbl[] = { /* Elkhart Lake IDs 0x4b60 & 0x4b62 https://sata-io.org/product/8803 not tested yet */ { PCI_VDEVICE(INTEL, 0x4b63), board_ahci_pcs_quirk }, /* Elkhart Lake AHCI */ + /* JMicron JMB582/585: force 32-bit DMA (broken 64-bit implementation) */ + { PCI_VDEVICE(JMICRON, 0x0582), board_ahci_jmb585 }, + { PCI_VDEVICE(JMICRON, 0x0585), board_ahci_jmb585 }, + /* JMicron 360/1/3/5/6, match class to avoid IDE function */ { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr }, @@ -2094,13 +2108,13 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (ap->flags & ATA_FLAG_EM) ap->em_message_type = hpriv->em_msg_type; - ahci_mark_external_port(ap); - - ahci_update_initial_lpm_policy(ap); - /* disabled/not-implemented port */ - if (!(hpriv->port_map & (1 << i))) + if (!(hpriv->port_map & (1 << i))) { ap->ops = &ata_dummy_port_ops; + } else { + ahci_mark_external_port(ap); + ahci_update_initial_lpm_policy(ap); + } } /* apply workaround for ASUS P5W DH Deluxe mainboard */ diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index f48fb63d7e854..4fb8990d22d55 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2358,6 +2358,24 @@ static bool ata_dev_check_adapter(struct ata_device *dev, return false; } +bool ata_adapter_is_online(struct ata_port *ap) +{ + struct device *dev; + + if (!ap || !ap->host) + return false; + + dev = ap->host->dev; + if (!dev) + return false; + + if (dev_is_pci(dev) && + pci_channel_offline(to_pci_dev(dev))) + return false; + + return true; +} + static int ata_dev_config_ncq(struct ata_device *dev, char *desc, size_t desc_sz) { @@ -2872,7 +2890,8 @@ static void ata_dev_config_lpm(struct ata_device *dev) static void ata_dev_print_features(struct ata_device *dev) { - if (!(dev->flags & ATA_DFLAG_FEATURES_MASK)) + if (!(dev->flags & ATA_DFLAG_FEATURES_MASK) && !dev->cpr_log && + !ata_id_has_hipm(dev->id) && !ata_id_has_dipm(dev->id)) return; ata_dev_info(dev, @@ -3116,6 +3135,11 @@ int ata_dev_configure(struct ata_device *dev) ata_mode_string(xfer_mask), cdb_intr_string, atapi_an_string, dma_dir_string); + + ata_dev_config_lpm(dev); + + if (print_info) + ata_dev_print_features(dev); } /* determine max_sectors */ @@ -4132,6 +4156,13 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "ST3320[68]13AS", "SD1[5-9]", ATA_QUIRK_NONCQ | ATA_QUIRK_FIRMWARE_WARN }, + /* ADATA devices with LPM issues. */ + { "ADATA SU680", NULL, ATA_QUIRK_NOLPM }, + + /* Seagate disks with LPM issues */ + { "ST1000DM010-2EP102", NULL, ATA_QUIRK_NOLPM }, + { "ST2000DM008-2FR102", NULL, ATA_QUIRK_NOLPM }, + /* drives which fail FPDMA_AA activation (some may freeze afterwards) the ST disks also have LPM issues */ { "ST1000LM024 HN-M101MBB", NULL, ATA_QUIRK_BROKEN_FPDMA_AA | @@ -4172,6 +4203,7 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { /* Devices that do not need bridging limits applied */ { "MTRON MSP-SATA*", NULL, ATA_QUIRK_BRIDGE_OK }, { "BUFFALO HD-QSU2/R5", NULL, ATA_QUIRK_BRIDGE_OK }, + { "QEMU HARDDISK", "2.5+", ATA_QUIRK_BRIDGE_OK }, /* Devices which aren't very happy with higher link speeds */ { "WD My Book", NULL, ATA_QUIRK_1_5_GBPS }, @@ -5058,6 +5090,12 @@ void ata_qc_issue(struct ata_queued_cmd *qc) qc->flags |= ATA_QCFLAG_ACTIVE; ap->qc_active |= 1ULL << qc->tag; + /* Make sure the device is still accessible. */ + if (!ata_adapter_is_online(ap)) { + qc->err_mask |= AC_ERR_HOST_BUS; + goto sys_err; + } + /* * We guarantee to LLDs that they will have at least one * non-zero sg if the command is a data command. @@ -5543,6 +5581,7 @@ struct ata_port *ata_port_alloc(struct ata_host *host) mutex_init(&ap->scsi_scan_mutex); INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug); INIT_DELAYED_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan); + INIT_WORK(&ap->deferred_qc_work, ata_scsi_deferred_qc_work); INIT_LIST_HEAD(&ap->eh_done_q); init_waitqueue_head(&ap->eh_wait_q); init_completion(&ap->park_req_pending); @@ -6163,9 +6202,11 @@ static void ata_port_detach(struct ata_port *ap) /* wait till EH commits suicide */ ata_port_wait_eh(ap); - /* it better be dead now */ + /* It better be dead now and not have any remaining deferred qc. */ WARN_ON(!(ap->pflags & ATA_PFLAG_UNLOADED)); + WARN_ON(ap->deferred_qc); + cancel_work_sync(&ap->deferred_qc_work); cancel_delayed_work_sync(&ap->hotplug_task); cancel_delayed_work_sync(&ap->scsi_rescan_task); diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 2586e77ebf45d..23be85418b3b1 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -640,12 +640,29 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, set_host_byte(scmd, DID_OK); ata_qc_for_each_raw(ap, qc, i) { - if (qc->flags & ATA_QCFLAG_ACTIVE && - qc->scsicmd == scmd) + if (qc->scsicmd != scmd) + continue; + if ((qc->flags & ATA_QCFLAG_ACTIVE) || + qc == ap->deferred_qc) break; } - if (i < ATA_MAX_QUEUE) { + if (i < ATA_MAX_QUEUE && qc == ap->deferred_qc) { + /* + * This is a deferred command that timed out while + * waiting for the command queue to drain. Since the qc + * is not active yet (deferred_qc is still set, so the + * deferred qc work has not issued the command yet), + * simply signal the timeout by finishing the SCSI + * command and clear the deferred qc to prevent the + * deferred qc work from issuing this qc. + */ + WARN_ON_ONCE(qc->flags & ATA_QCFLAG_ACTIVE); + ap->deferred_qc = NULL; + cancel_work(&ap->deferred_qc_work); + set_host_byte(scmd, DID_TIME_OUT); + scsi_eh_finish_cmd(scmd, &ap->eh_done_q); + } else if (i < ATA_MAX_QUEUE) { /* the scmd has an associated qc */ if (!(qc->flags & ATA_QCFLAG_EH)) { /* which hasn't failed yet, timeout */ @@ -736,7 +753,8 @@ void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap) spin_unlock_irqrestore(ap->lock, flags); /* invoke EH, skip if unloading or suspended */ - if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED))) + if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED)) && + ata_adapter_is_online(ap)) ap->ops->error_handler(ap); else { /* if unloading, commence suicide */ @@ -917,6 +935,12 @@ static void ata_eh_set_pending(struct ata_port *ap, bool fastdrain) ap->pflags |= ATA_PFLAG_EH_PENDING; + /* + * If we have a deferred qc, requeue it so that it is retried once EH + * completes. + */ + ata_scsi_requeue_deferred_qc(ap); + if (!fastdrain) return; diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c index b2817a2995d6b..04e1e774645e2 100644 --- a/drivers/ata/libata-sata.c +++ b/drivers/ata/libata-sata.c @@ -909,7 +909,7 @@ static bool ata_scsi_lpm_supported(struct ata_port *ap) struct ata_link *link; struct ata_device *dev; - if (ap->flags & ATA_FLAG_NO_LPM) + if ((ap->flags & ATA_FLAG_NO_LPM) || !ap->ops->set_lpm) return false; ata_for_each_link(link, ap, EDGE) { diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 434774e71fe61..88689369db03d 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -1657,8 +1657,78 @@ static void ata_qc_done(struct ata_queued_cmd *qc) done(cmd); } +void ata_scsi_deferred_qc_work(struct work_struct *work) +{ + struct ata_port *ap = + container_of(work, struct ata_port, deferred_qc_work); + struct ata_queued_cmd *qc; + unsigned long flags; + + spin_lock_irqsave(ap->lock, flags); + + /* + * If we still have a deferred qc and we are not in EH, issue it. In + * such case, we should not need any more deferring the qc, so warn if + * qc_defer() says otherwise. + */ + qc = ap->deferred_qc; + if (qc && !ata_port_eh_scheduled(ap)) { + WARN_ON_ONCE(ap->ops->qc_defer(qc)); + ap->deferred_qc = NULL; + ata_qc_issue(qc); + } + + spin_unlock_irqrestore(ap->lock, flags); +} + +void ata_scsi_requeue_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + struct scsi_cmnd *scmd; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc when a reset occurs or NCQ commands fail, + * do not try to be smart about what to do with this deferred command + * and simply retry it by completing it with DID_SOFT_ERROR. + */ + if (!qc) + return; + + scmd = qc->scsicmd; + ap->deferred_qc = NULL; + cancel_work(&ap->deferred_qc_work); + ata_qc_free(qc); + scmd->result = (DID_SOFT_ERROR << 16); + scsi_done(scmd); +} + +static void ata_scsi_schedule_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc, then qc_defer() is defined and we can use + * this callback to determine if this qc is good to go, unless EH has + * been scheduled. + */ + if (!qc) + return; + + if (ata_port_eh_scheduled(ap)) { + ata_scsi_requeue_deferred_qc(ap); + return; + } + if (!ap->ops->qc_defer(qc)) + queue_work(system_highpri_wq, &ap->deferred_qc_work); +} + static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) { + struct ata_port *ap = qc->ap; struct scsi_cmnd *cmd = qc->scsicmd; u8 *cdb = cmd->cmnd; bool have_sense = qc->flags & ATA_QCFLAG_SENSE_VALID; @@ -1688,6 +1758,66 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) } ata_qc_done(qc); + + ata_scsi_schedule_deferred_qc(ap); +} + +static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc) +{ + int ret; + + if (!ap->ops->qc_defer) + goto issue; + + /* + * If we already have a deferred qc, then rely on the SCSI layer to + * requeue and defer all incoming commands until the deferred qc is + * processed, once all on-going commands complete. + */ + if (ap->deferred_qc) { + ata_qc_free(qc); + return SCSI_MLQUEUE_DEVICE_BUSY; + } + + /* Check if the command needs to be deferred. */ + ret = ap->ops->qc_defer(qc); + switch (ret) { + case 0: + break; + case ATA_DEFER_LINK: + ret = SCSI_MLQUEUE_DEVICE_BUSY; + break; + case ATA_DEFER_PORT: + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + default: + WARN_ON_ONCE(1); + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + } + + if (ret) { + /* + * We must defer this qc: if this is not an NCQ command, keep + * this qc as a deferred one and report to the SCSI layer that + * we issued it so that it is not requeued. The deferred qc will + * be issued with the port deferred_qc_work once all on-going + * commands complete. + */ + if (!ata_is_ncq(qc->tf.protocol)) { + ap->deferred_qc = qc; + return 0; + } + + /* Force a requeue of the command to defer its execution. */ + ata_qc_free(qc); + return ret; + } + +issue: + ata_qc_issue(qc); + + return 0; } /** @@ -1713,66 +1843,49 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) * spin_lock_irqsave(host lock) * * RETURNS: - * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY if the command - * needs to be deferred. + * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY or SCSI_MLQUEUE_HOST_BUSY if the + * command needs to be deferred. */ static int ata_scsi_translate(struct ata_device *dev, struct scsi_cmnd *cmd, ata_xlat_func_t xlat_func) { struct ata_port *ap = dev->link->ap; struct ata_queued_cmd *qc; - int rc; + lockdep_assert_held(ap->lock); + + /* + * ata_scsi_qc_new() calls scsi_done(cmd) in case of failure. So we + * have nothing further to do when allocating a qc fails. + */ qc = ata_scsi_qc_new(dev, cmd); if (!qc) - goto err_mem; + return 0; /* data is present; dma-map it */ if (cmd->sc_data_direction == DMA_FROM_DEVICE || cmd->sc_data_direction == DMA_TO_DEVICE) { if (unlikely(scsi_bufflen(cmd) < 1)) { ata_dev_warn(dev, "WARNING: zero len r/w req\n"); - goto err_did; + cmd->result = (DID_ERROR << 16); + goto done; } ata_sg_init(qc, scsi_sglist(cmd), scsi_sg_count(cmd)); - qc->dma_dir = cmd->sc_data_direction; } qc->complete_fn = ata_scsi_qc_complete; if (xlat_func(qc)) - goto early_finish; - - if (ap->ops->qc_defer) { - if ((rc = ap->ops->qc_defer(qc))) - goto defer; - } - - /* select device, send command to hardware */ - ata_qc_issue(qc); - - return 0; + goto done; -early_finish: - ata_qc_free(qc); - scsi_done(cmd); - return 0; + return ata_scsi_qc_issue(ap, qc); -err_did: +done: ata_qc_free(qc); - cmd->result = (DID_ERROR << 16); scsi_done(cmd); -err_mem: return 0; - -defer: - ata_qc_free(qc); - if (rc == ATA_DEFER_LINK) - return SCSI_MLQUEUE_DEVICE_BUSY; - else - return SCSI_MLQUEUE_HOST_BUSY; } /** @@ -2981,6 +3094,9 @@ ata_scsi_find_dev(struct ata_port *ap, const struct scsi_device *scsidev) { struct ata_device *dev = __ata_scsi_find_dev(ap, scsidev); + if (!ata_adapter_is_online(ap)) + return NULL; + if (unlikely(!dev || !ata_dev_enabled(dev))) return NULL; @@ -3483,7 +3599,7 @@ static unsigned int ata_scsiop_maint_in(struct ata_device *dev, if (cdb[2] != 1 && cdb[2] != 3) { ata_dev_warn(dev, "invalid command format %d\n", cdb[2]); - ata_scsi_set_invalid_field(dev, cmd, 1, 0xff); + ata_scsi_set_invalid_field(dev, cmd, 2, 0xff); return 0; } diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index e5b977a8d3e1e..0002994ddfc9b 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -94,6 +94,7 @@ extern int atapi_check_dma(struct ata_queued_cmd *qc); extern void swap_buf_le16(u16 *buf, unsigned int buf_words); extern bool ata_phys_link_online(struct ata_link *link); extern bool ata_phys_link_offline(struct ata_link *link); +bool ata_adapter_is_online(struct ata_port *ap); extern void ata_dev_init(struct ata_device *dev); extern void ata_link_init(struct ata_port *ap, struct ata_link *link, int pmp); extern int sata_link_init_spd(struct ata_link *link); @@ -161,6 +162,8 @@ void ata_scsi_sdev_config(struct scsi_device *sdev); int ata_scsi_dev_config(struct scsi_device *sdev, struct queue_limits *lim, struct ata_device *dev); int __ata_scsi_queuecmd(struct scsi_cmnd *scmd, struct ata_device *dev); +void ata_scsi_deferred_qc_work(struct work_struct *work); +void ata_scsi_requeue_deferred_qc(struct ata_port *ap); /* libata-eh.c */ extern unsigned int ata_internal_cmd_timeout(struct ata_device *dev, u8 cmd); diff --git a/drivers/ata/pata_ftide010.c b/drivers/ata/pata_ftide010.c index c3a8384c3e04d..c41da296eb389 100644 --- a/drivers/ata/pata_ftide010.c +++ b/drivers/ata/pata_ftide010.c @@ -122,10 +122,10 @@ static const u8 mwdma_50_active_time[3] = {6, 2, 2}; static const u8 mwdma_50_recovery_time[3] = {6, 2, 1}; static const u8 mwdma_66_active_time[3] = {8, 3, 3}; static const u8 mwdma_66_recovery_time[3] = {8, 2, 1}; -static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 1}; +static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 9}; static const u8 udma_50_hold_time[6] = {3, 1, 1, 1, 1, 1}; -static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, }; -static const u8 udma_66_hold_time[7] = {}; +static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, 1, 9, 9}; +static const u8 udma_66_hold_time[7] = {4, 2, 1, 1, 1, 1, 1}; /* * We set 66 MHz for all MWDMA modes diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c index f62e385714403..fec081db36dc4 100644 --- a/drivers/atm/fore200e.c +++ b/drivers/atm/fore200e.c @@ -373,6 +373,10 @@ fore200e_shutdown(struct fore200e* fore200e) fallthrough; case FORE200E_STATE_IRQ: free_irq(fore200e->irq, fore200e->atm_dev); +#ifdef FORE200E_USE_TASKLET + tasklet_kill(&fore200e->tx_tasklet); + tasklet_kill(&fore200e->rx_tasklet); +#endif fallthrough; case FORE200E_STATE_ALLOC_BUF: diff --git a/drivers/atm/he.c b/drivers/atm/he.c index ad91cc6a34fc5..92a041d5387bd 100644 --- a/drivers/atm/he.c +++ b/drivers/atm/he.c @@ -1587,7 +1587,8 @@ he_stop(struct he_dev *he_dev) he_dev->tbrq_base, he_dev->tbrq_phys); if (he_dev->tpdrq_base) - dma_free_coherent(&he_dev->pci_dev->dev, CONFIG_TBRQ_SIZE * sizeof(struct he_tbrq), + dma_free_coherent(&he_dev->pci_dev->dev, + CONFIG_TPDRQ_SIZE * sizeof(struct he_tpdrq), he_dev->tpdrq_base, he_dev->tpdrq_phys); dma_pool_destroy(he_dev->tpd_pool); diff --git a/drivers/auxdisplay/arm-charlcd.c b/drivers/auxdisplay/arm-charlcd.c index a7eae99a48f77..4e22882f57c9c 100644 --- a/drivers/auxdisplay/arm-charlcd.c +++ b/drivers/auxdisplay/arm-charlcd.c @@ -323,7 +323,7 @@ static int __init charlcd_probe(struct platform_device *pdev) out_no_irq: iounmap(lcd->virtbase); out_no_memregion: - release_mem_region(lcd->phybase, SZ_4K); + release_mem_region(lcd->phybase, lcd->physize); out_no_resource: kfree(lcd); return ret; diff --git a/drivers/base/bus.c b/drivers/base/bus.c index 5e75e1bce5516..2653670f962f4 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -466,6 +466,36 @@ int bus_for_each_drv(const struct bus_type *bus, struct device_driver *start, } EXPORT_SYMBOL_GPL(bus_for_each_drv); +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int ret; + + ret = __device_set_driver_override(dev, buf, count); + if (ret) + return ret; + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + guard(spinlock)(&dev->driver_override.lock); + return sysfs_emit(buf, "%s\n", dev->driver_override.name); +} +static DEVICE_ATTR_RW(driver_override); + +static struct attribute *driver_override_dev_attrs[] = { + &dev_attr_driver_override.attr, + NULL, +}; + +static const struct attribute_group driver_override_dev_group = { + .attrs = driver_override_dev_attrs, +}; + /** * bus_add_device - add device to bus * @dev: device being added @@ -499,9 +529,15 @@ int bus_add_device(struct device *dev) if (error) goto out_put; + if (dev->bus->driver_override) { + error = device_add_group(dev, &driver_override_dev_group); + if (error) + goto out_groups; + } + error = sysfs_create_link(&sp->devices_kset->kobj, &dev->kobj, dev_name(dev)); if (error) - goto out_groups; + goto out_override; error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem"); if (error) @@ -512,6 +548,9 @@ int bus_add_device(struct device *dev) out_subsys: sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev)); +out_override: + if (dev->bus->driver_override) + device_remove_group(dev, &driver_override_dev_group); out_groups: device_remove_groups(dev, sp->bus->dev_groups); out_put: @@ -570,6 +609,8 @@ void bus_remove_device(struct device *dev) sysfs_remove_link(&dev->kobj, "subsystem"); sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev)); + if (dev->bus->driver_override) + device_remove_group(dev, &driver_override_dev_group); device_remove_groups(dev, dev->bus->dev_groups); if (klist_node_attached(&dev->p->knode_bus)) klist_del(&dev->p->knode_bus); diff --git a/drivers/base/core.c b/drivers/base/core.c index f69dc9c859545..3099dbca234ab 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2556,6 +2556,7 @@ static void device_release(struct kobject *kobj) devres_release_all(dev); kfree(dev->dma_range_map); + kfree(dev->driver_override.name); if (dev->release) dev->release(dev); @@ -3159,6 +3160,7 @@ void device_initialize(struct device *dev) kobject_init(&dev->kobj, &device_ktype); INIT_LIST_HEAD(&dev->dma_pools); mutex_init(&dev->mutex); + spin_lock_init(&dev->driver_override.lock); lockdep_set_novalidate_class(&dev->mutex); spin_lock_init(&dev->devres_lock); INIT_LIST_HEAD(&dev->devres_head); diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 13ab98e033eaa..bca427532041f 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -258,7 +258,7 @@ static int deferred_devs_show(struct seq_file *s, void *data) DEFINE_SHOW_ATTRIBUTE(deferred_devs); #ifdef CONFIG_MODULES -static int driver_deferred_probe_timeout = 10; +static int driver_deferred_probe_timeout = 15; #else static int driver_deferred_probe_timeout; #endif @@ -381,6 +381,66 @@ static void __exit deferred_probe_exit(void) } __exitcall(deferred_probe_exit); +int __device_set_driver_override(struct device *dev, const char *s, size_t len) +{ + const char *new, *old; + char *cp; + + if (!s) + return -EINVAL; + + /* + * The stored value will be used in sysfs show callback (sysfs_emit()), + * which has a length limit of PAGE_SIZE and adds a trailing newline. + * Thus we can store one character less to avoid truncation during sysfs + * show. + */ + if (len >= (PAGE_SIZE - 1)) + return -EINVAL; + + /* + * Compute the real length of the string in case userspace sends us a + * bunch of \0 characters like python likes to do. + */ + len = strlen(s); + + if (!len) { + /* Empty string passed - clear override */ + spin_lock(&dev->driver_override.lock); + old = dev->driver_override.name; + dev->driver_override.name = NULL; + spin_unlock(&dev->driver_override.lock); + kfree(old); + + return 0; + } + + cp = strnchr(s, len, '\n'); + if (cp) + len = cp - s; + + new = kstrndup(s, len, GFP_KERNEL); + if (!new) + return -ENOMEM; + + spin_lock(&dev->driver_override.lock); + old = dev->driver_override.name; + if (cp != s) { + dev->driver_override.name = new; + spin_unlock(&dev->driver_override.lock); + } else { + /* "\n" passed - clear override */ + dev->driver_override.name = NULL; + spin_unlock(&dev->driver_override.lock); + + kfree(new); + } + kfree(old); + + return 0; +} +EXPORT_SYMBOL_GPL(__device_set_driver_override); + /** * device_is_bound() - Check if device is bound to a driver * @dev: device to check diff --git a/drivers/base/faux.c b/drivers/base/faux.c index 21dd02124231a..23d7258172325 100644 --- a/drivers/base/faux.c +++ b/drivers/base/faux.c @@ -29,9 +29,7 @@ struct faux_object { }; #define to_faux_object(dev) container_of_const(dev, struct faux_object, faux_dev.dev) -static struct device faux_bus_root = { - .init_name = "faux", -}; +static struct device *faux_bus_root; static int faux_match(struct device *dev, const struct device_driver *drv) { @@ -152,7 +150,7 @@ struct faux_device *faux_device_create_with_groups(const char *name, if (parent) dev->parent = parent; else - dev->parent = &faux_bus_root; + dev->parent = faux_bus_root; dev->bus = &faux_bus_type; dev_set_name(dev, "%s", name); device_set_pm_not_required(dev); @@ -236,9 +234,15 @@ int __init faux_bus_init(void) { int ret; - ret = device_register(&faux_bus_root); + faux_bus_root = kzalloc(sizeof(*faux_bus_root), GFP_KERNEL); + if (!faux_bus_root) + return -ENOMEM; + + dev_set_name(faux_bus_root, "faux"); + + ret = device_register(faux_bus_root); if (ret) { - put_device(&faux_bus_root); + put_device(faux_bus_root); return ret; } @@ -256,6 +260,6 @@ int __init faux_bus_init(void) bus_unregister(&faux_bus_type); error_bus: - device_unregister(&faux_bus_root); + device_unregister(faux_bus_root); return ret; } diff --git a/drivers/base/firmware_loader/Kconfig b/drivers/base/firmware_loader/Kconfig index 752b9a9bea038..15eff8a4b5053 100644 --- a/drivers/base/firmware_loader/Kconfig +++ b/drivers/base/firmware_loader/Kconfig @@ -38,7 +38,7 @@ config FW_LOADER_DEBUG config RUST_FW_LOADER_ABSTRACTIONS bool "Rust Firmware Loader abstractions" depends on RUST - depends on FW_LOADER=y + select FW_LOADER help This enables the Rust abstractions for the firmware loader API. diff --git a/drivers/base/platform.c b/drivers/base/platform.c index 09450349cf323..019d3f26807d9 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -562,7 +562,6 @@ static void platform_device_release(struct device *dev) kfree(pa->pdev.dev.platform_data); kfree(pa->pdev.mfd_cell); kfree(pa->pdev.resource); - kfree(pa->pdev.driver_override); kfree(pa); } @@ -1265,38 +1264,9 @@ static ssize_t numa_node_show(struct device *dev, } static DEVICE_ATTR_RO(numa_node); -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct platform_device *pdev = to_platform_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", pdev->driver_override); - device_unlock(dev); - - return len; -} - -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct platform_device *pdev = to_platform_device(dev); - int ret; - - ret = driver_set_override(dev, &pdev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} -static DEVICE_ATTR_RW(driver_override); - static struct attribute *platform_dev_attrs[] = { &dev_attr_modalias.attr, &dev_attr_numa_node.attr, - &dev_attr_driver_override.attr, NULL, }; @@ -1336,10 +1306,12 @@ static int platform_match(struct device *dev, const struct device_driver *drv) { struct platform_device *pdev = to_platform_device(dev); struct platform_driver *pdrv = to_platform_driver(drv); + int ret; /* When driver_override is set, only bind to the matching driver */ - if (pdev->driver_override) - return !strcmp(pdev->driver_override, drv->name); + ret = device_match_driver_override(dev, drv); + if (ret >= 0) + return ret; /* Attempt an OF style match first */ if (of_driver_match_device(dev, drv)) @@ -1475,6 +1447,7 @@ static const struct dev_pm_ops platform_dev_pm_ops = { const struct bus_type platform_bus_type = { .name = "platform", .dev_groups = platform_dev_groups, + .driver_override = true, .match = platform_match, .uevent = platform_uevent, .probe = platform_probe, diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 1b11a3cd4acc2..6980a8dfced2c 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1869,16 +1869,18 @@ void pm_runtime_init(struct device *dev) */ void pm_runtime_reinit(struct device *dev) { - if (!pm_runtime_enabled(dev)) { - if (dev->power.runtime_status == RPM_ACTIVE) - pm_runtime_set_suspended(dev); - if (dev->power.irq_safe) { - spin_lock_irq(&dev->power.lock); - dev->power.irq_safe = 0; - spin_unlock_irq(&dev->power.lock); - if (dev->parent) - pm_runtime_put(dev->parent); - } + if (pm_runtime_enabled(dev)) + return; + + if (dev->power.runtime_status == RPM_ACTIVE) + pm_runtime_set_suspended(dev); + + if (dev->power.irq_safe) { + spin_lock_irq(&dev->power.lock); + dev->power.irq_safe = 0; + spin_unlock_irq(&dev->power.lock); + if (dev->parent) + pm_runtime_put(dev->parent); } /* * Clear power.needs_force_resume in case it has been set by @@ -1894,6 +1896,7 @@ void pm_runtime_reinit(struct device *dev) void pm_runtime_remove(struct device *dev) { __pm_runtime_disable(dev, false); + flush_work(&dev->power.work); pm_runtime_reinit(dev); } diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 8aa28c08b2891..c0809d18fc540 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -83,13 +83,16 @@ EXPORT_SYMBOL_GPL(dev_pm_set_wake_irq); */ void dev_pm_clear_wake_irq(struct device *dev) { - struct wake_irq *wirq = dev->power.wakeirq; + struct wake_irq *wirq; unsigned long flags; - if (!wirq) + spin_lock_irqsave(&dev->power.lock, flags); + wirq = dev->power.wakeirq; + if (!wirq) { + spin_unlock_irqrestore(&dev->power.lock, flags); return; + } - spin_lock_irqsave(&dev->power.lock, flags); device_wakeup_detach_irq(dev); dev->power.wakeirq = NULL; spin_unlock_irqrestore(&dev->power.lock, flags); diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index d1283ff1080bb..2f630df16bfe1 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -276,9 +276,7 @@ EXPORT_SYMBOL_GPL(wakeup_sources_read_unlock); */ struct wakeup_source *wakeup_sources_walk_start(void) { - struct list_head *ws_head = &wakeup_sources; - - return list_entry_rcu(ws_head->next, struct wakeup_source, entry); + return list_first_or_null_rcu(&wakeup_sources, struct wakeup_source, entry); } EXPORT_SYMBOL_GPL(wakeup_sources_walk_start); diff --git a/drivers/base/property.c b/drivers/base/property.c index 6a63860579dd3..8d9a34be57fbf 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -797,7 +797,18 @@ struct fwnode_handle * fwnode_get_next_child_node(const struct fwnode_handle *fwnode, struct fwnode_handle *child) { - return fwnode_call_ptr_op(fwnode, get_next_child_node, child); + struct fwnode_handle *next; + + if (IS_ERR_OR_NULL(fwnode)) + return NULL; + + /* Try to find a child in primary fwnode */ + next = fwnode_call_ptr_op(fwnode, get_next_child_node, child); + if (next) + return next; + + /* When no more children in primary, continue with secondary */ + return fwnode_call_ptr_op(fwnode->secondary, get_next_child_node, child); } EXPORT_SYMBOL_GPL(fwnode_get_next_child_node); @@ -841,19 +852,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_next_available_child_node); struct fwnode_handle *device_get_next_child_node(const struct device *dev, struct fwnode_handle *child) { - const struct fwnode_handle *fwnode = dev_fwnode(dev); - struct fwnode_handle *next; - - if (IS_ERR_OR_NULL(fwnode)) - return NULL; - - /* Try to find a child in primary fwnode */ - next = fwnode_get_next_child_node(fwnode, child); - if (next) - return next; - - /* When no more children in primary, continue with secondary */ - return fwnode_get_next_child_node(fwnode->secondary, child); + return fwnode_get_next_child_node(dev_fwnode(dev), child); } EXPORT_SYMBOL_GPL(device_get_next_child_node); diff --git a/drivers/base/regmap/regcache-maple.c b/drivers/base/regmap/regcache-maple.c index 2319c30283a6d..9cf0384ce7b95 100644 --- a/drivers/base/regmap/regcache-maple.c +++ b/drivers/base/regmap/regcache-maple.c @@ -95,12 +95,13 @@ static int regcache_maple_write(struct regmap *map, unsigned int reg, mas_unlock(&mas); - if (ret == 0) { - kfree(lower); - kfree(upper); + if (ret) { + kfree(entry); + return ret; } - - return ret; + kfree(lower); + kfree(upper); + return 0; } static int regcache_maple_drop(struct regmap *map, unsigned int min, diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index ce9be3989a218..a648218507236 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -408,9 +408,11 @@ static void regmap_lock_hwlock_irq(void *__map) static void regmap_lock_hwlock_irqsave(void *__map) { struct regmap *map = __map; + unsigned long flags = 0; hwspin_lock_timeout_irqsave(map->hwlock, UINT_MAX, - &map->spinlock_flags); + &flags); + map->spinlock_flags = flags; } static void regmap_unlock_hwlock(void *__map) @@ -1541,6 +1543,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, unsigned int val_num) { void *orig_work_buf; + unsigned int selector_reg; unsigned int win_offset; unsigned int win_page; bool page_chg; @@ -1559,10 +1562,31 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, return -EINVAL; } - /* It is possible to have selector register inside data window. - In that case, selector register is located on every page and - it needs no page switching, when accessed alone. */ + /* + * Calculate the address of the selector register in the corresponding + * data window if it is located on every page. + */ + page_chg = in_range(range->selector_reg, range->window_start, range->window_len); + if (page_chg) + selector_reg = range->range_min + win_page * range->window_len + + range->selector_reg - range->window_start; + + /* + * It is possible to have selector register inside data window. + * In that case, selector register is located on every page and it + * needs no page switching, when accessed alone. + * + * Nevertheless we should synchronize the cache values for it. + * This can't be properly achieved if the selector register is + * the first and the only one to be read inside the data window. + * That's why we update it in that case as well. + * + * However, we specifically avoid updating it for the default page, + * when it's overlapped with the real data window, to prevent from + * infinite looping. + */ if (val_num > 1 || + (page_chg && selector_reg != range->selector_reg) || range->window_start + win_offset != range->selector_reg) { /* Use separate work_buf during page switching */ orig_work_buf = map->work_buf; @@ -1571,7 +1595,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, ret = _regmap_update_bits(map, range->selector_reg, range->selector_mask, win_page << range->selector_shift, - &page_chg, false); + NULL, false); map->work_buf = orig_work_buf; diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index 742b2908ff686..b3dbf6c76e98f 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c @@ -483,38 +483,20 @@ void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i) int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i) { - struct lru_cache *al = device->act_log; /* for bios crossing activity log extent boundaries, * we may need to activate two extents in one go */ unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); - unsigned nr_al_extents; - unsigned available_update_slots; unsigned enr; - D_ASSERT(device, first <= last); - - nr_al_extents = 1 + last - first; /* worst case: all touched extends are cold. */ - available_update_slots = min(al->nr_elements - al->used, - al->max_pending_changes - al->pending_changes); - - /* We want all necessary updates for a given request within the same transaction - * We could first check how many updates are *actually* needed, - * and use that instead of the worst-case nr_al_extents */ - if (available_update_slots < nr_al_extents) { - /* Too many activity log extents are currently "hot". - * - * If we have accumulated pending changes already, - * we made progress. - * - * If we cannot get even a single pending change through, - * stop the fast path until we made some progress, - * or requests to "cold" extents could be starved. */ - if (!al->pending_changes) - __set_bit(__LC_STARVING, &device->act_log->flags); - return -ENOBUFS; + if (i->partially_in_al_next_enr) { + D_ASSERT(device, first < i->partially_in_al_next_enr); + D_ASSERT(device, last >= i->partially_in_al_next_enr); + first = i->partially_in_al_next_enr; } + D_ASSERT(device, first <= last); + /* Is resync active in this area? */ for (enr = first; enr <= last; enr++) { struct lc_element *tmp; @@ -529,14 +511,21 @@ int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval * } } - /* Checkout the refcounts. - * Given that we checked for available elements and update slots above, - * this has to be successful. */ + /* Try to checkout the refcounts. */ for (enr = first; enr <= last; enr++) { struct lc_element *al_ext; al_ext = lc_get_cumulative(device->act_log, enr); - if (!al_ext) - drbd_info(device, "LOGIC BUG for enr=%u\n", enr); + + if (!al_ext) { + /* Did not work. We may have exhausted the possible + * changes per transaction. Or raced with someone + * "locking" it against changes. + * Remember where to continue from. + */ + if (enr > first) + i->partially_in_al_next_enr = enr; + return -ENOBUFS; + } } return 0; } @@ -556,7 +545,11 @@ void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i) for (enr = first; enr <= last; enr++) { extent = lc_find(device->act_log, enr); - if (!extent) { + /* Yes, this masks a bug elsewhere. However, during normal + * operation this is harmless, so no need to crash the kernel + * by the BUG_ON(refcount == 0) in lc_put(). + */ + if (!extent || extent->refcnt == 0) { drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr); continue; } diff --git a/drivers/block/drbd/drbd_interval.h b/drivers/block/drbd/drbd_interval.h index 366489b72fe97..5d3213b81eede 100644 --- a/drivers/block/drbd/drbd_interval.h +++ b/drivers/block/drbd/drbd_interval.h @@ -8,12 +8,15 @@ struct drbd_interval { struct rb_node rb; sector_t sector; /* start sector of the interval */ - unsigned int size; /* size in bytes */ sector_t end; /* highest interval end in subtree */ + unsigned int size; /* size in bytes */ unsigned int local:1 /* local or remote request? */; unsigned int waiting:1; /* someone is waiting for completion */ unsigned int completed:1; /* this has been completed already; * ignore for conflict detection */ + + /* to resume a partially successful drbd_al_begin_io_nonblock(); */ + unsigned int partially_in_al_next_enr; }; static inline void drbd_clear_interval(struct drbd_interval *i) diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index c73376886e7a5..1f6ac9202b66a 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2659,9 +2659,6 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig * connect. */ .max_hw_sectors = DRBD_MAX_BIO_SIZE_SAFE >> 8, - .features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_ROTATIONAL | - BLK_FEAT_STABLE_WRITES, }; device = minor_to_device(minor); diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 91f3b8afb63ce..b502038be0a92 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -1296,6 +1296,8 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_segments = drbd_backing_dev_max_segments(device); } else { lim.max_segments = BLK_MAX_SEGMENTS; + lim.features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | + BLK_FEAT_ROTATIONAL | BLK_FEAT_STABLE_WRITES; } lim.max_hw_sectors = new >> SECTOR_SHIFT; @@ -1318,8 +1320,24 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_hw_discard_sectors = 0; } - if (bdev) + if (bdev) { blk_stack_limits(&lim, &b->limits, 0); + /* + * blk_set_stacking_limits() cleared the features, and + * blk_stack_limits() may or may not have inherited + * BLK_FEAT_STABLE_WRITES from the backing device. + * + * DRBD always requires stable writes because: + * 1. The same bio data is read for both local disk I/O and + * network transmission. If the page changes mid-flight, + * the local and remote copies could diverge. + * 2. When data integrity is enabled, DRBD calculates a + * checksum before sending the data. If the page changes + * between checksum calculation and transmission, the + * receiver will detect a checksum mismatch. + */ + lim.features |= BLK_FEAT_STABLE_WRITES; + } /* * If we can handle "zeroes" efficiently on the protocol, we want to do diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index d15826f6ee81d..70f75ef079457 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -621,7 +621,8 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, break; case READ_COMPLETED_WITH_ERROR: - drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size); + drbd_set_out_of_sync(first_peer_device(device), + req->i.sector, req->i.size); drbd_report_io_error(device, req); __drbd_chk_io_error(device, DRBD_READ_ERROR); fallthrough; diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 5336c3c5ca362..c28786e0fe1c8 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -329,7 +329,7 @@ static bool initialized; * This default is used whenever the current disk size is unknown. * [Now it is rather a minimum] */ -#define MAX_DISK_SIZE 4 /* 3984 */ +#define MAX_DISK_SIZE (PAGE_SIZE / 1024) /* * globals used by 'result()' diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index a853c65ac65df..a05ff68e58d06 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1024,9 +1024,9 @@ static void recv_work(struct work_struct *work) nbd_mark_nsock_dead(nbd, nsock, 1); mutex_unlock(&nsock->tx_lock); - nbd_config_put(nbd); atomic_dec(&config->recv_threads); wake_up(&config->recv_wq); + nbd_config_put(nbd); kfree(args); } @@ -2241,12 +2241,13 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) ret = nbd_start_device(nbd); out: - mutex_unlock(&nbd->config_lock); if (!ret) { set_bit(NBD_RT_HAS_CONFIG_REF, &config->runtime_flags); refcount_inc(&nbd->config_refs); nbd_connect_reply(info, nbd->index); } + mutex_unlock(&nbd->config_lock); + nbd_config_put(nbd); if (put_dev) nbd_put(nbd); diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 0ee55f889cfdd..50938881219bc 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -665,12 +665,22 @@ static void nullb_add_fault_config(struct nullb_device *dev) configfs_add_default_group(&dev->init_hctx_fault_config.group, &dev->group); } +static void nullb_del_fault_config(struct nullb_device *dev) +{ + config_item_put(&dev->init_hctx_fault_config.group.cg_item); + config_item_put(&dev->requeue_config.group.cg_item); + config_item_put(&dev->timeout_config.group.cg_item); +} + #else static void nullb_add_fault_config(struct nullb_device *dev) { } +static void nullb_del_fault_config(struct nullb_device *dev) +{ +} #endif static struct @@ -702,7 +712,7 @@ nullb_group_drop_item(struct config_group *group, struct config_item *item) null_del_dev(dev->nullb); mutex_unlock(&lock); } - + nullb_del_fault_config(dev); config_item_put(item); } diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c index dc9e4a14b8854..8892f218a8147 100644 --- a/drivers/block/ps3disk.c +++ b/drivers/block/ps3disk.c @@ -85,10 +85,14 @@ static void ps3disk_scatter_gather(struct ps3_storage_device *dev, struct bio_vec bvec; rq_for_each_segment(bvec, req, iter) { + dev_dbg(&dev->sbd.core, "%s:%u: %u sectors from %llu\n", + __func__, __LINE__, bio_sectors(iter.bio), + iter.bio->bi_iter.bi_sector); if (gather) memcpy_from_bvec(dev->bounce_buf + offset, &bvec); else memcpy_to_bvec(&bvec, dev->bounce_buf + offset); + offset += bvec.bv_len; } } diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index af0e21149dbc9..8f441eb8b1926 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -3495,11 +3495,29 @@ static void rbd_img_object_requests(struct rbd_img_request *img_req) rbd_assert(!need_exclusive_lock(img_req) || __rbd_is_lock_owner(rbd_dev)); - if (rbd_img_is_write(img_req)) { - rbd_assert(!img_req->snapc); + if (test_bit(IMG_REQ_CHILD, &img_req->flags)) { + rbd_assert(!rbd_img_is_write(img_req)); + } else { + struct request *rq = blk_mq_rq_from_pdu(img_req); + u64 off = (u64)blk_rq_pos(rq) << SECTOR_SHIFT; + u64 len = blk_rq_bytes(rq); + u64 mapping_size; + down_read(&rbd_dev->header_rwsem); - img_req->snapc = ceph_get_snap_context(rbd_dev->header.snapc); + mapping_size = rbd_dev->mapping.size; + if (rbd_img_is_write(img_req)) { + rbd_assert(!img_req->snapc); + img_req->snapc = + ceph_get_snap_context(rbd_dev->header.snapc); + } up_read(&rbd_dev->header_rwsem); + + if (unlikely(off + len > mapping_size)) { + rbd_warn(rbd_dev, "beyond EOD (%llu~%llu > %llu)", + off, len, mapping_size); + img_req->pending.result = -EIO; + return; + } } for_each_obj_request(img_req, obj_req) { @@ -4725,7 +4743,6 @@ static void rbd_queue_workfn(struct work_struct *work) struct request *rq = blk_mq_rq_from_pdu(img_request); u64 offset = (u64)blk_rq_pos(rq) << SECTOR_SHIFT; u64 length = blk_rq_bytes(rq); - u64 mapping_size; int result; /* Ignore/skip any zero-length requests */ @@ -4738,17 +4755,9 @@ static void rbd_queue_workfn(struct work_struct *work) blk_mq_start_request(rq); down_read(&rbd_dev->header_rwsem); - mapping_size = rbd_dev->mapping.size; rbd_img_capture_header(img_request); up_read(&rbd_dev->header_rwsem); - if (offset + length > mapping_size) { - rbd_warn(rbd_dev, "beyond EOD (%llu~%llu > %llu)", offset, - length, mapping_size); - result = -EIO; - goto err_img_request; - } - dout("%s rbd_dev %p img_req %p %s %llu~%llu\n", __func__, rbd_dev, img_request, obj_op_name(op_type), offset, length); diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index f1409e54010a6..d1c354636315d 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -1423,9 +1423,11 @@ static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess, goto out_alloc; } - ret = ida_alloc_max(&index_ida, (1 << (MINORBITS - RNBD_PART_BITS)) - 1, - GFP_KERNEL); - if (ret < 0) { + dev->clt_device_id = ida_alloc_max(&index_ida, + (1 << (MINORBITS - RNBD_PART_BITS)) - 1, + GFP_KERNEL); + if (dev->clt_device_id < 0) { + ret = dev->clt_device_id; pr_err("Failed to initialize device '%s' from session %s, allocating idr failed, err: %d\n", pathname, sess->sessname, ret); goto out_queues; @@ -1434,10 +1436,9 @@ static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess, dev->pathname = kstrdup(pathname, GFP_KERNEL); if (!dev->pathname) { ret = -ENOMEM; - goto out_queues; + goto out_ida; } - dev->clt_device_id = ret; dev->sess = sess; dev->access_mode = access_mode; dev->nr_poll_queues = nr_poll_queues; @@ -1453,6 +1454,8 @@ static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess, return dev; +out_ida: + ida_free(&index_ida, dev->clt_device_id); out_queues: kfree(dev->hw_queues); out_alloc: diff --git a/drivers/block/rnbd/rnbd-clt.h b/drivers/block/rnbd/rnbd-clt.h index a48e040abe639..fbc1ed766025c 100644 --- a/drivers/block/rnbd/rnbd-clt.h +++ b/drivers/block/rnbd/rnbd-clt.h @@ -112,7 +112,7 @@ struct rnbd_clt_dev { struct rnbd_queue *hw_queues; u32 device_id; /* local Idr index - used to track minor number allocations. */ - u32 clt_device_id; + int clt_device_id; struct mutex lock; enum rnbd_clt_dev_state dev_state; refcount_t refcount; diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index 2df8941a6b146..7eeb321d61402 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -145,18 +145,30 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, priv->sess_dev = sess_dev; priv->id = id; - bio = bio_alloc(file_bdev(sess_dev->bdev_file), 1, + bio = bio_alloc(file_bdev(sess_dev->bdev_file), !!datalen, rnbd_to_bio_flags(le32_to_cpu(msg->rw)), GFP_KERNEL); - bio_add_virt_nofail(bio, data, datalen); - - bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); - if (bio_has_data(bio) && - bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { - rnbd_srv_err_rl(sess_dev, "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", - bio->bi_iter.bi_size, msg->bi_size); - err = -EINVAL; - goto bio_put; + if (unlikely(!bio)) { + err = -ENOMEM; + goto put_sess_dev; } + + if (!datalen) { + /* + * For special requests like DISCARD and WRITE_ZEROES, the datalen is zero. + */ + bio->bi_iter.bi_size = le32_to_cpu(msg->bi_size); + } else { + bio_add_virt_nofail(bio, data, datalen); + bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); + if (bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { + rnbd_srv_err_rl(sess_dev, + "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", + bio->bi_iter.bi_size, msg->bi_size); + err = -EINVAL; + goto bio_put; + } + } + bio->bi_end_io = rnbd_dev_bi_end_io; bio->bi_private = priv; bio->bi_iter.bi_sector = le64_to_cpu(msg->sector); @@ -170,6 +182,7 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, bio_put: bio_put(bio); +put_sess_dev: rnbd_put_sess_dev(sess_dev); err: kfree(priv); @@ -538,6 +551,8 @@ static void rnbd_srv_fill_msg_open_rsp(struct rnbd_msg_open_rsp *rsp, { struct block_device *bdev = file_bdev(sess_dev->bdev_file); + memset(rsp, 0, sizeof(*rsp)); + rsp->hdr.type = cpu_to_le16(RNBD_MSG_OPEN_RSP); rsp->device_id = cpu_to_le32(sess_dev->device_id); rsp->nsectors = cpu_to_le64(bdev_nr_sectors(bdev)); @@ -644,6 +659,7 @@ static void process_msg_sess_info(struct rnbd_srv_session *srv_sess, trace_process_msg_sess_info(srv_sess, sess_info_msg); + memset(rsp, 0, sizeof(*rsp)); rsp->hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO_RSP); rsp->ver = srv_sess->ver; } diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 0c74a41a67530..2729b1556e810 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -155,12 +155,13 @@ struct ublk_uring_cmd_pdu { */ #define UBLK_REFCOUNT_INIT (REFCOUNT_MAX / 2) +union ublk_io_buf { + __u64 addr; + struct ublk_auto_buf_reg auto_reg; +}; + struct ublk_io { - /* userspace buffer address from io cmd */ - union { - __u64 addr; - struct ublk_auto_buf_reg buf; - }; + union ublk_io_buf buf; unsigned int flags; int res; @@ -209,9 +210,6 @@ struct ublk_queue { struct ublk_device { struct gendisk *ub_disk; - char *__queues; - - unsigned int queue_size; struct ublksrv_ctrl_dev_info dev_info; struct blk_mq_tag_set tag_set; @@ -239,6 +237,9 @@ struct ublk_device { bool canceling; pid_t ublksrv_tgid; struct delayed_work exit_work; + struct work_struct partition_scan_work; + + struct ublk_queue *queues[]; }; /* header of ublk_params */ @@ -499,7 +500,7 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, iod->op_flags = ublk_op | ublk_req_build_flags(req); iod->nr_sectors = blk_rq_sectors(req); iod->start_sector = blk_rq_pos(req); - iod->addr = io->addr; + iod->addr = io->buf.addr; return BLK_STS_OK; } @@ -781,7 +782,7 @@ static noinline void ublk_put_device(struct ublk_device *ub) static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev, int qid) { - return (struct ublk_queue *)&(dev->__queues[qid * dev->queue_size]); + return dev->queues[qid]; } static inline bool ublk_rq_has_data(const struct request *rq) @@ -1047,7 +1048,7 @@ static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req, struct iov_iter iter; const int dir = ITER_DEST; - import_ubuf(dir, u64_to_user_ptr(io->addr), rq_bytes, &iter); + import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter); return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1068,7 +1069,7 @@ static int ublk_unmap_io(bool need_map, WARN_ON_ONCE(io->res > rq_bytes); - import_ubuf(dir, u64_to_user_ptr(io->addr), io->res, &iter); + import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter); return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1134,7 +1135,7 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) iod->op_flags = ublk_op | ublk_req_build_flags(req); iod->nr_sectors = blk_rq_sectors(req); iod->start_sector = blk_rq_pos(req); - iod->addr = io->addr; + iod->addr = io->buf.addr; return BLK_STS_OK; } @@ -1145,12 +1146,20 @@ static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu( return io_uring_cmd_to_pdu(ioucmd, struct ublk_uring_cmd_pdu); } +static void ublk_end_request(struct request *req, blk_status_t error) +{ + local_bh_disable(); + blk_mq_end_request(req, error); + local_bh_enable(); +} + /* todo: handle partial completion */ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, bool need_map) { unsigned int unmapped_bytes; blk_status_t res = BLK_STS_OK; + bool requeue; /* failed read IO if nothing is read */ if (!io->res && req_op(req) == REQ_OP_READ) @@ -1182,14 +1191,30 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, if (unlikely(unmapped_bytes < io->res)) io->res = unmapped_bytes; - if (blk_update_request(req, BLK_STS_OK, io->res)) + /* + * Run bio->bi_end_io() with softirqs disabled. If the final fput + * happens off this path, then that will prevent ublk's blkdev_release() + * from being called on current's task work, see fput() implementation. + * + * Otherwise, ublk server may not provide forward progress in case of + * reading the partition table from bdev_open() with disk->open_mutex + * held, and causes dead lock as we could already be holding + * disk->open_mutex here. + * + * Preferably we would not be doing IO with a mutex held that is also + * used for release, but this work-around will suffice for now. + */ + local_bh_disable(); + requeue = blk_update_request(req, BLK_STS_OK, io->res); + local_bh_enable(); + if (requeue) blk_mq_requeue_request(req, true); else if (likely(!blk_should_fake_timeout(req->q))) __blk_mq_end_request(req, BLK_STS_OK); return; exit: - blk_mq_end_request(req, res); + ublk_end_request(req, res); } static struct io_uring_cmd *__ublk_prep_compl_io_cmd(struct ublk_io *io, @@ -1229,49 +1254,69 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq, if (ublk_nosrv_dev_should_queue_io(ubq->dev)) blk_mq_requeue_request(rq, false); else - blk_mq_end_request(rq, BLK_STS_IOERR); + ublk_end_request(rq, BLK_STS_IOERR); } static void -ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, struct ublk_io *io) +ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, unsigned tag) { - unsigned tag = io - ubq->ios; struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag); iod->op_flags |= UBLK_IO_F_NEED_REG_BUF; } -static bool ublk_auto_buf_reg(const struct ublk_queue *ubq, struct request *req, - struct ublk_io *io, unsigned int issue_flags) +enum auto_buf_reg_res { + AUTO_BUF_REG_FAIL, + AUTO_BUF_REG_FALLBACK, + AUTO_BUF_REG_OK, +}; + +static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq, + struct request *req, struct ublk_io *io, + struct io_uring_cmd *cmd, + enum auto_buf_reg_res res) +{ + if (res == AUTO_BUF_REG_OK) { + io->task_registered_buffers = 1; + io->buf_ctx_handle = io_uring_cmd_ctx_handle(cmd); + io->flags |= UBLK_IO_FLAG_AUTO_BUF_REG; + } + ublk_init_req_ref(ubq, io); + __ublk_prep_compl_io_cmd(io, req); +} + +static enum auto_buf_reg_res +__ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req, + struct ublk_io *io, struct io_uring_cmd *cmd, + unsigned int issue_flags) { int ret; - ret = io_buffer_register_bvec(io->cmd, req, ublk_io_release, - io->buf.index, issue_flags); + ret = io_buffer_register_bvec(cmd, req, ublk_io_release, + io->buf.auto_reg.index, issue_flags); if (ret) { - if (io->buf.flags & UBLK_AUTO_BUF_REG_FALLBACK) { - ublk_auto_buf_reg_fallback(ubq, io); - return true; + if (io->buf.auto_reg.flags & UBLK_AUTO_BUF_REG_FALLBACK) { + ublk_auto_buf_reg_fallback(ubq, req->tag); + return AUTO_BUF_REG_FALLBACK; } - blk_mq_end_request(req, BLK_STS_IOERR); - return false; + ublk_end_request(req, BLK_STS_IOERR); + return AUTO_BUF_REG_FAIL; } - io->task_registered_buffers = 1; - io->buf_ctx_handle = io_uring_cmd_ctx_handle(io->cmd); - io->flags |= UBLK_IO_FLAG_AUTO_BUF_REG; - return true; + return AUTO_BUF_REG_OK; } -static bool ublk_prep_auto_buf_reg(struct ublk_queue *ubq, - struct request *req, struct ublk_io *io, - unsigned int issue_flags) +static void ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req, + struct ublk_io *io, struct io_uring_cmd *cmd, + unsigned int issue_flags) { - ublk_init_req_ref(ubq, io); - if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) - return ublk_auto_buf_reg(ubq, req, io, issue_flags); + enum auto_buf_reg_res res = __ublk_do_auto_buf_reg(ubq, req, io, cmd, + issue_flags); - return true; + if (res != AUTO_BUF_REG_FAIL) { + ublk_prep_auto_buf_reg_io(ubq, req, io, cmd, res); + io_uring_cmd_done(cmd, UBLK_IO_RES_OK, issue_flags); + } } static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, @@ -1344,8 +1389,12 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, if (!ublk_start_io(ubq, req, io)) return; - if (ublk_prep_auto_buf_reg(ubq, req, io, issue_flags)) + if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) { + ublk_do_auto_buf_reg(ubq, req, io, io->cmd, issue_flags); + } else { + ublk_init_req_ref(ubq, io); ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags); + } } static void ublk_cmd_tw_cb(struct io_uring_cmd *cmd, @@ -1537,7 +1586,7 @@ static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) */ io->flags &= UBLK_IO_FLAG_CANCELED; io->cmd = NULL; - io->addr = 0; + io->buf.addr = 0; /* * old task is PF_EXITING, put it now @@ -1600,6 +1649,27 @@ static void ublk_put_disk(struct gendisk *disk) put_device(disk_to_dev(disk)); } +static void ublk_partition_scan_work(struct work_struct *work) +{ + struct ublk_device *ub = + container_of(work, struct ublk_device, partition_scan_work); + /* Hold disk reference to prevent UAF during concurrent teardown */ + struct gendisk *disk = ublk_get_disk(ub); + + if (!disk) + return; + + if (WARN_ON_ONCE(!test_and_clear_bit(GD_SUPPRESS_PART_SCAN, + &disk->state))) + goto out; + + mutex_lock(&disk->open_mutex); + bdev_disk_changed(disk, false); + mutex_unlock(&disk->open_mutex); +out: + ublk_put_disk(disk); +} + /* * Use this function to ensure that ->canceling is consistently set for * the device and all queues. Do not set these flags directly. @@ -1625,8 +1695,7 @@ static bool ublk_check_and_reset_active_ref(struct ublk_device *ub) { int i, j; - if (!(ub->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | - UBLK_F_AUTO_BUF_REG))) + if (!ublk_dev_need_req_ref(ub)) return false; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { @@ -2045,6 +2114,7 @@ static void ublk_stop_dev(struct ublk_device *ub) mutex_lock(&ub->mutex); ublk_stop_dev_unlocked(ub); mutex_unlock(&ub->mutex); + cancel_work_sync(&ub->partition_scan_work); ublk_cancel_dev(ub); } @@ -2098,13 +2168,16 @@ static inline int ublk_check_cmd_op(u32 cmd_op) static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd) { - io->buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + struct ublk_auto_buf_reg buf; + + buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); - if (io->buf.reserved0 || io->buf.reserved1) + if (buf.reserved0 || buf.reserved1) return -EINVAL; - if (io->buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) + if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) return -EINVAL; + io->buf.auto_reg = buf; return 0; } @@ -2126,7 +2199,7 @@ static int ublk_handle_auto_buf_reg(struct ublk_io *io, * this ublk request gets stuck. */ if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) - *buf_idx = io->buf.index; + *buf_idx = io->buf.auto_reg.index; } return ublk_set_auto_buf_reg(io, cmd); @@ -2154,7 +2227,7 @@ ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io, if (ublk_dev_support_auto_buf_reg(ub)) return ublk_handle_auto_buf_reg(io, cmd, buf_idx); - io->addr = buf_addr; + io->buf.addr = buf_addr; return 0; } @@ -2351,7 +2424,7 @@ static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io, */ io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA; /* update iod->addr because ublksrv may have passed a new io buffer */ - ublk_get_iod(ubq, req->tag)->addr = io->addr; + ublk_get_iod(ubq, req->tag)->addr = io->buf.addr; pr_devel("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n", __func__, ubq->q_id, req->tag, io->flags, ublk_get_iod(ubq, req->tag)->addr); @@ -2367,7 +2440,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, u16 buf_idx = UBLK_INVALID_BUF_IDX; struct ublk_device *ub = cmd->file->private_data; struct ublk_queue *ubq; - struct ublk_io *io; + struct ublk_io *io = NULL; u32 cmd_op = cmd->cmd_op; u16 q_id = READ_ONCE(ub_src->q_id); u16 tag = READ_ONCE(ub_src->tag); @@ -2453,11 +2526,11 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, io->res = result; req = ublk_fill_io_cmd(io, cmd); ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + if (buf_idx != UBLK_INVALID_BUF_IDX) + io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); /* can't touch 'ublk_io' any more */ - if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = addr; if (compl) @@ -2488,7 +2561,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, out: pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n", - __func__, cmd_op, tag, ret, io->flags); + __func__, cmd_op, tag, ret, io ? io->flags : 0); return ret; } @@ -2662,9 +2735,13 @@ static const struct file_operations ublk_ch_fops = { static void ublk_deinit_queue(struct ublk_device *ub, int q_id) { - int size = ublk_queue_cmd_buf_size(ub); - struct ublk_queue *ubq = ublk_get_queue(ub, q_id); - int i; + struct ublk_queue *ubq = ub->queues[q_id]; + int size, i; + + if (!ubq) + return; + + size = ublk_queue_cmd_buf_size(ub); for (i = 0; i < ubq->q_depth; i++) { struct ublk_io *io = &ubq->ios[i]; @@ -2676,57 +2753,76 @@ static void ublk_deinit_queue(struct ublk_device *ub, int q_id) if (ubq->io_cmd_buf) free_pages((unsigned long)ubq->io_cmd_buf, get_order(size)); + + kvfree(ubq); + ub->queues[q_id] = NULL; +} + +static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id) +{ + unsigned int cpu; + + /* Find first CPU mapped to this queue */ + for_each_possible_cpu(cpu) { + if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[cpu] == q_id) + return cpu_to_node(cpu); + } + + return NUMA_NO_NODE; } static int ublk_init_queue(struct ublk_device *ub, int q_id) { - struct ublk_queue *ubq = ublk_get_queue(ub, q_id); + int depth = ub->dev_info.queue_depth; + int ubq_size = sizeof(struct ublk_queue) + depth * sizeof(struct ublk_io); gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO; - void *ptr; + struct ublk_queue *ubq; + struct page *page; + int numa_node; int size; + /* Determine NUMA node based on queue's CPU affinity */ + numa_node = ublk_get_queue_numa_node(ub, q_id); + + /* Allocate queue structure on local NUMA node */ + ubq = kvzalloc_node(ubq_size, GFP_KERNEL, numa_node); + if (!ubq) + return -ENOMEM; + spin_lock_init(&ubq->cancel_lock); ubq->flags = ub->dev_info.flags; ubq->q_id = q_id; - ubq->q_depth = ub->dev_info.queue_depth; + ubq->q_depth = depth; size = ublk_queue_cmd_buf_size(ub); - ptr = (void *) __get_free_pages(gfp_flags, get_order(size)); - if (!ptr) + /* Allocate I/O command buffer on local NUMA node */ + page = alloc_pages_node(numa_node, gfp_flags, get_order(size)); + if (!page) { + kvfree(ubq); return -ENOMEM; + } + ubq->io_cmd_buf = page_address(page); - ubq->io_cmd_buf = ptr; + ub->queues[q_id] = ubq; ubq->dev = ub; return 0; } static void ublk_deinit_queues(struct ublk_device *ub) { - int nr_queues = ub->dev_info.nr_hw_queues; int i; - if (!ub->__queues) - return; - - for (i = 0; i < nr_queues; i++) + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) ublk_deinit_queue(ub, i); - kvfree(ub->__queues); } static int ublk_init_queues(struct ublk_device *ub) { - int nr_queues = ub->dev_info.nr_hw_queues; - int depth = ub->dev_info.queue_depth; - int ubq_size = sizeof(struct ublk_queue) + depth * sizeof(struct ublk_io); - int i, ret = -ENOMEM; - - ub->queue_size = ubq_size; - ub->__queues = kvcalloc(nr_queues, ubq_size, GFP_KERNEL); - if (!ub->__queues) - return ret; + int i, ret; - for (i = 0; i < nr_queues; i++) { - if (ublk_init_queue(ub, i)) + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { + ret = ublk_init_queue(ub, i); + if (ret) goto fail; } @@ -2858,6 +2954,15 @@ static struct ublk_device *ublk_get_device_from_id(int idx) return ub; } +static bool ublk_validate_user_pid(struct ublk_device *ub, pid_t ublksrv_pid) +{ + rcu_read_lock(); + ublksrv_pid = pid_nr(find_vpid(ublksrv_pid)); + rcu_read_unlock(); + + return ub->ublksrv_tgid == ublksrv_pid; +} + static int ublk_ctrl_start_dev(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) { @@ -2926,7 +3031,7 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub, if (wait_for_completion_interruptible(&ub->completion) != 0) return -EINTR; - if (ub->ublksrv_tgid != ublksrv_pid) + if (!ublk_validate_user_pid(ub, ublksrv_pid)) return -EINVAL; mutex_lock(&ub->mutex); @@ -2945,14 +3050,22 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub, disk->fops = &ub_fops; disk->private_data = ub; - ub->dev_info.ublksrv_pid = ublksrv_pid; + ub->dev_info.ublksrv_pid = ub->ublksrv_tgid; ub->ub_disk = disk; ublk_apply_params(ub); - /* don't probe partitions if any daemon task is un-trusted */ - if (ub->unprivileged_daemons) - set_bit(GD_SUPPRESS_PART_SCAN, &disk->state); + /* + * Suppress partition scan to avoid potential IO hang. + * + * If ublk server error occurs during partition scan, the IO may + * wait while holding ub->mutex, which can deadlock with other + * operations that need the mutex. Defer partition scan to async + * work. + * For unprivileged daemons, keep GD_SUPPRESS_PART_SCAN set + * permanently. + */ + set_bit(GD_SUPPRESS_PART_SCAN, &disk->state); ublk_get_device(ub); ub->dev_info.state = UBLK_S_DEV_LIVE; @@ -2969,6 +3082,10 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub, set_bit(UB_STATE_USED, &ub->state); + /* Schedule async partition scan for trusted daemons */ + if (!ub->unprivileged_daemons) + schedule_work(&ub->partition_scan_work); + out_put_cdev: if (ret) { ublk_detach_disk(ub); @@ -3128,12 +3245,13 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) goto out_unlock; ret = -ENOMEM; - ub = kzalloc(sizeof(*ub), GFP_KERNEL); + ub = kzalloc(struct_size(ub, queues, info.nr_hw_queues), GFP_KERNEL); if (!ub) goto out_unlock; mutex_init(&ub->mutex); spin_lock_init(&ub->lock); mutex_init(&ub->cancel_mutex); + INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work); ret = ublk_alloc_dev_number(ub, header->dev_id); if (ret < 0) @@ -3178,17 +3296,17 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) ub->dev_info.nr_hw_queues, nr_cpu_ids); ublk_align_max_io_size(ub); - ret = ublk_init_queues(ub); + ret = ublk_add_tag_set(ub); if (ret) goto out_free_dev_number; - ret = ublk_add_tag_set(ub); + ret = ublk_init_queues(ub); if (ret) - goto out_deinit_queues; + goto out_free_tag_set; ret = -EFAULT; if (copy_to_user(argp, &ub->dev_info, sizeof(info))) - goto out_free_tag_set; + goto out_deinit_queues; /* * Add the char dev so that ublksrv daemon can be setup. @@ -3197,10 +3315,10 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) ret = ublk_add_chdev(ub); goto out_unlock; -out_free_tag_set: - blk_mq_free_tag_set(&ub->tag_set); out_deinit_queues: ublk_deinit_queues(ub); +out_free_tag_set: + blk_mq_free_tag_set(&ub->tag_set); out_free_dev_number: ublk_free_dev_number(ub); out_free_ub: @@ -3280,12 +3398,32 @@ static int ublk_ctrl_stop_dev(struct ublk_device *ub) static int ublk_ctrl_get_dev_info(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) { + struct task_struct *p; + struct pid *pid; + struct ublksrv_ctrl_dev_info dev_info; + pid_t init_ublksrv_tgid = ub->dev_info.ublksrv_pid; void __user *argp = (void __user *)(unsigned long)header->addr; if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr) return -EINVAL; - if (copy_to_user(argp, &ub->dev_info, sizeof(ub->dev_info))) + memcpy(&dev_info, &ub->dev_info, sizeof(dev_info)); + dev_info.ublksrv_pid = -1; + + if (init_ublksrv_tgid > 0) { + rcu_read_lock(); + pid = find_pid_ns(init_ublksrv_tgid, &init_pid_ns); + p = pid_task(pid, PIDTYPE_TGID); + if (p) { + int vnr = task_tgid_vnr(p); + + if (vnr) + dev_info.ublksrv_pid = vnr; + } + rcu_read_unlock(); + } + + if (copy_to_user(argp, &dev_info, sizeof(dev_info))) return -EFAULT; return 0; @@ -3430,7 +3568,7 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub, pr_devel("%s: All FETCH_REQs received, dev id %d\n", __func__, header->dev_id); - if (ub->ublksrv_tgid != ublksrv_pid) + if (!ublk_validate_user_pid(ub, ublksrv_pid)) return -EINVAL; mutex_lock(&ub->mutex); @@ -3441,7 +3579,7 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub, ret = -EBUSY; goto out_unlock; } - ub->dev_info.ublksrv_pid = ublksrv_pid; + ub->dev_info.ublksrv_pid = ub->ublksrv_tgid; ub->dev_info.state = UBLK_S_DEV_LIVE; pr_devel("%s: new ublksrv_pid %d, dev id %d\n", __func__, ublksrv_pid, header->dev_id); @@ -3466,15 +3604,22 @@ static int ublk_ctrl_get_features(const struct ublksrv_ctrl_cmd *header) return 0; } -static void ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) +static int ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) { struct ublk_param_basic *p = &ub->params.basic; u64 new_size = header->data[0]; + int ret = 0; mutex_lock(&ub->mutex); + if (!ub->ub_disk) { + ret = -ENODEV; + goto out; + } p->dev_sectors = new_size; set_capacity_and_notify(ub->ub_disk, p->dev_sectors); +out: mutex_unlock(&ub->mutex); + return ret; } struct count_busy { @@ -3703,10 +3848,10 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; - ublk_ctrl_cmd_dump(cmd); - if (!(issue_flags & IO_URING_F_SQE128)) - goto out; + return -EINVAL; + + ublk_ctrl_cmd_dump(cmd); ret = ublk_check_cmd_op(cmd_op); if (ret) @@ -3764,8 +3909,7 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, ret = ublk_ctrl_end_recovery(ub, header); break; case UBLK_CMD_UPDATE_SIZE: - ublk_ctrl_set_size(ub, header); - ret = 0; + ret = ublk_ctrl_set_size(ub, header); break; case UBLK_CMD_QUIESCE_DEV: ret = ublk_ctrl_quiesce_dev(ub, header); diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index a423228e201ba..26364e43aeb34 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -177,7 +177,7 @@ static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no) zone->wp = zone->start; } else if (file_sectors == zlo->zone_capacity) { zone->cond = BLK_ZONE_COND_FULL; - zone->wp = zone->start + zlo->zone_size; + zone->wp = ULLONG_MAX; } else { zone->cond = BLK_ZONE_COND_CLOSED; zone->wp = zone->start + file_sectors; @@ -326,7 +326,7 @@ static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) } zone->cond = BLK_ZONE_COND_FULL; - zone->wp = zone->start + zlo->zone_size; + zone->wp = ULLONG_MAX; clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); unlock: @@ -407,6 +407,10 @@ static void zloop_rw(struct zloop_cmd *cmd) mutex_lock(&zone->lock); if (is_append) { + if (zone->cond == BLK_ZONE_COND_FULL) { + ret = -EIO; + goto unlock; + } sector = zone->wp; cmd->sector = sector; } @@ -433,8 +437,10 @@ static void zloop_rw(struct zloop_cmd *cmd) * copmpletes. */ zone->wp += nr_sectors; - if (zone->wp == zone_end) + if (zone->wp == zone_end) { zone->cond = BLK_ZONE_COND_FULL; + zone->wp = ULLONG_MAX; + } } rq_for_each_bvec(tmp, rq, rq_iter) @@ -493,6 +499,21 @@ static void zloop_rw(struct zloop_cmd *cmd) zloop_put_cmd(cmd); } +/* + * Sync the entire FS containing the zone files instead of walking all files. + */ +static int zloop_flush(struct zloop_device *zlo) +{ + struct super_block *sb = file_inode(zlo->data_dir)->i_sb; + int ret; + + down_read(&sb->s_umount); + ret = sync_filesystem(sb); + up_read(&sb->s_umount); + + return ret; +} + static void zloop_handle_cmd(struct zloop_cmd *cmd) { struct request *rq = blk_mq_rq_from_pdu(cmd); @@ -509,11 +530,7 @@ static void zloop_handle_cmd(struct zloop_cmd *cmd) zloop_rw(cmd); return; case REQ_OP_FLUSH: - /* - * Sync the entire FS containing the zone files instead of - * walking all files - */ - cmd->ret = sync_filesystem(file_inode(zlo->data_dir)->i_sb); + cmd->ret = zloop_flush(zlo); break; case REQ_OP_ZONE_RESET: cmd->ret = zloop_reset_zone(zlo, rq_zone_no(rq)); @@ -886,7 +903,8 @@ static int zloop_ctl_add(struct zloop_options *opts) .max_hw_sectors = SZ_1M >> SECTOR_SHIFT, .max_hw_zone_append_sectors = SZ_1M >> SECTOR_SHIFT, .chunk_sectors = opts->zone_size, - .features = BLK_FEAT_ZONED, + .features = BLK_FEAT_ZONED | BLK_FEAT_WRITE_CACHE, + }; unsigned int nr_zones, i, j; struct zloop_device *zlo; @@ -1058,7 +1076,12 @@ static int zloop_ctl_remove(struct zloop_options *opts) int ret; if (!(opts->mask & ZLOOP_OPT_ID)) { - pr_err("No ID specified\n"); + pr_err("No ID specified for remove\n"); + return -EINVAL; + } + + if (opts->mask & ~ZLOOP_OPT_ID) { + pr_err("Invalid option specified for remove\n"); return -EINVAL; } diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 9d29ab811f802..5e0a05edcbfd1 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -251,11 +251,13 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code) bt_dev_err(hdev, "Hardware error 0x%2.2x", code); + hci_req_sync_lock(hdev); + skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { bt_dev_err(hdev, "Reset after hardware error failed (%ld)", PTR_ERR(skb)); - return; + goto unlock; } kfree_skb(skb); @@ -263,18 +265,21 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code) if (IS_ERR(skb)) { bt_dev_err(hdev, "Retrieving Intel exception info failed (%ld)", PTR_ERR(skb)); - return; + goto unlock; } if (skb->len != 13) { bt_dev_err(hdev, "Exception info size mismatch"); kfree_skb(skb); - return; + goto unlock; } bt_dev_err(hdev, "Exception info %s", (char *)(skb->data + 1)); kfree_skb(skb); + +unlock: + hci_req_sync_unlock(hdev); } EXPORT_SYMBOL_GPL(btintel_hw_error); diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index a075d8ec46773..c68a8de3025b0 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -825,6 +825,11 @@ static inline bool btintel_pcie_in_d0(struct btintel_pcie_data *data) return !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY); } +static inline bool btintel_pcie_in_device_halt(struct btintel_pcie_data *data) +{ + return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_HALTED; +} + static void btintel_pcie_wr_sleep_cntrl(struct btintel_pcie_data *data, u32 dxstate) { @@ -1425,11 +1430,6 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) } } -static irqreturn_t btintel_pcie_msix_isr(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data) { return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; @@ -1531,9 +1531,9 @@ static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) err = devm_request_threaded_irq(&data->pdev->dev, msix_entry->vector, - btintel_pcie_msix_isr, + NULL, btintel_pcie_irq_msix_handler, - IRQF_SHARED, + IRQF_ONESHOT | IRQF_SHARED, KBUILD_MODNAME, msix_entry); if (err) { @@ -2532,6 +2532,8 @@ static int btintel_pcie_suspend_late(struct device *dev, pm_message_t mesg) dxstate = (mesg.event == PM_EVENT_SUSPEND ? BTINTEL_PCIE_STATE_D3_HOT : BTINTEL_PCIE_STATE_D3_COLD); + data->pm_sx_event = mesg.event; + data->gp0_received = false; start = ktime_get(); @@ -2581,6 +2583,20 @@ static int btintel_pcie_resume(struct device *dev) start = ktime_get(); + /* When the system enters S4 (hibernate) mode, bluetooth device loses + * power, which results in the erasure of its loaded firmware. + * Consequently, function level reset (flr) is required on system + * resume to bring the controller back into an operational state by + * initiating a new firmware download. + */ + + if (data->pm_sx_event == PM_EVENT_FREEZE || + data->pm_sx_event == PM_EVENT_HIBERNATE) { + set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); + btintel_pcie_reset(data->hdev); + return 0; + } + /* Refer: 6.4.11.7 -> Platform power management */ btintel_pcie_wr_sleep_cntrl(data, BTINTEL_PCIE_STATE_D0); err = wait_event_timeout(data->gp0_wait_q, data->gp0_received, @@ -2589,6 +2605,26 @@ static int btintel_pcie_resume(struct device *dev) bt_dev_err(data->hdev, "Timeout (%u ms) on alive interrupt for D0 entry", BTINTEL_DEFAULT_INTR_TIMEOUT_MS); + + /* Trigger function level reset if the controller is in error + * state during resume() to bring back the controller to + * operational mode + */ + + data->boot_stage_cache = btintel_pcie_rd_reg32(data, + BTINTEL_PCIE_CSR_BOOT_STAGE_REG); + if (btintel_pcie_in_error(data) || + btintel_pcie_in_device_halt(data)) { + bt_dev_err(data->hdev, "Controller in error state for D0 entry"); + if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, + &data->flags)) { + data->dmp_hdr.trigger_reason = + BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT; + queue_work(data->workqueue, &data->rx_work); + } + set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); + btintel_pcie_reset(data->hdev); + } return -EBUSY; } diff --git a/drivers/bluetooth/btintel_pcie.h b/drivers/bluetooth/btintel_pcie.h index 04b21f968ad30..48e1ae1793e5c 100644 --- a/drivers/bluetooth/btintel_pcie.h +++ b/drivers/bluetooth/btintel_pcie.h @@ -464,6 +464,7 @@ struct btintel_pcie_dump_header { * @txq: TX Queue struct * @rxq: RX Queue struct * @alive_intr_ctxt: Alive interrupt context + * @pm_sx_event: PM event on which system got suspended */ struct btintel_pcie_data { struct pci_dev *pdev; @@ -513,6 +514,7 @@ struct btintel_pcie_data { u32 alive_intr_ctxt; struct btintel_pcie_dbgc dbgc; struct btintel_pcie_dump_header dmp_hdr; + u8 pm_sx_event; }; static inline u32 btintel_pcie_rd_reg32(struct btintel_pcie_data *data, diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index 7c958d6065bec..3b06269201934 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -581,28 +581,11 @@ static int qca_download_firmware(struct hci_dev *hdev, ret = request_firmware(&fw, config->fwname, &hdev->dev); if (ret) { - /* For WCN6750, if mbn file is not present then check for - * tlv file. - */ - if (soc_type == QCA_WCN6750 && config->type == ELF_TYPE_PATCH) { - bt_dev_dbg(hdev, "QCA Failed to request file: %s (%d)", - config->fwname, ret); - config->type = TLV_TYPE_PATCH; - snprintf(config->fwname, sizeof(config->fwname), - "qca/msbtfw%02x.tlv", rom_ver); - bt_dev_info(hdev, "QCA Downloading %s", config->fwname); - ret = request_firmware(&fw, config->fwname, &hdev->dev); - if (ret) { - bt_dev_err(hdev, "QCA Failed to request file: %s (%d)", - config->fwname, ret); - return ret; - } - } /* If the board-specific file is missing, try loading the default * one, unless that was attempted already. */ - else if (config->type == TLV_TYPE_NVM && - qca_get_alt_nvm_file(config->fwname, sizeof(config->fwname))) { + if (config->type == TLV_TYPE_NVM && + qca_get_alt_nvm_file(config->fwname, sizeof(config->fwname))) { bt_dev_info(hdev, "QCA Downloading %s", config->fwname); ret = request_firmware(&fw, config->fwname, &hdev->dev); if (ret) { @@ -804,6 +787,8 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, */ if (soc_type == QCA_WCN3988) rom_ver = ((soc_ver & 0x00000f00) >> 0x05) | (soc_ver & 0x0000000f); + else if (soc_type == QCA_WCN3998) + rom_ver = ((soc_ver & 0x0000f000) >> 0x07) | (soc_ver & 0x0000000f); else rom_ver = ((soc_ver & 0x00000f00) >> 0x04) | (soc_ver & 0x0000000f); @@ -847,8 +832,12 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, "qca/msbtfw%02x.mbn", rom_ver); break; case QCA_WCN6855: + /* Due to historical reasons, WCN685x chip has been using firmware + * without the "wcn" prefix. The mapping between the chip and its + * corresponding firmware has now been corrected. + */ snprintf(config.fwname, sizeof(config.fwname), - "qca/hpbtfw%02x.tlv", rom_ver); + "qca/wcnhpbtfw%02x.tlv", rom_ver); break; case QCA_WCN7850: snprintf(config.fwname, sizeof(config.fwname), @@ -861,8 +850,26 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, } err = qca_download_firmware(hdev, &config, soc_type, rom_ver); + /* For WCN6750, if mbn file is not present then check for + * tlv file. + */ + if (err < 0 && soc_type == QCA_WCN6750) { + bt_dev_dbg(hdev, "QCA Failed to request file: %s (%d)", + config.fwname, err); + config.type = TLV_TYPE_PATCH; + snprintf(config.fwname, sizeof(config.fwname), + "qca/msbtfw%02x.tlv", rom_ver); + bt_dev_info(hdev, "QCA Downloading %s", config.fwname); + err = qca_download_firmware(hdev, &config, soc_type, rom_ver); + } else if (err < 0 && !rampatch_name && soc_type == QCA_WCN6855) { + snprintf(config.fwname, sizeof(config.fwname), + "qca/hpbtfw%02x.tlv", rom_ver); + err = qca_download_firmware(hdev, &config, soc_type, rom_ver); + } + if (err < 0) { - bt_dev_err(hdev, "QCA Failed to download patch (%d)", err); + bt_dev_err(hdev, "QCA Failed to request file: %s (%d)", + config.fwname, err); return err; } @@ -923,7 +930,7 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, case QCA_WCN6855: qca_read_fw_board_id(hdev, &boardid); qca_get_nvm_name_by_board(config.fwname, sizeof(config.fwname), - "hpnv", soc_type, ver, rom_ver, boardid); + "wcnhpnv", soc_type, ver, rom_ver, boardid); break; case QCA_WCN7850: qca_get_nvm_name_by_board(config.fwname, sizeof(config.fwname), @@ -936,8 +943,15 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, } err = qca_download_firmware(hdev, &config, soc_type, rom_ver); + if (err < 0 && !firmware_name && soc_type == QCA_WCN6855) { + qca_get_nvm_name_by_board(config.fwname, sizeof(config.fwname), + "hpnv", soc_type, ver, rom_ver, boardid); + err = qca_download_firmware(hdev, &config, soc_type, rom_ver); + } + if (err < 0) { - bt_dev_err(hdev, "QCA Failed to download NVM (%d)", err); + bt_dev_err(hdev, "QCA Failed to request file: %s (%d)", + config.fwname, err); return err; } diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index fa683bb7f0b49..3d20e1b5b6799 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -504,6 +504,8 @@ static const struct usb_device_id quirks_table[] = { /* Realtek 8821CE Bluetooth devices */ { USB_DEVICE(0x13d3, 0x3529), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3533), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8822CE Bluetooth devices */ { USB_DEVICE(0x0bda, 0xb00c), .driver_info = BTUSB_REALTEK | @@ -519,12 +521,16 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x0bda, 0xb850), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3600), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3601), .driver_info = BTUSB_REALTEK }, + { USB_DEVICE(0x0489, 0xe112), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8851BU Bluetooth devices */ { USB_DEVICE(0x3625, 0x010b), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2001, 0x332a), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x7392, 0xe611), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8852AE Bluetooth devices */ { USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK | @@ -555,6 +561,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3592), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3612), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe122), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, @@ -585,6 +593,8 @@ static const struct usb_device_id quirks_table[] = { /* Realtek 8852BT/8852BE-VT Bluetooth devices */ { USB_DEVICE(0x0bda, 0x8520), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe12f), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8922AE Bluetooth devices */ { USB_DEVICE(0x0bda, 0x8922), .driver_info = BTUSB_REALTEK | @@ -621,12 +631,16 @@ static const struct usb_device_id quirks_table[] = { /* Additional MediaTek MT7920 Bluetooth devices */ { USB_DEVICE(0x0489, 0xe134), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe135), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3620), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3621), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3622), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe158), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, /* Additional MediaTek MT7921 Bluetooth devices */ { USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK | @@ -685,6 +699,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe153), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe170), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x04ca, 0x3804), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x04ca, 0x38e4), .driver_info = BTUSB_MEDIATEK | @@ -761,6 +777,7 @@ static const struct usb_device_id quirks_table[] = { /* Additional Realtek 8723BU Bluetooth devices */ { USB_DEVICE(0x7392, 0xa611), .driver_info = BTUSB_REALTEK }, + { USB_DEVICE(0x2c0a, 0x8761), .driver_info = BTUSB_REALTEK }, /* Additional Realtek 8723DE Bluetooth devices */ { USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK }, @@ -781,6 +798,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2b89, 0x8761), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x2b89, 0x6275), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Additional Realtek 8821AE Bluetooth devices */ { USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK }, @@ -2337,8 +2356,11 @@ static void btusb_work(struct work_struct *work) if (data->air_mode == HCI_NOTIFY_ENABLE_SCO_CVSD) { if (hdev->voice_setting & 0x0020) { static const int alts[3] = { 2, 4, 5 }; + unsigned int sco_idx; - new_alts = alts[data->sco_num - 1]; + sco_idx = min_t(unsigned int, data->sco_num - 1, + ARRAY_SIZE(alts) - 1); + new_alts = alts[sco_idx]; } else { new_alts = data->sco_num; } @@ -3261,6 +3283,7 @@ static const struct qca_device_info qca_devices_table[] = { static const struct qca_custom_firmware qca_custom_btfws[] = { { 0x00130201, 0x030A, "QCA2066" }, + { 0x00130201, 0x030B, "QCA2066" }, { }, }; @@ -3455,7 +3478,10 @@ static int btusb_setup_qca_load_rampatch(struct hci_dev *hdev, "firmware rome 0x%x build 0x%x", rver_rom, rver_patch, ver_rom, ver_patch); - if (rver_rom != ver_rom || rver_patch <= ver_patch) { + /* Allow rampatch if version is greater than or equal to firmware version. + * Equal versions are acceptable for re-flashing or recovery scenarios. + */ + if (rver_rom != ver_rom || rver_patch < ver_patch) { bt_dev_err(hdev, "rampatch file version did not match with firmware"); err = -EINVAL; goto done; @@ -4006,7 +4032,7 @@ static int btusb_probe(struct usb_interface *intf, return -ENODEV; } - data = devm_kzalloc(&intf->dev, sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -4029,8 +4055,10 @@ static int btusb_probe(struct usb_interface *intf, } } - if (!data->intr_ep || !data->bulk_tx_ep || !data->bulk_rx_ep) + if (!data->intr_ep || !data->bulk_tx_ep || !data->bulk_rx_ep) { + kfree(data); return -ENODEV; + } if (id->driver_info & BTUSB_AMP) { data->cmdreq_type = USB_TYPE_CLASS | 0x01; @@ -4085,8 +4113,10 @@ static int btusb_probe(struct usb_interface *intf, data->recv_acl = hci_recv_frame; hdev = hci_alloc_dev_priv(priv_size); - if (!hdev) + if (!hdev) { + kfree(data); return -ENOMEM; + } hdev->bus = HCI_USB; hci_set_drvdata(hdev, data); @@ -4359,6 +4389,7 @@ static int btusb_probe(struct usb_interface *intf, if (data->reset_gpio) gpiod_put(data->reset_gpio); hci_free_dev(hdev); + kfree(data); return err; } @@ -4407,6 +4438,7 @@ static void btusb_disconnect(struct usb_interface *intf) } hci_free_dev(hdev); + kfree(data); } #ifdef CONFIG_PM diff --git a/drivers/bluetooth/hci_h4.c b/drivers/bluetooth/hci_h4.c index ec017df8572c8..1e9e2cad9ddf6 100644 --- a/drivers/bluetooth/hci_h4.c +++ b/drivers/bluetooth/hci_h4.c @@ -109,9 +109,6 @@ static int h4_recv(struct hci_uart *hu, const void *data, int count) { struct h4_struct *h4 = hu->priv; - if (!test_bit(HCI_UART_REGISTERED, &hu->flags)) - return -EUNATCH; - h4->rx_skb = h4_recv_buf(hu, h4->rx_skb, data, count, h4_recv_pkts, ARRAY_SIZE(h4_recv_pkts)); if (IS_ERR(h4->rx_skb)) { diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index d0adae3267b41..2b28515de92c4 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -685,6 +685,8 @@ static int hci_uart_register_dev(struct hci_uart *hu) return err; } + set_bit(HCI_UART_PROTO_INIT, &hu->flags); + if (test_bit(HCI_UART_INIT_PENDING, &hu->hdev_flags)) return 0; @@ -712,8 +714,6 @@ static int hci_uart_set_proto(struct hci_uart *hu, int id) hu->proto = p; - set_bit(HCI_UART_PROTO_INIT, &hu->flags); - err = hci_uart_register_dev(hu); if (err) { return err; diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c index 6f4e25917b863..c4584f4085766 100644 --- a/drivers/bluetooth/hci_ll.c +++ b/drivers/bluetooth/hci_ll.c @@ -541,6 +541,8 @@ static int download_firmware(struct ll_device *lldev) if (err || !fw->data || !fw->size) { bt_dev_err(lldev->hu.hdev, "request_firmware failed(errno %d) for %s", err, bts_scr_name); + if (!err) + release_firmware(fw); return -EINVAL; } ptr = (void *)fw->data; diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 888176b0faa90..a76c6e42005b3 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -87,6 +87,7 @@ enum qca_flags { enum qca_capabilities { QCA_CAP_WIDEBAND_SPEECH = BIT(0), QCA_CAP_VALID_LE_STATES = BIT(1), + QCA_CAP_HFP_HW_OFFLOAD = BIT(2), }; /* HCI_IBS transmit side sleep protocol states */ @@ -229,6 +230,7 @@ struct qca_serdev { u32 init_speed; u32 oper_speed; bool bdaddr_property_broken; + bool support_hfp_hw_offload; const char *firmware_name[2]; }; @@ -1653,6 +1655,39 @@ static void qca_hw_error(struct hci_dev *hdev, u8 code) skb_queue_purge(&qca->rx_memdump_q); } + /* + * If the BT chip's bt_en pin is connected to a 3.3V power supply via + * hardware and always stays high, driver cannot control the bt_en pin. + * As a result, during SSR (SubSystem Restart), QCA_SSR_TRIGGERED and + * QCA_IBS_DISABLED flags cannot be cleared, which leads to a reset + * command timeout. + * Add an msleep delay to ensure controller completes the SSR process. + * + * Host will not download the firmware after SSR, controller to remain + * in the IBS_WAKE state, and the host needs to synchronize with it + * + * Since the bluetooth chip has been reset, clear the memdump state. + */ + if (!hci_test_quirk(hu->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP)) { + /* + * When the SSR (SubSystem Restart) duration exceeds 2 seconds, + * it triggers host tx_idle_delay, which sets host TX state + * to sleep. Reset tx_idle_timer after SSR to prevent + * host enter TX IBS_Sleep mode. + */ + mod_timer(&qca->tx_idle_timer, jiffies + + msecs_to_jiffies(qca->tx_idle_delay)); + + /* Controller reset completion time is 50ms */ + msleep(50); + + clear_bit(QCA_SSR_TRIGGERED, &qca->flags); + clear_bit(QCA_IBS_DISABLED, &qca->flags); + + qca->tx_ibs_state = HCI_IBS_TX_AWAKE; + qca->memdump_state = QCA_MEMDUMP_IDLE; + } + clear_bit(QCA_HW_ERROR_EVENT, &qca->flags); } @@ -1879,7 +1914,7 @@ static int qca_setup(struct hci_uart *hu) const char *rampatch_name = qca_get_rampatch_name(hu); int ret; struct qca_btsoc_version ver; - struct qca_serdev *qcadev; + struct qca_serdev *qcadev = serdev_device_get_drvdata(hu->serdev); const char *soc_name; ret = qca_check_speeds(hu); @@ -1943,7 +1978,6 @@ static int qca_setup(struct hci_uart *hu) case QCA_WCN6750: case QCA_WCN6855: case QCA_WCN7850: - qcadev = serdev_device_get_drvdata(hu->serdev); if (qcadev->bdaddr_property_broken) hci_set_quirk(hdev, HCI_QUIRK_BDADDR_PROPERTY_BROKEN); @@ -2012,19 +2046,23 @@ static int qca_setup(struct hci_uart *hu) } out: - if (ret && retries < MAX_INIT_RETRIES) { - bt_dev_warn(hdev, "Retry BT power ON:%d", retries); + if (ret) { qca_power_shutdown(hu); - if (hu->serdev) { - serdev_device_close(hu->serdev); - ret = serdev_device_open(hu->serdev); - if (ret) { - bt_dev_err(hdev, "failed to open port"); - return ret; + + if (retries < MAX_INIT_RETRIES) { + bt_dev_warn(hdev, "Retry BT power ON:%d", retries); + if (hu->serdev) { + serdev_device_close(hu->serdev); + ret = serdev_device_open(hu->serdev); + if (ret) { + bt_dev_err(hdev, "failed to open port"); + return ret; + } } + retries++; + goto retry; } - retries++; - goto retry; + return ret; } /* Setup bdaddr */ @@ -2033,7 +2071,7 @@ static int qca_setup(struct hci_uart *hu) else hu->hdev->set_bdaddr = qca_set_bdaddr; - if (soc_type == QCA_QCA2066) + if (qcadev->support_hfp_hw_offload) qca_configure_hfp_offload(hdev); qca->fw_version = le16_to_cpu(ver.patch_ver); @@ -2117,7 +2155,8 @@ static const struct qca_device_data qca_soc_data_wcn3998 __maybe_unused = { static const struct qca_device_data qca_soc_data_qca2066 __maybe_unused = { .soc_type = QCA_QCA2066, .num_vregs = 0, - .capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES, + .capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES | + QCA_CAP_HFP_HW_OFFLOAD, }; static const struct qca_device_data qca_soc_data_qca6390 __maybe_unused = { @@ -2153,7 +2192,8 @@ static const struct qca_device_data qca_soc_data_wcn6855 __maybe_unused = { { "vddrfa1p2", 257000 }, }, .num_vregs = 6, - .capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES, + .capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES | + QCA_CAP_HFP_HW_OFFLOAD, }; static const struct qca_device_data qca_soc_data_wcn7850 __maybe_unused = { @@ -2167,7 +2207,8 @@ static const struct qca_device_data qca_soc_data_wcn7850 __maybe_unused = { { "vddrfa1p9", 302000 }, }, .num_vregs = 6, - .capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES, + .capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES | + QCA_CAP_HFP_HW_OFFLOAD, }; static void qca_power_shutdown(struct hci_uart *hu) @@ -2430,7 +2471,8 @@ static int qca_serdev_probe(struct serdev_device *serdev) if (!qcadev->bt_en && (data->soc_type == QCA_WCN6750 || - data->soc_type == QCA_WCN6855)) + data->soc_type == QCA_WCN6855 || + data->soc_type == QCA_WCN7850)) power_ctrl_enabled = false; qcadev->sw_ctrl = devm_gpiod_get_optional(&serdev->dev, "swctrl", @@ -2502,6 +2544,9 @@ static int qca_serdev_probe(struct serdev_device *serdev) if (!(data->capabilities & QCA_CAP_VALID_LE_STATES)) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_LE_STATES); + + if (data->capabilities & QCA_CAP_HFP_HW_OFFLOAD) + qcadev->support_hfp_hw_offload = true; } return 0; diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 25845c04e5620..eb7b6c0ba9e7c 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -202,8 +202,12 @@ static ssize_t driver_override_show(struct device *dev, struct device_attribute *attr, char *buf) { struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); + ssize_t len; - return sysfs_emit(buf, "%s\n", mc_dev->driver_override); + device_lock(dev); + len = sysfs_emit(buf, "%s\n", mc_dev->driver_override); + device_unlock(dev); + return len; } static DEVICE_ATTR_RW(driver_override); @@ -905,11 +909,7 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc, return 0; error_cleanup_dev: - kfree(mc_dev->regions); - if (mc_bus) - kfree(mc_bus); - else - kfree(mc_dev); + put_device(&mc_dev->dev); return error; } diff --git a/drivers/bus/mhi/host/boot.c b/drivers/bus/mhi/host/boot.c index 205d83ac069f1..c76db35bc29a9 100644 --- a/drivers/bus/mhi/host/boot.c +++ b/drivers/bus/mhi/host/boot.c @@ -584,6 +584,16 @@ void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl) * device transitioning into MHI READY state */ if (fw_load_type == MHI_FW_LOAD_FBC) { + /* + * Some FW combine two separate ELF images (SBL + WLAN FW) in a single + * file. Hence, check for the existence of the second ELF header after + * SBL. If present, load the second image separately. + */ + if (!memcmp(fw_data + mhi_cntrl->sbl_size, ELFMAG, SELFMAG)) { + fw_data += mhi_cntrl->sbl_size; + fw_sz -= mhi_cntrl->sbl_size; + } + ret = mhi_alloc_bhie_table(mhi_cntrl, &mhi_cntrl->fbc_image, fw_sz); if (ret) { release_firmware(firmware); diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c index b188bbf7de042..760bf923dc593 100644 --- a/drivers/bus/mhi/host/pci_generic.c +++ b/drivers/bus/mhi/host/pci_generic.c @@ -94,22 +94,6 @@ struct mhi_pci_dev_info { .doorbell_mode_switch = false, \ } -#define MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(ch_num, ch_name, el_count, ev_ring) \ - { \ - .num = ch_num, \ - .name = ch_name, \ - .num_elements = el_count, \ - .event_ring = ev_ring, \ - .dir = DMA_FROM_DEVICE, \ - .ee_mask = BIT(MHI_EE_AMSS), \ - .pollcfg = 0, \ - .doorbell = MHI_DB_BRST_DISABLE, \ - .lpm_notify = false, \ - .offload_channel = false, \ - .doorbell_mode_switch = false, \ - .auto_queue = true, \ - } - #define MHI_EVENT_CONFIG_CTRL(ev_ring, el_count) \ { \ .num_elements = el_count, \ @@ -329,7 +313,7 @@ static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 8, 0), + MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0), MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0), MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 64, 2), @@ -423,6 +407,16 @@ static const struct mhi_pci_dev_info mhi_qcom_sdx55_info = { .sideband_wake = false, }; +static const struct mhi_pci_dev_info mhi_qcom_sdx35_info = { + .name = "qcom-sdx35m", + .config = &modem_qcom_v2_mhiv_config, + .bar_num = MHI_PCI_DEFAULT_BAR_NUM, + .dma_data_width = 32, + .mru_default = 32768, + .sideband_wake = false, + .edl_trigger = true, +}; + static const struct mhi_pci_dev_info mhi_qcom_sdx24_info = { .name = "qcom-sdx24", .edl = "qcom/prog_firehose_sdx24.mbn", @@ -751,7 +745,7 @@ static const struct mhi_channel_config mhi_telit_fn980_hw_v1_channels[] = { MHI_CHANNEL_CONFIG_UL(14, "QMI", 32, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 32, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 16, 0), - MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 16, 0), + MHI_CHANNEL_CONFIG_DL(21, "IPCR", 16, 0), MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 1), MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 2), }; @@ -877,6 +871,16 @@ static const struct mhi_pci_dev_info mhi_telit_fn990b40_info = { .edl_trigger = true, }; +static const struct mhi_pci_dev_info mhi_telit_fe990b40_info = { + .name = "telit-fe990b40", + .config = &modem_telit_fn920c04_config, + .bar_num = MHI_PCI_DEFAULT_BAR_NUM, + .dma_data_width = 32, + .sideband_wake = false, + .mru_default = 32768, + .edl_trigger = true, +}; + static const struct mhi_pci_dev_info mhi_netprisma_lcur57_info = { .name = "netprisma-lcur57", .edl = "qcom/prog_firehose_sdx24.mbn", @@ -904,6 +908,8 @@ static const struct pci_device_id mhi_pci_id_table[] = { /* Telit FN920C04 (sdx35) */ {PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x011a, 0x1c5d, 0x2020), .driver_data = (kernel_ulong_t) &mhi_telit_fn920c04_info }, + { PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x011a), + .driver_data = (kernel_ulong_t) &mhi_qcom_sdx35_info }, { PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0304), .driver_data = (kernel_ulong_t) &mhi_qcom_sdx24_info }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0306, PCI_VENDOR_ID_QCOM, 0x010c), @@ -933,6 +939,9 @@ static const struct pci_device_id mhi_pci_id_table[] = { /* Telit FN990B40 (sdx72) */ { PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0309, 0x1c5d, 0x201a), .driver_data = (kernel_ulong_t) &mhi_telit_fn990b40_info }, + /* Telit FE990B40 (sdx72) */ + { PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0309, 0x1c5d, 0x2025), + .driver_data = (kernel_ulong_t) &mhi_telit_fe990b40_info }, { PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0309), .driver_data = (kernel_ulong_t) &mhi_qcom_sdx75_info }, /* QDU100, x100-DU */ @@ -1383,7 +1392,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_unregister; } - err = mhi_sync_power_up(mhi_cntrl); + err = mhi_async_power_up(mhi_cntrl); if (err) { dev_err(&pdev->dev, "failed to power up MHI controller\n"); goto err_unprepare; @@ -1437,6 +1446,7 @@ static void mhi_pci_remove(struct pci_dev *pdev) mhi_soc_reset(mhi_cntrl); mhi_unregister_controller(mhi_cntrl); + pm_runtime_forbid(&pdev->dev); } static void mhi_pci_shutdown(struct pci_dev *pdev) diff --git a/drivers/bus/omap-ocp2scp.c b/drivers/bus/omap-ocp2scp.c index e4dfda7b3b102..eee5ad191ea9c 100644 --- a/drivers/bus/omap-ocp2scp.c +++ b/drivers/bus/omap-ocp2scp.c @@ -17,15 +17,6 @@ #define OCP2SCP_TIMING 0x18 #define SYNC2_MASK 0xf -static int ocp2scp_remove_devices(struct device *dev, void *c) -{ - struct platform_device *pdev = to_platform_device(dev); - - platform_device_unregister(pdev); - - return 0; -} - static int omap_ocp2scp_probe(struct platform_device *pdev) { int ret; @@ -79,7 +70,7 @@ static int omap_ocp2scp_probe(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); err0: - device_for_each_child(&pdev->dev, NULL, ocp2scp_remove_devices); + of_platform_depopulate(&pdev->dev); return ret; } @@ -87,7 +78,7 @@ static int omap_ocp2scp_probe(struct platform_device *pdev) static void omap_ocp2scp_remove(struct platform_device *pdev) { pm_runtime_disable(&pdev->dev); - device_for_each_child(&pdev->dev, NULL, ocp2scp_remove_devices); + of_platform_depopulate(&pdev->dev); } #ifdef CONFIG_OF diff --git a/drivers/bus/simple-pm-bus.c b/drivers/bus/simple-pm-bus.c index d8e029e7e53f7..50f8b4e2ba5bc 100644 --- a/drivers/bus/simple-pm-bus.c +++ b/drivers/bus/simple-pm-bus.c @@ -36,7 +36,7 @@ static int simple_pm_bus_probe(struct platform_device *pdev) * that's not listed in simple_pm_bus_of_match. We don't want to do any * of the simple-pm-bus tasks for these devices, so return early. */ - if (pdev->driver_override) + if (device_has_driver_override(&pdev->dev)) return 0; match = of_match_device(dev->driver->of_match_table, dev); @@ -78,7 +78,7 @@ static void simple_pm_bus_remove(struct platform_device *pdev) { const void *data = of_device_get_match_data(&pdev->dev); - if (pdev->driver_override || data) + if (device_has_driver_override(&pdev->dev) || data) return; dev_dbg(&pdev->dev, "%s\n", __func__); diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index 5566ad11399e7..610354ce7f8f0 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -48,6 +48,7 @@ enum sysc_soc { SOC_UNKNOWN, SOC_2420, SOC_2430, + SOC_AM33, SOC_3430, SOC_AM35, SOC_3630, @@ -2912,6 +2913,7 @@ static void ti_sysc_idle(struct work_struct *work) static const struct soc_device_attribute sysc_soc_match[] = { SOC_FLAG("OMAP242*", SOC_2420), SOC_FLAG("OMAP243*", SOC_2430), + SOC_FLAG("AM33*", SOC_AM33), SOC_FLAG("AM35*", SOC_AM35), SOC_FLAG("OMAP3[45]*", SOC_3430), SOC_FLAG("OMAP3[67]*", SOC_3630), @@ -3117,10 +3119,15 @@ static int sysc_check_active_timer(struct sysc *ddata) * can be dropped if we stop supporting old beagleboard revisions * A to B4 at some point. */ - if (sysc_soc->soc == SOC_3430 || sysc_soc->soc == SOC_AM35) + switch (sysc_soc->soc) { + case SOC_AM33: + case SOC_3430: + case SOC_AM35: error = -ENXIO; - else + break; + default: error = -EBUSY; + } if ((ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT) && (ddata->cfg.quirks & SYSC_QUIRK_NO_IDLE)) diff --git a/drivers/cache/ax45mp_cache.c b/drivers/cache/ax45mp_cache.c index 1d7dd3d2c101c..934c5087ec2bd 100644 --- a/drivers/cache/ax45mp_cache.c +++ b/drivers/cache/ax45mp_cache.c @@ -178,11 +178,11 @@ static const struct of_device_id ax45mp_cache_ids[] = { static int __init ax45mp_cache_init(void) { - struct device_node *np; struct resource res; int ret; - np = of_find_matching_node(NULL, ax45mp_cache_ids); + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, ax45mp_cache_ids); if (!of_device_is_available(np)) return -ENODEV; diff --git a/drivers/cache/starfive_starlink_cache.c b/drivers/cache/starfive_starlink_cache.c index 24c7d078ca227..3a25d2d7c70ca 100644 --- a/drivers/cache/starfive_starlink_cache.c +++ b/drivers/cache/starfive_starlink_cache.c @@ -102,11 +102,11 @@ static const struct of_device_id starlink_cache_ids[] = { static int __init starlink_cache_init(void) { - struct device_node *np; u32 block_size; int ret; - np = of_find_matching_node(NULL, starlink_cache_ids); + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, starlink_cache_ids); if (!of_device_is_available(np)) return -ENODEV; diff --git a/drivers/cdx/cdx_msi.c b/drivers/cdx/cdx_msi.c index 91b95422b2634..6924e07c75283 100644 --- a/drivers/cdx/cdx_msi.c +++ b/drivers/cdx/cdx_msi.c @@ -121,23 +121,23 @@ static int cdx_msi_prepare(struct irq_domain *msi_domain, struct device *dev, int nvec, msi_alloc_info_t *info) { + struct of_phandle_args msi_spec = {}; struct cdx_device *cdx_dev = to_cdx_device(dev); struct device *parent = cdx_dev->cdx->dev; struct msi_domain_info *msi_info; - u32 dev_id; int ret; /* Retrieve device ID from requestor ID using parent device */ - ret = of_map_id(parent->of_node, cdx_dev->msi_dev_id, "msi-map", "msi-map-mask", - NULL, &dev_id); + ret = of_map_msi_id(parent->of_node, cdx_dev->msi_dev_id, NULL, &msi_spec); if (ret) { dev_err(dev, "of_map_id failed for MSI: %d\n", ret); return ret; } + of_node_put(msi_spec.np); #ifdef GENERIC_MSI_DOMAIN_OPS /* Set the device Id to be passed to the GIC-ITS */ - info->scratchpad[0].ul = dev_id; + info->scratchpad[0].ul = msi_spec.args[0]; #endif msi_info = msi_get_domain_info(msi_domain->parent); diff --git a/drivers/char/applicom.c b/drivers/char/applicom.c index 9fed9706d9cd2..c138c468f3a44 100644 --- a/drivers/char/applicom.c +++ b/drivers/char/applicom.c @@ -835,7 +835,10 @@ static long ac_ioctl(struct file *file, unsigned int cmd, unsigned long arg) ret = -ENOTTY; break; } - Dummy = readb(apbs[IndexCard].RamIO + VERS); + + if (cmd != 6) + Dummy = readb(apbs[IndexCard].RamIO + VERS); + kfree(adgl); mutex_unlock(&ac_mutex); return ret; diff --git a/drivers/char/hw_random/airoha-trng.c b/drivers/char/hw_random/airoha-trng.c index 1dbfa9505c214..9a648f6d9fd40 100644 --- a/drivers/char/hw_random/airoha-trng.c +++ b/drivers/char/hw_random/airoha-trng.c @@ -212,6 +212,7 @@ static int airoha_trng_probe(struct platform_device *pdev) trng->rng.init = airoha_trng_init; trng->rng.cleanup = airoha_trng_cleanup; trng->rng.read = airoha_trng_read; + trng->rng.quality = 900; ret = devm_hwrng_register(dev, &trng->rng); if (ret) { diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 018316f546215..036de7294bbda 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -20,23 +20,25 @@ #include #include #include +#include #include #include #include #include #include +#include #define RNG_MODULE_NAME "hw_random" #define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES) -static struct hwrng *current_rng; +static struct hwrng __rcu *current_rng; /* the current rng has been explicitly chosen by user via sysfs */ static int cur_rng_set_by_user; static struct task_struct *hwrng_fill; /* list of registered rngs */ static LIST_HEAD(rng_list); -/* Protects rng_list and current_rng */ +/* Protects rng_list, hwrng_fill and updating on current_rng */ static DEFINE_MUTEX(rng_mutex); /* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */ static DEFINE_MUTEX(reading_mutex); @@ -64,18 +66,39 @@ static size_t rng_buffer_size(void) return RNG_BUFFER_SIZE; } -static inline void cleanup_rng(struct kref *kref) +static void cleanup_rng_work(struct work_struct *work) { - struct hwrng *rng = container_of(kref, struct hwrng, ref); + struct hwrng *rng = container_of(work, struct hwrng, cleanup_work); + + /* + * Hold rng_mutex here so we serialize in case they set_current_rng + * on rng again immediately. + */ + mutex_lock(&rng_mutex); + + /* Skip if rng has been reinitialized. */ + if (kref_read(&rng->ref)) { + mutex_unlock(&rng_mutex); + return; + } if (rng->cleanup) rng->cleanup(rng); complete(&rng->cleanup_done); + mutex_unlock(&rng_mutex); +} + +static inline void cleanup_rng(struct kref *kref) +{ + struct hwrng *rng = container_of(kref, struct hwrng, ref); + + schedule_work(&rng->cleanup_work); } static int set_current_rng(struct hwrng *rng) { + struct hwrng *old_rng; int err; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -84,8 +107,14 @@ static int set_current_rng(struct hwrng *rng) if (err) return err; - drop_current_rng(); - current_rng = rng; + old_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + rcu_assign_pointer(current_rng, rng); + + if (old_rng) { + synchronize_rcu(); + kref_put(&old_rng->ref, cleanup_rng); + } /* if necessary, start hwrng thread */ if (!hwrng_fill) { @@ -101,47 +130,56 @@ static int set_current_rng(struct hwrng *rng) static void drop_current_rng(void) { - BUG_ON(!mutex_is_locked(&rng_mutex)); - if (!current_rng) + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!rng) return; + RCU_INIT_POINTER(current_rng, NULL); + synchronize_rcu(); + + if (hwrng_fill) { + kthread_stop(hwrng_fill); + hwrng_fill = NULL; + } + /* decrease last reference for triggering the cleanup */ - kref_put(¤t_rng->ref, cleanup_rng); - current_rng = NULL; + kref_put(&rng->ref, cleanup_rng); } -/* Returns ERR_PTR(), NULL or refcounted hwrng */ +/* Returns NULL or refcounted hwrng */ static struct hwrng *get_current_rng_nolock(void) { - if (current_rng) - kref_get(¤t_rng->ref); + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (rng) + kref_get(&rng->ref); - return current_rng; + return rng; } static struct hwrng *get_current_rng(void) { struct hwrng *rng; - if (mutex_lock_interruptible(&rng_mutex)) - return ERR_PTR(-ERESTARTSYS); + rcu_read_lock(); + rng = rcu_dereference(current_rng); + if (rng) + kref_get(&rng->ref); - rng = get_current_rng_nolock(); + rcu_read_unlock(); - mutex_unlock(&rng_mutex); return rng; } static void put_rng(struct hwrng *rng) { - /* - * Hold rng_mutex here so we serialize in case they set_current_rng - * on rng again immediately. - */ - mutex_lock(&rng_mutex); if (rng) kref_put(&rng->ref, cleanup_rng); - mutex_unlock(&rng_mutex); } static int hwrng_init(struct hwrng *rng) @@ -213,10 +251,6 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, while (size) { rng = get_current_rng(); - if (IS_ERR(rng)) { - err = PTR_ERR(rng); - goto out; - } if (!rng) { err = -ENODEV; goto out; @@ -303,7 +337,7 @@ static struct miscdevice rng_miscdev = { static int enable_best_rng(void) { - struct hwrng *rng, *new_rng = NULL; + struct hwrng *rng, *cur_rng, *new_rng = NULL; int ret = -ENODEV; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -321,7 +355,9 @@ static int enable_best_rng(void) new_rng = rng; } - ret = ((new_rng == current_rng) ? 0 : set_current_rng(new_rng)); + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + ret = ((new_rng == cur_rng) ? 0 : set_current_rng(new_rng)); if (!ret) cur_rng_set_by_user = 0; @@ -341,6 +377,9 @@ static ssize_t rng_current_store(struct device *dev, if (sysfs_streq(buf, "")) { err = enable_best_rng(); + } else if (sysfs_streq(buf, "none")) { + cur_rng_set_by_user = 1; + drop_current_rng(); } else { list_for_each_entry(rng, &rng_list, list) { if (sysfs_streq(rng->name, buf)) { @@ -368,8 +407,6 @@ static ssize_t rng_current_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); ret = sysfs_emit(buf, "%s\n", rng ? rng->name : "none"); put_rng(rng); @@ -392,7 +429,7 @@ static ssize_t rng_available_show(struct device *dev, strlcat(buf, rng->name, PAGE_SIZE); strlcat(buf, " ", PAGE_SIZE); } - strlcat(buf, "\n", PAGE_SIZE); + strlcat(buf, "none\n", PAGE_SIZE); mutex_unlock(&rng_mutex); return strlen(buf); @@ -413,8 +450,6 @@ static ssize_t rng_quality_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); if (!rng) /* no need to put_rng */ return -ENODEV; @@ -429,6 +464,7 @@ static ssize_t rng_quality_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { + struct hwrng *rng; u16 quality; int ret = -EINVAL; @@ -445,12 +481,13 @@ static ssize_t rng_quality_store(struct device *dev, goto out; } - if (!current_rng) { + rng = rcu_dereference_protected(current_rng, lockdep_is_held(&rng_mutex)); + if (!rng) { ret = -ENODEV; goto out; } - current_rng->quality = quality; + rng->quality = quality; current_quality = quality; /* obsolete */ /* the best available RNG may have changed */ @@ -486,8 +523,20 @@ static int hwrng_fillfn(void *unused) struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng) || !rng) + if (!rng) { + /* + * Keep the task_struct alive until kthread_stop() + * is called to avoid UAF in drop_current_rng(). + */ + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + } + set_current_state(TASK_RUNNING); break; + } + mutex_lock(&reading_mutex); rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), 1); @@ -515,14 +564,13 @@ static int hwrng_fillfn(void *unused) add_hwgenerator_randomness((void *)rng_fillbuf, rc, entropy >> 10, true); } - hwrng_fill = NULL; return 0; } int hwrng_register(struct hwrng *rng) { int err = -EINVAL; - struct hwrng *tmp; + struct hwrng *cur_rng, *tmp; if (!rng->name || (!rng->data_read && !rng->read)) goto out; @@ -537,6 +585,7 @@ int hwrng_register(struct hwrng *rng) } list_add_tail(&rng->list, &rng_list); + INIT_WORK(&rng->cleanup_work, cleanup_rng_work); init_completion(&rng->cleanup_done); complete(&rng->cleanup_done); init_completion(&rng->dying); @@ -544,16 +593,19 @@ int hwrng_register(struct hwrng *rng) /* Adjust quality field to always have a proper value */ rng->quality = min_t(u16, min_t(u16, default_quality, 1024), rng->quality ?: 1024); - if (!current_rng || - (!cur_rng_set_by_user && rng->quality > current_rng->quality)) { - /* - * Set new rng as current as the new rng source - * provides better entropy quality and was not - * chosen by userspace. - */ - err = set_current_rng(rng); - if (err) - goto out_unlock; + if (!cur_rng_set_by_user) { + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!cur_rng || rng->quality > cur_rng->quality) { + /* + * Set new rng as current as the new rng source + * provides better entropy quality and was not + * chosen by userspace. + */ + err = set_current_rng(rng); + if (err) + goto out_unlock; + } } mutex_unlock(&rng_mutex); return 0; @@ -566,14 +618,17 @@ EXPORT_SYMBOL_GPL(hwrng_register); void hwrng_unregister(struct hwrng *rng) { - struct hwrng *new_rng; + struct hwrng *cur_rng; int err; mutex_lock(&rng_mutex); list_del(&rng->list); complete_all(&rng->dying); - if (current_rng == rng) { + + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (cur_rng == rng) { err = enable_best_rng(); if (err) { drop_current_rng(); @@ -581,17 +636,7 @@ void hwrng_unregister(struct hwrng *rng) } } - new_rng = get_current_rng_nolock(); - if (list_empty(&rng_list)) { - mutex_unlock(&rng_mutex); - if (hwrng_fill) - kthread_stop(hwrng_fill); - } else - mutex_unlock(&rng_mutex); - - if (new_rng) - put_rng(new_rng); - + mutex_unlock(&rng_mutex); wait_for_completion(&rng->cleanup_done); } EXPORT_SYMBOL_GPL(hwrng_unregister); @@ -679,7 +724,7 @@ static int __init hwrng_modinit(void) static void __exit hwrng_modexit(void) { mutex_lock(&rng_mutex); - BUG_ON(current_rng); + WARN_ON(rcu_access_pointer(current_rng)); kfree(rng_buffer); kfree(rng_fillbuf); mutex_unlock(&rng_mutex); diff --git a/drivers/char/ipmi/ipmi_ipmb.c b/drivers/char/ipmi/ipmi_ipmb.c index 3a51e58b24875..28818952a7a4b 100644 --- a/drivers/char/ipmi/ipmi_ipmb.c +++ b/drivers/char/ipmi/ipmi_ipmb.c @@ -202,11 +202,16 @@ static int ipmi_ipmb_slave_cb(struct i2c_client *client, break; case I2C_SLAVE_READ_REQUESTED: + *val = 0xff; + ipmi_ipmb_check_msg_done(iidev); + break; + case I2C_SLAVE_STOP: ipmi_ipmb_check_msg_done(iidev); break; case I2C_SLAVE_READ_PROCESSED: + *val = 0xff; break; } diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 3700ab4eba3e7..17f7b3b551d5c 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -599,7 +599,24 @@ static void __ipmi_bmc_unregister(struct ipmi_smi *intf); static int __ipmi_bmc_register(struct ipmi_smi *intf, struct ipmi_device_id *id, bool guid_set, guid_t *guid, int intf_num); -static int __scan_channels(struct ipmi_smi *intf, struct ipmi_device_id *id); +static int __scan_channels(struct ipmi_smi *intf, + struct ipmi_device_id *id, bool rescan); + +static void ipmi_lock_xmit_msgs(struct ipmi_smi *intf, int run_to_completion, + unsigned long *flags) +{ + if (run_to_completion) + return; + spin_lock_irqsave(&intf->xmit_msgs_lock, *flags); +} + +static void ipmi_unlock_xmit_msgs(struct ipmi_smi *intf, int run_to_completion, + unsigned long *flags) +{ + if (run_to_completion) + return; + spin_unlock_irqrestore(&intf->xmit_msgs_lock, *flags); +} static void free_ipmi_user(struct kref *ref) { @@ -1870,21 +1887,32 @@ static struct ipmi_smi_msg *smi_add_send_msg(struct ipmi_smi *intf, return smi_msg; } -static void smi_send(struct ipmi_smi *intf, +static int smi_send(struct ipmi_smi *intf, const struct ipmi_smi_handlers *handlers, struct ipmi_smi_msg *smi_msg, int priority) { int run_to_completion = READ_ONCE(intf->run_to_completion); unsigned long flags = 0; + int rv = 0; - if (!run_to_completion) - spin_lock_irqsave(&intf->xmit_msgs_lock, flags); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); smi_msg = smi_add_send_msg(intf, smi_msg, priority); - if (!run_to_completion) - spin_unlock_irqrestore(&intf->xmit_msgs_lock, flags); + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); - if (smi_msg) - handlers->sender(intf->send_info, smi_msg); + if (smi_msg) { + rv = handlers->sender(intf->send_info, smi_msg); + if (rv) { + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); + intf->curr_msg = NULL; + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); + /* + * Something may have been added to the transmit + * queue, so schedule a check for that. + */ + queue_work(system_wq, &intf->smi_work); + } + } + return rv; } static bool is_maintenance_mode_cmd(struct kernel_ipmi_msg *msg) @@ -2297,6 +2325,7 @@ static int i_ipmi_request(struct ipmi_user *user, struct ipmi_recv_msg *recv_msg; int run_to_completion = READ_ONCE(intf->run_to_completion); int rv = 0; + bool in_seq_table = false; if (supplied_recv) { recv_msg = supplied_recv; @@ -2350,33 +2379,50 @@ static int i_ipmi_request(struct ipmi_user *user, rv = i_ipmi_req_ipmb(intf, addr, msgid, msg, smi_msg, recv_msg, source_address, source_lun, retries, retry_time_ms); + in_seq_table = true; } else if (is_ipmb_direct_addr(addr)) { rv = i_ipmi_req_ipmb_direct(intf, addr, msgid, msg, smi_msg, recv_msg, source_lun); } else if (is_lan_addr(addr)) { rv = i_ipmi_req_lan(intf, addr, msgid, msg, smi_msg, recv_msg, source_lun, retries, retry_time_ms); + in_seq_table = true; } else { - /* Unknown address type. */ + /* Unknown address type. */ ipmi_inc_stat(intf, sent_invalid_commands); rv = -EINVAL; } - if (rv) { -out_err: - if (!supplied_smi) - ipmi_free_smi_msg(smi_msg); - if (!supplied_recv) - ipmi_free_recv_msg(recv_msg); - } else { + if (!rv) { dev_dbg(intf->si_dev, "Send: %*ph\n", smi_msg->data_size, smi_msg->data); - smi_send(intf, intf->handlers, smi_msg, priority); + rv = smi_send(intf, intf->handlers, smi_msg, priority); + if (rv != IPMI_CC_NO_ERROR) + /* smi_send() returns an IPMI err, return a Linux one. */ + rv = -EIO; + if (rv && in_seq_table) { + /* + * If it's in the sequence table, it will be + * retried later, so ignore errors. + */ + rv = 0; + /* But we need to fix the timeout. */ + intf_start_seq_timer(intf, smi_msg->msgid); + ipmi_free_smi_msg(smi_msg); + smi_msg = NULL; + } } +out_err: if (!run_to_completion) mutex_unlock(&intf->users_mutex); + if (rv) { + if (!supplied_smi) + ipmi_free_smi_msg(smi_msg); + if (!supplied_recv) + ipmi_free_recv_msg(recv_msg); + } return rv; } @@ -2668,7 +2714,7 @@ static int __bmc_get_device_id(struct ipmi_smi *intf, struct bmc_device *bmc, if (__ipmi_bmc_register(intf, &id, guid_set, &guid, intf_num)) need_waiter(intf); /* Retry later on an error. */ else - __scan_channels(intf, &id); + __scan_channels(intf, &id, false); if (!intf_set) { @@ -2688,7 +2734,7 @@ static int __bmc_get_device_id(struct ipmi_smi *intf, struct bmc_device *bmc, goto out_noprocessing; } else if (memcmp(&bmc->fetch_id, &bmc->id, sizeof(bmc->id))) /* Version info changes, scan the channels again. */ - __scan_channels(intf, &bmc->fetch_id); + __scan_channels(intf, &bmc->fetch_id, true); bmc->dyn_id_expiry = jiffies + IPMI_DYN_DEV_ID_EXPIRY; @@ -3417,8 +3463,6 @@ channel_handler(struct ipmi_smi *intf, struct ipmi_recv_msg *msg) intf->channels_ready = true; wake_up(&intf->waitq); } else { - intf->channel_list = intf->wchannels + set; - intf->channels_ready = true; rv = send_channel_info_cmd(intf, intf->curr_channel); } @@ -3440,10 +3484,17 @@ channel_handler(struct ipmi_smi *intf, struct ipmi_recv_msg *msg) /* * Must be holding intf->bmc_reg_mutex to call this. */ -static int __scan_channels(struct ipmi_smi *intf, struct ipmi_device_id *id) +static int __scan_channels(struct ipmi_smi *intf, + struct ipmi_device_id *id, + bool rescan) { int rv; + if (rescan) { + /* Clear channels_ready to force channels rescan. */ + intf->channels_ready = false; + } + if (ipmi_version_major(id) > 1 || (ipmi_version_major(id) == 1 && ipmi_version_minor(id) >= 5)) { @@ -3658,7 +3709,7 @@ int ipmi_add_smi(struct module *owner, } mutex_lock(&intf->bmc_reg_mutex); - rv = __scan_channels(intf, &id); + rv = __scan_channels(intf, &id, false); mutex_unlock(&intf->bmc_reg_mutex); if (rv) goto out_err_bmc_reg; @@ -3941,12 +3992,12 @@ static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf, dev_dbg(intf->si_dev, "Invalid command: %*ph\n", msg->data_size, msg->data); - smi_send(intf, intf->handlers, msg, 0); - /* - * We used the message, so return the value that - * causes it to not be freed or queued. - */ - rv = -1; + if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR) + /* + * We used the message, so return the value that + * causes it to not be freed or queued. + */ + rv = -1; } else if (!IS_ERR(recv_msg)) { /* Extract the source address from the data. */ ipmb_addr = (struct ipmi_ipmb_addr *) &recv_msg->addr; @@ -4020,12 +4071,12 @@ static int handle_ipmb_direct_rcv_cmd(struct ipmi_smi *intf, msg->data[4] = IPMI_INVALID_CMD_COMPLETION_CODE; msg->data_size = 5; - smi_send(intf, intf->handlers, msg, 0); - /* - * We used the message, so return the value that - * causes it to not be freed or queued. - */ - rv = -1; + if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR) + /* + * We used the message, so return the value that + * causes it to not be freed or queued. + */ + rv = -1; } else if (!IS_ERR(recv_msg)) { /* Extract the source address from the data. */ daddr = (struct ipmi_ipmb_direct_addr *)&recv_msg->addr; @@ -4165,7 +4216,7 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf, struct ipmi_smi_msg *msg) { struct cmd_rcvr *rcvr; - int rv = 0; + int rv = 0; /* Free by default */ unsigned char netfn; unsigned char cmd; unsigned char chan; @@ -4218,12 +4269,12 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf, dev_dbg(intf->si_dev, "Invalid command: %*ph\n", msg->data_size, msg->data); - smi_send(intf, intf->handlers, msg, 0); - /* - * We used the message, so return the value that - * causes it to not be freed or queued. - */ - rv = -1; + if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR) + /* + * We used the message, so return the value that + * causes it to not be freed or queued. + */ + rv = -1; } else if (!IS_ERR(recv_msg)) { /* Extract the source address from the data. */ lan_addr = (struct ipmi_lan_addr *) &recv_msg->addr; @@ -4816,8 +4867,7 @@ static void smi_work(struct work_struct *t) * message delivery. */ restart: - if (!run_to_completion) - spin_lock_irqsave(&intf->xmit_msgs_lock, flags); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); if (intf->curr_msg == NULL && !intf->in_shutdown) { struct list_head *entry = NULL; @@ -4833,8 +4883,7 @@ static void smi_work(struct work_struct *t) intf->curr_msg = newmsg; } } - if (!run_to_completion) - spin_unlock_irqrestore(&intf->xmit_msgs_lock, flags); + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); if (newmsg) { cc = intf->handlers->sender(intf->send_info, newmsg); @@ -4842,8 +4891,11 @@ static void smi_work(struct work_struct *t) if (newmsg->recv_msg) deliver_err_response(intf, newmsg->recv_msg, cc); - else - ipmi_free_smi_msg(newmsg); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); + intf->curr_msg = NULL; + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); + ipmi_free_smi_msg(newmsg); + newmsg = NULL; goto restart; } } @@ -4911,16 +4963,14 @@ void ipmi_smi_msg_received(struct ipmi_smi *intf, spin_unlock_irqrestore(&intf->waiting_rcv_msgs_lock, flags); - if (!run_to_completion) - spin_lock_irqsave(&intf->xmit_msgs_lock, flags); + ipmi_lock_xmit_msgs(intf, run_to_completion, &flags); /* * We can get an asynchronous event or receive message in addition * to commands we send. */ if (msg == intf->curr_msg) intf->curr_msg = NULL; - if (!run_to_completion) - spin_unlock_irqrestore(&intf->xmit_msgs_lock, flags); + ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags); if (run_to_completion) smi_work(&intf->smi_work); @@ -5033,7 +5083,12 @@ static void check_msg_timeout(struct ipmi_smi *intf, struct seq_table *ent, ipmi_inc_stat(intf, retransmitted_ipmb_commands); - smi_send(intf, intf->handlers, smi_msg, 0); + /* If this fails we'll retry later or timeout. */ + if (smi_send(intf, intf->handlers, smi_msg, 0) != IPMI_CC_NO_ERROR) { + /* But fix the timeout. */ + intf_start_seq_timer(intf, smi_msg->msgid); + ipmi_free_smi_msg(smi_msg); + } } else ipmi_free_smi_msg(smi_msg); diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 70e55f5ff85e7..f3a4fa98b1efd 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -810,6 +810,12 @@ static enum si_sm_result smi_event_handler(struct smi_info *smi_info, */ return_hosed_msg(smi_info, IPMI_BUS_ERR); } + if (smi_info->waiting_msg != NULL) { + /* Also handle if there was a message waiting. */ + smi_info->curr_msg = smi_info->waiting_msg; + smi_info->waiting_msg = NULL; + return_hosed_msg(smi_info, IPMI_BUS_ERR); + } smi_mod_timer(smi_info, jiffies + SI_TIMEOUT_HOSED); goto out; } @@ -919,9 +925,14 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg) { struct smi_info *smi_info = send_info; unsigned long flags; + int rv = IPMI_CC_NO_ERROR; debug_timestamp(smi_info, "Enqueue"); + /* + * Check here for run to completion mode. A check under lock is + * later. + */ if (smi_info->si_state == SI_HOSED) return IPMI_BUS_ERR; @@ -935,18 +946,15 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg) } spin_lock_irqsave(&smi_info->si_lock, flags); - /* - * The following two lines don't need to be under the lock for - * the lock's sake, but they do need SMP memory barriers to - * avoid getting things out of order. We are already claiming - * the lock, anyway, so just do it under the lock to avoid the - * ordering problem. - */ - BUG_ON(smi_info->waiting_msg); - smi_info->waiting_msg = msg; - check_start_timer_thread(smi_info); + if (smi_info->si_state == SI_HOSED) { + rv = IPMI_BUS_ERR; + } else { + BUG_ON(smi_info->waiting_msg); + smi_info->waiting_msg = msg; + check_start_timer_thread(smi_info); + } spin_unlock_irqrestore(&smi_info->si_lock, flags); - return IPMI_CC_NO_ERROR; + return rv; } static void set_run_to_completion(void *send_info, bool i_run_to_completion) @@ -1114,7 +1122,9 @@ static void smi_timeout(struct timer_list *t) * SI_USEC_PER_JIFFY); smi_result = smi_event_handler(smi_info, time_diff); - if ((smi_info->io.irq) && (!smi_info->interrupt_disabled)) { + if (smi_info->si_state == SI_HOSED) { + timeout = jiffies + SI_TIMEOUT_HOSED; + } else if ((smi_info->io.irq) && (!smi_info->interrupt_disabled)) { /* Running with interrupts, only do long timeouts. */ timeout = jiffies + SI_TIMEOUT_JIFFIES; smi_inc_stat(smi_info, long_timeouts); @@ -2227,7 +2237,8 @@ static void wait_msg_processed(struct smi_info *smi_info) unsigned long jiffies_now; long time_diff; - while (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL)) { + while (smi_info->si_state != SI_HOSED && + (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL))) { jiffies_now = jiffies; time_diff = (((long)jiffies_now - (long)smi_info->last_timeout_jiffies) * SI_USEC_PER_JIFFY); diff --git a/drivers/char/misc_minor_kunit.c b/drivers/char/misc_minor_kunit.c index 6fc8b05169c57..e930c78e1ef97 100644 --- a/drivers/char/misc_minor_kunit.c +++ b/drivers/char/misc_minor_kunit.c @@ -166,7 +166,7 @@ static void __init miscdev_test_can_open(struct kunit *test, struct miscdevice * KUNIT_FAIL(test, "failed to create node\n"); filp = filp_open(devname, O_RDONLY, 0); - if (IS_ERR_OR_NULL(filp)) + if (IS_ERR(filp)) KUNIT_FAIL(test, "failed to open misc device: %ld\n", PTR_ERR(filp)); else fput(filp); diff --git a/drivers/char/random.c b/drivers/char/random.c index b8b24b6ed3fe4..f344f67c83f4e 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -96,8 +96,7 @@ static ATOMIC_NOTIFIER_HEAD(random_ready_notifier); /* Control how we warn userspace. */ static struct ratelimit_state urandom_warning = RATELIMIT_STATE_INIT_FLAGS("urandom_warning", HZ, 3, RATELIMIT_MSG_ON_RELEASE); -static int ratelimit_disable __read_mostly = - IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM); +static int ratelimit_disable __read_mostly = 0; module_param_named(ratelimit_disable, ratelimit_disable, int, 0644); MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression"); @@ -168,12 +167,6 @@ int __cold execute_with_initialized_rng(struct notifier_block *nb) return ret; } -#define warn_unseeded_randomness() \ - if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \ - printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", \ - __func__, (void *)_RET_IP_, crng_init) - - /********************************************************************* * * Fast key erasure RNG, the "crng". @@ -434,7 +427,6 @@ static void _get_random_bytes(void *buf, size_t len) */ void get_random_bytes(void *buf, size_t len) { - warn_unseeded_randomness(); _get_random_bytes(buf, len); } EXPORT_SYMBOL(get_random_bytes); @@ -523,8 +515,6 @@ type get_random_ ##type(void) \ struct batch_ ##type *batch; \ unsigned long next_gen; \ \ - warn_unseeded_randomness(); \ - \ if (!crng_ready()) { \ _get_random_bytes(&ret, sizeof(ret)); \ return ret; \ @@ -1296,6 +1286,7 @@ static void __cold try_to_generate_entropy(void) struct entropy_timer_state *stack = PTR_ALIGN((void *)stack_bytes, SMP_CACHE_BYTES); unsigned int i, num_different = 0; unsigned long last = random_get_entropy(); + cpumask_var_t timer_cpus; int cpu = -1; for (i = 0; i < NUM_TRIAL_SAMPLES - 1; ++i) { @@ -1310,13 +1301,15 @@ static void __cold try_to_generate_entropy(void) atomic_set(&stack->samples, 0); timer_setup_on_stack(&stack->timer, entropy_timer, 0); + if (!alloc_cpumask_var(&timer_cpus, GFP_KERNEL)) + goto out; + while (!crng_ready() && !signal_pending(current)) { /* * Check !timer_pending() and then ensure that any previous callback has finished * executing by checking timer_delete_sync_try(), before queueing the next one. */ if (!timer_pending(&stack->timer) && timer_delete_sync_try(&stack->timer) >= 0) { - struct cpumask timer_cpus; unsigned int num_cpus; /* @@ -1326,19 +1319,19 @@ static void __cold try_to_generate_entropy(void) preempt_disable(); /* Only schedule callbacks on timer CPUs that are online. */ - cpumask_and(&timer_cpus, housekeeping_cpumask(HK_TYPE_TIMER), cpu_online_mask); - num_cpus = cpumask_weight(&timer_cpus); + cpumask_and(timer_cpus, housekeeping_cpumask(HK_TYPE_TIMER), cpu_online_mask); + num_cpus = cpumask_weight(timer_cpus); /* In very bizarre case of misconfiguration, fallback to all online. */ if (unlikely(num_cpus == 0)) { - timer_cpus = *cpu_online_mask; - num_cpus = cpumask_weight(&timer_cpus); + *timer_cpus = *cpu_online_mask; + num_cpus = cpumask_weight(timer_cpus); } /* Basic CPU round-robin, which avoids the current CPU. */ do { - cpu = cpumask_next(cpu, &timer_cpus); + cpu = cpumask_next(cpu, timer_cpus); if (cpu >= nr_cpu_ids) - cpu = cpumask_first(&timer_cpus); + cpu = cpumask_first(timer_cpus); } while (cpu == smp_processor_id() && num_cpus > 1); /* Expiring the timer at `jiffies` means it's the next tick. */ @@ -1354,6 +1347,8 @@ static void __cold try_to_generate_entropy(void) } mix_pool_bytes(&stack->entropy, sizeof(stack->entropy)); + free_cpumask_var(timer_cpus); +out: timer_delete_sync(&stack->timer); timer_destroy_on_stack(&stack->timer); } diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index 2ed7815e4899b..e2b7451ea7ccd 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -328,8 +328,10 @@ static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf, for (i = 0; i < len - 1;) { burstcnt = get_burstcount(chip); - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + ret = burstcnt; + goto out_err; + } size = min_t(int, len - i - 1, burstcnt); ret = tpm_dev->ops->send(tpm_dev->phy_id, TPM_DATA_FIFO, buf + i, size); diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index e25daf2396d37..dfeb28866a327 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -282,7 +282,6 @@ static void tpm_dev_release(struct device *dev) kfree(chip->work_space.context_buf); kfree(chip->work_space.session_buf); - kfree(chip->allocated_banks); #ifdef CONFIG_TCG_TPM2_HMAC kfree(chip->auth); #endif diff --git a/drivers/char/tpm/tpm1-cmd.c b/drivers/char/tpm/tpm1-cmd.c index cf64c73851052..b49a790f1bd53 100644 --- a/drivers/char/tpm/tpm1-cmd.c +++ b/drivers/char/tpm/tpm1-cmd.c @@ -799,11 +799,6 @@ int tpm1_pm_suspend(struct tpm_chip *chip, u32 tpm_suspend_pcr) */ int tpm1_get_pcr_allocation(struct tpm_chip *chip) { - chip->allocated_banks = kcalloc(1, sizeof(*chip->allocated_banks), - GFP_KERNEL); - if (!chip->allocated_banks) - return -ENOMEM; - chip->allocated_banks[0].alg_id = TPM_ALG_SHA1; chip->allocated_banks[0].digest_size = hash_digest_size[HASH_ALGO_SHA1]; chip->allocated_banks[0].crypto_id = HASH_ALGO_SHA1; diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index 7d77f6fbc1520..58a8477cda851 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -11,8 +11,11 @@ * used by the kernel internally. */ +#include "linux/dev_printk.h" +#include "linux/tpm.h" #include "tpm.h" #include +#include static bool disable_pcr_integrity; module_param(disable_pcr_integrity, bool, 0444); @@ -187,7 +190,11 @@ int tpm2_pcr_extend(struct tpm_chip *chip, u32 pcr_idx, } if (!disable_pcr_integrity) { - tpm_buf_append_name(chip, &buf, pcr_idx, NULL); + rc = tpm_buf_append_name(chip, &buf, pcr_idx, NULL); + if (rc) { + tpm_buf_destroy(&buf); + return rc; + } tpm_buf_append_hmac_session(chip, &buf, 0, NULL, 0); } else { tpm_buf_append_handle(chip, &buf, pcr_idx); @@ -202,8 +209,14 @@ int tpm2_pcr_extend(struct tpm_chip *chip, u32 pcr_idx, chip->allocated_banks[i].digest_size); } - if (!disable_pcr_integrity) - tpm_buf_fill_hmac_session(chip, &buf); + if (!disable_pcr_integrity) { + rc = tpm_buf_fill_hmac_session(chip, &buf); + if (rc) { + tpm_buf_destroy(&buf); + return rc; + } + } + rc = tpm_transmit_cmd(chip, &buf, 0, "attempting extend a PCR value"); if (!disable_pcr_integrity) rc = tpm_buf_check_hmac_response(chip, &buf, rc); @@ -261,7 +274,12 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max) | TPM2_SA_CONTINUE_SESSION, NULL, 0); tpm_buf_append_u16(&buf, num_bytes); - tpm_buf_fill_hmac_session(chip, &buf); + err = tpm_buf_fill_hmac_session(chip, &buf); + if (err) { + tpm_buf_destroy(&buf); + return err; + } + err = tpm_transmit_cmd(chip, &buf, offsetof(struct tpm2_get_random_out, buffer), @@ -538,11 +556,9 @@ ssize_t tpm2_get_pcr_allocation(struct tpm_chip *chip) nr_possible_banks = be32_to_cpup( (__be32 *)&buf.data[TPM_HEADER_SIZE + 5]); - - chip->allocated_banks = kcalloc(nr_possible_banks, - sizeof(*chip->allocated_banks), - GFP_KERNEL); - if (!chip->allocated_banks) { + if (nr_possible_banks > TPM2_MAX_PCR_BANKS) { + pr_err("tpm: out of bank capacity: %u > %u\n", + nr_possible_banks, TPM2_MAX_PCR_BANKS); rc = -ENOMEM; goto out; } diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c index 6d03c224e6b21..3f389e2f6f580 100644 --- a/drivers/char/tpm/tpm2-sessions.c +++ b/drivers/char/tpm/tpm2-sessions.c @@ -144,59 +144,80 @@ struct tpm2_auth { /* * Name Size based on TPM algorithm (assumes no hash bigger than 255) */ -static u8 name_size(const u8 *name) +static int name_size(const u8 *name) { - static u8 size_map[] = { - [TPM_ALG_SHA1] = SHA1_DIGEST_SIZE, - [TPM_ALG_SHA256] = SHA256_DIGEST_SIZE, - [TPM_ALG_SHA384] = SHA384_DIGEST_SIZE, - [TPM_ALG_SHA512] = SHA512_DIGEST_SIZE, - }; - u16 alg = get_unaligned_be16(name); - return size_map[alg] + 2; -} - -static int tpm2_parse_read_public(char *name, struct tpm_buf *buf) -{ - struct tpm_header *head = (struct tpm_header *)buf->data; - off_t offset = TPM_HEADER_SIZE; - u32 tot_len = be32_to_cpu(head->length); - u32 val; - - /* we're starting after the header so adjust the length */ - tot_len -= TPM_HEADER_SIZE; - - /* skip public */ - val = tpm_buf_read_u16(buf, &offset); - if (val > tot_len) - return -EINVAL; - offset += val; - /* name */ - val = tpm_buf_read_u16(buf, &offset); - if (val != name_size(&buf->data[offset])) + u16 hash_alg = get_unaligned_be16(name); + + switch (hash_alg) { + case TPM_ALG_SHA1: + return SHA1_DIGEST_SIZE + 2; + case TPM_ALG_SHA256: + return SHA256_DIGEST_SIZE + 2; + case TPM_ALG_SHA384: + return SHA384_DIGEST_SIZE + 2; + case TPM_ALG_SHA512: + return SHA512_DIGEST_SIZE + 2; + default: + pr_warn("tpm: unsupported name algorithm: 0x%04x\n", hash_alg); return -EINVAL; - memcpy(name, &buf->data[offset], val); - /* forget the rest */ - return 0; + } } -static int tpm2_read_public(struct tpm_chip *chip, u32 handle, char *name) +static int tpm2_read_public(struct tpm_chip *chip, u32 handle, void *name) { + u32 mso = tpm2_handle_mso(handle); + off_t offset = TPM_HEADER_SIZE; + int rc, name_size_alg; struct tpm_buf buf; - int rc; + + if (mso != TPM2_MSO_PERSISTENT && mso != TPM2_MSO_VOLATILE && + mso != TPM2_MSO_NVRAM) { + memcpy(name, &handle, sizeof(u32)); + return sizeof(u32); + } rc = tpm_buf_init(&buf, TPM2_ST_NO_SESSIONS, TPM2_CC_READ_PUBLIC); if (rc) return rc; tpm_buf_append_u32(&buf, handle); - rc = tpm_transmit_cmd(chip, &buf, 0, "read public"); - if (rc == TPM2_RC_SUCCESS) - rc = tpm2_parse_read_public(name, &buf); - tpm_buf_destroy(&buf); + rc = tpm_transmit_cmd(chip, &buf, 0, "TPM2_ReadPublic"); + if (rc) { + tpm_buf_destroy(&buf); + return tpm_ret_to_err(rc); + } - return rc; + /* Skip TPMT_PUBLIC: */ + offset += tpm_buf_read_u16(&buf, &offset); + + /* + * Ensure space for the length field of TPM2B_NAME and hashAlg field of + * TPMT_HA (the extra four bytes). + */ + if (offset + 4 > tpm_buf_length(&buf)) { + tpm_buf_destroy(&buf); + return -EIO; + } + + rc = tpm_buf_read_u16(&buf, &offset); + name_size_alg = name_size(&buf.data[offset]); + + if (name_size_alg < 0) + return name_size_alg; + + if (rc != name_size_alg) { + tpm_buf_destroy(&buf); + return -EIO; + } + + if (offset + rc > tpm_buf_length(&buf)) { + tpm_buf_destroy(&buf); + return -EIO; + } + + memcpy(name, &buf.data[offset], rc); + return name_size_alg; } #endif /* CONFIG_TCG_TPM2_HMAC */ @@ -221,46 +242,70 @@ static int tpm2_read_public(struct tpm_chip *chip, u32 handle, char *name) * As with most tpm_buf operations, success is assumed because failure * will be caused by an incorrect programming model and indicated by a * kernel message. + * + * Ends the authorization session on failure. */ -void tpm_buf_append_name(struct tpm_chip *chip, struct tpm_buf *buf, - u32 handle, u8 *name) +int tpm_buf_append_name(struct tpm_chip *chip, struct tpm_buf *buf, + u32 handle, u8 *name) { #ifdef CONFIG_TCG_TPM2_HMAC enum tpm2_mso_type mso = tpm2_handle_mso(handle); struct tpm2_auth *auth; + u16 name_size_alg; int slot; + int ret; #endif if (!tpm2_chip_auth(chip)) { tpm_buf_append_handle(chip, buf, handle); - return; + return 0; } #ifdef CONFIG_TCG_TPM2_HMAC slot = (tpm_buf_length(buf) - TPM_HEADER_SIZE) / 4; if (slot >= AUTH_MAX_NAMES) { - dev_err(&chip->dev, "TPM: too many handles\n"); - return; + dev_err(&chip->dev, "too many handles\n"); + ret = -EIO; + goto err; } auth = chip->auth; - WARN(auth->session != tpm_buf_length(buf), - "name added in wrong place\n"); + if (auth->session != tpm_buf_length(buf)) { + dev_err(&chip->dev, "session state malformed"); + ret = -EIO; + goto err; + } tpm_buf_append_u32(buf, handle); auth->session += 4; if (mso == TPM2_MSO_PERSISTENT || mso == TPM2_MSO_VOLATILE || mso == TPM2_MSO_NVRAM) { - if (!name) - tpm2_read_public(chip, handle, auth->name[slot]); + if (!name) { + ret = tpm2_read_public(chip, handle, auth->name[slot]); + if (ret < 0) + goto err; + + name_size_alg = ret; + } } else { - if (name) - dev_err(&chip->dev, "TPM: Handle does not require name but one is specified\n"); + if (name) { + dev_err(&chip->dev, "handle 0x%08x does not use a name\n", + handle); + ret = -EIO; + goto err; + } } auth->name_h[slot] = handle; if (name) - memcpy(auth->name[slot], name, name_size(name)); + memcpy(auth->name[slot], name, name_size_alg); +#endif + return 0; + +#ifdef CONFIG_TCG_TPM2_HMAC +err: + tpm2_end_auth_session(chip); + return tpm_ret_to_err(ret); #endif } EXPORT_SYMBOL_GPL(tpm_buf_append_name); @@ -533,11 +578,9 @@ static void tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip, * encryption key and encrypts the first parameter of the command * buffer with it. * - * As with most tpm_buf operations, success is assumed because failure - * will be caused by an incorrect programming model and indicated by a - * kernel message. + * Ends the authorization session on failure. */ -void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) +int tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) { u32 cc, handles, val; struct tpm2_auth *auth = chip->auth; @@ -549,9 +592,12 @@ void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) u8 cphash[SHA256_DIGEST_SIZE]; struct sha256_ctx sctx; struct hmac_sha256_ctx hctx; + int ret; - if (!auth) - return; + if (!auth) { + ret = -EIO; + goto err; + } /* save the command code in BE format */ auth->ordinal = head->ordinal; @@ -560,9 +606,11 @@ void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) i = tpm2_find_cc(chip, cc); if (i < 0) { - dev_err(&chip->dev, "Command 0x%x not found in TPM\n", cc); - return; + dev_err(&chip->dev, "command 0x%08x not found\n", cc); + ret = -EIO; + goto err; } + attrs = chip->cc_attrs_tbl[i]; handles = (attrs >> TPM2_CC_ATTR_CHANDLES) & GENMASK(2, 0); @@ -576,9 +624,9 @@ void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) u32 handle = tpm_buf_read_u32(buf, &offset_s); if (auth->name_h[i] != handle) { - dev_err(&chip->dev, "TPM: handle %d wrong for name\n", - i); - return; + dev_err(&chip->dev, "invalid handle 0x%08x\n", handle); + ret = -EIO; + goto err; } } /* point offset_s to the start of the sessions */ @@ -609,12 +657,14 @@ void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) offset_s += len; } if (offset_s != offset_p) { - dev_err(&chip->dev, "TPM session length is incorrect\n"); - return; + dev_err(&chip->dev, "session length is incorrect\n"); + ret = -EIO; + goto err; } if (!hmac) { - dev_err(&chip->dev, "TPM could not find HMAC session\n"); - return; + dev_err(&chip->dev, "could not find HMAC session\n"); + ret = -EIO; + goto err; } /* encrypt before HMAC */ @@ -646,8 +696,11 @@ void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) if (mso == TPM2_MSO_PERSISTENT || mso == TPM2_MSO_VOLATILE || mso == TPM2_MSO_NVRAM) { - sha256_update(&sctx, auth->name[i], - name_size(auth->name[i])); + ret = name_size(auth->name[i]); + if (ret < 0) + goto err; + + sha256_update(&sctx, auth->name[i], ret); } else { __be32 h = cpu_to_be32(auth->name_h[i]); @@ -668,6 +721,11 @@ void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) hmac_sha256_update(&hctx, auth->tpm_nonce, sizeof(auth->tpm_nonce)); hmac_sha256_update(&hctx, &auth->attrs, 1); hmac_sha256_final(&hctx, hmac); + return 0; + +err: + tpm2_end_auth_session(chip); + return ret; } EXPORT_SYMBOL(tpm_buf_fill_hmac_session); diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c index bdf1f329a6794..8b7d32de0b2ef 100644 --- a/drivers/char/tpm/tpm_i2c_infineon.c +++ b/drivers/char/tpm/tpm_i2c_infineon.c @@ -544,8 +544,10 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t bufsiz, burstcnt = get_burstcount(chip); /* burstcnt < 0 = TPM is busy */ - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + rc = burstcnt; + goto out_err; + } if (burstcnt > (len - 1 - count)) burstcnt = len - 1 - count; diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c index fc6891a0b6936..b48cacacc0664 100644 --- a/drivers/char/tpm/tpm_tis_i2c_cr50.c +++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c @@ -749,8 +749,7 @@ static int tpm_cr50_i2c_probe(struct i2c_client *client) if (client->irq > 0) { rc = devm_request_irq(dev, client->irq, tpm_cr50_i2c_int_handler, - IRQF_TRIGGER_FALLING | IRQF_ONESHOT | - IRQF_NO_AUTOEN, + IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN, dev->driver->name, chip); if (rc < 0) { dev_err(dev, "Failed to probe IRQ %d\n", client->irq); diff --git a/drivers/char/tpm/tpm_tis_spi_cr50.c b/drivers/char/tpm/tpm_tis_spi_cr50.c index f4937280e9406..32920b4cecfb4 100644 --- a/drivers/char/tpm/tpm_tis_spi_cr50.c +++ b/drivers/char/tpm/tpm_tis_spi_cr50.c @@ -287,7 +287,7 @@ int cr50_spi_probe(struct spi_device *spi) if (spi->irq > 0) { ret = devm_request_irq(&spi->dev, spi->irq, cr50_spi_irq_handler, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_TRIGGER_RISING, "cr50_spi", cr50_phy); if (ret < 0) { if (ret == -EPROBE_DEFER) diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index b74a1767ca278..61ec08404442b 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -125,8 +125,7 @@ obj-$(CONFIG_ARCH_HISI) += hisilicon/ obj-y += imgtec/ obj-y += imx/ obj-y += ingenic/ -obj-$(CONFIG_ARCH_K3) += keystone/ -obj-$(CONFIG_ARCH_KEYSTONE) += keystone/ +obj-y += keystone/ obj-y += mediatek/ obj-$(CONFIG_ARCH_MESON) += meson/ obj-y += microchip/ diff --git a/drivers/clk/actions/owl-composite.c b/drivers/clk/actions/owl-composite.c index 00b74f8bc4375..9540444307d6c 100644 --- a/drivers/clk/actions/owl-composite.c +++ b/drivers/clk/actions/owl-composite.c @@ -57,15 +57,10 @@ static int owl_comp_div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_composite *comp = hw_to_owl_comp(hw); - long rate; - - rate = owl_divider_helper_round_rate(&comp->common, &comp->rate.div_hw, - req->rate, &req->best_parent_rate); - if (rate < 0) - return rate; + struct owl_divider_hw *div = &comp->rate.div_hw; - req->rate = rate; - return 0; + return divider_determine_rate(&comp->common.hw, req, div->table, + div->width, div->div_flags); } static unsigned long owl_comp_div_recalc_rate(struct clk_hw *hw, diff --git a/drivers/clk/actions/owl-divider.c b/drivers/clk/actions/owl-divider.c index 118f1393c6780..316ace80e87e3 100644 --- a/drivers/clk/actions/owl-divider.c +++ b/drivers/clk/actions/owl-divider.c @@ -13,26 +13,13 @@ #include "owl-divider.h" -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate) -{ - return divider_round_rate(&common->hw, rate, parent_rate, - div_hw->table, div_hw->width, - div_hw->div_flags); -} - static int owl_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_divider *div = hw_to_owl_divider(hw); - req->rate = owl_divider_helper_round_rate(&div->common, &div->div_hw, - req->rate, - &req->best_parent_rate); - - return 0; + return divider_determine_rate(hw, req, div->div_hw.table, + div->div_hw.width, div->div_hw.div_flags); } unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, diff --git a/drivers/clk/actions/owl-divider.h b/drivers/clk/actions/owl-divider.h index 083be6d80954d..2ba957740c381 100644 --- a/drivers/clk/actions/owl-divider.h +++ b/drivers/clk/actions/owl-divider.h @@ -56,11 +56,6 @@ static inline struct owl_divider *hw_to_owl_divider(const struct clk_hw *hw) return container_of(common, struct owl_divider, common); } -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate); - unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, const struct owl_divider_hw *div_hw, unsigned long parent_rate); diff --git a/drivers/clk/clk-apple-nco.c b/drivers/clk/clk-apple-nco.c index d3ced4a0f029e..434c067968bbc 100644 --- a/drivers/clk/clk-apple-nco.c +++ b/drivers/clk/clk-apple-nco.c @@ -320,6 +320,7 @@ static int applnco_probe(struct platform_device *pdev) } static const struct of_device_id applnco_ids[] = { + { .compatible = "apple,t8103-nco" }, { .compatible = "apple,nco" }, { } }; diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c index dac190bc6e19a..d2617fe16d2e4 100644 --- a/drivers/clk/clk-bm1880.c +++ b/drivers/clk/clk-bm1880.c @@ -629,10 +629,7 @@ static int bm1880_clk_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div->table, div->width, div->flags); - - return 0; + return divider_determine_rate(hw, req, div->table, div->width, div->flags); } static int bm1880_clk_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-loongson1.c b/drivers/clk/clk-loongson1.c index f9f060d08a5fa..1674181a1107d 100644 --- a/drivers/clk/clk-loongson1.c +++ b/drivers/clk/clk-loongson1.c @@ -99,10 +99,7 @@ static int ls1x_divider_determine_rate(struct clk_hw *hw, struct ls1x_clk *ls1x_clk = to_ls1x_clk(hw); const struct ls1x_clk_div_data *d = ls1x_clk->data; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - d->table, d->width, d->flags); - - return 0; + return divider_determine_rate(hw, req, d->table, d->width, d->flags); } static int ls1x_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-milbeaut.c b/drivers/clk/clk-milbeaut.c index b4f9b7143eaa6..bb94d02a76cf1 100644 --- a/drivers/clk/clk-milbeaut.c +++ b/drivers/clk/clk-milbeaut.c @@ -407,10 +407,7 @@ static int m10v_clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, divider->flags); } static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-renesas-pcie.c b/drivers/clk/clk-renesas-pcie.c index 4c3a5e4eb77ac..f94a9c4d0b670 100644 --- a/drivers/clk/clk-renesas-pcie.c +++ b/drivers/clk/clk-renesas-pcie.c @@ -64,7 +64,7 @@ struct rs9_driver_data { struct i2c_client *client; struct regmap *regmap; const struct rs9_chip_info *chip_info; - struct clk_hw *clk_dif[4]; + struct clk_hw *clk_dif[8]; u8 pll_amplitude; u8 pll_ssc; u8 clk_dif_sr; diff --git a/drivers/clk/clk-versaclock3.c b/drivers/clk/clk-versaclock3.c index 1849863dbd673..27b6cf70f3ae1 100644 --- a/drivers/clk/clk-versaclock3.c +++ b/drivers/clk/clk-versaclock3.c @@ -523,11 +523,8 @@ static int vc3_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div_data->table, - div_data->width, div_data->flags); - - return 0; + return divider_determine_rate(hw, req, div_data->table, div_data->width, + div_data->flags); } static int vc3_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c index 6bae18a84cb6c..fd7ceb92d6515 100644 --- a/drivers/clk/hisilicon/clkdivider-hi6220.c +++ b/drivers/clk/hisilicon/clkdivider-hi6220.c @@ -60,10 +60,8 @@ static int hi6220_clkdiv_determine_rate(struct clk_hw *hw, { struct hi6220_clk_divider *dclk = to_hi6220_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, dclk->table, - dclk->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int hi6220_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index 34c9dc1fb20e5..c03f7821824d1 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -696,8 +696,7 @@ struct clk_hw *imx_clk_scu_alloc_dev(const char *name, if (ret) goto put_device; - ret = driver_set_override(&pdev->dev, &pdev->driver_override, - "imx-scu-clk", strlen("imx-scu-clk")); + ret = device_set_driver_override(&pdev->dev, "imx-scu-clk"); if (ret) goto put_device; diff --git a/drivers/clk/keystone/syscon-clk.c b/drivers/clk/keystone/syscon-clk.c index c509929da8545..ecf180a7949c9 100644 --- a/drivers/clk/keystone/syscon-clk.c +++ b/drivers/clk/keystone/syscon-clk.c @@ -129,7 +129,7 @@ static int ti_syscon_gate_clk_probe(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - regmap = regmap_init_mmio(dev, base, &ti_syscon_regmap_cfg); + regmap = devm_regmap_init_mmio(dev, base, &ti_syscon_regmap_cfg); if (IS_ERR(regmap)) return dev_err_probe(dev, PTR_ERR(regmap), "failed to get regmap\n"); diff --git a/drivers/clk/mediatek/clk-mt7981-eth.c b/drivers/clk/mediatek/clk-mt7981-eth.c index 906aec9ddff54..0655ebb6c561f 100644 --- a/drivers/clk/mediatek/clk-mt7981-eth.c +++ b/drivers/clk/mediatek/clk-mt7981-eth.c @@ -31,7 +31,7 @@ static const struct mtk_gate_regs sgmii0_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii0_clks[] __initconst = { +static const struct mtk_gate sgmii0_clks[] = { GATE_SGMII0(CLK_SGM0_TX_EN, "sgm0_tx_en", "usb_tx250m", 2), GATE_SGMII0(CLK_SGM0_RX_EN, "sgm0_rx_en", "usb_eq_rx250m", 3), GATE_SGMII0(CLK_SGM0_CK0_EN, "sgm0_ck0_en", "usb_ln0", 4), @@ -53,7 +53,7 @@ static const struct mtk_gate_regs sgmii1_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii1_clks[] __initconst = { +static const struct mtk_gate sgmii1_clks[] = { GATE_SGMII1(CLK_SGM1_TX_EN, "sgm1_tx_en", "usb_tx250m", 2), GATE_SGMII1(CLK_SGM1_RX_EN, "sgm1_rx_en", "usb_eq_rx250m", 3), GATE_SGMII1(CLK_SGM1_CK1_EN, "sgm1_ck1_en", "usb_ln0", 4), @@ -75,7 +75,7 @@ static const struct mtk_gate_regs eth_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate eth_clks[] __initconst = { +static const struct mtk_gate eth_clks[] = { GATE_ETH(CLK_ETH_FE_EN, "eth_fe_en", "netsys_2x", 6), GATE_ETH(CLK_ETH_GP2_EN, "eth_gp2_en", "sgm_325m", 7), GATE_ETH(CLK_ETH_GP1_EN, "eth_gp1_en", "sgm_325m", 8), diff --git a/drivers/clk/mediatek/clk-mt8196-mfg.c b/drivers/clk/mediatek/clk-mt8196-mfg.c index ae1eb9de79ae2..f40795b47ff1f 100644 --- a/drivers/clk/mediatek/clk-mt8196-mfg.c +++ b/drivers/clk/mediatek/clk-mt8196-mfg.c @@ -58,24 +58,25 @@ .pcw_shift = _pcw_shift, \ .pcwbits = _pcwbits, \ .pcwibits = MT8196_INTEGER_BITS, \ + .parent_name = "mfg_eb", \ } static const struct mtk_pll_data mfg_ao_plls[] = { - PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, 0, - BIT(0), MFGPLL_CON1, 24, 0, 0, 0, + PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, + PLL_PARENT_EN, BIT(0), MFGPLL_CON1, 24, 0, 0, 0, MFGPLL_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc0_ao_plls[] = { PLL(CLK_MFGSC0_AO_MFGPLL_SC0, "mfgpll-sc0", MFGPLL_SC0_CON0, - MFGPLL_SC0_CON0, 0, 0, 0, BIT(0), MFGPLL_SC0_CON1, 24, 0, 0, 0, - MFGPLL_SC0_CON1, 0, 22), + MFGPLL_SC0_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC0_CON1, 24, + 0, 0, 0, MFGPLL_SC0_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc1_ao_plls[] = { PLL(CLK_MFGSC1_AO_MFGPLL_SC1, "mfgpll-sc1", MFGPLL_SC1_CON0, - MFGPLL_SC1_CON0, 0, 0, 0, BIT(0), MFGPLL_SC1_CON1, 24, 0, 0, 0, - MFGPLL_SC1_CON1, 0, 22), + MFGPLL_SC1_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC1_CON1, 24, + 0, 0, 0, MFGPLL_SC1_CON1, 0, 22), }; static const struct of_device_id of_match_clk_mt8196_mfg[] = { diff --git a/drivers/clk/mediatek/clk-mt8516.c b/drivers/clk/mediatek/clk-mt8516.c index 21eb052b0a539..342a59019fea9 100644 --- a/drivers/clk/mediatek/clk-mt8516.c +++ b/drivers/clk/mediatek/clk-mt8516.c @@ -544,7 +544,7 @@ static const struct mtk_gate_regs top5_cg_regs = { #define GATE_TOP5(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &top5_cg_regs, _shift, &mtk_clk_gate_ops_setclr) -static const struct mtk_gate top_clks[] __initconst = { +static const struct mtk_gate top_clks[] = { /* TOP1 */ GATE_TOP1(CLK_TOP_THEM, "them", "ahb_infra_sel", 1), GATE_TOP1(CLK_TOP_APDMA, "apdma", "ahb_infra_sel", 2), diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c index 19cd27941747a..deafe55a96cb1 100644 --- a/drivers/clk/mediatek/clk-mtk.c +++ b/drivers/clk/mediatek/clk-mtk.c @@ -497,14 +497,16 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, if (mcd->need_runtime_pm) { - devm_pm_runtime_enable(&pdev->dev); + r = devm_pm_runtime_enable(&pdev->dev); + if (r) + goto unmap_io; /* * Do a pm_runtime_resume_and_get() to workaround a possible * deadlock between clk_register() and the genpd framework. */ r = pm_runtime_resume_and_get(&pdev->dev); if (r) - return r; + goto unmap_io; } /* Calculate how many clk_hw_onecell_data entries to allocate */ @@ -618,11 +620,11 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, free_data: mtk_free_clk_data(clk_data); free_base: - if (mcd->shared_io && base) - iounmap(base); - if (mcd->need_runtime_pm) pm_runtime_put(&pdev->dev); +unmap_io: + if (mcd->shared_io && base) + iounmap(base); return r; } diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c index cd2b6ce551c6b..de3eb02670554 100644 --- a/drivers/clk/mediatek/clk-pll.c +++ b/drivers/clk/mediatek/clk-pll.c @@ -358,6 +358,9 @@ struct clk_hw *mtk_clk_register_pll_ops(struct mtk_clk_pll *pll, init.name = data->name; init.flags = (data->flags & PLL_AO) ? CLK_IS_CRITICAL : 0; + if (data->flags & PLL_PARENT_EN) + init.flags |= CLK_OPS_PARENT_ENABLE; + init.ops = pll_ops; if (data->parent_name) init.parent_names = &data->parent_name; diff --git a/drivers/clk/mediatek/clk-pll.h b/drivers/clk/mediatek/clk-pll.h index d71c150ce83e4..de5a8fb7cbcfe 100644 --- a/drivers/clk/mediatek/clk-pll.h +++ b/drivers/clk/mediatek/clk-pll.h @@ -21,6 +21,7 @@ struct mtk_pll_div_table { #define HAVE_RST_BAR BIT(0) #define PLL_AO BIT(1) +#define PLL_PARENT_EN BIT(2) #define POSTDIV_MASK GENMASK(2, 0) struct mtk_pll_data { diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 185b6348251db..d0d4c7b6dc827 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -777,12 +777,23 @@ static struct clk_regmap g12a_hdmi_pll_dco = { }, }; +/* + * G12/SM1 hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table g12a_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap g12a_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL0, .shift = 16, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -800,7 +811,7 @@ static struct clk_regmap g12a_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 18, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -818,7 +829,7 @@ static struct clk_regmap g12a_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 20, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 5a229c4ffae10..ec9a3414875ac 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -349,12 +349,23 @@ static struct clk_regmap gxbb_hdmi_pll = { }, }; +/* + * GXL hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table gxl_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap gxl_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 21, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -372,7 +383,7 @@ static struct clk_regmap gxl_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 23, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -390,7 +401,7 @@ static struct clk_regmap gxl_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 19, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/meson/s4-peripherals.c b/drivers/clk/meson/s4-peripherals.c index 6d69b132d1e1f..bab4f5700de47 100644 --- a/drivers/clk/meson/s4-peripherals.c +++ b/drivers/clk/meson/s4-peripherals.c @@ -1106,7 +1106,6 @@ static struct clk_regmap s4_cts_enci_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1122,7 +1121,6 @@ static struct clk_regmap s4_cts_encp_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1138,7 +1136,6 @@ static struct clk_regmap s4_cts_vdac_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1169,7 +1166,6 @@ static struct clk_regmap s4_hdmi_tx_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_hdmi_tx_parents, .num_parents = ARRAY_SIZE(s4_hdmi_tx_parents), - .flags = CLK_SET_RATE_PARENT, }, }; diff --git a/drivers/clk/microchip/clk-core.c b/drivers/clk/microchip/clk-core.c index b34348d491f3e..82f62731fc0ed 100644 --- a/drivers/clk/microchip/clk-core.c +++ b/drivers/clk/microchip/clk-core.c @@ -283,14 +283,13 @@ static u8 roclk_get_parent(struct clk_hw *hw) v = (readl(refo->ctrl_reg) >> REFO_SEL_SHIFT) & REFO_SEL_MASK; - if (!refo->parent_map) - return v; - - for (i = 0; i < clk_hw_get_num_parents(hw); i++) - if (refo->parent_map[i] == v) - return i; + if (refo->parent_map) { + for (i = 0; i < clk_hw_get_num_parents(hw); i++) + if (refo->parent_map[i] == v) + return i; + } - return -EINVAL; + return v; } static unsigned long roclk_calc_rate(unsigned long parent_rate, @@ -780,15 +779,6 @@ static unsigned long sclk_get_rate(struct clk_hw *hw, unsigned long parent_rate) return parent_rate / div; } -static int sclk_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - req->rate = calc_best_divided_rate(req->rate, req->best_parent_rate, - SLEW_SYSDIV, 1); - - return 0; -} - static int sclk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { @@ -826,13 +816,13 @@ static u8 sclk_get_parent(struct clk_hw *hw) v = (readl(sclk->mux_reg) >> OSC_CUR_SHIFT) & OSC_CUR_MASK; - if (!sclk->parent_map) - return v; + if (sclk->parent_map) { + for (i = 0; i < clk_hw_get_num_parents(hw); i++) + if (sclk->parent_map[i] == v) + return i; + } - for (i = 0; i < clk_hw_get_num_parents(hw); i++) - if (sclk->parent_map[i] == v) - return i; - return -EINVAL; + return v; } static int sclk_set_parent(struct clk_hw *hw, u8 index) @@ -912,7 +902,6 @@ static int sclk_init(struct clk_hw *hw) const struct clk_ops pic32_sclk_ops = { .get_parent = sclk_get_parent, .set_parent = sclk_set_parent, - .determine_rate = sclk_determine_rate, .set_rate = sclk_set_rate, .recalc_rate = sclk_get_rate, .init = sclk_init, diff --git a/drivers/clk/mvebu/cp110-system-controller.c b/drivers/clk/mvebu/cp110-system-controller.c index 03c59bf221060..b47c869060466 100644 --- a/drivers/clk/mvebu/cp110-system-controller.c +++ b/drivers/clk/mvebu/cp110-system-controller.c @@ -110,6 +110,25 @@ static const char * const gate_base_names[] = { [CP110_GATE_EIP197] = "eip197" }; +static unsigned long gate_flags(const u8 bit_idx) +{ + switch (bit_idx) { + case CP110_GATE_PCIE_X1_0: + case CP110_GATE_PCIE_X1_1: + case CP110_GATE_PCIE_X4: + /* + * If a port had an active link at boot time, stopping + * the clock creates a failed state from which controller + * driver can not recover. + * Prevent stopping this clock till after a driver has taken + * ownership. + */ + return CLK_IGNORE_UNUSED; + default: + return 0; + } +}; + struct cp110_gate_clk { struct clk_hw hw; struct regmap *regmap; @@ -171,6 +190,7 @@ static struct clk_hw *cp110_register_gate(const char *name, init.ops = &cp110_gate_ops; init.parent_names = &parent_name; init.num_parents = 1; + init.flags = gate_flags(bit_idx); gate->regmap = regmap; gate->bit_idx = bit_idx; diff --git a/drivers/clk/nuvoton/clk-ma35d1-divider.c b/drivers/clk/nuvoton/clk-ma35d1-divider.c index e39f53d5bf457..e992e7c303419 100644 --- a/drivers/clk/nuvoton/clk-ma35d1-divider.c +++ b/drivers/clk/nuvoton/clk-ma35d1-divider.c @@ -44,11 +44,8 @@ static int ma35d1_clkdiv_determine_rate(struct clk_hw *hw, { struct ma35d1_adc_clk_div *dclk = to_ma35d1_adc_clk_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - dclk->table, dclk->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int ma35d1_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c index 23f980cf6a2b5..ae2fa5341a2e4 100644 --- a/drivers/clk/nxp/clk-lpc32xx.c +++ b/drivers/clk/nxp/clk-lpc32xx.c @@ -975,10 +975,8 @@ static int clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 78a3038426136..a230e166811ba 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -46,6 +46,23 @@ config CLK_GLYMUR_TCSRCC Support for the TCSR clock controller on GLYMUR devices. Say Y if you want to use peripheral devices such as USB/PCIe/EDP. +config CLK_KAANAPALI_GCC + tristate "KAANAPALI Global Clock Controller" + depends on ARM64 || COMPILE_TEST + select QCOM_GDSC + help + Support for the global clock controller on Kaanapali devices. + Say Y if you want to use peripheral devices such as UART, + SPI, I2C, USB, SD/UFS, PCIe etc. + +config CLK_KAANAPALI_TCSRCC + tristate "KAANAPALI TCSR Clock Controller" + depends on ARM64 || COMPILE_TEST + select QCOM_GDSC + help + Support for the TCSR clock controller on Kaanapali devices. + Say Y if you want to use peripheral devices such as PCIe, USB, UFS. + config CLK_X1E80100_CAMCC tristate "X1E80100 Camera Clock Controller" depends on ARM64 || COMPILE_TEST @@ -91,6 +108,16 @@ config CLK_X1E80100_TCSRCC Support for the TCSR clock controller on X1E80100 devices. Say Y if you want to use peripheral devices such as SD/UFS. +config CLK_X1P42100_CAMCC + tristate "X1P42100 Camera Clock Controller" + depends on ARM64 || COMPILE_TEST + select CLK_X1E80100_GCC + help + Support for the camera clock controller on Qualcomm Technologies, Inc. + X1P42100 devices. + Say Y if you want to support camera devices and camera functionality + such as capturing pictures. + config CLK_X1P42100_GPUCC tristate "X1P42100 Graphics Clock Controller" depends on ARM64 || COMPILE_TEST @@ -100,6 +127,16 @@ config CLK_X1P42100_GPUCC Say Y if you want to support graphics controller devices and functionality such as 3D graphics. +config CLK_X1P42100_VIDEOCC + tristate "X1P42100 Video Clock Controller" + depends on ARM64 || COMPILE_TEST + select CLK_X1E80100_GCC + help + Support for the video clock controller on Qualcomm Technologies, Inc. + X1P42100 devices. + Say Y if you want to support video devices and functionality such as + video encode/decode. + config CLK_QCM2290_GPUCC tristate "QCM2290 Graphics Clock Controller" depends on ARM64 || COMPILE_TEST @@ -531,6 +568,7 @@ config QCM_DISPCC_2290 config QCS_DISPCC_615 tristate "QCS615 Display Clock Controller" + depends on ARM64 || COMPILE_TEST select QCS_GCC_615 help Support for the display clock controller on Qualcomm Technologies, Inc @@ -586,6 +624,7 @@ config QCS_GCC_615 config QCS_GPUCC_615 tristate "QCS615 Graphics clock controller" + depends on ARM64 || COMPILE_TEST select QCS_GCC_615 help Support for the graphics clock controller on QCS615 devices. @@ -594,6 +633,7 @@ config QCS_GPUCC_615 config QCS_VIDEOCC_615 tristate "QCS615 Video Clock Controller" + depends on ARM64 || COMPILE_TEST select QCS_GCC_615 help Support for the video clock controller on QCS615 devices. @@ -1448,6 +1488,7 @@ config SA_VIDEOCC_8775P config SM_VIDEOCC_6350 tristate "SM6350 Video Clock Controller" + depends on ARM64 || COMPILE_TEST select SM_GCC_6350 select QCOM_GDSC help diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index 8051d481c439b..c6d705c968a24 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -24,12 +24,16 @@ obj-$(CONFIG_CLK_GFM_LPASS_SM8250) += lpass-gfm-sm8250.o obj-$(CONFIG_CLK_GLYMUR_DISPCC) += dispcc-glymur.o obj-$(CONFIG_CLK_GLYMUR_GCC) += gcc-glymur.o obj-$(CONFIG_CLK_GLYMUR_TCSRCC) += tcsrcc-glymur.o +obj-$(CONFIG_CLK_KAANAPALI_GCC) += gcc-kaanapali.o +obj-$(CONFIG_CLK_KAANAPALI_TCSRCC) += tcsrcc-kaanapali.o obj-$(CONFIG_CLK_X1E80100_CAMCC) += camcc-x1e80100.o obj-$(CONFIG_CLK_X1E80100_DISPCC) += dispcc-x1e80100.o obj-$(CONFIG_CLK_X1E80100_GCC) += gcc-x1e80100.o obj-$(CONFIG_CLK_X1E80100_GPUCC) += gpucc-x1e80100.o obj-$(CONFIG_CLK_X1E80100_TCSRCC) += tcsrcc-x1e80100.o +obj-$(CONFIG_CLK_X1P42100_CAMCC) += camcc-x1p42100.o obj-$(CONFIG_CLK_X1P42100_GPUCC) += gpucc-x1p42100.o +obj-$(CONFIG_CLK_X1P42100_VIDEOCC) += videocc-x1p42100.o obj-$(CONFIG_CLK_QCM2290_GPUCC) += gpucc-qcm2290.o obj-$(CONFIG_IPQ_APSS_PLL) += apss-ipq-pll.o obj-$(CONFIG_IPQ_APSS_5424) += apss-ipq5424.o diff --git a/drivers/clk/qcom/camcc-sm6350.c b/drivers/clk/qcom/camcc-sm6350.c index 8aac97d29ce3f..7df12c1311c68 100644 --- a/drivers/clk/qcom/camcc-sm6350.c +++ b/drivers/clk/qcom/camcc-sm6350.c @@ -145,15 +145,11 @@ static struct clk_alpha_pll_postdiv camcc_pll1_out_even = { static const struct alpha_pll_config camcc_pll2_config = { .l = 0x64, .alpha = 0x0, - .post_div_val = 0x3 << 8, - .post_div_mask = 0x3 << 8, - .aux_output_mask = BIT(1), - .main_output_mask = BIT(0), - .early_output_mask = BIT(3), .config_ctl_val = 0x20000800, .config_ctl_hi_val = 0x400003d2, .test_ctl_val = 0x04000400, .test_ctl_hi_val = 0x00004000, + .user_ctl_val = 0x0000030b, }; static struct clk_alpha_pll camcc_pll2 = { @@ -1693,6 +1689,8 @@ static struct clk_branch camcc_sys_tmr_clk = { }, }; +static struct gdsc titan_top_gdsc; + static struct gdsc bps_gdsc = { .gdscr = 0x6004, .en_rest_wait_val = 0x2, @@ -1702,6 +1700,7 @@ static struct gdsc bps_gdsc = { .name = "bps_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &titan_top_gdsc.pd, .flags = VOTABLE, }; @@ -1714,6 +1713,7 @@ static struct gdsc ipe_0_gdsc = { .name = "ipe_0_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &titan_top_gdsc.pd, .flags = VOTABLE, }; @@ -1726,6 +1726,7 @@ static struct gdsc ife_0_gdsc = { .name = "ife_0_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &titan_top_gdsc.pd, }; static struct gdsc ife_1_gdsc = { @@ -1737,6 +1738,7 @@ static struct gdsc ife_1_gdsc = { .name = "ife_1_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &titan_top_gdsc.pd, }; static struct gdsc ife_2_gdsc = { @@ -1748,6 +1750,7 @@ static struct gdsc ife_2_gdsc = { .name = "ife_2_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &titan_top_gdsc.pd, }; static struct gdsc titan_top_gdsc = { diff --git a/drivers/clk/qcom/camcc-sm7150.c b/drivers/clk/qcom/camcc-sm7150.c index 4a3baf5d8e858..590548cac45bf 100644 --- a/drivers/clk/qcom/camcc-sm7150.c +++ b/drivers/clk/qcom/camcc-sm7150.c @@ -139,13 +139,9 @@ static struct clk_fixed_factor camcc_pll1_out_even = { /* 1920MHz configuration */ static const struct alpha_pll_config camcc_pll2_config = { .l = 0x64, - .post_div_val = 0x3 << 8, - .post_div_mask = 0x3 << 8, - .early_output_mask = BIT(3), - .aux_output_mask = BIT(1), - .main_output_mask = BIT(0), .config_ctl_hi_val = 0x400003d6, .config_ctl_val = 0x20000954, + .user_ctl_val = 0x0000030b, }; static struct clk_alpha_pll camcc_pll2 = { diff --git a/drivers/clk/qcom/camcc-sm8550.c b/drivers/clk/qcom/camcc-sm8550.c index 63aed9e4c362d..b8ece8a57a8a9 100644 --- a/drivers/clk/qcom/camcc-sm8550.c +++ b/drivers/clk/qcom/camcc-sm8550.c @@ -3204,6 +3204,8 @@ static struct clk_branch cam_cc_sfe_1_fast_ahb_clk = { }, }; +static struct gdsc cam_cc_titan_top_gdsc; + static struct gdsc cam_cc_bps_gdsc = { .gdscr = 0x10004, .en_rest_wait_val = 0x2, @@ -3213,6 +3215,7 @@ static struct gdsc cam_cc_bps_gdsc = { .name = "cam_cc_bps_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; @@ -3225,6 +3228,7 @@ static struct gdsc cam_cc_ife_0_gdsc = { .name = "cam_cc_ife_0_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; @@ -3237,6 +3241,7 @@ static struct gdsc cam_cc_ife_1_gdsc = { .name = "cam_cc_ife_1_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; @@ -3249,6 +3254,7 @@ static struct gdsc cam_cc_ife_2_gdsc = { .name = "cam_cc_ife_2_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; @@ -3261,6 +3267,7 @@ static struct gdsc cam_cc_ipe_0_gdsc = { .name = "cam_cc_ipe_0_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; @@ -3273,6 +3280,7 @@ static struct gdsc cam_cc_sbi_gdsc = { .name = "cam_cc_sbi_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; @@ -3285,6 +3293,7 @@ static struct gdsc cam_cc_sfe_0_gdsc = { .name = "cam_cc_sfe_0_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; @@ -3297,6 +3306,7 @@ static struct gdsc cam_cc_sfe_1_gdsc = { .name = "cam_cc_sfe_1_gdsc", }, .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; diff --git a/drivers/clk/qcom/camcc-x1e80100.c b/drivers/clk/qcom/camcc-x1e80100.c index cbcc1c9fcb341..7e3fc7aee854e 100644 --- a/drivers/clk/qcom/camcc-x1e80100.c +++ b/drivers/clk/qcom/camcc-x1e80100.c @@ -1052,6 +1052,31 @@ static struct clk_rcg2 cam_cc_mclk7_clk_src = { }, }; +static const struct freq_tbl ftbl_cam_cc_qdss_debug_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_CAM_CC_PLL8_OUT_EVEN, 8, 0, 0), + F(75000000, P_CAM_CC_PLL0_OUT_EVEN, 8, 0, 0), + F(150000000, P_CAM_CC_PLL0_OUT_EVEN, 4, 0, 0), + F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_qdss_debug_clk_src = { + .cmd_rcgr = 0x13938, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_qdss_debug_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + static const struct freq_tbl ftbl_cam_cc_sfe_0_clk_src[] = { F(345600000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0), F(432000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0), @@ -2182,6 +2207,42 @@ static struct clk_branch cam_cc_mclk7_clk = { }, }; +static struct clk_branch cam_cc_qdss_debug_clk = { + .halt_reg = 0x13a64, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13a64, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_qdss_debug_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_qdss_debug_xo_clk = { + .halt_reg = 0x13a68, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13a68, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_xo_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch cam_cc_sfe_0_clk = { .halt_reg = 0x133c0, .halt_check = BRANCH_HALT, @@ -2398,6 +2459,9 @@ static struct clk_regmap *cam_cc_x1e80100_clocks[] = { [CAM_CC_PLL6_OUT_EVEN] = &cam_cc_pll6_out_even.clkr, [CAM_CC_PLL8] = &cam_cc_pll8.clkr, [CAM_CC_PLL8_OUT_EVEN] = &cam_cc_pll8_out_even.clkr, + [CAM_CC_QDSS_DEBUG_CLK] = &cam_cc_qdss_debug_clk.clkr, + [CAM_CC_QDSS_DEBUG_CLK_SRC] = &cam_cc_qdss_debug_clk_src.clkr, + [CAM_CC_QDSS_DEBUG_XO_CLK] = &cam_cc_qdss_debug_xo_clk.clkr, [CAM_CC_SFE_0_CLK] = &cam_cc_sfe_0_clk.clkr, [CAM_CC_SFE_0_CLK_SRC] = &cam_cc_sfe_0_clk_src.clkr, [CAM_CC_SFE_0_FAST_AHB_CLK] = &cam_cc_sfe_0_fast_ahb_clk.clkr, diff --git a/drivers/clk/qcom/camcc-x1p42100.c b/drivers/clk/qcom/camcc-x1p42100.c new file mode 100644 index 0000000000000..c1a61c2679199 --- /dev/null +++ b/drivers/clk/qcom/camcc-x1p42100.c @@ -0,0 +1,2223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + DT_IFACE, + DT_BI_TCXO, + DT_BI_TCXO_AO, + DT_SLEEP_CLK, +}; + +enum { + P_BI_TCXO, + P_BI_TCXO_AO, + P_CAM_CC_PLL0_OUT_EVEN, + P_CAM_CC_PLL0_OUT_MAIN, + P_CAM_CC_PLL0_OUT_ODD, + P_CAM_CC_PLL1_OUT_EVEN, + P_CAM_CC_PLL2_OUT_EVEN, + P_CAM_CC_PLL2_OUT_MAIN, + P_CAM_CC_PLL3_OUT_EVEN, + P_CAM_CC_PLL6_OUT_EVEN, + P_SLEEP_CLK, +}; + +static const struct pll_vco lucid_ole_vco[] = { + { 249600000, 2300000000, 0 }, +}; + +static const struct pll_vco rivian_ole_vco[] = { + { 777000000, 1285000000, 0 }, +}; + +/* 1200.0 MHz Configuration */ +static const struct alpha_pll_config cam_cc_pll0_config = { + .l = 0x3e, + .alpha = 0x8000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x82aa299c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000003, + .test_ctl_hi1_val = 0x00009000, + .test_ctl_hi2_val = 0x00000034, + .user_ctl_val = 0x00008400, + .user_ctl_hi_val = 0x00000005, +}; + +static struct clk_alpha_pll cam_cc_pll0 = { + .offset = 0x0, + .config = &cam_cc_pll0_config, + .vco_table = lucid_ole_vco, + .num_vco = ARRAY_SIZE(lucid_ole_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll0", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_evo_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_cam_cc_pll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv cam_cc_pll0_out_even = { + .offset = 0x0, + .post_div_shift = 10, + .post_div_table = post_div_table_cam_cc_pll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll0_out_even", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_pll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_lucid_ole_ops, + }, +}; + +static const struct clk_div_table post_div_table_cam_cc_pll0_out_odd[] = { + { 0x2, 3 }, + { } +}; + +static struct clk_alpha_pll_postdiv cam_cc_pll0_out_odd = { + .offset = 0x0, + .post_div_shift = 14, + .post_div_table = post_div_table_cam_cc_pll0_out_odd, + .num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_odd), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll0_out_odd", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_pll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_lucid_ole_ops, + }, +}; + +/* 728.0 MHz Configuration */ +static const struct alpha_pll_config cam_cc_pll1_config = { + .l = 0x25, + .alpha = 0xeaaa, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x82aa299c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000003, + .test_ctl_hi1_val = 0x00009000, + .test_ctl_hi2_val = 0x00000034, + .user_ctl_val = 0x00000400, + .user_ctl_hi_val = 0x00000005, +}; + +static struct clk_alpha_pll cam_cc_pll1 = { + .offset = 0x1000, + .config = &cam_cc_pll1_config, + .vco_table = lucid_ole_vco, + .num_vco = ARRAY_SIZE(lucid_ole_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll1", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_evo_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_cam_cc_pll1_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv cam_cc_pll1_out_even = { + .offset = 0x1000, + .post_div_shift = 10, + .post_div_table = post_div_table_cam_cc_pll1_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll1_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll1_out_even", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_pll1.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_lucid_ole_ops, + }, +}; + +/* 960.0 MHz Configuration */ +static const struct alpha_pll_config cam_cc_pll2_config = { + .l = 0x32, + .alpha = 0x0, + .config_ctl_val = 0x10000030, + .config_ctl_hi_val = 0x80890263, + .config_ctl_hi1_val = 0x00000217, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00100000, +}; + +static struct clk_alpha_pll cam_cc_pll2 = { + .offset = 0x2000, + .config = &cam_cc_pll2_config, + .vco_table = rivian_ole_vco, + .num_vco = ARRAY_SIZE(rivian_ole_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_RIVIAN_EVO], + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll2", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_rivian_evo_ops, + }, + }, +}; + +/* 864.0 MHz Configuration */ +static const struct alpha_pll_config cam_cc_pll3_config = { + .l = 0x2d, + .alpha = 0x0, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x82aa299c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000003, + .test_ctl_hi1_val = 0x00009000, + .test_ctl_hi2_val = 0x00000034, + .user_ctl_val = 0x00000400, + .user_ctl_hi_val = 0x00000005, +}; + +static struct clk_alpha_pll cam_cc_pll3 = { + .offset = 0x3000, + .config = &cam_cc_pll3_config, + .vco_table = lucid_ole_vco, + .num_vco = ARRAY_SIZE(lucid_ole_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll3", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_evo_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_cam_cc_pll3_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv cam_cc_pll3_out_even = { + .offset = 0x3000, + .post_div_shift = 10, + .post_div_table = post_div_table_cam_cc_pll3_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll3_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll3_out_even", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_pll3.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_lucid_ole_ops, + }, +}; + +/* 960.0 MHz Configuration */ +static const struct alpha_pll_config cam_cc_pll6_config = { + .l = 0x32, + .alpha = 0x0, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x82aa299c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000003, + .test_ctl_hi1_val = 0x00009000, + .test_ctl_hi2_val = 0x00000034, + .user_ctl_val = 0x00000400, + .user_ctl_hi_val = 0x00000005, +}; + +static struct clk_alpha_pll cam_cc_pll6 = { + .offset = 0x6000, + .config = &cam_cc_pll6_config, + .vco_table = lucid_ole_vco, + .num_vco = ARRAY_SIZE(lucid_ole_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll6", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_evo_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_cam_cc_pll6_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv cam_cc_pll6_out_even = { + .offset = 0x6000, + .post_div_shift = 10, + .post_div_table = post_div_table_cam_cc_pll6_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll6_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_pll6_out_even", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_pll6.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_lucid_ole_ops, + }, +}; + +static const struct parent_map cam_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_CAM_CC_PLL0_OUT_MAIN, 1 }, + { P_CAM_CC_PLL0_OUT_EVEN, 2 }, + { P_CAM_CC_PLL0_OUT_ODD, 3 }, + { P_CAM_CC_PLL6_OUT_EVEN, 5 }, +}; + +static const struct clk_parent_data cam_cc_parent_data_0[] = { + { .index = DT_BI_TCXO }, + { .hw = &cam_cc_pll0.clkr.hw }, + { .hw = &cam_cc_pll0_out_even.clkr.hw }, + { .hw = &cam_cc_pll0_out_odd.clkr.hw }, + { .hw = &cam_cc_pll6_out_even.clkr.hw }, +}; + +static const struct parent_map cam_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_CAM_CC_PLL2_OUT_EVEN, 3 }, + { P_CAM_CC_PLL2_OUT_MAIN, 5 }, +}; + +static const struct clk_parent_data cam_cc_parent_data_1[] = { + { .index = DT_BI_TCXO }, + { .hw = &cam_cc_pll2.clkr.hw }, + { .hw = &cam_cc_pll2.clkr.hw }, +}; + +static const struct parent_map cam_cc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_CAM_CC_PLL3_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data cam_cc_parent_data_2[] = { + { .index = DT_BI_TCXO }, + { .hw = &cam_cc_pll3_out_even.clkr.hw }, +}; + +static const struct parent_map cam_cc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_CAM_CC_PLL1_OUT_EVEN, 4 }, +}; + +static const struct clk_parent_data cam_cc_parent_data_3[] = { + { .index = DT_BI_TCXO }, + { .hw = &cam_cc_pll1_out_even.clkr.hw }, +}; + +static const struct parent_map cam_cc_parent_map_4[] = { + { P_SLEEP_CLK, 0 }, +}; + +static const struct clk_parent_data cam_cc_parent_data_4[] = { + { .index = DT_SLEEP_CLK }, +}; + +static const struct parent_map cam_cc_parent_map_5[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data cam_cc_parent_data_5[] = { + { .index = DT_BI_TCXO }, +}; + +static const struct freq_tbl ftbl_cam_cc_bps_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_CAM_CC_PLL0_OUT_ODD, 2, 0, 0), + F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0), + F(600000000, P_CAM_CC_PLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_bps_clk_src = { + .cmd_rcgr = 0x10278, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_bps_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_bps_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_camnoc_axi_rt_clk_src[] = { + F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0), + F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_camnoc_axi_rt_clk_src = { + .cmd_rcgr = 0x138f8, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_camnoc_axi_rt_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_camnoc_axi_rt_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_cci_0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_CAM_CC_PLL0_OUT_EVEN, 16, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_cci_0_clk_src = { + .cmd_rcgr = 0x1365c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_cci_0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cci_0_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_cci_1_clk_src = { + .cmd_rcgr = 0x1378c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_cci_0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cci_1_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_cphy_rx_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_cphy_rx_clk_src = { + .cmd_rcgr = 0x11164, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_cphy_rx_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cphy_rx_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_csi0phytimer_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_csi0phytimer_clk_src = { + .cmd_rcgr = 0x150e0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi0phytimer_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_csi1phytimer_clk_src = { + .cmd_rcgr = 0x15104, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi1phytimer_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_csi2phytimer_clk_src = { + .cmd_rcgr = 0x15124, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi2phytimer_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_csi3phytimer_clk_src = { + .cmd_rcgr = 0x15258, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi3phytimer_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_csi4phytimer_clk_src = { + .cmd_rcgr = 0x1538c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi4phytimer_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_csi5phytimer_clk_src = { + .cmd_rcgr = 0x154c0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi5phytimer_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_csid_clk_src[] = { + F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_csid_clk_src = { + .cmd_rcgr = 0x138d4, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_csid_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csid_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_fast_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_CAM_CC_PLL0_OUT_EVEN, 6, 0, 0), + F(200000000, P_CAM_CC_PLL0_OUT_EVEN, 3, 0, 0), + F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0), + F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_fast_ahb_clk_src = { + .cmd_rcgr = 0x10018, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_fast_ahb_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_fast_ahb_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_icp_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0), + F(480000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0), + F(600000000, P_CAM_CC_PLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_icp_clk_src = { + .cmd_rcgr = 0x13520, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_icp_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_icp_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_ife_0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(432000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0), + F(594000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0), + F(675000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0), + F(727000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_ife_0_clk_src = { + .cmd_rcgr = 0x11018, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_2, + .freq_tbl = ftbl_cam_cc_ife_0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_0_clk_src", + .parent_data = cam_cc_parent_data_2, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_ife_lite_clk_src[] = { + F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0), + F(480000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_ife_lite_clk_src = { + .cmd_rcgr = 0x13000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_ife_lite_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_lite_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_ife_lite_csid_clk_src = { + .cmd_rcgr = 0x1313c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_ife_lite_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_lite_csid_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_ipe_nps_clk_src[] = { + F(364000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0), + F(500000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0), + F(600000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0), + F(700000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_ipe_nps_clk_src = { + .cmd_rcgr = 0x103cc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_3, + .freq_tbl = ftbl_cam_cc_ipe_nps_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ipe_nps_clk_src", + .parent_data = cam_cc_parent_data_3, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_jpeg_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_CAM_CC_PLL0_OUT_ODD, 2, 0, 0), + F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0), + F(480000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0), + F(600000000, P_CAM_CC_PLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_jpeg_clk_src = { + .cmd_rcgr = 0x133dc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_jpeg_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_jpeg_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_mclk0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(24000000, P_CAM_CC_PLL2_OUT_MAIN, 10, 1, 4), + F(68571429, P_CAM_CC_PLL2_OUT_MAIN, 14, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_mclk0_clk_src = { + .cmd_rcgr = 0x15000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk0_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_mclk1_clk_src = { + .cmd_rcgr = 0x1501c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk1_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_mclk2_clk_src = { + .cmd_rcgr = 0x15038, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk2_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_mclk3_clk_src = { + .cmd_rcgr = 0x15054, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk3_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_mclk4_clk_src = { + .cmd_rcgr = 0x15070, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk4_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_mclk5_clk_src = { + .cmd_rcgr = 0x1508c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk5_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_mclk6_clk_src = { + .cmd_rcgr = 0x150a8, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk6_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 cam_cc_mclk7_clk_src = { + .cmd_rcgr = 0x150c4, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_1, + .freq_tbl = ftbl_cam_cc_mclk0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk7_clk_src", + .parent_data = cam_cc_parent_data_1, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_qdss_debug_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(75000000, P_CAM_CC_PLL0_OUT_EVEN, 8, 0, 0), + F(150000000, P_CAM_CC_PLL0_OUT_EVEN, 4, 0, 0), + F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_qdss_debug_clk_src = { + .cmd_rcgr = 0x13938, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_qdss_debug_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_sleep_clk_src[] = { + F(32000, P_SLEEP_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_sleep_clk_src = { + .cmd_rcgr = 0x13aa0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_4, + .freq_tbl = ftbl_cam_cc_sleep_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_sleep_clk_src", + .parent_data = cam_cc_parent_data_4, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_4), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_slow_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(80000000, P_CAM_CC_PLL0_OUT_EVEN, 7.5, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_slow_ahb_clk_src = { + .cmd_rcgr = 0x10148, + .mnd_width = 8, + .hid_width = 5, + .parent_map = cam_cc_parent_map_0, + .freq_tbl = ftbl_cam_cc_slow_ahb_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_slow_ahb_clk_src", + .parent_data = cam_cc_parent_data_0, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_cam_cc_xo_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cam_cc_xo_clk_src = { + .cmd_rcgr = 0x13a84, + .mnd_width = 0, + .hid_width = 5, + .parent_map = cam_cc_parent_map_5, + .freq_tbl = ftbl_cam_cc_xo_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "cam_cc_xo_clk_src", + .parent_data = cam_cc_parent_data_5, + .num_parents = ARRAY_SIZE(cam_cc_parent_data_5), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_branch cam_cc_bps_ahb_clk = { + .halt_reg = 0x10274, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10274, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_bps_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_slow_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_bps_clk = { + .halt_reg = 0x103a4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x103a4, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_bps_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_bps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_bps_fast_ahb_clk = { + .halt_reg = 0x10144, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10144, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_bps_fast_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_fast_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_camnoc_axi_nrt_clk = { + .halt_reg = 0x13920, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x13920, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x13920, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_camnoc_axi_nrt_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_camnoc_axi_rt_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_camnoc_axi_rt_clk = { + .halt_reg = 0x13910, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13910, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_camnoc_axi_rt_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_camnoc_axi_rt_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_camnoc_dcd_xo_clk = { + .halt_reg = 0x1392c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1392c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_camnoc_dcd_xo_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_camnoc_xo_clk = { + .halt_reg = 0x13930, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13930, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_camnoc_xo_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cci_0_clk = { + .halt_reg = 0x13788, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13788, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cci_0_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cci_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cci_1_clk = { + .halt_reg = 0x138b8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x138b8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cci_1_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cci_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_core_ahb_clk = { + .halt_reg = 0x13a80, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x13a80, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_core_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_slow_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cpas_ahb_clk = { + .halt_reg = 0x138bc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x138bc, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cpas_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_slow_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cpas_bps_clk = { + .halt_reg = 0x103b0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x103b0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cpas_bps_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_bps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cpas_fast_ahb_clk = { + .halt_reg = 0x138c8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x138c8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cpas_fast_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_fast_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cpas_ife_0_clk = { + .halt_reg = 0x11150, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x11150, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cpas_ife_0_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ife_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cpas_ife_lite_clk = { + .halt_reg = 0x13138, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13138, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cpas_ife_lite_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ife_lite_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_cpas_ipe_nps_clk = { + .halt_reg = 0x10504, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10504, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_cpas_ipe_nps_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ipe_nps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csi0phytimer_clk = { + .halt_reg = 0x150f8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x150f8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi0phytimer_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_csi0phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csi1phytimer_clk = { + .halt_reg = 0x1511c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1511c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi1phytimer_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_csi1phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csi2phytimer_clk = { + .halt_reg = 0x15250, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15250, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi2phytimer_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_csi2phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csi3phytimer_clk = { + .halt_reg = 0x15384, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15384, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi3phytimer_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_csi3phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csi4phytimer_clk = { + .halt_reg = 0x154b8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x154b8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi4phytimer_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_csi4phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csi5phytimer_clk = { + .halt_reg = 0x155ec, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x155ec, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csi5phytimer_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_csi5phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csid_clk = { + .halt_reg = 0x138ec, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x138ec, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csid_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_csid_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csid_csiphy_rx_clk = { + .halt_reg = 0x15100, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15100, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csid_csiphy_rx_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csiphy0_clk = { + .halt_reg = 0x150fc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x150fc, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csiphy0_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csiphy1_clk = { + .halt_reg = 0x15120, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15120, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csiphy1_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csiphy2_clk = { + .halt_reg = 0x15254, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15254, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csiphy2_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csiphy3_clk = { + .halt_reg = 0x15388, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15388, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csiphy3_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csiphy4_clk = { + .halt_reg = 0x154bc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x154bc, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csiphy4_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_csiphy5_clk = { + .halt_reg = 0x155f0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x155f0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_csiphy5_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_icp_ahb_clk = { + .halt_reg = 0x13658, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13658, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_icp_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_slow_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_icp_clk = { + .halt_reg = 0x1364c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1364c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_icp_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_icp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ife_0_clk = { + .halt_reg = 0x11144, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x11144, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_0_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ife_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ife_0_dsp_clk = { + .halt_reg = 0x11154, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x11154, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_0_dsp_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ife_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ife_0_fast_ahb_clk = { + .halt_reg = 0x11160, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x11160, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_0_fast_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_fast_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ife_lite_ahb_clk = { + .halt_reg = 0x13278, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13278, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_lite_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_slow_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ife_lite_clk = { + .halt_reg = 0x1312c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1312c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_lite_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ife_lite_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ife_lite_cphy_rx_clk = { + .halt_reg = 0x13274, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13274, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_lite_cphy_rx_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ife_lite_csid_clk = { + .halt_reg = 0x13268, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13268, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ife_lite_csid_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ife_lite_csid_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ipe_nps_ahb_clk = { + .halt_reg = 0x1051c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1051c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ipe_nps_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_slow_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ipe_nps_clk = { + .halt_reg = 0x104f8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x104f8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ipe_nps_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ipe_nps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ipe_nps_fast_ahb_clk = { + .halt_reg = 0x10520, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10520, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ipe_nps_fast_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_fast_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ipe_pps_clk = { + .halt_reg = 0x10508, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10508, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ipe_pps_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_ipe_nps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_ipe_pps_fast_ahb_clk = { + .halt_reg = 0x10524, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10524, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_ipe_pps_fast_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_fast_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_jpeg_clk = { + .halt_reg = 0x13508, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13508, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_jpeg_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_jpeg_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk0_clk = { + .halt_reg = 0x15018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15018, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk0_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk1_clk = { + .halt_reg = 0x15034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15034, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk1_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk2_clk = { + .halt_reg = 0x15050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15050, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk2_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk3_clk = { + .halt_reg = 0x1506c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1506c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk3_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk4_clk = { + .halt_reg = 0x15088, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x15088, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk4_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk5_clk = { + .halt_reg = 0x150a4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x150a4, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk5_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk6_clk = { + .halt_reg = 0x150c0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x150c0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk6_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_mclk7_clk = { + .halt_reg = 0x150dc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x150dc, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_mclk7_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_mclk7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_qdss_debug_clk = { + .halt_reg = 0x13a64, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13a64, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_qdss_debug_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch cam_cc_qdss_debug_xo_clk = { + .halt_reg = 0x13a68, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x13a68, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "cam_cc_qdss_debug_xo_clk", + .parent_hws = (const struct clk_hw*[]) { + &cam_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc cam_cc_titan_top_gdsc = { + .gdscr = 0x13a6c, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "cam_cc_titan_top_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc cam_cc_bps_gdsc = { + .gdscr = 0x10004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "cam_cc_bps_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, + .flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc cam_cc_ife_0_gdsc = { + .gdscr = 0x11004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "cam_cc_ife_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc cam_cc_ipe_0_gdsc = { + .gdscr = 0x103b8, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "cam_cc_ipe_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .parent = &cam_cc_titan_top_gdsc.pd, + .flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct clk_regmap *cam_cc_x1p42100_clocks[] = { + [CAM_CC_BPS_AHB_CLK] = &cam_cc_bps_ahb_clk.clkr, + [CAM_CC_BPS_CLK] = &cam_cc_bps_clk.clkr, + [CAM_CC_BPS_CLK_SRC] = &cam_cc_bps_clk_src.clkr, + [CAM_CC_BPS_FAST_AHB_CLK] = &cam_cc_bps_fast_ahb_clk.clkr, + [CAM_CC_CAMNOC_AXI_NRT_CLK] = &cam_cc_camnoc_axi_nrt_clk.clkr, + [CAM_CC_CAMNOC_AXI_RT_CLK] = &cam_cc_camnoc_axi_rt_clk.clkr, + [CAM_CC_CAMNOC_AXI_RT_CLK_SRC] = &cam_cc_camnoc_axi_rt_clk_src.clkr, + [CAM_CC_CAMNOC_DCD_XO_CLK] = &cam_cc_camnoc_dcd_xo_clk.clkr, + [CAM_CC_CAMNOC_XO_CLK] = &cam_cc_camnoc_xo_clk.clkr, + [CAM_CC_CCI_0_CLK] = &cam_cc_cci_0_clk.clkr, + [CAM_CC_CCI_0_CLK_SRC] = &cam_cc_cci_0_clk_src.clkr, + [CAM_CC_CCI_1_CLK] = &cam_cc_cci_1_clk.clkr, + [CAM_CC_CCI_1_CLK_SRC] = &cam_cc_cci_1_clk_src.clkr, + [CAM_CC_CORE_AHB_CLK] = &cam_cc_core_ahb_clk.clkr, + [CAM_CC_CPAS_AHB_CLK] = &cam_cc_cpas_ahb_clk.clkr, + [CAM_CC_CPAS_BPS_CLK] = &cam_cc_cpas_bps_clk.clkr, + [CAM_CC_CPAS_FAST_AHB_CLK] = &cam_cc_cpas_fast_ahb_clk.clkr, + [CAM_CC_CPAS_IFE_0_CLK] = &cam_cc_cpas_ife_0_clk.clkr, + [CAM_CC_CPAS_IFE_LITE_CLK] = &cam_cc_cpas_ife_lite_clk.clkr, + [CAM_CC_CPAS_IPE_NPS_CLK] = &cam_cc_cpas_ipe_nps_clk.clkr, + [CAM_CC_CPHY_RX_CLK_SRC] = &cam_cc_cphy_rx_clk_src.clkr, + [CAM_CC_CSI0PHYTIMER_CLK] = &cam_cc_csi0phytimer_clk.clkr, + [CAM_CC_CSI0PHYTIMER_CLK_SRC] = &cam_cc_csi0phytimer_clk_src.clkr, + [CAM_CC_CSI1PHYTIMER_CLK] = &cam_cc_csi1phytimer_clk.clkr, + [CAM_CC_CSI1PHYTIMER_CLK_SRC] = &cam_cc_csi1phytimer_clk_src.clkr, + [CAM_CC_CSI2PHYTIMER_CLK] = &cam_cc_csi2phytimer_clk.clkr, + [CAM_CC_CSI2PHYTIMER_CLK_SRC] = &cam_cc_csi2phytimer_clk_src.clkr, + [CAM_CC_CSI3PHYTIMER_CLK] = &cam_cc_csi3phytimer_clk.clkr, + [CAM_CC_CSI3PHYTIMER_CLK_SRC] = &cam_cc_csi3phytimer_clk_src.clkr, + [CAM_CC_CSI4PHYTIMER_CLK] = &cam_cc_csi4phytimer_clk.clkr, + [CAM_CC_CSI4PHYTIMER_CLK_SRC] = &cam_cc_csi4phytimer_clk_src.clkr, + [CAM_CC_CSI5PHYTIMER_CLK] = &cam_cc_csi5phytimer_clk.clkr, + [CAM_CC_CSI5PHYTIMER_CLK_SRC] = &cam_cc_csi5phytimer_clk_src.clkr, + [CAM_CC_CSID_CLK] = &cam_cc_csid_clk.clkr, + [CAM_CC_CSID_CLK_SRC] = &cam_cc_csid_clk_src.clkr, + [CAM_CC_CSID_CSIPHY_RX_CLK] = &cam_cc_csid_csiphy_rx_clk.clkr, + [CAM_CC_CSIPHY0_CLK] = &cam_cc_csiphy0_clk.clkr, + [CAM_CC_CSIPHY1_CLK] = &cam_cc_csiphy1_clk.clkr, + [CAM_CC_CSIPHY2_CLK] = &cam_cc_csiphy2_clk.clkr, + [CAM_CC_CSIPHY3_CLK] = &cam_cc_csiphy3_clk.clkr, + [CAM_CC_CSIPHY4_CLK] = &cam_cc_csiphy4_clk.clkr, + [CAM_CC_CSIPHY5_CLK] = &cam_cc_csiphy5_clk.clkr, + [CAM_CC_FAST_AHB_CLK_SRC] = &cam_cc_fast_ahb_clk_src.clkr, + [CAM_CC_ICP_AHB_CLK] = &cam_cc_icp_ahb_clk.clkr, + [CAM_CC_ICP_CLK] = &cam_cc_icp_clk.clkr, + [CAM_CC_ICP_CLK_SRC] = &cam_cc_icp_clk_src.clkr, + [CAM_CC_IFE_0_CLK] = &cam_cc_ife_0_clk.clkr, + [CAM_CC_IFE_0_CLK_SRC] = &cam_cc_ife_0_clk_src.clkr, + [CAM_CC_IFE_0_DSP_CLK] = &cam_cc_ife_0_dsp_clk.clkr, + [CAM_CC_IFE_0_FAST_AHB_CLK] = &cam_cc_ife_0_fast_ahb_clk.clkr, + [CAM_CC_IFE_LITE_AHB_CLK] = &cam_cc_ife_lite_ahb_clk.clkr, + [CAM_CC_IFE_LITE_CLK] = &cam_cc_ife_lite_clk.clkr, + [CAM_CC_IFE_LITE_CLK_SRC] = &cam_cc_ife_lite_clk_src.clkr, + [CAM_CC_IFE_LITE_CPHY_RX_CLK] = &cam_cc_ife_lite_cphy_rx_clk.clkr, + [CAM_CC_IFE_LITE_CSID_CLK] = &cam_cc_ife_lite_csid_clk.clkr, + [CAM_CC_IFE_LITE_CSID_CLK_SRC] = &cam_cc_ife_lite_csid_clk_src.clkr, + [CAM_CC_IPE_NPS_AHB_CLK] = &cam_cc_ipe_nps_ahb_clk.clkr, + [CAM_CC_IPE_NPS_CLK] = &cam_cc_ipe_nps_clk.clkr, + [CAM_CC_IPE_NPS_CLK_SRC] = &cam_cc_ipe_nps_clk_src.clkr, + [CAM_CC_IPE_NPS_FAST_AHB_CLK] = &cam_cc_ipe_nps_fast_ahb_clk.clkr, + [CAM_CC_IPE_PPS_CLK] = &cam_cc_ipe_pps_clk.clkr, + [CAM_CC_IPE_PPS_FAST_AHB_CLK] = &cam_cc_ipe_pps_fast_ahb_clk.clkr, + [CAM_CC_JPEG_CLK] = &cam_cc_jpeg_clk.clkr, + [CAM_CC_JPEG_CLK_SRC] = &cam_cc_jpeg_clk_src.clkr, + [CAM_CC_MCLK0_CLK] = &cam_cc_mclk0_clk.clkr, + [CAM_CC_MCLK0_CLK_SRC] = &cam_cc_mclk0_clk_src.clkr, + [CAM_CC_MCLK1_CLK] = &cam_cc_mclk1_clk.clkr, + [CAM_CC_MCLK1_CLK_SRC] = &cam_cc_mclk1_clk_src.clkr, + [CAM_CC_MCLK2_CLK] = &cam_cc_mclk2_clk.clkr, + [CAM_CC_MCLK2_CLK_SRC] = &cam_cc_mclk2_clk_src.clkr, + [CAM_CC_MCLK3_CLK] = &cam_cc_mclk3_clk.clkr, + [CAM_CC_MCLK3_CLK_SRC] = &cam_cc_mclk3_clk_src.clkr, + [CAM_CC_MCLK4_CLK] = &cam_cc_mclk4_clk.clkr, + [CAM_CC_MCLK4_CLK_SRC] = &cam_cc_mclk4_clk_src.clkr, + [CAM_CC_MCLK5_CLK] = &cam_cc_mclk5_clk.clkr, + [CAM_CC_MCLK5_CLK_SRC] = &cam_cc_mclk5_clk_src.clkr, + [CAM_CC_MCLK6_CLK] = &cam_cc_mclk6_clk.clkr, + [CAM_CC_MCLK6_CLK_SRC] = &cam_cc_mclk6_clk_src.clkr, + [CAM_CC_MCLK7_CLK] = &cam_cc_mclk7_clk.clkr, + [CAM_CC_MCLK7_CLK_SRC] = &cam_cc_mclk7_clk_src.clkr, + [CAM_CC_PLL0] = &cam_cc_pll0.clkr, + [CAM_CC_PLL0_OUT_EVEN] = &cam_cc_pll0_out_even.clkr, + [CAM_CC_PLL0_OUT_ODD] = &cam_cc_pll0_out_odd.clkr, + [CAM_CC_PLL1] = &cam_cc_pll1.clkr, + [CAM_CC_PLL1_OUT_EVEN] = &cam_cc_pll1_out_even.clkr, + [CAM_CC_PLL2] = &cam_cc_pll2.clkr, + [CAM_CC_PLL3] = &cam_cc_pll3.clkr, + [CAM_CC_PLL3_OUT_EVEN] = &cam_cc_pll3_out_even.clkr, + [CAM_CC_PLL6] = &cam_cc_pll6.clkr, + [CAM_CC_PLL6_OUT_EVEN] = &cam_cc_pll6_out_even.clkr, + [CAM_CC_QDSS_DEBUG_CLK] = &cam_cc_qdss_debug_clk.clkr, + [CAM_CC_QDSS_DEBUG_CLK_SRC] = &cam_cc_qdss_debug_clk_src.clkr, + [CAM_CC_QDSS_DEBUG_XO_CLK] = &cam_cc_qdss_debug_xo_clk.clkr, + [CAM_CC_SLEEP_CLK_SRC] = &cam_cc_sleep_clk_src.clkr, + [CAM_CC_SLOW_AHB_CLK_SRC] = &cam_cc_slow_ahb_clk_src.clkr, + [CAM_CC_XO_CLK_SRC] = &cam_cc_xo_clk_src.clkr, +}; + +static struct gdsc *cam_cc_x1p42100_gdscs[] = { + [CAM_CC_BPS_GDSC] = &cam_cc_bps_gdsc, + [CAM_CC_IFE_0_GDSC] = &cam_cc_ife_0_gdsc, + [CAM_CC_IPE_0_GDSC] = &cam_cc_ipe_0_gdsc, + [CAM_CC_TITAN_TOP_GDSC] = &cam_cc_titan_top_gdsc, +}; + +static const struct qcom_reset_map cam_cc_x1p42100_resets[] = { + [CAM_CC_BPS_BCR] = { 0x10000 }, + [CAM_CC_ICP_BCR] = { 0x1351c }, + [CAM_CC_IFE_0_BCR] = { 0x11000 }, + [CAM_CC_IPE_0_BCR] = { 0x103b4 }, +}; + +static struct clk_alpha_pll *cam_cc_x1p42100_plls[] = { + &cam_cc_pll0, + &cam_cc_pll1, + &cam_cc_pll2, + &cam_cc_pll3, + &cam_cc_pll6, +}; + +static u32 cam_cc_x1p42100_critical_cbcrs[] = { + 0x13a9c, /* CAM_CC_GDSC_CLK */ + 0x13ab8, /* CAM_CC_SLEEP_CLK */ +}; + +static const struct regmap_config cam_cc_x1p42100_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1603c, + .fast_io = true, +}; + +static struct qcom_cc_driver_data cam_cc_x1p42100_driver_data = { + .alpha_plls = cam_cc_x1p42100_plls, + .num_alpha_plls = ARRAY_SIZE(cam_cc_x1p42100_plls), + .clk_cbcrs = cam_cc_x1p42100_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(cam_cc_x1p42100_critical_cbcrs), +}; + +static struct qcom_cc_desc cam_cc_x1p42100_desc = { + .config = &cam_cc_x1p42100_regmap_config, + .clks = cam_cc_x1p42100_clocks, + .num_clks = ARRAY_SIZE(cam_cc_x1p42100_clocks), + .resets = cam_cc_x1p42100_resets, + .num_resets = ARRAY_SIZE(cam_cc_x1p42100_resets), + .gdscs = cam_cc_x1p42100_gdscs, + .num_gdscs = ARRAY_SIZE(cam_cc_x1p42100_gdscs), + .use_rpm = true, + .driver_data = &cam_cc_x1p42100_driver_data, +}; + +static const struct of_device_id cam_cc_x1p42100_match_table[] = { + { .compatible = "qcom,x1p42100-camcc" }, + { } +}; +MODULE_DEVICE_TABLE(of, cam_cc_x1p42100_match_table); + +static int cam_cc_x1p42100_probe(struct platform_device *pdev) +{ + return qcom_cc_probe(pdev, &cam_cc_x1p42100_desc); +} + +static struct platform_driver cam_cc_x1p42100_driver = { + .probe = cam_cc_x1p42100_probe, + .driver = { + .name = "camcc-x1p42100", + .of_match_table = cam_cc_x1p42100_match_table, + }, +}; + +module_platform_driver(cam_cc_x1p42100_driver); + +MODULE_DESCRIPTION("QTI CAMCC X1P42100 Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 6aeba40358c11..a84e8bee65346 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -1257,11 +1257,8 @@ static int clk_alpha_pll_postdiv_determine_rate(struct clk_hw *hw, else table = clk_alpha_div_table; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - table, pll->width, - CLK_DIVIDER_POWER_OF_TWO); - - return 0; + return divider_determine_rate(hw, req, table, pll->width, + CLK_DIVIDER_POWER_OF_TWO); } static int clk_alpha_pll_postdiv_ro_determine_rate(struct clk_hw *hw, @@ -1617,11 +1614,8 @@ static int clk_trion_pll_postdiv_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); }; static int @@ -1657,11 +1651,8 @@ static int clk_alpha_pll_postdiv_fabia_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int clk_alpha_pll_postdiv_fabia_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index e18cb8807d735..d0a5847f91114 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -755,7 +755,7 @@ static int clk_rcg2_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) { struct clk_rcg2 *rcg = to_clk_rcg2(hw); - u32 notn_m, n, m, d, not2d, mask, duty_per, cfg; + u32 notn_m, n, m, d, not2d, mask, cfg; int ret; /* Duty-cycle cannot be modified for non-MND RCGs */ @@ -774,10 +774,8 @@ static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) n = (~(notn_m) + m) & mask; - duty_per = (duty->num * 100) / duty->den; - /* Calculate 2d value */ - d = DIV_ROUND_CLOSEST(n * duty_per * 2, 100); + d = DIV_ROUND_CLOSEST(n * duty->num * 2, duty->den); /* * Check bit widths of 2d. If D is too big reduce duty cycle. @@ -1266,6 +1264,7 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, if (req->max_rate < parent_req.max_rate) parent_req.max_rate = req->max_rate; + parent_req.best_parent_hw = req->best_parent_hw; ret = __clk_determine_rate(req->best_parent_hw, &parent_req); if (ret) return ret; diff --git a/drivers/clk/qcom/clk-regmap-divider.c b/drivers/clk/qcom/clk-regmap-divider.c index 4f5395f0ab6d0..672e82caf2050 100644 --- a/drivers/clk/qcom/clk-regmap-divider.c +++ b/drivers/clk/qcom/clk-regmap-divider.c @@ -26,24 +26,16 @@ static int div_ro_determine_rate(struct clk_hw *hw, val >>= divider->shift; val &= BIT(divider->width) - 1; - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST, val); - - return 0; + return divider_ro_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST, val); } static int div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct clk_regmap_div *divider = to_clk_regmap_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index 63c38cb47bc45..d8cf716a1dec7 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -395,6 +395,19 @@ DEFINE_CLK_RPMH_VRM(clk4, _a, "C4A_E0", 1); DEFINE_CLK_RPMH_VRM(clk5, _a, "C5A_E0", 1); DEFINE_CLK_RPMH_VRM(clk8, _a, "C8A_E0", 1); +DEFINE_CLK_RPMH_VRM(ln_bb_clk1, _a2_e0, "C6A_E0", 2); +DEFINE_CLK_RPMH_VRM(ln_bb_clk2, _a2_e0, "C7A_E0", 2); +DEFINE_CLK_RPMH_VRM(ln_bb_clk3, _a2_e0, "C8A_E0", 2); + +DEFINE_CLK_RPMH_VRM(rf_clk1, _a_e0, "C1A_E0", 1); +DEFINE_CLK_RPMH_VRM(rf_clk2, _a_e0, "C2A_E0", 1); + +DEFINE_CLK_RPMH_VRM(rf_clk3, _a2_e0, "C3A_E0", 2); +DEFINE_CLK_RPMH_VRM(rf_clk4, _a2_e0, "C4A_E0", 2); +DEFINE_CLK_RPMH_VRM(rf_clk5, _a2_e0, "C5A_E0", 2); + +DEFINE_CLK_RPMH_VRM(div_clk1, _a4_e0, "C11A_E0", 4); + DEFINE_CLK_RPMH_BCM(ce, "CE0"); DEFINE_CLK_RPMH_BCM(hwkm, "HK0"); DEFINE_CLK_RPMH_BCM(ipa, "IP0"); @@ -900,6 +913,34 @@ static const struct clk_rpmh_desc clk_rpmh_glymur = { .num_clks = ARRAY_SIZE(glymur_rpmh_clocks), }; +static struct clk_hw *kaanapali_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &clk_rpmh_bi_tcxo_div2.hw, + [RPMH_CXO_CLK_A] = &clk_rpmh_bi_tcxo_div2_ao.hw, + [RPMH_DIV_CLK1] = &clk_rpmh_div_clk1_a4_e0.hw, + [RPMH_LN_BB_CLK1] = &clk_rpmh_ln_bb_clk1_a2_e0.hw, + [RPMH_LN_BB_CLK1_A] = &clk_rpmh_ln_bb_clk1_a2_e0_ao.hw, + [RPMH_LN_BB_CLK2] = &clk_rpmh_ln_bb_clk2_a2_e0.hw, + [RPMH_LN_BB_CLK2_A] = &clk_rpmh_ln_bb_clk2_a2_e0_ao.hw, + [RPMH_LN_BB_CLK3] = &clk_rpmh_ln_bb_clk3_a2_e0.hw, + [RPMH_LN_BB_CLK3_A] = &clk_rpmh_ln_bb_clk3_a2_e0_ao.hw, + [RPMH_RF_CLK1] = &clk_rpmh_rf_clk1_a_e0.hw, + [RPMH_RF_CLK1_A] = &clk_rpmh_rf_clk1_a_e0_ao.hw, + [RPMH_RF_CLK2] = &clk_rpmh_rf_clk2_a_e0.hw, + [RPMH_RF_CLK2_A] = &clk_rpmh_rf_clk2_a_e0_ao.hw, + [RPMH_RF_CLK3] = &clk_rpmh_rf_clk3_a2_e0.hw, + [RPMH_RF_CLK3_A] = &clk_rpmh_rf_clk3_a2_e0_ao.hw, + [RPMH_RF_CLK4] = &clk_rpmh_rf_clk4_a2_e0.hw, + [RPMH_RF_CLK4] = &clk_rpmh_rf_clk4_a2_e0_ao.hw, + [RPMH_RF_CLK5_A] = &clk_rpmh_rf_clk5_a2_e0.hw, + [RPMH_RF_CLK5_A] = &clk_rpmh_rf_clk5_a2_e0_ao.hw, + [RPMH_IPA_CLK] = &clk_rpmh_ipa.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_kaanapali = { + .clks = kaanapali_rpmh_clocks, + .num_clks = ARRAY_SIZE(kaanapali_rpmh_clocks), +}; + static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec, void *data) { @@ -990,6 +1031,7 @@ static int clk_rpmh_probe(struct platform_device *pdev) static const struct of_device_id clk_rpmh_match_table[] = { { .compatible = "qcom,glymur-rpmh-clk", .data = &clk_rpmh_glymur}, + { .compatible = "qcom,kaanapali-rpmh-clk", .data = &clk_rpmh_kaanapali}, { .compatible = "qcom,milos-rpmh-clk", .data = &clk_rpmh_milos}, { .compatible = "qcom,qcs615-rpmh-clk", .data = &clk_rpmh_qcs615}, { .compatible = "qcom,qdu1000-rpmh-clk", .data = &clk_rpmh_qdu1000}, diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 1215918867741..eec369d2173b5 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -454,7 +454,7 @@ int qcom_cc_probe_by_index(struct platform_device *pdev, int index, base = devm_platform_ioremap_resource(pdev, index); if (IS_ERR(base)) - return -ENOMEM; + return PTR_ERR(base); regmap = devm_regmap_init_mmio(&pdev->dev, base, desc->config); if (IS_ERR(regmap)) diff --git a/drivers/clk/qcom/dispcc-sdm845.c b/drivers/clk/qcom/dispcc-sdm845.c index 2f9e9665d7e93..78e43f6d75026 100644 --- a/drivers/clk/qcom/dispcc-sdm845.c +++ b/drivers/clk/qcom/dispcc-sdm845.c @@ -280,7 +280,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { .name = "disp_cc_mdss_pclk0_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; @@ -295,7 +295,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = { .name = "disp_cc_mdss_pclk1_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm7150.c b/drivers/clk/qcom/dispcc-sm7150.c index bdfff246ed3fe..923f0f38e804c 100644 --- a/drivers/clk/qcom/dispcc-sm7150.c +++ b/drivers/clk/qcom/dispcc-sm7150.c @@ -356,7 +356,7 @@ static struct clk_rcg2 dispcc_mdss_pclk0_clk_src = { .name = "dispcc_mdss_pclk0_clk_src", .parent_data = dispcc_parent_data_4, .num_parents = ARRAY_SIZE(dispcc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; @@ -370,7 +370,7 @@ static struct clk_rcg2 dispcc_mdss_pclk1_clk_src = { .name = "dispcc_mdss_pclk1_clk_src", .parent_data = dispcc_parent_data_4, .num_parents = ARRAY_SIZE(dispcc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index 62059120f9720..17e860307fa10 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -2317,7 +2317,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -2339,7 +2339,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_3, .num_parents = ARRAY_SIZE(gcc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -6760,7 +6760,7 @@ static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { .halt_reg = 0x3f088, - .halt_check = BRANCH_HALT_DELAY, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x3f088, .hwcg_bit = 1, .clkr = { @@ -6816,7 +6816,7 @@ static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = { static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { .halt_reg = 0xe2078, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0xe2078, .hwcg_bit = 1, .clkr = { @@ -6872,7 +6872,7 @@ static struct clk_branch gcc_usb3_tert_phy_com_aux_clk = { static struct clk_branch gcc_usb3_tert_phy_pipe_clk = { .halt_reg = 0xe1078, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0xe1078, .hwcg_bit = 1, .clkr = { @@ -6961,7 +6961,7 @@ static struct clk_branch gcc_usb4_0_master_clk = { static struct clk_branch gcc_usb4_0_phy_p2rr2p_pipe_clk = { .halt_reg = 0x2b0f4, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x2b0f4, .enable_mask = BIT(0), @@ -6979,7 +6979,7 @@ static struct clk_branch gcc_usb4_0_phy_p2rr2p_pipe_clk = { static struct clk_branch gcc_usb4_0_phy_pcie_pipe_clk = { .halt_reg = 0x2b04c, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x62010, .enable_mask = BIT(11), @@ -7033,7 +7033,7 @@ static struct clk_branch gcc_usb4_0_phy_rx1_clk = { static struct clk_branch gcc_usb4_0_phy_usb_pipe_clk = { .halt_reg = 0x2b0bc, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x2b0bc, .hwcg_bit = 1, .clkr = { @@ -7196,7 +7196,7 @@ static struct clk_branch gcc_usb4_1_master_clk = { static struct clk_branch gcc_usb4_1_phy_p2rr2p_pipe_clk = { .halt_reg = 0x2d118, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x2d118, .enable_mask = BIT(0), @@ -7214,7 +7214,7 @@ static struct clk_branch gcc_usb4_1_phy_p2rr2p_pipe_clk = { static struct clk_branch gcc_usb4_1_phy_pcie_pipe_clk = { .halt_reg = 0x2d04c, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x62010, .enable_mask = BIT(12), @@ -7268,7 +7268,7 @@ static struct clk_branch gcc_usb4_1_phy_rx1_clk = { static struct clk_branch gcc_usb4_1_phy_usb_pipe_clk = { .halt_reg = 0x2d0e0, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x2d0e0, .hwcg_bit = 1, .clkr = { @@ -7431,7 +7431,7 @@ static struct clk_branch gcc_usb4_2_master_clk = { static struct clk_branch gcc_usb4_2_phy_p2rr2p_pipe_clk = { .halt_reg = 0xe00f8, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0xe00f8, .enable_mask = BIT(0), @@ -7449,7 +7449,7 @@ static struct clk_branch gcc_usb4_2_phy_p2rr2p_pipe_clk = { static struct clk_branch gcc_usb4_2_phy_pcie_pipe_clk = { .halt_reg = 0xe004c, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x62010, .enable_mask = BIT(13), @@ -7503,7 +7503,7 @@ static struct clk_branch gcc_usb4_2_phy_rx1_clk = { static struct clk_branch gcc_usb4_2_phy_usb_pipe_clk = { .halt_reg = 0xe00c0, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0xe00c0, .hwcg_bit = 1, .clkr = { diff --git a/drivers/clk/qcom/gcc-ipq5018.c b/drivers/clk/qcom/gcc-ipq5018.c index dcda2be8c1a51..64792cda06202 100644 --- a/drivers/clk/qcom/gcc-ipq5018.c +++ b/drivers/clk/qcom/gcc-ipq5018.c @@ -1340,6 +1340,7 @@ static struct clk_branch gcc_sleep_clk_src = { .name = "gcc_sleep_clk_src", .parent_data = gcc_sleep_clk_data, .num_parents = ARRAY_SIZE(gcc_sleep_clk_data), + .flags = CLK_IS_CRITICAL, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/gcc-ipq5424.c b/drivers/clk/qcom/gcc-ipq5424.c index 3d42f3d85c7a9..71afa1b86b723 100644 --- a/drivers/clk/qcom/gcc-ipq5424.c +++ b/drivers/clk/qcom/gcc-ipq5424.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2018,2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -3284,6 +3284,7 @@ static const struct qcom_cc_desc gcc_ipq5424_desc = { .num_clk_hws = ARRAY_SIZE(gcc_ipq5424_hws), .icc_hws = icc_ipq5424_hws, .num_icc_hws = ARRAY_SIZE(icc_ipq5424_hws), + .icc_first_node_id = IPQ_APPS_ID, }; static int gcc_ipq5424_probe(struct platform_device *pdev) diff --git a/drivers/clk/qcom/gcc-kaanapali.c b/drivers/clk/qcom/gcc-kaanapali.c new file mode 100644 index 0000000000000..1790e54df7b3a --- /dev/null +++ b/drivers/clk/qcom/gcc-kaanapali.c @@ -0,0 +1,3541 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "clk-regmap-phy-mux.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + DT_BI_TCXO, + DT_BI_TCXO_AO, + DT_SLEEP_CLK, + DT_PCIE_0_PIPE_CLK, + DT_UFS_PHY_RX_SYMBOL_0_CLK, + DT_UFS_PHY_RX_SYMBOL_1_CLK, + DT_UFS_PHY_TX_SYMBOL_0_CLK, + DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, +}; + +enum { + P_BI_TCXO, + P_GCC_GPLL0_OUT_EVEN, + P_GCC_GPLL0_OUT_MAIN, + P_GCC_GPLL1_OUT_MAIN, + P_GCC_GPLL4_OUT_MAIN, + P_GCC_GPLL7_OUT_MAIN, + P_GCC_GPLL9_OUT_MAIN, + P_PCIE_0_PIPE_CLK, + P_SLEEP_CLK, + P_UFS_PHY_RX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_1_CLK, + P_UFS_PHY_TX_SYMBOL_0_CLK, + P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, +}; + +static struct clk_alpha_pll gcc_gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T], + .clkr = { + .enable_reg = 0x52020, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpll0", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_taycan_eko_t_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 10, + .post_div_table = post_div_table_gcc_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T], + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_gpll0_out_even", + .parent_hws = (const struct clk_hw*[]) { + &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops, + }, +}; + +static struct clk_alpha_pll gcc_gpll1 = { + .offset = 0x1000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T], + .clkr = { + .enable_reg = 0x52020, + .enable_mask = BIT(1), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpll1", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_taycan_eko_t_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll4 = { + .offset = 0x4000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T], + .clkr = { + .enable_reg = 0x52020, + .enable_mask = BIT(4), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpll4", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_taycan_eko_t_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll7 = { + .offset = 0x7000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T], + .clkr = { + .enable_reg = 0x52020, + .enable_mask = BIT(7), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpll7", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_taycan_eko_t_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll9 = { + .offset = 0x9000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T], + .clkr = { + .enable_reg = 0x52020, + .enable_mask = BIT(9), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpll9", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_taycan_eko_t_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .index = DT_BI_TCXO }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL1_OUT_MAIN, 4 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .index = DT_BI_TCXO }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll1.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_SLEEP_CLK, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .index = DT_BI_TCXO }, + { .hw = &gcc_gpll0.clkr.hw }, + { .index = DT_SLEEP_CLK }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .index = DT_BI_TCXO }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data gcc_parent_data_4[] = { + { .index = DT_BI_TCXO }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL7_OUT_MAIN, 2 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .index = DT_BI_TCXO }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll7.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .index = DT_BI_TCXO }, + { .index = DT_SLEEP_CLK }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL9_OUT_MAIN, 2 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_8[] = { + { .index = DT_BI_TCXO }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll9.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_12[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_12[] = { + { .index = DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK }, + { .index = DT_BI_TCXO }, +}; + +static struct clk_regmap_phy_mux gcc_pcie_0_pipe_clk_src = { + .reg = 0x6b090, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_pipe_clk_src", + .parent_data = &(const struct clk_parent_data){ + .index = DT_PCIE_0_PIPE_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_0_clk_src = { + .reg = 0x77068, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_rx_symbol_0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .index = DT_UFS_PHY_RX_SYMBOL_0_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_1_clk_src = { + .reg = 0x770ec, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_rx_symbol_1_clk_src", + .parent_data = &(const struct clk_parent_data){ + .index = DT_UFS_PHY_RX_SYMBOL_1_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_ufs_phy_tx_symbol_0_clk_src = { + .reg = 0x77058, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_tx_symbol_0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .index = DT_UFS_PHY_TX_SYMBOL_0_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = { + .reg = 0x39074, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_12, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb3_prim_phy_pipe_clk_src", + .parent_data = gcc_parent_data_12, + .num_parents = ARRAY_SIZE(gcc_parent_data_12), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x64004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x65004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x66004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x6b094, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_0_phy_aux_clk_src = { + .cmd_rcgr = 0x6b0ac, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_phy_aux_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = { + .cmd_rcgr = 0x6b078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x33010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_i2c_s0_clk_src = { + .cmd_rcgr = 0x17008, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_i2c_s1_clk_src = { + .cmd_rcgr = 0x17024, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_i2c_s2_clk_src = { + .cmd_rcgr = 0x17040, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_i2c_s3_clk_src = { + .cmd_rcgr = 0x1705c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_i2c_s4_clk_src = { + .cmd_rcgr = 0x17078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0), + F(250000000, P_GCC_GPLL7_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap1_qspi_ref_clk_src_init = { + .name = "gcc_qupv3_wrap1_qspi_ref_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_qspi_ref_clk_src = { + .cmd_rcgr = 0x188c0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_qspi_ref_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375), + F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = { + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x18014, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = { + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x18150, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = { + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x182a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s4_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375), + F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = { + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x183dc, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = { + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x18518, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s6_clk_src_init = { + .name = "gcc_qupv3_wrap1_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = { + .cmd_rcgr = 0x18654, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s7_clk_src_init = { + .name = "gcc_qupv3_wrap1_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = { + .cmd_rcgr = 0x18790, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s7_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s0_clk_src_init = { + .name = "gcc_qupv3_wrap2_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = { + .cmd_rcgr = 0x1e014, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s1_clk_src_init = { + .name = "gcc_qupv3_wrap2_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = { + .cmd_rcgr = 0x1e150, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s2_clk_src_init = { + .name = "gcc_qupv3_wrap2_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = { + .cmd_rcgr = 0x1e28c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s3_clk_src_init = { + .name = "gcc_qupv3_wrap2_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = { + .cmd_rcgr = 0x1e3c8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s4_clk_src_init = { + .name = "gcc_qupv3_wrap2_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = { + .cmd_rcgr = 0x1e504, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s4_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap3_ibi_ctrl_0_clk_src[] = { + F(37500000, P_GCC_GPLL0_OUT_EVEN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_qupv3_wrap3_ibi_ctrl_0_clk_src = { + .cmd_rcgr = 0xa877c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap3_ibi_ctrl_0_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_ibi_ctrl_0_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_init_data gcc_qupv3_wrap3_s0_clk_src_init = { + .name = "gcc_qupv3_wrap3_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap3_s0_clk_src = { + .cmd_rcgr = 0xa8014, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap3_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap3_s1_clk_src_init = { + .name = "gcc_qupv3_wrap3_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap3_s1_clk_src = { + .cmd_rcgr = 0xa8150, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap3_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap3_s2_clk_src_init = { + .name = "gcc_qupv3_wrap3_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap3_s2_clk_src = { + .cmd_rcgr = 0xa828c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap3_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap3_s3_clk_src_init = { + .name = "gcc_qupv3_wrap3_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap3_s3_clk_src = { + .cmd_rcgr = 0xa83c8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap3_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap3_s4_clk_src_init = { + .name = "gcc_qupv3_wrap3_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap3_s4_clk_src = { + .cmd_rcgr = 0xa8504, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap3_s4_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap3_s5_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375), + F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0), + F(128000000, P_GCC_GPLL0_OUT_MAIN, 1, 16, 75), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap3_s5_clk_src_init = { + .name = "gcc_qupv3_wrap3_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap3_s5_clk_src = { + .cmd_rcgr = 0xa8640, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap3_s5_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap3_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap4_s0_clk_src_init = { + .name = "gcc_qupv3_wrap4_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap4_s0_clk_src = { + .cmd_rcgr = 0xa9014, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap4_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap4_s1_clk_src_init = { + .name = "gcc_qupv3_wrap4_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap4_s1_clk_src = { + .cmd_rcgr = 0xa9150, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap4_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap4_s2_clk_src_init = { + .name = "gcc_qupv3_wrap4_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap4_s2_clk_src = { + .cmd_rcgr = 0xa928c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap4_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap4_s3_clk_src_init = { + .name = "gcc_qupv3_wrap4_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap4_s3_clk_src = { + .cmd_rcgr = 0xa93c8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap4_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap4_s4_clk_src_init = { + .name = "gcc_qupv3_wrap4_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap4_s4_clk_src = { + .cmd_rcgr = 0xa9504, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap4_s4_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1401c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_8), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1601c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = { + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(201500000, P_GCC_GPLL4_OUT_MAIN, 4, 0, 0), + F(403000000, P_GCC_GPLL4_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x77034, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = { + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(201500000, P_GCC_GPLL4_OUT_MAIN, 4, 0, 0), + F(403000000, P_GCC_GPLL4_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x7708c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_phy_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x770c0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_ufs_phy_phy_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x770a4, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = { + F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0x39034, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0x3904c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0x39078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_no_init_park_ops, + }, +}; + +static struct clk_regmap_div gcc_qupv3_wrap1_s2_clk_src = { + .reg = 0x1828c, + .shift = 0, + .width = 4, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = { + .reg = 0x39064, + .shift = 0, + .width = 4, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_axi_clk = { + .halt_reg = 0x10068, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x10068, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(12), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_aggre_noc_pcie_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x770f0, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770f0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770f0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0x39094, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x39094, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x39094, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x38004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x38004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(18), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0x26014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26014, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0x26028, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26028, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26028, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_pcie_anoc_ahb_clk = { + .halt_reg = 0x10050, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x10050, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(20), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_cfg_noc_pcie_anoc_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0x39090, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x39090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x39090, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cnoc_pcie_sf_axi_clk = { + .halt_reg = 0x10058, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(6), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_cnoc_pcie_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_pcie_sf_qtb_clk = { + .halt_reg = 0x1007c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x1007c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(19), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ddrss_pcie_sf_qtb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0x2701c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x2701c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0x27008, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x27008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27008, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_eva_axi0_clk = { + .halt_reg = 0x9f008, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x9f008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9f008, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_eva_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_eva_axi0c_clk = { + .halt_reg = 0x9f01c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x9f01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9f01c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_eva_axi0c_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x64000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x64000, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gp1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x65000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x65000, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gp2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x66000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x66000, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gp3_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gemnoc_gfx_clk = { + .halt_reg = 0x71010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x71010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x71010, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpu_gemnoc_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(15), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpu_gpll0_clk_src", + .parent_hws = (const struct clk_hw*[]) { + &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(16), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_hws = (const struct clk_hw*[]) { + &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x6b044, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(3), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_aux_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_pcie_0_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x6b040, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b040, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(2), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x6b030, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x6b030, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(1), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_phy_aux_clk = { + .halt_reg = 0x6b054, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(31), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_phy_aux_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_pcie_0_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_phy_rchng_clk = { + .halt_reg = 0x6b074, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(22), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_phy_rchng_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_pcie_0_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x6b064, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(4), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_pcie_0_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x6b020, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b020, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x6b01c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(5), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3300c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pdm2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x33004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x33004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x33004, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x33008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33008, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_cmd_ahb_clk = { + .halt_reg = 0x26010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x26010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26010, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_camera_cmd_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0x26008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x26008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26008, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0x2600c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2600c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2600c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_dcp_sf_ahb_clk = { + .halt_reg = 0x27030, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x27030, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27030, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_disp_dcp_sf_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_gpu_ahb_clk = { + .halt_reg = 0x71008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x71008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x71008, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_gpu_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_pcie_ahb_clk = { + .halt_reg = 0x6b018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(19), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_pcie_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cv_cpu_ahb_clk = { + .halt_reg = 0x32014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x32014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x32014, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_video_cv_cpu_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cvp_ahb_clk = { + .halt_reg = 0x32008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x32008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x32008, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_video_cvp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_v_cpu_ahb_clk = { + .halt_reg = 0x32010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x32010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x32010, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_video_v_cpu_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0x3200c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x3200c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x3200c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_i2c_core_clk = { + .halt_reg = 0x23004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(8), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_i2c_s0_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(10), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_i2c_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_i2c_s1_clk = { + .halt_reg = 0x17020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(11), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_i2c_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_i2c_s2_clk = { + .halt_reg = 0x1703c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(12), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_i2c_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_i2c_s3_clk = { + .halt_reg = 0x17058, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(13), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s3_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_i2c_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_i2c_s4_clk = { + .halt_reg = 0x17074, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(14), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s4_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_i2c_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_i2c_s_ahb_clk = { + .halt_reg = 0x23000, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x23000, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(7), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_i2c_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = { + .halt_reg = 0x2315c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(18), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_clk = { + .halt_reg = 0x23148, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(19), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_qspi_ref_clk = { + .halt_reg = 0x188bc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(29), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_qspi_ref_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x18004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(22), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x18140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(23), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x1827c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(24), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x18290, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(25), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x183cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(26), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x18508, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(27), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s6_clk = { + .halt_reg = 0x18644, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(28), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s6_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s7_clk = { + .halt_reg = 0x18780, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(16), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap1_s7_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap1_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_2x_clk = { + .halt_reg = 0x232b4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(3), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap2_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_clk = { + .halt_reg = 0x232a0, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap2_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s0_clk = { + .halt_reg = 0x1e004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(4), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap2_s0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap2_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s1_clk = { + .halt_reg = 0x1e140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(5), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap2_s1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap2_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s2_clk = { + .halt_reg = 0x1e27c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(6), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap2_s2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap2_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s3_clk = { + .halt_reg = 0x1e3b8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(7), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap2_s3_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap2_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s4_clk = { + .halt_reg = 0x1e4f4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(8), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap2_s4_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap2_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_core_2x_clk = { + .halt_reg = 0x2340c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(11), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_core_clk = { + .halt_reg = 0x233f8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(10), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_ibi_ctrl_1_clk = { + .halt_reg = 0xa8774, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa8774, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(20), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_ibi_ctrl_1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_ibi_ctrl_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_ibi_ctrl_2_clk = { + .halt_reg = 0xa8778, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa8778, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(21), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_ibi_ctrl_2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_ibi_ctrl_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_s0_clk = { + .halt_reg = 0xa8004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(12), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_s0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_s1_clk = { + .halt_reg = 0xa8140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(13), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_s1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_s2_clk = { + .halt_reg = 0xa827c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(14), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_s2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_s3_clk = { + .halt_reg = 0xa83b8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(15), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_s3_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_s4_clk = { + .halt_reg = 0xa84f4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(16), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_s4_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap3_s5_clk = { + .halt_reg = 0xa8630, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(17), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap3_s5_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap3_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap4_core_2x_clk = { + .halt_reg = 0x23564, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(25), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap4_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap4_core_clk = { + .halt_reg = 0x23550, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(24), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap4_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap4_s0_clk = { + .halt_reg = 0xa9004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(26), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap4_s0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap4_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap4_s1_clk = { + .halt_reg = 0xa9140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(27), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap4_s1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap4_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap4_s2_clk = { + .halt_reg = 0xa927c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(28), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap4_s2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap4_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap4_s3_clk = { + .halt_reg = 0xa93b8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(29), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap4_s3_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap4_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap4_s4_clk = { + .halt_reg = 0xa94f4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(30), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap4_s4_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_qupv3_wrap4_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_axi_clk = { + .halt_reg = 0x23140, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x23140, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(20), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_1_m_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x23144, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x23144, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(21), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = { + .halt_reg = 0x23298, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x23298, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(2), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_2_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = { + .halt_reg = 0x2329c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2329c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(1), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_2_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_3_ibi_1_ahb_clk = { + .halt_reg = 0xa876c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa876c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(18), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_3_ibi_1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_3_ibi_2_ahb_clk = { + .halt_reg = 0xa8770, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa8770, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(19), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_3_ibi_2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_3_m_ahb_clk = { + .halt_reg = 0x233f0, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x233f0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(8), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_3_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_3_s_ahb_clk = { + .halt_reg = 0x233f4, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x233f4, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(9), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_3_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_4_m_ahb_clk = { + .halt_reg = 0x23548, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x23548, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(22), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_4_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_4_s_ahb_clk = { + .halt_reg = 0x2354c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2354c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(23), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_qupv3_wrap_4_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x14014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14014, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x14004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14004, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_sdcc2_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x16014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16014, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_sdcc4_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_sdcc4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x77028, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77028, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77028, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x77018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77018, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x7707c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7707c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7707c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_ice_core_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x770bc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770bc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770bc, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x77030, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x77030, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x770d8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x770d8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x7702c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7702c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x7706c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7706c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7706c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0x39018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x39018, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb30_prim_master_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0x3902c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3902c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0x39028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x39028, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0x39068, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x39068, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0x3906c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3906c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0x39070, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0x39070, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x39070, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb3_prim_phy_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0x32018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x32018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x32018, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi1_clk = { + .halt_reg = 0x3202c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x3202c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x3202c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "gcc_video_axi1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc gcc_pcie_0_gdsc = { + .gdscr = 0x6b004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .collapse_ctrl = 0x5214c, + .collapse_mask = BIT(0), + .pd = { + .name = "gcc_pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE, +}; + +static struct gdsc gcc_pcie_0_phy_gdsc = { + .gdscr = 0x6c000, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, + .collapse_ctrl = 0x5214c, + .collapse_mask = BIT(2), + .pd = { + .name = "gcc_pcie_0_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE, +}; + +static struct gdsc gcc_ufs_mem_phy_gdsc = { + .gdscr = 0x9e000, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, + .pd = { + .name = "gcc_ufs_mem_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc gcc_ufs_phy_gdsc = { + .gdscr = 0x77004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "gcc_ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc gcc_usb30_prim_gdsc = { + .gdscr = 0x39004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "gcc_usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc gcc_usb3_phy_gdsc = { + .gdscr = 0x50018, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, + .pd = { + .name = "gcc_usb3_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct clk_regmap *gcc_kaanapali_clocks[] = { + [GCC_AGGRE_NOC_PCIE_AXI_CLK] = &gcc_aggre_noc_pcie_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_PCIE_ANOC_AHB_CLK] = &gcc_cfg_noc_pcie_anoc_ahb_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CNOC_PCIE_SF_AXI_CLK] = &gcc_cnoc_pcie_sf_axi_clk.clkr, + [GCC_DDRSS_PCIE_SF_QTB_CLK] = &gcc_ddrss_pcie_sf_qtb_clk.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_EVA_AXI0_CLK] = &gcc_eva_axi0_clk.clkr, + [GCC_EVA_AXI0C_CLK] = &gcc_eva_axi0c_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPLL0] = &gcc_gpll0.clkr, + [GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr, + [GCC_GPLL1] = &gcc_gpll1.clkr, + [GCC_GPLL4] = &gcc_gpll4.clkr, + [GCC_GPLL7] = &gcc_gpll7.clkr, + [GCC_GPLL9] = &gcc_gpll9.clkr, + [GCC_GPU_GEMNOC_GFX_CLK] = &gcc_gpu_gemnoc_gfx_clk.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PHY_AUX_CLK] = &gcc_pcie_0_phy_aux_clk.clkr, + [GCC_PCIE_0_PHY_AUX_CLK_SRC] = &gcc_pcie_0_phy_aux_clk_src.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK] = &gcc_pcie_0_phy_rchng_clk.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_QUPV3_I2C_CORE_CLK] = &gcc_qupv3_i2c_core_clk.clkr, + [GCC_QUPV3_I2C_S0_CLK] = &gcc_qupv3_i2c_s0_clk.clkr, + [GCC_QUPV3_I2C_S0_CLK_SRC] = &gcc_qupv3_i2c_s0_clk_src.clkr, + [GCC_QUPV3_I2C_S1_CLK] = &gcc_qupv3_i2c_s1_clk.clkr, + [GCC_QUPV3_I2C_S1_CLK_SRC] = &gcc_qupv3_i2c_s1_clk_src.clkr, + [GCC_QUPV3_I2C_S2_CLK] = &gcc_qupv3_i2c_s2_clk.clkr, + [GCC_QUPV3_I2C_S2_CLK_SRC] = &gcc_qupv3_i2c_s2_clk_src.clkr, + [GCC_QUPV3_I2C_S3_CLK] = &gcc_qupv3_i2c_s3_clk.clkr, + [GCC_QUPV3_I2C_S3_CLK_SRC] = &gcc_qupv3_i2c_s3_clk_src.clkr, + [GCC_QUPV3_I2C_S4_CLK] = &gcc_qupv3_i2c_s4_clk.clkr, + [GCC_QUPV3_I2C_S4_CLK_SRC] = &gcc_qupv3_i2c_s4_clk_src.clkr, + [GCC_QUPV3_I2C_S_AHB_CLK] = &gcc_qupv3_i2c_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr, + [GCC_QUPV3_WRAP1_QSPI_REF_CLK] = &gcc_qupv3_wrap1_qspi_ref_clk.clkr, + [GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC] = &gcc_qupv3_wrap1_qspi_ref_clk_src.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr, + [GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr, + [GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr, + [GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr, + [GCC_QUPV3_WRAP2_CORE_2X_CLK] = &gcc_qupv3_wrap2_core_2x_clk.clkr, + [GCC_QUPV3_WRAP2_CORE_CLK] = &gcc_qupv3_wrap2_core_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr, + [GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr, + [GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr, + [GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr, + [GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr, + [GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr, + [GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr, + [GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr, + [GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr, + [GCC_QUPV3_WRAP3_CORE_2X_CLK] = &gcc_qupv3_wrap3_core_2x_clk.clkr, + [GCC_QUPV3_WRAP3_CORE_CLK] = &gcc_qupv3_wrap3_core_clk.clkr, + [GCC_QUPV3_WRAP3_IBI_CTRL_0_CLK_SRC] = &gcc_qupv3_wrap3_ibi_ctrl_0_clk_src.clkr, + [GCC_QUPV3_WRAP3_IBI_CTRL_1_CLK] = &gcc_qupv3_wrap3_ibi_ctrl_1_clk.clkr, + [GCC_QUPV3_WRAP3_IBI_CTRL_2_CLK] = &gcc_qupv3_wrap3_ibi_ctrl_2_clk.clkr, + [GCC_QUPV3_WRAP3_S0_CLK] = &gcc_qupv3_wrap3_s0_clk.clkr, + [GCC_QUPV3_WRAP3_S0_CLK_SRC] = &gcc_qupv3_wrap3_s0_clk_src.clkr, + [GCC_QUPV3_WRAP3_S1_CLK] = &gcc_qupv3_wrap3_s1_clk.clkr, + [GCC_QUPV3_WRAP3_S1_CLK_SRC] = &gcc_qupv3_wrap3_s1_clk_src.clkr, + [GCC_QUPV3_WRAP3_S2_CLK] = &gcc_qupv3_wrap3_s2_clk.clkr, + [GCC_QUPV3_WRAP3_S2_CLK_SRC] = &gcc_qupv3_wrap3_s2_clk_src.clkr, + [GCC_QUPV3_WRAP3_S3_CLK] = &gcc_qupv3_wrap3_s3_clk.clkr, + [GCC_QUPV3_WRAP3_S3_CLK_SRC] = &gcc_qupv3_wrap3_s3_clk_src.clkr, + [GCC_QUPV3_WRAP3_S4_CLK] = &gcc_qupv3_wrap3_s4_clk.clkr, + [GCC_QUPV3_WRAP3_S4_CLK_SRC] = &gcc_qupv3_wrap3_s4_clk_src.clkr, + [GCC_QUPV3_WRAP3_S5_CLK] = &gcc_qupv3_wrap3_s5_clk.clkr, + [GCC_QUPV3_WRAP3_S5_CLK_SRC] = &gcc_qupv3_wrap3_s5_clk_src.clkr, + [GCC_QUPV3_WRAP4_CORE_2X_CLK] = &gcc_qupv3_wrap4_core_2x_clk.clkr, + [GCC_QUPV3_WRAP4_CORE_CLK] = &gcc_qupv3_wrap4_core_clk.clkr, + [GCC_QUPV3_WRAP4_S0_CLK] = &gcc_qupv3_wrap4_s0_clk.clkr, + [GCC_QUPV3_WRAP4_S0_CLK_SRC] = &gcc_qupv3_wrap4_s0_clk_src.clkr, + [GCC_QUPV3_WRAP4_S1_CLK] = &gcc_qupv3_wrap4_s1_clk.clkr, + [GCC_QUPV3_WRAP4_S1_CLK_SRC] = &gcc_qupv3_wrap4_s1_clk_src.clkr, + [GCC_QUPV3_WRAP4_S2_CLK] = &gcc_qupv3_wrap4_s2_clk.clkr, + [GCC_QUPV3_WRAP4_S2_CLK_SRC] = &gcc_qupv3_wrap4_s2_clk_src.clkr, + [GCC_QUPV3_WRAP4_S3_CLK] = &gcc_qupv3_wrap4_s3_clk.clkr, + [GCC_QUPV3_WRAP4_S3_CLK_SRC] = &gcc_qupv3_wrap4_s3_clk_src.clkr, + [GCC_QUPV3_WRAP4_S4_CLK] = &gcc_qupv3_wrap4_s4_clk.clkr, + [GCC_QUPV3_WRAP4_S4_CLK_SRC] = &gcc_qupv3_wrap4_s4_clk_src.clkr, + [GCC_QUPV3_WRAP_1_M_AXI_CLK] = &gcc_qupv3_wrap_1_m_axi_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_3_IBI_1_AHB_CLK] = &gcc_qupv3_wrap_3_ibi_1_ahb_clk.clkr, + [GCC_QUPV3_WRAP_3_IBI_2_AHB_CLK] = &gcc_qupv3_wrap_3_ibi_2_ahb_clk.clkr, + [GCC_QUPV3_WRAP_3_M_AHB_CLK] = &gcc_qupv3_wrap_3_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_3_S_AHB_CLK] = &gcc_qupv3_wrap_3_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_4_M_AHB_CLK] = &gcc_qupv3_wrap_4_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_4_S_AHB_CLK] = &gcc_qupv3_wrap_4_s_ahb_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr, + [GCC_QMIP_CAMERA_CMD_AHB_CLK] = &gcc_qmip_camera_cmd_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QMIP_GPU_AHB_CLK] = &gcc_qmip_gpu_ahb_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_DCP_SF_AHB_CLK] = &gcc_qmip_disp_dcp_sf_ahb_clk.clkr, + [GCC_QMIP_PCIE_AHB_CLK] = &gcc_qmip_pcie_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CV_CPU_AHB_CLK] = &gcc_qmip_video_cv_cpu_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CVP_AHB_CLK] = &gcc_qmip_video_cvp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_V_CPU_AHB_CLK] = &gcc_qmip_video_v_cpu_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, +}; + +static struct gdsc *gcc_kaanapali_gdscs[] = { + [GCC_PCIE_0_GDSC] = &gcc_pcie_0_gdsc, + [GCC_PCIE_0_PHY_GDSC] = &gcc_pcie_0_phy_gdsc, + [GCC_UFS_MEM_PHY_GDSC] = &gcc_ufs_mem_phy_gdsc, + [GCC_UFS_PHY_GDSC] = &gcc_ufs_phy_gdsc, + [GCC_USB30_PRIM_GDSC] = &gcc_usb30_prim_gdsc, + [GCC_USB3_PHY_GDSC] = &gcc_usb3_phy_gdsc, +}; + +static const struct qcom_reset_map gcc_kaanapali_resets[] = { + [GCC_CAMERA_BCR] = { 0x26000 }, + [GCC_DISPLAY_BCR] = { 0x27000 }, + [GCC_EVA_AXI0_CLK_ARES] = { 0x9f008, 2 }, + [GCC_EVA_AXI0C_CLK_ARES] = { 0x9f01c, 2 }, + [GCC_EVA_BCR] = { 0x9f000 }, + [GCC_GPU_BCR] = { 0x71000 }, + [GCC_PCIE_0_BCR] = { 0x6b000 }, + [GCC_PCIE_0_LINK_DOWN_BCR] = { 0x6c014 }, + [GCC_PCIE_0_NOCSR_COM_PHY_BCR] = { 0x6c020 }, + [GCC_PCIE_0_PHY_BCR] = { 0x6c01c }, + [GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR] = { 0x6c028 }, + [GCC_PCIE_PHY_BCR] = { 0x6f000 }, + [GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x6f00c }, + [GCC_PCIE_PHY_COM_BCR] = { 0x6f010 }, + [GCC_PCIE_RSCC_BCR] = { 0x11000 }, + [GCC_PDM_BCR] = { 0x33000 }, + [GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 }, + [GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 }, + [GCC_QUPV3_WRAPPER_3_BCR] = { 0xa8000 }, + [GCC_QUPV3_WRAPPER_4_BCR] = { 0xa9000 }, + [GCC_QUPV3_WRAPPER_I2C_BCR] = { 0x17000 }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x12004 }, + [GCC_SDCC2_BCR] = { 0x14000 }, + [GCC_SDCC4_BCR] = { 0x16000 }, + [GCC_UFS_PHY_BCR] = { 0x77000 }, + [GCC_USB30_PRIM_BCR] = { 0x39000 }, + [GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 }, + [GCC_USB3_DP_PHY_SEC_BCR] = { 0x50014 }, + [GCC_USB3_PHY_PRIM_BCR] = { 0x50000 }, + [GCC_USB3_PHY_SEC_BCR] = { 0x5000c }, + [GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 }, + [GCC_USB3PHY_PHY_SEC_BCR] = { 0x50010 }, + [GCC_VIDEO_AXI0_CLK_ARES] = { 0x32018, 2 }, + [GCC_VIDEO_AXI1_CLK_ARES] = { 0x3202c, 2 }, + [GCC_VIDEO_BCR] = { 0x32000 }, + [GCC_VIDEO_XO_CLK_ARES] = { 0x32040, 2 }, +}; + +static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { + DEFINE_RCG_DFS(gcc_qupv3_wrap1_qspi_ref_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s7_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap3_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap3_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap3_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap3_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap3_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap3_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap4_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap4_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap4_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap4_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap4_s4_clk_src), +}; + +static u32 gcc_kaanapali_critical_cbcrs[] = { + 0xa0004, /* GCC_CAM_BIST_MCLK_AHB_CLK */ + 0x26004, /* GCC_CAMERA_AHB_CLK */ + 0x2603c, /* GCC_CAMERA_XO_CLK */ + 0x27004, /* GCC_DISP_AHB_CLK */ + 0x9f004, /* GCC_EVA_AHB_CLK */ + 0x9f024, /* GCC_EVA_XO_CLK */ + 0x71004, /* GCC_GPU_CFG_AHB_CLK */ + 0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */ + 0x52010, /* GCC_PCIE_RSCC_XO_CLK */ + 0x32004, /* GCC_VIDEO_AHB_CLK */ + 0x32040, /* GCC_VIDEO_XO_CLK */ +}; + +static const struct regmap_config gcc_kaanapali_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1f41f0, + .fast_io = true, +}; + +static void clk_kaanapali_regs_configure(struct device *dev, struct regmap *regmap) +{ + /* FORCE_MEM_CORE_ON for ufs phy ice core clocks */ + qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true); +} + +static struct qcom_cc_driver_data gcc_kaanapali_driver_data = { + .clk_cbcrs = gcc_kaanapali_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(gcc_kaanapali_critical_cbcrs), + .dfs_rcgs = gcc_dfs_clocks, + .num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks), + .clk_regs_configure = clk_kaanapali_regs_configure, +}; + +static const struct qcom_cc_desc gcc_kaanapali_desc = { + .config = &gcc_kaanapali_regmap_config, + .clks = gcc_kaanapali_clocks, + .num_clks = ARRAY_SIZE(gcc_kaanapali_clocks), + .resets = gcc_kaanapali_resets, + .num_resets = ARRAY_SIZE(gcc_kaanapali_resets), + .gdscs = gcc_kaanapali_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_kaanapali_gdscs), + .driver_data = &gcc_kaanapali_driver_data, +}; + +static const struct of_device_id gcc_kaanapali_match_table[] = { + { .compatible = "qcom,kaanapali-gcc" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_kaanapali_match_table); + +static int gcc_kaanapali_probe(struct platform_device *pdev) +{ + return qcom_cc_probe(pdev, &gcc_kaanapali_desc); +} + +static struct platform_driver gcc_kaanapali_driver = { + .probe = gcc_kaanapali_probe, + .driver = { + .name = "gcc-kaanapali", + .of_match_table = gcc_kaanapali_match_table, + }, +}; + +static int __init gcc_kaanapali_init(void) +{ + return platform_driver_register(&gcc_kaanapali_driver); +} +subsys_initcall(gcc_kaanapali_init); + +static void __exit gcc_kaanapali_exit(void) +{ + platform_driver_unregister(&gcc_kaanapali_driver); +} +module_exit(gcc_kaanapali_exit); + +MODULE_DESCRIPTION("QTI GCC KAANAPALI Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/qcom/gcc-milos.c b/drivers/clk/qcom/gcc-milos.c index c9d61b05bafa1..81fa09ec55d7f 100644 --- a/drivers/clk/qcom/gcc-milos.c +++ b/drivers/clk/qcom/gcc-milos.c @@ -917,7 +917,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -938,7 +938,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .name = "gcc_sdcc1_ice_core_clk_src", .parent_data = gcc_parent_data_10, .num_parents = ARRAY_SIZE(gcc_parent_data_10), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -962,7 +962,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-msm8917.c b/drivers/clk/qcom/gcc-msm8917.c index 0a1aa623cd49a..9d1c5a9953e2c 100644 --- a/drivers/clk/qcom/gcc-msm8917.c +++ b/drivers/clk/qcom/gcc-msm8917.c @@ -3409,7 +3409,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-msm8953.c b/drivers/clk/qcom/gcc-msm8953.c index 8f29ecc74c50b..8fe1d3e421440 100644 --- a/drivers/clk/qcom/gcc-msm8953.c +++ b/drivers/clk/qcom/gcc-msm8953.c @@ -3946,7 +3946,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-qcs615.c b/drivers/clk/qcom/gcc-qcs615.c index 9695446bc2a3c..5b3b8dd4f114b 100644 --- a/drivers/clk/qcom/gcc-qcs615.c +++ b/drivers/clk/qcom/gcc-qcs615.c @@ -784,7 +784,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_1, .num_parents = ARRAY_SIZE(gcc_parent_data_1), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -806,7 +806,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .name = "gcc_sdcc1_ice_core_clk_src", .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -830,7 +830,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-qcs8300.c b/drivers/clk/qcom/gcc-qcs8300.c index 80831c7dea3bc..7a7f4eca40d8d 100644 --- a/drivers/clk/qcom/gcc-qcs8300.c +++ b/drivers/clk/qcom/gcc-qcs8300.c @@ -3306,7 +3306,7 @@ static struct gdsc gcc_usb20_prim_gdsc = { .pd = { .name = "gcc_usb20_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3318,7 +3318,7 @@ static struct gdsc gcc_usb30_prim_gdsc = { .pd = { .name = "gcc_usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; diff --git a/drivers/clk/qcom/gcc-qdu1000.c b/drivers/clk/qcom/gcc-qdu1000.c index dbe9e9437939a..915bb9b4ff813 100644 --- a/drivers/clk/qcom/gcc-qdu1000.c +++ b/drivers/clk/qcom/gcc-qdu1000.c @@ -904,7 +904,7 @@ static struct clk_rcg2 gcc_sdcc5_apps_clk_src = { .name = "gcc_sdcc5_apps_clk_src", .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -923,7 +923,7 @@ static struct clk_rcg2 gcc_sdcc5_ice_core_clk_src = { .name = "gcc_sdcc5_ice_core_clk_src", .parent_data = gcc_parent_data_2, .num_parents = ARRAY_SIZE(gcc_parent_data_2), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sa8775p.c b/drivers/clk/qcom/gcc-sa8775p.c index e7425e82c54f2..d7d1ba6ad6af3 100644 --- a/drivers/clk/qcom/gcc-sa8775p.c +++ b/drivers/clk/qcom/gcc-sa8775p.c @@ -4261,7 +4261,7 @@ static struct gdsc usb20_prim_gdsc = { .pd = { .name = "usb20_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -4273,7 +4273,7 @@ static struct gdsc usb30_prim_gdsc = { .pd = { .name = "usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -4285,7 +4285,7 @@ static struct gdsc usb30_sec_gdsc = { .pd = { .name = "usb30_sec_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; diff --git a/drivers/clk/qcom/gcc-sdx75.c b/drivers/clk/qcom/gcc-sdx75.c index 453a6bf8e8786..1f3cd58483a2d 100644 --- a/drivers/clk/qcom/gcc-sdx75.c +++ b/drivers/clk/qcom/gcc-sdx75.c @@ -1033,7 +1033,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_18, .num_parents = ARRAY_SIZE(gcc_parent_data_18), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm4450.c b/drivers/clk/qcom/gcc-sm4450.c index e2d9e4691c5b7..023d840e9f4ef 100644 --- a/drivers/clk/qcom/gcc-sm4450.c +++ b/drivers/clk/qcom/gcc-sm4450.c @@ -769,7 +769,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -791,7 +791,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -815,7 +815,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_6, .num_parents = ARRAY_SIZE(gcc_parent_data_6), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8450.c b/drivers/clk/qcom/gcc-sm8450.c index 65d7d52bce034..b18bb34889ab2 100644 --- a/drivers/clk/qcom/gcc-sm8450.c +++ b/drivers/clk/qcom/gcc-sm8450.c @@ -1034,7 +1034,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_7, .num_parents = ARRAY_SIZE(gcc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8550.c b/drivers/clk/qcom/gcc-sm8550.c index 862a9bf73bcb5..36a5b7de5b55d 100644 --- a/drivers/clk/qcom/gcc-sm8550.c +++ b/drivers/clk/qcom/gcc-sm8550.c @@ -1025,7 +1025,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1048,7 +1048,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8650.c b/drivers/clk/qcom/gcc-sm8650.c index 24f98062b9dd5..2dd6444ce0365 100644 --- a/drivers/clk/qcom/gcc-sm8650.c +++ b/drivers/clk/qcom/gcc-sm8650.c @@ -1257,7 +1257,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1279,7 +1279,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8750.c b/drivers/clk/qcom/gcc-sm8750.c index 8092dd6b37b56..db81569dd4b17 100644 --- a/drivers/clk/qcom/gcc-sm8750.c +++ b/drivers/clk/qcom/gcc-sm8750.c @@ -1012,6 +1012,7 @@ static struct clk_rcg2 gcc_qupv3_wrap2_s7_clk_src = { static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { F(400000, P_BI_TCXO, 12, 1, 4), F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(37500000, P_GCC_GPLL0_OUT_EVEN, 8, 0, 0), F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0), @@ -1029,7 +1030,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1051,7 +1052,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-x1e80100.c b/drivers/clk/qcom/gcc-x1e80100.c index 301fc9fc32d8e..47a143507d419 100644 --- a/drivers/clk/qcom/gcc-x1e80100.c +++ b/drivers/clk/qcom/gcc-x1e80100.c @@ -32,6 +32,33 @@ enum { DT_USB3_PHY_0_WRAPPER_GCC_USB30_PIPE, DT_USB3_PHY_1_WRAPPER_GCC_USB30_PIPE, DT_USB3_PHY_2_WRAPPER_GCC_USB30_PIPE, + DT_GCC_USB4_0_PHY_DP0_GMUX_CLK_SRC, + DT_GCC_USB4_0_PHY_DP1_GMUX_CLK_SRC, + DT_GCC_USB4_0_PHY_PCIE_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_0_PHY_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_0_PHY_SYS_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_1_PHY_DP0_GMUX_CLK_SRC, + DT_GCC_USB4_1_PHY_DP1_GMUX_CLK_SRC, + DT_GCC_USB4_1_PHY_PCIE_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_1_PHY_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_1_PHY_SYS_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_2_PHY_DP0_GMUX_CLK_SRC, + DT_GCC_USB4_2_PHY_DP1_GMUX_CLK_SRC, + DT_GCC_USB4_2_PHY_PCIE_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_2_PHY_PIPEGMUX_CLK_SRC, + DT_GCC_USB4_2_PHY_SYS_PIPEGMUX_CLK_SRC, + DT_QUSB4PHY_0_GCC_USB4_RX0_CLK, + DT_QUSB4PHY_0_GCC_USB4_RX1_CLK, + DT_QUSB4PHY_1_GCC_USB4_RX0_CLK, + DT_QUSB4PHY_1_GCC_USB4_RX1_CLK, + DT_QUSB4PHY_2_GCC_USB4_RX0_CLK, + DT_QUSB4PHY_2_GCC_USB4_RX1_CLK, + DT_USB4_0_PHY_GCC_USB4_PCIE_PIPE_CLK, + DT_USB4_0_PHY_GCC_USB4RTR_MAX_PIPE_CLK, + DT_USB4_1_PHY_GCC_USB4_PCIE_PIPE_CLK, + DT_USB4_1_PHY_GCC_USB4RTR_MAX_PIPE_CLK, + DT_USB4_2_PHY_GCC_USB4_PCIE_PIPE_CLK, + DT_USB4_2_PHY_GCC_USB4RTR_MAX_PIPE_CLK, }; enum { @@ -42,10 +69,40 @@ enum { P_GCC_GPLL7_OUT_MAIN, P_GCC_GPLL8_OUT_MAIN, P_GCC_GPLL9_OUT_MAIN, + P_GCC_USB3_PRIM_PHY_PIPE_CLK_SRC, + P_GCC_USB3_SEC_PHY_PIPE_CLK_SRC, + P_GCC_USB3_TERT_PHY_PIPE_CLK_SRC, + P_GCC_USB4_0_PHY_DP0_GMUX_CLK_SRC, + P_GCC_USB4_0_PHY_DP1_GMUX_CLK_SRC, + P_GCC_USB4_0_PHY_PCIE_PIPEGMUX_CLK_SRC, + P_GCC_USB4_0_PHY_PIPEGMUX_CLK_SRC, + P_GCC_USB4_0_PHY_SYS_PIPEGMUX_CLK_SRC, + P_GCC_USB4_1_PHY_DP0_GMUX_CLK_SRC, + P_GCC_USB4_1_PHY_DP1_GMUX_CLK_SRC, + P_GCC_USB4_1_PHY_PCIE_PIPEGMUX_CLK_SRC, + P_GCC_USB4_1_PHY_PIPEGMUX_CLK_SRC, + P_GCC_USB4_1_PHY_SYS_PIPEGMUX_CLK_SRC, + P_GCC_USB4_2_PHY_DP0_GMUX_CLK_SRC, + P_GCC_USB4_2_PHY_DP1_GMUX_CLK_SRC, + P_GCC_USB4_2_PHY_PCIE_PIPEGMUX_CLK_SRC, + P_GCC_USB4_2_PHY_PIPEGMUX_CLK_SRC, + P_GCC_USB4_2_PHY_SYS_PIPEGMUX_CLK_SRC, + P_QUSB4PHY_0_GCC_USB4_RX0_CLK, + P_QUSB4PHY_0_GCC_USB4_RX1_CLK, + P_QUSB4PHY_1_GCC_USB4_RX0_CLK, + P_QUSB4PHY_1_GCC_USB4_RX1_CLK, + P_QUSB4PHY_2_GCC_USB4_RX0_CLK, + P_QUSB4PHY_2_GCC_USB4_RX1_CLK, P_SLEEP_CLK, P_USB3_PHY_0_WRAPPER_GCC_USB30_PIPE_CLK, P_USB3_PHY_1_WRAPPER_GCC_USB30_PIPE_CLK, P_USB3_PHY_2_WRAPPER_GCC_USB30_PIPE_CLK, + P_USB4_0_PHY_GCC_USB4_PCIE_PIPE_CLK, + P_USB4_0_PHY_GCC_USB4RTR_MAX_PIPE_CLK, + P_USB4_1_PHY_GCC_USB4_PCIE_PIPE_CLK, + P_USB4_1_PHY_GCC_USB4RTR_MAX_PIPE_CLK, + P_USB4_2_PHY_GCC_USB4_PCIE_PIPE_CLK, + P_USB4_2_PHY_GCC_USB4RTR_MAX_PIPE_CLK, }; static struct clk_alpha_pll gcc_gpll0 = { @@ -320,6 +377,342 @@ static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { { } }; +static const struct clk_parent_data gcc_parent_data_13[] = { + { .index = DT_GCC_USB4_0_PHY_DP0_GMUX_CLK_SRC }, + { .index = DT_USB4_0_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_14[] = { + { .index = DT_GCC_USB4_0_PHY_DP1_GMUX_CLK_SRC }, + { .index = DT_USB4_0_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_15[] = { + { .index = DT_USB4_0_PHY_GCC_USB4_PCIE_PIPE_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_16[] = { + { .index = DT_GCC_USB4_0_PHY_PCIE_PIPEGMUX_CLK_SRC }, + { .index = DT_USB4_0_PHY_GCC_USB4_PCIE_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_17[] = { + { .index = DT_QUSB4PHY_0_GCC_USB4_RX0_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_18[] = { + { .index = DT_QUSB4PHY_0_GCC_USB4_RX1_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_19[] = { + { .index = DT_GCC_USB4_0_PHY_SYS_PIPEGMUX_CLK_SRC }, + { .index = DT_USB4_0_PHY_GCC_USB4_PCIE_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_20[] = { + { .index = DT_GCC_USB4_1_PHY_DP0_GMUX_CLK_SRC }, + { .index = DT_USB4_1_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_21[] = { + { .index = DT_GCC_USB4_1_PHY_DP1_GMUX_CLK_SRC }, + { .index = DT_USB4_1_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_22[] = { + { .index = DT_USB4_1_PHY_GCC_USB4_PCIE_PIPE_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_23[] = { + { .index = DT_GCC_USB4_1_PHY_PCIE_PIPEGMUX_CLK_SRC }, + { .index = DT_USB4_1_PHY_GCC_USB4_PCIE_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_24[] = { + { .index = DT_QUSB4PHY_1_GCC_USB4_RX0_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_25[] = { + { .index = DT_QUSB4PHY_1_GCC_USB4_RX1_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_26[] = { + { .index = DT_GCC_USB4_1_PHY_SYS_PIPEGMUX_CLK_SRC }, + { .index = DT_USB4_1_PHY_GCC_USB4_PCIE_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_27[] = { + { .index = DT_GCC_USB4_2_PHY_DP0_GMUX_CLK_SRC }, + { .index = DT_USB4_2_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_28[] = { + { .index = DT_GCC_USB4_2_PHY_DP1_GMUX_CLK_SRC }, + { .index = DT_USB4_2_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_29[] = { + { .index = DT_USB4_2_PHY_GCC_USB4_PCIE_PIPE_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_30[] = { + { .index = DT_GCC_USB4_2_PHY_PCIE_PIPEGMUX_CLK_SRC }, + { .index = DT_USB4_2_PHY_GCC_USB4_PCIE_PIPE_CLK }, +}; + +static const struct clk_parent_data gcc_parent_data_31[] = { + { .index = DT_QUSB4PHY_2_GCC_USB4_RX0_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_32[] = { + { .index = DT_QUSB4PHY_2_GCC_USB4_RX1_CLK }, + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data gcc_parent_data_33[] = { + { .index = DT_GCC_USB4_2_PHY_SYS_PIPEGMUX_CLK_SRC }, + { .index = DT_USB4_2_PHY_GCC_USB4_PCIE_PIPE_CLK }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_0_phy_dp0_clk_src = { + .reg = 0x9f06c, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_0_phy_dp0_clk_src", + .parent_data = gcc_parent_data_13, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_0_phy_dp1_clk_src = { + .reg = 0x9f114, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_0_phy_dp1_clk_src", + .parent_data = gcc_parent_data_14, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_0_phy_p2rr2p_pipe_clk_src = { + .reg = 0x9f0d4, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_0_phy_p2rr2p_pipe_clk_src", + .parent_data = gcc_parent_data_15, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_0_phy_pcie_pipe_mux_clk_src = { + .reg = 0x9f104, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_0_phy_pcie_pipe_mux_clk_src", + .parent_data = gcc_parent_data_16, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_0_phy_rx0_clk_src = { + .reg = 0x9f0ac, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_0_phy_rx0_clk_src", + .parent_data = gcc_parent_data_17, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_0_phy_rx1_clk_src = { + .reg = 0x9f0bc, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_0_phy_rx1_clk_src", + .parent_data = gcc_parent_data_18, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_0_phy_sys_clk_src = { + .reg = 0x9f0e4, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_0_phy_sys_clk_src", + .parent_data = gcc_parent_data_19, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_1_phy_dp0_clk_src = { + .reg = 0x2b06c, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_1_phy_dp0_clk_src", + .parent_data = gcc_parent_data_20, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_1_phy_dp1_clk_src = { + .reg = 0x2b114, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_1_phy_dp1_clk_src", + .parent_data = gcc_parent_data_21, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_1_phy_p2rr2p_pipe_clk_src = { + .reg = 0x2b0d4, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_1_phy_p2rr2p_pipe_clk_src", + .parent_data = gcc_parent_data_22, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_1_phy_pcie_pipe_mux_clk_src = { + .reg = 0x2b104, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_1_phy_pcie_pipe_mux_clk_src", + .parent_data = gcc_parent_data_23, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_1_phy_rx0_clk_src = { + .reg = 0x2b0ac, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_1_phy_rx0_clk_src", + .parent_data = gcc_parent_data_24, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_1_phy_rx1_clk_src = { + .reg = 0x2b0bc, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_1_phy_rx1_clk_src", + .parent_data = gcc_parent_data_25, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_1_phy_sys_clk_src = { + .reg = 0x2b0e4, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_1_phy_sys_clk_src", + .parent_data = gcc_parent_data_26, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_2_phy_dp0_clk_src = { + .reg = 0x1106c, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_2_phy_dp0_clk_src", + .parent_data = gcc_parent_data_27, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_2_phy_dp1_clk_src = { + .reg = 0x11114, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_2_phy_dp1_clk_src", + .parent_data = gcc_parent_data_28, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_2_phy_p2rr2p_pipe_clk_src = { + .reg = 0x110d4, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_2_phy_p2rr2p_pipe_clk_src", + .parent_data = gcc_parent_data_29, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_2_phy_pcie_pipe_mux_clk_src = { + .reg = 0x11104, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_2_phy_pcie_pipe_mux_clk_src", + .parent_data = gcc_parent_data_30, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_2_phy_rx0_clk_src = { + .reg = 0x110ac, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_2_phy_rx0_clk_src", + .parent_data = gcc_parent_data_31, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_2_phy_rx1_clk_src = { + .reg = 0x110bc, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_2_phy_rx1_clk_src", + .parent_data = gcc_parent_data_32, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + +static struct clk_regmap_phy_mux gcc_usb4_2_phy_sys_clk_src = { + .reg = 0x110e4, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb4_2_phy_sys_clk_src", + .parent_data = gcc_parent_data_33, + .ops = &clk_regmap_phy_mux_ops, + }, + }, +}; + static struct clk_rcg2 gcc_gp1_clk_src = { .cmd_rcgr = 0x64004, .mnd_width = 16, @@ -1123,7 +1516,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1145,7 +1538,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -2790,6 +3183,11 @@ static struct clk_branch gcc_pcie_0_pipe_clk = { .enable_mask = BIT(25), .hw.init = &(const struct clk_init_data) { .name = "gcc_pcie_0_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_pcie_pipe_mux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2879,6 +3277,11 @@ static struct clk_branch gcc_pcie_1_pipe_clk = { .enable_mask = BIT(30), .hw.init = &(const struct clk_init_data) { .name = "gcc_pcie_1_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_pcie_pipe_mux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2968,6 +3371,11 @@ static struct clk_branch gcc_pcie_2_pipe_clk = { .enable_mask = BIT(23), .hw.init = &(const struct clk_init_data) { .name = "gcc_pcie_2_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_2_phy_pcie_pipe_mux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -4741,7 +5149,7 @@ static struct clk_branch gcc_ufs_phy_phy_aux_clk = { static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { .halt_reg = 0x7702c, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_DELAY, .clkr = { .enable_reg = 0x7702c, .enable_mask = BIT(0), @@ -4754,7 +5162,7 @@ static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { .halt_reg = 0x770cc, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_DELAY, .clkr = { .enable_reg = 0x770cc, .enable_mask = BIT(0), @@ -4767,7 +5175,7 @@ static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { .halt_reg = 0x77028, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_DELAY, .clkr = { .enable_reg = 0x77028, .enable_mask = BIT(0), @@ -5156,6 +5564,33 @@ static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = { }, }; +static const struct parent_map gcc_parent_map_34[] = { + { P_GCC_USB3_PRIM_PHY_PIPE_CLK_SRC, 0 }, + { P_USB4_0_PHY_GCC_USB4RTR_MAX_PIPE_CLK, 1 }, + { P_GCC_USB4_0_PHY_PIPEGMUX_CLK_SRC, 3 }, +}; + +static const struct clk_parent_data gcc_parent_data_34[] = { + { .hw = &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw }, + { .index = DT_USB4_0_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, + { .index = DT_GCC_USB4_0_PHY_PIPEGMUX_CLK_SRC }, +}; + +static struct clk_regmap_mux gcc_usb34_prim_phy_pipe_clk_src = { + .reg = 0x39070, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_34, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb34_prim_phy_pipe_clk_src", + .parent_data = gcc_parent_data_34, + .num_parents = ARRAY_SIZE(gcc_parent_data_34), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { .halt_reg = 0x39068, .halt_check = BRANCH_HALT_SKIP, @@ -5167,7 +5602,7 @@ static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { .hw.init = &(const struct clk_init_data) { .name = "gcc_usb3_prim_phy_pipe_clk", .parent_hws = (const struct clk_hw*[]) { - &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw, + &gcc_usb34_prim_phy_pipe_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -5227,6 +5662,33 @@ static struct clk_regmap_mux gcc_usb3_sec_phy_pipe_clk_src = { }, }; +static const struct parent_map gcc_parent_map_35[] = { + { P_GCC_USB3_SEC_PHY_PIPE_CLK_SRC, 0 }, + { P_USB4_1_PHY_GCC_USB4RTR_MAX_PIPE_CLK, 1 }, + { P_GCC_USB4_1_PHY_PIPEGMUX_CLK_SRC, 3 }, +}; + +static const struct clk_parent_data gcc_parent_data_35[] = { + { .hw = &gcc_usb3_sec_phy_pipe_clk_src.clkr.hw }, + { .index = DT_USB4_1_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, + { .index = DT_GCC_USB4_1_PHY_PIPEGMUX_CLK_SRC }, +}; + +static struct clk_regmap_mux gcc_usb34_sec_phy_pipe_clk_src = { + .reg = 0xa1070, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_35, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb34_sec_phy_pipe_clk_src", + .parent_data = gcc_parent_data_35, + .num_parents = ARRAY_SIZE(gcc_parent_data_35), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { .halt_reg = 0xa1068, .halt_check = BRANCH_HALT_SKIP, @@ -5238,7 +5700,7 @@ static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { .hw.init = &(const struct clk_init_data) { .name = "gcc_usb3_sec_phy_pipe_clk", .parent_hws = (const struct clk_hw*[]) { - &gcc_usb3_sec_phy_pipe_clk_src.clkr.hw, + &gcc_usb34_sec_phy_pipe_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -5298,6 +5760,33 @@ static struct clk_regmap_mux gcc_usb3_tert_phy_pipe_clk_src = { }, }; +static const struct parent_map gcc_parent_map_36[] = { + { P_GCC_USB3_TERT_PHY_PIPE_CLK_SRC, 0 }, + { P_USB4_2_PHY_GCC_USB4RTR_MAX_PIPE_CLK, 1 }, + { P_GCC_USB4_2_PHY_PIPEGMUX_CLK_SRC, 3 }, +}; + +static const struct clk_parent_data gcc_parent_data_36[] = { + { .hw = &gcc_usb3_tert_phy_pipe_clk_src.clkr.hw }, + { .index = DT_USB4_2_PHY_GCC_USB4RTR_MAX_PIPE_CLK }, + { .index = DT_GCC_USB4_2_PHY_PIPEGMUX_CLK_SRC }, +}; + +static struct clk_regmap_mux gcc_usb34_tert_phy_pipe_clk_src = { + .reg = 0xa2070, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_36, + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "gcc_usb34_tert_phy_pipe_clk_src", + .parent_data = gcc_parent_data_36, + .num_parents = ARRAY_SIZE(gcc_parent_data_36), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + static struct clk_branch gcc_usb3_tert_phy_pipe_clk = { .halt_reg = 0xa2068, .halt_check = BRANCH_HALT_SKIP, @@ -5309,7 +5798,7 @@ static struct clk_branch gcc_usb3_tert_phy_pipe_clk = { .hw.init = &(const struct clk_init_data) { .name = "gcc_usb3_tert_phy_pipe_clk", .parent_hws = (const struct clk_hw*[]) { - &gcc_usb3_tert_phy_pipe_clk_src.clkr.hw, + &gcc_usb34_tert_phy_pipe_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -5335,12 +5824,17 @@ static struct clk_branch gcc_usb4_0_cfg_ahb_clk = { static struct clk_branch gcc_usb4_0_dp0_clk = { .halt_reg = 0x9f060, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9f060, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_dp0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_dp0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5348,12 +5842,17 @@ static struct clk_branch gcc_usb4_0_dp0_clk = { static struct clk_branch gcc_usb4_0_dp1_clk = { .halt_reg = 0x9f108, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9f108, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_dp1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_dp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5385,6 +5884,11 @@ static struct clk_branch gcc_usb4_0_phy_p2rr2p_pipe_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_phy_p2rr2p_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_p2rr2p_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5398,6 +5902,11 @@ static struct clk_branch gcc_usb4_0_phy_pcie_pipe_clk = { .enable_mask = BIT(19), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_phy_pcie_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_pcie_pipe_mux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5405,12 +5914,17 @@ static struct clk_branch gcc_usb4_0_phy_pcie_pipe_clk = { static struct clk_branch gcc_usb4_0_phy_rx0_clk = { .halt_reg = 0x9f0b0, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9f0b0, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_phy_rx0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_rx0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5418,12 +5932,17 @@ static struct clk_branch gcc_usb4_0_phy_rx0_clk = { static struct clk_branch gcc_usb4_0_phy_rx1_clk = { .halt_reg = 0x9f0c0, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9f0c0, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_phy_rx1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_rx1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5439,6 +5958,11 @@ static struct clk_branch gcc_usb4_0_phy_usb_pipe_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_phy_usb_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb34_prim_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5470,6 +5994,11 @@ static struct clk_branch gcc_usb4_0_sys_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_0_sys_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_0_phy_sys_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5512,12 +6041,17 @@ static struct clk_branch gcc_usb4_1_cfg_ahb_clk = { static struct clk_branch gcc_usb4_1_dp0_clk = { .halt_reg = 0x2b060, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x2b060, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_dp0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_dp0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5525,12 +6059,17 @@ static struct clk_branch gcc_usb4_1_dp0_clk = { static struct clk_branch gcc_usb4_1_dp1_clk = { .halt_reg = 0x2b108, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x2b108, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_dp1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_dp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5562,6 +6101,11 @@ static struct clk_branch gcc_usb4_1_phy_p2rr2p_pipe_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_phy_p2rr2p_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_p2rr2p_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5575,6 +6119,11 @@ static struct clk_branch gcc_usb4_1_phy_pcie_pipe_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_phy_pcie_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_pcie_pipe_mux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5582,12 +6131,17 @@ static struct clk_branch gcc_usb4_1_phy_pcie_pipe_clk = { static struct clk_branch gcc_usb4_1_phy_rx0_clk = { .halt_reg = 0x2b0b0, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x2b0b0, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_phy_rx0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_rx0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5595,12 +6149,17 @@ static struct clk_branch gcc_usb4_1_phy_rx0_clk = { static struct clk_branch gcc_usb4_1_phy_rx1_clk = { .halt_reg = 0x2b0c0, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x2b0c0, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_phy_rx1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_rx1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5616,6 +6175,11 @@ static struct clk_branch gcc_usb4_1_phy_usb_pipe_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_phy_usb_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb34_sec_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5647,6 +6211,11 @@ static struct clk_branch gcc_usb4_1_sys_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_1_sys_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_1_phy_sys_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5689,12 +6258,17 @@ static struct clk_branch gcc_usb4_2_cfg_ahb_clk = { static struct clk_branch gcc_usb4_2_dp0_clk = { .halt_reg = 0x11060, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x11060, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_2_dp0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_2_phy_dp0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5702,12 +6276,17 @@ static struct clk_branch gcc_usb4_2_dp0_clk = { static struct clk_branch gcc_usb4_2_dp1_clk = { .halt_reg = 0x11108, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x11108, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_2_dp1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_2_phy_dp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5739,6 +6318,11 @@ static struct clk_branch gcc_usb4_2_phy_p2rr2p_pipe_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_2_phy_p2rr2p_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_2_phy_p2rr2p_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5752,6 +6336,11 @@ static struct clk_branch gcc_usb4_2_phy_pcie_pipe_clk = { .enable_mask = BIT(1), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_2_phy_pcie_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_2_phy_pcie_pipe_mux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5759,12 +6348,17 @@ static struct clk_branch gcc_usb4_2_phy_pcie_pipe_clk = { static struct clk_branch gcc_usb4_2_phy_rx0_clk = { .halt_reg = 0x110b0, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x110b0, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_2_phy_rx0_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_2_phy_rx0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5772,12 +6366,17 @@ static struct clk_branch gcc_usb4_2_phy_rx0_clk = { static struct clk_branch gcc_usb4_2_phy_rx1_clk = { .halt_reg = 0x110c0, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x110c0, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_2_phy_rx1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb4_2_phy_rx1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -5793,6 +6392,11 @@ static struct clk_branch gcc_usb4_2_phy_usb_pipe_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_usb4_2_phy_usb_pipe_clk", + .parent_hws = (const struct clk_hw*[]) { + &gcc_usb34_tert_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -6483,6 +7087,9 @@ static struct clk_regmap *gcc_x1e80100_clocks[] = { [GCC_USB30_TERT_MOCK_UTMI_CLK_SRC] = &gcc_usb30_tert_mock_utmi_clk_src.clkr, [GCC_USB30_TERT_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_tert_mock_utmi_postdiv_clk_src.clkr, [GCC_USB30_TERT_SLEEP_CLK] = &gcc_usb30_tert_sleep_clk.clkr, + [GCC_USB34_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb34_prim_phy_pipe_clk_src.clkr, + [GCC_USB34_SEC_PHY_PIPE_CLK_SRC] = &gcc_usb34_sec_phy_pipe_clk_src.clkr, + [GCC_USB34_TERT_PHY_PIPE_CLK_SRC] = &gcc_usb34_tert_phy_pipe_clk_src.clkr, [GCC_USB3_MP_PHY_AUX_CLK] = &gcc_usb3_mp_phy_aux_clk.clkr, [GCC_USB3_MP_PHY_AUX_CLK_SRC] = &gcc_usb3_mp_phy_aux_clk_src.clkr, [GCC_USB3_MP_PHY_COM_AUX_CLK] = &gcc_usb3_mp_phy_com_aux_clk.clkr, @@ -6508,11 +7115,18 @@ static struct clk_regmap *gcc_x1e80100_clocks[] = { [GCC_USB4_0_DP1_CLK] = &gcc_usb4_0_dp1_clk.clkr, [GCC_USB4_0_MASTER_CLK] = &gcc_usb4_0_master_clk.clkr, [GCC_USB4_0_MASTER_CLK_SRC] = &gcc_usb4_0_master_clk_src.clkr, + [GCC_USB4_0_PHY_DP0_CLK_SRC] = &gcc_usb4_0_phy_dp0_clk_src.clkr, + [GCC_USB4_0_PHY_DP1_CLK_SRC] = &gcc_usb4_0_phy_dp1_clk_src.clkr, [GCC_USB4_0_PHY_P2RR2P_PIPE_CLK] = &gcc_usb4_0_phy_p2rr2p_pipe_clk.clkr, + [GCC_USB4_0_PHY_P2RR2P_PIPE_CLK_SRC] = &gcc_usb4_0_phy_p2rr2p_pipe_clk_src.clkr, [GCC_USB4_0_PHY_PCIE_PIPE_CLK] = &gcc_usb4_0_phy_pcie_pipe_clk.clkr, [GCC_USB4_0_PHY_PCIE_PIPE_CLK_SRC] = &gcc_usb4_0_phy_pcie_pipe_clk_src.clkr, + [GCC_USB4_0_PHY_PCIE_PIPE_MUX_CLK_SRC] = &gcc_usb4_0_phy_pcie_pipe_mux_clk_src.clkr, [GCC_USB4_0_PHY_RX0_CLK] = &gcc_usb4_0_phy_rx0_clk.clkr, + [GCC_USB4_0_PHY_RX0_CLK_SRC] = &gcc_usb4_0_phy_rx0_clk_src.clkr, [GCC_USB4_0_PHY_RX1_CLK] = &gcc_usb4_0_phy_rx1_clk.clkr, + [GCC_USB4_0_PHY_RX1_CLK_SRC] = &gcc_usb4_0_phy_rx1_clk_src.clkr, + [GCC_USB4_0_PHY_SYS_CLK_SRC] = &gcc_usb4_0_phy_sys_clk_src.clkr, [GCC_USB4_0_PHY_USB_PIPE_CLK] = &gcc_usb4_0_phy_usb_pipe_clk.clkr, [GCC_USB4_0_SB_IF_CLK] = &gcc_usb4_0_sb_if_clk.clkr, [GCC_USB4_0_SB_IF_CLK_SRC] = &gcc_usb4_0_sb_if_clk_src.clkr, @@ -6524,11 +7138,18 @@ static struct clk_regmap *gcc_x1e80100_clocks[] = { [GCC_USB4_1_DP1_CLK] = &gcc_usb4_1_dp1_clk.clkr, [GCC_USB4_1_MASTER_CLK] = &gcc_usb4_1_master_clk.clkr, [GCC_USB4_1_MASTER_CLK_SRC] = &gcc_usb4_1_master_clk_src.clkr, + [GCC_USB4_1_PHY_DP0_CLK_SRC] = &gcc_usb4_1_phy_dp0_clk_src.clkr, + [GCC_USB4_1_PHY_DP1_CLK_SRC] = &gcc_usb4_1_phy_dp1_clk_src.clkr, [GCC_USB4_1_PHY_P2RR2P_PIPE_CLK] = &gcc_usb4_1_phy_p2rr2p_pipe_clk.clkr, + [GCC_USB4_1_PHY_P2RR2P_PIPE_CLK_SRC] = &gcc_usb4_1_phy_p2rr2p_pipe_clk_src.clkr, [GCC_USB4_1_PHY_PCIE_PIPE_CLK] = &gcc_usb4_1_phy_pcie_pipe_clk.clkr, [GCC_USB4_1_PHY_PCIE_PIPE_CLK_SRC] = &gcc_usb4_1_phy_pcie_pipe_clk_src.clkr, + [GCC_USB4_1_PHY_PCIE_PIPE_MUX_CLK_SRC] = &gcc_usb4_1_phy_pcie_pipe_mux_clk_src.clkr, [GCC_USB4_1_PHY_RX0_CLK] = &gcc_usb4_1_phy_rx0_clk.clkr, + [GCC_USB4_1_PHY_RX0_CLK_SRC] = &gcc_usb4_1_phy_rx0_clk_src.clkr, [GCC_USB4_1_PHY_RX1_CLK] = &gcc_usb4_1_phy_rx1_clk.clkr, + [GCC_USB4_1_PHY_RX1_CLK_SRC] = &gcc_usb4_1_phy_rx1_clk_src.clkr, + [GCC_USB4_1_PHY_SYS_CLK_SRC] = &gcc_usb4_1_phy_sys_clk_src.clkr, [GCC_USB4_1_PHY_USB_PIPE_CLK] = &gcc_usb4_1_phy_usb_pipe_clk.clkr, [GCC_USB4_1_SB_IF_CLK] = &gcc_usb4_1_sb_if_clk.clkr, [GCC_USB4_1_SB_IF_CLK_SRC] = &gcc_usb4_1_sb_if_clk_src.clkr, @@ -6540,11 +7161,18 @@ static struct clk_regmap *gcc_x1e80100_clocks[] = { [GCC_USB4_2_DP1_CLK] = &gcc_usb4_2_dp1_clk.clkr, [GCC_USB4_2_MASTER_CLK] = &gcc_usb4_2_master_clk.clkr, [GCC_USB4_2_MASTER_CLK_SRC] = &gcc_usb4_2_master_clk_src.clkr, + [GCC_USB4_2_PHY_DP0_CLK_SRC] = &gcc_usb4_2_phy_dp0_clk_src.clkr, + [GCC_USB4_2_PHY_DP1_CLK_SRC] = &gcc_usb4_2_phy_dp1_clk_src.clkr, [GCC_USB4_2_PHY_P2RR2P_PIPE_CLK] = &gcc_usb4_2_phy_p2rr2p_pipe_clk.clkr, + [GCC_USB4_2_PHY_P2RR2P_PIPE_CLK_SRC] = &gcc_usb4_2_phy_p2rr2p_pipe_clk_src.clkr, [GCC_USB4_2_PHY_PCIE_PIPE_CLK] = &gcc_usb4_2_phy_pcie_pipe_clk.clkr, [GCC_USB4_2_PHY_PCIE_PIPE_CLK_SRC] = &gcc_usb4_2_phy_pcie_pipe_clk_src.clkr, + [GCC_USB4_2_PHY_PCIE_PIPE_MUX_CLK_SRC] = &gcc_usb4_2_phy_pcie_pipe_mux_clk_src.clkr, [GCC_USB4_2_PHY_RX0_CLK] = &gcc_usb4_2_phy_rx0_clk.clkr, + [GCC_USB4_2_PHY_RX0_CLK_SRC] = &gcc_usb4_2_phy_rx0_clk_src.clkr, [GCC_USB4_2_PHY_RX1_CLK] = &gcc_usb4_2_phy_rx1_clk.clkr, + [GCC_USB4_2_PHY_RX1_CLK_SRC] = &gcc_usb4_2_phy_rx1_clk_src.clkr, + [GCC_USB4_2_PHY_SYS_CLK_SRC] = &gcc_usb4_2_phy_sys_clk_src.clkr, [GCC_USB4_2_PHY_USB_PIPE_CLK] = &gcc_usb4_2_phy_usb_pipe_clk.clkr, [GCC_USB4_2_SB_IF_CLK] = &gcc_usb4_2_sb_if_clk.clkr, [GCC_USB4_2_SB_IF_CLK_SRC] = &gcc_usb4_2_sb_if_clk_src.clkr, @@ -6660,16 +7288,52 @@ static const struct qcom_reset_map gcc_x1e80100_resets[] = { [GCC_USB3_UNIPHY_MP0_BCR] = { 0x19000 }, [GCC_USB3_UNIPHY_MP1_BCR] = { 0x54000 }, [GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 }, + [GCC_USB4PHY_PHY_PRIM_BCR] = { 0x5000c }, [GCC_USB3PHY_PHY_SEC_BCR] = { 0x2a004 }, + [GCC_USB4PHY_PHY_SEC_BCR] = { 0x2a00c }, [GCC_USB3PHY_PHY_TERT_BCR] = { 0xa3004 }, + [GCC_USB4PHY_PHY_TERT_BCR] = { 0xa300c }, [GCC_USB3UNIPHY_PHY_MP0_BCR] = { 0x19004 }, [GCC_USB3UNIPHY_PHY_MP1_BCR] = { 0x54004 }, [GCC_USB4_0_BCR] = { 0x9f000 }, [GCC_USB4_0_DP0_PHY_PRIM_BCR] = { 0x50010 }, - [GCC_USB4_1_DP0_PHY_SEC_BCR] = { 0x2a010 }, - [GCC_USB4_2_DP0_PHY_TERT_BCR] = { 0xa3010 }, + [GCC_USB4_0_MISC_USB4_SYS_BCR] = { .reg = 0xad0f8, .bit = 0 }, + [GCC_USB4_0_MISC_RX_CLK_0_BCR] = { .reg = 0xad0f8, .bit = 1 }, + [GCC_USB4_0_MISC_RX_CLK_1_BCR] = { .reg = 0xad0f8, .bit = 2 }, + [GCC_USB4_0_MISC_USB_PIPE_BCR] = { .reg = 0xad0f8, .bit = 3 }, + [GCC_USB4_0_MISC_PCIE_PIPE_BCR] = { .reg = 0xad0f8, .bit = 4 }, + [GCC_USB4_0_MISC_TMU_BCR] = { .reg = 0xad0f8, .bit = 5 }, + [GCC_USB4_0_MISC_SB_IF_BCR] = { .reg = 0xad0f8, .bit = 6 }, + [GCC_USB4_0_MISC_HIA_MSTR_BCR] = { .reg = 0xad0f8, .bit = 7 }, + [GCC_USB4_0_MISC_AHB_BCR] = { .reg = 0xad0f8, .bit = 8 }, + [GCC_USB4_0_MISC_DP0_MAX_PCLK_BCR] = { .reg = 0xad0f8, .bit = 9 }, + [GCC_USB4_0_MISC_DP1_MAX_PCLK_BCR] = { .reg = 0xad0f8, .bit = 10 }, [GCC_USB4_1_BCR] = { 0x2b000 }, + [GCC_USB4_1_DP0_PHY_SEC_BCR] = { 0x2a010 }, + [GCC_USB4_1_MISC_USB4_SYS_BCR] = { .reg = 0xae0f8, .bit = 0 }, + [GCC_USB4_1_MISC_RX_CLK_0_BCR] = { .reg = 0xae0f8, .bit = 1 }, + [GCC_USB4_1_MISC_RX_CLK_1_BCR] = { .reg = 0xae0f8, .bit = 2 }, + [GCC_USB4_1_MISC_USB_PIPE_BCR] = { .reg = 0xae0f8, .bit = 3 }, + [GCC_USB4_1_MISC_PCIE_PIPE_BCR] = { .reg = 0xae0f8, .bit = 4 }, + [GCC_USB4_1_MISC_TMU_BCR] = { .reg = 0xae0f8, .bit = 5 }, + [GCC_USB4_1_MISC_SB_IF_BCR] = { .reg = 0xae0f8, .bit = 6 }, + [GCC_USB4_1_MISC_HIA_MSTR_BCR] = { .reg = 0xae0f8, .bit = 7 }, + [GCC_USB4_1_MISC_AHB_BCR] = { .reg = 0xae0f8, .bit = 8 }, + [GCC_USB4_1_MISC_DP0_MAX_PCLK_BCR] = { .reg = 0xae0f8, .bit = 9 }, + [GCC_USB4_1_MISC_DP1_MAX_PCLK_BCR] = { .reg = 0xae0f8, .bit = 10 }, [GCC_USB4_2_BCR] = { 0x11000 }, + [GCC_USB4_2_DP0_PHY_TERT_BCR] = { 0xa3010 }, + [GCC_USB4_2_MISC_USB4_SYS_BCR] = { .reg = 0xaf0f8, .bit = 0 }, + [GCC_USB4_2_MISC_RX_CLK_0_BCR] = { .reg = 0xaf0f8, .bit = 1 }, + [GCC_USB4_2_MISC_RX_CLK_1_BCR] = { .reg = 0xaf0f8, .bit = 2 }, + [GCC_USB4_2_MISC_USB_PIPE_BCR] = { .reg = 0xaf0f8, .bit = 3 }, + [GCC_USB4_2_MISC_PCIE_PIPE_BCR] = { .reg = 0xaf0f8, .bit = 4 }, + [GCC_USB4_2_MISC_TMU_BCR] = { .reg = 0xaf0f8, .bit = 5 }, + [GCC_USB4_2_MISC_SB_IF_BCR] = { .reg = 0xaf0f8, .bit = 6 }, + [GCC_USB4_2_MISC_HIA_MSTR_BCR] = { .reg = 0xaf0f8, .bit = 7 }, + [GCC_USB4_2_MISC_AHB_BCR] = { .reg = 0xaf0f8, .bit = 8 }, + [GCC_USB4_2_MISC_DP0_MAX_PCLK_BCR] = { .reg = 0xaf0f8, .bit = 9 }, + [GCC_USB4_2_MISC_DP1_MAX_PCLK_BCR] = { .reg = 0xaf0f8, .bit = 10 }, [GCC_USB_0_PHY_BCR] = { 0x50020 }, [GCC_USB_1_PHY_BCR] = { 0x2a020 }, [GCC_USB_2_PHY_BCR] = { 0xa3020 }, diff --git a/drivers/clk/qcom/mmcc-sdm660.c b/drivers/clk/qcom/mmcc-sdm660.c index b723c536dfb6c..dbd3f561dc6d0 100644 --- a/drivers/clk/qcom/mmcc-sdm660.c +++ b/drivers/clk/qcom/mmcc-sdm660.c @@ -2781,6 +2781,7 @@ static struct gdsc *mmcc_sdm660_gdscs[] = { }; static const struct qcom_reset_map mmcc_660_resets[] = { + [MDSS_BCR] = { 0x2300 }, [CAMSS_MICRO_BCR] = { 0x3490 }, }; diff --git a/drivers/clk/qcom/tcsrcc-glymur.c b/drivers/clk/qcom/tcsrcc-glymur.c index c1f8b6d10b7fd..215bc2ac548da 100644 --- a/drivers/clk/qcom/tcsrcc-glymur.c +++ b/drivers/clk/qcom/tcsrcc-glymur.c @@ -28,10 +28,10 @@ enum { }; static struct clk_branch tcsr_edp_clkref_en = { - .halt_reg = 0x1c, + .halt_reg = 0x60, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x1c, + .enable_reg = 0x60, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_edp_clkref_en", @@ -45,10 +45,10 @@ static struct clk_branch tcsr_edp_clkref_en = { }; static struct clk_branch tcsr_pcie_1_clkref_en = { - .halt_reg = 0x4, + .halt_reg = 0x48, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x4, + .enable_reg = 0x48, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_pcie_1_clkref_en", @@ -62,10 +62,10 @@ static struct clk_branch tcsr_pcie_1_clkref_en = { }; static struct clk_branch tcsr_pcie_2_clkref_en = { - .halt_reg = 0x8, + .halt_reg = 0x4c, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x8, + .enable_reg = 0x4c, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_pcie_2_clkref_en", @@ -79,10 +79,10 @@ static struct clk_branch tcsr_pcie_2_clkref_en = { }; static struct clk_branch tcsr_pcie_3_clkref_en = { - .halt_reg = 0x10, + .halt_reg = 0x54, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x10, + .enable_reg = 0x54, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_pcie_3_clkref_en", @@ -96,10 +96,10 @@ static struct clk_branch tcsr_pcie_3_clkref_en = { }; static struct clk_branch tcsr_pcie_4_clkref_en = { - .halt_reg = 0x14, + .halt_reg = 0x58, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x14, + .enable_reg = 0x58, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_pcie_4_clkref_en", @@ -113,10 +113,10 @@ static struct clk_branch tcsr_pcie_4_clkref_en = { }; static struct clk_branch tcsr_usb2_1_clkref_en = { - .halt_reg = 0x28, + .halt_reg = 0x6c, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x28, + .enable_reg = 0x6c, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb2_1_clkref_en", @@ -130,10 +130,10 @@ static struct clk_branch tcsr_usb2_1_clkref_en = { }; static struct clk_branch tcsr_usb2_2_clkref_en = { - .halt_reg = 0x2c, + .halt_reg = 0x70, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x2c, + .enable_reg = 0x70, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb2_2_clkref_en", @@ -147,10 +147,10 @@ static struct clk_branch tcsr_usb2_2_clkref_en = { }; static struct clk_branch tcsr_usb2_3_clkref_en = { - .halt_reg = 0x30, + .halt_reg = 0x74, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x30, + .enable_reg = 0x74, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb2_3_clkref_en", @@ -164,10 +164,10 @@ static struct clk_branch tcsr_usb2_3_clkref_en = { }; static struct clk_branch tcsr_usb2_4_clkref_en = { - .halt_reg = 0x44, + .halt_reg = 0x88, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x44, + .enable_reg = 0x88, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb2_4_clkref_en", @@ -181,10 +181,10 @@ static struct clk_branch tcsr_usb2_4_clkref_en = { }; static struct clk_branch tcsr_usb3_0_clkref_en = { - .halt_reg = 0x20, + .halt_reg = 0x64, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x20, + .enable_reg = 0x64, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb3_0_clkref_en", @@ -198,10 +198,10 @@ static struct clk_branch tcsr_usb3_0_clkref_en = { }; static struct clk_branch tcsr_usb3_1_clkref_en = { - .halt_reg = 0x24, + .halt_reg = 0x68, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x24, + .enable_reg = 0x68, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb3_1_clkref_en", @@ -215,10 +215,10 @@ static struct clk_branch tcsr_usb3_1_clkref_en = { }; static struct clk_branch tcsr_usb4_1_clkref_en = { - .halt_reg = 0x0, + .halt_reg = 0x44, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x0, + .enable_reg = 0x44, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb4_1_clkref_en", @@ -232,10 +232,10 @@ static struct clk_branch tcsr_usb4_1_clkref_en = { }; static struct clk_branch tcsr_usb4_2_clkref_en = { - .halt_reg = 0x18, + .halt_reg = 0x5c, .halt_check = BRANCH_HALT_DELAY, .clkr = { - .enable_reg = 0x18, + .enable_reg = 0x5c, .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "tcsr_usb4_2_clkref_en", @@ -268,7 +268,7 @@ static const struct regmap_config tcsr_cc_glymur_regmap_config = { .reg_bits = 32, .reg_stride = 4, .val_bits = 32, - .max_register = 0x44, + .max_register = 0x94, .fast_io = true, }; diff --git a/drivers/clk/qcom/tcsrcc-kaanapali.c b/drivers/clk/qcom/tcsrcc-kaanapali.c new file mode 100644 index 0000000000000..14cfa75e892cc --- /dev/null +++ b/drivers/clk/qcom/tcsrcc-kaanapali.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include + +#include + +#include "clk-branch.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "common.h" + +enum { + DT_BI_TCXO_PAD, +}; + +static struct clk_branch tcsr_pcie_0_clkref_en = { + .halt_reg = 0x0, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "tcsr_pcie_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch tcsr_ufs_clkref_en = { + .halt_reg = 0x10, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x10, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "tcsr_ufs_clkref_en", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO_PAD, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch tcsr_usb2_clkref_en = { + .halt_reg = 0x18, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x18, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "tcsr_usb2_clkref_en", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO_PAD, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch tcsr_usb3_clkref_en = { + .halt_reg = 0x8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "tcsr_usb3_clkref_en", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO_PAD, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap *tcsr_cc_kaanapali_clocks[] = { + [TCSR_PCIE_0_CLKREF_EN] = &tcsr_pcie_0_clkref_en.clkr, + [TCSR_UFS_CLKREF_EN] = &tcsr_ufs_clkref_en.clkr, + [TCSR_USB2_CLKREF_EN] = &tcsr_usb2_clkref_en.clkr, + [TCSR_USB3_CLKREF_EN] = &tcsr_usb3_clkref_en.clkr, +}; + +static const struct regmap_config tcsr_cc_kaanapali_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x18, + .fast_io = true, +}; + +static const struct qcom_cc_desc tcsr_cc_kaanapali_desc = { + .config = &tcsr_cc_kaanapali_regmap_config, + .clks = tcsr_cc_kaanapali_clocks, + .num_clks = ARRAY_SIZE(tcsr_cc_kaanapali_clocks), +}; + +static const struct of_device_id tcsr_cc_kaanapali_match_table[] = { + { .compatible = "qcom,kaanapali-tcsr" }, + { } +}; +MODULE_DEVICE_TABLE(of, tcsr_cc_kaanapali_match_table); + +static int tcsr_cc_kaanapali_probe(struct platform_device *pdev) +{ + return qcom_cc_probe(pdev, &tcsr_cc_kaanapali_desc); +} + +static struct platform_driver tcsr_cc_kaanapali_driver = { + .probe = tcsr_cc_kaanapali_probe, + .driver = { + .name = "tcsr_cc-kaanapali", + .of_match_table = tcsr_cc_kaanapali_match_table, + }, +}; + +static int __init tcsr_cc_kaanapali_init(void) +{ + return platform_driver_register(&tcsr_cc_kaanapali_driver); +} +subsys_initcall(tcsr_cc_kaanapali_init); + +static void __exit tcsr_cc_kaanapali_exit(void) +{ + platform_driver_unregister(&tcsr_cc_kaanapali_driver); +} +module_exit(tcsr_cc_kaanapali_exit); + +MODULE_DESCRIPTION("QTI TCSR_CC KAANAPALI Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/qcom/videocc-x1p42100.c b/drivers/clk/qcom/videocc-x1p42100.c new file mode 100644 index 0000000000000..2bb40ac6fcc57 --- /dev/null +++ b/drivers/clk/qcom/videocc-x1p42100.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + DT_BI_TCXO, +}; + +enum { + P_BI_TCXO, + P_VIDEO_CC_PLL0_OUT_MAIN, + P_VIDEO_CC_PLL1_OUT_MAIN, +}; + +static const struct pll_vco lucid_ole_vco[] = { + { 249600000, 2300000000, 0 }, +}; + +/* 420.0 MHz Configuration */ +static const struct alpha_pll_config video_cc_pll0_config = { + .l = 0x15, + .alpha = 0xe000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x82aa299c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000003, + .test_ctl_hi1_val = 0x00009000, + .test_ctl_hi2_val = 0x00000034, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000005, +}; + +static struct clk_alpha_pll video_cc_pll0 = { + .offset = 0x0, + .config = &video_cc_pll0_config, + .vco_table = lucid_ole_vco, + .num_vco = ARRAY_SIZE(lucid_ole_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_pll0", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_evo_ops, + }, + }, +}; + +/* 1050.0 MHz Configuration */ +static const struct alpha_pll_config video_cc_pll1_config = { + .l = 0x36, + .alpha = 0xb000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x82aa299c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000003, + .test_ctl_hi1_val = 0x00009000, + .test_ctl_hi2_val = 0x00000034, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000005, +}; + +static struct clk_alpha_pll video_cc_pll1 = { + .offset = 0x1000, + .config = &video_cc_pll1_config, + .vco_table = lucid_ole_vco, + .num_vco = ARRAY_SIZE(lucid_ole_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE], + .clkr = { + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_pll1", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_evo_ops, + }, + }, +}; + +static const struct parent_map video_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data video_cc_parent_data_0[] = { + { .index = DT_BI_TCXO }, +}; + +static const struct parent_map video_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_VIDEO_CC_PLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data video_cc_parent_data_1[] = { + { .index = DT_BI_TCXO }, + { .hw = &video_cc_pll0.clkr.hw }, +}; + +static const struct parent_map video_cc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_VIDEO_CC_PLL1_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data video_cc_parent_data_2[] = { + { .index = DT_BI_TCXO }, + { .hw = &video_cc_pll1.clkr.hw }, +}; + +static const struct freq_tbl ftbl_video_cc_mvs0_bse_clk_src[] = { + F(420000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0), + F(600000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0), + F(670000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0), + F(848000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0), + F(920000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_mvs0_bse_clk_src = { + .cmd_rcgr = 0x8154, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_1, + .freq_tbl = ftbl_video_cc_mvs0_bse_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0_bse_clk_src", + .parent_data = video_cc_parent_data_1, + .num_parents = ARRAY_SIZE(video_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_video_cc_mvs0_clk_src[] = { + F(210000000, P_VIDEO_CC_PLL0_OUT_MAIN, 2, 0, 0), + F(300000000, P_VIDEO_CC_PLL0_OUT_MAIN, 2, 0, 0), + F(335000000, P_VIDEO_CC_PLL0_OUT_MAIN, 2, 0, 0), + F(424000000, P_VIDEO_CC_PLL0_OUT_MAIN, 2, 0, 0), + F(460000000, P_VIDEO_CC_PLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_mvs0_clk_src = { + .cmd_rcgr = 0x8000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_1, + .freq_tbl = ftbl_video_cc_mvs0_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0_clk_src", + .parent_data = video_cc_parent_data_1, + .num_parents = ARRAY_SIZE(video_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_video_cc_mvs1_clk_src[] = { + F(1050000000, P_VIDEO_CC_PLL1_OUT_MAIN, 1, 0, 0), + F(1350000000, P_VIDEO_CC_PLL1_OUT_MAIN, 1, 0, 0), + F(1650000000, P_VIDEO_CC_PLL1_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_mvs1_clk_src = { + .cmd_rcgr = 0x8018, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_2, + .freq_tbl = ftbl_video_cc_mvs1_clk_src, + .hw_clk_ctrl = true, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs1_clk_src", + .parent_data = video_cc_parent_data_2, + .num_parents = ARRAY_SIZE(video_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_video_cc_xo_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_xo_clk_src = { + .cmd_rcgr = 0x810c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_0, + .freq_tbl = ftbl_video_cc_xo_clk_src, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "video_cc_xo_clk_src", + .parent_data = video_cc_parent_data_0, + .num_parents = ARRAY_SIZE(video_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_regmap_div video_cc_mvs0_bse_div4_div_clk_src = { + .reg = 0x817c, + .shift = 0, + .width = 4, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0_bse_div4_div_clk_src", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs0_bse_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div video_cc_mvs1_div_clk_src = { + .reg = 0x80ec, + .shift = 0, + .width = 4, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs1_div_clk_src", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div video_cc_mvs1c_div2_div_clk_src = { + .reg = 0x809c, + .shift = 0, + .width = 4, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs1c_div2_div_clk_src", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch video_cc_mvs0_bse_clk = { + .halt_reg = 0x8170, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8170, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0_bse_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs0_bse_div4_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs0_clk = { + .halt_reg = 0x80b8, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x80b8, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x80b8, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs0_shift_clk = { + .halt_reg = 0x8128, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x8128, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0_shift_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs0c_clk = { + .halt_reg = 0x8064, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8064, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0c_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs0c_shift_clk = { + .halt_reg = 0x812c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x812c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs0c_shift_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs1_clk = { + .halt_reg = 0x80e0, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x80e0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x80e0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs1_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs1_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs1_shift_clk = { + .halt_reg = 0x8130, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x8130, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs1_shift_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs1c_clk = { + .halt_reg = 0x8090, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8090, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs1c_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_mvs1c_div2_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_mvs1c_shift_clk = { + .halt_reg = 0x8134, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x8134, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "video_cc_mvs1c_shift_clk", + .parent_hws = (const struct clk_hw*[]) { + &video_cc_xo_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc video_cc_mvs0c_gdsc = { + .gdscr = 0x804c, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x6, + .pd = { + .name = "video_cc_mvs0c_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc video_cc_mvs0_gdsc = { + .gdscr = 0x80a4, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x6, + .pd = { + .name = "video_cc_mvs0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .parent = &video_cc_mvs0c_gdsc.pd, + .flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc video_cc_mvs1c_gdsc = { + .gdscr = 0x8078, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "video_cc_mvs1c_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct gdsc video_cc_mvs1_gdsc = { + .gdscr = 0x80cc, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, + .pd = { + .name = "video_cc_mvs1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .parent = &video_cc_mvs1c_gdsc.pd, + .flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE, +}; + +static struct clk_regmap *video_cc_x1p42100_clocks[] = { + [VIDEO_CC_MVS0_BSE_CLK] = &video_cc_mvs0_bse_clk.clkr, + [VIDEO_CC_MVS0_BSE_CLK_SRC] = &video_cc_mvs0_bse_clk_src.clkr, + [VIDEO_CC_MVS0_BSE_DIV4_DIV_CLK_SRC] = &video_cc_mvs0_bse_div4_div_clk_src.clkr, + [VIDEO_CC_MVS0_CLK] = &video_cc_mvs0_clk.clkr, + [VIDEO_CC_MVS0_CLK_SRC] = &video_cc_mvs0_clk_src.clkr, + [VIDEO_CC_MVS0_SHIFT_CLK] = &video_cc_mvs0_shift_clk.clkr, + [VIDEO_CC_MVS0C_CLK] = &video_cc_mvs0c_clk.clkr, + [VIDEO_CC_MVS0C_SHIFT_CLK] = &video_cc_mvs0c_shift_clk.clkr, + [VIDEO_CC_MVS1_CLK] = &video_cc_mvs1_clk.clkr, + [VIDEO_CC_MVS1_CLK_SRC] = &video_cc_mvs1_clk_src.clkr, + [VIDEO_CC_MVS1_DIV_CLK_SRC] = &video_cc_mvs1_div_clk_src.clkr, + [VIDEO_CC_MVS1_SHIFT_CLK] = &video_cc_mvs1_shift_clk.clkr, + [VIDEO_CC_MVS1C_CLK] = &video_cc_mvs1c_clk.clkr, + [VIDEO_CC_MVS1C_DIV2_DIV_CLK_SRC] = &video_cc_mvs1c_div2_div_clk_src.clkr, + [VIDEO_CC_MVS1C_SHIFT_CLK] = &video_cc_mvs1c_shift_clk.clkr, + [VIDEO_CC_PLL0] = &video_cc_pll0.clkr, + [VIDEO_CC_PLL1] = &video_cc_pll1.clkr, + [VIDEO_CC_XO_CLK_SRC] = &video_cc_xo_clk_src.clkr, +}; + +static struct gdsc *video_cc_x1p42100_gdscs[] = { + [VIDEO_CC_MVS0_GDSC] = &video_cc_mvs0_gdsc, + [VIDEO_CC_MVS0C_GDSC] = &video_cc_mvs0c_gdsc, + [VIDEO_CC_MVS1_GDSC] = &video_cc_mvs1_gdsc, + [VIDEO_CC_MVS1C_GDSC] = &video_cc_mvs1c_gdsc, +}; + +static const struct qcom_reset_map video_cc_x1p42100_resets[] = { + [CVP_VIDEO_CC_INTERFACE_BCR] = { 0x80f0 }, + [CVP_VIDEO_CC_MVS0_BCR] = { 0x80a0 }, + [CVP_VIDEO_CC_MVS0C_BCR] = { 0x8048 }, + [CVP_VIDEO_CC_MVS1_BCR] = { 0x80c8 }, + [CVP_VIDEO_CC_MVS1C_BCR] = { 0x8074 }, + [VIDEO_CC_MVS0_BSE_BCR] = { 0x816c }, + [VIDEO_CC_MVS0C_CLK_ARES] = { 0x8064, 2 }, + [VIDEO_CC_MVS1C_CLK_ARES] = { 0x8090, 2 }, + [VIDEO_CC_XO_CLK_ARES] = { 0x8124, 2 }, +}; + +static struct clk_alpha_pll *video_cc_x1p42100_plls[] = { + &video_cc_pll0, + &video_cc_pll1, +}; + +static u32 video_cc_x1p42100_critical_cbcrs[] = { + 0x80f4, /* VIDEO_CC_AHB_CLK */ + 0x8150, /* VIDEO_CC_SLEEP_CLK */ + 0x8124, /* VIDEO_CC_XO_CLK */ +}; + +static const struct regmap_config video_cc_x1p42100_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x9f54, + .fast_io = true, +}; + +static struct qcom_cc_driver_data video_cc_x1p42100_driver_data = { + .alpha_plls = video_cc_x1p42100_plls, + .num_alpha_plls = ARRAY_SIZE(video_cc_x1p42100_plls), + .clk_cbcrs = video_cc_x1p42100_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(video_cc_x1p42100_critical_cbcrs), +}; + +static const struct qcom_cc_desc video_cc_x1p42100_desc = { + .config = &video_cc_x1p42100_regmap_config, + .clks = video_cc_x1p42100_clocks, + .num_clks = ARRAY_SIZE(video_cc_x1p42100_clocks), + .resets = video_cc_x1p42100_resets, + .num_resets = ARRAY_SIZE(video_cc_x1p42100_resets), + .gdscs = video_cc_x1p42100_gdscs, + .num_gdscs = ARRAY_SIZE(video_cc_x1p42100_gdscs), + .use_rpm = true, + .driver_data = &video_cc_x1p42100_driver_data, +}; + +static const struct of_device_id video_cc_x1p42100_match_table[] = { + { .compatible = "qcom,x1p42100-videocc" }, + { } +}; +MODULE_DEVICE_TABLE(of, video_cc_x1p42100_match_table); + +static int video_cc_x1p42100_probe(struct platform_device *pdev) +{ + return qcom_cc_probe(pdev, &video_cc_x1p42100_desc); +} + +static struct platform_driver video_cc_x1p42100_driver = { + .probe = video_cc_x1p42100_probe, + .driver = { + .name = "videocc-x1p42100", + .of_match_table = video_cc_x1p42100_match_table, + }, +}; + +module_platform_driver(video_cc_x1p42100_driver); + +MODULE_DESCRIPTION("QTI VIDEOCC X1P42100 Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/renesas/r9a06g032-clocks.c b/drivers/clk/renesas/r9a06g032-clocks.c index dcda19318b2a9..0f5c91b5dfa99 100644 --- a/drivers/clk/renesas/r9a06g032-clocks.c +++ b/drivers/clk/renesas/r9a06g032-clocks.c @@ -1333,9 +1333,9 @@ static int __init r9a06g032_clocks_probe(struct platform_device *pdev) if (IS_ERR(mclk)) return PTR_ERR(mclk); - clocks->reg = of_iomap(np, 0); - if (WARN_ON(!clocks->reg)) - return -ENOMEM; + clocks->reg = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(clocks->reg)) + return PTR_ERR(clocks->reg); r9a06g032_init_h2mode(clocks); diff --git a/drivers/clk/renesas/r9a09g077-cpg.c b/drivers/clk/renesas/r9a09g077-cpg.c index af3ef6d58c87c..d12975418a568 100644 --- a/drivers/clk/renesas/r9a09g077-cpg.c +++ b/drivers/clk/renesas/r9a09g077-cpg.c @@ -217,7 +217,7 @@ r9a09g077_cpg_div_clk_register(struct device *dev, if (core->dtable) clk_hw = clk_hw_register_divider_table(dev, core->name, - parent_name, 0, + parent_name, CLK_SET_RATE_PARENT, addr, GET_SHIFT(core->conf), GET_WIDTH(core->conf), @@ -226,7 +226,7 @@ r9a09g077_cpg_div_clk_register(struct device *dev, &pub->rmw_lock); else clk_hw = clk_hw_register_divider(dev, core->name, - parent_name, 0, + parent_name, CLK_SET_RATE_PARENT, addr, GET_SHIFT(core->conf), GET_WIDTH(core->conf), diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index de1cf7ba45b78..a0a68ec0490f7 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -676,53 +676,56 @@ static int __init cpg_mssr_add_clk_domain(struct device *dev, #define rcdev_to_priv(x) container_of(x, struct cpg_mssr_priv, rcdev) -static int cpg_mssr_reset(struct reset_controller_dev *rcdev, - unsigned long id) +static int cpg_mssr_reset_operate(struct reset_controller_dev *rcdev, + const char *func, bool set, unsigned long id) { struct cpg_mssr_priv *priv = rcdev_to_priv(rcdev); unsigned int reg = id / 32; unsigned int bit = id % 32; + const u16 off = set ? priv->reset_regs[reg] : priv->reset_clear_regs[reg]; u32 bitmask = BIT(bit); - dev_dbg(priv->dev, "reset %u%02u\n", reg, bit); - - /* Reset module */ - writel(bitmask, priv->pub.base0 + priv->reset_regs[reg]); - - /* Wait for at least one cycle of the RCLK clock (@ ca. 32 kHz) */ - udelay(35); + if (func) + dev_dbg(priv->dev, "%s %u%02u\n", func, reg, bit); - /* Release module from reset state */ - writel(bitmask, priv->pub.base0 + priv->reset_clear_regs[reg]); + writel(bitmask, priv->pub.base0 + off); + readl(priv->pub.base0 + off); + barrier_data(priv->pub.base0 + off); return 0; } -static int cpg_mssr_assert(struct reset_controller_dev *rcdev, unsigned long id) +static int cpg_mssr_reset(struct reset_controller_dev *rcdev, + unsigned long id) { struct cpg_mssr_priv *priv = rcdev_to_priv(rcdev); - unsigned int reg = id / 32; - unsigned int bit = id % 32; - u32 bitmask = BIT(bit); - dev_dbg(priv->dev, "assert %u%02u\n", reg, bit); + /* Reset module */ + cpg_mssr_reset_operate(rcdev, "reset", true, id); - writel(bitmask, priv->pub.base0 + priv->reset_regs[reg]); - return 0; + /* + * On R-Car Gen4, delay after SRCR has been written is 1ms. + * On older SoCs, delay after SRCR has been written is 35us + * (one cycle of the RCLK clock @ ca. 32 kHz). + */ + if (priv->reg_layout == CLK_REG_LAYOUT_RCAR_GEN4) + usleep_range(1000, 2000); + else + usleep_range(35, 1000); + + /* Release module from reset state */ + return cpg_mssr_reset_operate(rcdev, NULL, false, id); +} + +static int cpg_mssr_assert(struct reset_controller_dev *rcdev, unsigned long id) +{ + return cpg_mssr_reset_operate(rcdev, "assert", true, id); } static int cpg_mssr_deassert(struct reset_controller_dev *rcdev, unsigned long id) { - struct cpg_mssr_priv *priv = rcdev_to_priv(rcdev); - unsigned int reg = id / 32; - unsigned int bit = id % 32; - u32 bitmask = BIT(bit); - - dev_dbg(priv->dev, "deassert %u%02u\n", reg, bit); - - writel(bitmask, priv->pub.base0 + priv->reset_clear_regs[reg]); - return 0; + return cpg_mssr_reset_operate(rcdev, "deassert", false, id); } static int cpg_mssr_status(struct reset_controller_dev *rcdev, diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c index 07909e80bae24..edad47ca33ec5 100644 --- a/drivers/clk/renesas/rzg2l-cpg.c +++ b/drivers/clk/renesas/rzg2l-cpg.c @@ -122,8 +122,8 @@ struct div_hw_data { struct rzg2l_pll5_param { u32 pl5_fracin; + u16 pl5_intin; u8 pl5_refdiv; - u8 pl5_intin; u8 pl5_postdiv1; u8 pl5_postdiv2; u8 pl5_spread; @@ -572,8 +572,8 @@ rzg2l_cpg_get_foutpostdiv_rate(struct rzg2l_pll5_param *params, foutvco_rate = div_u64(mul_u32_u32(EXTAL_FREQ_IN_MEGA_HZ * MEGA, (params->pl5_intin << 24) + params->pl5_fracin), params->pl5_refdiv) >> 24; - foutpostdiv_rate = DIV_ROUND_CLOSEST_ULL(foutvco_rate, - params->pl5_postdiv1 * params->pl5_postdiv2); + foutpostdiv_rate = DIV_ROUND_CLOSEST(foutvco_rate, + params->pl5_postdiv1 * params->pl5_postdiv2); return foutpostdiv_rate; } @@ -1647,6 +1647,7 @@ static int __rzg2l_cpg_assert(struct reset_controller_dev *rcdev, u32 mask = BIT(info->resets[id].bit); s8 monbit = info->resets[id].monbit; u32 value = mask << 16; + u32 mon; int ret; dev_dbg(rcdev->dev, "%s id:%ld offset:0x%x\n", @@ -1667,10 +1668,10 @@ static int __rzg2l_cpg_assert(struct reset_controller_dev *rcdev, return 0; } - ret = readl_poll_timeout_atomic(priv->base + reg, value, - assert == !!(value & mask), 10, 200); - if (ret && !assert) { - value = mask << 16; + ret = readl_poll_timeout_atomic(priv->base + reg, mon, + assert == !!(mon & mask), 10, 200); + if (ret) { + value ^= mask; writel(value, priv->base + CLK_RST_R(info->resets[id].off)); } diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index 19caf26c991bd..2d30b1e24f013 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -693,7 +693,7 @@ void rockchip_clk_register_late_branches(struct device *dev, break; } - if (!pdev) + if (IS_ERR_OR_NULL(pdev)) dev_err(dev, "failed to register device for clock %s\n", list->name); } } diff --git a/drivers/clk/samsung/clk-exynos-clkout.c b/drivers/clk/samsung/clk-exynos-clkout.c index 5f1a4f5e2e594..5b21025338bd5 100644 --- a/drivers/clk/samsung/clk-exynos-clkout.c +++ b/drivers/clk/samsung/clk-exynos-clkout.c @@ -175,6 +175,7 @@ static int exynos_clkout_probe(struct platform_device *pdev) clkout->mux.shift = EXYNOS_CLKOUT_MUX_SHIFT; clkout->mux.lock = &clkout->slock; + clkout->data.num = EXYNOS_CLKOUT_NR_CLKS; clkout->data.hws[0] = clk_hw_register_composite(NULL, "clkout", parent_names, parent_count, &clkout->mux.hw, &clk_mux_ops, NULL, NULL, &clkout->gate.hw, @@ -185,7 +186,6 @@ static int exynos_clkout_probe(struct platform_device *pdev) goto err_unmap; } - clkout->data.num = EXYNOS_CLKOUT_NR_CLKS; ret = of_clk_add_hw_provider(clkout->np, of_clk_hw_onecell_get, &clkout->data); if (ret) goto err_clk_unreg; diff --git a/drivers/clk/sophgo/clk-sg2042-clkgen.c b/drivers/clk/sophgo/clk-sg2042-clkgen.c index 683661b71787c..9725ac4e050a4 100644 --- a/drivers/clk/sophgo/clk-sg2042-clkgen.c +++ b/drivers/clk/sophgo/clk-sg2042-clkgen.c @@ -180,7 +180,6 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct sg2042_divider_clock *divider = to_sg2042_clk_divider(hw); - unsigned long ret_rate; u32 bestdiv; /* if read only, just return current value */ @@ -191,17 +190,13 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, bestdiv = readl(divider->reg) >> divider->shift; bestdiv &= clk_div_mask(divider->width); } - ret_rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - } else { - ret_rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, NULL, - divider->width, divider->div_flags); - } + req->rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - pr_debug("--> %s: divider_round_rate: val = %ld\n", - clk_hw_get_name(hw), ret_rate); - req->rate = ret_rate; + return 0; + } - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + divider->div_flags); } static int sg2042_clk_divider_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/spacemit/Makefile b/drivers/clk/spacemit/Makefile index 5ec6da61db98e..ad2bf315109b8 100644 --- a/drivers/clk/spacemit/Makefile +++ b/drivers/clk/spacemit/Makefile @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_SPACEMIT_K1_CCU) = spacemit-ccu-k1.o -spacemit-ccu-k1-y = ccu_pll.o ccu_mix.o ccu_ddn.o +obj-$(CONFIG_SPACEMIT_CCU) += spacemit-ccu.o +spacemit-ccu-y += ccu_common.o +spacemit-ccu-y += ccu_pll.o +spacemit-ccu-y += ccu_mix.o +spacemit-ccu-y += ccu_ddn.o + +obj-$(CONFIG_SPACEMIT_K1_CCU) += spacemit-ccu-k1.o spacemit-ccu-k1-y += ccu-k1.o diff --git a/drivers/clk/spacemit/ccu-k1.c b/drivers/clk/spacemit/ccu-k1.c index f5a9fe6ba1859..01d9485b615d3 100644 --- a/drivers/clk/spacemit/ccu-k1.c +++ b/drivers/clk/spacemit/ccu-k1.c @@ -1018,6 +1018,8 @@ static int spacemit_ccu_register(struct device *dev, if (!clk_data) return -ENOMEM; + clk_data->num = data->num; + for (i = 0; i < data->num; i++) { struct clk_hw *hw = data->hws[i]; struct ccu_common *common; @@ -1044,8 +1046,6 @@ static int spacemit_ccu_register(struct device *dev, clk_data->hws[i] = hw; } - clk_data->num = data->num; - ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data); if (ret) dev_err(dev, "failed to add clock hardware provider (%d)\n", ret); @@ -1204,6 +1204,7 @@ static struct platform_driver k1_ccu_driver = { }; module_platform_driver(k1_ccu_driver); +MODULE_IMPORT_NS("CLK_SPACEMIT"); MODULE_DESCRIPTION("SpacemiT K1 CCU driver"); MODULE_AUTHOR("Haylen Chu "); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_common.c b/drivers/clk/spacemit/ccu_common.c new file mode 100644 index 0000000000000..4412c4104dabb --- /dev/null +++ b/drivers/clk/spacemit/ccu_common.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include + +MODULE_DESCRIPTION("SpacemiT CCU common clock driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_ddn.c b/drivers/clk/spacemit/ccu_ddn.c index 5b16e273bee5b..b5540e0781ffa 100644 --- a/drivers/clk/spacemit/ccu_ddn.c +++ b/drivers/clk/spacemit/ccu_ddn.c @@ -84,3 +84,4 @@ const struct clk_ops spacemit_ccu_ddn_ops = { .determine_rate = ccu_ddn_determine_rate, .set_rate = ccu_ddn_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_ddn_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_mix.c b/drivers/clk/spacemit/ccu_mix.c index 7b79908753723..67f8b12b4f5b7 100644 --- a/drivers/clk/spacemit/ccu_mix.c +++ b/drivers/clk/spacemit/ccu_mix.c @@ -198,24 +198,28 @@ const struct clk_ops spacemit_ccu_gate_ops = { .enable = ccu_gate_enable, .is_enabled = ccu_gate_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_ops = { .determine_rate = ccu_factor_determine_rate, .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_ops = { .determine_rate = ccu_mix_determine_rate, .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_ops = { .determine_rate = ccu_mix_determine_rate, .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_gate_ops = { .disable = ccu_gate_disable, @@ -226,6 +230,7 @@ const struct clk_ops spacemit_ccu_factor_gate_ops = { .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_gate_ops = { .disable = ccu_gate_disable, @@ -236,6 +241,7 @@ const struct clk_ops spacemit_ccu_mux_gate_ops = { .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_gate_ops = { .disable = ccu_gate_disable, @@ -246,6 +252,7 @@ const struct clk_ops spacemit_ccu_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .disable = ccu_gate_disable, @@ -259,6 +266,7 @@ const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_ops = { .get_parent = ccu_mux_get_parent, @@ -268,3 +276,4 @@ const struct clk_ops spacemit_ccu_mux_div_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_pll.c b/drivers/clk/spacemit/ccu_pll.c index d92f0dae65a49..76d0244873d87 100644 --- a/drivers/clk/spacemit/ccu_pll.c +++ b/drivers/clk/spacemit/ccu_pll.c @@ -157,3 +157,4 @@ const struct clk_ops spacemit_ccu_pll_ops = { .determine_rate = ccu_pll_determine_rate, .is_enabled = ccu_pll_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_pll_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/sprd/div.c b/drivers/clk/sprd/div.c index 0134238819680..cd57163a7204c 100644 --- a/drivers/clk/sprd/div.c +++ b/drivers/clk/sprd/div.c @@ -14,11 +14,7 @@ static int sprd_div_determine_rate(struct clk_hw *hw, { struct sprd_div *cd = hw_to_sprd_div(hw); - req->rate = divider_round_rate(&cd->common.hw, req->rate, - &req->best_parent_rate, - NULL, cd->div.width, 0); - - return 0; + return divider_determine_rate(&cd->common.hw, req, NULL, cd->div.width, 0); } unsigned long sprd_div_helper_recalc_rate(struct sprd_clk_common *common, diff --git a/drivers/clk/stm32/clk-stm32-core.c b/drivers/clk/stm32/clk-stm32-core.c index 72825b9c36a4d..e921c25a929c3 100644 --- a/drivers/clk/stm32/clk-stm32-core.c +++ b/drivers/clk/stm32/clk-stm32-core.c @@ -369,22 +369,14 @@ static int clk_stm32_divider_determine_rate(struct clk_hw *hw, val = readl(div->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, - divider->table, - divider->width, - divider->flags, val); - - return 0; + return divider_ro_determine_rate(hw, req, + divider->table, + divider->width, + divider->flags, val); } - req->rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, - &req->best_parent_rate, - divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static unsigned long clk_stm32_divider_recalc_rate(struct clk_hw *hw, @@ -441,7 +433,6 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, { struct clk_stm32_composite *composite = to_clk_stm32_composite(hw); const struct stm32_div_cfg *divider; - long rate; if (composite->div_id == NO_STM32_DIV) return 0; @@ -455,24 +446,13 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, val = readl(composite->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - rate = divider_ro_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags, - val); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_ro_determine_rate(hw, req, divider->table, + divider->width, divider->flags, + val); } - rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static u8 clk_stm32_composite_get_parent(struct clk_hw *hw) diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c index 2a6db04342815..5f1af6dfe7154 100644 --- a/drivers/clk/tegra/clk-tegra124-emc.c +++ b/drivers/clk/tegra/clk-tegra124-emc.c @@ -197,8 +197,8 @@ static struct tegra_emc *emc_ensure_emc_driver(struct tegra_clk_emc *tegra) tegra->emc_node = NULL; tegra->emc = platform_get_drvdata(pdev); + put_device(&pdev->dev); if (!tegra->emc) { - put_device(&pdev->dev); pr_err("%s: cannot find EMC driver\n", __func__); return NULL; } @@ -538,8 +538,10 @@ struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np tegra->hw.init = &init; clk = clk_register(NULL, &tegra->hw); - if (IS_ERR(clk)) + if (IS_ERR(clk)) { + kfree(tegra); return clk; + } tegra->prev_parent = clk_hw_get_parent_by_index( &tegra->hw, emc_get_parent(&tegra->hw))->clk; diff --git a/drivers/clk/thead/clk-th1520-ap.c b/drivers/clk/thead/clk-th1520-ap.c index 71ad03a998e8e..d870f0c665f8a 100644 --- a/drivers/clk/thead/clk-th1520-ap.c +++ b/drivers/clk/thead/clk-th1520-ap.c @@ -8,11 +8,14 @@ #include #include #include +#include #include #include #include #include +#define TH1520_PLL_STS 0x80 + #define TH1520_PLL_POSTDIV2 GENMASK(26, 24) #define TH1520_PLL_POSTDIV1 GENMASK(22, 20) #define TH1520_PLL_FBDIV GENMASK(19, 8) @@ -23,6 +26,13 @@ #define TH1520_PLL_FRAC GENMASK(23, 0) #define TH1520_PLL_FRAC_BITS 24 +/* + * All PLLs in TH1520 take 21250ns at maximum to lock, let's take its double + * for safety. + */ +#define TH1520_PLL_LOCK_TIMEOUT_US 44 +#define TH1520_PLL_STABLE_DELAY_US 30 + struct ccu_internal { u8 shift; u8 width; @@ -64,6 +74,7 @@ struct ccu_div { struct ccu_pll { struct ccu_common common; + u32 lock_sts_mask; }; #define TH_CCU_ARG(_shift, _width) \ @@ -299,9 +310,21 @@ static void ccu_pll_disable(struct clk_hw *hw) static int ccu_pll_enable(struct clk_hw *hw) { struct ccu_pll *pll = hw_to_ccu_pll(hw); + u32 reg; + int ret; - return regmap_clear_bits(pll->common.map, pll->common.cfg1, - TH1520_PLL_VCO_RST); + regmap_clear_bits(pll->common.map, pll->common.cfg1, + TH1520_PLL_VCO_RST); + + ret = regmap_read_poll_timeout_atomic(pll->common.map, TH1520_PLL_STS, + reg, reg & pll->lock_sts_mask, + 5, TH1520_PLL_LOCK_TIMEOUT_US); + if (ret) + return ret; + + udelay(TH1520_PLL_STABLE_DELAY_US); + + return 0; } static int ccu_pll_is_enabled(struct clk_hw *hw) @@ -389,6 +412,7 @@ static struct ccu_pll cpu_pll0_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(1), }; static struct ccu_pll cpu_pll1_clk = { @@ -401,6 +425,7 @@ static struct ccu_pll cpu_pll1_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(4), }; static struct ccu_pll gmac_pll_clk = { @@ -413,6 +438,7 @@ static struct ccu_pll gmac_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(3), }; static const struct clk_hw *gmac_pll_clk_parent[] = { @@ -433,6 +459,7 @@ static struct ccu_pll video_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(7), }; static const struct clk_hw *video_pll_clk_parent[] = { @@ -453,6 +480,7 @@ static struct ccu_pll dpu0_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(8), }; static const struct clk_hw *dpu0_pll_clk_parent[] = { @@ -469,6 +497,7 @@ static struct ccu_pll dpu1_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(9), }; static const struct clk_hw *dpu1_pll_clk_parent[] = { @@ -485,6 +514,7 @@ static struct ccu_pll tee_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(10), }; static const struct clk_parent_data c910_i0_parents[] = { diff --git a/drivers/clk/x86/clk-cgu.c b/drivers/clk/x86/clk-cgu.c index d099667355f8d..92ee05d75af2b 100644 --- a/drivers/clk/x86/clk-cgu.c +++ b/drivers/clk/x86/clk-cgu.c @@ -137,10 +137,8 @@ static int lgm_clk_divider_determine_rate(struct clk_hw *hw, { struct lgm_clk_divider *divider = to_lgm_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index c824eeacd8ebd..984e577ea6711 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -111,10 +111,9 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, } /** - * zynqmp_clk_divider_round_rate() - Round rate of divider clock + * zynqmp_clk_divider_determine_rate() - Determine rate of divider clock * @hw: handle between common and hardware-specific interfaces - * @rate: rate of clock to be set - * @prate: rate of parent clock + * @req: rate of clock to be set * * Return: 0 on success else error+reason */ @@ -151,8 +150,9 @@ static int zynqmp_clk_divider_determine_rate(struct clk_hw *hw, width = fls(divider->max_div); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, width, divider->flags); + ret = divider_determine_rate(hw, req, NULL, width, divider->flags); + if (ret != 0) + return ret; if (divider->is_frac && (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && (req->rate % req->best_parent_rate)) diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c index 630a3936c97c3..6bc2c3934f564 100644 --- a/drivers/clk/zynqmp/pll.c +++ b/drivers/clk/zynqmp/pll.c @@ -91,10 +91,9 @@ static inline void zynqmp_pll_set_mode(struct clk_hw *hw, bool on) } /** - * zynqmp_pll_round_rate() - Round a clock frequency + * zynqmp_pll_determine_rate() - Round a clock frequency * @hw: Handle between common and hardware-specific interfaces - * @rate: Desired clock frequency - * @prate: Clock frequency of parent clock + * @req: Desired clock frequency * * Return: Frequency closest to @rate the hardware can generate */ diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index ffcd23668763f..1119fbd52e1c9 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -254,6 +254,7 @@ config KEYSTONE_TIMER config INTEGRATOR_AP_TIMER bool "Integrator-AP timer driver" if COMPILE_TEST + depends on OF select CLKSRC_MMIO help Enables support for the Integrator-AP timer. diff --git a/drivers/clocksource/arm_arch_timer_mmio.c b/drivers/clocksource/arm_arch_timer_mmio.c index ebe1987d651eb..d10362692fdd3 100644 --- a/drivers/clocksource/arm_arch_timer_mmio.c +++ b/drivers/clocksource/arm_arch_timer_mmio.c @@ -426,6 +426,7 @@ static struct platform_driver arch_timer_mmio_drv = { .driver = { .name = "arch-timer-mmio", .of_match_table = arch_timer_mmio_of_table, + .suppress_bind_attrs = true, }, .probe = arch_timer_mmio_probe, }; @@ -434,6 +435,7 @@ builtin_platform_driver(arch_timer_mmio_drv); static struct platform_driver arch_timer_mmio_acpi_drv = { .driver = { .name = "gtdt-arm-mmio-timer", + .suppress_bind_attrs = true, }, .probe = arch_timer_mmio_probe, }; diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index beffff81c00f3..3fc6ed9b56300 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -143,16 +143,6 @@ static void sh_tmu_start_stop_ch(struct sh_tmu_channel *ch, int start) static int __sh_tmu_enable(struct sh_tmu_channel *ch) { - int ret; - - /* enable clock */ - ret = clk_enable(ch->tmu->clk); - if (ret) { - dev_err(&ch->tmu->pdev->dev, "ch%u: cannot enable clock\n", - ch->index); - return ret; - } - /* make sure channel is disabled */ sh_tmu_start_stop_ch(ch, 0); @@ -174,7 +164,6 @@ static int sh_tmu_enable(struct sh_tmu_channel *ch) if (ch->enable_count++ > 0) return 0; - pm_runtime_get_sync(&ch->tmu->pdev->dev); dev_pm_syscore_device(&ch->tmu->pdev->dev, true); return __sh_tmu_enable(ch); @@ -187,9 +176,6 @@ static void __sh_tmu_disable(struct sh_tmu_channel *ch) /* disable interrupts in TMU block */ sh_tmu_write(ch, TCR, TCR_TPSC_CLK4); - - /* stop clock */ - clk_disable(ch->tmu->clk); } static void sh_tmu_disable(struct sh_tmu_channel *ch) @@ -203,7 +189,6 @@ static void sh_tmu_disable(struct sh_tmu_channel *ch) __sh_tmu_disable(ch); dev_pm_syscore_device(&ch->tmu->pdev->dev, false); - pm_runtime_put(&ch->tmu->pdev->dev); } static void sh_tmu_set_next(struct sh_tmu_channel *ch, unsigned long delta, @@ -552,7 +537,6 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) goto err_clk_unprepare; tmu->rate = clk_get_rate(tmu->clk) / 4; - clk_disable(tmu->clk); /* Map the memory resource. */ ret = sh_tmu_map_memory(tmu); @@ -626,8 +610,6 @@ static int sh_tmu_probe(struct platform_device *pdev) out: if (tmu->has_clockevent || tmu->has_clocksource) pm_runtime_irq_safe(&pdev->dev); - else - pm_runtime_idle(&pdev->dev); return 0; } diff --git a/drivers/clocksource/timer-nxp-pit.c b/drivers/clocksource/timer-nxp-pit.c index 2d0a3554b6bf7..d1740f18f7180 100644 --- a/drivers/clocksource/timer-nxp-pit.c +++ b/drivers/clocksource/timer-nxp-pit.c @@ -374,9 +374,10 @@ static struct platform_driver nxp_pit_driver = { .driver = { .name = "nxp-pit", .of_match_table = pit_timer_of_match, + .suppress_bind_attrs = true, }, .probe = pit_timer_probe, }; -module_platform_driver(nxp_pit_driver); +builtin_platform_driver(nxp_pit_driver); TIMER_OF_DECLARE(vf610, "fsl,vf610-pit", pit_timer_init); diff --git a/drivers/clocksource/timer-nxp-stm.c b/drivers/clocksource/timer-nxp-stm.c index bbc40623728fa..1ab907233f481 100644 --- a/drivers/clocksource/timer-nxp-stm.c +++ b/drivers/clocksource/timer-nxp-stm.c @@ -177,15 +177,15 @@ static void nxp_stm_clocksource_resume(struct clocksource *cs) nxp_stm_clocksource_enable(cs); } -static void __init devm_clocksource_unregister(void *data) +static void devm_clocksource_unregister(void *data) { struct stm_timer *stm_timer = data; clocksource_unregister(&stm_timer->cs); } -static int __init nxp_stm_clocksource_init(struct device *dev, struct stm_timer *stm_timer, - const char *name, void __iomem *base, struct clk *clk) +static int nxp_stm_clocksource_init(struct device *dev, struct stm_timer *stm_timer, + const char *name, void __iomem *base, struct clk *clk) { int ret; @@ -208,10 +208,8 @@ static int __init nxp_stm_clocksource_init(struct device *dev, struct stm_timer return ret; ret = devm_add_action_or_reset(dev, devm_clocksource_unregister, stm_timer); - if (ret) { - clocksource_unregister(&stm_timer->cs); + if (ret) return ret; - } stm_sched_clock = stm_timer; @@ -298,9 +296,9 @@ static void nxp_stm_clockevent_resume(struct clock_event_device *ced) nxp_stm_module_get(stm_timer); } -static int __init nxp_stm_clockevent_per_cpu_init(struct device *dev, struct stm_timer *stm_timer, - const char *name, void __iomem *base, int irq, - struct clk *clk, int cpu) +static int nxp_stm_clockevent_per_cpu_init(struct device *dev, struct stm_timer *stm_timer, + const char *name, void __iomem *base, int irq, + struct clk *clk, int cpu) { stm_timer->base = base; stm_timer->rate = clk_get_rate(clk); @@ -388,7 +386,7 @@ static irqreturn_t nxp_stm_module_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static int __init nxp_stm_timer_probe(struct platform_device *pdev) +static int nxp_stm_timer_probe(struct platform_device *pdev) { struct stm_timer *stm_timer; struct device *dev = &pdev->dev; @@ -484,14 +482,15 @@ static const struct of_device_id nxp_stm_of_match[] = { }; MODULE_DEVICE_TABLE(of, nxp_stm_of_match); -static struct platform_driver nxp_stm_probe = { +static struct platform_driver nxp_stm_driver = { .probe = nxp_stm_timer_probe, .driver = { .name = "nxp-stm", .of_match_table = nxp_stm_of_match, + .suppress_bind_attrs = true, }, }; -module_platform_driver(nxp_stm_probe); +builtin_platform_driver(nxp_stm_driver); MODULE_DESCRIPTION("NXP System Timer Module driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/clocksource/timer-ralink.c b/drivers/clocksource/timer-ralink.c index 6ecdb4228f763..68434d9ed9107 100644 --- a/drivers/clocksource/timer-ralink.c +++ b/drivers/clocksource/timer-ralink.c @@ -130,14 +130,15 @@ static int __init ralink_systick_init(struct device_node *np) systick.dev.irq = irq_of_parse_and_map(np, 0); if (!systick.dev.irq) { pr_err("%pOFn: request_irq failed", np); - return -EINVAL; + ret = -EINVAL; + goto err_iounmap; } ret = clocksource_mmio_init(systick.membase + SYSTICK_COUNT, np->name, SYSTICK_FREQ, 301, 16, clocksource_mmio_readl_up); if (ret) - return ret; + goto err_free_irq; clockevents_register_device(&systick.dev); @@ -145,6 +146,12 @@ static int __init ralink_systick_init(struct device_node *np) np, systick.dev.mult, systick.dev.shift); return 0; + +err_free_irq: + irq_dispose_mapping(systick.dev.irq); +err_iounmap: + iounmap(systick.membase); + return ret; } TIMER_OF_DECLARE(systick, "ralink,cevt-systick", ralink_systick_init); diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index 4d7cf338824a3..cfc4d83c42c03 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -50,8 +50,9 @@ static int riscv_clock_next_event(unsigned long delta, if (static_branch_likely(&riscv_sstc_available)) { #if defined(CONFIG_32BIT) - csr_write(CSR_STIMECMP, next_tval & 0xFFFFFFFF); + csr_write(CSR_STIMECMP, ULONG_MAX); csr_write(CSR_STIMECMPH, next_tval >> 32); + csr_write(CSR_STIMECMP, next_tval & 0xFFFFFFFF); #else csr_write(CSR_STIMECMP, next_tval); #endif diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c index 7e2f2b1a1c362..8c5507d78914f 100644 --- a/drivers/comedi/comedi_fops.c +++ b/drivers/comedi/comedi_fops.c @@ -1099,7 +1099,7 @@ static int do_chaninfo_ioctl(struct comedi_device *dev, for (i = 0; i < s->n_chan; i++) { int x; - x = (dev->minor << 28) | (it->subdev << 24) | (i << 16) | + x = (it->subdev << 24) | (i << 16) | (s->range_table_list[i]->length); if (put_user(x, it->rangelist + i)) return -EFAULT; @@ -3023,7 +3023,12 @@ static int compat_chaninfo(struct file *file, unsigned long arg) chaninfo.rangelist = compat_ptr(chaninfo32.rangelist); mutex_lock(&dev->mutex); - err = do_chaninfo_ioctl(dev, &chaninfo); + if (!dev->attached) { + dev_dbg(dev->class_dev, "no driver attached\n"); + err = -ENODEV; + } else { + err = do_chaninfo_ioctl(dev, &chaninfo); + } mutex_unlock(&dev->mutex); return err; } @@ -3044,7 +3049,12 @@ static int compat_rangeinfo(struct file *file, unsigned long arg) rangeinfo.range_ptr = compat_ptr(rangeinfo32.range_ptr); mutex_lock(&dev->mutex); - err = do_rangeinfo_ioctl(dev, &rangeinfo); + if (!dev->attached) { + dev_dbg(dev->class_dev, "no driver attached\n"); + err = -ENODEV; + } else { + err = do_rangeinfo_ioctl(dev, &rangeinfo); + } mutex_unlock(&dev->mutex); return err; } @@ -3120,7 +3130,12 @@ static int compat_cmd(struct file *file, unsigned long arg) return rc; mutex_lock(&dev->mutex); - rc = do_cmd_ioctl(dev, &cmd, ©, file); + if (!dev->attached) { + dev_dbg(dev->class_dev, "no driver attached\n"); + rc = -ENODEV; + } else { + rc = do_cmd_ioctl(dev, &cmd, ©, file); + } mutex_unlock(&dev->mutex); if (copy) { /* Special case: copy cmd back to user. */ @@ -3145,7 +3160,12 @@ static int compat_cmdtest(struct file *file, unsigned long arg) return rc; mutex_lock(&dev->mutex); - rc = do_cmdtest_ioctl(dev, &cmd, ©, file); + if (!dev->attached) { + dev_dbg(dev->class_dev, "no driver attached\n"); + rc = -ENODEV; + } else { + rc = do_cmdtest_ioctl(dev, &cmd, ©, file); + } mutex_unlock(&dev->mutex); if (copy) { err = put_compat_cmd(compat_ptr(arg), &cmd); @@ -3205,7 +3225,12 @@ static int compat_insnlist(struct file *file, unsigned long arg) } mutex_lock(&dev->mutex); - rc = do_insnlist_ioctl(dev, insns, insnlist32.n_insns, file); + if (!dev->attached) { + dev_dbg(dev->class_dev, "no driver attached\n"); + rc = -ENODEV; + } else { + rc = do_insnlist_ioctl(dev, insns, insnlist32.n_insns, file); + } mutex_unlock(&dev->mutex); kfree(insns); return rc; @@ -3224,7 +3249,12 @@ static int compat_insn(struct file *file, unsigned long arg) return rc; mutex_lock(&dev->mutex); - rc = do_insn_ioctl(dev, &insn, file); + if (!dev->attached) { + dev_dbg(dev->class_dev, "no driver attached\n"); + rc = -ENODEV; + } else { + rc = do_insn_ioctl(dev, &insn, file); + } mutex_unlock(&dev->mutex); return rc; } diff --git a/drivers/comedi/drivers.c b/drivers/comedi/drivers.c index c9ebaadc5e82a..2b0776c8342d9 100644 --- a/drivers/comedi/drivers.c +++ b/drivers/comedi/drivers.c @@ -1001,6 +1001,14 @@ int comedi_device_attach(struct comedi_device *dev, struct comedi_devconfig *it) ret = -EIO; goto out; } + if (IS_ENABLED(CONFIG_LOCKDEP)) { + /* + * dev->spinlock is for private use by the attached low-level + * driver. Reinitialize it to stop lock-dependency tracking + * between attachments to different low-level drivers. + */ + spin_lock_init(&dev->spinlock); + } dev->driver = driv; dev->board_name = dev->board_ptr ? *(const char **)dev->board_ptr : dev->driver->driver_name; diff --git a/drivers/comedi/drivers/c6xdigio.c b/drivers/comedi/drivers/c6xdigio.c index 14b90d1c64dc1..8a38d97d463b2 100644 --- a/drivers/comedi/drivers/c6xdigio.c +++ b/drivers/comedi/drivers/c6xdigio.c @@ -249,9 +249,6 @@ static int c6xdigio_attach(struct comedi_device *dev, if (ret) return ret; - /* Make sure that PnP ports get activated */ - pnp_register_driver(&c6xdigio_pnp_driver); - s = &dev->subdevices[0]; /* pwm output subdevice */ s->type = COMEDI_SUBD_PWM; @@ -278,19 +275,46 @@ static int c6xdigio_attach(struct comedi_device *dev, return 0; } -static void c6xdigio_detach(struct comedi_device *dev) -{ - comedi_legacy_detach(dev); - pnp_unregister_driver(&c6xdigio_pnp_driver); -} - static struct comedi_driver c6xdigio_driver = { .driver_name = "c6xdigio", .module = THIS_MODULE, .attach = c6xdigio_attach, - .detach = c6xdigio_detach, + .detach = comedi_legacy_detach, }; -module_comedi_driver(c6xdigio_driver); + +static bool c6xdigio_pnp_registered = false; + +static int __init c6xdigio_module_init(void) +{ + int ret; + + ret = comedi_driver_register(&c6xdigio_driver); + if (ret) + return ret; + + if (IS_ENABLED(CONFIG_PNP)) { + /* Try to activate the PnP ports */ + ret = pnp_register_driver(&c6xdigio_pnp_driver); + if (ret) { + pr_warn("failed to register pnp driver - err %d\n", + ret); + ret = 0; /* ignore the error. */ + } else { + c6xdigio_pnp_registered = true; + } + } + + return 0; +} +module_init(c6xdigio_module_init); + +static void __exit c6xdigio_module_exit(void) +{ + if (c6xdigio_pnp_registered) + pnp_unregister_driver(&c6xdigio_pnp_driver); + comedi_driver_unregister(&c6xdigio_driver); +} +module_exit(c6xdigio_module_exit); MODULE_AUTHOR("Comedi https://www.comedi.org"); MODULE_DESCRIPTION("Comedi driver for the C6x_DIGIO DSP daughter card"); diff --git a/drivers/comedi/drivers/dmm32at.c b/drivers/comedi/drivers/dmm32at.c index 644e3b643c796..910cd24b1bed5 100644 --- a/drivers/comedi/drivers/dmm32at.c +++ b/drivers/comedi/drivers/dmm32at.c @@ -330,6 +330,7 @@ static int dmm32at_ai_cmdtest(struct comedi_device *dev, static void dmm32at_setaitimer(struct comedi_device *dev, unsigned int nansec) { + unsigned long irq_flags; unsigned char lo1, lo2, hi2; unsigned short both2; @@ -342,6 +343,9 @@ static void dmm32at_setaitimer(struct comedi_device *dev, unsigned int nansec) /* set counter clocks to 10MHz, disable all aux dio */ outb(0, dev->iobase + DMM32AT_CTRDIO_CFG_REG); + /* serialize access to control register and paged registers */ + spin_lock_irqsave(&dev->spinlock, irq_flags); + /* get access to the clock regs */ outb(DMM32AT_CTRL_PAGE_8254, dev->iobase + DMM32AT_CTRL_REG); @@ -354,6 +358,8 @@ static void dmm32at_setaitimer(struct comedi_device *dev, unsigned int nansec) outb(lo2, dev->iobase + DMM32AT_CLK2); outb(hi2, dev->iobase + DMM32AT_CLK2); + spin_unlock_irqrestore(&dev->spinlock, irq_flags); + /* enable the ai conversion interrupt and the clock to start scans */ outb(DMM32AT_INTCLK_ADINT | DMM32AT_INTCLK_CLKEN | DMM32AT_INTCLK_CLKSEL, @@ -363,13 +369,19 @@ static void dmm32at_setaitimer(struct comedi_device *dev, unsigned int nansec) static int dmm32at_ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s) { struct comedi_cmd *cmd = &s->async->cmd; + unsigned long irq_flags; int ret; dmm32at_ai_set_chanspec(dev, s, cmd->chanlist[0], cmd->chanlist_len); + /* serialize access to control register and paged registers */ + spin_lock_irqsave(&dev->spinlock, irq_flags); + /* reset the interrupt just in case */ outb(DMM32AT_CTRL_INTRST, dev->iobase + DMM32AT_CTRL_REG); + spin_unlock_irqrestore(&dev->spinlock, irq_flags); + /* * wait for circuit to settle * we don't have the 'insn' here but it's not needed @@ -429,8 +441,13 @@ static irqreturn_t dmm32at_isr(int irq, void *d) comedi_handle_events(dev, s); } + /* serialize access to control register and paged registers */ + spin_lock(&dev->spinlock); + /* reset the interrupt */ outb(DMM32AT_CTRL_INTRST, dev->iobase + DMM32AT_CTRL_REG); + + spin_unlock(&dev->spinlock); return IRQ_HANDLED; } @@ -481,14 +498,25 @@ static int dmm32at_ao_insn_write(struct comedi_device *dev, static int dmm32at_8255_io(struct comedi_device *dev, int dir, int port, int data, unsigned long regbase) { + unsigned long irq_flags; + int ret; + + /* serialize access to control register and paged registers */ + spin_lock_irqsave(&dev->spinlock, irq_flags); + /* get access to the DIO regs */ outb(DMM32AT_CTRL_PAGE_8255, dev->iobase + DMM32AT_CTRL_REG); if (dir) { outb(data, dev->iobase + regbase + port); - return 0; + ret = 0; + } else { + ret = inb(dev->iobase + regbase + port); } - return inb(dev->iobase + regbase + port); + + spin_unlock_irqrestore(&dev->spinlock, irq_flags); + + return ret; } /* Make sure the board is there and put it to a known state */ diff --git a/drivers/comedi/drivers/dt2815.c b/drivers/comedi/drivers/dt2815.c index 03ba2fd18a216..d066dc303520b 100644 --- a/drivers/comedi/drivers/dt2815.c +++ b/drivers/comedi/drivers/dt2815.c @@ -175,6 +175,18 @@ static int dt2815_attach(struct comedi_device *dev, struct comedi_devconfig *it) ? current_range_type : voltage_range_type; } + /* + * Check if hardware is present before attempting any I/O operations. + * Reading 0xff from status register typically indicates no hardware + * on the bus (floating bus reads as all 1s). + */ + if (inb(dev->iobase + DT2815_STATUS) == 0xff) { + dev_err(dev->class_dev, + "No hardware detected at I/O base 0x%lx\n", + dev->iobase); + return -ENODEV; + } + /* Init the 2815 */ outb(0x00, dev->iobase + DT2815_STATUS); for (i = 0; i < 100; i++) { diff --git a/drivers/comedi/drivers/me4000.c b/drivers/comedi/drivers/me4000.c index 7dd3a00718630..effe9fdbbafe3 100644 --- a/drivers/comedi/drivers/me4000.c +++ b/drivers/comedi/drivers/me4000.c @@ -315,6 +315,18 @@ static int me4000_xilinx_download(struct comedi_device *dev, unsigned int val; unsigned int i; + /* Get data stream length from header. */ + if (size >= 4) { + file_length = (((unsigned int)data[0] & 0xff) << 24) + + (((unsigned int)data[1] & 0xff) << 16) + + (((unsigned int)data[2] & 0xff) << 8) + + ((unsigned int)data[3] & 0xff); + } + if (size < 16 || file_length > size - 16) { + dev_err(dev->class_dev, "Firmware length inconsistency\n"); + return -EINVAL; + } + if (!xilinx_iobase) return -ENODEV; @@ -346,10 +358,6 @@ static int me4000_xilinx_download(struct comedi_device *dev, outl(val, devpriv->plx_regbase + PLX9052_CNTRL); /* Download Xilinx firmware */ - file_length = (((unsigned int)data[0] & 0xff) << 24) + - (((unsigned int)data[1] & 0xff) << 16) + - (((unsigned int)data[2] & 0xff) << 8) + - ((unsigned int)data[3] & 0xff); usleep_range(10, 1000); for (i = 0; i < file_length; i++) { diff --git a/drivers/comedi/drivers/me_daq.c b/drivers/comedi/drivers/me_daq.c index 076b15097afd9..2f2ea029cffc2 100644 --- a/drivers/comedi/drivers/me_daq.c +++ b/drivers/comedi/drivers/me_daq.c @@ -344,6 +344,25 @@ static int me2600_xilinx_download(struct comedi_device *dev, unsigned int file_length; unsigned int i; + /* + * Format of the firmware + * Build longs from the byte-wise coded header + * Byte 1-3: length of the array + * Byte 4-7: version + * Byte 8-11: date + * Byte 12-15: reserved + */ + if (size >= 4) { + file_length = (((unsigned int)data[0] & 0xff) << 24) + + (((unsigned int)data[1] & 0xff) << 16) + + (((unsigned int)data[2] & 0xff) << 8) + + ((unsigned int)data[3] & 0xff); + } + if (size < 16 || file_length > size - 16) { + dev_err(dev->class_dev, "Firmware length inconsistency\n"); + return -EINVAL; + } + /* disable irq's on PLX */ writel(0x00, devpriv->plx_regbase + PLX9052_INTCSR); @@ -357,22 +376,6 @@ static int me2600_xilinx_download(struct comedi_device *dev, writeb(0x00, dev->mmio + 0x0); sleep(1); - /* - * Format of the firmware - * Build longs from the byte-wise coded header - * Byte 1-3: length of the array - * Byte 4-7: version - * Byte 8-11: date - * Byte 12-15: reserved - */ - if (size < 16) - return -EINVAL; - - file_length = (((unsigned int)data[0] & 0xff) << 24) + - (((unsigned int)data[1] & 0xff) << 16) + - (((unsigned int)data[2] & 0xff) << 8) + - ((unsigned int)data[3] & 0xff); - /* * Loop for writing firmware byte by byte to xilinx * Firmware data start at offset 16 diff --git a/drivers/comedi/drivers/multiq3.c b/drivers/comedi/drivers/multiq3.c index 07ff5383da997..ac369e9a262d7 100644 --- a/drivers/comedi/drivers/multiq3.c +++ b/drivers/comedi/drivers/multiq3.c @@ -67,6 +67,11 @@ #define MULTIQ3_TRSFRCNTR_OL 0x10 /* xfer CNTR to OL (x and y) */ #define MULTIQ3_EFLAG_RESET 0x06 /* reset E bit of flag reg */ +/* + * Limit on the number of optional encoder channels + */ +#define MULTIQ3_MAX_ENC_CHANS 8 + static void multiq3_set_ctrl(struct comedi_device *dev, unsigned int bits) { /* @@ -312,6 +317,10 @@ static int multiq3_attach(struct comedi_device *dev, s->insn_read = multiq3_encoder_insn_read; s->insn_config = multiq3_encoder_insn_config; + /* sanity check for number of encoder channels */ + if (s->n_chan > MULTIQ3_MAX_ENC_CHANS) + s->n_chan = MULTIQ3_MAX_ENC_CHANS; + for (i = 0; i < s->n_chan; i++) multiq3_encoder_reset(dev, i); diff --git a/drivers/comedi/drivers/ni_atmio16d.c b/drivers/comedi/drivers/ni_atmio16d.c index e5e7cc423c879..b057b3b3582e1 100644 --- a/drivers/comedi/drivers/ni_atmio16d.c +++ b/drivers/comedi/drivers/ni_atmio16d.c @@ -698,7 +698,8 @@ static int atmio16d_attach(struct comedi_device *dev, static void atmio16d_detach(struct comedi_device *dev) { - reset_atmio16d(dev); + if (dev->private) + reset_atmio16d(dev); comedi_legacy_detach(dev); } diff --git a/drivers/comedi/drivers/pcl818.c b/drivers/comedi/drivers/pcl818.c index 4127adcfb2295..06fe06396f23a 100644 --- a/drivers/comedi/drivers/pcl818.c +++ b/drivers/comedi/drivers/pcl818.c @@ -1111,10 +1111,9 @@ static void pcl818_detach(struct comedi_device *dev) { struct pcl818_private *devpriv = dev->private; - if (devpriv) { - pcl818_ai_cancel(dev, dev->read_subdev); + if (devpriv) pcl818_reset(dev); - } + pcl818_free_dma(dev); comedi_legacy_detach(dev); } diff --git a/drivers/comedi/range.c b/drivers/comedi/range.c index 8f43cf88d784b..5b8f662365e35 100644 --- a/drivers/comedi/range.c +++ b/drivers/comedi/range.c @@ -52,7 +52,7 @@ int do_rangeinfo_ioctl(struct comedi_device *dev, const struct comedi_lrange *lr; struct comedi_subdevice *s; - subd = (it->range_type >> 24) & 0xf; + subd = (it->range_type >> 24) & 0xff; chan = (it->range_type >> 16) & 0xff; if (!dev->attached) diff --git a/drivers/counter/104-quad-8.c b/drivers/counter/104-quad-8.c index ce81fc4e1ae76..573b2fe93253e 100644 --- a/drivers/counter/104-quad-8.c +++ b/drivers/counter/104-quad-8.c @@ -1192,6 +1192,7 @@ static irqreturn_t quad8_irq_handler(int irq, void *private) { struct counter_device *counter = private; struct quad8 *const priv = counter_priv(counter); + struct device *dev = counter->parent; unsigned int status; unsigned long irq_status; unsigned long channel; @@ -1200,8 +1201,11 @@ static irqreturn_t quad8_irq_handler(int irq, void *private) int ret; ret = regmap_read(priv->map, QUAD8_INTERRUPT_STATUS, &status); - if (ret) - return ret; + if (ret) { + dev_WARN_ONCE(dev, true, + "Attempt to read Interrupt Status Register failed: %d\n", ret); + return IRQ_NONE; + } if (!status) return IRQ_NONE; @@ -1223,8 +1227,9 @@ static irqreturn_t quad8_irq_handler(int irq, void *private) break; default: /* should never reach this path */ - WARN_ONCE(true, "invalid interrupt trigger function %u configured for channel %lu\n", - flg_pins, channel); + dev_WARN_ONCE(dev, true, + "invalid interrupt trigger function %u configured for channel %lu\n", + flg_pins, channel); continue; } @@ -1232,8 +1237,11 @@ static irqreturn_t quad8_irq_handler(int irq, void *private) } ret = regmap_write(priv->map, QUAD8_CHANNEL_OPERATION, CLEAR_PENDING_INTERRUPTS); - if (ret) - return ret; + if (ret) { + dev_WARN_ONCE(dev, true, + "Attempt to clear pending interrupts by writing to Channel Operation Register failed: %d\n", ret); + return IRQ_HANDLED; + } return IRQ_HANDLED; } diff --git a/drivers/counter/interrupt-cnt.c b/drivers/counter/interrupt-cnt.c index 6c0c1d2d7027d..e6100b5fb082e 100644 --- a/drivers/counter/interrupt-cnt.c +++ b/drivers/counter/interrupt-cnt.c @@ -229,8 +229,7 @@ static int interrupt_cnt_probe(struct platform_device *pdev) irq_set_status_flags(priv->irq, IRQ_NOAUTOEN); ret = devm_request_irq(dev, priv->irq, interrupt_cnt_isr, - IRQF_TRIGGER_RISING | IRQF_NO_THREAD, - dev_name(dev), counter); + IRQF_TRIGGER_RISING, dev_name(dev), counter); if (ret) return ret; diff --git a/drivers/counter/rz-mtu3-cnt.c b/drivers/counter/rz-mtu3-cnt.c index e755d54dfece9..7bfb6979193ce 100644 --- a/drivers/counter/rz-mtu3-cnt.c +++ b/drivers/counter/rz-mtu3-cnt.c @@ -107,9 +107,9 @@ static bool rz_mtu3_is_counter_invalid(struct counter_device *counter, int id) struct rz_mtu3_cnt *const priv = counter_priv(counter); unsigned long tmdr; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); tmdr = rz_mtu3_shared_reg_read(priv->ch, RZ_MTU3_TMDR3); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); if (id == RZ_MTU3_32_BIT_CH && test_bit(RZ_MTU3_TMDR3_LWA, &tmdr)) return false; @@ -165,12 +165,12 @@ static int rz_mtu3_count_read(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); if (count->id == RZ_MTU3_32_BIT_CH) *val = rz_mtu3_32bit_ch_read(ch, RZ_MTU3_TCNTLW); else *val = rz_mtu3_16bit_ch_read(ch, RZ_MTU3_TCNT); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -187,26 +187,26 @@ static int rz_mtu3_count_write(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); if (count->id == RZ_MTU3_32_BIT_CH) rz_mtu3_32bit_ch_write(ch, RZ_MTU3_TCNTLW, val); else rz_mtu3_16bit_ch_write(ch, RZ_MTU3_TCNT, val); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; } static int rz_mtu3_count_function_read_helper(struct rz_mtu3_channel *const ch, - struct rz_mtu3_cnt *const priv, + struct counter_device *const counter, enum counter_function *function) { u8 timer_mode; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); timer_mode = rz_mtu3_8bit_ch_read(ch, RZ_MTU3_TMDR1); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); switch (timer_mode & RZ_MTU3_TMDR1_PH_CNT_MODE_MASK) { case RZ_MTU3_TMDR1_PH_CNT_MODE_1: @@ -240,7 +240,7 @@ static int rz_mtu3_count_function_read(struct counter_device *counter, if (ret) return ret; - ret = rz_mtu3_count_function_read_helper(ch, priv, function); + ret = rz_mtu3_count_function_read_helper(ch, counter, function); mutex_unlock(&priv->lock); return ret; @@ -279,9 +279,9 @@ static int rz_mtu3_count_function_write(struct counter_device *counter, return -EINVAL; } - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); rz_mtu3_8bit_ch_write(ch, RZ_MTU3_TMDR1, timer_mode); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -300,9 +300,9 @@ static int rz_mtu3_count_direction_read(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); tsr = rz_mtu3_8bit_ch_read(ch, RZ_MTU3_TSR); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); *direction = (tsr & RZ_MTU3_TSR_TCFD) ? COUNTER_COUNT_DIRECTION_FORWARD : COUNTER_COUNT_DIRECTION_BACKWARD; @@ -377,14 +377,14 @@ static int rz_mtu3_count_ceiling_write(struct counter_device *counter, return -EINVAL; } - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); if (count->id == RZ_MTU3_32_BIT_CH) rz_mtu3_32bit_ch_write(ch, RZ_MTU3_TGRALW, ceiling); else rz_mtu3_16bit_ch_write(ch, RZ_MTU3_TGRA, ceiling); rz_mtu3_8bit_ch_write(ch, RZ_MTU3_TCR, RZ_MTU3_TCR_CCLR_TGRA); - pm_runtime_put(ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -495,25 +495,28 @@ static int rz_mtu3_count_enable_read(struct counter_device *counter, static int rz_mtu3_count_enable_write(struct counter_device *counter, struct counter_count *count, u8 enable) { - struct rz_mtu3_channel *const ch = rz_mtu3_get_ch(counter, count->id); struct rz_mtu3_cnt *const priv = counter_priv(counter); int ret = 0; + mutex_lock(&priv->lock); + + if (priv->count_is_enabled[count->id] == enable) + goto exit; + if (enable) { - mutex_lock(&priv->lock); - pm_runtime_get_sync(ch->dev); + pm_runtime_get_sync(counter->parent); ret = rz_mtu3_initialize_counter(counter, count->id); if (ret == 0) priv->count_is_enabled[count->id] = true; - mutex_unlock(&priv->lock); } else { - mutex_lock(&priv->lock); rz_mtu3_terminate_counter(counter, count->id); priv->count_is_enabled[count->id] = false; - pm_runtime_put(ch->dev); - mutex_unlock(&priv->lock); + pm_runtime_put(counter->parent); } +exit: + mutex_unlock(&priv->lock); + return ret; } @@ -540,9 +543,9 @@ static int rz_mtu3_cascade_counts_enable_get(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); tmdr = rz_mtu3_shared_reg_read(priv->ch, RZ_MTU3_TMDR3); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); *cascade_enable = test_bit(RZ_MTU3_TMDR3_LWA, &tmdr); mutex_unlock(&priv->lock); @@ -559,10 +562,10 @@ static int rz_mtu3_cascade_counts_enable_set(struct counter_device *counter, if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); rz_mtu3_shared_reg_update_bit(priv->ch, RZ_MTU3_TMDR3, RZ_MTU3_TMDR3_LWA, cascade_enable); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -579,9 +582,9 @@ static int rz_mtu3_ext_input_phase_clock_select_get(struct counter_device *count if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); tmdr = rz_mtu3_shared_reg_read(priv->ch, RZ_MTU3_TMDR3); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); *ext_input_phase_clock_select = test_bit(RZ_MTU3_TMDR3_PHCKSEL, &tmdr); mutex_unlock(&priv->lock); @@ -598,11 +601,11 @@ static int rz_mtu3_ext_input_phase_clock_select_set(struct counter_device *count if (ret) return ret; - pm_runtime_get_sync(priv->ch->dev); + pm_runtime_get_sync(counter->parent); rz_mtu3_shared_reg_update_bit(priv->ch, RZ_MTU3_TMDR3, RZ_MTU3_TMDR3_PHCKSEL, ext_input_phase_clock_select); - pm_runtime_put(priv->ch->dev); + pm_runtime_put(counter->parent); mutex_unlock(&priv->lock); return 0; @@ -640,7 +643,7 @@ static int rz_mtu3_action_read(struct counter_device *counter, if (ret) return ret; - ret = rz_mtu3_count_function_read_helper(ch, priv, &function); + ret = rz_mtu3_count_function_read_helper(ch, counter, &function); if (ret) { mutex_unlock(&priv->lock); return ret; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b44f0f7a5ba1c..602e4fa81d6c5 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1282,7 +1282,7 @@ static int amd_pstate_change_mode_without_dvr_change(int mode) if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) return 0; - for_each_present_cpu(cpu) { + for_each_online_cpu(cpu) { cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); } diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index cd1816a12bb99..47dd76acd31e5 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -87,6 +87,7 @@ static const struct of_device_id allowlist[] __initconst = { { .compatible = "st-ericsson,u9540", }, { .compatible = "starfive,jh7110", }, + { .compatible = "starfive,jh7110s", }, { .compatible = "ti,omap2", }, { .compatible = "ti,omap4", }, @@ -168,8 +169,11 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,sdm845", }, { .compatible = "qcom,sdx75", }, { .compatible = "qcom,sm6115", }, + { .compatible = "qcom,sm6125", }, + { .compatible = "qcom,sm6150", }, { .compatible = "qcom,sm6350", }, { .compatible = "qcom,sm6375", }, + { .compatible = "qcom,sm7125", }, { .compatible = "qcom,sm7225", }, { .compatible = "qcom,sm7325", }, { .compatible = "qcom,sm8150", }, diff --git a/drivers/cpufreq/cpufreq-nforce2.c b/drivers/cpufreq/cpufreq-nforce2.c index fedad1081973f..fbbbe501cf2dc 100644 --- a/drivers/cpufreq/cpufreq-nforce2.c +++ b/drivers/cpufreq/cpufreq-nforce2.c @@ -145,6 +145,8 @@ static unsigned int nforce2_fsb_read(int bootfsb) pci_read_config_dword(nforce2_sub5, NFORCE2_BOOTFSB, &fsb); fsb /= 1000000; + pci_dev_put(nforce2_sub5); + /* Check if PLL register is already set */ pci_read_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8 *)&temp); @@ -426,6 +428,7 @@ static int __init nforce2_init(void) static void __exit nforce2_exit(void) { cpufreq_unregister_driver(&nforce2_driver); + pci_dev_put(nforce2_dev); } module_init(nforce2_init); diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index cce6a8d113e12..305a32b6b302e 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -313,6 +313,17 @@ static void cs_start(struct cpufreq_policy *policy) dbs_info->requested_freq = policy->cur; } +static void cs_limits(struct cpufreq_policy *policy) +{ + struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); + + /* + * The limits have changed, so may have the current frequency. Reset + * requested_freq to avoid any unintended outcomes due to the mismatch. + */ + dbs_info->requested_freq = policy->cur; +} + static struct dbs_governor cs_governor = { .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), .kobj_type = { .default_groups = cs_groups }, @@ -322,6 +333,7 @@ static struct dbs_governor cs_governor = { .init = cs_init, .exit = cs_exit, .start = cs_start, + .limits = cs_limits, }; #define CPU_FREQ_GOV_CONSERVATIVE (cs_governor.gov) diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 1a7fcaf39cc9b..00a26fa6292c5 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -468,13 +468,13 @@ int cpufreq_dbs_governor_init(struct cpufreq_policy *policy) /* Failure, so roll back. */ pr_err("initialization failed (dbs_data kobject init error %d)\n", ret); - kobject_put(&dbs_data->attr_set.kobj); - policy->governor_data = NULL; if (!have_governor_per_policy()) gov->gdbs_data = NULL; - gov->exit(dbs_data); + + kobject_put(&dbs_data->attr_set.kobj); + goto free_policy_dbs_info; free_dbs_data: kfree(dbs_data); @@ -563,6 +563,7 @@ EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop); void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) { + struct dbs_governor *gov = dbs_governor_of(policy); struct policy_dbs_info *policy_dbs; /* Protect gov->gdbs_data against cpufreq_dbs_governor_exit() */ @@ -574,6 +575,8 @@ void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) mutex_lock(&policy_dbs->update_mutex); cpufreq_policy_apply_limits(policy); gov_update_sample_delay(policy_dbs, 0); + if (gov->limits) + gov->limits(policy); mutex_unlock(&policy_dbs->update_mutex); out: diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index 168c23fd7fcac..1462d59277bd1 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -138,6 +138,7 @@ struct dbs_governor { int (*init)(struct dbs_data *dbs_data); void (*exit)(struct dbs_data *dbs_data); void (*start)(struct cpufreq_policy *policy); + void (*limits)(struct cpufreq_policy *policy); }; static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 492a10f1bdbfa..5efda8af4b708 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1152,7 +1152,7 @@ static void hybrid_init_cpu_capacity_scaling(bool refresh) * the capacity of SMT threads is not deterministic even approximately, * do not do that when SMT is in use. */ - if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { + if (hwp_is_hybrid && !cpu_smt_possible() && arch_enable_hybrid_capacity_scale()) { hybrid_refresh_cpu_capacity_scaling(); /* * Disabling ITMT causes sched domains to be rebuilt to disable asym @@ -1467,13 +1467,13 @@ static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, refresh_frequency_limits(policy); } -static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +static bool intel_pstate_update_max_freq(int cpu) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); if (!policy) return false; - __intel_pstate_update_max_freq(policy, cpudata); + __intel_pstate_update_max_freq(policy, all_cpu_data[cpu]); return true; } @@ -1492,7 +1492,7 @@ static void intel_pstate_update_limits_for_all(void) int cpu; for_each_possible_cpu(cpu) - intel_pstate_update_max_freq(all_cpu_data[cpu]); + intel_pstate_update_max_freq(cpu); mutex_lock(&hybrid_capacity_lock); @@ -1663,8 +1663,8 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, static void update_cpu_qos_request(int cpu, enum freq_qos_req_type type) { struct cpudata *cpudata = all_cpu_data[cpu]; - unsigned int freq = cpudata->pstate.turbo_freq; struct freq_qos_request *req; + unsigned int freq; struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); if (!policy) @@ -1677,6 +1677,8 @@ static void update_cpu_qos_request(int cpu, enum freq_qos_req_type type) if (hwp_active) intel_pstate_get_hwp_cap(cpudata); + freq = cpudata->pstate.turbo_freq; + if (type == FREQ_QOS_MIN) { freq = DIV_ROUND_UP(freq * global.min_perf_pct, 100); } else { @@ -1932,7 +1934,7 @@ static void intel_pstate_notify_work(struct work_struct *work) struct cpudata *cpudata = container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); - if (intel_pstate_update_max_freq(cpudata)) { + if (intel_pstate_update_max_freq(cpudata->cpu)) { /* * The driver will not be unregistered while this function is * running, so update the capacity without acquiring the driver diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index 4215621deb3fe..ba8a1c96427a1 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -518,7 +518,7 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) if (policy->cpu != 0) { ret = -EINVAL; - goto out_dmc1; + goto out; } /* @@ -530,7 +530,7 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) if ((mem_type != LPDDR) && (mem_type != LPDDR2)) { pr_err("CPUFreq doesn't support this memory type\n"); ret = -EINVAL; - goto out_dmc1; + goto out; } /* Find current refresh counter and frequency each DMC */ @@ -544,6 +544,8 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) cpufreq_generic_init(policy, s5pv210_freq_table, 40000); return 0; +out: + clk_put(dmc1_clk); out_dmc1: clk_put(dmc0_clk); out_dmc0: diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index d2a110079f5fd..c450cf9c881d4 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -101,6 +101,7 @@ static int scmi_cpu_domain_id(struct device *cpu_dev) return -EINVAL; } + of_node_put(domain_id.np); return domain_id.args[0]; } diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 56132e843c991..2ac0850abad84 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -222,6 +222,9 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP); ktime_t time_start, time_end; + if (cpus_peek_for_pending_ipi(cpumask_of(dev->cpu))) + return -EBUSY; + instrumentation_begin(); /* diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 23239b0c04f95..ca863ba03d454 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -239,7 +239,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, /* Find the shortest expected idle interval. */ predicted_ns = get_typical_interval(data) * NSEC_PER_USEC; - if (predicted_ns > RESIDENCY_THRESHOLD_NS) { + if (predicted_ns > RESIDENCY_THRESHOLD_NS || tick_nohz_tick_stopped()) { unsigned int timer_us; /* Determine the time till the closest timer. */ @@ -259,6 +259,16 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, RESOLUTION * DECAY * NSEC_PER_USEC); /* Use the lowest expected idle interval to pick the idle state. */ predicted_ns = min((u64)timer_us * NSEC_PER_USEC, predicted_ns); + /* + * If the tick is already stopped, the cost of possible short + * idle duration misprediction is much higher, because the CPU + * may be stuck in a shallow idle state for a long time as a + * result of it. In that case, say we might mispredict and use + * the known time till the closest timer event for the idle + * state selection. + */ + if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) + predicted_ns = data->next_timer_ns; } else { /* * Because the next timer event is not going to be determined @@ -284,16 +294,6 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, return 0; } - /* - * If the tick is already stopped, the cost of possible short idle - * duration misprediction is much higher, because the CPU may be stuck - * in a shallow idle state for a long time as a result of it. In that - * case, say we might mispredict and use the known time till the closest - * timer event for the idle state selection. - */ - if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) - predicted_ns = data->next_timer_ns; - /* * Find the idle state with the lowest power while satisfying * our constraints. @@ -317,12 +317,13 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, } /* - * Use a physical idle state, not busy polling, unless a timer - * is going to trigger soon enough or the exit latency of the - * idle state in question is greater than the predicted idle - * duration. + * Use a physical idle state instead of busy polling so long as + * its target residency is below the residency threshold, its + * exit latency is not greater than the predicted idle duration, + * and the next timer doesn't expire soon. */ if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && + s->target_residency_ns < RESIDENCY_THRESHOLD_NS && s->target_residency_ns <= data->next_timer_ns && s->exit_latency_ns <= predicted_ns) { predicted_ns = s->target_residency_ns; diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c index bfa55c1eab5bc..8078eca72ca53 100644 --- a/drivers/cpuidle/governors/teo.c +++ b/drivers/cpuidle/governors/teo.c @@ -458,11 +458,8 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, * If the closest expected timer is before the target residency of the * candidate state, a shallower one needs to be found. */ - if (drv->states[idx].target_residency_ns > duration_ns) { - i = teo_find_shallower_state(drv, dev, idx, duration_ns, false); - if (teo_state_ok(i, drv)) - idx = i; - } + if (drv->states[idx].target_residency_ns > duration_ns) + idx = teo_find_shallower_state(drv, dev, idx, duration_ns, false); /* * If the selected state's target residency is below the tick length diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index 0fcf4a39de279..a12653a658699 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -52,9 +52,10 @@ static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data, rng->priv = 0; } else { work_data = kmalloc(sizeof(*work_data), GFP_ATOMIC); - if (!work_data) + if (!work_data) { + atomic_dec(&i2c_priv->tfm_count); return -ENOMEM; - + } work_data->ctx = i2c_priv; work_data->client = i2c_priv->client; diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 107ccb2ade420..07665494c8758 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -3326,9 +3326,10 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, if (aligned_len < keylen) return -EOVERFLOW; - hashed_key = kmemdup(key, aligned_len, GFP_KERNEL); + hashed_key = kmalloc(aligned_len, GFP_KERNEL); if (!hashed_key) return -ENOMEM; + memcpy(hashed_key, key, keylen); ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); if (ret) goto bad_free_key; @@ -4814,7 +4815,8 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) { struct device *dev = priv->dev; struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); - int err; + struct dpaa2_caam_priv_per_cpu *ppriv; + int i, err; if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); @@ -4822,6 +4824,12 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) dev_err(dev, "dpseci_reset() failed\n"); } + for_each_cpu(i, priv->clean_mask) { + ppriv = per_cpu_ptr(priv->ppriv, i); + free_netdev(ppriv->net_dev); + } + free_cpumask_var(priv->clean_mask); + dpaa2_dpseci_congestion_free(priv); dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); } @@ -5007,16 +5015,15 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) struct device *dev = &ls_dev->dev; struct dpaa2_caam_priv *priv; struct dpaa2_caam_priv_per_cpu *ppriv; - cpumask_var_t clean_mask; int err, cpu; u8 i; err = -ENOMEM; - if (!zalloc_cpumask_var(&clean_mask, GFP_KERNEL)) - goto err_cpumask; - priv = dev_get_drvdata(dev); + if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) + goto err_cpumask; + priv->dev = dev; priv->dpsec_id = ls_dev->obj_desc.id; @@ -5118,7 +5125,7 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) err = -ENOMEM; goto err_alloc_netdev; } - cpumask_set_cpu(cpu, clean_mask); + cpumask_set_cpu(cpu, priv->clean_mask); ppriv->net_dev->dev = *dev; netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, @@ -5126,18 +5133,16 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) DPAA2_CAAM_NAPI_WEIGHT); } - err = 0; - goto free_cpumask; + return 0; err_alloc_netdev: - free_dpaa2_pcpu_netdev(priv, clean_mask); + free_dpaa2_pcpu_netdev(priv, priv->clean_mask); err_get_rx_queue: dpaa2_dpseci_congestion_free(priv); err_get_vers: dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); err_open: -free_cpumask: - free_cpumask_var(clean_mask); + free_cpumask_var(priv->clean_mask); err_cpumask: return err; } @@ -5182,7 +5187,6 @@ static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) ppriv = per_cpu_ptr(priv->ppriv, i); napi_disable(&ppriv->napi); netif_napi_del(&ppriv->napi); - free_netdev(ppriv->net_dev); } return 0; diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h index 61d1219a202fc..8e65b4b28c7ba 100644 --- a/drivers/crypto/caam/caamalg_qi2.h +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -42,6 +42,7 @@ * @mc_io: pointer to MC portal's I/O object * @domain: IOMMU domain * @ppriv: per CPU pointers to privata data + * @clean_mask: CPU mask of CPUs that have allocated netdevs */ struct dpaa2_caam_priv { int dpsec_id; @@ -65,6 +66,7 @@ struct dpaa2_caam_priv { struct dpaa2_caam_priv_per_cpu __percpu *ppriv; struct dentry *dfs_root; + cpumask_var_t clean_mask; }; /** diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 25c02e2672585..053af748be86d 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -441,9 +441,10 @@ static int ahash_setkey(struct crypto_ahash *ahash, if (aligned_len < keylen) return -EOVERFLOW; - hashed_key = kmemdup(key, keylen, GFP_KERNEL); + hashed_key = kmalloc(aligned_len, GFP_KERNEL); if (!hashed_key) return -ENOMEM; + memcpy(hashed_key, key, keylen); ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); if (ret) goto bad_free_key; diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index b3d14a7f4dd14..0eb43c8625164 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -181,7 +181,9 @@ static inline void test_len(struct hwrng *rng, size_t len, bool wait) struct device *dev = ctx->ctrldev; buf = kcalloc(CAAM_RNG_MAX_FIFO_STORE_SIZE, sizeof(u8), GFP_KERNEL); - + if (!buf) { + return; + } while (len > 0) { read_len = rng->read(rng, buf, len, wait); diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index c246920e6f540..bccd680c7f7ee 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -180,7 +180,8 @@ static void free_command_queues(struct cpt_vf *cptvf, hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, nextchunk) { - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index d78865d9d5f09..d0412e5847625 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -642,7 +642,7 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) struct ccp_data dst; struct ccp_data aad; struct ccp_op op; - } *wa __cleanup(kfree) = kzalloc(sizeof *wa, GFP_KERNEL); + } *wa __free(kfree) = kzalloc(sizeof(*wa), GFP_KERNEL); unsigned int dm_offset; unsigned int authsize; unsigned int jobid; diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 9e21da0e298ad..5c7f7e02a7d8a 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -351,6 +351,17 @@ struct psp_device *psp_get_master_device(void) return sp ? sp->psp_data : NULL; } +int psp_restore(struct sp_device *sp) +{ + struct psp_device *psp = sp->psp_data; + int ret = 0; + + if (psp->tee_data) + ret = tee_restore(psp); + + return ret; +} + void psp_pci_init(void) { psp_master = psp_get_master_device(); diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 0d13d47c164bb..5c7a5ff2d9eaa 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -119,13 +119,6 @@ static size_t sev_es_tmr_size = SEV_TMR_SIZE; #define NV_LENGTH (32 * 1024) static void *sev_init_ex_buffer; -/* - * SEV_DATA_RANGE_LIST: - * Array containing range of pages that firmware transitions to HV-fixed - * page state. - */ -static struct sev_data_range_list *snp_range_list; - static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); static int snp_shutdown_on_panic(struct notifier_block *nb, @@ -1116,15 +1109,12 @@ struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages) { struct psp_device *psp_master = psp_get_master_device(); struct snp_hv_fixed_pages_entry *entry; - struct sev_device *sev; unsigned int order; struct page *page; - if (!psp_master || !psp_master->sev_data) + if (!psp_master) return NULL; - sev = psp_master->sev_data; - order = get_order(PMD_SIZE * num_2mb_pages); /* @@ -1137,7 +1127,8 @@ struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages) * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed * page state, fail if SNP is already initialized. */ - if (sev->snp_initialized) + if (psp_master->sev_data && + ((struct sev_device *)psp_master->sev_data)->snp_initialized) return NULL; /* Re-use freed pages that match the request */ @@ -1173,7 +1164,7 @@ void snp_free_hv_fixed_pages(struct page *page) struct psp_device *psp_master = psp_get_master_device(); struct snp_hv_fixed_pages_entry *entry, *nentry; - if (!psp_master || !psp_master->sev_data) + if (!psp_master) return; /* @@ -1365,6 +1356,7 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { + struct sev_data_range_list *snp_range_list __free(kfree) = NULL; struct psp_device *psp = psp_master; struct sev_data_snp_init_ex data; struct sev_device *sev; @@ -1841,7 +1833,10 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error); - /* If we query the CSR length, FW responded with expected data. */ + /* + * Firmware will returns the length of the CSR blob (either the minimum + * required length or the actual length written), return it to the user. + */ input.length = data.len; if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { @@ -1849,6 +1844,9 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) goto e_free_blob; } + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free_blob; + if (blob) { if (copy_to_user(input_address, blob, input.length)) ret = -EFAULT; @@ -2199,6 +2197,9 @@ static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) goto e_free; } + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free; + if (id_blob) { if (copy_to_user(input_address, id_blob, data.len)) { ret = -EFAULT; @@ -2315,7 +2316,10 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error); - /* If we query the length, FW responded with expected data. */ + /* + * Firmware will return the length of the blobs (either the minimum + * required length or the actual length written), return 'em to the user. + */ input.cert_chain_len = data.cert_chain_len; input.pdh_cert_len = data.pdh_cert_len; @@ -2324,6 +2328,9 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) goto e_free_cert; } + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free_cert; + if (pdh_blob) { if (copy_to_user(input_pdh_cert_address, pdh_blob, input.pdh_cert_len)) { @@ -2351,11 +2358,10 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; - bool shutdown_required = false; struct sev_data_snp_addr buf; struct page *status_page; - int ret, error; void *data; + int ret; if (!argp->data) return -EINVAL; @@ -2366,31 +2372,33 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) data = page_address(status_page); - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - goto cleanup; - } - /* - * Firmware expects status page to be in firmware-owned state, otherwise - * it will report firmware error code INVALID_PAGE_STATE (0x1A). + * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed + * when SNP has been initialized, the status page must be firmware-owned. */ - if (rmp_mark_pages_firmware(__pa(data), 1, true)) { - ret = -EFAULT; - goto cleanup; + if (sev->snp_initialized) { + /* + * Firmware expects the status page to be in Firmware state, + * otherwise it will report an error INVALID_PAGE_STATE. + */ + if (rmp_mark_pages_firmware(__pa(data), 1, true)) { + ret = -EFAULT; + goto cleanup; + } } buf.address = __psp_pa(data); ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error); - /* - * Status page will be transitioned to Reclaim state upon success, or - * left in Firmware state in failure. Use snp_reclaim_pages() to - * transition either case back to Hypervisor-owned state. - */ - if (snp_reclaim_pages(__pa(data), 1, true)) - return -EFAULT; + if (sev->snp_initialized) { + /* + * The status page will be in Reclaim state on success, or left + * in Firmware state on failure. Use snp_reclaim_pages() to + * transition either case back to Hypervisor-owned state. + */ + if (snp_reclaim_pages(__pa(data), 1, true)) + return -EFAULT; + } if (ret) goto cleanup; @@ -2400,9 +2408,6 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) ret = -EFAULT; cleanup: - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - __free_pages(status_page, 0); return ret; } @@ -2753,11 +2758,6 @@ static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) sev_init_ex_buffer = NULL; } - if (snp_range_list) { - kfree(snp_range_list); - snp_range_list = NULL; - } - __sev_snp_shutdown_locked(&error, panic); } diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c index 3467f6db4f505..f204aa5df96e2 100644 --- a/drivers/crypto/ccp/sp-dev.c +++ b/drivers/crypto/ccp/sp-dev.c @@ -230,6 +230,18 @@ int sp_resume(struct sp_device *sp) return 0; } +int sp_restore(struct sp_device *sp) +{ + if (sp->psp_data) { + int ret = psp_restore(sp); + + if (ret) + return ret; + } + + return sp_resume(sp); +} + struct sp_device *sp_get_psp_master_device(void) { struct sp_device *i, *ret = NULL; diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 6f9d7063257d7..c8a611ef275b5 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -141,6 +141,7 @@ void sp_destroy(struct sp_device *sp); int sp_suspend(struct sp_device *sp); int sp_resume(struct sp_device *sp); +int sp_restore(struct sp_device *sp); int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, const char *name, void *data); void sp_free_ccp_irq(struct sp_device *sp, void *data); @@ -174,6 +175,7 @@ int psp_dev_init(struct sp_device *sp); void psp_pci_init(void); void psp_dev_destroy(struct sp_device *sp); void psp_pci_exit(void); +int psp_restore(struct sp_device *sp); #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ @@ -181,6 +183,7 @@ static inline int psp_dev_init(struct sp_device *sp) { return 0; } static inline void psp_pci_init(void) { } static inline void psp_dev_destroy(struct sp_device *sp) { } static inline void psp_pci_exit(void) { } +static inline int psp_restore(struct sp_device *sp) { return 0; } #endif /* CONFIG_CRYPTO_DEV_SP_PSP */ diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index e7bb803912a6d..6ac805d99ccb3 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -353,6 +353,13 @@ static int __maybe_unused sp_pci_resume(struct device *dev) return sp_resume(sp); } +static int __maybe_unused sp_pci_restore(struct device *dev) +{ + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_restore(sp); +} + #ifdef CONFIG_CRYPTO_DEV_SP_PSP static const struct sev_vdata sevv1 = { .cmdresp_reg = 0x10580, /* C2PMSG_32 */ @@ -459,6 +466,17 @@ static const struct psp_vdata pspv6 = { .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ }; +static const struct psp_vdata pspv7 = { + .tee = &teev2, + .cmdresp_reg = 0x10944, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10510, /* P2CMSG_INTEN */ + .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ +}; + #endif static const struct sp_dev_vdata dev_vdata[] = { @@ -525,6 +543,13 @@ static const struct sp_dev_vdata dev_vdata[] = { .psp_vdata = &pspv6, #endif }, + { /* 9 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv7, +#endif + }, + }; static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] }, @@ -539,12 +564,20 @@ static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] }, { PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x115A), (kernel_ulong_t)&dev_vdata[9] }, /* Last entry must be zero */ { 0, } }; MODULE_DEVICE_TABLE(pci, sp_pci_table); -static SIMPLE_DEV_PM_OPS(sp_pci_pm_ops, sp_pci_suspend, sp_pci_resume); +static const struct dev_pm_ops sp_pci_pm_ops = { + .suspend = pm_sleep_ptr(sp_pci_suspend), + .resume = pm_sleep_ptr(sp_pci_resume), + .freeze = pm_sleep_ptr(sp_pci_suspend), + .thaw = pm_sleep_ptr(sp_pci_resume), + .poweroff = pm_sleep_ptr(sp_pci_suspend), + .restore_early = pm_sleep_ptr(sp_pci_restore), +}; static struct pci_driver sp_pci_driver = { .name = "ccp", diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c index 5e1d80724678d..92ffa412622a2 100644 --- a/drivers/crypto/ccp/tee-dev.c +++ b/drivers/crypto/ccp/tee-dev.c @@ -86,10 +86,34 @@ static inline void tee_free_cmd_buffer(struct tee_init_ring_cmd *cmd) kfree(cmd); } +static bool tee_send_destroy_cmd(struct psp_tee_device *tee) +{ + unsigned int reg; + int ret; + + ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, + TEE_DEFAULT_CMD_TIMEOUT, ®); + if (ret) { + dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); + psp_dead = true; + return false; + } + + if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", + FIELD_GET(PSP_CMDRESP_STS, reg)); + psp_dead = true; + return false; + } + + return true; +} + static int tee_init_ring(struct psp_tee_device *tee) { int ring_size = MAX_RING_BUFFER_ENTRIES * sizeof(struct tee_ring_cmd); struct tee_init_ring_cmd *cmd; + bool retry = false; unsigned int reg; int ret; @@ -112,6 +136,7 @@ static int tee_init_ring(struct psp_tee_device *tee) /* Send command buffer details to Trusted OS by writing to * CPU-PSP message registers */ +retry_init: ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_INIT, cmd, TEE_DEFAULT_CMD_TIMEOUT, ®); if (ret) { @@ -122,9 +147,22 @@ static int tee_init_ring(struct psp_tee_device *tee) } if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + /* + * During the hibernate resume sequence driver may have gotten loaded + * but the ring not properly destroyed. If the ring doesn't work, try + * to destroy and re-init once. + */ + if (!retry && FIELD_GET(PSP_CMDRESP_STS, reg) == PSP_TEE_STS_RING_BUSY) { + dev_info(tee->dev, "tee: ring init command failed with busy status, retrying\n"); + if (tee_send_destroy_cmd(tee)) { + retry = true; + goto retry_init; + } + } dev_err(tee->dev, "tee: ring init command failed (%#010lx)\n", FIELD_GET(PSP_CMDRESP_STS, reg)); tee_free_ring(tee); + psp_dead = true; ret = -EIO; } @@ -136,24 +174,13 @@ static int tee_init_ring(struct psp_tee_device *tee) static void tee_destroy_ring(struct psp_tee_device *tee) { - unsigned int reg; - int ret; - if (!tee->rb_mgr.ring_start) return; if (psp_dead) goto free_ring; - ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, - TEE_DEFAULT_CMD_TIMEOUT, ®); - if (ret) { - dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); - psp_dead = true; - } else if (FIELD_GET(PSP_CMDRESP_STS, reg)) { - dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", - FIELD_GET(PSP_CMDRESP_STS, reg)); - } + tee_send_destroy_cmd(tee); free_ring: tee_free_ring(tee); @@ -365,3 +392,8 @@ int psp_check_tee_status(void) return 0; } EXPORT_SYMBOL(psp_check_tee_status); + +int tee_restore(struct psp_device *psp) +{ + return tee_init_ring(psp->tee_data); +} diff --git a/drivers/crypto/ccp/tee-dev.h b/drivers/crypto/ccp/tee-dev.h index ea9a2b7c05f57..c23416cb7bb37 100644 --- a/drivers/crypto/ccp/tee-dev.h +++ b/drivers/crypto/ccp/tee-dev.h @@ -111,5 +111,6 @@ struct tee_ring_cmd { int tee_dev_init(struct psp_device *psp); void tee_dev_destroy(struct psp_device *psp); +int tee_restore(struct psp_device *psp); #endif /* __TEE_DEV_H__ */ diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index 3963bb91321fc..dc7e0cd51c259 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -1235,6 +1235,7 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, int rc = 0; u32 dummy = 0; u32 mapped_nents = 0; + int sg_nents; dev_dbg(dev, " update params : curr_buff=%p curr_buff_cnt=0x%X nbytes=0x%X src=%p curr_index=%u\n", curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); @@ -1248,7 +1249,10 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, if (total_in_len < block_size) { dev_dbg(dev, " less than one block: curr_buff=%p *curr_buff_cnt=0x%X copy_to=%p\n", curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]); - areq_ctx->in_nents = sg_nents_for_len(src, nbytes); + sg_nents = sg_nents_for_len(src, nbytes); + if (sg_nents < 0) + return sg_nents; + areq_ctx->in_nents = sg_nents; sg_copy_to_buffer(src, areq_ctx->in_nents, &curr_buff[*curr_buff_cnt], nbytes); *curr_buff_cnt += nbytes; diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 4835bdebdbb38..a0cb1a8186ac9 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -57,6 +57,7 @@ config CRYPTO_DEV_HISI_ZIP depends on UACCE || UACCE=n depends on ACPI select CRYPTO_DEV_HISI_QM + select CRYPTO_DEFLATE help Support for HiSilicon ZIP Driver diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 0f3ddbadbcf99..021dbd9a1d48f 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -94,9 +94,8 @@ struct hpre_sqe { __le64 key; __le64 in; __le64 out; - __le16 tag; - __le16 resv2; -#define _HPRE_SQE_ALIGN_EXT 7 + __le64 tag; +#define _HPRE_SQE_ALIGN_EXT 6 __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 21ccf879f70c5..839c1f6771436 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -93,6 +93,7 @@ struct hpre_dh_ctx { char *g; /* m */ dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ecdh_ctx { @@ -103,17 +104,15 @@ struct hpre_ecdh_ctx { /* low address: x->y */ unsigned char *g; dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ctx { struct hisi_qp *qp; struct device *dev; - struct hpre_asym_request **req_list; struct hpre *hpre; - spinlock_t req_lock; unsigned int key_sz; bool crt_g2_mode; - struct idr req_idr; union { struct hpre_rsa_ctx rsa; struct hpre_dh_ctx dh; @@ -123,6 +122,7 @@ struct hpre_ctx { unsigned int curve_id; /* for high performance core */ u8 enable_hpcore; + bool fallback; }; struct hpre_asym_request { @@ -136,7 +136,6 @@ struct hpre_asym_request { struct kpp_request *ecdh; } areq; int err; - int req_id; hpre_cb cb; struct timespec64 req_time; }; @@ -151,79 +150,13 @@ static inline unsigned int hpre_align_pd(void) return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); } -static int hpre_alloc_req_id(struct hpre_ctx *ctx) +static void hpre_dfx_add_req_time(struct hpre_asym_request *hpre_req) { - unsigned long flags; - int id; - - spin_lock_irqsave(&ctx->req_lock, flags); - id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); - spin_unlock_irqrestore(&ctx->req_lock, flags); - - return id; -} - -static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) -{ - unsigned long flags; - - spin_lock_irqsave(&ctx->req_lock, flags); - idr_remove(&ctx->req_idr, req_id); - spin_unlock_irqrestore(&ctx->req_lock, flags); -} - -static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx; - struct hpre_dfx *dfx; - int id; - - ctx = hpre_req->ctx; - id = hpre_alloc_req_id(ctx); - if (unlikely(id < 0)) - return -EINVAL; - - ctx->req_list[id] = hpre_req; - hpre_req->req_id = id; + struct hpre_ctx *ctx = hpre_req->ctx; + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; - dfx = ctx->hpre->debug.dfx; if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) ktime_get_ts64(&hpre_req->req_time); - - return id; -} - -static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx = hpre_req->ctx; - int id = hpre_req->req_id; - - if (hpre_req->req_id >= 0) { - hpre_req->req_id = HPRE_INVLD_REQ_ID; - ctx->req_list[id] = NULL; - hpre_free_req_id(ctx, id); - } -} - -static struct hisi_qp *hpre_get_qp_and_start(u8 type) -{ - struct hisi_qp *qp; - int ret; - - qp = hpre_create_qp(type); - if (!qp) { - pr_err("Can not create hpre qp!\n"); - return ERR_PTR(-ENODEV); - } - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - hisi_qm_free_qps(&qp, 1); - pci_err(qp->qm->pdev, "Can not start qp!\n"); - return ERR_PTR(-EINVAL); - } - - return qp; } static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, @@ -340,26 +273,19 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { - struct hpre_asym_request *req; unsigned int err, done, alg; - int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 #define HREE_HW_ERR_MASK GENMASK(10, 0) #define HREE_SQE_DONE_MASK GENMASK(1, 0) #define HREE_ALG_TYPE_MASK GENMASK(4, 0) - id = (int)le16_to_cpu(sqe->tag); - req = ctx->req_list[id]; - hpre_rm_req_from_ctx(req); - *kreq = req; + *kreq = (void *)le64_to_cpu(sqe->tag); err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & HREE_HW_ERR_MASK; - done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & HREE_SQE_DONE_MASK; - if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) return 0; @@ -370,36 +296,10 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, return -EINVAL; } -static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) -{ - struct hpre *hpre; - - if (!ctx || !qp || qlen < 0) - return -EINVAL; - - spin_lock_init(&ctx->req_lock); - ctx->qp = qp; - ctx->dev = &qp->qm->pdev->dev; - - hpre = container_of(ctx->qp->qm, struct hpre, qm); - ctx->hpre = hpre; - ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); - if (!ctx->req_list) - return -ENOMEM; - ctx->key_sz = 0; - ctx->crt_g2_mode = false; - idr_init(&ctx->req_idr); - - return 0; -} - static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) { - if (is_clear_all) { - idr_destroy(&ctx->req_idr); - kfree(ctx->req_list); + if (is_clear_all) hisi_qm_free_qps(&ctx->qp, 1); - } ctx->crt_g2_mode = false; ctx->key_sz = 0; @@ -467,49 +367,44 @@ static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) static void hpre_alg_cb(struct hisi_qp *qp, void *resp) { - struct hpre_ctx *ctx = qp->qp_ctx; - struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *h_req; struct hpre_sqe *sqe = resp; - struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; - if (unlikely(!req)) { - atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); + h_req = (struct hpre_asym_request *)le64_to_cpu(sqe->tag); + if (unlikely(!h_req)) { + pr_err("Failed to get request, and qp_id is %u\n", qp->qp_id); return; } - req->cb(ctx, resp); -} - -static void hpre_stop_qp_and_put(struct hisi_qp *qp) -{ - hisi_qm_stop_qp(qp); - hisi_qm_free_qps(&qp, 1); + h_req->cb(h_req->ctx, resp); } static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) { struct hisi_qp *qp; - int ret; + struct hpre *hpre; - qp = hpre_get_qp_and_start(type); - if (IS_ERR(qp)) - return PTR_ERR(qp); + qp = hpre_create_qp(type); + if (!qp) { + ctx->qp = NULL; + return -ENODEV; + } - qp->qp_ctx = ctx; qp->req_cb = hpre_alg_cb; + ctx->qp = qp; + ctx->dev = &qp->qm->pdev->dev; + hpre = container_of(ctx->qp->qm, struct hpre, qm); + ctx->hpre = hpre; + ctx->key_sz = 0; + ctx->crt_g2_mode = false; - ret = hpre_ctx_set(ctx, qp, qp->sq_depth); - if (ret) - hpre_stop_qp_and_put(qp); - - return ret; + return 0; } static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (is_rsa) { @@ -549,11 +444,8 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -566,9 +458,7 @@ static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) do { atomic64_inc(&dfx[HPRE_SEND_CNT].value); - spin_lock_bh(&ctx->req_lock); ret = hisi_qp_send(ctx->qp, msg); - spin_unlock_bh(&ctx->req_lock); if (ret != -EBUSY) break; atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); @@ -619,12 +509,53 @@ static int hpre_dh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static struct kpp_request *hpre_dh_prepare_fb_req(struct kpp_request *req) +{ + struct kpp_request *fb_req = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + kpp_request_set_tfm(fb_req, ctx->dh.soft_tfm); + kpp_request_set_callback(fb_req, req->base.flags, req->base.complete, req->base.data); + kpp_request_set_input(fb_req, req->src, req->src_len); + kpp_request_set_output(fb_req, req->dst, req->dst_len); + + return fb_req; +} + +static int hpre_dh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_generate_public_key(fb_req); + } + + return hpre_dh_compute_value(req); +} + +static int hpre_dh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_compute_shared_secret(fb_req); + } + + return hpre_dh_compute_value(req); +} + static int hpre_is_dh_params_length_valid(unsigned int key_sz) { #define _HPRE_DH_GRP1 768 @@ -651,13 +582,6 @@ static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) struct device *dev = ctx->dev; unsigned int sz; - if (params->p_size > HPRE_DH_MAX_P_SZ) - return -EINVAL; - - if (hpre_is_dh_params_length_valid(params->p_size << - HPRE_BITS_2_BYTES_SHIFT)) - return -EINVAL; - sz = ctx->key_sz = params->p_size; ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, &ctx->dh.dma_xa_p, GFP_KERNEL); @@ -690,8 +614,8 @@ static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->dh.g) { dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); @@ -718,6 +642,13 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, if (crypto_dh_decode_key(buf, len, ¶ms) < 0) return -EINVAL; + if (!ctx->qp) + goto set_soft_secret; + + if (hpre_is_dh_params_length_valid(params.p_size << + HPRE_BITS_2_BYTES_SHIFT)) + goto set_soft_secret; + /* Free old secret if any */ hpre_dh_clear_ctx(ctx, false); @@ -728,27 +659,55 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, params.key_size); + ctx->fallback = false; return 0; err_clear_ctx: hpre_dh_clear_ctx(ctx, false); return ret; +set_soft_secret: + ctx->fallback = true; + return crypto_kpp_set_secret(ctx->dh.soft_tfm, buf, len); } static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->dh.soft_tfm); + return ctx->key_sz; } static int hpre_dh_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + unsigned int reqsize; + int ret; + + ctx->dh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->dh.soft_tfm)) { + pr_err("Failed to alloc dh tfm!\n"); + return PTR_ERR(ctx->dh.soft_tfm); + } - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + crypto_kpp_set_flags(ctx->dh.soft_tfm, crypto_kpp_get_flags(tfm)); - return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + reqsize = max(sizeof(struct hpre_asym_request) + hpre_align_pd(), + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->dh.soft_tfm)); + kpp_set_reqsize(tfm, reqsize); + + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + if (ret && ret != -ENODEV) { + crypto_free_kpp(ctx->dh.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } + + return 0; } static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) @@ -756,6 +715,7 @@ static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); hpre_dh_clear_ctx(ctx, true); + crypto_free_kpp(ctx->dh.soft_tfm); } static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) @@ -795,9 +755,8 @@ static int hpre_rsa_enc(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_encrypt(req); akcipher_request_set_tfm(req, tfm); @@ -828,7 +787,6 @@ static int hpre_rsa_enc(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -843,9 +801,8 @@ static int hpre_rsa_dec(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_decrypt(req); akcipher_request_set_tfm(req, tfm); @@ -883,7 +840,6 @@ static int hpre_rsa_dec(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -899,8 +855,10 @@ static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, ctx->key_sz = vlen; /* if invalid key size provided, we use software tfm */ - if (!hpre_rsa_key_size_is_support(ctx->key_sz)) + if (!hpre_rsa_key_size_is_support(ctx->key_sz)) { + ctx->fallback = true; return 0; + } ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_pubkey, @@ -1035,8 +993,8 @@ static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int half_key_sz = ctx->key_sz >> 1; struct device *dev = ctx->dev; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->rsa.pubkey) { dma_free_coherent(dev, ctx->key_sz << 1, @@ -1117,6 +1075,7 @@ static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, goto free; } + ctx->fallback = false; return 0; free: @@ -1134,6 +1093,9 @@ static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, false); } @@ -1147,6 +1109,9 @@ static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, true); } @@ -1154,9 +1119,8 @@ static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) { struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); return ctx->key_sz; @@ -1177,10 +1141,14 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) hpre_align_pd()); ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); - if (ret) + if (ret && ret != -ENODEV) { crypto_free_akcipher(ctx->rsa.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } - return ret; + return 0; } static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) @@ -1207,9 +1175,6 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); - if (ctx->ecdh.p) { /* ecdh: p->a->k->b */ memzero_explicit(ctx->ecdh.p + shift, sz); @@ -1346,7 +1311,7 @@ static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) return 0; } -static bool hpre_key_is_zero(char *key, unsigned short key_sz) +static bool hpre_key_is_zero(const char *key, unsigned short key_sz) { int i; @@ -1387,6 +1352,9 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, struct ecdh params; int ret; + if (ctx->fallback) + return crypto_kpp_set_secret(ctx->ecdh.soft_tfm, buf, len); + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { dev_err(dev, "failed to decode ecdh key!\n"); return -EINVAL; @@ -1488,7 +1456,6 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (req->dst_len < ctx->key_sz << 1) { @@ -1510,11 +1477,8 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -1612,28 +1576,86 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static int hpre_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_generate_public_key(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + +static int hpre_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_compute_shared_secret(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->ecdh.soft_tfm); + /* max size is the pub_key_size, include x and y */ return ctx->key_sz << 1; } +static int hpre_ecdh_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + int ret; + + ret = hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + if (!ret) { + kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + return 0; + } else if (ret && ret != -ENODEV) { + return ret; + } + + ctx->ecdh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ecdh.soft_tfm)) { + pr_err("Failed to alloc %s tfm!\n", alg); + return PTR_ERR(ctx->ecdh.soft_tfm); + } + + crypto_kpp_set_flags(ctx->ecdh.soft_tfm, crypto_kpp_get_flags(tfm)); + ctx->fallback = true; + + return 0; +} + static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); ctx->curve_id = ECC_CURVE_NIST_P192; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) @@ -1643,9 +1665,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P256; ctx->enable_hpcore = 1; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) @@ -1654,15 +1674,18 @@ static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P384; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) { + crypto_free_kpp(ctx->ecdh.soft_tfm); + return; + } + hpre_ecc_clear_ctx(ctx, true); } @@ -1680,13 +1703,14 @@ static struct akcipher_alg rsa = { .cra_name = "rsa", .cra_driver_name = "hpre-rsa", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg dh = { .set_secret = hpre_dh_set_secret, - .generate_public_key = hpre_dh_compute_value, - .compute_shared_secret = hpre_dh_compute_value, + .generate_public_key = hpre_dh_generate_public_key, + .compute_shared_secret = hpre_dh_compute_shared_secret, .max_size = hpre_dh_max_size, .init = hpre_dh_init_tfm, .exit = hpre_dh_exit_tfm, @@ -1696,14 +1720,15 @@ static struct kpp_alg dh = { .cra_name = "dh", .cra_driver_name = "hpre-dh", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg ecdh_curves[] = { { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p192_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1713,11 +1738,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p192", .cra_driver_name = "hpre-ecdh-nist-p192", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p256_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1727,11 +1753,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p256", .cra_driver_name = "hpre-ecdh-nist-p256", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p384_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1741,6 +1768,7 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p384", .cra_driver_name = "hpre-ecdh-nist-p384", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, } }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index b94fecd765eeb..884d5d0afaf41 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -465,7 +465,7 @@ struct hisi_qp *hpre_create_qp(u8 type) * type: 0 - RSA/DH. algorithm supported in V2, * 1 - ECC algorithm in V3. */ - ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp); + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, &type, node, &qp); if (!ret) return qp; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 3b391a1466353..b92ee2fcb18aa 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -609,9 +609,13 @@ static void qm_mb_write(struct hisi_qm *qm, const void *src) } #if IS_ENABLED(CONFIG_ARM64) + /* + * The dmb oshst instruction ensures that the data in the + * mailbox is written before it is sent to the hardware. + */ asm volatile("ldp %0, %1, %3\n" - "stp %0, %1, %2\n" "dmb oshst\n" + "stp %0, %1, %2\n" : "=&r" (tmp0), "=&r" (tmp1), "+Q" (*((char __iomem *)fun_base)) @@ -2210,6 +2214,7 @@ static void qp_stop_fail_cb(struct hisi_qp *qp) for (i = 0; i < qp_used; i++) { pos = (i + cur_head) % sq_depth; qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); + qm_cq_head_update(qp); atomic_dec(&qp->qp_status.used); } } @@ -2359,25 +2364,33 @@ EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg) { struct hisi_qp_status *qp_status = &qp->qp_status; - u16 sq_tail = qp_status->sq_tail; - u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth; - void *sqe = qm_get_avail_sqe(qp); + u16 sq_tail, sq_tail_next; + void *sqe; + spin_lock_bh(&qp->qp_lock); if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { + spin_unlock_bh(&qp->qp_lock); dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } - if (!sqe) + sqe = qm_get_avail_sqe(qp); + if (!sqe) { + spin_unlock_bh(&qp->qp_lock); return -EBUSY; + } + sq_tail = qp_status->sq_tail; + sq_tail_next = (sq_tail + 1) % qp->sq_depth; memcpy(sqe, msg, qp->qm->sqe_size); + qp->msg[sq_tail] = msg; qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); atomic_inc(&qp->qp_status.used); qp_status->sq_tail = sq_tail_next; + spin_unlock_bh(&qp->qp_lock); return 0; } @@ -2907,12 +2920,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) { struct device *dev = &qm->pdev->dev; - struct qm_dma *qdma; + struct hisi_qp *qp; int i; for (i = num - 1; i >= 0; i--) { - qdma = &qm->qp_array[i].qdma; - dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); + qp = &qm->qp_array[i]; + dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); + kfree(qp->msg); kfree(qm->poll_data[i].qp_finish_id); } @@ -2934,10 +2948,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, return -ENOMEM; qp = &qm->qp_array[id]; + qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); + if (!qp->msg) + goto err_free_qp_finish_id; + qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, GFP_KERNEL); if (!qp->qdma.va) - goto err_free_qp_finish_id; + goto err_free_qp_msg; qp->sqe = qp->qdma.va; qp->sqe_dma = qp->qdma.dma; @@ -2949,8 +2967,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, qp->qm = qm; qp->qp_id = id; + spin_lock_init(&qp->qp_lock); + spin_lock_init(&qp->backlog.lock); + INIT_LIST_HEAD(&qp->backlog.list); + return 0; +err_free_qp_msg: + kfree(qp->msg); err_free_qp_finish_id: kfree(qm->poll_data[id].qp_finish_id); return ret; @@ -3496,6 +3520,14 @@ void hisi_qm_dev_err_uninit(struct hisi_qm *qm) } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); +static void qm_release_qp_nolock(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + + qm->qp_in_used--; + idr_remove(&qm->qp_idr, qp->qp_id); +} + /** * hisi_qm_free_qps() - free multiple queue pairs. * @qps: The queue pairs need to be freed. @@ -3508,8 +3540,15 @@ void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) if (!qps || qp_num <= 0) return; - for (i = qp_num - 1; i >= 0; i--) - hisi_qm_release_qp(qps[i]); + down_write(&qps[0]->qm->qps_lock); + + for (i = qp_num - 1; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + + up_write(&qps[0]->qm->qps_lock); + qm_pm_put_sync(qps[0]->qm); } EXPORT_SYMBOL_GPL(hisi_qm_free_qps); @@ -3523,6 +3562,43 @@ static void free_list(struct list_head *head) } } +static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) +{ + int i, ret; + + ret = qm_pm_get_sync(qm); + if (ret) + return ret; + + down_write(&qm->qps_lock); + for (i = 0; i < qp_num; i++) { + qps[i] = qm_create_qp_nolock(qm, alg_type[i]); + if (IS_ERR(qps[i])) { + ret = -ENODEV; + goto stop_and_free; + } + + ret = qm_start_qp_nolock(qps[i], 0); + if (ret) { + qm_release_qp_nolock(qps[i]); + goto stop_and_free; + } + } + up_write(&qm->qps_lock); + + return 0; + +stop_and_free: + for (i--; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + up_write(&qm->qps_lock); + qm_pm_put_sync(qm); + + return ret; +} + static int hisi_qm_sort_devices(int node, struct list_head *head, struct hisi_qm_list *qm_list) { @@ -3571,12 +3647,11 @@ static int hisi_qm_sort_devices(int node, struct list_head *head, * not meet the requirements will return error. */ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps) + u8 *alg_type, int node, struct hisi_qp **qps) { struct hisi_qm_resource *tmp; int ret = -ENODEV; LIST_HEAD(head); - int i; if (!qps || !qm_list || qp_num <= 0) return -EINVAL; @@ -3588,24 +3663,15 @@ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, } list_for_each_entry(tmp, &head, list) { - for (i = 0; i < qp_num; i++) { - qps[i] = hisi_qm_create_qp(tmp->qm, alg_type); - if (IS_ERR(qps[i])) { - hisi_qm_free_qps(qps, i); - break; - } - } - - if (i == qp_num) { - ret = 0; + ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); + if (!ret) break; - } } mutex_unlock(&qm_list->lock); if (ret) - pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n", - node, alg_type, qp_num); + pr_info("Failed to create qps, node[%d], qp[%d]!\n", + node, qp_num); err: free_list(&head); @@ -3678,6 +3744,7 @@ static void qm_clear_vft_config(struct hisi_qm *qm) static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) { struct device *dev = &qm->pdev->dev; + struct qm_shaper_factor t_factor; u32 ir = qos * QM_QOS_RATE; int ret, total_vfs, i; @@ -3685,6 +3752,7 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) if (fun_index > total_vfs) return -EINVAL; + memcpy(&t_factor, &qm->factor[fun_index], sizeof(t_factor)); qm->factor[fun_index].func_qos = qos; ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); @@ -3698,11 +3766,21 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); if (ret) { dev_err(dev, "type: %d, failed to set shaper vft!\n", i); - return -EINVAL; + goto back_func_qos; } } return 0; + +back_func_qos: + memcpy(&qm->factor[fun_index], &t_factor, sizeof(t_factor)); + for (i--; i >= ALG_TYPE_0; i--) { + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) + dev_err(dev, "failed to restore shaper vft during rollback!\n"); + } + + return -EINVAL; } static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 81d0beda93b2b..0710977861f32 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -82,11 +82,6 @@ struct sec_aead_req { __u8 out_mac_buf[SEC_MAX_MAC_LEN]; }; -struct sec_instance_backlog { - struct list_head list; - spinlock_t lock; -}; - /* SEC request of Crypto */ struct sec_req { union { @@ -112,7 +107,6 @@ struct sec_req { bool use_pbuf; struct list_head list; - struct sec_instance_backlog *backlog; struct sec_request_buf buf; }; @@ -172,7 +166,6 @@ struct sec_qp_ctx { spinlock_t id_lock; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; - struct sec_instance_backlog backlog; u16 send_head; }; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 31590d01139a3..c462b58d30343 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -54,7 +54,6 @@ #define SEC_AUTH_CIPHER_V3 0x40 #define SEC_FLAG_OFFSET 7 #define SEC_FLAG_MASK 0x0780 -#define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 #define SEC_ICV_MASK 0x000E @@ -148,7 +147,7 @@ static void sec_free_req_id(struct sec_req *req) spin_unlock_bh(&qp_ctx->id_lock); } -static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) +static void pre_parse_finished_bd(struct bd_status *status, void *resp) { struct sec_sqe *bd = resp; @@ -158,11 +157,9 @@ static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le16_to_cpu(bd->type2.tag); status->err_type = bd->type2.error_type; - - return bd->type_cipher_auth & SEC_TYPE_MASK; } -static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) +static void pre_parse_finished_bd3(struct bd_status *status, void *resp) { struct sec_sqe3 *bd3 = resp; @@ -172,8 +169,6 @@ static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le64_to_cpu(bd3->tag); status->err_type = bd3->error_type; - - return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; } static int sec_cb_status_check(struct sec_req *req, @@ -244,7 +239,7 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp struct sec_req *req, *tmp; int ret; - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) { list_del(&req->list); ctx->req_op->buf_unmap(ctx, req); if (req->req_id >= 0) @@ -265,11 +260,12 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { + struct hisi_qp *qp = qp_ctx->qp; struct sec_req *req, *tmp; int ret; - spin_lock_bh(&qp_ctx->backlog.lock); - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + spin_lock_bh(&qp->backlog.lock); + list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) { ret = qp_send_message(req); switch (ret) { case -EINPROGRESS: @@ -287,42 +283,46 @@ static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) } unlock: - spin_unlock_bh(&qp_ctx->backlog.lock); + spin_unlock_bh(&qp->backlog.lock); } static void sec_req_cb(struct hisi_qp *qp, void *resp) { - struct sec_qp_ctx *qp_ctx = qp->qp_ctx; - struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; - u8 type_supported = qp_ctx->ctx->type_supported; + const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head]; + struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe); + struct sec_ctx *ctx = req->ctx; + struct sec_dfx *dfx = &ctx->sec->debug.dfx; struct bd_status status; - struct sec_ctx *ctx; - struct sec_req *req; int err; - u8 type; - if (type_supported == SEC_BD_TYPE2) { - type = pre_parse_finished_bd(&status, resp); - req = qp_ctx->req_list[status.tag]; - } else { - type = pre_parse_finished_bd3(&status, resp); - req = (void *)(uintptr_t)status.tag; - } + pre_parse_finished_bd(&status, resp); - if (unlikely(type != type_supported)) { - atomic64_inc(&dfx->err_bd_cnt); - pr_err("err bd type [%u]\n", type); - return; - } + req->err_type = status.err_type; + err = sec_cb_status_check(req, &status); + if (err) + atomic64_inc(&dfx->done_flag_cnt); - if (unlikely(!req)) { - atomic64_inc(&dfx->invalid_req_cnt); - atomic_inc(&qp->qp_status.used); - return; - } + atomic64_inc(&dfx->recv_cnt); + ctx->req_op->buf_unmap(ctx, req); + ctx->req_op->callback(ctx, req, err); +} + +static void sec_req_cb3(struct hisi_qp *qp, void *resp) +{ + struct bd_status status; + struct sec_ctx *ctx; + struct sec_dfx *dfx; + struct sec_req *req; + int err; + + pre_parse_finished_bd3(&status, resp); + + req = (void *)(uintptr_t)status.tag; req->err_type = status.err_type; ctx = req->ctx; + dfx = &ctx->sec->debug.dfx; + err = sec_cb_status_check(req, &status); if (err) atomic64_inc(&dfx->done_flag_cnt); @@ -330,7 +330,6 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) atomic64_inc(&dfx->recv_cnt); ctx->req_op->buf_unmap(ctx, req); - ctx->req_op->callback(ctx, req, err); } @@ -348,8 +347,10 @@ static int sec_alg_send_message_retry(struct sec_req *req) static int sec_alg_try_enqueue(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; + /* Check if any request is already backlogged */ - if (!list_empty(&req->backlog->list)) + if (!list_empty(&qp->backlog.list)) return -EBUSY; /* Try to enqueue to HW ring */ @@ -359,17 +360,18 @@ static int sec_alg_try_enqueue(struct sec_req *req) static int sec_alg_send_message_maybacklog(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; int ret; ret = sec_alg_try_enqueue(req); if (ret != -EBUSY) return ret; - spin_lock_bh(&req->backlog->lock); + spin_lock_bh(&qp->backlog.lock); ret = sec_alg_try_enqueue(req); if (ret == -EBUSY) - list_add_tail(&req->list, &req->backlog->list); - spin_unlock_bh(&req->backlog->lock); + list_add_tail(&req->list, &qp->backlog.list); + spin_unlock_bh(&qp->backlog.lock); return ret; } @@ -624,32 +626,25 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) qp_ctx = &ctx->qp_ctx[qp_ctx_id]; qp = ctx->qps[qp_ctx_id]; - qp->req_type = 0; - qp->qp_ctx = qp_ctx; qp_ctx->qp = qp; qp_ctx->ctx = ctx; - qp->req_cb = sec_req_cb; + if (ctx->type_supported == SEC_BD_TYPE3) + qp->req_cb = sec_req_cb3; + else + qp->req_cb = sec_req_cb; spin_lock_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); - spin_lock_init(&qp_ctx->backlog.lock); spin_lock_init(&qp_ctx->id_lock); - INIT_LIST_HEAD(&qp_ctx->backlog.list); qp_ctx->send_head = 0; ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); if (ret) goto err_destroy_idr; - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) - goto err_resource_free; - return 0; -err_resource_free: - sec_free_qp_ctx_resource(ctx, qp_ctx); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); return ret; @@ -658,7 +653,6 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) static void sec_release_qp_ctx(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { - hisi_qm_stop_qp(qp_ctx->qp); sec_free_qp_ctx_resource(ctx, qp_ctx); idr_destroy(&qp_ctx->req_idr); } @@ -669,10 +663,8 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) int i, ret; ctx->qps = sec_create_qps(); - if (!ctx->qps) { - pr_err("Can not create sec qps!\n"); + if (!ctx->qps) return -ENODEV; - } sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); ctx->sec = sec; @@ -708,6 +700,9 @@ static void sec_ctx_base_uninit(struct sec_ctx *ctx) { int i; + if (!ctx->qps) + return; + for (i = 0; i < ctx->sec->ctx_q_num; i++) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); @@ -719,6 +714,9 @@ static int sec_cipher_init(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return 0; + c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, &c_ctx->c_key_dma, GFP_KERNEL); if (!c_ctx->c_key) @@ -731,6 +729,9 @@ static void sec_cipher_uninit(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return; + memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, c_ctx->c_key, c_ctx->c_key_dma); @@ -752,6 +753,9 @@ static void sec_auth_uninit(struct sec_ctx *ctx) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + if (!ctx->qps) + return; + memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, a_ctx->a_key, a_ctx->a_key_dma); @@ -789,7 +793,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) } ret = sec_ctx_base_init(ctx); - if (ret) + if (ret && ret != -ENODEV) return ret; ret = sec_cipher_init(ctx); @@ -898,6 +902,9 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, struct device *dev = ctx->dev; int ret; + if (!ctx->qps) + goto set_soft_key; + if (c_mode == SEC_CMODE_XTS) { ret = xts_verify_key(tfm, key, keylen); if (ret) { @@ -928,13 +935,14 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (c_ctx->fbtfm) { - ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); - if (ret) { - dev_err(dev, "failed to set fallback skcipher key!\n"); - return ret; - } + +set_soft_key: + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); + if (ret) { + dev_err(dev, "failed to set fallback skcipher key!\n"); + return ret; } + return 0; } @@ -1398,6 +1406,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, struct crypto_authenc_keys keys; int ret; + if (!ctx->qps) + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + ctx->a_ctx.a_alg = a_alg; ctx->c_ctx.c_alg = c_alg; c_ctx->c_mode = c_mode; @@ -1952,7 +1963,6 @@ static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); req->qp_ctx = qp_ctx; - req->backlog = &qp_ctx->backlog; return 0; } @@ -2055,6 +2065,9 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) if (ret) return ret; + if (!ctx->qps) + return 0; + if (ctx->sec->qm.ver < QM_HW_V3) { ctx->type_supported = SEC_BD_TYPE2; ctx->req_op = &sec_skcipher_req_ops; @@ -2063,7 +2076,7 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) ctx->req_op = &sec_skcipher_req_ops_v3; } - return ret; + return 0; } static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) @@ -2131,7 +2144,7 @@ static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { pr_err("hisi_sec2: aead init error!\n"); return ret; } @@ -2173,7 +2186,7 @@ static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); return ret; } @@ -2318,6 +2331,9 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + if (!sk_req->cryptlen) { if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) return -EINVAL; @@ -2335,9 +2351,12 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) return -EINVAL; if (unlikely(ctx->c_ctx.fallback || need_fallback)) - return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); + goto soft_crypto; return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); } static int sec_skcipher_encrypt(struct skcipher_request *sk_req) @@ -2545,6 +2564,9 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + req->flag = a_req->base.flags; req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; @@ -2555,11 +2577,14 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) ret = sec_aead_param_check(ctx, req, &need_fallback); if (unlikely(ret)) { if (need_fallback) - return sec_aead_soft_crypto(ctx, a_req, encrypt); + goto soft_crypto; return -EINVAL; } return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_aead_soft_crypto(ctx, a_req, encrypt); } static int sec_aead_encrypt(struct aead_request *a_req) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 5eb2d68207426..7dd125f5f511f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -417,18 +417,29 @@ struct hisi_qp **sec_create_qps(void) int node = cpu_to_node(raw_smp_processor_id()); u32 ctx_num = ctx_q_num; struct hisi_qp **qps; + u8 *type; int ret; qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); if (!qps) return NULL; - ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps); - if (!ret) - return qps; + /* The type of SEC is all 0, so just allocated by kcalloc */ + type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); + if (!type) { + kfree(qps); + return NULL; + } - kfree(qps); - return NULL; + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); + if (ret) { + kfree(type); + kfree(qps); + return NULL; + } + + kfree(type); + return qps; } u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 7a9ef2a9972a2..848ad7b101d93 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -265,7 +265,7 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, return curr_hw_sgl; err_unmap: - dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, sgl, sg_n, dir); return ERR_PTR(ret); } diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index ac74df4a94712..5ca0b90859a81 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -40,6 +40,7 @@ #define SEED_SHIFT_24 24 #define SEED_SHIFT_16 16 #define SEED_SHIFT_8 8 +#define SW_MAX_RANDOM_BYTES 65520 struct hisi_trng_list { struct mutex lock; @@ -53,8 +54,10 @@ struct hisi_trng { struct list_head list; struct hwrng rng; u32 ver; - bool is_used; - struct mutex mutex; + u32 ctx_num; + /* The bytes of the random number generated since the last seeding. */ + u32 random_bytes; + struct mutex lock; }; struct hisi_trng_ctx { @@ -63,10 +66,14 @@ struct hisi_trng_ctx { static atomic_t trng_active_devs; static struct hisi_trng_list trng_devices; +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait); -static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) +static int hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) { u32 val, seed_reg, i; + int ret; + + writel(0x0, trng->base + SW_DRBG_BLOCKS); for (i = 0; i < SW_DRBG_SEED_SIZE; i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { @@ -78,6 +85,20 @@ static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; writel(val, trng->base + SW_DRBG_SEED(seed_reg)); } + + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), + trng->base + SW_DRBG_BLOCKS); + writel(0x1, trng->base + SW_DRBG_INIT); + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(0), SLEEP_US, TIMEOUT_US); + if (ret) { + pr_err("failed to init trng(%d)\n", ret); + return -EIO; + } + + trng->random_bytes = 0; + + return 0; } static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, @@ -85,8 +106,7 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, { struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); struct hisi_trng *trng = ctx->trng; - u32 val = 0; - int ret = 0; + int ret; if (slen < SW_DRBG_SEED_SIZE) { pr_err("slen(%u) is not matched with trng(%d)\n", slen, @@ -94,43 +114,45 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, return -EINVAL; } - writel(0x0, trng->base + SW_DRBG_BLOCKS); - hisi_trng_set_seed(trng, seed); + mutex_lock(&trng->lock); + ret = hisi_trng_set_seed(trng, seed); + mutex_unlock(&trng->lock); - writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), - trng->base + SW_DRBG_BLOCKS); - writel(0x1, trng->base + SW_DRBG_INIT); + return ret; +} - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(0), SLEEP_US, TIMEOUT_US); - if (ret) - pr_err("fail to init trng(%d)\n", ret); +static int hisi_trng_reseed(struct hisi_trng *trng) +{ + u8 seed[SW_DRBG_SEED_SIZE]; + int size; - return ret; + if (!trng->random_bytes) + return 0; + + size = hisi_trng_read(&trng->rng, seed, SW_DRBG_SEED_SIZE, false); + if (size != SW_DRBG_SEED_SIZE) + return -EIO; + + return hisi_trng_set_seed(trng, seed); } -static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) +static int hisi_trng_get_bytes(struct hisi_trng *trng, u8 *dstn, unsigned int dlen) { - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; u32 data[SW_DRBG_DATA_NUM]; u32 currsize = 0; u32 val = 0; int ret; u32 i; - if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { - pr_err("dlen(%u) exceeds limit(%d)!\n", dlen, - SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); - return -EINVAL; - } + ret = hisi_trng_reseed(trng); + if (ret) + return ret; do { ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(1), SLEEP_US, TIMEOUT_US); + val, val & BIT(1), SLEEP_US, TIMEOUT_US); if (ret) { - pr_err("fail to generate random number(%d)!\n", ret); + pr_err("failed to generate random number(%d)!\n", ret); break; } @@ -145,30 +167,57 @@ static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, currsize = dlen; } + trng->random_bytes += SW_DRBG_BYTES; writel(0x1, trng->base + SW_DRBG_GEN); } while (currsize < dlen); return ret; } +static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + unsigned int currsize = 0; + unsigned int block_size; + int ret; + + if (!dstn || !dlen) { + pr_err("output is error, dlen %u!\n", dlen); + return -EINVAL; + } + + do { + block_size = min_t(unsigned int, dlen - currsize, SW_MAX_RANDOM_BYTES); + mutex_lock(&trng->lock); + ret = hisi_trng_get_bytes(trng, dstn + currsize, block_size); + mutex_unlock(&trng->lock); + if (ret) + return ret; + currsize += block_size; + } while (currsize < dlen); + + return 0; +} + static int hisi_trng_init(struct crypto_tfm *tfm) { struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); struct hisi_trng *trng; - int ret = -EBUSY; + u32 ctx_num = ~0; mutex_lock(&trng_devices.lock); list_for_each_entry(trng, &trng_devices.list, list) { - if (!trng->is_used) { - trng->is_used = true; + if (trng->ctx_num < ctx_num) { + ctx_num = trng->ctx_num; ctx->trng = trng; - ret = 0; - break; } } + ctx->trng->ctx_num++; mutex_unlock(&trng_devices.lock); - return ret; + return 0; } static void hisi_trng_exit(struct crypto_tfm *tfm) @@ -176,7 +225,7 @@ static void hisi_trng_exit(struct crypto_tfm *tfm) struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); mutex_lock(&trng_devices.lock); - ctx->trng->is_used = false; + ctx->trng->ctx_num--; mutex_unlock(&trng_devices.lock); } @@ -238,7 +287,7 @@ static int hisi_trng_del_from_list(struct hisi_trng *trng) int ret = -EBUSY; mutex_lock(&trng_devices.lock); - if (!trng->is_used) { + if (!trng->ctx_num) { list_del(&trng->list); ret = 0; } @@ -262,7 +311,9 @@ static int hisi_trng_probe(struct platform_device *pdev) if (IS_ERR(trng->base)) return PTR_ERR(trng->base); - trng->is_used = false; + trng->ctx_num = 0; + trng->random_bytes = SW_MAX_RANDOM_BYTES; + mutex_init(&trng->lock); trng->ver = readl(trng->base + HISI_TRNG_VERSION); if (!trng_devices.is_init) { INIT_LIST_HEAD(&trng_devices.list); diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 9fb2a9c01132b..b83f228281ab1 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -99,7 +99,7 @@ enum zip_cap_table_type { ZIP_CORE5_BITMAP, }; -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index b97513981a3b7..e140d4f8afe0e 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -39,6 +39,7 @@ enum { HZIP_CTX_Q_NUM }; +#define GET_REQ_FROM_SQE(sqe) ((u64)(sqe)->dw26 | (u64)(sqe)->dw27 << 32) #define COMP_NAME_TO_TYPE(alg_name) \ (!strcmp((alg_name), "deflate") ? HZIP_ALG_TYPE_DEFLATE : 0) @@ -48,6 +49,7 @@ struct hisi_zip_req { struct hisi_acc_hw_sgl *hw_dst; dma_addr_t dma_src; dma_addr_t dma_dst; + struct hisi_zip_qp_ctx *qp_ctx; u16 req_id; }; @@ -64,6 +66,7 @@ struct hisi_zip_qp_ctx { struct hisi_acc_sgl_pool *sgl_pool; struct hisi_zip *zip_dev; struct hisi_zip_ctx *ctx; + u8 req_type; }; struct hisi_zip_sqe_ops { @@ -74,7 +77,6 @@ struct hisi_zip_sqe_ops { void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); - u32 (*get_tag)(struct hisi_zip_sqe *sqe); u32 (*get_status)(struct hisi_zip_sqe *sqe); u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); }; @@ -82,6 +84,7 @@ struct hisi_zip_sqe_ops { struct hisi_zip_ctx { struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; const struct hisi_zip_sqe_ops *ops; + bool fallback; }; static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) @@ -108,6 +111,24 @@ static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); +static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decompress) +{ + ACOMP_FBREQ_ON_STACK(fbreq, acomp_req); + int ret; + + if (!is_decompress) + ret = crypto_acomp_compress(fbreq); + else + ret = crypto_acomp_decompress(fbreq); + if (ret) { + pr_err("failed to do fallback work, ret=%d\n", ret); + return ret; + } + + acomp_req->dlen = fbreq->dlen; + return ret; +} + static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, struct acomp_req *req) { @@ -131,6 +152,7 @@ static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, req_cache = q + req_id; req_cache->req_id = req_id; req_cache->req = req; + req_cache->qp_ctx = qp_ctx; return req_cache; } @@ -181,7 +203,8 @@ static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) static void hisi_zip_fill_tag(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) { - sqe->dw26 = req->req_id; + sqe->dw26 = lower_32_bits((u64)req); + sqe->dw27 = upper_32_bits((u64)req); } static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) @@ -213,7 +236,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, { struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct acomp_req *a_req = req->req; struct hisi_qp *qp = qp_ctx->qp; struct device *dev = &qp->qm->pdev->dev; @@ -237,18 +259,16 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, &req->dma_dst, DMA_FROM_DEVICE); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); - dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n", ret); goto err_unmap_input; } - hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); + hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req); /* send command to start a task */ atomic64_inc(&dfx->send_cnt); - spin_lock_bh(&req_q->req_lock); ret = hisi_qp_send(qp, &zip_sqe); - spin_unlock_bh(&req_q->req_lock); if (unlikely(ret < 0)) { atomic64_inc(&dfx->send_busy_cnt); ret = -EAGAIN; @@ -265,11 +285,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, return ret; } -static u32 hisi_zip_get_tag(struct hisi_zip_sqe *sqe) -{ - return sqe->dw26; -} - static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) { return sqe->dw3 & HZIP_BD_STATUS_M; @@ -282,14 +297,12 @@ static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) { - struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; + struct hisi_zip_sqe *sqe = data; + struct hisi_zip_req *req = (struct hisi_zip_req *)GET_REQ_FROM_SQE(sqe); + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx; const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct device *dev = &qp->qm->pdev->dev; - struct hisi_zip_sqe *sqe = data; - u32 tag = ops->get_tag(sqe); - struct hisi_zip_req *req = req_q->q + tag; struct acomp_req *acomp_req = req->req; int err = 0; u32 status; @@ -319,10 +332,15 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 0); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -340,10 +358,15 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 1); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -358,33 +381,6 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return ret; } -static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, - int alg_type, int req_type) -{ - struct device *dev = &qp->qm->pdev->dev; - int ret; - - qp->req_type = req_type; - qp->alg_type = alg_type; - qp->qp_ctx = qp_ctx; - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - dev_err(dev, "failed to start qp (%d)!\n", ret); - return ret; - } - - qp_ctx->qp = qp; - - return 0; -} - -static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) -{ - hisi_qm_stop_qp(qp_ctx->qp); - hisi_qm_free_qps(&qp_ctx->qp, 1); -} - static const struct hisi_zip_sqe_ops hisi_zip_ops = { .sqe_type = 0x3, .fill_addr = hisi_zip_fill_addr, @@ -393,7 +389,6 @@ static const struct hisi_zip_sqe_ops hisi_zip_ops = { .fill_req_type = hisi_zip_fill_req_type, .fill_tag = hisi_zip_fill_tag, .fill_sqe_type = hisi_zip_fill_sqe_type, - .get_tag = hisi_zip_get_tag, .get_status = hisi_zip_get_status, .get_dstlen = hisi_zip_get_dstlen, }; @@ -402,10 +397,15 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int { struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; struct hisi_zip_qp_ctx *qp_ctx; + u8 alg_type[HZIP_CTX_Q_NUM]; struct hisi_zip *hisi_zip; - int ret, i, j; + int ret, i; - ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); + /* alg_type = 0 for compress, 1 for decompress in hw sqe */ + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + alg_type[i] = i; + + ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node, alg_type); if (ret) { pr_err("failed to create zip qps (%d)!\n", ret); return -ENODEV; @@ -414,19 +414,11 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); for (i = 0; i < HZIP_CTX_Q_NUM; i++) { - /* alg_type = 0 for compress, 1 for decompress in hw sqe */ qp_ctx = &hisi_zip_ctx->qp_ctx[i]; qp_ctx->ctx = hisi_zip_ctx; - ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); - if (ret) { - for (j = i - 1; j >= 0; j--) - hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); - - hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); - return ret; - } - qp_ctx->zip_dev = hisi_zip; + qp_ctx->req_type = req_type; + qp_ctx->qp = qps[i]; } hisi_zip_ctx->ops = &hisi_zip_ops; @@ -436,10 +428,13 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) { + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; int i; for (i = 0; i < HZIP_CTX_Q_NUM; i++) - hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); + qps[i] = hisi_zip_ctx->qp_ctx[i].qp; + + hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); } static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) @@ -549,7 +544,7 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); if (ret) { pr_err("failed to init ctx (%d)!\n", ret); - return ret; + goto switch_to_soft; } dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; @@ -574,17 +569,20 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) hisi_zip_release_req_q(ctx); err_ctx_exit: hisi_zip_ctx_exit(ctx); - return ret; +switch_to_soft: + ctx->fallback = true; + return 0; } static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); - hisi_zip_set_acomp_cb(ctx, NULL); - hisi_zip_release_sgl_pool(ctx); - hisi_zip_release_req_q(ctx); - hisi_zip_ctx_exit(ctx); + if (!ctx->fallback) { + hisi_zip_release_sgl_pool(ctx); + hisi_zip_release_req_q(ctx); + hisi_zip_ctx_exit(ctx); + } } static struct acomp_alg hisi_zip_acomp_deflate = { @@ -595,7 +593,8 @@ static struct acomp_alg hisi_zip_acomp_deflate = { .base = { .cra_name = "deflate", .cra_driver_name = "hisi-deflate-acomp", - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_module = THIS_MODULE, .cra_priority = HZIP_ALG_PRIORITY, .cra_ctxsize = sizeof(struct hisi_zip_ctx), diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4fcbe6bada066..85b26ef175485 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -446,12 +446,12 @@ static const struct pci_device_id hisi_zip_dev_ids[] = { }; MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type) { if (node == NUMA_NO_NODE) node = cpu_to_node(raw_smp_processor_id()); - return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); + return hisi_qm_alloc_qps_node(&zip_devices, qp_num, alg_type, node, qps); } bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c index 0b38a567da0e0..b7fd9795062d4 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-main.c +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -77,11 +77,44 @@ inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); } -static void eip93_unregister_algs(unsigned int i) +static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags) +{ + if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && + !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + return 0; + + if (IS_AES(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_AES)) + return 0; + + if (IS_HASH_MD5(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_MD5)) + return 0; + + if (IS_HASH_SHA1(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) + return 0; + + if (IS_HASH_SHA224(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) + return 0; + + if (IS_HASH_SHA256(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) + return 0; + + return 1; +} + +static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i) { unsigned int j; for (j = 0; j < i; j++) { + if (!eip93_algo_is_supported(eip93_algs[j]->flags, + supported_algo_flags)) + continue; + switch (eip93_algs[j]->type) { case EIP93_ALG_TYPE_SKCIPHER: crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); @@ -90,7 +123,7 @@ static void eip93_unregister_algs(unsigned int i) crypto_unregister_aead(&eip93_algs[j]->alg.aead); break; case EIP93_ALG_TYPE_HASH: - crypto_unregister_ahash(&eip93_algs[i]->alg.ahash); + crypto_unregister_ahash(&eip93_algs[j]->alg.ahash); break; } } @@ -106,49 +139,27 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl eip93_algs[i]->eip93 = eip93; - if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && - !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + if (!eip93_algo_is_supported(alg_flags, supported_algo_flags)) continue; - if (IS_AES(alg_flags)) { - if (!(supported_algo_flags & EIP93_PE_OPTION_AES)) - continue; + if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) { + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_128; - if (!IS_HMAC(alg_flags)) { - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_128; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_192; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_192; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_256; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_256; - - if (IS_RFC3686(alg_flags)) - eip93_algs[i]->alg.skcipher.max_keysize += - CTR_RFC3686_NONCE_SIZE; - } + if (IS_RFC3686(alg_flags)) + eip93_algs[i]->alg.skcipher.max_keysize += + CTR_RFC3686_NONCE_SIZE; } - if (IS_HASH_MD5(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_MD5)) - continue; - - if (IS_HASH_SHA1(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) - continue; - - if (IS_HASH_SHA224(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) - continue; - - if (IS_HASH_SHA256(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) - continue; - switch (eip93_algs[i]->type) { case EIP93_ALG_TYPE_SKCIPHER: ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); @@ -167,7 +178,7 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl return 0; fail: - eip93_unregister_algs(i); + eip93_unregister_algs(supported_algo_flags, i); return ret; } @@ -469,8 +480,11 @@ static int eip93_crypto_probe(struct platform_device *pdev) static void eip93_crypto_remove(struct platform_device *pdev) { struct eip93_device *eip93 = platform_get_drvdata(pdev); + u32 algo_flags; + + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); - eip93_unregister_algs(ARRAY_SIZE(eip93_algs)); + eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs)); eip93_cleanup(eip93); } diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 23f585219fb4b..da9b2bc515194 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -221,15 +221,13 @@ static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX]; static int find_empty_iaa_compression_mode(void) { - int i = -EINVAL; + int i; - for (i = 0; i < IAA_COMP_MODES_MAX; i++) { - if (iaa_compression_modes[i]) - continue; - break; - } + for (i = 0; i < IAA_COMP_MODES_MAX; i++) + if (!iaa_compression_modes[i]) + return i; - return i; + return -EINVAL; } static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx) @@ -805,7 +803,7 @@ static int save_iaa_wq(struct idxd_wq *wq) if (!cpus_per_iaa) cpus_per_iaa = 1; out: - return 0; + return ret; } static void remove_iaa_wq(struct idxd_wq *wq) diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 35679b21ff63b..f63e78724c7a7 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -41,8 +41,6 @@ static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, adf_error_notifier(accel_dev); adf_pf2vf_notify_fatal_error(accel_dev); adf_dev_restarting_notify(accel_dev); - adf_pf2vf_notify_restarting(accel_dev); - adf_pf2vf_wait_for_restarting_complete(accel_dev); pci_clear_master(pdev); adf_dev_down(accel_dev); @@ -105,7 +103,6 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) accel_dev->accel_id); hw_device->reset_device(accel_dev); pci_restore_state(pdev); - pci_save_state(pdev); } } @@ -204,7 +201,6 @@ static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) if (!pdev->is_busmaster) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); res = adf_dev_up(accel_dev, false); if (res && res != -EALREADY) return PCI_ERS_RESULT_DISCONNECT; diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c index b9b5e744a3f16..af8dbc7517cf8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c @@ -148,6 +148,16 @@ static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; break; + default: + dev_err(&GET_DEV(vf_info->accel_dev), + "Invalid BlockMsg type 0x%.4x received from VF%u\n", + req.type, vf_info->vf_nr); + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR) | + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, + ADF_PF2VF_UNSPECIFIED_ERROR); + return resp; } /* Is this a request for CRC or data? */ diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index 9f5601c0280bf..417a48f413505 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -1326,7 +1326,7 @@ static ssize_t ucode_load_store(struct device *dev, int del_grp_idx = -1; int ucode_idx = 0; - if (strlen(buf) > OTX_CPT_UCODE_NAME_LENGTH) + if (count >= OTX_CPT_UCODE_NAME_LENGTH) return -EINVAL; eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr); diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 88a41d1ca5f64..6c0bfb3ea1c9f 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -168,7 +168,8 @@ static void free_command_queues(struct otx_cptvf *cptvf, chunk = list_first_entry(&cqinfo->queue[i].chead, struct otx_cpt_cmd_chunk, nextchunk); - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + OTX_CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c index a4cc6bf146ec0..0345c9383d509 100644 --- a/drivers/crypto/omap-crypto.c +++ b/drivers/crypto/omap-crypto.c @@ -21,7 +21,7 @@ static int omap_crypto_copy_sg_lists(int total, int bs, struct scatterlist *tmp; if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) { - new_sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); + new_sg = kmalloc_array(n, sizeof(*new_sg), GFP_KERNEL); if (!new_sg) return -ENOMEM; diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index 329f60ad422e6..9214bbfc868f5 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -332,6 +332,13 @@ static int __init padlock_init(void) if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN)) return -ENODEV; + /* + * Skip family 0x07 and newer used by Zhaoxin processors, + * as the driver's self-tests fail on these CPUs. + */ + if (c->x86 >= 0x07) + return -ENODEV; + /* Register the newly added algorithm module if on * * VIA Nano processor, or else just do as before */ if (c->x86_model < 0x0f) { diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c index 97b56e92ea33f..1e6e1f5bcca45 100644 --- a/drivers/crypto/qce/aead.c +++ b/drivers/crypto/qce/aead.c @@ -63,6 +63,10 @@ static void qce_aead_done(void *data) sg_free_table(&rctx->dst_tbl); } + error = qce_bam_unlock(qce); + if (error) + dev_err(qce->dev, "aead: failed to unlock the BAM\n"); + error = qce_check_status(qce, &status); if (error < 0 && (error != -EBADMSG)) dev_err(qce->dev, "aead operation error (%x)\n", status); @@ -188,6 +192,8 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; unsigned int assoclen = rctx->assoclen; unsigned int adata_header_len, cryptlen, totallen; gfp_t gfp; @@ -200,6 +206,10 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) cryptlen = rctx->cryptlen; totallen = cryptlen + req->assoclen; + ret = qce_bam_lock(qce); + if (ret) + return ret; + /* Get the msg */ msg_sg = scatterwalk_ffwd(__sg, req->src, req->assoclen); diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index 04253a8d33409..930006aaba4ac 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -14,6 +14,7 @@ #include "cipher.h" #include "common.h" #include "core.h" +#include "dma.h" #include "regs-v5.h" #include "sha.h" #include "aead.h" @@ -25,7 +26,7 @@ static inline u32 qce_read(struct qce_device *qce, u32 offset) static inline void qce_write(struct qce_device *qce, u32 offset, u32 val) { - writel(val, qce->base + offset); + qce_write_dma(qce, offset, val); } static inline void qce_write_array(struct qce_device *qce, u32 offset, @@ -82,6 +83,8 @@ static void qce_setup_config(struct qce_device *qce) { u32 config; + qce_clear_bam_transaction(qce); + /* get big endianness */ config = qce_config_reg(qce, 0); @@ -90,12 +93,14 @@ static void qce_setup_config(struct qce_device *qce) qce_write(qce, REG_CONFIG, config); } -static inline void qce_crypto_go(struct qce_device *qce, bool result_dump) +static int qce_crypto_go(struct qce_device *qce, bool result_dump) { if (result_dump) qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT)); else qce_write(qce, REG_GOPROC, BIT(GO_SHIFT)); + + return qce_submit_cmd_desc(qce); } #if defined(CONFIG_CRYPTO_DEV_QCE_SHA) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD) @@ -223,9 +228,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req) config = qce_config_reg(qce, 1); qce_write(qce, REG_CONFIG, config); - qce_crypto_go(qce, true); - - return 0; + return qce_crypto_go(qce, true); } #endif @@ -386,9 +389,7 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) config = qce_config_reg(qce, 1); qce_write(qce, REG_CONFIG, config); - qce_crypto_go(qce, true); - - return 0; + return qce_crypto_go(qce, true); } #endif @@ -535,9 +536,7 @@ static int qce_setup_regs_aead(struct crypto_async_request *async_req) qce_write(qce, REG_CONFIG, config); /* Start the process */ - qce_crypto_go(qce, !IS_CCM(flags)); - - return 0; + return qce_crypto_go(qce, !IS_CCM(flags)); } #endif @@ -593,3 +592,21 @@ void qce_get_version(struct qce_device *qce, u32 *major, u32 *minor, u32 *step) *minor = (val & CORE_MINOR_REV_MASK) >> CORE_MINOR_REV_SHIFT; *step = (val & CORE_STEP_REV_MASK) >> CORE_STEP_REV_SHIFT; } + +int qce_bam_lock(struct qce_device *qce) +{ + qce_clear_bam_transaction(qce); + /* Dummy write to acquire the lock on the BAM pipe. */ + qce_write(qce, REG_AUTH_SEG_CFG, 0); + + return qce_submit_cmd_desc_lock(qce); +} + +int qce_bam_unlock(struct qce_device *qce) +{ + qce_clear_bam_transaction(qce); + /* Dummy write to release the lock on the BAM pipe. */ + qce_write(qce, REG_AUTH_SEG_CFG, 0); + + return qce_submit_cmd_desc_unlock(qce); +} diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index e95e84486d9ae..ae5beed600d32 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -12,8 +12,10 @@ #include #include #include +#include +#include +#include #include -#include #include #include "core.h" @@ -91,13 +93,17 @@ static int qce_handle_queue(struct qce_device *qce, struct crypto_async_request *async_req, *backlog; int ret = 0, err; + ret = pm_runtime_resume_and_get(qce->dev); + if (ret < 0) + return ret; + scoped_guard(mutex, &qce->lock) { if (req) ret = crypto_enqueue_request(&qce->queue, req); /* busy, do not dequeue request */ if (qce->req) - return ret; + goto qce_suspend; backlog = crypto_get_backlog(&qce->queue); async_req = crypto_dequeue_request(&qce->queue); @@ -106,7 +112,7 @@ static int qce_handle_queue(struct qce_device *qce, } if (!async_req) - return ret; + goto qce_suspend; if (backlog) { scoped_guard(mutex, &qce->lock) @@ -119,6 +125,8 @@ static int qce_handle_queue(struct qce_device *qce, schedule_work(&qce->done_work); } +qce_suspend: + pm_runtime_put_autosuspend(qce->dev); return ret; } @@ -187,10 +195,19 @@ static int qce_check_version(struct qce_device *qce) return 0; } +static void qce_crypto_unmap_dma(void *data) +{ + struct qce_device *qce = data; + + dma_unmap_resource(qce->dev, qce->base_dma, qce->dma_size, + DMA_BIDIRECTIONAL, 0); +} + static int qce_crypto_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct qce_device *qce; + struct resource *res; int ret; qce = devm_kzalloc(dev, sizeof(*qce), GFP_KERNEL); @@ -200,7 +217,7 @@ static int qce_crypto_probe(struct platform_device *pdev) qce->dev = dev; platform_set_drvdata(pdev, qce); - qce->base = devm_platform_ioremap_resource(pdev, 0); + qce->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(qce->base)) return PTR_ERR(qce->base); @@ -208,37 +225,47 @@ static int qce_crypto_probe(struct platform_device *pdev) if (ret < 0) return ret; - qce->core = devm_clk_get_optional_enabled(qce->dev, "core"); - if (IS_ERR(qce->core)) - return PTR_ERR(qce->core); + /* PM clock helpers: register device clocks */ + ret = devm_pm_clk_create(dev); + if (ret) + return ret; + + ret = pm_clk_add(dev, "core"); + if (ret) + return ret; - qce->iface = devm_clk_get_optional_enabled(qce->dev, "iface"); - if (IS_ERR(qce->iface)) - return PTR_ERR(qce->iface); + ret = pm_clk_add(dev, "iface"); + if (ret) + return ret; - qce->bus = devm_clk_get_optional_enabled(qce->dev, "bus"); - if (IS_ERR(qce->bus)) - return PTR_ERR(qce->bus); + ret = pm_clk_add(dev, "bus"); + if (ret) + return ret; - qce->mem_path = devm_of_icc_get(qce->dev, "memory"); + qce->mem_path = devm_of_icc_get(dev, "memory"); if (IS_ERR(qce->mem_path)) return PTR_ERR(qce->mem_path); - ret = icc_set_bw(qce->mem_path, QCE_DEFAULT_MEM_BANDWIDTH, QCE_DEFAULT_MEM_BANDWIDTH); + /* Enable runtime PM after clocks and ICC are acquired */ + ret = devm_pm_runtime_enable(dev); if (ret) return ret; - ret = devm_qce_dma_request(qce->dev, &qce->dma); + ret = pm_runtime_resume_and_get(dev); if (ret) return ret; + ret = devm_qce_dma_request(qce); + if (ret) + goto err_pm; + ret = qce_check_version(qce); if (ret) - return ret; + goto err_pm; ret = devm_mutex_init(qce->dev, &qce->lock); if (ret) - return ret; + goto err_pm; INIT_WORK(&qce->done_work, qce_req_done_work); crypto_init_queue(&qce->queue, QCE_QUEUE_LENGTH); @@ -246,9 +273,70 @@ static int qce_crypto_probe(struct platform_device *pdev) qce->async_req_enqueue = qce_async_request_enqueue; qce->async_req_done = qce_async_request_done; - return devm_qce_register_algs(qce); + ret = devm_qce_register_algs(qce); + if (ret) + goto err_pm; + + qce->dma_size = resource_size(res); + qce->base_dma = dma_map_resource(dev, res->start, qce->dma_size, + DMA_BIDIRECTIONAL, 0); + qce->base_phys = res->start; + ret = dma_mapping_error(dev, qce->base_dma); + if (ret) + goto err_pm; + + ret = devm_add_action_or_reset(qce->dev, qce_crypto_unmap_dma, qce); + if (ret) + goto err_pm; + + /* Configure autosuspend after successful init */ + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + return 0; + +err_pm: + pm_runtime_put(dev); + + return ret; +} + +static int __maybe_unused qce_runtime_suspend(struct device *dev) +{ + struct qce_device *qce = dev_get_drvdata(dev); + + icc_disable(qce->mem_path); + + return 0; +} + +static int __maybe_unused qce_runtime_resume(struct device *dev) +{ + struct qce_device *qce = dev_get_drvdata(dev); + int ret = 0; + + ret = icc_enable(qce->mem_path); + if (ret) + return ret; + + ret = icc_set_bw(qce->mem_path, QCE_DEFAULT_MEM_BANDWIDTH, QCE_DEFAULT_MEM_BANDWIDTH); + if (ret) + goto err_icc; + + return 0; + +err_icc: + icc_disable(qce->mem_path); + return ret; } +static const struct dev_pm_ops qce_crypto_pm_ops = { + SET_RUNTIME_PM_OPS(qce_runtime_suspend, qce_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + static const struct of_device_id qce_crypto_of_match[] = { { .compatible = "qcom,crypto-v5.1", }, { .compatible = "qcom,crypto-v5.4", }, @@ -262,6 +350,7 @@ static struct platform_driver qce_crypto_driver = { .driver = { .name = KBUILD_MODNAME, .of_match_table = qce_crypto_of_match, + .pm = &qce_crypto_pm_ops, }, }; module_platform_driver(qce_crypto_driver); diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h index eb6fa7a8b64a8..d238097f834e4 100644 --- a/drivers/crypto/qce/core.h +++ b/drivers/crypto/qce/core.h @@ -8,6 +8,7 @@ #include #include +#include #include "dma.h" @@ -26,6 +27,11 @@ * @dma: pointer to dma data * @burst_size: the crypto burst size * @pipe_pair_id: which pipe pair id the device using + * @base_dma: base DMA address + * @base_phys: base physical address + * @dma_size: size of memory mapped for DMA + * @read_buf: Buffer for DMA to write back to + * @read_buf_dma: Mapped address of the read buffer * @async_req_enqueue: invoked by every algorithm to enqueue a request * @async_req_done: invoked by every algorithm to finish its request */ @@ -42,6 +48,11 @@ struct qce_device { struct qce_dma_data dma; int burst_size; unsigned int pipe_pair_id; + dma_addr_t base_dma; + phys_addr_t base_phys; + size_t dma_size; + __le32 *read_buf; + dma_addr_t read_buf_dma; int (*async_req_enqueue)(struct qce_device *qce, struct crypto_async_request *req); void (*async_req_done)(struct qce_device *qce, int ret); diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c index 1dec7aea852dd..bfdc1397a289b 100644 --- a/drivers/crypto/qce/dma.c +++ b/drivers/crypto/qce/dma.c @@ -4,50 +4,156 @@ */ #include +#include +#include #include #include +#include "core.h" #include "dma.h" -static void qce_dma_release(void *data) +#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE) +#define QCE_BAM_CMD_SGL_SIZE 128 +#define QCE_BAM_CMD_ELEMENT_SIZE 128 +#define QCE_MAX_REG_READ 8 + +struct qce_desc_info { + struct dma_async_tx_descriptor *dma_desc; + enum dma_data_direction dir; +}; + +struct qce_bam_transaction { + struct bam_cmd_element bam_ce[QCE_BAM_CMD_ELEMENT_SIZE]; + struct scatterlist wr_sgl[QCE_BAM_CMD_SGL_SIZE]; + struct qce_desc_info *desc; + u32 bam_ce_idx; + u32 pre_bam_ce_idx; + u32 wr_sgl_cnt; +}; + +void qce_clear_bam_transaction(struct qce_device *qce) { - struct qce_dma_data *dma = data; + struct qce_bam_transaction *bam_txn = qce->dma.bam_txn; - dma_release_channel(dma->txchan); - dma_release_channel(dma->rxchan); - kfree(dma->result_buf); + bam_txn->bam_ce_idx = 0; + bam_txn->wr_sgl_cnt = 0; + bam_txn->bam_ce_idx = 0; + bam_txn->pre_bam_ce_idx = 0; } -int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma) +static int qce_do_submit_cmd_desc(struct qce_device *qce, unsigned long flags) { + struct qce_desc_info *qce_desc = qce->dma.bam_txn->desc; + struct qce_bam_transaction *bam_txn = qce->dma.bam_txn; + struct dma_async_tx_descriptor *dma_desc; + struct dma_chan *chan = qce->dma.rxchan; + unsigned long attrs = DMA_PREP_CMD | flags; + dma_cookie_t cookie; + unsigned int mapped; int ret; - dma->txchan = dma_request_chan(dev, "tx"); + mapped = dma_map_sg_attrs(qce->dev, bam_txn->wr_sgl, bam_txn->wr_sgl_cnt, + DMA_TO_DEVICE, attrs); + if (!mapped) + return -ENOMEM; + + dma_desc = dmaengine_prep_slave_sg(chan, bam_txn->wr_sgl, bam_txn->wr_sgl_cnt, + DMA_MEM_TO_DEV, attrs); + if (!dma_desc) { + dma_unmap_sg(qce->dev, bam_txn->wr_sgl, bam_txn->wr_sgl_cnt, DMA_TO_DEVICE); + return -ENOMEM; + } + + qce_desc->dma_desc = dma_desc; + cookie = dmaengine_submit(qce_desc->dma_desc); + + ret = dma_submit_error(cookie); + if (ret) + return ret; + + qce_dma_issue_pending(&qce->dma); + + return 0; +} + +int qce_submit_cmd_desc(struct qce_device *qce) +{ + return qce_do_submit_cmd_desc(qce, 0); +} + +int qce_submit_cmd_desc_lock(struct qce_device *qce) +{ + return qce_do_submit_cmd_desc(qce, DMA_PREP_LOCK); +} + +int qce_submit_cmd_desc_unlock(struct qce_device *qce) +{ + return qce_do_submit_cmd_desc(qce, DMA_PREP_UNLOCK); +} + +static void qce_prep_dma_cmd_desc(struct qce_device *qce, struct qce_dma_data *dma, + unsigned int addr, void *buf) +{ + struct qce_bam_transaction *bam_txn = dma->bam_txn; + struct bam_cmd_element *bam_ce_buf; + int bam_ce_size, cnt, idx; + + idx = bam_txn->bam_ce_idx; + bam_ce_buf = &bam_txn->bam_ce[idx]; + bam_prep_ce_le32(bam_ce_buf, addr, BAM_WRITE_COMMAND, *((__le32 *)buf)); + + bam_ce_buf = &bam_txn->bam_ce[bam_txn->pre_bam_ce_idx]; + bam_txn->bam_ce_idx++; + bam_ce_size = (bam_txn->bam_ce_idx - bam_txn->pre_bam_ce_idx) * sizeof(*bam_ce_buf); + + cnt = bam_txn->wr_sgl_cnt; + + sg_set_buf(&bam_txn->wr_sgl[cnt], bam_ce_buf, bam_ce_size); + + ++bam_txn->wr_sgl_cnt; + bam_txn->pre_bam_ce_idx = bam_txn->bam_ce_idx; +} + +void qce_write_dma(struct qce_device *qce, unsigned int offset, u32 val) +{ + unsigned int reg_addr = ((unsigned int)(qce->base_phys) + offset); + + qce_prep_dma_cmd_desc(qce, &qce->dma, reg_addr, &val); +} + +int devm_qce_dma_request(struct qce_device *qce) +{ + struct qce_dma_data *dma = &qce->dma; + struct device *dev = qce->dev; + + dma->txchan = devm_dma_request_chan(dev, "tx"); if (IS_ERR(dma->txchan)) return PTR_ERR(dma->txchan); - dma->rxchan = dma_request_chan(dev, "rx"); - if (IS_ERR(dma->rxchan)) { - ret = PTR_ERR(dma->rxchan); - goto error_rx; - } + dma->rxchan = devm_dma_request_chan(dev, "rx"); + if (IS_ERR(dma->rxchan)) + return PTR_ERR(dma->rxchan); - dma->result_buf = kmalloc(QCE_RESULT_BUF_SZ + QCE_IGNORE_BUF_SZ, - GFP_KERNEL); - if (!dma->result_buf) { - ret = -ENOMEM; - goto error_nomem; - } + dma->result_buf = devm_kmalloc(dev, QCE_RESULT_BUF_SZ + QCE_IGNORE_BUF_SZ, GFP_KERNEL); + if (!dma->result_buf) + return -ENOMEM; + + dma->bam_txn = devm_kzalloc(dev, sizeof(*dma->bam_txn), GFP_KERNEL); + if (!dma->bam_txn) + return -ENOMEM; + + dma->bam_txn->desc = devm_kzalloc(dev, sizeof(*dma->bam_txn->desc), GFP_KERNEL); + if (!dma->bam_txn->desc) + return -ENOMEM; - dma->ignore_buf = dma->result_buf + QCE_RESULT_BUF_SZ; + sg_init_table(dma->bam_txn->wr_sgl, QCE_BAM_CMD_SGL_SIZE); - return devm_add_action_or_reset(dev, qce_dma_release, dma); + qce->read_buf = dmam_alloc_coherent(qce->dev, QCE_MAX_REG_READ * sizeof(*qce->read_buf), + &qce->read_buf_dma, GFP_KERNEL); + if (!qce->read_buf) + return -ENOMEM; -error_nomem: - dma_release_channel(dma->rxchan); -error_rx: - dma_release_channel(dma->txchan); - return ret; + return 0; } struct scatterlist * diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h index 31629185000e1..4b3ee17db72e2 100644 --- a/drivers/crypto/qce/dma.h +++ b/drivers/crypto/qce/dma.h @@ -8,6 +8,9 @@ #include +struct qce_bam_transaction; +struct qce_device; + /* maximum data transfer block size between BAM and CE */ #define QCE_BAM_BURST_SIZE 64 @@ -23,7 +26,6 @@ struct qce_result_dump { u32 status2; }; -#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE) #define QCE_RESULT_BUF_SZ \ ALIGN(sizeof(struct qce_result_dump), QCE_BAM_BURST_SIZE) @@ -31,10 +33,10 @@ struct qce_dma_data { struct dma_chan *txchan; struct dma_chan *rxchan; struct qce_result_dump *result_buf; - void *ignore_buf; + struct qce_bam_transaction *bam_txn; }; -int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma); +int devm_qce_dma_request(struct qce_device *qce); int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in, int in_ents, struct scatterlist *sg_out, int out_ents, dma_async_tx_callback cb, void *cb_param); @@ -43,5 +45,12 @@ int qce_dma_terminate_all(struct qce_dma_data *dma); struct scatterlist * qce_sgtable_add(struct sg_table *sgt, struct scatterlist *sg_add, unsigned int max_len); +void qce_write_dma(struct qce_device *qce, unsigned int offset, u32 val); +int qce_submit_cmd_desc(struct qce_device *qce); +int qce_submit_cmd_desc_lock(struct qce_device *qce); +int qce_submit_cmd_desc_unlock(struct qce_device *qce); +void qce_clear_bam_transaction(struct qce_device *qce); +int qce_bam_lock(struct qce_device *qce); +int qce_bam_unlock(struct qce_device *qce); #endif /* _DMA_H_ */ diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 71b748183cfa8..a9db92b741c90 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -60,6 +60,10 @@ static void qce_ahash_done(void *data) rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]); rctx->byte_count[1] = cpu_to_be32(result->auth_byte_count[1]); + error = qce_bam_unlock(qce); + if (error) + dev_err(qce->dev, "ahash: failed to unlock the BAM\n"); + error = qce_check_status(qce, &status); if (error < 0) dev_dbg(qce->dev, "ahash operation error (%x)\n", status); @@ -90,6 +94,10 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) rctx->authklen = AES_KEYSIZE_128; } + ret = qce_bam_lock(qce); + if (ret) + return ret; + rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); if (rctx->src_nents < 0) { dev_err(qce->dev, "Invalid numbers of src SG.\n"); diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index ffb334eb5b346..a34a0b14a959b 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -51,6 +51,9 @@ static void qce_skcipher_done(void *data) dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); sg_free_table(&rctx->dst_tbl); + error = qce_bam_unlock(qce); + if (error) + dev_err(qce->dev, "skcipher: failed to unlock the BAM\n"); error = qce_check_status(qce, &status); if (error < 0) @@ -78,6 +81,10 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req) rctx->ivsize = crypto_skcipher_ivsize(skcipher); rctx->cryptlen = req->cryptlen; + ret = qce_bam_lock(qce); + if (ret) + return ret; + diff_dst = (req->src != req->dst) ? true : false; dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 426b24889af85..01195664cc7cd 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -669,8 +669,10 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq return -ENOMEM; if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), - rctx->adata, cryp->assoclen) != cryp->assoclen) + rctx->adata, cryp->assoclen) != cryp->assoclen) { + kfree(rctx->adata); return -EINVAL; + } } if (cryp->total_in) @@ -681,8 +683,11 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq ctx->rctx = rctx; ret = starfive_aes_hw_init(ctx); - if (ret) + if (ret) { + if (cryp->assoclen) + kfree(rctx->adata); return ret; + } if (!cryp->assoclen) goto write_text; diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index e6839c7bfb73a..54b7af4a7aee8 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -325,6 +325,7 @@ static int starfive_hash_digest(struct ahash_request *req) struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); struct starfive_cryp_dev *cryp = ctx->cryp; + int sg_len; memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx)); @@ -333,7 +334,10 @@ static int starfive_hash_digest(struct ahash_request *req) rctx->in_sg = req->src; rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); rctx->digsize = crypto_ahash_digestsize(tfm); - rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + if (sg_len < 0) + return sg_len; + rctx->in_sg_len = sg_len; ctx->rctx = rctx; return crypto_transfer_hash_request_to_engine(cryp->engine, req); diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 0e07d0523291a..9210cceb4b7b2 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -529,7 +529,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-tegra", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = 0xf, @@ -550,7 +550,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-tegra", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = 0xf, @@ -572,7 +572,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-tegra", .cra_priority = 500, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = 0xf, @@ -594,6 +594,7 @@ static struct tegra_se_alg tegra_aes_algs[] = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-tegra", .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_aes_ctx), .cra_alignmask = (__alignof__(u64) - 1), @@ -1922,6 +1923,7 @@ static struct tegra_se_alg tegra_aead_algs[] = { .cra_name = "gcm(aes)", .cra_driver_name = "gcm-aes-tegra", .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct tegra_aead_ctx), .cra_alignmask = 0xf, @@ -1944,6 +1946,7 @@ static struct tegra_se_alg tegra_aead_algs[] = { .cra_name = "ccm(aes)", .cra_driver_name = "ccm-aes-tegra", .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct tegra_aead_ctx), .cra_alignmask = 0xf, @@ -1971,7 +1974,7 @@ static struct tegra_se_alg tegra_cmac_algs[] = { .cra_name = "cmac(aes)", .cra_driver_name = "tegra-se-cmac", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_cmac_ctx), .cra_alignmask = 0, diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 4a298ace6e9f7..06bb5bf0fa335 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -761,7 +761,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha1", .cra_driver_name = "tegra-se-sha1", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -786,7 +786,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha224", .cra_driver_name = "tegra-se-sha224", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -811,7 +811,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha256", .cra_driver_name = "tegra-se-sha256", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -836,7 +836,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha384", .cra_driver_name = "tegra-se-sha384", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -861,7 +861,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha512", .cra_driver_name = "tegra-se-sha512", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -886,7 +886,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-224", .cra_driver_name = "tegra-se-sha3-224", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -911,7 +911,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-256", .cra_driver_name = "tegra-se-sha3-256", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -936,7 +936,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-384", .cra_driver_name = "tegra-se-sha3-384", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -961,7 +961,7 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "sha3-512", .cra_driver_name = "tegra-se-sha3-512", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA3_512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -988,7 +988,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha224)", .cra_driver_name = "tegra-se-hmac-sha224", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -1015,7 +1016,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha256)", .cra_driver_name = "tegra-se-hmac-sha256", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -1042,7 +1044,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha384)", .cra_driver_name = "tegra-se-hmac-sha384", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, @@ -1069,7 +1072,8 @@ static struct tegra_se_alg tegra_hash_algs[] = { .cra_name = "hmac(sha512)", .cra_driver_name = "tegra-se-hmac-sha512", .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_sha_ctx), .cra_alignmask = 0, diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 3d241446099cc..ccc6b5c1b24b3 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -75,15 +75,20 @@ static void virtcrypto_done_task(unsigned long data) struct data_queue *data_vq = (struct data_queue *)data; struct virtqueue *vq = data_vq->vq; struct virtio_crypto_request *vc_req; + unsigned long flags; unsigned int len; + spin_lock_irqsave(&data_vq->lock, flags); do { virtqueue_disable_cb(vq); while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { + spin_unlock_irqrestore(&data_vq->lock, flags); if (vc_req->alg_cb) vc_req->alg_cb(vc_req, len); + spin_lock_irqsave(&data_vq->lock, flags); } } while (!virtqueue_enable_cb(vq)); + spin_unlock_irqrestore(&data_vq->lock, flags); } static void virtcrypto_dataq_callback(struct virtqueue *vq) diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index 1b3fb21a2a7de..11053d1786d4d 100644 --- a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -541,8 +541,6 @@ int virtio_crypto_skcipher_crypt_req( if (ret < 0) return ret; - virtqueue_kick(data_vq->vq); - return 0; } diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 48b7314afdb88..599e126a18eb7 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -58,6 +58,7 @@ config CXL_ACPI tristate "CXL ACPI: Platform Support" depends on ACPI depends on ACPI_NUMA + depends on CXL_PMEM || !CXL_PMEM default CXL_BUS select ACPI_TABLE_LIB select ACPI_HMAT diff --git a/drivers/cxl/core/edac.c b/drivers/cxl/core/edac.c index 79994ca9bc9f3..81160260e26b7 100644 --- a/drivers/cxl/core/edac.c +++ b/drivers/cxl/core/edac.c @@ -1988,6 +1988,40 @@ static int cxl_memdev_soft_ppr_init(struct cxl_memdev *cxlmd, return 0; } +static void err_rec_free(void *_cxlmd) +{ + struct cxl_memdev *cxlmd = _cxlmd; + struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; + struct cxl_event_gen_media *rec_gen_media; + struct cxl_event_dram *rec_dram; + unsigned long index; + + cxlmd->err_rec_array = NULL; + xa_for_each(&array_rec->rec_dram, index, rec_dram) + kfree(rec_dram); + xa_destroy(&array_rec->rec_dram); + + xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) + kfree(rec_gen_media); + xa_destroy(&array_rec->rec_gen_media); + kfree(array_rec); +} + +static int devm_cxl_memdev_setup_err_rec(struct cxl_memdev *cxlmd) +{ + struct cxl_mem_err_rec *array_rec = + kzalloc(sizeof(*array_rec), GFP_KERNEL); + + if (!array_rec) + return -ENOMEM; + + xa_init(&array_rec->rec_gen_media); + xa_init(&array_rec->rec_dram); + cxlmd->err_rec_array = array_rec; + + return devm_add_action_or_reset(&cxlmd->dev, err_rec_free, cxlmd); +} + int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { struct edac_dev_feature ras_features[CXL_NR_EDAC_DEV_FEATURES]; @@ -2038,15 +2072,9 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) } if (repair_inst) { - struct cxl_mem_err_rec *array_rec = - devm_kzalloc(&cxlmd->dev, sizeof(*array_rec), - GFP_KERNEL); - if (!array_rec) - return -ENOMEM; - - xa_init(&array_rec->rec_gen_media); - xa_init(&array_rec->rec_dram); - cxlmd->err_rec_array = array_rec; + rc = devm_cxl_memdev_setup_err_rec(cxlmd); + if (rc) + return rc; } } @@ -2088,22 +2116,4 @@ int devm_cxl_region_edac_register(struct cxl_region *cxlr) } EXPORT_SYMBOL_NS_GPL(devm_cxl_region_edac_register, "CXL"); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ - struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; - struct cxl_event_gen_media *rec_gen_media; - struct cxl_event_dram *rec_dram; - unsigned long index; - - if (!IS_ENABLED(CONFIG_CXL_EDAC_MEM_REPAIR) || !array_rec) - return; - - xa_for_each(&array_rec->rec_dram, index, rec_dram) - kfree(rec_dram); - xa_destroy(&array_rec->rec_dram); - xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) - kfree(rec_gen_media); - xa_destroy(&array_rec->rec_gen_media); -} -EXPORT_SYMBOL_NS_GPL(devm_cxl_memdev_edac_release, "CXL"); diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index d3a094ca01ad9..4d5113b5e6316 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -94,7 +94,6 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) struct cxl_hdm *cxlhdm; void __iomem *hdm; u32 ctrl; - int i; if (!info) return false; @@ -113,22 +112,16 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) return false; /* - * If any decoders are committed already, there should not be any - * emulated DVSEC decoders. + * If HDM decoders are globally enabled, do not fall back to DVSEC + * range emulation. Zeroed decoder registers after region teardown + * do not imply absence of HDM capability. + * + * Falling back to DVSEC here would treat the decoder as AUTO and + * may incorrectly latch default interleave settings. */ - for (i = 0; i < cxlhdm->decoder_count; i++) { - ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); - dev_dbg(&info->port->dev, - "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n", - info->port->id, i, - FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl), - readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)), - readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)), - readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)), - readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i))); - if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) - return false; - } + ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); + if (ctrl & CXL_HDM_DECODER_ENABLE) + return false; return true; } @@ -403,7 +396,7 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, * is not set. */ if (cxled->part < 0) - for (int i = 0; cxlds->nr_partitions; i++) + for (int i = 0; i < cxlds->nr_partitions; i++) if (resource_contains(&cxlds->part[i].res, res)) { cxled->part = i; break; @@ -844,14 +837,13 @@ static int cxl_decoder_commit(struct cxl_decoder *cxld) scoped_guard(rwsem_read, &cxl_rwsem.dpa) setup_hw_decoder(cxld, hdm); - port->commit_end++; rc = cxld_await_commit(hdm, cxld->id); if (rc) { dev_dbg(&port->dev, "%s: error %d committing decoder\n", dev_name(&cxld->dev), rc); - cxld->reset(cxld); return rc; } + port->commit_end++; cxld->flags |= CXL_DECODER_F_ENABLE; return 0; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index e370d733e4400..4dff7f44d908e 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -27,7 +27,6 @@ static void cxl_memdev_release(struct device *dev) struct cxl_memdev *cxlmd = to_cxl_memdev(dev); ida_free(&cxl_memdev_ida, cxlmd->id); - devm_cxl_memdev_edac_release(cxlmd); kfree(cxlmd); } diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index 8853415c106a9..e3a8b8d813333 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -115,15 +115,17 @@ static void unregister_nvb(void *_cxl_nvb) device_unregister(&cxl_nvb->dev); } -/** - * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology - * @host: platform firmware root device - * @port: CXL port at the root of a CXL topology - * - * Return: bridge device that can host cxl_nvdimm objects - */ -struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, - struct cxl_port *port) +static bool cxl_nvdimm_bridge_failed_attach(struct cxl_nvdimm_bridge *cxl_nvb) +{ + struct device *dev = &cxl_nvb->dev; + + guard(device)(dev); + /* If the device has no driver, then it failed to attach. */ + return dev->driver == NULL; +} + +struct cxl_nvdimm_bridge *__devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port) { struct cxl_nvdimm_bridge *cxl_nvb; struct device *dev; @@ -145,6 +147,11 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, if (rc) goto err; + if (cxl_nvdimm_bridge_failed_attach(cxl_nvb)) { + unregister_nvb(cxl_nvb); + return ERR_PTR(-ENODEV); + } + rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb); if (rc) return ERR_PTR(rc); @@ -155,7 +162,7 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, put_device(dev); return ERR_PTR(rc); } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL"); +EXPORT_SYMBOL_FOR_MODULES(__devm_cxl_add_nvdimm_bridge, "cxl_pmem"); static void cxl_nvdimm_release(struct device *dev) { @@ -254,6 +261,21 @@ int devm_cxl_add_nvdimm(struct cxl_port *parent_port, if (!cxl_nvb) return -ENODEV; + /* + * Take the uport_dev lock to guard against race of nvdimm_bus object. + * cxl_acpi_probe() registers the nvdimm_bus and is done under the + * root port uport_dev lock. + * + * Take the cxl_nvb device lock to ensure that cxl_nvb driver is in a + * consistent state. And the driver registers nvdimm_bus. + */ + guard(device)(cxl_nvb->port->uport_dev); + guard(device)(&cxl_nvb->dev); + if (!cxl_nvb->nvdimm_bus) { + rc = -ENODEV; + goto err_alloc; + } + cxl_nvd = cxl_nvdimm_alloc(cxl_nvb, cxlmd); if (IS_ERR(cxl_nvd)) { rc = PTR_ERR(cxl_nvd); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 8128fd2b5b317..c53a13d4f1662 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -553,10 +553,13 @@ static void cxl_port_release(struct device *dev) xa_destroy(&port->dports); xa_destroy(&port->regions); ida_free(&cxl_port_ida, port->id); - if (is_cxl_root(port)) + + if (is_cxl_root(port)) { kfree(to_cxl_root(port)); - else + } else { + put_device(dev->parent); kfree(port); + } } static ssize_t decoders_committed_show(struct device *dev, @@ -722,6 +725,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, struct cxl_port *iter; dev->parent = &parent_port->dev; + get_device(dev->parent); port->depth = parent_port->depth + 1; port->parent_dport = parent_dport; @@ -823,16 +827,18 @@ DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) { + struct cxl_port *parent = parent_port_of(dport->port); struct dentry *dir; if (!einj_cxl_is_initialized()) return; /* - * dport_dev needs to be a PCIe port for CXL 2.0+ ports because - * EINJ expects a dport SBDF to be specified for 2.0 error injection. + * Protocol error injection is only available for CXL 2.0+ root ports + * and CXL 1.1 downstream ports */ - if (!dport->rch && !dev_is_pci(dport->dport_dev)) + if (!dport->rch && + !(dev_is_pci(dport->dport_dev) && parent && is_cxl_root(parent))) return; dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); @@ -1591,7 +1597,7 @@ static int update_decoder_targets(struct device *dev, void *data) cxlsd->target[i] = dport; dev_dbg(dev, "dport%d found in target list, index %d\n", dport->port_id, i); - return 1; + return 0; } } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 231ddccf89773..3a794278cc7fe 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -555,11 +555,16 @@ struct cxl_nvdimm_bridge { #define CXL_DEV_ID_LEN 19 +enum { + CXL_NVD_F_INVALIDATED = 0, +}; + struct cxl_nvdimm { struct device dev; struct cxl_memdev *cxlmd; u8 dev_id[CXL_DEV_ID_LEN]; /* for nvdimm, string of 'serial' */ u64 dirty_shutdowns; + unsigned long flags; }; struct cxl_pmem_region_mapping { @@ -866,6 +871,8 @@ void cxl_driver_unregister(struct cxl_driver *cxl_drv); struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev); struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, struct cxl_port *port); +struct cxl_nvdimm_bridge *__devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port); struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev); bool is_cxl_nvdimm(struct device *dev); int devm_cxl_add_nvdimm(struct cxl_port *parent_port, struct cxl_memdev *cxlmd); diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 434031a0c1f74..c12ab4fc95123 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -63,7 +63,7 @@ struct cxl_memdev { int depth; u8 scrub_cycle; int scrub_region_id; - void *err_rec_array; + struct cxl_mem_err_rec *err_rec_array; }; static inline struct cxl_memdev *to_cxl_memdev(struct device *dev) @@ -877,7 +877,6 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd); int devm_cxl_region_edac_register(struct cxl_region *cxlr); int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, union cxl_event *evt); int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd); #else static inline int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { return 0; } @@ -889,8 +888,6 @@ static inline int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, static inline int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt) { return 0; } -static inline void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ return; } #endif #ifdef CONFIG_CXL_SUSPEND diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index e197883690efc..3432fd83b1e2a 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -13,6 +13,20 @@ static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); +/** + * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology + * @host: platform firmware root device + * @port: CXL port at the root of a CXL topology + * + * Return: bridge device that can host cxl_nvdimm objects + */ +struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port) +{ + return __devm_cxl_add_nvdimm_bridge(host, port); +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL"); + static void clear_exclusive(void *mds) { clear_exclusive_cxl_commands(mds, exclusive_cmds); @@ -129,6 +143,9 @@ static int cxl_nvdimm_probe(struct device *dev) struct nvdimm *nvdimm; int rc; + if (test_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags)) + return -EBUSY; + set_exclusive_cxl_commands(mds, exclusive_cmds); rc = devm_add_action_or_reset(dev, clear_exclusive, mds); if (rc) @@ -309,8 +326,10 @@ static int detach_nvdimm(struct device *dev, void *data) scoped_guard(device, dev) { if (dev->driver) { cxl_nvd = to_cxl_nvdimm(dev); - if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) + if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) { release = true; + set_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags); + } } } if (release) @@ -353,6 +372,7 @@ static struct cxl_driver cxl_nvdimm_bridge_driver = { .probe = cxl_nvdimm_bridge_probe, .id = CXL_DEVICE_NVDIMM_BRIDGE, .drv = { + .probe_type = PROBE_FORCE_SYNCHRONOUS, .suppress_bind_attrs = true, }, }; @@ -534,7 +554,7 @@ static __exit void cxl_pmem_exit(void) MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support"); MODULE_LICENSE("GPL v2"); -module_init(cxl_pmem_init); +subsys_initcall(cxl_pmem_init); module_exit(cxl_pmem_exit); MODULE_IMPORT_NS("CXL"); MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h index 0867115aeef2e..c6ae27c982f43 100644 --- a/drivers/dax/dax-private.h +++ b/drivers/dax/dax-private.h @@ -67,14 +67,16 @@ struct dev_dax_range { /** * struct dev_dax - instance data for a subdivision of a dax region, and * data while the device is activated in the driver. - * @region - parent region - * @dax_dev - core dax functionality + * @region: parent region + * @dax_dev: core dax functionality + * @align: alignment of this instance * @target_node: effective numa node if dev_dax memory range is onlined * @dyn_id: is this a dynamic or statically created instance * @id: ida allocated id when the dax_region is not static * @ida: mapping id allocator - * @dev - device core - * @pgmap - pgmap for memmap setup / lifetime (driver owned) + * @dev: device core + * @pgmap: pgmap for memmap setup / lifetime (driver owned) + * @memmap_on_memory: allow kmem to put the memmap in the memory * @nr_range: size of @ranges * @ranges: range tuples of memory used */ diff --git a/drivers/devfreq/hisi_uncore_freq.c b/drivers/devfreq/hisi_uncore_freq.c index 96d1815059e32..c1ed70fa0a400 100644 --- a/drivers/devfreq/hisi_uncore_freq.c +++ b/drivers/devfreq/hisi_uncore_freq.c @@ -265,10 +265,11 @@ static int hisi_uncore_target(struct device *dev, unsigned long *freq, dev_err(dev, "Failed to get opp for freq %lu hz\n", *freq); return PTR_ERR(opp); } - dev_pm_opp_put(opp); data = (u32)(dev_pm_opp_get_freq(opp) / HZ_PER_MHZ); + dev_pm_opp_put(opp); + return hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_FREQ, &data); } diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index bd49f03742912..04bbd774b3b44 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -936,6 +936,7 @@ static void admac_remove(struct platform_device *pdev) } static const struct of_device_id admac_of_match[] = { + { .compatible = "apple,t8103-admac", }, { .compatible = "apple,admac", }, { } }; diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 2d147712cbc69..dffe5becd6c3c 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -1765,6 +1765,7 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) static void atc_free_chan_resources(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma_slave *atslave; BUG_ON(atc_chan_is_enabled(atchan)); @@ -1774,8 +1775,12 @@ static void atc_free_chan_resources(struct dma_chan *chan) /* * Free atslave allocated in at_dma_xlate() */ - kfree(chan->private); - chan->private = NULL; + atslave = chan->private; + if (atslave) { + put_device(atslave->dma_dev); + kfree(atslave); + chan->private = NULL; + } dev_vdbg(chan2dev(chan), "free_chan_resources: done\n"); } diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index 7f0e76439ce58..ed037fa883f6f 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -1699,7 +1699,7 @@ static int sba_probe(struct platform_device *pdev) /* Prealloc channel resource */ ret = sba_prealloc_channel_resources(sba); if (ret) - goto fail_free_mchan; + goto fail_put_mbox; /* Check availability of debugfs */ if (!debugfs_initialized()) @@ -1729,6 +1729,8 @@ static int sba_probe(struct platform_device *pdev) fail_free_resources: debugfs_remove_recursive(sba->root); sba_freeup_channel_resources(sba); +fail_put_mbox: + put_device(sba->mbox_dev); fail_free_mchan: mbox_free_channel(sba->mchan); return ret; @@ -1744,6 +1746,8 @@ static void sba_remove(struct platform_device *pdev) sba_freeup_channel_resources(sba); + put_device(sba->mbox_dev); + mbox_free_channel(sba->mchan); } diff --git a/drivers/dma/cv1800b-dmamux.c b/drivers/dma/cv1800b-dmamux.c index e900d65956171..f7a952fcbc7d7 100644 --- a/drivers/dma/cv1800b-dmamux.c +++ b/drivers/dma/cv1800b-dmamux.c @@ -102,11 +102,11 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec, struct llist_node *node; unsigned long flags; unsigned int chid, devid, cpuid; - int ret; + int ret = -EINVAL; if (dma_spec->args_count != DMAMUX_NCELLS) { dev_err(&pdev->dev, "invalid number of dma mux args\n"); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } devid = dma_spec->args[0]; @@ -115,18 +115,18 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec, if (devid > MAX_DMA_MAPPING_ID) { dev_err(&pdev->dev, "invalid device id: %u\n", devid); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } if (cpuid > MAX_DMA_CPU_ID) { dev_err(&pdev->dev, "invalid cpu id: %u\n", cpuid); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); if (!dma_spec->np) { dev_err(&pdev->dev, "can't get dma master\n"); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } spin_lock_irqsave(&dmamux->lock, flags); @@ -136,8 +136,6 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec, if (map->peripheral == devid && map->cpu == cpuid) goto found; } - - ret = -EINVAL; goto failed; } else { node = llist_del_first(&dmamux->free_maps); @@ -171,12 +169,17 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec, dev_dbg(&pdev->dev, "register channel %u for req %u (cpu %u)\n", chid, devid, cpuid); + put_device(&pdev->dev); + return map; failed: spin_unlock_irqrestore(&dmamux->lock, flags); of_node_put(dma_spec->np); dev_err(&pdev->dev, "errno %d\n", ret); +err_put_pdev: + put_device(&pdev->dev); + return ERR_PTR(ret); } diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 5b06b0dc67ee1..0f25f6d8ae71f 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -233,11 +233,9 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) unsigned int flags = 0; unsigned int val; - if (!chan->hw_sg) { - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); - if (val) /* Queue is full, wait for the next SOT IRQ */ - return; - } + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; desc = chan->next_desc; @@ -247,6 +245,7 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) return; list_move_tail(&vdesc->node, &chan->active_descs); desc = to_axi_dmac_desc(vdesc); + chan->next_desc = desc; } sg = &desc->sg[desc->num_submitted]; @@ -265,8 +264,6 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) else chan->next_desc = NULL; flags |= AXI_DMAC_FLAG_LAST; - } else { - chan->next_desc = desc; } sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index e3f8db4fe909a..ce8f7254bab21 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -252,10 +252,10 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) lower_32_bits(chunk->ll_region.paddr)); SET_CH_32(dw, chan->dir, chan->id, llp.msb, upper_32_bits(chunk->ll_region.paddr)); + /* Set consumer cycle */ + SET_CH_32(dw, chan->dir, chan->id, cycle_sync, + HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); } - /* Set consumer cycle */ - SET_CH_32(dw, chan->dir, chan->id, cycle_sync, - HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); dw_hdma_v0_sync_ll_data(chunk); diff --git a/drivers/dma/dw/rzn1-dmamux.c b/drivers/dma/dw/rzn1-dmamux.c index deadf135681b6..cbec277af4dd3 100644 --- a/drivers/dma/dw/rzn1-dmamux.c +++ b/drivers/dma/dw/rzn1-dmamux.c @@ -90,7 +90,7 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, if (test_and_set_bit(map->req_idx, dmamux->used_chans)) { ret = -EBUSY; - goto free_map; + goto put_dma_spec_np; } mask = BIT(map->req_idx); @@ -103,6 +103,8 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, clear_bitmap: clear_bit(map->req_idx, dmamux->used_chans); +put_dma_spec_np: + of_node_put(dma_spec->np); free_map: kfree(map); put_device: diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index 4976d7dde0809..11655dcc4d6c1 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -852,6 +852,7 @@ int fsl_edma_alloc_chan_resources(struct dma_chan *chan) free_irq(fsl_chan->txirq, fsl_chan); err_txirq: dma_pool_destroy(fsl_chan->tcd_pool); + clk_disable_unprepare(fsl_chan->clk); return ret; } diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 97583c7d51a2e..fbc12521da266 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -317,10 +317,8 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, return NULL; i = fsl_chan - fsl_edma->chans; - fsl_chan->priority = dma_spec->args[1]; - fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; - fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; - fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + if (!b_chmux && i != dma_spec->args[0]) + continue; if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) continue; @@ -328,17 +326,15 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) continue; - if (!b_chmux && i == dma_spec->args[0]) { - chan = dma_get_slave_channel(chan); - chan->device->privatecnt++; - return chan; - } else if (b_chmux && !fsl_chan->srcid) { - /* if controller support channel mux, choose a free channel */ - chan = dma_get_slave_channel(chan); - chan->device->privatecnt++; - fsl_chan->srcid = dma_spec->args[0]; - return chan; - } + fsl_chan->srcid = dma_spec->args[0]; + fsl_chan->priority = dma_spec->args[1]; + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + return chan; } return NULL; } @@ -915,7 +911,6 @@ static void fsl_edma_remove(struct platform_device *pdev) of_dma_controller_free(np); dma_async_device_unregister(&fsl_edma->dma_dev); fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); } static int fsl_edma_suspend_late(struct device *dev) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 7e4715f927732..4105688cf3f06 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -158,11 +158,7 @@ static const struct device_type idxd_cdev_file_type = { static void idxd_cdev_dev_release(struct device *dev) { struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); - struct idxd_cdev_context *cdev_ctx; - struct idxd_wq *wq = idxd_cdev->wq; - cdev_ctx = &ictx[wq->idxd->data->type]; - ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); kfree(idxd_cdev); } @@ -582,11 +578,15 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) void idxd_wq_del_cdev(struct idxd_wq *wq) { + struct idxd_cdev_context *cdev_ctx; struct idxd_cdev *idxd_cdev; idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); + + cdev_ctx = &ictx[wq->idxd->data->type]; + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); put_device(cdev_dev(idxd_cdev)); } diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c index eff9943f1a42e..95b8ef9586338 100644 --- a/drivers/dma/idxd/compat.c +++ b/drivers/dma/idxd/compat.c @@ -20,11 +20,16 @@ static ssize_t unbind_store(struct device_driver *drv, const char *buf, size_t c int rc = -ENODEV; dev = bus_find_device_by_name(bus, NULL, buf); - if (dev && dev->driver) { + if (!dev) + return -ENODEV; + + if (dev->driver) { device_driver_detach(dev); rc = count; } + put_device(dev); + return rc; } static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store); @@ -38,9 +43,12 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t cou struct idxd_dev *idxd_dev; dev = bus_find_device_by_name(bus, NULL, buf); - if (!dev || dev->driver || drv != &dsa_drv.drv) + if (!dev) return -ENODEV; + if (dev->driver || drv != &dsa_drv.drv) + goto err_put_dev; + idxd_dev = confdev_to_idxd_dev(dev); if (is_idxd_dev(idxd_dev)) { alt_drv = driver_find("idxd", bus); @@ -53,13 +61,20 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t cou alt_drv = driver_find("user", bus); } if (!alt_drv) - return -ENODEV; + goto err_put_dev; rc = device_driver_attach(alt_drv, dev); if (rc < 0) - return rc; + goto err_put_dev; + + put_device(dev); return count; + +err_put_dev: + put_device(dev); + + return rc; } static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store); diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 5cf419fe6b464..746d9edbba164 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -174,6 +174,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq) free_descs(wq); dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); sbitmap_queue_free(&wq->sbq); + wq->type = IDXD_WQT_NONE; } EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, "IDXD"); @@ -367,7 +368,6 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) lockdep_assert_held(&wq->wq_lock); wq->state = IDXD_WQ_DISABLED; memset(wq->wqcfg, 0, idxd->wqcfg_size); - wq->type = IDXD_WQT_NONE; wq->threshold = 0; wq->priority = 0; wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; @@ -815,8 +815,7 @@ static void idxd_device_evl_free(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; struct idxd_evl *evl = idxd->evl; - gencfg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); - if (!gencfg.evl_en) + if (!evl) return; mutex_lock(&evl->lock); @@ -1107,7 +1106,11 @@ int idxd_device_config(struct idxd_device *idxd) { int rc; - lockdep_assert_held(&idxd->dev_lock); + guard(spinlock)(&idxd->dev_lock); + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return 0; + rc = idxd_wqs_setup(idxd); if (rc < 0) return rc; @@ -1434,11 +1437,7 @@ int idxd_drv_enable_wq(struct idxd_wq *wq) } } - rc = 0; - spin_lock(&idxd->dev_lock); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock(&idxd->dev_lock); + rc = idxd_device_config(idxd); if (rc < 0) { dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); goto err; @@ -1513,7 +1512,6 @@ void idxd_drv_disable_wq(struct idxd_wq *wq) idxd_wq_reset(wq); idxd_wq_free_resources(wq); percpu_ref_exit(&wq->wq_active); - wq->type = IDXD_WQT_NONE; wq->client_count = 0; } EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, "IDXD"); @@ -1534,10 +1532,7 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) } /* Device configuration */ - spin_lock(&idxd->dev_lock); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock(&idxd->dev_lock); + rc = idxd_device_config(idxd); if (rc < 0) return -ENXIO; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 2acc34b3daff8..afba88f9c3e43 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -962,7 +962,8 @@ static void idxd_device_config_restore(struct idxd_device *idxd, idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; - idxd->evl->size = saved_evl->size; + if (idxd->evl) + idxd->evl->size = saved_evl->size; for (i = 0; i < idxd->max_groups; i++) { struct idxd_group *saved_group, *group; @@ -1093,12 +1094,10 @@ static void idxd_reset_done(struct pci_dev *pdev) idxd_device_config_restore(idxd, idxd->idxd_saved); /* Re-configure IDXD device if allowed. */ - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { - rc = idxd_device_config(idxd); - if (rc < 0) { - dev_err(dev, "HALT: %s config fails\n", idxd_name); - goto out; - } + rc = idxd_device_config(idxd); + if (rc < 0) { + dev_err(dev, "HALT: %s config fails\n", idxd_name); + goto out; } /* Bind IDXD device to driver. */ @@ -1136,6 +1135,7 @@ static void idxd_reset_done(struct pci_dev *pdev) } out: kfree(idxd->idxd_saved); + idxd->idxd_saved = NULL; } static const struct pci_error_handlers idxd_error_handler = { diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 6db1c5fcedc58..03217041b8b3e 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -138,7 +138,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, */ list_for_each_entry_safe(d, t, &flist, list) { list_del_init(&d->list); - idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true, + idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true, NULL, NULL); } } diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 9f0701021af0e..cdd7a59140d90 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1812,6 +1812,7 @@ static void idxd_conf_device_release(struct device *dev) { struct idxd_device *idxd = confdev_to_idxd(dev); + destroy_workqueue(idxd->wq); kfree(idxd->groups); bitmap_free(idxd->wq_enable_map); kfree(idxd->wqs); diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 02f68b3285112..227398673b731 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -1286,7 +1286,6 @@ static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_wake_from_d3(pdev, false); } diff --git a/drivers/dma/lpc18xx-dmamux.c b/drivers/dma/lpc18xx-dmamux.c index 2b6436f4b1937..d3ff521951b83 100644 --- a/drivers/dma/lpc18xx-dmamux.c +++ b/drivers/dma/lpc18xx-dmamux.c @@ -57,30 +57,31 @@ static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec, struct lpc18xx_dmamux_data *dmamux = platform_get_drvdata(pdev); unsigned long flags; unsigned mux; + int ret = -EINVAL; if (dma_spec->args_count != 3) { dev_err(&pdev->dev, "invalid number of dma mux args\n"); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } mux = dma_spec->args[0]; if (mux >= dmamux->dma_master_requests) { dev_err(&pdev->dev, "invalid mux number: %d\n", dma_spec->args[0]); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } if (dma_spec->args[1] > LPC18XX_DMAMUX_MAX_VAL) { dev_err(&pdev->dev, "invalid dma mux value: %d\n", dma_spec->args[1]); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } /* The of_node_put() will be done in the core for the node */ dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); if (!dma_spec->np) { dev_err(&pdev->dev, "can't get dma master\n"); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } spin_lock_irqsave(&dmamux->lock, flags); @@ -89,7 +90,8 @@ static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec, dev_err(&pdev->dev, "dma request %u busy with %u.%u\n", mux, mux, dmamux->muxes[mux].value); of_node_put(dma_spec->np); - return ERR_PTR(-EBUSY); + ret = -EBUSY; + goto err_put_pdev; } dmamux->muxes[mux].busy = true; @@ -106,7 +108,14 @@ static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec, dev_dbg(&pdev->dev, "mapping dmamux %u.%u to dma request %u\n", mux, dmamux->muxes[mux].value, mux); + put_device(&pdev->dev); + return &dmamux->muxes[mux]; + +err_put_pdev: + put_device(&pdev->dev); + + return ERR_PTR(ret); } static int lpc18xx_dmamux_probe(struct platform_device *pdev) diff --git a/drivers/dma/lpc32xx-dmamux.c b/drivers/dma/lpc32xx-dmamux.c index 351d7e23e6156..33be714740ddf 100644 --- a/drivers/dma/lpc32xx-dmamux.c +++ b/drivers/dma/lpc32xx-dmamux.c @@ -95,11 +95,12 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec, struct lpc32xx_dmamux_data *dmamux = platform_get_drvdata(pdev); unsigned long flags; struct lpc32xx_dmamux *mux = NULL; + int ret = -EINVAL; int i; if (dma_spec->args_count != 3) { dev_err(&pdev->dev, "invalid number of dma mux args\n"); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } for (i = 0; i < ARRAY_SIZE(lpc32xx_muxes); i++) { @@ -111,20 +112,20 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec, if (!mux) { dev_err(&pdev->dev, "invalid mux request number: %d\n", dma_spec->args[0]); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } if (dma_spec->args[2] > 1) { dev_err(&pdev->dev, "invalid dma mux value: %d\n", dma_spec->args[1]); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } /* The of_node_put() will be done in the core for the node */ dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); if (!dma_spec->np) { dev_err(&pdev->dev, "can't get dma master\n"); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } spin_lock_irqsave(&dmamux->lock, flags); @@ -133,7 +134,8 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec, dev_err(dev, "dma request signal %d busy, routed to %s\n", mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); of_node_put(dma_spec->np); - return ERR_PTR(-EBUSY); + ret = -EBUSY; + goto err_put_pdev; } mux->busy = true; @@ -148,7 +150,14 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec, dev_dbg(dev, "dma request signal %d routed to %s\n", mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); + put_device(&pdev->dev); + return mux; + +err_put_pdev: + put_device(&pdev->dev); + + return ERR_PTR(ret); } static int lpc32xx_dmamux_probe(struct platform_device *pdev) diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index 08e15177427b9..96c18c815f1df 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -41,7 +41,7 @@ #define VFF_STOP_CLR_B 0 #define VFF_EN_CLR_B 0 #define VFF_INT_EN_CLR_B 0 -#define VFF_4G_SUPPORT_CLR_B 0 +#define VFF_ADDR2_CLR_B 0 /* * interrupt trigger level for tx @@ -72,7 +72,7 @@ /* TX: the buffer size SW can write. RX: the buffer size HW can write. */ #define VFF_LEFT_SIZE 0x40 #define VFF_DEBUG_STATUS 0x50 -#define VFF_4G_SUPPORT 0x54 +#define VFF_ADDR2 0x54 struct mtk_uart_apdmadev { struct dma_device ddev; @@ -149,7 +149,7 @@ static void mtk_uart_apdma_start_tx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); @@ -192,7 +192,7 @@ static void mtk_uart_apdma_start_rx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B); @@ -298,7 +298,7 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) } if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B); + mtk_uart_apdma_write(c, VFF_ADDR2, VFF_ADDR2_CLR_B); err_pm: pm_runtime_put_noidle(mtkd->ddev.dev); diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index d07229a748868..d12e729ee12c5 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -152,8 +152,8 @@ struct mmp_pdma_phy { * * Controller Configuration: * @run_bits: Control bits in DCSR register for channel start/stop - * @dma_mask: DMA addressing capability of controller. 0 to use OF/platform - * settings, or explicit mask like DMA_BIT_MASK(32/64) + * @dma_width: DMA addressing width in bits (32 or 64). Determines the + * DMA mask capability of the controller hardware. */ struct mmp_pdma_ops { /* Hardware Register Operations */ @@ -173,7 +173,7 @@ struct mmp_pdma_ops { /* Controller Configuration */ u32 run_bits; - u64 dma_mask; + u32 dma_width; }; struct mmp_pdma_device { @@ -928,6 +928,7 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, { struct mmp_pdma_desc_sw *sw; struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device); + unsigned long flags; u64 curr; u32 residue = 0; bool passed = false; @@ -945,6 +946,8 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, else curr = pdev->ops->read_src_addr(chan->phy); + spin_lock_irqsave(&chan->desc_lock, flags); + list_for_each_entry(sw, &chan->chain_running, node) { u64 start, end; u32 len; @@ -989,6 +992,7 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, continue; if (sw->async_tx.cookie == cookie) { + spin_unlock_irqrestore(&chan->desc_lock, flags); return residue; } else { residue = 0; @@ -996,6 +1000,8 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, } } + spin_unlock_irqrestore(&chan->desc_lock, flags); + /* We should only get here in case of cyclic transactions */ return residue; } @@ -1172,7 +1178,7 @@ static const struct mmp_pdma_ops marvell_pdma_v1_ops = { .get_desc_src_addr = get_desc_src_addr_32, .get_desc_dst_addr = get_desc_dst_addr_32, .run_bits = (DCSR_RUN), - .dma_mask = 0, /* let OF/platform set DMA mask */ + .dma_width = 32, }; static const struct mmp_pdma_ops spacemit_k1_pdma_ops = { @@ -1185,7 +1191,7 @@ static const struct mmp_pdma_ops spacemit_k1_pdma_ops = { .get_desc_src_addr = get_desc_src_addr_64, .get_desc_dst_addr = get_desc_dst_addr_64, .run_bits = (DCSR_RUN | DCSR_LPAEEN), - .dma_mask = DMA_BIT_MASK(64), /* force 64-bit DMA addr capability */ + .dma_width = 64, }; static const struct of_device_id mmp_pdma_dt_ids[] = { @@ -1314,13 +1320,9 @@ static int mmp_pdma_probe(struct platform_device *op) pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - /* Set DMA mask based on ops->dma_mask, or OF/platform */ - if (pdev->ops->dma_mask) - dma_set_mask(pdev->dev, pdev->ops->dma_mask); - else if (pdev->dev->coherent_dma_mask) - dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask); - else - dma_set_mask(pdev->dev, DMA_BIT_MASK(64)); + /* Set DMA mask based on controller hardware capabilities */ + dma_set_mask_and_coherent(pdev->dev, + DMA_BIT_MASK(pdev->ops->dma_width)); ret = dma_async_device_register(&pdev->device); if (ret) { diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 2cf060174795f..68921d22ad7ab 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -58,6 +58,8 @@ struct bam_desc_hw { #define DESC_FLAG_EOB BIT(13) #define DESC_FLAG_NWD BIT(12) #define DESC_FLAG_CMD BIT(11) +#define DESC_FLAG_LOCK BIT(10) +#define DESC_FLAG_UNLOCK BIT(9) struct bam_async_desc { struct virt_dma_desc vd; @@ -111,6 +113,11 @@ struct reg_offset_data { unsigned int pipe_mult, evnt_mult, ee_mult; }; +struct bam_device_data { + const struct reg_offset_data *reg_info; + bool bam_pipe_lock; +}; + static const struct reg_offset_data bam_v1_3_reg_info[] = { [BAM_CTRL] = { 0x0F80, 0x00, 0x00, 0x00 }, [BAM_REVISION] = { 0x0F84, 0x00, 0x00, 0x00 }, @@ -140,6 +147,10 @@ static const struct reg_offset_data bam_v1_3_reg_info[] = { [BAM_P_FIFO_SIZES] = { 0x1020, 0x00, 0x40, 0x00 }, }; +static const struct bam_device_data bam_v1_3_data = { + .reg_info = bam_v1_3_reg_info, +}; + static const struct reg_offset_data bam_v1_4_reg_info[] = { [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 }, [BAM_REVISION] = { 0x0004, 0x00, 0x00, 0x00 }, @@ -169,6 +180,11 @@ static const struct reg_offset_data bam_v1_4_reg_info[] = { [BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 }, }; +static const struct bam_device_data bam_v1_4_data = { + .reg_info = bam_v1_4_reg_info, + .bam_pipe_lock = true, +}; + static const struct reg_offset_data bam_v1_7_reg_info[] = { [BAM_CTRL] = { 0x00000, 0x00, 0x00, 0x00 }, [BAM_REVISION] = { 0x01000, 0x00, 0x00, 0x00 }, @@ -198,6 +214,11 @@ static const struct reg_offset_data bam_v1_7_reg_info[] = { [BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 }, }; +static const struct bam_device_data bam_v1_7_data = { + .reg_info = bam_v1_7_reg_info, + .bam_pipe_lock = true, +}; + /* BAM CTRL */ #define BAM_SW_RST BIT(0) #define BAM_EN BIT(1) @@ -370,6 +391,9 @@ struct bam_chan { struct list_head desc_list; struct list_head node; + + /* Is the BAM currently locked? */ + bool locked; }; static inline struct bam_chan *to_bam_chan(struct dma_chan *common) @@ -391,7 +415,7 @@ struct bam_device { bool powered_remotely; u32 active_channels; - const struct reg_offset_data *layout; + const struct bam_device_data *dev_data; struct clk *bamclk; int irq; @@ -409,7 +433,7 @@ struct bam_device { static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe, enum bam_reg reg) { - const struct reg_offset_data r = bdev->layout[reg]; + const struct reg_offset_data r = bdev->dev_data->reg_info[reg]; return bdev->regs + r.base_offset + r.pipe_mult * pipe + @@ -651,6 +675,7 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, { struct bam_chan *bchan = to_bam_chan(chan); struct bam_device *bdev = bchan->bdev; + const struct bam_device_data *bdata = bdev->dev_data; struct bam_async_desc *async_desc; struct scatterlist *sg; u32 i; @@ -691,9 +716,34 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, unsigned int curr_offset = 0; do { - if (flags & DMA_PREP_CMD) + if (flags & DMA_PREP_CMD) { + if (!bdata->bam_pipe_lock && + (flags & (DMA_PREP_LOCK | DMA_PREP_UNLOCK))) { + dev_err(bdev->dev, "Device doesn't support BAM locking\n"); + return NULL; + } + desc->flags |= cpu_to_le16(DESC_FLAG_CMD); + if (bdata->bam_pipe_lock && (flags & DMA_PREP_LOCK)) { + if (bchan->locked) { + dev_err(bdev->dev, "BAM already locked\n"); + return NULL; + } + + desc->flags |= cpu_to_le16(DESC_FLAG_LOCK); + bchan->locked = true; + } else if (bdata->bam_pipe_lock && (flags & DMA_PREP_UNLOCK)) { + if (!bchan->locked) { + dev_err(bdev->dev, "BAM is not locked\n"); + return NULL; + } + + desc->flags |= cpu_to_le16(DESC_FLAG_UNLOCK); + bchan->locked = false; + } + } + desc->addr = cpu_to_le32(sg_dma_address(sg) + curr_offset); @@ -1225,9 +1275,9 @@ static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan, } static const struct of_device_id bam_of_match[] = { - { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info }, - { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info }, - { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info }, + { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_data }, + { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_data }, + { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_data }, {} }; @@ -1251,7 +1301,7 @@ static int bam_dma_probe(struct platform_device *pdev) return -ENODEV; } - bdev->layout = match->data; + bdev->dev_data = match->data; bdev->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(bdev->regs)) diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 8e87738086b25..67840ccca4293 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. * Copyright (c) 2020, Linaro Limited + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -67,6 +68,14 @@ #define TRE_DMA_LEN GENMASK(23, 0) #define TRE_DMA_IMMEDIATE_LEN GENMASK(3, 0) +/* Lock TRE */ +#define TRE_LOCK BIT(0) +#define TRE_MINOR_TYPE GENMASK(19, 16) +#define TRE_MAJOR_TYPE GENMASK(23, 20) + +/* Unlock TRE */ +#define TRE_UNLOCK BIT(8) + /* Register offsets from gpi-top */ #define GPII_n_CH_k_CNTXT_0_OFFS(n, k) (0x20000 + (0x4000 * (n)) + (0x80 * (k))) #define GPII_n_CH_k_CNTXT_0_EL_SIZE GENMASK(31, 24) @@ -518,7 +527,7 @@ struct gpii { bool ieob_set; }; -#define MAX_TRE 3 +#define MAX_TRE 5 struct gpi_desc { struct virt_dma_desc vd; @@ -1605,14 +1614,16 @@ static int gpi_peripheral_config(struct dma_chan *chan, struct dma_slave_config *config) { struct gchan *gchan = to_gchan(chan); + void *new_config; if (!config->peripheral_config) return -EINVAL; - gchan->config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT); - if (!gchan->config) + new_config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT); + if (!new_config) return -ENOMEM; + gchan->config = new_config; memcpy(gchan->config, config->peripheral_config, config->peripheral_size); return 0; @@ -1622,12 +1633,27 @@ static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, struct scatterlist *sgl, enum dma_transfer_direction direction) { struct gpi_i2c_config *i2c = chan->config; + enum gpi_lock_action lock_action = i2c->lock_action; struct device *dev = chan->gpii->gpi_dev->dev; unsigned int tre_idx = 0; dma_addr_t address; struct gpi_tre *tre; unsigned int i; + /* Optional lock TRE before transfer */ + if (lock_action == GPI_LOCK_ACQUIRE) { + tre = &desc->tre[tre_idx]; + tre_idx++; + + tre->dword[0] = 0; + tre->dword[1] = 0; + tre->dword[2] = 0; + tre->dword[3] = u32_encode_bits(1, TRE_LOCK); + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB); + tre->dword[3] |= u32_encode_bits(0, TRE_MINOR_TYPE); + tre->dword[3] |= u32_encode_bits(3, TRE_MAJOR_TYPE); + } + /* first create config tre if applicable */ if (i2c->set_config) { tre = &desc->tre[tre_idx]; @@ -1684,6 +1710,24 @@ static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + + /* If multi-owner and this is the release boundary, chain it */ + if (i2c->lock_action == GPI_LOCK_RELEASE) + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); + } + + /* Optional unlock TRE after transfer */ + if (lock_action == GPI_LOCK_RELEASE && i2c->op != I2C_READ) { + tre = &desc->tre[tre_idx]; + tre_idx++; + + tre->dword[0] = 0; + tre->dword[1] = 0; + tre->dword[2] = 0; + tre->dword[3] = u32_encode_bits(1, TRE_UNLOCK); + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB); + tre->dword[3] |= u32_encode_bits(1, TRE_MINOR_TYPE); + tre->dword[3] |= u32_encode_bits(3, TRE_MAJOR_TYPE); } for (i = 0; i < tre_idx; i++) diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index 1f687b08d6b86..818d1ef6f0bf9 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -297,13 +298,10 @@ static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); - unsigned long flags; dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); - local_irq_save(flags); rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); - local_irq_restore(flags); } static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) @@ -448,6 +446,7 @@ static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) if (!desc) break; + /* No need to lock. This is called only for the 1st client. */ list_add_tail(&desc->node, &channel->ld_free); channel->descs_allocated++; } @@ -503,18 +502,21 @@ rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", __func__, channel->index, &src, &dest, len); - if (list_empty(&channel->ld_free)) - return NULL; + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + if (list_empty(&channel->ld_free)) + return NULL; - desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); - desc->type = RZ_DMAC_DESC_MEMCPY; - desc->src = src; - desc->dest = dest; - desc->len = len; - desc->direction = DMA_MEM_TO_MEM; + desc->type = RZ_DMAC_DESC_MEMCPY; + desc->src = src; + desc->dest = dest; + desc->len = len; + desc->direction = DMA_MEM_TO_MEM; + + list_move_tail(channel->ld_free.next, &channel->ld_queue); + } - list_move_tail(channel->ld_free.next, &channel->ld_queue); return vchan_tx_prep(&channel->vc, &desc->vd, flags); } @@ -530,38 +532,45 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, int dma_length = 0; int i = 0; - if (list_empty(&channel->ld_free)) - return NULL; + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + if (list_empty(&channel->ld_free)) + return NULL; - desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); - for_each_sg(sgl, sg, sg_len, i) { - dma_length += sg_dma_len(sg); - } + for_each_sg(sgl, sg, sg_len, i) + dma_length += sg_dma_len(sg); - desc->type = RZ_DMAC_DESC_SLAVE_SG; - desc->sg = sgl; - desc->sgcount = sg_len; - desc->len = dma_length; - desc->direction = direction; + desc->type = RZ_DMAC_DESC_SLAVE_SG; + desc->sg = sgl; + desc->sgcount = sg_len; + desc->len = dma_length; + desc->direction = direction; - if (direction == DMA_DEV_TO_MEM) - desc->src = channel->src_per_address; - else - desc->dest = channel->dst_per_address; + if (direction == DMA_DEV_TO_MEM) + desc->src = channel->src_per_address; + else + desc->dest = channel->dst_per_address; + + list_move_tail(channel->ld_free.next, &channel->ld_queue); + } - list_move_tail(channel->ld_free.next, &channel->ld_queue); return vchan_tx_prep(&channel->vc, &desc->vd, flags); } static int rz_dmac_terminate_all(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_lmdesc *lmdesc = channel->lmdesc.base; unsigned long flags; + unsigned int i; LIST_HEAD(head); - rz_dmac_disable_hw(channel); spin_lock_irqsave(&channel->vc.lock, flags); + rz_dmac_disable_hw(channel); + for (i = 0; i < DMAC_NR_LMDESC; i++) + lmdesc[i].header = 0; + list_splice_tail_init(&channel->ld_active, &channel->ld_free); list_splice_tail_init(&channel->ld_queue, &channel->ld_free); vchan_get_all_descriptors(&channel->vc, &head); @@ -695,7 +704,9 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) if (chstat & CHSTAT_ER) { dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", channel->index, chstat); - rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); + + scoped_guard(spinlock_irqsave, &channel->vc.lock) + rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); goto done; } @@ -854,6 +865,13 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, return 0; } +static void rz_dmac_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) { struct device_node *np = dev->of_node; @@ -876,6 +894,10 @@ static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) return -ENODEV; } + ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev); + if (ret) + return ret; + dmac_index = args.args[0]; if (dmac_index > RZV2H_MAX_DMAC_INDEX) { dev_err(dev, "DMAC index %u invalid.\n", dmac_index); @@ -1055,8 +1077,6 @@ static void rz_dmac_remove(struct platform_device *pdev) reset_control_assert(dmac->rstc); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); - - platform_device_put(dmac->icu.pdev); } static const struct of_device_id of_rz_dmac_match[] = { diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c index 50e7106c5cb73..9500164c8f688 100644 --- a/drivers/dma/stm32/stm32-dma3.c +++ b/drivers/dma/stm32/stm32-dma3.c @@ -1914,12 +1914,7 @@ static struct platform_driver stm32_dma3_driver = { }, }; -static int __init stm32_dma3_init(void) -{ - return platform_driver_register(&stm32_dma3_driver); -} - -subsys_initcall(stm32_dma3_init); +module_platform_driver(stm32_dma3_driver); MODULE_DESCRIPTION("STM32 DMA3 controller driver"); MODULE_AUTHOR("Amelie Delaunay "); diff --git a/drivers/dma/stm32/stm32-dmamux.c b/drivers/dma/stm32/stm32-dmamux.c index 8d77e2a7939a0..2bd218dbabbb1 100644 --- a/drivers/dma/stm32/stm32-dmamux.c +++ b/drivers/dma/stm32/stm32-dmamux.c @@ -90,23 +90,25 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, struct stm32_dmamux_data *dmamux = platform_get_drvdata(pdev); struct stm32_dmamux *mux; u32 i, min, max; - int ret; + int ret = -EINVAL; unsigned long flags; if (dma_spec->args_count != 3) { dev_err(&pdev->dev, "invalid number of dma mux args\n"); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } if (dma_spec->args[0] > dmamux->dmamux_requests) { dev_err(&pdev->dev, "invalid mux request number: %d\n", dma_spec->args[0]); - return ERR_PTR(-EINVAL); + goto err_put_pdev; } mux = kzalloc(sizeof(*mux), GFP_KERNEL); - if (!mux) - return ERR_PTR(-ENOMEM); + if (!mux) { + ret = -ENOMEM; + goto err_put_pdev; + } spin_lock_irqsave(&dmamux->lock, flags); mux->chan_id = find_first_zero_bit(dmamux->dma_inuse, @@ -133,7 +135,6 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", i - 1); if (!dma_spec->np) { dev_err(&pdev->dev, "can't get dma master\n"); - ret = -EINVAL; goto error; } @@ -142,7 +143,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { spin_unlock_irqrestore(&dmamux->lock, flags); - goto error; + goto err_put_dma_spec_np; } spin_unlock_irqrestore(&dmamux->lock, flags); @@ -160,13 +161,20 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, dev_dbg(&pdev->dev, "Mapping DMAMUX(%u) to DMA%u(%u)\n", mux->request, mux->master, mux->chan_id); + put_device(&pdev->dev); + return mux; +err_put_dma_spec_np: + of_node_put(dma_spec->np); error: clear_bit(mux->chan_id, dmamux->dma_inuse); error_chan_id: kfree(mux); +err_put_pdev: + put_device(&pdev->dev); + return ERR_PTR(ret); } diff --git a/drivers/dma/stm32/stm32-mdma.c b/drivers/dma/stm32/stm32-mdma.c index 080c1c725216c..b87d41b234df1 100644 --- a/drivers/dma/stm32/stm32-mdma.c +++ b/drivers/dma/stm32/stm32-mdma.c @@ -731,7 +731,7 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct scatterlist *sg; dma_addr_t src_addr, dst_addr; - u32 m2m_hw_period, ccr, ctcr, ctbr; + u32 m2m_hw_period = 0, ccr = 0, ctcr, ctbr; int i, ret = 0; if (chan_config->m2m_hw) diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 2215ff877bf7d..f9d876deb1f05 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -583,6 +583,22 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) return ret; } +static u32 find_burst_size(const u32 burst_lengths, u32 maxburst) +{ + if (!maxburst) + return 1; + + if (BIT(maxburst) & burst_lengths) + return maxburst; + + /* Hardware only does power-of-two bursts. */ + for (u32 burst = rounddown_pow_of_two(maxburst); burst > 0; burst /= 2) + if (BIT(burst) & burst_lengths) + return burst; + + return 1; +} + static int set_config(struct sun6i_dma_dev *sdev, struct dma_slave_config *sconfig, enum dma_transfer_direction direction, @@ -616,15 +632,13 @@ static int set_config(struct sun6i_dma_dev *sdev, return -EINVAL; if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths)) return -EINVAL; - if (!(BIT(src_maxburst) & sdev->cfg->src_burst_lengths)) - return -EINVAL; - if (!(BIT(dst_maxburst) & sdev->cfg->dst_burst_lengths)) - return -EINVAL; src_width = convert_buswidth(src_addr_width); dst_width = convert_buswidth(dst_addr_width); - dst_burst = convert_burst(dst_maxburst); - src_burst = convert_burst(src_maxburst); + src_burst = find_burst_size(sdev->cfg->src_burst_lengths, src_maxburst); + dst_burst = find_burst_size(sdev->cfg->dst_burst_lengths, dst_maxburst); + dst_burst = convert_burst(dst_burst); + src_burst = convert_burst(src_burst); *p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) | DMA_CHAN_CFG_DST_WIDTH(dst_width); diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index fad896ff29a2d..812f64569e6d8 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -429,10 +429,17 @@ static void tegra_adma_stop(struct tegra_adma_chan *tdc) return; } - kfree(tdc->desc); + vchan_terminate_vdesc(&tdc->desc->vd); tdc->desc = NULL; } +static void tegra_adma_synchronize(struct dma_chan *dc) +{ + struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); + + vchan_synchronize(&tdc->vc); +} + static void tegra_adma_start(struct tegra_adma_chan *tdc) { struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc); @@ -1155,6 +1162,7 @@ static int tegra_adma_probe(struct platform_device *pdev) tdma->dma_dev.device_config = tegra_adma_slave_config; tdma->dma_dev.device_tx_status = tegra_adma_tx_status; tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all; + tdma->dma_dev.device_synchronize = tegra_adma_synchronize; tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c index 7f17ee87a6dce..ff05b150ad372 100644 --- a/drivers/dma/ti/dma-crossbar.c +++ b/drivers/dma/ti/dma-crossbar.c @@ -79,34 +79,35 @@ static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec, { struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); struct ti_am335x_xbar_data *xbar = platform_get_drvdata(pdev); - struct ti_am335x_xbar_map *map; + struct ti_am335x_xbar_map *map = ERR_PTR(-EINVAL); if (dma_spec->args_count != 3) - return ERR_PTR(-EINVAL); + goto out_put_pdev; if (dma_spec->args[2] >= xbar->xbar_events) { dev_err(&pdev->dev, "Invalid XBAR event number: %d\n", dma_spec->args[2]); - return ERR_PTR(-EINVAL); + goto out_put_pdev; } if (dma_spec->args[0] >= xbar->dma_requests) { dev_err(&pdev->dev, "Invalid DMA request line number: %d\n", dma_spec->args[0]); - return ERR_PTR(-EINVAL); + goto out_put_pdev; } /* The of_node_put() will be done in the core for the node */ dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); if (!dma_spec->np) { dev_err(&pdev->dev, "Can't get DMA master\n"); - return ERR_PTR(-EINVAL); + goto out_put_pdev; } map = kzalloc(sizeof(*map), GFP_KERNEL); if (!map) { of_node_put(dma_spec->np); - return ERR_PTR(-ENOMEM); + map = ERR_PTR(-ENOMEM); + goto out_put_pdev; } map->dma_line = (u16)dma_spec->args[0]; @@ -120,6 +121,9 @@ static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec, ti_am335x_xbar_write(xbar->iomem, map->dma_line, map->mux_val); +out_put_pdev: + put_device(&pdev->dev); + return map; } @@ -288,6 +292,8 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec, ti_dra7_xbar_write(xbar->iomem, map->xbar_out, map->xbar_in); + put_device(&pdev->dev); + return map; } diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index 05228bf000333..624360423ef17 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -42,9 +42,9 @@ struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property) } ud = platform_get_drvdata(pdev); + put_device(&pdev->dev); if (!ud) { pr_debug("UDMA has not been probed\n"); - put_device(&pdev->dev); return ERR_PTR(-EPROBE_DEFER); } diff --git a/drivers/dma/ti/omap-dma.c b/drivers/dma/ti/omap-dma.c index 8c023c6e623a5..73ed4b7946304 100644 --- a/drivers/dma/ti/omap-dma.c +++ b/drivers/dma/ti/omap-dma.c @@ -1808,6 +1808,8 @@ static int omap_dma_probe(struct platform_device *pdev) if (rc) { pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", rc); + if (od->ll123_supported) + dma_pool_destroy(od->desc_pool); omap_dma_free(od); return rc; } @@ -1823,6 +1825,8 @@ static int omap_dma_probe(struct platform_device *pdev) if (rc) { pr_warn("OMAP-DMA: failed to register DMA controller\n"); dma_async_device_unregister(&od->ddev); + if (od->ll123_supported) + dma_pool_destroy(od->desc_pool); omap_dma_free(od); } } diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h index 6ad08878e9386..70bca92621aa4 100644 --- a/drivers/dma/xilinx/xdma-regs.h +++ b/drivers/dma/xilinx/xdma-regs.h @@ -9,6 +9,7 @@ /* The length of register space exposed to host */ #define XDMA_REG_SPACE_LEN 65536 +#define XDMA_MAX_REG_OFFSET (XDMA_REG_SPACE_LEN - 4) /* * maximum number of DMA channels for each direction: diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 0d88b1a670e14..58e01e22b9765 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -38,7 +38,7 @@ static const struct regmap_config xdma_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, - .max_register = XDMA_REG_SPACE_LEN, + .max_register = XDMA_MAX_REG_OFFSET, }; /** @@ -1236,8 +1236,8 @@ static int xdma_probe(struct platform_device *pdev) xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base, &xdma_regmap_config); - if (!xdev->rmap) { - xdma_err(xdev, "config regmap failed: %d", ret); + if (IS_ERR(xdev->rmap)) { + xdma_err(xdev, "config regmap failed: %pe", xdev->rmap); goto failed; } INIT_LIST_HEAD(&xdev->dma_dev.channels); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index fabff602065f6..7dec5e6babe14 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -131,6 +131,7 @@ #define XILINX_MCDMA_MAX_CHANS_PER_DEVICE 0x20 #define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x2 #define XILINX_CDMA_MAX_CHANS_PER_DEVICE 0x1 +#define XILINX_DMA_DFAULT_ADDRWIDTH 0x20 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK \ (XILINX_DMA_DMASR_FRM_CNT_IRQ | \ @@ -996,16 +997,16 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, struct xilinx_cdma_tx_segment, node); cdma_hw = &cdma_seg->hw; - residue += (cdma_hw->control - cdma_hw->status) & - chan->xdev->max_buffer_len; + residue += (cdma_hw->control & chan->xdev->max_buffer_len) - + (cdma_hw->status & chan->xdev->max_buffer_len); } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { axidma_seg = list_entry(entry, struct xilinx_axidma_tx_segment, node); axidma_hw = &axidma_seg->hw; - residue += (axidma_hw->control - axidma_hw->status) & - chan->xdev->max_buffer_len; + residue += (axidma_hw->control & chan->xdev->max_buffer_len) - + (axidma_hw->status & chan->xdev->max_buffer_len); } else { aximcdma_seg = list_entry(entry, @@ -1013,8 +1014,8 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, node); aximcdma_hw = &aximcdma_seg->hw; residue += - (aximcdma_hw->control - aximcdma_hw->status) & - chan->xdev->max_buffer_len; + (aximcdma_hw->control & chan->xdev->max_buffer_len) - + (aximcdma_hw->status & chan->xdev->max_buffer_len); } } @@ -1216,14 +1217,6 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) dma_cookie_init(dchan); - if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { - /* For AXI DMA resetting once channel will reset the - * other channel as well so enable the interrupts here. - */ - dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, - XILINX_DMA_DMAXR_ALL_IRQ_MASK); - } - if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_CDMA_CR_SGMODE); @@ -1545,8 +1538,29 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) if (chan->err) return; - if (list_empty(&chan->pending_list)) + if (list_empty(&chan->pending_list)) { + if (chan->cyclic) { + struct xilinx_dma_tx_descriptor *desc; + struct list_head *entry; + + desc = list_last_entry(&chan->done_list, + struct xilinx_dma_tx_descriptor, node); + list_for_each(entry, &desc->segments) { + struct xilinx_axidma_tx_segment *axidma_seg; + struct xilinx_axidma_desc_hw *axidma_hw; + axidma_seg = list_entry(entry, + struct xilinx_axidma_tx_segment, + node); + axidma_hw = &axidma_seg->hw; + axidma_hw->status = 0; + } + + list_splice_tail_init(&chan->done_list, &chan->active_list); + chan->desc_pendingcount = 0; + chan->idle = false; + } return; + } if (!chan->idle) return; @@ -1572,6 +1586,7 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) head_desc->async_tx.phys); reg &= ~XILINX_DMA_CR_DELAY_MAX; reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; + reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK; dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); xilinx_dma_start(chan); @@ -3002,7 +3017,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, return -EINVAL; } - xdev->common.directions |= chan->direction; + xdev->common.directions |= BIT(chan->direction); /* Request the interrupt */ chan->irq = of_irq_get(node, chan->tdest); @@ -3159,7 +3174,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) struct device_node *node = pdev->dev.of_node; struct xilinx_dma_device *xdev; struct device_node *child, *np = pdev->dev.of_node; - u32 num_frames, addr_width, len_width; + u32 num_frames, addr_width = XILINX_DMA_DFAULT_ADDRWIDTH, len_width; int i, err; /* Allocate and initialize the DMA engine structure */ @@ -3235,7 +3250,9 @@ static int xilinx_dma_probe(struct platform_device *pdev) err = of_property_read_u32(node, "xlnx,addrwidth", &addr_width); if (err < 0) - dev_warn(xdev->dev, "missing xlnx,addrwidth property\n"); + dev_warn(xdev->dev, + "missing xlnx,addrwidth property, using default value %d\n", + XILINX_DMA_DFAULT_ADDRWIDTH); if (addr_width > 32) xdev->ext_addr = true; diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index a461095efd8ac..8879a72351561 100644 --- a/drivers/dpll/dpll_core.c +++ b/drivers/dpll/dpll_core.c @@ -83,10 +83,8 @@ dpll_xa_ref_pin_add(struct xarray *xa_pins, struct dpll_pin *pin, if (ref->pin != pin) continue; reg = dpll_pin_registration_find(ref, ops, priv, cookie); - if (reg) { - refcount_inc(&ref->refcount); - return 0; - } + if (reg) + return -EEXIST; ref_exists = true; break; } @@ -164,10 +162,8 @@ dpll_xa_ref_dpll_add(struct xarray *xa_dplls, struct dpll_device *dpll, if (ref->dpll != dpll) continue; reg = dpll_pin_registration_find(ref, ops, priv, cookie); - if (reg) { - refcount_inc(&ref->refcount); - return 0; - } + if (reg) + return -EEXIST; ref_exists = true; break; } diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index a4153bcb6dcfe..64944f601ee5a 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -637,6 +637,10 @@ dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin, ret = dpll_msg_add_pin_freq(msg, pin, ref, extack); if (ret) return ret; + if (prop->phase_gran && + nla_put_u32(msg, DPLL_A_PIN_PHASE_ADJUST_GRAN, + prop->phase_gran)) + return -EMSGSIZE; if (nla_put_s32(msg, DPLL_A_PIN_PHASE_ADJUST_MIN, prop->phase_range.min)) return -EMSGSIZE; @@ -1261,7 +1265,13 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, if (phase_adj > pin->prop.phase_range.max || phase_adj < pin->prop.phase_range.min) { NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr, - "phase adjust value not supported"); + "phase adjust value of out range"); + return -EINVAL; + } + if (pin->prop.phase_gran && phase_adj % (s32)pin->prop.phase_gran) { + NL_SET_ERR_MSG_ATTR_FMT(extack, phase_adj_attr, + "phase adjust value not multiple of %u", + pin->prop.phase_gran); return -EINVAL; } diff --git a/drivers/dpll/zl3073x/Makefile b/drivers/dpll/zl3073x/Makefile index 84e22aae57e5f..bd324c7fe7101 100644 --- a/drivers/dpll/zl3073x/Makefile +++ b/drivers/dpll/zl3073x/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ZL3073X) += zl3073x.o -zl3073x-objs := core.o devlink.o dpll.o flash.o fw.o prop.o +zl3073x-objs := core.o devlink.o dpll.o flash.o fw.o \ + out.o prop.o ref.o synth.o obj-$(CONFIG_ZL3073X_I2C) += zl3073x_i2c.o zl3073x_i2c-objs := i2c.o diff --git a/drivers/dpll/zl3073x/core.c b/drivers/dpll/zl3073x/core.c index e42e527813cf8..b9b7c751b7602 100644 --- a/drivers/dpll/zl3073x/core.c +++ b/drivers/dpll/zl3073x/core.c @@ -129,47 +129,6 @@ const struct regmap_config zl3073x_regmap_config = { }; EXPORT_SYMBOL_NS_GPL(zl3073x_regmap_config, "ZL3073X"); -/** - * zl3073x_ref_freq_factorize - factorize given frequency - * @freq: input frequency - * @base: base frequency - * @mult: multiplier - * - * Checks if the given frequency can be factorized using one of the - * supported base frequencies. If so the base frequency and multiplier - * are stored into appropriate parameters if they are not NULL. - * - * Return: 0 on success, -EINVAL if the frequency cannot be factorized - */ -int -zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult) -{ - static const u16 base_freqs[] = { - 1, 2, 4, 5, 8, 10, 16, 20, 25, 32, 40, 50, 64, 80, 100, 125, - 128, 160, 200, 250, 256, 320, 400, 500, 625, 640, 800, 1000, - 1250, 1280, 1600, 2000, 2500, 3125, 3200, 4000, 5000, 6250, - 6400, 8000, 10000, 12500, 15625, 16000, 20000, 25000, 31250, - 32000, 40000, 50000, 62500, - }; - u32 div; - int i; - - for (i = 0; i < ARRAY_SIZE(base_freqs); i++) { - div = freq / base_freqs[i]; - - if (div <= U16_MAX && (freq % base_freqs[i]) == 0) { - if (base) - *base = base_freqs[i]; - if (mult) - *mult = div; - - return 0; - } - } - - return -EINVAL; -} - static bool zl3073x_check_reg(struct zl3073x_dev *zldev, unsigned int reg, size_t size) { @@ -593,190 +552,6 @@ int zl3073x_write_hwreg_seq(struct zl3073x_dev *zldev, return rc; } -/** - * zl3073x_ref_state_fetch - get input reference state - * @zldev: pointer to zl3073x_dev structure - * @index: input reference index to fetch state for - * - * Function fetches information for the given input reference that are - * invariant and stores them for later use. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index) -{ - struct zl3073x_ref *input = &zldev->ref[index]; - u8 ref_config; - int rc; - - /* If the input is differential then the configuration for N-pin - * reference is ignored and P-pin config is used for both. - */ - if (zl3073x_is_n_pin(index) && - zl3073x_ref_is_diff(zldev, index - 1)) { - input->enabled = zl3073x_ref_is_enabled(zldev, index - 1); - input->diff = true; - - return 0; - } - - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(index)); - if (rc) - return rc; - - /* Read ref_config register */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_CONFIG, &ref_config); - if (rc) - return rc; - - input->enabled = FIELD_GET(ZL_REF_CONFIG_ENABLE, ref_config); - input->diff = FIELD_GET(ZL_REF_CONFIG_DIFF_EN, ref_config); - - dev_dbg(zldev->dev, "REF%u is %s and configured as %s\n", index, - str_enabled_disabled(input->enabled), - input->diff ? "differential" : "single-ended"); - - return rc; -} - -/** - * zl3073x_out_state_fetch - get output state - * @zldev: pointer to zl3073x_dev structure - * @index: output index to fetch state for - * - * Function fetches information for the given output (not output pin) - * that are invariant and stores them for later use. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_out_state_fetch(struct zl3073x_dev *zldev, u8 index) -{ - struct zl3073x_out *out = &zldev->out[index]; - u8 output_ctrl, output_mode; - int rc; - - /* Read output configuration */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_CTRL(index), &output_ctrl); - if (rc) - return rc; - - /* Store info about output enablement and synthesizer the output - * is connected to. - */ - out->enabled = FIELD_GET(ZL_OUTPUT_CTRL_EN, output_ctrl); - out->synth = FIELD_GET(ZL_OUTPUT_CTRL_SYNTH_SEL, output_ctrl); - - dev_dbg(zldev->dev, "OUT%u is %s and connected to SYNTH%u\n", index, - str_enabled_disabled(out->enabled), out->synth); - - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(index)); - if (rc) - return rc; - - /* Read output_mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &output_mode); - if (rc) - return rc; - - /* Extract and store output signal format */ - out->signal_format = FIELD_GET(ZL_OUTPUT_MODE_SIGNAL_FORMAT, - output_mode); - - dev_dbg(zldev->dev, "OUT%u has signal format 0x%02x\n", index, - out->signal_format); - - return rc; -} - -/** - * zl3073x_synth_state_fetch - get synth state - * @zldev: pointer to zl3073x_dev structure - * @index: synth index to fetch state for - * - * Function fetches information for the given synthesizer that are - * invariant and stores them for later use. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 index) -{ - struct zl3073x_synth *synth = &zldev->synth[index]; - u16 base, m, n; - u8 synth_ctrl; - u32 mult; - int rc; - - /* Read synth control register */ - rc = zl3073x_read_u8(zldev, ZL_REG_SYNTH_CTRL(index), &synth_ctrl); - if (rc) - return rc; - - /* Store info about synth enablement and DPLL channel the synth is - * driven by. - */ - synth->enabled = FIELD_GET(ZL_SYNTH_CTRL_EN, synth_ctrl); - synth->dpll = FIELD_GET(ZL_SYNTH_CTRL_DPLL_SEL, synth_ctrl); - - dev_dbg(zldev->dev, "SYNTH%u is %s and driven by DPLL%u\n", index, - str_enabled_disabled(synth->enabled), synth->dpll); - - guard(mutex)(&zldev->multiop_lock); - - /* Read synth configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_SYNTH_MB_SEM, ZL_SYNTH_MB_SEM_RD, - ZL_REG_SYNTH_MB_MASK, BIT(index)); - if (rc) - return rc; - - /* The output frequency is determined by the following formula: - * base * multiplier * numerator / denominator - * - * Read registers with these values - */ - rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_BASE, &base); - if (rc) - return rc; - - rc = zl3073x_read_u32(zldev, ZL_REG_SYNTH_FREQ_MULT, &mult); - if (rc) - return rc; - - rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_M, &m); - if (rc) - return rc; - - rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_N, &n); - if (rc) - return rc; - - /* Check denominator for zero to avoid div by 0 */ - if (!n) { - dev_err(zldev->dev, - "Zero divisor for SYNTH%u retrieved from device\n", - index); - return -EINVAL; - } - - /* Compute and store synth frequency */ - zldev->synth[index].freq = div_u64(mul_u32_u32(base * m, mult), n); - - dev_dbg(zldev->dev, "SYNTH%u frequency: %u Hz\n", index, - zldev->synth[index].freq); - - return rc; -} - static int zl3073x_dev_state_fetch(struct zl3073x_dev *zldev) { @@ -816,6 +591,21 @@ zl3073x_dev_state_fetch(struct zl3073x_dev *zldev) return rc; } +static void +zl3073x_dev_ref_status_update(struct zl3073x_dev *zldev) +{ + int i, rc; + + for (i = 0; i < ZL3073X_NUM_REFS; i++) { + rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(i), + &zldev->ref[i].mon_status); + if (rc) + dev_warn(zldev->dev, + "Failed to get REF%u status: %pe\n", i, + ERR_PTR(rc)); + } +} + /** * zl3073x_ref_phase_offsets_update - update reference phase offsets * @zldev: pointer to zl3073x_dev structure @@ -935,6 +725,9 @@ zl3073x_dev_periodic_work(struct kthread_work *work) struct zl3073x_dpll *zldpll; int rc; + /* Update input references status */ + zl3073x_dev_ref_status_update(zldev); + /* Update DPLL-to-connected-ref phase offsets registers */ rc = zl3073x_ref_phase_offsets_update(zldev, -1); if (rc) @@ -1185,11 +978,7 @@ zl3073x_devm_dpll_init(struct zl3073x_dev *zldev, u8 num_dplls) } /* Add devres action to release DPLL related resources */ - rc = devm_add_action_or_reset(zldev->dev, zl3073x_dev_dpll_fini, zldev); - if (rc) - goto error; - - return 0; + return devm_add_action_or_reset(zldev->dev, zl3073x_dev_dpll_fini, zldev); error: zl3073x_dev_dpll_fini(zldev); @@ -1230,6 +1019,7 @@ int zl3073x_dev_probe(struct zl3073x_dev *zldev, "Unknown or non-match chip ID: 0x%0x\n", id); } + zldev->chip_id = id; /* Read revision, firmware version and custom config version */ rc = zl3073x_read_u16(zldev, ZL_REG_REVISION, &revision); diff --git a/drivers/dpll/zl3073x/core.h b/drivers/dpll/zl3073x/core.h index 1dca4ddcf2350..08f1b49cccca4 100644 --- a/drivers/dpll/zl3073x/core.h +++ b/drivers/dpll/zl3073x/core.h @@ -9,7 +9,10 @@ #include #include +#include "out.h" +#include "ref.h" #include "regs.h" +#include "synth.h" struct device; struct regmap; @@ -27,47 +30,12 @@ struct zl3073x_dpll; #define ZL3073X_NUM_PINS (ZL3073X_NUM_INPUT_PINS + \ ZL3073X_NUM_OUTPUT_PINS) -/** - * struct zl3073x_ref - input reference invariant info - * @enabled: input reference is enabled or disabled - * @diff: true if input reference is differential - * @ffo: current fractional frequency offset - */ -struct zl3073x_ref { - bool enabled; - bool diff; - s64 ffo; -}; - -/** - * struct zl3073x_out - output invariant info - * @enabled: out is enabled or disabled - * @synth: synthesizer the out is connected to - * @signal_format: out signal format - */ -struct zl3073x_out { - bool enabled; - u8 synth; - u8 signal_format; -}; - -/** - * struct zl3073x_synth - synthesizer invariant info - * @freq: synthesizer frequency - * @dpll: ID of DPLL the synthesizer is driven by - * @enabled: synth is enabled or disabled - */ -struct zl3073x_synth { - u32 freq; - u8 dpll; - bool enabled; -}; - /** * struct zl3073x_dev - zl3073x device * @dev: pointer to device * @regmap: regmap to access device registers * @multiop_lock: to serialize multiple register operations + * @chip_id: chip ID read from hardware * @ref: array of input references' invariants * @out: array of outs' invariants * @synth: array of synths' invariants @@ -81,6 +49,7 @@ struct zl3073x_dev { struct device *dev; struct regmap *regmap; struct mutex multiop_lock; + u16 chip_id; /* Invariants */ struct zl3073x_ref ref[ZL3073X_NUM_REFS]; @@ -175,9 +144,34 @@ int zl3073x_write_hwreg_seq(struct zl3073x_dev *zldev, * Misc operations *****************/ -int zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult); int zl3073x_ref_phase_offsets_update(struct zl3073x_dev *zldev, int channel); +/** + * zl3073x_dev_is_ref_phase_comp_32bit - check ref phase comp register size + * @zldev: pointer to zl3073x device + * + * Some chip IDs have a 32-bit wide ref_phase_offset_comp register instead + * of the default 48-bit. + * + * Return: true if the register is 32-bit, false if 48-bit + */ +static inline bool +zl3073x_dev_is_ref_phase_comp_32bit(struct zl3073x_dev *zldev) +{ + switch (zldev->chip_id) { + case 0x0E30: + case 0x0E93: + case 0x0E94: + case 0x0E95: + case 0x0E96: + case 0x0E97: + case 0x1F60: + return true; + default: + return false; + } +} + static inline bool zl3073x_is_n_pin(u8 id) { @@ -217,172 +211,218 @@ zl3073x_output_pin_out_get(u8 id) } /** - * zl3073x_ref_ffo_get - get current fractional frequency offset + * zl3073x_dev_ref_ffo_get - get current fractional frequency offset * @zldev: pointer to zl3073x device * @index: input reference index * * Return: the latest measured fractional frequency offset */ static inline s64 -zl3073x_ref_ffo_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_ref_ffo_get(struct zl3073x_dev *zldev, u8 index) +{ + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_ffo_get(ref); +} + +/** + * zl3073x_dev_ref_freq_get - get input reference frequency + * @zldev: pointer to zl3073x device + * @index: input reference index + * + * Return: frequency of given input reference + */ +static inline u32 +zl3073x_dev_ref_freq_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->ref[index].ffo; + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_freq_get(ref); } /** - * zl3073x_ref_is_diff - check if the given input reference is differential + * zl3073x_dev_ref_is_diff - check if the given input reference is differential * @zldev: pointer to zl3073x device * @index: input reference index * * Return: true if reference is differential, false if reference is single-ended */ static inline bool -zl3073x_ref_is_diff(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_ref_is_diff(struct zl3073x_dev *zldev, u8 index) { - return zldev->ref[index].diff; + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_is_diff(ref); } /** - * zl3073x_ref_is_enabled - check if the given input reference is enabled + * zl3073x_dev_ref_is_enabled - check if the given input reference is enabled * @zldev: pointer to zl3073x device * @index: input reference index * * Return: true if input refernce is enabled, false otherwise */ static inline bool -zl3073x_ref_is_enabled(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_ref_is_enabled(struct zl3073x_dev *zldev, u8 index) { - return zldev->ref[index].enabled; + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_is_enabled(ref); +} + +/* + * zl3073x_dev_ref_is_status_ok - check the given input reference status + * @zldev: pointer to zl3073x device + * @index: input reference index + * + * Return: true if the status is ok, false otherwise + */ +static inline bool +zl3073x_dev_ref_is_status_ok(struct zl3073x_dev *zldev, u8 index) +{ + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_is_status_ok(ref); } /** - * zl3073x_synth_dpll_get - get DPLL ID the synth is driven by + * zl3073x_dev_synth_dpll_get - get DPLL ID the synth is driven by * @zldev: pointer to zl3073x device * @index: synth index * * Return: ID of DPLL the given synthetizer is driven by */ static inline u8 -zl3073x_synth_dpll_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_synth_dpll_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->synth[index].dpll; + const struct zl3073x_synth *synth; + + synth = zl3073x_synth_state_get(zldev, index); + return zl3073x_synth_dpll_get(synth); } /** - * zl3073x_synth_freq_get - get synth current freq + * zl3073x_dev_synth_freq_get - get synth current freq * @zldev: pointer to zl3073x device * @index: synth index * * Return: frequency of given synthetizer */ static inline u32 -zl3073x_synth_freq_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_synth_freq_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->synth[index].freq; + const struct zl3073x_synth *synth; + + synth = zl3073x_synth_state_get(zldev, index); + return zl3073x_synth_freq_get(synth); } /** - * zl3073x_synth_is_enabled - check if the given synth is enabled + * zl3073x_dev_synth_is_enabled - check if the given synth is enabled * @zldev: pointer to zl3073x device * @index: synth index * * Return: true if synth is enabled, false otherwise */ static inline bool -zl3073x_synth_is_enabled(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_synth_is_enabled(struct zl3073x_dev *zldev, u8 index) { - return zldev->synth[index].enabled; + const struct zl3073x_synth *synth; + + synth = zl3073x_synth_state_get(zldev, index); + return zl3073x_synth_is_enabled(synth); } /** - * zl3073x_out_synth_get - get synth connected to given output + * zl3073x_dev_out_synth_get - get synth connected to given output * @zldev: pointer to zl3073x device * @index: output index * * Return: index of synth connected to given output. */ static inline u8 -zl3073x_out_synth_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_synth_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->out[index].synth; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + + return zl3073x_out_synth_get(out); } /** - * zl3073x_out_is_enabled - check if the given output is enabled + * zl3073x_dev_out_is_enabled - check if the given output is enabled * @zldev: pointer to zl3073x device * @index: output index * * Return: true if the output is enabled, false otherwise */ static inline bool -zl3073x_out_is_enabled(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_is_enabled(struct zl3073x_dev *zldev, u8 index) { - u8 synth; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + const struct zl3073x_synth *synth; + u8 synth_id; /* Output is enabled only if associated synth is enabled */ - synth = zl3073x_out_synth_get(zldev, index); - if (zl3073x_synth_is_enabled(zldev, synth)) - return zldev->out[index].enabled; + synth_id = zl3073x_out_synth_get(out); + synth = zl3073x_synth_state_get(zldev, synth_id); - return false; + return zl3073x_synth_is_enabled(synth) && zl3073x_out_is_enabled(out); } /** - * zl3073x_out_signal_format_get - get output signal format + * zl3073x_dev_out_signal_format_get - get output signal format * @zldev: pointer to zl3073x device * @index: output index * * Return: signal format of given output */ static inline u8 -zl3073x_out_signal_format_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_signal_format_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->out[index].signal_format; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + + return zl3073x_out_signal_format_get(out); } /** - * zl3073x_out_dpll_get - get DPLL ID the output is driven by + * zl3073x_dev_out_dpll_get - get DPLL ID the output is driven by * @zldev: pointer to zl3073x device * @index: output index * * Return: ID of DPLL the given output is driven by */ static inline -u8 zl3073x_out_dpll_get(struct zl3073x_dev *zldev, u8 index) +u8 zl3073x_dev_out_dpll_get(struct zl3073x_dev *zldev, u8 index) { - u8 synth; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + const struct zl3073x_synth *synth; + u8 synth_id; /* Get synthesizer connected to given output */ - synth = zl3073x_out_synth_get(zldev, index); + synth_id = zl3073x_out_synth_get(out); + synth = zl3073x_synth_state_get(zldev, synth_id); /* Return DPLL that drives the synth */ - return zl3073x_synth_dpll_get(zldev, synth); + return zl3073x_synth_dpll_get(synth); } /** - * zl3073x_out_is_diff - check if the given output is differential + * zl3073x_dev_out_is_diff - check if the given output is differential * @zldev: pointer to zl3073x device * @index: output index * * Return: true if output is differential, false if output is single-ended */ static inline bool -zl3073x_out_is_diff(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_is_diff(struct zl3073x_dev *zldev, u8 index) { - switch (zl3073x_out_signal_format_get(zldev, index)) { - case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LVDS: - case ZL_OUTPUT_MODE_SIGNAL_FORMAT_DIFF: - case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LOWVCM: - return true; - default: - break; - } + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); - return false; + return zl3073x_out_is_diff(out); } /** - * zl3073x_output_pin_is_enabled - check if the given output pin is enabled + * zl3073x_dev_output_pin_is_enabled - check if the given output pin is enabled * @zldev: pointer to zl3073x device * @id: output pin id * @@ -392,16 +432,21 @@ zl3073x_out_is_diff(struct zl3073x_dev *zldev, u8 index) * Return: true if output pin is enabled, false if output pin is disabled */ static inline bool -zl3073x_output_pin_is_enabled(struct zl3073x_dev *zldev, u8 id) +zl3073x_dev_output_pin_is_enabled(struct zl3073x_dev *zldev, u8 id) { - u8 output = zl3073x_output_pin_out_get(id); + u8 out_id = zl3073x_output_pin_out_get(id); + const struct zl3073x_out *out; + + out = zl3073x_out_state_get(zldev, out_id); - /* Check if the whole output is enabled */ - if (!zl3073x_out_is_enabled(zldev, output)) + /* Check if the output is enabled - call _dev_ helper that + * additionally checks for attached synth enablement. + */ + if (!zl3073x_dev_out_is_enabled(zldev, out_id)) return false; /* Check signal format */ - switch (zl3073x_out_signal_format_get(zldev, output)) { + switch (zl3073x_out_signal_format_get(out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_DISABLED: /* Both output pins are disabled by signal format */ return false; diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c index f93f9a4583243..d7194418d1568 100644 --- a/drivers/dpll/zl3073x/dpll.c +++ b/drivers/dpll/zl3073x/dpll.c @@ -35,6 +35,7 @@ * @prio: pin priority <0, 14> * @selectable: pin is selectable in automatic mode * @esync_control: embedded sync is controllable + * @phase_gran: phase adjustment granularity * @pin_state: last saved pin state * @phase_offset: last saved pin phase offset * @freq_offset: last saved fractional frequency offset @@ -49,6 +50,7 @@ struct zl3073x_dpll_pin { u8 prio; bool selectable; bool esync_control; + s32 phase_gran; enum dpll_pin_state pin_state; s64 phase_offset; s64 freq_offset; @@ -98,60 +100,6 @@ zl3073x_dpll_pin_direction_get(const struct dpll_pin *dpll_pin, void *pin_priv, return 0; } -/** - * zl3073x_dpll_input_ref_frequency_get - get input reference frequency - * @zldpll: pointer to zl3073x_dpll - * @ref_id: reference id - * @frequency: pointer to variable to store frequency - * - * Reads frequency of given input reference. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_dpll_input_ref_frequency_get(struct zl3073x_dpll *zldpll, u8 ref_id, - u32 *frequency) -{ - struct zl3073x_dev *zldev = zldpll->dev; - u16 base, mult, num, denom; - int rc; - - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref_id)); - if (rc) - return rc; - - /* Read registers to compute resulting frequency */ - rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_BASE, &base); - if (rc) - return rc; - rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_MULT, &mult); - if (rc) - return rc; - rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_M, &num); - if (rc) - return rc; - rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_N, &denom); - if (rc) - return rc; - - /* Sanity check that HW has not returned zero denominator */ - if (!denom) { - dev_err(zldev->dev, - "Zero divisor for ref %u frequency got from device\n", - ref_id); - return -EINVAL; - } - - /* Compute the frequency */ - *frequency = mul_u64_u32_div(base * mult, num, denom); - - return rc; -} - static int zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, void *pin_priv, @@ -163,39 +111,15 @@ zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 ref, ref_sync_ctrl, sync_mode; - u32 esync_div, ref_freq; - int rc; - - /* Get reference frequency */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, pin->id, &ref_freq); - if (rc) - return rc; - - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; - - /* Get ref sync mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_SYNC_CTRL, &ref_sync_ctrl); - if (rc) - return rc; - - /* Get esync divisor */ - rc = zl3073x_read_u32(zldev, ZL_REG_REF_ESYNC_DIV, &esync_div); - if (rc) - return rc; + const struct zl3073x_ref *ref; + u8 ref_id; - sync_mode = FIELD_GET(ZL_REF_SYNC_CTRL_MODE, ref_sync_ctrl); + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); - switch (sync_mode) { + switch (FIELD_GET(ZL_REF_SYNC_CTRL_MODE, ref->sync_ctrl)) { case ZL_REF_SYNC_CTRL_MODE_50_50_ESYNC_25_75: - esync->freq = (esync_div == ZL_REF_ESYNC_DIV_1HZ) ? 1 : 0; + esync->freq = ref->esync_n_div == ZL_REF_ESYNC_DIV_1HZ ? 1 : 0; esync->pulse = 25; break; default: @@ -207,7 +131,7 @@ zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, /* If the pin supports esync control expose its range but only * if the current reference frequency is > 1 Hz. */ - if (pin->esync_control && ref_freq > 1) { + if (pin->esync_control && zl3073x_ref_freq_get(ref) > 1) { esync->range = esync_freq_ranges; esync->range_num = ARRAY_SIZE(esync_freq_ranges); } else { @@ -215,7 +139,7 @@ zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, esync->range_num = 0; } - return rc; + return 0; } static int @@ -228,22 +152,11 @@ zl3073x_dpll_input_pin_esync_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 ref, ref_sync_ctrl, sync_mode; - int rc; - - guard(mutex)(&zldev->multiop_lock); + struct zl3073x_ref ref; + u8 ref_id, sync_mode; - /* Read reference configuration into mailbox */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; - - /* Get ref sync mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_SYNC_CTRL, &ref_sync_ctrl); - if (rc) - return rc; + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = *zl3073x_ref_state_get(zldev, ref_id); /* Use freq == 0 to disable esync */ if (!freq) @@ -251,25 +164,16 @@ zl3073x_dpll_input_pin_esync_set(const struct dpll_pin *dpll_pin, else sync_mode = ZL_REF_SYNC_CTRL_MODE_50_50_ESYNC_25_75; - ref_sync_ctrl &= ~ZL_REF_SYNC_CTRL_MODE; - ref_sync_ctrl |= FIELD_PREP(ZL_REF_SYNC_CTRL_MODE, sync_mode); - - /* Update ref sync control register */ - rc = zl3073x_write_u8(zldev, ZL_REG_REF_SYNC_CTRL, ref_sync_ctrl); - if (rc) - return rc; + ref.sync_ctrl &= ~ZL_REF_SYNC_CTRL_MODE; + ref.sync_ctrl |= FIELD_PREP(ZL_REF_SYNC_CTRL_MODE, sync_mode); if (freq) { - /* 1 Hz is only supported frequnecy currently */ - rc = zl3073x_write_u32(zldev, ZL_REG_REF_ESYNC_DIV, - ZL_REF_ESYNC_DIV_1HZ); - if (rc) - return rc; + /* 1 Hz is only supported frequency now */ + ref.esync_n_div = ZL_REF_ESYNC_DIV_1HZ; } - /* Commit reference configuration */ - return zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, - ZL_REG_REF_MB_MASK, BIT(ref)); + /* Update reference configuration */ + return zl3073x_ref_state_set(zldev, ref_id, &ref); } static int @@ -293,17 +197,12 @@ zl3073x_dpll_input_pin_frequency_get(const struct dpll_pin *dpll_pin, { struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dpll_pin *pin = pin_priv; - u32 ref_freq; - u8 ref; - int rc; + u8 ref_id; - /* Read and return ref frequency */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, ref, &ref_freq); - if (!rc) - *frequency = ref_freq; + ref_id = zl3073x_input_pin_ref_get(pin->id); + *frequency = zl3073x_dev_ref_freq_get(zldpll->dev, ref_id); - return rc; + return 0; } static int @@ -316,39 +215,18 @@ zl3073x_dpll_input_pin_frequency_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u16 base, mult; - u8 ref; - int rc; + struct zl3073x_ref ref; + u8 ref_id; - /* Get base frequency and multiplier for the requested frequency */ - rc = zl3073x_ref_freq_factorize(frequency, &base, &mult); - if (rc) - return rc; + /* Get reference state */ + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = *zl3073x_ref_state_get(zldev, ref_id); - guard(mutex)(&zldev->multiop_lock); + /* Update frequency */ + zl3073x_ref_freq_set(&ref, frequency); - /* Load reference configuration */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - - /* Update base frequency, multiplier, numerator & denominator */ - rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_BASE, base); - if (rc) - return rc; - rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_MULT, mult); - if (rc) - return rc; - rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_M, 1); - if (rc) - return rc; - rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_N, 1); - if (rc) - return rc; - - /* Commit reference configuration */ - return zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, - ZL_REG_REF_MB_MASK, BIT(ref)); + /* Commit reference state */ + return zl3073x_ref_state_set(zldev, ref_id, &ref); } /** @@ -495,19 +373,10 @@ zl3073x_dpll_connected_ref_get(struct zl3073x_dpll *zldpll, u8 *ref) if (rc) return rc; - if (ZL3073X_DPLL_REF_IS_VALID(*ref)) { - u8 ref_status; - - /* Read the reference monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(*ref), - &ref_status); - if (rc) - return rc; - - /* If the monitor indicates an error nothing is connected */ - if (ref_status != ZL_REF_MON_STATUS_OK) - *ref = ZL3073X_DPLL_REF_NONE; - } + /* If the monitor indicates an error nothing is connected */ + if (ZL3073X_DPLL_REF_IS_VALID(*ref) && + !zl3073x_dev_ref_is_status_ok(zldev, *ref)) + *ref = ZL3073X_DPLL_REF_NONE; return 0; } @@ -522,34 +391,25 @@ zl3073x_dpll_input_pin_phase_offset_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 conn_ref, ref, ref_status; + const struct zl3073x_ref *ref; + u8 conn_id, ref_id; s64 ref_phase; int rc; /* Get currently connected reference */ - rc = zl3073x_dpll_connected_ref_get(zldpll, &conn_ref); + rc = zl3073x_dpll_connected_ref_get(zldpll, &conn_id); if (rc) return rc; /* Report phase offset only for currently connected pin if the phase - * monitor feature is disabled. + * monitor feature is disabled and only if the input pin signal is + * present. */ - ref = zl3073x_input_pin_ref_get(pin->id); - if (!zldpll->phase_monitor && ref != conn_ref) { + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); + if ((!zldpll->phase_monitor && ref_id != conn_id) || + !zl3073x_ref_is_status_ok(ref)) { *phase_offset = 0; - - return 0; - } - - /* Get this pin monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), &ref_status); - if (rc) - return rc; - - /* Report phase offset only if the input pin signal is present */ - if (ref_status != ZL_REF_MON_STATUS_OK) { - *phase_offset = 0; - return 0; } @@ -559,20 +419,12 @@ zl3073x_dpll_input_pin_phase_offset_get(const struct dpll_pin *dpll_pin, * the phase offset is modded to the period of the signal * the dpll is locked to. */ - if (ZL3073X_DPLL_REF_IS_VALID(conn_ref) && conn_ref != ref) { + if (ZL3073X_DPLL_REF_IS_VALID(conn_id) && conn_id != ref_id) { u32 conn_freq, ref_freq; - /* Get frequency of connected ref */ - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, conn_ref, - &conn_freq); - if (rc) - return rc; - - /* Get frequency of given ref */ - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, ref, - &ref_freq); - if (rc) - return rc; + /* Get frequency of connected and given ref */ + conn_freq = zl3073x_dev_ref_freq_get(zldev, conn_id); + ref_freq = zl3073x_ref_freq_get(ref); if (conn_freq > ref_freq) { s64 conn_period, div_factor; @@ -599,33 +451,26 @@ zl3073x_dpll_input_pin_phase_adjust_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; + const struct zl3073x_ref *ref; s64 phase_comp; - u8 ref; - int rc; - - guard(mutex)(&zldev->multiop_lock); + u8 ref_id; /* Read reference configuration */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); - /* Read current phase offset compensation */ - rc = zl3073x_read_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, &phase_comp); - if (rc) - return rc; - - /* Perform sign extension for 48bit signed value */ - phase_comp = sign_extend64(phase_comp, 47); + /* Perform sign extension based on register width */ + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) + phase_comp = sign_extend64(ref->phase_comp, 31); + else + phase_comp = sign_extend64(ref->phase_comp, 47); /* Reverse two's complement negation applied during set and convert * to 32bit signed int */ *phase_adjust = (s32)-phase_comp; - return rc; + return 0; } static int @@ -639,32 +484,20 @@ zl3073x_dpll_input_pin_phase_adjust_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - s64 phase_comp; - u8 ref; - int rc; + struct zl3073x_ref ref; + u8 ref_id; + + /* Read reference configuration */ + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = *zl3073x_ref_state_get(zldev, ref_id); /* The value in the register is stored as two's complement negation * of requested value. */ - phase_comp = -phase_adjust; + ref.phase_comp = -phase_adjust; - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; - - /* Write the requested value into the compensation register */ - rc = zl3073x_write_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, phase_comp); - if (rc) - return rc; - - /* Commit reference configuration */ - return zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, - ZL_REG_REF_MB_MASK, BIT(ref)); + /* Update reference configuration */ + return zl3073x_ref_state_set(zldev, ref_id, &ref); } /** @@ -775,7 +608,7 @@ zl3073x_dpll_ref_state_get(struct zl3073x_dpll_pin *pin, { struct zl3073x_dpll *zldpll = pin->dpll; struct zl3073x_dev *zldev = zldpll->dev; - u8 ref, ref_conn, status; + u8 ref, ref_conn; int rc; ref = zl3073x_input_pin_ref_get(pin->id); @@ -795,20 +628,9 @@ zl3073x_dpll_ref_state_get(struct zl3073x_dpll_pin *pin, * pin as selectable. */ if (zldpll->refsel_mode == ZL_DPLL_MODE_REFSEL_MODE_AUTO && - pin->selectable) { - /* Read reference monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), - &status); - if (rc) - return rc; - - /* If the monitor indicates errors report the reference - * as disconnected - */ - if (status == ZL_REF_MON_STATUS_OK) { - *state = DPLL_PIN_STATE_SELECTABLE; - return 0; - } + zl3073x_dev_ref_is_status_ok(zldev, ref) && pin->selectable) { + *state = DPLL_PIN_STATE_SELECTABLE; + return 0; } /* Otherwise report the pin as disconnected */ @@ -951,21 +773,19 @@ zl3073x_dpll_output_pin_esync_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - struct device *dev = zldev->dev; - u32 esync_period, esync_width; - u8 clock_type, synth; - u8 out, output_mode; - u32 output_div; + const struct zl3073x_synth *synth; + const struct zl3073x_out *out; + u8 clock_type, out_id; u32 synth_freq; - int rc; - out = zl3073x_output_pin_out_get(pin->id); + out_id = zl3073x_output_pin_out_get(pin->id); + out = zl3073x_out_state_get(zldev, out_id); /* If N-division is enabled, esync is not supported. The register used * for N-division is also used for the esync divider so both cannot * be used. */ - switch (zl3073x_out_signal_format_get(zldev, out)) { + switch (zl3073x_out_signal_format_get(out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV: case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV_INV: return -EOPNOTSUPP; @@ -973,38 +793,11 @@ zl3073x_dpll_output_pin_esync_get(const struct dpll_pin *dpll_pin, break; } - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Read output mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &output_mode); - if (rc) - return rc; - - /* Read output divisor */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &output_div); - if (rc) - return rc; - - /* Check output divisor for zero */ - if (!output_div) { - dev_err(dev, "Zero divisor for OUTPUT%u got from device\n", - out); - return -EINVAL; - } - - /* Get synth attached to output pin */ - synth = zl3073x_out_synth_get(zldev, out); + /* Get attached synth frequency */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(out)); + synth_freq = zl3073x_synth_freq_get(synth); - /* Get synth frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - clock_type = FIELD_GET(ZL_OUTPUT_MODE_CLOCK_TYPE, output_mode); + clock_type = FIELD_GET(ZL_OUTPUT_MODE_CLOCK_TYPE, out->mode); if (clock_type != ZL_OUTPUT_MODE_CLOCK_TYPE_ESYNC) { /* No need to read esync data if it is not enabled */ esync->freq = 0; @@ -1013,38 +806,21 @@ zl3073x_dpll_output_pin_esync_get(const struct dpll_pin *dpll_pin, goto finish; } - /* Read esync period */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, &esync_period); - if (rc) - return rc; - - /* Check esync divisor for zero */ - if (!esync_period) { - dev_err(dev, "Zero esync divisor for OUTPUT%u got from device\n", - out); - return -EINVAL; - } - - /* Get esync pulse width in units of half synth cycles */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, &esync_width); - if (rc) - return rc; - /* Compute esync frequency */ - esync->freq = synth_freq / output_div / esync_period; + esync->freq = synth_freq / out->div / out->esync_n_period; /* By comparing the esync_pulse_width to the half of the pulse width * the esync pulse percentage can be determined. * Note that half pulse width is in units of half synth cycles, which * is why it reduces down to be output_div. */ - esync->pulse = (50 * esync_width) / output_div; + esync->pulse = (50 * out->esync_n_width) / out->div; finish: /* Set supported esync ranges if the pin supports esync control and * if the output frequency is > 1 Hz. */ - if (pin->esync_control && (synth_freq / output_div) > 1) { + if (pin->esync_control && (synth_freq / out->div) > 1) { esync->range = esync_freq_ranges; esync->range_num = ARRAY_SIZE(esync_freq_ranges); } else { @@ -1062,21 +838,22 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, void *dpll_priv, u64 freq, struct netlink_ext_ack *extack) { - u32 esync_period, esync_width, output_div; struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 clock_type, out, output_mode, synth; + const struct zl3073x_synth *synth; + struct zl3073x_out out; + u8 clock_type, out_id; u32 synth_freq; - int rc; - out = zl3073x_output_pin_out_get(pin->id); + out_id = zl3073x_output_pin_out_get(pin->id); + out = *zl3073x_out_state_get(zldev, out_id); /* If N-division is enabled, esync is not supported. The register used * for N-division is also used for the esync divider so both cannot * be used. */ - switch (zl3073x_out_signal_format_get(zldev, out)) { + switch (zl3073x_out_signal_format_get(&out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV: case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV_INV: return -EOPNOTSUPP; @@ -1084,19 +861,6 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, break; } - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Read output mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &output_mode); - if (rc) - return rc; - /* Select clock type */ if (freq) clock_type = ZL_OUTPUT_MODE_CLOCK_TYPE_ESYNC; @@ -1104,38 +868,19 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, clock_type = ZL_OUTPUT_MODE_CLOCK_TYPE_NORMAL; /* Update clock type in output mode */ - output_mode &= ~ZL_OUTPUT_MODE_CLOCK_TYPE; - output_mode |= FIELD_PREP(ZL_OUTPUT_MODE_CLOCK_TYPE, clock_type); - rc = zl3073x_write_u8(zldev, ZL_REG_OUTPUT_MODE, output_mode); - if (rc) - return rc; + out.mode &= ~ZL_OUTPUT_MODE_CLOCK_TYPE; + out.mode |= FIELD_PREP(ZL_OUTPUT_MODE_CLOCK_TYPE, clock_type); /* If esync is being disabled just write mailbox and finish */ if (!freq) goto write_mailbox; - /* Get synth attached to output pin */ - synth = zl3073x_out_synth_get(zldev, out); - - /* Get synth frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &output_div); - if (rc) - return rc; - - /* Check output divisor for zero */ - if (!output_div) { - dev_err(zldev->dev, - "Zero divisor for OUTPUT%u got from device\n", out); - return -EINVAL; - } + /* Get attached synth frequency */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(&out)); + synth_freq = zl3073x_synth_freq_get(synth); /* Compute and update esync period */ - esync_period = synth_freq / (u32)freq / output_div; - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, esync_period); - if (rc) - return rc; + out.esync_n_period = synth_freq / (u32)freq / out.div; /* Half of the period in units of 1/2 synth cycle can be represented by * the output_div. To get the supported esync pulse width of 25% of the @@ -1143,15 +888,11 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, * assumes that output_div is even, otherwise some resolution will be * lost. */ - esync_width = output_div / 2; - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, esync_width); - if (rc) - return rc; + out.esync_n_width = out.div / 2; write_mailbox: /* Commit output configuration */ - return zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } static int @@ -1164,83 +905,46 @@ zl3073x_dpll_output_pin_frequency_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - struct device *dev = zldev->dev; - u8 out, signal_format, synth; - u32 output_div, synth_freq; - int rc; - - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &output_div); - if (rc) - return rc; + const struct zl3073x_synth *synth; + const struct zl3073x_out *out; + u32 synth_freq; + u8 out_id; - /* Check output divisor for zero */ - if (!output_div) { - dev_err(dev, "Zero divisor for output %u got from device\n", - out); - return -EINVAL; - } + out_id = zl3073x_output_pin_out_get(pin->id); + out = zl3073x_out_state_get(zldev, out_id); - /* Read used signal format for the given output */ - signal_format = zl3073x_out_signal_format_get(zldev, out); + /* Get attached synth frequency */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(out)); + synth_freq = zl3073x_synth_freq_get(synth); - switch (signal_format) { + switch (zl3073x_out_signal_format_get(out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV: case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV_INV: /* In case of divided format we have to distiguish between * given output pin type. + * + * For P-pin the resulting frequency is computed as simple + * division of synth frequency and output divisor. + * + * For N-pin we have to divide additionally by divisor stored + * in esync_n_period output mailbox register that is used as + * N-pin divisor for these modes. */ - if (zl3073x_dpll_is_p_pin(pin)) { - /* For P-pin the resulting frequency is computed as - * simple division of synth frequency and output - * divisor. - */ - *frequency = synth_freq / output_div; - } else { - /* For N-pin we have to divide additionally by - * divisor stored in esync_period output mailbox - * register that is used as N-pin divisor for these - * modes. - */ - u32 ndiv; - - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, - &ndiv); - if (rc) - return rc; + *frequency = synth_freq / out->div; - /* Check N-pin divisor for zero */ - if (!ndiv) { - dev_err(dev, - "Zero N-pin divisor for output %u got from device\n", - out); - return -EINVAL; - } + if (!zl3073x_dpll_is_p_pin(pin)) + *frequency = (u32)*frequency / out->esync_n_period; - /* Compute final divisor for N-pin */ - *frequency = synth_freq / output_div / ndiv; - } break; default: /* In other modes the resulting frequency is computed as * division of synth frequency and output divisor. */ - *frequency = synth_freq / output_div; + *frequency = synth_freq / out->div; break; } - return rc; + return 0; } static int @@ -1253,28 +957,21 @@ zl3073x_dpll_output_pin_frequency_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - struct device *dev = zldev->dev; - u32 output_n_freq, output_p_freq; - u8 out, signal_format, synth; - u32 cur_div, new_div, ndiv; - u32 synth_freq; - int rc; + const struct zl3073x_synth *synth; + u8 out_id, signal_format; + u32 new_div, synth_freq; + struct zl3073x_out out; - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); - synth_freq = zl3073x_synth_freq_get(zldev, synth); + out_id = zl3073x_output_pin_out_get(pin->id); + out = *zl3073x_out_state_get(zldev, out_id); + + /* Get attached synth frequency and compute new divisor */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(&out)); + synth_freq = zl3073x_synth_freq_get(synth); new_div = synth_freq / (u32)frequency; /* Get used signal format for the given output */ - signal_format = zl3073x_out_signal_format_get(zldev, out); - - guard(mutex)(&zldev->multiop_lock); - - /* Load output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; + signal_format = zl3073x_out_signal_format_get(&out); /* Check signal format */ if (signal_format != ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV && @@ -1282,99 +979,50 @@ zl3073x_dpll_output_pin_frequency_set(const struct dpll_pin *dpll_pin, /* For non N-divided signal formats the frequency is computed * as division of synth frequency and output divisor. */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_DIV, new_div); - if (rc) - return rc; + out.div = new_div; /* For 50/50 duty cycle the divisor is equal to width */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_WIDTH, new_div); - if (rc) - return rc; + out.width = new_div; /* Commit output configuration */ - return zl3073x_mb_op(zldev, - ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } - /* For N-divided signal format get current divisor */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &cur_div); - if (rc) - return rc; - - /* Check output divisor for zero */ - if (!cur_div) { - dev_err(dev, "Zero divisor for output %u got from device\n", - out); - return -EINVAL; - } - - /* Get N-pin divisor (shares the same register with esync */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, &ndiv); - if (rc) - return rc; - - /* Check N-pin divisor for zero */ - if (!ndiv) { - dev_err(dev, - "Zero N-pin divisor for output %u got from device\n", - out); - return -EINVAL; - } - - /* Compute current output frequency for P-pin */ - output_p_freq = synth_freq / cur_div; - - /* Compute current N-pin frequency */ - output_n_freq = output_p_freq / ndiv; - if (zl3073x_dpll_is_p_pin(pin)) { /* We are going to change output frequency for P-pin but * if the requested frequency is less than current N-pin * frequency then indicate a failure as we are not able * to compute N-pin divisor to keep its frequency unchanged. + * + * Update divisor for N-pin to keep N-pin frequency. */ - if (frequency <= output_n_freq) + out.esync_n_period = (out.esync_n_period * out.div) / new_div; + if (!out.esync_n_period) return -EINVAL; /* Update the output divisor */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_DIV, new_div); - if (rc) - return rc; + out.div = new_div; /* For 50/50 duty cycle the divisor is equal to width */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_WIDTH, new_div); - if (rc) - return rc; - - /* Compute new divisor for N-pin */ - ndiv = (u32)frequency / output_n_freq; + out.width = out.div; } else { /* We are going to change frequency of N-pin but if * the requested freq is greater or equal than freq of P-pin * in the output pair we cannot compute divisor for the N-pin. * In this case indicate a failure. + * + * Update divisor for N-pin */ - if (output_p_freq <= frequency) + out.esync_n_period = div64_u64(synth_freq, frequency * out.div); + if (!out.esync_n_period) return -EINVAL; - - /* Compute new divisor for N-pin */ - ndiv = output_p_freq / (u32)frequency; } - /* Update divisor for the N-pin */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, ndiv); - if (rc) - return rc; - /* For 50/50 duty cycle the divisor is equal to width */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, ndiv); - if (rc) - return rc; + out.esync_n_width = out.esync_n_period; /* Commit output configuration */ - return zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } static int @@ -1388,42 +1036,16 @@ zl3073x_dpll_output_pin_phase_adjust_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u32 synth_freq; - s32 phase_comp; - u8 out, synth; - int rc; + const struct zl3073x_out *out; + u8 out_id; - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); - synth_freq = zl3073x_synth_freq_get(zldev, synth); + out_id = zl3073x_output_pin_out_get(pin->id); + out = zl3073x_out_state_get(zldev, out_id); - /* Check synth freq for zero */ - if (!synth_freq) { - dev_err(zldev->dev, "Got zero synth frequency for output %u\n", - out); - return -EINVAL; - } - - guard(mutex)(&zldev->multiop_lock); + /* The value in the register is expressed in half synth clock cycles. */ + *phase_adjust = out->phase_comp * pin->phase_gran; - /* Read output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Read current output phase compensation */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, &phase_comp); - if (rc) - return rc; - - /* Value in register is expressed in half synth clock cycles */ - phase_comp *= (int)div_u64(PSEC_PER_SEC, 2 * synth_freq); - - /* Reverse two's complement negation applied during 'set' */ - *phase_adjust = -phase_comp; - - return rc; + return 0; } static int @@ -1437,52 +1059,17 @@ zl3073x_dpll_output_pin_phase_adjust_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - int half_synth_cycle; - u32 synth_freq; - u8 out, synth; - int rc; + struct zl3073x_out out; + u8 out_id; - /* Get attached synth */ - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); + out_id = zl3073x_output_pin_out_get(pin->id); + out = *zl3073x_out_state_get(zldev, out_id); - /* Get synth's frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - /* Value in register is expressed in half synth clock cycles so - * the given phase adjustment a multiple of half synth clock. - */ - half_synth_cycle = (int)div_u64(PSEC_PER_SEC, 2 * synth_freq); - - if ((phase_adjust % half_synth_cycle) != 0) { - NL_SET_ERR_MSG_FMT(extack, - "Phase adjustment value has to be multiple of %d", - half_synth_cycle); - return -EINVAL; - } - phase_adjust /= half_synth_cycle; - - /* The value in the register is stored as two's complement negation - * of requested value. - */ - phase_adjust = -phase_adjust; - - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Write the requested value into the compensation register */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, phase_adjust); - if (rc) - return rc; + /* The value in the register is expressed in half synth clock cycles. */ + out.phase_comp = phase_adjust / pin->phase_gran; /* Update output configuration from mailbox */ - return zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } static int @@ -1758,9 +1345,10 @@ zl3073x_dpll_pin_register(struct zl3073x_dpll_pin *pin, u32 index) if (IS_ERR(props)) return PTR_ERR(props); - /* Save package label & esync capability */ + /* Save package label, esync capability and phase adjust granularity */ strscpy(pin->label, props->package_label); pin->esync_control = props->esync_control; + pin->phase_gran = props->dpll_props.phase_gran; if (zl3073x_dpll_is_input_pin(pin)) { rc = zl3073x_dpll_ref_prio_get(pin, &pin->prio); @@ -1878,33 +1466,32 @@ zl3073x_dpll_pin_is_registrable(struct zl3073x_dpll *zldpll, const char *name; if (dir == DPLL_PIN_DIRECTION_INPUT) { - u8 ref = zl3073x_input_pin_ref_get(index); - - name = "REF"; + u8 ref_id = zl3073x_input_pin_ref_get(index); + const struct zl3073x_ref *ref; /* Skip the pin if the DPLL is running in NCO mode */ if (zldpll->refsel_mode == ZL_DPLL_MODE_REFSEL_MODE_NCO) return false; - is_diff = zl3073x_ref_is_diff(zldev, ref); - is_enabled = zl3073x_ref_is_enabled(zldev, ref); + name = "REF"; + ref = zl3073x_ref_state_get(zldev, ref_id); + is_diff = zl3073x_ref_is_diff(ref); + is_enabled = zl3073x_ref_is_enabled(ref); } else { /* Output P&N pair shares single HW output */ u8 out = zl3073x_output_pin_out_get(index); - name = "OUT"; - /* Skip the pin if it is connected to different DPLL channel */ - if (zl3073x_out_dpll_get(zldev, out) != zldpll->id) { + if (zl3073x_dev_out_dpll_get(zldev, out) != zldpll->id) { dev_dbg(zldev->dev, - "%s%u is driven by different DPLL\n", name, - out); + "OUT%u is driven by different DPLL\n", out); return false; } - is_diff = zl3073x_out_is_diff(zldev, out); - is_enabled = zl3073x_output_pin_is_enabled(zldev, index); + name = "OUT"; + is_diff = zl3073x_dev_out_is_diff(zldev, out); + is_enabled = zl3073x_dev_output_pin_is_enabled(zldev, index); } /* Skip N-pin if the corresponding input/output is differential */ @@ -2061,42 +1648,26 @@ zl3073x_dpll_pin_phase_offset_check(struct zl3073x_dpll_pin *pin) struct zl3073x_dev *zldev = zldpll->dev; unsigned int reg; s64 phase_offset; - u8 ref; + u8 ref_id; int rc; - ref = zl3073x_input_pin_ref_get(pin->id); + /* No phase offset if the ref monitor reports signal errors */ + ref_id = zl3073x_input_pin_ref_get(pin->id); + if (!zl3073x_dev_ref_is_status_ok(zldev, ref_id)) + return false; /* Select register to read phase offset value depending on pin and * phase monitor state: * 1) For connected pin use dpll_phase_err_data register * 2) For other pins use appropriate ref_phase register if the phase - * monitor feature is enabled and reference monitor does not - * report signal errors for given input pin + * monitor feature is enabled. */ - if (pin->pin_state == DPLL_PIN_STATE_CONNECTED) { + if (pin->pin_state == DPLL_PIN_STATE_CONNECTED) reg = ZL_REG_DPLL_PHASE_ERR_DATA(zldpll->id); - } else if (zldpll->phase_monitor) { - u8 status; - - /* Get reference monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), - &status); - if (rc) { - dev_err(zldev->dev, - "Failed to read %s refmon status: %pe\n", - pin->label, ERR_PTR(rc)); - - return false; - } - - if (status != ZL_REF_MON_STATUS_OK) - return false; - - reg = ZL_REG_REF_PHASE(ref); - } else { - /* The pin is not connected or phase monitor disabled */ + else if (zldpll->phase_monitor) + reg = ZL_REG_REF_PHASE(ref_id); + else return false; - } /* Read measured phase offset value */ rc = zl3073x_read_u48(zldev, reg, &phase_offset); @@ -2135,32 +1706,22 @@ zl3073x_dpll_pin_ffo_check(struct zl3073x_dpll_pin *pin) { struct zl3073x_dpll *zldpll = pin->dpll; struct zl3073x_dev *zldev = zldpll->dev; - u8 ref, status; - s64 ffo; - int rc; + const struct zl3073x_ref *ref; + u8 ref_id; /* Get reference monitor status */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), &status); - if (rc) { - dev_err(zldev->dev, "Failed to read %s refmon status: %pe\n", - pin->label, ERR_PTR(rc)); - - return false; - } + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); /* Do not report ffo changes if the reference monitor report errors */ - if (status != ZL_REF_MON_STATUS_OK) + if (!zl3073x_ref_is_status_ok(ref)) return false; - /* Get the latest measured ref's ffo */ - ffo = zl3073x_ref_ffo_get(zldev, ref); - /* Compare with previous value */ - if (pin->freq_offset != ffo) { + if (pin->freq_offset != ref->ffo) { dev_dbg(zldev->dev, "%s freq offset changed: %lld -> %lld\n", - pin->label, pin->freq_offset, ffo); - pin->freq_offset = ffo; + pin->label, pin->freq_offset, ref->ffo); + pin->freq_offset = ref->ffo; return true; } diff --git a/drivers/dpll/zl3073x/out.c b/drivers/dpll/zl3073x/out.c new file mode 100644 index 0000000000000..86829a0c1c022 --- /dev/null +++ b/drivers/dpll/zl3073x/out.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "out.h" + +/** + * zl3073x_out_state_fetch - fetch output state from hardware + * @zldev: pointer to zl3073x_dev structure + * @index: output index to fetch state for + * + * Function fetches state of the given output from hardware and stores it + * for later use. + * + * Return: 0 on success, <0 on error + */ +int zl3073x_out_state_fetch(struct zl3073x_dev *zldev, u8 index) +{ + struct zl3073x_out *out = &zldev->out[index]; + int rc; + + /* Read output configuration */ + rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_CTRL(index), &out->ctrl); + if (rc) + return rc; + + dev_dbg(zldev->dev, "OUT%u is %s and connected to SYNTH%u\n", index, + str_enabled_disabled(zl3073x_out_is_enabled(out)), + zl3073x_out_synth_get(out)); + + guard(mutex)(&zldev->multiop_lock); + + /* Read output configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, + ZL_REG_OUTPUT_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Read output mode */ + rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &out->mode); + if (rc) + return rc; + + dev_dbg(zldev->dev, "OUT%u has signal format 0x%02x\n", index, + zl3073x_out_signal_format_get(out)); + + /* Read output divisor */ + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &out->div); + if (rc) + return rc; + + if (!out->div) { + dev_err(zldev->dev, "Zero divisor for OUT%u got from device\n", + index); + return -EINVAL; + } + + dev_dbg(zldev->dev, "OUT%u divisor: %u\n", index, out->div); + + /* Read output width */ + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_WIDTH, &out->width); + if (rc) + return rc; + + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, + &out->esync_n_period); + if (rc) + return rc; + + if (!out->esync_n_period) { + dev_err(zldev->dev, + "Zero esync divisor for OUT%u got from device\n", + index); + return -EINVAL; + } + + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, + &out->esync_n_width); + if (rc) + return rc; + + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, + &out->phase_comp); + if (rc) + return rc; + + return rc; +} + +/** + * zl3073x_out_state_get - get current output state + * @zldev: pointer to zl3073x_dev structure + * @index: output index to get state for + * + * Return: pointer to given output state + */ +const struct zl3073x_out *zl3073x_out_state_get(struct zl3073x_dev *zldev, + u8 index) +{ + return &zldev->out[index]; +} + +int zl3073x_out_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_out *out) +{ + struct zl3073x_out *dout = &zldev->out[index]; + int rc; + + guard(mutex)(&zldev->multiop_lock); + + /* Read output configuration into mailbox */ + rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, + ZL_REG_OUTPUT_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Update mailbox with changed values */ + if (dout->div != out->div) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_DIV, out->div); + if (!rc && dout->width != out->width) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_WIDTH, out->width); + if (!rc && dout->esync_n_period != out->esync_n_period) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, + out->esync_n_period); + if (!rc && dout->esync_n_width != out->esync_n_width) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, + out->esync_n_width); + if (!rc && dout->mode != out->mode) + rc = zl3073x_write_u8(zldev, ZL_REG_OUTPUT_MODE, out->mode); + if (!rc && dout->phase_comp != out->phase_comp) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, + out->phase_comp); + if (rc) + return rc; + + /* Commit output configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, + ZL_REG_OUTPUT_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* After successful commit store new state */ + dout->div = out->div; + dout->width = out->width; + dout->esync_n_period = out->esync_n_period; + dout->esync_n_width = out->esync_n_width; + dout->mode = out->mode; + dout->phase_comp = out->phase_comp; + + return 0; +} diff --git a/drivers/dpll/zl3073x/out.h b/drivers/dpll/zl3073x/out.h new file mode 100644 index 0000000000000..e8ea7a0e0f071 --- /dev/null +++ b/drivers/dpll/zl3073x/out.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ZL3073X_OUT_H +#define _ZL3073X_OUT_H + +#include +#include + +#include "regs.h" + +struct zl3073x_dev; + +/** + * struct zl3073x_out - output state + * @div: output divisor + * @width: output pulse width + * @esync_n_period: embedded sync or n-pin period (for n-div formats) + * @esync_n_width: embedded sync or n-pin pulse width + * @phase_comp: phase compensation + * @ctrl: output control + * @mode: output mode + */ +struct zl3073x_out { + u32 div; + u32 width; + u32 esync_n_period; + u32 esync_n_width; + s32 phase_comp; + u8 ctrl; + u8 mode; +}; + +int zl3073x_out_state_fetch(struct zl3073x_dev *zldev, u8 index); +const struct zl3073x_out *zl3073x_out_state_get(struct zl3073x_dev *zldev, + u8 index); + +int zl3073x_out_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_out *out); + +/** + * zl3073x_out_signal_format_get - get output signal format + * @out: pointer to out state + * + * Return: signal format of given output + */ +static inline u8 zl3073x_out_signal_format_get(const struct zl3073x_out *out) +{ + return FIELD_GET(ZL_OUTPUT_MODE_SIGNAL_FORMAT, out->mode); +} + +/** + * zl3073x_out_is_diff - check if the given output is differential + * @out: pointer to out state + * + * Return: true if output is differential, false if output is single-ended + */ +static inline bool zl3073x_out_is_diff(const struct zl3073x_out *out) +{ + switch (zl3073x_out_signal_format_get(out)) { + case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LVDS: + case ZL_OUTPUT_MODE_SIGNAL_FORMAT_DIFF: + case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LOWVCM: + return true; + default: + break; + } + + return false; +} + +/** + * zl3073x_out_is_enabled - check if the given output is enabled + * @out: pointer to out state + * + * Return: true if output is enabled, false if output is disabled + */ +static inline bool zl3073x_out_is_enabled(const struct zl3073x_out *out) +{ + return !!FIELD_GET(ZL_OUTPUT_CTRL_EN, out->ctrl); +} + +/** + * zl3073x_out_synth_get - get synth connected to given output + * @out: pointer to out state + * + * Return: index of synth connected to given output. + */ +static inline u8 zl3073x_out_synth_get(const struct zl3073x_out *out) +{ + return FIELD_GET(ZL_OUTPUT_CTRL_SYNTH_SEL, out->ctrl); +} + +#endif /* _ZL3073X_OUT_H */ diff --git a/drivers/dpll/zl3073x/prop.c b/drivers/dpll/zl3073x/prop.c index 4cf7e8aefcb37..4ed153087570b 100644 --- a/drivers/dpll/zl3073x/prop.c +++ b/drivers/dpll/zl3073x/prop.c @@ -46,10 +46,10 @@ zl3073x_pin_check_freq(struct zl3073x_dev *zldev, enum dpll_pin_direction dir, /* Get output pin synthesizer */ out = zl3073x_output_pin_out_get(id); - synth = zl3073x_out_synth_get(zldev, out); + synth = zl3073x_dev_out_synth_get(zldev, out); /* Get synth frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); + synth_freq = zl3073x_dev_synth_freq_get(zldev, synth); /* Check the frequency divides synth frequency */ if (synth_freq % (u32)freq) @@ -93,13 +93,13 @@ zl3073x_prop_pin_package_label_set(struct zl3073x_dev *zldev, prefix = "REF"; ref = zl3073x_input_pin_ref_get(id); - is_diff = zl3073x_ref_is_diff(zldev, ref); + is_diff = zl3073x_dev_ref_is_diff(zldev, ref); } else { u8 out; prefix = "OUT"; out = zl3073x_output_pin_out_get(id); - is_diff = zl3073x_out_is_diff(zldev, out); + is_diff = zl3073x_dev_out_is_diff(zldev, out); } if (!is_diff) @@ -208,7 +208,18 @@ struct zl3073x_pin_props *zl3073x_pin_props_get(struct zl3073x_dev *zldev, DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE | DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE; } else { + u8 out, synth; + u32 f; + props->dpll_props.type = DPLL_PIN_TYPE_GNSS; + + /* The output pin phase adjustment granularity equals half of + * the synth frequency count. + */ + out = zl3073x_output_pin_out_get(index); + synth = zl3073x_dev_out_synth_get(zldev, out); + f = 2 * zl3073x_dev_synth_freq_get(zldev, synth); + props->dpll_props.phase_gran = f ? div_u64(PSEC_PER_SEC, f) : 1; } props->dpll_props.phase_range.min = S32_MIN; diff --git a/drivers/dpll/zl3073x/ref.c b/drivers/dpll/zl3073x/ref.c new file mode 100644 index 0000000000000..6b65e61039999 --- /dev/null +++ b/drivers/dpll/zl3073x/ref.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "ref.h" + +/** + * zl3073x_ref_freq_factorize - factorize given frequency + * @freq: input frequency + * @base: base frequency + * @mult: multiplier + * + * Checks if the given frequency can be factorized using one of the + * supported base frequencies. If so the base frequency and multiplier + * are stored into appropriate parameters if they are not NULL. + * + * Return: 0 on success, -EINVAL if the frequency cannot be factorized + */ +int +zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult) +{ + static const u16 base_freqs[] = { + 1, 2, 4, 5, 8, 10, 16, 20, 25, 32, 40, 50, 64, 80, 100, 125, + 128, 160, 200, 250, 256, 320, 400, 500, 625, 640, 800, 1000, + 1250, 1280, 1600, 2000, 2500, 3125, 3200, 4000, 5000, 6250, + 6400, 8000, 10000, 12500, 15625, 16000, 20000, 25000, 31250, + 32000, 40000, 50000, 62500, + }; + u32 div; + int i; + + for (i = 0; i < ARRAY_SIZE(base_freqs); i++) { + div = freq / base_freqs[i]; + + if (div <= U16_MAX && (freq % base_freqs[i]) == 0) { + if (base) + *base = base_freqs[i]; + if (mult) + *mult = div; + + return 0; + } + } + + return -EINVAL; +} + +/** + * zl3073x_ref_state_fetch - fetch input reference state from hardware + * @zldev: pointer to zl3073x_dev structure + * @index: input reference index to fetch state for + * + * Function fetches state for the given input reference from hardware and + * stores it for later use. + * + * Return: 0 on success, <0 on error + */ +int zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index) +{ + struct zl3073x_ref *ref = &zldev->ref[index]; + int rc; + + /* For differential type inputs the N-pin reference shares + * part of the configuration with the P-pin counterpart. + */ + if (zl3073x_is_n_pin(index) && zl3073x_ref_is_diff(ref - 1)) { + struct zl3073x_ref *p_ref = ref - 1; /* P-pin counterpart*/ + + /* Copy the shared items from the P-pin */ + ref->config = p_ref->config; + ref->esync_n_div = p_ref->esync_n_div; + ref->freq_base = p_ref->freq_base; + ref->freq_mult = p_ref->freq_mult; + ref->freq_ratio_m = p_ref->freq_ratio_m; + ref->freq_ratio_n = p_ref->freq_ratio_n; + ref->phase_comp = p_ref->phase_comp; + ref->sync_ctrl = p_ref->sync_ctrl; + + return 0; /* Finish - no non-shared items for now */ + } + + guard(mutex)(&zldev->multiop_lock); + + /* Read reference configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, + ZL_REG_REF_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Read ref_config register */ + rc = zl3073x_read_u8(zldev, ZL_REG_REF_CONFIG, &ref->config); + if (rc) + return rc; + + /* Read frequency related registers */ + rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_BASE, &ref->freq_base); + if (rc) + return rc; + rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_MULT, &ref->freq_mult); + if (rc) + return rc; + rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_M, &ref->freq_ratio_m); + if (rc) + return rc; + rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_N, &ref->freq_ratio_n); + if (rc) + return rc; + + /* Read eSync and N-div rated registers */ + rc = zl3073x_read_u32(zldev, ZL_REG_REF_ESYNC_DIV, &ref->esync_n_div); + if (rc) + return rc; + rc = zl3073x_read_u8(zldev, ZL_REG_REF_SYNC_CTRL, &ref->sync_ctrl); + if (rc) + return rc; + + /* Read phase compensation register */ + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) { + u32 val; + + rc = zl3073x_read_u32(zldev, ZL_REG_REF_PHASE_OFFSET_COMP_32, + &val); + ref->phase_comp = val; + } else { + rc = zl3073x_read_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, + &ref->phase_comp); + } + if (rc) + return rc; + + dev_dbg(zldev->dev, "REF%u is %s and configured as %s\n", index, + str_enabled_disabled(zl3073x_ref_is_enabled(ref)), + zl3073x_ref_is_diff(ref) ? "differential" : "single-ended"); + + return rc; +} + +/** + * zl3073x_ref_state_get - get current input reference state + * @zldev: pointer to zl3073x_dev structure + * @index: input reference index to get state for + * + * Return: pointer to given input reference state + */ +const struct zl3073x_ref * +zl3073x_ref_state_get(struct zl3073x_dev *zldev, u8 index) +{ + return &zldev->ref[index]; +} + +int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_ref *ref) +{ + struct zl3073x_ref *dref = &zldev->ref[index]; + int rc; + + guard(mutex)(&zldev->multiop_lock); + + /* Read reference configuration into mailbox */ + rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, + ZL_REG_REF_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Update mailbox with changed values */ + if (dref->freq_base != ref->freq_base) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_BASE, + ref->freq_base); + if (!rc && dref->freq_mult != ref->freq_mult) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_MULT, + ref->freq_mult); + if (!rc && dref->freq_ratio_m != ref->freq_ratio_m) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_M, + ref->freq_ratio_m); + if (!rc && dref->freq_ratio_n != ref->freq_ratio_n) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_N, + ref->freq_ratio_n); + if (!rc && dref->esync_n_div != ref->esync_n_div) + rc = zl3073x_write_u32(zldev, ZL_REG_REF_ESYNC_DIV, + ref->esync_n_div); + if (!rc && dref->sync_ctrl != ref->sync_ctrl) + rc = zl3073x_write_u8(zldev, ZL_REG_REF_SYNC_CTRL, + ref->sync_ctrl); + if (!rc && dref->phase_comp != ref->phase_comp) { + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) + rc = zl3073x_write_u32(zldev, + ZL_REG_REF_PHASE_OFFSET_COMP_32, + ref->phase_comp); + else + rc = zl3073x_write_u48(zldev, + ZL_REG_REF_PHASE_OFFSET_COMP, + ref->phase_comp); + } + if (rc) + return rc; + + /* Commit reference configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, + ZL_REG_REF_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* After successful commit store new state */ + dref->freq_base = ref->freq_base; + dref->freq_mult = ref->freq_mult; + dref->freq_ratio_m = ref->freq_ratio_m; + dref->freq_ratio_n = ref->freq_ratio_n; + dref->esync_n_div = ref->esync_n_div; + dref->sync_ctrl = ref->sync_ctrl; + dref->phase_comp = ref->phase_comp; + + return 0; +} diff --git a/drivers/dpll/zl3073x/ref.h b/drivers/dpll/zl3073x/ref.h new file mode 100644 index 0000000000000..efc7f59cd9f9c --- /dev/null +++ b/drivers/dpll/zl3073x/ref.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ZL3073X_REF_H +#define _ZL3073X_REF_H + +#include +#include +#include + +#include "regs.h" + +struct zl3073x_dev; + +/** + * struct zl3073x_ref - input reference state + * @ffo: current fractional frequency offset + * @phase_comp: phase compensation + * @esync_n_div: divisor for embedded sync or n-divided signal formats + * @freq_base: frequency base + * @freq_mult: frequnecy multiplier + * @freq_ratio_m: FEC mode multiplier + * @freq_ratio_n: FEC mode divisor + * @config: reference config + * @sync_ctrl: reference sync control + * @mon_status: reference monitor status + */ +struct zl3073x_ref { + s64 ffo; + u64 phase_comp; + u32 esync_n_div; + u16 freq_base; + u16 freq_mult; + u16 freq_ratio_m; + u16 freq_ratio_n; + u8 config; + u8 sync_ctrl; + u8 mon_status; +}; + +int zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index); + +const struct zl3073x_ref *zl3073x_ref_state_get(struct zl3073x_dev *zldev, + u8 index); + +int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_ref *ref); + +int zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult); + +/** + * zl3073x_ref_ffo_get - get current fractional frequency offset + * @ref: pointer to ref state + * + * Return: the latest measured fractional frequency offset + */ +static inline s64 +zl3073x_ref_ffo_get(const struct zl3073x_ref *ref) +{ + return ref->ffo; +} + +/** + * zl3073x_ref_freq_get - get given input reference frequency + * @ref: pointer to ref state + * + * Return: frequency of the given input reference + */ +static inline u32 +zl3073x_ref_freq_get(const struct zl3073x_ref *ref) +{ + return mul_u64_u32_div(ref->freq_base * ref->freq_mult, + ref->freq_ratio_m, ref->freq_ratio_n); +} + +/** + * zl3073x_ref_freq_set - set given input reference frequency + * @ref: pointer to ref state + * @freq: frequency to be set + * + * Return: 0 on success, <0 when frequency cannot be factorized + */ +static inline int +zl3073x_ref_freq_set(struct zl3073x_ref *ref, u32 freq) +{ + u16 base, mult; + int rc; + + rc = zl3073x_ref_freq_factorize(freq, &base, &mult); + if (rc) + return rc; + + ref->freq_base = base; + ref->freq_mult = mult; + + return 0; +} + +/** + * zl3073x_ref_is_diff - check if the given input reference is differential + * @ref: pointer to ref state + * + * Return: true if reference is differential, false if reference is single-ended + */ +static inline bool +zl3073x_ref_is_diff(const struct zl3073x_ref *ref) +{ + return !!FIELD_GET(ZL_REF_CONFIG_DIFF_EN, ref->config); +} + +/** + * zl3073x_ref_is_enabled - check if the given input reference is enabled + * @ref: pointer to ref state + * + * Return: true if input refernce is enabled, false otherwise + */ +static inline bool +zl3073x_ref_is_enabled(const struct zl3073x_ref *ref) +{ + return !!FIELD_GET(ZL_REF_CONFIG_ENABLE, ref->config); +} + +/** + * zl3073x_ref_is_status_ok - check the given input reference status + * @ref: pointer to ref state + * + * Return: true if the status is ok, false otherwise + */ +static inline bool +zl3073x_ref_is_status_ok(const struct zl3073x_ref *ref) +{ + return ref->mon_status == ZL_REF_MON_STATUS_OK; +} + +#endif /* _ZL3073X_REF_H */ diff --git a/drivers/dpll/zl3073x/regs.h b/drivers/dpll/zl3073x/regs.h index d837bee72b178..5573d7188406b 100644 --- a/drivers/dpll/zl3073x/regs.h +++ b/drivers/dpll/zl3073x/regs.h @@ -194,6 +194,7 @@ #define ZL_REF_CONFIG_DIFF_EN BIT(2) #define ZL_REG_REF_PHASE_OFFSET_COMP ZL_REG(10, 0x28, 6) +#define ZL_REG_REF_PHASE_OFFSET_COMP_32 ZL_REG(10, 0x28, 4) #define ZL_REG_REF_SYNC_CTRL ZL_REG(10, 0x2e, 1) #define ZL_REF_SYNC_CTRL_MODE GENMASK(2, 0) diff --git a/drivers/dpll/zl3073x/synth.c b/drivers/dpll/zl3073x/synth.c new file mode 100644 index 0000000000000..da839572dab26 --- /dev/null +++ b/drivers/dpll/zl3073x/synth.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "synth.h" + +/** + * zl3073x_synth_state_fetch - fetch synth state from hardware + * @zldev: pointer to zl3073x_dev structure + * @index: synth index to fetch state for + * + * Function fetches state of the given synthesizer from the hardware and + * stores it for later use. + * + * Return: 0 on success, <0 on error + */ +int zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 index) +{ + struct zl3073x_synth *synth = &zldev->synth[index]; + int rc; + + /* Read synth control register */ + rc = zl3073x_read_u8(zldev, ZL_REG_SYNTH_CTRL(index), &synth->ctrl); + if (rc) + return rc; + + guard(mutex)(&zldev->multiop_lock); + + /* Read synth configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_SYNTH_MB_SEM, ZL_SYNTH_MB_SEM_RD, + ZL_REG_SYNTH_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* The output frequency is determined by the following formula: + * base * multiplier * numerator / denominator + * + * Read registers with these values + */ + rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_BASE, &synth->freq_base); + if (rc) + return rc; + + rc = zl3073x_read_u32(zldev, ZL_REG_SYNTH_FREQ_MULT, &synth->freq_mult); + if (rc) + return rc; + + rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_M, &synth->freq_m); + if (rc) + return rc; + + rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_N, &synth->freq_n); + if (rc) + return rc; + + /* Check denominator for zero to avoid div by 0 */ + if (!synth->freq_n) { + dev_err(zldev->dev, + "Zero divisor for SYNTH%u retrieved from device\n", + index); + return -EINVAL; + } + + dev_dbg(zldev->dev, "SYNTH%u frequency: %u Hz\n", index, + zl3073x_synth_freq_get(synth)); + + return rc; +} + +/** + * zl3073x_synth_state_get - get current synth state + * @zldev: pointer to zl3073x_dev structure + * @index: synth index to get state for + * + * Return: pointer to given synth state + */ +const struct zl3073x_synth *zl3073x_synth_state_get(struct zl3073x_dev *zldev, + u8 index) +{ + return &zldev->synth[index]; +} diff --git a/drivers/dpll/zl3073x/synth.h b/drivers/dpll/zl3073x/synth.h new file mode 100644 index 0000000000000..6c55eb8a888c2 --- /dev/null +++ b/drivers/dpll/zl3073x/synth.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ZL3073X_SYNTH_H +#define _ZL3073X_SYNTH_H + +#include +#include +#include + +#include "regs.h" + +struct zl3073x_dev; + +/** + * struct zl3073x_synth - synthesizer state + * @freq_mult: frequency multiplier + * @freq_base: frequency base + * @freq_m: frequency numerator + * @freq_n: frequency denominator + * @ctrl: synth control + */ +struct zl3073x_synth { + u32 freq_mult; + u16 freq_base; + u16 freq_m; + u16 freq_n; + u8 ctrl; +}; + +int zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 synth_id); + +const struct zl3073x_synth *zl3073x_synth_state_get(struct zl3073x_dev *zldev, + u8 synth_id); + +int zl3073x_synth_state_set(struct zl3073x_dev *zldev, u8 synth_id, + const struct zl3073x_synth *synth); + +/** + * zl3073x_synth_dpll_get - get DPLL ID the synth is driven by + * @synth: pointer to synth state + * + * Return: ID of DPLL the given synthetizer is driven by + */ +static inline u8 zl3073x_synth_dpll_get(const struct zl3073x_synth *synth) +{ + return FIELD_GET(ZL_SYNTH_CTRL_DPLL_SEL, synth->ctrl); +} + +/** + * zl3073x_synth_freq_get - get synth current freq + * @synth: pointer to synth state + * + * Return: frequency of given synthetizer + */ +static inline u32 zl3073x_synth_freq_get(const struct zl3073x_synth *synth) +{ + return mul_u64_u32_div(synth->freq_base * synth->freq_m, + synth->freq_mult, synth->freq_n); +} + +/** + * zl3073x_synth_is_enabled - check if the given synth is enabled + * @synth: pointer to synth state + * + * Return: true if synth is enabled, false otherwise + */ +static inline bool zl3073x_synth_is_enabled(const struct zl3073x_synth *synth) +{ + return FIELD_GET(ZL_SYNTH_CTRL_EN, synth->ctrl); +} + +#endif /* _ZL3073X_SYNTH_H */ diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 0c5b94e64ea15..4edd2088c2db6 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -1563,8 +1563,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->sb_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB SBERR IRQ error\n"); @@ -1587,8 +1586,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->db_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n"); @@ -1970,8 +1968,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->sb_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, - ecc_name, altdev); + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No SBERR IRQ resource\n"); goto err_release_group1; @@ -1993,7 +1990,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index 0959320fe51c6..b510919e49c0e 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -370,13 +370,13 @@ struct mem_ctl_info *edac_mc_alloc(unsigned int mc_num, if (!mci->layers) goto error; + mci->dev.release = mci_release; + device_initialize(&mci->dev); + mci->pvt_info = kzalloc(sz_pvt, GFP_KERNEL); if (!mci->pvt_info) goto error; - mci->dev.release = mci_release; - device_initialize(&mci->dev); - /* setup index and various internal pointers */ mci->mc_idx = mc_num; mci->tot_dimms = tot_dimms; diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c index afccdebf5ac1e..6cade6d7ceff6 100644 --- a/drivers/edac/i3200_edac.c +++ b/drivers/edac/i3200_edac.c @@ -358,10 +358,11 @@ static int i3200_probe1(struct pci_dev *pdev, int dev_idx) layers[1].type = EDAC_MC_LAYER_CHANNEL; layers[1].size = nr_channels; layers[1].is_virt_csrow = false; - mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers, - sizeof(struct i3200_priv)); + + rc = -ENOMEM; + mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers, sizeof(struct i3200_priv)); if (!mci) - return -ENOMEM; + goto unmap; edac_dbg(3, "MC: init mci\n"); @@ -421,9 +422,9 @@ static int i3200_probe1(struct pci_dev *pdev, int dev_idx) return 0; fail: + edac_mc_free(mci); +unmap: iounmap(window); - if (mci) - edac_mc_free(mci); return rc; } diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c index 4a1bebc1ff14d..471b8540d18b0 100644 --- a/drivers/edac/i5000_edac.c +++ b/drivers/edac/i5000_edac.c @@ -1111,6 +1111,7 @@ static void calculate_dimm_size(struct i5000_pvt *pvt) n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c index b5cf25905b059..fb49a1d1df112 100644 --- a/drivers/edac/i5400_edac.c +++ b/drivers/edac/i5400_edac.c @@ -1026,13 +1026,13 @@ static void calculate_dimm_size(struct i5400_pvt *pvt) space -= n; } - space -= n; edac_dbg(2, "%s\n", mem_buffer); p = mem_buffer; space = PAGE_SIZE; n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index 2fc59f9eed691..463bbcd484c3c 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -246,6 +246,8 @@ static struct work_struct ecclog_work; /* Compute did IDs for Amston Lake with IBECC */ #define DID_ASL_SKU1 0x464a +#define DID_ASL_SKU2 0x4646 +#define DID_ASL_SKU3 0x4652 /* Compute die IDs for Raptor Lake-P with IBECC */ #define DID_RPL_P_SKU1 0xa706 @@ -274,6 +276,16 @@ static struct work_struct ecclog_work; #define DID_PTL_H_SKU1 0xb000 #define DID_PTL_H_SKU2 0xb001 #define DID_PTL_H_SKU3 0xb002 +#define DID_PTL_H_SKU4 0xb003 +#define DID_PTL_H_SKU5 0xb004 +#define DID_PTL_H_SKU6 0xb005 +#define DID_PTL_H_SKU7 0xb008 +#define DID_PTL_H_SKU8 0xb011 +#define DID_PTL_H_SKU9 0xb014 +#define DID_PTL_H_SKU10 0xb015 +#define DID_PTL_H_SKU11 0xb028 +#define DID_PTL_H_SKU12 0xb029 +#define DID_PTL_H_SKU13 0xb02a /* Compute die IDs for Wildcat Lake with IBECC */ #define DID_WCL_SKU1 0xfd00 @@ -618,6 +630,8 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_AZB_SKU1), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_ASL_SKU1), (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU2), (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU3), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), (kernel_ulong_t)&rpl_p_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), (kernel_ulong_t)&rpl_p_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), (kernel_ulong_t)&rpl_p_cfg }, @@ -636,6 +650,16 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_WCL_SKU1), (kernel_ulong_t)&wcl_cfg }, { }, }; diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c index 49ab5721aab25..292dda754c236 100644 --- a/drivers/edac/x38_edac.c +++ b/drivers/edac/x38_edac.c @@ -341,9 +341,12 @@ static int x38_probe1(struct pci_dev *pdev, int dev_idx) layers[1].type = EDAC_MC_LAYER_CHANNEL; layers[1].size = x38_channel_num; layers[1].is_virt_csrow = false; + + + rc = -ENOMEM; mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers, 0); if (!mci) - return -ENOMEM; + goto unmap; edac_dbg(3, "MC: init mci\n"); @@ -403,9 +406,9 @@ static int x38_probe1(struct pci_dev *pdev, int dev_idx) return 0; fail: + edac_mc_free(mci); +unmap: iounmap(window); - if (mci) - edac_mc_free(mci); return rc; } diff --git a/drivers/firewire/core-transaction.c b/drivers/firewire/core-transaction.c index c65f491c54d06..2e797430332c0 100644 --- a/drivers/firewire/core-transaction.c +++ b/drivers/firewire/core-transaction.c @@ -134,20 +134,14 @@ static void split_transaction_timeout_callback(struct timer_list *timer) } } -static void start_split_transaction_timeout(struct fw_transaction *t, - struct fw_card *card) +// card->transactions.lock should be acquired in advance for the linked list. +static void start_split_transaction_timeout(struct fw_transaction *t, unsigned int delta) { - unsigned long delta; - if (list_empty(&t->link) || WARN_ON(t->is_split_transaction)) return; t->is_split_transaction = true; - // NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for - // local destination never runs in any type of IRQ context. - scoped_guard(spinlock_irqsave, &card->split_timeout.lock) - delta = card->split_timeout.jiffies; mod_timer(&t->split_timeout_timer, jiffies + delta); } @@ -168,13 +162,20 @@ static void transmit_complete_callback(struct fw_packet *packet, break; case ACK_PENDING: { + unsigned int delta; + // NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->split_timeout.lock) { t->split_timeout_cycle = compute_split_timeout_timestamp(card, packet->timestamp) & 0xffff; + delta = card->split_timeout.jiffies; } - start_split_transaction_timeout(t, card); + + // NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for + // local destination never runs in any type of IRQ context. + scoped_guard(spinlock_irqsave, &card->transactions.lock) + start_split_transaction_timeout(t, delta); break; } case ACK_BUSY_X: diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c index 6d64467135395..e829454089550 100644 --- a/drivers/firewire/net.c +++ b/drivers/firewire/net.c @@ -257,9 +257,10 @@ static void fwnet_header_cache_update(struct hh_cache *hh, memcpy((u8 *)hh->hh_data + HH_DATA_OFF(FWNET_HLEN), haddr, net->addr_len); } -static int fwnet_header_parse(const struct sk_buff *skb, unsigned char *haddr) +static int fwnet_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr) { - memcpy(haddr, skb->dev->dev_addr, FWNET_ALEN); + memcpy(haddr, dev->dev_addr, FWNET_ALEN); return FWNET_ALEN; } diff --git a/drivers/firewire/nosy.c b/drivers/firewire/nosy.c index ea31ac7ac1ca9..e59053738a432 100644 --- a/drivers/firewire/nosy.c +++ b/drivers/firewire/nosy.c @@ -36,6 +36,8 @@ static char driver_name[] = KBUILD_MODNAME; +#define RCV_BUFFER_SIZE (16 * 1024) + /* this is the physical layout of a PCL, its size is 128 bytes */ struct pcl { __le32 next; @@ -517,16 +519,14 @@ remove_card(struct pci_dev *dev) lynx->rcv_start_pcl, lynx->rcv_start_pcl_bus); dma_free_coherent(&lynx->pci_device->dev, sizeof(struct pcl), lynx->rcv_pcl, lynx->rcv_pcl_bus); - dma_free_coherent(&lynx->pci_device->dev, PAGE_SIZE, lynx->rcv_buffer, - lynx->rcv_buffer_bus); + dma_free_coherent(&lynx->pci_device->dev, RCV_BUFFER_SIZE, + lynx->rcv_buffer, lynx->rcv_buffer_bus); iounmap(lynx->registers); pci_disable_device(dev); lynx_put(lynx); } -#define RCV_BUFFER_SIZE (16 * 1024) - static int add_card(struct pci_dev *dev, const struct pci_device_id *unused) { @@ -680,7 +680,7 @@ add_card(struct pci_dev *dev, const struct pci_device_id *unused) dma_free_coherent(&lynx->pci_device->dev, sizeof(struct pcl), lynx->rcv_pcl, lynx->rcv_pcl_bus); if (lynx->rcv_buffer) - dma_free_coherent(&lynx->pci_device->dev, PAGE_SIZE, + dma_free_coherent(&lynx->pci_device->dev, RCV_BUFFER_SIZE, lynx->rcv_buffer, lynx->rcv_buffer_bus); iounmap(lynx->registers); diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index c72ee47565856..f6ceae987acbc 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -205,12 +205,12 @@ static int ffa_rxtx_map(phys_addr_t tx_buf, phys_addr_t rx_buf, u32 pg_cnt) return 0; } -static int ffa_rxtx_unmap(u16 vm_id) +static int ffa_rxtx_unmap(void) { ffa_value_t ret; invoke_ffa_fn((ffa_value_t){ - .a0 = FFA_RXTX_UNMAP, .a1 = PACK_TARGET_INFO(vm_id, 0), + .a0 = FFA_RXTX_UNMAP, }, &ret); if (ret.a0 == FFA_ERROR) @@ -981,10 +981,27 @@ static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu) } } +/* + * Map logical ID index to the u16 index within the packed ID list. + * + * For native responses (FF-A width == kernel word size), IDs are + * tightly packed: idx -> idx. + * + * For 32-bit responses on a 64-bit kernel, each 64-bit register + * contributes 4 x u16 values but only the lower 2 are defined; the + * upper 2 are garbage. This mapping skips those upper halves: + * 0,1,2,3,4,5,... -> 0,1,4,5,8,9,... + */ +static int list_idx_to_u16_idx(int idx, bool is_native_resp) +{ + return is_native_resp ? idx : idx + 2 * (idx >> 1); +} + static void ffa_notification_info_get(void) { - int idx, list, max_ids, lists_cnt, ids_processed, ids_count[MAX_IDS_64]; - bool is_64b_resp; + int ids_processed, ids_count[MAX_IDS_64]; + int idx, list, max_ids, lists_cnt; + bool is_64b_resp, is_native_resp; ffa_value_t ret; u64 id_list; @@ -1001,6 +1018,7 @@ static void ffa_notification_info_get(void) } is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS); + is_native_resp = (ret.a0 == FFA_FN_NATIVE(SUCCESS)); ids_processed = 0; lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2); @@ -1017,12 +1035,16 @@ static void ffa_notification_info_get(void) /* Process IDs */ for (list = 0; list < lists_cnt; list++) { + int u16_idx; u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3; if (ids_processed >= max_ids - 1) break; - part_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + part_id = packed_id_list[u16_idx]; + ids_processed++; if (ids_count[list] == 1) { /* Global Notification */ __do_sched_recv_cb(part_id, 0, false); @@ -1034,7 +1056,10 @@ static void ffa_notification_info_get(void) if (ids_processed >= max_ids - 1) break; - vcpu_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + vcpu_id = packed_id_list[u16_idx]; + ids_processed++; __do_sched_recv_cb(part_id, vcpu_id, true); } @@ -2068,6 +2093,7 @@ static int __init ffa_init(void) pr_err("failed to setup partitions\n"); ffa_notifications_cleanup(); + ffa_rxtx_unmap(); free_pages: if (drv_info->tx_buffer) free_pages_exact(drv_info->tx_buffer, rxtx_bufsz); @@ -2082,7 +2108,7 @@ static void __exit ffa_exit(void) { ffa_notifications_cleanup(); ffa_partitions_cleanup(); - ffa_rxtx_unmap(drv_info->vm_id); + ffa_rxtx_unmap(); free_pages_exact(drv_info->tx_buffer, drv_info->rxtx_bufsz); free_pages_exact(drv_info->rx_buffer, drv_info->rxtx_bufsz); kfree(drv_info); diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c index dee9f238f6fdd..2047edbdc5f6b 100644 --- a/drivers/firmware/arm_scmi/notify.c +++ b/drivers/firmware/arm_scmi/notify.c @@ -1066,7 +1066,7 @@ static int scmi_register_event_handler(struct scmi_notify_instance *ni, * since at creation time we usually want to have all setup and ready before * events really start flowing. * - * Return: A properly refcounted handler on Success, NULL on Failure + * Return: A properly refcounted handler on Success, ERR_PTR on Failure */ static inline struct scmi_event_handler * __scmi_event_handler_get_ops(struct scmi_notify_instance *ni, @@ -1113,7 +1113,7 @@ __scmi_event_handler_get_ops(struct scmi_notify_instance *ni, } mutex_unlock(&ni->pending_mtx); - return hndl; + return hndl ?: ERR_PTR(-ENODEV); } static struct scmi_event_handler * diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c index 87c323de17b90..398642cc25d90 100644 --- a/drivers/firmware/arm_scpi.c +++ b/drivers/firmware/arm_scpi.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -940,13 +941,13 @@ static int scpi_probe(struct platform_device *pdev) int idx = scpi_drvinfo->num_chans; struct scpi_chan *pchan = scpi_drvinfo->channels + idx; struct mbox_client *cl = &pchan->cl; - struct device_node *shmem = of_parse_phandle(np, "shmem", idx); + struct device_node *shmem __free(device_node) = + of_parse_phandle(np, "shmem", idx); if (!of_match_node(shmem_of_match, shmem)) return -ENXIO; ret = of_address_to_resource(shmem, 0, &res); - of_node_put(shmem); if (ret) { dev_err(dev, "failed to get SCPI payload mem resource\n"); return ret; diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index f51047d8ea64f..7ca56777a1da5 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -9,6 +9,8 @@ * Cirrus Logic International Semiconductor Ltd. */ +#include +#include #include #include #include @@ -22,6 +24,41 @@ #include #include +#include "cs_dsp.h" + +/* + * When the KUnit test is running the error-case tests will cause a lot + * of messages. Rate-limit to prevent overflowing the kernel log buffer + * during KUnit test runs. + */ +#if IS_ENABLED(CONFIG_FW_CS_DSP_KUNIT_TEST) +bool cs_dsp_suppress_err_messages; +EXPORT_SYMBOL_IF_KUNIT(cs_dsp_suppress_err_messages); + +bool cs_dsp_suppress_warn_messages; +EXPORT_SYMBOL_IF_KUNIT(cs_dsp_suppress_warn_messages); + +bool cs_dsp_suppress_info_messages; +EXPORT_SYMBOL_IF_KUNIT(cs_dsp_suppress_info_messages); + +#define cs_dsp_err(_dsp, fmt, ...) \ + do { \ + if (!cs_dsp_suppress_err_messages) \ + dev_err_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \ + } while (false) +#define cs_dsp_warn(_dsp, fmt, ...) \ + do { \ + if (!cs_dsp_suppress_warn_messages) \ + dev_warn_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \ + } while (false) +#define cs_dsp_info(_dsp, fmt, ...) \ + do { \ + if (!cs_dsp_suppress_info_messages) \ + dev_info_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \ + } while (false) +#define cs_dsp_dbg(_dsp, fmt, ...) \ + dev_dbg_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__) +#else #define cs_dsp_err(_dsp, fmt, ...) \ dev_err(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__) #define cs_dsp_warn(_dsp, fmt, ...) \ @@ -30,6 +67,7 @@ dev_info(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__) #define cs_dsp_dbg(_dsp, fmt, ...) \ dev_dbg(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__) +#endif #define ADSP1_CONTROL_1 0x00 #define ADSP1_CONTROL_2 0x02 @@ -410,24 +448,30 @@ static void cs_dsp_debugfs_clear(struct cs_dsp *dsp) dsp->bin_file_name = NULL; } +static ssize_t cs_dsp_debugfs_string_read(struct cs_dsp *dsp, + char __user *user_buf, + size_t count, loff_t *ppos, + const char **pstr) +{ + const char *str; + + scoped_guard(mutex, &dsp->pwr_lock) { + str = *pstr; + if (!str) + return 0; + + return simple_read_from_buffer(user_buf, count, ppos, str, strlen(str)); + } +} + static ssize_t cs_dsp_debugfs_wmfw_read(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { struct cs_dsp *dsp = file->private_data; - ssize_t ret; - - mutex_lock(&dsp->pwr_lock); - - if (!dsp->wmfw_file_name || !dsp->booted) - ret = 0; - else - ret = simple_read_from_buffer(user_buf, count, ppos, - dsp->wmfw_file_name, - strlen(dsp->wmfw_file_name)); - mutex_unlock(&dsp->pwr_lock); - return ret; + return cs_dsp_debugfs_string_read(dsp, user_buf, count, ppos, + &dsp->wmfw_file_name); } static ssize_t cs_dsp_debugfs_bin_read(struct file *file, @@ -435,19 +479,9 @@ static ssize_t cs_dsp_debugfs_bin_read(struct file *file, size_t count, loff_t *ppos) { struct cs_dsp *dsp = file->private_data; - ssize_t ret; - - mutex_lock(&dsp->pwr_lock); - if (!dsp->bin_file_name || !dsp->booted) - ret = 0; - else - ret = simple_read_from_buffer(user_buf, count, ppos, - dsp->bin_file_name, - strlen(dsp->bin_file_name)); - - mutex_unlock(&dsp->pwr_lock); - return ret; + return cs_dsp_debugfs_string_read(dsp, user_buf, count, ppos, + &dsp->bin_file_name); } static const struct { diff --git a/drivers/firmware/cirrus/cs_dsp.h b/drivers/firmware/cirrus/cs_dsp.h new file mode 100644 index 0000000000000..adf543004aea3 --- /dev/null +++ b/drivers/firmware/cirrus/cs_dsp.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * cs_dsp.h -- Private header for cs_dsp driver. + * + * Copyright (C) 2026 Cirrus Logic, Inc. and + * Cirrus Logic International Semiconductor Ltd. + */ + +#ifndef FW_CS_DSP_H +#define FW_CS_DSP_H + +#if IS_ENABLED(CONFIG_KUNIT) +extern bool cs_dsp_suppress_err_messages; +extern bool cs_dsp_suppress_warn_messages; +extern bool cs_dsp_suppress_info_messages; +#endif + +#endif /* ifndef FW_CS_DSP_H */ diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_bin.c b/drivers/firmware/cirrus/test/cs_dsp_test_bin.c index 163b7faecff46..2c6486fa95758 100644 --- a/drivers/firmware/cirrus/test/cs_dsp_test_bin.c +++ b/drivers/firmware/cirrus/test/cs_dsp_test_bin.c @@ -17,6 +17,8 @@ #include #include +#include "../cs_dsp.h" + /* * Test method is: * @@ -2224,7 +2226,22 @@ static int cs_dsp_bin_test_common_init(struct kunit *test, struct cs_dsp *dsp) return ret; /* Automatically call cs_dsp_remove() when test case ends */ - return kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + ret = kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + if (ret) + return ret; + + /* + * The large number of test cases will cause an unusually large amount + * of dev_info() messages from cs_dsp, so suppress these. + */ + cs_dsp_suppress_info_messages = true; + + return 0; +} + +static void cs_dsp_bin_test_exit(struct kunit *test) +{ + cs_dsp_suppress_info_messages = false; } static int cs_dsp_bin_test_halo_init(struct kunit *test) @@ -2536,18 +2553,21 @@ static struct kunit_case cs_dsp_bin_test_cases_adsp2[] = { static struct kunit_suite cs_dsp_bin_test_halo = { .name = "cs_dsp_bin_halo", .init = cs_dsp_bin_test_halo_init, + .exit = cs_dsp_bin_test_exit, .test_cases = cs_dsp_bin_test_cases_halo, }; static struct kunit_suite cs_dsp_bin_test_adsp2_32bit = { .name = "cs_dsp_bin_adsp2_32bit", .init = cs_dsp_bin_test_adsp2_32bit_init, + .exit = cs_dsp_bin_test_exit, .test_cases = cs_dsp_bin_test_cases_adsp2, }; static struct kunit_suite cs_dsp_bin_test_adsp2_16bit = { .name = "cs_dsp_bin_adsp2_16bit", .init = cs_dsp_bin_test_adsp2_16bit_init, + .exit = cs_dsp_bin_test_exit, .test_cases = cs_dsp_bin_test_cases_adsp2, }; diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c b/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c index a7ec956d27249..631b9cb9eb250 100644 --- a/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c +++ b/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c @@ -18,6 +18,8 @@ #include #include +#include "../cs_dsp.h" + KUNIT_DEFINE_ACTION_WRAPPER(_put_device_wrapper, put_device, struct device *); KUNIT_DEFINE_ACTION_WRAPPER(_cs_dsp_remove_wrapper, cs_dsp_remove, struct cs_dsp *); @@ -380,11 +382,9 @@ static void bin_block_payload_len_garbage(struct kunit *test) static void cs_dsp_bin_err_test_exit(struct kunit *test) { - /* - * Testing error conditions can produce a lot of log output - * from cs_dsp error messages, so rate limit the test cases. - */ - usleep_range(200, 500); + cs_dsp_suppress_err_messages = false; + cs_dsp_suppress_warn_messages = false; + cs_dsp_suppress_info_messages = false; } static int cs_dsp_bin_err_test_common_init(struct kunit *test, struct cs_dsp *dsp, @@ -474,7 +474,19 @@ static int cs_dsp_bin_err_test_common_init(struct kunit *test, struct cs_dsp *ds return ret; /* Automatically call cs_dsp_remove() when test case ends */ - return kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + ret = kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + if (ret) + return ret; + + /* + * Testing error conditions can produce a lot of log output + * from cs_dsp error messages, so suppress messages. + */ + cs_dsp_suppress_err_messages = true; + cs_dsp_suppress_warn_messages = true; + cs_dsp_suppress_info_messages = true; + + return 0; } static int cs_dsp_bin_err_test_halo_init(struct kunit *test) diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c index 9e997c4ee2d67..f02cb6cf76386 100644 --- a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c +++ b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c @@ -18,6 +18,8 @@ #include #include +#include "../cs_dsp.h" + /* * Test method is: * @@ -1853,7 +1855,22 @@ static int cs_dsp_wmfw_test_common_init(struct kunit *test, struct cs_dsp *dsp, return ret; /* Automatically call cs_dsp_remove() when test case ends */ - return kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + ret = kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + if (ret) + return ret; + + /* + * The large number of test cases will cause an unusually large amount + * of dev_info() messages from cs_dsp, so suppress these. + */ + cs_dsp_suppress_info_messages = true; + + return 0; +} + +static void cs_dsp_wmfw_test_exit(struct kunit *test) +{ + cs_dsp_suppress_info_messages = false; } static int cs_dsp_wmfw_test_halo_init(struct kunit *test) @@ -2163,42 +2180,49 @@ static struct kunit_case cs_dsp_wmfw_test_cases_adsp2[] = { static struct kunit_suite cs_dsp_wmfw_test_halo = { .name = "cs_dsp_wmfwV3_halo", .init = cs_dsp_wmfw_test_halo_init, + .exit = cs_dsp_wmfw_test_exit, .test_cases = cs_dsp_wmfw_test_cases_halo, }; static struct kunit_suite cs_dsp_wmfw_test_adsp2_32bit_wmfw0 = { .name = "cs_dsp_wmfwV0_adsp2_32bit", .init = cs_dsp_wmfw_test_adsp2_32bit_wmfw0_init, + .exit = cs_dsp_wmfw_test_exit, .test_cases = cs_dsp_wmfw_test_cases_adsp2, }; static struct kunit_suite cs_dsp_wmfw_test_adsp2_32bit_wmfw1 = { .name = "cs_dsp_wmfwV1_adsp2_32bit", .init = cs_dsp_wmfw_test_adsp2_32bit_wmfw1_init, + .exit = cs_dsp_wmfw_test_exit, .test_cases = cs_dsp_wmfw_test_cases_adsp2, }; static struct kunit_suite cs_dsp_wmfw_test_adsp2_32bit_wmfw2 = { .name = "cs_dsp_wmfwV2_adsp2_32bit", .init = cs_dsp_wmfw_test_adsp2_32bit_wmfw2_init, + .exit = cs_dsp_wmfw_test_exit, .test_cases = cs_dsp_wmfw_test_cases_adsp2, }; static struct kunit_suite cs_dsp_wmfw_test_adsp2_16bit_wmfw0 = { .name = "cs_dsp_wmfwV0_adsp2_16bit", .init = cs_dsp_wmfw_test_adsp2_16bit_wmfw0_init, + .exit = cs_dsp_wmfw_test_exit, .test_cases = cs_dsp_wmfw_test_cases_adsp2, }; static struct kunit_suite cs_dsp_wmfw_test_adsp2_16bit_wmfw1 = { .name = "cs_dsp_wmfwV1_adsp2_16bit", .init = cs_dsp_wmfw_test_adsp2_16bit_wmfw1_init, + .exit = cs_dsp_wmfw_test_exit, .test_cases = cs_dsp_wmfw_test_cases_adsp2, }; static struct kunit_suite cs_dsp_wmfw_test_adsp2_16bit_wmfw2 = { .name = "cs_dsp_wmfwV2_adsp2_16bit", .init = cs_dsp_wmfw_test_adsp2_16bit_wmfw2_init, + .exit = cs_dsp_wmfw_test_exit, .test_cases = cs_dsp_wmfw_test_cases_adsp2, }; diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c index c309843261d72..37162d12e2fa7 100644 --- a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c +++ b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c @@ -18,6 +18,8 @@ #include #include +#include "../cs_dsp.h" + KUNIT_DEFINE_ACTION_WRAPPER(_put_device_wrapper, put_device, struct device *); KUNIT_DEFINE_ACTION_WRAPPER(_cs_dsp_remove_wrapper, cs_dsp_remove, struct cs_dsp *); @@ -989,11 +991,9 @@ static void wmfw_v2_coeff_description_exceeds_block(struct kunit *test) static void cs_dsp_wmfw_err_test_exit(struct kunit *test) { - /* - * Testing error conditions can produce a lot of log output - * from cs_dsp error messages, so rate limit the test cases. - */ - usleep_range(200, 500); + cs_dsp_suppress_err_messages = false; + cs_dsp_suppress_warn_messages = false; + cs_dsp_suppress_info_messages = false; } static int cs_dsp_wmfw_err_test_common_init(struct kunit *test, struct cs_dsp *dsp, @@ -1072,7 +1072,19 @@ static int cs_dsp_wmfw_err_test_common_init(struct kunit *test, struct cs_dsp *d return ret; /* Automatically call cs_dsp_remove() when test case ends */ - return kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + ret = kunit_add_action_or_reset(priv->test, _cs_dsp_remove_wrapper, dsp); + if (ret) + return ret; + + /* + * Testing error conditions can produce a lot of log output + * from cs_dsp error messages, so suppress messages. + */ + cs_dsp_suppress_err_messages = true; + cs_dsp_suppress_warn_messages = true; + cs_dsp_suppress_info_messages = true; + + return 0; } static int cs_dsp_wmfw_err_test_halo_init(struct kunit *test) diff --git a/drivers/firmware/cirrus/test/cs_dsp_tests.c b/drivers/firmware/cirrus/test/cs_dsp_tests.c index 7b829a03ca529..288675fdbdc53 100644 --- a/drivers/firmware/cirrus/test/cs_dsp_tests.c +++ b/drivers/firmware/cirrus/test/cs_dsp_tests.c @@ -12,3 +12,4 @@ MODULE_AUTHOR("Richard Fitzgerald "); MODULE_LICENSE("GPL"); MODULE_IMPORT_NS("FW_CS_DSP"); MODULE_IMPORT_NS("FW_CS_DSP_KUNIT_TEST_UTILS"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/firmware/efi/cper-arm.c b/drivers/firmware/efi/cper-arm.c index f0a63d09d3c49..b21cb1232d820 100644 --- a/drivers/firmware/efi/cper-arm.c +++ b/drivers/firmware/efi/cper-arm.c @@ -93,15 +93,11 @@ static void cper_print_arm_err_info(const char *pfx, u32 type, bool proc_context_corrupt, corrected, precise_pc, restartable_pc; bool time_out, access_mode; - /* If the type is unknown, bail. */ - if (type > CPER_ARM_MAX_TYPE) - return; - /* * Vendor type errors have error information values that are vendor * specific. */ - if (type == CPER_ARM_VENDOR_ERROR) + if (type & CPER_ARM_VENDOR_ERROR) return; if (error_info & CPER_ARM_ERR_VALID_TRANSACTION_TYPE) { @@ -116,43 +112,38 @@ static void cper_print_arm_err_info(const char *pfx, u32 type, if (error_info & CPER_ARM_ERR_VALID_OPERATION_TYPE) { op_type = ((error_info >> CPER_ARM_ERR_OPERATION_SHIFT) & CPER_ARM_ERR_OPERATION_MASK); - switch (type) { - case CPER_ARM_CACHE_ERROR: + if (type & CPER_ARM_CACHE_ERROR) { if (op_type < ARRAY_SIZE(arm_cache_err_op_strs)) { - printk("%soperation type: %s\n", pfx, + printk("%scache error, operation type: %s\n", pfx, arm_cache_err_op_strs[op_type]); } - break; - case CPER_ARM_TLB_ERROR: + } + if (type & CPER_ARM_TLB_ERROR) { if (op_type < ARRAY_SIZE(arm_tlb_err_op_strs)) { - printk("%soperation type: %s\n", pfx, + printk("%sTLB error, operation type: %s\n", pfx, arm_tlb_err_op_strs[op_type]); } - break; - case CPER_ARM_BUS_ERROR: + } + if (type & CPER_ARM_BUS_ERROR) { if (op_type < ARRAY_SIZE(arm_bus_err_op_strs)) { - printk("%soperation type: %s\n", pfx, + printk("%sbus error, operation type: %s\n", pfx, arm_bus_err_op_strs[op_type]); } - break; } } if (error_info & CPER_ARM_ERR_VALID_LEVEL) { level = ((error_info >> CPER_ARM_ERR_LEVEL_SHIFT) & CPER_ARM_ERR_LEVEL_MASK); - switch (type) { - case CPER_ARM_CACHE_ERROR: + if (type & CPER_ARM_CACHE_ERROR) printk("%scache level: %d\n", pfx, level); - break; - case CPER_ARM_TLB_ERROR: + + if (type & CPER_ARM_TLB_ERROR) printk("%sTLB level: %d\n", pfx, level); - break; - case CPER_ARM_BUS_ERROR: + + if (type & CPER_ARM_BUS_ERROR) printk("%saffinity level at which the bus error occurred: %d\n", pfx, level); - break; - } } if (error_info & CPER_ARM_ERR_VALID_PROC_CONTEXT_CORRUPT) { @@ -235,20 +226,25 @@ static void cper_print_arm_err_info(const char *pfx, u32 type, } void cper_print_proc_arm(const char *pfx, - const struct cper_sec_proc_arm *proc) + const struct cper_sec_proc_arm *proc, + u32 length) { int i, len, max_ctx_type; struct cper_arm_err_info *err_info; struct cper_arm_ctx_info *ctx_info; - char newpfx[64], infopfx[64]; + char newpfx[64], infopfx[ARRAY_SIZE(newpfx) + 1]; + char error_type[120]; printk("%sMIDR: 0x%016llx\n", pfx, proc->midr); len = proc->section_length - (sizeof(*proc) + proc->err_info_num * (sizeof(*err_info))); - if (len < 0) { - printk("%ssection length: %d\n", pfx, proc->section_length); - printk("%ssection length is too small\n", pfx); + + if (len < 0 || proc->section_length > length) { + printk("%ssection length: %d, CPER size: %d\n", + pfx, proc->section_length, length); + printk("%ssection length is too %s\n", pfx, + (len < 0) ? "small" : "big"); printk("%sfirmware-generated error record is incorrect\n", pfx); printk("%sERR_INFO_NUM is %d\n", pfx, proc->err_info_num); return; @@ -289,9 +285,15 @@ void cper_print_proc_arm(const char *pfx, newpfx); } - printk("%serror_type: %d, %s\n", newpfx, err_info->type, - err_info->type < ARRAY_SIZE(cper_proc_error_type_strs) ? - cper_proc_error_type_strs[err_info->type] : "unknown"); + cper_bits_to_str(error_type, sizeof(error_type), + FIELD_GET(CPER_ARM_ERR_TYPE_MASK, err_info->type), + cper_proc_error_type_strs, + ARRAY_SIZE(cper_proc_error_type_strs)); + + printk("%serror_type: 0x%02x: %s%s\n", newpfx, err_info->type, + error_type, + (err_info->type & ~CPER_ARM_ERR_TYPE_MASK) ? " with reserved bit(s)" : ""); + if (err_info->validation_bits & CPER_ARM_INFO_VALID_ERR_INFO) { printk("%serror_info: 0x%016llx\n", newpfx, err_info->error_info); diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index 928409199a1a4..d4dc69693ece4 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -12,6 +12,7 @@ * Specification version 2.4. */ +#include #include #include #include @@ -106,6 +107,65 @@ void cper_print_bits(const char *pfx, unsigned int bits, printk("%s\n", buf); } +/** + * cper_bits_to_str - return a string for set bits + * @buf: buffer to store the output string + * @buf_size: size of the output string buffer + * @bits: bit mask + * @strs: string array, indexed by bit position + * @strs_size: size of the string array: @strs + * + * Add to @buf the bitmask in hexadecimal. Then, for each set bit in @bits, + * add the corresponding string describing the bit in @strs to @buf. + * + * A typical example is:: + * + * const char * const bits[] = { + * "bit 3 name", + * "bit 4 name", + * "bit 5 name", + * }; + * char str[120]; + * unsigned int bitmask = BIT(3) | BIT(5); + * #define MASK GENMASK(5,3) + * + * cper_bits_to_str(str, sizeof(str), FIELD_GET(MASK, bitmask), + * bits, ARRAY_SIZE(bits)); + * + * The above code fills the string ``str`` with ``bit 3 name|bit 5 name``. + * + * Return: number of bytes stored or an error code if lower than zero. + */ +int cper_bits_to_str(char *buf, int buf_size, unsigned long bits, + const char * const strs[], unsigned int strs_size) +{ + int len = buf_size; + char *str = buf; + int i, size; + + *buf = '\0'; + + for_each_set_bit(i, &bits, strs_size) { + if (!(bits & BIT_ULL(i))) + continue; + + if (*buf && len > 0) { + *str = '|'; + len--; + str++; + } + + size = strscpy(str, strs[i], len); + if (size < 0) + return size; + + len -= size; + str += size; + } + return buf_size - len; +} +EXPORT_SYMBOL_GPL(cper_bits_to_str); + static const char * const proc_type_strs[] = { "IA32/X64", "IA64", @@ -500,6 +560,11 @@ static void cper_print_fw_err(const char *pfx, } else { offset = sizeof(*fw_err); } + if (offset > length) { + printk("%s""error section length is too small: offset=%d, length=%d\n", + pfx, offset, length); + return; + } buf += offset; length -= offset; @@ -599,7 +664,8 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata printk("%ssection_type: ARM processor error\n", newpfx); if (gdata->error_data_length >= sizeof(*arm_err)) - cper_print_proc_arm(newpfx, arm_err); + cper_print_proc_arm(newpfx, arm_err, + gdata->error_data_length); else goto err_section_too_small; #endif diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 1ce428e2ac8a0..c3cf5541ed682 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -74,6 +74,9 @@ struct mm_struct efi_mm = { .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock), .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), .cpu_bitmap = { [BITS_TO_LONGS(NR_CPUS)] = 0}, +#ifdef CONFIG_SCHED_MM_CID + .cpus_allowed_lock = __RAW_SPIN_LOCK_UNLOCKED(efi_mm.cpus_allowed_lock), +#endif }; struct workqueue_struct *efi_rts_wq; @@ -688,13 +691,13 @@ static __init int match_config_table(const efi_guid_t *guid, static __init void reserve_unaccepted(struct efi_unaccepted_memory *unaccepted) { - phys_addr_t start, size; + phys_addr_t start, end; start = PAGE_ALIGN_DOWN(efi.unaccepted); - size = PAGE_ALIGN(sizeof(*unaccepted) + unaccepted->size); + end = PAGE_ALIGN(efi.unaccepted + sizeof(*unaccepted) + unaccepted->size); - memblock_add(start, size); - memblock_reserve(start, size); + memblock_add(start, end - start); + memblock_reserve(start, end - start); } int __init efi_config_parse_tables(const efi_config_table_t *config_tables, diff --git a/drivers/firmware/efi/libstub/x86-5lvl.c b/drivers/firmware/efi/libstub/x86-5lvl.c index f1c5fb45d5f7c..c00d0ae7ed5d5 100644 --- a/drivers/firmware/efi/libstub/x86-5lvl.c +++ b/drivers/firmware/efi/libstub/x86-5lvl.c @@ -66,7 +66,7 @@ void efi_5level_switch(void) bool have_la57 = native_read_cr4() & X86_CR4_LA57; bool need_toggle = want_la57 ^ have_la57; u64 *pgt = (void *)la57_toggle + PAGE_SIZE; - u64 *cr3 = (u64 *)__native_read_cr3(); + pgd_t *cr3 = (pgd_t *)native_read_cr3_pa(); u64 *new_cr3; if (!la57_toggle || !need_toggle) @@ -82,7 +82,7 @@ void efi_5level_switch(void) new_cr3[0] = (u64)cr3 | _PAGE_TABLE_NOENC; } else { /* take the new root table pointer from the current entry #0 */ - new_cr3 = (u64 *)(cr3[0] & PAGE_MASK); + new_cr3 = (u64 *)(native_pgd_val(cr3[0]) & PTE_PFN_MASK); /* copy the new root table if it is not 32-bit addressable */ if ((u64)new_cr3 > U32_MAX) diff --git a/drivers/firmware/efi/mokvar-table.c b/drivers/firmware/efi/mokvar-table.c index aedbbd627706a..741674a0a70c5 100644 --- a/drivers/firmware/efi/mokvar-table.c +++ b/drivers/firmware/efi/mokvar-table.c @@ -85,7 +85,7 @@ static struct kobject *mokvar_kobj; * as an alternative to ordinary EFI variables, due to platform-dependent * limitations. The memory occupied by this table is marked as reserved. * - * This routine must be called before efi_free_boot_services() in order + * This routine must be called before efi_unmap_boot_services() in order * to guarantee that it can mark the table as reserved. * * Implicit inputs: diff --git a/drivers/firmware/imx/imx-scu-irq.c b/drivers/firmware/imx/imx-scu-irq.c index 6125cccc9ba79..f587abcd7ca3a 100644 --- a/drivers/firmware/imx/imx-scu-irq.c +++ b/drivers/firmware/imx/imx-scu-irq.c @@ -203,6 +203,18 @@ int imx_scu_enable_general_irq_channel(struct device *dev) struct mbox_chan *ch; int ret = 0, i = 0; + if (!of_parse_phandle_with_args(dev->of_node, "mboxes", + "#mbox-cells", 0, &spec)) { + i = of_alias_get_id(spec.np, "mu"); + of_node_put(spec.np); + } + + /* use mu1 as general mu irq channel if failed */ + if (i < 0) + i = 1; + + mu_resource_id = IMX_SC_R_MU_0A + i; + ret = imx_scu_get_handle(&imx_sc_irq_ipc_handle); if (ret) return ret; @@ -214,6 +226,8 @@ int imx_scu_enable_general_irq_channel(struct device *dev) cl->dev = dev; cl->rx_callback = imx_scu_irq_callback; + INIT_WORK(&imx_sc_irq_work, imx_scu_irq_work_handler); + /* SCU general IRQ uses general interrupt channel 3 */ ch = mbox_request_channel_byname(cl, "gip3"); if (IS_ERR(ch)) { @@ -223,18 +237,6 @@ int imx_scu_enable_general_irq_channel(struct device *dev) return ret; } - INIT_WORK(&imx_sc_irq_work, imx_scu_irq_work_handler); - - if (!of_parse_phandle_with_args(dev->of_node, "mboxes", - "#mbox-cells", 0, &spec)) - i = of_alias_get_id(spec.np, "mu"); - - /* use mu1 as general mu irq channel if failed */ - if (i < 0) - i = 1; - - mu_resource_id = IMX_SC_R_MU_0A + i; - /* Create directory under /sysfs/firmware */ wakeup_obj = kobject_create_and_add("scu_wakeup_source", firmware_kobj); if (!wakeup_obj) { diff --git a/drivers/firmware/microchip/mpfs-auto-update.c b/drivers/firmware/microchip/mpfs-auto-update.c index e194f7acb2a9b..8fc3749d4a709 100644 --- a/drivers/firmware/microchip/mpfs-auto-update.c +++ b/drivers/firmware/microchip/mpfs-auto-update.c @@ -113,10 +113,6 @@ static enum fw_upload_err mpfs_auto_update_prepare(struct fw_upload *fw_uploader * be added here. */ - priv->flash = mpfs_sys_controller_get_flash(priv->sys_controller); - if (!priv->flash) - return FW_UPLOAD_ERR_HW_ERROR; - erase_size = round_up(erase_size, (u64)priv->flash->erasesize); /* @@ -427,6 +423,12 @@ static int mpfs_auto_update_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(priv->sys_controller), "Could not register as a sub device of the system controller\n"); + priv->flash = mpfs_sys_controller_get_flash(priv->sys_controller); + if (IS_ERR_OR_NULL(priv->flash)) { + dev_dbg(dev, "No flash connected to the system controller, auto-update not supported\n"); + return -ENODEV; + } + priv->dev = dev; platform_set_drvdata(pdev, priv); diff --git a/drivers/firmware/psci/Kconfig b/drivers/firmware/psci/Kconfig index 97944168b5e66..93ff7b071a0c3 100644 --- a/drivers/firmware/psci/Kconfig +++ b/drivers/firmware/psci/Kconfig @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only config ARM_PSCI_FW bool + select POWER_RESET + select REBOOT_MODE config ARM_PSCI_CHECKER bool "ARM PSCI checker" diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c index 38ca190d4a22d..40a27bc2cc3a3 100644 --- a/drivers/firmware/psci/psci.c +++ b/drivers/firmware/psci/psci.c @@ -13,10 +13,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -51,6 +53,24 @@ static int resident_cpu = -1; struct psci_operations psci_ops; static enum arm_smccc_conduit psci_conduit = SMCCC_CONDUIT_NONE; +struct psci_vendor_sysreset2 { + u32 reset_type; + u32 cookie; + bool valid; +}; + +static struct psci_vendor_sysreset2 vendor_reset; + +static int psci_panic_event(struct notifier_block *nb, unsigned long v, void *p) +{ + vendor_reset.valid = false; + return NOTIFY_DONE; +} + +static struct notifier_block psci_panic_block = { + .notifier_call = psci_panic_event +}; + bool psci_tos_resident_on(int cpu) { return cpu == resident_cpu; @@ -309,7 +329,10 @@ static int get_set_conduit_method(const struct device_node *np) static int psci_sys_reset(struct notifier_block *nb, unsigned long action, void *data) { - if ((reboot_mode == REBOOT_WARM || reboot_mode == REBOOT_SOFT) && + if (vendor_reset.valid && psci_system_reset2_supported) { + invoke_psci_fn(PSCI_FN_NATIVE(1_1, SYSTEM_RESET2), vendor_reset.reset_type, + vendor_reset.cookie, 0); + } else if ((reboot_mode == REBOOT_WARM || reboot_mode == REBOOT_SOFT) && psci_system_reset2_supported) { /* * reset_type[31] = 0 (architectural) @@ -547,6 +570,70 @@ static const struct platform_suspend_ops psci_suspend_ops = { .enter = psci_system_suspend_enter, }; +static int psci_set_vendor_sys_reset2(struct reboot_mode_driver *reboot, u64 magic) +{ + u32 magic_32; + + if (psci_system_reset2_supported) { + magic_32 = magic & 0xffffffff; + vendor_reset.reset_type = PSCI_1_1_RESET_TYPE_VENDOR_START | magic_32; + vendor_reset.cookie = (magic >> 32) & 0xffffffff; + vendor_reset.valid = true; + } + + return NOTIFY_DONE; +} + +static int __init psci_init_vendor_reset(void) +{ + struct reboot_mode_driver *reboot; + struct device_node *psci_np; + struct device_node *np; + int ret; + + if (!psci_system_reset2_supported) + return -EINVAL; + + psci_np = of_find_compatible_node(NULL, NULL, "arm,psci-1.0"); + if (!psci_np) + return -ENODEV; + + np = of_find_node_by_name(psci_np, "reboot-mode"); + if (!np) { + of_node_put(psci_np); + return -ENODEV; + } + + ret = atomic_notifier_chain_register(&panic_notifier_list, &psci_panic_block); + if (ret) + goto err_notifier; + + reboot = kzalloc(sizeof(*reboot), GFP_KERNEL); + if (!reboot) { + ret = -ENOMEM; + goto err_kzalloc; + } + + reboot->write = psci_set_vendor_sys_reset2; + reboot->driver_name = "psci"; + + ret = reboot_mode_register(reboot, of_fwnode_handle(np)); + if (ret) + goto err_register; + + return 0; + +err_register: + kfree(reboot); +err_kzalloc: + atomic_notifier_chain_unregister(&panic_notifier_list, &psci_panic_block); +err_notifier: + of_node_put(psci_np); + of_node_put(np); + return ret; +} +late_initcall(psci_init_vendor_reset) + static void __init psci_init_system_reset2(void) { int ret; diff --git a/drivers/firmware/qcom/Kconfig b/drivers/firmware/qcom/Kconfig index b477d54b495a6..dd20fed163e3f 100644 --- a/drivers/firmware/qcom/Kconfig +++ b/drivers/firmware/qcom/Kconfig @@ -74,4 +74,20 @@ config QCOM_QSEECOM_UEFISECAPP Select Y here to provide access to EFI variables on the aforementioned platforms. +config QCOM_MEMORY_DUMP_V2 + tristate "QCOM Memory Dump V2 Support" + depends on QCOM_TZMEM + depends on CMA + help + This enables memory dump feature. It allows various client + subsystems to register respective dump regions. At the time + of deadlocks or cpu hangs these dump regions are captured to + give a snapshot of the system at the time of the crash. + +config QCOM_MEMORY_DUMP_DEV + tristate "QCOM Memory Dump V2 device stub" + depends on QCOM_MEMORY_DUMP_V2 + help + Device stub for memory dump V2 driver. + endmenu diff --git a/drivers/firmware/qcom/Makefile b/drivers/firmware/qcom/Makefile index 0be40a1abc13c..01b293cae79ed 100644 --- a/drivers/firmware/qcom/Makefile +++ b/drivers/firmware/qcom/Makefile @@ -8,3 +8,5 @@ qcom-scm-objs += qcom_scm.o qcom_scm-smc.o qcom_scm-legacy.o obj-$(CONFIG_QCOM_TZMEM) += qcom_tzmem.o obj-$(CONFIG_QCOM_QSEECOM) += qcom_qseecom.o obj-$(CONFIG_QCOM_QSEECOM_UEFISECAPP) += qcom_qseecom_uefisecapp.o +obj-$(CONFIG_QCOM_MEMORY_DUMP_V2) += memory_dump_v2.o +obj-$(CONFIG_QCOM_MEMORY_DUMP_DEV) += memory_dump_dev.o diff --git a/drivers/firmware/qcom/memory_dump_dev.c b/drivers/firmware/qcom/memory_dump_dev.c new file mode 100644 index 0000000000000..23b5d057a174c --- /dev/null +++ b/drivers/firmware/qcom/memory_dump_dev.c @@ -0,0 +1,511 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include + +#define DEV_NAME "msm_mem_dump" + +static struct platform_device *mem_dump_pdev; + +enum dump_ids { + C0_CONTEXT = 0x0, + C100_CONTEXT = 0x1, + C200_CONTEXT = 0x2, + C300_CONTEXT = 0x3, + C400_CONTEXT = 0x4, + C500_CONTEXT = 0x5, + C600_CONTEXT = 0x6, + C700_CONTEXT = 0x7, + L1_ITLB10000 = 0x24, + L1_ITLB10100 = 0x25, + L1_ITLB10200 = 0x26, + L1_ITLB10300 = 0x27, + L1_DTLB10000 = 0x44, + L1_DTLB10100 = 0x45, + L1_DTLB10200 = 0x46, + L1_DTLB10300 = 0x47, + L1_ICACHE0 = 0x60, + L1_ICACHE100 = 0x61, + L1_ICACHE200 = 0x62, + L1_ICACHE300 = 0x63, + L1_ICACHE10000 = 0x64, + L1_ICACHE10100 = 0x65, + L1_ICACHE10200 = 0x66, + L1_ICACHE10300 = 0x67, + L1_DCACHE0 = 0x80, + L1_DCACHE100 = 0x81, + L1_DCACHE200 = 0x82, + L1_DCACHE300 = 0x83, + L1_DCACHE10000 = 0x84, + L1_DCACHE10100 = 0x85, + L1_DCACHE10200 = 0x86, + L1_DCACHE10300 = 0x87, + L2_CACHE10000 = 0xc4, + L2_CACHE10100 = 0xc5, + L2_CACHE10200 = 0xc6, + L2_CACHE10300 = 0xc7, + PMIC = 0xe4, + MISC_DATA = 0xe8, + RPM_SW = 0xea, + RPMH = 0xec, + CPUSS_REG = 0xef, + TMC_ETF = 0xf0, + ETF_SWAO = 0xf1, + ETF_SLPI = 0xf3, + ETF_LPASS = 0xf4, + FCM = 0xee, + ETR_REG = 0x100, + ETF_REG = 0x101, + ETFSWAO_REG = 0x102, + ETFSLPI_REG = 0x103, + ETFLPASS_REG = 0x104, + ETR1_REG = 0x105, + L2_TLB0 = 0x120, + L2_TLB100 = 0x121, + L2_TLB200 = 0x122, + L2_TLB300 = 0x123, + L2_TLB10000 = 0x124, + L2_TLB10100 = 0x125, + L2_TLB10200 = 0x126, + L2_TLB10300 = 0x127, + C0_SCANDUMP = 0x130, + C100_SCANDUMP = 0x131, + C200_SCANDUMP = 0x132, + C300_SCANDUMP = 0x133, + C10000_SCANDUMP = 0x134, + C10100_SCANDUMP = 0x135, + C10200_SCANDUMP = 0x136, + C10300_SCANDUMP = 0x137, + LLCC1_D_CACHE = 0x140, + LLCC2_D_CACHE = 0x141, + MHM_SCAN = 0x161, + GEMNOC = 0x162, + OSM_REG = 0x163, + PCU_REG = 0x164, + FSM_DATA = 0x165, + SCANDUMP_SMMU = 0x220, +}; + +static const struct dump_item lemans_items[] = { + { C0_CONTEXT, 0x800, "c0-context" }, + { C0_SCANDUMP, 0x40000, "c0-scandump" }, + { C100_CONTEXT, 0x800, "c100-context" }, + { C100_SCANDUMP, 0x40000, "c100-scandump" }, + { C200_CONTEXT, 0x800, "c200-context" }, + { C200_SCANDUMP, 0x40000, "c200-scandump" }, + { C300_CONTEXT, 0x800, "c300-context" }, + { C300_SCANDUMP, 0x40000, "c300-scandump" }, + { C400_CONTEXT, 0x800, "c400-context" }, + { C500_CONTEXT, 0x800, "c500-context" }, + { C600_CONTEXT, 0x800, "c600-context" }, + { C700_CONTEXT, 0x800, "c700-context" }, + { C10000_SCANDUMP, 0x40000, "c10000-scandump" }, + { C10100_SCANDUMP, 0x40000, "c10100-scandump" }, + { C10200_SCANDUMP, 0x40000, "c10200-scandump" }, + { C10300_SCANDUMP, 0x40000, "c10300-scandump" }, + { CPUSS_REG, 0x20000, "cpuss-reg" }, + { ETF_SWAO, 0x10000, "etf-swao" }, + { ETFSWAO_REG, 0x1000, "etfswao-reg" }, + { ETR_REG, 0x1000, "etr-reg" }, + { FCM, 0x8400, "fcm" }, + { L1_DCACHE0, 0x12100, "l1-dcache0" }, + { L1_DCACHE100, 0x12100, "l1-dcache100" }, + { L1_DCACHE200, 0x12100, "l1-dcache200" }, + { L1_DCACHE300, 0x12100, "l1-dcache300" }, + { L1_DCACHE10000, 0x12100, "l1-dcache10000" }, + { L1_DCACHE10100, 0x12100, "l1-dcache10100" }, + { L1_DCACHE10200, 0x12100, "l1-dcache10200" }, + { L1_DCACHE10300, 0x12100, "l1-dcache10300" }, + { L1_DTLB10000, 0x300, "l1-dtlb10000" }, + { L1_DTLB10100, 0x300, "l1-dtlb10100" }, + { L1_DTLB10200, 0x300, "l1-dtlb10200" }, + { L1_DTLB10300, 0x300, "l1-dtlb10300" }, + { L1_ICACHE0, 0x26100, "l1-icache0" }, + { L1_ICACHE100, 0x26100, "l1-icache100" }, + { L1_ICACHE200, 0x26100, "l1-icache200" }, + { L1_ICACHE300, 0x26100, "l1-icache300" }, + { L1_ICACHE10000, 0x26100, "l1-icache10000" }, + { L1_ICACHE10100, 0x26100, "l1-icache10100" }, + { L1_ICACHE10200, 0x26100, "l1-icache10200" }, + { L1_ICACHE10300, 0x26100, "l1-icache10300" }, + { L1_ITLB10000, 0x300, "l1-itlb10000" }, + { L1_ITLB10100, 0x300, "l1-itlb10100" }, + { L1_ITLB10200, 0x300, "l1-itlb10200" }, + { L1_ITLB10300, 0x300, "l1-itlb10300" }, + { L2_CACHE10000, 0x90100, "l2-cache10000" }, + { L2_CACHE10100, 0x90100, "l2-cache10100" }, + { L2_CACHE10200, 0x90100, "l2-cache10200" }, + { L2_CACHE10300, 0x90100, "l2-cache10300" }, + { L2_TLB0, 0x6100, "l2-tlb0" }, + { L2_TLB100, 0x6100, "l2-tlb100" }, + { L2_TLB200, 0x6100, "l2-tlb200" }, + { L2_TLB300, 0x6100, "l2-tlb300" }, + { L2_TLB10000, 0x6100, "l2-tlb10000" }, + { L2_TLB10100, 0x6100, "l2-tlb10100" }, + { L2_TLB10200, 0x6100, "l2-tlb10200" }, + { L2_TLB10300, 0x6100, "l2-tlb10300" }, + { MISC_DATA, 0x1000, "misc-data" }, + { PMIC, 0x80000, "pmic" }, + { RPM_SW, 0x28000, "rpm-sw" }, + { RPMH, 0x2000000, "rpmh" }, +}; + +static const struct dump_item talos_items[] = { + { C0_CONTEXT, 0x800, "c0-context" }, + { C100_CONTEXT, 0x800, "c100-context" }, + { C200_CONTEXT, 0x800, "c200-context" }, + { C300_CONTEXT, 0x800, "c300-context" }, + { C400_CONTEXT, 0x800, "c400-context" }, + { C500_CONTEXT, 0x800, "c500-context" }, + { C600_CONTEXT, 0x800, "c600-context" }, + { C700_CONTEXT, 0x800, "c700-context" }, + { RPMH, 0x2000000, "rpmh" }, + { RPM_SW, 0x28000, "rpm-sw" }, + { PMIC, 0x10000, "pmic" }, + { FCM, 0x8400, "fcm" }, + { TMC_ETF, 0x8000, "tmc-etf" }, + { ETF_SWAO, 0x8000, "etf-swao" }, + { ETR_REG, 0x1000, "etr-reg" }, + { ETF_REG, 0x1000, "etf-reg" }, + { ETFSWAO_REG, 0x1000, "etfswao-reg" }, + { MISC_DATA, 0x1000, "misc-data" }, + { L1_ICACHE0, 0x8800, "l1-icache0" }, + { L1_ICACHE100, 0x8800, "l1-icache100" }, + { L1_ICACHE200, 0x8800, "l1-icache200" }, + { L1_ICACHE300, 0x8800, "l1-icache300" }, + { L1_ICACHE10000, 0x8800, "l1-icache400" }, + { L1_ICACHE10100, 0x8800, "l1-icache500" }, + { L1_ICACHE10200, 0x11000, "l1-icache600" }, + { L1_ICACHE10300, 0x11000, "l1-icache700" }, + { L1_DCACHE0, 0x9000, "l1-dcache0" }, + { L1_DCACHE100, 0x9000, "l1-dcache100" }, + { L1_DCACHE200, 0x9000, "l1-dcache200" }, + { L1_DCACHE300, 0x9000, "l1-dcache300" }, + { L1_DCACHE10000, 0x9000, "l1-dcache400" }, + { L1_DCACHE10100, 0x9000, "l1-dcache500" }, + { L1_DCACHE10200, 0x12000, "l1-dcache600" }, + { L1_DCACHE10300, 0x12000, "l1-dcache700" }, + { L1_ITLB10200, 0x300, "l1-itlb600" }, + { L1_ITLB10300, 0x300, "l1-itlb700" }, + { L1_DTLB10200, 0x480, "l1-dtlb600" }, + { L1_DTLB10300, 0x480, "l1-dtlb700" }, + { L2_CACHE10200, 0x48000, "l2-cache600" }, + { L2_CACHE10300, 0x48000, "l2-cache700" }, + { L2_TLB0, 0x5000, "l2-tlb0" }, + { L2_TLB100, 0x5000, "l2-tlb100" }, + { L2_TLB200, 0x5000, "l2-tlb200" }, + { L2_TLB300, 0x5000, "l2-tlb300" }, + { L2_TLB10000, 0x5000, "l2-tlb400" }, + { L2_TLB10100, 0x5000, "l2-tlb500" }, + { L2_TLB10200, 0x7800, "l2-tlb600" }, + { L2_TLB10300, 0x7800, "l2-tlb700" }, + { LLCC1_D_CACHE, 0x6c000, "llcc1-d-cache" }, +}; + +static const struct dump_item kodiak_items[] = { + { C0_CONTEXT, 0x800, "c0-context" }, + { C0_SCANDUMP, 0x10100, "c0-scandump" }, + { C100_CONTEXT, 0x800, "c100-context" }, + { C100_SCANDUMP, 0x10100, "c100-scandump" }, + { C200_CONTEXT, 0x800, "c200-context" }, + { C200_SCANDUMP, 0x10100, "c200-scandump" }, + { C300_CONTEXT, 0x800, "c300-context" }, + { C300_SCANDUMP, 0x10100, "c300-scandump" }, + { C400_CONTEXT, 0x800, "c400-context" }, + { C10000_SCANDUMP, 0x40000, "c400-scandump" }, + { C500_CONTEXT, 0x800, "c500-context" }, + { C10100_SCANDUMP, 0x40000, "c500-scandump" }, + { C600_CONTEXT, 0x800, "c600-context" }, + { C10200_SCANDUMP, 0x40000, "c600-scandump" }, + { C700_CONTEXT, 0x800, "c700-context" }, + { C10300_SCANDUMP, 0x40000, "c700-scandump" }, + { CPUSS_REG, 0x30000, "cpuss-reg" }, + { ETF_LPASS, 0x4000, "etf-lpass" }, + { ETFLPASS_REG, 0x1000, "etflpass-reg" }, + { ETF_SWAO, 0x10000, "etf-swao" }, + { ETFSWAO_REG, 0x1000, "etfswao-reg" }, + { ETR_REG, 0x1000, "etr-reg" }, + { FCM, 0x8400, "fcm" }, + { FSM_DATA, 0x400, "fsm-data" }, + { GEMNOC, 0x100000, "gemnoc" }, + { L1_DCACHE0, 0x9100, "l1-dcache0" }, + { L1_DCACHE100, 0x9100, "l1-dcache100" }, + { L1_DCACHE200, 0x9100, "l1-dcache200" }, + { L1_DCACHE300, 0x9100, "l1-dcache300" }, + { L1_DCACHE10000, 0x9100, "l1-dcache400" }, + { L1_DCACHE10100, 0x9100, "l1-dcache500" }, + { L1_DCACHE10200, 0x9100, "l1-dcache600" }, + { L1_DCACHE10300, 0x12100, "l1-dcache700" }, + { L1_DTLB10000, 0x300, "l1-dtlb400" }, + { L1_DTLB10100, 0x300, "l1-dtlb500" }, + { L1_DTLB10200, 0x300, "l1-dtlb600" }, + { L1_DTLB10300, 0x3a0, "l1-dtlb700" }, + { L1_ICACHE0, 0x10900, "l1-icache0" }, + { L1_ICACHE100, 0x10900, "l1-icache100" }, + { L1_ICACHE200, 0x10900, "l1-icache200" }, + { L1_ICACHE300, 0x10900, "l1-icache300" }, + { L1_ICACHE10000, 0x15100, "l1-icache400" }, + { L1_ICACHE10100, 0x15100, "l1-icache500" }, + { L1_ICACHE10200, 0x15100, "l1-icache600" }, + { L1_ICACHE10300, 0x32100, "l1-icache700" }, + { L1_ITLB10000, 0x300, "l1-itlb400" }, + { L1_ITLB10100, 0x300, "l1-itlb500" }, + { L1_ITLB10200, 0x300, "l1-itlb600" }, + { L1_ITLB10300, 0x400, "l1-itlb700" }, + { L2_CACHE10000, 0x90100, "l2-cache400" }, + { L2_CACHE10100, 0x90100, "l2-cache500" }, + { L2_CACHE10200, 0x90100, "l2-cache600" }, + { L2_CACHE10300, 0x120100, "l2-cache700" }, + { L2_TLB0, 0x5b00, "l2-tlb0" }, + { L2_TLB100, 0x5b00, "l2-tlb100" }, + { L2_TLB200, 0x5b00, "l2-tlb200" }, + { L2_TLB300, 0x5b00, "l2-tlb300" }, + { L2_TLB10000, 0x6100, "l2-tlb400" }, + { L2_TLB10100, 0x6100, "l2-tlb500" }, + { L2_TLB10200, 0x6100, "l2-tlb600" }, + { L2_TLB10300, 0xc100, "l2-tlb700" }, + { LLCC1_D_CACHE, 0x1141c0, "llcc1-d-cache" }, + { LLCC2_D_CACHE, 0x1141c0, "llcc2-d-cache" }, + { MHM_SCAN, 0x20000, "mhm-scan" }, + { MISC_DATA, 0x1000, "misc-data" }, + { OSM_REG, 0x400, "osm-reg" }, + { PCU_REG, 0x400, "pcu-reg" }, + { PMIC, 0x200000, "pmic" }, + { RPM_SW, 0x28000, "rpm-sw" }, + { RPMH, 0x2000000, "rpmh" }, +}; + +static const struct dump_item pakala_items[] = { + { C0_CONTEXT, 0x800, "c0-context" }, + { C100_CONTEXT, 0x800, "c100-context" }, + { C200_CONTEXT, 0x800, "c200-context" }, + { C300_CONTEXT, 0x800, "c300-context" }, + { C400_CONTEXT, 0x800, "c400-context" }, + { C500_CONTEXT, 0x800, "c500-context" }, + { C600_CONTEXT, 0x800, "c600-context" }, + { C700_CONTEXT, 0x800, "c700-context" }, + { RPMH, 0x400000, "rpmh" }, + { RPM_SW, 0x28000, "rpm-sw" }, + { PMIC, 0x200000, "pmic" }, + { FCM, 0x8400, "fcm" }, + { MISC_DATA, 0x1000, "misc-data" }, + { ETF_SWAO, 0x10000, "etf-swao" }, + { ETR_REG, 0x1000, "etr-reg" }, + { ETFSWAO_REG, 0x1000, "etfswao-reg" }, + { ETR1_REG, 0x1000, "etr1-reg" }, + { ETF_SLPI, 0x4000, "etf-slpi" }, + { ETFSLPI_REG, 0x1000, "etfslpi-reg" }, + { ETF_LPASS, 0x4000, "etf-lpass" }, + { ETFLPASS_REG, 0x1000, "etflpass-reg" }, + { OSM_REG, 0x400, "osm-reg" }, + { PCU_REG, 0x400, "pcu-reg" }, + { FSM_DATA, 0x400, "fsm-data" }, + { SCANDUMP_SMMU, 0x40000, "scandump-smmu" }, + { C0_SCANDUMP, 0x380000, "apps-scandump" }, +}; + +static const struct dump_item kaanapali_items[] = { + { C0_CONTEXT, 0x800, "c0-context" }, + { C100_CONTEXT, 0x800, "c100-context" }, + { C200_CONTEXT, 0x800, "c200-context" }, + { C300_CONTEXT, 0x800, "c300-context" }, + { C400_CONTEXT, 0x800, "c400-context" }, + { C500_CONTEXT, 0x800, "c500-context" }, + { C600_CONTEXT, 0x800, "c600-context" }, + { C700_CONTEXT, 0x800, "c700-context" }, + { RPMH, 0x400000, "rpmh" }, + { RPM_SW, 0x28000, "rpm-sw" }, + { PMIC, 0x200000, "pmic" }, + { FCM, 0x8400, "fcm" }, + { MISC_DATA, 0x1000, "misc-data" }, + { ETF_SWAO, 0x10000, "etf-swao" }, + { ETR_REG, 0x1000, "etr-reg" }, + { ETFSWAO_REG, 0x1000, "etfswao-reg" }, + { ETR1_REG, 0x1000, "etr1-reg" }, + { ETF_SLPI, 0x4000, "etf-slpi" }, + { ETFSLPI_REG, 0x1000, "etfslpi-reg" }, + { ETF_LPASS, 0x4000, "etf-lpass" }, + { ETFLPASS_REG, 0x1000, "etflpass-reg" }, + { OSM_REG, 0x400, "osm-reg" }, + { PCU_REG, 0x400, "pcu-reg" }, + { FSM_DATA, 0x400, "fsm-data" }, + { SCANDUMP_SMMU, 0x40000, "scandump-smmu" }, + { C0_SCANDUMP, 0x380000, "apps-scandump" }, +}; + +static const struct dump_item hamoa_items[] = { + { C0_CONTEXT, 0x800, "c0-context" }, + { C100_CONTEXT, 0x800, "c100-context" }, + { C200_CONTEXT, 0x800, "c200-context" }, + { C300_CONTEXT, 0x800, "c300-context" }, + { C400_CONTEXT, 0x800, "c400-context" }, + { C500_CONTEXT, 0x800, "c500-context" }, + { C600_CONTEXT, 0x800, "c600-context" }, + { C700_CONTEXT, 0x800, "c700-context" }, + { RPMH, 0xc10000, "rpmh" }, + { PMIC, 0x200000, "pmic" }, + { ETF_SWAO, 0x10000, "etf-swao" }, + { ETR_REG, 0x1000, "etr-reg" }, + { ETFSWAO_REG, 0x1000, "etfswao-reg" }, +}; + +static const struct dump_table lemans_dump_table = { + .items = lemans_items, + .num_of_items = ARRAY_SIZE(lemans_items), + .imem_base = 0x146d8010, + .imem_size = 0x8, +}; + +static const struct dump_table talos_dump_table = { + .items = talos_items, + .num_of_items = ARRAY_SIZE(talos_items), + .imem_base = 0x146aa010, + .imem_size = 0x8, +}; + +static const struct dump_table kodiak_dump_table = { + .items = kodiak_items, + .num_of_items = ARRAY_SIZE(kodiak_items), + .imem_base = 0x146aa010, + .imem_size = 0x8, +}; + +static const struct dump_table pakala_dump_table = { + .items = pakala_items, + .num_of_items = ARRAY_SIZE(pakala_items), + .imem_base = 0x14680010, + .imem_size = 0x8, +}; + +static const struct dump_table kaanapali_dump_table = { + .items = kaanapali_items, + .num_of_items = ARRAY_SIZE(kaanapali_items), + .imem_base = 0x14680010, + .imem_size = 0x8, +}; + +static const struct dump_table hamoa_dump_table = { + .items = hamoa_items, + .num_of_items = ARRAY_SIZE(hamoa_items), + .imem_base = 0x146aa010, + .imem_size = 0x8, +}; + +static int __init mem_dump_dev_init(void) +{ + int ret; + u32 soc_id; + + mem_dump_pdev = platform_device_alloc(DEV_NAME, -1); + if (!mem_dump_pdev) + return -ENOMEM; + + ret = qcom_smem_get_soc_id(&soc_id); + if (ret) + goto fail; + + switch (soc_id) { + case 377: + case 380: + case 384: + case 401: + case 406: + case 680: + ret = platform_device_add_data(mem_dump_pdev, + &talos_dump_table, sizeof(talos_dump_table)); + if (ret) + goto fail; + + break; + case 534: + case 606: + case 667: + case 674: + case 675: + case 676: + ret = platform_device_add_data(mem_dump_pdev, + &lemans_dump_table, sizeof(lemans_dump_table)); + if (ret) + goto fail; + + break; + case 475: + case 497: + case 498: + case 515: + ret = platform_device_add_data(mem_dump_pdev, + &kodiak_dump_table, sizeof(kodiak_dump_table)); + if (ret) + goto fail; + + break; + case 618: + case 639: + case 705: + case 706: + ret = platform_device_add_data(mem_dump_pdev, + &pakala_dump_table, sizeof(pakala_dump_table)); + if (ret) + goto fail; + + break; + case 660: + case 661: + case 704: + case 722: + case 723: + case 730: + case 743: + ret = platform_device_add_data(mem_dump_pdev, + &kaanapali_dump_table, sizeof(kaanapali_dump_table)); + if (ret) + goto fail; + + break; + case 555: + case 615: + case 616: + case 709: + case 710: + ret = platform_device_add_data(mem_dump_pdev, + &hamoa_dump_table, sizeof(hamoa_dump_table)); + if (ret) + goto fail; + + break; + default: + dev_err(&mem_dump_pdev->dev, "Invalid SoC ID\n"); + ret = -EINVAL; + goto fail; + } + + ret = platform_device_add(mem_dump_pdev); + if (ret) + goto fail; + + dev_info(&mem_dump_pdev->dev, "Register platform device successfully\n"); + + return 0; + +fail: + dev_err(&mem_dump_pdev->dev, + "Failed to register memory dump platform device\n"); + platform_device_put(mem_dump_pdev); + + return ret; +} + +static void __exit mem_dump_dev_exit(void) +{ + platform_device_unregister(mem_dump_pdev); +} + +late_initcall(mem_dump_dev_init); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Qualcomm Technologies Inc. Memory Dump driver V2, device stub"); diff --git a/drivers/firmware/qcom/memory_dump_v2.c b/drivers/firmware/qcom/memory_dump_v2.c new file mode 100644 index 0000000000000..831d1aea86e09 --- /dev/null +++ b/drivers/firmware/qcom/memory_dump_v2.c @@ -0,0 +1,1102 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014-2017, 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MSM_DUMP_TABLE_VERSION MSM_DUMP_MAKE_VERSION(2, 0) + +#define SCM_CMD_DEBUG_LAR_UNLOCK 0x4 + +#define CPUSS_REGDUMP 0xEF +#define SPR_DUMP_CPU0 0x1F0 +#define SPR_DUMP_CPU1 0x1F1 +#define SPR_DUMP_CPU2 0x1F2 +#define SPR_DUMP_CPU3 0x1F3 +#define SPR_DUMP_CPU4 0x1F4 +#define SPR_DUMP_CPU5 0x1F5 +#define SPR_DUMP_CPU6 0x1F6 +#define SPR_DUMP_CPU7 0x1F7 +#define SPR_DATA_HEADER_SIZE 5 +#define SPR_DATA_HEADER_TAIL_SIZE 1 +#define SPR_INPUT_DATA_TAIL_SIZE 1 +#define SPR_INPUT_DATA_SIZE 1 +#define SPR_OUTPUT_DATA_SIZE 2 +#define MAX_CORE_NUM 8 + +#define INPUT_DATA_BY_HLOS 0x00C0FFEE +#define FORMAT_VERSION_1 0x1 +#define FORMAT_VERSION_2 0x2 +#define CORE_REG_NUM_DEFAULT 0x1 + +#define MAGIC_INDEX 0 +#define FORMAT_VERSION_INDEX 1 +#define SYS_REG_INPUT_INDEX 2 +#define OUTPUT_DUMP_INDEX 3 +#define PERCORE_INDEX 4 +#define SYSTEM_REGS_INPUT_INDEX 5 + +#define CMD_REPEAT_READ (0x2 << 24) +#define CMD_DELAY (0x1 << 24) +#define CMD_READ 0x0 +#define CMD_READ_WORD 0x1 +#define CMD_WRITE 0x2 +#define CMD_EXTRA 0x3 + +#define CMD_MASK 0x3 +#define OFFSET_MASK GENMASK(31, 2) +#define EXTRA_CMD_MASK GENMASK(31, 24) +#define EXTRA_VALUE_MASK GENMASK(23, 0) +#define MAX_EXTRA_VALUE 0xffffff + +struct sprs_dump_data { + void *dump_vaddr; + u32 size; + u32 sprs_data_index; + u32 used_memory; +}; + +struct cpuss_regdump_data { + void *dump_vaddr; + u32 size; + u32 core_reg_num; + u32 core_reg_used_num; + u32 core_reg_end_index; + u32 sys_reg_size; + u32 used_memory; +}; + +struct cpuss_dump_data { + struct mutex mutex; + struct cpuss_regdump_data *cpussregdata; + struct sprs_dump_data *sprdata[MAX_CORE_NUM]; +}; + +struct reg_dump_data { + uint32_t magic; + uint32_t version; + uint32_t system_regs_input_index; + uint32_t regdump_output_byte_offset; +}; + +struct msm_dump_table { + uint32_t version; + uint32_t num_entries; + struct msm_dump_entry entries[MAX_NUM_ENTRIES]; +}; + +struct msm_memory_dump { + uint64_t table_phys; + struct msm_dump_table *table; +}; + +/** + * Set bit 0 if percore reg dump initialized. + * Set bit 1 if spr dump initialized. + */ +#define PERCORE_REG_INITIALIZED BIT(0) +#define SPRS_INITIALIZED BIT(1) + +static struct msm_memory_dump memdump; + +/** + * reset_sprs_dump_table - reset the sprs dump table + * + * This function calculates system_regs_input_index and + * regdump_output_byte_offset to store into the dump memory. + * It also updates members of cpudata by the parameter core_reg_num. + * + * Returns 0 on success, or -ENOMEM on error of no enough memory. + */ +static int reset_sprs_dump_table(struct device *dev) +{ + int ret = 0; + struct reg_dump_data *p; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + int i = 0; + + if (!cpudata) + return -EFAULT; + + mutex_lock(&cpudata->mutex); + + for (i = 0; i < MAX_CORE_NUM; i++) { + if (cpudata->sprdata[i]) { + cpudata->sprdata[i]->sprs_data_index = 0; + cpudata->sprdata[i]->used_memory = (SPR_DATA_HEADER_SIZE + + SPR_INPUT_DATA_TAIL_SIZE) * sizeof(uint32_t); + memset(cpudata->sprdata[i]->dump_vaddr, 0xDE, + cpudata->sprdata[i]->size); + p = (struct reg_dump_data *)cpudata->sprdata[i]->dump_vaddr; + p->magic = INPUT_DATA_BY_HLOS; + p->version = FORMAT_VERSION_1; + p->system_regs_input_index = SYSTEM_REGS_INPUT_INDEX; + p->regdump_output_byte_offset = (SPR_DATA_HEADER_SIZE + + SPR_INPUT_DATA_TAIL_SIZE) * sizeof(uint32_t); + memset((uint32_t *)cpudata->sprdata[i]->dump_vaddr + + PERCORE_INDEX, 0x0, (SPR_DATA_HEADER_TAIL_SIZE + + SPR_INPUT_DATA_TAIL_SIZE) * sizeof(uint32_t)); + } + } + + mutex_unlock(&cpudata->mutex); + return ret; +} + + +/** + * update_reg_dump_table - update the register dump table + * @core_reg_num: the number of per-core registers + * + * This function calculates system_regs_input_index and + * regdump_output_byte_offset to store into the dump memory. + * It also updates members of cpudata by the parameter core_reg_num. + * + * Returns 0 on success, or -ENOMEM on error of no enough memory. + */ +static int update_reg_dump_table(struct device *dev, u32 core_reg_num) +{ + int ret = 0; + u32 system_regs_input_index = SYSTEM_REGS_INPUT_INDEX + + core_reg_num * 2; + u32 regdump_output_byte_offset = (system_regs_input_index + 1) + * sizeof(uint32_t); + struct reg_dump_data *p; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + mutex_lock(&cpudata->mutex); + + if (regdump_output_byte_offset >= cpudata->cpussregdata->size || + regdump_output_byte_offset / sizeof(uint32_t) + < system_regs_input_index + 1) { + ret = -ENOMEM; + goto err; + } + + cpudata->cpussregdata->core_reg_num = core_reg_num; + cpudata->cpussregdata->core_reg_used_num = 0; + cpudata->cpussregdata->core_reg_end_index = PERCORE_INDEX; + cpudata->cpussregdata->sys_reg_size = 0; + cpudata->cpussregdata->used_memory = regdump_output_byte_offset; + + memset(cpudata->cpussregdata->dump_vaddr, 0xDE, cpudata->cpussregdata->size); + p = (struct reg_dump_data *)cpudata->cpussregdata->dump_vaddr; + p->magic = INPUT_DATA_BY_HLOS; + p->version = FORMAT_VERSION_2; + p->system_regs_input_index = system_regs_input_index; + p->regdump_output_byte_offset = regdump_output_byte_offset; + memset((uint32_t *)cpudata->cpussregdata->dump_vaddr + PERCORE_INDEX, 0x0, + (system_regs_input_index - PERCORE_INDEX + 1) + * sizeof(uint32_t)); + +err: + mutex_unlock(&cpudata->mutex); + return ret; +} + +static ssize_t core_reg_num_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + if (!cpudata) + return -EFAULT; + + mutex_lock(&cpudata->mutex); + + ret = scnprintf(buf, PAGE_SIZE, "%u\n", cpudata->cpussregdata->core_reg_num); + + mutex_unlock(&cpudata->mutex); + return ret; +} + +static ssize_t core_reg_num_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned int val; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + if (kstrtouint(buf, 16, &val)) + return -EINVAL; + + mutex_lock(&cpudata->mutex); + + if (cpudata->cpussregdata->core_reg_used_num || cpudata->cpussregdata->sys_reg_size) { + dev_err(dev, "Couldn't set core_reg_num, register available in list\n"); + ret = -EPERM; + goto err; + } + if (val == cpudata->cpussregdata->core_reg_num) { + mutex_unlock(&cpudata->mutex); + return size; + } + + mutex_unlock(&cpudata->mutex); + + ret = update_reg_dump_table(dev, val); + if (ret) { + dev_err(dev, "Couldn't set core_reg_num, no enough memory\n"); + return ret; + } + + return size; + +err: + mutex_unlock(&cpudata->mutex); + return ret; +} +static DEVICE_ATTR_RW(core_reg_num); + +/** + * This function shows configs of per-core and system registers. + */ +static ssize_t register_config_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + char local_buf[64]; + int len = 0, count = 0; + int index, system_index_start, index_end; + uint32_t register_offset, val; + uint32_t *p, cmd; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + buf[0] = '\0'; + + if (!cpudata) + return -EFAULT; + + mutex_lock(&cpudata->mutex); + + p = (uint32_t *)cpudata->cpussregdata->dump_vaddr; + + /* print per-core & system registers */ + len = scnprintf(local_buf, 64, "per-core registers:\n"); + strlcat(buf, local_buf, PAGE_SIZE); + count += len; + + system_index_start = *(p + SYS_REG_INPUT_INDEX); + index_end = system_index_start + + cpudata->cpussregdata->sys_reg_size / sizeof(uint32_t) + 1; + for (index = PERCORE_INDEX; index < index_end;) { + if (index == system_index_start) { + len = scnprintf(local_buf, 64, "system registers:\n"); + if ((count + len) > PAGE_SIZE) { + dev_err(dev, "Couldn't write complete config\n"); + break; + } + + strlcat(buf, local_buf, PAGE_SIZE); + count += len; + } + + register_offset = *(p + index); + if (register_offset == 0) { + index++; + continue; + } + + cmd = register_offset & CMD_MASK; + register_offset &= OFFSET_MASK; + + switch (cmd) { + case CMD_READ: + val = *(p + index + 1); + len = scnprintf(local_buf, 64, + "0x%x, 0x%x, r\n", + register_offset, val); + index += 2; + break; + case CMD_READ_WORD: + len = scnprintf(local_buf, 64, + "0x%x, 0x%x, r\n", + register_offset, 0x4); + index++; + break; + case CMD_WRITE: + val = *(p + index + 1); + len = scnprintf(local_buf, 64, + "0x%x, 0x%x, w\n", + register_offset, val); + index += 2; + break; + case CMD_EXTRA: + val = *(p + index + 1); + cmd = val & EXTRA_CMD_MASK; + val &= EXTRA_VALUE_MASK; + if (cmd == CMD_DELAY) + len = scnprintf(local_buf, 64, + "0x%x, 0x%x, d\n", + register_offset, val); + else + len = scnprintf(local_buf, 64, + "0x%x, 0x%x, R\n", + register_offset, val); + index += 2; + break; + } + + if ((count + len) > PAGE_SIZE) { + dev_err(dev, "Couldn't write complete config\n"); + break; + } + + strlcat(buf, local_buf, PAGE_SIZE); + count += len; + } + + mutex_unlock(&cpudata->mutex); + return count; +} + +static int config_cpuss_register(struct device *dev, + uint32_t *p, uint32_t index, char cmd, + uint32_t register_offset, uint32_t val) +{ + int ret = 0; + + switch (cmd) { + case 'r': + if (val > 4) { + *(p + index) = register_offset; + *(p + index + 1) = val; + } else { + *(p + index) = register_offset | CMD_READ_WORD; + } + break; + case 'R': + if (val > MAX_EXTRA_VALUE) { + dev_err(dev, "repeat read time exceeded the limit\n"); + ret = -EINVAL; + return ret; + } + *(p + index) = register_offset | CMD_EXTRA; + *(p + index + 1) = val | CMD_REPEAT_READ; + break; + case 'd': + if (val > MAX_EXTRA_VALUE) { + dev_err(dev, "sleep time exceeded the limit\n"); + ret = -EINVAL; + return ret; + } + *(p + index) = CMD_EXTRA; + *(p + index + 1) = val | CMD_DELAY; + break; + case 'w': + *(p + index) = register_offset | CMD_WRITE; + *(p + index + 1) = val; + break; + default: + dev_err(dev, "Don't support this command\n"); + ret = -EINVAL; + } + return ret; +} +/** + * This function sets configs of per-core or system registers. + */ +static ssize_t register_config_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + uint32_t register_offset, val, reserve_size = 4, per_core = 0; + int nval; + char cmd; + uint32_t num_cores; + u32 extra_memory; + u32 used_memory; + u32 system_reg_end_index; + uint32_t *p; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + nval = sscanf(buf, "%x %x %c %u", ®ister_offset, + &val, &cmd, &per_core); + if (nval < 2) + return -EINVAL; + if (nval == 2) + cmd = 'r'; + if (per_core > 1) + return -EINVAL; + if (register_offset & 0x3) { + dev_err(dev, "Invalid address, must be 4 byte aligned\n"); + return -EINVAL; + } + + if (cmd == 'r' || cmd == 'R') { + if (val == 0) { + dev_err(dev, "Invalid length of 0\n"); + return -EINVAL; + } + if (cmd == 'r' && val & 0x3) { + dev_err(dev, "Invalid length, must be 4 byte aligned\n"); + return -EINVAL; + } + if (cmd == 'R') + reserve_size = val * 4; + else + reserve_size = val; + } + + mutex_lock(&cpudata->mutex); + + p = (uint32_t *)cpudata->cpussregdata->dump_vaddr; + if (per_core) { /* per-core register */ + if (cpudata->cpussregdata->core_reg_used_num == + cpudata->cpussregdata->core_reg_num) { + dev_err(dev, "Couldn't add per-core config, out of range\n"); + ret = -EINVAL; + goto err; + } + + num_cores = num_possible_cpus(); + extra_memory = reserve_size * num_cores; + used_memory = cpudata->cpussregdata->used_memory + extra_memory; + if (extra_memory / num_cores < reserve_size || + used_memory > cpudata->cpussregdata->size || + used_memory < cpudata->cpussregdata->used_memory) { + dev_err(dev, "Couldn't add per-core reg config, no enough memory\n"); + ret = -ENOMEM; + goto err; + } + + ret = config_cpuss_register(dev, p, cpudata->cpussregdata->core_reg_end_index, + cmd, register_offset, val); + if (ret) + goto err; + + if (cmd == 'r' && val == 4) + cpudata->cpussregdata->core_reg_end_index++; + else + cpudata->cpussregdata->core_reg_end_index += 2; + + cpudata->cpussregdata->core_reg_used_num++; + cpudata->cpussregdata->used_memory = used_memory; + } else { /* system register */ + system_reg_end_index = *(p + SYS_REG_INPUT_INDEX) + + cpudata->cpussregdata->sys_reg_size / sizeof(uint32_t); + + if (cmd == 'r' && reserve_size == 4) + extra_memory = sizeof(uint32_t) + reserve_size; + else + extra_memory = sizeof(uint32_t) * 2 + reserve_size; + + used_memory = cpudata->cpussregdata->used_memory + extra_memory; + if (extra_memory < reserve_size || + used_memory > cpudata->cpussregdata->size || + used_memory < cpudata->cpussregdata->used_memory) { + dev_err(dev, "Couldn't add system reg config, no enough memory\n"); + ret = -ENOMEM; + goto err; + } + + ret = config_cpuss_register(dev, p, system_reg_end_index, + cmd, register_offset, val); + if (ret) + goto err; + + if (cmd == 'r' && val == 4) { + system_reg_end_index++; + cpudata->cpussregdata->sys_reg_size += sizeof(uint32_t); + } else { + system_reg_end_index += 2; + cpudata->cpussregdata->sys_reg_size += sizeof(uint32_t) * 2; + } + + cpudata->cpussregdata->used_memory = used_memory; + *(p + system_reg_end_index) = 0x0; + *(p + OUTPUT_DUMP_INDEX) = (system_reg_end_index + 1) + * sizeof(uint32_t); + } + + ret = size; + +err: + mutex_unlock(&cpudata->mutex); + return ret; +} +static DEVICE_ATTR_RW(register_config); + +static ssize_t format_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + struct reg_dump_data *p; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + if (!cpudata) + return -EFAULT; + + mutex_lock(&cpudata->mutex); + p = (struct reg_dump_data *)cpudata->cpussregdata->dump_vaddr; + ret = scnprintf(buf, PAGE_SIZE, "%u\n", p->version); + + mutex_unlock(&cpudata->mutex); + return ret; +} +static DEVICE_ATTR_RO(format_version); +/** + * This function resets the register dump table. + */ +static ssize_t register_reset_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + unsigned int val; + + if (kstrtouint(buf, 16, &val)) + return -EINVAL; + if (val != 1) + return -EINVAL; + + update_reg_dump_table(dev, CORE_REG_NUM_DEFAULT); + + return size; +} +static DEVICE_ATTR_WO(register_reset); + +/** + * This function shows configs of per-core spr dump. + */ +static ssize_t spr_config_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + char local_buf[64]; + int len = 0, count = 0; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + int i = 0, index = 0; + uint32_t *p; + + buf[0] = '\0'; + + if (!cpudata) + return -EFAULT; + + mutex_lock(&cpudata->mutex); + + len = scnprintf(local_buf, 64, "spr data list below:\n"); + strlcat(buf, local_buf, PAGE_SIZE); + count += len; + + for (i = 0; i < MAX_CORE_NUM; i++) { + if (count > PAGE_SIZE) { + dev_err(dev, "Couldn't write complete config\n"); + break; + } + if (!cpudata->sprdata[i]) { + dev_err(dev, "SPR data pinter for CPU%d is empty\n", i); + continue; + } + p = (uint32_t *)cpudata->sprdata[i]->dump_vaddr; + len = scnprintf(local_buf, 64, "spr data for CPU[%d] below:\n", i); + strlcat(buf, local_buf, PAGE_SIZE); + count += len; + index = 0; + while (index < cpudata->sprdata[i]->sprs_data_index) { + if (count > PAGE_SIZE) { + dev_err(dev, "Couldn't write complete config\n"); + break; + } + len = scnprintf(local_buf, 64, "%d\n", *(p + SPR_DATA_HEADER_SIZE + index)); + strlcat(buf, local_buf, PAGE_SIZE); + count += len; + index++; + } + } + + mutex_unlock(&cpudata->mutex); + return count; +} + +/** + * This function sets configs for sprs dump. + */ +static ssize_t spr_config_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret = 0; + uint32_t spr_data, cpu_num; + uint32_t index; + int nval; + uint32_t *p; + u32 reserved = 0; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + nval = sscanf(buf, "%d %d", &spr_data, &cpu_num); + if (nval != 2) + return -EINVAL; + if (!cpudata) + return -EFAULT; + + if (cpu_num >= MAX_CORE_NUM) { + dev_err(dev, "Input the wrong CPU number\n"); + return -EINVAL; + } + reserved = (SPR_INPUT_DATA_SIZE + SPR_OUTPUT_DATA_SIZE) * sizeof(uint32_t); + + mutex_lock(&cpudata->mutex); + if (cpudata->sprdata[cpu_num]) { + p = (uint32_t *)cpudata->sprdata[cpu_num]->dump_vaddr; + index = cpudata->sprdata[cpu_num]->sprs_data_index; + + if (cpudata->sprdata[cpu_num]->size > + cpudata->sprdata[cpu_num]->used_memory + reserved) { + p = (uint32_t *)cpudata->sprdata[cpu_num]->dump_vaddr; + *(p + OUTPUT_DUMP_INDEX) = (SPR_DATA_HEADER_SIZE + + index + SPR_INPUT_DATA_TAIL_SIZE + 1) * sizeof(uint32_t); + *(p + SPR_DATA_HEADER_SIZE + index) = spr_data; + *(p + SPR_DATA_HEADER_SIZE + index + 1) = 0; + cpudata->sprdata[cpu_num]->sprs_data_index++; + cpudata->sprdata[cpu_num]->used_memory = + cpudata->sprdata[cpu_num]->used_memory + reserved; + } else { + dev_err(dev, "Couldn't add SPR config, no enough memory\n"); + ret = -ENOMEM; + goto err; + } + } + ret = size; + +err: + mutex_unlock(&cpudata->mutex); + return ret; +} +static DEVICE_ATTR_RW(spr_config); + +/** + * This function resets the sprs dump table. + */ +static ssize_t sprs_register_reset_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + unsigned int val; + + if (kstrtouint(buf, 16, &val)) + return -EINVAL; + if (val != 1) + return -EINVAL; + + reset_sprs_dump_table(dev); + + return size; +} +static DEVICE_ATTR_WO(sprs_register_reset); + +static ssize_t sprs_format_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + char local_buf[64]; + int len = 0, count = 0, i = 0; + struct reg_dump_data *p; + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + buf[0] = '\0'; + + if (!cpudata) + return -EFAULT; + + mutex_lock(&cpudata->mutex); + + for (i = 0; i < MAX_CORE_NUM; i++) { + if (cpudata->sprdata[i]) { + p = (struct reg_dump_data *)cpudata->sprdata[i]->dump_vaddr; + len = scnprintf(local_buf, 64, + "SPR data format version for cpu%d is %d\n", i, p->version); + strlcat(buf, local_buf, PAGE_SIZE); + count += len; + } + } + + mutex_unlock(&cpudata->mutex); + return count; +} +static DEVICE_ATTR_RO(sprs_format_version); + + +static const struct device_attribute *register_dump_attrs[] = { + &dev_attr_core_reg_num, + &dev_attr_register_config, + &dev_attr_register_reset, + &dev_attr_format_version, + NULL, +}; + +static const struct device_attribute *spr_dump_attrs[] = { + &dev_attr_spr_config, + &dev_attr_sprs_register_reset, + &dev_attr_sprs_format_version, + NULL, +}; + +static int memory_dump_create_files(struct device *dev, + const struct device_attribute **attrs) +{ + int ret = 0; + int i, j; + + for (i = 0; attrs[i] != NULL; i++) { + ret = device_create_file(dev, attrs[i]); + if (ret) { + dev_err(dev, "Couldn't create sysfs attribute: %s\n", + attrs[i]->attr.name); + for (j = 0; j < i; j++) + device_remove_file(dev, attrs[j]); + break; + } + } + return ret; +} + +static void cpuss_create_nodes(struct platform_device *pdev, + int initialized) +{ + if (initialized & PERCORE_REG_INITIALIZED) { + if (memory_dump_create_files(&pdev->dev, register_dump_attrs)) + dev_err(&pdev->dev, "Fail to create files for cpuss register dump\n"); + } + if (initialized & SPRS_INITIALIZED) { + if (memory_dump_create_files(&pdev->dev, spr_dump_attrs)) + dev_err(&pdev->dev, "Fail to create files for spr dump\n"); + } +} + +uint32_t msm_dump_table_version(void) +{ + return MSM_DUMP_TABLE_VERSION; +} +EXPORT_SYMBOL_GPL(msm_dump_table_version); + +static int msm_dump_table_register(struct msm_dump_entry *entry) +{ + struct msm_dump_entry *e; + struct msm_dump_table *table = memdump.table; + + if (!table || table->num_entries >= MAX_NUM_ENTRIES) + return -EINVAL; + + e = &table->entries[table->num_entries]; + e->id = entry->id; + e->type = MSM_DUMP_TYPE_TABLE; + e->addr = entry->addr; + table->num_entries++; + + return 0; +} + +static struct msm_dump_table *msm_dump_get_table(enum msm_dump_table_ids id) +{ + struct msm_dump_table *table = memdump.table; + int i; + unsigned long offset; + + if (!table) { + pr_err("mem dump base table does not exist\n"); + return ERR_PTR(-EINVAL); + } + + for (i = 0; i < MAX_NUM_ENTRIES; i++) { + if (table->entries[i].id == id) + break; + } + if (i == MAX_NUM_ENTRIES || !table->entries[i].addr) { + pr_err("mem dump base table entry %d invalid\n", id); + return ERR_PTR(-EINVAL); + } + + offset = table->entries[i].addr - memdump.table_phys; + /* Get the apps table pointer */ + table = (void *)memdump.table + offset; + + return table; +} + +static int register_dump_table_entry(enum msm_dump_table_ids id, + struct msm_dump_entry *entry) +{ + struct msm_dump_entry *e; + struct msm_dump_table *table; + + table = msm_dump_get_table(id); + if (IS_ERR(table)) + return PTR_ERR(table); + + if (!table || table->num_entries >= MAX_NUM_ENTRIES) + return -EINVAL; + + e = &table->entries[table->num_entries]; + e->id = entry->id; + e->type = MSM_DUMP_TYPE_DATA; + e->addr = entry->addr; + table->num_entries++; + + return 0; +} + +/** + * msm_dump_data_register_nominidump - register to dump data framework + * @id: ID of the dump table. + * @entry: dump entry to be registered + * This api will register the entry passed to dump table only + */ +int msm_dump_data_register_nominidump(enum msm_dump_table_ids id, + struct msm_dump_entry *entry) +{ + return register_dump_table_entry(id, entry); +} +EXPORT_SYMBOL_GPL(msm_dump_data_register_nominidump); + +#define MSM_DUMP_TOTAL_SIZE_OFFSET 0x724 +static int init_memdump_imem_area(const struct dump_table *table, size_t size) +{ + void __iomem *imem_base; + + imem_base = ioremap(table->imem_base, table->imem_size); + if (!imem_base) { + pr_err("mem dump base table imem offset mapping failed\n"); + return -ENOMEM; + } + + memcpy_toio(imem_base, &memdump.table_phys, + sizeof(memdump.table_phys)); + memcpy_toio(imem_base + MSM_DUMP_TOTAL_SIZE_OFFSET, + &size, sizeof(size_t)); + + /* Ensure write to imem_base is complete before unmapping */ + mb(); + pr_info("MSM Memory Dump base table set up in IMEM\n"); + + iounmap(imem_base); + return 0; +} + +static int init_memory_dump(void *dump_vaddr, phys_addr_t phys_addr) +{ + struct msm_dump_table *table; + struct msm_dump_entry entry; + int ret; + + memdump.table = dump_vaddr; + memdump.table->version = MSM_DUMP_TABLE_VERSION; + memdump.table_phys = phys_addr; + dump_vaddr += sizeof(*table); + phys_addr += sizeof(*table); + table = dump_vaddr; + table->version = MSM_DUMP_TABLE_VERSION; + entry.id = MSM_DUMP_TABLE_APPS; + entry.addr = phys_addr; + ret = msm_dump_table_register(&entry); + if (ret) { + pr_err("mem dump apps data table register failed\n"); + return ret; + } + pr_info("MSM Memory Dump apps data table set up\n"); + + return 0; +} + +static int cpuss_regdump_init(struct device *dev, + void *dump_vaddr, u32 size) +{ + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + + cpudata->cpussregdata = devm_kzalloc(dev, + sizeof(struct cpuss_regdump_data), GFP_KERNEL); + + if (cpudata->cpussregdata) { + cpudata->cpussregdata->dump_vaddr = dump_vaddr; + cpudata->cpussregdata->size = size; + return 0; + } + return -ENOMEM; +} + +static int sprs_dump_init(struct device *dev, + void *dump_vaddr, u32 size, u32 id) +{ + struct cpuss_dump_data *cpudata = dev_get_drvdata(dev); + int core_num = 0; + + core_num = id - SPR_DUMP_CPU0; + + cpudata->sprdata[core_num] = devm_kzalloc(dev, + sizeof(struct sprs_dump_data), GFP_KERNEL); + if (cpudata->sprdata[core_num]) { + cpudata->sprdata[core_num]->dump_vaddr = dump_vaddr; + cpudata->sprdata[core_num]->size = size; + return 0; + } + return -ENOMEM; +} + +static int cpuss_dump_init(struct platform_device *pdev, + void *dump_vaddr, u32 size, u32 id) +{ + struct cpuss_dump_data *cpudata = dev_get_drvdata(&pdev->dev); + static int initialized; + + if (!cpudata) { + cpudata = devm_kzalloc(&pdev->dev, + sizeof(struct cpuss_dump_data), GFP_KERNEL); + if (cpudata) { + mutex_init(&cpudata->mutex); + platform_set_drvdata(pdev, cpudata); + } else + return initialized; + } + + if (id == CPUSS_REGDUMP) { + if (!cpuss_regdump_init(&pdev->dev, dump_vaddr, size)) + initialized |= PERCORE_REG_INITIALIZED; + } else { + if (!sprs_dump_init(&pdev->dev, dump_vaddr, size, id)) + initialized |= SPRS_INITIALIZED; + } + + return initialized; +} + +struct cma *memdump_cma; +void __init reserve_memdump_cma(void) +{ + unsigned long long cma_size = 0x3000000; + unsigned long long request_size = roundup(cma_size, PAGE_SIZE); + + if (cma_declare_contiguous(0, request_size, 0, 0, 0, false, + "memdump", &memdump_cma)) { + pr_warn("memdump CMA reservation failed\n"); + } +} + +#define MSM_DUMP_DATA_SIZE sizeof(struct msm_dump_data) +static int mem_dump_alloc(struct platform_device *pdev) +{ + struct msm_dump_data *dump_data; + struct msm_dump_entry dump_entry; + size_t total_size; + u32 size, id; + int i, ret, no_of_nodes; + phys_addr_t phys_addr; + void *dump_vaddr; + u64 shm_bridge_handle; + int initialized = 0; + const struct dump_table *table = dev_get_platdata(&pdev->dev); + struct page *reserved_page; + + total_size = size = ret = no_of_nodes = 0; + /* For dump table registration with IMEM */ + total_size = sizeof(struct msm_dump_table) * 2; + for (i = 0; i < table->num_of_items; i++) { + total_size += table->items[i].size; + no_of_nodes++; + } + + total_size += (MSM_DUMP_DATA_SIZE * no_of_nodes); + total_size = ALIGN(total_size, SZ_4K); + reserved_page = cma_alloc(memdump_cma, total_size >> PAGE_SHIFT, + 0, false); + if (!reserved_page) + return -ENOMEM; + phys_addr = page_to_phys(reserved_page); + dump_vaddr = page_to_virt(reserved_page); + + memset(dump_vaddr, 0x0, total_size); + ret = qcom_tzmem_shm_bridge_create(phys_addr, total_size, &shm_bridge_handle); + if (ret) { + dev_err(&pdev->dev, "Failed to create shm bridge.ret=%d\n", ret); + return ret; + } + + ret = init_memory_dump(dump_vaddr, phys_addr); + if (ret) { + dev_err(&pdev->dev, "Memory Dump table set up is failed\n"); + qcom_tzmem_shm_bridge_delete(shm_bridge_handle); + return ret; + } + + ret = init_memdump_imem_area(table, total_size); + if (ret) { + qcom_tzmem_shm_bridge_delete(shm_bridge_handle); + return ret; + } + + dump_vaddr += (sizeof(struct msm_dump_table) * 2); + phys_addr += (sizeof(struct msm_dump_table) * 2); + for (i = 0; i < table->num_of_items; i++) { + size = table->items[i].size; + id = table->items[i].dump_id; + dump_data = dump_vaddr; + dump_data->addr = phys_addr + MSM_DUMP_DATA_SIZE; + dump_data->len = size; + dump_entry.id = id; + strscpy(dump_data->name, table->items[i].name, + sizeof(table->items[i].name)); + dump_entry.addr = phys_addr; + ret = msm_dump_data_register_nominidump(MSM_DUMP_TABLE_APPS, + &dump_entry); + if (ret) + dev_err(&pdev->dev, "Data dump setup failed, id = %d\n", + id); + + if ((id == CPUSS_REGDUMP) || + ((id >= SPR_DUMP_CPU0) && (id <= SPR_DUMP_CPU7))) + initialized = cpuss_dump_init(pdev, + (dump_vaddr + MSM_DUMP_DATA_SIZE), size, id); + + dump_vaddr += (size + MSM_DUMP_DATA_SIZE); + phys_addr += (size + MSM_DUMP_DATA_SIZE); + } + + cpuss_create_nodes(pdev, initialized); + + if (initialized & SPRS_INITIALIZED) + reset_sprs_dump_table(&pdev->dev); + + return ret; +} + +static int mem_dump_probe(struct platform_device *pdev) +{ + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret < 0) + return ret; + + ret = mem_dump_alloc(pdev); + return ret; +} + +static struct platform_driver mem_dump_driver = { + .probe = mem_dump_probe, + .driver = { + .name = "msm_mem_dump", + }, +}; + +module_platform_driver(mem_dump_driver); + +MODULE_DESCRIPTION("Memory Dump V2 Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c index e777b7cb9b127..18ca9a22ffe99 100644 --- a/drivers/firmware/qcom/qcom_scm.c +++ b/drivers/firmware/qcom/qcom_scm.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -111,6 +112,8 @@ enum qcom_scm_qseecom_tz_cmd_info { QSEECOM_TZ_CMD_INFO_VERSION = 3, }; +#define RSCTABLE_BUFFER_NOT_SUFFICIENT 20 + #define QSEECOM_MAX_APP_NAME_SIZE 64 #define SHMBRIDGE_RESULT_NOTSUPP 4 @@ -558,16 +561,105 @@ static void qcom_scm_set_download_mode(u32 dload_mode) dev_err(__scm->dev, "failed to set download mode: %d\n", ret); } +/** + * devm_qcom_scm_pas_context_alloc() - Allocate peripheral authentication service + * context for a given peripheral + * + * PAS context is device-resource managed, so the caller does not need + * to worry about freeing the context memory. + * + * @dev: PAS firmware device + * @pas_id: peripheral authentication service id + * @mem_phys: Subsystem reserve memory start address + * @mem_size: Subsystem reserve memory size + * + * Returns: The new PAS context, or ERR_PTR() on failure. + */ +struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev, + u32 pas_id, + phys_addr_t mem_phys, + size_t mem_size) +{ + struct qcom_scm_pas_context *ctx; + + ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + ctx->dev = dev; + ctx->pas_id = pas_id; + ctx->mem_phys = mem_phys; + ctx->mem_size = mem_size; + + return ctx; +} +EXPORT_SYMBOL_GPL(devm_qcom_scm_pas_context_alloc); + +static int __qcom_scm_pas_init_image(u32 pas_id, dma_addr_t mdata_phys, + struct qcom_scm_res *res) +{ + struct qcom_scm_desc desc = { + .svc = QCOM_SCM_SVC_PIL, + .cmd = QCOM_SCM_PIL_PAS_INIT_IMAGE, + .arginfo = QCOM_SCM_ARGS(2, QCOM_SCM_VAL, QCOM_SCM_RW), + .args[0] = pas_id, + .owner = ARM_SMCCC_OWNER_SIP, + }; + int ret; + + ret = qcom_scm_clk_enable(); + if (ret) + return ret; + + ret = qcom_scm_bw_enable(); + if (ret) + goto disable_clk; + + desc.args[1] = mdata_phys; + + ret = qcom_scm_call(__scm->dev, &desc, res); + qcom_scm_bw_disable(); + +disable_clk: + qcom_scm_clk_disable(); + + return ret; +} + +static int qcom_scm_pas_prep_and_init_image(struct qcom_scm_pas_context *ctx, + const void *metadata, size_t size) +{ + struct qcom_scm_res res; + phys_addr_t mdata_phys; + void *mdata_buf; + int ret; + + mdata_buf = qcom_tzmem_alloc(__scm->mempool, size, GFP_KERNEL); + if (!mdata_buf) + return -ENOMEM; + + memcpy(mdata_buf, metadata, size); + mdata_phys = qcom_tzmem_to_phys(mdata_buf); + + ret = __qcom_scm_pas_init_image(ctx->pas_id, mdata_phys, &res); + if (ret < 0) + qcom_tzmem_free(mdata_buf); + else + ctx->ptr = mdata_buf; + + return ret ? : res.result[0]; +} + /** * qcom_scm_pas_init_image() - Initialize peripheral authentication service * state machine for a given peripheral, using the * metadata - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * @metadata: pointer to memory containing ELF header, program header table * and optional blob of data used for authenticating the metadata * and the rest of the firmware * @size: size of the metadata - * @ctx: optional metadata context + * @ctx: optional pas context * * Return: 0 on success. * @@ -575,20 +667,16 @@ static void qcom_scm_set_download_mode(u32 dload_mode) * track the metadata allocation, this needs to be released by invoking * qcom_scm_pas_metadata_release() by the caller. */ -int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, - struct qcom_scm_pas_metadata *ctx) +int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size, + struct qcom_scm_pas_context *ctx) { + struct qcom_scm_res res; dma_addr_t mdata_phys; void *mdata_buf; int ret; - struct qcom_scm_desc desc = { - .svc = QCOM_SCM_SVC_PIL, - .cmd = QCOM_SCM_PIL_PAS_INIT_IMAGE, - .arginfo = QCOM_SCM_ARGS(2, QCOM_SCM_VAL, QCOM_SCM_RW), - .args[0] = peripheral, - .owner = ARM_SMCCC_OWNER_SIP, - }; - struct qcom_scm_res res; + + if (ctx && ctx->use_tzmem) + return qcom_scm_pas_prep_and_init_image(ctx, metadata, size); /* * During the scm call memory protection will be enabled for the meta @@ -609,23 +697,7 @@ int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, memcpy(mdata_buf, metadata, size); - ret = qcom_scm_clk_enable(); - if (ret) - goto out; - - ret = qcom_scm_bw_enable(); - if (ret) - goto disable_clk; - - desc.args[1] = mdata_phys; - - ret = qcom_scm_call(__scm->dev, &desc, &res); - qcom_scm_bw_disable(); - -disable_clk: - qcom_scm_clk_disable(); - -out: + ret = __qcom_scm_pas_init_image(pas_id, mdata_phys, &res); if (ret < 0 || !ctx) { dma_free_coherent(__scm->dev, size, mdata_buf, mdata_phys); } else if (ctx) { @@ -640,38 +712,39 @@ EXPORT_SYMBOL_GPL(qcom_scm_pas_init_image); /** * qcom_scm_pas_metadata_release() - release metadata context - * @ctx: metadata context + * @ctx: pas context */ -void qcom_scm_pas_metadata_release(struct qcom_scm_pas_metadata *ctx) +void qcom_scm_pas_metadata_release(struct qcom_scm_pas_context *ctx) { if (!ctx->ptr) return; - dma_free_coherent(__scm->dev, ctx->size, ctx->ptr, ctx->phys); + if (ctx->use_tzmem) + qcom_tzmem_free(ctx->ptr); + else + dma_free_coherent(__scm->dev, ctx->size, ctx->ptr, ctx->phys); ctx->ptr = NULL; - ctx->phys = 0; - ctx->size = 0; } EXPORT_SYMBOL_GPL(qcom_scm_pas_metadata_release); /** * qcom_scm_pas_mem_setup() - Prepare the memory related to a given peripheral * for firmware loading - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * @addr: start address of memory area to prepare * @size: size of the memory area to prepare * * Returns 0 on success. */ -int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size) +int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_MEM_SETUP, .arginfo = QCOM_SCM_ARGS(3), - .args[0] = peripheral, + .args[0] = pas_id, .args[1] = addr, .args[2] = size, .owner = ARM_SMCCC_OWNER_SIP, @@ -696,21 +769,189 @@ int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size) } EXPORT_SYMBOL_GPL(qcom_scm_pas_mem_setup); +static void *__qcom_scm_pas_get_rsc_table(u32 pas_id, void *input_rt_tzm, + size_t input_rt_size, + size_t *output_rt_size) +{ + struct qcom_scm_desc desc = { + .svc = QCOM_SCM_SVC_PIL, + .cmd = QCOM_SCM_PIL_PAS_GET_RSCTABLE, + .arginfo = QCOM_SCM_ARGS(5, QCOM_SCM_VAL, QCOM_SCM_RO, QCOM_SCM_VAL, + QCOM_SCM_RW, QCOM_SCM_VAL), + .args[0] = pas_id, + .owner = ARM_SMCCC_OWNER_SIP, + }; + struct qcom_scm_res res; + void *output_rt_tzm; + int ret; + + output_rt_tzm = qcom_tzmem_alloc(__scm->mempool, *output_rt_size, GFP_KERNEL); + if (!output_rt_tzm) + return ERR_PTR(-ENOMEM); + + desc.args[1] = qcom_tzmem_to_phys(input_rt_tzm); + desc.args[2] = input_rt_size; + desc.args[3] = qcom_tzmem_to_phys(output_rt_tzm); + desc.args[4] = *output_rt_size; + + /* + * Whether SMC fail or pass, res.result[2] will hold actual resource table + * size. + * + * If passed 'output_rt_size' buffer size is not sufficient to hold the + * resource table TrustZone sends, response code in res.result[1] as + * RSCTABLE_BUFFER_NOT_SUFFICIENT so that caller can retry this SMC call + * with output_rt_tzm buffer with res.result[2] size however, It should not + * be of unresonable size. + */ + ret = qcom_scm_call(__scm->dev, &desc, &res); + if (!ret && res.result[2] > SZ_1G) { + ret = -E2BIG; + goto free_output_rt; + } + + *output_rt_size = res.result[2]; + if (ret && res.result[1] == RSCTABLE_BUFFER_NOT_SUFFICIENT) + ret = -EOVERFLOW; + +free_output_rt: + if (ret) + qcom_tzmem_free(output_rt_tzm); + + return ret ? ERR_PTR(ret) : output_rt_tzm; +} + +/** + * qcom_scm_pas_get_rsc_table() - Retrieve the resource table in passed output buffer + * for a given peripheral. + * + * Qualcomm remote processor may rely on both static and dynamic resources for + * its functionality. Static resources typically refer to memory-mapped addresses + * required by the subsystem and are often embedded within the firmware binary + * and dynamic resources, such as shared memory in DDR etc., are determined at + * runtime during the boot process. + * + * On Qualcomm Technologies devices, it's possible that static resources are not + * embedded in the firmware binary and instead are provided by TrustZone However, + * dynamic resources are always expected to come from TrustZone. This indicates + * that for Qualcomm devices, all resources (static and dynamic) will be provided + * by TrustZone via the SMC call. + * + * If the remote processor firmware binary does contain static resources, they + * should be passed in input_rt. These will be forwarded to TrustZone for + * authentication. TrustZone will then append the dynamic resources and return + * the complete resource table in output_rt_tzm. + * + * If the remote processor firmware binary does not include a resource table, + * the caller of this function should set input_rt as NULL and input_rt_size + * as zero respectively. + * + * More about documentation on resource table data structures can be found in + * include/linux/remoteproc.h + * + * @ctx: PAS context + * @pas_id: peripheral authentication service id + * @input_rt: resource table buffer which is present in firmware binary + * @input_rt_size: size of the resource table present in firmware binary + * @output_rt_size: TrustZone expects caller should pass worst case size for + * the output_rt_tzm. + * + * Return: + * On success, returns a pointer to the allocated buffer containing the final + * resource table and output_rt_size will have actual resource table size from + * TrustZone. The caller is responsible for freeing the buffer. On failure, + * returns ERR_PTR(-errno). + */ +struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *ctx, + void *input_rt, + size_t input_rt_size, + size_t *output_rt_size) +{ + struct resource_table empty_rsc = {}; + size_t size = SZ_16K; + void *output_rt_tzm; + void *input_rt_tzm; + void *tbl_ptr; + int ret; + + ret = qcom_scm_clk_enable(); + if (ret) + return ERR_PTR(ret); + + ret = qcom_scm_bw_enable(); + if (ret) + goto disable_clk; + + /* + * TrustZone can not accept buffer as NULL value as argument hence, + * we need to pass a input buffer indicating that subsystem firmware + * does not have resource table by filling resource table structure. + */ + if (!input_rt) { + input_rt = &empty_rsc; + input_rt_size = sizeof(empty_rsc); + } + + input_rt_tzm = qcom_tzmem_alloc(__scm->mempool, input_rt_size, GFP_KERNEL); + if (!input_rt_tzm) { + ret = -ENOMEM; + goto disable_scm_bw; + } + + memcpy(input_rt_tzm, input_rt, input_rt_size); + + output_rt_tzm = __qcom_scm_pas_get_rsc_table(ctx->pas_id, input_rt_tzm, + input_rt_size, &size); + if (PTR_ERR(output_rt_tzm) == -EOVERFLOW) + /* Try again with the size requested by the TZ */ + output_rt_tzm = __qcom_scm_pas_get_rsc_table(ctx->pas_id, + input_rt_tzm, + input_rt_size, + &size); + if (IS_ERR(output_rt_tzm)) { + ret = PTR_ERR(output_rt_tzm); + goto free_input_rt; + } + + tbl_ptr = kzalloc(size, GFP_KERNEL); + if (!tbl_ptr) { + qcom_tzmem_free(output_rt_tzm); + ret = -ENOMEM; + goto free_input_rt; + } + + memcpy(tbl_ptr, output_rt_tzm, size); + *output_rt_size = size; + qcom_tzmem_free(output_rt_tzm); + +free_input_rt: + qcom_tzmem_free(input_rt_tzm); + +disable_scm_bw: + qcom_scm_bw_disable(); + +disable_clk: + qcom_scm_clk_disable(); + + return ret ? ERR_PTR(ret) : tbl_ptr; +} +EXPORT_SYMBOL_GPL(qcom_scm_pas_get_rsc_table); + /** * qcom_scm_pas_auth_and_reset() - Authenticate the given peripheral firmware * and reset the remote processor - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * * Return 0 on success. */ -int qcom_scm_pas_auth_and_reset(u32 peripheral) +int qcom_scm_pas_auth_and_reset(u32 pas_id) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_AUTH_AND_RESET, .arginfo = QCOM_SCM_ARGS(1), - .args[0] = peripheral, + .args[0] = pas_id, .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; @@ -733,20 +974,67 @@ int qcom_scm_pas_auth_and_reset(u32 peripheral) } EXPORT_SYMBOL_GPL(qcom_scm_pas_auth_and_reset); +/** + * qcom_scm_pas_prepare_and_auth_reset() - Prepare, authenticate, and reset the + * remote processor + * + * @ctx: Context saved during call to qcom_scm_pas_context_init() + * + * This function performs the necessary steps to prepare a PAS subsystem, + * authenticate it using the provided metadata, and initiate a reset sequence. + * + * It should be used when Linux is in control setting up the IOMMU hardware + * for remote subsystem during secure firmware loading processes. The preparation + * step sets up a shmbridge over the firmware memory before TrustZone accesses the + * firmware memory region for authentication. The authentication step verifies + * the integrity and authenticity of the firmware or configuration using secure + * metadata. Finally, the reset step ensures the subsystem starts in a clean and + * sane state. + * + * Return: 0 on success, negative errno on failure. + */ +int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx) +{ + u64 handle; + int ret; + + /* + * When Linux running @ EL1, Gunyah hypervisor running @ EL2 traps the + * auth_and_reset call and create an shmbridge on the remote subsystem + * memory region and then invokes a call to TrustZone to authenticate. + */ + if (!ctx->use_tzmem) + return qcom_scm_pas_auth_and_reset(ctx->pas_id); + + /* + * When Linux runs @ EL2 Linux must create the shmbridge itself and then + * subsequently call TrustZone for authenticate and reset. + */ + ret = qcom_tzmem_shm_bridge_create(ctx->mem_phys, ctx->mem_size, &handle); + if (ret) + return ret; + + ret = qcom_scm_pas_auth_and_reset(ctx->pas_id); + qcom_tzmem_shm_bridge_delete(handle); + + return ret; +} +EXPORT_SYMBOL_GPL(qcom_scm_pas_prepare_and_auth_reset); + /** * qcom_scm_pas_shutdown() - Shut down the remote processor - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * * Returns 0 on success. */ -int qcom_scm_pas_shutdown(u32 peripheral) +int qcom_scm_pas_shutdown(u32 pas_id) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_SHUTDOWN, .arginfo = QCOM_SCM_ARGS(1), - .args[0] = peripheral, + .args[0] = pas_id, .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; @@ -772,18 +1060,18 @@ EXPORT_SYMBOL_GPL(qcom_scm_pas_shutdown); /** * qcom_scm_pas_supported() - Check if the peripheral authentication service is * available for the given peripherial - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * * Returns true if PAS is supported for this peripheral, otherwise false. */ -bool qcom_scm_pas_supported(u32 peripheral) +bool qcom_scm_pas_supported(u32 pas_id) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_IS_SUPPORTED, .arginfo = QCOM_SCM_ARGS(1), - .args[0] = peripheral, + .args[0] = pas_id, .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; @@ -2010,6 +2298,7 @@ static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = { { .compatible = "microsoft,romulus13", }, { .compatible = "microsoft,romulus15", }, { .compatible = "qcom,hamoa-iot-evk" }, + { .compatible = "qcom,purwa-iot-evk" }, { .compatible = "qcom,sc8180x-primus" }, { .compatible = "qcom,x1e001de-devkit" }, { .compatible = "qcom,x1e80100-crd" }, @@ -2182,6 +2471,56 @@ int qcom_scm_qtee_callback_response(phys_addr_t buf, size_t buf_size, } EXPORT_SYMBOL(qcom_scm_qtee_callback_response); +static void qcom_scm_gunyah_wdt_free(void *data) +{ + struct platform_device *gunyah_wdt_dev = data; + + platform_device_unregister(gunyah_wdt_dev); +} + +static void qcom_scm_gunyah_wdt_init(struct qcom_scm *scm) +{ + struct platform_device *gunyah_wdt_dev; + struct device_node *np; + bool of_wdt_available; + int i; + static const uuid_t gunyah_uuid = UUID_INIT(0xc1d58fcd, 0xa453, 0x5fdb, + 0x92, 0x65, 0xce, 0x36, + 0x67, 0x3d, 0x5f, 0x14); + static const char * const of_wdt_compatible[] = { + "qcom,kpss-wdt", + "arm,sbsa-gwdt", + }; + + /* Bail out if we are not running under Gunyah */ + if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC_DISCOVERY) || + !arm_smccc_hypervisor_has_uuid(&gunyah_uuid)) + return; + + /* + * Gunyah emulates either of Qualcomm watchdog or ARM SBSA watchdog on + * newer platforms. Bail out if we find them in the devicetree. + */ + for (i = 0; i < ARRAY_SIZE(of_wdt_compatible); i++) { + np = of_find_compatible_node(NULL, NULL, of_wdt_compatible[i]); + of_wdt_available = of_device_is_available(np); + of_node_put(np); + if (of_wdt_available) + return; + } + + gunyah_wdt_dev = platform_device_register_simple("gunyah-wdt", -1, + NULL, 0); + if (IS_ERR(gunyah_wdt_dev)) { + dev_err(scm->dev, "Failed to register Gunyah watchdog device: %ld\n", + PTR_ERR(gunyah_wdt_dev)); + return; + } + + devm_add_action_or_reset(scm->dev, qcom_scm_gunyah_wdt_free, + gunyah_wdt_dev); +} + static void qcom_scm_qtee_free(void *data) { struct platform_device *qtee_dev = data; @@ -2223,6 +2562,51 @@ bool qcom_scm_is_available(void) } EXPORT_SYMBOL_GPL(qcom_scm_is_available); +int qcom_scm_camera_update_camnoc_qos(uint32_t use_case_id, + uint32_t cam_qos_cnt, struct qcom_scm_camera_qos *cam_qos) +{ + int ret; + dma_addr_t payload_phys; + u32 *payload_buf = NULL; + u32 payload_size = 0; + + if ((cam_qos_cnt > QCOM_SCM_CAMERA_MAX_QOS_CNT) || (cam_qos_cnt && !cam_qos)) { + pr_err("Invalid input SmartQoS count: %d\n", cam_qos_cnt); + return -EINVAL; + } + + struct qcom_scm_desc desc = { + .svc = QCOM_SCM_SVC_CAMERA, + .cmd = QCOM_SCM_CAMERA_UPDATE_CAMNOC_QOS, + .owner = ARM_SMCCC_OWNER_SIP, + .args[0] = use_case_id, + .args[2] = payload_size, + .arginfo = QCOM_SCM_ARGS(3, QCOM_SCM_VAL, QCOM_SCM_RW, QCOM_SCM_VAL), + }; + + payload_size = cam_qos_cnt * sizeof(struct qcom_scm_camera_qos); + + /* fill all required qos settings */ + if (use_case_id && payload_size && cam_qos) { + payload_buf = dma_alloc_coherent(__scm->dev, + payload_size, &payload_phys, GFP_KERNEL); + if (!payload_buf) + return -ENOMEM; + + memcpy(payload_buf, cam_qos, payload_size); + desc.args[1] = payload_phys; + desc.args[2] = payload_size; + + } + ret = qcom_scm_call(__scm->dev, &desc, NULL); + + if (payload_buf) + dma_free_coherent(__scm->dev, payload_size, payload_buf, payload_phys); + + return ret; +} +EXPORT_SYMBOL_GPL(qcom_scm_camera_update_camnoc_qos); + static int qcom_scm_assert_valid_wq_ctx(u32 wq_ctx) { /* FW currently only supports a single wq_ctx (zero). @@ -2448,6 +2832,9 @@ static int qcom_scm_probe(struct platform_device *pdev) /* Initialize the QTEE object interface. */ qcom_scm_qtee_init(scm); + /* Initialize the Gunyah watchdog platform device. */ + qcom_scm_gunyah_wdt_init(scm); + return 0; } diff --git a/drivers/firmware/qcom/qcom_scm.h b/drivers/firmware/qcom/qcom_scm.h index a56c8212cc0c4..de7e8ce6a7b21 100644 --- a/drivers/firmware/qcom/qcom_scm.h +++ b/drivers/firmware/qcom/qcom_scm.h @@ -105,6 +105,7 @@ int qcom_scm_shm_bridge_enable(struct device *scm_dev); #define QCOM_SCM_PIL_PAS_SHUTDOWN 0x06 #define QCOM_SCM_PIL_PAS_IS_SUPPORTED 0x07 #define QCOM_SCM_PIL_PAS_MSS_RESET 0x0a +#define QCOM_SCM_PIL_PAS_GET_RSCTABLE 0x21 #define QCOM_SCM_SVC_IO 0x05 #define QCOM_SCM_IO_READ 0x01 @@ -173,6 +174,9 @@ int qcom_scm_shm_bridge_enable(struct device *scm_dev); #define QCOM_SCM_INTERRUPTED 1 #define QCOM_SCM_WAITQ_SLEEP 2 +#define QCOM_SCM_SVC_CAMERA 0x18 +#define QCOM_SCM_CAMERA_UPDATE_CAMNOC_QOS 0xA + static inline int qcom_scm_remap_error(int err) { switch (err) { diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 00f58e27f6de5..1d24e82f3d09d 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2017-2018, Intel Corporation + * Copyright (C) 2025, Altera Corporation */ #include @@ -52,6 +53,7 @@ struct stratix10_svc_chan; /** * struct stratix10_svc - svc private data * @stratix10_svc_rsu: pointer to stratix10 RSU device + * @intel_svc_fcs: pointer to the FCS device */ struct stratix10_svc { struct platform_device *stratix10_svc_rsu; @@ -174,6 +176,12 @@ struct stratix10_svc_chan { static LIST_HEAD(svc_ctrl); static LIST_HEAD(svc_data_mem); +/** + * svc_mem_lock protects access to the svc_data_mem list for + * concurrent multi-client operations + */ +static DEFINE_MUTEX(svc_mem_lock); + /** * svc_pa_to_va() - translate physical address to virtual address * @addr: to be translated physical address @@ -186,6 +194,7 @@ static void *svc_pa_to_va(unsigned long addr) struct stratix10_svc_data_mem *pmem; pr_debug("claim back P-addr=0x%016x\n", (unsigned int)addr); + guard(mutex)(&svc_mem_lock); list_for_each_entry(pmem, &svc_data_mem, node) if (pmem->paddr == addr) return pmem->vaddr; @@ -992,6 +1001,7 @@ int stratix10_svc_send(struct stratix10_svc_chan *chan, void *msg) p_data->flag = ct->flags; } } else { + guard(mutex)(&svc_mem_lock); list_for_each_entry(p_mem, &svc_data_mem, node) if (p_mem->vaddr == p_msg->payload) { p_data->paddr = p_mem->paddr; @@ -1074,6 +1084,7 @@ void *stratix10_svc_allocate_memory(struct stratix10_svc_chan *chan, if (!pmem) return ERR_PTR(-ENOMEM); + guard(mutex)(&svc_mem_lock); va = gen_pool_alloc(genpool, s); if (!va) return ERR_PTR(-ENOMEM); @@ -1102,6 +1113,7 @@ EXPORT_SYMBOL_GPL(stratix10_svc_allocate_memory); void stratix10_svc_free_memory(struct stratix10_svc_chan *chan, void *kaddr) { struct stratix10_svc_data_mem *pmem; + guard(mutex)(&svc_mem_lock); list_for_each_entry(pmem, &svc_data_mem, node) if (pmem->vaddr == kaddr) { diff --git a/drivers/firmware/thead,th1520-aon.c b/drivers/firmware/thead,th1520-aon.c index 38f812ac9920e..b87d4e8235b12 100644 --- a/drivers/firmware/thead,th1520-aon.c +++ b/drivers/firmware/thead,th1520-aon.c @@ -170,10 +170,9 @@ int th1520_aon_power_update(struct th1520_aon_chan *aon_chan, u16 rsrc, hdr->func = TH1520_AON_PM_FUNC_SET_RESOURCE_POWER_MODE; hdr->size = TH1520_AON_RPC_MSG_NUM; - RPC_SET_BE16(&msg.resource, 0, rsrc); - RPC_SET_BE16(&msg.resource, 2, - (power_on ? TH1520_AON_PM_PW_MODE_ON : - TH1520_AON_PM_PW_MODE_OFF)); + msg.resource = cpu_to_be16(rsrc); + msg.mode = cpu_to_be16(power_on ? TH1520_AON_PM_PW_MODE_ON : + TH1520_AON_PM_PW_MODE_OFF); ret = th1520_aon_call_rpc(aon_chan, &msg); if (ret) diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c index 49fd2ae01055d..8d96a3c12b36a 100644 --- a/drivers/firmware/ti_sci.c +++ b/drivers/firmware/ti_sci.c @@ -3751,9 +3751,11 @@ static int __maybe_unused ti_sci_suspend_noirq(struct device *dev) struct ti_sci_info *info = dev_get_drvdata(dev); int ret = 0; - ret = ti_sci_cmd_set_io_isolation(&info->handle, TISCI_MSG_VALUE_IO_ENABLE); - if (ret) - return ret; + if (info->fw_caps & MSG_FLAG_CAPS_IO_ISOLATION) { + ret = ti_sci_cmd_set_io_isolation(&info->handle, TISCI_MSG_VALUE_IO_ENABLE); + if (ret) + return ret; + } return 0; } @@ -3767,9 +3769,11 @@ static int __maybe_unused ti_sci_resume_noirq(struct device *dev) u8 pin; u8 mode; - ret = ti_sci_cmd_set_io_isolation(&info->handle, TISCI_MSG_VALUE_IO_DISABLE); - if (ret) - return ret; + if (info->fw_caps & MSG_FLAG_CAPS_IO_ISOLATION) { + ret = ti_sci_cmd_set_io_isolation(&info->handle, TISCI_MSG_VALUE_IO_DISABLE); + if (ret) + return ret; + } ret = ti_sci_msg_cmd_lpm_wake_reason(&info->handle, &source, &time, &pin, &mode); /* Do not fail to resume on error as the wake reason is not critical */ @@ -3928,11 +3932,12 @@ static int ti_sci_probe(struct platform_device *pdev) } ti_sci_msg_cmd_query_fw_caps(&info->handle, &info->fw_caps); - dev_dbg(dev, "Detected firmware capabilities: %s%s%s%s\n", + dev_dbg(dev, "Detected firmware capabilities: %s%s%s%s%s\n", info->fw_caps & MSG_FLAG_CAPS_GENERIC ? "Generic" : "", info->fw_caps & MSG_FLAG_CAPS_LPM_PARTIAL_IO ? " Partial-IO" : "", info->fw_caps & MSG_FLAG_CAPS_LPM_DM_MANAGED ? " DM-Managed" : "", - info->fw_caps & MSG_FLAG_CAPS_LPM_ABORT ? " LPM-Abort" : "" + info->fw_caps & MSG_FLAG_CAPS_LPM_ABORT ? " LPM-Abort" : "", + info->fw_caps & MSG_FLAG_CAPS_IO_ISOLATION ? " IO-Isolation" : "" ); ti_sci_setup_ops(info); diff --git a/drivers/firmware/ti_sci.h b/drivers/firmware/ti_sci.h index 701c416b2e78f..7559cde17b6cc 100644 --- a/drivers/firmware/ti_sci.h +++ b/drivers/firmware/ti_sci.h @@ -149,6 +149,7 @@ struct ti_sci_msg_req_reboot { * MSG_FLAG_CAPS_LPM_PARTIAL_IO: Partial IO in LPM * MSG_FLAG_CAPS_LPM_DM_MANAGED: LPM can be managed by DM * MSG_FLAG_CAPS_LPM_ABORT: Abort entry to LPM + * MSG_FLAG_CAPS_IO_ISOLATION: IO Isolation support * * Response to a generic message with message type TI_SCI_MSG_QUERY_FW_CAPS * providing currently available SOC/firmware capabilities. SoC that don't @@ -160,6 +161,7 @@ struct ti_sci_msg_resp_query_fw_caps { #define MSG_FLAG_CAPS_LPM_PARTIAL_IO TI_SCI_MSG_FLAG(4) #define MSG_FLAG_CAPS_LPM_DM_MANAGED TI_SCI_MSG_FLAG(5) #define MSG_FLAG_CAPS_LPM_ABORT TI_SCI_MSG_FLAG(9) +#define MSG_FLAG_CAPS_IO_ISOLATION TI_SCI_MSG_FLAG(7) #define MSG_MASK_CAPS_LPM GENMASK_ULL(4, 1) u64 fw_caps; } __packed; diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index 7022657243c0a..449c3a082e232 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -2018,7 +2018,7 @@ static void __exit dfl_fpga_exit(void) bus_unregister(&dfl_bus_type); } -module_init(dfl_fpga_init); +subsys_initcall(dfl_fpga_init); module_exit(dfl_fpga_exit); MODULE_DESCRIPTION("FPGA Device Feature List (DFL) Support"); diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c index 43db4bb77138a..caa091224dc54 100644 --- a/drivers/fpga/of-fpga-region.c +++ b/drivers/fpga/of-fpga-region.c @@ -83,7 +83,7 @@ static struct fpga_manager *of_fpga_region_get_mgr(struct device_node *np) * done with the bridges. * * Return: 0 for success (even if there are no bridges specified) - * or -EBUSY if any of the bridges are in use. + * or an error code if any of the bridges are not available. */ static int of_fpga_region_get_bridges(struct fpga_region *region) { @@ -130,10 +130,10 @@ static int of_fpga_region_get_bridges(struct fpga_region *region) ®ion->bridge_list); of_node_put(br); - /* If any of the bridges are in use, give up */ - if (ret == -EBUSY) { + /* If any of the bridges are not available, give up */ + if (ret) { fpga_bridges_put(®ion->bridge_list); - return -EBUSY; + return ret; } } diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 7ee3afbc2b05d..4d644dcecad93 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -738,7 +738,6 @@ config GPIO_TB10X depends on ARC_PLAT_TB10X || COMPILE_TEST select GPIO_GENERIC select GENERIC_IRQ_CHIP - select OF_GPIO config GPIO_TEGRA tristate "NVIDIA Tegra GPIO support" @@ -1195,11 +1194,11 @@ config GPIO_PCA953X 8 bits: max7310, max7315, pca6107, pca9534, pca9538, pca9554, pca9556, pca9557, pca9574, tca6408, tca9554, xra1202, - pcal6408, pcal9554b, tca9538 + pcal6408, pcal9554b, tca9538, tcal6408 16 bits: max7312, max7313, pca9535, pca9539, pca9555, pca9575, tca6416, pca6416, pcal6416, pcal9535, pcal9555a, max7318, - tca9539 + tca9539, tcal6416 18 bits: tca6418 diff --git a/drivers/gpio/gpio-amd-fch.c b/drivers/gpio/gpio-amd-fch.c index e6c6c3ec7656e..9f329938202bf 100644 --- a/drivers/gpio/gpio-amd-fch.c +++ b/drivers/gpio/gpio-amd-fch.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -120,15 +121,15 @@ static int amd_fch_gpio_get(struct gpio_chip *gc, unsigned int offset) { unsigned long flags; - int ret; + u32 val; struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc); void __iomem *ptr = amd_fch_gpio_addr(priv, offset); spin_lock_irqsave(&priv->lock, flags); - ret = (readl_relaxed(ptr) & AMD_FCH_GPIO_FLAG_READ); + val = readl_relaxed(ptr); spin_unlock_irqrestore(&priv->lock, flags); - return ret; + return FIELD_GET(AMD_FCH_GPIO_FLAG_READ, val); } static int amd_fch_gpio_request(struct gpio_chip *chip, diff --git a/drivers/gpio/gpio-aspeed-sgpio.c b/drivers/gpio/gpio-aspeed-sgpio.c index 7622f9e9f54af..318cd0e397416 100644 --- a/drivers/gpio/gpio-aspeed-sgpio.c +++ b/drivers/gpio/gpio-aspeed-sgpio.c @@ -516,7 +516,7 @@ static const struct of_device_id aspeed_sgpio_of_table[] = { MODULE_DEVICE_TABLE(of, aspeed_sgpio_of_table); -static int __init aspeed_sgpio_probe(struct platform_device *pdev) +static int aspeed_sgpio_probe(struct platform_device *pdev) { u32 nr_gpios, sgpio_freq, sgpio_clk_div, gpio_cnt_regval, pin_mask; const struct aspeed_sgpio_pdata *pdata; @@ -611,11 +611,12 @@ static int __init aspeed_sgpio_probe(struct platform_device *pdev) } static struct platform_driver aspeed_sgpio_driver = { + .probe = aspeed_sgpio_probe, .driver = { .name = KBUILD_MODNAME, .of_match_table = aspeed_sgpio_of_table, }, }; -module_platform_driver_probe(aspeed_sgpio_driver, aspeed_sgpio_probe); +module_platform_driver(aspeed_sgpio_driver); MODULE_DESCRIPTION("Aspeed Serial GPIO Driver"); diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c index 7953a9c4e36d7..3da37a0fda3fb 100644 --- a/drivers/gpio/gpio-aspeed.c +++ b/drivers/gpio/gpio-aspeed.c @@ -24,7 +24,7 @@ /* * These two headers aren't meant to be used by GPIO drivers. We need - * them in order to access gpio_chip_hwgpio() which we need to implement + * them in order to access gpiod_hwgpio() which we need to implement * the aspeed specific API which allows the coprocessor to request * access to some GPIOs and to arbitrate between coprocessor and ARM. */ @@ -942,7 +942,7 @@ int aspeed_gpio_copro_grab_gpio(struct gpio_desc *desc, { struct gpio_chip *chip = gpiod_to_chip(desc); struct aspeed_gpio *gpio = gpiochip_get_data(chip); - int rc = 0, bindex, offset = gpio_chip_hwgpio(desc); + int rc = 0, bindex, offset = gpiod_hwgpio(desc); const struct aspeed_gpio_bank *bank = to_bank(offset); if (!aspeed_gpio_support_copro(gpio)) @@ -987,7 +987,7 @@ int aspeed_gpio_copro_release_gpio(struct gpio_desc *desc) { struct gpio_chip *chip = gpiod_to_chip(desc); struct aspeed_gpio *gpio = gpiochip_get_data(chip); - int rc = 0, bindex, offset = gpio_chip_hwgpio(desc); + int rc = 0, bindex, offset = gpiod_hwgpio(desc); if (!aspeed_gpio_support_copro(gpio)) return -EOPNOTSUPP; diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c index f40c9472588bc..f0cb1991b326c 100644 --- a/drivers/gpio/gpio-brcmstb.c +++ b/drivers/gpio/gpio-brcmstb.c @@ -301,12 +301,10 @@ static struct brcmstb_gpio_bank *brcmstb_gpio_hwirq_to_bank( struct brcmstb_gpio_priv *priv, irq_hw_number_t hwirq) { struct brcmstb_gpio_bank *bank; - int i = 0; - /* banks are in descending order */ - list_for_each_entry_reverse(bank, &priv->bank_list, node) { - i += bank->chip.gc.ngpio; - if (hwirq < i) + list_for_each_entry(bank, &priv->bank_list, node) { + if (hwirq >= bank->chip.gc.offset && + hwirq < (bank->chip.gc.offset + bank->chip.gc.ngpio)) return bank; } return NULL; diff --git a/drivers/gpio/gpio-it87.c b/drivers/gpio/gpio-it87.c index 5d677bcfccf26..2ad3c239367bc 100644 --- a/drivers/gpio/gpio-it87.c +++ b/drivers/gpio/gpio-it87.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -241,23 +242,17 @@ static int it87_gpio_direction_out(struct gpio_chip *chip, mask = 1 << (gpio_num % 8); group = (gpio_num / 8); - spin_lock(&it87_gpio->lock); + guard(spinlock)(&it87_gpio->lock); rc = superio_enter(); if (rc) - goto exit; + return rc; /* set the output enable bit */ superio_set_mask(mask, group + it87_gpio->output_base); rc = it87_gpio_set(chip, gpio_num, val); - if (rc) - goto exit; - superio_exit(); - -exit: - spin_unlock(&it87_gpio->lock); return rc; } diff --git a/drivers/gpio/gpio-loongson-64bit.c b/drivers/gpio/gpio-loongson-64bit.c index 02f181cb219e9..d5573fb0616ce 100644 --- a/drivers/gpio/gpio-loongson-64bit.c +++ b/drivers/gpio/gpio-loongson-64bit.c @@ -263,7 +263,7 @@ static int loongson_gpio_init_irqchip(struct platform_device *pdev, chip->irq.num_parents = data->intr_num; chip->irq.parents = devm_kcalloc(&pdev->dev, data->intr_num, sizeof(*chip->irq.parents), GFP_KERNEL); - if (!chip->parent) + if (!chip->irq.parents) return -ENOMEM; for (i = 0; i < data->intr_num; i++) { @@ -407,11 +407,11 @@ static const struct loongson_gpio_chip_data loongson_gpio_ls2k2000_data0 = { static const struct loongson_gpio_chip_data loongson_gpio_ls2k2000_data1 = { .label = "ls2k2000_gpio", - .mode = BIT_CTRL_MODE, - .conf_offset = 0x0, - .in_offset = 0x20, - .out_offset = 0x10, - .inten_offset = 0x30, + .mode = BYTE_CTRL_MODE, + .conf_offset = 0x800, + .in_offset = 0xa00, + .out_offset = 0x900, + .inten_offset = 0xb00, }; static const struct loongson_gpio_chip_data loongson_gpio_ls2k2000_data2 = { diff --git a/drivers/gpio/gpio-mpsse.c b/drivers/gpio/gpio-mpsse.c index 9f42bb30b4ec7..120b27183b1d9 100644 --- a/drivers/gpio/gpio-mpsse.c +++ b/drivers/gpio/gpio-mpsse.c @@ -10,6 +10,7 @@ #include #include #include +#include #include struct mpsse_priv { @@ -17,8 +18,10 @@ struct mpsse_priv { struct usb_device *udev; /* USB device encompassing all MPSSEs */ struct usb_interface *intf; /* USB interface for this MPSSE */ u8 intf_id; /* USB interface number for this MPSSE */ - struct work_struct irq_work; /* polling work thread */ + struct list_head workers; /* polling work threads */ struct mutex irq_mutex; /* lock over irq_data */ + struct mutex irq_race; /* race for polling worker teardown */ + raw_spinlock_t irq_spin; /* protects worker list */ atomic_t irq_type[16]; /* pin -> edge detection type */ atomic_t irq_enabled; int id; @@ -26,6 +29,9 @@ struct mpsse_priv { u8 gpio_outputs[2]; /* Output states for GPIOs [L, H] */ u8 gpio_dir[2]; /* Directions for GPIOs [L, H] */ + unsigned long dir_in; /* Bitmask of valid input pins */ + unsigned long dir_out; /* Bitmask of valid output pins */ + u8 *bulk_in_buf; /* Extra recv buffer to grab status bytes */ struct usb_endpoint_descriptor *bulk_in; @@ -34,6 +40,14 @@ struct mpsse_priv { struct mutex io_mutex; /* sync I/O with disconnect */ }; +struct mpsse_worker { + struct mpsse_priv *priv; + struct work_struct work; + atomic_t cancelled; + struct list_head list; /* linked list */ + struct list_head destroy; /* teardown linked list */ +}; + struct bulk_desc { bool tx; /* direction of bulk transfer */ u8 *data; /* input (tx) or output (rx) */ @@ -43,6 +57,14 @@ struct bulk_desc { int timeout; }; +#define MPSSE_NGPIO 16 + +struct mpsse_quirk { + const char *names[MPSSE_NGPIO]; /* Pin names, if applicable */ + unsigned long dir_in; /* Bitmask of valid input pins */ + unsigned long dir_out; /* Bitmask of valid output pins */ +}; + static const struct usb_device_id gpio_mpsse_table[] = { { USB_DEVICE(0x0c52, 0xa064) }, /* SeaLevel Systems, Inc. */ { } /* Terminating entry */ @@ -160,6 +182,32 @@ static int gpio_mpsse_get_bank(struct mpsse_priv *priv, u8 bank) return buf; } +static int mpsse_ensure_supported(struct gpio_chip *chip, + unsigned long mask, int direction) +{ + unsigned long supported, unsupported; + char *type = "input"; + struct mpsse_priv *priv = gpiochip_get_data(chip); + + supported = priv->dir_in; + if (direction == GPIO_LINE_DIRECTION_OUT) { + supported = priv->dir_out; + type = "output"; + } + + /* An invalid bit was in the provided mask */ + unsupported = mask & ~supported; + if (unsupported) { + dev_err(&priv->udev->dev, + "mpsse: GPIO %lu doesn't support %s\n", + find_first_bit(&unsupported, sizeof(unsupported) * 8), + type); + return -EOPNOTSUPP; + } + + return 0; +} + static int gpio_mpsse_set_multiple(struct gpio_chip *chip, unsigned long *mask, unsigned long *bits) { @@ -167,6 +215,10 @@ static int gpio_mpsse_set_multiple(struct gpio_chip *chip, unsigned long *mask, int ret; struct mpsse_priv *priv = gpiochip_get_data(chip); + ret = mpsse_ensure_supported(chip, *mask, GPIO_LINE_DIRECTION_OUT); + if (ret) + return ret; + guard(mutex)(&priv->io_mutex); for_each_set_clump8(i, bank_mask, mask, chip->ngpio) { bank = i / 8; @@ -194,6 +246,10 @@ static int gpio_mpsse_get_multiple(struct gpio_chip *chip, unsigned long *mask, int ret; struct mpsse_priv *priv = gpiochip_get_data(chip); + ret = mpsse_ensure_supported(chip, *mask, GPIO_LINE_DIRECTION_IN); + if (ret) + return ret; + guard(mutex)(&priv->io_mutex); for_each_set_clump8(i, bank_mask, mask, chip->ngpio) { bank = i / 8; @@ -242,10 +298,15 @@ static int gpio_mpsse_gpio_set(struct gpio_chip *chip, unsigned int offset, static int gpio_mpsse_direction_output(struct gpio_chip *chip, unsigned int offset, int value) { + int ret; struct mpsse_priv *priv = gpiochip_get_data(chip); int bank = (offset & 8) >> 3; int bank_offset = offset & 7; + ret = mpsse_ensure_supported(chip, BIT(offset), GPIO_LINE_DIRECTION_OUT); + if (ret) + return ret; + scoped_guard(mutex, &priv->io_mutex) priv->gpio_dir[bank] |= BIT(bank_offset); @@ -255,10 +316,15 @@ static int gpio_mpsse_direction_output(struct gpio_chip *chip, static int gpio_mpsse_direction_input(struct gpio_chip *chip, unsigned int offset) { + int ret; struct mpsse_priv *priv = gpiochip_get_data(chip); int bank = (offset & 8) >> 3; int bank_offset = offset & 7; + ret = mpsse_ensure_supported(chip, BIT(offset), GPIO_LINE_DIRECTION_IN); + if (ret) + return ret; + guard(mutex)(&priv->io_mutex); priv->gpio_dir[bank] &= ~BIT(bank_offset); gpio_mpsse_set_bank(priv, bank); @@ -284,18 +350,62 @@ static int gpio_mpsse_get_direction(struct gpio_chip *chip, return ret; } -static void gpio_mpsse_poll(struct work_struct *work) +/* + * Stops all workers except `my_worker`. + * Safe to call only when `irq_race` is held. + */ +static void gpio_mpsse_stop_all_except(struct mpsse_priv *priv, + struct mpsse_worker *my_worker) +{ + struct mpsse_worker *worker, *worker_tmp; + struct list_head destructors = LIST_HEAD_INIT(destructors); + + scoped_guard(raw_spinlock_irqsave, &priv->irq_spin) { + list_for_each_entry_safe(worker, worker_tmp, + &priv->workers, list) { + /* Don't stop ourselves */ + if (worker == my_worker) + continue; + + list_del(&worker->list); + + /* Give worker a chance to terminate itself */ + atomic_set(&worker->cancelled, 1); + /* Keep track of stuff to cancel */ + INIT_LIST_HEAD(&worker->destroy); + list_add(&worker->destroy, &destructors); + } + } + + list_for_each_entry_safe(worker, worker_tmp, + &destructors, destroy) { + list_del(&worker->destroy); + cancel_work_sync(&worker->work); + kfree(worker); + } +} + +static void gpio_mpsse_poll(struct work_struct *my_work) { unsigned long pin_mask, pin_states, flags; int irq_enabled, offset, err, value, fire_irq, irq, old_value[16], irq_type[16]; - struct mpsse_priv *priv = container_of(work, struct mpsse_priv, - irq_work); + struct mpsse_worker *my_worker = container_of(my_work, struct mpsse_worker, work); + struct mpsse_priv *priv = my_worker->priv; for (offset = 0; offset < priv->gpio.ngpio; ++offset) old_value[offset] = -1; - while ((irq_enabled = atomic_read(&priv->irq_enabled))) { + /* + * We only want one worker. Workers race to acquire irq_race and tear + * down all other workers. This is a cond guard so that we don't deadlock + * trying to cancel a worker. + */ + scoped_cond_guard(mutex_try, return, &priv->irq_race) + gpio_mpsse_stop_all_except(priv, my_worker); + + while ((irq_enabled = atomic_read(&priv->irq_enabled)) && + !atomic_read(&my_worker->cancelled)) { usleep_range(MPSSE_POLL_INTERVAL, MPSSE_POLL_INTERVAL + 1000); /* Cleanup will trigger at the end of the loop */ guard(mutex)(&priv->irq_mutex); @@ -370,21 +480,45 @@ static int gpio_mpsse_set_irq_type(struct irq_data *irqd, unsigned int type) static void gpio_mpsse_irq_disable(struct irq_data *irqd) { + struct mpsse_worker *worker; struct mpsse_priv *priv = irq_data_get_irq_chip_data(irqd); atomic_and(~BIT(irqd->hwirq), &priv->irq_enabled); gpiochip_disable_irq(&priv->gpio, irqd->hwirq); + + /* + * Can't actually do teardown in IRQ context (it blocks). + * As a result, these workers will stick around until irq is reenabled + * or device gets disconnected + */ + scoped_guard(raw_spinlock_irqsave, &priv->irq_spin) + list_for_each_entry(worker, &priv->workers, list) + atomic_set(&worker->cancelled, 1); } static void gpio_mpsse_irq_enable(struct irq_data *irqd) { + struct mpsse_worker *worker; struct mpsse_priv *priv = irq_data_get_irq_chip_data(irqd); gpiochip_enable_irq(&priv->gpio, irqd->hwirq); /* If no-one else was using the IRQ, enable it */ if (!atomic_fetch_or(BIT(irqd->hwirq), &priv->irq_enabled)) { - INIT_WORK(&priv->irq_work, gpio_mpsse_poll); - schedule_work(&priv->irq_work); + /* + * Can't be devm because it uses a non-raw spinlock (illegal in + * this context, where a raw spinlock is held by our caller) + */ + worker = kzalloc(sizeof(*worker), GFP_NOWAIT); + if (!worker) + return; + + worker->priv = priv; + INIT_LIST_HEAD(&worker->list); + INIT_WORK(&worker->work, gpio_mpsse_poll); + schedule_work(&worker->work); + + scoped_guard(raw_spinlock_irqsave, &priv->irq_spin) + list_add(&worker->list, &priv->workers); } } @@ -404,19 +538,61 @@ static void gpio_mpsse_ida_remove(void *data) ida_free(&gpio_mpsse_ida, priv->id); } +static void gpio_mpsse_usb_put_dev(void *data) +{ + struct mpsse_priv *priv = data; + + usb_put_dev(priv->udev); +} + +static int mpsse_init_valid_mask(struct gpio_chip *chip, + unsigned long *valid_mask, + unsigned int ngpios) +{ + struct mpsse_priv *priv = gpiochip_get_data(chip); + + if (WARN_ON(priv == NULL)) + return -ENODEV; + + *valid_mask = priv->dir_in | priv->dir_out; + + return 0; +} + +static void mpsse_irq_init_valid_mask(struct gpio_chip *chip, + unsigned long *valid_mask, + unsigned int ngpios) +{ + struct mpsse_priv *priv = gpiochip_get_data(chip); + + if (WARN_ON(priv == NULL)) + return; + + /* Can only use IRQ on input capable pins */ + *valid_mask = priv->dir_in; +} + static int gpio_mpsse_probe(struct usb_interface *interface, const struct usb_device_id *id) { struct mpsse_priv *priv; struct device *dev; + char *serial; int err; + struct mpsse_quirk *quirk = (void *)id->driver_info; dev = &interface->dev; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; + INIT_LIST_HEAD(&priv->workers); + priv->udev = usb_get_dev(interface_to_usbdev(interface)); + err = devm_add_action_or_reset(dev, gpio_mpsse_usb_put_dev, priv); + if (err) + return err; + priv->intf = interface; priv->intf_id = interface->cur_altsetting->desc.bInterfaceNumber; @@ -436,9 +612,21 @@ static int gpio_mpsse_probe(struct usb_interface *interface, if (err) return err; + err = devm_mutex_init(dev, &priv->irq_race); + if (err) + return err; + + raw_spin_lock_init(&priv->irq_spin); + + serial = priv->udev->serial; + if (!serial) + serial = "NONE"; + priv->gpio.label = devm_kasprintf(dev, GFP_KERNEL, - "gpio-mpsse.%d.%d", - priv->id, priv->intf_id); + "MPSSE%04x:%04x.%d.%d.%s", + id->idVendor, id->idProduct, + priv->intf_id, priv->id, + serial); if (!priv->gpio.label) return -ENOMEM; @@ -452,10 +640,20 @@ static int gpio_mpsse_probe(struct usb_interface *interface, priv->gpio.get_multiple = gpio_mpsse_get_multiple; priv->gpio.set_multiple = gpio_mpsse_set_multiple; priv->gpio.base = -1; - priv->gpio.ngpio = 16; + priv->gpio.ngpio = MPSSE_NGPIO; priv->gpio.offset = priv->intf_id * priv->gpio.ngpio; priv->gpio.can_sleep = 1; + if (quirk) { + priv->dir_out = quirk->dir_out; + priv->dir_in = quirk->dir_in; + priv->gpio.names = quirk->names; + priv->gpio.init_valid_mask = mpsse_init_valid_mask; + } else { + priv->dir_in = U16_MAX; + priv->dir_out = U16_MAX; + } + err = usb_find_common_endpoints(interface->cur_altsetting, &priv->bulk_in, &priv->bulk_out, NULL, NULL); @@ -494,6 +692,7 @@ static int gpio_mpsse_probe(struct usb_interface *interface, priv->gpio.irq.parents = NULL; priv->gpio.irq.default_type = IRQ_TYPE_NONE; priv->gpio.irq.handler = handle_simple_irq; + priv->gpio.irq.init_valid_mask = mpsse_irq_init_valid_mask; err = devm_gpiochip_add_data(dev, &priv->gpio, priv); if (err) @@ -506,9 +705,15 @@ static void gpio_mpsse_disconnect(struct usb_interface *intf) { struct mpsse_priv *priv = usb_get_intfdata(intf); + /* + * Lock prevents double-free of worker from here and the teardown + * step at the beginning of gpio_mpsse_poll + */ + scoped_guard(mutex, &priv->irq_race) + gpio_mpsse_stop_all_except(priv, NULL); + priv->intf = NULL; usb_set_intfdata(intf, NULL); - usb_put_dev(priv->udev); } static struct usb_driver gpio_mpsse_driver = { diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c index 52060b3ec7458..441ba95b38cf9 100644 --- a/drivers/gpio/gpio-mxc.c +++ b/drivers/gpio/gpio-mxc.c @@ -584,12 +584,13 @@ static bool mxc_gpio_set_pad_wakeup(struct mxc_gpio_port *port, bool enable) unsigned long config; bool ret = false; int i, type; + bool is_imx8qm = of_device_is_compatible(port->dev->of_node, "fsl,imx8qm-gpio"); static const u32 pad_type_map[] = { IMX_SCU_WAKEUP_OFF, /* 0 */ IMX_SCU_WAKEUP_RISE_EDGE, /* IRQ_TYPE_EDGE_RISING */ IMX_SCU_WAKEUP_FALL_EDGE, /* IRQ_TYPE_EDGE_FALLING */ - IMX_SCU_WAKEUP_FALL_EDGE, /* IRQ_TYPE_EDGE_BOTH */ + IMX_SCU_WAKEUP_RISE_EDGE, /* IRQ_TYPE_EDGE_BOTH */ IMX_SCU_WAKEUP_HIGH_LVL, /* IRQ_TYPE_LEVEL_HIGH */ IMX_SCU_WAKEUP_OFF, /* 5 */ IMX_SCU_WAKEUP_OFF, /* 6 */ @@ -604,6 +605,13 @@ static bool mxc_gpio_set_pad_wakeup(struct mxc_gpio_port *port, bool enable) config = pad_type_map[type]; else config = IMX_SCU_WAKEUP_OFF; + + if (is_imx8qm && config == IMX_SCU_WAKEUP_FALL_EDGE) { + dev_warn_once(port->dev, + "No falling-edge support for wakeup on i.MX8QM\n"); + config = IMX_SCU_WAKEUP_OFF; + } + ret |= mxc_gpio_generic_config(port, i, config); } } diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c index 97c5cd33279d5..e22b713166d71 100644 --- a/drivers/gpio/gpio-nomadik.c +++ b/drivers/gpio/gpio-nomadik.c @@ -430,6 +430,9 @@ void nmk_gpio_dbg_show_one(struct seq_file *s, struct pinctrl_dev *pctldev, #ifdef CONFIG_PINCTRL_NOMADIK if (mode == NMK_GPIO_ALT_C && pctldev) { desc = gpio_device_get_desc(chip->gpiodev, offset); + if (IS_ERR(desc)) + return; + mode = nmk_prcm_gpiocr_get_mode(pctldev, desc_to_gpio(desc)); } #endif diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c index a268c76bdca62..8693544304fa8 100644 --- a/drivers/gpio/gpio-omap.c +++ b/drivers/gpio/gpio-omap.c @@ -799,10 +799,13 @@ static struct platform_device omap_mpuio_device = { static inline void omap_mpuio_init(struct gpio_bank *bank) { - platform_set_drvdata(&omap_mpuio_device, bank); + static bool registered; - if (platform_driver_register(&omap_mpuio_driver) == 0) - (void) platform_device_register(&omap_mpuio_device); + platform_set_drvdata(&omap_mpuio_device, bank); + if (!registered) { + (void)platform_device_register(&omap_mpuio_device); + registered = true; + } } /*---------------------------------------------------------------------*/ @@ -1576,13 +1579,24 @@ static struct platform_driver omap_gpio_driver = { */ static int __init omap_gpio_drv_reg(void) { - return platform_driver_register(&omap_gpio_driver); + int ret; + + ret = platform_driver_register(&omap_mpuio_driver); + if (ret) + return ret; + + ret = platform_driver_register(&omap_gpio_driver); + if (ret) + platform_driver_unregister(&omap_mpuio_driver); + + return ret; } postcore_initcall(omap_gpio_drv_reg); static void __exit omap_gpio_exit(void) { platform_driver_unregister(&omap_gpio_driver); + platform_driver_unregister(&omap_mpuio_driver); } module_exit(omap_gpio_exit); diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index b46927f550389..f7daeadaea57c 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -126,6 +126,9 @@ static const struct i2c_device_id pca953x_id[] = { { "tca9539", 16 | PCA953X_TYPE | PCA_INT, }, { "tca9554", 8 | PCA953X_TYPE | PCA_INT, }, { "xra1202", 8 | PCA953X_TYPE }, + + { "tcal6408", 8 | PCA953X_TYPE | PCA_LATCH_INT, }, + { "tcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, }, { } }; MODULE_DEVICE_TABLE(i2c, pca953x_id); @@ -911,6 +914,8 @@ static void pca953x_irq_shutdown(struct irq_data *d) clear_bit(hwirq, chip->irq_trig_fall); clear_bit(hwirq, chip->irq_trig_level_low); clear_bit(hwirq, chip->irq_trig_level_high); + + pca953x_irq_mask(d); } static void pca953x_irq_print_chip(struct irq_data *data, struct seq_file *p) @@ -940,14 +945,35 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, unsigned long *pendin DECLARE_BITMAP(old_stat, MAX_LINE); DECLARE_BITMAP(cur_stat, MAX_LINE); DECLARE_BITMAP(new_stat, MAX_LINE); + DECLARE_BITMAP(int_stat, MAX_LINE); DECLARE_BITMAP(trigger, MAX_LINE); DECLARE_BITMAP(edges, MAX_LINE); int ret; + if (chip->driver_data & PCA_PCAL) { + /* Read INT_STAT before it is cleared by the input-port read. */ + ret = pca953x_read_regs(chip, PCAL953X_INT_STAT, int_stat); + if (ret) + return false; + } + ret = pca953x_read_regs(chip, chip->regs->input, cur_stat); if (ret) return false; + if (chip->driver_data & PCA_PCAL) { + /* Detect short pulses via INT_STAT. */ + bitmap_and(trigger, int_stat, chip->irq_mask, gc->ngpio); + + /* Apply filter for rising/falling edge selection. */ + bitmap_replace(new_stat, chip->irq_trig_fall, chip->irq_trig_raise, + cur_stat, gc->ngpio); + + bitmap_and(int_stat, new_stat, trigger, gc->ngpio); + } else { + bitmap_zero(int_stat, gc->ngpio); + } + /* Remove output pins from the equation */ pca953x_read_regs(chip, chip->regs->direction, reg_direction); @@ -961,7 +987,8 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, unsigned long *pendin if (bitmap_empty(chip->irq_trig_level_high, gc->ngpio) && bitmap_empty(chip->irq_trig_level_low, gc->ngpio)) { - if (bitmap_empty(trigger, gc->ngpio)) + if (bitmap_empty(trigger, gc->ngpio) && + bitmap_empty(int_stat, gc->ngpio)) return false; } @@ -969,6 +996,7 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, unsigned long *pendin bitmap_and(old_stat, chip->irq_trig_raise, new_stat, gc->ngpio); bitmap_or(edges, old_stat, cur_stat, gc->ngpio); bitmap_and(pending, edges, trigger, gc->ngpio); + bitmap_or(pending, pending, int_stat, gc->ngpio); bitmap_and(cur_stat, new_stat, chip->irq_trig_level_high, gc->ngpio); bitmap_and(cur_stat, cur_stat, chip->irq_mask, gc->ngpio); @@ -1441,6 +1469,9 @@ static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "ti,tca9538", .data = OF_953X( 8, PCA_INT), }, { .compatible = "ti,tca9539", .data = OF_953X(16, PCA_INT), }, + { .compatible = "ti,tcal6408", .data = OF_953X( 8, PCA_LATCH_INT), }, + { .compatible = "ti,tcal6416", .data = OF_953X(16, PCA_LATCH_INT), }, + { .compatible = "onnn,cat9554", .data = OF_953X( 8, PCA_INT), }, { .compatible = "onnn,pca9654", .data = OF_953X( 8, PCA_INT), }, { .compatible = "onnn,pca9655", .data = OF_953X(16, PCA_INT), }, diff --git a/drivers/gpio/gpio-regmap.c b/drivers/gpio/gpio-regmap.c index f4267af00027e..c64805dcb9f88 100644 --- a/drivers/gpio/gpio-regmap.c +++ b/drivers/gpio/gpio-regmap.c @@ -328,7 +328,7 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config config->regmap_irq_line, config->regmap_irq_flags, 0, config->regmap_irq_chip, &gpio->irq_chip_data); if (ret) - goto err_free_bitmap; + goto err_remove_gpiochip; irq_domain = regmap_irq_get_domain(gpio->irq_chip_data); } else diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c index 47174eb3ba76f..0fff4a699f12d 100644 --- a/drivers/gpio/gpio-rockchip.c +++ b/drivers/gpio/gpio-rockchip.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -164,12 +163,6 @@ static int rockchip_gpio_set_direction(struct gpio_chip *chip, unsigned long flags; u32 data = input ? 0 : 1; - - if (input) - pinctrl_gpio_direction_input(chip, offset); - else - pinctrl_gpio_direction_output(chip, offset); - raw_spin_lock_irqsave(&bank->slock, flags); rockchip_gpio_writel_bit(bank, offset, data, bank->gpio_regs->port_ddr); raw_spin_unlock_irqrestore(&bank->slock, flags); diff --git a/drivers/gpio/gpio-sprd.c b/drivers/gpio/gpio-sprd.c index 413bcd0a42405..2cc8abe705cdb 100644 --- a/drivers/gpio/gpio-sprd.c +++ b/drivers/gpio/gpio-sprd.c @@ -35,7 +35,7 @@ struct sprd_gpio { struct gpio_chip chip; void __iomem *base; - spinlock_t lock; + raw_spinlock_t lock; int irq; }; @@ -54,7 +54,7 @@ static void sprd_gpio_update(struct gpio_chip *chip, unsigned int offset, unsigned long flags; u32 tmp; - spin_lock_irqsave(&sprd_gpio->lock, flags); + raw_spin_lock_irqsave(&sprd_gpio->lock, flags); tmp = readl_relaxed(base + reg); if (val) @@ -63,7 +63,7 @@ static void sprd_gpio_update(struct gpio_chip *chip, unsigned int offset, tmp &= ~BIT(SPRD_GPIO_BIT(offset)); writel_relaxed(tmp, base + reg); - spin_unlock_irqrestore(&sprd_gpio->lock, flags); + raw_spin_unlock_irqrestore(&sprd_gpio->lock, flags); } static int sprd_gpio_read(struct gpio_chip *chip, unsigned int offset, u16 reg) @@ -236,7 +236,7 @@ static int sprd_gpio_probe(struct platform_device *pdev) if (IS_ERR(sprd_gpio->base)) return PTR_ERR(sprd_gpio->base); - spin_lock_init(&sprd_gpio->lock); + raw_spin_lock_init(&sprd_gpio->lock); sprd_gpio->chip.label = dev_name(&pdev->dev); sprd_gpio->chip.ngpio = SPRD_GPIO_NR; diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index 15a5762a82c25..b14052fe64ac6 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -595,7 +595,7 @@ static void tegra_gpio_irq_release_resources(struct irq_data *d) struct tegra_gpio_info *tgi = gpiochip_get_data(chip); gpiochip_relres_irq(chip, d->hwirq); - tegra_gpio_enable(tgi, d->hwirq); + tegra_gpio_disable(tgi, d->hwirq); } static void tegra_gpio_irq_print_chip(struct irq_data *d, struct seq_file *s) diff --git a/drivers/gpio/gpio-virtuser.c b/drivers/gpio/gpio-virtuser.c index a10eab7d2617e..252fec5ea3835 100644 --- a/drivers/gpio/gpio-virtuser.c +++ b/drivers/gpio/gpio-virtuser.c @@ -1684,10 +1684,10 @@ static void gpio_virtuser_device_config_group_release(struct config_item *item) { struct gpio_virtuser_device *dev = to_gpio_virtuser_device(item); - guard(mutex)(&dev->lock); - - if (gpio_virtuser_device_is_live(dev)) - gpio_virtuser_device_deactivate(dev); + scoped_guard(mutex, &dev->lock) { + if (gpio_virtuser_device_is_live(dev)) + gpio_virtuser_device_deactivate(dev); + } mutex_destroy(&dev->lock); ida_free(&gpio_virtuser_ida, dev->id); diff --git a/drivers/gpio/gpiolib-acpi-core.c b/drivers/gpio/gpiolib-acpi-core.c index d441c1236d8ca..8110690ea69d0 100644 --- a/drivers/gpio/gpiolib-acpi-core.c +++ b/drivers/gpio/gpiolib-acpi-core.c @@ -1104,6 +1104,7 @@ acpi_gpio_adr_space_handler(u32 function, acpi_physical_address address, unsigned int pin = agpio->pin_table[i]; struct acpi_gpio_connection *conn; struct gpio_desc *desc; + u16 word, shift; bool found; mutex_lock(&achip->conn_lock); @@ -1158,10 +1159,22 @@ acpi_gpio_adr_space_handler(u32 function, acpi_physical_address address, mutex_unlock(&achip->conn_lock); - if (function == ACPI_WRITE) - gpiod_set_raw_value_cansleep(desc, !!(*value & BIT(i))); - else - *value |= (u64)gpiod_get_raw_value_cansleep(desc) << i; + /* + * For the cases when OperationRegion() consists of more than + * 64 bits calculate the word and bit shift to use that one to + * access the value. + */ + word = i / 64; + shift = i % 64; + + if (function == ACPI_WRITE) { + gpiod_set_raw_value_cansleep(desc, value[word] & BIT_ULL(shift)); + } else { + if (gpiod_get_raw_value_cansleep(desc)) + value[word] |= BIT_ULL(shift); + else + value[word] &= ~BIT_ULL(shift); + } } out: @@ -1346,6 +1359,7 @@ static int acpi_gpio_package_count(const union acpi_object *obj) while (element < end) { switch (element->type) { case ACPI_TYPE_LOCAL_REFERENCE: + case ACPI_TYPE_STRING: element += 3; fallthrough; case ACPI_TYPE_INTEGER: diff --git a/drivers/gpio/gpiolib-acpi-quirks.c b/drivers/gpio/gpiolib-acpi-quirks.c index 7b95d1b033614..a0116f004975a 100644 --- a/drivers/gpio/gpiolib-acpi-quirks.c +++ b/drivers/gpio/gpiolib-acpi-quirks.c @@ -370,6 +370,28 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] __initconst = { .ignore_wake = "ASCP1A00:00@8", }, }, + { + /* + * Spurious wakeups, likely from touchpad controller + * Dell Precision 7780 + * Found in BIOS 1.24.1 + * + * Found in touchpad firmware, installed by Dell Touchpad Firmware Update Utility version 1160.4196.9, A01 + * ( Dell-Touchpad-Firmware-Update-Utility_VYGNN_WIN64_1160.4196.9_A00.EXE ), + * released on 11 Jul 2024 + * + * https://lore.kernel.org/linux-i2c/197ae95ffd8.dc819e60457077.7692120488609091556@zohomail.com/ + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_FAMILY, "Precision"), + DMI_MATCH(DMI_PRODUCT_NAME, "Precision 7780"), + DMI_MATCH(DMI_BOARD_NAME, "0C6JVW"), + }, + .driver_data = &(struct acpi_gpiolib_dmi_quirk) { + .ignore_wake = "VEN_0488:00@355", + }, + }, {} /* Terminating entry */ }; diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index d8d93059ac04c..986312c71678f 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -676,7 +676,7 @@ static enum hte_return process_hw_ts_thread(void *p) } le.line_seqno = line->line_seqno; le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno; - le.offset = gpio_chip_hwgpio(line->desc); + le.offset = gpiod_hwgpio(line->desc); linereq_put_event(lr, &le); @@ -793,7 +793,7 @@ static irqreturn_t edge_irq_thread(int irq, void *p) line->line_seqno++; le.line_seqno = line->line_seqno; le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno; - le.offset = gpio_chip_hwgpio(line->desc); + le.offset = gpiod_hwgpio(line->desc); linereq_put_event(lr, &le); @@ -891,7 +891,7 @@ static void debounce_work_func(struct work_struct *work) lr = line->req; le.timestamp_ns = line_event_timestamp(line); - le.offset = gpio_chip_hwgpio(line->desc); + le.offset = gpiod_hwgpio(line->desc); #ifdef CONFIG_HTE if (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) { /* discard events except the last one */ @@ -1591,7 +1591,7 @@ static void linereq_show_fdinfo(struct seq_file *out, struct file *file) for (i = 0; i < lr->num_lines; i++) seq_printf(out, "gpio-line:\t%d\n", - gpio_chip_hwgpio(lr->lines[i].desc)); + gpiod_hwgpio(lr->lines[i].desc)); } #endif @@ -2244,7 +2244,7 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, return; memset(info, 0, sizeof(*info)); - info->offset = gpio_chip_hwgpio(desc); + info->offset = gpiod_hwgpio(desc); if (desc->name) strscpy(info->name, desc->name, sizeof(info->name)); @@ -2550,7 +2550,7 @@ static int lineinfo_changed_notify(struct notifier_block *nb, struct gpio_desc *desc = data; struct file *fp; - if (!test_bit(gpio_chip_hwgpio(desc), cdev->watched_lines)) + if (!test_bit(gpiod_hwgpio(desc), cdev->watched_lines)) return NOTIFY_DONE; /* Keep the file descriptor alive for the duration of the notification. */ @@ -2573,6 +2573,7 @@ static int lineinfo_changed_notify(struct notifier_block *nb, ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC); if (!ctx) { pr_err("Failed to allocate memory for line info notification\n"); + fput(fp); return NOTIFY_DONE; } @@ -2720,7 +2721,7 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file) cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); if (!cdev) - return -ENODEV; + return -ENOMEM; cdev->watched_lines = bitmap_zalloc(gdev->ngpio, GFP_KERNEL); if (!cdev->watched_lines) @@ -2820,15 +2821,20 @@ int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) return -ENOMEM; ret = cdev_device_add(&gdev->chrdev, &gdev->dev); - if (ret) + if (ret) { + destroy_workqueue(gdev->line_state_wq); return ret; + } guard(srcu)(&gdev->srcu); gc = srcu_dereference(gdev->chip, &gdev->srcu); - if (!gc) + if (!gc) { + cdev_device_del(&gdev->chrdev, &gdev->dev); + destroy_workqueue(gdev->line_state_wq); return -ENODEV; + } - chip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id); + gpiochip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id); return 0; } diff --git a/drivers/gpio/gpiolib-swnode.c b/drivers/gpio/gpiolib-swnode.c index e3806db1c0e07..f21dbc28cf2c8 100644 --- a/drivers/gpio/gpiolib-swnode.c +++ b/drivers/gpio/gpiolib-swnode.c @@ -41,7 +41,7 @@ static struct gpio_device *swnode_get_gpio_device(struct fwnode_handle *fwnode) !strcmp(gdev_node->name, GPIOLIB_SWNODE_UNDEFINED_NAME)) return ERR_PTR(-ENOENT); - gdev = gpio_device_find_by_fwnode(fwnode); + gdev = gpio_device_find_by_label(gdev_node->name); return gdev ?: ERR_PTR(-EPROBE_DEFER); } diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 9a849245b3588..d4a46a0a37d8f 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -244,7 +244,7 @@ static int gpio_sysfs_request_irq(struct gpiod_data *data, unsigned char flags) * Remove this redundant call (along with the corresponding unlock) * when those drivers have been fixed. */ - ret = gpiochip_lock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); + ret = gpiochip_lock_as_irq(guard.gc, gpiod_hwgpio(desc)); if (ret < 0) goto err_clr_bits; @@ -258,7 +258,7 @@ static int gpio_sysfs_request_irq(struct gpiod_data *data, unsigned char flags) return 0; err_unlock: - gpiochip_unlock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); + gpiochip_unlock_as_irq(guard.gc, gpiod_hwgpio(desc)); err_clr_bits: clear_bit(GPIOD_FLAG_EDGE_RISING, &desc->flags); clear_bit(GPIOD_FLAG_EDGE_FALLING, &desc->flags); @@ -280,7 +280,7 @@ static void gpio_sysfs_free_irq(struct gpiod_data *data) data->irq_flags = 0; free_irq(data->irq, data); - gpiochip_unlock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); + gpiochip_unlock_as_irq(guard.gc, gpiod_hwgpio(desc)); clear_bit(GPIOD_FLAG_EDGE_RISING, &desc->flags); clear_bit(GPIOD_FLAG_EDGE_FALLING, &desc->flags); } @@ -478,10 +478,10 @@ static int export_gpio_desc(struct gpio_desc *desc) if (!guard.gc) return -ENODEV; - offset = gpio_chip_hwgpio(desc); + offset = gpiod_hwgpio(desc); if (!gpiochip_line_is_valid(guard.gc, offset)) { pr_debug_ratelimited("%s: GPIO %d masked\n", __func__, - gpio_chip_hwgpio(desc)); + gpiod_hwgpio(desc)); return -EINVAL; } @@ -823,7 +823,7 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) } desc_data->chip_attr_group.name = kasprintf(GFP_KERNEL, "gpio%u", - gpio_chip_hwgpio(desc)); + gpiod_hwgpio(desc)); if (!desc_data->chip_attr_group.name) { status = -ENOMEM; goto err_put_dirent; @@ -843,7 +843,7 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) if (status) goto err_free_name; - path = kasprintf(GFP_KERNEL, "gpio%u/value", gpio_chip_hwgpio(desc)); + path = kasprintf(GFP_KERNEL, "gpio%u/value", gpiod_hwgpio(desc)); if (!path) { status = -ENOMEM; goto err_remove_groups; @@ -919,63 +919,68 @@ int gpiod_export_link(struct device *dev, const char *name, } EXPORT_SYMBOL_GPL(gpiod_export_link); -/** - * gpiod_unexport - reverse effect of gpiod_export() - * @desc: GPIO to make unavailable - * - * This is implicit on gpiod_free(). - */ -void gpiod_unexport(struct gpio_desc *desc) +static void gpiod_unexport_unlocked(struct gpio_desc *desc) { struct gpiod_data *tmp, *desc_data = NULL; struct gpiodev_data *gdev_data; struct gpio_device *gdev; - if (!desc) { - pr_warn("%s: invalid GPIO\n", __func__); + if (!test_bit(GPIOD_FLAG_EXPORT, &desc->flags)) return; - } - scoped_guard(mutex, &sysfs_lock) { - if (!test_bit(GPIOD_FLAG_EXPORT, &desc->flags)) - return; - - gdev = gpiod_to_gpio_device(desc); - gdev_data = gdev_get_data(gdev); - if (!gdev_data) - return; + gdev = gpiod_to_gpio_device(desc); + gdev_data = gdev_get_data(gdev); + if (!gdev_data) + return; - list_for_each_entry(tmp, &gdev_data->exported_lines, list) { - if (gpiod_is_equal(desc, tmp->desc)) { - desc_data = tmp; - break; - } + list_for_each_entry(tmp, &gdev_data->exported_lines, list) { + if (gpiod_is_equal(desc, tmp->desc)) { + desc_data = tmp; + break; } + } - if (!desc_data) - return; + if (!desc_data) + return; - list_del(&desc_data->list); - clear_bit(GPIOD_FLAG_EXPORT, &desc->flags); + list_del(&desc_data->list); + clear_bit(GPIOD_FLAG_EXPORT, &desc->flags); #if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) - sysfs_put(desc_data->value_kn); - device_unregister(desc_data->dev); - - /* - * Release irq after deregistration to prevent race with - * edge_store. - */ - if (desc_data->irq_flags) - gpio_sysfs_free_irq(desc_data); + sysfs_put(desc_data->value_kn); + device_unregister(desc_data->dev); + + /* + * Release irq after deregistration to prevent race with + * edge_store. + */ + if (desc_data->irq_flags) + gpio_sysfs_free_irq(desc_data); #endif /* CONFIG_GPIO_SYSFS_LEGACY */ - sysfs_remove_groups(desc_data->parent, - desc_data->chip_attr_groups); - } + sysfs_remove_groups(desc_data->parent, + desc_data->chip_attr_groups); mutex_destroy(&desc_data->mutex); kfree(desc_data); } + +/** + * gpiod_unexport - reverse effect of gpiod_export() + * @desc: GPIO to make unavailable + * + * This is implicit on gpiod_free(). + */ +void gpiod_unexport(struct gpio_desc *desc) +{ + if (!desc) { + pr_warn("%s: invalid GPIO\n", __func__); + return; + } + + guard(mutex)(&sysfs_lock); + + gpiod_unexport_unlocked(desc); +} EXPORT_SYMBOL_GPL(gpiod_unexport); int gpiochip_sysfs_register(struct gpio_device *gdev) @@ -1054,29 +1059,28 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) struct gpio_desc *desc; struct gpio_chip *chip; - scoped_guard(mutex, &sysfs_lock) { - data = gdev_get_data(gdev); - if (!data) - return; + guard(mutex)(&sysfs_lock); -#if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) - device_unregister(data->cdev_base); -#endif /* CONFIG_GPIO_SYSFS_LEGACY */ - device_unregister(data->cdev_id); - kfree(data); - } + data = gdev_get_data(gdev); + if (!data) + return; guard(srcu)(&gdev->srcu); - chip = srcu_dereference(gdev->chip, &gdev->srcu); if (!chip) return; /* unregister gpiod class devices owned by sysfs */ for_each_gpio_desc_with_flag(chip, desc, GPIOD_FLAG_SYSFS) { - gpiod_unexport(desc); + gpiod_unexport_unlocked(desc); gpiod_free(desc); } + +#if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) + device_unregister(data->cdev_base); +#endif /* CONFIG_GPIO_SYSFS_LEGACY */ + device_unregister(data->cdev_id); + kfree(data); } /* @@ -1091,7 +1095,7 @@ static int gpiofind_sysfs_register(struct gpio_chip *gc, const void *data) ret = gpiochip_sysfs_register(gdev); if (ret) - chip_err(gc, "failed to register the sysfs entry: %d\n", ret); + gpiochip_err(gc, "failed to register the sysfs entry: %d\n", ret); return 0; } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index cd8800ba5825f..67f362d53d53f 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -443,7 +443,7 @@ int gpiod_get_direction(struct gpio_desc *desc) if (!guard.gc) return -ENODEV; - offset = gpio_chip_hwgpio(desc); + offset = gpiod_hwgpio(desc); flags = READ_ONCE(desc->flags); /* @@ -882,13 +882,15 @@ static const struct device_type gpio_dev_type = { #define gcdev_unregister(gdev) device_del(&(gdev)->dev) #endif +/* + * An initial reference count has been held in gpiochip_add_data_with_key(). + * The caller should drop the reference via gpio_device_put() on errors. + */ static int gpiochip_setup_dev(struct gpio_device *gdev) { struct fwnode_handle *fwnode = dev_fwnode(&gdev->dev); int ret; - device_initialize(&gdev->dev); - /* * If fwnode doesn't belong to another device, it's safe to clear its * initialized flag. @@ -921,8 +923,8 @@ static void gpiochip_machine_hog(struct gpio_chip *gc, struct gpiod_hog *hog) desc = gpiochip_get_desc(gc, hog->chip_hwnum); if (IS_ERR(desc)) { - chip_err(gc, "%s: unable to get GPIO desc: %ld\n", __func__, - PTR_ERR(desc)); + gpiochip_err(gc, "%s: unable to get GPIO desc: %ld\n", + __func__, PTR_ERR(desc)); return; } @@ -954,9 +956,11 @@ static void gpiochip_setup_devs(void) list_for_each_entry_srcu(gdev, &gpio_devices, list, srcu_read_lock_held(&gpio_devices_srcu)) { ret = gpiochip_setup_dev(gdev); - if (ret) + if (ret) { + gpio_device_put(gdev); dev_err(&gdev->dev, "Failed to initialize gpio device (%d)\n", ret); + } } } @@ -1037,59 +1041,72 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, int base = 0; int ret; - /* - * First: allocate and populate the internal stat container, and - * set up the struct device. - */ gdev = kzalloc(sizeof(*gdev), GFP_KERNEL); if (!gdev) return -ENOMEM; - - gdev->dev.type = &gpio_dev_type; - gdev->dev.bus = &gpio_bus_type; - gdev->dev.parent = gc->parent; - rcu_assign_pointer(gdev->chip, gc); - gc->gpiodev = gdev; gpiochip_set_data(gc, data); - device_set_node(&gdev->dev, gpiochip_choose_fwnode(gc)); - ret = ida_alloc(&gpio_ida, GFP_KERNEL); if (ret < 0) goto err_free_gdev; gdev->id = ret; - ret = dev_set_name(&gdev->dev, GPIOCHIP_NAME "%d", gdev->id); + ret = init_srcu_struct(&gdev->srcu); if (ret) goto err_free_ida; + rcu_assign_pointer(gdev->chip, gc); - if (gc->parent && gc->parent->driver) - gdev->owner = gc->parent->driver->owner; - else if (gc->owner) - /* TODO: remove chip->owner */ - gdev->owner = gc->owner; - else - gdev->owner = THIS_MODULE; + ret = init_srcu_struct(&gdev->desc_srcu); + if (ret) + goto err_cleanup_gdev_srcu; + + ret = dev_set_name(&gdev->dev, GPIOCHIP_NAME "%d", gdev->id); + if (ret) + goto err_cleanup_desc_srcu; + + device_initialize(&gdev->dev); + /* + * After this point any allocated resources to `gdev` will be + * free():ed by gpiodev_release(). If you add new resources + * then make sure they get free():ed there. + */ + gdev->dev.type = &gpio_dev_type; + gdev->dev.bus = &gpio_bus_type; + gdev->dev.parent = gc->parent; + device_set_node(&gdev->dev, gpiochip_choose_fwnode(gc)); ret = gpiochip_get_ngpios(gc, &gdev->dev); if (ret) - goto err_free_dev_name; + goto err_put_device; + gdev->ngpio = gc->ngpio; gdev->descs = kcalloc(gc->ngpio, sizeof(*gdev->descs), GFP_KERNEL); if (!gdev->descs) { ret = -ENOMEM; - goto err_free_dev_name; + goto err_put_device; } gdev->label = kstrdup_const(gc->label ?: "unknown", GFP_KERNEL); if (!gdev->label) { ret = -ENOMEM; - goto err_free_descs; + goto err_put_device; } - gdev->ngpio = gc->ngpio; gdev->can_sleep = gc->can_sleep; + rwlock_init(&gdev->line_state_lock); + RAW_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier); + BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); +#ifdef CONFIG_PINCTRL + INIT_LIST_HEAD(&gdev->pin_ranges); +#endif + if (gc->parent && gc->parent->driver) + gdev->owner = gc->parent->driver->owner; + else if (gc->owner) + /* TODO: remove chip->owner */ + gdev->owner = gc->owner; + else + gdev->owner = THIS_MODULE; scoped_guard(mutex, &gpio_devices_lock) { /* @@ -1105,7 +1122,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (base < 0) { ret = base; base = 0; - goto err_free_label; + goto err_put_device; } /* @@ -1124,37 +1141,21 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, ret = gpiodev_add_to_list_unlocked(gdev); if (ret) { - chip_err(gc, "GPIO integer space overlap, cannot add chip\n"); - goto err_free_label; + gpiochip_err(gc, "GPIO integer space overlap, cannot add chip\n"); + goto err_put_device; } } - rwlock_init(&gdev->line_state_lock); - RAW_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier); - BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); - - ret = init_srcu_struct(&gdev->srcu); - if (ret) - goto err_remove_from_list; - - ret = init_srcu_struct(&gdev->desc_srcu); - if (ret) - goto err_cleanup_gdev_srcu; - -#ifdef CONFIG_PINCTRL - INIT_LIST_HEAD(&gdev->pin_ranges); -#endif - if (gc->names) gpiochip_set_desc_names(gc); ret = gpiochip_set_names(gc); if (ret) - goto err_cleanup_desc_srcu; + goto err_remove_from_list; ret = gpiochip_init_valid_mask(gc); if (ret) - goto err_cleanup_desc_srcu; + goto err_remove_from_list; for (desc_index = 0; desc_index < gc->ngpio; desc_index++) { struct gpio_desc *desc = &gdev->descs[desc_index]; @@ -1227,29 +1228,23 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, of_gpiochip_remove(gc); err_free_valid_mask: gpiochip_free_valid_mask(gc); -err_cleanup_desc_srcu: - cleanup_srcu_struct(&gdev->desc_srcu); -err_cleanup_gdev_srcu: - cleanup_srcu_struct(&gdev->srcu); err_remove_from_list: scoped_guard(mutex, &gpio_devices_lock) list_del_rcu(&gdev->list); synchronize_srcu(&gpio_devices_srcu); - if (gdev->dev.release) { - /* release() has been registered by gpiochip_setup_dev() */ - gpio_device_put(gdev); - goto err_print_message; - } -err_free_label: - kfree_const(gdev->label); -err_free_descs: - kfree(gdev->descs); -err_free_dev_name: - kfree(dev_name(&gdev->dev)); +err_put_device: + gpio_device_put(gdev); + goto err_print_message; + +err_cleanup_desc_srcu: + cleanup_srcu_struct(&gdev->desc_srcu); +err_cleanup_gdev_srcu: + cleanup_srcu_struct(&gdev->srcu); err_free_ida: ida_free(&gpio_ida, gdev->id); err_free_gdev: kfree(gdev); + err_print_message: /* failures here can mean systems won't boot... */ if (ret != -EPROBE_DEFER) { @@ -1528,8 +1523,7 @@ static void gpiochip_set_hierarchical_irqchip(struct gpio_chip *gc, &parent_hwirq, &parent_type); if (ret) { - chip_err(gc, "skip set-up on hwirq %d\n", - i); + gpiochip_err(gc, "skip set-up on hwirq %d\n", i); continue; } @@ -1542,15 +1536,14 @@ static void gpiochip_set_hierarchical_irqchip(struct gpio_chip *gc, ret = irq_domain_alloc_irqs(gc->irq.domain, 1, NUMA_NO_NODE, &fwspec); if (ret < 0) { - chip_err(gc, - "can not allocate irq for GPIO line %d parent hwirq %d in hierarchy domain: %d\n", - i, parent_hwirq, - ret); + gpiochip_err(gc, + "can not allocate irq for GPIO line %d parent hwirq %d in hierarchy domain: %d\n", + i, parent_hwirq, ret); } } } - chip_err(gc, "%s unknown fwnode type proceed anyway\n", __func__); + gpiochip_err(gc, "%s unknown fwnode type proceed anyway\n", __func__); return; } @@ -1602,15 +1595,15 @@ static int gpiochip_hierarchy_irq_domain_alloc(struct irq_domain *d, if (ret) return ret; - chip_dbg(gc, "allocate IRQ %d, hwirq %lu\n", irq, hwirq); + gpiochip_dbg(gc, "allocate IRQ %d, hwirq %lu\n", irq, hwirq); ret = girq->child_to_parent_hwirq(gc, hwirq, type, &parent_hwirq, &parent_type); if (ret) { - chip_err(gc, "can't look up hwirq %lu\n", hwirq); + gpiochip_err(gc, "can't look up hwirq %lu\n", hwirq); return ret; } - chip_dbg(gc, "found parent hwirq %u\n", parent_hwirq); + gpiochip_dbg(gc, "found parent hwirq %u\n", parent_hwirq); /* * We set handle_bad_irq because the .set_type() should @@ -1631,8 +1624,8 @@ static int gpiochip_hierarchy_irq_domain_alloc(struct irq_domain *d, if (ret) return ret; - chip_dbg(gc, "alloc_irqs_parent for %d parent hwirq %d\n", - irq, parent_hwirq); + gpiochip_dbg(gc, "alloc_irqs_parent for %d parent hwirq %d\n", + irq, parent_hwirq); irq_set_lockdep_class(irq, gc->irq.lock_key, gc->irq.request_key); ret = irq_domain_alloc_irqs_parent(d, irq, 1, &gpio_parent_fwspec); /* @@ -1642,9 +1635,9 @@ static int gpiochip_hierarchy_irq_domain_alloc(struct irq_domain *d, if (irq_domain_is_msi(d->parent) && (ret == -EEXIST)) ret = 0; if (ret) - chip_err(gc, - "failed to allocate parent hwirq %d for hwirq %lu\n", - parent_hwirq, hwirq); + gpiochip_err(gc, + "failed to allocate parent hwirq %d for hwirq %lu\n", + parent_hwirq, hwirq); return ret; } @@ -1720,7 +1713,7 @@ static struct irq_domain *gpiochip_hierarchy_create_domain(struct gpio_chip *gc) if (!gc->irq.child_to_parent_hwirq || !gc->irq.fwnode) { - chip_err(gc, "missing irqdomain vital data\n"); + gpiochip_err(gc, "missing irqdomain vital data\n"); return ERR_PTR(-EINVAL); } @@ -1993,7 +1986,7 @@ static void gpiochip_set_irq_hooks(struct gpio_chip *gc) if (irqchip->flags & IRQCHIP_IMMUTABLE) return; - chip_warn(gc, "not an immutable chip, please consider fixing it!\n"); + gpiochip_warn(gc, "not an immutable chip, please consider fixing it!\n"); if (!irqchip->irq_request_resources && !irqchip->irq_release_resources) { @@ -2009,8 +2002,8 @@ static void gpiochip_set_irq_hooks(struct gpio_chip *gc) * ...and if so, give a gentle warning that this is bad * practice. */ - chip_info(gc, - "detected irqchip that is shared with multiple gpiochips: please fix the driver.\n"); + gpiochip_info(gc, + "detected irqchip that is shared with multiple gpiochips: please fix the driver.\n"); return; } @@ -2039,7 +2032,8 @@ static int gpiochip_irqchip_add_allocated_domain(struct gpio_chip *gc, return -EINVAL; if (gc->to_irq) - chip_warn(gc, "to_irq is redefined in %s and you shouldn't rely on it\n", __func__); + gpiochip_warn(gc, "to_irq is redefined in %s and you shouldn't rely on it\n", + __func__); gc->to_irq = gpiochip_to_irq; gc->irq.domain = domain; @@ -2080,7 +2074,7 @@ static int gpiochip_add_irqchip(struct gpio_chip *gc, return 0; if (gc->irq.parent_handler && gc->can_sleep) { - chip_err(gc, "you cannot have chained interrupts on a chip that may sleep\n"); + gpiochip_err(gc, "you cannot have chained interrupts on a chip that may sleep\n"); return -EINVAL; } @@ -2316,10 +2310,8 @@ int gpiochip_add_pingroup_range(struct gpio_chip *gc, int ret; pin_range = kzalloc(sizeof(*pin_range), GFP_KERNEL); - if (!pin_range) { - chip_err(gc, "failed to allocate pin ranges\n"); + if (!pin_range) return -ENOMEM; - } /* Use local offset as range ID */ pin_range->range.id = gpio_offset; @@ -2338,7 +2330,7 @@ int gpiochip_add_pingroup_range(struct gpio_chip *gc, pinctrl_add_gpio_range(pctldev, &pin_range->range); - chip_dbg(gc, "created GPIO range %d->%d ==> %s PINGRP %s\n", + gpiochip_dbg(gc, "created GPIO range %d->%d ==> %s PINGRP %s\n", gpio_offset, gpio_offset + pin_range->range.npins - 1, pinctrl_dev_get_devname(pctldev), pin_group); @@ -2379,10 +2371,8 @@ int gpiochip_add_pin_range_with_pins(struct gpio_chip *gc, int ret; pin_range = kzalloc(sizeof(*pin_range), GFP_KERNEL); - if (!pin_range) { - chip_err(gc, "failed to allocate pin ranges\n"); + if (!pin_range) return -ENOMEM; - } /* Use local offset as range ID */ pin_range->range.id = gpio_offset; @@ -2396,19 +2386,18 @@ int gpiochip_add_pin_range_with_pins(struct gpio_chip *gc, &pin_range->range); if (IS_ERR(pin_range->pctldev)) { ret = PTR_ERR(pin_range->pctldev); - chip_err(gc, "could not create pin range\n"); + gpiochip_err(gc, "could not create pin range\n"); kfree(pin_range); return ret; } if (pin_range->range.pins) - chip_dbg(gc, "created GPIO range %d->%d ==> %s %d sparse PIN range { %d, ... }", - gpio_offset, gpio_offset + npins - 1, - pinctl_name, npins, pins[0]); + gpiochip_dbg(gc, "created GPIO range %d->%d ==> %s %d sparse PIN range { %d, ... }", + gpio_offset, gpio_offset + npins - 1, + pinctl_name, npins, pins[0]); else - chip_dbg(gc, "created GPIO range %d->%d ==> %s PIN %d->%d\n", - gpio_offset, gpio_offset + npins - 1, - pinctl_name, - pin_offset, pin_offset + npins - 1); + gpiochip_dbg(gc, "created GPIO range %d->%d ==> %s PIN %d->%d\n", + gpio_offset, gpio_offset + npins - 1, pinctl_name, + pin_offset, pin_offset + npins - 1); list_add_tail(&pin_range->node, &gdev->pin_ranges); @@ -2452,9 +2441,11 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) if (test_and_set_bit(GPIOD_FLAG_REQUESTED, &desc->flags)) return -EBUSY; - offset = gpio_chip_hwgpio(desc); - if (!gpiochip_line_is_valid(guard.gc, offset)) - return -EINVAL; + offset = gpiod_hwgpio(desc); + if (!gpiochip_line_is_valid(guard.gc, offset)) { + ret = -EINVAL; + goto out_clear_bit; + } /* NOTE: gpio_request() can be called in early boot, * before IRQs are enabled, for non-sleeping (SOC) GPIOs. @@ -2514,7 +2505,7 @@ static void gpiod_free_commit(struct gpio_desc *desc) if (guard.gc && test_bit(GPIOD_FLAG_REQUESTED, &flags)) { if (guard.gc->free) - guard.gc->free(guard.gc, gpio_chip_hwgpio(desc)); + guard.gc->free(guard.gc, gpiod_hwgpio(desc)); clear_bit(GPIOD_FLAG_ACTIVE_LOW, &flags); clear_bit(GPIOD_FLAG_REQUESTED, &flags); @@ -2618,7 +2609,7 @@ struct gpio_desc *gpiochip_request_own_desc(struct gpio_chip *gc, int ret; if (IS_ERR(desc)) { - chip_err(gc, "failed to get GPIO %s descriptor\n", name); + gpiochip_err(gc, "failed to get GPIO %s descriptor\n", name); return desc; } @@ -2629,7 +2620,7 @@ struct gpio_desc *gpiochip_request_own_desc(struct gpio_chip *gc, ret = gpiod_configure_flags(desc, label, lflags, dflags); if (ret) { gpiod_free_commit(desc); - chip_err(gc, "setup of own GPIO %s failed\n", name); + gpiochip_err(gc, "setup of own GPIO %s failed\n", name); return ERR_PTR(ret); } @@ -2674,7 +2665,7 @@ int gpio_do_set_config(struct gpio_desc *desc, unsigned long config) if (!guard.gc->set_config) return -ENOTSUPP; - ret = guard.gc->set_config(guard.gc, gpio_chip_hwgpio(desc), config); + ret = guard.gc->set_config(guard.gc, gpiod_hwgpio(desc), config); if (ret > 0) ret = -EBADE; @@ -2705,7 +2696,7 @@ static int gpio_set_config_with_argument_optional(struct gpio_desc *desc, u32 argument) { struct device *dev = &desc->gdev->dev; - int gpio = gpio_chip_hwgpio(desc); + int gpio = gpiod_hwgpio(desc); int ret; ret = gpio_set_config_with_argument(desc, mode, argument); @@ -2868,9 +2859,9 @@ int gpiod_direction_input_nonotify(struct gpio_desc *desc) */ if (guard.gc->direction_input) { ret = gpiochip_direction_input(guard.gc, - gpio_chip_hwgpio(desc)); + gpiod_hwgpio(desc)); } else if (guard.gc->get_direction) { - dir = gpiochip_get_direction(guard.gc, gpio_chip_hwgpio(desc)); + dir = gpiochip_get_direction(guard.gc, gpiod_hwgpio(desc)); if (dir < 0) return dir; @@ -2929,12 +2920,12 @@ static int gpiod_direction_output_raw_commit(struct gpio_desc *desc, int value) if (guard.gc->direction_output) { ret = gpiochip_direction_output(guard.gc, - gpio_chip_hwgpio(desc), val); + gpiod_hwgpio(desc), val); } else { /* Check that we are in output mode if we can */ if (guard.gc->get_direction) { dir = gpiochip_get_direction(guard.gc, - gpio_chip_hwgpio(desc)); + gpiod_hwgpio(desc)); if (dir < 0) return dir; @@ -2949,7 +2940,7 @@ static int gpiod_direction_output_raw_commit(struct gpio_desc *desc, int value) * If we can't actively set the direction, we are some * output-only chip, so just drive the output as desired. */ - ret = gpiochip_set(guard.gc, gpio_chip_hwgpio(desc), val); + ret = gpiochip_set(guard.gc, gpiod_hwgpio(desc), val); if (ret) return ret; } @@ -3100,7 +3091,7 @@ int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags) } ret = guard.gc->en_hw_timestamp(guard.gc, - gpio_chip_hwgpio(desc), flags); + gpiod_hwgpio(desc), flags); if (ret) gpiod_warn(desc, "%s: hw ts request failed\n", __func__); @@ -3132,7 +3123,7 @@ int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags) return -ENOTSUPP; } - ret = guard.gc->dis_hw_timestamp(guard.gc, gpio_chip_hwgpio(desc), + ret = guard.gc->dis_hw_timestamp(guard.gc, gpiod_hwgpio(desc), flags); if (ret) gpiod_warn(desc, "%s: hw ts release failed\n", __func__); @@ -3255,15 +3246,19 @@ static int gpiochip_get(struct gpio_chip *gc, unsigned int offset) /* Make sure this is called after checking for gc->get(). */ ret = gc->get(gc, offset); - if (ret > 1) - ret = -EBADE; + if (ret > 1) { + gpiochip_warn(gc, + "invalid return value from gc->get(): %d, consider fixing the driver\n", + ret); + ret = !!ret; + } return ret; } static int gpio_chip_get_value(struct gpio_chip *gc, const struct gpio_desc *desc) { - return gc->get ? gpiochip_get(gc, gpio_chip_hwgpio(desc)) : -EIO; + return gc->get ? gpiochip_get(gc, gpiod_hwgpio(desc)) : -EIO; } /* I/O calls are only valid after configuration completed; the relevant @@ -3423,7 +3418,7 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep, first = i; do { const struct gpio_desc *desc = desc_array[i]; - int hwgpio = gpio_chip_hwgpio(desc); + int hwgpio = gpiod_hwgpio(desc); __set_bit(hwgpio, mask); i++; @@ -3445,7 +3440,7 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep, for (j = first; j < i; ) { const struct gpio_desc *desc = desc_array[j]; - int hwgpio = gpio_chip_hwgpio(desc); + int hwgpio = gpiod_hwgpio(desc); int value = test_bit(hwgpio, bits); if (!raw && test_bit(GPIOD_FLAG_ACTIVE_LOW, &desc->flags)) @@ -3582,7 +3577,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_array_value); */ static int gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value) { - int ret = 0, offset = gpio_chip_hwgpio(desc); + int ret = 0, offset = gpiod_hwgpio(desc); CLASS(gpio_chip_guard, guard)(desc); if (!guard.gc) @@ -3611,7 +3606,7 @@ static int gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value) */ static int gpio_set_open_source_value_commit(struct gpio_desc *desc, bool value) { - int ret = 0, offset = gpio_chip_hwgpio(desc); + int ret = 0, offset = gpiod_hwgpio(desc); CLASS(gpio_chip_guard, guard)(desc); if (!guard.gc) @@ -3643,7 +3638,7 @@ static int gpiod_set_raw_value_commit(struct gpio_desc *desc, bool value) return -ENODEV; trace_gpio_value(desc_to_gpio(desc), 0, value); - return gpiochip_set(guard.gc, gpio_chip_hwgpio(desc), value); + return gpiochip_set(guard.gc, gpiod_hwgpio(desc), value); } /* @@ -3766,7 +3761,7 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, do { struct gpio_desc *desc = desc_array[i]; - int hwgpio = gpio_chip_hwgpio(desc); + int hwgpio = gpiod_hwgpio(desc); int value = test_bit(i, value_bitmap); if (unlikely(!test_bit(GPIOD_FLAG_IS_OUT, &desc->flags))) @@ -4006,7 +4001,7 @@ int gpiod_to_irq(const struct gpio_desc *desc) if (!gc) return -ENODEV; - offset = gpio_chip_hwgpio(desc); + offset = gpiod_hwgpio(desc); if (gc->to_irq) { ret = gc->to_irq(gc, offset); if (ret) @@ -4056,8 +4051,8 @@ int gpiochip_lock_as_irq(struct gpio_chip *gc, unsigned int offset) int dir = gpiod_get_direction(desc); if (dir < 0) { - chip_err(gc, "%s: cannot get GPIO direction\n", - __func__); + gpiochip_err(gc, "%s: cannot get GPIO direction\n", + __func__); return dir; } } @@ -4065,9 +4060,9 @@ int gpiochip_lock_as_irq(struct gpio_chip *gc, unsigned int offset) /* To be valid for IRQ the line needs to be input or open drain */ if (test_bit(GPIOD_FLAG_IS_OUT, &desc->flags) && !test_bit(GPIOD_FLAG_OPEN_DRAIN, &desc->flags)) { - chip_err(gc, - "%s: tried to flag a GPIO set as output for IRQ\n", - __func__); + gpiochip_err(gc, + "%s: tried to flag a GPIO set as output for IRQ\n", + __func__); return -EIO; } @@ -4144,7 +4139,7 @@ int gpiochip_reqres_irq(struct gpio_chip *gc, unsigned int offset) ret = gpiochip_lock_as_irq(gc, offset); if (ret) { - chip_err(gc, "unable to lock HW IRQ %u for IRQ\n", offset); + gpiochip_err(gc, "unable to lock HW IRQ %u for IRQ\n", offset); module_put(gc->gpiodev->owner); return ret; } @@ -4963,7 +4958,7 @@ int gpiod_hog(struct gpio_desc *desc, const char *name, if (test_and_set_bit(GPIOD_FLAG_IS_HOGGED, &desc->flags)) return 0; - hwnum = gpio_chip_hwgpio(desc); + hwnum = gpiod_hwgpio(desc); local_desc = gpiochip_request_own_desc(guard.gc, hwnum, name, lflags, dflags); @@ -5044,7 +5039,7 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev, * If pin hardware number of array member 0 is also 0, select * its chip as a candidate for fast bitmap processing path. */ - if (descs->ndescs == 0 && gpio_chip_hwgpio(desc) == 0) { + if (descs->ndescs == 0 && gpiod_hwgpio(desc) == 0) { struct gpio_descs *array; bitmap_size = BITS_TO_LONGS(gdev->ngpio > count ? @@ -5089,7 +5084,7 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev, * Detect array members which belong to the 'fast' chip * but their pins are not in hardware order. */ - else if (gpio_chip_hwgpio(desc) != descs->ndescs) { + else if (gpiod_hwgpio(desc) != descs->ndescs) { /* * Don't use fast path if all array members processed so * far belong to the same chip as this one but its pin diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 2a003a7311e7a..2b4f479d32123 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -276,7 +276,7 @@ const char *gpiod_get_label(struct gpio_desc *desc); /* * Return the GPIO number of the passed descriptor relative to its chip */ -static inline int gpio_chip_hwgpio(const struct gpio_desc *desc) +static inline int gpiod_hwgpio(const struct gpio_desc *desc) { return desc - &desc->gdev->descs[0]; } @@ -309,13 +309,13 @@ do { \ /* With chip prefix */ -#define chip_err(gc, fmt, ...) \ +#define gpiochip_err(gc, fmt, ...) \ dev_err(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) -#define chip_warn(gc, fmt, ...) \ +#define gpiochip_warn(gc, fmt, ...) \ dev_warn(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) -#define chip_info(gc, fmt, ...) \ +#define gpiochip_info(gc, fmt, ...) \ dev_info(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) -#define chip_dbg(gc, fmt, ...) \ +#define gpiochip_dbg(gc, fmt, ...) \ dev_dbg(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) #endif /* GPIOLIB_H */ diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 7e6bc0b3a589c..ed85d0ceee3ba 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -210,7 +210,7 @@ config DRM_GPUVM config DRM_GPUSVM tristate - depends on DRM && DEVICE_PRIVATE + depends on DRM select HMM_MIRROR select MMU_NOTIFIER help diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index da2565e6de71d..742f0d590c5af 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -106,8 +106,10 @@ obj-$(CONFIG_DRM_EXEC) += drm_exec.o obj-$(CONFIG_DRM_GPUVM) += drm_gpuvm.o drm_gpusvm_helper-y := \ - drm_gpusvm.o\ + drm_gpusvm.o +drm_gpusvm_helper-$(CONFIG_ZONE_DEVICE) += \ drm_pagemap.o + obj-$(CONFIG_DRM_GPUSVM) += drm_gpusvm_helper.o obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 6f5b4a0e0a343..803b6fc360a07 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -274,6 +274,8 @@ extern int amdgpu_rebar; extern int amdgpu_wbrf; extern int amdgpu_user_queue; +extern uint amdgpu_hdmi_hpd_debounce_delay_ms; + #define AMDGPU_VM_MAX_NUM_CTX 4096 #define AMDGPU_SG_THRESHOLD (256*1024*1024) #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c index 9b31804491500..3f9b094e93a29 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c @@ -641,6 +641,7 @@ static void aca_error_fini(struct aca_error *aerr) aca_bank_error_remove(aerr, bank_error); out_unlock: + mutex_unlock(&aerr->lock); mutex_destroy(&aerr->lock); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index 6c62e27b98002..67db986eda3f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -1136,8 +1136,10 @@ static int amdgpu_acpi_enumerate_xcc(void) if (!dev_info) ret = amdgpu_acpi_dev_init(&dev_info, xcc_info, sbdf); - if (ret == -ENOMEM) + if (ret == -ENOMEM) { + kfree(xcc_info); return ret; + } if (!dev_info) { kfree(xcc_info); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index a2879d2b7c8ec..1ec26be82f30e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -687,9 +687,9 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev, goto err_ib_sched; } - /* Drop the initial kref_init count (see drm_sched_main as example) */ - dma_fence_put(f); ret = dma_fence_wait(f, false); + /* Drop the returned fence reference after the wait completes */ + dma_fence_put(f); err_ib_sched: amdgpu_job_free(job); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 923f0fa7350cf..d3f541d3108c5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1421,7 +1421,10 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, *process_info = info; } - vm->process_info = *process_info; + if (cmpxchg(&vm->process_info, NULL, *process_info) != NULL) { + ret = -EINVAL; + goto already_acquired; + } /* Validate page directory and attach eviction fence */ ret = amdgpu_bo_reserve(vm->root.bo, true); @@ -1461,6 +1464,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, amdgpu_bo_unreserve(vm->root.bo); reserve_pd_fail: vm->process_info = NULL; +already_acquired: if (info) { dma_fence_put(&info->eviction_fence->base); *process_info = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index 66fb37b643882..ded22f244adab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -36,6 +36,7 @@ #define AMDGPU_BO_LIST_MAX_PRIORITY 32u #define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1) +#define AMDGPU_BO_LIST_MAX_ENTRIES (128 * 1024) static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu) { @@ -190,6 +191,9 @@ int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in, const uint32_t bo_number = in->bo_number; struct drm_amdgpu_bo_list_entry *info; + if (bo_number > AMDGPU_BO_LIST_MAX_ENTRIES) + return -EINVAL; + /* copy the handle array from userspace to a kernel buffer */ if (likely(info_size == bo_info_size)) { info = vmemdup_array_user(uptr, bo_number, info_size); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 96b6738e62526..c22aea46efcdf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2814,8 +2814,10 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) break; default: r = amdgpu_discovery_set_ip_blocks(adev); - if (r) + if (r) { + adev->num_ip_blocks = 0; return r; + } break; } @@ -3280,7 +3282,8 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) if (r) goto init_failed; - if (adev->mman.buffer_funcs_ring->sched.ready) + if (adev->mman.buffer_funcs_ring && + adev->mman.buffer_funcs_ring->sched.ready) amdgpu_ttm_set_buffer_funcs_status(adev, true); /* Don't init kfd if whole hive need to be reset during init */ @@ -3372,6 +3375,8 @@ int amdgpu_device_set_cg_state(struct amdgpu_device *adev, i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; if (!adev->ip_blocks[i].status.late_initialized) continue; + if (!adev->ip_blocks[i].version) + continue; /* skip CG for GFX, SDMA on S0ix */ if (adev->in_s0ix && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || @@ -3411,16 +3416,17 @@ int amdgpu_device_set_pg_state(struct amdgpu_device *adev, i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; if (!adev->ip_blocks[i].status.late_initialized) continue; + if (!adev->ip_blocks[i].version) + continue; /* skip PG for GFX, SDMA on S0ix */ if (adev->in_s0ix && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) continue; - /* skip CG for VCE/UVD/VPE, it's handled specially */ + /* skip CG for VCE/UVD, it's handled specially */ if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && - adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VPE && adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && adev->ip_blocks[i].version->funcs->set_powergating_state) { /* enable powergating to save power */ @@ -3619,6 +3625,8 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) int i, r; for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].version) + continue; if (!adev->ip_blocks[i].version->funcs->early_fini) continue; @@ -3630,9 +3638,6 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) } } - amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); - amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); - amdgpu_amdkfd_suspend(adev, true); amdgpu_userq_suspend(adev); @@ -3684,6 +3689,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) if (!adev->ip_blocks[i].status.sw) continue; + if (!adev->ip_blocks[i].version) + continue; if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { amdgpu_ucode_free_bo(adev); amdgpu_free_static_csa(&adev->virt.csa_obj); @@ -3710,6 +3717,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) for (i = adev->num_ip_blocks - 1; i >= 0; i--) { if (!adev->ip_blocks[i].status.late_initialized) continue; + if (!adev->ip_blocks[i].version) + continue; if (adev->ip_blocks[i].version->funcs->late_fini) adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]); adev->ip_blocks[i].status.late_initialized = false; @@ -4961,6 +4970,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_virt_fini_data_exchange(adev); } + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + /* disable all interrupts */ amdgpu_irq_disable_all(adev); if (adev->mode_info.mode_config_initialized) { @@ -4986,6 +4998,14 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_ttm_set_buffer_funcs_status(adev, false); + /* + * device went through surprise hotplug; we need to destroy topology + * before ip_fini_early to prevent kfd locking refcount issues by calling + * amdgpu_amdkfd_suspend() + */ + if (pci_dev_is_disconnected(adev->pdev)) + amdgpu_amdkfd_device_fini_sw(adev); + amdgpu_device_ip_fini_early(adev); amdgpu_irq_fini_hw(adev); @@ -4995,7 +5015,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_gart_dummy_page_fini(adev); - if (drm_dev_is_unplugged(adev_to_drm(adev))) + if (pci_dev_is_disconnected(adev->pdev)) amdgpu_device_unmap_mmio(adev); } @@ -5732,6 +5752,12 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev) if (ret) goto mode1_reset_failed; + /* enable mmio access after mode 1 reset completed */ + adev->no_hw_access = false; + + /* ensure no_hw_access is updated before we access hw */ + smp_mb(); + amdgpu_device_load_pci_state(adev->pdev); ret = amdgpu_psp_wait_for_bootloader(adev); if (ret) @@ -6476,6 +6502,8 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, struct amdgpu_hive_info *hive = NULL; int r = 0; bool need_emergency_restart = false; + /* save the pasid here as the job may be freed before the end of the reset */ + int pasid = job ? job->pasid : -EINVAL; /* * If it reaches here because of hang/timeout and a RAS error is @@ -6572,8 +6600,12 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, if (!r) { struct amdgpu_task_info *ti = NULL; - if (job) - ti = amdgpu_vm_get_task_info_pasid(adev, job->pasid); + /* + * The job may already be freed at this point via the sched tdr workqueue so + * use the cached pasid. + */ + if (pasid >= 0) + ti = amdgpu_vm_get_task_info_pasid(adev, pasid); drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, ti ? &ti->task : NULL); @@ -7045,6 +7077,15 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) dev_info(adev->dev, "PCI error: slot reset callback!!\n"); memset(&reset_context, 0, sizeof(reset_context)); + INIT_LIST_HEAD(&device_list); + hive = amdgpu_get_xgmi_hive(adev); + if (hive) { + mutex_lock(&hive->hive_lock); + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) + list_add_tail(&tmp_adev->reset_list, &device_list); + } else { + list_add_tail(&adev->reset_list, &device_list); + } if (adev->pcie_reset_ctx.swus) link_dev = adev->pcie_reset_ctx.swus; @@ -7085,19 +7126,13 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) reset_context.reset_req_dev = adev; set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); - INIT_LIST_HEAD(&device_list); - hive = amdgpu_get_xgmi_hive(adev); if (hive) { - mutex_lock(&hive->hive_lock); reset_context.hive = hive; - list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) tmp_adev->pcie_reset_ctx.in_link_reset = true; - list_add_tail(&tmp_adev->reset_list, &device_list); - } } else { set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); - list_add_tail(&adev->reset_list, &device_list); } r = amdgpu_device_asic_reset(adev, &device_list, &reset_context); @@ -7337,6 +7372,9 @@ void amdgpu_device_halt(struct amdgpu_device *adev) amdgpu_xcp_dev_unplug(adev); drm_dev_unplug(ddev); + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + amdgpu_irq_disable_all(adev); amdgpu_fence_driver_hw_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index dd7b2b796427c..54a045a0bda96 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -2995,6 +2995,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(6, 0, 0): case IP_VERSION(6, 0, 1): case IP_VERSION(6, 1, 0): + case IP_VERSION(6, 1, 1): adev->hdp.funcs = &hdp_v6_0_funcs; break; case IP_VERSION(7, 0, 0): diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index 51bab32fd8c6f..2f416d12e2e7e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -1824,7 +1824,12 @@ int amdgpu_display_get_scanout_buffer(struct drm_plane *plane, struct drm_scanout_buffer *sb) { struct amdgpu_bo *abo; - struct drm_framebuffer *fb = plane->state->fb; + struct drm_framebuffer *fb; + + if (drm_drv_uses_atomic_modeset(plane->dev)) + fb = plane->state->fb; + else + fb = plane->fb; if (!fb) return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index ed3bef1edfe44..f20d4bc58904d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -489,8 +489,15 @@ amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach) r = dma_resv_reserve_fences(resv, 2); if (!r) r = amdgpu_vm_clear_freed(adev, vm, NULL); + + /* Don't pass 'ticket' to amdgpu_vm_handle_moved: we want the clear=true + * path to be used otherwise we might update the PT of another process + * while it's using the BO. + * With clear=true, amdgpu_vm_bo_update will sync to command submission + * from the same VM. + */ if (!r) - r = amdgpu_vm_handle_moved(adev, vm, ticket); + r = amdgpu_vm_handle_moved(adev, vm, NULL); if (r && r != -EBUSY) DRM_ERROR("Failed to invalidate VM page tables (%d))\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 7333e19291cf9..7248947835f10 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -246,6 +246,7 @@ int amdgpu_damage_clips = -1; /* auto */ int amdgpu_umsch_mm_fwlog; int amdgpu_rebar = -1; /* auto */ int amdgpu_user_queue = -1; +uint amdgpu_hdmi_hpd_debounce_delay_ms; DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, "DRM_UT_CORE", @@ -1128,6 +1129,16 @@ module_param_named(rebar, amdgpu_rebar, int, 0444); MODULE_PARM_DESC(user_queue, "Enable user queues (-1 = auto (default), 0 = disable, 1 = enable, 2 = enable UQs and disable KQs)"); module_param_named(user_queue, amdgpu_user_queue, int, 0444); +/* + * DOC: hdmi_hpd_debounce_delay_ms (uint) + * HDMI HPD disconnect debounce delay in milliseconds. + * + * Used to filter short disconnect->reconnect HPD toggles some HDMI sinks + * generate while entering/leaving power save. Set to 0 to disable by default. + */ +MODULE_PARM_DESC(hdmi_hpd_debounce_delay_ms, "HDMI HPD disconnect debounce delay in milliseconds (0 to disable (by default), 1500 is common)"); +module_param_named(hdmi_hpd_debounce_delay_ms, amdgpu_hdmi_hpd_debounce_delay_ms, uint, 0644); + /* These devices are not supported by amdgpu. * They are supported by the mach64, r128, radeon drivers */ @@ -2334,9 +2345,6 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, return -ENODEV; } - if (amdgpu_aspm == -1 && !pcie_aspm_enabled(pdev)) - amdgpu_aspm = 0; - if (amdgpu_virtual_display || amdgpu_device_asic_has_dc_support(pdev, flags & AMD_ASIC_MASK)) supports_atomic = true; @@ -2675,8 +2683,12 @@ static int amdgpu_pmops_freeze(struct device *dev) if (r) return r; - if (amdgpu_acpi_should_gpu_reset(adev)) - return amdgpu_asic_reset(adev); + if (amdgpu_acpi_should_gpu_reset(adev)) { + amdgpu_device_lock_reset_domain(adev->reset_domain); + r = amdgpu_asic_reset(adev); + amdgpu_device_unlock_reset_domain(adev->reset_domain); + return r; + } return 0; } @@ -3158,7 +3170,6 @@ static int __init amdgpu_init(void) if (r) goto error_fence; - DRM_INFO("amdgpu kernel modesetting enabled.\n"); amdgpu_register_atpx_handler(); amdgpu_acpi_detect(); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 18a7829122d24..89a639044d520 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -804,7 +804,7 @@ void amdgpu_fence_save_wptr(struct dma_fence *fence) } static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring, - u64 start_wptr, u32 end_wptr) + u64 start_wptr, u64 end_wptr) { unsigned int first_idx = start_wptr & ring->buf_mask; unsigned int last_idx = end_wptr & ring->buf_mask; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index b7ebae289beab..1631a0431ea8e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -112,47 +112,6 @@ amdgpu_gem_update_timeline_node(struct drm_file *filp, return 0; } -static void -amdgpu_gem_update_bo_mapping(struct drm_file *filp, - struct amdgpu_bo_va *bo_va, - uint32_t operation, - uint64_t point, - struct dma_fence *fence, - struct drm_syncobj *syncobj, - struct dma_fence_chain *chain) -{ - struct amdgpu_bo *bo = bo_va ? bo_va->base.bo : NULL; - struct amdgpu_fpriv *fpriv = filp->driver_priv; - struct amdgpu_vm *vm = &fpriv->vm; - struct dma_fence *last_update; - - if (!syncobj) - return; - - /* Find the last update fence */ - switch (operation) { - case AMDGPU_VA_OP_MAP: - case AMDGPU_VA_OP_REPLACE: - if (bo && (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)) - last_update = vm->last_update; - else - last_update = bo_va->last_pt_update; - break; - case AMDGPU_VA_OP_UNMAP: - case AMDGPU_VA_OP_CLEAR: - last_update = fence; - break; - default: - return; - } - - /* Add fence to timeline */ - if (!point) - drm_syncobj_replace_fence(syncobj, last_update); - else - drm_syncobj_add_point(syncobj, chain, last_update, point); -} - static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) { struct ttm_buffer_object *bo = vmf->vma->vm_private_data; @@ -761,16 +720,27 @@ amdgpu_gem_va_update_vm(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, uint32_t operation) { - struct dma_fence *fence = dma_fence_get_stub(); - int r; + struct dma_fence *fence; + int r = 0; + + /* Always start from the VM's existing last update fence. */ + fence = dma_fence_get(vm->last_update); if (!amdgpu_vm_ready(vm)) return fence; + /* + * First clean up any freed mappings in the VM. + * + * amdgpu_vm_clear_freed() may replace @fence with a new fence if it + * schedules GPU work. If nothing needs clearing, @fence can remain as + * the original vm->last_update. + */ r = amdgpu_vm_clear_freed(adev, vm, &fence); if (r) goto error; + /* For MAP/REPLACE we also need to update the BO mappings. */ if (operation == AMDGPU_VA_OP_MAP || operation == AMDGPU_VA_OP_REPLACE) { r = amdgpu_vm_bo_update(adev, bo_va, false); @@ -778,7 +748,46 @@ amdgpu_gem_va_update_vm(struct amdgpu_device *adev, goto error; } + /* Always update PDEs after we touched the mappings. */ r = amdgpu_vm_update_pdes(adev, vm, false); + if (r) + goto error; + + /* + * Decide which fence best represents the last update: + * + * MAP/REPLACE: + * - For always-valid mappings, use vm->last_update. + * - Otherwise, export bo_va->last_pt_update. + * + * UNMAP/CLEAR: + * Keep the fence returned by amdgpu_vm_clear_freed(). If no work was + * needed, it can remain as vm->last_pt_update. + * + * The VM and BO update fences are always initialized to a valid value. + * vm->last_update and bo_va->last_pt_update always start as valid fences. + * and are never expected to be NULL. + */ + switch (operation) { + case AMDGPU_VA_OP_MAP: + case AMDGPU_VA_OP_REPLACE: + /* + * For MAP/REPLACE, return the page table update fence for the + * mapping we just modified. bo_va is expected to be valid here. + */ + dma_fence_put(fence); + + if (amdgpu_vm_is_bo_always_valid(vm, bo_va->base.bo)) + fence = dma_fence_get(vm->last_update); + else + fence = dma_fence_get(bo_va->last_pt_update); + break; + case AMDGPU_VA_OP_UNMAP: + case AMDGPU_VA_OP_CLEAR: + default: + /* keep @fence as returned by amdgpu_vm_clear_freed() */ + break; + } error: if (r && r != -ERESTARTSYS) @@ -810,6 +819,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, uint64_t vm_size; int r = 0; + /* Validate virtual address range against reserved regions. */ if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) { dev_dbg(dev->dev, "va_address 0x%llx is in reserved area 0x%llx\n", @@ -843,6 +853,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, return -EINVAL; } + /* Validate operation type. */ switch (args->operation) { case AMDGPU_VA_OP_MAP: case AMDGPU_VA_OP_UNMAP: @@ -866,6 +877,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, abo = NULL; } + /* Add input syncobj fences (if any) for synchronization. */ r = amdgpu_gem_add_input_fence(filp, args->input_fence_syncobj_handles, args->num_syncobj_handles); @@ -888,6 +900,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, goto error; } + /* Resolve the BO-VA mapping for this VM/BO combination. */ if (abo) { bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo); if (!bo_va) { @@ -900,6 +913,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, bo_va = NULL; } + /* + * Prepare the timeline syncobj node if the user requested a VM + * timeline update. This only allocates/looks up the syncobj and + * chain node; the actual fence is attached later. + */ r = amdgpu_gem_update_timeline_node(filp, args->vm_timeline_syncobj_out, args->vm_timeline_point, @@ -931,18 +949,30 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, default: break; } + + /* + * Once the VA operation is done, update the VM and obtain the fence + * that represents the last relevant update for this mapping. This + * fence can then be exported to the user-visible VM timeline. + */ if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) { fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, args->operation); - if (timeline_syncobj) - amdgpu_gem_update_bo_mapping(filp, bo_va, - args->operation, - args->vm_timeline_point, - fence, timeline_syncobj, - timeline_chain); - else - dma_fence_put(fence); + if (timeline_syncobj && fence) { + if (!args->vm_timeline_point) { + /* Replace the existing fence when no point is given. */ + drm_syncobj_replace_fence(timeline_syncobj, + fence); + } else { + /* Attach the last-update fence at a specific point. */ + drm_syncobj_add_point(timeline_syncobj, + timeline_chain, + fence, + args->vm_timeline_point); + } + } + dma_fence_put(fence); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 869bceb0fe2c6..1e3b8a506d1b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -498,8 +498,13 @@ void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr, if (adev->irq.retry_cam_enabled) return; + else if (adev->irq.ih1.ring_size) + ih = &adev->irq.ih1; + else if (adev->irq.ih_soft.enabled) + ih = &adev->irq.ih_soft; + else + return; - ih = &adev->irq.ih1; /* Get the WPTR of the last entry in IH ring */ last_wptr = amdgpu_ih_get_wptr(adev, ih); /* Order wptr with ring data. */ @@ -1008,6 +1013,16 @@ void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev) case CHIP_RENOIR: adev->mman.keep_stolen_vga_memory = true; break; + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + /* MacBookPros with switchable graphics put VRAM at 0 when + * the iGPU is enabled which results in cursor issues if + * the cursor ends up at 0. Reserve vram at 0 in that case. + */ + if (adev->gmc.vram_start == 0) + adev->mman.keep_stolen_vga_memory = true; + break; default: adev->mman.keep_stolen_vga_memory = false; break; @@ -1376,7 +1391,7 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, if (!*exp_ranges) *exp_ranges = range_cnt; err: - kfree(ranges); + kvfree(ranges); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 7d9bcb72e8dd3..9581643dbb726 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -228,7 +228,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, amdgpu_ring_ib_begin(ring); - if (ring->funcs->emit_gfx_shadow) + if (ring->funcs->emit_gfx_shadow && adev->gfx.cp_gfx_shadow) amdgpu_ring_emit_gfx_shadow(ring, shadow_va, csa_va, gds_va, init_shadow, vmid); @@ -284,7 +284,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, fence_flags | AMDGPU_FENCE_FLAG_64BIT); } - if (ring->funcs->emit_gfx_shadow && ring->funcs->init_cond_exec) { + if (ring->funcs->emit_gfx_shadow && ring->funcs->init_cond_exec && + adev->gfx.cp_gfx_shadow) { amdgpu_ring_emit_gfx_shadow(ring, 0, 0, 0, false, 0); amdgpu_ring_init_cond_exec(ring, ring->cond_exe_gpu_addr); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 3ef5bc95642ca..81cf9bf999a40 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -22,7 +22,7 @@ */ #include "amdgpu_ids.h" -#include +#include #include @@ -35,10 +35,13 @@ * PASIDs are global address space identifiers that can be shared * between the GPU, an IOMMU and the driver. VMs on different devices * may use the same PASID if they share the same address - * space. Therefore PASIDs are allocated using a global IDA. VMs are - * looked up from the PASID per amdgpu_device. + * space. Therefore PASIDs are allocated using IDR cyclic allocator + * (similar to kernel PID allocation) which naturally delays reuse. + * VMs are looked up from the PASID per amdgpu_device. */ -static DEFINE_IDA(amdgpu_pasid_ida); + +static DEFINE_XARRAY_FLAGS(amdgpu_pasid_xa, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ALLOC1); +static u32 amdgpu_pasid_xa_next; /* Helper to free pasid from a fence callback */ struct amdgpu_pasid_cb { @@ -50,8 +53,8 @@ struct amdgpu_pasid_cb { * amdgpu_pasid_alloc - Allocate a PASID * @bits: Maximum width of the PASID in bits, must be at least 1 * - * Allocates a PASID of the given width while keeping smaller PASIDs - * available if possible. + * Uses kernel's IDR cyclic allocator (same as PID allocation). + * Allocates sequentially with automatic wrap-around. * * Returns a positive integer on success. Returns %-EINVAL if bits==0. * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on @@ -59,29 +62,37 @@ struct amdgpu_pasid_cb { */ int amdgpu_pasid_alloc(unsigned int bits) { - int pasid = -EINVAL; + u32 pasid; + int r; - for (bits = min(bits, 31U); bits > 0; bits--) { - pasid = ida_alloc_range(&amdgpu_pasid_ida, 1U << (bits - 1), - (1U << bits) - 1, GFP_KERNEL); - if (pasid != -ENOSPC) - break; - } + if (bits == 0) + return -EINVAL; - if (pasid >= 0) - trace_amdgpu_pasid_allocated(pasid); + r = xa_alloc_cyclic_irq(&amdgpu_pasid_xa, &pasid, xa_mk_value(0), + XA_LIMIT(1, (1U << bits) - 1), + &amdgpu_pasid_xa_next, GFP_KERNEL); + if (r < 0) + return r; + trace_amdgpu_pasid_allocated(pasid); return pasid; } /** * amdgpu_pasid_free - Free a PASID * @pasid: PASID to free + * + * Called in IRQ context. */ void amdgpu_pasid_free(u32 pasid) { + unsigned long flags; + trace_amdgpu_pasid_freed(pasid); - ida_free(&amdgpu_pasid_ida, pasid); + + xa_lock_irqsave(&amdgpu_pasid_xa, flags); + __xa_erase(&amdgpu_pasid_xa, pasid); + xa_unlock_irqrestore(&amdgpu_pasid_xa, flags); } static void amdgpu_pasid_free_cb(struct dma_fence *fence, @@ -640,3 +651,13 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) } } } + +/** + * amdgpu_pasid_mgr_cleanup - cleanup PASID manager + * + * Cleanup the IDR allocator. + */ +void amdgpu_pasid_mgr_cleanup(void) +{ + xa_destroy(&amdgpu_pasid_xa); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h index b3649cd3af569..a57919478d3bd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h @@ -74,6 +74,7 @@ int amdgpu_pasid_alloc(unsigned int bits); void amdgpu_pasid_free(u32 pasid); void amdgpu_pasid_free_delayed(struct dma_resv *resv, u32 pasid); +void amdgpu_pasid_mgr_cleanup(void); bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev, struct amdgpu_vmid *id); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index d020a890a0ea4..ffc720d189007 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -130,11 +130,9 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) } /* attempt a per ring reset */ - if (unlikely(adev->debug_disable_gpu_ring_reset)) { - dev_err(adev->dev, "Ring reset disabled by debug mask\n"); - } else if (amdgpu_gpu_recovery && - amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_PER_QUEUE) && - ring->funcs->reset) { + if (amdgpu_gpu_recovery && + amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_PER_QUEUE) && + ring->funcs->reset) { dev_err(adev->dev, "Starting %s ring reset\n", s_job->sched->name); r = amdgpu_ring_reset(ring, job->vmid, &job->hw_fence); @@ -149,7 +147,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name); } - dma_fence_set_error(&s_job->s_fence->finished, -ETIME); + if (dma_fence_get_status(&s_job->s_fence->finished) == 0) + dma_fence_set_error(&s_job->s_fence->finished, -ETIME); if (amdgpu_device_should_recover_gpu(ring->adev)) { struct amdgpu_reset_context reset_context; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index b3e6b3fcdf2cb..d8c0154c5297d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -83,7 +83,7 @@ void amdgpu_driver_unload_kms(struct drm_device *dev) { struct amdgpu_device *adev = drm_to_adev(dev); - if (adev == NULL) + if (adev == NULL || !adev->num_ip_blocks) return; amdgpu_unregister_gpu_instance(adev); @@ -201,6 +201,9 @@ static enum amd_ip_block_type type = (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_JPEG)) ? AMD_IP_BLOCK_TYPE_JPEG : AMD_IP_BLOCK_TYPE_VCN; break; + case AMDGPU_HW_IP_VPE: + type = AMD_IP_BLOCK_TYPE_VPE; + break; default: type = AMD_IP_BLOCK_TYPE_NUM; break; @@ -721,6 +724,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) case AMD_IP_BLOCK_TYPE_UVD: count = adev->uvd.num_uvd_inst; break; + case AMD_IP_BLOCK_TYPE_VPE: + count = adev->vpe.num_instances; + break; /* For all other IP block types not listed in the switch statement * the ip status is valid here and the instance count is one. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 656b8a931dae8..52c2d1731aabf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -96,6 +96,7 @@ struct amdgpu_bo_va { * if non-zero, cannot unmap from GPU because user queues may still access it */ unsigned int queue_refcount; + atomic_t userq_va_mapped; }; struct amdgpu_bo { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index aa7987d0806c6..5f7aa840b2151 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -726,7 +726,7 @@ psp_cmd_submit_buf(struct psp_context *psp, ras_intr = amdgpu_ras_intr_triggered(); if (ras_intr) break; - usleep_range(10, 100); + usleep_range(60, 100); amdgpu_device_invalidate_hdp(psp->adev, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c index 6e8aad91bcd30..0d3c18f04ac36 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c @@ -332,13 +332,13 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size if (!context || !context->initialized) { dev_err(adev->dev, "TA is not initialized\n"); ret = -EINVAL; - goto err_free_shared_buf; + goto free_shared_buf; } if (!psp->ta_funcs || !psp->ta_funcs->fn_ta_invoke) { dev_err(adev->dev, "Unsupported function to invoke TA\n"); ret = -EOPNOTSUPP; - goto err_free_shared_buf; + goto free_shared_buf; } context->session_id = ta_id; @@ -346,7 +346,7 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size mutex_lock(&psp->ras_context.mutex); ret = prep_ta_mem_context(&context->mem_context, shared_buf, shared_buf_len); if (ret) - goto err_free_shared_buf; + goto unlock; ret = psp_fn_ta_invoke(psp, cmd_id); if (ret || context->resp_status) { @@ -354,15 +354,17 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size ret, context->resp_status); if (!ret) { ret = -EINVAL; - goto err_free_shared_buf; + goto unlock; } } if (copy_to_user((char *)&buf[copy_pos], context->mem_context.shared_buf, shared_buf_len)) ret = -EFAULT; -err_free_shared_buf: +unlock: mutex_unlock(&psp->ras_context.mutex); + +free_shared_buf: kfree(shared_buf); return ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index e0ee211508607..e9ed1a3b135cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -2892,6 +2892,11 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct ras_err_handler_data *data = con->eh_data; for (j = 0; j < count; j++) { + if (!data->space_left && + amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { + return -ENOMEM; + } + if (amdgpu_ras_check_bad_page_unlock(con, bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { data->count++; @@ -2899,11 +2904,6 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, continue; } - if (!data->space_left && - amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { - return -ENOMEM; - } - amdgpu_ras_reserve_page(adev, bps[j].retired_page); memcpy(&data->bps[data->count], &(bps[j]), @@ -3056,8 +3056,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, /* deal with retire_unit records a time */ ret = __amdgpu_ras_convert_rec_array_from_rom(adev, &bps[i], &err_data, nps); - if (ret) - con->bad_page_num -= adev->umc.retire_unit; i += (adev->umc.retire_unit - 1); } else { break; @@ -3070,8 +3068,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, for (; i < pages; i++) { ret = __amdgpu_ras_convert_rec_from_rom(adev, &bps[i], &err_data, nps); - if (ret) - con->bad_page_num -= adev->umc.retire_unit; } con->eh_data->count_saved = con->eh_data->count; @@ -4137,7 +4133,7 @@ int amdgpu_ras_init(struct amdgpu_device *adev) * to handle fatal error */ r = amdgpu_nbio_ras_sw_init(adev); if (r) - return r; + goto release_con; if (adev->nbio.ras && adev->nbio.ras->init_ras_controller_interrupt) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 3eb3fb55ccb05..dafa46a9656ca 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -1475,10 +1475,12 @@ int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control) } res = __verify_ras_table_checksum(control); - if (res) + if (res) { dev_err(adev->dev, "RAS table incorrect checksum or error:%d\n", res); + return -EINVAL; + } /* Warn if we are at 90% of the threshold or above */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 5ec5c3ff22bb0..304564ec2f59a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -460,9 +460,6 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, ktime_t deadline; bool ret; - if (unlikely(ring->adev->debug_disable_soft_recovery)) - return false; - deadline = ktime_add_us(ktime_get(), 10000); if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c index 8b8a04138711c..321310ba2c08e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c @@ -558,6 +558,9 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id, struct amdgpu_ring *gfx_ring = &sdma_instance->ring; struct amdgpu_ring *page_ring = &sdma_instance->page; + if (amdgpu_sriov_vf(adev)) + return -EOPNOTSUPP; + mutex_lock(&sdma_instance->engine_reset_mutex); if (!caller_handles_kernel_queues) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 9d568c16beb1e..4183e5301cffc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1529,6 +1529,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo, if (r) goto out; + mutex_lock(&adev->mman.gtt_window_lock); amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm); src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) + src_mm.start; @@ -1543,6 +1544,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo, WARN_ON(job->ibs[0].length_dw > num_dw); fence = amdgpu_job_submit(job); + mutex_unlock(&adev->mman.gtt_window_lock); if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout)) r = -ETIMEDOUT; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 1add21160d218..7e3175f82a20d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -44,10 +44,29 @@ u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev) return userq_ip_mask; } -int amdgpu_userq_input_va_validate(struct amdgpu_vm *vm, u64 addr, - u64 expected_size) +static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue, + struct amdgpu_bo_va_mapping *va_map, u64 addr) +{ + struct amdgpu_userq_va_cursor *va_cursor; + struct userq_va_list; + + va_cursor = kzalloc(sizeof(*va_cursor), GFP_KERNEL); + if (!va_cursor) + return -ENOMEM; + + INIT_LIST_HEAD(&va_cursor->list); + va_cursor->gpu_addr = addr; + atomic_set(&va_map->bo_va->userq_va_mapped, 1); + list_add(&va_cursor->list, &queue->userq_va_list); + + return 0; +} + +int amdgpu_userq_input_va_validate(struct amdgpu_usermode_queue *queue, + u64 addr, u64 expected_size) { struct amdgpu_bo_va_mapping *va_map; + struct amdgpu_vm *vm = queue->vm; u64 user_addr; u64 size; int r = 0; @@ -67,6 +86,7 @@ int amdgpu_userq_input_va_validate(struct amdgpu_vm *vm, u64 addr, /* Only validate the userq whether resident in the VM mapping range */ if (user_addr >= va_map->start && va_map->last - user_addr + 1 >= size) { + amdgpu_userq_buffer_va_list_add(queue, va_map, user_addr); amdgpu_bo_unreserve(vm->root.bo); return 0; } @@ -185,6 +205,7 @@ amdgpu_userq_cleanup(struct amdgpu_userq_mgr *uq_mgr, uq_funcs->mqd_destroy(uq_mgr, queue); amdgpu_userq_fence_driver_free(queue); idr_remove(&uq_mgr->userq_idr, queue_id); + list_del(&queue->userq_va_list); kfree(queue); } @@ -337,6 +358,13 @@ amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, goto unpin_bo; } + /* Validate doorbell_offset is within the doorbell BO */ + if ((u64)db_info->doorbell_offset * db_size + db_size > + amdgpu_bo_size(db_obj->obj)) { + r = -EINVAL; + goto unpin_bo; + } + index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj, db_info->doorbell_offset, db_size); drm_dbg_driver(adev_to_drm(uq_mgr->adev), @@ -505,14 +533,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) goto unlock; } - /* Validate the userq virtual address.*/ - if (amdgpu_userq_input_va_validate(&fpriv->vm, args->in.queue_va, args->in.queue_size) || - amdgpu_userq_input_va_validate(&fpriv->vm, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) || - amdgpu_userq_input_va_validate(&fpriv->vm, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) { - r = -EINVAL; - kfree(queue); - goto unlock; - } + INIT_LIST_HEAD(&queue->userq_va_list); queue->doorbell_handle = args->in.doorbell_handle; queue->queue_type = args->in.ip_type; queue->vm = &fpriv->vm; @@ -523,6 +544,15 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) db_info.db_obj = &queue->db_obj; db_info.doorbell_offset = args->in.doorbell_offset; + /* Validate the userq virtual address.*/ + if (amdgpu_userq_input_va_validate(queue, args->in.queue_va, args->in.queue_size) || + amdgpu_userq_input_va_validate(queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) || + amdgpu_userq_input_va_validate(queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) { + r = -EINVAL; + kfree(queue); + goto unlock; + } + /* Convert relative doorbell offset into absolute doorbell index */ index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp); if (index == (uint64_t)-EINVAL) { @@ -548,7 +578,6 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) goto unlock; } - qid = idr_alloc(&uq_mgr->userq_idr, queue, 1, AMDGPU_MAX_USERQ_COUNT, GFP_KERNEL); if (qid < 0) { drm_file_err(uq_mgr->file, "Failed to allocate a queue id\n"); @@ -634,6 +663,17 @@ static int amdgpu_userq_input_args_validate(struct drm_device *dev, drm_file_err(filp, "invalidate userq queue va or size\n"); return -EINVAL; } + + if (!is_power_of_2(args->in.queue_size)) { + drm_file_err(filp, "Queue size must be a power of 2\n"); + return -EINVAL; + } + + if (args->in.queue_size < AMDGPU_GPU_PAGE_SIZE) { + drm_file_err(filp, "Queue size smaller than AMDGPU_GPU_PAGE_SIZE\n"); + return -EINVAL; + } + if (!args->in.wptr_va || !args->in.rptr_va) { drm_file_err(filp, "invalidate userq queue rptr or wptr\n"); return -EINVAL; @@ -659,12 +699,28 @@ static int amdgpu_userq_input_args_validate(struct drm_device *dev, return 0; } +bool amdgpu_userq_enabled(struct drm_device *dev) +{ + struct amdgpu_device *adev = drm_to_adev(dev); + int i; + + for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { + if (adev->userq_funcs[i]) + return true; + } + + return false; +} + int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { union drm_amdgpu_userq *args = data; int r; + if (!amdgpu_userq_enabled(dev)) + return -ENOTSUPP; + if (amdgpu_userq_input_args_validate(dev, args, filp) < 0) return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h index c027dd9166727..d78532f9d507b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h @@ -47,6 +47,11 @@ struct amdgpu_userq_obj { struct amdgpu_bo *obj; }; +struct amdgpu_userq_va_cursor { + u64 gpu_addr; + struct list_head list; +}; + struct amdgpu_usermode_queue { int queue_type; enum amdgpu_userq_state state; @@ -66,6 +71,8 @@ struct amdgpu_usermode_queue { u32 xcp_id; int priority; struct dentry *debugfs_queue; + + struct list_head userq_va_list; }; struct amdgpu_userq_funcs { @@ -128,6 +135,7 @@ uint64_t amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, struct drm_file *filp); u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev); +bool amdgpu_userq_enabled(struct drm_device *dev); int amdgpu_userq_suspend(struct amdgpu_device *adev); int amdgpu_userq_resume(struct amdgpu_device *adev); @@ -136,7 +144,6 @@ int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, u32 idx); int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, u32 idx); - -int amdgpu_userq_input_va_validate(struct amdgpu_vm *vm, u64 addr, - u64 expected_size); +int amdgpu_userq_input_va_validate(struct amdgpu_usermode_queue *queue, + u64 addr, u64 expected_size); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index 4d0096d0baa9d..73e1827816a0c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -35,6 +35,8 @@ static const struct dma_fence_ops amdgpu_userq_fence_ops; static struct kmem_cache *amdgpu_userq_fence_slab; +#define AMDGPU_USERQ_MAX_HANDLES (1U << 16) + int amdgpu_userq_fence_slab_init(void) { amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence", @@ -141,6 +143,8 @@ static void amdgpu_userq_walk_and_drop_fence_drv(struct xarray *xa) void amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq) { + dma_fence_put(userq->last_fence); + amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa); xa_destroy(&userq->fence_drv_xa); /* Drop the fence_drv reference held by user queue */ @@ -470,6 +474,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, struct drm_exec exec; u64 wptr; + if (!amdgpu_userq_enabled(dev)) + return -ENOTSUPP; + + if (args->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES || + args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES || + args->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES) + return -EINVAL; + num_syncobj_handles = args->num_syncobj_handles; syncobj_handles = memdup_user(u64_to_user_ptr(args->syncobj_handles), size_mul(sizeof(u32), num_syncobj_handles)); @@ -652,6 +664,14 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, int r, i, rentry, wentry, cnt; struct drm_exec exec; + if (!amdgpu_userq_enabled(dev)) + return -ENOTSUPP; + + if (wait_info->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES || + wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES || + wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES) + return -EINVAL; + num_read_bo_handles = wait_info->num_bo_read_handles; bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles), size_mul(sizeof(u32), num_read_bo_handles)); @@ -821,7 +841,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv, DMA_RESV_USAGE_READ, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } @@ -838,7 +858,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv, DMA_RESV_USAGE_WRITE, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } @@ -862,8 +882,9 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, goto free_fences; dma_fence_unwrap_for_each(f, &iter, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; + dma_fence_put(fence); goto free_fences; } @@ -886,8 +907,9 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, if (r) goto free_fences; - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; + dma_fence_put(fence); goto free_fences; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index f96beb96c75cc..38183b4f03dfd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -972,6 +972,7 @@ bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_i || ucode_id == AMDGPU_UCODE_ID_SDMA5 || ucode_id == AMDGPU_UCODE_ID_SDMA6 || ucode_id == AMDGPU_UCODE_ID_SDMA7 + || ucode_id == AMDGPU_UCODE_ID_SDMA_RS64 || ucode_id == AMDGPU_UCODE_ID_RLC_G || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 676e24fb88647..69080e3734891 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1066,7 +1066,9 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } /* Prepare a TLB flush fence to be attached to PTs */ - if (!params->unlocked) { + if (!params->unlocked && + /* SI doesn't support pasid or KIQ/MES */ + params->adev->family > AMDGPU_FAMILY_SI) { amdgpu_vm_tlb_fence_create(params->adev, vm, fence); /* Makes sure no PD/PT is freed before the flush */ @@ -2882,6 +2884,7 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev) xa_destroy(&adev->vm_manager.pasids); amdgpu_vmid_mgr_fini(adev); + amdgpu_pasid_mgr_cleanup(); } /** @@ -2908,8 +2911,7 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) switch (args->in.op) { case AMDGPU_VM_OP_RESERVE_VMID: /* We only have requirement to reserve vmid from gfxhub */ - amdgpu_vmid_alloc_reserved(adev, vm, AMDGPU_GFXHUB(0)); - break; + return amdgpu_vmid_alloc_reserved(adev, vm, AMDGPU_GFXHUB(0)); case AMDGPU_VM_OP_UNRESERVE_VMID: amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(0)); break; @@ -2958,14 +2960,14 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, if (!root) return false; - addr /= AMDGPU_GPU_PAGE_SIZE; - if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid, - node_id, addr, ts, write_fault)) { + node_id, addr >> PAGE_SHIFT, ts, write_fault)) { amdgpu_bo_unref(&root); return true; } + addr /= AMDGPU_GPU_PAGE_SIZE; + r = amdgpu_bo_reserve(root, true); if (r) goto error_unref; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index cf0ec94e8a075..70fec516a2908 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -172,7 +172,7 @@ struct amdgpu_bo_vm; #define AMDGPU_VA_RESERVED_SEQ64_SIZE (2ULL << 20) #define AMDGPU_VA_RESERVED_SEQ64_START(adev) (AMDGPU_VA_RESERVED_CSA_START(adev) \ - AMDGPU_VA_RESERVED_SEQ64_SIZE) -#define AMDGPU_VA_RESERVED_TRAP_SIZE (2ULL << 12) +#define AMDGPU_VA_RESERVED_TRAP_SIZE (1ULL << 16) #define AMDGPU_VA_RESERVED_TRAP_START(adev) (AMDGPU_VA_RESERVED_SEQ64_START(adev) \ - AMDGPU_VA_RESERVED_TRAP_SIZE) #define AMDGPU_VA_RESERVED_BOTTOM (1ULL << 16) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 8841d7213de45..003bcece715eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4956,7 +4956,8 @@ static int gfx_v10_0_sw_init(struct amdgpu_ip_block *ip_block) amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); adev->gfx.compute_supported_reset = amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); - if (!amdgpu_sriov_vf(adev)) { + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } @@ -6878,7 +6879,7 @@ static int gfx_v10_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset) memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); /* reset the ring */ ring->wptr = 0; - *ring->wptr_cpu_addr = 0; + atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); amdgpu_ring_clear_ring(ring); } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index f2be16e700c41..1dd9fd486eecf 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -1821,13 +1821,15 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 0, 3): if ((adev->gfx.me_fw_version >= 2280) && (adev->gfx.mec_fw_version >= 2410) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } break; default: - if (!amdgpu_sriov_vf(adev)) { + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } @@ -4199,7 +4201,7 @@ static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset) memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); /* reset the ring */ ring->wptr = 0; - *ring->wptr_cpu_addr = 0; + atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); amdgpu_ring_clear_ring(ring); } @@ -6821,11 +6823,12 @@ static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, struct amdgpu_fence *timedout_fence) { struct amdgpu_device *adev = ring->adev; + bool use_mmio = false; int r; amdgpu_ring_reset_helper_begin(ring, timedout_fence); - r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false); + r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio); if (r) { dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r); @@ -6834,16 +6837,18 @@ static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, return r; } - r = gfx_v11_0_kgq_init_queue(ring, true); - if (r) { - dev_err(adev->dev, "failed to init kgq\n"); - return r; - } + if (use_mmio) { + r = gfx_v11_0_kgq_init_queue(ring, true); + if (r) { + dev_err(adev->dev, "failed to init kgq\n"); + return r; + } - r = amdgpu_mes_map_legacy_queue(adev, ring); - if (r) { - dev_err(adev->dev, "failed to remap kgq\n"); - return r; + r = amdgpu_mes_map_legacy_queue(adev, ring); + if (r) { + dev_err(adev->dev, "failed to remap kgq\n"); + return r; + } } return amdgpu_ring_reset_helper_end(ring, timedout_fence); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index 93fde0f9af87f..50e39b9d9df6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -278,7 +278,6 @@ static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance, int xcc_id); static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev); -static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val); static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); @@ -1548,7 +1547,8 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(12, 0, 1): if ((adev->gfx.me_fw_version >= 2660) && (adev->gfx.mec_fw_version >= 2920) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } @@ -3079,7 +3079,7 @@ static int gfx_v12_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset) memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); /* reset the ring */ ring->wptr = 0; - *ring->wptr_cpu_addr = 0; + atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); amdgpu_ring_clear_ring(ring); } @@ -4633,16 +4633,6 @@ static int gfx_v12_0_ring_preempt_ib(struct amdgpu_ring *ring) return r; } -static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, - bool start, - bool secure) -{ - uint32_t v = secure ? FRAME_TMZ : 0; - - amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); - amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); -} - static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t reg_val_offs) { @@ -5307,11 +5297,12 @@ static int gfx_v12_0_reset_kgq(struct amdgpu_ring *ring, struct amdgpu_fence *timedout_fence) { struct amdgpu_device *adev = ring->adev; + bool use_mmio = false; int r; amdgpu_ring_reset_helper_begin(ring, timedout_fence); - r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false); + r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio); if (r) { dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r); r = gfx_v12_reset_gfx_pipe(ring); @@ -5319,16 +5310,18 @@ static int gfx_v12_0_reset_kgq(struct amdgpu_ring *ring, return r; } - r = gfx_v12_0_kgq_init_queue(ring, true); - if (r) { - dev_err(adev->dev, "failed to init kgq\n"); - return r; - } + if (use_mmio) { + r = gfx_v12_0_kgq_init_queue(ring, true); + if (r) { + dev_err(adev->dev, "failed to init kgq\n"); + return r; + } - r = amdgpu_mes_map_legacy_queue(adev, ring); - if (r) { - dev_err(adev->dev, "failed to remap kgq\n"); - return r; + r = amdgpu_mes_map_legacy_queue(adev, ring); + if (r) { + dev_err(adev->dev, "failed to remap kgq\n"); + return r; + } } return amdgpu_ring_reset_helper_end(ring, timedout_fence); @@ -5519,7 +5512,6 @@ static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = { .emit_cntxcntl = gfx_v12_0_ring_emit_cntxcntl, .init_cond_exec = gfx_v12_0_ring_emit_init_cond_exec, .preempt_ib = gfx_v12_0_ring_preempt_ib, - .emit_frame_cntl = gfx_v12_0_ring_emit_frame_cntl, .emit_wreg = gfx_v12_0_ring_emit_wreg, .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait, .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index dd19a97436db9..7d0a2d239b78d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -2409,7 +2409,7 @@ static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block) amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); adev->gfx.compute_supported_reset = amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); - if (!amdgpu_sriov_vf(adev)) + if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_gpu_ring_reset) adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index c90cbe053ef37..a4ebb6c5af557 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -1149,14 +1149,16 @@ static int gfx_v9_4_3_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(9, 4, 3): case IP_VERSION(9, 4, 4): if ((adev->gfx.mec_fw_version >= 155) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE; } break; case IP_VERSION(9, 5, 0): if ((adev->gfx.mec_fw_version >= 21) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 7bc389d9f5c48..8acc912e7e4cb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -103,12 +103,39 @@ static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev, uint32_t vmhub_index = entry->client_id == SOC21_IH_CLIENTID_VMC ? AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0); struct amdgpu_vmhub *hub = &adev->vmhub[vmhub_index]; + bool retry_fault = !!(entry->src_data[1] & 0x80); + bool write_fault = !!(entry->src_data[1] & 0x20); uint32_t status = 0; u64 addr; addr = (u64)entry->src_data[0] << 12; addr |= ((u64)entry->src_data[1] & 0xf) << 44; + if (retry_fault) { + /* Returning 1 here also prevents sending the IV to the KFD */ + + /* Process it only if it's the first fault for this address */ + if (entry->ih != &adev->irq.ih_soft && + amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, + entry->timestamp)) + return 1; + + /* Delegate it to a different ring if the hardware hasn't + * already done it. + */ + if (entry->ih == &adev->irq.ih) { + amdgpu_irq_delegate(adev, entry, 8); + return 1; + } + + /* Try to handle the recoverable page faults by filling page + * tables + */ + if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr, + entry->timestamp, write_fault)) + return 1; + } + if (!amdgpu_sriov_vf(adev)) { /* * Issue a dummy read to wait for the status register to diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index f4a19357ccbc6..3746d39182707 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -91,6 +91,8 @@ static int gmc_v12_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { struct amdgpu_vmhub *hub; + bool retry_fault = !!(entry->src_data[1] & 0x80); + bool write_fault = !!(entry->src_data[1] & 0x20); uint32_t status = 0; u64 addr; @@ -102,6 +104,31 @@ static int gmc_v12_0_process_interrupt(struct amdgpu_device *adev, else hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; + if (retry_fault) { + /* Returning 1 here also prevents sending the IV to the KFD */ + + /* Process it only if it's the first fault for this address */ + if (entry->ih != &adev->irq.ih_soft && + amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, + entry->timestamp)) + return 1; + + /* Delegate it to a different ring if the hardware hasn't + * already done it. + */ + if (entry->ih == &adev->irq.ih) { + amdgpu_irq_delegate(adev, entry, 8); + return 1; + } + + /* Try to handle the recoverable page faults by filling page + * tables + */ + if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr, + entry->timestamp, write_fault)) + return 1; + } + if (!amdgpu_sriov_vf(adev)) { /* * Issue a dummy read to wait for the status register to diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 0d1dd587db5f4..0f0d6cd52d916 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -691,28 +691,35 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, } else { switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(9, 0, 0): - mmhub_cid = mmhub_client_ids_vega10[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vega10) ? + mmhub_client_ids_vega10[cid][rw] : NULL; break; case IP_VERSION(9, 3, 0): - mmhub_cid = mmhub_client_ids_vega12[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vega12) ? + mmhub_client_ids_vega12[cid][rw] : NULL; break; case IP_VERSION(9, 4, 0): - mmhub_cid = mmhub_client_ids_vega20[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vega20) ? + mmhub_client_ids_vega20[cid][rw] : NULL; break; case IP_VERSION(9, 4, 1): - mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_arcturus) ? + mmhub_client_ids_arcturus[cid][rw] : NULL; break; case IP_VERSION(9, 1, 0): case IP_VERSION(9, 2, 0): - mmhub_cid = mmhub_client_ids_raven[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_raven) ? + mmhub_client_ids_raven[cid][rw] : NULL; break; case IP_VERSION(1, 5, 0): case IP_VERSION(2, 4, 0): - mmhub_cid = mmhub_client_ids_renoir[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_renoir) ? + mmhub_client_ids_renoir[cid][rw] : NULL; break; case IP_VERSION(1, 8, 0): case IP_VERSION(9, 4, 2): - mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_aldebaran) ? + mmhub_client_ids_aldebaran[cid][rw] : NULL; break; default: mmhub_cid = NULL; @@ -1233,16 +1240,16 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, *flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_NC); break; case AMDGPU_VM_MTYPE_WC: - *flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_WC); + *flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_WC); break; case AMDGPU_VM_MTYPE_RW: - *flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_RW); + *flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_RW); break; case AMDGPU_VM_MTYPE_CC: - *flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_CC); + *flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_CC); break; case AMDGPU_VM_MTYPE_UC: - *flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_UC); + *flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_UC); break; } diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index 1cd9eaeef38f9..f5aa83ff57f35 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -254,7 +254,6 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type]; struct drm_amdgpu_userq_in *mqd_user = args_in; struct amdgpu_mqd_prop *userq_props; - struct amdgpu_gfx_shadow_info shadow_info; int r; /* Structure to initialize MQD for userqueue using generic MQD init function */ @@ -280,8 +279,6 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, userq_props->doorbell_index = queue->doorbell_index; userq_props->fence_address = queue->fence_drv->gpu_addr; - if (adev->gfx.funcs->get_gfx_shadow_info) - adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) { struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd; @@ -298,8 +295,9 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, goto free_mqd; } - if (amdgpu_userq_input_va_validate(queue->vm, compute_mqd->eop_va, - max_t(u32, PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE))) + r = amdgpu_userq_input_va_validate(queue, compute_mqd->eop_va, + 2048); + if (r) goto free_mqd; userq_props->eop_gpu_addr = compute_mqd->eop_va; @@ -311,6 +309,14 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, kfree(compute_mqd); } else if (queue->queue_type == AMDGPU_HW_IP_GFX) { struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11; + struct amdgpu_gfx_shadow_info shadow_info; + + if (adev->gfx.funcs->get_gfx_shadow_info) { + adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); + } else { + r = -EINVAL; + goto free_mqd; + } if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) { DRM_ERROR("Invalid GFX MQD\n"); @@ -330,8 +336,13 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, userq_props->tmz_queue = mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; - if (amdgpu_userq_input_va_validate(queue->vm, mqd_gfx_v11->shadow_va, - shadow_info.shadow_size)) + r = amdgpu_userq_input_va_validate(queue, mqd_gfx_v11->shadow_va, + shadow_info.shadow_size); + if (r) + goto free_mqd; + r = amdgpu_userq_input_va_validate(queue, mqd_gfx_v11->csa_va, + shadow_info.csa_size); + if (r) goto free_mqd; kfree(mqd_gfx_v11); @@ -350,9 +361,9 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, r = -ENOMEM; goto free_mqd; } - - if (amdgpu_userq_input_va_validate(queue->vm, mqd_sdma_v11->csa_va, - shadow_info.csa_size)) + r = amdgpu_userq_input_va_validate(queue, mqd_sdma_v11->csa_va, + 32); + if (r) goto free_mqd; userq_props->csa_addr = mqd_sdma_v11->csa_va; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index da575bb1377f1..d5cc32fa78485 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -714,11 +714,6 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes) mes_set_hw_res_pkt.enable_reg_active_poll = 1; mes_set_hw_res_pkt.enable_level_process_quantum_check = 1; mes_set_hw_res_pkt.oversubscription_timer = 50; - if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x7f) - mes_set_hw_res_pkt.enable_lr_compute_wa = 1; - else - dev_info_once(mes->adev->dev, - "MES FW version must be >= 0x7f to enable LR compute workaround.\n"); if (amdgpu_mes_log_enable) { mes_set_hw_res_pkt.enable_mes_event_int_logging = 1; @@ -1667,7 +1662,7 @@ static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block) if (r) goto failure; - if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x50) { + if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x52) { r = mes_v11_0_set_hw_resources_1(&adev->mes); if (r) { DRM_ERROR("failed mes_v11_0_set_hw_resources_1, r=%d\n", r); diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c index 7f3512d9de07d..c40aed5a8fc21 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c @@ -727,6 +727,9 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe) int i; struct amdgpu_device *adev = mes->adev; union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt; + uint32_t mes_rev = (pipe == AMDGPU_MES_SCHED_PIPE) ? + (mes->sched_version & AMDGPU_MES_VERSION_MASK) : + (mes->kiq_version & AMDGPU_MES_VERSION_MASK); memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt)); @@ -775,18 +778,13 @@ static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe) mes_set_hw_res_pkt.use_different_vmid_compute = 1; mes_set_hw_res_pkt.enable_reg_active_poll = 1; mes_set_hw_res_pkt.enable_level_process_quantum_check = 1; - if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x82) - mes_set_hw_res_pkt.enable_lr_compute_wa = 1; - else - dev_info_once(adev->dev, - "MES FW version must be >= 0x82 to enable LR compute workaround.\n"); /* * Keep oversubscribe timer for sdma . When we have unmapped doorbell * handling support, other queue will not use the oversubscribe timer. * handling mode - 0: disabled; 1: basic version; 2: basic+ version */ - mes_set_hw_res_pkt.oversubscription_timer = 50; + mes_set_hw_res_pkt.oversubscription_timer = mes_rev < 0x8b ? 0 : 50; mes_set_hw_res_pkt.unmapped_doorbell_handling = 1; if (amdgpu_mes_log_enable) { diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c index a0cc8e218ca1e..534cb4c544dc4 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c @@ -154,14 +154,17 @@ mmhub_v2_0_print_l2_protection_fault_status(struct amdgpu_device *adev, switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(2, 0, 0): case IP_VERSION(2, 0, 2): - mmhub_cid = mmhub_client_ids_navi1x[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_navi1x) ? + mmhub_client_ids_navi1x[cid][rw] : NULL; break; case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): - mmhub_cid = mmhub_client_ids_sienna_cichlid[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_sienna_cichlid) ? + mmhub_client_ids_sienna_cichlid[cid][rw] : NULL; break; case IP_VERSION(2, 1, 2): - mmhub_cid = mmhub_client_ids_beige_goby[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_beige_goby) ? + mmhub_client_ids_beige_goby[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c index 5eb8122e27469..ceb2f6b46de52 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c @@ -94,7 +94,8 @@ mmhub_v2_3_print_l2_protection_fault_status(struct amdgpu_device *adev, case IP_VERSION(2, 3, 0): case IP_VERSION(2, 4, 0): case IP_VERSION(2, 4, 1): - mmhub_cid = mmhub_client_ids_vangogh[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_vangogh) ? + mmhub_client_ids_vangogh[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c index 7d5242df58a51..ab966e69a342a 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c @@ -110,7 +110,8 @@ mmhub_v3_0_print_l2_protection_fault_status(struct amdgpu_device *adev, switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(3, 0, 0): case IP_VERSION(3, 0, 1): - mmhub_cid = mmhub_client_ids_v3_0_0[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_0_0) ? + mmhub_client_ids_v3_0_0[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c index 910337dc28d10..14a742d3a99d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c @@ -117,7 +117,8 @@ mmhub_v3_0_1_print_l2_protection_fault_status(struct amdgpu_device *adev, switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(3, 0, 1): - mmhub_cid = mmhub_client_ids_v3_0_1[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_0_1) ? + mmhub_client_ids_v3_0_1[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c index f0f182f033b98..e1f07f2a18527 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c @@ -108,7 +108,8 @@ mmhub_v3_0_2_print_l2_protection_fault_status(struct amdgpu_device *adev, "MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", status); - mmhub_cid = mmhub_client_ids_v3_0_2[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_0_2) ? + mmhub_client_ids_v3_0_2[cid][rw] : NULL; dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", mmhub_cid ? mmhub_cid : "unknown", cid); dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c index 951998454b257..88bfe321f83aa 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c @@ -102,7 +102,8 @@ mmhub_v4_1_0_print_l2_protection_fault_status(struct amdgpu_device *adev, status); switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(4, 1, 0): - mmhub_cid = mmhub_client_ids_v4_1_0[cid][rw]; + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v4_1_0) ? + mmhub_client_ids_v4_1_0[cid][rw] : NULL; break; default: mmhub_cid = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index a9be7a5050268..27d883fda5fa9 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -170,7 +170,8 @@ static int psp_v11_0_wait_for_bootloader(struct psp_context *psp) int retry_loop; /* For a reset done at the end of S3, only wait for TOS to be unloaded */ - if (adev->in_s3 && !(adev->flags & AMD_IS_APU) && amdgpu_in_reset(adev)) + if ((adev->in_s4 || adev->in_s3) && !(adev->flags & AMD_IS_APU) && + amdgpu_in_reset(adev)) return psp_v11_wait_for_tos_unload(psp); for (retry_loop = 0; retry_loop < 20; retry_loop++) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c index 36b1ca73c2ed3..a1443990d5c60 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c @@ -2361,11 +2361,15 @@ static void sdma_v4_4_2_update_reset_mask(struct amdgpu_device *adev) switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { case IP_VERSION(9, 4, 3): case IP_VERSION(9, 4, 4): - if ((adev->gfx.mec_fw_version >= 0xb0) && amdgpu_dpm_reset_sdma_is_supported(adev)) + if ((adev->gfx.mec_fw_version >= 0xb0) && + amdgpu_dpm_reset_sdma_is_supported(adev) && + !adev->debug_disable_gpu_ring_reset) adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; break; case IP_VERSION(9, 5, 0): - if ((adev->gfx.mec_fw_version >= 0xf) && amdgpu_dpm_reset_sdma_is_supported(adev)) + if ((adev->gfx.mec_fw_version >= 0xf) && + amdgpu_dpm_reset_sdma_is_supported(adev) && + !adev->debug_disable_gpu_ring_reset) adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; break; default: diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 7dc67a22a7a01..45e2933214a80 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -1424,17 +1424,9 @@ static int sdma_v5_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 0, 0): - case IP_VERSION(5, 0, 2): - case IP_VERSION(5, 0, 5): - if ((adev->sdma.instance[0].fw_version >= 35) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 3bd44c24f692d..5b982cc91af39 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -1342,23 +1342,9 @@ static int sdma_v5_2_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 0): - case IP_VERSION(5, 2, 2): - case IP_VERSION(5, 2, 3): - case IP_VERSION(5, 2, 4): - if ((adev->sdma.instance[0].fw_version >= 76) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - case IP_VERSION(5, 2, 5): - if ((adev->sdma.instance[0].fw_version >= 34) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index db6e41967f126..6809c6d4be5b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -1351,17 +1351,9 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(6, 0, 0): - case IP_VERSION(6, 0, 2): - case IP_VERSION(6, 0, 3): - if ((adev->sdma.instance[0].fw_version >= 21) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; if (amdgpu_sdma_ras_sw_init(adev)) { dev_err(adev->dev, "Failed to initialize sdma ras block!\n"); @@ -1389,7 +1381,7 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block) adev->userq_funcs[AMDGPU_HW_IP_DMA] = &userq_mes_funcs; break; case IP_VERSION(6, 0, 3): - if ((adev->sdma.instance[0].fw_version >= 27) && !adev->sdma.disable_uq) + if (adev->sdma.instance[0].fw_version >= 29 && !adev->sdma.disable_uq) adev->userq_funcs[AMDGPU_HW_IP_DMA] = &userq_mes_funcs; break; case IP_VERSION(6, 1, 0): diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index 326ecc8d37d21..2b81344dcd668 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -1337,7 +1337,8 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - if (!amdgpu_sriov_vf(adev)) + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; r = amdgpu_sdma_sysfs_reset_mask_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index ad36c96478a82..25536d89635d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -225,7 +225,13 @@ static u32 soc21_get_config_memsize(struct amdgpu_device *adev) static u32 soc21_get_xclk(struct amdgpu_device *adev) { - return adev->clock.spll.reference_freq; + u32 reference_clock = adev->clock.spll.reference_freq; + + /* reference clock is actually 99.81 Mhz rather than 100 Mhz */ + if ((adev->flags & AMD_IS_APU) && reference_clock == 10000) + return 9981; + + return reference_clock; } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c index 8897dcc9c1a0a..e35fae9cdaf66 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c @@ -1964,7 +1964,8 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) struct mmsch_v2_0_cmd_end end = { {0} }; struct mmsch_v2_0_init_header *header; uint32_t *init_table = adev->virt.mm_table.cpu_addr; - uint8_t i = 0; + + /* This path only programs VCN instance 0. */ header = (struct mmsch_v2_0_init_header *)init_table; direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE; @@ -1983,93 +1984,93 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[0].fw->size + 4); MMSCH_V2_0_INSERT_DIRECT_RD_MOD_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS), + SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0xFFFFFFFF, 0x00000004); /* mc resume*/ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi); offset = 0; } else { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr)); offset = size; } MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); for (r = 0; r < adev->vcn.inst[0].num_enc_rings; ++r) { ring = &adev->vcn.inst->ring_enc[r]; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4); } ring = &adev->vcn.inst->ring_dec; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH), upper_32_bits(ring->gpu_addr)); /* force RBC into idle state */ @@ -2080,7 +2081,7 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp); + SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp); /* add end packet */ tmp = sizeof(struct mmsch_v2_0_cmd_end); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c index cebee453871c1..006a154511971 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c @@ -521,7 +521,9 @@ static int vcn_v2_5_hw_fini(struct amdgpu_ip_block *ip_block) RREG32_SOC15(VCN, i, mmUVD_STATUS))) vinst->set_pg_state(vinst, AMD_PG_STATE_GATE); - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) + /* VF doesn't enable interrupt operations for RAS */ + if (!amdgpu_sriov_vf(adev) && + amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) amdgpu_irq_put(adev, &vinst->ras_poison_irq, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c index b107ee80e4728..1f6a22983c0dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c @@ -265,6 +265,8 @@ static int vcn_v4_0_5_sw_fini(struct amdgpu_ip_block *ip_block) if (amdgpu_sriov_vf(adev)) amdgpu_virt_free_mm_table(adev); + amdgpu_vcn_sysfs_reset_mask_fini(adev); + for (i = 0; i < adev->vcn.num_vcn_inst; i++) { r = amdgpu_vcn_suspend(adev, i); if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c index 0202df5db1e12..6109124f852e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c @@ -174,6 +174,10 @@ static int vcn_v5_0_0_sw_init(struct amdgpu_ip_block *ip_block) fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); fw_shared->sq.is_enabled = 1; + fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG); + fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ? + AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU; + if (amdgpu_vcnfw_log) amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]); diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h index 0320163b6e740..f98c735b2905f 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h @@ -3644,14 +3644,18 @@ static const uint32_t cwsr_trap_gfx9_4_3_hex[] = { }; static const uint32_t cwsr_trap_gfx12_hex[] = { - 0xbfa00001, 0xbfa002a2, - 0xb0804009, 0xb8f8f804, + 0xbfa00001, 0xbfa002b2, + 0xb0804009, 0xb8eef81a, + 0xbf880000, 0xb980081a, + 0x00000000, 0xb8f8f804, + 0x9177ff77, 0x0c000000, + 0x846e9a6e, 0x8c776e77, 0x9178ff78, 0x00008c00, 0xb8fbf811, 0x8b6eff78, 0x00004000, 0xbfa10008, 0x8b6eff7b, 0x00000080, 0xbfa20018, 0x8b6ea07b, - 0xbfa20042, 0xbf830010, + 0xbfa2004a, 0xbf830010, 0xb8fbf811, 0xbfa0fffb, 0x8b6eff7b, 0x00000bd0, 0xbfa20010, 0xb8eef812, @@ -3662,28 +3666,32 @@ static const uint32_t cwsr_trap_gfx12_hex[] = { 0xf0000000, 0xbfa20005, 0x8b6fff6f, 0x00000200, 0xbfa20002, 0x8b6ea07b, - 0xbfa2002c, 0xbefa4d82, + 0xbfa20034, 0xbefa4d82, 0xbf8a0000, 0x84fa887a, 0xbf0d8f7b, 0xbfa10002, 0x8c7bff7b, 0xffff0000, - 0xf4601bbd, 0xf8000010, - 0xbf8a0000, 0x846e976e, - 0x9177ff77, 0x00800000, - 0x8c776e77, 0xf4603bbd, - 0xf8000000, 0xbf8a0000, - 0xf4603ebd, 0xf8000008, - 0xbf8a0000, 0x8bee6e6e, - 0xbfa10001, 0xbe80486e, - 0x8b6eff6d, 0xf0000000, - 0xbfa20009, 0xb8eef811, - 0x8b6eff6e, 0x00000080, - 0xbfa20007, 0x8c78ff78, - 0x00004000, 0x80ec886c, - 0x82ed806d, 0xbfa00002, - 0x806c846c, 0x826d806d, - 0x8b6dff6d, 0x0000ffff, - 0x8bfe7e7e, 0x8bea6a6a, - 0x85788978, 0xb9783244, + 0x8b6eff77, 0x0c000000, + 0x916dff6d, 0x0c000000, + 0x8c6d6e6d, 0xf4601bbd, + 0xf8000010, 0xbf8a0000, + 0x846e976e, 0x9177ff77, + 0x00800000, 0x8c776e77, + 0xf4603bbd, 0xf8000000, + 0xbf8a0000, 0xf4603ebd, + 0xf8000008, 0xbf8a0000, + 0x8bee6e6e, 0xbfa10001, + 0xbe80486e, 0x8b6eff6d, + 0xf0000000, 0xbfa20009, + 0xb8eef811, 0x8b6eff6e, + 0x00000080, 0xbfa20007, + 0x8c78ff78, 0x00004000, + 0x80ec886c, 0x82ed806d, + 0xbfa00002, 0x806c846c, + 0x826d806d, 0x8b6dff6d, + 0x0000ffff, 0x8bfe7e7e, + 0x8bea6a6a, 0x85788978, + 0x936eff77, 0x0002001a, + 0xb96ef81a, 0xb9783244, 0xbe804a6c, 0xb8faf802, 0xbf0d987a, 0xbfa10001, 0xbfb00000, 0x8b6dff6d, @@ -3981,7 +3989,7 @@ static const uint32_t cwsr_trap_gfx12_hex[] = { 0x008ce800, 0x00000000, 0x807d817d, 0x8070ff70, 0x00000080, 0xbf0a7b7d, - 0xbfa2fff7, 0xbfa0016e, + 0xbfa2fff7, 0xbfa00171, 0xbef4007e, 0x8b75ff7f, 0x0000ffff, 0x8c75ff75, 0x00040000, 0xbef60080, @@ -4163,12 +4171,14 @@ static const uint32_t cwsr_trap_gfx12_hex[] = { 0xf8000074, 0xbf8a0000, 0x8b6dff6d, 0x0000ffff, 0x8bfe7e7e, 0x8bea6a6a, - 0xb97af804, 0xbe804ec2, - 0xbf94fffe, 0xbe804a6c, + 0x936eff77, 0x0002001a, + 0xb96ef81a, 0xb97af804, 0xbe804ec2, 0xbf94fffe, - 0xbfb10000, 0xbf9f0000, + 0xbe804a6c, 0xbe804ec2, + 0xbf94fffe, 0xbfb10000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, + 0xbf9f0000, 0x00000000, }; static const uint32_t cwsr_trap_gfx9_5_0_hex[] = { diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm index 5a1a1b1f897fe..07999b4649ded 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm @@ -78,9 +78,16 @@ var SQ_WAVE_EXCP_FLAG_PRIV_RESTORE_PART_2_SHIFT = SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL var SQ_WAVE_EXCP_FLAG_PRIV_RESTORE_PART_2_SIZE = SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_SHIFT - SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT var SQ_WAVE_EXCP_FLAG_PRIV_RESTORE_PART_3_SHIFT = SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT var SQ_WAVE_EXCP_FLAG_PRIV_RESTORE_PART_3_SIZE = 32 - SQ_WAVE_EXCP_FLAG_PRIV_RESTORE_PART_3_SHIFT + +var SQ_WAVE_SCHED_MODE_DEP_MODE_SHIFT = 0 +var SQ_WAVE_SCHED_MODE_DEP_MODE_SIZE = 2 + var BARRIER_STATE_SIGNAL_OFFSET = 16 var BARRIER_STATE_VALID_OFFSET = 0 +var TTMP11_SCHED_MODE_SHIFT = 26 +var TTMP11_SCHED_MODE_SIZE = 2 +var TTMP11_SCHED_MODE_MASK = 0xC000000 var TTMP11_DEBUG_TRAP_ENABLED_SHIFT = 23 var TTMP11_DEBUG_TRAP_ENABLED_MASK = 0x800000 @@ -160,8 +167,19 @@ L_JUMP_TO_RESTORE: s_branch L_RESTORE L_SKIP_RESTORE: + // Assume most relaxed scheduling mode is set. Save and revert to normal mode. + s_getreg_b32 ttmp2, hwreg(HW_REG_WAVE_SCHED_MODE) + s_wait_alu 0 + s_setreg_imm32_b32 hwreg(HW_REG_WAVE_SCHED_MODE, \ + SQ_WAVE_SCHED_MODE_DEP_MODE_SHIFT, SQ_WAVE_SCHED_MODE_DEP_MODE_SIZE), 0 + s_getreg_b32 s_save_state_priv, hwreg(HW_REG_WAVE_STATE_PRIV) //save STATUS since we will change SCC + // Save SCHED_MODE[1:0] into ttmp11[27:26]. + s_andn2_b32 ttmp11, ttmp11, TTMP11_SCHED_MODE_MASK + s_lshl_b32 ttmp2, ttmp2, TTMP11_SCHED_MODE_SHIFT + s_or_b32 ttmp11, ttmp11, ttmp2 + // Clear SPI_PRIO: do not save with elevated priority. // Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd. s_andn2_b32 s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_ALWAYS_CLEAR_MASK @@ -238,6 +256,13 @@ L_FETCH_2ND_TRAP: s_cbranch_scc0 L_NO_SIGN_EXTEND_TMA s_or_b32 ttmp15, ttmp15, 0xFFFF0000 L_NO_SIGN_EXTEND_TMA: +#if ASIC_FAMILY == CHIP_GFX12 + // Move SCHED_MODE[1:0] from ttmp11 to unused bits in ttmp1[27:26] (return PC_HI). + // The second-level trap will restore from ttmp1 for backwards compatibility. + s_and_b32 ttmp2, ttmp11, TTMP11_SCHED_MODE_MASK + s_andn2_b32 ttmp1, ttmp1, TTMP11_SCHED_MODE_MASK + s_or_b32 ttmp1, ttmp1, ttmp2 +#endif s_load_dword ttmp2, [ttmp14, ttmp15], 0x10 scope:SCOPE_SYS // debug trap enabled flag s_wait_idle @@ -287,6 +312,10 @@ L_EXIT_TRAP: // STATE_PRIV.BARRIER_COMPLETE may have changed since we read it. // Only restore fields which the trap handler changes. s_lshr_b32 s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_SCC_SHIFT + + // Assume relaxed scheduling mode after this point. + restore_sched_mode(ttmp2) + s_setreg_b32 hwreg(HW_REG_WAVE_STATE_PRIV, SQ_WAVE_STATE_PRIV_SCC_SHIFT, \ SQ_WAVE_STATE_PRIV_POISON_ERR_SHIFT - SQ_WAVE_STATE_PRIV_SCC_SHIFT + 1), s_save_state_priv @@ -1043,6 +1072,9 @@ L_SKIP_BARRIER_RESTORE: s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 + // Assume relaxed scheduling mode after this point. + restore_sched_mode(s_restore_tmp) + s_setreg_b32 hwreg(HW_REG_WAVE_STATE_PRIV), s_restore_state_priv // SCC is included, which is changed by previous salu // Make barrier and LDS state visible to all waves in the group. @@ -1134,3 +1166,8 @@ function valu_sgpr_hazard end #endif end + +function restore_sched_mode(s_tmp) + s_bfe_u32 s_tmp, ttmp11, (TTMP11_SCHED_MODE_SHIFT | (TTMP11_SCHED_MODE_SIZE << 0x10)) + s_setreg_b32 hwreg(HW_REG_WAVE_SCHED_MODE), s_tmp +end diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index ba99e0f258aee..986cb297de8f8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -401,27 +401,25 @@ static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_i return -ENOMEM; } -static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { spin_lock(&pdd->dev->watch_points_lock); /* process owns device watch point so safe to clear */ - if ((pdd->alloc_watch_ids >> watch_id) & 0x1) { - pdd->alloc_watch_ids &= ~(0x1 << watch_id); - pdd->dev->alloc_watch_ids &= ~(0x1 << watch_id); + if (pdd->alloc_watch_ids & BIT(watch_id)) { + pdd->alloc_watch_ids &= ~BIT(watch_id); + pdd->dev->alloc_watch_ids &= ~BIT(watch_id); } spin_unlock(&pdd->dev->watch_points_lock); } -static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { bool owns_watch_id = false; spin_lock(&pdd->dev->watch_points_lock); - owns_watch_id = watch_id < MAX_WATCH_ADDRESSES && - ((pdd->alloc_watch_ids >> watch_id) & 0x1); - + owns_watch_id = pdd->alloc_watch_ids & BIT(watch_id); spin_unlock(&pdd->dev->watch_points_lock); return owns_watch_id; @@ -432,6 +430,9 @@ int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd, { int r; + if (watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id)) return -EINVAL; @@ -469,6 +470,9 @@ int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd, if (r) return r; + if (*watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 6e7bc983fc0b6..58c5acf50a220 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -1209,14 +1209,8 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, pr_debug_ratelimited("Evicting process pid %d queues\n", pdd->process->lead_thread->pid); - if (dqm->dev->kfd->shared_resources.enable_mes) { + if (dqm->dev->kfd->shared_resources.enable_mes) pdd->last_evict_timestamp = get_jiffies_64(); - retval = suspend_all_queues_mes(dqm); - if (retval) { - dev_err(dev, "Suspending all queues failed"); - goto out; - } - } /* Mark all queues as evicted. Deactivate all active queues on * the qpd. @@ -1246,10 +1240,6 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); - } else { - retval = resume_all_queues_mes(dqm); - if (retval) - dev_err(dev, "Resuming all queues failed"); } out: @@ -2915,6 +2905,14 @@ static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) return retval; } +static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, + struct kfd_mem_obj *mqd) +{ + WARN(!mqd, "No hiq sdma mqd trunk to free"); + + amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem); +} + struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) { struct device_queue_manager *dqm; @@ -3038,19 +3036,14 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) return dqm; } + if (!dev->kfd->shared_resources.enable_mes) + deallocate_hiq_sdma_mqd(dev, &dqm->hiq_sdma_mqd); + out_free: kfree(dqm); return NULL; } -static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, - struct kfd_mem_obj *mqd) -{ - WARN(!mqd, "No hiq sdma mqd trunk to free"); - - amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem); -} - void device_queue_manager_uninit(struct device_queue_manager *dqm) { dqm->ops.stop(dqm); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 82905f3e54ddd..261db87e86fe6 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -331,6 +331,12 @@ static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, if (p->signal_page) return -EBUSY; + if (size < KFD_SIGNAL_EVENT_LIMIT * 8) { + pr_err("Event page size %llu is too small, need at least %lu bytes\n", + size, (unsigned long)(KFD_SIGNAL_EVENT_LIMIT * 8)); + return -EINVAL; + } + page = kzalloc(sizeof(*page), GFP_KERNEL); if (!page) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 59a5a3fea65df..ea8377071c390 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -62,7 +62,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring, u64 npages, *gart_addr = adev->gmc.gart_start; num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); - num_bytes = npages * 8; + num_bytes = npages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE; r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, AMDGPU_FENCE_OWNER_UNDEFINED, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 70ef051511bb1..4f4eb0791138a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -102,8 +102,8 @@ * The first chunk is the TBA used for the CWSR ISA code. The second * chunk is used as TMA for user-mode trap handler setup in daisy-chain mode. */ -#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2) -#define KFD_CWSR_TMA_OFFSET (PAGE_SIZE + 2048) +#define KFD_CWSR_TBA_TMA_SIZE (AMDGPU_GPU_PAGE_SIZE * 2) +#define KFD_CWSR_TMA_OFFSET (AMDGPU_GPU_PAGE_SIZE + 2048) #define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ (KFD_MAX_NUM_OF_PROCESSES * \ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index ddfe30c13e9d6..8ed513a77d38f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1992,7 +1992,7 @@ static int signal_eviction_fence(struct kfd_process *p) ef = dma_fence_get_rcu_safe(&p->ef); rcu_read_unlock(); if (!ef) - return -EINVAL; + return true; ret = dma_fence_signal(ef); dma_fence_put(ef); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 7fbb5c274ccc4..7bf712032c52c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -606,6 +606,7 @@ int pqm_update_queue_properties(struct process_queue_manager *pqm, p->queue_size)) { pr_debug("ring buf 0x%llx size 0x%llx not mapped on GPU\n", p->queue_address, p->queue_size); + amdgpu_bo_unreserve(vm->root.bo); return -EFAULT; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c index f1e7583650c41..b97f4a51db6e3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c @@ -275,8 +275,8 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope /* EOP buffer is not required for all ASICs */ if (properties->eop_ring_buffer_address) { - if (properties->eop_ring_buffer_size != topo_dev->node_props.eop_buffer_size) { - pr_debug("queue eop bo size 0x%x not equal to node eop buf size 0x%x\n", + if (properties->eop_ring_buffer_size < topo_dev->node_props.eop_buffer_size) { + pr_debug("queue eop bo size 0x%x is less than node eop buf size 0x%x\n", properties->eop_ring_buffer_size, topo_dev->node_props.eop_buffer_size); err = -EINVAL; @@ -284,7 +284,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope } err = kfd_queue_buffer_get(vm, (void *)properties->eop_ring_buffer_address, &properties->eop_buf_bo, - properties->eop_ring_buffer_size); + ALIGN(properties->eop_ring_buffer_size, PAGE_SIZE)); if (err) goto out_err_unreserve; } @@ -409,6 +409,7 @@ static u32 kfd_get_vgpr_size_per_cu(u32 gfxv) vgpr_size = 0x80000; else if (gfxv == 110000 || /* GFX_VERSION_PLUM_BONITO */ gfxv == 110001 || /* GFX_VERSION_WHEAT_NAS */ + gfxv == 110501 || /* GFX_VERSION_GFX1151 */ gfxv == 120000 || /* GFX_VERSION_GFX1200 */ gfxv == 120001) /* GFX_VERSION_GFX1201 */ vgpr_size = 0x60000; @@ -443,10 +444,11 @@ void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev) min(cu_num * 40, props->array_count / props->simd_arrays_per_engine * 512) : cu_num * 32; - wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props), PAGE_SIZE); + wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props), + AMDGPU_GPU_PAGE_SIZE); ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(gfxv) + 8; ctl_stack_size = ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size, - PAGE_SIZE); + AMDGPU_GPU_PAGE_SIZE); if ((gfxv / 10000 * 10000) == 100000) { /* HW design limits control stack size to 0x7000. @@ -458,7 +460,7 @@ void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev) props->ctl_stack_size = ctl_stack_size; props->debug_memory_size = ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN); - props->cwsr_size = ctl_stack_size + wg_data_size; + props->cwsr_size = ALIGN(ctl_stack_size + wg_data_size, PAGE_SIZE); if (gfxv == 80002) /* GFX_VERSION_TONGA */ props->eop_buffer_size = 0x8000; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c index a499449fcb068..d2bc169e84b0b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c @@ -312,7 +312,7 @@ void kfd_smi_event_queue_restore(struct kfd_node *node, pid_t pid) { kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_RESTORE, KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), pid, - node->id, 0)); + node->id, '0')); } void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 74a1d3e1d52be..6daa70ace261f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -33,6 +33,7 @@ #include "amdgpu_hmm.h" #include "amdgpu.h" #include "amdgpu_xgmi.h" +#include "amdgpu_reset.h" #include "kfd_priv.h" #include "kfd_svm.h" #include "kfd_migrate.h" @@ -1144,30 +1145,48 @@ static int svm_range_split_tail(struct svm_range *prange, uint64_t new_last, struct list_head *insert_list, struct list_head *remap_list) { + unsigned long last_align_down = ALIGN_DOWN(prange->last, 512); + unsigned long start_align = ALIGN(prange->start, 512); + bool huge_page_mapping = last_align_down > start_align; struct svm_range *tail = NULL; - int r = svm_range_split(prange, prange->start, new_last, &tail); + int r; - if (!r) { - list_add(&tail->list, insert_list); - if (!IS_ALIGNED(new_last + 1, 1UL << prange->granularity)) - list_add(&tail->update_list, remap_list); - } - return r; + r = svm_range_split(prange, prange->start, new_last, &tail); + + if (r) + return r; + + list_add(&tail->list, insert_list); + + if (huge_page_mapping && tail->start > start_align && + tail->start < last_align_down && (!IS_ALIGNED(tail->start, 512))) + list_add(&tail->update_list, remap_list); + + return 0; } static int svm_range_split_head(struct svm_range *prange, uint64_t new_start, struct list_head *insert_list, struct list_head *remap_list) { + unsigned long last_align_down = ALIGN_DOWN(prange->last, 512); + unsigned long start_align = ALIGN(prange->start, 512); + bool huge_page_mapping = last_align_down > start_align; struct svm_range *head = NULL; - int r = svm_range_split(prange, new_start, prange->last, &head); + int r; - if (!r) { - list_add(&head->list, insert_list); - if (!IS_ALIGNED(new_start, 1UL << prange->granularity)) - list_add(&head->update_list, remap_list); - } - return r; + r = svm_range_split(prange, new_start, prange->last, &head); + + if (r) + return r; + + list_add(&head->list, insert_list); + + if (huge_page_mapping && head->last + 1 > start_align && + head->last + 1 < last_align_down && (!IS_ALIGNED(head->last, 512))) + list_add(&head->update_list, remap_list); + + return 0; } static void @@ -2325,6 +2344,9 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) pr_debug("drain retry fault gpu %d svms %p\n", i, svms); + if (!down_read_trylock(&pdd->dev->adev->reset_domain->sem)) + continue; + amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, pdd->dev->adev->irq.retry_cam_enabled ? &pdd->dev->adev->irq.ih : @@ -2334,6 +2356,7 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, &pdd->dev->adev->irq.ih_soft); + up_read(&pdd->dev->adev->reset_domain->sem); pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms); } @@ -2517,7 +2540,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, adev = pdd->dev->adev; /* Check and drain ih1 ring if cam not available */ - if (adev->irq.ih1.ring_size) { + if (!adev->irq.retry_cam_enabled && adev->irq.ih1.ring_size) { ih = &adev->irq.ih1; checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); if (ih->rptr != checkpoint_wptr) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index 5c98746eb72df..3eb32d58a1200 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -491,6 +491,10 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, dev->node_props.num_sdma_queues_per_engine); sysfs_show_32bit_prop(buffer, offs, "num_cp_queues", dev->node_props.num_cp_queues); + sysfs_show_32bit_prop(buffer, offs, "cwsr_size", + dev->node_props.cwsr_size); + sysfs_show_32bit_prop(buffer, offs, "ctl_stack_size", + dev->node_props.ctl_stack_size); if (dev->gpu) { log_max_watch_addr = @@ -530,7 +534,9 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version", dev->gpu->kfd->sdma_fw_version); sysfs_show_64bit_prop(buffer, offs, "unique_id", - dev->gpu->xcp ? + dev->gpu->xcp && + (dev->gpu->xcp->xcp_mgr->mode != + AMDGPU_SPX_PARTITION_MODE) ? dev->gpu->xcp->unique_id : dev->gpu->adev->unique_id); sysfs_show_32bit_prop(buffer, offs, "num_xcc", diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 7fe40bbba2658..ae10e8365ae15 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -3407,7 +3407,17 @@ static int dm_resume(struct amdgpu_ip_block *ip_block) struct dc_commit_streams_params commit_params = {}; if (dm->dc->caps.ips_support) { + if (!amdgpu_in_reset(adev)) + mutex_lock(&dm->dc_lock); + + /* Need to set POWER_STATE_D0 first or it will not execute + * idle_power_optimizations command to DMUB. + */ + dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0); dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false); + + if (!amdgpu_in_reset(adev)) + mutex_unlock(&dm->dc_lock); } if (amdgpu_in_reset(adev)) { @@ -3812,8 +3822,9 @@ void amdgpu_dm_update_connector_after_detect( aconnector->dc_sink = sink; dc_sink_retain(aconnector->dc_sink); + drm_edid_free(aconnector->drm_edid); + aconnector->drm_edid = NULL; if (sink->dc_edid.length == 0) { - aconnector->drm_edid = NULL; hdmi_cec_unset_edid(aconnector); if (aconnector->dc_link->aux_mode) { drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux); @@ -5193,6 +5204,8 @@ amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector) struct amdgpu_dm_backlight_caps *caps; char bl_name[16]; int min, max; + int real_brightness; + int init_brightness; if (aconnector->bl_idx == -1) return; @@ -5217,6 +5230,8 @@ amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector) } else props.brightness = props.max_brightness = MAX_BACKLIGHT_LEVEL; + init_brightness = props.brightness; + if (caps->data_points && !(amdgpu_dc_debug_mask & DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE)) { drm_info(drm, "Using custom brightness curve\n"); props.scale = BACKLIGHT_SCALE_NON_LINEAR; @@ -5235,8 +5250,20 @@ amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector) if (IS_ERR(dm->backlight_dev[aconnector->bl_idx])) { drm_err(drm, "DM: Backlight registration failed!\n"); dm->backlight_dev[aconnector->bl_idx] = NULL; - } else + } else { + /* + * dm->brightness[x] can be inconsistent just after startup until + * ops.get_brightness is called. + */ + real_brightness = + amdgpu_dm_backlight_ops.get_brightness(dm->backlight_dev[aconnector->bl_idx]); + + if (real_brightness != init_brightness) { + dm->actual_brightness[aconnector->bl_idx] = real_brightness; + dm->brightness[aconnector->bl_idx] = real_brightness; + } drm_dbg_driver(drm, "DM: Registered Backlight device: %s\n", bl_name); + } } static int initialize_plane(struct amdgpu_display_manager *dm, @@ -5545,7 +5572,8 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev) if (psr_feature_enabled) { amdgpu_dm_set_psr_caps(link); - drm_info(adev_to_drm(adev), "PSR support %d, DC PSR ver %d, sink PSR ver %d DPCD caps 0x%x su_y_granularity %d\n", + drm_info(adev_to_drm(adev), "%s: PSR support %d, DC PSR ver %d, sink PSR ver %d DPCD caps 0x%x su_y_granularity %d\n", + aconnector->base.name, link->psr_settings.psr_feature_enabled, link->psr_settings.psr_version, link->dpcd_caps.psr_info.psr_version, @@ -7531,10 +7559,12 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector) drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr); /* Cancel and flush any pending HDMI HPD debounce work */ - cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work); - if (aconnector->hdmi_prev_sink) { - dc_sink_release(aconnector->hdmi_prev_sink); - aconnector->hdmi_prev_sink = NULL; + if (aconnector->hdmi_hpd_debounce_delay_ms) { + cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work); + if (aconnector->hdmi_prev_sink) { + dc_sink_release(aconnector->hdmi_prev_sink); + aconnector->hdmi_prev_sink = NULL; + } } if (aconnector->bl_idx != -1) { @@ -8698,9 +8728,18 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm, mutex_init(&aconnector->hpd_lock); mutex_init(&aconnector->handle_mst_msg_ready); - aconnector->hdmi_hpd_debounce_delay_ms = AMDGPU_DM_HDMI_HPD_DEBOUNCE_MS; - INIT_DELAYED_WORK(&aconnector->hdmi_hpd_debounce_work, hdmi_hpd_debounce_work); - aconnector->hdmi_prev_sink = NULL; + /* + * If HDMI HPD debounce delay is set, use the minimum between selected + * value and AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS + */ + if (amdgpu_hdmi_hpd_debounce_delay_ms) { + aconnector->hdmi_hpd_debounce_delay_ms = min(amdgpu_hdmi_hpd_debounce_delay_ms, + AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS); + INIT_DELAYED_WORK(&aconnector->hdmi_hpd_debounce_work, hdmi_hpd_debounce_work); + aconnector->hdmi_prev_sink = NULL; + } else { + aconnector->hdmi_hpd_debounce_delay_ms = 0; + } /* * configure support HPD hot plug connector_>polled default value is 0 @@ -10383,10 +10422,10 @@ static void dm_set_writeback(struct amdgpu_display_manager *dm, wb_info->dwb_params.capture_rate = dwb_capture_rate_0; - wb_info->dwb_params.scaler_taps.h_taps = 4; - wb_info->dwb_params.scaler_taps.v_taps = 4; - wb_info->dwb_params.scaler_taps.h_taps_c = 2; - wb_info->dwb_params.scaler_taps.v_taps_c = 2; + wb_info->dwb_params.scaler_taps.h_taps = 1; + wb_info->dwb_params.scaler_taps.v_taps = 1; + wb_info->dwb_params.scaler_taps.h_taps_c = 1; + wb_info->dwb_params.scaler_taps.v_taps_c = 1; wb_info->dwb_params.subsample_position = DWB_INTERSTITIAL_SUBSAMPLING; wb_info->mcif_buf_params.luma_pitch = afb->base.pitches[0]; @@ -10686,7 +10725,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) continue; } for (j = 0; j < status->plane_count; j++) - dummy_updates[j].surface = status->plane_states[0]; + dummy_updates[j].surface = status->plane_states[j]; sort(dummy_updates, status->plane_count, sizeof(*dummy_updates), dm_plane_layer_index_cmp, NULL); @@ -11402,6 +11441,8 @@ static bool should_reset_plane(struct drm_atomic_state *state, struct drm_crtc_state *old_crtc_state, *new_crtc_state; struct dm_crtc_state *old_dm_crtc_state, *new_dm_crtc_state; struct amdgpu_device *adev = drm_to_adev(plane->dev); + struct drm_connector_state *new_con_state; + struct drm_connector *connector; int i; /* @@ -11412,6 +11453,15 @@ static bool should_reset_plane(struct drm_atomic_state *state, state->allow_modeset) return true; + /* Check for writeback commit */ + for_each_new_connector_in_state(state, connector, new_con_state, i) { + if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) + continue; + + if (new_con_state->writeback_job) + return true; + } + if (amdgpu_in_reset(adev) && state->allow_modeset) return true; @@ -11588,7 +11638,7 @@ static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc, * check tiling flags when the FB doesn't have a modifier. */ if (!(fb->flags & DRM_MODE_FB_MODIFIERS)) { - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { + if (adev->family == AMDGPU_FAMILY_GC_12_0_0) { linear = AMDGPU_TILING_GET(afb->tiling_flags, GFX12_SWIZZLE_MODE) == 0; } else if (adev->family >= AMDGPU_FAMILY_AI) { linear = AMDGPU_TILING_GET(afb->tiling_flags, SWIZZLE_MODE) == 0; @@ -12010,10 +12060,9 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev, /* Overlay cursor not supported on HW before DCN * DCN401 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions - * as previous DCN generations, so enable native mode on DCN401 in addition to DCE + * as previous DCN generations, so enable native mode on DCN401 */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0 || - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { *cursor_mode = DM_CURSOR_NATIVE_MODE; return 0; } @@ -12220,6 +12269,11 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, } if (dc_resource_is_dsc_encoding_supported(dc)) { + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { + dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); + dm_new_crtc_state->mode_changed_independent_from_dsc = new_crtc_state->mode_changed; + } + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { if (drm_atomic_crtc_needs_modeset(new_crtc_state)) { ret = add_affected_mst_dsc_crtcs(state, crtc); @@ -12333,6 +12387,12 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, * need to be added for DC to not disable a plane by mistake */ if (dm_new_crtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { + drm_dbg(dev, "Overlay cursor not supported on DCE\n"); + ret = -EINVAL; + goto fail; + } + ret = drm_atomic_add_affected_planes(state, crtc); if (ret) goto fail; @@ -12810,7 +12870,7 @@ static void parse_edid_displayid_vrr(struct drm_connector *connector, u16 min_vfreq; u16 max_vfreq; - if (edid == NULL || edid->extensions == 0) + if (!edid || !edid->extensions) return; /* Find DisplayID extension */ @@ -12820,7 +12880,7 @@ static void parse_edid_displayid_vrr(struct drm_connector *connector, break; } - if (edid_ext == NULL) + if (i == edid->extensions) return; while (j < EDID_LENGTH) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index 8ca7389575989..cd362d22a2774 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -59,7 +59,10 @@ #define AMDGPU_HDR_MULT_DEFAULT (0x100000000LL) -#define AMDGPU_DM_HDMI_HPD_DEBOUNCE_MS 1500 +/* + * Maximum HDMI HPD debounce delay in milliseconds + */ +#define AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS 5000 /* #include "include/amdgpu_dal_power_if.h" #include "amdgpu_dm_irq.h" @@ -962,6 +965,7 @@ struct dm_crtc_state { bool freesync_vrr_info_changed; + bool mode_changed_independent_from_dsc; bool dsc_force_changed; bool vrr_supported; struct mod_freesync_config freesync_config; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 38f9ea313dcbb..2e7ee77c010e1 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -248,8 +248,6 @@ static void amdgpu_dm_crtc_vblank_control_worker(struct work_struct *work) struct vblank_control_work *vblank_work = container_of(work, struct vblank_control_work, work); struct amdgpu_display_manager *dm = vblank_work->dm; - struct amdgpu_device *adev = drm_to_adev(dm->ddev); - int r; mutex_lock(&dm->dc_lock); @@ -279,16 +277,7 @@ static void amdgpu_dm_crtc_vblank_control_worker(struct work_struct *work) if (dm->active_vblank_irq_count == 0) { dc_post_update_surfaces_to_stream(dm->dc); - - r = amdgpu_dpm_pause_power_profile(adev, true); - if (r) - dev_warn(adev->dev, "failed to set default power profile mode\n"); - dc_allow_idle_optimizations(dm->dc, true); - - r = amdgpu_dpm_pause_power_profile(adev, false); - if (r) - dev_warn(adev->dev, "failed to restore the power profile mode\n"); } mutex_unlock(&dm->dc_lock); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index dbd1da4d85d32..2e0895f4f9b10 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -884,28 +884,26 @@ struct dsc_mst_fairness_params { }; #if defined(CONFIG_DRM_AMD_DC_FP) -static uint64_t kbps_to_pbn(int kbps, bool is_peak_pbn) +static uint16_t get_fec_overhead_multiplier(struct dc_link *dc_link) { - uint64_t effective_kbps = (uint64_t)kbps; + u8 link_coding_cap; + uint16_t fec_overhead_multiplier_x1000 = PBN_FEC_OVERHEAD_MULTIPLIER_8B_10B; - if (is_peak_pbn) { // add 0.6% (1006/1000) overhead into effective kbps - effective_kbps *= 1006; - effective_kbps = div_u64(effective_kbps, 1000); - } + link_coding_cap = dc_link_dp_mst_decide_link_encoding_format(dc_link); + if (link_coding_cap == DP_128b_132b_ENCODING) + fec_overhead_multiplier_x1000 = PBN_FEC_OVERHEAD_MULTIPLIER_128B_132B; - return (uint64_t) DIV64_U64_ROUND_UP(effective_kbps * 64, (54 * 8 * 1000)); + return fec_overhead_multiplier_x1000; } -static uint32_t pbn_to_kbps(unsigned int pbn, bool with_margin) +static int kbps_to_peak_pbn(int kbps, uint16_t fec_overhead_multiplier_x1000) { - uint64_t pbn_effective = (uint64_t)pbn; - - if (with_margin) // deduct 0.6% (994/1000) overhead from effective pbn - pbn_effective *= (1000000 / PEAK_FACTOR_X1000); - else - pbn_effective *= 1000; + u64 peak_kbps = kbps; - return DIV_U64_ROUND_UP(pbn_effective * 8 * 54, 64); + peak_kbps *= 1006; + peak_kbps *= fec_overhead_multiplier_x1000; + peak_kbps = div_u64(peak_kbps, 1000 * 1000); + return (int) DIV64_U64_ROUND_UP(peak_kbps * 64, (54 * 8 * 1000)); } static void set_dsc_configs_from_fairness_vars(struct dsc_mst_fairness_params *params, @@ -976,7 +974,7 @@ static int bpp_x16_from_pbn(struct dsc_mst_fairness_params param, int pbn) dc_dsc_get_default_config_option(param.sink->ctx->dc, &dsc_options); dsc_options.max_target_bpp_limit_override_x16 = drm_connector->display_info.max_dsc_bpp * 16; - kbps = pbn_to_kbps(pbn, false); + kbps = div_u64((u64)pbn * 994 * 8 * 54, 64); dc_dsc_compute_config( param.sink->ctx->dc->res_pool->dscs[0], ¶m.sink->dsc_caps.dsc_dec_caps, @@ -1005,11 +1003,12 @@ static int increase_dsc_bpp(struct drm_atomic_state *state, int link_timeslots_used; int fair_pbn_alloc; int ret = 0; + uint16_t fec_overhead_multiplier_x1000 = get_fec_overhead_multiplier(dc_link); for (i = 0; i < count; i++) { if (vars[i + k].dsc_enabled) { initial_slack[i] = - kbps_to_pbn(params[i].bw_range.max_kbps, false) - vars[i + k].pbn; + kbps_to_peak_pbn(params[i].bw_range.max_kbps, fec_overhead_multiplier_x1000) - vars[i + k].pbn; bpp_increased[i] = false; remaining_to_increase += 1; } else { @@ -1105,6 +1104,7 @@ static int try_disable_dsc(struct drm_atomic_state *state, int next_index; int remaining_to_try = 0; int ret; + uint16_t fec_overhead_multiplier_x1000 = get_fec_overhead_multiplier(dc_link); int var_pbn; for (i = 0; i < count; i++) { @@ -1137,7 +1137,7 @@ static int try_disable_dsc(struct drm_atomic_state *state, DRM_DEBUG_DRIVER("MST_DSC index #%d, try no compression\n", next_index); var_pbn = vars[next_index].pbn; - vars[next_index].pbn = kbps_to_pbn(params[next_index].bw_range.stream_kbps, true); + vars[next_index].pbn = kbps_to_peak_pbn(params[next_index].bw_range.stream_kbps, fec_overhead_multiplier_x1000); ret = drm_dp_atomic_find_time_slots(state, params[next_index].port->mgr, params[next_index].port, @@ -1197,6 +1197,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state, int count = 0; int i, k, ret; bool debugfs_overwrite = false; + uint16_t fec_overhead_multiplier_x1000 = get_fec_overhead_multiplier(dc_link); struct drm_connector_state *new_conn_state; memset(params, 0, sizeof(params)); @@ -1277,7 +1278,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state, DRM_DEBUG_DRIVER("MST_DSC Try no compression\n"); for (i = 0; i < count; i++) { vars[i + k].aconnector = params[i].aconnector; - vars[i + k].pbn = kbps_to_pbn(params[i].bw_range.stream_kbps, false); + vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps, fec_overhead_multiplier_x1000); vars[i + k].dsc_enabled = false; vars[i + k].bpp_x16 = 0; ret = drm_dp_atomic_find_time_slots(state, params[i].port->mgr, params[i].port, @@ -1299,7 +1300,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state, DRM_DEBUG_DRIVER("MST_DSC Try max compression\n"); for (i = 0; i < count; i++) { if (params[i].compression_possible && params[i].clock_force_enable != DSC_CLK_FORCE_DISABLE) { - vars[i + k].pbn = kbps_to_pbn(params[i].bw_range.min_kbps, false); + vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.min_kbps, fec_overhead_multiplier_x1000); vars[i + k].dsc_enabled = true; vars[i + k].bpp_x16 = params[i].bw_range.min_target_bpp_x16; ret = drm_dp_atomic_find_time_slots(state, params[i].port->mgr, @@ -1307,7 +1308,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state, if (ret < 0) return ret; } else { - vars[i + k].pbn = kbps_to_pbn(params[i].bw_range.stream_kbps, false); + vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps, fec_overhead_multiplier_x1000); vars[i + k].dsc_enabled = false; vars[i + k].bpp_x16 = 0; ret = drm_dp_atomic_find_time_slots(state, params[i].port->mgr, @@ -1743,9 +1744,11 @@ int pre_validate_dsc(struct drm_atomic_state *state, int ind = find_crtc_index_in_state_by_stream(state, stream); if (ind >= 0) { + struct dm_crtc_state *dm_new_crtc_state = to_dm_crtc_state(state->crtcs[ind].new_state); + DRM_INFO_ONCE("%s:%d MST_DSC no mode changed for stream 0x%p\n", __func__, __LINE__, stream); - state->crtcs[ind].new_state->mode_changed = 0; + dm_new_crtc_state->base.mode_changed = dm_new_crtc_state->mode_changed_independent_from_dsc; } } } @@ -1762,6 +1765,18 @@ int pre_validate_dsc(struct drm_atomic_state *state, return ret; } +static uint32_t kbps_from_pbn(unsigned int pbn) +{ + uint64_t kbps = (uint64_t)pbn; + + kbps *= (1000000 / PEAK_FACTOR_X1000); + kbps *= 8; + kbps *= 54; + kbps /= 64; + + return (uint32_t)kbps; +} + static bool is_dsc_common_config_possible(struct dc_stream_state *stream, struct dc_dsc_bw_range *bw_range) { @@ -1860,7 +1875,7 @@ enum dc_status dm_dp_mst_is_port_support_mode( dc_link_get_highest_encoding_format(stream->link)); cur_link_settings = stream->link->verified_link_cap; root_link_bw_in_kbps = dc_link_bandwidth_kbps(aconnector->dc_link, &cur_link_settings); - virtual_channel_bw_in_kbps = pbn_to_kbps(aconnector->mst_output_port->full_pbn, true); + virtual_channel_bw_in_kbps = kbps_from_pbn(aconnector->mst_output_port->full_pbn); /* pick the end to end bw bottleneck */ end_to_end_bw_in_kbps = min(root_link_bw_in_kbps, virtual_channel_bw_in_kbps); @@ -1913,7 +1928,7 @@ enum dc_status dm_dp_mst_is_port_support_mode( immediate_upstream_port = aconnector->mst_output_port->parent->port_parent; if (immediate_upstream_port) { - virtual_channel_bw_in_kbps = pbn_to_kbps(immediate_upstream_port->full_pbn, true); + virtual_channel_bw_in_kbps = kbps_from_pbn(immediate_upstream_port->full_pbn); virtual_channel_bw_in_kbps = min(root_link_bw_in_kbps, virtual_channel_bw_in_kbps); } else { /* For topology LCT 1 case - only one mstb*/ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index e027798ece032..2ecebf9a00fa4 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -277,7 +277,7 @@ static int amdgpu_dm_plane_validate_dcc(struct amdgpu_device *adev, if (!dcc->enable) return 0; - if (adev->family < AMDGPU_FAMILY_GC_12_0_0 && + if (adev->family != AMDGPU_FAMILY_GC_12_0_0 && format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) return -EINVAL; @@ -900,7 +900,7 @@ int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev, upper_32_bits(chroma_addr); } - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { + if (adev->family == AMDGPU_FAMILY_GC_12_0_0) { ret = amdgpu_dm_plane_fill_gfx12_plane_attributes_from_modifiers(adev, afb, format, rotation, plane_size, tiling_info, dcc, @@ -1059,10 +1059,15 @@ static void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev, *min_downscale = plane_cap->max_downscale_factor.nv12; break; + /* All 64 bpp formats have the same fp16 scaling limits */ case DRM_FORMAT_XRGB16161616F: case DRM_FORMAT_ARGB16161616F: case DRM_FORMAT_XBGR16161616F: case DRM_FORMAT_ABGR16161616F: + case DRM_FORMAT_XRGB16161616: + case DRM_FORMAT_ARGB16161616: + case DRM_FORMAT_XBGR16161616: + case DRM_FORMAT_ABGR16161616: *max_upscale = plane_cap->max_upscale_factor.fp16; *min_downscale = plane_cap->max_downscale_factor.fp16; break; diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c index 04eb647acc4e1..550a9f1d03f82 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -1480,10 +1480,10 @@ static enum bp_result get_embedded_panel_info_v2_1( /* not provided by VBIOS */ info->lcd_timing.misc_info.HORIZONTAL_CUT_OFF = 0; - info->lcd_timing.misc_info.H_SYNC_POLARITY = ~(uint32_t) (lvds->lcd_timing.miscinfo - & ATOM_HSYNC_POLARITY); - info->lcd_timing.misc_info.V_SYNC_POLARITY = ~(uint32_t) (lvds->lcd_timing.miscinfo - & ATOM_VSYNC_POLARITY); + info->lcd_timing.misc_info.H_SYNC_POLARITY = !(lvds->lcd_timing.miscinfo & + ATOM_HSYNC_POLARITY); + info->lcd_timing.misc_info.V_SYNC_POLARITY = !(lvds->lcd_timing.miscinfo & + ATOM_VSYNC_POLARITY); /* not provided by VBIOS */ info->lcd_timing.misc_info.VERTICAL_CUT_OFF = 0; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c index 15cf13ec53026..c450feae5fa5b 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c @@ -255,6 +255,10 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p BREAK_TO_DEBUGGER(); return NULL; } + if (ctx->dce_version == DCN_VERSION_2_01) { + dcn201_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); + return &clk_mgr->base; + } if (ASICREV_IS_SIENNA_CICHLID_P(asic_id.hw_internal_rev)) { dcn3_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); return &clk_mgr->base; @@ -267,10 +271,6 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p dcn3_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); return &clk_mgr->base; } - if (ctx->dce_version == DCN_VERSION_2_01) { - dcn201_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); - return &clk_mgr->base; - } dcn20_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg); return &clk_mgr->base; } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c index b315ed91e010b..c49268db85f68 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c @@ -138,7 +138,7 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, if (dc->work_arounds.skip_clock_update) return; - clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; + display_count = dcn315_get_active_display_cnt_wa(dc, context); /* * if it is safe to lower, but we are already in the lower state, we don't have to do anything * also if safe to lower is false, we just go in the higher state @@ -151,7 +151,6 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, } /* check that we're not already in lower */ if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_LOW_POWER) { - display_count = dcn315_get_active_display_cnt_wa(dc, context); /* if we can go lower, go lower */ if (display_count == 0) { union display_idle_optimization_u idle_info = { 0 }; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c index 1eb04772f5da2..817a0253d10e5 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c @@ -768,32 +768,32 @@ static struct wm_table ddr5_wm_table = { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index 0a46e834357a1..0886bef32a5de 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -169,7 +169,7 @@ struct dc_stream_state *dc_create_stream_for_sink( if (sink == NULL) return NULL; - stream = kzalloc(sizeof(struct dc_stream_state), GFP_KERNEL); + stream = kzalloc(sizeof(struct dc_stream_state), GFP_ATOMIC); if (stream == NULL) goto alloc_fail; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c index 922f23557f5d9..0971dfa258454 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c @@ -86,7 +86,7 @@ uint8_t dc_plane_get_pipe_mask(struct dc_state *dc_state, const struct dc_plane struct dc_plane_state *dc_create_plane_state(const struct dc *dc) { struct dc_plane_state *plane_state = kvzalloc(sizeof(*plane_state), - GFP_KERNEL); + GFP_ATOMIC); if (NULL == plane_state) return NULL; diff --git a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h index b015e80672ec9..fcd3ab4b00459 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h @@ -41,7 +41,7 @@ /* kHZ*/ #define DP_ADAPTOR_DVI_MAX_TMDS_CLK 165000 /* kHZ*/ -#define DP_ADAPTOR_HDMI_SAFE_MAX_TMDS_CLK 165000 +#define DP_ADAPTOR_HDMI_SAFE_MAX_TMDS_CLK 340000 struct dp_hdmi_dongle_signature_data { int8_t id[15];/* "DP-HDMI ADAPTOR"*/ diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c index 0690c346f2c52..227aa8672d17b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c @@ -105,9 +105,12 @@ void cm_helper_program_gamcor_xfer_func( #define NUMBER_REGIONS 32 #define NUMBER_SW_SEGMENTS 16 -bool cm3_helper_translate_curve_to_hw_format( - const struct dc_transfer_func *output_tf, - struct pwl_params *lut_params, bool fixpoint) +#define DC_LOGGER \ + ctx->logger + +bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx, + const struct dc_transfer_func *output_tf, + struct pwl_params *lut_params, bool fixpoint) { struct curve_points3 *corner_points; struct pwl_result_data *rgb_resulted; @@ -163,6 +166,11 @@ bool cm3_helper_translate_curve_to_hw_format( hw_points += (1 << seg_distr[k]); } + // DCN3+ have 257 pts in lieu of no separate slope registers + // Prior HW had 256 base+slope pairs + // Shaper LUT (i.e. fixpoint == true) is still 256 bases and 256 deltas + hw_points = fixpoint ? (hw_points - 1) : hw_points; + j = 0; for (k = 0; k < (region_end - region_start); k++) { increment = NUMBER_SW_SEGMENTS / (1 << seg_distr[k]); @@ -223,8 +231,6 @@ bool cm3_helper_translate_curve_to_hw_format( corner_points[1].green.slope = dc_fixpt_zero; corner_points[1].blue.slope = dc_fixpt_zero; - // DCN3+ have 257 pts in lieu of no separate slope registers - // Prior HW had 256 base+slope pairs lut_params->hw_points_num = hw_points + 1; k = 0; @@ -248,6 +254,10 @@ bool cm3_helper_translate_curve_to_hw_format( if (fixpoint == true) { i = 1; while (i != hw_points + 2) { + uint32_t red_clamp; + uint32_t green_clamp; + uint32_t blue_clamp; + if (i >= hw_points) { if (dc_fixpt_lt(rgb_plus_1->red, rgb->red)) rgb_plus_1->red = dc_fixpt_add(rgb->red, @@ -260,9 +270,20 @@ bool cm3_helper_translate_curve_to_hw_format( rgb_minus_1->delta_blue); } - rgb->delta_red_reg = dc_fixpt_clamp_u0d10(rgb->delta_red); - rgb->delta_green_reg = dc_fixpt_clamp_u0d10(rgb->delta_green); - rgb->delta_blue_reg = dc_fixpt_clamp_u0d10(rgb->delta_blue); + rgb->delta_red = dc_fixpt_sub(rgb_plus_1->red, rgb->red); + rgb->delta_green = dc_fixpt_sub(rgb_plus_1->green, rgb->green); + rgb->delta_blue = dc_fixpt_sub(rgb_plus_1->blue, rgb->blue); + + red_clamp = dc_fixpt_clamp_u0d14(rgb->delta_red); + green_clamp = dc_fixpt_clamp_u0d14(rgb->delta_green); + blue_clamp = dc_fixpt_clamp_u0d14(rgb->delta_blue); + + if (red_clamp >> 10 || green_clamp >> 10 || blue_clamp >> 10) + DC_LOG_ERROR("Losing delta precision while programming shaper LUT."); + + rgb->delta_red_reg = red_clamp & 0x3ff; + rgb->delta_green_reg = green_clamp & 0x3ff; + rgb->delta_blue_reg = blue_clamp & 0x3ff; rgb->red_reg = dc_fixpt_clamp_u0d14(rgb->red); rgb->green_reg = dc_fixpt_clamp_u0d14(rgb->green); rgb->blue_reg = dc_fixpt_clamp_u0d14(rgb->blue); diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c index 06907e8a4eda1..ddc736af776c9 100644 --- a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c @@ -188,9 +188,18 @@ void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc, if (!query_dp_alt_from_dmub(enc, &cmd)) return; - if (cmd.query_dp_alt.data.is_usb && - cmd.query_dp_alt.data.is_dp4 == 0) - link_settings->lane_count = MIN(LANE_COUNT_TWO, link_settings->lane_count); + /* + * USB-C DisplayPort Alt Mode lane count limitation logic: + * When USB and DP share the same USB-C connector, hardware must allocate + * some lanes for USB data, limiting DP to maximum 2 lanes instead of 4. + * This ensures USB functionality remains available while DP is active. + */ + if (cmd.query_dp_alt.data.is_dp_alt_disable == 0 && + cmd.query_dp_alt.data.is_usb && + cmd.query_dp_alt.data.is_dp4 == 0) { + link_settings->lane_count = + MIN(LANE_COUNT_TWO, link_settings->lane_count); + } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index b357683b4255a..268b5fbdb48bd 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -30,7 +30,11 @@ dml_rcflags := $(CC_FLAGS_NO_FPU) ifneq ($(CONFIG_FRAME_WARN),0) ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y) - frame_warn_limit := 3072 + ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_COMPILE_TEST),yy) + frame_warn_limit := 4096 + else + frame_warn_limit := 3072 + endif else frame_warn_limit := 2048 endif diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c index 8d24763938ea6..2d19bb8de59c8 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c @@ -77,32 +77,14 @@ static unsigned int dscceComputeDelay( static unsigned int dscComputeDelay( enum output_format_class pixelFormat, enum output_encoder_class Output); -// Super monster function with some 45 argument static bool CalculatePrefetchSchedule( struct display_mode_lib *mode_lib, - double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData, - double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly, + unsigned int k, Pipe *myPipe, unsigned int DSCDelay, - double DPPCLKDelaySubtotalPlusCNVCFormater, - double DPPCLKDelaySCL, - double DPPCLKDelaySCLLBOnly, - double DPPCLKDelayCNVCCursor, - double DISPCLKDelaySubtotal, unsigned int DPP_RECOUT_WIDTH, - enum output_format_class OutputFormat, - unsigned int MaxInterDCNTileRepeaters, unsigned int VStartup, unsigned int MaxVStartup, - unsigned int GPUVMPageTableLevels, - bool GPUVMEnable, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - double HostVMMinPageSize, - bool DynamicMetadataEnable, - bool DynamicMetadataVMEnabled, - int DynamicMetadataLinesBeforeActiveRequired, - unsigned int DynamicMetadataTransmittedBytes, double UrgentLatency, double UrgentExtraLatency, double TCalc, @@ -116,7 +98,6 @@ static bool CalculatePrefetchSchedule( unsigned int MaxNumSwathY, double PrefetchSourceLinesC, unsigned int SwathWidthC, - int BytePerPixelC, double VInitPreFillC, unsigned int MaxNumSwathC, long swath_width_luma_ub, @@ -124,9 +105,6 @@ static bool CalculatePrefetchSchedule( unsigned int SwathHeightY, unsigned int SwathHeightC, double TWait, - bool ProgressiveToInterlaceUnitInOPP, - double *DSTXAfterScaler, - double *DSTYAfterScaler, double *DestinationLinesForPrefetch, double *PrefetchBandwidth, double *DestinationLinesToRequestVMInVBlank, @@ -135,14 +113,7 @@ static bool CalculatePrefetchSchedule( double *VRatioPrefetchC, double *RequiredPrefetchPixDataBWLuma, double *RequiredPrefetchPixDataBWChroma, - bool *NotEnoughTimeForDynamicMetadata, - double *Tno_bw, - double *prefetch_vmrow_bw, - double *Tdmdl_vm, - double *Tdmdl, - unsigned int *VUpdateOffsetPix, - double *VUpdateWidthPix, - double *VReadyOffsetPix); + bool *NotEnoughTimeForDynamicMetadata); static double RoundToDFSGranularityUp(double Clock, double VCOSpeed); static double RoundToDFSGranularityDown(double Clock, double VCOSpeed); static void CalculateDCCConfiguration( @@ -810,29 +781,12 @@ static unsigned int dscComputeDelay(enum output_format_class pixelFormat, enum o static bool CalculatePrefetchSchedule( struct display_mode_lib *mode_lib, - double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData, - double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly, + unsigned int k, Pipe *myPipe, unsigned int DSCDelay, - double DPPCLKDelaySubtotalPlusCNVCFormater, - double DPPCLKDelaySCL, - double DPPCLKDelaySCLLBOnly, - double DPPCLKDelayCNVCCursor, - double DISPCLKDelaySubtotal, unsigned int DPP_RECOUT_WIDTH, - enum output_format_class OutputFormat, - unsigned int MaxInterDCNTileRepeaters, unsigned int VStartup, unsigned int MaxVStartup, - unsigned int GPUVMPageTableLevels, - bool GPUVMEnable, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - double HostVMMinPageSize, - bool DynamicMetadataEnable, - bool DynamicMetadataVMEnabled, - int DynamicMetadataLinesBeforeActiveRequired, - unsigned int DynamicMetadataTransmittedBytes, double UrgentLatency, double UrgentExtraLatency, double TCalc, @@ -846,7 +800,6 @@ static bool CalculatePrefetchSchedule( unsigned int MaxNumSwathY, double PrefetchSourceLinesC, unsigned int SwathWidthC, - int BytePerPixelC, double VInitPreFillC, unsigned int MaxNumSwathC, long swath_width_luma_ub, @@ -854,9 +807,6 @@ static bool CalculatePrefetchSchedule( unsigned int SwathHeightY, unsigned int SwathHeightC, double TWait, - bool ProgressiveToInterlaceUnitInOPP, - double *DSTXAfterScaler, - double *DSTYAfterScaler, double *DestinationLinesForPrefetch, double *PrefetchBandwidth, double *DestinationLinesToRequestVMInVBlank, @@ -865,15 +815,10 @@ static bool CalculatePrefetchSchedule( double *VRatioPrefetchC, double *RequiredPrefetchPixDataBWLuma, double *RequiredPrefetchPixDataBWChroma, - bool *NotEnoughTimeForDynamicMetadata, - double *Tno_bw, - double *prefetch_vmrow_bw, - double *Tdmdl_vm, - double *Tdmdl, - unsigned int *VUpdateOffsetPix, - double *VUpdateWidthPix, - double *VReadyOffsetPix) + bool *NotEnoughTimeForDynamicMetadata) { + struct vba_vars_st *v = &mode_lib->vba; + double DPPCLKDelaySubtotalPlusCNVCFormater = v->DPPCLKDelaySubtotal + v->DPPCLKDelayCNVCFormater; bool MyError = false; unsigned int DPPCycles = 0, DISPCLKCycles = 0; double DSTTotalPixelsAfterScaler = 0; @@ -905,26 +850,26 @@ static bool CalculatePrefetchSchedule( double Tdmec = 0; double Tdmsks = 0; - if (GPUVMEnable == true && HostVMEnable == true) { - HostVMInefficiencyFactor = PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData / PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly; - HostVMDynamicLevelsTrips = HostVMMaxNonCachedPageTableLevels; + if (v->GPUVMEnable == true && v->HostVMEnable == true) { + HostVMInefficiencyFactor = v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData / v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly; + HostVMDynamicLevelsTrips = v->HostVMMaxNonCachedPageTableLevels; } else { HostVMInefficiencyFactor = 1; HostVMDynamicLevelsTrips = 0; } CalculateDynamicMetadataParameters( - MaxInterDCNTileRepeaters, + v->MaxInterDCNTileRepeaters, myPipe->DPPCLK, myPipe->DISPCLK, myPipe->DCFCLKDeepSleep, myPipe->PixelClock, myPipe->HTotal, myPipe->VBlank, - DynamicMetadataTransmittedBytes, - DynamicMetadataLinesBeforeActiveRequired, + v->DynamicMetadataTransmittedBytes[k], + v->DynamicMetadataLinesBeforeActiveRequired[k], myPipe->InterlaceEnable, - ProgressiveToInterlaceUnitInOPP, + v->ProgressiveToInterlaceUnitInOPP, &Tsetup, &Tdmbf, &Tdmec, @@ -932,16 +877,16 @@ static bool CalculatePrefetchSchedule( LineTime = myPipe->HTotal / myPipe->PixelClock; trip_to_mem = UrgentLatency; - Tvm_trips = UrgentExtraLatency + trip_to_mem * (GPUVMPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1); + Tvm_trips = UrgentExtraLatency + trip_to_mem * (v->GPUVMMaxPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1); - if (DynamicMetadataVMEnabled == true && GPUVMEnable == true) { - *Tdmdl = TWait + Tvm_trips + trip_to_mem; + if (v->DynamicMetadataVMEnabled == true && v->GPUVMEnable == true) { + v->Tdmdl[k] = TWait + Tvm_trips + trip_to_mem; } else { - *Tdmdl = TWait + UrgentExtraLatency; + v->Tdmdl[k] = TWait + UrgentExtraLatency; } - if (DynamicMetadataEnable == true) { - if (VStartup * LineTime < Tsetup + *Tdmdl + Tdmbf + Tdmec + Tdmsks) { + if (v->DynamicMetadataEnable[k] == true) { + if (VStartup * LineTime < Tsetup + v->Tdmdl[k] + Tdmbf + Tdmec + Tdmsks) { *NotEnoughTimeForDynamicMetadata = true; } else { *NotEnoughTimeForDynamicMetadata = false; @@ -949,39 +894,39 @@ static bool CalculatePrefetchSchedule( dml_print("DML: Tdmbf: %fus - time for dmd transfer from dchub to dio output buffer\n", Tdmbf); dml_print("DML: Tdmec: %fus - time dio takes to transfer dmd\n", Tdmec); dml_print("DML: Tdmsks: %fus - time before active dmd must complete transmission at dio\n", Tdmsks); - dml_print("DML: Tdmdl: %fus - time for fabric to become ready and fetch dmd \n", *Tdmdl); + dml_print("DML: Tdmdl: %fus - time for fabric to become ready and fetch dmd \n", v->Tdmdl[k]); } } else { *NotEnoughTimeForDynamicMetadata = false; } - *Tdmdl_vm = (DynamicMetadataEnable == true && DynamicMetadataVMEnabled == true && GPUVMEnable == true ? TWait + Tvm_trips : 0); + v->Tdmdl_vm[k] = (v->DynamicMetadataEnable[k] == true && v->DynamicMetadataVMEnabled == true && v->GPUVMEnable == true ? TWait + Tvm_trips : 0); if (myPipe->ScalerEnabled) - DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCL; + DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCL; else - DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCLLBOnly; + DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCLLBOnly; - DPPCycles = DPPCycles + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor; + DPPCycles = DPPCycles + myPipe->NumberOfCursors * v->DPPCLKDelayCNVCCursor; - DISPCLKCycles = DISPCLKDelaySubtotal; + DISPCLKCycles = v->DISPCLKDelaySubtotal; if (myPipe->DPPCLK == 0.0 || myPipe->DISPCLK == 0.0) return true; - *DSTXAfterScaler = DPPCycles * myPipe->PixelClock / myPipe->DPPCLK + DISPCLKCycles * myPipe->PixelClock / myPipe->DISPCLK + v->DSTXAfterScaler[k] = DPPCycles * myPipe->PixelClock / myPipe->DPPCLK + DISPCLKCycles * myPipe->PixelClock / myPipe->DISPCLK + DSCDelay; - *DSTXAfterScaler = *DSTXAfterScaler + ((myPipe->ODMCombineEnabled)?18:0) + (myPipe->DPPPerPlane - 1) * DPP_RECOUT_WIDTH; + v->DSTXAfterScaler[k] = v->DSTXAfterScaler[k] + ((myPipe->ODMCombineEnabled)?18:0) + (myPipe->DPPPerPlane - 1) * DPP_RECOUT_WIDTH; - if (OutputFormat == dm_420 || (myPipe->InterlaceEnable && ProgressiveToInterlaceUnitInOPP)) - *DSTYAfterScaler = 1; + if (v->OutputFormat[k] == dm_420 || (myPipe->InterlaceEnable && v->ProgressiveToInterlaceUnitInOPP)) + v->DSTYAfterScaler[k] = 1; else - *DSTYAfterScaler = 0; + v->DSTYAfterScaler[k] = 0; - DSTTotalPixelsAfterScaler = *DSTYAfterScaler * myPipe->HTotal + *DSTXAfterScaler; - *DSTYAfterScaler = dml_floor(DSTTotalPixelsAfterScaler / myPipe->HTotal, 1); - *DSTXAfterScaler = DSTTotalPixelsAfterScaler - ((double) (*DSTYAfterScaler * myPipe->HTotal)); + DSTTotalPixelsAfterScaler = v->DSTYAfterScaler[k] * myPipe->HTotal + v->DSTXAfterScaler[k]; + v->DSTYAfterScaler[k] = dml_floor(DSTTotalPixelsAfterScaler / myPipe->HTotal, 1); + v->DSTXAfterScaler[k] = DSTTotalPixelsAfterScaler - ((double) (v->DSTYAfterScaler[k] * myPipe->HTotal)); MyError = false; @@ -990,33 +935,33 @@ static bool CalculatePrefetchSchedule( Tvm_trips_rounded = dml_ceil(4.0 * Tvm_trips / LineTime, 1) / 4 * LineTime; Tr0_trips_rounded = dml_ceil(4.0 * Tr0_trips / LineTime, 1) / 4 * LineTime; - if (GPUVMEnable) { - if (GPUVMPageTableLevels >= 3) { - *Tno_bw = UrgentExtraLatency + trip_to_mem * ((GPUVMPageTableLevels - 2) - 1); + if (v->GPUVMEnable) { + if (v->GPUVMMaxPageTableLevels >= 3) { + v->Tno_bw[k] = UrgentExtraLatency + trip_to_mem * ((v->GPUVMMaxPageTableLevels - 2) - 1); } else - *Tno_bw = 0; + v->Tno_bw[k] = 0; } else if (!myPipe->DCCEnable) - *Tno_bw = LineTime; + v->Tno_bw[k] = LineTime; else - *Tno_bw = LineTime / 4; + v->Tno_bw[k] = LineTime / 4; - dst_y_prefetch_equ = VStartup - (Tsetup + dml_max(TWait + TCalc, *Tdmdl)) / LineTime - - (*DSTYAfterScaler + *DSTXAfterScaler / myPipe->HTotal); + dst_y_prefetch_equ = VStartup - (Tsetup + dml_max(TWait + TCalc, v->Tdmdl[k])) / LineTime + - (v->DSTYAfterScaler[k] + v->DSTXAfterScaler[k] / myPipe->HTotal); dst_y_prefetch_equ = dml_min(dst_y_prefetch_equ, 63.75); // limit to the reg limit of U6.2 for DST_Y_PREFETCH Lsw_oto = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC); Tsw_oto = Lsw_oto * LineTime; - prefetch_bw_oto = (PrefetchSourceLinesY * swath_width_luma_ub * BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * BytePerPixelC) / Tsw_oto; + prefetch_bw_oto = (PrefetchSourceLinesY * swath_width_luma_ub * BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * v->BytePerPixelC[k]) / Tsw_oto; - if (GPUVMEnable == true) { - Tvm_oto = dml_max3(*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_oto, + if (v->GPUVMEnable == true) { + Tvm_oto = dml_max3(v->Tno_bw[k] + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_oto, Tvm_trips, LineTime / 4.0); } else Tvm_oto = LineTime / 4.0; - if ((GPUVMEnable == true || myPipe->DCCEnable == true)) { + if ((v->GPUVMEnable == true || myPipe->DCCEnable == true)) { Tr0_oto = dml_max3( (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / prefetch_bw_oto, LineTime - Tvm_oto, LineTime / 4); @@ -1042,10 +987,10 @@ static bool CalculatePrefetchSchedule( dml_print("DML: Tdmbf: %fus - time for dmd transfer from dchub to dio output buffer\n", Tdmbf); dml_print("DML: Tdmec: %fus - time dio takes to transfer dmd\n", Tdmec); dml_print("DML: Tdmsks: %fus - time before active dmd must complete transmission at dio\n", Tdmsks); - dml_print("DML: Tdmdl_vm: %fus - time for vm stages of dmd \n", *Tdmdl_vm); - dml_print("DML: Tdmdl: %fus - time for fabric to become ready and fetch dmd \n", *Tdmdl); - dml_print("DML: dst_x_after_scl: %f pixels - number of pixel clocks pipeline and buffer delay after scaler \n", *DSTXAfterScaler); - dml_print("DML: dst_y_after_scl: %d lines - number of lines of pipeline and buffer delay after scaler \n", (int)*DSTYAfterScaler); + dml_print("DML: Tdmdl_vm: %fus - time for vm stages of dmd \n", v->Tdmdl_vm[k]); + dml_print("DML: Tdmdl: %fus - time for fabric to become ready and fetch dmd \n", v->Tdmdl[k]); + dml_print("DML: dst_x_after_scl: %f pixels - number of pixel clocks pipeline and buffer delay after scaler \n", v->DSTXAfterScaler[k]); + dml_print("DML: dst_y_after_scl: %d lines - number of lines of pipeline and buffer delay after scaler \n", (int)v->DSTYAfterScaler[k]); *PrefetchBandwidth = 0; *DestinationLinesToRequestVMInVBlank = 0; @@ -1059,26 +1004,26 @@ static bool CalculatePrefetchSchedule( double PrefetchBandwidth3 = 0; double PrefetchBandwidth4 = 0; - if (Tpre_rounded - *Tno_bw > 0) + if (Tpre_rounded - v->Tno_bw[k] > 0) PrefetchBandwidth1 = (PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor + 2 * MetaRowByte + 2 * PixelPTEBytesPerRow * HostVMInefficiencyFactor + PrefetchSourceLinesY * swath_width_luma_ub * BytePerPixelY - + PrefetchSourceLinesC * swath_width_chroma_ub * BytePerPixelC) - / (Tpre_rounded - *Tno_bw); + + PrefetchSourceLinesC * swath_width_chroma_ub * v->BytePerPixelC[k]) + / (Tpre_rounded - v->Tno_bw[k]); else PrefetchBandwidth1 = 0; - if (VStartup == MaxVStartup && (PrefetchBandwidth1 > 4 * prefetch_bw_oto) && (Tpre_rounded - Tsw_oto / 4 - 0.75 * LineTime - *Tno_bw) > 0) { - PrefetchBandwidth1 = (PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor + 2 * MetaRowByte + 2 * PixelPTEBytesPerRow * HostVMInefficiencyFactor) / (Tpre_rounded - Tsw_oto / 4 - 0.75 * LineTime - *Tno_bw); + if (VStartup == MaxVStartup && (PrefetchBandwidth1 > 4 * prefetch_bw_oto) && (Tpre_rounded - Tsw_oto / 4 - 0.75 * LineTime - v->Tno_bw[k]) > 0) { + PrefetchBandwidth1 = (PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor + 2 * MetaRowByte + 2 * PixelPTEBytesPerRow * HostVMInefficiencyFactor) / (Tpre_rounded - Tsw_oto / 4 - 0.75 * LineTime - v->Tno_bw[k]); } - if (Tpre_rounded - *Tno_bw - 2 * Tr0_trips_rounded > 0) + if (Tpre_rounded - v->Tno_bw[k] - 2 * Tr0_trips_rounded > 0) PrefetchBandwidth2 = (PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor + PrefetchSourceLinesY * swath_width_luma_ub * BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * - BytePerPixelC) / - (Tpre_rounded - *Tno_bw - 2 * Tr0_trips_rounded); + v->BytePerPixelC[k]) / + (Tpre_rounded - v->Tno_bw[k] - 2 * Tr0_trips_rounded); else PrefetchBandwidth2 = 0; @@ -1086,7 +1031,7 @@ static bool CalculatePrefetchSchedule( PrefetchBandwidth3 = (2 * MetaRowByte + 2 * PixelPTEBytesPerRow * HostVMInefficiencyFactor + PrefetchSourceLinesY * swath_width_luma_ub * BytePerPixelY + PrefetchSourceLinesC * - swath_width_chroma_ub * BytePerPixelC) / (Tpre_rounded - + swath_width_chroma_ub * v->BytePerPixelC[k]) / (Tpre_rounded - Tvm_trips_rounded); else PrefetchBandwidth3 = 0; @@ -1096,7 +1041,7 @@ static bool CalculatePrefetchSchedule( } if (Tpre_rounded - Tvm_trips_rounded - 2 * Tr0_trips_rounded > 0) - PrefetchBandwidth4 = (PrefetchSourceLinesY * swath_width_luma_ub * BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * BytePerPixelC) + PrefetchBandwidth4 = (PrefetchSourceLinesY * swath_width_luma_ub * BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * v->BytePerPixelC[k]) / (Tpre_rounded - Tvm_trips_rounded - 2 * Tr0_trips_rounded); else PrefetchBandwidth4 = 0; @@ -1107,7 +1052,7 @@ static bool CalculatePrefetchSchedule( bool Case3OK; if (PrefetchBandwidth1 > 0) { - if (*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / PrefetchBandwidth1 + if (v->Tno_bw[k] + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / PrefetchBandwidth1 >= Tvm_trips_rounded && (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / PrefetchBandwidth1 >= Tr0_trips_rounded) { Case1OK = true; } else { @@ -1118,7 +1063,7 @@ static bool CalculatePrefetchSchedule( } if (PrefetchBandwidth2 > 0) { - if (*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / PrefetchBandwidth2 + if (v->Tno_bw[k] + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / PrefetchBandwidth2 >= Tvm_trips_rounded && (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / PrefetchBandwidth2 < Tr0_trips_rounded) { Case2OK = true; } else { @@ -1129,7 +1074,7 @@ static bool CalculatePrefetchSchedule( } if (PrefetchBandwidth3 > 0) { - if (*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / PrefetchBandwidth3 + if (v->Tno_bw[k] + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / PrefetchBandwidth3 < Tvm_trips_rounded && (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / PrefetchBandwidth3 >= Tr0_trips_rounded) { Case3OK = true; } else { @@ -1152,13 +1097,13 @@ static bool CalculatePrefetchSchedule( dml_print("DML: prefetch_bw_equ: %f\n", prefetch_bw_equ); if (prefetch_bw_equ > 0) { - if (GPUVMEnable) { - Tvm_equ = dml_max3(*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_equ, Tvm_trips, LineTime / 4); + if (v->GPUVMEnable) { + Tvm_equ = dml_max3(v->Tno_bw[k] + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_equ, Tvm_trips, LineTime / 4); } else { Tvm_equ = LineTime / 4; } - if ((GPUVMEnable || myPipe->DCCEnable)) { + if ((v->GPUVMEnable || myPipe->DCCEnable)) { Tr0_equ = dml_max4( (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / prefetch_bw_equ, Tr0_trips, @@ -1227,7 +1172,7 @@ static bool CalculatePrefetchSchedule( } *RequiredPrefetchPixDataBWLuma = (double) PrefetchSourceLinesY / LinesToRequestPrefetchPixelData * BytePerPixelY * swath_width_luma_ub / LineTime; - *RequiredPrefetchPixDataBWChroma = (double) PrefetchSourceLinesC / LinesToRequestPrefetchPixelData * BytePerPixelC * swath_width_chroma_ub / LineTime; + *RequiredPrefetchPixDataBWChroma = (double) PrefetchSourceLinesC / LinesToRequestPrefetchPixelData * v->BytePerPixelC[k] * swath_width_chroma_ub / LineTime; } else { MyError = true; dml_print("DML: MyErr set %s:%d\n", __FILE__, __LINE__); @@ -1243,9 +1188,9 @@ static bool CalculatePrefetchSchedule( dml_print("DML: Tr0: %fus - time to fetch first row of data pagetables and first row of meta data (done in parallel)\n", TimeForFetchingRowInVBlank); dml_print("DML: Tr1: %fus - time to fetch second row of data pagetables and second row of meta data (done in parallel)\n", TimeForFetchingRowInVBlank); dml_print("DML: Tsw: %fus = time to fetch enough pixel data and cursor data to feed the scalers init position and detile\n", (double)LinesToRequestPrefetchPixelData * LineTime); - dml_print("DML: To: %fus - time for propagation from scaler to optc\n", (*DSTYAfterScaler + ((*DSTXAfterScaler) / (double) myPipe->HTotal)) * LineTime); + dml_print("DML: To: %fus - time for propagation from scaler to optc\n", (v->DSTYAfterScaler[k] + ((v->DSTXAfterScaler[k]) / (double) myPipe->HTotal)) * LineTime); dml_print("DML: Tvstartup - Tsetup - Tcalc - Twait - Tpre - To > 0\n"); - dml_print("DML: Tslack(pre): %fus - time left over in schedule\n", VStartup * LineTime - TimeForFetchingMetaPTE - 2 * TimeForFetchingRowInVBlank - (*DSTYAfterScaler + ((*DSTXAfterScaler) / (double) myPipe->HTotal)) * LineTime - TWait - TCalc - Tsetup); + dml_print("DML: Tslack(pre): %fus - time left over in schedule\n", VStartup * LineTime - TimeForFetchingMetaPTE - 2 * TimeForFetchingRowInVBlank - (v->DSTYAfterScaler[k] + ((v->DSTXAfterScaler[k]) / (double) myPipe->HTotal)) * LineTime - TWait - TCalc - Tsetup); dml_print("DML: row_bytes = dpte_row_bytes (per_pipe) = PixelPTEBytesPerRow = : %d\n", PixelPTEBytesPerRow); } else { @@ -1276,7 +1221,7 @@ static bool CalculatePrefetchSchedule( dml_print("DML: MyErr set %s:%d\n", __FILE__, __LINE__); } - *prefetch_vmrow_bw = dml_max(prefetch_vm_bw, prefetch_row_bw); + v->prefetch_vmrow_bw[k] = dml_max(prefetch_vm_bw, prefetch_row_bw); } if (MyError) { @@ -2437,30 +2382,12 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->ErrorResult[k] = CalculatePrefetchSchedule( mode_lib, - v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData, - v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly, + k, &myPipe, v->DSCDelay[k], - v->DPPCLKDelaySubtotal - + v->DPPCLKDelayCNVCFormater, - v->DPPCLKDelaySCL, - v->DPPCLKDelaySCLLBOnly, - v->DPPCLKDelayCNVCCursor, - v->DISPCLKDelaySubtotal, (unsigned int) (v->SwathWidthY[k] / v->HRatio[k]), - v->OutputFormat[k], - v->MaxInterDCNTileRepeaters, dml_min(v->VStartupLines, v->MaxVStartupLines[k]), v->MaxVStartupLines[k], - v->GPUVMMaxPageTableLevels, - v->GPUVMEnable, - v->HostVMEnable, - v->HostVMMaxNonCachedPageTableLevels, - v->HostVMMinPageSize, - v->DynamicMetadataEnable[k], - v->DynamicMetadataVMEnabled, - v->DynamicMetadataLinesBeforeActiveRequired[k], - v->DynamicMetadataTransmittedBytes[k], v->UrgentLatency, v->UrgentExtraLatency, v->TCalc, @@ -2474,7 +2401,6 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->MaxNumSwathY[k], v->PrefetchSourceLinesC[k], v->SwathWidthC[k], - v->BytePerPixelC[k], v->VInitPreFillC[k], v->MaxNumSwathC[k], v->swath_width_luma_ub[k], @@ -2482,9 +2408,6 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->SwathHeightY[k], v->SwathHeightC[k], TWait, - v->ProgressiveToInterlaceUnitInOPP, - &v->DSTXAfterScaler[k], - &v->DSTYAfterScaler[k], &v->DestinationLinesForPrefetch[k], &v->PrefetchBandwidth[k], &v->DestinationLinesToRequestVMInVBlank[k], @@ -2493,14 +2416,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman &v->VRatioPrefetchC[k], &v->RequiredPrefetchPixDataBWLuma[k], &v->RequiredPrefetchPixDataBWChroma[k], - &v->NotEnoughTimeForDynamicMetadata[k], - &v->Tno_bw[k], - &v->prefetch_vmrow_bw[k], - &v->Tdmdl_vm[k], - &v->Tdmdl[k], - &v->VUpdateOffsetPix[k], - &v->VUpdateWidthPix[k], - &v->VReadyOffsetPix[k]); + &v->NotEnoughTimeForDynamicMetadata[k]); if (v->BlendingAndTiming[k] == k) { double TotalRepeaterDelayTime = v->MaxInterDCNTileRepeaters * (2 / v->DPPCLK[k] + 3 / v->DISPCLK); v->VUpdateWidthPix[k] = (14 / v->DCFCLKDeepSleep + 12 / v->DPPCLK[k] + TotalRepeaterDelayTime) * v->PixelClock[k]; @@ -4770,29 +4686,12 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l v->NoTimeForPrefetch[i][j][k] = CalculatePrefetchSchedule( mode_lib, - v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData, - v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly, + k, &myPipe, v->DSCDelayPerState[i][k], - v->DPPCLKDelaySubtotal + v->DPPCLKDelayCNVCFormater, - v->DPPCLKDelaySCL, - v->DPPCLKDelaySCLLBOnly, - v->DPPCLKDelayCNVCCursor, - v->DISPCLKDelaySubtotal, v->SwathWidthYThisState[k] / v->HRatio[k], - v->OutputFormat[k], - v->MaxInterDCNTileRepeaters, dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]), v->MaximumVStartup[i][j][k], - v->GPUVMMaxPageTableLevels, - v->GPUVMEnable, - v->HostVMEnable, - v->HostVMMaxNonCachedPageTableLevels, - v->HostVMMinPageSize, - v->DynamicMetadataEnable[k], - v->DynamicMetadataVMEnabled, - v->DynamicMetadataLinesBeforeActiveRequired[k], - v->DynamicMetadataTransmittedBytes[k], v->UrgLatency[i], v->ExtraLatency, v->TimeCalc, @@ -4806,7 +4705,6 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l v->MaxNumSwY[k], v->PrefetchLinesC[i][j][k], v->SwathWidthCThisState[k], - v->BytePerPixelC[k], v->PrefillC[k], v->MaxNumSwC[k], v->swath_width_luma_ub_this_state[k], @@ -4814,9 +4712,6 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l v->SwathHeightYThisState[k], v->SwathHeightCThisState[k], v->TWait, - v->ProgressiveToInterlaceUnitInOPP, - &v->DSTXAfterScaler[k], - &v->DSTYAfterScaler[k], &v->LineTimesForPrefetch[k], &v->PrefetchBW[k], &v->LinesForMetaPTE[k], @@ -4825,14 +4720,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l &v->VRatioPreC[i][j][k], &v->RequiredPrefetchPixelDataBWLuma[i][j][k], &v->RequiredPrefetchPixelDataBWChroma[i][j][k], - &v->NoTimeForDynamicMetadata[i][j][k], - &v->Tno_bw[k], - &v->prefetch_vmrow_bw[k], - &v->Tdmdl_vm[k], - &v->Tdmdl[k], - &v->VUpdateOffsetPix[k], - &v->VUpdateWidthPix[k], - &v->VReadyOffsetPix[k]); + &v->NoTimeForDynamicMetadata[i][j][k]); } for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c index 817a370e80a77..8a177d5ae213e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c @@ -164,8 +164,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_5_soc = { }, }, .num_states = 5, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .sr_exit_z8_time_us = 250.0, .sr_enter_plus_exit_z8_time_us = 350.0, .fclk_change_latency_us = 24.0, diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c index 4f569cd8a5d61..272ebdd8b9ebd 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c @@ -360,10 +360,10 @@ void dpp3_cnv_setup ( tbl_entry.color_space = input_color_space; - if (color_space >= COLOR_SPACE_YCBCR601) - select = INPUT_CSC_SELECT_ICSC; - else + if (dpp3_should_bypass_post_csc_for_colorspace(color_space)) select = INPUT_CSC_SELECT_BYPASS; + else + select = INPUT_CSC_SELECT_ICSC; dpp3_program_post_csc(dpp_base, color_space, select, &tbl_entry); @@ -1521,3 +1521,18 @@ bool dpp3_construct( return true; } +bool dpp3_should_bypass_post_csc_for_colorspace(enum dc_color_space dc_color_space) +{ + switch (dc_color_space) { + case COLOR_SPACE_UNKNOWN: + case COLOR_SPACE_SRGB: + case COLOR_SPACE_XR_RGB: + case COLOR_SPACE_SRGB_LIMITED: + case COLOR_SPACE_MSREF_SCRGB: + case COLOR_SPACE_2020_RGB_FULLRANGE: + case COLOR_SPACE_2020_RGB_LIMITEDRANGE: + return true; + default: + return false; + } +} diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h index f236824126e94..2a76105fa9b1c 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h @@ -642,4 +642,8 @@ void dpp3_program_cm_dealpha( void dpp3_cm_get_gamut_remap(struct dpp *dpp_base, struct dpp_grph_csc_adjustment *adjust); + +bool dpp3_should_bypass_post_csc_for_colorspace( + enum dc_color_space dc_color_space); + #endif /* __DC_HWSS_DCN30_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c index 36187f890d5d0..b62bbadb0d440 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c @@ -206,10 +206,10 @@ void dpp401_dpp_setup( tbl_entry.color_space = input_color_space; - if (color_space >= COLOR_SPACE_YCBCR601) - select = INPUT_CSC_SELECT_ICSC; - else + if (dpp3_should_bypass_post_csc_for_colorspace(color_space)) select = INPUT_CSC_SELECT_BYPASS; + else + select = INPUT_CSC_SELECT_ICSC; dpp3_program_post_csc(dpp_base, color_space, select, &tbl_entry); diff --git a/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h b/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h index b86347c9b0389..95f9318a54efc 100644 --- a/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h +++ b/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h @@ -59,7 +59,7 @@ void cm_helper_program_gamcor_xfer_func( const struct pwl_params *params, const struct dcn3_xfer_func_reg *reg); -bool cm3_helper_translate_curve_to_hw_format( +bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx, const struct dc_transfer_func *output_tf, struct pwl_params *lut_params, bool fixpoint); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c index ebc220b29d142..8f86177de48dc 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c @@ -59,6 +59,7 @@ #include "dc_state_priv.h" #include "dpcd_defs.h" #include "dsc.h" +#include "dc_dp_types.h" /* include DCE11 register header files */ #include "dce/dce_11_0_d.h" #include "dce/dce_11_0_sh_mask.h" @@ -1098,13 +1099,13 @@ void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx) if (dc->current_state->res_ctx.pipe_ctx[i].stream_res.audio != NULL) num_audio++; } + if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa) { + /*wake AZ from D3 first before access az endpoint*/ + clk_mgr->funcs->enable_pme_wa(clk_mgr); + } pipe_ctx->stream_res.audio->funcs->az_enable(pipe_ctx->stream_res.audio); - if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa) - /*this is the first audio. apply the PME w/a in order to wake AZ from D3*/ - clk_mgr->funcs->enable_pme_wa(clk_mgr); - link_hwss->enable_audio_packet(pipe_ctx); if (pipe_ctx->stream_res.audio) @@ -1458,9 +1459,6 @@ void build_audio_output( state->clk_mgr); } - audio_output->pll_info.feed_back_divider = - pipe_ctx->pll_settings.feedback_divider; - audio_output->pll_info.dto_source = translate_to_dto_source( pipe_ctx->stream_res.tg->inst + 1); @@ -1732,20 +1730,25 @@ static void power_down_encoders(struct dc *dc) int i; for (i = 0; i < dc->link_count; i++) { - enum signal_type signal = dc->links[i]->connector_signal; - - dc->link_srv->blank_dp_stream(dc->links[i], false); + struct dc_link *link = dc->links[i]; + struct link_encoder *link_enc = link->link_enc; + enum signal_type signal = link->connector_signal; + dc->link_srv->blank_dp_stream(link, false); if (signal != SIGNAL_TYPE_EDP) signal = SIGNAL_TYPE_NONE; - if (dc->links[i]->ep_type == DISPLAY_ENDPOINT_PHY) - dc->links[i]->link_enc->funcs->disable_output( - dc->links[i]->link_enc, signal); + if (link->ep_type == DISPLAY_ENDPOINT_PHY) + link_enc->funcs->disable_output(link_enc, signal); + + if (link->fec_state == dc_link_fec_enabled) { + link_enc->funcs->fec_set_enable(link_enc, false); + link_enc->funcs->fec_set_ready(link_enc, false); + link->fec_state = dc_link_fec_not_ready; + } - dc->links[i]->link_status.link_active = false; - memset(&dc->links[i]->cur_link_settings, 0, - sizeof(dc->links[i]->cur_link_settings)); + link->link_status.link_active = false; + memset(&link->cur_link_settings, 0, sizeof(link->cur_link_settings)); } } @@ -1793,6 +1796,9 @@ static void disable_vga_and_power_gate_all_controllers( struct timing_generator *tg; struct dc_context *ctx = dc->ctx; + if (dc->caps.ips_support) + return; + for (i = 0; i < dc->res_pool->timing_generator_count; i++) { tg = dc->res_pool->timing_generators[i]; @@ -1869,13 +1875,16 @@ static void clean_up_dsc_blocks(struct dc *dc) /* disable DSC in OPTC */ if (i < dc->res_pool->timing_generator_count) { tg = dc->res_pool->timing_generators[i]; - tg->funcs->set_dsc_config(tg, OPTC_DSC_DISABLED, 0, 0); + if (tg->funcs->set_dsc_config) + tg->funcs->set_dsc_config(tg, OPTC_DSC_DISABLED, 0, 0); } /* disable DSC in stream encoder */ if (i < dc->res_pool->stream_enc_count) { se = dc->res_pool->stream_enc[i]; - se->funcs->dp_set_dsc_config(se, OPTC_DSC_DISABLED, 0, 0); - se->funcs->dp_set_dsc_pps_info_packet(se, false, NULL, true); + if (se->funcs->dp_set_dsc_config) + se->funcs->dp_set_dsc_config(se, OPTC_DSC_DISABLED, 0, 0); + if (se->funcs->dp_set_dsc_pps_info_packet) + se->funcs->dp_set_dsc_pps_info_packet(se, false, NULL, true); } /* disable DSC block */ if (dccg->funcs->set_ref_dscclk) @@ -1924,8 +1933,8 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context) get_edp_streams(context, edp_streams, &edp_stream_num); - /* Check fastboot support, disable on DCE 6-8 because of blank screens */ - if (edp_num && edp_stream_num && dc->ctx->dce_version < DCE_VERSION_10_0) { + /* Check fastboot support, disable on DCE 6-8-10 because of blank screens */ + if (edp_num && edp_stream_num && dc->ctx->dce_version > DCE_VERSION_10_0) { for (i = 0; i < edp_num; i++) { edp_link = edp_links[i]; if (edp_link != edp_streams[0]->link) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c index 56c1ab6c73308..a4025a09a38a3 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c @@ -3055,9 +3055,17 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx) dccg->funcs->enable_symclk32_se(dccg, dp_hpo_inst, phyd32clk); } } else { - if (dccg->funcs->enable_symclk_se) - dccg->funcs->enable_symclk_se(dccg, stream_enc->stream_enc_inst, + if (dccg->funcs->enable_symclk_se && link_enc) { + if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA + && link->cur_link_settings.link_rate == LINK_RATE_UNKNOWN + && !link->link_status.link_active) { + if (dccg->funcs->disable_symclk_se) + dccg->funcs->disable_symclk_se(dccg, stream_enc->stream_enc_inst, link_enc->transmitter - TRANSMITTER_UNIPHY_A); + } else + dccg->funcs->enable_symclk_se(dccg, stream_enc->stream_enc_inst, + link_enc->transmitter - TRANSMITTER_UNIPHY_A); + } } if (dc->res_pool->dccg->funcs->set_pixel_rate_div) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c index e47ed5571dfdd..731645a2ab9aa 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c @@ -238,7 +238,7 @@ bool dcn30_set_blend_lut( if (plane_state->blend_tf.type == TF_TYPE_HWPWL) blend_lut = &plane_state->blend_tf.pwl; else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { - result = cm3_helper_translate_curve_to_hw_format( + result = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, &plane_state->blend_tf, &dpp_base->regamma_params, false); if (!result) return result; @@ -333,8 +333,9 @@ bool dcn30_set_input_transfer_func(struct dc *dc, if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL) params = &plane_state->in_transfer_func.pwl; else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format(&plane_state->in_transfer_func, - &dpp_base->degamma_params, false)) + cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_transfer_func, + &dpp_base->degamma_params, false)) params = &dpp_base->degamma_params; result = dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params); @@ -405,7 +406,7 @@ bool dcn30_set_output_transfer_func(struct dc *dc, params = &stream->out_transfer_func.pwl; else if (pipe_ctx->stream->out_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format( + cm3_helper_translate_curve_to_hw_format(stream->ctx, &stream->out_transfer_func, &mpc->blender_params, false)) params = &mpc->blender_params; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c index b822f2dffff0e..e9bd43a72ce58 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c @@ -546,8 +546,22 @@ static void dcn31_reset_back_end_for_pipe( if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass) pipe_ctx->stream_res.tg->funcs->set_odm_bypass( pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing); + /* + * TODO - convert symclk_ref_cnts for otg to a bit map to solve + * the case where the same symclk is shared across multiple otg + * instances + */ if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal)) - pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0; + link->phy_state.symclk_ref_cnts.otg = 0; + + if (pipe_ctx->top_pipe == NULL) { + if (link->phy_state.symclk_state == SYMCLK_ON_TX_OFF) { + const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res); + + link_hwss->disable_link_output(link, &pipe_ctx->link_res, pipe_ctx->stream->signal); + link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF; + } + } set_drr_and_clear_adjust_pending(pipe_ctx, pipe_ctx->stream, NULL); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c index f39292952702f..c6fde355ac823 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c @@ -486,8 +486,9 @@ bool dcn32_set_mcm_luts( if (plane_state->blend_tf.type == TF_TYPE_HWPWL) lut_params = &plane_state->blend_tf.pwl; else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { - result = cm3_helper_translate_curve_to_hw_format(&plane_state->blend_tf, - &dpp_base->regamma_params, false); + result = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->blend_tf, + &dpp_base->regamma_params, false); if (!result) return result; @@ -501,9 +502,9 @@ bool dcn32_set_mcm_luts( lut_params = &plane_state->in_shaper_func.pwl; else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { // TODO: dpp_base replace - ASSERT(false); - cm3_helper_translate_curve_to_hw_format(&plane_state->in_shaper_func, - &dpp_base->shaper_params, true); + cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_shaper_func, + &dpp_base->shaper_params, true); lut_params = &dpp_base->shaper_params; } @@ -543,8 +544,9 @@ bool dcn32_set_input_transfer_func(struct dc *dc, if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL) params = &plane_state->in_transfer_func.pwl; else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format(&plane_state->in_transfer_func, - &dpp_base->degamma_params, false)) + cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_transfer_func, + &dpp_base->degamma_params, false)) params = &dpp_base->degamma_params; dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params); @@ -575,7 +577,7 @@ bool dcn32_set_output_transfer_func(struct dc *dc, params = &stream->out_transfer_func.pwl; else if (pipe_ctx->stream->out_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format( + cm3_helper_translate_curve_to_hw_format(stream->ctx, &stream->out_transfer_func, &mpc->blender_params, false)) params = &mpc->blender_params; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c index 68e48a2492c9e..914c30a42ecfa 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c @@ -68,7 +68,11 @@ void dcn401_initialize_min_clocks(struct dc *dc) * audio corruption. Read current DISPCLK from DENTIST and request the same * freq to ensure that the timing is valid and unchanged. */ - clocks->dispclk_khz = dc->clk_mgr->funcs->get_dispclk_from_dentist(dc->clk_mgr); + if (dc->clk_mgr->funcs->get_dispclk_from_dentist) { + clocks->dispclk_khz = dc->clk_mgr->funcs->get_dispclk_from_dentist(dc->clk_mgr); + } else { + clocks->dispclk_khz = dc->clk_mgr->boot_snapshot.dispclk * 1000; + } } clocks->ref_dtbclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dtbclk_mhz * 1000; clocks->fclk_p_state_change_support = true; @@ -139,6 +143,7 @@ void dcn401_init_hw(struct dc *dc) int edp_num; uint32_t backlight = MAX_BACKLIGHT_LEVEL; uint32_t user_level = MAX_BACKLIGHT_LEVEL; + bool dchub_ref_freq_changed; int current_dchub_ref_freq = 0; if (dc->clk_mgr && dc->clk_mgr->funcs && dc->clk_mgr->funcs->init_clocks) { @@ -284,6 +289,8 @@ void dcn401_init_hw(struct dc *dc) for (i = 0; i < dc->link_count; i++) { struct dc_link *link = dc->links[i]; + if (link->ep_type != DISPLAY_ENDPOINT_PHY) + continue; if (link->link_enc->funcs->is_dig_enabled && link->link_enc->funcs->is_dig_enabled(link->link_enc) && hws->funcs.power_down) { @@ -351,14 +358,18 @@ void dcn401_init_hw(struct dc *dc) dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr; dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver > 0; dc->caps.dmub_caps.fams_ver = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver; + + /* sw and fw FAMS versions must match for support */ dc->debug.fams2_config.bits.enable &= - dc->caps.dmub_caps.fams_ver == dc->debug.fams_version.ver; // sw & fw fams versions must match for support - if ((!dc->debug.fams2_config.bits.enable && dc->res_pool->funcs->update_bw_bounding_box) - || res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000 != current_dchub_ref_freq) { + dc->caps.dmub_caps.fams_ver == dc->debug.fams_version.ver; + dchub_ref_freq_changed = + res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000 != current_dchub_ref_freq; + if ((!dc->debug.fams2_config.bits.enable || dchub_ref_freq_changed) && + dc->res_pool->funcs->update_bw_bounding_box && + dc->clk_mgr && dc->clk_mgr->bw_params) { /* update bounding box if FAMS2 disabled, or if dchub clk has changed */ - if (dc->clk_mgr) - dc->res_pool->funcs->update_bw_bounding_box(dc, - dc->clk_mgr->bw_params); + dc->res_pool->funcs->update_bw_bounding_box(dc, + dc->clk_mgr->bw_params); } } } @@ -427,7 +438,7 @@ void dcn401_populate_mcm_luts(struct dc *dc, if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL) m_lut_params.pwl = &mcm_luts.lut1d_func->pwl; else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) { - rval = cm3_helper_translate_curve_to_hw_format( + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, mcm_luts.lut1d_func, &dpp_base->regamma_params, false); m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; @@ -447,7 +458,7 @@ void dcn401_populate_mcm_luts(struct dc *dc, m_lut_params.pwl = &mcm_luts.shaper->pwl; else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) { ASSERT(false); - rval = cm3_helper_translate_curve_to_hw_format( + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, mcm_luts.shaper, &dpp_base->regamma_params, true); m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; @@ -624,8 +635,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx, if (plane_state->blend_tf.type == TF_TYPE_HWPWL) lut_params = &plane_state->blend_tf.pwl; else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { - rval = cm3_helper_translate_curve_to_hw_format(&plane_state->blend_tf, - &dpp_base->regamma_params, false); + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->blend_tf, + &dpp_base->regamma_params, false); lut_params = rval ? &dpp_base->regamma_params : NULL; } result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id); @@ -636,8 +648,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx, lut_params = &plane_state->in_shaper_func.pwl; else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { // TODO: dpp_base replace - rval = cm3_helper_translate_curve_to_hw_format(&plane_state->in_shaper_func, - &dpp_base->shaper_params, true); + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_shaper_func, + &dpp_base->shaper_params, true); lut_params = rval ? &dpp_base->shaper_params : NULL; } result &= mpc->funcs->program_shaper(mpc, lut_params, mpcc_id); @@ -671,7 +684,7 @@ bool dcn401_set_output_transfer_func(struct dc *dc, params = &stream->out_transfer_func.pwl; else if (pipe_ctx->stream->out_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format( + cm3_helper_translate_curve_to_hw_format(stream->ctx, &stream->out_transfer_func, &mpc->blender_params, false)) params = &mpc->blender_params; @@ -912,10 +925,10 @@ static void dcn401_enable_stream_calc( pipe_ctx->stream->link->cur_link_settings.lane_count; uint32_t active_total_with_borders; - if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) + if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) { *dp_hpo_inst = pipe_ctx->stream_res.hpo_dp_stream_enc->inst; - - *phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link); + *phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link); + } if (dc_is_tmds_signal(pipe_ctx->stream->signal)) dcn401_calculate_dccg_tmds_div_value(pipe_ctx, tmds_div); @@ -1626,7 +1639,8 @@ void dcn401_unblank_stream(struct pipe_ctx *pipe_ctx, void dcn401_hardware_release(struct dc *dc) { if (!dc->debug.disable_force_pstate_allow_on_hw_release) { - dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); + if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable) + dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); /* If pstate unsupported, or still supported * by firmware, force it supported by dcn @@ -1646,7 +1660,9 @@ void dcn401_hardware_release(struct dc *dc) dc->clk_mgr->clks.p_state_change_support = false; dc->clk_mgr->funcs->update_clocks(dc->clk_mgr, dc->current_state, true); } - dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); + + if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable) + dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); } } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c index 1173c53359b00..31b404e9b9ae0 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c @@ -332,7 +332,7 @@ static void query_dp_dual_mode_adaptor( /* Assume we have no valid DP passive dongle connected */ *dongle = DISPLAY_DONGLE_NONE; - sink_cap->max_hdmi_pixel_clock = DP_ADAPTOR_HDMI_SAFE_MAX_TMDS_CLK; + sink_cap->max_hdmi_pixel_clock = DP_ADAPTOR_DVI_MAX_TMDS_CLK; /* Read DP-HDMI dongle I2c (no response interpreted as DP-DVI dongle)*/ if (!i2c_read( @@ -388,6 +388,8 @@ static void query_dp_dual_mode_adaptor( } } + if (is_valid_hdmi_signature) + sink_cap->max_hdmi_pixel_clock = DP_ADAPTOR_HDMI_SAFE_MAX_TMDS_CLK; if (is_type2_dongle) { uint32_t max_tmds_clk = diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index b66fbcb0040d6..a084c698ed6ef 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -1928,7 +1928,7 @@ static void disable_link_dp(struct dc_link *link, link->dc->hwss.edp_power_control(link, false); } - if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST) + if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST && link->sink_count == 0) /* set the sink to SST mode after disabling the link */ enable_mst_on_sink(link, false); @@ -2079,7 +2079,12 @@ static enum dc_status enable_link_dp(struct dc_state *state, pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT && link->dc->debug.set_mst_en_for_sst) { enable_mst_on_sink(link, true); + } else if (link->dpcd_caps.is_mst_capable && + pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT) { + /* disable mst on sink */ + enable_mst_on_sink(link, false); } + if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP) { /*in case it is not on*/ if (!link->dc->config.edp_no_power_sequencing) @@ -2358,9 +2363,9 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx) if (pipe_ctx->stream->sink) { if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { - DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__, + DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, pipe_ctx->stream->sink->edid_caps.display_name, - pipe_ctx->stream->signal, link->link_index); + pipe_ctx->stream->signal, link->link_index, link->sink_count); } } @@ -2474,10 +2479,11 @@ void link_set_dpms_on( if (pipe_ctx->stream->sink) { if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { - DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__, + DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, pipe_ctx->stream->sink->edid_caps.display_name, pipe_ctx->stream->signal, - link->link_index); + link->link_index, + link->sink_count); } } diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c index ce174ce5579c0..6a7c4a59ff4c7 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c @@ -271,7 +271,7 @@ enum link_training_result dp_perform_fixed_vs_pe_training_sequence( rate = get_dpcd_link_rate(<_settings->link_settings); // Only perform toggle if FIXED_VS LTTPR reports no IEEE OUI - if (memcmp("\x0,\x0,\x0", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) { + if (memcmp("\x00\x00\x00", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) { /* Vendor specific: Toggle link rate */ toggle_rate = (rate == 0x6) ? 0xA : 0x6; diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c index 6f0e017a8ae29..a9d2aa0b5390f 100644 --- a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c @@ -724,8 +724,7 @@ bool mpc32_program_shaper( return false; } - if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) - mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true); + mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true); current_mode = mpc32_get_shaper_current(mpc, mpcc_id); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c index c4b4dc3ad8c9d..ef146f253577a 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c @@ -624,7 +624,7 @@ static struct link_encoder *dce100_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -635,7 +635,8 @@ static struct link_encoder *dce100_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c index cccde5a6f3cdf..234da1e2ae21d 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c @@ -668,7 +668,7 @@ static struct link_encoder *dce110_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -679,7 +679,8 @@ static struct link_encoder *dce110_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c index 869a8e515fc09..2d5a5e1691971 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c @@ -629,7 +629,7 @@ static struct link_encoder *dce112_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -640,7 +640,8 @@ static struct link_encoder *dce112_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c index 540e04ec1e2d9..0af20d2bd8b90 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c @@ -713,7 +713,7 @@ static struct link_encoder *dce120_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -724,7 +724,8 @@ static struct link_encoder *dce120_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c index b75be6ad64f6c..f2e47f1ff4b33 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c @@ -718,18 +718,19 @@ static struct link_encoder *dce60_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = map_transmitter_id_to_phy_instance(enc_init_data->transmitter); dce60_link_encoder_construct(enc110, - enc_init_data, - &link_enc_feature, - &link_enc_regs[link_regs_id], - &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data, + &link_enc_feature, + &link_enc_regs[link_regs_id], + &link_enc_aux_regs[enc_init_data->channel - 1], + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c index 5b77697452022..8822d5818d29e 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c @@ -724,7 +724,7 @@ static struct link_encoder *dce80_link_encoder_create( kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); int link_regs_id; - if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) + if (!enc110) return NULL; link_regs_id = @@ -735,7 +735,8 @@ static struct link_encoder *dce80_link_encoder_create( &link_enc_feature, &link_enc_regs[link_regs_id], &link_enc_aux_regs[enc_init_data->channel - 1], - &link_enc_hpd_regs[enc_init_data->hpd_source]); + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c index 82cc78c291d82..12c2a0d9fb2a3 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c @@ -1226,12 +1226,12 @@ static struct stream_encoder *dcn315_stream_encoder_create( /*PHYB is wired off in HW, allow front end to remapping, otherwise needs more changes*/ /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c index 636110e48d01b..3c77c14c5a5ed 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c @@ -1220,12 +1220,12 @@ static struct stream_encoder *dcn316_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c index 3965a7f1b64b7..8f054d9b5d57a 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c @@ -1208,12 +1208,12 @@ static struct stream_encoder *dcn32_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn32_vpg_create(ctx, vpg_inst); afmt = dcn32_afmt_create(ctx, afmt_inst); @@ -1786,7 +1786,10 @@ static bool dml1_validate(struct dc *dc, struct dc_state *context, enum dc_valid dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel); + DC_FP_START(); dcn32_override_min_req_memclk(dc, context); + DC_FP_END(); + dcn32_override_min_req_dcfclk(dc, context); BW_VAL_TRACE_END_WATERMARKS(); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c index ad214986f7ac7..26fd5c03c0147 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c @@ -1189,12 +1189,12 @@ static struct stream_encoder *dcn321_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn321_vpg_create(ctx, vpg_inst); afmt = dcn321_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c index fff57f23f4f7a..e8d74ceb9dc29 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c @@ -203,12 +203,12 @@ enum dcn35_clk_src_array_id { NBIO_BASE_INNER(seg) #define NBIO_SR(reg_name)\ - REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \ - regBIF_BX2_ ## reg_name + REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \ + regBIF_BX1_ ## reg_name #define NBIO_SR_ARR(reg_name, id)\ - REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \ - regBIF_BX2_ ## reg_name + REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \ + regBIF_BX1_ ## reg_name #define bios_regs_init() \ ( \ @@ -1271,12 +1271,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c index 0abd163b425e5..532e5d9bc4337 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c @@ -183,12 +183,12 @@ enum dcn351_clk_src_array_id { NBIO_BASE_INNER(seg) #define NBIO_SR(reg_name)\ - REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \ - regBIF_BX2_ ## reg_name + REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \ + regBIF_BX1_ ## reg_name #define NBIO_SR_ARR(reg_name, id)\ - REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \ - regBIF_BX2_ ## reg_name + REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \ + regBIF_BX1_ ## reg_name #define bios_regs_init() \ ( \ @@ -1251,12 +1251,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/include/audio_types.h b/drivers/gpu/drm/amd/display/include/audio_types.h index e4a26143f14c9..6699ad4fa825e 100644 --- a/drivers/gpu/drm/amd/display/include/audio_types.h +++ b/drivers/gpu/drm/amd/display/include/audio_types.h @@ -47,15 +47,15 @@ struct audio_crtc_info { uint32_t h_total; uint32_t h_active; uint32_t v_active; - uint32_t pixel_repetition; uint32_t requested_pixel_clock_100Hz; /* in 100Hz */ uint32_t calculated_pixel_clock_100Hz; /* in 100Hz */ - uint32_t refresh_rate; + uint32_t dsc_bits_per_pixel; + uint32_t dsc_num_slices; enum dc_color_depth color_depth; enum dc_pixel_encoding pixel_encoding; + uint16_t refresh_rate; + uint8_t pixel_repetition; bool interlaced; - uint32_t dsc_bits_per_pixel; - uint32_t dsc_num_slices; }; struct azalia_clock_info { uint32_t pixel_clock_in_10khz; @@ -78,11 +78,9 @@ enum audio_dto_source { struct audio_pll_info { uint32_t audio_dto_source_clock_in_khz; - uint32_t feed_back_divider; + uint32_t ss_percentage; enum audio_dto_source dto_source; bool ss_enabled; - uint32_t ss_percentage; - uint32_t ss_percentage_divider; }; struct audio_channel_associate_info { diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c index bc29a923fa6e5..8253d2977408d 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c @@ -80,15 +80,15 @@ int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev, enum ip_power_state pwr_state = gate ? POWER_STATE_OFF : POWER_STATE_ON; bool is_vcn = block_type == AMD_IP_BLOCK_TYPE_VCN; + mutex_lock(&adev->pm.mutex); + if (atomic_read(&adev->pm.pwr_state[block_type]) == pwr_state && (!is_vcn || adev->vcn.num_vcn_inst == 1)) { dev_dbg(adev->dev, "IP block%d already in the target %s state!", block_type, gate ? "gate" : "ungate"); - return 0; + goto out_unlock; } - mutex_lock(&adev->pm.mutex); - switch (block_type) { case AMD_IP_BLOCK_TYPE_UVD: case AMD_IP_BLOCK_TYPE_VCE: @@ -115,6 +115,7 @@ int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev, if (!ret) atomic_set(&adev->pm.pwr_state[block_type], pwr_state); +out_unlock: mutex_unlock(&adev->pm.mutex); return ret; diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c index 3a9522c17fee3..c25e72e31d8bf 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c @@ -2273,8 +2273,6 @@ static int si_populate_smc_tdp_limits(struct amdgpu_device *adev, if (scaling_factor == 0) return -EINVAL; - memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE)); - ret = si_calculate_adjusted_tdp_limits(adev, false, /* ??? */ adev->pm.dpm.tdp_adjustment, @@ -2283,6 +2281,12 @@ static int si_populate_smc_tdp_limits(struct amdgpu_device *adev, if (ret) return ret; + if (adev->pdev->device == 0x6611 && adev->pdev->revision == 0x87) { + /* Workaround buggy powertune on Radeon 430 and 520. */ + tdp_limit = 32; + near_tdp_limit = 28; + } + smc_table->dpm2Params.TDPLimit = cpu_to_be32(si_scale_power_for_smc(tdp_limit, scaling_factor) * 1000); smc_table->dpm2Params.NearTDPLimit = @@ -2328,16 +2332,8 @@ static int si_populate_smc_tdp_limits_2(struct amdgpu_device *adev, if (ni_pi->enable_power_containment) { SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable; - u32 scaling_factor = si_get_smc_power_scaling_factor(adev); int ret; - memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE)); - - smc_table->dpm2Params.NearTDPLimit = - cpu_to_be32(si_scale_power_for_smc(adev->pm.dpm.near_tdp_limit_adjusted, scaling_factor) * 1000); - smc_table->dpm2Params.SafePowerLimit = - cpu_to_be32(si_scale_power_for_smc((adev->pm.dpm.near_tdp_limit_adjusted * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000); - ret = amdgpu_si_copy_bytes_to_smc(adev, (si_pi->state_table_start + offsetof(SISLANDS_SMC_STATETABLE, dpm2Params) + @@ -3463,9 +3459,11 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, if (adev->asic_type == CHIP_HAINAN) { if ((adev->pdev->revision == 0x81) || (adev->pdev->revision == 0xC3) || + (adev->pdev->device == 0x6660) || (adev->pdev->device == 0x6664) || (adev->pdev->device == 0x6665) || - (adev->pdev->device == 0x6667)) { + (adev->pdev->device == 0x6667) || + (adev->pdev->device == 0x666F)) { max_sclk = 75000; } if ((adev->pdev->revision == 0xC3) || @@ -3473,15 +3471,25 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, max_sclk = 60000; max_mclk = 80000; } + if ((adev->pdev->device == 0x666f) && + (adev->pdev->revision == 0x00)) { + max_sclk = 80000; + max_mclk = 95000; + } } else if (adev->asic_type == CHIP_OLAND) { if ((adev->pdev->revision == 0xC7) || (adev->pdev->revision == 0x80) || (adev->pdev->revision == 0x81) || (adev->pdev->revision == 0x83) || - (adev->pdev->revision == 0x87) || + (adev->pdev->revision == 0x87 && + adev->pdev->device != 0x6611) || (adev->pdev->device == 0x6604) || (adev->pdev->device == 0x6605)) { max_sclk = 75000; + } else if (adev->pdev->revision == 0x87 && + adev->pdev->device == 0x6611) { + /* Radeon 430 and 520 */ + max_sclk = 78000; } } @@ -7600,12 +7608,12 @@ static int si_dpm_set_interrupt_state(struct amdgpu_device *adev, case AMDGPU_IRQ_STATE_DISABLE: cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT); cg_thermal_int |= CG_THERMAL_INT__THERM_INT_MASK_HIGH_MASK; - WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int); + WREG32(mmCG_THERMAL_INT, cg_thermal_int); break; case AMDGPU_IRQ_STATE_ENABLE: cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT); cg_thermal_int &= ~CG_THERMAL_INT__THERM_INT_MASK_HIGH_MASK; - WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int); + WREG32(mmCG_THERMAL_INT, cg_thermal_int); break; default: break; @@ -7617,12 +7625,12 @@ static int si_dpm_set_interrupt_state(struct amdgpu_device *adev, case AMDGPU_IRQ_STATE_DISABLE: cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT); cg_thermal_int |= CG_THERMAL_INT__THERM_INT_MASK_LOW_MASK; - WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int); + WREG32(mmCG_THERMAL_INT, cg_thermal_int); break; case AMDGPU_IRQ_STATE_ENABLE: cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT); cg_thermal_int &= ~CG_THERMAL_INT__THERM_INT_MASK_LOW_MASK; - WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int); + WREG32(mmCG_THERMAL_INT, cg_thermal_int); break; default: break; diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 244b8c364d45f..cfa4ff5d21b27 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -1655,9 +1655,12 @@ static int smu_smc_hw_setup(struct smu_context *smu) if (adev->in_suspend && smu_is_dpm_running(smu)) { dev_info(adev->dev, "dpm has been enabled\n"); ret = smu_system_features_control(smu, true); - if (ret) + if (ret) { dev_err(adev->dev, "Failed system features control!\n"); - return ret; + return ret; + } + + return smu_enable_thermal_alert(smu); } break; default: diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h index 4263798d716b7..8e592a477c33a 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h @@ -56,6 +56,7 @@ #define SMUQ10_TO_UINT(x) ((x) >> 10) #define SMUQ10_FRAC(x) ((x) & 0x3ff) #define SMUQ10_ROUND(x) ((SMUQ10_TO_UINT(x)) + ((SMUQ10_FRAC(x)) >= 0x200)) +#define SMU_V13_SOFT_FREQ_ROUND(x) ((x) + 1) extern const int pmfw_decoded_link_speed[5]; extern const int pmfw_decoded_link_width[7]; diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h index 29a4583db8734..0b1e6f25e6112 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h @@ -57,6 +57,7 @@ extern const int decoded_link_width[8]; #define DECODE_GEN_SPEED(gen_speed_idx) (decoded_link_speed[gen_speed_idx]) #define DECODE_LANE_WIDTH(lane_width_idx) (decoded_link_width[lane_width_idx]) +#define SMU_V14_SOFT_FREQ_ROUND(x) ((x) + 1) struct smu_14_0_max_sustainable_clocks { uint32_t display_clock; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c index 0028f10ead423..a2fcf678182b4 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c @@ -2454,24 +2454,21 @@ static int navi10_update_pcie_parameters(struct smu_context *smu, } for (i = 0; i < NUM_LINK_LEVELS; i++) { - if (pptable->PcieGenSpeed[i] > pcie_gen_cap || - pptable->PcieLaneCount[i] > pcie_width_cap) { - dpm_context->dpm_tables.pcie_table.pcie_gen[i] = - pptable->PcieGenSpeed[i] > pcie_gen_cap ? - pcie_gen_cap : pptable->PcieGenSpeed[i]; - dpm_context->dpm_tables.pcie_table.pcie_lane[i] = - pptable->PcieLaneCount[i] > pcie_width_cap ? - pcie_width_cap : pptable->PcieLaneCount[i]; - smu_pcie_arg = i << 16; - smu_pcie_arg |= pcie_gen_cap << 8; - smu_pcie_arg |= pcie_width_cap; - ret = smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_OverridePcieParameters, - smu_pcie_arg, - NULL); - if (ret) - break; - } + dpm_context->dpm_tables.pcie_table.pcie_gen[i] = + pptable->PcieGenSpeed[i] > pcie_gen_cap ? + pcie_gen_cap : pptable->PcieGenSpeed[i]; + dpm_context->dpm_tables.pcie_table.pcie_lane[i] = + pptable->PcieLaneCount[i] > pcie_width_cap ? + pcie_width_cap : pptable->PcieLaneCount[i]; + smu_pcie_arg = i << 16; + smu_pcie_arg |= dpm_context->dpm_tables.pcie_table.pcie_gen[i] << 8; + smu_pcie_arg |= dpm_context->dpm_tables.pcie_table.pcie_lane[i]; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_OverridePcieParameters, + smu_pcie_arg, + NULL); + if (ret) + return ret; } return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index 78e4186d06cc0..48bf000f12e5c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -262,7 +262,6 @@ int smu_v11_0_check_fw_version(struct smu_context *smu) "smu fw program = %d, version = 0x%08x (%d.%d.%d)\n", smu->smc_driver_if_version, if_version, smu_program, smu_version, smu_major, smu_minor, smu_debug); - dev_info(smu->adev->dev, "SMU driver if version not matched\n"); } return ret; @@ -1022,7 +1021,12 @@ int smu_v11_0_enable_thermal_alert(struct smu_context *smu) int smu_v11_0_disable_thermal_alert(struct smu_context *smu) { - return amdgpu_irq_put(smu->adev, &smu->irq_source, 0); + int ret = 0; + + if (smu->smu_table.thermal_controller_type) + ret = amdgpu_irq_put(smu->adev, &smu->irq_source, 0); + + return ret; } static uint16_t convert_to_vddc(uint8_t vid) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c index 3d3cd546f0adc..63098ed9fcfe5 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c @@ -101,7 +101,6 @@ int smu_v12_0_check_fw_version(struct smu_context *smu) "smu fw program = %d, smu fw version = 0x%08x (%d.%d.%d)\n", smu->smc_driver_if_version, if_version, smu_program, smu_version, smu_major, smu_minor, smu_debug); - dev_info(smu->adev->dev, "SMU driver if version not matched\n"); } return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index a89075e257174..2efd914d81e5b 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -1555,6 +1555,7 @@ int smu_v13_0_set_soft_freq_limited_range(struct smu_context *smu, return clk_id; if (max > 0) { + max = SMU_V13_SOFT_FREQ_ROUND(max); if (automatic) param = (uint32_t)((clk_id << 16) | 0xffff); else diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index c1062e5f03936..75b542e03e2d0 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -59,6 +59,10 @@ #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) +static void smu_v13_0_0_get_od_setting_limits(struct smu_context *smu, + int od_feature_bit, + int32_t *min, int32_t *max); + #define FEATURE_MASK(feature) (1ULL << feature) #define SMC_DPM_FEATURE ( \ FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ @@ -1061,8 +1065,35 @@ static bool smu_v13_0_0_is_od_feature_supported(struct smu_context *smu, PPTable_t *pptable = smu->smu_table.driver_pptable; const OverDriveLimits_t * const overdrive_upperlimits = &pptable->SkuTable.OverDriveLimitsBasicMax; + int32_t min_value, max_value; + bool feature_enabled; + + switch (od_feature_bit) { + case PP_OD_FEATURE_FAN_CURVE_BIT: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + if (feature_enabled) { + smu_v13_0_0_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_TEMP, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + + smu_v13_0_0_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_PWM, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + } + break; + default: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + break; + } - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); +out: + return feature_enabled; } static void smu_v13_0_0_get_od_setting_limits(struct smu_context *smu, @@ -2109,6 +2140,7 @@ static ssize_t smu_v13_0_0_get_gpu_metrics(struct smu_context *smu, (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; SmuMetricsExternal_t metrics_ext; SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; + uint32_t mp1_ver = amdgpu_ip_version(smu->adev, MP1_HWIP, 0); int ret = 0; ret = smu_cmn_get_metrics_table(smu, @@ -2133,7 +2165,12 @@ static ssize_t smu_v13_0_0_get_gpu_metrics(struct smu_context *smu, metrics->Vcn1ActivityPercentage); gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; + + if ((mp1_ver == IP_VERSION(13, 0, 0) && smu->smc_fw_version <= 0x004e1e00) || + (mp1_ver == IP_VERSION(13, 0, 10) && smu->smc_fw_version <= 0x00500800)) + gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; + else + gpu_metrics->energy_accumulator = UINT_MAX; if (metrics->AverageGfxActivity <= SMU_13_0_0_BUSY_THRESHOLD) gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; @@ -2289,7 +2326,8 @@ static int smu_v13_0_0_restore_user_od_settings(struct smu_context *smu) user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | BIT(PP_OD_FEATURE_UCLK_BIT) | BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | + BIT(PP_OD_FEATURE_ZERO_FAN_BIT); res = smu_v13_0_0_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) @@ -2922,8 +2960,13 @@ static int smu_v13_0_0_mode1_reset(struct smu_context *smu) break; } - if (!ret) + if (!ret) { + /* disable mmio access while doing mode 1 reset*/ + smu->adev->no_hw_access = true; + /* ensure no_hw_access is globally visible before any MMIO */ + smp_mb(); msleep(SMU13_MODE1_RESET_WAIT_TIME_IN_MS); + } return ret; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c index 285cf79796939..43965b1135fe7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c @@ -1494,7 +1494,7 @@ static int smu_v13_0_6_print_clk_levels(struct smu_context *smu, case SMU_OD_MCLK: if (!smu_v13_0_6_cap_supported(smu, SMU_CAP(SET_UCLK_MAX))) - return 0; + return -EOPNOTSUPP; size += sysfs_emit_at(buf, size, "%s:\n", "OD_MCLK"); size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index c96fa5e49ed65..f355ede317d83 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -59,6 +59,10 @@ #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) +static void smu_v13_0_7_get_od_setting_limits(struct smu_context *smu, + int od_feature_bit, + int32_t *min, int32_t *max); + #define FEATURE_MASK(feature) (1ULL << feature) #define SMC_DPM_FEATURE ( \ FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ @@ -1050,8 +1054,35 @@ static bool smu_v13_0_7_is_od_feature_supported(struct smu_context *smu, PPTable_t *pptable = smu->smu_table.driver_pptable; const OverDriveLimits_t * const overdrive_upperlimits = &pptable->SkuTable.OverDriveLimitsBasicMax; + int32_t min_value, max_value; + bool feature_enabled; - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); + switch (od_feature_bit) { + case PP_OD_FEATURE_FAN_CURVE_BIT: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + if (feature_enabled) { + smu_v13_0_7_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_TEMP, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + + smu_v13_0_7_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_PWM, + &min_value, &max_value); + if (!min_value && !max_value) { + feature_enabled = false; + goto out; + } + } + break; + default: + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); + break; + } + +out: + return feature_enabled; } static void smu_v13_0_7_get_od_setting_limits(struct smu_context *smu, @@ -2119,7 +2150,8 @@ static ssize_t smu_v13_0_7_get_gpu_metrics(struct smu_context *smu, metrics->Vcn1ActivityPercentage); gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; + gpu_metrics->energy_accumulator = smu->smc_fw_version <= 0x00521400 ? + metrics->EnergyAccumulator : UINT_MAX; if (metrics->AverageGfxActivity <= SMU_13_0_7_BUSY_THRESHOLD) gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; @@ -2275,7 +2307,8 @@ static int smu_v13_0_7_restore_user_od_settings(struct smu_context *smu) user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | BIT(PP_OD_FEATURE_UCLK_BIT) | BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | + BIT(PP_OD_FEATURE_ZERO_FAN_BIT); res = smu_v13_0_7_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c index f9b0938c57ea7..71885876c463e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c @@ -284,7 +284,6 @@ int smu_v14_0_check_fw_version(struct smu_context *smu) "smu fw program = %d, smu fw version = 0x%08x (%d.%d.%d)\n", smu->smc_driver_if_version, if_version, smu_program, smu_version, smu_major, smu_minor, smu_debug); - dev_info(adev->dev, "SMU driver if version not matched\n"); } return ret; @@ -1178,6 +1177,7 @@ int smu_v14_0_set_soft_freq_limited_range(struct smu_context *smu, return clk_id; if (max > 0) { + max = SMU_V14_SOFT_FREQ_ROUND(max); if (automatic) param = (uint32_t)((clk_id << 16) | 0xffff); else diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c index 086501cc5213b..470a901926f33 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c @@ -1701,8 +1701,9 @@ static int smu_v14_0_2_get_power_limit(struct smu_context *smu, table_context->power_play_table; PPTable_t *pptable = table_context->driver_pptable; CustomSkuTable_t *skutable = &pptable->CustomSkuTable; - uint32_t power_limit, od_percent_upper = 0, od_percent_lower = 0; + int16_t od_percent_upper = 0, od_percent_lower = 0; uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; + uint32_t power_limit; if (smu_v14_0_get_current_power_limit(smu, &power_limit)) power_limit = smu->adev->pm.ac_power ? @@ -2142,10 +2143,15 @@ static int smu_v14_0_2_mode1_reset(struct smu_context *smu) ret = smu_cmn_send_debug_smc_msg(smu, DEBUGSMC_MSG_Mode1Reset); if (!ret) { - if (amdgpu_emu_mode == 1) + if (amdgpu_emu_mode == 1) { msleep(50000); - else + } else { + /* disable mmio access while doing mode 1 reset*/ + smu->adev->no_hw_access = true; + /* ensure no_hw_access is globally visible before any MMIO */ + smp_mb(); msleep(1000); + } } return ret; @@ -2406,7 +2412,8 @@ static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu) user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | BIT(PP_OD_FEATURE_UCLK_BIT) | BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | + BIT(PP_OD_FEATURE_ZERO_FAN_BIT); res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) diff --git a/drivers/gpu/drm/ast/ast_dp501.c b/drivers/gpu/drm/ast/ast_dp501.c index 9e19d8c177308..677c52c0d99a3 100644 --- a/drivers/gpu/drm/ast/ast_dp501.c +++ b/drivers/gpu/drm/ast/ast_dp501.c @@ -436,7 +436,7 @@ static void ast_init_analog(struct ast_device *ast) /* Finally, clear bits [17:16] of SCU2c */ data = ast_read32(ast, 0x1202c); data &= 0xfffcffff; - ast_write32(ast, 0, data); + ast_write32(ast, 0x1202c, data); /* Disable DVO */ ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa3, 0xcf, 0x00); diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c index 4a7ba0918eca1..ae8d7b017968d 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c @@ -78,8 +78,6 @@ drm_plane_state_to_atmel_hlcdc_plane_state(struct drm_plane_state *s) return container_of(s, struct atmel_hlcdc_plane_state, base); } -#define SUBPIXEL_MASK 0xffff - static uint32_t rgb_formats[] = { DRM_FORMAT_C8, DRM_FORMAT_XRGB4444, @@ -365,13 +363,34 @@ void atmel_xlcdc_plane_setup_scaler(struct atmel_hlcdc_plane *plane, xfactor); /* - * With YCbCr 4:2:2 and YCbYcr 4:2:0 window resampling, configuration - * register LCDC_HEOCFG25.VXSCFACT and LCDC_HEOCFG27.HXSCFACT is half + * With YCbCr 4:2:0 window resampling, configuration register + * LCDC_HEOCFG25.VXSCFACT and LCDC_HEOCFG27.HXSCFACT values are half * the value of yfactor and xfactor. + * + * On the other hand, with YCbCr 4:2:2 window resampling, only the + * configuration register LCDC_HEOCFG27.HXSCFACT value is half the value + * of the xfactor; the value of LCDC_HEOCFG25.VXSCFACT is yfactor (no + * division by 2). */ - if (state->base.fb->format->format == DRM_FORMAT_YUV420) { + switch (state->base.fb->format->format) { + /* YCbCr 4:2:2 */ + case DRM_FORMAT_YUYV: + case DRM_FORMAT_UYVY: + case DRM_FORMAT_YVYU: + case DRM_FORMAT_VYUY: + case DRM_FORMAT_YUV422: + case DRM_FORMAT_NV61: + xfactor /= 2; + break; + + /* YCbCr 4:2:0 */ + case DRM_FORMAT_YUV420: + case DRM_FORMAT_NV21: yfactor /= 2; xfactor /= 2; + break; + default: + break; } atmel_hlcdc_layer_write_cfg(&plane->layer, desc->layout.scaler_config + 2, @@ -723,24 +742,15 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p, if (ret || !s->visible) return ret; - hstate->src_x = s->src.x1; - hstate->src_y = s->src.y1; - hstate->src_w = drm_rect_width(&s->src); - hstate->src_h = drm_rect_height(&s->src); + hstate->src_x = s->src.x1 >> 16; + hstate->src_y = s->src.y1 >> 16; + hstate->src_w = drm_rect_width(&s->src) >> 16; + hstate->src_h = drm_rect_height(&s->src) >> 16; hstate->crtc_x = s->dst.x1; hstate->crtc_y = s->dst.y1; hstate->crtc_w = drm_rect_width(&s->dst); hstate->crtc_h = drm_rect_height(&s->dst); - if ((hstate->src_x | hstate->src_y | hstate->src_w | hstate->src_h) & - SUBPIXEL_MASK) - return -EINVAL; - - hstate->src_x >>= 16; - hstate->src_y >>= 16; - hstate->src_w >>= 16; - hstate->src_h >>= 16; - hstate->nplanes = fb->format->num_planes; if (hstate->nplanes > ATMEL_HLCDC_LAYER_MAX_PLANES) return -EINVAL; @@ -1164,8 +1174,7 @@ atmel_hlcdc_plane_atomic_duplicate_state(struct drm_plane *p) return NULL; } - if (copy->base.fb) - drm_framebuffer_get(copy->base.fb); + __drm_atomic_helper_plane_duplicate_state(p, ©->base); return ©->base; } @@ -1183,8 +1192,7 @@ static void atmel_hlcdc_plane_atomic_destroy_state(struct drm_plane *p, state->dscrs[i]->self); } - if (s->fb) - drm_framebuffer_put(s->fb); + __drm_atomic_helper_plane_destroy_state(s); kfree(state); } diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig index a250afd8d6622..7fef383ed7cb7 100644 --- a/drivers/gpu/drm/bridge/Kconfig +++ b/drivers/gpu/drm/bridge/Kconfig @@ -184,6 +184,16 @@ config DRM_LONTIUM_LT9611UXC HDMI signals Please say Y if you have such hardware. +config DRM_LONTIUM_LT8713SX + tristate "Lontium LT8713SX DP MST bridge" + depends on OF + select REGMAP_I2C + help + Driver for Lontium LT8713SX DP MST bridge + chip firmware upgrade, which converts Type-C/DP1.4 + to 3 configurable Type-C/DP1.4/HDMI2.0 outputs + Please say Y if you have such hardware. + config DRM_ITE_IT66121 tristate "ITE IT66121 HDMI bridge" depends on OF diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile index c7dc03182e592..07eeb13fa4978 100644 --- a/drivers/gpu/drm/bridge/Makefile +++ b/drivers/gpu/drm/bridge/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_DRM_LONTIUM_LT8912B) += lontium-lt8912b.o obj-$(CONFIG_DRM_LONTIUM_LT9211) += lontium-lt9211.o obj-$(CONFIG_DRM_LONTIUM_LT9611) += lontium-lt9611.o obj-$(CONFIG_DRM_LONTIUM_LT9611UXC) += lontium-lt9611uxc.o +obj-$(CONFIG_DRM_LONTIUM_LT8713SX) += lontium-lt8713sx.o obj-$(CONFIG_DRM_LVDS_CODEC) += lvds-codec.o obj-$(CONFIG_DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW) += megachips-stdpxxxx-ge-b850v3-fw.o obj-$(CONFIG_DRM_MICROCHIP_LVDS_SERIALIZER) += microchip-lvds.o diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c index 6f3fdcb6afdb9..4e49e4f28d552 100644 --- a/drivers/gpu/drm/bridge/analogix/anx7625.c +++ b/drivers/gpu/drm/bridge/analogix/anx7625.c @@ -1801,7 +1801,7 @@ static const struct drm_edid *anx7625_edid_read(struct anx7625_data *ctx) return NULL; } - ctx->cached_drm_edid = drm_edid_alloc(edid_buf, FOUR_BLOCK_SIZE); + ctx->cached_drm_edid = drm_edid_alloc(edid_buf, edid_num * ONE_BLOCK_SIZE); kfree(edid_buf); out: diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c index e9f16dbc95353..27866ba158aa5 100644 --- a/drivers/gpu/drm/bridge/display-connector.c +++ b/drivers/gpu/drm/bridge/display-connector.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ struct display_connector { struct regulator *supply; struct gpio_desc *ddc_en; + + struct work_struct hpd_work; }; static inline struct display_connector * @@ -87,6 +90,34 @@ display_connector_bridge_detect(struct drm_bridge *bridge, struct drm_connector return display_connector_detect(bridge); } +static void display_connector_hpd_enable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + enable_irq(conn->hpd_irq); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + schedule_work(&conn->hpd_work); +} + +static void display_connector_hpd_disable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + cancel_work_sync(&conn->hpd_work); + + disable_irq(conn->hpd_irq); +} + +static void display_connector_hpd_work(struct work_struct *work) +{ + struct display_connector *conn = container_of(work, struct display_connector, hpd_work); + struct drm_bridge *bridge = &conn->bridge; + + drm_bridge_hpd_notify(bridge, display_connector_detect(bridge)); +} + static const struct drm_edid *display_connector_edid_read(struct drm_bridge *bridge, struct drm_connector *connector) { @@ -178,6 +209,8 @@ static u32 *display_connector_get_input_bus_fmts(struct drm_bridge *bridge, static const struct drm_bridge_funcs display_connector_bridge_funcs = { .attach = display_connector_attach, .detect = display_connector_bridge_detect, + .hpd_enable = display_connector_hpd_enable, + .hpd_disable = display_connector_hpd_disable, .edid_read = display_connector_edid_read, .atomic_get_output_bus_fmts = display_connector_get_output_bus_fmts, .atomic_get_input_bus_fmts = display_connector_get_input_bus_fmts, @@ -307,6 +340,7 @@ static int display_connector_probe(struct platform_device *pdev) NULL, display_connector_hpd_irq, IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | + IRQF_NO_AUTOEN | IRQF_ONESHOT, "HPD", conn); if (ret) { @@ -378,6 +412,8 @@ static int display_connector_probe(struct platform_device *pdev) conn->bridge.ops |= DRM_BRIDGE_OP_DETECT; if (conn->hpd_irq >= 0) conn->bridge.ops |= DRM_BRIDGE_OP_HPD; + if (conn->hpd_irq >= 0 && type == DRM_MODE_CONNECTOR_DisplayPort) + INIT_WORK(&conn->hpd_work, display_connector_hpd_work); dev_dbg(&pdev->dev, "Found %s display connector '%s' %s DDC bus and %s HPD GPIO (ops 0x%x)\n", diff --git a/drivers/gpu/drm/bridge/lontium-lt8713sx.c b/drivers/gpu/drm/bridge/lontium-lt8713sx.c new file mode 100644 index 0000000000000..6ea54ff3733d4 --- /dev/null +++ b/drivers/gpu/drm/bridge/lontium-lt8713sx.c @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define FW_FILE "lt8713sx_fw.bin" + +#define REG_PAGE_CONTROL 0xff + +#define MAX_OUTPUT_PORTS 3 +#define LT8713SX_PAGE_SIZE 256 + +DECLARE_CRC8_TABLE(lt8713sx_crc_table); + +struct lt8713sx { + struct device *dev; + struct drm_bridge bridge; + struct drm_bridge *next_bridge[MAX_OUTPUT_PORTS]; + int num_outputs; + + struct regmap *regmap; + /* Protects all accesses to registers by stopping the on-chip MCU */ + struct mutex ocm_lock; + + struct gpio_desc *reset_gpio; + struct gpio_desc *enable_gpio; + + struct i2c_client *client; + const struct firmware *fw; + + u8 *fw_buffer; + + u32 main_crc_value; + u32 bank_crc_value[17]; + + int bank_num; +}; + +static void lt8713sx_reset(struct lt8713sx *lt8713sx); + +static const struct regmap_range lt8713sx_ranges[] = { + { + .range_min = 0x0000, + .range_max = 0xffff + }, +}; + +static const struct regmap_access_table lt8713sx_table = { + .yes_ranges = lt8713sx_ranges, + .n_yes_ranges = ARRAY_SIZE(lt8713sx_ranges), +}; + +static const struct regmap_range_cfg lt8713sx_range_cfg = { + .name = "lt8713sx", + .range_min = 0x0000, + .range_max = 0xffff, + .selector_reg = REG_PAGE_CONTROL, + .selector_mask = 0xff, + .selector_shift = 0, + .window_start = 0, + .window_len = 0x100, +}; + +static const struct regmap_config lt8713sx_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .volatile_table = <8713sx_table, + .ranges = <8713sx_range_cfg, + .num_ranges = 1, + .cache_type = REGCACHE_NONE, + .max_register = 0xffff, +}; + +static void lt8713sx_i2c_enable(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe0ee, 0x01); +} + +static void lt8713sx_i2c_disable(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe0ee, 0x00); +} + +static u32 calculate_crc(const u8 *upgrade_data, u64 len, u64 crc_size) +{ + u8 crc = 0x00; + u8 pad = 0xff; + + crc = crc8(lt8713sx_crc_table, upgrade_data, len, crc); + + /* pad remaining bytes */ + crc_size -= len; + while (crc_size--) + crc = crc8(lt8713sx_crc_table, &pad, 1, crc); + + return crc; +} + +static int lt8713sx_prepare_firmware_data(struct lt8713sx *lt8713sx) +{ + int ret = 0; + u64 sz_12k = 12 * SZ_1K; + + ret = request_firmware(<8713sx->fw, FW_FILE, lt8713sx->dev); + if (ret < 0) { + dev_err(lt8713sx->dev, "request firmware failed\n"); + return ret; + } + + dev_dbg(lt8713sx->dev, "Firmware size: %zu bytes\n", lt8713sx->fw->size); + + if (lt8713sx->fw->size > SZ_256K - 1) { + dev_err(lt8713sx->dev, "Firmware size exceeds 256KB limit\n"); + release_firmware(lt8713sx->fw); + return -EINVAL; + } + + lt8713sx->fw_buffer = kvmalloc(SZ_256K, GFP_KERNEL); + if (!lt8713sx->fw_buffer) { + release_firmware(lt8713sx->fw); + return -ENOMEM; + } + + memset(lt8713sx->fw_buffer, 0xff, SZ_256K); + + if (lt8713sx->fw->size < SZ_64K) { + memcpy(lt8713sx->fw_buffer, lt8713sx->fw->data, lt8713sx->fw->size); + lt8713sx->fw_buffer[SZ_64K - 1] = + calculate_crc(lt8713sx->fw->data, lt8713sx->fw->size, SZ_64K - 1); + lt8713sx->main_crc_value = lt8713sx->fw_buffer[SZ_64K - 1]; + dev_dbg(lt8713sx->dev, + "Main Firmware Data Crc=0x%02X\n", lt8713sx->main_crc_value); + + } else { + /* main firmware */ + memcpy(lt8713sx->fw_buffer, lt8713sx->fw->data, SZ_64K - 1); + lt8713sx->fw_buffer[SZ_64K - 1] = + calculate_crc(lt8713sx->fw_buffer, SZ_64K - 1, SZ_64K - 1); + lt8713sx->main_crc_value = lt8713sx->fw_buffer[SZ_64K - 1]; + dev_dbg(lt8713sx->dev, + "Main Firmware Data Crc=0x%02X\n", lt8713sx->main_crc_value); + + /* bank firmware */ + memcpy(lt8713sx->fw_buffer + SZ_64K, + lt8713sx->fw->data + SZ_64K, + lt8713sx->fw->size - SZ_64K); + + lt8713sx->bank_num = (lt8713sx->fw->size - SZ_64K + sz_12k - 1) / sz_12k; + dev_dbg(lt8713sx->dev, "Bank Number Total is %d.\n", lt8713sx->bank_num); + + for (int i = 0; i < lt8713sx->bank_num; i++) { + lt8713sx->bank_crc_value[i] = + calculate_crc(lt8713sx->fw_buffer + SZ_64K + i * sz_12k, + sz_12k, sz_12k); + dev_dbg(lt8713sx->dev, "Bank number:%d; Firmware Data Crc:0x%02X\n", + i, lt8713sx->bank_crc_value[i]); + } + } + return 0; +} + +static void lt8713sx_config_parameters(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe0ee, 0x01); + regmap_write(lt8713sx->regmap, 0xe05e, 0xc1); + regmap_write(lt8713sx->regmap, 0xe058, 0x00); + regmap_write(lt8713sx->regmap, 0xe059, 0x50); + regmap_write(lt8713sx->regmap, 0xe05a, 0x10); + regmap_write(lt8713sx->regmap, 0xe05a, 0x00); + regmap_write(lt8713sx->regmap, 0xe058, 0x21); +} + +static void lt8713sx_wren(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe103, 0xbf); + regmap_write(lt8713sx->regmap, 0xe103, 0xff); + regmap_write(lt8713sx->regmap, 0xe05a, 0x04); + regmap_write(lt8713sx->regmap, 0xe05a, 0x00); +} + +static void lt8713sx_wrdi(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe05a, 0x08); + regmap_write(lt8713sx->regmap, 0xe05a, 0x00); +} + +static void lt8713sx_fifo_reset(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe103, 0xbf); + regmap_write(lt8713sx->regmap, 0xe103, 0xff); +} + +static void lt8713sx_disable_sram_write(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe055, 0x00); +} + +static void lt8713sx_sram_to_flash(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe05a, 0x30); + regmap_write(lt8713sx->regmap, 0xe05a, 0x00); +} + +static void lt8713sx_i2c_to_sram(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe055, 0x80); + regmap_write(lt8713sx->regmap, 0xe05e, 0xc0); + regmap_write(lt8713sx->regmap, 0xe058, 0x21); +} + +static u8 lt8713sx_read_flash_status(struct lt8713sx *lt8713sx) +{ + u32 flash_status = 0; + + regmap_write(lt8713sx->regmap, 0xe103, 0x3f); + regmap_write(lt8713sx->regmap, 0xe103, 0xff); + + regmap_write(lt8713sx->regmap, 0xe05e, 0x40); + regmap_write(lt8713sx->regmap, 0xe056, 0x05); /* opcode=read status register */ + regmap_write(lt8713sx->regmap, 0xe055, 0x25); + regmap_write(lt8713sx->regmap, 0xe055, 0x01); + regmap_write(lt8713sx->regmap, 0xe058, 0x21); + + regmap_read(lt8713sx->regmap, 0xe05f, &flash_status); + dev_dbg(lt8713sx->dev, "flash_status:%x\n", flash_status); + + return flash_status; +} + +static void lt8713sx_block_erase(struct lt8713sx *lt8713sx) +{ + u32 i = 0; + u8 flash_status = 0; + u8 blocknum = 0x00; + u32 flashaddr = 0x00; + + for (blocknum = 0; blocknum < 8; blocknum++) { + flashaddr = blocknum * SZ_32K; + regmap_write(lt8713sx->regmap, 0xe0ee, 0x01); + regmap_write(lt8713sx->regmap, 0xe05a, 0x04); + regmap_write(lt8713sx->regmap, 0xe05a, 0x00); + regmap_write(lt8713sx->regmap, 0xe05b, flashaddr >> 16); + regmap_write(lt8713sx->regmap, 0xe05c, flashaddr >> 8); + regmap_write(lt8713sx->regmap, 0xe05d, flashaddr); + regmap_write(lt8713sx->regmap, 0xe05a, 0x01); + regmap_write(lt8713sx->regmap, 0xe05a, 0x00); + msleep(100); + i = 0; + while (1) { + flash_status = lt8713sx_read_flash_status(lt8713sx); + if ((flash_status & 0x01) == 0) + break; + + if (i > 50) + break; + + i++; + msleep(50); + } + } + dev_dbg(lt8713sx->dev, "erase flash done.\n"); +} + +static void lt8713sx_load_main_fw_to_sram(struct lt8713sx *lt8713sx) +{ + regmap_write(lt8713sx->regmap, 0xe0ee, 0x01); + regmap_write(lt8713sx->regmap, 0xe068, 0x00); + regmap_write(lt8713sx->regmap, 0xe069, 0x00); + regmap_write(lt8713sx->regmap, 0xe06a, 0x00); + regmap_write(lt8713sx->regmap, 0xe065, 0x00); + regmap_write(lt8713sx->regmap, 0xe066, 0xff); + regmap_write(lt8713sx->regmap, 0xe067, 0xff); + regmap_write(lt8713sx->regmap, 0xe06b, 0x00); + regmap_write(lt8713sx->regmap, 0xe06c, 0x00); + regmap_write(lt8713sx->regmap, 0xe060, 0x01); + msleep(200); + regmap_write(lt8713sx->regmap, 0xe060, 0x00); +} + +static void lt8713sx_load_bank_fw_to_sram(struct lt8713sx *lt8713sx, u64 addr) +{ + regmap_write(lt8713sx->regmap, 0xe0ee, 0x01); + regmap_write(lt8713sx->regmap, 0xe068, ((addr & 0xff0000) >> 16)); + regmap_write(lt8713sx->regmap, 0xe069, ((addr & 0x00ff00) >> 8)); + regmap_write(lt8713sx->regmap, 0xe06a, (addr & 0x0000ff)); + regmap_write(lt8713sx->regmap, 0xe065, 0x00); + regmap_write(lt8713sx->regmap, 0xe066, 0x30); + regmap_write(lt8713sx->regmap, 0xe067, 0x00); + regmap_write(lt8713sx->regmap, 0xe06b, 0x00); + regmap_write(lt8713sx->regmap, 0xe06c, 0x00); + regmap_write(lt8713sx->regmap, 0xe060, 0x01); + msleep(50); + regmap_write(lt8713sx->regmap, 0xe060, 0x00); +} + +static int lt8713sx_write_data(struct lt8713sx *lt8713sx, const u8 *data, u64 filesize) +{ + int page = 0, num = 0, i = 0, val; + + page = (filesize % LT8713SX_PAGE_SIZE) ? + ((filesize / LT8713SX_PAGE_SIZE) + 1) : (filesize / LT8713SX_PAGE_SIZE); + + dev_dbg(lt8713sx->dev, + "Writing to Sram=%u pages, total size = %llu bytes\n", page, filesize); + + for (num = 0; num < page; num++) { + dev_dbg(lt8713sx->dev, "page[%d]\n", num); + lt8713sx_i2c_to_sram(lt8713sx); + + for (i = 0; i < LT8713SX_PAGE_SIZE; i++) { + if ((num * LT8713SX_PAGE_SIZE + i) < filesize) + val = *(data + (num * LT8713SX_PAGE_SIZE + i)); + else + val = 0xff; + regmap_write(lt8713sx->regmap, 0xe059, val); + } + + lt8713sx_wren(lt8713sx); + lt8713sx_sram_to_flash(lt8713sx); + } + + lt8713sx_wrdi(lt8713sx); + lt8713sx_disable_sram_write(lt8713sx); + + return 0; +} + +static void lt8713sx_main_upgrade_result(struct lt8713sx *lt8713sx) +{ + u32 main_crc_result; + + regmap_read(lt8713sx->regmap, 0xe023, &main_crc_result); + + dev_dbg(lt8713sx->dev, "Main CRC HW: 0x%02X\n", main_crc_result); + dev_dbg(lt8713sx->dev, "Main CRC FW: 0x%02X\n", lt8713sx->main_crc_value); + + if (main_crc_result == lt8713sx->main_crc_value) + dev_dbg(lt8713sx->dev, "Main Firmware Upgrade Success.\n"); + else + dev_err(lt8713sx->dev, "Main Firmware Upgrade Failed.\n"); +} + +static void lt8713sx_bank_upgrade_result(struct lt8713sx *lt8713sx, u8 banknum) +{ + u32 bank_crc_result; + + regmap_read(lt8713sx->regmap, 0xe023, &bank_crc_result); + + dev_dbg(lt8713sx->dev, "Bank %d CRC Result: 0x%02X\n", banknum, bank_crc_result); + + if (bank_crc_result == lt8713sx->bank_crc_value[banknum]) + dev_dbg(lt8713sx->dev, "Bank %d Firmware Upgrade Success.\n", banknum); + else + dev_err(lt8713sx->dev, "Bank %d Firmware Upgrade Failed.\n", banknum); +} + +static void lt8713sx_bank_result_check(struct lt8713sx *lt8713sx) +{ + int i; + u64 addr = 0x010000; + + for (i = 0; i < lt8713sx->bank_num; i++) { + lt8713sx_load_bank_fw_to_sram(lt8713sx, addr); + lt8713sx_bank_upgrade_result(lt8713sx, i); + addr += 0x3000; + } +} + +static int lt8713sx_firmware_upgrade(struct lt8713sx *lt8713sx) +{ + int ret; + + lt8713sx_config_parameters(lt8713sx); + + lt8713sx_block_erase(lt8713sx); + + if (lt8713sx->fw->size < SZ_64K) { + ret = lt8713sx_write_data(lt8713sx, lt8713sx->fw_buffer, SZ_64K); + if (ret < 0) { + dev_err(lt8713sx->dev, "Failed to write firmware data: %d\n", ret); + return ret; + } + } else { + ret = lt8713sx_write_data(lt8713sx, lt8713sx->fw_buffer, lt8713sx->fw->size); + if (ret < 0) { + dev_err(lt8713sx->dev, "Failed to write firmware data: %d\n", ret); + return ret; + } + } + dev_dbg(lt8713sx->dev, "Write Data done.\n"); + + return 0; +} + +static int lt8713sx_firmware_update(struct lt8713sx *lt8713sx) +{ + int ret = 0; + + mutex_lock(<8713sx->ocm_lock); + lt8713sx_i2c_enable(lt8713sx); + + ret = lt8713sx_prepare_firmware_data(lt8713sx); + if (ret < 0) { + dev_err(lt8713sx->dev, "Failed to prepare firmware data: %d\n", ret); + goto error; + } + + ret = lt8713sx_firmware_upgrade(lt8713sx); + if (ret < 0) { + dev_err(lt8713sx->dev, "Upgrade failure.\n"); + goto error; + } else { + /* Validate CRC */ + lt8713sx_load_main_fw_to_sram(lt8713sx); + lt8713sx_main_upgrade_result(lt8713sx); + lt8713sx_wrdi(lt8713sx); + lt8713sx_fifo_reset(lt8713sx); + lt8713sx_bank_result_check(lt8713sx); + lt8713sx_wrdi(lt8713sx); + } + +error: + lt8713sx_i2c_disable(lt8713sx); + if (!ret) + lt8713sx_reset(lt8713sx); + + kvfree(lt8713sx->fw_buffer); + lt8713sx->fw_buffer = NULL; + + if (lt8713sx->fw) { + release_firmware(lt8713sx->fw); + lt8713sx->fw = NULL; + } + mutex_unlock(<8713sx->ocm_lock); + + return ret; +} + +static void lt8713sx_reset(struct lt8713sx *lt8713sx) +{ + dev_dbg(lt8713sx->dev, "reset bridge.\n"); + gpiod_set_value_cansleep(lt8713sx->reset_gpio, 1); + msleep(20); + + gpiod_set_value_cansleep(lt8713sx->reset_gpio, 0); + msleep(20); + + dev_dbg(lt8713sx->dev, "reset done.\n"); +} + +static int lt8713sx_regulator_enable(struct lt8713sx *lt8713sx) +{ + int ret; + + ret = devm_regulator_get_enable(lt8713sx->dev, "vdd"); + if (ret < 0) + return dev_err_probe(lt8713sx->dev, ret, "failed to enable vdd regulator\n"); + + usleep_range(1000, 10000); + + ret = devm_regulator_get_enable(lt8713sx->dev, "vcc"); + if (ret < 0) + return dev_err_probe(lt8713sx->dev, ret, "failed to enable vcc regulator\n"); + return 0; +} + +static int lt8713sx_bridge_attach(struct drm_bridge *bridge, + struct drm_encoder *encoder, + enum drm_bridge_attach_flags flags) +{ + struct lt8713sx *lt8713sx = container_of(bridge, struct lt8713sx, bridge); + int i, ret; + + for (i = 0; i < lt8713sx->num_outputs; i++) { + if (!lt8713sx->next_bridge[i]) + continue; + + ret = drm_bridge_attach(encoder, + lt8713sx->next_bridge[i], + bridge, flags); + if (ret) + return ret; + } + + return 0; +} + +static int lt8713sx_get_ports(struct lt8713sx *lt8713sx) +{ + struct device *dev = lt8713sx->dev; + struct device_node *port, *ports, *ep, *remote; + int i = 0; + u32 reg; + + ports = of_get_child_by_name(dev->of_node, "ports"); + if (!ports) + return -ENODEV; + + for_each_child_of_node(ports, port) { + if (of_property_read_u32(port, "reg", ®)) + continue; + + if (reg == 0) + continue; + + if (i >= ARRAY_SIZE(lt8713sx->next_bridge)) { + of_node_put(port); + break; + } + + ep = of_graph_get_next_endpoint(port, NULL); + if (!ep) + continue; + + remote = of_graph_get_remote_port_parent(ep); + of_node_put(ep); + + if (!remote) + continue; + + lt8713sx->next_bridge[i] = of_drm_find_bridge(remote); + of_node_put(remote); + if (lt8713sx->next_bridge[i]) + i++; + } + lt8713sx->num_outputs = i; + dev_dbg(dev, "Enabled %d output ports", i); + + of_node_put(ports); + return 0; +}; + +static int lt8713sx_gpio_init(struct lt8713sx *lt8713sx) +{ + struct device *dev = lt8713sx->dev; + + lt8713sx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(lt8713sx->reset_gpio)) + return dev_err_probe(dev, PTR_ERR(lt8713sx->reset_gpio), + "failed to acquire reset gpio\n"); + + /* power enable gpio */ + lt8713sx->enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH); + if (IS_ERR(lt8713sx->enable_gpio)) + return dev_err_probe(dev, PTR_ERR(lt8713sx->enable_gpio), + "failed to acquire enable gpio\n"); + return 0; +} + +static ssize_t lt8713sx_firmware_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct lt8713sx *lt8713sx = dev_get_drvdata(dev); + int ret; + + ret = lt8713sx_firmware_update(lt8713sx); + if (ret < 0) + return ret; + return len; +} + +static DEVICE_ATTR_WO(lt8713sx_firmware); + +static struct attribute *lt8713sx_attrs[] = { + &dev_attr_lt8713sx_firmware.attr, + NULL, +}; + +static const struct attribute_group lt8713sx_attr_group = { + .attrs = lt8713sx_attrs, +}; + +static const struct attribute_group *lt8713sx_attr_groups[] = { + <8713sx_attr_group, + NULL, +}; + +static const struct drm_bridge_funcs lt8713sx_bridge_funcs = { + .attach = lt8713sx_bridge_attach, +}; + +static int lt8713sx_probe(struct i2c_client *client) +{ + struct lt8713sx *lt8713sx; + struct device *dev = &client->dev; + int ret; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return dev_err_probe(dev, -ENODEV, "device doesn't support I2C\n"); + + lt8713sx = devm_drm_bridge_alloc(dev, struct lt8713sx, bridge, <8713sx_bridge_funcs); + if (IS_ERR(lt8713sx)) + return PTR_ERR(lt8713sx); + + lt8713sx->dev = dev; + lt8713sx->client = client; + i2c_set_clientdata(client, lt8713sx); + + ret = devm_mutex_init(lt8713sx->dev, <8713sx->ocm_lock); + if (ret) + return ret; + + lt8713sx->regmap = devm_regmap_init_i2c(client, <8713sx_regmap_config); + if (IS_ERR(lt8713sx->regmap)) + return dev_err_probe(dev, PTR_ERR(lt8713sx->regmap), "regmap i2c init failed\n"); + + ret = lt8713sx_get_ports(lt8713sx); + if (ret < 0) + return ret; + + ret = lt8713sx_gpio_init(lt8713sx); + if (ret < 0) + return ret; + + ret = lt8713sx_regulator_enable(lt8713sx); + if (ret) + return ret; + + lt8713sx_reset(lt8713sx); + + lt8713sx->bridge.funcs = <8713sx_bridge_funcs; + lt8713sx->bridge.of_node = dev->of_node; + lt8713sx->bridge.type = DRM_MODE_CONNECTOR_DisplayPort; + drm_bridge_add(<8713sx->bridge); + + crc8_populate_msb(lt8713sx_crc_table, 0x31); + + return 0; +} + +static void lt8713sx_remove(struct i2c_client *client) +{ + struct lt8713sx *lt8713sx = i2c_get_clientdata(client); + + drm_bridge_remove(<8713sx->bridge); +} + +static struct i2c_device_id lt8713sx_id[] = { + { "lontium,lt8713sx", 0 }, + { /* sentinel */ } +}; + +static const struct of_device_id lt8713sx_match_table[] = { + { .compatible = "lontium,lt8713sx" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, lt8713sx_match_table); + +static struct i2c_driver lt8713sx_driver = { + .driver = { + .name = "lt8713sx", + .of_match_table = lt8713sx_match_table, + .dev_groups = lt8713sx_attr_groups, + }, + .probe = lt8713sx_probe, + .remove = lt8713sx_remove, + .id_table = lt8713sx_id, +}; + +module_i2c_driver(lt8713sx_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("lt8713sx drm bridge driver"); +MODULE_AUTHOR("Tony "); +MODULE_FIRMWARE(FW_FILE); diff --git a/drivers/gpu/drm/bridge/lontium-lt9211.c b/drivers/gpu/drm/bridge/lontium-lt9211.c index 03fc8fd10f20a..95fefab9c0216 100644 --- a/drivers/gpu/drm/bridge/lontium-lt9211.c +++ b/drivers/gpu/drm/bridge/lontium-lt9211.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -28,28 +29,48 @@ #include #include -#define REG_PAGE_CONTROL 0xff -#define REG_CHIPID0 0x8100 -#define REG_CHIPID0_VALUE 0x18 -#define REG_CHIPID1 0x8101 -#define REG_CHIPID1_VALUE 0x01 -#define REG_CHIPID2 0x8102 -#define REG_CHIPID2_VALUE 0xe3 +#define REG_PAGE_CONTROL 0xff +#define REG_CHIPID0 0x8100 +#define REG_CHIPID0_VALUE 0x18 +#define REG_CHIPID1 0x8101 +#define REG_CHIPID1_VALUE 0x01 +#define REG_CHIPID2 0x8102 +#define REG_CHIPID2_VALUE 0xe3 -#define REG_DSI_LANE 0xd000 +/* LT9211C chip ID values */ +#define REG_CHIPID0_LT9211C_VALUE 0x21 +#define REG_CHIPID1_LT9211C_VALUE 0x03 +#define REG_CHIPID2_LT9211C_VALUE 0xe1 + +#define REG_DSI_LANE 0xd000 /* DSI lane count - 0 means 4 lanes ; 1, 2, 3 means 1, 2, 3 lanes. */ -#define REG_DSI_LANE_COUNT(n) ((n) & 3) +#define REG_DSI_LANE_COUNT(n) ((n) & 3) + +/* Chip type enum */ +enum lt9211_chip_type { + LT9211, + LT9211C, +}; struct lt9211 { - struct drm_bridge bridge; - struct device *dev; - struct regmap *regmap; - struct mipi_dsi_device *dsi; - struct drm_bridge *panel_bridge; - struct gpio_desc *reset_gpio; - struct regulator *vccio; - bool lvds_dual_link; - bool lvds_dual_link_even_odd_swap; + struct drm_bridge bridge; + struct device *dev; + struct regmap *regmap; + struct mipi_dsi_device *dsi; + struct drm_bridge *panel_bridge; + struct gpio_desc *reset_gpio; + struct regulator *vccio; + bool lvds_dual_link; + bool lvds_dual_link_even_odd_swap; + + /* LT9211C specific fields */ + enum lt9211_chip_type chip_type; + struct workqueue_struct *wq; + struct delayed_work lt9211_dw; + struct drm_display_mode mode; + bool bpp24; + bool jeida; + bool de; }; static const struct regmap_range lt9211_rw_ranges[] = { @@ -86,13 +107,56 @@ static const struct regmap_config lt9211_regmap_config = { .val_bits = 8, .rd_table = <9211_rw_table, .wr_table = <9211_rw_table, - .volatile_table = <9211_rw_table, + .volatile_table = <9211_rw_table, .ranges = <9211_range, .num_ranges = 1, .cache_type = REGCACHE_MAPLE, .max_register = 0xda00, }; +static const struct regmap_range lt9211c_rw_ranges[] = { + regmap_reg_range(0xff, 0xff), + regmap_reg_range(0x8100, 0x8182), + regmap_reg_range(0x8200, 0x82aa), + regmap_reg_range(0x8500, 0x85ff), + regmap_reg_range(0x8600, 0x86a0), + regmap_reg_range(0x8700, 0x8746), + regmap_reg_range(0xd000, 0xd0a7), + regmap_reg_range(0xd400, 0xd42c), + regmap_reg_range(0xd800, 0xd838), + regmap_reg_range(0xd9c0, 0xd9d5), +}; + +static const struct regmap_access_table lt9211c_rw_table = { + .yes_ranges = lt9211c_rw_ranges, + .n_yes_ranges = ARRAY_SIZE(lt9211c_rw_ranges), +}; + +static const struct regmap_range_cfg lt9211c_range = { + .name = "lt9211c", + .range_min = 0x0000, + .range_max = 0xda00, + .selector_reg = REG_PAGE_CONTROL, + .selector_mask = 0xff, + .selector_shift = 0, + .window_start = 0, + .window_len = 0x100, +}; + +static const struct regmap_config lt9211c_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = <9211c_rw_table, + .wr_table = <9211c_rw_table, + .volatile_table = <9211c_rw_table, + .ranges = <9211c_range, + .num_ranges = 1, + .cache_type = REGCACHE_RBTREE, + .max_register = 0xda00, +}; + +static void lt9211_delayed_work_func(struct work_struct *work); + static struct lt9211 *bridge_to_lt9211(struct drm_bridge *bridge) { return container_of(bridge, struct lt9211, bridge); @@ -120,14 +184,24 @@ static int lt9211_read_chipid(struct lt9211 *ctx) return ret; } - /* Test for known Chip ID. */ - if (chipid[0] != REG_CHIPID0_VALUE || chipid[1] != REG_CHIPID1_VALUE) { - dev_err(ctx->dev, "Unknown Chip ID: 0x%02x 0x%02x 0x%02x\n", - chipid[0], chipid[1], chipid[2]); - return -EINVAL; + /* Test for LT9211 Chip ID. */ + if (chipid[0] == REG_CHIPID0_VALUE && chipid[1] == REG_CHIPID1_VALUE && + chipid[2] == REG_CHIPID2_VALUE) { + dev_dbg(ctx->dev, "Detected LT9211 chip\n"); + return 0; } - return 0; + /* Test for LT9211C Chip ID. */ + if (chipid[0] == REG_CHIPID0_LT9211C_VALUE && + chipid[1] == REG_CHIPID1_LT9211C_VALUE && + chipid[2] == REG_CHIPID2_LT9211C_VALUE) { + dev_dbg(ctx->dev, "Detected LT9211C chip\n"); + return 0; + } + + dev_err(ctx->dev, "Unknown Chip ID: 0x%02x 0x%02x 0x%02x\n", chipid[0], + chipid[1], chipid[2]); + return -EINVAL; } static int lt9211_system_init(struct lt9211 *ctx) @@ -250,9 +324,9 @@ static int lt9211_autodetect_rx(struct lt9211 *ctx, height = (buf[3] << 8) | buf[4]; format = buf[2] & 0xf; - if (format == 0x3) { /* YUV422 16bit */ + if (format == 0x3) { /* YUV422 16bit */ width /= 2; - } else if (format == 0xa) { /* RGB888 24bit */ + } else if (format == 0xa) { /* RGB888 24bit */ width /= 3; } else { dev_err(ctx->dev, "Unsupported DSI pixel format 0x%01x\n", @@ -476,7 +550,7 @@ static void lt9211_atomic_enable(struct drm_bridge *bridge, /* Deassert reset */ gpiod_set_value(ctx->reset_gpio, 1); - usleep_range(20000, 21000); /* Very long post-reset delay. */ + usleep_range(20000, 21000); /* Very long post-reset delay. */ /* Get the LVDS format from the bridge state. */ bridge_state = drm_atomic_get_new_bridge_state(state, bridge); @@ -504,8 +578,8 @@ static void lt9211_atomic_enable(struct drm_bridge *bridge, lvds_format_24bpp = true; lvds_format_jeida = false; dev_warn(ctx->dev, - "Unsupported LVDS bus format 0x%04x, please check output bridge driver. Falling back to SPWG24.\n", - bridge_state->output_bus_cfg.format); + "Unsupported LVDS bus format 0x%04x, please check output bridge driver. Falling back to SPWG24.\n", + bridge_state->output_bus_cfg.format); break; } @@ -518,35 +592,42 @@ static void lt9211_atomic_enable(struct drm_bridge *bridge, crtc = drm_atomic_get_new_connector_state(state, connector)->crtc; crtc_state = drm_atomic_get_new_crtc_state(state, crtc); mode = &crtc_state->adjusted_mode; - ret = lt9211_read_chipid(ctx); if (ret) return; - ret = lt9211_system_init(ctx); - if (ret) + if (ctx->chip_type == LT9211C && ctx->wq) { + drm_mode_copy(&ctx->mode, mode); + /* LT9211C must enable after mipi clock enable */ + queue_delayed_work(ctx->wq, &ctx->lt9211_dw, + msecs_to_jiffies(100)); + dev_dbg(ctx->dev, "LT9211C enabled.\n"); return; + } + ret = lt9211_system_init(ctx); + if (ret) + return; - ret = lt9211_configure_rx(ctx); - if (ret) - return; + ret = lt9211_configure_rx(ctx); + if (ret) + return; - ret = lt9211_autodetect_rx(ctx, mode); - if (ret) - return; + ret = lt9211_autodetect_rx(ctx, mode); + if (ret) + return; - ret = lt9211_configure_timing(ctx, mode); - if (ret) - return; + ret = lt9211_configure_timing(ctx, mode); + if (ret) + return; - ret = lt9211_configure_plls(ctx, mode); - if (ret) - return; + ret = lt9211_configure_plls(ctx, mode); + if (ret) + return; ret = lt9211_configure_tx(ctx, lvds_format_jeida, lvds_format_24bpp, - bus_flags & DRM_BUS_FLAG_DE_HIGH); - if (ret) - return; + bus_flags & DRM_BUS_FLAG_DE_HIGH); + if (ret) + return; dev_dbg(ctx->dev, "LT9211 enabled.\n"); } @@ -562,7 +643,7 @@ static void lt9211_atomic_disable(struct drm_bridge *bridge, * and assure lengthy 10ms reset low timing. */ gpiod_set_value(ctx->reset_gpio, 0); - usleep_range(10000, 11000); /* Very long reset duration. */ + usleep_range(10000, 11000); /* Very long reset duration. */ ret = regulator_disable(ctx->vccio); if (ret) @@ -585,7 +666,7 @@ lt9211_mode_valid(struct drm_bridge *bridge, return MODE_OK; } -#define MAX_INPUT_SEL_FORMATS 1 +#define MAX_INPUT_SEL_FORMATS 1 static u32 * lt9211_atomic_get_input_bus_fmts(struct drm_bridge *bridge, @@ -612,14 +693,14 @@ lt9211_atomic_get_input_bus_fmts(struct drm_bridge *bridge, } static const struct drm_bridge_funcs lt9211_funcs = { - .attach = lt9211_attach, - .mode_valid = lt9211_mode_valid, - .atomic_enable = lt9211_atomic_enable, - .atomic_disable = lt9211_atomic_disable, + .attach = lt9211_attach, + .mode_valid = lt9211_mode_valid, + .atomic_enable = lt9211_atomic_enable, + .atomic_disable = lt9211_atomic_disable, .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, .atomic_get_input_bus_fmts = lt9211_atomic_get_input_bus_fmts, - .atomic_reset = drm_atomic_helper_bridge_reset, + .atomic_reset = drm_atomic_helper_bridge_reset, }; static int lt9211_parse_dt(struct lt9211 *ctx) @@ -671,11 +752,6 @@ static int lt9211_parse_dt(struct lt9211 *ctx) static int lt9211_host_attach(struct lt9211 *ctx) { - const struct mipi_dsi_device_info info = { - .type = "lt9211", - .channel = 0, - .node = NULL, - }; struct device *dev = ctx->dev; struct device_node *host_node; struct device_node *endpoint; @@ -684,6 +760,8 @@ static int lt9211_host_attach(struct lt9211 *ctx) int dsi_lanes; int ret; + /* Check if the compatible string matches lt9211c */ + endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, -1); dsi_lanes = drm_of_get_data_lanes_count(endpoint, 1, 4); host_node = of_graph_get_remote_port_parent(endpoint); @@ -697,7 +775,22 @@ static int lt9211_host_attach(struct lt9211 *ctx) if (dsi_lanes < 0) return dsi_lanes; - dsi = devm_mipi_dsi_device_register_full(dev, host, &info); + if (ctx->chip_type == LT9211C) { + const struct mipi_dsi_device_info info = { + .type = "lt9211c", + .channel = 0, + .node = NULL, + }; + dsi = devm_mipi_dsi_device_register_full(dev, host, &info); + } else { + const struct mipi_dsi_device_info info = { + .type = "lt9211", + .channel = 0, + .node = NULL, + }; + dsi = devm_mipi_dsi_device_register_full(dev, host, &info); + } + if (IS_ERR(dsi)) return dev_err_probe(dev, PTR_ERR(dsi), "failed to create dsi device\n"); @@ -706,10 +799,16 @@ static int lt9211_host_attach(struct lt9211 *ctx) dsi->lanes = dsi_lanes; dsi->format = MIPI_DSI_FMT_RGB888; - dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE | - MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO_NO_HSA | - MIPI_DSI_MODE_VIDEO_NO_HFP | MIPI_DSI_MODE_VIDEO_NO_HBP | - MIPI_DSI_MODE_NO_EOT_PACKET; + + if (ctx->chip_type == LT9211C) { + dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_LPM; + } else { + dsi->mode_flags = + MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE | + MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO_NO_HSA | + MIPI_DSI_MODE_VIDEO_NO_HFP | MIPI_DSI_MODE_VIDEO_NO_HBP | + MIPI_DSI_MODE_NO_EOT_PACKET; + } ret = devm_mipi_dsi_attach(dev, dsi); if (ret < 0) { @@ -731,6 +830,7 @@ static int lt9211_probe(struct i2c_client *client) return PTR_ERR(ctx); ctx->dev = dev; + ctx->chip_type = LT9211; /* * Put the chip in reset, pull nRST line low, @@ -741,16 +841,32 @@ static int lt9211_probe(struct i2c_client *client) if (IS_ERR(ctx->reset_gpio)) return PTR_ERR(ctx->reset_gpio); - usleep_range(10000, 11000); /* Very long reset duration. */ + usleep_range(10000, 11000); /* Very long reset duration. */ ret = lt9211_parse_dt(ctx); + if (ret) return ret; - ctx->regmap = devm_regmap_init_i2c(client, <9211_regmap_config); + if (of_device_is_compatible(dev->of_node, "lontium,lt9211c")) { + ctx->chip_type = LT9211C; + ctx->regmap = + devm_regmap_init_i2c(client, <9211c_regmap_config); + } else { + ctx->chip_type = LT9211; + ctx->regmap = + devm_regmap_init_i2c(client, <9211_regmap_config); + } if (IS_ERR(ctx->regmap)) return PTR_ERR(ctx->regmap); + /* Initialize LT9211C-specific fields */ + ctx->wq = create_workqueue("lt9211_work"); + if (!ctx->wq) + return -ENOMEM; + + INIT_DELAYED_WORK(&ctx->lt9211_dw, lt9211_delayed_work_func); + dev_set_drvdata(dev, ctx); i2c_set_clientdata(client, ctx); @@ -768,17 +884,668 @@ static void lt9211_remove(struct i2c_client *client) { struct lt9211 *ctx = i2c_get_clientdata(client); + if (ctx->wq) + destroy_workqueue(ctx->wq); + drm_bridge_remove(&ctx->bridge); } +static int lt9211c_configure_rx(struct lt9211 *ctx) +{ + unsigned int pval; + + const struct reg_sequence lt9211c_rx_phy_seq[] = { + { REG_DSI_LANE, REG_DSI_LANE_COUNT(ctx->dsi->lanes) }, + { 0x8201, 0x11 }, + { 0x8218, 0x48 }, + { 0x8201, 0x91 }, + { 0x8202, 0x00 }, + { 0x8203, 0xee }, + { 0x8209, 0x21 }, + { 0x8204, 0x44 }, + { 0x8205, 0xc4 }, + { 0x8206, 0x44 }, + { 0x8213, 0x0c }, + + { 0xd001, 0x00 }, + { 0xd002, 0x0e }, + { 0xd005, 0x00 }, + { 0xd00a, 0x59 }, + { 0xd00b, 0x20 }, + }; + + const struct reg_sequence lt9211c_rx_phy_reset_seq[] = { + { 0x8109, 0xde }, + { 0x8109, 0xdf }, + }; + + const struct reg_sequence lt9211c_rx_clk_sel_seq[] = { + { 0x85e9, 0x88 }, + { 0x8180, 0x51 }, + { 0x8181, 0x10 }, + { 0x8632, 0x03 }, + }; + + const struct reg_sequence lt9211c_rx_input_sel_seq[] = { + { 0xd004, 0x00 }, + { 0xd021, 0x46 }, + }; + + const struct reg_sequence lt9211c_rx_dig_seq[] = { + { 0x853f, 0x08 }, + { 0x8540, 0x04 }, + { 0x8541, 0x03 }, + { 0x8542, 0x02 }, + { 0x8543, 0x01 }, + { 0x8545, 0x04 }, + { 0x8546, 0x03 }, + { 0x8547, 0x02 }, + { 0x8548, 0x01 }, + { 0x8544, 0x00 }, + { 0x8549, 0x00 }, + }; + + int ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_rx_phy_seq, + ARRAY_SIZE(lt9211c_rx_phy_seq)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_rx_phy_reset_seq, + ARRAY_SIZE(lt9211c_rx_phy_reset_seq)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_rx_clk_sel_seq, + ARRAY_SIZE(lt9211c_rx_clk_sel_seq)); + if (ret) + return ret; + + ret = regmap_read(ctx->regmap, 0x8180, &pval); + if (ret) + return ret; + + ret = regmap_write(ctx->regmap, 0x8180, (pval & 0xfc | 0x03)); + if (ret) + return ret; + + ret = regmap_read(ctx->regmap, 0x8680, &pval); + if (ret) + return ret; + + ret = regmap_write(ctx->regmap, 0x863f, (pval & 0xf8)); + if (ret) + return ret; + + ret = regmap_write(ctx->regmap, 0x863f, 0x05); + if (ret) + return ret; + + ret = regmap_read(ctx->regmap, 0x8530, &pval); + if (ret) + return ret; + + ret = regmap_write(ctx->regmap, 0x8530, (pval & 0xf8 | 0x11)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_rx_input_sel_seq, + ARRAY_SIZE(lt9211c_rx_input_sel_seq)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_rx_dig_seq, + ARRAY_SIZE(lt9211c_rx_dig_seq)); + if (ret) + return ret; + + /* Give the chip time to lock onto RX stream. */ + msleep(100); + + return 0; +} + +static int lt9211c_autodetect_rx(struct lt9211 *ctx, + const struct drm_display_mode *mode) +{ + u16 width, height; + u8 buf[5]; + u8 format; + u8 bc[3]; + u8 sot[8]; + int ret; + + /* Read the SOT from the chip. */ + ret = regmap_bulk_read(ctx->regmap, 0xd088, sot, sizeof(sot)); + if (ret) + return ret; + + dev_dbg(ctx->dev, "Sot Num = 0x%02x, 0x%02x, 0x%02x, 0x%02x\n", sot[0], + sot[2], sot[4], sot[6]); + + dev_dbg(ctx->dev, "Sot Data = 0x%02x, 0x%02x, 0x%02x, 0x%02x\n", sot[1], + sot[3], sot[5], sot[7]); + /* HS Settle Set */ + if (sot[0] > 0x10 && sot[0] < 0x50) + regmap_write(ctx->regmap, 0xd002, sot[0] - 5); + else + regmap_write(ctx->regmap, 0xd002, 0x08); + + /* Width/Height/Format Auto-detection */ + ret = regmap_bulk_read(ctx->regmap, 0xd082, buf, sizeof(buf)); + if (ret) + return ret; + + width = (buf[0] << 8) | buf[1]; + height = (buf[3] << 8) | buf[4]; + format = buf[2] & 0xf; + + if (format == 0x3) { /* YUV422 16bit */ + width /= 2; + } else if (format == 0xa) { /* RGB888 24bit */ + width /= 3; + } else { + dev_err(ctx->dev, "Unsupported DSI format 0x%01x\n", format); + return -EINVAL; + } + + if (width != mode->hdisplay) { + dev_err(ctx->dev, + "RX: Detected DSI width (%d) does not match mode hdisplay (%d)\n", + width, mode->hdisplay); + return -EINVAL; + } + + if (height != mode->vdisplay) { + dev_err(ctx->dev, + "RX: Detected DSI height (%d) does not match mode vdisplay (%d)\n", + height, mode->vdisplay); + return -EINVAL; + } + + dev_dbg(ctx->dev, "RX: %dx%d format=0x%01x\n", width, height, format); + return 0; +} + +static int lt9211c_configure_timing(struct lt9211 *ctx, + const struct drm_display_mode *mode) +{ + const struct reg_sequence lt9211c_timing[] = { + { 0xd00d, (mode->vtotal >> 8) & 0xff }, + { 0xd00e, mode->vtotal & 0xff }, + { 0xd00f, (mode->vdisplay >> 8) & 0xff }, + { 0xd010, mode->vdisplay & 0xff }, + { 0xd011, (mode->htotal >> 8) & 0xff }, + { 0xd012, mode->htotal & 0xff }, + { 0xd013, (mode->hdisplay >> 8) & 0xff }, + { 0xd014, mode->hdisplay & 0xff }, + { 0xd015, (mode->vsync_end - mode->vsync_start) & 0xff }, + { 0xd04c, ((mode->hsync_end - mode->hsync_start) >> 8) & 0xff }, + { 0xd016, (mode->hsync_end - mode->hsync_start) & 0xff }, + { 0xd017, ((mode->vsync_start - mode->vdisplay) >> 8) & 0xff }, + { 0xd018, (mode->vsync_start - mode->vdisplay) & 0xff }, + { 0xd019, ((mode->hsync_start - mode->hdisplay) >> 8) & 0xff }, + { 0xd01a, (mode->hsync_start - mode->hdisplay) & 0xff }, + }; + + return regmap_multi_reg_write(ctx->regmap, lt9211c_timing, + ARRAY_SIZE(lt9211c_timing)); +} + +static int lt9211c_configure_plls(struct lt9211 *ctx, + const struct drm_display_mode *mode) +{ + const struct reg_sequence lt9211c_dessc_pll_reset[] = { + { 0x8103, 0xfe, 2000 }, + { 0x8103, 0xff, 0 }, + }; + + const struct reg_sequence lt9211c_pcr_cali_seq[] = { + { 0xd00a, 0x5f }, + { 0xd01e, 0x51 }, + { 0xd023, 0x80 }, + { 0xd024, 0x70 }, + { 0xd025, 0x80 }, + { 0xd02a, 0x10 }, + { 0xd021, 0x4f }, + { 0xd022, 0xf0 }, + { 0xd038, 0x04 }, + { 0xd039, 0x08 }, + { 0xd03a, 0x10 }, + { 0xd03b, 0x20 }, + { 0xd03f, 0x04 }, + { 0xd040, 0x08 }, + { 0xd041, 0x10 }, + { 0xd042, 0x20 }, + { 0xd02b, 0xA0 }, + }; + + const struct reg_sequence lt9211c_pcr_reset_seq[] = { + { 0xd009, 0xdb }, + { 0xd009, 0xdf }, + { 0xd008, 0x80 }, + { 0xd008, 0x00 }, + }; + + unsigned int pval; + int ret; + u8 div; + u32 pcr_m; + u32 pcr_k; + u32 pcr_up; + u32 pcr_down; + + /* DeSSC PLL reference clock is 25 MHz XTal. */ + ret = regmap_write(ctx->regmap, 0x8226, 0x20); + if (ret) + return ret; + + /* Prediv = 0 */ + ret = regmap_write(ctx->regmap, 0x8227, 0x40); + if (ret) + return ret; + + if (mode->clock < 22000) { + ret = regmap_write(ctx->regmap, 0x822f, 0x07); + ret |= regmap_write(ctx->regmap, 0x822c, 0x01); + div = 16; + } else if (mode->clock < 44000) { + ret = regmap_write(ctx->regmap, 0x822f, 0x07); + div = 16; + } else if (mode->clock < 88000) { + ret = regmap_write(ctx->regmap, 0x822f, 0x06); + div = 8; + } else if (mode->clock < 176000) { + ret = regmap_write(ctx->regmap, 0x822f, 0x05); + div = 4; + } else { + ret = regmap_write(ctx->regmap, 0x822f, 0x04); + div = 2; + } + + if (ret) + return ret; + + pcr_m = (mode->clock * div) / 25; + pcr_k = pcr_m % 1000; + pcr_m /= 1000; + + pcr_up = pcr_m + 1; + pcr_down = pcr_m - 1; + + pcr_k <<= 14; + + ret = regmap_write(ctx->regmap, 0xd008, 0x00); + if (ret < 0) + return ret; + + /* 0xd026: pcr_m */ + ret = regmap_write(ctx->regmap, 0xd026, (0x80 | (u8)pcr_m) & 0x7f); + if (ret < 0) + return ret; + + /* 0xd027 0xd028 0xd029: pcr_k */ + ret = regmap_write(ctx->regmap, 0xd027, (pcr_k >> 16) & 0xff); + if (ret < 0) + return ret; + + ret = regmap_write(ctx->regmap, 0xd028, (pcr_k >> 8) & 0xff); + if (ret < 0) + return ret; + + ret = regmap_write(ctx->regmap, 0xd029, pcr_k & 0xff); + if (ret < 0) + return ret; + + /* 0xd02d: pcr_m overflow limit setting */ + ret = regmap_write(ctx->regmap, 0xd02d, pcr_up); + if (ret < 0) + return ret; + + /* 0xd031: pcr_m underflow limit setting */ + ret = regmap_write(ctx->regmap, 0xd031, pcr_down); + if (ret < 0) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_dessc_pll_reset, + ARRAY_SIZE(lt9211c_dessc_pll_reset)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_pcr_cali_seq, + ARRAY_SIZE(lt9211c_pcr_cali_seq)); + if (ret) + return ret; + + if (mode->clock < 44000) { + ret = regmap_write(ctx->regmap, 0xd00c, 0x60); + ret |= regmap_write(ctx->regmap, 0xd01b, 0x00); + ret |= regmap_write(ctx->regmap, 0xd01c, 0x60); + } else { + ret = regmap_write(ctx->regmap, 0xd00c, 0x40); + ret |= regmap_write(ctx->regmap, 0xd01b, 0x00); + ret |= regmap_write(ctx->regmap, 0xd01c, 0x40); + } + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_pcr_reset_seq, + ARRAY_SIZE(lt9211c_pcr_reset_seq)); + if (ret) + return ret; + + /* PCR stability test takes seconds. */ + ret = regmap_read_poll_timeout(ctx->regmap, 0xd087, pval, + ((pval & 0x18) == 0x18), 20000, 3000000); + if (ret) + dev_err(ctx->dev, "PCR unstable, ret=%i\n", ret); + + ret = regmap_write(ctx->regmap, 0x8180, 0x51); + if (ret) + return ret; + + ret = regmap_write(ctx->regmap, 0x863f, 0x00); + if (ret) + return ret; + + ret = regmap_write(ctx->regmap, 0x863f, 0x01); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(ctx->regmap, 0x8640, pval, + ((pval & 0x01) == 0x01), 50000, 250000); + if (ret) + dev_err(ctx->dev, "Video check not stable, ret=%i\n", ret); + + return ret; +} + +static int lt9211c_configure_tx(struct lt9211 *ctx, + const struct drm_display_mode *mode) +{ + const struct reg_sequence lt9211c_tx_phy_off_seq[] = { + { 0x8236, 0x00 }, + { 0x8237, 0x00 }, + { 0x8108, 0x6f }, + { 0x8103, 0xbf }, + }; + + const struct reg_sequence lt9211c_tx_phy_seq[] = { + { 0x8236, 0x03 }, + { 0x8237, 0x44 }, + { 0x8238, 0x14 }, + { 0x8239, 0x31 }, + { 0x823a, 0xc8 }, + { 0x823b, 0x00 }, + { 0x823c, 0x0f }, + { 0x8246, 0x40 }, + { 0x8247, 0x40 }, + { 0x8248, 0x40 }, + { 0x8249, 0x40 }, + { 0x824a, 0x40 }, + { 0x824b, 0x40 }, + { 0x824c, 0x40 }, + { 0x824d, 0x40 }, + { 0x824e, 0x40 }, + { 0x824f, 0x40 }, + { 0x8250, 0x40 }, + { 0x8251, 0x40 }, + }; + + const struct reg_sequence lt9211c_tx_mltx_reset[] = { + { 0x8103, 0xbf }, + { 0x8103, 0xff }, + }; + + const struct reg_sequence lt9211c_tx_dig_seq[] = { + { 0x854a, 0x01 }, + { 0x854b, 0x00 }, + { 0x854c, 0x10 }, + { 0x854d, 0x20 }, + { 0x854e, 0x50 }, + { 0x854f, 0x30 }, + { 0x8550, 0x46 }, + { 0x8551, 0x10 }, + { 0x8552, 0x20 }, + { 0x8553, 0x50 }, + { 0x8554, 0x30 }, + { 0x8555, 0x00 }, + { 0x8556, 0x20 }, + + { 0x8568, 0x00 }, + { 0x856e, 0x10 | (ctx->de ? BIT(6) : 0) }, + { 0x856f, 0x81 | (ctx->jeida ? BIT(6) : 0) | + (ctx->lvds_dual_link ? BIT(4) : 0) | + (ctx->bpp24 ? BIT(2) : 0) }, + }; + + const struct reg_sequence lt9211c_tx_ssc_seq[] = { + { 0x8234, 0x00 }, + { 0x856e, 0x10 }, + { 0x8181, 0x15 }, + { 0x871e, 0x00 }, + { 0x8717, 0x02 }, + { 0x8718, 0x04 }, + { 0x8719, 0xd4 }, + { 0x871A, 0x00 }, + { 0x871B, 0x12 }, + { 0x871C, 0x00 }, + { 0x871D, 0x24 }, + { 0x871F, 0x1c }, + { 0x8720, 0x00 }, + { 0x8721, 0x00 }, + { 0x871e, 0x02 }, + }; + + const struct reg_sequence lt9211c_tx_pll_reset_seq[] = { + { 0x810c, 0xfe, 2000 }, + { 0x810c, 0xff, 0 }, + }; + + const struct reg_sequence lt9211c_tx_sw_reset_seq[] = { + { 0x8108, 0x6f, 2000 }, + { 0x8108, 0x7f, 0 }, + }; + + unsigned int pval; + int ret; + u32 phy_clk; + u8 pixclk_div; + u8 pre_div; + u8 div_set; + u8 sericlk_div; + u8 val; + + dev_info(ctx->dev, + "dual_link=%d,even_odd_swap=%d,bpp24=%d,jeida=%d,de=%d\n", + ctx->lvds_dual_link, ctx->lvds_dual_link_even_odd_swap, + ctx->bpp24, ctx->jeida, ctx->de); + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_tx_phy_off_seq, + ARRAY_SIZE(lt9211c_tx_phy_off_seq)); + if (ret) + return ret; + + ret = regmap_read(ctx->regmap, 0x8530, &pval); + if (ret) + return ret; + + ret = regmap_write(ctx->regmap, 0x8530, (pval & 0x3f | 0x40)); + if (ret) + return ret; + + /* [7]0:txpll normal work; txpll ref clk sel pix clk */ + ret = regmap_write(ctx->regmap, 0x8230, 0x00); + if (ret) + return ret; + + if (ctx->lvds_dual_link) + phy_clk = (u32)(mode->clock * 7 / 2); + else + phy_clk = (u32)(mode->clock * 7); + + /* 0x8231: prediv sel */ + if (mode->clock < 20000) { + val = 0x28; + pre_div = 1; + } else if (mode->clock < 40000) { + val = 0x28; + pre_div = 1; + } else if (mode->clock < 80000) { + val = 0x29; + pre_div = 2; + } else if (mode->clock < 160000) { + val = 0x2a; + pre_div = 4; + } else if (mode->clock < 320000) { + val = 0x2b; + pre_div = 8; + } else { + val = 0x2f; + pre_div = 16; + } + ret = regmap_write(ctx->regmap, 0x8231, val); + if (ret < 0) + return ret; + + /* 0x8232: serickdiv sel */ + if (phy_clk < 80000) { + val = 0x32; + sericlk_div = 16; + } else if (phy_clk < 160000) { + val = 0x22; + sericlk_div = 8; + } else if (phy_clk < 320000) { + val = 0x12; + sericlk_div = 4; + } else if (phy_clk < 640000) { + val = 0x02; + sericlk_div = 2; + } else { + val = 0x42; + sericlk_div = 1; + } + ret = regmap_write(ctx->regmap, 0x8232, val); + if (ret < 0) + return ret; + + /* 0x8233: pix_mux sel & pix_div sel + * To avoid floating point operations, The pixclk_div is enlarged by 10 times + */ + if (mode->clock > 150000) { + val = 0x04; + pixclk_div = 35; + } else { + pixclk_div = + (u8)((phy_clk * sericlk_div * 10) / (mode->clock * 7)); + if (pixclk_div <= 10) + val = 0x00; + else if (pixclk_div <= 20) + val = 0x01; + else if (pixclk_div <= 40) + val = 0x02; + else + val = 0x03; + } + ret = regmap_write(ctx->regmap, 0x8233, val); + if (ret < 0) + return ret; + + ret = regmap_write(ctx->regmap, 0x8234, 0x01); + if (ret < 0) + return ret; + + /* 0x8235: div set */ + div_set = (u8)(phy_clk * sericlk_div / mode->clock / pre_div); + ret = regmap_write(ctx->regmap, 0x8235, div_set); + if (ret < 0) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_tx_ssc_seq, + ARRAY_SIZE(lt9211c_tx_ssc_seq)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_tx_pll_reset_seq, + ARRAY_SIZE(lt9211c_tx_pll_reset_seq)); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(ctx->regmap, 0x8739, pval, pval & 0x04, + 10000, 1000000); + if (ret) { + dev_err(ctx->dev, "TX PLL unstable, ret=%i\n", ret); + return ret; + } + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_tx_phy_seq, + ARRAY_SIZE(lt9211c_tx_phy_seq)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_tx_mltx_reset, + ARRAY_SIZE(lt9211c_tx_mltx_reset)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_tx_dig_seq, + ARRAY_SIZE(lt9211c_tx_dig_seq)); + if (ret) + return ret; + + ret = regmap_multi_reg_write(ctx->regmap, lt9211c_tx_sw_reset_seq, + ARRAY_SIZE(lt9211c_tx_sw_reset_seq)); + if (ret) + return ret; + + return 0; +} + +static void lt9211_delayed_work_func(struct work_struct *work) +{ + struct delayed_work *dw = to_delayed_work(work); + struct lt9211 *ctx = container_of(dw, struct lt9211, lt9211_dw); + int ret; + const struct drm_display_mode *mode = &ctx->mode; + + /* For LT9211C */ + if (ctx->chip_type != LT9211C) { + dev_err(ctx->dev, "LT9211: Delayed work called for non-LT9211C chip\n"); + return; + } + ret = lt9211c_configure_rx(ctx); + if (ret) + return; + + ret = lt9211c_autodetect_rx(ctx, mode); + if (ret) + return; + + ret = lt9211c_configure_timing(ctx, mode); + if (ret) + return; + + ret = lt9211c_configure_plls(ctx, mode); + if (ret) + return; + + ret = lt9211c_configure_tx(ctx, mode); + if (ret) + return; + +} + static const struct i2c_device_id lt9211_id[] = { { "lontium,lt9211" }, + { "lontium,lt9211c" }, {}, }; MODULE_DEVICE_TABLE(i2c, lt9211_id); static const struct of_device_id lt9211_match_table[] = { { .compatible = "lontium,lt9211" }, + { .compatible = "lontium,lt9211c" }, {}, }; MODULE_DEVICE_TABLE(of, lt9211_match_table); diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c index 38fb8776c0f44..875893f617ade 100644 --- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c +++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c @@ -89,7 +89,9 @@ static const struct regmap_config lt9611uxc_regmap_config = { struct lt9611uxc_mode { u16 hdisplay; + u16 htotal; u16 vdisplay; + u16 vtotal; u8 vrefresh; }; @@ -98,22 +100,23 @@ struct lt9611uxc_mode { * Enumerate them here to check whether the mode is supported. */ static struct lt9611uxc_mode lt9611uxc_modes[] = { - { 1920, 1080, 60 }, - { 1920, 1080, 30 }, - { 1920, 1080, 25 }, - { 1366, 768, 60 }, - { 1360, 768, 60 }, - { 1280, 1024, 60 }, - { 1280, 800, 60 }, - { 1280, 720, 60 }, - { 1280, 720, 50 }, - { 1280, 720, 30 }, - { 1152, 864, 60 }, - { 1024, 768, 60 }, - { 800, 600, 60 }, - { 720, 576, 50 }, - { 720, 480, 60 }, - { 640, 480, 60 }, + { 3840, 4400, 2160, 2250, 30 }, + { 1920, 2200, 1080, 1125, 60 }, + { 1920, 2200, 1080, 1125, 30 }, + { 1920, 2640, 1080, 1125, 25 }, + { 1366, 1792, 768, 798, 60 }, + { 1360, 1792, 768, 795, 60 }, + { 1280, 1688, 1024, 1066, 60 }, + { 1280, 1680, 800, 831, 60 }, + { 1280, 1650, 720, 750, 60 }, + { 1280, 1980, 720, 750, 50 }, + { 1280, 3300, 720, 750, 30 }, + { 1152, 1600, 864, 900, 60 }, + { 1024, 1344, 768, 806, 60 }, + { 800, 1056, 600, 628, 60 }, + { 720, 864, 576, 625, 50 }, + { 720, 858, 480, 525, 60 }, + { 640, 800, 480, 525, 60 }, }; static struct lt9611uxc *bridge_to_lt9611uxc(struct drm_bridge *bridge) @@ -171,6 +174,9 @@ static void lt9611uxc_hpd_work(struct work_struct *work) connected = lt9611uxc->hdmi_connected; mutex_unlock(<9611uxc->ocm_lock); + if (!connected) + lt9611uxc->edid_read = false; + drm_bridge_hpd_notify(<9611uxc->bridge, connected ? connector_status_connected : @@ -237,7 +243,9 @@ static struct lt9611uxc_mode *lt9611uxc_find_mode(const struct drm_display_mode for (i = 0; i < ARRAY_SIZE(lt9611uxc_modes); i++) { if (lt9611uxc_modes[i].hdisplay == mode->hdisplay && + lt9611uxc_modes[i].htotal == mode->htotal && lt9611uxc_modes[i].vdisplay == mode->vdisplay && + lt9611uxc_modes[i].vtotal == mode->vtotal && lt9611uxc_modes[i].vrefresh == drm_mode_vrefresh(mode)) { return <9611uxc_modes[i]; } @@ -382,7 +390,7 @@ lt9611uxc_bridge_detect(struct drm_bridge *bridge, struct drm_connector *connect static int lt9611uxc_wait_for_edid(struct lt9611uxc *lt9611uxc) { return wait_event_interruptible_timeout(lt9611uxc->wq, lt9611uxc->edid_read, - msecs_to_jiffies(500)); + msecs_to_jiffies(1000)); } static int lt9611uxc_get_edid_block(void *data, u8 *buf, unsigned int block, size_t len) diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c index eabc4c32f6ab4..ad8c6aa49d48a 100644 --- a/drivers/gpu/drm/bridge/samsung-dsim.c +++ b/drivers/gpu/drm/bridge/samsung-dsim.c @@ -1881,6 +1881,14 @@ static int samsung_dsim_register_te_irq(struct samsung_dsim *dsi, struct device return 0; } +static void samsung_dsim_unregister_te_irq(struct samsung_dsim *dsi) +{ + if (dsi->te_gpio) { + free_irq(gpiod_to_irq(dsi->te_gpio), dsi); + gpiod_put(dsi->te_gpio); + } +} + static int samsung_dsim_host_attach(struct mipi_dsi_host *host, struct mipi_dsi_device *device) { @@ -1955,13 +1963,13 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host, if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO)) { ret = samsung_dsim_register_te_irq(dsi, &device->dev); if (ret) - return ret; + goto err_remove_bridge; } if (pdata->host_ops && pdata->host_ops->attach) { ret = pdata->host_ops->attach(dsi, device); if (ret) - return ret; + goto err_unregister_te_irq; } dsi->lanes = device->lanes; @@ -1969,14 +1977,13 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host, dsi->mode_flags = device->mode_flags; return 0; -} -static void samsung_dsim_unregister_te_irq(struct samsung_dsim *dsi) -{ - if (dsi->te_gpio) { - free_irq(gpiod_to_irq(dsi->te_gpio), dsi); - gpiod_put(dsi->te_gpio); - } +err_unregister_te_irq: + if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO)) + samsung_dsim_unregister_te_irq(dsi); +err_remove_bridge: + drm_bridge_remove(&dsi->bridge); + return ret; } static int samsung_dsim_host_detach(struct mipi_dsi_host *host, diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c index 9bbfe8da3de02..e82960163018a 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c @@ -2060,33 +2060,41 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, } ret = drm_bridge_attach(encoder, bridge, NULL, DRM_BRIDGE_ATTACH_NO_CONNECTOR); - if (ret) + if (ret) { dev_err_probe(dev, ret, "Failed to attach bridge\n"); + goto unregister_aux; + } dw_dp_init_hw(dp); ret = phy_init(dp->phy); if (ret) { dev_err_probe(dev, ret, "phy init failed\n"); - return ERR_PTR(ret); + goto unregister_aux; } ret = devm_add_action_or_reset(dev, dw_dp_phy_exit, dp); if (ret) - return ERR_PTR(ret); + goto unregister_aux; dp->irq = platform_get_irq(pdev, 0); - if (dp->irq < 0) - return ERR_PTR(ret); + if (dp->irq < 0) { + ret = dp->irq; + goto unregister_aux; + } ret = devm_request_threaded_irq(dev, dp->irq, NULL, dw_dp_irq, IRQF_ONESHOT, dev_name(dev), dp); if (ret) { dev_err_probe(dev, ret, "failed to request irq\n"); - return ERR_PTR(ret); + goto unregister_aux; } return dp; + +unregister_aux: + drm_dp_aux_unregister(&dp->aux); + return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(dw_dp_bind); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c index 39332c57f2c54..d302455875167 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c @@ -143,6 +143,7 @@ struct dw_hdmi_qp { } phy; struct regmap *regm; + int main_irq; unsigned long tmds_char_rate; }; @@ -817,7 +818,7 @@ static int dw_hdmi_qp_config_audio_infoframe(struct dw_hdmi_qp *hdmi, regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS0, &header_bytes, 1); regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS1, &buffer[3], 1); - regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS2, &buffer[4], 1); + regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS2, &buffer[7], 1); /* Enable ACR, AUDI, AMD */ dw_hdmi_qp_mod(hdmi, @@ -1068,6 +1069,7 @@ struct dw_hdmi_qp *dw_hdmi_qp_bind(struct platform_device *pdev, dw_hdmi_qp_init_hw(hdmi); + hdmi->main_irq = plat_data->main_irq; ret = devm_request_threaded_irq(dev, plat_data->main_irq, dw_hdmi_qp_main_hardirq, NULL, IRQF_SHARED, dev_name(dev), hdmi); @@ -1106,9 +1108,16 @@ struct dw_hdmi_qp *dw_hdmi_qp_bind(struct platform_device *pdev, } EXPORT_SYMBOL_GPL(dw_hdmi_qp_bind); +void dw_hdmi_qp_suspend(struct device *dev, struct dw_hdmi_qp *hdmi) +{ + disable_irq(hdmi->main_irq); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_suspend); + void dw_hdmi_qp_resume(struct device *dev, struct dw_hdmi_qp *hdmi) { dw_hdmi_qp_init_hw(hdmi); + enable_irq(hdmi->main_irq); } EXPORT_SYMBOL_GPL(dw_hdmi_qp_resume); diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c index 033c44326552a..43344c2e15b77 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c @@ -351,9 +351,9 @@ static u8 sn65dsi83_get_dsi_range(struct sn65dsi83 *ctx, * DSI_CLK = mode clock * bpp / dsi_data_lanes / 2 * the 2 is there because the bus is DDR. */ - return DIV_ROUND_UP(clamp((unsigned int)mode->clock * - mipi_dsi_pixel_format_to_bpp(ctx->dsi->format) / - ctx->dsi->lanes / 2, 40000U, 500000U), 5000U); + return clamp((unsigned int)mode->clock * + mipi_dsi_pixel_format_to_bpp(ctx->dsi->format) / + ctx->dsi->lanes / 2, 40000U, 500000U) / 5000U; } static u8 sn65dsi83_get_dsi_div(struct sn65dsi83 *ctx) @@ -429,7 +429,14 @@ static void sn65dsi83_handle_errors(struct sn65dsi83 *ctx) */ ret = regmap_read(ctx->regmap, REG_IRQ_STAT, &irq_stat); - if (ret || irq_stat) { + + /* + * Some hardware (Toradex Verdin AM62) is known to report the + * PLL_UNLOCK error interrupt while working without visible + * problems. In lack of a reliable way to discriminate such cases + * from user-visible PLL_UNLOCK cases, ignore that bit entirely. + */ + if (ret || irq_stat & ~REG_IRQ_STAT_CHA_PLL_UNLOCK) { /* * IRQ acknowledged is not always possible (the bridge can be in * a state where it doesn't answer anymore). To prevent an @@ -467,6 +474,7 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge, struct drm_atomic_state *state) { struct sn65dsi83 *ctx = bridge_to_sn65dsi83(bridge); + const unsigned int dual_factor = ctx->lvds_dual_link ? 2 : 1; const struct drm_bridge_state *bridge_state; const struct drm_crtc_state *crtc_state; const struct drm_display_mode *mode; @@ -599,18 +607,18 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge, /* 32 + 1 pixel clock to ensure proper operation */ le16val = cpu_to_le16(32 + 1); regmap_bulk_write(ctx->regmap, REG_VID_CHA_SYNC_DELAY_LOW, &le16val, 2); - le16val = cpu_to_le16(mode->hsync_end - mode->hsync_start); + le16val = cpu_to_le16((mode->hsync_end - mode->hsync_start) / dual_factor); regmap_bulk_write(ctx->regmap, REG_VID_CHA_HSYNC_PULSE_WIDTH_LOW, &le16val, 2); le16val = cpu_to_le16(mode->vsync_end - mode->vsync_start); regmap_bulk_write(ctx->regmap, REG_VID_CHA_VSYNC_PULSE_WIDTH_LOW, &le16val, 2); regmap_write(ctx->regmap, REG_VID_CHA_HORIZONTAL_BACK_PORCH, - mode->htotal - mode->hsync_end); + (mode->htotal - mode->hsync_end) / dual_factor); regmap_write(ctx->regmap, REG_VID_CHA_VERTICAL_BACK_PORCH, mode->vtotal - mode->vsync_end); regmap_write(ctx->regmap, REG_VID_CHA_HORIZONTAL_FRONT_PORCH, - mode->hsync_start - mode->hdisplay); + (mode->hsync_start - mode->hdisplay) / dual_factor); regmap_write(ctx->regmap, REG_VID_CHA_VERTICAL_FRONT_PORCH, mode->vsync_start - mode->vdisplay); regmap_write(ctx->regmap, REG_VID_CHA_TEST_PATTERN, 0x00); @@ -654,7 +662,7 @@ static void sn65dsi83_atomic_enable(struct drm_bridge *bridge, if (ctx->irq) { /* Enable irq to detect errors */ regmap_write(ctx->regmap, REG_IRQ_GLOBAL, REG_IRQ_GLOBAL_IRQ_EN); - regmap_write(ctx->regmap, REG_IRQ_EN, 0xff); + regmap_write(ctx->regmap, REG_IRQ_EN, 0xff & ~REG_IRQ_EN_CHA_PLL_UNLOCK_EN); } else { /* Use the polling task */ sn65dsi83_monitor_start(ctx); diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index ae0d08e5e9600..98d64ad791d04 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -106,10 +106,21 @@ #define SN_PWM_EN_INV_REG 0xA5 #define SN_PWM_INV_MASK BIT(0) #define SN_PWM_EN_MASK BIT(1) + +#define SN_IRQ_EN_REG 0xE0 +#define IRQ_EN BIT(0) + +#define SN_IRQ_EVENTS_EN_REG 0xE6 +#define HPD_INSERTION_EN BIT(1) +#define HPD_REMOVAL_EN BIT(2) + #define SN_AUX_CMD_STATUS_REG 0xF4 #define AUX_IRQ_STATUS_AUX_RPLY_TOUT BIT(3) #define AUX_IRQ_STATUS_AUX_SHORT BIT(5) #define AUX_IRQ_STATUS_NAT_I2C_FAIL BIT(6) +#define SN_IRQ_STATUS_REG 0xF5 +#define HPD_REMOVAL_STATUS BIT(2) +#define HPD_INSERTION_STATUS BIT(1) #define MIN_DSI_CLK_FREQ_MHZ 40 @@ -152,7 +163,9 @@ * @ln_assign: Value to program to the LN_ASSIGN register. * @ln_polrs: Value for the 4-bit LN_POLRS field of SN_ENH_FRAME_REG. * @comms_enabled: If true then communication over the aux channel is enabled. + * @hpd_enabled: If true then HPD events are enabled. * @comms_mutex: Protects modification of comms_enabled. + * @hpd_mutex: Protects modification of hpd_enabled. * * @gchip: If we expose our GPIOs, this is used. * @gchip_output: A cache of whether we've set GPIOs to output. This @@ -190,7 +203,9 @@ struct ti_sn65dsi86 { u8 ln_assign; u8 ln_polrs; bool comms_enabled; + bool hpd_enabled; struct mutex comms_mutex; + struct mutex hpd_mutex; #if defined(CONFIG_OF_GPIO) struct gpio_chip gchip; @@ -221,6 +236,23 @@ static const struct regmap_config ti_sn65dsi86_regmap_config = { .max_register = 0xFF, }; +static int ti_sn65dsi86_read_u8(struct ti_sn65dsi86 *pdata, unsigned int reg, + u8 *val) +{ + int ret; + unsigned int reg_val; + + ret = regmap_read(pdata->regmap, reg, ®_val); + if (ret) { + dev_err(pdata->dev, "fail to read raw reg %#x: %d\n", + reg, ret); + return ret; + } + *val = (u8)reg_val; + + return 0; +} + static int __maybe_unused ti_sn65dsi86_read_u16(struct ti_sn65dsi86 *pdata, unsigned int reg, u16 *val) { @@ -379,6 +411,7 @@ static void ti_sn65dsi86_disable_comms(struct ti_sn65dsi86 *pdata) static int __maybe_unused ti_sn65dsi86_resume(struct device *dev) { struct ti_sn65dsi86 *pdata = dev_get_drvdata(dev); + const struct i2c_client *client = to_i2c_client(pdata->dev); int ret; ret = regulator_bulk_enable(SN_REGULATOR_SUPPLY_NUM, pdata->supplies); @@ -413,6 +446,13 @@ static int __maybe_unused ti_sn65dsi86_resume(struct device *dev) if (pdata->refclk) ti_sn65dsi86_enable_comms(pdata, NULL); + if (client->irq) { + ret = regmap_update_bits(pdata->regmap, SN_IRQ_EN_REG, IRQ_EN, + IRQ_EN); + if (ret) + dev_err(pdata->dev, "Failed to enable IRQ events: %d\n", ret); + } + return ret; } @@ -1211,6 +1251,8 @@ static void ti_sn65dsi86_debugfs_init(struct drm_bridge *bridge, struct dentry * static void ti_sn_bridge_hpd_enable(struct drm_bridge *bridge) { struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge); + const struct i2c_client *client = to_i2c_client(pdata->dev); + int ret; /* * Device needs to be powered on before reading the HPD state @@ -1219,11 +1261,35 @@ static void ti_sn_bridge_hpd_enable(struct drm_bridge *bridge) */ pm_runtime_get_sync(pdata->dev); + + mutex_lock(&pdata->hpd_mutex); + pdata->hpd_enabled = true; + mutex_unlock(&pdata->hpd_mutex); + + if (client->irq) { + ret = regmap_set_bits(pdata->regmap, SN_IRQ_EVENTS_EN_REG, + HPD_REMOVAL_EN | HPD_INSERTION_EN); + if (ret) + dev_err(pdata->dev, "Failed to enable HPD events: %d\n", ret); + } } static void ti_sn_bridge_hpd_disable(struct drm_bridge *bridge) { struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge); + const struct i2c_client *client = to_i2c_client(pdata->dev); + int ret; + + if (client->irq) { + ret = regmap_clear_bits(pdata->regmap, SN_IRQ_EVENTS_EN_REG, + HPD_REMOVAL_EN | HPD_INSERTION_EN); + if (ret) + dev_err(pdata->dev, "Failed to disable HPD events: %d\n", ret); + } + + mutex_lock(&pdata->hpd_mutex); + pdata->hpd_enabled = false; + mutex_unlock(&pdata->hpd_mutex); pm_runtime_put_autosuspend(pdata->dev); } @@ -1309,11 +1375,47 @@ static int ti_sn_bridge_parse_dsi_host(struct ti_sn65dsi86 *pdata) return 0; } +static irqreturn_t ti_sn_bridge_interrupt(int irq, void *private) +{ + struct ti_sn65dsi86 *pdata = private; + struct drm_device *dev = pdata->bridge.dev; + u8 status; + int ret; + bool hpd_event; + + ret = ti_sn65dsi86_read_u8(pdata, SN_IRQ_STATUS_REG, &status); + if (ret) { + dev_err(pdata->dev, "Failed to read IRQ status: %d\n", ret); + return IRQ_NONE; + } + + hpd_event = status & (HPD_REMOVAL_STATUS | HPD_INSERTION_STATUS); + + dev_dbg(pdata->dev, "(SN_IRQ_STATUS_REG = %#x)\n", status); + if (!status) + return IRQ_NONE; + + ret = regmap_write(pdata->regmap, SN_IRQ_STATUS_REG, status); + if (ret) { + dev_err(pdata->dev, "Failed to clear IRQ status: %d\n", ret); + return IRQ_NONE; + } + + /* Only send the HPD event if we are bound with a device. */ + mutex_lock(&pdata->hpd_mutex); + if (pdata->hpd_enabled && hpd_event) + drm_kms_helper_hotplug_event(dev); + mutex_unlock(&pdata->hpd_mutex); + + return IRQ_HANDLED; +} + static int ti_sn_bridge_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { struct ti_sn65dsi86 *pdata = dev_get_drvdata(adev->dev.parent); struct device_node *np = pdata->dev->of_node; + const struct i2c_client *client = to_i2c_client(pdata->dev); int ret; pdata->next_bridge = devm_drm_of_get_bridge(&adev->dev, np, 1, 0); @@ -1332,8 +1434,9 @@ static int ti_sn_bridge_probe(struct auxiliary_device *adev, ? DRM_MODE_CONNECTOR_DisplayPort : DRM_MODE_CONNECTOR_eDP; if (pdata->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) { - pdata->bridge.ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT | - DRM_BRIDGE_OP_HPD; + pdata->bridge.ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT; + if (client->irq) + pdata->bridge.ops |= DRM_BRIDGE_OP_HPD; /* * If comms were already enabled they would have been enabled * with the wrong value of HPD_DISABLE. Update it now. Comms @@ -1931,6 +2034,7 @@ static int ti_sn65dsi86_probe(struct i2c_client *client) dev_set_drvdata(dev, pdata); pdata->dev = dev; + mutex_init(&pdata->hpd_mutex); mutex_init(&pdata->comms_mutex); pdata->regmap = devm_regmap_init_i2c(client, @@ -1971,6 +2075,16 @@ static int ti_sn65dsi86_probe(struct i2c_client *client) if (strncmp(id_buf, "68ISD ", ARRAY_SIZE(id_buf))) return dev_err_probe(dev, -EOPNOTSUPP, "unsupported device id\n"); + if (client->irq) { + ret = devm_request_threaded_irq(pdata->dev, client->irq, NULL, + ti_sn_bridge_interrupt, + IRQF_ONESHOT, + dev_name(pdata->dev), pdata); + + if (ret) + return dev_err_probe(dev, ret, "failed to request interrupt\n"); + } + /* * Break ourselves up into a collection of aux devices. The only real * motiviation here is to solve the chicken-and-egg problem of probe diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c index 64e5c176d5cce..be749dcad3b58 100644 --- a/drivers/gpu/drm/display/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c @@ -4572,7 +4572,8 @@ int drm_dp_atomic_release_time_slots(struct drm_atomic_state *state, if (!payload->delete) { payload->pbn = 0; payload->delete = true; - topology_state->payload_mask &= ~BIT(payload->vcpi - 1); + if (payload->vcpi > 0) + topology_state->payload_mask &= ~BIT(payload->vcpi - 1); } return 0; diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index d5ebe6ea0acbc..bbec1c184f652 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -1162,8 +1162,18 @@ crtc_needs_disable(struct drm_crtc_state *old_state, new_state->self_refresh_active; } -static void -encoder_bridge_disable(struct drm_device *dev, struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_encoder_bridge_disable - disable bridges and encoder + * @dev: DRM device + * @state: the driver state object + * + * Loops over all connectors in the current state and if the CRTC needs + * it, disables the bridge chain all the way, then disables the encoder + * afterwards. + */ +void +drm_atomic_helper_commit_encoder_bridge_disable(struct drm_device *dev, + struct drm_atomic_state *state) { struct drm_connector *connector; struct drm_connector_state *old_conn_state, *new_conn_state; @@ -1229,9 +1239,18 @@ encoder_bridge_disable(struct drm_device *dev, struct drm_atomic_state *state) } } } +EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_disable); -static void -crtc_disable(struct drm_device *dev, struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_crtc_disable - disable CRTSs + * @dev: DRM device + * @state: the driver state object + * + * Loops over all CRTCs in the current state and if the CRTC needs + * it, disables it. + */ +void +drm_atomic_helper_commit_crtc_disable(struct drm_device *dev, struct drm_atomic_state *state) { struct drm_crtc *crtc; struct drm_crtc_state *old_crtc_state, *new_crtc_state; @@ -1282,9 +1301,18 @@ crtc_disable(struct drm_device *dev, struct drm_atomic_state *state) drm_crtc_vblank_put(crtc); } } +EXPORT_SYMBOL(drm_atomic_helper_commit_crtc_disable); -static void -encoder_bridge_post_disable(struct drm_device *dev, struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_encoder_bridge_post_disable - post-disable encoder bridges + * @dev: DRM device + * @state: the driver state object + * + * Loops over all connectors in the current state and if the CRTC needs + * it, post-disables all encoder bridges. + */ +void +drm_atomic_helper_commit_encoder_bridge_post_disable(struct drm_device *dev, struct drm_atomic_state *state) { struct drm_connector *connector; struct drm_connector_state *old_conn_state, *new_conn_state; @@ -1335,15 +1363,16 @@ encoder_bridge_post_disable(struct drm_device *dev, struct drm_atomic_state *sta drm_bridge_put(bridge); } } +EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_post_disable); static void disable_outputs(struct drm_device *dev, struct drm_atomic_state *state) { - encoder_bridge_disable(dev, state); + drm_atomic_helper_commit_encoder_bridge_disable(dev, state); - crtc_disable(dev, state); + drm_atomic_helper_commit_encoder_bridge_post_disable(dev, state); - encoder_bridge_post_disable(dev, state); + drm_atomic_helper_commit_crtc_disable(dev, state); } /** @@ -1446,8 +1475,17 @@ void drm_atomic_helper_calc_timestamping_constants(struct drm_atomic_state *stat } EXPORT_SYMBOL(drm_atomic_helper_calc_timestamping_constants); -static void -crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_crtc_set_mode - set the new mode + * @dev: DRM device + * @state: the driver state object + * + * Loops over all connectors in the current state and if the mode has + * changed, change the mode of the CRTC, then call down the bridge + * chain and change the mode in all bridges as well. + */ +void +drm_atomic_helper_commit_crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *state) { struct drm_crtc *crtc; struct drm_crtc_state *new_crtc_state; @@ -1508,6 +1546,7 @@ crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *state) drm_bridge_put(bridge); } } +EXPORT_SYMBOL(drm_atomic_helper_commit_crtc_set_mode); /** * drm_atomic_helper_commit_modeset_disables - modeset commit to disable outputs @@ -1531,12 +1570,21 @@ void drm_atomic_helper_commit_modeset_disables(struct drm_device *dev, drm_atomic_helper_update_legacy_modeset_state(dev, state); drm_atomic_helper_calc_timestamping_constants(state); - crtc_set_mode(dev, state); + drm_atomic_helper_commit_crtc_set_mode(dev, state); } EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_disables); -static void drm_atomic_helper_commit_writebacks(struct drm_device *dev, - struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_writebacks - issue writebacks + * @dev: DRM device + * @state: atomic state object being committed + * + * This loops over the connectors, checks if the new state requires + * a writeback job to be issued and in that case issues an atomic + * commit on each connector. + */ +void drm_atomic_helper_commit_writebacks(struct drm_device *dev, + struct drm_atomic_state *state) { struct drm_connector *connector; struct drm_connector_state *new_conn_state; @@ -1555,9 +1603,18 @@ static void drm_atomic_helper_commit_writebacks(struct drm_device *dev, } } } +EXPORT_SYMBOL(drm_atomic_helper_commit_writebacks); -static void -encoder_bridge_pre_enable(struct drm_device *dev, struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_encoder_bridge_pre_enable - pre-enable bridges + * @dev: DRM device + * @state: atomic state object being committed + * + * This loops over the connectors and if the CRTC needs it, pre-enables + * the entire bridge chain. + */ +void +drm_atomic_helper_commit_encoder_bridge_pre_enable(struct drm_device *dev, struct drm_atomic_state *state) { struct drm_connector *connector; struct drm_connector_state *new_conn_state; @@ -1588,9 +1645,18 @@ encoder_bridge_pre_enable(struct drm_device *dev, struct drm_atomic_state *state drm_bridge_put(bridge); } } +EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_pre_enable); -static void -crtc_enable(struct drm_device *dev, struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_crtc_enable - enables the CRTCs + * @dev: DRM device + * @state: atomic state object being committed + * + * This loops over CRTCs in the new state, and of the CRTC needs + * it, enables it. + */ +void +drm_atomic_helper_commit_crtc_enable(struct drm_device *dev, struct drm_atomic_state *state) { struct drm_crtc *crtc; struct drm_crtc_state *old_crtc_state; @@ -1619,9 +1685,18 @@ crtc_enable(struct drm_device *dev, struct drm_atomic_state *state) } } } +EXPORT_SYMBOL(drm_atomic_helper_commit_crtc_enable); -static void -encoder_bridge_enable(struct drm_device *dev, struct drm_atomic_state *state) +/** + * drm_atomic_helper_commit_encoder_bridge_enable - enables the bridges + * @dev: DRM device + * @state: atomic state object being committed + * + * This loops over all connectors in the new state, and of the CRTC needs + * it, enables the entire bridge chain. + */ +void +drm_atomic_helper_commit_encoder_bridge_enable(struct drm_device *dev, struct drm_atomic_state *state) { struct drm_connector *connector; struct drm_connector_state *new_conn_state; @@ -1664,6 +1739,7 @@ encoder_bridge_enable(struct drm_device *dev, struct drm_atomic_state *state) drm_bridge_put(bridge); } } +EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_enable); /** * drm_atomic_helper_commit_modeset_enables - modeset commit to enable outputs @@ -1682,11 +1758,11 @@ encoder_bridge_enable(struct drm_device *dev, struct drm_atomic_state *state) void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev, struct drm_atomic_state *state) { - encoder_bridge_pre_enable(dev, state); + drm_atomic_helper_commit_crtc_enable(dev, state); - crtc_enable(dev, state); + drm_atomic_helper_commit_encoder_bridge_pre_enable(dev, state); - encoder_bridge_enable(dev, state); + drm_atomic_helper_commit_encoder_bridge_enable(dev, state); drm_atomic_helper_commit_writebacks(dev, state); } diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index a94061f373de5..640d93070bb7c 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -12,8 +12,17 @@ #include +enum drm_buddy_free_tree { + DRM_BUDDY_CLEAR_TREE = 0, + DRM_BUDDY_DIRTY_TREE, + DRM_BUDDY_MAX_FREE_TREES, +}; + static struct kmem_cache *slab_blocks; +#define for_each_free_tree(tree) \ + for ((tree) = 0; (tree) < DRM_BUDDY_MAX_FREE_TREES; (tree)++) + static struct drm_buddy_block *drm_block_alloc(struct drm_buddy *mm, struct drm_buddy_block *parent, unsigned int order, @@ -31,6 +40,8 @@ static struct drm_buddy_block *drm_block_alloc(struct drm_buddy *mm, block->header |= order; block->parent = parent; + RB_CLEAR_NODE(&block->rb); + BUG_ON(block->header & DRM_BUDDY_HEADER_UNUSED); return block; } @@ -41,23 +52,64 @@ static void drm_block_free(struct drm_buddy *mm, kmem_cache_free(slab_blocks, block); } -static void list_insert_sorted(struct drm_buddy *mm, - struct drm_buddy_block *block) +static enum drm_buddy_free_tree +get_block_tree(struct drm_buddy_block *block) { - struct drm_buddy_block *node; - struct list_head *head; + return drm_buddy_block_is_clear(block) ? + DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; +} - head = &mm->free_list[drm_buddy_block_order(block)]; - if (list_empty(head)) { - list_add(&block->link, head); - return; - } +static struct drm_buddy_block * +rbtree_get_free_block(const struct rb_node *node) +{ + return node ? rb_entry(node, struct drm_buddy_block, rb) : NULL; +} - list_for_each_entry(node, head, link) - if (drm_buddy_block_offset(block) < drm_buddy_block_offset(node)) - break; +static struct drm_buddy_block * +rbtree_last_free_block(struct rb_root *root) +{ + return rbtree_get_free_block(rb_last(root)); +} - __list_add(&block->link, node->link.prev, &node->link); +static bool rbtree_is_empty(struct rb_root *root) +{ + return RB_EMPTY_ROOT(root); +} + +static bool drm_buddy_block_offset_less(const struct drm_buddy_block *block, + const struct drm_buddy_block *node) +{ + return drm_buddy_block_offset(block) < drm_buddy_block_offset(node); +} + +static bool rbtree_block_offset_less(struct rb_node *block, + const struct rb_node *node) +{ + return drm_buddy_block_offset_less(rbtree_get_free_block(block), + rbtree_get_free_block(node)); +} + +static void rbtree_insert(struct drm_buddy *mm, + struct drm_buddy_block *block, + enum drm_buddy_free_tree tree) +{ + rb_add(&block->rb, + &mm->free_trees[tree][drm_buddy_block_order(block)], + rbtree_block_offset_less); +} + +static void rbtree_remove(struct drm_buddy *mm, + struct drm_buddy_block *block) +{ + unsigned int order = drm_buddy_block_order(block); + enum drm_buddy_free_tree tree; + struct rb_root *root; + + tree = get_block_tree(block); + root = &mm->free_trees[tree][order]; + + rb_erase(&block->rb, root); + RB_CLEAR_NODE(&block->rb); } static void clear_reset(struct drm_buddy_block *block) @@ -70,29 +122,34 @@ static void mark_cleared(struct drm_buddy_block *block) block->header |= DRM_BUDDY_HEADER_CLEAR; } -static void mark_allocated(struct drm_buddy_block *block) +static void mark_allocated(struct drm_buddy *mm, + struct drm_buddy_block *block) { block->header &= ~DRM_BUDDY_HEADER_STATE; block->header |= DRM_BUDDY_ALLOCATED; - list_del(&block->link); + rbtree_remove(mm, block); } static void mark_free(struct drm_buddy *mm, struct drm_buddy_block *block) { + enum drm_buddy_free_tree tree; + block->header &= ~DRM_BUDDY_HEADER_STATE; block->header |= DRM_BUDDY_FREE; - list_insert_sorted(mm, block); + tree = get_block_tree(block); + rbtree_insert(mm, block, tree); } -static void mark_split(struct drm_buddy_block *block) +static void mark_split(struct drm_buddy *mm, + struct drm_buddy_block *block) { block->header &= ~DRM_BUDDY_HEADER_STATE; block->header |= DRM_BUDDY_SPLIT; - list_del(&block->link); + rbtree_remove(mm, block); } static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) @@ -148,7 +205,7 @@ static unsigned int __drm_buddy_free(struct drm_buddy *mm, mark_cleared(parent); } - list_del(&buddy->link); + rbtree_remove(mm, buddy); if (force_merge && drm_buddy_block_is_clear(buddy)) mm->clear_avail -= drm_buddy_block_size(mm, buddy); @@ -169,7 +226,7 @@ static int __force_merge(struct drm_buddy *mm, u64 end, unsigned int min_order) { - unsigned int order; + unsigned int tree, order; int i; if (!min_order) @@ -178,44 +235,48 @@ static int __force_merge(struct drm_buddy *mm, if (min_order > mm->max_order) return -EINVAL; - for (i = min_order - 1; i >= 0; i--) { - struct drm_buddy_block *block, *prev; + for_each_free_tree(tree) { + for (i = min_order - 1; i >= 0; i--) { + struct rb_node *iter = rb_last(&mm->free_trees[tree][i]); - list_for_each_entry_safe_reverse(block, prev, &mm->free_list[i], link) { - struct drm_buddy_block *buddy; - u64 block_start, block_end; + while (iter) { + struct drm_buddy_block *block, *buddy; + u64 block_start, block_end; - if (!block->parent) - continue; + block = rbtree_get_free_block(iter); + iter = rb_prev(iter); - block_start = drm_buddy_block_offset(block); - block_end = block_start + drm_buddy_block_size(mm, block) - 1; + if (!block || !block->parent) + continue; - if (!contains(start, end, block_start, block_end)) - continue; + block_start = drm_buddy_block_offset(block); + block_end = block_start + drm_buddy_block_size(mm, block) - 1; - buddy = __get_buddy(block); - if (!drm_buddy_block_is_free(buddy)) - continue; + if (!contains(start, end, block_start, block_end)) + continue; - WARN_ON(drm_buddy_block_is_clear(block) == - drm_buddy_block_is_clear(buddy)); + buddy = __get_buddy(block); + if (!drm_buddy_block_is_free(buddy)) + continue; - /* - * If the prev block is same as buddy, don't access the - * block in the next iteration as we would free the - * buddy block as part of the free function. - */ - if (prev == buddy) - prev = list_prev_entry(prev, link); + WARN_ON(drm_buddy_block_is_clear(block) == + drm_buddy_block_is_clear(buddy)); - list_del(&block->link); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail -= drm_buddy_block_size(mm, block); + /* + * Advance to the next node when the current node is the buddy, + * as freeing the block will also remove its buddy from the tree. + */ + if (iter == &buddy->rb) + iter = rb_prev(iter); - order = __drm_buddy_free(mm, block, true); - if (order >= min_order) - return 0; + rbtree_remove(mm, block); + if (drm_buddy_block_is_clear(block)) + mm->clear_avail -= drm_buddy_block_size(mm, block); + + order = __drm_buddy_free(mm, block, true); + if (order >= min_order) + return 0; + } } } @@ -236,8 +297,8 @@ static int __force_merge(struct drm_buddy *mm, */ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) { - unsigned int i; - u64 offset; + unsigned int i, j, root_count = 0; + u64 offset = 0; if (size < chunk_size) return -EINVAL; @@ -258,14 +319,22 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) BUG_ON(mm->max_order > DRM_BUDDY_MAX_ORDER); - mm->free_list = kmalloc_array(mm->max_order + 1, - sizeof(struct list_head), - GFP_KERNEL); - if (!mm->free_list) + mm->free_trees = kmalloc_array(DRM_BUDDY_MAX_FREE_TREES, + sizeof(*mm->free_trees), + GFP_KERNEL); + if (!mm->free_trees) return -ENOMEM; - for (i = 0; i <= mm->max_order; ++i) - INIT_LIST_HEAD(&mm->free_list[i]); + for_each_free_tree(i) { + mm->free_trees[i] = kmalloc_array(mm->max_order + 1, + sizeof(struct rb_root), + GFP_KERNEL); + if (!mm->free_trees[i]) + goto out_free_tree; + + for (j = 0; j <= mm->max_order; ++j) + mm->free_trees[i][j] = RB_ROOT; + } mm->n_roots = hweight64(size); @@ -273,10 +342,7 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) sizeof(struct drm_buddy_block *), GFP_KERNEL); if (!mm->roots) - goto out_free_list; - - offset = 0; - i = 0; + goto out_free_tree; /* * Split into power-of-two blocks, in case we are given a size that is @@ -296,24 +362,26 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) mark_free(mm, root); - BUG_ON(i > mm->max_order); + BUG_ON(root_count > mm->max_order); BUG_ON(drm_buddy_block_size(mm, root) < chunk_size); - mm->roots[i] = root; + mm->roots[root_count] = root; offset += root_size; size -= root_size; - i++; + root_count++; } while (size); return 0; out_free_roots: - while (i--) - drm_block_free(mm, mm->roots[i]); + while (root_count--) + drm_block_free(mm, mm->roots[root_count]); kfree(mm->roots); -out_free_list: - kfree(mm->free_list); +out_free_tree: + while (i--) + kfree(mm->free_trees[i]); + kfree(mm->free_trees); return -ENOMEM; } EXPORT_SYMBOL(drm_buddy_init); @@ -323,7 +391,7 @@ EXPORT_SYMBOL(drm_buddy_init); * * @mm: DRM buddy manager to free * - * Cleanup memory manager resources and the freelist + * Cleanup memory manager resources and the freetree */ void drm_buddy_fini(struct drm_buddy *mm) { @@ -349,8 +417,10 @@ void drm_buddy_fini(struct drm_buddy *mm) WARN_ON(mm->avail != mm->size); + for_each_free_tree(i) + kfree(mm->free_trees[i]); + kfree(mm->free_trees); kfree(mm->roots); - kfree(mm->free_list); } EXPORT_SYMBOL(drm_buddy_fini); @@ -374,8 +444,7 @@ static int split_block(struct drm_buddy *mm, return -ENOMEM; } - mark_free(mm, block->left); - mark_free(mm, block->right); + mark_split(mm, block); if (drm_buddy_block_is_clear(block)) { mark_cleared(block->left); @@ -383,7 +452,8 @@ static int split_block(struct drm_buddy *mm, clear_reset(block); } - mark_split(block); + mark_free(mm, block->left); + mark_free(mm, block->right); return 0; } @@ -412,10 +482,11 @@ EXPORT_SYMBOL(drm_get_buddy); * @is_clear: blocks clear state * * Reset the clear state based on @is_clear value for each block - * in the freelist. + * in the freetree. */ void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear) { + enum drm_buddy_free_tree src_tree, dst_tree; u64 root_size, size, start; unsigned int order; int i; @@ -430,19 +501,24 @@ void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear) size -= root_size; } + src_tree = is_clear ? DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE; + dst_tree = is_clear ? DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; + for (i = 0; i <= mm->max_order; ++i) { - struct drm_buddy_block *block; - - list_for_each_entry_reverse(block, &mm->free_list[i], link) { - if (is_clear != drm_buddy_block_is_clear(block)) { - if (is_clear) { - mark_cleared(block); - mm->clear_avail += drm_buddy_block_size(mm, block); - } else { - clear_reset(block); - mm->clear_avail -= drm_buddy_block_size(mm, block); - } + struct rb_root *root = &mm->free_trees[src_tree][i]; + struct drm_buddy_block *block, *tmp; + + rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { + rbtree_remove(mm, block); + if (is_clear) { + mark_cleared(block); + mm->clear_avail += drm_buddy_block_size(mm, block); + } else { + clear_reset(block); + mm->clear_avail -= drm_buddy_block_size(mm, block); } + + rbtree_insert(mm, block, dst_tree); } } } @@ -632,23 +708,17 @@ __drm_buddy_alloc_range_bias(struct drm_buddy *mm, } static struct drm_buddy_block * -get_maxblock(struct drm_buddy *mm, unsigned int order, - unsigned long flags) +get_maxblock(struct drm_buddy *mm, + unsigned int order, + enum drm_buddy_free_tree tree) { struct drm_buddy_block *max_block = NULL, *block = NULL; + struct rb_root *root; unsigned int i; for (i = order; i <= mm->max_order; ++i) { - struct drm_buddy_block *tmp_block; - - list_for_each_entry_reverse(tmp_block, &mm->free_list[i], link) { - if (block_incompatible(tmp_block, flags)) - continue; - - block = tmp_block; - break; - } - + root = &mm->free_trees[tree][i]; + block = rbtree_last_free_block(root); if (!block) continue; @@ -667,46 +737,44 @@ get_maxblock(struct drm_buddy *mm, unsigned int order, } static struct drm_buddy_block * -alloc_from_freelist(struct drm_buddy *mm, +alloc_from_freetree(struct drm_buddy *mm, unsigned int order, unsigned long flags) { struct drm_buddy_block *block = NULL; + struct rb_root *root; + enum drm_buddy_free_tree tree; unsigned int tmp; int err; + tree = (flags & DRM_BUDDY_CLEAR_ALLOCATION) ? + DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; + if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) { - block = get_maxblock(mm, order, flags); + block = get_maxblock(mm, order, tree); if (block) /* Store the obtained block order */ tmp = drm_buddy_block_order(block); } else { for (tmp = order; tmp <= mm->max_order; ++tmp) { - struct drm_buddy_block *tmp_block; - - list_for_each_entry_reverse(tmp_block, &mm->free_list[tmp], link) { - if (block_incompatible(tmp_block, flags)) - continue; - - block = tmp_block; - break; - } - + /* Get RB tree root for this order and tree */ + root = &mm->free_trees[tree][tmp]; + block = rbtree_last_free_block(root); if (block) break; } } if (!block) { - /* Fallback method */ + /* Try allocating from the other tree */ + tree = (tree == DRM_BUDDY_CLEAR_TREE) ? + DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE; + for (tmp = order; tmp <= mm->max_order; ++tmp) { - if (!list_empty(&mm->free_list[tmp])) { - block = list_last_entry(&mm->free_list[tmp], - struct drm_buddy_block, - link); - if (block) - break; - } + root = &mm->free_trees[tree][tmp]; + block = rbtree_last_free_block(root); + if (block) + break; } if (!block) @@ -771,7 +839,7 @@ static int __alloc_range(struct drm_buddy *mm, if (contains(start, end, block_start, block_end)) { if (drm_buddy_block_is_free(block)) { - mark_allocated(block); + mark_allocated(mm, block); total_allocated += drm_buddy_block_size(mm, block); mm->avail -= drm_buddy_block_size(mm, block); if (drm_buddy_block_is_clear(block)) @@ -849,10 +917,9 @@ static int __alloc_contig_try_harder(struct drm_buddy *mm, { u64 rhs_offset, lhs_offset, lhs_size, filled; struct drm_buddy_block *block; - struct list_head *list; + unsigned int tree, order; LIST_HEAD(blocks_lhs); unsigned long pages; - unsigned int order; u64 modify_size; int err; @@ -862,35 +929,45 @@ static int __alloc_contig_try_harder(struct drm_buddy *mm, if (order == 0) return -ENOSPC; - list = &mm->free_list[order]; - if (list_empty(list)) - return -ENOSPC; + for_each_free_tree(tree) { + struct rb_root *root; + struct rb_node *iter; + + root = &mm->free_trees[tree][order]; + if (rbtree_is_empty(root)) + continue; - list_for_each_entry_reverse(block, list, link) { - /* Allocate blocks traversing RHS */ - rhs_offset = drm_buddy_block_offset(block); - err = __drm_buddy_alloc_range(mm, rhs_offset, size, - &filled, blocks); - if (!err || err != -ENOSPC) - return err; - - lhs_size = max((size - filled), min_block_size); - if (!IS_ALIGNED(lhs_size, min_block_size)) - lhs_size = round_up(lhs_size, min_block_size); - - /* Allocate blocks traversing LHS */ - lhs_offset = drm_buddy_block_offset(block) - lhs_size; - err = __drm_buddy_alloc_range(mm, lhs_offset, lhs_size, - NULL, &blocks_lhs); - if (!err) { - list_splice(&blocks_lhs, blocks); - return 0; - } else if (err != -ENOSPC) { + iter = rb_last(root); + while (iter) { + block = rbtree_get_free_block(iter); + + /* Allocate blocks traversing RHS */ + rhs_offset = drm_buddy_block_offset(block); + err = __drm_buddy_alloc_range(mm, rhs_offset, size, + &filled, blocks); + if (!err || err != -ENOSPC) + return err; + + lhs_size = max((size - filled), min_block_size); + if (!IS_ALIGNED(lhs_size, min_block_size)) + lhs_size = round_up(lhs_size, min_block_size); + + /* Allocate blocks traversing LHS */ + lhs_offset = drm_buddy_block_offset(block) - lhs_size; + err = __drm_buddy_alloc_range(mm, lhs_offset, lhs_size, + NULL, &blocks_lhs); + if (!err) { + list_splice(&blocks_lhs, blocks); + return 0; + } else if (err != -ENOSPC) { + drm_buddy_free_list_internal(mm, blocks); + return err; + } + /* Free blocks for the next iteration */ drm_buddy_free_list_internal(mm, blocks); - return err; + + iter = rb_prev(iter); } - /* Free blocks for the next iteration */ - drm_buddy_free_list_internal(mm, blocks); } return -ENOSPC; @@ -976,7 +1053,7 @@ int drm_buddy_block_trim(struct drm_buddy *mm, list_add(&block->tmp_link, &dfs); err = __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL); if (err) { - mark_allocated(block); + mark_allocated(mm, block); mm->avail -= drm_buddy_block_size(mm, block); if (drm_buddy_block_is_clear(block)) mm->clear_avail -= drm_buddy_block_size(mm, block); @@ -999,8 +1076,8 @@ __drm_buddy_alloc_blocks(struct drm_buddy *mm, return __drm_buddy_alloc_range_bias(mm, start, end, order, flags); else - /* Allocate from freelist */ - return alloc_from_freelist(mm, order, flags); + /* Allocate from freetree */ + return alloc_from_freetree(mm, order, flags); } /** @@ -1017,8 +1094,8 @@ __drm_buddy_alloc_blocks(struct drm_buddy *mm, * alloc_range_bias() called on range limitations, which traverses * the tree and returns the desired block. * - * alloc_from_freelist() called when *no* range restrictions - * are enforced, which picks the block from the freelist. + * alloc_from_freetree() called when *no* range restrictions + * are enforced, which picks the block from the freetree. * * Returns: * 0 on success, error code on failure. @@ -1078,6 +1155,15 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm, order = fls(pages) - 1; min_order = ilog2(min_block_size) - ilog2(mm->chunk_size); + if (order > mm->max_order || size > mm->size) { + if ((flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) && + !(flags & DRM_BUDDY_RANGE_ALLOCATION)) + return __alloc_contig_try_harder(mm, original_size, + original_min_size, blocks); + + return -EINVAL; + } + do { order = min(order, (unsigned int)fls(pages) - 1); BUG_ON(order > mm->max_order); @@ -1120,7 +1206,7 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm, } } while (1); - mark_allocated(block); + mark_allocated(mm, block); mm->avail -= drm_buddy_block_size(mm, block); if (drm_buddy_block_is_clear(block)) mm->clear_avail -= drm_buddy_block_size(mm, block); @@ -1201,12 +1287,18 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p) mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20); for (order = mm->max_order; order >= 0; order--) { - struct drm_buddy_block *block; + struct drm_buddy_block *block, *tmp; + struct rb_root *root; u64 count = 0, free; + unsigned int tree; + + for_each_free_tree(tree) { + root = &mm->free_trees[tree][order]; - list_for_each_entry(block, &mm->free_list[order], link) { - BUG_ON(!drm_buddy_block_is_free(block)); - count++; + rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { + BUG_ON(!drm_buddy_block_is_free(block)); + count++; + } } drm_printf(p, "order-%2d ", order); diff --git a/drivers/gpu/drm/drm_client_modeset.c b/drivers/gpu/drm/drm_client_modeset.c index 9c2c3b0c8c470..eaf71c9ecf136 100644 --- a/drivers/gpu/drm/drm_client_modeset.c +++ b/drivers/gpu/drm/drm_client_modeset.c @@ -930,7 +930,8 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width, mutex_unlock(&client->modeset_mutex); out: kfree(crtcs); - modes_destroy(dev, modes, connector_count); + if (modes) + modes_destroy(dev, modes, connector_count); kfree(modes); kfree(offsets); kfree(enabled); diff --git a/drivers/gpu/drm/drm_displayid.c b/drivers/gpu/drm/drm_displayid.c index b4fd43783c509..58d0bb6d26768 100644 --- a/drivers/gpu/drm/drm_displayid.c +++ b/drivers/gpu/drm/drm_displayid.c @@ -9,6 +9,34 @@ #include "drm_crtc_internal.h" #include "drm_displayid_internal.h" +enum { + QUIRK_IGNORE_CHECKSUM, +}; + +struct displayid_quirk { + const struct drm_edid_ident ident; + u8 quirks; +}; + +static const struct displayid_quirk quirks[] = { + { + .ident = DRM_EDID_IDENT_INIT('C', 'S', 'O', 5142, "MNE007ZA1-5"), + .quirks = BIT(QUIRK_IGNORE_CHECKSUM), + }, +}; + +static u8 get_quirks(const struct drm_edid *drm_edid) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(quirks); i++) { + if (drm_edid_match(drm_edid, &quirks[i].ident)) + return quirks[i].quirks; + } + + return 0; +} + static const struct displayid_header * displayid_get_header(const u8 *displayid, int length, int index) { @@ -23,7 +51,7 @@ displayid_get_header(const u8 *displayid, int length, int index) } static const struct displayid_header * -validate_displayid(const u8 *displayid, int length, int idx) +validate_displayid(const u8 *displayid, int length, int idx, bool ignore_checksum) { int i, dispid_length; u8 csum = 0; @@ -41,33 +69,35 @@ validate_displayid(const u8 *displayid, int length, int idx) for (i = 0; i < dispid_length; i++) csum += displayid[idx + i]; if (csum) { - DRM_NOTE("DisplayID checksum invalid, remainder is %d\n", csum); - return ERR_PTR(-EINVAL); + DRM_NOTE("DisplayID checksum invalid, remainder is %d%s\n", csum, + ignore_checksum ? " (ignoring)" : ""); + + if (!ignore_checksum) + return ERR_PTR(-EINVAL); } return base; } -static const u8 *drm_find_displayid_extension(const struct drm_edid *drm_edid, - int *length, int *idx, - int *ext_index) +static const u8 *find_next_displayid_extension(struct displayid_iter *iter) { const struct displayid_header *base; const u8 *displayid; + bool ignore_checksum = iter->quirks & BIT(QUIRK_IGNORE_CHECKSUM); - displayid = drm_edid_find_extension(drm_edid, DISPLAYID_EXT, ext_index); + displayid = drm_edid_find_extension(iter->drm_edid, DISPLAYID_EXT, &iter->ext_index); if (!displayid) return NULL; /* EDID extensions block checksum isn't for us */ - *length = EDID_LENGTH - 1; - *idx = 1; + iter->length = EDID_LENGTH - 1; + iter->idx = 1; - base = validate_displayid(displayid, *length, *idx); + base = validate_displayid(displayid, iter->length, iter->idx, ignore_checksum); if (IS_ERR(base)) return NULL; - *length = *idx + sizeof(*base) + base->bytes; + iter->length = iter->idx + sizeof(*base) + base->bytes; return displayid; } @@ -78,6 +108,7 @@ void displayid_iter_edid_begin(const struct drm_edid *drm_edid, memset(iter, 0, sizeof(*iter)); iter->drm_edid = drm_edid; + iter->quirks = get_quirks(drm_edid); } static const struct displayid_block * @@ -126,10 +157,7 @@ __displayid_iter_next(struct displayid_iter *iter) /* The first section we encounter is the base section */ bool base_section = !iter->section; - iter->section = drm_find_displayid_extension(iter->drm_edid, - &iter->length, - &iter->idx, - &iter->ext_index); + iter->section = find_next_displayid_extension(iter); if (!iter->section) { iter->drm_edid = NULL; return NULL; diff --git a/drivers/gpu/drm/drm_displayid_internal.h b/drivers/gpu/drm/drm_displayid_internal.h index 957dd0619f5c5..5b1b32f735166 100644 --- a/drivers/gpu/drm/drm_displayid_internal.h +++ b/drivers/gpu/drm/drm_displayid_internal.h @@ -167,6 +167,8 @@ struct displayid_iter { u8 version; u8 primary_use; + + u8 quirks; }; void displayid_iter_edid_begin(const struct drm_edid *drm_edid, diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index f884d155a832a..11e7141c1524b 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -970,33 +970,41 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, { struct drm_gem_change_handle *args = data; struct drm_gem_object *obj; - int ret; + int handle, ret; if (!drm_core_check_feature(dev, DRIVER_GEM)) return -EOPNOTSUPP; + /* idr_alloc() limitation. */ + if (args->new_handle > INT_MAX) + return -EINVAL; + handle = args->new_handle; + obj = drm_gem_object_lookup(file_priv, args->handle); if (!obj) return -ENOENT; - if (args->handle == args->new_handle) - return 0; + if (args->handle == handle) { + ret = 0; + goto out; + } mutex_lock(&file_priv->prime.lock); spin_lock(&file_priv->table_lock); - ret = idr_alloc(&file_priv->object_idr, obj, - args->new_handle, args->new_handle + 1, GFP_NOWAIT); + ret = idr_alloc(&file_priv->object_idr, obj, handle, handle + 1, + GFP_NOWAIT); spin_unlock(&file_priv->table_lock); if (ret < 0) goto out_unlock; if (obj->dma_buf) { - ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf, args->new_handle); + ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf, + handle); if (ret < 0) { spin_lock(&file_priv->table_lock); - idr_remove(&file_priv->object_idr, args->new_handle); + idr_remove(&file_priv->object_idr, handle); spin_unlock(&file_priv->table_lock); goto out_unlock; } @@ -1012,6 +1020,8 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, out_unlock: mutex_unlock(&file_priv->prime.lock); +out: + drm_gem_object_put(obj); return ret; } diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 5d1349c34afd3..5c6da91bbba17 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -15,6 +15,8 @@ #include #endif +#include + #include #include #include @@ -861,6 +863,67 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_import_no_map); +/* + * Kunit helpers + */ + +#if IS_ENABLED(CONFIG_KUNIT) +int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + ret = drm_gem_shmem_vmap_locked(shmem, map); + dma_resv_unlock(obj->resv); + + return ret; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_vmap); + +void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_lock_interruptible(obj->resv, NULL); + drm_gem_shmem_vunmap_locked(shmem, map); + dma_resv_unlock(obj->resv); +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_vunmap); + +int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + ret = drm_gem_shmem_madvise_locked(shmem, madv); + dma_resv_unlock(obj->resv); + + return ret; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_madvise); + +int drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + drm_gem_shmem_purge_locked(shmem); + dma_resv_unlock(obj->resv); + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_purge); +#endif + MODULE_DESCRIPTION("DRM SHMEM memory-management helpers"); MODULE_IMPORT_NS("DMA_BUF"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c index e6b5b06de1487..f3e40d1e6098b 100644 --- a/drivers/gpu/drm/drm_ioc32.c +++ b/drivers/gpu/drm/drm_ioc32.c @@ -28,6 +28,7 @@ * IN THE SOFTWARE. */ #include +#include #include #include @@ -374,6 +375,7 @@ long drm_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (nr >= ARRAY_SIZE(drm_compat_ioctls)) return drm_ioctl(filp, cmd, arg); + nr = array_index_nospec(nr, ARRAY_SIZE(drm_compat_ioctls)); fn = drm_compat_ioctls[nr].fn; if (!fn) return drm_ioctl(filp, cmd, arg); diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 22c44807e3fe9..445cf0e6589fa 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -3,6 +3,7 @@ * Copyright © 2024-2025 Intel Corporation */ +#include #include #include #include @@ -408,10 +409,14 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, drm_pagemap_get_devmem_page(page, zdd); } - err = ops->copy_to_devmem(pages, pagemap_addr, npages); + err = ops->copy_to_devmem(pages, pagemap_addr, npages, + devmem_allocation->pre_migrate_fence); if (err) goto err_finalize; + dma_fence_put(devmem_allocation->pre_migrate_fence); + devmem_allocation->pre_migrate_fence = NULL; + /* Upon success bind devmem allocation to range and zdd */ devmem_allocation->timeslice_expiration = get_jiffies_64() + msecs_to_jiffies(timeslice_ms); @@ -596,7 +601,7 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) for (i = 0; i < npages; ++i) pages[i] = migrate_pfn_to_page(src[i]); - err = ops->copy_to_ram(pages, pagemap_addr, npages); + err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); if (err) goto err_finalize; @@ -732,7 +737,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, for (i = 0; i < npages; ++i) pages[i] = migrate_pfn_to_page(migrate.src[i]); - err = ops->copy_to_ram(pages, pagemap_addr, npages); + err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); if (err) goto err_finalize; @@ -813,11 +818,14 @@ EXPORT_SYMBOL_GPL(drm_pagemap_pagemap_ops_get); * @ops: Pointer to the operations structure for GPU SVM device memory * @dpagemap: The struct drm_pagemap we're allocating from. * @size: Size of device memory allocation + * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. + * (May be NULL). */ void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, struct device *dev, struct mm_struct *mm, const struct drm_pagemap_devmem_ops *ops, - struct drm_pagemap *dpagemap, size_t size) + struct drm_pagemap *dpagemap, size_t size, + struct dma_fence *pre_migrate_fence) { init_completion(&devmem_allocation->detached); devmem_allocation->dev = dev; @@ -825,6 +833,7 @@ void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, devmem_allocation->ops = ops; devmem_allocation->dpagemap = dpagemap; devmem_allocation->size = size; + devmem_allocation->pre_migrate_fence = pre_migrate_fence; } EXPORT_SYMBOL_GPL(drm_pagemap_devmem_init); diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c index a30493ed97157..4cadea997129d 100644 --- a/drivers/gpu/drm/drm_plane.c +++ b/drivers/gpu/drm/drm_plane.c @@ -338,14 +338,14 @@ static int drm_plane_create_hotspot_properties(struct drm_plane *plane) prop_x = drm_property_create_signed_range(plane->dev, 0, "HOTSPOT_X", INT_MIN, INT_MAX); - if (IS_ERR(prop_x)) - return PTR_ERR(prop_x); + if (!prop_x) + return -ENOMEM; prop_y = drm_property_create_signed_range(plane->dev, 0, "HOTSPOT_Y", INT_MIN, INT_MAX); - if (IS_ERR(prop_y)) { + if (!prop_y) { drm_property_destroy(plane->dev, prop_x); - return PTR_ERR(prop_y); + return -ENOMEM; } drm_object_attach_property(&plane->base, prop_x, 0); diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c index 596272149a359..3c88b5fbdf28c 100644 --- a/drivers/gpu/drm/drm_property.c +++ b/drivers/gpu/drm/drm_property.c @@ -562,7 +562,7 @@ drm_property_create_blob(struct drm_device *dev, size_t length, if (!length || length > INT_MAX - sizeof(struct drm_property_blob)) return ERR_PTR(-EINVAL); - blob = kvzalloc(sizeof(struct drm_property_blob)+length, GFP_KERNEL); + blob = kvzalloc(sizeof(struct drm_property_blob) + length, GFP_KERNEL_ACCOUNT); if (!blob) return ERR_PTR(-ENOMEM); diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index e1b0fa4000cdd..7eb2cdbc574a0 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -900,7 +900,7 @@ drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, return drm_syncobj_export_sync_file(file_private, args->handle, point, &args->fd); - if (args->point) + if (point) return -EINVAL; return drm_syncobj_handle_to_fd(file_private, args->handle, @@ -934,7 +934,7 @@ drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, args->handle, point); - if (args->point) + if (point) return -EINVAL; return drm_syncobj_fd_to_handle(file_private, args->fd, diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h index 23646e55f142c..06c29ff2aac0e 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_drv.h +++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h @@ -199,6 +199,7 @@ struct drm_exynos_file_private { struct exynos_drm_private { struct device *g2d_dev; struct device *dma_dev; + struct device *vidi_dev; void *mapping; /* for atomic commit */ diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c index e094b8bbc0f12..37733f2ac0e7a 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c +++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c @@ -186,29 +186,37 @@ static ssize_t vidi_store_connection(struct device *dev, const char *buf, size_t len) { struct vidi_context *ctx = dev_get_drvdata(dev); - int ret; + int ret, new_connected; - ret = kstrtoint(buf, 0, &ctx->connected); + ret = kstrtoint(buf, 0, &new_connected); if (ret) return ret; - - if (ctx->connected > 1) + if (new_connected > 1) return -EINVAL; + mutex_lock(&ctx->lock); + /* * Use fake edid data for test. If raw_edid is set then it can't be * tested. */ if (ctx->raw_edid) { DRM_DEV_DEBUG_KMS(dev, "edid data is not fake data.\n"); - return -EINVAL; + ret = -EINVAL; + goto fail; } + ctx->connected = new_connected; + mutex_unlock(&ctx->lock); + DRM_DEV_DEBUG_KMS(dev, "requested connection.\n"); drm_helper_hpd_irq_event(ctx->drm_dev); return len; +fail: + mutex_unlock(&ctx->lock); + return ret; } static DEVICE_ATTR(connection, 0644, vidi_show_connection, @@ -223,9 +231,14 @@ ATTRIBUTE_GROUPS(vidi); int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, struct drm_file *file_priv) { - struct vidi_context *ctx = dev_get_drvdata(drm_dev->dev); + struct exynos_drm_private *priv = drm_dev->dev_private; + struct device *dev = priv ? priv->vidi_dev : NULL; + struct vidi_context *ctx = dev ? dev_get_drvdata(dev) : NULL; struct drm_exynos_vidi_connection *vidi = data; + if (!ctx) + return -ENODEV; + if (!vidi) { DRM_DEV_DEBUG_KMS(ctx->dev, "user data for vidi is null.\n"); @@ -238,21 +251,38 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, return -EINVAL; } + mutex_lock(&ctx->lock); if (ctx->connected == vidi->connection) { + mutex_unlock(&ctx->lock); DRM_DEV_DEBUG_KMS(ctx->dev, "same connection request.\n"); return -EINVAL; } + mutex_unlock(&ctx->lock); if (vidi->connection) { const struct drm_edid *drm_edid; - const struct edid *raw_edid; + const void __user *edid_userptr = u64_to_user_ptr(vidi->edid); + void *edid_buf; + struct edid hdr; size_t size; - raw_edid = (const struct edid *)(unsigned long)vidi->edid; - size = (raw_edid->extensions + 1) * EDID_LENGTH; + if (copy_from_user(&hdr, edid_userptr, sizeof(hdr))) + return -EFAULT; + + size = (hdr.extensions + 1) * EDID_LENGTH; + + edid_buf = kmalloc(size, GFP_KERNEL); + if (!edid_buf) + return -ENOMEM; - drm_edid = drm_edid_alloc(raw_edid, size); + if (copy_from_user(edid_buf, edid_userptr, size)) { + kfree(edid_buf); + return -EFAULT; + } + + drm_edid = drm_edid_alloc(edid_buf, size); + kfree(edid_buf); if (!drm_edid) return -ENOMEM; @@ -262,14 +292,21 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, "edid data is invalid.\n"); return -EINVAL; } + mutex_lock(&ctx->lock); ctx->raw_edid = drm_edid; + mutex_unlock(&ctx->lock); } else { /* with connection = 0, free raw_edid */ + mutex_lock(&ctx->lock); drm_edid_free(ctx->raw_edid); ctx->raw_edid = NULL; + mutex_unlock(&ctx->lock); } + mutex_lock(&ctx->lock); ctx->connected = vidi->connection; + mutex_unlock(&ctx->lock); + drm_helper_hpd_irq_event(ctx->drm_dev); return 0; @@ -284,7 +321,7 @@ static enum drm_connector_status vidi_detect(struct drm_connector *connector, * connection request would come from user side * to do hotplug through specific ioctl. */ - return ctx->connected ? connector_status_connected : + return READ_ONCE(ctx->connected) ? connector_status_connected : connector_status_disconnected; } @@ -307,11 +344,15 @@ static int vidi_get_modes(struct drm_connector *connector) const struct drm_edid *drm_edid; int count; + mutex_lock(&ctx->lock); + if (ctx->raw_edid) drm_edid = drm_edid_dup(ctx->raw_edid); else drm_edid = drm_edid_alloc(fake_edid_info, sizeof(fake_edid_info)); + mutex_unlock(&ctx->lock); + drm_edid_connector_update(connector, drm_edid); count = drm_edid_connector_add_modes(connector); @@ -371,6 +412,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; struct drm_encoder *encoder = &ctx->encoder; struct exynos_drm_plane *exynos_plane; struct exynos_drm_plane_config plane_config = { 0 }; @@ -378,6 +420,8 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) int ret; ctx->drm_dev = drm_dev; + if (priv) + priv->vidi_dev = dev; plane_config.pixel_formats = formats; plane_config.num_pixel_formats = ARRAY_SIZE(formats); @@ -423,8 +467,12 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) static void vidi_unbind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); + struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; timer_delete_sync(&ctx->timer); + if (priv) + priv->vidi_dev = NULL; } static const struct component_ops vidi_component_ops = { @@ -456,9 +504,13 @@ static void vidi_remove(struct platform_device *pdev) { struct vidi_context *ctx = platform_get_drvdata(pdev); + mutex_lock(&ctx->lock); + drm_edid_free(ctx->raw_edid); ctx->raw_edid = NULL; + mutex_unlock(&ctx->lock); + component_del(&pdev->dev, &vidi_component_ops); } diff --git a/drivers/gpu/drm/gma500/fbdev.c b/drivers/gpu/drm/gma500/fbdev.c index 32d31e5f5f1a7..a6af21514cff7 100644 --- a/drivers/gpu/drm/gma500/fbdev.c +++ b/drivers/gpu/drm/gma500/fbdev.c @@ -50,48 +50,6 @@ static const struct vm_operations_struct psb_fbdev_vm_ops = { * struct fb_ops */ -#define CMAP_TOHW(_val, _width) ((((_val) << (_width)) + 0x7FFF - (_val)) >> 16) - -static int psb_fbdev_fb_setcolreg(unsigned int regno, - unsigned int red, unsigned int green, - unsigned int blue, unsigned int transp, - struct fb_info *info) -{ - struct drm_fb_helper *fb_helper = info->par; - struct drm_framebuffer *fb = fb_helper->fb; - uint32_t v; - - if (!fb) - return -ENOMEM; - - if (regno > 255) - return 1; - - red = CMAP_TOHW(red, info->var.red.length); - blue = CMAP_TOHW(blue, info->var.blue.length); - green = CMAP_TOHW(green, info->var.green.length); - transp = CMAP_TOHW(transp, info->var.transp.length); - - v = (red << info->var.red.offset) | - (green << info->var.green.offset) | - (blue << info->var.blue.offset) | - (transp << info->var.transp.offset); - - if (regno < 16) { - switch (fb->format->cpp[0] * 8) { - case 16: - ((uint32_t *) info->pseudo_palette)[regno] = v; - break; - case 24: - case 32: - ((uint32_t *) info->pseudo_palette)[regno] = v; - break; - } - } - - return 0; -} - static int psb_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma) { if (vma->vm_pgoff != 0) @@ -135,7 +93,6 @@ static const struct fb_ops psb_fbdev_fb_ops = { .owner = THIS_MODULE, __FB_DEFAULT_IOMEM_OPS_RDWR, DRM_FB_HELPER_DEFAULT_OPS, - .fb_setcolreg = psb_fbdev_fb_setcolreg, __FB_DEFAULT_IOMEM_OPS_DRAW, .fb_mmap = psb_fbdev_fb_mmap, .fb_destroy = psb_fbdev_fb_destroy, diff --git a/drivers/gpu/drm/gud/gud_drv.c b/drivers/gpu/drm/gud/gud_drv.c index b7345c8d823de..a2000991ecbe4 100644 --- a/drivers/gpu/drm/gud/gud_drv.c +++ b/drivers/gpu/drm/gud/gud_drv.c @@ -249,7 +249,7 @@ int gud_usb_set_u8(struct gud_device *gdrm, u8 request, u8 val) return gud_usb_set(gdrm, request, 0, &val, sizeof(val)); } -static int gud_get_properties(struct gud_device *gdrm) +static int gud_plane_add_properties(struct gud_device *gdrm) { struct gud_property_req *properties; unsigned int i, num_properties; @@ -339,7 +339,9 @@ static int gud_stats_debugfs(struct seq_file *m, void *data) } static const struct drm_crtc_helper_funcs gud_crtc_helper_funcs = { - .atomic_check = drm_crtc_helper_atomic_check + .atomic_check = drm_crtc_helper_atomic_check, + .atomic_enable = gud_crtc_atomic_enable, + .atomic_disable = gud_crtc_atomic_disable, }; static const struct drm_crtc_funcs gud_crtc_funcs = { @@ -364,6 +366,10 @@ static const struct drm_plane_funcs gud_plane_funcs = { DRM_GEM_SHADOW_PLANE_FUNCS, }; +static const struct drm_mode_config_helper_funcs gud_mode_config_helpers = { + .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, +}; + static const struct drm_mode_config_funcs gud_mode_config_funcs = { .fb_create = drm_gem_fb_create_with_dirty, .atomic_check = drm_atomic_helper_check, @@ -463,10 +469,6 @@ static int gud_probe(struct usb_interface *intf, const struct usb_device_id *id) return PTR_ERR(gdrm); drm = &gdrm->drm; - drm->mode_config.funcs = &gud_mode_config_funcs; - ret = drmm_mode_config_init(drm); - if (ret) - return ret; gdrm->flags = le32_to_cpu(desc.flags); gdrm->compression = desc.compression & GUD_COMPRESSION_LZ4; @@ -483,11 +485,29 @@ static int gud_probe(struct usb_interface *intf, const struct usb_device_id *id) if (ret) return ret; + usb_set_intfdata(intf, gdrm); + + dma_dev = usb_intf_get_dma_device(intf); + if (dma_dev) { + drm_dev_set_dma_dev(drm, dma_dev); + put_device(dma_dev); + } else { + dev_warn(dev, "buffer sharing not supported"); /* not an error */ + } + + /* Mode config init */ + ret = drmm_mode_config_init(drm); + if (ret) + return ret; + drm->mode_config.min_width = le32_to_cpu(desc.min_width); drm->mode_config.max_width = le32_to_cpu(desc.max_width); drm->mode_config.min_height = le32_to_cpu(desc.min_height); drm->mode_config.max_height = le32_to_cpu(desc.max_height); + drm->mode_config.funcs = &gud_mode_config_funcs; + drm->mode_config.helper_private = &gud_mode_config_helpers; + /* Format init */ formats_dev = devm_kmalloc(dev, GUD_FORMATS_MAX_NUM, GFP_KERNEL); /* Add room for emulated XRGB8888 */ formats = devm_kmalloc_array(dev, GUD_FORMATS_MAX_NUM + 1, sizeof(*formats), GFP_KERNEL); @@ -587,6 +607,7 @@ static int gud_probe(struct usb_interface *intf, const struct usb_device_id *id) return -ENOMEM; } + /* Pipeline init */ ret = drm_universal_plane_init(drm, &gdrm->plane, 0, &gud_plane_funcs, formats, num_formats, @@ -598,12 +619,9 @@ static int gud_probe(struct usb_interface *intf, const struct usb_device_id *id) drm_plane_helper_add(&gdrm->plane, &gud_plane_helper_funcs); drm_plane_enable_fb_damage_clips(&gdrm->plane); - devm_kfree(dev, formats); - devm_kfree(dev, formats_dev); - - ret = gud_get_properties(gdrm); + ret = gud_plane_add_properties(gdrm); if (ret) { - dev_err(dev, "Failed to get properties (error=%d)\n", ret); + dev_err(dev, "Failed to add properties (error=%d)\n", ret); return ret; } @@ -621,16 +639,7 @@ static int gud_probe(struct usb_interface *intf, const struct usb_device_id *id) } drm_mode_config_reset(drm); - - usb_set_intfdata(intf, gdrm); - - dma_dev = usb_intf_get_dma_device(intf); - if (dma_dev) { - drm_dev_set_dma_dev(drm, dma_dev); - put_device(dma_dev); - } else { - dev_warn(dev, "buffer sharing not supported"); /* not an error */ - } + drm_kms_helper_poll_init(drm); drm_debugfs_add_file(drm, "stats", gud_stats_debugfs, NULL); @@ -638,7 +647,8 @@ static int gud_probe(struct usb_interface *intf, const struct usb_device_id *id) if (ret) return ret; - drm_kms_helper_poll_init(drm); + devm_kfree(dev, formats); + devm_kfree(dev, formats_dev); drm_client_setup(drm, NULL); diff --git a/drivers/gpu/drm/gud/gud_internal.h b/drivers/gpu/drm/gud/gud_internal.h index d27c31648341c..8eec8335f5f90 100644 --- a/drivers/gpu/drm/gud/gud_internal.h +++ b/drivers/gpu/drm/gud/gud_internal.h @@ -62,6 +62,10 @@ int gud_usb_set_u8(struct gud_device *gdrm, u8 request, u8 val); void gud_clear_damage(struct gud_device *gdrm); void gud_flush_work(struct work_struct *work); +void gud_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state); +void gud_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state); int gud_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_state *state); void gud_plane_atomic_update(struct drm_plane *plane, diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c index 54d9aa9998e59..0d38eecd152da 100644 --- a/drivers/gpu/drm/gud/gud_pipe.c +++ b/drivers/gpu/drm/gud/gud_pipe.c @@ -457,27 +457,20 @@ int gud_plane_atomic_check(struct drm_plane *plane, struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane); struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane); struct drm_crtc *crtc = new_plane_state->crtc; - struct drm_crtc_state *crtc_state; + struct drm_crtc_state *crtc_state = NULL; const struct drm_display_mode *mode; struct drm_framebuffer *old_fb = old_plane_state->fb; struct drm_connector_state *connector_state = NULL; struct drm_framebuffer *fb = new_plane_state->fb; - const struct drm_format_info *format = fb->format; + const struct drm_format_info *format; struct drm_connector *connector; unsigned int i, num_properties; struct gud_state_req *req; int idx, ret; size_t len; - if (drm_WARN_ON_ONCE(plane->dev, !fb)) - return -EINVAL; - - if (drm_WARN_ON_ONCE(plane->dev, !crtc)) - return -EINVAL; - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - - mode = &crtc_state->mode; + if (crtc) + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state, DRM_PLANE_NO_SCALING, @@ -492,6 +485,9 @@ int gud_plane_atomic_check(struct drm_plane *plane, if (old_plane_state->rotation != new_plane_state->rotation) crtc_state->mode_changed = true; + mode = &crtc_state->mode; + format = fb->format; + if (old_fb && old_fb->format != format) crtc_state->mode_changed = true; @@ -584,6 +580,39 @@ int gud_plane_atomic_check(struct drm_plane *plane, return ret; } +void gud_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_device *drm = crtc->dev; + struct gud_device *gdrm = to_gud_device(drm); + int idx; + + if (!drm_dev_enter(drm, &idx)) + return; + + gud_usb_set_u8(gdrm, GUD_REQ_SET_CONTROLLER_ENABLE, 1); + gud_usb_set(gdrm, GUD_REQ_SET_STATE_COMMIT, 0, NULL, 0); + gud_usb_set_u8(gdrm, GUD_REQ_SET_DISPLAY_ENABLE, 1); + + drm_dev_exit(idx); +} + +void gud_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_device *drm = crtc->dev; + struct gud_device *gdrm = to_gud_device(drm); + int idx; + + if (!drm_dev_enter(drm, &idx)) + return; + + gud_usb_set_u8(gdrm, GUD_REQ_SET_DISPLAY_ENABLE, 0); + gud_usb_set_u8(gdrm, GUD_REQ_SET_CONTROLLER_ENABLE, 0); + + drm_dev_exit(idx); +} + void gud_plane_atomic_update(struct drm_plane *plane, struct drm_atomic_state *atomic_state) { @@ -598,7 +627,7 @@ void gud_plane_atomic_update(struct drm_plane *plane, struct drm_atomic_helper_damage_iter iter; int ret, idx; - if (crtc->state->mode_changed || !crtc->state->enable) { + if (!crtc || crtc->state->mode_changed || !crtc->state->enable) { cancel_work_sync(&gdrm->work); mutex_lock(&gdrm->damage_lock); if (gdrm->fb) { @@ -611,24 +640,12 @@ void gud_plane_atomic_update(struct drm_plane *plane, mutex_unlock(&gdrm->damage_lock); } - if (!drm_dev_enter(drm, &idx)) + if (!crtc || !drm_dev_enter(drm, &idx)) return; - if (!old_state->fb) - gud_usb_set_u8(gdrm, GUD_REQ_SET_CONTROLLER_ENABLE, 1); - - if (fb && (crtc->state->mode_changed || crtc->state->connectors_changed)) - gud_usb_set(gdrm, GUD_REQ_SET_STATE_COMMIT, 0, NULL, 0); - - if (crtc->state->active_changed) - gud_usb_set_u8(gdrm, GUD_REQ_SET_DISPLAY_ENABLE, crtc->state->active); - - if (!fb) - goto ctrl_disable; - ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE); if (ret) - goto ctrl_disable; + goto out; drm_atomic_helper_damage_iter_init(&iter, old_state, new_state); drm_atomic_for_each_plane_damage(&iter, &damage) @@ -636,9 +653,6 @@ void gud_plane_atomic_update(struct drm_plane *plane, drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); -ctrl_disable: - if (!crtc->state->enable) - gud_usb_set_u8(gdrm, GUD_REQ_SET_CONTROLLER_ENABLE, 0); - +out: drm_dev_exit(idx); } diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h index 4add05c7f161a..f9ee7ebfec55c 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h @@ -40,6 +40,10 @@ struct hibmc_dp_dev { struct mutex lock; /* protects concurrent RW in hibmc_dp_reg_write_field() */ struct hibmc_dp_link link; u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; + struct drm_dp_desc desc; + bool is_branch; + int hpd_status; void __iomem *serdes_base; }; diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h index 08f9e1caf7fcb..efb30a7584758 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h @@ -17,5 +17,7 @@ #define HIBMC_DP_LINK_RATE_CAL 27 #define HIBMC_DP_SYNC_DELAY(lanes) ((lanes) == 0x2 ? 86 : 46) #define HIBMC_DP_INT_ENABLE 0xc +/* HIBMC_DP_LINK_RATE_CAL * 10000 * 80% = 216000 */ +#define DP_MODE_VALI_CAL 216000 #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c index 8f0daec7d1749..d5bd3c45649b2 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c @@ -2,6 +2,7 @@ // Copyright (c) 2024 Hisilicon Limited. #include +#include #include #include "dp_config.h" #include "dp_comm.h" @@ -176,13 +177,18 @@ int hibmc_dp_hw_init(struct hibmc_dp *dp) dp_dev->link.cap.lanes = 0x2; dp_dev->link.cap.link_rate = DP_LINK_BW_8_1; - /* hdcp data */ - writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* int init */ writel(0, dp_dev->base + HIBMC_DP_INTR_ENABLE); writel(HIBMC_DP_INT_RST, dp_dev->base + HIBMC_DP_INTR_ORIGINAL_STATUS); + /* clr colorbar */ + writel(0, dp_dev->base + HIBMC_DP_COLOR_BAR_CTRL); /* rst */ + writel(0, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + usleep_range(30, 50); + /* de-rst */ writel(HIBMC_DP_DPTX_RST, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + /* hdcp data */ + writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* clock enable */ writel(HIBMC_DP_CLK_EN, dp_dev->base + HIBMC_DP_DPTX_CLK_CTRL); @@ -263,6 +269,16 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp) dp->dp_dev->link.status.channel_equalized = false; } +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.link_rate; +} + +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.lanes; +} + static const struct hibmc_dp_color_raw g_rgb_raw[] = { {CBAR_COLOR_BAR, 0x000, 0x000, 0x000}, {CBAR_WHITE, 0xfff, 0xfff, 0xfff}, @@ -305,3 +321,21 @@ void hibmc_dp_set_cbar(struct hibmc_dp *dp, const struct hibmc_dp_cbar_cfg *cfg) hibmc_dp_reg_write_field(dp_dev, HIBMC_DP_COLOR_BAR_CTRL, BIT(0), cfg->enable); writel(HIBMC_DP_SYNC_EN_MASK, dp_dev->base + HIBMC_DP_TIMING_SYNC_CTRL); } + +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status) +{ + u32 status; + int ret; + + ret = readl_poll_timeout(dp->dp_dev->base + HIBMC_DP_HPD_STATUS, status, + FIELD_GET(HIBMC_DP_HPD_CUR_STATE, status) == exp_status, + 1000, 100000); /* DP spec says 100ms */ + if (ret) { + drm_dbg_dp(dp->drm_dev, "wait hpd status timeout"); + return false; + } + + dp->dp_dev->hpd_status = exp_status; + + return true; +} diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h index 665f5b166dfb5..31316fe1ea8db 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h @@ -14,6 +14,11 @@ struct hibmc_dp_dev; +enum hibmc_hpd_status { + HIBMC_HPD_OUT, + HIBMC_HPD_IN, +}; + enum hibmc_dp_cbar_pattern { CBAR_COLOR_BAR, CBAR_WHITE, @@ -60,5 +65,8 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp); void hibmc_dp_hpd_cfg(struct hibmc_dp *dp); void hibmc_dp_enable_int(struct hibmc_dp *dp); void hibmc_dp_disable_int(struct hibmc_dp *dp); +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status); +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp); +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp); #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h index 394b1e933c3ae..64306abcd9866 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h @@ -24,6 +24,9 @@ #define HIBMC_DP_CFG_AUX_READY_DATA_BYTE GENMASK(16, 12) #define HIBMC_DP_CFG_AUX GENMASK(24, 17) +#define HIBMC_DP_HPD_STATUS 0x98 +#define HIBMC_DP_HPD_CUR_STATE GENMASK(7, 4) + #define HIBMC_DP_PHYIF_CTRL0 0xa0 #define HIBMC_DP_CFG_SCRAMBLE_EN BIT(0) #define HIBMC_DP_CFG_PAT_SEL GENMASK(7, 4) diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c index d06832e62e966..616821e3c933b 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c @@ -12,6 +12,8 @@ #include "hibmc_drm_drv.h" #include "dp/dp_hw.h" +#include "dp/dp_comm.h" +#include "dp/dp_config.h" #define DP_MASKED_SINK_HPD_PLUG_INT BIT(2) @@ -31,17 +33,76 @@ static int hibmc_dp_connector_get_modes(struct drm_connector *connector) return count; } +static bool hibmc_dp_get_dpcd(struct hibmc_dp_dev *dp_dev) +{ + int ret; + + ret = drm_dp_read_dpcd_caps(dp_dev->aux, dp_dev->dpcd); + if (ret) + return false; + + dp_dev->is_branch = drm_dp_is_branch(dp_dev->dpcd); + + ret = drm_dp_read_desc(dp_dev->aux, &dp_dev->desc, dp_dev->is_branch); + if (ret) + return false; + + ret = drm_dp_read_downstream_info(dp_dev->aux, dp_dev->dpcd, dp_dev->downstream_ports); + if (ret) + return false; + + return true; +} + static int hibmc_dp_detect(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx, bool force) { - mdelay(200); + struct hibmc_dp *dp = to_hibmc_dp(connector); + struct hibmc_dp_dev *dp_dev = dp->dp_dev; + int ret; + + if (dp->irq_status) { + if (dp_dev->hpd_status != HIBMC_HPD_IN) + return connector_status_disconnected; + } + + if (!hibmc_dp_get_dpcd(dp_dev)) + return connector_status_disconnected; + + if (!dp_dev->is_branch) + return connector_status_connected; + + if (drm_dp_read_sink_count_cap(connector, dp_dev->dpcd, &dp_dev->desc) && + dp_dev->downstream_ports[0] & DP_DS_PORT_HPD) { + ret = drm_dp_read_sink_count(dp_dev->aux); + if (ret > 0) + return connector_status_connected; + } + + return connector_status_disconnected; +} + +static int hibmc_dp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode, + struct drm_modeset_acquire_ctx *ctx, + enum drm_mode_status *status) +{ + struct hibmc_dp *dp = to_hibmc_dp(connector); + u64 cur_val, max_val; - return drm_connector_helper_detect_from_ddc(connector, ctx, force); + /* check DP link BW */ + cur_val = (u64)mode->clock * HIBMC_DP_BPP; + max_val = (u64)hibmc_dp_get_link_rate(dp) * DP_MODE_VALI_CAL * hibmc_dp_get_lanes(dp); + + *status = cur_val > max_val ? MODE_CLOCK_HIGH : MODE_OK; + + return 0; } static const struct drm_connector_helper_funcs hibmc_dp_conn_helper_funcs = { .get_modes = hibmc_dp_connector_get_modes, .detect_ctx = hibmc_dp_detect, + .mode_valid_ctx = hibmc_dp_mode_valid, }; static int hibmc_dp_late_register(struct drm_connector *connector) @@ -115,7 +176,7 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) { struct drm_device *dev = (struct drm_device *)arg; struct hibmc_drm_private *priv = to_hibmc_drm_private(dev); - int idx; + int idx, exp_status; if (!drm_dev_enter(dev, &idx)) return -ENODEV; @@ -123,12 +184,14 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) if (priv->dp.irq_status & DP_MASKED_SINK_HPD_PLUG_INT) { drm_dbg_dp(&priv->dev, "HPD IN isr occur!\n"); hibmc_dp_hpd_cfg(&priv->dp); + exp_status = HIBMC_HPD_IN; } else { drm_dbg_dp(&priv->dev, "HPD OUT isr occur!\n"); hibmc_dp_reset_link(&priv->dp); + exp_status = HIBMC_HPD_OUT; } - if (dev->registered) + if (hibmc_dp_check_hpd_status(&priv->dp, exp_status)) drm_connector_helper_hpd_irq_event(&priv->dp.connector); drm_dev_exit(idx); diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c index aa159f9ce12fd..b4929078b37f9 100644 --- a/drivers/gpu/drm/i915/display/g4x_dp.c +++ b/drivers/gpu/drm/i915/display/g4x_dp.c @@ -137,7 +137,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder, intel_dp->DP |= DP_SYNC_VS_HIGH; intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; - if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) + if (pipe_config->enhanced_framing) intel_dp->DP |= DP_ENHANCED_FRAMING; intel_dp->DP |= DP_PIPE_SEL_IVB(crtc->pipe); diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index 37faa8f19f6e4..951f30a641af8 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -890,7 +890,7 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, * non-compressed link speeds, and simplifies down to the ratio between * compressed and non-compressed bpp. */ - if (crtc_state->dsc.compression_enable) { + if (is_vid_mode(intel_dsi) && crtc_state->dsc.compression_enable) { mul = fxp_q4_to_int(crtc_state->dsc.compressed_bpp_x16); div = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); } @@ -1506,7 +1506,7 @@ static void gen11_dsi_get_timings(struct intel_encoder *encoder, struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; - if (pipe_config->dsc.compressed_bpp_x16) { + if (is_vid_mode(intel_dsi) && pipe_config->dsc.compressed_bpp_x16) { int div = fxp_q4_to_int(pipe_config->dsc.compressed_bpp_x16); int mul = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); diff --git a/drivers/gpu/drm/i915/display/intel_acpi.c b/drivers/gpu/drm/i915/display/intel_acpi.c index 1addd62882413..1e8b9d1756988 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.c +++ b/drivers/gpu/drm/i915/display/intel_acpi.c @@ -96,6 +96,7 @@ static void intel_dsm_platform_mux_info(acpi_handle dhandle) if (!pkg->package.count) { DRM_DEBUG_DRIVER("no connection in _DSM\n"); + ACPI_FREE(pkg); return; } diff --git a/drivers/gpu/drm/i915/display/intel_alpm.c b/drivers/gpu/drm/i915/display/intel_alpm.c index ed7a7ed486b55..2aed3861d8edd 100644 --- a/drivers/gpu/drm/i915/display/intel_alpm.c +++ b/drivers/gpu/drm/i915/display/intel_alpm.c @@ -577,12 +577,7 @@ void intel_alpm_disable(struct intel_dp *intel_dp) mutex_lock(&intel_dp->alpm_parameters.lock); intel_de_rmw(display, ALPM_CTL(display, cpu_transcoder), - ALPM_CTL_ALPM_ENABLE | ALPM_CTL_LOBF_ENABLE | - ALPM_CTL_ALPM_AUX_LESS_ENABLE, 0); - - intel_de_rmw(display, - PORT_ALPM_CTL(cpu_transcoder), - PORT_ALPM_CTL_ALPM_AUX_LESS_ENABLE, 0); + ALPM_CTL_ALPM_ENABLE | ALPM_CTL_LOBF_ENABLE, 0); drm_dbg_kms(display->drm, "Disabling ALPM\n"); mutex_unlock(&intel_dp->alpm_parameters.lock); diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 0d527cf228666..3f40028e79cfc 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -1642,8 +1642,6 @@ static void hsw_configure_cpu_transcoder(const struct intel_crtc_state *crtc_sta } intel_set_transcoder_timings(crtc_state); - if (HAS_VRR(display)) - intel_vrr_set_transcoder_timings(crtc_state); if (cpu_transcoder != TRANSCODER_EDP) intel_de_write(display, TRANS_MULT(display, cpu_transcoder), @@ -4570,6 +4568,7 @@ intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct intel_crtc_state *saved_state; + int err; saved_state = intel_crtc_state_alloc(crtc); if (!saved_state) @@ -4578,7 +4577,12 @@ intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, /* free the old crtc_state->hw members */ intel_crtc_free_hw_state(crtc_state); - intel_dp_tunnel_atomic_clear_stream_bw(state, crtc_state); + err = intel_dp_tunnel_atomic_clear_stream_bw(state, crtc_state); + if (err) { + kfree(saved_state); + + return err; + } /* FIXME: before the switch to atomic started, a new pipe_config was * kzalloc'd. Code that depends on any field being zero should be diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c index 5e88b930f5aae..5b7152cf324e8 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power_well.c +++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c @@ -796,7 +796,7 @@ void gen9_set_dc_state(struct intel_display *display, u32 state) power_domains->dc_state, val & mask); enable_dc6 = state & DC_STATE_EN_UPTO_DC6; - dc6_was_enabled = val & DC_STATE_EN_UPTO_DC6; + dc6_was_enabled = power_domains->dc_state & DC_STATE_EN_UPTO_DC6; if (!dc6_was_enabled && enable_dc6) intel_dmc_update_dc6_allowed_count(display, true); diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 358ab922d7a76..39dd7389f1a71 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -1127,6 +1127,7 @@ struct intel_crtc_state { u32 dc3co_exitline; u16 su_y_granularity; u8 active_non_psr_pipes; + u8 entry_setup_frames; /* * Frequency the dpll for the port should run at. Differs from the diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c index e1455fd7277fd..9b6b90be02fef 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.c +++ b/drivers/gpu/drm/i915/display/intel_dmc.c @@ -1569,8 +1569,7 @@ static bool intel_dmc_get_dc6_allowed_count(struct intel_display *display, u32 * return false; mutex_lock(&power_domains->lock); - dc6_enabled = intel_de_read(display, DC_STATE_EN) & - DC_STATE_EN_UPTO_DC6; + dc6_enabled = power_domains->dc_state & DC_STATE_EN_UPTO_DC6; if (dc6_enabled) intel_dmc_update_dc6_allowed_count(display, false); diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 2eab591a8ef56..be3d54729a440 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -2523,16 +2523,30 @@ intel_dp_compute_config_link_bpp_limits(struct intel_dp *intel_dp, return true; } -static void -intel_dp_dsc_compute_pipe_bpp_limits(struct intel_dp *intel_dp, +static bool +intel_dp_dsc_compute_pipe_bpp_limits(struct intel_connector *connector, struct link_config_limits *limits) { - struct intel_display *display = to_intel_display(intel_dp); + struct intel_display *display = to_intel_display(connector); + const struct link_config_limits orig_limits = *limits; int dsc_min_bpc = intel_dp_dsc_min_src_input_bpc(); int dsc_max_bpc = intel_dp_dsc_max_src_input_bpc(display); - limits->pipe.max_bpp = clamp(limits->pipe.max_bpp, dsc_min_bpc * 3, dsc_max_bpc * 3); - limits->pipe.min_bpp = clamp(limits->pipe.min_bpp, dsc_min_bpc * 3, dsc_max_bpc * 3); + limits->pipe.min_bpp = max(limits->pipe.min_bpp, dsc_min_bpc * 3); + limits->pipe.max_bpp = min(limits->pipe.max_bpp, dsc_max_bpc * 3); + + if (limits->pipe.min_bpp <= 0 || + limits->pipe.min_bpp > limits->pipe.max_bpp) { + drm_dbg_kms(display->drm, + "[CONNECTOR:%d:%s] Invalid DSC src/sink input BPP (src:%d-%d pipe:%d-%d)\n", + connector->base.base.id, connector->base.name, + dsc_min_bpc * 3, dsc_max_bpc * 3, + orig_limits.pipe.min_bpp, orig_limits.pipe.max_bpp); + + return false; + } + + return true; } bool @@ -2543,6 +2557,7 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, bool dsc, struct link_config_limits *limits) { + struct intel_display *display = to_intel_display(intel_dp); bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); struct intel_connector *connector = to_intel_connector(conn_state->connector); @@ -2555,8 +2570,7 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, limits->min_lane_count = intel_dp_min_lane_count(intel_dp); limits->max_lane_count = intel_dp_max_lane_count(intel_dp); - limits->pipe.min_bpp = intel_dp_in_hdr_mode(conn_state) ? 30 : - intel_dp_min_bpp(crtc_state->output_format); + limits->pipe.min_bpp = intel_dp_min_bpp(crtc_state->output_format); if (is_mst) { /* * FIXME: If all the streams can't fit into the link with their @@ -2572,8 +2586,21 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, respect_downstream_limits); } - if (dsc) - intel_dp_dsc_compute_pipe_bpp_limits(intel_dp, limits); + if (!dsc && intel_dp_in_hdr_mode(conn_state)) { + if (intel_dp_supports_dsc(intel_dp, connector, crtc_state) && + limits->pipe.max_bpp >= 30) + limits->pipe.min_bpp = max(limits->pipe.min_bpp, 30); + else + drm_dbg_kms(display->drm, + "[CONNECTOR:%d:%s] Can't force 30 bpp for HDR (pipe bpp: %d-%d DSC-support: %s)\n", + connector->base.base.id, connector->base.name, + limits->pipe.min_bpp, limits->pipe.max_bpp, + str_yes_no(intel_dp_supports_dsc(intel_dp, connector, + crtc_state))); + } + + if (dsc && !intel_dp_dsc_compute_pipe_bpp_limits(connector, limits)) + return false; if (is_mst || intel_dp->use_max_params) { /* @@ -2702,10 +2729,11 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, } drm_dbg_kms(display->drm, - "DP lane count %d clock %d bpp input %d compressed " FXP_Q4_FMT " link rate required %d available %d\n", + "DP lane count %d clock %d bpp input %d compressed " FXP_Q4_FMT " HDR %s link rate required %d available %d\n", pipe_config->lane_count, pipe_config->port_clock, pipe_config->pipe_bpp, FXP_Q4_ARGS(pipe_config->dsc.compressed_bpp_x16), + str_yes_no(intel_dp_in_hdr_mode(conn_state)), intel_dp_config_required_rate(pipe_config), intel_dp_max_link_data_rate(intel_dp, pipe_config->port_clock, diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c index faa2b7a46699d..0d15739eaa05d 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c +++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c @@ -622,19 +622,27 @@ int intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, * * Clear any DP tunnel stream BW requirement set by * intel_dp_tunnel_atomic_compute_stream_bw(). + * + * Returns 0 in case of success, a negative error code otherwise. */ -void intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state) +int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, + struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + int err; if (!crtc_state->dp_tunnel_ref.tunnel) - return; + return 0; + + err = drm_dp_tunnel_atomic_set_stream_bw(&state->base, + crtc_state->dp_tunnel_ref.tunnel, + crtc->pipe, 0); + if (err) + return err; - drm_dp_tunnel_atomic_set_stream_bw(&state->base, - crtc_state->dp_tunnel_ref.tunnel, - crtc->pipe, 0); drm_dp_tunnel_ref_put(&crtc_state->dp_tunnel_ref); + + return 0; } /** diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.h b/drivers/gpu/drm/i915/display/intel_dp_tunnel.h index 7f0f720e8dcad..10ab9eebcef69 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.h +++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.h @@ -40,8 +40,8 @@ int intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, struct intel_dp *intel_dp, const struct intel_connector *connector, struct intel_crtc_state *crtc_state); -void intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state); +int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, + struct intel_crtc_state *crtc_state); int intel_dp_tunnel_atomic_add_state_for_crtc(struct intel_atomic_state *state, struct intel_crtc *crtc); @@ -88,9 +88,12 @@ intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, return 0; } -static inline void +static inline int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state) {} + struct intel_crtc_state *crtc_state) +{ + return 0; +} static inline int intel_dp_tunnel_atomic_add_state_for_crtc(struct intel_atomic_state *state, diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index 7c4709d58aa34..7daf72b69bae4 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -207,6 +207,35 @@ static const struct drm_fb_helper_funcs intel_fb_helper_funcs = { .fb_set_suspend = intelfb_set_suspend, }; +static void intel_fbdev_fill_mode_cmd(struct drm_fb_helper_surface_size *sizes, + struct drm_mode_fb_cmd2 *mode_cmd) +{ + /* we don't do packed 24bpp */ + if (sizes->surface_bpp == 24) + sizes->surface_bpp = 32; + + mode_cmd->width = sizes->surface_width; + mode_cmd->height = sizes->surface_height; + + mode_cmd->pitches[0] = ALIGN(mode_cmd->width * DIV_ROUND_UP(sizes->surface_bpp, 8), 64); + mode_cmd->pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, + sizes->surface_depth); +} + +static struct intel_framebuffer * +__intel_fbdev_fb_alloc(struct intel_display *display, + struct drm_fb_helper_surface_size *sizes) +{ + struct drm_mode_fb_cmd2 mode_cmd = {}; + struct intel_framebuffer *fb; + + intel_fbdev_fill_mode_cmd(sizes, &mode_cmd); + + fb = intel_fbdev_fb_alloc(display->drm, &mode_cmd); + + return fb; +} + int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -234,12 +263,18 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, drm_framebuffer_put(&fb->base); fb = NULL; } + + wakeref = intel_display_rpm_get(display); + if (!fb || drm_WARN_ON(display->drm, !intel_fb_bo(&fb->base))) { drm_dbg_kms(display->drm, "no BIOS fb, allocating a new one\n"); - fb = intel_fbdev_fb_alloc(helper, sizes); - if (IS_ERR(fb)) - return PTR_ERR(fb); + + fb = __intel_fbdev_fb_alloc(display, sizes); + if (IS_ERR(fb)) { + ret = PTR_ERR(fb); + goto out_unlock; + } } else { drm_dbg_kms(display->drm, "re-using BIOS fb\n"); prealloc = true; @@ -247,8 +282,6 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, sizes->fb_height = fb->base.height; } - wakeref = intel_display_rpm_get(display); - /* Pin the GGTT vma for our access via info->screen_base. * This also validates that any existing fb inherited from the * BIOS is suitable for own access. diff --git a/drivers/gpu/drm/i915/display/intel_fbdev_fb.c b/drivers/gpu/drm/i915/display/intel_fbdev_fb.c index 210aee9ae88b8..685612e6afc53 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev_fb.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev_fb.c @@ -3,7 +3,7 @@ * Copyright © 2023 Intel Corporation */ -#include +#include #include "gem/i915_gem_lmem.h" @@ -13,29 +13,16 @@ #include "intel_fb.h" #include "intel_fbdev_fb.h" -struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper, - struct drm_fb_helper_surface_size *sizes) +struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_device *drm, + struct drm_mode_fb_cmd2 *mode_cmd) { - struct intel_display *display = to_intel_display(helper->dev); - struct drm_i915_private *dev_priv = to_i915(display->drm); + struct intel_display *display = to_intel_display(drm); + struct drm_i915_private *dev_priv = to_i915(drm); struct drm_framebuffer *fb; - struct drm_mode_fb_cmd2 mode_cmd = {}; struct drm_i915_gem_object *obj; int size; - /* we don't do packed 24bpp */ - if (sizes->surface_bpp == 24) - sizes->surface_bpp = 32; - - mode_cmd.width = sizes->surface_width; - mode_cmd.height = sizes->surface_height; - - mode_cmd.pitches[0] = ALIGN(mode_cmd.width * - DIV_ROUND_UP(sizes->surface_bpp, 8), 64); - mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, - sizes->surface_depth); - - size = mode_cmd.pitches[0] * mode_cmd.height; + size = mode_cmd->pitches[0] * mode_cmd->height; size = PAGE_ALIGN(size); obj = ERR_PTR(-ENODEV); @@ -58,18 +45,25 @@ struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper, } if (IS_ERR(obj)) { - drm_err(display->drm, "failed to allocate framebuffer (%pe)\n", obj); + drm_err(drm, "failed to allocate framebuffer (%pe)\n", obj); return ERR_PTR(-ENOMEM); } fb = intel_framebuffer_create(intel_bo_to_drm_bo(obj), - drm_get_format_info(display->drm, - mode_cmd.pixel_format, - mode_cmd.modifier[0]), - &mode_cmd); + drm_get_format_info(drm, + mode_cmd->pixel_format, + mode_cmd->modifier[0]), + mode_cmd); + if (IS_ERR(fb)) { + i915_gem_object_put(obj); + goto err; + } + i915_gem_object_put(obj); return to_intel_framebuffer(fb); +err: + return ERR_CAST(fb); } int intel_fbdev_fb_fill_info(struct intel_display *display, struct fb_info *info, diff --git a/drivers/gpu/drm/i915/display/intel_fbdev_fb.h b/drivers/gpu/drm/i915/display/intel_fbdev_fb.h index cb79572727150..83454ffbf79cd 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev_fb.h +++ b/drivers/gpu/drm/i915/display/intel_fbdev_fb.h @@ -6,15 +6,15 @@ #ifndef __INTEL_FBDEV_FB_H__ #define __INTEL_FBDEV_FB_H__ -struct drm_fb_helper; -struct drm_fb_helper_surface_size; +struct drm_device; struct drm_gem_object; +struct drm_mode_fb_cmd2; struct fb_info; struct i915_vma; struct intel_display; -struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper, - struct drm_fb_helper_surface_size *sizes); +struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_device *drm, + struct drm_mode_fb_cmd2 *mode_cmd); int intel_fbdev_fb_fill_info(struct intel_display *display, struct fb_info *info, struct drm_gem_object *obj, struct i915_vma *vma); diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index 358210adb8f84..10b3f3969c960 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -497,8 +497,10 @@ gmbus_xfer_read_chunk(struct intel_display *display, val = intel_de_read_fw(display, GMBUS3(display)); do { - if (extra_byte_added && len == 1) + if (extra_byte_added && len == 1) { + len--; break; + } *buf++ = val & 0xff; val >>= 8; diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c index 2329f09d413d0..91deeaec61100 100644 --- a/drivers/gpu/drm/i915/display/intel_plane.c +++ b/drivers/gpu/drm/i915/display/intel_plane.c @@ -421,11 +421,16 @@ void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, drm_framebuffer_get(plane_state->hw.fb); } +static void unlink_nv12_plane(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state); + void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + unlink_nv12_plane(crtc_state, plane_state); + crtc_state->active_planes &= ~BIT(plane->id); crtc_state->scaled_planes &= ~BIT(plane->id); crtc_state->nv12_planes &= ~BIT(plane->id); @@ -1498,6 +1503,9 @@ static void unlink_nv12_plane(struct intel_crtc_state *crtc_state, struct intel_display *display = to_intel_display(plane_state); struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + if (!plane_state->planar_linked_plane) + return; + plane_state->planar_linked_plane = NULL; if (!plane_state->is_y_plane) @@ -1535,8 +1543,7 @@ static int icl_check_nv12_planes(struct intel_atomic_state *state, if (plane->pipe != crtc->pipe) continue; - if (plane_state->planar_linked_plane) - unlink_nv12_plane(crtc_state, plane_state); + unlink_nv12_plane(crtc_state, plane_state); } if (!crtc_state->nv12_planes) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 6d9c95e5c0255..504f6228bf35d 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -50,6 +50,7 @@ #include "intel_snps_phy.h" #include "intel_step.h" #include "intel_vblank.h" +#include "intel_vdsc.h" #include "intel_vrr.h" #include "skl_universal_plane.h" @@ -813,7 +814,12 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp, void intel_psr_panel_replay_enable_sink(struct intel_dp *intel_dp) { - if (CAN_PANEL_REPLAY(intel_dp)) + /* + * NOTE: We might want to trigger mode set when + * disabling/enabling Panel Replay via debugfs interface to + * ensure this bit is cleared/set accordingly. + */ + if (CAN_PANEL_REPLAY(intel_dp) && panel_replay_global_enabled(intel_dp)) drm_dp_dpcd_writeb(&intel_dp->aux, PANEL_REPLAY_CONFIG, DP_PANEL_REPLAY_ENABLE); } @@ -1579,7 +1585,7 @@ static bool _psr_compute_config(struct intel_dp *intel_dp, entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, adjusted_mode); if (entry_setup_frames >= 0) { - intel_dp->psr.entry_setup_frames = entry_setup_frames; + crtc_state->entry_setup_frames = entry_setup_frames; } else { drm_dbg_kms(display->drm, "PSR condition failed: PSR setup timing not met\n"); @@ -1651,7 +1657,7 @@ static bool intel_psr_needs_wa_18037818876(struct intel_dp *intel_dp, { struct intel_display *display = to_intel_display(intel_dp); - return (DISPLAY_VER(display) == 20 && intel_dp->psr.entry_setup_frames > 0 && + return (DISPLAY_VER(display) == 20 && crtc_state->entry_setup_frames > 0 && !crtc_state->has_sel_update); } @@ -2021,6 +2027,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, crtc_state->req_psr2_sdp_prior_scanline; intel_dp->psr.active_non_psr_pipes = crtc_state->active_non_psr_pipes; intel_dp->psr.pkg_c_latency_used = crtc_state->pkg_c_latency_used; + intel_dp->psr.entry_setup_frames = crtc_state->entry_setup_frames; if (!psr_interrupt_error_check(intel_dp)) return; @@ -2484,6 +2491,12 @@ void intel_psr2_program_trans_man_trk_ctl(struct intel_dsb *dsb, intel_de_write_dsb(display, dsb, PIPE_SRCSZ_ERLY_TPT(crtc->pipe), crtc_state->pipe_srcsz_early_tpt); + + if (!crtc_state->dsc.compression_enable) + return; + + intel_dsc_su_et_parameters_configure(dsb, encoder, crtc_state, + drm_rect_height(&crtc_state->psr2_su_area)); } static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state, @@ -2536,9 +2549,9 @@ static u32 psr2_pipe_srcsz_early_tpt_calc(struct intel_crtc_state *crtc_state, static void clip_area_update(struct drm_rect *overlap_damage_area, struct drm_rect *damage_area, - struct drm_rect *pipe_src) + struct drm_rect *display_area) { - if (!drm_rect_intersect(damage_area, pipe_src)) + if (!drm_rect_intersect(damage_area, display_area)) return; if (overlap_damage_area->y1 == -1) { @@ -2554,11 +2567,12 @@ static void clip_area_update(struct drm_rect *overlap_damage_area, overlap_damage_area->y2 = damage_area->y2; } -static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_state) +static bool intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_state) { struct intel_display *display = to_intel_display(crtc_state); const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; u16 y_alignment; + bool su_area_changed = false; /* ADLP aligns the SU region to vdsc slice height in case dsc is enabled */ if (crtc_state->dsc.compression_enable && @@ -2567,10 +2581,18 @@ static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_st else y_alignment = crtc_state->su_y_granularity; - crtc_state->psr2_su_area.y1 -= crtc_state->psr2_su_area.y1 % y_alignment; - if (crtc_state->psr2_su_area.y2 % y_alignment) + if (crtc_state->psr2_su_area.y1 % y_alignment) { + crtc_state->psr2_su_area.y1 -= crtc_state->psr2_su_area.y1 % y_alignment; + su_area_changed = true; + } + + if (crtc_state->psr2_su_area.y2 % y_alignment) { crtc_state->psr2_su_area.y2 = ((crtc_state->psr2_su_area.y2 / y_alignment) + 1) * y_alignment; + su_area_changed = true; + } + + return su_area_changed; } /* @@ -2580,6 +2602,7 @@ static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_st static void intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state, struct intel_crtc *crtc, + struct drm_rect *display_area, bool *cursor_in_su_area) { struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); @@ -2607,7 +2630,7 @@ intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state, continue; clip_area_update(&crtc_state->psr2_su_area, &new_plane_state->uapi.dst, - &crtc_state->pipe_src); + display_area); *cursor_in_su_area = true; } } @@ -2703,7 +2726,13 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct intel_plane_state *new_plane_state, *old_plane_state; struct intel_plane *plane; - bool full_update = false, cursor_in_su_area = false; + struct drm_rect display_area = { + .x1 = 0, + .y1 = 0, + .x2 = crtc_state->hw.adjusted_mode.crtc_hdisplay, + .y2 = crtc_state->hw.adjusted_mode.crtc_vdisplay, + }; + bool full_update = false, su_area_changed; int i, ret; if (!crtc_state->enable_psr2_sel_fetch) @@ -2716,7 +2745,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, crtc_state->psr2_su_area.x1 = 0; crtc_state->psr2_su_area.y1 = -1; - crtc_state->psr2_su_area.x2 = drm_rect_width(&crtc_state->pipe_src); + crtc_state->psr2_su_area.x2 = drm_rect_width(&display_area); crtc_state->psr2_su_area.y2 = -1; /* @@ -2754,14 +2783,14 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, damaged_area.y1 = old_plane_state->uapi.dst.y1; damaged_area.y2 = old_plane_state->uapi.dst.y2; clip_area_update(&crtc_state->psr2_su_area, &damaged_area, - &crtc_state->pipe_src); + &display_area); } if (new_plane_state->uapi.visible) { damaged_area.y1 = new_plane_state->uapi.dst.y1; damaged_area.y2 = new_plane_state->uapi.dst.y2; clip_area_update(&crtc_state->psr2_su_area, &damaged_area, - &crtc_state->pipe_src); + &display_area); } continue; } else if (new_plane_state->uapi.alpha != old_plane_state->uapi.alpha) { @@ -2769,7 +2798,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, damaged_area.y1 = new_plane_state->uapi.dst.y1; damaged_area.y2 = new_plane_state->uapi.dst.y2; clip_area_update(&crtc_state->psr2_su_area, &damaged_area, - &crtc_state->pipe_src); + &display_area); continue; } @@ -2785,7 +2814,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, damaged_area.x1 += new_plane_state->uapi.dst.x1 - src.x1; damaged_area.x2 += new_plane_state->uapi.dst.x1 - src.x1; - clip_area_update(&crtc_state->psr2_su_area, &damaged_area, &crtc_state->pipe_src); + clip_area_update(&crtc_state->psr2_su_area, &damaged_area, &display_area); } /* @@ -2810,15 +2839,33 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, if (ret) return ret; - /* - * Adjust su area to cover cursor fully as necessary (early - * transport). This needs to be done after - * drm_atomic_add_affected_planes to ensure visible cursor is added into - * affected planes even when cursor is not updated by itself. - */ - intel_psr2_sel_fetch_et_alignment(state, crtc, &cursor_in_su_area); + do { + bool cursor_in_su_area; + + /* + * Adjust su area to cover cursor fully as necessary + * (early transport). This needs to be done after + * drm_atomic_add_affected_planes to ensure visible + * cursor is added into affected planes even when + * cursor is not updated by itself. + */ + intel_psr2_sel_fetch_et_alignment(state, crtc, &display_area, + &cursor_in_su_area); + + su_area_changed = intel_psr2_sel_fetch_pipe_alignment(crtc_state); - intel_psr2_sel_fetch_pipe_alignment(crtc_state); + /* + * If the cursor was outside the SU area before + * alignment, the alignment step (which only expands + * SU) may pull the cursor partially inside, so we + * must run ET alignment again to fully cover it. But + * if the cursor was already fully inside before + * alignment, expanding the SU area won't change that, + * so no further work is needed. + */ + if (cursor_in_su_area) + break; + } while (su_area_changed); /* * Now that we have the pipe damaged area check if it intersect with @@ -2878,6 +2925,10 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, } skip_sel_fetch_set_loop: + if (full_update) + clip_area_update(&crtc_state->psr2_su_area, &display_area, + &display_area); + psr2_man_trk_ctl_calc(crtc_state, full_update); crtc_state->pipe_srcsz_early_tpt = psr2_pipe_srcsz_early_tpt_calc(crtc_state, full_update); diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index d2e16b79d6be1..1abbdd426e587 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -239,7 +239,7 @@ static struct intel_quirk intel_quirks[] = { { 0x0f31, 0x103c, 0x220f, quirk_invert_brightness }, /* Dell XPS 13 7390 2-in-1 */ - { 0x8a12, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, + { 0x8a52, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, }; static const struct intel_dpcd_quirk intel_dpcd_quirks[] = { diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index 8e799e225af17..4e701ea0bb233 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -751,6 +751,29 @@ void intel_dsc_dp_pps_write(struct intel_encoder *encoder, sizeof(dp_dsc_pps_sdp)); } +void intel_dsc_su_et_parameters_configure(struct intel_dsb *dsb, struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, int su_lines) +{ + struct intel_display *display = to_intel_display(crtc_state); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; + enum pipe pipe = crtc->pipe; + int vdsc_instances_per_pipe = intel_dsc_get_vdsc_per_pipe(crtc_state); + int slice_row_per_frame = su_lines / vdsc_cfg->slice_height; + u32 val; + + drm_WARN_ON_ONCE(display->drm, su_lines % vdsc_cfg->slice_height); + drm_WARN_ON_ONCE(display->drm, vdsc_instances_per_pipe > 2); + + val = DSC_SUPS0_SU_SLICE_ROW_PER_FRAME(slice_row_per_frame); + val |= DSC_SUPS0_SU_PIC_HEIGHT(su_lines); + + intel_de_write_dsb(display, dsb, LNL_DSC0_SU_PARAMETER_SET_0(pipe), val); + + if (vdsc_instances_per_pipe == 2) + intel_de_write_dsb(display, dsb, LNL_DSC1_SU_PARAMETER_SET_0(pipe), val); +} + static i915_reg_t dss_ctl1_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder) { return is_pipe_dsc(crtc, cpu_transcoder) ? diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.h b/drivers/gpu/drm/i915/display/intel_vdsc.h index 9e2812f99dd74..b77d30b61a32f 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.h +++ b/drivers/gpu/drm/i915/display/intel_vdsc.h @@ -13,6 +13,7 @@ struct drm_printer; enum transcoder; struct intel_crtc; struct intel_crtc_state; +struct intel_dsb; struct intel_encoder; bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state); @@ -29,6 +30,8 @@ void intel_dsc_dsi_pps_write(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state); void intel_dsc_dp_pps_write(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state); +void intel_dsc_su_et_parameters_configure(struct intel_dsb *dsb, struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, int su_lines); void intel_vdsc_state_dump(struct drm_printer *p, int indent, const struct intel_crtc_state *crtc_state); int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state); diff --git a/drivers/gpu/drm/i915/display/intel_vdsc_regs.h b/drivers/gpu/drm/i915/display/intel_vdsc_regs.h index 2d478a84b07c4..2b2e3c1b8138a 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc_regs.h +++ b/drivers/gpu/drm/i915/display/intel_vdsc_regs.h @@ -196,6 +196,18 @@ #define DSC_PPS18_NSL_BPG_OFFSET(offset) REG_FIELD_PREP(DSC_PPS18_NSL_BPG_OFFSET_MASK, offset) #define DSC_PPS18_SL_OFFSET_ADJ(offset) REG_FIELD_PREP(DSC_PPS18_SL_OFFSET_ADJ_MASK, offset) +#define _LNL_DSC0_SU_PARAMETER_SET_0_PA 0x78064 +#define _LNL_DSC1_SU_PARAMETER_SET_0_PA 0x78164 +#define _LNL_DSC0_SU_PARAMETER_SET_0_PB 0x78264 +#define _LNL_DSC1_SU_PARAMETER_SET_0_PB 0x78364 +#define LNL_DSC0_SU_PARAMETER_SET_0(pipe) _MMIO_PIPE((pipe), _LNL_DSC0_SU_PARAMETER_SET_0_PA, _LNL_DSC0_SU_PARAMETER_SET_0_PB) +#define LNL_DSC1_SU_PARAMETER_SET_0(pipe) _MMIO_PIPE((pipe), _LNL_DSC1_SU_PARAMETER_SET_0_PA, _LNL_DSC1_SU_PARAMETER_SET_0_PB) + +#define DSC_SUPS0_SU_SLICE_ROW_PER_FRAME_MASK REG_GENMASK(31, 20) +#define DSC_SUPS0_SU_SLICE_ROW_PER_FRAME(rows) REG_FIELD_PREP(DSC_SUPS0_SU_SLICE_ROW_PER_FRAME_MASK, (rows)) +#define DSC_SUPS0_SU_PIC_HEIGHT_MASK REG_GENMASK(15, 0) +#define DSC_SUPS0_SU_PIC_HEIGHT(h) REG_FIELD_PREP(DSC_SUPS0_SU_PIC_HEIGHT_MASK, (h)) + /* Icelake Rate Control Buffer Threshold Registers */ #define DSCA_RC_BUF_THRESH_0 _MMIO(0x6B230) #define DSCA_RC_BUF_THRESH_0_UDW _MMIO(0x6B230 + 4) diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c index 3eed37f271b02..04a2ea086fb2e 100644 --- a/drivers/gpu/drm/i915/display/intel_vrr.c +++ b/drivers/gpu/drm/i915/display/intel_vrr.c @@ -461,6 +461,21 @@ void intel_vrr_set_transcoder_timings(const struct intel_crtc_state *crtc_state) struct intel_display *display = to_intel_display(crtc_state); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + if (!HAS_VRR(display)) + return; + + /* + * Bspec says: + * "(note: VRR needs to be programmed after + * TRANS_DDI_FUNC_CTL and before TRANS_CONF)." + * + * In practice it turns out that ICL can hang if + * TRANS_VRR_VMAX/FLIPLINE are written before + * enabling TRANS_DDI_FUNC_CTL. + */ + drm_WARN_ON(display->drm, + !(intel_de_read(display, TRANS_DDI_FUNC_CTL(display, cpu_transcoder)) & TRANS_DDI_FUNC_ENABLE)); + /* * This bit seems to have two meanings depending on the platform: * TGL: generate VRR "safe window" for DSB vblank waits @@ -655,6 +670,8 @@ void intel_vrr_transcoder_enable(const struct intel_crtc_state *crtc_state) if (!HAS_VRR(display)) return; + intel_vrr_set_transcoder_timings(crtc_state); + if (!intel_vrr_possible(crtc_state)) return; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 39c7c32e1e74e..bcae382acdf56 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -950,13 +950,13 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) vma = eb_lookup_vma(eb, eb->exec[i].handle); if (IS_ERR(vma)) { err = PTR_ERR(vma); - goto err; + return err; } err = eb_validate_vma(eb, &eb->exec[i], vma); if (unlikely(err)) { i915_vma_put(vma); - goto err; + return err; } err = eb_add_vma(eb, ¤t_batch, i, vma); @@ -965,19 +965,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) if (i915_gem_object_is_userptr(vma->obj)) { err = i915_gem_object_userptr_submit_init(vma->obj); - if (err) { - if (i + 1 < eb->buffer_count) { - /* - * Execbuffer code expects last vma entry to be NULL, - * since we already initialized this entry, - * set the next value to NULL or we mess up - * cleanup handling. - */ - eb->vma[i + 1].vma = NULL; - } - + if (err) return err; - } eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT; eb->args->flags |= __EXEC_USERPTR_USED; @@ -985,10 +974,6 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) } return 0; - -err: - eb->vma[i].vma = NULL; - return err; } static int eb_lock_vmas(struct i915_execbuffer *eb) @@ -3374,7 +3359,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, eb.exec = exec; eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); - eb.vma[0].vma = NULL; + memset(eb.vma, 0, (args->buffer_count + 1) * sizeof(struct eb_vma)); + eb.batch_pool = NULL; eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; @@ -3583,7 +3569,18 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, if (err) return err; - /* Allocate extra slots for use by the command parser */ + /* + * Allocate extra slots for use by the command parser. + * + * Note that this allocation handles two different arrays (the + * exec2_list array, and the eventual eb.vma array introduced in + * i915_gem_do_execbuffer()), that reside in virtually contiguous + * memory. Also note that the allocation intentionally doesn't fill the + * area with zeros, because the exec2_list part doesn't need to be, as + * it's immediately overwritten by user data a few lines below. + * However, the eb.vma part is explicitly zeroed later in + * i915_gem_do_execbuffer(). + */ exec2_list = kvmalloc_array(count + 2, eb_element_size(), __GFP_NOWARN | GFP_KERNEL); if (exec2_list == NULL) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index b9dae15c1d166..1944db5082114 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -151,8 +151,12 @@ int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, } } while (1); - nr_pages = min_t(unsigned long, - folio_nr_pages(folio), page_count - i); + nr_pages = min_array(((unsigned long[]) { + folio_nr_pages(folio), + page_count - i, + max_segment / PAGE_SIZE, + }), 3); + if (!i || sg->length >= max_segment || folio_pfn(folio) != next_pfn) { @@ -162,7 +166,9 @@ int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, st->nents++; sg_set_folio(sg, folio, nr_pages * PAGE_SIZE, 0); } else { - /* XXX: could overflow? */ + nr_pages = min_t(unsigned long, nr_pages, + (max_segment - sg->length) / PAGE_SIZE); + sg->length += nr_pages * PAGE_SIZE; } next_pfn = folio_pfn(folio) + nr_pages; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index b721bbd233567..ce8cdd517daa1 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -1969,7 +1969,8 @@ void intel_engines_reset_default_submission(struct intel_gt *gt) if (engine->sanitize) engine->sanitize(engine); - engine->set_default_submission(engine); + if (engine->set_default_submission) + engine->set_default_submission(engine); } } diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c index 8d4bb95f84248..cad81592c8481 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c @@ -145,10 +145,12 @@ static void heartbeat(struct work_struct *wrk) /* Just in case everything has gone horribly wrong, give it a kick */ intel_engine_flush_submission(engine); - rq = engine->heartbeat.systole; - if (rq && i915_request_completed(rq)) { - i915_request_put(rq); - engine->heartbeat.systole = NULL; + rq = xchg(&engine->heartbeat.systole, NULL); + if (rq) { + if (i915_request_completed(rq)) + i915_request_put(rq); + else + engine->heartbeat.systole = rq; } if (!intel_engine_pm_get_if_awake(engine)) @@ -229,8 +231,11 @@ static void heartbeat(struct work_struct *wrk) unlock: mutex_unlock(&ce->timeline->mutex); out: - if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine)) - i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); + if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine)) { + rq = xchg(&engine->heartbeat.systole, NULL); + if (rq) + i915_request_put(rq); + } intel_engine_pm_put(engine); } @@ -244,8 +249,13 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine) void intel_engine_park_heartbeat(struct intel_engine_cs *engine) { - if (cancel_delayed_work(&engine->heartbeat.work)) - i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); + if (cancel_delayed_work(&engine->heartbeat.work)) { + struct i915_request *rq; + + rq = xchg(&engine->heartbeat.systole, NULL); + if (rq) + i915_request_put(rq); + } } void intel_gt_unpark_heartbeats(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index e14a0c3db999b..f07eeef0f6ec7 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -519,7 +519,7 @@ ggtt_write(struct io_mapping *mapping, /* We can use the cpu mem copy function because this is X86. */ vaddr = io_mapping_map_atomic_wc(mapping, base); - unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, + unwritten = copy_from_user_inatomic_nontemporal((void __force *)vaddr + offset, user_data, length); io_mapping_unmap_atomic(vaddr); if (unwritten) { diff --git a/drivers/gpu/drm/i915/i915_wait_util.h b/drivers/gpu/drm/i915/i915_wait_util.h index 7376898e3bf83..e1ed7921ec701 100644 --- a/drivers/gpu/drm/i915/i915_wait_util.h +++ b/drivers/gpu/drm/i915/i915_wait_util.h @@ -25,9 +25,9 @@ might_sleep(); \ for (;;) { \ const bool expired__ = ktime_after(ktime_get_raw(), end__); \ - OP; \ /* Guarantee COND check prior to timeout */ \ barrier(); \ + OP; \ if (COND) { \ ret__ = 0; \ break; \ diff --git a/drivers/gpu/drm/i915/intel_memory_region.h b/drivers/gpu/drm/i915/intel_memory_region.h index b3b75be9ced5d..e9a4e6090fe0a 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.h +++ b/drivers/gpu/drm/i915/intel_memory_region.h @@ -72,7 +72,7 @@ struct intel_memory_region { u16 instance; enum intel_region_id id; char name[16]; - char uabi_name[16]; + char uabi_name[20]; bool private; /* not for userspace */ struct { diff --git a/drivers/gpu/drm/i915/intel_wakeref.c b/drivers/gpu/drm/i915/intel_wakeref.c index 7fa194de5d35b..5bd75a35d0a39 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.c +++ b/drivers/gpu/drm/i915/intel_wakeref.c @@ -78,7 +78,7 @@ void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) /* Assume we are not in process context and so cannot sleep. */ if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { mod_delayed_work(wf->i915->unordered_wq, &wf->work, - FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); + FIELD_GET(INTEL_WAKEREF_PUT_DELAY_MASK, flags)); return; } diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h index a2894a56e18fc..81308bac34bab 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.h +++ b/drivers/gpu/drm/i915/intel_wakeref.h @@ -128,17 +128,16 @@ intel_wakeref_get_if_active(struct intel_wakeref *wf) return atomic_inc_not_zero(&wf->count); } -enum { - INTEL_WAKEREF_PUT_ASYNC_BIT = 0, - __INTEL_WAKEREF_PUT_LAST_BIT__ -}; - static inline void intel_wakeref_might_get(struct intel_wakeref *wf) { might_lock(&wf->mutex); } +/* flags for __intel_wakeref_put() and __intel_wakeref_put_last */ +#define INTEL_WAKEREF_PUT_ASYNC BIT(0) +#define INTEL_WAKEREF_PUT_DELAY_MASK GENMASK(BITS_PER_LONG - 1, 1) + /** * __intel_wakeref_put: Release the wakeref * @wf: the wakeref @@ -154,9 +153,6 @@ intel_wakeref_might_get(struct intel_wakeref *wf) */ static inline void __intel_wakeref_put(struct intel_wakeref *wf, unsigned long flags) -#define INTEL_WAKEREF_PUT_ASYNC BIT(INTEL_WAKEREF_PUT_ASYNC_BIT) -#define INTEL_WAKEREF_PUT_DELAY \ - GENMASK(BITS_PER_LONG - 1, __INTEL_WAKEREF_PUT_LAST_BIT__) { INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); if (unlikely(!atomic_add_unless(&wf->count, -1, 1))) @@ -181,7 +177,7 @@ intel_wakeref_put_delay(struct intel_wakeref *wf, unsigned long delay) { __intel_wakeref_put(wf, INTEL_WAKEREF_PUT_ASYNC | - FIELD_PREP(INTEL_WAKEREF_PUT_DELAY, delay)); + FIELD_PREP(INTEL_WAKEREF_PUT_DELAY_MASK, delay)); } static inline void diff --git a/drivers/gpu/drm/imagination/pvr_device.c b/drivers/gpu/drm/imagination/pvr_device.c index 294b6019b4155..b2275cb7360e4 100644 --- a/drivers/gpu/drm/imagination/pvr_device.c +++ b/drivers/gpu/drm/imagination/pvr_device.c @@ -48,7 +48,7 @@ * * Return: * * 0 on success, or - * * Any error returned by devm_platform_ioremap_resource(). + * * Any error returned by devm_platform_get_and_ioremap_resource(). */ static int pvr_device_reg_init(struct pvr_device *pvr_dev) @@ -224,29 +224,12 @@ static irqreturn_t pvr_device_irq_thread_handler(int irq, void *data) } if (pvr_dev->has_safety_events) { - int err; - - /* - * Ensure the GPU is powered on since some safety events (such - * as ECC faults) can happen outside of job submissions, which - * are otherwise the only time a power reference is held. - */ - err = pvr_power_get(pvr_dev); - if (err) { - drm_err_ratelimited(drm_dev, - "%s: could not take power reference (%d)\n", - __func__, err); - return ret; - } - while (pvr_device_safety_irq_pending(pvr_dev)) { pvr_device_safety_irq_clear(pvr_dev); pvr_device_handle_safety_events(pvr_dev); ret = IRQ_HANDLED; } - - pvr_power_put(pvr_dev); } return ret; diff --git a/drivers/gpu/drm/imagination/pvr_fw_trace.c b/drivers/gpu/drm/imagination/pvr_fw_trace.c index a1098b5214856..e7f4554510fd4 100644 --- a/drivers/gpu/drm/imagination/pvr_fw_trace.c +++ b/drivers/gpu/drm/imagination/pvr_fw_trace.c @@ -136,6 +136,7 @@ update_logtype(struct pvr_device *pvr_dev, u32 group_mask) struct rogue_fwif_kccb_cmd cmd; int idx; int err; + int slot; if (group_mask) fw_trace->tracebuf_ctrl->log_type = ROGUE_FWIF_LOG_TYPE_TRACE | group_mask; @@ -153,8 +154,13 @@ update_logtype(struct pvr_device *pvr_dev, u32 group_mask) cmd.cmd_type = ROGUE_FWIF_KCCB_CMD_LOGTYPE_UPDATE; cmd.kccb_flags = 0; - err = pvr_kccb_send_cmd(pvr_dev, &cmd, NULL); + err = pvr_kccb_send_cmd(pvr_dev, &cmd, &slot); + if (err) + goto err_drm_dev_exit; + + err = pvr_kccb_wait_for_completion(pvr_dev, slot, HZ, NULL); +err_drm_dev_exit: drm_dev_exit(idx); err_up_read: diff --git a/drivers/gpu/drm/imagination/pvr_gem.c b/drivers/gpu/drm/imagination/pvr_gem.c index a66cf082af244..c07c9a9151903 100644 --- a/drivers/gpu/drm/imagination/pvr_gem.c +++ b/drivers/gpu/drm/imagination/pvr_gem.c @@ -28,6 +28,16 @@ static void pvr_gem_object_free(struct drm_gem_object *obj) drm_gem_shmem_object_free(obj); } +static struct dma_buf *pvr_gem_export(struct drm_gem_object *obj, int flags) +{ + struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(obj); + + if (pvr_obj->flags & DRM_PVR_BO_PM_FW_PROTECT) + return ERR_PTR(-EPERM); + + return drm_gem_prime_export(obj, flags); +} + static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *vma) { struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(gem_obj); @@ -42,6 +52,7 @@ static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *v static const struct drm_gem_object_funcs pvr_gem_object_funcs = { .free = pvr_gem_object_free, .print_info = drm_gem_shmem_object_print_info, + .export = pvr_gem_export, .pin = drm_gem_shmem_object_pin, .unpin = drm_gem_shmem_object_unpin, .get_sg_table = drm_gem_shmem_object_get_sg_table, diff --git a/drivers/gpu/drm/imagination/pvr_power.c b/drivers/gpu/drm/imagination/pvr_power.c index c6e7ff9e935d3..a7994457900d0 100644 --- a/drivers/gpu/drm/imagination/pvr_power.c +++ b/drivers/gpu/drm/imagination/pvr_power.c @@ -89,7 +89,7 @@ pvr_power_request_pwr_off(struct pvr_device *pvr_dev) } static int -pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset) +pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset, bool rpm_suspend) { if (!hard_reset) { int err; @@ -105,6 +105,11 @@ pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset) return err; } + if (rpm_suspend) { + /* Wait for late processing of GPU or firmware IRQs in other cores */ + synchronize_irq(pvr_dev->irq); + } + return pvr_fw_stop(pvr_dev); } @@ -360,7 +365,7 @@ pvr_power_device_suspend(struct device *dev) return -EIO; if (pvr_dev->fw_dev.booted) { - err = pvr_power_fw_disable(pvr_dev, false); + err = pvr_power_fw_disable(pvr_dev, false, true); if (err) goto err_drm_dev_exit; } @@ -509,7 +514,16 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset) } /* Disable IRQs for the duration of the reset. */ - disable_irq(pvr_dev->irq); + if (hard_reset) { + disable_irq(pvr_dev->irq); + } else { + /* + * Soft reset is triggered as a response to a FW command to the Host and is + * processed from the threaded IRQ handler. This code cannot (nor needs to) + * wait for any IRQ processing to complete. + */ + disable_irq_nosync(pvr_dev->irq); + } do { if (hard_reset) { @@ -517,7 +531,7 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset) queues_disabled = true; } - err = pvr_power_fw_disable(pvr_dev, hard_reset); + err = pvr_power_fw_disable(pvr_dev, hard_reset, false); if (!err) { if (hard_reset) { pvr_dev->fw_dev.booted = false; diff --git a/drivers/gpu/drm/imx/ipuv3/imx-tve.c b/drivers/gpu/drm/imx/ipuv3/imx-tve.c index c5629e155d25a..99c2e20fa3f76 100644 --- a/drivers/gpu/drm/imx/ipuv3/imx-tve.c +++ b/drivers/gpu/drm/imx/ipuv3/imx-tve.c @@ -525,6 +525,13 @@ static const struct component_ops imx_tve_ops = { .bind = imx_tve_bind, }; +static void imx_tve_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + static int imx_tve_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -546,6 +553,12 @@ static int imx_tve_probe(struct platform_device *pdev) if (ddc_node) { tve->ddc = of_find_i2c_adapter_by_node(ddc_node); of_node_put(ddc_node); + if (tve->ddc) { + ret = devm_add_action_or_reset(dev, imx_tve_put_device, + &tve->ddc->dev); + if (ret) + return ret; + } } tve->mode = of_get_tve_mode(np); diff --git a/drivers/gpu/drm/imx/ipuv3/parallel-display.c b/drivers/gpu/drm/imx/ipuv3/parallel-display.c index 7fc6af7033078..d5f2ee41c03fe 100644 --- a/drivers/gpu/drm/imx/ipuv3/parallel-display.c +++ b/drivers/gpu/drm/imx/ipuv3/parallel-display.c @@ -256,7 +256,9 @@ static int imx_pd_probe(struct platform_device *pdev) platform_set_drvdata(pdev, imxpd); - devm_drm_bridge_add(dev, &imxpd->bridge); + ret = devm_drm_bridge_add(dev, &imxpd->bridge); + if (ret) + return ret; return component_add(dev, &imx_pd_ops); } diff --git a/drivers/gpu/drm/logicvc/logicvc_drm.c b/drivers/gpu/drm/logicvc/logicvc_drm.c index 204b0fee55d0b..bbebf4fc7f51a 100644 --- a/drivers/gpu/drm/logicvc/logicvc_drm.c +++ b/drivers/gpu/drm/logicvc/logicvc_drm.c @@ -92,7 +92,6 @@ static int logicvc_drm_config_parse(struct logicvc_drm *logicvc) struct device *dev = drm_dev->dev; struct device_node *of_node = dev->of_node; struct logicvc_drm_config *config = &logicvc->config; - struct device_node *layers_node; int ret; logicvc_of_property_parse_bool(of_node, LOGICVC_OF_PROPERTY_DITHERING, @@ -128,7 +127,8 @@ static int logicvc_drm_config_parse(struct logicvc_drm *logicvc) if (ret) return ret; - layers_node = of_get_child_by_name(of_node, "layers"); + struct device_node *layers_node __free(device_node) = + of_get_child_by_name(of_node, "layers"); if (!layers_node) { drm_err(drm_dev, "Missing non-optional layers node\n"); return -EINVAL; diff --git a/drivers/gpu/drm/mediatek/mtk_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_ddp_comp.c index ac6620e10262e..9672ea1f91a2b 100644 --- a/drivers/gpu/drm/mediatek/mtk_ddp_comp.c +++ b/drivers/gpu/drm/mediatek/mtk_ddp_comp.c @@ -621,15 +621,27 @@ int mtk_find_possible_crtcs(struct drm_device *drm, struct device *dev) return ret; } -int mtk_ddp_comp_init(struct device_node *node, struct mtk_ddp_comp *comp, +static void mtk_ddp_comp_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + +static void mtk_ddp_comp_clk_put(void *_clk) +{ + struct clk *clk = _clk; + + clk_put(clk); +} + +int mtk_ddp_comp_init(struct device *dev, struct device_node *node, struct mtk_ddp_comp *comp, unsigned int comp_id) { struct platform_device *comp_pdev; enum mtk_ddp_comp_type type; struct mtk_ddp_comp_dev *priv; -#if IS_REACHABLE(CONFIG_MTK_CMDQ) int ret; -#endif if (comp_id >= DDP_COMPONENT_DRM_ID_MAX) return -EINVAL; @@ -651,6 +663,10 @@ int mtk_ddp_comp_init(struct device_node *node, struct mtk_ddp_comp *comp, } comp->dev = &comp_pdev->dev; + ret = devm_add_action_or_reset(dev, mtk_ddp_comp_put_device, comp->dev); + if (ret) + return ret; + if (type == MTK_DISP_AAL || type == MTK_DISP_BLS || type == MTK_DISP_CCORR || @@ -666,15 +682,22 @@ int mtk_ddp_comp_init(struct device_node *node, struct mtk_ddp_comp *comp, type == MTK_DSI) return 0; - priv = devm_kzalloc(comp->dev, sizeof(*priv), GFP_KERNEL); + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - priv->regs = of_iomap(node, 0); + priv->regs = devm_of_iomap(dev, node, 0, NULL); + if (IS_ERR(priv->regs)) + return PTR_ERR(priv->regs); + priv->clk = of_clk_get(node, 0); if (IS_ERR(priv->clk)) return PTR_ERR(priv->clk); + ret = devm_add_action_or_reset(dev, mtk_ddp_comp_clk_put, priv->clk); + if (ret) + return ret; + #if IS_REACHABLE(CONFIG_MTK_CMDQ) ret = cmdq_dev_get_client_reg(comp->dev, &priv->cmdq_reg, 0); if (ret) diff --git a/drivers/gpu/drm/mediatek/mtk_ddp_comp.h b/drivers/gpu/drm/mediatek/mtk_ddp_comp.h index 7289b3dcf22f2..3f3d43f4330da 100644 --- a/drivers/gpu/drm/mediatek/mtk_ddp_comp.h +++ b/drivers/gpu/drm/mediatek/mtk_ddp_comp.h @@ -350,7 +350,7 @@ static inline void mtk_ddp_comp_encoder_index_set(struct mtk_ddp_comp *comp) int mtk_ddp_comp_get_id(struct device_node *node, enum mtk_ddp_comp_type comp_type); int mtk_find_possible_crtcs(struct drm_device *drm, struct device *dev); -int mtk_ddp_comp_init(struct device_node *comp_node, struct mtk_ddp_comp *comp, +int mtk_ddp_comp_init(struct device *dev, struct device_node *comp_node, struct mtk_ddp_comp *comp, unsigned int comp_id); enum mtk_ddp_comp_type mtk_ddp_comp_get_type(unsigned int comp_id); void mtk_ddp_write(struct cmdq_pkt *cmdq_pkt, unsigned int value, diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c b/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c index 10d60d2c2a568..6d7bf4afa78d3 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c @@ -80,27 +80,6 @@ void mtk_ccorr_stop(struct device *dev) writel_relaxed(0x0, ccorr->regs + DISP_CCORR_EN); } -/* Converts a DRM S31.32 value to the HW S1.n format. */ -static u16 mtk_ctm_s31_32_to_s1_n(u64 in, u32 n) -{ - u16 r; - - /* Sign bit. */ - r = in & BIT_ULL(63) ? BIT(n + 1) : 0; - - if ((in & GENMASK_ULL(62, 33)) > 0) { - /* identity value 0x100000000 -> 0x400(mt8183), */ - /* identity value 0x100000000 -> 0x800(mt8192), */ - /* if bigger this, set it to max 0x7ff. */ - r |= GENMASK(n, 0); - } else { - /* take the n+1 most important bits. */ - r |= (in >> (32 - n)) & GENMASK(n, 0); - } - - return r; -} - void mtk_ccorr_ctm_set(struct device *dev, struct drm_crtc_state *state) { struct mtk_disp_ccorr *ccorr = dev_get_drvdata(dev); @@ -119,7 +98,7 @@ void mtk_ccorr_ctm_set(struct device *dev, struct drm_crtc_state *state) input = ctm->matrix; for (i = 0; i < ARRAY_SIZE(coeffs); i++) - coeffs[i] = mtk_ctm_s31_32_to_s1_n(input[i], matrix_bits); + coeffs[i] = drm_color_ctm_s31_32_to_qm_n(input[i], 2, matrix_bits); mtk_ddp_write(cmdq_pkt, coeffs[0] << 16 | coeffs[1], &ccorr->cmdq_reg, ccorr->regs, DISP_CCORR_COEF_0); diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c index fe97bb97e004a..c0af3e3b51d55 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c @@ -527,6 +527,13 @@ bool mtk_ovl_adaptor_is_comp_present(struct device_node *node) type == OVL_ADAPTOR_TYPE_PADDING; } +static void ovl_adaptor_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + static int ovl_adaptor_comp_init(struct device *dev, struct component_match **match) { struct mtk_disp_ovl_adaptor *priv = dev_get_drvdata(dev); @@ -560,6 +567,11 @@ static int ovl_adaptor_comp_init(struct device *dev, struct component_match **ma if (!comp_pdev) return -EPROBE_DEFER; + ret = devm_add_action_or_reset(dev, ovl_adaptor_put_device, + &comp_pdev->dev); + if (ret) + return ret; + priv->ovl_adaptor_comp[id] = &comp_pdev->dev; drm_of_component_match_add(dev, match, component_compare_of, node); diff --git a/drivers/gpu/drm/mediatek/mtk_dp.c b/drivers/gpu/drm/mediatek/mtk_dp.c index bef6eeb30d3ec..b0b1e158600f3 100644 --- a/drivers/gpu/drm/mediatek/mtk_dp.c +++ b/drivers/gpu/drm/mediatek/mtk_dp.c @@ -2087,6 +2087,7 @@ static int mtk_dp_dt_parse(struct mtk_dp *mtk_dp, endpoint = of_graph_get_endpoint_by_regs(pdev->dev.of_node, 1, -1); len = of_property_count_elems_of_size(endpoint, "data-lanes", sizeof(u32)); + of_node_put(endpoint); if (len < 0 || len > 4 || len == 3) { dev_err(dev, "invalid data lane size: %d\n", len); return -EINVAL; diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c index 61cab32e213af..53360b5d12ba5 100644 --- a/drivers/gpu/drm/mediatek/mtk_dpi.c +++ b/drivers/gpu/drm/mediatek/mtk_dpi.c @@ -836,20 +836,6 @@ static int mtk_dpi_bridge_attach(struct drm_bridge *bridge, enum drm_bridge_attach_flags flags) { struct mtk_dpi *dpi = bridge_to_dpi(bridge); - int ret; - - dpi->next_bridge = devm_drm_of_get_bridge(dpi->dev, dpi->dev->of_node, 1, -1); - if (IS_ERR(dpi->next_bridge)) { - ret = PTR_ERR(dpi->next_bridge); - if (ret == -EPROBE_DEFER) - return ret; - - /* Old devicetree has only one endpoint */ - dpi->next_bridge = devm_drm_of_get_bridge(dpi->dev, dpi->dev->of_node, 0, 0); - if (IS_ERR(dpi->next_bridge)) - return dev_err_probe(dpi->dev, PTR_ERR(dpi->next_bridge), - "Failed to get bridge\n"); - } return drm_bridge_attach(encoder, dpi->next_bridge, &dpi->bridge, flags); @@ -1319,6 +1305,15 @@ static int mtk_dpi_probe(struct platform_device *pdev) if (dpi->irq < 0) return dpi->irq; + dpi->next_bridge = devm_drm_of_get_bridge(dpi->dev, dpi->dev->of_node, 1, -1); + if (IS_ERR(dpi->next_bridge) && PTR_ERR(dpi->next_bridge) == -ENODEV) { + /* Old devicetree has only one endpoint */ + dpi->next_bridge = devm_drm_of_get_bridge(dpi->dev, dpi->dev->of_node, 0, 0); + } + if (IS_ERR(dpi->next_bridge)) + return dev_err_probe(dpi->dev, PTR_ERR(dpi->next_bridge), + "Failed to get bridge\n"); + platform_set_drvdata(pdev, dpi); dpi->bridge.of_node = dev->of_node; diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c index 31ff2922758a4..a94c51a832616 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c @@ -1123,7 +1123,7 @@ static int mtk_drm_probe(struct platform_device *pdev) (void *)private->mmsys_dev, sizeof(*private->mmsys_dev)); private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR].dev = &ovl_adaptor->dev; - mtk_ddp_comp_init(NULL, &private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR], + mtk_ddp_comp_init(dev, NULL, &private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR], DDP_COMPONENT_DRM_OVL_ADAPTOR); component_match_add(dev, &match, compare_dev, &ovl_adaptor->dev); } @@ -1189,7 +1189,7 @@ static int mtk_drm_probe(struct platform_device *pdev) node); } - ret = mtk_ddp_comp_init(node, &private->ddp_comp[comp_id], comp_id); + ret = mtk_ddp_comp_init(dev, node, &private->ddp_comp[comp_id], comp_id); if (ret) { of_node_put(node); goto err_node; diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c index 0e2bcd5f67b76..acee2227275b7 100644 --- a/drivers/gpu/drm/mediatek/mtk_dsi.c +++ b/drivers/gpu/drm/mediatek/mtk_dsi.c @@ -1002,12 +1002,6 @@ static int mtk_dsi_host_attach(struct mipi_dsi_host *host, return PTR_ERR(dsi->next_bridge); } - /* - * set flag to request the DSI host bridge be pre-enabled before device bridge - * in the chain, so the DSI host is ready when the device bridge is pre-enabled - */ - dsi->next_bridge->pre_enable_prev_first = true; - drm_bridge_add(&dsi->bridge); ret = component_add(host->dev, &mtk_dsi_component_ops); @@ -1238,6 +1232,11 @@ static int mtk_dsi_probe(struct platform_device *pdev) dsi->host.ops = &mtk_dsi_ops; dsi->host.dev = dev; + + init_waitqueue_head(&dsi->irq_wait_queue); + + platform_set_drvdata(pdev, dsi); + ret = mipi_dsi_host_register(&dsi->host); if (ret < 0) return dev_err_probe(dev, ret, "Failed to register DSI host\n"); @@ -1249,10 +1248,6 @@ static int mtk_dsi_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "Failed to request DSI irq\n"); } - init_waitqueue_head(&dsi->irq_wait_queue); - - platform_set_drvdata(pdev, dsi); - dsi->bridge.of_node = dev->of_node; dsi->bridge.type = DRM_MODE_CONNECTOR_DSI; diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c index b766dd5e6c8de..306e2c9073114 100644 --- a/drivers/gpu/drm/mediatek/mtk_hdmi.c +++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c @@ -1345,6 +1345,13 @@ static const struct drm_bridge_funcs mtk_hdmi_bridge_funcs = { .edid_read = mtk_hdmi_bridge_edid_read, }; +static void mtk_hdmi_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + static int mtk_hdmi_get_cec_dev(struct mtk_hdmi *hdmi, struct device *dev, struct device_node *np) { struct platform_device *cec_pdev; @@ -1369,6 +1376,10 @@ static int mtk_hdmi_get_cec_dev(struct mtk_hdmi *hdmi, struct device *dev, struc } of_node_put(cec_np); + ret = devm_add_action_or_reset(dev, mtk_hdmi_put_device, &cec_pdev->dev); + if (ret) + return ret; + /* * The mediatek,syscon-hdmi property contains a phandle link to the * MMSYS_CONFIG device and the register offset of the HDMI_SYS_CFG @@ -1423,6 +1434,10 @@ static int mtk_hdmi_dt_parse_pdata(struct mtk_hdmi *hdmi, if (!hdmi->ddc_adpt) return dev_err_probe(dev, -EINVAL, "Failed to get ddc i2c adapter by node\n"); + ret = devm_add_action_or_reset(dev, mtk_hdmi_put_device, &hdmi->ddc_adpt->dev); + if (ret) + return ret; + ret = mtk_hdmi_get_cec_dev(hdmi, dev, np); if (ret) return ret; diff --git a/drivers/gpu/drm/mgag200/mgag200_bmc.c b/drivers/gpu/drm/mgag200/mgag200_bmc.c index a689c71ff1653..bbdeb791c5b38 100644 --- a/drivers/gpu/drm/mgag200/mgag200_bmc.c +++ b/drivers/gpu/drm/mgag200/mgag200_bmc.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only #include +#include #include #include @@ -12,7 +13,7 @@ void mgag200_bmc_stop_scanout(struct mga_device *mdev) { u8 tmp; - int iter_max; + int ret; /* * 1 - The first step is to inform the BMC of an upcoming mode @@ -42,30 +43,22 @@ void mgag200_bmc_stop_scanout(struct mga_device *mdev) /* * 3a- The third step is to verify if there is an active scan. - * We are waiting for a 0 on remhsyncsts ). + * We are waiting for a 0 on remhsyncsts (). */ - iter_max = 300; - while (!(tmp & 0x1) && iter_max) { - WREG8(DAC_INDEX, MGA1064_SPAREREG); - tmp = RREG8(DAC_DATA); - udelay(1000); - iter_max--; - } + ret = read_poll_timeout(RREG_DAC, tmp, !(tmp & 0x1), + 1000, 300000, false, + MGA1064_SPAREREG); + if (ret == -ETIMEDOUT) + return; /* - * 3b- This step occurs only if the remove is actually + * 3b- This step occurs only if the remote BMC is actually * scanning. We are waiting for the end of the frame which is * a 1 on remvsyncsts (XSPAREREG<1>) */ - if (iter_max) { - iter_max = 300; - while ((tmp & 0x2) && iter_max) { - WREG8(DAC_INDEX, MGA1064_SPAREREG); - tmp = RREG8(DAC_DATA); - udelay(1000); - iter_max--; - } - } + (void)read_poll_timeout(RREG_DAC, tmp, (tmp & 0x2), + 1000, 300000, false, + MGA1064_SPAREREG); } void mgag200_bmc_start_scanout(struct mga_device *mdev) diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h index f4bf40cd7c88a..a875c4bf8cbe4 100644 --- a/drivers/gpu/drm/mgag200/mgag200_drv.h +++ b/drivers/gpu/drm/mgag200/mgag200_drv.h @@ -111,6 +111,12 @@ #define DAC_INDEX 0x3c00 #define DAC_DATA 0x3c0a +#define RREG_DAC(reg) \ + ({ \ + WREG8(DAC_INDEX, reg); \ + RREG8(DAC_DATA); \ + }) \ + #define WREG_DAC(reg, v) \ do { \ WREG8(DAC_INDEX, reg); \ diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c index 951d715dea301..d019177462cf7 100644 --- a/drivers/gpu/drm/mgag200/mgag200_mode.c +++ b/drivers/gpu/drm/mgag200/mgag200_mode.c @@ -161,6 +161,30 @@ static void mgag200_set_startadd(struct mga_device *mdev, WREG_ECRT(0x00, crtcext0); } +/* + * Set the opmode for the hardware swapper for Big-Endian processor + * support for the frame buffer aperture and DMAWIN space. + */ +static void mgag200_set_datasiz(struct mga_device *mdev, u32 format) +{ +#if defined(__BIG_ENDIAN) + u32 opmode = RREG32(MGAREG_OPMODE); + + opmode &= ~(GENMASK(17, 16) | GENMASK(9, 8) | GENMASK(3, 2)); + + /* Big-endian byte-swapping */ + switch (format) { + case DRM_FORMAT_RGB565: + opmode |= 0x10100; + break; + case DRM_FORMAT_XRGB8888: + opmode |= 0x20200; + break; + } + WREG32(MGAREG_OPMODE, opmode); +#endif +} + void mgag200_init_registers(struct mga_device *mdev) { u8 crtc11, misc; @@ -496,6 +520,7 @@ void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane, struct drm_atomic_helper_damage_iter iter; struct drm_rect damage; + mgag200_set_datasiz(mdev, fb->format->format); drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state); drm_atomic_for_each_plane_damage(&iter, &damage) { mgag200_handle_damage(mdev, shadow_plane_state->data, fb, &damage); diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile index 0c0dfb25f01b1..5a7fc4ba365b8 100644 --- a/drivers/gpu/drm/msm/Makefile +++ b/drivers/gpu/drm/msm/Makefile @@ -85,6 +85,7 @@ msm-display-$(CONFIG_DRM_MSM_DPU) += \ disp/dpu1/dpu_hw_lm.o \ disp/dpu1/dpu_hw_pingpong.o \ disp/dpu1/dpu_hw_sspp.o \ + disp/dpu1/dpu_hw_sspp_v13.o \ disp/dpu1/dpu_hw_dspp.o \ disp/dpu1/dpu_hw_merge3d.o \ disp/dpu1/dpu_hw_top.o \ @@ -142,7 +143,8 @@ msm-display-$(CONFIG_DRM_MSM_DP)+= dp/dp_aux.o \ dp/dp_link.o \ dp/dp_panel.o \ dp/dp_audio.o \ - dp/dp_utils.o + dp/dp_utils.o \ + dp/dp_mst_drm.o msm-display-$(CONFIG_DRM_MSM_HDMI_HDCP) += hdmi/hdmi_hdcp.o diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c index ec38db45d8a36..e67ed58aa3d8f 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c @@ -77,7 +77,10 @@ static bool a2xx_me_init(struct msm_gpu *gpu) /* Vertex and Pixel Shader Start Addresses in instructions * (3 DWORDS per instruction) */ - OUT_RING(ring, 0x80000180); + if (adreno_is_a225(adreno_gpu)) + OUT_RING(ring, 0x80000300); + else + OUT_RING(ring, 0x80000180); /* Maximum Contexts */ OUT_RING(ring, 0x00000001); /* Write Confirm Interval and The CP will wait the @@ -234,7 +237,7 @@ static int a2xx_hw_init(struct msm_gpu *gpu) * word (0x20xxxx for A200, 0x220xxx for A220, 0x225xxx for A225). * Older firmware files, which lack protection support, have 0 instead. */ - if (ptr[1] == 0) { + if (ptr[1] == 0 && !a2xx_gpu->protection_disabled) { dev_warn(gpu->dev->dev, "Legacy firmware detected, disabling protection support\n"); a2xx_gpu->protection_disabled = true; diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c index 0407c9bc8c1bf..4467b04527cd1 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c @@ -78,7 +78,7 @@ static void a2xx_gpummu_destroy(struct msm_mmu *mmu) { struct a2xx_gpummu *gpummu = to_a2xx_gpummu(mmu); - dma_free_attrs(mmu->dev, TABLE_SIZE, gpummu->table, gpummu->pt_base, + dma_free_attrs(mmu->dev, TABLE_SIZE + 32, gpummu->table, gpummu->pt_base, DMA_ATTR_FORCE_CONTIGUOUS); kfree(gpummu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c index 44df6410bce17..426fc3ba62020 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c @@ -501,8 +501,6 @@ static const struct adreno_reglist a690_hwcg[] = { {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, - {REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 0x10111}, - {REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 0x5555}, {} }; @@ -705,6 +703,22 @@ static const struct adreno_info a6xx_gpus[] = { { 157, 3 }, { 127, 4 }, ), + }, { + .chip_ids = ADRENO_CHIP_IDS(0x06010200), + .family = ADRENO_6XX_GEN1, + .fw = { + [ADRENO_FW_SQE] = "a630_sqe.fw", + [ADRENO_FW_GMU] = "a612_rgmu.bin", + }, + .gmem = (SZ_128K + SZ_4K), + .inactive_period = DRM_MSM_INACTIVE_PERIOD, + .init = a6xx_gpu_init, + .a6xx = &(const struct a6xx_info) { + .hwcg = a612_hwcg, + .protect = &a630_protect, + .gmu_cgc_mode = 0x00000022, + .prim_fifo_threshold = 0x00080000, + }, }, { .chip_ids = ADRENO_CHIP_IDS(0x06010500), .family = ADRENO_6XX_GEN1, @@ -1360,6 +1374,7 @@ static const u32 a750_ifpc_reglist_regs[] = { REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE(2), REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE(3), REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE(4), + REG_A6XX_RBBM_PERFCTR_CNTL, REG_A6XX_TPL1_NC_MODE_CNTL, REG_A6XX_SP_NC_MODE_CNTL, REG_A6XX_CP_DBG_ECO_CNTL, diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 4e6dc16e4a4c4..82d0a4c797a3f 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -350,12 +350,18 @@ static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = { /* Trigger a OOB (out of band) request to the GMU */ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) { + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; int ret; u32 val; int request, ack; WARN_ON_ONCE(!mutex_is_locked(&gmu->lock)); + /* Skip OOB calls since RGMU is not enabled */ + if (adreno_has_rgmu(adreno_gpu)) + return 0; + if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits)) return -EINVAL; @@ -395,10 +401,16 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) /* Clear a pending OOB state in the GMU */ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) { + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; int bit; WARN_ON_ONCE(!mutex_is_locked(&gmu->lock)); + /* Skip OOB calls since RGMU is not enabled */ + if (adreno_has_rgmu(adreno_gpu)) + return; + if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits)) return; @@ -485,8 +497,9 @@ static void a6xx_gemnoc_workaround(struct a6xx_gmu *gmu) * in the power down sequence not being fully executed. That in turn can * prevent CX_GDSC from collapsing. Assert Qactive to avoid this. */ - if (adreno_is_a621(adreno_gpu) || adreno_is_7c3(adreno_gpu)) - gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, BIT(0)); + if (adreno_is_a7xx(adreno_gpu) || (adreno_is_a621(adreno_gpu) || + adreno_is_7c3(adreno_gpu))) + gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, BIT(0)); } /* Let the GMU know that we are about to go into slumber */ @@ -522,10 +535,9 @@ static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu) } out: - a6xx_gemnoc_workaround(gmu); - /* Put fence into allow mode */ gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); + a6xx_gemnoc_workaround(gmu); return ret; } @@ -584,22 +596,19 @@ static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value) writel(value, ptr + (offset << 2)); } -static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, - const char *name); - static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) { struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; struct platform_device *pdev = to_platform_device(gmu->dev); - void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc"); + void __iomem *pdcptr = devm_platform_ioremap_resource_byname(pdev, "gmu_pdc"); u32 seqmem0_drv0_reg = REG_A6XX_RSCC_SEQ_MEM_0_DRV0; void __iomem *seqptr = NULL; uint32_t pdc_address_offset; bool pdc_in_aop = false; if (IS_ERR(pdcptr)) - goto err; + return; if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) @@ -612,9 +621,9 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) pdc_address_offset = 0x30080; if (!pdc_in_aop) { - seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq"); + seqptr = devm_platform_ioremap_resource_byname(pdev, "gmu_pdc_seq"); if (IS_ERR(seqptr)) - goto err; + return; } /* Disable SDE clock gating */ @@ -704,12 +713,6 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) /* ensure no writes happen before the uCode is fully written */ wmb(); - -err: - if (!IS_ERR_OR_NULL(pdcptr)) - iounmap(pdcptr); - if (!IS_ERR_OR_NULL(seqptr)) - iounmap(seqptr); } /* @@ -1492,13 +1495,14 @@ static unsigned int a6xx_gmu_get_arc_level(struct device *dev, } static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes, - unsigned long *freqs, int freqs_count, const char *id) + unsigned long *freqs, int freqs_count, + const char *pri_id, const char *sec_id) { int i, j; const u16 *pri, *sec; size_t pri_count, sec_count; - pri = cmd_db_read_aux_data(id, &pri_count); + pri = cmd_db_read_aux_data(pri_id, &pri_count); if (IS_ERR(pri)) return PTR_ERR(pri); /* @@ -1509,13 +1513,7 @@ static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes, if (!pri_count) return -EINVAL; - /* - * Some targets have a separate gfx mxc rail. So try to read that first and then fall back - * to regular mx rail if it is missing - */ - sec = cmd_db_read_aux_data("gmxc.lvl", &sec_count); - if (IS_ERR(sec) && sec != ERR_PTR(-EPROBE_DEFER)) - sec = cmd_db_read_aux_data("mx.lvl", &sec_count); + sec = cmd_db_read_aux_data(sec_id, &sec_count); if (IS_ERR(sec)) return PTR_ERR(sec); @@ -1583,15 +1581,24 @@ static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu) struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; const struct a6xx_info *info = adreno_gpu->info->a6xx; struct msm_gpu *gpu = &adreno_gpu->base; + const char *sec_id; + const u16 *gmxc; int ret; + gmxc = cmd_db_read_aux_data("gmxc.lvl", NULL); + if (gmxc == ERR_PTR(-EPROBE_DEFER)) + return -EPROBE_DEFER; + + /* If GMxC is present, prefer that as secondary rail for GX votes */ + sec_id = IS_ERR_OR_NULL(gmxc) ? "mx.lvl" : "gmxc.lvl"; + /* Build the GX votes */ ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes, - gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl"); + gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl", sec_id); /* Build the CX votes */ ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes, - gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl"); + gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl", "mx.lvl"); /* Build the interconnect votes */ if (info->bcms && gmu->nr_gpu_bws > 1) @@ -1795,27 +1802,6 @@ static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu) return 0; } -static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, - const char *name) -{ - void __iomem *ret; - struct resource *res = platform_get_resource_byname(pdev, - IORESOURCE_MEM, name); - - if (!res) { - DRM_DEV_ERROR(&pdev->dev, "Unable to find the %s registers\n", name); - return ERR_PTR(-EINVAL); - } - - ret = ioremap(res->start, resource_size(res)); - if (!ret) { - DRM_DEV_ERROR(&pdev->dev, "Unable to map the %s registers\n", name); - return ERR_PTR(-EINVAL); - } - - return ret; -} - static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev, const char *name, irq_handler_t handler) { @@ -1866,7 +1852,6 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) { struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; struct a6xx_gmu *gmu = &a6xx_gpu->gmu; - struct platform_device *pdev = to_platform_device(gmu->dev); mutex_lock(&gmu->lock); if (!gmu->initialized) { @@ -1895,12 +1880,11 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) qmp_put(gmu->qmp); iounmap(gmu->mmio); - if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) - iounmap(gmu->rscc); gmu->mmio = NULL; gmu->rscc = NULL; - if (!adreno_has_gmu_wrapper(adreno_gpu)) { + if (!adreno_has_gmu_wrapper(adreno_gpu) && + !adreno_has_rgmu(adreno_gpu)) { a6xx_gmu_memory_free(gmu); free_irq(gmu->gmu_irq, gmu); @@ -1922,10 +1906,36 @@ static int cxpd_notifier_cb(struct notifier_block *nb, return 0; } +static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, resource_size_t *start) +{ + struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + void __iomem *ret; + + if (!res) { + DRM_DEV_ERROR(&pdev->dev, "Unable to find the gmu core registers\n"); + return ERR_PTR(-EINVAL); + } + + ret = ioremap(res->start, resource_size(res)); + if (!ret) { + DRM_DEV_ERROR(&pdev->dev, "Unable to map the gmu core registers\n"); + return ERR_PTR(-EINVAL); + } + + if (start) + *start = res->start; + + return ret; +} + int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) { struct platform_device *pdev = of_find_device_by_node(node); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + resource_size_t start; + struct resource *res; int ret; if (!pdev) @@ -1942,13 +1952,29 @@ int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) /* Mark legacy for manual SPTPRAC control */ gmu->legacy = true; + /* RGMU requires clocks */ + ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks); + if (ret < 0) + goto err_clk; + + gmu->nr_clocks = ret; + /* Map the GMU registers */ - gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); + gmu->mmio = a6xx_gmu_get_mmio(pdev, &start); if (IS_ERR(gmu->mmio)) { ret = PTR_ERR(gmu->mmio); goto err_mmio; } + res = platform_get_resource_byname(gpu->pdev, IORESOURCE_MEM, "kgsl_3d0_reg_memory"); + if (!res) { + ret = -EINVAL; + goto err_mmio; + } + + /* Identify gmu base offset from gpu base address */ + gmu->mmio_offset = (u32)(start - res->start); + gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx"); if (IS_ERR(gmu->cxpd)) { ret = PTR_ERR(gmu->cxpd); @@ -1981,6 +2007,7 @@ int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) err_mmio: iounmap(gmu->mmio); +err_clk: /* Drop reference taken in of_find_device_by_node */ put_device(gmu->dev); @@ -1989,10 +2016,13 @@ int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) { + struct platform_device *pdev = of_find_device_by_node(node); struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; struct a6xx_gmu *gmu = &a6xx_gpu->gmu; - struct platform_device *pdev = of_find_device_by_node(node); struct device_link *link; + resource_size_t start; + struct resource *res; int ret; if (!pdev) @@ -2087,15 +2117,24 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) goto err_memory; /* Map the GMU registers */ - gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); + gmu->mmio = a6xx_gmu_get_mmio(pdev, &start); if (IS_ERR(gmu->mmio)) { ret = PTR_ERR(gmu->mmio); goto err_memory; } + res = platform_get_resource_byname(gpu->pdev, IORESOURCE_MEM, "kgsl_3d0_reg_memory"); + if (!res) { + ret = -EINVAL; + goto err_mmio; + } + + /* Identify gmu base offset from gpu base address */ + gmu->mmio_offset = (u32)(start - res->start); + if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) { - gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc"); + gmu->rscc = devm_platform_ioremap_resource_byname(pdev, "rscc"); if (IS_ERR(gmu->rscc)) { ret = -ENODEV; goto err_mmio; @@ -2173,8 +2212,6 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) err_mmio: iounmap(gmu->mmio); - if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) - iounmap(gmu->rscc); free_irq(gmu->gmu_irq, gmu); free_irq(gmu->hfi_irq, gmu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h index 06cfc294016f5..55b1c78daa8b5 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -68,6 +68,7 @@ struct a6xx_gmu { struct drm_gpuvm *vm; void __iomem *mmio; + u32 mmio_offset; void __iomem *rscc; int hfi_irq; @@ -130,20 +131,23 @@ struct a6xx_gmu { unsigned long status; }; +#define GMU_BYTE_OFFSET(gmu, offset) (((offset) << 2) - (gmu)->mmio_offset) + static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset) { - return readl(gmu->mmio + (offset << 2)); + /* The 'offset' is based on GPU's start address. Adjust it */ + return readl(gmu->mmio + GMU_BYTE_OFFSET(gmu, offset)); } static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value) { - writel(value, gmu->mmio + (offset << 2)); + writel(value, gmu->mmio + GMU_BYTE_OFFSET(gmu, offset)); } static inline void gmu_write_bulk(struct a6xx_gmu *gmu, u32 offset, const u32 *data, u32 size) { - memcpy_toio(gmu->mmio + (offset << 2), data, size); + memcpy_toio(gmu->mmio + GMU_BYTE_OFFSET(gmu, offset), data, size); wmb(); } @@ -160,17 +164,17 @@ static inline u64 gmu_read64(struct a6xx_gmu *gmu, u32 lo, u32 hi) { u64 val; - val = (u64) readl(gmu->mmio + (lo << 2)); - val |= ((u64) readl(gmu->mmio + (hi << 2)) << 32); + val = gmu_read(gmu, lo); + val |= ((u64) gmu_read(gmu, hi) << 32); return val; } #define gmu_poll_timeout(gmu, addr, val, cond, interval, timeout) \ - readl_poll_timeout((gmu)->mmio + ((addr) << 2), val, cond, \ - interval, timeout) + readl_poll_timeout((gmu)->mmio + (GMU_BYTE_OFFSET(gmu, addr)), val, \ + cond, interval, timeout) #define gmu_poll_timeout_atomic(gmu, addr, val, cond, interval, timeout) \ - readl_poll_timeout_atomic((gmu)->mmio + ((addr) << 2), val, cond, \ + readl_poll_timeout_atomic((gmu)->mmio + (GMU_BYTE_OFFSET(gmu, addr)), val, cond, \ interval, timeout) static inline u32 gmu_read_rscc(struct a6xx_gmu *gmu, u32 offset) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index b8f8ae940b55f..881c5fe64ea01 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -224,7 +224,7 @@ static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu, OUT_RING(ring, submit->seqno - 1); OUT_PKT7(ring, CP_THREAD_CONTROL, 1); - OUT_RING(ring, CP_SET_THREAD_BOTH); + OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BOTH); /* Reset state used to synchronize BR and BV */ OUT_PKT7(ring, CP_RESET_CONTEXT_STATE, 1); @@ -235,7 +235,13 @@ static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu, CP_RESET_CONTEXT_STATE_0_RESET_GLOBAL_LOCAL_TS); OUT_PKT7(ring, CP_THREAD_CONTROL, 1); - OUT_RING(ring, CP_SET_THREAD_BR); + OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BOTH); + + OUT_PKT7(ring, CP_EVENT_WRITE, 1); + OUT_RING(ring, LRZ_FLUSH); + + OUT_PKT7(ring, CP_THREAD_CONTROL, 1); + OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR); } if (!sysprof) { @@ -612,15 +618,26 @@ static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state) if (adreno_is_a630(adreno_gpu)) clock_cntl_on = 0x8aa8aa02; - else if (adreno_is_a610(adreno_gpu)) + else if (adreno_is_a610(adreno_gpu) || adreno_is_a612(adreno_gpu)) clock_cntl_on = 0xaaa8aa82; else if (adreno_is_a702(adreno_gpu)) clock_cntl_on = 0xaaaaaa82; else clock_cntl_on = 0x8aa8aa82; - cgc_delay = adreno_is_a615_family(adreno_gpu) ? 0x111 : 0x10111; - cgc_hyst = adreno_is_a615_family(adreno_gpu) ? 0x555 : 0x5555; + if (adreno_is_a612(adreno_gpu)) + cgc_delay = 0x11; + else if (adreno_is_a615_family(adreno_gpu)) + cgc_delay = 0x111; + else + cgc_delay = 0x10111; + + if (adreno_is_a612(adreno_gpu)) + cgc_hyst = 0x55; + else if (adreno_is_a615_family(adreno_gpu)) + cgc_hyst = 0x555; + else + cgc_hyst = 0x5555; gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, state ? adreno_gpu->info->a6xx->gmu_cgc_mode : 0); @@ -714,6 +731,9 @@ static int a6xx_calc_ubwc_config(struct adreno_gpu *gpu) cfg->ubwc_swizzle = 0x7; } + if (adreno_is_a612(gpu)) + cfg->highest_bank_bit = 14; + if (adreno_is_a618(gpu)) cfg->highest_bank_bit = 14; @@ -831,15 +851,17 @@ static void a7xx_patch_pwrup_reglist(struct msm_gpu *gpu) lock->gpu_req = lock->cpu_req = lock->turn = 0; reglist = adreno_gpu->info->a6xx->ifpc_reglist; - lock->ifpc_list_len = reglist->count; + if (reglist) { + lock->ifpc_list_len = reglist->count; - /* - * For each entry in each of the lists, write the offset and the current - * register value into the GPU buffer - */ - for (i = 0; i < reglist->count; i++) { - *dest++ = reglist->regs[i]; - *dest++ = gpu_read(gpu, reglist->regs[i]); + /* + * For each entry in each of the lists, write the offset and the current + * register value into the GPU buffer + */ + for (i = 0; i < reglist->count; i++) { + *dest++ = reglist->regs[i]; + *dest++ = gpu_read(gpu, reglist->regs[i]); + } } reglist = adreno_gpu->info->a6xx->pwrup_reglist; @@ -1288,7 +1310,7 @@ static int hw_init(struct msm_gpu *gpu) gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020); /* Setting the mem pool size */ - if (adreno_is_a610(adreno_gpu)) { + if (adreno_is_a610(adreno_gpu) || adreno_is_a612(adreno_gpu)) { gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 48); gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 47); } else if (adreno_is_a702(adreno_gpu)) { @@ -1321,7 +1343,8 @@ static int hw_init(struct msm_gpu *gpu) a6xx_set_ubwc_config(gpu); /* Enable fault detection */ - if (adreno_is_a730(adreno_gpu) || + if (adreno_is_a612(adreno_gpu) || + adreno_is_a730(adreno_gpu) || adreno_is_a740_family(adreno_gpu)) gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0xcfffff); else if (adreno_is_a690(adreno_gpu)) @@ -1576,7 +1599,7 @@ static void a6xx_recover(struct msm_gpu *gpu) */ gpu->active_submits = 0; - if (adreno_has_gmu_wrapper(adreno_gpu)) { + if (adreno_has_gmu_wrapper(adreno_gpu) || adreno_has_rgmu(adreno_gpu)) { /* Drain the outstanding traffic on memory buses */ a6xx_bus_clear_pending_transactions(adreno_gpu, true); @@ -2229,6 +2252,12 @@ static int a6xx_pm_resume(struct msm_gpu *gpu) if (ret) goto err_bulk_clk; + ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks); + if (ret) { + clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks); + goto err_bulk_clk; + } + if (adreno_is_a619_holi(adreno_gpu)) a6xx_sptprac_enable(gmu); @@ -2242,8 +2271,10 @@ static int a6xx_pm_resume(struct msm_gpu *gpu) err_set_opp: mutex_unlock(&a6xx_gpu->gmu.lock); - if (!ret) + if (!ret) { msm_devfreq_resume(gpu); + a6xx_llc_activate(a6xx_gpu); + } return ret; } @@ -2284,6 +2315,8 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu) trace_msm_gpu_suspend(0); + a6xx_llc_deactivate(a6xx_gpu); + msm_devfreq_suspend(gpu); mutex_lock(&a6xx_gpu->gmu.lock); @@ -2295,6 +2328,7 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu) a6xx_sptprac_disable(gmu); clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks); + clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks); pm_runtime_put_sync(gmu->gxpd); dev_pm_opp_set_opp(&gpu->pdev->dev, NULL); @@ -2673,7 +2707,8 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_a7xx, 4); else if (is_a7xx) ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_a7xx, 1); - else if (adreno_has_gmu_wrapper(adreno_gpu)) + else if (adreno_has_gmu_wrapper(adreno_gpu) || + of_device_is_compatible(node, "qcom,adreno-rgmu")) ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_gmuwrapper, 1); else ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); @@ -2689,7 +2724,7 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) if (adreno_is_a618(adreno_gpu) || adreno_is_7c3(adreno_gpu)) priv->gpu_clamp_to_idle = true; - if (adreno_has_gmu_wrapper(adreno_gpu)) + if (adreno_has_gmu_wrapper(adreno_gpu) || adreno_has_rgmu(adreno_gpu)) ret = a6xx_gmu_wrapper_init(a6xx_gpu, node); else ret = a6xx_gmu_init(a6xx_gpu, node); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 4c7f3c642f6ac..d2d6b2fd3cba3 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -1255,7 +1255,7 @@ static void a6xx_get_gmu_registers(struct msm_gpu *gpu, return; /* Set the fence to ALLOW mode so we can access the registers */ - gpu_write(gpu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); + gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gmu_reglist[2], &a6xx_state->gmu_registers[3], false); @@ -1596,7 +1596,8 @@ struct msm_gpu_state *a6xx_gpu_state_get(struct msm_gpu *gpu) /* Get the generic state from the adreno core */ adreno_gpu_state_get(gpu, &a6xx_state->base); - if (!adreno_has_gmu_wrapper(adreno_gpu)) { + if (!adreno_has_gmu_wrapper(adreno_gpu) && + !adreno_has_rgmu(adreno_gpu)) { a6xx_get_gmu_registers(gpu, a6xx_state); a6xx_state->gmu_log = a6xx_snapshot_gmu_bo(a6xx_state, &a6xx_gpu->gmu.log); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h index 1c18499b60bb9..4753b71837f33 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h @@ -343,48 +343,48 @@ static const struct a6xx_registers a6xx_gbif_reglist = static const u32 a6xx_gmu_gx_registers[] = { /* GMU GX */ - 0x0000, 0x0000, 0x0010, 0x0013, 0x0016, 0x0016, 0x0018, 0x001b, - 0x001e, 0x001e, 0x0020, 0x0023, 0x0026, 0x0026, 0x0028, 0x002b, - 0x002e, 0x002e, 0x0030, 0x0033, 0x0036, 0x0036, 0x0038, 0x003b, - 0x003e, 0x003e, 0x0040, 0x0043, 0x0046, 0x0046, 0x0080, 0x0084, - 0x0100, 0x012b, 0x0140, 0x0140, + 0x1a800, 0x1a800, 0x1a810, 0x1a813, 0x1a816, 0x1a816, 0x1a818, 0x1a81b, + 0x1a81e, 0x1a81e, 0x1a820, 0x1a823, 0x1a826, 0x1a826, 0x1a828, 0x1a82b, + 0x1a82e, 0x1a82e, 0x1a830, 0x1a833, 0x1a836, 0x1a836, 0x1a838, 0x1a83b, + 0x1a83e, 0x1a83e, 0x1a840, 0x1a843, 0x1a846, 0x1a846, 0x1a880, 0x1a884, + 0x1a900, 0x1a92b, 0x1a940, 0x1a940, }; static const u32 a6xx_gmu_cx_registers[] = { /* GMU CX */ - 0x4c00, 0x4c07, 0x4c10, 0x4c12, 0x4d00, 0x4d00, 0x4d07, 0x4d0a, - 0x5000, 0x5004, 0x5007, 0x5008, 0x500b, 0x500c, 0x500f, 0x501c, - 0x5024, 0x502a, 0x502d, 0x5030, 0x5040, 0x5053, 0x5087, 0x5089, - 0x50a0, 0x50a2, 0x50a4, 0x50af, 0x50c0, 0x50c3, 0x50d0, 0x50d0, - 0x50e4, 0x50e4, 0x50e8, 0x50ec, 0x5100, 0x5103, 0x5140, 0x5140, - 0x5142, 0x5144, 0x514c, 0x514d, 0x514f, 0x5151, 0x5154, 0x5154, - 0x5157, 0x5158, 0x515d, 0x515d, 0x5162, 0x5162, 0x5164, 0x5165, - 0x5180, 0x5186, 0x5190, 0x519e, 0x51c0, 0x51c0, 0x51c5, 0x51cc, - 0x51e0, 0x51e2, 0x51f0, 0x51f0, 0x5200, 0x5201, + 0x1f400, 0x1f407, 0x1f410, 0x1f412, 0x1f500, 0x1f500, 0x1f507, 0x1f50a, + 0x1f800, 0x1f804, 0x1f807, 0x1f808, 0x1f80b, 0x1f80c, 0x1f80f, 0x1f81c, + 0x1f824, 0x1f82a, 0x1f82d, 0x1f830, 0x1f840, 0x1f853, 0x1f887, 0x1f889, + 0x1f8a0, 0x1f8a2, 0x1f8a4, 0x1f8af, 0x1f8c0, 0x1f8c3, 0x1f8d0, 0x1f8d0, + 0x1f8e4, 0x1f8e4, 0x1f8e8, 0x1f8ec, 0x1f900, 0x1f903, 0x1f940, 0x1f940, + 0x1f942, 0x1f944, 0x1f94c, 0x1f94d, 0x1f94f, 0x1f951, 0x1f954, 0x1f954, + 0x1f957, 0x1f958, 0x1f95d, 0x1f95d, 0x1f962, 0x1f962, 0x1f964, 0x1f965, + 0x1f980, 0x1f986, 0x1f990, 0x1f99e, 0x1f9c0, 0x1f9c0, 0x1f9c5, 0x1f9cc, + 0x1f9e0, 0x1f9e2, 0x1f9f0, 0x1f9f0, 0x1fa00, 0x1fa01, /* GMU AO */ - 0x9300, 0x9316, 0x9400, 0x9400, + 0x23b00, 0x23b16, 0x23c00, 0x23c00, }; static const u32 a6xx_gmu_gpucc_registers[] = { /* GPU CC */ - 0x9800, 0x9812, 0x9840, 0x9852, 0x9c00, 0x9c04, 0x9c07, 0x9c0b, - 0x9c15, 0x9c1c, 0x9c1e, 0x9c2d, 0x9c3c, 0x9c3d, 0x9c3f, 0x9c40, - 0x9c42, 0x9c49, 0x9c58, 0x9c5a, 0x9d40, 0x9d5e, 0xa000, 0xa002, - 0xa400, 0xa402, 0xac00, 0xac02, 0xb000, 0xb002, 0xb400, 0xb402, - 0xb800, 0xb802, + 0x24000, 0x24012, 0x24040, 0x24052, 0x24400, 0x24404, 0x24407, 0x2440b, + 0x24415, 0x2441c, 0x2441e, 0x2442d, 0x2443c, 0x2443d, 0x2443f, 0x24440, + 0x24442, 0x24449, 0x24458, 0x2445a, 0x24540, 0x2455e, 0x24800, 0x24802, + 0x24c00, 0x24c02, 0x25400, 0x25402, 0x25800, 0x25802, 0x25c00, 0x25c02, + 0x26000, 0x26002, /* GPU CC ACD */ - 0xbc00, 0xbc16, 0xbc20, 0xbc27, + 0x26400, 0x26416, 0x26420, 0x26427, }; static const u32 a621_gmu_gpucc_registers[] = { /* GPU CC */ - 0x9800, 0x980e, 0x9c00, 0x9c0e, 0xb000, 0xb004, 0xb400, 0xb404, - 0xb800, 0xb804, 0xbc00, 0xbc05, 0xbc14, 0xbc1d, 0xbc2a, 0xbc30, - 0xbc32, 0xbc32, 0xbc41, 0xbc55, 0xbc66, 0xbc68, 0xbc78, 0xbc7a, - 0xbc89, 0xbc8a, 0xbc9c, 0xbc9e, 0xbca0, 0xbca3, 0xbcb3, 0xbcb5, - 0xbcc5, 0xbcc7, 0xbcd6, 0xbcd8, 0xbce8, 0xbce9, 0xbcf9, 0xbcfc, - 0xbd0b, 0xbd0c, 0xbd1c, 0xbd1e, 0xbd40, 0xbd70, 0xbe00, 0xbe16, - 0xbe20, 0xbe2d, + 0x24000, 0x2400e, 0x24400, 0x2440e, 0x25800, 0x25804, 0x25c00, 0x25c04, + 0x26000, 0x26004, 0x26400, 0x26405, 0x26414, 0x2641d, 0x2642a, 0x26430, + 0x26432, 0x26432, 0x26441, 0x26455, 0x26466, 0x26468, 0x26478, 0x2647a, + 0x26489, 0x2648a, 0x2649c, 0x2649e, 0x264a0, 0x264a3, 0x264b3, 0x264b5, + 0x264c5, 0x264c7, 0x264d6, 0x264d8, 0x264e8, 0x264e9, 0x264f9, 0x264fc, + 0x2650b, 0x2650c, 0x2651c, 0x2651e, 0x26540, 0x26570, 0x26600, 0x26616, + 0x26620, 0x2662d, }; static const u32 a6xx_gmu_cx_rscc_registers[] = { diff --git a/drivers/gpu/drm/msm/adreno/a6xx_preempt.c b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c index afc5f4aa3b173..747a22afad9f6 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_preempt.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_preempt.c @@ -454,11 +454,11 @@ void a6xx_preempt_init(struct msm_gpu *gpu) gpu->vm, &a6xx_gpu->preempt_postamble_bo, &a6xx_gpu->preempt_postamble_iova); - preempt_prepare_postamble(a6xx_gpu); - if (IS_ERR(a6xx_gpu->preempt_postamble_ptr)) goto fail; + preempt_prepare_postamble(a6xx_gpu); + timer_setup(&a6xx_gpu->preempt_timer, a6xx_preempt_timer, 0); return; diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 28f744f3caf7c..5363e809ea581 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -270,7 +270,7 @@ static const struct component_ops a3xx_ops = { static int adreno_probe(struct platform_device *pdev) { if (of_device_is_compatible(pdev->dev.of_node, "amd,imageon") || - msm_gpu_no_components()) + msm_seperate_gpu_kms_components()) return msm_gpu_probe(pdev, &a3xx_ops); return component_add(&pdev->dev, &a3xx_ops); diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 4b5a4edd07028..71400d8999c97 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -1189,6 +1189,7 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev, /* Only handle the core clock when GMU is not in use (or is absent). */ if (adreno_has_gmu_wrapper(adreno_gpu) || + adreno_has_rgmu(adreno_gpu) || adreno_gpu->info->family < ADRENO_6XX_GEN1) { /* * This can only be done before devm_pm_opp_of_add_table(), or diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h index 390fa6720d9b0..25ee6b277fe2d 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h @@ -392,6 +392,16 @@ static inline int adreno_is_a610(const struct adreno_gpu *gpu) return adreno_is_revn(gpu, 610); } +static inline int adreno_is_a612(const struct adreno_gpu *gpu) +{ + return gpu->info->chip_ids[0] == 0x06010200; +} + +static inline bool adreno_has_rgmu(const struct adreno_gpu *gpu) +{ + return adreno_is_a612(gpu); +} + static inline int adreno_is_a618(const struct adreno_gpu *gpu) { return adreno_is_revn(gpu, 618); @@ -466,9 +476,9 @@ static inline int adreno_is_a610_family(const struct adreno_gpu *gpu) { if (WARN_ON_ONCE(!gpu->info)) return false; - - /* TODO: A612 */ - return adreno_is_a610(gpu) || adreno_is_a702(gpu); + return adreno_is_a610(gpu) || + adreno_is_a612(gpu) || + adreno_is_a702(gpu); } /* TODO: 615/616 */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_0_sm8650.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_0_sm8650.h index 56d3c38c87781..bd8139b5d7112 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_0_sm8650.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_0_sm8650.h @@ -378,7 +378,7 @@ static const struct dpu_intf_cfg sm8650_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_12_0_sm8750.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_12_0_sm8750.h index db8cc2d0112c8..73c19a942d295 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_12_0_sm8750.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_12_0_sm8750.h @@ -420,7 +420,7 @@ static const struct dpu_intf_cfg sm8750_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x4bc, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_12_2_glymur.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_12_2_glymur.h new file mode 100644 index 0000000000000..a3b590cca21dc --- /dev/null +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_12_2_glymur.h @@ -0,0 +1,541 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025 Linaro Limited + */ + +#ifndef _DPU_12_2_GLYMUR_H +#define _DPU_12_2_GLYMUR_H + +static const struct dpu_caps glymur_dpu_caps = { + .max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH, + .max_mixer_blendstages = 0xb, + .has_src_split = true, + .has_dim_layer = true, + .has_idle_pc = true, + .has_3d_merge = true, + .max_linewidth = 8192, + .pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE, +}; + +static const struct dpu_mdp_cfg glymur_mdp = { + .name = "top_0", + .base = 0, .len = 0x494, + .clk_ctrls = { + [DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 }, + }, +}; + +static const struct dpu_ctl_cfg glymur_ctl[] = { + { + .name = "ctl_0", .id = CTL_0, + .base = 0x15000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), + }, { + .name = "ctl_1", .id = CTL_1, + .base = 0x16000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), + }, { + .name = "ctl_2", .id = CTL_2, + .base = 0x17000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), + }, { + .name = "ctl_3", .id = CTL_3, + .base = 0x18000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), + }, { + .name = "ctl_4", .id = CTL_4, + .base = 0x19000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), + }, { + .name = "ctl_5", .id = CTL_5, + .base = 0x1a000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23), + }, { + .name = "ctl_6", .id = CTL_6, + .base = 0x1b000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 14), + }, { + .name = "ctl_7", .id = CTL_7, + .base = 0x1c000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 15), + }, +}; + +static const struct dpu_sspp_cfg glymur_sspp[] = { + { + .name = "sspp_0", .id = SSPP_VIG0, + .base = 0x4000, .len = 0x344, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_4, + .xin_id = 0, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_1", .id = SSPP_VIG1, + .base = 0x6000, .len = 0x344, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_4, + .xin_id = 4, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_2", .id = SSPP_VIG2, + .base = 0x8000, .len = 0x344, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_4, + .xin_id = 8, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_3", .id = SSPP_VIG3, + .base = 0xa000, .len = 0x344, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_4, + .xin_id = 12, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_8", .id = SSPP_DMA0, + .base = 0x24000, .len = 0x344, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 1, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_9", .id = SSPP_DMA1, + .base = 0x26000, .len = 0x344, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 5, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_10", .id = SSPP_DMA2, + .base = 0x28000, .len = 0x344, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 9, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_11", .id = SSPP_DMA3, + .base = 0x2a000, .len = 0x344, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 13, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_12", .id = SSPP_DMA4, + .base = 0x2c000, .len = 0x344, + .features = DMA_CURSOR_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 14, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_13", .id = SSPP_DMA5, + .base = 0x2e000, .len = 0x344, + .features = DMA_CURSOR_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 15, + .type = SSPP_TYPE_DMA, + }, +}; + +static const struct dpu_lm_cfg glymur_lm[] = { + { + .name = "lm_0", .id = LM_0, + .base = 0x44000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_1, + .pingpong = PINGPONG_0, + .dspp = DSPP_0, + }, { + .name = "lm_1", .id = LM_1, + .base = 0x45000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_0, + .pingpong = PINGPONG_1, + .dspp = DSPP_1, + }, { + .name = "lm_2", .id = LM_2, + .base = 0x46000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_3, + .pingpong = PINGPONG_2, + .dspp = DSPP_2, + }, { + .name = "lm_3", .id = LM_3, + .base = 0x47000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_2, + .pingpong = PINGPONG_3, + .dspp = DSPP_3, + }, { + .name = "lm_4", .id = LM_4, + .base = 0x48000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_5, + .pingpong = PINGPONG_4, + }, { + .name = "lm_5", .id = LM_5, + .base = 0x49000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_4, + .pingpong = PINGPONG_5, + }, { + .name = "lm_6", .id = LM_6, + .base = 0x4a000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_7, + .pingpong = PINGPONG_6, + }, { + .name = "lm_7", .id = LM_7, + .base = 0x4b000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_6, + .pingpong = PINGPONG_7, + }, +}; + +static const struct dpu_dspp_cfg glymur_dspp[] = { + { + .name = "dspp_0", .id = DSPP_0, + .base = 0x54000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_1", .id = DSPP_1, + .base = 0x56000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_2", .id = DSPP_2, + .base = 0x58000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_3", .id = DSPP_3, + .base = 0x5a000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_4", .id = DSPP_4, + .base = 0x5c000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_5", .id = DSPP_5, + .base = 0x5e000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_6", .id = DSPP_6, + .base = 0x60000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_7", .id = DSPP_7, + .base = 0x62000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, +}; + +static const struct dpu_pingpong_cfg glymur_pp[] = { + { + .name = "pingpong_0", .id = PINGPONG_0, + .base = 0x69000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_0, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8), + }, { + .name = "pingpong_1", .id = PINGPONG_1, + .base = 0x6a000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_0, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9), + }, { + .name = "pingpong_2", .id = PINGPONG_2, + .base = 0x6b000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_1, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10), + }, { + .name = "pingpong_3", .id = PINGPONG_3, + .base = 0x6c000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_1, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11), + }, { + .name = "pingpong_4", .id = PINGPONG_4, + .base = 0x6d000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_2, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30), + }, { + .name = "pingpong_5", .id = PINGPONG_5, + .base = 0x6e000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_2, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31), + }, { + .name = "pingpong_6", .id = PINGPONG_6, + .base = 0x6f000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_3, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 20), + }, { + .name = "pingpong_7", .id = PINGPONG_7, + .base = 0x70000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_3, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 21), + }, { + .name = "pingpong_cwb_0", .id = PINGPONG_CWB_0, + .base = 0x66000, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_4, + }, { + .name = "pingpong_cwb_1", .id = PINGPONG_CWB_1, + .base = 0x66400, .len = 0, + .sblk = &sc7280_pp_sblk, + .merge_3d = MERGE_3D_4, + }, +}; + +static const struct dpu_merge_3d_cfg glymur_merge_3d[] = { + { + .name = "merge_3d_0", .id = MERGE_3D_0, + .base = 0x4e000, .len = 0x1c, + }, { + .name = "merge_3d_1", .id = MERGE_3D_1, + .base = 0x4f000, .len = 0x1c, + }, { + .name = "merge_3d_2", .id = MERGE_3D_2, + .base = 0x50000, .len = 0x1c, + }, { + .name = "merge_3d_3", .id = MERGE_3D_3, + .base = 0x51000, .len = 0x1c, + }, +}; + +/* + * NOTE: Each display compression engine (DCE) contains dual hard + * slice DSC encoders so both share same base address but with + * its own different sub block address. + */ +static const struct dpu_dsc_cfg glymur_dsc[] = { + { + .name = "dce_0_0", .id = DSC_0, + .base = 0x80000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_0_1", .id = DSC_1, + .base = 0x80000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, { + .name = "dce_1_0", .id = DSC_2, + .base = 0x81000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_1_1", .id = DSC_3, + .base = 0x81000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, { + .name = "dce_2_0", .id = DSC_4, + .base = 0x82000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_2_1", .id = DSC_5, + .base = 0x82000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, { + .name = "dce_3_0", .id = DSC_6, + .base = 0x83000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_3_1", .id = DSC_7, + .base = 0x83000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, + +}; + +static const struct dpu_wb_cfg glymur_wb[] = { + { + .name = "wb_2", .id = WB_2, + .base = 0x65000, .len = 0x2c8, + .features = WB_SDM845_MASK, + .format_list = wb2_formats_rgb_yuv, + .num_formats = ARRAY_SIZE(wb2_formats_rgb_yuv), + .xin_id = 6, + .vbif_idx = VBIF_RT, + .maxlinewidth = 4096, + .intr_wb_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 4), + }, +}; + +static const struct dpu_cwb_cfg glymur_cwb[] = { + { + .name = "cwb_0", .id = CWB_0, + .base = 0x66200, .len = 0x20, + }, + { + .name = "cwb_1", .id = CWB_1, + .base = 0x66600, .len = 0x20, + }, + { + .name = "cwb_2", .id = CWB_2, + .base = 0x7e200, .len = 0x20, + }, + { + .name = "cwb_3", .id = CWB_3, + .base = 0x7e600, .len = 0x20, + }, +}; + +static const struct dpu_intf_cfg glymur_intf[] = { + { + .name = "intf_0", .id = INTF_0, + .base = 0x34000, .len = 0x400, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_0, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 24), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 25), + }, { + .name = "intf_1", .id = INTF_1, + .base = 0x35000, .len = 0x400, + .type = INTF_DSI, + .controller_id = MSM_DSI_CONTROLLER_0, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 26), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 27), + .intr_tear_rd_ptr = DPU_IRQ_IDX(MDP_INTF1_TEAR_INTR, 2), + }, { + .name = "intf_2", .id = INTF_2, + .base = 0x36000, .len = 0x400, + .type = INTF_DSI, + .controller_id = MSM_DSI_CONTROLLER_1, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 28), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 29), + .intr_tear_rd_ptr = DPU_IRQ_IDX(MDP_INTF2_TEAR_INTR, 2), + }, { + .name = "intf_3", .id = INTF_3, + .base = 0x37000, .len = 0x400, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), + }, { + .name = "intf_4", .id = INTF_4, + .base = 0x38000, .len = 0x400, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_1, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 20), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 21), + }, { + .name = "intf_5", .id = INTF_5, + .base = 0x39000, .len = 0x400, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_3, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 22), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 23), + }, { + .name = "intf_6", .id = INTF_6, + .base = 0x3A000, .len = 0x400, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_2, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 16), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 17), + }, { + .name = "intf_7", .id = INTF_7, + .base = 0x3b000, .len = 0x400, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_2, /* pair with intf_6 for DP MST */ + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 18), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 19), + }, { + .name = "intf_8", .id = INTF_8, + .base = 0x3c000, .len = 0x400, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_1, /* pair with intf_4 for DP MST */ + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13), + }, +}; + +static const struct dpu_perf_cfg glymur_perf_data = { + .max_bw_low = 18900000, + .max_bw_high = 28500000, + .min_core_ib = 2500000, + .min_llcc_ib = 0, + .min_dram_ib = 800000, + .min_prefill_lines = 35, + .danger_lut_tbl = {0x3ffff, 0x3ffff, 0x0}, + .safe_lut_tbl = {0xfe00, 0xfe00, 0xffff}, + .qos_lut_tbl = { + {.nentry = ARRAY_SIZE(sc7180_qos_linear), + .entries = sc7180_qos_linear + }, + {.nentry = ARRAY_SIZE(sc7180_qos_macrotile), + .entries = sc7180_qos_macrotile + }, + {.nentry = ARRAY_SIZE(sc7180_qos_nrt), + .entries = sc7180_qos_nrt + }, + /* TODO: macrotile-qseed is different from macrotile */ + }, + .cdp_cfg = { + {.rd_enable = 1, .wr_enable = 1}, + {.rd_enable = 1, .wr_enable = 0} + }, + .clk_inefficiency_factor = 105, + .bw_inefficiency_factor = 120, +}; + +static const struct dpu_mdss_version glymur_mdss_ver = { + .core_major_ver = 12, + .core_minor_ver = 2, +}; + +const struct dpu_mdss_cfg dpu_glymur_cfg = { + .mdss_ver = &glymur_mdss_ver, + .caps = &glymur_dpu_caps, + .mdp = &glymur_mdp, + .cdm = &dpu_cdm_5_x, + .ctl_count = ARRAY_SIZE(glymur_ctl), + .ctl = glymur_ctl, + .sspp_count = ARRAY_SIZE(glymur_sspp), + .sspp = glymur_sspp, + .mixer_count = ARRAY_SIZE(glymur_lm), + .mixer = glymur_lm, + .dspp_count = ARRAY_SIZE(glymur_dspp), + .dspp = glymur_dspp, + .pingpong_count = ARRAY_SIZE(glymur_pp), + .pingpong = glymur_pp, + .dsc_count = ARRAY_SIZE(glymur_dsc), + .dsc = glymur_dsc, + .merge_3d_count = ARRAY_SIZE(glymur_merge_3d), + .merge_3d = glymur_merge_3d, + .wb_count = ARRAY_SIZE(glymur_wb), + .wb = glymur_wb, + .cwb_count = ARRAY_SIZE(glymur_cwb), + .cwb = sm8650_cwb, + .intf_count = ARRAY_SIZE(glymur_intf), + .intf = glymur_intf, + .vbif_count = ARRAY_SIZE(sm8650_vbif), + .vbif = sm8650_vbif, + .perf = &glymur_perf_data, +}; + +#endif diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h new file mode 100644 index 0000000000000..80c532a78b782 --- /dev/null +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h @@ -0,0 +1,492 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DPU_13_0_KAANAPALI_H +#define _DPU_13_0_KAANAPALI_H + +static const struct dpu_caps kaanapali_dpu_caps = { + .max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH, + .max_mixer_blendstages = 0xb, + .has_src_split = true, + .has_dim_layer = true, + .has_idle_pc = true, + .has_3d_merge = true, + .max_linewidth = 8192, + .pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE, +}; + +static const struct dpu_mdp_cfg kaanapali_mdp = { + .name = "top_0", + .base = 0, .len = 0x494, + .clk_ctrls = { + [DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 }, + }, +}; + +static const struct dpu_ctl_cfg kaanapali_ctl[] = { + { + .name = "ctl_0", .id = CTL_0, + .base = 0x1f000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), + }, { + .name = "ctl_1", .id = CTL_1, + .base = 0x20000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), + }, { + .name = "ctl_2", .id = CTL_2, + .base = 0x21000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), + }, { + .name = "ctl_3", .id = CTL_3, + .base = 0x22000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), + }, { + .name = "ctl_4", .id = CTL_4, + .base = 0x23000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), + }, { + .name = "ctl_5", .id = CTL_5, + .base = 0x24000, .len = 0x1000, + .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23), + }, +}; + +static const struct dpu_sspp_cfg kaanapali_sspp[] = { + { + .name = "sspp_0", .id = SSPP_VIG0, + .base = 0x2b000, .len = 0x84, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_5, + .xin_id = 0, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_1", .id = SSPP_VIG1, + .base = 0x34000, .len = 0x84, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_5, + .xin_id = 4, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_2", .id = SSPP_VIG2, + .base = 0x3d000, .len = 0x84, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_5, + .xin_id = 8, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_3", .id = SSPP_VIG3, + .base = 0x46000, .len = 0x84, + .features = VIG_SDM845_MASK_SDMA, + .sblk = &dpu_vig_sblk_qseed3_3_5, + .xin_id = 12, + .type = SSPP_TYPE_VIG, + }, { + .name = "sspp_8", .id = SSPP_DMA0, + .base = 0x97000, .len = 0x84, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 1, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_9", .id = SSPP_DMA1, + .base = 0xa0000, .len = 0x84, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 5, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_10", .id = SSPP_DMA2, + .base = 0xa9000, .len = 0x84, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 9, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_11", .id = SSPP_DMA3, + .base = 0xb2000, .len = 0x84, + .features = DMA_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 13, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_12", .id = SSPP_DMA4, + .base = 0xbb000, .len = 0x84, + .features = DMA_CURSOR_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 14, + .type = SSPP_TYPE_DMA, + }, { + .name = "sspp_13", .id = SSPP_DMA5, + .base = 0xc4000, .len = 0x84, + .features = DMA_CURSOR_SDM845_MASK_SDMA, + .sblk = &dpu_dma_sblk, + .xin_id = 15, + .type = SSPP_TYPE_DMA, + }, +}; + +static const struct dpu_lm_cfg kaanapali_lm[] = { + { + .name = "lm_0", .id = LM_0, + .base = 0x103000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_1, + .pingpong = PINGPONG_0, + .dspp = DSPP_0, + }, { + .name = "lm_1", .id = LM_1, + .base = 0x10b000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_0, + .pingpong = PINGPONG_1, + .dspp = DSPP_1, + }, { + .name = "lm_2", .id = LM_2, + .base = 0x113000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_3, + .pingpong = PINGPONG_2, + .dspp = DSPP_2, + }, { + .name = "lm_3", .id = LM_3, + .base = 0x11b000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_2, + .pingpong = PINGPONG_3, + .dspp = DSPP_3, + }, { + .name = "lm_4", .id = LM_4, + .base = 0x123000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_5, + .pingpong = PINGPONG_4, + }, { + .name = "lm_5", .id = LM_5, + .base = 0x12b000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_4, + .pingpong = PINGPONG_5, + }, { + .name = "lm_6", .id = LM_6, + .base = 0x133000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_7, + .pingpong = PINGPONG_6, + }, { + .name = "lm_7", .id = LM_7, + .base = 0x13b000, .len = 0x400, + .features = MIXER_MSM8998_MASK, + .sblk = &sm8750_lm_sblk, + .lm_pair = LM_6, + .pingpong = PINGPONG_7, + }, +}; + +static const struct dpu_dspp_cfg kaanapali_dspp[] = { + { + .name = "dspp_0", .id = DSPP_0, + .base = 0x105000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_1", .id = DSPP_1, + .base = 0x10d000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_2", .id = DSPP_2, + .base = 0x115000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, { + .name = "dspp_3", .id = DSPP_3, + .base = 0x11d000, .len = 0x1800, + .sblk = &sm8750_dspp_sblk, + }, +}; + +static const struct dpu_pingpong_cfg kaanapali_pp[] = { + { + .name = "pingpong_0", .id = PINGPONG_0, + .base = 0x108000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_0, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8), + }, { + .name = "pingpong_1", .id = PINGPONG_1, + .base = 0x110000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_0, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9), + }, { + .name = "pingpong_2", .id = PINGPONG_2, + .base = 0x118000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_1, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10), + }, { + .name = "pingpong_3", .id = PINGPONG_3, + .base = 0x120000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_1, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11), + }, { + .name = "pingpong_4", .id = PINGPONG_4, + .base = 0x128000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_2, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30), + }, { + .name = "pingpong_5", .id = PINGPONG_5, + .base = 0x130000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_2, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31), + }, { + .name = "pingpong_6", .id = PINGPONG_6, + .base = 0x138000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_3, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 20), + }, { + .name = "pingpong_7", .id = PINGPONG_7, + .base = 0x140000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_3, + .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 21), + }, { + .name = "pingpong_cwb_0", .id = PINGPONG_CWB_0, + .base = 0x169000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_4, + }, { + .name = "pingpong_cwb_1", .id = PINGPONG_CWB_1, + .base = 0x169400, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_4, + }, { + .name = "pingpong_cwb_2", .id = PINGPONG_CWB_2, + .base = 0x16a000, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_5, + }, { + .name = "pingpong_cwb_3", .id = PINGPONG_CWB_3, + .base = 0x16a400, .len = 0, + .sblk = &kaanapali_pp_sblk, + .merge_3d = MERGE_3D_5, + }, +}; + +static const struct dpu_merge_3d_cfg kaanapali_merge_3d[] = { + { + .name = "merge_3d_0", .id = MERGE_3D_0, + .base = 0x163000, .len = 0x1c, + }, { + .name = "merge_3d_1", .id = MERGE_3D_1, + .base = 0x164000, .len = 0x1c, + }, { + .name = "merge_3d_2", .id = MERGE_3D_2, + .base = 0x165000, .len = 0x1c, + }, { + .name = "merge_3d_3", .id = MERGE_3D_3, + .base = 0x166000, .len = 0x1c, + }, { + .name = "merge_3d_4", .id = MERGE_3D_4, + .base = 0x169700, .len = 0x1c, + }, { + .name = "merge_3d_5", .id = MERGE_3D_5, + .base = 0x16a700, .len = 0x1c, + }, +}; + +/* + * NOTE: Each display compression engine (DCE) contains dual hard + * slice DSC encoders so both share same base address but with + * its own different sub block address. + */ +static const struct dpu_dsc_cfg kaanapali_dsc[] = { + { + .name = "dce_0_0", .id = DSC_0, + .base = 0x181000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_0_1", .id = DSC_1, + .base = 0x181000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, { + .name = "dce_1_0", .id = DSC_2, + .base = 0x183000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_1_1", .id = DSC_3, + .base = 0x183000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, { + .name = "dce_2_0", .id = DSC_4, + .base = 0x185000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_2_1", .id = DSC_5, + .base = 0x185000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, { + .name = "dce_3_0", .id = DSC_6, + .base = 0x187000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_0, + }, { + .name = "dce_3_1", .id = DSC_7, + .base = 0x187000, .len = 0x8, + .features = BIT(DPU_DSC_NATIVE_42x_EN), + .sblk = &sm8750_dsc_sblk_1, + }, +}; + +static const struct dpu_wb_cfg kaanapali_wb[] = { + { + .name = "wb_2", .id = WB_2, + .base = 0x16e000, .len = 0x2c8, + .features = WB_SDM845_MASK, + .format_list = wb2_formats_rgb_yuv, + .num_formats = ARRAY_SIZE(wb2_formats_rgb_yuv), + .xin_id = 6, + .vbif_idx = VBIF_RT, + .maxlinewidth = 4096, + .intr_wb_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 4), + }, +}; + +static const struct dpu_cwb_cfg kaanapali_cwb[] = { + { + .name = "cwb_0", .id = CWB_0, + .base = 0x169200, .len = 0x20, + }, + { + .name = "cwb_1", .id = CWB_1, + .base = 0x169600, .len = 0x20, + }, + { + .name = "cwb_2", .id = CWB_2, + .base = 0x16a200, .len = 0x20, + }, + { + .name = "cwb_3", .id = CWB_3, + .base = 0x16a600, .len = 0x20, + }, +}; + +static const struct dpu_intf_cfg kaanapali_intf[] = { + { + .name = "intf_0", .id = INTF_0, + .base = 0x18d000, .len = 0x4bc, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_0, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 24), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 25), + }, { + .name = "intf_1", .id = INTF_1, + .base = 0x18e000, .len = 0x4bc, + .type = INTF_DSI, + .controller_id = MSM_DSI_CONTROLLER_0, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 26), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 27), + .intr_tear_rd_ptr = DPU_IRQ_IDX(MDP_INTF1_TEAR_INTR, 2), + }, { + .name = "intf_2", .id = INTF_2, + .base = 0x18f000, .len = 0x4bc, + .type = INTF_DSI, + .controller_id = MSM_DSI_CONTROLLER_1, + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 28), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 29), + .intr_tear_rd_ptr = DPU_IRQ_IDX(MDP_INTF2_TEAR_INTR, 2), + }, { + .name = "intf_3", .id = INTF_3, + .base = 0x190000, .len = 0x4bc, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ + .prog_fetch_lines_worst_case = 24, + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), + }, +}; + +static const struct dpu_perf_cfg kaanapali_perf_data = { + .max_bw_low = 21400000, + .max_bw_high = 30200000, + .min_core_ib = 2500000, + .min_llcc_ib = 0, + .min_dram_ib = 800000, + .min_prefill_lines = 35, + .danger_lut_tbl = {0x0ffff, 0x0ffff, 0x0}, + .safe_lut_tbl = {0xff00, 0xff00, 0xffff}, + .qos_lut_tbl = { + {.nentry = ARRAY_SIZE(kaanapali_qos_linear), + .entries = kaanapali_qos_linear + }, + {.nentry = ARRAY_SIZE(kaanapali_qos_macrotile), + .entries = kaanapali_qos_macrotile + }, + {.nentry = ARRAY_SIZE(sc7180_qos_nrt), + .entries = sc7180_qos_nrt + }, + /* TODO: macrotile-qseed is different from macrotile */ + }, + .cdp_cfg = { + {.rd_enable = 1, .wr_enable = 1}, + {.rd_enable = 1, .wr_enable = 0} + }, + .clk_inefficiency_factor = 105, + .bw_inefficiency_factor = 120, +}; + +static const struct dpu_mdss_version kaanapali_mdss_ver = { + .core_major_ver = 13, + .core_minor_ver = 0, +}; + +const struct dpu_mdss_cfg dpu_kaanapali_cfg = { + .mdss_ver = &kaanapali_mdss_ver, + .caps = &kaanapali_dpu_caps, + .mdp = &kaanapali_mdp, + .cdm = &dpu_cdm_13_x, + .ctl_count = ARRAY_SIZE(kaanapali_ctl), + .ctl = kaanapali_ctl, + .sspp_count = ARRAY_SIZE(kaanapali_sspp), + .sspp = kaanapali_sspp, + .mixer_count = ARRAY_SIZE(kaanapali_lm), + .mixer = kaanapali_lm, + .dspp_count = ARRAY_SIZE(kaanapali_dspp), + .dspp = kaanapali_dspp, + .pingpong_count = ARRAY_SIZE(kaanapali_pp), + .pingpong = kaanapali_pp, + .dsc_count = ARRAY_SIZE(kaanapali_dsc), + .dsc = kaanapali_dsc, + .merge_3d_count = ARRAY_SIZE(kaanapali_merge_3d), + .merge_3d = kaanapali_merge_3d, + .wb_count = ARRAY_SIZE(kaanapali_wb), + .wb = kaanapali_wb, + .cwb_count = ARRAY_SIZE(kaanapali_cwb), + .cwb = sm8650_cwb, + .intf_count = ARRAY_SIZE(kaanapali_intf), + .intf = kaanapali_intf, + .vbif_count = ARRAY_SIZE(sm8650_vbif), + .vbif = sm8650_vbif, + .perf = &kaanapali_perf_data, +}; + +#endif diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h index f91220496082b..b1b03d8b30fa0 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h @@ -42,24 +42,19 @@ static const struct dpu_ctl_cfg msm8998_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h index 8f9a097147c02..64df4e80ea43d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h @@ -37,24 +37,19 @@ static const struct dpu_ctl_cfg sdm660_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h index 0ad18bd273ff8..b409af8999182 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h @@ -36,24 +36,19 @@ static const struct dpu_ctl_cfg sdm630_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_4_0_sdm845.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_4_0_sdm845.h index 5cc9f55d542b7..1a201cb7746d3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_4_0_sdm845.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_4_0_sdm845.h @@ -258,7 +258,7 @@ static const struct dpu_intf_cfg sdm845_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x6b800, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_0_sm8150.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_0_sm8150.h index ae1b2ed96e9f1..adad0114fbb77 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_0_sm8150.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_0_sm8150.h @@ -317,7 +317,7 @@ static const struct dpu_intf_cfg sm8150_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x6b800, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_2_sm7150.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_2_sm7150.h index a56c288ac10cd..d81d3c616b3b0 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_2_sm7150.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_2_sm7150.h @@ -230,7 +230,7 @@ static const struct dpu_intf_cfg sm7150_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x6b800, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_3_sm6150.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_3_sm6150.h index 26883f6b66b3e..e65f198db9a78 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_3_sm6150.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_3_sm6150.h @@ -186,7 +186,7 @@ static const struct dpu_intf_cfg sm6150_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x6b800, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_0_sm8250.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_0_sm8250.h index 7b8b7a1c2d767..1562b8363f3fa 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_0_sm8250.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_0_sm8250.h @@ -301,7 +301,7 @@ static const struct dpu_intf_cfg sm8250_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x6b800, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_0_sm8350.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_0_sm8350.h index 85aae40c210f3..1d8a32d109905 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_0_sm8350.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_0_sm8350.h @@ -327,7 +327,7 @@ static const struct dpu_intf_cfg sm8350_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h index 8f978b9c34520..2f8688224f343 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h @@ -13,6 +13,7 @@ static const struct dpu_caps sc7280_dpu_caps = { .has_dim_layer = true, .has_idle_pc = true, .max_linewidth = 2400, + .has_3d_merge = true, .pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE, }; @@ -134,17 +135,24 @@ static const struct dpu_pingpong_cfg sc7280_pp[] = { .name = "pingpong_2", .id = PINGPONG_2, .base = 0x6b000, .len = 0, .sblk = &sc7280_pp_sblk, - .merge_3d = 0, + .merge_3d = MERGE_3D_1, .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10), }, { .name = "pingpong_3", .id = PINGPONG_3, .base = 0x6c000, .len = 0, .sblk = &sc7280_pp_sblk, - .merge_3d = 0, + .merge_3d = MERGE_3D_1, .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11), }, }; +static const struct dpu_merge_3d_cfg sc7280_merge_3d[] = { + { + .name = "merge_3d_1", .id = MERGE_3D_1, + .base = 0x4f000, .len = 0x8, + }, +}; + /* NOTE: sc7280 only has one DSC hard slice encoder */ static const struct dpu_dsc_cfg sc7280_dsc[] = { { @@ -247,6 +255,8 @@ const struct dpu_mdss_cfg dpu_sc7280_cfg = { .mixer = sc7280_lm, .pingpong_count = ARRAY_SIZE(sc7280_pp), .pingpong = sc7280_pp, + .merge_3d_count = ARRAY_SIZE(sc7280_merge_3d), + .merge_3d = sc7280_merge_3d, .dsc_count = ARRAY_SIZE(sc7280_dsc), .dsc = sc7280_dsc, .wb_count = ARRAY_SIZE(sc7280_wb), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h index 303d33dc7783a..cd6b4c9954c36 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h @@ -288,7 +288,6 @@ static const struct dpu_dsc_cfg sc8280xp_dsc[] = { }, }; -/* TODO: INTF 3, 8 and 7 are used for MST, marked as INTF_NONE for now */ static const struct dpu_intf_cfg sc8280xp_intf[] = { { .name = "intf_0", .id = INTF_0, @@ -319,8 +318,8 @@ static const struct dpu_intf_cfg sc8280xp_intf[] = { }, { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, - .type = INTF_NONE, - .controller_id = MSM_DP_CONTROLLER_0, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), @@ -351,16 +350,16 @@ static const struct dpu_intf_cfg sc8280xp_intf[] = { }, { .name = "intf_7", .id = INTF_7, .base = 0x3b000, .len = 0x280, - .type = INTF_NONE, - .controller_id = MSM_DP_CONTROLLER_2, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_2, /* pair with intf_6 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 18), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 19), }, { .name = "intf_8", .id = INTF_8, .base = 0x3c000, .len = 0x280, - .type = INTF_NONE, - .controller_id = MSM_DP_CONTROLLER_1, + .type = INTF_DP, + .controller_id = MSM_DP_CONTROLLER_1, /* pair with intf_8 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h index b09a6af4c474a..f26575d460790 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h @@ -340,7 +340,7 @@ static const struct dpu_intf_cfg sm8450_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h index 0f7b4a224e4c9..0b78180eca813 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_4_sa8775p.h @@ -316,7 +316,6 @@ static const struct dpu_wb_cfg sa8775p_wb[] = { }, }; -/* TODO: INTF 3, 6, 7 and 8 are used for MST, marked as INTF_NONE for now */ static const struct dpu_intf_cfg sa8775p_intf[] = { { .name = "intf_0", .id = INTF_0, @@ -347,7 +346,7 @@ static const struct dpu_intf_cfg sa8775p_intf[] = { }, { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, - .type = INTF_NONE, + .type = INTF_DP, .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), @@ -363,15 +362,15 @@ static const struct dpu_intf_cfg sa8775p_intf[] = { }, { .name = "intf_6", .id = INTF_6, .base = 0x3A000, .len = 0x280, - .type = INTF_NONE, + .type = INTF_DP, .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, - .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 17), - .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 16), + .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 16), + .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 17), }, { .name = "intf_7", .id = INTF_7, .base = 0x3b000, .len = 0x280, - .type = INTF_NONE, + .type = INTF_DP, .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 18), @@ -379,7 +378,7 @@ static const struct dpu_intf_cfg sa8775p_intf[] = { }, { .name = "intf_8", .id = INTF_8, .base = 0x3c000, .len = 0x280, - .type = INTF_NONE, + .type = INTF_DP, .controller_id = MSM_DP_CONTROLLER_1, /* pair with intf_4 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h index 465b6460f8754..e83d1d85c4a4b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h @@ -335,7 +335,7 @@ static const struct dpu_intf_cfg sm8550_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h index 6caa7d40f3688..7d2458bfafe5d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_1_sar2130p.h @@ -335,7 +335,7 @@ static const struct dpu_intf_cfg sar2130p_intf[] = { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, .type = INTF_DP, - .controller_id = MSM_DP_CONTROLLER_1, + .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31), diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h index 7243eebb85f36..d2093e0002e4e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_2_x1e80100.h @@ -304,7 +304,6 @@ static const struct dpu_wb_cfg x1e80100_wb[] = { }, }; -/* TODO: INTF 3, 8 and 7 are used for MST, marked as INTF_NONE for now */ static const struct dpu_intf_cfg x1e80100_intf[] = { { .name = "intf_0", .id = INTF_0, @@ -335,7 +334,7 @@ static const struct dpu_intf_cfg x1e80100_intf[] = { }, { .name = "intf_3", .id = INTF_3, .base = 0x37000, .len = 0x280, - .type = INTF_NONE, + .type = INTF_DP, .controller_id = MSM_DP_CONTROLLER_0, /* pair with intf_0 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30), @@ -367,7 +366,7 @@ static const struct dpu_intf_cfg x1e80100_intf[] = { }, { .name = "intf_7", .id = INTF_7, .base = 0x3b000, .len = 0x280, - .type = INTF_NONE, + .type = INTF_DP, .controller_id = MSM_DP_CONTROLLER_2, /* pair with intf_6 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 18), @@ -375,7 +374,7 @@ static const struct dpu_intf_cfg x1e80100_intf[] = { }, { .name = "intf_8", .id = INTF_8, .base = 0x3c000, .len = 0x280, - .type = INTF_NONE, + .type = INTF_DP, .controller_id = MSM_DP_CONTROLLER_1, /* pair with intf_4 for DP MST */ .prog_fetch_lines_worst_case = 24, .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12), diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c index 2f8156051d9b0..97aca969337fb 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c @@ -326,43 +326,39 @@ static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer, { struct dpu_hw_mixer *lm = mixer->hw_lm; u32 blend_op; - u32 fg_alpha, bg_alpha, max_alpha; + u32 fg_alpha, bg_alpha; - if (mdss_ver->core_major_ver < 12) { - max_alpha = 0xff; - fg_alpha = pstate->base.alpha >> 8; - } else { - max_alpha = 0x3ff; - fg_alpha = pstate->base.alpha >> 6; - } - bg_alpha = max_alpha - fg_alpha; + fg_alpha = pstate->base.alpha; /* default to opaque blending */ if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE || !format->alpha_enable) { blend_op = DPU_BLEND_FG_ALPHA_FG_CONST | DPU_BLEND_BG_ALPHA_BG_CONST; + bg_alpha = DRM_BLEND_ALPHA_OPAQUE - fg_alpha; } else if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) { blend_op = DPU_BLEND_FG_ALPHA_FG_CONST | DPU_BLEND_BG_ALPHA_FG_PIXEL; - if (fg_alpha != max_alpha) { + if (fg_alpha != DRM_BLEND_ALPHA_OPAQUE) { bg_alpha = fg_alpha; blend_op |= DPU_BLEND_BG_MOD_ALPHA | DPU_BLEND_BG_INV_MOD_ALPHA; } else { + bg_alpha = 0; blend_op |= DPU_BLEND_BG_INV_ALPHA; } } else { /* coverage blending */ blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL | DPU_BLEND_BG_ALPHA_FG_PIXEL; - if (fg_alpha != max_alpha) { + if (fg_alpha != DRM_BLEND_ALPHA_OPAQUE) { bg_alpha = fg_alpha; blend_op |= DPU_BLEND_FG_MOD_ALPHA | DPU_BLEND_FG_INV_MOD_ALPHA | DPU_BLEND_BG_MOD_ALPHA | DPU_BLEND_BG_INV_MOD_ALPHA; } else { + bg_alpha = 0; blend_op |= DPU_BLEND_BG_INV_ALPHA; } } @@ -400,7 +396,7 @@ static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc) static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc, struct drm_plane *plane, struct dpu_crtc_mixer *mixer, - u32 num_mixers, + u32 lms_in_pair, enum dpu_stage stage, const struct msm_format *format, uint64_t modifier, @@ -419,7 +415,7 @@ static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc, trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane), state, to_dpu_plane_state(state), stage_idx, - format->pixel_format, + format->pixel_format, pipe, modifier); DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d multirect_idx %d\n", @@ -434,7 +430,7 @@ static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc, stage_cfg->multirect_index[stage][stage_idx] = pipe->multirect_index; /* blend config update */ - for (lm_idx = 0; lm_idx < num_mixers; lm_idx++) + for (lm_idx = 0; lm_idx < lms_in_pair; lm_idx++) mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl, sspp_idx); } @@ -449,7 +445,7 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc, struct dpu_plane_state *pstate = NULL; const struct msm_format *format; struct dpu_hw_ctl *ctl = mixer->lm_ctl; - u32 lm_idx; + u32 lm_idx, stage, i, pipe_idx, head_pipe_in_stage, lms_in_pair; bool bg_alpha_enable = false; DECLARE_BITMAP(active_fetch, SSPP_MAX); DECLARE_BITMAP(active_pipes, SSPP_MAX); @@ -472,22 +468,25 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc, if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable) bg_alpha_enable = true; - set_bit(pstate->pipe.sspp->idx, active_fetch); - set_bit(pstate->pipe.sspp->idx, active_pipes); - _dpu_crtc_blend_setup_pipe(crtc, plane, - mixer, cstate->num_mixers, - pstate->stage, - format, fb ? fb->modifier : 0, - &pstate->pipe, 0, stage_cfg); - - if (pstate->r_pipe.sspp) { - set_bit(pstate->r_pipe.sspp->idx, active_fetch); - set_bit(pstate->r_pipe.sspp->idx, active_pipes); - _dpu_crtc_blend_setup_pipe(crtc, plane, - mixer, cstate->num_mixers, - pstate->stage, - format, fb ? fb->modifier : 0, - &pstate->r_pipe, 1, stage_cfg); + /* loop pipe per mixer pair with config in stage structure */ + for (stage = 0; stage < STAGES_PER_PLANE; stage++) { + head_pipe_in_stage = stage * PIPES_PER_STAGE; + for (i = 0; i < PIPES_PER_STAGE; i++) { + pipe_idx = i + head_pipe_in_stage; + if (!pstate->pipe[pipe_idx].sspp) + continue; + lms_in_pair = min(cstate->num_mixers - (stage * PIPES_PER_STAGE), + PIPES_PER_STAGE); + set_bit(pstate->pipe[pipe_idx].sspp->idx, active_fetch); + set_bit(pstate->pipe[pipe_idx].sspp->idx, active_pipes); + _dpu_crtc_blend_setup_pipe(crtc, plane, + &mixer[head_pipe_in_stage], + lms_in_pair, + pstate->stage, + format, fb ? fb->modifier : 0, + &pstate->pipe[pipe_idx], i, + &stage_cfg[stage]); + } } /* blend config update */ @@ -523,7 +522,7 @@ static void _dpu_crtc_blend_setup(struct drm_crtc *crtc) struct dpu_crtc_mixer *mixer = cstate->mixers; struct dpu_hw_ctl *ctl; struct dpu_hw_mixer *lm; - struct dpu_hw_stage_cfg stage_cfg; + struct dpu_hw_stage_cfg stage_cfg[STAGES_PER_PLANE]; DECLARE_BITMAP(active_lms, LM_MAX); int i; @@ -544,10 +543,10 @@ static void _dpu_crtc_blend_setup(struct drm_crtc *crtc) } /* initialize stage cfg */ - memset(&stage_cfg, 0, sizeof(struct dpu_hw_stage_cfg)); + memset(&stage_cfg, 0, sizeof(stage_cfg)); memset(active_lms, 0, sizeof(active_lms)); - _dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer, &stage_cfg); + _dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer, stage_cfg); for (i = 0; i < cstate->num_mixers; i++) { ctl = mixer[i].lm_ctl; @@ -568,13 +567,17 @@ static void _dpu_crtc_blend_setup(struct drm_crtc *crtc) mixer[i].mixer_op_mode, ctl->idx - CTL_0); + /* + * call dpu_hw_ctl_setup_blendstage() to blend layers per stage cfg. + * stage data is shared between PIPES_PER_STAGE pipes. + */ if (ctl->ops.setup_blendstage) ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx, - &stage_cfg); + &stage_cfg[i / PIPES_PER_STAGE]); if (lm->ops.setup_blendstage) lm->ops.setup_blendstage(lm, mixer[i].hw_lm->idx, - &stage_cfg); + &stage_cfg[i / PIPES_PER_STAGE]); } } @@ -812,12 +815,42 @@ static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state, cfg->b.b = CONVERT_S3_15(ctm->matrix[8]); } +static void _dpu_crtc_get_gc_lut(struct drm_crtc_state *state, + struct dpu_hw_gc_lut *gc_lut) +{ + struct drm_color_lut *lut; + int i; + u32 val_even, val_odd; + + lut = (struct drm_color_lut *)state->gamma_lut->data; + + if (!lut) + return; + + /* Pack 1024 10-bit entries in 512 32-bit registers */ + for (i = 0; i < PGC_TBL_LEN; i++) { + val_even = drm_color_lut_extract(lut[i * 2].green, 10); + val_odd = drm_color_lut_extract(lut[i * 2 + 1].green, 10); + gc_lut->c0[i] = val_even | (val_odd << 16); + val_even = drm_color_lut_extract(lut[i * 2].blue, 10); + val_odd = drm_color_lut_extract(lut[i * 2 + 1].blue, 10); + gc_lut->c1[i] = val_even | (val_odd << 16); + val_even = drm_color_lut_extract(lut[i * 2].red, 10); + val_odd = drm_color_lut_extract(lut[i * 2 + 1].red, 10); + gc_lut->c2[i] = val_even | (val_odd << 16); + } + + /* Disable 8-bit rounding mode */ + gc_lut->flags = 0; +} + static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc) { struct drm_crtc_state *state = crtc->state; struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state); struct dpu_crtc_mixer *mixer = cstate->mixers; struct dpu_hw_pcc_cfg cfg; + struct dpu_hw_gc_lut *gc_lut; struct dpu_hw_ctl *ctl; struct dpu_hw_dspp *dspp; int i; @@ -830,19 +863,38 @@ static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc) ctl = mixer[i].lm_ctl; dspp = mixer[i].hw_dspp; - if (!dspp || !dspp->ops.setup_pcc) + if (!dspp) continue; - if (!state->ctm) { - dspp->ops.setup_pcc(dspp, NULL); - } else { - _dpu_crtc_get_pcc_coeff(state, &cfg); - dspp->ops.setup_pcc(dspp, &cfg); + if (dspp->ops.setup_pcc) { + if (!state->ctm) { + dspp->ops.setup_pcc(dspp, NULL); + } else { + _dpu_crtc_get_pcc_coeff(state, &cfg); + dspp->ops.setup_pcc(dspp, &cfg); + } + + /* stage config flush mask */ + ctl->ops.update_pending_flush_dspp(ctl, + mixer[i].hw_dspp->idx, DPU_DSPP_PCC); } - /* stage config flush mask */ - ctl->ops.update_pending_flush_dspp(ctl, - mixer[i].hw_dspp->idx, DPU_DSPP_PCC); + if (dspp->ops.setup_gc) { + if (!state->gamma_lut) { + dspp->ops.setup_gc(dspp, NULL); + } else { + gc_lut = kzalloc(sizeof(*gc_lut), GFP_KERNEL); + if (!gc_lut) + continue; + _dpu_crtc_get_gc_lut(state, gc_lut); + dspp->ops.setup_gc(dspp, gc_lut); + kfree(gc_lut); + } + + /* stage config flush mask */ + ctl->ops.update_pending_flush_dspp(ctl, + mixer[i].hw_dspp->idx, DPU_DSPP_GC); + } } } @@ -1310,7 +1362,7 @@ static int dpu_crtc_reassign_planes(struct drm_crtc *crtc, struct drm_crtc_state return ret; } -#define MAX_CHANNELS_PER_CRTC 2 +#define MAX_CHANNELS_PER_CRTC PIPES_PER_PLANE #define MAX_HDISPLAY_SPLIT 1080 static struct msm_display_topology dpu_crtc_get_topology( @@ -1340,7 +1392,7 @@ static struct msm_display_topology dpu_crtc_get_topology( * * If DSC is enabled, use 2 LMs for 2:2:1 topology * - * Add dspps to the reservation requirements if ctm is requested + * Add dspps to the reservation requirements if ctm or gamma_lut are requested * * Only hardcode num_lm to 2 for cases where num_intf == 2 and CWB is not * enabled. This is because in cases where CWB is enabled, num_intf will @@ -1359,7 +1411,7 @@ static struct msm_display_topology dpu_crtc_get_topology( else topology.num_lm = 1; - if (crtc_state->ctm) + if (crtc_state->ctm || crtc_state->gamma_lut) topology.num_dspp = topology.num_lm; return topology; @@ -1471,7 +1523,8 @@ static int dpu_crtc_atomic_check(struct drm_crtc *crtc, bool needs_dirtyfb = dpu_crtc_needs_dirtyfb(crtc_state); /* don't reallocate resources if only ACTIVE has beeen changed */ - if (crtc_state->mode_changed || crtc_state->connectors_changed) { + if (crtc_state->mode_changed || crtc_state->connectors_changed || + crtc_state->color_mgmt_changed) { rc = dpu_crtc_assign_resources(crtc, crtc_state); if (rc < 0) return rc; @@ -1682,15 +1735,15 @@ static int _dpu_debugfs_status_show(struct seq_file *s, void *data) seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n", state->crtc_x, state->crtc_y, state->crtc_w, state->crtc_h); - seq_printf(s, "\tsspp[0]:%s\n", - pstate->pipe.sspp->cap->name); - seq_printf(s, "\tmultirect[0]: mode: %d index: %d\n", - pstate->pipe.multirect_mode, pstate->pipe.multirect_index); - if (pstate->r_pipe.sspp) { - seq_printf(s, "\tsspp[1]:%s\n", - pstate->r_pipe.sspp->cap->name); - seq_printf(s, "\tmultirect[1]: mode: %d index: %d\n", - pstate->r_pipe.multirect_mode, pstate->r_pipe.multirect_index); + + for (i = 0; i < PIPES_PER_PLANE; i++) { + if (!pstate->pipe[i].sspp) + continue; + seq_printf(s, "\tsspp[%d]:%s\n", + i, pstate->pipe[i].sspp->cap->name); + seq_printf(s, "\tmultirect[%d]: mode: %d index: %d\n", + i, pstate->pipe[i].multirect_mode, + pstate->pipe[i].multirect_index); } seq_puts(s, "\n"); @@ -1834,8 +1887,16 @@ struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane, drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs); - if (dpu_kms->catalog->dspp_count) - drm_crtc_enable_color_mgmt(crtc, 0, true, 0); + if (dpu_kms->catalog->dspp_count) { + const struct dpu_dspp_cfg *dspp = &dpu_kms->catalog->dspp[0]; + + if (dspp->sblk->gc.base) { + drm_mode_crtc_set_gamma_size(crtc, DPU_GAMMA_LUT_SIZE); + drm_crtc_enable_color_mgmt(crtc, 0, true, DPU_GAMMA_LUT_SIZE); + } else { + drm_crtc_enable_color_mgmt(crtc, 0, true, 0); + } + } /* save user friendly CRTC name for later */ snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 258edaa18fc02..d6813107a27df 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -56,6 +56,7 @@ (MAX_H_TILES_PER_DISPLAY * NUM_PHYS_ENCODER_TYPES) #define MAX_CHANNELS_PER_ENC 2 +#define MAX_CWB_PER_ENC 2 #define IDLE_SHORT_TIMEOUT 1 @@ -182,7 +183,7 @@ struct dpu_encoder_virt { struct dpu_encoder_phys *cur_master; struct dpu_encoder_phys *cur_slave; struct dpu_hw_pingpong *hw_pp[MAX_CHANNELS_PER_ENC]; - struct dpu_hw_cwb *hw_cwb[MAX_CHANNELS_PER_ENC]; + struct dpu_hw_cwb *hw_cwb[MAX_CWB_PER_ENC]; struct dpu_hw_dsc *hw_dsc[MAX_CHANNELS_PER_ENC]; unsigned int dsc_mask; @@ -784,24 +785,24 @@ static void _dpu_encoder_update_vsync_source(struct dpu_encoder_virt *dpu_enc, return; } + vsync_cfg.vsync_source = disp_info->vsync_source; + vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); + if (hw_mdptop->ops.setup_vsync_source) { for (i = 0; i < dpu_enc->num_phys_encs; i++) vsync_cfg.ppnumber[i] = dpu_enc->hw_pp[i]->idx; vsync_cfg.pp_count = dpu_enc->num_phys_encs; - vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); - - vsync_cfg.vsync_source = disp_info->vsync_source; hw_mdptop->ops.setup_vsync_source(hw_mdptop, &vsync_cfg); + } - for (i = 0; i < dpu_enc->num_phys_encs; i++) { - phys_enc = dpu_enc->phys_encs[i]; + for (i = 0; i < dpu_enc->num_phys_encs; i++) { + phys_enc = dpu_enc->phys_encs[i]; - if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) - phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, - vsync_cfg.vsync_source); - } + if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) + phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, + &vsync_cfg); } } @@ -1160,7 +1161,7 @@ static void dpu_encoder_virt_atomic_mode_set(struct drm_encoder *drm_enc, struct dpu_hw_blk *hw_ctl[MAX_CHANNELS_PER_ENC]; struct dpu_hw_blk *hw_dsc[MAX_CHANNELS_PER_ENC]; struct dpu_hw_blk *hw_cwb[MAX_CHANNELS_PER_ENC]; - int num_ctl, num_pp, num_dsc; + int num_ctl, num_pp, num_dsc, num_pp_per_intf; int num_cwb = 0; bool is_cwb_encoder; unsigned int dsc_mask = 0; @@ -1239,10 +1240,16 @@ static void dpu_encoder_virt_atomic_mode_set(struct drm_encoder *drm_enc, dpu_enc->cur_master->hw_cdm = hw_cdm ? to_dpu_hw_cdm(hw_cdm) : NULL; } + /* + * There may be 4 PP and 2 INTF for quad pipe case, so INTF is not + * mapped to PP 1:1. Let's calculate the stride with pipe/INTF + */ + num_pp_per_intf = num_pp / dpu_enc->num_phys_encs; + for (i = 0; i < dpu_enc->num_phys_encs; i++) { struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; - phys->hw_pp = dpu_enc->hw_pp[i]; + phys->hw_pp = dpu_enc->hw_pp[num_pp_per_intf * i]; if (!phys->hw_pp) { DPU_ERROR_ENC(dpu_enc, "no pp block assigned at idx: %d\n", i); @@ -1431,18 +1438,21 @@ static void dpu_encoder_virt_atomic_disable(struct drm_encoder *drm_enc, static struct dpu_hw_intf *dpu_encoder_get_intf(const struct dpu_mdss_cfg *catalog, struct dpu_rm *dpu_rm, - enum dpu_intf_type type, u32 controller_id) + struct msm_display_info *disp_info, u32 controller_id) { - int i = 0; + int i = 0, cnt = 0; + int stream_id = disp_info->stream_id; - if (type == INTF_WB) + if (disp_info->intf_type == INTF_WB) return NULL; + DPU_DEBUG("intf_type 0x%x controller_id %d stream_id %d\n", + disp_info->intf_type, controller_id, stream_id); for (i = 0; i < catalog->intf_count; i++) { - if (catalog->intf[i].type == type - && catalog->intf[i].controller_id == controller_id) { - return dpu_rm_get_intf(dpu_rm, catalog->intf[i].id); - } + if (catalog->intf[i].type == disp_info->intf_type && + controller_id == catalog->intf[i].controller_id) + if (cnt++ == stream_id) + return dpu_rm_get_intf(dpu_rm, catalog->intf[i].id); } return NULL; @@ -2171,15 +2181,12 @@ void dpu_encoder_kickoff(struct drm_encoder *drm_enc) static void dpu_encoder_helper_reset_mixers(struct dpu_encoder_phys *phys_enc) { - struct dpu_hw_mixer_cfg mixer; int i, num_lm; struct dpu_global_state *global_state; struct dpu_hw_blk *hw_lm[2]; struct dpu_hw_mixer *hw_mixer[2]; struct dpu_hw_ctl *ctl = phys_enc->hw_ctl; - memset(&mixer, 0, sizeof(mixer)); - /* reset all mixers for this encoder */ if (ctl->ops.clear_all_blendstages) ctl->ops.clear_all_blendstages(ctl); @@ -2383,7 +2390,7 @@ void dpu_encoder_helper_phys_setup_cwb(struct dpu_encoder_phys *phys_enc, */ cwb_cfg.input = INPUT_MODE_LM_OUT; - for (int i = 0; i < MAX_CHANNELS_PER_ENC; i++) { + for (int i = 0; i < MAX_CWB_PER_ENC; i++) { hw_cwb = dpu_enc->hw_cwb[i]; if (!hw_cwb) continue; @@ -2671,8 +2678,7 @@ static int dpu_encoder_setup_display(struct dpu_encoder_virt *dpu_enc, i, controller_id, phys_params.split_role); phys_params.hw_intf = dpu_encoder_get_intf(dpu_kms->catalog, &dpu_kms->rm, - disp_info->intf_type, - controller_id); + disp_info, controller_id); if (disp_info->intf_type == INTF_WB && controller_id < WB_MAX) phys_params.hw_wb = dpu_rm_get_wb(&dpu_kms->rm, controller_id); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h index ca1ca2e51d7ea..2eb4c39b111c1 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h @@ -28,6 +28,7 @@ * @h_tile_instance: Controller instance used per tile. Number of elements is * based on num_of_h_tiles * @is_cmd_mode Boolean to indicate if the CMD mode is requested + * @stream_id stream id for which the interface needs to be acquired * @vsync_source: Source of the TE signal for DSI CMD devices */ struct msm_display_info { @@ -35,6 +36,7 @@ struct msm_display_info { uint32_t num_of_h_tiles; uint32_t h_tile_instance[MAX_H_TILES_PER_DISPLAY]; bool is_cmd_mode; + int stream_id; enum dpu_vsync_source vsync_source; }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c index 0ec6d67c7c70b..45079ee59cf67 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c @@ -257,6 +257,12 @@ static int dpu_encoder_phys_cmd_control_vblank_irq( if (!dpu_encoder_phys_cmd_is_master(phys_enc)) goto end; + /* IRQ not yet initialized */ + if (!phys_enc->irq[INTR_IDX_RDPTR]) { + ret = -EINVAL; + goto end; + } + /* protect against negative */ if (!enable && refcount == 0) { ret = -EINVAL; @@ -681,10 +687,11 @@ static int dpu_encoder_phys_cmd_wait_for_commit_done( if (!dpu_encoder_phys_cmd_is_master(phys_enc)) return 0; - if (phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) - return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); + if (phys_enc->irq[INTR_IDX_CTL_START] && + !phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) + return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); - return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); + return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); } static void dpu_encoder_phys_cmd_handle_post_kickoff( diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c index 46f348972a975..6d28f2281c765 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c @@ -247,14 +247,12 @@ static void dpu_encoder_phys_wb_setup_ctl(struct dpu_encoder_phys *phys_enc) if (hw_cdm) intf_cfg.cdm = hw_cdm->idx; - if (phys_enc->hw_pp->merge_3d && phys_enc->hw_pp->merge_3d->ops.setup_3d_mode) - phys_enc->hw_pp->merge_3d->ops.setup_3d_mode(phys_enc->hw_pp->merge_3d, - mode_3d); + if (hw_pp && hw_pp->merge_3d && hw_pp->merge_3d->ops.setup_3d_mode) + hw_pp->merge_3d->ops.setup_3d_mode(hw_pp->merge_3d, mode_3d); /* setup which pp blk will connect to this wb */ - if (hw_pp && phys_enc->hw_wb->ops.bind_pingpong_blk) - phys_enc->hw_wb->ops.bind_pingpong_blk(phys_enc->hw_wb, - phys_enc->hw_pp->idx); + if (hw_pp && hw_wb->ops.bind_pingpong_blk) + hw_wb->ops.bind_pingpong_blk(hw_wb, hw_pp->idx); phys_enc->hw_ctl->ops.setup_intf_cfg(phys_enc->hw_ctl, &intf_cfg); } else if (phys_enc->hw_ctl && phys_enc->hw_ctl->ops.setup_intf_cfg) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c index b0d585c5315ca..6e8883dbfad43 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c @@ -7,7 +7,6 @@ #include #include -#include "msm_media_info.h" #include "dpu_kms.h" #include "dpu_formats.h" @@ -54,116 +53,90 @@ static void _dpu_get_v_h_subsample_rate( } } -static int _dpu_format_get_media_color_ubwc(const struct msm_format *fmt) -{ - static const struct dpu_media_color_map dpu_media_ubwc_map[] = { - {DRM_FORMAT_ABGR8888, COLOR_FMT_RGBA8888_UBWC}, - {DRM_FORMAT_ARGB8888, COLOR_FMT_RGBA8888_UBWC}, - {DRM_FORMAT_XBGR8888, COLOR_FMT_RGBA8888_UBWC}, - {DRM_FORMAT_XRGB8888, COLOR_FMT_RGBA8888_UBWC}, - {DRM_FORMAT_ABGR2101010, COLOR_FMT_RGBA1010102_UBWC}, - {DRM_FORMAT_ARGB2101010, COLOR_FMT_RGBA1010102_UBWC}, - {DRM_FORMAT_XRGB2101010, COLOR_FMT_RGBA1010102_UBWC}, - {DRM_FORMAT_XBGR2101010, COLOR_FMT_RGBA1010102_UBWC}, - {DRM_FORMAT_BGR565, COLOR_FMT_RGB565_UBWC}, - }; - int color_fmt = -1; - int i; - - if (fmt->pixel_format == DRM_FORMAT_NV12 || - fmt->pixel_format == DRM_FORMAT_P010) { - if (MSM_FORMAT_IS_DX(fmt)) { - if (fmt->flags & MSM_FORMAT_FLAG_UNPACK_TIGHT) - color_fmt = COLOR_FMT_NV12_BPP10_UBWC; - else - color_fmt = COLOR_FMT_P010_UBWC; - } else - color_fmt = COLOR_FMT_NV12_UBWC; - return color_fmt; - } - - for (i = 0; i < ARRAY_SIZE(dpu_media_ubwc_map); ++i) - if (fmt->pixel_format == dpu_media_ubwc_map[i].format) { - color_fmt = dpu_media_ubwc_map[i].color; - break; - } - return color_fmt; -} - static int _dpu_format_populate_plane_sizes_ubwc( const struct msm_format *fmt, struct drm_framebuffer *fb, struct dpu_hw_fmt_layout *layout) { - int i; - int color; bool meta = MSM_FORMAT_IS_UBWC(fmt); - memset(layout, 0, sizeof(struct dpu_hw_fmt_layout)); - layout->width = fb->width; - layout->height = fb->height; - layout->num_planes = fmt->num_planes; + if (MSM_FORMAT_IS_YUV(fmt)) { + unsigned int stride, sclines; + unsigned int y_tile_width, y_tile_height; + unsigned int y_meta_stride, y_meta_scanlines; + unsigned int uv_meta_stride, uv_meta_scanlines; - color = _dpu_format_get_media_color_ubwc(fmt); - if (color < 0) { - DRM_ERROR("UBWC format not supported for fmt: %p4cc\n", - &fmt->pixel_format); - return -EINVAL; - } + if (MSM_FORMAT_IS_DX(fmt)) { + if (fmt->flags & MSM_FORMAT_FLAG_UNPACK_TIGHT) { + /* can't use round_up() here because 192 is NPoT */ + stride = roundup(fb->width, 192); + stride = round_up(stride * 4 / 3, 256); + y_tile_width = 48; + } else { + stride = round_up(fb->width * 2, 256); + y_tile_width = 32; + } + + sclines = round_up(fb->height, 16); + y_tile_height = 4; + } else { + stride = round_up(fb->width, 128); + y_tile_width = 32; - if (MSM_FORMAT_IS_YUV(fmt)) { - uint32_t y_sclines, uv_sclines; - uint32_t y_meta_scanlines = 0; - uint32_t uv_meta_scanlines = 0; + sclines = round_up(fb->height, 32); + y_tile_height = 8; + } - layout->num_planes = 2; - layout->plane_pitch[0] = VENUS_Y_STRIDE(color, fb->width); - y_sclines = VENUS_Y_SCANLINES(color, fb->height); - layout->plane_size[0] = MSM_MEDIA_ALIGN(layout->plane_pitch[0] * - y_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); + layout->plane_pitch[0] = stride; + layout->plane_size[0] = round_up(layout->plane_pitch[0] * + sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); - layout->plane_pitch[1] = VENUS_UV_STRIDE(color, fb->width); - uv_sclines = VENUS_UV_SCANLINES(color, fb->height); - layout->plane_size[1] = MSM_MEDIA_ALIGN(layout->plane_pitch[1] * - uv_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); + layout->plane_pitch[1] = stride; + layout->plane_size[1] = round_up(layout->plane_pitch[1] * + sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); if (!meta) - goto done; + return 0; + + y_meta_stride = DIV_ROUND_UP(fb->width, y_tile_width); + layout->plane_pitch[2] = round_up(y_meta_stride, 64); - layout->num_planes += 2; - layout->plane_pitch[2] = VENUS_Y_META_STRIDE(color, fb->width); - y_meta_scanlines = VENUS_Y_META_SCANLINES(color, fb->height); - layout->plane_size[2] = MSM_MEDIA_ALIGN(layout->plane_pitch[2] * + y_meta_scanlines = DIV_ROUND_UP(fb->height, y_tile_height); + y_meta_scanlines = round_up(y_meta_scanlines, 16); + layout->plane_size[2] = round_up(layout->plane_pitch[2] * y_meta_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); - layout->plane_pitch[3] = VENUS_UV_META_STRIDE(color, fb->width); - uv_meta_scanlines = VENUS_UV_META_SCANLINES(color, fb->height); - layout->plane_size[3] = MSM_MEDIA_ALIGN(layout->plane_pitch[3] * - uv_meta_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); + uv_meta_stride = DIV_ROUND_UP((fb->width+1)>>1, y_tile_width / 2); + layout->plane_pitch[3] = round_up(uv_meta_stride, 64); + uv_meta_scanlines = DIV_ROUND_UP((fb->height+1)>>1, y_tile_height); + uv_meta_scanlines = round_up(uv_meta_scanlines, 16); + layout->plane_size[3] = round_up(layout->plane_pitch[3] * + uv_meta_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); } else { - uint32_t rgb_scanlines, rgb_meta_scanlines; - - layout->num_planes = 1; + unsigned int rgb_scanlines, rgb_meta_scanlines, rgb_meta_stride; - layout->plane_pitch[0] = VENUS_RGB_STRIDE(color, fb->width); - rgb_scanlines = VENUS_RGB_SCANLINES(color, fb->height); - layout->plane_size[0] = MSM_MEDIA_ALIGN(layout->plane_pitch[0] * + layout->plane_pitch[0] = round_up(fb->width * fmt->bpp, 256); + rgb_scanlines = round_up(fb->height, 16); + layout->plane_size[0] = round_up(layout->plane_pitch[0] * rgb_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); if (!meta) - goto done; - layout->num_planes += 2; - layout->plane_pitch[2] = VENUS_RGB_META_STRIDE(color, fb->width); - rgb_meta_scanlines = VENUS_RGB_META_SCANLINES(color, fb->height); - layout->plane_size[2] = MSM_MEDIA_ALIGN(layout->plane_pitch[2] * + return 0; + + /* uAPI leaves plane[1] empty and plane[2] as meta */ + layout->num_planes += 1; + + rgb_meta_stride = DIV_ROUND_UP(fb->width, 16); + layout->plane_pitch[2] = round_up(rgb_meta_stride, 64); + + rgb_meta_scanlines = DIV_ROUND_UP(fb->height, 4); + rgb_meta_scanlines = round_up(rgb_meta_scanlines, 16); + + layout->plane_size[2] = round_up(layout->plane_pitch[2] * rgb_meta_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT); } -done: - for (i = 0; i < DPU_MAX_PLANES; i++) - layout->total_size += layout->plane_size[i]; - return 0; } @@ -174,14 +147,8 @@ static int _dpu_format_populate_plane_sizes_linear( { int i; - memset(layout, 0, sizeof(struct dpu_hw_fmt_layout)); - layout->width = fb->width; - layout->height = fb->height; - layout->num_planes = fmt->num_planes; - /* Due to memset above, only need to set planes of interest */ if (fmt->fetch_type == MDP_PLANE_INTERLEAVED) { - layout->num_planes = 1; layout->plane_size[0] = fb->width * fb->height * fmt->bpp; layout->plane_pitch[0] = fb->width * fmt->bpp; } else { @@ -208,12 +175,10 @@ static int _dpu_format_populate_plane_sizes_linear( (fb->height / v_subsample); if (fmt->fetch_type == MDP_PLANE_PSEUDO_PLANAR) { - layout->num_planes = 2; layout->plane_size[1] *= 2; layout->plane_pitch[1] *= 2; } else { /* planar */ - layout->num_planes = 3; layout->plane_size[2] = layout->plane_size[1]; layout->plane_pitch[2] = layout->plane_pitch[1]; } @@ -235,9 +200,6 @@ static int _dpu_format_populate_plane_sizes_linear( } } - for (i = 0; i < DPU_MAX_PLANES; i++) - layout->total_size += layout->plane_size[i]; - return 0; } @@ -254,6 +216,7 @@ int dpu_format_populate_plane_sizes( struct dpu_hw_fmt_layout *layout) { const struct msm_format *fmt; + int ret, i; if (!layout || !fb) { DRM_ERROR("invalid pointer\n"); @@ -268,10 +231,23 @@ int dpu_format_populate_plane_sizes( fmt = msm_framebuffer_format(fb); + memset(layout, 0, sizeof(struct dpu_hw_fmt_layout)); + layout->width = fb->width; + layout->height = fb->height; + layout->num_planes = fmt->num_planes; + if (MSM_FORMAT_IS_UBWC(fmt) || MSM_FORMAT_IS_TILE(fmt)) - return _dpu_format_populate_plane_sizes_ubwc(fmt, fb, layout); + ret = _dpu_format_populate_plane_sizes_ubwc(fmt, fb, layout); + else + ret = _dpu_format_populate_plane_sizes_linear(fmt, fb, layout); + + if (ret) + return ret; + + for (i = 0; i < DPU_MAX_PLANES; i++) + layout->total_size += layout->plane_size[i]; - return _dpu_format_populate_plane_sizes_linear(fmt, fb, layout); + return 0; } static void _dpu_format_populate_addrs_ubwc(struct drm_framebuffer *fb, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index 9f8d1bba9139a..c4e1f6b7345db 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -241,6 +241,23 @@ static const u32 wb2_formats_rgb_yuv[] = { .rotation_cfg = NULL, \ } +/* kaanapali SSPP common configuration */ +#define _VIG_SBLK_REC0_REC1(scaler_ver) \ + { \ + .sspp_rec0_blk = {.name = "sspp_rec0", \ + .base = 0x1000, .len = 0x180,}, \ + .csc_blk = {.name = "csc", \ + .base = 0x1800, .len = 0x100,}, \ + .scaler_blk = {.name = "scaler", \ + .version = scaler_ver, \ + .base = 0x2000, .len = 0xec,}, \ + .sspp_rec1_blk = {.name = "sspp_rec1", \ + .base = 0x3000, .len = 0x180,}, \ + .format_list = plane_formats_yuv, \ + .num_formats = ARRAY_SIZE(plane_formats_yuv), \ + .rotation_cfg = NULL, \ + } + #define _VIG_SBLK_ROT(scaler_ver, rot_cfg) \ { \ .scaler_blk = {.name = "scaler", \ @@ -329,6 +346,9 @@ static const struct dpu_sspp_sub_blks dpu_vig_sblk_qseed3_3_3 = static const struct dpu_sspp_sub_blks dpu_vig_sblk_qseed3_3_4 = _VIG_SBLK(SSPP_SCALER_VER(3, 4)); +static const struct dpu_sspp_sub_blks dpu_vig_sblk_qseed3_3_5 = + _VIG_SBLK_REC0_REC1(SSPP_SCALER_VER(3, 5)); + static const struct dpu_sspp_sub_blks dpu_rgb_sblk = _RGB_SBLK(); static const struct dpu_sspp_sub_blks dpu_dma_sblk = _DMA_SBLK(); @@ -382,11 +402,15 @@ static const struct dpu_lm_sub_blks qcm2290_lm_sblk = { static const struct dpu_dspp_sub_blks msm8998_dspp_sblk = { .pcc = {.name = "pcc", .base = 0x1700, .len = 0x90, .version = 0x10007}, + .gc = {.name = "gc", .base = 0x17c0, + .len = 0x40, .version = 0x10007}, }; static const struct dpu_dspp_sub_blks sdm845_dspp_sblk = { .pcc = {.name = "pcc", .base = 0x1700, .len = 0x90, .version = 0x40000}, + .gc = {.name = "gc", .base = 0x17c0, + .len = 0x40, .version = 0x10008}, }; static const struct dpu_dspp_sub_blks sm8750_dspp_sblk = { @@ -412,6 +436,11 @@ static const struct dpu_pingpong_sub_blks sc7280_pp_sblk = { .len = 0x20, .version = 0x20000}, }; +static const struct dpu_pingpong_sub_blks kaanapali_pp_sblk = { + .dither = {.name = "dither", .base = 0xc0, + .len = 0x40, .version = 0x30000}, +}; + /************************************************************* * DSC sub blocks config *************************************************************/ @@ -452,6 +481,13 @@ static const struct dpu_cdm_cfg dpu_cdm_5_x = { .base = 0x79200, }; +static const struct dpu_cdm_cfg dpu_cdm_13_x = { + .name = "cdm_0", + .id = CDM_0, + .len = 0x240, + .base = 0x19e000, +}; + /************************************************************* * VBIF sub blocks config *************************************************************/ @@ -639,6 +675,10 @@ static const struct dpu_qos_lut_entry sc7180_qos_linear[] = { {.fl = 0, .lut = 0x0011222222335777}, }; +static const struct dpu_qos_lut_entry kaanapali_qos_linear[] = { + {.fl = 0, .lut = 0x0011223344556666}, +}; + static const struct dpu_qos_lut_entry sm6350_qos_linear_macrotile[] = { {.fl = 0, .lut = 0x0011223445566777 }, }; @@ -668,6 +708,10 @@ static const struct dpu_qos_lut_entry sc7180_qos_macrotile[] = { {.fl = 0, .lut = 0x0011223344556677}, }; +static const struct dpu_qos_lut_entry kaanapali_qos_macrotile[] = { + {.fl = 0, .lut = 0x0011223344556666}, +}; + static const struct dpu_qos_lut_entry sc8180x_qos_macrotile[] = { {.fl = 10, .lut = 0x0000000344556677}, }; @@ -726,3 +770,5 @@ static const struct dpu_qos_lut_entry sc7180_qos_nrt[] = { #include "catalog/dpu_10_0_sm8650.h" #include "catalog/dpu_12_0_sm8750.h" +#include "catalog/dpu_12_2_glymur.h" +#include "catalog/dpu_13_0_kaanapali.h" diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h index f0768f54e9b3d..70d5ed4732f2e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h @@ -77,9 +77,11 @@ enum { /** * DSPP sub-blocks * @DPU_DSPP_PCC Panel color correction block + * @DPU_DSPP_GC Gamma correction block */ enum { DPU_DSPP_PCC = 0x1, + DPU_DSPP_GC, DPU_DSPP_MAX }; @@ -208,6 +210,18 @@ struct dpu_dsc_blk { u32 len; }; +/** + * struct dpu_sspp_v13_rec_blk - SSPP REC sub-blk information + * @name: string name for debug purposes + * @base: offset of this sub-block relative to the block offset + * @len: register block length of this sub-block + */ +struct dpu_sspp_v13_rec_blk { + char name[DPU_HW_BLK_NAME_LEN]; + u32 base; + u32 len; +}; + /** * enum dpu_qos_lut_usage - define QoS LUT use cases */ @@ -294,6 +308,8 @@ struct dpu_sspp_sub_blks { u32 qseed_ver; struct dpu_scaler_blk scaler_blk; struct dpu_pp_blk csc_blk; + struct dpu_sspp_v13_rec_blk sspp_rec0_blk; + struct dpu_sspp_v13_rec_blk sspp_rec1_blk; const u32 *format_list; u32 num_formats; @@ -314,9 +330,11 @@ struct dpu_lm_sub_blks { /** * struct dpu_dspp_sub_blks: Information of DSPP block * @pcc: pixel color correction block + * @gc: gamma correction block */ struct dpu_dspp_sub_blks { struct dpu_pp_blk pcc; + struct dpu_pp_blk gc; }; struct dpu_pingpong_sub_blks { @@ -749,6 +767,8 @@ struct dpu_mdss_cfg { const struct dpu_format_extended *vig_formats; }; +extern const struct dpu_mdss_cfg dpu_glymur_cfg; +extern const struct dpu_mdss_cfg dpu_kaanapali_cfg; extern const struct dpu_mdss_cfg dpu_msm8917_cfg; extern const struct dpu_mdss_cfg dpu_msm8937_cfg; extern const struct dpu_mdss_cfg dpu_msm8953_cfg; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h index 6bb3476a05f80..75e6dae0fcd9b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h @@ -89,13 +89,13 @@ enum dpu_hw_cdwn_op_mode_method_h_v { */ struct dpu_hw_cdm_ops { /** - * Enable the CDM module + * @enable: Enable the CDM module * @cdm Pointer to chroma down context */ int (*enable)(struct dpu_hw_cdm *cdm, struct dpu_hw_cdm_cfg *cfg); /** - * Enable/disable the connection with pingpong + * @bind_pingpong_blk: Enable/disable the connection with pingpong * @cdm Pointer to chroma down context * @pp pingpong block id. */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c index ac834db2e4c16..36a497f1d6c12 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c @@ -399,6 +399,9 @@ static void dpu_hw_ctl_update_pending_flush_dspp_sub_blocks( case DPU_DSPP_PCC: ctx->pending_dspp_flush_mask[dspp - DSPP_0] |= BIT(4); break; + case DPU_DSPP_GC: + ctx->pending_dspp_flush_mask[dspp - DSPP_0] |= BIT(5); + break; default: return; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h index 15931b22ec941..e535bf013825b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h @@ -12,9 +12,9 @@ #include "dpu_hw_sspp.h" /** - * dpu_ctl_mode_sel: Interface mode selection - * DPU_CTL_MODE_SEL_VID: Video mode interface - * DPU_CTL_MODE_SEL_CMD: Command mode interface + * enum dpu_ctl_mode_sel: Interface mode selection + * @DPU_CTL_MODE_SEL_VID: Video mode interface + * @DPU_CTL_MODE_SEL_CMD: Command mode interface */ enum dpu_ctl_mode_sel { DPU_CTL_MODE_SEL_VID = 0, @@ -37,6 +37,7 @@ struct dpu_hw_stage_cfg { * struct dpu_hw_intf_cfg :Describes how the DPU writes data to output interface * @intf : Interface id * @intf_master: Master interface id in the dual pipe topology + * @wb: Writeback mode * @mode_3d: 3d mux configuration * @merge_3d: 3d merge block used * @intf_mode_sel: Interface mode, cmd / vid @@ -64,21 +65,21 @@ struct dpu_hw_intf_cfg { */ struct dpu_hw_ctl_ops { /** - * kickoff hw operation for Sw controlled interfaces + * @trigger_start: kickoff hw operation for Sw controlled interfaces * DSI cmd mode and WB interface are SW controlled * @ctx : ctl path ctx pointer */ void (*trigger_start)(struct dpu_hw_ctl *ctx); /** - * check if the ctl is started + * @is_started: check if the ctl is started * @ctx : ctl path ctx pointer * @Return: true if started, false if stopped */ bool (*is_started)(struct dpu_hw_ctl *ctx); /** - * kickoff prepare is in progress hw operation for sw + * @trigger_pending: kickoff prepare is in progress hw operation for sw * controlled interfaces: DSI cmd mode and WB interface * are SW controlled * @ctx : ctl path ctx pointer @@ -86,7 +87,7 @@ struct dpu_hw_ctl_ops { void (*trigger_pending)(struct dpu_hw_ctl *ctx); /** - * Clear the value of the cached pending_flush_mask + * @clear_pending_flush: Clear the value of the cached pending_flush_mask * No effect on hardware. * Required to be implemented. * @ctx : ctl path ctx pointer @@ -94,14 +95,15 @@ struct dpu_hw_ctl_ops { void (*clear_pending_flush)(struct dpu_hw_ctl *ctx); /** - * Query the value of the cached pending_flush_mask + * @get_pending_flush: Query the value of the cached pending_flush_mask * No effect on hardware * @ctx : ctl path ctx pointer */ u32 (*get_pending_flush)(struct dpu_hw_ctl *ctx); /** - * OR in the given flushbits to the cached pending_flush_mask + * @update_pending_flush: OR in the given flushbits to the cached + * pending_flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @flushbits : module flushmask @@ -110,7 +112,8 @@ struct dpu_hw_ctl_ops { u32 flushbits); /** - * OR in the given flushbits to the cached pending_(wb_)flush_mask + * @update_pending_flush_wb: OR in the given flushbits to the + * cached pending_(wb_)flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : writeback block index @@ -119,7 +122,8 @@ struct dpu_hw_ctl_ops { enum dpu_wb blk); /** - * OR in the given flushbits to the cached pending_(cwb_)flush_mask + * @update_pending_flush_cwb: OR in the given flushbits to the + * cached pending_(cwb_)flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : concurrent writeback block index @@ -128,7 +132,8 @@ struct dpu_hw_ctl_ops { enum dpu_cwb blk); /** - * OR in the given flushbits to the cached pending_(intf_)flush_mask + * @update_pending_flush_intf: OR in the given flushbits to the + * cached pending_(intf_)flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : interface block index @@ -137,7 +142,8 @@ struct dpu_hw_ctl_ops { enum dpu_intf blk); /** - * OR in the given flushbits to the cached pending_(periph_)flush_mask + * @update_pending_flush_periph: OR in the given flushbits to the + * cached pending_(periph_)flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : interface block index @@ -146,7 +152,8 @@ struct dpu_hw_ctl_ops { enum dpu_intf blk); /** - * OR in the given flushbits to the cached pending_(merge_3d_)flush_mask + * @update_pending_flush_merge_3d: OR in the given flushbits to the + * cached pending_(merge_3d_)flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : interface block index @@ -155,7 +162,8 @@ struct dpu_hw_ctl_ops { enum dpu_merge_3d blk); /** - * OR in the given flushbits to the cached pending_flush_mask + * @update_pending_flush_sspp: OR in the given flushbits to the + * cached pending_flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : SSPP block index @@ -164,7 +172,8 @@ struct dpu_hw_ctl_ops { enum dpu_sspp blk); /** - * OR in the given flushbits to the cached pending_flush_mask + * @update_pending_flush_mixer: OR in the given flushbits to the + * cached pending_flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : LM block index @@ -173,7 +182,8 @@ struct dpu_hw_ctl_ops { enum dpu_lm blk); /** - * OR in the given flushbits to the cached pending_flush_mask + * @update_pending_flush_dspp: OR in the given flushbits to the + * cached pending_flush_mask. * No effect on hardware * @ctx : ctl path ctx pointer * @blk : DSPP block index @@ -183,7 +193,8 @@ struct dpu_hw_ctl_ops { enum dpu_dspp blk, u32 dspp_sub_blk); /** - * OR in the given flushbits to the cached pending_(dsc_)flush_mask + * @update_pending_flush_dsc: OR in the given flushbits to the + * cached pending_(dsc_)flush_mask. * No effect on hardware * @ctx: ctl path ctx pointer * @blk: interface block index @@ -192,7 +203,8 @@ struct dpu_hw_ctl_ops { enum dpu_dsc blk); /** - * OR in the given flushbits to the cached pending_(cdm_)flush_mask + * @update_pending_flush_cdm: OR in the given flushbits to the + * cached pending_(cdm_)flush_mask. * No effect on hardware * @ctx: ctl path ctx pointer * @cdm_num: idx of cdm to be flushed @@ -200,20 +212,20 @@ struct dpu_hw_ctl_ops { void (*update_pending_flush_cdm)(struct dpu_hw_ctl *ctx, enum dpu_cdm cdm_num); /** - * Write the value of the pending_flush_mask to hardware + * @trigger_flush: Write the value of the pending_flush_mask to hardware * @ctx : ctl path ctx pointer */ void (*trigger_flush)(struct dpu_hw_ctl *ctx); /** - * Read the value of the flush register + * @get_flush_register: Read the value of the flush register * @ctx : ctl path ctx pointer * @Return: value of the ctl flush register. */ u32 (*get_flush_register)(struct dpu_hw_ctl *ctx); /** - * Setup ctl_path interface config + * @setup_intf_cfg: Setup ctl_path interface config * @ctx * @cfg : interface config structure pointer */ @@ -221,17 +233,20 @@ struct dpu_hw_ctl_ops { struct dpu_hw_intf_cfg *cfg); /** - * reset ctl_path interface config + * @reset_intf_cfg: reset ctl_path interface config * @ctx : ctl path ctx pointer * @cfg : interface config structure pointer */ void (*reset_intf_cfg)(struct dpu_hw_ctl *ctx, struct dpu_hw_intf_cfg *cfg); + /** + * @reset: reset function for this ctl type + */ int (*reset)(struct dpu_hw_ctl *c); - /* - * wait_reset_status - checks ctl reset status + /** + * @wait_reset_status: checks ctl reset status * @ctx : ctl path ctx pointer * * This function checks the ctl reset status bit. @@ -242,13 +257,13 @@ struct dpu_hw_ctl_ops { int (*wait_reset_status)(struct dpu_hw_ctl *ctx); /** - * Set all blend stages to disabled + * @clear_all_blendstages: Set all blend stages to disabled * @ctx : ctl path ctx pointer */ void (*clear_all_blendstages)(struct dpu_hw_ctl *ctx); /** - * Configure layer mixer to pipe configuration + * @setup_blendstage: Configure layer mixer to pipe configuration * @ctx : ctl path ctx pointer * @lm : layer mixer enumeration * @cfg : blend stage configuration @@ -256,11 +271,16 @@ struct dpu_hw_ctl_ops { void (*setup_blendstage)(struct dpu_hw_ctl *ctx, enum dpu_lm lm, struct dpu_hw_stage_cfg *cfg); + /** + * @set_active_fetch_pipes: Set active pipes attached to this CTL + * @ctx: ctl path ctx pointer + * @active_pipes: bitmap of enum dpu_sspp + */ void (*set_active_fetch_pipes)(struct dpu_hw_ctl *ctx, unsigned long *fetch_active); /** - * Set active pipes attached to this CTL + * @set_active_pipes: Set active pipes attached to this CTL * @ctx: ctl path ctx pointer * @active_pipes: bitmap of enum dpu_sspp */ @@ -268,13 +288,12 @@ struct dpu_hw_ctl_ops { unsigned long *active_pipes); /** - * Set active layer mixers attached to this CTL + * @set_active_lms: Set active layer mixers attached to this CTL * @ctx: ctl path ctx pointer * @active_lms: bitmap of enum dpu_lm */ void (*set_active_lms)(struct dpu_hw_ctl *ctx, unsigned long *active_lms); - }; /** @@ -289,6 +308,9 @@ struct dpu_hw_ctl_ops { * @pending_intf_flush_mask: pending INTF flush * @pending_wb_flush_mask: pending WB flush * @pending_cwb_flush_mask: pending CWB flush + * @pending_periph_flush_mask: pending PERIPH flush + * @pending_merge_3d_flush_mask: pending MERGE 3D flush + * @pending_dspp_flush_mask: pending DSPP flush * @pending_dsc_flush_mask: pending DSC flush * @pending_cdm_flush_mask: pending CDM flush * @mdss_ver: MDSS revision information @@ -320,7 +342,7 @@ struct dpu_hw_ctl { }; /** - * dpu_hw_ctl - convert base object dpu_hw_base to container + * to_dpu_hw_ctl - convert base object dpu_hw_base to container * @hw: Pointer to base hardware block * return: Pointer to hardware block container */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cwb.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cwb.h index 96b6edf6b2bbf..ed7bfcee7f1cc 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cwb.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cwb.h @@ -28,7 +28,6 @@ struct dpu_hw_cwb_setup_cfg { }; /** - * * struct dpu_hw_cwb_ops : Interface to the cwb hw driver functions * @config_cwb: configure CWB mux */ @@ -54,7 +53,7 @@ struct dpu_hw_cwb { }; /** - * dpu_hw_cwb - convert base object dpu_hw_base to container + * to_dpu_hw_cwb - convert base object dpu_hw_base to container * @hw: Pointer to base hardware block * return: Pointer to hardware block container */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h index b7013c9822d23..39d93b9df0515 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h @@ -21,13 +21,13 @@ struct dpu_hw_dsc; */ struct dpu_hw_dsc_ops { /** - * dsc_disable - disable dsc + * @dsc_disable: disable dsc * @hw_dsc: Pointer to dsc context */ void (*dsc_disable)(struct dpu_hw_dsc *hw_dsc); /** - * dsc_config - configures dsc encoder + * @dsc_config: configures dsc encoder * @hw_dsc: Pointer to dsc context * @dsc: panel dsc parameters * @mode: dsc topology mode to be set @@ -39,13 +39,17 @@ struct dpu_hw_dsc_ops { u32 initial_lines); /** - * dsc_config_thresh - programs panel thresholds + * @dsc_config_thresh: programs panel thresholds * @hw_dsc: Pointer to dsc context * @dsc: panel dsc parameters */ void (*dsc_config_thresh)(struct dpu_hw_dsc *hw_dsc, struct drm_dsc_config *dsc); + /** + * @dsc_bind_pingpong_blk: binds pixel output from a DSC block + * to a pingpong block + */ void (*dsc_bind_pingpong_blk)(struct dpu_hw_dsc *hw_dsc, enum dpu_pingpong pp); }; @@ -71,12 +75,6 @@ struct dpu_hw_dsc *dpu_hw_dsc_init_1_2(struct drm_device *dev, const struct dpu_dsc_cfg *cfg, void __iomem *addr); -/** - * dpu_hw_dsc_destroy - destroys dsc driver context - * @dsc: Pointer to dsc driver context returned by dpu_hw_dsc_init - */ -void dpu_hw_dsc_destroy(struct dpu_hw_dsc *dsc); - static inline struct dpu_hw_dsc *to_dpu_hw_dsc(struct dpu_hw_blk *hw) { return container_of(hw, struct dpu_hw_dsc, base); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c index 54b20faa0b697..23dcbe1ce1b83 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c @@ -24,6 +24,18 @@ #define PCC_BLUE_G_OFF 0x24 #define PCC_BLUE_B_OFF 0x30 +/* DSPP_GC */ +#define GC_EN BIT(0) +#define GC_DIS 0 +#define GC_8B_ROUND_EN BIT(1) +#define GC_LUT_SWAP_OFF 0x1c +#define GC_C0_OFF 0x4 +#define GC_C1_OFF 0xc +#define GC_C2_OFF 0x14 +#define GC_C0_INDEX_OFF 0x8 +#define GC_C1_INDEX_OFF 0x10 +#define GC_C2_INDEX_OFF 0x18 + static void dpu_setup_dspp_pcc(struct dpu_hw_dspp *ctx, struct dpu_hw_pcc_cfg *cfg) { @@ -63,6 +75,46 @@ static void dpu_setup_dspp_pcc(struct dpu_hw_dspp *ctx, DPU_REG_WRITE(&ctx->hw, base, PCC_EN); } +static void dpu_setup_dspp_gc(struct dpu_hw_dspp *ctx, + struct dpu_hw_gc_lut *gc_lut) +{ + int i = 0; + u32 base, reg; + + if (!ctx) { + DRM_ERROR("invalid ctx\n"); + return; + } + + base = ctx->cap->sblk->gc.base; + + if (!base) { + DRM_ERROR("invalid ctx %p gc base\n", ctx); + return; + } + + if (!gc_lut) { + DRM_DEBUG_DRIVER("disable gc feature\n"); + DPU_REG_WRITE(&ctx->hw, base, GC_DIS); + return; + } + + DPU_REG_WRITE(&ctx->hw, base + GC_C0_INDEX_OFF, 0); + DPU_REG_WRITE(&ctx->hw, base + GC_C1_INDEX_OFF, 0); + DPU_REG_WRITE(&ctx->hw, base + GC_C2_INDEX_OFF, 0); + + for (i = 0; i < PGC_TBL_LEN; i++) { + DPU_REG_WRITE(&ctx->hw, base + GC_C0_OFF, gc_lut->c0[i]); + DPU_REG_WRITE(&ctx->hw, base + GC_C1_OFF, gc_lut->c1[i]); + DPU_REG_WRITE(&ctx->hw, base + GC_C2_OFF, gc_lut->c2[i]); + } + + DPU_REG_WRITE(&ctx->hw, base + GC_LUT_SWAP_OFF, BIT(0)); + + reg = GC_EN | ((gc_lut->flags & PGC_8B_ROUND) ? GC_8B_ROUND_EN : 0); + DPU_REG_WRITE(&ctx->hw, base, reg); +} + /** * dpu_hw_dspp_init() - Initializes the DSPP hw driver object. * should be called once before accessing every DSPP. @@ -92,6 +144,8 @@ struct dpu_hw_dspp *dpu_hw_dspp_init(struct drm_device *dev, c->cap = cfg; if (c->cap->sblk->pcc.base) c->ops.setup_pcc = dpu_setup_dspp_pcc; + if (c->cap->sblk->gc.base) + c->ops.setup_gc = dpu_setup_dspp_gc; return c; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h index 45c26cd49fa3e..b47b7788064b2 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h @@ -22,7 +22,7 @@ struct dpu_hw_pcc_coeff { }; /** - * struct dpu_hw_pcc - pcc feature structure + * struct dpu_hw_pcc_cfg - pcc feature structure * @r: red coefficients. * @g: green coefficients. * @b: blue coefficients. @@ -33,6 +33,25 @@ struct dpu_hw_pcc_cfg { struct dpu_hw_pcc_coeff b; }; +#define DPU_GAMMA_LUT_SIZE 1024 +#define PGC_TBL_LEN 512 +#define PGC_8B_ROUND BIT(0) + +/** + * struct dpu_hw_gc_lut - gc lut feature structure + * @flags: flags for the feature values can be: + * - PGC_8B_ROUND + * @c0: color0 component lut + * @c1: color1 component lut + * @c2: color2 component lut + */ +struct dpu_hw_gc_lut { + __u64 flags; + __u32 c0[PGC_TBL_LEN]; + __u32 c1[PGC_TBL_LEN]; + __u32 c2[PGC_TBL_LEN]; +}; + /** * struct dpu_hw_dspp_ops - interface to the dspp hardware driver functions * Caller must call the init function to get the dspp context for each dspp @@ -40,12 +59,19 @@ struct dpu_hw_pcc_cfg { */ struct dpu_hw_dspp_ops { /** - * setup_pcc - setup dspp pcc + * @setup_pcc: setup_pcc - setup dspp pcc * @ctx: Pointer to dspp context * @cfg: Pointer to configuration */ void (*setup_pcc)(struct dpu_hw_dspp *ctx, struct dpu_hw_pcc_cfg *cfg); + /** + * setup_gc - setup dspp gc + * @ctx: Pointer to dspp context + * @gc_lut: Pointer to lut content + */ + void (*setup_gc)(struct dpu_hw_dspp *ctx, struct dpu_hw_gc_lut *gc_lut); + }; /** @@ -69,7 +95,7 @@ struct dpu_hw_dspp { }; /** - * dpu_hw_dspp - convert base object dpu_hw_base to container + * to_dpu_hw_dspp - convert base object dpu_hw_base to container * @hw: Pointer to base hardware block * return: Pointer to hardware block container */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c index 49bd77a755aa5..5b7cd5241f450 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c @@ -40,6 +40,15 @@ #define MDP_INTF_REV_7xxx_INTR_TEAR_STATUS(intf) (MDP_INTF_REV_7xxx_TEAR_OFF(intf) + 0x004) #define MDP_INTF_REV_7xxx_INTR_TEAR_CLEAR(intf) (MDP_INTF_REV_7xxx_TEAR_OFF(intf) + 0x008) +#define MDP_INTF_REV_13xx_OFF(intf) (0x18d000 + 0x1000 * (intf)) +#define MDP_INTF_REV_13xx_INTR_EN(intf) (MDP_INTF_REV_13xx_OFF(intf) + 0x1c0) +#define MDP_INTF_REV_13xx_INTR_STATUS(intf) (MDP_INTF_REV_13xx_OFF(intf) + 0x1c4) +#define MDP_INTF_REV_13xx_INTR_CLEAR(intf) (MDP_INTF_REV_13xx_OFF(intf) + 0x1c8) +#define MDP_INTF_REV_13xx_TEAR_OFF(intf) (0x18d800 + 0x1000 * (intf)) +#define MDP_INTF_REV_13xx_INTR_TEAR_EN(intf) (MDP_INTF_REV_13xx_TEAR_OFF(intf) + 0x000) +#define MDP_INTF_REV_13xx_INTR_TEAR_STATUS(intf) (MDP_INTF_REV_13xx_TEAR_OFF(intf) + 0x004) +#define MDP_INTF_REV_13xx_INTR_TEAR_CLEAR(intf) (MDP_INTF_REV_13xx_TEAR_OFF(intf) + 0x008) + /** * struct dpu_intr_reg - array of DPU register sets * @clr_off: offset to CLEAR reg @@ -199,6 +208,82 @@ static const struct dpu_intr_reg dpu_intr_set_7xxx[] = { }, }; +/* + * dpu_intr_set_13xx - List of DPU interrupt registers for DPU >= 13.0 + */ +static const struct dpu_intr_reg dpu_intr_set_13xx[] = { + [MDP_SSPP_TOP0_INTR] = { + INTR_CLEAR, + INTR_EN, + INTR_STATUS + }, + [MDP_SSPP_TOP0_INTR2] = { + INTR2_CLEAR, + INTR2_EN, + INTR2_STATUS + }, + [MDP_SSPP_TOP0_HIST_INTR] = { + HIST_INTR_CLEAR, + HIST_INTR_EN, + HIST_INTR_STATUS + }, + [MDP_INTF0_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(0), + MDP_INTF_REV_13xx_INTR_EN(0), + MDP_INTF_REV_13xx_INTR_STATUS(0) + }, + [MDP_INTF1_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(1), + MDP_INTF_REV_13xx_INTR_EN(1), + MDP_INTF_REV_13xx_INTR_STATUS(1) + }, + [MDP_INTF1_TEAR_INTR] = { + MDP_INTF_REV_13xx_INTR_TEAR_CLEAR(1), + MDP_INTF_REV_13xx_INTR_TEAR_EN(1), + MDP_INTF_REV_13xx_INTR_TEAR_STATUS(1) + }, + [MDP_INTF2_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(2), + MDP_INTF_REV_13xx_INTR_EN(2), + MDP_INTF_REV_13xx_INTR_STATUS(2) + }, + [MDP_INTF2_TEAR_INTR] = { + MDP_INTF_REV_13xx_INTR_TEAR_CLEAR(2), + MDP_INTF_REV_13xx_INTR_TEAR_EN(2), + MDP_INTF_REV_13xx_INTR_TEAR_STATUS(2) + }, + [MDP_INTF3_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(3), + MDP_INTF_REV_13xx_INTR_EN(3), + MDP_INTF_REV_13xx_INTR_STATUS(3) + }, + [MDP_INTF4_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(4), + MDP_INTF_REV_13xx_INTR_EN(4), + MDP_INTF_REV_13xx_INTR_STATUS(4) + }, + [MDP_INTF5_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(5), + MDP_INTF_REV_13xx_INTR_EN(5), + MDP_INTF_REV_13xx_INTR_STATUS(5) + }, + [MDP_INTF6_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(6), + MDP_INTF_REV_13xx_INTR_EN(6), + MDP_INTF_REV_13xx_INTR_STATUS(6) + }, + [MDP_INTF7_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(7), + MDP_INTF_REV_13xx_INTR_EN(7), + MDP_INTF_REV_13xx_INTR_STATUS(7) + }, + [MDP_INTF8_INTR] = { + MDP_INTF_REV_13xx_INTR_CLEAR(8), + MDP_INTF_REV_13xx_INTR_EN(8), + MDP_INTF_REV_13xx_INTR_STATUS(8) + }, +}; + #define DPU_IRQ_MASK(irq_idx) (BIT(DPU_IRQ_BIT(irq_idx))) static inline bool dpu_core_irq_is_valid(unsigned int irq_idx) @@ -507,7 +592,9 @@ struct dpu_hw_intr *dpu_hw_intr_init(struct drm_device *dev, if (!intr) return ERR_PTR(-ENOMEM); - if (m->mdss_ver->core_major_ver >= 7) + if (m->mdss_ver->core_major_ver >= 13) + intr->intr_set = dpu_intr_set_13xx; + else if (m->mdss_ver->core_major_ver >= 7) intr->intr_set = dpu_intr_set_7xxx; else intr->intr_set = dpu_intr_set_legacy; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c index a80ac82a96255..7e620f5909849 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c @@ -67,6 +67,10 @@ #define INTF_MISR_CTRL 0x180 #define INTF_MISR_SIGNATURE 0x184 +#define INTF_WD_TIMER_0_CTL 0x230 +#define INTF_WD_TIMER_0_CTL2 0x234 +#define INTF_WD_TIMER_0_LOAD_VALUE 0x238 + #define INTF_MUX 0x25C #define INTF_STATUS 0x26C #define INTF_AVR_CONTROL 0x270 @@ -475,7 +479,20 @@ static int dpu_hw_intf_get_vsync_info(struct dpu_hw_intf *intf, } static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, - enum dpu_vsync_source vsync_source) + struct dpu_vsync_source_cfg *cfg) +{ + struct dpu_hw_blk_reg_map *c; + + if (!intf) + return; + + c = &intf->hw; + + DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (cfg->vsync_source & 0xf)); +} + +static void dpu_hw_intf_vsync_sel_v8(struct dpu_hw_intf *intf, + struct dpu_vsync_source_cfg *cfg) { struct dpu_hw_blk_reg_map *c; @@ -484,7 +501,30 @@ static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, c = &intf->hw; - DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (vsync_source & 0xf)); + if (cfg->vsync_source >= DPU_VSYNC_SOURCE_WD_TIMER_4 && + cfg->vsync_source <= DPU_VSYNC_SOURCE_WD_TIMER_1) { + pr_warn_once("DPU 8.x supports only GPIOs and timer0 as TE sources\n"); + return; + } + + if (cfg->vsync_source == DPU_VSYNC_SOURCE_WD_TIMER_0) { + u32 reg; + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_LOAD_VALUE, + CALCULATE_WD_LOAD_VALUE(cfg->frame_rate)); + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL, BIT(0)); /* clear timer */ + + reg = BIT(8); /* enable heartbeat timer */ + reg |= BIT(0); /* enable WD timer */ + reg |= BIT(1); /* select default 16 clock ticks */ + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL2, reg); + + /* make sure that timers are enabled/disabled for vsync state */ + wmb(); + } + + dpu_hw_intf_vsync_sel(intf, cfg); } static void dpu_hw_intf_disable_autorefresh(struct dpu_hw_intf *intf, @@ -598,7 +638,10 @@ struct dpu_hw_intf *dpu_hw_intf_init(struct drm_device *dev, c->ops.enable_tearcheck = dpu_hw_intf_enable_te; c->ops.disable_tearcheck = dpu_hw_intf_disable_te; c->ops.connect_external_te = dpu_hw_intf_connect_external_te; - c->ops.vsync_sel = dpu_hw_intf_vsync_sel; + if (mdss_rev->core_major_ver >= 8) + c->ops.vsync_sel = dpu_hw_intf_vsync_sel_v8; + else + c->ops.vsync_sel = dpu_hw_intf_vsync_sel; c->ops.disable_autorefresh = dpu_hw_intf_disable_autorefresh; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h index f31067a9aaf1d..f6ef2c21b66d4 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h @@ -12,6 +12,7 @@ #include "dpu_hw_util.h" struct dpu_hw_intf; +struct dpu_vsync_source_cfg; /* intf timing settings */ struct dpu_hw_intf_timing_params { @@ -57,11 +58,11 @@ struct dpu_hw_intf_cmd_mode_cfg { /** * struct dpu_hw_intf_ops : Interface to the interface Hw driver functions * Assumption is these functions will be called after clocks are enabled - * @ setup_timing_gen : programs the timing engine - * @ setup_prog_fetch : enables/disables the programmable fetch logic - * @ enable_timing: enable/disable timing engine - * @ get_status: returns if timing engine is enabled or not - * @ get_line_count: reads current vertical line counter + * @setup_timing_gen : programs the timing engine + * @setup_prg_fetch : enables/disables the programmable fetch logic + * @enable_timing: enable/disable timing engine + * @get_status: returns if timing engine is enabled or not + * @get_line_count: reads current vertical line counter * @bind_pingpong_blk: enable/disable the connection with pingpong which will * feed pixels to this interface * @setup_misr: enable/disable MISR @@ -70,12 +71,9 @@ struct dpu_hw_intf_cmd_mode_cfg { * pointer and programs the tear check configuration * @disable_tearcheck: Disables tearcheck block * @connect_external_te: Read, modify, write to either set or clear listening to external TE - * Return: 1 if TE was originally connected, 0 if not, or -ERROR - * @get_vsync_info: Provides the programmed and current line_count - * @setup_autorefresh: Configure and enable the autorefresh config - * @get_autorefresh: Retrieve autorefresh config from hardware - * Return: 0 on success, -ETIMEDOUT on timeout + * Returns 1 if TE was originally connected, 0 if not, or -ERROR * @vsync_sel: Select vsync signal for tear-effect configuration + * @disable_autorefresh: Disable autorefresh if enabled * @program_intf_cmd_cfg: Program the DPU to interface datapath for command mode */ struct dpu_hw_intf_ops { @@ -107,11 +105,8 @@ struct dpu_hw_intf_ops { int (*connect_external_te)(struct dpu_hw_intf *intf, bool enable_external_te); - void (*vsync_sel)(struct dpu_hw_intf *intf, enum dpu_vsync_source vsync_source); + void (*vsync_sel)(struct dpu_hw_intf *intf, struct dpu_vsync_source_cfg *cfg); - /** - * Disable autorefresh if enabled - */ void (*disable_autorefresh)(struct dpu_hw_intf *intf, uint32_t encoder_id, u16 vdisplay); void (*program_intf_cmd_cfg)(struct dpu_hw_intf *intf, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.c index e8a76d5192c23..b7779726bf106 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.c @@ -126,7 +126,9 @@ static int dpu_hw_lm_collect_misr(struct dpu_hw_mixer *ctx, u32 *misr_value) } static void dpu_hw_lm_setup_blend_config_combined_alpha(struct dpu_hw_mixer *ctx, - u32 stage, u32 fg_alpha, u32 bg_alpha, u32 blend_op) + u32 stage, + u16 fg_alpha, u16 bg_alpha, + u32 blend_op) { struct dpu_hw_blk_reg_map *c = &ctx->hw; int stage_off; @@ -139,15 +141,16 @@ static void dpu_hw_lm_setup_blend_config_combined_alpha(struct dpu_hw_mixer *ctx if (WARN_ON(stage_off < 0)) return; - const_alpha = (bg_alpha & 0xFF) | ((fg_alpha & 0xFF) << 16); + const_alpha = (bg_alpha >> 8) | ((fg_alpha >> 8) << 16); DPU_REG_WRITE(c, LM_BLEND0_CONST_ALPHA + stage_off, const_alpha); DPU_REG_WRITE(c, LM_BLEND0_OP + stage_off, blend_op); } static void dpu_hw_lm_setup_blend_config_combined_alpha_v12(struct dpu_hw_mixer *ctx, - u32 stage, u32 fg_alpha, - u32 bg_alpha, u32 blend_op) + u32 stage, + u16 fg_alpha, u16 bg_alpha, + u32 blend_op) { struct dpu_hw_blk_reg_map *c = &ctx->hw; int stage_off; @@ -160,13 +163,15 @@ dpu_hw_lm_setup_blend_config_combined_alpha_v12(struct dpu_hw_mixer *ctx, if (WARN_ON(stage_off < 0)) return; - const_alpha = (bg_alpha & 0x3ff) | ((fg_alpha & 0x3ff) << 16); + const_alpha = (bg_alpha >> 6) | ((fg_alpha >> 6) << 16); DPU_REG_WRITE(c, LM_BLEND0_CONST_ALPHA_V12 + stage_off, const_alpha); DPU_REG_WRITE(c, LM_BLEND0_OP + stage_off, blend_op); } static void dpu_hw_lm_setup_blend_config(struct dpu_hw_mixer *ctx, - u32 stage, u32 fg_alpha, u32 bg_alpha, u32 blend_op) + u32 stage, + u16 fg_alpha, u16 bg_alpha, + u32 blend_op) { struct dpu_hw_blk_reg_map *c = &ctx->hw; int stage_off; @@ -178,8 +183,8 @@ static void dpu_hw_lm_setup_blend_config(struct dpu_hw_mixer *ctx, if (WARN_ON(stage_off < 0)) return; - DPU_REG_WRITE(c, LM_BLEND0_FG_ALPHA + stage_off, fg_alpha); - DPU_REG_WRITE(c, LM_BLEND0_BG_ALPHA + stage_off, bg_alpha); + DPU_REG_WRITE(c, LM_BLEND0_FG_ALPHA + stage_off, fg_alpha >> 8); + DPU_REG_WRITE(c, LM_BLEND0_BG_ALPHA + stage_off, bg_alpha >> 8); DPU_REG_WRITE(c, LM_BLEND0_OP + stage_off, blend_op); } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.h index 1b9ecd082d7fd..380ca673f6de4 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.h @@ -25,39 +25,38 @@ struct dpu_hw_color3_cfg { }; /** - * * struct dpu_hw_lm_ops : Interface to the mixer Hw driver functions * Assumption is these functions will be called after clocks are enabled */ struct dpu_hw_lm_ops { - /* - * Sets up mixer output width and height + /** + * @setup_mixer_out: Sets up mixer output width and height * and border color if enabled */ void (*setup_mixer_out)(struct dpu_hw_mixer *ctx, struct dpu_hw_mixer_cfg *cfg); - /* - * Alpha blending configuration + /** + * @setup_blend_config: Alpha blending configuration * for the specified stage */ void (*setup_blend_config)(struct dpu_hw_mixer *ctx, uint32_t stage, - uint32_t fg_alpha, uint32_t bg_alpha, uint32_t blend_op); + u16 fg_alpha, u16 bg_alpha, uint32_t blend_op); - /* - * Alpha color component selection from either fg or bg + /** + * @setup_alpha_out: Alpha color component selection from either fg or bg */ void (*setup_alpha_out)(struct dpu_hw_mixer *ctx, uint32_t mixer_op); /** - * Clear layer mixer to pipe configuration + * @clear_all_blendstages: Clear layer mixer to pipe configuration * @ctx : mixer ctx pointer * Returns: 0 on success or -error */ int (*clear_all_blendstages)(struct dpu_hw_mixer *ctx); /** - * Configure layer mixer to pipe configuration + * @setup_blendstage: Configure layer mixer to pipe configuration * @ctx : mixer ctx pointer * @lm : layer mixer enumeration * @stage_cfg : blend stage configuration @@ -67,19 +66,19 @@ struct dpu_hw_lm_ops { struct dpu_hw_stage_cfg *stage_cfg); /** - * setup_border_color : enable/disable border color + * @setup_border_color : enable/disable border color */ void (*setup_border_color)(struct dpu_hw_mixer *ctx, struct dpu_mdss_color *color, u8 border_en); /** - * setup_misr: Enable/disable MISR + * @setup_misr: Enable/disable MISR */ void (*setup_misr)(struct dpu_hw_mixer *ctx); /** - * collect_misr: Read MISR signature + * @collect_misr: Read MISR signature */ int (*collect_misr)(struct dpu_hw_mixer *ctx, u32 *misr_value); }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h index 175639c8bfbb9..046b683d4c66d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h @@ -34,7 +34,9 @@ #define DPU_MAX_PLANES 4 #endif +#define STAGES_PER_PLANE 1 #define PIPES_PER_STAGE 2 +#define PIPES_PER_PLANE (PIPES_PER_STAGE * STAGES_PER_PLANE) #ifndef DPU_MAX_DE_CURVES #define DPU_MAX_DE_CURVES 3 #endif @@ -149,6 +151,10 @@ enum dpu_dspp { DSPP_1, DSPP_2, DSPP_3, + DSPP_4, + DSPP_5, + DSPP_6, + DSPP_7, DSPP_MAX }; @@ -159,6 +165,8 @@ enum dpu_ctl { CTL_3, CTL_4, CTL_5, + CTL_6, + CTL_7, CTL_MAX }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_merge3d.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_merge3d.h index 6833c02075236..b57f88187148b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_merge3d.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_merge3d.h @@ -12,7 +12,6 @@ struct dpu_hw_merge_3d; /** - * * struct dpu_hw_merge_3d_ops : Interface to the merge_3d Hw driver functions * Assumption is these functions will be called after clocks are enabled * @setup_3d_mode : enable 3D merge diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h index dd99e1c21a1ee..effd012d864af 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h @@ -34,7 +34,6 @@ struct dpu_hw_dither_cfg { }; /** - * * struct dpu_hw_pingpong_ops : Interface to the pingpong Hw driver functions * Assumption is these functions will be called after clocks are enabled * @enable_tearcheck: program and enable tear check block @@ -44,51 +43,52 @@ struct dpu_hw_dither_cfg { */ struct dpu_hw_pingpong_ops { /** - * enables vysnc generation and sets up init value of + * @enable_tearcheck: enables vysnc generation and sets up init value of * read pointer and programs the tear check cofiguration */ int (*enable_tearcheck)(struct dpu_hw_pingpong *pp, struct dpu_hw_tear_check *cfg); /** - * disables tear check block + * @disable_tearcheck: disables tear check block */ int (*disable_tearcheck)(struct dpu_hw_pingpong *pp); /** - * read, modify, write to either set or clear listening to external TE + * @connect_external_te: read, modify, write to either set or clear + * listening to external TE * @Return: 1 if TE was originally connected, 0 if not, or -ERROR */ int (*connect_external_te)(struct dpu_hw_pingpong *pp, bool enable_external_te); /** - * Obtain current vertical line counter + * @get_line_count: Obtain current vertical line counter */ u32 (*get_line_count)(struct dpu_hw_pingpong *pp); /** - * Disable autorefresh if enabled + * @disable_autorefresh: Disable autorefresh if enabled */ void (*disable_autorefresh)(struct dpu_hw_pingpong *pp, uint32_t encoder_id, u16 vdisplay); /** - * Setup dither matix for pingpong block + * @setup_dither: Setup dither matix for pingpong block */ void (*setup_dither)(struct dpu_hw_pingpong *pp, struct dpu_hw_dither_cfg *cfg); /** - * Enable DSC + * @enable_dsc: Enable DSC */ int (*enable_dsc)(struct dpu_hw_pingpong *pp); /** - * Disable DSC + * @disable_dsc: Disable DSC */ void (*disable_dsc)(struct dpu_hw_pingpong *pp); /** - * Setup DSC + * @setup_dsc: Setup DSC */ int (*setup_dsc)(struct dpu_hw_pingpong *pp); }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c index 6f1fc790ad6d8..d3da700092343 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c @@ -72,6 +72,8 @@ #define SSPP_EXCL_REC_XY_REC1 0x188 #define SSPP_EXCL_REC_SIZE 0x1B4 #define SSPP_EXCL_REC_XY 0x1B8 +#define SSPP_UBWC_STATIC_CTRL_REC1 0x1c0 +#define SSPP_UBWC_ERROR_STATUS_REC1 0x1c8 #define SSPP_CLK_CTRL 0x330 /* SSPP_SRC_OP_MODE & OP_MODE_REC1 */ @@ -145,11 +147,18 @@ static void dpu_hw_sspp_setup_multirect(struct dpu_sw_pipe *pipe) { struct dpu_hw_sspp *ctx = pipe->sspp; - u32 mode_mask; if (!ctx) return; + dpu_hw_setup_multirect_impl(pipe, ctx, SSPP_MULTIRECT_OPMODE); +} + +void dpu_hw_setup_multirect_impl(struct dpu_sw_pipe *pipe, + struct dpu_hw_sspp *ctx, u32 op_mode_off) +{ + u32 mode_mask; + if (pipe->multirect_index == DPU_SSPP_RECT_SOLO) { /** * if rect index is RECT_SOLO, we cannot expect a @@ -158,7 +167,7 @@ static void dpu_hw_sspp_setup_multirect(struct dpu_sw_pipe *pipe) */ mode_mask = 0; } else { - mode_mask = DPU_REG_READ(&ctx->hw, SSPP_MULTIRECT_OPMODE); + mode_mask = DPU_REG_READ(&ctx->hw, op_mode_off); mode_mask |= pipe->multirect_index; if (pipe->multirect_mode == DPU_SSPP_MULTIRECT_TIME_MX) mode_mask |= BIT(2); @@ -166,10 +175,10 @@ static void dpu_hw_sspp_setup_multirect(struct dpu_sw_pipe *pipe) mode_mask &= ~BIT(2); } - DPU_REG_WRITE(&ctx->hw, SSPP_MULTIRECT_OPMODE, mode_mask); + DPU_REG_WRITE(&ctx->hw, op_mode_off, mode_mask); } -static void _sspp_setup_opmode(struct dpu_hw_sspp *ctx, +void dpu_hw_sspp_setup_opmode(struct dpu_hw_sspp *ctx, u32 mask, u8 en) { const struct dpu_sspp_sub_blks *sblk = ctx->cap->sblk; @@ -189,7 +198,7 @@ static void _sspp_setup_opmode(struct dpu_hw_sspp *ctx, DPU_REG_WRITE(&ctx->hw, sblk->scaler_blk.base + SSPP_VIG_OP_MODE, opmode); } -static void _sspp_setup_csc10_opmode(struct dpu_hw_sspp *ctx, +void dpu_hw_sspp_setup_csc10_opmode(struct dpu_hw_sspp *ctx, u32 mask, u8 en) { const struct dpu_sspp_sub_blks *sblk = ctx->cap->sblk; @@ -211,11 +220,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, const struct msm_format *fmt, u32 flags) { struct dpu_hw_sspp *ctx = pipe->sspp; - struct dpu_hw_blk_reg_map *c; - u32 chroma_samp, unpack, src_format; - u32 opmode = 0; - u32 fast_clear = 0; - u32 op_mode_off, unpack_pat_off, format_off; + u32 op_mode_off, unpack_pat_off, format_off, ubwc_ctrl_off, ubwc_error_off; if (!ctx || !fmt) return; @@ -225,12 +230,46 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, op_mode_off = SSPP_SRC_OP_MODE; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN; format_off = SSPP_SRC_FORMAT; + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; } else { op_mode_off = SSPP_SRC_OP_MODE_REC1; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN_REC1; format_off = SSPP_SRC_FORMAT_REC1; + + /* reg wasn't present before DPU 8.0 */ + if (ctx->mdss_ver->core_major_ver >= 8) { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL_REC1; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS_REC1; + } else { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; + } } + if (fmt->fetch_mode != MDP_FETCH_LINEAR) { + u32 hbb = ctx->ubwc->highest_bank_bit - 13; + + DPU_REG_WRITE(&ctx->hw, SSPP_FETCH_CONFIG, + DPU_FETCH_CONFIG_RESET_VALUE | + hbb << 18); + } + + dpu_hw_setup_format_impl(pipe, fmt, flags, ctx, op_mode_off, + unpack_pat_off, format_off, + ubwc_ctrl_off, ubwc_error_off); +} + +void dpu_hw_setup_format_impl(struct dpu_sw_pipe *pipe, const struct msm_format *fmt, + u32 flags, struct dpu_hw_sspp *ctx, u32 op_mode_off, + u32 unpack_pat_off, u32 format_off, u32 ubwc_ctrl_off, + u32 ubwc_error_off) +{ + struct dpu_hw_blk_reg_map *c; + u32 chroma_samp, unpack, src_format; + u32 opmode; + u32 fast_clear; + c = &ctx->hw; opmode = DPU_REG_READ(c, op_mode_off); opmode &= ~(MDSS_MDP_OP_FLIP_LR | MDSS_MDP_OP_FLIP_UD | @@ -270,36 +309,38 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, ((fmt->bpp - 1) << 9); if (fmt->fetch_mode != MDP_FETCH_LINEAR) { + u32 hbb = ctx->ubwc->highest_bank_bit - 13; + u32 ctrl_val; + if (MSM_FORMAT_IS_UBWC(fmt)) opmode |= MDSS_MDP_OP_BWC_EN; src_format |= (fmt->fetch_mode & 3) << 30; /*FRAME_FORMAT */ - DPU_REG_WRITE(c, SSPP_FETCH_CONFIG, - DPU_FETCH_CONFIG_RESET_VALUE | - ctx->ubwc->highest_bank_bit << 18); - switch (ctx->ubwc->ubwc_enc_version) { - case UBWC_1_0: + + if (ctx->ubwc->ubwc_enc_version == UBWC_1_0) { fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) | - BIT(8) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_2_0: + ctrl_val = fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) | + BIT(8) | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_2_0) { fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - fast_clear | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_3_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - BIT(30) | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_4_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30)); - break; + ctrl_val = fast_clear | ctx->ubwc->ubwc_swizzle | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_3_0) { + ctrl_val = BIT(30) | (ctx->ubwc->ubwc_swizzle) | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_4_0) { + ctrl_val = MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30); + } else if (ctx->ubwc->ubwc_enc_version <= UBWC_6_0) { + if (MSM_FORMAT_IS_YUV(fmt)) + ctrl_val = 0; + else if (MSM_FORMAT_IS_DX(fmt)) /* or FP16, but it's unsupported */ + ctrl_val = BIT(30); + else + ctrl_val = BIT(30) | BIT(31); + /* SDE also sets bits for lossy formats, but we don't support them yet */ + } else { + DRM_WARN_ONCE("Unsupported UBWC version %x\n", ctx->ubwc->ubwc_enc_version); + ctrl_val = 0; } + + DPU_REG_WRITE(c, ubwc_ctrl_off, ctrl_val); } opmode |= MDSS_MDP_OP_PE_OVERRIDE; @@ -313,10 +354,10 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, /* update scaler opmode, if appropriate */ if (test_bit(DPU_SSPP_CSC, &ctx->cap->features)) - _sspp_setup_opmode(ctx, VIG_OP_CSC_EN | VIG_OP_CSC_SRC_DATAFMT, + dpu_hw_sspp_setup_opmode(ctx, VIG_OP_CSC_EN | VIG_OP_CSC_SRC_DATAFMT, MSM_FORMAT_IS_YUV(fmt)); else if (test_bit(DPU_SSPP_CSC_10BIT, &ctx->cap->features)) - _sspp_setup_csc10_opmode(ctx, + dpu_hw_sspp_setup_csc10_opmode(ctx, VIG_CSC_10_EN | VIG_CSC_10_SRC_DATAFMT, MSM_FORMAT_IS_YUV(fmt)); @@ -325,7 +366,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, DPU_REG_WRITE(c, op_mode_off, opmode); /* clear previous UBWC error */ - DPU_REG_WRITE(c, SSPP_UBWC_ERROR_STATUS, BIT(31)); + DPU_REG_WRITE(c, ubwc_error_off, BIT(31)); } static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_sspp *ctx, @@ -385,7 +426,7 @@ static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_sspp *ctx, tot_req_pixels[3]); } -static void _dpu_hw_sspp_setup_scaler3(struct dpu_hw_sspp *ctx, +void dpu_hw_sspp_setup_scaler3(struct dpu_hw_sspp *ctx, struct dpu_hw_scaler3_cfg *scaler3_cfg, const struct msm_format *format) { @@ -405,15 +446,11 @@ static void dpu_hw_sspp_setup_rects(struct dpu_sw_pipe *pipe, struct dpu_sw_pipe_cfg *cfg) { struct dpu_hw_sspp *ctx = pipe->sspp; - struct dpu_hw_blk_reg_map *c; - u32 src_size, src_xy, dst_size, dst_xy; u32 src_size_off, src_xy_off, out_size_off, out_xy_off; if (!ctx || !cfg) return; - c = &ctx->hw; - if (pipe->multirect_index == DPU_SSPP_RECT_SOLO || pipe->multirect_index == DPU_SSPP_RECT_0) { src_size_off = SSPP_SRC_SIZE; @@ -427,20 +464,8 @@ static void dpu_hw_sspp_setup_rects(struct dpu_sw_pipe *pipe, out_xy_off = SSPP_OUT_XY_REC1; } - - /* src and dest rect programming */ - src_xy = (cfg->src_rect.y1 << 16) | cfg->src_rect.x1; - src_size = (drm_rect_height(&cfg->src_rect) << 16) | - drm_rect_width(&cfg->src_rect); - dst_xy = (cfg->dst_rect.y1 << 16) | cfg->dst_rect.x1; - dst_size = (drm_rect_height(&cfg->dst_rect) << 16) | - drm_rect_width(&cfg->dst_rect); - - /* rectangle register programming */ - DPU_REG_WRITE(c, src_size_off, src_size); - DPU_REG_WRITE(c, src_xy_off, src_xy); - DPU_REG_WRITE(c, out_size_off, dst_size); - DPU_REG_WRITE(c, out_xy_off, dst_xy); + dpu_hw_setup_rects_impl(pipe, cfg, ctx, src_size_off, + src_xy_off, out_size_off, out_xy_off); } static void dpu_hw_sspp_setup_sourceaddress(struct dpu_sw_pipe *pipe, @@ -497,7 +522,7 @@ static void dpu_hw_sspp_setup_sourceaddress(struct dpu_sw_pipe *pipe, DPU_REG_WRITE(&ctx->hw, SSPP_SRC_YSTRIDE1, ystride1); } -static void dpu_hw_sspp_setup_csc(struct dpu_hw_sspp *ctx, +void dpu_hw_sspp_setup_csc(struct dpu_hw_sspp *ctx, const struct dpu_csc_cfg *data) { u32 offset; @@ -519,21 +544,31 @@ static void dpu_hw_sspp_setup_csc(struct dpu_hw_sspp *ctx, static void dpu_hw_sspp_setup_solidfill(struct dpu_sw_pipe *pipe, u32 color) { struct dpu_hw_sspp *ctx = pipe->sspp; - struct dpu_hw_fmt_layout cfg; + u32 const_clr_off; if (!ctx) return; + if (pipe->multirect_index == DPU_SSPP_RECT_SOLO || + pipe->multirect_index == DPU_SSPP_RECT_0) + const_clr_off = SSPP_SRC_CONSTANT_COLOR; + else + const_clr_off = SSPP_SRC_CONSTANT_COLOR_REC1; + + dpu_hw_setup_solidfill_impl(pipe, color, ctx, const_clr_off); +} + +void dpu_hw_setup_solidfill_impl(struct dpu_sw_pipe *pipe, + u32 color, struct dpu_hw_sspp *ctx, + u32 const_clr_off) +{ + struct dpu_hw_fmt_layout cfg; + /* cleanup source addresses */ memset(&cfg, 0, sizeof(cfg)); ctx->ops.setup_sourceaddress(pipe, &cfg); - if (pipe->multirect_index == DPU_SSPP_RECT_SOLO || - pipe->multirect_index == DPU_SSPP_RECT_0) - DPU_REG_WRITE(&ctx->hw, SSPP_SRC_CONSTANT_COLOR, color); - else - DPU_REG_WRITE(&ctx->hw, SSPP_SRC_CONSTANT_COLOR_REC1, - color); + DPU_REG_WRITE(&ctx->hw, const_clr_off, color); } static void dpu_hw_sspp_setup_qos_lut(struct dpu_hw_sspp *ctx, @@ -547,14 +582,20 @@ static void dpu_hw_sspp_setup_qos_lut(struct dpu_hw_sspp *ctx, cfg); } +void dpu_hw_sspp_setup_qos_ctrl_impl(struct dpu_hw_sspp *ctx, + bool danger_safe_en, u32 ctrl_off) +{ + DPU_REG_WRITE(&ctx->hw, ctrl_off, + danger_safe_en ? SSPP_QOS_CTRL_DANGER_SAFE_EN : 0); +} + static void dpu_hw_sspp_setup_qos_ctrl(struct dpu_hw_sspp *ctx, bool danger_safe_en) { if (!ctx) return; - DPU_REG_WRITE(&ctx->hw, SSPP_QOS_CTRL, - danger_safe_en ? SSPP_QOS_CTRL_DANGER_SAFE_EN : 0); + dpu_hw_sspp_setup_qos_ctrl_impl(ctx, danger_safe_en, SSPP_QOS_CTRL); } static void dpu_hw_sspp_setup_cdp(struct dpu_sw_pipe *pipe, @@ -609,7 +650,7 @@ static void _setup_layer_ops(struct dpu_hw_sspp *c, c->ops.setup_multirect = dpu_hw_sspp_setup_multirect; if (test_bit(DPU_SSPP_SCALER_QSEED3_COMPATIBLE, &features)) - c->ops.setup_scaler = _dpu_hw_sspp_setup_scaler3; + c->ops.setup_scaler = dpu_hw_sspp_setup_scaler3; if (test_bit(DPU_SSPP_CDP, &features)) c->ops.setup_cdp = dpu_hw_sspp_setup_cdp; @@ -706,7 +747,10 @@ struct dpu_hw_sspp *dpu_hw_sspp_init(struct drm_device *dev, hw_pipe->mdss_ver = mdss_rev; - _setup_layer_ops(hw_pipe, hw_pipe->cap->features, mdss_rev); + if (mdss_rev->core_major_ver >= 13) + dpu_hw_sspp_init_v13(hw_pipe, hw_pipe->cap->features, mdss_rev); + else + _setup_layer_ops(hw_pipe, hw_pipe->cap->features, mdss_rev); return hw_pipe; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h index bdac5c04bf790..69d68cc916b2e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h @@ -14,7 +14,7 @@ struct dpu_hw_sspp; #define DPU_SSPP_MAX_PITCH_SIZE 0xffff -/** +/* * Flags */ #define DPU_SSPP_FLIP_LR BIT(0) @@ -23,7 +23,7 @@ struct dpu_hw_sspp; #define DPU_SSPP_ROT_90 BIT(3) #define DPU_SSPP_SOLID_FILL BIT(4) -/** +/* * Component indices */ enum { @@ -36,9 +36,10 @@ enum { }; /** - * DPU_SSPP_RECT_SOLO - multirect disabled - * DPU_SSPP_RECT_0 - rect0 of a multirect pipe - * DPU_SSPP_RECT_1 - rect1 of a multirect pipe + * enum dpu_sspp_multirect_index - multirect mode + * @DPU_SSPP_RECT_SOLO: multirect disabled + * @DPU_SSPP_RECT_0: rect0 of a multirect pipe + * @DPU_SSPP_RECT_1: rect1 of a multirect pipe * * Note: HW supports multirect with either RECT0 or * RECT1. Considering no benefit of such configs over @@ -143,7 +144,7 @@ struct dpu_hw_pixel_ext { * struct dpu_sw_pipe_cfg : software pipe configuration * @src_rect: src ROI, caller takes into account the different operations * such as decimation, flip etc to program this field - * @dest_rect: destination ROI. + * @dst_rect: destination ROI. * @rotation: simplified drm rotation hint */ struct dpu_sw_pipe_cfg { @@ -165,8 +166,8 @@ struct dpu_hw_pipe_ts_cfg { /** * struct dpu_sw_pipe - software pipe description * @sspp: backing SSPP pipe - * @index: index of the rectangle of SSPP - * @mode: parallel or time multiplex multirect mode + * @multirect_index: index of the rectangle of SSPP + * @multirect_mode: parallel or time multiplex multirect mode */ struct dpu_sw_pipe { struct dpu_hw_sspp *sspp; @@ -181,7 +182,7 @@ struct dpu_sw_pipe { */ struct dpu_hw_sspp_ops { /** - * setup_format - setup pixel format cropping rectangle, flip + * @setup_format: setup pixel format cropping rectangle, flip * @pipe: Pointer to software pipe context * @cfg: Pointer to pipe config structure * @flags: Extra flags for format config @@ -190,7 +191,7 @@ struct dpu_hw_sspp_ops { const struct msm_format *fmt, u32 flags); /** - * setup_rects - setup pipe ROI rectangles + * @setup_rects: setup pipe ROI rectangles * @pipe: Pointer to software pipe context * @cfg: Pointer to pipe config structure */ @@ -198,7 +199,7 @@ struct dpu_hw_sspp_ops { struct dpu_sw_pipe_cfg *cfg); /** - * setup_pe - setup pipe pixel extension + * @setup_pe: setup pipe pixel extension * @ctx: Pointer to pipe context * @pe_ext: Pointer to pixel ext settings */ @@ -206,7 +207,7 @@ struct dpu_hw_sspp_ops { struct dpu_hw_pixel_ext *pe_ext); /** - * setup_sourceaddress - setup pipe source addresses + * @setup_sourceaddress: setup pipe source addresses * @pipe: Pointer to software pipe context * @layout: format layout information for programming buffer to hardware */ @@ -214,14 +215,14 @@ struct dpu_hw_sspp_ops { struct dpu_hw_fmt_layout *layout); /** - * setup_csc - setup color space coversion + * @setup_csc: setup color space coversion * @ctx: Pointer to pipe context * @data: Pointer to config structure */ void (*setup_csc)(struct dpu_hw_sspp *ctx, const struct dpu_csc_cfg *data); /** - * setup_solidfill - enable/disable colorfill + * @setup_solidfill: enable/disable colorfill * @pipe: Pointer to software pipe context * @const_color: Fill color value * @flags: Pipe flags @@ -229,23 +230,22 @@ struct dpu_hw_sspp_ops { void (*setup_solidfill)(struct dpu_sw_pipe *pipe, u32 color); /** - * setup_multirect - setup multirect configuration + * @setup_multirect: setup multirect configuration * @pipe: Pointer to software pipe context */ void (*setup_multirect)(struct dpu_sw_pipe *pipe); /** - * setup_sharpening - setup sharpening + * @setup_sharpening: setup sharpening * @ctx: Pointer to pipe context * @cfg: Pointer to config structure */ void (*setup_sharpening)(struct dpu_hw_sspp *ctx, struct dpu_hw_sharp_cfg *cfg); - /** - * setup_qos_lut - setup QoS LUTs + * @setup_qos_lut: setup QoS LUTs * @ctx: Pointer to pipe context * @cfg: LUT configuration */ @@ -253,7 +253,7 @@ struct dpu_hw_sspp_ops { struct dpu_hw_qos_cfg *cfg); /** - * setup_qos_ctrl - setup QoS control + * @setup_qos_ctrl: setup QoS control * @ctx: Pointer to pipe context * @danger_safe_en: flags controlling enabling of danger/safe QoS/LUT */ @@ -261,7 +261,7 @@ struct dpu_hw_sspp_ops { bool danger_safe_en); /** - * setup_clk_force_ctrl - setup clock force control + * @setup_clk_force_ctrl: setup clock force control * @ctx: Pointer to pipe context * @enable: enable clock force if true */ @@ -269,7 +269,7 @@ struct dpu_hw_sspp_ops { bool enable); /** - * setup_histogram - setup histograms + * @setup_histogram: setup histograms * @ctx: Pointer to pipe context * @cfg: Pointer to histogram configuration */ @@ -277,7 +277,7 @@ struct dpu_hw_sspp_ops { void *cfg); /** - * setup_scaler - setup scaler + * @setup_scaler: setup scaler * @scaler3_cfg: Pointer to scaler configuration * @format: pixel format parameters */ @@ -286,7 +286,7 @@ struct dpu_hw_sspp_ops { const struct msm_format *format); /** - * setup_cdp - setup client driven prefetch + * @setup_cdp: setup client driven prefetch * @pipe: Pointer to software pipe context * @fmt: format used by the sw pipe * @enable: whether the CDP should be enabled for this pipe @@ -303,6 +303,7 @@ struct dpu_hw_sspp_ops { * @ubwc: UBWC configuration data * @idx: pipe index * @cap: pointer to layer_cfg + * @mdss_ver: MDSS version info to use for feature checks * @ops: pointer to operations possible for this pipe */ struct dpu_hw_sspp { @@ -331,5 +332,61 @@ struct dpu_hw_sspp *dpu_hw_sspp_init(struct drm_device *dev, int _dpu_hw_sspp_init_debugfs(struct dpu_hw_sspp *hw_pipe, struct dpu_kms *kms, struct dentry *entry); +void dpu_hw_sspp_setup_opmode(struct dpu_hw_sspp *ctx, + u32 mask, u8 en); + +void dpu_hw_sspp_setup_csc10_opmode(struct dpu_hw_sspp *ctx, + u32 mask, u8 en); + +void dpu_hw_sspp_setup_scaler3(struct dpu_hw_sspp *ctx, + struct dpu_hw_scaler3_cfg *scaler3_cfg, + const struct msm_format *format); + +void dpu_hw_sspp_setup_csc(struct dpu_hw_sspp *ctx, + const struct dpu_csc_cfg *data); + +void dpu_hw_setup_multirect_impl(struct dpu_sw_pipe *pipe, + struct dpu_hw_sspp *ctx, + u32 op_mode_off); + +void dpu_hw_setup_format_impl(struct dpu_sw_pipe *pipe, const struct msm_format *fmt, + u32 flags, struct dpu_hw_sspp *ctx, + u32 op_mode_off, u32 unpack_pat_off, u32 format_off, + u32 ubwc_ctrl_off, u32 ubwc_err_off); + +static inline void dpu_hw_setup_rects_impl(struct dpu_sw_pipe *pipe, struct dpu_sw_pipe_cfg *cfg, + struct dpu_hw_sspp *ctx, u32 src_size_off, + u32 src_xy_off, u32 out_size_off, u32 out_xy_off) +{ + struct dpu_hw_blk_reg_map *c; + u32 src_size, src_xy, dst_size, dst_xy; + + c = &ctx->hw; + + /* src and dest rect programming */ + src_xy = (cfg->src_rect.y1 << 16) | cfg->src_rect.x1; + src_size = (drm_rect_height(&cfg->src_rect) << 16) | + drm_rect_width(&cfg->src_rect); + dst_xy = (cfg->dst_rect.y1 << 16) | cfg->dst_rect.x1; + dst_size = (drm_rect_height(&cfg->dst_rect) << 16) | + drm_rect_width(&cfg->dst_rect); + + /* rectangle register programming */ + DPU_REG_WRITE(c, src_size_off, src_size); + DPU_REG_WRITE(c, src_xy_off, src_xy); + DPU_REG_WRITE(c, out_size_off, dst_size); + DPU_REG_WRITE(c, out_xy_off, dst_xy); +} + +void dpu_hw_setup_solidfill_impl(struct dpu_sw_pipe *pipe, + u32 color, struct dpu_hw_sspp *ctx, u32 const_clr_off); + +void dpu_hw_sspp_setup_qos_ctrl_impl(struct dpu_hw_sspp *ctx, + bool danger_safe_en, u32 ctrl_off); + +void dpu_hw_sspp_init_v13(struct dpu_hw_sspp *c, + unsigned long features, + const struct dpu_mdss_version *mdss_rev); + #endif /*_DPU_HW_SSPP_H */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp_v13.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp_v13.c new file mode 100644 index 0000000000000..f8f96ad971d78 --- /dev/null +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp_v13.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include + +#include "dpu_hw_sspp.h" + +/* >= v13 DPU */ +/* CMN Registers -> Source Surface Processing Pipe Common SSPP registers */ +/* Name Offset */ +#define SSPP_CMN_CLK_CTRL 0x0 +#define SSPP_CMN_CLK_STATUS 0x4 +#define SSPP_CMN_MULTI_REC_OP_MODE 0x10 +#define SSPP_CMN_ADDR_CONFIG 0x14 +#define SSPP_CMN_CAC_CTRL 0x20 +#define SSPP_CMN_SYS_CACHE_MODE 0x24 +#define SSPP_CMN_QOS_CTRL 0x28 + +#define SSPP_CMN_FILL_LEVEL_SCALE 0x3c +#define SSPP_CMN_FILL_LEVELS 0x40 +#define SSPP_CMN_STATUS 0x44 +#define SSPP_CMN_FETCH_DMA_RD_OTS 0x48 +#define SSPP_CMN_FETCH_DTB_WR_PLANE0 0x4c +#define SSPP_CMN_FETCH_DTB_WR_PLANE1 0x50 +#define SSPP_CMN_FETCH_DTB_WR_PLANE2 0x54 +#define SSPP_CMN_DTB_UNPACK_RD_PLANE0 0x58 +#define SSPP_CMN_DTB_UNPACK_RD_PLANE1 0x5c +#define SSPP_CMN_DTB_UNPACK_RD_PLANE2 0x60 +#define SSPP_CMN_UNPACK_LINE_COUNT 0x64 +#define SSPP_CMN_TPG_CONTROL 0x68 +#define SSPP_CMN_TPG_CONFIG 0x6c +#define SSPP_CMN_TPG_COMPONENT_LIMITS 0x70 +#define SSPP_CMN_TPG_RECTANGLE 0x74 +#define SSPP_CMN_TPG_BLACK_WHITE_PATTERN_FRAMES 0x78 +#define SSPP_CMN_TPG_RGB_MAPPING 0x7c +#define SSPP_CMN_TPG_PATTERN_GEN_INIT_VAL 0x80 + +/*RECRegisterset*/ +/*Name Offset*/ +#define SSPP_REC_SRC_FORMAT 0x0 +#define SSPP_REC_SRC_UNPACK_PATTERN 0x4 +#define SSPP_REC_SRC_OP_MODE 0x8 +#define SSPP_REC_SRC_CONSTANT_COLOR 0xc +#define SSPP_REC_SRC_IMG_SIZE 0x10 +#define SSPP_REC_SRC_SIZE 0x14 +#define SSPP_REC_SRC_XY 0x18 +#define SSPP_REC_OUT_SIZE 0x1c +#define SSPP_REC_OUT_XY 0x20 +#define SSPP_REC_SW_PIX_EXT_LR 0x24 +#define SSPP_REC_SW_PIX_EXT_TB 0x28 +#define SSPP_REC_SRC_SIZE_ODX 0x30 +#define SSPP_REC_SRC_XY_ODX 0x34 +#define SSPP_REC_OUT_SIZE_ODX 0x38 +#define SSPP_REC_OUT_XY_ODX 0x3c +#define SSPP_REC_SW_PIX_EXT_LR_ODX 0x40 +#define SSPP_REC_SW_PIX_EXT_TB_ODX 0x44 +#define SSPP_REC_PRE_DOWN_SCALE 0x48 +#define SSPP_REC_SRC0_ADDR 0x4c +#define SSPP_REC_SRC1_ADDR 0x50 +#define SSPP_REC_SRC2_ADDR 0x54 +#define SSPP_REC_SRC3_ADDR 0x58 +#define SSPP_REC_SRC_YSTRIDE0 0x5c +#define SSPP_REC_SRC_YSTRIDE1 0x60 +#define SSPP_REC_CURRENT_SRC0_ADDR 0x64 +#define SSPP_REC_CURRENT_SRC1_ADDR 0x68 +#define SSPP_REC_CURRENT_SRC2_ADDR 0x6c +#define SSPP_REC_CURRENT_SRC3_ADDR 0x70 +#define SSPP_REC_SRC_ADDR_SW_STATUS 0x74 +#define SSPP_REC_CDP_CNTL 0x78 +#define SSPP_REC_TRAFFIC_SHAPER 0x7c +#define SSPP_REC_TRAFFIC_SHAPER_PREFILL 0x80 +#define SSPP_REC_PD_MEM_ALLOC 0x84 +#define SSPP_REC_QOS_CLAMP 0x88 +#define SSPP_REC_UIDLE_CTRL_VALUE 0x8c +#define SSPP_REC_UBWC_STATIC_CTRL 0x90 +#define SSPP_REC_UBWC_STATIC_CTRL_OVERRIDE 0x94 +#define SSPP_REC_UBWC_STATS_ROI 0x98 +#define SSPP_REC_UBWC_STATS_WORST_TILE_ROW_BW_ROI0 0x9c +#define SSPP_REC_UBWC_STATS_TOTAL_BW_ROI0 0xa0 +#define SSPP_REC_UBWC_STATS_WORST_TILE_ROW_BW_ROI1 0xa4 +#define SSPP_REC_UBWC_STATS_TOTAL_BW_ROI1 0xa8 +#define SSPP_REC_UBWC_STATS_WORST_TILE_ROW_BW_ROI2 0xac +#define SSPP_REC_UBWC_STATS_TOTAL_BW_ROI2 0xb0 +#define SSPP_REC_EXCL_REC_CTRL 0xb4 +#define SSPP_REC_EXCL_REC_SIZE 0xb8 +#define SSPP_REC_EXCL_REC_XY 0xbc +#define SSPP_REC_LINE_INSERTION_CTRL 0xc0 +#define SSPP_REC_LINE_INSERTION_OUT_SIZE 0xc4 +#define SSPP_REC_FETCH_PIPE_ACTIVE 0xc8 +#define SSPP_REC_META_ERROR_STATUS 0xcc +#define SSPP_REC_UBWC_ERROR_STATUS 0xd0 +#define SSPP_REC_FLUSH_CTRL 0xd4 +#define SSPP_REC_INTR_EN 0xd8 +#define SSPP_REC_INTR_STATUS 0xdc +#define SSPP_REC_INTR_CLEAR 0xe0 +#define SSPP_REC_HSYNC_STATUS 0xe4 +#define SSPP_REC_FP16_CONFIG 0x150 +#define SSPP_REC_FP16_CSC_MATRIX_COEFF_R_0 0x154 +#define SSPP_REC_FP16_CSC_MATRIX_COEFF_R_1 0x158 +#define SSPP_REC_FP16_CSC_MATRIX_COEFF_G_0 0x15c +#define SSPP_REC_FP16_CSC_MATRIX_COEFF_G_1 0x160 +#define SSPP_REC_FP16_CSC_MATRIX_COEFF_B_0 0x164 +#define SSPP_REC_FP16_CSC_MATRIX_COEFF_B_1 0x168 +#define SSPP_REC_FP16_CSC_PRE_CLAMP_R 0x16c +#define SSPP_REC_FP16_CSC_PRE_CLAMP_G 0x170 +#define SSPP_REC_FP16_CSC_PRE_CLAMP_B 0x174 +#define SSPP_REC_FP16_CSC_POST_CLAMP 0x178 + +static inline u32 dpu_hw_sspp_calculate_rect_off(enum dpu_sspp_multirect_index rect_index, + struct dpu_hw_sspp *ctx) +{ + return (rect_index == DPU_SSPP_RECT_SOLO || rect_index == DPU_SSPP_RECT_0) ? + ctx->cap->sblk->sspp_rec0_blk.base : ctx->cap->sblk->sspp_rec1_blk.base; +} + +static void dpu_hw_sspp_setup_multirect_v13(struct dpu_sw_pipe *pipe) +{ + struct dpu_hw_sspp *ctx = pipe->sspp; + + if (!ctx) + return; + + dpu_hw_setup_multirect_impl(pipe, ctx, SSPP_CMN_MULTI_REC_OP_MODE); +} + +static void dpu_hw_sspp_setup_format_v13(struct dpu_sw_pipe *pipe, + const struct msm_format *fmt, u32 flags) +{ + struct dpu_hw_sspp *ctx = pipe->sspp; + u32 op_mode_off, unpack_pat_off, format_off; + u32 ubwc_ctrl_off, ubwc_err_off; + u32 offset; + + if (!ctx || !fmt) + return; + + offset = dpu_hw_sspp_calculate_rect_off(pipe->multirect_index, ctx); + + op_mode_off = offset + SSPP_REC_SRC_OP_MODE; + unpack_pat_off = offset + SSPP_REC_SRC_UNPACK_PATTERN; + format_off = offset + SSPP_REC_SRC_FORMAT; + ubwc_ctrl_off = offset + SSPP_REC_UBWC_STATIC_CTRL; + ubwc_err_off = offset + SSPP_REC_UBWC_ERROR_STATUS; + + dpu_hw_setup_format_impl(pipe, fmt, flags, ctx, op_mode_off, + unpack_pat_off, format_off, ubwc_ctrl_off, ubwc_err_off); +} + +static void dpu_hw_sspp_setup_pe_config_v13(struct dpu_hw_sspp *ctx, + struct dpu_hw_pixel_ext *pe_ext) +{ + struct dpu_hw_blk_reg_map *c; + u8 color; + u32 lr_pe[4], tb_pe[4]; + const u32 bytemask = 0xff; + u32 offset; + + if (!ctx || !pe_ext) + return; + + offset = ctx->cap->sblk->sspp_rec0_blk.base; + + c = &ctx->hw; + /* program SW pixel extension override for all pipes*/ + for (color = 0; color < DPU_MAX_PLANES; color++) { + /* color 2 has the same set of registers as color 1 */ + if (color == 2) + continue; + + lr_pe[color] = ((pe_ext->right_ftch[color] & bytemask) << 24) | + ((pe_ext->right_rpt[color] & bytemask) << 16) | + ((pe_ext->left_ftch[color] & bytemask) << 8) | + (pe_ext->left_rpt[color] & bytemask); + + tb_pe[color] = ((pe_ext->btm_ftch[color] & bytemask) << 24) | + ((pe_ext->btm_rpt[color] & bytemask) << 16) | + ((pe_ext->top_ftch[color] & bytemask) << 8) | + (pe_ext->top_rpt[color] & bytemask); + } + + /* color 0 */ + DPU_REG_WRITE(c, SSPP_REC_SW_PIX_EXT_LR + offset, lr_pe[0]); + DPU_REG_WRITE(c, SSPP_REC_SW_PIX_EXT_TB + offset, tb_pe[0]); + + /* color 1 and color 2 */ + DPU_REG_WRITE(c, SSPP_REC_SW_PIX_EXT_LR_ODX + offset, lr_pe[1]); + DPU_REG_WRITE(c, SSPP_REC_SW_PIX_EXT_TB_ODX + offset, tb_pe[1]); +} + +static void dpu_hw_sspp_setup_rects_v13(struct dpu_sw_pipe *pipe, + struct dpu_sw_pipe_cfg *cfg) +{ + struct dpu_hw_sspp *ctx = pipe->sspp; + u32 src_size_off, src_xy_off, out_size_off, out_xy_off; + u32 offset; + + if (!ctx || !cfg) + return; + + offset = dpu_hw_sspp_calculate_rect_off(pipe->multirect_index, ctx); + + src_size_off = offset + SSPP_REC_SRC_SIZE; + src_xy_off = offset + SSPP_REC_SRC_XY; + out_size_off = offset + SSPP_REC_OUT_SIZE; + out_xy_off = offset + SSPP_REC_OUT_XY; + + dpu_hw_setup_rects_impl(pipe, cfg, ctx, src_size_off, + src_xy_off, out_size_off, out_xy_off); +} + +static void dpu_hw_sspp_setup_sourceaddress_v13(struct dpu_sw_pipe *pipe, + struct dpu_hw_fmt_layout *layout) +{ + struct dpu_hw_sspp *ctx = pipe->sspp; + int i; + u32 offset, ystride0, ystride1; + + if (!ctx) + return; + + offset = dpu_hw_sspp_calculate_rect_off(pipe->multirect_index, ctx); + + for (i = 0; i < ARRAY_SIZE(layout->plane_addr); i++) + DPU_REG_WRITE(&ctx->hw, offset + SSPP_REC_SRC0_ADDR + i * 0x4, + layout->plane_addr[i]); + + ystride0 = (layout->plane_pitch[0]) | (layout->plane_pitch[2] << 16); + ystride1 = (layout->plane_pitch[1]) | (layout->plane_pitch[3] << 16); + + DPU_REG_WRITE(&ctx->hw, offset + SSPP_REC_SRC_YSTRIDE0, ystride0); + DPU_REG_WRITE(&ctx->hw, offset + SSPP_REC_SRC_YSTRIDE1, ystride1); +} + +static void dpu_hw_sspp_setup_solidfill_v13(struct dpu_sw_pipe *pipe, u32 color) +{ + struct dpu_hw_sspp *ctx = pipe->sspp; + u32 const_clr_off; + u32 offset; + + if (!ctx) + return; + + offset = dpu_hw_sspp_calculate_rect_off(pipe->multirect_index, ctx); + const_clr_off = offset + SSPP_REC_SRC_CONSTANT_COLOR; + + dpu_hw_setup_solidfill_impl(pipe, color, ctx, const_clr_off); +} + +static void dpu_hw_sspp_setup_qos_lut_v13(struct dpu_hw_sspp *ctx, + struct dpu_hw_qos_cfg *cfg) +{ + if (!ctx || !cfg) + return; + + dpu_hw_setup_qos_lut_v13(&ctx->hw, cfg); +} + +static void dpu_hw_sspp_setup_qos_ctrl_v13(struct dpu_hw_sspp *ctx, + bool danger_safe_en) +{ + if (!ctx) + return; + + dpu_hw_sspp_setup_qos_ctrl_impl(ctx, danger_safe_en, SSPP_CMN_QOS_CTRL); +} + +static void dpu_hw_sspp_setup_cdp_v13(struct dpu_sw_pipe *pipe, + const struct msm_format *fmt, + bool enable) +{ + struct dpu_hw_sspp *ctx = pipe->sspp; + u32 offset = 0; + + if (!ctx) + return; + + offset = dpu_hw_sspp_calculate_rect_off(pipe->multirect_index, ctx); + dpu_setup_cdp(&ctx->hw, offset + SSPP_REC_CDP_CNTL, fmt, enable); +} + +static bool dpu_hw_sspp_setup_clk_force_ctrl_v13(struct dpu_hw_sspp *ctx, bool enable) +{ + static const struct dpu_clk_ctrl_reg sspp_clk_ctrl = { + .reg_off = SSPP_CMN_CLK_CTRL, + .bit_off = 0 + }; + + return dpu_hw_clk_force_ctrl(&ctx->hw, &sspp_clk_ctrl, enable); +} + +void dpu_hw_sspp_init_v13(struct dpu_hw_sspp *c, + unsigned long features, const struct dpu_mdss_version *mdss_rev) +{ + c->ops.setup_format = dpu_hw_sspp_setup_format_v13; + c->ops.setup_rects = dpu_hw_sspp_setup_rects_v13; + c->ops.setup_sourceaddress = dpu_hw_sspp_setup_sourceaddress_v13; + c->ops.setup_solidfill = dpu_hw_sspp_setup_solidfill_v13; + c->ops.setup_pe = dpu_hw_sspp_setup_pe_config_v13; + + if (test_bit(DPU_SSPP_QOS, &features)) { + c->ops.setup_qos_lut = dpu_hw_sspp_setup_qos_lut_v13; + c->ops.setup_qos_ctrl = dpu_hw_sspp_setup_qos_ctrl_v13; + } + + if (test_bit(DPU_SSPP_CSC, &features) || + test_bit(DPU_SSPP_CSC_10BIT, &features)) + c->ops.setup_csc = dpu_hw_sspp_setup_csc; + + if (test_bit(DPU_SSPP_SMART_DMA_V1, &c->cap->features) || + test_bit(DPU_SSPP_SMART_DMA_V2, &c->cap->features)) + c->ops.setup_multirect = dpu_hw_sspp_setup_multirect_v13; + + if (test_bit(DPU_SSPP_SCALER_QSEED3_COMPATIBLE, &features)) + c->ops.setup_scaler = dpu_hw_sspp_setup_scaler3; + + if (test_bit(DPU_SSPP_CDP, &features)) + c->ops.setup_cdp = dpu_hw_sspp_setup_cdp_v13; + + c->ops.setup_clk_force_ctrl = dpu_hw_sspp_setup_clk_force_ctrl_v13; +} diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c index 96dc10589bee6..1ebd75d4f9be8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c @@ -22,13 +22,6 @@ #define TRAFFIC_SHAPER_WR_CLIENT(num) (0x060 + (num * 4)) #define TRAFFIC_SHAPER_FIXPOINT_FACTOR 4 -#define MDP_TICK_COUNT 16 -#define XO_CLK_RATE 19200 -#define MS_TICKS_IN_SEC 1000 - -#define CALCULATE_WD_LOAD_VALUE(fps) \ - ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) - static void dpu_hw_setup_split_pipe(struct dpu_hw_mdp *mdp, struct split_pipe_cfg *cfg) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.h index 04efdcd21ceb0..80279d87c2cde 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.h @@ -77,12 +77,11 @@ enum dpu_dp_phy_sel { /** * struct dpu_hw_mdp_ops - interface to the MDP TOP Hw driver functions * Assumption is these functions will be called after clocks are enabled. - * @setup_split_pipe : Programs the pipe control registers - * @setup_pp_split : Programs the pp split control registers - * @setup_traffic_shaper : programs traffic shaper control */ struct dpu_hw_mdp_ops { - /** setup_split_pipe() : Registers are not double buffered, thisk + /** + * @setup_split_pipe : Programs the pipe control registers. + * Registers are not double buffered, this * function should be called before timing control enable * @mdp : mdp top context driver * @cfg : upper and lower part of pipe configuration @@ -91,7 +90,7 @@ struct dpu_hw_mdp_ops { struct split_pipe_cfg *p); /** - * setup_traffic_shaper() : Setup traffic shaper control + * @setup_traffic_shaper : programs traffic shaper control. * @mdp : mdp top context driver * @cfg : traffic shaper configuration */ @@ -99,7 +98,7 @@ struct dpu_hw_mdp_ops { struct traffic_shaper_cfg *cfg); /** - * setup_clk_force_ctrl - set clock force control + * @setup_clk_force_ctrl: set clock force control * @mdp: mdp top context driver * @clk_ctrl: clock to be controlled * @enable: force on enable @@ -109,7 +108,7 @@ struct dpu_hw_mdp_ops { enum dpu_clk_ctrl_type clk_ctrl, bool enable); /** - * get_danger_status - get danger status + * @get_danger_status: get danger status * @mdp: mdp top context driver * @status: Pointer to danger safe status */ @@ -117,7 +116,7 @@ struct dpu_hw_mdp_ops { struct dpu_danger_safe_status *status); /** - * setup_vsync_source - setup vsync source configuration details + * @setup_vsync_source: setup vsync source configuration details * @mdp: mdp top context driver * @cfg: vsync source selection configuration */ @@ -125,7 +124,7 @@ struct dpu_hw_mdp_ops { struct dpu_vsync_source_cfg *cfg); /** - * get_safe_status - get safe status + * @get_safe_status: get safe status * @mdp: mdp top context driver * @status: Pointer to danger safe status */ @@ -133,14 +132,14 @@ struct dpu_hw_mdp_ops { struct dpu_danger_safe_status *status); /** - * dp_phy_intf_sel - configure intf to phy mapping + * @dp_phy_intf_sel: configure intf to phy mapping * @mdp: mdp top context driver * @phys: list of phys the DP interfaces should be connected to. 0 disables the INTF. */ void (*dp_phy_intf_sel)(struct dpu_hw_mdp *mdp, enum dpu_dp_phy_sel phys[2]); /** - * intf_audio_select - select the external interface for audio + * @intf_audio_select: select the external interface for audio * @mdp: mdp top context driver */ void (*intf_audio_select)(struct dpu_hw_mdp *mdp); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c index 486be346d40d8..c7a5f4a0d0541 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c @@ -81,6 +81,13 @@ static u32 dpu_hw_util_log_mask = DPU_DBG_MASK_NONE; #define QOS_CREQ_LUT_0 0x14 #define QOS_CREQ_LUT_1 0x18 +/* CMN_QOS_LUT */ +#define SSPP_CMN_QOS_CTRL 0x28 +#define SSPP_CMN_DANGER_LUT 0x2c +#define SSPP_CMN_SAFE_LUT 0x30 +#define SSPP_CMN_CREQ_LUT_0 0x34 +#define SSPP_CMN_CREQ_LUT_1 0x38 + /* QOS_QOS_CTRL */ #define QOS_QOS_CTRL_DANGER_SAFE_EN BIT(0) #define QOS_QOS_CTRL_DANGER_VBLANK_MASK GENMASK(5, 4) @@ -475,6 +482,17 @@ void _dpu_hw_setup_qos_lut(struct dpu_hw_blk_reg_map *c, u32 offset, cfg->danger_safe_en ? QOS_QOS_CTRL_DANGER_SAFE_EN : 0); } +void dpu_hw_setup_qos_lut_v13(struct dpu_hw_blk_reg_map *c, + const struct dpu_hw_qos_cfg *cfg) +{ + DPU_REG_WRITE(c, SSPP_CMN_DANGER_LUT, cfg->danger_lut); + DPU_REG_WRITE(c, SSPP_CMN_SAFE_LUT, cfg->safe_lut); + DPU_REG_WRITE(c, SSPP_CMN_CREQ_LUT_0, cfg->creq_lut); + DPU_REG_WRITE(c, SSPP_CMN_CREQ_LUT_1, cfg->creq_lut >> 32); + DPU_REG_WRITE(c, SSPP_CMN_QOS_CTRL, + cfg->danger_safe_en ? QOS_QOS_CTRL_DANGER_SAFE_EN : 0); +} + /* * note: Aside from encoders, input_sel should be set to 0x0 by default */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h index 67b08e99335dc..628befc65ce35 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h @@ -21,6 +21,13 @@ #define TO_S15D16(_x_)((_x_) << 7) +#define MDP_TICK_COUNT 16 +#define XO_CLK_RATE 19200 +#define MS_TICKS_IN_SEC 1000 + +#define CALCULATE_WD_LOAD_VALUE(fps) \ + ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) + extern const struct dpu_csc_cfg dpu_csc_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_rgb2yuv_601l; @@ -360,6 +367,9 @@ void _dpu_hw_setup_qos_lut(struct dpu_hw_blk_reg_map *c, u32 offset, bool qos_8lvl, const struct dpu_hw_qos_cfg *cfg); +void dpu_hw_setup_qos_lut_v13(struct dpu_hw_blk_reg_map *c, + const struct dpu_hw_qos_cfg *cfg); + void dpu_hw_setup_misr(struct dpu_hw_blk_reg_map *c, u32 misr_ctrl_offset, u8 input_sel); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.h index 285121ec804cc..9ac49448e4325 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.h @@ -17,7 +17,7 @@ struct dpu_hw_vbif; */ struct dpu_hw_vbif_ops { /** - * set_limit_conf - set transaction limit config + * @set_limit_conf: set transaction limit config * @vbif: vbif context driver * @xin_id: client interface identifier * @rd: true for read limit; false for write limit @@ -27,7 +27,7 @@ struct dpu_hw_vbif_ops { u32 xin_id, bool rd, u32 limit); /** - * get_limit_conf - get transaction limit config + * @get_limit_conf: get transaction limit config * @vbif: vbif context driver * @xin_id: client interface identifier * @rd: true for read limit; false for write limit @@ -37,7 +37,7 @@ struct dpu_hw_vbif_ops { u32 xin_id, bool rd); /** - * set_halt_ctrl - set halt control + * @set_halt_ctrl: set halt control * @vbif: vbif context driver * @xin_id: client interface identifier * @enable: halt control enable @@ -46,7 +46,7 @@ struct dpu_hw_vbif_ops { u32 xin_id, bool enable); /** - * get_halt_ctrl - get halt control + * @get_halt_ctrl: get halt control * @vbif: vbif context driver * @xin_id: client interface identifier * @return: halt control enable @@ -55,7 +55,7 @@ struct dpu_hw_vbif_ops { u32 xin_id); /** - * set_qos_remap - set QoS priority remap + * @set_qos_remap: set QoS priority remap * @vbif: vbif context driver * @xin_id: client interface identifier * @level: priority level @@ -65,7 +65,7 @@ struct dpu_hw_vbif_ops { u32 xin_id, u32 level, u32 remap_level); /** - * set_mem_type - set memory type + * @set_mem_type: set memory type * @vbif: vbif context driver * @xin_id: client interface identifier * @value: memory type value @@ -74,7 +74,7 @@ struct dpu_hw_vbif_ops { u32 xin_id, u32 value); /** - * clear_errors - clear any vbif errors + * @clear_errors: clear any vbif errors * This function clears any detected pending/source errors * on the VBIF interface, and optionally returns the detected * error mask(s). @@ -86,7 +86,7 @@ struct dpu_hw_vbif_ops { u32 *pnd_errors, u32 *src_errors); /** - * set_write_gather_en - set write_gather enable + * @set_write_gather_en: set write_gather enable * @vbif: vbif context driver * @xin_id: client interface identifier */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c index 478a091aeccfc..4da4bd6a997c5 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c @@ -148,6 +148,15 @@ static void dpu_hw_wb_setup_qos_lut(struct dpu_hw_wb *ctx, cfg); } +static void dpu_hw_wb_setup_qos_lut_v13(struct dpu_hw_wb *ctx, + struct dpu_hw_qos_cfg *cfg) +{ + if (!ctx || !cfg) + return; + + dpu_hw_setup_qos_lut_v13(&ctx->hw, cfg); +} + static void dpu_hw_wb_setup_cdp(struct dpu_hw_wb *ctx, const struct msm_format *fmt, bool enable) @@ -202,8 +211,12 @@ static void _setup_wb_ops(struct dpu_hw_wb_ops *ops, if (test_bit(DPU_WB_XY_ROI_OFFSET, &features)) ops->setup_roi = dpu_hw_wb_roi; - if (test_bit(DPU_WB_QOS, &features)) - ops->setup_qos_lut = dpu_hw_wb_setup_qos_lut; + if (test_bit(DPU_WB_QOS, &features)) { + if (mdss_rev->core_major_ver >= 13) + ops->setup_qos_lut = dpu_hw_wb_setup_qos_lut_v13; + else + ops->setup_qos_lut = dpu_hw_wb_setup_qos_lut; + } if (test_bit(DPU_WB_CDP, &features)) ops->setup_cdp = dpu_hw_wb_setup_cdp; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h index ee5e5ab786e1b..cfdbb5bb2a0f3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h @@ -22,11 +22,11 @@ struct dpu_hw_wb_cfg { }; /** - * * struct dpu_hw_wb_ops : Interface to the wb hw driver functions * Assumption is these functions will be called after clocks are enabled * @setup_outaddress: setup output address from the writeback job * @setup_outformat: setup output format of writeback block from writeback job + * @setup_roi: setup ROI (Region of Interest) parameters * @setup_qos_lut: setup qos LUT for writeback block based on input * @setup_cdp: setup chroma down prefetch block for writeback block * @setup_clk_force_ctrl: setup clock force control @@ -61,7 +61,7 @@ struct dpu_hw_wb_ops { * struct dpu_hw_wb : WB driver object * @hw: block hardware details * @idx: hardware index number within type - * @wb_hw_caps: hardware capabilities + * @caps: hardware capabilities * @ops: function pointers */ struct dpu_hw_wb { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index 4e5a8ecd31f75..938f50ac655b8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -52,7 +52,7 @@ #define DPU_DEBUGFS_DIR "msm_dpu" #define DPU_DEBUGFS_HWMASKNAME "hw_log_mask" -bool dpu_use_virtual_planes; +bool dpu_use_virtual_planes = true; module_param(dpu_use_virtual_planes, bool, 0); static int dpu_kms_hw_init(struct msm_kms *kms); @@ -612,6 +612,7 @@ static int _dpu_kms_initialize_dsi(struct drm_device *dev, info.h_tile_instance[info.num_of_h_tiles++] = other; info.is_cmd_mode = msm_dsi_is_cmd_mode(priv->kms->dsi[i]); + info.stream_id = 0; rc = dpu_kms_dsi_set_te_source(&info, priv->kms->dsi[i]); if (rc) { @@ -653,7 +654,7 @@ static int _dpu_kms_initialize_displayport(struct drm_device *dev, struct msm_display_info info; bool yuv_supported; int rc; - int i; + int i, stream_id, stream_cnt; for (i = 0; i < ARRAY_SIZE(priv->kms->dp); i++) { if (!priv->kms->dp[i]) @@ -676,6 +677,31 @@ static int _dpu_kms_initialize_displayport(struct drm_device *dev, DPU_ERROR("modeset_init failed for DP, rc = %d\n", rc); return rc; } + + stream_cnt = msm_dp_get_mst_max_stream(priv->kms->dp[i]); + + if (stream_cnt > 1) { + rc = msm_dp_mst_register(priv->kms->dp[i]); + if (rc) { + DPU_ERROR("dp_mst_init failed for DP, rc = %d\n", rc); + return rc; + } + + for (stream_id = 0; stream_id < stream_cnt; stream_id++) { + info.stream_id = stream_id; + encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_DPMST, &info); + if (IS_ERR(encoder)) { + DPU_ERROR("encoder init failed for dp mst display\n"); + return PTR_ERR(encoder); + } + + rc = msm_dp_mst_attach_encoder(priv->kms->dp[i], encoder); + if (rc) { + DPU_ERROR("DP MST init failed, %d\n", rc); + continue; + } + } + } } return 0; @@ -696,6 +722,7 @@ static int _dpu_kms_initialize_hdmi(struct drm_device *dev, info.num_of_h_tiles = 1; info.h_tile_instance[0] = 0; info.intf_type = INTF_HDMI; + info.stream_id = 0; encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_TMDS, &info); if (IS_ERR(encoder)) { @@ -728,6 +755,7 @@ static int _dpu_kms_initialize_writeback(struct drm_device *dev, /* use only WB idx 2 instance for DPU */ info.h_tile_instance[0] = wb_idx; info.intf_type = INTF_WB; + info.stream_id = 0; maxlinewidth = dpu_rm_get_wb(&dpu_kms->rm, info.h_tile_instance[0])->caps->maxlinewidth; @@ -1461,8 +1489,6 @@ static int __maybe_unused dpu_runtime_suspend(struct device *dev) struct msm_drm_private *priv = platform_get_drvdata(pdev); struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms); - /* Drop the performance state vote */ - dev_pm_opp_set_rate(dev, 0); clk_bulk_disable_unprepare(dpu_kms->num_clocks, dpu_kms->clocks); for (i = 0; i < dpu_kms->num_paths; i++) @@ -1505,6 +1531,8 @@ static const struct dev_pm_ops dpu_pm_ops = { }; static const struct of_device_id dpu_dt_match[] = { + { .compatible = "qcom,glymur-dpu", .data = &dpu_glymur_cfg, }, + { .compatible = "qcom,kaanapali-dpu", .data = &dpu_kaanapali_cfg, }, { .compatible = "qcom,msm8917-mdp5", .data = &dpu_msm8917_cfg, }, { .compatible = "qcom,msm8937-mdp5", .data = &dpu_msm8937_cfg, }, { .compatible = "qcom,msm8953-mdp5", .data = &dpu_msm8953_cfg, }, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c index 905524ceeb1f1..9b7a8b46bfa91 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c @@ -622,6 +622,7 @@ static void _dpu_plane_color_fill(struct dpu_plane *pdpu, struct msm_drm_private *priv = plane->dev->dev_private; struct dpu_plane_state *pstate = to_dpu_plane_state(plane->state); u32 fill_color = (color & 0xFFFFFF) | ((alpha & 0xFF) << 24); + int i; DPU_DEBUG_PLANE(pdpu, "\n"); @@ -635,12 +636,13 @@ static void _dpu_plane_color_fill(struct dpu_plane *pdpu, return; /* update sspp */ - _dpu_plane_color_fill_pipe(pstate, &pstate->pipe, &pstate->pipe_cfg.dst_rect, - fill_color, fmt); - - if (pstate->r_pipe.sspp) - _dpu_plane_color_fill_pipe(pstate, &pstate->r_pipe, &pstate->r_pipe_cfg.dst_rect, + for (i = 0; i < PIPES_PER_PLANE; i++) { + if (!pstate->pipe[i].sspp) + continue; + _dpu_plane_color_fill_pipe(pstate, &pstate->pipe[i], + &pstate->pipe_cfg[i].dst_rect, fill_color, fmt); + } } static int dpu_plane_prepare_fb(struct drm_plane *plane, @@ -822,8 +824,8 @@ static int dpu_plane_atomic_check_nosspp(struct drm_plane *plane, struct dpu_kms *kms = _dpu_plane_get_kms(&pdpu->base); u64 max_mdp_clk_rate = kms->perf.max_core_clk_rate; struct dpu_plane_state *pstate = to_dpu_plane_state(new_plane_state); - struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg; - struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg; + struct dpu_sw_pipe_cfg *pipe_cfg; + struct dpu_sw_pipe_cfg *r_pipe_cfg; struct drm_rect fb_rect = { 0 }; uint32_t max_linewidth; @@ -848,6 +850,9 @@ static int dpu_plane_atomic_check_nosspp(struct drm_plane *plane, return -EINVAL; } + /* move the assignment here, to ease handling to another pairs later */ + pipe_cfg = &pstate->pipe_cfg[0]; + r_pipe_cfg = &pstate->pipe_cfg[1]; /* state->src is 16.16, src_rect is not */ drm_rect_fp_to_int(&pipe_cfg->src_rect, &new_plane_state->src); @@ -954,6 +959,23 @@ static int dpu_plane_is_multirect_parallel_capable(struct dpu_hw_sspp *sspp, dpu_plane_is_parallel_capable(pipe_cfg, fmt, max_linewidth); } +static bool dpu_plane_get_single_pipe_in_stage(struct dpu_plane_state *pstate, + struct dpu_sw_pipe **single_pipe, + struct dpu_sw_pipe_cfg **single_pipe_cfg, + int stage_index) +{ + int pipe_idx; + + pipe_idx = stage_index * PIPES_PER_STAGE; + if (drm_rect_width(&pstate->pipe_cfg[pipe_idx].src_rect) != 0 && + drm_rect_width(&pstate->pipe_cfg[pipe_idx + 1].src_rect) == 0) { + *single_pipe = &pstate->pipe[pipe_idx]; + *single_pipe_cfg = &pstate->pipe_cfg[pipe_idx]; + return true; + } + + return false; +} static int dpu_plane_atomic_check_sspp(struct drm_plane *plane, struct drm_atomic_state *state, @@ -963,10 +985,10 @@ static int dpu_plane_atomic_check_sspp(struct drm_plane *plane, drm_atomic_get_new_plane_state(state, plane); struct dpu_plane *pdpu = to_dpu_plane(plane); struct dpu_plane_state *pstate = to_dpu_plane_state(new_plane_state); - struct dpu_sw_pipe *pipe = &pstate->pipe; - struct dpu_sw_pipe *r_pipe = &pstate->r_pipe; - struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg; - struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg; + struct dpu_sw_pipe *pipe = &pstate->pipe[0]; + struct dpu_sw_pipe *r_pipe = &pstate->pipe[1]; + struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg[0]; + struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->pipe_cfg[1]; int ret = 0; ret = dpu_plane_atomic_check_pipe(pdpu, pipe, pipe_cfg, @@ -1019,17 +1041,20 @@ static bool dpu_plane_try_multirect_parallel(struct dpu_sw_pipe *pipe, struct dp static int dpu_plane_try_multirect_shared(struct dpu_plane_state *pstate, struct dpu_plane_state *prev_adjacent_pstate, const struct msm_format *fmt, - uint32_t max_linewidth) + uint32_t max_linewidth, int stage_index) { - struct dpu_sw_pipe *pipe = &pstate->pipe; - struct dpu_sw_pipe *r_pipe = &pstate->r_pipe; - struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg; - struct dpu_sw_pipe *prev_pipe = &prev_adjacent_pstate->pipe; - struct dpu_sw_pipe_cfg *prev_pipe_cfg = &prev_adjacent_pstate->pipe_cfg; + struct dpu_sw_pipe *pipe, *prev_pipe; + struct dpu_sw_pipe_cfg *pipe_cfg, *prev_pipe_cfg; const struct msm_format *prev_fmt = msm_framebuffer_format(prev_adjacent_pstate->base.fb); u16 max_tile_height = 1; - if (prev_adjacent_pstate->r_pipe.sspp != NULL || + if (!dpu_plane_get_single_pipe_in_stage(pstate, &pipe, + &pipe_cfg, stage_index)) + return false; + + if (!dpu_plane_get_single_pipe_in_stage(prev_adjacent_pstate, + &prev_pipe, &prev_pipe_cfg, + stage_index) || prev_pipe->multirect_mode != DPU_SSPP_MULTIRECT_NONE) return false; @@ -1044,11 +1069,6 @@ static int dpu_plane_try_multirect_shared(struct dpu_plane_state *pstate, if (MSM_FORMAT_IS_UBWC(prev_fmt)) max_tile_height = max(max_tile_height, prev_fmt->tile_height); - r_pipe->multirect_index = DPU_SSPP_RECT_SOLO; - r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; - - r_pipe->sspp = NULL; - if (dpu_plane_is_parallel_capable(pipe_cfg, fmt, max_linewidth) && dpu_plane_is_parallel_capable(prev_pipe_cfg, prev_fmt, max_linewidth) && (pipe_cfg->dst_rect.x1 >= prev_pipe_cfg->dst_rect.x2 || @@ -1089,10 +1109,10 @@ static int dpu_plane_atomic_check(struct drm_plane *plane, struct dpu_plane *pdpu = to_dpu_plane(plane); struct dpu_plane_state *pstate = to_dpu_plane_state(new_plane_state); struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane); - struct dpu_sw_pipe *pipe = &pstate->pipe; - struct dpu_sw_pipe *r_pipe = &pstate->r_pipe; - struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg; - struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg; + struct dpu_sw_pipe *pipe = &pstate->pipe[0]; + struct dpu_sw_pipe *r_pipe = &pstate->pipe[1]; + struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg[0]; + struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->pipe_cfg[1]; const struct drm_crtc_state *crtc_state = NULL; uint32_t max_linewidth = dpu_kms->catalog->caps->max_linewidth; @@ -1136,7 +1156,7 @@ static int dpu_plane_virtual_atomic_check(struct drm_plane *plane, drm_atomic_get_old_plane_state(state, plane); struct dpu_plane_state *pstate = to_dpu_plane_state(plane_state); struct drm_crtc_state *crtc_state = NULL; - int ret; + int ret, i; if (IS_ERR(plane_state)) return PTR_ERR(plane_state); @@ -1154,8 +1174,8 @@ static int dpu_plane_virtual_atomic_check(struct drm_plane *plane, * resources are freed by dpu_crtc_assign_plane_resources(), * but clean them here. */ - pstate->pipe.sspp = NULL; - pstate->r_pipe.sspp = NULL; + for (i = 0; i < PIPES_PER_PLANE; i++) + pstate->pipe[i].sspp = NULL; return 0; } @@ -1177,37 +1197,72 @@ static int dpu_plane_virtual_atomic_check(struct drm_plane *plane, return 0; } +static int dpu_plane_assign_resource_in_stage(struct dpu_sw_pipe *pipe, + struct dpu_sw_pipe_cfg *pipe_cfg, + struct drm_plane_state *plane_state, + struct dpu_global_state *global_state, + struct drm_crtc *crtc, + struct dpu_rm_sspp_requirements *reqs) +{ + struct drm_plane *plane = plane_state->plane; + struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane); + struct dpu_sw_pipe *r_pipe = pipe + 1; + struct dpu_sw_pipe_cfg *r_pipe_cfg = pipe_cfg + 1; + + if (drm_rect_width(&pipe_cfg->src_rect) == 0) + return 0; + + pipe->sspp = dpu_rm_reserve_sspp(&dpu_kms->rm, global_state, crtc, reqs); + if (!pipe->sspp) + return -ENODEV; + pipe->multirect_index = DPU_SSPP_RECT_SOLO; + pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; + + if (drm_rect_width(&r_pipe_cfg->src_rect) == 0) + return 0; + + if (dpu_plane_try_multirect_parallel(pipe, pipe_cfg, r_pipe, r_pipe_cfg, + pipe->sspp, + msm_framebuffer_format(plane_state->fb), + dpu_kms->catalog->caps->max_linewidth)) + return 0; + + r_pipe->sspp = dpu_rm_reserve_sspp(&dpu_kms->rm, global_state, crtc, reqs); + if (!r_pipe->sspp) + return -ENODEV; + r_pipe->multirect_index = DPU_SSPP_RECT_SOLO; + r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; + + return 0; +} + static int dpu_plane_virtual_assign_resources(struct drm_crtc *crtc, struct dpu_global_state *global_state, struct drm_atomic_state *state, struct drm_plane_state *plane_state, - struct drm_plane_state *prev_adjacent_plane_state) + struct drm_plane_state **prev_adjacent_plane_state) { const struct drm_crtc_state *crtc_state = NULL; struct drm_plane *plane = plane_state->plane; struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane); struct dpu_rm_sspp_requirements reqs; - struct dpu_plane_state *pstate, *prev_adjacent_pstate; + struct dpu_plane_state *pstate, *prev_adjacent_pstate[STAGES_PER_PLANE]; struct dpu_sw_pipe *pipe; - struct dpu_sw_pipe *r_pipe; struct dpu_sw_pipe_cfg *pipe_cfg; - struct dpu_sw_pipe_cfg *r_pipe_cfg; const struct msm_format *fmt; + int i, ret; if (plane_state->crtc) crtc_state = drm_atomic_get_new_crtc_state(state, plane_state->crtc); pstate = to_dpu_plane_state(plane_state); - prev_adjacent_pstate = prev_adjacent_plane_state ? - to_dpu_plane_state(prev_adjacent_plane_state) : NULL; - pipe = &pstate->pipe; - r_pipe = &pstate->r_pipe; - pipe_cfg = &pstate->pipe_cfg; - r_pipe_cfg = &pstate->r_pipe_cfg; - - pipe->sspp = NULL; - r_pipe->sspp = NULL; + for (i = 0; i < STAGES_PER_PLANE; i++) + prev_adjacent_pstate[i] = prev_adjacent_plane_state[i] ? + to_dpu_plane_state(prev_adjacent_plane_state[i]) : NULL; + + for (i = 0; i < PIPES_PER_PLANE; i++) + pstate->pipe[i].sspp = NULL; if (!plane_state->fb) return -EINVAL; @@ -1219,42 +1274,24 @@ static int dpu_plane_virtual_assign_resources(struct drm_crtc *crtc, reqs.rot90 = drm_rotation_90_or_270(plane_state->rotation); - if (drm_rect_width(&r_pipe_cfg->src_rect) == 0) { - if (!prev_adjacent_pstate || - !dpu_plane_try_multirect_shared(pstate, prev_adjacent_pstate, fmt, - dpu_kms->catalog->caps->max_linewidth)) { - pipe->sspp = dpu_rm_reserve_sspp(&dpu_kms->rm, global_state, crtc, &reqs); - if (!pipe->sspp) - return -ENODEV; - - r_pipe->sspp = NULL; + for (i = 0; i < STAGES_PER_PLANE; i++) { + if (prev_adjacent_pstate[i] && + dpu_plane_try_multirect_shared(pstate, prev_adjacent_pstate[i], fmt, + dpu_kms->catalog->caps->max_linewidth, + i)) + continue; - pipe->multirect_index = DPU_SSPP_RECT_SOLO; - pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; + if (dpu_plane_get_single_pipe_in_stage(pstate, &pipe, &pipe_cfg, i)) + prev_adjacent_plane_state[i] = plane_state; - r_pipe->multirect_index = DPU_SSPP_RECT_SOLO; - r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; - } - } else { - pipe->sspp = dpu_rm_reserve_sspp(&dpu_kms->rm, global_state, crtc, &reqs); - if (!pipe->sspp) - return -ENODEV; - - if (!dpu_plane_try_multirect_parallel(pipe, pipe_cfg, r_pipe, r_pipe_cfg, - pipe->sspp, - msm_framebuffer_format(plane_state->fb), - dpu_kms->catalog->caps->max_linewidth)) { - /* multirect is not possible, use two SSPP blocks */ - r_pipe->sspp = dpu_rm_reserve_sspp(&dpu_kms->rm, global_state, crtc, &reqs); - if (!r_pipe->sspp) - return -ENODEV; - - pipe->multirect_index = DPU_SSPP_RECT_SOLO; - pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; - - r_pipe->multirect_index = DPU_SSPP_RECT_SOLO; - r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; - } + pipe = &pstate->pipe[i * PIPES_PER_STAGE]; + pipe_cfg = &pstate->pipe_cfg[i * PIPES_PER_STAGE]; + ret = dpu_plane_assign_resource_in_stage(pipe, pipe_cfg, + plane_state, + global_state, + crtc, &reqs); + if (ret) + return ret; } return dpu_plane_atomic_check_sspp(plane, state, crtc_state); @@ -1267,7 +1304,7 @@ int dpu_assign_plane_resources(struct dpu_global_state *global_state, unsigned int num_planes) { unsigned int i; - struct drm_plane_state *prev_adjacent_plane_state = NULL; + struct drm_plane_state *prev_adjacent_plane_state[STAGES_PER_PLANE] = { NULL }; for (i = 0; i < num_planes; i++) { struct drm_plane_state *plane_state = states[i]; @@ -1281,8 +1318,6 @@ int dpu_assign_plane_resources(struct dpu_global_state *global_state, prev_adjacent_plane_state); if (ret) return ret; - - prev_adjacent_plane_state = plane_state; } return 0; @@ -1318,6 +1353,7 @@ void dpu_plane_flush(struct drm_plane *plane) { struct dpu_plane *pdpu; struct dpu_plane_state *pstate; + int i; if (!plane || !plane->state) { DPU_ERROR("invalid plane\n"); @@ -1338,8 +1374,8 @@ void dpu_plane_flush(struct drm_plane *plane) /* force 100% alpha */ _dpu_plane_color_fill(pdpu, pdpu->color_fill, 0xFF); else { - dpu_plane_flush_csc(pdpu, &pstate->pipe); - dpu_plane_flush_csc(pdpu, &pstate->r_pipe); + for (i = 0; i < PIPES_PER_PLANE; i++) + dpu_plane_flush_csc(pdpu, &pstate->pipe[i]); } /* flag h/w flush complete */ @@ -1440,15 +1476,12 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane, struct dpu_plane *pdpu = to_dpu_plane(plane); struct drm_plane_state *state = plane->state; struct dpu_plane_state *pstate = to_dpu_plane_state(state); - struct dpu_sw_pipe *pipe = &pstate->pipe; - struct dpu_sw_pipe *r_pipe = &pstate->r_pipe; struct drm_crtc *crtc = state->crtc; struct drm_framebuffer *fb = state->fb; bool is_rt_pipe; const struct msm_format *fmt = msm_framebuffer_format(fb); - struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg; - struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg; + int i; pstate->pending = true; @@ -1463,12 +1496,11 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane, crtc->base.id, DRM_RECT_ARG(&state->dst), &fmt->pixel_format, MSM_FORMAT_IS_UBWC(fmt)); - dpu_plane_sspp_update_pipe(plane, pipe, pipe_cfg, fmt, - drm_mode_vrefresh(&crtc->mode), - &pstate->layout); - - if (r_pipe->sspp) { - dpu_plane_sspp_update_pipe(plane, r_pipe, r_pipe_cfg, fmt, + for (i = 0; i < PIPES_PER_PLANE; i++) { + if (!drm_rect_width(&pstate->pipe_cfg[i].src_rect)) + continue; + dpu_plane_sspp_update_pipe(plane, &pstate->pipe[i], + &pstate->pipe_cfg[i], fmt, drm_mode_vrefresh(&crtc->mode), &pstate->layout); } @@ -1476,15 +1508,17 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane, if (pstate->needs_qos_remap) pstate->needs_qos_remap = false; - pstate->plane_fetch_bw = _dpu_plane_calc_bw(pdpu->catalog, fmt, - &crtc->mode, pipe_cfg); - - pstate->plane_clk = _dpu_plane_calc_clk(&crtc->mode, pipe_cfg); - - if (r_pipe->sspp) { - pstate->plane_fetch_bw += _dpu_plane_calc_bw(pdpu->catalog, fmt, &crtc->mode, r_pipe_cfg); + pstate->plane_fetch_bw = 0; + pstate->plane_clk = 0; + for (i = 0; i < PIPES_PER_PLANE; i++) { + if (!drm_rect_width(&pstate->pipe_cfg[i].src_rect)) + continue; + pstate->plane_fetch_bw += _dpu_plane_calc_bw(pdpu->catalog, fmt, + &crtc->mode, &pstate->pipe_cfg[i]); - pstate->plane_clk = max(pstate->plane_clk, _dpu_plane_calc_clk(&crtc->mode, r_pipe_cfg)); + pstate->plane_clk = max(pstate->plane_clk, + _dpu_plane_calc_clk(&crtc->mode, + &pstate->pipe_cfg[i])); } } @@ -1492,17 +1526,28 @@ static void _dpu_plane_atomic_disable(struct drm_plane *plane) { struct drm_plane_state *state = plane->state; struct dpu_plane_state *pstate = to_dpu_plane_state(state); - struct dpu_sw_pipe *r_pipe = &pstate->r_pipe; + struct dpu_sw_pipe *pipe; + int i; - trace_dpu_plane_disable(DRMID(plane), false, - pstate->pipe.multirect_mode); + for (i = 0; i < PIPES_PER_PLANE; i += 1) { + pipe = &pstate->pipe[i]; + if (!pipe->sspp) + continue; - if (r_pipe->sspp) { - r_pipe->multirect_index = DPU_SSPP_RECT_SOLO; - r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; + trace_dpu_plane_disable(DRMID(plane), false, + pstate->pipe[i].multirect_mode); - if (r_pipe->sspp->ops.setup_multirect) - r_pipe->sspp->ops.setup_multirect(r_pipe); + if (i % PIPES_PER_STAGE == 0) + continue; + + /* + * clear multirect for the right pipe so that the SSPP + * can be further reused in the solo mode + */ + pipe->multirect_index = DPU_SSPP_RECT_SOLO; + pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE; + if (pipe->sspp->ops.setup_multirect) + pipe->sspp->ops.setup_multirect(pipe); } pstate->pending = true; @@ -1597,31 +1642,26 @@ static void dpu_plane_atomic_print_state(struct drm_printer *p, const struct drm_plane_state *state) { const struct dpu_plane_state *pstate = to_dpu_plane_state(state); - const struct dpu_sw_pipe *pipe = &pstate->pipe; - const struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg; - const struct dpu_sw_pipe *r_pipe = &pstate->r_pipe; - const struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg; + const struct dpu_sw_pipe *pipe; + const struct dpu_sw_pipe_cfg *pipe_cfg; + int i; drm_printf(p, "\tstage=%d\n", pstate->stage); - if (pipe->sspp) { - drm_printf(p, "\tsspp[0]=%s\n", pipe->sspp->cap->name); - drm_printf(p, "\tmultirect_mode[0]=%s\n", + for (i = 0; i < PIPES_PER_PLANE; i++) { + pipe = &pstate->pipe[i]; + if (!pipe->sspp) + continue; + pipe_cfg = &pstate->pipe_cfg[i]; + drm_printf(p, "\tsspp[%d]=%s\n", i, pipe->sspp->cap->name); + drm_printf(p, "\tmultirect_mode[%d]=%s\n", i, dpu_get_multirect_mode(pipe->multirect_mode)); - drm_printf(p, "\tmultirect_index[0]=%s\n", + drm_printf(p, "\tmultirect_index[%d]=%s\n", i, dpu_get_multirect_index(pipe->multirect_index)); - drm_printf(p, "\tsrc[0]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&pipe_cfg->src_rect)); - drm_printf(p, "\tdst[0]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&pipe_cfg->dst_rect)); - } - - if (r_pipe->sspp) { - drm_printf(p, "\tsspp[1]=%s\n", r_pipe->sspp->cap->name); - drm_printf(p, "\tmultirect_mode[1]=%s\n", - dpu_get_multirect_mode(r_pipe->multirect_mode)); - drm_printf(p, "\tmultirect_index[1]=%s\n", - dpu_get_multirect_index(r_pipe->multirect_index)); - drm_printf(p, "\tsrc[1]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&r_pipe_cfg->src_rect)); - drm_printf(p, "\tdst[1]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&r_pipe_cfg->dst_rect)); + drm_printf(p, "\tsrc[%d]=" DRM_RECT_FMT "\n", i, + DRM_RECT_ARG(&pipe_cfg->src_rect)); + drm_printf(p, "\tdst[%d]=" DRM_RECT_FMT "\n", i, + DRM_RECT_ARG(&pipe_cfg->dst_rect)); } } @@ -1659,14 +1699,17 @@ void dpu_plane_danger_signal_ctrl(struct drm_plane *plane, bool enable) struct dpu_plane *pdpu = to_dpu_plane(plane); struct dpu_plane_state *pstate = to_dpu_plane_state(plane->state); struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane); + int i; if (!pdpu->is_rt_pipe) return; pm_runtime_get_sync(&dpu_kms->pdev->dev); - _dpu_plane_set_qos_ctrl(plane, &pstate->pipe, enable); - if (pstate->r_pipe.sspp) - _dpu_plane_set_qos_ctrl(plane, &pstate->r_pipe, enable); + for (i = 0; i < PIPES_PER_PLANE; i++) { + if (!pstate->pipe[i].sspp) + continue; + _dpu_plane_set_qos_ctrl(plane, &pstate->pipe[i], enable); + } pm_runtime_put_sync(&dpu_kms->pdev->dev); } #endif diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h index a3a6e90283330..1ef5a041b8aca 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h @@ -17,10 +17,8 @@ /** * struct dpu_plane_state: Define dpu extension of drm plane state object * @base: base drm plane state object - * @pipe: software pipe description - * @r_pipe: software pipe description of the second pipe - * @pipe_cfg: software pipe configuration - * @r_pipe_cfg: software pipe configuration for the second pipe + * @pipe: software pipe description array + * @pipe_cfg: software pipe configuration array * @stage: assigned by crtc blender * @needs_qos_remap: qos remap settings need to be updated * @multirect_index: index of the rectangle of SSPP @@ -33,10 +31,8 @@ */ struct dpu_plane_state { struct drm_plane_state base; - struct dpu_sw_pipe pipe; - struct dpu_sw_pipe r_pipe; - struct dpu_sw_pipe_cfg pipe_cfg; - struct dpu_sw_pipe_cfg r_pipe_cfg; + struct dpu_sw_pipe pipe[PIPES_PER_PLANE]; + struct dpu_sw_pipe_cfg pipe_cfg[PIPES_PER_PLANE]; enum dpu_stage stage; bool needs_qos_remap; bool pending; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c index d9c3b0a1d0914..7e77d88f89592 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c @@ -315,20 +315,19 @@ static bool _dpu_rm_check_lm_and_get_connected_blks(struct dpu_rm *rm, /* Already reserved? */ if (reserved_by_other(global_state->mixer_to_crtc_id, lm_idx, crtc_id)) { - DPU_DEBUG("lm %d already reserved\n", lm_idx + LM_0); + DPU_DEBUG("LM_%d already reserved\n", lm_idx); return false; } lm_cfg = to_dpu_hw_mixer(rm->mixer_blks[lm_idx])->cap; idx = lm_cfg->pingpong - PINGPONG_0; - if (idx < 0 || idx >= ARRAY_SIZE(rm->pingpong_blks)) { - DPU_ERROR("failed to get pp on lm %d\n", lm_cfg->pingpong); + if (idx < 0 || idx >= ARRAY_SIZE(rm->pingpong_blks) || !rm->pingpong_blks[idx]) { + DPU_ERROR("LM_%d, invalid PP_%d\n", lm_idx, idx); return false; } if (reserved_by_other(global_state->pingpong_to_crtc_id, idx, crtc_id)) { - DPU_DEBUG("lm %d pp %d already reserved\n", lm_cfg->id, - lm_cfg->pingpong); + DPU_DEBUG("LM_%d PP_%d already reserved\n", lm_idx, idx); return false; } *pp_idx = idx; @@ -337,14 +336,13 @@ static bool _dpu_rm_check_lm_and_get_connected_blks(struct dpu_rm *rm, return true; idx = lm_cfg->dspp - DSPP_0; - if (idx < 0 || idx >= ARRAY_SIZE(rm->dspp_blks)) { - DPU_ERROR("failed to get dspp on lm %d\n", lm_cfg->dspp); + if (idx < 0 || idx >= ARRAY_SIZE(rm->dspp_blks) || !rm->dspp_blks[idx]) { + DPU_ERROR("LM_%d, invalid DSPP_%d\n", lm_idx, idx); return false; } if (reserved_by_other(global_state->dspp_to_crtc_id, idx, crtc_id)) { - DPU_DEBUG("lm %d dspp %d already reserved\n", lm_cfg->id, - lm_cfg->dspp); + DPU_DEBUG("LM_%d DSPP_%d already reserved\n", lm_idx, idx); return false; } *dspp_idx = idx; @@ -364,7 +362,7 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm, int i, lm_count = 0; if (!topology->num_lm) { - DPU_ERROR("invalid number of lm: %d\n", topology->num_lm); + DPU_ERROR("zero LMs in topology\n"); return -EINVAL; } @@ -374,7 +372,11 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm, if (!rm->mixer_blks[i]) continue; - lm_count = 0; + /* + * Reset lm_count to an even index. This will drop the previous + * primary mixer if failed to find its peer. + */ + lm_count &= ~1; lm_idx[lm_count] = i; if (!_dpu_rm_check_lm_and_get_connected_blks(rm, global_state, @@ -464,13 +466,13 @@ static int _dpu_rm_reserve_ctls( features = ctl->caps->features; has_split_display = BIT(DPU_CTL_SPLIT_DISPLAY) & features; - DPU_DEBUG("ctl %d caps 0x%lX\n", j + CTL_0, features); + DPU_DEBUG("CTL_%d caps 0x%lX\n", j, features); if (needs_split_display != has_split_display) continue; ctl_idx[i] = j; - DPU_DEBUG("ctl %d match\n", j + CTL_0); + DPU_DEBUG("CTL_%d match\n", j); if (++i == num_ctls) break; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h index 5307cbc2007c5..cb24ad2a6d8d3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h @@ -651,9 +651,9 @@ TRACE_EVENT(dpu_crtc_setup_mixer, TP_PROTO(uint32_t crtc_id, uint32_t plane_id, struct drm_plane_state *state, struct dpu_plane_state *pstate, uint32_t stage_idx, uint32_t pixel_format, - uint64_t modifier), + struct dpu_sw_pipe *pipe, uint64_t modifier), TP_ARGS(crtc_id, plane_id, state, pstate, stage_idx, - pixel_format, modifier), + pixel_format, pipe, modifier), TP_STRUCT__entry( __field( uint32_t, crtc_id ) __field( uint32_t, plane_id ) @@ -676,9 +676,9 @@ TRACE_EVENT(dpu_crtc_setup_mixer, __entry->dst_rect = drm_plane_state_dest(state); __entry->stage_idx = stage_idx; __entry->stage = pstate->stage; - __entry->sspp = pstate->pipe.sspp->idx; - __entry->multirect_idx = pstate->pipe.multirect_index; - __entry->multirect_mode = pstate->pipe.multirect_mode; + __entry->sspp = pipe->sspp->idx; + __entry->multirect_idx = pipe->multirect_index; + __entry->multirect_mode = pipe->multirect_mode; __entry->pixel_format = pixel_format; __entry->modifier = modifier; ), diff --git a/drivers/gpu/drm/msm/disp/dpu1/msm_media_info.h b/drivers/gpu/drm/msm/disp/dpu1/msm_media_info.h deleted file mode 100644 index 9fc9dbde8a27c..0000000000000 --- a/drivers/gpu/drm/msm/disp/dpu1/msm_media_info.h +++ /dev/null @@ -1,1155 +0,0 @@ -#ifndef __MEDIA_INFO_H__ -#define __MEDIA_INFO_H__ - -#ifndef MSM_MEDIA_ALIGN -#define MSM_MEDIA_ALIGN(__sz, __align) (((__align) & ((__align) - 1)) ?\ - ((((__sz) + (__align) - 1) / (__align)) * (__align)) :\ - (((__sz) + (__align) - 1) & (~((__align) - 1)))) -#endif - -#ifndef MSM_MEDIA_ROUNDUP -#define MSM_MEDIA_ROUNDUP(__sz, __r) (((__sz) + ((__r) - 1)) / (__r)) -#endif - -#ifndef MSM_MEDIA_MAX -#define MSM_MEDIA_MAX(__a, __b) ((__a) > (__b)?(__a):(__b)) -#endif - -enum color_fmts { - /* Venus NV12: - * YUV 4:2:0 image with a plane of 8 bit Y samples followed - * by an interleaved U/V plane containing 8 bit 2x2 subsampled - * colour difference samples. - * - * <-------- Y/UV_Stride --------> - * <------- Width -------> - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . ^ ^ - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . Height | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | Y_Scanlines - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * U V U V U V U V U V U V . . . . ^ - * U V U V U V U V U V U V . . . . | - * U V U V U V U V U V U V . . . . | - * U V U V U V U V U V U V . . . . UV_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . --> Buffer size alignment - * - * Y_Stride : Width aligned to 128 - * UV_Stride : Width aligned to 128 - * Y_Scanlines: Height aligned to 32 - * UV_Scanlines: Height/2 aligned to 16 - * Extradata: Arbitrary (software-imposed) padding - * Total size = align((Y_Stride * Y_Scanlines - * + UV_Stride * UV_Scanlines - * + max(Extradata, Y_Stride * 8), 4096) - */ - COLOR_FMT_NV12, - - /* Venus NV21: - * YUV 4:2:0 image with a plane of 8 bit Y samples followed - * by an interleaved V/U plane containing 8 bit 2x2 subsampled - * colour difference samples. - * - * <-------- Y/UV_Stride --------> - * <------- Width -------> - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . ^ ^ - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . Height | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | Y_Scanlines - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * V U V U V U V U V U V U . . . . ^ - * V U V U V U V U V U V U . . . . | - * V U V U V U V U V U V U . . . . | - * V U V U V U V U V U V U . . . . UV_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . --> Padding & Buffer size alignment - * - * Y_Stride : Width aligned to 128 - * UV_Stride : Width aligned to 128 - * Y_Scanlines: Height aligned to 32 - * UV_Scanlines: Height/2 aligned to 16 - * Extradata: Arbitrary (software-imposed) padding - * Total size = align((Y_Stride * Y_Scanlines - * + UV_Stride * UV_Scanlines - * + max(Extradata, Y_Stride * 8), 4096) - */ - COLOR_FMT_NV21, - /* Venus NV12_MVTB: - * Two YUV 4:2:0 images/views one after the other - * in a top-bottom layout, same as NV12 - * with a plane of 8 bit Y samples followed - * by an interleaved U/V plane containing 8 bit 2x2 subsampled - * colour difference samples. - * - * - * <-------- Y/UV_Stride --------> - * <------- Width -------> - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . ^ ^ ^ - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . Height | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | Y_Scanlines | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . V | | - * . . . . . . . . . . . . . . . . | View_1 - * . . . . . . . . . . . . . . . . | | - * . . . . . . . . . . . . . . . . | | - * . . . . . . . . . . . . . . . . V | - * U V U V U V U V U V U V . . . . ^ | - * U V U V U V U V U V U V . . . . | | - * U V U V U V U V U V U V . . . . | | - * U V U V U V U V U V U V . . . . UV_Scanlines | - * . . . . . . . . . . . . . . . . | | - * . . . . . . . . . . . . . . . . V V - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . ^ ^ ^ - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . Height | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | Y_Scanlines | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . V | | - * . . . . . . . . . . . . . . . . | View_2 - * . . . . . . . . . . . . . . . . | | - * . . . . . . . . . . . . . . . . | | - * . . . . . . . . . . . . . . . . V | - * U V U V U V U V U V U V . . . . ^ | - * U V U V U V U V U V U V . . . . | | - * U V U V U V U V U V U V . . . . | | - * U V U V U V U V U V U V . . . . UV_Scanlines | - * . . . . . . . . . . . . . . . . | | - * . . . . . . . . . . . . . . . . V V - * . . . . . . . . . . . . . . . . --> Buffer size alignment - * - * Y_Stride : Width aligned to 128 - * UV_Stride : Width aligned to 128 - * Y_Scanlines: Height aligned to 32 - * UV_Scanlines: Height/2 aligned to 16 - * View_1 begin at: 0 (zero) - * View_2 begin at: Y_Stride * Y_Scanlines + UV_Stride * UV_Scanlines - * Extradata: Arbitrary (software-imposed) padding - * Total size = align((2*(Y_Stride * Y_Scanlines) - * + 2*(UV_Stride * UV_Scanlines) + Extradata), 4096) - */ - COLOR_FMT_NV12_MVTB, - /* - * The buffer can be of 2 types: - * (1) Venus NV12 UBWC Progressive - * (2) Venus NV12 UBWC Interlaced - * - * (1) Venus NV12 UBWC Progressive Buffer Format: - * Compressed Macro-tile format for NV12. - * Contains 4 planes in the following order - - * (A) Y_Meta_Plane - * (B) Y_UBWC_Plane - * (C) UV_Meta_Plane - * (D) UV_UBWC_Plane - * - * Y_Meta_Plane consists of meta information to decode compressed - * tile data in Y_UBWC_Plane. - * Y_UBWC_Plane consists of Y data in compressed macro-tile format. - * UBWC decoder block will use the Y_Meta_Plane data together with - * Y_UBWC_Plane data to produce loss-less uncompressed 8 bit Y samples. - * - * UV_Meta_Plane consists of meta information to decode compressed - * tile data in UV_UBWC_Plane. - * UV_UBWC_Plane consists of UV data in compressed macro-tile format. - * UBWC decoder block will use UV_Meta_Plane data together with - * UV_UBWC_Plane data to produce loss-less uncompressed 8 bit 2x2 - * subsampled color difference samples. - * - * Each tile in Y_UBWC_Plane/UV_UBWC_Plane is independently decodable - * and randomly accessible. There is no dependency between tiles. - * - * <----- Y_Meta_Stride ----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Height | - * M M M M M M M M M M M M . . | Meta_Y_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <--Compressed tile Y Stride---> - * <------- Width -------> - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . ^ ^ - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . Height | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | Macro_tile_Y_Scanlines - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * <----- UV_Meta_Stride ----> - * M M M M M M M M M M M M . . ^ - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . M_UV_Scanlines - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * <--Compressed tile UV Stride---> - * U* V* U* V* U* V* U* V* . . . . ^ - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . UV_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * - * Y_Stride = align(Width, 128) - * UV_Stride = align(Width, 128) - * Y_Scanlines = align(Height, 32) - * UV_Scanlines = align(Height/2, 16) - * Y_UBWC_Plane_size = align(Y_Stride * Y_Scanlines, 4096) - * UV_UBWC_Plane_size = align(UV_Stride * UV_Scanlines, 4096) - * Y_Meta_Stride = align(roundup(Width, Y_TileWidth), 64) - * Y_Meta_Scanlines = align(roundup(Height, Y_TileHeight), 16) - * Y_Meta_Plane_size = align(Y_Meta_Stride * Y_Meta_Scanlines, 4096) - * UV_Meta_Stride = align(roundup(Width, UV_TileWidth), 64) - * UV_Meta_Scanlines = align(roundup(Height, UV_TileHeight), 16) - * UV_Meta_Plane_size = align(UV_Meta_Stride * UV_Meta_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align( Y_UBWC_Plane_size + UV_UBWC_Plane_size + - * Y_Meta_Plane_size + UV_Meta_Plane_size - * + max(Extradata, Y_Stride * 48), 4096) - * - * - * (2) Venus NV12 UBWC Interlaced Buffer Format: - * Compressed Macro-tile format for NV12 interlaced. - * Contains 8 planes in the following order - - * (A) Y_Meta_Top_Field_Plane - * (B) Y_UBWC_Top_Field_Plane - * (C) UV_Meta_Top_Field_Plane - * (D) UV_UBWC_Top_Field_Plane - * (E) Y_Meta_Bottom_Field_Plane - * (F) Y_UBWC_Bottom_Field_Plane - * (G) UV_Meta_Bottom_Field_Plane - * (H) UV_UBWC_Bottom_Field_Plane - * Y_Meta_Top_Field_Plane consists of meta information to decode - * compressed tile data for Y_UBWC_Top_Field_Plane. - * Y_UBWC_Top_Field_Plane consists of Y data in compressed macro-tile - * format for top field of an interlaced frame. - * UBWC decoder block will use the Y_Meta_Top_Field_Plane data together - * with Y_UBWC_Top_Field_Plane data to produce loss-less uncompressed - * 8 bit Y samples for top field of an interlaced frame. - * - * UV_Meta_Top_Field_Plane consists of meta information to decode - * compressed tile data in UV_UBWC_Top_Field_Plane. - * UV_UBWC_Top_Field_Plane consists of UV data in compressed macro-tile - * format for top field of an interlaced frame. - * UBWC decoder block will use UV_Meta_Top_Field_Plane data together - * with UV_UBWC_Top_Field_Plane data to produce loss-less uncompressed - * 8 bit subsampled color difference samples for top field of an - * interlaced frame. - * - * Each tile in Y_UBWC_Top_Field_Plane/UV_UBWC_Top_Field_Plane is - * independently decodable and randomly accessible. There is no - * dependency between tiles. - * - * Y_Meta_Bottom_Field_Plane consists of meta information to decode - * compressed tile data for Y_UBWC_Bottom_Field_Plane. - * Y_UBWC_Bottom_Field_Plane consists of Y data in compressed macro-tile - * format for bottom field of an interlaced frame. - * UBWC decoder block will use the Y_Meta_Bottom_Field_Plane data - * together with Y_UBWC_Bottom_Field_Plane data to produce loss-less - * uncompressed 8 bit Y samples for bottom field of an interlaced frame. - * - * UV_Meta_Bottom_Field_Plane consists of meta information to decode - * compressed tile data in UV_UBWC_Bottom_Field_Plane. - * UV_UBWC_Bottom_Field_Plane consists of UV data in compressed - * macro-tile format for bottom field of an interlaced frame. - * UBWC decoder block will use UV_Meta_Bottom_Field_Plane data together - * with UV_UBWC_Bottom_Field_Plane data to produce loss-less - * uncompressed 8 bit subsampled color difference samples for bottom - * field of an interlaced frame. - * - * Each tile in Y_UBWC_Bottom_Field_Plane/UV_UBWC_Bottom_Field_Plane is - * independently decodable and randomly accessible. There is no - * dependency between tiles. - * - * <-----Y_TF_Meta_Stride----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Half_height | - * M M M M M M M M M M M M . . | Meta_Y_TF_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <-Compressed tile Y_TF Stride-> - * <------- Width -------> - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . ^ ^ - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . Half_height | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | Macro_tile_Y_TF_Scanlines - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * <----UV_TF_Meta_Stride----> - * M M M M M M M M M M M M . . ^ - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . M_UV_TF_Scanlines - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * <-Compressed tile UV_TF Stride-> - * U* V* U* V* U* V* U* V* . . . . ^ - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . UV_TF_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * <-----Y_BF_Meta_Stride----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Half_height | - * M M M M M M M M M M M M . . | Meta_Y_BF_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <-Compressed tile Y_BF Stride-> - * <------- Width -------> - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . ^ ^ - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . Half_height | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | Macro_tile_Y_BF_Scanlines - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * <----UV_BF_Meta_Stride----> - * M M M M M M M M M M M M . . ^ - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . M_UV_BF_Scanlines - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * <-Compressed tile UV_BF Stride-> - * U* V* U* V* U* V* U* V* . . . . ^ - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . UV_BF_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * - * Half_height = (Height+1)>>1 - * Y_TF_Stride = align(Width, 128) - * UV_TF_Stride = align(Width, 128) - * Y_TF_Scanlines = align(Half_height, 32) - * UV_TF_Scanlines = align((Half_height+1)/2, 32) - * Y_UBWC_TF_Plane_size = align(Y_TF_Stride * Y_TF_Scanlines, 4096) - * UV_UBWC_TF_Plane_size = align(UV_TF_Stride * UV_TF_Scanlines, 4096) - * Y_TF_Meta_Stride = align(roundup(Width, Y_TileWidth), 64) - * Y_TF_Meta_Scanlines = align(roundup(Half_height, Y_TileHeight), 16) - * Y_TF_Meta_Plane_size = - * align(Y_TF_Meta_Stride * Y_TF_Meta_Scanlines, 4096) - * UV_TF_Meta_Stride = align(roundup(Width, UV_TileWidth), 64) - * UV_TF_Meta_Scanlines = align(roundup(Half_height, UV_TileHeight), 16) - * UV_TF_Meta_Plane_size = - * align(UV_TF_Meta_Stride * UV_TF_Meta_Scanlines, 4096) - * Y_BF_Stride = align(Width, 128) - * UV_BF_Stride = align(Width, 128) - * Y_BF_Scanlines = align(Half_height, 32) - * UV_BF_Scanlines = align((Half_height+1)/2, 32) - * Y_UBWC_BF_Plane_size = align(Y_BF_Stride * Y_BF_Scanlines, 4096) - * UV_UBWC_BF_Plane_size = align(UV_BF_Stride * UV_BF_Scanlines, 4096) - * Y_BF_Meta_Stride = align(roundup(Width, Y_TileWidth), 64) - * Y_BF_Meta_Scanlines = align(roundup(Half_height, Y_TileHeight), 16) - * Y_BF_Meta_Plane_size = - * align(Y_BF_Meta_Stride * Y_BF_Meta_Scanlines, 4096) - * UV_BF_Meta_Stride = align(roundup(Width, UV_TileWidth), 64) - * UV_BF_Meta_Scanlines = align(roundup(Half_height, UV_TileHeight), 16) - * UV_BF_Meta_Plane_size = - * align(UV_BF_Meta_Stride * UV_BF_Meta_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align( Y_UBWC_TF_Plane_size + UV_UBWC_TF_Plane_size + - * Y_TF_Meta_Plane_size + UV_TF_Meta_Plane_size + - * Y_UBWC_BF_Plane_size + UV_UBWC_BF_Plane_size + - * Y_BF_Meta_Plane_size + UV_BF_Meta_Plane_size + - * + max(Extradata, Y_TF_Stride * 48), 4096) - */ - COLOR_FMT_NV12_UBWC, - /* Venus NV12 10-bit UBWC: - * Compressed Macro-tile format for NV12. - * Contains 4 planes in the following order - - * (A) Y_Meta_Plane - * (B) Y_UBWC_Plane - * (C) UV_Meta_Plane - * (D) UV_UBWC_Plane - * - * Y_Meta_Plane consists of meta information to decode compressed - * tile data in Y_UBWC_Plane. - * Y_UBWC_Plane consists of Y data in compressed macro-tile format. - * UBWC decoder block will use the Y_Meta_Plane data together with - * Y_UBWC_Plane data to produce loss-less uncompressed 10 bit Y samples. - * - * UV_Meta_Plane consists of meta information to decode compressed - * tile data in UV_UBWC_Plane. - * UV_UBWC_Plane consists of UV data in compressed macro-tile format. - * UBWC decoder block will use UV_Meta_Plane data together with - * UV_UBWC_Plane data to produce loss-less uncompressed 10 bit 2x2 - * subsampled color difference samples. - * - * Each tile in Y_UBWC_Plane/UV_UBWC_Plane is independently decodable - * and randomly accessible. There is no dependency between tiles. - * - * <----- Y_Meta_Stride -----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Height | - * M M M M M M M M M M M M . . | Meta_Y_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <--Compressed tile Y Stride---> - * <------- Width -------> - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . ^ ^ - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . Height | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | Macro_tile_Y_Scanlines - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * <----- UV_Meta_Stride ----> - * M M M M M M M M M M M M . . ^ - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . M_UV_Scanlines - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * <--Compressed tile UV Stride---> - * U* V* U* V* U* V* U* V* . . . . ^ - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . UV_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * - * - * Y_Stride = align(Width * 4/3, 128) - * UV_Stride = align(Width * 4/3, 128) - * Y_Scanlines = align(Height, 32) - * UV_Scanlines = align(Height/2, 16) - * Y_UBWC_Plane_Size = align(Y_Stride * Y_Scanlines, 4096) - * UV_UBWC_Plane_Size = align(UV_Stride * UV_Scanlines, 4096) - * Y_Meta_Stride = align(roundup(Width, Y_TileWidth), 64) - * Y_Meta_Scanlines = align(roundup(Height, Y_TileHeight), 16) - * Y_Meta_Plane_size = align(Y_Meta_Stride * Y_Meta_Scanlines, 4096) - * UV_Meta_Stride = align(roundup(Width, UV_TileWidth), 64) - * UV_Meta_Scanlines = align(roundup(Height, UV_TileHeight), 16) - * UV_Meta_Plane_size = align(UV_Meta_Stride * UV_Meta_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align(Y_UBWC_Plane_size + UV_UBWC_Plane_size + - * Y_Meta_Plane_size + UV_Meta_Plane_size - * + max(Extradata, Y_Stride * 48), 4096) - */ - COLOR_FMT_NV12_BPP10_UBWC, - /* Venus RGBA8888 format: - * Contains 1 plane in the following order - - * (A) RGBA plane - * - * <-------- RGB_Stride --------> - * <------- Width -------> - * R R R R R R R R R R R R . . . . ^ ^ - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . Height | - * R R R R R R R R R R R R . . . . | RGB_Scanlines - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * - * RGB_Stride = align(Width * 4, 128) - * RGB_Scanlines = align(Height, 32) - * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align(RGB_Plane_size + Extradata, 4096) - */ - COLOR_FMT_RGBA8888, - /* Venus RGBA8888 UBWC format: - * Contains 2 planes in the following order - - * (A) Meta plane - * (B) RGBA plane - * - * <--- RGB_Meta_Stride ----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Height | - * M M M M M M M M M M M M . . | Meta_RGB_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <-------- RGB_Stride --------> - * <------- Width -------> - * R R R R R R R R R R R R . . . . ^ ^ - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . Height | - * R R R R R R R R R R R R . . . . | RGB_Scanlines - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * - * RGB_Stride = align(Width * 4, 128) - * RGB_Scanlines = align(Height, 32) - * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096) - * RGB_Meta_Stride = align(roundup(Width, RGB_TileWidth), 64) - * RGB_Meta_Scanline = align(roundup(Height, RGB_TileHeight), 16) - * RGB_Meta_Plane_size = align(RGB_Meta_Stride * - * RGB_Meta_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align(RGB_Meta_Plane_size + RGB_Plane_size + - * Extradata, 4096) - */ - COLOR_FMT_RGBA8888_UBWC, - /* Venus RGBA1010102 UBWC format: - * Contains 2 planes in the following order - - * (A) Meta plane - * (B) RGBA plane - * - * <--- RGB_Meta_Stride ----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Height | - * M M M M M M M M M M M M . . | Meta_RGB_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <-------- RGB_Stride --------> - * <------- Width -------> - * R R R R R R R R R R R R . . . . ^ ^ - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . Height | - * R R R R R R R R R R R R . . . . | RGB_Scanlines - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * - * RGB_Stride = align(Width * 4, 256) - * RGB_Scanlines = align(Height, 16) - * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096) - * RGB_Meta_Stride = align(roundup(Width, RGB_TileWidth), 64) - * RGB_Meta_Scanline = align(roundup(Height, RGB_TileHeight), 16) - * RGB_Meta_Plane_size = align(RGB_Meta_Stride * - * RGB_Meta_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align(RGB_Meta_Plane_size + RGB_Plane_size + - * Extradata, 4096) - */ - COLOR_FMT_RGBA1010102_UBWC, - /* Venus RGB565 UBWC format: - * Contains 2 planes in the following order - - * (A) Meta plane - * (B) RGB plane - * - * <--- RGB_Meta_Stride ----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Height | - * M M M M M M M M M M M M . . | Meta_RGB_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <-------- RGB_Stride --------> - * <------- Width -------> - * R R R R R R R R R R R R . . . . ^ ^ - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . Height | - * R R R R R R R R R R R R . . . . | RGB_Scanlines - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . | | - * R R R R R R R R R R R R . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * - * RGB_Stride = align(Width * 2, 128) - * RGB_Scanlines = align(Height, 16) - * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096) - * RGB_Meta_Stride = align(roundup(Width, RGB_TileWidth), 64) - * RGB_Meta_Scanline = align(roundup(Height, RGB_TileHeight), 16) - * RGB_Meta_Plane_size = align(RGB_Meta_Stride * - * RGB_Meta_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align(RGB_Meta_Plane_size + RGB_Plane_size + - * Extradata, 4096) - */ - COLOR_FMT_RGB565_UBWC, - /* P010 UBWC: - * Compressed Macro-tile format for NV12. - * Contains 4 planes in the following order - - * (A) Y_Meta_Plane - * (B) Y_UBWC_Plane - * (C) UV_Meta_Plane - * (D) UV_UBWC_Plane - * - * Y_Meta_Plane consists of meta information to decode compressed - * tile data in Y_UBWC_Plane. - * Y_UBWC_Plane consists of Y data in compressed macro-tile format. - * UBWC decoder block will use the Y_Meta_Plane data together with - * Y_UBWC_Plane data to produce loss-less uncompressed 10 bit Y samples. - * - * UV_Meta_Plane consists of meta information to decode compressed - * tile data in UV_UBWC_Plane. - * UV_UBWC_Plane consists of UV data in compressed macro-tile format. - * UBWC decoder block will use UV_Meta_Plane data together with - * UV_UBWC_Plane data to produce loss-less uncompressed 10 bit 2x2 - * subsampled color difference samples. - * - * Each tile in Y_UBWC_Plane/UV_UBWC_Plane is independently decodable - * and randomly accessible. There is no dependency between tiles. - * - * <----- Y_Meta_Stride -----> - * <-------- Width ------> - * M M M M M M M M M M M M . . ^ ^ - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . Height | - * M M M M M M M M M M M M . . | Meta_Y_Scanlines - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . | | - * M M M M M M M M M M M M . . V | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . V - * <--Compressed tile Y Stride---> - * <------- Width -------> - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . ^ ^ - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . Height | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | Macro_tile_Y_Scanlines - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . | | - * Y* Y* Y* Y* Y* Y* Y* Y* . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * . . . . . . . . . . . . . . . . V - * <----- UV_Meta_Stride ----> - * M M M M M M M M M M M M . . ^ - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . | - * M M M M M M M M M M M M . . M_UV_Scanlines - * . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * <--Compressed tile UV Stride---> - * U* V* U* V* U* V* U* V* . . . . ^ - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . | - * U* V* U* V* U* V* U* V* . . . . UV_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . -------> Buffer size aligned to 4k - * - * - * Y_Stride = align(Width * 2, 256) - * UV_Stride = align(Width * 2, 256) - * Y_Scanlines = align(Height, 16) - * UV_Scanlines = align(Height/2, 16) - * Y_UBWC_Plane_Size = align(Y_Stride * Y_Scanlines, 4096) - * UV_UBWC_Plane_Size = align(UV_Stride * UV_Scanlines, 4096) - * Y_Meta_Stride = align(roundup(Width, Y_TileWidth), 64) - * Y_Meta_Scanlines = align(roundup(Height, Y_TileHeight), 16) - * Y_Meta_Plane_size = align(Y_Meta_Stride * Y_Meta_Scanlines, 4096) - * UV_Meta_Stride = align(roundup(Width, UV_TileWidth), 64) - * UV_Meta_Scanlines = align(roundup(Height, UV_TileHeight), 16) - * UV_Meta_Plane_size = align(UV_Meta_Stride * UV_Meta_Scanlines, 4096) - * Extradata = 8k - * - * Total size = align(Y_UBWC_Plane_size + UV_UBWC_Plane_size + - * Y_Meta_Plane_size + UV_Meta_Plane_size - * + max(Extradata, Y_Stride * 48), 4096) - */ - COLOR_FMT_P010_UBWC, - /* Venus P010: - * YUV 4:2:0 image with a plane of 10 bit Y samples followed - * by an interleaved U/V plane containing 10 bit 2x2 subsampled - * colour difference samples. - * - * <-------- Y/UV_Stride --------> - * <------- Width -------> - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . ^ ^ - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . Height | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | Y_Scanlines - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . | | - * Y Y Y Y Y Y Y Y Y Y Y Y . . . . V | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * U V U V U V U V U V U V . . . . ^ - * U V U V U V U V U V U V . . . . | - * U V U V U V U V U V U V . . . . | - * U V U V U V U V U V U V . . . . UV_Scanlines - * . . . . . . . . . . . . . . . . | - * . . . . . . . . . . . . . . . . V - * . . . . . . . . . . . . . . . . --> Buffer size alignment - * - * Y_Stride : Width * 2 aligned to 128 - * UV_Stride : Width * 2 aligned to 128 - * Y_Scanlines: Height aligned to 32 - * UV_Scanlines: Height/2 aligned to 16 - * Extradata: Arbitrary (software-imposed) padding - * Total size = align((Y_Stride * Y_Scanlines - * + UV_Stride * UV_Scanlines - * + max(Extradata, Y_Stride * 8), 4096) - */ - COLOR_FMT_P010, -}; - -#define COLOR_FMT_RGBA1010102_UBWC COLOR_FMT_RGBA1010102_UBWC -#define COLOR_FMT_RGB565_UBWC COLOR_FMT_RGB565_UBWC -#define COLOR_FMT_P010_UBWC COLOR_FMT_P010_UBWC -#define COLOR_FMT_P010 COLOR_FMT_P010 - -/* - * Function arguments: - * @color_fmt - * @width - * Progressive: width - * Interlaced: width - */ -static unsigned int VENUS_Y_STRIDE(int color_fmt, int width) -{ - unsigned int stride = 0; - - if (!width) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV21: - case COLOR_FMT_NV12: - case COLOR_FMT_NV12_MVTB: - case COLOR_FMT_NV12_UBWC: - stride = MSM_MEDIA_ALIGN(width, 128); - break; - case COLOR_FMT_NV12_BPP10_UBWC: - stride = MSM_MEDIA_ALIGN(width, 192); - stride = MSM_MEDIA_ALIGN(stride * 4 / 3, 256); - break; - case COLOR_FMT_P010_UBWC: - stride = MSM_MEDIA_ALIGN(width * 2, 256); - break; - case COLOR_FMT_P010: - stride = MSM_MEDIA_ALIGN(width * 2, 128); - break; - } - - return stride; -} - -/* - * Function arguments: - * @color_fmt - * @width - * Progressive: width - * Interlaced: width - */ -static unsigned int VENUS_UV_STRIDE(int color_fmt, int width) -{ - unsigned int stride = 0; - - if (!width) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV21: - case COLOR_FMT_NV12: - case COLOR_FMT_NV12_MVTB: - case COLOR_FMT_NV12_UBWC: - stride = MSM_MEDIA_ALIGN(width, 128); - break; - case COLOR_FMT_NV12_BPP10_UBWC: - stride = MSM_MEDIA_ALIGN(width, 192); - stride = MSM_MEDIA_ALIGN(stride * 4 / 3, 256); - break; - case COLOR_FMT_P010_UBWC: - stride = MSM_MEDIA_ALIGN(width * 2, 256); - break; - case COLOR_FMT_P010: - stride = MSM_MEDIA_ALIGN(width * 2, 128); - break; - } - - return stride; -} - -/* - * Function arguments: - * @color_fmt - * @height - * Progressive: height - * Interlaced: (height+1)>>1 - */ -static unsigned int VENUS_Y_SCANLINES(int color_fmt, int height) -{ - unsigned int sclines = 0; - - if (!height) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV21: - case COLOR_FMT_NV12: - case COLOR_FMT_NV12_MVTB: - case COLOR_FMT_NV12_UBWC: - case COLOR_FMT_P010: - sclines = MSM_MEDIA_ALIGN(height, 32); - break; - case COLOR_FMT_NV12_BPP10_UBWC: - case COLOR_FMT_P010_UBWC: - sclines = MSM_MEDIA_ALIGN(height, 16); - break; - } - - return sclines; -} - -/* - * Function arguments: - * @color_fmt - * @height - * Progressive: height - * Interlaced: (height+1)>>1 - */ -static unsigned int VENUS_UV_SCANLINES(int color_fmt, int height) -{ - unsigned int sclines = 0; - - if (!height) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV21: - case COLOR_FMT_NV12: - case COLOR_FMT_NV12_MVTB: - case COLOR_FMT_NV12_BPP10_UBWC: - case COLOR_FMT_P010_UBWC: - case COLOR_FMT_P010: - sclines = MSM_MEDIA_ALIGN((height + 1) >> 1, 16); - break; - case COLOR_FMT_NV12_UBWC: - sclines = MSM_MEDIA_ALIGN((height + 1) >> 1, 32); - break; - } - - return sclines; -} - -/* - * Function arguments: - * @color_fmt - * @width - * Progressive: width - * Interlaced: width - */ -static unsigned int VENUS_Y_META_STRIDE(int color_fmt, int width) -{ - int y_tile_width = 0, y_meta_stride; - - if (!width) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV12_UBWC: - case COLOR_FMT_P010_UBWC: - y_tile_width = 32; - break; - case COLOR_FMT_NV12_BPP10_UBWC: - y_tile_width = 48; - break; - default: - return 0; - } - - y_meta_stride = MSM_MEDIA_ROUNDUP(width, y_tile_width); - return MSM_MEDIA_ALIGN(y_meta_stride, 64); -} - -/* - * Function arguments: - * @color_fmt - * @height - * Progressive: height - * Interlaced: (height+1)>>1 - */ -static unsigned int VENUS_Y_META_SCANLINES(int color_fmt, int height) -{ - int y_tile_height = 0, y_meta_scanlines; - - if (!height) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV12_UBWC: - y_tile_height = 8; - break; - case COLOR_FMT_NV12_BPP10_UBWC: - case COLOR_FMT_P010_UBWC: - y_tile_height = 4; - break; - default: - return 0; - } - - y_meta_scanlines = MSM_MEDIA_ROUNDUP(height, y_tile_height); - return MSM_MEDIA_ALIGN(y_meta_scanlines, 16); -} - -/* - * Function arguments: - * @color_fmt - * @width - * Progressive: width - * Interlaced: width - */ -static unsigned int VENUS_UV_META_STRIDE(int color_fmt, int width) -{ - int uv_tile_width = 0, uv_meta_stride; - - if (!width) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV12_UBWC: - case COLOR_FMT_P010_UBWC: - uv_tile_width = 16; - break; - case COLOR_FMT_NV12_BPP10_UBWC: - uv_tile_width = 24; - break; - default: - return 0; - } - - uv_meta_stride = MSM_MEDIA_ROUNDUP((width+1)>>1, uv_tile_width); - return MSM_MEDIA_ALIGN(uv_meta_stride, 64); -} - -/* - * Function arguments: - * @color_fmt - * @height - * Progressive: height - * Interlaced: (height+1)>>1 - */ -static unsigned int VENUS_UV_META_SCANLINES(int color_fmt, int height) -{ - int uv_tile_height = 0, uv_meta_scanlines; - - if (!height) - return 0; - - switch (color_fmt) { - case COLOR_FMT_NV12_UBWC: - uv_tile_height = 8; - break; - case COLOR_FMT_NV12_BPP10_UBWC: - case COLOR_FMT_P010_UBWC: - uv_tile_height = 4; - break; - default: - return 0; - } - - uv_meta_scanlines = MSM_MEDIA_ROUNDUP((height+1)>>1, uv_tile_height); - return MSM_MEDIA_ALIGN(uv_meta_scanlines, 16); -} - -static unsigned int VENUS_RGB_STRIDE(int color_fmt, int width) -{ - unsigned int alignment = 0, bpp = 4; - - if (!width) - return 0; - - switch (color_fmt) { - case COLOR_FMT_RGBA8888: - alignment = 128; - break; - case COLOR_FMT_RGB565_UBWC: - alignment = 256; - bpp = 2; - break; - case COLOR_FMT_RGBA8888_UBWC: - case COLOR_FMT_RGBA1010102_UBWC: - alignment = 256; - break; - default: - return 0; - } - - return MSM_MEDIA_ALIGN(width * bpp, alignment); -} - -static unsigned int VENUS_RGB_SCANLINES(int color_fmt, int height) -{ - unsigned int alignment = 0; - - if (!height) - return 0; - - switch (color_fmt) { - case COLOR_FMT_RGBA8888: - alignment = 32; - break; - case COLOR_FMT_RGBA8888_UBWC: - case COLOR_FMT_RGBA1010102_UBWC: - case COLOR_FMT_RGB565_UBWC: - alignment = 16; - break; - default: - return 0; - } - - return MSM_MEDIA_ALIGN(height, alignment); -} - -static unsigned int VENUS_RGB_META_STRIDE(int color_fmt, int width) -{ - int rgb_meta_stride; - - if (!width) - return 0; - - switch (color_fmt) { - case COLOR_FMT_RGBA8888_UBWC: - case COLOR_FMT_RGBA1010102_UBWC: - case COLOR_FMT_RGB565_UBWC: - rgb_meta_stride = MSM_MEDIA_ROUNDUP(width, 16); - return MSM_MEDIA_ALIGN(rgb_meta_stride, 64); - } - - return 0; -} - -static unsigned int VENUS_RGB_META_SCANLINES(int color_fmt, int height) -{ - int rgb_meta_scanlines; - - if (!height) - return 0; - - switch (color_fmt) { - case COLOR_FMT_RGBA8888_UBWC: - case COLOR_FMT_RGBA1010102_UBWC: - case COLOR_FMT_RGB565_UBWC: - rgb_meta_scanlines = MSM_MEDIA_ROUNDUP(height, 4); - return MSM_MEDIA_ALIGN(rgb_meta_scanlines, 16); - } - - return 0; -} - -#endif diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c index da53ca88251e7..e8066f9fd5343 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c @@ -527,13 +527,14 @@ static void mdp4_crtc_wait_for_flush_done(struct drm_crtc *crtc) struct drm_device *dev = crtc->dev; struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc); struct mdp4_kms *mdp4_kms = get_kms(crtc); + wait_queue_head_t *queue = drm_crtc_vblank_waitqueue(crtc); int ret; ret = drm_crtc_vblank_get(crtc); if (ret) return; - ret = wait_event_timeout(dev->vblank[drm_crtc_index(crtc)].queue, + ret = wait_event_timeout(*queue, !(mdp4_read(mdp4_kms, REG_MDP4_OVERLAY_FLUSH) & mdp4_crtc->flushed_mask), msecs_to_jiffies(50)); diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c index 4c4900a7beda8..373ae7d9bf01c 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c @@ -1234,6 +1234,7 @@ static void mdp5_crtc_wait_for_flush_done(struct drm_crtc *crtc) struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc); struct mdp5_crtc_state *mdp5_cstate = to_mdp5_crtc_state(crtc->state); struct mdp5_ctl *ctl = mdp5_cstate->ctl; + wait_queue_head_t *queue = drm_crtc_vblank_waitqueue(crtc); int ret; /* Should not call this function if crtc is disabled. */ @@ -1244,7 +1245,7 @@ static void mdp5_crtc_wait_for_flush_done(struct drm_crtc *crtc) if (ret) return; - ret = wait_event_timeout(dev->vblank[drm_crtc_index(crtc)].queue, + ret = wait_event_timeout(*queue, ((mdp5_ctl_get_commit_status(ctl) & mdp5_crtc->flushed_mask) == 0), msecs_to_jiffies(50)); diff --git a/drivers/gpu/drm/msm/disp/mdp_format.c b/drivers/gpu/drm/msm/disp/mdp_format.c index 426782d50cb49..33da569eae5ef 100644 --- a/drivers/gpu/drm/msm/disp/mdp_format.c +++ b/drivers/gpu/drm/msm/disp/mdp_format.c @@ -66,87 +66,205 @@ static struct csc_cfg csc_convert[CSC_MAX] = { #define MDP_TILE_HEIGHT_UBWC 4 #define MDP_TILE_HEIGHT_NV12 8 -#define INTERLEAVED_RGB_FMT(fmt, a, r, g, b, e0, e1, e2, e3, uc, alpha, \ -bp, flg, fm, np) \ +#define INTERLEAVED_RGB_FMT(fmt, bp, r, g, b, e0, e1, e2) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ .fetch_type = MDP_PLANE_INTERLEAVED, \ - .alpha_enable = alpha, \ + .alpha_enable = false, \ + .element = { (e0), (e1), (e2), 0 }, \ + .bpc_g_y = g, \ + .bpc_b_cb = b, \ + .bpc_r_cr = r, \ + .bpc_a = 0, \ + .chroma_sample = CHROMA_FULL, \ + .unpack_count = 3, \ + .bpp = bp, \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT, \ + .num_planes = 1, \ + .tile_height = MDP_TILE_HEIGHT_DEFAULT \ +} + +#define INTERLEAVED_RGBA_FMT(fmt, bp, a, r, g, b, e0, e1, e2, e3) \ +{ \ + .pixel_format = DRM_FORMAT_ ## fmt, \ + .fetch_type = MDP_PLANE_INTERLEAVED, \ + .alpha_enable = true, \ .element = { (e0), (e1), (e2), (e3) }, \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ .bpc_a = a, \ .chroma_sample = CHROMA_FULL, \ - .unpack_count = uc, \ + .unpack_count = 4, \ .bpp = bp, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg, \ - .num_planes = np, \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT, \ + .num_planes = 1, \ .tile_height = MDP_TILE_HEIGHT_DEFAULT \ } -#define INTERLEAVED_RGB_FMT_TILED(fmt, a, r, g, b, e0, e1, e2, e3, uc, \ -alpha, bp, flg, fm, np, th) \ +#define INTERLEAVED_RGBX_FMT(fmt, bp, a, r, g, b, e0, e1, e2, e3) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ .fetch_type = MDP_PLANE_INTERLEAVED, \ - .alpha_enable = alpha, \ + .alpha_enable = false, \ .element = { (e0), (e1), (e2), (e3) }, \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ .bpc_a = a, \ .chroma_sample = CHROMA_FULL, \ - .unpack_count = uc, \ + .unpack_count = 4, \ .bpp = bp, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg, \ - .num_planes = np, \ - .tile_height = th \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT, \ + .num_planes = 1, \ + .tile_height = MDP_TILE_HEIGHT_DEFAULT \ } -#define INTERLEAVED_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, e3, \ -alpha, chroma, count, bp, flg, fm, np) \ +#define INTERLEAVED_RGBA_DX_FMT(fmt, bp, a, r, g, b, e0, e1, e2, e3) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ .fetch_type = MDP_PLANE_INTERLEAVED, \ - .alpha_enable = alpha, \ - .element = { (e0), (e1), (e2), (e3)}, \ + .alpha_enable = true, \ + .element = { (e0), (e1), (e2), (e3) }, \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ .bpc_a = a, \ - .chroma_sample = chroma, \ - .unpack_count = count, \ + .chroma_sample = CHROMA_FULL, \ + .unpack_count = 4, \ .bpp = bp, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg, \ - .num_planes = np, \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_DX, \ + .num_planes = 1, \ .tile_height = MDP_TILE_HEIGHT_DEFAULT \ } -#define PSEUDO_YUV_FMT(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np) \ +#define INTERLEAVED_RGBX_DX_FMT(fmt, bp, a, r, g, b, e0, e1, e2, e3) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ - .fetch_type = MDP_PLANE_PSEUDO_PLANAR, \ - .alpha_enable = 0, \ - .element = { (e0), (e1), 0, 0 }, \ + .fetch_type = MDP_PLANE_INTERLEAVED, \ + .alpha_enable = false, \ + .element = { (e0), (e1), (e2), (e3) }, \ + .bpc_g_y = g, \ + .bpc_b_cb = b, \ + .bpc_r_cr = r, \ + .bpc_a = a, \ + .chroma_sample = CHROMA_FULL, \ + .unpack_count = 4, \ + .bpp = bp, \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_DX, \ + .num_planes = 1, \ + .tile_height = MDP_TILE_HEIGHT_DEFAULT \ +} + +#define INTERLEAVED_RGB_FMT_TILED(fmt, bp, r, g, b, e0, e1, e2) \ +{ \ + .pixel_format = DRM_FORMAT_ ## fmt, \ + .fetch_type = MDP_PLANE_INTERLEAVED, \ + .alpha_enable = false, \ + .element = { (e0), (e1), (e2), 0 }, \ + .bpc_g_y = g, \ + .bpc_b_cb = b, \ + .bpc_r_cr = r, \ + .bpc_a = 0, \ + .chroma_sample = CHROMA_FULL, \ + .unpack_count = 3, \ + .bpp = bp, \ + .fetch_mode = MDP_FETCH_UBWC, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_COMPRESSED, \ + .num_planes = 2, \ + .tile_height = MDP_TILE_HEIGHT_UBWC, \ +} + +#define INTERLEAVED_RGBA_FMT_TILED(fmt, bp, a, r, g, b, e0, e1, e2, e3) \ +{ \ + .pixel_format = DRM_FORMAT_ ## fmt, \ + .fetch_type = MDP_PLANE_INTERLEAVED, \ + .alpha_enable = true, \ + .element = { (e0), (e1), (e2), (e3) }, \ + .bpc_g_y = g, \ + .bpc_b_cb = b, \ + .bpc_r_cr = r, \ + .bpc_a = a, \ + .chroma_sample = CHROMA_FULL, \ + .unpack_count = 4, \ + .bpp = bp, \ + .fetch_mode = MDP_FETCH_UBWC, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_COMPRESSED, \ + .num_planes = 2, \ + .tile_height = MDP_TILE_HEIGHT_UBWC, \ +} + +#define INTERLEAVED_RGBX_FMT_TILED(fmt, bp, a, r, g, b, e0, e1, e2, e3) \ +{ \ + .pixel_format = DRM_FORMAT_ ## fmt, \ + .fetch_type = MDP_PLANE_INTERLEAVED, \ + .alpha_enable = false, \ + .element = { (e0), (e1), (e2), (e3) }, \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ .bpc_a = a, \ + .chroma_sample = CHROMA_FULL, \ + .unpack_count = 4, \ + .bpp = bp, \ + .fetch_mode = MDP_FETCH_UBWC, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_COMPRESSED, \ + .num_planes = 2, \ + .tile_height = MDP_TILE_HEIGHT_UBWC, \ +} + +#define INTERLEAVED_RGBA_DX_FMT_TILED(fmt, bp, a, r, g, b, e0, e1, e2, e3) \ +{ \ + .pixel_format = DRM_FORMAT_ ## fmt, \ + .fetch_type = MDP_PLANE_INTERLEAVED, \ + .alpha_enable = true, \ + .element = { (e0), (e1), (e2), (e3) }, \ + .bpc_g_y = g, \ + .bpc_b_cb = b, \ + .bpc_r_cr = r, \ + .bpc_a = a, \ + .chroma_sample = CHROMA_FULL, \ + .unpack_count = 4, \ + .bpp = bp, \ + .fetch_mode = MDP_FETCH_UBWC, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_DX | \ + MSM_FORMAT_FLAG_COMPRESSED, \ + .num_planes = 2, \ + .tile_height = MDP_TILE_HEIGHT_UBWC, \ +} + +#define INTERLEAVED_YUV_FMT(fmt, bp, r, g, b, e0, e1, e2, e3, chroma) \ +{ \ + .pixel_format = DRM_FORMAT_ ## fmt, \ + .fetch_type = MDP_PLANE_INTERLEAVED, \ + .alpha_enable = false, \ + .element = { (e0), (e1), (e2), (e3)}, \ + .bpc_g_y = g, \ + .bpc_b_cb = b, \ + .bpc_r_cr = r, \ + .bpc_a = 0, \ .chroma_sample = chroma, \ - .unpack_count = 2, \ - .bpp = 2, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg, \ - .num_planes = np, \ + .unpack_count = 4, \ + .bpp = bp, \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_YUV, \ + .num_planes = 1, \ .tile_height = MDP_TILE_HEIGHT_DEFAULT \ } -#define PSEUDO_YUV_FMT_TILED(fmt, a, r, g, b, e0, e1, chroma, \ -flg, fm, np, th) \ +#define PSEUDO_YUV_FMT(fmt, r, g, b, e0, e1, chroma) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ .fetch_type = MDP_PLANE_PSEUDO_PLANAR, \ @@ -155,17 +273,18 @@ flg, fm, np, th) \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ - .bpc_a = a, \ + .bpc_a = 0, \ .chroma_sample = chroma, \ .unpack_count = 2, \ .bpp = 2, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg, \ - .num_planes = np, \ - .tile_height = th \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_YUV, \ + .num_planes = 2, \ + .tile_height = MDP_TILE_HEIGHT_DEFAULT \ } -#define PSEUDO_YUV_FMT_LOOSE(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np)\ +#define PSEUDO_YUV_FMT_TILED(fmt, r, g, b, e0, e1, chroma, flg, th) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ .fetch_type = MDP_PLANE_PSEUDO_PLANAR, \ @@ -174,18 +293,19 @@ flg, fm, np, th) \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ - .bpc_a = a, \ + .bpc_a = 0, \ .chroma_sample = chroma, \ .unpack_count = 2, \ .bpp = 2, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB | flg, \ - .num_planes = np, \ - .tile_height = MDP_TILE_HEIGHT_DEFAULT \ + .fetch_mode = MDP_FETCH_UBWC, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_YUV | \ + MSM_FORMAT_FLAG_COMPRESSED | flg, \ + .num_planes = 4, \ + .tile_height = th \ } -#define PSEUDO_YUV_FMT_LOOSE_TILED(fmt, a, r, g, b, e0, e1, chroma, \ -flg, fm, np, th) \ +#define PSEUDO_YUV_FMT_LOOSE(fmt, r, g, b, e0, e1, chroma) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ .fetch_type = MDP_PLANE_PSEUDO_PLANAR, \ @@ -194,323 +314,242 @@ flg, fm, np, th) \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ - .bpc_a = a, \ + .bpc_a = 0, \ .chroma_sample = chroma, \ .unpack_count = 2, \ .bpp = 2, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB | flg, \ - .num_planes = np, \ - .tile_height = th \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB | \ + MSM_FORMAT_FLAG_DX | \ + MSM_FORMAT_FLAG_YUV, \ + .num_planes = 2, \ + .tile_height = MDP_TILE_HEIGHT_DEFAULT \ } -#define PLANAR_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, alpha, chroma, bp, \ -flg, fm, np) \ +#define PLANAR_YUV_FMT(fmt, bp, r, g, b, e0, e1, e2, chroma) \ { \ .pixel_format = DRM_FORMAT_ ## fmt, \ .fetch_type = MDP_PLANE_PLANAR, \ - .alpha_enable = alpha, \ + .alpha_enable = false, \ .element = { (e0), (e1), (e2), 0 }, \ .bpc_g_y = g, \ .bpc_b_cb = b, \ .bpc_r_cr = r, \ - .bpc_a = a, \ + .bpc_a = 0, \ .chroma_sample = chroma, \ .unpack_count = 1, \ .bpp = bp, \ - .fetch_mode = fm, \ - .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg, \ - .num_planes = np, \ + .fetch_mode = MDP_FETCH_LINEAR, \ + .flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | \ + MSM_FORMAT_FLAG_YUV, \ + .num_planes = 3, \ .tile_height = MDP_TILE_HEIGHT_DEFAULT \ } static const struct msm_format mdp_formats[] = { - INTERLEAVED_RGB_FMT(ARGB8888, + INTERLEAVED_RGBA_FMT(ARGB8888, 4, BPC8A, BPC8, BPC8, BPC8, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - true, 4, 0, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(ABGR8888, + INTERLEAVED_RGBA_FMT(ABGR8888, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, 0, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(XBGR8888, + INTERLEAVED_RGBX_FMT(XBGR8888, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - false, 4, 0, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(RGBA8888, + INTERLEAVED_RGBA_FMT(RGBA8888, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - true, 4, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), - INTERLEAVED_RGB_FMT(BGRA8888, + INTERLEAVED_RGBA_FMT(BGRA8888, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - true, 4, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(BGRX8888, + INTERLEAVED_RGBX_FMT(BGRX8888, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - false, 4, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(XRGB8888, + INTERLEAVED_RGBX_FMT(XRGB8888, 4, BPC8A, BPC8, BPC8, BPC8, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - false, 4, 0, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(RGBX8888, + INTERLEAVED_RGBX_FMT(RGBX8888, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - false, 4, 0, - MDP_FETCH_LINEAR, 1), - - INTERLEAVED_RGB_FMT(RGB888, - 0, BPC8, BPC8, BPC8, - C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3, - false, 3, 0, - MDP_FETCH_LINEAR, 1), - - INTERLEAVED_RGB_FMT(BGR888, - 0, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3, - false, 3, 0, - MDP_FETCH_LINEAR, 1), - - INTERLEAVED_RGB_FMT(RGB565, - 0, BPC5, BPC6, BPC5, - C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3, - false, 2, 0, - MDP_FETCH_LINEAR, 1), - - INTERLEAVED_RGB_FMT(BGR565, - 0, BPC5, BPC6, BPC5, - C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3, - false, 2, 0, - MDP_FETCH_LINEAR, 1), - - INTERLEAVED_RGB_FMT(ARGB1555, + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), + + INTERLEAVED_RGB_FMT(RGB888, 3, + BPC8, BPC8, BPC8, + C1_B_Cb, C0_G_Y, C2_R_Cr), + + INTERLEAVED_RGB_FMT(BGR888, 3, + BPC8, BPC8, BPC8, + C2_R_Cr, C0_G_Y, C1_B_Cb), + + INTERLEAVED_RGB_FMT(RGB565, 2, + BPC5, BPC6, BPC5, + C1_B_Cb, C0_G_Y, C2_R_Cr), + + INTERLEAVED_RGB_FMT(BGR565, 2, + BPC5, BPC6, BPC5, + C2_R_Cr, C0_G_Y, C1_B_Cb), + + INTERLEAVED_RGBA_FMT(ARGB1555, 2, BPC1A, BPC5, BPC5, BPC5, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(ABGR1555, + INTERLEAVED_RGBA_FMT(ABGR1555, 2, BPC1A, BPC5, BPC5, BPC5, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(RGBA5551, + INTERLEAVED_RGBA_FMT(RGBA5551, 2, BPC1A, BPC5, BPC5, BPC5, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), - INTERLEAVED_RGB_FMT(BGRA5551, + INTERLEAVED_RGBA_FMT(BGRA5551, 2, BPC1A, BPC5, BPC5, BPC5, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(XRGB1555, + INTERLEAVED_RGBX_FMT(XRGB1555, 2, BPC1A, BPC5, BPC5, BPC5, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(XBGR1555, + INTERLEAVED_RGBX_FMT(XBGR1555, 2, BPC1A, BPC5, BPC5, BPC5, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(RGBX5551, + INTERLEAVED_RGBX_FMT(RGBX5551, 2, BPC1A, BPC5, BPC5, BPC5, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), - INTERLEAVED_RGB_FMT(BGRX5551, + INTERLEAVED_RGBX_FMT(BGRX5551, 2, BPC1A, BPC5, BPC5, BPC5, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(ARGB4444, + INTERLEAVED_RGBA_FMT(ARGB4444, 2, BPC4A, BPC4, BPC4, BPC4, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(ABGR4444, + INTERLEAVED_RGBA_FMT(ABGR4444, 2, BPC4A, BPC4, BPC4, BPC4, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(RGBA4444, + INTERLEAVED_RGBA_FMT(RGBA4444, 2, BPC4A, BPC4, BPC4, BPC4, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), - INTERLEAVED_RGB_FMT(BGRA4444, + INTERLEAVED_RGBA_FMT(BGRA4444, 2, BPC4A, BPC4, BPC4, BPC4, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - true, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(XRGB4444, + INTERLEAVED_RGBX_FMT(XRGB4444, 2, BPC4A, BPC4, BPC4, BPC4, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(XBGR4444, + INTERLEAVED_RGBX_FMT(XBGR4444, 2, BPC4A, BPC4, BPC4, BPC4, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(RGBX4444, + INTERLEAVED_RGBX_FMT(RGBX4444, 2, BPC4A, BPC4, BPC4, BPC4, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), - INTERLEAVED_RGB_FMT(BGRX4444, + INTERLEAVED_RGBX_FMT(BGRX4444, 2, BPC4A, BPC4, BPC4, BPC4, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - false, 2, 0, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(BGRA1010102, + INTERLEAVED_RGBA_DX_FMT(BGRA1010102, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - true, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(RGBA1010102, + INTERLEAVED_RGBA_DX_FMT(RGBA1010102, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - true, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), - INTERLEAVED_RGB_FMT(ABGR2101010, + INTERLEAVED_RGBA_DX_FMT(ABGR2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(ARGB2101010, + INTERLEAVED_RGBA_DX_FMT(ARGB2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(XRGB2101010, + INTERLEAVED_RGBX_DX_FMT(XRGB2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4, - false, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA), - INTERLEAVED_RGB_FMT(BGRX1010102, + INTERLEAVED_RGBX_DX_FMT(BGRX1010102, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4, - false, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT(XBGR2101010, + INTERLEAVED_RGBX_DX_FMT(XBGR2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - false, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT(RGBX1010102, + INTERLEAVED_RGBX_DX_FMT(RGBX1010102, 4, BPC8A, BPC8, BPC8, BPC8, - C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4, - false, 4, MSM_FORMAT_FLAG_DX, - MDP_FETCH_LINEAR, 1), + C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr), /* --- RGB formats above / YUV formats below this line --- */ /* 2 plane YUV */ PSEUDO_YUV_FMT(NV12, - 0, BPC8, BPC8, BPC8, + BPC8, BPC8, BPC8, C1_B_Cb, C2_R_Cr, - CHROMA_420, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_420), PSEUDO_YUV_FMT(NV21, - 0, BPC8, BPC8, BPC8, + BPC8, BPC8, BPC8, C2_R_Cr, C1_B_Cb, - CHROMA_420, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_420), PSEUDO_YUV_FMT(NV16, - 0, BPC8, BPC8, BPC8, + BPC8, BPC8, BPC8, C1_B_Cb, C2_R_Cr, - CHROMA_H2V1, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_H2V1), PSEUDO_YUV_FMT(NV61, - 0, BPC8, BPC8, BPC8, + BPC8, BPC8, BPC8, C2_R_Cr, C1_B_Cb, - CHROMA_H2V1, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_H2V1), PSEUDO_YUV_FMT_LOOSE(P010, - 0, BPC8, BPC8, BPC8, + BPC8, BPC8, BPC8, C1_B_Cb, C2_R_Cr, - CHROMA_420, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_420), /* 1 plane YUV */ - INTERLEAVED_YUV_FMT(VYUY, - 0, BPC8, BPC8, BPC8, + INTERLEAVED_YUV_FMT(VYUY, 2, + BPC8, BPC8, BPC8, C2_R_Cr, C0_G_Y, C1_B_Cb, C0_G_Y, - false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_H2V1), - INTERLEAVED_YUV_FMT(UYVY, - 0, BPC8, BPC8, BPC8, + INTERLEAVED_YUV_FMT(UYVY, 2, + BPC8, BPC8, BPC8, C1_B_Cb, C0_G_Y, C2_R_Cr, C0_G_Y, - false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_H2V1), - INTERLEAVED_YUV_FMT(YUYV, - 0, BPC8, BPC8, BPC8, + INTERLEAVED_YUV_FMT(YUYV, 2, + BPC8, BPC8, BPC8, C0_G_Y, C1_B_Cb, C0_G_Y, C2_R_Cr, - false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_H2V1), - INTERLEAVED_YUV_FMT(YVYU, - 0, BPC8, BPC8, BPC8, + INTERLEAVED_YUV_FMT(YVYU, 2, + BPC8, BPC8, BPC8, C0_G_Y, C2_R_Cr, C0_G_Y, C1_B_Cb, - false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + CHROMA_H2V1), /* 3 plane YUV */ - PLANAR_YUV_FMT(YUV420, - 0, BPC8, BPC8, BPC8, + PLANAR_YUV_FMT(YUV420, 1, + BPC8, BPC8, BPC8, C2_R_Cr, C1_B_Cb, C0_G_Y, - false, CHROMA_420, 1, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 3), + CHROMA_420), - PLANAR_YUV_FMT(YVU420, - 0, BPC8, BPC8, BPC8, + PLANAR_YUV_FMT(YVU420, 1, + BPC8, BPC8, BPC8, C1_B_Cb, C2_R_Cr, C0_G_Y, - false, CHROMA_420, 1, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 3), + CHROMA_420), }; /* @@ -520,82 +559,61 @@ static const struct msm_format mdp_formats[] = { * the data will be passed by user-space. */ static const struct msm_format mdp_formats_ubwc[] = { - INTERLEAVED_RGB_FMT_TILED(BGR565, - 0, BPC5, BPC6, BPC5, - C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3, - false, 2, MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + INTERLEAVED_RGB_FMT_TILED(BGR565, 2, + BPC5, BPC6, BPC5, + C2_R_Cr, C0_G_Y, C1_B_Cb), - INTERLEAVED_RGB_FMT_TILED(ABGR8888, + INTERLEAVED_RGBA_FMT_TILED(ABGR8888, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), /* ARGB8888 and ABGR8888 purposely have the same color * ordering. The hardware only supports ABGR8888 UBWC * natively. */ - INTERLEAVED_RGB_FMT_TILED(ARGB8888, + INTERLEAVED_RGBA_FMT_TILED(ARGB8888, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT_TILED(XBGR8888, + INTERLEAVED_RGBX_FMT_TILED(XBGR8888, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - false, 4, MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT_TILED(XRGB8888, + INTERLEAVED_RGBX_FMT_TILED(XRGB8888, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - false, 4, MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT_TILED(ABGR2101010, + INTERLEAVED_RGBA_DX_FMT_TILED(ABGR2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT_TILED(XBGR2101010, + INTERLEAVED_RGBA_DX_FMT_TILED(XBGR2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), - INTERLEAVED_RGB_FMT_TILED(XRGB2101010, + INTERLEAVED_RGBA_DX_FMT_TILED(XRGB2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), /* XRGB2101010 and ARGB2101010 purposely have the same color * ordering. The hardware only supports ARGB2101010 UBWC * natively. */ - INTERLEAVED_RGB_FMT_TILED(ARGB2101010, + INTERLEAVED_RGBA_DX_FMT_TILED(ARGB2101010, 4, BPC8A, BPC8, BPC8, BPC8, - C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4, - true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC), + C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA), PSEUDO_YUV_FMT_TILED(NV12, - 0, BPC8, BPC8, BPC8, + BPC8, BPC8, BPC8, C1_B_Cb, C2_R_Cr, - CHROMA_420, MSM_FORMAT_FLAG_YUV | - MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 4, MDP_TILE_HEIGHT_NV12), + CHROMA_420, 0, + MDP_TILE_HEIGHT_NV12), PSEUDO_YUV_FMT_TILED(P010, - 0, BPC8, BPC8, BPC8, + BPC8, BPC8, BPC8, C1_B_Cb, C2_R_Cr, - CHROMA_420, MSM_FORMAT_FLAG_DX | - MSM_FORMAT_FLAG_YUV | - MSM_FORMAT_FLAG_COMPRESSED, - MDP_FETCH_UBWC, 4, MDP_TILE_HEIGHT_UBWC), + CHROMA_420, MSM_FORMAT_FLAG_DX, + MDP_TILE_HEIGHT_UBWC), }; const struct msm_format *mdp_get_format(struct msm_kms *kms, uint32_t format, diff --git a/drivers/gpu/drm/msm/disp/mdp_format.h b/drivers/gpu/drm/msm/disp/mdp_format.h index a00d646ff4d47..915954bf5dc76 100644 --- a/drivers/gpu/drm/msm/disp/mdp_format.h +++ b/drivers/gpu/drm/msm/disp/mdp_format.h @@ -24,7 +24,7 @@ enum msm_format_flags { #define MSM_FORMAT_FLAG_UNPACK_TIGHT BIT(MSM_FORMAT_FLAG_UNPACK_TIGHT_BIT) #define MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB BIT(MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB_BIT) -/** +/* * DPU HW,Component order color map */ enum { @@ -37,6 +37,10 @@ enum { /** * struct msm_format: defines the format configuration * @pixel_format: format fourcc + * @bpc_g_y: element bit widths: BPC for G or Y + * @bpc_b_cb: element bit widths: BPC for B or Cb + * @bpc_r_cr: element bit widths: BPC for R or Cr + * @bpc_a: element bit widths: BPC for the alpha channel * @element: element color ordering * @fetch_type: how the color components are packed in pixel format * @chroma_sample: chroma sub-samplng type diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h index b5f452bd7ada5..53bd1dcde15fb 100644 --- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h +++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h @@ -38,6 +38,7 @@ * struct msm_disp_state - structure to store current dpu state * @dev: device pointer * @drm_dev: drm device pointer + * @blocks: list head for hardware state blocks * @atomic_state: atomic state duplicated at the time of the error * @time: timestamp at which the coredump was captured */ @@ -55,7 +56,7 @@ struct msm_disp_state { /** * struct msm_disp_state_block - structure to store each hardware block state * @name: name of the block - * @drm_dev: handle to the linked list head + * @node: handle to the linked list head * @size: size of the register space of this hardware block * @state: array holding the register dump of this hardware block * @base_addr: starting address of this hardware block's register space @@ -88,8 +89,9 @@ void msm_disp_snapshot_destroy(struct drm_device *drm_dev); * msm_disp_snapshot_state_sync - synchronously snapshot display state * @kms: the kms object * - * Returns state or error + * Returns: state or error * + * Context: * Must be called with &kms->dump_mutex held */ struct msm_disp_state *msm_disp_snapshot_state_sync(struct msm_kms *kms); @@ -97,7 +99,7 @@ struct msm_disp_state *msm_disp_snapshot_state_sync(struct msm_kms *kms); /** * msm_disp_snapshot_state - trigger to dump the display snapshot * @drm_dev: handle to drm device - + * * Returns: none */ void msm_disp_snapshot_state(struct drm_device *drm_dev); @@ -114,7 +116,7 @@ void msm_disp_state_print(struct msm_disp_state *disp_state, struct drm_printer /** * msm_disp_snapshot_capture_state - utility to capture atomic state and hw registers * @disp_state: handle to msm_disp_state struct - + * * Returns: none */ void msm_disp_snapshot_capture_state(struct msm_disp_state *disp_state); @@ -122,7 +124,7 @@ void msm_disp_snapshot_capture_state(struct msm_disp_state *disp_state); /** * msm_disp_state_free - free the memory after the coredump has been read * @data: handle to struct msm_disp_state - + * * Returns: none */ void msm_disp_state_free(void *data); @@ -130,7 +132,6 @@ void msm_disp_state_free(void *data); /** * msm_disp_snapshot_add_block - add a hardware block with its register dump * @disp_state: handle to struct msm_disp_state - * @name: name of the hardware block * @len: size of the register space of the hardware block * @base_addr: starting address of the register space of the hardware block * @fmt: format in which the block names need to be printed diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c index 071bcdea80f71..19b470968f4db 100644 --- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c +++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c @@ -82,8 +82,7 @@ void msm_disp_state_print(struct msm_disp_state *state, struct drm_printer *p) drm_printf(p, "kernel: " UTS_RELEASE "\n"); drm_printf(p, "module: " KBUILD_MODNAME "\n"); drm_printf(p, "dpu devcoredump\n"); - drm_printf(p, "time: %lld.%09ld\n", - state->time.tv_sec, state->time.tv_nsec); + drm_printf(p, "time: %ptSp\n", &state->time); list_for_each_entry_safe(block, tmp, &state->blocks, node) { drm_printf(p, "====================%s================\n", block->name); diff --git a/drivers/gpu/drm/msm/dp/dp_audio.c b/drivers/gpu/drm/msm/dp/dp_audio.c index 41018e82efa10..035e230201fd9 100644 --- a/drivers/gpu/drm/msm/dp/dp_audio.c +++ b/drivers/gpu/drm/msm/dp/dp_audio.c @@ -284,7 +284,7 @@ int msm_dp_audio_prepare(struct drm_bridge *bridge, * such cases check for connection status and bail out if not * connected. */ - if (!msm_dp_display->power_on) { + if (!msm_dp_display->active_stream_cnt) { rc = -EINVAL; goto end; } diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index c42fd2c17a328..ee8ec87b1a277 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -65,14 +65,24 @@ (PSR_UPDATE_MASK | PSR_CAPTURE_MASK | PSR_EXIT_MASK | \ PSR_UPDATE_ERROR_MASK | PSR_WAKE_ERROR_MASK) +#define DP_INTERRUPT_STATUS5 \ + (DP_INTR_DP0_VCPF_SENT | DP_INTR_DP1_VCPF_SENT) +#define DP_INTERRUPT_STATUS5_MASK \ + (DP_INTERRUPT_STATUS5 << DP_INTERRUPT_STATUS_MASK_SHIFT) + #define DP_CTRL_INTR_READY_FOR_VIDEO BIT(0) #define DP_CTRL_INTR_IDLE_PATTERN_SENT BIT(3) +#define DP_DP0_PUSH_VCPF BIT(12) +#define DP_DP1_PUSH_VCPF BIT(14) +#define DP_MSTLINK_PUSH_VCPF BIT(12) + #define MR_LINK_TRAINING1 0x8 #define MR_LINK_SYMBOL_ERM 0x80 #define MR_LINK_PRBS7 0x100 #define MR_LINK_CUSTOM80 0x200 #define MR_LINK_TRAINING4 0x40 +#define DP_MAX_TIME_SLOTS 64 enum { DP_TRAINING_NONE, @@ -109,6 +119,11 @@ struct msm_dp_vc_tu_mapping_table { u8 tu_size_minus1; }; +struct msm_dp_mst_ch_slot_info { + u32 start_slot; + u32 tot_slots; +}; + struct msm_dp_ctrl_private { struct msm_dp_ctrl msm_dp_ctrl; struct drm_device *drm_dev; @@ -118,6 +133,8 @@ struct msm_dp_ctrl_private { struct msm_dp_link *link; void __iomem *ahb_base; void __iomem *link_base; + void __iomem *mst2link_base; + void __iomem *mst3link_base; struct phy *phy; @@ -127,7 +144,8 @@ struct msm_dp_ctrl_private { unsigned int num_link_clks; struct clk_bulk_data *link_clks; - struct clk *pixel_clk; + struct clk *pixel_clk[DP_STREAM_MAX]; + unsigned int num_pixel_clks; union phy_configure_opts phy_opts; @@ -139,7 +157,10 @@ struct msm_dp_ctrl_private { bool core_clks_on; bool link_clks_on; - bool stream_clks_on; + bool stream_clks_on[DP_STREAM_MAX]; + bool mst_active; + + struct msm_dp_mst_ch_slot_info mst_ch_info[DP_STREAM_MAX]; }; static inline u32 msm_dp_read_ahb(const struct msm_dp_ctrl_private *ctrl, u32 offset) @@ -157,40 +178,180 @@ static inline void msm_dp_write_ahb(struct msm_dp_ctrl_private *ctrl, writel(data, ctrl->ahb_base + offset); } -static inline u32 msm_dp_read_link(struct msm_dp_ctrl_private *ctrl, u32 offset) +static inline u32 msm_dp_read_link(struct msm_dp_ctrl_private *ctrl, + enum msm_dp_stream_id stream_id, u32 offset) { - return readl_relaxed(ctrl->link_base + offset); + switch (stream_id) { + case DP_STREAM_0: + case DP_STREAM_1: + return readl_relaxed(ctrl->link_base + offset); + case DP_STREAM_2: + return readl_relaxed(ctrl->mst2link_base + offset); + case DP_STREAM_3: + return readl_relaxed(ctrl->mst3link_base + offset); + default: + DRM_ERROR("error stream_id\n"); + return 0; + } } static inline void msm_dp_write_link(struct msm_dp_ctrl_private *ctrl, - u32 offset, u32 data) + enum msm_dp_stream_id stream_id, u32 offset, u32 data) { /* * To make sure link reg writes happens before any other operation, * this function uses writel() instread of writel_relaxed() */ - writel(data, ctrl->link_base + offset); + switch (stream_id) { + case DP_STREAM_0: + case DP_STREAM_1: + writel(data, ctrl->link_base + offset); + break; + case DP_STREAM_2: + writel(data, ctrl->mst2link_base + offset); + break; + case DP_STREAM_3: + writel(data, ctrl->mst3link_base + offset); + break; + default: + DRM_ERROR("error stream_id\n"); + break; + } } static int msm_dp_aux_link_configure(struct drm_dp_aux *aux, struct msm_dp_link_info *link) { - u8 values[2]; + u8 lane_count, bw_code; int err; - values[0] = drm_dp_link_rate_to_bw_code(link->rate); - values[1] = link->num_lanes; + lane_count = link->num_lanes; if (link->capabilities & DP_LINK_CAP_ENHANCED_FRAMING) - values[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN; + lane_count |= DP_LANE_COUNT_ENHANCED_FRAME_EN; - err = drm_dp_dpcd_write(aux, DP_LINK_BW_SET, values, sizeof(values)); + err = drm_dp_dpcd_writeb(aux, DP_LANE_COUNT_SET, lane_count); if (err < 0) return err; + if (link->use_rate_set) { + DRM_DEBUG_DP("using LINK_RATE_SET: 0x%02x", link->rate_set); + err = drm_dp_dpcd_writeb(aux, DP_LINK_RATE_SET, link->rate_set); + } else { + bw_code = drm_dp_link_rate_to_bw_code(link->rate); + DRM_DEBUG_DP("using LINK_BW_SET: 0x%02x", bw_code); + err = drm_dp_dpcd_writeb(aux, DP_LINK_BW_SET, bw_code); + } + + return err; +} + +int msm_dp_ctrl_mst_send_act(struct msm_dp_ctrl *msm_dp_ctrl) +{ + struct msm_dp_ctrl_private *ctrl; + bool act_complete; + + ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); + + if (!ctrl->mst_active) + return 0; + + msm_dp_write_link(ctrl, 0, REG_DP_MST_ACT, 0x1); + /* make sure ACT signal is performed */ + wmb(); + + msleep(20); /* needs 1 frame time */ + + act_complete = msm_dp_read_link(ctrl, 0, REG_DP_MST_ACT); + + if (!act_complete) { + drm_dbg_dp(ctrl->drm_dev, "MST ACT trigger complete failed\n"); + return 0; + } + return 0; } +static void msm_dp_ctrl_mst_config(struct msm_dp_ctrl_private *ctrl, bool enable) +{ + u32 mainlink_ctrl; + + mainlink_ctrl = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); + if (enable) + mainlink_ctrl |= DP_MAINLINK_CTRL_MST_EN; + else + mainlink_ctrl &= ~DP_MAINLINK_CTRL_MST_EN; + + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, mainlink_ctrl); +} + +static void msm_dp_ctrl_mst_channel_alloc(struct msm_dp_ctrl_private *ctrl, + enum msm_dp_stream_id stream_id, u32 ch_start_slot, + u32 tot_slot_cnt) +{ + u32 i, slot; + u32 slot_reg_1, slot_reg_2; + u32 reg_off = 0; + int const num_slots_per_reg = 32; + + if (ch_start_slot > DP_MAX_TIME_SLOTS || + (ch_start_slot + tot_slot_cnt > DP_MAX_TIME_SLOTS)) { + DRM_ERROR("invalid slots start %d, tot %d\n", + ch_start_slot, tot_slot_cnt); + return; + } + + drm_dbg_dp(ctrl->drm_dev, "stream_id %d, start_slot %d, tot_slot %d\n", + stream_id, ch_start_slot, tot_slot_cnt); + + if (stream_id == DP_STREAM_1) + reg_off = REG_DP_DP1_TIMESLOT_1_32 - REG_DP_DP0_TIMESLOT_1_32; + + slot_reg_1 = 0; + slot_reg_2 = 0; + + if (ch_start_slot && tot_slot_cnt) { + ch_start_slot--; + for (i = 0; i < tot_slot_cnt; i++) { + if (ch_start_slot < num_slots_per_reg) { + slot_reg_1 |= BIT(ch_start_slot); + } else { + slot = ch_start_slot - num_slots_per_reg; + slot_reg_2 |= BIT(slot); + } + ch_start_slot++; + } + } + + drm_dbg_dp(ctrl->drm_dev, "stream_id:%d slot_reg_1:%d, slot_reg_2:%d\n", stream_id, + slot_reg_1, slot_reg_2); + + msm_dp_write_link(ctrl, stream_id, stream_id > DP_STREAM_1 ? + REG_DP_MSTLINK_TIMESLOT_1_32 : REG_DP_DP0_TIMESLOT_1_32 + reg_off, + slot_reg_1); + msm_dp_write_link(ctrl, stream_id, stream_id > DP_STREAM_1 ? + REG_DP_MSTLINK_TIMESLOT_33_63 : REG_DP_DP0_TIMESLOT_33_63 + reg_off, + slot_reg_2); +} + +static void msm_dp_ctrl_update_rg(struct msm_dp_ctrl_private *ctrl, + enum msm_dp_stream_id stream_id, u32 x_int, u32 y_frac_enum) +{ + u32 rg, reg_off = 0; + + rg = y_frac_enum; + rg |= (x_int << 16); + + drm_dbg_dp(ctrl->drm_dev, "stream_id: %d x_int:%d y_frac_enum:%d rg:%d\n", + stream_id, x_int, y_frac_enum, rg); + + if (stream_id == DP_STREAM_1) + reg_off = REG_DP_DP1_RG - REG_DP_DP0_RG; + + msm_dp_write_link(ctrl, stream_id, stream_id > 1 ? + REG_DP_MSTLINK_DP_RG : REG_DP_DP0_RG + reg_off, rg); +} + /* * NOTE: resetting DP controller will also clear any pending HPD related interrupts */ @@ -253,6 +414,8 @@ void msm_dp_ctrl_enable_irq(struct msm_dp_ctrl *msm_dp_ctrl) DP_INTERRUPT_STATUS1_MASK); msm_dp_write_ahb(ctrl, REG_DP_INTR_STATUS2, DP_INTERRUPT_STATUS2_MASK); + msm_dp_write_ahb(ctrl, REG_DP_INTR_STATUS5, + DP_INTERRUPT_STATUS5_MASK); } void msm_dp_ctrl_disable_irq(struct msm_dp_ctrl *msm_dp_ctrl) @@ -262,6 +425,7 @@ void msm_dp_ctrl_disable_irq(struct msm_dp_ctrl *msm_dp_ctrl) msm_dp_write_ahb(ctrl, REG_DP_INTR_STATUS, 0x00); msm_dp_write_ahb(ctrl, REG_DP_INTR_STATUS2, 0x00); + msm_dp_write_ahb(ctrl, REG_DP_INTR_STATUS5, 0x00); } static u32 msm_dp_ctrl_get_psr_interrupt(struct msm_dp_ctrl_private *ctrl) @@ -281,22 +445,36 @@ static void msm_dp_ctrl_config_psr_interrupt(struct msm_dp_ctrl_private *ctrl) msm_dp_write_ahb(ctrl, REG_DP_INTR_MASK4, DP_INTERRUPT_MASK4); } +static u32 msm_dp_ctrl_get_mst_interrupt(struct msm_dp_ctrl_private *ctrl) +{ + u32 intr, intr_ack; + + intr = msm_dp_read_ahb(ctrl, REG_DP_INTR_STATUS5); + intr &= ~DP_INTERRUPT_STATUS5_MASK; + intr_ack = (intr & DP_INTERRUPT_STATUS5) + << DP_INTERRUPT_STATUS_ACK_SHIFT; + msm_dp_write_ahb(ctrl, REG_DP_INTR_STATUS5, + intr_ack | DP_INTERRUPT_STATUS5_MASK); + + return intr; +} + static void msm_dp_ctrl_psr_mainlink_enable(struct msm_dp_ctrl_private *ctrl) { u32 val; - val = msm_dp_read_link(ctrl, REG_DP_MAINLINK_CTRL); + val = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); val |= DP_MAINLINK_CTRL_ENABLE; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, val); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, val); } static void msm_dp_ctrl_psr_mainlink_disable(struct msm_dp_ctrl_private *ctrl) { u32 val; - val = msm_dp_read_link(ctrl, REG_DP_MAINLINK_CTRL); + val = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); val &= ~DP_MAINLINK_CTRL_ENABLE; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, val); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, val); } static void msm_dp_ctrl_mainlink_enable(struct msm_dp_ctrl_private *ctrl) @@ -305,21 +483,21 @@ static void msm_dp_ctrl_mainlink_enable(struct msm_dp_ctrl_private *ctrl) drm_dbg_dp(ctrl->drm_dev, "enable\n"); - mainlink_ctrl = msm_dp_read_link(ctrl, REG_DP_MAINLINK_CTRL); + mainlink_ctrl = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); mainlink_ctrl &= ~(DP_MAINLINK_CTRL_RESET | DP_MAINLINK_CTRL_ENABLE); - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, mainlink_ctrl); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, mainlink_ctrl); mainlink_ctrl |= DP_MAINLINK_CTRL_RESET; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, mainlink_ctrl); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, mainlink_ctrl); mainlink_ctrl &= ~DP_MAINLINK_CTRL_RESET; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, mainlink_ctrl); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, mainlink_ctrl); mainlink_ctrl |= (DP_MAINLINK_CTRL_ENABLE | DP_MAINLINK_FB_BOUNDARY_SEL); - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, mainlink_ctrl); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, mainlink_ctrl); } static void msm_dp_ctrl_mainlink_disable(struct msm_dp_ctrl_private *ctrl) @@ -328,23 +506,23 @@ static void msm_dp_ctrl_mainlink_disable(struct msm_dp_ctrl_private *ctrl) drm_dbg_dp(ctrl->drm_dev, "disable\n"); - mainlink_ctrl = msm_dp_read_link(ctrl, REG_DP_MAINLINK_CTRL); + mainlink_ctrl = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); mainlink_ctrl &= ~DP_MAINLINK_CTRL_ENABLE; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, mainlink_ctrl); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, mainlink_ctrl); } static void msm_dp_setup_peripheral_flush(struct msm_dp_ctrl_private *ctrl) { u32 mainlink_ctrl; - mainlink_ctrl = msm_dp_read_link(ctrl, REG_DP_MAINLINK_CTRL); + mainlink_ctrl = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); if (ctrl->hw_revision >= DP_HW_VERSION_1_2) mainlink_ctrl |= DP_MAINLINK_FLUSH_MODE_SDE_PERIPH_UPDATE; else mainlink_ctrl |= DP_MAINLINK_FLUSH_MODE_UPDATE_SDP; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, mainlink_ctrl); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, mainlink_ctrl); } static bool msm_dp_ctrl_mainlink_ready(struct msm_dp_ctrl_private *ctrl) @@ -364,14 +542,28 @@ static bool msm_dp_ctrl_mainlink_ready(struct msm_dp_ctrl_private *ctrl) return true; } -void msm_dp_ctrl_push_idle(struct msm_dp_ctrl *msm_dp_ctrl) +void msm_dp_ctrl_push_idle(struct msm_dp_ctrl *msm_dp_ctrl, struct msm_dp_panel *msm_dp_panel) { struct msm_dp_ctrl_private *ctrl; + u32 state = 0x0; ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); + if (!ctrl->mst_active) + state |= DP_STATE_CTRL_PUSH_IDLE; + else if (msm_dp_panel->stream_id == DP_STREAM_0) + state |= DP_DP0_PUSH_VCPF; + else if (msm_dp_panel->stream_id == DP_STREAM_1) + state |= DP_DP1_PUSH_VCPF; + else + state |= DP_MSTLINK_PUSH_VCPF; + reinit_completion(&ctrl->idle_comp); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, DP_STATE_CTRL_PUSH_IDLE); + + msm_dp_write_link(ctrl, msm_dp_panel->stream_id, + msm_dp_panel->stream_id > 1 ? + REG_DP_MSTLINK_STATE_CTRL : REG_DP_STATE_CTRL, + state); if (!wait_for_completion_timeout(&ctrl->idle_comp, IDLE_PATTERN_COMPLETION_TIMEOUT_JIFFIES)) @@ -380,26 +572,49 @@ void msm_dp_ctrl_push_idle(struct msm_dp_ctrl *msm_dp_ctrl) drm_dbg_dp(ctrl->drm_dev, "mainlink off\n"); } -static void msm_dp_ctrl_config_ctrl(struct msm_dp_ctrl_private *ctrl) +static void msm_dp_ctrl_config_ctrl_streams(struct msm_dp_ctrl_private *ctrl, + struct msm_dp_panel *msm_dp_panel) { u32 config = 0, tbd; + u32 reg_offset = 0; + + if (msm_dp_panel->stream_id == DP_STREAM_0) + config = msm_dp_read_link(ctrl, 0, REG_DP_CONFIGURATION_CTRL); + + if (msm_dp_panel->stream_id == DP_STREAM_1) + reg_offset = REG_DP1_CONFIGURATION_CTRL - REG_DP_CONFIGURATION_CTRL; + + if (msm_dp_panel->msm_dp_mode.out_fmt_is_yuv_420) + config |= DP_CONFIGURATION_CTRL_RGB_YUV; /* YUV420 */ + + tbd = msm_dp_link_get_test_bits_depth(ctrl->link, + msm_dp_panel->msm_dp_mode.bpp); + + config |= tbd << DP_CONFIGURATION_CTRL_BPC_SHIFT; + + if (msm_dp_panel->psr_cap.version) + config |= DP_CONFIGURATION_CTRL_SEND_VSC; + + drm_dbg_dp(ctrl->drm_dev, "stream DP_CONFIGURATION_CTRL=0x%x\n", config); + + msm_dp_write_link(ctrl, msm_dp_panel->stream_id, msm_dp_panel->stream_id > 1 ? + REG_DP_MSTLINK_CONFIGURATION_CTRL : + REG_DP_CONFIGURATION_CTRL + reg_offset, config); + +} + +static void msm_dp_ctrl_config_ctrl_link(struct msm_dp_ctrl_private *ctrl) +{ + u32 config = 0; const u8 *dpcd = ctrl->panel->dpcd; /* Default-> LSCLK DIV: 1/4 LCLK */ config |= (2 << DP_CONFIGURATION_CTRL_LSCLK_DIV_SHIFT); - if (ctrl->panel->msm_dp_mode.out_fmt_is_yuv_420) - config |= DP_CONFIGURATION_CTRL_RGB_YUV; /* YUV420 */ - /* Scrambler reset enable */ if (drm_dp_alternate_scrambler_reset_cap(dpcd)) config |= DP_CONFIGURATION_CTRL_ASSR; - tbd = msm_dp_link_get_test_bits_depth(ctrl->link, - ctrl->panel->msm_dp_mode.bpp); - - config |= tbd << DP_CONFIGURATION_CTRL_BPC_SHIFT; - /* Num of Lanes */ config |= ((ctrl->link->link_params.num_lanes - 1) << DP_CONFIGURATION_CTRL_NUM_OF_LANES_SHIFT); @@ -413,41 +628,40 @@ static void msm_dp_ctrl_config_ctrl(struct msm_dp_ctrl_private *ctrl) config |= DP_CONFIGURATION_CTRL_STATIC_DYNAMIC_CN; config |= DP_CONFIGURATION_CTRL_SYNC_ASYNC_CLK; - if (ctrl->panel->psr_cap.version) - config |= DP_CONFIGURATION_CTRL_SEND_VSC; - - drm_dbg_dp(ctrl->drm_dev, "DP_CONFIGURATION_CTRL=0x%x\n", config); + drm_dbg_dp(ctrl->drm_dev, "link DP_CONFIGURATION_CTRL=0x%x\n", config); - msm_dp_write_link(ctrl, REG_DP_CONFIGURATION_CTRL, config); + msm_dp_write_link(ctrl, 0, REG_DP_CONFIGURATION_CTRL, config); } static void msm_dp_ctrl_lane_mapping(struct msm_dp_ctrl_private *ctrl) { - u32 ln_0 = 0, ln_1 = 1, ln_2 = 2, ln_3 = 3; /* One-to-One mapping */ + u32 *lane_map = ctrl->link->lane_map; u32 ln_mapping; - ln_mapping = ln_0 << LANE0_MAPPING_SHIFT; - ln_mapping |= ln_1 << LANE1_MAPPING_SHIFT; - ln_mapping |= ln_2 << LANE2_MAPPING_SHIFT; - ln_mapping |= ln_3 << LANE3_MAPPING_SHIFT; + ln_mapping = lane_map[0] << LANE0_MAPPING_SHIFT; + ln_mapping |= lane_map[1] << LANE1_MAPPING_SHIFT; + ln_mapping |= lane_map[2] << LANE2_MAPPING_SHIFT; + ln_mapping |= lane_map[3] << LANE3_MAPPING_SHIFT; - msm_dp_write_link(ctrl, REG_DP_LOGICAL2PHYSICAL_LANE_MAPPING, - ln_mapping); + msm_dp_write_link(ctrl, 0, REG_DP_LOGICAL2PHYSICAL_LANE_MAPPING, + ln_mapping); } -static void msm_dp_ctrl_configure_source_params(struct msm_dp_ctrl_private *ctrl) +static void msm_dp_ctrl_config_misc1_misc0(struct msm_dp_ctrl_private *ctrl, + struct msm_dp_panel *msm_dp_panel) { u32 colorimetry_cfg, test_bits_depth, misc_val; + u32 reg_offset = 0; - msm_dp_ctrl_lane_mapping(ctrl); - msm_dp_setup_peripheral_flush(ctrl); - - msm_dp_ctrl_config_ctrl(ctrl); - - test_bits_depth = msm_dp_link_get_test_bits_depth(ctrl->link, ctrl->panel->msm_dp_mode.bpp); + test_bits_depth = msm_dp_link_get_test_bits_depth(ctrl->link, + msm_dp_panel->msm_dp_mode.bpp); colorimetry_cfg = msm_dp_link_get_colorimetry_config(ctrl->link); - misc_val = msm_dp_read_link(ctrl, REG_DP_MISC1_MISC0); + if (msm_dp_panel->stream_id == DP_STREAM_1) + reg_offset = REG_DP1_MISC1_MISC0 - REG_DP_MISC1_MISC0; + + misc_val = msm_dp_read_link(ctrl, msm_dp_panel->stream_id, msm_dp_panel->stream_id > 1 ? + REG_DP_MSTLINK_MISC1_MISC0 : REG_DP_MISC1_MISC0 + reg_offset); /* clear bpp bits */ misc_val &= ~(0x07 << DP_MISC0_TEST_BITS_DEPTH_SHIFT); @@ -457,9 +671,22 @@ static void msm_dp_ctrl_configure_source_params(struct msm_dp_ctrl_private *ctrl misc_val |= DP_MISC0_SYNCHRONOUS_CLK; drm_dbg_dp(ctrl->drm_dev, "misc settings = 0x%x\n", misc_val); - msm_dp_write_link(ctrl, REG_DP_MISC1_MISC0, misc_val); + msm_dp_write_link(ctrl, msm_dp_panel->stream_id, + msm_dp_panel->stream_id > 1 ? + REG_DP_MSTLINK_MISC1_MISC0 : REG_DP_MISC1_MISC0 + reg_offset, + misc_val); +} + +static void msm_dp_ctrl_configure_source_params(struct msm_dp_ctrl_private *ctrl, + struct msm_dp_panel *msm_dp_panel) +{ + msm_dp_ctrl_config_ctrl_streams(ctrl, msm_dp_panel); + + msm_dp_ctrl_config_misc1_misc0(ctrl, msm_dp_panel); + + msm_dp_panel_timing_cfg(msm_dp_panel, ctrl->msm_dp_ctrl.wide_bus_en); - msm_dp_panel_timing_cfg(ctrl->panel, ctrl->msm_dp_ctrl.wide_bus_en); + msm_dp_panel_mst_async_fifo(msm_dp_panel, ctrl->mst_active); } /* @@ -1275,9 +1502,9 @@ static void msm_dp_ctrl_setup_tr_unit(struct msm_dp_ctrl_private *ctrl) pr_debug("dp_tu=0x%x, valid_boundary=0x%x, valid_boundary2=0x%x\n", msm_dp_tu, valid_boundary, valid_boundary2); - msm_dp_write_link(ctrl, REG_DP_VALID_BOUNDARY, valid_boundary); - msm_dp_write_link(ctrl, REG_DP_TU, msm_dp_tu); - msm_dp_write_link(ctrl, REG_DP_VALID_BOUNDARY_2, valid_boundary2); + msm_dp_write_link(ctrl, 0, REG_DP_VALID_BOUNDARY, valid_boundary); + msm_dp_write_link(ctrl, 0, REG_DP_TU, msm_dp_tu); + msm_dp_write_link(ctrl, 0, REG_DP_VALID_BOUNDARY_2, valid_boundary2); } static int msm_dp_ctrl_wait4video_ready(struct msm_dp_ctrl_private *ctrl) @@ -1394,7 +1621,7 @@ static int msm_dp_ctrl_set_pattern_state_bit(struct msm_dp_ctrl_private *ctrl, bit = BIT(state_bit - 1); drm_dbg_dp(ctrl->drm_dev, "hw: bit=%d train=%d\n", bit, state_bit); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, bit); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, bit); bit = BIT(state_bit - 1) << DP_MAINLINK_READY_LINK_TRAINING_SHIFT; @@ -1421,7 +1648,7 @@ static int msm_dp_ctrl_link_train_1(struct msm_dp_ctrl_private *ctrl, delay_us = drm_dp_read_clock_recovery_delay(ctrl->aux, ctrl->panel->dpcd, dp_phy, false); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, 0); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, 0); *training_step = DP_TRAINING_1; @@ -1474,26 +1701,32 @@ static int msm_dp_ctrl_link_train_1(struct msm_dp_ctrl_private *ctrl, static int msm_dp_ctrl_link_rate_down_shift(struct msm_dp_ctrl_private *ctrl) { int ret = 0; + struct msm_dp_link_info *link_params = &ctrl->link->link_params; - switch (ctrl->link->link_params.rate) { - case 810000: - ctrl->link->link_params.rate = 540000; - break; - case 540000: - ctrl->link->link_params.rate = 270000; - break; - case 270000: - ctrl->link->link_params.rate = 162000; - break; - case 162000: - default: - ret = -EINVAL; - break; + if (link_params->rate_set) { + --link_params->rate_set; + link_params->rate = link_params->supported_rates[link_params->rate_set]; + } else { + switch (link_params->rate) { + case 810000: + link_params->rate = 540000; + break; + case 540000: + link_params->rate = 270000; + break; + case 270000: + link_params->rate = 162000; + break; + case 162000: + default: + ret = -EINVAL; + break; + } } if (!ret) { drm_dbg_dp(ctrl->drm_dev, "new rate=0x%x\n", - ctrl->link->link_params.rate); + link_params->rate); } return ret; @@ -1539,7 +1772,7 @@ static int msm_dp_ctrl_link_train_2(struct msm_dp_ctrl_private *ctrl, delay_us = drm_dp_read_channel_eq_delay(ctrl->aux, ctrl->panel->dpcd, dp_phy, false); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, 0); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, 0); *training_step = DP_TRAINING_2; @@ -1614,7 +1847,7 @@ static int msm_dp_ctrl_link_train(struct msm_dp_ctrl_private *ctrl, u8 assr; struct msm_dp_link_info link_info = {0}; - msm_dp_ctrl_config_ctrl(ctrl); + msm_dp_ctrl_config_ctrl_link(ctrl); link_info.num_lanes = ctrl->link->link_params.num_lanes; link_info.rate = ctrl->link->link_params.rate; @@ -1656,7 +1889,7 @@ static int msm_dp_ctrl_link_train(struct msm_dp_ctrl_private *ctrl, } end: - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, 0); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, 0); return ret; } @@ -1802,34 +2035,34 @@ static int msm_dp_ctrl_enable_mainlink_clocks(struct msm_dp_ctrl_private *ctrl) static void msm_dp_ctrl_enable_sdp(struct msm_dp_ctrl_private *ctrl) { /* trigger sdp */ - msm_dp_write_link(ctrl, MMSS_DP_SDP_CFG3, UPDATE_SDP); - msm_dp_write_link(ctrl, MMSS_DP_SDP_CFG3, 0x0); + msm_dp_write_link(ctrl, 0, MMSS_DP_SDP_CFG3, UPDATE_SDP); + msm_dp_write_link(ctrl, 0, MMSS_DP_SDP_CFG3, 0x0); } static void msm_dp_ctrl_psr_enter(struct msm_dp_ctrl_private *ctrl) { u32 cmd; - cmd = msm_dp_read_link(ctrl, REG_PSR_CMD); + cmd = msm_dp_read_link(ctrl, 0, REG_PSR_CMD); cmd &= ~(PSR_ENTER | PSR_EXIT); cmd |= PSR_ENTER; msm_dp_ctrl_enable_sdp(ctrl); - msm_dp_write_link(ctrl, REG_PSR_CMD, cmd); + msm_dp_write_link(ctrl, 0, REG_PSR_CMD, cmd); } static void msm_dp_ctrl_psr_exit(struct msm_dp_ctrl_private *ctrl) { u32 cmd; - cmd = msm_dp_read_link(ctrl, REG_PSR_CMD); + cmd = msm_dp_read_link(ctrl, 0, REG_PSR_CMD); cmd &= ~(PSR_ENTER | PSR_EXIT); cmd |= PSR_EXIT; msm_dp_ctrl_enable_sdp(ctrl); - msm_dp_write_link(ctrl, REG_PSR_CMD, cmd); + msm_dp_write_link(ctrl, 0, REG_PSR_CMD, cmd); } void msm_dp_ctrl_config_psr(struct msm_dp_ctrl *msm_dp_ctrl) @@ -1842,9 +2075,9 @@ void msm_dp_ctrl_config_psr(struct msm_dp_ctrl *msm_dp_ctrl) return; /* enable PSR1 function */ - cfg = msm_dp_read_link(ctrl, REG_PSR_CONFIG); + cfg = msm_dp_read_link(ctrl, 0, REG_PSR_CONFIG); cfg |= PSR1_SUPPORTED; - msm_dp_write_link(ctrl, REG_PSR_CONFIG, cfg); + msm_dp_write_link(ctrl, 0, REG_PSR_CONFIG, cfg); msm_dp_ctrl_config_psr_interrupt(ctrl); msm_dp_ctrl_enable_sdp(ctrl); @@ -1882,17 +2115,17 @@ void msm_dp_ctrl_set_psr(struct msm_dp_ctrl *msm_dp_ctrl, bool enter) return; } - msm_dp_ctrl_push_idle(msm_dp_ctrl); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, 0); + msm_dp_ctrl_push_idle(msm_dp_ctrl, ctrl->panel); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, 0); msm_dp_ctrl_psr_mainlink_disable(ctrl); } else { msm_dp_ctrl_psr_mainlink_enable(ctrl); msm_dp_ctrl_psr_exit(ctrl); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, DP_STATE_CTRL_SEND_VIDEO); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, DP_STATE_CTRL_SEND_VIDEO); msm_dp_ctrl_wait4video_ready(ctrl); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, 0); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, 0); } } @@ -1992,7 +2225,7 @@ static int msm_dp_ctrl_link_maintenance(struct msm_dp_ctrl_private *ctrl) int ret = 0; int training_step = DP_TRAINING_NONE; - msm_dp_ctrl_push_idle(&ctrl->msm_dp_ctrl); + msm_dp_ctrl_push_idle(&ctrl->msm_dp_ctrl, ctrl->panel); ctrl->link->phy_params.p_level = 0; ctrl->link->phy_params.v_level = 0; @@ -2003,7 +2236,11 @@ static int msm_dp_ctrl_link_maintenance(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_clear_training_pattern(ctrl, DP_PHY_DPRX); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, DP_STATE_CTRL_SEND_VIDEO); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, DP_STATE_CTRL_SEND_VIDEO); + + ret = msm_dp_ctrl_mst_send_act(&ctrl->msm_dp_ctrl); + if (ret) + return ret; ret = msm_dp_ctrl_wait4video_ready(ctrl); end: @@ -2018,72 +2255,72 @@ static void msm_dp_ctrl_send_phy_pattern(struct msm_dp_ctrl_private *ctrl, u32 value = 0x0; /* Make sure to clear the current pattern before starting a new one */ - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, 0x0); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, 0x0); drm_dbg_dp(ctrl->drm_dev, "pattern: %#x\n", pattern); switch (pattern) { case DP_PHY_TEST_PATTERN_D10_2: - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, - DP_STATE_CTRL_LINK_TRAINING_PATTERN1); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, + DP_STATE_CTRL_LINK_TRAINING_PATTERN1); break; case DP_PHY_TEST_PATTERN_ERROR_COUNT: value &= ~(1 << 16); - msm_dp_write_link(ctrl, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, - value); + msm_dp_write_link(ctrl, 0, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, + value); value |= SCRAMBLER_RESET_COUNT_VALUE; - msm_dp_write_link(ctrl, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, - value); - msm_dp_write_link(ctrl, REG_DP_MAINLINK_LEVELS, - DP_MAINLINK_SAFE_TO_EXIT_LEVEL_2); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, - DP_STATE_CTRL_LINK_SYMBOL_ERR_MEASURE); + msm_dp_write_link(ctrl, 0, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, + value); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_LEVELS, + DP_MAINLINK_SAFE_TO_EXIT_LEVEL_2); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, + DP_STATE_CTRL_LINK_SYMBOL_ERR_MEASURE); break; case DP_PHY_TEST_PATTERN_PRBS7: - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, - DP_STATE_CTRL_LINK_PRBS7); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, + DP_STATE_CTRL_LINK_PRBS7); break; case DP_PHY_TEST_PATTERN_80BIT_CUSTOM: - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, - DP_STATE_CTRL_LINK_TEST_CUSTOM_PATTERN); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, + DP_STATE_CTRL_LINK_TEST_CUSTOM_PATTERN); /* 00111110000011111000001111100000 */ - msm_dp_write_link(ctrl, REG_DP_TEST_80BIT_CUSTOM_PATTERN_REG0, - 0x3E0F83E0); + msm_dp_write_link(ctrl, 0, REG_DP_TEST_80BIT_CUSTOM_PATTERN_REG0, + 0x3E0F83E0); /* 00001111100000111110000011111000 */ - msm_dp_write_link(ctrl, REG_DP_TEST_80BIT_CUSTOM_PATTERN_REG1, - 0x0F83E0F8); + msm_dp_write_link(ctrl, 0, REG_DP_TEST_80BIT_CUSTOM_PATTERN_REG1, + 0x0F83E0F8); /* 1111100000111110 */ - msm_dp_write_link(ctrl, REG_DP_TEST_80BIT_CUSTOM_PATTERN_REG2, - 0x0000F83E); + msm_dp_write_link(ctrl, 0, REG_DP_TEST_80BIT_CUSTOM_PATTERN_REG2, + 0x0000F83E); break; case DP_PHY_TEST_PATTERN_CP2520: - value = msm_dp_read_link(ctrl, REG_DP_MAINLINK_CTRL); + value = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); value &= ~DP_MAINLINK_CTRL_SW_BYPASS_SCRAMBLER; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, value); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, value); value = DP_HBR2_ERM_PATTERN; - msm_dp_write_link(ctrl, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, - value); + msm_dp_write_link(ctrl, 0, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, + value); value |= SCRAMBLER_RESET_COUNT_VALUE; - msm_dp_write_link(ctrl, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, - value); - msm_dp_write_link(ctrl, REG_DP_MAINLINK_LEVELS, - DP_MAINLINK_SAFE_TO_EXIT_LEVEL_2); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, - DP_STATE_CTRL_LINK_SYMBOL_ERR_MEASURE); - value = msm_dp_read_link(ctrl, REG_DP_MAINLINK_CTRL); + msm_dp_write_link(ctrl, 0, REG_DP_HBR2_COMPLIANCE_SCRAMBLER_RESET, + value); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_LEVELS, + DP_MAINLINK_SAFE_TO_EXIT_LEVEL_2); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, + DP_STATE_CTRL_LINK_SYMBOL_ERR_MEASURE); + value = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_CTRL); value |= DP_MAINLINK_CTRL_ENABLE; - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, value); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, value); break; case DP_PHY_TEST_PATTERN_SEL_MASK: - msm_dp_write_link(ctrl, REG_DP_MAINLINK_CTRL, - DP_MAINLINK_CTRL_ENABLE); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, - DP_STATE_CTRL_LINK_TRAINING_PATTERN4); + msm_dp_write_link(ctrl, 0, REG_DP_MAINLINK_CTRL, + DP_MAINLINK_CTRL_ENABLE); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, + DP_STATE_CTRL_LINK_TRAINING_PATTERN4); break; default: @@ -2111,7 +2348,7 @@ static bool msm_dp_ctrl_send_phy_test_pattern(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_update_phy_vx_px(ctrl, DP_PHY_DPRX); msm_dp_link_send_test_response(ctrl->link); - pattern_sent = msm_dp_read_link(ctrl, REG_DP_MAINLINK_READY); + pattern_sent = msm_dp_read_link(ctrl, 0, REG_DP_MAINLINK_READY); switch (pattern_sent) { case MR_LINK_TRAINING1: @@ -2145,6 +2382,43 @@ static bool msm_dp_ctrl_send_phy_test_pattern(struct msm_dp_ctrl_private *ctrl) return success; } +static int msm_dp_ctrl_on_pixel_clk(struct msm_dp_ctrl_private *ctrl, unsigned long pixel_rate, + enum msm_dp_stream_id stream_id) +{ + int ret; + + ret = clk_set_rate(ctrl->pixel_clk[stream_id], pixel_rate * 1000); + if (ret) { + DRM_ERROR("Failed to set pixel clock rate. ret=%d\n", ret); + return ret; + } + + if (ctrl->stream_clks_on[stream_id]) { + drm_dbg_dp(ctrl->drm_dev, "pixel clks already enabled\n"); + } else { + ret = clk_prepare_enable(ctrl->pixel_clk[stream_id]); + if (ret) { + DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret); + return ret; + } + ctrl->stream_clks_on[stream_id] = true; + } + + return ret; +} + +void msm_dp_ctrl_off_pixel_clk(struct msm_dp_ctrl *msm_dp_ctrl, enum msm_dp_stream_id stream_id) +{ + struct msm_dp_ctrl_private *ctrl; + + ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); + + if (ctrl->stream_clks_on[stream_id]) { + clk_disable_unprepare(ctrl->pixel_clk[stream_id]); + ctrl->stream_clks_on[stream_id] = false; + } +} + static int msm_dp_ctrl_process_phy_test_request(struct msm_dp_ctrl_private *ctrl) { int ret; @@ -2161,31 +2435,17 @@ static int msm_dp_ctrl_process_phy_test_request(struct msm_dp_ctrl_private *ctrl * running. Add the global reset just before disabling the * link clocks and core clocks. */ - msm_dp_ctrl_off(&ctrl->msm_dp_ctrl); + msm_dp_ctrl_off_pixel_clk(&ctrl->msm_dp_ctrl, ctrl->panel->stream_id); + msm_dp_ctrl_off_link(&ctrl->msm_dp_ctrl); - ret = msm_dp_ctrl_on_link(&ctrl->msm_dp_ctrl); + ret = msm_dp_ctrl_on_link(&ctrl->msm_dp_ctrl, false); if (ret) { DRM_ERROR("failed to enable DP link controller\n"); return ret; } pixel_rate = ctrl->panel->msm_dp_mode.drm_mode.clock; - ret = clk_set_rate(ctrl->pixel_clk, pixel_rate * 1000); - if (ret) { - DRM_ERROR("Failed to set pixel clock rate. ret=%d\n", ret); - return ret; - } - - if (ctrl->stream_clks_on) { - drm_dbg_dp(ctrl->drm_dev, "pixel clks already enabled\n"); - } else { - ret = clk_prepare_enable(ctrl->pixel_clk); - if (ret) { - DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret); - return ret; - } - ctrl->stream_clks_on = true; - } + ret = msm_dp_ctrl_on_pixel_clk(ctrl, pixel_rate, ctrl->panel->stream_id); msm_dp_ctrl_send_phy_test_pattern(ctrl); @@ -2258,7 +2518,7 @@ static bool msm_dp_ctrl_channel_eq_ok(struct msm_dp_ctrl_private *ctrl) return drm_dp_channel_eq_ok(link_status, num_lanes); } -int msm_dp_ctrl_on_link(struct msm_dp_ctrl *msm_dp_ctrl) +int msm_dp_ctrl_on_link(struct msm_dp_ctrl *msm_dp_ctrl, bool mst_active) { int rc = 0; struct msm_dp_ctrl_private *ctrl; @@ -2276,6 +2536,7 @@ int msm_dp_ctrl_on_link(struct msm_dp_ctrl *msm_dp_ctrl) rate = ctrl->panel->link_info.rate; pixel_rate = ctrl->panel->msm_dp_mode.drm_mode.clock; + ctrl->mst_active = mst_active; msm_dp_ctrl_core_clk_enable(&ctrl->msm_dp_ctrl); @@ -2391,24 +2652,43 @@ static int msm_dp_ctrl_link_retrain(struct msm_dp_ctrl_private *ctrl) } static void msm_dp_ctrl_config_msa(struct msm_dp_ctrl_private *ctrl, + struct msm_dp_panel *msm_dp_panel, u32 rate, u32 stream_rate_khz, bool is_ycbcr_420) { u32 pixel_m, pixel_n; - u32 mvid, nvid, pixel_div = 0, dispcc_input_rate; + u32 mvid, nvid, pixel_div, dispcc_input_rate; u32 const nvid_fixed = DP_LINK_CONSTANT_N_VALUE; u32 const link_rate_hbr2 = 540000; u32 const link_rate_hbr3 = 810000; unsigned long den, num; + u32 mvid_reg_off = 0, nvid_reg_off = 0; + + if (msm_dp_panel->stream_id == DP_STREAM_1) { + mvid_reg_off = REG_DP1_SOFTWARE_MVID - REG_DP_SOFTWARE_MVID; + nvid_reg_off = REG_DP1_SOFTWARE_NVID - REG_DP_SOFTWARE_NVID; + } - if (rate == link_rate_hbr3) + switch (rate) { + case link_rate_hbr3: pixel_div = 6; - else if (rate == 162000 || rate == 270000) - pixel_div = 2; - else if (rate == link_rate_hbr2) + break; + case link_rate_hbr2: pixel_div = 4; - else + break; + case 162000: + case 270000: + pixel_div = 2; + break; + default: + /* + * This cannot be reached but the compiler is not able to know + * that statically so return early to avoid a possibly invalid + * division. + */ DRM_ERROR("Invalid pixel mux divider\n"); + return; + } dispcc_input_rate = (rate * 10) / pixel_div; @@ -2442,59 +2722,136 @@ static void msm_dp_ctrl_config_msa(struct msm_dp_ctrl_private *ctrl, nvid *= 3; drm_dbg_dp(ctrl->drm_dev, "mvid=0x%x, nvid=0x%x\n", mvid, nvid); - msm_dp_write_link(ctrl, REG_DP_SOFTWARE_MVID, mvid); - msm_dp_write_link(ctrl, REG_DP_SOFTWARE_NVID, nvid); + msm_dp_write_link(ctrl, msm_dp_panel->stream_id, + msm_dp_panel->stream_id > 1 ? + REG_MSTLINK_SOFTWARE_MVID : REG_DP_SOFTWARE_MVID + mvid_reg_off, + mvid); + msm_dp_write_link(ctrl, msm_dp_panel->stream_id, + msm_dp_panel->stream_id > 1 ? + REG_MSTLINK_SOFTWARE_NVID : REG_DP_SOFTWARE_NVID + nvid_reg_off, + nvid); +} + +/* TODO: comments here. */ +static void msm_dp_ctrl_mst_calculate_rg(struct msm_dp_ctrl_private *ctrl, + struct msm_dp_panel *panel, + u32 *p_x_int, u32 *p_y_frac_enum) +{ + u64 min_slot_cnt, max_slot_cnt; + u64 raw_target_sc, target_sc_fixp; + u64 ts_denom, ts_enum, ts_int; + u64 pclk = panel->msm_dp_mode.drm_mode.clock; + u64 lclk = 0; + u64 lanes = ctrl->link->link_params.num_lanes; + u64 bpp = panel->msm_dp_mode.bpp; + u64 pbn = panel->pbn; + u64 numerator, denominator, temp, temp1, temp2; + u32 x_int = 0, y_frac_enum = 0; + u64 target_strm_sym, ts_int_fixp, ts_frac_fixp, y_frac_enum_fixp; + + lclk = ctrl->link->link_params.rate; + + /* min_slot_cnt */ + numerator = pclk * bpp * 64 * 1000; + denominator = lclk * lanes * 8 * 1000; + min_slot_cnt = drm_fixp_from_fraction(numerator, denominator); + + /* max_slot_cnt */ + numerator = pbn * 54 * 1000; + denominator = lclk * lanes; + max_slot_cnt = drm_fixp_from_fraction(numerator, denominator); + + /* raw_target_sc */ + numerator = max_slot_cnt + min_slot_cnt; + denominator = drm_fixp_from_fraction(2, 1); + raw_target_sc = drm_fixp_div(numerator, denominator); + + /* target_sc */ + temp = drm_fixp_from_fraction(256 * lanes, 1); + numerator = drm_fixp_mul(raw_target_sc, temp); + denominator = drm_fixp_from_fraction(256 * lanes, 1); + target_sc_fixp = drm_fixp_div(numerator, denominator); + + ts_enum = 256 * lanes; + ts_denom = drm_fixp_from_fraction(256 * lanes, 1); + ts_int = drm_fixp2int(target_sc_fixp); + + temp = drm_fixp2int_ceil(raw_target_sc); + if (temp != ts_int) { + temp = drm_fixp_from_fraction(ts_int, 1); + temp1 = raw_target_sc - temp; + temp2 = drm_fixp_mul(temp1, ts_denom); + ts_enum = drm_fixp2int(temp2); + } + + /* target_strm_sym */ + ts_int_fixp = drm_fixp_from_fraction(ts_int, 1); + ts_frac_fixp = drm_fixp_from_fraction(ts_enum, drm_fixp2int(ts_denom)); + temp = ts_int_fixp + ts_frac_fixp; + temp1 = drm_fixp_from_fraction(lanes, 1); + target_strm_sym = drm_fixp_mul(temp, temp1); + + /* x_int */ + x_int = drm_fixp2int(target_strm_sym); + + /* y_enum_frac */ + temp = drm_fixp_from_fraction(x_int, 1); + temp1 = target_strm_sym - temp; + temp2 = drm_fixp_from_fraction(256, 1); + y_frac_enum_fixp = drm_fixp_mul(temp1, temp2); + + temp1 = drm_fixp2int(y_frac_enum_fixp); + temp2 = drm_fixp2int_ceil(y_frac_enum_fixp); + + y_frac_enum = (u32)((temp1 == temp2) ? temp1 : temp1 + 1); + + *p_x_int = x_int; + *p_y_frac_enum = y_frac_enum; + + drm_dbg_dp(ctrl->drm_dev, "MST lane_cnt:%llu, rate:%llu x_int:%d, y_frac:%d\n", + lanes, lclk, x_int, y_frac_enum); } -int msm_dp_ctrl_on_stream(struct msm_dp_ctrl *msm_dp_ctrl, bool force_link_train) +static void msm_dp_ctrl_mst_stream_setup(struct msm_dp_ctrl_private *ctrl, + struct msm_dp_panel *panel) +{ + u32 x_int, y_frac_enum; + + if (!ctrl->mst_active) + return; + + drm_dbg_dp(ctrl->drm_dev, "MST stream channel allocation\n"); + + msm_dp_ctrl_mst_stream_channel_slot_setup(&ctrl->msm_dp_ctrl); + + msm_dp_ctrl_mst_calculate_rg(ctrl, panel, &x_int, &y_frac_enum); + + msm_dp_ctrl_update_rg(ctrl, panel->stream_id, x_int, y_frac_enum); +} + +int msm_dp_ctrl_prepare_stream_on(struct msm_dp_ctrl *msm_dp_ctrl, bool force_link_train) { int ret = 0; - bool mainlink_ready = false; struct msm_dp_ctrl_private *ctrl; - unsigned long pixel_rate; - unsigned long pixel_rate_orig; if (!msm_dp_ctrl) return -EINVAL; ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); - pixel_rate = pixel_rate_orig = ctrl->panel->msm_dp_mode.drm_mode.clock; + drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d\n", + ctrl->link->link_params.rate, + ctrl->link->link_params.num_lanes); - if (msm_dp_ctrl->wide_bus_en || ctrl->panel->msm_dp_mode.out_fmt_is_yuv_420) - pixel_rate >>= 1; - - drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%lu\n", - ctrl->link->link_params.rate, - ctrl->link->link_params.num_lanes, pixel_rate); - - drm_dbg_dp(ctrl->drm_dev, - "core_clk_on=%d link_clk_on=%d stream_clk_on=%d\n", - ctrl->core_clks_on, ctrl->link_clks_on, ctrl->stream_clks_on); + drm_dbg_dp(ctrl->drm_dev, "core_clk_on=%d link_clk_on=%d\n", + ctrl->core_clks_on, ctrl->link_clks_on); if (!ctrl->link_clks_on) { /* link clk is off */ ret = msm_dp_ctrl_enable_mainlink_clocks(ctrl); if (ret) { DRM_ERROR("Failed to start link clocks. ret=%d\n", ret); - goto end; - } - } - - ret = clk_set_rate(ctrl->pixel_clk, pixel_rate * 1000); - if (ret) { - DRM_ERROR("Failed to set pixel clock rate. ret=%d\n", ret); - goto end; - } - - if (ctrl->stream_clks_on) { - drm_dbg_dp(ctrl->drm_dev, "pixel clks already enabled\n"); - } else { - ret = clk_prepare_enable(ctrl->pixel_clk); - if (ret) { - DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret); - goto end; + return ret; } - ctrl->stream_clks_on = true; } if (force_link_train || !msm_dp_ctrl_channel_eq_ok(ctrl)) @@ -2503,24 +2860,68 @@ int msm_dp_ctrl_on_stream(struct msm_dp_ctrl *msm_dp_ctrl, bool force_link_train /* stop txing train pattern to end link training */ msm_dp_ctrl_clear_training_pattern(ctrl, DP_PHY_DPRX); + return ret; +} + +int msm_dp_ctrl_on_stream(struct msm_dp_ctrl *msm_dp_ctrl, struct msm_dp_panel *msm_dp_panel) +{ + int ret = 0; + bool mainlink_ready = false; + struct msm_dp_ctrl_private *ctrl; + unsigned long pixel_rate; + unsigned long pixel_rate_orig; + + if (!msm_dp_ctrl) + return -EINVAL; + + ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); + + pixel_rate_orig = msm_dp_panel->msm_dp_mode.drm_mode.clock; + pixel_rate = pixel_rate_orig; + + if (msm_dp_ctrl->wide_bus_en || msm_dp_panel->msm_dp_mode.out_fmt_is_yuv_420) + pixel_rate >>= 1; + + drm_dbg_dp(ctrl->drm_dev, "pixel_rate=%lu\n", pixel_rate); + + ret = msm_dp_ctrl_on_pixel_clk(ctrl, pixel_rate, msm_dp_panel->stream_id); + if (ret) + return ret; + /* * Set up transfer unit values and set controller state to send * video. */ reinit_completion(&ctrl->video_comp); - msm_dp_ctrl_configure_source_params(ctrl); + msm_dp_ctrl_lane_mapping(ctrl); + msm_dp_setup_peripheral_flush(ctrl); + if (ctrl->mst_active) + msm_dp_ctrl_mst_config(ctrl, true); + + if (msm_dp_panel->stream_id == DP_STREAM_0) + msm_dp_ctrl_config_ctrl_link(ctrl); + + msm_dp_ctrl_configure_source_params(ctrl, msm_dp_panel); msm_dp_ctrl_config_msa(ctrl, + msm_dp_panel, ctrl->link->link_params.rate, pixel_rate_orig, - ctrl->panel->msm_dp_mode.out_fmt_is_yuv_420); + msm_dp_panel->msm_dp_mode.out_fmt_is_yuv_420); + + msm_dp_panel_clear_dsc_dto(msm_dp_panel); - msm_dp_panel_clear_dsc_dto(ctrl->panel); + if (!ctrl->mst_active) + msm_dp_ctrl_setup_tr_unit(ctrl); - msm_dp_ctrl_setup_tr_unit(ctrl); + msm_dp_ctrl_mst_stream_setup(ctrl, msm_dp_panel); - msm_dp_write_link(ctrl, REG_DP_STATE_CTRL, DP_STATE_CTRL_SEND_VIDEO); + msm_dp_write_link(ctrl, 0, REG_DP_STATE_CTRL, DP_STATE_CTRL_SEND_VIDEO); + + ret = msm_dp_ctrl_mst_send_act(msm_dp_ctrl); + if (ret) + return ret; ret = msm_dp_ctrl_wait4video_ready(ctrl); if (ret) @@ -2530,11 +2931,10 @@ int msm_dp_ctrl_on_stream(struct msm_dp_ctrl *msm_dp_ctrl, bool force_link_train drm_dbg_dp(ctrl->drm_dev, "mainlink %s\n", mainlink_ready ? "READY" : "NOT READY"); -end: return ret; } -void msm_dp_ctrl_off_link_stream(struct msm_dp_ctrl *msm_dp_ctrl) +void msm_dp_ctrl_reinit_phy(struct msm_dp_ctrl *msm_dp_ctrl) { struct msm_dp_ctrl_private *ctrl; struct phy *phy; @@ -2542,23 +2942,6 @@ void msm_dp_ctrl_off_link_stream(struct msm_dp_ctrl *msm_dp_ctrl) ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); phy = ctrl->phy; - msm_dp_panel_disable_vsc_sdp(ctrl->panel); - - /* set dongle to D3 (power off) mode */ - msm_dp_link_psm_config(ctrl->link, &ctrl->panel->link_info, true); - - msm_dp_ctrl_mainlink_disable(ctrl); - - if (ctrl->stream_clks_on) { - clk_disable_unprepare(ctrl->pixel_clk); - ctrl->stream_clks_on = false; - } - - dev_pm_opp_set_rate(ctrl->dev, 0); - msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); - - phy_power_off(phy); - /* aux channel down, reinit phy */ phy_exit(phy); phy_init(phy); @@ -2576,44 +2959,51 @@ void msm_dp_ctrl_off_link(struct msm_dp_ctrl *msm_dp_ctrl) phy = ctrl->phy; msm_dp_ctrl_mainlink_disable(ctrl); + msm_dp_ctrl_mst_config(ctrl, false); + + msm_dp_ctrl_reset(&ctrl->msm_dp_ctrl); + + ctrl->mst_active = false; dev_pm_opp_set_rate(ctrl->dev, 0); msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); - DRM_DEBUG_DP("Before, phy=%p init_count=%d power_on=%d\n", - phy, phy->init_count, phy->power_count); - phy_power_off(phy); - - DRM_DEBUG_DP("After, phy=%p init_count=%d power_on=%d\n", - phy, phy->init_count, phy->power_count); + drm_dbg_dp(ctrl->drm_dev, "phy=%p init=%d power_on=%d\n", + phy, phy->init_count, phy->power_count); } -void msm_dp_ctrl_off(struct msm_dp_ctrl *msm_dp_ctrl) +void msm_dp_ctrl_set_mst_channel_info(struct msm_dp_ctrl *msm_dp_ctrl, + enum msm_dp_stream_id stream_id, + u32 start_slot, u32 tot_slots) { struct msm_dp_ctrl_private *ctrl; - struct phy *phy; - ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); - phy = ctrl->phy; + if (!msm_dp_ctrl || stream_id >= DP_STREAM_MAX) { + DRM_ERROR("invalid input\n"); + return; + } - msm_dp_panel_disable_vsc_sdp(ctrl->panel); + ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); - msm_dp_ctrl_mainlink_disable(ctrl); + ctrl->mst_ch_info[stream_id].start_slot = start_slot; + ctrl->mst_ch_info[stream_id].tot_slots = tot_slots; +} - msm_dp_ctrl_reset(&ctrl->msm_dp_ctrl); +void msm_dp_ctrl_mst_stream_channel_slot_setup(struct msm_dp_ctrl *msm_dp_ctrl) +{ + struct msm_dp_ctrl_private *ctrl; + int i; - if (ctrl->stream_clks_on) { - clk_disable_unprepare(ctrl->pixel_clk); - ctrl->stream_clks_on = false; - } + ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); - dev_pm_opp_set_rate(ctrl->dev, 0); - msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); + if (!ctrl->mst_active) + return; - phy_power_off(phy); - drm_dbg_dp(ctrl->drm_dev, "phy=%p init=%d power_on=%d\n", - phy, phy->init_count, phy->power_count); + for (i = DP_STREAM_0; i < ctrl->num_pixel_clks; i++) { + msm_dp_ctrl_mst_channel_alloc(ctrl, i, ctrl->mst_ch_info[i].start_slot, + ctrl->mst_ch_info[i].tot_slots); + } } irqreturn_t msm_dp_ctrl_isr(struct msm_dp_ctrl *msm_dp_ctrl) @@ -2657,6 +3047,13 @@ irqreturn_t msm_dp_ctrl_isr(struct msm_dp_ctrl *msm_dp_ctrl) ret = IRQ_HANDLED; } + isr = msm_dp_ctrl_get_mst_interrupt(ctrl); + if (isr & (DP_INTR_DP0_VCPF_SENT | DP_INTR_DP1_VCPF_SENT)) { + drm_dbg_dp(ctrl->drm_dev, "vcpf sent\n"); + complete(&ctrl->idle_comp); + ret = IRQ_HANDLED; + } + /* DP aux isr */ isr = msm_dp_ctrl_get_aux_interrupt(ctrl); if (isr) @@ -2675,7 +3072,14 @@ static const char *ctrl_clks[] = { "ctrl_link_iface", }; -static int msm_dp_ctrl_clk_init(struct msm_dp_ctrl *msm_dp_ctrl) +static const char * const pixel_clks[] = { + "stream_pixel", + "stream_1_pixel", + "stream_2_pixel", + "stream_3_pixel", +}; + +static int msm_dp_ctrl_clk_init(struct msm_dp_ctrl *msm_dp_ctrl, int max_stream) { struct msm_dp_ctrl_private *ctrl; struct device *dev; @@ -2708,18 +3112,41 @@ static int msm_dp_ctrl_clk_init(struct msm_dp_ctrl *msm_dp_ctrl) if (rc) return rc; - ctrl->pixel_clk = devm_clk_get(dev, "stream_pixel"); - if (IS_ERR(ctrl->pixel_clk)) - return PTR_ERR(ctrl->pixel_clk); + ctrl->num_pixel_clks = 0; + for (i = DP_STREAM_0; i < max_stream; i++) { + ctrl->pixel_clk[i] = devm_clk_get(dev, pixel_clks[i]); + + if (i == 0 && IS_ERR(ctrl->pixel_clk[i])) + return PTR_ERR(ctrl->pixel_clk[i]); + + if (IS_ERR(ctrl->pixel_clk[i])) { + DRM_DEBUG_DP("stream %d pixel clock not exist", i); + break; + } + + ctrl->num_pixel_clks++; + } return 0; } +int msm_dp_ctrl_get_stream_cnt(struct msm_dp_ctrl *msm_dp_ctrl) +{ + struct msm_dp_ctrl_private *ctrl; + + ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl); + + return ctrl->num_pixel_clks; +} + struct msm_dp_ctrl *msm_dp_ctrl_get(struct device *dev, struct msm_dp_link *link, struct msm_dp_panel *panel, struct drm_dp_aux *aux, struct phy *phy, + int max_stream, void __iomem *ahb_base, - void __iomem *link_base) + void __iomem *link_base, + void __iomem *mst2link_base, + void __iomem *mst3link_base) { struct msm_dp_ctrl_private *ctrl; int ret; @@ -2759,8 +3186,11 @@ struct msm_dp_ctrl *msm_dp_ctrl_get(struct device *dev, struct msm_dp_link *link ctrl->phy = phy; ctrl->ahb_base = ahb_base; ctrl->link_base = link_base; + ctrl->mst2link_base = mst2link_base; + ctrl->mst3link_base = mst3link_base; + ctrl->mst_active = false; - ret = msm_dp_ctrl_clk_init(&ctrl->msm_dp_ctrl); + ret = msm_dp_ctrl_clk_init(&ctrl->msm_dp_ctrl, max_stream); if (ret) { dev_err(dev, "failed to init clocks\n"); return ERR_PTR(ret); diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.h b/drivers/gpu/drm/msm/dp/dp_ctrl.h index 124b9b21bb7f2..cfe7e44969437 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.h +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.h @@ -16,12 +16,13 @@ struct msm_dp_ctrl { struct phy; -int msm_dp_ctrl_on_link(struct msm_dp_ctrl *msm_dp_ctrl); -int msm_dp_ctrl_on_stream(struct msm_dp_ctrl *msm_dp_ctrl, bool force_link_train); -void msm_dp_ctrl_off_link_stream(struct msm_dp_ctrl *msm_dp_ctrl); +int msm_dp_ctrl_on_link(struct msm_dp_ctrl *msm_dp_ctrl, bool mst_active); +int msm_dp_ctrl_on_stream(struct msm_dp_ctrl *msm_dp_ctrl, + struct msm_dp_panel *msm_dp_panel); +int msm_dp_ctrl_prepare_stream_on(struct msm_dp_ctrl *msm_dp_ctrl, bool force_link_train); void msm_dp_ctrl_off_link(struct msm_dp_ctrl *msm_dp_ctrl); -void msm_dp_ctrl_off(struct msm_dp_ctrl *msm_dp_ctrl); -void msm_dp_ctrl_push_idle(struct msm_dp_ctrl *msm_dp_ctrl); +void msm_dp_ctrl_off_pixel_clk(struct msm_dp_ctrl *msm_dp_ctrl, enum msm_dp_stream_id stream_id); +void msm_dp_ctrl_push_idle(struct msm_dp_ctrl *msm_dp_ctrl, struct msm_dp_panel *msm_dp_panel); irqreturn_t msm_dp_ctrl_isr(struct msm_dp_ctrl *msm_dp_ctrl); void msm_dp_ctrl_handle_sink_request(struct msm_dp_ctrl *msm_dp_ctrl); struct msm_dp_ctrl *msm_dp_ctrl_get(struct device *dev, @@ -29,8 +30,11 @@ struct msm_dp_ctrl *msm_dp_ctrl_get(struct device *dev, struct msm_dp_panel *panel, struct drm_dp_aux *aux, struct phy *phy, + int max_stream, void __iomem *ahb_base, - void __iomem *link_base); + void __iomem *link_base, + void __iomem *mst2link_base, + void __iomem *mst3link_base); void msm_dp_ctrl_reset(struct msm_dp_ctrl *msm_dp_ctrl); void msm_dp_ctrl_phy_init(struct msm_dp_ctrl *msm_dp_ctrl); @@ -46,4 +50,11 @@ void msm_dp_ctrl_core_clk_disable(struct msm_dp_ctrl *msm_dp_ctrl); void msm_dp_ctrl_enable_irq(struct msm_dp_ctrl *msm_dp_ctrl); void msm_dp_ctrl_disable_irq(struct msm_dp_ctrl *msm_dp_ctrl); +void msm_dp_ctrl_reinit_phy(struct msm_dp_ctrl *msm_dp_ctrl); +int msm_dp_ctrl_get_stream_cnt(struct msm_dp_ctrl *dp_ctrl); +int msm_dp_ctrl_mst_send_act(struct msm_dp_ctrl *msm_dp_ctrl); +void msm_dp_ctrl_mst_stream_channel_slot_setup(struct msm_dp_ctrl *msm_dp_ctrl); +void msm_dp_ctrl_set_mst_channel_info(struct msm_dp_ctrl *msm_dp_ctrl, + enum msm_dp_stream_id stream_id, + u32 start_slot, u32 tot_slots); #endif /* _DP_CTRL_H_ */ diff --git a/drivers/gpu/drm/msm/dp/dp_debug.h b/drivers/gpu/drm/msm/dp/dp_debug.h index 6dc0ff4f0f650..a90083fec856d 100644 --- a/drivers/gpu/drm/msm/dp/dp_debug.h +++ b/drivers/gpu/drm/msm/dp/dp_debug.h @@ -12,7 +12,7 @@ #if defined(CONFIG_DEBUG_FS) /** - * msm_dp_debug_get() - configure and get the DisplayPlot debug module data + * msm_dp_debug_init() - configure and get the DisplayPlot debug module data * * @dev: device instance of the caller * @panel: instance of panel module diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index d87d47cc7ec3e..77ea491efbd58 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "msm_drv.h" @@ -27,6 +28,7 @@ #include "dp_drm.h" #include "dp_audio.h" #include "dp_debug.h" +#include "dp_mst_drm.h" static bool psr_enabled = false; module_param(psr_enabled, bool, 0); @@ -38,40 +40,9 @@ enum { ISR_DISCONNECTED, ISR_CONNECT_PENDING, ISR_CONNECTED, - ISR_HPD_REPLUG_COUNT, + ISR_HPD_IO_GLITCH_COUNT, ISR_IRQ_HPD_PULSE_COUNT, - ISR_HPD_LO_GLITH_COUNT, -}; - -/* event thread connection state */ -enum { - ST_DISCONNECTED, - ST_MAINLINK_READY, - ST_CONNECTED, - ST_DISCONNECT_PENDING, - ST_DISPLAY_OFF, -}; - -enum { - EV_NO_EVENT, - /* hpd events */ - EV_HPD_PLUG_INT, - EV_IRQ_HPD_INT, - EV_HPD_UNPLUG_INT, - EV_USER_NOTIFICATION, -}; - -#define EVENT_TIMEOUT (HZ/10) /* 100ms */ -#define DP_EVENT_Q_MAX 8 - -#define DP_TIMEOUT_NONE 0 - -#define WAIT_FOR_RESUME_TIMEOUT_JIFFIES (HZ / 2) - -struct msm_dp_event { - u32 event_id; - u32 data; - u32 delay; + ISR_HPD_REPLUG_COUNT, }; struct msm_dp_display_private { @@ -84,6 +55,9 @@ struct msm_dp_display_private { bool phy_initialized; bool audio_supported; + struct mutex plugged_lock; + bool plugged; + struct drm_device *drm_dev; struct drm_dp_aux *aux; @@ -91,21 +65,14 @@ struct msm_dp_display_private { struct msm_dp_panel *panel; struct msm_dp_ctrl *ctrl; - struct msm_dp_display_mode msm_dp_mode; struct msm_dp msm_dp_display; /* wait for audio signaling */ struct completion audio_comp; - /* event related only access by event thread */ - struct mutex event_mutex; - wait_queue_head_t event_q; - u32 hpd_state; - u32 event_pndx; - u32 event_gndx; - struct task_struct *ev_tsk; - struct msm_dp_event event_list[DP_EVENT_Q_MAX]; - spinlock_t event_lock; + /* HPD IRQ handling */ + spinlock_t irq_thread_lock; + u32 hpd_isr_status; bool wide_bus_supported; @@ -120,21 +87,40 @@ struct msm_dp_display_private { void __iomem *link_base; size_t link_len; - void __iomem *p0_base; - size_t p0_len; + void __iomem *mst2link_base; + size_t mst2link_len; + + void __iomem *mst3link_base; + size_t mst3link_len; + + void __iomem *pixel_base[DP_STREAM_MAX]; + size_t pixel_len; + + int max_stream; }; struct msm_dp_desc { phys_addr_t io_start; unsigned int id; bool wide_bus_supported; + int mst_streams; }; -static const struct msm_dp_desc msm_dp_desc_sa8775p[] = { +static const struct msm_dp_desc msm_dp_desc_glymur[] = { { .io_start = 0x0af54000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true }, { .io_start = 0x0af5c000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true }, - { .io_start = 0x22154000, .id = MSM_DP_CONTROLLER_2, .wide_bus_supported = true }, - { .io_start = 0x2215c000, .id = MSM_DP_CONTROLLER_3, .wide_bus_supported = true }, + { .io_start = 0x0af64000, .id = MSM_DP_CONTROLLER_2, .wide_bus_supported = true }, + { .io_start = 0x0af6c000, .id = MSM_DP_CONTROLLER_3, .wide_bus_supported = true }, + {} +}; + +static const struct msm_dp_desc msm_dp_desc_sa8775p[] = { + { .io_start = 0x0af54000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true, + .mst_streams = 4}, + { .io_start = 0x0af5c000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true, + .mst_streams = 2}, + { .io_start = 0x22154000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true }, + { .io_start = 0x2215c000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true }, {} }; @@ -187,6 +173,7 @@ static const struct msm_dp_desc msm_dp_desc_x1e80100[] = { }; static const struct of_device_id msm_dp_dt_match[] = { + { .compatible = "qcom,glymur-dp", .data = &msm_dp_desc_glymur }, { .compatible = "qcom,sa8775p-dp", .data = &msm_dp_desc_sa8775p }, { .compatible = "qcom,sc7180-dp", .data = &msm_dp_desc_sc7180 }, { .compatible = "qcom,sc7280-dp", .data = &msm_dp_desc_sc7280 }, @@ -201,6 +188,7 @@ static const struct of_device_id msm_dp_dt_match[] = { { .compatible = "qcom,x1e80100-dp", .data = &msm_dp_desc_x1e80100 }, {} }; +MODULE_DEVICE_TABLE(of, msm_dp_dt_match); static struct msm_dp_display_private *dev_get_dp_display_private(struct device *dev) { @@ -209,60 +197,6 @@ static struct msm_dp_display_private *dev_get_dp_display_private(struct device * return container_of(dp, struct msm_dp_display_private, msm_dp_display); } -static int msm_dp_add_event(struct msm_dp_display_private *msm_dp_priv, u32 event, - u32 data, u32 delay) -{ - unsigned long flag; - struct msm_dp_event *todo; - int pndx; - - spin_lock_irqsave(&msm_dp_priv->event_lock, flag); - pndx = msm_dp_priv->event_pndx + 1; - pndx %= DP_EVENT_Q_MAX; - if (pndx == msm_dp_priv->event_gndx) { - pr_err("event_q is full: pndx=%d gndx=%d\n", - msm_dp_priv->event_pndx, msm_dp_priv->event_gndx); - spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag); - return -EPERM; - } - todo = &msm_dp_priv->event_list[msm_dp_priv->event_pndx++]; - msm_dp_priv->event_pndx %= DP_EVENT_Q_MAX; - todo->event_id = event; - todo->data = data; - todo->delay = delay; - wake_up(&msm_dp_priv->event_q); - spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag); - - return 0; -} - -static int msm_dp_del_event(struct msm_dp_display_private *msm_dp_priv, u32 event) -{ - unsigned long flag; - struct msm_dp_event *todo; - u32 gndx; - - spin_lock_irqsave(&msm_dp_priv->event_lock, flag); - if (msm_dp_priv->event_pndx == msm_dp_priv->event_gndx) { - spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag); - return -ENOENT; - } - - gndx = msm_dp_priv->event_gndx; - while (msm_dp_priv->event_pndx != gndx) { - todo = &msm_dp_priv->event_list[gndx]; - if (todo->event_id == event) { - todo->event_id = EV_NO_EVENT; /* deleted */ - todo->delay = 0; - } - gndx++; - gndx %= DP_EVENT_Q_MAX; - } - spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag); - - return 0; -} - void msm_dp_display_signal_audio_start(struct msm_dp *msm_dp_display) { struct msm_dp_display_private *dp; @@ -281,8 +215,6 @@ void msm_dp_display_signal_audio_complete(struct msm_dp *msm_dp_display) complete_all(&dp->audio_comp); } -static int msm_dp_hpd_event_thread_start(struct msm_dp_display_private *msm_dp_priv); - static int msm_dp_display_bind(struct device *dev, struct device *master, void *data) { @@ -302,12 +234,6 @@ static int msm_dp_display_bind(struct device *dev, struct device *master, goto end; } - rc = msm_dp_hpd_event_thread_start(dp); - if (rc) { - DRM_ERROR("Event thread create failed\n"); - goto end; - } - return 0; end: return rc; @@ -319,8 +245,6 @@ static void msm_dp_display_unbind(struct device *dev, struct device *master, struct msm_dp_display_private *dp = dev_get_dp_display_private(dev); struct msm_drm_private *priv = dev_get_drvdata(master); - kthread_stop(dp->ev_tsk); - of_dp_aux_depopulate_bus(dp->aux); msm_dp_aux_unregister(dp->aux); @@ -334,45 +258,6 @@ static const struct component_ops msm_dp_display_comp_ops = { .unbind = msm_dp_display_unbind, }; -static void msm_dp_display_send_hpd_event(struct msm_dp *msm_dp_display) -{ - struct msm_dp_display_private *dp; - struct drm_connector *connector; - - dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - - connector = dp->msm_dp_display.connector; - drm_helper_hpd_irq_event(connector->dev); -} - -static int msm_dp_display_send_hpd_notification(struct msm_dp_display_private *dp, - bool hpd) -{ - if ((hpd && dp->msm_dp_display.link_ready) || - (!hpd && !dp->msm_dp_display.link_ready)) { - drm_dbg_dp(dp->drm_dev, "HPD already %s\n", str_on_off(hpd)); - return 0; - } - - /* reset video pattern flag on disconnect */ - if (!hpd) { - dp->panel->video_test = false; - if (!dp->msm_dp_display.is_edp) - drm_dp_set_subconnector_property(dp->msm_dp_display.connector, - connector_status_disconnected, - dp->panel->dpcd, - dp->panel->downstream_ports); - } - - dp->msm_dp_display.link_ready = hpd; - - drm_dbg_dp(dp->drm_dev, "type=%d hpd=%d\n", - dp->msm_dp_display.connector_type, hpd); - msm_dp_display_send_hpd_event(&dp->msm_dp_display); - - return 0; -} - static int msm_dp_display_lttpr_init(struct msm_dp_display_private *dp, u8 *dpcd) { int rc, lttpr_count; @@ -390,12 +275,47 @@ static int msm_dp_display_lttpr_init(struct msm_dp_display_private *dp, u8 *dpcd return lttpr_count; } +static void msm_dp_display_mst_init(struct msm_dp_display_private *dp) +{ + const unsigned long clear_mstm_ctrl_timeout_us = 100000; + u8 old_mstm_ctrl; + struct msm_dp *msm_dp = &dp->msm_dp_display; + int ret; + + /* clear sink MST state */ + drm_dp_dpcd_read_byte(dp->aux, DP_MSTM_CTRL, &old_mstm_ctrl); + + ret = drm_dp_dpcd_write_byte(dp->aux, DP_MSTM_CTRL, 0); + if (ret < 0) { + DRM_ERROR("failed to clear DP_MSTM_CTRL, ret=%d\n", ret); + return; + } + + /* add extra delay if MST old state is on*/ + if (old_mstm_ctrl) { + drm_dbg_dp(dp->drm_dev, "wait %luus to set DP_MSTM_CTRL set 0\n", + clear_mstm_ctrl_timeout_us); + usleep_range(clear_mstm_ctrl_timeout_us, + clear_mstm_ctrl_timeout_us + 1000); + } + + ret = drm_dp_dpcd_write_byte(dp->aux, DP_MSTM_CTRL, + DP_MST_EN | DP_UP_REQ_EN | DP_UPSTREAM_IS_SRC); + if (ret < 0) { + DRM_ERROR("sink MST enablement failed\n"); + return; + } + + msm_dp->mst_active = true; +} + static int msm_dp_display_process_hpd_high(struct msm_dp_display_private *dp) { struct drm_connector *connector = dp->msm_dp_display.connector; const struct drm_display_info *info = &connector->display_info; int rc = 0; u8 dpcd[DP_RECEIVER_CAP_SIZE]; + const struct drm_edid *drm_edid; rc = drm_dp_read_dpcd_caps(dp->aux, dpcd); if (rc) @@ -403,10 +323,25 @@ static int msm_dp_display_process_hpd_high(struct msm_dp_display_private *dp) dp->link->lttpr_count = msm_dp_display_lttpr_init(dp, dpcd); - rc = msm_dp_panel_read_sink_caps(dp->panel, connector); + rc = msm_dp_panel_read_link_caps(dp->panel, connector); if (rc) goto end; + if (!(dp->max_stream > 1) || !drm_dp_read_mst_cap(dp->aux, dp->panel->dpcd)) { + drm_edid = drm_edid_read_ddc(connector, &dp->aux->ddc); + drm_edid_connector_update(connector, drm_edid); + + if (!drm_edid) { + DRM_ERROR("panel edid read failed\n"); + /* check edid read fail is due to unplug */ + if (!msm_dp_aux_is_link_connected(dp->aux)) + return -ETIMEDOUT; + } + + if (rc) + goto end; + } + msm_dp_link_process_request(dp->link); if (!dp->msm_dp_display.is_edp) @@ -418,7 +353,7 @@ static int msm_dp_display_process_hpd_high(struct msm_dp_display_private *dp) dp->msm_dp_display.psr_supported = dp->panel->psr_cap.version && psr_enabled; dp->audio_supported = info->has_audio; - msm_dp_panel_handle_sink_request(dp->panel); + msm_dp_panel_handle_sink_request(dp->panel, drm_edid); /* * set sink to normal operation mode -- D0 @@ -426,20 +361,19 @@ static int msm_dp_display_process_hpd_high(struct msm_dp_display_private *dp) */ msm_dp_link_psm_config(dp->link, &dp->panel->link_info, false); - msm_dp_link_reset_phy_params_vx_px(dp->link); - rc = msm_dp_ctrl_on_link(dp->ctrl); - if (rc) { - DRM_ERROR("failed to complete DP link training\n"); - goto end; - } + if (dp->max_stream > 1 && drm_dp_read_mst_cap(dp->aux, dp->panel->dpcd)) + msm_dp_display_mst_init(dp); + + if (dp->msm_dp_display.mst_active) + msm_dp_mst_display_set_mgr_state(&dp->msm_dp_display, true); - msm_dp_add_event(dp, EV_USER_NOTIFICATION, true, 0); + msm_dp_link_reset_phy_params_vx_px(dp->link); end: return rc; } -static void msm_dp_display_host_phy_init(struct msm_dp_display_private *dp) +static bool msm_dp_display_host_phy_init(struct msm_dp_display_private *dp) { drm_dbg_dp(dp->drm_dev, "type=%d core_init=%d phy_init=%d\n", dp->msm_dp_display.connector_type, dp->core_initialized, @@ -448,7 +382,10 @@ static void msm_dp_display_host_phy_init(struct msm_dp_display_private *dp) if (!dp->phy_initialized) { msm_dp_ctrl_phy_init(dp->ctrl); dp->phy_initialized = true; + return true; } + + return false; } static void msm_dp_display_host_phy_exit(struct msm_dp_display_private *dp) @@ -489,24 +426,6 @@ static void msm_dp_display_host_deinit(struct msm_dp_display_private *dp) dp->core_initialized = false; } -static int msm_dp_display_usbpd_configure_cb(struct device *dev) -{ - struct msm_dp_display_private *dp = dev_get_dp_display_private(dev); - - msm_dp_display_host_phy_init(dp); - - return msm_dp_display_process_hpd_high(dp); -} - -static int msm_dp_display_notify_disconnect(struct device *dev) -{ - struct msm_dp_display_private *dp = dev_get_dp_display_private(dev); - - msm_dp_add_event(dp, EV_USER_NOTIFICATION, false, 0); - - return 0; -} - static void msm_dp_display_handle_video_request(struct msm_dp_display_private *dp) { if (dp->link->sink_request & DP_TEST_LINK_VIDEO_PATTERN) { @@ -515,41 +434,11 @@ static void msm_dp_display_handle_video_request(struct msm_dp_display_private *d } } -static int msm_dp_display_handle_port_status_changed(struct msm_dp_display_private *dp) -{ - int rc = 0; - - if (drm_dp_is_branch(dp->panel->dpcd) && dp->link->sink_count == 0) { - drm_dbg_dp(dp->drm_dev, "sink count is zero, nothing to do\n"); - if (dp->hpd_state != ST_DISCONNECTED) { - dp->hpd_state = ST_DISCONNECT_PENDING; - msm_dp_add_event(dp, EV_USER_NOTIFICATION, false, 0); - } - } else { - if (dp->hpd_state == ST_DISCONNECTED) { - dp->hpd_state = ST_MAINLINK_READY; - rc = msm_dp_display_process_hpd_high(dp); - if (rc) - dp->hpd_state = ST_DISCONNECTED; - } - } - - return rc; -} - static int msm_dp_display_handle_irq_hpd(struct msm_dp_display_private *dp) { u32 sink_request = dp->link->sink_request; drm_dbg_dp(dp->drm_dev, "%d\n", sink_request); - if (dp->hpd_state == ST_DISCONNECTED) { - if (sink_request & DP_LINK_STATUS_UPDATED) { - drm_dbg_dp(dp->drm_dev, "Disconnected sink_request: %d\n", - sink_request); - DRM_ERROR("Disconnected, no DP_LINK_STATUS_UPDATED\n"); - return -EINVAL; - } - } msm_dp_ctrl_handle_sink_request(dp->ctrl); @@ -559,79 +448,42 @@ static int msm_dp_display_handle_irq_hpd(struct msm_dp_display_private *dp) return 0; } -static int msm_dp_display_usbpd_attention_cb(struct device *dev) +static int msm_dp_hpd_plug_handle(struct msm_dp_display_private *dp) { - int rc = 0; - u32 sink_request; - struct msm_dp_display_private *dp = dev_get_dp_display_private(dev); - - /* check for any test request issued by sink */ - rc = msm_dp_link_process_request(dp->link); - if (!rc) { - sink_request = dp->link->sink_request; - drm_dbg_dp(dp->drm_dev, "hpd_state=%d sink_request=%d\n", - dp->hpd_state, sink_request); - if (sink_request & DS_PORT_STATUS_CHANGED) - rc = msm_dp_display_handle_port_status_changed(dp); - else - rc = msm_dp_display_handle_irq_hpd(dp); - } - - return rc; -} - -static int msm_dp_hpd_plug_handle(struct msm_dp_display_private *dp, u32 data) -{ - u32 state; int ret; struct platform_device *pdev = dp->msm_dp_display.pdev; - msm_dp_aux_enable_xfers(dp->aux, true); - - mutex_lock(&dp->event_mutex); + drm_dbg_dp(dp->drm_dev, "Before, type=%d sink_count=%d\n", + dp->msm_dp_display.connector_type, + dp->link->sink_count); - state = dp->hpd_state; - drm_dbg_dp(dp->drm_dev, "Before, type=%d hpd_state=%d\n", - dp->msm_dp_display.connector_type, state); - - if (state == ST_DISPLAY_OFF) { - mutex_unlock(&dp->event_mutex); + if (dp->plugged && dp->msm_dp_display.mst_active) return 0; - } - if (state == ST_MAINLINK_READY || state == ST_CONNECTED) { - mutex_unlock(&dp->event_mutex); - return 0; - } - - if (state == ST_DISCONNECT_PENDING) { - /* wait until ST_DISCONNECTED */ - msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 1); /* delay = 1 */ - mutex_unlock(&dp->event_mutex); - return 0; - } + mutex_lock(&dp->plugged_lock); ret = pm_runtime_resume_and_get(&pdev->dev); if (ret) { + mutex_unlock(&dp->plugged_lock); DRM_ERROR("failed to pm_runtime_resume\n"); - mutex_unlock(&dp->event_mutex); return ret; } - ret = msm_dp_display_usbpd_configure_cb(&pdev->dev); - if (ret) { /* link train failed */ - dp->hpd_state = ST_DISCONNECTED; - pm_runtime_put_sync(&pdev->dev); - } else { - dp->hpd_state = ST_MAINLINK_READY; - } + msm_dp_aux_enable_xfers(dp->aux, true); + + msm_dp_display_host_phy_init(dp); - drm_dbg_dp(dp->drm_dev, "After, type=%d hpd_state=%d\n", - dp->msm_dp_display.connector_type, state); - mutex_unlock(&dp->event_mutex); + ret = msm_dp_display_process_hpd_high(dp); - /* uevent will complete connection part */ - return 0; + drm_dbg_dp(dp->drm_dev, "After, type=%d sink_count=%d\n", + dp->msm_dp_display.connector_type, + dp->link->sink_count); + + dp->plugged = true; + + mutex_unlock(&dp->plugged_lock); + + return ret; }; static void msm_dp_display_handle_plugged_change(struct msm_dp *msm_dp_display, @@ -648,105 +500,126 @@ static void msm_dp_display_handle_plugged_change(struct msm_dp *msm_dp_display, plugged); } -static int msm_dp_hpd_unplug_handle(struct msm_dp_display_private *dp, u32 data) +static int msm_dp_hpd_unplug_handle(struct msm_dp_display_private *dp) { - u32 state; struct platform_device *pdev = dp->msm_dp_display.pdev; + dp->panel->video_test = false; + msm_dp_aux_enable_xfers(dp->aux, false); - mutex_lock(&dp->event_mutex); + drm_dbg_dp(dp->drm_dev, "Before, type=%d sink_count=%d\n", + dp->msm_dp_display.connector_type, + dp->link->sink_count); - state = dp->hpd_state; + mutex_lock(&dp->plugged_lock); + if (!dp->plugged) { + mutex_unlock(&dp->plugged_lock); - drm_dbg_dp(dp->drm_dev, "Before, type=%d hpd_state=%d\n", - dp->msm_dp_display.connector_type, state); + return 0; + } - /* unplugged, no more irq_hpd handle */ - msm_dp_del_event(dp, EV_IRQ_HPD_INT); + /* Don't forget modes for eDP */ + if (!dp->msm_dp_display.is_edp) + drm_edid_connector_update(dp->msm_dp_display.connector, NULL); - if (state == ST_DISCONNECTED) { - /* triggered by irq_hdp with sink_count = 0 */ - if (dp->link->sink_count == 0) { - msm_dp_display_host_phy_exit(dp); - } - msm_dp_display_notify_disconnect(&dp->msm_dp_display.pdev->dev); - mutex_unlock(&dp->event_mutex); - return 0; - } else if (state == ST_DISCONNECT_PENDING) { - mutex_unlock(&dp->event_mutex); - return 0; - } else if (state == ST_MAINLINK_READY) { - msm_dp_ctrl_off_link(dp->ctrl); + /* triggered by irq_hdp with sink_count = 0 */ + if (dp->link->sink_count == 0) msm_dp_display_host_phy_exit(dp); - dp->hpd_state = ST_DISCONNECTED; - msm_dp_display_notify_disconnect(&dp->msm_dp_display.pdev->dev); - pm_runtime_put_sync(&pdev->dev); - mutex_unlock(&dp->event_mutex); - return 0; - } /* * We don't need separate work for disconnect as * connect/attention interrupts are disabled */ - msm_dp_display_notify_disconnect(&dp->msm_dp_display.pdev->dev); + if (!dp->msm_dp_display.is_edp) + drm_dp_set_subconnector_property(dp->msm_dp_display.connector, + connector_status_disconnected, + dp->panel->dpcd, + dp->panel->downstream_ports); - if (state == ST_DISPLAY_OFF) { - dp->hpd_state = ST_DISCONNECTED; - } else { - dp->hpd_state = ST_DISCONNECT_PENDING; + if (dp->msm_dp_display.mst_active) { + msm_dp_mst_display_set_mgr_state(&dp->msm_dp_display, false); + dp->msm_dp_display.mst_active = false; } /* signal the disconnect event early to ensure proper teardown */ msm_dp_display_handle_plugged_change(&dp->msm_dp_display, false); - drm_dbg_dp(dp->drm_dev, "After, type=%d hpd_state=%d\n", - dp->msm_dp_display.connector_type, state); + drm_dbg_dp(dp->drm_dev, "After, type=%d, sink_count=%d\n", + dp->msm_dp_display.connector_type, + dp->link->sink_count); + + if (dp->plugged) { + pm_runtime_put_sync(&pdev->dev); + dp->plugged = false; + } + mutex_unlock(&dp->plugged_lock); - /* uevent will complete disconnection part */ - pm_runtime_put_sync(&pdev->dev); - mutex_unlock(&dp->event_mutex); return 0; } -static int msm_dp_irq_hpd_handle(struct msm_dp_display_private *dp, u32 data) +static int msm_dp_irq_hpd_handle(struct msm_dp_display_private *dp) { - u32 state; - - mutex_lock(&dp->event_mutex); + u32 sink_request; + int rc = 0; + struct msm_dp *msm_dp_display = &dp->msm_dp_display; /* irq_hpd can happen at either connected or disconnected state */ - state = dp->hpd_state; - drm_dbg_dp(dp->drm_dev, "Before, type=%d hpd_state=%d\n", - dp->msm_dp_display.connector_type, state); + drm_dbg_dp(dp->drm_dev, "Before, type=%d, sink_count=%d\n", + dp->msm_dp_display.connector_type, + dp->link->sink_count); - if (state == ST_DISPLAY_OFF) { - mutex_unlock(&dp->event_mutex); + if (msm_dp_display->mst_active) { + if (msm_dp_aux_is_link_connected(dp->aux) != ISR_DISCONNECTED) + msm_dp_mst_display_hpd_irq(&dp->msm_dp_display); return 0; } - if (state == ST_MAINLINK_READY || state == ST_DISCONNECT_PENDING) { - /* wait until ST_CONNECTED */ - msm_dp_add_event(dp, EV_IRQ_HPD_INT, 0, 1); /* delay = 1 */ - mutex_unlock(&dp->event_mutex); - return 0; + /* check for any test request issued by sink */ + rc = msm_dp_link_process_request(dp->link); + if (!rc) { + sink_request = dp->link->sink_request; + drm_dbg_dp(dp->drm_dev, "sink_request=%d\n", sink_request); + if (sink_request & DS_PORT_STATUS_CHANGED) + rc = msm_dp_display_process_hpd_high(dp); + else + rc = msm_dp_display_handle_irq_hpd(dp); } - msm_dp_display_usbpd_attention_cb(&dp->msm_dp_display.pdev->dev); + drm_dbg_dp(dp->drm_dev, "After, type=%d, sink_count=%d\n", + dp->msm_dp_display.connector_type, + dp->link->sink_count); - drm_dbg_dp(dp->drm_dev, "After, type=%d hpd_state=%d\n", - dp->msm_dp_display.connector_type, state); + return rc; +} - mutex_unlock(&dp->event_mutex); +struct msm_dp_panel *msm_dp_display_get_panel(struct msm_dp *msm_dp_display) +{ + struct msm_dp_display_private *dp; + struct msm_dp_panel *dp_panel; - return 0; + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); + + dp_panel = msm_dp_panel_get(&dp->msm_dp_display.pdev->dev, dp->aux, dp->link, + dp->link_base, dp->mst2link_base, dp->mst3link_base, + dp->pixel_base[0]); + + if (IS_ERR(dp->panel)) { + DRM_ERROR("failed to initialize panel\n"); + return NULL; + } + + /* FIXME: move out of panel */ + memcpy(dp_panel->dpcd, dp->panel->dpcd, DP_RECEIVER_CAP_SIZE); + memcpy(&dp_panel->link_info, &dp->panel->link_info, + sizeof(dp->panel->link_info)); + + return dp_panel; } static void msm_dp_display_deinit_sub_modules(struct msm_dp_display_private *dp) { msm_dp_audio_put(dp->audio); - msm_dp_panel_put(dp->panel); msm_dp_aux_put(dp->aux); } @@ -783,7 +656,8 @@ static int msm_dp_init_sub_modules(struct msm_dp_display_private *dp) goto error_link; } - dp->panel = msm_dp_panel_get(dev, dp->aux, dp->link, dp->link_base, dp->p0_base); + dp->panel = msm_dp_panel_get(dev, dp->aux, dp->link, dp->link_base, + dp->mst2link_base, dp->mst3link_base, dp->pixel_base[0]); if (IS_ERR(dp->panel)) { rc = PTR_ERR(dp->panel); DRM_ERROR("failed to initialize panel, rc = %d\n", rc); @@ -792,26 +666,27 @@ static int msm_dp_init_sub_modules(struct msm_dp_display_private *dp) } dp->ctrl = msm_dp_ctrl_get(dev, dp->link, dp->panel, dp->aux, - phy, dp->ahb_base, dp->link_base); + phy, dp->max_stream, dp->ahb_base, + dp->link_base, dp->mst2link_base, dp->mst3link_base); if (IS_ERR(dp->ctrl)) { rc = PTR_ERR(dp->ctrl); DRM_ERROR("failed to initialize ctrl, rc = %d\n", rc); dp->ctrl = NULL; - goto error_ctrl; + goto error_link; } + if (dp->max_stream != msm_dp_ctrl_get_stream_cnt(dp->ctrl)) + dp->max_stream = 1; dp->audio = msm_dp_audio_get(dp->msm_dp_display.pdev, dp->link_base); if (IS_ERR(dp->audio)) { rc = PTR_ERR(dp->audio); pr_err("failed to initialize audio, rc = %d\n", rc); dp->audio = NULL; - goto error_ctrl; + goto error_link; } return rc; -error_ctrl: - msm_dp_panel_put(dp->panel); error_link: msm_dp_aux_put(dp->aux); error: @@ -819,34 +694,90 @@ static int msm_dp_init_sub_modules(struct msm_dp_display_private *dp) } static int msm_dp_display_set_mode(struct msm_dp *msm_dp_display, - struct msm_dp_display_mode *mode) + const struct drm_display_mode *adjusted_mode, + struct msm_dp_panel *msm_dp_panel) { struct msm_dp_display_private *dp; + u32 bpp; dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - drm_mode_copy(&dp->panel->msm_dp_mode.drm_mode, &mode->drm_mode); - dp->panel->msm_dp_mode.bpp = mode->bpp; - dp->panel->msm_dp_mode.out_fmt_is_yuv_420 = mode->out_fmt_is_yuv_420; - msm_dp_panel_init_panel_info(dp->panel); + drm_mode_copy(&msm_dp_panel->msm_dp_mode.drm_mode, adjusted_mode); + if (msm_dp_display_check_video_test(msm_dp_display)) + bpp = msm_dp_display_get_test_bpp(msm_dp_display); + else + bpp = msm_dp_panel->connector->display_info.bpc * 3; + + msm_dp_panel->msm_dp_mode.bpp = bpp ? bpp : 24; /* Default bpp */ + msm_dp_panel->msm_dp_mode.v_active_low = + !!(adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC); + msm_dp_panel->msm_dp_mode.h_active_low = + !!(adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC); + msm_dp_panel->msm_dp_mode.out_fmt_is_yuv_420 = + drm_mode_is_420_only(&msm_dp_panel->connector->display_info, adjusted_mode) && + msm_dp_panel->vsc_sdp_supported; + msm_dp_panel_init_panel_info(msm_dp_panel); + + /* populate wide_bus_support to different layers */ + dp->ctrl->wide_bus_en = + msm_dp_panel->msm_dp_mode.out_fmt_is_yuv_420 ? false : dp->wide_bus_supported; return 0; } -static int msm_dp_display_enable(struct msm_dp_display_private *dp, bool force_link_train) +int msm_dp_display_prepare(struct msm_dp *msm_dp_display) { + struct msm_dp_display_private *dp; int rc = 0; - struct msm_dp *msm_dp_display = &dp->msm_dp_display; + bool force_link_train = false; + + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); drm_dbg_dp(dp->drm_dev, "sink_count=%d\n", dp->link->sink_count); - if (msm_dp_display->power_on) { + + if (msm_dp_display->is_edp) + msm_dp_hpd_plug_handle(dp); + + if (msm_dp_display->prepared) { drm_dbg_dp(dp->drm_dev, "Link already setup, return\n"); return 0; } - rc = msm_dp_ctrl_on_stream(dp->ctrl, force_link_train); + rc = pm_runtime_resume_and_get(&msm_dp_display->pdev->dev); + if (rc) { + DRM_ERROR("failed to pm_runtime_resume\n"); + return rc; + } + + if (dp->link->sink_count == 0) + return rc; + + if (!msm_dp_display->active_stream_cnt) { + msm_dp_display_host_phy_init(dp); + force_link_train = true; + + rc = msm_dp_ctrl_on_link(dp->ctrl, msm_dp_display->mst_active); + if (rc) + DRM_ERROR("Failed link training (rc=%d)\n", rc); + // TODO: schedule drm_connector_set_link_status_property() + } + + rc = msm_dp_ctrl_prepare_stream_on(dp->ctrl, force_link_train); if (!rc) - msm_dp_display->power_on = true; + msm_dp_display->prepared = true; + + return rc; +} +static int msm_dp_display_enable(struct msm_dp_display_private *dp, + struct msm_dp_panel *msm_dp_panel) +{ + int rc = 0; + + drm_dbg_dp(dp->drm_dev, "sink_count=%d\n", dp->link->sink_count); + + rc = msm_dp_ctrl_on_stream(dp->ctrl, msm_dp_panel); + + dp->msm_dp_display.active_stream_cnt++; return rc; } @@ -873,13 +804,10 @@ static int msm_dp_display_post_enable(struct msm_dp *msm_dp_display) return 0; } -static int msm_dp_display_disable(struct msm_dp_display_private *dp) +static void msm_dp_display_audio_notify_disable(struct msm_dp_display_private *dp) { struct msm_dp *msm_dp_display = &dp->msm_dp_display; - if (!msm_dp_display->power_on) - return 0; - /* wait only if audio was enabled */ if (msm_dp_display->audio_enabled) { /* signal the disconnect event */ @@ -890,48 +818,71 @@ static int msm_dp_display_disable(struct msm_dp_display_private *dp) } msm_dp_display->audio_enabled = false; +} - if (dp->link->sink_count == 0) { - /* - * irq_hpd with sink_count = 0 - * hdmi unplugged out of dongle - */ - msm_dp_ctrl_off_link_stream(dp->ctrl); - } else { - /* - * unplugged interrupt - * dongle unplugged out of DUT - */ - msm_dp_ctrl_off(dp->ctrl); - msm_dp_display_host_phy_exit(dp); - } +static int msm_dp_display_disable(struct msm_dp_display_private *dp, + struct msm_dp_panel *msm_dp_panel) +{ + if (!dp->msm_dp_display.active_stream_cnt) + return 0; + + msm_dp_panel_disable_vsc_sdp(msm_dp_panel); - msm_dp_display->power_on = false; + msm_dp_ctrl_off_pixel_clk(dp->ctrl, msm_dp_panel->stream_id); + + dp->msm_dp_display.active_stream_cnt--; drm_dbg_dp(dp->drm_dev, "sink count: %d\n", dp->link->sink_count); return 0; } +int msm_dp_display_set_stream_info(struct msm_dp *msm_dp_display, struct msm_dp_panel *panel, + enum msm_dp_stream_id stream_id, u32 start_slot, + u32 num_slots, u32 pbn) +{ + int rc = 0; + struct msm_dp_display_private *dp; + const int max_slots = 64; + + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); + + if (!dp) { + DRM_ERROR("invalid input\n"); + return -EINVAL; + } + + if (start_slot + num_slots > max_slots) { + DRM_ERROR("invalid channel info received. start:%d, slots:%d\n", + start_slot, num_slots); + return -EINVAL; + } + + msm_dp_ctrl_set_mst_channel_info(dp->ctrl, stream_id, start_slot, num_slots); + + panel->stream_id = stream_id; + panel->pbn = pbn; + msm_dp_panel_set_pixel_base(panel, dp->pixel_base[stream_id]); + + return rc; +} + /** * msm_dp_bridge_mode_valid - callback to determine if specified mode is valid - * @bridge: Pointer to drm bridge structure + * @dp: Pointer to dp display structure * @info: display info * @mode: Pointer to drm mode structure * Returns: Validity status for specified mode */ -enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, - const struct drm_display_info *info, - const struct drm_display_mode *mode) +enum drm_mode_status msm_dp_display_mode_valid(struct msm_dp *dp, + const struct drm_display_info *info, + const struct drm_display_mode *mode) { const u32 num_components = 3, default_bpp = 24; struct msm_dp_display_private *msm_dp_display; struct msm_dp_link_info *link_info; u32 mode_rate_khz = 0, supported_rate_khz = 0, mode_bpp = 0; - struct msm_dp *dp; int mode_pclk_khz = mode->clock; - dp = to_dp_bridge(bridge)->msm_dp_display; - if (!dp || !mode_pclk_khz || !dp->connector) { DRM_ERROR("invalid params\n"); return -EINVAL; @@ -975,8 +926,7 @@ int msm_dp_display_get_modes(struct msm_dp *dp) msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); - return msm_dp_panel_get_modes(msm_dp_display->panel, - dp->connector); + return drm_edid_connector_add_modes(msm_dp_display->panel->connector); } bool msm_dp_display_check_video_test(struct msm_dp *dp) @@ -1016,12 +966,8 @@ void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp) * power_on status before dumping DP registers to avoid crash due * to unclocked access */ - mutex_lock(&msm_dp_display->event_mutex); - - if (!dp->power_on) { - mutex_unlock(&msm_dp_display->event_mutex); + if (!dp->active_stream_cnt) return; - } msm_disp_snapshot_add_block(disp_state, msm_dp_display->ahb_len, msm_dp_display->ahb_base, "dp_ahb"); @@ -1029,10 +975,18 @@ void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp) msm_dp_display->aux_base, "dp_aux"); msm_disp_snapshot_add_block(disp_state, msm_dp_display->link_len, msm_dp_display->link_base, "dp_link"); - msm_disp_snapshot_add_block(disp_state, msm_dp_display->p0_len, - msm_dp_display->p0_base, "dp_p0"); - - mutex_unlock(&msm_dp_display->event_mutex); + msm_disp_snapshot_add_block(disp_state, msm_dp_display->mst2link_len, + msm_dp_display->mst2link_base, "dp_mst2link"); + msm_disp_snapshot_add_block(disp_state, msm_dp_display->mst3link_len, + msm_dp_display->mst3link_base, "dp_mst3link"); + msm_disp_snapshot_add_block(disp_state, msm_dp_display->pixel_len, + msm_dp_display->pixel_base[0], "dp_p0"); + msm_disp_snapshot_add_block(disp_state, msm_dp_display->pixel_len, + msm_dp_display->pixel_base[1], "dp_p1"); + msm_disp_snapshot_add_block(disp_state, msm_dp_display->pixel_len, + msm_dp_display->pixel_base[2], "dp_p2"); + msm_disp_snapshot_add_block(disp_state, msm_dp_display->pixel_len, + msm_dp_display->pixel_base[3], "dp_p3"); } void msm_dp_display_set_psr(struct msm_dp *msm_dp_display, bool enter) @@ -1048,137 +1002,153 @@ void msm_dp_display_set_psr(struct msm_dp *msm_dp_display, bool enter) msm_dp_ctrl_set_psr(dp->ctrl, enter); } -static int hpd_event_thread(void *data) +/** + * msm_dp_bridge_detect - callback to determine if connector is connected + * @bridge: Pointer to drm bridge structure + * @connector: Pointer to drm connector structure + * Returns: Bridge's 'is connected' status + */ +enum drm_connector_status msm_dp_bridge_detect(struct drm_bridge *bridge, + struct drm_connector *connector) { - struct msm_dp_display_private *msm_dp_priv; - unsigned long flag; - struct msm_dp_event *todo; - int timeout_mode = 0; + struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(bridge); + struct msm_dp *dp = msm_dp_bridge->msm_dp_display; + struct msm_dp_display_private *priv; + int ret = 0; + int status = connector_status_disconnected; + u8 dpcd[DP_RECEIVER_CAP_SIZE]; + struct drm_dp_desc desc; + bool phy_deinit; - msm_dp_priv = (struct msm_dp_display_private *)data; + dp = to_dp_bridge(bridge)->msm_dp_display; - while (1) { - if (timeout_mode) { - wait_event_timeout(msm_dp_priv->event_q, - (msm_dp_priv->event_pndx == msm_dp_priv->event_gndx) || - kthread_should_stop(), EVENT_TIMEOUT); - } else { - wait_event_interruptible(msm_dp_priv->event_q, - (msm_dp_priv->event_pndx != msm_dp_priv->event_gndx) || - kthread_should_stop()); - } + priv = container_of(dp, struct msm_dp_display_private, msm_dp_display); - if (kthread_should_stop()) - break; + if (dp->mst_active) + return status; - spin_lock_irqsave(&msm_dp_priv->event_lock, flag); - todo = &msm_dp_priv->event_list[msm_dp_priv->event_gndx]; - if (todo->delay) { - struct msm_dp_event *todo_next; - - msm_dp_priv->event_gndx++; - msm_dp_priv->event_gndx %= DP_EVENT_Q_MAX; - - /* re enter delay event into q */ - todo_next = &msm_dp_priv->event_list[msm_dp_priv->event_pndx++]; - msm_dp_priv->event_pndx %= DP_EVENT_Q_MAX; - todo_next->event_id = todo->event_id; - todo_next->data = todo->data; - todo_next->delay = todo->delay - 1; - - /* clean up older event */ - todo->event_id = EV_NO_EVENT; - todo->delay = 0; - - /* switch to timeout mode */ - timeout_mode = 1; - spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag); - continue; - } + mutex_lock(&priv->plugged_lock); + ret = pm_runtime_resume_and_get(&dp->pdev->dev); + if (ret) { + DRM_ERROR("failed to pm_runtime_resume\n"); + mutex_unlock(&priv->plugged_lock); + return status; + } - /* timeout with no events in q */ - if (msm_dp_priv->event_pndx == msm_dp_priv->event_gndx) { - spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag); - continue; - } + phy_deinit = msm_dp_display_host_phy_init(priv); - msm_dp_priv->event_gndx++; - msm_dp_priv->event_gndx %= DP_EVENT_Q_MAX; - timeout_mode = 0; - spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag); + msm_dp_aux_enable_xfers(priv->aux, true); - switch (todo->event_id) { - case EV_HPD_PLUG_INT: - msm_dp_hpd_plug_handle(msm_dp_priv, todo->data); - break; - case EV_HPD_UNPLUG_INT: - msm_dp_hpd_unplug_handle(msm_dp_priv, todo->data); - break; - case EV_IRQ_HPD_INT: - msm_dp_irq_hpd_handle(msm_dp_priv, todo->data); - break; - case EV_USER_NOTIFICATION: - msm_dp_display_send_hpd_notification(msm_dp_priv, - todo->data); - break; - default: - break; - } + ret = msm_dp_aux_is_link_connected(priv->aux); + DRM_DEBUG_DP("aux link status: %x\n", ret); + if (!priv->plugged && !ret) { + DRM_DEBUG_DP("aux not connected\n"); + priv->plugged = false; + goto end; } - return 0; -} + ret = drm_dp_read_dpcd_caps(priv->aux, dpcd); + if (ret) { + DRM_DEBUG_DP("failed to read caps\n"); + priv->plugged = false; + goto end; + } -static int msm_dp_hpd_event_thread_start(struct msm_dp_display_private *msm_dp_priv) -{ - /* set event q to empty */ - msm_dp_priv->event_gndx = 0; - msm_dp_priv->event_pndx = 0; + ret = drm_dp_read_desc(priv->aux, &desc, drm_dp_is_branch(dpcd)); + if (ret) { + DRM_DEBUG_DP("failed to read desc\n"); + priv->plugged = false; + goto end; + } - msm_dp_priv->ev_tsk = kthread_run(hpd_event_thread, msm_dp_priv, "dp_hpd_handler"); - if (IS_ERR(msm_dp_priv->ev_tsk)) - return PTR_ERR(msm_dp_priv->ev_tsk); + status = connector_status_connected; + priv->plugged = true; - return 0; + if (drm_dp_read_sink_count_cap(connector, dpcd, &desc)) { + int sink_count = drm_dp_read_sink_count(priv->aux); + + drm_dbg_dp(dp->drm_dev, "sink_count = %d\n", sink_count); + + if (sink_count <= 0) + status = connector_status_disconnected; + } + + if (priv->max_stream > 1 && drm_dp_read_mst_cap(priv->aux, dpcd)) + status = connector_status_disconnected; +end: + /* + * If we detected the DPRX, leave the controller on so that it doesn't + * loose the state. + */ + if (!priv->plugged) { + if (phy_deinit) { + msm_dp_aux_enable_xfers(priv->aux, false); + msm_dp_display_host_phy_exit(priv); + } + + pm_runtime_put_sync(&dp->pdev->dev); + } + + mutex_unlock(&priv->plugged_lock); + + return status; } static irqreturn_t msm_dp_display_irq_handler(int irq, void *dev_id) { struct msm_dp_display_private *dp = dev_id; - irqreturn_t ret = IRQ_NONE; u32 hpd_isr_status; - - if (!dp) { - DRM_ERROR("invalid data\n"); - return IRQ_NONE; - } + unsigned long flags; + irqreturn_t ret = IRQ_HANDLED; hpd_isr_status = msm_dp_aux_get_hpd_intr_status(dp->aux); if (hpd_isr_status & 0x0F) { drm_dbg_dp(dp->drm_dev, "type=%d isr=0x%x\n", dp->msm_dp_display.connector_type, hpd_isr_status); - /* hpd related interrupts */ - if (hpd_isr_status & DP_DP_HPD_PLUG_INT_MASK) - msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 0); - if (hpd_isr_status & DP_DP_IRQ_HPD_INT_MASK) { - msm_dp_add_event(dp, EV_IRQ_HPD_INT, 0, 0); - } + spin_lock_irqsave(&dp->irq_thread_lock, flags); + dp->hpd_isr_status |= hpd_isr_status; + ret = IRQ_WAKE_THREAD; + spin_unlock_irqrestore(&dp->irq_thread_lock, flags); + } - if (hpd_isr_status & DP_DP_HPD_REPLUG_INT_MASK) { - msm_dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0); - msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 3); - } + /* DP controller isr */ + ret |= msm_dp_ctrl_isr(dp->ctrl); + + return ret; +} - if (hpd_isr_status & DP_DP_HPD_UNPLUG_INT_MASK) - msm_dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0); +static irqreturn_t msm_dp_display_irq_thread(int irq, void *dev_id) +{ + struct msm_dp_display_private *dp = dev_id; + irqreturn_t ret = IRQ_NONE; + unsigned long flags; + u32 hpd_isr_status; + + spin_lock_irqsave(&dp->irq_thread_lock, flags); + hpd_isr_status = dp->hpd_isr_status; + dp->hpd_isr_status = 0; + spin_unlock_irqrestore(&dp->irq_thread_lock, flags); + + if (hpd_isr_status & DP_DP_HPD_UNPLUG_INT_MASK) + drm_bridge_hpd_notify(dp->msm_dp_display.bridge, + connector_status_disconnected); + + if (hpd_isr_status & DP_DP_HPD_PLUG_INT_MASK) + drm_bridge_hpd_notify(dp->msm_dp_display.bridge, + connector_status_connected); - ret = IRQ_HANDLED; + /* Send HPD as connected and distinguish it in the notifier */ + if (hpd_isr_status & DP_DP_IRQ_HPD_INT_MASK) { + if (dp->msm_dp_display.mst_active) + msm_dp_irq_hpd_handle(dp); + else + drm_bridge_hpd_notify(dp->msm_dp_display.bridge, + connector_status_connected); } - /* DP controller isr */ - ret |= msm_dp_ctrl_isr(dp->ctrl); + ret = IRQ_HANDLED; return ret; } @@ -1194,9 +1164,13 @@ static int msm_dp_display_request_irq(struct msm_dp_display_private *dp) return dp->irq; } - rc = devm_request_irq(&pdev->dev, dp->irq, msm_dp_display_irq_handler, - IRQF_TRIGGER_HIGH|IRQF_NO_AUTOEN, - "dp_display_isr", dp); + spin_lock_init(&dp->irq_thread_lock); + irq_set_status_flags(dp->irq, IRQ_NOAUTOEN); + rc = devm_request_threaded_irq(&pdev->dev, dp->irq, + msm_dp_display_irq_handler, + msm_dp_display_irq_thread, + IRQ_TYPE_LEVEL_HIGH, + "dp_display_isr", dp); if (rc < 0) { DRM_ERROR("failed to request IRQ%u: %d\n", @@ -1301,6 +1275,7 @@ static void __iomem *msm_dp_ioremap(struct platform_device *pdev, int idx, size_ static int msm_dp_display_get_io(struct msm_dp_display_private *display) { struct platform_device *pdev = display->msm_dp_display.pdev; + int i; display->ahb_base = msm_dp_ioremap(pdev, 0, &display->ahb_len); if (IS_ERR(display->ahb_base)) @@ -1330,8 +1305,8 @@ static int msm_dp_display_get_io(struct msm_dp_display_private *display) display->aux_len = DP_DEFAULT_AUX_SIZE; display->link_base = display->ahb_base + DP_DEFAULT_LINK_OFFSET; display->link_len = DP_DEFAULT_LINK_SIZE; - display->p0_base = display->ahb_base + DP_DEFAULT_P0_OFFSET; - display->p0_len = DP_DEFAULT_P0_SIZE; + display->pixel_base[0] = display->ahb_base + DP_DEFAULT_P0_OFFSET; + display->pixel_len = DP_DEFAULT_P0_SIZE; return 0; } @@ -1342,15 +1317,42 @@ static int msm_dp_display_get_io(struct msm_dp_display_private *display) return PTR_ERR(display->link_base); } - display->p0_base = msm_dp_ioremap(pdev, 3, &display->p0_len); - if (IS_ERR(display->p0_base)) { - DRM_ERROR("unable to remap p0 region: %pe\n", display->p0_base); - return PTR_ERR(display->p0_base); + display->pixel_base[0] = msm_dp_ioremap(pdev, 3, &display->pixel_len); + if (IS_ERR(display->pixel_base[0])) { + DRM_ERROR("unable to remap p0 region: %pe\n", display->pixel_base[0]); + return PTR_ERR(display->pixel_base[0]); } + for (i = DP_STREAM_1; i < display->max_stream; i++) { + /* pixels clk reg index start from 3*/ + display->pixel_base[i] = msm_dp_ioremap(pdev, i + 3, &display->pixel_len); + if (IS_ERR(display->pixel_base[i])) { + DRM_DEBUG_DP("unable to remap p%d region: %pe\n", i, + display->pixel_base[i]); + break; + } + } + + display->mst2link_base = msm_dp_ioremap(pdev, 7, &display->mst2link_len); + if (IS_ERR(display->mst2link_base)) + DRM_DEBUG_DP("unable to remap link region: %pe\n", display->mst2link_base); + + display->mst3link_base = msm_dp_ioremap(pdev, 8, &display->mst3link_len); + if (IS_ERR(display->mst3link_base)) + DRM_DEBUG_DP("unable to remap link region: %pe\n", display->mst3link_base); + return 0; } +int msm_dp_get_mst_max_stream(struct msm_dp *msm_dp_display) +{ + struct msm_dp_display_private *dp; + + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); + + return dp->max_stream; +} + static int msm_dp_display_probe(struct platform_device *pdev) { int rc = 0; @@ -1376,6 +1378,13 @@ static int msm_dp_display_probe(struct platform_device *pdev) dp->wide_bus_supported = desc->wide_bus_supported; dp->msm_dp_display.is_edp = (dp->msm_dp_display.connector_type == DRM_MODE_CONNECTOR_eDP); + dp->hpd_isr_status = 0; + dp->max_stream = 1; + + if (desc->mst_streams > 1) + dp->max_stream = desc->mst_streams; + + mutex_init(&dp->plugged_lock); rc = msm_dp_display_get_io(dp); if (rc) @@ -1387,11 +1396,6 @@ static int msm_dp_display_probe(struct platform_device *pdev) return -EPROBE_DEFER; } - /* setup event q */ - mutex_init(&dp->event_mutex); - init_waitqueue_head(&dp->event_q); - spin_lock_init(&dp->event_lock); - /* Store DP audio handle inside DP display */ dp->msm_dp_display.msm_dp_audio = dp->audio; @@ -1529,7 +1533,7 @@ bool msm_dp_wide_bus_available(const struct msm_dp *msm_dp_display) dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - if (dp->msm_dp_mode.out_fmt_is_yuv_420) + if (dp->panel->msm_dp_mode.out_fmt_is_yuv_420) return false; return dp->wide_bus_supported; @@ -1580,152 +1584,168 @@ int msm_dp_modeset_init(struct msm_dp *msm_dp_display, struct drm_device *dev, return 0; } -void msm_dp_bridge_atomic_enable(struct drm_bridge *drm_bridge, - struct drm_atomic_state *state) +int msm_dp_mst_register(struct msm_dp *msm_dp_display) { - struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(drm_bridge); - struct msm_dp *dp = msm_dp_bridge->msm_dp_display; - int rc = 0; - struct msm_dp_display_private *msm_dp_display; - u32 hpd_state; - bool force_link_train = false; + struct msm_dp_display_private *dp; - msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); - if (!msm_dp_display->msm_dp_mode.drm_mode.clock) { - DRM_ERROR("invalid params\n"); - return; - } + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - if (dp->is_edp) - msm_dp_hpd_plug_handle(msm_dp_display, 0); + return msm_dp_mst_init(msm_dp_display, dp->max_stream, dp->aux); +} - mutex_lock(&msm_dp_display->event_mutex); - if (pm_runtime_resume_and_get(&dp->pdev->dev)) { - DRM_ERROR("failed to pm_runtime_resume\n"); - mutex_unlock(&msm_dp_display->event_mutex); - return; - } +int msm_dp_display_set_mode_helper(struct msm_dp *msm_dp_display, + struct drm_atomic_state *state, + struct drm_encoder *drm_encoder, + struct msm_dp_panel *msm_dp_panel) +{ + struct drm_crtc *crtc; + struct drm_crtc_state *crtc_state; - hpd_state = msm_dp_display->hpd_state; - if (hpd_state != ST_DISPLAY_OFF && hpd_state != ST_MAINLINK_READY) { - mutex_unlock(&msm_dp_display->event_mutex); - return; - } + crtc = drm_atomic_get_new_crtc_for_encoder(state, drm_encoder); + if (!crtc) + return 0; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + return msm_dp_display_set_mode(msm_dp_display, &crtc_state->adjusted_mode, msm_dp_panel); +} + +void msm_dp_display_atomic_prepare(struct msm_dp *msm_dp_display, + struct drm_atomic_state *state) +{ + int rc = 0; + struct msm_dp_display_private *dp; + + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - rc = msm_dp_display_set_mode(dp, &msm_dp_display->msm_dp_mode); + rc = msm_dp_display_set_mode_helper(msm_dp_display, state, + msm_dp_display->bridge->encoder, dp->panel); if (rc) { DRM_ERROR("Failed to perform a mode set, rc=%d\n", rc); - mutex_unlock(&msm_dp_display->event_mutex); return; } - hpd_state = msm_dp_display->hpd_state; + rc = msm_dp_display_prepare(msm_dp_display); + if (rc) + DRM_ERROR("DP display prepare failed, rc=%d\n", rc); +} - if (hpd_state == ST_DISPLAY_OFF) { - msm_dp_display_host_phy_init(msm_dp_display); - force_link_train = true; - } +void msm_dp_display_enable_helper(struct msm_dp *msm_dp_display, struct msm_dp_panel *msm_dp_panel) +{ + struct msm_dp_display_private *dp; + int rc = 0; - msm_dp_display_enable(msm_dp_display, force_link_train); + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - rc = msm_dp_display_post_enable(dp); - if (rc) { - DRM_ERROR("DP display post enable failed, rc=%d\n", rc); - msm_dp_display_disable(msm_dp_display); + if (msm_dp_display->prepared) { + rc = msm_dp_display_enable(dp, msm_dp_panel); + if (rc) + DRM_ERROR("DP display enable failed, rc=%d\n", rc); + + rc = msm_dp_display_post_enable(msm_dp_display); + if (rc) { + DRM_ERROR("DP display post enable failed, rc=%d\n", rc); + msm_dp_display_disable(dp, msm_dp_panel); + } } - /* completed connection */ - msm_dp_display->hpd_state = ST_CONNECTED; + drm_dbg_dp(msm_dp_display->drm_dev, "type=%d Done\n", msm_dp_display->connector_type); +} - drm_dbg_dp(dp->drm_dev, "type=%d Done\n", dp->connector_type); - mutex_unlock(&msm_dp_display->event_mutex); +void msm_dp_display_atomic_enable(struct msm_dp *msm_dp_display) +{ + struct msm_dp_display_private *dp; + + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); + + msm_dp_display_set_stream_info(msm_dp_display, dp->panel, 0, 0, 0, 0); + + msm_dp_display_enable_helper(msm_dp_display, dp->panel); } -void msm_dp_bridge_atomic_disable(struct drm_bridge *drm_bridge, - struct drm_atomic_state *state) +void msm_dp_display_disable_helper(struct msm_dp *msm_dp_display, + struct msm_dp_panel *msm_dp_panel) { - struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(drm_bridge); - struct msm_dp *dp = msm_dp_bridge->msm_dp_display; - struct msm_dp_display_private *msm_dp_display; + struct msm_dp_display_private *dp; - msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); + + if (!msm_dp_display->active_stream_cnt) { + drm_dbg_dp(dp->drm_dev, "no active streams\n"); + return; + } - msm_dp_ctrl_push_idle(msm_dp_display->ctrl); + msm_dp_ctrl_push_idle(dp->ctrl, msm_dp_panel); + msm_dp_ctrl_mst_stream_channel_slot_setup(dp->ctrl); + msm_dp_ctrl_mst_send_act(dp->ctrl); } -void msm_dp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge, - struct drm_atomic_state *state) +void msm_dp_display_atomic_disable(struct msm_dp *msm_dp_display) { - struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(drm_bridge); - struct msm_dp *dp = msm_dp_bridge->msm_dp_display; - u32 hpd_state; - struct msm_dp_display_private *msm_dp_display; + struct msm_dp_display_private *dp; - msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - if (dp->is_edp) - msm_dp_hpd_unplug_handle(msm_dp_display, 0); + msm_dp_display_disable_helper(msm_dp_display, dp->panel); +} - mutex_lock(&msm_dp_display->event_mutex); +void msm_dp_display_unprepare(struct msm_dp *msm_dp_display) +{ + struct msm_dp_display_private *dp; - hpd_state = msm_dp_display->hpd_state; - if (hpd_state != ST_DISCONNECT_PENDING && hpd_state != ST_CONNECTED) - drm_dbg_dp(dp->drm_dev, "type=%d wrong hpd_state=%d\n", - dp->connector_type, hpd_state); + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - msm_dp_display_disable(msm_dp_display); + if (!msm_dp_display->prepared) { + drm_dbg_dp(dp->drm_dev, "Link already setup, return\n"); + return; + } - hpd_state = msm_dp_display->hpd_state; - if (hpd_state == ST_DISCONNECT_PENDING) { - /* completed disconnection */ - msm_dp_display->hpd_state = ST_DISCONNECTED; - } else { - msm_dp_display->hpd_state = ST_DISPLAY_OFF; + if (msm_dp_display->active_stream_cnt) { + drm_dbg_dp(dp->drm_dev, "stream still active, return\n"); + return; } - drm_dbg_dp(dp->drm_dev, "type=%d Done\n", dp->connector_type); + /* dongle is still connected but sinks are disconnected */ + if (dp->link->sink_count == 0) + msm_dp_link_psm_config(dp->link, &dp->panel->link_info, true); + + msm_dp_ctrl_off_link(dp->ctrl); - pm_runtime_put_sync(&dp->pdev->dev); - mutex_unlock(&msm_dp_display->event_mutex); + /* re-init the PHY so that we can listen to Dongle disconnect */ + if (dp->link->sink_count == 0) + msm_dp_ctrl_reinit_phy(dp->ctrl); + else + msm_dp_display_host_phy_exit(dp); + + pm_runtime_put_sync(&msm_dp_display->pdev->dev); + + msm_dp_display->prepared = false; } -void msm_dp_bridge_mode_set(struct drm_bridge *drm_bridge, - const struct drm_display_mode *mode, - const struct drm_display_mode *adjusted_mode) +void msm_dp_display_atomic_post_disable_helper(struct msm_dp *dp, struct msm_dp_panel *msm_dp_panel) { - struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(drm_bridge); - struct msm_dp *dp = msm_dp_bridge->msm_dp_display; struct msm_dp_display_private *msm_dp_display; - struct msm_dp_panel *msm_dp_panel; msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); - msm_dp_panel = msm_dp_display->panel; - memset(&msm_dp_display->msm_dp_mode, 0x0, sizeof(struct msm_dp_display_mode)); + if (dp->is_edp) + msm_dp_hpd_unplug_handle(msm_dp_display); - if (msm_dp_display_check_video_test(dp)) - msm_dp_display->msm_dp_mode.bpp = msm_dp_display_get_test_bpp(dp); - else /* Default num_components per pixel = 3 */ - msm_dp_display->msm_dp_mode.bpp = dp->connector->display_info.bpc * 3; + msm_dp_display_audio_notify_disable(msm_dp_display); - if (!msm_dp_display->msm_dp_mode.bpp) - msm_dp_display->msm_dp_mode.bpp = 24; /* Default bpp */ + msm_dp_display_disable(msm_dp_display, msm_dp_panel); - drm_mode_copy(&msm_dp_display->msm_dp_mode.drm_mode, adjusted_mode); + drm_dbg_dp(dp->drm_dev, "type=%d Done\n", dp->connector_type); +} - msm_dp_display->msm_dp_mode.v_active_low = - !!(msm_dp_display->msm_dp_mode.drm_mode.flags & DRM_MODE_FLAG_NVSYNC); +void msm_dp_display_atomic_post_disable(struct msm_dp *msm_dp_display) +{ + struct msm_dp_display_private *dp; - msm_dp_display->msm_dp_mode.h_active_low = - !!(msm_dp_display->msm_dp_mode.drm_mode.flags & DRM_MODE_FLAG_NHSYNC); + dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - msm_dp_display->msm_dp_mode.out_fmt_is_yuv_420 = - drm_mode_is_420_only(&dp->connector->display_info, adjusted_mode) && - msm_dp_panel->vsc_sdp_supported; + msm_dp_display_atomic_post_disable_helper(msm_dp_display, dp->panel); - /* populate wide_bus_support to different layers */ - msm_dp_display->ctrl->wide_bus_en = - msm_dp_display->msm_dp_mode.out_fmt_is_yuv_420 ? false : msm_dp_display->wide_bus_supported; + msm_dp_display_unprepare(msm_dp_display); } void msm_dp_bridge_hpd_enable(struct drm_bridge *bridge) @@ -1742,18 +1762,13 @@ void msm_dp_bridge_hpd_enable(struct drm_bridge *bridge) * step-4: DP PHY is initialized at plugin handler before link training * */ - mutex_lock(&dp->event_mutex); if (pm_runtime_resume_and_get(&msm_dp_display->pdev->dev)) { DRM_ERROR("failed to resume power\n"); - mutex_unlock(&dp->event_mutex); return; } msm_dp_aux_hpd_enable(dp->aux); msm_dp_aux_hpd_intr_enable(dp->aux); - - msm_dp_display->internal_hpd = true; - mutex_unlock(&dp->event_mutex); } void msm_dp_bridge_hpd_disable(struct drm_bridge *bridge) @@ -1762,15 +1777,10 @@ void msm_dp_bridge_hpd_disable(struct drm_bridge *bridge) struct msm_dp *msm_dp_display = msm_dp_bridge->msm_dp_display; struct msm_dp_display_private *dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - mutex_lock(&dp->event_mutex); - msm_dp_aux_hpd_intr_disable(dp->aux); msm_dp_aux_hpd_disable(dp->aux); - msm_dp_display->internal_hpd = false; - pm_runtime_put_sync(&msm_dp_display->pdev->dev); - mutex_unlock(&dp->event_mutex); } void msm_dp_bridge_hpd_notify(struct drm_bridge *bridge, @@ -1779,13 +1789,31 @@ void msm_dp_bridge_hpd_notify(struct drm_bridge *bridge, struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(bridge); struct msm_dp *msm_dp_display = msm_dp_bridge->msm_dp_display; struct msm_dp_display_private *dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); + u32 hpd_link_status = 0; - /* Without next_bridge interrupts are handled by the DP core directly */ - if (msm_dp_display->internal_hpd) + if (pm_runtime_resume_and_get(&msm_dp_display->pdev->dev)) { + DRM_ERROR("failed to pm_runtime_resume\n"); return; + } + + hpd_link_status = msm_dp_aux_is_link_connected(dp->aux); - if (!msm_dp_display->link_ready && status == connector_status_connected) - msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 0); - else if (msm_dp_display->link_ready && status == connector_status_disconnected) - msm_dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0); + drm_dbg_dp(dp->drm_dev, "type=%d link hpd_link_status=0x%x, status=%d\n", + msm_dp_display->connector_type, hpd_link_status, status); + + if (status == connector_status_connected) { + if (hpd_link_status == ISR_HPD_REPLUG_COUNT) { + msm_dp_hpd_unplug_handle(dp); + msm_dp_hpd_plug_handle(dp); + } else if (hpd_link_status == ISR_IRQ_HPD_PULSE_COUNT) { + msm_dp_irq_hpd_handle(dp); + } else { + msm_dp_hpd_plug_handle(dp); + } + } else { + if (hpd_link_status == ISR_DISCONNECTED) + msm_dp_hpd_unplug_handle(dp); + } + + pm_runtime_put_sync(&msm_dp_display->pdev->dev); } diff --git a/drivers/gpu/drm/msm/dp/dp_display.h b/drivers/gpu/drm/msm/dp/dp_display.h index cc6e2cab36e9c..55874daf41c44 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.h +++ b/drivers/gpu/drm/msm/dp/dp_display.h @@ -16,17 +16,21 @@ struct msm_dp { struct platform_device *pdev; struct drm_connector *connector; struct drm_bridge *next_bridge; - bool link_ready; + struct drm_bridge *bridge; bool audio_enabled; - bool power_on; + u32 active_stream_cnt; + bool mst_active; unsigned int connector_type; bool is_edp; - bool internal_hpd; + bool prepared; + + void *msm_dp_mst; struct msm_dp_audio *msm_dp_audio; bool psr_supported; }; +int msm_dp_get_mst_max_stream(struct msm_dp *msm_dp_display); int msm_dp_display_get_modes(struct msm_dp *msm_dp_display); bool msm_dp_display_check_video_test(struct msm_dp *msm_dp_display); int msm_dp_display_get_test_bpp(struct msm_dp *msm_dp_display); @@ -34,5 +38,29 @@ void msm_dp_display_signal_audio_start(struct msm_dp *msm_dp_display); void msm_dp_display_signal_audio_complete(struct msm_dp *msm_dp_display); void msm_dp_display_set_psr(struct msm_dp *dp, bool enter); void msm_dp_display_debugfs_init(struct msm_dp *msm_dp_display, struct dentry *dentry, bool is_edp); +void msm_dp_display_atomic_post_disable(struct msm_dp *dp_display); +void msm_dp_display_atomic_disable(struct msm_dp *dp_display); +void msm_dp_display_atomic_prepare(struct msm_dp *dp_display, + struct drm_atomic_state *state); +void msm_dp_display_atomic_enable(struct msm_dp *dp_display); +enum drm_mode_status msm_dp_display_mode_valid(struct msm_dp *dp, + const struct drm_display_info *info, + const struct drm_display_mode *mode); +int msm_dp_display_set_stream_info(struct msm_dp *msm_dp_display, struct msm_dp_panel *panel, + enum msm_dp_stream_id stream_id, + u32 start_slot, u32 num_slots, u32 pbn); +void msm_dp_display_enable_helper(struct msm_dp *msm_dp_display, + struct msm_dp_panel *msm_dp_panel); +void msm_dp_display_disable_helper(struct msm_dp *msm_dp_display, + struct msm_dp_panel *msm_dp_panel); +void msm_dp_display_atomic_post_disable_helper(struct msm_dp *msm_dp_display, + struct msm_dp_panel *msm_dp_panel); +int msm_dp_display_set_mode_helper(struct msm_dp *msm_dp_display, + struct drm_atomic_state *state, + struct drm_encoder *drm_encoder, + struct msm_dp_panel *msm_dp_panel); +int msm_dp_display_prepare(struct msm_dp *msm_dp_display); +void msm_dp_display_unprepare(struct msm_dp *dp); +struct msm_dp_panel *msm_dp_display_get_panel(struct msm_dp *msm_dp_display); #endif /* _DP_DISPLAY_H_ */ diff --git a/drivers/gpu/drm/msm/dp/dp_drm.c b/drivers/gpu/drm/msm/dp/dp_drm.c index 9a461ab2f32fc..0feb757e2db90 100644 --- a/drivers/gpu/drm/msm/dp/dp_drm.c +++ b/drivers/gpu/drm/msm/dp/dp_drm.c @@ -15,55 +15,6 @@ #include "dp_audio.h" #include "dp_drm.h" -/** - * msm_dp_bridge_detect - callback to determine if connector is connected - * @bridge: Pointer to drm bridge structure - * Returns: Bridge's 'is connected' status - */ -static enum drm_connector_status -msm_dp_bridge_detect(struct drm_bridge *bridge, struct drm_connector *connector) -{ - struct msm_dp *dp; - - dp = to_dp_bridge(bridge)->msm_dp_display; - - drm_dbg_dp(dp->drm_dev, "link_ready = %s\n", - str_true_false(dp->link_ready)); - - return (dp->link_ready) ? connector_status_connected : - connector_status_disconnected; -} - -static int msm_dp_bridge_atomic_check(struct drm_bridge *bridge, - struct drm_bridge_state *bridge_state, - struct drm_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct msm_dp *dp; - - dp = to_dp_bridge(bridge)->msm_dp_display; - - drm_dbg_dp(dp->drm_dev, "link_ready = %s\n", - str_true_false(dp->link_ready)); - - /* - * There is no protection in the DRM framework to check if the display - * pipeline has been already disabled before trying to disable it again. - * Hence if the sink is unplugged, the pipeline gets disabled, but the - * crtc->active is still true. Any attempt to set the mode or manually - * disable this encoder will result in the crash. - * - * TODO: add support for telling the DRM subsystem that the pipeline is - * disabled by the hardware and thus all access to it should be forbidden. - * After that this piece of code can be removed. - */ - if (bridge->ops & DRM_BRIDGE_OP_HPD) - return (dp->link_ready) ? 0 : -ENOTCONN; - - return 0; -} - - /** * msm_dp_bridge_get_modes - callback to add drm modes via drm_mode_probed_add() * @bridge: Poiner to drm bridge @@ -81,12 +32,10 @@ static int msm_dp_bridge_get_modes(struct drm_bridge *bridge, struct drm_connect dp = to_dp_bridge(bridge)->msm_dp_display; /* pluggable case assumes EDID is read when HPD */ - if (dp->link_ready) { - rc = msm_dp_display_get_modes(dp); - if (rc <= 0) { - DRM_ERROR("failed to get DP sink modes, rc=%d\n", rc); - return rc; - } + rc = msm_dp_display_get_modes(dp); + if (rc <= 0) { + DRM_ERROR("failed to get DP sink modes, rc=%d\n", rc); + return rc; } else { drm_dbg_dp(connector->dev, "No sink connected\n"); } @@ -100,6 +49,44 @@ static void msm_dp_bridge_debugfs_init(struct drm_bridge *bridge, struct dentry msm_dp_display_debugfs_init(dp, root, false); } +static void msm_dp_bridge_atomic_enable(struct drm_bridge *drm_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge); + struct msm_dp *dp = dp_bridge->msm_dp_display; + + msm_dp_display_atomic_prepare(dp, state); + msm_dp_display_atomic_enable(dp); +} + +static void msm_dp_bridge_atomic_disable(struct drm_bridge *drm_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge); + struct msm_dp *dp = dp_bridge->msm_dp_display; + + msm_dp_display_atomic_disable(dp); +} + +static void msm_dp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge); + struct msm_dp *dp = dp_bridge->msm_dp_display; + + msm_dp_display_atomic_post_disable(dp); +} + +static enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *drm_bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) +{ + struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge); + struct msm_dp *dp = dp_bridge->msm_dp_display; + + return msm_dp_display_mode_valid(dp, info, mode); +} + static const struct drm_bridge_funcs msm_dp_bridge_ops = { .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, @@ -107,11 +94,9 @@ static const struct drm_bridge_funcs msm_dp_bridge_ops = { .atomic_enable = msm_dp_bridge_atomic_enable, .atomic_disable = msm_dp_bridge_atomic_disable, .atomic_post_disable = msm_dp_bridge_atomic_post_disable, - .mode_set = msm_dp_bridge_mode_set, .mode_valid = msm_dp_bridge_mode_valid, .get_modes = msm_dp_bridge_get_modes, .detect = msm_dp_bridge_detect, - .atomic_check = msm_dp_bridge_atomic_check, .hpd_enable = msm_dp_bridge_hpd_enable, .hpd_disable = msm_dp_bridge_hpd_disable, .hpd_notify = msm_dp_bridge_hpd_notify, @@ -168,7 +153,8 @@ static void msm_edp_bridge_atomic_enable(struct drm_bridge *drm_bridge, return; } - msm_dp_bridge_atomic_enable(drm_bridge, state); + msm_dp_display_atomic_prepare(dp, state); + msm_dp_display_atomic_enable(dp); } static void msm_edp_bridge_atomic_disable(struct drm_bridge *drm_bridge, @@ -285,7 +271,6 @@ static const struct drm_bridge_funcs msm_edp_bridge_ops = { .atomic_enable = msm_edp_bridge_atomic_enable, .atomic_disable = msm_edp_bridge_atomic_disable, .atomic_post_disable = msm_edp_bridge_atomic_post_disable, - .mode_set = msm_dp_bridge_mode_set, .mode_valid = msm_edp_bridge_mode_valid, .atomic_reset = drm_atomic_helper_bridge_reset, .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, @@ -359,6 +344,8 @@ int msm_dp_bridge_init(struct msm_dp *msm_dp_display, struct drm_device *dev, } } + msm_dp_display->bridge = bridge; + return 0; } diff --git a/drivers/gpu/drm/msm/dp/dp_drm.h b/drivers/gpu/drm/msm/dp/dp_drm.h index d8c9b905f8bfb..111df516e4bd9 100644 --- a/drivers/gpu/drm/msm/dp/dp_drm.h +++ b/drivers/gpu/drm/msm/dp/dp_drm.h @@ -25,18 +25,8 @@ int msm_dp_bridge_init(struct msm_dp *msm_dp_display, struct drm_device *dev, struct drm_encoder *encoder, bool yuv_supported); -void msm_dp_bridge_atomic_enable(struct drm_bridge *drm_bridge, - struct drm_atomic_state *state); -void msm_dp_bridge_atomic_disable(struct drm_bridge *drm_bridge, - struct drm_atomic_state *state); -void msm_dp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge, - struct drm_atomic_state *state); -enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, - const struct drm_display_info *info, - const struct drm_display_mode *mode); -void msm_dp_bridge_mode_set(struct drm_bridge *drm_bridge, - const struct drm_display_mode *mode, - const struct drm_display_mode *adjusted_mode); +enum drm_connector_status msm_dp_bridge_detect(struct drm_bridge *bridge, + struct drm_connector *connector); void msm_dp_bridge_hpd_enable(struct drm_bridge *bridge); void msm_dp_bridge_hpd_disable(struct drm_bridge *bridge); void msm_dp_bridge_hpd_notify(struct drm_bridge *bridge, diff --git a/drivers/gpu/drm/msm/dp/dp_link.c b/drivers/gpu/drm/msm/dp/dp_link.c index 66e1bbd80db3a..2ae78d33fffd7 100644 --- a/drivers/gpu/drm/msm/dp/dp_link.c +++ b/drivers/gpu/drm/msm/dp/dp_link.c @@ -6,12 +6,14 @@ #define pr_fmt(fmt) "[drm-dp] %s: " fmt, __func__ #include +#include #include #include "dp_reg.h" #include "dp_link.h" #include "dp_panel.h" +#define DP_LINK_RATE_HBR2 540000 /* kbytes */ #define DP_TEST_REQUEST_MASK 0x7F enum audio_sample_rate { @@ -1210,10 +1212,122 @@ u32 msm_dp_link_get_test_bits_depth(struct msm_dp_link *msm_dp_link, u32 bpp) return tbd; } +static u32 msm_dp_link_link_frequencies(struct device_node *of_node) +{ + struct device_node *endpoint; + u64 frequency = 0; + int cnt; + + endpoint = of_graph_get_endpoint_by_regs(of_node, 1, 0); /* port@1 */ + if (!endpoint) + return 0; + + cnt = of_property_count_u64_elems(endpoint, "link-frequencies"); + + if (cnt > 0) + of_property_read_u64_index(endpoint, "link-frequencies", + cnt - 1, &frequency); + of_node_put(endpoint); + + do_div(frequency, + 10 * /* from symbol rate to link rate */ + 1000); /* kbytes */ + + return frequency; +} + +/* + * Always populate msm_dp_link->lane_map with 4 lanes. + * - Use DTS "data-lanes" if present; otherwise fall back to default mapping. + * - For partial definitions, fill remaining entries with unused lanes in + * ascending order. + */ +static int msm_dp_link_lane_map(struct device *dev, struct msm_dp_link *msm_dp_link) +{ + struct device_node *of_node = dev->of_node; + struct device_node *endpoint; + int cnt = msm_dp_link->max_dp_lanes; + u32 tmp[DP_MAX_NUM_DP_LANES]; + u32 map[DP_MAX_NUM_DP_LANES] = {0, 1, 2, 3}; /* default 1:1 mapping */ + bool used[DP_MAX_NUM_DP_LANES] = {false}; + int i, ret = -EINVAL; + + endpoint = of_graph_get_endpoint_by_regs(of_node, 1, -1); + if (endpoint) { + ret = of_property_read_u32_array(endpoint, "data-lanes", tmp, cnt); + if (ret) + dev_dbg(dev, "endpoint data-lanes read failed (ret=%d)\n", ret); + } + + if (ret) { + ret = of_property_read_u32_array(of_node, "data-lanes", tmp, cnt); + if (ret) { + dev_info(dev, "data-lanes not defined, set to default\n"); + goto out; + } + } + + for (i = 0; i < cnt; i++) { + if (tmp[i] >= DP_MAX_NUM_DP_LANES) { + dev_err(dev, "data-lanes[%d]=%u out of range\n", i, tmp[i]); + return -EINVAL; + } + used[tmp[i]] = true; + map[i] = tmp[i]; + } + + /* Fill the remaining entries with unused physical lanes (ascending) */ + i = cnt; + for (int j = 0; i < DP_MAX_NUM_DP_LANES && j < DP_MAX_NUM_DP_LANES; j++) { + if (!used[j]) + map[i++] = j; + } + +out: + if (endpoint) + of_node_put(endpoint); + + dev_dbg(dev, "data-lanes count %d <%d %d %d %d>\n", cnt, map[0], map[1], map[2], map[3]); + memcpy(msm_dp_link->lane_map, map, sizeof(map)); + return 0; +} + +static int msm_dp_link_parse_dt(struct device *dev, struct msm_dp_link *msm_dp_link) +{ + struct device_node *of_node = dev->of_node; + int cnt; + + /* + * data-lanes is the property of msm_dp_out endpoint + */ + cnt = drm_of_get_data_lanes_count_ep(of_node, 1, 0, 1, DP_MAX_NUM_DP_LANES); + if (cnt < 0) { + /* legacy code, data-lanes is the property of mdss_dp node */ + cnt = drm_of_get_data_lanes_count(of_node, 1, DP_MAX_NUM_DP_LANES); + } + + if (cnt > 0) + msm_dp_link->max_dp_lanes = cnt; + else + msm_dp_link->max_dp_lanes = DP_MAX_NUM_DP_LANES; /* 4 lanes */ + + if (msm_dp_link_lane_map(dev, msm_dp_link)) { + dev_err(dev, "failed to parse data-lanes\n"); + return -EINVAL; + } + + msm_dp_link->max_dp_link_rate = msm_dp_link_link_frequencies(of_node); + if (!msm_dp_link->max_dp_link_rate) + msm_dp_link->max_dp_link_rate = DP_LINK_RATE_HBR2; + + return 0; +} + struct msm_dp_link *msm_dp_link_get(struct device *dev, struct drm_dp_aux *aux) { struct msm_dp_link_private *link; struct msm_dp_link *msm_dp_link; + int ret; if (!dev || !aux) { DRM_ERROR("invalid input\n"); @@ -1229,5 +1343,9 @@ struct msm_dp_link *msm_dp_link_get(struct device *dev, struct drm_dp_aux *aux) mutex_init(&link->psm_mutex); msm_dp_link = &link->msm_dp_link; + ret = msm_dp_link_parse_dt(dev, msm_dp_link); + if (ret) + return ERR_PTR(ret); + return msm_dp_link; } diff --git a/drivers/gpu/drm/msm/dp/dp_link.h b/drivers/gpu/drm/msm/dp/dp_link.h index ba47c6d19fbfa..76125e9c07e9c 100644 --- a/drivers/gpu/drm/msm/dp/dp_link.h +++ b/drivers/gpu/drm/msm/dp/dp_link.h @@ -12,10 +12,14 @@ #define DS_PORT_STATUS_CHANGED 0x200 #define DP_TEST_BIT_DEPTH_UNKNOWN 0xFFFFFFFF #define DP_LINK_CAP_ENHANCED_FRAMING (1 << 0) +#define DP_MAX_NUM_DP_LANES 4 struct msm_dp_link_info { unsigned char revision; unsigned int rate; + unsigned int supported_rates[DP_MAX_SUPPORTED_RATES]; + unsigned int rate_set; + bool use_rate_set; unsigned int num_lanes; unsigned long capabilities; }; @@ -72,14 +76,18 @@ struct msm_dp_link { struct msm_dp_link_test_audio test_audio; struct msm_dp_link_phy_params phy_params; struct msm_dp_link_info link_params; + + u32 lane_map[DP_MAX_NUM_DP_LANES]; + u32 max_dp_lanes; + u32 max_dp_link_rate; }; /** - * mdss_dp_test_bit_depth_to_bpp() - convert test bit depth to bpp + * msm_dp_link_bit_depth_to_bpp() - convert test bit depth to bpp * @tbd: test bit depth * - * Returns the bits per pixel (bpp) to be used corresponding to the - * git bit depth value. This function assumes that bit depth has + * Returns: the bits per pixel (bpp) to be used corresponding to the + * bit depth value. This function assumes that bit depth has * already been validated. */ static inline u32 msm_dp_link_bit_depth_to_bpp(u32 tbd) @@ -115,7 +123,8 @@ bool msm_dp_link_send_edid_checksum(struct msm_dp_link *msm_dp_link, u8 checksum /** * msm_dp_link_get() - get the functionalities of dp test module - * + * @dev: kernel device structure + * @aux: DisplayPort AUX channel * * return: a pointer to msm_dp_link struct */ diff --git a/drivers/gpu/drm/msm/dp/dp_mst_drm.c b/drivers/gpu/drm/msm/dp/dp_mst_drm.c new file mode 100644 index 0000000000000..cb5da84ead146 --- /dev/null +++ b/drivers/gpu/drm/msm/dp/dp_mst_drm.c @@ -0,0 +1,869 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "dp_mst_drm.h" +#include "dp_panel.h" + +#define MAX_DPCD_TRANSACTION_BYTES 16 + +#define to_dp_mst_bridge(x) container_of((x), struct msm_dp_mst_bridge, base) +#define to_dp_mst_bridge_state_priv(x) \ + container_of((x), struct msm_dp_mst_bridge_state, base) +#define to_dp_mst_bridge_state(x) \ + to_dp_mst_bridge_state_priv((x)->obj.state) +#define to_dp_mst_connector(x) \ + container_of((x), struct msm_dp_mst_connector, connector) + +#define DP_MST_CONN_ID(x) ((x)->connector ? \ + (x)->connector->base.id : 0) + +struct msm_dp_mst_bridge { + struct drm_bridge base; + struct drm_private_obj obj; + u32 id; + + bool initialized; + + struct msm_dp *display; + struct drm_encoder *encoder; + + struct drm_connector *connector; + struct msm_dp_panel *msm_dp_panel; +}; + +struct msm_dp_mst_bridge_state { + struct drm_private_state base; + struct drm_connector *connector; + struct msm_dp_panel *msm_dp_panel; +}; + +struct msm_dp_mst_connector { + struct drm_connector connector; + struct drm_dp_mst_port *mst_port; + struct msm_dp_mst *dp_mst; + struct msm_dp_panel *dp_panel; +}; + +struct msm_dp_mst { + struct drm_dp_mst_topology_mgr mst_mgr; + struct msm_dp_mst_bridge *mst_bridge[DP_STREAM_MAX]; + struct msm_dp *msm_dp; + struct drm_dp_aux *dp_aux; + u32 max_streams; + /* Protects MST bridge enable/disable handling. */ + struct mutex mst_lock; +}; + +static struct drm_private_state *msm_dp_mst_duplicate_bridge_state(struct drm_private_obj *obj) +{ + struct msm_dp_mst_bridge_state *mst_bridge_state; + + mst_bridge_state = kmemdup(obj->state, sizeof(*mst_bridge_state), GFP_KERNEL); + if (!mst_bridge_state) + return NULL; + + __drm_atomic_helper_private_obj_duplicate_state(obj, &mst_bridge_state->base); + + return &mst_bridge_state->base; +} + +static void msm_dp_mst_destroy_bridge_state(struct drm_private_obj *obj, + struct drm_private_state *state) +{ + struct msm_dp_mst_bridge_state *mst_bridge_state = + to_dp_mst_bridge_state_priv(state); + + kfree(mst_bridge_state); +} + +static const struct drm_private_state_funcs msm_dp_mst_bridge_state_funcs = { + .atomic_duplicate_state = msm_dp_mst_duplicate_bridge_state, + .atomic_destroy_state = msm_dp_mst_destroy_bridge_state, +}; + +static struct msm_dp_mst_bridge_state *msm_dp_mst_br_priv_state(struct drm_atomic_state *st, + struct msm_dp_mst_bridge *bridge) +{ + struct drm_device *dev = bridge->base.dev; + struct drm_private_state *obj_state = drm_atomic_get_private_obj_state(st, &bridge->obj); + + WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); + + return to_dp_mst_bridge_state_priv(obj_state); +} + +static void msm_dp_mst_update_timeslots(struct msm_dp_mst *mst, + struct msm_dp_mst_bridge *mst_bridge, + struct drm_atomic_state *state, + struct drm_dp_mst_port *port) +{ + struct drm_dp_mst_topology_state *mst_state; + struct drm_dp_mst_atomic_payload *payload; + + mst_state = to_drm_dp_mst_topology_state(mst->mst_mgr.base.state); + payload = drm_atomic_get_mst_payload_state(mst_state, port); + + if (!payload) { + DRM_ERROR("MST bridge [%d] update_timeslots failed, null payload\n", + mst_bridge->id); + return; + } + + if (payload->vc_start_slot < 0) + msm_dp_display_set_stream_info(mst->msm_dp, mst_bridge->msm_dp_panel, + mst_bridge->id, 1, 0, 0); + else + msm_dp_display_set_stream_info(mst->msm_dp, mst_bridge->msm_dp_panel, + mst_bridge->id, payload->vc_start_slot, + payload->time_slots, payload->pbn); +} + +static int msm_dp_mst_bridge_pre_enable_part1(struct msm_dp_mst_bridge *dp_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp *dp_display = dp_bridge->display; + struct msm_dp_mst *mst = dp_display->msm_dp_mst; + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(dp_bridge->connector); + struct drm_dp_mst_port *port = mst_conn->mst_port; + struct drm_dp_mst_topology_state *mst_state; + struct drm_dp_mst_atomic_payload *payload; + struct msm_dp_panel *dp_panel = mst_conn->dp_panel; + int pbn; + int rc = 0; + + mst_state = drm_atomic_get_new_mst_topology_state(state, &mst->mst_mgr); + + pbn = drm_dp_calc_pbn_mode(dp_panel->msm_dp_mode.drm_mode.clock, + (mst_conn->connector.display_info.bpc * 3) << 4); + + payload = drm_atomic_get_mst_payload_state(mst_state, port); + if (!payload || payload->time_slots <= 0) { + DRM_ERROR("time slots not allocated for conn:%d\n", DP_MST_CONN_ID(dp_bridge)); + rc = -EINVAL; + return rc; + } + + drm_dbg_dp(dp_display->drm_dev, "conn:%d pbn:%d, slots:%d\n", DP_MST_CONN_ID(dp_bridge), + pbn, payload->time_slots); + + drm_dp_mst_update_slots(mst_state, DP_CAP_ANSI_8B10B); + + rc = drm_dp_add_payload_part1(&mst->mst_mgr, mst_state, payload); + if (rc) { + DRM_ERROR("payload allocation failure for conn:%d\n", DP_MST_CONN_ID(dp_bridge)); + return rc; + } + + msm_dp_mst_update_timeslots(mst, dp_bridge, state, port); + + return rc; +} + +static void _msm_dp_mst_bridge_pre_enable_part2(struct msm_dp_mst_bridge *dp_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp *dp_display = dp_bridge->display; + struct msm_dp_mst *mst = dp_display->msm_dp_mst; + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(dp_bridge->connector); + struct drm_dp_mst_port *port = mst_conn->mst_port; + struct drm_dp_mst_topology_state *mst_state; + struct drm_dp_mst_atomic_payload *payload; + + drm_dp_check_act_status(&mst->mst_mgr); + + mst_state = to_drm_dp_mst_topology_state(mst->mst_mgr.base.state); + payload = drm_atomic_get_mst_payload_state(mst_state, port); + + drm_dp_add_payload_part2(&mst->mst_mgr, payload); + + drm_dbg_dp(dp_display->drm_dev, "MST bridge [%d] _pre enable part-2 complete\n", + dp_bridge->id); +} + +static void msm_dp_mst_bridge_pre_disable_part1(struct msm_dp_mst_bridge *dp_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp *dp_display = dp_bridge->display; + struct msm_dp_mst *mst = dp_display->msm_dp_mst; + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(dp_bridge->connector); + struct drm_dp_mst_port *port = mst_conn->mst_port; + struct drm_dp_mst_topology_state *old_mst_state; + struct drm_dp_mst_topology_state *new_mst_state; + const struct drm_dp_mst_atomic_payload *old_payload; + struct drm_dp_mst_atomic_payload *new_payload; + + old_mst_state = drm_atomic_get_old_mst_topology_state(state, &mst->mst_mgr); + new_mst_state = drm_atomic_get_new_mst_topology_state(state, &mst->mst_mgr); + + old_payload = drm_atomic_get_mst_payload_state(old_mst_state, port); + new_payload = drm_atomic_get_mst_payload_state(new_mst_state, port); + + if (!old_payload || !new_payload) { + DRM_ERROR("MST bridge [%d] _pre disable part-1 failed, null payload\n", + dp_bridge->id); + return; + } + + drm_dp_remove_payload_part1(&mst->mst_mgr, new_mst_state, new_payload); + drm_dp_remove_payload_part2(&mst->mst_mgr, new_mst_state, old_payload, new_payload); + + msm_dp_mst_update_timeslots(mst, dp_bridge, state, port); + + drm_dbg_dp(dp_display->drm_dev, "MST bridge [%d] _pre disable part-1 complete\n", + dp_bridge->id); +} + +static void msm_dp_mst_bridge_atomic_pre_enable(struct drm_bridge *drm_bridge, + struct drm_atomic_state *state) +{ + int rc = 0; + struct msm_dp_mst_bridge *bridge; + struct msm_dp *dp_display; + struct msm_dp_mst_bridge_state *mst_bridge_state; + struct msm_dp_mst *dp_mst; + struct msm_dp_panel *msm_dp_panel; + + if (!drm_bridge) { + DRM_ERROR("Invalid params\n"); + return; + } + + bridge = to_dp_mst_bridge(drm_bridge); + mst_bridge_state = to_dp_mst_bridge_state(bridge); + dp_display = bridge->display; + dp_mst = dp_display->msm_dp_mst; + + /* to cover cases of bridge_disable/bridge_enable without modeset */ + bridge->connector = mst_bridge_state->connector; + bridge->msm_dp_panel = mst_bridge_state->msm_dp_panel; + + if (!bridge->connector) { + DRM_ERROR("Invalid connector\n"); + return; + } + + msm_dp_panel = bridge->msm_dp_panel; + mutex_lock(&dp_mst->mst_lock); + + rc = msm_dp_display_set_mode_helper(dp_display, state, drm_bridge->encoder, msm_dp_panel); + if (rc) { + DRM_ERROR("Failed to perform a mode set, rc=%d\n", rc); + mutex_unlock(&dp_mst->mst_lock); + return; + } + msm_dp_panel->pbn = drm_dp_calc_pbn_mode(msm_dp_panel->msm_dp_mode.drm_mode.clock, + msm_dp_panel->msm_dp_mode.bpp << 4); + rc = msm_dp_display_prepare(dp_display); + if (rc) { + DRM_ERROR("[%d] DP display pre-enable failed, rc=%d\n", bridge->id, rc); + msm_dp_display_unprepare(dp_display); + mutex_unlock(&dp_mst->mst_lock); + return; + } + + rc = msm_dp_mst_bridge_pre_enable_part1(bridge, state); + if (rc) { + DRM_ERROR("[%d] DP display pre-enable failed, rc=%d\n", bridge->id, rc); + mutex_unlock(&dp_mst->mst_lock); + return; + } + + msm_dp_display_enable_helper(dp_display, bridge->msm_dp_panel); + + _msm_dp_mst_bridge_pre_enable_part2(bridge, state); + + mutex_unlock(&dp_mst->mst_lock); + + drm_dbg_dp(dp_display->drm_dev, "conn:%d mode:%s pre enable done\n", + DP_MST_CONN_ID(bridge), bridge->msm_dp_panel->msm_dp_mode.drm_mode.name); +} + +static void msm_dp_mst_bridge_atomic_disable(struct drm_bridge *drm_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp_mst_bridge *bridge; + struct msm_dp *dp_display; + struct msm_dp_mst *mst; + + if (!drm_bridge) { + DRM_ERROR("Invalid params\n"); + return; + } + + bridge = to_dp_mst_bridge(drm_bridge); + if (!bridge->connector) { + DRM_ERROR("Invalid connector\n"); + return; + } + + dp_display = bridge->display; + mst = dp_display->msm_dp_mst; + + mutex_lock(&mst->mst_lock); + + msm_dp_mst_bridge_pre_disable_part1(bridge, state); + + msm_dp_display_disable_helper(dp_display, bridge->msm_dp_panel); + + drm_dp_check_act_status(&mst->mst_mgr); + + mutex_unlock(&mst->mst_lock); + + drm_dbg_dp(dp_display->drm_dev, "MST bridge:%d disable complete\n", bridge->id); +} + +static void msm_dp_mst_bridge_atomic_post_disable(struct drm_bridge *drm_bridge, + struct drm_atomic_state *state) +{ + struct msm_dp_mst_bridge *bridge; + struct msm_dp *dp_display; + struct msm_dp_mst *mst; + + if (!drm_bridge) { + DRM_ERROR("Invalid params\n"); + return; + } + + bridge = to_dp_mst_bridge(drm_bridge); + if (!bridge->connector) { + DRM_ERROR("Invalid connector\n"); + return; + } + + dp_display = bridge->display; + mst = dp_display->msm_dp_mst; + + mutex_lock(&mst->mst_lock); + + msm_dp_display_atomic_post_disable_helper(dp_display, bridge->msm_dp_panel); + + if (!dp_display->mst_active) + msm_dp_display_unprepare(dp_display); + + bridge->connector = NULL; + bridge->msm_dp_panel = NULL; + + mutex_unlock(&mst->mst_lock); + + drm_dbg_dp(dp_display->drm_dev, "MST bridge:%d conn:%d post disable complete\n", + bridge->id, DP_MST_CONN_ID(bridge)); +} + +static int msm_dp_mst_bridge_atomic_check(struct drm_bridge *drm_bridge, + struct drm_bridge_state *bridge_state, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + struct drm_atomic_state *state = crtc_state->state; + struct drm_connector *connector = conn_state->connector; + struct drm_dp_mst_topology_state *mst_state; + struct msm_dp_mst_connector *mst_conn; + struct msm_dp_mst *mst; + int rc = 0, pbn, slots; + struct msm_dp_mst_bridge_state *mst_bridge_state; + u32 bpp; + + if (!drm_atomic_crtc_needs_modeset(crtc_state) || !crtc_state->enable) + return 0; + + mst_conn = to_dp_mst_connector(connector); + mst = mst_conn->dp_mst; + + bpp = connector->display_info.bpc * 3; + + if (!bpp) + bpp = 24; + + pbn = drm_dp_calc_pbn_mode(crtc_state->mode.clock, bpp << 4); + + mst_state = to_drm_dp_mst_topology_state(mst->mst_mgr.base.state); + if (IS_ERR(mst_state)) + return PTR_ERR(mst_state); + + if (!dfixed_trunc(mst_state->pbn_div)) { + mst_state->pbn_div = + drm_dp_get_vc_payload_bw(mst_conn->dp_panel->link_info.rate, + mst_conn->dp_panel->link_info.num_lanes); + } + + slots = drm_dp_atomic_find_time_slots(state, &mst->mst_mgr, mst_conn->mst_port, pbn); + + drm_dbg_dp(drm_bridge->dev, "add slots, conn:%d pbn:%d slots:%d rc:%d\n", + connector->base.id, pbn, slots, rc); + + if (!conn_state->crtc) { + mst_bridge_state = msm_dp_mst_br_priv_state(state, to_dp_mst_bridge(drm_bridge)); + mst_bridge_state->connector = NULL; + mst_bridge_state->msm_dp_panel = NULL; + } + + return 0; +} + +/* DP MST Bridge APIs */ +static const struct drm_bridge_funcs msm_dp_mst_bridge_ops = { + .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, + .atomic_reset = drm_atomic_helper_bridge_reset, + .atomic_pre_enable = msm_dp_mst_bridge_atomic_pre_enable, + .atomic_disable = msm_dp_mst_bridge_atomic_disable, + .atomic_post_disable = msm_dp_mst_bridge_atomic_post_disable, + .atomic_check = msm_dp_mst_bridge_atomic_check, +}; + +int msm_dp_mst_attach_encoder(struct msm_dp *dp_display, struct drm_encoder *encoder) +{ + int rc = 0; + struct msm_dp_mst_bridge *bridge = NULL; + struct msm_dp_mst_bridge_state *mst_bridge_state; + struct drm_device *dev; + struct msm_dp_mst *mst = dp_display->msm_dp_mst; + int i; + + for (i = 0; i < mst->max_streams; i++) { + if (!mst->mst_bridge[i]->initialized) { + bridge = mst->mst_bridge[i]; + bridge->encoder = encoder; + bridge->initialized = true; + bridge->id = i; + break; + } + } + + if (i == mst->max_streams) { + DRM_ERROR("MST supports only %d bridges\n", mst->max_streams); + rc = -EACCES; + goto end; + } + + dev = dp_display->drm_dev; + bridge->display = dp_display; + bridge->base.encoder = encoder; + bridge->base.type = dp_display->connector_type; + bridge->base.ops = DRM_BRIDGE_OP_MODES; + drm_bridge_add(&bridge->base); + + rc = drm_bridge_attach(encoder, &bridge->base, NULL, 0); + if (rc) { + DRM_ERROR("failed to attach bridge, rc=%d\n", rc); + goto end; + } + + mst_bridge_state = kzalloc(sizeof(*mst_bridge_state), GFP_KERNEL); + if (!mst_bridge_state) { + rc = -ENOMEM; + goto end; + } + + drm_atomic_private_obj_init(dev, &bridge->obj, + &mst_bridge_state->base, + &msm_dp_mst_bridge_state_funcs); + + drm_dbg_dp(dp_display->drm_dev, "MST drm bridge init. bridge id:%d\n", i); + + return 0; + +end: + return rc; +} + +int msm_dp_mst_display_set_mgr_state(struct msm_dp *dp_display, bool state) +{ + struct msm_dp_mst *mst = dp_display->msm_dp_mst; + int rc; + + rc = drm_dp_mst_topology_mgr_set_mst(&mst->mst_mgr, state); + if (rc < 0) { + DRM_ERROR("failed to set topology mgr state to %d. rc %d\n", + state, rc); + } + + drm_dbg_dp(dp_display->drm_dev, "dp_mst_display_set_mgr_state state:%d\n", state); + return rc; +} + +/* DP MST HPD IRQ callback */ +void msm_dp_mst_display_hpd_irq(struct msm_dp *dp_display) +{ + int rc; + struct msm_dp_mst *mst = dp_display->msm_dp_mst; + u8 ack[8] = {}; + u8 esi[4]; + unsigned int esi_res = DP_SINK_COUNT_ESI + 1; + bool handled; + + rc = drm_dp_dpcd_read_data(mst->dp_aux, DP_SINK_COUNT_ESI, esi, 4); + if (rc < 0) { + DRM_ERROR("DPCD sink status read failed, rlen=%d\n", rc); + return; + } + + drm_dbg_dp(dp_display->drm_dev, "MST irq: esi1[0x%x] esi2[0x%x] esi3[%x]\n", + esi[1], esi[2], esi[3]); + + rc = drm_dp_mst_hpd_irq_handle_event(&mst->mst_mgr, esi, ack, &handled); + + /* ack the request */ + if (handled) { + rc = drm_dp_dpcd_write_byte(mst->dp_aux, esi_res, ack[1]); + if (rc < 0) { + DRM_ERROR("DPCD esi_res failed. rc=%d\n", rc); + return; + } + + drm_dp_mst_hpd_irq_send_new_request(&mst->mst_mgr); + } + drm_dbg_dp(dp_display->drm_dev, "MST display hpd_irq handled:%d rc:%d\n", handled, rc); +} + +/* DP MST Connector OPs */ +static int +msm_dp_mst_connector_detect(struct drm_connector *connector, + struct drm_modeset_acquire_ctx *ctx, + bool force) +{ + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(connector); + struct msm_dp_mst *mst = mst_conn->dp_mst; + struct msm_dp *dp_display = mst->msm_dp; + struct device *dev = dp_display->drm_dev->dev; + enum drm_connector_status status = connector_status_disconnected; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return status; + + if (dp_display->mst_active) + status = drm_dp_mst_detect_port(connector, + ctx, &mst->mst_mgr, mst_conn->mst_port); + + pm_runtime_put_autosuspend(dev); + + return status; +} + +static int msm_dp_mst_connector_get_modes(struct drm_connector *connector) +{ + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(connector); + struct msm_dp_mst *mst = mst_conn->dp_mst; + const struct drm_edid *drm_edid; + + drm_edid = drm_dp_mst_edid_read(connector, &mst->mst_mgr, mst_conn->mst_port); + drm_edid_connector_update(connector, drm_edid); + + return drm_edid_connector_add_modes(connector); +} + +static enum drm_mode_status msm_dp_mst_connector_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode) +{ + struct msm_dp_mst_connector *mst_conn; + struct drm_dp_mst_port *mst_port; + struct msm_dp *dp_display; + int required_pbn; + + if (drm_connector_is_unregistered(connector)) + return 0; + + mst_conn = to_dp_mst_connector(connector); + mst_port = mst_conn->mst_port; + dp_display = mst_conn->dp_mst->msm_dp; + + if (!mst_port) + return MODE_ERROR; + + required_pbn = drm_dp_calc_pbn_mode(mode->clock, (6 * 3) << 4); + + if (required_pbn > mst_port->full_pbn) { + drm_dbg_dp(dp_display->drm_dev, "mode:%s not supported.\n", mode->name); + return MODE_CLOCK_HIGH; + } + + return msm_dp_display_mode_valid(dp_display, &connector->display_info, mode); +} + +static struct drm_encoder * +msm_dp_mst_atomic_best_encoder(struct drm_connector *connector, struct drm_atomic_state *state) +{ + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(connector); + struct msm_dp_mst *mst = mst_conn->dp_mst; + struct msm_dp *dp_display = mst->msm_dp; + struct drm_encoder *enc = NULL; + struct msm_dp_mst_bridge_state *mst_bridge_state; + u32 i; + struct drm_connector_state *conn_state = drm_atomic_get_new_connector_state(state, + connector); + + if (conn_state && conn_state->best_encoder) + return conn_state->best_encoder; + + for (i = 0; i < mst->max_streams; i++) { + mst_bridge_state = msm_dp_mst_br_priv_state(state, mst->mst_bridge[i]); + if (IS_ERR(mst_bridge_state)) + goto end; + + if (mst_bridge_state->connector == connector) { + enc = mst->mst_bridge[i]->encoder; + goto end; + } + } + + for (i = 0; i < mst->max_streams; i++) { + mst_bridge_state = msm_dp_mst_br_priv_state(state, mst->mst_bridge[i]); + + if (!mst_bridge_state->connector) { + mst_bridge_state->connector = connector; + mst_bridge_state->msm_dp_panel = mst_conn->dp_panel; + enc = mst->mst_bridge[i]->encoder; + break; + } + } + +end: + if (enc) + drm_dbg_dp(dp_display->drm_dev, "MST connector:%d atomic best encoder:%d\n", + connector->base.id, i); + else + drm_dbg_dp(dp_display->drm_dev, "MST connector:%d atomic best encoder failed\n", + connector->base.id); + + return enc; +} + +static int msm_dp_mst_connector_atomic_check(struct drm_connector *connector, + struct drm_atomic_state *state) +{ + int rc = 0, slots; + struct drm_connector_state *old_conn_state; + struct drm_connector_state *new_conn_state; + struct drm_crtc *old_crtc; + struct drm_crtc_state *crtc_state; + struct msm_dp_mst_bridge *bridge; + struct msm_dp_mst_bridge_state *mst_bridge_state; + struct drm_bridge *drm_bridge; + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(connector); + struct msm_dp_mst *mst = mst_conn->dp_mst; + struct msm_dp *dp_display = mst->msm_dp; + struct drm_dp_mst_atomic_payload *payload; + struct drm_dp_mst_topology_state *mst_state; + + if (!state) + return rc; + + new_conn_state = drm_atomic_get_new_connector_state(state, connector); + if (!new_conn_state) + return rc; + + old_conn_state = drm_atomic_get_old_connector_state(state, connector); + if (!old_conn_state) + goto end; + + old_crtc = old_conn_state->crtc; + if (!old_crtc) + goto end; + + crtc_state = drm_atomic_get_new_crtc_state(state, old_crtc); + + /* attempt to release vcpi slots on a modeset change for crtc state */ + if (drm_atomic_crtc_needs_modeset(crtc_state)) { + if (WARN_ON(!old_conn_state->best_encoder)) { + rc = -EINVAL; + goto end; + } + + drm_bridge = drm_bridge_chain_get_first_bridge(old_conn_state->best_encoder); + if (WARN_ON(!drm_bridge)) { + rc = -EINVAL; + goto end; + } + bridge = to_dp_mst_bridge(drm_bridge); + + mst_bridge_state = msm_dp_mst_br_priv_state(state, bridge); + + mst_state = to_drm_dp_mst_topology_state(mst->mst_mgr.base.state); + + payload = drm_atomic_get_mst_payload_state(mst_state, mst_conn->mst_port); + + slots = payload->time_slots; + if (slots > 0) { + rc = drm_dp_atomic_release_time_slots(state, + &mst->mst_mgr, + mst_conn->mst_port); + if (rc) { + DRM_ERROR("failed releasing %d vcpi slots %d\n", slots, rc); + goto end; + } + } + + if (!new_conn_state->crtc) { + /* for cases where crtc is not disabled the slots are not + * freed by drm_dp_atomic_release_time_slots. this results + * in subsequent atomic_check failing since internal slots + * were freed but not the DP MST mgr's + */ + mst_bridge_state->connector = NULL; + mst_bridge_state->msm_dp_panel = NULL; + drm_dbg_dp(dp_display->drm_dev, "clear best encoder: %d\n", bridge->id); + } + } + +end: + drm_dbg_dp(dp_display->drm_dev, "mst connector:%d atomic check ret %d\n", + connector->base.id, rc); + return rc; +} + +static void dp_mst_connector_destroy(struct drm_connector *connector) +{ + struct msm_dp_mst_connector *mst_conn = to_dp_mst_connector(connector); + + drm_connector_cleanup(connector); + drm_dp_mst_put_port_malloc(mst_conn->mst_port); + kfree(mst_conn); +} + +/* DRM MST callbacks */ +static const struct drm_connector_helper_funcs msm_dp_drm_mst_connector_helper_funcs = { + .get_modes = msm_dp_mst_connector_get_modes, + .detect_ctx = msm_dp_mst_connector_detect, + .mode_valid = msm_dp_mst_connector_mode_valid, + .atomic_best_encoder = msm_dp_mst_atomic_best_encoder, + .atomic_check = msm_dp_mst_connector_atomic_check, +}; + +static const struct drm_connector_funcs msm_dp_drm_mst_connector_funcs = { + .reset = drm_atomic_helper_connector_reset, + .destroy = dp_mst_connector_destroy, + .fill_modes = drm_helper_probe_single_connector_modes, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static struct drm_connector * +msm_dp_mst_add_connector(struct drm_dp_mst_topology_mgr *mgr, + struct drm_dp_mst_port *port, const char *pathprop) +{ + struct msm_dp_mst *dp_mst; + struct drm_device *dev; + struct msm_dp *dp_display; + struct msm_dp_mst_connector *mst_conn; + struct drm_connector *connector; + int rc, i; + + dp_mst = container_of(mgr, struct msm_dp_mst, mst_mgr); + + dp_display = dp_mst->msm_dp; + dev = dp_display->drm_dev; + + mst_conn = kzalloc(sizeof(*mst_conn), GFP_KERNEL); + + if (!mst_conn) + return NULL; + + drm_modeset_lock_all(dev); + + connector = &mst_conn->connector; + rc = drm_connector_dynamic_init(dev, connector, + &msm_dp_drm_mst_connector_funcs, + DRM_MODE_CONNECTOR_DisplayPort, NULL); + if (rc) { + kfree(mst_conn); + drm_modeset_unlock_all(dev); + return NULL; + } + + mst_conn->dp_panel = msm_dp_display_get_panel(dp_display); + if (!mst_conn->dp_panel) { + DRM_ERROR("failed to get dp_panel for connector\n"); + kfree(mst_conn); + drm_modeset_unlock_all(dev); + return NULL; + } + + mst_conn->dp_panel->connector = connector; + mst_conn->dp_mst = dp_mst; + + drm_connector_helper_add(connector, &msm_dp_drm_mst_connector_helper_funcs); + + if (connector->funcs->reset) + connector->funcs->reset(connector); + + /* add all encoders as possible encoders */ + for (i = 0; i < dp_mst->max_streams; i++) { + rc = drm_connector_attach_encoder(connector, dp_mst->mst_bridge[i]->encoder); + + if (rc) { + DRM_ERROR("failed to attach encoder to connector, %d\n", rc); + kfree(mst_conn); + drm_modeset_unlock_all(dev); + return NULL; + } + } + + mst_conn->mst_port = port; + drm_dp_mst_get_port_malloc(mst_conn->mst_port); + + drm_object_attach_property(&connector->base, + dev->mode_config.path_property, 0); + drm_object_attach_property(&connector->base, + dev->mode_config.tile_property, 0); + drm_connector_set_path_property(connector, pathprop); + drm_modeset_unlock_all(dev); + + drm_dbg_dp(dp_display->drm_dev, "add MST connector id:%d\n", connector->base.id); + + return connector; +} + +static const struct drm_dp_mst_topology_cbs msm_dp_mst_drm_cbs = { + .add_connector = msm_dp_mst_add_connector, +}; + +int msm_dp_mst_init(struct msm_dp *dp_display, u32 max_streams, struct drm_dp_aux *drm_aux) +{ + struct drm_device *dev = dp_display->drm_dev; + int conn_base_id = 0; + int ret; + struct msm_dp_mst *msm_dp_mst; + + msm_dp_mst = devm_kzalloc(dev->dev, sizeof(*msm_dp_mst), GFP_KERNEL); + if (!msm_dp_mst) + return -ENOMEM; + + memset(&msm_dp_mst->mst_mgr, 0, sizeof(msm_dp_mst->mst_mgr)); + msm_dp_mst->mst_mgr.cbs = &msm_dp_mst_drm_cbs; + conn_base_id = dp_display->connector->base.id; + msm_dp_mst->msm_dp = dp_display; + msm_dp_mst->max_streams = max_streams; + + for (int i = 0; i < DP_STREAM_MAX; i++) { + msm_dp_mst->mst_bridge[i] = + devm_drm_bridge_alloc(dev->dev, struct msm_dp_mst_bridge, base, + &msm_dp_mst_bridge_ops); + } + + msm_dp_mst->dp_aux = drm_aux; + + ret = drm_dp_mst_topology_mgr_init(&msm_dp_mst->mst_mgr, dev, + drm_aux, + MAX_DPCD_TRANSACTION_BYTES, + max_streams, + conn_base_id); + if (ret) { + DRM_ERROR("DP DRM MST topology manager init failed\n"); + return ret; + } + + dp_display->msm_dp_mst = msm_dp_mst; + + mutex_init(&msm_dp_mst->mst_lock); + return ret; +} diff --git a/drivers/gpu/drm/msm/dp/dp_mst_drm.h b/drivers/gpu/drm/msm/dp/dp_mst_drm.h new file mode 100644 index 0000000000000..08e145399cfc7 --- /dev/null +++ b/drivers/gpu/drm/msm/dp/dp_mst_drm.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _DP_MST_DRM_H_ +#define _DP_MST_DRM_H_ + +#include "dp_display.h" + +int msm_dp_mst_init(struct msm_dp *dp_display, u32 max_streams, struct drm_dp_aux *drm_aux); +int msm_dp_mst_display_set_mgr_state(struct msm_dp *dp_display, bool state); +void msm_dp_mst_display_hpd_irq(struct msm_dp *dp_display); + +#endif /* _DP_MST_DRM_H_ */ diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c index 15b7f6c7146e1..e05d96f33c433 100644 --- a/drivers/gpu/drm/msm/dp/dp_panel.c +++ b/drivers/gpu/drm/msm/dp/dp_panel.c @@ -13,12 +13,11 @@ #include #include +#include +#include #define DP_INTF_CONFIG_DATABUS_WIDEN BIT(4) -#define DP_MAX_NUM_DP_LANES 4 -#define DP_LINK_RATE_HBR2 540000 /* kbytes */ - struct msm_dp_panel_private { struct device *dev; struct drm_device *drm_dev; @@ -26,43 +25,70 @@ struct msm_dp_panel_private { struct drm_dp_aux *aux; struct msm_dp_link *link; void __iomem *link_base; - void __iomem *p0_base; + void __iomem *mst2link_base; + void __iomem *mst3link_base; + void __iomem *pixel_base; bool panel_on; }; static inline u32 msm_dp_read_link(struct msm_dp_panel_private *panel, u32 offset) { - return readl_relaxed(panel->link_base + offset); + switch (panel->msm_dp_panel.stream_id) { + case DP_STREAM_0: + case DP_STREAM_1: + return readl_relaxed(panel->link_base + offset); + case DP_STREAM_2: + return readl_relaxed(panel->mst2link_base + offset); + case DP_STREAM_3: + return readl_relaxed(panel->mst3link_base + offset); + default: + DRM_ERROR("error stream_id\n"); + return 0; + } } static inline void msm_dp_write_link(struct msm_dp_panel_private *panel, - u32 offset, u32 data) + u32 offset, u32 data) { /* * To make sure link reg writes happens before any other operation, * this function uses writel() instread of writel_relaxed() */ - writel(data, panel->link_base + offset); + switch (panel->msm_dp_panel.stream_id) { + case DP_STREAM_0: + case DP_STREAM_1: + writel(data, panel->link_base + offset); + break; + case DP_STREAM_2: + writel(data, panel->mst2link_base + offset); + break; + case DP_STREAM_3: + writel(data, panel->mst3link_base + offset); + break; + default: + DRM_ERROR("error stream_id\n"); + break; + } } -static inline void msm_dp_write_p0(struct msm_dp_panel_private *panel, - u32 offset, u32 data) +static inline void msm_dp_write_pn(struct msm_dp_panel_private *panel, + u32 offset, u32 data) { /* * To make sure interface reg writes happens before any other operation, * this function uses writel() instread of writel_relaxed() */ - writel(data, panel->p0_base + offset); + writel(data, panel->pixel_base + offset); } -static inline u32 msm_dp_read_p0(struct msm_dp_panel_private *panel, - u32 offset) +static inline u32 msm_dp_read_pn(struct msm_dp_panel_private *panel, + u32 offset) { /* * To make sure interface reg writes happens before any other operation, * this function uses writel() instread of writel_relaxed() */ - return readl_relaxed(panel->p0_base + offset); + return readl_relaxed(panel->pixel_base + offset); } static void msm_dp_panel_read_psr_cap(struct msm_dp_panel_private *panel) @@ -91,6 +117,7 @@ static int msm_dp_panel_read_dpcd(struct msm_dp_panel *msm_dp_panel) int rc, max_lttpr_lanes, max_lttpr_rate; struct msm_dp_panel_private *panel; struct msm_dp_link_info *link_info; + struct msm_dp_link *link; u8 *dpcd, major, minor; panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); @@ -105,29 +132,102 @@ static int msm_dp_panel_read_dpcd(struct msm_dp_panel *msm_dp_panel) major = (link_info->revision >> 4) & 0x0f; minor = link_info->revision & 0x0f; - link_info->rate = drm_dp_max_link_rate(dpcd); + link = panel->link; + drm_dbg_dp(panel->drm_dev, "max_lanes=%d max_link_rate=%d\n", + link->max_dp_lanes, link->max_dp_link_rate); + + max_lttpr_lanes = drm_dp_lttpr_max_lane_count(link->lttpr_common_caps); + max_lttpr_rate = drm_dp_lttpr_max_link_rate(link->lttpr_common_caps); + + /* eDP sink */ + if (msm_dp_panel->dpcd[DP_EDP_CONFIGURATION_CAP]) { + u8 edp_rev; + + rc = drm_dp_dpcd_read_byte(panel->aux, DP_EDP_DPCD_REV, &edp_rev); + if (rc) + return rc; + + drm_dbg_dp(panel->drm_dev, "edp_rev=0x%x\n", edp_rev); + + /* For eDP v1.4+, parse the SUPPORTED_LINK_RATES table */ + if (edp_rev >= DP_EDP_14) { + __le16 rates[DP_MAX_SUPPORTED_RATES]; + u8 bw_set; + int i; + + rc = drm_dp_dpcd_read_data(panel->aux, DP_SUPPORTED_LINK_RATES, + rates, sizeof(rates)); + if (rc) + return rc; + + rc = drm_dp_dpcd_read_byte(panel->aux, DP_LINK_BW_SET, &bw_set); + if (rc) + return rc; + + /* Find index of max supported link rate that does not exceed dtsi limits */ + for (i = 0; i < ARRAY_SIZE(rates); i++) { + /* + * The value from the DPCD multiplied by 200 gives + * the link rate in kHz. Divide by 10 to convert to + * symbol rate, accounting for 8b/10b encoding. + */ + u32 rate = (le16_to_cpu(rates[i]) * 200) / 10; + + if (!rate) + break; + + drm_dbg_dp(panel->drm_dev, + "SUPPORTED_LINK_RATES[%d]: %d\n", i, rate); + + /* + * Limit link rate from link-frequencies of endpoint + * property of dtsi + */ + if (rate > link->max_dp_link_rate) + break; + + /* Limit link rate from LTTPR capabilities, if any */ + if (max_lttpr_rate && rate > max_lttpr_rate) + break; + + link_info->rate = rate; + link_info->supported_rates[i] = rate; + link_info->rate_set = i; + } + + /* Only use LINK_RATE_SET if LINK_BW_SET hasn't already been written to */ + if (!bw_set && link_info->rate) + link_info->use_rate_set = true; + } + } + + /* Fall back on MAX_LINK_RATE/LINK_BW_SET (DP, eDP <= v1.3) */ + if (!link_info->rate) { + link_info->rate = drm_dp_max_link_rate(dpcd); + + /* Limit link rate from link-frequencies of endpoint property of dtsi */ + if (link_info->rate > link->max_dp_link_rate) + link_info->rate = link->max_dp_link_rate; + + /* Limit link rate from LTTPR capabilities, if any */ + if (max_lttpr_rate && max_lttpr_rate < link_info->rate) + link_info->rate = max_lttpr_rate; + } + link_info->num_lanes = drm_dp_max_lane_count(dpcd); /* Limit data lanes from data-lanes of endpoint property of dtsi */ - if (link_info->num_lanes > msm_dp_panel->max_dp_lanes) - link_info->num_lanes = msm_dp_panel->max_dp_lanes; - - /* Limit link rate from link-frequencies of endpoint property of dtsi */ - if (link_info->rate > msm_dp_panel->max_dp_link_rate) - link_info->rate = msm_dp_panel->max_dp_link_rate; + if (link_info->num_lanes > link->max_dp_lanes) + link_info->num_lanes = link->max_dp_lanes; /* Limit data lanes from LTTPR capabilities, if any */ - max_lttpr_lanes = drm_dp_lttpr_max_lane_count(panel->link->lttpr_common_caps); if (max_lttpr_lanes && max_lttpr_lanes < link_info->num_lanes) link_info->num_lanes = max_lttpr_lanes; - /* Limit link rate from LTTPR capabilities, if any */ - max_lttpr_rate = drm_dp_lttpr_max_link_rate(panel->link->lttpr_common_caps); - if (max_lttpr_rate && max_lttpr_rate < link_info->rate) - link_info->rate = max_lttpr_rate; - drm_dbg_dp(panel->drm_dev, "version: %d.%d\n", major, minor); drm_dbg_dp(panel->drm_dev, "link_rate=%d\n", link_info->rate); + drm_dbg_dp(panel->drm_dev, "link_rate_set=%d\n", link_info->rate_set); + drm_dbg_dp(panel->drm_dev, "use_rate_set=%d\n", link_info->use_rate_set); drm_dbg_dp(panel->drm_dev, "lane_count=%d\n", link_info->num_lanes); if (drm_dp_enhanced_frame_cap(dpcd)) @@ -159,8 +259,8 @@ static u32 msm_dp_panel_get_supported_bpp(struct msm_dp_panel *msm_dp_panel, return min_supported_bpp; } -int msm_dp_panel_read_sink_caps(struct msm_dp_panel *msm_dp_panel, - struct drm_connector *connector) +int msm_dp_panel_read_link_caps(struct msm_dp_panel *msm_dp_panel, + struct drm_connector *connector) { int rc, bw_code; int count; @@ -173,9 +273,6 @@ int msm_dp_panel_read_sink_caps(struct msm_dp_panel *msm_dp_panel, panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); - drm_dbg_dp(panel->drm_dev, "max_lanes=%d max_link_rate=%d\n", - msm_dp_panel->max_dp_lanes, msm_dp_panel->max_dp_link_rate); - rc = msm_dp_panel_read_dpcd(msm_dp_panel); if (rc) { DRM_ERROR("read dpcd failed %d\n", rc); @@ -201,25 +298,6 @@ int msm_dp_panel_read_sink_caps(struct msm_dp_panel *msm_dp_panel, rc = drm_dp_read_downstream_info(panel->aux, msm_dp_panel->dpcd, msm_dp_panel->downstream_ports); - if (rc) - return rc; - - drm_edid_free(msm_dp_panel->drm_edid); - - msm_dp_panel->drm_edid = drm_edid_read_ddc(connector, &panel->aux->ddc); - - drm_edid_connector_update(connector, msm_dp_panel->drm_edid); - - if (!msm_dp_panel->drm_edid) { - DRM_ERROR("panel edid read failed\n"); - /* check edid read fail is due to unplug */ - if (!msm_dp_aux_is_link_connected(panel->aux)) { - rc = -ETIMEDOUT; - goto end; - } - } - -end: return rc; } @@ -246,20 +324,6 @@ u32 msm_dp_panel_get_mode_bpp(struct msm_dp_panel *msm_dp_panel, return bpp; } -int msm_dp_panel_get_modes(struct msm_dp_panel *msm_dp_panel, - struct drm_connector *connector) -{ - if (!msm_dp_panel) { - DRM_ERROR("invalid input\n"); - return -EINVAL; - } - - if (msm_dp_panel->drm_edid) - return drm_edid_connector_add_modes(connector); - - return 0; -} - static u8 msm_dp_panel_get_edid_checksum(const struct edid *edid) { edid += edid->extensions; @@ -267,7 +331,8 @@ static u8 msm_dp_panel_get_edid_checksum(const struct edid *edid) return edid->checksum; } -void msm_dp_panel_handle_sink_request(struct msm_dp_panel *msm_dp_panel) +void msm_dp_panel_handle_sink_request(struct msm_dp_panel *msm_dp_panel, + const struct drm_edid *drm_edid) { struct msm_dp_panel_private *panel; @@ -280,7 +345,7 @@ void msm_dp_panel_handle_sink_request(struct msm_dp_panel *msm_dp_panel) if (panel->link->sink_request & DP_TEST_LINK_EDID_READ) { /* FIXME: get rid of drm_edid_raw() */ - const struct edid *edid = drm_edid_raw(msm_dp_panel->drm_edid); + const struct edid *edid = drm_edid_raw(drm_edid); u8 checksum; if (edid) @@ -329,34 +394,34 @@ static void msm_dp_panel_tpg_enable(struct msm_dp_panel *msm_dp_panel, display_hctl = (hsync_end_x << 16) | hsync_start_x; - msm_dp_write_p0(panel, MMSS_DP_INTF_HSYNC_CTL, hsync_ctl); - msm_dp_write_p0(panel, MMSS_DP_INTF_VSYNC_PERIOD_F0, vsync_period * + msm_dp_write_pn(panel, MMSS_DP_INTF_HSYNC_CTL, hsync_ctl); + msm_dp_write_pn(panel, MMSS_DP_INTF_VSYNC_PERIOD_F0, vsync_period * hsync_period); - msm_dp_write_p0(panel, MMSS_DP_INTF_VSYNC_PULSE_WIDTH_F0, v_sync_width * + msm_dp_write_pn(panel, MMSS_DP_INTF_VSYNC_PULSE_WIDTH_F0, v_sync_width * hsync_period); - msm_dp_write_p0(panel, MMSS_DP_INTF_VSYNC_PERIOD_F1, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_VSYNC_PULSE_WIDTH_F1, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_DISPLAY_HCTL, display_hctl); - msm_dp_write_p0(panel, MMSS_DP_INTF_ACTIVE_HCTL, 0); - msm_dp_write_p0(panel, MMSS_INTF_DISPLAY_V_START_F0, display_v_start); - msm_dp_write_p0(panel, MMSS_DP_INTF_DISPLAY_V_END_F0, display_v_end); - msm_dp_write_p0(panel, MMSS_INTF_DISPLAY_V_START_F1, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_DISPLAY_V_END_F1, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_ACTIVE_V_START_F0, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_ACTIVE_V_END_F0, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_ACTIVE_V_START_F1, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_ACTIVE_V_END_F1, 0); - msm_dp_write_p0(panel, MMSS_DP_INTF_POLARITY_CTL, 0); - - msm_dp_write_p0(panel, MMSS_DP_TPG_MAIN_CONTROL, - DP_TPG_CHECKERED_RECT_PATTERN); - msm_dp_write_p0(panel, MMSS_DP_TPG_VIDEO_CONFIG, - DP_TPG_VIDEO_CONFIG_BPP_8BIT | - DP_TPG_VIDEO_CONFIG_RGB); - msm_dp_write_p0(panel, MMSS_DP_BIST_ENABLE, - DP_BIST_ENABLE_DPBIST_EN); - msm_dp_write_p0(panel, MMSS_DP_TIMING_ENGINE_EN, - DP_TIMING_ENGINE_EN_EN); + msm_dp_write_pn(panel, MMSS_DP_INTF_VSYNC_PERIOD_F1, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_VSYNC_PULSE_WIDTH_F1, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_DISPLAY_HCTL, display_hctl); + msm_dp_write_pn(panel, MMSS_DP_INTF_ACTIVE_HCTL, 0); + msm_dp_write_pn(panel, MMSS_INTF_DISPLAY_V_START_F0, display_v_start); + msm_dp_write_pn(panel, MMSS_DP_INTF_DISPLAY_V_END_F0, display_v_end); + msm_dp_write_pn(panel, MMSS_INTF_DISPLAY_V_START_F1, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_DISPLAY_V_END_F1, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_ACTIVE_V_START_F0, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_ACTIVE_V_END_F0, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_ACTIVE_V_START_F1, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_ACTIVE_V_END_F1, 0); + msm_dp_write_pn(panel, MMSS_DP_INTF_POLARITY_CTL, 0); + + msm_dp_write_pn(panel, MMSS_DP_TPG_MAIN_CONTROL, + DP_TPG_CHECKERED_RECT_PATTERN); + msm_dp_write_pn(panel, MMSS_DP_TPG_VIDEO_CONFIG, + DP_TPG_VIDEO_CONFIG_BPP_8BIT | + DP_TPG_VIDEO_CONFIG_RGB); + msm_dp_write_pn(panel, MMSS_DP_BIST_ENABLE, + DP_BIST_ENABLE_DPBIST_EN); + msm_dp_write_pn(panel, MMSS_DP_TIMING_ENGINE_EN, + DP_TIMING_ENGINE_EN_EN); drm_dbg_dp(panel->drm_dev, "%s: enabled tpg\n", __func__); } @@ -365,9 +430,9 @@ static void msm_dp_panel_tpg_disable(struct msm_dp_panel *msm_dp_panel) struct msm_dp_panel_private *panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); - msm_dp_write_p0(panel, MMSS_DP_TPG_MAIN_CONTROL, 0x0); - msm_dp_write_p0(panel, MMSS_DP_BIST_ENABLE, 0x0); - msm_dp_write_p0(panel, MMSS_DP_TIMING_ENGINE_EN, 0x0); + msm_dp_write_pn(panel, MMSS_DP_TPG_MAIN_CONTROL, 0x0); + msm_dp_write_pn(panel, MMSS_DP_BIST_ENABLE, 0x0); + msm_dp_write_pn(panel, MMSS_DP_TIMING_ENGINE_EN, 0x0); } void msm_dp_panel_tpg_config(struct msm_dp_panel *msm_dp_panel, bool enable) @@ -401,35 +466,56 @@ void msm_dp_panel_clear_dsc_dto(struct msm_dp_panel *msm_dp_panel) struct msm_dp_panel_private *panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); - msm_dp_write_p0(panel, MMSS_DP_DSC_DTO, 0x0); + msm_dp_write_pn(panel, MMSS_DP_DSC_DTO, 0x0); } static void msm_dp_panel_send_vsc_sdp(struct msm_dp_panel_private *panel, struct dp_sdp *vsc_sdp) { + u32 id = panel->msm_dp_panel.stream_id; u32 header[2]; u32 val; int i; + u32 offset = 0; + + if (id == DP_STREAM_1) + offset = MMSS_DP1_GENERIC0_0 - MMSS_DP_GENERIC0_0; msm_dp_utils_pack_sdp_header(&vsc_sdp->sdp_header, header); - msm_dp_write_link(panel, MMSS_DP_GENERIC0_0, header[0]); - msm_dp_write_link(panel, MMSS_DP_GENERIC0_1, header[1]); + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_GENERIC0_0 : MMSS_DP_GENERIC0_0 + offset, + header[0]); + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_GENERIC0_1 : MMSS_DP_GENERIC0_1 + offset, + header[1]); for (i = 0; i < sizeof(vsc_sdp->db); i += 4) { val = ((vsc_sdp->db[i]) | (vsc_sdp->db[i + 1] << 8) | (vsc_sdp->db[i + 2] << 16) | (vsc_sdp->db[i + 3] << 24)); - msm_dp_write_link(panel, MMSS_DP_GENERIC0_2 + i, val); + + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_GENERIC0_2 + i : MMSS_DP_GENERIC0_2 + i + offset, + val); } } static void msm_dp_panel_update_sdp(struct msm_dp_panel_private *panel) { + u32 id = panel->msm_dp_panel.stream_id; u32 hw_revision = panel->msm_dp_panel.hw_revision; + u32 offset = 0; + + if (id == DP_STREAM_1) + offset = MMSS_DP1_SDP_CFG3 - MMSS_DP_SDP_CFG3; if (hw_revision >= DP_HW_VERSION_1_0 && hw_revision < DP_HW_VERSION_1_2) { - msm_dp_write_link(panel, MMSS_DP_SDP_CFG3, UPDATE_SDP); - msm_dp_write_link(panel, MMSS_DP_SDP_CFG3, 0x0); + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG3 : MMSS_DP_SDP_CFG3 + offset, + UPDATE_SDP); + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG3 : MMSS_DP_SDP_CFG3 + offset, + 0x0); } } @@ -437,17 +523,34 @@ void msm_dp_panel_enable_vsc_sdp(struct msm_dp_panel *msm_dp_panel, struct dp_sd { struct msm_dp_panel_private *panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); + u32 id = msm_dp_panel->stream_id; u32 cfg, cfg2, misc; + u32 misc_reg_offset = 0; + u32 sdp_cfg_offset = 0; + u32 sdp_cfg2_offset = 0; + + if (id == DP_STREAM_1) { + misc_reg_offset = REG_DP1_MISC1_MISC0 - REG_DP_MISC1_MISC0; + sdp_cfg_offset = MMSS_DP1_SDP_CFG - MMSS_DP_SDP_CFG; + sdp_cfg2_offset = MMSS_DP1_SDP_CFG2 - MMSS_DP_SDP_CFG2; + } - cfg = msm_dp_read_link(panel, MMSS_DP_SDP_CFG); - cfg2 = msm_dp_read_link(panel, MMSS_DP_SDP_CFG2); - misc = msm_dp_read_link(panel, REG_DP_MISC1_MISC0); + cfg = msm_dp_read_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG : MMSS_DP_SDP_CFG + sdp_cfg_offset); + cfg2 = msm_dp_read_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG2 : MMSS_DP_SDP_CFG2 + sdp_cfg2_offset); + misc = msm_dp_read_link(panel, id > 1 ? + REG_DP_MSTLINK_MISC1_MISC0 : REG_DP_MISC1_MISC0 + misc_reg_offset); cfg |= GEN0_SDP_EN; - msm_dp_write_link(panel, MMSS_DP_SDP_CFG, cfg); - cfg2 |= GENERIC0_SDPSIZE_VALID; - msm_dp_write_link(panel, MMSS_DP_SDP_CFG2, cfg2); + + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG : MMSS_DP_SDP_CFG + sdp_cfg_offset, + cfg); + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG2 : MMSS_DP_SDP_CFG2 + sdp_cfg2_offset, + cfg2); msm_dp_panel_send_vsc_sdp(panel, vsc_sdp); @@ -457,7 +560,9 @@ void msm_dp_panel_enable_vsc_sdp(struct msm_dp_panel *msm_dp_panel, struct dp_sd drm_dbg_dp(panel->drm_dev, "vsc sdp enable=1\n"); pr_debug("misc settings = 0x%x\n", misc); - msm_dp_write_link(panel, REG_DP_MISC1_MISC0, misc); + msm_dp_write_link(panel, id > 1 ? + REG_DP_MSTLINK_MISC1_MISC0 : REG_DP_MISC1_MISC0 + misc_reg_offset, + misc); msm_dp_panel_update_sdp(panel); } @@ -466,17 +571,34 @@ void msm_dp_panel_disable_vsc_sdp(struct msm_dp_panel *msm_dp_panel) { struct msm_dp_panel_private *panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); + u32 id = msm_dp_panel->stream_id; u32 cfg, cfg2, misc; + u32 misc_reg_offset = 0; + u32 sdp_cfg_offset = 0; + u32 sdp_cfg2_offset = 0; + + if (id == DP_STREAM_1) { + misc_reg_offset = REG_DP1_MISC1_MISC0 - REG_DP_MISC1_MISC0; + sdp_cfg_offset = MMSS_DP1_SDP_CFG - MMSS_DP_SDP_CFG; + sdp_cfg2_offset = MMSS_DP1_SDP_CFG2 - MMSS_DP_SDP_CFG2; + } - cfg = msm_dp_read_link(panel, MMSS_DP_SDP_CFG); - cfg2 = msm_dp_read_link(panel, MMSS_DP_SDP_CFG2); - misc = msm_dp_read_link(panel, REG_DP_MISC1_MISC0); + cfg = msm_dp_read_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG : MMSS_DP_SDP_CFG + sdp_cfg_offset); + cfg2 = msm_dp_read_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG2 : MMSS_DP_SDP_CFG2 + sdp_cfg2_offset); + misc = msm_dp_read_link(panel, id > 1 ? + REG_DP_MSTLINK_MISC1_MISC0 : REG_DP_MISC1_MISC0 + misc_reg_offset); cfg &= ~GEN0_SDP_EN; - msm_dp_write_link(panel, MMSS_DP_SDP_CFG, cfg); - cfg2 &= ~GENERIC0_SDPSIZE_VALID; - msm_dp_write_link(panel, MMSS_DP_SDP_CFG2, cfg2); + + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG : MMSS_DP_SDP_CFG + sdp_cfg_offset, + cfg); + msm_dp_write_link(panel, id > 1 ? + MMSS_DP_MSTLINK_SDP_CFG2 : MMSS_DP_SDP_CFG2 + sdp_cfg2_offset, + cfg2); /* switch back to MSA */ misc &= ~DP_MISC1_VSC_SDP; @@ -484,7 +606,9 @@ void msm_dp_panel_disable_vsc_sdp(struct msm_dp_panel *msm_dp_panel) drm_dbg_dp(panel->drm_dev, "vsc sdp enable=0\n"); pr_debug("misc settings = 0x%x\n", misc); - msm_dp_write_link(panel, REG_DP_MISC1_MISC0, misc); + msm_dp_write_link(panel, id > 1 ? + REG_DP_MSTLINK_MISC1_MISC0 : REG_DP_MISC1_MISC0 + misc_reg_offset, + misc); msm_dp_panel_update_sdp(panel); } @@ -534,6 +658,7 @@ static int msm_dp_panel_setup_vsc_sdp_yuv_420(struct msm_dp_panel *msm_dp_panel) int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en) { + u32 id = msm_dp_panel->stream_id; u32 data, total_ver, total_hor; struct msm_dp_panel_private *panel; struct drm_display_mode *drm_mode; @@ -542,6 +667,7 @@ int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en) u32 msm_dp_active; u32 total; u32 reg; + u32 offset = 0; panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); drm_mode = &panel->msm_dp_panel.msm_dp_mode.drm_mode; @@ -556,6 +682,9 @@ int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en) drm_mode->vsync_start - drm_mode->vdisplay, drm_mode->vsync_end - drm_mode->vsync_start); + if (id == DP_STREAM_1) + offset = REG_DP1_TOTAL_HOR_VER - REG_DP_TOTAL_HOR_VER; + total_hor = drm_mode->htotal; total_ver = drm_mode->vtotal; @@ -586,12 +715,20 @@ int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en) msm_dp_active = data; - msm_dp_write_link(panel, REG_DP_TOTAL_HOR_VER, total); - msm_dp_write_link(panel, REG_DP_START_HOR_VER_FROM_SYNC, sync_start); - msm_dp_write_link(panel, REG_DP_HSYNC_VSYNC_WIDTH_POLARITY, width_blanking); - msm_dp_write_link(panel, REG_DP_ACTIVE_HOR_VER, msm_dp_active); - - reg = msm_dp_read_p0(panel, MMSS_DP_INTF_CONFIG); + msm_dp_write_link(panel, + id > 1 ? REG_DP_MSTLINK_TOTAL_HOR_VER : + REG_DP_TOTAL_HOR_VER + offset, total); + msm_dp_write_link(panel, + id > 1 ? REG_DP_MSTLINK_START_HOR_VER_FROM_SYNC : + REG_DP_START_HOR_VER_FROM_SYNC + offset, sync_start); + msm_dp_write_link(panel, + id > 1 ? REG_DP_MSTLINK_HSYNC_VSYNC_WIDTH_POLARITY : + REG_DP_HSYNC_VSYNC_WIDTH_POLARITY + offset, width_blanking); + msm_dp_write_link(panel, + id > 1 ? REG_DP_MSTLINK_ACTIVE_HOR_VER : + REG_DP_ACTIVE_HOR_VER + offset, msm_dp_active); + + reg = msm_dp_read_pn(panel, MMSS_DP_INTF_CONFIG); if (wide_bus_en) reg |= DP_INTF_CONFIG_DATABUS_WIDEN; else @@ -599,7 +736,7 @@ int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en) drm_dbg_dp(panel->drm_dev, "wide_bus_en=%d reg=%#x\n", wide_bus_en, reg); - msm_dp_write_p0(panel, MMSS_DP_INTF_CONFIG, reg); + msm_dp_write_pn(panel, MMSS_DP_INTF_CONFIG, reg); if (msm_dp_panel->msm_dp_mode.out_fmt_is_yuv_420) msm_dp_panel_setup_vsc_sdp_yuv_420(msm_dp_panel); @@ -609,6 +746,25 @@ int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en) return 0; } +void msm_dp_panel_set_pixel_base(struct msm_dp_panel *msm_dp_panel, void __iomem *pixel_base) +{ + struct msm_dp_panel_private *panel = + container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); + panel->pixel_base = pixel_base; +} + +void msm_dp_panel_mst_async_fifo(struct msm_dp_panel *msm_dp_panel, bool mst_en) +{ + struct msm_dp_panel_private *panel; + + panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); + + if (mst_en) + msm_dp_write_pn(panel, MMSS_DP_ASYNC_FIFO_CONFIG, 0x01); + else + msm_dp_write_pn(panel, MMSS_DP_ASYNC_FIFO_CONFIG, 0x00); +} + int msm_dp_panel_init_panel_info(struct msm_dp_panel *msm_dp_panel) { struct drm_display_mode *drm_mode; @@ -648,68 +804,15 @@ int msm_dp_panel_init_panel_info(struct msm_dp_panel *msm_dp_panel) return 0; } -static u32 msm_dp_panel_link_frequencies(struct device_node *of_node) -{ - struct device_node *endpoint; - u64 frequency = 0; - int cnt; - - endpoint = of_graph_get_endpoint_by_regs(of_node, 1, 0); /* port@1 */ - if (!endpoint) - return 0; - - cnt = of_property_count_u64_elems(endpoint, "link-frequencies"); - - if (cnt > 0) - of_property_read_u64_index(endpoint, "link-frequencies", - cnt - 1, &frequency); - of_node_put(endpoint); - - do_div(frequency, - 10 * /* from symbol rate to link rate */ - 1000); /* kbytes */ - - return frequency; -} - -static int msm_dp_panel_parse_dt(struct msm_dp_panel *msm_dp_panel) -{ - struct msm_dp_panel_private *panel; - struct device_node *of_node; - int cnt; - - panel = container_of(msm_dp_panel, struct msm_dp_panel_private, msm_dp_panel); - of_node = panel->dev->of_node; - - /* - * data-lanes is the property of msm_dp_out endpoint - */ - cnt = drm_of_get_data_lanes_count_ep(of_node, 1, 0, 1, DP_MAX_NUM_DP_LANES); - if (cnt < 0) { - /* legacy code, data-lanes is the property of mdss_dp node */ - cnt = drm_of_get_data_lanes_count(of_node, 1, DP_MAX_NUM_DP_LANES); - } - - if (cnt > 0) - msm_dp_panel->max_dp_lanes = cnt; - else - msm_dp_panel->max_dp_lanes = DP_MAX_NUM_DP_LANES; /* 4 lanes */ - - msm_dp_panel->max_dp_link_rate = msm_dp_panel_link_frequencies(of_node); - if (!msm_dp_panel->max_dp_link_rate) - msm_dp_panel->max_dp_link_rate = DP_LINK_RATE_HBR2; - - return 0; -} - struct msm_dp_panel *msm_dp_panel_get(struct device *dev, struct drm_dp_aux *aux, struct msm_dp_link *link, void __iomem *link_base, - void __iomem *p0_base) + void __iomem *mst2link_base, + void __iomem *mst3link_base, + void __iomem *pixel_base) { struct msm_dp_panel_private *panel; struct msm_dp_panel *msm_dp_panel; - int ret; if (!dev || !aux || !link) { DRM_ERROR("invalid input\n"); @@ -724,22 +827,13 @@ struct msm_dp_panel *msm_dp_panel_get(struct device *dev, struct drm_dp_aux *aux panel->aux = aux; panel->link = link; panel->link_base = link_base; - panel->p0_base = p0_base; + panel->pixel_base = pixel_base; + panel->mst2link_base = mst2link_base; + panel->mst3link_base = mst3link_base; msm_dp_panel = &panel->msm_dp_panel; msm_dp_panel->max_bw_code = DP_LINK_BW_8_1; - ret = msm_dp_panel_parse_dt(msm_dp_panel); - if (ret) - return ERR_PTR(ret); - return msm_dp_panel; } -void msm_dp_panel_put(struct msm_dp_panel *msm_dp_panel) -{ - if (!msm_dp_panel) - return; - - drm_edid_free(msm_dp_panel->drm_edid); -} diff --git a/drivers/gpu/drm/msm/dp/dp_panel.h b/drivers/gpu/drm/msm/dp/dp_panel.h index d2cf401506dcb..8bab27520439c 100644 --- a/drivers/gpu/drm/msm/dp/dp_panel.h +++ b/drivers/gpu/drm/msm/dp/dp_panel.h @@ -27,13 +27,21 @@ struct msm_dp_panel_psr { u8 capabilities; }; +/* stream id */ +enum msm_dp_stream_id { + DP_STREAM_0, + DP_STREAM_1, + DP_STREAM_2, + DP_STREAM_3, + DP_STREAM_MAX, +}; + struct msm_dp_panel { /* dpcd raw data */ u8 dpcd[DP_RECEIVER_CAP_SIZE]; u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; struct msm_dp_link_info link_info; - const struct drm_edid *drm_edid; struct drm_connector *connector; struct msm_dp_display_mode msm_dp_mode; struct msm_dp_panel_psr psr_cap; @@ -41,8 +49,8 @@ struct msm_dp_panel { bool vsc_sdp_supported; u32 hw_revision; - u32 max_dp_lanes; - u32 max_dp_link_rate; + enum msm_dp_stream_id stream_id; + u32 pbn; u32 max_bw_code; }; @@ -50,25 +58,27 @@ struct msm_dp_panel { int msm_dp_panel_init_panel_info(struct msm_dp_panel *msm_dp_panel); int msm_dp_panel_deinit(struct msm_dp_panel *msm_dp_panel); int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en); -int msm_dp_panel_read_sink_caps(struct msm_dp_panel *msm_dp_panel, - struct drm_connector *connector); +int msm_dp_panel_read_link_caps(struct msm_dp_panel *msm_dp_panel, + struct drm_connector *connector); u32 msm_dp_panel_get_mode_bpp(struct msm_dp_panel *msm_dp_panel, u32 mode_max_bpp, u32 mode_pclk_khz); -int msm_dp_panel_get_modes(struct msm_dp_panel *msm_dp_panel, - struct drm_connector *connector); -void msm_dp_panel_handle_sink_request(struct msm_dp_panel *msm_dp_panel); +void msm_dp_panel_handle_sink_request(struct msm_dp_panel *msm_dp_panel, + const struct drm_edid *drm_edid); void msm_dp_panel_tpg_config(struct msm_dp_panel *msm_dp_panel, bool enable); void msm_dp_panel_clear_dsc_dto(struct msm_dp_panel *msm_dp_panel); +void msm_dp_panel_set_pixel_base(struct msm_dp_panel *msm_dp_panel, void __iomem *pixel_base); void msm_dp_panel_enable_vsc_sdp(struct msm_dp_panel *msm_dp_panel, struct dp_sdp *vsc_sdp); void msm_dp_panel_disable_vsc_sdp(struct msm_dp_panel *msm_dp_panel); +void msm_dp_panel_mst_async_fifo(struct msm_dp_panel *msm_dp_panel, bool mst_en); + /** * is_link_rate_valid() - validates the link rate - * @lane_rate: link rate requested by the sink + * @bw_code: link rate requested by the sink * - * Returns true if the requested link rate is supported. + * Returns: true if the requested link rate is supported. */ static inline bool is_link_rate_valid(u32 bw_code) { @@ -79,10 +89,10 @@ static inline bool is_link_rate_valid(u32 bw_code) } /** - * msm_dp_link_is_lane_count_valid() - validates the lane count + * is_lane_count_valid() - validates the lane count * @lane_count: lane count requested by the sink * - * Returns true if the requested lane count is supported. + * Returns: true if the requested lane count is supported. */ static inline bool is_lane_count_valid(u32 lane_count) { @@ -94,6 +104,7 @@ static inline bool is_lane_count_valid(u32 lane_count) struct msm_dp_panel *msm_dp_panel_get(struct device *dev, struct drm_dp_aux *aux, struct msm_dp_link *link, void __iomem *link_base, - void __iomem *p0_base); -void msm_dp_panel_put(struct msm_dp_panel *msm_dp_panel); + void __iomem *mst2link_base, + void __iomem *mst3link_base, + void __iomem *pixel_base); #endif /* _DP_PANEL_H_ */ diff --git a/drivers/gpu/drm/msm/dp/dp_reg.h b/drivers/gpu/drm/msm/dp/dp_reg.h index 7c44d4e2cf139..65695fcb48d0f 100644 --- a/drivers/gpu/drm/msm/dp/dp_reg.h +++ b/drivers/gpu/drm/msm/dp/dp_reg.h @@ -42,9 +42,13 @@ #define DP_INTR_FRAME_END BIT(6) #define DP_INTR_CRC_UPDATED BIT(9) +#define DP_INTR_DP0_VCPF_SENT BIT(0) +#define DP_INTR_DP1_VCPF_SENT BIT(3) + #define REG_DP_INTR_STATUS3 (0x00000028) #define REG_DP_INTR_STATUS4 (0x0000002C) +#define REG_DP_INTR_STATUS5 (0x00000034) #define PSR_UPDATE_INT (0x00000001) #define PSR_CAPTURE_INT (0x00000004) #define PSR_EXIT_INT (0x00000010) @@ -68,8 +72,8 @@ #define DP_DP_IRQ_HPD_INT_ACK (0x00000002) #define DP_DP_HPD_REPLUG_INT_ACK (0x00000004) #define DP_DP_HPD_UNPLUG_INT_ACK (0x00000008) -#define DP_DP_HPD_STATE_STATUS_BITS_MASK (0x0000000F) -#define DP_DP_HPD_STATE_STATUS_BITS_SHIFT (0x1C) +#define DP_DP_HPD_STATE_STATUS_BITS_MASK (0x00000007) +#define DP_DP_HPD_STATE_STATUS_BITS_SHIFT (0x1D) #define REG_DP_DP_HPD_INT_MASK (0x0000000C) #define DP_DP_HPD_PLUG_INT_MASK (0x00000001) @@ -128,6 +132,10 @@ #define DP_MAINLINK_FLUSH_MODE_UPDATE_SDP FIELD_PREP(DP_MAINLINK_CTRL_FLUSH_MODE_MASK, 1) #define DP_MAINLINK_FLUSH_MODE_SDE_PERIPH_UPDATE FIELD_PREP(DP_MAINLINK_CTRL_FLUSH_MODE_MASK, 3) #define DP_MAINLINK_FB_BOUNDARY_SEL (0x02000000) +#define DP_MAINLINK_CTRL_ECF_MODE BIT(26) +#define DP_MAINLINK_CTRL_MST_ACTIVE BIT(8) +#define DP_MAINLINK_CTRL_MST_EN (DP_MAINLINK_CTRL_ECF_MODE | \ + DP_MAINLINK_CTRL_MST_ACTIVE) #define REG_DP_STATE_CTRL (0x00000004) #define DP_STATE_CTRL_LINK_TRAINING_PATTERN1 (0x00000001) @@ -156,13 +164,14 @@ #define DP_CONFIGURATION_CTRL_BPC_SHIFT (0x08) #define DP_CONFIGURATION_CTRL_LSCLK_DIV_SHIFT (0x0D) +#define REG_DP_MST_ACT (0x00000500) + #define REG_DP_SOFTWARE_MVID (0x00000010) #define REG_DP_SOFTWARE_NVID (0x00000018) #define REG_DP_TOTAL_HOR_VER (0x0000001C) #define REG_DP_START_HOR_VER_FROM_SYNC (0x00000020) #define REG_DP_HSYNC_VSYNC_WIDTH_POLARITY (0x00000024) #define REG_DP_ACTIVE_HOR_VER (0x00000028) - #define REG_DP_MISC1_MISC0 (0x0000002C) #define DP_MISC0_SYNCHRONOUS_CLK (0x00000001) #define DP_MISC0_COLORIMETRY_CFG_SHIFT (0x00000001) @@ -332,6 +341,43 @@ #define DP_TPG_VIDEO_CONFIG_BPP_8BIT (0x00000001) #define DP_TPG_VIDEO_CONFIG_RGB (0x00000004) +/* DP MST registers */ + +#define REG_DP_MSTLINK_DP_RG (0X0000011C) +#define REG_DP1_CONFIGURATION_CTRL (0x00000400) +#define REG_DP_DP0_TIMESLOT_1_32 (0x00000404) +#define REG_DP_DP0_TIMESLOT_33_63 (0x00000408) +#define REG_DP_DP1_TIMESLOT_1_32 (0x0000040C) +#define REG_DP_DP1_TIMESLOT_33_63 (0x00000410) +#define REG_DP1_SOFTWARE_MVID (0x00000414) +#define REG_DP1_SOFTWARE_NVID (0x00000418) +#define REG_DP1_TOTAL_HOR_VER (0x0000041C) +#define REG_DP1_MISC1_MISC0 (0x0000042C) +#define MMSS_DP1_GENERIC0_0 (0x00000490) +#define MMSS_DP1_SDP_CFG (0x000004E0) +#define MMSS_DP1_SDP_CFG2 (0x000004E4) +#define MMSS_DP1_SDP_CFG3 (0x000004E8) +#define REG_DP_DP0_RG (0x000004F8) +#define REG_DP_DP1_RG (0x000004FC) + +#define REG_DP_MSTLINK_STATE_CTRL (0x00000000) +#define REG_DP_MSTLINK_CONFIGURATION_CTRL (0x00000034) +#define REG_DP_MSTLINK_TIMESLOT_1_32 (0x00000038) +#define REG_DP_MSTLINK_TIMESLOT_33_63 (0x0000003C) +#define REG_MSTLINK_SOFTWARE_MVID (0x00000040) +#define REG_MSTLINK_SOFTWARE_NVID (0x00000044) +#define REG_DP_MSTLINK_TOTAL_HOR_VER (0x00000048) +#define REG_DP_MSTLINK_START_HOR_VER_FROM_SYNC (0x0000004C) +#define REG_DP_MSTLINK_HSYNC_VSYNC_WIDTH_POLARITY (0x00000050) +#define REG_DP_MSTLINK_ACTIVE_HOR_VER (0x00000054) +#define REG_DP_MSTLINK_MISC1_MISC0 (0x00000058) +#define MMSS_DP_MSTLINK_GENERIC0_0 (0x000000BC) +#define MMSS_DP_MSTLINK_GENERIC0_1 (0x000000C0) +#define MMSS_DP_MSTLINK_GENERIC0_2 (0x000000C4) +#define MMSS_DP_MSTLINK_SDP_CFG (0x0000010c) +#define MMSS_DP_MSTLINK_SDP_CFG2 (0x0000011c) +#define MMSS_DP_MSTLINK_SDP_CFG3 (0x00000114) + #define MMSS_DP_ASYNC_FIFO_CONFIG (0x00000088) #define REG_DP_PHY_AUX_INTERRUPT_CLEAR (0x0000004C) diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c index d8bb40ef820e2..3c9f01ed62713 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.c +++ b/drivers/gpu/drm/msm/dsi/dsi.c @@ -198,6 +198,7 @@ static const struct of_device_id dt_match[] = { { .compatible = "qcom,dsi-ctrl-6g-qcm2290" }, {} }; +MODULE_DEVICE_TABLE(of, dt_match); static const struct dev_pm_ops dsi_pm_ops = { SET_RUNTIME_PM_OPS(msm_dsi_runtime_suspend, msm_dsi_runtime_resume, NULL) diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c index fed8e9b67011c..bd3c51c350e73 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c @@ -205,6 +205,17 @@ static const struct msm_dsi_config sm8650_dsi_cfg = { }, }; +static const struct msm_dsi_config kaanapali_dsi_cfg = { + .io_offset = DSI_6G_REG_SHIFT, + .regulator_data = sm8650_dsi_regulators, + .num_regulators = ARRAY_SIZE(sm8650_dsi_regulators), + .bus_clk_names = dsi_v2_4_clk_names, + .num_bus_clks = ARRAY_SIZE(dsi_v2_4_clk_names), + .io_start = { + { 0x9ac0000, 0x9ac3000 }, + }, +}; + static const struct regulator_bulk_data sc7280_dsi_regulators[] = { { .supply = "vdda", .init_load_uA = 8350 }, /* 1.2 V */ { .supply = "refgen" }, @@ -332,6 +343,8 @@ static const struct msm_dsi_cfg_handler dsi_cfg_handlers[] = { &sm8650_dsi_cfg, &msm_dsi_6g_v2_host_ops}, {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_9_0, &sm8650_dsi_cfg, &msm_dsi_6g_v2_9_host_ops}, + {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_10_0, + &kaanapali_dsi_cfg, &msm_dsi_6g_v2_9_host_ops}, }; const struct msm_dsi_cfg_handler *msm_dsi_cfg_get(u32 major, u32 minor) diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h index 38f303f2ed04c..5dc812028bd54 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h @@ -32,6 +32,7 @@ #define MSM_DSI_6G_VER_MINOR_V2_7_0 0x20070000 #define MSM_DSI_6G_VER_MINOR_V2_8_0 0x20080000 #define MSM_DSI_6G_VER_MINOR_V2_9_0 0x20090000 +#define MSM_DSI_6G_VER_MINOR_V2_10_0 0x200a0000 #define MSM_DSI_V2_VER_MINOR_8064 0x0 diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index e0de545d40775..db6da99375a18 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -584,13 +584,30 @@ void dsi_link_clk_disable_v2(struct msm_dsi_host *msm_host) * FIXME: Reconsider this if/when CMD mode handling is rewritten to use * transfer time and data overhead as a starting point of the calculations. */ -static unsigned long dsi_adjust_pclk_for_compression(const struct drm_display_mode *mode, - const struct drm_dsc_config *dsc) +static unsigned long +dsi_adjust_pclk_for_compression(const struct drm_display_mode *mode, + const struct drm_dsc_config *dsc, + bool is_bonded_dsi) { - int new_hdisplay = DIV_ROUND_UP(mode->hdisplay * drm_dsc_get_bpp_int(dsc), - dsc->bits_per_component * 3); + int hdisplay, new_hdisplay, new_htotal; - int new_htotal = mode->htotal - mode->hdisplay + new_hdisplay; + /* + * For bonded DSI, split hdisplay across two links and round up each + * half separately, passing the full hdisplay would only round up once. + * This also aligns with the hdisplay we program later in + * dsi_timing_setup() + */ + hdisplay = mode->hdisplay; + if (is_bonded_dsi) + hdisplay /= 2; + + new_hdisplay = DIV_ROUND_UP(hdisplay * drm_dsc_get_bpp_int(dsc), + dsc->bits_per_component * 3); + + if (is_bonded_dsi) + new_hdisplay *= 2; + + new_htotal = mode->htotal - mode->hdisplay + new_hdisplay; return mult_frac(mode->clock * 1000u, new_htotal, mode->htotal); } @@ -603,7 +620,7 @@ static unsigned long dsi_get_pclk_rate(const struct drm_display_mode *mode, pclk_rate = mode->clock * 1000u; if (dsc) - pclk_rate = dsi_adjust_pclk_for_compression(mode, dsc); + pclk_rate = dsi_adjust_pclk_for_compression(mode, dsc, is_bonded_dsi); /* * For bonded DSI mode, the current DRM mode has the complete width of the @@ -993,7 +1010,7 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) if (msm_host->dsc) { struct drm_dsc_config *dsc = msm_host->dsc; - u32 bytes_per_pclk; + u32 bits_per_pclk; /* update dsc params with timing params */ if (!dsc || !mode->hdisplay || !mode->vdisplay) { @@ -1015,7 +1032,9 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) /* * DPU sends 3 bytes per pclk cycle to DSI. If widebus is - * enabled, bus width is extended to 6 bytes. + * enabled, MDP always sends out 48-bit compressed data per + * pclk and on average, DSI consumes an amount of compressed + * data equivalent to the uncompressed pixel depth per pclk. * * Calculate the number of pclks needed to transmit one line of * the compressed data. @@ -1027,12 +1046,12 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) * unused anyway. */ h_total -= hdisplay; - if (wide_bus_enabled && !(msm_host->mode_flags & MIPI_DSI_MODE_VIDEO)) - bytes_per_pclk = 6; + if (wide_bus_enabled) + bits_per_pclk = mipi_dsi_pixel_format_to_bpp(msm_host->format); else - bytes_per_pclk = 3; + bits_per_pclk = 24; - hdisplay = DIV_ROUND_UP(msm_dsc_get_bytes_per_line(msm_host->dsc), bytes_per_pclk); + hdisplay = DIV_ROUND_UP(msm_dsc_get_bytes_per_line(msm_host->dsc) * 8, bits_per_pclk); h_total += hdisplay; ha_end = ha_start + hdisplay; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c index 4ea681130dbaf..c59375aaae197 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c @@ -577,9 +577,12 @@ static const struct of_device_id dsi_phy_dt_match[] = { .data = &dsi_phy_4nm_8650_cfgs }, { .compatible = "qcom,sm8750-dsi-phy-3nm", .data = &dsi_phy_3nm_8750_cfgs }, + { .compatible = "qcom,kaanapali-dsi-phy-3nm", + .data = &dsi_phy_3nm_kaanapali_cfgs }, #endif {} }; +MODULE_DEVICE_TABLE(of, dsi_phy_dt_match); /* * Currently, we only support one SoC for each PHY type. When we have multiple diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h index 3cbf082314924..c01784ca38edc 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h @@ -64,6 +64,7 @@ extern const struct msm_dsi_phy_cfg dsi_phy_5nm_sar2130p_cfgs; extern const struct msm_dsi_phy_cfg dsi_phy_4nm_8550_cfgs; extern const struct msm_dsi_phy_cfg dsi_phy_4nm_8650_cfgs; extern const struct msm_dsi_phy_cfg dsi_phy_3nm_8750_cfgs; +extern const struct msm_dsi_phy_cfg dsi_phy_3nm_kaanapali_cfgs; struct msm_dsi_dphy_timing { u32 clk_zero; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c index fdefcbd9c2848..a156c7e7cea83 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c @@ -628,12 +628,7 @@ static int dsi_pll_14nm_postdiv_determine_rate(struct clk_hw *hw, DBG("DSI%d PLL parent rate=%lu", pll_14nm->phy->id, req->rate); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - postdiv->width, - postdiv->flags); - - return 0; + return divider_determine_rate(hw, req, NULL, postdiv->width, postdiv->flags); } static int dsi_pll_14nm_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c index c5e1d2016bcca..1cc4061465cd6 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c @@ -595,6 +595,12 @@ static unsigned long dsi_pll_7nm_vco_recalc_rate(struct clk_hw *hw, pll_freq += div_u64(tmp64, multiplier); vco_rate = pll_freq; + /* + * Recalculating the rate from dec and frac doesn't end up the rate + * we originally set. Convert the freq to KHz, round it up and + * convert it back to Hz. + */ + vco_rate = DIV_ROUND_UP_ULL(vco_rate, 1000) * 1000; pll_7nm->vco_current_rate = vco_rate; DBG("DSI PLL%d returning vco rate = %lu, dec = %x, frac = %x", @@ -1504,3 +1510,26 @@ const struct msm_dsi_phy_cfg dsi_phy_3nm_8750_cfgs = { .num_dsi_phy = 2, .quirks = DSI_PHY_7NM_QUIRK_V7_0, }; + +const struct msm_dsi_phy_cfg dsi_phy_3nm_kaanapali_cfgs = { + .has_phy_lane = true, + .regulator_data = dsi_phy_7nm_98000uA_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_7nm_98000uA_regulators), + .ops = { + .enable = dsi_7nm_phy_enable, + .disable = dsi_7nm_phy_disable, + .pll_init = dsi_pll_7nm_init, + .save_pll_state = dsi_7nm_pll_save_state, + .restore_pll_state = dsi_7nm_pll_restore_state, + .set_continuous_clock = dsi_7nm_set_continuous_clock, + }, + .min_pll_rate = 600000000UL, +#ifdef CONFIG_64BIT + .max_pll_rate = 5000000000UL, +#else + .max_pll_rate = ULONG_MAX, +#endif + .io_start = { 0x9ac1000, 0x9ac4000 }, + .num_dsi_phy = 2, + .quirks = DSI_PHY_7NM_QUIRK_V7_0, +}; diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c index 5afac09c0d334..d5ef5089c9e9c 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi.c @@ -441,6 +441,7 @@ static const struct of_device_id msm_hdmi_dt_match[] = { { .compatible = "qcom,hdmi-tx-8660", .data = &hdmi_tx_8960_config }, {} }; +MODULE_DEVICE_TABLE(of, msm_hdmi_dt_match); static struct platform_driver msm_hdmi_driver = { .probe = msm_hdmi_dev_probe, diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_phy.c b/drivers/gpu/drm/msm/hdmi/hdmi_phy.c index 667573f1db7c6..f726555bb6810 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi_phy.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi_phy.c @@ -204,6 +204,7 @@ static const struct of_device_id msm_hdmi_phy_dt_match[] = { .data = &msm_hdmi_phy_8998_cfg }, {} }; +MODULE_DEVICE_TABLE(of, msm_hdmi_phy_dt_match); static struct platform_driver msm_hdmi_phy_platform_driver = { .probe = msm_hdmi_phy_probe, diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c index 87a91148a731d..ea064aa6d8fc1 100644 --- a/drivers/gpu/drm/msm/msm_atomic.c +++ b/drivers/gpu/drm/msm/msm_atomic.c @@ -4,6 +4,7 @@ * Author: Rob Clark */ +#include #include #include @@ -207,7 +208,11 @@ int msm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state) if (ret) return ret; - return drm_atomic_helper_check(dev, state); + ret = drm_atomic_helper_check(dev, state); + if (ret) + return ret; + + return drm_dp_mst_atomic_check(state); } void msm_atomic_commit_tail(struct drm_atomic_state *state) @@ -221,6 +226,8 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state) trace_msm_atomic_commit_tail_start(async, crtc_mask); + drm_dp_mst_atomic_wait_for_dependencies(state); + kms->funcs->enable_commit(kms); /* diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 7e977fec41007..9ea1a1269c5ed 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -54,16 +54,46 @@ static bool modeset = true; MODULE_PARM_DESC(modeset, "Use kernel modesetting [KMS] (1=on (default), 0=disable)"); module_param(modeset, bool, 0600); -static bool separate_gpu_kms; -MODULE_PARM_DESC(separate_gpu_drm, "Use separate DRM device for the GPU (0=single DRM device for both GPU and display (default), 1=two DRM devices)"); -module_param(separate_gpu_kms, bool, 0400); +static int separate_gpu_kms = -1; +MODULE_PARM_DESC(separate_gpu_kms, "Use separate DRM device for the GPU (-1=auto (default), 0=single DRM device, 1=separate DRM devices)"); +module_param(separate_gpu_kms, int, 0400); DECLARE_FAULT_ATTR(fail_gem_alloc); DECLARE_FAULT_ATTR(fail_gem_iova); -bool msm_gpu_no_components(void) +static const struct of_device_id msm_gpu_match[]; + +static int msm_count_gpus(void) +{ + struct device_node *np; + int count = 0; + + for_each_matching_node(np, msm_gpu_match) { + if (of_device_is_available(np) && adreno_has_gpu(np)) + count++; + } + + return count; +} + +static bool msm_separate_gpu_kms_auto(void) +{ + int gpus = msm_count_gpus(); + int mdss = msm_mdss_count_masters(); + + if (mdss == 2 && gpus == 1) + return true; + + return false; +} + +bool msm_seperate_gpu_kms_components(void) { - return separate_gpu_kms; + if (separate_gpu_kms == 1) + return true; + if (separate_gpu_kms == 0) + return false; + return msm_separate_gpu_kms_auto(); } static int msm_drm_uninit(struct device *dev, const struct component_ops *gpu_ops) @@ -1018,7 +1048,7 @@ static int add_gpu_components(struct device *dev, static int msm_drm_bind(struct device *dev) { return msm_drm_init(dev, - msm_gpu_no_components() ? + msm_seperate_gpu_kms_components() ? &msm_kms_driver : &msm_driver, NULL); @@ -1057,7 +1087,7 @@ int msm_drv_probe(struct device *master_dev, return ret; } - if (!msm_gpu_no_components()) { + if (!msm_seperate_gpu_kms_components()) { ret = add_gpu_components(master_dev, &match); if (ret) return ret; diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 6d847d593f1ae..111007a241717 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -362,6 +362,9 @@ bool msm_dp_is_yuv_420_enabled(const struct msm_dp *dp_display, bool msm_dp_needs_periph_flush(const struct msm_dp *dp_display, const struct drm_display_mode *mode); bool msm_dp_wide_bus_available(const struct msm_dp *dp_display); +int msm_dp_get_mst_max_stream(struct msm_dp *dp_display); +int msm_dp_mst_register(struct msm_dp *dp_display); +int msm_dp_mst_attach_encoder(struct msm_dp *dp_display, struct drm_encoder *encoder); #else static inline int __init msm_dp_register(void) @@ -379,6 +382,21 @@ static inline int msm_dp_modeset_init(struct msm_dp *dp_display, return -EINVAL; } +static inline int msm_dp_get_mst_max_stream(struct msm_dp *dp_display) +{ + return -EINVAL; +} + +static inline int msm_dp_mst_register(struct msm_dp *dp_display) +{ + return -EINVAL; +} + +static inline int msm_dp_mst_attach_encoder(struct msm_dp *dp_display, struct drm_encoder *encoder) +{ + return -EINVAL; +} + static inline void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp_display) { } @@ -555,6 +573,8 @@ void msm_kms_shutdown(struct platform_device *pdev); bool msm_disp_drv_should_bind(struct device *dev, bool dpu_driver); -bool msm_gpu_no_components(void); +bool msm_seperate_gpu_kms_components(void); + +int msm_mdss_count_masters(void); #endif /* __MSM_DRV_H__ */ diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 17759abc46d7d..995549d0bbbc5 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -197,8 +197,7 @@ static ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset, drm_printf(&p, "---\n"); drm_printf(&p, "kernel: " UTS_RELEASE "\n"); drm_printf(&p, "module: " KBUILD_MODNAME "\n"); - drm_printf(&p, "time: %lld.%09ld\n", - state->time.tv_sec, state->time.tv_nsec); + drm_printf(&p, "time: %ptSp\n", &state->time); if (state->comm) drm_printf(&p, "comm: %s\n", state->comm); if (state->cmd) @@ -287,16 +286,17 @@ static void crashstate_get_bos(struct msm_gpu_state *state, struct msm_gem_submi state->bos = kcalloc(cnt, sizeof(struct msm_gpu_state_bo), GFP_KERNEL); - drm_gpuvm_for_each_va (vma, submit->vm) { - bool dump = rd_full || (vma->flags & MSM_VMA_DUMP); + if (state->bos) + drm_gpuvm_for_each_va(vma, submit->vm) { + bool dump = rd_full || (vma->flags & MSM_VMA_DUMP); - /* Skip MAP_NULL/PRR VMAs: */ - if (!vma->gem.obj) - continue; + /* Skip MAP_NULL/PRR VMAs: */ + if (!vma->gem.obj) + continue; - msm_gpu_crashstate_get_bo(state, vma->gem.obj, vma->va.addr, - dump, vma->gem.offset, vma->va.range); - } + msm_gpu_crashstate_get_bo(state, vma->gem.obj, vma->va.addr, + dump, vma->gem.offset, vma->va.range); + } drm_exec_fini(&exec); } else { @@ -348,6 +348,10 @@ static void crashstate_get_vm_logs(struct msm_gpu_state *state, struct msm_gem_v state->vm_logs = kmalloc_array( state->nr_vm_logs, sizeof(vm->log[0]), GFP_KERNEL); + if (!state->vm_logs) { + state->nr_vm_logs = 0; + } + for (int i = 0; i < state->nr_vm_logs; i++) { int idx = (i + first) & vm_log_mask; diff --git a/drivers/gpu/drm/msm/msm_kms.c b/drivers/gpu/drm/msm/msm_kms.c index 6e5e94f5c9a74..a4f339db3a754 100644 --- a/drivers/gpu/drm/msm/msm_kms.c +++ b/drivers/gpu/drm/msm/msm_kms.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,7 @@ static const struct drm_mode_config_funcs mode_config_funcs = { static const struct drm_mode_config_helper_funcs mode_config_helper_funcs = { .atomic_commit_tail = msm_atomic_commit_tail, + .atomic_commit_setup = drm_dp_mst_atomic_setup_commit, }; static irqreturn_t msm_irq(int irq, void *arg) diff --git a/drivers/gpu/drm/msm/msm_mdss.c b/drivers/gpu/drm/msm/msm_mdss.c index 2d0e3e784c044..9eb6a69d7d1e0 100644 --- a/drivers/gpu/drm/msm/msm_mdss.c +++ b/drivers/gpu/drm/msm/msm_mdss.c @@ -73,6 +73,21 @@ static int msm_mdss_parse_data_bus_icc_path(struct device *dev, return 0; } +static const struct of_device_id mdss_dt_match[]; + +int msm_mdss_count_masters(void) +{ + struct device_node *np; + int count = 0; + + for_each_matching_node(np, mdss_dt_match) { + if (of_device_is_available(np)) + count++; + } + + return count; +} + static void msm_mdss_irq(struct irq_desc *desc) { struct msm_mdss *msm_mdss = irq_desc_get_handler_data(desc); @@ -229,7 +244,7 @@ static void msm_mdss_setup_ubwc_dec_50(struct msm_mdss *msm_mdss) { const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) | - MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit); + MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13); if (data->ubwc_bank_spread) value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD; @@ -555,6 +570,7 @@ static const struct of_device_id mdss_dt_match[] = { { .compatible = "qcom,mdss", .data = &data_153k6 }, { .compatible = "qcom,msm8998-mdss", .data = &data_76k8 }, { .compatible = "qcom,qcm2290-mdss", .data = &data_76k8 }, + { .compatible = "qcom,qcs8300-mdss", .data = &data_74k }, { .compatible = "qcom,sa8775p-mdss", .data = &data_74k }, { .compatible = "qcom,sar2130p-mdss", .data = &data_74k }, { .compatible = "qcom,sdm670-mdss", .data = &data_76k8 }, diff --git a/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml index b15a242d974d6..09b8a0b9c0de7 100644 --- a/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml +++ b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml @@ -40,56 +40,56 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + @@ -99,15 +99,15 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - + - + - + @@ -119,71 +119,71 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -191,27 +191,27 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - - - - - + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/drivers/gpu/drm/nouveau/dispnv04/nouveau_i2c_encoder.c b/drivers/gpu/drm/nouveau/dispnv04/nouveau_i2c_encoder.c index e2bf99c433366..a60209097a20a 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/nouveau_i2c_encoder.c +++ b/drivers/gpu/drm/nouveau/dispnv04/nouveau_i2c_encoder.c @@ -94,26 +94,6 @@ int nouveau_i2c_encoder_init(struct drm_device *dev, return err; } -/** - * nouveau_i2c_encoder_destroy - Unregister the I2C device backing an encoder - * @drm_encoder: Encoder to be unregistered. - * - * This should be called from the @destroy method of an I2C slave - * encoder driver once I2C access is no longer needed. - */ -void nouveau_i2c_encoder_destroy(struct drm_encoder *drm_encoder) -{ - struct nouveau_i2c_encoder *encoder = to_encoder_i2c(drm_encoder); - struct i2c_client *client = nouveau_i2c_encoder_get_client(drm_encoder); - struct module *module = client->dev.driver->owner; - - i2c_unregister_device(client); - encoder->i2c_client = NULL; - - module_put(module); -} -EXPORT_SYMBOL(nouveau_i2c_encoder_destroy); - /* * Wrapper fxns which can be plugged in to drm_encoder_helper_funcs: */ diff --git a/drivers/gpu/drm/nouveau/dispnv50/atom.h b/drivers/gpu/drm/nouveau/dispnv50/atom.h index 93f8f4f645784..b43c4f9bbcdf5 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/atom.h +++ b/drivers/gpu/drm/nouveau/dispnv50/atom.h @@ -152,8 +152,21 @@ static inline struct nv50_head_atom * nv50_head_atom_get(struct drm_atomic_state *state, struct drm_crtc *crtc) { struct drm_crtc_state *statec = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(statec)) return (void *)statec; + + return nv50_head_atom(statec); +} + +static inline struct nv50_head_atom * +nv50_head_atom_get_new(struct drm_atomic_state *state, struct drm_crtc *crtc) +{ + struct drm_crtc_state *statec = drm_atomic_get_new_crtc_state(state, crtc); + + if (!statec) + return NULL; + return nv50_head_atom(statec); } diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs507a.c b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c index a95ee5dcc2e39..1a889139cb053 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/curs507a.c +++ b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c @@ -84,6 +84,7 @@ curs507a_prepare(struct nv50_wndw *wndw, struct nv50_head_atom *asyh, asyh->curs.handle = handle; asyh->curs.offset = offset; asyh->set.curs = asyh->curs.visible; + nv50_atom(asyh->state.state)->lock_core = true; } } diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c index ef9e410babbfb..9a2c20fce0f3e 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c +++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c @@ -583,7 +583,7 @@ nv50_wndw_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state) asyw->image.offset[0] = nvbo->offset; if (wndw->func->prepare) { - asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc); + asyh = nv50_head_atom_get_new(asyw->state.state, asyw->state.crtc); if (IS_ERR(asyh)) return PTR_ERR(asyh); diff --git a/drivers/gpu/drm/nouveau/include/dispnv04/i2c/encoder_i2c.h b/drivers/gpu/drm/nouveau/include/dispnv04/i2c/encoder_i2c.h index 31334aa90781b..869820701a56e 100644 --- a/drivers/gpu/drm/nouveau/include/dispnv04/i2c/encoder_i2c.h +++ b/drivers/gpu/drm/nouveau/include/dispnv04/i2c/encoder_i2c.h @@ -202,7 +202,24 @@ static inline struct i2c_client *nouveau_i2c_encoder_get_client(struct drm_encod return to_encoder_i2c(encoder)->i2c_client; } -void nouveau_i2c_encoder_destroy(struct drm_encoder *encoder); +/** + * nouveau_i2c_encoder_destroy - Unregister the I2C device backing an encoder + * @drm_encoder: Encoder to be unregistered. + * + * This should be called from the @destroy method of an I2C slave + * encoder driver once I2C access is no longer needed. + */ +static __always_inline void nouveau_i2c_encoder_destroy(struct drm_encoder *drm_encoder) +{ + struct nouveau_i2c_encoder *encoder = to_encoder_i2c(drm_encoder); + struct i2c_client *client = nouveau_i2c_encoder_get_client(drm_encoder); + struct module *module = client->dev.driver->owner; + + i2c_unregister_device(client); + encoder->i2c_client = NULL; + + module_put(module); +} /* * Wrapper fxns which can be plugged in to drm_encoder_helper_funcs: diff --git a/drivers/gpu/drm/nouveau/include/nvif/client.h b/drivers/gpu/drm/nouveau/include/nvif/client.h index 03f1d564eb124..b698c74306f8f 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/client.h +++ b/drivers/gpu/drm/nouveau/include/nvif/client.h @@ -11,7 +11,7 @@ struct nvif_client { int nvif_client_ctor(struct nvif_client *parent, const char *name, struct nvif_client *); void nvif_client_dtor(struct nvif_client *); -int nvif_client_suspend(struct nvif_client *); +int nvif_client_suspend(struct nvif_client *, bool); int nvif_client_resume(struct nvif_client *); /*XXX*/ diff --git a/drivers/gpu/drm/nouveau/include/nvif/driver.h b/drivers/gpu/drm/nouveau/include/nvif/driver.h index 7b08ff769039a..61c8a177b28fd 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/driver.h +++ b/drivers/gpu/drm/nouveau/include/nvif/driver.h @@ -8,7 +8,7 @@ struct nvif_driver { const char *name; int (*init)(const char *name, u64 device, const char *cfg, const char *dbg, void **priv); - int (*suspend)(void *priv); + int (*suspend)(void *priv, bool runtime); int (*resume)(void *priv); int (*ioctl)(void *priv, void *data, u32 size, void **hack); void __iomem *(*map)(void *priv, u64 handle, u32 size); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h index 99579e7b93760..954a89d43bada 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h @@ -2,6 +2,7 @@ #ifndef __NVKM_DEVICE_H__ #define __NVKM_DEVICE_H__ #include +#include #include enum nvkm_subdev_type; @@ -93,7 +94,7 @@ struct nvkm_device_func { void *(*dtor)(struct nvkm_device *); int (*preinit)(struct nvkm_device *); int (*init)(struct nvkm_device *); - void (*fini)(struct nvkm_device *, bool suspend); + void (*fini)(struct nvkm_device *, enum nvkm_suspend_state suspend); int (*irq)(struct nvkm_device *); resource_size_t (*resource_addr)(struct nvkm_device *, enum nvkm_bar_id); resource_size_t (*resource_size)(struct nvkm_device *, enum nvkm_bar_id); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h index 738899fcf30b6..1e97be6c65644 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h @@ -20,7 +20,7 @@ struct nvkm_engine_func { int (*oneinit)(struct nvkm_engine *); int (*info)(struct nvkm_engine *, u64 mthd, u64 *data); int (*init)(struct nvkm_engine *); - int (*fini)(struct nvkm_engine *, bool suspend); + int (*fini)(struct nvkm_engine *, enum nvkm_suspend_state suspend); int (*reset)(struct nvkm_engine *); int (*nonstall)(struct nvkm_engine *); void (*intr)(struct nvkm_engine *); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h index 10107ef3ca49e..54d356154274f 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h @@ -2,6 +2,7 @@ #ifndef __NVKM_OBJECT_H__ #define __NVKM_OBJECT_H__ #include +#include struct nvkm_event; struct nvkm_gpuobj; struct nvkm_uevent; @@ -27,7 +28,7 @@ enum nvkm_object_map { struct nvkm_object_func { void *(*dtor)(struct nvkm_object *); int (*init)(struct nvkm_object *); - int (*fini)(struct nvkm_object *, bool suspend); + int (*fini)(struct nvkm_object *, enum nvkm_suspend_state suspend); int (*mthd)(struct nvkm_object *, u32 mthd, void *data, u32 size); int (*ntfy)(struct nvkm_object *, u32 mthd, struct nvkm_event **); int (*map)(struct nvkm_object *, void *argv, u32 argc, @@ -49,7 +50,7 @@ int nvkm_object_new(const struct nvkm_oclass *, void *data, u32 size, void nvkm_object_del(struct nvkm_object **); void *nvkm_object_dtor(struct nvkm_object *); int nvkm_object_init(struct nvkm_object *); -int nvkm_object_fini(struct nvkm_object *, bool suspend); +int nvkm_object_fini(struct nvkm_object *, enum nvkm_suspend_state); int nvkm_object_mthd(struct nvkm_object *, u32 mthd, void *data, u32 size); int nvkm_object_ntfy(struct nvkm_object *, u32 mthd, struct nvkm_event **); int nvkm_object_map(struct nvkm_object *, void *argv, u32 argc, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/oproxy.h b/drivers/gpu/drm/nouveau/include/nvkm/core/oproxy.h index 0e70a9afba33d..cf66aee4d111d 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/oproxy.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/oproxy.h @@ -13,7 +13,7 @@ struct nvkm_oproxy { struct nvkm_oproxy_func { void (*dtor[2])(struct nvkm_oproxy *); int (*init[2])(struct nvkm_oproxy *); - int (*fini[2])(struct nvkm_oproxy *, bool suspend); + int (*fini[2])(struct nvkm_oproxy *, enum nvkm_suspend_state suspend); }; void nvkm_oproxy_ctor(const struct nvkm_oproxy_func *, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h index bce6e1ba09ea0..bd6b1b658e40f 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h @@ -40,7 +40,7 @@ struct nvkm_subdev_func { int (*oneinit)(struct nvkm_subdev *); int (*info)(struct nvkm_subdev *, u64 mthd, u64 *data); int (*init)(struct nvkm_subdev *); - int (*fini)(struct nvkm_subdev *, bool suspend); + int (*fini)(struct nvkm_subdev *, enum nvkm_suspend_state suspend); void (*intr)(struct nvkm_subdev *); }; @@ -65,7 +65,7 @@ void nvkm_subdev_unref(struct nvkm_subdev *); int nvkm_subdev_preinit(struct nvkm_subdev *); int nvkm_subdev_oneinit(struct nvkm_subdev *); int nvkm_subdev_init(struct nvkm_subdev *); -int nvkm_subdev_fini(struct nvkm_subdev *, bool suspend); +int nvkm_subdev_fini(struct nvkm_subdev *, enum nvkm_suspend_state suspend); int nvkm_subdev_info(struct nvkm_subdev *, u64, u64 *); void nvkm_subdev_intr(struct nvkm_subdev *); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/suspend_state.h b/drivers/gpu/drm/nouveau/include/nvkm/core/suspend_state.h new file mode 100644 index 0000000000000..134120fb71f4e --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/suspend_state.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVKM_SUSPEND_STATE_H__ +#define __NVKM_SUSPEND_STATE_H__ + +enum nvkm_suspend_state { + NVKM_POWEROFF, + NVKM_SUSPEND, + NVKM_RUNTIME_SUSPEND, +}; + +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h index d1beaad0c82b6..834ed6587aa52 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h @@ -1,28 +1,81 @@ /* SPDX-License-Identifier: MIT */ #ifndef __NVBIOS_CONN_H__ #define __NVBIOS_CONN_H__ + +/* + * An enumerator representing all of the possible VBIOS connector types defined + * by Nvidia at + * https://nvidia.github.io/open-gpu-doc/DCB/DCB-4.x-Specification.html. + * + * [1] Nvidia's documentation actually claims DCB_CONNECTOR_HDMI_0 is a "3-Pin + * DIN Stereo Connector". This seems very likely to be a documentation typo + * or some sort of funny historical baggage, because we've treated this + * connector type as HDMI for years without issue. + * TODO: Check with Nvidia what's actually happening here. + */ enum dcb_connector_type { - DCB_CONNECTOR_VGA = 0x00, - DCB_CONNECTOR_TV_0 = 0x10, - DCB_CONNECTOR_TV_1 = 0x11, - DCB_CONNECTOR_TV_3 = 0x13, - DCB_CONNECTOR_DVI_I = 0x30, - DCB_CONNECTOR_DVI_D = 0x31, - DCB_CONNECTOR_DMS59_0 = 0x38, - DCB_CONNECTOR_DMS59_1 = 0x39, - DCB_CONNECTOR_LVDS = 0x40, - DCB_CONNECTOR_LVDS_SPWG = 0x41, - DCB_CONNECTOR_DP = 0x46, - DCB_CONNECTOR_eDP = 0x47, - DCB_CONNECTOR_mDP = 0x48, - DCB_CONNECTOR_HDMI_0 = 0x60, - DCB_CONNECTOR_HDMI_1 = 0x61, - DCB_CONNECTOR_HDMI_C = 0x63, - DCB_CONNECTOR_DMS59_DP0 = 0x64, - DCB_CONNECTOR_DMS59_DP1 = 0x65, - DCB_CONNECTOR_WFD = 0x70, - DCB_CONNECTOR_USB_C = 0x71, - DCB_CONNECTOR_NONE = 0xff + /* Analog outputs */ + DCB_CONNECTOR_VGA = 0x00, // VGA 15-pin connector + DCB_CONNECTOR_DVI_A = 0x01, // DVI-A + DCB_CONNECTOR_POD_VGA = 0x02, // Pod - VGA 15-pin connector + DCB_CONNECTOR_TV_0 = 0x10, // TV - Composite Out + DCB_CONNECTOR_TV_1 = 0x11, // TV - S-Video Out + DCB_CONNECTOR_TV_2 = 0x12, // TV - S-Video Breakout - Composite + DCB_CONNECTOR_TV_3 = 0x13, // HDTV Component - YPrPb + DCB_CONNECTOR_TV_SCART = 0x14, // TV - SCART Connector + DCB_CONNECTOR_TV_SCART_D = 0x16, // TV - Composite SCART over D-connector + DCB_CONNECTOR_TV_DTERM = 0x17, // HDTV - D-connector (EIAJ4120) + DCB_CONNECTOR_POD_TV_3 = 0x18, // Pod - HDTV - YPrPb + DCB_CONNECTOR_POD_TV_1 = 0x19, // Pod - S-Video + DCB_CONNECTOR_POD_TV_0 = 0x1a, // Pod - Composite + + /* DVI digital outputs */ + DCB_CONNECTOR_DVI_I_TV_1 = 0x20, // DVI-I-TV-S-Video + DCB_CONNECTOR_DVI_I_TV_0 = 0x21, // DVI-I-TV-Composite + DCB_CONNECTOR_DVI_I_TV_2 = 0x22, // DVI-I-TV-S-Video Breakout-Composite + DCB_CONNECTOR_DVI_I = 0x30, // DVI-I + DCB_CONNECTOR_DVI_D = 0x31, // DVI-D + DCB_CONNECTOR_DVI_ADC = 0x32, // Apple Display Connector (ADC) + DCB_CONNECTOR_DMS59_0 = 0x38, // LFH-DVI-I-1 + DCB_CONNECTOR_DMS59_1 = 0x39, // LFH-DVI-I-2 + DCB_CONNECTOR_BNC = 0x3c, // BNC Connector [for SDI?] + + /* LVDS / TMDS digital outputs */ + DCB_CONNECTOR_LVDS = 0x40, // LVDS-SPWG-Attached [is this name correct?] + DCB_CONNECTOR_LVDS_SPWG = 0x41, // LVDS-OEM-Attached (non-removable) + DCB_CONNECTOR_LVDS_REM = 0x42, // LVDS-SPWG-Detached [following naming above] + DCB_CONNECTOR_LVDS_SPWG_REM = 0x43, // LVDS-OEM-Detached (removable) + DCB_CONNECTOR_TMDS = 0x45, // TMDS-OEM-Attached (non-removable) + + /* DP digital outputs */ + DCB_CONNECTOR_DP = 0x46, // DisplayPort External Connector + DCB_CONNECTOR_eDP = 0x47, // DisplayPort Internal Connector + DCB_CONNECTOR_mDP = 0x48, // DisplayPort (Mini) External Connector + + /* Dock outputs (not used) */ + DCB_CONNECTOR_DOCK_VGA_0 = 0x50, // VGA 15-pin if not docked + DCB_CONNECTOR_DOCK_VGA_1 = 0x51, // VGA 15-pin if docked + DCB_CONNECTOR_DOCK_DVI_I_0 = 0x52, // DVI-I if not docked + DCB_CONNECTOR_DOCK_DVI_I_1 = 0x53, // DVI-I if docked + DCB_CONNECTOR_DOCK_DVI_D_0 = 0x54, // DVI-D if not docked + DCB_CONNECTOR_DOCK_DVI_D_1 = 0x55, // DVI-D if docked + DCB_CONNECTOR_DOCK_DP_0 = 0x56, // DisplayPort if not docked + DCB_CONNECTOR_DOCK_DP_1 = 0x57, // DisplayPort if docked + DCB_CONNECTOR_DOCK_mDP_0 = 0x58, // DisplayPort (Mini) if not docked + DCB_CONNECTOR_DOCK_mDP_1 = 0x59, // DisplayPort (Mini) if docked + + /* HDMI? digital outputs */ + DCB_CONNECTOR_HDMI_0 = 0x60, // HDMI? See [1] in top-level enum comment above + DCB_CONNECTOR_HDMI_1 = 0x61, // HDMI-A connector + DCB_CONNECTOR_SPDIF = 0x62, // Audio S/PDIF connector + DCB_CONNECTOR_HDMI_C = 0x63, // HDMI-C (Mini) connector + + /* Misc. digital outputs */ + DCB_CONNECTOR_DMS59_DP0 = 0x64, // LFH-DP-1 + DCB_CONNECTOR_DMS59_DP1 = 0x65, // LFH-DP-2 + DCB_CONNECTOR_WFD = 0x70, // Virtual connector for Wifi Display (WFD) + DCB_CONNECTOR_USB_C = 0x71, // [DP over USB-C; not present in docs] + DCB_CONNECTOR_NONE = 0xff // Skip Entry }; struct nvbios_connT { diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h index 226c7ec56b8ed..64fed208e4cf8 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h @@ -44,6 +44,9 @@ typedef void (*nvkm_gsp_event_func)(struct nvkm_gsp_event *, void *repv, u32 rep * NVKM_GSP_RPC_REPLY_NOWAIT - If specified, immediately return to the * caller after the GSP RPC command is issued. * + * NVKM_GSP_RPC_REPLY_NOSEQ - If specified, exactly like NOWAIT + * but don't emit RPC sequence number. + * * NVKM_GSP_RPC_REPLY_RECV - If specified, wait and receive the entire GSP * RPC message after the GSP RPC command is issued. * @@ -53,6 +56,7 @@ typedef void (*nvkm_gsp_event_func)(struct nvkm_gsp_event *, void *repv, u32 rep */ enum nvkm_gsp_rpc_reply_policy { NVKM_GSP_RPC_REPLY_NOWAIT = 0, + NVKM_GSP_RPC_REPLY_NOSEQ, NVKM_GSP_RPC_REPLY_RECV, NVKM_GSP_RPC_REPLY_POLL, }; @@ -73,6 +77,10 @@ struct nvkm_gsp { const struct firmware *bl; const struct firmware *rm; + + struct { + struct nvkm_falcon_fw sb; + } falcon; } fws; struct nvkm_firmware fw; @@ -238,6 +246,8 @@ struct nvkm_gsp { /* The size of the registry RPC */ size_t registry_rpc_size; + u32 rpc_seq; + #ifdef CONFIG_DEBUG_FS /* * Logging buffers in debugfs. The wrapper objects need to remain diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index 63621b1510f60..902e0e93e9682 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -1230,6 +1230,9 @@ nouveau_connector_aux_xfer(struct drm_dp_aux *obj, struct drm_dp_aux_msg *msg) u8 size = msg->size; int ret; + if (pm_runtime_suspended(nv_connector->base.dev->dev)) + return -EBUSY; + nv_encoder = find_encoder(&nv_connector->base, DCB_OUTPUT_DP); if (!nv_encoder) return -ENODEV; diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 1527b801f013a..dc469e571c0aa 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -983,7 +983,7 @@ nouveau_do_suspend(struct nouveau_drm *drm, bool runtime) } NV_DEBUG(drm, "suspending object tree...\n"); - ret = nvif_client_suspend(&drm->_client); + ret = nvif_client_suspend(&drm->_client, runtime); if (ret) goto fail_client; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index 869d4335c0f45..4a193b7d6d9e4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -183,11 +183,11 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha fctx->context = drm->runl[chan->runlist].context_base + chan->chid; if (chan == drm->cechan) - strcpy(fctx->name, "copy engine channel"); + strscpy(fctx->name, "copy engine channel"); else if (chan == drm->channel) - strcpy(fctx->name, "generic kernel channel"); + strscpy(fctx->name, "generic kernel channel"); else - strcpy(fctx->name, cli->name); + strscpy(fctx->name, cli->name); kref_init(&fctx->fence_ref); if (!priv->uevent) diff --git a/drivers/gpu/drm/nouveau/nouveau_nvif.c b/drivers/gpu/drm/nouveau/nouveau_nvif.c index adb802421fda5..eeb4ebbc16bf6 100644 --- a/drivers/gpu/drm/nouveau/nouveau_nvif.c +++ b/drivers/gpu/drm/nouveau/nouveau_nvif.c @@ -62,10 +62,16 @@ nvkm_client_resume(void *priv) } static int -nvkm_client_suspend(void *priv) +nvkm_client_suspend(void *priv, bool runtime) { struct nvkm_client *client = priv; - return nvkm_object_fini(&client->object, true); + enum nvkm_suspend_state state; + + if (runtime) + state = NVKM_RUNTIME_SUSPEND; + else + state = NVKM_SUSPEND; + return nvkm_object_fini(&client->object, state); } static int diff --git a/drivers/gpu/drm/nouveau/nvif/client.c b/drivers/gpu/drm/nouveau/nvif/client.c index fdf5054ed7d83..36d3c99786bd5 100644 --- a/drivers/gpu/drm/nouveau/nvif/client.c +++ b/drivers/gpu/drm/nouveau/nvif/client.c @@ -30,9 +30,9 @@ #include int -nvif_client_suspend(struct nvif_client *client) +nvif_client_suspend(struct nvif_client *client, bool runtime) { - return client->driver->suspend(client->object.priv); + return client->driver->suspend(client->object.priv, runtime); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c index 36a31e9eea228..5bf62940d7be1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c @@ -41,7 +41,7 @@ nvkm_engine_reset(struct nvkm_engine *engine) if (engine->func->reset) return engine->func->reset(engine); - nvkm_subdev_fini(&engine->subdev, false); + nvkm_subdev_fini(&engine->subdev, NVKM_POWEROFF); return nvkm_subdev_init(&engine->subdev); } @@ -98,7 +98,7 @@ nvkm_engine_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data) } static int -nvkm_engine_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_engine_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_engine *engine = nvkm_engine(subdev); if (engine->func->fini) diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c index 45051a1249dac..b8fc9be678515 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c @@ -141,7 +141,7 @@ nvkm_ioctl_new(struct nvkm_client *client, } ret = -EEXIST; } - nvkm_object_fini(object, false); + nvkm_object_fini(object, NVKM_POWEROFF); } nvkm_object_del(&object); @@ -160,7 +160,7 @@ nvkm_ioctl_del(struct nvkm_client *client, nvif_ioctl(object, "delete size %d\n", size); if (!(ret = nvif_unvers(ret, &data, &size, args->none))) { nvif_ioctl(object, "delete\n"); - nvkm_object_fini(object, false); + nvkm_object_fini(object, NVKM_POWEROFF); nvkm_object_del(&object); } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/object.c b/drivers/gpu/drm/nouveau/nvkm/core/object.c index 390c265cf8af4..af9f00f74c288 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/object.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/object.c @@ -142,13 +142,25 @@ nvkm_object_bind(struct nvkm_object *object, struct nvkm_gpuobj *gpuobj, } int -nvkm_object_fini(struct nvkm_object *object, bool suspend) +nvkm_object_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { - const char *action = suspend ? "suspend" : "fini"; + const char *action; struct nvkm_object *child; s64 time; int ret; + switch (suspend) { + case NVKM_POWEROFF: + default: + action = "fini"; + break; + case NVKM_SUSPEND: + action = "suspend"; + break; + case NVKM_RUNTIME_SUSPEND: + action = "runtime"; + break; + } nvif_debug(object, "%s children...\n", action); time = ktime_to_us(ktime_get()); list_for_each_entry_reverse(child, &object->tree, head) { @@ -212,11 +224,11 @@ nvkm_object_init(struct nvkm_object *object) fail_child: list_for_each_entry_continue_reverse(child, &object->tree, head) - nvkm_object_fini(child, false); + nvkm_object_fini(child, NVKM_POWEROFF); fail: nvif_error(object, "init failed with %d\n", ret); if (object->func->fini) - object->func->fini(object, false); + object->func->fini(object, NVKM_POWEROFF); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c index 5db80d1780f08..7c9edf7527686 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c @@ -87,7 +87,7 @@ nvkm_oproxy_uevent(struct nvkm_object *object, void *argv, u32 argc, } static int -nvkm_oproxy_fini(struct nvkm_object *object, bool suspend) +nvkm_oproxy_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nvkm_oproxy *oproxy = nvkm_oproxy(object); int ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c index 6c20e827a069a..b7045d1c84158 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c @@ -51,12 +51,24 @@ nvkm_subdev_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data) } int -nvkm_subdev_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_subdev_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_device *device = subdev->device; - const char *action = suspend ? "suspend" : subdev->use.enabled ? "fini" : "reset"; + const char *action; s64 time; + switch (suspend) { + case NVKM_POWEROFF: + default: + action = subdev->use.enabled ? "fini" : "reset"; + break; + case NVKM_SUSPEND: + action = "suspend"; + break; + case NVKM_RUNTIME_SUSPEND: + action = "runtime"; + break; + } nvkm_trace(subdev, "%s running...\n", action); time = ktime_to_us(ktime_get()); @@ -186,7 +198,7 @@ void nvkm_subdev_unref(struct nvkm_subdev *subdev) { if (refcount_dec_and_mutex_lock(&subdev->use.refcount, &subdev->use.mutex)) { - nvkm_subdev_fini(subdev, false); + nvkm_subdev_fini(subdev, NVKM_POWEROFF); mutex_unlock(&subdev->use.mutex); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/uevent.c b/drivers/gpu/drm/nouveau/nvkm/core/uevent.c index cc254c390a571..46beb6e470ee3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/uevent.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/uevent.c @@ -73,7 +73,7 @@ nvkm_uevent_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) } static int -nvkm_uevent_fini(struct nvkm_object *object, bool suspend) +nvkm_uevent_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nvkm_uevent *uevent = nvkm_uevent(object); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c index 1c0c601387060..1a3caf6976082 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c @@ -46,7 +46,7 @@ ga100_ce_nonstall(struct nvkm_engine *engine) } int -ga100_ce_fini(struct nvkm_engine *engine, bool suspend) +ga100_ce_fini(struct nvkm_engine *engine, enum nvkm_suspend_state suspend) { nvkm_inth_block(&engine->subdev.inth); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h index 34fd2657134b8..f07b458533104 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h @@ -14,7 +14,7 @@ extern const struct nvkm_object_func gv100_ce_cclass; int ga100_ce_oneinit(struct nvkm_engine *); int ga100_ce_init(struct nvkm_engine *); -int ga100_ce_fini(struct nvkm_engine *, bool); +int ga100_ce_fini(struct nvkm_engine *, enum nvkm_suspend_state); int ga100_ce_nonstall(struct nvkm_engine *); u32 gb202_ce_grce_mask(struct nvkm_device *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 3375a59ebf1a4..a965914f1c2fb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2935,13 +2935,25 @@ nvkm_device_engine(struct nvkm_device *device, int type, int inst) } int -nvkm_device_fini(struct nvkm_device *device, bool suspend) +nvkm_device_fini(struct nvkm_device *device, enum nvkm_suspend_state suspend) { - const char *action = suspend ? "suspend" : "fini"; + const char *action; struct nvkm_subdev *subdev; int ret; s64 time; + switch (suspend) { + case NVKM_POWEROFF: + default: + action = "fini"; + break; + case NVKM_SUSPEND: + action = "suspend"; + break; + case NVKM_RUNTIME_SUSPEND: + action = "runtime"; + break; + } nvdev_trace(device, "%s running...\n", action); time = ktime_to_us(ktime_get()); @@ -3031,7 +3043,7 @@ nvkm_device_init(struct nvkm_device *device) if (ret) return ret; - nvkm_device_fini(device, false); + nvkm_device_fini(device, NVKM_POWEROFF); nvdev_trace(device, "init running...\n"); time = ktime_to_us(ktime_get()); @@ -3059,9 +3071,9 @@ nvkm_device_init(struct nvkm_device *device) fail_subdev: list_for_each_entry_from(subdev, &device->subdev, head) - nvkm_subdev_fini(subdev, false); + nvkm_subdev_fini(subdev, NVKM_POWEROFF); fail: - nvkm_device_fini(device, false); + nvkm_device_fini(device, NVKM_POWEROFF); nvdev_error(device, "init failed with %d\n", ret); return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c index 8f0261a0d618b..4c29b60460d48 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c @@ -1605,10 +1605,10 @@ nvkm_device_pci_irq(struct nvkm_device *device) } static void -nvkm_device_pci_fini(struct nvkm_device *device, bool suspend) +nvkm_device_pci_fini(struct nvkm_device *device, enum nvkm_suspend_state suspend) { struct nvkm_device_pci *pdev = nvkm_device_pci(device); - if (suspend) { + if (suspend != NVKM_POWEROFF) { pci_disable_device(pdev->pdev); pdev->suspend = true; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h index 75ee7506d4432..d0c40f0342442 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h @@ -56,5 +56,5 @@ int nvkm_device_ctor(const struct nvkm_device_func *, const char *name, const char *cfg, const char *dbg, struct nvkm_device *); int nvkm_device_init(struct nvkm_device *); -int nvkm_device_fini(struct nvkm_device *, bool suspend); +int nvkm_device_fini(struct nvkm_device *, enum nvkm_suspend_state suspend); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index 58191b7a0494e..32ff3181f47b6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -218,7 +218,7 @@ nvkm_udevice_map(struct nvkm_object *object, void *argv, u32 argc, } static int -nvkm_udevice_fini(struct nvkm_object *object, bool suspend) +nvkm_udevice_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nvkm_udevice *udev = nvkm_udevice(object); struct nvkm_device *device = udev->device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c index b24eb1e560bc7..84745f60912e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c @@ -99,13 +99,13 @@ nvkm_disp_intr(struct nvkm_engine *engine) } static int -nvkm_disp_fini(struct nvkm_engine *engine, bool suspend) +nvkm_disp_fini(struct nvkm_engine *engine, enum nvkm_suspend_state suspend) { struct nvkm_disp *disp = nvkm_disp(engine); struct nvkm_outp *outp; if (disp->func->fini) - disp->func->fini(disp, suspend); + disp->func->fini(disp, suspend != NVKM_POWEROFF); list_for_each_entry(outp, &disp->outps, head) { if (outp->func->fini) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/chan.c index 9b84e357d3544..57a62a2de7c7d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/chan.c @@ -128,7 +128,7 @@ nvkm_disp_chan_child_get(struct nvkm_object *object, int index, struct nvkm_ocla } static int -nvkm_disp_chan_fini(struct nvkm_object *object, bool suspend) +nvkm_disp_chan_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nvkm_disp_chan *chan = nvkm_disp_chan(object); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c index 2dab6612c4fc8..23d1e5c27bb1e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c @@ -191,27 +191,60 @@ nvkm_uconn_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nv spin_lock(&disp->client.lock); if (!conn->object.func) { switch (conn->info.type) { - case DCB_CONNECTOR_VGA : args->v0.type = NVIF_CONN_V0_VGA; break; - case DCB_CONNECTOR_TV_0 : - case DCB_CONNECTOR_TV_1 : - case DCB_CONNECTOR_TV_3 : args->v0.type = NVIF_CONN_V0_TV; break; - case DCB_CONNECTOR_DMS59_0 : - case DCB_CONNECTOR_DMS59_1 : - case DCB_CONNECTOR_DVI_I : args->v0.type = NVIF_CONN_V0_DVI_I; break; - case DCB_CONNECTOR_DVI_D : args->v0.type = NVIF_CONN_V0_DVI_D; break; - case DCB_CONNECTOR_LVDS : args->v0.type = NVIF_CONN_V0_LVDS; break; - case DCB_CONNECTOR_LVDS_SPWG: args->v0.type = NVIF_CONN_V0_LVDS_SPWG; break; - case DCB_CONNECTOR_DMS59_DP0: - case DCB_CONNECTOR_DMS59_DP1: - case DCB_CONNECTOR_DP : - case DCB_CONNECTOR_mDP : - case DCB_CONNECTOR_USB_C : args->v0.type = NVIF_CONN_V0_DP; break; - case DCB_CONNECTOR_eDP : args->v0.type = NVIF_CONN_V0_EDP; break; - case DCB_CONNECTOR_HDMI_0 : - case DCB_CONNECTOR_HDMI_1 : - case DCB_CONNECTOR_HDMI_C : args->v0.type = NVIF_CONN_V0_HDMI; break; + /* VGA */ + case DCB_CONNECTOR_DVI_A : + case DCB_CONNECTOR_POD_VGA : + case DCB_CONNECTOR_VGA : args->v0.type = NVIF_CONN_V0_VGA; break; + + /* TV */ + case DCB_CONNECTOR_TV_0 : + case DCB_CONNECTOR_TV_1 : + case DCB_CONNECTOR_TV_2 : + case DCB_CONNECTOR_TV_SCART : + case DCB_CONNECTOR_TV_SCART_D : + case DCB_CONNECTOR_TV_DTERM : + case DCB_CONNECTOR_POD_TV_3 : + case DCB_CONNECTOR_POD_TV_1 : + case DCB_CONNECTOR_POD_TV_0 : + case DCB_CONNECTOR_TV_3 : args->v0.type = NVIF_CONN_V0_TV; break; + + /* DVI */ + case DCB_CONNECTOR_DVI_I_TV_1 : + case DCB_CONNECTOR_DVI_I_TV_0 : + case DCB_CONNECTOR_DVI_I_TV_2 : + case DCB_CONNECTOR_DVI_ADC : + case DCB_CONNECTOR_DMS59_0 : + case DCB_CONNECTOR_DMS59_1 : + case DCB_CONNECTOR_DVI_I : args->v0.type = NVIF_CONN_V0_DVI_I; break; + case DCB_CONNECTOR_TMDS : + case DCB_CONNECTOR_DVI_D : args->v0.type = NVIF_CONN_V0_DVI_D; break; + + /* LVDS */ + case DCB_CONNECTOR_LVDS : args->v0.type = NVIF_CONN_V0_LVDS; break; + case DCB_CONNECTOR_LVDS_SPWG : args->v0.type = NVIF_CONN_V0_LVDS_SPWG; break; + + /* DP */ + case DCB_CONNECTOR_DMS59_DP0 : + case DCB_CONNECTOR_DMS59_DP1 : + case DCB_CONNECTOR_DP : + case DCB_CONNECTOR_mDP : + case DCB_CONNECTOR_USB_C : args->v0.type = NVIF_CONN_V0_DP; break; + case DCB_CONNECTOR_eDP : args->v0.type = NVIF_CONN_V0_EDP; break; + + /* HDMI */ + case DCB_CONNECTOR_HDMI_0 : + case DCB_CONNECTOR_HDMI_1 : + case DCB_CONNECTOR_HDMI_C : args->v0.type = NVIF_CONN_V0_HDMI; break; + + /* + * Dock & unused outputs. + * BNC, SPDIF, WFD, and detached LVDS go here. + */ default: - WARN_ON(1); + nvkm_warn(&disp->engine.subdev, + "unimplemented connector type 0x%02x\n", + conn->info.type); + args->v0.type = NVIF_CONN_V0_VGA; ret = -EINVAL; break; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c index fd5ee9f0af360..cf8e356867b4f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c @@ -93,13 +93,13 @@ nvkm_falcon_intr(struct nvkm_engine *engine) } static int -nvkm_falcon_fini(struct nvkm_engine *engine, bool suspend) +nvkm_falcon_fini(struct nvkm_engine *engine, enum nvkm_suspend_state suspend) { struct nvkm_falcon *falcon = nvkm_falcon(engine); struct nvkm_device *device = falcon->engine.subdev.device; const u32 base = falcon->addr; - if (!suspend) { + if (suspend == NVKM_POWEROFF) { nvkm_memory_unref(&falcon->core); if (falcon->external) { vfree(falcon->data.data); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 6fd4e60634fbe..1561287a32f20 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -122,7 +122,7 @@ nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_dev } static int -nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend) +nvkm_fifo_fini(struct nvkm_engine *engine, enum nvkm_suspend_state suspend) { struct nvkm_fifo *fifo = nvkm_fifo(engine); struct nvkm_runl *runl; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 52420a1edca52..c978b97e10c64 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -72,7 +72,7 @@ struct nvkm_uobj { }; static int -nvkm_uchan_object_fini_1(struct nvkm_oproxy *oproxy, bool suspend) +nvkm_uchan_object_fini_1(struct nvkm_oproxy *oproxy, enum nvkm_suspend_state suspend) { struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); struct nvkm_chan *chan = uobj->chan; @@ -87,7 +87,7 @@ nvkm_uchan_object_fini_1(struct nvkm_oproxy *oproxy, bool suspend) nvkm_chan_cctx_bind(chan, ectx->engn, NULL); if (refcount_dec_and_test(&ectx->uses)) - nvkm_object_fini(ectx->object, false); + nvkm_object_fini(ectx->object, NVKM_POWEROFF); mutex_unlock(&chan->cgrp->mutex); } @@ -269,7 +269,7 @@ nvkm_uchan_map(struct nvkm_object *object, void *argv, u32 argc, } static int -nvkm_uchan_fini(struct nvkm_object *object, bool suspend) +nvkm_uchan_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nvkm_chan *chan = nvkm_uchan(object)->chan; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c index f5e68f09df768..cd4908b1b4dff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c @@ -168,11 +168,11 @@ nvkm_gr_init(struct nvkm_engine *engine) } static int -nvkm_gr_fini(struct nvkm_engine *engine, bool suspend) +nvkm_gr_fini(struct nvkm_engine *engine, enum nvkm_suspend_state suspend) { struct nvkm_gr *gr = nvkm_gr(engine); if (gr->func->fini) - return gr->func->fini(gr, suspend); + return gr->func->fini(gr, suspend != NVKM_POWEROFF); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 3ea447f6a45b5..3608215f0f11c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2330,7 +2330,7 @@ gf100_gr_reset(struct nvkm_gr *base) WARN_ON(gf100_gr_fecs_halt_pipeline(gr)); - subdev->func->fini(subdev, false); + subdev->func->fini(subdev, NVKM_POWEROFF); nvkm_mc_disable(device, subdev->type, subdev->inst); if (gr->func->gpccs.reset) gr->func->gpccs.reset(gr); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c index ca822f07b63e9..82937df8b8c0d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c @@ -1158,7 +1158,7 @@ nv04_gr_chan_dtor(struct nvkm_object *object) } static int -nv04_gr_chan_fini(struct nvkm_object *object, bool suspend) +nv04_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nv04_gr_chan *chan = nv04_gr_chan(object); struct nv04_gr *gr = chan->gr; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c index 92ef7c9b29101..fcb4e4fce83f7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c @@ -951,7 +951,7 @@ nv10_gr_context_switch(struct nv10_gr *gr) } static int -nv10_gr_chan_fini(struct nvkm_object *object, bool suspend) +nv10_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nv10_gr_chan *chan = nv10_gr_chan(object); struct nv10_gr *gr = chan->gr; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c index 13407fafe9479..ab57b3b40228e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c @@ -27,7 +27,7 @@ nv20_gr_chan_init(struct nvkm_object *object) } int -nv20_gr_chan_fini(struct nvkm_object *object, bool suspend) +nv20_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nv20_gr_chan *chan = nv20_gr_chan(object); struct nv20_gr *gr = chan->gr; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h index c0d2be53413e0..786c7832f7acd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h @@ -31,5 +31,5 @@ struct nv20_gr_chan { void *nv20_gr_chan_dtor(struct nvkm_object *); int nv20_gr_chan_init(struct nvkm_object *); -int nv20_gr_chan_fini(struct nvkm_object *, bool); +int nv20_gr_chan_fini(struct nvkm_object *, enum nvkm_suspend_state); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c index b609b0150ba14..e3e797cf30349 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c @@ -89,7 +89,7 @@ nv40_gr_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent, } static int -nv40_gr_chan_fini(struct nvkm_object *object, bool suspend) +nv40_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nv40_gr_chan *chan = nv40_gr_chan(object); struct nv40_gr *gr = chan->gr; @@ -101,7 +101,7 @@ nv40_gr_chan_fini(struct nvkm_object *object, bool suspend) nvkm_mask(device, 0x400720, 0x00000001, 0x00000000); if (nvkm_rd32(device, 0x40032c) == inst) { - if (suspend) { + if (suspend != NVKM_POWEROFF) { nvkm_wr32(device, 0x400720, 0x00000000); nvkm_wr32(device, 0x400784, inst); nvkm_mask(device, 0x400310, 0x00000020, 0x00000020); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c index 4b1374adbda3a..38146f9cc81c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c @@ -65,7 +65,7 @@ nv44_mpeg_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent, } static int -nv44_mpeg_chan_fini(struct nvkm_object *object, bool suspend) +nv44_mpeg_chan_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nv44_mpeg_chan *chan = nv44_mpeg_chan(object); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c index f2c60da5d1e86..3e4d6a680ee95 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c @@ -37,7 +37,7 @@ nvkm_sec2_finimsg(void *priv, struct nvfw_falcon_msg *hdr) } static int -nvkm_sec2_fini(struct nvkm_engine *engine, bool suspend) +nvkm_sec2_fini(struct nvkm_engine *engine, enum nvkm_suspend_state suspend) { struct nvkm_sec2 *sec2 = nvkm_sec2(engine); struct nvkm_subdev *subdev = &sec2->engine.subdev; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c b/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c index f7d3ba0afb556..910a5bb2d191d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c @@ -76,7 +76,7 @@ nvkm_xtensa_intr(struct nvkm_engine *engine) } static int -nvkm_xtensa_fini(struct nvkm_engine *engine, bool suspend) +nvkm_xtensa_fini(struct nvkm_engine *engine, enum nvkm_suspend_state suspend) { struct nvkm_xtensa *xtensa = nvkm_xtensa(engine); struct nvkm_device *device = xtensa->engine.subdev.device; @@ -85,7 +85,7 @@ nvkm_xtensa_fini(struct nvkm_engine *engine, bool suspend) nvkm_wr32(device, base + 0xd84, 0); /* INTR_EN */ nvkm_wr32(device, base + 0xd94, 0); /* FIFO_CTRL */ - if (!suspend) + if (suspend == NVKM_POWEROFF) nvkm_memory_unref(&xtensa->gpu_fw); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c index 9b8ca4e898f90..13d829593180c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c @@ -182,7 +182,7 @@ nvkm_acr_managed_falcon(struct nvkm_device *device, enum nvkm_acr_lsf_id id) } static int -nvkm_acr_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_acr_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { if (!subdev->use.enabled) return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index 91bc53be97ffc..7dee55bf9ada9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -90,7 +90,7 @@ nvkm_bar_bar2_init(struct nvkm_device *device) } static int -nvkm_bar_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_bar_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_bar *bar = nvkm_bar(subdev); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c index 178dc56909c27..71420f81714b3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c @@ -577,7 +577,7 @@ nvkm_clk_read(struct nvkm_clk *clk, enum nv_clk_src src) } static int -nvkm_clk_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_clk_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_clk *clk = nvkm_clk(subdev); flush_work(&clk->work); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/base.c index 3d9319c319c64..ad5ec9ee1294d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/base.c @@ -67,11 +67,11 @@ nvkm_devinit_post(struct nvkm_devinit *init) } static int -nvkm_devinit_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_devinit_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_devinit *init = nvkm_devinit(subdev); /* force full reinit on resume */ - if (suspend) + if (suspend != NVKM_POWEROFF) init->post = true; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c index b53ac9a2552fa..d8d32bb5bcd9e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c @@ -51,7 +51,7 @@ nvkm_fault_intr(struct nvkm_subdev *subdev) } static int -nvkm_fault_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_fault_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_fault *fault = nvkm_fault(subdev); if (fault->func->fini) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c index cd2fbc0472d8e..8ab052d18e5d3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c @@ -56,7 +56,7 @@ nvkm_ufault_map(struct nvkm_object *object, void *argv, u32 argc, } static int -nvkm_ufault_fini(struct nvkm_object *object, bool suspend) +nvkm_ufault_fini(struct nvkm_object *object, enum nvkm_suspend_state suspend) { struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); buffer->fault->func->buffer.fini(buffer); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index 8a286a9349ac6..7ce1b65e2c1c2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -279,7 +279,7 @@ nvkm_fb_ctor(const struct nvkm_fb_func *func, struct nvkm_device *device, mutex_init(&fb->tags.mutex); if (func->sysmem.flush_page_init) { - fb->sysmem.flush_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + fb->sysmem.flush_page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); if (!fb->sysmem.flush_page) return -ENOMEM; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c index b196baa376dcd..b2c34878a68f6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c @@ -144,7 +144,7 @@ nvkm_gpio_intr(struct nvkm_subdev *subdev) } static int -nvkm_gpio_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_gpio_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_gpio *gpio = nvkm_gpio(subdev); u32 mask = (1ULL << gpio->func->lines) - 1; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ad102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ad102.c index 35d1fcef520bf..c456a96268231 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ad102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ad102.c @@ -30,6 +30,9 @@ ad102_gsp = { .booter.ctor = ga102_gsp_booter_ctor, + .fwsec_sb.ctor = tu102_gsp_fwsec_sb_ctor, + .fwsec_sb.dtor = tu102_gsp_fwsec_sb_dtor, + .dtor = r535_gsp_dtor, .oneinit = tu102_gsp_oneinit, .init = tu102_gsp_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c index 7ccb417610665..30cb843ba35cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c @@ -48,7 +48,7 @@ nvkm_gsp_intr_stall(struct nvkm_gsp *gsp, enum nvkm_subdev_type type, int inst) } static int -nvkm_gsp_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_gsp_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_gsp *gsp = nvkm_gsp(subdev); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/fwsec.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/fwsec.c index 5b721bd9d7994..851140e801220 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/fwsec.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/fwsec.c @@ -259,18 +259,16 @@ nvkm_gsp_fwsec_v3(struct nvkm_gsp *gsp, const char *name, } static int -nvkm_gsp_fwsec(struct nvkm_gsp *gsp, const char *name, u32 init_cmd) +nvkm_gsp_fwsec_init(struct nvkm_gsp *gsp, struct nvkm_falcon_fw *fw, const char *name, u32 init_cmd) { struct nvkm_subdev *subdev = &gsp->subdev; struct nvkm_device *device = subdev->device; struct nvkm_bios *bios = device->bios; const union nvfw_falcon_ucode_desc *desc; struct nvbios_pmuE flcn_ucode; - u8 idx, ver, hdr; u32 data; u16 size, vers; - struct nvkm_falcon_fw fw = {}; - u32 mbox0 = 0; + u8 idx, ver, hdr; int ret; /* Lookup in VBIOS. */ @@ -291,8 +289,8 @@ nvkm_gsp_fwsec(struct nvkm_gsp *gsp, const char *name, u32 init_cmd) vers = (desc->v2.Hdr & 0x0000ff00) >> 8; switch (vers) { - case 2: ret = nvkm_gsp_fwsec_v2(gsp, name, &desc->v2, size, init_cmd, &fw); break; - case 3: ret = nvkm_gsp_fwsec_v3(gsp, name, &desc->v3, size, init_cmd, &fw); break; + case 2: ret = nvkm_gsp_fwsec_v2(gsp, name, &desc->v2, size, init_cmd, fw); break; + case 3: ret = nvkm_gsp_fwsec_v3(gsp, name, &desc->v3, size, init_cmd, fw); break; default: nvkm_error(subdev, "%s(v%d): version unknown\n", name, vers); return -EINVAL; @@ -303,15 +301,19 @@ nvkm_gsp_fwsec(struct nvkm_gsp *gsp, const char *name, u32 init_cmd) return ret; } - /* Boot. */ - ret = nvkm_falcon_fw_boot(&fw, subdev, true, &mbox0, NULL, 0, 0); - nvkm_falcon_fw_dtor(&fw); - if (ret) - return ret; - return 0; } +static int +nvkm_gsp_fwsec_boot(struct nvkm_gsp *gsp, struct nvkm_falcon_fw *fw) +{ + struct nvkm_subdev *subdev = &gsp->subdev; + u32 mbox0 = 0; + + /* Boot */ + return nvkm_falcon_fw_boot(fw, subdev, true, &mbox0, NULL, 0, 0); +} + int nvkm_gsp_fwsec_sb(struct nvkm_gsp *gsp) { @@ -320,7 +322,7 @@ nvkm_gsp_fwsec_sb(struct nvkm_gsp *gsp) int ret; u32 err; - ret = nvkm_gsp_fwsec(gsp, "fwsec-sb", NVFW_FALCON_APPIF_DMEMMAPPER_CMD_SB); + ret = nvkm_gsp_fwsec_boot(gsp, &gsp->fws.falcon.sb); if (ret) return ret; @@ -334,27 +336,42 @@ nvkm_gsp_fwsec_sb(struct nvkm_gsp *gsp) return 0; } +int +nvkm_gsp_fwsec_sb_init(struct nvkm_gsp *gsp) +{ + return nvkm_gsp_fwsec_init(gsp, &gsp->fws.falcon.sb, "fwsec-sb", + NVFW_FALCON_APPIF_DMEMMAPPER_CMD_SB); +} + int nvkm_gsp_fwsec_frts(struct nvkm_gsp *gsp) { struct nvkm_subdev *subdev = &gsp->subdev; struct nvkm_device *device = subdev->device; + struct nvkm_falcon_fw fw = {}; int ret; u32 err, wpr2_lo, wpr2_hi; - ret = nvkm_gsp_fwsec(gsp, "fwsec-frts", NVFW_FALCON_APPIF_DMEMMAPPER_CMD_FRTS); + ret = nvkm_gsp_fwsec_init(gsp, &fw, "fwsec-frts", NVFW_FALCON_APPIF_DMEMMAPPER_CMD_FRTS); if (ret) return ret; + ret = nvkm_gsp_fwsec_boot(gsp, &fw); + if (ret) + goto fwsec_dtor; + /* Verify. */ err = nvkm_rd32(device, 0x001400 + (0xe * 4)) >> 16; if (err) { nvkm_error(subdev, "fwsec-frts: 0x%04x\n", err); - return -EIO; + ret = -EIO; + } else { + wpr2_lo = nvkm_rd32(device, 0x1fa824); + wpr2_hi = nvkm_rd32(device, 0x1fa828); + nvkm_debug(subdev, "fwsec-frts: WPR2 @ %08x - %08x\n", wpr2_lo, wpr2_hi); } - wpr2_lo = nvkm_rd32(device, 0x1fa824); - wpr2_hi = nvkm_rd32(device, 0x1fa828); - nvkm_debug(subdev, "fwsec-frts: WPR2 @ %08x - %08x\n", wpr2_lo, wpr2_hi); - return 0; +fwsec_dtor: + nvkm_falcon_fw_dtor(&fw); + return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c index d201e8697226b..27a13aeccd3cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c @@ -47,6 +47,9 @@ ga100_gsp = { .booter.ctor = tu102_gsp_booter_ctor, + .fwsec_sb.ctor = tu102_gsp_fwsec_sb_ctor, + .fwsec_sb.dtor = tu102_gsp_fwsec_sb_dtor, + .dtor = r535_gsp_dtor, .oneinit = tu102_gsp_oneinit, .init = tu102_gsp_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c index 917f7e2f6c466..b6b3eb6f4c006 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c @@ -158,6 +158,9 @@ ga102_gsp_r535 = { .booter.ctor = ga102_gsp_booter_ctor, + .fwsec_sb.ctor = tu102_gsp_fwsec_sb_ctor, + .fwsec_sb.dtor = tu102_gsp_fwsec_sb_dtor, + .dtor = r535_gsp_dtor, .oneinit = tu102_gsp_oneinit, .init = tu102_gsp_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gh100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gh100.c index b0dd5fce7bad3..88436a2641779 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gh100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gh100.c @@ -17,7 +17,7 @@ #include int -gh100_gsp_fini(struct nvkm_gsp *gsp, bool suspend) +gh100_gsp_fini(struct nvkm_gsp *gsp, enum nvkm_suspend_state suspend) { struct nvkm_falcon *falcon = &gsp->falcon; int ret, time = 4000; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h index c3494b7ac572b..71b7203bef507 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h @@ -6,7 +6,9 @@ enum nvkm_acr_lsf_id; int nvkm_gsp_fwsec_frts(struct nvkm_gsp *); + int nvkm_gsp_fwsec_sb(struct nvkm_gsp *); +int nvkm_gsp_fwsec_sb_init(struct nvkm_gsp *gsp); struct nvkm_gsp_fwif { int version; @@ -49,10 +51,15 @@ struct nvkm_gsp_func { struct nvkm_falcon *, struct nvkm_falcon_fw *); } booter; + struct { + int (*ctor)(struct nvkm_gsp *); + void (*dtor)(struct nvkm_gsp *); + } fwsec_sb; + void (*dtor)(struct nvkm_gsp *); int (*oneinit)(struct nvkm_gsp *); int (*init)(struct nvkm_gsp *); - int (*fini)(struct nvkm_gsp *, bool suspend); + int (*fini)(struct nvkm_gsp *, enum nvkm_suspend_state suspend); int (*reset)(struct nvkm_gsp *); struct { @@ -64,9 +71,11 @@ extern const struct nvkm_falcon_func tu102_gsp_flcn; extern const struct nvkm_falcon_fw_func tu102_gsp_fwsec; int tu102_gsp_booter_ctor(struct nvkm_gsp *, const char *, const struct firmware *, struct nvkm_falcon *, struct nvkm_falcon_fw *); +int tu102_gsp_fwsec_sb_ctor(struct nvkm_gsp *); +void tu102_gsp_fwsec_sb_dtor(struct nvkm_gsp *); int tu102_gsp_oneinit(struct nvkm_gsp *); int tu102_gsp_init(struct nvkm_gsp *); -int tu102_gsp_fini(struct nvkm_gsp *, bool suspend); +int tu102_gsp_fini(struct nvkm_gsp *, enum nvkm_suspend_state suspend); int tu102_gsp_reset(struct nvkm_gsp *); u64 tu102_gsp_wpr_heap_size(struct nvkm_gsp *); @@ -78,15 +87,28 @@ int ga102_gsp_reset(struct nvkm_gsp *); int gh100_gsp_oneinit(struct nvkm_gsp *); int gh100_gsp_init(struct nvkm_gsp *); -int gh100_gsp_fini(struct nvkm_gsp *, bool suspend); +int gh100_gsp_fini(struct nvkm_gsp *, enum nvkm_suspend_state suspend); void r535_gsp_dtor(struct nvkm_gsp *); int r535_gsp_oneinit(struct nvkm_gsp *); int r535_gsp_init(struct nvkm_gsp *); -int r535_gsp_fini(struct nvkm_gsp *, bool suspend); +int r535_gsp_fini(struct nvkm_gsp *, enum nvkm_suspend_state suspend); int nvkm_gsp_new_(const struct nvkm_gsp_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **); +static inline int nvkm_gsp_fwsec_sb_ctor(struct nvkm_gsp *gsp) +{ + if (gsp->func->fwsec_sb.ctor) + return gsp->func->fwsec_sb.ctor(gsp); + return 0; +} + +static inline void nvkm_gsp_fwsec_sb_dtor(struct nvkm_gsp *gsp) +{ + if (gsp->func->fwsec_sb.dtor) + gsp->func->fwsec_sb.dtor(gsp); +} + extern const struct nvkm_gsp_func gv100_gsp; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c index 150e22fde2ac2..e962d0e8f8377 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c @@ -208,7 +208,7 @@ r535_fbsr_resume(struct nvkm_gsp *gsp) } static int -r535_fbsr_suspend(struct nvkm_gsp *gsp) +r535_fbsr_suspend(struct nvkm_gsp *gsp, bool runtime) { struct nvkm_subdev *subdev = &gsp->subdev; struct nvkm_device *device = subdev->device; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c index 32e6a065d6d7a..a575a8dbf727d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c @@ -704,7 +704,7 @@ r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp) build_registry(gsp, rpc); - return nvkm_gsp_rpc_wr(gsp, rpc, NVKM_GSP_RPC_REPLY_NOWAIT); + return nvkm_gsp_rpc_wr(gsp, rpc, NVKM_GSP_RPC_REPLY_NOSEQ); fail: clean_registry(gsp); @@ -737,8 +737,8 @@ r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps) if (!obj) goto done; - if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || - WARN_ON(obj->buffer.length != 4)) + if (obj->type != ACPI_TYPE_BUFFER || + obj->buffer.length != 4) goto done; caps->status = 0; @@ -773,8 +773,8 @@ r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt) if (!obj) goto done; - if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || - WARN_ON(obj->buffer.length != 4)) + if (obj->type != ACPI_TYPE_BUFFER || + obj->buffer.length != 4) goto done; jt->status = 0; @@ -861,8 +861,8 @@ r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod) _DOD = output.pointer; - if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) || - WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList))) + if (_DOD->type != ACPI_TYPE_PACKAGE || + _DOD->package.count > ARRAY_SIZE(dod->acpiIdList)) return; for (int i = 0; i < _DOD->package.count; i++) { @@ -921,7 +921,7 @@ r535_gsp_set_system_info(struct nvkm_gsp *gsp) info->pciConfigMirrorSize = device->pci->func->cfg.size; r535_gsp_acpi_info(gsp, &info->acpiMethodData); - return nvkm_gsp_rpc_wr(gsp, info, NVKM_GSP_RPC_REPLY_NOWAIT); + return nvkm_gsp_rpc_wr(gsp, info, NVKM_GSP_RPC_REPLY_NOSEQ); } static int @@ -1721,7 +1721,7 @@ r535_gsp_sr_data_size(struct nvkm_gsp *gsp) } int -r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend) +r535_gsp_fini(struct nvkm_gsp *gsp, enum nvkm_suspend_state suspend) { struct nvkm_rm *rm = gsp->rm; int ret; @@ -1748,7 +1748,7 @@ r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend) sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr; sr->sizeOfSuspendResumeData = len; - ret = rm->api->fbsr->suspend(gsp); + ret = rm->api->fbsr->suspend(gsp, suspend == NVKM_RUNTIME_SUSPEND); if (ret) { nvkm_gsp_mem_dtor(&gsp->sr.meta); nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3); @@ -1817,12 +1817,16 @@ r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp) RM_RISCV_UCODE_DESC *desc; int ret; + ret = nvkm_gsp_fwsec_sb_ctor(gsp); + if (ret) + return ret; + hdr = nvfw_bin_hdr(&gsp->subdev, fw->data); desc = (void *)fw->data + hdr->header_offset; ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw); if (ret) - return ret; + goto dtor_fwsec; memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size); @@ -1831,6 +1835,9 @@ r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp) gsp->boot.manifest_offset = desc->manifestOffset; gsp->boot.app_version = desc->appVersion; return 0; +dtor_fwsec: + nvkm_gsp_fwsec_sb_dtor(gsp); + return ret; } static const struct nvkm_firmware_func @@ -2101,6 +2108,7 @@ r535_gsp_dtor(struct nvkm_gsp *gsp) mutex_destroy(&gsp->cmdq.mutex); nvkm_gsp_dtor_fws(gsp); + nvkm_gsp_fwsec_sb_dtor(gsp); nvkm_gsp_mem_dtor(&gsp->rmargs); nvkm_gsp_mem_dtor(&gsp->wpr_meta); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/rpc.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/rpc.c index 0dc4782df8c0c..3ca3de8f43408 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/rpc.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/rpc.c @@ -557,6 +557,7 @@ r535_gsp_rpc_handle_reply(struct nvkm_gsp *gsp, u32 fn, switch (policy) { case NVKM_GSP_RPC_REPLY_NOWAIT: + case NVKM_GSP_RPC_REPLY_NOSEQ: break; case NVKM_GSP_RPC_REPLY_RECV: reply = r535_gsp_msg_recv(gsp, fn, gsp_rpc_len); @@ -588,6 +589,11 @@ r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *payload, rpc->data, rpc->length - sizeof(*rpc), true); } + if (policy == NVKM_GSP_RPC_REPLY_NOSEQ) + rpc->sequence = 0; + else + rpc->sequence = gsp->rpc_seq++; + ret = r535_gsp_cmdq_push(gsp, rpc); if (ret) return ERR_PTR(ret); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c index 2945d5b4e5707..8ef8b4f655883 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c @@ -62,7 +62,7 @@ r570_fbsr_resume(struct nvkm_gsp *gsp) } static int -r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size) +r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size, bool runtime) { NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS *ctrl; struct nvkm_gsp_object memlist; @@ -81,7 +81,7 @@ r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size) ctrl->hClient = gsp->internal.client.object.handle; ctrl->hSysMem = memlist.handle; ctrl->sysmemAddrOfSuspendResumeData = gsp->sr.meta.addr; - ctrl->bEnteringGcoffState = 1; + ctrl->bEnteringGcoffState = runtime ? 1 : 0; ret = nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl); if (ret) @@ -92,7 +92,7 @@ r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size) } static int -r570_fbsr_suspend(struct nvkm_gsp *gsp) +r570_fbsr_suspend(struct nvkm_gsp *gsp, bool runtime) { struct nvkm_subdev *subdev = &gsp->subdev; struct nvkm_device *device = subdev->device; @@ -133,7 +133,7 @@ r570_fbsr_suspend(struct nvkm_gsp *gsp) return ret; /* Initialise FBSR on RM. */ - ret = r570_fbsr_init(gsp, &gsp->sr.fbsr, size); + ret = r570_fbsr_init(gsp, &gsp->sr.fbsr, size, runtime); if (ret) { nvkm_gsp_sg_free(device, &gsp->sr.fbsr); return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c index 9d2fa4e66d598..996941c668ba9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c @@ -176,7 +176,7 @@ r570_gsp_set_system_info(struct nvkm_gsp *gsp) info->bIsPrimary = video_is_primary_device(device->dev); info->bPreserveVideoMemoryAllocations = false; - return nvkm_gsp_rpc_wr(gsp, info, NVKM_GSP_RPC_REPLY_NOWAIT); + return nvkm_gsp_rpc_wr(gsp, info, NVKM_GSP_RPC_REPLY_NOSEQ); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h index 393ea775941fa..4f0ae6cc085c3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h @@ -78,7 +78,7 @@ struct nvkm_rm_api { } *device; const struct nvkm_rm_api_fbsr { - int (*suspend)(struct nvkm_gsp *); + int (*suspend)(struct nvkm_gsp *, bool runtime); void (*resume)(struct nvkm_gsp *); } *fbsr; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c index 81e56da0474a1..19cb269e7a265 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c @@ -30,6 +30,18 @@ #include #include +int +tu102_gsp_fwsec_sb_ctor(struct nvkm_gsp *gsp) +{ + return nvkm_gsp_fwsec_sb_init(gsp); +} + +void +tu102_gsp_fwsec_sb_dtor(struct nvkm_gsp *gsp) +{ + nvkm_falcon_fw_dtor(&gsp->fws.falcon.sb); +} + static int tu102_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1) { @@ -149,7 +161,7 @@ tu102_gsp_reset(struct nvkm_gsp *gsp) } int -tu102_gsp_fini(struct nvkm_gsp *gsp, bool suspend) +tu102_gsp_fini(struct nvkm_gsp *gsp, enum nvkm_suspend_state suspend) { u32 mbox0 = 0xff, mbox1 = 0xff; int ret; @@ -370,6 +382,9 @@ tu102_gsp = { .booter.ctor = tu102_gsp_booter_ctor, + .fwsec_sb.ctor = tu102_gsp_fwsec_sb_ctor, + .fwsec_sb.dtor = tu102_gsp_fwsec_sb_dtor, + .dtor = r535_gsp_dtor, .oneinit = tu102_gsp_oneinit, .init = tu102_gsp_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu116.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu116.c index 97eb046c25d07..58cf258424218 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu116.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu116.c @@ -30,6 +30,9 @@ tu116_gsp = { .booter.ctor = tu102_gsp_booter_ctor, + .fwsec_sb.ctor = tu102_gsp_fwsec_sb_ctor, + .fwsec_sb.dtor = tu102_gsp_fwsec_sb_dtor, + .dtor = r535_gsp_dtor, .oneinit = tu102_gsp_oneinit, .init = tu102_gsp_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c index 7ec17e8435a13..454bb21815a27 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c @@ -135,7 +135,7 @@ nvkm_i2c_intr(struct nvkm_subdev *subdev) } static int -nvkm_i2c_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_i2c_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_i2c *i2c = nvkm_i2c(subdev); struct nvkm_i2c_pad *pad; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 2f55bab8e132f..6b9ed61684a02 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -176,7 +176,7 @@ nvkm_instmem_boot(struct nvkm_instmem *imem) } static int -nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_instmem_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_instmem *imem = nvkm_instmem(subdev); int ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c index 6867934256a76..0f3e0d324a529 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c @@ -74,7 +74,7 @@ nvkm_pci_rom_shadow(struct nvkm_pci *pci, bool shadow) } static int -nvkm_pci_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_pci_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_pci *pci = nvkm_pci(subdev); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c index 8f2f50ad4ded1..9e9004ec4588b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c @@ -77,7 +77,7 @@ nvkm_pmu_intr(struct nvkm_subdev *subdev) } static int -nvkm_pmu_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_pmu_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_pmu *pmu = nvkm_pmu(subdev); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c index fc5ee118e9106..1510aba339560 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c @@ -341,15 +341,15 @@ nvkm_therm_intr(struct nvkm_subdev *subdev) } static int -nvkm_therm_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_therm_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_therm *therm = nvkm_therm(subdev); if (therm->func->fini) therm->func->fini(therm); - nvkm_therm_fan_fini(therm, suspend); - nvkm_therm_sensor_fini(therm, suspend); + nvkm_therm_fan_fini(therm, suspend != NVKM_POWEROFF); + nvkm_therm_sensor_fini(therm, suspend != NVKM_POWEROFF); if (suspend) { therm->suspend = therm->mode; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c index 8b0da0c062685..a5c3c282b5d07 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c @@ -149,7 +149,7 @@ nvkm_timer_intr(struct nvkm_subdev *subdev) } static int -nvkm_timer_fini(struct nvkm_subdev *subdev, bool suspend) +nvkm_timer_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { struct nvkm_timer *tmr = nvkm_timer(subdev); tmr->func->alarm_fini(tmr); diff --git a/drivers/gpu/drm/nova/Kconfig b/drivers/gpu/drm/nova/Kconfig index cca6a3fea879b..3e637ad7b5bad 100644 --- a/drivers/gpu/drm/nova/Kconfig +++ b/drivers/gpu/drm/nova/Kconfig @@ -1,9 +1,11 @@ config DRM_NOVA tristate "Nova DRM driver" + depends on 64BIT depends on DRM=y depends on PCI depends on RUST select AUXILIARY_BUS + select NOVA_CORE default n help Choose this if you want to build the Nova DRM driver for Nvidia diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index 62435e3cd9f4d..d1e68c464e308 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -1730,6 +1730,12 @@ static const struct panel_delay delay_200_500_p2e100 = { .prepare_to_enable = 100, }; +static const struct panel_delay delay_200_500_p2e200 = { + .hpd_absent = 200, + .unprepare = 500, + .prepare_to_enable = 200, +}; + static const struct panel_delay delay_200_500_e50 = { .hpd_absent = 200, .unprepare = 500, @@ -1880,6 +1886,7 @@ static const struct panel_delay delay_80_500_e50_d50 = { */ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x04a4, &delay_200_500_e50, "B122UAN01.0"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x0ba4, &delay_200_500_e50, "B140QAX01.H"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x105c, &delay_200_500_e50, "B116XTN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x1062, &delay_200_500_e50, "B120XAN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x125c, &delay_200_500_e50, "Unknown"), @@ -1903,6 +1910,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x643d, &delay_200_500_e50, "B140HAN06.4"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x73aa, &delay_200_500_e50, "B116XTN02.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"), @@ -1972,6 +1980,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b66, &delay_200_500_e80, "NE140WUM-N6G"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"), + EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c26, &delay_200_500_p2e200, "NV140WUM-T08"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c93, &delay_200_500_e200, "Unknown"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cb6, &delay_200_500_e200, "NT116WHM-N44"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf6, &delay_200_500_e200, "NV140WUM-N64"), diff --git a/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c b/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c index 23462065d726b..ea975170fafff 100644 --- a/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c +++ b/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c @@ -434,8 +434,10 @@ static void jdi_panel_dsi_remove(struct mipi_dsi_device *dsi) int err; /* only detach from host for the DSI-LINK2 interface */ - if (!jdi) + if (!jdi) { mipi_dsi_detach(dsi); + return; + } err = jdi_panel_disable(&jdi->base); if (err < 0) diff --git a/drivers/gpu/drm/panel/panel-lg-sw43408.c b/drivers/gpu/drm/panel/panel-lg-sw43408.c index 46a56ea92ad9f..6e307fba658f7 100644 --- a/drivers/gpu/drm/panel/panel-lg-sw43408.c +++ b/drivers/gpu/drm/panel/panel-lg-sw43408.c @@ -294,10 +294,6 @@ static void sw43408_remove(struct mipi_dsi_device *dsi) struct sw43408_panel *ctx = mipi_dsi_get_drvdata(dsi); int ret; - ret = sw43408_unprepare(&ctx->base); - if (ret < 0) - dev_err(&dsi->dev, "failed to unprepare panel: %d\n", ret); - ret = mipi_dsi_detach(dsi); if (ret < 0) dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret); diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35560.c b/drivers/gpu/drm/panel/panel-novatek-nt35560.c index 561e6643dcbb6..6e5173f98a226 100644 --- a/drivers/gpu/drm/panel/panel-novatek-nt35560.c +++ b/drivers/gpu/drm/panel/panel-novatek-nt35560.c @@ -213,7 +213,7 @@ static const struct backlight_properties nt35560_bl_props = { static void nt35560_read_id(struct mipi_dsi_multi_context *dsi_ctx) { - struct device dev = dsi_ctx->dsi->dev; + struct device *dev = &dsi_ctx->dsi->dev; u8 vendor, version, panel; u16 val; @@ -225,7 +225,7 @@ static void nt35560_read_id(struct mipi_dsi_multi_context *dsi_ctx) return; if (vendor == 0x00) { - dev_err(&dev, "device vendor ID is zero\n"); + dev_err(dev, "device vendor ID is zero\n"); dsi_ctx->accum_err = -ENODEV; return; } @@ -236,12 +236,12 @@ static void nt35560_read_id(struct mipi_dsi_multi_context *dsi_ctx) case DISPLAY_SONY_ACX424AKP_ID2: case DISPLAY_SONY_ACX424AKP_ID3: case DISPLAY_SONY_ACX424AKP_ID4: - dev_info(&dev, + dev_info(dev, "MTP vendor: %02x, version: %02x, panel: %02x\n", vendor, version, panel); break; default: - dev_info(&dev, + dev_info(dev, "unknown vendor: %02x, version: %02x, panel: %02x\n", vendor, version, panel); break; diff --git a/drivers/gpu/drm/panel/panel-novatek-nt37801.c b/drivers/gpu/drm/panel/panel-novatek-nt37801.c index d6a37d7e0cc63..7eda16e0c1f94 100644 --- a/drivers/gpu/drm/panel/panel-novatek-nt37801.c +++ b/drivers/gpu/drm/panel/panel-novatek-nt37801.c @@ -267,6 +267,11 @@ static int novatek_nt37801_probe(struct mipi_dsi_device *dsi) if (ret < 0) return ret; + ret = regulator_set_voltage(ctx->supplies[0].consumer, + 1650000, 1950000); + if (ret < 0) + return ret; + ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(ctx->reset_gpio)) return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio), diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 0019de93be1b6..271f933991937 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -623,49 +623,6 @@ static struct panel_simple *panel_simple_probe(struct device *dev) if (IS_ERR(desc)) return ERR_CAST(desc); - panel = devm_drm_panel_alloc(dev, struct panel_simple, base, - &panel_simple_funcs, desc->connector_type); - if (IS_ERR(panel)) - return ERR_CAST(panel); - - panel->desc = desc; - - panel->supply = devm_regulator_get(dev, "power"); - if (IS_ERR(panel->supply)) - return ERR_CAST(panel->supply); - - panel->enable_gpio = devm_gpiod_get_optional(dev, "enable", - GPIOD_OUT_LOW); - if (IS_ERR(panel->enable_gpio)) - return dev_err_cast_probe(dev, panel->enable_gpio, - "failed to request GPIO\n"); - - err = of_drm_get_panel_orientation(dev->of_node, &panel->orientation); - if (err) { - dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, err); - return ERR_PTR(err); - } - - ddc = of_parse_phandle(dev->of_node, "ddc-i2c-bus", 0); - if (ddc) { - panel->ddc = of_find_i2c_adapter_by_node(ddc); - of_node_put(ddc); - - if (!panel->ddc) - return ERR_PTR(-EPROBE_DEFER); - } - - if (!of_device_is_compatible(dev->of_node, "panel-dpi") && - !of_get_display_timing(dev->of_node, "panel-timing", &dt)) - panel_simple_parse_panel_timing_node(dev, panel, &dt); - - if (desc->connector_type == DRM_MODE_CONNECTOR_LVDS) { - /* Optional data-mapping property for overriding bus format */ - err = panel_simple_override_nondefault_lvds_datamapping(dev, panel); - if (err) - goto free_ddc; - } - connector_type = desc->connector_type; /* Catch common mistakes for panels. */ switch (connector_type) { @@ -690,8 +647,7 @@ static struct panel_simple *panel_simple_probe(struct device *dev) break; case DRM_MODE_CONNECTOR_eDP: dev_warn(dev, "eDP panels moved to panel-edp\n"); - err = -EINVAL; - goto free_ddc; + return ERR_PTR(-EINVAL); case DRM_MODE_CONNECTOR_DSI: if (desc->bpc != 6 && desc->bpc != 8) dev_warn(dev, "Expected bpc in {6,8} but got: %u\n", desc->bpc); @@ -720,6 +676,49 @@ static struct panel_simple *panel_simple_probe(struct device *dev) break; } + panel = devm_drm_panel_alloc(dev, struct panel_simple, base, + &panel_simple_funcs, connector_type); + if (IS_ERR(panel)) + return ERR_CAST(panel); + + panel->desc = desc; + + panel->supply = devm_regulator_get(dev, "power"); + if (IS_ERR(panel->supply)) + return ERR_CAST(panel->supply); + + panel->enable_gpio = devm_gpiod_get_optional(dev, "enable", + GPIOD_OUT_LOW); + if (IS_ERR(panel->enable_gpio)) + return dev_err_cast_probe(dev, panel->enable_gpio, + "failed to request GPIO\n"); + + err = of_drm_get_panel_orientation(dev->of_node, &panel->orientation); + if (err) { + dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, err); + return ERR_PTR(err); + } + + ddc = of_parse_phandle(dev->of_node, "ddc-i2c-bus", 0); + if (ddc) { + panel->ddc = of_find_i2c_adapter_by_node(ddc); + of_node_put(ddc); + + if (!panel->ddc) + return ERR_PTR(-EPROBE_DEFER); + } + + if (!of_device_is_compatible(dev->of_node, "panel-dpi") && + !of_get_display_timing(dev->of_node, "panel-timing", &dt)) + panel_simple_parse_panel_timing_node(dev, panel, &dt); + + if (desc->connector_type == DRM_MODE_CONNECTOR_LVDS) { + /* Optional data-mapping property for overriding bus format */ + err = panel_simple_override_nondefault_lvds_datamapping(dev, panel); + if (err) + goto free_ddc; + } + dev_set_drvdata(dev, panel); /* @@ -1900,6 +1899,7 @@ static const struct panel_desc dataimage_scf0700c48ggu18 = { }, .bus_format = MEDIA_BUS_FMT_RGB888_1X24, .bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE, + .connector_type = DRM_MODE_CONNECTOR_DPI, }; static const struct display_timing dlc_dlc0700yzg_1_timing = { diff --git a/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c index 7c989b70ab513..a14c86c60d19d 100644 --- a/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c +++ b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c @@ -212,6 +212,8 @@ static int sony_td4353_jdi_probe(struct mipi_dsi_device *dsi) if (ret) return dev_err_probe(dev, ret, "Failed to get backlight\n"); + ctx->panel.prepare_prev_first = true; + drm_panel_add(&ctx->panel); ret = mipi_dsi_attach(dsi); diff --git a/drivers/gpu/drm/panel/panel-visionox-rm69299.c b/drivers/gpu/drm/panel/panel-visionox-rm69299.c index 909c280eab1fb..66c30db3b73a7 100644 --- a/drivers/gpu/drm/panel/panel-visionox-rm69299.c +++ b/drivers/gpu/drm/panel/panel-visionox-rm69299.c @@ -192,7 +192,7 @@ static int visionox_rm69299_unprepare(struct drm_panel *panel) struct visionox_rm69299 *ctx = panel_to_ctx(panel); struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi }; - ctx->dsi->mode_flags = 0; + ctx->dsi->mode_flags &= ~MIPI_DSI_MODE_LPM; mipi_dsi_dcs_set_display_off_multi(&dsi_ctx); @@ -247,7 +247,7 @@ static const struct drm_display_mode visionox_rm69299_1080x2248_60hz = { }; static const struct drm_display_mode visionox_rm69299_1080x2160_60hz = { - .clock = 158695, + .clock = (2160 + 8 + 4 + 4) * (1080 + 26 + 2 + 36) * 60 / 1000, .hdisplay = 1080, .hsync_start = 1080 + 26, .hsync_end = 1080 + 26 + 2, diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c index 81df49880bd87..962a10e00848e 100644 --- a/drivers/gpu/drm/panthor/panthor_device.c +++ b/drivers/gpu/drm/panthor/panthor_device.c @@ -83,6 +83,8 @@ void panthor_device_unplug(struct panthor_device *ptdev) return; } + drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0); + /* Call drm_dev_unplug() so any access to HW blocks happening after * that point get rejected. */ @@ -93,8 +95,6 @@ void panthor_device_unplug(struct panthor_device *ptdev) */ mutex_unlock(&ptdev->unplug.lock); - drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0); - /* Now, try to cleanly shutdown the GPU before the device resources * get reclaimed. */ diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c index 3f43686f01958..43471babff86c 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@ -86,7 +86,6 @@ static void panthor_gem_free_object(struct drm_gem_object *obj) void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo) { struct panthor_vm *vm; - int ret; if (IS_ERR_OR_NULL(bo)) return; @@ -94,18 +93,11 @@ void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo) vm = bo->vm; panthor_kernel_bo_vunmap(bo); - if (drm_WARN_ON(bo->obj->dev, - to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm))) - goto out_free_bo; - - ret = panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size); - if (ret) - goto out_free_bo; - + drm_WARN_ON(bo->obj->dev, + to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm)); + panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size); panthor_vm_free_va(vm, &bo->va_node); drm_gem_object_put(bo->obj); - -out_free_bo: panthor_vm_put(vm); kfree(bo); } @@ -152,6 +144,9 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, bo = to_panthor_bo(&obj->base); kbo->obj = &obj->base; bo->flags = bo_flags; + bo->exclusive_vm_root_gem = panthor_vm_root_gem(vm); + drm_gem_object_get(bo->exclusive_vm_root_gem); + bo->base.base.resv = bo->exclusive_vm_root_gem->resv; if (vm == panthor_fw_vm(ptdev)) debug_flags |= PANTHOR_DEBUGFS_GEM_USAGE_FLAG_FW_MAPPED; @@ -175,9 +170,6 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, goto err_free_va; kbo->vm = panthor_vm_get(vm); - bo->exclusive_vm_root_gem = panthor_vm_root_gem(vm); - drm_gem_object_get(bo->exclusive_vm_root_gem); - bo->base.base.resv = bo->exclusive_vm_root_gem->resv; return kbo; err_free_va: diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c index db69449a5be09..77d0f4ced1206 100644 --- a/drivers/gpu/drm/panthor/panthor_gpu.c +++ b/drivers/gpu/drm/panthor/panthor_gpu.c @@ -49,7 +49,7 @@ struct panthor_gpu { static void panthor_gpu_coherency_set(struct panthor_device *ptdev) { gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, - ptdev->coherent ? GPU_COHERENCY_PROT_BIT(ACE_LITE) : GPU_COHERENCY_NONE); + ptdev->coherent ? GPU_COHERENCY_ACE_LITE : GPU_COHERENCY_NONE); } static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) @@ -259,38 +259,42 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev) int panthor_gpu_flush_caches(struct panthor_device *ptdev, u32 l2, u32 lsc, u32 other) { - bool timedout = false; unsigned long flags; + int ret = 0; /* Serialize cache flush operations. */ guard(mutex)(&ptdev->gpu->cache_flush_lock); spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); - if (!drm_WARN_ON(&ptdev->base, - ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { + if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); + } else { + ret = -EIO; } spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); + if (ret) + return ret; + if (!wait_event_timeout(ptdev->gpu->reqs_acked, !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), msecs_to_jiffies(100))) { spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) - timedout = true; + ret = -ETIMEDOUT; else ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); } - if (timedout) { + if (ret) { + panthor_device_schedule_reset(ptdev); drm_err(&ptdev->base, "Flush caches timeout"); - return -ETIMEDOUT; } - return 0; + return ret; } /** @@ -330,6 +334,7 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev) return -ETIMEDOUT; } + ptdev->gpu->pending_reqs = 0; return 0; } diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index 7870e7dbaa5d4..5647a24741095 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -523,27 +523,29 @@ static int wait_ready(struct panthor_device *ptdev, u32 as_nr) return ret; } -static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd) +static int as_send_cmd_and_wait(struct panthor_device *ptdev, u32 as_nr, u32 cmd) { int status; /* write AS_COMMAND when MMU is ready to accept another command */ status = wait_ready(ptdev, as_nr); - if (!status) + if (!status) { gpu_write(ptdev, AS_COMMAND(as_nr), cmd); + status = wait_ready(ptdev, as_nr); + } return status; } -static void lock_region(struct panthor_device *ptdev, u32 as_nr, - u64 region_start, u64 size) +static int lock_region(struct panthor_device *ptdev, u32 as_nr, + u64 region_start, u64 size) { u8 region_width; u64 region; u64 region_end = region_start + size; if (!size) - return; + return 0; /* * The locked region is a naturally aligned power of 2 block encoded as @@ -566,7 +568,7 @@ static void lock_region(struct panthor_device *ptdev, u32 as_nr, /* Lock the region that needs to be updated */ gpu_write64(ptdev, AS_LOCKADDR(as_nr), region); - write_cmd(ptdev, as_nr, AS_COMMAND_LOCK); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_LOCK); } static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, @@ -599,9 +601,7 @@ static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, * power it up */ - lock_region(ptdev, as_nr, iova, size); - - ret = wait_ready(ptdev, as_nr); + ret = lock_region(ptdev, as_nr, iova, size); if (ret) return ret; @@ -614,10 +614,7 @@ static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, * at the end of the GPU_CONTROL cache flush command, unlike * AS_COMMAND_FLUSH_MEM or AS_COMMAND_FLUSH_PT. */ - write_cmd(ptdev, as_nr, AS_COMMAND_UNLOCK); - - /* Wait for the unlock command to complete */ - return wait_ready(ptdev, as_nr); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UNLOCK); } static int mmu_hw_do_operation(struct panthor_vm *vm, @@ -646,7 +643,7 @@ static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr, gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr); gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg); - return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr) @@ -661,7 +658,7 @@ static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr) gpu_write64(ptdev, AS_MEMATTR(as_nr), 0); gpu_write64(ptdev, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED); - return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value) @@ -1146,6 +1143,20 @@ static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx, } } +static void +panthor_vm_op_ctx_return_vma(struct panthor_vm_op_ctx *op_ctx, + struct panthor_vma *vma) +{ + for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) { + if (!op_ctx->preallocated_vmas[i]) { + op_ctx->preallocated_vmas[i] = vma; + return; + } + } + + WARN_ON_ONCE(1); +} + static struct panthor_vma * panthor_vm_op_ctx_get_vma(struct panthor_vm_op_ctx *op_ctx) { @@ -1547,6 +1558,10 @@ static void panthor_vm_destroy(struct panthor_vm *vm) vm->destroyed = true; + /* Tell scheduler to stop all GPU work related to this VM */ + if (refcount_read(&vm->as.active_cnt) > 0) + panthor_sched_prepare_for_vm_destruction(vm->ptdev); + mutex_lock(&vm->heaps.lock); panthor_heap_pool_destroy(vm->heaps.pool); vm->heaps.pool = NULL; @@ -2085,8 +2100,10 @@ static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv) ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags), op_ctx->map.sgt, op->map.gem.offset, op->map.va.range); - if (ret) + if (ret) { + panthor_vm_op_ctx_return_vma(op_ctx, vma); return ret; + } /* Ref owned by the mapping now, clear the obj field so we don't release the * pinning/obj ref behind GPUVA's back. diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c index 3d1f57e3990f4..6930053009d5c 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.c +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "panthor_devfreq.h" #include "panthor_device.h" @@ -772,6 +773,12 @@ struct panthor_job_profiling_data { */ #define MAX_GROUPS_PER_POOL 128 +/* + * Mark added on an entry of group pool Xarray to identify if the group has + * been fully initialized and can be accessed elsewhere in the driver code. + */ +#define GROUP_REGISTERED XA_MARK_1 + /** * struct panthor_group_pool - Group pool * @@ -895,7 +902,8 @@ static void group_free_queue(struct panthor_group *group, struct panthor_queue * if (IS_ERR_OR_NULL(queue)) return; - drm_sched_entity_destroy(&queue->entity); + if (queue->entity.fence_context) + drm_sched_entity_destroy(&queue->entity); if (queue->scheduler.ops) drm_sched_fini(&queue->scheduler); @@ -919,6 +927,9 @@ static void group_release_work(struct work_struct *work) release_work); u32 i; + /* dma-fences may still be accessing group->queues under rcu lock. */ + synchronize_rcu(); + for (i = 0; i < group->queue_count; i++) group_free_queue(group, group->queues[i]); @@ -1895,10 +1906,10 @@ struct panthor_sched_tick_ctx { struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT]; u32 idle_group_count; u32 group_count; - enum panthor_csg_priority min_priority; struct panthor_vm *vms[MAX_CS_PER_CSG]; u32 as_count; bool immediate_tick; + bool stop_tick; u32 csg_upd_failed_mask; }; @@ -1963,17 +1974,21 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, if (!owned_by_tick_ctx) group_get(group); - list_move_tail(&group->run_node, &ctx->groups[group->priority]); ctx->group_count++; + + /* If we have more than one active group with the same priority, + * we need to keep ticking to rotate the CSG priority. + */ if (group_is_idle(group)) ctx->idle_group_count++; + else if (!list_empty(&ctx->groups[group->priority])) + ctx->stop_tick = false; + + list_move_tail(&group->run_node, &ctx->groups[group->priority]); if (i == ctx->as_count) ctx->vms[ctx->as_count++] = group->vm; - if (ctx->min_priority > group->priority) - ctx->min_priority = group->priority; - if (tick_ctx_is_full(sched, ctx)) return; } @@ -1982,31 +1997,22 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, static void tick_ctx_insert_old_group(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx, - struct panthor_group *group, - bool full_tick) + struct panthor_group *group) { struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; struct panthor_group *other_group; - if (!full_tick) { - list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); - return; - } - - /* Rotate to make sure groups with lower CSG slot - * priorities have a chance to get a higher CSG slot - * priority next time they get picked. This priority - * has an impact on resource request ordering, so it's - * important to make sure we don't let one group starve - * all other groups with the same group priority. - */ + /* Class groups in descending priority order so we can easily rotate. */ list_for_each_entry(other_group, &ctx->old_groups[csg_slot->group->priority], run_node) { struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id]; - if (other_csg_slot->priority > csg_slot->priority) { - list_add_tail(&csg_slot->group->run_node, &other_group->run_node); + /* Our group has a higher prio than the one we're testing against, + * place it just before. + */ + if (csg_slot->priority > other_csg_slot->priority) { + list_add_tail(&group->run_node, &other_group->run_node); return; } } @@ -2016,8 +2022,7 @@ tick_ctx_insert_old_group(struct panthor_scheduler *sched, static void tick_ctx_init(struct panthor_scheduler *sched, - struct panthor_sched_tick_ctx *ctx, - bool full_tick) + struct panthor_sched_tick_ctx *ctx) { struct panthor_device *ptdev = sched->ptdev; struct panthor_csg_slots_upd_ctx upd_ctx; @@ -2027,7 +2032,7 @@ tick_ctx_init(struct panthor_scheduler *sched, memset(ctx, 0, sizeof(*ctx)); csgs_upd_ctx_init(&upd_ctx); - ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT; + ctx->stop_tick = true; for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { INIT_LIST_HEAD(&ctx->groups[i]); INIT_LIST_HEAD(&ctx->old_groups[i]); @@ -2055,7 +2060,7 @@ tick_ctx_init(struct panthor_scheduler *sched, group->fatal_queues |= GENMASK(group->queue_count - 1, 0); } - tick_ctx_insert_old_group(sched, ctx, group, full_tick); + tick_ctx_insert_old_group(sched, ctx, group); csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, csg_iface->output->ack ^ CSG_STATUS_UPDATE, CSG_STATUS_UPDATE); @@ -2339,32 +2344,18 @@ static u64 tick_ctx_update_resched_target(struct panthor_scheduler *sched, const struct panthor_sched_tick_ctx *ctx) { - /* We had space left, no need to reschedule until some external event happens. */ - if (!tick_ctx_is_full(sched, ctx)) - goto no_tick; - - /* If idle groups were scheduled, no need to wake up until some external - * event happens (group unblocked, new job submitted, ...). - */ - if (ctx->idle_group_count) - goto no_tick; + u64 resched_target; - if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT)) + if (ctx->stop_tick) goto no_tick; - /* If there are groups of the same priority waiting, we need to - * keep the scheduler ticking, otherwise, we'll just wait for - * new groups with higher priority to be queued. - */ - if (!list_empty(&sched->groups.runnable[ctx->min_priority])) { - u64 resched_target = sched->last_tick + sched->tick_period; + resched_target = sched->last_tick + sched->tick_period; - if (time_before64(sched->resched_target, sched->last_tick) || - time_before64(resched_target, sched->resched_target)) - sched->resched_target = resched_target; + if (time_before64(sched->resched_target, sched->last_tick) || + time_before64(resched_target, sched->resched_target)) + sched->resched_target = resched_target; - return sched->resched_target - sched->last_tick; - } + return sched->resched_target - sched->last_tick; no_tick: sched->resched_target = U64_MAX; @@ -2377,9 +2368,11 @@ static void tick_work(struct work_struct *work) tick_work.work); struct panthor_device *ptdev = sched->ptdev; struct panthor_sched_tick_ctx ctx; + u64 resched_target = sched->resched_target; u64 remaining_jiffies = 0, resched_delay; u64 now = get_jiffies_64(); int prio, ret, cookie; + bool full_tick; if (!drm_dev_enter(&ptdev->base, &cookie)) return; @@ -2388,18 +2381,24 @@ static void tick_work(struct work_struct *work) if (drm_WARN_ON(&ptdev->base, ret)) goto out_dev_exit; - if (time_before64(now, sched->resched_target)) - remaining_jiffies = sched->resched_target - now; + /* If the tick is stopped, calculate when the next tick would be */ + if (resched_target == U64_MAX) + resched_target = sched->last_tick + sched->tick_period; + + if (time_before64(now, resched_target)) + remaining_jiffies = resched_target - now; + + full_tick = remaining_jiffies == 0; mutex_lock(&sched->lock); if (panthor_device_reset_is_pending(sched->ptdev)) goto out_unlock; - tick_ctx_init(sched, &ctx, remaining_jiffies != 0); + tick_ctx_init(sched, &ctx); if (ctx.csg_upd_failed_mask) goto out_cleanup_ctx; - if (remaining_jiffies) { + if (!full_tick) { /* Scheduling forced in the middle of a tick. Only RT groups * can preempt non-RT ones. Currently running RT groups can't be * preempted. @@ -2421,9 +2420,29 @@ static void tick_work(struct work_struct *work) for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0 && !tick_ctx_is_full(sched, &ctx); prio--) { + struct panthor_group *old_highest_prio_group = + list_first_entry_or_null(&ctx.old_groups[prio], + struct panthor_group, run_node); + + /* Pull out the group with the highest prio for rotation. */ + if (old_highest_prio_group) + list_del(&old_highest_prio_group->run_node); + + /* Re-insert old active groups so they get a chance to run with higher prio. */ + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Fill the remaining slots with runnable groups. */ tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio], true, false); - tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Re-insert the old group with the highest prio, and give it a chance to be + * scheduled again (but with a lower prio) if there's room left. + */ + if (old_highest_prio_group) { + list_add_tail(&old_highest_prio_group->run_node, &ctx.old_groups[prio]); + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], + true, true); + } } /* If we have free CSG slots left, pick idle groups */ @@ -2548,14 +2567,33 @@ static void sync_upd_work(struct work_struct *work) sched_queue_delayed_work(sched, tick, 0); } +static void sched_resume_tick(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + u64 delay_jiffies, now; + + drm_WARN_ON(&ptdev->base, sched->resched_target != U64_MAX); + + /* Scheduler tick was off, recalculate the resched_target based on the + * last tick event, and queue the scheduler work. + */ + now = get_jiffies_64(); + sched->resched_target = sched->last_tick + sched->tick_period; + if (sched->used_csg_slot_count == sched->csg_slot_count && + time_before64(now, sched->resched_target)) + delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); + else + delay_jiffies = 0; + + sched_queue_delayed_work(sched, tick, delay_jiffies); +} + static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) { struct panthor_device *ptdev = group->ptdev; struct panthor_scheduler *sched = ptdev->scheduler; struct list_head *queue = &sched->groups.runnable[group->priority]; - u64 delay_jiffies = 0; bool was_idle; - u64 now; if (!group_can_run(group)) return; @@ -2600,13 +2638,7 @@ static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) /* Scheduler tick was off, recalculate the resched_target based on the * last tick event, and queue the scheduler work. */ - now = get_jiffies_64(); - sched->resched_target = sched->last_tick + sched->tick_period; - if (sched->used_csg_slot_count == sched->csg_slot_count && - time_before64(now, sched->resched_target)) - delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); - - sched_queue_delayed_work(sched, tick, delay_jiffies); + sched_resume_tick(ptdev); } static void queue_stop(struct panthor_queue *queue, @@ -2679,6 +2711,20 @@ void panthor_sched_report_mmu_fault(struct panthor_device *ptdev) panthor_sched_immediate_tick(ptdev); } +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev) +{ + /* FW can write out internal state, like the heap context, during CSG + * suspend. It is therefore important that the scheduler has fully + * evicted any pending and related groups before VM destruction can + * safely continue. Failure to do so can lead to GPU page faults. + * A controlled termination of a Panthor instance involves destroying + * the group(s) before the VM. This means any relevant group eviction + * has already been initiated by this point, and we just need to + * ensure that any pending tick_work() has been completed. + */ + flush_work(&ptdev->scheduler->tick_work.work); +} + void panthor_sched_resume(struct panthor_device *ptdev) { /* Force a tick to re-evaluate after a resume. */ @@ -2899,7 +2945,7 @@ void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile) return; xa_lock(&gpool->xa); - xa_for_each(&gpool->xa, i, group) { + xa_for_each_marked(&gpool->xa, i, group, GROUP_REGISTERED) { guard(spinlock)(&group->fdinfo.lock); pfile->stats.cycles += group->fdinfo.data.cycles; pfile->stats.time += group->fdinfo.data.time; @@ -3207,6 +3253,18 @@ queue_run_job(struct drm_sched_job *sched_job) group_schedule_locked(group, BIT(job->queue_idx)); } else { + u32 queue_mask = BIT(job->queue_idx); + bool resume_tick = group_is_idle(group) && + (group->idle_queues & queue_mask) && + !(group->blocked_queues & queue_mask) && + sched->resched_target == U64_MAX; + + /* We just added something to the queue, so it's no longer idle. */ + group->idle_queues &= ~queue_mask; + + if (resume_tick) + sched_resume_tick(ptdev); + gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); if (!sched->pm.has_ref && !(group->blocked_queues & BIT(job->queue_idx))) { @@ -3403,6 +3461,8 @@ group_create_queue(struct panthor_group *group, drm_sched = &queue->scheduler; ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL); + if (ret) + goto err_free_queue; return queue; @@ -3572,6 +3632,8 @@ int panthor_group_create(struct panthor_file *pfile, group_init_task_info(group); + xa_set_mark(&gpool->xa, gid, GROUP_REGISTERED); + return gid; err_put_group: @@ -3586,6 +3648,9 @@ int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle) struct panthor_scheduler *sched = ptdev->scheduler; struct panthor_group *group; + if (!xa_get_mark(&gpool->xa, group_handle, GROUP_REGISTERED)) + return -EINVAL; + group = xa_erase(&gpool->xa, group_handle); if (!group) return -EINVAL; @@ -3611,12 +3676,12 @@ int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle) } static struct panthor_group *group_from_handle(struct panthor_group_pool *pool, - u32 group_handle) + unsigned long group_handle) { struct panthor_group *group; xa_lock(&pool->xa); - group = group_get(xa_load(&pool->xa, group_handle)); + group = group_get(xa_find(&pool->xa, &group_handle, group_handle, GROUP_REGISTERED)); xa_unlock(&pool->xa); return group; @@ -3703,7 +3768,7 @@ panthor_fdinfo_gather_group_mem_info(struct panthor_file *pfile, return; xa_lock(&gpool->xa); - xa_for_each(&gpool->xa, i, group) { + xa_for_each_marked(&gpool->xa, i, group, GROUP_REGISTERED) { stats->resident += group->fdinfo.kbo_sizes; if (group->csg_id >= 0) stats->active += group->fdinfo.kbo_sizes; @@ -3856,6 +3921,7 @@ void panthor_sched_unplug(struct panthor_device *ptdev) struct panthor_scheduler *sched = ptdev->scheduler; cancel_delayed_work_sync(&sched->tick_work); + disable_work_sync(&sched->fw_events_work); mutex_lock(&sched->lock); if (sched->pm.has_ref) { diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h index 742b0b4ff3a3c..6a560ab0a5b31 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.h +++ b/drivers/gpu/drm/panthor/panthor_sched.h @@ -49,6 +49,7 @@ void panthor_sched_suspend(struct panthor_device *ptdev); void panthor_sched_resume(struct panthor_device *ptdev); void panthor_sched_report_mmu_fault(struct panthor_device *ptdev); +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev); void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events); void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile); diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c index 56ff6a3fb4834..d7dc83cf7b00a 100644 --- a/drivers/gpu/drm/pl111/pl111_drv.c +++ b/drivers/gpu/drm/pl111/pl111_drv.c @@ -295,7 +295,7 @@ static int pl111_amba_probe(struct amba_device *amba_dev, variant->name, priv); if (ret != 0) { dev_err(dev, "%s failed irq %d\n", __func__, ret); - return ret; + goto dev_put; } ret = pl111_modeset_init(drm); diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c index 506ae1f5e099f..cd1901d5c7c0a 100644 --- a/drivers/gpu/drm/qxl/qxl_ioctl.c +++ b/drivers/gpu/drm/qxl/qxl_ioctl.c @@ -182,7 +182,7 @@ static int qxl_process_single_command(struct qxl_device *qdev, /* TODO copy slow path code from i915 */ fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_MASK)); - unwritten = __copy_from_user_inatomic_nocache + unwritten = copy_from_user_inatomic_nontemporal (fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_MASK), u64_to_user_ptr(cmd->command), cmd->command_size); diff --git a/drivers/gpu/drm/radeon/pptable.h b/drivers/gpu/drm/radeon/pptable.h index 969a8fb0ee9e0..f4e71046dc91c 100644 --- a/drivers/gpu/drm/radeon/pptable.h +++ b/drivers/gpu/drm/radeon/pptable.h @@ -450,7 +450,7 @@ typedef struct _ClockInfoArray{ //sizeof(ATOM_PPLIB_CLOCK_INFO) UCHAR ucEntrySize; - UCHAR clockInfo[] __counted_by(ucNumEntries); + UCHAR clockInfo[] /*__counted_by(ucNumEntries)*/; }ClockInfoArray; typedef struct _NonClockInfoArray{ diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index 9deb91970d4df..0342d095d44ce 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -2915,9 +2915,11 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, if (rdev->family == CHIP_HAINAN) { if ((rdev->pdev->revision == 0x81) || (rdev->pdev->revision == 0xC3) || + (rdev->pdev->device == 0x6660) || (rdev->pdev->device == 0x6664) || (rdev->pdev->device == 0x6665) || - (rdev->pdev->device == 0x6667)) { + (rdev->pdev->device == 0x6667) || + (rdev->pdev->device == 0x666F)) { max_sclk = 75000; } if ((rdev->pdev->revision == 0xC3) || @@ -2925,6 +2927,11 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, max_sclk = 60000; max_mclk = 80000; } + if ((rdev->pdev->device == 0x666f) && + (rdev->pdev->revision == 0x00)) { + max_sclk = 80000; + max_mclk = 95000; + } } else if (rdev->family == CHIP_OLAND) { if ((rdev->pdev->revision == 0xC7) || (rdev->pdev->revision == 0x80) || diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h index 76521276e2af8..dd871e17dcf53 100644 --- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h +++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h @@ -50,8 +50,8 @@ #define TXCMPPD3R 0x16c #define RXSETR 0x200 -#define RXSETR_CRCEN (((n) & 0xf) << 24) -#define RXSETR_ECCEN (((n) & 0xf) << 16) +#define RXSETR_CRCEN(n) (((n) & 0xf) << 24) +#define RXSETR_ECCEN(n) (((n) & 0xf) << 16) #define RXPSETR 0x210 #define RXPSETR_LPPDACC (1 << 0) #define RXPSR 0x220 diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c index 3b52dfc0ea1e0..b164e3a62cc2f 100644 --- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c +++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c @@ -646,6 +646,13 @@ static void rzg2l_mipi_dsi_atomic_disable(struct drm_bridge *bridge, rzg2l_mipi_dsi_stop_video(dsi); rzg2l_mipi_dsi_stop_hs_clock(dsi); +} + +static void rzg2l_mipi_dsi_atomic_post_disable(struct drm_bridge *bridge, + struct drm_atomic_state *state) +{ + struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge); + rzg2l_mipi_dsi_stop(dsi); } @@ -681,6 +688,7 @@ static const struct drm_bridge_funcs rzg2l_mipi_dsi_bridge_ops = { .atomic_pre_enable = rzg2l_mipi_dsi_atomic_pre_enable, .atomic_enable = rzg2l_mipi_dsi_atomic_enable, .atomic_disable = rzg2l_mipi_dsi_atomic_disable, + .atomic_post_disable = rzg2l_mipi_dsi_atomic_post_disable, .mode_valid = rzg2l_mipi_dsi_bridge_mode_valid, }; diff --git a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c index ed6e8f036f4b3..409f1a1e82a06 100644 --- a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c @@ -267,12 +267,7 @@ static irqreturn_t dw_hdmi_qp_rk3576_hardirq(int irq, void *dev_id) static irqreturn_t dw_hdmi_qp_rk3576_irq(int irq, void *dev_id) { struct rockchip_hdmi_qp *hdmi = dev_id; - u32 intr_stat, val; - - regmap_read(hdmi->regmap, RK3576_IOC_HDMI_HPD_STATUS, &intr_stat); - - if (!intr_stat) - return IRQ_NONE; + u32 val; val = FIELD_PREP_WM16(RK3576_HDMI_HPD_INT_CLR, 1); regmap_write(hdmi->regmap, RK3576_IOC_MISC_CON0, val); @@ -597,6 +592,15 @@ static void dw_hdmi_qp_rockchip_remove(struct platform_device *pdev) component_del(&pdev->dev, &dw_hdmi_qp_rockchip_ops); } +static int __maybe_unused dw_hdmi_qp_rockchip_suspend(struct device *dev) +{ + struct rockchip_hdmi_qp *hdmi = dev_get_drvdata(dev); + + dw_hdmi_qp_suspend(dev, hdmi->hdmi); + + return 0; +} + static int __maybe_unused dw_hdmi_qp_rockchip_resume(struct device *dev) { struct rockchip_hdmi_qp *hdmi = dev_get_drvdata(dev); @@ -612,7 +616,8 @@ static int __maybe_unused dw_hdmi_qp_rockchip_resume(struct device *dev) } static const struct dev_pm_ops dw_hdmi_qp_rockchip_pm = { - SET_SYSTEM_SLEEP_PM_OPS(NULL, dw_hdmi_qp_rockchip_resume) + SET_SYSTEM_SLEEP_PM_OPS(dw_hdmi_qp_rockchip_suspend, + dw_hdmi_qp_rockchip_resume) }; struct platform_driver dw_hdmi_qp_rockchip_pltfm_driver = { diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c index eb77bde9f6283..e693160e9b7f5 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c @@ -96,6 +96,9 @@ void rockchip_drm_dma_init_device(struct drm_device *drm_dev, private->iommu_dev = ERR_PTR(-ENODEV); else if (!private->iommu_dev) private->iommu_dev = dev; + + if (!IS_ERR(private->iommu_dev)) + drm_dev_set_dma_dev(drm_dev, private->iommu_dev); } static int rockchip_drm_init_iommu(struct drm_device *drm_dev) diff --git a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c index 38c49030c7ab6..f3950e8476a75 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c +++ b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c @@ -1369,6 +1369,25 @@ static const struct vop2_regs_dump rk3588_regs_dump[] = { }, }; +/* + * phys_id is used to identify a main window(Cluster Win/Smart Win, not + * include the sub win of a cluster or the multi area) that can do overlay + * in main overlay stage. + */ +static struct vop2_win *vop2_find_win_by_phys_id(struct vop2 *vop2, uint8_t phys_id) +{ + struct vop2_win *win; + int i; + + for (i = 0; i < vop2->data->win_size; i++) { + win = &vop2->win[i]; + if (win->data->phys_id == phys_id) + return win; + } + + return NULL; +} + static unsigned long rk3568_set_intf_mux(struct vop2_video_port *vp, int id, u32 polflags) { struct vop2 *vop2 = vp->vop2; @@ -1842,15 +1861,31 @@ static void vop2_parse_alpha(struct vop2_alpha_config *alpha_config, alpha->dst_alpha_ctrl.bits.factor_mode = ALPHA_SRC_INVERSE; } -static int vop2_find_start_mixer_id_for_vp(struct vop2 *vop2, u8 port_id) +static int vop2_find_start_mixer_id_for_vp(struct vop2_video_port *vp) { - struct vop2_video_port *vp; - int used_layer = 0; + struct vop2 *vop2 = vp->vop2; + struct vop2_win *win; + u32 layer_sel = vop2->old_layer_sel; + u32 used_layer = 0; + unsigned long win_mask = vp->win_mask; + unsigned long phys_id; + bool match; int i; - for (i = 0; i < port_id; i++) { - vp = &vop2->vps[i]; - used_layer += hweight32(vp->win_mask); + for (i = 0; i < 31; i += 4) { + match = false; + for_each_set_bit(phys_id, &win_mask, ROCKCHIP_VOP2_ESMART3) { + win = vop2_find_win_by_phys_id(vop2, phys_id); + if (win->data->layer_sel_id[vp->id] == ((layer_sel >> i) & 0xf)) { + match = true; + break; + } + } + + if (!match) + used_layer += 1; + else + break; } return used_layer; @@ -1935,7 +1970,7 @@ static void vop2_setup_alpha(struct vop2_video_port *vp) u32 dst_global_alpha = DRM_BLEND_ALPHA_OPAQUE; if (vop2->version <= VOP_VERSION_RK3588) - mixer_id = vop2_find_start_mixer_id_for_vp(vop2, vp->id); + mixer_id = vop2_find_start_mixer_id_for_vp(vp); else mixer_id = 0; @@ -2069,7 +2104,7 @@ static void rk3568_vop2_wait_for_port_mux_done(struct vop2 *vop2) * Spin until the previous port_mux figuration is done. */ ret = readx_poll_timeout_atomic(rk3568_vop2_read_port_mux, vop2, port_mux_sel, - port_mux_sel == vop2->old_port_sel, 0, 50 * 1000); + port_mux_sel == vop2->old_port_sel, 10, 50 * 1000); if (ret) DRM_DEV_ERROR(vop2->dev, "wait port_mux done timeout: 0x%x--0x%x\n", port_mux_sel, vop2->old_port_sel); @@ -2089,7 +2124,7 @@ static void rk3568_vop2_wait_for_layer_cfg_done(struct vop2 *vop2, u32 cfg) * Spin until the previous layer configuration is done. */ ret = readx_poll_timeout_atomic(rk3568_vop2_read_layer_cfg, vop2, atv_layer_cfg, - atv_layer_cfg == cfg, 0, 50 * 1000); + atv_layer_cfg == cfg, 10, 50 * 1000); if (ret) DRM_DEV_ERROR(vop2->dev, "wait layer cfg done timeout: 0x%x--0x%x\n", atv_layer_cfg, cfg); @@ -2109,6 +2144,7 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) u8 layer_sel_id; unsigned int ofs; u32 ovl_ctrl; + u32 cfg_done; int i; struct vop2_video_port *vp0 = &vop2->vps[0]; struct vop2_video_port *vp1 = &vop2->vps[1]; @@ -2263,8 +2299,16 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) rk3568_vop2_wait_for_port_mux_done(vop2); } - if (layer_sel != old_layer_sel && atv_layer_sel != old_layer_sel) - rk3568_vop2_wait_for_layer_cfg_done(vop2, vop2->old_layer_sel); + if (layer_sel != old_layer_sel && atv_layer_sel != old_layer_sel) { + cfg_done = vop2_readl(vop2, RK3568_REG_CFG_DONE); + cfg_done &= (BIT(vop2->data->nr_vps) - 1); + cfg_done &= ~BIT(vp->id); + /* + * Changes of other VPs' overlays have not taken effect + */ + if (cfg_done) + rk3568_vop2_wait_for_layer_cfg_done(vop2, vop2->old_layer_sel); + } vop2_writel(vop2, RK3568_OVL_LAYER_SEL, layer_sel); mutex_unlock(&vop2->ovl_lock); diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index c39f0245e3a97..3f138776d35fb 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -361,6 +361,7 @@ static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched) /** * drm_sched_job_done - complete a job * @s_job: pointer to the job which is done + * @result: 0 on success, -ERRNO on error * * Finish the job's fence and resubmit the work items. */ diff --git a/drivers/gpu/drm/sitronix/st7586.c b/drivers/gpu/drm/sitronix/st7586.c index a29672d84eded..055383497aae7 100644 --- a/drivers/gpu/drm/sitronix/st7586.c +++ b/drivers/gpu/drm/sitronix/st7586.c @@ -346,6 +346,12 @@ static int st7586_probe(struct spi_device *spi) if (ret) return ret; + /* + * Override value set by mipi_dbi_spi_init(). This driver is a bit + * non-standard, so best to set it explicitly here. + */ + dbi->write_memory_bpw = 8; + /* Cannot read from this controller via SPI */ dbi->read_commands = NULL; @@ -355,15 +361,6 @@ static int st7586_probe(struct spi_device *spi) if (ret) return ret; - /* - * we are using 8-bit data, so we are not actually swapping anything, - * but setting mipi->swap_bytes makes mipi_dbi_typec3_command() do the - * right thing and not use 16-bit transfers (which results in swapped - * bytes on little-endian systems and causes out of order data to be - * sent to the display). - */ - dbi->swap_bytes = true; - drm_mode_config_reset(drm); ret = drm_dev_register(drm, 0); diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c index eec43d1a55951..18d2294c526de 100644 --- a/drivers/gpu/drm/solomon/ssd130x.c +++ b/drivers/gpu/drm/solomon/ssd130x.c @@ -736,6 +736,7 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, unsigned int height = drm_rect_height(rect); unsigned int line_length = DIV_ROUND_UP(width, 8); unsigned int page_height = SSD130X_PAGE_HEIGHT; + u8 page_start = ssd130x->page_offset + y / page_height; unsigned int pages = DIV_ROUND_UP(height, page_height); struct drm_device *drm = &ssd130x->drm; u32 array_idx = 0; @@ -773,14 +774,11 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, */ if (!ssd130x->page_address_mode) { - u8 page_start; - /* Set address range for horizontal addressing mode */ ret = ssd130x_set_col_range(ssd130x, ssd130x->col_offset + x, width); if (ret < 0) return ret; - page_start = ssd130x->page_offset + y / page_height; ret = ssd130x_set_page_range(ssd130x, page_start, pages); if (ret < 0) return ret; @@ -812,7 +810,7 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, */ if (ssd130x->page_address_mode) { ret = ssd130x_set_page_pos(ssd130x, - ssd130x->page_offset + i, + page_start + i, ssd130x->col_offset + x); if (ret < 0) return ret; diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h index 89633e30ca625..265a1b8a93493 100644 --- a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h +++ b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h @@ -47,15 +47,6 @@ const struct drm_format_info *drm_sysfb_get_format_si(struct drm_device *dev, const struct screen_info *si); #endif -/* - * Input parsing - */ - -int drm_sysfb_get_validated_int(struct drm_device *dev, const char *name, - u64 value, u32 max); -int drm_sysfb_get_validated_int0(struct drm_device *dev, const char *name, - u64 value, u32 max); - /* * Display modes */ diff --git a/drivers/gpu/drm/sysfb/efidrm.c b/drivers/gpu/drm/sysfb/efidrm.c index 1883c4a8604c2..97a3711e79337 100644 --- a/drivers/gpu/drm/sysfb/efidrm.c +++ b/drivers/gpu/drm/sysfb/efidrm.c @@ -150,7 +150,6 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv, struct drm_sysfb_device *sysfb; struct drm_device *dev; struct resource *mem = NULL; - void __iomem *screen_base = NULL; struct drm_plane *primary_plane; struct drm_crtc *crtc; struct drm_encoder *encoder; @@ -235,21 +234,38 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv, mem_flags = efidrm_get_mem_flags(dev, res->start, vsize); - if (mem_flags & EFI_MEMORY_WC) - screen_base = devm_ioremap_wc(&pdev->dev, mem->start, resource_size(mem)); - else if (mem_flags & EFI_MEMORY_UC) - screen_base = devm_ioremap(&pdev->dev, mem->start, resource_size(mem)); - else if (mem_flags & EFI_MEMORY_WT) - screen_base = devm_memremap(&pdev->dev, mem->start, resource_size(mem), - MEMREMAP_WT); - else if (mem_flags & EFI_MEMORY_WB) - screen_base = devm_memremap(&pdev->dev, mem->start, resource_size(mem), - MEMREMAP_WB); - else + if (mem_flags & EFI_MEMORY_WC) { + void __iomem *screen_base = devm_ioremap_wc(&pdev->dev, mem->start, + resource_size(mem)); + + if (!screen_base) + return ERR_PTR(-ENXIO); + iosys_map_set_vaddr_iomem(&sysfb->fb_addr, screen_base); + } else if (mem_flags & EFI_MEMORY_UC) { + void __iomem *screen_base = devm_ioremap(&pdev->dev, mem->start, + resource_size(mem)); + + if (!screen_base) + return ERR_PTR(-ENXIO); + iosys_map_set_vaddr_iomem(&sysfb->fb_addr, screen_base); + } else if (mem_flags & EFI_MEMORY_WT) { + void *screen_base = devm_memremap(&pdev->dev, mem->start, + resource_size(mem), MEMREMAP_WT); + + if (IS_ERR(screen_base)) + return ERR_CAST(screen_base); + iosys_map_set_vaddr(&sysfb->fb_addr, screen_base); + } else if (mem_flags & EFI_MEMORY_WB) { + void *screen_base = devm_memremap(&pdev->dev, mem->start, + resource_size(mem), MEMREMAP_WB); + + if (IS_ERR(screen_base)) + return ERR_CAST(screen_base); + iosys_map_set_vaddr(&sysfb->fb_addr, screen_base); + } else { drm_err(dev, "invalid mem_flags: 0x%llx\n", mem_flags); - if (!screen_base) - return ERR_PTR(-ENOMEM); - iosys_map_set_vaddr_iomem(&sysfb->fb_addr, screen_base); + return ERR_PTR(-EINVAL); + } /* * Modesetting diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c index ddfb2858acbf1..5aa78b902adac 100644 --- a/drivers/gpu/drm/tegra/dsi.c +++ b/drivers/gpu/drm/tegra/dsi.c @@ -1540,11 +1540,9 @@ static int tegra_dsi_ganged_probe(struct tegra_dsi *dsi) return -EPROBE_DEFER; dsi->slave = platform_get_drvdata(gangster); - - if (!dsi->slave) { - put_device(&gangster->dev); + put_device(&gangster->dev); + if (!dsi->slave) return -EPROBE_DEFER; - } dsi->slave->master = dsi; } diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c index 8cd2969e7d4bf..c4820f5e76581 100644 --- a/drivers/gpu/drm/tegra/hdmi.c +++ b/drivers/gpu/drm/tegra/hdmi.c @@ -658,7 +658,7 @@ static void tegra_hdmi_write_infopack(struct tegra_hdmi *hdmi, const void *data, { const u8 *ptr = data; unsigned long offset; - size_t i, j; + size_t i; u32 value; switch (ptr[0]) { @@ -691,7 +691,7 @@ static void tegra_hdmi_write_infopack(struct tegra_hdmi *hdmi, const void *data, * - subpack_low: bytes 0 - 3 * - subpack_high: bytes 4 - 6 (with byte 7 padded to 0x00) */ - for (i = 3, j = 0; i < size; i += 7, j += 8) { + for (i = 3; i < size; i += 7) { size_t rem = size - i, num = min_t(size_t, rem, 4); value = tegra_hdmi_subpack(&ptr[i], num); diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c index 21f3dfdcc5c95..bc7dd562cf6b6 100644 --- a/drivers/gpu/drm/tegra/sor.c +++ b/drivers/gpu/drm/tegra/sor.c @@ -1864,7 +1864,7 @@ static void tegra_sor_hdmi_write_infopack(struct tegra_sor *sor, { const u8 *ptr = data; unsigned long offset; - size_t i, j; + size_t i; u32 value; switch (ptr[0]) { @@ -1897,7 +1897,7 @@ static void tegra_sor_hdmi_write_infopack(struct tegra_sor *sor, * - subpack_low: bytes 0 - 3 * - subpack_high: bytes 4 - 6 (with byte 7 padded to 0x00) */ - for (i = 3, j = 0; i < size; i += 7, j += 8) { + for (i = 3; i < size; i += 7) { size_t rem = size - i, num = min_t(size_t, rem, 4); value = tegra_sor_hdmi_subpack(&ptr[i], num); diff --git a/drivers/gpu/drm/tests/drm_atomic_state_test.c b/drivers/gpu/drm/tests/drm_atomic_state_test.c index 2f6ac7a09f445..bc27f65b28233 100644 --- a/drivers/gpu/drm/tests/drm_atomic_state_test.c +++ b/drivers/gpu/drm/tests/drm_atomic_state_test.c @@ -156,24 +156,29 @@ static int set_up_atomic_state(struct kunit *test, if (connector) { conn_state = drm_atomic_get_connector_state(state, connector); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state); + if (IS_ERR(conn_state)) + return PTR_ERR(conn_state); ret = drm_atomic_set_crtc_for_connector(conn_state, crtc); - KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + return ret; } crtc_state = drm_atomic_get_crtc_state(state, crtc); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); ret = drm_atomic_set_mode_for_crtc(crtc_state, &drm_atomic_test_mode); - KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + return ret; crtc_state->enable = true; crtc_state->active = true; if (connector) { ret = drm_atomic_commit(state); - KUNIT_ASSERT_EQ(test, ret, 0); + if (ret) + return ret; } else { // dummy connector mask crtc_state->connector_mask = DRM_TEST_CONN_0; @@ -206,7 +211,13 @@ static void drm_test_check_connector_changed_modeset(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); // first modeset to enable +retry_set_up: ret = set_up_atomic_state(test, priv, old_conn, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_set_up; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -277,13 +288,26 @@ static void drm_test_check_valid_clones(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_set_up: ret = set_up_atomic_state(test, priv, NULL, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_set_up; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state); +retry: crtc_state = drm_atomic_get_crtc_state(state, priv->crtc); + if (PTR_ERR(crtc_state) == -EDEADLK) { + drm_atomic_state_clear(state); + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry; + } KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); crtc_state->encoder_mask = param->encoder_mask; @@ -292,6 +316,12 @@ static void drm_test_check_valid_clones(struct kunit *test) crtc_state->mode_changed = true; ret = drm_atomic_helper_check_modeset(drm, state); + if (ret == -EDEADLK) { + drm_atomic_state_clear(state); + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry; + } KUNIT_ASSERT_EQ(test, ret, param->expected_result); drm_modeset_drop_locks(&ctx); diff --git a/drivers/gpu/drm/tests/drm_gem_shmem_test.c b/drivers/gpu/drm/tests/drm_gem_shmem_test.c index 68f2c31623547..4b459f21acfd9 100644 --- a/drivers/gpu/drm/tests/drm_gem_shmem_test.c +++ b/drivers/gpu/drm/tests/drm_gem_shmem_test.c @@ -19,6 +19,8 @@ #include #include +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); + #define TEST_SIZE SZ_1M #define TEST_BYTE 0xae @@ -34,6 +36,9 @@ KUNIT_DEFINE_ACTION_WRAPPER(sg_free_table_wrapper, sg_free_table, KUNIT_DEFINE_ACTION_WRAPPER(drm_gem_shmem_free_wrapper, drm_gem_shmem_free, struct drm_gem_shmem_object *); +KUNIT_DEFINE_ACTION_WRAPPER(drm_gem_shmem_unpin_wrapper, drm_gem_shmem_unpin, + struct drm_gem_shmem_object *); + /* * Test creating a shmem GEM object backed by shmem buffer. The test * case succeeds if the GEM object is successfully allocated with the @@ -173,7 +178,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); - ret = drm_gem_shmem_vmap_locked(shmem, &map); + ret = drm_gem_shmem_vmap(shmem, &map); KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_ASSERT_NOT_NULL(test, shmem->vaddr); KUNIT_ASSERT_FALSE(test, iosys_map_is_null(&map)); @@ -183,7 +188,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) for (i = 0; i < TEST_SIZE; i++) KUNIT_EXPECT_EQ(test, iosys_map_rd(&map, i, u8), TEST_BYTE); - drm_gem_shmem_vunmap_locked(shmem, &map); + drm_gem_shmem_vunmap(shmem, &map); KUNIT_EXPECT_NULL(test, shmem->vaddr); KUNIT_EXPECT_EQ(test, refcount_read(&shmem->vmap_use_count), 0); } @@ -194,7 +199,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) * scatter/gather table large enough to accommodate the backing memory * is successfully exported. */ -static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) +static void drm_gem_shmem_test_get_sg_table(struct kunit *test) { struct drm_device *drm_dev = test->priv; struct drm_gem_shmem_object *shmem; @@ -212,6 +217,9 @@ static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) ret = drm_gem_shmem_pin(shmem); KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, drm_gem_shmem_unpin_wrapper, shmem); + KUNIT_ASSERT_EQ(test, ret, 0); + sgt = drm_gem_shmem_get_sg_table(shmem); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sgt); KUNIT_EXPECT_NULL(test, shmem->sgt); @@ -236,7 +244,7 @@ static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) * backing pages are pinned and a scatter/gather table large enough to * accommodate the backing memory is successfully exported. */ -static void drm_gem_shmem_test_get_sg_table(struct kunit *test) +static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) { struct drm_device *drm_dev = test->priv; struct drm_gem_shmem_object *shmem; @@ -284,17 +292,17 @@ static void drm_gem_shmem_test_madvise(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); - ret = drm_gem_shmem_madvise_locked(shmem, 1); + ret = drm_gem_shmem_madvise(shmem, 1); KUNIT_EXPECT_TRUE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, 1); /* Set madv to a negative value */ - ret = drm_gem_shmem_madvise_locked(shmem, -1); + ret = drm_gem_shmem_madvise(shmem, -1); KUNIT_EXPECT_FALSE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, -1); /* Check that madv cannot be set back to a positive value */ - ret = drm_gem_shmem_madvise_locked(shmem, 0); + ret = drm_gem_shmem_madvise(shmem, 0); KUNIT_EXPECT_FALSE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, -1); } @@ -322,7 +330,7 @@ static void drm_gem_shmem_test_purge(struct kunit *test) ret = drm_gem_shmem_is_purgeable(shmem); KUNIT_EXPECT_FALSE(test, ret); - ret = drm_gem_shmem_madvise_locked(shmem, 1); + ret = drm_gem_shmem_madvise(shmem, 1); KUNIT_EXPECT_TRUE(test, ret); /* The scatter/gather table will be freed by drm_gem_shmem_free */ @@ -332,7 +340,9 @@ static void drm_gem_shmem_test_purge(struct kunit *test) ret = drm_gem_shmem_is_purgeable(shmem); KUNIT_EXPECT_TRUE(test, ret); - drm_gem_shmem_purge_locked(shmem); + ret = drm_gem_shmem_purge(shmem); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, shmem->pages); KUNIT_EXPECT_NULL(test, shmem->sgt); KUNIT_EXPECT_EQ(test, shmem->madv, -1); @@ -366,8 +376,8 @@ static struct kunit_case drm_gem_shmem_test_cases[] = { KUNIT_CASE(drm_gem_shmem_test_obj_create_private), KUNIT_CASE(drm_gem_shmem_test_pin_pages), KUNIT_CASE(drm_gem_shmem_test_vmap), - KUNIT_CASE(drm_gem_shmem_test_get_pages_sgt), KUNIT_CASE(drm_gem_shmem_test_get_sg_table), + KUNIT_CASE(drm_gem_shmem_test_get_pages_sgt), KUNIT_CASE(drm_gem_shmem_test_madvise), KUNIT_CASE(drm_gem_shmem_test_purge), {} diff --git a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c index 8bd412735000c..70f9aa7021430 100644 --- a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c +++ b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c @@ -257,10 +257,16 @@ static void drm_test_check_broadcast_rgb_crtc_mode_changed(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -326,10 +332,16 @@ static void drm_test_check_broadcast_rgb_crtc_mode_not_changed(struct kunit *tes drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -397,10 +409,16 @@ static void drm_test_check_broadcast_rgb_auto_cea_mode(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -457,10 +475,17 @@ static void drm_test_check_broadcast_rgb_auto_cea_mode_vic_1(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, mode); crtc = priv->crtc; + +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, mode, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -518,10 +543,16 @@ static void drm_test_check_broadcast_rgb_full_cea_mode(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -580,10 +611,17 @@ static void drm_test_check_broadcast_rgb_full_cea_mode_vic_1(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, mode); crtc = priv->crtc; + +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, mode, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -643,10 +681,16 @@ static void drm_test_check_broadcast_rgb_limited_cea_mode(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -705,10 +749,17 @@ static void drm_test_check_broadcast_rgb_limited_cea_mode_vic_1(struct kunit *te KUNIT_ASSERT_NOT_NULL(test, mode); crtc = priv->crtc; + +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, mode, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -870,10 +921,16 @@ static void drm_test_check_output_bpc_crtc_mode_changed(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -946,10 +1003,16 @@ static void drm_test_check_output_bpc_crtc_mode_not_changed(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); @@ -1022,10 +1085,16 @@ static void drm_test_check_output_bpc_dvi(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); conn_state = conn->state; @@ -1069,10 +1138,16 @@ static void drm_test_check_tmds_char_rate_rgb_8bpc(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); conn_state = conn->state; @@ -1118,10 +1193,16 @@ static void drm_test_check_tmds_char_rate_rgb_10bpc(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); conn_state = conn->state; @@ -1167,10 +1248,16 @@ static void drm_test_check_tmds_char_rate_rgb_12bpc(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); conn_state = conn->state; @@ -1218,10 +1305,16 @@ static void drm_test_check_hdmi_funcs_reject_rate(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); /* You shouldn't be doing that at home. */ @@ -1292,10 +1385,16 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_rgb(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_EXPECT_EQ(test, ret, 0); conn_state = conn->state; @@ -1440,10 +1539,16 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv422(struct kunit drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_EXPECT_EQ(test, ret, 0); conn_state = conn->state; @@ -1669,10 +1774,17 @@ static void drm_test_check_output_bpc_format_vic_1(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); crtc = priv->crtc; + +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, mode, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_EXPECT_EQ(test, ret, 0); conn_state = conn->state; @@ -1736,10 +1848,16 @@ static void drm_test_check_output_bpc_format_driver_rgb_only(struct kunit *test) drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_EXPECT_EQ(test, ret, 0); conn_state = conn->state; @@ -1805,10 +1923,16 @@ static void drm_test_check_output_bpc_format_display_rgb_only(struct kunit *test drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_EXPECT_EQ(test, ret, 0); conn_state = conn->state; @@ -1865,10 +1989,16 @@ static void drm_test_check_output_bpc_format_driver_8bpc_only(struct kunit *test drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_EXPECT_EQ(test, ret, 0); conn_state = conn->state; @@ -1927,10 +2057,16 @@ static void drm_test_check_output_bpc_format_display_8bpc_only(struct kunit *tes drm_modeset_acquire_init(&ctx, 0); +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_EXPECT_EQ(test, ret, 0); conn_state = conn->state; @@ -1970,10 +2106,17 @@ static void drm_test_check_disable_connector(struct kunit *test) drm = &priv->drm; crtc = priv->crtc; + +retry_conn_enable: ret = drm_kunit_helper_enable_crtc_connector(test, drm, crtc, conn, preferred, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry_conn_enable; + } KUNIT_ASSERT_EQ(test, ret, 0); state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx); diff --git a/drivers/gpu/drm/tidss/tidss_dispc.c b/drivers/gpu/drm/tidss/tidss_dispc.c index 7c8c15a5c39b3..6bf1171d0bc2d 100644 --- a/drivers/gpu/drm/tidss/tidss_dispc.c +++ b/drivers/gpu/drm/tidss/tidss_dispc.c @@ -57,12 +57,6 @@ static const u16 tidss_k2g_common_regs[DISPC_COMMON_REG_TABLE_LEN] = { }; const struct dispc_features dispc_k2g_feats = { - .min_pclk_khz = 4375, - - .max_pclk_khz = { - [DISPC_VP_DPI] = 150000, - }, - /* * XXX According TRM the RGB input buffer width up to 2560 should * work on 3 taps, but in practice it only works up to 1280. @@ -145,11 +139,6 @@ static const u16 tidss_am65x_common_regs[DISPC_COMMON_REG_TABLE_LEN] = { }; const struct dispc_features dispc_am65x_feats = { - .max_pclk_khz = { - [DISPC_VP_DPI] = 165000, - [DISPC_VP_OLDI_AM65X] = 165000, - }, - .scaling = { .in_width_max_5tap_rgb = 1280, .in_width_max_3tap_rgb = 2560, @@ -245,11 +234,6 @@ static const u16 tidss_j721e_common_regs[DISPC_COMMON_REG_TABLE_LEN] = { }; const struct dispc_features dispc_j721e_feats = { - .max_pclk_khz = { - [DISPC_VP_DPI] = 170000, - [DISPC_VP_INTERNAL] = 600000, - }, - .scaling = { .in_width_max_5tap_rgb = 2048, .in_width_max_3tap_rgb = 4096, @@ -316,11 +300,6 @@ const struct dispc_features dispc_j721e_feats = { }; const struct dispc_features dispc_am625_feats = { - .max_pclk_khz = { - [DISPC_VP_DPI] = 165000, - [DISPC_VP_INTERNAL] = 170000, - }, - .scaling = { .in_width_max_5tap_rgb = 1280, .in_width_max_3tap_rgb = 2560, @@ -377,15 +356,6 @@ const struct dispc_features dispc_am625_feats = { }; const struct dispc_features dispc_am62a7_feats = { - /* - * if the code reaches dispc_mode_valid with VP1, - * it should return MODE_BAD. - */ - .max_pclk_khz = { - [DISPC_VP_TIED_OFF] = 0, - [DISPC_VP_DPI] = 165000, - }, - .scaling = { .in_width_max_5tap_rgb = 1280, .in_width_max_3tap_rgb = 2560, @@ -442,10 +412,6 @@ const struct dispc_features dispc_am62a7_feats = { }; const struct dispc_features dispc_am62l_feats = { - .max_pclk_khz = { - [DISPC_VP_DPI] = 165000, - }, - .subrev = DISPC_AM62L, .common = "common", @@ -1331,33 +1297,61 @@ static void dispc_vp_set_default_color(struct dispc_device *dispc, DISPC_OVR_DEFAULT_COLOR2, (v >> 32) & 0xffff); } +/* + * Calculate the percentage difference between the requested pixel clock rate + * and the effective rate resulting from calculating the clock divider value. + */ +unsigned int dispc_pclk_diff(unsigned long rate, unsigned long real_rate) +{ + int r = rate / 100, rr = real_rate / 100; + + return (unsigned int)(abs(((rr - r) * 100) / r)); +} + +static int check_pixel_clock(struct dispc_device *dispc, u32 hw_videoport, + unsigned long clock) +{ + unsigned long round_clock; + + /* + * For VP's with external clocking, clock operations must be + * delegated to respective driver, so we skip the check here. + */ + if (dispc->tidss->is_ext_vp_clk[hw_videoport]) + return 0; + + round_clock = clk_round_rate(dispc->vp_clk[hw_videoport], clock); + /* + * To keep the check consistent with dispc_vp_set_clk_rate(), we + * use the same 5% check here. + */ + if (dispc_pclk_diff(clock, round_clock) > 5) + return -EINVAL; + + return 0; +} + enum drm_mode_status dispc_vp_mode_valid(struct dispc_device *dispc, u32 hw_videoport, const struct drm_display_mode *mode) { u32 hsw, hfp, hbp, vsw, vfp, vbp; enum dispc_vp_bus_type bus_type; - int max_pclk; bus_type = dispc->feat->vp_bus_type[hw_videoport]; - max_pclk = dispc->feat->max_pclk_khz[bus_type]; - - if (WARN_ON(max_pclk == 0)) + if (WARN_ON(bus_type == DISPC_VP_TIED_OFF)) return MODE_BAD; - if (mode->clock < dispc->feat->min_pclk_khz) - return MODE_CLOCK_LOW; - - if (mode->clock > max_pclk) - return MODE_CLOCK_HIGH; - if (mode->hdisplay > 4096) return MODE_BAD; if (mode->vdisplay > 4096) return MODE_BAD; + if (check_pixel_clock(dispc, hw_videoport, mode->clock * 1000)) + return MODE_CLOCK_RANGE; + /* TODO: add interlace support */ if (mode->flags & DRM_MODE_FLAG_INTERLACE) return MODE_NO_INTERLACE; @@ -1421,17 +1415,6 @@ void dispc_vp_disable_clk(struct dispc_device *dispc, u32 hw_videoport) clk_disable_unprepare(dispc->vp_clk[hw_videoport]); } -/* - * Calculate the percentage difference between the requested pixel clock rate - * and the effective rate resulting from calculating the clock divider value. - */ -unsigned int dispc_pclk_diff(unsigned long rate, unsigned long real_rate) -{ - int r = rate / 100, rr = real_rate / 100; - - return (unsigned int)(abs(((rr - r) * 100) / r)); -} - int dispc_vp_set_clk_rate(struct dispc_device *dispc, u32 hw_videoport, unsigned long rate) { diff --git a/drivers/gpu/drm/tidss/tidss_dispc.h b/drivers/gpu/drm/tidss/tidss_dispc.h index 60c1b400eb893..42279312dcc1b 100644 --- a/drivers/gpu/drm/tidss/tidss_dispc.h +++ b/drivers/gpu/drm/tidss/tidss_dispc.h @@ -77,9 +77,6 @@ enum dispc_dss_subrevision { }; struct dispc_features { - int min_pclk_khz; - int max_pclk_khz[DISPC_VP_MAX_BUS_TYPE]; - struct dispc_features_scaling scaling; enum dispc_dss_subrevision subrev; diff --git a/drivers/gpu/drm/tidss/tidss_drv.h b/drivers/gpu/drm/tidss/tidss_drv.h index 84454a4855d11..e1c1f41d8b4be 100644 --- a/drivers/gpu/drm/tidss/tidss_drv.h +++ b/drivers/gpu/drm/tidss/tidss_drv.h @@ -24,6 +24,8 @@ struct tidss_device { const struct dispc_features *feat; struct dispc_device *dispc; + bool is_ext_vp_clk[TIDSS_MAX_PORTS]; + unsigned int num_crtcs; struct drm_crtc *crtcs[TIDSS_MAX_PORTS]; diff --git a/drivers/gpu/drm/tidss/tidss_kms.c b/drivers/gpu/drm/tidss/tidss_kms.c index c34eb90cddbea..9f5f98e707f2c 100644 --- a/drivers/gpu/drm/tidss/tidss_kms.c +++ b/drivers/gpu/drm/tidss/tidss_kms.c @@ -28,9 +28,33 @@ static void tidss_atomic_commit_tail(struct drm_atomic_state *old_state) tidss_runtime_get(tidss); - drm_atomic_helper_commit_modeset_disables(ddev, old_state); - drm_atomic_helper_commit_planes(ddev, old_state, DRM_PLANE_COMMIT_ACTIVE_ONLY); - drm_atomic_helper_commit_modeset_enables(ddev, old_state); + /* + * TI's OLDI and DSI encoders need to be set up before the crtc is + * enabled. Thus drm_atomic_helper_commit_modeset_enables() and + * drm_atomic_helper_commit_modeset_disables() cannot be used here, as + * they enable the crtc before bridges' pre-enable, and disable the crtc + * after bridges' post-disable. + * + * Open code the functions here and first call the bridges' pre-enables, + * then crtc enable, then bridges' post-enable (and vice versa for + * disable). + */ + + drm_atomic_helper_commit_encoder_bridge_disable(ddev, old_state); + drm_atomic_helper_commit_crtc_disable(ddev, old_state); + drm_atomic_helper_commit_encoder_bridge_post_disable(ddev, old_state); + + drm_atomic_helper_update_legacy_modeset_state(ddev, old_state); + drm_atomic_helper_calc_timestamping_constants(old_state); + drm_atomic_helper_commit_crtc_set_mode(ddev, old_state); + + drm_atomic_helper_commit_planes(ddev, old_state, + DRM_PLANE_COMMIT_ACTIVE_ONLY); + + drm_atomic_helper_commit_encoder_bridge_pre_enable(ddev, old_state); + drm_atomic_helper_commit_crtc_enable(ddev, old_state); + drm_atomic_helper_commit_encoder_bridge_enable(ddev, old_state); + drm_atomic_helper_commit_writebacks(ddev, old_state); drm_atomic_helper_commit_hw_done(old_state); drm_atomic_helper_wait_for_flip_done(ddev, old_state); diff --git a/drivers/gpu/drm/tidss/tidss_oldi.c b/drivers/gpu/drm/tidss/tidss_oldi.c index 7688251beba28..17c535bfa0576 100644 --- a/drivers/gpu/drm/tidss/tidss_oldi.c +++ b/drivers/gpu/drm/tidss/tidss_oldi.c @@ -309,6 +309,25 @@ static u32 *tidss_oldi_atomic_get_input_bus_fmts(struct drm_bridge *bridge, return input_fmts; } +static enum drm_mode_status +tidss_oldi_mode_valid(struct drm_bridge *bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) +{ + struct tidss_oldi *oldi = drm_bridge_to_tidss_oldi(bridge); + unsigned long round_clock; + + round_clock = clk_round_rate(oldi->serial, mode->clock * 7 * 1000); + /* + * To keep the check consistent with dispc_vp_set_clk_rate(), + * we use the same 5% check here. + */ + if (dispc_pclk_diff(mode->clock * 7 * 1000, round_clock) > 5) + return -EINVAL; + + return 0; +} + static const struct drm_bridge_funcs tidss_oldi_bridge_funcs = { .attach = tidss_oldi_bridge_attach, .atomic_pre_enable = tidss_oldi_atomic_pre_enable, @@ -317,6 +336,7 @@ static const struct drm_bridge_funcs tidss_oldi_bridge_funcs = { .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, .atomic_reset = drm_atomic_helper_bridge_reset, + .mode_valid = tidss_oldi_mode_valid, }; static int get_oldi_mode(struct device_node *oldi_tx, int *companion_instance) @@ -430,6 +450,7 @@ void tidss_oldi_deinit(struct tidss_device *tidss) for (int i = 0; i < tidss->num_oldis; i++) { if (tidss->oldis[i]) { drm_bridge_remove(&tidss->oldis[i]->bridge); + tidss->is_ext_vp_clk[tidss->oldis[i]->parent_vp] = false; tidss->oldis[i] = NULL; } } @@ -580,6 +601,7 @@ int tidss_oldi_init(struct tidss_device *tidss) oldi->bridge.timings = &default_tidss_oldi_timings; tidss->oldis[tidss->num_oldis++] = oldi; + tidss->is_ext_vp_clk[oldi->parent_vp] = true; oldi->tidss = tidss; drm_bridge_add(&oldi->bridge); diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c index b5f60b2b2d0e7..41802c9bd1475 100644 --- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c +++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c @@ -586,7 +586,7 @@ static void tilcdc_crtc_recover_work(struct work_struct *work) drm_modeset_unlock(&crtc->mutex); } -static void tilcdc_crtc_destroy(struct drm_crtc *crtc) +void tilcdc_crtc_destroy(struct drm_crtc *crtc) { struct tilcdc_drm_private *priv = crtc->dev->dev_private; diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c index 7caec4d38ddf0..3dcbec312bacb 100644 --- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c +++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c @@ -172,8 +172,7 @@ static void tilcdc_fini(struct drm_device *dev) if (priv->crtc) tilcdc_crtc_shutdown(priv->crtc); - if (priv->is_registered) - drm_dev_unregister(dev); + drm_dev_unregister(dev); drm_kms_helper_poll_fini(dev); drm_atomic_helper_shutdown(dev); @@ -220,21 +219,21 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev) priv->wq = alloc_ordered_workqueue("tilcdc", 0); if (!priv->wq) { ret = -ENOMEM; - goto init_failed; + goto put_drm; } priv->mmio = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->mmio)) { dev_err(dev, "failed to request / ioremap\n"); ret = PTR_ERR(priv->mmio); - goto init_failed; + goto free_wq; } priv->clk = clk_get(dev, "fck"); if (IS_ERR(priv->clk)) { dev_err(dev, "failed to get functional clock\n"); ret = -ENODEV; - goto init_failed; + goto free_wq; } pm_runtime_enable(dev); @@ -313,7 +312,7 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev) ret = tilcdc_crtc_create(ddev); if (ret < 0) { dev_err(dev, "failed to create crtc\n"); - goto init_failed; + goto disable_pm; } modeset_init(ddev); @@ -324,46 +323,46 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev) if (ret) { dev_err(dev, "failed to register cpufreq notifier\n"); priv->freq_transition.notifier_call = NULL; - goto init_failed; + goto destroy_crtc; } #endif if (priv->is_componentized) { ret = component_bind_all(dev, ddev); if (ret < 0) - goto init_failed; + goto unregister_cpufreq_notif; ret = tilcdc_add_component_encoder(ddev); if (ret < 0) - goto init_failed; + goto unbind_component; } else { ret = tilcdc_attach_external_device(ddev); if (ret) - goto init_failed; + goto unregister_cpufreq_notif; } if (!priv->external_connector && ((priv->num_encoders == 0) || (priv->num_connectors == 0))) { dev_err(dev, "no encoders/connectors found\n"); ret = -EPROBE_DEFER; - goto init_failed; + goto unbind_component; } ret = drm_vblank_init(ddev, 1); if (ret < 0) { dev_err(dev, "failed to initialize vblank\n"); - goto init_failed; + goto unbind_component; } ret = platform_get_irq(pdev, 0); if (ret < 0) - goto init_failed; + goto unbind_component; priv->irq = ret; ret = tilcdc_irq_install(ddev, priv->irq); if (ret < 0) { dev_err(dev, "failed to install IRQ handler\n"); - goto init_failed; + goto unbind_component; } drm_mode_config_reset(ddev); @@ -372,16 +371,34 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev) ret = drm_dev_register(ddev, 0); if (ret) - goto init_failed; - priv->is_registered = true; + goto stop_poll; drm_client_setup_with_color_mode(ddev, bpp); return 0; -init_failed: - tilcdc_fini(ddev); +stop_poll: + drm_kms_helper_poll_fini(ddev); + tilcdc_irq_uninstall(ddev); +unbind_component: + if (priv->is_componentized) + component_unbind_all(dev, ddev); +unregister_cpufreq_notif: +#ifdef CONFIG_CPU_FREQ + cpufreq_unregister_notifier(&priv->freq_transition, + CPUFREQ_TRANSITION_NOTIFIER); +destroy_crtc: +#endif + tilcdc_crtc_destroy(priv->crtc); +disable_pm: + pm_runtime_disable(dev); + clk_put(priv->clk); +free_wq: + destroy_workqueue(priv->wq); +put_drm: platform_set_drvdata(pdev, NULL); + ddev->dev_private = NULL; + drm_dev_put(ddev); return ret; } diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.h b/drivers/gpu/drm/tilcdc/tilcdc_drv.h index b818448c83f61..58b276f82a669 100644 --- a/drivers/gpu/drm/tilcdc/tilcdc_drv.h +++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.h @@ -82,7 +82,6 @@ struct tilcdc_drm_private { struct drm_encoder *external_encoder; struct drm_connector *external_connector; - bool is_registered; bool is_componentized; bool irq_enabled; }; @@ -164,6 +163,7 @@ void tilcdc_crtc_set_panel_info(struct drm_crtc *crtc, void tilcdc_crtc_set_simulate_vesa_sync(struct drm_crtc *crtc, bool simulate_vesa_sync); void tilcdc_crtc_shutdown(struct drm_crtc *crtc); +void tilcdc_crtc_destroy(struct drm_crtc *crtc); int tilcdc_crtc_update_fb(struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_pending_vblank_event *event); diff --git a/drivers/gpu/drm/tiny/sharp-memory.c b/drivers/gpu/drm/tiny/sharp-memory.c index 64272cd0f6e22..cbf69460ebf32 100644 --- a/drivers/gpu/drm/tiny/sharp-memory.c +++ b/drivers/gpu/drm/tiny/sharp-memory.c @@ -541,8 +541,8 @@ static int sharp_memory_probe(struct spi_device *spi) smd = devm_drm_dev_alloc(dev, &sharp_memory_drm_driver, struct sharp_memory_device, drm); - if (!smd) - return -ENOMEM; + if (IS_ERR(smd)) + return PTR_ERR(smd); spi_set_drvdata(spi, smd); diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c index 6c77550c51af1..40a55f81bf47d 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -222,13 +222,13 @@ static void ttm_bo_reserve_interrupted(struct kunit *test) KUNIT_FAIL(test, "Couldn't create ttm bo reserve task\n"); /* Take a lock so the threaded reserve has to wait */ - mutex_lock(&bo->base.resv->lock.base); + dma_resv_lock(bo->base.resv, NULL); wake_up_process(task); msleep(20); err = kthread_stop(task); - mutex_unlock(&bo->base.resv->lock.base); + dma_resv_unlock(bo->base.resv); KUNIT_ASSERT_EQ(test, err, -ERESTARTSYS); } diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index b47020fca1992..e6abc7b40b189 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -434,6 +434,11 @@ int ttm_bo_access(struct ttm_buffer_object *bo, unsigned long offset, if (ret) return ret; + if (!bo->resource) { + ret = -ENODATA; + goto unlock; + } + switch (bo->resource->mem_type) { case TTM_PL_SYSTEM: fallthrough; @@ -448,6 +453,7 @@ int ttm_bo_access(struct ttm_buffer_object *bo, unsigned long offset, ret = -EIO; } +unlock: ttm_bo_unreserve(bo); return ret; diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig index 4b55308fd2eb4..e933e64780272 100644 --- a/drivers/gpu/drm/tyr/Kconfig +++ b/drivers/gpu/drm/tyr/Kconfig @@ -6,6 +6,7 @@ config DRM_TYR depends on RUST depends on ARM || ARM64 || COMPILE_TEST depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE + depends on COMMON_CLK default n help Rust DRM driver for ARM Mali CSF-based GPUs. diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index d5625dd1e41c8..0052ebe957199 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -76,7 +76,7 @@ fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { dev_err!(dev, "GPU reset failed with errno\n"); dev_err!( dev, - "GPU_INT_RAWSTAT is {}\n", + "GPU_IRQ_RAWSTAT is {}\n", regs::GPU_IRQ_RAWSTAT.read(dev, iomem)? ); diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c index c5a3bbbc74c5c..f4da7a94e4016 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.c +++ b/drivers/gpu/drm/v3d/v3d_drv.c @@ -377,6 +377,8 @@ static int v3d_platform_drm_probe(struct platform_device *pdev) if (ret) goto clk_disable; + dma_set_max_seg_size(&pdev->dev, UINT_MAX); + v3d->va_width = 30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_VA_WIDTH); ident1 = V3D_READ(V3D_HUB_IDENT1); diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c index 4aaa587be3a5e..a1efda9c39f92 100644 --- a/drivers/gpu/drm/vc4/vc4_bo.c +++ b/drivers/gpu/drm/vc4/vc4_bo.c @@ -738,12 +738,15 @@ static int vc4_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct return -EINVAL; } + mutex_lock(&bo->madv_lock); if (bo->madv != VC4_MADV_WILLNEED) { DRM_DEBUG("mmapping of %s BO not allowed\n", bo->madv == VC4_MADV_DONTNEED ? "purgeable" : "purged"); + mutex_unlock(&bo->madv_lock); return -EINVAL; } + mutex_unlock(&bo->madv_lock); return drm_gem_dma_mmap(&bo->base, vma); } diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c index 255e5817618e3..6887631f2d8be 100644 --- a/drivers/gpu/drm/vc4/vc4_gem.c +++ b/drivers/gpu/drm/vc4/vc4_gem.c @@ -61,6 +61,7 @@ vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state) for (i = 0; i < state->user_state.bo_count; i++) drm_gem_object_put(state->bo[i]); + kfree(state->bo); kfree(state); } @@ -169,10 +170,8 @@ vc4_save_hang_state(struct drm_device *dev) spin_lock_irqsave(&vc4->job_lock, irqflags); exec[0] = vc4_first_bin_job(vc4); exec[1] = vc4_first_render_job(vc4); - if (!exec[0] && !exec[1]) { - spin_unlock_irqrestore(&vc4->job_lock, irqflags); - return; - } + if (!exec[0] && !exec[1]) + goto err_free_state; /* Get the bos from both binner and renderer into hang state. */ state->bo_count = 0; @@ -189,10 +188,8 @@ vc4_save_hang_state(struct drm_device *dev) kernel_state->bo = kcalloc(state->bo_count, sizeof(*kernel_state->bo), GFP_ATOMIC); - if (!kernel_state->bo) { - spin_unlock_irqrestore(&vc4->job_lock, irqflags); - return; - } + if (!kernel_state->bo) + goto err_free_state; k = 0; for (i = 0; i < 2; i++) { @@ -284,6 +281,12 @@ vc4_save_hang_state(struct drm_device *dev) vc4->hang_state = kernel_state; spin_unlock_irqrestore(&vc4->job_lock, irqflags); } + + return; + +err_free_state: + spin_unlock_irqrestore(&vc4->job_lock, irqflags); + kfree(kernel_state); } static void diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index 07c91b450f93a..34b974f46f87d 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -2355,17 +2355,23 @@ static int vc4_hdmi_hotplug_init(struct vc4_hdmi *vc4_hdmi) int ret; if (vc4_hdmi->variant->external_irq_controller) { - unsigned int hpd_con = platform_get_irq_byname(pdev, "hpd-connected"); - unsigned int hpd_rm = platform_get_irq_byname(pdev, "hpd-removed"); + int hpd = platform_get_irq_byname(pdev, "hpd-connected"); - ret = devm_request_threaded_irq(&pdev->dev, hpd_con, + if (hpd < 0) + return hpd; + + ret = devm_request_threaded_irq(&pdev->dev, hpd, NULL, vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT, "vc4 hdmi hpd connected", vc4_hdmi); if (ret) return ret; - ret = devm_request_threaded_irq(&pdev->dev, hpd_rm, + hpd = platform_get_irq_byname(pdev, "hpd-removed"); + if (hpd < 0) + return hpd; + + ret = devm_request_threaded_irq(&pdev->dev, hpd, NULL, vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT, "vc4 hdmi hpd disconnected", vc4_hdmi); diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c index bb09df5000bda..e470412851cc8 100644 --- a/drivers/gpu/drm/vc4/vc4_v3d.c +++ b/drivers/gpu/drm/vc4/vc4_v3d.c @@ -479,6 +479,7 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data) pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, 40); /* a little over 2 frames. */ + pm_runtime_put_autosuspend(dev); return 0; diff --git a/drivers/gpu/drm/vgem/vgem_fence.c b/drivers/gpu/drm/vgem/vgem_fence.c index fd76730fd38c0..07db319c3d7f9 100644 --- a/drivers/gpu/drm/vgem/vgem_fence.c +++ b/drivers/gpu/drm/vgem/vgem_fence.c @@ -79,7 +79,7 @@ static struct dma_fence *vgem_fence_create(struct vgem_file *vfile, dma_fence_init(&fence->base, &vgem_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1); - timer_setup(&fence->timer, vgem_fence_timeout, 0); + timer_setup(&fence->timer, vgem_fence_timeout, TIMER_IRQSAFE); /* We force the fence to expire within 10s to prevent driver hangs */ mod_timer(&fence->timer, jiffies + VGEM_FENCE_TIMEOUT); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c index f031a312c7835..b22887e8c8815 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c @@ -32,9 +32,15 @@ #include -static void vmw_bo_release(struct vmw_bo *vbo) +/** + * vmw_bo_free - vmw_bo destructor + * + * @bo: Pointer to the embedded struct ttm_buffer_object + */ +static void vmw_bo_free(struct ttm_buffer_object *bo) { struct vmw_resource *res; + struct vmw_bo *vbo = to_vmw_bo(&bo->base); WARN_ON(kref_read(&vbo->tbo.base.refcount) != 0); vmw_bo_unmap(vbo); @@ -62,20 +68,8 @@ static void vmw_bo_release(struct vmw_bo *vbo) } vmw_surface_unreference(&vbo->dumb_surface); } - drm_gem_object_release(&vbo->tbo.base); -} - -/** - * vmw_bo_free - vmw_bo destructor - * - * @bo: Pointer to the embedded struct ttm_buffer_object - */ -static void vmw_bo_free(struct ttm_buffer_object *bo) -{ - struct vmw_bo *vbo = to_vmw_bo(&bo->base); - WARN_ON(!RB_EMPTY_ROOT(&vbo->res_tree)); - vmw_bo_release(vbo); + drm_gem_object_release(&vbo->tbo.base); WARN_ON(vbo->dirty); kfree(vbo); } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index 3057f8baa7d25..e1f18020170ab 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -1143,7 +1143,7 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv, ret = vmw_user_bo_lookup(sw_context->filp, handle, &vmw_bo); if (ret != 0) { drm_dbg(&dev_priv->drm, "Could not find or use MOB buffer.\n"); - return PTR_ERR(vmw_bo); + return ret; } vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_MOB, VMW_BO_DOMAIN_MOB); ret = vmw_validation_add_bo(sw_context->ctx, vmw_bo); @@ -1199,7 +1199,7 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv, ret = vmw_user_bo_lookup(sw_context->filp, handle, &vmw_bo); if (ret != 0) { drm_dbg(&dev_priv->drm, "Could not find or use GMR region.\n"); - return PTR_ERR(vmw_bo); + return ret; } vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index 54ea1b513950a..3e8a2f4a907da 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -763,13 +763,16 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev, return ERR_PTR(ret); } - ttm_bo_reserve(&bo->tbo, false, false, NULL); - ret = vmw_bo_dirty_add(bo); - if (!ret && surface && surface->res.func->dirty_alloc) { - surface->res.coherent = true; - ret = surface->res.func->dirty_alloc(&surface->res); + if (bo) { + ttm_bo_reserve(&bo->tbo, false, false, NULL); + ret = vmw_bo_dirty_add(bo); + if (!ret && surface && surface->res.func->dirty_alloc) { + surface->res.coherent = true; + if (surface->res.dirty == NULL) + ret = surface->res.func->dirty_alloc(&surface->res); + } + ttm_bo_unreserve(&bo->tbo); } - ttm_bo_unreserve(&bo->tbo); return &vfb->base; } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c index fd4e76486f2d1..45561bc1c9eff 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c @@ -260,6 +260,13 @@ int vmw_bo_dirty_add(struct vmw_bo *vbo) return ret; } +static void vmw_bo_dirty_free(struct kref *kref) +{ + struct vmw_bo_dirty *dirty = container_of(kref, struct vmw_bo_dirty, ref_count); + + kvfree(dirty); +} + /** * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object * @vbo: The buffer object @@ -274,7 +281,7 @@ void vmw_bo_dirty_release(struct vmw_bo *vbo) { struct vmw_bo_dirty *dirty = vbo->dirty; - if (dirty && kref_put(&dirty->ref_count, (void *)kvfree)) + if (dirty && kref_put(&dirty->ref_count, vmw_bo_dirty_free)) vbo->dirty = NULL; } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c index 69dfe69ce0f87..a8c8c9375d297 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c @@ -923,8 +923,10 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv, ttm_bo_unreserve(&buf->tbo); res = vmw_shader_alloc(dev_priv, buf, size, 0, shader_type); - if (unlikely(ret != 0)) + if (IS_ERR(res)) { + ret = PTR_ERR(res); goto no_reserve; + } ret = vmw_cmdbuf_res_add(man, vmw_cmdbuf_res_shader, vmw_shader_key(user_key, shader_type), diff --git a/drivers/gpu/drm/xe/Kconfig b/drivers/gpu/drm/xe/Kconfig index 4b288eb3f5b0e..c34be1be155b3 100644 --- a/drivers/gpu/drm/xe/Kconfig +++ b/drivers/gpu/drm/xe/Kconfig @@ -39,7 +39,7 @@ config DRM_XE select DRM_TTM select DRM_TTM_HELPER select DRM_EXEC - select DRM_GPUSVM if !UML && DEVICE_PRIVATE + select DRM_GPUSVM if !UML select DRM_GPUVM select DRM_SCHED select MMU_NOTIFIER diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c index 8ea9a472113c4..8eaf1cc7fdf93 100644 --- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c +++ b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c @@ -3,7 +3,7 @@ * Copyright © 2023 Intel Corporation */ -#include +#include #include "intel_display_core.h" #include "intel_display_types.h" @@ -15,29 +15,15 @@ #include -struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper, - struct drm_fb_helper_surface_size *sizes) +struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_device *drm, + struct drm_mode_fb_cmd2 *mode_cmd) { struct drm_framebuffer *fb; - struct drm_device *dev = helper->dev; - struct xe_device *xe = to_xe_device(dev); - struct drm_mode_fb_cmd2 mode_cmd = {}; + struct xe_device *xe = to_xe_device(drm); struct xe_bo *obj; int size; - /* we don't do packed 24bpp */ - if (sizes->surface_bpp == 24) - sizes->surface_bpp = 32; - - mode_cmd.width = sizes->surface_width; - mode_cmd.height = sizes->surface_height; - - mode_cmd.pitches[0] = ALIGN(mode_cmd.width * - DIV_ROUND_UP(sizes->surface_bpp, 8), XE_PAGE_SIZE); - mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, - sizes->surface_depth); - - size = mode_cmd.pitches[0] * mode_cmd.height; + size = mode_cmd->pitches[0] * mode_cmd->height; size = PAGE_ALIGN(size); obj = ERR_PTR(-ENODEV); @@ -67,10 +53,10 @@ struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper, } fb = intel_framebuffer_create(&obj->ttm.base, - drm_get_format_info(dev, - mode_cmd.pixel_format, - mode_cmd.modifier[0]), - &mode_cmd); + drm_get_format_info(drm, + mode_cmd->pixel_format, + mode_cmd->modifier[0]), + mode_cmd); if (IS_ERR(fb)) { xe_bo_unpin_map_no_vm(obj); goto err; diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 1fd4a815e784b..b18d15cc3c53d 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -378,7 +378,7 @@ intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, { *out_flags = 0; - return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment); + return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment); } void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index f4c3e1187a00a..27fba92301c4e 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -96,6 +96,12 @@ #define ENABLE_SEMAPHORE_POLL_BIT REG_BIT(13) #define RING_CMD_CCTL(base) XE_REG((base) + 0xc4, XE_REG_OPTION_MASKED) + +#define CS_MMIO_GROUP_INSTANCE_SELECT(base) XE_REG((base) + 0xcc) +#define SELECTIVE_READ_ADDRESSING REG_BIT(30) +#define SELECTIVE_READ_GROUP REG_GENMASK(29, 23) +#define SELECTIVE_READ_INSTANCE REG_GENMASK(22, 16) + /* * CMD_CCTL read/write fields take a MOCS value and _not_ a table index. * The lsb of each can be considered a separate enabling bit for encryption. diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index f680c8b8f2587..7d90e4dd86d0e 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -540,6 +540,7 @@ #define ENABLE_SMP_LD_RENDER_SURFACE_CONTROL REG_BIT(44 - 32) #define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32) #define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32) +#define L3_128B_256B_WRT_DIS REG_BIT(40 - 32) #define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32) #define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32) diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h index 2118f7dec287f..87984713dd126 100644 --- a/drivers/gpu/drm/xe/regs/xe_guc_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h @@ -90,6 +90,9 @@ #define GUC_SEND_INTERRUPT XE_REG(0xc4c8) #define GUC_SEND_TRIGGER REG_BIT(0) +#define GUC_INTR_CHICKEN XE_REG(0xc50c) +#define DISABLE_SIGNALING_ENGINES REG_BIT(1) + #define GUC_BCS_RCS_IER XE_REG(0xc550) #define GUC_VCS2_VCS1_IER XE_REG(0xc554) #define GUC_WD_VECS_IER XE_REG(0xc558) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 4410e28dee54b..a270aef7c4980 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1008,6 +1008,7 @@ static long xe_bo_shrink_purge(struct ttm_operation_ctx *ctx, unsigned long *scanned) { struct xe_device *xe = ttm_to_xe_device(bo->bdev); + struct ttm_tt *tt = bo->ttm; long lret; /* Fake move to system, without copying data. */ @@ -1032,8 +1033,10 @@ static long xe_bo_shrink_purge(struct ttm_operation_ctx *ctx, .writeback = false, .allow_move = false}); - if (lret > 0) + if (lret > 0) { xe_ttm_tt_account_subtract(xe, bo->ttm); + update_global_total_pages(bo->bdev, -(long)tt->num_pages); + } return lret; } @@ -1119,8 +1122,10 @@ long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, if (needs_rpm) xe_pm_runtime_put(xe); - if (lret > 0) + if (lret > 0) { xe_ttm_tt_account_subtract(xe, tt); + update_global_total_pages(bo->bdev, -(long)tt->num_pages); + } out_unref: xe_bo_put(xe_bo); @@ -1480,7 +1485,7 @@ static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo) * always succeed here, as long as we hold the lru lock. */ spin_lock(&ttm_bo->bdev->lru_lock); - locked = dma_resv_trylock(ttm_bo->base.resv); + locked = dma_resv_trylock(&ttm_bo->base._resv); spin_unlock(&ttm_bo->bdev->lru_lock); xe_assert(xe, locked); @@ -1500,13 +1505,6 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) bo = ttm_to_xe_bo(ttm_bo); xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount))); - /* - * Corner case where TTM fails to allocate memory and this BOs resv - * still points the VMs resv - */ - if (ttm_bo->base.resv != &ttm_bo->base._resv) - return; - if (!xe_ttm_bo_lock_in_destructor(ttm_bo)) return; @@ -1516,14 +1514,14 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) * TODO: Don't do this for external bos once we scrub them after * unbind. */ - dma_resv_for_each_fence(&cursor, ttm_bo->base.resv, + dma_resv_for_each_fence(&cursor, &ttm_bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, fence) { if (xe_fence_is_xe_preempt(fence) && !dma_fence_is_signaled(fence)) { if (!replacement) replacement = dma_fence_get_stub(); - dma_resv_replace_fences(ttm_bo->base.resv, + dma_resv_replace_fences(&ttm_bo->base._resv, fence->context, replacement, DMA_RESV_USAGE_BOOKKEEP); @@ -1531,7 +1529,7 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) } dma_fence_put(replacement); - dma_resv_unlock(ttm_bo->base.resv); + dma_resv_unlock(&ttm_bo->base._resv); } static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo) @@ -1897,7 +1895,7 @@ static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf) int err = 0; int idx; - if (!drm_dev_enter(&xe->drm, &idx)) + if (xe_device_wedged(xe) || !drm_dev_enter(&xe->drm, &idx)) return ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm); diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 1396634231857..08f379cb5321f 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -258,11 +258,10 @@ static bool is_bound(struct xe_config_group_device *dev) return false; ret = pci_get_drvdata(pdev); - pci_dev_put(pdev); - if (ret) pci_dbg(pdev, "Already bound to driver\n"); + pci_dev_put(pdev); return ret; } @@ -689,6 +688,7 @@ static void xe_config_device_release(struct config_item *item) mutex_destroy(&dev->lock); + kfree(dev->config.ctx_restore_mid_bb[0].cs); kfree(dev->config.ctx_restore_post_bb[0].cs); kfree(dev); } diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index c61e0e47ed94c..08cce375ae0f3 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -19,9 +19,11 @@ void xe_configfs_check_device(struct pci_dev *pdev); bool xe_configfs_get_survivability_mode(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); -u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); -u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); #else static inline int xe_configfs_init(void) { return 0; } @@ -30,9 +32,11 @@ static inline void xe_configfs_check_device(struct pci_dev *pdev) { } static inline bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { return false; } static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } -static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } -static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } #endif diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index cd977dbd1ef63..7b48bf90cab8f 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -255,14 +255,64 @@ static ssize_t wedged_mode_show(struct file *f, char __user *ubuf, return simple_read_from_buffer(ubuf, size, pos, buf, len); } +static int __wedged_mode_set_reset_policy(struct xe_gt *gt, enum xe_wedged_mode mode) +{ + bool enable_engine_reset; + int ret; + + enable_engine_reset = (mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET); + ret = xe_guc_ads_scheduler_policy_toggle_reset(>->uc.guc.ads, + enable_engine_reset); + if (ret) + xe_gt_err(gt, "Failed to update GuC ADS scheduler policy (%pe)\n", ERR_PTR(ret)); + + return ret; +} + +static int wedged_mode_set_reset_policy(struct xe_device *xe, enum xe_wedged_mode mode) +{ + struct xe_gt *gt; + int ret; + u8 id; + + guard(xe_pm_runtime)(xe); + for_each_gt(gt, xe, id) { + ret = __wedged_mode_set_reset_policy(gt, mode); + if (ret) { + if (id > 0) { + xe->wedged.inconsistent_reset = true; + drm_err(&xe->drm, "Inconsistent reset policy state between GTs\n"); + } + return ret; + } + } + + xe->wedged.inconsistent_reset = false; + + return 0; +} + +static bool wedged_mode_needs_policy_update(struct xe_device *xe, enum xe_wedged_mode mode) +{ + if (xe->wedged.inconsistent_reset) + return true; + + if (xe->wedged.mode == mode) + return false; + + if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET || + mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) + return true; + + return false; +} + static ssize_t wedged_mode_set(struct file *f, const char __user *ubuf, size_t size, loff_t *pos) { struct xe_device *xe = file_inode(f)->i_private; - struct xe_gt *gt; u32 wedged_mode; ssize_t ret; - u8 id; ret = kstrtouint_from_user(ubuf, size, 0, &wedged_mode); if (ret) @@ -271,22 +321,14 @@ static ssize_t wedged_mode_set(struct file *f, const char __user *ubuf, if (wedged_mode > 2) return -EINVAL; - if (xe->wedged.mode == wedged_mode) - return size; + if (wedged_mode_needs_policy_update(xe, wedged_mode)) { + ret = wedged_mode_set_reset_policy(xe, wedged_mode); + if (ret) + return ret; + } xe->wedged.mode = wedged_mode; - xe_pm_runtime_get(xe); - for_each_gt(gt, xe, id) { - ret = xe_guc_ads_scheduler_policy_toggle_reset(>->uc.guc.ads); - if (ret) { - xe_gt_err(gt, "Failed to update GuC ADS scheduler policy. GuC may still cause engine reset even with wedged_mode=2\n"); - xe_pm_runtime_put(xe); - return -EIO; - } - } - xe_pm_runtime_put(xe); - return size; } diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 456899238377e..0d69cd0e4e798 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -966,6 +966,7 @@ int xe_device_probe(struct xe_device *xe) err_unregister_display: xe_display_unregister(xe); + drm_dev_unregister(&xe->drm); return err; } @@ -974,8 +975,6 @@ void xe_device_remove(struct xe_device *xe) { xe_display_unregister(xe); - xe_nvm_fini(xe); - drm_dev_unplug(&xe->drm); xe_bo_pci_dev_remove_all(xe); @@ -1046,7 +1045,7 @@ static void tdf_request_sync(struct xe_device *xe) * transient and need to be flushed.. */ if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, - 150, NULL, false)) + 300, NULL, false)) xe_gt_err_once(gt, "TD flush timeout\n"); xe_force_wake_put(gt_to_fw(gt), fw_ref); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 74d7af830b85d..0e80f2940c996 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -42,6 +42,22 @@ struct xe_pat_ops; struct xe_pxp; struct xe_vram_region; +/** + * enum xe_wedged_mode - possible wedged modes + * @XE_WEDGED_MODE_NEVER: Device will never be declared wedged. + * @XE_WEDGED_MODE_UPON_CRITICAL_ERROR: Device will be declared wedged only + * when critical error occurs like GT reset failure or firmware failure. + * This is the default mode. + * @XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: Device will be declared wedged on + * any hang. In this mode, engine resets are disabled to avoid automatic + * recovery attempts. This mode is primarily intended for debugging hangs. + */ +enum xe_wedged_mode { + XE_WEDGED_MODE_NEVER = 0, + XE_WEDGED_MODE_UPON_CRITICAL_ERROR = 1, + XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET = 2, +}; + #define XE_BO_INVALID_OFFSET LONG_MAX #define GRAPHICS_VER(xe) ((xe)->info.graphics_verx100 / 100) @@ -556,6 +572,8 @@ struct xe_device { int mode; /** @wedged.method: Recovery method to be sent in the drm device wedged uevent */ unsigned long method; + /** @wedged.inconsistent_reset: Inconsistent reset policy state between GTs */ + bool inconsistent_reset; } wedged; /** @bo_device: Struct to control async free of BOs */ diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index a7d67725c3ee1..2650c5abb3654 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -113,7 +113,7 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, case XE_PL_TT: sgt = drm_prime_pages_to_sg(obj->dev, bo->ttm.ttm->pages, - bo->ttm.ttm->num_pages); + obj->size >> PAGE_SHIFT); if (IS_ERR(sgt)) return sgt; diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index f5cfdf29fde34..33a3764e3e71b 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -290,7 +290,7 @@ static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension return -EFAULT; if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) || - XE_IOCTL_DBG(xe, ext.pad)) + XE_IOCTL_DBG(xe, !ext.property) || XE_IOCTL_DBG(xe, ext.pad)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs)); diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index a8ab363a80465..87a32b61bece6 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -130,7 +130,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) || + XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) return -EINVAL; q = xe_exec_queue_lookup(xef, args->exec_queue_id); @@ -181,9 +182,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto err_syncs; } - if (xe_exec_queue_is_parallel(q)) { - err = copy_from_user(addresses, addresses_user, sizeof(u64) * - q->width); + if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) { + err = copy_from_user(addresses, addresses_user, + sizeof(u64) * q->width); if (err) { err = -EFAULT; goto err_syncs; diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index cb5f204c08ed6..231d1fbe5eefa 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -284,6 +284,7 @@ struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe * @xe: Xe device. * @tile: tile which bind exec queue belongs to. * @flags: exec queue creation flags + * @user_vm: The user VM which this exec queue belongs to * @extensions: exec queue creation extensions * * Normalize bind exec queue creation. Bind exec queue is tied to migration VM @@ -297,6 +298,7 @@ struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe */ struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *user_vm, u32 flags, u64 extensions) { struct xe_gt *gt = tile->primary_gt; @@ -333,6 +335,9 @@ struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, xe_exec_queue_put(q); return ERR_PTR(err); } + + if (user_vm) + q->user_vm = xe_vm_get(user_vm); } return q; @@ -357,6 +362,11 @@ void xe_exec_queue_destroy(struct kref *ref) xe_exec_queue_put(eq); } + if (q->user_vm) { + xe_vm_put(q->user_vm); + q->user_vm = NULL; + } + q->ops->destroy(q); } @@ -692,6 +702,22 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) return -EINVAL; + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -ENOENT; + + err = down_read_interruptible(&vm->lock); + if (err) { + xe_vm_put(vm); + return err; + } + + if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { + up_read(&vm->lock); + xe_vm_put(vm); + return -ENOENT; + } + for_each_tile(tile, xe, id) { struct xe_exec_queue *new; @@ -699,9 +725,11 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, if (id) flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; - new = xe_exec_queue_create_bind(xe, tile, flags, + new = xe_exec_queue_create_bind(xe, tile, vm, flags, args->extensions); if (IS_ERR(new)) { + up_read(&vm->lock); + xe_vm_put(vm); err = PTR_ERR(new); if (q) goto put_exec_queue; @@ -713,6 +741,8 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, list_add_tail(&new->multi_gt_list, &q->multi_gt_link); } + up_read(&vm->lock); + xe_vm_put(vm); } else { logical_mask = calc_validate_logical_mask(xe, eci, args->width, diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 15ec852e7f7e7..5343c1b8cab54 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -24,6 +24,7 @@ struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe u32 flags, u64 extensions); struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *user_vm, u32 flags, u64 extensions); void xe_exec_queue_fini(struct xe_exec_queue *q); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index df1c69dc81f17..38906cb7608ca 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -54,6 +54,12 @@ struct xe_exec_queue { struct kref refcount; /** @vm: VM (address space) for this exec queue */ struct xe_vm *vm; + /** + * @user_vm: User VM (address space) for this exec queue (bind queues + * only) + */ + struct xe_vm *user_vm; + /** @class: class of this exec queue */ enum xe_engine_class class; /** diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index 5edc0cad47e20..8f722dd842da8 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -243,6 +243,8 @@ static void dev_fini_ggtt(void *arg) { struct xe_ggtt *ggtt = arg; + scoped_guard(mutex, &ggtt->lock) + ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE; drain_workqueue(ggtt->wq); } @@ -291,7 +293,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt) else ggtt->pt_ops = &xelp_pt_ops; - ggtt->wq = alloc_workqueue("xe-ggtt-wq", 0, WQ_MEM_RECLAIM); + ggtt->wq = alloc_workqueue("xe-ggtt-wq", WQ_MEM_RECLAIM, 0); if (!ggtt->wq) return -ENOMEM; @@ -301,6 +303,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt) if (err) return err; + ggtt->flags |= XE_GGTT_FLAGS_ONLINE; err = devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt); if (err) return err; @@ -334,13 +337,10 @@ static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) static void ggtt_node_remove(struct xe_ggtt_node *node) { struct xe_ggtt *ggtt = node->ggtt; - struct xe_device *xe = tile_to_xe(ggtt->tile); bool bound; - int idx; - - bound = drm_dev_enter(&xe->drm, &idx); mutex_lock(&ggtt->lock); + bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE; if (bound) xe_ggtt_clear(ggtt, node->base.start, node->base.size); drm_mm_remove_node(&node->base); @@ -353,8 +353,6 @@ static void ggtt_node_remove(struct xe_ggtt_node *node) if (node->invalidate_on_remove) xe_ggtt_invalidate(ggtt); - drm_dev_exit(idx); - free_node: xe_ggtt_node_fini(node); } @@ -365,9 +363,8 @@ static void ggtt_node_remove_work_func(struct work_struct *work) delayed_removal_work); struct xe_device *xe = tile_to_xe(node->ggtt->tile); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ggtt_node_remove(node); - xe_pm_runtime_put(xe); } /** diff --git a/drivers/gpu/drm/xe/xe_ggtt_types.h b/drivers/gpu/drm/xe/xe_ggtt_types.h index c5e999d58ff2a..445141d64701e 100644 --- a/drivers/gpu/drm/xe/xe_ggtt_types.h +++ b/drivers/gpu/drm/xe/xe_ggtt_types.h @@ -25,11 +25,14 @@ struct xe_ggtt { /** @size: Total size of this GGTT */ u64 size; -#define XE_GGTT_FLAGS_64K BIT(0) +#define XE_GGTT_FLAGS_64K BIT(0) +#define XE_GGTT_FLAGS_ONLINE BIT(1) /** * @flags: Flags for this GGTT * Acceptable flags: * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K. + * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock + * after init */ unsigned int flags; /** @scratch: Internal object allocation used as a scratch page */ diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.c b/drivers/gpu/drm/xe/xe_gsc_proxy.c index 464282a89eef3..a6f6f0ea56526 100644 --- a/drivers/gpu/drm/xe/xe_gsc_proxy.c +++ b/drivers/gpu/drm/xe/xe_gsc_proxy.c @@ -435,16 +435,12 @@ static int proxy_channel_alloc(struct xe_gsc *gsc) return 0; } -static void xe_gsc_proxy_remove(void *arg) +static void xe_gsc_proxy_stop(struct xe_gsc *gsc) { - struct xe_gsc *gsc = arg; struct xe_gt *gt = gsc_to_gt(gsc); struct xe_device *xe = gt_to_xe(gt); unsigned int fw_ref = 0; - if (!gsc->proxy.component_added) - return; - /* disable HECI2 IRQs */ xe_pm_runtime_get(xe); fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); @@ -458,6 +454,30 @@ static void xe_gsc_proxy_remove(void *arg) xe_pm_runtime_put(xe); xe_gsc_wait_for_worker_completion(gsc); + gsc->proxy.started = false; +} + +static void xe_gsc_proxy_remove(void *arg) +{ + struct xe_gsc *gsc = arg; + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + + if (!gsc->proxy.component_added) + return; + + /* + * GSC proxy start is an async process that can be ongoing during + * Xe module load/unload. Using devm managed action to register + * xe_gsc_proxy_stop could cause issues if Xe module unload has + * already started when the action is registered, potentially leading + * to the cleanup being called at the wrong time. Therefore, instead + * of registering a separate devm action to undo what is done in + * proxy start, we call it from here, but only if the start has + * completed successfully (tracked with the 'started' flag). + */ + if (gsc->proxy.started) + xe_gsc_proxy_stop(gsc); component_del(xe->drm.dev, &xe_gsc_proxy_component_ops); gsc->proxy.component_added = false; @@ -513,6 +533,7 @@ int xe_gsc_proxy_init(struct xe_gsc *gsc) */ int xe_gsc_proxy_start(struct xe_gsc *gsc) { + struct xe_gt *gt = gsc_to_gt(gsc); int err; /* enable the proxy interrupt in the GSC shim layer */ @@ -524,12 +545,18 @@ int xe_gsc_proxy_start(struct xe_gsc *gsc) */ err = xe_gsc_proxy_request_handler(gsc); if (err) - return err; + goto err_irq_disable; if (!xe_gsc_proxy_init_done(gsc)) { - xe_gt_err(gsc_to_gt(gsc), "GSC FW reports proxy init not completed\n"); - return -EIO; + xe_gt_err(gt, "GSC FW reports proxy init not completed\n"); + err = -EIO; + goto err_irq_disable; } + gsc->proxy.started = true; return 0; + +err_irq_disable: + gsc_proxy_irq_toggle(gsc, false); + return err; } diff --git a/drivers/gpu/drm/xe/xe_gsc_types.h b/drivers/gpu/drm/xe/xe_gsc_types.h index 97c056656df05..5aaa2a75861fd 100644 --- a/drivers/gpu/drm/xe/xe_gsc_types.h +++ b/drivers/gpu/drm/xe/xe_gsc_types.h @@ -58,6 +58,8 @@ struct xe_gsc { struct mutex mutex; /** @proxy.component_added: whether the component has been added */ bool component_added; + /** @proxy.started: whether the proxy has been started */ + bool started; /** @proxy.bo: object to store message to and from the GSC */ struct xe_bo *bo; /** @proxy.to_gsc: map of the memory used to send messages to the GSC */ diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 6d3db5e55d98a..1099335298471 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -187,11 +187,15 @@ static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) return ret; } +/* Dwords required to emit a RMW of a register */ +#define EMIT_RMW_DW 20 + static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) { - struct xe_reg_sr *sr = &q->hwe->reg_lrc; + struct xe_hw_engine *hwe = q->hwe; + struct xe_reg_sr *sr = &hwe->reg_lrc; struct xe_reg_sr_entry *entry; - int count_rmw = 0, count = 0, ret; + int count_rmw = 0, count_rmw_mcr = 0, count = 0, ret; unsigned long idx; struct xe_bb *bb; size_t bb_len = 0; @@ -201,6 +205,8 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) xa_for_each(&sr->xa, idx, entry) { if (entry->reg.masked || entry->clr_bits == ~0) ++count; + else if (entry->reg.mcr) + ++count_rmw_mcr; else ++count_rmw; } @@ -208,17 +214,35 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) if (count) bb_len += count * 2 + 1; - if (count_rmw) - bb_len += count_rmw * 20 + 7; + /* + * RMW of MCR registers is the same as a normal RMW, except an + * additional LRI (3 dwords) is required per register to steer the read + * to a nom-terminated instance. + * + * We could probably shorten the batch slightly by eliding the + * steering for consecutive MCR registers that have the same + * group/instance target, but it's not worth the extra complexity to do + * so. + */ + bb_len += count_rmw * EMIT_RMW_DW; + bb_len += count_rmw_mcr * (EMIT_RMW_DW + 3); + + /* + * After doing all RMW, we need 7 trailing dwords to clean up, + * plus an additional 3 dwords to reset steering if any of the + * registers were MCR. + */ + if (count_rmw || count_rmw_mcr) + bb_len += 7 + (count_rmw_mcr ? 3 : 0); - if (q->hwe->class == XE_ENGINE_CLASS_RENDER) + if (hwe->class == XE_ENGINE_CLASS_RENDER) /* * Big enough to emit all of the context's 3DSTATE via * xe_lrc_emit_hwe_state_instructions() */ - bb_len += xe_gt_lrc_size(gt, q->hwe->class) / sizeof(u32); + bb_len += xe_gt_lrc_size(gt, hwe->class) / sizeof(u32); - xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", q->hwe->name, bb_len); + xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", hwe->name, bb_len); bb = xe_bb_new(gt, bb_len, false); if (IS_ERR(bb)) @@ -253,13 +277,23 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) } } - if (count_rmw) { - /* Emit MI_MATH for each RMW reg: 20dw per reg + 7 trailing dw */ - + if (count_rmw || count_rmw_mcr) { xa_for_each(&sr->xa, idx, entry) { if (entry->reg.masked || entry->clr_bits == ~0) continue; + if (entry->reg.mcr) { + struct xe_reg_mcr reg = { .__reg.raw = entry->reg.raw }; + u8 group, instance; + + xe_gt_mcr_get_nonterminated_steering(gt, reg, &group, &instance); + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(hwe->mmio_base).addr; + *cs++ = SELECTIVE_READ_ADDRESSING | + REG_FIELD_PREP(SELECTIVE_READ_GROUP, group) | + REG_FIELD_PREP(SELECTIVE_READ_INSTANCE, instance); + } + *cs++ = MI_LOAD_REGISTER_REG | MI_LRR_DST_CS_MMIO; *cs++ = entry->reg.addr; *cs++ = CS_GPR_REG(0, 0).addr; @@ -285,8 +319,9 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) *cs++ = CS_GPR_REG(0, 0).addr; *cs++ = entry->reg.addr; - xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x\n", - entry->reg.addr, entry->clr_bits, entry->set_bits); + xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x%s\n", + entry->reg.addr, entry->clr_bits, entry->set_bits, + entry->reg.mcr ? " (MCR)" : ""); } /* reset used GPR */ @@ -298,6 +333,13 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) *cs++ = 0; *cs++ = CS_GPR_REG(0, 2).addr; *cs++ = 0; + + /* reset steering */ + if (count_rmw_mcr) { + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(q->hwe->mmio_base).addr; + *cs++ = 0; + } } cs = xe_lrc_emit_hwe_state_instructions(q, cs); @@ -784,9 +826,6 @@ static int do_gt_restart(struct xe_gt *gt) xe_gt_sriov_pf_init_hw(gt); xe_mocs_init(gt); - err = xe_uc_start(>->uc); - if (err) - return err; for_each_hw_engine(hwe, gt, id) xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); @@ -794,6 +833,10 @@ static int do_gt_restart(struct xe_gt *gt) /* Get CCS mode in sync between sw/hw */ xe_gt_apply_ccs_mode(gt); + err = xe_uc_start(>->uc); + if (err) + return err; + /* Restore GT freq to expected values */ xe_gt_sanitize_freq(gt); diff --git a/drivers/gpu/drm/xe/xe_gt_freq.c b/drivers/gpu/drm/xe/xe_gt_freq.c index 4ff1b6b58d6be..e8e70fd2e8c42 100644 --- a/drivers/gpu/drm/xe/xe_gt_freq.c +++ b/drivers/gpu/drm/xe/xe_gt_freq.c @@ -296,8 +296,10 @@ int xe_gt_freq_init(struct xe_gt *gt) return -ENOMEM; err = sysfs_create_files(gt->freq, freq_attrs); - if (err) + if (err) { + kobject_put(gt->freq); return err; + } err = devm_add_action_or_reset(xe->drm.dev, freq_fini, gt->freq); if (err) diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index bdc9d9877ec49..3e3d1d52f6302 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -5,6 +5,7 @@ #include +#include #include "xe_force_wake.h" #include "xe_device.h" #include "xe_gt.h" @@ -16,6 +17,7 @@ #include "xe_mmio.h" #include "xe_pm.h" #include "xe_sriov.h" +#include "xe_wa.h" /** * DOC: Xe GT Idle @@ -145,6 +147,12 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25); } + if (XE_GT_WA(gt, 14020316580)) + gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) | + VDN_MFXVDENC_POWERGATE_ENABLE(0) | + VDN_HCP_POWERGATE_ENABLE(2) | + VDN_MFXVDENC_POWERGATE_ENABLE(2)); + xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); xe_force_wake_put(gt_to_fw(gt), fw_ref); } diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index 00789844ea4d0..ae0c88da422b2 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -1632,7 +1632,7 @@ int xe_guc_start(struct xe_guc *guc) return xe_guc_submit_start(guc); } -void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) +int xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) { struct xe_gt *gt = guc_to_gt(guc); unsigned int fw_ref; @@ -1644,7 +1644,7 @@ void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) if (!IS_SRIOV_VF(gt_to_xe(gt))) { fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); if (!fw_ref) - return; + return -EIO; status = xe_mmio_read32(>->mmio, GUC_STATUS); @@ -1672,6 +1672,8 @@ void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) drm_puts(p, "\n"); xe_guc_submit_print(guc, p); + + return 0; } /** diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h index 1cca05967e621..3b858933749bd 100644 --- a/drivers/gpu/drm/xe/xe_guc.h +++ b/drivers/gpu/drm/xe/xe_guc.h @@ -45,7 +45,7 @@ int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val); int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val); void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir); void xe_guc_sanitize(struct xe_guc *guc); -void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p); +int xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p); int xe_guc_reset_prepare(struct xe_guc *guc); void xe_guc_reset_wait(struct xe_guc *guc); void xe_guc_stop_prepare(struct xe_guc *guc); diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 58e0b0294a5bc..0e2bece1d8b83 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -979,16 +979,17 @@ static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_off /** * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy * @ads: Additional data structures object + * @enable_engine_reset: true to enable engine resets, false otherwise * - * This function update the GuC's engine reset policy based on wedged.mode. + * This function update the GuC's engine reset policy. * * Return: 0 on success, and negative error code otherwise. */ -int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads) +int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads, + bool enable_engine_reset) { struct guc_policies *policies; struct xe_guc *guc = ads_to_guc(ads); - struct xe_device *xe = ads_to_xe(ads); CLASS(xe_guc_buf, buf)(&guc->buf, sizeof(*policies)); if (!xe_guc_buf_is_valid(buf)) @@ -1000,10 +1001,11 @@ int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads) policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time); policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items); policies->is_valid = 1; - if (xe->wedged.mode == 2) - policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; - else + + if (enable_engine_reset) policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET; + else + policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; return guc_ads_action_update_policies(ads, xe_guc_buf_flush(buf)); } diff --git a/drivers/gpu/drm/xe/xe_guc_ads.h b/drivers/gpu/drm/xe/xe_guc_ads.h index 2e6674c760ff9..7a39f361cb17d 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.h +++ b/drivers/gpu/drm/xe/xe_guc_ads.h @@ -6,6 +6,8 @@ #ifndef _XE_GUC_ADS_H_ #define _XE_GUC_ADS_H_ +#include + struct xe_guc_ads; int xe_guc_ads_init(struct xe_guc_ads *ads); @@ -13,6 +15,7 @@ int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads); void xe_guc_ads_populate(struct xe_guc_ads *ads); void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads); void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads); -int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads); +int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads, + bool enable_engine_reset); #endif diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index b7afe8e983cb7..4158e8af13908 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -106,7 +106,9 @@ static void g2h_fence_cancel(struct g2h_fence *g2h_fence) { g2h_fence->cancel = true; g2h_fence->fail = true; - g2h_fence->done = true; + + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ + WRITE_ONCE(g2h_fence->done, true); } static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) @@ -264,6 +266,7 @@ static void guc_action_disable_ct(void *arg) { struct xe_guc_ct *ct = arg; + xe_guc_ct_stop(ct); guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); } @@ -1128,10 +1131,13 @@ static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, return ret; } - ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); + /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response + * and g2h_fence_cancel. + */ + ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); if (!ret) { LNL_FLUSH_WORK(&ct->g2h_worker); - if (g2h_fence.done) { + if (READ_ONCE(g2h_fence.done)) { xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", g2h_fence.seqno, action[0]); ret = 1; @@ -1375,7 +1381,8 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); - g2h_fence->done = true; + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ + WRITE_ONCE(g2h_fence->done, true); smp_mb(); wake_up_all(&ct->g2h_fence_wq); diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 53fdf59524c41..a3e9796e6430f 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -1231,6 +1231,36 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf) return ret; } +static int pc_action_set_dcc(struct xe_guc_pc *pc, bool enable) +{ + int ret; + + ret = pc_action_set_param(pc, + SLPC_PARAM_TASK_ENABLE_DCC, + enable); + if (!ret) + return pc_action_set_param(pc, + SLPC_PARAM_TASK_DISABLE_DCC, + !enable); + else + return ret; +} + +static int pc_modify_defaults(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + int ret = 0; + + if (xe->info.platform == XE_PANTHERLAKE) { + ret = pc_action_set_dcc(pc, false); + if (unlikely(ret)) + xe_gt_err(gt, "Failed to modify DCC default: %pe\n", ERR_PTR(ret)); + } + + return ret; +} + /** * xe_guc_pc_start - Start GuC's Power Conservation component * @pc: Xe_GuC_PC instance @@ -1288,6 +1318,10 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) ktime_ms_delta(ktime_get(), earlier)); } + ret = pc_modify_defaults(pc); + if (ret) + return ret; + ret = pc_init_freqs(pc); if (ret) goto out; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 94ed8159496f1..ecee50d827108 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -670,6 +670,24 @@ static u32 wq_space_until_wrap(struct xe_exec_queue *q) return (WQ_SIZE - q->guc->wqi_tail); } +static inline void relaxed_ms_sleep(unsigned int delay_ms) +{ + unsigned long min_us, max_us; + + if (!delay_ms) + return; + + if (delay_ms > 20) { + msleep(delay_ms); + return; + } + + min_us = mul_u32_u32(delay_ms, 1000); + max_us = min_us + 500; + + usleep_range(min_us, max_us); +} + static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) { struct xe_guc *guc = exec_queue_to_guc(q); @@ -1280,7 +1298,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (exec_queue_reset(q)) err = -EIO; - if (!exec_queue_destroyed(q)) { + if (!exec_queue_destroyed(q) && xe_uc_fw_is_running(&guc->fw)) { /* * Wait for any pending G2H to flush out before * modifying state @@ -1312,6 +1330,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) */ smp_rmb(); ret = wait_event_timeout(guc->ct.wq, + !xe_uc_fw_is_running(&guc->fw) || !exec_queue_pending_disable(q) || xe_guc_read_stopped(guc), HZ * 5); if (!ret || xe_guc_read_stopped(guc)) { @@ -1559,7 +1578,7 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) since_resume_ms; if (wait_ms > 0 && q->guc->resume_time) - msleep(wait_ms); + relaxed_ms_sleep(wait_ms); set_exec_queue_suspended(q); disable_scheduling(q, false); diff --git a/drivers/gpu/drm/xe/xe_heci_gsc.c b/drivers/gpu/drm/xe/xe_heci_gsc.c index a415ca4887914..32d509b113915 100644 --- a/drivers/gpu/drm/xe/xe_heci_gsc.c +++ b/drivers/gpu/drm/xe/xe_heci_gsc.c @@ -221,7 +221,7 @@ void xe_heci_gsc_irq_handler(struct xe_device *xe, u32 iir) if (xe->heci_gsc.irq < 0) return; - ret = generic_handle_irq(xe->heci_gsc.irq); + ret = generic_handle_irq_safe(xe->heci_gsc.irq); if (ret) drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret); } @@ -241,7 +241,7 @@ void xe_heci_csc_irq_handler(struct xe_device *xe, u32 iir) if (xe->heci_gsc.irq < 0) return; - ret = generic_handle_irq(xe->heci_gsc.irq); + ret = generic_handle_irq_safe(xe->heci_gsc.irq); if (ret) drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret); } diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 1cf623b4a5bcc..d8f16e25b817d 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -587,9 +587,8 @@ static void adjust_idledly(struct xe_hw_engine *hwe) maxcnt *= maxcnt_units_ns; if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) { - idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * maxcnt_units_ns), + idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000), idledly_units_ps); - idledly = DIV_ROUND_CLOSEST(idledly, 1000); xe_mmio_write32(>->mmio, RING_IDLEDLY(hwe->mmio_base), idledly); } } diff --git a/drivers/gpu/drm/xe/xe_late_bind_fw_types.h b/drivers/gpu/drm/xe/xe_late_bind_fw_types.h index 0f5da89ce98b8..2a8a985c37e71 100644 --- a/drivers/gpu/drm/xe/xe_late_bind_fw_types.h +++ b/drivers/gpu/drm/xe/xe_late_bind_fw_types.h @@ -15,10 +15,12 @@ #define XE_LB_MAX_PAYLOAD_SIZE SZ_4K /** - * xe_late_bind_fw_id - enum to determine late binding fw index + * enum xe_late_bind_fw_id - enum to determine late binding fw index */ enum xe_late_bind_fw_id { + /** @XE_LB_FW_FAN_CONTROL: Fan control */ XE_LB_FW_FAN_CONTROL = 0, + /** @XE_LB_FW_MAX_ID: Number of IDs */ XE_LB_FW_MAX_ID }; diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 47e9df7750725..9f29dbe890f66 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1050,6 +1050,9 @@ static ssize_t setup_utilization_wa(struct xe_lrc *lrc, { u32 *cmd = batch; + if (IS_SRIOV_VF(gt_to_xe(lrc->gt))) + return 0; + if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) return -ENOSPC; @@ -1182,7 +1185,7 @@ static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, return -ENOSPC; *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); - *cmd++ = CS_DEBUG_MODE1(0).addr; + *cmd++ = CS_DEBUG_MODE2(0).addr; *cmd++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); return cmd - batch; diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index a36ce7dce8cc0..b6905f35d6c81 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -1813,6 +1813,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, unsigned long sram_offset, struct drm_pagemap_addr *sram_addr, u64 vram_addr, + struct dma_fence *deps, const enum xe_migrate_copy_dir dir) { struct xe_gt *gt = m->tile->primary_gt; @@ -1890,6 +1891,14 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); + if (deps && !dma_fence_is_signaled(deps)) { + dma_fence_get(deps); + err = drm_sched_job_add_dependency(&job->drm, deps); + if (err) + dma_fence_wait(deps, false); + err = 0; + } + mutex_lock(&m->job_mutex); xe_sched_job_arm(job); fence = dma_fence_get(&job->drm.s_fence->finished); @@ -1915,6 +1924,8 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, * @npages: Number of pages to migrate. * @src_addr: Array of DMA information (source of migrate) * @dst_addr: Device physical address of VRAM (destination of migrate) + * @deps: struct dma_fence representing the dependencies that need + * to be signaled before migration. * * Copy from an array dma addresses to a VRAM device physical address * @@ -1924,10 +1935,11 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, unsigned long npages, struct drm_pagemap_addr *src_addr, - u64 dst_addr) + u64 dst_addr, + struct dma_fence *deps) { return xe_migrate_vram(m, npages * PAGE_SIZE, 0, src_addr, dst_addr, - XE_MIGRATE_COPY_TO_VRAM); + deps, XE_MIGRATE_COPY_TO_VRAM); } /** @@ -1936,6 +1948,8 @@ struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, * @npages: Number of pages to migrate. * @src_addr: Device physical address of VRAM (source of migrate) * @dst_addr: Array of DMA information (destination of migrate) + * @deps: struct dma_fence representing the dependencies that need + * to be signaled before migration. * * Copy from a VRAM device physical address to an array dma addresses * @@ -1945,10 +1959,11 @@ struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, unsigned long npages, u64 src_addr, - struct drm_pagemap_addr *dst_addr) + struct drm_pagemap_addr *dst_addr, + struct dma_fence *deps) { return xe_migrate_vram(m, npages * PAGE_SIZE, 0, dst_addr, src_addr, - XE_MIGRATE_COPY_TO_SRAM); + deps, XE_MIGRATE_COPY_TO_SRAM); } static void xe_migrate_dma_unmap(struct xe_device *xe, @@ -2121,7 +2136,7 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, __fence = xe_migrate_vram(m, current_bytes, (unsigned long)buf & ~PAGE_MASK, &pagemap_addr[current_page], - vram_addr, write ? + vram_addr, NULL, write ? XE_MIGRATE_COPY_TO_VRAM : XE_MIGRATE_COPY_TO_SRAM); if (IS_ERR(__fence)) { @@ -2167,7 +2182,7 @@ void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q) if (is_migrate) mutex_lock(&m->job_mutex); else - xe_vm_assert_held(q->vm); /* User queues VM's should be locked */ + xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ } /** @@ -2185,7 +2200,7 @@ void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q) if (is_migrate) mutex_unlock(&m->job_mutex); else - xe_vm_assert_held(q->vm); /* User queues VM's should be locked */ + xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ } #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 4fad324b62535..bc55f650204b8 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -111,12 +111,14 @@ int xe_migrate_init(struct xe_migrate *m); struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, unsigned long npages, struct drm_pagemap_addr *src_addr, - u64 dst_addr); + u64 dst_addr, + struct dma_fence *deps); struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, unsigned long npages, u64 src_addr, - struct drm_pagemap_addr *dst_addr); + struct drm_pagemap_addr *dst_addr, + struct dma_fence *deps); struct dma_fence *xe_migrate_copy(struct xe_migrate *m, struct xe_bo *src_bo, diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index ef6f3ea573a2c..6752881af093a 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -260,11 +260,11 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; u32 ldw, udw, oldudw, retries; - reg.addr = xe_mmio_adjusted_addr(mmio, reg.addr); - reg_udw.addr = xe_mmio_adjusted_addr(mmio, reg_udw.addr); - - /* we shouldn't adjust just one register address */ - xe_tile_assert(mmio->tile, reg_udw.addr == reg.addr + 0x4); + /* + * The two dwords of a 64-bit register can never straddle the offset + * adjustment cutoff. + */ + xe_tile_assert(mmio->tile, !in_range(mmio->adj_limit, reg.addr + 1, 7)); oldudw = xe_mmio_read32(mmio, reg_udw); for (retries = 5; retries; --retries) { diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h index 5a3bfea8b7b4c..b668495392701 100644 --- a/drivers/gpu/drm/xe/xe_module.h +++ b/drivers/gpu/drm/xe/xe_module.h @@ -12,7 +12,7 @@ struct xe_modparam { bool force_execlist; bool probe_display; - u32 force_vram_bar_size; + int force_vram_bar_size; int guc_log_level; char *guc_firmware_path; char *huc_firmware_path; diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c index 33f4ac82fc80a..6da42b2b5e467 100644 --- a/drivers/gpu/drm/xe/xe_nvm.c +++ b/drivers/gpu/drm/xe/xe_nvm.c @@ -83,6 +83,27 @@ static bool xe_nvm_writable_override(struct xe_device *xe) return writable_override; } +static void xe_nvm_fini(void *arg) +{ + struct xe_device *xe = arg; + struct intel_dg_nvm_dev *nvm = xe->nvm; + + if (!xe->info.has_gsc_nvm) + return; + + /* No access to internal NVM from VFs */ + if (IS_SRIOV_VF(xe)) + return; + + /* Nvm pointer should not be NULL here */ + if (WARN_ON(!nvm)) + return; + + auxiliary_device_delete(&nvm->aux_dev); + auxiliary_device_uninit(&nvm->aux_dev); + xe->nvm = NULL; +} + int xe_nvm_init(struct xe_device *xe) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); @@ -132,39 +153,17 @@ int xe_nvm_init(struct xe_device *xe) ret = auxiliary_device_init(aux_dev); if (ret) { drm_err(&xe->drm, "xe-nvm aux init failed %d\n", ret); - goto err; + kfree(nvm); + xe->nvm = NULL; + return ret; } ret = auxiliary_device_add(aux_dev); if (ret) { drm_err(&xe->drm, "xe-nvm aux add failed %d\n", ret); auxiliary_device_uninit(aux_dev); - goto err; + xe->nvm = NULL; + return ret; } - return 0; - -err: - kfree(nvm); - xe->nvm = NULL; - return ret; -} - -void xe_nvm_fini(struct xe_device *xe) -{ - struct intel_dg_nvm_dev *nvm = xe->nvm; - - if (!xe->info.has_gsc_nvm) - return; - - /* No access to internal NVM from VFs */ - if (IS_SRIOV_VF(xe)) - return; - - /* Nvm pointer should not be NULL here */ - if (WARN_ON(!nvm)) - return; - - auxiliary_device_delete(&nvm->aux_dev); - auxiliary_device_uninit(&nvm->aux_dev); - xe->nvm = NULL; + return devm_add_action_or_reset(xe->drm.dev, xe_nvm_fini, xe); } diff --git a/drivers/gpu/drm/xe/xe_nvm.h b/drivers/gpu/drm/xe/xe_nvm.h index 7f3d5f57bed08..fd3467ad35a4d 100644 --- a/drivers/gpu/drm/xe/xe_nvm.h +++ b/drivers/gpu/drm/xe/xe_nvm.h @@ -10,6 +10,4 @@ struct xe_device; int xe_nvm_init(struct xe_device *xe); -void xe_nvm_fini(struct xe_device *xe); - #endif diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 125698a9ecf1c..98bfb127eafca 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -543,8 +543,7 @@ static ssize_t xe_oa_read(struct file *file, char __user *buf, size_t offset = 0; int ret; - /* Can't read from disabled streams */ - if (!stream->enabled || !stream->sample) + if (!stream->sample) return -EINVAL; if (!(file->f_flags & O_NONBLOCK)) { @@ -1104,11 +1103,12 @@ static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) oag_buf_size_select(stream) | oag_configure_mmio_trigger(stream, true)); - xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ? - (OAG_OAGLBCTXCTRL_COUNTER_RESUME | + xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, + OAG_OAGLBCTXCTRL_COUNTER_RESUME | + (stream->periodic ? OAG_OAGLBCTXCTRL_TIMER_ENABLE | REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK, - stream->period_exponent)) : 0); + stream->period_exponent) : 0)); /* * Initialize Super Queue Internal Cnt Register @@ -1253,6 +1253,9 @@ static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value, static int xe_oa_set_prop_num_syncs(struct xe_oa *oa, u64 value, struct xe_oa_open_param *param) { + if (XE_IOCTL_DBG(oa->xe, value > DRM_XE_MAX_SYNCS)) + return -EINVAL; + param->num_syncs = value; return 0; } @@ -1342,7 +1345,7 @@ static int xe_oa_user_ext_set_property(struct xe_oa *oa, enum xe_oa_user_extn_fr ARRAY_SIZE(xe_oa_set_property_funcs_config)); if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) || - XE_IOCTL_DBG(oa->xe, ext.pad)) + XE_IOCTL_DBG(oa->xe, !ext.property) || XE_IOCTL_DBG(oa->xe, ext.pad)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open)); @@ -1455,6 +1458,10 @@ static void xe_oa_stream_disable(struct xe_oa_stream *stream) if (stream->sample) hrtimer_cancel(&stream->poll_check_timer); + + /* Update stream->oa_buffer.tail to allow any final reports to be read */ + if (xe_oa_buffer_check_unlocked(stream)) + wake_up(&stream->poll_wq); } static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) @@ -2403,11 +2410,13 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi goto sysfs_err; } - mutex_unlock(&oa->metrics_lock); + id = oa_config->id; - drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, oa_config->id); + drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, id); + + mutex_unlock(&oa->metrics_lock); - return oa_config->id; + return id; sysfs_err: mutex_unlock(&oa->metrics_lock); diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 2c5a44377994b..a74e800846ffa 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -300,9 +301,15 @@ ALLOW_ERROR_INJECTION(xe_pm_init_early, ERRNO); /* See xe_pci_probe() */ static u32 vram_threshold_value(struct xe_device *xe) { - /* FIXME: D3Cold temporarily disabled by default on BMG */ - if (xe->info.platform == XE_BATTLEMAGE) - return 0; + if (xe->info.platform == XE_BATTLEMAGE) { + const char *product_name; + + product_name = dmi_get_system_info(DMI_PRODUCT_NAME); + if (product_name && strstr(product_name, "NUC13RNG")) { + drm_warn(&xe->drm, "BMG + D3Cold not supported on this platform\n"); + return 0; + } + } return DEFAULT_VRAM_THRESHOLD; } @@ -660,6 +667,13 @@ static void xe_pm_runtime_lockdep_prime(void) /** * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously * @xe: xe device instance + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime)) is + * be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which + * can break scope-based handling, or when the lifetime of the runtime PM + * reference does not match a specific scope (e.g., runtime PM obtained in one + * function and released in a different one). */ void xe_pm_runtime_get(struct xe_device *xe) { @@ -692,6 +706,13 @@ void xe_pm_runtime_put(struct xe_device *xe) * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl * @xe: xe device instance * + * When possible, scope-based runtime PM (through + * ACQUIRE(xe_pm_runtime_ioctl, ...)) is be preferred over direct usage of this + * function. Manual get/put handling should only be used when the function + * contains goto-based logic which can break scope-based handling, or when the + * lifetime of the runtime PM reference does not match a specific scope (e.g., + * runtime PM obtained in one function and released in a different one). + * * Returns: Any number greater than or equal to 0 for success, negative error * code otherwise. */ @@ -761,6 +782,13 @@ static bool xe_pm_suspending_or_resuming(struct xe_device *xe) * It will warn if not protected. * The reference should be put back after this function regardless, since it * will always bump the usage counter, regardless. + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime_noresume)) + * is be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which can + * break scope-based handling, or when the lifetime of the runtime PM reference + * does not match a specific scope (e.g., runtime PM obtained in one function + * and released in a different one). */ void xe_pm_runtime_get_noresume(struct xe_device *xe) { diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 59678b310e55f..e8005775be39e 100644 --- a/drivers/gpu/drm/xe/xe_pm.h +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -6,6 +6,7 @@ #ifndef _XE_PM_H_ #define _XE_PM_H_ +#include #include #define DEFAULT_VRAM_THRESHOLD 300 /* in MB */ @@ -35,4 +36,20 @@ bool xe_rpm_reclaim_safe(const struct xe_device *xe); struct task_struct *xe_pm_read_callback_task(struct xe_device *xe); int xe_pm_module_init(void); +static inline void __xe_pm_runtime_noop(struct xe_device *xe) {} + +DEFINE_GUARD(xe_pm_runtime, struct xe_device *, + xe_pm_runtime_get(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD(xe_pm_runtime_noresume, struct xe_device *, + xe_pm_runtime_get_noresume(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD_COND(xe_pm_runtime, _ioctl, xe_pm_runtime_get_ioctl(_T), _RET >= 0) + +/* + * Used when a function needs to release runtime PM in all possible cases + * and error paths, but the wakeref was already acquired by a different + * function (i.e., get() has already happened so only a put() is needed). + */ +DEFINE_GUARD(xe_pm_runtime_release_only, struct xe_device *, + __xe_pm_runtime_noop(_T), xe_pm_runtime_put(_T)); + #endif diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 07f96bda638af..a21383fba1e81 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -1451,9 +1451,9 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, err = vma_check_userptr(vm, op->map.vma, pt_update); break; case DRM_GPUVA_OP_REMAP: - if (op->remap.prev) + if (op->remap.prev && !op->remap.skip_prev) err = vma_check_userptr(vm, op->remap.prev, pt_update); - if (!err && op->remap.next) + if (!err && op->remap.next && !op->remap.skip_next) err = vma_check_userptr(vm, op->remap.next, pt_update); break; case DRM_GPUVA_OP_UNMAP: @@ -2038,12 +2038,12 @@ static int op_prepare(struct xe_vm *vm, err = unbind_op_prepare(tile, pt_update_ops, old); - if (!err && op->remap.prev) { + if (!err && op->remap.prev && !op->remap.skip_prev) { err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.prev, false); pt_update_ops->wait_vm_bookkeep = true; } - if (!err && op->remap.next) { + if (!err && op->remap.next && !op->remap.skip_next) { err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.next, false); pt_update_ops->wait_vm_bookkeep = true; @@ -2267,10 +2267,10 @@ static void op_commit(struct xe_vm *vm, unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2); - if (op->remap.prev) + if (op->remap.prev && !op->remap.skip_prev) bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, fence, fence2, false); - if (op->remap.next) + if (op->remap.next && !op->remap.skip_next) bind_op_commit(vm, tile, pt_update_ops, op->remap.next, fence, fence2, false); break; diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index bdbdbbf6a6781..9261a8412b64f 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -532,7 +532,7 @@ static int __exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q) static int pxp_start(struct xe_pxp *pxp, u8 type) { int ret = 0; - bool restart = false; + bool restart; if (!xe_pxp_is_enabled(pxp)) return -ENODEV; @@ -561,6 +561,8 @@ static int pxp_start(struct xe_pxp *pxp, u8 type) msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS))) return -ETIMEDOUT; + restart = false; + mutex_lock(&pxp->mutex); /* If PXP is not already active, turn it on */ @@ -603,6 +605,7 @@ static int pxp_start(struct xe_pxp *pxp, u8 type) drm_err(&pxp->xe->drm, "PXP termination failed before start\n"); mutex_lock(&pxp->mutex); pxp->status = XE_PXP_ERROR; + complete_all(&pxp->termination); goto out_unlock; } @@ -890,11 +893,6 @@ int xe_pxp_pm_suspend(struct xe_pxp *pxp) pxp->key_instance++; needs_queue_inval = true; break; - default: - drm_err(&pxp->xe->drm, "unexpected state during PXP suspend: %u", - pxp->status); - ret = -EIO; - goto out; } /* @@ -919,7 +917,6 @@ int xe_pxp_pm_suspend(struct xe_pxp *pxp) pxp->last_suspend_key_instance = pxp->key_instance; -out: return ret; } diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 2e9ff33ed2fe2..856089f64c341 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -491,7 +491,7 @@ static int copy_mask(void __user **ptr, if (copy_to_user(*ptr, topo, sizeof(*topo))) return -EFAULT; - *ptr += sizeof(topo); + *ptr += sizeof(*topo); if (copy_to_user(*ptr, mask, mask_size)) return -EFAULT; diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c index fc8447a838c4f..6b9edc7ca4115 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.c +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -101,10 +101,12 @@ int xe_reg_sr_add(struct xe_reg_sr *sr, *pentry = *e; ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); if (ret) - goto fail; + goto fail_free; return 0; +fail_free: + kfree(pentry); fail: xe_gt_err(gt, "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index d71837773d6c6..e0082b55e2162 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -265,6 +265,9 @@ static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + if (job->user_fence.used) { i = emit_flush_dw(dw, i); i = emit_store_imm_ppgtt_posted(job->user_fence.addr, @@ -328,6 +331,9 @@ static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + if (job->user_fence.used) { i = emit_flush_dw(dw, i); i = emit_store_imm_ppgtt_posted(job->user_fence.addr, @@ -377,6 +383,9 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + i = emit_render_cache_flush(job, dw, i); if (job->user_fence.used) diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 8dec616c37c98..739a3eb180b53 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -276,7 +276,7 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) flags = EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_PERMANENT | EXEC_QUEUE_FLAG_MIGRATE; - q = xe_exec_queue_create_bind(xe, tile, flags, 0); + q = xe_exec_queue_create_bind(xe, tile, NULL, flags, 0); if (IS_ERR(q)) { err = PTR_ERR(q); goto err_ret; diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 129e7818565c8..65452c20609ed 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -477,7 +477,8 @@ static void xe_svm_copy_us_stats_incr(struct xe_gt *gt, static int xe_svm_copy(struct page **pages, struct drm_pagemap_addr *pagemap_addr, - unsigned long npages, const enum xe_svm_copy_dir dir) + unsigned long npages, const enum xe_svm_copy_dir dir, + struct dma_fence *pre_migrate_fence) { struct xe_vram_region *vr = NULL; struct xe_gt *gt = NULL; @@ -566,7 +567,8 @@ static int xe_svm_copy(struct page **pages, __fence = xe_migrate_from_vram(vr->migrate, i - pos + incr, vram_addr, - &pagemap_addr[pos]); + &pagemap_addr[pos], + pre_migrate_fence); } else { vm_dbg(&xe->drm, "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", @@ -575,13 +577,14 @@ static int xe_svm_copy(struct page **pages, __fence = xe_migrate_to_vram(vr->migrate, i - pos + incr, &pagemap_addr[pos], - vram_addr); + vram_addr, + pre_migrate_fence); } if (IS_ERR(__fence)) { err = PTR_ERR(__fence); goto err_out; } - + pre_migrate_fence = NULL; dma_fence_put(fence); fence = __fence; } @@ -604,20 +607,22 @@ static int xe_svm_copy(struct page **pages, vram_addr, (u64)pagemap_addr[pos].addr, 1); __fence = xe_migrate_from_vram(vr->migrate, 1, vram_addr, - &pagemap_addr[pos]); + &pagemap_addr[pos], + pre_migrate_fence); } else { vm_dbg(&xe->drm, "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", (u64)pagemap_addr[pos].addr, vram_addr, 1); __fence = xe_migrate_to_vram(vr->migrate, 1, &pagemap_addr[pos], - vram_addr); + vram_addr, + pre_migrate_fence); } if (IS_ERR(__fence)) { err = PTR_ERR(__fence); goto err_out; } - + pre_migrate_fence = NULL; dma_fence_put(fence); fence = __fence; } @@ -630,6 +635,8 @@ static int xe_svm_copy(struct page **pages, dma_fence_wait(fence, false); dma_fence_put(fence); } + if (pre_migrate_fence) + dma_fence_wait(pre_migrate_fence, false); /* * XXX: We can't derive the GT here (or anywhere in this functions, but @@ -646,16 +653,20 @@ static int xe_svm_copy(struct page **pages, static int xe_svm_copy_to_devmem(struct page **pages, struct drm_pagemap_addr *pagemap_addr, - unsigned long npages) + unsigned long npages, + struct dma_fence *pre_migrate_fence) { - return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM); + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM, + pre_migrate_fence); } static int xe_svm_copy_to_ram(struct page **pages, struct drm_pagemap_addr *pagemap_addr, - unsigned long npages) + unsigned long npages, + struct dma_fence *pre_migrate_fence) { - return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM); + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM, + pre_migrate_fence); } static struct xe_bo *to_xe_bo(struct drm_pagemap_devmem *devmem_allocation) @@ -668,6 +679,7 @@ static void xe_svm_devmem_release(struct drm_pagemap_devmem *devmem_allocation) struct xe_bo *bo = to_xe_bo(devmem_allocation); struct xe_device *xe = xe_bo_device(bo); + dma_fence_put(devmem_allocation->pre_migrate_fence); xe_bo_put_async(bo); xe_pm_runtime_put(xe); } @@ -862,6 +874,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, unsigned long timeslice_ms) { struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap); + struct dma_fence *pre_migrate_fence = NULL; struct xe_device *xe = vr->xe; struct device *dev = xe->drm.dev; struct drm_buddy_block *block; @@ -888,8 +901,20 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, break; } + /* Ensure that any clearing or async eviction will complete before migration. */ + if (!dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL)) { + err = dma_resv_get_singleton(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, + &pre_migrate_fence); + if (err) + dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, + false, MAX_SCHEDULE_TIMEOUT); + else if (pre_migrate_fence) + dma_fence_enable_sw_signaling(pre_migrate_fence); + } + drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm, - &dpagemap_devmem_ops, dpagemap, end - start); + &dpagemap_devmem_ops, dpagemap, end - start, + pre_migrate_fence); blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; list_for_each_entry(block, blocks, link) @@ -942,7 +967,7 @@ bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vm xe_assert(vm->xe, IS_DGFX(vm->xe)); if (xe_svm_range_in_vram(range)) { - drm_info(&vm->xe->drm, "Range is already in VRAM\n"); + drm_dbg(&vm->xe->drm, "Range is already in VRAM\n"); return false; } diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 0955d2ac8d744..fa757dd07954d 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -214,7 +214,7 @@ int xe_svm_init(struct xe_vm *vm) { #if IS_ENABLED(CONFIG_DRM_GPUSVM) return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", &vm->xe->drm, - NULL, NULL, 0, 0, 0, NULL, NULL, 0); + NULL, 0, 0, 0, NULL, NULL, 0); #else return 0; #endif diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index d48ab7b32ca51..90944218a5906 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -146,8 +146,10 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, if (!signal) { sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; + if (XE_IOCTL_DBG(xe, !sync->fence)) { + err = -EINVAL; + goto free_sync; + } } break; @@ -167,17 +169,21 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, if (signal) { sync->chain_fence = dma_fence_chain_alloc(); - if (!sync->chain_fence) - return -ENOMEM; + if (!sync->chain_fence) { + err = -ENOMEM; + goto free_sync; + } } else { sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; + if (XE_IOCTL_DBG(xe, !sync->fence)) { + err = -EINVAL; + goto free_sync; + } err = dma_fence_chain_find_seqno(&sync->fence, sync_in.timeline_value); if (err) - return err; + goto free_sync; } break; @@ -200,8 +206,10 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) return PTR_ERR(sync->ufence); sync->ufence_chain_fence = dma_fence_chain_alloc(); - if (!sync->ufence_chain_fence) - return -ENOMEM; + if (!sync->ufence_chain_fence) { + err = -ENOMEM; + goto free_sync; + } sync->ufence_syncobj = ufence_syncobj; } @@ -216,6 +224,10 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, sync->timeline_value = sync_in.timeline_value; return 0; + +free_sync: + xe_sync_entry_cleanup(sync); + return err; } ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index cdd1dc540a59e..8f7b8f2da06ba 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1481,7 +1481,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) INIT_WORK(&vm->destroy_work, vm_destroy_work_func); INIT_LIST_HEAD(&vm->preempt.exec_queues); - vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ + if (flags & XE_VM_FLAG_FAULT_MODE) + vm->preempt.min_run_period_ms = 0; + else + vm->preempt.min_run_period_ms = 5; for_each_tile(tile, xe, id) xe_range_fence_tree_init(&vm->rftree[id]); @@ -1587,7 +1590,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) if (!vm->pt_root[id]) continue; - q = xe_exec_queue_create_bind(xe, tile, create_flags, 0); + q = xe_exec_queue_create_bind(xe, tile, vm, create_flags, 0); if (IS_ERR(q)) { err = PTR_ERR(q); goto err_close; @@ -2419,8 +2422,17 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, if (IS_ERR(vma)) return vma; - if (xe_vma_is_userptr(vma)) + if (xe_vma_is_userptr(vma)) { err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); + /* + * -EBUSY has dedicated meaning that a user fence + * attached to the VMA is busy, in practice + * xe_vma_userptr_pin_pages can only fail with -EBUSY if + * we are low on memory so convert this to -ENOMEM. + */ + if (err == -EBUSY) + err = -ENOMEM; + } } if (err) { prep_vma_destroy(vm, vma, false); @@ -2491,7 +2503,6 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) if (!err && op->remap.skip_prev) { op->remap.prev->tile_present = tile_present; - op->remap.prev = NULL; } } if (op->remap.next) { @@ -2501,11 +2512,13 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) if (!err && op->remap.skip_next) { op->remap.next->tile_present = tile_present; - op->remap.next = NULL; } } - /* Adjust for partial unbind after removing VMA from VM */ + /* + * Adjust for partial unbind after removing VMA from VM. In case + * of unwind we might need to undo this later. + */ if (!err) { op->base.remap.unmap->va->va.addr = op->remap.start; op->base.remap.unmap->va->va.range = op->remap.range; @@ -2624,6 +2637,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, op->remap.start = xe_vma_start(old); op->remap.range = xe_vma_size(old); + op->remap.old_start = op->remap.start; + op->remap.old_range = op->remap.range; flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK; if (op->base.remap.prev) { @@ -2771,8 +2786,19 @@ static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, xe_svm_notifier_lock(vm); vma->gpuva.flags &= ~XE_VMA_DESTROYED; xe_svm_notifier_unlock(vm); - if (post_commit) + if (post_commit) { + /* + * Restore the old va range, in case of the + * prev/next skip optimisation. Otherwise what + * we re-insert here could be smaller than the + * original range. + */ + op->base.remap.unmap->va->va.addr = + op->remap.old_start; + op->base.remap.unmap->va->va.range = + op->remap.old_range; xe_vm_insert_vma(vm, vma); + } } break; } @@ -3167,7 +3193,8 @@ static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op, { switch (op->base.op) { case DRM_GPUVA_OP_MAP: - vma_add_ufence(op->map.vma, ufence); + if (!xe_vma_is_cpu_addr_mirror(op->map.vma)) + vma_add_ufence(op->map.vma, ufence); break; case DRM_GPUVA_OP_REMAP: if (op->remap.prev) @@ -3282,6 +3309,9 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, if (XE_IOCTL_DBG(xe, args->extensions)) return -EINVAL; + if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) + return -EINVAL; + if (args->num_binds > 1) { u64 __user *bind_user = u64_to_user_ptr(args->vector_of_binds); @@ -3530,6 +3560,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) } } + if (XE_IOCTL_DBG(xe, q && vm != q->user_vm)) { + err = -EINVAL; + goto put_exec_queue; + } + /* Ensure all UNMAPs visible */ xe_svm_flush(vm); diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index ef8a5019574e6..016f6786134cb 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -379,7 +379,7 @@ static inline void xe_vm_set_validation_exec(struct xe_vm *vm, struct drm_exec * } /** - * xe_vm_set_validation_exec() - Accessor to read the drm_exec object + * xe_vm_validation_exec() - Accessor to read the drm_exec object * @vm: The vm we want to register a drm_exec object with. * * Return: The drm_exec object used to lock the vm's resv. The value diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index cad3cf627c3f2..9dc801f657129 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -268,8 +268,13 @@ static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madv break; case DRM_XE_MEM_RANGE_ATTR_PAT: { - u16 coh_mode = xe_pat_index_get_coh_mode(xe, args->pat_index.val); + u16 pat_index, coh_mode; + if (XE_IOCTL_DBG(xe, args->pat_index.val >= xe->pat.n_entries)) + return false; + + pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries); + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); if (XE_IOCTL_DBG(xe, !coh_mode)) return false; @@ -385,7 +390,7 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil madvise_range.num_vmas, args->atomic.val)) { err = -EINVAL; - goto unlock_vm; + goto free_vmas; } } @@ -421,6 +426,7 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil err_fini: if (madvise_range.has_bo_vmas) drm_exec_fini(&exec); +free_vmas: kfree(madvise_range.vmas); madvise_range.vmas = NULL; unlock_vm: diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index d6e2a0fdd4b35..9c9c2209bdcf6 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -268,7 +268,7 @@ struct xe_vm { * @min_run_period_ms: The minimum run period before preempting * an engine again */ - s64 min_run_period_ms; + unsigned int min_run_period_ms; /** @exec_queues: list of exec queues attached to this VM */ struct list_head exec_queues; /** @num_exec_queues: number exec queues attached to this VM */ @@ -365,6 +365,10 @@ struct xe_vma_op_remap { u64 start; /** @range: range of the VMA unmap */ u64 range; + /** @old_start: Original start of the VMA we unmap */ + u64 old_start; + /** @old_range: Original range of the VMA we unmap */ + u64 old_range; /** @skip_prev: skip prev rebind */ bool skip_prev; /** @skip_next: skip next rebind */ diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 3cf30718b2003..51c5985830b17 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -15,6 +15,7 @@ #include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" +#include "regs/xe_guc_regs.h" #include "regs/xe_regs.h" #include "xe_device_types.h" #include "xe_force_wake.h" @@ -254,12 +255,14 @@ static const struct xe_rtp_entry_sr gt_was[] = { { XE_RTP_NAME("16025250150"), XE_RTP_RULES(GRAPHICS_VERSION(2001)), - XE_RTP_ACTIONS(SET(LSN_VC_REG2, - LSN_LNI_WGT(1) | - LSN_LNE_WGT(1) | - LSN_DIM_X_WGT(1) | - LSN_DIM_Y_WGT(1) | - LSN_DIM_Z_WGT(1))) + XE_RTP_ACTIONS(FIELD_SET(LSN_VC_REG2, + LSN_LNI_WGT_MASK | LSN_LNE_WGT_MASK | + LSN_DIM_X_WGT_MASK | LSN_DIM_Y_WGT_MASK | + LSN_DIM_Z_WGT_MASK, + LSN_LNI_WGT(1) | LSN_LNE_WGT(1) | + LSN_DIM_X_WGT(1) | LSN_DIM_Y_WGT(1) | + LSN_DIM_Z_WGT(1)), + SET(LSC_CHICKEN_BIT_0_UDW, L3_128B_256B_WRT_DIS)) }, /* Xe2_HPM */ @@ -270,14 +273,6 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)), XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), }, - { XE_RTP_NAME("14020316580"), - XE_RTP_RULES(MEDIA_VERSION(1301)), - XE_RTP_ACTIONS(CLR(POWERGATE_ENABLE, - VDN_HCP_POWERGATE_ENABLE(0) | - VDN_MFXVDENC_POWERGATE_ENABLE(0) | - VDN_HCP_POWERGATE_ENABLE(2) | - VDN_MFXVDENC_POWERGATE_ENABLE(2))), - }, { XE_RTP_NAME("14019449301"), XE_RTP_RULES(MEDIA_VERSION(1301), ENGINE_CLASS(VIDEO_DECODE)), XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)), @@ -323,6 +318,10 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)), XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), }, + { XE_RTP_NAME("16028005424"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)), + XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES)) + }, }; static const struct xe_rtp_entry_sr engine_was[] = { @@ -575,16 +574,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS)) }, - { XE_RTP_NAME("14019988906"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) - }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, { XE_RTP_NAME("14020338487"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), FUNC(xe_rtp_match_first_render_or_compute)), @@ -881,6 +870,14 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) }, + { XE_RTP_NAME("14019988906"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) + }, + { XE_RTP_NAME("14019877138"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) + }, { XE_RTP_NAME("14021490052"), XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(FF_MODE, diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index f3a6d5d239cec..1d32fa1c3b2d7 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -81,3 +81,4 @@ 15015404425_disable PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER) 16026007364 MEDIA_VERSION(3000) +14020316580 MEDIA_VERSION(1301) diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c index f63d14a57a1d9..acc7d82e0585e 100644 --- a/drivers/gpu/host1x/syncpt.c +++ b/drivers/gpu/host1x/syncpt.c @@ -345,8 +345,6 @@ static void syncpt_release(struct kref *ref) sp->locked = false; - mutex_lock(&sp->host->syncpt_mutex); - host1x_syncpt_base_free(sp->base); kfree(sp->name); sp->base = NULL; @@ -369,7 +367,7 @@ void host1x_syncpt_put(struct host1x_syncpt *sp) if (!sp) return; - kref_put(&sp->ref, syncpt_release); + kref_put_mutex(&sp->ref, syncpt_release, &sp->host->syncpt_mutex); } EXPORT_SYMBOL(host1x_syncpt_put); diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index 20d3e6d0d796e..527920f9c4d39 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -3,7 +3,7 @@ config NOVA_CORE depends on 64BIT depends on PCI depends on RUST - depends on RUST_FW_LOADER_ABSTRACTIONS + select RUST_FW_LOADER_ABSTRACTIONS select AUXILIARY_BUS default n help diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 9b70095434c61..6b0761460a57d 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -202,10 +202,10 @@ impl GspFirmware { let mut level0_data = kvec![0u8; GSP_PAGE_SIZE]?; // Fill level 1 page entry. - #[allow(clippy::useless_conversion)] - let level1_entry = u64::from(level1.iter().next().unwrap().dma_address()); - let dst = &mut level0_data[..size_of_val(&level1_entry)]; - dst.copy_from_slice(&level1_entry.to_le_bytes()); + let level1_entry = level1.iter().next().ok_or(EINVAL)?; + let level1_entry_addr = level1_entry.dma_address(); + let dst = &mut level0_data[..size_of_val(&level1_entry_addr)]; + dst.copy_from_slice(&level1_entry_addr.to_le_bytes()); // Turn the level0 page table into a [`DmaObject`]. DmaObject::from_data(dev, &level0_data)? diff --git a/drivers/greybus/gb-beagleplay.c b/drivers/greybus/gb-beagleplay.c index 9610f878da1b6..87186f891a6ac 100644 --- a/drivers/greybus/gb-beagleplay.c +++ b/drivers/greybus/gb-beagleplay.c @@ -644,8 +644,8 @@ static int cc1352_bootloader_wait_for_ack(struct gb_beagleplay *bg) ret = wait_for_completion_timeout( &bg->fwl_ack_com, msecs_to_jiffies(CC1352_BOOTLOADER_TIMEOUT)); - if (ret < 0) - return dev_err_probe(&bg->sd->dev, ret, + if (!ret) + return dev_err_probe(&bg->sd->dev, -ETIMEDOUT, "Failed to acquire ack semaphore"); switch (READ_ONCE(bg->fwl_ack)) { @@ -683,8 +683,8 @@ static int cc1352_bootloader_get_status(struct gb_beagleplay *bg) ret = wait_for_completion_timeout( &bg->fwl_cmd_response_com, msecs_to_jiffies(CC1352_BOOTLOADER_TIMEOUT)); - if (ret < 0) - return dev_err_probe(&bg->sd->dev, ret, + if (!ret) + return dev_err_probe(&bg->sd->dev, -ETIMEDOUT, "Failed to acquire last status semaphore"); switch (READ_ONCE(bg->fwl_cmd_response)) { @@ -768,8 +768,8 @@ static int cc1352_bootloader_crc32(struct gb_beagleplay *bg, u32 *crc32) ret = wait_for_completion_timeout( &bg->fwl_cmd_response_com, msecs_to_jiffies(CC1352_BOOTLOADER_TIMEOUT)); - if (ret < 0) - return dev_err_probe(&bg->sd->dev, ret, + if (!ret) + return dev_err_probe(&bg->sd->dev, -ETIMEDOUT, "Failed to acquire last status semaphore"); *crc32 = READ_ONCE(bg->fwl_cmd_response); diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 04420a713be08..03ad8b5c29a4d 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -369,6 +369,7 @@ config HID_ELECOM - EX-G Trackballs (M-XT3DRBK, M-XT3URBK) - DEFT Trackballs (M-DT1DRBK, M-DT1URBK, M-DT2DRBK, M-DT2URBK) - HUGE Trackballs (M-HT1DRBK, M-HT1URBK) + - HUGE Plus Trackball (M-HT1MRBK) config HID_ELO tristate "ELO USB 4000/4500 touchscreen" diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c index 1d9f955573aa4..4b81cebdc3359 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c @@ -413,7 +413,8 @@ static void sfh_init_work(struct work_struct *work) rc = amd_sfh_hid_client_init(mp2); if (rc) { amd_sfh_clear_intr(mp2); - dev_err(&pdev->dev, "amd_sfh_hid_client_init failed err %d\n", rc); + if (rc != -EOPNOTSUPP) + dev_err(&pdev->dev, "amd_sfh_hid_client_init failed err %d\n", rc); return; } diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c index 9a06f9b0e4ef3..cf465a5fe43af 100644 --- a/drivers/hid/bpf/hid_bpf_dispatch.c +++ b/drivers/hid/bpf/hid_bpf_dispatch.c @@ -447,6 +447,8 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz, (u64)(long)ctx, true); /* prevent infinite recursions */ + if (ret > size) + ret = size; if (ret > 0) memcpy(buf, dma_data, ret); diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c index ba00f6e6324b0..8c3f83532ce92 100644 --- a/drivers/hid/hid-alps.c +++ b/drivers/hid/hid-alps.c @@ -437,6 +437,9 @@ static int alps_raw_event(struct hid_device *hdev, int ret = 0; struct alps_dev *hdata = hid_get_drvdata(hdev); + if (!(hdev->claimed & HID_CLAIMED_INPUT) || !hdata->input) + return 0; + switch (hdev->product) { case HID_PRODUCT_ID_T4_BTNLESS: ret = t4_raw_event(hdata, data, size); diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 57da4f86a9fa7..9dcb252c5d6c7 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -354,6 +354,7 @@ static const struct apple_key_translation swapped_fn_leftctrl_keys[] = { }; static const struct apple_non_apple_keyboard non_apple_keyboards[] = { + { "SONiX KN85 Keyboard" }, { "SONiX USB DEVICE" }, { "SONiX AK870 PRO" }, { "Keychron" }, @@ -364,6 +365,9 @@ static const struct apple_non_apple_keyboard non_apple_keyboards[] = { { "A3R" }, { "hfd.cn" }, { "WKB603" }, + { "TH87" }, /* EPOMAKER TH87 BT mode */ + { "HFD Epomaker TH87" }, /* EPOMAKER TH87 USB mode */ + { "2.4G Wireless Receiver" }, /* EPOMAKER TH87 dongle */ }; static bool apple_is_non_apple_keyboard(struct hid_device *hdev) @@ -685,9 +689,7 @@ static const __u8 *apple_report_fixup(struct hid_device *hdev, __u8 *rdesc, hid_info(hdev, "fixing up Magic Keyboard battery report descriptor\n"); *rsize = *rsize - 1; - rdesc = kmemdup(rdesc + 1, *rsize, GFP_KERNEL); - if (!rdesc) - return NULL; + rdesc = rdesc + 1; rdesc[0] = 0x05; rdesc[1] = 0x01; diff --git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c index b00687e67ce8e..0b10cff465e17 100644 --- a/drivers/hid/hid-appletb-kbd.c +++ b/drivers/hid/hid-appletb-kbd.c @@ -477,7 +477,7 @@ static int appletb_kbd_suspend(struct hid_device *hdev, pm_message_t msg) return 0; } -static int appletb_kbd_reset_resume(struct hid_device *hdev) +static int appletb_kbd_resume(struct hid_device *hdev) { struct appletb_kbd *kbd = hid_get_drvdata(hdev); @@ -503,7 +503,8 @@ static struct hid_driver appletb_kbd_hid_driver = { .input_configured = appletb_kbd_input_configured, #ifdef CONFIG_PM .suspend = appletb_kbd_suspend, - .reset_resume = appletb_kbd_reset_resume, + .resume = appletb_kbd_resume, + .reset_resume = appletb_kbd_resume, #endif .driver.dev_groups = appletb_kbd_groups, }; diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index a444d41e53b6c..5a068f6fd2ce5 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -1305,14 +1305,21 @@ static const __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ if (*rsize == rsize_orig && rdesc[offs] == 0x09 && rdesc[offs + 1] == 0x76) { - *rsize = rsize_orig + 1; - rdesc = kmemdup(rdesc, *rsize, GFP_KERNEL); - if (!rdesc) - return NULL; + __u8 *new_rdesc; + + new_rdesc = devm_kzalloc(&hdev->dev, rsize_orig + 1, + GFP_KERNEL); + if (!new_rdesc) + return rdesc; hid_info(hdev, "Fixing up %s keyb report descriptor\n", drvdata->quirks & QUIRK_T100CHI ? "T100CHI" : "T90CHI"); + + memcpy(new_rdesc, rdesc, rsize_orig); + *rsize = rsize_orig + 1; + rdesc = new_rdesc; + memmove(rdesc + offs + 4, rdesc + offs + 2, 12); rdesc[offs] = 0x19; rdesc[offs + 1] = 0x00; @@ -1396,6 +1403,9 @@ static const struct hid_device_id asus_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X), QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_ALLY_XPAD }, + { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, + USB_DEVICE_ID_ASUSTEK_XGM_2023), + }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD), QUIRK_ROG_CLAYMORE_II_KEYBOARD }, diff --git a/drivers/hid/hid-cmedia.c b/drivers/hid/hid-cmedia.c index 528d7f3612157..8bf5649b0c793 100644 --- a/drivers/hid/hid-cmedia.c +++ b/drivers/hid/hid-cmedia.c @@ -99,7 +99,7 @@ static int cmhid_raw_event(struct hid_device *hid, struct hid_report *report, { struct cmhid *cm = hid_get_drvdata(hid); - if (len != CM6533_JD_RAWEV_LEN) + if (len != CM6533_JD_RAWEV_LEN || !(hid->claimed & HID_CLAIMED_INPUT)) goto out; if (memcmp(data+CM6533_JD_SFX_OFFSET, ji_sfx, sizeof(ji_sfx))) goto out; diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index a5b3a8ca2fcbc..8be4e06af4636 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -71,6 +71,9 @@ static u32 s32ton(__s32 value, unsigned int n) if (!value || !n) return 0; + if (n > 32) + n = 32; + a = value >> (n - 1); if (a && a != -1) return value < 0 ? 1 << (n - 1) : (1 << (n - 1)) - 1; @@ -2057,9 +2060,10 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 * rsize = max_buffer_size; if (csize < rsize) { - dbg_hid("report %d is too short, (%d < %d)\n", report->id, - csize, rsize); - memset(cdata + csize, 0, rsize - csize); + hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %d)\n", + report->id, rsize, csize); + ret = -EINVAL; + goto out; } if ((hid->claimed & HID_CLAIMED_HIDDEV) && hid->hiddev_report_event) diff --git a/drivers/hid/hid-creative-sb0540.c b/drivers/hid/hid-creative-sb0540.c index b4c8e7a5d3e02..dfd6add353d19 100644 --- a/drivers/hid/hid-creative-sb0540.c +++ b/drivers/hid/hid-creative-sb0540.c @@ -153,7 +153,7 @@ static int creative_sb0540_raw_event(struct hid_device *hid, u64 code, main_code; int key; - if (len != 6) + if (len != 6 || !(hid->claimed & HID_CLAIMED_INPUT)) return 0; /* From daemons/hw_hiddev.c sb0540_rec() in lirc */ diff --git a/drivers/hid/hid-elecom.c b/drivers/hid/hid-elecom.c index 981d1b6e96589..37d88ce57f671 100644 --- a/drivers/hid/hid-elecom.c +++ b/drivers/hid/hid-elecom.c @@ -5,6 +5,7 @@ * - EX-G Trackballs (M-XT3DRBK, M-XT3URBK, M-XT4DRBK) * - DEFT Trackballs (M-DT1DRBK, M-DT1URBK, M-DT2DRBK, M-DT2URBK) * - HUGE Trackballs (M-HT1DRBK, M-HT1URBK) + * - HUGE Plus Trackball (M-HT1MRBK) * * Copyright (c) 2010 Richard Nauber * Copyright (c) 2016 Yuxuan Shui @@ -77,7 +78,7 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc, break; case USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB: case USB_DEVICE_ID_ELECOM_M_XT3URBK_018F: - case USB_DEVICE_ID_ELECOM_M_XT3DRBK: + case USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC: case USB_DEVICE_ID_ELECOM_M_XT4DRBK: /* * Report descriptor format: @@ -102,6 +103,16 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ mouse_button_fixup(hdev, rdesc, *rsize, 12, 30, 14, 20, 8); break; + case USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C: + /* + * Report descriptor format: + * 22: button bit count + * 30: padding bit count + * 24: button report size + * 16: button usage maximum + */ + mouse_button_fixup(hdev, rdesc, *rsize, 22, 30, 24, 16, 6); + break; case USB_DEVICE_ID_ELECOM_M_DT2DRBK: case USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C: /* @@ -113,16 +124,30 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ mouse_button_fixup(hdev, rdesc, *rsize, 22, 30, 24, 16, 8); break; + case USB_DEVICE_ID_ELECOM_M_HT1MRBK: + case USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB: + case USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC: + /* + * Report descriptor format: + * 24: button bit count + * 28: padding bit count + * 22: button report size + * 16: button usage maximum + */ + mouse_button_fixup(hdev, rdesc, *rsize, 24, 28, 22, 16, 8); + break; } return rdesc; } static const struct hid_device_id elecom_devices[] = { { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, - { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1URBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1DRBK) }, @@ -131,6 +156,8 @@ static const struct hid_device_id elecom_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1URBK_019B) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB) }, { } }; MODULE_DEVICE_TABLE(hid, elecom_devices); diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index c4589075a5ed6..a245928933454 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -22,6 +22,9 @@ #define USB_DEVICE_ID_3M2256 0x0502 #define USB_DEVICE_ID_3M3266 0x0506 +#define USB_VENDOR_ID_8BITDO 0x2dc8 +#define USB_DEVICE_ID_8BITDO_PRO_3 0x6009 + #define USB_VENDOR_ID_A4TECH 0x09da #define USB_DEVICE_ID_A4TECH_WCP32PU 0x0006 #define USB_DEVICE_ID_A4TECH_X5_005D 0x000a @@ -229,6 +232,7 @@ #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X 0x1b4c #define USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD 0x196b #define USB_DEVICE_ID_ASUSTEK_FX503VD_KEYBOARD 0x1869 +#define USB_DEVICE_ID_ASUSTEK_XGM_2023 0x1a9a #define USB_VENDOR_ID_ATEN 0x0557 #define USB_DEVICE_ID_ATEN_UC100KM 0x2004 @@ -317,6 +321,7 @@ #define USB_DEVICE_ID_CHICONY_ACER_SWITCH12 0x1421 #define USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA 0xb824 #define USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA2 0xb82c +#define USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA3 0xb882 #define USB_VENDOR_ID_CHUNGHWAT 0x2247 #define USB_DEVICE_ID_CHUNGHWAT_MULTITOUCH 0x0001 @@ -436,8 +441,12 @@ #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_7349 0x7349 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_73F7 0x73f7 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_A001 0xa001 +#define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C000 0xc000 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C002 0xc002 +#define USB_VENDOR_ID_EDIFIER 0x2d99 +#define USB_DEVICE_ID_EDIFIER_QR30 0xa101 /* EDIFIER Hal0 2.0 SE */ + #define USB_VENDOR_ID_ELAN 0x04f3 #define USB_DEVICE_ID_TOSHIBA_CLICK_L9W 0x0401 #define USB_DEVICE_ID_HP_X2 0x074d @@ -451,7 +460,8 @@ #define USB_DEVICE_ID_ELECOM_M_XGL20DLBK 0x00e6 #define USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB 0x00fb #define USB_DEVICE_ID_ELECOM_M_XT3URBK_018F 0x018f -#define USB_DEVICE_ID_ELECOM_M_XT3DRBK 0x00fc +#define USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC 0x00fc +#define USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C 0x018c #define USB_DEVICE_ID_ELECOM_M_XT4DRBK 0x00fd #define USB_DEVICE_ID_ELECOM_M_DT1URBK 0x00fe #define USB_DEVICE_ID_ELECOM_M_DT1DRBK 0x00ff @@ -460,6 +470,9 @@ #define USB_DEVICE_ID_ELECOM_M_HT1URBK_019B 0x019b #define USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D 0x010d #define USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C 0x011c +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK 0x01aa +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB 0x01ab +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC 0x01ac #define USB_VENDOR_ID_DREAM_CHEEKY 0x1d34 #define USB_DEVICE_ID_DREAM_CHEEKY_WN 0x0004 @@ -835,6 +848,7 @@ #define USB_DEVICE_ID_LENOVO_X1_TAB3 0x60b5 #define USB_DEVICE_ID_LENOVO_X12_TAB 0x60fe #define USB_DEVICE_ID_LENOVO_X12_TAB2 0x61ae +#define USB_DEVICE_ID_LENOVO_YOGABOOK9I 0x6161 #define USB_DEVICE_ID_LENOVO_OPTICAL_USB_MOUSE_600E 0x600e #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_608D 0x608d #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6019 0x6019 diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 2bbb645c2ff41..32a96ae2946f1 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -878,7 +878,7 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel switch (usage->hid) { /* These usage IDs map directly to the usage codes. */ - case HID_GD_X: case HID_GD_Y: case HID_GD_Z: + case HID_GD_X: case HID_GD_Y: case HID_GD_RX: case HID_GD_RY: case HID_GD_RZ: if (field->flags & HID_MAIN_ITEM_RELATIVE) map_rel(usage->hid & 0xf); @@ -886,6 +886,22 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel map_abs_clear(usage->hid & 0xf); break; + case HID_GD_Z: + /* HID_GD_Z is mapped to ABS_DISTANCE for stylus/pen */ + if (field->flags & HID_MAIN_ITEM_RELATIVE) { + map_rel(usage->hid & 0xf); + } else { + if (field->application == HID_DG_PEN || + field->physical == HID_DG_PEN || + field->logical == HID_DG_STYLUS || + field->physical == HID_DG_STYLUS || + field->application == HID_DG_DIGITIZER) + map_abs_clear(ABS_DISTANCE); + else + map_abs_clear(usage->hid & 0xf); + } + break; + case HID_GD_WHEEL: if (field->flags & HID_MAIN_ITEM_RELATIVE) { set_bit(REL_WHEEL, input->relbit); diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c index cce54dd9884a3..3b5412541c925 100644 --- a/drivers/hid/hid-logitech-dj.c +++ b/drivers/hid/hid-logitech-dj.c @@ -805,7 +805,6 @@ static void delayedwork_callback(struct work_struct *work) struct dj_workitem workitem; unsigned long flags; int count; - int retval; dbg_hid("%s\n", __func__); @@ -842,11 +841,7 @@ static void delayedwork_callback(struct work_struct *work) logi_dj_recv_destroy_djhid_device(djrcv_dev, &workitem); break; case WORKITEM_TYPE_UNKNOWN: - retval = logi_dj_recv_query_paired_devices(djrcv_dev); - if (retval) { - hid_err(djrcv_dev->hidpp, "%s: logi_dj_recv_query_paired_devices error: %d\n", - __func__, retval); - } + logi_dj_recv_query_paired_devices(djrcv_dev); break; case WORKITEM_TYPE_EMPTY: dbg_hid("%s: device list is empty\n", __func__); @@ -1239,8 +1234,10 @@ static int logi_dj_recv_query_paired_devices(struct dj_receiver_dev *djrcv_dev) djrcv_dev->last_query = jiffies; - if (djrcv_dev->type != recvr_type_dj) - return logi_dj_recv_query_hidpp_devices(djrcv_dev); + if (djrcv_dev->type != recvr_type_dj) { + retval = logi_dj_recv_query_hidpp_devices(djrcv_dev); + goto out; + } dj_report = kzalloc(sizeof(struct dj_report), GFP_KERNEL); if (!dj_report) @@ -1250,6 +1247,10 @@ static int logi_dj_recv_query_paired_devices(struct dj_receiver_dev *djrcv_dev) dj_report->report_type = REPORT_TYPE_CMD_GET_PAIRED_DEVICES; retval = logi_dj_recv_send_report(djrcv_dev, dj_report); kfree(dj_report); +out: + if (retval < 0) + hid_err(djrcv_dev->hidpp, "%s error:%d\n", __func__, retval); + return retval; } @@ -1275,6 +1276,8 @@ static int logi_dj_recv_switch_to_dj_mode(struct dj_receiver_dev *djrcv_dev, (u8)timeout; retval = logi_dj_recv_send_report(djrcv_dev, dj_report); + if (retval) + goto out; /* * Ugly sleep to work around a USB 3.0 bug when the receiver is @@ -1283,11 +1286,6 @@ static int logi_dj_recv_switch_to_dj_mode(struct dj_receiver_dev *djrcv_dev, * 50 msec should gives enough time to the receiver to be ready. */ msleep(50); - - if (retval) { - kfree(dj_report); - return retval; - } } /* @@ -1313,7 +1311,12 @@ static int logi_dj_recv_switch_to_dj_mode(struct dj_receiver_dev *djrcv_dev, HIDPP_REPORT_SHORT_LENGTH, HID_OUTPUT_REPORT, HID_REQ_SET_REPORT); +out: kfree(dj_report); + + if (retval < 0) + hid_err(hdev, "%s error:%d\n", __func__, retval); + return retval; } @@ -1835,11 +1838,8 @@ static int logi_dj_probe(struct hid_device *hdev, if (has_hidpp) { retval = logi_dj_recv_switch_to_dj_mode(djrcv_dev, 0); - if (retval < 0) { - hid_err(hdev, "%s: logi_dj_recv_switch_to_dj_mode returned error:%d\n", - __func__, retval); + if (retval < 0) goto switch_to_dj_mode_fail; - } } /* This is enabling the polling urb on the IN endpoint */ @@ -1857,15 +1857,11 @@ static int logi_dj_probe(struct hid_device *hdev, spin_lock_irqsave(&djrcv_dev->lock, flags); djrcv_dev->ready = true; spin_unlock_irqrestore(&djrcv_dev->lock, flags); - retval = logi_dj_recv_query_paired_devices(djrcv_dev); - if (retval < 0) { - hid_err(hdev, "%s: logi_dj_recv_query_paired_devices error:%d\n", - __func__, retval); - /* - * This can happen with a KVM, let the probe succeed, - * logi_dj_recv_queue_unknown_work will retry later. - */ - } + /* + * This can fail with a KVM. Ignore errors to let the probe + * succeed, logi_dj_recv_queue_unknown_work will retry later. + */ + logi_dj_recv_query_paired_devices(djrcv_dev); } return 0; @@ -1882,18 +1878,12 @@ static int logi_dj_probe(struct hid_device *hdev, #ifdef CONFIG_PM static int logi_dj_reset_resume(struct hid_device *hdev) { - int retval; struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev); if (!djrcv_dev || djrcv_dev->hidpp != hdev) return 0; - retval = logi_dj_recv_switch_to_dj_mode(djrcv_dev, 0); - if (retval < 0) { - hid_err(hdev, "%s: logi_dj_recv_switch_to_dj_mode returned error:%d\n", - __func__, retval); - } - + logi_dj_recv_switch_to_dj_mode(djrcv_dev, 0); return 0; } #endif diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 5e763de4b94fd..faef80cb2adbd 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -352,10 +352,15 @@ static int hidpp_send_message_sync(struct hidpp_device *hidpp, do { ret = __do_hidpp_send_message_sync(hidpp, message, response); - if (ret != HIDPP20_ERROR_BUSY) + if (response->report_id == REPORT_ID_HIDPP_SHORT && + ret != HIDPP_ERROR_BUSY) + break; + if ((response->report_id == REPORT_ID_HIDPP_LONG || + response->report_id == REPORT_ID_HIDPP_VERY_LONG) && + ret != HIDPP20_ERROR_BUSY) break; - dbg_hid("%s:got busy hidpp 2.0 error %02X, retrying\n", __func__, ret); + dbg_hid("%s:got busy hidpp error %02X, retrying\n", __func__, ret); } while (--max_retries); mutex_unlock(&hidpp->send_mutex); @@ -4308,7 +4313,7 @@ static int hidpp_get_report_length(struct hid_device *hdev, int id) re = &(hdev->report_enum[HID_OUTPUT_REPORT]); report = re->report_id_hash[id]; - if (!report) + if (!report || !report->maxfield) return 0; return report->field[0]->report_count + 1; @@ -4481,10 +4486,12 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id) if (!ret) ret = hidpp_ff_init(hidpp, &data); - if (ret) + if (ret) { hid_warn(hidpp->hid_dev, "Unable to initialize force feedback support, errno %d\n", ret); + ret = 0; + } } /* @@ -4656,8 +4663,14 @@ static const struct hid_device_id hidpp_devices[] = { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb025) }, { /* MX Master 3S mouse over Bluetooth */ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb034) }, + { /* MX Anywhere 3S mouse over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb037) }, { /* MX Anywhere 3SB mouse over Bluetooth */ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb038) }, + { /* Slim Solar+ K980 Keyboard over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb391) }, + { /* MX Master 4 mouse over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb042) }, {} }; diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 7d4a25c6de0eb..9eadf3252d0dc 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -725,6 +725,11 @@ static int magicmouse_input_configured(struct hid_device *hdev, struct magicmouse_sc *msc = hid_get_drvdata(hdev); int ret; + if (!msc->input) { + hid_err(hdev, "magicmouse setup input failed (no input)"); + return -EINVAL; + } + ret = magicmouse_setup_input(msc->input, hdev); if (ret) { hid_err(hdev, "magicmouse setup input failed (%d)\n", ret); @@ -985,13 +990,11 @@ static const __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ if ((is_usb_magicmouse2(hdev->vendor, hdev->product) || is_usb_magictrackpad2(hdev->vendor, hdev->product)) && - *rsize == 83 && rdesc[46] == 0x84 && rdesc[58] == 0x85) { + *rsize >= 83 && rdesc[46] == 0x84 && rdesc[58] == 0x85) { hid_info(hdev, "fixing up magicmouse battery report descriptor\n"); *rsize = *rsize - 1; - rdesc = kmemdup(rdesc + 1, *rsize, GFP_KERNEL); - if (!rdesc) - return NULL; + rdesc = rdesc + 1; rdesc[0] = 0x05; rdesc[1] = 0x01; diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c index 33603b019f975..ef3b5c77c38e3 100644 --- a/drivers/hid/hid-mcp2221.c +++ b/drivers/hid/hid-mcp2221.c @@ -353,6 +353,8 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, usleep_range(90, 100); retries++; } else { + usleep_range(980, 1000); + mcp_cancel_last_cmd(mcp); return ret; } } else { diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index 179dc316b4b51..21bfaf9bbd733 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -76,6 +76,8 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_DISABLE_WAKEUP BIT(21) #define MT_QUIRK_ORIENTATION_INVERT BIT(22) #define MT_QUIRK_APPLE_TOUCHBAR BIT(23) +#define MT_QUIRK_YOGABOOK9I BIT(24) +#define MT_QUIRK_KEEP_LATENCY_ON_CLOSE BIT(25) #define MT_INPUTMODE_TOUCHSCREEN 0x02 #define MT_INPUTMODE_TOUCHPAD 0x03 @@ -211,6 +213,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_WIN_8_DISABLE_WAKEUP 0x0016 #define MT_CLS_WIN_8_NO_STICKY_FINGERS 0x0017 #define MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU 0x0018 +#define MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE 0x0019 /* vendor specific classes */ #define MT_CLS_3M 0x0101 @@ -229,6 +232,8 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_RAZER_BLADE_STEALTH 0x0112 #define MT_CLS_SMART_TECH 0x0113 #define MT_CLS_APPLE_TOUCHBAR 0x0114 +#define MT_CLS_YOGABOOK9I 0x0115 +#define MT_CLS_EGALAX_P80H84 0x0116 #define MT_CLS_SIS 0x0457 #define MT_DEFAULT_MAXCONTACT 10 @@ -330,6 +335,15 @@ static const struct mt_class mt_classes[] = { MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_WIN8_PTP_BUTTONS, .export_all_inputs = true }, + { .name = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_HOVERING | + MT_QUIRK_CONTACT_CNT_ACCURATE | + MT_QUIRK_STICKY_FINGERS | + MT_QUIRK_WIN8_PTP_BUTTONS | + MT_QUIRK_KEEP_LATENCY_ON_CLOSE, + .export_all_inputs = true }, /* * vendor specific classes @@ -393,6 +407,7 @@ static const struct mt_class mt_classes[] = { { .name = MT_CLS_VTL, .quirks = MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE | + MT_QUIRK_STICKY_FINGERS | MT_QUIRK_FORCE_GET_FEATURE, }, { .name = MT_CLS_GOOGLE, @@ -424,6 +439,19 @@ static const struct mt_class mt_classes[] = { .quirks = MT_QUIRK_NOT_SEEN_MEANS_UP | MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE, + }, + { .name = MT_CLS_YOGABOOK9I, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_FORCE_MULTI_INPUT | + MT_QUIRK_SEPARATE_APP_REPORT | + MT_QUIRK_HOVERING | + MT_QUIRK_YOGABOOK9I, + .export_all_inputs = true + }, + { .name = MT_CLS_EGALAX_P80H84, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_CONTACT_CNT_ACCURATE, }, { } }; @@ -496,12 +524,19 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report) dev_warn(&hdev->dev, "failed to fetch feature %d\n", report->id); } else { + /* The report ID in the request and the response should match */ + if (report->id != buf[0]) { + hid_err(hdev, "Returned feature report did not match the request\n"); + goto free; + } + ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, buf, size, 0); if (ret) dev_warn(&hdev->dev, "failed to report feature\n"); } +free: kfree(buf); } @@ -830,7 +865,8 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, if ((cls->name == MT_CLS_WIN_8 || cls->name == MT_CLS_WIN_8_FORCE_MULTI_INPUT || cls->name == MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU || - cls->name == MT_CLS_WIN_8_DISABLE_WAKEUP) && + cls->name == MT_CLS_WIN_8_DISABLE_WAKEUP || + cls->name == MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE) && (field->application == HID_DG_TOUCHPAD || field->application == HID_DG_TOUCHSCREEN)) app->quirks |= MT_QUIRK_CONFIDENCE; @@ -1565,6 +1601,38 @@ static void mt_report(struct hid_device *hid, struct hid_report *report) if (rdata && rdata->is_mt_collection) return mt_touch_report(hid, rdata); + /* Lenovo Yoga Book 9i requires consuming and dropping certain bogus reports */ + if (rdata && rdata->application && + (rdata->application->quirks & MT_QUIRK_YOGABOOK9I)) { + + bool all_zero_report = true; + + for (int f = 0; f < report->maxfield && all_zero_report; f++) { + struct hid_field *fld = report->field[f]; + + for (int i = 0; i < fld->report_count; i++) { + unsigned int usage = fld->usage[i].hid; + + if (usage == HID_DG_INRANGE || + usage == HID_DG_TIPSWITCH || + usage == HID_DG_BARRELSWITCH || + usage == HID_DG_BARRELSWITCH2 || + usage == HID_DG_CONTACTID || + usage == HID_DG_TILT_X || + usage == HID_DG_TILT_Y) { + + if (fld->value[i] != 0) { + all_zero_report = false; + break; + } + } + } + } + + if (all_zero_report) + return; + } + if (field && field->hidinput && field->hidinput->input) input_sync(field->hidinput->input); } @@ -1709,7 +1777,8 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi) int ret; if (td->is_haptic_touchpad && (td->mtclass.name == MT_CLS_WIN_8 || - td->mtclass.name == MT_CLS_WIN_8_FORCE_MULTI_INPUT)) { + td->mtclass.name == MT_CLS_WIN_8_FORCE_MULTI_INPUT || + td->mtclass.name == MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE)) { if (hid_haptic_input_configured(hdev, td->haptic, hi) == 0) td->is_haptic_touchpad = false; } else { @@ -1761,6 +1830,30 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi) break; } + /* Lenovo Yoga Book 9i requires custom naming to allow differentiation in udev */ + if (hi->report && td->mtclass.quirks & MT_QUIRK_YOGABOOK9I) { + switch (hi->report->id) { + case 48: + suffix = "Touchscreen Top"; + break; + case 56: + suffix = "Touchscreen Bottom"; + break; + case 20: + suffix = "Stylus Top"; + break; + case 40: + suffix = "Stylus Bottom"; + break; + case 80: + suffix = "Emulated Touchpad"; + break; + default: + suffix = ""; + break; + } + } + if (suffix) { hi->input->name = devm_kasprintf(&hdev->dev, GFP_KERNEL, "%s %s", hdev->name, suffix); @@ -1998,7 +2091,12 @@ static void mt_on_hid_hw_open(struct hid_device *hdev) static void mt_on_hid_hw_close(struct hid_device *hdev) { - mt_set_modes(hdev, HID_LATENCY_HIGH, TOUCHPAD_REPORT_NONE); + struct mt_device *td = hid_get_drvdata(hdev); + + if (td->mtclass.quirks & MT_QUIRK_KEEP_LATENCY_ON_CLOSE) + mt_set_modes(hdev, HID_LATENCY_NORMAL, TOUCHPAD_REPORT_NONE); + else + mt_set_modes(hdev, HID_LATENCY_HIGH, TOUCHPAD_REPORT_NONE); } /* @@ -2135,8 +2233,12 @@ static const struct hid_device_id mt_devices[] = { { .driver_data = MT_CLS_EGALAX_SERIAL, MT_USB_DEVICE(USB_VENDOR_ID_DWAV, USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_A001) }, - { .driver_data = MT_CLS_EGALAX, + { .driver_data = MT_CLS_EGALAX_SERIAL, MT_USB_DEVICE(USB_VENDOR_ID_DWAV, + USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C000) }, + { .driver_data = MT_CLS_EGALAX_P80H84, + HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_DWAV, USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C002) }, /* Elan devices */ @@ -2266,6 +2368,12 @@ static const struct hid_device_id mt_devices[] = { USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X12_TAB2) }, + /* Lenovo Yoga Book 9i */ + { .driver_data = MT_CLS_YOGABOOK9I, + HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_LENOVO, + USB_DEVICE_ID_LENOVO_YOGABOOK9I) }, + /* Logitech devices */ { .driver_data = MT_CLS_NSMU, HID_DEVICE(BUS_BLUETOOTH, HID_GROUP_MULTITOUCH_WIN_8, @@ -2375,6 +2483,14 @@ static const struct hid_device_id mt_devices[] = { MT_USB_DEVICE(USB_VENDOR_ID_UNITEC, USB_DEVICE_ID_UNITEC_USB_TOUCH_0A19) }, + /* Uniwill touchpads */ + { .driver_data = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_PIXART, 0x0255) }, + { .driver_data = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_PIXART, 0x0274) }, + /* VTL panels */ { .driver_data = MT_CLS_VTL, MT_USB_DEVICE(USB_VENDOR_ID_VTL, diff --git a/drivers/hid/hid-pl.c b/drivers/hid/hid-pl.c index 3c8827081deae..dc11d5322fc0f 100644 --- a/drivers/hid/hid-pl.c +++ b/drivers/hid/hid-pl.c @@ -194,9 +194,14 @@ static int pl_probe(struct hid_device *hdev, const struct hid_device_id *id) goto err; } - plff_init(hdev); + ret = plff_init(hdev); + if (ret) + goto stop; return 0; + +stop: + hid_hw_stop(hdev); err: return ret; } diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c index 128aa6abd10be..2ec6d4445e84b 100644 --- a/drivers/hid/hid-playstation.c +++ b/drivers/hid/hid-playstation.c @@ -753,11 +753,16 @@ ps_gamepad_create(struct hid_device *hdev, if (IS_ERR(gamepad)) return ERR_CAST(gamepad); + /* Set initial resting state for joysticks to 128 (center) */ input_set_abs_params(gamepad, ABS_X, 0, 255, 0, 0); + gamepad->absinfo[ABS_X].value = 128; input_set_abs_params(gamepad, ABS_Y, 0, 255, 0, 0); + gamepad->absinfo[ABS_Y].value = 128; input_set_abs_params(gamepad, ABS_Z, 0, 255, 0, 0); input_set_abs_params(gamepad, ABS_RX, 0, 255, 0, 0); + gamepad->absinfo[ABS_RX].value = 128; input_set_abs_params(gamepad, ABS_RY, 0, 255, 0, 0); + gamepad->absinfo[ABS_RY].value = 128; input_set_abs_params(gamepad, ABS_RZ, 0, 255, 0, 0); input_set_abs_params(gamepad, ABS_HAT0X, -1, 1, 0, 0); @@ -769,7 +774,9 @@ ps_gamepad_create(struct hid_device *hdev, #if IS_ENABLED(CONFIG_PLAYSTATION_FF) if (play_effect) { input_set_capability(gamepad, EV_FF, FF_RUMBLE); - input_ff_create_memless(gamepad, NULL, play_effect); + ret = input_ff_create_memless(gamepad, NULL, play_effect); + if (ret) + return ERR_PTR(ret); } #endif diff --git a/drivers/hid/hid-prodikeys.c b/drivers/hid/hid-prodikeys.c index 74bddb2c3e82e..6e413df38358a 100644 --- a/drivers/hid/hid-prodikeys.c +++ b/drivers/hid/hid-prodikeys.c @@ -378,6 +378,10 @@ static int pcmidi_handle_report4(struct pcmidi_snd *pm, u8 *data) bit_mask = (bit_mask << 8) | data[2]; bit_mask = (bit_mask << 8) | data[3]; + /* robustness in case input_mapping hook does not get called */ + if (!pm->input_ep82) + return 0; + /* break keys */ for (bit_index = 0; bit_index < 24; bit_index++) { if (!((0x01 << bit_index) & bit_mask)) { diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index c89a015686c07..f6be3ffee0232 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -25,6 +25,7 @@ */ static const struct hid_device_id hid_quirks[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_8BITDO, USB_DEVICE_ID_8BITDO_PRO_3), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_GAMEPAD), HID_QUIRK_BADPAD }, { HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_PREDATOR), HID_QUIRK_BADPAD }, { HID_USB_DEVICE(USB_VENDOR_ID_ADATA_XPG, USB_VENDOR_ID_ADATA_XPG_WL_GAMING_MOUSE), HID_QUIRK_ALWAYS_POLL }, @@ -81,6 +82,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_WIIU), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_DWAV, USB_DEVICE_ID_EGALAX_TOUCHCONTROLLER), HID_QUIRK_MULTI_INPUT | HID_QUIRK_NOGET }, + { HID_USB_DEVICE(USB_VENDOR_ID_EDIFIER, USB_DEVICE_ID_EDIFIER_QR30), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_ELAN, HID_ANY_ID), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_ELO, USB_DEVICE_ID_ELO_TS2700), HID_QUIRK_NOGET }, { HID_USB_DEVICE(USB_VENDOR_ID_EMS, USB_DEVICE_ID_EMS_TRIO_LINKER_PLUS_II), HID_QUIRK_MULTI_INPUT }, @@ -232,6 +234,15 @@ static const struct hid_device_id hid_quirks[] = { * used as a driver. See hid_scan_report(). */ static const struct hid_device_id hid_have_special_driver[] = { +#if IS_ENABLED(CONFIG_APPLEDISPLAY) + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x9218) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x9219) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x921c) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x921d) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x9222) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x9226) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x9236) }, +#endif #if IS_ENABLED(CONFIG_HID_A4TECH) { HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_WCP32PU) }, { HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_X5_005D) }, @@ -410,9 +421,11 @@ static const struct hid_device_id hid_have_special_driver[] = { #if IS_ENABLED(CONFIG_HID_ELECOM) { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, - { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1URBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1DRBK) }, @@ -421,6 +434,8 @@ static const struct hid_device_id hid_have_special_driver[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1URBK_019B) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB) }, #endif #if IS_ENABLED(CONFIG_HID_ELO) { HID_USB_DEVICE(USB_VENDOR_ID_ELO, 0x0009) }, @@ -769,6 +784,7 @@ static const struct hid_device_id hid_ignore_list[] = { { HID_USB_DEVICE(USB_VENDOR_ID_BERKSHIRE, USB_DEVICE_ID_BERKSHIRE_PCWD) }, { HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA) }, { HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA2) }, + { HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA3) }, { HID_USB_DEVICE(USB_VENDOR_ID_CIDC, 0x0103) }, { HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI470X) }, { HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI4713) }, diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c index c7f7562e22e56..e413662f75082 100644 --- a/drivers/hid/hid-roccat.c +++ b/drivers/hid/hid-roccat.c @@ -257,6 +257,7 @@ int roccat_report_event(int minor, u8 const *data) if (!new_value) return -ENOMEM; + mutex_lock(&device->readers_lock); mutex_lock(&device->cbuf_lock); report = &device->cbuf[device->cbuf_end]; @@ -279,6 +280,7 @@ int roccat_report_event(int minor, u8 const *data) } mutex_unlock(&device->cbuf_lock); + mutex_unlock(&device->readers_lock); wake_up_interruptible(&device->wait); return 0; diff --git a/drivers/hid/hid-zydacron.c b/drivers/hid/hid-zydacron.c index 3bdb26f455925..1aae80f848f50 100644 --- a/drivers/hid/hid-zydacron.c +++ b/drivers/hid/hid-zydacron.c @@ -114,7 +114,7 @@ static int zc_raw_event(struct hid_device *hdev, struct hid_report *report, unsigned key; unsigned short index; - if (report->id == data[0]) { + if (report->id == data[0] && (hdev->claimed & HID_CLAIMED_INPUT)) { /* break keys */ for (index = 0; index < 4; index++) { diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index 63f46a2e57882..5a183af3d5c6a 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -286,6 +286,7 @@ static int i2c_hid_get_report(struct i2c_hid *ihid, * In addition to report data device will supply data length * in the first 2 bytes of the response, so adjust . */ + recv_len = min(recv_len, ihid->bufsize - sizeof(__le16)); error = i2c_hid_xfer(ihid, ihid->cmdbuf, length, ihid->rawbuf, recv_len + sizeof(__le16)); if (error) { diff --git a/drivers/hid/i2c-hid/i2c-hid-of-elan.c b/drivers/hid/i2c-hid/i2c-hid-of-elan.c index 0215f217f6d86..b81fcc6ff49ee 100644 --- a/drivers/hid/i2c-hid/i2c-hid-of-elan.c +++ b/drivers/hid/i2c-hid/i2c-hid-of-elan.c @@ -168,6 +168,13 @@ static const struct elan_i2c_hid_chip_data elan_ekth6a12nay_chip_data = { .power_after_backlight = true, }; +static const struct elan_i2c_hid_chip_data focaltech_ft8112_chip_data = { + .post_power_delay_ms = 10, + .post_gpio_reset_on_delay_ms = 150, + .hid_descriptor_address = 0x0001, + .main_supply_name = "vcc33", +}; + static const struct elan_i2c_hid_chip_data ilitek_ili9882t_chip_data = { .post_power_delay_ms = 1, .post_gpio_reset_on_delay_ms = 200, @@ -191,6 +198,7 @@ static const struct elan_i2c_hid_chip_data ilitek_ili2901_chip_data = { static const struct of_device_id elan_i2c_hid_of_match[] = { { .compatible = "elan,ekth6915", .data = &elan_ekth6915_chip_data }, { .compatible = "elan,ekth6a12nay", .data = &elan_ekth6a12nay_chip_data }, + { .compatible = "focaltech,ft8112", .data = &focaltech_ft8112_chip_data }, { .compatible = "ilitek,ili9882t", .data = &ilitek_ili9882t_chip_data }, { .compatible = "ilitek,ili2901", .data = &ilitek_ili2901_chip_data }, { } diff --git a/drivers/hid/intel-ish-hid/ipc/hw-ish.h b/drivers/hid/intel-ish-hid/ipc/hw-ish.h index fa5d68c363134..27389971b96cc 100644 --- a/drivers/hid/intel-ish-hid/ipc/hw-ish.h +++ b/drivers/hid/intel-ish-hid/ipc/hw-ish.h @@ -39,6 +39,8 @@ #define PCI_DEVICE_ID_INTEL_ISH_PTL_H 0xE345 #define PCI_DEVICE_ID_INTEL_ISH_PTL_P 0xE445 #define PCI_DEVICE_ID_INTEL_ISH_WCL 0x4D45 +#define PCI_DEVICE_ID_INTEL_ISH_NVL_H 0xD354 +#define PCI_DEVICE_ID_INTEL_ISH_NVL_S 0x6E78 #define REVISION_ID_CHT_A0 0x6 #define REVISION_ID_CHT_Ax_SI 0x0 diff --git a/drivers/hid/intel-ish-hid/ipc/ipc.c b/drivers/hid/intel-ish-hid/ipc/ipc.c index 3ddaa2cd39d55..3692d1db3bc76 100644 --- a/drivers/hid/intel-ish-hid/ipc/ipc.c +++ b/drivers/hid/intel-ish-hid/ipc/ipc.c @@ -628,7 +628,7 @@ static void recv_ipc(struct ishtp_device *dev, uint32_t doorbell_val) if (!ishtp_dev) { ishtp_dev = dev; } - schedule_work(&fw_reset_work); + queue_work(dev->unbound_wq, &fw_reset_work); break; case MNG_RESET_NOTIFY_ACK: @@ -933,6 +933,25 @@ static const struct ishtp_hw_ops ish_hw_ops = { .dma_no_cache_snooping = _dma_no_cache_snooping }; +static void ishtp_free_workqueue(void *wq) +{ + destroy_workqueue(wq); +} + +static struct workqueue_struct *devm_ishtp_alloc_workqueue(struct device *dev) +{ + struct workqueue_struct *wq; + + wq = alloc_workqueue("ishtp_unbound_%d", WQ_UNBOUND, 0, dev->id); + if (!wq) + return NULL; + + if (devm_add_action_or_reset(dev, ishtp_free_workqueue, wq)) + return NULL; + + return wq; +} + /** * ish_dev_init() -Initialize ISH devoce * @pdev: PCI device @@ -953,6 +972,10 @@ struct ishtp_device *ish_dev_init(struct pci_dev *pdev) if (!dev) return NULL; + dev->unbound_wq = devm_ishtp_alloc_workqueue(&pdev->dev); + if (!dev->unbound_wq) + return NULL; + dev->devc = &pdev->dev; ishtp_device_init(dev); diff --git a/drivers/hid/intel-ish-hid/ipc/pci-ish.c b/drivers/hid/intel-ish-hid/ipc/pci-ish.c index 9d150ce234f25..51a41a28541c5 100644 --- a/drivers/hid/intel-ish-hid/ipc/pci-ish.c +++ b/drivers/hid/intel-ish-hid/ipc/pci-ish.c @@ -28,11 +28,15 @@ enum ishtp_driver_data_index { ISHTP_DRIVER_DATA_LNL_M, ISHTP_DRIVER_DATA_PTL, ISHTP_DRIVER_DATA_WCL, + ISHTP_DRIVER_DATA_NVL_H, + ISHTP_DRIVER_DATA_NVL_S, }; #define ISH_FW_GEN_LNL_M "lnlm" #define ISH_FW_GEN_PTL "ptl" #define ISH_FW_GEN_WCL "wcl" +#define ISH_FW_GEN_NVL_H "nvlh" +#define ISH_FW_GEN_NVL_S "nvls" #define ISH_FIRMWARE_PATH(gen) "intel/ish/ish_" gen ".bin" #define ISH_FIRMWARE_PATH_ALL "intel/ish/ish_*.bin" @@ -47,6 +51,12 @@ static struct ishtp_driver_data ishtp_driver_data[] = { [ISHTP_DRIVER_DATA_WCL] = { .fw_generation = ISH_FW_GEN_WCL, }, + [ISHTP_DRIVER_DATA_NVL_H] = { + .fw_generation = ISH_FW_GEN_NVL_H, + }, + [ISHTP_DRIVER_DATA_NVL_S] = { + .fw_generation = ISH_FW_GEN_NVL_S, + }, }; static const struct pci_device_id ish_pci_tbl[] = { @@ -76,6 +86,8 @@ static const struct pci_device_id ish_pci_tbl[] = { {PCI_DEVICE_DATA(INTEL, ISH_PTL_H, ISHTP_DRIVER_DATA_PTL)}, {PCI_DEVICE_DATA(INTEL, ISH_PTL_P, ISHTP_DRIVER_DATA_PTL)}, {PCI_DEVICE_DATA(INTEL, ISH_WCL, ISHTP_DRIVER_DATA_WCL)}, + {PCI_DEVICE_DATA(INTEL, ISH_NVL_H, ISHTP_DRIVER_DATA_NVL_H)}, + {PCI_DEVICE_DATA(INTEL, ISH_NVL_S, ISHTP_DRIVER_DATA_NVL_S)}, {} }; MODULE_DEVICE_TABLE(pci, ish_pci_tbl); @@ -384,7 +396,7 @@ static int __maybe_unused ish_resume(struct device *device) ish_resume_device = device; dev->resume_flag = 1; - schedule_work(&resume_work); + queue_work(dev->unbound_wq, &resume_work); return 0; } diff --git a/drivers/hid/intel-ish-hid/ishtp-hid-client.c b/drivers/hid/intel-ish-hid/ishtp-hid-client.c index d8c3c54a8c0f2..12a43c64e8156 100644 --- a/drivers/hid/intel-ish-hid/ishtp-hid-client.c +++ b/drivers/hid/intel-ish-hid/ishtp-hid-client.c @@ -495,6 +495,7 @@ static int ishtp_enum_enum_devices(struct ishtp_cl *hid_ishtp_cl) int rv; /* Send HOSTIF_DM_ENUM_DEVICES */ + client_data->enum_devices_done = false; memset(&msg, 0, sizeof(struct hostif_msg)); msg.hdr.command = HOSTIF_DM_ENUM_DEVICES; rv = ishtp_cl_send(hid_ishtp_cl, (unsigned char *)&msg, @@ -860,7 +861,7 @@ static int hid_ishtp_cl_reset(struct ishtp_cl_device *cl_device) hid_ishtp_trace(client_data, "%s hid_ishtp_cl %p\n", __func__, hid_ishtp_cl); - schedule_work(&client_data->work); + queue_work(ishtp_get_workqueue(cl_device), &client_data->work); return 0; } @@ -902,7 +903,7 @@ static int hid_ishtp_cl_resume(struct device *device) hid_ishtp_trace(client_data, "%s hid_ishtp_cl %p\n", __func__, hid_ishtp_cl); - schedule_work(&client_data->resume_work); + queue_work(ishtp_get_workqueue(cl_device), &client_data->resume_work); return 0; } diff --git a/drivers/hid/intel-ish-hid/ishtp/bus.c b/drivers/hid/intel-ish-hid/ishtp/bus.c index 93a0432e70581..b890fbf97a75c 100644 --- a/drivers/hid/intel-ish-hid/ishtp/bus.c +++ b/drivers/hid/intel-ish-hid/ishtp/bus.c @@ -240,9 +240,17 @@ static int ishtp_cl_bus_match(struct device *dev, const struct device_driver *dr { struct ishtp_cl_device *device = to_ishtp_cl_device(dev); struct ishtp_cl_driver *driver = to_ishtp_cl_driver(drv); + struct ishtp_fw_client *client = device->fw_client; + const struct ishtp_device_id *id; - return(device->fw_client ? guid_equal(&driver->id[0].guid, - &device->fw_client->props.protocol_name) : 0); + if (client) { + for (id = driver->id; !guid_is_null(&id->guid); id++) { + if (guid_equal(&id->guid, &client->props.protocol_name)) + return 1; + } + } + + return 0; } /** @@ -541,7 +549,7 @@ void ishtp_cl_bus_rx_event(struct ishtp_cl_device *device) return; if (device->event_cb) - schedule_work(&device->event_work); + queue_work(device->ishtp_dev->unbound_wq, &device->event_work); } /** @@ -722,7 +730,7 @@ void ishtp_bus_remove_all_clients(struct ishtp_device *ishtp_dev, spin_lock_irqsave(&ishtp_dev->cl_list_lock, flags); list_for_each_entry(cl, &ishtp_dev->cl_list, link) { cl->state = ISHTP_CL_DISCONNECTED; - if (warm_reset && cl->device->reference_count) + if (warm_reset && cl->device && cl->device->reference_count) continue; /* @@ -876,6 +884,22 @@ struct device *ishtp_get_pci_device(struct ishtp_cl_device *device) } EXPORT_SYMBOL(ishtp_get_pci_device); +/** + * ishtp_get_workqueue - Retrieve the workqueue associated with an ISHTP device + * @cl_device: Pointer to the ISHTP client device structure + * + * Returns the workqueue_struct pointer (unbound_wq) associated with the given + * ISHTP client device. This workqueue is typically used for scheduling work + * related to the device. + * + * Return: Pointer to struct workqueue_struct. + */ +struct workqueue_struct *ishtp_get_workqueue(struct ishtp_cl_device *cl_device) +{ + return cl_device->ishtp_dev->unbound_wq; +} +EXPORT_SYMBOL(ishtp_get_workqueue); + /** * ishtp_trace_callback() - Return trace callback * @cl_device: ISH-TP client device instance diff --git a/drivers/hid/intel-ish-hid/ishtp/hbm.c b/drivers/hid/intel-ish-hid/ishtp/hbm.c index 8ee5467127d87..97c4fcd9e3c6f 100644 --- a/drivers/hid/intel-ish-hid/ishtp/hbm.c +++ b/drivers/hid/intel-ish-hid/ishtp/hbm.c @@ -573,7 +573,7 @@ void ishtp_hbm_dispatch(struct ishtp_device *dev, /* Start firmware loading process if it has loader capability */ if (version_res->host_version_supported & ISHTP_SUPPORT_CAP_LOADER) - schedule_work(&dev->work_fw_loader); + queue_work(dev->unbound_wq, &dev->work_fw_loader); dev->version.major_version = HBM_MAJOR_VERSION; dev->version.minor_version = HBM_MINOR_VERSION; @@ -864,7 +864,7 @@ void recv_hbm(struct ishtp_device *dev, struct ishtp_msg_hdr *ishtp_hdr) dev->rd_msg_fifo_tail = (dev->rd_msg_fifo_tail + IPC_PAYLOAD_SIZE) % (RD_INT_FIFO_SIZE * IPC_PAYLOAD_SIZE); spin_unlock_irqrestore(&dev->rd_msg_spinlock, flags); - schedule_work(&dev->bh_hbm_work); + queue_work(dev->unbound_wq, &dev->bh_hbm_work); eoi: return; } diff --git a/drivers/hid/intel-ish-hid/ishtp/ishtp-dev.h b/drivers/hid/intel-ish-hid/ishtp/ishtp-dev.h index 23db97ecf21cd..4b0596eadf1cc 100644 --- a/drivers/hid/intel-ish-hid/ishtp/ishtp-dev.h +++ b/drivers/hid/intel-ish-hid/ishtp/ishtp-dev.h @@ -175,6 +175,9 @@ struct ishtp_device { struct hbm_version version; int transfer_path; /* Choice of transfer path: IPC or DMA */ + /* Alloc a dedicated unbound workqueue for ishtp device */ + struct workqueue_struct *unbound_wq; + /* work structure for scheduling firmware loading tasks */ struct work_struct work_fw_loader; /* waitq for waiting for command response from the firmware loader */ diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c index 14cabd5dc6ddb..f0830a56d556b 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c @@ -37,6 +37,10 @@ struct quickspi_driver_data arl = { .max_packet_size_value = MAX_PACKET_SIZE_VALUE_MTL, }; +struct quickspi_driver_data nvl = { + .max_packet_size_value = MAX_PACKET_SIZE_VALUE_LNL, +}; + /* THC QuickSPI ACPI method to get device properties */ /* HIDSPI Method: {6e2ac436-0fcf-41af-a265-b32a220dcfab} */ static guid_t hidspi_guid = @@ -984,6 +988,8 @@ static const struct pci_device_id quickspi_pci_tbl[] = { {PCI_DEVICE_DATA(INTEL, THC_WCL_DEVICE_ID_SPI_PORT2, &ptl), }, {PCI_DEVICE_DATA(INTEL, THC_ARL_DEVICE_ID_SPI_PORT1, &arl), }, {PCI_DEVICE_DATA(INTEL, THC_ARL_DEVICE_ID_SPI_PORT2, &arl), }, + {PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_SPI_PORT1, &nvl), }, + {PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_SPI_PORT2, &nvl), }, {} }; MODULE_DEVICE_TABLE(pci, quickspi_pci_tbl); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h index c30e1a42eb098..bf5e18f5a5f42 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h @@ -23,6 +23,8 @@ #define PCI_DEVICE_ID_INTEL_THC_WCL_DEVICE_ID_SPI_PORT2 0x4D4B #define PCI_DEVICE_ID_INTEL_THC_ARL_DEVICE_ID_SPI_PORT1 0x7749 #define PCI_DEVICE_ID_INTEL_THC_ARL_DEVICE_ID_SPI_PORT2 0x774B +#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_SPI_PORT1 0xD349 +#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_SPI_PORT2 0xD34B /* HIDSPI special ACPI parameters DSM methods */ #define ACPI_QUICKSPI_REVISION_NUM 2 diff --git a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c index 636a683065015..7e220a4c5ded7 100644 --- a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c +++ b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c @@ -1593,7 +1593,7 @@ int thc_i2c_set_rx_max_size(struct thc_device *dev, u32 max_rx_size) if (!max_rx_size) return -EOPNOTSUPP; - ret = regmap_read(dev->thc_regmap, THC_M_PRT_SW_SEQ_STS_OFFSET, &val); + ret = regmap_read(dev->thc_regmap, THC_M_PRT_SPI_ICRRD_OPCODE_OFFSET, &val); if (ret) return ret; @@ -1662,7 +1662,7 @@ int thc_i2c_set_rx_int_delay(struct thc_device *dev, u32 delay_us) if (!delay_us) return -EOPNOTSUPP; - ret = regmap_read(dev->thc_regmap, THC_M_PRT_SW_SEQ_STS_OFFSET, &val); + ret = regmap_read(dev->thc_regmap, THC_M_PRT_SPI_ICRRD_OPCODE_OFFSET, &val); if (ret) return ret; diff --git a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.c b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.c index 82b8854843e05..6ee675e0a7384 100644 --- a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.c +++ b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.c @@ -232,6 +232,7 @@ static int setup_dma_buffers(struct thc_device *dev, return 0; memset(config->sgls, 0, sizeof(config->sgls)); + memset(config->sgls_nent_pages, 0, sizeof(config->sgls_nent_pages)); memset(config->sgls_nent, 0, sizeof(config->sgls_nent)); cpu_addr = dma_alloc_coherent(dev->dev, prd_tbls_size, @@ -254,6 +255,7 @@ static int setup_dma_buffers(struct thc_device *dev, } count = dma_map_sg(dev->dev, config->sgls[i], nent, dir); + config->sgls_nent_pages[i] = nent; config->sgls_nent[i] = count; } @@ -299,7 +301,7 @@ static void release_dma_buffers(struct thc_device *dev, continue; dma_unmap_sg(dev->dev, config->sgls[i], - config->sgls_nent[i], + config->sgls_nent_pages[i], config->dir); sgl_free(config->sgls[i]); @@ -573,6 +575,11 @@ static int read_dma_buffer(struct thc_device *dev, return -EINVAL; } + if (!read_config->prd_tbls || !read_config->sgls[prd_table_index]) { + dev_err_once(dev->dev, "PRD tables are not ready yet\n"); + return -EINVAL; + } + prd_tbl = &read_config->prd_tbls[prd_table_index]; mes_len = calc_message_len(prd_tbl, &nent); if (mes_len > read_config->max_packet_size) { diff --git a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.h b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.h index 78917400492ca..541d33995baf3 100644 --- a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.h +++ b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dma.h @@ -91,6 +91,7 @@ struct thc_prd_table { * @dir: Direction of DMA for this config * @prd_tbls: PRD tables for current DMA * @sgls: Array of pointers to scatter-gather lists + * @sgls_nent_pages: Number of pages per scatter-gather list * @sgls_nent: Actual number of entries per scatter-gather list * @prd_tbl_num: Actual number of PRD tables * @max_packet_size: Size of the buffer needed for 1 DMA message (1 PRD table) @@ -107,6 +108,7 @@ struct thc_dma_configuration { struct thc_prd_table *prd_tbls; struct scatterlist *sgls[PRD_TABLES_NUM]; + u8 sgls_nent_pages[PRD_TABLES_NUM]; u8 sgls_nent[PRD_TABLES_NUM]; u8 prd_tbl_num; diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c index aac0051a2cf65..758eb21430cda 100644 --- a/drivers/hid/usbhid/hid-core.c +++ b/drivers/hid/usbhid/hid-core.c @@ -985,6 +985,7 @@ static int usbhid_parse(struct hid_device *hid) struct usb_device *dev = interface_to_usbdev (intf); struct hid_descriptor *hdesc; struct hid_class_descriptor *hcdesc; + __u8 fixed_opt_descriptors_size; u32 quirks = 0; unsigned int rsize = 0; char *rdesc; @@ -1015,7 +1016,21 @@ static int usbhid_parse(struct hid_device *hid) (hdesc->bNumDescriptors - 1) * sizeof(*hcdesc)) { dbg_hid("hid descriptor invalid, bLen=%hhu bNum=%hhu\n", hdesc->bLength, hdesc->bNumDescriptors); - return -EINVAL; + + /* + * Some devices may expose a wrong number of descriptors compared + * to the provided length. + * However, we ignore the optional hid class descriptors entirely + * so we can safely recompute the proper field. + */ + if (hdesc->bLength >= sizeof(*hdesc)) { + fixed_opt_descriptors_size = hdesc->bLength - sizeof(*hdesc); + + hid_warn(intf, "fixing wrong optional hid class descriptors count\n"); + hdesc->bNumDescriptors = fixed_opt_descriptors_size / sizeof(*hcdesc) + 1; + } else { + return -EINVAL; + } } hid->version = le16_to_cpu(hdesc->bcdHID); diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c index 95377c5f63356..56d6af39ba81e 100644 --- a/drivers/hid/usbhid/hid-pidff.c +++ b/drivers/hid/usbhid/hid-pidff.c @@ -523,9 +523,19 @@ static void pidff_set_effect_report(struct pidff_device *pidff, pidff_set_duration(&pidff->set_effect[PID_DURATION], effect->replay.length); - pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = effect->trigger.button; - pidff_set_time(&pidff->set_effect[PID_TRIGGER_REPEAT_INT], - effect->trigger.interval); + /* Some games set this to random values that can be out of range */ + s32 trigger_button_max = + pidff->set_effect[PID_TRIGGER_BUTTON].field->logical_maximum; + if (effect->trigger.button <= trigger_button_max) { + pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = + effect->trigger.button; + pidff_set_time(&pidff->set_effect[PID_TRIGGER_REPEAT_INT], + effect->trigger.interval); + } else { + pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = 0; + pidff->set_effect[PID_TRIGGER_REPEAT_INT].value[0] = 0; + } + pidff->set_effect[PID_GAIN].value[0] = pidff->set_effect[PID_GAIN].field->logical_maximum; @@ -1442,10 +1452,13 @@ static int pidff_init_fields(struct pidff_device *pidff, struct input_dev *dev) hid_warn(pidff->hid, "unknown ramp effect layout\n"); if (PIDFF_FIND_FIELDS(set_condition, PID_SET_CONDITION, 1)) { - if (test_and_clear_bit(FF_SPRING, dev->ffbit) || - test_and_clear_bit(FF_DAMPER, dev->ffbit) || - test_and_clear_bit(FF_FRICTION, dev->ffbit) || - test_and_clear_bit(FF_INERTIA, dev->ffbit)) + bool test = false; + + test |= test_and_clear_bit(FF_SPRING, dev->ffbit); + test |= test_and_clear_bit(FF_DAMPER, dev->ffbit); + test |= test_and_clear_bit(FF_FRICTION, dev->ffbit); + test |= test_and_clear_bit(FF_INERTIA, dev->ffbit); + if (test) hid_warn(pidff->hid, "unknown condition effect layout\n"); } diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index 9b2c710f8da18..da1f0ea85625d 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -1208,10 +1208,20 @@ static int wacom_intuos_bt_irq(struct wacom_wac *wacom, size_t len) switch (data[0]) { case 0x04: + if (len < 32) { + dev_warn(wacom->pen_input->dev.parent, + "Report 0x04 too short: %zu bytes\n", len); + break; + } wacom_intuos_bt_process_data(wacom, data + i); i += 10; fallthrough; case 0x03: + if (i == 1 && len < 22) { + dev_warn(wacom->pen_input->dev.parent, + "Report 0x03 too short: %zu bytes\n", len); + break; + } wacom_intuos_bt_process_data(wacom, data + i); i += 10; wacom_intuos_bt_process_data(wacom, data + i); diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index e109a620c83fc..71fd3ea4fa8bd 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -195,13 +195,15 @@ static void hv_kmsg_dump(struct kmsg_dumper *dumper, /* * Write dump contents to the page. No need to synchronize; panic should - * be single-threaded. + * be single-threaded. Ignore failures from kmsg_dump_get_buffer() since + * panic notification should be done even if there is no message data. + * Don't assume bytes_written is set in case of failure, so initialize it. */ kmsg_dump_rewind(&iter); - kmsg_dump_get_buffer(&iter, false, hv_panic_page, HV_HYP_PAGE_SIZE, + bytes_written = 0; + (void)kmsg_dump_get_buffer(&iter, false, hv_panic_page, HV_HYP_PAGE_SIZE, &bytes_written); - if (!bytes_written) - return; + /* * P3 to contain the physical address of the panic page & P4 to * contain the size of the panic data in that page. Rest of the @@ -210,7 +212,7 @@ static void hv_kmsg_dump(struct kmsg_dumper *dumper, hv_set_msr(HV_MSR_CRASH_P0, 0); hv_set_msr(HV_MSR_CRASH_P1, 0); hv_set_msr(HV_MSR_CRASH_P2, 0); - hv_set_msr(HV_MSR_CRASH_P3, virt_to_phys(hv_panic_page)); + hv_set_msr(HV_MSR_CRASH_P3, bytes_written ? virt_to_phys(hv_panic_page) : 0); hv_set_msr(HV_MSR_CRASH_P4, bytes_written); /* diff --git a/drivers/hv/mshv_eventfd.c b/drivers/hv/mshv_eventfd.c index 806674722868b..40b7179fee805 100644 --- a/drivers/hv/mshv_eventfd.c +++ b/drivers/hv/mshv_eventfd.c @@ -87,8 +87,9 @@ static void mshv_irqfd_resampler_ack(struct mshv_irq_ack_notifier *mian) idx = srcu_read_lock(&partition->pt_irq_srcu); - hlist_for_each_entry_rcu(irqfd, &resampler->rsmplr_irqfd_list, - irqfd_resampler_hnode) { + hlist_for_each_entry_srcu(irqfd, &resampler->rsmplr_irqfd_list, + irqfd_resampler_hnode, + srcu_read_lock_held(&partition->pt_irq_srcu)) { if (hv_should_clear_interrupt(irqfd->irqfd_lapic_irq.lapic_control.interrupt_type)) hv_call_clear_virtual_interrupt(partition->pt_id); @@ -243,12 +244,13 @@ static void mshv_irqfd_shutdown(struct work_struct *work) { struct mshv_irqfd *irqfd = container_of(work, struct mshv_irqfd, irqfd_shutdown); + u64 cnt; /* * Synchronize with the wait-queue and unhook ourselves to prevent * further events. */ - remove_wait_queue(irqfd->irqfd_wqh, &irqfd->irqfd_wait); + eventfd_ctx_remove_wait_queue(irqfd->irqfd_eventfd_ctx, &irqfd->irqfd_wait, &cnt); if (irqfd->irqfd_resampler) { mshv_irqfd_resampler_shutdown(irqfd); @@ -367,8 +369,6 @@ static void mshv_irqfd_queue_proc(struct file *file, wait_queue_head_t *wqh, struct mshv_irqfd *irqfd = container_of(polltbl, struct mshv_irqfd, irqfd_polltbl); - irqfd->irqfd_wqh = wqh; - /* * TODO: Ensure there isn't already an exclusive, priority waiter, e.g. * that the irqfd isn't already bound to another partition. Only the diff --git a/drivers/hv/mshv_eventfd.h b/drivers/hv/mshv_eventfd.h index 332e7670a3442..464c6b81ab336 100644 --- a/drivers/hv/mshv_eventfd.h +++ b/drivers/hv/mshv_eventfd.h @@ -32,7 +32,6 @@ struct mshv_irqfd { struct mshv_lapic_irq irqfd_lapic_irq; struct hlist_node irqfd_hnode; poll_table irqfd_polltbl; - wait_queue_head_t *irqfd_wqh; wait_queue_entry_t irqfd_wait; struct work_struct irqfd_shutdown; struct mshv_irqfd_resampler *irqfd_resampler; diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c index e3b2bd417c464..4e04bef544379 100644 --- a/drivers/hv/mshv_root_main.c +++ b/drivers/hv/mshv_root_main.c @@ -159,6 +159,7 @@ static int mshv_ioctl_passthru_hvcall(struct mshv_partition *partition, unsigned int pages_order; void *input_pg = NULL; void *output_pg = NULL; + u16 reps_completed; if (copy_from_user(&args, user_args, sizeof(args))) return -EFAULT; @@ -210,41 +211,42 @@ static int mshv_ioctl_passthru_hvcall(struct mshv_partition *partition, */ *(u64 *)input_pg = partition->pt_id; - if (args.reps) - status = hv_do_rep_hypercall(args.code, args.reps, 0, - input_pg, output_pg); - else - status = hv_do_hypercall(args.code, input_pg, output_pg); - - if (hv_result(status) == HV_STATUS_CALL_PENDING) { - if (is_async) { - mshv_async_hvcall_handler(partition, &status); - } else { /* Paranoia check. This shouldn't happen! */ - ret = -EBADFD; - goto free_pages_out; + reps_completed = 0; + do { + if (args.reps) { + status = hv_do_rep_hypercall_ex(args.code, args.reps, + 0, reps_completed, + input_pg, output_pg); + reps_completed = hv_repcomp(status); + } else { + status = hv_do_hypercall(args.code, input_pg, output_pg); } - } - if (hv_result(status) == HV_STATUS_INSUFFICIENT_MEMORY) { - ret = hv_call_deposit_pages(NUMA_NO_NODE, partition->pt_id, 1); - if (!ret) - ret = -EAGAIN; - } else if (!hv_result_success(status)) { - ret = hv_result_to_errno(status); - } + if (hv_result(status) == HV_STATUS_CALL_PENDING) { + if (is_async) { + mshv_async_hvcall_handler(partition, &status); + } else { /* Paranoia check. This shouldn't happen! */ + ret = -EBADFD; + goto free_pages_out; + } + } + + if (hv_result_success(status)) + break; + + if (hv_result(status) != HV_STATUS_INSUFFICIENT_MEMORY) + ret = hv_result_to_errno(status); + else + ret = hv_call_deposit_pages(NUMA_NO_NODE, + partition->pt_id, 1); + } while (!ret); - /* - * Always return the status and output data regardless of result. - * The VMM may need it to determine how to proceed. E.g. the status may - * contain the number of reps completed if a rep hypercall partially - * succeeded. - */ args.status = hv_result(status); - args.reps = args.reps ? hv_repcomp(status) : 0; + args.reps = reps_completed; if (copy_to_user(user_args, &args, sizeof(args))) ret = -EFAULT; - if (output_pg && + if (!ret && output_pg && copy_to_user((void __user *)args.out_ptr, output_pg, args.out_sz)) ret = -EFAULT; @@ -1172,21 +1174,6 @@ mshv_partition_region_by_gfn(struct mshv_partition *partition, u64 gfn) return NULL; } -static struct mshv_mem_region * -mshv_partition_region_by_uaddr(struct mshv_partition *partition, u64 uaddr) -{ - struct mshv_mem_region *region; - - hlist_for_each_entry(region, &partition->pt_mem_regions, hnode) { - if (uaddr >= region->start_uaddr && - uaddr < region->start_uaddr + - (region->nr_pages << HV_HYP_PAGE_SHIFT)) - return region; - } - - return NULL; -} - /* * NB: caller checks and makes sure mem->size is page aligned * Returns: 0 with regionpp updated on success, or -errno @@ -1196,15 +1183,21 @@ static int mshv_partition_create_region(struct mshv_partition *partition, struct mshv_mem_region **regionpp, bool is_mmio) { - struct mshv_mem_region *region; + struct mshv_mem_region *region, *rg; u64 nr_pages = HVPFN_DOWN(mem->size); /* Reject overlapping regions */ - if (mshv_partition_region_by_gfn(partition, mem->guest_pfn) || - mshv_partition_region_by_gfn(partition, mem->guest_pfn + nr_pages - 1) || - mshv_partition_region_by_uaddr(partition, mem->userspace_addr) || - mshv_partition_region_by_uaddr(partition, mem->userspace_addr + mem->size - 1)) + hlist_for_each_entry(rg, &partition->pt_mem_regions, hnode) { + u64 rg_size = rg->nr_pages << HV_HYP_PAGE_SHIFT; + + if ((mem->guest_pfn + nr_pages <= rg->start_gfn || + rg->start_gfn + rg->nr_pages <= mem->guest_pfn) && + (mem->userspace_addr + mem->size <= rg->start_uaddr || + rg->start_uaddr + rg_size <= mem->userspace_addr)) + continue; + return -EEXIST; + } region = vzalloc(sizeof(*region) + sizeof(struct page *) * nr_pages); if (!region) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 69591dc7bad26..3ab62277b6be6 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -1306,7 +1307,7 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) } } -static void vmbus_isr(void) +static void __vmbus_isr(void) { struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context); @@ -1330,6 +1331,53 @@ static void vmbus_isr(void) add_interrupt_randomness(vmbus_interrupt); } +static DEFINE_PER_CPU(bool, vmbus_irq_pending); +static DEFINE_PER_CPU(struct task_struct *, vmbus_irqd); + +static void vmbus_irqd_wake(void) +{ + struct task_struct *tsk = __this_cpu_read(vmbus_irqd); + + __this_cpu_write(vmbus_irq_pending, true); + wake_up_process(tsk); +} + +static void vmbus_irqd_setup(unsigned int cpu) +{ + sched_set_fifo(current); +} + +static int vmbus_irqd_should_run(unsigned int cpu) +{ + return __this_cpu_read(vmbus_irq_pending); +} + +static void run_vmbus_irqd(unsigned int cpu) +{ + __this_cpu_write(vmbus_irq_pending, false); + __vmbus_isr(); +} + +static bool vmbus_irq_initialized; + +static struct smp_hotplug_thread vmbus_irq_threads = { + .store = &vmbus_irqd, + .setup = vmbus_irqd_setup, + .thread_should_run = vmbus_irqd_should_run, + .thread_fn = run_vmbus_irqd, + .thread_comm = "vmbus_irq/%u", +}; + +static void vmbus_isr(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + vmbus_irqd_wake(); + } else { + lockdep_hardirq_threaded(); + __vmbus_isr(); + } +} + static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id) { vmbus_isr(); @@ -1375,6 +1423,13 @@ static int vmbus_bus_init(void) * the VMbus interrupt handler. */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && !vmbus_irq_initialized) { + ret = smpboot_register_percpu_thread(&vmbus_irq_threads); + if (ret) + goto err_kthread; + vmbus_irq_initialized = true; + } + if (vmbus_irq == -1) { hv_setup_vmbus_handler(vmbus_isr); } else { @@ -1449,6 +1504,11 @@ static int vmbus_bus_init(void) free_percpu(vmbus_evt); } err_setup: + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } +err_kthread: bus_unregister(&hv_bus); return ret; } @@ -2914,6 +2974,10 @@ static void __exit vmbus_exit(void) free_percpu_irq(vmbus_irq, vmbus_evt); free_percpu(vmbus_evt); } + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } for_each_online_cpu(cpu) { struct hv_per_cpu_context *hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 2760feb9f83b5..fdb3ecc2bb041 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -245,12 +245,12 @@ config SENSORS_ADT7475 will be called adt7475. config SENSORS_AHT10 - tristate "Aosong AHT10, AHT20" + tristate "Aosong AHT10, AHT20, DHT20" depends on I2C select CRC8 help - If you say yes here, you get support for the Aosong AHT10 and AHT20 - temperature and humidity sensors + If you say yes here, you get support for the Aosong AHT10, AHT20 and + DHT20 temperature and humidity sensors This driver can also be built as a module. If so, the module will be called aht10. @@ -1893,6 +1893,15 @@ config SENSORS_PWM_FAN This driver can also be built as a module. If so, the module will be called pwm-fan. +config SENSORS_QCOM_BCL + tristate "Qualcomm BCL hardware monitoring" + help + Say yes here to enable support for Qualcomm battery over current + and under voltage alarms monitor. + + This driver can also be built as a module. If so, the module + will be called qcom-bcl-hwmon. + config SENSORS_QNAP_MCU_HWMON tristate "QNAP MCU hardware monitoring" depends on MFD_QNAP_MCU diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 73b2abdcc6dd9..e2d01688fdb75 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -196,6 +196,7 @@ obj-$(CONFIG_SENSORS_POWERZ) += powerz.o obj-$(CONFIG_SENSORS_POWR1220) += powr1220.o obj-$(CONFIG_SENSORS_PT5161L) += pt5161l.o obj-$(CONFIG_SENSORS_PWM_FAN) += pwm-fan.o +obj-$(CONFIG_SENSORS_QCOM_BCL) += qcom-bcl-hwmon.o obj-$(CONFIG_SENSORS_QNAP_MCU_HWMON) += qnap-mcu-hwmon.o obj-$(CONFIG_SENSORS_RASPBERRYPI_HWMON) += raspberrypi-hwmon.o obj-$(CONFIG_SENSORS_SA67MCU) += sa67mcu-hwmon.o diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c index 29ccdc2fb7ff8..de408df0c4d78 100644 --- a/drivers/hwmon/acpi_power_meter.c +++ b/drivers/hwmon/acpi_power_meter.c @@ -47,6 +47,8 @@ static int cap_in_hardware; static bool force_cap_on; +static DEFINE_MUTEX(acpi_notify_lock); + static int can_cap_in_hardware(void) { return force_cap_on || cap_in_hardware; @@ -823,18 +825,26 @@ static void acpi_power_meter_notify(struct acpi_device *device, u32 event) resource = acpi_driver_data(device); + guard(mutex)(&acpi_notify_lock); + switch (event) { case METER_NOTIFY_CONFIG: + if (!IS_ERR(resource->hwmon_dev)) + hwmon_device_unregister(resource->hwmon_dev); + mutex_lock(&resource->lock); + free_capabilities(resource); remove_domain_devices(resource); - hwmon_device_unregister(resource->hwmon_dev); res = read_capabilities(resource); if (res) dev_err_once(&device->dev, "read capabilities failed.\n"); res = read_domain_devices(resource); if (res && res != -ENODEV) dev_err_once(&device->dev, "read domain devices failed.\n"); + + mutex_unlock(&resource->lock); + resource->hwmon_dev = hwmon_device_register_with_info(&device->dev, ACPI_POWER_METER_NAME, @@ -843,7 +853,7 @@ static void acpi_power_meter_notify(struct acpi_device *device, u32 event) power_extra_groups); if (IS_ERR(resource->hwmon_dev)) dev_err_once(&device->dev, "register hwmon device failed.\n"); - mutex_unlock(&resource->lock); + break; case METER_NOTIFY_TRIP: sysfs_notify(&device->dev.kobj, NULL, POWER_AVERAGE_NAME); @@ -953,7 +963,8 @@ static void acpi_power_meter_remove(struct acpi_device *device) return; resource = acpi_driver_data(device); - hwmon_device_unregister(resource->hwmon_dev); + if (!IS_ERR(resource->hwmon_dev)) + hwmon_device_unregister(resource->hwmon_dev); remove_domain_devices(resource); free_capabilities(resource); diff --git a/drivers/hwmon/adm1177.c b/drivers/hwmon/adm1177.c index 8b2c965480e3f..7888afe8dafd6 100644 --- a/drivers/hwmon/adm1177.c +++ b/drivers/hwmon/adm1177.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include @@ -33,7 +35,7 @@ struct adm1177_state { struct i2c_client *client; u32 r_sense_uohm; - u32 alert_threshold_ua; + u64 alert_threshold_ua; bool vrange_high; }; @@ -48,7 +50,7 @@ static int adm1177_write_cmd(struct adm1177_state *st, u8 cmd) } static int adm1177_write_alert_thr(struct adm1177_state *st, - u32 alert_threshold_ua) + u64 alert_threshold_ua) { u64 val; int ret; @@ -91,8 +93,8 @@ static int adm1177_read(struct device *dev, enum hwmon_sensor_types type, *val = div_u64((105840000ull * dummy), 4096 * st->r_sense_uohm); return 0; - case hwmon_curr_max_alarm: - *val = st->alert_threshold_ua; + case hwmon_curr_max: + *val = div_u64(st->alert_threshold_ua, 1000); return 0; default: return -EOPNOTSUPP; @@ -126,9 +128,10 @@ static int adm1177_write(struct device *dev, enum hwmon_sensor_types type, switch (type) { case hwmon_curr: switch (attr) { - case hwmon_curr_max_alarm: - adm1177_write_alert_thr(st, val); - return 0; + case hwmon_curr_max: + val = clamp_val(val, 0, + div_u64(105840000ULL, st->r_sense_uohm)); + return adm1177_write_alert_thr(st, (u64)val * 1000); default: return -EOPNOTSUPP; } @@ -156,7 +159,7 @@ static umode_t adm1177_is_visible(const void *data, if (st->r_sense_uohm) return 0444; return 0; - case hwmon_curr_max_alarm: + case hwmon_curr_max: if (st->r_sense_uohm) return 0644; return 0; @@ -170,7 +173,7 @@ static umode_t adm1177_is_visible(const void *data, static const struct hwmon_channel_info * const adm1177_info[] = { HWMON_CHANNEL_INFO(curr, - HWMON_C_INPUT | HWMON_C_MAX_ALARM), + HWMON_C_INPUT | HWMON_C_MAX), HWMON_CHANNEL_INFO(in, HWMON_I_INPUT), NULL @@ -192,7 +195,8 @@ static int adm1177_probe(struct i2c_client *client) struct device *dev = &client->dev; struct device *hwmon_dev; struct adm1177_state *st; - u32 alert_threshold_ua; + u64 alert_threshold_ua; + u32 prop; int ret; st = devm_kzalloc(dev, sizeof(*st), GFP_KERNEL); @@ -208,22 +212,26 @@ static int adm1177_probe(struct i2c_client *client) if (device_property_read_u32(dev, "shunt-resistor-micro-ohms", &st->r_sense_uohm)) st->r_sense_uohm = 0; - if (device_property_read_u32(dev, "adi,shutdown-threshold-microamp", - &alert_threshold_ua)) { - if (st->r_sense_uohm) - /* - * set maximum default value from datasheet based on - * shunt-resistor - */ - alert_threshold_ua = div_u64(105840000000, - st->r_sense_uohm); - else - alert_threshold_ua = 0; + if (!device_property_read_u32(dev, "adi,shutdown-threshold-microamp", + &prop)) { + alert_threshold_ua = prop; + } else if (st->r_sense_uohm) { + /* + * set maximum default value from datasheet based on + * shunt-resistor + */ + alert_threshold_ua = div_u64(105840000000ULL, + st->r_sense_uohm); + } else { + alert_threshold_ua = 0; } st->vrange_high = device_property_read_bool(dev, "adi,vrange-high-enable"); - if (alert_threshold_ua && st->r_sense_uohm) - adm1177_write_alert_thr(st, alert_threshold_ua); + if (alert_threshold_ua && st->r_sense_uohm) { + ret = adm1177_write_alert_thr(st, alert_threshold_ua); + if (ret) + return ret; + } ret = adm1177_write_cmd(st, ADM1177_CMD_V_CONT | ADM1177_CMD_I_CONT | diff --git a/drivers/hwmon/aht10.c b/drivers/hwmon/aht10.c index d1c55e2eb4799..7ba00bafb57d6 100644 --- a/drivers/hwmon/aht10.c +++ b/drivers/hwmon/aht10.c @@ -37,6 +37,10 @@ #define AHT10_CMD_MEAS 0b10101100 #define AHT10_CMD_RST 0b10111010 +#define AHT20_CMD_INIT 0b10111110 + +#define DHT20_CMD_INIT 0b01110001 + /* * Flags in the answer byte/command */ @@ -48,11 +52,12 @@ #define AHT10_MAX_POLL_INTERVAL_LEN 30 -enum aht10_variant { aht10, aht20 }; +enum aht10_variant { aht10, aht20, dht20}; static const struct i2c_device_id aht10_id[] = { { "aht10", aht10 }, { "aht20", aht20 }, + { "dht20", dht20 }, { }, }; MODULE_DEVICE_TABLE(i2c, aht10_id); @@ -77,6 +82,7 @@ MODULE_DEVICE_TABLE(i2c, aht10_id); * AHT10/AHT20 * @crc8: crc8 support flag * @meas_size: measurements data size + * @init_cmd: Initialization command */ struct aht10_data { @@ -92,6 +98,7 @@ struct aht10_data { int humidity; bool crc8; unsigned int meas_size; + u8 init_cmd; }; /* @@ -101,13 +108,13 @@ struct aht10_data { */ static int aht10_init(struct aht10_data *data) { - const u8 cmd_init[] = {AHT10_CMD_INIT, AHT10_CAL_ENABLED | AHT10_MODE_CYC, + const u8 cmd_init[] = {data->init_cmd, AHT10_CAL_ENABLED | AHT10_MODE_CYC, 0x00}; int res; u8 status; struct i2c_client *client = data->client; - res = i2c_master_send(client, cmd_init, 3); + res = i2c_master_send(client, cmd_init, sizeof(cmd_init)); if (res < 0) return res; @@ -352,9 +359,17 @@ static int aht10_probe(struct i2c_client *client) data->meas_size = AHT20_MEAS_SIZE; data->crc8 = true; crc8_populate_msb(crc8_table, AHT20_CRC8_POLY); + data->init_cmd = AHT20_CMD_INIT; + break; + case dht20: + data->meas_size = AHT20_MEAS_SIZE; + data->crc8 = true; + crc8_populate_msb(crc8_table, AHT20_CRC8_POLY); + data->init_cmd = DHT20_CMD_INIT; break; default: data->meas_size = AHT10_MEAS_SIZE; + data->init_cmd = AHT10_CMD_INIT; break; } diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c index 34a8f6b834c97..95c50d3a788ce 100644 --- a/drivers/hwmon/asus-ec-sensors.c +++ b/drivers/hwmon/asus-ec-sensors.c @@ -111,6 +111,8 @@ enum ec_sensors { ec_sensor_temp_mb, /* "T_Sensor" temperature sensor reading [℃] */ ec_sensor_temp_t_sensor, + /* like ec_sensor_temp_t_sensor, but at an alternate address [℃] */ + ec_sensor_temp_t_sensor_alt1, /* VRM temperature [℃] */ ec_sensor_temp_vrm, /* CPU Core voltage [mV] */ @@ -156,6 +158,7 @@ enum ec_sensors { #define SENSOR_TEMP_CPU_PACKAGE BIT(ec_sensor_temp_cpu_package) #define SENSOR_TEMP_MB BIT(ec_sensor_temp_mb) #define SENSOR_TEMP_T_SENSOR BIT(ec_sensor_temp_t_sensor) +#define SENSOR_TEMP_T_SENSOR_ALT1 BIT(ec_sensor_temp_t_sensor_alt1) #define SENSOR_TEMP_VRM BIT(ec_sensor_temp_vrm) #define SENSOR_IN_CPU_CORE BIT(ec_sensor_in_cpu_core) #define SENSOR_FAN_CPU_OPT BIT(ec_sensor_fan_cpu_opt) @@ -272,6 +275,8 @@ static const struct ec_sensor_info sensors_family_amd_600[] = { EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x33), [ec_sensor_temp_t_sensor] = EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x36), + [ec_sensor_temp_t_sensor_alt1] = + EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x37), [ec_sensor_fan_cpu_opt] = EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0), [ec_sensor_temp_water_in] = @@ -489,7 +494,7 @@ static const struct ec_board_info board_info_prime_x570_pro = { static const struct ec_board_info board_info_prime_x670e_pro_wifi = { .sensors = SENSOR_TEMP_CPU | SENSOR_TEMP_CPU_PACKAGE | SENSOR_TEMP_MB | SENSOR_TEMP_VRM | - SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CPU_OPT, + SENSOR_TEMP_T_SENSOR_ALT1 | SENSOR_FAN_CPU_OPT, .mutex_path = ACPI_GLOBAL_LOCK_PSEUDO_PATH, .family = family_amd_600_series, }; diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c index b7bb325c3ad96..01590dfa55e60 100644 --- a/drivers/hwmon/axi-fan-control.c +++ b/drivers/hwmon/axi-fan-control.c @@ -507,7 +507,7 @@ static int axi_fan_control_probe(struct platform_device *pdev) ret = devm_request_threaded_irq(&pdev->dev, ctl->irq, NULL, axi_fan_control_irq_handler, IRQF_ONESHOT | IRQF_TRIGGER_HIGH, - pdev->driver_override, ctl); + NULL, ctl); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to request an irq\n"); diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index cbe1a74a3deea..768690a597f4d 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -76,6 +76,9 @@ #define DELL_SMM_NO_TEMP 10 #define DELL_SMM_NO_FANS 4 +/* limit fan multiplier to avoid overflow */ +#define DELL_SMM_MAX_FAN_MULT (INT_MAX / U16_MAX) + struct smm_regs { unsigned int eax; unsigned int ebx; @@ -861,9 +864,9 @@ static umode_t dell_smm_is_visible(const void *drvdata, enum hwmon_sensor_types if (auto_fan) { /* * The setting affects all fans, so only create a - * single attribute. + * single attribute for the first fan channel. */ - if (channel != 1) + if (channel != 0) return 0; /* @@ -1253,6 +1256,12 @@ static int dell_smm_init_data(struct device *dev, const struct dell_smm_ops *ops data->ops = ops; /* All options must not be 0 */ data->i8k_fan_mult = fan_mult ? : I8K_FAN_MULT; + if (data->i8k_fan_mult > DELL_SMM_MAX_FAN_MULT) { + dev_err(dev, + "fan multiplier %u is too large (max %u)\n", + data->i8k_fan_mult, DELL_SMM_MAX_FAN_MULT); + return -EINVAL; + } data->i8k_fan_max = fan_max ? : I8K_FAN_HIGH; data->i8k_pwm_mult = DIV_ROUND_UP(255, data->i8k_fan_max); @@ -1316,6 +1325,13 @@ static const struct dmi_system_id i8k_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "MP061"), }, }, + { + .ident = "Dell OptiPlex 7080", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7080"), + }, + }, { .ident = "Dell OptiPlex 7060", .matches = { @@ -1621,6 +1637,14 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = { }, .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3], }, + { + .ident = "Dell G15 5510", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Dell G15 5510"), + }, + .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3], + }, { .ident = "Dell G15 5511", .matches = { diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c index 60809289f8169..67e82021da210 100644 --- a/drivers/hwmon/emc2305.c +++ b/drivers/hwmon/emc2305.c @@ -578,6 +578,7 @@ static int emc2305_of_parse_pwm_child(struct device *dev, data->pwm_output_mask |= EMC2305_OPEN_DRAIN << ch; } + of_node_put(args.np); return 0; } @@ -593,10 +594,8 @@ static int emc2305_probe_childs_from_dt(struct device *dev) for_each_child_of_node(dev->of_node, child) { if (of_property_present(child, "reg")) { ret = emc2305_of_parse_pwm_child(dev, child, data); - if (ret) { - of_node_put(child); + if (ret) continue; - } count++; } } @@ -685,8 +684,10 @@ static int emc2305_probe(struct i2c_client *client) i = 0; for_each_child_of_node(dev->of_node, child) { ret = emc2305_set_single_tz(dev, child, i); - if (ret != 0) + if (ret != 0) { + of_node_put(child); return ret; + } i++; } } else { diff --git a/drivers/hwmon/f71882fg.c b/drivers/hwmon/f71882fg.c index df83f9866fbcf..204059d2de6cd 100644 --- a/drivers/hwmon/f71882fg.c +++ b/drivers/hwmon/f71882fg.c @@ -51,6 +51,7 @@ #define SIO_F81866_ID 0x1010 /* Chipset ID */ #define SIO_F71858AD_ID 0x0903 /* Chipset ID */ #define SIO_F81966_ID 0x1502 /* Chipset ID */ +#define SIO_F81968_ID 0x1806 /* Chipset ID */ #define REGION_LENGTH 8 #define ADDR_REG_OFFSET 5 @@ -2570,6 +2571,7 @@ static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) break; case SIO_F81866_ID: case SIO_F81966_ID: + case SIO_F81968_ID: sio_data->type = f81866a; break; default: @@ -2599,9 +2601,9 @@ static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) address &= ~(REGION_LENGTH - 1); /* Ignore 3 LSB */ err = address; - pr_info("Found %s chip at %#x, revision %d\n", + pr_info("Found %s chip at %#x, revision %d, devid: %04x\n", f71882fg_names[sio_data->type], (unsigned int)address, - (int)superio_inb(sioaddr, SIO_REG_DEVREV)); + (int)superio_inb(sioaddr, SIO_REG_DEVREV), devid); exit: superio_exit(sioaddr); return err; diff --git a/drivers/hwmon/gpio-fan.c b/drivers/hwmon/gpio-fan.c index 516c34bb61c9c..a8892ced1e549 100644 --- a/drivers/hwmon/gpio-fan.c +++ b/drivers/hwmon/gpio-fan.c @@ -148,7 +148,7 @@ static int set_fan_speed(struct gpio_fan_data *fan_data, int speed_index) int ret; ret = pm_runtime_put_sync(fan_data->dev); - if (ret < 0) + if (ret < 0 && ret != -ENOSYS) return ret; } @@ -291,7 +291,7 @@ static ssize_t set_rpm(struct device *dev, struct device_attribute *attr, { struct gpio_fan_data *fan_data = dev_get_drvdata(dev); unsigned long rpm; - int ret = count; + int ret; if (kstrtoul(buf, 10, &rpm)) return -EINVAL; @@ -308,7 +308,7 @@ static ssize_t set_rpm(struct device *dev, struct device_attribute *attr, exit_unlock: mutex_unlock(&fan_data->lock); - return ret; + return ret ? ret : count; } static DEVICE_ATTR_RW(pwm1); diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c index e233aafa8856c..5cfb98a0512f0 100644 --- a/drivers/hwmon/it87.c +++ b/drivers/hwmon/it87.c @@ -3590,10 +3590,13 @@ static int it87_resume(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct it87_data *data = dev_get_drvdata(dev); + int err; it87_resume_sio(pdev); - it87_lock(data); + err = it87_lock(data); + if (err) + return err; it87_check_pwm(dev); it87_check_limit_regs(data); diff --git a/drivers/hwmon/ltc4282.c b/drivers/hwmon/ltc4282.c index 1d664a2d7b3cb..58c2d3a624329 100644 --- a/drivers/hwmon/ltc4282.c +++ b/drivers/hwmon/ltc4282.c @@ -1018,8 +1018,9 @@ static umode_t ltc4282_in_is_visible(const struct ltc4282_state *st, u32 attr) case hwmon_in_max: case hwmon_in_min: case hwmon_in_enable: - case hwmon_in_reset_history: return 0644; + case hwmon_in_reset_history: + return 0200; default: return 0; } @@ -1038,8 +1039,9 @@ static umode_t ltc4282_curr_is_visible(u32 attr) return 0444; case hwmon_curr_max: case hwmon_curr_min: - case hwmon_curr_reset_history: return 0644; + case hwmon_curr_reset_history: + return 0200; default: return 0; } @@ -1057,8 +1059,9 @@ static umode_t ltc4282_power_is_visible(u32 attr) return 0444; case hwmon_power_max: case hwmon_power_min: - case hwmon_power_reset_history: return 0644; + case hwmon_power_reset_history: + return 0200; default: return 0; } diff --git a/drivers/hwmon/max16065.c b/drivers/hwmon/max16065.c index 0ccb5eb596fc4..43fbb9b26b102 100644 --- a/drivers/hwmon/max16065.c +++ b/drivers/hwmon/max16065.c @@ -151,27 +151,27 @@ static struct max16065_data *max16065_update_device(struct device *dev) int i; for (i = 0; i < data->num_adc; i++) - data->adc[i] - = max16065_read_adc(client, MAX16065_ADC(i)); + WRITE_ONCE(data->adc[i], + max16065_read_adc(client, MAX16065_ADC(i))); if (data->have_current) { - data->adc[MAX16065_NUM_ADC] - = max16065_read_adc(client, MAX16065_CSP_ADC); - data->curr_sense - = i2c_smbus_read_byte_data(client, - MAX16065_CURR_SENSE); + WRITE_ONCE(data->adc[MAX16065_NUM_ADC], + max16065_read_adc(client, MAX16065_CSP_ADC)); + WRITE_ONCE(data->curr_sense, + i2c_smbus_read_byte_data(client, MAX16065_CURR_SENSE)); } for (i = 0; i < 2; i++) - data->fault[i] - = i2c_smbus_read_byte_data(client, MAX16065_FAULT(i)); + WRITE_ONCE(data->fault[i], + i2c_smbus_read_byte_data(client, MAX16065_FAULT(i))); /* * MAX16067 and MAX16068 have separate undervoltage and * overvoltage alarm bits. Squash them together. */ if (data->chip == max16067 || data->chip == max16068) - data->fault[0] |= data->fault[1]; + WRITE_ONCE(data->fault[0], + data->fault[0] | data->fault[1]); data->last_updated = jiffies; data->valid = true; @@ -185,7 +185,7 @@ static ssize_t max16065_alarm_show(struct device *dev, { struct sensor_device_attribute_2 *attr2 = to_sensor_dev_attr_2(da); struct max16065_data *data = max16065_update_device(dev); - int val = data->fault[attr2->nr]; + int val = READ_ONCE(data->fault[attr2->nr]); if (val < 0) return val; @@ -203,7 +203,7 @@ static ssize_t max16065_input_show(struct device *dev, { struct sensor_device_attribute *attr = to_sensor_dev_attr(da); struct max16065_data *data = max16065_update_device(dev); - int adc = data->adc[attr->index]; + int adc = READ_ONCE(data->adc[attr->index]); if (unlikely(adc < 0)) return adc; @@ -216,12 +216,13 @@ static ssize_t max16065_current_show(struct device *dev, struct device_attribute *da, char *buf) { struct max16065_data *data = max16065_update_device(dev); + int curr_sense = READ_ONCE(data->curr_sense); - if (unlikely(data->curr_sense < 0)) - return data->curr_sense; + if (unlikely(curr_sense < 0)) + return curr_sense; return sysfs_emit(buf, "%d\n", - ADC_TO_CURR(data->curr_sense, data->curr_gain)); + ADC_TO_CURR(curr_sense, data->curr_gain)); } static ssize_t max16065_limit_store(struct device *dev, diff --git a/drivers/hwmon/max6639.c b/drivers/hwmon/max6639.c index a06346496e1d9..447e8cdbc74c6 100644 --- a/drivers/hwmon/max6639.c +++ b/drivers/hwmon/max6639.c @@ -235,7 +235,7 @@ static int max6639_read_fan(struct device *dev, u32 attr, int channel, static int max6639_set_ppr(struct max6639_data *data, int channel, u8 ppr) { /* Decrement the PPR value and shift left by 6 to match the register format */ - return regmap_write(data->regmap, MAX6639_REG_FAN_PPR(channel), ppr-- << 6); + return regmap_write(data->regmap, MAX6639_REG_FAN_PPR(channel), --ppr << 6); } static int max6639_write_fan(struct device *dev, u32 attr, int channel, @@ -537,8 +537,8 @@ static int max6639_probe_child_from_dt(struct i2c_client *client, { struct device *dev = &client->dev; - u32 i; - int err, val; + u32 i, val; + int err; err = of_property_read_u32(child, "reg", &i); if (err) { @@ -553,8 +553,8 @@ static int max6639_probe_child_from_dt(struct i2c_client *client, err = of_property_read_u32(child, "pulses-per-revolution", &val); if (!err) { - if (val < 1 || val > 5) { - dev_err(dev, "invalid pulses-per-revolution %d of %pOFn\n", val, child); + if (val < 1 || val > 4) { + dev_err(dev, "invalid pulses-per-revolution %u of %pOFn\n", val, child); return -EINVAL; } data->ppr[i] = val; @@ -623,7 +623,7 @@ static int max6639_init_client(struct i2c_client *client, return err; /* Fans PWM polarity high by default */ - err = regmap_write(data->regmap, MAX6639_REG_FAN_CONFIG2a(i), 0x00); + err = regmap_write(data->regmap, MAX6639_REG_FAN_CONFIG2a(i), 0x02); if (err) return err; diff --git a/drivers/hwmon/max6697.c b/drivers/hwmon/max6697.c index 0735a1d2c20fd..6926d787b5ad1 100644 --- a/drivers/hwmon/max6697.c +++ b/drivers/hwmon/max6697.c @@ -548,7 +548,7 @@ static int max6697_probe(struct i2c_client *client) struct regmap *regmap; int err; - regmap = regmap_init_i2c(client, &max6697_regmap_config); + regmap = devm_regmap_init_i2c(client, &max6697_regmap_config); if (IS_ERR(regmap)) return PTR_ERR(regmap); diff --git a/drivers/hwmon/nct6683.c b/drivers/hwmon/nct6683.c index 6cda35388b24c..4a83804140386 100644 --- a/drivers/hwmon/nct6683.c +++ b/drivers/hwmon/nct6683.c @@ -181,6 +181,7 @@ superio_exit(int ioreg) #define NCT6683_CUSTOMER_ID_ASROCK2 0xe1b #define NCT6683_CUSTOMER_ID_ASROCK3 0x1631 #define NCT6683_CUSTOMER_ID_ASROCK4 0x163e +#define NCT6683_CUSTOMER_ID_ASROCK5 0x1621 #define NCT6683_REG_BUILD_YEAR 0x604 #define NCT6683_REG_BUILD_MONTH 0x605 @@ -1242,6 +1243,8 @@ static int nct6683_probe(struct platform_device *pdev) break; case NCT6683_CUSTOMER_ID_ASROCK4: break; + case NCT6683_CUSTOMER_ID_ASROCK5: + break; default: if (!force) return -ENODEV; diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c index 407945d2cd6a8..0b529a542b0d4 100644 --- a/drivers/hwmon/nct6775-platform.c +++ b/drivers/hwmon/nct6775-platform.c @@ -1357,6 +1357,7 @@ static const char * const asus_msi_boards[] = { "Pro WS W680-ACE IPMI", "Pro WS W790-ACE", "Pro WS W790E-SAGE SE", + "Pro WS WRX90E-SAGE SE", "ProArt B650-CREATOR", "ProArt B660-CREATOR D4", "ProArt B760-CREATOR D4", diff --git a/drivers/hwmon/nct7363.c b/drivers/hwmon/nct7363.c index e13ab918b1abe..56ac42e4a8653 100644 --- a/drivers/hwmon/nct7363.c +++ b/drivers/hwmon/nct7363.c @@ -351,6 +351,7 @@ static int nct7363_present_pwm_fanin(struct device *dev, if (ret) return ret; + of_node_put(args.np); if (args.args[0] >= NCT7363_PWM_COUNT) return -EINVAL; data->pwm_mask |= BIT(args.args[0]); diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index b3694a4209b97..42cc6068bb08b 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -420,6 +420,12 @@ static ssize_t occ_show_freq_2(struct device *dev, return sysfs_emit(buf, "%u\n", val); } +static u64 occ_get_powr_avg(u64 accum, u32 samples) +{ + return (samples == 0) ? 0 : + mul_u64_u32_div(accum, 1000000UL, samples); +} + static ssize_t occ_show_power_1(struct device *dev, struct device_attribute *attr, char *buf) { @@ -441,9 +447,8 @@ static ssize_t occ_show_power_1(struct device *dev, val = get_unaligned_be16(&power->sensor_id); break; case 1: - val = get_unaligned_be32(&power->accumulator) / - get_unaligned_be32(&power->update_tag); - val *= 1000000ULL; + val = occ_get_powr_avg(get_unaligned_be32(&power->accumulator), + get_unaligned_be32(&power->update_tag)); break; case 2: val = (u64)get_unaligned_be32(&power->update_tag) * @@ -459,12 +464,6 @@ static ssize_t occ_show_power_1(struct device *dev, return sysfs_emit(buf, "%llu\n", val); } -static u64 occ_get_powr_avg(u64 accum, u32 samples) -{ - return (samples == 0) ? 0 : - mul_u64_u32_div(accum, 1000000UL, samples); -} - static ssize_t occ_show_power_2(struct device *dev, struct device_attribute *attr, char *buf) { @@ -725,7 +724,7 @@ static ssize_t occ_show_extended(struct device *dev, switch (sattr->nr) { case 0: if (extn->flags & EXTN_FLAG_SENSOR_ID) { - rc = sysfs_emit(buf, "%u", + rc = sysfs_emit(buf, "%u\n", get_unaligned_be32(&extn->sensor_id)); } else { rc = sysfs_emit(buf, "%4phN\n", extn->name); @@ -749,6 +748,7 @@ static ssize_t occ_show_extended(struct device *dev, * are dynamically allocated, we cannot use the existing kernel macros which * stringify the name argument. */ +__printf(7, 8) static void occ_init_attribute(struct occ_attribute *attr, int mode, ssize_t (*show)(struct device *dev, struct device_attribute *attr, char *buf), ssize_t (*store)(struct device *dev, struct device_attribute *attr, diff --git a/drivers/hwmon/peci/cputemp.c b/drivers/hwmon/peci/cputemp.c index c7112dbf008b7..007fa621e10f8 100644 --- a/drivers/hwmon/peci/cputemp.c +++ b/drivers/hwmon/peci/cputemp.c @@ -133,7 +133,7 @@ static int get_temp_target(struct peci_cputemp *priv, enum peci_temp_target_type *val = priv->temp.target.tjmax; break; case crit_hyst_type: - *val = priv->temp.target.tjmax - priv->temp.target.tcontrol; + *val = priv->temp.target.tcontrol; break; default: ret = -EOPNOTSUPP; @@ -339,7 +339,7 @@ static umode_t cputemp_is_visible(const void *data, enum hwmon_sensor_types type { const struct peci_cputemp *priv = data; - if (channel > CPUTEMP_CHANNEL_NUMS) + if (channel >= CPUTEMP_CHANNEL_NUMS) return 0; if (channel < channel_core) diff --git a/drivers/hwmon/pmbus/ina233.c b/drivers/hwmon/pmbus/ina233.c index dde1e16783943..7aebd854763a6 100644 --- a/drivers/hwmon/pmbus/ina233.c +++ b/drivers/hwmon/pmbus/ina233.c @@ -67,10 +67,13 @@ static int ina233_read_word_data(struct i2c_client *client, int page, switch (reg) { case PMBUS_VIRT_READ_VMON: ret = pmbus_read_word_data(client, 0, 0xff, MFR_READ_VSHUNT); + if (ret < 0) + return ret; /* Adjust returned value to match VIN coefficients */ /* VIN: 1.25 mV VSHUNT: 2.5 uV LSB */ - ret = DIV_ROUND_CLOSEST(ret * 25, 12500); + ret = clamp_val(DIV_ROUND_CLOSEST((s16)ret * 25, 12500), + S16_MIN, S16_MAX) & 0xffff; break; default: ret = -ENODATA; diff --git a/drivers/hwmon/pmbus/isl68137.c b/drivers/hwmon/pmbus/isl68137.c index 6bba9b50c51bf..ab54184b4878e 100644 --- a/drivers/hwmon/pmbus/isl68137.c +++ b/drivers/hwmon/pmbus/isl68137.c @@ -94,10 +94,21 @@ static ssize_t isl68137_avs_enable_show_page(struct i2c_client *client, int page, char *buf) { - int val = pmbus_read_byte_data(client, page, PMBUS_OPERATION); + int val; - return sprintf(buf, "%d\n", - (val & ISL68137_VOUT_AVS) == ISL68137_VOUT_AVS ? 1 : 0); + val = pmbus_lock_interruptible(client); + if (val) + return val; + + val = pmbus_read_byte_data(client, page, PMBUS_OPERATION); + + pmbus_unlock(client); + + if (val < 0) + return val; + + return sysfs_emit(buf, "%d\n", + (val & ISL68137_VOUT_AVS) == ISL68137_VOUT_AVS); } static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, @@ -113,6 +124,10 @@ static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, op_val = result ? ISL68137_VOUT_AVS : 0; + rc = pmbus_lock_interruptible(client); + if (rc) + return rc; + /* * Writes to VOUT setpoint over AVSBus will persist after the VRM is * switched to PMBus control. Switching back to AVSBus control @@ -124,17 +139,20 @@ static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, rc = pmbus_read_word_data(client, page, 0xff, PMBUS_VOUT_COMMAND); if (rc < 0) - return rc; + goto unlock; rc = pmbus_write_word_data(client, page, PMBUS_VOUT_COMMAND, rc); if (rc < 0) - return rc; + goto unlock; } rc = pmbus_update_byte_data(client, page, PMBUS_OPERATION, ISL68137_VOUT_AVS, op_val); +unlock: + pmbus_unlock(client); + return (rc < 0) ? rc : count; } diff --git a/drivers/hwmon/pmbus/ltc4286.c b/drivers/hwmon/pmbus/ltc4286.c index aabd0bcdfeee3..8715d380784a0 100644 --- a/drivers/hwmon/pmbus/ltc4286.c +++ b/drivers/hwmon/pmbus/ltc4286.c @@ -173,3 +173,4 @@ module_i2c_driver(ltc4286_driver); MODULE_AUTHOR("Delphine CC Chiu "); MODULE_DESCRIPTION("PMBUS driver for LTC4286 and compatibles"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("PMBUS"); diff --git a/drivers/hwmon/pmbus/mp2869.c b/drivers/hwmon/pmbus/mp2869.c index cc69a1e91dfe8..4647892e51121 100644 --- a/drivers/hwmon/pmbus/mp2869.c +++ b/drivers/hwmon/pmbus/mp2869.c @@ -165,7 +165,7 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg) { const struct pmbus_driver_info *info = pmbus_get_driver_info(client); struct mp2869_data *data = to_mp2869_data(info); - int ret; + int ret, mfr; switch (reg) { case PMBUS_VOUT_MODE: @@ -188,11 +188,14 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg) if (ret < 0) return ret; + mfr = pmbus_read_byte_data(client, page, + PMBUS_STATUS_MFR_SPECIFIC); + if (mfr < 0) + return mfr; + ret = (ret & ~GENMASK(2, 2)) | FIELD_PREP(GENMASK(2, 2), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))); + FIELD_GET(GENMASK(1, 1), mfr)); break; case PMBUS_STATUS_TEMPERATURE: /* @@ -207,15 +210,16 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg) if (ret < 0) return ret; + mfr = pmbus_read_byte_data(client, page, + PMBUS_STATUS_MFR_SPECIFIC); + if (mfr < 0) + return mfr; + ret = (ret & ~GENMASK(7, 6)) | FIELD_PREP(GENMASK(6, 6), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))) | + FIELD_GET(GENMASK(1, 1), mfr)) | FIELD_PREP(GENMASK(7, 7), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))); + FIELD_GET(GENMASK(1, 1), mfr)); break; default: ret = -ENODATA; @@ -230,7 +234,7 @@ static int mp2869_read_word_data(struct i2c_client *client, int page, int phase, { const struct pmbus_driver_info *info = pmbus_get_driver_info(client); struct mp2869_data *data = to_mp2869_data(info); - int ret; + int ret, mfr; switch (reg) { case PMBUS_STATUS_WORD: @@ -246,11 +250,14 @@ static int mp2869_read_word_data(struct i2c_client *client, int page, int phase, if (ret < 0) return ret; + mfr = pmbus_read_byte_data(client, page, + PMBUS_STATUS_MFR_SPECIFIC); + if (mfr < 0) + return mfr; + ret = (ret & ~GENMASK(2, 2)) | FIELD_PREP(GENMASK(2, 2), - FIELD_GET(GENMASK(1, 1), - pmbus_read_byte_data(client, page, - PMBUS_STATUS_MFR_SPECIFIC))); + FIELD_GET(GENMASK(1, 1), mfr)); break; case PMBUS_READ_VIN: /* diff --git a/drivers/hwmon/pmbus/mp2975.c b/drivers/hwmon/pmbus/mp2975.c index c31982d851962..d0bc47b12cb07 100644 --- a/drivers/hwmon/pmbus/mp2975.c +++ b/drivers/hwmon/pmbus/mp2975.c @@ -313,6 +313,8 @@ static int mp2973_read_word_data(struct i2c_client *client, int page, case PMBUS_STATUS_WORD: /* MP2973 & MP2971 return PGOOD instead of PB_STATUS_POWER_GOOD_N. */ ret = pmbus_read_word_data(client, page, phase, reg); + if (ret < 0) + return ret; ret ^= PB_STATUS_POWER_GOOD_N; break; case PMBUS_OT_FAULT_LIMIT: diff --git a/drivers/hwmon/pmbus/mpq8785.c b/drivers/hwmon/pmbus/mpq8785.c index 1f56aaf4dde80..87bd039c77b9b 100644 --- a/drivers/hwmon/pmbus/mpq8785.c +++ b/drivers/hwmon/pmbus/mpq8785.c @@ -47,6 +47,33 @@ static int mpq8785_identify(struct i2c_client *client, return 0; }; +static int mpq8785_read_byte_data(struct i2c_client *client, int page, int reg) +{ + int ret; + + switch (reg) { + case PMBUS_VOUT_MODE: + ret = pmbus_read_byte_data(client, page, reg); + if (ret < 0) + return ret; + + if ((ret >> 5) == 1) { + /* + * The MPQ8785 chip reports VOUT_MODE as VID mode, but the driver + * treats VID as direct mode. Without this, identification would fail + * due to mode mismatch. + * This override ensures the reported mode matches the driver + * configuration, allowing successful initialization. + */ + return PB_VOUT_MODE_DIRECT; + } + + return ret; + default: + return -ENODATA; + } +} + static int mpm82504_read_word_data(struct i2c_client *client, int page, int phase, int reg) { @@ -129,6 +156,7 @@ static int mpq8785_probe(struct i2c_client *client) break; case mpq8785: info->identify = mpq8785_identify; + info->read_byte_data = mpq8785_read_byte_data; break; default: return -ENODEV; diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index be6d05def1152..572be3ebc03df 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -6,6 +6,7 @@ * Copyright (c) 2012 Guenter Roeck */ +#include #include #include #include @@ -21,8 +22,8 @@ #include #include #include -#include #include +#include #include "pmbus.h" /* @@ -112,6 +113,11 @@ struct pmbus_data { struct mutex update_lock; +#if IS_ENABLED(CONFIG_REGULATOR) + atomic_t regulator_events[PMBUS_PAGES]; + struct work_struct regulator_notify_work; +#endif + bool has_status_word; /* device uses STATUS_WORD register */ int (*read_status)(struct i2c_client *client, int page); @@ -1209,6 +1215,12 @@ static ssize_t pmbus_show_boolean(struct device *dev, return sysfs_emit(buf, "%d\n", val); } +static ssize_t pmbus_show_zero(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + return sysfs_emit(buf, "0\n"); +} + static ssize_t pmbus_show_sensor(struct device *dev, struct device_attribute *devattr, char *buf) { @@ -1407,7 +1419,7 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, int reg, enum pmbus_sensor_classes class, bool update, bool readonly, - bool convert) + bool writeonly, bool convert) { struct pmbus_sensor *sensor; struct device_attribute *a; @@ -1436,7 +1448,8 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, sensor->data = -ENODATA; pmbus_dev_attr_init(a, sensor->name, readonly ? 0444 : 0644, - pmbus_show_sensor, pmbus_set_sensor); + writeonly ? pmbus_show_zero : pmbus_show_sensor, + pmbus_set_sensor); if (pmbus_add_attribute(data, &a->attr)) return NULL; @@ -1495,8 +1508,10 @@ static int pmbus_add_label(struct pmbus_data *data, struct pmbus_limit_attr { u16 reg; /* Limit register */ u16 sbit; /* Alarm attribute status bit */ - bool update; /* True if register needs updates */ - bool low; /* True if low limit; for limits with compare functions only */ + bool readonly:1; /* True if the attribute is read-only */ + bool writeonly:1; /* True if the attribute is write-only */ + bool update:1; /* True if register needs updates */ + bool low:1; /* True if low limit; for limits with compare functions only */ const char *attr; /* Attribute name */ const char *alarm; /* Alarm attribute name */ }; @@ -1511,9 +1526,9 @@ struct pmbus_sensor_attr { u8 nlimit; /* # of limit registers */ enum pmbus_sensor_classes class;/* sensor class */ const char *label; /* sensor label */ - bool paged; /* true if paged sensor */ - bool update; /* true if update needed */ - bool compare; /* true if compare function needed */ + bool paged:1; /* true if paged sensor */ + bool update:1; /* true if update needed */ + bool compare:1; /* true if compare function needed */ u32 func; /* sensor mask */ u32 sfunc; /* sensor status mask */ int sreg; /* status register */ @@ -1544,7 +1559,7 @@ static int pmbus_add_limit_attrs(struct i2c_client *client, curr = pmbus_add_sensor(data, name, l->attr, index, page, 0xff, l->reg, attr->class, attr->update || l->update, - false, true); + l->readonly, l->writeonly, true); if (!curr) return -ENOMEM; if (l->sbit && (info->func[page] & attr->sfunc)) { @@ -1584,7 +1599,7 @@ static int pmbus_add_sensor_attrs_one(struct i2c_client *client, return ret; } base = pmbus_add_sensor(data, name, "input", index, page, phase, - attr->reg, attr->class, true, true, true); + attr->reg, attr->class, true, true, false, true); if (!base) return -ENOMEM; /* No limit and alarm attributes for phase specific sensors */ @@ -1707,23 +1722,29 @@ static const struct pmbus_limit_attr vin_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_VIN_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_VIN_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_VIN_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_VIN_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_VIN_MIN, + .readonly = true, .attr = "rated_min", }, { .reg = PMBUS_MFR_VIN_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1776,23 +1797,29 @@ static const struct pmbus_limit_attr vout_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_VOUT_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_VOUT_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_VOUT_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_VOUT_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_VOUT_MIN, + .readonly = true, .attr = "rated_min", }, { .reg = PMBUS_MFR_VOUT_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1852,20 +1879,25 @@ static const struct pmbus_limit_attr iin_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_IIN_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_IIN_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_IIN_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_IIN_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_IIN_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1889,20 +1921,25 @@ static const struct pmbus_limit_attr iout_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_IOUT_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_IOUT_MIN, .update = true, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_IOUT_MAX, .update = true, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_IOUT_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_IOUT_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1943,20 +1980,25 @@ static const struct pmbus_limit_attr pin_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_PIN_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_PIN_MIN, .update = true, + .readonly = true, .attr = "input_lowest", }, { .reg = PMBUS_VIRT_READ_PIN_MAX, .update = true, + .readonly = true, .attr = "input_highest", }, { .reg = PMBUS_VIRT_RESET_PIN_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_PIN_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -1980,20 +2022,25 @@ static const struct pmbus_limit_attr pout_limit_attrs[] = { }, { .reg = PMBUS_VIRT_READ_POUT_AVG, .update = true, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_POUT_MIN, .update = true, + .readonly = true, .attr = "input_lowest", }, { .reg = PMBUS_VIRT_READ_POUT_MAX, .update = true, + .readonly = true, .attr = "input_highest", }, { .reg = PMBUS_VIRT_RESET_POUT_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_POUT_MAX, + .readonly = true, .attr = "rated_max", }, }; @@ -2049,18 +2096,23 @@ static const struct pmbus_limit_attr temp_limit_attrs[] = { .sbit = PB_TEMP_OT_FAULT, }, { .reg = PMBUS_VIRT_READ_TEMP_MIN, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_TEMP_AVG, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_TEMP_MAX, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_TEMP_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_MAX_TEMP_1, + .readonly = true, .attr = "rated_max", }, }; @@ -2090,18 +2142,23 @@ static const struct pmbus_limit_attr temp_limit_attrs2[] = { .sbit = PB_TEMP_OT_FAULT, }, { .reg = PMBUS_VIRT_READ_TEMP2_MIN, + .readonly = true, .attr = "lowest", }, { .reg = PMBUS_VIRT_READ_TEMP2_AVG, + .readonly = true, .attr = "average", }, { .reg = PMBUS_VIRT_READ_TEMP2_MAX, + .readonly = true, .attr = "highest", }, { .reg = PMBUS_VIRT_RESET_TEMP2_HISTORY, + .writeonly = true, .attr = "reset_history", }, { .reg = PMBUS_MFR_MAX_TEMP_2, + .readonly = true, .attr = "rated_max", }, }; @@ -2131,6 +2188,7 @@ static const struct pmbus_limit_attr temp_limit_attrs3[] = { .sbit = PB_TEMP_OT_FAULT, }, { .reg = PMBUS_MFR_MAX_TEMP_3, + .readonly = true, .attr = "rated_max", }, }; @@ -2214,7 +2272,7 @@ static int pmbus_add_fan_ctrl(struct i2c_client *client, sensor = pmbus_add_sensor(data, "fan", "target", index, page, 0xff, PMBUS_VIRT_FAN_TARGET_1 + id, PSC_FAN, - false, false, true); + false, false, false, true); if (!sensor) return -ENOMEM; @@ -2225,14 +2283,14 @@ static int pmbus_add_fan_ctrl(struct i2c_client *client, sensor = pmbus_add_sensor(data, "pwm", NULL, index, page, 0xff, PMBUS_VIRT_PWM_1 + id, PSC_PWM, - false, false, true); + false, false, false, true); if (!sensor) return -ENOMEM; sensor = pmbus_add_sensor(data, "pwm", "enable", index, page, 0xff, PMBUS_VIRT_PWM_ENABLE_1 + id, PSC_PWM, - true, false, false); + true, false, false, false); if (!sensor) return -ENOMEM; @@ -2274,7 +2332,7 @@ static int pmbus_add_fan_attributes(struct i2c_client *client, if (pmbus_add_sensor(data, "fan", "input", index, page, 0xff, pmbus_fan_registers[f], - PSC_FAN, true, true, true) == NULL) + PSC_FAN, true, true, false, true) == NULL) return -ENOMEM; /* Fan control */ @@ -3176,12 +3234,19 @@ static int pmbus_regulator_get_voltage(struct regulator_dev *rdev) .class = PSC_VOLTAGE_OUT, .convert = true, }; + int ret; + mutex_lock(&data->update_lock); s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_READ_VOUT); - if (s.data < 0) - return s.data; + if (s.data < 0) { + ret = s.data; + goto unlock; + } - return (int)pmbus_reg2data(data, &s) * 1000; /* unit is uV */ + ret = (int)pmbus_reg2data(data, &s) * 1000; /* unit is uV */ +unlock: + mutex_unlock(&data->update_lock); + return ret; } static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, @@ -3198,16 +3263,22 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, }; int val = DIV_ROUND_CLOSEST(min_uv, 1000); /* convert to mV */ int low, high; + int ret; *selector = 0; + mutex_lock(&data->update_lock); low = pmbus_regulator_get_low_margin(client, s.page); - if (low < 0) - return low; + if (low < 0) { + ret = low; + goto unlock; + } high = pmbus_regulator_get_high_margin(client, s.page); - if (high < 0) - return high; + if (high < 0) { + ret = high; + goto unlock; + } /* Make sure we are within margins */ if (low > val) @@ -3217,7 +3288,10 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, val = pmbus_data2reg(data, &s, val); - return _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); + ret = _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); +unlock: + mutex_unlock(&data->update_lock); + return ret; } static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, @@ -3227,6 +3301,7 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, struct i2c_client *client = to_i2c_client(dev->parent); struct pmbus_data *data = i2c_get_clientdata(client); int val, low, high; + int ret; if (data->flags & PMBUS_VOUT_PROTECTED) return 0; @@ -3239,18 +3314,29 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, val = DIV_ROUND_CLOSEST(rdev->desc->min_uV + (rdev->desc->uV_step * selector), 1000); /* convert to mV */ + mutex_lock(&data->update_lock); + low = pmbus_regulator_get_low_margin(client, rdev_get_id(rdev)); - if (low < 0) - return low; + if (low < 0) { + ret = low; + goto unlock; + } high = pmbus_regulator_get_high_margin(client, rdev_get_id(rdev)); - if (high < 0) - return high; + if (high < 0) { + ret = high; + goto unlock; + } - if (val >= low && val <= high) - return val * 1000; /* unit is uV */ + if (val >= low && val <= high) { + ret = val * 1000; /* unit is uV */ + goto unlock; + } - return 0; + ret = 0; +unlock: + mutex_unlock(&data->update_lock); + return ret; } const struct regulator_ops pmbus_regulator_ops = { @@ -3281,12 +3367,42 @@ int pmbus_regulator_init_cb(struct regulator_dev *rdev, } EXPORT_SYMBOL_NS_GPL(pmbus_regulator_init_cb, "PMBUS"); +static void pmbus_regulator_notify_work_cancel(void *data) +{ + struct pmbus_data *pdata = data; + + cancel_work_sync(&pdata->regulator_notify_work); +} + +static void pmbus_regulator_notify_worker(struct work_struct *work) +{ + struct pmbus_data *data = + container_of(work, struct pmbus_data, regulator_notify_work); + int i, j; + + for (i = 0; i < data->info->pages; i++) { + int event; + + event = atomic_xchg(&data->regulator_events[i], 0); + if (!event) + continue; + + for (j = 0; j < data->info->num_regulators; j++) { + if (i == rdev_get_id(data->rdevs[j])) { + regulator_notifier_call_chain(data->rdevs[j], + event, NULL); + break; + } + } + } +} + static int pmbus_regulator_register(struct pmbus_data *data) { struct device *dev = data->dev; const struct pmbus_driver_info *info = data->info; const struct pmbus_platform_data *pdata = dev_get_platdata(dev); - int i; + int i, ret; data->rdevs = devm_kzalloc(dev, sizeof(struct regulator_dev *) * info->num_regulators, GFP_KERNEL); @@ -3310,19 +3426,19 @@ static int pmbus_regulator_register(struct pmbus_data *data) info->reg_desc[i].name); } + INIT_WORK(&data->regulator_notify_work, pmbus_regulator_notify_worker); + + ret = devm_add_action_or_reset(dev, pmbus_regulator_notify_work_cancel, data); + if (ret) + return ret; + return 0; } static void pmbus_regulator_notify(struct pmbus_data *data, int page, int event) { - int j; - - for (j = 0; j < data->info->num_regulators; j++) { - if (page == rdev_get_id(data->rdevs[j])) { - regulator_notifier_call_chain(data->rdevs[j], event, NULL); - break; - } - } + atomic_or(event, &data->regulator_events[page]); + schedule_work(&data->regulator_notify_work); } #else static int pmbus_regulator_register(struct pmbus_data *data) diff --git a/drivers/hwmon/pmbus/pxe1610.c b/drivers/hwmon/pmbus/pxe1610.c index 6a4a978eca7e8..24c1f961c7668 100644 --- a/drivers/hwmon/pmbus/pxe1610.c +++ b/drivers/hwmon/pmbus/pxe1610.c @@ -104,7 +104,10 @@ static int pxe1610_probe(struct i2c_client *client) * By default this device doesn't boot to page 0, so set page 0 * to access all pmbus registers. */ - i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0); + ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0); + if (ret < 0) + return dev_err_probe(&client->dev, ret, + "Failed to set page 0\n"); /* Read Manufacturer id */ ret = i2c_smbus_read_block_data(client, PMBUS_MFR_ID, buf); diff --git a/drivers/hwmon/pmbus/q54sj108a2.c b/drivers/hwmon/pmbus/q54sj108a2.c index 4d7086d83aa36..7b0d292a425ae 100644 --- a/drivers/hwmon/pmbus/q54sj108a2.c +++ b/drivers/hwmon/pmbus/q54sj108a2.c @@ -78,7 +78,8 @@ static ssize_t q54sj108a2_debugfs_read(struct file *file, char __user *buf, int idx = *idxp; struct q54sj108a2_data *psu = to_psu(idxp, idx); char data[I2C_SMBUS_BLOCK_MAX + 2] = { 0 }; - char data_char[I2C_SMBUS_BLOCK_MAX + 2] = { 0 }; + char data_char[I2C_SMBUS_BLOCK_MAX * 2 + 2] = { 0 }; + char *out = data; char *res; switch (idx) { @@ -149,27 +150,27 @@ static ssize_t q54sj108a2_debugfs_read(struct file *file, char __user *buf, if (rc < 0) return rc; - res = bin2hex(data, data_char, 32); - rc = res - data; - + res = bin2hex(data_char, data, rc); + rc = res - data_char; + out = data_char; break; case Q54SJ108A2_DEBUGFS_FLASH_KEY: rc = i2c_smbus_read_block_data(psu->client, PMBUS_FLASH_KEY_WRITE, data); if (rc < 0) return rc; - res = bin2hex(data, data_char, 4); - rc = res - data; - + res = bin2hex(data_char, data, rc); + rc = res - data_char; + out = data_char; break; default: return -EINVAL; } - data[rc] = '\n'; + out[rc] = '\n'; rc += 2; - return simple_read_from_buffer(buf, count, ppos, data, rc); + return simple_read_from_buffer(buf, count, ppos, out, rc); } static ssize_t q54sj108a2_debugfs_write(struct file *file, const char __user *buf, diff --git a/drivers/hwmon/pmbus/tps53679.c b/drivers/hwmon/pmbus/tps53679.c index ca2bfa25eb04c..249974c13aa39 100644 --- a/drivers/hwmon/pmbus/tps53679.c +++ b/drivers/hwmon/pmbus/tps53679.c @@ -103,10 +103,10 @@ static int tps53679_identify_chip(struct i2c_client *client, } ret = i2c_smbus_read_block_data(client, PMBUS_IC_DEVICE_ID, buf); - if (ret < 0) - return ret; + if (ret <= 0) + return ret < 0 ? ret : -EIO; - /* Adjust length if null terminator if present */ + /* Adjust length if null terminator is present */ buf_len = (buf[ret - 1] != '\x00' ? ret : ret - 1); id_len = strlen(id); @@ -175,8 +175,8 @@ static int tps53676_identify(struct i2c_client *client, ret = i2c_smbus_read_block_data(client, PMBUS_IC_DEVICE_ID, buf); if (ret < 0) return ret; - if (strncmp("TI\x53\x67\x60", buf, 5)) { - dev_err(&client->dev, "Unexpected device ID: %s\n", buf); + if (ret != 6 || memcmp(buf, "TI\x53\x67\x60\x00", 6)) { + dev_err(&client->dev, "Unexpected device ID: %*ph\n", ret, buf); return -ENODEV; } diff --git a/drivers/hwmon/powerz.c b/drivers/hwmon/powerz.c index 4e663d5b4e330..a75b941bd6e2f 100644 --- a/drivers/hwmon/powerz.c +++ b/drivers/hwmon/powerz.c @@ -108,6 +108,9 @@ static int powerz_read_data(struct usb_device *udev, struct powerz_priv *priv) { int ret; + if (!priv->urb) + return -ENODEV; + priv->status = -ETIMEDOUT; reinit_completion(&priv->completion); @@ -224,6 +227,8 @@ static int powerz_probe(struct usb_interface *intf, mutex_init(&priv->mutex); init_completion(&priv->completion); + usb_set_intfdata(intf, priv); + hwmon_dev = devm_hwmon_device_register_with_info(parent, DRIVER_NAME, priv, &powerz_chip_info, NULL); @@ -232,8 +237,6 @@ static int powerz_probe(struct usb_interface *intf, return PTR_ERR(hwmon_dev); } - usb_set_intfdata(intf, priv); - return 0; } @@ -244,6 +247,7 @@ static void powerz_disconnect(struct usb_interface *intf) mutex_lock(&priv->mutex); usb_kill_urb(priv->urb); usb_free_urb(priv->urb); + priv->urb = NULL; mutex_unlock(&priv->mutex); } diff --git a/drivers/hwmon/qcom-bcl-hwmon.c b/drivers/hwmon/qcom-bcl-hwmon.c new file mode 100644 index 0000000000000..9894b1d14d124 --- /dev/null +++ b/drivers/hwmon/qcom-bcl-hwmon.c @@ -0,0 +1,982 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm pmic BCL hardware driver for battery overcurrent and + * battery or system under voltage monitor + * + * Copyright (c) 2026, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom-bcl-hwmon.h" + +ADD_BCL_HWMON_ALARM_MAPS(in, min, lcrit); +ADD_BCL_HWMON_ALARM_MAPS(curr, max, crit); + +/* Interrupt names for each alarm level */ +static const char * const bcl_int_names[ALARM_MAX] = { + [LVL0] = "bcl-max-min", + [LVL1] = "bcl-critical", +}; + +static const char * const bcl_channel_label[CHANNEL_MAX] = { + "BCL Voltage", + "BCL Current", +}; + +/* Common Reg Fields */ +static const struct reg_field common_reg_fields[COMMON_FIELD_MAX] = { + [F_V_MAJOR] = REG_FIELD(REVISION2, 0, 7), + [F_V_MINOR] = REG_FIELD(REVISION1, 0, 7), + [F_CTL_EN] = REG_FIELD(EN_CTL1, 7, 7), + [F_LVL0_ALARM] = REG_FIELD(STATUS, 0, 0), + [F_LVL1_ALARM] = REG_FIELD(STATUS, 1, 1), +}; + +/* BCL Version/Modes specific fields */ +static const struct reg_field bcl_v1_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(MODE_CTL1, 0, 2), + [F_IN_L0_THR] = REG_FIELD(VADC_L0_THR, 0, 7), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 5), + [F_IN_INPUT_EN] = REG_FIELD(VADC_CONV_REQ, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 7), + [F_CURR_MON_EN] = REG_FIELD(IADC_CONV_REQ, 0, 0), + [F_CURR_H0_THR] = REG_FIELD(IADC_H0_THR, 0, 7), + [F_CURR_H1_THR] = REG_FIELD(IADC_H1_THR, 0, 7), + [F_CURR_INPUT] = REG_FIELD(IADC_DATA1, 0, 7), +}; + +static const struct reg_field bcl_v2_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(VCMP_CTL, 0, 1), + [F_IN_L0_THR] = REG_FIELD(VADC_L0_THR, 0, 7), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 5), + [F_IN_INPUT_EN] = REG_FIELD(VADC_CONV_REQ, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 7), + [F_CURR_MON_EN] = REG_FIELD(IADC_CONV_REQ, 0, 0), + [F_CURR_H0_THR] = REG_FIELD(IADC_H0_THR, 0, 7), + [F_CURR_H1_THR] = REG_FIELD(IADC_H1_THR, 0, 7), + [F_CURR_INPUT] = REG_FIELD(IADC_DATA1, 0, 7), +}; + +static const struct reg_field bcl_v3_bmx_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(VCMP_CTL, 0, 2), + [F_IN_L0_THR] = REG_FIELD(VADC_L0_THR, 0, 7), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 5), + [F_IN_INPUT_EN] = REG_FIELD(PARAM_1, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 7), + [F_CURR_MON_EN] = REG_FIELD(PARAM_1, 1, 1), + [F_CURR_H0_THR] = REG_FIELD(IADC_H0_THR, 0, 7), + [F_CURR_H1_THR] = REG_FIELD(IADC_H1_THR_GEN3, 0, 7), + [F_CURR_INPUT] = REG_FIELD(IADC_DATA1, 0, 7), +}; + +static const struct reg_field bcl_v3_wb_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(VCMP_CTL, 0, 2), + [F_IN_L0_THR] = REG_FIELD(VADC_L0_THR, 0, 7), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 5), + [F_IN_INPUT_EN] = REG_FIELD(PARAM_1, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 7), + [F_CURR_MON_EN] = REG_FIELD(PARAM_1, 1, 1), + [F_CURR_H0_THR] = REG_FIELD(IADC_H0_THR, 0, 7), + [F_CURR_H1_THR] = REG_FIELD(IADC_H1_THR, 0, 3), + [F_CURR_INPUT] = REG_FIELD(IADC_DATA1, 0, 7), +}; + +static const struct reg_field bcl_v3_core_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(VCMP_CTL, 0, 2), + [F_IN_L0_THR] = REG_FIELD(VCMP_L0_THR, 0, 5), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 5), + [F_IN_INPUT_EN] = REG_FIELD(PARAM_1, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 7), + [F_CURR_MON_EN] = REG_FIELD(PARAM_1, 1, 1), +}; + +static const struct reg_field bcl_v4_bmx_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(VCMP_CTL, 0, 2), + [F_IN_L0_THR] = REG_FIELD(VADC_L0_THR, 0, 7), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 5), + [F_IN_INPUT_EN] = REG_FIELD(PARAM_1, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 15), + [F_CURR_MON_EN] = REG_FIELD(PARAM_1, 1, 1), + [F_CURR_H0_THR] = REG_FIELD(IADC_H0_THR, 0, 7), + [F_CURR_H1_THR] = REG_FIELD(IADC_H1_THR_GEN3, 0, 7), + [F_CURR_INPUT] = REG_FIELD(IADC_DATA1, 0, 15), +}; + +static const struct reg_field bcl_v4_wb_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(VCMP_CTL, 0, 2), + [F_IN_L0_THR] = REG_FIELD(VADC_L0_THR, 0, 7), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 6), + [F_IN_INPUT_EN] = REG_FIELD(PARAM_1, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 15), + [F_CURR_MON_EN] = REG_FIELD(PARAM_1, 1, 1), + [F_CURR_H0_THR] = REG_FIELD(IADC_H0_THR, 0, 7), + [F_CURR_H1_THR] = REG_FIELD(IADC_H1_THR, 0, 4), + [F_CURR_INPUT] = REG_FIELD(IADC_DATA1, 0, 15), +}; + +static const struct reg_field bcl_v4_core_reg_fields[] = { + [F_IN_MON_EN] = REG_FIELD(VCMP_CTL, 0, 2), + [F_IN_L0_THR] = REG_FIELD(VCMP_L0_THR, 0, 6), + [F_IN_L1_THR] = REG_FIELD(VCMP_L1_THR, 0, 6), + [F_IN_INPUT_EN] = REG_FIELD(PARAM_1, 0, 0), + [F_IN_INPUT] = REG_FIELD(VADC_DATA1, 0, 15), + [F_CURR_MON_EN] = REG_FIELD(PARAM_1, 1, 1), +}; + +/* V1 BMX and core */ +static const struct bcl_desc pm7250b_data = { + .reg_fields = bcl_v1_reg_fields, + .num_reg_fields = F_MAX_FIELDS, + .data_field_bits_size = 8, + .thresh_field_bits_size = 7, + .channel[IN] = { + .base = 2250, + .max = 3600, + .step = 25, + .default_scale_nu = 194637, + .thresh_type = {ADC, INDEX}, + }, + .channel[CURR] = { + .max = 10000, + .default_scale_nu = 305180, + .thresh_type = {ADC, ADC}, + }, +}; + +/* V2 BMX and core */ +static const struct bcl_desc pm8350_data = { + .reg_fields = bcl_v2_reg_fields, + .num_reg_fields = F_MAX_FIELDS, + .data_field_bits_size = 8, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 2250, + .max = 3600, + .step = 25, + .default_scale_nu = 194637, + .thresh_type = {ADC, INDEX}, + }, + .channel[CURR] = { + .max = 10000, + .default_scale_nu = 305180, + .thresh_type = {ADC, ADC}, + }, +}; + +/* V3 BMX */ +static const struct bcl_desc pm8550b_data = { + .reg_fields = bcl_v3_bmx_reg_fields, + .num_reg_fields = F_MAX_FIELDS, + .data_field_bits_size = 8, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 2250, + .max = 3600, + .step = 25, + .default_scale_nu = 194637, + .thresh_type = {ADC, INDEX}, + }, + .channel[CURR] = { + .max = 12000, + .default_scale_nu = 366220, + .thresh_type = {ADC, ADC}, + }, +}; + +/* V3 WB */ +static const struct bcl_desc pmw5100_data = { + .reg_fields = bcl_v3_wb_reg_fields, + .num_reg_fields = F_MAX_FIELDS, + .data_field_bits_size = 8, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 2250, + .max = 3600, + .step = 25, + .default_scale_nu = 194637, + .thresh_type = {ADC, INDEX}, + }, + .channel[CURR] = { + .base = 800, + .max = 2000, + .step = 100, + .default_scale_nu = 61035, + .thresh_type = {ADC, INDEX}, + }, +}; + +/* V3 CORE */ +static const struct bcl_desc pm8550_data = { + .reg_fields = bcl_v3_core_reg_fields, + .num_reg_fields = F_CURR_MON_EN + 1, + .data_field_bits_size = 8, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 2250, + .max = 3600, + .step = 25, + .thresh_type = {INDEX, INDEX}, + }, +}; + +/* V4 BMX */ +static const struct bcl_desc pmih010_data = { + .reg_fields = bcl_v4_bmx_reg_fields, + .num_reg_fields = F_MAX_FIELDS, + .data_field_bits_size = 16, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 2250, + .max = 3600, + .step = 25, + .default_scale_nu = 194637, + .thresh_type = {ADC, INDEX}, + }, + .channel[CURR] = { + .max = 20000, + .default_scale_nu = 610370, + .thresh_type = {ADC, ADC}, + }, +}; + +/* V4 WB */ +static const struct bcl_desc pmw6100_data = { + .reg_fields = bcl_v4_wb_reg_fields, + .num_reg_fields = F_MAX_FIELDS, + .data_field_bits_size = 16, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 1500, + .max = 4000, + .step = 25, + .default_scale_nu = 194637, + .thresh_type = {ADC, INDEX}, + }, + .channel[CURR] = { + .base = 900, + .max = 3300, + .step = 150, + .default_scale_nu = 152586, + .thresh_type = {ADC, INDEX}, + }, +}; + +/* V4 CORE */ +static const struct bcl_desc pmh010_data = { + .reg_fields = bcl_v4_core_reg_fields, + .num_reg_fields = F_CURR_MON_EN + 1, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 1500, + .max = 4000, + .step = 25, + .thresh_type = {INDEX, INDEX}, + }, +}; + +/* V4 BMX with different scale */ +static const struct bcl_desc pmv010_data = { + .reg_fields = bcl_v4_bmx_reg_fields, + .num_reg_fields = F_MAX_FIELDS, + .data_field_bits_size = 16, + .thresh_field_bits_size = 8, + .channel[IN] = { + .base = 2250, + .max = 3600, + .step = 25, + .default_scale_nu = 194637, + .thresh_type = {ADC, INDEX}, + }, + .channel[CURR] = { + .max = 12000, + .default_scale_nu = 366220, + .thresh_type = {ADC, ADC}, + }, +}; + +/** + * bcl_convert_raw_to_milliunit - Convert raw value to milli unit + * @desc: BCL device descriptor + * @raw_val: Raw ADC value from hardware + * @type: type of the channel, in or curr + * @field_width: bits size for data or threshold field + * + * Return: value in milli unit + */ +static unsigned int bcl_convert_raw_to_milliunit(const struct bcl_desc *desc, int raw_val, + enum bcl_channel_type type, u8 field_width) +{ + u32 def_scale = desc->channel[type].default_scale_nu; + u32 lsb_weight = field_width > 8 ? 1 : 1 << field_width; + u32 scaling_factor = def_scale * lsb_weight; + + return div_s64((s64)raw_val * scaling_factor, 1000000); +} + +/** + * bcl_convert_milliunit_to_raw - Convert milli unit to raw value + * @desc: BCL device descriptor + * @ma_val: threshold value in milli unit + * @type: type of the channel, in or curr + * @field_width: bits size for data or threshold field + * + * Return: Raw ADC value for hardware + */ +static unsigned int bcl_convert_milliunit_to_raw(const struct bcl_desc *desc, int mval, + enum bcl_channel_type type, u8 field_width) +{ + u32 def_scale = desc->channel[type].default_scale_nu; + u32 lsb_weight = field_width > 8 ? 1 : 1 << field_width; + u32 scaling_factor = def_scale * lsb_weight; + + return div_s64((s64)mval * 1000000, scaling_factor); +} + +/** + * bcl_convert_milliunit_to_index - Convert milliunit to in or curr index + * @desc: BCL device descriptor + * @val: in or curr value in milli unit + * @type: type of the channel, in or curr + * + * Converts a value in milli unit to an index for BCL that use indexed thresholds. + * + * Return: Voltage index value + */ +static unsigned int bcl_convert_milliunit_to_index(const struct bcl_desc *desc, int val, + enum bcl_channel_type type) +{ + return div_s64((s64)val - desc->channel[type].base, desc->channel[type].step); +} + +/** + * bcl_convert_index_to_milliunit - Convert in or curr index to milli unit + * @desc: BCL device descriptor + * @val: index value + * @type: type of the channel, in or curr + * + * Converts an index value to milli unit for BCL that use indexed thresholds. + * + * Return: Voltage value in millivolts + */ +static unsigned int bcl_convert_index_to_milliunit(const struct bcl_desc *desc, int val, + enum bcl_channel_type type) +{ + return desc->channel[type].base + val * desc->channel[type].step; +} + +static int bcl_in_thresh_write(struct bcl_device *bcl, long value, enum bcl_limit_alarm lvl) +{ + const struct bcl_desc *desc = bcl->desc; + u32 raw_val; + + int thresh = clamp_val(value, desc->channel[IN].base, desc->channel[IN].max); + + if (desc->channel[IN].thresh_type[lvl] == ADC) + raw_val = bcl_convert_milliunit_to_raw(desc, thresh, IN, + desc->thresh_field_bits_size); + else + raw_val = bcl_convert_milliunit_to_index(desc, thresh, IN); + + return regmap_field_write(bcl->fields[F_IN_L0_THR + lvl], raw_val); +} + +static int bcl_curr_thresh_write(struct bcl_device *bcl, long value, enum bcl_limit_alarm lvl) +{ + const struct bcl_desc *desc = bcl->desc; + u32 raw_val; + + /* Clamp only to curr max */ + int thresh = clamp_val(value, value, desc->channel[CURR].max); + + if (desc->channel[CURR].thresh_type[lvl] == ADC) + raw_val = bcl_convert_milliunit_to_raw(desc, thresh, CURR, + desc->thresh_field_bits_size); + else + raw_val = bcl_convert_milliunit_to_index(desc, thresh, CURR); + + return regmap_field_write(bcl->fields[F_CURR_H0_THR + lvl], raw_val); +} + +static int bcl_in_thresh_read(struct bcl_device *bcl, enum bcl_limit_alarm lvl, long *out) +{ + int ret, thresh; + u32 raw_val = 0; + const struct bcl_desc *desc = bcl->desc; + + ret = bcl_read_field_value(bcl, F_IN_L0_THR + lvl, &raw_val); + if (ret) + return ret; + + if (desc->channel[IN].thresh_type[lvl] == ADC) + thresh = bcl_convert_raw_to_milliunit(desc, raw_val, IN, + desc->thresh_field_bits_size); + else + thresh = bcl_convert_index_to_milliunit(desc, raw_val, IN); + + *out = thresh; + + return 0; +} + +static int bcl_curr_thresh_read(struct bcl_device *bcl, enum bcl_limit_alarm lvl, long *out) +{ + int ret, thresh; + u32 raw_val = 0; + const struct bcl_desc *desc = bcl->desc; + + ret = bcl_read_field_value(bcl, F_CURR_H0_THR + lvl, &raw_val); + if (ret) + return ret; + + if (desc->channel[CURR].thresh_type[lvl] == ADC) + thresh = bcl_convert_raw_to_milliunit(desc, raw_val, CURR, + desc->thresh_field_bits_size); + else + thresh = bcl_convert_index_to_milliunit(desc, raw_val, CURR); + + *out = thresh; + + return 0; +} + +static int bcl_curr_input_read(struct bcl_device *bcl, long *out) +{ + int ret; + u32 raw_val = 0; + const struct bcl_desc *desc = bcl->desc; + + ret = bcl_read_field_value(bcl, F_CURR_INPUT, &raw_val); + if (ret) + return ret; + + /* + * The sensor sometime can read a value 0 if there are + * consecutive reads + */ + if (raw_val != 0) + bcl->last_curr_input = + bcl_convert_raw_to_milliunit(desc, raw_val, CURR, + desc->data_field_bits_size); + + *out = bcl->last_curr_input; + + return 0; +} + +static int bcl_in_input_read(struct bcl_device *bcl, long *out) +{ + int ret; + u32 raw_val = 0; + const struct bcl_desc *desc = bcl->desc; + + ret = bcl_read_field_value(bcl, F_IN_INPUT, &raw_val); + if (ret) + return ret; + + if (raw_val < GENMASK(desc->data_field_bits_size - 1, 0)) + bcl->last_in_input = + bcl_convert_raw_to_milliunit(desc, raw_val, IN, + desc->data_field_bits_size); + + *out = bcl->last_in_input; + + return 0; +} + +static int bcl_read_alarm_status(struct bcl_device *bcl, + enum bcl_limit_alarm lvl, long *status) +{ + int ret; + u32 raw_val = 0; + + ret = bcl_read_field_value(bcl, F_LVL0_ALARM + lvl, &raw_val); + if (ret) + return ret; + + *status = raw_val; + + return 0; +} + +static unsigned int bcl_get_version_major(const struct bcl_device *bcl) +{ + u32 raw_val = 0; + + bcl_read_field_value(bcl, F_V_MAJOR, &raw_val); + + return raw_val; +} + +static unsigned int bcl_get_version_minor(const struct bcl_device *bcl) +{ + u32 raw_val = 0; + + bcl_read_field_value(bcl, F_V_MINOR, &raw_val); + + return raw_val; +} + +static void bcl_hwmon_notify_event(struct bcl_device *bcl, enum bcl_limit_alarm alarm) +{ + if (bcl->in_mon_enabled) + hwmon_notify_event(bcl->hwmon_dev, hwmon_in, + in_lvl_to_attr_map[alarm], 0); + if (bcl->curr_mon_enabled) + hwmon_notify_event(bcl->hwmon_dev, hwmon_curr, + curr_lvl_to_attr_map[alarm], 0); +} + +static void bcl_alarm_enable_poll(struct work_struct *work) +{ + struct bcl_alarm_data *alarm = container_of(work, struct bcl_alarm_data, + alarm_poll_work.work); + struct bcl_device *bcl = alarm->device; + long status; + + guard(mutex)(&bcl->lock); + + if (bcl_read_alarm_status(bcl, alarm->type, &status)) + goto re_schedule; + + if (!status & !alarm->irq_enabled) { + bcl_enable_irq(alarm); + bcl_hwmon_notify_event(bcl, alarm->type); + return; + } + +re_schedule: + schedule_delayed_work(&alarm->alarm_poll_work, + msecs_to_jiffies(BCL_ALARM_POLLING_MS)); +} + +static irqreturn_t bcl_handle_alarm(int irq, void *data) +{ + struct bcl_alarm_data *alarm = data; + struct bcl_device *bcl = alarm->device; + long status; + + guard(mutex)(&bcl->lock); + + if (bcl_read_alarm_status(bcl, alarm->type, &status) || !status) + return IRQ_HANDLED; + + if (!bcl->hwmon_dev) + return IRQ_HANDLED; + + bcl_hwmon_notify_event(bcl, alarm->type); + + bcl_disable_irq(alarm); + schedule_delayed_work(&alarm->alarm_poll_work, + msecs_to_jiffies(BCL_ALARM_POLLING_MS)); + + dev_dbg(bcl->dev, "Irq:%d triggered for bcl type:%d\n", + irq, alarm->type); + + return IRQ_HANDLED; +} + +static umode_t bcl_hwmon_is_visible(const void *data, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + const struct bcl_device *bcl = data; + + switch (type) { + case hwmon_in: + if (!bcl->in_mon_enabled) + return 0; + switch (attr) { + case hwmon_in_input: + return bcl->in_input_enabled ? 0444 : 0; + case hwmon_in_label: + case hwmon_in_min_alarm: + case hwmon_in_lcrit_alarm: + return 0444; + case hwmon_in_min: + case hwmon_in_lcrit: + return 0644; + default: + return 0; + } + case hwmon_curr: + if (!bcl->curr_mon_enabled) + return 0; + switch (attr) { + case hwmon_curr_input: + case hwmon_curr_label: + case hwmon_curr_max_alarm: + case hwmon_curr_crit_alarm: + return 0444; + case hwmon_curr_max: + case hwmon_curr_crit: + return 0644; + default: + return 0; + } + default: + return 0; + } +} + +static int bcl_hwmon_write(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long val) +{ + struct bcl_device *bcl = dev_get_drvdata(dev); + int ret = -EOPNOTSUPP; + + guard(mutex)(&bcl->lock); + + switch (type) { + case hwmon_in: + switch (attr) { + case hwmon_in_min: + case hwmon_in_lcrit: + ret = bcl_in_thresh_write(bcl, val, in_attr_to_lvl_map[attr]); + break; + default: + ret = -EOPNOTSUPP; + } + break; + case hwmon_curr: + switch (attr) { + case hwmon_curr_max: + case hwmon_curr_crit: + ret = bcl_curr_thresh_write(bcl, val, curr_attr_to_lvl_map[attr]); + break; + default: + ret = -EOPNOTSUPP; + } + break; + default: + break; + } + + return ret; +} + +static int bcl_hwmon_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *value) +{ + struct bcl_device *bcl = dev_get_drvdata(dev); + int ret; + + guard(mutex)(&bcl->lock); + + switch (type) { + case hwmon_in: + switch (attr) { + case hwmon_in_input: + ret = bcl_in_input_read(bcl, value); + break; + case hwmon_in_min: + case hwmon_in_lcrit: + ret = bcl_in_thresh_read(bcl, in_attr_to_lvl_map[attr], value); + break; + case hwmon_in_min_alarm: + case hwmon_in_lcrit_alarm: + ret = bcl_read_alarm_status(bcl, in_attr_to_lvl_map[attr], value); + break; + default: + ret = -EOPNOTSUPP; + } + break; + case hwmon_curr: + switch (attr) { + case hwmon_curr_input: + ret = bcl_curr_input_read(bcl, value); + break; + case hwmon_curr_max: + case hwmon_curr_crit: + ret = bcl_curr_thresh_read(bcl, curr_attr_to_lvl_map[attr], value); + break; + case hwmon_curr_max_alarm: + case hwmon_curr_crit_alarm: + ret = bcl_read_alarm_status(bcl, curr_attr_to_lvl_map[attr], value); + break; + default: + ret = -EOPNOTSUPP; + } + break; + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +static int bcl_hwmon_read_string(struct device *dev, + enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + switch (type) { + case hwmon_in: + if (attr != hwmon_in_label) + break; + *str = bcl_channel_label[IN]; + return 0; + case hwmon_curr: + if (attr != hwmon_curr_label) + break; + *str = bcl_channel_label[CURR]; + return 0; + default: + break; + } + + return -EOPNOTSUPP; +} + +static const struct hwmon_ops bcl_hwmon_ops = { + .is_visible = bcl_hwmon_is_visible, + .read = bcl_hwmon_read, + .read_string = bcl_hwmon_read_string, + .write = bcl_hwmon_write, +}; + +static const struct hwmon_channel_info *bcl_hwmon_info[] = { + HWMON_CHANNEL_INFO(in, + HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_MIN | + HWMON_I_LCRIT | HWMON_I_MIN_ALARM | + HWMON_I_LCRIT_ALARM), + HWMON_CHANNEL_INFO(curr, + HWMON_C_INPUT | HWMON_C_LABEL | HWMON_C_MAX | + HWMON_C_CRIT | HWMON_C_MAX_ALARM | + HWMON_C_CRIT_ALARM), + NULL, +}; + +static const struct hwmon_chip_info bcl_hwmon_chip_info = { + .ops = &bcl_hwmon_ops, + .info = bcl_hwmon_info, +}; + +static int bcl_curr_thresh_update(struct bcl_device *bcl) +{ + int ret, i; + + if (!bcl->curr_thresholds[0]) + return 0; + + for (i = 0; i < ALARM_MAX; i++) { + ret = bcl_curr_thresh_write(bcl, bcl->curr_thresholds[i], i); + if (ret < 0) + return ret; + } + + return 0; +} + +static void bcl_hw_channel_mon_init(struct bcl_device *bcl) +{ + bcl->in_mon_enabled = bcl_in_monitor_enabled(bcl); + bcl->in_input_enabled = bcl_in_input_enabled(bcl); + bcl->curr_mon_enabled = bcl_curr_monitor_enabled(bcl); +} + +static int bcl_alarm_irq_init(struct platform_device *pdev, + struct bcl_device *bcl) +{ + int ret = 0, irq_num = 0, i = 0; + struct bcl_alarm_data *alarm; + + for (i = LVL0; i < ALARM_MAX; i++) { + alarm = &bcl->bcl_alarms[i]; + alarm->type = i; + alarm->device = bcl; + + ret = devm_delayed_work_autocancel(bcl->dev, &alarm->alarm_poll_work, + bcl_alarm_enable_poll); + if (ret) + return ret; + + irq_num = platform_get_irq_byname(pdev, bcl_int_names[i]); + if (irq_num <= 0) + continue; + + ret = devm_request_threaded_irq(&pdev->dev, irq_num, NULL, + bcl_handle_alarm, IRQF_ONESHOT, + bcl_int_names[i], alarm); + if (ret) { + dev_err(&pdev->dev, "Error requesting irq(%s).err:%d\n", + bcl_int_names[i], ret); + return ret; + } + alarm->irq = irq_num; + enable_irq_wake(alarm->irq); + alarm->irq_enabled = true; + } + + return 0; +} + +static int bcl_regmap_field_init(struct device *dev, struct bcl_device *bcl, + const struct bcl_desc *data) +{ + int i; + struct reg_field fields[F_MAX_FIELDS]; + + BUILD_BUG_ON(ARRAY_SIZE(common_reg_fields) != COMMON_FIELD_MAX); + + for (i = 0; i < data->num_reg_fields; i++) { + if (i < COMMON_FIELD_MAX) + fields[i] = common_reg_fields[i]; + else + fields[i] = data->reg_fields[i]; + + /* Need to adjust BCL base from regmap dynamically */ + fields[i].reg += bcl->base; + } + + return devm_regmap_field_bulk_alloc(dev, bcl->regmap, bcl->fields, + fields, data->num_reg_fields); +} + +static int bcl_get_device_property_data(struct platform_device *pdev, + struct bcl_device *bcl) +{ + struct device *dev = &pdev->dev; + int ret; + u32 reg; + + ret = device_property_read_u32(dev, "reg", ®); + if (ret < 0) + return ret; + + bcl->base = reg; + + device_property_read_u32_array(dev, "overcurrent-thresholds-milliamp", + bcl->curr_thresholds, 2); + return 0; +} + +static int bcl_probe(struct platform_device *pdev) +{ + struct bcl_device *bcl; + int ret; + + bcl = devm_kzalloc(&pdev->dev, sizeof(*bcl), GFP_KERNEL); + if (!bcl) + return -ENOMEM; + + bcl->dev = &pdev->dev; + bcl->desc = device_get_match_data(&pdev->dev); + if (!bcl->desc) + return -EINVAL; + + ret = devm_mutex_init(bcl->dev, &bcl->lock); + if (ret) + return ret; + + bcl->regmap = dev_get_regmap(pdev->dev.parent, NULL); + if (!bcl->regmap) { + dev_err(&pdev->dev, "Couldn't get parent's regmap\n"); + return -EINVAL; + } + + ret = bcl_get_device_property_data(pdev, bcl); + if (ret < 0) + return ret; + + ret = bcl_regmap_field_init(bcl->dev, bcl, bcl->desc); + if (ret < 0) { + dev_err(&pdev->dev, "Unable to allocate regmap fields, err:%d\n", ret); + return ret; + } + + if (!bcl_hw_is_enabled(bcl)) + return -ENODEV; + + ret = bcl_curr_thresh_update(bcl); + if (ret < 0) + return ret; + + ret = bcl_alarm_irq_init(pdev, bcl); + if (ret < 0) + return ret; + + bcl_hw_channel_mon_init(bcl); + + dev_set_drvdata(&pdev->dev, bcl); + + bcl->hwmon_name = devm_hwmon_sanitize_name(&pdev->dev, + dev_name(bcl->dev)); + if (IS_ERR(bcl->hwmon_name)) { + dev_err(&pdev->dev, "Failed to sanitize hwmon name\n"); + return PTR_ERR(bcl->hwmon_name); + } + + bcl->hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev, + bcl->hwmon_name, + bcl, + &bcl_hwmon_chip_info, + NULL); + if (IS_ERR(bcl->hwmon_dev)) { + dev_err(&pdev->dev, "Failed to register hwmon device: %ld\n", + PTR_ERR(bcl->hwmon_dev)); + return PTR_ERR(bcl->hwmon_dev); + } + + dev_dbg(&pdev->dev, "BCL hwmon device with version: %u.%u registered\n", + bcl_get_version_major(bcl), bcl_get_version_minor(bcl)); + + return 0; +} + +static const struct of_device_id bcl_match[] = { + { + .compatible = "qcom,bcl-v1", + .data = &pm7250b_data, + }, { + .compatible = "qcom,bcl-v2", + .data = &pm8350_data, + }, { + .compatible = "qcom,bcl-v3-bmx", + .data = &pm8550b_data, + }, { + .compatible = "qcom,bcl-v3-wb", + .data = &pmw5100_data, + }, { + .compatible = "qcom,bcl-v3-core", + .data = &pm8550_data, + }, { + .compatible = "qcom,bcl-v4-bmx", + .data = &pmih010_data, + }, { + .compatible = "qcom,bcl-v4-wb", + .data = &pmw6100_data, + }, { + .compatible = "qcom,bcl-v4-core", + .data = &pmh010_data, + }, { + .compatible = "qcom,bcl-v4-pmv010", + .data = &pmv010_data, + }, + { } +}; +MODULE_DEVICE_TABLE(of, bcl_match); + +static struct platform_driver bcl_driver = { + .probe = bcl_probe, + .driver = { + .name = BCL_DRIVER_NAME, + .of_match_table = bcl_match, + }, +}; + +MODULE_AUTHOR("Manaf Meethalavalappu Pallikunhi "); +MODULE_DESCRIPTION("QCOM BCL HWMON driver"); +module_platform_driver(bcl_driver); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/qcom-bcl-hwmon.h b/drivers/hwmon/qcom-bcl-hwmon.h new file mode 100644 index 0000000000000..28a7154d9486a --- /dev/null +++ b/drivers/hwmon/qcom-bcl-hwmon.h @@ -0,0 +1,311 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef __QCOM_BCL_HWMON_H__ +#define __QCOM_BCL_HWMON_H__ + +#define BCL_DRIVER_NAME "qcom-bcl-hwmon" + +/* BCL common regmap offset */ +#define REVISION1 0x0 +#define REVISION2 0x1 +#define STATUS 0x8 +#define INT_RT_STS 0x10 +#define EN_CTL1 0x46 + +/* BCL GEN1 regmap offsets */ +#define MODE_CTL1 0x41 +#define VADC_L0_THR 0x48 +#define VCMP_L1_THR 0x49 +#define IADC_H0_THR 0x4b +#define IADC_H1_THR 0x4c +#define VADC_CONV_REQ 0x72 +#define IADC_CONV_REQ 0x82 +#define VADC_DATA1 0x76 +#define IADC_DATA1 0x86 + +/* BCL GEN3 regmap offsets */ +#define VCMP_CTL 0x44 +#define VCMP_L0_THR 0x47 +#define PARAM_1 0x0e +#define IADC_H1_THR_GEN3 0x4d + +#define BCL_IN_INC_MV 25 +#define BCL_ALARM_POLLING_MS 50 + +/** + * enum bcl_limit_alarm - BCL alarm threshold levels + * @LVL0: Level 0 alarm threshold (mapped to in_min_alarm or curr_max_alarm) + * @LVL1: Level 1 alarm threshold (mapped to in_lcrit_alarm or curr_crit_alarm) + * @ALARM_MAX: sentinel value + * + * Defines the three threshold levels for BCL monitoring. Each level corresponds + * to different severity of in or curr conditions. + */ +enum bcl_limit_alarm { + LVL0, + LVL1, + + ALARM_MAX, +}; + +/** + * enum bcl_channel_type - BCL supported sensor channel type + * @IN: in (voltage) channel + * @CURR: curr (current) channel + * @CHANNEL_MAX: sentinel value + * + * Defines the supported channel types for bcl. + */ +enum bcl_channel_type { + IN, + CURR, + + CHANNEL_MAX, +}; + +/** + * enum bcl_thresh_type - voltage or current threshold representation type + * @ADC: Raw ADC value representation + * @INDEX: Index-based voltage or current representation + * + * Specifies how voltage or current thresholds are stored and interpreted in + * registers. Some PMICs use raw ADC values while others use indexed values. + */ +enum bcl_thresh_type { + ADC, + INDEX, +}; + +/** + * enum bcl_fields - BCL register field identifiers + * @F_V_MAJOR: Major revision info field + * @F_V_MINOR: Minor revision info field + * @F_CTL_EN: Monitor enable control field + * @F_LVL0_ALARM: Level 0 alarm status field + * @F_LVL1_ALARM: Level 1 alarm status field + * @COMMON_FIELD_MAX: sentinel value for common fields + * @F_IN_MON_EN: voltage monitor enable control field + * @F_IN_L0_THR: voltage level 0 threshold field + * @F_IN_L1_THR: voltage level 1 threshold field + * @F_IN_INPUT_EN: voltage input enable control field + * @F_IN_INPUT: voltage input data field + * @F_CURR_MON_EN: current monitor enable control field + * @F_CURR_H0_THR: current level 0 threshold field + * @F_CURR_H1_THR: current level 1 threshold field + * @F_CURR_INPUT: current input data field + * @F_MAX_FIELDS: sentinel value + * + * Enumeration of all register fields used by the BCL driver for accessing + * registers through regmap fields. + */ +enum bcl_fields { + F_V_MAJOR, + F_V_MINOR, + + F_CTL_EN, + + /* common alarm for in and curr channel */ + F_LVL0_ALARM, + F_LVL1_ALARM, + + COMMON_FIELD_MAX, + + F_IN_MON_EN = COMMON_FIELD_MAX, + F_IN_L0_THR, + F_IN_L1_THR, + + F_IN_INPUT_EN, + F_IN_INPUT, + + F_CURR_MON_EN, + F_CURR_H0_THR, + F_CURR_H1_THR, + + F_CURR_INPUT, + + F_MAX_FIELDS +}; + +#define ADD_BCL_HWMON_ALARM_MAPS(_type, lvl0_attr, lvl1_attr) \ + \ +static const u8 _type##_attr_to_lvl_map[] = { \ + [hwmon_##_type##_##lvl0_attr] = LVL0, \ + [hwmon_##_type##_##lvl1_attr] = LVL1, \ + [hwmon_##_type##_##lvl0_attr##_alarm] = LVL0, \ + [hwmon_##_type##_##lvl1_attr##_alarm] = LVL1, \ +}; \ + \ +static const u8 _type##_lvl_to_attr_map[ALARM_MAX] = { \ + [LVL0] = hwmon_##_type##_##lvl0_attr##_alarm, \ + [LVL1] = hwmon_##_type##_##lvl1_attr##_alarm, \ +} + +/** + * struct bcl_channel_cfg - BCL channel related configuration + * @default_scale_nu: Default scaling factor in nano unit + * @base: Base threshold value in milli unit + * @max: Maximum threshold value in milli unit + * @step: step increment value between two indexed threshold value + * @thresh_type: Array specifying threshold representation type for each alarm level + * + * Contains hardware-specific configuration and scaling parameters for different + * channel(voltage and current).. + */ + +struct bcl_channel_cfg { + u32 default_scale_nu; + u32 base; + u32 max; + u32 step; + u8 thresh_type[ALARM_MAX]; +}; + +/** + * struct bcl_desc - BCL device descriptor + * @reg_fields: Array of register field definitions for this device variant + * @channel: Each channel specific(voltage or current) configuration + * @num_reg_fields: Number of register field definitions for this device variant + * @data_field_bits_size: data read register bit size + * @thresh_field_bits_size: lsb bit size those are not included in threshold register + * + * Contains hardware-specific configuration and scaling parameters for different + * BCL variants. Each PMIC model may have different register layouts and + * conversion factors. + */ + +struct bcl_desc { + const struct reg_field *reg_fields; + struct bcl_channel_cfg channel[CHANNEL_MAX]; + u8 num_reg_fields; + u8 data_field_bits_size; + u8 thresh_field_bits_size; +}; + +struct bcl_device; + +/** + * struct bcl_alarm_data - BCL alarm interrupt data + * @irq: IRQ number assigned to this alarm + * @irq_enabled: Flag indicating if IRQ is enabled + * @type: Alarm level type (LVL0, or LVL1) + * @device: Pointer to parent BCL device structure + * @alarm_poll_work: delayed_work to poll alarm status + * + * Stores interrupt-related information for each alarm threshold level. + * Used by the IRQ handler to identify which alarm triggered. + */ +struct bcl_alarm_data { + int irq; + bool irq_enabled; + enum bcl_limit_alarm type; + struct bcl_device *device; + struct delayed_work alarm_poll_work; +}; + +/** + * struct bcl_device - Main BCL device structure + * @dev: Pointer to device structure + * @regmap: Regmap for accessing PMIC registers + * @fields: Array of regmap fields for register access + * @bcl_alarms: Array of alarm data structures for each threshold level + * @lock: Mutex for protecting concurrent hardware access + * @in_mon_enabled: Flag indicating if voltage monitoring is enabled + * @curr_mon_enabled: Flag indicating if current monitoring is enabled + * @curr_thresholds: Current threshold values in milliamps from dt-binding(LVL0 and LVL1) + * @base: the BCL regbase offset from regmap + * @in_input_enabled: Flag indicating if voltage input reading is enabled + * @last_in_input: Last valid voltage input reading in millivolts + * @last_curr_input: Last valid current input reading in milliamps + * @desc: Pointer to device descriptor with hardware-specific parameters + * @hwmon_dev: Pointer to registered hwmon device + * @hwmon_name: Sanitized name for hwmon device + * + * Main driver structure containing all state and configuration for a BCL + * monitoring instance. Manages voltage and current monitoring, thresholds, + * and alarm handling. + */ +struct bcl_device { + struct device *dev; + struct regmap *regmap; + u16 base; + struct regmap_field *fields[F_MAX_FIELDS]; + struct bcl_alarm_data bcl_alarms[ALARM_MAX]; + struct mutex lock; + u32 curr_thresholds[ALARM_MAX]; + u32 last_in_input; + u32 last_curr_input; + bool in_mon_enabled; + bool curr_mon_enabled; + bool in_input_enabled; + const struct bcl_desc *desc; + struct device *hwmon_dev; + char *hwmon_name; +}; + +/** + * bcl_read_field_value - Read alarm status for a given level + * @bcl: BCL device structure + * @id: Index in bcl->fields[] + * @val: Pointer to store val + * + * Return: 0 on success or regmap error code + */ +static inline int bcl_read_field_value(const struct bcl_device *bcl, enum bcl_fields id, u32 *val) +{ + return regmap_field_read(bcl->fields[id], val); +} + +/** + * bcl_field_enabled - Generic helper to check if a regmap field is enabled + * @bcl: BCL device structure + * @field: Index in bcl->fields[] + * + * Return: true if field is non-zero, false otherwise + */ +static inline bool bcl_field_enabled(const struct bcl_device *bcl, enum bcl_fields id) +{ + int ret; + u32 val = 0; + + ret = regmap_field_read(bcl->fields[id], &val); + if (ret) + return false; + + return !!val; +} + +#define bcl_in_input_enabled(bcl) bcl_field_enabled(bcl, F_IN_INPUT_EN) +#define bcl_curr_monitor_enabled(bcl) bcl_field_enabled(bcl, F_CURR_MON_EN) +#define bcl_in_monitor_enabled(bcl) bcl_field_enabled(bcl, F_IN_MON_EN) +#define bcl_hw_is_enabled(bcl) bcl_field_enabled(bcl, F_CTL_EN) + +/** + * bcl_enable_irq - Generic helper to enable alarm irq + * @alarm: BCL level alarm data + */ +static inline void bcl_enable_irq(struct bcl_alarm_data *alarm) +{ + if (alarm->irq_enabled) + return; + alarm->irq_enabled = true; + enable_irq(alarm->irq); + enable_irq_wake(alarm->irq); +} + +/** + * bcl_disable_irq - Generic helper to disable alarm irq + * @alarm: BCL level alarm data + */ +static inline void bcl_disable_irq(struct bcl_alarm_data *alarm) +{ + if (!alarm->irq_enabled) + return; + alarm->irq_enabled = false; + disable_irq_nosync(alarm->irq); + disable_irq_wake(alarm->irq); +} + +#endif /* __QCOM_BCL_HWMON_H__ */ diff --git a/drivers/hwmon/sy7636a-hwmon.c b/drivers/hwmon/sy7636a-hwmon.c index a12fc0ce70e76..d51daaf63d632 100644 --- a/drivers/hwmon/sy7636a-hwmon.c +++ b/drivers/hwmon/sy7636a-hwmon.c @@ -66,18 +66,13 @@ static const struct hwmon_chip_info sy7636a_chip_info = { static int sy7636a_sensor_probe(struct platform_device *pdev) { struct regmap *regmap = dev_get_regmap(pdev->dev.parent, NULL); - struct regulator *regulator; struct device *hwmon_dev; int err; if (!regmap) return -EPROBE_DEFER; - regulator = devm_regulator_get(&pdev->dev, "vcom"); - if (IS_ERR(regulator)) - return PTR_ERR(regulator); - - err = regulator_enable(regulator); + err = devm_regulator_get_enable(&pdev->dev, "vcom"); if (err) return err; diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c index 02c5a3bb1071c..84aaf817144cf 100644 --- a/drivers/hwmon/tmp401.c +++ b/drivers/hwmon/tmp401.c @@ -401,7 +401,7 @@ static int tmp401_chip_read(struct device *dev, u32 attr, int channel, long *val ret = regmap_read(data->regmap, TMP401_CONVERSION_RATE, ®val); if (ret < 0) return ret; - *val = (1 << (7 - regval)) * 125; + *val = (1 << (7 - min(regval, 7))) * 125; break; case hwmon_chip_temp_reset_history: *val = 0; diff --git a/drivers/hwmon/w83791d.c b/drivers/hwmon/w83791d.c index ace854b370a05..996e36951f9da 100644 --- a/drivers/hwmon/w83791d.c +++ b/drivers/hwmon/w83791d.c @@ -218,9 +218,14 @@ static u8 fan_to_reg(long rpm, int div) return clamp_val((1350000 + rpm * div / 2) / (rpm * div), 1, 254); } -#define FAN_FROM_REG(val, div) ((val) == 0 ? -1 : \ - ((val) == 255 ? 0 : \ - 1350000 / ((val) * (div)))) +static int fan_from_reg(int val, int div) +{ + if (val == 0) + return -1; + if (val == 255) + return 0; + return 1350000 / (val * div); +} /* for temp1 which is 8-bit resolution, LSB = 1 degree Celsius */ #define TEMP1_FROM_REG(val) ((val) * 1000) @@ -521,7 +526,7 @@ static ssize_t show_##reg(struct device *dev, struct device_attribute *attr, \ struct w83791d_data *data = w83791d_update_device(dev); \ int nr = sensor_attr->index; \ return sprintf(buf, "%d\n", \ - FAN_FROM_REG(data->reg[nr], DIV_FROM_REG(data->fan_div[nr]))); \ + fan_from_reg(data->reg[nr], DIV_FROM_REG(data->fan_div[nr]))); \ } show_fan_reg(fan); @@ -585,10 +590,10 @@ static ssize_t store_fan_div(struct device *dev, struct device_attribute *attr, if (err) return err; + mutex_lock(&data->update_lock); /* Save fan_min */ - min = FAN_FROM_REG(data->fan_min[nr], DIV_FROM_REG(data->fan_div[nr])); + min = fan_from_reg(data->fan_min[nr], DIV_FROM_REG(data->fan_div[nr])); - mutex_lock(&data->update_lock); data->fan_div[nr] = div_to_reg(nr, val); switch (nr) { diff --git a/drivers/hwmon/w83l786ng.c b/drivers/hwmon/w83l786ng.c index 9b81bd406e059..1d9109ca1585e 100644 --- a/drivers/hwmon/w83l786ng.c +++ b/drivers/hwmon/w83l786ng.c @@ -76,15 +76,25 @@ FAN_TO_REG(long rpm, int div) return clamp_val((1350000 + rpm * div / 2) / (rpm * div), 1, 254); } -#define FAN_FROM_REG(val, div) ((val) == 0 ? -1 : \ - ((val) == 255 ? 0 : \ - 1350000 / ((val) * (div)))) +static int fan_from_reg(int val, int div) +{ + if (val == 0) + return -1; + if (val == 255) + return 0; + return 1350000 / (val * div); +} /* for temp */ #define TEMP_TO_REG(val) (clamp_val(((val) < 0 ? (val) + 0x100 * 1000 \ : (val)) / 1000, 0, 0xff)) -#define TEMP_FROM_REG(val) (((val) & 0x80 ? \ - (val) - 0x100 : (val)) * 1000) + +static int temp_from_reg(int val) +{ + if (val & 0x80) + return (val - 0x100) * 1000; + return val * 1000; +} /* * The analog voltage inputs have 8mV LSB. Since the sysfs output is @@ -280,7 +290,7 @@ static ssize_t show_##reg(struct device *dev, struct device_attribute *attr, \ int nr = to_sensor_dev_attr(attr)->index; \ struct w83l786ng_data *data = w83l786ng_update_device(dev); \ return sprintf(buf, "%d\n", \ - FAN_FROM_REG(data->reg[nr], DIV_FROM_REG(data->fan_div[nr]))); \ + fan_from_reg(data->reg[nr], DIV_FROM_REG(data->fan_div[nr]))); \ } show_fan_reg(fan); @@ -347,7 +357,7 @@ store_fan_div(struct device *dev, struct device_attribute *attr, /* Save fan_min */ mutex_lock(&data->update_lock); - min = FAN_FROM_REG(data->fan_min[nr], DIV_FROM_REG(data->fan_div[nr])); + min = fan_from_reg(data->fan_min[nr], DIV_FROM_REG(data->fan_div[nr])); data->fan_div[nr] = DIV_TO_REG(val); @@ -409,7 +419,7 @@ show_temp(struct device *dev, struct device_attribute *attr, char *buf) int nr = sensor_attr->nr; int index = sensor_attr->index; struct w83l786ng_data *data = w83l786ng_update_device(dev); - return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp[nr][index])); + return sprintf(buf, "%d\n", temp_from_reg(data->temp[nr][index])); } static ssize_t diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 27b47b8623c09..2d8de835bc242 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -88,7 +88,9 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) * make sure the module is enabled and clocked before reading * the module SYSSTATUS register */ - devm_pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/hwtracing/Kconfig b/drivers/hwtracing/Kconfig index 911ee977103c0..8a640218eed84 100644 --- a/drivers/hwtracing/Kconfig +++ b/drivers/hwtracing/Kconfig @@ -7,4 +7,6 @@ source "drivers/hwtracing/intel_th/Kconfig" source "drivers/hwtracing/ptt/Kconfig" +source "drivers/hwtracing/qcom/Kconfig" + endmenu diff --git a/drivers/hwtracing/coresight/Makefile b/drivers/hwtracing/coresight/Makefile index ab16d06783a57..821a1b06b20c2 100644 --- a/drivers/hwtracing/coresight/Makefile +++ b/drivers/hwtracing/coresight/Makefile @@ -55,5 +55,5 @@ coresight-cti-y := coresight-cti-core.o coresight-cti-platform.o \ obj-$(CONFIG_ULTRASOC_SMB) += ultrasoc-smb.o obj-$(CONFIG_CORESIGHT_DUMMY) += coresight-dummy.o obj-$(CONFIG_CORESIGHT_CTCU) += coresight-ctcu.o -coresight-ctcu-y := coresight-ctcu-core.o +coresight-ctcu-y := coresight-ctcu-core.o coresight-ctcu-byte-cntr.o obj-$(CONFIG_CORESIGHT_KUNIT_TESTS) += coresight-kunit-tests.o diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 3267192f0c1c6..7db1801d74320 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -313,7 +313,8 @@ static void coresight_disable_sink(struct coresight_device *csdev) static int coresight_enable_link(struct coresight_device *csdev, struct coresight_device *parent, struct coresight_device *child, - struct coresight_device *source) + struct coresight_device *source, + enum cs_mode mode) { int link_subtype; struct coresight_connection *inconn, *outconn; @@ -330,7 +331,7 @@ static int coresight_enable_link(struct coresight_device *csdev, if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_SPLIT && IS_ERR(outconn)) return PTR_ERR(outconn); - return link_ops(csdev)->enable(csdev, inconn, outconn); + return link_ops(csdev)->enable(csdev, inconn, outconn, mode); } static void coresight_disable_link(struct coresight_device *csdev, @@ -546,7 +547,7 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode, case CORESIGHT_DEV_TYPE_LINK: parent = list_prev_entry(nd, link)->csdev; child = list_next_entry(nd, link)->csdev; - ret = coresight_enable_link(csdev, parent, child, source); + ret = coresight_enable_link(csdev, parent, child, source, mode); if (ret) goto err_disable_helpers; break; @@ -581,6 +582,65 @@ struct coresight_device *coresight_get_sink(struct coresight_path *path) } EXPORT_SYMBOL_GPL(coresight_get_sink); +/** + * coresight_get_helper: find the helper device of the assigned csdev. + * + * @csdev: The csdev the helper device is conntected to. + * @type: helper_subtype of the expected helper device. + * + * Retrieve the helper device for the specific csdev based on its + * helper_subtype. + * + * Return: the helper's csdev upon success or NULL for fail. + */ +struct coresight_device *coresight_get_helper(struct coresight_device *csdev, + int type) +{ + int i; + struct coresight_device *helper; + + /* protect the connections */ + mutex_lock(&coresight_mutex); + for (i = 0; i < csdev->pdata->nr_outconns; ++i) { + helper = csdev->pdata->out_conns[i]->dest_dev; + if (!helper || !coresight_is_helper(helper)) + continue; + + if (helper->subtype.helper_subtype == type) { + mutex_unlock(&coresight_mutex); + return helper; + } + } + mutex_unlock(&coresight_mutex); + + return NULL; +} +EXPORT_SYMBOL_GPL(coresight_get_helper); + +/** + * coresight_get_in_port_dest: get the in-port number of the dest device + * that is connected to the src device. + * + * @src: csdev of the source device. + * @dest: csdev of the destination device. + * + * Return: port number upon success or -EINVAL for fail. + */ +int coresight_get_in_port_dest(struct coresight_device *src, + struct coresight_device *dest) +{ + struct coresight_platform_data *pdata = dest->pdata; + int i; + + for (i = 0; i < pdata->nr_inconns; ++i) { + if (pdata->in_conns[i]->src_dev == src) + return pdata->in_conns[i]->dest_port; + } + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(coresight_get_in_port_dest); + u32 coresight_get_sink_id(struct coresight_device *csdev) { if (!csdev->ea) diff --git a/drivers/hwtracing/coresight/coresight-ctcu-byte-cntr.c b/drivers/hwtracing/coresight/coresight-ctcu-byte-cntr.c new file mode 100644 index 0000000000000..d611d26f6e8e7 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-ctcu-byte-cntr.c @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include + +#include "coresight-ctcu.h" +#include "coresight-priv.h" +#include "coresight-tmc.h" + +static irqreturn_t byte_cntr_handler(int irq, void *data) +{ + struct ctcu_byte_cntr *byte_cntr_data = (struct ctcu_byte_cntr *)data; + + atomic_inc(&byte_cntr_data->irq_cnt); + wake_up(&byte_cntr_data->wq); + + return IRQ_HANDLED; +} + +static void ctcu_reset_sysfs_buf(struct tmc_drvdata *drvdata) +{ + u32 sts; + + CS_UNLOCK(drvdata->base); + tmc_write_rrp(drvdata, drvdata->sysfs_buf->hwaddr); + tmc_write_rwp(drvdata, drvdata->sysfs_buf->hwaddr); + sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL; + writel_relaxed(sts, drvdata->base + TMC_STS); + CS_LOCK(drvdata->base); +} + +static void ctcu_cfg_byte_cntr_reg(struct tmc_drvdata *drvdata, u32 val, u32 offset) +{ + struct ctcu_drvdata *ctcu_drvdata; + struct coresight_device *helper; + + helper = coresight_get_helper(drvdata->csdev, CORESIGHT_DEV_SUBTYPE_HELPER_CTCU); + if (!helper) + return; + + ctcu_drvdata = dev_get_drvdata(helper->dev.parent); + /* A one value for IRQCTRL register represents 8 bytes */ + ctcu_program_register(ctcu_drvdata, val / 8, offset); +} + +static struct ctcu_byte_cntr *ctcu_get_byte_cntr_data(struct tmc_drvdata *drvdata) +{ + struct ctcu_byte_cntr *byte_cntr_data; + struct ctcu_drvdata *ctcu_drvdata; + struct coresight_device *helper; + int port; + + helper = coresight_get_helper(drvdata->csdev, CORESIGHT_DEV_SUBTYPE_HELPER_CTCU); + if (!helper) + return NULL; + + port = coresight_get_in_port_dest(drvdata->csdev, helper); + if (port < 0) + return NULL; + + ctcu_drvdata = dev_get_drvdata(helper->dev.parent); + byte_cntr_data = &ctcu_drvdata->byte_cntr_data[port]; + return byte_cntr_data; +} + +static bool ctcu_byte_cntr_switch_buffer(struct tmc_drvdata *drvdata, + struct ctcu_byte_cntr *byte_cntr_data) +{ + struct etr_buf_node *nd, *next, *curr_node, *picked_node; + struct etr_buf *curr_buf = drvdata->sysfs_buf; + bool found_free_buf = false; + + if (WARN_ON(!drvdata || !byte_cntr_data)) + return found_free_buf; + + /* Stop the ETR before we start the switch */ + if (coresight_get_mode(drvdata->csdev) != CS_MODE_DISABLED) + tmc_etr_enable_disable_hw(drvdata, false); + + list_for_each_entry_safe(nd, next, &drvdata->etr_buf_list, node) { + /* curr_buf is free for next round */ + if (nd->sysfs_buf == curr_buf) { + nd->is_free = true; + curr_node = nd; + } + + if (!found_free_buf && nd->is_free && nd->sysfs_buf != curr_buf) { + picked_node = nd; + found_free_buf = true; + } + } + + if (found_free_buf) { + curr_node->pos = 0; + drvdata->reading_node = curr_node; + drvdata->sysfs_buf = picked_node->sysfs_buf; + drvdata->etr_buf = picked_node->sysfs_buf; + picked_node->is_free = false; + /* Reset irq_cnt for next etr_buf */ + atomic_set(&byte_cntr_data->irq_cnt, 0); + /* Reset rrp and rwp when the system has switched the buffer*/ + ctcu_reset_sysfs_buf(drvdata); + /* Restart the ETR when we find a free buffer */ + if (coresight_get_mode(drvdata->csdev) != CS_MODE_DISABLED) + tmc_etr_enable_disable_hw(drvdata, true); + } + + return found_free_buf; +} + +/* + * ctcu_byte_cntr_get_data() - reads data from the deactivated and filled buffer. + * The byte-cntr reading work reads data from the deactivated and filled buffer. + * The read operation waits for a buffer to become available, either filled or + * upon timeout, and then reads trace data from the synced buffer. + */ +static ssize_t ctcu_byte_cntr_get_data(struct tmc_drvdata *drvdata, loff_t pos, + size_t len, char **bufpp) +{ + struct etr_buf *sysfs_buf = drvdata->sysfs_buf; + struct device *dev = &drvdata->csdev->dev; + ssize_t actual, size = sysfs_buf->size; + struct ctcu_byte_cntr *byte_cntr_data; + size_t thresh_val; + atomic_t *irq_cnt; + int ret; + + byte_cntr_data = ctcu_get_byte_cntr_data(drvdata); + if (!byte_cntr_data) + return -EINVAL; + + thresh_val = byte_cntr_data->thresh_val; + irq_cnt = &byte_cntr_data->irq_cnt; + +wait_buffer: + if (!byte_cntr_data->reading_buf) { + ret = wait_event_interruptible_timeout(byte_cntr_data->wq, + ((atomic_read(irq_cnt) + 1) * thresh_val >= size) || + !byte_cntr_data->enable, + BYTE_CNTR_TIMEOUT); + if (ret < 0) + return ret; + /* + * The current etr_buf is almost full or timeout is triggered, + * so switch the buffer and mark the switched buffer as reading. + */ + if (byte_cntr_data->enable) { + if (!ctcu_byte_cntr_switch_buffer(drvdata, byte_cntr_data)) { + dev_err(dev, "Switch buffer failed for byte-cntr\n"); + return -EINVAL; + } + + byte_cntr_data->reading_buf = true; + } else { + /* + * TMC-ETR has been disabled, so directly reads data from + * the drvdata->sysfs_buf. + */ + actual = drvdata->sysfs_ops->get_trace_data(drvdata, pos, len, bufpp); + if (actual > 0) { + byte_cntr_data->total_size += actual; + return actual; + } + + /* Exit byte-cntr reading */ + return 0; + } + } + + /* Check the status of current etr_buf*/ + if ((atomic_read(irq_cnt) + 1) * thresh_val >= size) + /* + * Unlikely to find a free buffer to switch, so just disable + * the ETR for a while. + */ + if (!ctcu_byte_cntr_switch_buffer(drvdata, byte_cntr_data)) + dev_warn(dev, "No available buffer to store data, disable ETR\n"); + + pos = drvdata->reading_node->pos; + actual = drvdata->sysfs_ops->get_trace_data(drvdata, pos, len, bufpp); + if (actual <= 0) { + /* Reset flags upon reading is finished or failed */ + byte_cntr_data->reading_buf = false; + drvdata->reading_node = NULL; + + /* + * Nothing in the buffer, waiting for the next buffer + * to be filled. + */ + if (actual == 0) + goto wait_buffer; + } else + byte_cntr_data->total_size += actual; + + return actual; +} + +static int ctcu_read_prepare_byte_cntr(struct tmc_drvdata *drvdata) +{ + struct ctcu_byte_cntr *byte_cntr_data; + unsigned long flags; + int ret = 0; + + /* config types are set a boot time and never change */ + if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR)) + return -EINVAL; + + /* + * Byte counter reading should start only after the TMC-ETR has been + * enabled, which implies that the sysfs_buf has already been setup + * in drvdata. + */ + if (!drvdata->sysfs_buf) + return -EINVAL; + + byte_cntr_data = ctcu_get_byte_cntr_data(drvdata); + if (!byte_cntr_data) + return -EINVAL; + + /* + * The threshold value must not exceed the buffer size. + * A margin should be maintained between the two values to account + * for the time gap between the interrupt and buffer switching. + */ + if (byte_cntr_data->thresh_val + SZ_16K >= drvdata->size) { + dev_err(&drvdata->csdev->dev, "The threshold value is too large\n"); + return -EINVAL; + } + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + if (byte_cntr_data->reading) { + ret = -EBUSY; + goto out_unlock; + } + + byte_cntr_data->reading = true; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + /* Setup an available etr_buf_list for byte-cntr */ + ret = tmc_create_etr_buf_list(drvdata, 2); + if (ret) + goto out; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + atomic_set(&byte_cntr_data->irq_cnt, 0); + /* Configure the byte-cntr register to enable IRQ */ + ctcu_cfg_byte_cntr_reg(drvdata, byte_cntr_data->thresh_val, + byte_cntr_data->irq_ctrl_offset); + enable_irq_wake(byte_cntr_data->irq); + byte_cntr_data->total_size = 0; + +out_unlock: + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + +out: + return ret; +} + +static int ctcu_read_unprepare_byte_cntr(struct tmc_drvdata *drvdata) +{ + struct device *dev = &drvdata->csdev->dev; + struct ctcu_byte_cntr *byte_cntr_data; + unsigned long flags; + + byte_cntr_data = ctcu_get_byte_cntr_data(drvdata); + if (!byte_cntr_data) + return -EINVAL; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + /* Configure the byte-cntr register to disable IRQ */ + ctcu_cfg_byte_cntr_reg(drvdata, 0, byte_cntr_data->irq_ctrl_offset); + disable_irq_wake(byte_cntr_data->irq); + byte_cntr_data->reading = false; + byte_cntr_data->reading_buf = false; + drvdata->reading_node = NULL; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + dev_dbg(dev, "send data total size:%llu bytes\n", byte_cntr_data->total_size); + tmc_clean_etr_buf_list(drvdata); + + return 0; +} + +static const struct sysfs_read_ops byte_cntr_sysfs_read_ops = { + .read_prepare = ctcu_read_prepare_byte_cntr, + .read_unprepare = ctcu_read_unprepare_byte_cntr, + .get_trace_data = ctcu_byte_cntr_get_data, +}; + +/* Start the byte-cntr function when the path is enabled. */ +void ctcu_byte_cntr_start(struct coresight_device *csdev, struct coresight_path *path) +{ + struct ctcu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct coresight_device *sink = coresight_get_sink(path); + struct ctcu_byte_cntr *byte_cntr_data; + int port_num; + + if (!sink) + return; + + port_num = coresight_get_in_port_dest(sink, csdev); + if (port_num < 0) + return; + + byte_cntr_data = &drvdata->byte_cntr_data[port_num]; + /* Don't start byte-cntr function when threshold is not set. */ + if (!byte_cntr_data->thresh_val || byte_cntr_data->enable) + return; + + guard(raw_spinlock_irqsave)(&byte_cntr_data->spin_lock); + byte_cntr_data->enable = true; + byte_cntr_data->reading_buf = false; +} + +/* Stop the byte-cntr function when the path is disabled. */ +void ctcu_byte_cntr_stop(struct coresight_device *csdev, struct coresight_path *path) +{ + struct ctcu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct coresight_device *sink = coresight_get_sink(path); + struct ctcu_byte_cntr *byte_cntr_data; + int port_num; + + if (!sink || coresight_get_mode(sink) == CS_MODE_SYSFS) + return; + + port_num = coresight_get_in_port_dest(sink, csdev); + if (port_num < 0) + return; + + byte_cntr_data = &drvdata->byte_cntr_data[port_num]; + guard(raw_spinlock_irqsave)(&byte_cntr_data->spin_lock); + byte_cntr_data->enable = false; +} + +void ctcu_byte_cntr_init(struct device *dev, struct ctcu_drvdata *drvdata, int etr_num) +{ + struct ctcu_byte_cntr *byte_cntr_data; + struct device_node *nd = dev->of_node; + int irq_num, ret, i; + + drvdata->byte_cntr_sysfs_read_ops = &byte_cntr_sysfs_read_ops; + for (i = 0; i < etr_num; i++) { + byte_cntr_data = &drvdata->byte_cntr_data[i]; + irq_num = of_irq_get_byname(nd, byte_cntr_data->irq_name); + if (irq_num < 0) { + dev_err(dev, "Failed to get IRQ from DT for %s\n", + byte_cntr_data->irq_name); + continue; + } + + ret = devm_request_irq(dev, irq_num, byte_cntr_handler, + IRQF_TRIGGER_RISING | IRQF_SHARED, + dev_name(dev), byte_cntr_data); + if (ret) { + dev_err(dev, "Failed to register IRQ for %s\n", + byte_cntr_data->irq_name); + continue; + } + + byte_cntr_data->irq = irq_num; + init_waitqueue_head(&byte_cntr_data->wq); + } +} diff --git a/drivers/hwtracing/coresight/coresight-ctcu-core.c b/drivers/hwtracing/coresight/coresight-ctcu-core.c index c586495e9a088..46e71319adc79 100644 --- a/drivers/hwtracing/coresight/coresight-ctcu-core.c +++ b/drivers/hwtracing/coresight/coresight-ctcu-core.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "coresight-ctcu.h" #include "coresight-priv.h" @@ -45,17 +46,23 @@ DEFINE_CORESIGHT_DEVLIST(ctcu_devs, "ctcu"); #define CTCU_ATID_REG_BIT(traceid) (traceid % 32) #define CTCU_ATID_REG_SIZE 0x10 +#define CTCU_ETR0_IRQCTRL 0x6c +#define CTCU_ETR1_IRQCTRL 0x70 #define CTCU_ETR0_ATID0 0xf8 #define CTCU_ETR1_ATID0 0x108 static const struct ctcu_etr_config sa8775p_etr_cfgs[] = { { - .atid_offset = CTCU_ETR0_ATID0, - .port_num = 0, + .atid_offset = CTCU_ETR0_ATID0, + .irq_ctrl_offset = CTCU_ETR0_IRQCTRL, + .irq_name = "etr0", + .port_num = 0, }, { - .atid_offset = CTCU_ETR1_ATID0, - .port_num = 1, + .atid_offset = CTCU_ETR1_ATID0, + .irq_ctrl_offset = CTCU_ETR1_IRQCTRL, + .irq_name = "etr1", + .port_num = 1, }, }; @@ -64,6 +71,71 @@ static const struct ctcu_config sa8775p_cfgs = { .num_etr_config = ARRAY_SIZE(sa8775p_etr_cfgs), }; +void ctcu_program_register(struct ctcu_drvdata *drvdata, u32 val, u32 offset) +{ + CS_UNLOCK(drvdata->base); + ctcu_writel(drvdata, val, offset); + CS_LOCK(drvdata->base); +} + +static ssize_t irq_threshold_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ctcu_drvdata *drvdata = dev_get_drvdata(dev->parent); + int i, len = 0; + + for (i = 0; i < ETR_MAX_NUM; i++) { + if (drvdata->byte_cntr_data[i].irq_ctrl_offset) + len += scnprintf(buf + len, PAGE_SIZE - len, "%u ", + drvdata->byte_cntr_data[i].thresh_val); + } + + len += scnprintf(buf + len, PAGE_SIZE - len, "\n"); + + return len; +} + +/* Program a valid value into IRQCTRL register will enable byte-cntr interrupt */ +static ssize_t irq_threshold_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + struct ctcu_drvdata *drvdata = dev_get_drvdata(dev->parent); + u32 thresh_vals[ETR_MAX_NUM] = { 0 }; + int num, i; + + num = sscanf(buf, "%i %i", &thresh_vals[0], &thresh_vals[1]); + if (num <= 0 || num > ETR_MAX_NUM) + return -EINVAL; + + /* Threshold 0 disables the interruption. */ + guard(raw_spinlock_irqsave)(&drvdata->spin_lock); + for (i = 0; i < num; i++) { + /* A small threshold will result in a large number of interruptions */ + if (thresh_vals[i] && thresh_vals[i] < SZ_4K) + return -EINVAL; + + if (drvdata->byte_cntr_data[i].irq_ctrl_offset) + drvdata->byte_cntr_data[i].thresh_val = thresh_vals[i]; + } + + return size; +} +static DEVICE_ATTR_RW(irq_threshold); + +static struct attribute *ctcu_attrs[] = { + &dev_attr_irq_threshold.attr, + NULL, +}; + +static struct attribute_group ctcu_attr_grp = { + .attrs = ctcu_attrs, +}; + +static const struct attribute_group *ctcu_attr_grps[] = { + &ctcu_attr_grp, + NULL, +}; + static void ctcu_program_atid_register(struct ctcu_drvdata *drvdata, u32 reg_offset, u8 bit, bool enable) { @@ -118,23 +190,6 @@ static int __ctcu_set_etr_traceid(struct coresight_device *csdev, u8 traceid, in return 0; } -/* - * Searching the sink device from helper's view in case there are multiple helper devices - * connected to the sink device. - */ -static int ctcu_get_active_port(struct coresight_device *sink, struct coresight_device *helper) -{ - struct coresight_platform_data *pdata = helper->pdata; - int i; - - for (i = 0; i < pdata->nr_inconns; ++i) { - if (pdata->in_conns[i]->src_dev == sink) - return pdata->in_conns[i]->dest_port; - } - - return -EINVAL; -} - static int ctcu_set_etr_traceid(struct coresight_device *csdev, struct coresight_path *path, bool enable) { @@ -147,7 +202,7 @@ static int ctcu_set_etr_traceid(struct coresight_device *csdev, struct coresight return -EINVAL; } - port_num = ctcu_get_active_port(sink, csdev); + port_num = coresight_get_in_port_dest(sink, csdev); if (port_num < 0) return -EINVAL; @@ -160,6 +215,8 @@ static int ctcu_enable(struct coresight_device *csdev, enum cs_mode mode, void * { struct coresight_path *path = (struct coresight_path *)data; + ctcu_byte_cntr_start(csdev, path); + return ctcu_set_etr_traceid(csdev, path, true); } @@ -167,6 +224,8 @@ static int ctcu_disable(struct coresight_device *csdev, void *data) { struct coresight_path *path = (struct coresight_path *)data; + ctcu_byte_cntr_stop(csdev, path); + return ctcu_set_etr_traceid(csdev, path, false); } @@ -217,7 +276,11 @@ static int ctcu_probe(struct platform_device *pdev) for (i = 0; i < cfgs->num_etr_config; i++) { etr_cfg = &cfgs->etr_cfgs[i]; drvdata->atid_offset[i] = etr_cfg->atid_offset; + drvdata->byte_cntr_data[i].irq_name = etr_cfg->irq_name; + drvdata->byte_cntr_data[i].irq_ctrl_offset = + etr_cfg->irq_ctrl_offset; } + ctcu_byte_cntr_init(dev, drvdata, cfgs->num_etr_config); } } @@ -229,6 +292,7 @@ static int ctcu_probe(struct platform_device *pdev) desc.subtype.helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CTCU; desc.pdata = pdata; desc.dev = dev; + desc.groups = ctcu_attr_grps; desc.ops = &ctcu_ops; desc.access = CSDEV_ACCESS_IOMEM(base); diff --git a/drivers/hwtracing/coresight/coresight-ctcu.h b/drivers/hwtracing/coresight/coresight-ctcu.h index e9594c38dd91c..21938a201dbb1 100644 --- a/drivers/hwtracing/coresight/coresight-ctcu.h +++ b/drivers/hwtracing/coresight/coresight-ctcu.h @@ -5,19 +5,28 @@ #ifndef _CORESIGHT_CTCU_H #define _CORESIGHT_CTCU_H + +#include #include "coresight-trace-id.h" +#include "coresight-tmc.h" /* Maximum number of supported ETR devices for a single CTCU. */ #define ETR_MAX_NUM 2 +#define BYTE_CNTR_TIMEOUT (5 * HZ) + /** * struct ctcu_etr_config * @atid_offset: offset to the ATID0 Register. - * @port_num: in-port number of CTCU device that connected to ETR. + * @port_num: in-port number of the CTCU device that connected to ETR. + * @irq_ctrl_offset: offset to the BYTECNTRVAL register. + * @irq_name: IRQ name in dt node. */ struct ctcu_etr_config { const u32 atid_offset; const u32 port_num; + const u32 irq_ctrl_offset; + const char *irq_name; }; struct ctcu_config { @@ -25,15 +34,52 @@ struct ctcu_config { int num_etr_config; }; -struct ctcu_drvdata { - void __iomem *base; - struct clk *apb_clk; - struct device *dev; - struct coresight_device *csdev; +/** + * struct ctcu_byte_cntr + * @enable: indicates that byte_cntr function is enabled or not. + * @reading: indicates that byte-cntr reading is started. + * @reading_buf: indicates that byte-cntr is reading data from the buffer. + * @thresh_val: threshold to trigger a interruption. + * @total_size: total size of transferred data. + * @irq: allocated number of the IRQ. + * @irq_cnt: IRQ count number for triggered interruptions. + * @wq: waitqueue for reading data from ETR buffer. + * @spin_lock: spinlock of byte_cntr_data. + * @irq_ctrl_offset: offset to the BYTECNTVAL Register. + * @irq_name: IRQ name defined in DT. + */ +struct ctcu_byte_cntr { + bool enable; + bool reading; + bool reading_buf; + u32 thresh_val; + u64 total_size; + int irq; + atomic_t irq_cnt; + wait_queue_head_t wq; raw_spinlock_t spin_lock; - u32 atid_offset[ETR_MAX_NUM]; + u32 irq_ctrl_offset; + const char *irq_name; +}; + +struct ctcu_drvdata { + void __iomem *base; + struct clk *apb_clk; + struct device *dev; + struct coresight_device *csdev; + struct ctcu_byte_cntr byte_cntr_data[ETR_MAX_NUM]; + raw_spinlock_t spin_lock; + u32 atid_offset[ETR_MAX_NUM]; /* refcnt for each traceid of each sink */ - u8 traceid_refcnt[ETR_MAX_NUM][CORESIGHT_TRACE_ID_RES_TOP]; + u8 traceid_refcnt[ETR_MAX_NUM][CORESIGHT_TRACE_ID_RES_TOP]; + const struct sysfs_read_ops *byte_cntr_sysfs_read_ops; }; +void ctcu_program_register(struct ctcu_drvdata *drvdata, u32 val, u32 offset); + +/* Byte-cntr functions */ +void ctcu_byte_cntr_start(struct coresight_device *csdev, struct coresight_path *path); +void ctcu_byte_cntr_stop(struct coresight_device *csdev, struct coresight_path *path); +void ctcu_byte_cntr_init(struct device *dev, struct ctcu_drvdata *drvdata, int port_num); + #endif diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 8fb30dd73fd20..d6a28df484846 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,54 @@ #include "coresight-priv.h" #include "coresight-cti.h" +#define CTI_EXTENDED "qcom,coresight-cti-extended" + +static const int cti_normal_offset[] = { + [CTIINTACK] = 0x010, + [CTIAPPSET] = 0x014, + [CTIAPPCLEAR] = 0x018, + [CTIAPPPULSE] = 0x01C, + [CTIINEN] = 0x020, + [CTIOUTEN] = 0x0A0, + [CTITRIGINSTATUS] = 0x130, + [CTITRIGOUTSTATUS] = 0x134, + [CTICHINSTATUS] = 0x138, + [CTICHOUTSTATUS] = 0x13C, + [CTIGATE] = 0x140, + [ASICCTL] = 0x144, + [ITCHINACK] = 0xEDC, + [ITTRIGINACK] = 0xEE0, + [ITCHOUT] = 0xEE4, + [ITTRIGOUT] = 0xEE8, + [ITCHOUTACK] = 0xEEC, + [ITTRIGOUTACK] = 0xEF0, + [ITCHIN] = 0xEF4, + [ITTRIGIN] = 0xEF8, +}; + +static const int cti_extended_offset[] = { + [CTIINTACK] = 0x020, + [CTIAPPSET] = 0x004, + [CTIAPPCLEAR] = 0x008, + [CTIAPPPULSE] = 0x00C, + [CTIINEN] = 0x400, + [CTIOUTEN] = 0x800, + [CTITRIGINSTATUS] = 0x040, + [CTITRIGOUTSTATUS] = 0x060, + [CTICHINSTATUS] = 0x080, + [CTICHOUTSTATUS] = 0x084, + [CTIGATE] = 0x088, + [ASICCTL] = 0x08c, + [ITCHINACK] = 0xE70, + [ITTRIGINACK] = 0xE80, + [ITCHOUT] = 0xE74, + [ITTRIGOUT] = 0xEA0, + [ITCHOUTACK] = 0xE78, + [ITTRIGOUTACK] = 0xEC0, + [ITCHIN] = 0xE7C, + [ITTRIGIN] = 0xEE0, +}; + /* * CTI devices can be associated with a PE, or be connected to CoreSight * hardware. We have a list of all CTIs irrespective of CPU bound or @@ -57,6 +106,12 @@ static struct cti_drvdata *cti_cpu_drvdata[NR_CPUS]; */ DEFINE_CORESIGHT_DEVLIST(cti_sys_devs, "cti_sys"); +u32 cti_offset(struct cti_drvdata *drvdata, int index, int num) +{ + return (drvdata->is_extended_cti ? cti_extended_offset[index] + : cti_normal_offset[index]) + (4 * num); +} + /* write set of regs to hardware - call with spinlock claimed */ void cti_write_all_hw_regs(struct cti_drvdata *drvdata) { @@ -70,15 +125,15 @@ void cti_write_all_hw_regs(struct cti_drvdata *drvdata) /* write the CTI trigger registers */ for (i = 0; i < config->nr_trig_max; i++) { - writel_relaxed(config->ctiinen[i], drvdata->base + CTIINEN(i)); + writel_relaxed(config->ctiinen[i], drvdata->base + cti_offset(drvdata, CTIINEN, i)); writel_relaxed(config->ctiouten[i], - drvdata->base + CTIOUTEN(i)); + drvdata->base + cti_offset(drvdata, CTIOUTEN, i)); } /* other regs */ - writel_relaxed(config->ctigate, drvdata->base + CTIGATE); - writel_relaxed(config->asicctl, drvdata->base + ASICCTL); - writel_relaxed(config->ctiappset, drvdata->base + CTIAPPSET); + writel_relaxed(config->ctigate, drvdata->base + cti_offset(drvdata, CTIGATE, 0)); + writel_relaxed(config->asicctl, drvdata->base + cti_offset(drvdata, ASICCTL, 0)); + writel_relaxed(config->ctiappset, drvdata->base + cti_offset(drvdata, CTIAPPSET, 0)); /* re-enable CTI */ writel_relaxed(1, drvdata->base + CTICONTROL); @@ -99,10 +154,13 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) if (config->hw_enabled || !config->hw_powered) goto cti_state_unchanged; - /* claim the device */ - rc = coresight_claim_device(drvdata->csdev); - if (rc) - goto cti_err_not_enabled; + /* There is no relationship between the CLR and SET pair for extended CTI. */ + if (!drvdata->is_extended_cti) { + /* claim the device */ + rc = coresight_claim_device(drvdata->csdev); + if (rc) + goto cti_err_not_enabled; + } cti_write_all_hw_regs(drvdata); @@ -175,7 +233,8 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) writel_relaxed(0, drvdata->base + CTICONTROL); config->hw_enabled = false; - coresight_disclaim_device_unlocked(csdev); + if (!drvdata->is_extended_cti) + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); raw_spin_unlock(&drvdata->spinlock); return ret; @@ -270,8 +329,10 @@ int cti_add_connection_entry(struct device *dev, struct cti_drvdata *drvdata, cti_dev->nr_trig_con++; /* add connection usage bit info to overall info */ - drvdata->config.trig_in_use |= tc->con_in->used_mask; - drvdata->config.trig_out_use |= tc->con_out->used_mask; + bitmap_or(drvdata->config.trig_in_use, drvdata->config.trig_in_use, + tc->con_in->used_mask, drvdata->config.nr_trig_max); + bitmap_or(drvdata->config.trig_out_use, drvdata->config.trig_out_use, + tc->con_out->used_mask, drvdata->config.nr_trig_max); return 0; } @@ -314,7 +375,6 @@ int cti_add_default_connection(struct device *dev, struct cti_drvdata *drvdata) { int ret = 0; int n_trigs = drvdata->config.nr_trig_max; - u32 n_trig_mask = GENMASK(n_trigs - 1, 0); struct cti_trig_con *tc = NULL; /* @@ -325,8 +385,9 @@ int cti_add_default_connection(struct device *dev, struct cti_drvdata *drvdata) if (!tc) return -ENOMEM; - tc->con_in->used_mask = n_trig_mask; - tc->con_out->used_mask = n_trig_mask; + bitmap_fill(tc->con_in->used_mask, n_trigs); + bitmap_fill(tc->con_out->used_mask, n_trigs); + ret = cti_add_connection_entry(dev, drvdata, tc, NULL, "default"); return ret; } @@ -339,7 +400,6 @@ int cti_channel_trig_op(struct device *dev, enum cti_chan_op op, { struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); struct cti_config *config = &drvdata->config; - u32 trig_bitmask; u32 chan_bitmask; u32 reg_value; int reg_offset; @@ -349,25 +409,23 @@ int cti_channel_trig_op(struct device *dev, enum cti_chan_op op, (trigger_idx >= config->nr_trig_max)) return -EINVAL; - trig_bitmask = BIT(trigger_idx); - /* ensure registered triggers and not out filtered */ if (direction == CTI_TRIG_IN) { - if (!(trig_bitmask & config->trig_in_use)) + if (!(test_bit(trigger_idx, config->trig_in_use))) return -EINVAL; } else { - if (!(trig_bitmask & config->trig_out_use)) + if (!(test_bit(trigger_idx, config->trig_out_use))) return -EINVAL; if ((config->trig_filter_enable) && - (config->trig_out_filter & trig_bitmask)) + test_bit(trigger_idx, config->trig_out_filter)) return -EINVAL; } /* update the local register values */ chan_bitmask = BIT(channel_idx); - reg_offset = (direction == CTI_TRIG_IN ? CTIINEN(trigger_idx) : - CTIOUTEN(trigger_idx)); + reg_offset = (direction == CTI_TRIG_IN ? cti_offset(drvdata, CTIINEN, trigger_idx) : + cti_offset(drvdata, CTIOUTEN, trigger_idx)); raw_spin_lock(&drvdata->spinlock); @@ -451,19 +509,19 @@ int cti_channel_setop(struct device *dev, enum cti_chan_set_op op, case CTI_CHAN_SET: config->ctiappset |= chan_bitmask; reg_value = config->ctiappset; - reg_offset = CTIAPPSET; + reg_offset = cti_offset(drvdata, CTIAPPSET, 0); break; case CTI_CHAN_CLR: config->ctiappset &= ~chan_bitmask; reg_value = chan_bitmask; - reg_offset = CTIAPPCLEAR; + reg_offset = cti_offset(drvdata, CTIAPPCLEAR, 0); break; case CTI_CHAN_PULSE: config->ctiappset &= ~chan_bitmask; reg_value = chan_bitmask; - reg_offset = CTIAPPPULSE; + reg_offset = cti_offset(drvdata, CTIAPPPULSE, 0); break; default: @@ -857,6 +915,19 @@ static void cti_remove(struct amba_device *adev) coresight_unregister(drvdata->csdev); } +static bool of_is_extended_cti(struct device *dev) +{ + struct fwnode_handle *fwnode = dev_fwnode(dev); + + if (is_of_node(fwnode)) { + if (of_device_is_compatible(to_of_node(fwnode), + CTI_EXTENDED)) + return true; + } + + return false; +} + static int cti_probe(struct amba_device *adev, const struct amba_id *id) { int ret = 0; @@ -950,9 +1021,11 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) drvdata->csdev_release = drvdata->csdev->dev.release; drvdata->csdev->dev.release = cti_device_release; + drvdata->is_extended_cti = of_is_extended_cti(dev); + /* all done - dec pm refcount */ pm_runtime_put(&adev->dev); - dev_info(&drvdata->csdev->dev, "CTI initialized\n"); + dev_info(&drvdata->csdev->dev, "CTI initialized %d\n", drvdata->is_extended_cti); return 0; pm_release: diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c index d0ae10bf61281..4bef860a0484d 100644 --- a/drivers/hwtracing/coresight/coresight-cti-platform.c +++ b/drivers/hwtracing/coresight/coresight-cti-platform.c @@ -136,8 +136,8 @@ static int cti_plat_create_v8_etm_connection(struct device *dev, goto create_v8_etm_out; /* build connection data */ - tc->con_in->used_mask = 0xF0; /* sigs <4,5,6,7> */ - tc->con_out->used_mask = 0xF0; /* sigs <4,5,6,7> */ + bitmap_set(tc->con_in->used_mask, 4, 4); /* sigs <4,5,6,7> */ + bitmap_set(tc->con_out->used_mask, 4, 4); /* sigs <4,5,6,7> */ /* * The EXTOUT type signals from the ETM are connected to a set of input @@ -194,10 +194,10 @@ static int cti_plat_create_v8_connections(struct device *dev, goto of_create_v8_out; /* Set the v8 PE CTI connection data */ - tc->con_in->used_mask = 0x3; /* sigs <0 1> */ + bitmap_set(tc->con_in->used_mask, 0, 2); /* sigs <0 1> */ tc->con_in->sig_types[0] = PE_DBGTRIGGER; tc->con_in->sig_types[1] = PE_PMUIRQ; - tc->con_out->used_mask = 0x7; /* sigs <0 1 2 > */ + bitmap_set(tc->con_out->used_mask, 0, 3); /* sigs <0 1 2 > */ tc->con_out->sig_types[0] = PE_EDBGREQ; tc->con_out->sig_types[1] = PE_DBGRESTART; tc->con_out->sig_types[2] = PE_CTIIRQ; @@ -213,7 +213,7 @@ static int cti_plat_create_v8_connections(struct device *dev, goto of_create_v8_out; /* filter pe_edbgreq - PE trigout sig <0> */ - drvdata->config.trig_out_filter |= 0x1; + set_bit(0, drvdata->config.trig_out_filter); of_create_v8_out: return ret; @@ -257,7 +257,7 @@ static int cti_plat_read_trig_group(struct cti_trig_grp *tgrp, if (!err) { /* set the signal usage mask */ for (idx = 0; idx < tgrp->nr_sigs; idx++) - tgrp->used_mask |= BIT(values[idx]); + set_bit(values[idx], tgrp->used_mask); } kfree(values); @@ -331,7 +331,9 @@ static int cti_plat_process_filter_sigs(struct cti_drvdata *drvdata, err = cti_plat_read_trig_group(tg, fwnode, CTI_DT_FILTER_OUT_SIGS); if (!err) - drvdata->config.trig_out_filter |= tg->used_mask; + bitmap_or(drvdata->config.trig_out_filter, + drvdata->config.trig_out_filter, + tg->used_mask, drvdata->config.nr_trig_max); kfree(tg); return err; diff --git a/drivers/hwtracing/coresight/coresight-cti-sysfs.c b/drivers/hwtracing/coresight/coresight-cti-sysfs.c index 572b80ee96fbf..cc680a4b900a7 100644 --- a/drivers/hwtracing/coresight/coresight-cti-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-cti-sysfs.c @@ -172,9 +172,8 @@ static struct attribute *coresight_cti_attrs[] = { /* register based attributes */ -/* Read registers with power check only (no enable check). */ -static ssize_t coresight_cti_reg_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t coresight_cti_mgmt_reg_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); struct cs_off_attribute *cti_attr = container_of(attr, struct cs_off_attribute, attr); @@ -189,6 +188,53 @@ static ssize_t coresight_cti_reg_show(struct device *dev, return sysfs_emit(buf, "0x%x\n", val); } +/* Read registers with power check only (no enable check). */ +static ssize_t coresight_cti_reg_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct cs_off_attribute *cti_attr = container_of(attr, struct cs_off_attribute, attr); + struct cti_config *cfg = &drvdata->config; + u32 val = 0; + int i, num; + ssize_t size = 0; + + pm_runtime_get_sync(dev->parent); + raw_spin_lock(&drvdata->spinlock); + if (drvdata->config.hw_powered) { + switch (cti_attr->off) { + case CTICHINSTATUS: + case CTICHOUTSTATUS: + case ITCHINACK: + case ITCHOUT: + case ITCHOUTACK: + case ITCHIN: + val = readl_relaxed(drvdata->base + cti_offset(drvdata, cti_attr->off, 0)); + size += sysfs_emit(buf, "0x%x\n", val); + break; + + case CTITRIGINSTATUS: + case CTITRIGOUTSTATUS: + case ITTRIGINACK: + case ITTRIGOUT: + case ITTRIGOUTACK: + case ITTRIGIN: + num = (cfg->nr_trig_max - 1) / 32; + for (i = 0; i <= num; i++) { + val = readl_relaxed(drvdata->base + + cti_offset(drvdata, cti_attr->off, i)); + size += sysfs_emit_at(buf, size, "0x%x ", val); + } + if (size > 0) + buf[size - 1] = '\n'; + break; + } + } + raw_spin_unlock(&drvdata->spinlock); + pm_runtime_put_sync(dev->parent); + return size; +} + /* Write registers with power check only (no enable check). */ static __maybe_unused ssize_t coresight_cti_reg_store(struct device *dev, struct device_attribute *attr, @@ -197,19 +243,45 @@ static __maybe_unused ssize_t coresight_cti_reg_store(struct device *dev, struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); struct cs_off_attribute *cti_attr = container_of(attr, struct cs_off_attribute, attr); unsigned long val = 0; + int num, i; if (kstrtoul(buf, 0, &val)) return -EINVAL; pm_runtime_get_sync(dev->parent); raw_spin_lock(&drvdata->spinlock); - if (drvdata->config.hw_powered) - cti_write_single_reg(drvdata, cti_attr->off, val); + if (drvdata->config.hw_powered) { + switch (cti_attr->off) { + case ITCHINACK: + case ITCHOUT: + cti_write_single_reg(drvdata, cti_offset(drvdata, cti_attr->off, 0), val); + break; + + case ITTRIGINACK: + case ITTRIGOUT: + case ITTRIGOUTACK: + num = val / 32; + i = val % 32; + for (i = 0; i <= num; i++) + cti_write_single_reg(drvdata, + cti_offset(drvdata, cti_attr->off, i), BIT(i)); + break; + } + } + raw_spin_unlock(&drvdata->spinlock); pm_runtime_put_sync(dev->parent); return size; } +#define coresight_cti_mgmt_reg(name, offset) \ + (&((struct cs_off_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_cti_mgmt_reg_show, NULL), \ + offset \ + } \ + })[0].attr.attr) + #define coresight_cti_reg(name, offset) \ (&((struct cs_off_attribute[]) { \ { \ @@ -237,17 +309,17 @@ static __maybe_unused ssize_t coresight_cti_reg_store(struct device *dev, /* coresight management registers */ static struct attribute *coresight_cti_mgmt_attrs[] = { - coresight_cti_reg(devaff0, CTIDEVAFF0), - coresight_cti_reg(devaff1, CTIDEVAFF1), - coresight_cti_reg(authstatus, CORESIGHT_AUTHSTATUS), - coresight_cti_reg(devarch, CORESIGHT_DEVARCH), - coresight_cti_reg(devid, CORESIGHT_DEVID), - coresight_cti_reg(devtype, CORESIGHT_DEVTYPE), - coresight_cti_reg(pidr0, CORESIGHT_PERIPHIDR0), - coresight_cti_reg(pidr1, CORESIGHT_PERIPHIDR1), - coresight_cti_reg(pidr2, CORESIGHT_PERIPHIDR2), - coresight_cti_reg(pidr3, CORESIGHT_PERIPHIDR3), - coresight_cti_reg(pidr4, CORESIGHT_PERIPHIDR4), + coresight_cti_mgmt_reg(devaff0, CTIDEVAFF0), + coresight_cti_mgmt_reg(devaff1, CTIDEVAFF1), + coresight_cti_mgmt_reg(authstatus, CORESIGHT_AUTHSTATUS), + coresight_cti_mgmt_reg(devarch, CORESIGHT_DEVARCH), + coresight_cti_mgmt_reg(devid, CORESIGHT_DEVID), + coresight_cti_mgmt_reg(devtype, CORESIGHT_DEVTYPE), + coresight_cti_mgmt_reg(pidr0, CORESIGHT_PERIPHIDR0), + coresight_cti_mgmt_reg(pidr1, CORESIGHT_PERIPHIDR1), + coresight_cti_mgmt_reg(pidr2, CORESIGHT_PERIPHIDR2), + coresight_cti_mgmt_reg(pidr3, CORESIGHT_PERIPHIDR3), + coresight_cti_mgmt_reg(pidr4, CORESIGHT_PERIPHIDR4), NULL, }; @@ -284,11 +356,12 @@ static ssize_t cti_reg32_show(struct device *dev, char *buf, * if reg_offset >= 0 then write through if enabled. */ static ssize_t cti_reg32_store(struct device *dev, const char *buf, - size_t size, u32 *pcached_val, int reg_offset) + size_t size, u32 *pcached_val, int index) { unsigned long val; struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); struct cti_config *config = &drvdata->config; + int reg_offset; if (kstrtoul(buf, 0, &val)) return -EINVAL; @@ -298,6 +371,7 @@ static ssize_t cti_reg32_store(struct device *dev, const char *buf, if (pcached_val) *pcached_val = (u32)val; + reg_offset = cti_offset(drvdata, index, 0); /* write through if offset and enabled */ if ((reg_offset >= 0) && cti_active(config)) cti_write_single_reg(drvdata, reg_offset, val); @@ -306,14 +380,14 @@ static ssize_t cti_reg32_store(struct device *dev, const char *buf, } /* Standard macro for simple rw cti config registers */ -#define cti_config_reg32_rw(name, cfgname, offset) \ +#define cti_config_reg32_rw(name, cfgname, index) \ static ssize_t name##_show(struct device *dev, \ struct device_attribute *attr, \ char *buf) \ { \ struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); \ return cti_reg32_show(dev, buf, \ - &drvdata->config.cfgname, offset); \ + &drvdata->config.cfgname, index); \ } \ \ static ssize_t name##_store(struct device *dev, \ @@ -322,7 +396,7 @@ static ssize_t name##_store(struct device *dev, \ { \ struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); \ return cti_reg32_store(dev, buf, size, \ - &drvdata->config.cfgname, offset); \ + &drvdata->config.cfgname, index); \ } \ static DEVICE_ATTR_RW(name) @@ -389,7 +463,7 @@ static ssize_t inen_store(struct device *dev, /* write through if enabled */ if (cti_active(config)) - cti_write_single_reg(drvdata, CTIINEN(index), val); + cti_write_single_reg(drvdata, cti_offset(drvdata, CTIINEN, index), val); raw_spin_unlock(&drvdata->spinlock); return size; } @@ -428,7 +502,7 @@ static ssize_t outen_store(struct device *dev, /* write through if enabled */ if (cti_active(config)) - cti_write_single_reg(drvdata, CTIOUTEN(index), val); + cti_write_single_reg(drvdata, cti_offset(drvdata, CTIOUTEN, index), val); raw_spin_unlock(&drvdata->spinlock); return size; } @@ -711,10 +785,8 @@ static ssize_t trigout_filtered_show(struct device *dev, struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); struct cti_config *cfg = &drvdata->config; int size = 0, nr_trig_max = cfg->nr_trig_max; - unsigned long mask = cfg->trig_out_filter; - if (mask) - size = bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); + size = bitmap_print_to_pagebuf(true, buf, cfg->trig_out_filter, nr_trig_max); return size; } static DEVICE_ATTR_RO(trigout_filtered); @@ -926,9 +998,8 @@ static ssize_t trigin_sig_show(struct device *dev, struct cti_trig_con *con = (struct cti_trig_con *)ext_attr->var; struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); struct cti_config *cfg = &drvdata->config; - unsigned long mask = con->con_in->used_mask; - return bitmap_print_to_pagebuf(true, buf, &mask, cfg->nr_trig_max); + return bitmap_print_to_pagebuf(true, buf, con->con_in->used_mask, cfg->nr_trig_max); } static ssize_t trigout_sig_show(struct device *dev, @@ -940,9 +1011,8 @@ static ssize_t trigout_sig_show(struct device *dev, struct cti_trig_con *con = (struct cti_trig_con *)ext_attr->var; struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); struct cti_config *cfg = &drvdata->config; - unsigned long mask = con->con_out->used_mask; - return bitmap_print_to_pagebuf(true, buf, &mask, cfg->nr_trig_max); + return bitmap_print_to_pagebuf(true, buf, con->con_out->used_mask, cfg->nr_trig_max); } /* convert a sig type id to a name */ diff --git a/drivers/hwtracing/coresight/coresight-cti.h b/drivers/hwtracing/coresight/coresight-cti.h index 8362a47c939c6..7d052e76d116d 100644 --- a/drivers/hwtracing/coresight/coresight-cti.h +++ b/drivers/hwtracing/coresight/coresight-cti.h @@ -18,46 +18,42 @@ struct fwnode_handle; -/* - * Device registers - * 0x000 - 0x144: CTI programming and status - * 0xEDC - 0xEF8: CTI integration test. - * 0xF00 - 0xFFC: Coresight management registers. - */ -/* CTI programming registers */ -#define CTICONTROL 0x000 -#define CTIINTACK 0x010 -#define CTIAPPSET 0x014 -#define CTIAPPCLEAR 0x018 -#define CTIAPPPULSE 0x01C -#define CTIINEN(n) (0x020 + (4 * n)) -#define CTIOUTEN(n) (0x0A0 + (4 * n)) -#define CTITRIGINSTATUS 0x130 -#define CTITRIGOUTSTATUS 0x134 -#define CTICHINSTATUS 0x138 -#define CTICHOUTSTATUS 0x13C -#define CTIGATE 0x140 -#define ASICCTL 0x144 -/* Integration test registers */ -#define ITCHINACK 0xEDC /* WO CTI CSSoc 400 only*/ -#define ITTRIGINACK 0xEE0 /* WO CTI CSSoc 400 only*/ -#define ITCHOUT 0xEE4 /* WO RW-600 */ -#define ITTRIGOUT 0xEE8 /* WO RW-600 */ -#define ITCHOUTACK 0xEEC /* RO CTI CSSoc 400 only*/ -#define ITTRIGOUTACK 0xEF0 /* RO CTI CSSoc 400 only*/ -#define ITCHIN 0xEF4 /* RO */ -#define ITTRIGIN 0xEF8 /* RO */ -/* management registers */ -#define CTIDEVAFF0 0xFA8 -#define CTIDEVAFF1 0xFAC - /* * CTI CSSoc 600 has a max of 32 trigger signals per direction. * CTI CSSoc 400 has 8 IO triggers - other CTIs can be impl def. * Max of in and out defined in the DEVID register. * - pick up actual number used from .dts parameters if present. */ -#define CTIINOUTEN_MAX 32 +#define CTIINOUTEN_MAX 128 + +#define CTICONTROL 0x000 + +/* management registers */ +#define CTIDEVAFF0 0xFA8 +#define CTIDEVAFF1 0xFAC + +enum cti_offset_index { + CTIINTACK, + CTIAPPSET, + CTIAPPCLEAR, + CTIAPPPULSE, + CTIINEN, + CTIOUTEN, + CTITRIGINSTATUS, + CTITRIGOUTSTATUS, + CTICHINSTATUS, + CTICHOUTSTATUS, + CTIGATE, + ASICCTL, + ITCHINACK, + ITTRIGINACK, + ITCHOUT, + ITTRIGOUT, + ITCHOUTACK, + ITTRIGOUTACK, + ITCHIN, + ITTRIGIN, +}; /** * Group of related trigger signals @@ -68,7 +64,7 @@ struct fwnode_handle; */ struct cti_trig_grp { int nr_sigs; - u32 used_mask; + DECLARE_BITMAP(used_mask, CTIINOUTEN_MAX); int sig_types[]; }; @@ -147,9 +143,10 @@ struct cti_config { bool hw_powered; /* registered triggers and filtering */ - u32 trig_in_use; - u32 trig_out_use; - u32 trig_out_filter; + DECLARE_BITMAP(trig_in_use, CTIINOUTEN_MAX); + DECLARE_BITMAP(trig_out_use, CTIINOUTEN_MAX); + DECLARE_BITMAP(trig_out_filter, CTIINOUTEN_MAX); + bool trig_filter_enable; u8 xtrig_rchan_sel; @@ -180,6 +177,7 @@ struct cti_drvdata { struct cti_config config; struct list_head node; void (*csdev_release)(struct device *dev); + bool is_extended_cti; }; /* @@ -232,6 +230,7 @@ int cti_create_cons_sysfs(struct device *dev, struct cti_drvdata *drvdata); struct coresight_platform_data * coresight_cti_get_platform_data(struct device *dev); const char *cti_plat_get_node_name(struct fwnode_handle *fwnode); +u32 cti_offset(struct cti_drvdata *drvdata, int index, int num); /* cti powered and enabled */ static inline bool cti_active(struct cti_config *cfg) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index f677c08233ba1..5c256af6e54af 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -520,6 +520,7 @@ static void etm_event_start(struct perf_event *event, int flags) goto out; path = etm_event_cpu_path(event_data, cpu); + path->handle = handle; /* We need a sink, no need to continue without one */ sink = coresight_get_sink(path); if (WARN_ON_ONCE(!sink)) diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 45630a1cd32fb..0c011b7041696 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -439,13 +439,26 @@ struct etm_enable_arg { int rc; }; -static void etm_enable_hw_smp_call(void *info) +static void etm_enable_sysfs_smp_call(void *info) { struct etm_enable_arg *arg = info; + struct coresight_device *csdev; if (WARN_ON(!arg)) return; + + csdev = arg->drvdata->csdev; + if (!coresight_take_mode(csdev, CS_MODE_SYSFS)) { + /* Someone is already using the tracer */ + arg->rc = -EBUSY; + return; + } + arg->rc = etm_enable_hw(arg->drvdata); + + /* The tracer didn't start */ + if (arg->rc) + coresight_set_mode(csdev, CS_MODE_DISABLED); } static int etm_cpu_id(struct coresight_device *csdev) @@ -465,16 +478,26 @@ static int etm_enable_perf(struct coresight_device *csdev, struct coresight_path *path) { struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + int ret; if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) return -EINVAL; + if (!coresight_take_mode(csdev, CS_MODE_PERF)) + return -EBUSY; + /* Configure the tracer based on the session's specifics */ etm_parse_event_config(drvdata, event); drvdata->traceid = path->trace_id; /* And enable it */ - return etm_enable_hw(drvdata); + ret = etm_enable_hw(drvdata); + + /* Failed to start tracer; roll back to DISABLED mode */ + if (ret) + coresight_set_mode(csdev, CS_MODE_DISABLED); + + return ret; } static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path) @@ -494,7 +517,7 @@ static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_pat if (cpu_online(drvdata->cpu)) { arg.drvdata = drvdata; ret = smp_call_function_single(drvdata->cpu, - etm_enable_hw_smp_call, &arg, 1); + etm_enable_sysfs_smp_call, &arg, 1); if (!ret) ret = arg.rc; if (!ret) @@ -517,12 +540,6 @@ static int etm_enable(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode, struct coresight_path *path) { int ret; - struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - - if (!coresight_take_mode(csdev, mode)) { - /* Someone is already using the tracer */ - return -EBUSY; - } switch (mode) { case CS_MODE_SYSFS: @@ -535,17 +552,12 @@ static int etm_enable(struct coresight_device *csdev, struct perf_event *event, ret = -EINVAL; } - /* The tracer didn't start */ - if (ret) - coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); - return ret; } -static void etm_disable_hw(void *info) +static void etm_disable_hw(struct etm_drvdata *drvdata) { int i; - struct etm_drvdata *drvdata = info; struct etm_config *config = &drvdata->config; struct coresight_device *csdev = drvdata->csdev; @@ -567,6 +579,15 @@ static void etm_disable_hw(void *info) "cpu: %d disable smp call done\n", drvdata->cpu); } +static void etm_disable_sysfs_smp_call(void *info) +{ + struct etm_drvdata *drvdata = info; + + etm_disable_hw(drvdata); + + coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); +} + static void etm_disable_perf(struct coresight_device *csdev) { struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); @@ -588,6 +609,8 @@ static void etm_disable_perf(struct coresight_device *csdev) CS_LOCK(drvdata->csa.base); + coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); + /* * perf will release trace ids when _free_aux() * is called at the end of the session @@ -612,7 +635,8 @@ static void etm_disable_sysfs(struct coresight_device *csdev) * Executing etm_disable_hw on the cpu whose ETM is being disabled * ensures that register writes occur when cpu is powered. */ - smp_call_function_single(drvdata->cpu, etm_disable_hw, drvdata, 1); + smp_call_function_single(drvdata->cpu, etm_disable_sysfs_smp_call, + drvdata, 1); spin_unlock(&drvdata->spinlock); cpus_read_unlock(); @@ -652,9 +676,6 @@ static void etm_disable(struct coresight_device *csdev, WARN_ON_ONCE(mode); return; } - - if (mode) - coresight_set_mode(csdev, CS_MODE_DISABLED); } static const struct coresight_ops_source etm_source_ops = { @@ -774,16 +795,16 @@ static int __init etm_hp_setup(void) { int ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight:starting", - etm_starting_cpu, etm_dying_cpu); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, + "arm/coresight:starting", + etm_starting_cpu, etm_dying_cpu); if (ret) return ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, - "arm/coresight:online", - etm_online_cpu, NULL); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm/coresight:online", + etm_online_cpu, NULL); /* HP dyn state ID returned in ret on success */ if (ret > 0) { diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 020f070bf17dc..fdda924a2c711 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -446,10 +446,24 @@ static int etm4_enable_trace_unit(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR); etm4x_allow_trace(drvdata); + + /* + * According to software usage PKLXF in Arm ARM (ARM DDI 0487 L.a), + * execute a Context synchronization event to guarantee the trace unit + * will observe the new values of the System registers. + */ + if (!csa->io_mem) + isb(); + /* Enable the trace unit */ etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); - /* Synchronize the register updates for sysreg access */ + /* + * As recommended by section 4.3.7 ("Synchronization when using system + * instructions to progrom the trace unit") of ARM IHI 0064H.b, the + * self-hosted trace analyzer must perform a Context synchronization + * event between writing to the TRCPRGCTLR and reading the TRCSTATR. + */ if (!csa->io_mem) isb(); @@ -589,13 +603,26 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) return rc; } -static void etm4_enable_hw_smp_call(void *info) +static void etm4_enable_sysfs_smp_call(void *info) { struct etm4_enable_arg *arg = info; + struct coresight_device *csdev; if (WARN_ON(!arg)) return; + + csdev = arg->drvdata->csdev; + if (!coresight_take_mode(csdev, CS_MODE_SYSFS)) { + /* Someone is already using the tracer */ + arg->rc = -EBUSY; + return; + } + arg->rc = etm4_enable_hw(arg->drvdata); + + /* The tracer didn't start */ + if (arg->rc) + coresight_set_mode(csdev, CS_MODE_DISABLED); } /* @@ -808,13 +835,14 @@ static int etm4_enable_perf(struct coresight_device *csdev, struct perf_event *event, struct coresight_path *path) { - int ret = 0; struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + int ret; - if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) { - ret = -EINVAL; - goto out; - } + if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) + return -EINVAL; + + if (!coresight_take_mode(csdev, CS_MODE_PERF)) + return -EBUSY; /* Configure the tracer based on the session's specifics */ ret = etm4_parse_event_config(csdev, event); @@ -830,6 +858,9 @@ static int etm4_enable_perf(struct coresight_device *csdev, ret = etm4_enable_hw(drvdata); out: + /* Failed to start tracer; roll back to DISABLED mode */ + if (ret) + coresight_set_mode(csdev, CS_MODE_DISABLED); return ret; } @@ -861,7 +892,7 @@ static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_pa */ arg.drvdata = drvdata; ret = smp_call_function_single(drvdata->cpu, - etm4_enable_hw_smp_call, &arg, 1); + etm4_enable_sysfs_smp_call, &arg, 1); if (!ret) ret = arg.rc; if (!ret) @@ -882,11 +913,6 @@ static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, { int ret; - if (!coresight_take_mode(csdev, mode)) { - /* Someone is already using the tracer */ - return -EBUSY; - } - switch (mode) { case CS_MODE_SYSFS: ret = etm4_enable_sysfs(csdev, path); @@ -898,10 +924,6 @@ static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, ret = -EINVAL; } - /* The tracer didn't start */ - if (ret) - coresight_set_mode(csdev, CS_MODE_DISABLED); - return ret; } @@ -923,11 +945,16 @@ static void etm4_disable_trace_unit(struct etmv4_drvdata *drvdata) */ etm4x_prohibit_trace(drvdata); /* - * Make sure everything completes before disabling, as recommended - * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register, - * SSTATUS") of ARM IHI 0064D + * Prevent being speculative at the point of disabling the trace unit, + * as recommended by section 7.3.77 ("TRCVICTLR, ViewInst Main Control + * Register, SSTATUS") of ARM IHI 0064D */ dsb(sy); + /* + * According to software usage VKHHY in Arm ARM (ARM DDI 0487 L.a), + * execute a Context synchronization event to guarantee no new + * program-flow trace is generated. + */ isb(); /* Trace synchronization barrier, is a nop if not supported */ tsb_csync(); @@ -953,10 +980,9 @@ static void etm4_disable_trace_unit(struct etmv4_drvdata *drvdata) isb(); } -static void etm4_disable_hw(void *info) +static void etm4_disable_hw(struct etmv4_drvdata *drvdata) { u32 control; - struct etmv4_drvdata *drvdata = info; struct etmv4_config *config = &drvdata->config; struct coresight_device *csdev = drvdata->csdev; struct csdev_access *csa = &csdev->access; @@ -993,6 +1019,15 @@ static void etm4_disable_hw(void *info) "cpu: %d disable smp call done\n", drvdata->cpu); } +static void etm4_disable_sysfs_smp_call(void *info) +{ + struct etmv4_drvdata *drvdata = info; + + etm4_disable_hw(drvdata); + + coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); +} + static int etm4_disable_perf(struct coresight_device *csdev, struct perf_event *event) { @@ -1022,6 +1057,8 @@ static int etm4_disable_perf(struct coresight_device *csdev, /* TRCVICTLR::SSSTATUS, bit[9] */ filters->ssstatus = (control & BIT(9)); + coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); + /* * perf will release trace ids when _free_aux() is * called at the end of the session. @@ -1047,7 +1084,8 @@ static void etm4_disable_sysfs(struct coresight_device *csdev) * Executing etm4_disable_hw on the cpu whose ETM is being disabled * ensures that register writes occur when cpu is powered. */ - smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1); + smp_call_function_single(drvdata->cpu, etm4_disable_sysfs_smp_call, + drvdata, 1); raw_spin_unlock(&drvdata->spinlock); @@ -1087,9 +1125,6 @@ static void etm4_disable(struct coresight_device *csdev, etm4_disable_perf(csdev, event); break; } - - if (mode) - coresight_set_mode(csdev, CS_MODE_DISABLED); } static int etm4_resume_perf(struct coresight_device *csdev) @@ -1823,9 +1858,11 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) goto out; } + if (!drvdata->paused) + etm4_disable_trace_unit(drvdata); + state = drvdata->save_state; - state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); if (drvdata->nr_pe) state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); @@ -1908,7 +1945,7 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcpdcr = etm4x_read32(csa, TRCPDCR); /* wait for TRCSTATR.IDLE to go up */ - if (etm4x_wait_status(csa, TRCSTATR_PMSTABLE_BIT, 1)) { + if (etm4x_wait_status(csa, TRCSTATR_IDLE_BIT, 1)) { dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); etm4_os_unlock(drvdata); @@ -1935,9 +1972,6 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) { int ret = 0; - /* Save the TRFCR irrespective of whether the ETM is ON */ - if (drvdata->trfcr) - drvdata->save_trfcr = read_trfcr(); /* * Save and restore the ETM Trace registers only if * the ETM is active. @@ -1959,7 +1993,6 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4_cs_unlock(drvdata, csa); etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); - etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); if (drvdata->nr_pe) etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); @@ -2044,13 +2077,15 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) /* Unlock the OS lock to re-enable trace and external debug access */ etm4_os_unlock(drvdata); + + if (!drvdata->paused) + etm4_enable_trace_unit(drvdata); + etm4_cs_lock(drvdata, csa); } static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) { - if (drvdata->trfcr) - write_trfcr(drvdata->save_trfcr); if (drvdata->state_needs_restore) __etm4_cpu_restore(drvdata); } diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 13ec9ecef46f5..b8796b4271025 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -866,7 +866,6 @@ struct etmv4_config { * struct etm4_save_state - state to be preserved when ETM is without power */ struct etmv4_save_state { - u32 trcprgctlr; u32 trcprocselr; u32 trcconfigr; u32 trcauxctlr; @@ -980,7 +979,6 @@ struct etmv4_save_state { * at runtime, due to the additional setting of TRFCR_CX when * in EL2. Otherwise, 0. * @config: structure holding configuration parameters. - * @save_trfcr: Saved TRFCR_EL1 register during a CPU PM event. * @save_state: State to be preserved across power loss * @state_needs_restore: True when there is context to restore after PM exit * @skip_power_up: Indicates if an implementation can skip powering up @@ -1037,7 +1035,6 @@ struct etmv4_drvdata { bool lpoverride; u64 trfcr; struct etmv4_config config; - u64 save_trfcr; struct etmv4_save_state *save_state; bool state_needs_restore; bool skip_power_up; diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c index 3b248e54471a3..fd8dcd541454b 100644 --- a/drivers/hwtracing/coresight/coresight-funnel.c +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,9 @@ #define FUNNEL_ENSx_MASK 0xff DEFINE_CORESIGHT_DEVLIST(funnel_devs, "funnel"); +static LIST_HEAD(funnel_delay_probe); +static enum cpuhp_state hp_online; +static DEFINE_SPINLOCK(delay_lock); /** * struct funnel_drvdata - specifics associated to a funnel component @@ -40,6 +44,9 @@ DEFINE_CORESIGHT_DEVLIST(funnel_devs, "funnel"); * @csdev: component vitals needed by the framework. * @priority: port selection order. * @spinlock: serialize enable/disable operations. + * @cpumask: CPU mask representing the CPUs related to this funnel. + * @dev: pointer to the device associated with this funnel. + * @link: list node for adding this funnel to the delayed probe list. */ struct funnel_drvdata { void __iomem *base; @@ -48,6 +55,15 @@ struct funnel_drvdata { struct coresight_device *csdev; unsigned long priority; raw_spinlock_t spinlock; + struct cpumask *cpumask; + struct device *dev; + struct list_head link; +}; + +struct funnel_smp_arg { + struct funnel_drvdata *drvdata; + int port; + int rc; }; static int dynamic_funnel_enable_hw(struct funnel_drvdata *drvdata, int port) @@ -76,9 +92,37 @@ static int dynamic_funnel_enable_hw(struct funnel_drvdata *drvdata, int port) return rc; } +static void funnel_enable_hw_smp_call(void *info) +{ + struct funnel_smp_arg *arg = info; + + arg->rc = dynamic_funnel_enable_hw(arg->drvdata, arg->port); +} + +static int funnel_enable_hw(struct funnel_drvdata *drvdata, int port) +{ + int cpu, ret; + struct funnel_smp_arg arg = { 0 }; + + if (!drvdata->cpumask) + return dynamic_funnel_enable_hw(drvdata, port); + + arg.drvdata = drvdata; + arg.port = port; + + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + funnel_enable_hw_smp_call, &arg, 1); + if (!ret) + return arg.rc; + } + return ret; +} + static int funnel_enable(struct coresight_device *csdev, struct coresight_connection *in, - struct coresight_connection *out) + struct coresight_connection *out, + enum cs_mode mode) { int rc = 0; struct funnel_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); @@ -86,19 +130,41 @@ static int funnel_enable(struct coresight_device *csdev, bool first_enable = false; raw_spin_lock_irqsave(&drvdata->spinlock, flags); - if (in->dest_refcnt == 0) { - if (drvdata->base) - rc = dynamic_funnel_enable_hw(drvdata, in->dest_port); - if (!rc) - first_enable = true; - } - if (!rc) + + if (in->dest_refcnt == 0) + first_enable = true; + else in->dest_refcnt++; + + if (mode == CS_MODE_PERF) { + if (first_enable) { + if (drvdata->cpumask && + !cpumask_test_cpu(smp_processor_id(), drvdata->cpumask)) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return -EINVAL; + } + + if (drvdata->base) + rc = dynamic_funnel_enable_hw(drvdata, in->dest_port); + if (!rc) + in->dest_refcnt++; + } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return rc; + } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); - if (first_enable) - dev_dbg(&csdev->dev, "FUNNEL inport %d enabled\n", - in->dest_port); + if (first_enable) { + if (drvdata->base) + rc = funnel_enable_hw(drvdata, in->dest_port); + if (!rc) { + in->dest_refcnt++; + dev_dbg(&csdev->dev, "FUNNEL inport %d enabled\n", + in->dest_port); + } + } + return rc; } @@ -135,6 +201,7 @@ static void funnel_disable(struct coresight_device *csdev, dynamic_funnel_disable_hw(drvdata, in->dest_port); last_disable = true; } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); if (last_disable) @@ -188,15 +255,39 @@ static u32 get_funnel_ctrl_hw(struct funnel_drvdata *drvdata) return functl; } +static void get_funnel_ctrl_smp_call(void *info) +{ + struct funnel_smp_arg *arg = info; + + arg->rc = get_funnel_ctrl_hw(arg->drvdata); +} + static ssize_t funnel_ctrl_show(struct device *dev, struct device_attribute *attr, char *buf) { u32 val; + int cpu, ret; struct funnel_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct funnel_smp_arg arg = { 0 }; pm_runtime_get_sync(dev->parent); - - val = get_funnel_ctrl_hw(drvdata); + if (!drvdata->cpumask) { + val = get_funnel_ctrl_hw(drvdata); + } else { + arg.drvdata = drvdata; + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + get_funnel_ctrl_smp_call, &arg, 1); + if (!ret) + break; + } + if (!ret) { + val = arg.rc; + } else { + pm_runtime_put(dev->parent); + return ret; + } + } pm_runtime_put(dev->parent); @@ -211,22 +302,68 @@ static struct attribute *coresight_funnel_attrs[] = { }; ATTRIBUTE_GROUPS(coresight_funnel); +static void funnel_clear_self_claim_tag(struct funnel_drvdata *drvdata) +{ + struct csdev_access access = CSDEV_ACCESS_IOMEM(drvdata->base); + + coresight_clear_self_claim_tag(&access); +} + +static void funnel_init_on_cpu(void *info) +{ + struct funnel_drvdata *drvdata = info; + + funnel_clear_self_claim_tag(drvdata); +} + +static int funnel_add_coresight_dev(struct device *dev) +{ + struct coresight_desc desc = { 0 }; + struct funnel_drvdata *drvdata = dev_get_drvdata(dev); + + if (drvdata->base) { + desc.groups = coresight_funnel_groups; + desc.access = CSDEV_ACCESS_IOMEM(drvdata->base); + } + + desc.name = coresight_alloc_device_name(&funnel_devs, dev); + if (!desc.name) + return -ENOMEM; + + desc.type = CORESIGHT_DEV_TYPE_LINK; + desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_MERG; + desc.ops = &funnel_cs_ops; + desc.pdata = dev->platform_data; + desc.dev = dev; + + drvdata->csdev = coresight_register(&desc); + if (IS_ERR(drvdata->csdev)) + return PTR_ERR(drvdata->csdev); + return 0; +} + +static struct cpumask *funnel_get_cpumask(struct device *dev) +{ + struct generic_pm_domain *pd; + + pd = pd_to_genpd(dev->pm_domain); + if (pd) + return pd->cpus; + + return NULL; +} + static int funnel_probe(struct device *dev, struct resource *res) { void __iomem *base; struct coresight_platform_data *pdata = NULL; struct funnel_drvdata *drvdata; - struct coresight_desc desc = { 0 }; - int ret; + int cpu, ret; if (is_of_node(dev_fwnode(dev)) && of_device_is_compatible(dev->of_node, "arm,coresight-funnel")) dev_warn_once(dev, "Uses OBSOLETE CoreSight funnel binding\n"); - desc.name = coresight_alloc_device_name(&funnel_devs, dev); - if (!desc.name) - return -ENOMEM; - drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) return -ENOMEM; @@ -244,9 +381,6 @@ static int funnel_probe(struct device *dev, struct resource *res) if (IS_ERR(base)) return PTR_ERR(base); drvdata->base = base; - desc.groups = coresight_funnel_groups; - desc.access = CSDEV_ACCESS_IOMEM(base); - coresight_clear_self_claim_tag(&desc.access); } dev_set_drvdata(dev, drvdata); @@ -258,24 +392,46 @@ static int funnel_probe(struct device *dev, struct resource *res) dev->platform_data = pdata; raw_spin_lock_init(&drvdata->spinlock); - desc.type = CORESIGHT_DEV_TYPE_LINK; - desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_MERG; - desc.ops = &funnel_cs_ops; - desc.pdata = pdata; - desc.dev = dev; - drvdata->csdev = coresight_register(&desc); - if (IS_ERR(drvdata->csdev)) - return PTR_ERR(drvdata->csdev); - return 0; + if (is_of_node(dev_fwnode(dev)) && + of_device_is_compatible(dev->of_node, "arm,coresight-cpu-funnel")) { + drvdata->cpumask = funnel_get_cpumask(dev); + if (!drvdata->cpumask) + return -EINVAL; + drvdata->dev = dev; + cpus_read_lock(); + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + funnel_init_on_cpu, drvdata, 1); + if (!ret) + break; + } + + if (ret) { + scoped_guard(spinlock, &delay_lock) + list_add(&drvdata->link, &funnel_delay_probe); + cpus_read_unlock(); + return 0; + } + + cpus_read_unlock(); + } else if (res) { + funnel_clear_self_claim_tag(drvdata); + } + + return funnel_add_coresight_dev(dev); } static int funnel_remove(struct device *dev) { struct funnel_drvdata *drvdata = dev_get_drvdata(dev); - coresight_unregister(drvdata->csdev); - + if (drvdata->csdev) { + coresight_unregister(drvdata->csdev); + } else { + scoped_guard(spinlock, &delay_lock) + list_del(&drvdata->link); + } return 0; } @@ -341,6 +497,7 @@ static void funnel_platform_remove(struct platform_device *pdev) static const struct of_device_id funnel_match[] = { {.compatible = "arm,coresight-static-funnel"}, + {.compatible = "arm,coresight-cpu-funnel"}, {} }; @@ -412,8 +569,41 @@ static struct amba_driver dynamic_funnel_driver = { .id_table = dynamic_funnel_ids, }; +static int funnel_online_cpu(unsigned int cpu) +{ + struct funnel_drvdata *drvdata, *tmp; + int ret; + + list_for_each_entry_safe(drvdata, tmp, &funnel_delay_probe, link) { + if (cpumask_test_cpu(cpu, drvdata->cpumask)) { + scoped_guard(spinlock, &delay_lock) + list_del(&drvdata->link); + + ret = pm_runtime_resume_and_get(drvdata->dev); + if (ret < 0) + return 0; + + funnel_clear_self_claim_tag(drvdata); + funnel_add_coresight_dev(drvdata->dev); + pm_runtime_put(drvdata->dev); + } + } + return 0; +} + static int __init funnel_init(void) { + int ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm/coresight-funnel:online", + funnel_online_cpu, NULL); + + if (ret > 0) + hp_online = ret; + else + return ret; + return coresight_init_driver("funnel", &dynamic_funnel_driver, &funnel_driver, THIS_MODULE); } @@ -421,6 +611,10 @@ static int __init funnel_init(void) static void __exit funnel_exit(void) { coresight_remove_driver(&dynamic_funnel_driver, &funnel_driver); + if (hp_online) { + cpuhp_remove_state_nocalls(hp_online); + hp_online = 0; + } } module_init(funnel_init); diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c index 0db64c5f49959..2c2571652a788 100644 --- a/drivers/hwtracing/coresight/coresight-platform.c +++ b/drivers/hwtracing/coresight/coresight-platform.c @@ -220,6 +220,8 @@ static int of_coresight_parse_endpoint(struct device *dev, rparent = of_coresight_get_port_parent(rep); if (!rparent) break; + if (!of_device_is_available(rparent)) + break; if (of_graph_parse_endpoint(rep, &rendpoint)) break; diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index 33e22b1ba0432..f80122827934d 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -156,6 +156,10 @@ void coresight_remove_links(struct coresight_device *orig, u32 coresight_get_sink_id(struct coresight_device *csdev); void coresight_path_assign_trace_id(struct coresight_path *path, enum cs_mode mode); +int coresight_get_in_port_dest(struct coresight_device *src, + struct coresight_device *dest); +struct coresight_device *coresight_get_helper(struct coresight_device *csdev, + int type); #if IS_ENABLED(CONFIG_CORESIGHT_SOURCE_ETM3X) int etm_readl_cp14(u32 off, unsigned int *val); diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index e6472658235dc..6cb57763f9b10 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,9 @@ #define REPLICATOR_IDFILTER1 0x004 DEFINE_CORESIGHT_DEVLIST(replicator_devs, "replicator"); +static LIST_HEAD(replicator_delay_probe); +static enum cpuhp_state hp_online; +static DEFINE_SPINLOCK(delay_lock); /** * struct replicator_drvdata - specifics associated to a replicator component @@ -35,6 +39,9 @@ DEFINE_CORESIGHT_DEVLIST(replicator_devs, "replicator"); * @csdev: component vitals needed by the framework * @spinlock: serialize enable/disable operations. * @check_idfilter_val: check if the context is lost upon clock removal. + * @cpumask: CPU mask representing the CPUs related to this replicator. + * @dev: pointer to the device associated with this replicator. + * @link: link to the delay_probed list. */ struct replicator_drvdata { void __iomem *base; @@ -43,18 +50,64 @@ struct replicator_drvdata { struct coresight_device *csdev; raw_spinlock_t spinlock; bool check_idfilter_val; + struct cpumask *cpumask; + struct device *dev; + struct list_head link; }; -static void dynamic_replicator_reset(struct replicator_drvdata *drvdata) +struct replicator_smp_arg { + struct replicator_drvdata *drvdata; + int outport; + u32 offset; + int rc; +}; + +static void replicator_clear_self_claim_tag(struct replicator_drvdata *drvdata) +{ + struct csdev_access access = CSDEV_ACCESS_IOMEM(drvdata->base); + + coresight_clear_self_claim_tag(&access); +} + +static int replicator_claim_device_unlocked(struct replicator_drvdata *drvdata) +{ + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access access = CSDEV_ACCESS_IOMEM(drvdata->base); + u32 claim_tag; + + if (csdev) + return coresight_claim_device_unlocked(csdev); + + writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, drvdata->base + CORESIGHT_CLAIMSET); + + claim_tag = readl_relaxed(drvdata->base + CORESIGHT_CLAIMCLR); + if (claim_tag != CORESIGHT_CLAIM_SELF_HOSTED) { + coresight_clear_self_claim_tag_unlocked(&access); + return -EBUSY; + } + + return 0; +} + +static void replicator_disclaim_device_unlocked(struct replicator_drvdata *drvdata) { struct coresight_device *csdev = drvdata->csdev; + struct csdev_access access = CSDEV_ACCESS_IOMEM(drvdata->base); + if (csdev) + return coresight_disclaim_device_unlocked(csdev); + + coresight_clear_self_claim_tag_unlocked(&access); +} + +static void dynamic_replicator_reset(struct replicator_drvdata *drvdata) +{ CS_UNLOCK(drvdata->base); - if (!coresight_claim_device_unlocked(csdev)) { + if (!replicator_claim_device_unlocked(drvdata)) { writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER0); writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER1); - coresight_disclaim_device_unlocked(csdev); + replicator_disclaim_device_unlocked(drvdata); } CS_LOCK(drvdata->base); @@ -116,9 +169,38 @@ static int dynamic_replicator_enable(struct replicator_drvdata *drvdata, return rc; } +static void replicator_enable_hw_smp_call(void *info) +{ + struct replicator_smp_arg *arg = info; + + arg->rc = dynamic_replicator_enable(arg->drvdata, 0, arg->outport); +} + +static int replicator_enable_hw(struct replicator_drvdata *drvdata, + int inport, int outport) +{ + int cpu, ret; + struct replicator_smp_arg arg = { 0 }; + + if (!drvdata->cpumask) + return dynamic_replicator_enable(drvdata, 0, outport); + + arg.drvdata = drvdata; + arg.outport = outport; + + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, replicator_enable_hw_smp_call, &arg, 1); + if (!ret) + return arg.rc; + } + + return ret; +} + static int replicator_enable(struct coresight_device *csdev, struct coresight_connection *in, - struct coresight_connection *out) + struct coresight_connection *out, + enum cs_mode mode) { int rc = 0; struct replicator_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); @@ -126,19 +208,43 @@ static int replicator_enable(struct coresight_device *csdev, bool first_enable = false; raw_spin_lock_irqsave(&drvdata->spinlock, flags); - if (out->src_refcnt == 0) { - if (drvdata->base) - rc = dynamic_replicator_enable(drvdata, in->dest_port, - out->src_port); - if (!rc) - first_enable = true; - } - if (!rc) + + if (out->src_refcnt == 0) + first_enable = true; + else out->src_refcnt++; + + if (mode == CS_MODE_PERF) { + if (first_enable) { + if (drvdata->cpumask && + !cpumask_test_cpu(smp_processor_id(), drvdata->cpumask)) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return -EINVAL; + } + + if (drvdata->base) + rc = dynamic_replicator_enable(drvdata, in->dest_port, + out->src_port); + if (!rc) + out->src_refcnt++; + } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return rc; + } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); - if (first_enable) - dev_dbg(&csdev->dev, "REPLICATOR enabled\n"); + if (first_enable) { + if (drvdata->base) + rc = replicator_enable_hw(drvdata, in->dest_port, + out->src_port); + if (!rc) { + out->src_refcnt++; + dev_dbg(&csdev->dev, "REPLICATOR enabled\n"); + return rc; + } + } + return rc; } @@ -186,6 +292,7 @@ static void replicator_disable(struct coresight_device *csdev, out->src_port); last_disable = true; } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); if (last_disable) @@ -201,9 +308,65 @@ static const struct coresight_ops replicator_cs_ops = { .link_ops = &replicator_link_ops, }; +static void replicator_read_register_smp_call(void *info) +{ + struct replicator_smp_arg *arg = info; + + arg->rc = readl_relaxed(arg->drvdata->base + arg->offset); +} + +static ssize_t coresight_replicator_reg32_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct replicator_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct cs_off_attribute *cs_attr = container_of(attr, struct cs_off_attribute, attr); + unsigned long flags; + struct replicator_smp_arg arg = { 0 }; + u32 val; + int ret, cpu; + + pm_runtime_get_sync(dev->parent); + + if (!drvdata->cpumask) { + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + val = readl_relaxed(drvdata->base + cs_attr->off); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + } else { + arg.drvdata = drvdata; + arg.offset = cs_attr->off; + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + replicator_read_register_smp_call, + &arg, 1); + if (!ret) + break; + } + if (!ret) { + val = arg.rc; + } else { + pm_runtime_put_sync(dev->parent); + return ret; + } + } + + pm_runtime_put_sync(dev->parent); + + return sysfs_emit(buf, "0x%x\n", val); +} + +#define coresight_replicator_reg32(name, offset) \ + (&((struct cs_off_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_replicator_reg32_show, NULL), \ + offset \ + } \ + })[0].attr.attr) + static struct attribute *replicator_mgmt_attrs[] = { - coresight_simple_reg32(idfilter0, REPLICATOR_IDFILTER0), - coresight_simple_reg32(idfilter1, REPLICATOR_IDFILTER1), + coresight_replicator_reg32(idfilter0, REPLICATOR_IDFILTER0), + coresight_replicator_reg32(idfilter1, REPLICATOR_IDFILTER1), NULL, }; @@ -217,23 +380,69 @@ static const struct attribute_group *replicator_groups[] = { NULL, }; +static int replicator_add_coresight_dev(struct device *dev) +{ + struct coresight_desc desc = { 0 }; + struct replicator_drvdata *drvdata = dev_get_drvdata(dev); + + if (drvdata->base) { + desc.groups = replicator_groups; + desc.access = CSDEV_ACCESS_IOMEM(drvdata->base); + } + + desc.name = coresight_alloc_device_name(&replicator_devs, dev); + if (!desc.name) + return -ENOMEM; + + desc.type = CORESIGHT_DEV_TYPE_LINK; + desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_SPLIT; + desc.ops = &replicator_cs_ops; + desc.pdata = dev->platform_data; + desc.dev = dev; + + drvdata->csdev = coresight_register(&desc); + if (IS_ERR(drvdata->csdev)) + return PTR_ERR(drvdata->csdev); + + return 0; +} + +static void replicator_init_hw(struct replicator_drvdata *drvdata) +{ + replicator_clear_self_claim_tag(drvdata); + replicator_reset(drvdata); +} + +static void replicator_init_on_cpu(void *info) +{ + struct replicator_drvdata *drvdata = info; + + replicator_init_hw(drvdata); +} + +static struct cpumask *replicator_get_cpumask(struct device *dev) +{ + struct generic_pm_domain *pd; + + pd = pd_to_genpd(dev->pm_domain); + if (pd) + return pd->cpus; + + return NULL; +} + static int replicator_probe(struct device *dev, struct resource *res) { struct coresight_platform_data *pdata = NULL; struct replicator_drvdata *drvdata; - struct coresight_desc desc = { 0 }; void __iomem *base; - int ret; + int cpu, ret; if (is_of_node(dev_fwnode(dev)) && of_device_is_compatible(dev->of_node, "arm,coresight-replicator")) dev_warn_once(dev, "Uses OBSOLETE CoreSight replicator binding\n"); - desc.name = coresight_alloc_device_name(&replicator_devs, dev); - if (!desc.name) - return -ENOMEM; - drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) return -ENOMEM; @@ -251,9 +460,6 @@ static int replicator_probe(struct device *dev, struct resource *res) if (IS_ERR(base)) return PTR_ERR(base); drvdata->base = base; - desc.groups = replicator_groups; - desc.access = CSDEV_ACCESS_IOMEM(base); - coresight_clear_self_claim_tag(&desc.access); } if (fwnode_property_present(dev_fwnode(dev), @@ -268,25 +474,49 @@ static int replicator_probe(struct device *dev, struct resource *res) dev->platform_data = pdata; raw_spin_lock_init(&drvdata->spinlock); - desc.type = CORESIGHT_DEV_TYPE_LINK; - desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_SPLIT; - desc.ops = &replicator_cs_ops; - desc.pdata = dev->platform_data; - desc.dev = dev; - drvdata->csdev = coresight_register(&desc); - if (IS_ERR(drvdata->csdev)) - return PTR_ERR(drvdata->csdev); + if (is_of_node(dev_fwnode(dev)) && + of_device_is_compatible(dev->of_node, "arm,coresight-cpu-replicator")) { + drvdata->cpumask = replicator_get_cpumask(dev); + if (!drvdata->cpumask) + return -EINVAL; + drvdata->dev = dev; + cpus_read_lock(); + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + replicator_init_on_cpu, drvdata, 1); + if (!ret) + break; + } - replicator_reset(drvdata); - return 0; + if (ret) { + scoped_guard(spinlock, &delay_lock) + list_add(&drvdata->link, &replicator_delay_probe); + cpus_read_unlock(); + return 0; + } + + cpus_read_unlock(); + } else if (res) { + replicator_init_hw(drvdata); + } + + ret = replicator_add_coresight_dev(dev); + + return ret; } static int replicator_remove(struct device *dev) { struct replicator_drvdata *drvdata = dev_get_drvdata(dev); - coresight_unregister(drvdata->csdev); + if (drvdata->csdev) { + coresight_unregister(drvdata->csdev); + } else { + scoped_guard(spinlock, &delay_lock) + list_del(&drvdata->link); + } + return 0; } @@ -354,6 +584,7 @@ static const struct dev_pm_ops replicator_dev_pm_ops = { static const struct of_device_id replicator_match[] = { {.compatible = "arm,coresight-replicator"}, {.compatible = "arm,coresight-static-replicator"}, + {.compatible = "arm,coresight-cpu-replicator"}, {} }; @@ -418,8 +649,44 @@ static struct amba_driver dynamic_replicator_driver = { .id_table = dynamic_replicator_ids, }; +static int replicator_online_cpu(unsigned int cpu) +{ + struct replicator_drvdata *drvdata, *tmp; + int ret; + + spin_lock(&delay_lock); + list_for_each_entry_safe(drvdata, tmp, &replicator_delay_probe, link) { + if (cpumask_test_cpu(cpu, drvdata->cpumask)) { + list_del(&drvdata->link); + spin_unlock(&delay_lock); + ret = pm_runtime_resume_and_get(drvdata->dev); + if (ret < 0) + return 0; + + replicator_clear_self_claim_tag(drvdata); + replicator_reset(drvdata); + replicator_add_coresight_dev(drvdata->dev); + pm_runtime_put(drvdata->dev); + spin_lock(&delay_lock); + } + } + spin_unlock(&delay_lock); + return 0; +} + static int __init replicator_init(void) { + int ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm/coresight-replicator:online", + replicator_online_cpu, NULL); + + if (ret > 0) + hp_online = ret; + else + return ret; + return coresight_init_driver("replicator", &dynamic_replicator_driver, &replicator_driver, THIS_MODULE); } @@ -427,6 +694,10 @@ static int __init replicator_init(void) static void __exit replicator_exit(void) { coresight_remove_driver(&dynamic_replicator_driver, &replicator_driver); + if (hp_online) { + cpuhp_remove_state_nocalls(hp_online); + hp_online = 0; + } } module_init(replicator_init); diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index 36599c431be62..984be10c10fd2 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -31,10 +32,14 @@ #include "coresight-priv.h" #include "coresight-tmc.h" +#include "coresight-ctcu.h" DEFINE_CORESIGHT_DEVLIST(etb_devs, "tmc_etb"); DEFINE_CORESIGHT_DEVLIST(etf_devs, "tmc_etf"); DEFINE_CORESIGHT_DEVLIST(etr_devs, "tmc_etr"); +static LIST_HEAD(tmc_delay_probe); +static enum cpuhp_state hp_online; +static DEFINE_SPINLOCK(delay_lock); int tmc_wait_for_tmcready(struct tmc_drvdata *drvdata) { @@ -228,22 +233,47 @@ static int tmc_prepare_crashdata(struct tmc_drvdata *drvdata) return 0; } +/* Return the byte-cntr's sysfs_read_ops if in use */ +static const struct sysfs_read_ops *tmc_get_byte_cntr_sysfs_ops(struct tmc_drvdata *drvdata) +{ + struct ctcu_byte_cntr *byte_cntr_data; + struct ctcu_drvdata *ctcu_drvdata; + struct coresight_device *helper; + int port; + + helper = coresight_get_helper(drvdata->csdev, CORESIGHT_DEV_SUBTYPE_HELPER_CTCU); + if (!helper) + return NULL; + + port = coresight_get_in_port_dest(drvdata->csdev, helper); + if (port < 0) + return NULL; + + ctcu_drvdata = dev_get_drvdata(helper->dev.parent); + byte_cntr_data = &ctcu_drvdata->byte_cntr_data[port]; + if (byte_cntr_data && byte_cntr_data->thresh_val) + return ctcu_drvdata->byte_cntr_sysfs_read_ops; + + return NULL; +} + static int tmc_read_prepare(struct tmc_drvdata *drvdata) { + const struct sysfs_read_ops *byte_cntr_sysfs_read_ops; int ret = 0; - switch (drvdata->config_type) { - case TMC_CONFIG_TYPE_ETB: - case TMC_CONFIG_TYPE_ETF: - ret = tmc_read_prepare_etb(drvdata); - break; - case TMC_CONFIG_TYPE_ETR: - ret = tmc_read_prepare_etr(drvdata); - break; - default: - ret = -EINVAL; + byte_cntr_sysfs_read_ops = tmc_get_byte_cntr_sysfs_ops(drvdata); + if (byte_cntr_sysfs_read_ops) { + ret = byte_cntr_sysfs_read_ops->read_prepare(drvdata); + goto out; } + if (drvdata->sysfs_ops) + ret = drvdata->sysfs_ops->read_prepare(drvdata); + else + ret = -EINVAL; + +out: if (!ret) dev_dbg(&drvdata->csdev->dev, "TMC read start\n"); @@ -252,20 +282,21 @@ static int tmc_read_prepare(struct tmc_drvdata *drvdata) static int tmc_read_unprepare(struct tmc_drvdata *drvdata) { + const struct sysfs_read_ops *byte_cntr_sysfs_read_ops; int ret = 0; - switch (drvdata->config_type) { - case TMC_CONFIG_TYPE_ETB: - case TMC_CONFIG_TYPE_ETF: - ret = tmc_read_unprepare_etb(drvdata); - break; - case TMC_CONFIG_TYPE_ETR: - ret = tmc_read_unprepare_etr(drvdata); - break; - default: - ret = -EINVAL; + byte_cntr_sysfs_read_ops = tmc_get_byte_cntr_sysfs_ops(drvdata); + if (byte_cntr_sysfs_read_ops) { + ret = byte_cntr_sysfs_read_ops->read_unprepare(drvdata); + goto out; } + if (drvdata->sysfs_ops) + ret = drvdata->sysfs_ops->read_unprepare(drvdata); + else + ret = -EINVAL; + +out: if (!ret) dev_dbg(&drvdata->csdev->dev, "TMC read end\n"); @@ -291,13 +322,14 @@ static int tmc_open(struct inode *inode, struct file *file) static ssize_t tmc_get_sysfs_trace(struct tmc_drvdata *drvdata, loff_t pos, size_t len, char **bufpp) { - switch (drvdata->config_type) { - case TMC_CONFIG_TYPE_ETB: - case TMC_CONFIG_TYPE_ETF: - return tmc_etb_get_sysfs_trace(drvdata, pos, len, bufpp); - case TMC_CONFIG_TYPE_ETR: - return tmc_etr_get_sysfs_trace(drvdata, pos, len, bufpp); - } + const struct sysfs_read_ops *byte_cntr_sysfs_read_ops; + + byte_cntr_sysfs_read_ops = tmc_get_byte_cntr_sysfs_ops(drvdata); + if (byte_cntr_sysfs_read_ops) + return byte_cntr_sysfs_read_ops->get_trace_data(drvdata, pos, len, bufpp); + + if (drvdata->sysfs_ops) + return drvdata->sysfs_ops->get_trace_data(drvdata, pos, len, bufpp); return -EINVAL; } @@ -319,7 +351,11 @@ static ssize_t tmc_read(struct file *file, char __user *data, size_t len, return -EFAULT; } - *ppos += actual; + if (drvdata->reading_node) + drvdata->reading_node->pos += actual; + else + *ppos += actual; + dev_dbg(&drvdata->csdev->dev, "%zu bytes copied\n", actual); return actual; @@ -457,21 +493,130 @@ static enum tmc_mem_intf_width tmc_get_memwidth(u32 devid) return memwidth; } +struct tmc_smp_arg { + struct tmc_drvdata *drvdata; + u32 offset; + int rc; +}; + +static void tmc_read_reg_smp_call(void *info) +{ + struct tmc_smp_arg *arg = info; + + arg->rc = readl_relaxed(arg->drvdata->base + arg->offset); +} + +static u32 cpu_tmc_read_reg(struct tmc_drvdata *drvdata, u32 offset) +{ + struct tmc_smp_arg arg = { + .drvdata = drvdata, + .offset = offset, + }; + int cpu, ret = 0; + + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + tmc_read_reg_smp_call, &arg, 1); + if (!ret) + return arg.rc; + } + + return ret; +} + +static ssize_t coresight_tmc_reg32_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct cs_off_attribute *cs_attr = container_of(attr, struct cs_off_attribute, attr); + int ret; + u32 val; + + ret = pm_runtime_resume_and_get(dev->parent); + if (ret < 0) + return ret; + + if (!drvdata->cpumask) + val = readl_relaxed(drvdata->base + cs_attr->off); + else + val = cpu_tmc_read_reg(drvdata, cs_attr->off); + + pm_runtime_put(dev->parent); + + if (ret < 0) + return ret; + else + return sysfs_emit(buf, "0x%x\n", val); +} + +static ssize_t coresight_tmc_reg64_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct cs_pair_attribute *cs_attr = container_of(attr, struct cs_pair_attribute, attr); + int ret; + u64 val; + + ret = pm_runtime_resume_and_get(dev->parent); + if (ret < 0) + return ret; + if (!drvdata->cpumask) { + val = readl_relaxed(drvdata->base + cs_attr->lo_off) | + ((u64)readl_relaxed(drvdata->base + cs_attr->hi_off) << 32); + } else { + ret = cpu_tmc_read_reg(drvdata, cs_attr->lo_off); + + if (ret < 0) + goto out; + + val = ret; + + ret = cpu_tmc_read_reg(drvdata, cs_attr->hi_off); + if (ret < 0) + goto out; + + val |= ((u64)ret << 32); + } + +out: + pm_runtime_put_sync(dev->parent); + if (ret < 0) + return ret; + else + return sysfs_emit(buf, "0x%llx\n", val); +} + +#define coresight_tmc_reg32(name, offset) \ + (&((struct cs_off_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_tmc_reg32_show, NULL), \ + offset \ + } \ + })[0].attr.attr) +#define coresight_tmc_reg64(name, lo_off, hi_off) \ + (&((struct cs_pair_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_tmc_reg64_show, NULL), \ + lo_off, hi_off \ + } \ + })[0].attr.attr) static struct attribute *coresight_tmc_mgmt_attrs[] = { - coresight_simple_reg32(rsz, TMC_RSZ), - coresight_simple_reg32(sts, TMC_STS), - coresight_simple_reg64(rrp, TMC_RRP, TMC_RRPHI), - coresight_simple_reg64(rwp, TMC_RWP, TMC_RWPHI), - coresight_simple_reg32(trg, TMC_TRG), - coresight_simple_reg32(ctl, TMC_CTL), - coresight_simple_reg32(ffsr, TMC_FFSR), - coresight_simple_reg32(ffcr, TMC_FFCR), - coresight_simple_reg32(mode, TMC_MODE), - coresight_simple_reg32(pscr, TMC_PSCR), - coresight_simple_reg32(devid, CORESIGHT_DEVID), - coresight_simple_reg64(dba, TMC_DBALO, TMC_DBAHI), - coresight_simple_reg32(axictl, TMC_AXICTL), - coresight_simple_reg32(authstatus, TMC_AUTHSTATUS), + coresight_tmc_reg32(rsz, TMC_RSZ), + coresight_tmc_reg32(sts, TMC_STS), + coresight_tmc_reg64(rrp, TMC_RRP, TMC_RRPHI), + coresight_tmc_reg64(rwp, TMC_RWP, TMC_RWPHI), + coresight_tmc_reg32(trg, TMC_TRG), + coresight_tmc_reg32(ctl, TMC_CTL), + coresight_tmc_reg32(ffsr, TMC_FFSR), + coresight_tmc_reg32(ffcr, TMC_FFCR), + coresight_tmc_reg32(mode, TMC_MODE), + coresight_tmc_reg32(pscr, TMC_PSCR), + coresight_tmc_reg32(devid, CORESIGHT_DEVID), + coresight_tmc_reg64(dba, TMC_DBALO, TMC_DBAHI), + coresight_tmc_reg32(axictl, TMC_AXICTL), + coresight_tmc_reg32(authstatus, TMC_AUTHSTATUS), NULL, }; @@ -769,56 +914,26 @@ static void register_crash_dev_interface(struct tmc_drvdata *drvdata, "Valid crash tracedata found\n"); } -static int __tmc_probe(struct device *dev, struct resource *res) +static const struct sysfs_read_ops tmc_etb_sysfs_read_ops = { + .read_prepare = tmc_read_prepare_etb, + .read_unprepare = tmc_read_unprepare_etb, + .get_trace_data = tmc_etb_get_sysfs_trace, +}; + +static const struct sysfs_read_ops tmc_etr_sysfs_read_ops = { + .read_prepare = tmc_read_prepare_etr, + .read_unprepare = tmc_read_unprepare_etr, + .get_trace_data = tmc_etr_get_sysfs_trace, +}; + +static int tmc_add_coresight_dev(struct device *dev) { - int ret = 0; - u32 devid; - void __iomem *base; - struct coresight_platform_data *pdata = NULL; - struct tmc_drvdata *drvdata; + struct tmc_drvdata *drvdata = dev_get_drvdata(dev); struct coresight_desc desc = { 0 }; struct coresight_dev_list *dev_list = NULL; + int ret = 0; - drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); - if (!drvdata) - return -ENOMEM; - - dev_set_drvdata(dev, drvdata); - - ret = coresight_get_enable_clocks(dev, &drvdata->pclk, &drvdata->atclk); - if (ret) - return ret; - - ret = -ENOMEM; - - /* Validity for the resource is already checked by the AMBA core */ - base = devm_ioremap_resource(dev, res); - if (IS_ERR(base)) { - ret = PTR_ERR(base); - goto out; - } - - drvdata->base = base; - desc.access = CSDEV_ACCESS_IOMEM(base); - - raw_spin_lock_init(&drvdata->spinlock); - - devid = readl_relaxed(drvdata->base + CORESIGHT_DEVID); - drvdata->config_type = BMVAL(devid, 6, 7); - drvdata->memwidth = tmc_get_memwidth(devid); - /* This device is not associated with a session */ - drvdata->pid = -1; - drvdata->etr_mode = ETR_MODE_AUTO; - - if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { - drvdata->size = tmc_etr_get_default_buffer_size(dev); - drvdata->max_burst_size = tmc_etr_get_max_burst_size(dev); - } else { - drvdata->size = readl_relaxed(drvdata->base + TMC_RSZ) * 4; - } - - tmc_get_reserved_region(dev); - + desc.access = CSDEV_ACCESS_IOMEM(drvdata->base); desc.dev = dev; switch (drvdata->config_type) { @@ -828,18 +943,21 @@ static int __tmc_probe(struct device *dev, struct resource *res) desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER; desc.ops = &tmc_etb_cs_ops; dev_list = &etb_devs; + drvdata->sysfs_ops = &tmc_etb_sysfs_read_ops; break; case TMC_CONFIG_TYPE_ETR: desc.groups = coresight_etr_groups; desc.type = CORESIGHT_DEV_TYPE_SINK; desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_SYSMEM; desc.ops = &tmc_etr_cs_ops; - ret = tmc_etr_setup_caps(dev, devid, &desc.access); + ret = tmc_etr_setup_caps(dev, drvdata->devid, &desc.access); if (ret) - goto out; + return ret; idr_init(&drvdata->idr); mutex_init(&drvdata->idr_mutex); dev_list = &etr_devs; + INIT_LIST_HEAD(&drvdata->etr_buf_list); + drvdata->sysfs_ops = &tmc_etr_sysfs_read_ops; break; case TMC_CONFIG_TYPE_ETF: desc.groups = coresight_etf_groups; @@ -848,47 +966,153 @@ static int __tmc_probe(struct device *dev, struct resource *res) desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO; desc.ops = &tmc_etf_cs_ops; dev_list = &etf_devs; + drvdata->sysfs_ops = &tmc_etb_sysfs_read_ops; break; default: pr_err("%s: Unsupported TMC config\n", desc.name); - ret = -EINVAL; - goto out; + return -EINVAL; } desc.name = coresight_alloc_device_name(dev_list, dev); - if (!desc.name) { - ret = -ENOMEM; + if (!desc.name) + return -ENOMEM; + + drvdata->desc_name = desc.name; + + desc.pdata = dev->platform_data; + + drvdata->csdev = coresight_register(&desc); + if (IS_ERR(drvdata->csdev)) + return PTR_ERR(drvdata->csdev); + + drvdata->miscdev.name = desc.name; + drvdata->miscdev.minor = MISC_DYNAMIC_MINOR; + drvdata->miscdev.fops = &tmc_fops; + ret = misc_register(&drvdata->miscdev); + if (ret) + coresight_unregister(drvdata->csdev); + + return ret; +} + +static void tmc_clear_self_claim_tag(struct tmc_drvdata *drvdata) +{ + struct csdev_access access = CSDEV_ACCESS_IOMEM(drvdata->base); + + coresight_clear_self_claim_tag(&access); +} + +static void tmc_init_hw_config(struct tmc_drvdata *drvdata) +{ + u32 devid; + + devid = readl_relaxed(drvdata->base + CORESIGHT_DEVID); + drvdata->config_type = BMVAL(devid, 6, 7); + drvdata->memwidth = tmc_get_memwidth(devid); + drvdata->devid = devid; + drvdata->size = readl_relaxed(drvdata->base + TMC_RSZ) * 4; + tmc_clear_self_claim_tag(drvdata); +} + +static void tmc_init_on_cpu(void *info) +{ + struct tmc_drvdata *drvdata = info; + + tmc_init_hw_config(drvdata); +} + +static struct cpumask *tmc_get_cpumask(struct device *dev) +{ + struct generic_pm_domain *pd; + + pd = pd_to_genpd(dev->pm_domain); + if (pd) + return pd->cpus; + + return NULL; +} + +static int __tmc_probe(struct device *dev, struct resource *res) +{ + int cpu, ret = 0; + void __iomem *base; + struct coresight_platform_data *pdata = NULL; + struct tmc_drvdata *drvdata; + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + dev_set_drvdata(dev, drvdata); + + ret = coresight_get_enable_clocks(dev, &drvdata->pclk, &drvdata->atclk); + if (ret) + return ret; + + ret = -ENOMEM; + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) { + ret = PTR_ERR(base); goto out; } + drvdata->base = base; + + raw_spin_lock_init(&drvdata->spinlock); + /* This device is not associated with a session */ + drvdata->pid = -1; + drvdata->etr_mode = ETR_MODE_AUTO; + tmc_get_reserved_region(dev); + pdata = coresight_get_platform_data(dev); if (IS_ERR(pdata)) { ret = PTR_ERR(pdata); goto out; } dev->platform_data = pdata; - desc.pdata = pdata; - coresight_clear_self_claim_tag(&desc.access); - drvdata->csdev = coresight_register(&desc); - if (IS_ERR(drvdata->csdev)) { - ret = PTR_ERR(drvdata->csdev); - goto out; + if (is_of_node(dev_fwnode(dev)) && + of_device_is_compatible(dev->of_node, "arm,coresight-cpu-tmc")) { + drvdata->cpumask = tmc_get_cpumask(dev); + if (!drvdata->cpumask) + return -EINVAL; + + drvdata->dev = dev; + + cpus_read_lock(); + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + tmc_init_on_cpu, drvdata, 1); + if (!ret) + break; + } + + if (ret) { + scoped_guard(spinlock, &delay_lock) + list_add(&drvdata->link, &tmc_delay_probe); + cpus_read_unlock(); + ret = 0; + goto out; + } + + cpus_read_unlock(); + } else { + tmc_init_hw_config(drvdata); } - drvdata->miscdev.name = desc.name; - drvdata->miscdev.minor = MISC_DYNAMIC_MINOR; - drvdata->miscdev.fops = &tmc_fops; - ret = misc_register(&drvdata->miscdev); - if (ret) { - coresight_unregister(drvdata->csdev); - goto out; + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + drvdata->size = tmc_etr_get_default_buffer_size(dev); + drvdata->max_burst_size = tmc_etr_get_max_burst_size(dev); } + ret = tmc_add_coresight_dev(dev); + out: if (is_tmc_crashdata_valid(drvdata) && !tmc_prepare_crashdata(drvdata)) - register_crash_dev_interface(drvdata, desc.name); + register_crash_dev_interface(drvdata, drvdata->desc_name); return ret; } @@ -934,10 +1158,16 @@ static void __tmc_remove(struct device *dev) * etb fops in this case, device is there until last file * handler to this device is closed. */ - misc_deregister(&drvdata->miscdev); + if (!drvdata->cpumask) + misc_deregister(&drvdata->miscdev); if (drvdata->crashdev.fops) misc_deregister(&drvdata->crashdev); - coresight_unregister(drvdata->csdev); + if (drvdata->csdev) { + coresight_unregister(drvdata->csdev); + } else { + scoped_guard(spinlock, &delay_lock) + list_del(&drvdata->link); + } } static void tmc_remove(struct amba_device *adev) @@ -992,7 +1222,6 @@ static void tmc_platform_remove(struct platform_device *pdev) if (WARN_ON(!drvdata)) return; - __tmc_remove(&pdev->dev); pm_runtime_disable(&pdev->dev); } @@ -1029,6 +1258,13 @@ static const struct dev_pm_ops tmc_dev_pm_ops = { SET_RUNTIME_PM_OPS(tmc_runtime_suspend, tmc_runtime_resume, NULL) }; +static const struct of_device_id tmc_match[] = { + {.compatible = "arm,coresight-cpu-tmc"}, + {} +}; + +MODULE_DEVICE_TABLE(of, tmc_match); + #ifdef CONFIG_ACPI static const struct acpi_device_id tmc_acpi_ids[] = { {"ARMHC501", 0, 0, 0}, /* ARM CoreSight ETR */ @@ -1043,20 +1279,62 @@ static struct platform_driver tmc_platform_driver = { .remove = tmc_platform_remove, .driver = { .name = "coresight-tmc-platform", + .of_match_table = tmc_match, .acpi_match_table = ACPI_PTR(tmc_acpi_ids), .suppress_bind_attrs = true, .pm = &tmc_dev_pm_ops, }, }; +static int tmc_online_cpu(unsigned int cpu) +{ + struct tmc_drvdata *drvdata, *tmp; + int ret; + + spin_lock(&delay_lock); + list_for_each_entry_safe(drvdata, tmp, &tmc_delay_probe, link) { + if (cpumask_test_cpu(cpu, drvdata->cpumask)) { + list_del(&drvdata->link); + + spin_unlock(&delay_lock); + ret = pm_runtime_resume_and_get(drvdata->dev); + if (ret < 0) + return 0; + + tmc_init_hw_config(drvdata); + tmc_clear_self_claim_tag(drvdata); + tmc_add_coresight_dev(drvdata->dev); + pm_runtime_put(drvdata->dev); + spin_lock(&delay_lock); + } + } + spin_unlock(&delay_lock); + return 0; +} + static int __init tmc_init(void) { + int ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm/coresight-tmc:online", + tmc_online_cpu, NULL); + + if (ret > 0) + hp_online = ret; + else + return ret; + return coresight_init_driver("tmc", &tmc_driver, &tmc_platform_driver, THIS_MODULE); } static void __exit tmc_exit(void) { coresight_remove_driver(&tmc_driver, &tmc_platform_driver); + if (hp_online) { + cpuhp_remove_state_nocalls(hp_online); + hp_online = 0; + } } module_init(tmc_init); module_exit(tmc_exit); diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 0f45ab5e52499..281c3b316dc3c 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -47,7 +47,7 @@ static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) return rc; } -static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +static int tmc_etb_enable_hw_local(struct tmc_drvdata *drvdata) { int rc = coresight_claim_device(drvdata->csdev); @@ -60,6 +60,36 @@ static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) return rc; } +struct tmc_smp_arg { + struct tmc_drvdata *drvdata; + int rc; +}; + +static void tmc_etb_enable_hw_smp_call(void *info) +{ + struct tmc_smp_arg *arg = info; + + arg->rc = tmc_etb_enable_hw_local(arg->drvdata); +} + +static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +{ + int cpu, ret; + struct tmc_smp_arg arg = { 0 }; + + if (!drvdata->cpumask) + return tmc_etb_enable_hw_local(drvdata); + + arg.drvdata = drvdata; + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + tmc_etb_enable_hw_smp_call, &arg, 1); + if (!ret) + return arg.rc; + } + return ret; +} + static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata) { char *bufp; @@ -130,7 +160,7 @@ static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) return rc; } -static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +static int tmc_etf_enable_hw_local(struct tmc_drvdata *drvdata) { int rc = coresight_claim_device(drvdata->csdev); @@ -143,6 +173,32 @@ static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) return rc; } +static void tmc_etf_enable_hw_smp_call(void *info) +{ + struct tmc_smp_arg *arg = info; + + arg->rc = tmc_etf_enable_hw_local(arg->drvdata); +} + +static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +{ + int cpu, ret; + struct tmc_smp_arg arg = { 0 }; + + if (!drvdata->cpumask) + return tmc_etf_enable_hw_local(drvdata); + + arg.drvdata = drvdata; + + for_each_cpu(cpu, drvdata->cpumask) { + ret = smp_call_function_single(cpu, + tmc_etf_enable_hw_smp_call, &arg, 1); + if (!ret) + return arg.rc; + } + return ret; +} + static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) { struct coresight_device *csdev = drvdata->csdev; @@ -228,7 +284,11 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) used = true; drvdata->buf = buf; } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + ret = tmc_etb_enable_hw(drvdata); + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); if (!ret) { coresight_set_mode(csdev, CS_MODE_SYSFS); csdev->refcnt++; @@ -290,7 +350,10 @@ static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) break; } - ret = tmc_etb_enable_hw(drvdata); + if (drvdata->cpumask && !cpumask_test_cpu(smp_processor_id(), drvdata->cpumask)) + break; + + ret = tmc_etb_enable_hw_local(drvdata); if (!ret) { /* Associate with monitored process. */ drvdata->pid = pid; @@ -361,7 +424,8 @@ static int tmc_disable_etf_sink(struct coresight_device *csdev) static int tmc_enable_etf_link(struct coresight_device *csdev, struct coresight_connection *in, - struct coresight_connection *out) + struct coresight_connection *out, + enum cs_mode mode) { int ret = 0; unsigned long flags; @@ -374,19 +438,38 @@ static int tmc_enable_etf_link(struct coresight_device *csdev, return -EBUSY; } - if (csdev->refcnt == 0) { + if (csdev->refcnt == 0) + first_enable = true; + + if (!first_enable) + csdev->refcnt++; + + if (mode == CS_MODE_PERF) { + if (first_enable) { + if (drvdata->cpumask && + !cpumask_test_cpu(smp_processor_id(), drvdata->cpumask)) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return -EINVAL; + } + + ret = tmc_etf_enable_hw_local(drvdata); + if (!ret) + csdev->refcnt++; + } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return ret; + } + + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + if (first_enable) { ret = tmc_etf_enable_hw(drvdata); if (!ret) { coresight_set_mode(csdev, CS_MODE_SYSFS); - first_enable = true; + csdev->refcnt++; + dev_dbg(&csdev->dev, "TMC-ETF enabled\n"); } } - if (!ret) - csdev->refcnt++; - raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); - if (first_enable) - dev_dbg(&csdev->dev, "TMC-ETF enabled\n"); return ret; } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index b07fcdb3fe1a8..10853a83ecb02 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1163,6 +1163,10 @@ ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata, ssize_t actual = len; struct etr_buf *etr_buf = drvdata->sysfs_buf; + /* Reading the buffer from the buf_node if it exists*/ + if (drvdata->reading_node) + etr_buf = drvdata->reading_node->sysfs_buf; + if (pos + actual > etr_buf->len) actual = etr_buf->len - pos; if (actual <= 0) @@ -1226,6 +1230,20 @@ static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata) } +/** + * tmc_etr_enable_disable_hw - enable/disable the ETR hw. + * @drvdata: drvdata of the TMC device. + * @enable: indicates enable/disable. + */ +void tmc_etr_enable_disable_hw(struct tmc_drvdata *drvdata, bool enable) +{ + if (enable) + __tmc_etr_enable_hw(drvdata); + else + __tmc_etr_disable_hw(drvdata); +} +EXPORT_SYMBOL_GPL(tmc_etr_enable_disable_hw); + void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) { __tmc_etr_disable_hw(drvdata); @@ -1250,6 +1268,13 @@ static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev) * with the lock released. */ raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + /* + * If the ETR is already enabled, continue with the existing buffer. + */ + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) + goto out; + sysfs_buf = READ_ONCE(drvdata->sysfs_buf); if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) { raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); @@ -1299,6 +1324,19 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) raw_spin_lock_irqsave(&drvdata->spinlock, flags); + /* + * Since the sysfs buffer allocation and the hardware enablement is not + * in the same critical region, it's possible to race with the perf. + */ + if (coresight_get_mode(csdev) == CS_MODE_PERF) { + drvdata->sysfs_buf = NULL; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Free allocated memory out side of the spinlock */ + tmc_etr_free_sysfs_buf(sysfs_buf); + return -EBUSY; + } + /* * In sysFS mode we can have multiple writers per sink. Since this * sink is already enabled no memory is needed and the HW need not be @@ -1327,7 +1365,8 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev, enum cs_mode mode, void *data) { - struct perf_output_handle *handle = data; + struct coresight_path *path = data; + struct perf_output_handle *handle = path->handle; struct etr_perf_buffer *etr_perf; switch (mode) { @@ -1909,6 +1948,100 @@ const struct coresight_ops tmc_etr_cs_ops = { .panic_ops = &tmc_etr_sync_ops, }; +/** + * tmc_clean_etr_buf_list - clean the etr_buf_list. + * @drvdata: driver data of the TMC device. + * + * Remove the allocated node from the list and free the extra buffer. + */ +void tmc_clean_etr_buf_list(struct tmc_drvdata *drvdata) +{ + struct etr_buf_node *nd, *next; + + list_for_each_entry_safe(nd, next, &drvdata->etr_buf_list, node) { + if (nd->sysfs_buf == drvdata->sysfs_buf) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_DISABLED) { + drvdata->sysfs_buf = NULL; + tmc_free_etr_buf(nd->sysfs_buf); + nd->sysfs_buf = NULL; + } + list_del(&nd->node); + kfree(nd); + } else { + /* Free allocated buffers which are not utilized by ETR */ + list_del(&nd->node); + tmc_free_etr_buf(nd->sysfs_buf); + nd->sysfs_buf = NULL; + kfree(nd); + } + } +} +EXPORT_SYMBOL_GPL(tmc_clean_etr_buf_list); + +/** + * tmc_create_etr_buf_list - create a list to manage the etr_buf_node. + * @drvdata: driver data of the TMC device. + * @num_nodes: number of nodes want to create with the list. + * + * Return 0 upon success and return the error number if fail. + */ +int tmc_create_etr_buf_list(struct tmc_drvdata *drvdata, int num_nodes) +{ + struct etr_buf_node *new_node; + struct etr_buf *sysfs_buf; + int i = 0, ret = 0; + + /* We dont need a list if there is only one node */ + if (num_nodes < 2) + return -EINVAL; + + /* We expect that sysfs_buf in drvdata has already been allocated. */ + if (drvdata->sysfs_buf) { + /* Directly insert the allocated sysfs_buf into the list first */ + new_node = kzalloc(sizeof(struct etr_buf_node), GFP_KERNEL); + if (IS_ERR(new_node)) + return PTR_ERR(new_node); + + new_node->sysfs_buf = drvdata->sysfs_buf; + new_node->is_free = false; + list_add(&new_node->node, &drvdata->etr_buf_list); + i++; + } + + while (i < num_nodes) { + new_node = kzalloc(sizeof(struct etr_buf_node), GFP_KERNEL); + if (IS_ERR(new_node)) { + ret = PTR_ERR(new_node); + break; + } + + sysfs_buf = tmc_alloc_etr_buf(drvdata, drvdata->size, 0, cpu_to_node(0), NULL); + if (IS_ERR(sysfs_buf)) { + kfree(new_node); + ret = PTR_ERR(new_node); + break; + } + + /* We dont have a available sysfs_buf in drvdata, setup one */ + if (!drvdata->sysfs_buf) { + drvdata->sysfs_buf = sysfs_buf; + new_node->is_free = false; + } else + new_node->is_free = true; + + new_node->sysfs_buf = sysfs_buf; + list_add(&new_node->node, &drvdata->etr_buf_list); + i++; + } + + /* Clean the list if there is an error */ + if (ret) + tmc_clean_etr_buf_list(drvdata); + + return ret; +} +EXPORT_SYMBOL_GPL(tmc_create_etr_buf_list); + int tmc_read_prepare_etr(struct tmc_drvdata *drvdata) { int ret = 0; diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h index cbb4ba4391585..a5c8783bfc8d1 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.h +++ b/drivers/hwtracing/coresight/coresight-tmc.h @@ -208,6 +208,21 @@ struct tmc_resrv_buf { s64 len; }; +/** + * @sysfs_buf: Allocated sysfs_buf. + * @is_free: Indicates whether the buffer is free to choose. + * @pos: Position of the buffer. + * @node: Node in etr_buf_list. + */ +struct etr_buf_node { + struct etr_buf *sysfs_buf; + bool is_free; + loff_t pos; + struct list_head node; +}; + +struct sysfs_read_ops; + /** * struct tmc_drvdata - specifics associated to an TMC component * @atclk: optional clock for the core parts of the TMC. @@ -243,6 +258,14 @@ struct tmc_resrv_buf { * (after crash) by default. * @crash_mdata: Reserved memory for storing tmc crash metadata. * Used by ETR/ETF. + * @cpumask: CPU mask representing the CPUs related to this TMC. + * @devid: TMC variant ID inferred from the device configuration register. + * @desc_name: Name to be used while creating crash interface. + * @dev: pointer to the device associated with this TMC. + * @link: link to the delay_probed list. + * @etr_buf_list: List that is used to manage allocated etr_buf. + * @reading_node: Available buffer_node for byte-cntr reading. + * @sysfs_ops: Read operations for sysfs mode. */ struct tmc_drvdata { struct clk *atclk; @@ -273,6 +296,27 @@ struct tmc_drvdata { struct etr_buf *perf_buf; struct tmc_resrv_buf resrv_buf; struct tmc_resrv_buf crash_mdata; + struct cpumask *cpumask; + u32 devid; + const char *desc_name; + struct device *dev; + struct list_head link; + struct list_head etr_buf_list; + struct etr_buf_node *reading_node; + const struct sysfs_read_ops *sysfs_ops; +}; + +/** + * struct sysfs_read_ops - read operations for TMC and its helper devices + * @read_prepare: prepare operation. + * @read_unprepare: unprepare operation. + * @get_trace_data: read operation. + */ +struct sysfs_read_ops { + int (*read_prepare)(struct tmc_drvdata *drvdata); + int (*read_unprepare)(struct tmc_drvdata *drvdata); + ssize_t (*get_trace_data)(struct tmc_drvdata *drvdata, loff_t pos, + size_t len, char **bufpp); }; struct etr_buf_operations { @@ -444,5 +488,8 @@ void tmc_etr_remove_catu_ops(void); struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev, enum cs_mode mode, void *data); extern const struct attribute_group coresight_etr_group; +void tmc_clean_etr_buf_list(struct tmc_drvdata *drvdata); +int tmc_create_etr_buf_list(struct tmc_drvdata *drvdata, int num_nodes); +void tmc_etr_enable_disable_hw(struct tmc_drvdata *drvdata, bool enable); #endif diff --git a/drivers/hwtracing/coresight/coresight-tnoc.c b/drivers/hwtracing/coresight/coresight-tnoc.c index ff9a0a9cfe96e..48e9e685b9439 100644 --- a/drivers/hwtracing/coresight/coresight-tnoc.c +++ b/drivers/hwtracing/coresight/coresight-tnoc.c @@ -73,7 +73,8 @@ static void trace_noc_enable_hw(struct trace_noc_drvdata *drvdata) } static int trace_noc_enable(struct coresight_device *csdev, struct coresight_connection *inport, - struct coresight_connection *outport) + struct coresight_connection *outport, + enum cs_mode mode) { struct trace_noc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); diff --git a/drivers/hwtracing/coresight/coresight-tpda.c b/drivers/hwtracing/coresight/coresight-tpda.c index 333b3cb236859..03eb6c7fc84f4 100644 --- a/drivers/hwtracing/coresight/coresight-tpda.c +++ b/drivers/hwtracing/coresight/coresight-tpda.c @@ -68,6 +68,15 @@ static int tpdm_read_element_size(struct tpda_drvdata *drvdata, int rc = -EINVAL; struct tpdm_drvdata *tpdm_data = dev_get_drvdata(csdev->dev.parent); + if (coresight_is_static_tpdm(csdev)) { + rc = fwnode_property_read_u32(dev_fwnode(csdev->dev.parent), + "qcom,dsb-element-bits", &drvdata->dsb_esize); + rc &= fwnode_property_read_u32(dev_fwnode(csdev->dev.parent), + "qcom,cmb-element-bits", &drvdata->cmb_esize); + + goto out; + } + if (tpdm_data->dsb) { rc = fwnode_property_read_u32(dev_fwnode(csdev->dev.parent), "qcom,dsb-element-bits", &drvdata->dsb_esize); @@ -147,9 +156,44 @@ static void tpda_enable_pre_port(struct tpda_drvdata *drvdata) u32 val; val = readl_relaxed(drvdata->base + TPDA_CR); + val &= ~TPDA_CR_MID; val &= ~TPDA_CR_ATID; val |= FIELD_PREP(TPDA_CR_ATID, drvdata->atid); + if (drvdata->trig_async) + val |= TPDA_CR_SRIE; + else + val &= ~TPDA_CR_SRIE; + if (drvdata->trig_flag_ts) + val |= TPDA_CR_FLRIE; + else + val &= ~TPDA_CR_FLRIE; + if (drvdata->trig_freq) + val |= TPDA_CR_FRIE; + else + val &= ~TPDA_CR_FRIE; + if (drvdata->freq_ts) + val |= TPDA_CR_FREQTS; + else + val &= ~TPDA_CR_FREQTS; + if (drvdata->cmbchan_mode) + val |= TPDA_CR_CMBCHANMODE; + else + val &= ~TPDA_CR_CMBCHANMODE; writel_relaxed(val, drvdata->base + TPDA_CR); + + /* + * If FLRIE bit is set, set the master and channel + * id as zero + */ + if (drvdata->trig_flag_ts) + writel_relaxed(0x0, drvdata->base + TPDA_FPID_CR); + + /* Program the counter value for TPDA_SYNCR */ + val = readl_relaxed(drvdata->base + TPDA_SYNCR); + /* Clear the mode */ + val &= ~TPDA_SYNCR_MODE_CTRL; + val |= TPDA_SYNCR_COUNTER_MASK; + writel_relaxed(val, drvdata->base + TPDA_SYNCR); } static int tpda_enable_port(struct tpda_drvdata *drvdata, int port) @@ -197,7 +241,8 @@ static int __tpda_enable(struct tpda_drvdata *drvdata, int port) static int tpda_enable(struct coresight_device *csdev, struct coresight_connection *in, - struct coresight_connection *out) + struct coresight_connection *out, + enum cs_mode mode) { struct tpda_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); int ret = 0; @@ -265,6 +310,247 @@ static const struct coresight_ops tpda_cs_ops = { .link_ops = &tpda_link_ops, }; +static ssize_t trig_async_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sysfs_emit(buf, "%u\n", (unsigned int)drvdata->trig_async); +} + +static ssize_t trig_async_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + drvdata->trig_async = !!val; + + return size; +} +static DEVICE_ATTR_RW(trig_async_enable); + +static ssize_t trig_flag_ts_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sysfs_emit(buf, "%u\n", (unsigned int)drvdata->trig_flag_ts); +} + +static ssize_t trig_flag_ts_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + drvdata->trig_flag_ts = !!val; + + return size; +} +static DEVICE_ATTR_RW(trig_flag_ts_enable); + +static ssize_t trig_freq_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sysfs_emit(buf, "%u\n", (unsigned int)drvdata->trig_freq); +} + +static ssize_t trig_freq_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + drvdata->trig_freq = !!val; + + return size; +} +static DEVICE_ATTR_RW(trig_freq_enable); + +static ssize_t freq_ts_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sysfs_emit(buf, "%u\n", (unsigned int)drvdata->freq_ts); +} + +static ssize_t freq_ts_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + drvdata->freq_ts = !!val; + + return size; +} +static DEVICE_ATTR_RW(freq_ts_enable); + +static ssize_t global_flush_req_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (!drvdata->csdev->refcnt) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + CS_UNLOCK(drvdata->base); + val = readl_relaxed(drvdata->base + TPDA_CR); + CS_LOCK(drvdata->base); + /* Only read value for bit 0 */ + val &= BIT(0); + + return sysfs_emit(buf, "%lu\n", val); +} + +static ssize_t global_flush_req_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + if (!drvdata->csdev->refcnt || !val) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + CS_UNLOCK(drvdata->base); + val = readl_relaxed(drvdata->base + TPDA_CR); + /* Only set bit 0 */ + val |= BIT(0); + writel_relaxed(val, drvdata->base + TPDA_CR); + CS_LOCK(drvdata->base); + + return size; +} +static DEVICE_ATTR_RW(global_flush_req); + +static ssize_t cmbchan_mode_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sysfs_emit(buf, "%u\n", (unsigned int)drvdata->cmbchan_mode); +} + +static ssize_t cmbchan_mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + drvdata->cmbchan_mode = !!val; + + return size; +} +static DEVICE_ATTR_RW(cmbchan_mode); + +static ssize_t port_flush_req_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val; + + if (!drvdata->csdev->refcnt) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + CS_UNLOCK(drvdata->base); + val = readl_relaxed(drvdata->base + TPDA_FLUSH_CR); + CS_LOCK(drvdata->base); + return sysfs_emit(buf, "0x%lx\n", val); +} + +static ssize_t port_flush_req_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tpda_drvdata *drvdata = dev_get_drvdata(dev->parent); + u32 val; + + if (kstrtou32(buf, 0, &val)) + return -EINVAL; + + if (!drvdata->csdev->refcnt || !val) + return -EINVAL; + + guard(spinlock)(&drvdata->spinlock); + CS_UNLOCK(drvdata->base); + writel_relaxed(val, drvdata->base + TPDA_FLUSH_CR); + CS_LOCK(drvdata->base); + + return size; +} +static DEVICE_ATTR_RW(port_flush_req); + +static struct attribute *tpda_attrs[] = { + &dev_attr_trig_async_enable.attr, + &dev_attr_trig_flag_ts_enable.attr, + &dev_attr_trig_freq_enable.attr, + &dev_attr_freq_ts_enable.attr, + &dev_attr_global_flush_req.attr, + &dev_attr_cmbchan_mode.attr, + &dev_attr_port_flush_req.attr, + NULL, +}; + +static struct attribute_group tpda_attr_grp = { + .attrs = tpda_attrs, +}; + +static const struct attribute_group *tpda_attr_grps[] = { + &tpda_attr_grp, + NULL, +}; + static int tpda_init_default_data(struct tpda_drvdata *drvdata) { int atid; @@ -280,6 +566,7 @@ static int tpda_init_default_data(struct tpda_drvdata *drvdata) return atid; drvdata->atid = atid; + drvdata->freq_ts = true; return 0; } @@ -323,6 +610,7 @@ static int tpda_probe(struct amba_device *adev, const struct amba_id *id) desc.ops = &tpda_cs_ops; desc.pdata = adev->dev.platform_data; desc.dev = &adev->dev; + desc.groups = tpda_attr_grps; desc.access = CSDEV_ACCESS_IOMEM(base); drvdata->csdev = coresight_register(&desc); if (IS_ERR(drvdata->csdev)) diff --git a/drivers/hwtracing/coresight/coresight-tpda.h b/drivers/hwtracing/coresight/coresight-tpda.h index c6af3d2da3efe..284ac63699ad6 100644 --- a/drivers/hwtracing/coresight/coresight-tpda.h +++ b/drivers/hwtracing/coresight/coresight-tpda.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2023,2025 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _CORESIGHT_CORESIGHT_TPDA_H @@ -8,17 +8,38 @@ #define TPDA_CR (0x000) #define TPDA_Pn_CR(n) (0x004 + (n * 4)) +#define TPDA_FPID_CR (0x084) +#define TPDA_SYNCR (0x08C) +#define TPDA_FLUSH_CR (0x090) + +/* Cross trigger FREQ packets timestamp bit */ +#define TPDA_CR_FREQTS BIT(2) +/* Cross trigger FREQ packet request bit */ +#define TPDA_CR_FRIE BIT(3) +/* Cross trigger FLAG packet request interface bit */ +#define TPDA_CR_FLRIE BIT(4) +/* Cross trigger synchronization bit */ +#define TPDA_CR_SRIE BIT(5) +/* Packetize CMB/MCMB traffic bit */ +#define TPDA_CR_CMBCHANMODE BIT(20) + /* Aggregator port enable bit */ #define TPDA_Pn_CR_ENA BIT(0) /* Aggregator port CMB data set element size bit */ #define TPDA_Pn_CR_CMBSIZE GENMASK(7, 6) /* Aggregator port DSB data set element size bit */ #define TPDA_Pn_CR_DSBSIZE BIT(8) +/* TPDA_SYNCR mode control bit */ +#define TPDA_SYNCR_MODE_CTRL BIT(12) +/* TPDA_SYNCR counter mask */ +#define TPDA_SYNCR_COUNTER_MASK GENMASK(11, 0) #define TPDA_MAX_INPORTS 32 /* Bits 6 ~ 12 is for atid value */ #define TPDA_CR_ATID GENMASK(12, 6) +/* Bits 13 ~ 19 is for mid value */ +#define TPDA_CR_MID GENMASK(19, 13) /** * struct tpda_drvdata - specifics associated to an TPDA component @@ -29,6 +50,11 @@ * @enable: enable status of the component. * @dsb_esize Record the DSB element size. * @cmb_esize Record the CMB element size. + * @trig_async: Enable/disable cross trigger synchronization sequence interface. + * @trig_flag_ts: Enable/disable cross trigger FLAG packet request interface. + * @trig_freq: Enable/disable cross trigger FREQ packet request interface. + * @freq_ts: Enable/disable the timestamp for all FREQ packets. + * @cmbchan_mode: Configure the CMB/MCMB channel mode. */ struct tpda_drvdata { void __iomem *base; @@ -38,6 +64,11 @@ struct tpda_drvdata { u8 atid; u32 dsb_esize; u32 cmb_esize; + bool trig_async; + bool trig_flag_ts; + bool trig_freq; + bool freq_ts; + bool cmbchan_mode; }; #endif /* _CORESIGHT_CORESIGHT_TPDA_H */ diff --git a/drivers/hwtracing/coresight/coresight-tpdm.c b/drivers/hwtracing/coresight/coresight-tpdm.c index 7214e65097ec9..1766b01828192 100644 --- a/drivers/hwtracing/coresight/coresight-tpdm.c +++ b/drivers/hwtracing/coresight/coresight-tpdm.c @@ -495,7 +495,9 @@ static int tpdm_enable(struct coresight_device *csdev, struct perf_event *event, return -EBUSY; } - __tpdm_enable(drvdata); + if (!coresight_is_static_tpdm(csdev)) + __tpdm_enable(drvdata); + drvdata->enable = true; spin_unlock(&drvdata->spinlock); @@ -551,7 +553,9 @@ static void tpdm_disable(struct coresight_device *csdev, return; } - __tpdm_disable(drvdata); + if (!coresight_is_static_tpdm(csdev)) + __tpdm_disable(drvdata); + coresight_set_mode(csdev, CS_MODE_DISABLED); drvdata->enable = false; spin_unlock(&drvdata->spinlock); @@ -1342,10 +1346,9 @@ static const struct attribute_group *tpdm_attr_grps[] = { NULL, }; -static int tpdm_probe(struct amba_device *adev, const struct amba_id *id) +static int tpdm_probe(struct device *dev, struct resource *res) { void __iomem *base; - struct device *dev = &adev->dev; struct coresight_platform_data *pdata; struct tpdm_drvdata *drvdata; struct coresight_desc desc = { 0 }; @@ -1354,32 +1357,33 @@ static int tpdm_probe(struct amba_device *adev, const struct amba_id *id) pdata = coresight_get_platform_data(dev); if (IS_ERR(pdata)) return PTR_ERR(pdata); - adev->dev.platform_data = pdata; + dev->platform_data = pdata; /* driver data*/ drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) return -ENOMEM; - drvdata->dev = &adev->dev; + drvdata->dev = dev; dev_set_drvdata(dev, drvdata); - base = devm_ioremap_resource(dev, &adev->res); - if (IS_ERR(base)) - return PTR_ERR(base); + if (res) { + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); - drvdata->base = base; + drvdata->base = base; + ret = tpdm_datasets_setup(drvdata); + if (ret) + return ret; - ret = tpdm_datasets_setup(drvdata); - if (ret) - return ret; + if (drvdata && tpdm_has_dsb_dataset(drvdata)) + of_property_read_u32(drvdata->dev->of_node, + "qcom,dsb-msrs-num", &drvdata->dsb_msr_num); - if (drvdata && tpdm_has_dsb_dataset(drvdata)) - of_property_read_u32(drvdata->dev->of_node, - "qcom,dsb-msrs-num", &drvdata->dsb_msr_num); - - if (drvdata && tpdm_has_cmb_dataset(drvdata)) - of_property_read_u32(drvdata->dev->of_node, - "qcom,cmb-msrs-num", &drvdata->cmb_msr_num); + if (drvdata && tpdm_has_cmb_dataset(drvdata)) + of_property_read_u32(drvdata->dev->of_node, + "qcom,cmb-msrs-num", &drvdata->cmb_msr_num); + } /* Set up coresight component description */ desc.name = coresight_alloc_device_name(&tpdm_devs, dev); @@ -1388,34 +1392,51 @@ static int tpdm_probe(struct amba_device *adev, const struct amba_id *id) desc.type = CORESIGHT_DEV_TYPE_SOURCE; desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM; desc.ops = &tpdm_cs_ops; - desc.pdata = adev->dev.platform_data; - desc.dev = &adev->dev; + desc.pdata = dev->platform_data; + desc.dev = dev; desc.access = CSDEV_ACCESS_IOMEM(base); - desc.groups = tpdm_attr_grps; + if (res) + desc.groups = tpdm_attr_grps; drvdata->csdev = coresight_register(&desc); if (IS_ERR(drvdata->csdev)) return PTR_ERR(drvdata->csdev); spin_lock_init(&drvdata->spinlock); - /* Decrease pm refcount when probe is done.*/ - pm_runtime_put(&adev->dev); - return 0; } -static void tpdm_remove(struct amba_device *adev) +static int tpdm_remove(struct device *dev) { - struct tpdm_drvdata *drvdata = dev_get_drvdata(&adev->dev); + struct tpdm_drvdata *drvdata = dev_get_drvdata(dev); coresight_unregister(drvdata->csdev); + + return 0; +} + +static int dynamic_tpdm_probe(struct amba_device *adev, + const struct amba_id *id) +{ + int ret; + + ret = tpdm_probe(&adev->dev, &adev->res); + if (!ret) + pm_runtime_put(&adev->dev); + + return ret; +} + +static void dynamic_tpdm_remove(struct amba_device *adev) +{ + tpdm_remove(&adev->dev); } /* * Different TPDM has different periph id. * The difference is 0-7 bits' value. So ignore 0-7 bits. */ -static const struct amba_id tpdm_ids[] = { +static const struct amba_id dynamic_tpdm_ids[] = { { .id = 0x001f0e00, .mask = 0x00ffff00, @@ -1423,17 +1444,76 @@ static const struct amba_id tpdm_ids[] = { { 0, 0, NULL }, }; -static struct amba_driver tpdm_driver = { +MODULE_DEVICE_TABLE(amba, dynamic_tpdm_ids); + +static struct amba_driver dynamic_tpdm_driver = { .drv = { .name = "coresight-tpdm", .suppress_bind_attrs = true, }, - .probe = tpdm_probe, - .id_table = tpdm_ids, - .remove = tpdm_remove, + .probe = dynamic_tpdm_probe, + .id_table = dynamic_tpdm_ids, + .remove = dynamic_tpdm_remove, }; -module_amba_driver(tpdm_driver); +static int tpdm_platform_probe(struct platform_device *pdev) +{ + struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + int ret; + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + ret = tpdm_probe(&pdev->dev, res); + pm_runtime_put(&pdev->dev); + if (ret) + pm_runtime_disable(&pdev->dev); + + return ret; +} + +static void tpdm_platform_remove(struct platform_device *pdev) +{ + struct tpdm_drvdata *drvdata = dev_get_drvdata(&pdev->dev); + + if (WARN_ON(!drvdata)) + return; + + tpdm_remove(&pdev->dev); + pm_runtime_disable(&pdev->dev); +} + +static const struct of_device_id static_tpdm_match[] = { + {.compatible = "qcom,coresight-static-tpdm"}, + {} +}; + +MODULE_DEVICE_TABLE(of, static_tpdm_match); + +static struct platform_driver static_tpdm_driver = { + .probe = tpdm_platform_probe, + .remove = tpdm_platform_remove, + .driver = { + .name = "coresight-static-tpdm", + .of_match_table = static_tpdm_match, + .suppress_bind_attrs = true, + }, +}; + +static int __init tpdm_init(void) +{ + return coresight_init_driver("tpdm", &dynamic_tpdm_driver, &static_tpdm_driver, + THIS_MODULE); +} + +static void __exit tpdm_exit(void) +{ + coresight_remove_driver(&dynamic_tpdm_driver, &static_tpdm_driver); +} + +module_init(tpdm_init); +module_exit(tpdm_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Trace, Profiling & Diagnostic Monitor driver"); diff --git a/drivers/hwtracing/coresight/coresight-tpdm.h b/drivers/hwtracing/coresight/coresight-tpdm.h index b117543897344..9f52c88ce5c12 100644 --- a/drivers/hwtracing/coresight/coresight-tpdm.h +++ b/drivers/hwtracing/coresight/coresight-tpdm.h @@ -343,4 +343,12 @@ struct tpdm_dataset_attribute { enum dataset_mem mem; u32 idx; }; + +static inline bool coresight_is_static_tpdm(struct coresight_device *csdev) +{ + struct device_node *node = csdev->dev.parent->of_node; + + return (csdev && + of_device_is_compatible(node, "qcom,coresight-static-tpdm")); +} #endif /* _CORESIGHT_CORESIGHT_TPDM_H */ diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c index 47d9e6c3bac04..e3a7ab112ea95 100644 --- a/drivers/hwtracing/intel_th/core.c +++ b/drivers/hwtracing/intel_th/core.c @@ -810,13 +810,20 @@ static int intel_th_output_open(struct inode *inode, struct file *file) int err; dev = bus_find_device_by_devt(&intel_th_bus, inode->i_rdev); - if (!dev || !dev->driver) + if (!dev) return -ENODEV; + if (!dev->driver) { + err = -ENODEV; + goto out_put_device; + } + thdrv = to_intel_th_driver(dev->driver); fops = fops_get(thdrv->fops); - if (!fops) - return -ENODEV; + if (!fops) { + err = -ENODEV; + goto out_put_device; + } replace_fops(file, fops); @@ -824,14 +831,30 @@ static int intel_th_output_open(struct inode *inode, struct file *file) if (file->f_op->open) { err = file->f_op->open(inode, file); - return err; + if (err) + goto out_put_device; } + return 0; + +out_put_device: + put_device(dev); + + return err; +} + +static int intel_th_output_release(struct inode *inode, struct file *file) +{ + struct intel_th_device *thdev = file->private_data; + + put_device(&thdev->dev); + return 0; } static const struct file_operations intel_th_output_fops = { .open = intel_th_output_open, + .release = intel_th_output_release, .llseek = noop_llseek, }; diff --git a/drivers/hwtracing/qcom/Kconfig b/drivers/hwtracing/qcom/Kconfig new file mode 100644 index 0000000000000..5c94c75ffa396 --- /dev/null +++ b/drivers/hwtracing/qcom/Kconfig @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# QCOM specific hwtracing drivers +# +menu "Qualcomm specific hwtracing drivers" + +config QCOM_TGU + tristate "QCOM Trigger Generation Unit driver" + depends on ARCH_QCOM || COMPILE_TEST + depends on ARM_AMBA + help + This driver provides support for Trigger Generation Unit that is + used to detect patterns or sequences on a given set of signals. + TGU is used to monitor a particular bus within a given region to + detect illegal transaction sequences or slave responses. It is also + used to monitor a data stream to detect protocol violations and to + provide a trigger point for centering data around a specific event + within the trace data buffer. + +endmenu diff --git a/drivers/hwtracing/qcom/Makefile b/drivers/hwtracing/qcom/Makefile new file mode 100644 index 0000000000000..5a0a868c1ea0d --- /dev/null +++ b/drivers/hwtracing/qcom/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_QCOM_TGU) += tgu.o diff --git a/drivers/hwtracing/qcom/tgu.c b/drivers/hwtracing/qcom/tgu.c new file mode 100644 index 0000000000000..9fb51f2a912f9 --- /dev/null +++ b/drivers/hwtracing/qcom/tgu.c @@ -0,0 +1,704 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tgu.h" + +static int calculate_array_location(struct tgu_drvdata *drvdata, + int step_index, int operation_index, + int reg_index) +{ + switch (operation_index) { + case TGU_PRIORITY0: + case TGU_PRIORITY1: + case TGU_PRIORITY2: + case TGU_PRIORITY3: + return operation_index * (drvdata->num_step) * + (drvdata->num_reg) + + step_index * (drvdata->num_reg) + reg_index; + case TGU_CONDITION_DECODE: + return step_index * (drvdata->num_condition_decode) + + reg_index; + case TGU_CONDITION_SELECT: + return step_index * (drvdata->num_condition_select) + + reg_index; + case TGU_COUNTER: + return step_index * (drvdata->num_counter) + reg_index; + case TGU_TIMER: + return step_index * (drvdata->num_timer) + reg_index; + default: + break; + } + + return -EINVAL; +} + +static int check_array_location(struct tgu_drvdata *drvdata, int step, + int ops, int reg) +{ + int result = calculate_array_location(drvdata, step, ops, reg); + + if (result == -EINVAL) + dev_err(drvdata->dev, "check array location - Fail\n"); + + return result; +} + +static ssize_t tgu_dataset_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tgu_drvdata *drvdata = dev_get_drvdata(dev); + struct tgu_attribute *tgu_attr = + container_of(attr, struct tgu_attribute, attr); + int index; + + index = check_array_location(drvdata, tgu_attr->step_index, + tgu_attr->operation_index, tgu_attr->reg_num); + + if (index == -EINVAL) + return index; + + switch (tgu_attr->operation_index) { + case TGU_PRIORITY0: + case TGU_PRIORITY1: + case TGU_PRIORITY2: + case TGU_PRIORITY3: + return sysfs_emit(buf, "0x%x\n", + drvdata->value_table->priority[index]); + case TGU_CONDITION_DECODE: + return sysfs_emit(buf, "0x%x\n", + drvdata->value_table->condition_decode[index]); + case TGU_CONDITION_SELECT: + return sysfs_emit(buf, "0x%x\n", + drvdata->value_table->condition_select[index]); + case TGU_TIMER: + return sysfs_emit(buf, "0x%x\n", + drvdata->value_table->timer[index]); + case TGU_COUNTER: + return sysfs_emit(buf, "0x%x\n", + drvdata->value_table->counter[index]); + default: + break; + } + return -EINVAL; +} + +static ssize_t tgu_dataset_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct tgu_drvdata *tgu_drvdata = dev_get_drvdata(dev); + struct tgu_attribute *tgu_attr = + container_of(attr, struct tgu_attribute, attr); + unsigned long val; + int index; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret) + return ret; + + guard(spinlock)(&tgu_drvdata->lock); + index = check_array_location(tgu_drvdata, tgu_attr->step_index, + tgu_attr->operation_index, + tgu_attr->reg_num); + + if (index == -EINVAL) + return index; + + switch (tgu_attr->operation_index) { + case TGU_PRIORITY0: + case TGU_PRIORITY1: + case TGU_PRIORITY2: + case TGU_PRIORITY3: + tgu_drvdata->value_table->priority[index] = val; + ret = size; + break; + case TGU_CONDITION_DECODE: + tgu_drvdata->value_table->condition_decode[index] = val; + ret = size; + break; + case TGU_CONDITION_SELECT: + tgu_drvdata->value_table->condition_select[index] = val; + ret = size; + break; + case TGU_TIMER: + tgu_drvdata->value_table->timer[index] = val; + ret = size; + break; + case TGU_COUNTER: + tgu_drvdata->value_table->counter[index] = val; + ret = size; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static umode_t tgu_node_visible(struct kobject *kobject, + struct attribute *attr, + int n) +{ + struct device *dev = kobj_to_dev(kobject); + struct tgu_drvdata *drvdata = dev_get_drvdata(dev); + struct device_attribute *dev_attr = + container_of(attr, struct device_attribute, attr); + struct tgu_attribute *tgu_attr = + container_of(dev_attr, struct tgu_attribute, attr); + + if (tgu_attr->step_index >= drvdata->num_step) + return SYSFS_GROUP_INVISIBLE; + + switch (tgu_attr->operation_index) { + case TGU_PRIORITY0: + case TGU_PRIORITY1: + case TGU_PRIORITY2: + case TGU_PRIORITY3: + if (tgu_attr->reg_num < drvdata->num_reg) + return attr->mode; + break; + case TGU_CONDITION_DECODE: + if (tgu_attr->reg_num < drvdata->num_condition_decode) + return attr->mode; + break; + case TGU_CONDITION_SELECT: + /* 'default' register is at the end of 'select' region */ + if (tgu_attr->reg_num == drvdata->num_condition_select - 1) + attr->name = "default"; + if (tgu_attr->reg_num < drvdata->num_condition_select) + return attr->mode; + break; + case TGU_COUNTER: + if (!drvdata->num_counter) + break; + if (tgu_attr->reg_num < drvdata->num_counter) + return attr->mode; + break; + case TGU_TIMER: + if (!drvdata->num_timer) + break; + if (tgu_attr->reg_num < drvdata->num_timer) + return attr->mode; + break; + default: + break; + } + + return 0; +} + +static ssize_t tgu_write_all_hw_regs(struct tgu_drvdata *drvdata) +{ + int i, j, k, index; + + TGU_UNLOCK(drvdata->base); + for (i = 0; i < drvdata->num_step; i++) { + for (j = 0; j < MAX_PRIORITY; j++) { + for (k = 0; k < drvdata->num_reg; k++) { + index = check_array_location( + drvdata, i, j, k); + if (index == -EINVAL) + goto exit; + + writel(drvdata->value_table->priority[index], + drvdata->base + + PRIORITY_REG_STEP(i, j, k)); + } + } + } + + for (i = 0; i < drvdata->num_step; i++) { + for (j = 0; j < drvdata->num_condition_decode; j++) { + index = check_array_location(drvdata, i, + TGU_CONDITION_DECODE, j); + if (index == -EINVAL) + goto exit; + + writel(drvdata->value_table->condition_decode[index], + drvdata->base + CONDITION_DECODE_STEP(i, j)); + } + } + + for (i = 0; i < drvdata->num_step; i++) { + for (j = 0; j < drvdata->num_condition_select; j++) { + index = check_array_location(drvdata, i, + TGU_CONDITION_SELECT, j); + if (index == -EINVAL) + goto exit; + + writel(drvdata->value_table->condition_select[index], + drvdata->base + CONDITION_SELECT_STEP(i, j)); + } + } + + for (i = 0; i < drvdata->num_step; i++) { + for (j = 0; j < drvdata->num_timer; j++) { + index = check_array_location(drvdata, i, TGU_TIMER, j); + + if (index == -EINVAL) + goto exit; + + writel(drvdata->value_table->timer[index], + drvdata->base + TIMER_COMPARE_STEP(i, j)); + } + } + + for (i = 0; i < drvdata->num_step; i++) { + for (j = 0; j < drvdata->num_counter; j++) { + index = check_array_location(drvdata, i, TGU_COUNTER, j); + + if (index == -EINVAL) + goto exit; + + writel(drvdata->value_table->counter[index], + drvdata->base + COUNTER_COMPARE_STEP(i, j)); + } + } + /* Enable TGU to program the triggers */ + writel(1, drvdata->base + TGU_CONTROL); +exit: + TGU_LOCK(drvdata->base); + return index >= 0 ? 0 : -EINVAL; +} + +static void tgu_set_reg_number(struct tgu_drvdata *drvdata) +{ + int num_sense_input; + int num_reg; + u32 devid; + + devid = readl(drvdata->base + TGU_DEVID); + + num_sense_input = TGU_DEVID_SENSE_INPUT(devid); + num_reg = (num_sense_input * TGU_BITS_PER_SIGNAL) / LENGTH_REGISTER; + + if ((num_sense_input * TGU_BITS_PER_SIGNAL) % LENGTH_REGISTER) + num_reg++; + + drvdata->num_reg = num_reg; +} + +static void tgu_set_steps(struct tgu_drvdata *drvdata) +{ + u32 devid; + + devid = readl(drvdata->base + TGU_DEVID); + + drvdata->num_step = TGU_DEVID_STEPS(devid); +} + +static void tgu_set_conditions(struct tgu_drvdata *drvdata) +{ + u32 devid; + + devid = readl(drvdata->base + TGU_DEVID); + drvdata->num_condition_decode = TGU_DEVID_CONDITIONS(devid); + /* select region has an additional 'default' register */ + drvdata->num_condition_select = TGU_DEVID_CONDITIONS(devid) + 1; +} + +static void tgu_set_timer_counter(struct tgu_drvdata *drvdata) +{ + int num_timers = 0, num_counters = 0; + u32 devid2; + + devid2 = readl(drvdata->base + CORESIGHT_DEVID2); + + if (TGU_DEVID2_TIMER0(devid2)) + num_timers++; + if (TGU_DEVID2_TIMER1(devid2)) + num_timers++; + + if (TGU_DEVID2_COUNTER0(devid2)) + num_counters++; + if (TGU_DEVID2_COUNTER1(devid2)) + num_counters++; + + drvdata->num_timer = num_timers; + drvdata->num_counter = num_counters; +} + +static int tgu_enable(struct device *dev) +{ + struct tgu_drvdata *drvdata = dev_get_drvdata(dev); + int ret; + + guard(spinlock)(&drvdata->lock); + + ret = tgu_write_all_hw_regs(drvdata); + if (!ret) + drvdata->enabled = true; + + return ret; +} + +static void tgu_do_disable(struct tgu_drvdata *drvdata) +{ + TGU_UNLOCK(drvdata->base); + writel(0, drvdata->base + TGU_CONTROL); + TGU_LOCK(drvdata->base); + + drvdata->enabled = false; +} + +static void tgu_disable(struct device *dev) +{ + struct tgu_drvdata *drvdata = dev_get_drvdata(dev); + + guard(spinlock)(&drvdata->lock); + if (!drvdata->enabled) + return; + + tgu_do_disable(drvdata); +} + +static ssize_t enable_tgu_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tgu_drvdata *drvdata = dev_get_drvdata(dev); + bool enabled; + + guard(spinlock)(&drvdata->lock); + enabled = drvdata->enabled; + + return sysfs_emit(buf, "%d\n", !!enabled); +} + +/* enable_tgu_store - Configure Trace and Gating Unit (TGU) triggers. */ +static ssize_t enable_tgu_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t size) +{ + struct tgu_drvdata *drvdata = dev_get_drvdata(dev); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret || val > 1) + return -EINVAL; + + if (val) { + scoped_guard(spinlock, &drvdata->lock) { + if (drvdata->enabled) + return -EBUSY; + } + + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; + + ret = tgu_enable(dev); + if (ret) { + pm_runtime_put(dev); + return ret; + } + } else { + scoped_guard(spinlock, &drvdata->lock) { + if (!drvdata->enabled) + return -EINVAL; + } + + tgu_disable(dev); + pm_runtime_put(dev); + } + + return size; +} +static DEVICE_ATTR_RW(enable_tgu); + +/* reset_tgu_store - Reset Trace and Gating Unit (TGU) configuration. */ +static ssize_t reset_tgu_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t size) +{ + struct tgu_drvdata *drvdata = dev_get_drvdata(dev); + struct value_table *vt = drvdata->value_table; + u32 *cond_decode = drvdata->value_table->condition_decode; + unsigned long value; + int i, j, ret; + + if (kstrtoul(buf, 0, &value) || value != 1) + return -EINVAL; + + spin_lock(&drvdata->lock); + if (!drvdata->enabled) { + spin_unlock(&drvdata->lock); + ret = pm_runtime_resume_and_get(drvdata->dev); + if (ret) + return ret; + spin_lock(&drvdata->lock); + } + + tgu_do_disable(drvdata); + + if (vt->priority) { + size_t size = MAX_PRIORITY * drvdata->num_step * + drvdata->num_reg * sizeof(unsigned int); + memset(vt->priority, 0, size); + } + + if (vt->condition_decode) { + size_t size = drvdata->num_condition_decode * + drvdata->num_step * sizeof(unsigned int); + memset(vt->condition_decode, 0, size); + } + + /* Initialize all condition registers to NOT(value=0x1000000) */ + for (i = 0; i < drvdata->num_step; i++) { + for (j = 0; j < drvdata->num_condition_decode; j++) { + cond_decode[calculate_array_location(drvdata, i, + TGU_CONDITION_DECODE, j)] = 0x1000000; + } + } + + if (vt->condition_select) { + size_t size = drvdata->num_condition_select * + drvdata->num_step * sizeof(unsigned int); + memset(vt->condition_select, 0, size); + } + + if (vt->timer) { + size_t size = (drvdata->num_step) * (drvdata->num_timer) * + sizeof(unsigned int); + memset(vt->timer, 0, size); + } + + if (vt->counter) { + size_t size = (drvdata->num_step) * (drvdata->num_counter) * + sizeof(unsigned int); + memset(vt->counter, 0, size); + } + + spin_unlock(&drvdata->lock); + + dev_dbg(dev, "Qualcomm-TGU reset complete\n"); + + pm_runtime_put(drvdata->dev); + + return size; +} +static DEVICE_ATTR_WO(reset_tgu); + +static struct attribute *tgu_common_attrs[] = { + &dev_attr_enable_tgu.attr, + &dev_attr_reset_tgu.attr, + NULL, +}; + +static const struct attribute_group tgu_common_grp = { + .attrs = tgu_common_attrs, + NULL, +}; + +static const struct attribute_group *tgu_attr_groups[] = { + &tgu_common_grp, + PRIORITY_ATTRIBUTE_GROUP_INIT(0, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(0, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(0, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(0, 3), + PRIORITY_ATTRIBUTE_GROUP_INIT(1, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(1, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(1, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(1, 3), + PRIORITY_ATTRIBUTE_GROUP_INIT(2, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(2, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(2, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(2, 3), + PRIORITY_ATTRIBUTE_GROUP_INIT(3, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(3, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(3, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(3, 3), + PRIORITY_ATTRIBUTE_GROUP_INIT(4, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(4, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(4, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(4, 3), + PRIORITY_ATTRIBUTE_GROUP_INIT(5, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(5, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(5, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(5, 3), + PRIORITY_ATTRIBUTE_GROUP_INIT(6, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(6, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(6, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(6, 3), + PRIORITY_ATTRIBUTE_GROUP_INIT(7, 0), + PRIORITY_ATTRIBUTE_GROUP_INIT(7, 1), + PRIORITY_ATTRIBUTE_GROUP_INIT(7, 2), + PRIORITY_ATTRIBUTE_GROUP_INIT(7, 3), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(0), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(1), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(2), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(3), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(4), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(5), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(6), + CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(7), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(0), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(1), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(2), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(3), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(4), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(5), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(6), + CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(7), + TIMER_ATTRIBUTE_GROUP_INIT(0), + TIMER_ATTRIBUTE_GROUP_INIT(1), + TIMER_ATTRIBUTE_GROUP_INIT(2), + TIMER_ATTRIBUTE_GROUP_INIT(3), + TIMER_ATTRIBUTE_GROUP_INIT(4), + TIMER_ATTRIBUTE_GROUP_INIT(5), + TIMER_ATTRIBUTE_GROUP_INIT(6), + TIMER_ATTRIBUTE_GROUP_INIT(7), + COUNTER_ATTRIBUTE_GROUP_INIT(0), + COUNTER_ATTRIBUTE_GROUP_INIT(1), + COUNTER_ATTRIBUTE_GROUP_INIT(2), + COUNTER_ATTRIBUTE_GROUP_INIT(3), + COUNTER_ATTRIBUTE_GROUP_INIT(4), + COUNTER_ATTRIBUTE_GROUP_INIT(5), + COUNTER_ATTRIBUTE_GROUP_INIT(6), + COUNTER_ATTRIBUTE_GROUP_INIT(7), + NULL, +}; + +static int tgu_probe(struct amba_device *adev, const struct amba_id *id) +{ + struct device *dev = &adev->dev; + struct tgu_drvdata *drvdata; + unsigned int *priority, *condition, *select, *timer, *counter; + size_t priority_size, condition_size, select_size, timer_size, counter_size; + int ret; + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + drvdata->dev = &adev->dev; + dev_set_drvdata(dev, drvdata); + + drvdata->base = devm_ioremap_resource(dev, &adev->res); + if (IS_ERR(drvdata->base)) + return PTR_ERR(drvdata->base); + + spin_lock_init(&drvdata->lock); + + tgu_set_reg_number(drvdata); + tgu_set_steps(drvdata); + tgu_set_conditions(drvdata); + tgu_set_timer_counter(drvdata); + + ret = sysfs_create_groups(&dev->kobj, tgu_attr_groups); + if (ret) { + dev_err(dev, "failed to create sysfs groups: %d\n", ret); + return ret; + } + + drvdata->value_table = + devm_kzalloc(dev, sizeof(*drvdata->value_table), GFP_KERNEL); + if (!drvdata->value_table) + return -ENOMEM; + + priority_size = MAX_PRIORITY * drvdata->num_reg * drvdata->num_step; + + priority = devm_kcalloc(dev, priority_size, + sizeof(*drvdata->value_table->priority), + GFP_KERNEL); + if (!priority) + return -ENOMEM; + + drvdata->value_table->priority = priority; + + condition_size = drvdata->num_condition_decode * drvdata->num_step; + + condition = devm_kcalloc(dev, condition_size, + sizeof(*(drvdata->value_table->condition_decode)), + GFP_KERNEL); + if (!condition) + return -ENOMEM; + + drvdata->value_table->condition_decode = condition; + + select_size = drvdata->num_condition_select * drvdata->num_step; + + select = devm_kcalloc(dev, select_size, + sizeof(*(drvdata->value_table->condition_select)), + GFP_KERNEL); + if (!select) + return -ENOMEM; + + drvdata->value_table->condition_select = select; + + timer_size = drvdata->num_step * drvdata->num_timer; + + timer = devm_kcalloc(dev, timer_size, + sizeof(*(drvdata->value_table->timer)), + GFP_KERNEL); + if (!timer) + return -ENOMEM; + + drvdata->value_table->timer = timer; + + counter_size = drvdata->num_step * drvdata->num_counter; + + counter = devm_kcalloc(dev, counter_size, + sizeof(*(drvdata->value_table->counter)), + GFP_KERNEL); + if (!counter) + return -ENOMEM; + + drvdata->value_table->counter = counter; + + drvdata->enabled = false; + + pm_runtime_put(&adev->dev); + + return 0; +} + +static void tgu_remove(struct amba_device *adev) +{ + struct device *dev = &adev->dev; + + sysfs_remove_groups(&dev->kobj, tgu_attr_groups); + + tgu_disable(dev); +} + +static const struct amba_id tgu_ids[] = { + { + .id = 0x000f0e00, + .mask = 0x000fffff, + }, + { 0, 0, NULL }, +}; + +MODULE_DEVICE_TABLE(amba, tgu_ids); + +static struct amba_driver tgu_driver = { + .drv = { + .name = "qcom-tgu", + .suppress_bind_attrs = true, + }, + .probe = tgu_probe, + .remove = tgu_remove, + .id_table = tgu_ids, +}; + +module_amba_driver(tgu_driver); + +MODULE_AUTHOR("Songwei Chai "); +MODULE_AUTHOR("Jinlong Mao "); +MODULE_DESCRIPTION("Qualcomm Trigger Generation Unit driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwtracing/qcom/tgu.h b/drivers/hwtracing/qcom/tgu.h new file mode 100644 index 0000000000000..1bcbc99169def --- /dev/null +++ b/drivers/hwtracing/qcom/tgu.h @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _QCOM_TGU_H +#define _QCOM_TGU_H + +/* Register addresses */ +#define TGU_CONTROL 0x0000 +#define TGU_LAR 0xfb0 +#define TGU_UNLOCK_OFFSET 0xc5acce55 +#define TGU_DEVID 0xfc8 +#define CORESIGHT_DEVID2 0xfc0 + +#define TGU_DEVID_SENSE_INPUT(devid_val) \ + ((int)FIELD_GET(GENMASK(17, 10), devid_val)) +#define TGU_DEVID_STEPS(devid_val) \ + ((int)FIELD_GET(GENMASK(6, 3), devid_val)) +#define TGU_DEVID_CONDITIONS(devid_val) \ + ((int)FIELD_GET(GENMASK(2, 0), devid_val)) +#define TGU_DEVID2_TIMER0(devid_val) \ + ((int)FIELD_GET(GENMASK(23, 18), devid_val)) +#define TGU_DEVID2_TIMER1(devid_val) \ + ((int)FIELD_GET(GENMASK(17, 13), devid_val)) +#define TGU_DEVID2_COUNTER0(devid_val) \ + ((int)FIELD_GET(GENMASK(11, 6), devid_val)) +#define TGU_DEVID2_COUNTER1(devid_val) \ + ((int)FIELD_GET(GENMASK(5, 0), devid_val)) + + +#define TGU_BITS_PER_SIGNAL 4 +#define LENGTH_REGISTER 32 + +/* + * TGU configuration space Step configuration + * offset table space layout + * x-------------------------x$ x-------------x$ + * | |$ | |$ + * | | | reserve |$ + * | | | |$ + * |coresight management | |-------------|base+n*0x1D8+0x1F4$ + * | registers | |---> |priority[3] |$ + * | | | |-------------|base+n*0x1D8+0x194$ + * | | | |priority[2] |$ + * |-------------------------| | |-------------|base+n*0x1D8+0x134$ + * | | | |priority[1] |$ + * | step[7] | | |-------------|base+n*0x1D8+0xD4$ + * |-------------------------|->base+0x40+7*0x1D8 | |priority[0] |$ + * | | | |-------------|base+n*0x1D8+0x74$ + * | ... | | | condition |$ + * | | | | select |$ + * |-------------------------|->base+0x40+1*0x1D8 | |-------------|base+n*0x1D8+0x60$ + * | | | | condition |$ + * | step[0] |--------------------> | decode |$ + * |-------------------------|-> base+0x40 |-------------|base+n*0x1D8+0x50$ + * | | | |$ + * | Control and status space| |Timer/Counter|$ + * | space | | |$ + * x-------------------------x->base x-------------x base+n*0x1D8+0x40$ + * + */ +#define STEP_OFFSET 0x1D8 +#define PRIORITY_START_OFFSET 0x0074 +#define CONDITION_DECODE_OFFSET 0x0050 +#define CONDITION_SELECT_OFFSET 0x0060 +#define TIMER_START_OFFSET 0x0040 +#define COUNTER_START_OFFSET 0x0048 +#define PRIORITY_OFFSET 0x60 +#define REG_OFFSET 0x4 + +/* Calculate compare step addresses */ +#define PRIORITY_REG_STEP(step, priority, reg)\ + (PRIORITY_START_OFFSET + PRIORITY_OFFSET * priority +\ + REG_OFFSET * reg + STEP_OFFSET * step) + +#define CONDITION_DECODE_STEP(step, decode) \ + (CONDITION_DECODE_OFFSET + REG_OFFSET * decode + STEP_OFFSET * step) + +#define CONDITION_SELECT_STEP(step, select) \ + (CONDITION_SELECT_OFFSET + REG_OFFSET * select + STEP_OFFSET * step) + +#define TIMER_COMPARE_STEP(step, timer) \ + (TIMER_START_OFFSET + REG_OFFSET * timer + STEP_OFFSET * step) + +#define COUNTER_COMPARE_STEP(step, counter) \ + (COUNTER_START_OFFSET + REG_OFFSET * counter + STEP_OFFSET * step) + +#define tgu_dataset_rw(name, step_index, type, reg_num) \ + (&((struct tgu_attribute[]){ { \ + __ATTR(name, 0644, tgu_dataset_show, tgu_dataset_store), \ + step_index, \ + type, \ + reg_num, \ + } })[0].attr.attr) + +#define STEP_PRIORITY(step_index, reg_num, priority) \ + tgu_dataset_rw(reg##reg_num, step_index, TGU_PRIORITY##priority, \ + reg_num) +#define STEP_DECODE(step_index, reg_num) \ + tgu_dataset_rw(reg##reg_num, step_index, TGU_CONDITION_DECODE, reg_num) +#define STEP_SELECT(step_index, reg_num) \ + tgu_dataset_rw(reg##reg_num, step_index, TGU_CONDITION_SELECT, reg_num) +#define STEP_TIMER(step_index, reg_num) \ + tgu_dataset_rw(reg##reg_num, step_index, TGU_TIMER, reg_num) +#define STEP_COUNTER(step_index, reg_num) \ + tgu_dataset_rw(reg##reg_num, step_index, TGU_COUNTER, reg_num) + +#define STEP_PRIORITY_LIST(step_index, priority) \ + {STEP_PRIORITY(step_index, 0, priority), \ + STEP_PRIORITY(step_index, 1, priority), \ + STEP_PRIORITY(step_index, 2, priority), \ + STEP_PRIORITY(step_index, 3, priority), \ + STEP_PRIORITY(step_index, 4, priority), \ + STEP_PRIORITY(step_index, 5, priority), \ + STEP_PRIORITY(step_index, 6, priority), \ + STEP_PRIORITY(step_index, 7, priority), \ + STEP_PRIORITY(step_index, 8, priority), \ + STEP_PRIORITY(step_index, 9, priority), \ + STEP_PRIORITY(step_index, 10, priority), \ + STEP_PRIORITY(step_index, 11, priority), \ + STEP_PRIORITY(step_index, 12, priority), \ + STEP_PRIORITY(step_index, 13, priority), \ + STEP_PRIORITY(step_index, 14, priority), \ + STEP_PRIORITY(step_index, 15, priority), \ + STEP_PRIORITY(step_index, 16, priority), \ + STEP_PRIORITY(step_index, 17, priority), \ + NULL \ + } + +#define STEP_DECODE_LIST(n) \ + {STEP_DECODE(n, 0), \ + STEP_DECODE(n, 1), \ + STEP_DECODE(n, 2), \ + STEP_DECODE(n, 3), \ + NULL \ + } + +#define STEP_SELECT_LIST(n) \ + {STEP_SELECT(n, 0), \ + STEP_SELECT(n, 1), \ + STEP_SELECT(n, 2), \ + STEP_SELECT(n, 3), \ + STEP_SELECT(n, 4), \ + NULL \ + } + +#define STEP_TIMER_LIST(n) \ + {STEP_TIMER(n, 0), \ + STEP_TIMER(n, 1), \ + NULL \ + } + +#define STEP_COUNTER_LIST(n) \ + {STEP_COUNTER(n, 0), \ + STEP_COUNTER(n, 1), \ + NULL \ + } + +#define PRIORITY_ATTRIBUTE_GROUP_INIT(step, priority)\ + (&(const struct attribute_group){\ + .attrs = (struct attribute*[])STEP_PRIORITY_LIST(step, priority),\ + .is_visible = tgu_node_visible,\ + .name = "step" #step "_priority" #priority \ + }) + +#define CONDITION_DECODE_ATTRIBUTE_GROUP_INIT(step)\ + (&(const struct attribute_group){\ + .attrs = (struct attribute*[])STEP_DECODE_LIST(step),\ + .is_visible = tgu_node_visible,\ + .name = "step" #step "_condition_decode" \ + }) + +#define CONDITION_SELECT_ATTRIBUTE_GROUP_INIT(step)\ + (&(const struct attribute_group){\ + .attrs = (struct attribute*[])STEP_SELECT_LIST(step),\ + .is_visible = tgu_node_visible,\ + .name = "step" #step "_condition_select" \ + }) + +#define TIMER_ATTRIBUTE_GROUP_INIT(step)\ + (&(const struct attribute_group){\ + .attrs = (struct attribute*[])STEP_TIMER_LIST(step),\ + .is_visible = tgu_node_visible,\ + .name = "step" #step "_timer" \ + }) + +#define COUNTER_ATTRIBUTE_GROUP_INIT(step)\ + (&(const struct attribute_group){\ + .attrs = (struct attribute*[])STEP_COUNTER_LIST(step),\ + .is_visible = tgu_node_visible,\ + .name = "step" #step "_counter" \ + }) + +enum operation_index { + TGU_PRIORITY0, + TGU_PRIORITY1, + TGU_PRIORITY2, + TGU_PRIORITY3, + TGU_CONDITION_DECODE, + TGU_CONDITION_SELECT, + TGU_TIMER, + TGU_COUNTER, +}; + +/* Maximum priority that TGU supports */ +#define MAX_PRIORITY 4 + +struct tgu_attribute { + struct device_attribute attr; + u32 step_index; + enum operation_index operation_index; + u32 reg_num; +}; + +struct value_table { + unsigned int *priority; + unsigned int *condition_decode; + unsigned int *condition_select; + unsigned int *timer; + unsigned int *counter; +}; + +static inline void TGU_LOCK(void __iomem *addr) +{ + do { + /* Wait for things to settle */ + mb(); + writel_relaxed(0x0, addr + TGU_LAR); + } while (0); +} + +static inline void TGU_UNLOCK(void __iomem *addr) +{ + do { + writel_relaxed(TGU_UNLOCK_OFFSET, addr + TGU_LAR); + /* Make sure everyone has seen this */ + mb(); + } while (0); +} + +/** + * struct tgu_drvdata - Data structure for a TGU (Trigger Generator Unit) + * @base: Memory-mapped base address of the TGU device + * @dev: Pointer to the associated device structure + * @lock: Spinlock for handling concurrent access to private data + * @enabled: Flag indicating whether the TGU device is enabled + * @value_table: Store given value based on relevant parameters + * @num_reg: Maximum number of registers + * @num_step: Maximum step size + * @num_condition_decode: Maximum number of condition_decode + * @num_condition_select: Maximum number of condition_select + * @num_timer: Maximum number of timers + * @num_counter: Maximum number of counters + * + * This structure defines the data associated with a TGU device, + * including its base address, device pointers, clock, spinlock for + * synchronization, trigger data pointers, maximum limits for various + * trigger-related parameters, and enable status. + */ +struct tgu_drvdata { + void __iomem *base; + struct device *dev; + spinlock_t lock; + bool enabled; + struct value_table *value_table; + int num_reg; + int num_step; + int num_condition_decode; + int num_condition_select; + int num_timer; + int num_counter; +}; + +#endif diff --git a/drivers/hwtracing/stm/Kconfig b/drivers/hwtracing/stm/Kconfig index eda6b11d40a1f..daa4aa09f64dc 100644 --- a/drivers/hwtracing/stm/Kconfig +++ b/drivers/hwtracing/stm/Kconfig @@ -40,6 +40,20 @@ config STM_PROTO_SYS_T If you don't know what this is, say N. +config STM_PROTO_OST + tristate "MIPI OST STM framing protocol driver" + default CONFIG_STM + help + This is an implementation of MIPI OST protocol to be used + over the STP transport. In addition to the data payload, it + also carries additional metadata for entity, better + means of trace source identification, etc. + + The receiving side must be able to decode this protocol in + addition to the MIPI STP, in order to extract the data. + + If you don't know what this is, say N. + config STM_DUMMY tristate "Dummy STM driver" help diff --git a/drivers/hwtracing/stm/Makefile b/drivers/hwtracing/stm/Makefile index 1692fcd292779..d9c8615849b95 100644 --- a/drivers/hwtracing/stm/Makefile +++ b/drivers/hwtracing/stm/Makefile @@ -5,9 +5,11 @@ stm_core-y := core.o policy.o obj-$(CONFIG_STM_PROTO_BASIC) += stm_p_basic.o obj-$(CONFIG_STM_PROTO_SYS_T) += stm_p_sys-t.o +obj-$(CONFIG_STM_PROTO_OST) += stm_p_ost.o stm_p_basic-y := p_basic.o stm_p_sys-t-y := p_sys-t.o +stm_p_ost-y := p_ost.o obj-$(CONFIG_STM_DUMMY) += dummy_stm.o diff --git a/drivers/hwtracing/stm/p_ost.c b/drivers/hwtracing/stm/p_ost.c new file mode 100644 index 0000000000000..ac9492ed5c2c0 --- /dev/null +++ b/drivers/hwtracing/stm/p_ost.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + * + * MIPI OST framing protocol for STM devices. + */ + +#include +#include +#include +#include +#include "stm.h" + +/* + * OST Base Protocol Header + * + * Position Bits Field Name + * 0 8 STARTSIMPLE + * 1 8 Version + * 2 8 Entity ID + * 3 8 protocol ID + */ +#define OST_FIELD_STARTSIMPLE 0 +#define OST_FIELD_VERSION 8 +#define OST_FIELD_ENTITY 16 +#define OST_FIELD_PROTOCOL 24 + +#define OST_TOKEN_STARTSIMPLE 0x10 +#define OST_VERSION_MIPI1 0x10 + +/* entity id to identify the source*/ +#define OST_ENTITY_FTRACE 0x01 +#define OST_ENTITY_CONSOLE 0x02 +#define OST_ENTITY_DIAG 0xEE + +#define OST_CONTROL_PROTOCOL 0x0 + +#define DATA_HEADER ((OST_TOKEN_STARTSIMPLE << OST_FIELD_STARTSIMPLE) | \ + (OST_VERSION_MIPI1 << OST_FIELD_PROTOCOL) | \ + (OST_CONTROL_PROTOCOL << OST_FIELD_PROTOCOL)) + +#define STM_MAKE_VERSION(ma, mi) (((ma) << 8) | (mi)) +#define STM_HEADER_MAGIC (0x5953) + +enum ost_entity_type { + OST_ENTITY_TYPE_NONE, + OST_ENTITY_TYPE_FTRACE, + OST_ENTITY_TYPE_CONSOLE, + OST_ENTITY_TYPE_DIAG, +}; + +static const char * const str_ost_entity_type[] = { + [OST_ENTITY_TYPE_NONE] = "none", + [OST_ENTITY_TYPE_FTRACE] = "ftrace", + [OST_ENTITY_TYPE_CONSOLE] = "console", + [OST_ENTITY_TYPE_DIAG] = "diag", +}; + +static const u32 ost_entity_value[] = { + [OST_ENTITY_TYPE_NONE] = 0, + [OST_ENTITY_TYPE_FTRACE] = OST_ENTITY_FTRACE, + [OST_ENTITY_TYPE_CONSOLE] = OST_ENTITY_CONSOLE, + [OST_ENTITY_TYPE_DIAG] = OST_ENTITY_DIAG, +}; + +struct ost_policy_node { + enum ost_entity_type entity_type; +}; + +struct ost_output { + struct ost_policy_node node; +}; + +/* Set default entity type as none */ +static void ost_policy_node_init(void *priv) +{ + struct ost_policy_node *pn = priv; + + pn->entity_type = OST_ENTITY_TYPE_NONE; +} + +static int ost_output_open(void *priv, struct stm_output *output) +{ + struct ost_policy_node *pn = priv; + struct ost_output *opriv; + + opriv = kzalloc(sizeof(*opriv), GFP_ATOMIC); + if (!opriv) + return -ENOMEM; + + memcpy(&opriv->node, pn, sizeof(opriv->node)); + output->pdrv_private = opriv; + return 0; +} + +static void ost_output_close(struct stm_output *output) +{ + kfree(output->pdrv_private); +} + +static ssize_t ost_t_policy_entity_show(struct config_item *item, + char *page) +{ + struct ost_policy_node *pn = to_pdrv_policy_node(item); + ssize_t sz = 0; + int i; + + for (i = 1; i < ARRAY_SIZE(str_ost_entity_type); i++) { + if (i == pn->entity_type) + sz += sysfs_emit_at(page, sz, "[%s] ", str_ost_entity_type[i]); + else + sz += sysfs_emit_at(page, sz, "%s ", str_ost_entity_type[i]); + } + + sz += sysfs_emit_at(page, sz, "\n"); + return sz; +} + +static int entity_index(const char *str) +{ + int i; + + for (i = 1; i < ARRAY_SIZE(str_ost_entity_type); i++) { + if (sysfs_streq(str, str_ost_entity_type[i])) + return i; + } + + return 0; +} + +static ssize_t +ost_t_policy_entity_store(struct config_item *item, const char *page, + size_t count) +{ + struct ost_policy_node *pn = to_pdrv_policy_node(item); + int i; + + i = entity_index(page); + if (i) + pn->entity_type = i; + else + return -EINVAL; + + return count; +} +CONFIGFS_ATTR(ost_t_policy_, entity); + +static struct configfs_attribute *ost_t_policy_attrs[] = { + &ost_t_policy_attr_entity, + NULL, +}; + +static ssize_t +notrace ost_write(struct stm_data *data, struct stm_output *output, + unsigned int chan, const char *buf, size_t count, + struct stm_source_data *source) +{ + struct ost_output *op = output->pdrv_private; + unsigned int c = output->channel + chan; + unsigned int m = output->master; + const unsigned char nil = 0; + u32 header = DATA_HEADER; + struct trc_hdr { + u16 version; + u16 magic; + u32 cpu; + u64 timestamp; + u64 tgid; + } hdr; + ssize_t sz; + + /* + * Identify the source by entity type. + * If entity type is not set, return error value. + */ + if (op->node.entity_type) + header |= ost_entity_value[op->node.entity_type]; + else + return -EINVAL; + + /* + * STP framing rules for OST frames: + * * the first packet of the OST frame is marked; + * * the last packet is a FLAG with timestamped tag. + */ + /* Message layout: HEADER / DATA / TAIL */ + /* HEADER */ + sz = data->packet(data, m, c, STP_PACKET_DATA, STP_PACKET_MARKED, + 4, (u8 *)&header); + if (sz <= 0) + return sz; + + /* DATA */ + hdr.version = STM_MAKE_VERSION(0, 3); + hdr.magic = STM_HEADER_MAGIC; + hdr.cpu = raw_smp_processor_id(); + hdr.timestamp = sched_clock(); + hdr.tgid = task_tgid_nr(current); + sz = stm_data_write(data, m, c, false, &hdr, sizeof(hdr)); + if (sz <= 0) + return sz; + + sz = stm_data_write(data, m, c, false, buf, count); + + /* TAIL */ + if (sz > 0) + data->packet(data, m, c, STP_PACKET_FLAG, + STP_PACKET_TIMESTAMPED, 0, &nil); + + return sz; +} + +static const struct stm_protocol_driver ost_pdrv = { + .owner = THIS_MODULE, + .name = "p_ost", + .write = ost_write, + .policy_attr = ost_t_policy_attrs, + .output_open = ost_output_open, + .output_close = ost_output_close, + .policy_node_init = ost_policy_node_init, +}; + +static int ost_stm_init(void) +{ + return stm_register_protocol(&ost_pdrv); +} +module_init(ost_stm_init); + +static void ost_stm_exit(void) +{ + stm_unregister_protocol(&ost_pdrv); +} +module_exit(ost_stm_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MIPI Open System Trace STM framing protocol driver"); diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index fd81e49638aaa..4515ded4338c5 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -1218,6 +1218,8 @@ config I2C_TEGRA tristate "NVIDIA Tegra internal I2C controller" depends on ARCH_TEGRA || (COMPILE_TEST && (ARC || ARM || ARM64 || M68K || RISCV || SUPERH || SPARC)) # COMPILE_TEST needs architectures with readsX()/writesX() primitives + depends on PINCTRL + # ARCH_TEGRA implies PINCTRL, but the COMPILE_TEST side doesn't. help If you say yes to this option, support will be included for the I2C controller embedded in NVIDIA Tegra SOCs diff --git a/drivers/i2c/busses/i2c-amd-mp2-pci.c b/drivers/i2c/busses/i2c-amd-mp2-pci.c index ef7370d3dbea3..60edbabc2986b 100644 --- a/drivers/i2c/busses/i2c-amd-mp2-pci.c +++ b/drivers/i2c/busses/i2c-amd-mp2-pci.c @@ -458,13 +458,16 @@ struct amd_mp2_dev *amd_mp2_find_device(void) { struct device *dev; struct pci_dev *pci_dev; + struct amd_mp2_dev *mp2_dev; dev = driver_find_next_device(&amd_mp2_pci_driver.driver, NULL); if (!dev) return NULL; pci_dev = to_pci_dev(dev); - return (struct amd_mp2_dev *)pci_get_drvdata(pci_dev); + mp2_dev = (struct amd_mp2_dev *)pci_get_drvdata(pci_dev); + put_device(dev); + return mp2_dev; } EXPORT_SYMBOL_GPL(amd_mp2_find_device); diff --git a/drivers/i2c/busses/i2c-cp2615.c b/drivers/i2c/busses/i2c-cp2615.c index e7720ea4045eb..7b62ba115eb9f 100644 --- a/drivers/i2c/busses/i2c-cp2615.c +++ b/drivers/i2c/busses/i2c-cp2615.c @@ -298,6 +298,9 @@ cp2615_i2c_probe(struct usb_interface *usbif, const struct usb_device_id *id) if (!adap) return -ENOMEM; + if (!usbdev->serial) + return -EINVAL; + strscpy(adap->name, usbdev->serial, sizeof(adap->name)); adap->owner = THIS_MODULE; adap->dev.parent = &usbif->dev; diff --git a/drivers/i2c/busses/i2c-designware-amdisp.c b/drivers/i2c/busses/i2c-designware-amdisp.c index 450793d5f8392..e0c3669bab08e 100644 --- a/drivers/i2c/busses/i2c-designware-amdisp.c +++ b/drivers/i2c/busses/i2c-designware-amdisp.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -82,22 +83,20 @@ static int amd_isp_dw_i2c_plat_probe(struct platform_device *pdev) if (isp_i2c_dev->shared_with_punit) pm_runtime_get_noresume(&pdev->dev); - pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); - + dev_pm_genpd_resume(&pdev->dev); ret = i2c_dw_probe(isp_i2c_dev); if (ret) { dev_err_probe(&pdev->dev, ret, "i2c_dw_probe failed\n"); goto error_release_rpm; } - - pm_runtime_put_sync(&pdev->dev); + dev_pm_genpd_suspend(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_enable(&pdev->dev); return 0; error_release_rpm: amd_isp_dw_i2c_plat_pm_cleanup(isp_i2c_dev); - pm_runtime_put_sync(&pdev->dev); return ret; } diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index 347843b4f5dd7..436555543c79d 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -78,6 +78,7 @@ #define DW_IC_TX_ABRT_SOURCE 0x80 #define DW_IC_ENABLE_STATUS 0x9c #define DW_IC_CLR_RESTART_DET 0xa8 +#define DW_IC_SMBUS_INTR_MASK 0xcc #define DW_IC_COMP_PARAM_1 0xf4 #define DW_IC_COMP_VERSION 0xf8 #define DW_IC_SDA_HOLD_MIN_VERS 0x3131312A /* "111*" == v1.11* */ diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index 41e9b5ecad201..45bfca05bb300 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -220,6 +220,13 @@ static int i2c_dw_init_master(struct dw_i2c_dev *dev) /* Disable the adapter */ __i2c_dw_disable(dev); + /* + * Mask SMBus interrupts to block storms from broken + * firmware that leaves IC_SMBUS=1; the handler never + * services them. + */ + regmap_write(dev->map, DW_IC_SMBUS_INTR_MASK, 0); + /* Write standard speed timing parameters */ regmap_write(dev->map, DW_IC_SS_SCL_HCNT, dev->ss_hcnt); regmap_write(dev->map, DW_IC_SS_SCL_LCNT, dev->ss_lcnt); diff --git a/drivers/i2c/busses/i2c-fsi.c b/drivers/i2c/busses/i2c-fsi.c index ae016a9431dac..6a9245423d2b5 100644 --- a/drivers/i2c/busses/i2c-fsi.c +++ b/drivers/i2c/busses/i2c-fsi.c @@ -728,6 +728,7 @@ static int fsi_i2c_probe(struct device *dev) rc = i2c_add_adapter(&port->adapter); if (rc < 0) { dev_err(dev, "Failed to register adapter: %d\n", rc); + of_node_put(np); kfree(port); continue; } diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 57fbec1259bea..506d69b156f7c 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -306,9 +306,10 @@ struct i801_priv { /* * If set to true the host controller registers are reserved for - * ACPI AML use. + * ACPI AML use. Needs extra protection by acpi_lock. */ bool acpi_reserved; + struct mutex acpi_lock; }; #define FEATURE_SMBUS_PEC BIT(0) @@ -890,8 +891,11 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, int hwpec, ret; struct i801_priv *priv = i2c_get_adapdata(adap); - if (priv->acpi_reserved) + mutex_lock(&priv->acpi_lock); + if (priv->acpi_reserved) { + mutex_unlock(&priv->acpi_lock); return -EBUSY; + } pm_runtime_get_sync(&priv->pci_dev->dev); @@ -931,6 +935,7 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, iowrite8(SMBHSTSTS_INUSE_STS | STATUS_FLAGS, SMBHSTSTS(priv)); pm_runtime_put_autosuspend(&priv->pci_dev->dev); + mutex_unlock(&priv->acpi_lock); return ret; } @@ -1459,7 +1464,7 @@ i801_acpi_io_handler(u32 function, acpi_physical_address address, u32 bits, * further access from the driver itself. This device is now owned * by the system firmware. */ - i2c_lock_bus(&priv->adapter, I2C_LOCK_SEGMENT); + mutex_lock(&priv->acpi_lock); if (!priv->acpi_reserved && i801_acpi_is_smbus_ioport(priv, address)) { priv->acpi_reserved = true; @@ -1479,7 +1484,7 @@ i801_acpi_io_handler(u32 function, acpi_physical_address address, u32 bits, else status = acpi_os_write_port(address, (u32)*value, bits); - i2c_unlock_bus(&priv->adapter, I2C_LOCK_SEGMENT); + mutex_unlock(&priv->acpi_lock); return status; } @@ -1539,6 +1544,7 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id) priv->adapter.dev.parent = &dev->dev; acpi_use_parent_companion(&priv->adapter.dev); priv->adapter.retries = 3; + mutex_init(&priv->acpi_lock); priv->pci_dev = dev; priv->features = id->driver_data; diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index 2a0962a0b4417..519a1ac832a41 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -5,6 +5,7 @@ * Copyright 2016 Freescale Semiconductor, Inc. */ +#include #include #include #include @@ -90,6 +91,7 @@ #define MRDR_RXEMPTY BIT(14) #define MDER_TDDE BIT(0) #define MDER_RDDE BIT(1) +#define MSR_RDF_ASSERTED(x) FIELD_GET(MSR_RDF, (x)) #define SCR_SEN BIT(0) #define SCR_RST BIT(1) @@ -461,7 +463,7 @@ static bool lpi2c_imx_write_txfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atom static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomic) { - unsigned int blocklen, remaining; + unsigned int remaining; unsigned int temp, data; do { @@ -472,15 +474,6 @@ static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomi lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff; } while (1); - /* - * First byte is the length of remaining packet in the SMBus block - * data read. Add it to msgs->len. - */ - if (lpi2c_imx->block_data) { - blocklen = lpi2c_imx->rx_buf[0]; - lpi2c_imx->msglen += blocklen; - } - remaining = lpi2c_imx->msglen - lpi2c_imx->delivered; if (!remaining) { @@ -493,12 +486,7 @@ static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomi lpi2c_imx_set_rx_watermark(lpi2c_imx); /* multiple receive commands */ - if (lpi2c_imx->block_data) { - lpi2c_imx->block_data = 0; - temp = remaining; - temp |= (RECV_DATA << 8); - writel(temp, lpi2c_imx->base + LPI2C_MTDR); - } else if (!(lpi2c_imx->delivered & 0xff)) { + if (!(lpi2c_imx->delivered & 0xff)) { temp = (remaining > CHUNK_DATA ? CHUNK_DATA : remaining) - 1; temp |= (RECV_DATA << 8); writel(temp, lpi2c_imx->base + LPI2C_MTDR); @@ -536,18 +524,77 @@ static int lpi2c_imx_write_atomic(struct lpi2c_imx_struct *lpi2c_imx, return err; } -static void lpi2c_imx_read_init(struct lpi2c_imx_struct *lpi2c_imx, - struct i2c_msg *msgs) +static unsigned int lpi2c_SMBus_block_read_length_byte(struct lpi2c_imx_struct *lpi2c_imx) { - unsigned int temp; + unsigned int data; + + data = readl(lpi2c_imx->base + LPI2C_MRDR); + lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff; + + return data; +} + +static int lpi2c_imx_read_init(struct lpi2c_imx_struct *lpi2c_imx, + struct i2c_msg *msgs) +{ + unsigned int temp, val, block_len; + int ret; lpi2c_imx->rx_buf = msgs->buf; lpi2c_imx->block_data = msgs->flags & I2C_M_RECV_LEN; lpi2c_imx_set_rx_watermark(lpi2c_imx); - temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1; - temp |= (RECV_DATA << 8); - writel(temp, lpi2c_imx->base + LPI2C_MTDR); + + if (!lpi2c_imx->block_data) { + temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1; + temp |= (RECV_DATA << 8); + writel(temp, lpi2c_imx->base + LPI2C_MTDR); + } else { + /* + * The LPI2C controller automatically sends a NACK after the last byte of a + * receive command, unless the next command in MTDR is also a receive command. + * If MTDR is empty when a receive completes, a NACK is sent by default. + * + * To comply with the SMBus block read spec, we start with a 2-byte read: + * The first byte in RXFIFO is the block length. Once this byte arrives, the + * controller immediately updates MTDR with the next read command, ensuring + * continuous ACK instead of NACK. + * + * The second byte is the first block data byte. Therefore, the subsequent + * read command should request (block_len - 1) bytes, since one data byte + * has already been read. + */ + + writel((RECV_DATA << 8) | 0x01, lpi2c_imx->base + LPI2C_MTDR); + + ret = readl_poll_timeout(lpi2c_imx->base + LPI2C_MSR, val, + MSR_RDF_ASSERTED(val), 1, 1000); + if (ret) { + dev_err(&lpi2c_imx->adapter.dev, "SMBus read count failed %d\n", ret); + return ret; + } + + /* Read block length byte and confirm this SMBus transfer meets protocol */ + block_len = lpi2c_SMBus_block_read_length_byte(lpi2c_imx); + if (block_len == 0 || block_len > I2C_SMBUS_BLOCK_MAX) { + dev_err(&lpi2c_imx->adapter.dev, "Invalid SMBus block read length\n"); + return -EPROTO; + } + + /* + * When block_len shows more bytes need to be read, update second read command to + * keep MTDR non-empty and ensuring continuous ACKs. Only update command register + * here. All block bytes will be read out at IRQ handler or lpi2c_imx_read_atomic() + * function. + */ + if (block_len > 1) + writel((RECV_DATA << 8) | (block_len - 2), lpi2c_imx->base + LPI2C_MTDR); + + lpi2c_imx->msglen += block_len; + msgs->len += block_len; + } + + return 0; } static bool lpi2c_imx_read_chunk_atomic(struct lpi2c_imx_struct *lpi2c_imx) @@ -592,6 +639,17 @@ static bool is_use_dma(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) if (!lpi2c_imx->can_use_dma) return false; + /* DMA is not suitable for SMBus block read */ + if (msg->flags & I2C_M_RECV_LEN) + return false; + + /* + * A system-wide suspend or resume transition is in progress. LPI2C should use PIO to + * transfer data to avoid issue caused by no ready DMA HW resource. + */ + if (pm_suspend_in_progress()) + return false; + /* * When the length of data is less than I2C_DMA_THRESHOLD, * cpu mode is used directly to avoid low performance. @@ -602,10 +660,14 @@ static bool is_use_dma(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) static int lpi2c_imx_pio_xfer(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) { + int ret; + reinit_completion(&lpi2c_imx->complete); if (msg->flags & I2C_M_RD) { - lpi2c_imx_read_init(lpi2c_imx, msg); + ret = lpi2c_imx_read_init(lpi2c_imx, msg); + if (ret) + return ret; lpi2c_imx_intctrl(lpi2c_imx, MIER_RDIE | MIER_NDIE); } else { lpi2c_imx_write(lpi2c_imx, msg); @@ -617,8 +679,12 @@ static int lpi2c_imx_pio_xfer(struct lpi2c_imx_struct *lpi2c_imx, static int lpi2c_imx_pio_xfer_atomic(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) { + int ret; + if (msg->flags & I2C_M_RD) { - lpi2c_imx_read_init(lpi2c_imx, msg); + ret = lpi2c_imx_read_init(lpi2c_imx, msg); + if (ret) + return ret; return lpi2c_imx_read_atomic(lpi2c_imx, msg); } diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index dcce882f3ebaa..a208fefd3c3b3 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -401,7 +401,7 @@ static void i2c_imx_reset_regs(struct imx_i2c_struct *i2c_imx) static int i2c_imx_dma_request(struct imx_i2c_struct *i2c_imx, dma_addr_t phy_addr) { struct imx_i2c_dma *dma; - struct dma_slave_config dma_sconfig; + struct dma_slave_config dma_sconfig = {}; struct device *dev = i2c_imx->adapter.dev.parent; int ret; @@ -1018,8 +1018,9 @@ static inline int i2c_imx_isr_read(struct imx_i2c_struct *i2c_imx) return 0; } -static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) +static inline enum imx_i2c_state i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) { + enum imx_i2c_state next_state = IMX_I2C_STATE_READ_CONTINUE; unsigned int temp; if ((i2c_imx->msg->len - 1) == i2c_imx->msg_buf_idx) { @@ -1033,18 +1034,20 @@ static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) i2c_imx->stopped = 1; temp &= ~(I2CR_MSTA | I2CR_MTX); imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); - } else { - /* - * For i2c master receiver repeat restart operation like: - * read -> repeat MSTA -> read/write - * The controller must set MTX before read the last byte in - * the first read operation, otherwise the first read cost - * one extra clock cycle. - */ - temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); - temp |= I2CR_MTX; - imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); + + return IMX_I2C_STATE_DONE; } + /* + * For i2c master receiver repeat restart operation like: + * read -> repeat MSTA -> read/write + * The controller must set MTX before read the last byte in + * the first read operation, otherwise the first read cost + * one extra clock cycle. + */ + temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); + temp |= I2CR_MTX; + imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); + next_state = IMX_I2C_STATE_DONE; } else if (i2c_imx->msg_buf_idx == (i2c_imx->msg->len - 2)) { temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); temp |= I2CR_TXAK; @@ -1052,6 +1055,7 @@ static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) } i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); + return next_state; } static inline void i2c_imx_isr_read_block_data_len(struct imx_i2c_struct *i2c_imx) @@ -1088,11 +1092,9 @@ static irqreturn_t i2c_imx_master_isr(struct imx_i2c_struct *i2c_imx, unsigned i break; case IMX_I2C_STATE_READ_CONTINUE: - i2c_imx_isr_read_continue(i2c_imx); - if (i2c_imx->msg_buf_idx == i2c_imx->msg->len) { - i2c_imx->state = IMX_I2C_STATE_DONE; + i2c_imx->state = i2c_imx_isr_read_continue(i2c_imx); + if (i2c_imx->state == IMX_I2C_STATE_DONE) wake_up(&i2c_imx->queue); - } break; case IMX_I2C_STATE_READ_BLOCK_DATA: @@ -1103,7 +1105,8 @@ static irqreturn_t i2c_imx_master_isr(struct imx_i2c_struct *i2c_imx, unsigned i case IMX_I2C_STATE_READ_BLOCK_DATA_LEN: i2c_imx_isr_read_block_data_len(i2c_imx); - i2c_imx->state = IMX_I2C_STATE_READ_CONTINUE; + if (i2c_imx->state == IMX_I2C_STATE_READ_BLOCK_DATA_LEN) + i2c_imx->state = IMX_I2C_STATE_READ_CONTINUE; break; case IMX_I2C_STATE_WRITE: @@ -1489,6 +1492,7 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bool is_lastmsg) { int block_data = msgs->flags & I2C_M_RECV_LEN; + int ret = 0; dev_dbg(&i2c_imx->adapter.dev, "<%s> write slave address: addr=0x%x\n", @@ -1521,10 +1525,20 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, dev_err(&i2c_imx->adapter.dev, "<%s> read timedout\n", __func__); return -ETIMEDOUT; } - if (!i2c_imx->stopped) - return i2c_imx_bus_busy(i2c_imx, 0, false); + if (i2c_imx->is_lastmsg) { + if (!i2c_imx->stopped) + ret = i2c_imx_bus_busy(i2c_imx, 0, false); + /* + * Only read the last byte of the last message after the bus is + * not busy. Else the controller generates another clock which + * might confuse devices. + */ + if (!ret) + i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = imx_i2c_read_reg(i2c_imx, + IMX_I2C_I2DR); + } - return 0; + return ret; } static int i2c_imx_xfer_common(struct i2c_adapter *adapter, diff --git a/drivers/i2c/busses/i2c-k1.c b/drivers/i2c/busses/i2c-k1.c index 6b918770e612e..8ef6d5d1927b7 100644 --- a/drivers/i2c/busses/i2c-k1.c +++ b/drivers/i2c/busses/i2c-k1.c @@ -158,11 +158,16 @@ static int spacemit_i2c_handle_err(struct spacemit_i2c_dev *i2c) { dev_dbg(i2c->dev, "i2c error status: 0x%08x\n", i2c->status); - if (i2c->status & (SPACEMIT_SR_BED | SPACEMIT_SR_ALD)) { + /* Arbitration Loss Detected */ + if (i2c->status & SPACEMIT_SR_ALD) { spacemit_i2c_reset(i2c); return -EAGAIN; } + /* Bus Error No ACK/NAK */ + if (i2c->status & SPACEMIT_SR_BED) + spacemit_i2c_reset(i2c); + return i2c->status & SPACEMIT_SR_ACKNAK ? -ENXIO : -EIO; } @@ -224,6 +229,12 @@ static void spacemit_i2c_check_bus_release(struct spacemit_i2c_dev *i2c) } } +static inline void +spacemit_i2c_clear_int_status(struct spacemit_i2c_dev *i2c, u32 mask) +{ + writel(mask & SPACEMIT_I2C_INT_STATUS_MASK, i2c->base + SPACEMIT_ISR); +} + static void spacemit_i2c_init(struct spacemit_i2c_dev *i2c) { u32 val; @@ -267,12 +278,8 @@ static void spacemit_i2c_init(struct spacemit_i2c_dev *i2c) val = readl(i2c->base + SPACEMIT_IRCR); val |= SPACEMIT_RCR_SDA_GLITCH_NOFIX; writel(val, i2c->base + SPACEMIT_IRCR); -} -static inline void -spacemit_i2c_clear_int_status(struct spacemit_i2c_dev *i2c, u32 mask) -{ - writel(mask & SPACEMIT_I2C_INT_STATUS_MASK, i2c->base + SPACEMIT_ISR); + spacemit_i2c_clear_int_status(i2c, SPACEMIT_I2C_INT_STATUS_MASK); } static void spacemit_i2c_start(struct spacemit_i2c_dev *i2c) @@ -559,7 +566,7 @@ static int spacemit_i2c_probe(struct platform_device *pdev) return dev_err_probe(dev, i2c->irq, "failed to get irq resource"); ret = devm_request_irq(i2c->dev, i2c->irq, spacemit_i2c_irq_handler, - IRQF_NO_SUSPEND | IRQF_ONESHOT, dev_name(i2c->dev), i2c); + IRQF_NO_SUSPEND, dev_name(i2c->dev), i2c); if (ret) return dev_err_probe(dev, ret, "failed to request irq"); diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c index 968a8b8794dac..70acf33e1d573 100644 --- a/drivers/i2c/busses/i2c-pxa.c +++ b/drivers/i2c/busses/i2c-pxa.c @@ -268,6 +268,7 @@ struct pxa_i2c { struct pinctrl *pinctrl; struct pinctrl_state *pinctrl_default; struct pinctrl_state *pinctrl_recovery; + bool reset_before_xfer; }; #define _IBMR(i2c) ((i2c)->reg_ibmr) @@ -1144,6 +1145,11 @@ static int i2c_pxa_xfer(struct i2c_adapter *adap, { struct pxa_i2c *i2c = adap->algo_data; + if (i2c->reset_before_xfer) { + i2c_pxa_reset(i2c); + i2c->reset_before_xfer = false; + } + return i2c_pxa_internal_xfer(i2c, msgs, num, i2c_pxa_do_xfer); } @@ -1521,7 +1527,16 @@ static int i2c_pxa_probe(struct platform_device *dev) } } - i2c_pxa_reset(i2c); + /* + * Skip reset on Armada 3700 when recovery is used to avoid + * controller hang due to the pinctrl state changes done by + * the generic recovery initialization code. The reset will + * be performed later, prior to the first transfer. + */ + if (i2c_type == REGS_A3700 && i2c->adap.bus_recovery_info) + i2c->reset_before_xfer = true; + else + i2c_pxa_reset(i2c); ret = i2c_add_numbered_adapter(&i2c->adap); if (ret < 0) diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index 43fdd89b8bebc..c098379743f84 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -97,6 +97,7 @@ struct geni_i2c_dev { dma_addr_t dma_addr; struct dma_chan *tx_c; struct dma_chan *rx_c; + bool no_dma; bool gpi_mode; bool abort_done; }; @@ -425,7 +426,7 @@ static int geni_i2c_rx_one_msg(struct geni_i2c_dev *gi2c, struct i2c_msg *msg, size_t len = msg->len; struct i2c_msg *cur; - dma_buf = i2c_get_dma_safe_msg_buf(msg, 32); + dma_buf = gi2c->no_dma ? NULL : i2c_get_dma_safe_msg_buf(msg, 32); if (dma_buf) geni_se_select_mode(se, GENI_SE_DMA); else @@ -464,7 +465,7 @@ static int geni_i2c_tx_one_msg(struct geni_i2c_dev *gi2c, struct i2c_msg *msg, size_t len = msg->len; struct i2c_msg *cur; - dma_buf = i2c_get_dma_safe_msg_buf(msg, 32); + dma_buf = gi2c->no_dma ? NULL : i2c_get_dma_safe_msg_buf(msg, 32); if (dma_buf) geni_se_select_mode(se, GENI_SE_DMA); else @@ -626,6 +627,14 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i if (i < num - 1) peripheral.stretch = 1; + peripheral.lock_action = GPI_LOCK_NONE; + if (gi2c->se.multi_owner) { + if (i == 0) + peripheral.lock_action = GPI_LOCK_ACQUIRE; + else if (i == num - 1) + peripheral.lock_action = GPI_LOCK_RELEASE; + } + peripheral.addr = msgs[i].addr; ret = geni_i2c_gpi(gi2c, &msgs[i], &config, @@ -813,6 +822,17 @@ static int geni_i2c_probe(struct platform_device *pdev) gi2c->clk_freq_out = I2C_MAX_STANDARD_MODE_FREQ; } + if (of_property_read_bool(pdev->dev.of_node, "qcom,qup-multi-owner")) { + /* + * Multi-owner controller configuration: the controller may be + * used by another system processor. Mark the SE as shared so + * common GENI resource handling can avoid pin state changes + * that would disrupt the other user. + */ + gi2c->se.multi_owner = true; + dev_dbg(&pdev->dev, "I2C controller is shared with another system processor\n"); + } + if (has_acpi_companion(dev)) ACPI_COMPANION_SET(&gi2c->adap.dev, ACPI_COMPANION(dev)); @@ -830,8 +850,14 @@ static int geni_i2c_probe(struct platform_device *pdev) spin_lock_init(&gi2c->lock); platform_set_drvdata(pdev, gi2c); - /* Keep interrupts disabled initially to allow for low-power modes */ - ret = devm_request_irq(dev, gi2c->irq, geni_i2c_irq, IRQF_NO_AUTOEN, + /* + * Keep interrupts disabled initially to allow for low-power modes. + * IRQF_NO_SUSPEND: Keep IRQ enabled during suspend to handle I2C transfers + * in noirq phase (e.g., from PCIe driver's noirq_resume). + * IRQF_EARLY_RESUME: Enable IRQ early during resume sequence. + */ + ret = devm_request_irq(dev, gi2c->irq, geni_i2c_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND | IRQF_EARLY_RESUME, dev_name(dev), gi2c); if (ret) return dev_err_probe(dev, ret, @@ -880,13 +906,17 @@ static int geni_i2c_probe(struct platform_device *pdev) goto err_resources; } - if (desc && desc->no_dma_support) + if (desc && desc->no_dma_support) { fifo_disable = false; - else + gi2c->no_dma = true; + } else { fifo_disable = readl_relaxed(gi2c->se.base + GENI_IF_DISABLE_RO) & FIFO_IF_DISABLE; + } if (fifo_disable) { - /* FIFO is disabled, so we can only use GPI DMA */ + /* FIFO is disabled, so we can only use GPI DMA. + * SE can be shared in GSI mode between subsystems, each SS owns a GPII. + */ gi2c->gpi_mode = true; ret = setup_gpi_dma(gi2c); if (ret) @@ -895,6 +925,10 @@ static int geni_i2c_probe(struct platform_device *pdev) dev_dbg(dev, "Using GPI DMA mode for I2C\n"); } else { gi2c->gpi_mode = false; + + if (gi2c->se.multi_owner) + dev_err_probe(dev, -EINVAL, "I2C sharing not supported in non GSI mode\n"); + tx_depth = geni_se_get_tx_fifo_depth(&gi2c->se); /* I2C Master Hub Serial Elements doesn't have the HW_PARAM_0 register */ @@ -1041,6 +1075,20 @@ static int __maybe_unused geni_i2c_suspend_noirq(struct device *dev) static int __maybe_unused geni_i2c_resume_noirq(struct device *dev) { struct geni_i2c_dev *gi2c = dev_get_drvdata(dev); + int ret = 0; + + /* + * Resume hardware to handle I2C transfers from other drivers' + * noirq_resume callbacks (e.g., PCIe driver). + * pm_runtime_force_resume() properly handles PM state and usage_count. + */ + if (gi2c->suspended) { + ret = pm_runtime_force_resume(dev); + if (ret) { + dev_err(dev, "Failed to resume I2C during noirq: %d\n", ret); + return ret; + } + } i2c_mark_adapter_resumed(&gi2c->adap); return 0; diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c index 3e8f126cb7f74..9e3595b3623e4 100644 --- a/drivers/i2c/busses/i2c-riic.c +++ b/drivers/i2c/busses/i2c-riic.c @@ -670,12 +670,39 @@ static const struct riic_of_data riic_rz_t2h_info = { static int riic_i2c_suspend(struct device *dev) { - struct riic_dev *riic = dev_get_drvdata(dev); - int ret; + /* + * Some I2C devices may need the I2C controller to remain active + * during resume_noirq() or suspend_noirq(). If the controller is + * autosuspended, there is no way to wake it up once runtime PM is + * disabled (in suspend_late()). + * + * During system resume, the I2C controller will be available only + * after runtime PM is re-enabled (in resume_early()). However, this + * may be too late for some devices. + * + * Wake up the controller in the suspend() callback while runtime PM + * is still enabled. The I2C controller will remain available until + * the suspend_noirq() callback (pm_runtime_force_suspend()) is + * called. During resume, the I2C controller can be restored by the + * resume_noirq() callback (pm_runtime_force_resume()). + * + * Finally, the resume() callback re-enables autosuspend, ensuring + * the I2C controller remains available until the system enters + * suspend_noirq() and from resume_noirq(). + */ + return pm_runtime_resume_and_get(dev); +} - ret = pm_runtime_resume_and_get(dev); - if (ret) - return ret; +static int riic_i2c_resume(struct device *dev) +{ + pm_runtime_put_autosuspend(dev); + + return 0; +} + +static int riic_i2c_suspend_noirq(struct device *dev) +{ + struct riic_dev *riic = dev_get_drvdata(dev); i2c_mark_adapter_suspended(&riic->adapter); @@ -683,12 +710,12 @@ static int riic_i2c_suspend(struct device *dev) riic_clear_set_bit(riic, ICCR1_ICE, 0, RIIC_ICCR1); pm_runtime_mark_last_busy(dev); - pm_runtime_put_sync(dev); + pm_runtime_force_suspend(dev); return reset_control_assert(riic->rstc); } -static int riic_i2c_resume(struct device *dev) +static int riic_i2c_resume_noirq(struct device *dev) { struct riic_dev *riic = dev_get_drvdata(dev); int ret; @@ -697,6 +724,10 @@ static int riic_i2c_resume(struct device *dev) if (ret) return ret; + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + ret = riic_init_hw(riic); if (ret) { /* @@ -714,6 +745,7 @@ static int riic_i2c_resume(struct device *dev) } static const struct dev_pm_ops riic_i2c_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(riic_i2c_suspend_noirq, riic_i2c_resume_noirq) SYSTEM_SLEEP_PM_OPS(riic_i2c_suspend, riic_i2c_resume) }; diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index 8138f5ef40f06..15e14a6fe6dce 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -503,8 +503,13 @@ static void i2c_s3c_irq_nextbyte(struct s3c24xx_i2c *i2c, unsigned long iicstat) i2c->msg->buf[i2c->msg_ptr++] = byte; /* Add actual length to read for smbus block read */ - if (i2c->msg->flags & I2C_M_RECV_LEN && i2c->msg->len == 1) + if (i2c->msg->flags & I2C_M_RECV_LEN && i2c->msg->len == 1) { + if (byte == 0 || byte > I2C_SMBUS_BLOCK_MAX) { + s3c24xx_i2c_stop(i2c, -EPROTO); + break; + } i2c->msg->len += byte; + } prepare_read: if (is_msglast(i2c)) { /* last byte of buffer */ diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index e533460bccc39..a9aed411e3190 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -1837,8 +1837,11 @@ static int tegra_i2c_probe(struct platform_device *pdev) * * VI I2C device shouldn't be marked as IRQ-safe because VI I2C won't * be used for atomic transfers. ACPI device is not IRQ safe also. + * + * Devices with pinctrl states cannot be marked IRQ-safe as the pinctrl + * state transitions during runtime PM require mutexes. */ - if (!IS_VI(i2c_dev) && !has_acpi_companion(i2c_dev->dev)) + if (!IS_VI(i2c_dev) && !has_acpi_companion(i2c_dev->dev) && !i2c_dev->dev->pins) pm_runtime_irq_safe(i2c_dev->dev); pm_runtime_enable(i2c_dev->dev); diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index d946db75df706..425e36b36009b 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -620,7 +620,8 @@ static int i3c_set_hotjoin(struct i3c_master_controller *master, bool enable) else ret = master->ops->disable_hotjoin(master); - master->hotjoin = enable; + if (!ret) + master->hotjoin = enable; i3c_bus_normaluse_unlock(&master->bus); @@ -2883,17 +2884,18 @@ int i3c_master_register(struct i3c_master_controller *master, INIT_LIST_HEAD(&master->boardinfo.i2c); INIT_LIST_HEAD(&master->boardinfo.i3c); - ret = i3c_bus_init(i3cbus, master->dev.of_node); - if (ret) - return ret; - device_initialize(&master->dev); - dev_set_name(&master->dev, "i3c-%d", i3cbus->id); master->dev.dma_mask = parent->dma_mask; master->dev.coherent_dma_mask = parent->coherent_dma_mask; master->dev.dma_parms = parent->dma_parms; + ret = i3c_bus_init(i3cbus, master->dev.of_node); + if (ret) + goto err_put_dev; + + dev_set_name(&master->dev, "i3c-%d", i3cbus->id); + ret = of_populate_i3c_bus(master); if (ret) goto err_put_dev; diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 9ceedf09c3b6a..825eb2d20e9eb 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -1005,7 +1005,7 @@ static int dw_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, master->free_pos &= ~BIT(pos); } - writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(dev->info.dyn_addr), + writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(dev->info.dyn_addr) | DEV_ADDR_TABLE_SIR_REJECT, master->regs + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); @@ -1034,7 +1034,7 @@ static int dw_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev) master->free_pos &= ~BIT(pos); i3c_dev_set_master_data(dev, data); - writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr), + writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr) | DEV_ADDR_TABLE_SIR_REJECT, master->regs + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); @@ -1094,6 +1094,7 @@ static int dw_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, dev_err(master->dev, "<%s> cannot resume i3c bus master, err: %d\n", __func__, ret); + dw_i3c_master_free_xfer(xfer); return ret; } @@ -1563,6 +1564,8 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); + spin_lock_init(&master->devs_lock); + writel(INTR_ALL, master->regs + INTR_STATUS); irq = platform_get_irq(pdev, 0); ret = devm_request_irq(&pdev->dev, irq, @@ -1593,6 +1596,8 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, master->quirks = (unsigned long)device_get_match_data(&pdev->dev); INIT_WORK(&master->hj_work, dw_i3c_hj_work); + + device_set_of_node_from_dev(&master->base.i2c.dev, &pdev->dev); ret = i3c_master_register(&master->base, &pdev->dev, &dw_mipi_i3c_ops, false); if (ret) diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd.h b/drivers/i3c/master/mipi-i3c-hci/cmd.h index 1d6dd2c5d01a5..b1bf87daa6516 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd.h +++ b/drivers/i3c/master/mipi-i3c-hci/cmd.h @@ -17,6 +17,7 @@ #define CMD_0_TOC W0_BIT_(31) #define CMD_0_ROC W0_BIT_(30) #define CMD_0_ATTR W0_MASK(2, 0) +#define CMD_0_TID W0_MASK(6, 3) /* * Response Descriptor Structure diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c index eb8a3ae2990d7..efb7a1f92641c 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c @@ -336,7 +336,7 @@ static int hci_cmd_v1_daa(struct i3c_hci *hci) hci->io->queue_xfer(hci, xfer, 1); if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, 1)) { - ret = -ETIME; + ret = -ETIMEDOUT; break; } if ((RESP_STATUS(xfer->response) == RESP_ERR_ADDR_HEADER || diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c index efb4326a25b73..5fc2e4c55ebb0 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c @@ -277,7 +277,7 @@ static int hci_cmd_v2_daa(struct i3c_hci *hci) hci->io->queue_xfer(hci, xfer, 2); if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, 2)) { - ret = -ETIME; + ret = -ETIMEDOUT; break; } if (RESP_STATUS(xfer[0].response) != RESP_SUCCESS) { diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c index 47e42cb4dbe71..c355f4a2baf57 100644 --- a/drivers/i3c/master/mipi-i3c-hci/core.c +++ b/drivers/i3c/master/mipi-i3c-hci/core.c @@ -230,7 +230,7 @@ static int i3c_hci_send_ccc_cmd(struct i3c_master_controller *m, goto out; if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, nxfers)) { - ret = -ETIME; + ret = -ETIMEDOUT; goto out; } for (i = prefixed; i < nxfers; i++) { @@ -309,7 +309,7 @@ static int i3c_hci_priv_xfers(struct i3c_dev_desc *dev, goto out; if (!wait_for_completion_timeout(&done, HZ) && hci->io->dequeue_xfer(hci, xfer, nxfers)) { - ret = -ETIME; + ret = -ETIMEDOUT; goto out; } for (i = 0; i < nxfers; i++) { @@ -357,7 +357,7 @@ static int i3c_hci_i2c_xfers(struct i2c_dev_desc *dev, goto out; if (!wait_for_completion_timeout(&done, m->i2c.timeout) && hci->io->dequeue_xfer(hci, xfer, nxfers)) { - ret = -ETIME; + ret = -ETIMEDOUT; goto out; } for (i = 0; i < nxfers; i++) { @@ -631,6 +631,9 @@ static int i3c_hci_init(struct i3c_hci *hci) if (ret) return ret; + spin_lock_init(&hci->lock); + mutex_init(&hci->control_mutex); + /* * Now let's reset the hardware. * SOFT_RST must be clear before we write to it. diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c index c401a9425cdc5..fe8894f6fe607 100644 --- a/drivers/i3c/master/mipi-i3c-hci/dma.c +++ b/drivers/i3c/master/mipi-i3c-hci/dma.c @@ -133,7 +133,6 @@ struct hci_rh_data { unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz; unsigned int done_ptr, ibi_chunk_ptr; struct hci_xfer **src_xfers; - spinlock_t lock; struct completion op_done; }; @@ -240,7 +239,6 @@ static int hci_dma_init(struct i3c_hci *hci) goto err_out; rh = &rings->headers[i]; rh->regs = hci->base_regs + offset; - spin_lock_init(&rh->lock); init_completion(&rh->op_done); rh->xfer_entries = XFER_RING_ENTRIES; @@ -342,6 +340,14 @@ static int hci_dma_init(struct i3c_hci *hci) rh_reg_write(INTR_SIGNAL_ENABLE, regval); ring_ready: + /* + * The MIPI I3C HCI specification does not document reset values for + * RING_OPERATION1 fields and some controllers (e.g. Intel controllers) + * do not reset the values, so ensure the ring pointers are set to zero + * here. + */ + rh_reg_write(RING_OPERATION1, 0); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); } @@ -367,6 +373,33 @@ static void hci_dma_unmap_xfer(struct i3c_hci *hci, } } +static struct i3c_dma *hci_dma_map_xfer(struct device *dev, struct hci_xfer *xfer) +{ + enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + bool need_bounce = device_iommu_mapped(dev) && xfer->rnw && (xfer->data_len & 3); + + return i3c_master_dma_map_single(dev, xfer->data, xfer->data_len, need_bounce, dir); +} + +static int hci_dma_map_xfer_list(struct i3c_hci *hci, struct device *dev, + struct hci_xfer *xfer_list, int n) +{ + for (int i = 0; i < n; i++) { + struct hci_xfer *xfer = xfer_list + i; + + if (!xfer->data) + continue; + + xfer->dma = hci_dma_map_xfer(dev, xfer); + if (!xfer->dma) { + hci_dma_unmap_xfer(hci, xfer_list, i); + return -ENOMEM; + } + } + + return 0; +} + static int hci_dma_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer_list, int n) { @@ -374,6 +407,11 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, struct hci_rh_data *rh; unsigned int i, ring, enqueue_ptr; u32 op1_val, op2_val; + int ret; + + ret = hci_dma_map_xfer_list(hci, rings->sysdev, xfer_list, n); + if (ret) + return ret; /* For now we only use ring 0 */ ring = 0; @@ -384,9 +422,6 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, for (i = 0; i < n; i++) { struct hci_xfer *xfer = xfer_list + i; u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr; - enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : - DMA_TO_DEVICE; - bool need_bounce; /* store cmd descriptor */ *ring_data++ = xfer->cmd_desc[0]; @@ -405,18 +440,6 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, /* 2nd and 3rd words of Data Buffer Descriptor Structure */ if (xfer->data) { - need_bounce = device_iommu_mapped(rings->sysdev) && - xfer->rnw && - xfer->data_len != ALIGN(xfer->data_len, 4); - xfer->dma = i3c_master_dma_map_single(rings->sysdev, - xfer->data, - xfer->data_len, - need_bounce, - dir); - if (!xfer->dma) { - hci_dma_unmap_xfer(hci, xfer_list, i); - return -ENOMEM; - } *ring_data++ = lower_32_bits(xfer->dma->addr); *ring_data++ = upper_32_bits(xfer->dma->addr); } else { @@ -439,18 +462,18 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci, op2_val = rh_reg_read(RING_OPERATION2); if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) { /* the ring is full */ - hci_dma_unmap_xfer(hci, xfer_list, i + 1); + hci_dma_unmap_xfer(hci, xfer_list, n); return -EBUSY; } } /* take care to update the hardware enqueue pointer atomically */ - spin_lock_irq(&rh->lock); + spin_lock_irq(&hci->lock); op1_val = rh_reg_read(RING_OPERATION1); op1_val &= ~RING_OP1_CR_ENQ_PTR; op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr); rh_reg_write(RING_OPERATION1, op1_val); - spin_unlock_irq(&rh->lock); + spin_unlock_irq(&hci->lock); return 0; } @@ -462,16 +485,25 @@ static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number]; unsigned int i; bool did_unqueue = false; - - /* stop the ring */ - rh_reg_write(RING_CONTROL, RING_CTRL_ABORT); - if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) { - /* - * We're deep in it if ever this condition is ever met. - * Hardware might still be writing to memory, etc. - */ - dev_crit(&hci->master.dev, "unable to abort the ring\n"); - WARN_ON(1); + u32 ring_status; + + guard(mutex)(&hci->control_mutex); + + ring_status = rh_reg_read(RING_STATUS); + if (ring_status & RING_STATUS_RUNNING) { + /* stop the ring */ + reinit_completion(&rh->op_done); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_ABORT); + wait_for_completion_timeout(&rh->op_done, HZ); + ring_status = rh_reg_read(RING_STATUS); + if (ring_status & RING_STATUS_RUNNING) { + /* + * We're deep in it if ever this condition is ever met. + * Hardware might still be writing to memory, etc. + */ + dev_crit(&hci->master.dev, "unable to abort the ring\n"); + WARN_ON(1); + } } for (i = 0; i < n; i++) { @@ -487,7 +519,7 @@ static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx; /* store no-op cmd descriptor */ - *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7); + *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7) | FIELD_PREP(CMD_0_TID, xfer->cmd_tid); *ring_data++ = 0; if (hci->cmd == &mipi_i3c_hci_cmd_v2) { *ring_data++ = 0; @@ -505,7 +537,9 @@ static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, } /* restart the ring */ + mipi_i3c_hci_resume(hci); rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); return did_unqueue; } @@ -548,12 +582,12 @@ static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh) } /* take care to update the software dequeue pointer atomically */ - spin_lock(&rh->lock); + spin_lock(&hci->lock); op1_val = rh_reg_read(RING_OPERATION1); op1_val &= ~RING_OP1_CR_SW_DEQ_PTR; op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr); rh_reg_write(RING_OPERATION1, op1_val); - spin_unlock(&rh->lock); + spin_unlock(&hci->lock); } static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, @@ -734,12 +768,12 @@ static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) done: /* take care to update the ibi dequeue pointer atomically */ - spin_lock(&rh->lock); + spin_lock(&hci->lock); op1_val = rh_reg_read(RING_OPERATION1); op1_val &= ~RING_OP1_IBI_DEQ_PTR; op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr); rh_reg_write(RING_OPERATION1, op1_val); - spin_unlock(&rh->lock); + spin_unlock(&hci->lock); /* update the chunk pointer */ rh->ibi_chunk_ptr += ibi_chunks; diff --git a/drivers/i3c/master/mipi-i3c-hci/ext_caps.c b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c index 7714f00ea9cc0..533a495e14c86 100644 --- a/drivers/i3c/master/mipi-i3c-hci/ext_caps.c +++ b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c @@ -272,7 +272,7 @@ int i3c_hci_parse_ext_caps(struct i3c_hci *hci) cap_length = FIELD_GET(CAP_HEADER_LENGTH, cap_header); dev_dbg(&hci->master.dev, "id=0x%02x length=%d", cap_id, cap_length); - if (!cap_length) + if (!cap_id || !cap_length) break; if (curr_cap + cap_length * 4 >= end) { dev_err(&hci->master.dev, diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h index 249ccb13c9092..32c8aecde9f76 100644 --- a/drivers/i3c/master/mipi-i3c-hci/hci.h +++ b/drivers/i3c/master/mipi-i3c-hci/hci.h @@ -45,6 +45,8 @@ struct i3c_hci { const struct hci_io_ops *io; void *io_data; const struct hci_cmd_ops *cmd; + spinlock_t lock; + struct mutex control_mutex; atomic_t next_cmd_tid; u32 caps; unsigned int quirks; diff --git a/drivers/i3c/master/mipi-i3c-hci/pio.c b/drivers/i3c/master/mipi-i3c-hci/pio.c index 710faa46a00fa..67dc34163d51b 100644 --- a/drivers/i3c/master/mipi-i3c-hci/pio.c +++ b/drivers/i3c/master/mipi-i3c-hci/pio.c @@ -124,7 +124,6 @@ struct hci_pio_ibi_data { }; struct hci_pio_data { - spinlock_t lock; struct hci_xfer *curr_xfer, *xfer_queue; struct hci_xfer *curr_rx, *rx_queue; struct hci_xfer *curr_tx, *tx_queue; @@ -146,7 +145,6 @@ static int hci_pio_init(struct i3c_hci *hci) return -ENOMEM; hci->io_data = pio; - spin_lock_init(&pio->lock); size_val = pio_reg_read(QUEUE_SIZE); dev_info(&hci->master.dev, "CMD/RESP FIFO = %ld entries\n", @@ -609,7 +607,7 @@ static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) xfer[i].data_left = xfer[i].data_len; } - spin_lock_irq(&pio->lock); + spin_lock_irq(&hci->lock); prev_queue_tail = pio->xfer_queue; pio->xfer_queue = &xfer[n - 1]; if (pio->curr_xfer) { @@ -623,7 +621,7 @@ static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); } - spin_unlock_irq(&pio->lock); + spin_unlock_irq(&hci->lock); return 0; } @@ -694,14 +692,14 @@ static bool hci_pio_dequeue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int struct hci_pio_data *pio = hci->io_data; int ret; - spin_lock_irq(&pio->lock); + spin_lock_irq(&hci->lock); dev_dbg(&hci->master.dev, "n=%d status=%#x/%#x", n, pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); dev_dbg(&hci->master.dev, "main_status = %#x/%#x", readl(hci->base_regs + 0x20), readl(hci->base_regs + 0x28)); ret = hci_pio_dequeue_xfer_common(hci, pio, xfer, n); - spin_unlock_irq(&pio->lock); + spin_unlock_irq(&hci->lock); return ret; } @@ -994,13 +992,13 @@ static bool hci_pio_irq_handler(struct i3c_hci *hci) struct hci_pio_data *pio = hci->io_data; u32 status; - spin_lock(&pio->lock); + spin_lock(&hci->lock); status = pio_reg_read(INTR_STATUS); dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", status, pio->enabled_irqs); status &= pio->enabled_irqs | STAT_LATENCY_WARNINGS; if (!status) { - spin_unlock(&pio->lock); + spin_unlock(&hci->lock); return false; } @@ -1036,7 +1034,7 @@ static bool hci_pio_irq_handler(struct i3c_hci *hci) pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); - spin_unlock(&pio->lock); + spin_unlock(&hci->lock); return true; } diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index 9641e66a4e5f2..93531cb216d16 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -406,21 +406,27 @@ static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master, int ret, val; u8 *buf; - slot = i3c_generic_ibi_get_free_slot(data->ibi_pool); - if (!slot) - return -ENOSPC; - - slot->len = 0; - buf = slot->data; - + /* + * Wait for transfer to complete before returning. Otherwise, the EmitStop + * request might be sent when the transfer is not complete. + */ ret = readl_relaxed_poll_timeout(master->regs + SVC_I3C_MSTATUS, val, SVC_I3C_MSTATUS_COMPLETE(val), 0, 1000); if (ret) { dev_err(master->dev, "Timeout when polling for COMPLETE\n"); - i3c_generic_ibi_recycle_slot(data->ibi_pool, slot); return ret; } + slot = i3c_generic_ibi_get_free_slot(data->ibi_pool); + if (!slot) { + dev_dbg(master->dev, "No free ibi slot, drop the data\n"); + writel(SVC_I3C_MDATACTRL_FLUSHRB, master->regs + SVC_I3C_MDATACTRL); + return -ENOSPC; + } + + slot->len = 0; + buf = slot->data; + while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS)) && slot->len < SVC_I3C_FIFO_SIZE) { mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL); @@ -490,8 +496,8 @@ static int svc_i3c_master_handle_ibi_won(struct svc_i3c_master *master, u32 msta static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) { struct svc_i3c_i2c_dev_data *data; + struct i3c_dev_desc *dev = NULL; unsigned int ibitype, ibiaddr; - struct i3c_dev_desc *dev; u32 status, val; int ret; @@ -584,7 +590,7 @@ static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) * for the slave to interrupt again. */ if (svc_i3c_master_error(master)) { - if (master->ibi.tbq_slot) { + if (master->ibi.tbq_slot && dev) { data = i3c_dev_get_master_data(dev); i3c_generic_ibi_recycle_slot(data->ibi_pool, master->ibi.tbq_slot); diff --git a/drivers/iio/accel/adxl313_core.c b/drivers/iio/accel/adxl313_core.c index 9f5d4d2cb325b..83dcac17a0425 100644 --- a/drivers/iio/accel/adxl313_core.c +++ b/drivers/iio/accel/adxl313_core.c @@ -998,6 +998,8 @@ static int adxl313_buffer_predisable(struct iio_dev *indio_dev) ret = regmap_write(data->regmap, ADXL313_REG_FIFO_CTL, FIELD_PREP(ADXL313_REG_FIFO_CTL_MODE_MSK, ADXL313_FIFO_BYPASS)); + if (ret) + return ret; ret = regmap_write(data->regmap, ADXL313_REG_INT_ENABLE, 0); if (ret) diff --git a/drivers/iio/accel/adxl355_core.c b/drivers/iio/accel/adxl355_core.c index 5fc7f814b9077..5aadc04771025 100644 --- a/drivers/iio/accel/adxl355_core.c +++ b/drivers/iio/accel/adxl355_core.c @@ -745,7 +745,7 @@ static const struct iio_chan_spec adxl355_channels[] = { BIT(IIO_CHAN_INFO_OFFSET), .scan_index = 3, .scan_type = { - .sign = 's', + .sign = 'u', .realbits = 12, .storagebits = 16, .endianness = IIO_BE, diff --git a/drivers/iio/accel/adxl380.c b/drivers/iio/accel/adxl380.c index 0cf3c68158293..c1188d3b06d4a 100644 --- a/drivers/iio/accel/adxl380.c +++ b/drivers/iio/accel/adxl380.c @@ -888,7 +888,7 @@ static int adxl380_set_fifo_samples(struct adxl380_state *st) ret = regmap_update_bits(st->regmap, ADXL380_FIFO_CONFIG_0_REG, ADXL380_FIFO_SAMPLES_8_MSK, FIELD_PREP(ADXL380_FIFO_SAMPLES_8_MSK, - (fifo_samples & BIT(8)))); + !!(fifo_samples & BIT(8)))); if (ret) return ret; @@ -977,6 +977,7 @@ static irqreturn_t adxl380_irq_handler(int irq, void *p) if (ret) return IRQ_HANDLED; + fifo_entries = rounddown(fifo_entries, st->fifo_set_size); for (i = 0; i < fifo_entries; i += st->fifo_set_size) { ret = regmap_noinc_read(st->regmap, ADXL380_FIFO_DATA, &st->fifo_buf[i], @@ -1728,9 +1729,9 @@ static int adxl380_config_irq(struct iio_dev *indio_dev) st->int_map[1] = ADXL380_INT0_MAP1_REG; } else { st->irq = fwnode_irq_get_byname(dev_fwnode(st->dev), "INT1"); - if (st->irq > 0) - return dev_err_probe(st->dev, -ENODEV, - "no interrupt name specified"); + if (st->irq < 0) + return dev_err_probe(st->dev, st->irq, + "no interrupt name specified\n"); st->int_map[0] = ADXL380_INT1_MAP0_REG; st->int_map[1] = ADXL380_INT1_MAP1_REG; } diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c index 8925f5279e627..7bc6761f51354 100644 --- a/drivers/iio/accel/bma180.c +++ b/drivers/iio/accel/bma180.c @@ -986,8 +986,9 @@ static int bma180_probe(struct i2c_client *client) } ret = devm_request_irq(dev, client->irq, - iio_trigger_generic_data_rdy_poll, IRQF_TRIGGER_RISING, - "bma180_event", data->trig); + iio_trigger_generic_data_rdy_poll, + IRQF_TRIGGER_RISING | IRQF_NO_THREAD, + "bma180_event", data->trig); if (ret) { dev_err(dev, "unable to request IRQ\n"); goto err_trigger_free; diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c index bfa8a3f5a92f4..9ef4d6e274669 100644 --- a/drivers/iio/accel/sca3000.c +++ b/drivers/iio/accel/sca3000.c @@ -1489,7 +1489,11 @@ static int sca3000_probe(struct spi_device *spi) if (ret) goto error_free_irq; - return iio_device_register(indio_dev); + ret = iio_device_register(indio_dev); + if (ret) + goto error_free_irq; + + return 0; error_free_irq: if (spi->irq) diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c index a7961c610ed20..1a9447c81b0f7 100644 --- a/drivers/iio/accel/st_accel_core.c +++ b/drivers/iio/accel/st_accel_core.c @@ -517,7 +517,6 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = { .wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS, .sensors_supported = { [0] = H3LIS331DL_ACCEL_DEV_NAME, - [1] = IIS328DQ_ACCEL_DEV_NAME, }, .ch = (struct iio_chan_spec *)st_accel_12bit_channels, .odr = { @@ -584,6 +583,77 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = { .multi_read_bit = true, .bootime = 2, }, + { + .wai = 0x32, + .wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS, + .sensors_supported = { + [0] = IIS328DQ_ACCEL_DEV_NAME, + }, + .ch = (struct iio_chan_spec *)st_accel_12bit_channels, + .odr = { + .addr = 0x20, + .mask = 0x18, + .odr_avl = { + { .hz = 50, .value = 0x00, }, + { .hz = 100, .value = 0x01, }, + { .hz = 400, .value = 0x02, }, + { .hz = 1000, .value = 0x03, }, + }, + }, + .pw = { + .addr = 0x20, + .mask = 0x20, + .value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE, + .value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE, + }, + .enable_axis = { + .addr = ST_SENSORS_DEFAULT_AXIS_ADDR, + .mask = ST_SENSORS_DEFAULT_AXIS_MASK, + }, + .fs = { + .addr = 0x23, + .mask = 0x30, + .fs_avl = { + [0] = { + .num = ST_ACCEL_FS_AVL_100G, + .value = 0x00, + .gain = IIO_G_TO_M_S_2(980), + }, + [1] = { + .num = ST_ACCEL_FS_AVL_200G, + .value = 0x01, + .gain = IIO_G_TO_M_S_2(1950), + }, + [2] = { + .num = ST_ACCEL_FS_AVL_400G, + .value = 0x03, + .gain = IIO_G_TO_M_S_2(3910), + }, + }, + }, + .bdu = { + .addr = 0x23, + .mask = 0x80, + }, + .drdy_irq = { + .int1 = { + .addr = 0x22, + .mask = 0x02, + }, + .int2 = { + .addr = 0x22, + .mask = 0x10, + }, + .addr_ihl = 0x22, + .mask_ihl = 0x80, + }, + .sim = { + .addr = 0x23, + .value = BIT(0), + }, + .multi_read_bit = true, + .bootime = 2, + }, { /* No WAI register present */ .sensors_supported = { diff --git a/drivers/iio/adc/ad4080.c b/drivers/iio/adc/ad4080.c index 6e61787ed3213..e15310fcd21a3 100644 --- a/drivers/iio/adc/ad4080.c +++ b/drivers/iio/adc/ad4080.c @@ -125,7 +125,7 @@ /* Miscellaneous Definitions */ #define AD4080_SPI_READ BIT(7) -#define AD4080_CHIP_ID GENMASK(2, 0) +#define AD4080_CHIP_ID 0x0050 #define AD4080_LVDS_CNV_CLK_CNT_MAX 7 @@ -445,7 +445,8 @@ static int ad4080_setup(struct iio_dev *indio_dev) { struct ad4080_state *st = iio_priv(indio_dev); struct device *dev = regmap_get_device(st->regmap); - unsigned int id; + __le16 id_le; + u16 id; int ret; ret = regmap_write(st->regmap, AD4080_REG_INTERFACE_CONFIG_A, @@ -458,10 +459,12 @@ static int ad4080_setup(struct iio_dev *indio_dev) if (ret) return ret; - ret = regmap_read(st->regmap, AD4080_REG_CHIP_TYPE, &id); + ret = regmap_bulk_read(st->regmap, AD4080_REG_PRODUCT_ID_L, &id_le, + sizeof(id_le)); if (ret) return ret; + id = le16_to_cpu(id_le); if (id != AD4080_CHIP_ID) dev_info(dev, "Unrecognized CHIP_ID 0x%X\n", id); diff --git a/drivers/iio/adc/ad7280a.c b/drivers/iio/adc/ad7280a.c index 50a6ff7c8b1c7..ba12a3796e2b1 100644 --- a/drivers/iio/adc/ad7280a.c +++ b/drivers/iio/adc/ad7280a.c @@ -1024,7 +1024,9 @@ static int ad7280_probe(struct spi_device *spi) st->spi->max_speed_hz = AD7280A_MAX_SPI_CLK_HZ; st->spi->mode = SPI_MODE_1; - spi_setup(st->spi); + ret = spi_setup(st->spi); + if (ret < 0) + return ret; st->ctrl_lb = FIELD_PREP(AD7280A_CTRL_LB_ACQ_TIME_MSK, st->acquisition_time) | FIELD_PREP(AD7280A_CTRL_LB_THERMISTOR_MSK, st->thermistor_term_en); diff --git a/drivers/iio/adc/ad7606_par.c b/drivers/iio/adc/ad7606_par.c index 634852c4bbd2c..b81e707ab40c5 100644 --- a/drivers/iio/adc/ad7606_par.c +++ b/drivers/iio/adc/ad7606_par.c @@ -43,7 +43,8 @@ static int ad7606_par_bus_setup_iio_backend(struct device *dev, struct iio_dev *indio_dev) { struct ad7606_state *st = iio_priv(indio_dev); - unsigned int ret, c; + unsigned int c; + int ret; struct iio_backend_data_fmt data = { .sign_extend = true, .enable = true, diff --git a/drivers/iio/adc/ad7766.c b/drivers/iio/adc/ad7766.c index 4d570383ef025..1e6bfe8765ab3 100644 --- a/drivers/iio/adc/ad7766.c +++ b/drivers/iio/adc/ad7766.c @@ -261,7 +261,7 @@ static int ad7766_probe(struct spi_device *spi) * don't enable the interrupt to avoid extra load on the system */ ret = devm_request_irq(&spi->dev, spi->irq, ad7766_irq, - IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN, + IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN | IRQF_NO_THREAD, dev_name(&spi->dev), ad7766->trig); if (ret < 0) diff --git a/drivers/iio/adc/ad9467.c b/drivers/iio/adc/ad9467.c index f7a9f46ea0dc4..2d8f8da3671da 100644 --- a/drivers/iio/adc/ad9467.c +++ b/drivers/iio/adc/ad9467.c @@ -95,7 +95,7 @@ #define CHIPID_AD9434 0x6A #define AD9434_DEF_OUTPUT_MODE 0x00 -#define AD9434_REG_VREF_MASK 0xC0 +#define AD9434_REG_VREF_MASK GENMASK(4, 0) /* * Analog Devices AD9467 16-Bit, 200/250 MSPS ADC diff --git a/drivers/iio/adc/ade9000.c b/drivers/iio/adc/ade9000.c index 94e05e11abd90..f87fa64562eb1 100644 --- a/drivers/iio/adc/ade9000.c +++ b/drivers/iio/adc/ade9000.c @@ -787,7 +787,7 @@ static int ade9000_iio_push_streaming(struct iio_dev *indio_dev) ADE9000_MIDDLE_PAGE_BIT); if (ret) { dev_err_ratelimited(dev, "IRQ0 WFB write fail"); - return IRQ_HANDLED; + return ret; } ade9000_configure_scan(indio_dev, ADE9000_REG_WF_BUFF); @@ -1123,7 +1123,7 @@ static int ade9000_write_raw(struct iio_dev *indio_dev, tmp &= ~ADE9000_PHASE_C_POS_BIT; switch (tmp) { - case ADE9000_REG_AWATTOS: + case ADE9000_REG_AWATT: return regmap_write(st->regmap, ADE9000_ADDR_ADJUST(ADE9000_REG_AWATTOS, chan->channel), val); @@ -1706,19 +1706,19 @@ static int ade9000_probe(struct spi_device *spi) init_completion(&st->reset_completion); - ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev); + ret = devm_mutex_init(dev, &st->lock); if (ret) return ret; - ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev); + ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev); if (ret) return ret; - ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev); + ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev); if (ret) return ret; - ret = devm_mutex_init(dev, &st->lock); + ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev); if (ret) return ret; diff --git a/drivers/iio/adc/aspeed_adc.c b/drivers/iio/adc/aspeed_adc.c index 1d5fd5f534b8a..8ab29948214a5 100644 --- a/drivers/iio/adc/aspeed_adc.c +++ b/drivers/iio/adc/aspeed_adc.c @@ -415,6 +415,7 @@ static int aspeed_adc_vref_config(struct iio_dev *indio_dev) } adc_engine_control_reg_val = readl(data->base + ASPEED_REG_ENGINE_CONTROL); + adc_engine_control_reg_val &= ~ASPEED_ADC_REF_VOLTAGE; ret = devm_regulator_get_enable_read_voltage(data->dev, "vref"); if (ret < 0 && ret != -ENODEV) diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c index b4c36e6a7490a..aa4ba3f5a5060 100644 --- a/drivers/iio/adc/at91-sama5d2_adc.c +++ b/drivers/iio/adc/at91-sama5d2_adc.c @@ -2481,6 +2481,7 @@ static void at91_adc_remove(struct platform_device *pdev) struct at91_adc_state *st = iio_priv(indio_dev); iio_device_unregister(indio_dev); + cancel_work_sync(&st->touch_st.workq); at91_adc_dma_disable(st); diff --git a/drivers/iio/adc/exynos_adc.c b/drivers/iio/adc/exynos_adc.c index 1484adff00df3..f2400897818c5 100644 --- a/drivers/iio/adc/exynos_adc.c +++ b/drivers/iio/adc/exynos_adc.c @@ -540,15 +540,6 @@ static const struct iio_chan_spec exynos_adc_iio_channels[] = { ADC_CHANNEL(9, "adc9"), }; -static int exynos_adc_remove_devices(struct device *dev, void *c) -{ - struct platform_device *pdev = to_platform_device(dev); - - platform_device_unregister(pdev); - - return 0; -} - static int exynos_adc_probe(struct platform_device *pdev) { struct exynos_adc *info = NULL; @@ -660,8 +651,7 @@ static int exynos_adc_probe(struct platform_device *pdev) return 0; err_of_populate: - device_for_each_child(&indio_dev->dev, NULL, - exynos_adc_remove_devices); + of_platform_depopulate(&indio_dev->dev); iio_device_unregister(indio_dev); err_irq: free_irq(info->irq, info); @@ -681,8 +671,7 @@ static void exynos_adc_remove(struct platform_device *pdev) struct iio_dev *indio_dev = platform_get_drvdata(pdev); struct exynos_adc *info = iio_priv(indio_dev); - device_for_each_child(&indio_dev->dev, NULL, - exynos_adc_remove_devices); + of_platform_depopulate(&indio_dev->dev); iio_device_unregister(indio_dev); free_irq(info->irq, info); if (info->data->exit_hw) diff --git a/drivers/iio/adc/pac1934.c b/drivers/iio/adc/pac1934.c index 48df165092605..edaff72e9d792 100644 --- a/drivers/iio/adc/pac1934.c +++ b/drivers/iio/adc/pac1934.c @@ -665,9 +665,9 @@ static int pac1934_reg_snapshot(struct pac1934_chip_info *info, /* add the power_acc field */ curr_energy += inc; - clamp(curr_energy, PAC_193X_MIN_POWER_ACC, PAC_193X_MAX_POWER_ACC); - - reg_data->energy_sec_acc[cnt] = curr_energy; + reg_data->energy_sec_acc[cnt] = clamp(curr_energy, + PAC_193X_MIN_POWER_ACC, + PAC_193X_MAX_POWER_ACC); } offset_reg_data_p += PAC1934_VPOWER_ACC_REG_LEN; diff --git a/drivers/iio/adc/ti-adc161s626.c b/drivers/iio/adc/ti-adc161s626.c index 28aa6b80160c8..be1cc2e77862b 100644 --- a/drivers/iio/adc/ti-adc161s626.c +++ b/drivers/iio/adc/ti-adc161s626.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -70,8 +71,7 @@ struct ti_adc_data { u8 read_size; u8 shift; - - u8 buffer[16] __aligned(IIO_DMA_MINALIGN); + u8 buf[3] __aligned(IIO_DMA_MINALIGN); }; static int ti_adc_read_measurement(struct ti_adc_data *data, @@ -80,26 +80,20 @@ static int ti_adc_read_measurement(struct ti_adc_data *data, int ret; switch (data->read_size) { - case 2: { - __be16 buf; - - ret = spi_read(data->spi, (void *) &buf, 2); + case 2: + ret = spi_read(data->spi, data->buf, 2); if (ret) return ret; - *val = be16_to_cpu(buf); + *val = get_unaligned_be16(data->buf); break; - } - case 3: { - __be32 buf; - - ret = spi_read(data->spi, (void *) &buf, 3); + case 3: + ret = spi_read(data->spi, data->buf, 3); if (ret) return ret; - *val = be32_to_cpu(buf) >> 8; + *val = get_unaligned_be24(data->buf); break; - } default: return -EINVAL; } @@ -114,15 +108,20 @@ static irqreturn_t ti_adc_trigger_handler(int irq, void *private) struct iio_poll_func *pf = private; struct iio_dev *indio_dev = pf->indio_dev; struct ti_adc_data *data = iio_priv(indio_dev); - int ret; + struct { + s16 data; + aligned_s64 timestamp; + } scan = { }; + int ret, val; + + ret = ti_adc_read_measurement(data, &indio_dev->channels[0], &val); + if (ret) + goto exit_notify_done; - ret = ti_adc_read_measurement(data, &indio_dev->channels[0], - (int *) &data->buffer); - if (!ret) - iio_push_to_buffers_with_timestamp(indio_dev, - data->buffer, - iio_get_time_ns(indio_dev)); + scan.data = val; + iio_push_to_buffers_with_timestamp(indio_dev, &scan, iio_get_time_ns(indio_dev)); + exit_notify_done: iio_trigger_notify_done(indio_dev->trig); return IRQ_HANDLED; diff --git a/drivers/iio/adc/ti-ads1119.c b/drivers/iio/adc/ti-ads1119.c index c9cedc59cdcdb..79be71b4de963 100644 --- a/drivers/iio/adc/ti-ads1119.c +++ b/drivers/iio/adc/ti-ads1119.c @@ -274,12 +274,15 @@ static int ads1119_single_conversion(struct ads1119_state *st, ret = pm_runtime_resume_and_get(dev); if (ret) - goto pdown; + return ret; ret = ads1119_configure_channel(st, mux, gain, datarate); if (ret) goto pdown; + if (st->client->irq) + reinit_completion(&st->completion); + ret = i2c_smbus_write_byte(st->client, ADS1119_CMD_START_SYNC); if (ret) goto pdown; @@ -735,10 +738,8 @@ static int ads1119_probe(struct i2c_client *client) return dev_err_probe(dev, ret, "Failed to setup IIO buffer\n"); if (client->irq > 0) { - ret = devm_request_threaded_irq(dev, client->irq, - ads1119_irq_handler, - NULL, IRQF_ONESHOT, - "ads1119", indio_dev); + ret = devm_request_irq(dev, client->irq, ads1119_irq_handler, + IRQF_NO_THREAD, "ads1119", indio_dev); if (ret) return dev_err_probe(dev, ret, "Failed to allocate irq\n"); diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c index bbe1ce5777895..cdc6248895593 100644 --- a/drivers/iio/adc/ti-ads7950.c +++ b/drivers/iio/adc/ti-ads7950.c @@ -427,13 +427,15 @@ static int ti_ads7950_set(struct gpio_chip *chip, unsigned int offset, static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset) { struct ti_ads7950_state *st = gpiochip_get_data(chip); + bool state; int ret; mutex_lock(&st->slock); /* If set as output, return the output */ if (st->gpio_cmd_settings_bitmask & BIT(offset)) { - ret = st->cmd_settings_bitmask & BIT(offset); + state = st->cmd_settings_bitmask & BIT(offset); + ret = 0; goto out; } @@ -444,7 +446,7 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset) if (ret) goto out; - ret = ((st->single_rx >> 12) & BIT(offset)) ? 1 : 0; + state = (st->single_rx >> 12) & BIT(offset); /* Revert back to original settings */ st->cmd_settings_bitmask &= ~TI_ADS7950_CR_GPIO_DATA; @@ -456,7 +458,7 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset) out: mutex_unlock(&st->slock); - return ret; + return ret ?: state; } static int ti_ads7950_get_direction(struct gpio_chip *chip, diff --git a/drivers/iio/adc/ti_am335x_adc.c b/drivers/iio/adc/ti_am335x_adc.c index 99f274adc870d..a1a28584de930 100644 --- a/drivers/iio/adc/ti_am335x_adc.c +++ b/drivers/iio/adc/ti_am335x_adc.c @@ -123,7 +123,7 @@ static void tiadc_step_config(struct iio_dev *indio_dev) chan = adc_dev->channel_line[i]; - if (adc_dev->step_avg[i]) + if (adc_dev->step_avg[i] && adc_dev->step_avg[i] <= STEPCONFIG_AVG_16) stepconfig = STEPCONFIG_AVG(ffs(adc_dev->step_avg[i]) - 1) | STEPCONFIG_FIFO1; else diff --git a/drivers/iio/chemical/bme680_core.c b/drivers/iio/chemical/bme680_core.c index 70f81c4a96bac..24e0b59e2fdf0 100644 --- a/drivers/iio/chemical/bme680_core.c +++ b/drivers/iio/chemical/bme680_core.c @@ -613,7 +613,7 @@ static int bme680_wait_for_eoc(struct bme680_data *data) * + heater duration */ int wait_eoc_us = ((data->oversampling_temp + data->oversampling_press + - data->oversampling_humid) * 1936) + (477 * 4) + + data->oversampling_humid) * 1963) + (477 * 4) + (477 * 5) + 1000 + (data->heater_dur * 1000); fsleep(wait_eoc_us); diff --git a/drivers/iio/chemical/scd4x.c b/drivers/iio/chemical/scd4x.c index 8859f89fb2a94..0fd839176e264 100644 --- a/drivers/iio/chemical/scd4x.c +++ b/drivers/iio/chemical/scd4x.c @@ -584,7 +584,7 @@ static const struct iio_chan_spec scd4x_channels[] = { .sign = 'u', .realbits = 16, .storagebits = 16, - .endianness = IIO_BE, + .endianness = IIO_CPU, }, }, { @@ -599,7 +599,7 @@ static const struct iio_chan_spec scd4x_channels[] = { .sign = 'u', .realbits = 16, .storagebits = 16, - .endianness = IIO_BE, + .endianness = IIO_CPU, }, }, { @@ -612,7 +612,7 @@ static const struct iio_chan_spec scd4x_channels[] = { .sign = 'u', .realbits = 16, .storagebits = 16, - .endianness = IIO_BE, + .endianness = IIO_CPU, }, }, }; diff --git a/drivers/iio/chemical/sps30_i2c.c b/drivers/iio/chemical/sps30_i2c.c index f692c089d17b4..c92f04990c34c 100644 --- a/drivers/iio/chemical/sps30_i2c.c +++ b/drivers/iio/chemical/sps30_i2c.c @@ -171,7 +171,7 @@ static int sps30_i2c_read_meas(struct sps30_state *state, __be32 *meas, size_t n if (!sps30_i2c_meas_ready(state)) return -ETIMEDOUT; - return sps30_i2c_command(state, SPS30_I2C_READ_MEAS, NULL, 0, meas, sizeof(num) * num); + return sps30_i2c_command(state, SPS30_I2C_READ_MEAS, NULL, 0, meas, sizeof(*meas) * num); } static int sps30_i2c_clean_fan(struct sps30_state *state) diff --git a/drivers/iio/chemical/sps30_serial.c b/drivers/iio/chemical/sps30_serial.c index 008bc88590f37..a5e6bc08d5fd4 100644 --- a/drivers/iio/chemical/sps30_serial.c +++ b/drivers/iio/chemical/sps30_serial.c @@ -303,7 +303,7 @@ static int sps30_serial_read_meas(struct sps30_state *state, __be32 *meas, size_ if (msleep_interruptible(1000)) return -EINTR; - ret = sps30_serial_command(state, SPS30_SERIAL_READ_MEAS, NULL, 0, meas, num * sizeof(num)); + ret = sps30_serial_command(state, SPS30_SERIAL_READ_MEAS, NULL, 0, meas, num * sizeof(*meas)); if (ret < 0) return ret; /* if measurements aren't ready sensor returns empty frame */ diff --git a/drivers/iio/dac/ad3552r-hs.c b/drivers/iio/dac/ad3552r-hs.c index 41b96b48ba98b..a9578afa7015d 100644 --- a/drivers/iio/dac/ad3552r-hs.c +++ b/drivers/iio/dac/ad3552r-hs.c @@ -549,12 +549,15 @@ static ssize_t ad3552r_hs_write_data_source(struct file *f, guard(mutex)(&st->lock); + if (count >= sizeof(buf)) + return -ENOSPC; + ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, userbuf, count); if (ret < 0) return ret; - buf[count] = '\0'; + buf[ret] = '\0'; ret = match_string(dbgfs_attr_source, ARRAY_SIZE(dbgfs_attr_source), buf); diff --git a/drivers/iio/dac/ad5686.c b/drivers/iio/dac/ad5686.c index d9cae9555e5df..4b18498aa0749 100644 --- a/drivers/iio/dac/ad5686.c +++ b/drivers/iio/dac/ad5686.c @@ -434,6 +434,12 @@ static const struct ad5686_chip_info ad5686_chip_info_tbl[] = { .num_channels = 4, .regmap_type = AD5686_REGMAP, }, + [ID_AD5695R] = { + .channels = ad5685r_channels, + .int_vref_mv = 2500, + .num_channels = 4, + .regmap_type = AD5686_REGMAP, + }, [ID_AD5696] = { .channels = ad5686_channels, .num_channels = 4, diff --git a/drivers/iio/dac/ad5770r.c b/drivers/iio/dac/ad5770r.c index cd47cb1c685c4..6027e8d88b278 100644 --- a/drivers/iio/dac/ad5770r.c +++ b/drivers/iio/dac/ad5770r.c @@ -322,7 +322,7 @@ static int ad5770r_read_raw(struct iio_dev *indio_dev, chan->address, st->transf_buf, 2); if (ret) - return 0; + return ret; buf16 = get_unaligned_le16(st->transf_buf); *val = buf16 >> 2; diff --git a/drivers/iio/dac/ds4424.c b/drivers/iio/dac/ds4424.c index a8198ba4f98a6..059acca45f64f 100644 --- a/drivers/iio/dac/ds4424.c +++ b/drivers/iio/dac/ds4424.c @@ -141,7 +141,7 @@ static int ds4424_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: - if (val < S8_MIN || val > S8_MAX) + if (val <= S8_MIN || val > S8_MAX) return -EINVAL; if (val > 0) { diff --git a/drivers/iio/frequency/adf4377.c b/drivers/iio/frequency/adf4377.c index 08833b7035e4b..48aa4b015a141 100644 --- a/drivers/iio/frequency/adf4377.c +++ b/drivers/iio/frequency/adf4377.c @@ -501,7 +501,7 @@ static int adf4377_soft_reset(struct adf4377_state *st) return ret; return regmap_read_poll_timeout(st->regmap, 0x0, read_val, - !(read_val & (ADF4377_0000_SOFT_RESET_R_MSK | + !(read_val & (ADF4377_0000_SOFT_RESET_MSK | ADF4377_0000_SOFT_RESET_R_MSK)), 200, 200 * 100); } diff --git a/drivers/iio/gyro/itg3200_buffer.c b/drivers/iio/gyro/itg3200_buffer.c index a624400a239cb..cf97adfa97274 100644 --- a/drivers/iio/gyro/itg3200_buffer.c +++ b/drivers/iio/gyro/itg3200_buffer.c @@ -118,11 +118,9 @@ int itg3200_probe_trigger(struct iio_dev *indio_dev) if (!st->trig) return -ENOMEM; - ret = request_irq(st->i2c->irq, - &iio_trigger_generic_data_rdy_poll, - IRQF_TRIGGER_RISING, - "itg3200_data_rdy", - st->trig); + ret = request_irq(st->i2c->irq, &iio_trigger_generic_data_rdy_poll, + IRQF_TRIGGER_RISING | IRQF_NO_THREAD, + "itg3200_data_rdy", st->trig); if (ret) goto error_free_trig; diff --git a/drivers/iio/gyro/itg3200_core.c b/drivers/iio/gyro/itg3200_core.c index cd8a2dae56cd9..bfe95ec1abda9 100644 --- a/drivers/iio/gyro/itg3200_core.c +++ b/drivers/iio/gyro/itg3200_core.c @@ -93,6 +93,8 @@ static int itg3200_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_RAW: reg = (u8)chan->address; ret = itg3200_read_reg_s16(indio_dev, reg, val); + if (ret) + return ret; return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: *val = 0; diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index 67ae7d1012bc2..d84e04e4b4314 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -322,7 +322,9 @@ static int mpu3050_read_raw(struct iio_dev *indio_dev, } case IIO_CHAN_INFO_RAW: /* Resume device */ - pm_runtime_get_sync(mpu3050->dev); + ret = pm_runtime_resume_and_get(mpu3050->dev); + if (ret) + return ret; mutex_lock(&mpu3050->lock); ret = mpu3050_set_8khz_samplerate(mpu3050); @@ -647,14 +649,20 @@ static irqreturn_t mpu3050_trigger_handler(int irq, void *p) static int mpu3050_buffer_preenable(struct iio_dev *indio_dev) { struct mpu3050 *mpu3050 = iio_priv(indio_dev); + int ret; - pm_runtime_get_sync(mpu3050->dev); + ret = pm_runtime_resume_and_get(mpu3050->dev); + if (ret) + return ret; /* Unless we have OUR trigger active, run at full speed */ - if (!mpu3050->hw_irq_trigger) - return mpu3050_set_8khz_samplerate(mpu3050); + if (!mpu3050->hw_irq_trigger) { + ret = mpu3050_set_8khz_samplerate(mpu3050); + if (ret) + pm_runtime_put_autosuspend(mpu3050->dev); + } - return 0; + return ret; } static int mpu3050_buffer_postdisable(struct iio_dev *indio_dev) @@ -1121,11 +1129,16 @@ static int mpu3050_trigger_probe(struct iio_dev *indio_dev, int irq) ret = iio_trigger_register(mpu3050->trig); if (ret) - return ret; + goto err_iio_trigger; indio_dev->trig = iio_trigger_get(mpu3050->trig); return 0; + +err_iio_trigger: + free_irq(mpu3050->irq, mpu3050->trig); + + return ret; } int mpu3050_common_probe(struct device *dev, @@ -1162,10 +1175,8 @@ int mpu3050_common_probe(struct device *dev, mpu3050->regs[1].supply = mpu3050_reg_vlogic; ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(mpu3050->regs), mpu3050->regs); - if (ret) { - dev_err(dev, "Cannot get regulators\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Cannot get regulators\n"); ret = mpu3050_power_up(mpu3050); if (ret) @@ -1215,12 +1226,6 @@ int mpu3050_common_probe(struct device *dev, goto err_power_down; } - ret = iio_device_register(indio_dev); - if (ret) { - dev_err(dev, "device register failed\n"); - goto err_cleanup_buffer; - } - dev_set_drvdata(dev, indio_dev); /* Check if we have an assigned IRQ to use as trigger */ @@ -1243,9 +1248,20 @@ int mpu3050_common_probe(struct device *dev, pm_runtime_use_autosuspend(dev); pm_runtime_put(dev); + ret = iio_device_register(indio_dev); + if (ret) { + dev_err(dev, "device register failed\n"); + goto err_iio_device_register; + } + return 0; -err_cleanup_buffer: +err_iio_device_register: + pm_runtime_get_sync(dev); + pm_runtime_put_noidle(dev); + pm_runtime_disable(dev); + if (irq) + free_irq(mpu3050->irq, mpu3050->trig); iio_triggered_buffer_cleanup(indio_dev); err_power_down: mpu3050_power_down(mpu3050); @@ -1258,13 +1274,13 @@ void mpu3050_common_remove(struct device *dev) struct iio_dev *indio_dev = dev_get_drvdata(dev); struct mpu3050 *mpu3050 = iio_priv(indio_dev); + iio_device_unregister(indio_dev); pm_runtime_get_sync(dev); pm_runtime_put_noidle(dev); pm_runtime_disable(dev); - iio_triggered_buffer_cleanup(indio_dev); if (mpu3050->irq) - free_irq(mpu3050->irq, mpu3050); - iio_device_unregister(indio_dev); + free_irq(mpu3050->irq, mpu3050->trig); + iio_triggered_buffer_cleanup(indio_dev); mpu3050_power_down(mpu3050); } diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c index 092878f2c8869..6549b22e643d8 100644 --- a/drivers/iio/gyro/mpu3050-i2c.c +++ b/drivers/iio/gyro/mpu3050-i2c.c @@ -19,8 +19,7 @@ static int mpu3050_i2c_bypass_select(struct i2c_mux_core *mux, u32 chan_id) struct mpu3050 *mpu3050 = i2c_mux_priv(mux); /* Just power up the device, that is all that is needed */ - pm_runtime_get_sync(mpu3050->dev); - return 0; + return pm_runtime_resume_and_get(mpu3050->dev); } static int mpu3050_i2c_bypass_deselect(struct i2c_mux_core *mux, u32 chan_id) diff --git a/drivers/iio/imu/adis.c b/drivers/iio/imu/adis.c index d160147cce0ba..a2bc1d14ed91d 100644 --- a/drivers/iio/imu/adis.c +++ b/drivers/iio/imu/adis.c @@ -526,7 +526,7 @@ int adis_init(struct adis *adis, struct iio_dev *indio_dev, adis->spi = spi; adis->data = data; - if (!adis->ops->write && !adis->ops->read && !adis->ops->reset) + if (!adis->ops) adis->ops = &adis_default_ops; else if (!adis->ops->write || !adis->ops->read || !adis->ops->reset) return -EINVAL; diff --git a/drivers/iio/imu/adis16550.c b/drivers/iio/imu/adis16550.c index 28f0dbd0226cb..1f2af506f4bdd 100644 --- a/drivers/iio/imu/adis16550.c +++ b/drivers/iio/imu/adis16550.c @@ -643,12 +643,12 @@ static int adis16550_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: switch (chan->type) { case IIO_ANGL_VEL: - ret = adis16550_get_accl_filter_freq(st, val); + ret = adis16550_get_gyro_filter_freq(st, val); if (ret) return ret; return IIO_VAL_INT; case IIO_ACCEL: - ret = adis16550_get_gyro_filter_freq(st, val); + ret = adis16550_get_accl_filter_freq(st, val); if (ret) return ret; return IIO_VAL_INT; @@ -681,9 +681,9 @@ static int adis16550_write_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: switch (chan->type) { case IIO_ANGL_VEL: - return adis16550_set_accl_filter_freq(st, val); - case IIO_ACCEL: return adis16550_set_gyro_filter_freq(st, val); + case IIO_ACCEL: + return adis16550_set_accl_filter_freq(st, val); default: return -EINVAL; } diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c index 5f47708b4c5dc..4abb83b75e2e8 100644 --- a/drivers/iio/imu/bmi160/bmi160_core.c +++ b/drivers/iio/imu/bmi160/bmi160_core.c @@ -573,12 +573,16 @@ static int bmi160_config_pin(struct regmap *regmap, enum bmi160_int_pin pin, int_out_ctrl_shift = BMI160_INT1_OUT_CTRL_SHIFT; int_latch_mask = BMI160_INT1_LATCH_MASK; int_map_mask = BMI160_INT1_MAP_DRDY_EN; + pin_name = "INT1"; break; case BMI160_PIN_INT2: int_out_ctrl_shift = BMI160_INT2_OUT_CTRL_SHIFT; int_latch_mask = BMI160_INT2_LATCH_MASK; int_map_mask = BMI160_INT2_MAP_DRDY_EN; + pin_name = "INT2"; break; + default: + return -EINVAL; } int_out_ctrl_mask = BMI160_INT_OUT_CTRL_MASK << int_out_ctrl_shift; @@ -612,17 +616,8 @@ static int bmi160_config_pin(struct regmap *regmap, enum bmi160_int_pin pin, ret = bmi160_write_conf_reg(regmap, BMI160_REG_INT_MAP, int_map_mask, int_map_mask, write_usleep); - if (ret) { - switch (pin) { - case BMI160_PIN_INT1: - pin_name = "INT1"; - break; - case BMI160_PIN_INT2: - pin_name = "INT2"; - break; - } + if (ret) dev_err(dev, "Failed to configure %s IRQ pin", pin_name); - } return ret; } diff --git a/drivers/iio/imu/bmi270/bmi270_i2c.c b/drivers/iio/imu/bmi270/bmi270_i2c.c index b909a421ad017..b92da4e0776fa 100644 --- a/drivers/iio/imu/bmi270/bmi270_i2c.c +++ b/drivers/iio/imu/bmi270/bmi270_i2c.c @@ -37,6 +37,7 @@ static const struct i2c_device_id bmi270_i2c_id[] = { { "bmi270", (kernel_ulong_t)&bmi270_chip_info }, { } }; +MODULE_DEVICE_TABLE(i2c, bmi270_i2c_id); static const struct acpi_device_id bmi270_acpi_match[] = { /* GPD Win Mini, Aya Neo AIR Pro, OXP Mini Pro, etc. */ @@ -45,12 +46,14 @@ static const struct acpi_device_id bmi270_acpi_match[] = { { "BMI0260", (kernel_ulong_t)&bmi260_chip_info }, { } }; +MODULE_DEVICE_TABLE(acpi, bmi270_acpi_match); static const struct of_device_id bmi270_of_match[] = { { .compatible = "bosch,bmi260", .data = &bmi260_chip_info }, { .compatible = "bosch,bmi270", .data = &bmi270_chip_info }, { } }; +MODULE_DEVICE_TABLE(of, bmi270_of_match); static struct i2c_driver bmi270_i2c_driver = { .driver = { diff --git a/drivers/iio/imu/bmi270/bmi270_spi.c b/drivers/iio/imu/bmi270/bmi270_spi.c index 19dd7734f9d07..80c9fa1d685ab 100644 --- a/drivers/iio/imu/bmi270/bmi270_spi.c +++ b/drivers/iio/imu/bmi270/bmi270_spi.c @@ -60,7 +60,7 @@ static int bmi270_spi_probe(struct spi_device *spi) &bmi270_spi_regmap_config); if (IS_ERR(regmap)) return dev_err_probe(dev, PTR_ERR(regmap), - "Failed to init i2c regmap"); + "Failed to init spi regmap\n"); return bmi270_core_probe(dev, regmap, chip_info); } diff --git a/drivers/iio/imu/bno055/bno055.c b/drivers/iio/imu/bno055/bno055.c index 303bc308f80a8..c96fec2ebb3e7 100644 --- a/drivers/iio/imu/bno055/bno055.c +++ b/drivers/iio/imu/bno055/bno055.c @@ -64,7 +64,7 @@ #define BNO055_GRAVITY_DATA_X_LSB_REG 0x2E #define BNO055_GRAVITY_DATA_Y_LSB_REG 0x30 #define BNO055_GRAVITY_DATA_Z_LSB_REG 0x32 -#define BNO055_SCAN_CH_COUNT ((BNO055_GRAVITY_DATA_Z_LSB_REG - BNO055_ACC_DATA_X_LSB_REG) / 2) +#define BNO055_SCAN_CH_COUNT ((BNO055_GRAVITY_DATA_Z_LSB_REG - BNO055_ACC_DATA_X_LSB_REG) / 2 + 1) #define BNO055_TEMP_REG 0x34 #define BNO055_CALIB_STAT_REG 0x35 #define BNO055_CALIB_STAT_MAGN_SHIFT 0 diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c index 54760d8f92a27..0ab6eddf0543f 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c @@ -651,6 +651,8 @@ static int inv_icm42600_accel_write_odr(struct iio_dev *indio_dev, return -EINVAL; conf.odr = inv_icm42600_accel_odr_conv[idx / 2]; + if (conf.odr == st->conf.accel.odr) + return 0; pm_runtime_get_sync(dev); mutex_lock(&st->lock); diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c index ada968be954d4..68a3957580318 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c @@ -371,6 +371,8 @@ static int inv_icm42600_buffer_predisable(struct iio_dev *indio_dev) static int inv_icm42600_buffer_postdisable(struct iio_dev *indio_dev) { struct inv_icm42600_state *st = iio_device_get_drvdata(indio_dev); + struct inv_icm42600_sensor_state *sensor_st = iio_priv(indio_dev); + struct inv_sensors_timestamp *ts = &sensor_st->ts; struct device *dev = regmap_get_device(st->map); unsigned int sensor; unsigned int *watermark; @@ -392,6 +394,8 @@ static int inv_icm42600_buffer_postdisable(struct iio_dev *indio_dev) mutex_lock(&st->lock); + inv_sensors_timestamp_apply_odr(ts, 0, 0, 0); + ret = inv_icm42600_buffer_set_fifo_en(st, st->fifo.en & ~sensor); if (ret) goto out_unlock; diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c index 7ef0a25ec74f6..11339ddf1da36 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c @@ -358,6 +358,8 @@ static int inv_icm42600_gyro_write_odr(struct iio_dev *indio_dev, return -EINVAL; conf.odr = inv_icm42600_gyro_odr_conv[idx / 2]; + if (conf.odr == st->conf.gyro.odr) + return 0; pm_runtime_get_sync(dev); mutex_lock(&st->lock); diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c index b2fa1f4957a5b..5796896d54cd8 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c @@ -1943,6 +1943,14 @@ int inv_mpu_core_probe(struct regmap *regmap, int irq, const char *name, irq_type); return -EINVAL; } + + /* + * Acking interrupts by status register does not work reliably + * but seem to work when this bit is set. + */ + if (st->chip_type == INV_MPU9150) + st->irq_mask |= INV_MPU6050_INT_RD_CLEAR; + device_set_wakeup_capable(dev, true); st->vdd_supply = devm_regulator_get(dev, "vdd"); diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h index 211901f8b8eb6..6239b1a803f77 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h @@ -390,6 +390,8 @@ struct inv_mpu6050_state { /* enable level triggering */ #define INV_MPU6050_LATCH_INT_EN 0x20 #define INV_MPU6050_BIT_BYPASS_EN 0x2 +/* allow acking interrupts by any register read */ +#define INV_MPU6050_INT_RD_CLEAR 0x10 /* Allowed timestamp period jitter in percent */ #define INV_MPU6050_TS_PERIOD_JITTER 4 diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c index 10a4733420759..22c1ce66f99ee 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c @@ -248,7 +248,6 @@ static irqreturn_t inv_mpu6050_interrupt_handle(int irq, void *p) switch (st->chip_type) { case INV_MPU6000: case INV_MPU6050: - case INV_MPU9150: /* * WoM is not supported and interrupt status read seems to be broken for * some chips. Since data ready is the only interrupt, bypass interrupt @@ -257,6 +256,10 @@ static irqreturn_t inv_mpu6050_interrupt_handle(int irq, void *p) wom_bits = 0; int_status = INV_MPU6050_BIT_RAW_DATA_RDY_INT; goto data_ready_interrupt; + case INV_MPU9150: + /* IRQ needs to be acked */ + wom_bits = 0; + break; case INV_MPU6500: case INV_MPU6515: case INV_MPU6880: diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h index 381b016fa5243..56244d49ab2fc 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h @@ -383,7 +383,7 @@ enum st_lsm6dsx_fifo_mode { * @id: Sensor identifier. * @hw: Pointer to instance of struct st_lsm6dsx_hw. * @gain: Configured sensor sensitivity. - * @odr: Output data rate of the sensor [Hz]. + * @odr: Output data rate of the sensor [mHz]. * @samples_to_discard: Number of samples to discard for filters settling time. * @watermark: Sensor watermark level. * @decimator: Sensor decimation factor. diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c index 8a9d2593576a2..07b81e523e635 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c @@ -224,6 +224,10 @@ static int st_lsm6dsx_set_fifo_odr(struct st_lsm6dsx_sensor *sensor, const struct st_lsm6dsx_reg *batch_reg; u8 data; + /* Only internal sensors have a FIFO ODR configuration register. */ + if (sensor->id >= ARRAY_SIZE(hw->settings->batch)) + return 0; + batch_reg = &hw->settings->batch[sensor->id]; if (batch_reg->addr) { int val; diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c index a2daf0c14d965..28ebb27d89249 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c @@ -101,6 +101,13 @@ static const struct iio_chan_spec st_lsm6dsx_acc_channels[] = { IIO_CHAN_SOFT_TIMESTAMP(3), }; +static const struct iio_chan_spec st_lsm6ds0_acc_channels[] = { + ST_LSM6DSX_CHANNEL(IIO_ACCEL, 0x28, IIO_MOD_X, 0), + ST_LSM6DSX_CHANNEL(IIO_ACCEL, 0x2a, IIO_MOD_Y, 1), + ST_LSM6DSX_CHANNEL(IIO_ACCEL, 0x2c, IIO_MOD_Z, 2), + IIO_CHAN_SOFT_TIMESTAMP(3), +}; + static const struct iio_chan_spec st_lsm6dsx_gyro_channels[] = { ST_LSM6DSX_CHANNEL(IIO_ANGL_VEL, 0x22, IIO_MOD_X, 0), ST_LSM6DSX_CHANNEL(IIO_ANGL_VEL, 0x24, IIO_MOD_Y, 1), @@ -142,8 +149,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = { }, .channels = { [ST_LSM6DSX_ID_ACC] = { - .chan = st_lsm6dsx_acc_channels, - .len = ARRAY_SIZE(st_lsm6dsx_acc_channels), + .chan = st_lsm6ds0_acc_channels, + .len = ARRAY_SIZE(st_lsm6ds0_acc_channels), }, [ST_LSM6DSX_ID_GYRO] = { .chan = st_lsm6ds0_gyro_channels, @@ -1449,8 +1456,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = { }, .channels = { [ST_LSM6DSX_ID_ACC] = { - .chan = st_lsm6dsx_acc_channels, - .len = ARRAY_SIZE(st_lsm6dsx_acc_channels), + .chan = st_lsm6ds0_acc_channels, + .len = ARRAY_SIZE(st_lsm6ds0_acc_channels), }, [ST_LSM6DSX_ID_GYRO] = { .chan = st_lsm6dsx_gyro_channels, diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c index 96ea0f039dfb1..c46a5f37ccc98 100644 --- a/drivers/iio/industrialio-buffer.c +++ b/drivers/iio/industrialio-buffer.c @@ -228,8 +228,10 @@ static ssize_t iio_buffer_write(struct file *filp, const char __user *buf, written = 0; add_wait_queue(&rb->pollq, &wait); do { - if (!indio_dev->info) - return -ENODEV; + if (!indio_dev->info) { + ret = -ENODEV; + break; + } if (!iio_buffer_space_available(rb)) { if (signal_pending(current)) { diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 88c3d585a1bd0..117ffad4f3769 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -1654,6 +1654,10 @@ static void iio_dev_release(struct device *device) iio_device_detach_buffers(indio_dev); + mutex_destroy(&iio_dev_opaque->info_exist_lock); + mutex_destroy(&iio_dev_opaque->mlock); + + lockdep_unregister_key(&iio_dev_opaque->info_exist_key); lockdep_unregister_key(&iio_dev_opaque->mlock_key); ida_free(&iio_ida, iio_dev_opaque->id); @@ -1694,12 +1698,6 @@ struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv) ACCESS_PRIVATE(indio_dev, priv) = (char *)iio_dev_opaque + ALIGN(sizeof(*iio_dev_opaque), IIO_DMA_MINALIGN); - indio_dev->dev.parent = parent; - indio_dev->dev.type = &iio_device_type; - indio_dev->dev.bus = &iio_bus_type; - device_initialize(&indio_dev->dev); - mutex_init(&iio_dev_opaque->mlock); - mutex_init(&iio_dev_opaque->info_exist_lock); INIT_LIST_HEAD(&iio_dev_opaque->channel_attr_list); iio_dev_opaque->id = ida_alloc(&iio_ida, GFP_KERNEL); @@ -1720,7 +1718,15 @@ struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv) INIT_LIST_HEAD(&iio_dev_opaque->ioctl_handlers); lockdep_register_key(&iio_dev_opaque->mlock_key); - lockdep_set_class(&iio_dev_opaque->mlock, &iio_dev_opaque->mlock_key); + lockdep_register_key(&iio_dev_opaque->info_exist_key); + + mutex_init_with_key(&iio_dev_opaque->mlock, &iio_dev_opaque->mlock_key); + mutex_init_with_key(&iio_dev_opaque->info_exist_lock, &iio_dev_opaque->info_exist_key); + + indio_dev->dev.parent = parent; + indio_dev->dev.type = &iio_device_type; + indio_dev->dev.bus = &iio_bus_type; + device_initialize(&indio_dev->dev); return indio_dev; } diff --git a/drivers/iio/light/bh1780.c b/drivers/iio/light/bh1780.c index 5d3c6d5276bac..a740d1f992a8a 100644 --- a/drivers/iio/light/bh1780.c +++ b/drivers/iio/light/bh1780.c @@ -109,9 +109,9 @@ static int bh1780_read_raw(struct iio_dev *indio_dev, case IIO_LIGHT: pm_runtime_get_sync(&bh1780->client->dev); value = bh1780_read_word(bh1780, BH1780_REG_DLOW); + pm_runtime_put_autosuspend(&bh1780->client->dev); if (value < 0) return value; - pm_runtime_put_autosuspend(&bh1780->client->dev); *val = value; return IIO_VAL_INT; diff --git a/drivers/iio/light/si1145.c b/drivers/iio/light/si1145.c index f8eb251eca8dc..ef0abc4499b74 100644 --- a/drivers/iio/light/si1145.c +++ b/drivers/iio/light/si1145.c @@ -1248,7 +1248,7 @@ static int si1145_probe_trigger(struct iio_dev *indio_dev) ret = devm_request_irq(&client->dev, client->irq, iio_trigger_generic_data_rdy_poll, - IRQF_TRIGGER_FALLING, + IRQF_TRIGGER_FALLING | IRQF_NO_THREAD, "si1145_irq", trig); if (ret < 0) { diff --git a/drivers/iio/light/vcnl4035.c b/drivers/iio/light/vcnl4035.c index 963747927425f..16aeb17067bc0 100644 --- a/drivers/iio/light/vcnl4035.c +++ b/drivers/iio/light/vcnl4035.c @@ -103,17 +103,23 @@ static irqreturn_t vcnl4035_trigger_consumer_handler(int irq, void *p) struct iio_dev *indio_dev = pf->indio_dev; struct vcnl4035_data *data = iio_priv(indio_dev); /* Ensure naturally aligned timestamp */ - u8 buffer[ALIGN(sizeof(u16), sizeof(s64)) + sizeof(s64)] __aligned(8) = { }; + struct { + u16 als_data; + aligned_s64 timestamp; + } buffer = { }; + unsigned int val; int ret; - ret = regmap_read(data->regmap, VCNL4035_ALS_DATA, (int *)buffer); + ret = regmap_read(data->regmap, VCNL4035_ALS_DATA, &val); if (ret < 0) { dev_err(&data->client->dev, "Trigger consumer can't read from sensor.\n"); goto fail_read; } - iio_push_to_buffers_with_timestamp(indio_dev, buffer, - iio_get_time_ns(indio_dev)); + + buffer.als_data = val; + iio_push_to_buffers_with_timestamp(indio_dev, &buffer, + iio_get_time_ns(indio_dev)); fail_read: iio_trigger_notify_done(indio_dev->trig); @@ -381,7 +387,7 @@ static const struct iio_chan_spec vcnl4035_channels[] = { .sign = 'u', .realbits = 16, .storagebits = 16, - .endianness = IIO_LE, + .endianness = IIO_CPU, }, }, { @@ -395,7 +401,7 @@ static const struct iio_chan_spec vcnl4035_channels[] = { .sign = 'u', .realbits = 16, .storagebits = 16, - .endianness = IIO_LE, + .endianness = IIO_CPU, }, }, }; diff --git a/drivers/iio/light/veml6070.c b/drivers/iio/light/veml6070.c index 6d4483c85f30c..74d7246e5225e 100644 --- a/drivers/iio/light/veml6070.c +++ b/drivers/iio/light/veml6070.c @@ -134,9 +134,7 @@ static int veml6070_read(struct veml6070_data *data) if (ret < 0) return ret; - ret = (msb << 8) | lsb; - - return 0; + return (msb << 8) | lsb; } static const struct iio_chan_spec veml6070_channels[] = { diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c index 3fd0171e5d69b..d30315ad85ded 100644 --- a/drivers/iio/magnetometer/ak8975.c +++ b/drivers/iio/magnetometer/ak8975.c @@ -581,7 +581,7 @@ static int ak8975_setup_irq(struct ak8975_data *data) irq = gpiod_to_irq(data->eoc_gpiod); rc = devm_request_irq(&client->dev, irq, ak8975_irq_handler, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_TRIGGER_RISING, dev_name(&client->dev), data); if (rc < 0) { dev_err(&client->dev, "irq %d request failed: %d\n", irq, rc); diff --git a/drivers/iio/magnetometer/tlv493d.c b/drivers/iio/magnetometer/tlv493d.c index ec53fd40277b0..e5e050af2b74c 100644 --- a/drivers/iio/magnetometer/tlv493d.c +++ b/drivers/iio/magnetometer/tlv493d.c @@ -171,7 +171,7 @@ static s16 tlv493d_get_channel_data(u8 *b, enum tlv493d_channels ch) switch (ch) { case TLV493D_AXIS_X: val = FIELD_GET(TLV493D_BX_MAG_X_AXIS_MSB, b[TLV493D_RD_REG_BX]) << 4 | - FIELD_GET(TLV493D_BX2_MAG_X_AXIS_LSB, b[TLV493D_RD_REG_BX2]) >> 4; + FIELD_GET(TLV493D_BX2_MAG_X_AXIS_LSB, b[TLV493D_RD_REG_BX2]); break; case TLV493D_AXIS_Y: val = FIELD_GET(TLV493D_BY_MAG_Y_AXIS_MSB, b[TLV493D_RD_REG_BY]) << 4 | diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c index e759f91a710a2..5a5e6e4fbe34b 100644 --- a/drivers/iio/orientation/hid-sensor-rotation.c +++ b/drivers/iio/orientation/hid-sensor-rotation.c @@ -19,8 +19,13 @@ struct dev_rot_state { struct hid_sensor_common common_attributes; struct hid_sensor_hub_attribute_info quaternion; struct { - s32 sampled_vals[4]; - aligned_s64 timestamp; + IIO_DECLARE_QUATERNION(s32, sampled_vals); + /* + * ABI regression avoidance: There are two copies of the same + * timestamp in case of userspace depending on broken alignment + * from older kernels. + */ + aligned_s64 timestamp[2]; } scan; int scale_pre_decml; int scale_post_decml; @@ -154,8 +159,19 @@ static int dev_rot_proc_event(struct hid_sensor_hub_device *hsdev, if (!rot_state->timestamp) rot_state->timestamp = iio_get_time_ns(indio_dev); - iio_push_to_buffers_with_timestamp(indio_dev, &rot_state->scan, - rot_state->timestamp); + /* + * ABI regression avoidance: IIO previously had an incorrect + * implementation of iio_push_to_buffers_with_timestamp() that + * put the timestamp in the last 8 bytes of the buffer, which + * was incorrect according to the IIO ABI. To avoid breaking + * userspace that may be depending on this broken behavior, we + * put the timestamp in both the correct place [0] and the old + * incorrect place [1]. + */ + rot_state->scan.timestamp[0] = rot_state->timestamp; + rot_state->scan.timestamp[1] = rot_state->timestamp; + + iio_push_to_buffers(indio_dev, &rot_state->scan); rot_state->timestamp = 0; } diff --git a/drivers/iio/potentiometer/mcp4131.c b/drivers/iio/potentiometer/mcp4131.c index ad082827aad5e..56c9111ef5e81 100644 --- a/drivers/iio/potentiometer/mcp4131.c +++ b/drivers/iio/potentiometer/mcp4131.c @@ -221,7 +221,7 @@ static int mcp4131_write_raw(struct iio_dev *indio_dev, mutex_lock(&data->lock); - data->buf[0] = address << MCP4131_WIPER_SHIFT; + data->buf[0] = address; data->buf[0] |= MCP4131_WRITE | (val >> 8); data->buf[1] = val & 0xFF; /* 8 bits here */ diff --git a/drivers/iio/pressure/mprls0025pa.c b/drivers/iio/pressure/mprls0025pa.c index 2336f2760eaeb..d4133fef91fac 100644 --- a/drivers/iio/pressure/mprls0025pa.c +++ b/drivers/iio/pressure/mprls0025pa.c @@ -59,7 +59,7 @@ * * Values given to the userspace in sysfs interface: * * raw - press_cnt - * * offset - (-1 * outputmin) - pmin / scale + * * offset - (-1 * outputmin) + pmin / scale * note: With all sensors from the datasheet pmin = 0 * which reduces the offset to (-1 * outputmin) */ @@ -160,8 +160,8 @@ static const struct iio_chan_spec mpr_channels[] = { BIT(IIO_CHAN_INFO_OFFSET), .scan_index = 0, .scan_type = { - .sign = 's', - .realbits = 32, + .sign = 'u', + .realbits = 24, .storagebits = 32, .endianness = IIO_CPU, }, @@ -313,8 +313,7 @@ static int mpr_read_raw(struct iio_dev *indio_dev, return IIO_VAL_INT_PLUS_NANO; case IIO_CHAN_INFO_OFFSET: *val = data->offset; - *val2 = data->offset2; - return IIO_VAL_INT_PLUS_NANO; + return IIO_VAL_INT; default: return -EINVAL; } @@ -330,8 +329,9 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) struct mpr_data *data; struct iio_dev *indio_dev; const char *triplet; - s64 scale, offset; + s64 odelta, pdelta; u32 func; + s32 tmp; indio_dev = devm_iio_device_alloc(dev, sizeof(*data)); if (!indio_dev) @@ -405,23 +405,17 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) data->outmin = mpr_func_spec[data->function].output_min; data->outmax = mpr_func_spec[data->function].output_max; - /* use 64 bit calculation for preserving a reasonable precision */ - scale = div_s64(((s64)(data->pmax - data->pmin)) * NANO, - data->outmax - data->outmin); - data->scale = div_s64_rem(scale, NANO, &data->scale2); - /* - * multiply with NANO before dividing by scale and later divide by NANO - * again. - */ - offset = ((-1LL) * (s64)data->outmin) * NANO - - div_s64(div_s64((s64)data->pmin * NANO, scale), NANO); - data->offset = div_s64_rem(offset, NANO, &data->offset2); + odelta = data->outmax - data->outmin; + pdelta = data->pmax - data->pmin; + + data->scale = div_s64_rem(div_s64(pdelta * NANO, odelta), NANO, &tmp); + data->scale2 = tmp; + + data->offset = div_s64(odelta * data->pmin, pdelta) - data->outmin; if (data->irq > 0) { - ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, - IRQF_TRIGGER_RISING, - dev_name(dev), - data); + ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, 0, + dev_name(dev), data); if (ret) return dev_err_probe(dev, ret, "request irq %d failed\n", data->irq); diff --git a/drivers/iio/pressure/mprls0025pa.h b/drivers/iio/pressure/mprls0025pa.h index d62a018eaff32..b6944b3051267 100644 --- a/drivers/iio/pressure/mprls0025pa.h +++ b/drivers/iio/pressure/mprls0025pa.h @@ -53,7 +53,6 @@ enum mpr_func_id { * @scale: pressure scale * @scale2: pressure scale, decimal number * @offset: pressure offset - * @offset2: pressure offset, decimal number * @gpiod_reset: reset * @irq: end of conversion irq. used to distinguish between irq mode and * reading in a loop until data is ready @@ -75,7 +74,6 @@ struct mpr_data { int scale; int scale2; int offset; - int offset2; struct gpio_desc *gpiod_reset; int irq; struct completion completion; diff --git a/drivers/iio/pressure/mprls0025pa_spi.c b/drivers/iio/pressure/mprls0025pa_spi.c index d04102f8a4a03..cf17eb2e72083 100644 --- a/drivers/iio/pressure/mprls0025pa_spi.c +++ b/drivers/iio/pressure/mprls0025pa_spi.c @@ -8,6 +8,7 @@ * https://prod-edam.honeywell.com/content/dam/honeywell-edam/sps/siot/en-us/products/sensors/pressure-sensors/board-mount-pressure-sensors/micropressure-mpr-series/documents/sps-siot-mpr-series-datasheet-32332628-ciid-172626.pdf */ +#include #include #include #include @@ -40,17 +41,25 @@ static int mpr_spi_xfer(struct mpr_data *data, const u8 cmd, const u8 pkt_len) { struct spi_device *spi = to_spi_device(data->dev); struct mpr_spi_buf *buf = spi_get_drvdata(spi); - struct spi_transfer xfer; + struct spi_transfer xfers[2] = { }; if (pkt_len > MPR_MEASUREMENT_RD_SIZE) return -EOVERFLOW; buf->tx[0] = cmd; - xfer.tx_buf = buf->tx; - xfer.rx_buf = data->buffer; - xfer.len = pkt_len; - return spi_sync_transfer(spi, &xfer, 1); + /* + * Dummy transfer with no data, just cause a 2.5us+ delay between the CS assert + * and the first clock edge as per the datasheet tHDSS timing requirement. + */ + xfers[0].delay.value = 2500; + xfers[0].delay.unit = SPI_DELAY_UNIT_NSECS; + + xfers[1].tx_buf = buf->tx; + xfers[1].rx_buf = data->buffer; + xfers[1].len = pkt_len; + + return spi_sync_transfer(spi, xfers, ARRAY_SIZE(xfers)); } static const struct mpr_ops mpr_spi_ops = { diff --git a/drivers/iio/proximity/hx9023s.c b/drivers/iio/proximity/hx9023s.c index 2918dfc0df547..17e00ee2b6f84 100644 --- a/drivers/iio/proximity/hx9023s.c +++ b/drivers/iio/proximity/hx9023s.c @@ -719,6 +719,9 @@ static int hx9023s_set_samp_freq(struct hx9023s_data *data, int val, int val2) struct device *dev = regmap_get_device(data->regmap); unsigned int i, period_ms; + if (!val && !val2) + return -EINVAL; + period_ms = div_u64(NANO, (val * MEGA + val2)); for (i = 0; i < ARRAY_SIZE(hx9023s_samp_freq_table); i++) { @@ -1034,9 +1037,8 @@ static int hx9023s_send_cfg(const struct firmware *fw, struct hx9023s_data *data if (!bin) return -ENOMEM; - memcpy(bin->data, fw->data, fw->size); - bin->fw_size = fw->size; + memcpy(bin->data, fw->data, bin->fw_size); bin->fw_ver = bin->data[FW_VER_OFFSET]; bin->reg_count = get_unaligned_le16(bin->data + FW_REG_CNT_OFFSET); diff --git a/drivers/iio/test/Kconfig b/drivers/iio/test/Kconfig index 6e65e929791ca..4fc17dd0dcd77 100644 --- a/drivers/iio/test/Kconfig +++ b/drivers/iio/test/Kconfig @@ -8,7 +8,6 @@ config IIO_GTS_KUNIT_TEST tristate "Test IIO gain-time-scale helpers" if !KUNIT_ALL_TESTS depends on KUNIT select IIO_GTS_HELPER - select TEST_KUNIT_DEVICE_HELPERS default KUNIT_ALL_TESTS help build unit tests for the IIO light sensor gain-time-scale helpers. diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 61596cda2b65f..35ba852a172aa 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -80,37 +80,25 @@ static const struct nla_policy ib_nl_addr_policy[LS_NLA_TYPE_MAX] = { .min = sizeof(struct rdma_nla_ls_gid)}, }; -static inline bool ib_nl_is_good_ip_resp(const struct nlmsghdr *nlh) +static void ib_nl_process_ip_rsep(const struct nlmsghdr *nlh) { struct nlattr *tb[LS_NLA_TYPE_MAX] = {}; + union ib_gid gid; + struct addr_req *req; + int found = 0; int ret; if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR) - return false; + return; ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh), nlmsg_len(nlh), ib_nl_addr_policy, NULL); if (ret) - return false; - - return true; -} - -static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh) -{ - const struct nlattr *head, *curr; - union ib_gid gid; - struct addr_req *req; - int len, rem; - int found = 0; - - head = (const struct nlattr *)nlmsg_data(nlh); - len = nlmsg_len(nlh); + return; - nla_for_each_attr(curr, head, len, rem) { - if (curr->nla_type == LS_NLA_TYPE_DGID) - memcpy(&gid, nla_data(curr), nla_len(curr)); - } + if (!tb[LS_NLA_TYPE_DGID]) + return; + memcpy(&gid, nla_data(tb[LS_NLA_TYPE_DGID]), sizeof(gid)); spin_lock_bh(&lock); list_for_each_entry(req, &req_list, list) { @@ -137,8 +125,7 @@ int ib_nl_handle_ip_res_resp(struct sk_buff *skb, !(NETLINK_CB(skb).sk)) return -EPERM; - if (ib_nl_is_good_ip_resp(nlh)) - ib_nl_process_good_ip_rsep(nlh); + ib_nl_process_ip_rsep(nlh); return 0; } diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 81cf3c902e819..78bc7d83edc65 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -927,6 +927,13 @@ static int gid_table_setup_one(struct ib_device *ib_dev) if (err) return err; + /* + * Mark the device as ready for GID cache updates. This allows netdev + * event handlers to update the GID cache even before the device is + * fully registered. + */ + ib_device_enable_gid_updates(ib_dev); + rdma_roce_rescan_device(ib_dev); return err; @@ -1537,7 +1544,8 @@ static void ib_cache_event_task(struct work_struct *_work) * the cache. */ ret = ib_cache_update(work->event.device, work->event.element.port_num, - work->event.event == IB_EVENT_GID_CHANGE, + work->event.event == IB_EVENT_GID_CHANGE || + work->event.event == IB_EVENT_CLIENT_REREGISTER, work->event.event == IB_EVENT_PKEY_CHANGE, work->enforce_security); @@ -1638,6 +1646,12 @@ void ib_cache_release_one(struct ib_device *device) void ib_cache_cleanup_one(struct ib_device *device) { + /* + * Clear the GID updates mark first to prevent event handlers from + * accessing the device while it's being torn down. + */ + ib_device_disable_gid_updates(device); + /* The cleanup function waits for all in-progress workqueue * elements and cleans up the GID cache. This function should be * called after the device was removed from the devices list and diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 5b2d3ae3f9fce..ce511800b056c 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -2009,6 +2009,7 @@ static void destroy_mc(struct rdma_id_private *id_priv, ib_sa_free_multicast(mc->sa_mc); if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) { + struct rdma_cm_event *event = &mc->iboe_join.event; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct net_device *ndev = NULL; @@ -2031,6 +2032,8 @@ static void destroy_mc(struct rdma_id_private *id_priv, dev_put(ndev); cancel_work_sync(&mc->iboe_join.work); + if (event->event == RDMA_CM_EVENT_MULTICAST_JOIN) + rdma_destroy_ah_attr(&event->param.ud.ah_attr); } kfree(mc); } diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index 05102769a918a..a2c36666e6fcb 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -100,6 +100,9 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter, roce_netdev_callback cb, void *cookie); +void ib_device_enable_gid_updates(struct ib_device *device); +void ib_device_disable_gid_updates(struct ib_device *device); + typedef int (*nldev_callback)(struct ib_device *device, struct sk_buff *skb, struct netlink_callback *cb, diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index b4f3c835844a9..ac9aaef1e5e61 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -93,6 +93,7 @@ static struct workqueue_struct *ib_unreg_wq; static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC); static DECLARE_RWSEM(devices_rwsem); #define DEVICE_REGISTERED XA_MARK_1 +#define DEVICE_GID_UPDATES XA_MARK_2 static u32 highest_client_id; #define CLIENT_REGISTERED XA_MARK_1 @@ -2441,11 +2442,42 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter, unsigned long index; down_read(&devices_rwsem); - xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) + xa_for_each_marked(&devices, index, dev, DEVICE_GID_UPDATES) ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); up_read(&devices_rwsem); } +/** + * ib_device_enable_gid_updates - Mark device as ready for GID cache updates + * @device: Device to mark + * + * Called after GID table is allocated and initialized. After this mark is set, + * netdevice event handlers can update the device's GID cache. This allows + * events that arrive during device registration to be processed, avoiding + * stale GID entries when netdev properties change during the device + * registration process. + */ +void ib_device_enable_gid_updates(struct ib_device *device) +{ + down_write(&devices_rwsem); + xa_set_mark(&devices, device->index, DEVICE_GID_UPDATES); + up_write(&devices_rwsem); +} + +/** + * ib_device_disable_gid_updates - Clear the GID updates mark + * @device: Device to unmark + * + * Called before GID table cleanup to prevent event handlers from accessing + * the device while it's being torn down. + */ +void ib_device_disable_gid_updates(struct ib_device *device) +{ + down_write(&devices_rwsem); + xa_clear_mark(&devices, device->index, DEVICE_GID_UPDATES); + up_write(&devices_rwsem); +} + /* * ib_enum_all_devs - enumerate all ib_devices * @cb: Callback to call for each found ib_device @@ -2881,8 +2913,10 @@ int ib_del_sub_device_and_put(struct ib_device *sub) { struct ib_device *parent = sub->parent; - if (!parent) + if (!parent) { + ib_device_put(sub); return -EOPNOTSUPP; + } mutex_lock(&parent->subdev_lock); list_del(&sub->subdev_list); diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 62410578dec37..eb942ab9c4055 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -95,7 +95,6 @@ static struct workqueue_struct *iwcm_wq; struct iwcm_work { struct work_struct work; struct iwcm_id_private *cm_id; - struct list_head list; struct iw_cm_event event; struct list_head free_list; }; @@ -178,7 +177,6 @@ static int alloc_work_entries(struct iwcm_id_private *cm_id_priv, int count) return -ENOMEM; } work->cm_id = cm_id_priv; - INIT_LIST_HEAD(&work->list); put_work(work); } return 0; @@ -213,7 +211,6 @@ static void free_cm_id(struct iwcm_id_private *cm_id_priv) static bool iwcm_deref_id(struct iwcm_id_private *cm_id_priv) { if (refcount_dec_and_test(&cm_id_priv->refcount)) { - BUG_ON(!list_empty(&cm_id_priv->work_list)); free_cm_id(cm_id_priv); return true; } @@ -260,7 +257,6 @@ struct iw_cm_id *iw_create_cm_id(struct ib_device *device, refcount_set(&cm_id_priv->refcount, 1); init_waitqueue_head(&cm_id_priv->connect_wait); init_completion(&cm_id_priv->destroy_comp); - INIT_LIST_HEAD(&cm_id_priv->work_list); INIT_LIST_HEAD(&cm_id_priv->work_free_list); return &cm_id_priv->id; @@ -1007,13 +1003,13 @@ static int process_event(struct iwcm_id_private *cm_id_priv, } /* - * Process events on the work_list for the cm_id. If the callback - * function requests that the cm_id be deleted, a flag is set in the - * cm_id flags to indicate that when the last reference is - * removed, the cm_id is to be destroyed. This is necessary to - * distinguish between an object that will be destroyed by the app - * thread asleep on the destroy_comp list vs. an object destroyed - * here synchronously when the last reference is removed. + * Process events for the cm_id. If the callback function requests + * that the cm_id be deleted, a flag is set in the cm_id flags to + * indicate that when the last reference is removed, the cm_id is + * to be destroyed. This is necessary to distinguish between an + * object that will be destroyed by the app thread asleep on the + * destroy_comp list vs. an object destroyed here synchronously + * when the last reference is removed. */ static void cm_work_handler(struct work_struct *_work) { @@ -1024,35 +1020,26 @@ static void cm_work_handler(struct work_struct *_work) int ret = 0; spin_lock_irqsave(&cm_id_priv->lock, flags); - while (!list_empty(&cm_id_priv->work_list)) { - work = list_first_entry(&cm_id_priv->work_list, - struct iwcm_work, list); - list_del_init(&work->list); - levent = work->event; - put_work(work); - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - - if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { - ret = process_event(cm_id_priv, &levent); - if (ret) { - destroy_cm_id(&cm_id_priv->id); - WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); - } - } else - pr_debug("dropping event %d\n", levent.event); - if (iwcm_deref_id(cm_id_priv)) - return; - spin_lock_irqsave(&cm_id_priv->lock, flags); - } + levent = work->event; + put_work(work); spin_unlock_irqrestore(&cm_id_priv->lock, flags); + + if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { + ret = process_event(cm_id_priv, &levent); + if (ret) { + destroy_cm_id(&cm_id_priv->id); + WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); + } + } else + pr_debug("dropping event %d\n", levent.event); + if (iwcm_deref_id(cm_id_priv)) + return; } /* * This function is called on interrupt context. Schedule events on * the iwcm_wq thread to allow callback functions to downcall into - * the CM and/or block. Events are queued to a per-CM_ID - * work_list. If this is the first event on the work_list, the work - * element is also queued on the iwcm_wq thread. + * the CM and/or block. * * Each event holds a reference on the cm_id. Until the last posted * event has been delivered and processed, the cm_id cannot be @@ -1094,7 +1081,6 @@ static int cm_event_handler(struct iw_cm_id *cm_id, } refcount_inc(&cm_id_priv->refcount); - list_add_tail(&work->list, &cm_id_priv->work_list); queue_work(iwcm_wq, &work->work); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); diff --git a/drivers/infiniband/core/iwcm.h b/drivers/infiniband/core/iwcm.h index bf74639be1287..b56fb12edece4 100644 --- a/drivers/infiniband/core/iwcm.h +++ b/drivers/infiniband/core/iwcm.h @@ -50,7 +50,6 @@ struct iwcm_id_private { struct ib_qp *qp; struct completion destroy_comp; wait_queue_head_t connect_wait; - struct list_head work_list; spinlock_t lock; refcount_t refcount; struct list_head work_free_list; diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c index 6354ddf2a274c..49fbfe1cef689 100644 --- a/drivers/infiniband/core/rw.c +++ b/drivers/infiniband/core/rw.c @@ -326,14 +326,29 @@ int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, if (rdma_rw_io_needs_mr(qp->device, port_num, dir, sg_cnt)) { ret = rdma_rw_init_mr_wrs(ctx, qp, port_num, sg, sg_cnt, sg_offset, remote_addr, rkey, dir); - } else if (sg_cnt > 1) { + /* + * If MR init succeeded or failed for a reason other + * than pool exhaustion, that result is final. + * + * Pool exhaustion (-EAGAIN) from the max_sgl_rd + * optimization is recoverable: fall back to + * direct SGE posting. iWARP and force_mr require + * MRs unconditionally, so -EAGAIN is terminal. + */ + if (ret != -EAGAIN || + rdma_protocol_iwarp(qp->device, port_num) || + unlikely(rdma_rw_force_mr)) + goto out; + } + + if (sg_cnt > 1) ret = rdma_rw_init_map_wrs(ctx, qp, sg, sg_cnt, sg_offset, remote_addr, rkey, dir); - } else { + else ret = rdma_rw_init_single_wr(ctx, qp, sg, sg_offset, remote_addr, rkey, dir); - } +out: if (ret < 0) goto out_unmap_sg; return ret; @@ -651,34 +666,57 @@ unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, } EXPORT_SYMBOL(rdma_rw_mr_factor); +/** + * rdma_rw_max_send_wr - compute max Send WRs needed for RDMA R/W contexts + * @dev: RDMA device + * @port_num: port number + * @max_rdma_ctxs: number of rdma_rw_ctx structures + * @create_flags: QP create flags (pass IB_QP_CREATE_INTEGRITY_EN if + * data integrity will be enabled on the QP) + * + * Returns the total number of Send Queue entries needed for + * @max_rdma_ctxs. The result accounts for memory registration and + * invalidation work requests when the device requires them. + * + * ULPs use this to size Send Queues and Send CQs before creating a + * Queue Pair. + */ +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags) +{ + unsigned int factor = 1; + unsigned int result; + + if (create_flags & IB_QP_CREATE_INTEGRITY_EN || + rdma_rw_can_use_mr(dev, port_num)) + factor += 2; /* reg + inv */ + + if (check_mul_overflow(factor, max_rdma_ctxs, &result)) + return UINT_MAX; + return result; +} +EXPORT_SYMBOL(rdma_rw_max_send_wr); + void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr) { - u32 factor; + unsigned int factor = 1; WARN_ON_ONCE(attr->port_num == 0); /* - * Each context needs at least one RDMA READ or WRITE WR. - * - * For some hardware we might need more, eventually we should ask the - * HCA driver for a multiplier here. - */ - factor = 1; - - /* - * If the device needs MRs to perform RDMA READ or WRITE operations, - * we'll need two additional MRs for the registrations and the - * invalidation. + * If the device uses MRs to perform RDMA READ or WRITE operations, + * or if data integrity is enabled, account for registration and + * invalidation work requests. */ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN || rdma_rw_can_use_mr(dev, attr->port_num)) - factor += 2; /* inv + reg */ + factor += 2; /* reg + inv */ attr->cap.max_send_wr += factor * attr->cap.max_rdma_ctxs; /* - * But maybe we were just too high in the sky and the device doesn't - * even support all we need, and we'll have to live with what we get.. + * The device might not support all we need, and we'll have to + * live with what we get. */ attr->cap.max_send_wr = min_t(u32, attr->cap.max_send_wr, dev->attrs.max_qp_wr); diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c index 0ec2e4120cc94..17b16fe0e49d9 100644 --- a/drivers/infiniband/core/umem_dmabuf.c +++ b/drivers/infiniband/core/umem_dmabuf.c @@ -221,13 +221,11 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, err = ib_umem_dmabuf_map_pages(umem_dmabuf); if (err) - goto err_unpin; + goto err_release; dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); return umem_dmabuf; -err_unpin: - dma_buf_unpin(umem_dmabuf->attach); err_release: dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); ib_umem_release(&umem_dmabuf->umem); diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index fd67fc9fe85a4..2f7e3c4483fc5 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -514,7 +514,8 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, struct rdma_ah_attr ah_attr; struct ib_ah *ah; __be64 *tid; - int ret, data_len, hdr_len, copy_offset, rmpp_active; + int ret, hdr_len, copy_offset, rmpp_active; + size_t data_len; u8 base_version; if (count < hdr_size(file) + IB_MGMT_RMPP_HDR) @@ -588,7 +589,10 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, } base_version = ((struct ib_mad_hdr *)&packet->mad.data)->base_version; - data_len = count - hdr_size(file) - hdr_len; + if (check_sub_overflow(count, hdr_size(file) + hdr_len, &data_len)) { + ret = -EINVAL; + goto err_ah; + } packet->msg = ib_create_send_mad(agent, be32_to_cpu(packet->mad.hdr.qpn), packet->mad.hdr.pkey_index, rmpp_active, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index ce16404cdfb8c..f4616deeca545 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -2049,7 +2049,10 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) if (ret) return ret; - user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); + if (cmd.wqe_size < sizeof(struct ib_uverbs_send_wr)) + return -EINVAL; + + user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return -ENOMEM; @@ -2239,7 +2242,7 @@ ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count, if (ret) return ERR_PTR(ret); - user_wr = kmalloc(wqe_size, GFP_KERNEL); + user_wr = kmalloc(wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return ERR_PTR(-ENOMEM); diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 3a5f81402d2f8..d279e301f5a1a 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -735,7 +735,7 @@ int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr, (struct in6_addr *)dgid); return 0; } else if (net_type == RDMA_NETWORK_IPV6 || - net_type == RDMA_NETWORK_IB || RDMA_NETWORK_ROCE_V1) { + net_type == RDMA_NETWORK_IB || net_type == RDMA_NETWORK_ROCE_V1) { *dgid = hdr->ibgrh.dgid; *sgid = hdr->ibgrh.sgid; return 0; diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.h b/drivers/infiniband/hw/bnxt_re/hw_counters.h index 09d371d442aa7..cebec033f4a01 100644 --- a/drivers/infiniband/hw/bnxt_re/hw_counters.h +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.h @@ -89,6 +89,9 @@ enum bnxt_re_hw_stats { BNXT_RE_RES_SRQ_LOAD_ERR, BNXT_RE_RES_TX_PCI_ERR, BNXT_RE_RES_RX_PCI_ERR, + BNXT_RE_REQ_CQE_ERROR, + BNXT_RE_RESP_CQE_ERROR, + BNXT_RE_RESP_REMOTE_ACCESS_ERRS, BNXT_RE_OUT_OF_SEQ_ERR, BNXT_RE_TX_ATOMIC_REQ, BNXT_RE_TX_READ_REQ, @@ -110,9 +113,6 @@ enum bnxt_re_hw_stats { BNXT_RE_TX_CNP, BNXT_RE_RX_CNP, BNXT_RE_RX_ECN, - BNXT_RE_REQ_CQE_ERROR, - BNXT_RE_RESP_CQE_ERROR, - BNXT_RE_RESP_REMOTE_ACCESS_ERRS, BNXT_RE_NUM_EXT_COUNTERS }; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 84ce3fce2826b..ff91511bd3389 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -601,7 +601,8 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) mr->qplib_mr.va = (u64)(unsigned long)fence->va; mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, - BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE); + BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE, + _is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)); if (rc) { ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n"); goto fail; @@ -2918,14 +2919,9 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, wqe.rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; } - switch (wr->send_flags) { - case IB_SEND_IP_CSUM: + if (wr->send_flags & IB_SEND_IP_CSUM) wqe.rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; - break; - default: - break; - } fallthrough; case IB_WR_SEND_WITH_INV: rc = bnxt_re_build_send_wqe(qp, wr, &wqe); @@ -4027,7 +4023,7 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) mr->qplib_mr.hwq.level = PBL_LVL_MAX; mr->qplib_mr.total_size = -1; /* Infinte length */ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0, - PAGE_SIZE); + PAGE_SIZE, false); if (rc) goto fail_mr; @@ -4257,7 +4253,8 @@ static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 umem_pgs = ib_umem_num_dma_blocks(umem, page_size); rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem, - umem_pgs, page_size); + umem_pgs, page_size, + _is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)); if (rc) { ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc); rc = -EIO; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index 295a9610f3e67..4dad0cfcfa98d 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -1112,7 +1112,7 @@ static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt) creq_db->dbinfo.flags = 0; creq_db->reg.bar_id = RCFW_COMM_CONS_PCI_BAR_REGION; creq_db->reg.bar_base = pci_resource_start(pdev, creq_db->reg.bar_id); - if (!creq_db->reg.bar_id) + if (!creq_db->reg.bar_base) dev_err(&pdev->dev, "QPLIB: CREQ BAR region %d resc start is 0!", creq_db->reg.bar_id); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 875d7b52c06ab..4d674a3aee1aa 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -64,9 +64,7 @@ static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl, for (i = 0; i < pbl->pg_count; i++) { if (pbl->pg_arr[i]) dma_free_coherent(&pdev->dev, pbl->pg_size, - (void *)((unsigned long) - pbl->pg_arr[i] & - PAGE_MASK), + pbl->pg_arr[i], pbl->pg_map_arr[i]); else dev_warn(&pdev->dev, @@ -237,7 +235,7 @@ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, if (npbl % BIT(MAX_PDL_LVL_SHIFT)) npde++; /* Alloc PDE pages */ - sginfo.pgsize = npde * pg_size; + sginfo.pgsize = npde * ROCE_PG_SIZE_4K; sginfo.npages = 1; rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo); if (rc) @@ -245,7 +243,7 @@ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, /* Alloc PBL pages */ sginfo.npages = npbl; - sginfo.pgsize = PAGE_SIZE; + sginfo.pgsize = ROCE_PG_SIZE_4K; rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo); if (rc) goto fail; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c index 9ef581ed785c8..408a34df26672 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -162,7 +162,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; attr->max_srq_sges = sb->max_srq_sge; attr->max_pkey = 1; - attr->max_inline_data = le32_to_cpu(sb->max_inline_data); + attr->max_inline_data = attr->max_qp_sges * sizeof(struct sq_sge); if (!bnxt_qplib_is_chip_gen_p7(rcfw->res->cctx)) attr->l2_db_size = (sb->l2_db_space_size + 1) * (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); @@ -578,7 +578,7 @@ int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, } int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, - struct ib_umem *umem, int num_pbls, u32 buf_pg_size) + struct ib_umem *umem, int num_pbls, u32 buf_pg_size, bool unified_mr) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct bnxt_qplib_hwq_attr hwq_attr = {}; @@ -640,7 +640,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, req.access = (mr->access_flags & BNXT_QPLIB_MR_ACCESS_MASK); req.va = cpu_to_le64(mr->va); req.key = cpu_to_le32(mr->lkey); - if (_is_alloc_mr_unified(res->dattr->dev_cap_flags)) + if (unified_mr) req.key = cpu_to_le32(mr->pd->id); req.flags = cpu_to_le16(mr->flags); req.mr_size = cpu_to_le64(mr->total_size); @@ -651,7 +651,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, if (rc) goto fail; - if (_is_alloc_mr_unified(res->dattr->dev_cap_flags)) { + if (unified_mr) { mr->lkey = le32_to_cpu(resp.xid); mr->rkey = mr->lkey; } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index 147b5d9c03138..5a45c55c6464c 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -341,7 +341,7 @@ int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, bool block); int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, - struct ib_umem *umem, int num_pbls, u32 buf_pg_size); + struct ib_umem *umem, int num_pbls, u32 buf_pg_size, bool unified_mr); int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, int max); diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index 0e979ca10d240..e97b5f0d70038 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* - * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ +#include + #include "efa_com.h" #include "efa_regs_defs.h" @@ -21,6 +23,8 @@ #define EFA_CTRL_SUB_MINOR 1 enum efa_cmd_status { + EFA_CMD_UNUSED, + EFA_CMD_ALLOCATED, EFA_CMD_SUBMITTED, EFA_CMD_COMPLETED, }; @@ -32,7 +36,6 @@ struct efa_comp_ctx { enum efa_cmd_status status; u16 cmd_id; u8 cmd_opcode; - u8 occupied; }; static const char *efa_com_cmd_str(u8 cmd) @@ -241,7 +244,6 @@ static int efa_com_admin_init_aenq(struct efa_com_dev *edev, return 0; } -/* ID to be used with efa_com_get_comp_ctx */ static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq) { u16 ctx_id; @@ -263,48 +265,59 @@ static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, spin_unlock(&aq->comp_ctx_lock); } -static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq, - struct efa_comp_ctx *comp_ctx) +static struct efa_comp_ctx *efa_com_alloc_comp_ctx(struct efa_com_admin_queue *aq) { - u16 cmd_id = EFA_GET(&comp_ctx->user_cqe->acq_common_descriptor.command, - EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); - u16 ctx_id = cmd_id & (aq->depth - 1); + struct efa_comp_ctx *comp_ctx; + u16 ctx_id; - ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id); - comp_ctx->occupied = 0; - efa_com_dealloc_ctx_id(aq, ctx_id); + ctx_id = efa_com_alloc_ctx_id(aq); + + comp_ctx = &aq->comp_ctx[ctx_id]; + if (comp_ctx->status != EFA_CMD_UNUSED) { + efa_com_dealloc_ctx_id(aq, ctx_id); + ibdev_err_ratelimited(aq->efa_dev, + "Completion context[%u] is used[%u]\n", + ctx_id, comp_ctx->status); + return NULL; + } + + comp_ctx->status = EFA_CMD_ALLOCATED; + ibdev_dbg(aq->efa_dev, "Take completion context[%u]\n", ctx_id); + return comp_ctx; } -static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq, - u16 cmd_id, bool capture) +static inline u16 efa_com_get_comp_ctx_id(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx) { - u16 ctx_id = cmd_id & (aq->depth - 1); + return comp_ctx - aq->comp_ctx; +} - if (aq->comp_ctx[ctx_id].occupied && capture) { - ibdev_err_ratelimited( - aq->efa_dev, - "Completion context for command_id %#x is occupied\n", - cmd_id); - return NULL; - } +static inline void efa_com_dealloc_comp_ctx(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx) +{ + u16 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); - if (capture) { - aq->comp_ctx[ctx_id].occupied = 1; - ibdev_dbg(aq->efa_dev, - "Take completion ctxt for command_id %#x\n", cmd_id); - } + ibdev_dbg(aq->efa_dev, "Put completion context[%u]\n", ctx_id); + comp_ctx->status = EFA_CMD_UNUSED; + efa_com_dealloc_ctx_id(aq, ctx_id); +} + +static inline struct efa_comp_ctx *efa_com_get_comp_ctx_by_cmd_id(struct efa_com_admin_queue *aq, + u16 cmd_id) +{ + u16 ctx_id = cmd_id & (aq->depth - 1); return &aq->comp_ctx[ctx_id]; } -static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, - struct efa_admin_aq_entry *cmd, - size_t cmd_size_in_bytes, - struct efa_admin_acq_entry *comp, - size_t comp_size_in_bytes) +static void __efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx, + struct efa_admin_aq_entry *cmd, + size_t cmd_size_in_bytes, + struct efa_admin_acq_entry *comp, + size_t comp_size_in_bytes) { struct efa_admin_aq_entry *aqe; - struct efa_comp_ctx *comp_ctx; u16 queue_size_mask; u16 cmd_id; u16 ctx_id; @@ -312,24 +325,17 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu queue_size_mask = aq->depth - 1; pi = aq->sq.pc & queue_size_mask; - - ctx_id = efa_com_alloc_ctx_id(aq); + ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); /* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */ cmd_id = ctx_id & queue_size_mask; - cmd_id |= aq->sq.pc & ~queue_size_mask; + cmd_id |= aq->sq.pc << ilog2(aq->depth); cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; cmd->aq_common_descriptor.command_id = cmd_id; EFA_SET(&cmd->aq_common_descriptor.flags, EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase); - comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true); - if (!comp_ctx) { - efa_com_dealloc_ctx_id(aq, ctx_id); - return ERR_PTR(-EINVAL); - } - comp_ctx->status = EFA_CMD_SUBMITTED; comp_ctx->comp_size = comp_size_in_bytes; comp_ctx->user_cqe = comp; @@ -350,8 +356,6 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu /* barrier not needed in case of writel */ writel(aq->sq.pc, aq->sq.db_addr); - - return comp_ctx; } static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) @@ -370,9 +374,9 @@ static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) } for (i = 0; i < aq->depth; i++) { - comp_ctx = efa_com_get_comp_ctx(aq, i, false); - if (comp_ctx) - init_completion(&comp_ctx->wait_event); + comp_ctx = &aq->comp_ctx[i]; + comp_ctx->status = EFA_CMD_UNUSED; + init_completion(&comp_ctx->wait_event); aq->comp_ctx_pool[i] = i; } @@ -384,28 +388,25 @@ static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) return 0; } -static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, - struct efa_admin_aq_entry *cmd, - size_t cmd_size_in_bytes, - struct efa_admin_acq_entry *comp, - size_t comp_size_in_bytes) +static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, + struct efa_comp_ctx *comp_ctx, + struct efa_admin_aq_entry *cmd, + size_t cmd_size_in_bytes, + struct efa_admin_acq_entry *comp, + size_t comp_size_in_bytes) { - struct efa_comp_ctx *comp_ctx; - spin_lock(&aq->sq.lock); if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); spin_unlock(&aq->sq.lock); - return ERR_PTR(-ENODEV); + return -ENODEV; } - comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp, - comp_size_in_bytes); + __efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size_in_bytes, comp, + comp_size_in_bytes); spin_unlock(&aq->sq.lock); - if (IS_ERR(comp_ctx)) - clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - return comp_ctx; + return 0; } static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, @@ -417,11 +418,12 @@ static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); - comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false); - if (comp_ctx->status != EFA_CMD_SUBMITTED) { + comp_ctx = efa_com_get_comp_ctx_by_cmd_id(aq, cmd_id); + if (comp_ctx->status != EFA_CMD_SUBMITTED || comp_ctx->cmd_id != cmd_id) { ibdev_err(aq->efa_dev, - "Received completion with unexpected command id[%d], sq producer: %d, sq consumer: %d, cq consumer: %d\n", - cmd_id, aq->sq.pc, aq->sq.cc, aq->cq.cc); + "Received completion with unexpected command id[%x], status[%d] sq producer[%d], sq consumer[%d], cq consumer[%d]\n", + cmd_id, comp_ctx->status, aq->sq.pc, aq->sq.cc, + aq->cq.cc); return -EINVAL; } @@ -501,7 +503,6 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c { unsigned long timeout; unsigned long flags; - int err; timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); @@ -521,24 +522,20 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c atomic64_inc(&aq->stats.no_completion); clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - err = -ETIME; - goto out; + return -ETIME; } msleep(aq->poll_interval); } - err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); -out: - efa_com_put_comp_ctx(aq, comp_ctx); - return err; + return efa_com_comp_status_to_errno( + comp_ctx->user_cqe->acq_common_descriptor.status); } static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, struct efa_com_admin_queue *aq) { unsigned long flags; - int err; wait_for_completion_timeout(&comp_ctx->wait_event, usecs_to_jiffies(aq->completion_timeout)); @@ -574,14 +571,11 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com aq->cq.cc); clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - err = -ETIME; - goto out; + return -ETIME; } - err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); -out: - efa_com_put_comp_ctx(aq, comp_ctx); - return err; + return efa_com_comp_status_to_errno( + comp_ctx->user_cqe->acq_common_descriptor.status); } /* @@ -631,30 +625,39 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq, ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", efa_com_cmd_str(cmd->aq_common_descriptor.opcode), cmd->aq_common_descriptor.opcode); - comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size); - if (IS_ERR(comp_ctx)) { + + comp_ctx = efa_com_alloc_comp_ctx(aq); + if (!comp_ctx) { + clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); + up(&aq->avail_cmds); + return -EINVAL; + } + + err = efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size, comp, comp_size); + if (err) { ibdev_err_ratelimited( aq->efa_dev, - "Failed to submit command %s (opcode %u) err %pe\n", + "Failed to submit command %s (opcode %u) err %d\n", efa_com_cmd_str(cmd->aq_common_descriptor.opcode), - cmd->aq_common_descriptor.opcode, comp_ctx); + cmd->aq_common_descriptor.opcode, err); + efa_com_dealloc_comp_ctx(aq, comp_ctx); up(&aq->avail_cmds); atomic64_inc(&aq->stats.cmd_err); - return PTR_ERR(comp_ctx); + return err; } err = efa_com_wait_and_process_admin_cq(comp_ctx, aq); if (err) { ibdev_err_ratelimited( aq->efa_dev, - "Failed to process command %s (opcode %u) comp_status %d err %d\n", + "Failed to process command %s (opcode %u) err %d\n", efa_com_cmd_str(cmd->aq_common_descriptor.opcode), - cmd->aq_common_descriptor.opcode, - comp_ctx->user_cqe->acq_common_descriptor.status, err); + cmd->aq_common_descriptor.opcode, err); atomic64_inc(&aq->stats.cmd_err); } + efa_com_dealloc_comp_ctx(aq, comp_ctx); up(&aq->avail_cmds); return err; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 22d3e25c3b9d1..5cab7dd70aebf 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1320,13 +1320,9 @@ static int umem_to_page_list(struct efa_dev *dev, u32 hp_cnt, u8 hp_shift) { - u32 pages_in_hp = BIT(hp_shift - PAGE_SHIFT); struct ib_block_iter biter; unsigned int hp_idx = 0; - ibdev_dbg(&dev->ibdev, "hp_cnt[%u], pages_in_hp[%u]\n", - hp_cnt, pages_in_hp); - rdma_umem_for_each_dma_block(umem, &biter, BIT(hp_shift)) page_list[hp_idx++] = rdma_block_iter_dma_address(&biter); @@ -1667,7 +1663,7 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, struct efa_mr *mr; if (udata && udata->inlen && - !ib_is_udata_cleared(udata, 0, sizeof(udata->inlen))) { + !ib_is_udata_cleared(udata, 0, udata->inlen)) { ibdev_dbg(&dev->ibdev, "Incompatible ABI params, udata not cleared\n"); return ERR_PTR(-EINVAL); diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 307c35888b300..3b6c6a6e9f977 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -61,7 +61,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, u8 tclass = get_tclass(grh); u8 priority = 0; u8 tc_mode = 0; - int ret; + int ret = 0; if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata) { ret = -EOPNOTSUPP; @@ -78,19 +78,18 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, ah->av.flowlabel = grh->flow_label; ah->av.udp_sport = get_ah_udp_sport(ah_attr); ah->av.tclass = tclass; + ah->av.sl = rdma_ah_get_sl(ah_attr); - ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); - if (ret == -EOPNOTSUPP) - ret = 0; - - if (ret && grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - goto err_out; + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); + if (ret == -EOPNOTSUPP) + ret = 0; + else if (ret) + goto err_out; - if (tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - ah->av.sl = priority; - else - ah->av.sl = rdma_ah_get_sl(ah_attr); + if (tc_mode == HNAE3_TC_MAP_MODE_DSCP) + ah->av.sl = priority; + } if (!check_sl_valid(hr_dev, ah->av.sl)) { ret = -EINVAL; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 63052c0e76133..f895731ad74a0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3661,6 +3661,23 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); } +static bool left_sw_wc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct hns_roce_qp *hr_qp; + + list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { + if (hr_qp->sq.head != hr_qp->sq.tail) + return true; + } + + list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { + if (hr_qp->rq.head != hr_qp->rq.tail) + return true; + } + + return false; +} + static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { @@ -3669,6 +3686,12 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, struct hns_roce_v2_db cq_db = {}; u32 notify_flag; + if (hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN) { + if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && + left_sw_wc(hr_dev, hr_cq)) + return 1; + return 0; + } /* * flags = 0, then notify_flag : next * flags = 1, then notify flag : solocited @@ -4975,20 +4998,22 @@ static int hns_roce_set_sl(struct ib_qp *ibqp, struct ib_device *ibdev = &hr_dev->ib_dev; int ret; - ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh), - &hr_qp->tc_mode, &hr_qp->priority); - if (ret && ret != -EOPNOTSUPP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { - ibdev_err_ratelimited(ibdev, - "failed to get dscp, ret = %d.\n", ret); - return ret; - } + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); - if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - hr_qp->sl = hr_qp->priority; - else - hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hns_roce_hw_v2_get_dscp(hr_dev, + get_tclass(&attr->ah_attr.grh), + &hr_qp->tc_mode, &hr_qp->priority); + if (ret && ret != -EOPNOTSUPP) { + ibdev_err_ratelimited(ibdev, + "failed to get dscp, ret = %d.\n", + ret); + return ret; + } + + if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP) + hr_qp->sl = hr_qp->priority; + } if (!check_sl_valid(hr_dev, hr_qp->sl)) return -EINVAL; @@ -6878,7 +6903,8 @@ static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work); - hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); + hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", + WQ_MEM_RECLAIM); if (!hr_dev->irq_workq) { dev_err(dev, "failed to create irq workqueue.\n"); ret = -ENOMEM; diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index ea12d9b8e125f..38d57bc2ba525 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -508,6 +508,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, { const struct ib_global_route *grh; enum rdma_network_type net; + u8 smac[ETH_ALEN]; u16 vlan; int rc; @@ -518,7 +519,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, grh = rdma_ah_read_grh(attr); - rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, &hdr->eth.smac_h[0]); + rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac); if (rc) return rc; @@ -536,6 +537,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, if (rc) return rc; + ether_addr_copy(hdr->eth.smac_h, smac); ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac); if (net == RDMA_NETWORK_IPV4) { @@ -1218,7 +1220,7 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); struct ionic_tbl_buf buf = {}; - struct ionic_cq_resp resp; + struct ionic_cq_resp resp = {}; struct ionic_cq_req req; int udma_idx = 0, rc; diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index 164046d00e5d4..bd4c73e530d08 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -81,6 +81,8 @@ static int ionic_query_port(struct ib_device *ibdev, u32 port, return -EINVAL; ndev = ib_device_get_netdev(ibdev, port); + if (!ndev) + return -ENODEV; if (netif_running(ndev) && netif_carrier_ok(ndev)) { attr->state = IB_PORT_ACTIVE; diff --git a/drivers/infiniband/hw/irdma/cm.c b/drivers/infiniband/hw/irdma/cm.c index c6a0a661d6e7e..d14a381beb661 100644 --- a/drivers/infiniband/hw/irdma/cm.c +++ b/drivers/infiniband/hw/irdma/cm.c @@ -2241,11 +2241,12 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, int oldarpindex; int arpindex; struct net_device *netdev = iwdev->netdev; + int ret; /* create an hte and cm_node for this instance */ cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); if (!cm_node) - return NULL; + return ERR_PTR(-ENOMEM); /* set our node specific transport info */ cm_node->ipv4 = cm_info->ipv4; @@ -2348,8 +2349,10 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, arpindex = -EINVAL; } - if (arpindex < 0) + if (arpindex < 0) { + ret = -EINVAL; goto err; + } ether_addr_copy(cm_node->rem_mac, iwdev->rf->arp_table[arpindex].mac_addr); @@ -2360,7 +2363,7 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, err: kfree(cm_node); - return NULL; + return ERR_PTR(ret); } static void irdma_destroy_connection(struct irdma_cm_node *cm_node) @@ -3021,8 +3024,8 @@ static int irdma_create_cm_node(struct irdma_cm_core *cm_core, /* create a CM connection node */ cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL); - if (!cm_node) - return -ENOMEM; + if (IS_ERR(cm_node)) + return PTR_ERR(cm_node); /* set our node side to client (active) side */ cm_node->tcp_cntxt.client = 1; @@ -3219,9 +3222,9 @@ void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf) cm_info.cm_id = listener->cm_id; cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, listener); - if (!cm_node) { + if (IS_ERR(cm_node)) { ibdev_dbg(&cm_core->iwdev->ibdev, - "CM: allocate node failed\n"); + "CM: allocate node failed ret=%ld\n", PTR_ERR(cm_node)); refcount_dec(&listener->refcnt); return; } @@ -3710,7 +3713,7 @@ int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) iwpd = iwqp->iwpd; tagged_offset = (uintptr_t)iwqp->ietf_mem.va; ibmr = irdma_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len, - IB_ACCESS_LOCAL_WRITE, &tagged_offset); + IB_ACCESS_LOCAL_WRITE, &tagged_offset, false); if (IS_ERR(ibmr)) { ret = -ENOMEM; goto error; @@ -4239,21 +4242,21 @@ static void irdma_cm_event_handler(struct work_struct *work) irdma_cm_event_reset(event); break; case IRDMA_CM_EVENT_CONNECTED: - if (!event->cm_node->cm_id || - event->cm_node->state != IRDMA_CM_STATE_OFFLOADED) + if (!cm_node->cm_id || + cm_node->state != IRDMA_CM_STATE_OFFLOADED) break; irdma_cm_event_connected(event); break; case IRDMA_CM_EVENT_MPA_REJECT: - if (!event->cm_node->cm_id || + if (!cm_node->cm_id || cm_node->state == IRDMA_CM_STATE_OFFLOADED) break; irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); break; case IRDMA_CM_EVENT_ABORTED: - if (!event->cm_node->cm_id || - event->cm_node->state == IRDMA_CM_STATE_OFFLOADED) + if (!cm_node->cm_id || + cm_node->state == IRDMA_CM_STATE_OFFLOADED) break; irdma_event_connect_error(event); break; @@ -4263,7 +4266,7 @@ static void irdma_cm_event_handler(struct work_struct *work) break; } - irdma_rem_ref_cm_node(event->cm_node); + irdma_rem_ref_cm_node(cm_node); kfree(event); } diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c index 4ef1c29032f77..cb970c9090ee0 100644 --- a/drivers/infiniband/hw/irdma/ctrl.c +++ b/drivers/infiniband/hw/irdma/ctrl.c @@ -3950,11 +3950,13 @@ int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp) */ void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq) { + unsigned long flags; u64 temp_val; u16 sw_cq_sel; u8 arm_next_se; u8 arm_seq_num; + spin_lock_irqsave(&ccq->dev->cqp_lock, flags); get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val); sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); @@ -3965,6 +3967,7 @@ void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq) FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1); set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val); + spin_unlock_irqrestore(&ccq->dev->cqp_lock, flags); dma_wmb(); /* make sure shadow area is updated before arming */ @@ -4731,7 +4734,8 @@ static int irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch, u64 hdr; dev = aeq->dev; - if (dev->privileged) + + if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]); cqp = dev->cqp; diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c index 27b191f61caf4..b49fd9cf24765 100644 --- a/drivers/infiniband/hw/irdma/icrdma_if.c +++ b/drivers/infiniband/hw/irdma/icrdma_if.c @@ -302,7 +302,8 @@ static int icrdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary err_ctrl_init: icrdma_deinit_interrupts(rf, cdev_info); err_init_interrupts: - kfree(iwdev->rf); + mutex_destroy(&rf->ah_tbl_lock); + kfree(rf); ib_dealloc_device(&iwdev->ibdev); return err; @@ -319,6 +320,9 @@ static void icrdma_remove(struct auxiliary_device *aux_dev) ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false); irdma_ib_unregister_device(iwdev); icrdma_deinit_interrupts(iwdev->rf, cdev_info); + mutex_destroy(&iwdev->rf->ah_tbl_lock); + + kfree(iwdev->rf); pr_debug("INIT: Gen[%d] func[%d] device remove success\n", rdma_ver, PCI_FUNC(cdev_info->pdev->devfn)); diff --git a/drivers/infiniband/hw/irdma/ig3rdma_if.c b/drivers/infiniband/hw/irdma/ig3rdma_if.c index 1bb42eb298ba6..e1d6670d9396e 100644 --- a/drivers/infiniband/hw/irdma/ig3rdma_if.c +++ b/drivers/infiniband/hw/irdma/ig3rdma_if.c @@ -55,6 +55,7 @@ static int ig3rdma_vchnl_init(struct irdma_pci_f *rf, ret = irdma_sc_vchnl_init(&rf->sc_dev, &virt_info); if (ret) { destroy_workqueue(rf->vchnl_wq); + mutex_destroy(&rf->sc_dev.vchnl_mutex); return ret; } @@ -124,7 +125,9 @@ static void ig3rdma_decfg_rf(struct irdma_pci_f *rf) { struct irdma_hw *hw = &rf->hw; + mutex_destroy(&rf->ah_tbl_lock); destroy_workqueue(rf->vchnl_wq); + mutex_destroy(&rf->sc_dev.vchnl_mutex); kfree(hw->io_regs); iounmap(hw->rdma_reg.addr); } @@ -149,6 +152,7 @@ static int ig3rdma_cfg_rf(struct irdma_pci_f *rf, err = ig3rdma_cfg_regions(&rf->hw, cdev_info); if (err) { destroy_workqueue(rf->vchnl_wq); + mutex_destroy(&rf->sc_dev.vchnl_mutex); return err; } diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h index 886b30da188ae..65ce4924dbfa6 100644 --- a/drivers/infiniband/hw/irdma/main.h +++ b/drivers/infiniband/hw/irdma/main.h @@ -556,7 +556,7 @@ void irdma_copy_ip_htonl(__be32 *dst, u32 *src); u16 irdma_get_vlan_ipv4(u32 *addr); void irdma_get_vlan_mac_ipv6(u32 *addr, u16 *vlan_id, u8 *mac); struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size, - int acc, u64 *iova_start); + int acc, u64 *iova_start, bool dma_mr); int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw); void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq); int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c index fa6325adaedec..28dfad7f940c2 100644 --- a/drivers/infiniband/hw/irdma/pble.c +++ b/drivers/infiniband/hw/irdma/pble.c @@ -506,12 +506,14 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, struct irdma_pble_alloc *palloc) { - pble_rsrc->freedpbles += palloc->total_cnt; - if (palloc->level == PBLE_LEVEL_2) free_lvl2(pble_rsrc, palloc); else irdma_prm_return_pbles(&pble_rsrc->pinfo, &palloc->level1.chunkinfo); + + mutex_lock(&pble_rsrc->pble_mutex_lock); + pble_rsrc->freedpbles += palloc->total_cnt; pble_rsrc->stats_alloc_freed++; + mutex_unlock(&pble_rsrc->pble_mutex_lock); } diff --git a/drivers/infiniband/hw/irdma/puda.c b/drivers/infiniband/hw/irdma/puda.c index 694e5a9ed15d0..9cd14a50f1a93 100644 --- a/drivers/infiniband/hw/irdma/puda.c +++ b/drivers/infiniband/hw/irdma/puda.c @@ -685,7 +685,6 @@ static int irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc) ukqp->rq_size = rsrc->rq_size; IRDMA_RING_INIT(ukqp->sq_ring, ukqp->sq_size); - IRDMA_RING_INIT(ukqp->initial_ring, ukqp->sq_size); IRDMA_RING_INIT(ukqp->rq_ring, ukqp->rq_size); ukqp->wqe_alloc_db = qp->pd->dev->wqe_alloc_db; diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c index ce1ae10c30fca..4cc81d61be7fa 100644 --- a/drivers/infiniband/hw/irdma/uk.c +++ b/drivers/infiniband/hw/irdma/uk.c @@ -114,33 +114,8 @@ void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx) */ void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp) { - u64 temp; - u32 hw_sq_tail; - u32 sw_sq_head; - - /* valid bit is written and loads completed before reading shadow */ - mb(); - - /* read the doorbell shadow area */ - get_64bit_val(qp->shadow_area, 0, &temp); - - hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp); - sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); - if (sw_sq_head != qp->initial_ring.head) { - if (sw_sq_head != hw_sq_tail) { - if (sw_sq_head > qp->initial_ring.head) { - if (hw_sq_tail >= qp->initial_ring.head && - hw_sq_tail < sw_sq_head) - writel(qp->qp_id, qp->wqe_alloc_db); - } else { - if (hw_sq_tail >= qp->initial_ring.head || - hw_sq_tail < sw_sq_head) - writel(qp->qp_id, qp->wqe_alloc_db); - } - } - } - - qp->initial_ring.head = qp->sq_ring.head; + dma_wmb(); + writel(qp->qp_id, qp->wqe_alloc_db); } /** @@ -1435,7 +1410,7 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, * irdma_round_up_wq - return round up qp wq depth * @wqdepth: wq depth in quanta to round up */ -static int irdma_round_up_wq(u32 wqdepth) +static u64 irdma_round_up_wq(u64 wqdepth) { int scount = 1; @@ -1488,15 +1463,16 @@ void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth) { - u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; + u64 hw_quanta = + irdma_round_up_wq(((u64)sq_size << shift) + IRDMA_SQ_RSVD); - *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD); - - if (*sqdepth < min_size) - *sqdepth = min_size; - else if (*sqdepth > uk_attrs->max_hw_wq_quanta) + if (hw_quanta < min_hw_quanta) + hw_quanta = min_hw_quanta; + else if (hw_quanta > uk_attrs->max_hw_wq_quanta) return -EINVAL; + *sqdepth = hw_quanta; return 0; } @@ -1510,15 +1486,16 @@ int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth) { - u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; - - *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD); + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; + u64 hw_quanta = + irdma_round_up_wq(((u64)rq_size << shift) + IRDMA_RQ_RSVD); - if (*rqdepth < min_size) - *rqdepth = min_size; - else if (*rqdepth > uk_attrs->max_hw_rq_quanta) + if (hw_quanta < min_hw_quanta) + hw_quanta = min_hw_quanta; + else if (hw_quanta > uk_attrs->max_hw_rq_quanta) return -EINVAL; + *rqdepth = hw_quanta; return 0; } @@ -1532,13 +1509,16 @@ int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth) { - *srqdepth = irdma_round_up_wq((srq_size << shift) + IRDMA_RQ_RSVD); + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; + u64 hw_quanta = + irdma_round_up_wq(((u64)srq_size << shift) + IRDMA_RQ_RSVD); - if (*srqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) - *srqdepth = uk_attrs->min_hw_wq_size << shift; - else if (*srqdepth > uk_attrs->max_hw_srq_quanta) + if (hw_quanta < min_hw_quanta) + hw_quanta = min_hw_quanta; + else if (hw_quanta > uk_attrs->max_hw_srq_quanta) return -EINVAL; + *srqdepth = hw_quanta; return 0; } @@ -1574,7 +1554,6 @@ static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp, qp->conn_wqes = move_cnt; IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt); IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt); - IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt); } /** @@ -1719,7 +1698,6 @@ int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) qp->max_sq_frag_cnt = info->max_sq_frag_cnt; sq_ring_size = qp->sq_size << info->sq_shift; IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); - IRDMA_RING_INIT(qp->initial_ring, sq_ring_size); if (info->first_sq_wq) { irdma_setup_connection_wqes(qp, info); qp->swqe_polarity = 1; diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h index ab57f689827a0..aeebf768174ab 100644 --- a/drivers/infiniband/hw/irdma/user.h +++ b/drivers/infiniband/hw/irdma/user.h @@ -456,7 +456,6 @@ struct irdma_srq_uk { struct irdma_uk_attrs *uk_attrs; __le64 *shadow_area; struct irdma_ring srq_ring; - struct irdma_ring initial_ring; u32 srq_id; u32 srq_size; u32 max_srq_frag_cnt; diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 8b94d87b01923..e5e226b346211 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -251,7 +251,7 @@ int irdma_net_event(struct notifier_block *notifier, unsigned long event, void *ptr) { struct neighbour *neigh = ptr; - struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; + struct net_device *real_dev, *netdev; struct irdma_device *iwdev; struct ib_device *ibdev; __be32 *p; @@ -260,6 +260,7 @@ int irdma_net_event(struct notifier_block *notifier, unsigned long event, switch (event) { case NETEVENT_NEIGH_UPDATE: + netdev = neigh->dev; real_dev = rdma_vlan_dev_real_dev(netdev); if (!real_dev) real_dev = netdev; @@ -2324,8 +2325,6 @@ void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) struct irdma_qp *qp = sc_qp->qp_uk.back_qp; struct ib_qp_attr attr; - if (qp->iwdev->rf->reset) - return; attr.qp_state = IB_QPS_ERR; if (rdma_protocol_roce(qp->ibqp.device, 1)) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index c883c9ea5a831..c399aa07bcae8 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -27,7 +27,8 @@ static int irdma_query_device(struct ib_device *ibdev, irdma_fw_minor_ver(&rf->sc_dev); props->device_cap_flags = IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS; - props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; + if (hw_attrs->uk_attrs.hw_rev < IRDMA_GEN_3) + props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; props->vendor_id = pcidev->vendor; props->vendor_part_id = pcidev->device; @@ -557,7 +558,8 @@ static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) } irdma_qp_rem_ref(&iwqp->ibqp); - wait_for_completion(&iwqp->free_qp); + if (!iwdev->rf->reset) + wait_for_completion(&iwqp->free_qp); irdma_free_lsmm_rsrc(iwqp); irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); @@ -1105,6 +1107,7 @@ static int irdma_create_qp(struct ib_qp *ibqp, spin_lock_init(&iwqp->sc_qp.pfpdu.lock); iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; rf->qp_table[qp_num] = iwqp; + init_completion(&iwqp->free_qp); if (udata) { /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ @@ -1129,7 +1132,6 @@ static int irdma_create_qp(struct ib_qp *ibqp, } } - init_completion(&iwqp->free_qp); return 0; error: @@ -1462,8 +1464,6 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, ctx_info->remote_atomics_en = true; } - wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); - ibdev_dbg(&iwdev->ibdev, "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", __builtin_return_address(0), ibqp->qp_num, attr->qp_state, @@ -1540,6 +1540,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, case IB_QPS_ERR: case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { + iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, @@ -1745,6 +1746,7 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, case IB_QPS_ERR: case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { + iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, @@ -2306,8 +2308,8 @@ static int irdma_setup_kmode_srq(struct irdma_device *iwdev, ukinfo->srq_size = depth >> shift; ukinfo->shadow_area = mem->va + ring_size; - info->shadow_area_pa = info->srq_pa + ring_size; info->srq_pa = mem->pa; + info->shadow_area_pa = info->srq_pa + ring_size; return 0; } @@ -3108,7 +3110,6 @@ static int irdma_hw_alloc_stag(struct irdma_device *iwdev, info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; info->pd_id = iwpd->sc_pd.pd_id; info->total_len = iwmr->len; - info->all_memory = pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY; info->remote_access = true; cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; cqp_info->post_sq = 1; @@ -3119,7 +3120,7 @@ static int irdma_hw_alloc_stag(struct irdma_device *iwdev, if (status) return status; - iwmr->is_hwreg = 1; + iwmr->is_hwreg = true; return 0; } @@ -3263,7 +3264,7 @@ static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) stag_info->remote_atomics_en = (access & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; stag_info->pd_id = iwpd->sc_pd.pd_id; - stag_info->all_memory = pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY; + stag_info->all_memory = iwmr->dma_mr; if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; else @@ -3290,7 +3291,7 @@ static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); if (!ret) - iwmr->is_hwreg = 1; + iwmr->is_hwreg = true; return ret; } @@ -3663,7 +3664,7 @@ static int irdma_hwdereg_mr(struct ib_mr *ib_mr) if (status) return status; - iwmr->is_hwreg = 0; + iwmr->is_hwreg = false; return 0; } @@ -3713,6 +3714,7 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, err: ib_umem_release(region); + iwmr->region = NULL; return err; } @@ -3786,9 +3788,10 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, * @size: size of memory to register * @access: Access rights * @iova_start: start of virtual address for physical buffers + * @dma_mr: Flag indicating whether this region is a PD DMA MR */ struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, - u64 *iova_start) + u64 *iova_start, bool dma_mr) { struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_pbl *iwpbl; @@ -3805,6 +3808,7 @@ struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->type = IRDMA_MEMREG_TYPE_MEM; + iwmr->dma_mr = dma_mr; iwpbl->user_base = *iova_start; stag = irdma_create_stag(iwdev); if (!stag) { @@ -3843,7 +3847,7 @@ static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) { u64 kva = 0; - return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); + return irdma_reg_phys_mr(pd, 0, 0, acc, &kva, true); } /** @@ -5200,7 +5204,7 @@ static int irdma_create_user_ah(struct ib_ah *ibah, #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); struct irdma_device *iwdev = to_iwdev(ibah->pd->device); - struct irdma_create_ah_resp uresp; + struct irdma_create_ah_resp uresp = {}; struct irdma_ah *parent_ah; int err; @@ -5500,7 +5504,9 @@ void irdma_ib_dealloc_device(struct ib_device *ibdev) irdma_rt_deinit_hw(iwdev); if (!iwdev->is_vport) { irdma_ctrl_deinit_hw(iwdev->rf); - if (iwdev->rf->vchnl_wq) + if (iwdev->rf->vchnl_wq) { destroy_workqueue(iwdev->rf->vchnl_wq); + mutex_destroy(&iwdev->rf->sc_dev.vchnl_mutex); + } } } diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index ac8b387018350..aabbb3442098b 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -111,7 +111,8 @@ struct irdma_mr { }; struct ib_umem *region; int access; - u8 is_hwreg; + bool is_hwreg:1; + bool dma_mr:1; u16 type; u32 page_cnt; u64 page_size; diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 1becc87791235..7600412b0739f 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -56,6 +56,10 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, doorbell = mana_ucontext->doorbell; } else { is_rnic_cq = true; + if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) { + ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); + return -EINVAL; + } buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); cq->cqe = buf_size / COMP_ENTRY_SIZE; err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index fc1e86f6c4097..5899bd5cb1623 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -557,12 +557,20 @@ static int mlx5_query_port_roce(struct ib_device *device, u32 port_num, * of an error it will still be zeroed out. * Use native port in case of reps */ - if (dev->is_rep) - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - 1, 0); - else - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - mdev_port_num, 0); + if (dev->is_rep) { + struct mlx5_eswitch_rep *rep; + + rep = dev->port[port_num - 1].rep; + if (rep) { + mdev = mlx5_eswitch_get_core_dev(rep->esw); + WARN_ON(!mdev); + } + mdev_port_num = 1; + } + + err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, + mdev_port_num, 0); + if (err) goto out; ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability); @@ -2874,7 +2882,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) container_of(_work, struct mlx5_ib_event_work, work); struct mlx5_ib_dev *ibdev; struct ib_event ibev; - bool fatal = false; if (work->is_slave) { ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi); @@ -2885,12 +2892,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) } switch (work->event) { - case MLX5_DEV_EVENT_SYS_ERROR: - ibev.event = IB_EVENT_DEVICE_FATAL; - mlx5_ib_handle_internal_error(ibdev); - ibev.element.port_num = (u8)(unsigned long)work->param; - fatal = true; - break; case MLX5_EVENT_TYPE_PORT_CHANGE: if (handle_port_change(ibdev, work->param, &ibev)) goto out; @@ -2912,8 +2913,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) if (ibdev->ib_active) ib_dispatch_event(&ibev); - if (fatal) - ibdev->ib_active = false; out: kfree(work); } @@ -2957,6 +2956,66 @@ static int mlx5_ib_event_slave_port(struct notifier_block *nb, return NOTIFY_OK; } +static void mlx5_ib_handle_sys_error_event(struct work_struct *_work) +{ + struct mlx5_ib_event_work *work = + container_of(_work, struct mlx5_ib_event_work, work); + struct mlx5_ib_dev *ibdev = work->dev; + struct ib_event ibev; + + ibev.event = IB_EVENT_DEVICE_FATAL; + mlx5_ib_handle_internal_error(ibdev); + ibev.element.port_num = (u8)(unsigned long)work->param; + ibev.device = &ibdev->ib_dev; + + if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) { + mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num); + goto out; + } + + if (ibdev->ib_active) + ib_dispatch_event(&ibev); + + ibdev->ib_active = false; +out: + kfree(work); +} + +static int mlx5_ib_sys_error_event(struct notifier_block *nb, + unsigned long event, void *param) +{ + struct mlx5_ib_event_work *work; + + if (event != MLX5_DEV_EVENT_SYS_ERROR) + return NOTIFY_DONE; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + + INIT_WORK(&work->work, mlx5_ib_handle_sys_error_event); + work->dev = container_of(nb, struct mlx5_ib_dev, sys_error_events); + work->is_slave = false; + work->param = param; + work->event = event; + + queue_work(mlx5_ib_event_wq, &work->work); + + return NOTIFY_OK; +} + +static int mlx5_ib_stage_sys_error_notifier_init(struct mlx5_ib_dev *dev) +{ + dev->sys_error_events.notifier_call = mlx5_ib_sys_error_event; + mlx5_notifier_register(dev->mdev, &dev->sys_error_events); + return 0; +} + +static void mlx5_ib_stage_sys_error_notifier_cleanup(struct mlx5_ib_dev *dev) +{ + mlx5_notifier_unregister(dev->mdev, &dev->sys_error_events); +} + static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane) { struct mlx5_hca_vport_context vport_ctx; @@ -4462,12 +4521,16 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) { err = mlx5_ib_init_ucaps(dev); if (err) - return err; + goto err_ucaps; } dev->ib_dev.use_cq_dim = true; return 0; + +err_ucaps: + bitmap_free(dev->var_table.bitmap); + return err; } static const struct ib_device_ops mlx5_ib_dev_port_ops = { @@ -4803,6 +4866,9 @@ static const struct mlx5_ib_profile pf_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), @@ -4860,6 +4926,9 @@ const struct mlx5_ib_profile raw_eth_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 09d82d5f95e35..fbccb0362590b 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1007,6 +1007,7 @@ enum mlx5_ib_stages { MLX5_IB_STAGE_BFREG, MLX5_IB_STAGE_PRE_IB_REG_UMR, MLX5_IB_STAGE_WHITELIST_UID, + MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, MLX5_IB_STAGE_IB_REG, MLX5_IB_STAGE_DEVICE_NOTIFIER, MLX5_IB_STAGE_POST_IB_REG_UMR, @@ -1165,6 +1166,7 @@ struct mlx5_ib_dev { /* protect accessing data_direct_dev */ struct mutex data_direct_lock; struct notifier_block mdev_events; + struct notifier_block sys_error_events; struct notifier_block lag_events; int num_ports; /* serialize update of capability mask diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 2fcf553044e15..1ee31611b4b3f 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -195,7 +195,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( int out_len = uverbs_attr_get_len(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH); u32 dev_path_len; - char *dev_path; + char *dev_path = NULL; int ret; c = to_mucontext(ib_uverbs_get_ucontext(attrs)); @@ -223,9 +223,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( ret = uverbs_copy_to(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH, dev_path, dev_path_len); - kfree(dev_path); end: + kfree(dev_path); mutex_unlock(&dev->data_direct_lock); return ret; } diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index dd572d76866c2..e095873b381b6 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -428,6 +428,8 @@ static int mthca_create_srq(struct ib_srq *ibsrq, if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) { mthca_free_srq(to_mdev(ibsrq->device), srq); + mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar, + context->db_tab, ucmd.db_index); return -EFAULT; } @@ -436,6 +438,7 @@ static int mthca_create_srq(struct ib_srq *ibsrq, static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) { + mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); if (udata) { struct mthca_ucontext *context = rdma_udata_to_drv_context( @@ -446,8 +449,6 @@ static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) mthca_unmap_user_db(to_mdev(srq->device), &context->uar, context->db_tab, to_msrq(srq)->db_index); } - - mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); return 0; } diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index 134a79eecfcb8..3467797b5b01d 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -92,12 +92,10 @@ static int rvt_wss_llc_size(void) static void cacheless_memcpy(void *dst, void *src, size_t n) { /* - * Use the only available X64 cacheless copy. Add a __user cast - * to quiet sparse. The src agument is already in the kernel so - * there are no security issues. The extra fault recovery machinery - * is not invoked. + * Use the only available X64 cacheless copy. + * The extra fault recovery machinery is not invoked. */ - __copy_user_nocache(dst, (void __user *)src, n); + copy_to_nontemporal(dst, src, n); } void rvt_wss_exit(struct rvt_dev_info *rdi) diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index a5b2b62f596b0..1390e861bd1d7 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -119,12 +119,15 @@ void retransmit_timer(struct timer_list *t) rxe_dbg_qp(qp, "retransmit timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { qp->comp.timeout = 1; rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index bcb97b3ea58ac..2c486bb616a7c 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -72,14 +72,46 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr) mr->ibmr.type = IB_MR_TYPE_DMA; } +/* + * Convert iova to page_info index. The page_info stores pages of size + * PAGE_SIZE, but MRs can have different page sizes. This function + * handles the conversion for all cases: + * + * 1. mr->page_size > PAGE_SIZE: + * The MR's iova may not be aligned to mr->page_size. We use the + * aligned base (iova & page_mask) as reference, then calculate + * which PAGE_SIZE sub-page the iova falls into. + * + * 2. mr->page_size <= PAGE_SIZE: + * Use simple shift arithmetic since each page_info entry corresponds + * to one or more MR pages. + */ static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova) { - return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); + int idx; + + if (mr_page_size(mr) > PAGE_SIZE) + idx = (iova - (mr->ibmr.iova & mr->page_mask)) >> PAGE_SHIFT; + else + idx = (iova >> mr->page_shift) - + (mr->ibmr.iova >> mr->page_shift); + + WARN_ON(idx >= mr->nbuf); + return idx; } +/* + * Convert iova to offset within the page_info entry. + * + * For mr_page_size > PAGE_SIZE, the offset is within the system page. + * For mr_page_size <= PAGE_SIZE, the offset is within the MR page size. + */ static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova) { - return iova & (mr_page_size(mr) - 1); + if (mr_page_size(mr) > PAGE_SIZE) + return iova & (PAGE_SIZE - 1); + else + return iova & (mr_page_size(mr) - 1); } static bool is_pmem_page(struct page *pg) @@ -93,37 +125,69 @@ static bool is_pmem_page(struct page *pg) static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) { - XA_STATE(xas, &mr->page_list, 0); struct sg_page_iter sg_iter; struct page *page; bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); + WARN_ON(mr_page_size(mr) != PAGE_SIZE); + __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0); if (!__sg_page_iter_next(&sg_iter)) return 0; - do { - xas_lock(&xas); - while (true) { - page = sg_page_iter_page(&sg_iter); - - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page can't be persistent\n"); - xas_set_err(&xas, -EINVAL); - break; - } + while (true) { + page = sg_page_iter_page(&sg_iter); - xas_store(&xas, page); - if (xas_error(&xas)) - break; - xas_next(&xas); - if (!__sg_page_iter_next(&sg_iter)) - break; + if (persistent && !is_pmem_page(page)) { + rxe_dbg_mr(mr, "Page can't be persistent\n"); + return -EINVAL; } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - return xas_error(&xas); + mr->page_info[mr->nbuf].page = page; + mr->page_info[mr->nbuf].offset = 0; + mr->nbuf++; + + if (!__sg_page_iter_next(&sg_iter)) + break; + } + + return 0; +} + +static int __alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + mr->page_info = kcalloc(num_pages, sizeof(struct rxe_mr_page), + GFP_KERNEL); + if (!mr->page_info) + return -ENOMEM; + + mr->max_allowed_buf = num_pages; + mr->nbuf = 0; + + return 0; +} + +static int alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + int ret; + + WARN_ON(mr->num_buf); + ret = __alloc_mr_page_info(mr, num_pages); + if (ret) + return ret; + + mr->num_buf = num_pages; + + return 0; +} + +static void free_mr_page_info(struct rxe_mr *mr) +{ + if (!mr->page_info) + return; + + kfree(mr->page_info); + mr->page_info = NULL; } int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, @@ -134,8 +198,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, rxe_mr_init(access, mr); - xa_init(&mr->page_list); - umem = ib_umem_get(&rxe->ib_dev, start, length, access); if (IS_ERR(umem)) { rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n", @@ -143,46 +205,24 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, return PTR_ERR(umem); } + err = alloc_mr_page_info(mr, ib_umem_num_pages(umem)); + if (err) + goto err2; + err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt); - if (err) { - ib_umem_release(umem); - return err; - } + if (err) + goto err1; mr->umem = umem; mr->ibmr.type = IB_MR_TYPE_USER; mr->state = RXE_MR_STATE_VALID; return 0; -} - -static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf) -{ - XA_STATE(xas, &mr->page_list, 0); - int i = 0; - int err; - - xa_init(&mr->page_list); - - do { - xas_lock(&xas); - while (i != num_buf) { - xas_store(&xas, XA_ZERO_ENTRY); - if (xas_error(&xas)) - break; - xas_next(&xas); - i++; - } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - - err = xas_error(&xas); - if (err) - return err; - - mr->num_buf = num_buf; - - return 0; +err1: + free_mr_page_info(mr); +err2: + ib_umem_release(umem); + return err; } int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) @@ -192,7 +232,7 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) /* always allow remote access for FMRs */ rxe_mr_init(RXE_ACCESS_REMOTE, mr); - err = rxe_mr_alloc(mr, max_pages); + err = alloc_mr_page_info(mr, max_pages); if (err) goto err1; @@ -205,26 +245,43 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) return err; } +/* + * I) MRs with page_size >= PAGE_SIZE, + * Split a large MR page (mr->page_size) into multiple PAGE_SIZE + * sub-pages and store them in page_info, offset is always 0. + * + * Called when mr->page_size > PAGE_SIZE. Each call to rxe_set_page() + * represents one mr->page_size region, which we must split into + * (mr->page_size >> PAGE_SHIFT) individual pages. + * + * II) MRs with page_size < PAGE_SIZE, + * Save each PAGE_SIZE page and its offset within the system page in page_info. + */ static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr) { struct rxe_mr *mr = to_rmr(ibmr); - struct page *page = ib_virt_dma_to_page(dma_addr); bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); - int err; + u32 i, pages_per_mr = mr_page_size(mr) >> PAGE_SHIFT; - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page cannot be persistent\n"); - return -EINVAL; - } + pages_per_mr = MAX(1, pages_per_mr); - if (unlikely(mr->nbuf == mr->num_buf)) - return -ENOMEM; + for (i = 0; i < pages_per_mr; i++) { + u64 addr = dma_addr + i * PAGE_SIZE; + struct page *sub_page = ib_virt_dma_to_page(addr); - err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL)); - if (err) - return err; + if (unlikely(mr->nbuf >= mr->max_allowed_buf)) + return -ENOMEM; + + if (persistent && !is_pmem_page(sub_page)) { + rxe_dbg_mr(mr, "Page cannot be persistent\n"); + return -EINVAL; + } + + mr->page_info[mr->nbuf].page = sub_page; + mr->page_info[mr->nbuf].offset = addr & (PAGE_SIZE - 1); + mr->nbuf++; + } - mr->nbuf++; return 0; } @@ -234,6 +291,31 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, struct rxe_mr *mr = to_rmr(ibmr); unsigned int page_size = mr_page_size(mr); + /* + * Ensure page_size and PAGE_SIZE are compatible for mapping. + * We require one to be a multiple of the other for correct + * iova-to-page conversion. + */ + if (!IS_ALIGNED(page_size, PAGE_SIZE) && + !IS_ALIGNED(PAGE_SIZE, page_size)) { + rxe_dbg_mr(mr, "MR page size %u must be compatible with PAGE_SIZE %lu\n", + page_size, PAGE_SIZE); + return -EINVAL; + } + + if (mr_page_size(mr) > PAGE_SIZE) { + /* resize page_info if needed */ + u32 map_mr_pages = (page_size >> PAGE_SHIFT) * mr->num_buf; + + if (map_mr_pages > mr->max_allowed_buf) { + rxe_dbg_mr(mr, "requested pages %u exceed max %u\n", + map_mr_pages, mr->max_allowed_buf); + free_mr_page_info(mr); + if (__alloc_mr_page_info(mr, map_mr_pages)) + return -ENOMEM; + } + } + mr->nbuf = 0; mr->page_shift = ilog2(page_size); mr->page_mask = ~((u64)page_size - 1); @@ -245,30 +327,30 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir) { - unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); - unsigned long index = rxe_mr_iova_to_index(mr, iova); unsigned int bytes; - struct page *page; - void *va; + u8 *va; while (length) { - page = xa_load(&mr->page_list, index); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + va = kmap_local_page(info->page); + if (dir == RXE_FROM_MR_OBJ) memcpy(addr, va + page_offset, bytes); else memcpy(va + page_offset, addr, bytes); kunmap_local(va); - page_offset = 0; addr += bytes; + iova += bytes; length -= bytes; - index++; } return 0; @@ -426,9 +508,6 @@ int copy_data( static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) { - unsigned int page_offset; - unsigned long index; - struct page *page; unsigned int bytes; int err; u8 *va; @@ -438,15 +517,17 @@ static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int leng return err; while (length > 0) { - index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - page_offset = rxe_mr_iova_to_page_offset(mr, iova); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + + va = kmap_local_page(info->page); arch_wb_cache_pmem(va + page_offset, bytes); kunmap_local(va); @@ -502,6 +583,7 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } else { unsigned long index; int err; + struct rxe_mr_page *info; err = mr_check_range(mr, iova, sizeof(value)); if (err) { @@ -510,9 +592,12 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } if (unlikely(page_offset & 0x7)) { @@ -551,6 +636,7 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } else { unsigned long index; int err; + struct rxe_mr_page *info; /* See IBA oA19-28 */ err = mr_check_range(mr, iova, sizeof(value)); @@ -560,9 +646,12 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } /* See IBA A19.4.2 */ @@ -726,5 +815,5 @@ void rxe_mr_cleanup(struct rxe_pool_elem *elem) ib_umem_release(mr->umem); if (mr->ibmr.type != IB_MR_TYPE_DMA) - xa_destroy(&mr->page_list); + free_mr_page_info(mr); } diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index f58e3ec6252f5..4d4e3b324dd29 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -179,8 +179,10 @@ static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u return err; need_fault = rxe_check_pagefault(umem_odp, iova, length); - if (need_fault) + if (need_fault) { + mutex_unlock(&umem_odp->umem_mutex); return -EFAULT; + } } return 0; diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 373b03f223beb..12d03f390b097 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -102,6 +102,8 @@ void rnr_nak_timer(struct timer_list *t) rxe_dbg_qp(qp, "nak timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { /* request a send queue retry */ @@ -110,6 +112,7 @@ void rnr_nak_timer(struct timer_list *t) rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } static void req_check_sq_drain_done(struct rxe_qp *qp) diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c index 3661cb627d28a..c9a7cd38953d3 100644 --- a/drivers/infiniband/sw/rxe/rxe_srq.c +++ b/drivers/infiniband/sw/rxe/rxe_srq.c @@ -77,9 +77,6 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, goto err_free; } - srq->rq.queue = q; - init->attr.max_wr = srq->rq.max_wr; - if (uresp) { if (copy_to_user(&uresp->srq_num, &srq->srq_num, sizeof(uresp->srq_num))) { @@ -88,6 +85,9 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, } } + srq->rq.queue = q; + init->attr.max_wr = srq->rq.max_wr; + return 0; err_free: @@ -171,7 +171,7 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq, udata, mi, &srq->rq.producer_lock, &srq->rq.consumer_lock); if (err) - goto err_free; + return err; srq->rq.max_wr = attr->max_wr; } @@ -180,11 +180,6 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq, srq->limit = attr->srq_limit; return 0; - -err_free: - rxe_queue_cleanup(q); - srq->rq.queue = NULL; - return err; } void rxe_srq_cleanup(struct rxe_pool_elem *elem) diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index fd48075810dd1..1b8ed1031bd57 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -335,6 +335,11 @@ static inline int rkey_is_mw(u32 rkey) return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX); } +struct rxe_mr_page { + struct page *page; + unsigned int offset; /* offset in system page */ +}; + struct rxe_mr { struct rxe_pool_elem elem; struct ib_mr ibmr; @@ -351,10 +356,13 @@ struct rxe_mr { unsigned int page_shift; u64 page_mask; + /* size of page_info when mr allocated */ u32 num_buf; + /* real size of page_info */ + u32 max_allowed_buf; u32 nbuf; - struct xarray page_list; + struct rxe_mr_page *page_info; }; static inline unsigned int mr_page_size(struct rxe_mr *mr) diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index a10820e338878..e8a88b378d51d 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -1435,7 +1435,8 @@ int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb, } if (unlikely(rv != 0 && rv != -EAGAIN)) { if ((srx->state > SIW_GET_HDR || - qp->rx_fpdu->more_ddp_segs) && run_completion) + (qp->rx_fpdu && qp->rx_fpdu->more_ddp_segs)) && + run_completion) siw_rdmap_complete(qp, rv); siw_dbg_qp(qp, "rx error %d, rx state %d\n", rv, diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 71387811b2815..8fa1d72bd20a4 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1464,6 +1464,7 @@ static void query_fast_reg_mode(struct rtrs_clt_path *clt_path) mr_page_shift = max(12, ffs(ib_dev->attrs.page_size_cap) - 1); max_pages_per_mr = ib_dev->attrs.max_mr_size; do_div(max_pages_per_mr, (1ull << mr_page_shift)); + max_pages_per_mr = min_not_zero((u32)max_pages_per_mr, U32_MAX); clt_path->max_pages_per_mr = min3(clt_path->max_pages_per_mr, (u32)max_pages_per_mr, ib_dev->attrs.max_fast_reg_page_list_len); @@ -1922,7 +1923,7 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, struct rtrs_path *s = con->c.path; const struct rtrs_msg_conn_rsp *msg; const char *rej_msg; - int status, errno; + int status, errno = -ECONNRESET; u8 data_len; status = ev->status; @@ -1944,7 +1945,7 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, status, rej_msg); } - return -ECONNRESET; + return errno; } void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index ef4abdea3c2d2..adb798e2a54ae 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -208,7 +208,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) size_t sg_cnt; int err, offset; bool need_inval; - u32 rkey = 0; struct ib_reg_wr rwr; struct ib_sge *plist; struct ib_sge list; @@ -240,11 +239,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) wr->wr.num_sge = 1; wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); - if (rkey == 0) - rkey = wr->rkey; - else - /* Only one key is actually used */ - WARN_ON_ONCE(rkey != wr->rkey); wr->wr.opcode = IB_WR_RDMA_WRITE; wr->wr.wr_cqe = &io_comp_cqe; @@ -277,7 +271,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) inv_wr.opcode = IB_WR_SEND_WITH_INV; inv_wr.wr_cqe = &io_comp_cqe; inv_wr.send_flags = 0; - inv_wr.ex.invalidate_rkey = rkey; + inv_wr.ex.invalidate_rkey = wr->rkey; } imm_wr.wr.next = NULL; @@ -601,7 +595,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) srv_path->mrs_num++) { struct rtrs_srv_mr *srv_mr = &srv_path->mrs[srv_path->mrs_num]; struct scatterlist *s; - int nr, nr_sgt, chunks; + int nr, nr_sgt, chunks, ind; sgt = &srv_mr->sgt; chunks = chunks_per_mr * srv_path->mrs_num; @@ -631,7 +625,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) } nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt, NULL, max_chunk_size); - if (nr != nr_sgt) { + if (nr < nr_sgt) { err = nr < 0 ? nr : -EINVAL; goto dereg_mr; } @@ -647,9 +641,24 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) goto dereg_mr; } } - /* Eventually dma addr for each chunk can be cached */ - for_each_sg(sgt->sgl, s, nr_sgt, i) - srv_path->dma_addr[chunks + i] = sg_dma_address(s); + + /* + * Cache DMA addresses by traversing sg entries. If + * regions were merged, an inner loop is required to + * populate the DMA address array by traversing larger + * regions. + */ + ind = chunks; + for_each_sg(sgt->sgl, s, nr_sgt, i) { + unsigned int dma_len = sg_dma_len(s); + u64 dma_addr = sg_dma_address(s); + u64 dma_addr_end = dma_addr + dma_len; + + do { + srv_path->dma_addr[ind++] = dma_addr; + dma_addr += max_chunk_size; + } while (dma_addr < dma_addr_end); + } ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); srv_mr->mr = mr; @@ -1450,7 +1459,7 @@ static struct rtrs_srv_sess *get_or_create_srv(struct rtrs_srv_ctx *ctx, kfree(srv->chunks); err_free_srv: - kfree(srv); + put_device(&srv->dev); return ERR_PTR(-ENOMEM); } diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index d72e89c25e503..627e8950e451e 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -133,6 +133,8 @@ static const struct xpad_device { } xpad_device[] = { /* Please keep this list sorted by vendor and product ID. */ { 0x0079, 0x18d4, "GPD Win 2 X-Box Controller", 0, XTYPE_XBOX360 }, + { 0x0351, 0x1000, "CRKD LP Blueberry Burst Pro Edition (Xbox)", 0, XTYPE_XBOX360 }, + { 0x0351, 0x2000, "CRKD LP Black Tribal Edition (Xbox) ", 0, XTYPE_XBOX360 }, { 0x03eb, 0xff01, "Wooting One (Legacy)", 0, XTYPE_XBOX360 }, { 0x03eb, 0xff02, "Wooting Two (Legacy)", 0, XTYPE_XBOX360 }, { 0x03f0, 0x038D, "HyperX Clutch", 0, XTYPE_XBOX360 }, /* wired */ @@ -311,6 +313,8 @@ static const struct xpad_device { { 0x1532, 0x0a00, "Razer Atrox Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, { 0x1532, 0x0a03, "Razer Wildcat", 0, XTYPE_XBOXONE }, { 0x1532, 0x0a29, "Razer Wolverine V2", 0, XTYPE_XBOXONE }, + { 0x1532, 0x0a57, "Razer Wolverine V3 Pro (Wired)", 0, XTYPE_XBOX360 }, + { 0x1532, 0x0a59, "Razer Wolverine V3 Pro (2.4 GHz Dongle)", 0, XTYPE_XBOX360 }, { 0x15e4, 0x3f00, "Power A Mini Pro Elite", 0, XTYPE_XBOX360 }, { 0x15e4, 0x3f0a, "Xbox Airflo wired controller", 0, XTYPE_XBOX360 }, { 0x15e4, 0x3f10, "Batarang Xbox 360 controller", 0, XTYPE_XBOX360 }, @@ -358,6 +362,8 @@ static const struct xpad_device { { 0x1bad, 0xfd00, "Razer Onza TE", 0, XTYPE_XBOX360 }, { 0x1bad, 0xfd01, "Razer Onza", 0, XTYPE_XBOX360 }, { 0x1ee9, 0x1590, "ZOTAC Gaming Zone", 0, XTYPE_XBOX360 }, + { 0x20bc, 0x5134, "BETOP BTP-KP50B Xinput Dongle", 0, XTYPE_XBOX360 }, + { 0x20bc, 0x514a, "BETOP BTP-KP50C Xinput Dongle", 0, XTYPE_XBOX360 }, { 0x20d6, 0x2001, "BDA Xbox Series X Wired Controller", 0, XTYPE_XBOXONE }, { 0x20d6, 0x2009, "PowerA Enhanced Wired Controller for Xbox Series X|S", 0, XTYPE_XBOXONE }, { 0x20d6, 0x2064, "PowerA Wired Controller for Xbox", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, @@ -420,6 +426,7 @@ static const struct xpad_device { { 0x3285, 0x0663, "Nacon Evol-X", 0, XTYPE_XBOXONE }, { 0x3537, 0x1004, "GameSir T4 Kaleid", 0, XTYPE_XBOX360 }, { 0x3537, 0x1010, "GameSir G7 SE", 0, XTYPE_XBOXONE }, + { 0x3651, 0x1000, "CRKD SG", 0, XTYPE_XBOX360 }, { 0x366c, 0x0005, "ByoWave Proteus Controller", MAP_SHARE_BUTTON, XTYPE_XBOXONE, FLAG_DELAY_INIT }, { 0x3767, 0x0101, "Fanatec Speedster 3 Forceshock Wheel", 0, XTYPE_XBOX }, { 0x37d7, 0x2501, "Flydigi Apex 5", 0, XTYPE_XBOX360 }, @@ -518,6 +525,7 @@ static const struct usb_device_id xpad_table[] = { */ { USB_INTERFACE_INFO('X', 'B', 0) }, /* Xbox USB-IF not-approved class */ XPAD_XBOX360_VENDOR(0x0079), /* GPD Win 2 controller */ + XPAD_XBOX360_VENDOR(0x0351), /* CRKD Controllers */ XPAD_XBOX360_VENDOR(0x03eb), /* Wooting Keyboards (Legacy) */ XPAD_XBOX360_VENDOR(0x03f0), /* HP HyperX Xbox 360 controllers */ XPAD_XBOXONE_VENDOR(0x03f0), /* HP HyperX Xbox One controllers */ @@ -558,6 +566,7 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOX360_VENDOR(0x1a86), /* Nanjing Qinheng Microelectronics (WCH) */ XPAD_XBOX360_VENDOR(0x1bad), /* Harmonix Rock Band guitar and drums */ XPAD_XBOX360_VENDOR(0x1ee9), /* ZOTAC Technology Limited */ + XPAD_XBOX360_VENDOR(0x20bc), /* BETOP wireless dongles */ XPAD_XBOX360_VENDOR(0x20d6), /* PowerA controllers */ XPAD_XBOXONE_VENDOR(0x20d6), /* PowerA controllers */ XPAD_XBOX360_VENDOR(0x2345), /* Machenike Controllers */ @@ -578,6 +587,7 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOXONE_VENDOR(0x3285), /* Nacon Evol-X */ XPAD_XBOX360_VENDOR(0x3537), /* GameSir Controllers */ XPAD_XBOXONE_VENDOR(0x3537), /* GameSir Controllers */ + XPAD_XBOX360_VENDOR(0x3651), /* CRKD Controllers */ XPAD_XBOXONE_VENDOR(0x366c), /* ByoWave controllers */ XPAD_XBOX360_VENDOR(0x37d7), /* Flydigi Controllers */ XPAD_XBOX360_VENDOR(0x413d), /* Black Shark Green Ghost Controller */ diff --git a/drivers/input/keyboard/lkkbd.c b/drivers/input/keyboard/lkkbd.c index c035216dd27c1..2f130f819363c 100644 --- a/drivers/input/keyboard/lkkbd.c +++ b/drivers/input/keyboard/lkkbd.c @@ -670,7 +670,8 @@ static int lkkbd_connect(struct serio *serio, struct serio_driver *drv) return 0; - fail3: serio_close(serio); + fail3: disable_work_sync(&lk->tq); + serio_close(serio); fail2: serio_set_drvdata(serio, NULL); fail1: input_free_device(input_dev); kfree(lk); @@ -684,6 +685,8 @@ static void lkkbd_disconnect(struct serio *serio) { struct lkkbd *lk = serio_get_drvdata(serio); + disable_work_sync(&lk->tq); + input_get_device(lk->dev); input_unregister_device(lk->dev); serio_close(serio); diff --git a/drivers/input/misc/uinput.c b/drivers/input/misc/uinput.c index 13336a2fd49c8..0e9544a98e672 100644 --- a/drivers/input/misc/uinput.c +++ b/drivers/input/misc/uinput.c @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include #include #include "../input-compat.h" @@ -57,6 +59,7 @@ struct uinput_device { struct input_dev *dev; struct mutex mutex; enum uinput_state state; + spinlock_t state_lock; wait_queue_head_t waitq; unsigned char ready; unsigned char head; @@ -75,6 +78,8 @@ static int uinput_dev_event(struct input_dev *dev, struct uinput_device *udev = input_get_drvdata(dev); struct timespec64 ts; + lockdep_assert_held(&dev->event_lock); + ktime_get_ts64(&ts); udev->buff[udev->head] = (struct input_event) { @@ -146,27 +151,26 @@ static void uinput_request_release_slot(struct uinput_device *udev, static int uinput_request_send(struct uinput_device *udev, struct uinput_request *request) { - int retval; + unsigned long flags; + int retval = 0; - retval = mutex_lock_interruptible(&udev->mutex); - if (retval) - return retval; + spin_lock(&udev->state_lock); if (udev->state != UIST_CREATED) { retval = -ENODEV; goto out; } - init_completion(&request->done); - /* * Tell our userspace application about this new request * by queueing an input event. */ + spin_lock_irqsave(&udev->dev->event_lock, flags); uinput_dev_event(udev->dev, EV_UINPUT, request->code, request->id); + spin_unlock_irqrestore(&udev->dev->event_lock, flags); out: - mutex_unlock(&udev->mutex); + spin_unlock(&udev->state_lock); return retval; } @@ -175,6 +179,13 @@ static int uinput_request_submit(struct uinput_device *udev, { int retval; + /* + * Initialize completion before allocating the request slot. + * Once the slot is allocated, uinput_flush_requests() may + * complete it at any time, so it must be initialized first. + */ + init_completion(&request->done); + retval = uinput_request_reserve_slot(udev, request); if (retval) return retval; @@ -289,7 +300,14 @@ static void uinput_destroy_device(struct uinput_device *udev) struct input_dev *dev = udev->dev; enum uinput_state old_state = udev->state; + /* + * Update state under state_lock so that concurrent + * uinput_request_send() sees the state change before we + * flush pending requests and tear down the device. + */ + spin_lock(&udev->state_lock); udev->state = UIST_NEW_DEVICE; + spin_unlock(&udev->state_lock); if (dev) { name = dev->name; @@ -366,7 +384,9 @@ static int uinput_create_device(struct uinput_device *udev) if (error) goto fail2; + spin_lock(&udev->state_lock); udev->state = UIST_CREATED; + spin_unlock(&udev->state_lock); return 0; @@ -384,6 +404,7 @@ static int uinput_open(struct inode *inode, struct file *file) return -ENOMEM; mutex_init(&newdev->mutex); + spin_lock_init(&newdev->state_lock); spin_lock_init(&newdev->requests_lock); init_waitqueue_head(&newdev->requests_waitq); init_waitqueue_head(&newdev->waitq); diff --git a/drivers/input/mouse/alps.c b/drivers/input/mouse/alps.c index d0cb9fb948218..df8953a5196e1 100644 --- a/drivers/input/mouse/alps.c +++ b/drivers/input/mouse/alps.c @@ -2975,6 +2975,7 @@ static void alps_disconnect(struct psmouse *psmouse) psmouse_reset(psmouse); timer_shutdown_sync(&priv->timer); + disable_delayed_work_sync(&priv->dev3_register_work); if (priv->dev2) input_unregister_device(priv->dev2); if (!IS_ERR_OR_NULL(priv->dev3)) diff --git a/drivers/input/mouse/bcm5974.c b/drivers/input/mouse/bcm5974.c index dfdfb59cc8b59..fe52f15c0c10d 100644 --- a/drivers/input/mouse/bcm5974.c +++ b/drivers/input/mouse/bcm5974.c @@ -286,6 +286,8 @@ struct bcm5974 { const struct tp_finger *index[MAX_FINGERS]; /* finger index data */ struct input_mt_pos pos[MAX_FINGERS]; /* position array */ int slots[MAX_FINGERS]; /* slot assignments */ + struct work_struct mode_reset_work; + unsigned long last_mode_reset; }; /* trackpad finger block data, le16-aligned */ @@ -696,6 +698,32 @@ static int bcm5974_wellspring_mode(struct bcm5974 *dev, bool on) return retval; } +/* + * Mode switches sent before the control response are ignored. + * Fixing this state requires switching to normal mode and waiting + * about 1ms before switching back to wellspring mode. + */ +static void bcm5974_mode_reset_work(struct work_struct *work) +{ + struct bcm5974 *dev = container_of(work, struct bcm5974, mode_reset_work); + int error; + + guard(mutex)(&dev->pm_mutex); + dev->last_mode_reset = jiffies; + + error = bcm5974_wellspring_mode(dev, false); + if (error) { + dev_err(&dev->intf->dev, "reset to normal mode failed\n"); + return; + } + + fsleep(1000); + + error = bcm5974_wellspring_mode(dev, true); + if (error) + dev_err(&dev->intf->dev, "mode switch after reset failed\n"); +} + static void bcm5974_irq_button(struct urb *urb) { struct bcm5974 *dev = urb->context; @@ -752,10 +780,20 @@ static void bcm5974_irq_trackpad(struct urb *urb) if (dev->tp_urb->actual_length == 2) goto exit; - if (report_tp_state(dev, dev->tp_urb->actual_length)) + if (report_tp_state(dev, dev->tp_urb->actual_length)) { dprintk(1, "bcm5974: bad trackpad package, length: %d\n", dev->tp_urb->actual_length); + /* + * Receiving a HID packet means we aren't in wellspring mode. + * If we haven't tried a reset in the last second, try now. + */ + if (dev->tp_urb->actual_length == 8 && + time_after(jiffies, dev->last_mode_reset + msecs_to_jiffies(1000))) { + schedule_work(&dev->mode_reset_work); + } + } + exit: error = usb_submit_urb(dev->tp_urb, GFP_ATOMIC); if (error) @@ -906,6 +944,7 @@ static int bcm5974_probe(struct usb_interface *iface, dev->intf = iface; dev->input = input_dev; dev->cfg = *cfg; + INIT_WORK(&dev->mode_reset_work, bcm5974_mode_reset_work); mutex_init(&dev->pm_mutex); /* setup urbs */ @@ -998,6 +1037,7 @@ static void bcm5974_disconnect(struct usb_interface *iface) { struct bcm5974 *dev = usb_get_intfdata(iface); + disable_work_sync(&dev->mode_reset_work); usb_set_intfdata(iface, NULL); input_unregister_device(dev->input); diff --git a/drivers/input/mouse/synaptics_i2c.c b/drivers/input/mouse/synaptics_i2c.c index a0d707e47d932..29da66af36d74 100644 --- a/drivers/input/mouse/synaptics_i2c.c +++ b/drivers/input/mouse/synaptics_i2c.c @@ -372,7 +372,7 @@ static irqreturn_t synaptics_i2c_irq(int irq, void *dev_id) { struct synaptics_i2c *touch = dev_id; - mod_delayed_work(system_wq, &touch->dwork, 0); + mod_delayed_work(system_dfl_wq, &touch->dwork, 0); return IRQ_HANDLED; } @@ -448,7 +448,7 @@ static void synaptics_i2c_work_handler(struct work_struct *work) * We poll the device once in THREAD_IRQ_SLEEP_SECS and * if error is detected, we try to reset and reconfigure the touchpad. */ - mod_delayed_work(system_wq, &touch->dwork, delay); + mod_delayed_work(system_dfl_wq, &touch->dwork, delay); } static int synaptics_i2c_open(struct input_dev *input) @@ -461,7 +461,7 @@ static int synaptics_i2c_open(struct input_dev *input) return ret; if (polling_req) - mod_delayed_work(system_wq, &touch->dwork, + mod_delayed_work(system_dfl_wq, &touch->dwork, msecs_to_jiffies(NO_DATA_SLEEP_MSECS)); return 0; @@ -615,13 +615,16 @@ static int synaptics_i2c_resume(struct device *dev) int ret; struct i2c_client *client = to_i2c_client(dev); struct synaptics_i2c *touch = i2c_get_clientdata(client); + struct input_dev *input = touch->input; ret = synaptics_i2c_reset_config(client); if (ret) return ret; - mod_delayed_work(system_wq, &touch->dwork, - msecs_to_jiffies(NO_DATA_SLEEP_MSECS)); + guard(mutex)(&input->mutex); + if (input_device_enabled(input)) + mod_delayed_work(system_dfl_wq, &touch->dwork, + msecs_to_jiffies(NO_DATA_SLEEP_MSECS)); return 0; } diff --git a/drivers/input/rmi4/rmi_f54.c b/drivers/input/rmi4/rmi_f54.c index ac4041a69fcd3..61909e1a39e24 100644 --- a/drivers/input/rmi4/rmi_f54.c +++ b/drivers/input/rmi4/rmi_f54.c @@ -538,6 +538,8 @@ static void rmi_f54_work(struct work_struct *work) int error; int i; + mutex_lock(&f54->data_mutex); + report_size = rmi_f54_get_report_size(f54); if (report_size == 0) { dev_err(&fn->dev, "Bad report size, report type=%d\n", @@ -546,8 +548,6 @@ static void rmi_f54_work(struct work_struct *work) goto error; /* retry won't help */ } - mutex_lock(&f54->data_mutex); - /* * Need to check if command has completed. * If not try again later. diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h index 1caa6c4ca435c..8ebdf4fb90308 100644 --- a/drivers/input/serio/i8042-acpipnpio.h +++ b/drivers/input/serio/i8042-acpipnpio.h @@ -115,6 +115,17 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = { }, .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_NEVER) }, + { + /* + * ASUS Zenbook UX425QA_UM425QA + * Some Zenbooks report "Zenbook" with a lowercase b. + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "Zenbook UX425QA_UM425QA"), + }, + .driver_data = (void *)(SERIO_QUIRK_PROBE_DEFER | SERIO_QUIRK_RESET_NEVER) + }, { /* ASUS ZenBook UX425UA/QA */ .matches = { @@ -1169,6 +1180,27 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = { .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "X5KK45xS_X5SP45xS"), + }, + .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | + SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "X6KK45xU_X6SP45xU"), + }, + .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | + SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "WUJIE Series-X5SP4NAG"), + }, + .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | + SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) + }, /* * A lot of modern Clevo barebones have touchpad and/or keyboard issues * after suspend fixable with the forcenorestore quirk. diff --git a/drivers/input/touchscreen/apple_z2.c b/drivers/input/touchscreen/apple_z2.c index 0de161eae59a0..271ababf0ad55 100644 --- a/drivers/input/touchscreen/apple_z2.c +++ b/drivers/input/touchscreen/apple_z2.c @@ -21,6 +21,7 @@ #define APPLE_Z2_TOUCH_STARTED 3 #define APPLE_Z2_TOUCH_MOVED 4 #define APPLE_Z2_CMD_READ_INTERRUPT_DATA 0xEB +#define APPLE_Z2_REPLY_INTERRUPT_DATA 0xE1 #define APPLE_Z2_HBPP_CMD_BLOB 0x3001 #define APPLE_Z2_FW_MAGIC 0x5746325A #define LOAD_COMMAND_INIT_PAYLOAD 0 @@ -142,6 +143,9 @@ static int apple_z2_read_packet(struct apple_z2 *z2) if (error) return error; + if (z2->rx_buf[0] != APPLE_Z2_REPLY_INTERRUPT_DATA) + return 0; + pkt_len = (get_unaligned_le16(z2->rx_buf + 1) + 8) & 0xfffffffc; error = spi_read(z2->spidev, z2->rx_buf, pkt_len); diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c index 93d659ff90aa9..73980142f492b 100644 --- a/drivers/input/touchscreen/ti_am335x_tsc.c +++ b/drivers/input/touchscreen/ti_am335x_tsc.c @@ -85,7 +85,7 @@ static int titsc_config_wires(struct titsc *ts_dev) wire_order[i] = ts_dev->config_inp[i] & 0x0F; if (WARN_ON(analog_line[i] > 7)) return -EINVAL; - if (WARN_ON(wire_order[i] > ARRAY_SIZE(config_pins))) + if (WARN_ON(wire_order[i] >= ARRAY_SIZE(config_pins))) return -EINVAL; } diff --git a/drivers/interconnect/debugfs-client.c b/drivers/interconnect/debugfs-client.c index bc3fd8a7b9eb4..24d7b5a577945 100644 --- a/drivers/interconnect/debugfs-client.c +++ b/drivers/interconnect/debugfs-client.c @@ -117,7 +117,12 @@ static int icc_commit_set(void *data, u64 val) mutex_lock(&debugfs_lock); - if (IS_ERR_OR_NULL(cur_path)) { + if (!cur_path) { + ret = -EINVAL; + goto out; + } + + if (IS_ERR(cur_path)) { ret = PTR_ERR(cur_path); goto out; } @@ -145,6 +150,11 @@ int icc_debugfs_client_init(struct dentry *icc_dir) return ret; } + src_node = devm_kstrdup(&pdev->dev, "", GFP_KERNEL); + dst_node = devm_kstrdup(&pdev->dev, "", GFP_KERNEL); + if (!src_node || !dst_node) + return -ENOMEM; + client_dir = debugfs_create_dir("test_client", icc_dir); debugfs_create_str("src_node", 0600, client_dir, &src_node); diff --git a/drivers/interconnect/mediatek/icc-emi.c b/drivers/interconnect/mediatek/icc-emi.c index 7da740b5fa8d6..dfa3a9cd93998 100644 --- a/drivers/interconnect/mediatek/icc-emi.c +++ b/drivers/interconnect/mediatek/icc-emi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,9 @@ static int mtk_emi_icc_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, { struct mtk_icc_node *in = node->data; - *agg_avg += avg_bw; + if (check_add_overflow(*agg_avg, avg_bw, agg_avg)) + *agg_avg = U32_MAX; + *agg_peak = max_t(u32, *agg_peak, peak_bw); in->sum_avg = *agg_avg; @@ -40,7 +43,7 @@ static int mtk_emi_icc_set(struct icc_node *src, struct icc_node *dst) if (unlikely(!src->provider)) return -EINVAL; - dev = src->provider->dev; + dev = src->provider->dev->parent; switch (node->ep) { case 0: @@ -97,7 +100,7 @@ int mtk_emi_icc_probe(struct platform_device *pdev) if (!data) return -ENOMEM; - provider->dev = pdev->dev.parent; + provider->dev = dev; provider->set = mtk_emi_icc_set; provider->aggregate = mtk_emi_icc_aggregate; provider->xlate = of_icc_xlate_onecell; diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig index 5b4bb9f1382b2..bb1cb8a640c17 100644 --- a/drivers/interconnect/qcom/Kconfig +++ b/drivers/interconnect/qcom/Kconfig @@ -17,6 +17,15 @@ config INTERCONNECT_QCOM_GLYMUR This is a driver for the Qualcomm Network-on-Chip on glymur-based platforms. +config INTERCONNECT_QCOM_KAANAPALI + tristate "Qualcomm KAANAPALI interconnect driver" + depends on INTERCONNECT_QCOM_RPMH_POSSIBLE + select INTERCONNECT_QCOM_RPMH + select INTERCONNECT_QCOM_BCM_VOTER + help + This is a driver for the Qualcomm Network-on-Chip on kaanapali-based + platforms. + config INTERCONNECT_QCOM_MSM8909 tristate "Qualcomm MSM8909 interconnect driver" depends on INTERCONNECT_QCOM diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile index cf8cba73ee3e6..6eedff043b412 100644 --- a/drivers/interconnect/qcom/Makefile +++ b/drivers/interconnect/qcom/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_INTERCONNECT_QCOM) += interconnect_qcom.o interconnect_qcom-y := icc-common.o icc-bcm-voter-objs := bcm-voter.o qnoc-glymur-objs := glymur.o +qnoc-kaanapali-objs := kaanapali.o qnoc-milos-objs := milos.o qnoc-msm8909-objs := msm8909.o qnoc-msm8916-objs := msm8916.o @@ -48,6 +49,7 @@ icc-smd-rpm-objs := smd-rpm.o icc-rpm.o icc-rpm-clocks.o obj-$(CONFIG_INTERCONNECT_QCOM_BCM_VOTER) += icc-bcm-voter.o obj-$(CONFIG_INTERCONNECT_QCOM_GLYMUR) += qnoc-glymur.o +obj-$(CONFIG_INTERCONNECT_QCOM_KAANAPALI) += qnoc-kaanapali.o obj-$(CONFIG_INTERCONNECT_QCOM_MILOS) += qnoc-milos.o obj-$(CONFIG_INTERCONNECT_QCOM_MSM8909) += qnoc-msm8909.o obj-$(CONFIG_INTERCONNECT_QCOM_MSM8916) += qnoc-msm8916.o diff --git a/drivers/interconnect/qcom/kaanapali.c b/drivers/interconnect/qcom/kaanapali.c new file mode 100644 index 0000000000000..c6b4902e057f5 --- /dev/null +++ b/drivers/interconnect/qcom/kaanapali.c @@ -0,0 +1,1868 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "bcm-voter.h" +#include "icc-rpmh.h" + +static struct qcom_icc_node qup0_core_slave = { + .name = "qup0_core_slave", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qup1_core_slave = { + .name = "qup1_core_slave", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qup2_core_slave = { + .name = "qup2_core_slave", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qup3_core_slave = { + .name = "qup3_core_slave", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qup4_core_slave = { + .name = "qup4_core_slave", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_ahb2phy0 = { + .name = "qhs_ahb2phy0", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_ahb2phy1 = { + .name = "qhs_ahb2phy1", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_camera_cfg = { + .name = "qhs_camera_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_clk_ctl = { + .name = "qhs_clk_ctl", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_crypto0_cfg = { + .name = "qhs_crypto0_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_display_cfg = { + .name = "qhs_display_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_eva_cfg = { + .name = "qhs_eva_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_gpuss_cfg = { + .name = "qhs_gpuss_cfg", + .channels = 1, + .buswidth = 8, +}; + +static struct qcom_icc_node qhs_i2c = { + .name = "qhs_i2c", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_i3c_ibi0_cfg = { + .name = "qhs_i3c_ibi0_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_i3c_ibi1_cfg = { + .name = "qhs_i3c_ibi1_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_imem_cfg = { + .name = "qhs_imem_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_ipc_router = { + .name = "qhs_ipc_router", + .channels = 4, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_mss_cfg = { + .name = "qhs_mss_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_pcie_cfg = { + .name = "qhs_pcie_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_prng = { + .name = "qhs_prng", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_qdss_cfg = { + .name = "qhs_qdss_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_qspi = { + .name = "qhs_qspi", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_qup1 = { + .name = "qhs_qup1", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_qup2 = { + .name = "qhs_qup2", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_qup3 = { + .name = "qhs_qup3", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_qup4 = { + .name = "qhs_qup4", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_sdc2 = { + .name = "qhs_sdc2", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_sdc4 = { + .name = "qhs_sdc4", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_spss_cfg = { + .name = "qhs_spss_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_tcsr = { + .name = "qhs_tcsr", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_tlmm = { + .name = "qhs_tlmm", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_ufs_mem_cfg = { + .name = "qhs_ufs_mem_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_usb3 = { + .name = "qhs_usb3", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_venus_cfg = { + .name = "qhs_venus_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_vsense_ctrl_cfg = { + .name = "qhs_vsense_ctrl_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node xs_qdss_stm = { + .name = "xs_qdss_stm", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node xs_sys_tcu_cfg = { + .name = "xs_sys_tcu_cfg", + .channels = 1, + .buswidth = 8, +}; + +static struct qcom_icc_node qhs_aoss = { + .name = "qhs_aoss", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_ipa = { + .name = "qhs_ipa", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_ipc_router_fence = { + .name = "qhs_ipc_router_fence", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_soccp = { + .name = "qhs_soccp", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qhs_tme_cfg = { + .name = "qhs_tme_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qns_apss = { + .name = "qns_apss", + .channels = 1, + .buswidth = 8, +}; + +static struct qcom_icc_node qss_ddrss_cfg = { + .name = "qss_ddrss_cfg", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qxs_boot_imem = { + .name = "qxs_boot_imem", + .channels = 1, + .buswidth = 16, +}; + +static struct qcom_icc_node qxs_imem = { + .name = "qxs_imem", + .channels = 1, + .buswidth = 8, +}; + +static struct qcom_icc_node xs_pcie = { + .name = "xs_pcie", + .channels = 1, + .buswidth = 16, +}; + +static struct qcom_icc_node ebi = { + .name = "ebi", + .channels = 4, + .buswidth = 4, +}; + +static struct qcom_icc_node srvc_mnoc = { + .name = "srvc_mnoc", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node srvc_pcie_aggre_noc = { + .name = "srvc_pcie_aggre_noc", + .channels = 1, + .buswidth = 4, +}; + +static struct qcom_icc_node qup0_core_master = { + .name = "qup0_core_master", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qup0_core_slave }, +}; + +static struct qcom_icc_node qup1_core_master = { + .name = "qup1_core_master", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qup1_core_slave }, +}; + +static struct qcom_icc_node qup2_core_master = { + .name = "qup2_core_master", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qup2_core_slave }, +}; + +static struct qcom_icc_node qup3_core_master = { + .name = "qup3_core_master", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qup3_core_slave }, +}; + +static struct qcom_icc_node qup4_core_master = { + .name = "qup4_core_master", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qup4_core_slave }, +}; + +static struct qcom_icc_node qnm_gemnoc_pcie = { + .name = "qnm_gemnoc_pcie", + .channels = 1, + .buswidth = 8, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &xs_pcie }, +}; + +static struct qcom_icc_node llcc_mc = { + .name = "llcc_mc", + .channels = 4, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &ebi }, +}; + +static struct qcom_icc_node qsm_mnoc_cfg = { + .name = "qsm_mnoc_cfg", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &srvc_mnoc }, +}; + +static struct qcom_icc_node qsm_pcie_anoc_cfg = { + .name = "qsm_pcie_anoc_cfg", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &srvc_pcie_aggre_noc }, +}; + +static struct qcom_icc_node qss_mnoc_cfg = { + .name = "qss_mnoc_cfg", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qsm_mnoc_cfg }, +}; + +static struct qcom_icc_node qss_pcie_anoc_cfg = { + .name = "qss_pcie_anoc_cfg", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qsm_pcie_anoc_cfg }, +}; + +static struct qcom_icc_node qns_llcc = { + .name = "qns_llcc", + .channels = 4, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &llcc_mc }, +}; + +static struct qcom_icc_node qns_pcie = { + .name = "qns_pcie", + .channels = 1, + .buswidth = 8, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_gemnoc_pcie }, +}; + +static struct qcom_icc_node qsm_cfg = { + .name = "qsm_cfg", + .channels = 1, + .buswidth = 4, + .num_links = 35, + .link_nodes = (struct qcom_icc_node *[]) { &qhs_ahb2phy0, &qhs_ahb2phy1, + &qhs_camera_cfg, &qhs_clk_ctl, + &qhs_crypto0_cfg, &qhs_display_cfg, + &qhs_eva_cfg, &qhs_gpuss_cfg, + &qhs_i2c, &qhs_i3c_ibi0_cfg, + &qhs_i3c_ibi1_cfg, &qhs_imem_cfg, + &qhs_ipc_router, &qhs_mss_cfg, + &qhs_pcie_cfg, &qhs_prng, + &qhs_qdss_cfg, &qhs_qspi, + &qhs_qup1, &qhs_qup2, + &qhs_qup3, &qhs_qup4, + &qhs_sdc2, &qhs_sdc4, + &qhs_spss_cfg, &qhs_tcsr, + &qhs_tlmm, &qhs_ufs_mem_cfg, + &qhs_usb3, &qhs_venus_cfg, + &qhs_vsense_ctrl_cfg, &qss_mnoc_cfg, + &qss_pcie_anoc_cfg, &xs_qdss_stm, + &xs_sys_tcu_cfg }, +}; + +static struct qcom_icc_node qnm_qpace = { + .name = "qnm_qpace", + .channels = 1, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x14e000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_llcc }, +}; + +static struct qcom_icc_node xm_gic = { + .name = "xm_gic", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x145000 }, + .prio = 4, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_llcc }, +}; + +static struct qcom_icc_node qss_cfg = { + .name = "qss_cfg", + .channels = 1, + .buswidth = 4, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qsm_cfg }, +}; + +static struct qcom_icc_node qnm_gemnoc_cnoc = { + .name = "qnm_gemnoc_cnoc", + .channels = 1, + .buswidth = 16, + .num_links = 10, + .link_nodes = (struct qcom_icc_node *[]) { &qhs_aoss, &qhs_ipa, + &qhs_ipc_router_fence, &qhs_soccp, + &qhs_tme_cfg, &qns_apss, + &qss_cfg, &qss_ddrss_cfg, + &qxs_boot_imem, &qxs_imem }, +}; + +static struct qcom_icc_node qns_gem_noc_cnoc = { + .name = "qns_gem_noc_cnoc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_gemnoc_cnoc }, +}; + +static struct qcom_icc_node alm_gpu_tcu = { + .name = "alm_gpu_tcu", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x13d000 }, + .prio = 1, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 2, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc }, +}; + +static struct qcom_icc_node alm_sys_tcu = { + .name = "alm_sys_tcu", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x13f000 }, + .prio = 6, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 2, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc }, +}; + +static struct qcom_icc_node chm_apps = { + .name = "chm_apps", + .channels = 4, + .buswidth = 32, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_gpu = { + .name = "qnm_gpu", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x31000, 0xb1000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_lpass_gemnoc = { + .name = "qnm_lpass_gemnoc", + .channels = 1, + .buswidth = 16, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x141000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_mdsp = { + .name = "qnm_mdsp", + .channels = 1, + .buswidth = 16, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_mnoc_hf = { + .name = "qnm_mnoc_hf", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x33000, 0xb3000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_mnoc_sf = { + .name = "qnm_mnoc_sf", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x35000, 0xb5000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_nsp_gemnoc = { + .name = "qnm_nsp_gemnoc", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x37000, 0xb7000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_pcie = { + .name = "qnm_pcie", + .channels = 1, + .buswidth = 16, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x143000 }, + .prio = 2, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 2, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc }, +}; + +static struct qcom_icc_node qnm_snoc_sf = { + .name = "qnm_snoc_sf", + .channels = 1, + .buswidth = 16, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x147000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qnm_wlan_q6 = { + .name = "qnm_wlan_q6", + .channels = 1, + .buswidth = 8, + .num_links = 3, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gem_noc_cnoc, &qns_llcc, + &qns_pcie }, +}; + +static struct qcom_icc_node qns_lpass_ag_noc_gemnoc = { + .name = "qns_lpass_ag_noc_gemnoc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_lpass_gemnoc }, +}; + +static struct qcom_icc_node qns_mem_noc_hf = { + .name = "qns_mem_noc_hf", + .channels = 2, + .buswidth = 32, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_mnoc_hf }, +}; + +static struct qcom_icc_node qns_mem_noc_sf = { + .name = "qns_mem_noc_sf", + .channels = 2, + .buswidth = 32, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_mnoc_sf }, +}; + +static struct qcom_icc_node qns_nsp_gemnoc = { + .name = "qns_nsp_gemnoc", + .channels = 2, + .buswidth = 32, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_nsp_gemnoc }, +}; + +static struct qcom_icc_node qns_pcie_gemnoc = { + .name = "qns_pcie_gemnoc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_pcie }, +}; + +static struct qcom_icc_node qns_gemnoc_sf = { + .name = "qns_gemnoc_sf", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_snoc_sf }, +}; + +static struct qcom_icc_node qnm_lpiaon_noc = { + .name = "qnm_lpiaon_noc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_lpass_ag_noc_gemnoc }, +}; + +static struct qcom_icc_node qnm_camnoc_hf = { + .name = "qnm_camnoc_hf", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x2a000, 0x2b000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_hf }, +}; + +static struct qcom_icc_node qnm_camnoc_nrt_icp_sf = { + .name = "qnm_camnoc_nrt_icp_sf", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x2c000 }, + .prio = 4, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_camnoc_rt_cdm_sf = { + .name = "qnm_camnoc_rt_cdm_sf", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x38000 }, + .prio = 2, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_camnoc_sf = { + .name = "qnm_camnoc_sf", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x2d000, 0x2e000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_mdp = { + .name = "qnm_mdp", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x2f000, 0x30000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_hf }, +}; + +static struct qcom_icc_node qnm_mdss_dcp = { + .name = "qnm_mdss_dcp", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x39000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_vapss_hcp = { + .name = "qnm_vapss_hcp", + .channels = 1, + .buswidth = 32, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_video_cv_cpu = { + .name = "qnm_video_cv_cpu", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x34000 }, + .prio = 4, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_video_eva = { + .name = "qnm_video_eva", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x35000, 0x36000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_video_mvp = { + .name = "qnm_video_mvp", + .channels = 2, + .buswidth = 32, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 2, + .port_offsets = { 0x32000, 0x33000 }, + .prio = 0, + .urg_fwd = 1, + .prio_fwd_disable = 0, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_video_v_cpu = { + .name = "qnm_video_v_cpu", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x37000 }, + .prio = 4, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_mem_noc_sf }, +}; + +static struct qcom_icc_node qnm_nsp = { + .name = "qnm_nsp", + .channels = 2, + .buswidth = 32, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_nsp_gemnoc }, +}; + +static struct qcom_icc_node xm_pcie = { + .name = "xm_pcie", + .channels = 1, + .buswidth = 16, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0xb000 }, + .prio = 3, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_pcie_gemnoc }, +}; + +static struct qcom_icc_node qnm_aggre1_noc = { + .name = "qnm_aggre1_noc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gemnoc_sf }, +}; + +static struct qcom_icc_node qnm_aggre2_noc = { + .name = "qnm_aggre2_noc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gemnoc_sf }, +}; + +static struct qcom_icc_node qnm_apss_noc = { + .name = "qnm_apss_noc", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x1e000 }, + .prio = 2, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gemnoc_sf }, +}; + +static struct qcom_icc_node qnm_cnoc_data = { + .name = "qnm_cnoc_data", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x1f000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_gemnoc_sf }, +}; + +static struct qcom_icc_node qns_a1noc_snoc = { + .name = "qns_a1noc_snoc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_aggre1_noc }, +}; + +static struct qcom_icc_node qns_a2noc_snoc = { + .name = "qns_a2noc_snoc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_aggre2_noc }, +}; + +static struct qcom_icc_node qns_lpass_aggnoc = { + .name = "qns_lpass_aggnoc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_lpiaon_noc }, +}; + +static struct qcom_icc_node qhm_qspi = { + .name = "qhm_qspi", + .channels = 1, + .buswidth = 4, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0xc000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a1noc_snoc }, +}; + +static struct qcom_icc_node qxm_crypto = { + .name = "qxm_crypto", + .channels = 1, + .buswidth = 16, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x36000 }, + .prio = 2, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a1noc_snoc }, +}; + +static struct qcom_icc_node qxm_qup1 = { + .name = "qxm_qup1", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x11000 }, + .prio = 2, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a1noc_snoc }, +}; + +static struct qcom_icc_node xm_sdc4 = { + .name = "xm_sdc4", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0xe000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a1noc_snoc }, +}; + +static struct qcom_icc_node xm_ufs_mem = { + .name = "xm_ufs_mem", + .channels = 1, + .buswidth = 16, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0xf000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a1noc_snoc }, +}; + +static struct qcom_icc_node xm_usb3 = { + .name = "xm_usb3", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x10000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a1noc_snoc }, +}; + +static struct qcom_icc_node qhm_qup2 = { + .name = "qhm_qup2", + .channels = 1, + .buswidth = 4, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x35000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node qhm_qup3 = { + .name = "qhm_qup3", + .channels = 1, + .buswidth = 4, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x3c000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node qhm_qup4 = { + .name = "qhm_qup4", + .channels = 1, + .buswidth = 4, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x3d000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node qxm_ipa = { + .name = "qxm_ipa", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x37000 }, + .prio = 2, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node qxm_soccp = { + .name = "qxm_soccp", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x3b000 }, + .prio = 2, + .urg_fwd = 1, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node qxm_sp = { + .name = "qxm_sp", + .channels = 1, + .buswidth = 8, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node xm_qdss_etr_0 = { + .name = "xm_qdss_etr_0", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x38000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node xm_qdss_etr_1 = { + .name = "xm_qdss_etr_1", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x39000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node xm_sdc2 = { + .name = "xm_sdc2", + .channels = 1, + .buswidth = 8, + .qosbox = &(const struct qcom_icc_qosbox) { + .num_ports = 1, + .port_offsets = { 0x3a000 }, + .prio = 2, + .urg_fwd = 0, + .prio_fwd_disable = 1, + }, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_a2noc_snoc }, +}; + +static struct qcom_icc_node qnm_lpass_lpinoc = { + .name = "qnm_lpass_lpinoc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_lpass_aggnoc }, +}; + +static struct qcom_icc_node qns_lpi_aon_noc = { + .name = "qns_lpi_aon_noc", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qnm_lpass_lpinoc }, +}; + +static struct qcom_icc_node qnm_lpinoc_dsp_qns4m = { + .name = "qnm_lpinoc_dsp_qns4m", + .channels = 1, + .buswidth = 16, + .num_links = 1, + .link_nodes = (struct qcom_icc_node *[]) { &qns_lpi_aon_noc }, +}; + +static struct qcom_icc_bcm bcm_acv = { + .name = "ACV", + .enable_mask = BIT(3), + .num_nodes = 1, + .nodes = { &ebi }, +}; + +static struct qcom_icc_bcm bcm_ce0 = { + .name = "CE0", + .num_nodes = 1, + .nodes = { &qxm_crypto }, +}; + +static struct qcom_icc_bcm bcm_cn0 = { + .name = "CN0", + .enable_mask = BIT(0), + .keepalive = true, + .num_nodes = 43, + .nodes = { &qsm_cfg, &qhs_ahb2phy0, + &qhs_ahb2phy1, &qhs_camera_cfg, + &qhs_clk_ctl, &qhs_crypto0_cfg, + &qhs_eva_cfg, &qhs_gpuss_cfg, + &qhs_i3c_ibi0_cfg, &qhs_i3c_ibi1_cfg, + &qhs_imem_cfg, &qhs_ipc_router, + &qhs_mss_cfg, &qhs_pcie_cfg, + &qhs_prng, &qhs_qdss_cfg, + &qhs_qspi, &qhs_sdc2, + &qhs_sdc4, &qhs_spss_cfg, + &qhs_tcsr, &qhs_tlmm, + &qhs_ufs_mem_cfg, &qhs_usb3, + &qhs_venus_cfg, &qhs_vsense_ctrl_cfg, + &qss_mnoc_cfg, &qss_pcie_anoc_cfg, + &xs_qdss_stm, &xs_sys_tcu_cfg, + &qnm_gemnoc_cnoc, &qnm_gemnoc_pcie, + &qhs_aoss, &qhs_ipa, + &qhs_ipc_router_fence, &qhs_soccp, + &qhs_tme_cfg, &qns_apss, + &qss_cfg, &qss_ddrss_cfg, + &qxs_boot_imem, &qxs_imem, + &xs_pcie }, +}; + +static struct qcom_icc_bcm bcm_cn1 = { + .name = "CN1", + .num_nodes = 6, + .nodes = { &qhs_display_cfg, &qhs_i2c, + &qhs_qup1, &qhs_qup2, + &qhs_qup3, &qhs_qup4 }, +}; + +static struct qcom_icc_bcm bcm_co0 = { + .name = "CO0", + .enable_mask = BIT(0), + .num_nodes = 2, + .nodes = { &qnm_nsp, &qns_nsp_gemnoc }, +}; + +static struct qcom_icc_bcm bcm_lp0 = { + .name = "LP0", + .num_nodes = 2, + .nodes = { &qnm_lpass_lpinoc, &qns_lpass_aggnoc }, +}; + +static struct qcom_icc_bcm bcm_mc0 = { + .name = "MC0", + .keepalive = true, + .num_nodes = 1, + .nodes = { &ebi }, +}; + +static struct qcom_icc_bcm bcm_mm0 = { + .name = "MM0", + .num_nodes = 1, + .nodes = { &qns_mem_noc_hf }, +}; + +static struct qcom_icc_bcm bcm_mm1 = { + .name = "MM1", + .enable_mask = BIT(0), + .num_nodes = 9, + .nodes = { &qnm_camnoc_hf, &qnm_camnoc_nrt_icp_sf, + &qnm_camnoc_rt_cdm_sf, &qnm_camnoc_sf, + &qnm_vapss_hcp, &qnm_video_cv_cpu, + &qnm_video_mvp, &qnm_video_v_cpu, + &qns_mem_noc_sf }, +}; + +static struct qcom_icc_bcm bcm_qpc0 = { + .name = "QPC0", + .num_nodes = 1, + .nodes = { &qnm_qpace }, +}; + +static struct qcom_icc_bcm bcm_qup0 = { + .name = "QUP0", + .keepalive = true, + .vote_scale = 1, + .num_nodes = 1, + .nodes = { &qup0_core_slave }, +}; + +static struct qcom_icc_bcm bcm_qup1 = { + .name = "QUP1", + .keepalive = true, + .vote_scale = 1, + .num_nodes = 1, + .nodes = { &qup1_core_slave }, +}; + +static struct qcom_icc_bcm bcm_qup2 = { + .name = "QUP2", + .keepalive = true, + .vote_scale = 1, + .num_nodes = 1, + .nodes = { &qup2_core_slave }, +}; + +static struct qcom_icc_bcm bcm_qup3 = { + .name = "QUP3", + .keepalive = true, + .vote_scale = 1, + .num_nodes = 1, + .nodes = { &qup3_core_slave }, +}; + +static struct qcom_icc_bcm bcm_qup4 = { + .name = "QUP4", + .keepalive = true, + .vote_scale = 1, + .num_nodes = 1, + .nodes = { &qup4_core_slave }, +}; + +static struct qcom_icc_bcm bcm_sh0 = { + .name = "SH0", + .keepalive = true, + .num_nodes = 1, + .nodes = { &qns_llcc }, +}; + +static struct qcom_icc_bcm bcm_sh1 = { + .name = "SH1", + .enable_mask = BIT(0), + .num_nodes = 14, + .nodes = { &alm_gpu_tcu, &alm_sys_tcu, + &chm_apps, &qnm_gpu, + &qnm_mdsp, &qnm_mnoc_hf, + &qnm_mnoc_sf, &qnm_nsp_gemnoc, + &qnm_pcie, &qnm_snoc_sf, + &qnm_wlan_q6, &xm_gic, + &qns_gem_noc_cnoc, &qns_pcie }, +}; + +static struct qcom_icc_bcm bcm_sn0 = { + .name = "SN0", + .keepalive = true, + .num_nodes = 1, + .nodes = { &qns_gemnoc_sf }, +}; + +static struct qcom_icc_bcm bcm_sn2 = { + .name = "SN2", + .num_nodes = 1, + .nodes = { &qnm_aggre1_noc }, +}; + +static struct qcom_icc_bcm bcm_sn3 = { + .name = "SN3", + .num_nodes = 1, + .nodes = { &qnm_aggre2_noc }, +}; + +static struct qcom_icc_bcm bcm_sn4 = { + .name = "SN4", + .num_nodes = 1, + .nodes = { &qns_pcie_gemnoc }, +}; + +static struct qcom_icc_bcm * const aggre_noc_bcms[] = { + &bcm_ce0, +}; + +static struct qcom_icc_node * const aggre_noc_nodes[] = { + [MASTER_QSPI_0] = &qhm_qspi, + [MASTER_CRYPTO] = &qxm_crypto, + [MASTER_QUP_1] = &qxm_qup1, + [MASTER_SDCC_4] = &xm_sdc4, + [MASTER_UFS_MEM] = &xm_ufs_mem, + [MASTER_USB3] = &xm_usb3, + [MASTER_QUP_2] = &qhm_qup2, + [MASTER_QUP_3] = &qhm_qup3, + [MASTER_QUP_4] = &qhm_qup4, + [MASTER_IPA] = &qxm_ipa, + [MASTER_SOCCP_PROC] = &qxm_soccp, + [MASTER_SP] = &qxm_sp, + [MASTER_QDSS_ETR] = &xm_qdss_etr_0, + [MASTER_QDSS_ETR_1] = &xm_qdss_etr_1, + [MASTER_SDCC_2] = &xm_sdc2, + [SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc, + [SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc, +}; + +static const struct regmap_config kaanapali_aggre_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x42400, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_aggre_noc = { + .config = &kaanapali_aggre_noc_regmap_config, + .nodes = aggre_noc_nodes, + .num_nodes = ARRAY_SIZE(aggre_noc_nodes), + .bcms = aggre_noc_bcms, + .num_bcms = ARRAY_SIZE(aggre_noc_bcms), + .qos_requires_clocks = true, + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const clk_virt_bcms[] = { + &bcm_qup0, + &bcm_qup1, + &bcm_qup2, + &bcm_qup3, + &bcm_qup4, +}; + +static struct qcom_icc_node * const clk_virt_nodes[] = { + [MASTER_QUP_CORE_0] = &qup0_core_master, + [MASTER_QUP_CORE_1] = &qup1_core_master, + [MASTER_QUP_CORE_2] = &qup2_core_master, + [MASTER_QUP_CORE_3] = &qup3_core_master, + [MASTER_QUP_CORE_4] = &qup4_core_master, + [SLAVE_QUP_CORE_0] = &qup0_core_slave, + [SLAVE_QUP_CORE_1] = &qup1_core_slave, + [SLAVE_QUP_CORE_2] = &qup2_core_slave, + [SLAVE_QUP_CORE_3] = &qup3_core_slave, + [SLAVE_QUP_CORE_4] = &qup4_core_slave, +}; + +static const struct qcom_icc_desc kaanapali_clk_virt = { + .nodes = clk_virt_nodes, + .num_nodes = ARRAY_SIZE(clk_virt_nodes), + .bcms = clk_virt_bcms, + .num_bcms = ARRAY_SIZE(clk_virt_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const cnoc_cfg_bcms[] = { + &bcm_cn0, + &bcm_cn1, +}; + +static struct qcom_icc_node * const cnoc_cfg_nodes[] = { + [MASTER_CNOC_CFG] = &qsm_cfg, + [SLAVE_AHB2PHY_SOUTH] = &qhs_ahb2phy0, + [SLAVE_AHB2PHY_NORTH] = &qhs_ahb2phy1, + [SLAVE_CAMERA_CFG] = &qhs_camera_cfg, + [SLAVE_CLK_CTL] = &qhs_clk_ctl, + [SLAVE_CRYPTO_0_CFG] = &qhs_crypto0_cfg, + [SLAVE_DISPLAY_CFG] = &qhs_display_cfg, + [SLAVE_EVA_CFG] = &qhs_eva_cfg, + [SLAVE_GFX3D_CFG] = &qhs_gpuss_cfg, + [SLAVE_I2C] = &qhs_i2c, + [SLAVE_I3C_IBI0_CFG] = &qhs_i3c_ibi0_cfg, + [SLAVE_I3C_IBI1_CFG] = &qhs_i3c_ibi1_cfg, + [SLAVE_IMEM_CFG] = &qhs_imem_cfg, + [SLAVE_IPC_ROUTER_CFG] = &qhs_ipc_router, + [SLAVE_CNOC_MSS] = &qhs_mss_cfg, + [SLAVE_PCIE_CFG] = &qhs_pcie_cfg, + [SLAVE_PRNG] = &qhs_prng, + [SLAVE_QDSS_CFG] = &qhs_qdss_cfg, + [SLAVE_QSPI_0] = &qhs_qspi, + [SLAVE_QUP_1] = &qhs_qup1, + [SLAVE_QUP_2] = &qhs_qup2, + [SLAVE_QUP_3] = &qhs_qup3, + [SLAVE_QUP_4] = &qhs_qup4, + [SLAVE_SDCC_2] = &qhs_sdc2, + [SLAVE_SDCC_4] = &qhs_sdc4, + [SLAVE_SPSS_CFG] = &qhs_spss_cfg, + [SLAVE_TCSR] = &qhs_tcsr, + [SLAVE_TLMM] = &qhs_tlmm, + [SLAVE_UFS_MEM_CFG] = &qhs_ufs_mem_cfg, + [SLAVE_USB3] = &qhs_usb3, + [SLAVE_VENUS_CFG] = &qhs_venus_cfg, + [SLAVE_VSENSE_CTRL_CFG] = &qhs_vsense_ctrl_cfg, + [SLAVE_CNOC_MNOC_CFG] = &qss_mnoc_cfg, + [SLAVE_PCIE_ANOC_CFG] = &qss_pcie_anoc_cfg, + [SLAVE_QDSS_STM] = &xs_qdss_stm, + [SLAVE_TCU] = &xs_sys_tcu_cfg, +}; + +static const struct regmap_config kaanapali_cnoc_cfg_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x6200, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_cnoc_cfg = { + .config = &kaanapali_cnoc_cfg_regmap_config, + .nodes = cnoc_cfg_nodes, + .num_nodes = ARRAY_SIZE(cnoc_cfg_nodes), + .bcms = cnoc_cfg_bcms, + .num_bcms = ARRAY_SIZE(cnoc_cfg_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const cnoc_main_bcms[] = { + &bcm_cn0, +}; + +static struct qcom_icc_node * const cnoc_main_nodes[] = { + [MASTER_GEM_NOC_CNOC] = &qnm_gemnoc_cnoc, + [MASTER_GEM_NOC_PCIE_SNOC] = &qnm_gemnoc_pcie, + [SLAVE_AOSS] = &qhs_aoss, + [SLAVE_IPA_CFG] = &qhs_ipa, + [SLAVE_IPC_ROUTER_FENCE] = &qhs_ipc_router_fence, + [SLAVE_SOCCP] = &qhs_soccp, + [SLAVE_TME_CFG] = &qhs_tme_cfg, + [SLAVE_APPSS] = &qns_apss, + [SLAVE_CNOC_CFG] = &qss_cfg, + [SLAVE_DDRSS_CFG] = &qss_ddrss_cfg, + [SLAVE_BOOT_IMEM] = &qxs_boot_imem, + [SLAVE_IMEM] = &qxs_imem, + [SLAVE_PCIE_0] = &xs_pcie, +}; + +static const struct regmap_config kaanapali_cnoc_main_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1a080, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_cnoc_main = { + .config = &kaanapali_cnoc_main_regmap_config, + .nodes = cnoc_main_nodes, + .num_nodes = ARRAY_SIZE(cnoc_main_nodes), + .bcms = cnoc_main_bcms, + .num_bcms = ARRAY_SIZE(cnoc_main_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const gem_noc_bcms[] = { + &bcm_qpc0, + &bcm_sh0, + &bcm_sh1, +}; + +static struct qcom_icc_node * const gem_noc_nodes[] = { + [MASTER_GPU_TCU] = &alm_gpu_tcu, + [MASTER_SYS_TCU] = &alm_sys_tcu, + [MASTER_APPSS_PROC] = &chm_apps, + [MASTER_GFX3D] = &qnm_gpu, + [MASTER_LPASS_GEM_NOC] = &qnm_lpass_gemnoc, + [MASTER_MSS_PROC] = &qnm_mdsp, + [MASTER_MNOC_HF_MEM_NOC] = &qnm_mnoc_hf, + [MASTER_MNOC_SF_MEM_NOC] = &qnm_mnoc_sf, + [MASTER_COMPUTE_NOC] = &qnm_nsp_gemnoc, + [MASTER_ANOC_PCIE_GEM_NOC] = &qnm_pcie, + [MASTER_QPACE] = &qnm_qpace, + [MASTER_SNOC_SF_MEM_NOC] = &qnm_snoc_sf, + [MASTER_WLAN_Q6] = &qnm_wlan_q6, + [MASTER_GIC] = &xm_gic, + [SLAVE_GEM_NOC_CNOC] = &qns_gem_noc_cnoc, + [SLAVE_LLCC] = &qns_llcc, + [SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie, +}; + +static const struct regmap_config kaanapali_gem_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x153080, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_gem_noc = { + .config = &kaanapali_gem_noc_regmap_config, + .nodes = gem_noc_nodes, + .num_nodes = ARRAY_SIZE(gem_noc_nodes), + .bcms = gem_noc_bcms, + .num_bcms = ARRAY_SIZE(gem_noc_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_node * const lpass_ag_noc_nodes[] = { + [MASTER_LPIAON_NOC] = &qnm_lpiaon_noc, + [SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc, +}; + +static const struct regmap_config kaanapali_lpass_ag_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0xe080, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_lpass_ag_noc = { + .config = &kaanapali_lpass_ag_noc_regmap_config, + .nodes = lpass_ag_noc_nodes, + .num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const lpass_lpiaon_noc_bcms[] = { + &bcm_lp0, +}; + +static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = { + [MASTER_LPASS_LPINOC] = &qnm_lpass_lpinoc, + [SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc, +}; + +static const struct regmap_config kaanapali_lpass_lpiaon_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x19080, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_lpass_lpiaon_noc = { + .config = &kaanapali_lpass_lpiaon_noc_regmap_config, + .nodes = lpass_lpiaon_noc_nodes, + .num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes), + .bcms = lpass_lpiaon_noc_bcms, + .num_bcms = ARRAY_SIZE(lpass_lpiaon_noc_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = { + [MASTER_LPASS_PROC] = &qnm_lpinoc_dsp_qns4m, + [SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc, +}; + +static const struct regmap_config kaanapali_lpass_lpicx_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x44080, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_lpass_lpicx_noc = { + .config = &kaanapali_lpass_lpicx_noc_regmap_config, + .nodes = lpass_lpicx_noc_nodes, + .num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const mc_virt_bcms[] = { + &bcm_acv, + &bcm_mc0, +}; + +static struct qcom_icc_node * const mc_virt_nodes[] = { + [MASTER_LLCC] = &llcc_mc, + [SLAVE_EBI1] = &ebi, +}; + +static const struct qcom_icc_desc kaanapali_mc_virt = { + .nodes = mc_virt_nodes, + .num_nodes = ARRAY_SIZE(mc_virt_nodes), + .bcms = mc_virt_bcms, + .num_bcms = ARRAY_SIZE(mc_virt_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const mmss_noc_bcms[] = { + &bcm_mm0, + &bcm_mm1, +}; + +static struct qcom_icc_node * const mmss_noc_nodes[] = { + [MASTER_CAMNOC_HF] = &qnm_camnoc_hf, + [MASTER_CAMNOC_NRT_ICP_SF] = &qnm_camnoc_nrt_icp_sf, + [MASTER_CAMNOC_RT_CDM_SF] = &qnm_camnoc_rt_cdm_sf, + [MASTER_CAMNOC_SF] = &qnm_camnoc_sf, + [MASTER_MDP] = &qnm_mdp, + [MASTER_MDSS_DCP] = &qnm_mdss_dcp, + [MASTER_CDSP_HCP] = &qnm_vapss_hcp, + [MASTER_VIDEO_CV_PROC] = &qnm_video_cv_cpu, + [MASTER_VIDEO_EVA] = &qnm_video_eva, + [MASTER_VIDEO_MVP] = &qnm_video_mvp, + [MASTER_VIDEO_V_PROC] = &qnm_video_v_cpu, + [MASTER_CNOC_MNOC_CFG] = &qsm_mnoc_cfg, + [SLAVE_MNOC_HF_MEM_NOC] = &qns_mem_noc_hf, + [SLAVE_MNOC_SF_MEM_NOC] = &qns_mem_noc_sf, + [SLAVE_SERVICE_MNOC] = &srvc_mnoc, +}; + +static const struct regmap_config kaanapali_mmss_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x5b800, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_mmss_noc = { + .config = &kaanapali_mmss_noc_regmap_config, + .nodes = mmss_noc_nodes, + .num_nodes = ARRAY_SIZE(mmss_noc_nodes), + .bcms = mmss_noc_bcms, + .num_bcms = ARRAY_SIZE(mmss_noc_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const nsp_noc_bcms[] = { + &bcm_co0, +}; + +static struct qcom_icc_node * const nsp_noc_nodes[] = { + [MASTER_CDSP_PROC] = &qnm_nsp, + [SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc, +}; + +static const struct regmap_config kaanapali_nsp_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x21280, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_nsp_noc = { + .config = &kaanapali_nsp_noc_regmap_config, + .nodes = nsp_noc_nodes, + .num_nodes = ARRAY_SIZE(nsp_noc_nodes), + .bcms = nsp_noc_bcms, + .num_bcms = ARRAY_SIZE(nsp_noc_bcms), + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const pcie_anoc_bcms[] = { + &bcm_sn4, +}; + +static struct qcom_icc_node * const pcie_anoc_nodes[] = { + [MASTER_PCIE_ANOC_CFG] = &qsm_pcie_anoc_cfg, + [MASTER_PCIE_0] = &xm_pcie, + [SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_gemnoc, + [SLAVE_SERVICE_PCIE_ANOC] = &srvc_pcie_aggre_noc, +}; + +static const struct regmap_config kaanapali_pcie_anoc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x11400, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_pcie_anoc = { + .config = &kaanapali_pcie_anoc_regmap_config, + .nodes = pcie_anoc_nodes, + .num_nodes = ARRAY_SIZE(pcie_anoc_nodes), + .bcms = pcie_anoc_bcms, + .num_bcms = ARRAY_SIZE(pcie_anoc_bcms), + .qos_requires_clocks = true, + .alloc_dyn_id = true, +}; + +static struct qcom_icc_bcm * const system_noc_bcms[] = { + &bcm_sn0, + &bcm_sn2, + &bcm_sn3, +}; + +static struct qcom_icc_node * const system_noc_nodes[] = { + [MASTER_A1NOC_SNOC] = &qnm_aggre1_noc, + [MASTER_A2NOC_SNOC] = &qnm_aggre2_noc, + [MASTER_APSS_NOC] = &qnm_apss_noc, + [MASTER_CNOC_SNOC] = &qnm_cnoc_data, + [SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf, +}; + +static const struct regmap_config kaanapali_system_noc_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1f080, + .fast_io = true, +}; + +static const struct qcom_icc_desc kaanapali_system_noc = { + .config = &kaanapali_system_noc_regmap_config, + .nodes = system_noc_nodes, + .num_nodes = ARRAY_SIZE(system_noc_nodes), + .bcms = system_noc_bcms, + .num_bcms = ARRAY_SIZE(system_noc_bcms), + .alloc_dyn_id = true, +}; + +static const struct of_device_id qnoc_of_match[] = { + { .compatible = "qcom,kaanapali-aggre-noc", .data = &kaanapali_aggre_noc }, + { .compatible = "qcom,kaanapali-clk-virt", .data = &kaanapali_clk_virt }, + { .compatible = "qcom,kaanapali-cnoc-cfg", .data = &kaanapali_cnoc_cfg }, + { .compatible = "qcom,kaanapali-cnoc-main", .data = &kaanapali_cnoc_main }, + { .compatible = "qcom,kaanapali-gem-noc", .data = &kaanapali_gem_noc }, + { .compatible = "qcom,kaanapali-lpass-ag-noc", .data = &kaanapali_lpass_ag_noc }, + { .compatible = "qcom,kaanapali-lpass-lpiaon-noc", .data = &kaanapali_lpass_lpiaon_noc }, + { .compatible = "qcom,kaanapali-lpass-lpicx-noc", .data = &kaanapali_lpass_lpicx_noc }, + { .compatible = "qcom,kaanapali-mc-virt", .data = &kaanapali_mc_virt }, + { .compatible = "qcom,kaanapali-mmss-noc", .data = &kaanapali_mmss_noc }, + { .compatible = "qcom,kaanapali-nsp-noc", .data = &kaanapali_nsp_noc }, + { .compatible = "qcom,kaanapali-pcie-anoc", .data = &kaanapali_pcie_anoc }, + { .compatible = "qcom,kaanapali-system-noc", .data = &kaanapali_system_noc }, + { } +}; +MODULE_DEVICE_TABLE(of, qnoc_of_match); + +static struct platform_driver qnoc_driver = { + .probe = qcom_icc_rpmh_probe, + .remove = qcom_icc_rpmh_remove, + .driver = { + .name = "qnoc-kaanapali", + .of_match_table = qnoc_of_match, + .sync_state = icc_sync_state, + }, +}; + +static int __init qnoc_driver_init(void) +{ + return platform_driver_register(&qnoc_driver); +} +core_initcall(qnoc_driver_init); + +static void __exit qnoc_driver_exit(void) +{ + platform_driver_unregister(&qnoc_driver); +} +module_exit(qnoc_driver_exit); + +MODULE_DESCRIPTION("Qualcomm Kaanapali NoC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/interconnect/qcom/msm8996.c b/drivers/interconnect/qcom/msm8996.c index b73566c9b21f9..84cfafb22aa17 100644 --- a/drivers/interconnect/qcom/msm8996.c +++ b/drivers/interconnect/qcom/msm8996.c @@ -552,6 +552,7 @@ static struct qcom_icc_node mas_venus_vmem = { static const u16 mas_snoc_pnoc_links[] = { MSM8996_SLAVE_BLSP_1, MSM8996_SLAVE_BLSP_2, + MSM8996_SLAVE_USB_HS, MSM8996_SLAVE_SDCC_1, MSM8996_SLAVE_SDCC_2, MSM8996_SLAVE_SDCC_4, diff --git a/drivers/interconnect/qcom/sdx75.c b/drivers/interconnect/qcom/sdx75.c index 7ef1f17f3292e..2def75f67eb8e 100644 --- a/drivers/interconnect/qcom/sdx75.c +++ b/drivers/interconnect/qcom/sdx75.c @@ -16,15 +16,6 @@ #include "icc-rpmh.h" #include "sdx75.h" -static struct qcom_icc_node qpic_core_master = { - .name = "qpic_core_master", - .id = SDX75_MASTER_QPIC_CORE, - .channels = 1, - .buswidth = 4, - .num_links = 1, - .links = { SDX75_SLAVE_QPIC_CORE }, -}; - static struct qcom_icc_node qup0_core_master = { .name = "qup0_core_master", .id = SDX75_MASTER_QUP_CORE_0, @@ -375,14 +366,6 @@ static struct qcom_icc_node xm_usb3 = { .links = { SDX75_SLAVE_A1NOC_CFG }, }; -static struct qcom_icc_node qpic_core_slave = { - .name = "qpic_core_slave", - .id = SDX75_SLAVE_QPIC_CORE, - .channels = 1, - .buswidth = 4, - .num_links = 0, -}; - static struct qcom_icc_node qup0_core_slave = { .name = "qup0_core_slave", .id = SDX75_SLAVE_QUP_CORE_0, @@ -831,12 +814,6 @@ static struct qcom_icc_bcm bcm_mc0 = { .nodes = { &ebi }, }; -static struct qcom_icc_bcm bcm_qp0 = { - .name = "QP0", - .num_nodes = 1, - .nodes = { &qpic_core_slave }, -}; - static struct qcom_icc_bcm bcm_qup0 = { .name = "QUP0", .keepalive = true, @@ -898,14 +875,11 @@ static struct qcom_icc_bcm bcm_sn4 = { }; static struct qcom_icc_bcm * const clk_virt_bcms[] = { - &bcm_qp0, &bcm_qup0, }; static struct qcom_icc_node * const clk_virt_nodes[] = { - [MASTER_QPIC_CORE] = &qpic_core_master, [MASTER_QUP_CORE_0] = &qup0_core_master, - [SLAVE_QPIC_CORE] = &qpic_core_slave, [SLAVE_QUP_CORE_0] = &qup0_core_slave, }; diff --git a/drivers/interconnect/qcom/sdx75.h b/drivers/interconnect/qcom/sdx75.h index 24e8871599201..34f51add59dc0 100644 --- a/drivers/interconnect/qcom/sdx75.h +++ b/drivers/interconnect/qcom/sdx75.h @@ -33,7 +33,6 @@ #define SDX75_MASTER_QDSS_ETR 24 #define SDX75_MASTER_QDSS_ETR_1 25 #define SDX75_MASTER_QPIC 26 -#define SDX75_MASTER_QPIC_CORE 27 #define SDX75_MASTER_QUP_0 28 #define SDX75_MASTER_QUP_CORE_0 29 #define SDX75_MASTER_SDCC_1 30 @@ -76,7 +75,6 @@ #define SDX75_SLAVE_QDSS_CFG 67 #define SDX75_SLAVE_QDSS_STM 68 #define SDX75_SLAVE_QPIC 69 -#define SDX75_SLAVE_QPIC_CORE 70 #define SDX75_SLAVE_QUP_0 71 #define SDX75_SLAVE_QUP_CORE_0 72 #define SDX75_SLAVE_SDCC_1 73 diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 9b4b589a54b57..bf77fdf5529f4 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -15,7 +15,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data); irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_galog(int irq, void *data); -irqreturn_t amd_iommu_int_handler(int irq, void *data); void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, u8 cntrl_intr, u8 cntrl_log, u32 status_run_mask, u32 status_overflow_mask); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index a698a2e7ce2a6..b0d919cd1a8fb 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -791,7 +791,7 @@ struct amd_iommu { u32 flags; volatile u64 *cmd_sem; - atomic64_t cmd_sem_val; + u64 cmd_sem_val; /* * Track physical address to directly use it in build_completion_wait() * and avoid adding any special checks and handling for kdump. diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 10fa217a71199..20b04996441d6 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -37,7 +37,7 @@ static ssize_t iommu_mmio_write(struct file *filp, const char __user *ubuf, if (ret) return ret; - if (iommu->dbg_mmio_offset > iommu->mmio_phys_end - 4) { + if (iommu->dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) { iommu->dbg_mmio_offset = -1; return -EINVAL; } diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index f2991c11867cb..76efd74124b33 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1136,9 +1136,13 @@ static void set_dte_bit(struct dev_table_entry *dte, u8 bit) static bool __reuse_device_table(struct amd_iommu *iommu) { struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; - u32 lo, hi, old_devtb_size; + struct dev_table_entry *old_dev_tbl_entry; + u32 lo, hi, old_devtb_size, devid; phys_addr_t old_devtb_phys; + u16 dom_id; + bool dte_v; u64 entry; + int ret; /* Each IOMMU use separate device table with the same size */ lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET); @@ -1173,6 +1177,23 @@ static bool __reuse_device_table(struct amd_iommu *iommu) return false; } + for (devid = 0; devid <= pci_seg->last_bdf; devid++) { + old_dev_tbl_entry = &pci_seg->old_dev_tbl_cpy[devid]; + dte_v = FIELD_GET(DTE_FLAG_V, old_dev_tbl_entry->data[0]); + dom_id = FIELD_GET(DEV_DOMID_MASK, old_dev_tbl_entry->data[1]); + + if (!dte_v || !dom_id) + continue; + /* + * ID reservation can fail with -ENOSPC when there + * are multiple devices present in the same domain, + * hence check only for -ENOMEM. + */ + ret = ida_alloc_range(&pdom_ids, dom_id, dom_id, GFP_KERNEL); + if (ret == -ENOMEM) + return false; + } + return true; } @@ -1710,13 +1731,22 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list); if (alloc_dev_table(pci_seg)) - return NULL; + goto err_free_pci_seg; if (alloc_alias_table(pci_seg)) - return NULL; + goto err_free_dev_table; if (alloc_rlookup_table(pci_seg)) - return NULL; + goto err_free_alias_table; return pci_seg; + +err_free_alias_table: + free_alias_table(pci_seg); +err_free_dev_table: + free_dev_table(pci_seg); +err_free_pci_seg: + list_del(&pci_seg->list); + kfree(pci_seg); + return NULL; } static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id, @@ -1849,7 +1879,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, iommu->pci_seg = pci_seg; raw_spin_lock_init(&iommu->lock); - atomic64_set(&iommu->cmd_sem_val, 0); + iommu->cmd_sem_val = 0; /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); @@ -2325,12 +2355,8 @@ static int iommu_setup_msi(struct amd_iommu *iommu) if (r) return r; - r = request_threaded_irq(iommu->dev->irq, - amd_iommu_int_handler, - amd_iommu_int_thread, - 0, "AMD-Vi", - iommu); - + r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread, + IRQF_ONESHOT, "AMD-Vi", iommu); if (r) { pci_disable_msi(iommu->dev); return r; @@ -2504,8 +2530,8 @@ static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname, return irq; } - ret = request_threaded_irq(irq, amd_iommu_int_handler, - thread_fn, 0, devname, iommu); + ret = request_threaded_irq(irq, NULL, thread_fn, IRQF_ONESHOT, devname, + iommu); if (ret) { irq_domain_free_irqs(irq, 1); irq_domain_remove(domain); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 2e1865daa1cee..4beef73139611 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1146,11 +1146,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data) return IRQ_HANDLED; } -irqreturn_t amd_iommu_int_handler(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - /**************************************************************************** * * IOMMU command queuing functions @@ -1161,7 +1156,12 @@ static int wait_on_sem(struct amd_iommu *iommu, u64 data) { int i = 0; - while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) { + /* + * cmd_sem holds a monotonically non-decreasing completion sequence + * number. + */ + while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 && + i < LOOP_TIMEOUT) { udelay(1); i += 1; } @@ -1386,6 +1386,12 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) return iommu_queue_command_sync(iommu, cmd, true); } +static u64 get_cmdsem_val(struct amd_iommu *iommu) +{ + lockdep_assert_held(&iommu->lock); + return ++iommu->cmd_sem_val; +} + /* * This function queues a completion wait command into the command * buffer of an IOMMU @@ -1400,20 +1406,19 @@ static int iommu_completion_wait(struct amd_iommu *iommu) if (!iommu->need_sync) return 0; - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd, iommu, data); - raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + if (ret) - goto out_unlock; + return ret; ret = wait_on_sem(iommu, data); -out_unlock: - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; } @@ -2426,8 +2431,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) goto out_err; } -out_err: - iommu_completion_wait(iommu); if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) @@ -2438,6 +2441,7 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) if (dev_is_pci(dev)) pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); +out_err: return iommu_dev; } @@ -3088,18 +3092,23 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) return; build_inv_irt(&cmd, devid); - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd2, iommu, data); raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd2, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, true); if (ret) - goto out; + goto out_err; ret = __iommu_queue_command_sync(iommu, &cmd2, false); if (ret) - goto out; + goto out_err; + raw_spin_unlock_irqrestore(&iommu->lock, flags); + wait_on_sem(iommu, data); -out: + return; + +out_err: raw_spin_unlock_irqrestore(&iommu->lock, flags); } @@ -3354,7 +3363,7 @@ static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, struct irte_ga *irte) { - bool ret; + int ret; ret = __modify_irte_ga(iommu, devid, index, irte); if (ret) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 95a4e62b8f63c..9804022c7f590 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -802,6 +802,8 @@ static int apple_dart_of_xlate(struct device *dev, struct apple_dart *cfg_dart; int i, sid; + put_device(&iommu_pdev->dev); + if (args->args_count != 1) return -EINVAL; sid = args->args[0]; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index 8cd8929bbfdf8..9ec5591565083 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -165,7 +165,9 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, * config bit here base this off the EATS value in the STE. If the EATS * is set then the VM must generate ATC flushes. */ - state.disable_ats = !nested_domain->enable_ats; + if (FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(nested_domain->ste[0])) == + STRTAB_STE_0_CFG_S1_TRANS) + state.disable_ats = !nested_domain->enable_ats; ret = arm_smmu_attach_prepare(&state, domain); if (ret) { mutex_unlock(&arm_smmu_asid_lock); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index d2671bfd37981..b254a94b2003d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -38,13 +38,16 @@ enum arm_smmu_test_master_feat { static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry, const __le64 *used_bits, const __le64 *target, + const __le64 *safe, unsigned int length) { bool differs = false; unsigned int i; for (i = 0; i < length; i++) { - if ((entry[i] & used_bits[i]) != target[i]) + __le64 used = used_bits[i] & ~safe[i]; + + if ((entry[i] & used) != (target[i] & used)) differs = true; } return differs; @@ -56,12 +59,24 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) struct arm_smmu_test_writer *test_writer = container_of(writer, struct arm_smmu_test_writer, writer); __le64 *entry_used_bits; + __le64 *safe_target; + __le64 *safe_init; entry_used_bits = kunit_kzalloc( test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits); + safe_target = kunit_kzalloc(test_writer->test, + sizeof(*safe_target) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_target); + + safe_init = kunit_kzalloc(test_writer->test, + sizeof(*safe_init) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_init); + pr_debug("STE value is now set to: "); print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, test_writer->entry, @@ -79,14 +94,23 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) * configuration. */ writer->ops->get_used(test_writer->entry, entry_used_bits); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(test_writer->entry, + test_writer->init_entry, + safe_init); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(test_writer->entry, + test_writer->target_entry, + safe_target); KUNIT_EXPECT_FALSE( test_writer->test, arm_smmu_entry_differs_in_used_bits( test_writer->entry, entry_used_bits, - test_writer->init_entry, NUM_ENTRY_QWORDS) && + test_writer->init_entry, safe_init, + NUM_ENTRY_QWORDS) && arm_smmu_entry_differs_in_used_bits( test_writer->entry, entry_used_bits, - test_writer->target_entry, + test_writer->target_entry, safe_target, NUM_ENTRY_QWORDS)); } } @@ -106,6 +130,7 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, static const struct arm_smmu_entry_writer_ops test_ste_ops = { .sync = arm_smmu_test_writer_record_syncs, .get_used = arm_smmu_get_ste_used, + .get_update_safe = arm_smmu_get_ste_update_safe, }; static const struct arm_smmu_entry_writer_ops test_cd_ops = { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 2a8b46b948f05..1e47da0ce6b9b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -487,20 +487,26 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) */ static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq) { - int val; - /* - * We can try to avoid the cmpxchg() loop by simply incrementing the - * lock counter. When held in exclusive state, the lock counter is set - * to INT_MIN so these increments won't hurt as the value will remain - * negative. + * When held in exclusive state, the lock counter is set to INT_MIN + * so these increments won't hurt as the value will remain negative. + * The increment will also signal the exclusive locker that there are + * shared waiters. */ if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) return; - do { - val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0); - } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val); + /* + * Someone else is holding the lock in exclusive state, so wait + * for them to finish. Since we already incremented the lock counter, + * no exclusive lock can be acquired until we finish. We don't need + * the return value since we only care that the exclusive lock is + * released (i.e. the lock counter is non-negative). + * Once the exclusive locker releases the lock, the sign bit will + * be cleared and our increment will make the lock counter positive, + * allowing us to proceed. + */ + atomic_cond_read_relaxed(&cmdq->lock, VAL > 0); } static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq) @@ -527,9 +533,14 @@ static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq) __ret; \ }) +/* + * Only clear the sign bit when releasing the exclusive lock this will + * allow any shared_lock() waiters to proceed without the possibility + * of entering the exclusive lock in a tight loop. + */ #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \ ({ \ - atomic_set_release(&cmdq->lock, 0); \ + atomic_fetch_andnot_release(INT_MIN, &cmdq->lock); \ local_irq_restore(flags); \ }) @@ -1082,6 +1093,49 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used); +VISIBLE_IF_KUNIT +void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, + __le64 *safe_bits) +{ + const __le64 eats_s1chk = + FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK); + const __le64 eats_trans = + FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS); + + /* + * When an STE changes EATS_TRANS, the sequencing code in the attach + * logic already will have the PCI cap for ATS disabled. Thus at this + * moment we can expect that the device will not generate ATS queries + * and so we don't care about the sequencing of EATS. The purpose of + * EATS_TRANS is to protect the system from hostile untrusted devices + * that issue ATS when the PCI config space is disabled. However, if + * EATS_TRANS is being changed, then we must have already trusted the + * device as the EATS_TRANS security block is being disabled. + * + * Note: now the EATS_TRANS update is moved to the first entry_set(). + * Changing S2S and EATS might transiently result in S2S=1 and EATS=1 + * which is a bad STE (see "5.2 Stream Table Entry"). In such a case, + * we can't do a hitless update. Also, it should not be added to the + * safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be + * effectively an errant 0b00 configuration. + */ + if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) && + !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S))) + safe_bits[1] |= cpu_to_le64(eats_trans); + + /* + * MEV does not meaningfully impact the operation of the HW, it only + * changes how many fault events are generated, thus we can relax it + * when computing the ordering. The spec notes the device can act like + * MEV=1 anyhow: + * + * Note: Software must expect, and be able to deal with, coalesced + * fault records even when MEV == 0. + */ + safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV); +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe); + /* * Figure out if we can do a hitless update of entry to become target. Returns a * bit mask where 1 indicates that qword needs to be set disruptively. @@ -1094,13 +1148,22 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, { __le64 target_used[NUM_ENTRY_QWORDS] = {}; __le64 cur_used[NUM_ENTRY_QWORDS] = {}; + __le64 safe[NUM_ENTRY_QWORDS] = {}; u8 used_qword_diff = 0; unsigned int i; writer->ops->get_used(entry, cur_used); writer->ops->get_used(target, target_used); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(entry, target, safe); for (i = 0; i != NUM_ENTRY_QWORDS; i++) { + /* + * Safe is only used for bits that are used by both entries, + * otherwise it is sequenced according to the unused entry. + */ + safe[i] &= target_used[i] & cur_used[i]; + /* * Check that masks are up to date, the make functions are not * allowed to set a bit to 1 if the used function doesn't say it @@ -1109,6 +1172,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, WARN_ON_ONCE(target[i] & ~target_used[i]); /* Bits can change because they are not currently being used */ + cur_used[i] &= ~safe[i]; unused_update[i] = (entry[i] & cur_used[i]) | (target[i] & ~cur_used[i]); /* @@ -1121,7 +1185,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, return used_qword_diff; } -static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, +static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, const __le64 *target, unsigned int start, unsigned int len) { @@ -1137,7 +1201,6 @@ static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, if (changed) writer->ops->sync(writer); - return changed; } /* @@ -1207,12 +1270,9 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, entry_set(writer, entry, target, 0, 1); } else { /* - * No inuse bit changed. Sanity check that all unused bits are 0 - * in the entry. The target was already sanity checked by - * compute_qword_diff(). + * No inuse bit changed, though safe bits may have changed. */ - WARN_ON_ONCE( - entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS)); + entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS); } } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry); @@ -1464,7 +1524,7 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size, &cd_table->cdtab_dma, GFP_KERNEL); - if (!cd_table->l2.l2ptrs) { + if (!cd_table->l2.l1tab) { ret = -ENOMEM; goto err_free_l2ptrs; } @@ -1543,6 +1603,7 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = { .sync = arm_smmu_ste_writer_sync_entry, .get_used = arm_smmu_get_ste_used, + .get_update_safe = arm_smmu_get_ste_update_safe, }; static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ae23aacc38402..287e223c054d1 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -900,6 +900,8 @@ struct arm_smmu_entry_writer { struct arm_smmu_entry_writer_ops { void (*get_used)(const __le64 *entry, __le64 *used); + void (*get_update_safe)(const __le64 *cur, const __le64 *target, + __le64 *safe_bits); void (*sync)(struct arm_smmu_entry_writer *writer); }; @@ -911,6 +913,8 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, #if IS_ENABLED(CONFIG_KUNIT) void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits); +void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, + __le64 *safe_bits); void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur, const __le64 *target); void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits); diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 378104cd395e5..04cc7a9036e43 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -1078,6 +1078,9 @@ static int tegra241_vcmdq_hw_init_user(struct tegra241_vcmdq *vcmdq) { char header[64]; + /* Reset VCMDQ */ + tegra241_vcmdq_hw_deinit(vcmdq); + /* Configure the vcmdq only; User space does the enabling */ writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE)); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c index db9b9a8e139c8..4565a58bb213f 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c @@ -228,3 +228,17 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init arm_smmu_impl_module_init(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + return qcom_smmu_module_init(); + + return 0; +} + +void __exit arm_smmu_impl_module_exit(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + qcom_smmu_module_exit(); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 57c097e876130..156e9d89b885d 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -39,14 +39,42 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,adreno-smmu", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, - { .compatible = "qcom,fastrpc", + { .compatible = "qcom,fastrpc-compute-cb", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, + { .compatible = "qcom,qcm2290-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sa8775p-mdss", + .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, { .compatible = "qcom,sc7280-mdss", .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, { .compatible = "qcom,sc7280-venus", .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sc8180x-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sc8280xp-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sm6115-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sm6125-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sm6350-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sm8150-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sm8250-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sm8350-mdss", + .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, + { .compatible = "qcom,sm8450-mdss", + .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, { .compatible = "qcom,sm8550-mdss", .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, + { .compatible = "qcom,sm8650-mdss", + .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, + { .compatible = "qcom,sm8750-mdss", + .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, + { .compatible = "qcom,x1e80100-mdss", + .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, { } }; @@ -431,17 +459,19 @@ static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu) /* * Some platforms support more than the Arm SMMU architected maximum of - * 128 stream matching groups. For unknown reasons, the additional - * groups don't exhibit the same behavior as the architected registers, - * so limit the groups to 128 until the behavior is fixed for the other - * groups. + * 128 stream matching groups. The additional registers appear to have + * the same behavior as the architected registers in the hardware. + * However, on some firmware versions, the hypervisor does not + * correctly trap and emulate accesses to the additional registers, + * resulting in unexpected behavior. + * + * If there are more than 128 groups, use the last reliable group to + * detect if we need to apply the bypass quirk. */ - if (smmu->num_mapping_groups > 128) { - dev_notice(smmu->dev, "\tLimiting the stream matching groups to 128\n"); - smmu->num_mapping_groups = 128; - } - - last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1); + if (smmu->num_mapping_groups > 128) + last_s2cr = ARM_SMMU_GR0_S2CR(127); + else + last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1); /* * With some firmware versions writes to S2CR of type FAULT are @@ -464,6 +494,11 @@ static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu) reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS); arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg); + + if (smmu->num_mapping_groups > 128) { + dev_notice(smmu->dev, "\tLimiting the stream matching groups to 128\n"); + smmu->num_mapping_groups = 128; + } } for (i = 0; i < smmu->num_mapping_groups; i++) { @@ -766,10 +801,6 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { const struct device_node *np = smmu->dev->of_node; const struct of_device_id *match; - static u8 tbu_registered; - - if (!tbu_registered++) - platform_driver_register(&qcom_smmu_tbu_driver); #ifdef CONFIG_ACPI if (np == NULL) { @@ -794,3 +825,13 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init qcom_smmu_module_init(void) +{ + return platform_driver_register(&qcom_smmu_tbu_driver); +} + +void __exit qcom_smmu_module_exit(void) +{ + platform_driver_unregister(&qcom_smmu_tbu_driver); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 4ced4b5bee4df..54ddcb709885d 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -462,10 +462,20 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) int idx = smmu_domain->cfg.cbndx; int ret; + if (!pm_runtime_get_if_active(smmu->dev)) + return IRQ_NONE; + + if (smmu->impl && smmu->impl->context_fault) { + ret = smmu->impl->context_fault(irq, dev); + goto out_power_off; + } + arm_smmu_read_context_fault_info(smmu, idx, &cfi); - if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT)) - return IRQ_NONE; + if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT)) { + ret = IRQ_NONE; + goto out_power_off; + } ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova, cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); @@ -480,7 +490,12 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE); } - return IRQ_HANDLED; + ret = IRQ_HANDLED; + +out_power_off: + pm_runtime_put_autosuspend(smmu->dev); + + return ret; } static irqreturn_t arm_smmu_global_fault(int irq, void *dev) @@ -489,14 +504,25 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) struct arm_smmu_device *smmu = dev; static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); + int ret; + + if (!pm_runtime_get_if_active(smmu->dev)) + return IRQ_NONE; + + if (smmu->impl && smmu->impl->global_fault) { + ret = smmu->impl->global_fault(irq, dev); + goto out_power_off; + } gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); - if (!gfsr) - return IRQ_NONE; + if (!gfsr) { + ret = IRQ_NONE; + goto out_power_off; + } if (__ratelimit(&rs)) { if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && @@ -513,7 +539,11 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) } arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); - return IRQ_HANDLED; + ret = IRQ_HANDLED; + +out_power_off: + pm_runtime_put_autosuspend(smmu->dev); + return ret; } static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, @@ -683,7 +713,6 @@ static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain, enum io_pgtable_fmt fmt; struct iommu_domain *domain = &smmu_domain->domain; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - irqreturn_t (*context_fault)(int irq, void *dev); mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) @@ -850,19 +879,14 @@ static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain, */ irq = smmu->irqs[cfg->irptndx]; - if (smmu->impl && smmu->impl->context_fault) - context_fault = smmu->impl->context_fault; - else - context_fault = arm_smmu_context_fault; - if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq) ret = devm_request_threaded_irq(smmu->dev, irq, NULL, - context_fault, + arm_smmu_context_fault, IRQF_ONESHOT | IRQF_SHARED, "arm-smmu-context-fault", smmu_domain); else - ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, + ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, IRQF_SHARED, "arm-smmu-context-fault", smmu_domain); if (ret < 0) { @@ -2122,7 +2146,6 @@ static int arm_smmu_device_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; int num_irqs, i, err; u32 global_irqs, pmu_irqs; - irqreturn_t (*global_fault)(int irq, void *dev); smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { @@ -2202,18 +2225,13 @@ static int arm_smmu_device_probe(struct platform_device *pdev) smmu->num_context_irqs = smmu->num_context_banks; } - if (smmu->impl && smmu->impl->global_fault) - global_fault = smmu->impl->global_fault; - else - global_fault = arm_smmu_global_fault; - for (i = 0; i < global_irqs; i++) { int irq = platform_get_irq(pdev, i); if (irq < 0) return irq; - err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED, + err = devm_request_irq(dev, irq, arm_smmu_global_fault, IRQF_SHARED, "arm-smmu global fault", smmu); if (err) return dev_err_probe(dev, err, @@ -2362,7 +2380,29 @@ static struct platform_driver arm_smmu_driver = { .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -module_platform_driver(arm_smmu_driver); + +static int __init arm_smmu_init(void) +{ + int ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) + return ret; + + ret = arm_smmu_impl_module_init(); + if (ret) + platform_driver_unregister(&arm_smmu_driver); + + return ret; +} +module_init(arm_smmu_init); + +static void __exit arm_smmu_exit(void) +{ + arm_smmu_impl_module_exit(); + platform_driver_unregister(&arm_smmu_driver); +} +module_exit(arm_smmu_exit); MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 2dbf3243b5ad2..26d2e33cd328b 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -540,6 +540,11 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); +int __init arm_smmu_impl_module_init(void); +void __exit arm_smmu_impl_module_exit(void); +int __init qcom_smmu_module_init(void); +void __exit qcom_smmu_module_exit(void); + void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); int arm_mmu500_reset(struct arm_smmu_device *smmu); diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index c5be95e560317..9c1166a3af6c9 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -565,14 +565,14 @@ static int qcom_iommu_of_xlate(struct device *dev, qcom_iommu = platform_get_drvdata(iommu_pdev); + put_device(&iommu_pdev->dev); + /* make sure the asid specified in dt is valid, so we don't have * to sanity check this elsewhere: */ if (WARN_ON(asid > qcom_iommu->max_asid) || - WARN_ON(qcom_iommu->ctxs[asid] == NULL)) { - put_device(&iommu_pdev->dev); + WARN_ON(qcom_iommu->ctxs[asid] == NULL)) return -EINVAL; - } if (!dev_iommu_priv_get(dev)) { dev_iommu_priv_set(dev, qcom_iommu); @@ -581,10 +581,8 @@ static int qcom_iommu_of_xlate(struct device *dev, * multiple different iommu devices. Multiple context * banks are ok, but multiple devices are not: */ - if (WARN_ON(qcom_iommu != dev_iommu_priv_get(dev))) { - put_device(&iommu_pdev->dev); + if (WARN_ON(qcom_iommu != dev_iommu_priv_get(dev))) return -EINVAL; - } } return iommu_fwspec_add_ids(dev, &asid, 1); diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index b6edd178fe25e..ce9e935cb84c3 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1446,17 +1446,14 @@ static int exynos_iommu_of_xlate(struct device *dev, return -ENODEV; data = platform_get_drvdata(sysmmu); - if (!data) { - put_device(&sysmmu->dev); + put_device(&sysmmu->dev); + if (!data) return -ENODEV; - } if (!owner) { owner = kzalloc(sizeof(*owner), GFP_KERNEL); - if (!owner) { - put_device(&sysmmu->dev); + if (!owner) return -ENOMEM; - } INIT_LIST_HEAD(&owner->controllers); mutex_init(&owner->rpm_lock); diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index f2f538c706503..17a91f881b2e9 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -66,7 +66,7 @@ config INTEL_IOMMU_DEFAULT_ON config INTEL_IOMMU_FLOPPY_WA def_bool y - depends on X86 + depends on X86 && BLK_DEV_FD help Floppy disk drivers are known to bypass DMA API calls thereby failing to work when IOMMU is enabled. This diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 265e7290256b5..385ae5cfb30d4 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -363,6 +363,13 @@ static void qi_batch_add_pasid_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qi_batch_increment_index(iommu, batch); } +static bool intel_domain_use_piotlb(struct dmar_domain *domain) +{ + return domain->domain.type == IOMMU_DOMAIN_SVA || + domain->domain.type == IOMMU_DOMAIN_NESTED || + intel_domain_is_fs_paging(domain); +} + static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *tag, unsigned long addr, unsigned long pages, unsigned long mask, int ih) @@ -370,7 +377,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag * struct intel_iommu *iommu = tag->iommu; u64 type = DMA_TLB_PSI_FLUSH; - if (intel_domain_is_fs_paging(domain)) { + if (intel_domain_use_piotlb(domain)) { qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr, pages, ih, domain->qi_batch); return; diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index ec975c73cfe6c..6938800e98845 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1314,7 +1314,6 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) if (fault & DMA_FSTS_ITE) { head = readl(iommu->reg + DMAR_IQH_REG); head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH; - head |= 1; tail = readl(iommu->reg + DMAR_IQT_REG); tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH; @@ -1331,7 +1330,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) do { if (qi->desc_status[head] == QI_IN_USE) qi->desc_status[head] = QI_ABORT; - head = (head - 2 + QI_LENGTH) % QI_LENGTH; + head = (head - 1 + QI_LENGTH) % QI_LENGTH; } while (head != tail); /* diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index e236c7ec221f4..49e83c8566a32 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1722,10 +1722,12 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 } did = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); intel_context_flush_no_pasid(info, context, did); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); } int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev, diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 3056583d7f56b..9198ac7f6bbab 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -969,7 +969,26 @@ static inline unsigned long lvl_to_nr_pages(unsigned int lvl) static inline void context_set_present(struct context_entry *context) { - context->lo |= 1; + u64 val; + + dma_wmb(); + val = READ_ONCE(context->lo) | 1; + WRITE_ONCE(context->lo, val); +} + +/* + * Clear the Present (P) bit (bit 0) of a context table entry. This initiates + * the transition of the entry's ownership from hardware to software. The + * caller is responsible for fulfilling the invalidation handshake recommended + * by the VT-d spec, Section 6.5.3.3 (Guidance to Software for Invalidations). + */ +static inline void context_clear_present(struct context_entry *context) +{ + u64 val; + + val = READ_ONCE(context->lo) & GENMASK_ULL(63, 1); + WRITE_ONCE(context->lo, val); + dma_wmb(); } static inline void context_set_fault_enable(struct context_entry *context) @@ -1097,7 +1116,7 @@ static inline void qi_desc_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, struct qi_desc *desc) { u8 dw = 0, dr = 0; - int ih = 0; + int ih = addr & 1; if (cap_write_drain(iommu->cap)) dw = 1; diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 4f9b01dc91e86..8bcbfe3d9c722 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -1303,17 +1303,17 @@ static struct irq_chip intel_ir_chip = { * irq_enter(); * handle_edge_irq() * irq_chip_ack_parent() - * irq_move_irq(); // No EOI + * intel_ack_posted_msi_irq(); // No EOI * handle_irq_event() * driver_handler() * handle_edge_irq() * irq_chip_ack_parent() - * irq_move_irq(); // No EOI + * intel_ack_posted_msi_irq(); // No EOI * handle_irq_event() * driver_handler() * handle_edge_irq() * irq_chip_ack_parent() - * irq_move_irq(); // No EOI + * intel_ack_posted_msi_irq(); // No EOI * handle_irq_event() * driver_handler() * apic_eoi() @@ -1322,7 +1322,7 @@ static struct irq_chip intel_ir_chip = { */ static struct irq_chip intel_ir_chip_post_msi = { .name = "INTEL-IR-POST", - .irq_ack = irq_move_irq, + .irq_ack = intel_ack_posted_msi_irq, .irq_set_affinity = intel_ir_set_affinity, .irq_compose_msi_msg = intel_ir_compose_msi_msg, .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 52f678975da74..7a64a55fb5887 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -153,6 +153,9 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) if (!entries) return NULL; + if (!ecap_coherent(info->iommu->ecap)) + clflush_cache_range(entries, VTD_PAGE_SIZE); + /* * The pasid directory table entry won't be freed after * allocation. No worry about the race with free and @@ -165,10 +168,8 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) iommu_free_pages(entries); goto retry; } - if (!ecap_coherent(info->iommu->ecap)) { - clflush_cache_range(entries, VTD_PAGE_SIZE); + if (!ecap_coherent(info->iommu->ecap)) clflush_cache_range(&dir[dir_index].val, sizeof(*dir)); - } } return &entries[index]; @@ -218,7 +219,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, if (!info || !info->ats_enabled) return; - if (pci_dev_is_disconnected(to_pci_dev(dev))) + if (!pci_device_is_present(to_pci_dev(dev))) return; sid = PCI_DEVID(info->bus, info->devfn); @@ -272,7 +273,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, did = pasid_get_domain_id(pte); pgtt = pasid_pte_get_pgtt(pte); - intel_pasid_clear_entry(dev, pasid, fault_ignore); + pasid_clear_present(pte); spin_unlock(&iommu->lock); if (!ecap_coherent(iommu->ecap)) @@ -286,6 +287,10 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, pasid); + intel_pasid_clear_entry(dev, pasid, fault_ignore); + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + if (!fault_ignore) intel_iommu_drain_pasid_prq(dev, pasid); } @@ -1023,7 +1028,7 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) } if (context_copied(iommu, bus, devfn)) { - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); /* @@ -1043,6 +1048,9 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + /* * At this point, the device is supposed to finish reset at * its driver probe stage, so no in-flight DMA will exist, @@ -1106,6 +1114,14 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) if (!info->ats_enabled) return; + /* + * Skip dev-IOTLB flush for inaccessible PCIe devices to prevent the + * Intel IOMMU from waiting indefinitely for an ATS invalidation that + * cannot complete. + */ + if (!pci_device_is_present(to_pci_dev(info->dev))) + return; + qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn), info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH); diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index a771a77d4239c..637373995be80 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -233,9 +233,23 @@ static inline void pasid_set_wpe(struct pasid_entry *pe) */ static inline void pasid_set_present(struct pasid_entry *pe) { + dma_wmb(); pasid_set_bits(&pe->val[0], 1 << 0, 1); } +/* + * Clear the Present (P) bit (bit 0) of a scalable-mode PASID table entry. + * This initiates the transition of the entry's ownership from hardware + * to software. The caller is responsible for fulfilling the invalidation + * handshake recommended by the VT-d spec, Section 6.5.3.3 (Guidance to + * Software for Invalidations). + */ +static inline void pasid_clear_present(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[0], 1 << 0, 0); + dma_wmb(); +} + /* * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID * entry. diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e147f71f91b72..4cbbebc027496 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -164,9 +164,12 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, if (IS_ERR(dev_pasid)) return PTR_ERR(dev_pasid); - ret = iopf_for_domain_replace(domain, old, dev); - if (ret) - goto out_remove_dev_pasid; + /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ + if (info->pri_supported) { + ret = iopf_for_domain_replace(domain, old, dev); + if (ret) + goto out_remove_dev_pasid; + } /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; @@ -180,7 +183,8 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, return 0; out_unwind_iopf: - iopf_for_domain_replace(old, domain, dev); + if (info->pri_supported) + iopf_for_domain_replace(old, domain, dev); out_remove_dev_pasid: domain_remove_dev_pasid(domain, dev, pasid); return ret; diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 7e8e2216c294a..040f5c4672564 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -639,7 +639,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, pte = READ_ONCE(*ptep); if (!pte) { WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); - return -ENOENT; + return 0; } /* If the size matches this level, we're in the right place */ diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 1a51cfd82808d..fd735aaae9e3f 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -3,6 +3,7 @@ * Helpers for IOMMU drivers implementing SVA */ #include +#include #include #include #include @@ -10,6 +11,8 @@ #include "iommu-priv.h" static DEFINE_MUTEX(iommu_sva_lock); +static bool iommu_sva_present; +static LIST_HEAD(iommu_sva_mms); static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm); @@ -42,6 +45,7 @@ static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct de return ERR_PTR(-ENOSPC); } iommu_mm->pasid = pasid; + iommu_mm->mm = mm; INIT_LIST_HEAD(&iommu_mm->sva_domains); /* * Make sure the write to mm->iommu_mm is not reordered in front of @@ -132,8 +136,13 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm if (ret) goto out_free_domain; domain->users = 1; - list_add(&domain->next, &mm->iommu_mm->sva_domains); + if (list_empty(&iommu_mm->sva_domains)) { + if (list_empty(&iommu_sva_mms)) + iommu_sva_present = true; + list_add(&iommu_mm->mm_list_elm, &iommu_sva_mms); + } + list_add(&domain->next, &iommu_mm->sva_domains); out: refcount_set(&handle->users, 1); mutex_unlock(&iommu_sva_lock); @@ -173,8 +182,15 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) iommu_detach_device_pasid(domain, dev, iommu_mm->pasid); if (--domain->users == 0) { list_del(&domain->next); + if (list_empty(&iommu_mm->sva_domains)) { + list_del(&iommu_mm->mm_list_elm); + if (list_empty(&iommu_sva_mms)) + iommu_sva_present = false; + } + iommu_domain_free(domain); } + mutex_unlock(&iommu_sva_lock); kfree(handle); } @@ -312,3 +328,15 @@ static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, return domain; } + +void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end) +{ + struct iommu_mm_data *iommu_mm; + + guard(mutex)(&iommu_sva_lock); + if (!iommu_sva_present) + return; + + list_for_each_entry(iommu_mm, &iommu_sva_mms, mm_list_elm) + mmu_notifier_arch_invalidate_secondary_tlbs(iommu_mm->mm, start, end); +} diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index de178827a078a..dc0947aaac625 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -1257,14 +1257,20 @@ static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, unsigned long start, size_t length) { + unsigned long last; struct iommufd_ioas *ioas; int rc; + if (!length) + return -EINVAL; + if (check_add_overflow(start, length - 1, &last)) + return -EOVERFLOW; + ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); down_write(&ioas->iopt.iova_rwsem); - rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL); + rc = iopt_reserve_iova(&ioas->iopt, start, last, NULL); up_write(&ioas->iopt.iova_rwsem); iommufd_put_object(ucmd->ictx, &ioas->obj); return rc; diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index ffa892f657140..02a2a55ffa0a0 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -720,6 +720,8 @@ static int ipmmu_init_platform_device(struct device *dev, dev_iommu_priv_set(dev, platform_get_drvdata(ipmmu_pdev)); + put_device(&ipmmu_pdev->dev); + return 0; } diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 0e0285348d2b8..64ce041238fd3 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -974,6 +974,8 @@ static int mtk_iommu_of_xlate(struct device *dev, return -EINVAL; dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); + + put_device(&m4updev->dev); } return iommu_fwspec_add_ids(dev, args->args, 1); @@ -1211,16 +1213,19 @@ static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **m } component_match_add(dev, match, component_compare_dev, &plarbdev->dev); - platform_device_put(plarbdev); } - if (!frst_avail_smicomm_node) - return -EINVAL; + if (!frst_avail_smicomm_node) { + ret = -EINVAL; + goto err_larbdev_put; + } pcommdev = of_find_device_by_node(frst_avail_smicomm_node); of_node_put(frst_avail_smicomm_node); - if (!pcommdev) - return -ENODEV; + if (!pcommdev) { + ret = -ENODEV; + goto err_larbdev_put; + } data->smicomm_dev = &pcommdev->dev; link = device_link_add(data->smicomm_dev, dev, @@ -1228,7 +1233,8 @@ static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **m platform_device_put(pcommdev); if (!link) { dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); - return -EINVAL; + ret = -EINVAL; + goto err_larbdev_put; } return 0; @@ -1400,8 +1406,12 @@ static int mtk_iommu_probe(struct platform_device *pdev) iommu_device_sysfs_remove(&data->iommu); out_list_del: list_del(&data->list); - if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { device_link_remove(data->smicomm_dev, dev); + + for (i = 0; i < MTK_LARB_NR_MAX; i++) + put_device(data->larb_imu[i].dev); + } out_runtime_disable: pm_runtime_disable(dev); return ret; @@ -1421,6 +1431,9 @@ static void mtk_iommu_remove(struct platform_device *pdev) if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { device_link_remove(data->smicomm_dev, &pdev->dev); component_master_del(&pdev->dev, &mtk_iommu_com_ops); + + for (i = 0; i < MTK_LARB_NR_MAX; i++) + put_device(data->larb_imu[i].dev); } pm_runtime_disable(&pdev->dev); for (i = 0; i < data->plat_data->banks_num; i++) { diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 10cc0b1197e80..44b965a2db923 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -435,6 +435,8 @@ static int mtk_iommu_v1_create_mapping(struct device *dev, return -EINVAL; dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); + + put_device(&m4updev->dev); } ret = iommu_fwspec_add_ids(dev, args->args, 1); @@ -646,8 +648,10 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) struct platform_device *plarbdev; larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); - if (!larbnode) - return -EINVAL; + if (!larbnode) { + ret = -EINVAL; + goto out_put_larbs; + } if (!of_device_is_available(larbnode)) { of_node_put(larbnode); @@ -657,11 +661,14 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) plarbdev = of_find_device_by_node(larbnode); if (!plarbdev) { of_node_put(larbnode); - return -ENODEV; + ret = -ENODEV; + goto out_put_larbs; } if (!plarbdev->dev.driver) { of_node_put(larbnode); - return -EPROBE_DEFER; + put_device(&plarbdev->dev); + ret = -EPROBE_DEFER; + goto out_put_larbs; } data->larb_imu[i].dev = &plarbdev->dev; @@ -673,7 +680,7 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) ret = mtk_iommu_v1_hw_init(data); if (ret) - return ret; + goto out_put_larbs; ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL, dev_name(&pdev->dev)); @@ -695,12 +702,17 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) iommu_device_sysfs_remove(&data->iommu); out_clk_unprepare: clk_disable_unprepare(data->bclk); +out_put_larbs: + for (i = 0; i < MTK_LARB_NR_MAX; i++) + put_device(data->larb_imu[i].dev); + return ret; } static void mtk_iommu_v1_remove(struct platform_device *pdev) { struct mtk_iommu_v1_data *data = platform_get_drvdata(pdev); + int i; iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); @@ -708,6 +720,9 @@ static void mtk_iommu_v1_remove(struct platform_device *pdev) clk_disable_unprepare(data->bclk); devm_free_irq(&pdev->dev, data->irq, data); component_master_del(&pdev->dev, &mtk_iommu_v1_com_ops); + + for (i = 0; i < MTK_LARB_NR_MAX; i++) + put_device(data->larb_imu[i].dev); } static int __maybe_unused mtk_iommu_v1_suspend(struct device *dev) diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 6b989a62def20..a18bb60f6f3df 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -45,12 +45,10 @@ static int of_iommu_configure_dev_id(struct device_node *master_np, struct device *dev, const u32 *id) { - struct of_phandle_args iommu_spec = { .args_count = 1 }; + struct of_phandle_args iommu_spec = {}; int err; - err = of_map_id(master_np, *id, "iommu-map", - "iommu-map-mask", &iommu_spec.np, - iommu_spec.args); + err = of_map_iommu_id(master_np, *id, &iommu_spec); if (err) return err; diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 5c6f5943f44b1..c0315c86cd187 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1675,6 +1675,7 @@ static struct iommu_device *omap_iommu_probe_device(struct device *dev) } oiommu = platform_get_drvdata(pdev); + put_device(&pdev->dev); if (!oiommu) { of_node_put(np); kfree(arch_data); @@ -1682,7 +1683,6 @@ static struct iommu_device *omap_iommu_probe_device(struct device *dev) } tmp->iommu_dev = oiommu; - tmp->dev = &pdev->dev; of_node_put(np); } diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h index 27697109ec79a..50b39be61abc7 100644 --- a/drivers/iommu/omap-iommu.h +++ b/drivers/iommu/omap-iommu.h @@ -88,7 +88,6 @@ struct omap_iommu { /** * struct omap_iommu_arch_data - omap iommu private data * @iommu_dev: handle of the OMAP iommu device - * @dev: handle of the iommu device * * This is an omap iommu private data object, which binds an iommu user * to its iommu device. This object should be placed at the iommu user's @@ -97,7 +96,6 @@ struct omap_iommu { */ struct omap_iommu_arch_data { struct omap_iommu *iommu_dev; - struct device *dev; }; struct cr_regs { diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index de10b569d9a94..6306570d57db4 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -839,6 +839,8 @@ static int sun50i_iommu_of_xlate(struct device *dev, dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev)); + put_device(&iommu_pdev->dev); + return iommu_fwspec_add_ids(dev, &id, 1); } diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 36cdd5fbab077..f6f26a0728208 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -830,10 +830,9 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np) return NULL; mc = platform_get_drvdata(pdev); - if (!mc) { - put_device(&pdev->dev); + put_device(&pdev->dev); + if (!mc) return NULL; - } return mc->smmu; } diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index a61c6dc63c29c..13eaafbdb1867 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -504,6 +504,7 @@ config GOLDFISH_PIC config QCOM_PDC tristate "QCOM PDC" depends on ARCH_QCOM + depends on QCOM_AOSS_QMP select IRQ_DOMAIN_HIERARCHY help Power Domain Controller driver to manage and configure wakeup diff --git a/drivers/irqchip/irq-bcm2712-mip.c b/drivers/irqchip/irq-bcm2712-mip.c index 9bd7bc0bf6d59..4761974ad650a 100644 --- a/drivers/irqchip/irq-bcm2712-mip.c +++ b/drivers/irqchip/irq-bcm2712-mip.c @@ -232,17 +232,12 @@ static int mip_parse_dt(struct mip_priv *mip, struct device_node *np) return ret; } -static int __init mip_of_msi_init(struct device_node *node, struct device_node *parent) +static int mip_msi_probe(struct platform_device *pdev, struct device_node *parent) { - struct platform_device *pdev; + struct device_node *node = pdev->dev.of_node; struct mip_priv *mip; int ret; - pdev = of_find_device_by_node(node); - of_node_put(node); - if (!pdev) - return -EPROBE_DEFER; - mip = kzalloc(sizeof(*mip), GFP_KERNEL); if (!mip) return -ENOMEM; @@ -285,7 +280,7 @@ static int __init mip_of_msi_init(struct device_node *node, struct device_node * } IRQCHIP_PLATFORM_DRIVER_BEGIN(mip_msi) -IRQCHIP_MATCH("brcm,bcm2712-mip", mip_of_msi_init) +IRQCHIP_MATCH("brcm,bcm2712-mip", mip_msi_probe) IRQCHIP_PLATFORM_DRIVER_END(mip_msi) MODULE_DESCRIPTION("Broadcom BCM2712 MSI-X interrupt controller"); MODULE_AUTHOR("Phil Elwell "); diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c index 04fac0cc857fd..821b288587cab 100644 --- a/drivers/irqchip/irq-bcm7038-l1.c +++ b/drivers/irqchip/irq-bcm7038-l1.c @@ -219,9 +219,8 @@ static int bcm7038_l1_set_affinity(struct irq_data *d, } #endif -static int __init bcm7038_l1_init_one(struct device_node *dn, - unsigned int idx, - struct bcm7038_l1_chip *intc) +static int bcm7038_l1_init_one(struct device_node *dn, unsigned int idx, + struct bcm7038_l1_chip *intc) { struct resource res; resource_size_t sz; @@ -395,9 +394,9 @@ static const struct irq_domain_ops bcm7038_l1_domain_ops = { .map = bcm7038_l1_map, }; -static int __init bcm7038_l1_of_init(struct device_node *dn, - struct device_node *parent) +static int bcm7038_l1_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *dn = pdev->dev.of_node; struct bcm7038_l1_chip *intc; int idx, ret; @@ -455,7 +454,7 @@ static int __init bcm7038_l1_of_init(struct device_node *dn, } IRQCHIP_PLATFORM_DRIVER_BEGIN(bcm7038_l1) -IRQCHIP_MATCH("brcm,bcm7038-l1-intc", bcm7038_l1_of_init) +IRQCHIP_MATCH("brcm,bcm7038-l1-intc", bcm7038_l1_probe) IRQCHIP_PLATFORM_DRIVER_END(bcm7038_l1) MODULE_DESCRIPTION("Broadcom STB 7038-style L1/L2 interrupt controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c index ff22c31044018..518c9d4366a55 100644 --- a/drivers/irqchip/irq-bcm7120-l2.c +++ b/drivers/irqchip/irq-bcm7120-l2.c @@ -143,8 +143,7 @@ static int bcm7120_l2_intc_init_one(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_iomap_7120(struct device_node *dn, - struct bcm7120_l2_intc_data *data) +static int bcm7120_l2_intc_iomap_7120(struct device_node *dn, struct bcm7120_l2_intc_data *data) { int ret; @@ -177,8 +176,7 @@ static int __init bcm7120_l2_intc_iomap_7120(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_iomap_3380(struct device_node *dn, - struct bcm7120_l2_intc_data *data) +static int bcm7120_l2_intc_iomap_3380(struct device_node *dn, struct bcm7120_l2_intc_data *data) { unsigned int gc_idx; @@ -208,15 +206,14 @@ static int __init bcm7120_l2_intc_iomap_3380(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_probe(struct device_node *dn, - struct device_node *parent, +static int bcm7120_l2_intc_probe(struct platform_device *pdev, struct device_node *parent, int (*iomap_regs_fn)(struct device_node *, - struct bcm7120_l2_intc_data *), + struct bcm7120_l2_intc_data *), const char *intc_name) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; + struct device_node *dn = pdev->dev.of_node; struct bcm7120_l2_intc_data *data; - struct platform_device *pdev; struct irq_chip_generic *gc; struct irq_chip_type *ct; int ret = 0; @@ -227,14 +224,7 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn, if (!data) return -ENOMEM; - pdev = of_find_device_by_node(dn); - if (!pdev) { - ret = -ENODEV; - goto out_free_data; - } - data->num_parent_irqs = platform_irq_count(pdev); - put_device(&pdev->dev); if (data->num_parent_irqs <= 0) { pr_err("invalid number of parent interrupts\n"); ret = -ENOMEM; @@ -334,22 +324,19 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn, if (data->map_base[idx]) iounmap(data->map_base[idx]); } -out_free_data: kfree(data); return ret; } -static int __init bcm7120_l2_intc_probe_7120(struct device_node *dn, - struct device_node *parent) +static int bcm7120_l2_intc_probe_7120(struct platform_device *pdev, struct device_node *parent) { - return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_7120, + return bcm7120_l2_intc_probe(pdev, parent, bcm7120_l2_intc_iomap_7120, "BCM7120 L2"); } -static int __init bcm7120_l2_intc_probe_3380(struct device_node *dn, - struct device_node *parent) +static int bcm7120_l2_intc_probe_3380(struct platform_device *pdev, struct device_node *parent) { - return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_3380, + return bcm7120_l2_intc_probe(pdev, parent, bcm7120_l2_intc_iomap_3380, "BCM3380 L2"); } diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index 1bec5b2cd3f0e..bb7078d6524f9 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -138,13 +138,12 @@ static void brcmstb_l2_intc_resume(struct irq_data *d) irq_reg_writel(gc, ~b->saved_mask, ct->regs.enable); } -static int __init brcmstb_l2_intc_of_init(struct device_node *np, - struct device_node *parent, - const struct brcmstb_intc_init_params - *init_params) +static int brcmstb_l2_intc_probe(struct platform_device *pdev, struct device_node *parent, + const struct brcmstb_intc_init_params *init_params) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; unsigned int set = 0; + struct device_node *np = pdev->dev.of_node; struct brcmstb_l2_intc_data *data; struct irq_chip_type *ct; int ret; @@ -257,23 +256,21 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, return ret; } -static int __init brcmstb_l2_edge_intc_of_init(struct device_node *np, - struct device_node *parent) +static int brcmstb_l2_edge_intc_probe(struct platform_device *pdev, struct device_node *parent) { - return brcmstb_l2_intc_of_init(np, parent, &l2_edge_intc_init); + return brcmstb_l2_intc_probe(pdev, parent, &l2_edge_intc_init); } -static int __init brcmstb_l2_lvl_intc_of_init(struct device_node *np, - struct device_node *parent) +static int brcmstb_l2_lvl_intc_probe(struct platform_device *pdev, struct device_node *parent) { - return brcmstb_l2_intc_of_init(np, parent, &l2_lvl_intc_init); + return brcmstb_l2_intc_probe(pdev, parent, &l2_lvl_intc_init); } IRQCHIP_PLATFORM_DRIVER_BEGIN(brcmstb_l2) -IRQCHIP_MATCH("brcm,l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,hif-spi-l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,upg-aux-aon-l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,bcm7271-l2-intc", brcmstb_l2_lvl_intc_of_init) +IRQCHIP_MATCH("brcm,l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,hif-spi-l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,upg-aux-aon-l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,bcm7271-l2-intc", brcmstb_l2_lvl_intc_probe) IRQCHIP_PLATFORM_DRIVER_END(brcmstb_l2) MODULE_DESCRIPTION("Broadcom STB generic L2 interrupt controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-gic-its-msi-parent.c b/drivers/irqchip/irq-gic-its-msi-parent.c index eb1473f1448ac..429e146ab3338 100644 --- a/drivers/irqchip/irq-gic-its-msi-parent.c +++ b/drivers/irqchip/irq-gic-its-msi-parent.c @@ -164,11 +164,13 @@ static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev, } while (!ret); if (ret) { - struct device_node *np = NULL; + struct of_phandle_args msi_spec = {}; - ret = of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &np, dev_id); - if (np) - of_node_put(np); + ret = of_map_msi_id(dev->of_node, dev->id, NULL, &msi_spec); + if (!ret) { + of_node_put(msi_spec.np); + *dev_id = msi_spec.args[0]; + } } return ret; @@ -209,12 +211,13 @@ static int of_v5_pmsi_get_msi_info(struct irq_domain *domain, struct device *dev } while (!ret); if (ret) { - struct device_node *np = NULL; + struct of_phandle_args msi_spec = {}; - ret = of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &np, dev_id); - if (np) { - ret = its_translate_frame_address(np, pa); - of_node_put(np); + ret = of_map_msi_id(dev->of_node, dev->id, NULL, &msi_spec); + if (!ret) { + *dev_id = msi_spec.args[0]; + ret = its_translate_frame_address(msi_spec.np, pa); + of_node_put(msi_spec.np); } } diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 467cb78435a9b..23158fc8d3921 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -709,7 +709,7 @@ static struct its_collection *its_build_mapd_cmd(struct its_node *its, struct its_cmd_block *cmd, struct its_cmd_desc *desc) { - unsigned long itt_addr; + phys_addr_t itt_addr; u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); @@ -879,7 +879,7 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, struct its_cmd_desc *desc) { struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe); - unsigned long vpt_addr, vconf_addr; + phys_addr_t vpt_addr, vconf_addr; u64 target; bool alloc; @@ -2477,10 +2477,10 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, baser->psz = psz; tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz; - pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n", + pr_info("ITS@%pa: allocated %d %s @%llx (%s, esz %d, psz %dK, shr %d)\n", &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), its_base_type_string[type], - (unsigned long)virt_to_phys(base), + (u64)virt_to_phys(base), indirect ? "indirect" : "flat", (int)esz, psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); @@ -3475,6 +3475,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, int lpi_base; int nr_lpis; int nr_ites; + int id_bits; int sz; if (!its_alloc_device_table(its, dev_id)) @@ -3486,7 +3487,10 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, /* * Even if the device wants a single LPI, the ITT must be * sized as a power of two (and you need at least one bit...). + * Also honor the ITS's own EID limit. */ + id_bits = FIELD_GET(GITS_TYPER_IDBITS, its->typer) + 1; + nvecs = min_t(unsigned int, nvecs, BIT(id_bits)); nr_ites = max(2, nvecs); sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); sz = max(sz, ITS_ITT_ALIGN); diff --git a/drivers/irqchip/irq-gic-v5-its.c b/drivers/irqchip/irq-gic-v5-its.c index 554485f0be1fb..8e22134b9f486 100644 --- a/drivers/irqchip/irq-gic-v5-its.c +++ b/drivers/irqchip/irq-gic-v5-its.c @@ -849,7 +849,7 @@ static int gicv5_its_map_event(struct gicv5_its_dev *its_dev, u16 event_id, u32 itte = gicv5_its_device_get_itte_ref(its_dev, event_id); - if (FIELD_GET(GICV5_ITTL2E_VALID, *itte)) + if (FIELD_GET(GICV5_ITTL2E_VALID, le64_to_cpu(*itte))) return -EEXIST; itt_entry = FIELD_PREP(GICV5_ITTL2E_LPI_ID, lpi) | diff --git a/drivers/irqchip/irq-imx-mu-msi.c b/drivers/irqchip/irq-imx-mu-msi.c index d2a4e8a61a42b..c598f2f52fc6f 100644 --- a/drivers/irqchip/irq-imx-mu-msi.c +++ b/drivers/irqchip/irq-imx-mu-msi.c @@ -296,11 +296,9 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { }, }; -static int __init imx_mu_of_init(struct device_node *dn, - struct device_node *parent, - const struct imx_mu_dcfg *cfg) +static int imx_mu_probe(struct platform_device *pdev, struct device_node *parent, + const struct imx_mu_dcfg *cfg) { - struct platform_device *pdev = of_find_device_by_node(dn); struct device_link *pd_link_a; struct device_link *pd_link_b; struct imx_mu_msi *msi_data; @@ -416,31 +414,27 @@ static const struct dev_pm_ops imx_mu_pm_ops = { imx_mu_runtime_resume, NULL) }; -static int __init imx_mu_imx7ulp_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx7ulp_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx7ulp); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx7ulp); } -static int __init imx_mu_imx6sx_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx6sx_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx6sx); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx6sx); } -static int __init imx_mu_imx8ulp_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx8ulp_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx8ulp); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx8ulp); } IRQCHIP_PLATFORM_DRIVER_BEGIN(imx_mu_msi) -IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_of_init) -IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_of_init) -IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_of_init) +IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_probe) +IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_probe) +IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_probe) IRQCHIP_PLATFORM_DRIVER_END(imx_mu_msi, .pm = &imx_mu_pm_ops) - MODULE_AUTHOR("Frank Li "); MODULE_DESCRIPTION("Freescale MU MSI controller driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-mchp-eic.c b/drivers/irqchip/irq-mchp-eic.c index 516a3a0e359cc..979bb86929f8e 100644 --- a/drivers/irqchip/irq-mchp-eic.c +++ b/drivers/irqchip/irq-mchp-eic.c @@ -166,7 +166,7 @@ static int mchp_eic_domain_alloc(struct irq_domain *domain, unsigned int virq, ret = irq_domain_translate_twocell(domain, fwspec, &hwirq, &type); if (ret || hwirq >= MCHP_EIC_NIRQ) - return ret; + return ret ?: -EINVAL; switch (type) { case IRQ_TYPE_EDGE_RISING: @@ -199,8 +199,9 @@ static const struct irq_domain_ops mchp_eic_domain_ops = { .free = irq_domain_free_irqs_common, }; -static int mchp_eic_init(struct device_node *node, struct device_node *parent) +static int mchp_eic_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *node = pdev->dev.of_node; struct irq_domain *parent_domain = NULL; int ret, i; @@ -273,7 +274,7 @@ static int mchp_eic_init(struct device_node *node, struct device_node *parent) } IRQCHIP_PLATFORM_DRIVER_BEGIN(mchp_eic) -IRQCHIP_MATCH("microchip,sama7g5-eic", mchp_eic_init) +IRQCHIP_MATCH("microchip,sama7g5-eic", mchp_eic_probe) IRQCHIP_PLATFORM_DRIVER_END(mchp_eic) MODULE_DESCRIPTION("Microchip External Interrupt Controller"); diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c index 7d177626d64ba..09ebf1d9c21b0 100644 --- a/drivers/irqchip/irq-meson-gpio.c +++ b/drivers/irqchip/irq-meson-gpio.c @@ -572,8 +572,9 @@ static int meson_gpio_irq_parse_dt(struct device_node *node, struct meson_gpio_i return 0; } -static int meson_gpio_irq_of_init(struct device_node *node, struct device_node *parent) +static int meson_gpio_irq_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *node = pdev->dev.of_node; struct irq_domain *domain, *parent_domain; struct meson_gpio_irq_controller *ctl; int ret; @@ -630,7 +631,7 @@ static int meson_gpio_irq_of_init(struct device_node *node, struct device_node * } IRQCHIP_PLATFORM_DRIVER_BEGIN(meson_gpio_intc) -IRQCHIP_MATCH("amlogic,meson-gpio-intc", meson_gpio_irq_of_init) +IRQCHIP_MATCH("amlogic,meson-gpio-intc", meson_gpio_irq_probe) IRQCHIP_PLATFORM_DRIVER_END(meson_gpio_intc) MODULE_AUTHOR("Jerome Brunet "); diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c index 8d569f7c5a7aa..181320528a47a 100644 --- a/drivers/irqchip/irq-qcom-mpm.c +++ b/drivers/irqchip/irq-qcom-mpm.c @@ -306,6 +306,8 @@ static int mpm_pd_power_off(struct generic_pm_domain *genpd) if (ret < 0) return ret; + mbox_client_txdone(priv->mbox_chan, 0); + return 0; } @@ -320,9 +322,9 @@ static bool gic_hwirq_is_mapped(struct mpm_gic_map *maps, int cnt, u32 hwirq) return false; } -static int qcom_mpm_init(struct device_node *np, struct device_node *parent) +static int qcom_mpm_probe(struct platform_device *pdev, struct device_node *parent) { - struct platform_device *pdev = of_find_device_by_node(np); + struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; struct irq_domain *parent_domain; struct generic_pm_domain *genpd; @@ -434,6 +436,7 @@ static int qcom_mpm_init(struct device_node *np, struct device_node *parent) } priv->mbox_client.dev = dev; + priv->mbox_client.knows_txdone = true; priv->mbox_chan = mbox_request_channel(&priv->mbox_client, 0); if (IS_ERR(priv->mbox_chan)) { ret = PTR_ERR(priv->mbox_chan); @@ -478,7 +481,7 @@ static int qcom_mpm_init(struct device_node *np, struct device_node *parent) } IRQCHIP_PLATFORM_DRIVER_BEGIN(qcom_mpm) -IRQCHIP_MATCH("qcom,mpm", qcom_mpm_init) +IRQCHIP_MATCH("qcom,mpm", qcom_mpm_probe) IRQCHIP_PLATFORM_DRIVER_END(qcom_mpm) MODULE_DESCRIPTION("Qualcomm Technologies, Inc. MSM Power Manager"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index 2a54adeb4cc71..1bf19deb02c4e 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -528,18 +527,15 @@ static int rzg2l_irqc_parse_interrupts(struct rzg2l_irqc_priv *priv, return 0; } -static int rzg2l_irqc_common_init(struct device_node *node, struct device_node *parent, - const struct irq_chip *irq_chip) +static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_node *parent, + const struct irq_chip *irq_chip) { - struct platform_device *pdev = of_find_device_by_node(node); - struct device *dev __free(put_device) = pdev ? &pdev->dev : NULL; struct irq_domain *irq_domain, *parent_domain; + struct device_node *node = pdev->dev.of_node; + struct device *dev = &pdev->dev; struct reset_control *resetn; int ret; - if (!pdev) - return -ENODEV; - parent_domain = irq_find_host(parent); if (!parent_domain) return dev_err_probe(dev, -ENODEV, "cannot find parent domain\n"); @@ -583,35 +579,22 @@ static int rzg2l_irqc_common_init(struct device_node *node, struct device_node * register_syscore_ops(&rzg2l_irqc_syscore_ops); - /* - * Prevent the cleanup function from invoking put_device by assigning - * NULL to dev. - * - * make coccicheck will complain about missing put_device calls, but - * those are false positives, as dev will be automatically "put" via - * __free_put_device on the failing path. - * On the successful path we don't actually want to "put" dev. - */ - dev = NULL; - return 0; } -static int __init rzg2l_irqc_init(struct device_node *node, - struct device_node *parent) +static int rzg2l_irqc_probe(struct platform_device *pdev, struct device_node *parent) { - return rzg2l_irqc_common_init(node, parent, &rzg2l_irqc_chip); + return rzg2l_irqc_common_probe(pdev, parent, &rzg2l_irqc_chip); } -static int __init rzfive_irqc_init(struct device_node *node, - struct device_node *parent) +static int rzfive_irqc_probe(struct platform_device *pdev, struct device_node *parent) { - return rzg2l_irqc_common_init(node, parent, &rzfive_irqc_chip); + return rzg2l_irqc_common_probe(pdev, parent, &rzfive_irqc_chip); } IRQCHIP_PLATFORM_DRIVER_BEGIN(rzg2l_irqc) -IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_init) -IRQCHIP_MATCH("renesas,r9a07g043f-irqc", rzfive_irqc_init) +IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_probe) +IRQCHIP_MATCH("renesas,r9a07g043f-irqc", rzfive_irqc_probe) IRQCHIP_PLATFORM_DRIVER_END(rzg2l_irqc) MODULE_AUTHOR("Lad Prabhakar "); MODULE_DESCRIPTION("Renesas RZ/G2L IRQC Driver"); diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c index 9018d9c3911e3..c9f9099a51294 100644 --- a/drivers/irqchip/irq-renesas-rzv2h.c +++ b/drivers/irqchip/irq-renesas-rzv2h.c @@ -89,6 +89,8 @@ #define ICU_RZG3E_TSSEL_MAX_VAL 0x8c #define ICU_RZV2H_TSSEL_MAX_VAL 0x55 +#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) + /** * struct rzv2h_hw_info - Interrupt Control Unit controller hardware info structure. * @tssel_lut: TINT lookup table @@ -328,6 +330,7 @@ static int rzv2h_tint_set_type(struct irq_data *d, unsigned int type) u32 titsr, titsr_k, titsel_n, tien; struct rzv2h_icu_priv *priv; u32 tssr, tssr_k, tssel_n; + u32 titsr_cur, tssr_cur; unsigned int hwirq; u32 tint, sense; int tint_nr; @@ -376,12 +379,18 @@ static int rzv2h_tint_set_type(struct irq_data *d, unsigned int type) guard(raw_spinlock)(&priv->lock); tssr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSSR(tssr_k)); + titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(titsr_k)); + + tssr_cur = field_get(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width), tssr); + titsr_cur = field_get(ICU_TITSR_TITSEL_MASK(titsel_n), titsr); + if (tssr_cur == tint && titsr_cur == sense) + return 0; + tssr &= ~(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width) | tien); tssr |= ICU_TSSR_TSSEL_PREP(tint, tssel_n, priv->info->field_width); writel_relaxed(tssr, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k)); - titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(titsr_k)); titsr &= ~ICU_TITSR_TITSEL_MASK(titsel_n); titsr |= ICU_TITSR_TITSEL_PREP(sense, titsel_n); @@ -490,29 +499,15 @@ static int rzv2h_icu_parse_interrupts(struct rzv2h_icu_priv *priv, struct device return 0; } -static void rzv2h_icu_put_device(void *data) -{ - put_device(data); -} - -static int rzv2h_icu_init_common(struct device_node *node, struct device_node *parent, - const struct rzv2h_hw_info *hw_info) +static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_node *parent, + const struct rzv2h_hw_info *hw_info) { struct irq_domain *irq_domain, *parent_domain; + struct device_node *node = pdev->dev.of_node; struct rzv2h_icu_priv *rzv2h_icu_data; - struct platform_device *pdev; struct reset_control *resetn; int ret; - pdev = of_find_device_by_node(node); - if (!pdev) - return -ENODEV; - - ret = devm_add_action_or_reset(&pdev->dev, rzv2h_icu_put_device, - &pdev->dev); - if (ret < 0) - return ret; - parent_domain = irq_find_host(parent); if (!parent_domain) { dev_err(&pdev->dev, "cannot find parent domain\n"); @@ -574,7 +569,7 @@ static int rzv2h_icu_init_common(struct device_node *node, struct device_node *p return 0; pm_put: - pm_runtime_put(&pdev->dev); + pm_runtime_put_sync(&pdev->dev); return ret; } @@ -618,19 +613,19 @@ static const struct rzv2h_hw_info rzv2h_hw_params = { .field_width = 8, }; -static int rzg3e_icu_init(struct device_node *node, struct device_node *parent) +static int rzg3e_icu_probe(struct platform_device *pdev, struct device_node *parent) { - return rzv2h_icu_init_common(node, parent, &rzg3e_hw_params); + return rzv2h_icu_probe_common(pdev, parent, &rzg3e_hw_params); } -static int rzv2h_icu_init(struct device_node *node, struct device_node *parent) +static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *parent) { - return rzv2h_icu_init_common(node, parent, &rzv2h_hw_params); + return rzv2h_icu_probe_common(pdev, parent, &rzv2h_hw_params); } IRQCHIP_PLATFORM_DRIVER_BEGIN(rzv2h_icu) -IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_init) -IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_init) +IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_probe) +IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_probe) IRQCHIP_PLATFORM_DRIVER_END(rzv2h_icu) MODULE_AUTHOR("Fabrizio Castro "); MODULE_DESCRIPTION("Renesas RZ/V2H(P) ICU Driver"); diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c index 2c4c682627b8c..d1727c343c38f 100644 --- a/drivers/irqchip/irq-riscv-imsic-early.c +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "riscv-imsic: " fmt #include #include +#include #include #include #include @@ -128,14 +129,8 @@ static void imsic_handle_irq(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static int imsic_starting_cpu(unsigned int cpu) +static void imsic_hw_states_init(void) { - /* Mark per-CPU IMSIC state as online */ - imsic_state_online(); - - /* Enable per-CPU parent interrupt */ - enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); - /* Setup IPIs */ imsic_ipi_starting_cpu(); @@ -147,6 +142,18 @@ static int imsic_starting_cpu(unsigned int cpu) /* Enable local interrupt delivery */ imsic_local_delivery(true); +} + +static int imsic_starting_cpu(unsigned int cpu) +{ + /* Mark per-CPU IMSIC state as online */ + imsic_state_online(); + + /* Enable per-CPU parent interrupt */ + enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); + + /* Initialize the IMSIC registers to enable the interrupt delivery */ + imsic_hw_states_init(); return 0; } @@ -162,6 +169,22 @@ static int imsic_dying_cpu(unsigned int cpu) return 0; } +static int imsic_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) +{ + switch (cmd) { + case CPU_PM_EXIT: + /* Initialize the IMSIC registers to enable the interrupt delivery */ + imsic_hw_states_init(); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block imsic_pm_notifier_block = { + .notifier_call = imsic_pm_notifier, +}; + static int __init imsic_early_probe(struct fwnode_handle *fwnode) { struct irq_domain *domain; @@ -199,7 +222,7 @@ static int __init imsic_early_probe(struct fwnode_handle *fwnode) cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting", imsic_starting_cpu, imsic_dying_cpu); - return 0; + return cpu_pm_register_notifier(&imsic_pm_notifier_block); } static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent) diff --git a/drivers/irqchip/irq-riscv-rpmi-sysmsi.c b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c index 5c74c561ce316..612f3972f7af0 100644 --- a/drivers/irqchip/irq-riscv-rpmi-sysmsi.c +++ b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c @@ -250,6 +250,7 @@ static int rpmi_sysmsi_probe(struct platform_device *pdev) rc = riscv_acpi_get_gsi_info(fwnode, &priv->gsi_base, &id, &nr_irqs, NULL); if (rc) { + mbox_free_channel(priv->chan); dev_err(dev, "failed to find GSI mapping\n"); return rc; } diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index cbd7697bc1481..0799c15c745d4 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -154,8 +154,13 @@ static void plic_irq_disable(struct irq_data *d) static void plic_irq_eoi(struct irq_data *d) { struct plic_handler *handler = this_cpu_ptr(&plic_handlers); + u32 __iomem *reg; + bool enabled; + + reg = handler->enable_base + (d->hwirq / 32) * sizeof(u32); + enabled = readl(reg) & BIT(d->hwirq % 32); - if (unlikely(irqd_irq_disabled(d))) { + if (unlikely(!enabled)) { plic_toggle(handler, d->hwirq, 1); writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM); plic_toggle(handler, d->hwirq, 0); diff --git a/drivers/irqchip/irq-starfive-jh8100-intc.c b/drivers/irqchip/irq-starfive-jh8100-intc.c index 2460798ec158b..705361b4ebe07 100644 --- a/drivers/irqchip/irq-starfive-jh8100-intc.c +++ b/drivers/irqchip/irq-starfive-jh8100-intc.c @@ -114,9 +114,9 @@ static void starfive_intc_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static int __init starfive_intc_init(struct device_node *intc, - struct device_node *parent) +static int starfive_intc_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *intc = pdev->dev.of_node; struct starfive_irq_chip *irqc; struct reset_control *rst; struct clk *clk; @@ -199,7 +199,7 @@ static int __init starfive_intc_init(struct device_node *intc, } IRQCHIP_PLATFORM_DRIVER_BEGIN(starfive_intc) -IRQCHIP_MATCH("starfive,jh8100-intc", starfive_intc_init) +IRQCHIP_MATCH("starfive,jh8100-intc", starfive_intc_probe) IRQCHIP_PLATFORM_DRIVER_END(starfive_intc) MODULE_DESCRIPTION("StarFive JH8100 External Interrupt Controller"); diff --git a/drivers/irqchip/irqchip.c b/drivers/irqchip/irqchip.c index 0ee7b6b71f5fa..689c8e4489019 100644 --- a/drivers/irqchip/irqchip.c +++ b/drivers/irqchip/irqchip.c @@ -36,11 +36,10 @@ int platform_irqchip_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct device_node *par_np __free(device_node) = of_irq_find_parent(np); - of_irq_init_cb_t irq_init_cb = of_device_get_match_data(&pdev->dev); + platform_irq_probe_t irq_probe = of_device_get_match_data(&pdev->dev); - if (!irq_init_cb) { + if (!irq_probe) return -EINVAL; - } if (par_np == np) par_np = NULL; @@ -53,10 +52,9 @@ int platform_irqchip_probe(struct platform_device *pdev) * interrupt controller. The actual initialization callback of this * interrupt controller can check for specific domains as necessary. */ - if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) { + if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) return -EPROBE_DEFER; - } - return irq_init_cb(np, par_np); + return irq_probe(pdev, par_np); } EXPORT_SYMBOL_GPL(platform_irqchip_probe); diff --git a/drivers/irqchip/qcom-irq-combiner.c b/drivers/irqchip/qcom-irq-combiner.c index 18e696dc7f4d6..9308088773be7 100644 --- a/drivers/irqchip/qcom-irq-combiner.c +++ b/drivers/irqchip/qcom-irq-combiner.c @@ -222,7 +222,7 @@ static int get_registers(struct platform_device *pdev, struct combiner *comb) return 0; } -static int __init combiner_probe(struct platform_device *pdev) +static int combiner_probe(struct platform_device *pdev) { struct combiner *combiner; int nregs; diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c index 52d77546aacb9..7122f75c1f7d5 100644 --- a/drivers/irqchip/qcom-pdc.c +++ b/drivers/irqchip/qcom-pdc.c @@ -19,6 +19,8 @@ #include #include #include +#include +#include #define PDC_MAX_GPIO_IRQS 256 #define PDC_DRV_OFFSET 0x10000 @@ -26,9 +28,11 @@ /* Valid only on HW version < 3.2 */ #define IRQ_ENABLE_BANK 0x10 #define IRQ_ENABLE_BANK_MAX (IRQ_ENABLE_BANK + BITS_TO_BYTES(PDC_MAX_GPIO_IRQS)) +#define IRQ_i_CFG_IRQ_MASK_3_0 3 #define IRQ_i_CFG 0x110 /* Valid only on HW version >= 3.2 */ +#define IRQ_i_CFG_IRQ_MASK_3_2 4 #define IRQ_i_CFG_IRQ_ENABLE 3 #define IRQ_i_CFG_TYPE_MASK GENMASK(2, 0) @@ -36,8 +40,11 @@ #define PDC_VERSION_REG 0x1000 /* Notable PDC versions */ +#define PDC_VERSION_3_0 0x30000 #define PDC_VERSION_3_2 0x30200 +#define PDC_PASS_THROUGH_MODE 0 + struct pdc_pin_region { u32 pin_base; u32 parent_base; @@ -97,6 +104,33 @@ static void pdc_x1e_irq_enable_write(u32 bank, u32 enable) pdc_base_reg_write(base, IRQ_ENABLE_BANK, bank, enable); } +/* + * The new mask bit controls whether the interrupt is to be forwarded to the + * parent GIC in secondary controller mode. Writing the mask is do not care + * when the PDC is set to pass through mode. + * + * As linux only makes so far make use of pass through mode set all IRQs + * masked during probe. + */ +static void __pdc_mask_intr(int pin_out, bool mask) +{ + unsigned long irq_cfg; + int mask_bit; + + /* Mask bit available from v3.0 */ + if (pdc_version < PDC_VERSION_3_0) + return; + + if (pdc_version < PDC_VERSION_3_2) + mask_bit = IRQ_i_CFG_IRQ_MASK_3_0; + else + mask_bit = IRQ_i_CFG_IRQ_MASK_3_2; + + irq_cfg = pdc_reg_read(IRQ_i_CFG, pin_out); + __assign_bit(mask_bit, &irq_cfg, mask); + pdc_reg_write(IRQ_i_CFG, pin_out, irq_cfg); +} + static void __pdc_enable_intr(int pin_out, bool on) { unsigned long enable; @@ -312,7 +346,6 @@ static const struct irq_domain_ops qcom_pdc_ops = { static int pdc_setup_pin_mapping(struct device_node *np) { int ret, n, i; - n = of_property_count_elems_of_size(np, "qcom,pdc-ranges", sizeof(u32)); if (n <= 0 || n % 3) return -EINVAL; @@ -341,8 +374,10 @@ static int pdc_setup_pin_mapping(struct device_node *np) if (ret) return ret; - for (i = 0; i < pdc_region[n].cnt; i++) + for (i = 0; i < pdc_region[n].cnt; i++) { __pdc_enable_intr(i + pdc_region[n].pin_base, 0); + __pdc_mask_intr(i + pdc_region[n].pin_base, true); + } } return 0; @@ -350,11 +385,15 @@ static int pdc_setup_pin_mapping(struct device_node *np) #define QCOM_PDC_SIZE 0x30000 -static int qcom_pdc_init(struct device_node *node, struct device_node *parent) +static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *parent) { + static const char buf[64] = "{class: cx_mol, res: cx, val: mol}"; + unsigned int domain_flag = IRQ_DOMAIN_FLAG_QCOM_PDC_WAKEUP; struct irq_domain *parent_domain, *pdc_domain; + struct device_node *node = pdev->dev.of_node; resource_size_t res_size; struct resource res; + struct qmp *pdc_qmp; int ret; /* compat with old sm8150 DT which had very small region for PDC */ @@ -365,6 +404,13 @@ static int qcom_pdc_init(struct device_node *node, struct device_node *parent) if (res_size > resource_size(&res)) pr_warn("%pOF: invalid reg size, please fix DT\n", node); + pdc_base = ioremap(res.start, res_size); + if (!pdc_base) { + pr_err("%pOF: unable to map PDC registers\n", node); + ret = -ENXIO; + goto fail; + } + /* * PDC has multiple DRV regions, each one provides the same set of * registers for a particular client in the system. Due to a hardware @@ -381,15 +427,71 @@ static int qcom_pdc_init(struct device_node *node, struct device_node *parent) } pdc_x1e_quirk = true; - } - pdc_base = ioremap(res.start, res_size); - if (!pdc_base) { - pr_err("%pOF: unable to map PDC registers\n", node); - ret = -ENXIO; - goto fail; + /* + * There are two modes PDC irqchip can work in + * - pass through mode + * - secondary controller mode + * + * All PDC irqchip supports pass through mode in which both + * Direct SPIs and GPIO IRQs (as SPIs) are sent to GIC + * without latching at PDC. + * + * Newer PDCs (v3.0 onwards) also support additional + * secondary controller mode where PDC latches GPIO IRQs + * and sends to GIC as level type IRQ. Direct SPIs still + * works same as pass through mode without latching at PDC + * even in secondary controller mode. + * + * All the SoCs so far default uses pass through mode with + * the exception of x1e. + * + * x1e modes: + * + * x1e PDC may be set to secondary controller mode for + * builds on CRD boards whereas it may be set to pass + * through mode for IoT-EVK boards. + * + * There is no way to read which current mode it is set to + * and make PDC work in respective mode as the read access + * is not opened up for non secure world. There is though + * write access opened up via SCM write API to set the mode. + * + * Configure PDC mode to pass through mode for all x1e based + * boards. + * + * For successful write: + * - Nothing more to be done + * + * For unsuccessful write: + * - Inform TLMM to monitor GPIO IRQs (same as MPM) + * - Prevent SoC low power mode (CxPC) as PDC is not + * monitoring GPIO IRQs which may be needed to wake + * the SoC from low power mode. + */ + ret = of_address_to_resource(node, 2, &res); + if (ret) { + domain_flag = IRQ_DOMAIN_FLAG_QCOM_MPM_WAKEUP; + goto skip_scm_write; + } + + ret = qcom_scm_io_writel(res.start, PDC_PASS_THROUGH_MODE); + if (ret) { + pdc_qmp = qmp_get(&pdev->dev); + if (IS_ERR(pdc_qmp)) { + ret = PTR_ERR(pdc_qmp); + goto fail; + } else { + ret = qmp_send(pdc_qmp, buf, sizeof(buf)); + qmp_put(pdc_qmp); + if (ret) + goto fail; + } + domain_flag = IRQ_DOMAIN_FLAG_QCOM_MPM_WAKEUP; + } } +skip_scm_write: pdc_version = pdc_reg_read(PDC_VERSION_REG, 0); parent_domain = irq_find_host(parent); @@ -406,7 +508,7 @@ static int qcom_pdc_init(struct device_node *node, struct device_node *parent) } pdc_domain = irq_domain_create_hierarchy(parent_domain, - IRQ_DOMAIN_FLAG_QCOM_PDC_WAKEUP, + domain_flag, PDC_MAX_GPIO_IRQS, of_fwnode_handle(node), &qcom_pdc_ops, NULL); @@ -428,7 +530,7 @@ static int qcom_pdc_init(struct device_node *node, struct device_node *parent) } IRQCHIP_PLATFORM_DRIVER_BEGIN(qcom_pdc) -IRQCHIP_MATCH("qcom,pdc", qcom_pdc_init) +IRQCHIP_MATCH("qcom,pdc", qcom_pdc_probe) IRQCHIP_PLATFORM_DRIVER_END(qcom_pdc) MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Power Domain Controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/isdn/mISDN/timerdev.c b/drivers/isdn/mISDN/timerdev.c index df98144a95394..33521c328a827 100644 --- a/drivers/isdn/mISDN/timerdev.c +++ b/drivers/isdn/mISDN/timerdev.c @@ -109,14 +109,14 @@ mISDN_read(struct file *filep, char __user *buf, size_t count, loff_t *off) spin_unlock_irq(&dev->lock); if (filep->f_flags & O_NONBLOCK) return -EAGAIN; - wait_event_interruptible(dev->wait, (dev->work || + wait_event_interruptible(dev->wait, (READ_ONCE(dev->work) || !list_empty(list))); if (signal_pending(current)) return -ERESTARTSYS; spin_lock_irq(&dev->lock); } if (dev->work) - dev->work = 0; + WRITE_ONCE(dev->work, 0); if (!list_empty(list)) { timer = list_first_entry(list, struct mISDNtimer, list); list_del(&timer->list); @@ -141,13 +141,16 @@ mISDN_poll(struct file *filep, poll_table *wait) if (*debug & DEBUG_TIMER) printk(KERN_DEBUG "%s(%p, %p)\n", __func__, filep, wait); if (dev) { + u32 work; + poll_wait(filep, &dev->wait, wait); mask = 0; - if (dev->work || !list_empty(&dev->expired)) + work = READ_ONCE(dev->work); + if (work || !list_empty(&dev->expired)) mask |= (EPOLLIN | EPOLLRDNORM); if (*debug & DEBUG_TIMER) printk(KERN_DEBUG "%s work(%d) empty(%d)\n", __func__, - dev->work, list_empty(&dev->expired)); + work, list_empty(&dev->expired)); } return mask; } @@ -172,7 +175,7 @@ misdn_add_timer(struct mISDNtimerdev *dev, int timeout) struct mISDNtimer *timer; if (!timeout) { - dev->work = 1; + WRITE_ONCE(dev->work, 1); wake_up_interruptible(&dev->wait); id = 0; } else { diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c index f3faf37f9a08a..6b9fa060c3a11 100644 --- a/drivers/leds/led-class.c +++ b/drivers/leds/led-class.c @@ -560,11 +560,6 @@ int led_classdev_register_ext(struct device *parent, #ifdef CONFIG_LEDS_BRIGHTNESS_HW_CHANGED led_cdev->brightness_hw_changed = -1; #endif - /* add to the list of leds */ - down_write(&leds_list_lock); - list_add_tail(&led_cdev->node, &leds_list); - up_write(&leds_list_lock); - if (!led_cdev->max_brightness) led_cdev->max_brightness = LED_FULL; @@ -574,6 +569,11 @@ int led_classdev_register_ext(struct device *parent, led_init_core(led_cdev); + /* add to the list of leds */ + down_write(&leds_list_lock); + list_add_tail(&led_cdev->node, &leds_list); + up_write(&leds_list_lock); + #ifdef CONFIG_LEDS_TRIGGERS led_trigger_set_default(led_cdev); #endif diff --git a/drivers/leds/leds-cros_ec.c b/drivers/leds/leds-cros_ec.c index 377cf04e202a1..bea3cc3fbfd2d 100644 --- a/drivers/leds/leds-cros_ec.c +++ b/drivers/leds/leds-cros_ec.c @@ -142,9 +142,6 @@ static int cros_ec_led_count_subleds(struct device *dev, } } - if (!num_subleds) - return -EINVAL; - *max_brightness = common_range; return num_subleds; } @@ -189,6 +186,8 @@ static int cros_ec_led_probe_one(struct device *dev, struct cros_ec_device *cros &priv->led_mc_cdev.led_cdev.max_brightness); if (num_subleds < 0) return num_subleds; + if (num_subleds == 0) + return 0; /* LED without any colors, skip */ priv->cros_ec = cros_ec; priv->led_id = id; diff --git a/drivers/leds/leds-expresswire.c b/drivers/leds/leds-expresswire.c index bb69be228a6d3..25c6b159a6ee9 100644 --- a/drivers/leds/leds-expresswire.c +++ b/drivers/leds/leds-expresswire.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -16,37 +17,41 @@ void expresswire_power_off(struct expresswire_common_props *props) { gpiod_set_value_cansleep(props->ctrl_gpio, 0); - usleep_range(props->timing.poweroff_us, props->timing.poweroff_us * 2); + fsleep(props->timing.poweroff_us); } EXPORT_SYMBOL_NS_GPL(expresswire_power_off, "EXPRESSWIRE"); void expresswire_enable(struct expresswire_common_props *props) { + unsigned long flags; + + local_irq_save(flags); + gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.detect_delay_us); gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.detect_us); gpiod_set_value(props->ctrl_gpio, 1); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_enable, "EXPRESSWIRE"); -void expresswire_start(struct expresswire_common_props *props) +static void expresswire_start(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.data_start_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_start, "EXPRESSWIRE"); -void expresswire_end(struct expresswire_common_props *props) +static void expresswire_end(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.end_of_data_low_us); gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.end_of_data_high_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_end, "EXPRESSWIRE"); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit) +static void expresswire_set_bit(struct expresswire_common_props *props, bool bit) { if (bit) { gpiod_set_value(props->ctrl_gpio, 0); @@ -60,13 +65,18 @@ void expresswire_set_bit(struct expresswire_common_props *props, bool bit) udelay(props->timing.short_bitset_us); } } -EXPORT_SYMBOL_NS_GPL(expresswire_set_bit, "EXPRESSWIRE"); void expresswire_write_u8(struct expresswire_common_props *props, u8 val) { + unsigned long flags; + + local_irq_save(flags); + expresswire_start(props); for (int i = 7; i >= 0; i--) expresswire_set_bit(props, val & BIT(i)); expresswire_end(props); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_write_u8, "EXPRESSWIRE"); diff --git a/drivers/leds/leds-lp50xx.c b/drivers/leds/leds-lp50xx.c index 94f8ef6b482c9..e2a9c8592953c 100644 --- a/drivers/leds/leds-lp50xx.c +++ b/drivers/leds/leds-lp50xx.c @@ -50,11 +50,17 @@ #define LP50XX_SW_RESET 0xff #define LP50XX_CHIP_EN BIT(6) +#define LP50XX_CHIP_DISABLE 0x00 +#define LP50XX_START_TIME_US 500 +#define LP50XX_RESET_TIME_US 3 + +#define LP50XX_EN_GPIO_LOW 0 +#define LP50XX_EN_GPIO_HIGH 1 /* There are 3 LED outputs per bank */ #define LP50XX_LEDS_PER_MODULE 3 -#define LP5009_MAX_LED_MODULES 2 +#define LP5009_MAX_LED_MODULES 3 #define LP5012_MAX_LED_MODULES 4 #define LP5018_MAX_LED_MODULES 6 #define LP5024_MAX_LED_MODULES 8 @@ -341,17 +347,15 @@ static int lp50xx_brightness_set(struct led_classdev *cdev, return ret; } -static int lp50xx_set_banks(struct lp50xx *priv, u32 led_banks[]) +static int lp50xx_set_banks(struct lp50xx *priv, u32 led_banks[], int num_leds) { u8 led_config_lo, led_config_hi; u32 bank_enable_mask = 0; int ret; int i; - for (i = 0; i < priv->chip_info->max_modules; i++) { - if (led_banks[i]) - bank_enable_mask |= (1 << led_banks[i]); - } + for (i = 0; i < num_leds; i++) + bank_enable_mask |= (1 << led_banks[i]); led_config_lo = bank_enable_mask; led_config_hi = bank_enable_mask >> 8; @@ -371,19 +375,42 @@ static int lp50xx_reset(struct lp50xx *priv) return regmap_write(priv->regmap, priv->chip_info->reset_reg, LP50XX_SW_RESET); } -static int lp50xx_enable_disable(struct lp50xx *priv, int enable_disable) +static int lp50xx_enable(struct lp50xx *priv) { int ret; - ret = gpiod_direction_output(priv->enable_gpio, enable_disable); + if (priv->enable_gpio) { + ret = gpiod_direction_output(priv->enable_gpio, LP50XX_EN_GPIO_HIGH); + if (ret) + return ret; + + udelay(LP50XX_START_TIME_US); + } + + ret = lp50xx_reset(priv); if (ret) return ret; - if (enable_disable) - return regmap_write(priv->regmap, LP50XX_DEV_CFG0, LP50XX_CHIP_EN); - else - return regmap_write(priv->regmap, LP50XX_DEV_CFG0, 0); + return regmap_write(priv->regmap, LP50XX_DEV_CFG0, LP50XX_CHIP_EN); +} + +static int lp50xx_disable(struct lp50xx *priv) +{ + int ret; + + ret = regmap_write(priv->regmap, LP50XX_DEV_CFG0, LP50XX_CHIP_DISABLE); + if (ret) + return ret; + if (priv->enable_gpio) { + ret = gpiod_direction_output(priv->enable_gpio, LP50XX_EN_GPIO_LOW); + if (ret) + return ret; + + udelay(LP50XX_RESET_TIME_US); + } + + return 0; } static int lp50xx_probe_leds(struct fwnode_handle *child, struct lp50xx *priv, @@ -405,7 +432,7 @@ static int lp50xx_probe_leds(struct fwnode_handle *child, struct lp50xx *priv, return ret; } - ret = lp50xx_set_banks(priv, led_banks); + ret = lp50xx_set_banks(priv, led_banks, num_leds); if (ret) { dev_err(priv->dev, "Cannot setup banked LEDs\n"); return ret; @@ -447,6 +474,10 @@ static int lp50xx_probe_dt(struct lp50xx *priv) return dev_err_probe(priv->dev, PTR_ERR(priv->enable_gpio), "Failed to get enable GPIO\n"); + ret = lp50xx_enable(priv); + if (ret) + return ret; + priv->regulator = devm_regulator_get(priv->dev, "vled"); if (IS_ERR(priv->regulator)) priv->regulator = NULL; @@ -547,14 +578,6 @@ static int lp50xx_probe(struct i2c_client *client) return ret; } - ret = lp50xx_reset(led); - if (ret) - return ret; - - ret = lp50xx_enable_disable(led, 1); - if (ret) - return ret; - return lp50xx_probe_dt(led); } @@ -563,7 +586,7 @@ static void lp50xx_remove(struct i2c_client *client) struct lp50xx *led = i2c_get_clientdata(client); int ret; - ret = lp50xx_enable_disable(led, 0); + ret = lp50xx_disable(led); if (ret) dev_err(led->dev, "Failed to disable chip\n"); diff --git a/drivers/leds/leds-netxbig.c b/drivers/leds/leds-netxbig.c index e95287416ef87..99df46f2d9f52 100644 --- a/drivers/leds/leds-netxbig.c +++ b/drivers/leds/leds-netxbig.c @@ -364,6 +364,9 @@ static int netxbig_gpio_ext_get(struct device *dev, if (!addr) return -ENOMEM; + gpio_ext->addr = addr; + gpio_ext->num_addr = 0; + /* * We cannot use devm_ managed resources with these GPIO descriptors * since they are associated with the "GPIO extension device" which @@ -375,45 +378,58 @@ static int netxbig_gpio_ext_get(struct device *dev, gpiod = gpiod_get_index(gpio_ext_dev, "addr", i, GPIOD_OUT_LOW); if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); + goto err_set_code; gpiod_set_consumer_name(gpiod, "GPIO extension addr"); addr[i] = gpiod; + gpio_ext->num_addr++; } - gpio_ext->addr = addr; - gpio_ext->num_addr = num_addr; ret = gpiod_count(gpio_ext_dev, "data"); if (ret < 0) { dev_err(dev, "Failed to count GPIOs in DT property data-gpios\n"); - return ret; + goto err_free_addr; } num_data = ret; data = devm_kcalloc(dev, num_data, sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; + if (!data) { + ret = -ENOMEM; + goto err_free_addr; + } + + gpio_ext->data = data; + gpio_ext->num_data = 0; for (i = 0; i < num_data; i++) { gpiod = gpiod_get_index(gpio_ext_dev, "data", i, GPIOD_OUT_LOW); if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); + goto err_free_data; gpiod_set_consumer_name(gpiod, "GPIO extension data"); data[i] = gpiod; + gpio_ext->num_data++; } - gpio_ext->data = data; - gpio_ext->num_data = num_data; gpiod = gpiod_get(gpio_ext_dev, "enable", GPIOD_OUT_LOW); if (IS_ERR(gpiod)) { dev_err(dev, "Failed to get GPIO from DT property enable-gpio\n"); - return PTR_ERR(gpiod); + goto err_free_data; } gpiod_set_consumer_name(gpiod, "GPIO extension enable"); gpio_ext->enable = gpiod; return devm_add_action_or_reset(dev, netxbig_gpio_ext_remove, gpio_ext); + +err_free_data: + for (i = 0; i < gpio_ext->num_data; i++) + gpiod_put(gpio_ext->data[i]); +err_set_code: + ret = PTR_ERR(gpiod); +err_free_addr: + for (i = 0; i < gpio_ext->num_addr; i++) + gpiod_put(gpio_ext->addr[i]); + return ret; } static int netxbig_leds_get_of_pdata(struct device *dev, diff --git a/drivers/leds/leds-upboard.c b/drivers/leds/leds-upboard.c index b350eb294280f..12989b2f19530 100644 --- a/drivers/leds/leds-upboard.c +++ b/drivers/leds/leds-upboard.c @@ -123,4 +123,4 @@ MODULE_AUTHOR("Gary Wang "); MODULE_AUTHOR("Thomas Richard "); MODULE_DESCRIPTION("UP Board LED driver"); MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:upboard-led"); +MODULE_ALIAS("platform:upboard-leds"); diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c index 4f2a178e3d265..a460782dadca4 100644 --- a/drivers/leds/rgb/leds-qcom-lpg.c +++ b/drivers/leds/rgb/leds-qcom-lpg.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2017-2022 Linaro Ltd * Copyright (c) 2010-2012, The Linux Foundation. All rights reserved. - * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include @@ -369,7 +369,7 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, { unsigned int idx; u16 val; - int i; + int i, ret; idx = bitmap_find_next_zero_area(lpg->lut_bitmap, lpg->lut_size, 0, len, 0); @@ -379,8 +379,10 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, for (i = 0; i < len; i++) { val = pattern[i].brightness; - regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), - &val, sizeof(val)); + ret = regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), + &val, sizeof(val)); + if (ret) + return ret; } bitmap_set(lpg->lut_bitmap, idx, len); @@ -1247,8 +1249,6 @@ static int lpg_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, lpg_apply(chan); - triled_set(lpg, chan->triled_mask, chan->enabled ? chan->triled_mask : 0); - out_unlock: mutex_unlock(&lpg->lock); diff --git a/drivers/macintosh/mac_hid.c b/drivers/macintosh/mac_hid.c index 369d72f59b3c1..06fd910b3fd1a 100644 --- a/drivers/macintosh/mac_hid.c +++ b/drivers/macintosh/mac_hid.c @@ -187,13 +187,14 @@ static int mac_hid_toggle_emumouse(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int *valp = table->data; - int old_val = *valp; + int old_val; int rc; rc = mutex_lock_killable(&mac_hid_emumouse_mutex); if (rc) return rc; + old_val = *valp; rc = proc_dointvec(table, write, buffer, lenp, ppos); if (rc == 0 && write && *valp != old_val) { diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c index 41f79e51d9e5a..4255fefc3a5a0 100644 --- a/drivers/mailbox/bcm-flexrm-mailbox.c +++ b/drivers/mailbox/bcm-flexrm-mailbox.c @@ -1173,14 +1173,6 @@ static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset) /* ====== FlexRM interrupt handler ===== */ -static irqreturn_t flexrm_irq_event(int irq, void *dev_id) -{ - /* We only have MSI for completions so just wakeup IRQ thread */ - /* Ring related errors will be informed via completion descriptors */ - - return IRQ_WAKE_THREAD; -} - static irqreturn_t flexrm_irq_thread(int irq, void *dev_id) { flexrm_process_completions(dev_id); @@ -1271,10 +1263,8 @@ static int flexrm_startup(struct mbox_chan *chan) ret = -ENODEV; goto fail_free_cmpl_memory; } - ret = request_threaded_irq(ring->irq, - flexrm_irq_event, - flexrm_irq_thread, - 0, dev_name(ring->mbox->dev), ring); + ret = request_threaded_irq(ring->irq, NULL, flexrm_irq_thread, + IRQF_ONESHOT, dev_name(ring->mbox->dev), ring); if (ret) { dev_err(ring->mbox->dev, "failed to request ring%d IRQ\n", ring->num); diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index 6778afc64a048..003f9236c35e0 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -122,6 +122,7 @@ struct imx_mu_dcfg { u32 xRR; /* Receive Register0 */ u32 xSR[IMX_MU_xSR_MAX]; /* Status Registers */ u32 xCR[IMX_MU_xCR_MAX]; /* Control Registers */ + bool skip_suspend_flag; }; #define IMX_MU_xSR_GIPn(type, x) (type & IMX_MU_V2 ? BIT(x) : BIT(28 + (3 - (x)))) @@ -988,6 +989,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx7ulp = { .xRR = 0x40, .xSR = {0x60, 0x60, 0x60, 0x60}, .xCR = {0x64, 0x64, 0x64, 0x64, 0x64}, + .skip_suspend_flag = true, }; static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { @@ -1071,7 +1073,8 @@ static int __maybe_unused imx_mu_suspend_noirq(struct device *dev) priv->xcr[i] = imx_mu_read(priv, priv->dcfg->xCR[i]); } - priv->suspend = true; + if (!priv->dcfg->skip_suspend_flag) + priv->suspend = true; return 0; } @@ -1094,7 +1097,8 @@ static int __maybe_unused imx_mu_resume_noirq(struct device *dev) imx_mu_write(priv, priv->xcr[i], priv->dcfg->xCR[i]); } - priv->suspend = false; + if (!priv->dcfg->skip_suspend_flag) + priv->suspend = false; return 0; } diff --git a/drivers/mailbox/mailbox-mchp-ipc-sbi.c b/drivers/mailbox/mailbox-mchp-ipc-sbi.c index a6e52009a4245..b87bf2fb4b9b9 100644 --- a/drivers/mailbox/mailbox-mchp-ipc-sbi.c +++ b/drivers/mailbox/mailbox-mchp-ipc-sbi.c @@ -174,26 +174,30 @@ static irqreturn_t mchp_ipc_cluster_aggr_isr(int irq, void *data) struct mchp_ipc_msg ipc_msg; struct mchp_ipc_status status_msg; int ret; - unsigned long hartid; u32 i, chan_index, chan_id; + bool found = false; /* Find out the hart that originated the irq */ for_each_online_cpu(i) { - hartid = cpuid_to_hartid_map(i); - if (irq == ipc->cluster_cfg[hartid].irq) + if (irq == ipc->cluster_cfg[i].irq) { + found = true; break; + } } - status_msg.cluster = hartid; - memcpy(ipc->cluster_cfg[hartid].buf_base, &status_msg, sizeof(struct mchp_ipc_status)); + if (unlikely(!found)) + return IRQ_NONE; - ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[hartid].buf_base_addr); + status_msg.cluster = cpuid_to_hartid_map(i); + memcpy(ipc->cluster_cfg[i].buf_base, &status_msg, sizeof(struct mchp_ipc_status)); + + ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[i].buf_base_addr); if (ret < 0) { dev_err_ratelimited(ipc->dev, "could not get IHC irq status ret=%d\n", ret); return IRQ_HANDLED; } - memcpy(&status_msg, ipc->cluster_cfg[hartid].buf_base, sizeof(struct mchp_ipc_status)); + memcpy(&status_msg, ipc->cluster_cfg[i].buf_base, sizeof(struct mchp_ipc_status)); /* * Iterate over each bit set in the IHC interrupt status register (IRQ_STATUS) to identify @@ -321,13 +325,6 @@ static int mchp_ipc_startup(struct mbox_chan *chan) goto fail_free_buf_msg_rx; } - if (ret) { - dev_err(ipc->dev, "failed to register interrupt(s)\n"); - goto fail_free_buf_msg_rx; - } - - return ret; - fail_free_buf_msg_rx: kfree(chan_info->msg_buf_rx); fail_free_buf_msg_tx: @@ -385,21 +382,21 @@ static int mchp_ipc_get_cluster_aggr_irq(struct mchp_ipc_sbi_mbox *ipc) if (ret <= 0) continue; - ipc->cluster_cfg[hartid].irq = ret; - ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[hartid].irq, + ipc->cluster_cfg[cpuid].irq = ret; + ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[cpuid].irq, mchp_ipc_cluster_aggr_isr, IRQF_SHARED, "miv-ihc-irq", ipc); if (ret) return ret; - ipc->cluster_cfg[hartid].buf_base = devm_kmalloc(ipc->dev, - sizeof(struct mchp_ipc_status), - GFP_KERNEL); + ipc->cluster_cfg[cpuid].buf_base = devm_kmalloc(ipc->dev, + sizeof(struct mchp_ipc_status), + GFP_KERNEL); - if (!ipc->cluster_cfg[hartid].buf_base) + if (!ipc->cluster_cfg[cpuid].buf_base) return -ENOMEM; - ipc->cluster_cfg[hartid].buf_base_addr = __pa(ipc->cluster_cfg[hartid].buf_base); + ipc->cluster_cfg[cpuid].buf_base_addr = __pa(ipc->cluster_cfg[cpuid].buf_base); irq_found = true; } @@ -419,7 +416,7 @@ static int mchp_ipc_probe(struct platform_device *pdev) ret = sbi_probe_extension(SBI_EXT_MICROCHIP_TECHNOLOGY); if (ret <= 0) - return dev_err_probe(dev, ret, "Microchip SBI extension not detected\n"); + return dev_err_probe(dev, -ENODEV, "Microchip SBI extension not detected\n"); ipc = devm_kzalloc(dev, sizeof(*ipc), GFP_KERNEL); if (!ipc) diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 2acc6ec229a45..617ba505691d3 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -489,12 +489,10 @@ EXPORT_SYMBOL_GPL(mbox_free_channel); static struct mbox_chan *fw_mbox_index_xlate(struct mbox_controller *mbox, const struct fwnode_reference_args *sp) { - int ind = sp->args[0]; - - if (ind >= mbox->num_chans) + if (sp->nargs < 1 || sp->args[0] >= mbox->num_chans) return ERR_PTR(-EINVAL); - return &mbox->chans[ind]; + return &mbox->chans[sp->args[0]]; } /** diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index ff292b9e0be9e..713022aed2e2f 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -305,22 +305,6 @@ static void pcc_chan_acknowledge(struct pcc_chan_info *pchan) pcc_chan_reg_read_modify_write(&pchan->db); } -static void *write_response(struct pcc_chan_info *pchan) -{ - struct pcc_header pcc_header; - void *buffer; - int data_len; - - memcpy_fromio(&pcc_header, pchan->chan.shmem, - sizeof(pcc_header)); - data_len = pcc_header.length - sizeof(u32) + sizeof(struct pcc_header); - - buffer = pchan->chan.rx_alloc(pchan->chan.mchan->cl, data_len); - if (buffer != NULL) - memcpy_fromio(buffer, pchan->chan.shmem, data_len); - return buffer; -} - /** * pcc_mbox_irq - PCC mailbox interrupt handler * @irq: interrupt number @@ -332,8 +316,6 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) { struct pcc_chan_info *pchan; struct mbox_chan *chan = p; - struct pcc_header *pcc_header = chan->active_req; - void *handle = NULL; pchan = chan->con_priv; @@ -357,17 +339,7 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) * required to avoid any possible race in updatation of this flag. */ pchan->chan_in_use = false; - - if (pchan->chan.rx_alloc) - handle = write_response(pchan); - - if (chan->active_req) { - pcc_header = chan->active_req; - if (pcc_header->flags & PCC_CMD_COMPLETION_NOTIFY) - mbox_chan_txdone(chan, 0); - } - - mbox_chan_received_data(chan, handle); + mbox_chan_received_data(chan, NULL); pcc_chan_acknowledge(pchan); @@ -411,24 +383,9 @@ pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) pcc_mchan = &pchan->chan; pcc_mchan->shmem = acpi_os_ioremap(pcc_mchan->shmem_base_addr, pcc_mchan->shmem_size); - if (!pcc_mchan->shmem) - goto err; - - pcc_mchan->manage_writes = false; - - /* This indicates that the channel is ready to accept messages. - * This needs to happen after the channel has registered - * its callback. There is no access point to do that in - * the mailbox API. That implies that the mailbox client must - * have set the allocate callback function prior to - * sending any messages. - */ - if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE) - pcc_chan_reg_read_modify_write(&pchan->cmd_update); - - return pcc_mchan; + if (pcc_mchan->shmem) + return pcc_mchan; -err: mbox_free_channel(chan); return ERR_PTR(-ENXIO); } @@ -459,38 +416,8 @@ void pcc_mbox_free_channel(struct pcc_mbox_chan *pchan) } EXPORT_SYMBOL_GPL(pcc_mbox_free_channel); -static int pcc_write_to_buffer(struct mbox_chan *chan, void *data) -{ - struct pcc_chan_info *pchan = chan->con_priv; - struct pcc_mbox_chan *pcc_mbox_chan = &pchan->chan; - struct pcc_header *pcc_header = data; - - if (!pchan->chan.manage_writes) - return 0; - - /* The PCC header length includes the command field - * but not the other values from the header. - */ - int len = pcc_header->length - sizeof(u32) + sizeof(struct pcc_header); - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) { - pr_info("%s pchan->cmd_complete not set", __func__); - return -1; - } - memcpy_toio(pcc_mbox_chan->shmem, data, len); - return 0; -} - - /** - * pcc_send_data - Called from Mailbox Controller code. If - * pchan->chan.rx_alloc is set, then the command complete - * flag is checked and the data is written to the shared - * buffer io memory. - * - * If pchan->chan.rx_alloc is not set, then it is used + * pcc_send_data - Called from Mailbox Controller code. Used * here only to ring the channel doorbell. The PCC client * specific read/write is done in the client driver in * order to maintain atomicity over PCC channel once @@ -506,37 +433,17 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) int ret; struct pcc_chan_info *pchan = chan->con_priv; - ret = pcc_write_to_buffer(chan, data); - if (ret) - return ret; - ret = pcc_chan_reg_read_modify_write(&pchan->cmd_update); if (ret) return ret; ret = pcc_chan_reg_read_modify_write(&pchan->db); - if (!ret && pchan->plat_irq > 0) pchan->chan_in_use = true; return ret; } - -static bool pcc_last_tx_done(struct mbox_chan *chan) -{ - struct pcc_chan_info *pchan = chan->con_priv; - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) - return false; - else - return true; -} - - - /** * pcc_startup - Called from Mailbox Controller code. Used here * to request the interrupt. @@ -552,7 +459,7 @@ static int pcc_startup(struct mbox_chan *chan) if (pchan->plat_irq > 0) { irqflags = pcc_chan_plat_irq_can_be_shared(pchan) ? - IRQF_SHARED | IRQF_ONESHOT : 0; + IRQF_SHARED : 0; rc = devm_request_irq(chan->mbox->dev, pchan->plat_irq, pcc_mbox_irq, irqflags, MBOX_IRQ_NAME, chan); if (unlikely(rc)) { @@ -582,7 +489,6 @@ static const struct mbox_chan_ops pcc_chan_ops = { .send_data = pcc_send_data, .startup = pcc_startup, .shutdown = pcc_shutdown, - .last_tx_done = pcc_last_tx_done, }; /** diff --git a/drivers/mailbox/sprd-mailbox.c b/drivers/mailbox/sprd-mailbox.c index ee8539dfcef54..46d0c34177ab9 100644 --- a/drivers/mailbox/sprd-mailbox.c +++ b/drivers/mailbox/sprd-mailbox.c @@ -166,6 +166,11 @@ static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data) return IRQ_NONE; } + /* Clear FIFO delivery and overflow status first */ + writel(fifo_sts & + (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK), + priv->inbox_base + SPRD_MBOX_FIFO_RST); + while (send_sts) { id = __ffs(send_sts); send_sts &= (send_sts - 1); @@ -181,11 +186,6 @@ static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data) mbox_chan_txdone(chan, 0); } - /* Clear FIFO delivery and overflow status */ - writel(fifo_sts & - (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK), - priv->inbox_base + SPRD_MBOX_FIFO_RST); - /* Clear irq status */ writel(SPRD_MBOX_IRQ_CLR, priv->inbox_base + SPRD_MBOX_IRQ_STS); @@ -243,21 +243,19 @@ static int sprd_mbox_startup(struct mbox_chan *chan) /* Select outbox FIFO mode and reset the outbox FIFO status */ writel(0x0, priv->outbox_base + SPRD_MBOX_FIFO_RST); - /* Enable inbox FIFO overflow and delivery interrupt */ - val = readl(priv->inbox_base + SPRD_MBOX_IRQ_MSK); - val &= ~(SPRD_INBOX_FIFO_OVERFLOW_IRQ | SPRD_INBOX_FIFO_DELIVER_IRQ); + /* Enable inbox FIFO delivery interrupt */ + val = SPRD_INBOX_FIFO_IRQ_MASK; + val &= ~SPRD_INBOX_FIFO_DELIVER_IRQ; writel(val, priv->inbox_base + SPRD_MBOX_IRQ_MSK); /* Enable outbox FIFO not empty interrupt */ - val = readl(priv->outbox_base + SPRD_MBOX_IRQ_MSK); + val = SPRD_OUTBOX_FIFO_IRQ_MASK; val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; writel(val, priv->outbox_base + SPRD_MBOX_IRQ_MSK); /* Enable supplementary outbox as the fundamental one */ if (priv->supp_base) { writel(0x0, priv->supp_base + SPRD_MBOX_FIFO_RST); - val = readl(priv->supp_base + SPRD_MBOX_IRQ_MSK); - val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; writel(val, priv->supp_base + SPRD_MBOX_IRQ_MSK); } } diff --git a/drivers/mcb/mcb-core.c b/drivers/mcb/mcb-core.c index c1367223e71a0..3d487d75c483d 100644 --- a/drivers/mcb/mcb-core.c +++ b/drivers/mcb/mcb-core.c @@ -85,7 +85,8 @@ static void mcb_remove(struct device *dev) struct mcb_device *mdev = to_mcb_device(dev); struct module *carrier_mod; - mdrv->remove(mdev); + if (mdrv->remove) + mdrv->remove(mdev); carrier_mod = mdev->dev.parent->driver->owner; module_put(carrier_mod); @@ -176,13 +177,13 @@ static const struct device_type mcb_carrier_device_type = { * @owner: The @mcb_driver's module * @mod_name: The name of the @mcb_driver's module * - * Register a @mcb_driver at the system. Perform some sanity checks, if - * the .probe and .remove methods are provided by the driver. + * Register a @mcb_driver at the system. Perform a sanity check, if + * .probe method is provided by the driver. */ int __mcb_register_driver(struct mcb_driver *drv, struct module *owner, const char *mod_name) { - if (!drv->probe || !drv->remove) + if (!drv->probe) return -EINVAL; drv->driver.owner = owner; diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 1d33e40d26ea5..cca5756030d71 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -273,6 +273,8 @@ struct bcache_device { struct bio_set bio_split; + struct bio_set bio_detached; + unsigned int data_csum:1; int (*cache_miss)(struct btree *b, struct search *s, @@ -755,6 +757,13 @@ struct bbio { struct bio bio; }; +struct detached_dev_io_private { + struct bcache_device *d; + unsigned long start_time; + struct bio *orig_bio; + struct bio bio; +}; + #define BTREE_PRIO USHRT_MAX #define INITIAL_PRIO 32768U diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index af345dc6fde14..6cba1180be8aa 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -1077,68 +1077,59 @@ static CLOSURE_CALLBACK(cached_dev_nodata) continue_at(cl, cached_dev_bio_complete, NULL); } -struct detached_dev_io_private { - struct bcache_device *d; - unsigned long start_time; - bio_end_io_t *bi_end_io; - void *bi_private; - struct block_device *orig_bdev; -}; - static void detached_dev_end_io(struct bio *bio) { - struct detached_dev_io_private *ddip; - - ddip = bio->bi_private; - bio->bi_end_io = ddip->bi_end_io; - bio->bi_private = ddip->bi_private; + struct detached_dev_io_private *ddip = + container_of(bio, struct detached_dev_io_private, bio); + struct bio *orig_bio = ddip->orig_bio; /* Count on the bcache device */ - bio_end_io_acct_remapped(bio, ddip->start_time, ddip->orig_bdev); + bio_end_io_acct(orig_bio, ddip->start_time); if (bio->bi_status) { - struct cached_dev *dc = container_of(ddip->d, - struct cached_dev, disk); + struct cached_dev *dc = bio->bi_private; + /* should count I/O error for backing device here */ bch_count_backing_io_errors(dc, bio); + orig_bio->bi_status = bio->bi_status; } - kfree(ddip); - bio->bi_end_io(bio); + bio_put(bio); + bio_endio(orig_bio); } -static void detached_dev_do_request(struct bcache_device *d, struct bio *bio, - struct block_device *orig_bdev, unsigned long start_time) +static void detached_dev_do_request(struct bcache_device *d, + struct bio *orig_bio, unsigned long start_time) { struct detached_dev_io_private *ddip; struct cached_dev *dc = container_of(d, struct cached_dev, disk); + struct bio *clone_bio; - /* - * no need to call closure_get(&dc->disk.cl), - * because upper layer had already opened bcache device, - * which would call closure_get(&dc->disk.cl) - */ - ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO); - if (!ddip) { - bio->bi_status = BLK_STS_RESOURCE; - bio->bi_end_io(bio); + if (bio_op(orig_bio) == REQ_OP_DISCARD && + !bdev_max_discard_sectors(dc->bdev)) { + bio_end_io_acct(orig_bio, start_time); + bio_endio(orig_bio); return; } - ddip->d = d; + clone_bio = bio_alloc_clone(dc->bdev, orig_bio, GFP_NOIO, + &d->bio_detached); + if (!clone_bio) { + orig_bio->bi_status = BLK_STS_RESOURCE; + bio_endio(orig_bio); + return; + } + + ddip = container_of(clone_bio, struct detached_dev_io_private, bio); /* Count on the bcache device */ - ddip->orig_bdev = orig_bdev; + ddip->d = d; ddip->start_time = start_time; - ddip->bi_end_io = bio->bi_end_io; - ddip->bi_private = bio->bi_private; - bio->bi_end_io = detached_dev_end_io; - bio->bi_private = ddip; - - if ((bio_op(bio) == REQ_OP_DISCARD) && - !bdev_max_discard_sectors(dc->bdev)) - bio->bi_end_io(bio); - else - submit_bio_noacct(bio); + ddip->orig_bio = orig_bio; + + clone_bio->bi_end_io = detached_dev_end_io; + clone_bio->bi_private = dc; + + submit_bio_noacct(clone_bio); } static void quit_max_writeback_rate(struct cache_set *c, @@ -1214,10 +1205,10 @@ void cached_dev_submit_bio(struct bio *bio) start_time = bio_start_io_acct(bio); - bio_set_dev(bio, dc->bdev); bio->bi_iter.bi_sector += dc->sb.data_offset; if (cached_dev_get(dc)) { + bio_set_dev(bio, dc->bdev); s = search_alloc(bio, d, orig_bdev, start_time); trace_bcache_request_start(s->d, bio); @@ -1237,9 +1228,10 @@ void cached_dev_submit_bio(struct bio *bio) else cached_dev_read(dc, s); } - } else + } else { /* I/O request sent to backing device */ - detached_dev_do_request(d, bio, orig_bdev, start_time); + detached_dev_do_request(d, bio, start_time); + } } static int cached_dev_ioctl(struct bcache_device *d, blk_mode_t mode, diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 6d250e366412c..2f06945533d67 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -887,6 +887,7 @@ static void bcache_device_free(struct bcache_device *d) } bioset_exit(&d->bio_split); + bioset_exit(&d->bio_detached); kvfree(d->full_dirty_stripes); kvfree(d->stripe_sectors_dirty); @@ -949,6 +950,11 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size, BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER)) goto out_ida_remove; + if (bioset_init(&d->bio_detached, 4, + offsetof(struct detached_dev_io_private, bio), + BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER)) + goto out_bioset_split_exit; + if (lim.logical_block_size > PAGE_SIZE && cached_bdev) { /* * This should only happen with BCACHE_SB_VERSION_BDEV. @@ -964,7 +970,7 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size, d->disk = blk_alloc_disk(&lim, NUMA_NO_NODE); if (IS_ERR(d->disk)) - goto out_bioset_exit; + goto out_bioset_detach_exit; set_capacity(d->disk, sectors); snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", idx); @@ -976,7 +982,9 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size, d->disk->private_data = d; return 0; -out_bioset_exit: +out_bioset_detach_exit: + bioset_exit(&d->bio_detached); +out_bioset_split_exit: bioset_exit(&d->bio_split); out_ida_remove: ida_free(&bcache_device_idx, idx); @@ -1365,6 +1373,13 @@ static CLOSURE_CALLBACK(cached_dev_free) mutex_unlock(&bch_register_lock); + /* + * Wait for any pending sb_write to complete before free. + * The sb_bio is embedded in struct cached_dev, so we must + * ensure no I/O is in progress. + */ + closure_sync(&dc->sb_write); + if (dc->sb_disk) folio_put(virt_to_folio(dc->sb_disk)); diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index e6d28be11c5c6..5235f3e4924b7 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1374,7 +1374,7 @@ static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio { unsigned int n_sectors; sector_t sector; - unsigned int offset, end; + unsigned int offset, end, align; b->end_io = end_io; @@ -1388,9 +1388,11 @@ static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio b->c->write_callback(b); offset = b->write_start; end = b->write_end; - offset &= -DM_BUFIO_WRITE_ALIGN; - end += DM_BUFIO_WRITE_ALIGN - 1; - end &= -DM_BUFIO_WRITE_ALIGN; + align = max(DM_BUFIO_WRITE_ALIGN, + bdev_physical_block_size(b->c->bdev)); + offset &= -align; + end += align - 1; + end &= -align; if (unlikely(end > b->c->block_size)) end = b->c->block_size; diff --git a/drivers/md/dm-ebs-target.c b/drivers/md/dm-ebs-target.c index 6abb31ca96623..b354e74a670e0 100644 --- a/drivers/md/dm-ebs-target.c +++ b/drivers/md/dm-ebs-target.c @@ -103,7 +103,7 @@ static int __ebs_rw_bvec(struct ebs_c *ec, enum req_op op, struct bio_vec *bv, } else { flush_dcache_page(bv->bv_page); memcpy(ba, pa, cur_len); - dm_bufio_mark_partial_buffer_dirty(b, buf_off, buf_off + cur_len); + dm_bufio_mark_buffer_dirty(b); } dm_bufio_release(b); diff --git a/drivers/md/dm-exception-store.c b/drivers/md/dm-exception-store.c index c3799757bf4a0..88f119a0a2ae0 100644 --- a/drivers/md/dm-exception-store.c +++ b/drivers/md/dm-exception-store.c @@ -116,7 +116,7 @@ int dm_exception_store_type_register(struct dm_exception_store_type *type) if (!__find_exception_store_type(type->name)) list_add(&type->list, &_exception_store_types); else - r = -EEXIST; + r = -EBUSY; spin_unlock(&_lock); return r; diff --git a/drivers/md/dm-exception-store.h b/drivers/md/dm-exception-store.h index b679766375381..061b4d3108132 100644 --- a/drivers/md/dm-exception-store.h +++ b/drivers/md/dm-exception-store.h @@ -29,7 +29,7 @@ typedef sector_t chunk_t; * chunk within the device. */ struct dm_exception { - struct hlist_bl_node hash_list; + struct hlist_node hash_list; chunk_t old_chunk; chunk_t new_chunk; diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 170bf67a2edd9..ba52631052503 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -2411,7 +2411,7 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); if (unlikely(new_pos != NOT_FOUND) || - unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { + unlikely(next_sector < dio->range.logical_sector + dio->range.n_sectors)) { remove_range_unlocked(ic, &dio->range); spin_unlock_irq(&ic->endio_wait.lock); queue_work(ic->commit_wq, &ic->commit_work); @@ -3788,14 +3788,27 @@ static void dm_integrity_resume(struct dm_target *ti) struct dm_integrity_c *ic = ti->private; __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); int r; + __le32 flags; DEBUG_print("resume\n"); ic->wrote_to_journal = false; + flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING); + r = sync_rw_sb(ic, REQ_OP_READ); + if (r) + dm_integrity_io_error(ic, "reading superblock", r); + if ((ic->sb->flags & flags) != flags) { + ic->sb->flags |= flags; + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); + } + if (ic->provided_data_sectors != old_provided_data_sectors) { if (ic->provided_data_sectors > old_provided_data_sectors && ic->mode == 'B' && + ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP) && ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { rw_journal_sectors(ic, REQ_OP_READ, 0, ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index 7bb7174f8f4f8..f0c84e7a5daa6 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c @@ -432,6 +432,7 @@ static int log_writes_kthread(void *arg) struct log_writes_c *lc = arg; sector_t sector = 0; + set_freezable(); while (!kthread_should_stop()) { bool super = false; bool logging_enabled; diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 9d85d045f9d9d..bced5a783ee33 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -121,7 +121,7 @@ int dm_dirty_log_type_register(struct dm_dirty_log_type *type) if (!__find_dirty_log_type(type->name)) list_add(&type->list, &_log_types); else - r = -EEXIST; + r = -EBUSY; spin_unlock(&_lock); return r; diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index aaf4a0a4b0ebb..7d3fdd96f4edf 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -225,6 +225,7 @@ static struct multipath *alloc_multipath(struct dm_target *ti) mutex_init(&m->work_mutex); m->queue_mode = DM_TYPE_NONE; + m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; m->ti = ti; ti->private = m; @@ -257,7 +258,6 @@ static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) set_bit(MPATHF_QUEUE_IO, &m->flags); atomic_set(&m->pg_init_in_progress, 0); atomic_set(&m->pg_init_count, 0); - m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; init_waitqueue_head(&m->pg_init_wait); init_waitqueue_head(&m->probe_wait); diff --git a/drivers/md/dm-path-selector.c b/drivers/md/dm-path-selector.c index d0b883fabfeb6..2b0ac200f1c02 100644 --- a/drivers/md/dm-path-selector.c +++ b/drivers/md/dm-path-selector.c @@ -107,7 +107,7 @@ int dm_register_path_selector(struct path_selector_type *pst) if (__find_path_selector_type(pst->name)) { kfree(psi); - r = -EEXIST; + r = -EBUSY; } else list_add(&psi->list, &_path_selectors); diff --git a/drivers/md/dm-pcache/cache.c b/drivers/md/dm-pcache/cache.c index 698697a7a73c6..534bf07b794f1 100644 --- a/drivers/md/dm-pcache/cache.c +++ b/drivers/md/dm-pcache/cache.c @@ -10,7 +10,8 @@ struct kmem_cache *key_cache; static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache) { - return cache->cache_info_addr + cache->info_index; + return (struct pcache_cache_info *)((char *)cache->cache_info_addr + + (size_t)cache->info_index * PCACHE_CACHE_INFO_SIZE); } static void cache_info_write(struct pcache_cache *cache) @@ -21,10 +22,10 @@ static void cache_info_write(struct pcache_cache *cache) cache_info->header.crc = pcache_meta_crc(&cache_info->header, sizeof(struct pcache_cache_info)); + cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX; memcpy_flushcache(get_cache_info_addr(cache), cache_info, sizeof(struct pcache_cache_info)); - - cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX; + pmem_wmb(); } static void cache_info_init_default(struct pcache_cache *cache); @@ -49,6 +50,8 @@ static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_optio return -EINVAL; } + cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO_SIZE; + return 0; } @@ -93,10 +96,10 @@ void cache_pos_encode(struct pcache_cache *cache, pos_onmedia.header.seq = seq; pos_onmedia.header.crc = cache_pos_onmedia_crc(&pos_onmedia); + *index = (*index + 1) % PCACHE_META_INDEX_MAX; + memcpy_flushcache(pos_onmedia_addr, &pos_onmedia, sizeof(struct pcache_cache_pos_onmedia)); pmem_wmb(); - - *index = (*index + 1) % PCACHE_META_INDEX_MAX; } int cache_pos_decode(struct pcache_cache *cache, diff --git a/drivers/md/dm-pcache/cache_segment.c b/drivers/md/dm-pcache/cache_segment.c index f0b58980806e0..9d92e2b067ed0 100644 --- a/drivers/md/dm-pcache/cache_segment.c +++ b/drivers/md/dm-pcache/cache_segment.c @@ -26,11 +26,11 @@ static void cache_seg_info_write(struct pcache_cache_segment *cache_seg) seg_info->header.seq++; seg_info->header.crc = pcache_meta_crc(&seg_info->header, sizeof(struct pcache_segment_info)); + cache_seg->info_index = (cache_seg->info_index + 1) % PCACHE_META_INDEX_MAX; + seg_info_addr = get_seg_info_addr(cache_seg); memcpy_flushcache(seg_info_addr, seg_info, sizeof(struct pcache_segment_info)); pmem_wmb(); - - cache_seg->info_index = (cache_seg->info_index + 1) % PCACHE_META_INDEX_MAX; mutex_unlock(&cache_seg->info_lock); } @@ -56,7 +56,10 @@ static int cache_seg_info_load(struct pcache_cache_segment *cache_seg) ret = -EIO; goto out; } - cache_seg->info_index = cache_seg_info_addr - cache_seg_info_addr_base; + + cache_seg->info_index = + ((char *)cache_seg_info_addr - (char *)cache_seg_info_addr_base) / + PCACHE_SEG_INFO_SIZE; out: mutex_unlock(&cache_seg->info_lock); @@ -129,10 +132,10 @@ static void cache_seg_ctrl_write(struct pcache_cache_segment *cache_seg) cache_seg_gen.header.crc = pcache_meta_crc(&cache_seg_gen.header, sizeof(struct pcache_cache_seg_gen)); + cache_seg->gen_index = (cache_seg->gen_index + 1) % PCACHE_META_INDEX_MAX; + memcpy_flushcache(get_cache_seg_gen_addr(cache_seg), &cache_seg_gen, sizeof(struct pcache_cache_seg_gen)); pmem_wmb(); - - cache_seg->gen_index = (cache_seg->gen_index + 1) % PCACHE_META_INDEX_MAX; } static void cache_seg_ctrl_init(struct pcache_cache_segment *cache_seg) diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index c6f7129e43d34..4bacdc499984b 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -2287,6 +2287,8 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) mddev->reshape_position = le64_to_cpu(sb->reshape_position); rs->raid_type = get_raid_type_by_ll(mddev->level, mddev->layout); + if (!rs->raid_type) + return -EINVAL; } } else { diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index a6ca92049c10e..923252fb57aec 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -109,14 +109,21 @@ static void end_clone_bio(struct bio *clone) */ tio->completed += nr_bytes; + if (!is_last) + return; + /* + * At this moment we know this is the last bio of the cloned request, + * and all cloned bios have been released, so reset the clone request's + * bio pointer to avoid double free. + */ + tio->clone->bio = NULL; + exit: /* * Update the original request. * Do not use blk_mq_end_request() here, because it may complete * the original request before the clone, and break the ordering. */ - if (is_last) - exit: - blk_update_request(tio->orig, BLK_STS_OK, tio->completed); + blk_update_request(tio->orig, BLK_STS_OK, tio->completed); } static struct dm_rq_target_io *tio_from_request(struct request *rq) @@ -278,8 +285,7 @@ static void dm_complete_request(struct request *rq, blk_status_t error) struct dm_rq_target_io *tio = tio_from_request(rq); tio->error = error; - if (likely(!blk_should_fake_timeout(rq->q))) - blk_mq_complete_request(rq); + blk_mq_complete_request(rq); } /* diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index f40c18da40000..dbd148967de42 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -40,10 +40,15 @@ static const char dm_snapshot_merge_target_name[] = "snapshot-merge"; #define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \ (DM_TRACKED_CHUNK_HASH_SIZE - 1)) +struct dm_hlist_head { + struct hlist_head head; + spinlock_t lock; +}; + struct dm_exception_table { uint32_t hash_mask; unsigned int hash_shift; - struct hlist_bl_head *table; + struct dm_hlist_head *table; }; struct dm_snapshot { @@ -628,8 +633,8 @@ static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk); /* Lock to protect access to the completed and pending exception hash tables. */ struct dm_exception_table_lock { - struct hlist_bl_head *complete_slot; - struct hlist_bl_head *pending_slot; + spinlock_t *complete_slot; + spinlock_t *pending_slot; }; static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk, @@ -638,20 +643,20 @@ static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk, struct dm_exception_table *complete = &s->complete; struct dm_exception_table *pending = &s->pending; - lock->complete_slot = &complete->table[exception_hash(complete, chunk)]; - lock->pending_slot = &pending->table[exception_hash(pending, chunk)]; + lock->complete_slot = &complete->table[exception_hash(complete, chunk)].lock; + lock->pending_slot = &pending->table[exception_hash(pending, chunk)].lock; } static void dm_exception_table_lock(struct dm_exception_table_lock *lock) { - hlist_bl_lock(lock->complete_slot); - hlist_bl_lock(lock->pending_slot); + spin_lock_nested(lock->complete_slot, 1); + spin_lock_nested(lock->pending_slot, 2); } static void dm_exception_table_unlock(struct dm_exception_table_lock *lock) { - hlist_bl_unlock(lock->pending_slot); - hlist_bl_unlock(lock->complete_slot); + spin_unlock(lock->pending_slot); + spin_unlock(lock->complete_slot); } static int dm_exception_table_init(struct dm_exception_table *et, @@ -661,13 +666,15 @@ static int dm_exception_table_init(struct dm_exception_table *et, et->hash_shift = hash_shift; et->hash_mask = size - 1; - et->table = kvmalloc_array(size, sizeof(struct hlist_bl_head), + et->table = kvmalloc_array(size, sizeof(struct dm_hlist_head), GFP_KERNEL); if (!et->table) return -ENOMEM; - for (i = 0; i < size; i++) - INIT_HLIST_BL_HEAD(et->table + i); + for (i = 0; i < size; i++) { + INIT_HLIST_HEAD(&et->table[i].head); + spin_lock_init(&et->table[i].lock); + } return 0; } @@ -675,16 +682,17 @@ static int dm_exception_table_init(struct dm_exception_table *et, static void dm_exception_table_exit(struct dm_exception_table *et, struct kmem_cache *mem) { - struct hlist_bl_head *slot; + struct dm_hlist_head *slot; struct dm_exception *ex; - struct hlist_bl_node *pos, *n; + struct hlist_node *pos; int i, size; size = et->hash_mask + 1; for (i = 0; i < size; i++) { slot = et->table + i; - hlist_bl_for_each_entry_safe(ex, pos, n, slot, hash_list) { + hlist_for_each_entry_safe(ex, pos, &slot->head, hash_list) { + hlist_del(&ex->hash_list); kmem_cache_free(mem, ex); cond_resched(); } @@ -700,7 +708,7 @@ static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk) static void dm_remove_exception(struct dm_exception *e) { - hlist_bl_del(&e->hash_list); + hlist_del(&e->hash_list); } /* @@ -710,12 +718,11 @@ static void dm_remove_exception(struct dm_exception *e) static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et, chunk_t chunk) { - struct hlist_bl_head *slot; - struct hlist_bl_node *pos; + struct hlist_head *slot; struct dm_exception *e; - slot = &et->table[exception_hash(et, chunk)]; - hlist_bl_for_each_entry(e, pos, slot, hash_list) + slot = &et->table[exception_hash(et, chunk)].head; + hlist_for_each_entry(e, slot, hash_list) if (chunk >= e->old_chunk && chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) return e; @@ -762,18 +769,17 @@ static void free_pending_exception(struct dm_snap_pending_exception *pe) static void dm_insert_exception(struct dm_exception_table *eh, struct dm_exception *new_e) { - struct hlist_bl_head *l; - struct hlist_bl_node *pos; + struct hlist_head *l; struct dm_exception *e = NULL; - l = &eh->table[exception_hash(eh, new_e->old_chunk)]; + l = &eh->table[exception_hash(eh, new_e->old_chunk)].head; /* Add immediately if this table doesn't support consecutive chunks */ if (!eh->hash_shift) goto out; /* List is ordered by old_chunk */ - hlist_bl_for_each_entry(e, pos, l, hash_list) { + hlist_for_each_entry(e, l, hash_list) { /* Insert after an existing chunk? */ if (new_e->old_chunk == (e->old_chunk + dm_consecutive_chunk_count(e) + 1) && @@ -804,13 +810,13 @@ static void dm_insert_exception(struct dm_exception_table *eh, * Either the table doesn't support consecutive chunks or slot * l is empty. */ - hlist_bl_add_head(&new_e->hash_list, l); + hlist_add_head(&new_e->hash_list, l); } else if (new_e->old_chunk < e->old_chunk) { /* Add before an existing exception */ - hlist_bl_add_before(&new_e->hash_list, &e->hash_list); + hlist_add_before(&new_e->hash_list, &e->hash_list); } else { /* Add to l's tail: e is the last exception in this slot */ - hlist_bl_add_behind(&new_e->hash_list, &e->hash_list); + hlist_add_behind(&new_e->hash_list, &e->hash_list); } } @@ -820,7 +826,6 @@ static void dm_insert_exception(struct dm_exception_table *eh, */ static int dm_add_exception(void *context, chunk_t old, chunk_t new) { - struct dm_exception_table_lock lock; struct dm_snapshot *s = context; struct dm_exception *e; @@ -833,17 +838,7 @@ static int dm_add_exception(void *context, chunk_t old, chunk_t new) /* Consecutive_count is implicitly initialised to zero */ e->new_chunk = new; - /* - * Although there is no need to lock access to the exception tables - * here, if we don't then hlist_bl_add_head(), called by - * dm_insert_exception(), will complain about accessing the - * corresponding list without locking it first. - */ - dm_exception_table_lock_init(s, old, &lock); - - dm_exception_table_lock(&lock); dm_insert_exception(&s->complete, e); - dm_exception_table_unlock(&lock); return 0; } @@ -873,7 +868,7 @@ static int calc_max_buckets(void) /* use a fixed size of 2MB */ unsigned long mem = 2 * 1024 * 1024; - mem /= sizeof(struct hlist_bl_head); + mem /= sizeof(struct dm_hlist_head); return mem; } diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index ad0a60a07b935..81265ed204b54 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -1237,9 +1237,6 @@ static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev, bdev_get_queue(bdev)->crypto_profile; int err = -EOPNOTSUPP; - if (!args->err) - return 0; - switch (args->op) { case DERIVE_SW_SECRET: err = blk_crypto_derive_sw_secret( @@ -1266,9 +1263,7 @@ static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev, break; } args->err = err; - - /* Try another device in case this fails. */ - return 0; + return 1; /* No need to continue the iteration. */ } static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile, @@ -1294,14 +1289,13 @@ static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile, * declared on all underlying devices. Thus, all the underlying devices * should support all wrapped key operations and they should behave * identically, i.e. work with the same keys. So, just executing the - * operation on the first device on which it works suffices for now. + * operation on the first device suffices for now. */ for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->type->iterate_devices) continue; - ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args); - if (!args->err) + if (ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args) != 0) break; } out: diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c index 8fede41adec00..1fd41289de367 100644 --- a/drivers/md/dm-target.c +++ b/drivers/md/dm-target.c @@ -88,7 +88,7 @@ int dm_register_target(struct target_type *tt) if (__find_target_type(tt->name)) { DMERR("%s: '%s' target already registered", __func__, tt->name); - rv = -EEXIST; + rv = -EBUSY; } else { list_add(&tt->list, &_targets); } diff --git a/drivers/md/dm-unstripe.c b/drivers/md/dm-unstripe.c index e8a9432057dce..17be483595642 100644 --- a/drivers/md/dm-unstripe.c +++ b/drivers/md/dm-unstripe.c @@ -117,7 +117,7 @@ static void unstripe_dtr(struct dm_target *ti) static sector_t map_to_core(struct dm_target *ti, struct bio *bio) { struct unstripe_c *uc = ti->private; - sector_t sector = bio->bi_iter.bi_sector; + sector_t sector = dm_target_offset(ti, bio->bi_iter.bi_sector); sector_t tmp_sector = sector; /* Shift us up to the right "row" on the stripe */ diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 72047b47a7a0a..5365b3987374a 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -413,10 +413,8 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, if (!verity_fec_is_enabled(v)) return -EOPNOTSUPP; - if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { - DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); + if (fio->level) return -EIO; - } fio->level++; @@ -535,9 +533,9 @@ void verity_fec_dtr(struct dm_verity *v) mempool_exit(&f->output_pool); kmem_cache_destroy(f->cache); - if (f->data_bufio) + if (!IS_ERR_OR_NULL(f->data_bufio)) dm_bufio_client_destroy(f->data_bufio); - if (f->bufio) + if (!IS_ERR_OR_NULL(f->bufio)) dm_bufio_client_destroy(f->bufio); if (f->dev) diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h index 09123a6129538..ec37e607cb3f0 100644 --- a/drivers/md/dm-verity-fec.h +++ b/drivers/md/dm-verity-fec.h @@ -23,9 +23,6 @@ #define DM_VERITY_FEC_BUF_MAX \ (1 << (PAGE_SHIFT - DM_VERITY_FEC_BUF_RS_BITS)) -/* maximum recursion level for verity_fec_decode */ -#define DM_VERITY_FEC_MAX_RECURSION 4 - #define DM_VERITY_OPT_FEC_DEV "use_fec_from_device" #define DM_VERITY_OPT_FEC_BLOCKS "fec_blocks" #define DM_VERITY_OPT_FEC_START "fec_start" diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 66a00a8ccb398..c8695c079cfe0 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -1690,7 +1690,7 @@ static struct target_type verity_target = { .name = "verity", /* Note: the LSMs depend on the singleton and immutable features */ .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, - .version = {1, 12, 0}, + .version = {1, 13, 0}, .module = THIS_MODULE, .ctr = verity_ctr, .dtr = verity_dtr, diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index 78e17dd4d01b8..f4950b5f766d3 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -56,7 +56,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, { struct mapped_device *md = disk->private_data; struct dm_table *map; - struct dm_table *zone_revalidate_map = md->zone_revalidate_map; + struct dm_table *zone_revalidate_map = READ_ONCE(md->zone_revalidate_map); int srcu_idx, ret = -EIO; bool put_table = false; @@ -66,11 +66,13 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, * Zone revalidation during __bind() is in progress, but this * call is from a different process */ - if (dm_suspended_md(md)) - return -EAGAIN; - map = dm_get_live_table(md, &srcu_idx); put_table = true; + + if (dm_suspended_md(md)) { + ret = -EAGAIN; + goto do_put_table; + } } else { /* Zone revalidation during __bind() */ map = zone_revalidate_map; @@ -80,6 +82,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data); +do_put_table: if (put_table) dm_put_live_table(md, srcu_idx); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 6c83ab940af75..52f01c44e73a6 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1363,6 +1363,8 @@ void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) if (!tgt_clone) tgt_clone = clone; + bio_clone_blkg_association(tgt_clone, io->orig_bio); + /* * Account io->origin_bio to DM dev on behalf of target * that took ownership of IO with DM_MAPIO_SUBMITTED. diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 84b7e2af6dbaa..7bb56d0491a2f 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2453,6 +2453,7 @@ static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, memcpy(page_address(store.sb_page), page_address(bitmap->storage.sb_page), sizeof(bitmap_super_t)); + mutex_lock(&bitmap->mddev->bitmap_info.mutex); spin_lock_irq(&bitmap->counts.lock); md_bitmap_file_unmap(&bitmap->storage); bitmap->storage = store; @@ -2560,7 +2561,7 @@ static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); } spin_unlock_irq(&bitmap->counts.lock); - + mutex_unlock(&bitmap->mddev->bitmap_info.mutex); if (!init) { __bitmap_unplug(bitmap); bitmap->mddev->pers->quiesce(bitmap->mddev, 0); diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c index 11f1e91d387d8..896279988dfd5 100644 --- a/drivers/md/md-cluster.c +++ b/drivers/md/md-cluster.c @@ -549,8 +549,13 @@ static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR); - /* daemaon thread must exist */ thread = rcu_dereference_protected(mddev->thread, true); + if (!thread) { + pr_warn("md-cluster: Received metadata update but MD thread is not ready\n"); + dlm_unlock_sync(cinfo->no_new_dev_lockres); + return; + } + wait_event(thread->wqueue, (got_lock = mddev_trylock(mddev)) || test_bit(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state)); diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1eb434306162a..bcb6eae1c711f 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -712,8 +712,10 @@ static int llbitmap_suspend_timeout(struct llbitmap *llbitmap, int page_idx) percpu_ref_kill(&pctl->active); if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active), - llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) + llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) { + percpu_ref_resurrect(&pctl->active); return -ETIMEDOUT; + } return 0; } diff --git a/drivers/md/md.c b/drivers/md/md.c index 41c476b40c7a3..92ec4be20db85 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -99,7 +99,7 @@ static int remove_and_add_spares(struct mddev *mddev, struct md_rdev *this); static void mddev_detach(struct mddev *mddev); static void export_rdev(struct md_rdev *rdev, struct mddev *mddev); -static void md_wakeup_thread_directly(struct md_thread __rcu *thread); +static void md_wakeup_thread_directly(struct md_thread __rcu **thread); /* * Default number of read corrections we'll attempt on an rdev @@ -730,6 +730,8 @@ static void mddev_clear_bitmap_ops(struct mddev *mddev) int mddev_init(struct mddev *mddev) { + int err = 0; + if (!IS_ENABLED(CONFIG_MD_BITMAP)) mddev->bitmap_id = ID_BITMAP_NONE; else @@ -741,10 +743,23 @@ int mddev_init(struct mddev *mddev) if (percpu_ref_init(&mddev->writes_pending, no_op, PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { - percpu_ref_exit(&mddev->active_io); - return -ENOMEM; + err = -ENOMEM; + goto exit_acitve_io; } + err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); + if (err) + goto exit_writes_pending; + + err = bioset_init(&mddev->sync_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); + if (err) + goto exit_bio_set; + + err = bioset_init(&mddev->io_clone_set, BIO_POOL_SIZE, + offsetof(struct md_io_clone, bio_clone), 0); + if (err) + goto exit_sync_set; + /* We want to start with the refcount at zero */ percpu_ref_put(&mddev->writes_pending); @@ -773,11 +788,24 @@ int mddev_init(struct mddev *mddev) INIT_WORK(&mddev->del_work, mddev_delayed_delete); return 0; + +exit_sync_set: + bioset_exit(&mddev->sync_set); +exit_bio_set: + bioset_exit(&mddev->bio_set); +exit_writes_pending: + percpu_ref_exit(&mddev->writes_pending); +exit_acitve_io: + percpu_ref_exit(&mddev->active_io); + return err; } EXPORT_SYMBOL_GPL(mddev_init); void mddev_destroy(struct mddev *mddev) { + bioset_exit(&mddev->bio_set); + bioset_exit(&mddev->sync_set); + bioset_exit(&mddev->io_clone_set); percpu_ref_exit(&mddev->active_io); percpu_ref_exit(&mddev->writes_pending); } @@ -941,8 +969,11 @@ void mddev_unlock(struct mddev *mddev) * do_md_stop. dm raid only uses md_stop to stop. So dm raid * doesn't need to check MD_DELETED when getting reconfig lock */ - if (test_bit(MD_DELETED, &mddev->flags)) + if (test_bit(MD_DELETED, &mddev->flags) && + !test_and_set_bit(MD_DO_DELETE, &mddev->flags)) { + kobject_del(&mddev->kobj); del_gendisk(mddev->gendisk); + } } } EXPORT_SYMBOL_GPL(mddev_unlock); @@ -3843,7 +3874,6 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe static int analyze_sbs(struct mddev *mddev) { - int i; struct md_rdev *rdev, *freshest, *tmp; freshest = NULL; @@ -3870,11 +3900,9 @@ static int analyze_sbs(struct mddev *mddev) super_types[mddev->major_version]. validate_super(mddev, NULL/*freshest*/, freshest); - i = 0; rdev_for_each_safe(rdev, tmp, mddev) { if (mddev->max_disks && - (rdev->desc_nr >= mddev->max_disks || - i > mddev->max_disks)) { + rdev->desc_nr >= mddev->max_disks) { pr_warn("md: %s: %pg: only %d devices permitted\n", mdname(mddev), rdev->bdev, mddev->max_disks); @@ -4368,7 +4396,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) if (err < 0) return err; - err = mddev_lock(mddev); + err = mddev_suspend_and_lock(mddev); if (err) return err; if (mddev->pers) @@ -4393,7 +4421,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) } else mddev->raid_disks = n; out_unlock: - mddev_unlock(mddev); + mddev_unlock_and_resume(mddev); return err ? err : len; } static struct md_sysfs_entry md_raid_disks = @@ -5134,7 +5162,7 @@ static void stop_sync_thread(struct mddev *mddev, bool locked) * Thread might be blocked waiting for metadata update which will now * never happen */ - md_wakeup_thread_directly(mddev->sync_thread); + md_wakeup_thread_directly(&mddev->sync_thread); if (work_pending(&mddev->sync_work)) flush_work(&mddev->sync_work); @@ -6384,29 +6412,9 @@ int md_run(struct mddev *mddev) nowait = nowait && bdev_nowait(rdev->bdev); } - if (!bioset_initialized(&mddev->bio_set)) { - err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); - if (err) - return err; - } - if (!bioset_initialized(&mddev->sync_set)) { - err = bioset_init(&mddev->sync_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); - if (err) - goto exit_bio_set; - } - - if (!bioset_initialized(&mddev->io_clone_set)) { - err = bioset_init(&mddev->io_clone_set, BIO_POOL_SIZE, - offsetof(struct md_io_clone, bio_clone), 0); - if (err) - goto exit_sync_set; - } - pers = get_pers(mddev->level, mddev->clevel); - if (!pers) { - err = -EINVAL; - goto abort; - } + if (!pers) + return -EINVAL; if (mddev->level != pers->head.id) { mddev->level = pers->head.id; mddev->new_level = pers->head.id; @@ -6417,8 +6425,7 @@ int md_run(struct mddev *mddev) pers->start_reshape == NULL) { /* This personality cannot handle reshaping... */ put_pers(pers); - err = -EINVAL; - goto abort; + return -EINVAL; } if (pers->sync_request) { @@ -6545,12 +6552,6 @@ int md_run(struct mddev *mddev) mddev->private = NULL; put_pers(pers); md_bitmap_destroy(mddev); -abort: - bioset_exit(&mddev->io_clone_set); -exit_sync_set: - bioset_exit(&mddev->sync_set); -exit_bio_set: - bioset_exit(&mddev->bio_set); return err; } EXPORT_SYMBOL_GPL(md_run); @@ -6715,13 +6716,15 @@ static void __md_stop_writes(struct mddev *mddev) { timer_delete_sync(&mddev->safemode_timer); - if (mddev->pers && mddev->pers->quiesce) { - mddev->pers->quiesce(mddev, 1); - mddev->pers->quiesce(mddev, 0); - } + if (md_is_rdwr(mddev) || !mddev_is_dm(mddev)) { + if (mddev->pers && mddev->pers->quiesce) { + mddev->pers->quiesce(mddev, 1); + mddev->pers->quiesce(mddev, 0); + } - if (md_bitmap_enabled(mddev, true)) - mddev->bitmap_ops->flush(mddev); + if (md_bitmap_enabled(mddev, true)) + mddev->bitmap_ops->flush(mddev); + } if (md_is_rdwr(mddev) && ((!mddev->in_sync && !mddev_is_clustered(mddev)) || @@ -6775,10 +6778,6 @@ static void __md_stop(struct mddev *mddev) mddev->private = NULL; put_pers(pers); clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery); - - bioset_exit(&mddev->bio_set); - bioset_exit(&mddev->sync_set); - bioset_exit(&mddev->io_clone_set); } void md_stop(struct mddev *mddev) @@ -6869,6 +6868,10 @@ static int do_md_stop(struct mddev *mddev, int mode) if (!md_is_rdwr(mddev)) set_disk_ro(disk, 0); + if (mode == 2 && mddev->pers->sync_request && + mddev->to_remove == NULL) + mddev->to_remove = &md_redundancy_group; + __md_stop_writes(mddev); __md_stop(mddev); @@ -8373,22 +8376,21 @@ static int md_thread(void *arg) return 0; } -static void md_wakeup_thread_directly(struct md_thread __rcu *thread) +static void md_wakeup_thread_directly(struct md_thread __rcu **thread) { struct md_thread *t; rcu_read_lock(); - t = rcu_dereference(thread); + t = rcu_dereference(*thread); if (t) wake_up_process(t->tsk); rcu_read_unlock(); } -void md_wakeup_thread(struct md_thread __rcu *thread) +void __md_wakeup_thread(struct md_thread __rcu *thread) { struct md_thread *t; - rcu_read_lock(); t = rcu_dereference(thread); if (t) { pr_debug("md: waking up MD thread %s.\n", t->tsk->comm); @@ -8396,9 +8398,8 @@ void md_wakeup_thread(struct md_thread __rcu *thread) if (wq_has_sleeper(&t->wqueue)) wake_up(&t->wqueue); } - rcu_read_unlock(); } -EXPORT_SYMBOL(md_wakeup_thread); +EXPORT_SYMBOL(__md_wakeup_thread); struct md_thread *md_register_thread(void (*run) (struct md_thread *), struct mddev *mddev, const char *name) diff --git a/drivers/md/md.h b/drivers/md/md.h index 1979c2d4fe89e..9d66afb8cc6e6 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -354,6 +354,7 @@ enum mddev_flags { MD_HAS_MULTIPLE_PPLS, MD_NOT_READY, MD_BROKEN, + MD_DO_DELETE, MD_DELETED, }; @@ -735,8 +736,8 @@ static inline int mddev_trylock(struct mddev *mddev) int ret; ret = mutex_trylock(&mddev->reconfig_mutex); - if (!ret && test_bit(MD_DELETED, &mddev->flags)) { - ret = -ENODEV; + if (ret && test_bit(MD_DELETED, &mddev->flags)) { + ret = 0; mutex_unlock(&mddev->reconfig_mutex); } return ret; @@ -882,6 +883,12 @@ struct md_io_clone { #define THREAD_WAKEUP 0 +#define md_wakeup_thread(thread) do { \ + rcu_read_lock(); \ + __md_wakeup_thread(thread); \ + rcu_read_unlock(); \ +} while (0) + static inline void safe_put_page(struct page *p) { if (p) put_page(p); @@ -895,7 +902,7 @@ extern struct md_thread *md_register_thread( struct mddev *mddev, const char *name); extern void md_unregister_thread(struct mddev *mddev, struct md_thread __rcu **threadp); -extern void md_wakeup_thread(struct md_thread __rcu *thread); +extern void __md_wakeup_thread(struct md_thread __rcu *thread); extern void md_check_recovery(struct mddev *mddev); extern void md_reap_sync_thread(struct mddev *mddev); extern enum sync_action md_sync_action(struct mddev *mddev); diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 592a402330047..ce7fd68869566 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3253,6 +3253,7 @@ static int raid1_run(struct mddev *mddev) if (!mddev_is_dm(mddev)) { ret = raid1_set_limits(mddev); if (ret) { + md_unregister_thread(mddev, &conf->thread); if (!mddev->private) raid1_free(mddev, conf); return ret; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 14dcd5142eb47..d58ae150b4502 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -3402,7 +3402,6 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, !test_bit(In_sync, &rdev->flags)) continue; /* This is where we read from */ - any_working = 1; sector = r10_bio->devs[j].addr; if (is_badblock(rdev, sector, max_sync, @@ -3417,6 +3416,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, continue; } } + any_working = 1; bio = r10_bio->devs[0].bio; bio->bi_next = biolist; biolist = bio; diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 24b32a0c95b40..3b711a1198adb 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3751,9 +3751,14 @@ static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, struct r5dev *dev = &sh->dev[disk_idx]; struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], &sh->dev[s->failed_num[1]] }; + struct mddev *mddev = sh->raid_conf->mddev; + bool force_rcw = false; int i; - bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW); + if (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW || + (mddev->bitmap_ops && mddev->bitmap_ops->blocks_synced && + !mddev->bitmap_ops->blocks_synced(mddev, sh->sector))) + force_rcw = true; if (test_bit(R5_LOCKED, &dev->flags) || test_bit(R5_UPTODATE, &dev->flags)) @@ -4956,7 +4961,8 @@ static void handle_stripe(struct stripe_head *sh) goto finish; if (s.handle_bad_blocks || - test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) { + (md_is_rdwr(conf->mddev) && + test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags))) { set_bit(STRIPE_HANDLE, &sh->state); goto finish; } @@ -6768,7 +6774,8 @@ static void raid5d(struct md_thread *thread) int batch_size, released; unsigned int offset; - if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) + if (md_is_rdwr(mddev) && + test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) break; released = release_stripe_list(conf, conf->temp_inactive_list); @@ -7185,12 +7192,14 @@ raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + conf = mddev->private; + if (!conf) { + mddev_unlock_and_resume(mddev); + return -ENODEV; + } raid5_quiesce(mddev, true); - conf = mddev->private; - if (!conf) - err = -ENODEV; - else if (new != conf->worker_cnt_per_group) { + if (new != conf->worker_cnt_per_group) { old_groups = conf->worker_groups; if (old_groups) flush_workqueue(raid5_wq); @@ -8052,7 +8061,8 @@ static int raid5_run(struct mddev *mddev) goto abort; } - if (log_init(conf, journal_dev, raid5_has_ppl(conf))) + ret = log_init(conf, journal_dev, raid5_has_ppl(conf)); + if (ret) goto abort; return 0; diff --git a/drivers/media/cec/core/cec-core.c b/drivers/media/cec/core/cec-core.c index d7259599029f8..dd6e24a0899bd 100644 --- a/drivers/media/cec/core/cec-core.c +++ b/drivers/media/cec/core/cec-core.c @@ -421,6 +421,7 @@ static int __init cec_devnode_init(void) ret = bus_register(&cec_bus_type); if (ret < 0) { + debugfs_remove_recursive(top_cec_dir); unregister_chrdev_region(cec_dev_t, CEC_NUM_DEVICES); pr_warn("cec: bus_register failed\n"); return -EIO; diff --git a/drivers/media/common/videobuf2/videobuf2-dma-contig.c b/drivers/media/common/videobuf2/videobuf2-dma-contig.c index a13ec569c82f6..7123c5fae92ce 100644 --- a/drivers/media/common/videobuf2/videobuf2-dma-contig.c +++ b/drivers/media/common/videobuf2/videobuf2-dma-contig.c @@ -258,6 +258,7 @@ static void *vb2_dc_alloc(struct vb2_buffer *vb, if (ret) { dev_err(dev, "dma alloc of size %lu failed\n", size); + put_device(buf->dev); kfree(buf); return ERR_PTR(-ENOMEM); } diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c index 151177e5a06d8..ab34be22cd1aa 100644 --- a/drivers/media/dvb-core/dmxdev.c +++ b/drivers/media/dvb-core/dmxdev.c @@ -168,7 +168,9 @@ static int dvb_dvr_open(struct inode *inode, struct file *file) mutex_unlock(&dmxdev->mutex); return -ENOMEM; } - dvb_ringbuffer_init(&dmxdev->dvr_buffer, mem, DVR_BUFFER_SIZE); + dmxdev->dvr_buffer.data = mem; + dmxdev->dvr_buffer.size = DVR_BUFFER_SIZE; + dvb_ringbuffer_reset(&dmxdev->dvr_buffer); if (dmxdev->may_do_mmap) dvb_vb2_init(&dmxdev->dvr_vb2_ctx, "dvr", file->f_flags & O_NONBLOCK); @@ -397,11 +399,11 @@ static int dvb_dmxdev_section_callback(const u8 *buffer1, size_t buffer1_len, if (dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx)) { ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx, buffer1, buffer1_len, - buffer_flags); + buffer_flags, true); if (ret == buffer1_len) ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx, buffer2, buffer2_len, - buffer_flags); + buffer_flags, true); } else { ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer, buffer1, buffer1_len); @@ -452,10 +454,10 @@ static int dvb_dmxdev_ts_callback(const u8 *buffer1, size_t buffer1_len, if (dvb_vb2_is_streaming(ctx)) { ret = dvb_vb2_fill_buffer(ctx, buffer1, buffer1_len, - buffer_flags); + buffer_flags, false); if (ret == buffer1_len) ret = dvb_vb2_fill_buffer(ctx, buffer2, buffer2_len, - buffer_flags); + buffer_flags, false); } else { if (buffer->error) { spin_unlock(&dmxdevfilter->dev->lock); diff --git a/drivers/media/dvb-core/dvb_net.c b/drivers/media/dvb-core/dvb_net.c index 8bb8dd34c223e..a2159b2bc1766 100644 --- a/drivers/media/dvb-core/dvb_net.c +++ b/drivers/media/dvb-core/dvb_net.c @@ -228,6 +228,9 @@ static int handle_one_ule_extension( struct dvb_net_priv *p ) unsigned char hlen = (p->ule_sndu_type & 0x0700) >> 8; unsigned char htype = p->ule_sndu_type & 0x00FF; + if (htype >= ARRAY_SIZE(ule_mandatory_ext_handlers)) + return -1; + /* Discriminate mandatory and optional extension headers. */ if (hlen == 0) { /* Mandatory extension header */ diff --git a/drivers/media/dvb-core/dvb_vb2.c b/drivers/media/dvb-core/dvb_vb2.c index 29edaaff7a5c9..7444bbc2f24d9 100644 --- a/drivers/media/dvb-core/dvb_vb2.c +++ b/drivers/media/dvb-core/dvb_vb2.c @@ -249,7 +249,8 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx) int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, const unsigned char *src, int len, - enum dmx_buffer_flags *buffer_flags) + enum dmx_buffer_flags *buffer_flags, + bool flush) { unsigned long flags = 0; void *vbuf = NULL; @@ -306,7 +307,7 @@ int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, } } - if (ctx->nonblocking && ctx->buf) { + if (flush && ctx->buf) { vb2_set_plane_payload(&ctx->buf->vb, 0, ll); vb2_buffer_done(&ctx->buf->vb, VB2_BUF_STATE_DONE); list_del(&ctx->buf->list); diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c index 378f4e6af12cb..5cbc973df684d 100644 --- a/drivers/media/i2c/adv7180.c +++ b/drivers/media/i2c/adv7180.c @@ -507,6 +507,13 @@ static int adv7180_get_frame_interval(struct v4l2_subdev *sd, fi->interval.denominator = 25; } + /* + * If the de-interlacer is active, the chip produces full video frames + * at the field rate. + */ + if (state->field == V4L2_FIELD_NONE) + fi->interval.denominator *= 2; + return 0; } diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c index 8fe7c2f728839..516553fb17e98 100644 --- a/drivers/media/i2c/adv7604.c +++ b/drivers/media/i2c/adv7604.c @@ -3670,7 +3670,7 @@ static int adv76xx_probe(struct i2c_client *client) err = media_entity_pads_init(&sd->entity, state->source_pad + 1, state->pads); if (err) - goto err_work_queues; + goto err_i2c; /* Configure regmaps */ err = configure_regmaps(state); @@ -3711,8 +3711,6 @@ static int adv76xx_probe(struct i2c_client *client) err_entity: media_entity_cleanup(&sd->entity); -err_work_queues: - cancel_delayed_work(&state->delayed_work_enable_hotplug); err_i2c: adv76xx_unregister_clients(state); err_hdl: diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c index 9780082db8415..21c3d3682e0b4 100644 --- a/drivers/media/i2c/adv7842.c +++ b/drivers/media/i2c/adv7842.c @@ -2699,6 +2699,7 @@ static int adv7842_cp_log_status(struct v4l2_subdev *sd) /* CP block */ struct adv7842_state *state = to_state(sd); struct v4l2_dv_timings timings; + int temp; u8 reg_io_0x02 = io_read(sd, 0x02); u8 reg_io_0x21 = io_read(sd, 0x21); u8 reg_rep_0x77 = rep_read(sd, 0x77); @@ -2821,8 +2822,9 @@ static int adv7842_cp_log_status(struct v4l2_subdev *sd) (((reg_io_0x02 >> 2) & 0x01) ^ (reg_io_0x02 & 0x01)) ? "(16-235)" : "(0-255)", (reg_io_0x02 & 0x08) ? "enabled" : "disabled"); + temp = cp_read(sd, 0xf4) >> 4; v4l2_info(sd, "Color space conversion: %s\n", - csc_coeff_sel_rb[cp_read(sd, 0xf4) >> 4]); + temp < 0 ? "" : csc_coeff_sel_rb[temp]); if (!is_digital_input(sd)) return 0; @@ -2852,8 +2854,9 @@ static int adv7842_cp_log_status(struct v4l2_subdev *sd) hdmi_read(sd, 0x5f)); v4l2_info(sd, "AV Mute: %s\n", (hdmi_read(sd, 0x04) & 0x40) ? "on" : "off"); + temp = hdmi_read(sd, 0x0b) >> 6; v4l2_info(sd, "Deep color mode: %s\n", - deep_color_mode_txt[hdmi_read(sd, 0x0b) >> 6]); + temp < 0 ? "" : deep_color_mode_txt[temp]); adv7842_log_infoframes(sd); @@ -3626,7 +3629,7 @@ static int adv7842_probe(struct i2c_client *client) err = media_entity_pads_init(&sd->entity, ADV7842_PAD_SOURCE + 1, state->pads); if (err) - goto err_work_queues; + goto err_i2c; err = adv7842_core_init(sd); if (err) @@ -3647,8 +3650,6 @@ static int adv7842_probe(struct i2c_client *client) err_entity: media_entity_cleanup(&sd->entity); -err_work_queues: - cancel_delayed_work(&state->delayed_work_enable_hotplug); err_i2c: adv7842_unregister_clients(sd); err_hdl: diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index 1c889c878abd3..3f7162f3d1e32 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -2346,7 +2346,7 @@ static void ccs_set_compose_scaler(struct v4l2_subdev *subdev, * CCS_LIM(sensor, SCALER_N_MIN) / sel->r.height; max_m = crops[CCS_PAD_SINK]->width * CCS_LIM(sensor, SCALER_N_MIN) - / CCS_LIM(sensor, MIN_X_OUTPUT_SIZE); + / (CCS_LIM(sensor, MIN_X_OUTPUT_SIZE) ?: 1); a = clamp(a, CCS_LIM(sensor, SCALER_M_MIN), CCS_LIM(sensor, SCALER_M_MAX)); @@ -2940,6 +2940,8 @@ static void ccs_cleanup(struct ccs_sensor *sensor) ccs_free_controls(sensor); } +static const struct v4l2_subdev_internal_ops ccs_internal_ops; + static int ccs_init_subdev(struct ccs_sensor *sensor, struct ccs_subdev *ssd, const char *name, unsigned short num_pads, u32 function, @@ -2952,8 +2954,10 @@ static int ccs_init_subdev(struct ccs_sensor *sensor, if (!ssd) return 0; - if (ssd != sensor->src) + if (ssd != sensor->src) { v4l2_subdev_init(&ssd->sd, &ccs_ops); + ssd->sd.internal_ops = &ccs_internal_ops; + } ssd->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; ssd->sd.entity.function = function; @@ -3062,6 +3066,10 @@ static const struct media_entity_operations ccs_entity_ops = { .link_validate = v4l2_subdev_link_validate, }; +static const struct v4l2_subdev_internal_ops ccs_internal_ops = { + .init_state = ccs_init_state, +}; + static const struct v4l2_subdev_internal_ops ccs_internal_src_ops = { .init_state = ccs_init_state, .registered = ccs_registered, @@ -3425,7 +3433,21 @@ static int ccs_probe(struct i2c_client *client) sensor->scale_m = CCS_LIM(sensor, SCALER_N_MIN); /* prepare PLL configuration input values */ - sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + switch (sensor->hwcfg.csi_signalling_mode) { + case CCS_CSI_SIGNALING_MODE_CSI_2_CPHY: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_CPHY; + break; + case CCS_CSI_SIGNALING_MODE_CSI_2_DPHY: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_CLOCK: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_STROBE: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + break; + default: + dev_err(&client->dev, "unsupported signalling mode %u\n", + sensor->hwcfg.csi_signalling_mode); + rval = -EINVAL; + goto out_cleanup; + } sensor->pll.csi2.lanes = sensor->hwcfg.lanes; if (CCS_LIM(sensor, CLOCK_CALCULATION) & CCS_CLOCK_CALCULATION_LANE_SPEED) { diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c index 1e7ad355a388c..3288de539452e 100644 --- a/drivers/media/i2c/dw9714.c +++ b/drivers/media/i2c/dw9714.c @@ -149,7 +149,7 @@ static int dw9714_power_up(struct dw9714_device *dw9714_dev) gpiod_set_value_cansleep(dw9714_dev->powerdown_gpio, 0); - usleep_range(1000, 2000); + usleep_range(12000, 14000); return 0; } diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c index c680aa6c3a55a..300935b1ef249 100644 --- a/drivers/media/i2c/imx219.c +++ b/drivers/media/i2c/imx219.c @@ -856,7 +856,7 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd, const struct imx219_mode *mode; struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; - u8 bin_h, bin_v; + u8 bin_h, bin_v, binning; u32 prev_line_len; format = v4l2_subdev_state_get_format(state, 0); @@ -877,9 +877,12 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd, bin_h = min(IMX219_PIXEL_ARRAY_WIDTH / format->width, 2U); bin_v = min(IMX219_PIXEL_ARRAY_HEIGHT / format->height, 2U); + /* Ensure bin_h and bin_v are same to avoid 1:2 or 2:1 stretching */ + binning = min(bin_h, bin_v); + crop = v4l2_subdev_state_get_crop(state, 0); - crop->width = format->width * bin_h; - crop->height = format->height * bin_v; + crop->width = format->width * binning; + crop->height = format->height * binning; crop->left = (IMX219_NATIVE_WIDTH - crop->width) / 2; crop->top = (IMX219_NATIVE_HEIGHT - crop->height) / 2; diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c index 7bbd639a9ddfa..e25e0a9ff65c3 100644 --- a/drivers/media/i2c/imx412.c +++ b/drivers/media/i2c/imx412.c @@ -925,10 +925,10 @@ static int imx412_parse_hw_config(struct imx412 *imx412) /* Request optional reset pin */ imx412->reset_gpio = devm_gpiod_get_optional(imx412->dev, "reset", - GPIOD_OUT_LOW); + GPIOD_OUT_HIGH); if (IS_ERR(imx412->reset_gpio)) { - dev_err(imx412->dev, "failed to get reset gpio %ld\n", - PTR_ERR(imx412->reset_gpio)); + dev_err(imx412->dev, "failed to get reset gpio %pe\n", + imx412->reset_gpio); return PTR_ERR(imx412->reset_gpio); } @@ -1037,7 +1037,11 @@ static int imx412_power_on(struct device *dev) goto error_reset; } - usleep_range(1000, 1200); + /* + * Certain Arducam IMX577 module variants require a longer reset settle + * time. Increasing the delay from 1ms to 10ms ensures reliable startup. + */ + usleep_range(10000, 12000); return 0; diff --git a/drivers/media/i2c/msp3400-kthreads.c b/drivers/media/i2c/msp3400-kthreads.c index ecabc0e1d32e6..1d9f41dd7c212 100644 --- a/drivers/media/i2c/msp3400-kthreads.c +++ b/drivers/media/i2c/msp3400-kthreads.c @@ -596,6 +596,8 @@ int msp3400c_thread(void *data) "carrier2 val: %5d / %s\n", val, cd[i].name); } + if (max1 < 0 || max1 > 3) + goto restart; /* program the msp3400 according to the results */ state->main = msp3400c_carrier_detect_main[max1].cdo; switch (max1) { diff --git a/drivers/media/i2c/mt9m114.c b/drivers/media/i2c/mt9m114.c index 51ebbe7ae9969..b1325e2cd1321 100644 --- a/drivers/media/i2c/mt9m114.c +++ b/drivers/media/i2c/mt9m114.c @@ -2360,11 +2360,17 @@ static int mt9m114_parse_dt(struct mt9m114 *sensor) struct fwnode_handle *ep; int ret; + /* + * On ACPI systems the fwnode graph can be initialized by a bridge + * driver, which may not have probed yet. Wait for this. + * + * TODO: Return an error once bridge driver code will have moved + * to the ACPI core. + */ ep = fwnode_graph_get_next_endpoint(fwnode, NULL); - if (!ep) { - dev_err(&sensor->client->dev, "No endpoint found\n"); - return -EINVAL; - } + if (!ep) + return dev_err_probe(&sensor->client->dev, -EPROBE_DEFER, + "waiting for fwnode graph endpoint\n"); sensor->bus_cfg.bus_type = V4L2_MBUS_UNKNOWN; ret = v4l2_fwnode_endpoint_alloc_parse(ep, &sensor->bus_cfg); @@ -2434,7 +2440,7 @@ static int mt9m114_probe(struct i2c_client *client) goto error_ep_free; } - sensor->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + sensor->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(sensor->reset)) { ret = PTR_ERR(sensor->reset); dev_err_probe(dev, ret, "Failed to get reset GPIO\n"); diff --git a/drivers/media/i2c/ov01a10.c b/drivers/media/i2c/ov01a10.c index 141cb6f75b555..c1a7373a6311c 100644 --- a/drivers/media/i2c/ov01a10.c +++ b/drivers/media/i2c/ov01a10.c @@ -16,7 +16,7 @@ #include #define OV01A10_LINK_FREQ_400MHZ 400000000ULL -#define OV01A10_SCLK 40000000LL +#define OV01A10_SCLK 80000000LL #define OV01A10_DATA_LANES 1 #define OV01A10_REG_CHIP_ID 0x300a @@ -48,7 +48,7 @@ /* analog gain controls */ #define OV01A10_REG_ANALOG_GAIN 0x3508 #define OV01A10_ANAL_GAIN_MIN 0x100 -#define OV01A10_ANAL_GAIN_MAX 0xffff +#define OV01A10_ANAL_GAIN_MAX 0x3fff #define OV01A10_ANAL_GAIN_STEP 1 /* digital gain controls */ @@ -57,7 +57,7 @@ #define OV01A10_REG_DIGITAL_GAIN_GR 0x3513 #define OV01A10_REG_DIGITAL_GAIN_R 0x3516 #define OV01A10_DGTL_GAIN_MIN 0 -#define OV01A10_DGTL_GAIN_MAX 0x3ffff +#define OV01A10_DGTL_GAIN_MAX 0x3fff #define OV01A10_DGTL_GAIN_STEP 1 #define OV01A10_DGTL_GAIN_DEFAULT 1024 @@ -75,6 +75,15 @@ #define OV01A10_REG_X_WIN 0x3811 #define OV01A10_REG_Y_WIN 0x3813 +/* + * The native ov01a10 bayer-pattern is GBRG, but there was a driver bug enabling + * hflip/mirroring by default resulting in BGGR. Because of this bug Intel's + * proprietary IPU6 userspace stack expects BGGR. So we report BGGR to not break + * userspace and fix things up by shifting the crop window-x coordinate by 1 + * when hflip is *disabled*. + */ +#define OV01A10_MEDIA_BUS_FMT MEDIA_BUS_FMT_SBGGR10_1X10 + struct ov01a10_reg { u16 address; u8 val; @@ -185,14 +194,14 @@ static const struct ov01a10_reg sensor_1280x800_setting[] = { {0x380e, 0x03}, {0x380f, 0x80}, {0x3810, 0x00}, - {0x3811, 0x08}, + {0x3811, 0x09}, {0x3812, 0x00}, {0x3813, 0x08}, {0x3814, 0x01}, {0x3815, 0x01}, {0x3816, 0x01}, {0x3817, 0x01}, - {0x3820, 0xa0}, + {0x3820, 0xa8}, {0x3822, 0x13}, {0x3832, 0x28}, {0x3833, 0x10}, @@ -240,9 +249,8 @@ static const struct ov01a10_reg sensor_1280x800_setting[] = { static const char * const ov01a10_test_pattern_menu[] = { "Disabled", "Color Bar", - "Top-Bottom Darker Color Bar", - "Right-Left Darker Color Bar", - "Color Bar type 4", + "Left-Right Darker Color Bar", + "Bottom-Top Darker Color Bar", }; static const s64 link_freq_menu_items[] = { @@ -397,10 +405,8 @@ static int ov01a10_update_digital_gain(struct ov01a10 *ov01a10, u32 d_gain) static int ov01a10_test_pattern(struct ov01a10 *ov01a10, u32 pattern) { - if (!pattern) - return 0; - - pattern = (pattern - 1) | OV01A10_TEST_PATTERN_ENABLE; + if (pattern) + pattern |= OV01A10_TEST_PATTERN_ENABLE; return ov01a10_write_reg(ov01a10, OV01A10_REG_TEST_PATTERN, 1, pattern); } @@ -411,7 +417,7 @@ static int ov01a10_set_hflip(struct ov01a10 *ov01a10, u32 hflip) int ret; u32 val, offset; - offset = hflip ? 0x9 : 0x8; + offset = hflip ? 0x8 : 0x9; ret = ov01a10_write_reg(ov01a10, OV01A10_REG_X_WIN, 1, offset); if (ret) return ret; @@ -420,8 +426,8 @@ static int ov01a10_set_hflip(struct ov01a10 *ov01a10, u32 hflip) if (ret) return ret; - val = hflip ? val | FIELD_PREP(OV01A10_HFLIP_MASK, 0x1) : - val & ~OV01A10_HFLIP_MASK; + val = hflip ? val & ~OV01A10_HFLIP_MASK : + val | FIELD_PREP(OV01A10_HFLIP_MASK, 0x1); return ov01a10_write_reg(ov01a10, OV01A10_REG_FORMAT1, 1, val); } @@ -610,7 +616,7 @@ static void ov01a10_update_pad_format(const struct ov01a10_mode *mode, { fmt->width = mode->width; fmt->height = mode->height; - fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10; + fmt->code = OV01A10_MEDIA_BUS_FMT; fmt->field = V4L2_FIELD_NONE; fmt->colorspace = V4L2_COLORSPACE_RAW; } @@ -722,7 +728,7 @@ static int ov01a10_set_format(struct v4l2_subdev *sd, h_blank); } - format = v4l2_subdev_state_get_format(sd_state, fmt->stream); + format = v4l2_subdev_state_get_format(sd_state, fmt->pad); *format = fmt->format; return 0; @@ -751,7 +757,7 @@ static int ov01a10_enum_mbus_code(struct v4l2_subdev *sd, if (code->index > 0) return -EINVAL; - code->code = MEDIA_BUS_FMT_SBGGR10_1X10; + code->code = OV01A10_MEDIA_BUS_FMT; return 0; } @@ -761,7 +767,7 @@ static int ov01a10_enum_frame_size(struct v4l2_subdev *sd, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes) || - fse->code != MEDIA_BUS_FMT_SBGGR10_1X10) + fse->code != OV01A10_MEDIA_BUS_FMT) return -EINVAL; fse->min_width = supported_modes[fse->index].width; @@ -855,6 +861,7 @@ static void ov01a10_remove(struct i2c_client *client) struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_async_unregister_subdev(sd); + v4l2_subdev_cleanup(sd); media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(sd->ctrl_handler); @@ -925,6 +932,7 @@ static int ov01a10_probe(struct i2c_client *client) err_pm_disable: pm_runtime_disable(dev); pm_runtime_set_suspended(&client->dev); + v4l2_subdev_cleanup(&ov01a10->sd); err_media_entity_cleanup: media_entity_cleanup(&ov01a10->sd.entity); diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c index e193fef4fcedf..5fb10e02ba6e2 100644 --- a/drivers/media/i2c/ov5647.c +++ b/drivers/media/i2c/ov5647.c @@ -69,11 +69,11 @@ #define OV5647_NATIVE_HEIGHT 1956U #define OV5647_PIXEL_ARRAY_LEFT 16U -#define OV5647_PIXEL_ARRAY_TOP 16U +#define OV5647_PIXEL_ARRAY_TOP 6U #define OV5647_PIXEL_ARRAY_WIDTH 2592U #define OV5647_PIXEL_ARRAY_HEIGHT 1944U -#define OV5647_VBLANK_MIN 4 +#define OV5647_VBLANK_MIN 24 #define OV5647_VTS_MAX 32767 #define OV5647_EXPOSURE_MIN 4 @@ -508,7 +508,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 2592, .height = 1944 @@ -529,7 +529,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 1920, .height = 1080 @@ -550,7 +550,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 1296, .height = 972 @@ -571,7 +571,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 640, .height = 480 @@ -582,7 +582,7 @@ static const struct ov5647_mode ov5647_modes[] = { .width = 2560, .height = 1920, }, - .pixel_rate = 55000000, + .pixel_rate = 58333000, .hts = 1852, .vts = 0x1f8, .reg_list = ov5647_640x480_10bpp, @@ -1291,6 +1291,8 @@ static int ov5647_init_controls(struct ov5647 *sensor) v4l2_ctrl_handler_init(&sensor->ctrls, 9); + sensor->ctrls.lock = &sensor->lock; + v4l2_ctrl_new_std(&sensor->ctrls, &ov5647_ctrl_ops, V4L2_CID_AUTOGAIN, 0, 1, 1, 0); @@ -1420,15 +1422,15 @@ static int ov5647_probe(struct i2c_client *client) sensor->mode = OV5647_DEFAULT_MODE; - ret = ov5647_init_controls(sensor); - if (ret) - goto mutex_destroy; - sd = &sensor->sd; v4l2_i2c_subdev_init(sd, client, &ov5647_subdev_ops); sd->internal_ops = &ov5647_subdev_internal_ops; sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS; + ret = ov5647_init_controls(sensor); + if (ret) + goto mutex_destroy; + sensor->pad.flags = MEDIA_PAD_FL_SOURCE; sd->entity.function = MEDIA_ENT_F_CAM_SENSOR; ret = media_entity_pads_init(&sd->entity, 1, &sensor->pad); diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c index 1087d2bddaf29..3532766cd795d 100644 --- a/drivers/media/i2c/tda1997x.c +++ b/drivers/media/i2c/tda1997x.c @@ -2797,7 +2797,6 @@ static int tda1997x_probe(struct i2c_client *client) err_free_handler: v4l2_ctrl_handler_free(&state->hdl); err_free_mutex: - cancel_delayed_work(&state->delayed_work_enable_hpd); mutex_destroy(&state->page_lock); mutex_destroy(&state->lock); tda1997x_set_power(state, 0); diff --git a/drivers/media/i2c/tw9903.c b/drivers/media/i2c/tw9903.c index b996a05e56f28..c3eafd5d5dc82 100644 --- a/drivers/media/i2c/tw9903.c +++ b/drivers/media/i2c/tw9903.c @@ -228,6 +228,7 @@ static int tw9903_probe(struct i2c_client *client) if (write_regs(sd, initial_registers) < 0) { v4l2_err(client, "error initializing TW9903\n"); + v4l2_ctrl_handler_free(hdl); return -EINVAL; } diff --git a/drivers/media/i2c/tw9906.c b/drivers/media/i2c/tw9906.c index 6220f4fddbabc..0ab43fe42d7f4 100644 --- a/drivers/media/i2c/tw9906.c +++ b/drivers/media/i2c/tw9906.c @@ -196,6 +196,7 @@ static int tw9906_probe(struct i2c_client *client) if (write_regs(sd, initial_registers) < 0) { v4l2_err(client, "error initializing TW9906\n"); + v4l2_ctrl_handler_free(hdl); return -EINVAL; } diff --git a/drivers/media/mc/mc-request.c b/drivers/media/mc/mc-request.c index f66f728b1b43d..4188d4c8f1d44 100644 --- a/drivers/media/mc/mc-request.c +++ b/drivers/media/mc/mc-request.c @@ -190,6 +190,8 @@ static long media_request_ioctl_reinit(struct media_request *req) struct media_device *mdev = req->mdev; unsigned long flags; + mutex_lock(&mdev->req_queue_mutex); + spin_lock_irqsave(&req->lock, flags); if (req->state != MEDIA_REQUEST_STATE_IDLE && req->state != MEDIA_REQUEST_STATE_COMPLETE) { @@ -197,6 +199,7 @@ static long media_request_ioctl_reinit(struct media_request *req) "request: %s not in idle or complete state, cannot reinit\n", req->debug_str); spin_unlock_irqrestore(&req->lock, flags); + mutex_unlock(&mdev->req_queue_mutex); return -EBUSY; } if (req->access_count) { @@ -204,6 +207,7 @@ static long media_request_ioctl_reinit(struct media_request *req) "request: %s is being accessed, cannot reinit\n", req->debug_str); spin_unlock_irqrestore(&req->lock, flags); + mutex_unlock(&mdev->req_queue_mutex); return -EBUSY; } req->state = MEDIA_REQUEST_STATE_CLEANING; @@ -214,6 +218,7 @@ static long media_request_ioctl_reinit(struct media_request *req) spin_lock_irqsave(&req->lock, flags); req->state = MEDIA_REQUEST_STATE_IDLE; spin_unlock_irqrestore(&req->lock, flags); + mutex_unlock(&mdev->req_queue_mutex); return 0; } diff --git a/drivers/media/pci/cx23885/cx23885-alsa.c b/drivers/media/pci/cx23885/cx23885-alsa.c index 25dc8d4dc5b73..717fc6c9ef21f 100644 --- a/drivers/media/pci/cx23885/cx23885-alsa.c +++ b/drivers/media/pci/cx23885/cx23885-alsa.c @@ -392,8 +392,10 @@ static int snd_cx23885_hw_params(struct snd_pcm_substream *substream, ret = cx23885_risc_databuffer(chip->pci, &buf->risc, buf->sglist, chip->period_size, chip->num_periods, 1); - if (ret < 0) + if (ret < 0) { + cx23885_alsa_dma_unmap(chip); goto error; + } /* Loop back to start of program */ buf->risc.jmp[0] = cpu_to_le32(RISC_JUMP|RISC_IRQ1|RISC_CNT_INC); diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c index a42f0c03a7ca8..f463365163b7e 100644 --- a/drivers/media/pci/cx25821/cx25821-alsa.c +++ b/drivers/media/pci/cx25821/cx25821-alsa.c @@ -535,6 +535,7 @@ static int snd_cx25821_hw_params(struct snd_pcm_substream *substream, chip->period_size, chip->num_periods, 1); if (ret < 0) { pr_info("DEBUG: ERROR after cx25821_risc_databuffer_audio()\n"); + cx25821_alsa_dma_unmap(chip); goto error; } diff --git a/drivers/media/pci/cx25821/cx25821-core.c b/drivers/media/pci/cx25821/cx25821-core.c index 6627fa9166d30..a7336be444748 100644 --- a/drivers/media/pci/cx25821/cx25821-core.c +++ b/drivers/media/pci/cx25821/cx25821-core.c @@ -908,6 +908,7 @@ static int cx25821_dev_setup(struct cx25821_dev *dev) if (!dev->lmmio) { CX25821_ERR("ioremap failed, maybe increasing __VMALLOC_RESERVE in page.h\n"); + release_mem_region(dev->base_io_addr, pci_resource_len(dev->pci, 0)); cx25821_iounmap(dev); return -ENOMEM; } diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c index 29fb1311e4434..4e574d8390b4d 100644 --- a/drivers/media/pci/cx88/cx88-alsa.c +++ b/drivers/media/pci/cx88/cx88-alsa.c @@ -483,8 +483,10 @@ static int snd_cx88_hw_params(struct snd_pcm_substream *substream, ret = cx88_risc_databuffer(chip->pci, &buf->risc, buf->sglist, chip->period_size, chip->num_periods, 1); - if (ret < 0) + if (ret < 0) { + cx88_alsa_dma_unmap(chip); goto error; + } /* Loop back to start of program */ buf->risc.jmp[0] = cpu_to_le32(RISC_JUMP | RISC_IRQ1 | RISC_CNT_INC); diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c index aa2cf7287477c..8f05987cdb4e7 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c @@ -3,6 +3,7 @@ * Copyright (C) 2013--2024 Intel Corporation */ #include +#include #include #include #include @@ -201,6 +202,8 @@ static int buffer_list_get(struct ipu6_isys_stream *stream, unsigned long flags; unsigned long buf_flag = IPU6_ISYS_BUFFER_LIST_FL_INCOMING; + lockdep_assert_held(&stream->mutex); + bl->nbufs = 0; INIT_LIST_HEAD(&bl->head); @@ -294,9 +297,8 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av, struct ipu6_isys_buffer_list __bl; int ret; - mutex_lock(&stream->isys->stream_mutex); + guard(mutex)(&stream->isys->stream_mutex); ret = ipu6_isys_video_set_streaming(av, 1, bl); - mutex_unlock(&stream->isys->stream_mutex); if (ret) goto out_requeue; @@ -637,10 +639,10 @@ static void stop_streaming(struct vb2_queue *q) mutex_lock(&av->isys->stream_mutex); if (stream->nr_streaming == stream->nr_queues && stream->streaming) ipu6_isys_video_set_streaming(av, 0, NULL); + list_del(&aq->node); mutex_unlock(&av->isys->stream_mutex); stream->nr_streaming--; - list_del(&aq->node); stream->streaming = 0; mutex_unlock(&stream->mutex); diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c index f3f3bc0615e5d..b96ae563b82f0 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c @@ -1022,11 +1022,10 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state, sd->name, r_pad->index, stream_mask); ret = v4l2_subdev_disable_streams(sd, r_pad->index, stream_mask); - if (ret) { + if (ret) dev_err(dev, "stream off %s failed with %d\n", sd->name, ret); - return ret; - } + close_streaming_firmware(av); } else { ret = start_stream_firmware(av, bl); @@ -1052,6 +1051,7 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state, out_media_entity_stop_streaming_firmware: stop_streaming_firmware(av); + close_streaming_firmware(av); return ret; } diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu.c b/drivers/media/pci/intel/ipu6/ipu6-mmu.c index 6d1c0b90169d4..85cc6d5b4dd11 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-mmu.c +++ b/drivers/media/pci/intel/ipu6/ipu6-mmu.c @@ -102,7 +102,7 @@ static void page_table_dump(struct ipu6_mmu_info *mmu_info) if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) continue; - l2_phys = TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx];) + l2_phys = TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]); dev_dbg(mmu_info->dev, "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pap\n", l1_idx, iova, iova + ISP_PAGE_SIZE, &l2_phys); @@ -248,7 +248,7 @@ static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info) dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt); - for (i = 0; i < ISP_L1PT_PTES; i++) + for (i = 0; i < ISP_L2PT_PTES; i++) pt[i] = mmu_info->dummy_page_pteval; return pt; diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c index 1f4f20b9c94dc..a2768f44017a5 100644 --- a/drivers/media/pci/intel/ipu6/ipu6.c +++ b/drivers/media/pci/intel/ipu6/ipu6.c @@ -630,21 +630,21 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) { dev_err_probe(&isp->pdev->dev, ret, "Failed to set MMU hardware\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw, &isp->psys->fw_sgt); if (ret) { dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data); if (ret) { dev_err_probe(&isp->pdev->dev, ret, "failed to create pkg dir\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr, @@ -652,7 +652,7 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) IRQF_SHARED, IPU6_NAME, isp); if (ret) { dev_err_probe(dev, ret, "Requesting irq failed\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_buttress_authenticate(isp); @@ -683,6 +683,8 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) out_free_irq: devm_free_irq(dev, pdev->irq, isp); +out_ipu6_rpm_put: + pm_runtime_put_sync(&isp->psys->auxdev.dev); out_ipu6_bus_del_devices: if (isp->psys) { ipu6_cpd_free_pkg_dir(isp->psys); diff --git a/drivers/media/pci/mgb4/mgb4_trigger.c b/drivers/media/pci/mgb4/mgb4_trigger.c index d7dddc5c8728e..10c23f0c833d5 100644 --- a/drivers/media/pci/mgb4/mgb4_trigger.c +++ b/drivers/media/pci/mgb4/mgb4_trigger.c @@ -114,7 +114,7 @@ static int probe_trigger(struct iio_dev *indio_dev, int irq) if (!st->trig) return -ENOMEM; - ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, 0, + ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, IRQF_NO_THREAD, "mgb4-trigger", st->trig); if (ret) goto error_free_trig; diff --git a/drivers/media/pci/solo6x10/solo6x10-tw28.c b/drivers/media/pci/solo6x10/solo6x10-tw28.c index 1b7c22a9bc94f..8f53946c67928 100644 --- a/drivers/media/pci/solo6x10/solo6x10-tw28.c +++ b/drivers/media/pci/solo6x10/solo6x10-tw28.c @@ -166,7 +166,7 @@ static const u8 tbl_tw2865_pal_template[] = { 0x64, 0x51, 0x40, 0xaf, 0xFF, 0xF0, 0x00, 0xC0, }; -#define is_tw286x(__solo, __id) (!(__solo->tw2815 & (1 << __id))) +#define is_tw286x(__solo, __id) (!((__solo)->tw2815 & (1U << (__id)))) static u8 tw_readbyte(struct solo_dev *solo_dev, int chip_id, u8 tw6x_off, u8 tw_off) @@ -686,6 +686,9 @@ int tw28_set_ctrl_val(struct solo_dev *solo_dev, u32 ctrl, u8 ch, chip_num = ch / 4; ch %= 4; + if (chip_num >= TW_NUM_CHIP) + return -EINVAL; + if (val > 255 || val < 0) return -ERANGE; @@ -758,6 +761,9 @@ int tw28_get_ctrl_val(struct solo_dev *solo_dev, u32 ctrl, u8 ch, chip_num = ch / 4; ch %= 4; + if (chip_num >= TW_NUM_CHIP) + return -EINVAL; + switch (ctrl) { case V4L2_CID_SHARPNESS: /* Only 286x has sharpness */ diff --git a/drivers/media/platform/amphion/vdec.c b/drivers/media/platform/amphion/vdec.c index 32eef2fd1f2a9..6b63e8ca0cb40 100644 --- a/drivers/media/platform/amphion/vdec.c +++ b/drivers/media/platform/amphion/vdec.c @@ -726,6 +726,7 @@ static int vdec_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder_cmd switch (cmd->cmd) { case V4L2_DEC_CMD_START: vdec_cmd_start(inst); + vb2_clear_last_buffer_dequeued(v4l2_m2m_get_dst_vq(inst->fh.m2m_ctx)); break; case V4L2_DEC_CMD_STOP: vdec_cmd_stop(inst); diff --git a/drivers/media/platform/amphion/vpu_core.c b/drivers/media/platform/amphion/vpu_core.c index da00f5fc0e5da..168f0514851e4 100644 --- a/drivers/media/platform/amphion/vpu_core.c +++ b/drivers/media/platform/amphion/vpu_core.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include @@ -542,47 +542,30 @@ const struct vpu_core_resources *vpu_get_resource(struct vpu_inst *inst) static int vpu_core_parse_dt(struct vpu_core *core, struct device_node *np) { - struct device_node *node; struct resource res; int ret; - if (of_count_phandle_with_args(np, "memory-region", NULL) < 2) { - dev_err(core->dev, "need 2 memory-region for boot and rpc\n"); - return -ENODEV; + ret = of_reserved_mem_region_to_resource(np, 0, &res); + if (ret) { + dev_err(core->dev, "Cannot get boot-region\n"); + return ret; } - node = of_parse_phandle(np, "memory-region", 0); - if (!node) { - dev_err(core->dev, "boot-region of_parse_phandle error\n"); - return -ENODEV; - } - if (of_address_to_resource(node, 0, &res)) { - dev_err(core->dev, "boot-region of_address_to_resource error\n"); - of_node_put(node); - return -EINVAL; - } core->fw.phys = res.start; core->fw.length = resource_size(&res); - of_node_put(node); - - node = of_parse_phandle(np, "memory-region", 1); - if (!node) { - dev_err(core->dev, "rpc-region of_parse_phandle error\n"); - return -ENODEV; - } - if (of_address_to_resource(node, 0, &res)) { - dev_err(core->dev, "rpc-region of_address_to_resource error\n"); - of_node_put(node); - return -EINVAL; + ret = of_reserved_mem_region_to_resource(np, 1, &res); + if (ret) { + dev_err(core->dev, "Cannot get rpc-region\n"); + return ret; } + core->rpc.phys = res.start; core->rpc.length = resource_size(&res); if (core->rpc.length < core->res->rpc_size + core->res->fwlog_size) { dev_err(core->dev, "the rpc-region <%pad, 0x%x> is not enough\n", &core->rpc.phys, core->rpc.length); - of_node_put(node); return -EINVAL; } @@ -594,7 +577,6 @@ static int vpu_core_parse_dt(struct vpu_core *core, struct device_node *np) if (ret != VPU_CORE_MEMORY_UNCACHED) { dev_err(core->dev, "rpc region<%pad, 0x%x> isn't uncached\n", &core->rpc.phys, core->rpc.length); - of_node_put(node); return -EINVAL; } @@ -606,8 +588,6 @@ static int vpu_core_parse_dt(struct vpu_core *core, struct device_node *np) core->act.length = core->rpc.length - core->res->rpc_size - core->log.length; core->rpc.length = core->res->rpc_size; - of_node_put(node); - return 0; } diff --git a/drivers/media/platform/amphion/vpu_malone.c b/drivers/media/platform/amphion/vpu_malone.c index ba688566dffd0..80802975c4f16 100644 --- a/drivers/media/platform/amphion/vpu_malone.c +++ b/drivers/media/platform/amphion/vpu_malone.c @@ -1337,22 +1337,18 @@ static int vpu_malone_insert_scode_vc1_g_seq(struct malone_scode_t *scode) { if (!scode->inst->total_input_count) return 0; - if (vpu_vb_is_codecconfig(to_vb2_v4l2_buffer(scode->vb))) - scode->need_data = 0; return 0; } static int vpu_malone_insert_scode_vc1_g_pic(struct malone_scode_t *scode) { - struct vb2_v4l2_buffer *vbuf; u8 nal_hdr[MALONE_VC1_NAL_HEADER_LEN]; u32 *data = NULL; int ret; - vbuf = to_vb2_v4l2_buffer(scode->vb); data = vb2_plane_vaddr(scode->vb, 0); - if (scode->inst->total_input_count == 0 || vpu_vb_is_codecconfig(vbuf)) + if (scode->inst->total_input_count == 0) return 0; if (MALONE_VC1_CONTAIN_NAL(*data)) return 0; @@ -1373,8 +1369,6 @@ static int vpu_malone_insert_scode_vc1_l_seq(struct malone_scode_t *scode) int size = 0; u8 rcv_seqhdr[MALONE_VC1_RCV_SEQ_HEADER_LEN]; - if (vpu_vb_is_codecconfig(to_vb2_v4l2_buffer(scode->vb))) - scode->need_data = 0; if (scode->inst->total_input_count) return 0; scode->need_data = 0; @@ -1560,7 +1554,7 @@ static int vpu_malone_input_frame_data(struct vpu_malone_str_buffer __iomem *str scode.vb = vb; scode.wptr = wptr; scode.need_data = 1; - if (vbuf->sequence == 0 || vpu_vb_is_codecconfig(vbuf)) + if (vbuf->sequence == 0) ret = vpu_malone_insert_scode(&scode, SCODE_SEQUENCE); if (ret < 0) @@ -1596,7 +1590,7 @@ static int vpu_malone_input_frame_data(struct vpu_malone_str_buffer __iomem *str * This module is currently only supported for the H264 and HEVC formats, * for other formats, vpu_malone_add_scode() will return 0. */ - if ((disp_imm || low_latency) && !vpu_vb_is_codecconfig(vbuf)) { + if (disp_imm || low_latency) { ret = vpu_malone_add_scode(inst->core->iface, inst->id, &inst->stream_buffer, @@ -1643,7 +1637,6 @@ int vpu_malone_input_frame(struct vpu_shared_addr *shared, struct vpu_inst *inst, struct vb2_buffer *vb) { struct vpu_dec_ctrl *hc = shared->priv; - struct vb2_v4l2_buffer *vbuf; struct vpu_malone_str_buffer __iomem *str_buf = hc->str_buf[inst->id]; u32 disp_imm = hc->codec_param[inst->id].disp_imm; u32 size; @@ -1657,16 +1650,6 @@ int vpu_malone_input_frame(struct vpu_shared_addr *shared, return ret; size = ret; - /* - * if buffer only contain codec data, and the timestamp is invalid, - * don't put the invalid timestamp to resync - * merge the data to next frame - */ - vbuf = to_vb2_v4l2_buffer(vb); - if (vpu_vb_is_codecconfig(vbuf)) { - inst->extra_size += size; - return 0; - } if (inst->extra_size) { size += inst->extra_size; inst->extra_size = 0; diff --git a/drivers/media/platform/amphion/vpu_v4l2.c b/drivers/media/platform/amphion/vpu_v4l2.c index fcb2eff813ac4..1fb887b9098c6 100644 --- a/drivers/media/platform/amphion/vpu_v4l2.c +++ b/drivers/media/platform/amphion/vpu_v4l2.c @@ -349,16 +349,6 @@ struct vb2_v4l2_buffer *vpu_next_src_buf(struct vpu_inst *inst) if (!src_buf || vpu_get_buffer_state(src_buf) == VPU_BUF_STATE_IDLE) return NULL; - while (vpu_vb_is_codecconfig(src_buf)) { - v4l2_m2m_src_buf_remove(inst->fh.m2m_ctx); - vpu_set_buffer_state(src_buf, VPU_BUF_STATE_IDLE); - v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE); - - src_buf = v4l2_m2m_next_src_buf(inst->fh.m2m_ctx); - if (!src_buf || vpu_get_buffer_state(src_buf) == VPU_BUF_STATE_IDLE) - return NULL; - } - return src_buf; } @@ -680,7 +670,6 @@ static int vpu_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_q src_vq->mem_ops = &vb2_vmalloc_memops; src_vq->drv_priv = inst; src_vq->buf_struct_size = sizeof(struct vpu_vb2_buffer); - src_vq->min_queued_buffers = 1; src_vq->dev = inst->vpu->dev; src_vq->lock = &inst->lock; ret = vb2_queue_init(src_vq); @@ -697,7 +686,6 @@ static int vpu_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_q dst_vq->mem_ops = &vb2_vmalloc_memops; dst_vq->drv_priv = inst; dst_vq->buf_struct_size = sizeof(struct vpu_vb2_buffer); - dst_vq->min_queued_buffers = 1; dst_vq->dev = inst->vpu->dev; dst_vq->lock = &inst->lock; ret = vb2_queue_init(dst_vq); @@ -713,15 +701,15 @@ static int vpu_v4l2_release(struct vpu_inst *inst) { vpu_trace(inst->vpu->dev, "%p\n", inst); - vpu_release_core(inst->core); - put_device(inst->dev); - if (inst->workqueue) { cancel_work_sync(&inst->msg_work); destroy_workqueue(inst->workqueue); inst->workqueue = NULL; } + vpu_release_core(inst->core); + put_device(inst->dev); + v4l2_ctrl_handler_free(&inst->ctrl_handler); mutex_destroy(&inst->lock); diff --git a/drivers/media/platform/amphion/vpu_v4l2.h b/drivers/media/platform/amphion/vpu_v4l2.h index 4a87b06ae5203..da9945f25e327 100644 --- a/drivers/media/platform/amphion/vpu_v4l2.h +++ b/drivers/media/platform/amphion/vpu_v4l2.h @@ -39,14 +39,4 @@ static inline struct vpu_format *vpu_get_format(struct vpu_inst *inst, u32 type) else return &inst->cap_format; } - -static inline int vpu_vb_is_codecconfig(struct vb2_v4l2_buffer *vbuf) -{ -#ifdef V4L2_BUF_FLAG_CODECCONFIG - return (vbuf->flags & V4L2_BUF_FLAG_CODECCONFIG) ? 1 : 0; -#else - return 0; -#endif -} - #endif diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.c b/drivers/media/platform/chips-media/wave5/wave5-helper.c index f03ad9c0de221..53a0ac068c2e2 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-helper.c +++ b/drivers/media/platform/chips-media/wave5/wave5-helper.c @@ -27,6 +27,11 @@ const char *state_to_str(enum vpu_instance_state state) } } +int wave5_kfifo_alloc(struct vpu_instance *inst) +{ + return kfifo_alloc(&inst->irq_status, 16 * sizeof(int), GFP_KERNEL); +} + void wave5_cleanup_instance(struct vpu_instance *inst, struct file *filp) { int i; @@ -49,7 +54,7 @@ void wave5_cleanup_instance(struct vpu_instance *inst, struct file *filp) v4l2_fh_del(&inst->v4l2_fh, filp); v4l2_fh_exit(&inst->v4l2_fh); } - list_del_init(&inst->list); + kfifo_free(&inst->irq_status); ida_free(&inst->dev->inst_ida, inst->id); kfree(inst->codec_info); kfree(inst); @@ -61,8 +66,29 @@ int wave5_vpu_release_device(struct file *filp, { struct vpu_instance *inst = file_to_vpu_inst(filp); int ret = 0; + unsigned long flags; v4l2_m2m_ctx_release(inst->v4l2_fh.m2m_ctx); + /* + * To prevent Null reference exception, the existing irq handler were + * separated to two modules. + * One is to queue interrupt reason into the irq handler, + * the other is irq_thread to call the wave5_vpu_dec_finish_decode + * to get decoded frame. + * The list of instances should be protected between all flow of the + * decoding process, but to protect the list in the irq_handler, spin lock + * should be used, and mutex should be used in the irq_thread because spin lock + * is not able to be used because mutex is already being used + * in the wave5_vpu_dec_finish_decode. + * So the spin lock and mutex were used to protect the list in the release function. + */ + ret = mutex_lock_interruptible(&inst->dev->irq_lock); + if (ret) + return ret; + spin_lock_irqsave(&inst->dev->irq_spinlock, flags); + list_del_init(&inst->list); + spin_unlock_irqrestore(&inst->dev->irq_spinlock, flags); + mutex_unlock(&inst->dev->irq_lock); if (inst->state != VPU_INST_STATE_NONE) { u32 fail_res; diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.h b/drivers/media/platform/chips-media/wave5/wave5-helper.h index 976a402e426ff..d61fdbda359d6 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-helper.h +++ b/drivers/media/platform/chips-media/wave5/wave5-helper.h @@ -33,4 +33,5 @@ void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned int width, unsigned int height, const struct v4l2_frmsize_stepwise *frmsize); +int wave5_kfifo_alloc(struct vpu_instance *inst); #endif diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c index e3038c18ca362..cff2fa17c3f59 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c @@ -1243,6 +1243,7 @@ static void wave5_vpu_dec_buf_queue_dst(struct vb2_buffer *vb) struct vpu_instance *inst = vb2_get_drv_priv(vb->vb2_queue); struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx; + pm_runtime_resume_and_get(inst->dev->dev); vbuf->sequence = inst->queued_dst_buf_num++; if (inst->state == VPU_INST_STATE_PIC_RUN) { @@ -1275,6 +1276,7 @@ static void wave5_vpu_dec_buf_queue_dst(struct vb2_buffer *vb) } else { v4l2_m2m_buf_queue(m2m_ctx, vbuf); } + pm_runtime_put_autosuspend(inst->dev->dev); } static void wave5_vpu_dec_buf_queue(struct vb2_buffer *vb) @@ -1753,8 +1755,10 @@ static int wave5_vpu_open_dec(struct file *filp) spin_lock_init(&inst->state_spinlock); inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); @@ -1806,6 +1810,11 @@ static int wave5_vpu_open_dec(struct file *filp) inst->xfer_func = V4L2_XFER_FUNC_DEFAULT; init_completion(&inst->irq_done); + ret = wave5_kfifo_alloc(inst); + if (ret) { + dev_err(inst->dev->dev, "failed to allocate fifo\n"); + goto cleanup_inst; + } inst->id = ida_alloc(&inst->dev->inst_ida, GFP_KERNEL); if (inst->id < 0) { @@ -1825,9 +1834,6 @@ static int wave5_vpu_open_dec(struct file *filp) if (ret) goto cleanup_inst; - if (list_empty(&dev->instances)) - pm_runtime_use_autosuspend(inst->dev->dev); - list_add_tail(&inst->list, &dev->instances); mutex_unlock(&dev->dev_lock); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c index 9bfaa9fb3ceb3..24fc0d0d3f4aa 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c @@ -649,6 +649,8 @@ static int wave5_vpu_enc_encoder_cmd(struct file *file, void *fh, struct v4l2_en m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx); m2m_ctx->is_draining = true; + + v4l2_m2m_try_schedule(m2m_ctx); break; case V4L2_ENC_CMD_START: break; @@ -1367,7 +1369,8 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count if (ret) goto return_buffers; } - if (inst->state == VPU_INST_STATE_OPEN && m2m_ctx->cap_q_ctx.q.streaming) { + if (inst->state == VPU_INST_STATE_OPEN && + (m2m_ctx->cap_q_ctx.q.streaming || q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) { ret = initialize_sequence(inst); if (ret) { dev_warn(inst->dev->dev, "Sequence not found: %d\n", ret); @@ -1578,8 +1581,10 @@ static int wave5_vpu_open_enc(struct file *filp) inst->ops = &wave5_vpu_enc_inst_ops; inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); @@ -1754,6 +1759,11 @@ static int wave5_vpu_open_enc(struct file *filp) inst->frame_rate = 30; init_completion(&inst->irq_done); + ret = wave5_kfifo_alloc(inst); + if (ret) { + dev_err(inst->dev->dev, "failed to allocate fifo\n"); + goto cleanup_inst; + } inst->id = ida_alloc(&inst->dev->inst_ida, GFP_KERNEL); if (inst->id < 0) { @@ -1768,9 +1778,6 @@ static int wave5_vpu_open_enc(struct file *filp) if (ret) goto cleanup_inst; - if (list_empty(&dev->instances)) - pm_runtime_use_autosuspend(inst->dev->dev); - list_add_tail(&inst->list, &dev->instances); mutex_unlock(&dev->dev_lock); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c b/drivers/media/platform/chips-media/wave5/wave5-vpu.c index e1715d3f43b0d..3216b49976447 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c @@ -51,8 +51,11 @@ static void wave5_vpu_handle_irq(void *dev_id) u32 seq_done; u32 cmd_done; u32 irq_reason; - struct vpu_instance *inst; + u32 irq_subreason; + struct vpu_instance *inst, *tmp; struct vpu_device *dev = dev_id; + int val; + unsigned long flags; irq_reason = wave5_vdi_read_register(dev, W5_VPU_VINT_REASON); seq_done = wave5_vdi_read_register(dev, W5_RET_SEQ_DONE_INSTANCE_INFO); @@ -60,7 +63,8 @@ static void wave5_vpu_handle_irq(void *dev_id) wave5_vdi_write_register(dev, W5_VPU_VINT_REASON_CLR, irq_reason); wave5_vdi_write_register(dev, W5_VPU_VINT_CLEAR, 0x1); - list_for_each_entry(inst, &dev->instances, list) { + spin_lock_irqsave(&dev->irq_spinlock, flags); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { if (irq_reason & BIT(INT_WAVE5_INIT_SEQ) || irq_reason & BIT(INT_WAVE5_ENC_SET_PARAM)) { @@ -82,22 +86,54 @@ static void wave5_vpu_handle_irq(void *dev_id) irq_reason & BIT(INT_WAVE5_ENC_PIC)) { if (cmd_done & BIT(inst->id)) { cmd_done &= ~BIT(inst->id); - wave5_vdi_write_register(dev, W5_RET_QUEUE_CMD_DONE_INST, - cmd_done); - inst->ops->finish_process(inst); + if (dev->irq >= 0) { + irq_subreason = + wave5_vdi_read_register(dev, W5_VPU_VINT_REASON); + if (!(irq_subreason & BIT(INT_WAVE5_DEC_PIC))) + wave5_vdi_write_register(dev, + W5_RET_QUEUE_CMD_DONE_INST, + cmd_done); + } + val = BIT(INT_WAVE5_DEC_PIC); + kfifo_in(&inst->irq_status, &val, sizeof(int)); } } + } + spin_unlock_irqrestore(&dev->irq_spinlock, flags); + + if (dev->irq < 0) + up(&dev->irq_sem); +} - wave5_vpu_clear_interrupt(inst, irq_reason); +static irqreturn_t wave5_vpu_irq(int irq, void *dev_id) +{ + struct vpu_device *dev = dev_id; + + if (wave5_vdi_read_register(dev, W5_VPU_VPU_INT_STS)) { + wave5_vpu_handle_irq(dev); + return IRQ_WAKE_THREAD; } + + return IRQ_HANDLED; } static irqreturn_t wave5_vpu_irq_thread(int irq, void *dev_id) { struct vpu_device *dev = dev_id; + struct vpu_instance *inst, *tmp; + int irq_status, ret; - if (wave5_vdi_read_register(dev, W5_VPU_VPU_INT_STS)) - wave5_vpu_handle_irq(dev); + mutex_lock(&dev->irq_lock); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { + while (kfifo_len(&inst->irq_status)) { + ret = kfifo_out(&inst->irq_status, &irq_status, sizeof(int)); + if (!ret) + break; + + inst->ops->finish_process(inst); + } + } + mutex_unlock(&dev->irq_lock); return IRQ_HANDLED; } @@ -121,6 +157,35 @@ static enum hrtimer_restart wave5_vpu_timer_callback(struct hrtimer *timer) return HRTIMER_RESTART; } +static int irq_thread(void *data) +{ + struct vpu_device *dev = (struct vpu_device *)data; + struct vpu_instance *inst, *tmp; + int irq_status, ret; + + while (!kthread_should_stop()) { + if (down_interruptible(&dev->irq_sem)) + continue; + + if (kthread_should_stop()) + break; + + mutex_lock(&dev->irq_lock); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { + while (kfifo_len(&inst->irq_status)) { + ret = kfifo_out(&inst->irq_status, &irq_status, sizeof(int)); + if (!ret) + break; + + inst->ops->finish_process(inst); + } + } + mutex_unlock(&dev->irq_lock); + } + + return 0; +} + static int wave5_vpu_load_firmware(struct device *dev, const char *fw_name, u32 *revision) { @@ -224,6 +289,8 @@ static int wave5_vpu_probe(struct platform_device *pdev) mutex_init(&dev->dev_lock); mutex_init(&dev->hw_lock); + mutex_init(&dev->irq_lock); + spin_lock_init(&dev->irq_spinlock); dev_set_drvdata(&pdev->dev, dev); dev->dev = &pdev->dev; @@ -266,9 +333,13 @@ static int wave5_vpu_probe(struct platform_device *pdev) } dev->product = wave5_vpu_get_product_id(dev); + INIT_LIST_HEAD(&dev->instances); + dev->irq = platform_get_irq(pdev, 0); if (dev->irq < 0) { dev_err(&pdev->dev, "failed to get irq resource, falling back to polling\n"); + sema_init(&dev->irq_sem, 1); + dev->irq_thread = kthread_run(irq_thread, dev, "irq thread"); hrtimer_setup(&dev->hrtimer, &wave5_vpu_timer_callback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); dev->worker = kthread_run_worker(0, "vpu_irq_thread"); @@ -280,7 +351,7 @@ static int wave5_vpu_probe(struct platform_device *pdev) dev->vpu_poll_interval = vpu_poll_interval; kthread_init_work(&dev->work, wave5_vpu_irq_work_fn); } else { - ret = devm_request_threaded_irq(&pdev->dev, dev->irq, NULL, + ret = devm_request_threaded_irq(&pdev->dev, dev->irq, wave5_vpu_irq, wave5_vpu_irq_thread, IRQF_ONESHOT, "vpu_irq", dev); if (ret) { dev_err(&pdev->dev, "Register interrupt handler, fail: %d\n", ret); @@ -288,7 +359,6 @@ static int wave5_vpu_probe(struct platform_device *pdev) } } - INIT_LIST_HEAD(&dev->instances); ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev); if (ret) { dev_err(&pdev->dev, "v4l2_device_register, fail: %d\n", ret); @@ -322,7 +392,7 @@ static int wave5_vpu_probe(struct platform_device *pdev) dev_info(&pdev->dev, "Product Code: 0x%x\n", dev->product_code); dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision); - pm_runtime_set_autosuspend_delay(&pdev->dev, 100); + pm_runtime_set_autosuspend_delay(&pdev->dev, 500); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_enable(&pdev->dev); wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0); @@ -351,21 +421,30 @@ static void wave5_vpu_remove(struct platform_device *pdev) { struct vpu_device *dev = dev_get_drvdata(&pdev->dev); + wave5_vpu_enc_unregister_device(dev); + wave5_vpu_dec_unregister_device(dev); + v4l2_device_unregister(&dev->v4l2_dev); + if (dev->irq < 0) { - kthread_destroy_worker(dev->worker); + if (dev->irq_thread) { + kthread_stop(dev->irq_thread); + up(&dev->irq_sem); + dev->irq_thread = NULL; + } + hrtimer_cancel(&dev->hrtimer); + kthread_cancel_work_sync(&dev->work); + kthread_destroy_worker(dev->worker); } - pm_runtime_put_sync(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); mutex_destroy(&dev->dev_lock); mutex_destroy(&dev->hw_lock); + mutex_destroy(&dev->irq_lock); reset_control_assert(dev->resets); clk_bulk_disable_unprepare(dev->num_clks, dev->clks); - wave5_vpu_enc_unregister_device(dev); - wave5_vpu_dec_unregister_device(dev); - v4l2_device_unregister(&dev->v4l2_dev); wave5_vdi_release(&pdev->dev); ida_destroy(&dev->inst_ida); } diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c index e5e879a13e8b8..e94d6ebc9f816 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c @@ -207,8 +207,6 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res) int retry = 0; struct vpu_device *vpu_dev = inst->dev; int i; - int inst_count = 0; - struct vpu_instance *inst_elm; *fail_res = 0; if (!inst->codec_info) @@ -250,11 +248,6 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res) wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task); - list_for_each_entry(inst_elm, &vpu_dev->instances, list) - inst_count++; - if (inst_count == 1) - pm_runtime_dont_use_autosuspend(vpu_dev->dev); - unlock_and_return: mutex_unlock(&vpu_dev->hw_lock); pm_runtime_put_sync(inst->dev->dev); @@ -720,8 +713,6 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res) int ret; int retry = 0; struct vpu_device *vpu_dev = inst->dev; - int inst_count = 0; - struct vpu_instance *inst_elm; *fail_res = 0; if (!inst->codec_info) @@ -764,12 +755,6 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res) } wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task); - - list_for_each_entry(inst_elm, &vpu_dev->instances, list) - inst_count++; - if (inst_count == 1) - pm_runtime_dont_use_autosuspend(vpu_dev->dev); - mutex_unlock(&vpu_dev->hw_lock); pm_runtime_put_sync(inst->dev->dev); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h index 45615c15beca3..bc101397204da 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h @@ -8,6 +8,7 @@ #ifndef VPUAPI_H_INCLUDED #define VPUAPI_H_INCLUDED +#include #include #include #include @@ -747,6 +748,7 @@ struct vpu_device { struct video_device *video_dev_enc; struct mutex dev_lock; /* lock for the src, dst v4l2 queues */ struct mutex hw_lock; /* lock hw configurations */ + struct mutex irq_lock; int irq; enum product_id product; struct vpu_attr attr; @@ -764,7 +766,10 @@ struct vpu_device { struct kthread_worker *worker; int vpu_poll_interval; int num_clks; + struct task_struct *irq_thread; + struct semaphore irq_sem; /* signal to irq_thread when interrupt happens*/ struct reset_control *resets; + spinlock_t irq_spinlock; /* protect instances list */ }; struct vpu_instance; @@ -788,6 +793,7 @@ struct vpu_instance { enum v4l2_ycbcr_encoding ycbcr_enc; enum v4l2_quantization quantization; + struct kfifo irq_status; enum vpu_instance_state state; enum vpu_instance_type type; const struct vpu_instance_ops *ops; diff --git a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c index 80fdc6ff57e0e..8432833814f31 100644 --- a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c +++ b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c @@ -194,11 +194,17 @@ static int mtk_mdp_probe(struct platform_device *pdev) } mdp->vpu_dev = vpu_get_plat_device(pdev); + if (!mdp->vpu_dev) { + dev_err(&pdev->dev, "Failed to get vpu device\n"); + ret = -ENODEV; + goto err_vpu_get_dev; + } + ret = vpu_wdt_reg_handler(mdp->vpu_dev, mtk_mdp_reset_handler, mdp, VPU_RST_MDP); if (ret) { dev_err(&pdev->dev, "Failed to register reset handler\n"); - goto err_m2m_register; + goto err_reg_handler; } platform_set_drvdata(pdev, mdp); @@ -206,7 +212,7 @@ static int mtk_mdp_probe(struct platform_device *pdev) ret = vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { dev_err(&pdev->dev, "Failed to set vb2 dma mag seg size\n"); - goto err_m2m_register; + goto err_reg_handler; } pm_runtime_enable(dev); @@ -214,6 +220,12 @@ static int mtk_mdp_probe(struct platform_device *pdev) return 0; +err_reg_handler: + platform_device_put(mdp->vpu_dev); + +err_vpu_get_dev: + mtk_mdp_unregister_m2m_device(mdp); + err_m2m_register: v4l2_device_unregister(&mdp->v4l2_dev); @@ -242,6 +254,7 @@ static void mtk_mdp_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); vb2_dma_contig_clear_max_seg_size(&pdev->dev); + platform_device_put(mdp->vpu_dev); mtk_mdp_unregister_m2m_device(mdp); v4l2_device_unregister(&mdp->v4l2_dev); diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c index 6559d72d5d427..6d26d4aa1eef2 100644 --- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c @@ -157,10 +157,18 @@ void mdp_video_device_release(struct video_device *vdev) kfree(mdp); } +static void mdp_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + static int mdp_mm_subsys_deploy(struct mdp_dev *mdp, enum mdp_infra_id id) { struct platform_device *mm_pdev = NULL; struct device **dev; + int ret; int i; if (!mdp) @@ -194,6 +202,11 @@ static int mdp_mm_subsys_deploy(struct mdp_dev *mdp, enum mdp_infra_id id) if (WARN_ON(!mm_pdev)) return -ENODEV; + ret = devm_add_action_or_reset(&mdp->pdev->dev, mdp_put_device, + &mm_pdev->dev); + if (ret) + return ret; + *dev = &mm_pdev->dev; } @@ -279,6 +292,7 @@ static int mdp_probe(struct platform_device *pdev) goto err_destroy_clock_wq; } mdp->scp = platform_get_drvdata(mm_pdev); + put_device(&mm_pdev->dev); } mdp->rproc_handle = scp_get_rproc(mdp->scp); diff --git a/drivers/media/platform/mediatek/vcodec/common/mtk_vcodec_fw_vpu.c b/drivers/media/platform/mediatek/vcodec/common/mtk_vcodec_fw_vpu.c index d7027d600208f..3632037f78f50 100644 --- a/drivers/media/platform/mediatek/vcodec/common/mtk_vcodec_fw_vpu.c +++ b/drivers/media/platform/mediatek/vcodec/common/mtk_vcodec_fw_vpu.c @@ -47,30 +47,32 @@ static void mtk_vcodec_vpu_reset_dec_handler(void *priv) { struct mtk_vcodec_dec_dev *dev = priv; struct mtk_vcodec_dec_ctx *ctx; + unsigned long flags; dev_err(&dev->plat_dev->dev, "Watchdog timeout!!"); - mutex_lock(&dev->dev_ctx_lock); + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_for_each_entry(ctx, &dev->ctx_list, list) { ctx->state = MTK_STATE_ABORT; mtk_v4l2_vdec_dbg(0, ctx, "[%d] Change to state MTK_STATE_ABORT", ctx->id); } - mutex_unlock(&dev->dev_ctx_lock); + spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); } static void mtk_vcodec_vpu_reset_enc_handler(void *priv) { struct mtk_vcodec_enc_dev *dev = priv; struct mtk_vcodec_enc_ctx *ctx; + unsigned long flags; dev_err(&dev->plat_dev->dev, "Watchdog timeout!!"); - mutex_lock(&dev->dev_ctx_lock); + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_for_each_entry(ctx, &dev->ctx_list, list) { ctx->state = MTK_STATE_ABORT; mtk_v4l2_vdec_dbg(0, ctx, "[%d] Change to state MTK_STATE_ABORT", ctx->id); } - mutex_unlock(&dev->dev_ctx_lock); + spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); } static const struct mtk_vcodec_fw_ops mtk_vcodec_vpu_msg = { @@ -117,8 +119,10 @@ struct mtk_vcodec_fw *mtk_vcodec_fw_vpu_init(void *priv, enum mtk_vcodec_fw_use vpu_wdt_reg_handler(fw_pdev, mtk_vcodec_vpu_reset_enc_handler, priv, rst_id); fw = devm_kzalloc(&plat_dev->dev, sizeof(*fw), GFP_KERNEL); - if (!fw) + if (!fw) { + put_device(&fw_pdev->dev); return ERR_PTR(-ENOMEM); + } fw->type = VPU; fw->ops = &mtk_vcodec_vpu_msg; fw->pdev = fw_pdev; diff --git a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.c b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.c index 46d176e6de63e..3b81fae9f9135 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.c @@ -198,6 +198,7 @@ static int fops_vcodec_open(struct file *file) struct mtk_vcodec_dec_ctx *ctx = NULL; int ret = 0, i, hw_count; struct vb2_queue *src_vq; + unsigned long flags; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) @@ -267,9 +268,9 @@ static int fops_vcodec_open(struct file *file) ctx->dev->vdec_pdata->init_vdec_params(ctx); - mutex_lock(&dev->dev_ctx_lock); + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_add(&ctx->list, &dev->ctx_list); - mutex_unlock(&dev->dev_ctx_lock); + spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); mtk_vcodec_dbgfs_create(ctx); mutex_unlock(&dev->dev_mutex); @@ -294,6 +295,7 @@ static int fops_vcodec_release(struct file *file) { struct mtk_vcodec_dec_dev *dev = video_drvdata(file); struct mtk_vcodec_dec_ctx *ctx = file_to_dec_ctx(file); + unsigned long flags; mtk_v4l2_vdec_dbg(0, ctx, "[%d] decoder", ctx->id); mutex_lock(&dev->dev_mutex); @@ -312,9 +314,9 @@ static int fops_vcodec_release(struct file *file) v4l2_ctrl_handler_free(&ctx->ctrl_hdl); mtk_vcodec_dbgfs_remove(dev, ctx->id); - mutex_lock(&dev->dev_ctx_lock); + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_del_init(&ctx->list); - mutex_unlock(&dev->dev_ctx_lock); + spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); kfree(ctx); mutex_unlock(&dev->dev_mutex); return 0; @@ -407,7 +409,7 @@ static int mtk_vcodec_probe(struct platform_device *pdev) for (i = 0; i < MTK_VDEC_HW_MAX; i++) mutex_init(&dev->dec_mutex[i]); mutex_init(&dev->dev_mutex); - mutex_init(&dev->dev_ctx_lock); + spin_lock_init(&dev->dev_ctx_lock); spin_lock_init(&dev->irqlock); snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name), "%s", diff --git a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.h b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.h index d047d7c580fb6..9d68808e8f9c3 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.h +++ b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_drv.h @@ -285,7 +285,7 @@ struct mtk_vcodec_dec_dev { /* decoder hardware mutex lock */ struct mutex dec_mutex[MTK_VDEC_HW_MAX]; struct mutex dev_mutex; - struct mutex dev_ctx_lock; + spinlock_t dev_ctx_lock; struct workqueue_struct *decode_workqueue; spinlock_t irqlock; diff --git a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c index d873159b9b306..9eef3ff2b1278 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c @@ -502,6 +502,12 @@ static int mtk_vdec_s_ctrl(struct v4l2_ctrl *ctrl) mtk_v4l2_vdec_err(ctx, "VP9: bit_depth:%d", frame->bit_depth); return -EINVAL; } + + if (!(frame->flags & V4L2_VP9_FRAME_FLAG_X_SUBSAMPLING) || + !(frame->flags & V4L2_VP9_FRAME_FLAG_Y_SUBSAMPLING)) { + mtk_v4l2_vdec_err(ctx, "VP9: only 420 subsampling is supported"); + return -EINVAL; + } break; case V4L2_CID_STATELESS_AV1_SEQUENCE: seq = (struct v4l2_ctrl_av1_sequence *)hdr_ctrl->p_new.p; diff --git a/drivers/media/platform/mediatek/vcodec/decoder/vdec_vpu_if.c b/drivers/media/platform/mediatek/vcodec/decoder/vdec_vpu_if.c index 145958206e38a..40b97f114cf6f 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/vdec_vpu_if.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/vdec_vpu_if.c @@ -75,16 +75,17 @@ static void handle_get_param_msg_ack(const struct vdec_vpu_ipi_get_param_ack *ms static bool vpu_dec_check_ap_inst(struct mtk_vcodec_dec_dev *dec_dev, struct vdec_vpu_inst *vpu) { struct mtk_vcodec_dec_ctx *ctx; + unsigned long flags; int ret = false; - mutex_lock(&dec_dev->dev_ctx_lock); + spin_lock_irqsave(&dec_dev->dev_ctx_lock, flags); list_for_each_entry(ctx, &dec_dev->ctx_list, list) { if (!IS_ERR_OR_NULL(ctx) && ctx->vpu_inst == vpu) { ret = true; break; } } - mutex_unlock(&dec_dev->dev_ctx_lock); + spin_unlock_irqrestore(&dec_dev->dev_ctx_lock, flags); return ret; } diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c index d815e962ab898..0212ff0462fcc 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c +++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c @@ -864,7 +864,7 @@ static void vb2ops_venc_buf_queue(struct vb2_buffer *vb) static int vb2ops_venc_start_streaming(struct vb2_queue *q, unsigned int count) { struct mtk_vcodec_enc_ctx *ctx = vb2_get_drv_priv(q); - struct venc_enc_param param; + struct venc_enc_param param = { }; int ret; int i; @@ -1018,7 +1018,7 @@ static int mtk_venc_encode_header(void *priv) int ret; struct vb2_v4l2_buffer *src_buf, *dst_buf; struct mtk_vcodec_mem bs_buf; - struct venc_done_result enc_result; + struct venc_done_result enc_result = { }; dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); if (!dst_buf) { @@ -1139,7 +1139,7 @@ static void mtk_venc_worker(struct work_struct *work) struct vb2_v4l2_buffer *src_buf, *dst_buf; struct venc_frm_buf frm_buf; struct mtk_vcodec_mem bs_buf; - struct venc_done_result enc_result; + struct venc_done_result enc_result = { }; int ret, i; /* check dst_buf, dst_buf may be removed in device_run diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c index fb1c3bdc2daeb..4ac667a8de4cd 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c +++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c @@ -117,6 +117,7 @@ static int fops_vcodec_open(struct file *file) struct mtk_vcodec_enc_ctx *ctx = NULL; int ret = 0; struct vb2_queue *src_vq; + unsigned long flags; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) @@ -176,9 +177,9 @@ static int fops_vcodec_open(struct file *file) mtk_v4l2_venc_dbg(2, ctx, "Create instance [%d]@%p m2m_ctx=%p ", ctx->id, ctx, ctx->m2m_ctx); - mutex_lock(&dev->dev_ctx_lock); + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_add(&ctx->list, &dev->ctx_list); - mutex_unlock(&dev->dev_ctx_lock); + spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); mutex_unlock(&dev->dev_mutex); mtk_v4l2_venc_dbg(0, ctx, "%s encoder [%d]", dev_name(&dev->plat_dev->dev), @@ -203,6 +204,7 @@ static int fops_vcodec_release(struct file *file) { struct mtk_vcodec_enc_dev *dev = video_drvdata(file); struct mtk_vcodec_enc_ctx *ctx = file_to_enc_ctx(file); + unsigned long flags; mtk_v4l2_venc_dbg(1, ctx, "[%d] encoder", ctx->id); mutex_lock(&dev->dev_mutex); @@ -213,9 +215,18 @@ static int fops_vcodec_release(struct file *file) v4l2_fh_exit(&ctx->fh); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); - mutex_lock(&dev->dev_ctx_lock); + /* + * Cancel any pending encode work before freeing the context. + * Although v4l2_m2m_ctx_release() waits for m2m job completion, + * the workqueue handler (mtk_venc_worker) may still be accessing + * the context after v4l2_m2m_job_finish() returns. Without this, + * a use-after-free occurs when the worker accesses ctx after kfree. + */ + cancel_work_sync(&ctx->encode_work); + + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_del_init(&ctx->list); - mutex_unlock(&dev->dev_ctx_lock); + spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); kfree(ctx); mutex_unlock(&dev->dev_mutex); return 0; @@ -297,7 +308,7 @@ static int mtk_vcodec_probe(struct platform_device *pdev) mutex_init(&dev->enc_mutex); mutex_init(&dev->dev_mutex); - mutex_init(&dev->dev_ctx_lock); + spin_lock_init(&dev->dev_ctx_lock); spin_lock_init(&dev->irqlock); snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name), "%s", diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.h b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.h index 5b304a5512366..0cddfa13594f5 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.h +++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.h @@ -206,7 +206,7 @@ struct mtk_vcodec_enc_dev { /* encoder hardware mutex lock */ struct mutex enc_mutex; struct mutex dev_mutex; - struct mutex dev_ctx_lock; + spinlock_t dev_ctx_lock; struct workqueue_struct *encode_workqueue; int enc_irq; diff --git a/drivers/media/platform/mediatek/vcodec/encoder/venc_vpu_if.c b/drivers/media/platform/mediatek/vcodec/encoder/venc_vpu_if.c index 51bb7ee141b9e..3c229b1f6b21f 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/venc_vpu_if.c +++ b/drivers/media/platform/mediatek/vcodec/encoder/venc_vpu_if.c @@ -45,16 +45,17 @@ static void handle_enc_encode_msg(struct venc_vpu_inst *vpu, const void *data) static bool vpu_enc_check_ap_inst(struct mtk_vcodec_enc_dev *enc_dev, struct venc_vpu_inst *vpu) { struct mtk_vcodec_enc_ctx *ctx; + unsigned long flags; int ret = false; - mutex_lock(&enc_dev->dev_ctx_lock); + spin_lock_irqsave(&enc_dev->dev_ctx_lock, flags); list_for_each_entry(ctx, &enc_dev->ctx_list, list) { if (!IS_ERR_OR_NULL(ctx) && ctx->vpu_inst == vpu) { ret = true; break; } } - mutex_unlock(&enc_dev->dev_ctx_lock); + spin_unlock_irqrestore(&enc_dev->dev_ctx_lock, flags); return ret; } diff --git a/drivers/media/platform/qcom/camss/Kconfig b/drivers/media/platform/qcom/camss/Kconfig index 4eda48cb1adf0..1edc5e5a1829e 100644 --- a/drivers/media/platform/qcom/camss/Kconfig +++ b/drivers/media/platform/qcom/camss/Kconfig @@ -7,3 +7,4 @@ config VIDEO_QCOM_CAMSS select VIDEO_V4L2_SUBDEV_API select VIDEOBUF2_DMA_SG select V4L2_FWNODE + select PHY_QCOM_MIPI_CSI2 diff --git a/drivers/media/platform/qcom/camss/Makefile b/drivers/media/platform/qcom/camss/Makefile index 23960d02877de..27898b3cc7d3c 100644 --- a/drivers/media/platform/qcom/camss/Makefile +++ b/drivers/media/platform/qcom/camss/Makefile @@ -10,10 +10,14 @@ qcom-camss-objs += \ camss-csid-680.o \ camss-csid-gen2.o \ camss-csid-gen3.o \ + camss-csiphy.o \ camss-csiphy-2ph-1-0.o \ camss-csiphy-3ph-1-0.o \ - camss-csiphy.o \ + camss-format.o \ camss-ispif.o \ + camss-tpg.o \ + camss-tpg-gen1.o \ + camss-vfe.o \ camss-vfe-4-1.o \ camss-vfe-4-7.o \ camss-vfe-4-8.o \ @@ -21,10 +25,9 @@ qcom-camss-objs += \ camss-vfe-340.o \ camss-vfe-480.o \ camss-vfe-680.o \ - camss-vfe-gen3.o \ camss-vfe-gen1.o \ - camss-vfe.o \ - camss-video.o \ - camss-format.o \ + camss-vfe-gen3.o \ + camss-vfe-vbif.o \ + camss-video.o obj-$(CONFIG_VIDEO_QCOM_CAMSS) += qcom-camss.o diff --git a/drivers/media/platform/qcom/camss/camss-csid-340.c b/drivers/media/platform/qcom/camss/camss-csid-340.c index 22a30510fb792..2b50f9b96a34e 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-340.c +++ b/drivers/media/platform/qcom/camss/camss-csid-340.c @@ -55,8 +55,7 @@ #define CSID_RDI_CTRL_HALT_AT_FRAME_BOUNDARY 0 #define CSID_RDI_CTRL_RESUME_AT_FRAME_BOUNDARY 1 -static void __csid_configure_rx(struct csid_device *csid, - struct csid_phy_config *phy, int vc) +static void __csid_configure_rx(struct csid_device *csid, struct csid_phy_config *phy) { u32 val; @@ -81,13 +80,9 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats, csid->res->formats->nformats, input_format->code); - u8 lane_cnt = csid->phy.lane_cnt; u8 dt_id; u32 val; - if (!lane_cnt) - lane_cnt = 4; - /* * DT_ID is a two bit bitfield that is concatenated with * the four least significant bits of the five bit VC @@ -120,10 +115,11 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable) { int i; + __csid_configure_rx(csid, &csid->phy); + for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++) { if (csid->phy.en_vc & BIT(i)) { __csid_configure_rdi_stream(csid, enable, i); - __csid_configure_rx(csid, &csid->phy, i); __csid_ctrl_rdi(csid, enable, i); } } diff --git a/drivers/media/platform/qcom/camss/camss-csid-680.c b/drivers/media/platform/qcom/camss/camss-csid-680.c index 3ad3a174bcfb8..2e4547455d229 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-680.c +++ b/drivers/media/platform/qcom/camss/camss-csid-680.c @@ -103,6 +103,8 @@ #define CSI2_RX_CFG0_PHY_NUM_SEL 20 #define CSI2_RX_CFG0_PHY_SEL_BASE_IDX 1 #define CSI2_RX_CFG0_PHY_TYPE_SEL 24 +#define CSI2_RX_CFG0_TPG_MUX_EN BIT(27) +#define CSI2_RX_CFG0_TPG_MUX_SEL GENMASK(29, 28) #define CSID_CSI2_RX_CFG1 0x204 #define CSI2_RX_CFG1_PACKET_ECC_CORRECTION_EN BIT(0) @@ -185,10 +187,20 @@ static void __csid_configure_rx(struct csid_device *csid, struct csid_phy_config *phy, int vc) { u32 val; + struct camss *camss; + camss = csid->camss; val = (phy->lane_cnt - 1) << CSI2_RX_CFG0_NUM_ACTIVE_LANES; val |= phy->lane_assign << CSI2_RX_CFG0_DL0_INPUT_SEL; - val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX) << CSI2_RX_CFG0_PHY_NUM_SEL; + + if (camss->tpg && csid->tpg_linked && + camss->tpg[phy->csiphy_id].testgen.mode != TPG_PAYLOAD_MODE_DISABLED) { + val |= FIELD_PREP(CSI2_RX_CFG0_TPG_MUX_SEL, phy->csiphy_id + 1); + val |= CSI2_RX_CFG0_TPG_MUX_EN; + } else { + val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX) + << CSI2_RX_CFG0_PHY_NUM_SEL; + } writel(val, csid->base + CSID_CSI2_RX_CFG0); diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen3.c b/drivers/media/platform/qcom/camss/camss-csid-gen3.c index 664245cf6eb0c..222765fe7914d 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-gen3.c +++ b/drivers/media/platform/qcom/camss/camss-csid-gen3.c @@ -48,9 +48,9 @@ #define IS_CSID_690(csid) ((csid->camss->res->version == CAMSS_8775P) \ || (csid->camss->res->version == CAMSS_8300)) #define CSID_BUF_DONE_IRQ_STATUS 0x8C -#define BUF_DONE_IRQ_STATUS_RDI_OFFSET (csid_is_lite(csid) ?\ - 1 : (IS_CSID_690(csid) ?\ - 13 : 14)) +#define BUF_DONE_IRQ_STATUS_RDI_OFFSET (csid_is_lite(csid) ? \ + ((IS_CSID_690(csid) ? 0 : 1)) : \ + ((IS_CSID_690(csid) ? 13 : 14))) #define CSID_BUF_DONE_IRQ_MASK 0x90 #define CSID_BUF_DONE_IRQ_CLEAR 0x94 #define CSID_BUF_DONE_IRQ_SET 0x98 @@ -66,6 +66,8 @@ #define CSI2_RX_CFG0_VC_MODE 3 #define CSI2_RX_CFG0_DL0_INPUT_SEL 4 #define CSI2_RX_CFG0_PHY_NUM_SEL 20 +#define CSI2_RX_CFG0_TPG_MUX_EN BIT(27) +#define CSI2_RX_CFG0_TPG_MUX_SEL GENMASK(29, 28) #define CSID_CSI2_RX_CFG1 0x204 #define CSI2_RX_CFG1_ECC_CORRECTION_EN BIT(0) @@ -109,10 +111,20 @@ static void __csid_configure_rx(struct csid_device *csid, struct csid_phy_config *phy, int vc) { int val; + struct camss *camss; + camss = csid->camss; val = (phy->lane_cnt - 1) << CSI2_RX_CFG0_NUM_ACTIVE_LANES; val |= phy->lane_assign << CSI2_RX_CFG0_DL0_INPUT_SEL; - val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX) << CSI2_RX_CFG0_PHY_NUM_SEL; + + if (camss->tpg && csid->tpg_linked && + camss->tpg[phy->csiphy_id].testgen.mode != TPG_PAYLOAD_MODE_DISABLED) { + val |= FIELD_PREP(CSI2_RX_CFG0_TPG_MUX_SEL, phy->csiphy_id + 1); + val |= CSI2_RX_CFG0_TPG_MUX_EN; + } else { + val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX) + << CSI2_RX_CFG0_PHY_NUM_SEL; + } writel(val, csid->base + CSID_CSI2_RX_CFG0); diff --git a/drivers/media/platform/qcom/camss/camss-csid.c b/drivers/media/platform/qcom/camss/camss-csid.c index 5284b5857368c..48459b46a981b 100644 --- a/drivers/media/platform/qcom/camss/camss-csid.c +++ b/drivers/media/platform/qcom/camss/camss-csid.c @@ -35,6 +35,8 @@ #define HW_VERSION_REVISION 16 #define HW_VERSION_GENERATION 28 +#define LANE_CFG_BITWIDTH 4 + #define MSM_CSID_NAME "msm_csid" const char * const csid_testgen_modes[] = { @@ -1187,24 +1189,12 @@ int msm_csid_subdev_init(struct camss *camss, struct csid_device *csid, /* Regulator */ for (i = 0; i < ARRAY_SIZE(res->regulators); i++) { - if (res->regulators[i]) + if (res->regulators[i].supply) csid->num_supplies++; } - if (csid->num_supplies) { - csid->supplies = devm_kmalloc_array(camss->dev, - csid->num_supplies, - sizeof(*csid->supplies), - GFP_KERNEL); - if (!csid->supplies) - return -ENOMEM; - } - - for (i = 0; i < csid->num_supplies; i++) - csid->supplies[i].supply = res->regulators[i]; - - ret = devm_regulator_bulk_get(camss->dev, csid->num_supplies, - csid->supplies); + ret = devm_regulator_bulk_get_const(camss->dev, csid->num_supplies, + res->regulators, &csid->supplies); if (ret) return ret; @@ -1227,18 +1217,22 @@ void msm_csid_get_csid_id(struct media_entity *entity, u8 *id) } /* - * csid_get_lane_assign - Calculate CSI2 lane assign configuration parameter - * @lane_cfg - CSI2 lane configuration + * csid_get_lane_assign - Calculate lane assign by csiphy/tpg lane num + * @lane_cfg: CSI2 lane configuration + * @num_lanes: lane num * * Return lane assign */ -static u32 csid_get_lane_assign(struct csiphy_lanes_cfg *lane_cfg) +static u32 csid_get_lane_assign(struct csiphy_lanes_cfg *lane_cfg, int num_lanes) { u32 lane_assign = 0; + int pos; int i; - for (i = 0; i < lane_cfg->num_data; i++) - lane_assign |= lane_cfg->data[i].pos << (i * 4); + for (i = 0; i < num_lanes; i++) { + pos = lane_cfg ? lane_cfg->data[i].pos : i; + lane_assign |= pos << (i * LANE_CFG_BITWIDTH); + } return lane_assign; } @@ -1263,6 +1257,7 @@ static int csid_link_setup(struct media_entity *entity, if ((local->flags & MEDIA_PAD_FL_SINK) && (flags & MEDIA_LNK_FL_ENABLED)) { struct v4l2_subdev *sd; + struct tpg_device *tpg; struct csid_device *csid; struct csiphy_device *csiphy; struct csiphy_lanes_cfg *lane_cfg; @@ -1277,18 +1272,28 @@ static int csid_link_setup(struct media_entity *entity, return -EBUSY; sd = media_entity_to_v4l2_subdev(remote->entity); - csiphy = v4l2_get_subdevdata(sd); + if (sd->grp_id == TPG_GRP_ID) { + tpg = v4l2_get_subdevdata(sd); + + csid->phy.lane_cnt = tpg->res->lane_cnt; + csid->phy.csiphy_id = tpg->id; + csid->phy.lane_assign = csid_get_lane_assign(NULL, csid->phy.lane_cnt); + csid->tpg_linked = true; + } else { + csiphy = v4l2_get_subdevdata(sd); - /* If a sensor is not linked to CSIPHY */ - /* do no allow a link from CSIPHY to CSID */ - if (!csiphy->cfg.csi2) - return -EPERM; + /* If a sensor is not linked to CSIPHY */ + /* do no allow a link from CSIPHY to CSID */ + if (!csiphy->cfg.csi2) + return -EPERM; - csid->phy.csiphy_id = csiphy->id; + csid->phy.csiphy_id = csiphy->id; - lane_cfg = &csiphy->cfg.csi2->lane_cfg; - csid->phy.lane_cnt = lane_cfg->num_data; - csid->phy.lane_assign = csid_get_lane_assign(lane_cfg); + lane_cfg = &csiphy->cfg.csi2->lane_cfg; + csid->phy.lane_cnt = lane_cfg->num_data; + csid->phy.lane_assign = csid_get_lane_assign(lane_cfg, lane_cfg->num_data); + csid->tpg_linked = false; + } } /* Decide which virtual channels to enable based on which source pads are enabled */ if (local->flags & MEDIA_PAD_FL_SOURCE) { diff --git a/drivers/media/platform/qcom/camss/camss-csid.h b/drivers/media/platform/qcom/camss/camss-csid.h index aedc96ed84b2f..5296b10f6bac8 100644 --- a/drivers/media/platform/qcom/camss/camss-csid.h +++ b/drivers/media/platform/qcom/camss/camss-csid.h @@ -161,6 +161,7 @@ struct csid_device { int num_supplies; struct completion reset_complete; struct csid_testgen_config testgen; + bool tpg_linked; struct csid_phy_config phy; struct v4l2_mbus_framefmt fmt[MSM_CSID_PADS_NUM]; struct v4l2_ctrl_handler ctrls; diff --git a/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c b/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c index a229ba04b1587..d37f71de0f42c 100644 --- a/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c +++ b/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c @@ -46,7 +46,8 @@ #define CSIPHY_3PH_CMN_CSI_COMMON_CTRL5_CLK_ENABLE BIT(7) #define CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_COMMON_PWRDN_B BIT(0) #define CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_SHOW_REV_ID BIT(1) -#define CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(offset, n) ((offset) + 0xb0 + 0x4 * (n)) +#define CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(offset, common_status_offset, n) \ + ((offset) + (common_status_offset) + 0x4 * (n)) #define CSIPHY_DEFAULT_PARAMS 0 #define CSIPHY_LANE_ENABLE 1 @@ -587,6 +588,102 @@ csiphy_lane_regs lane_regs_sm8550[] = { {0x0C64, 0x7F, 0x00, CSIPHY_DEFAULT_PARAMS}, }; +/* GEN2 2.2.0 2PH 4 lane DPHY mode */ +static const struct +csiphy_lane_regs lane_regs_sm8650[] = { + {0x0e94, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0ea0, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e90, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e98, 0x08, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e94, 0x07, 0xd1, CSIPHY_DEFAULT_PARAMS}, + {0x0e30, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e28, 0x04, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e00, 0x80, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e0c, 0xff, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e38, 0x1f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e2c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e34, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e1c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e14, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e3c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e04, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e20, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0e08, 0x19, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {0x0e10, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS}, + + {0x0094, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x00a0, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0090, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0098, 0x08, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0094, 0x07, 0xd1, CSIPHY_DEFAULT_PARAMS}, + {0x0030, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0000, 0x8e, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0038, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x002c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0034, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x001c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0014, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x003c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0004, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0020, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0008, 0x19, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {0x0010, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS}, + + {0x0494, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x04a0, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0490, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0498, 0x08, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0494, 0x07, 0xd1, CSIPHY_DEFAULT_PARAMS}, + {0x0430, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0400, 0x8e, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0438, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x042c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0434, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x041c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0414, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x043c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0404, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0420, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0408, 0x19, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {0x0410, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS}, + + {0x0894, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x08a0, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0890, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0898, 0x08, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0894, 0x07, 0xd1, CSIPHY_DEFAULT_PARAMS}, + {0x0830, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0800, 0x8e, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0838, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x082c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0834, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x081c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0814, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x083c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0804, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0820, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0808, 0x19, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {0x0810, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS}, + + {0x0c94, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0ca0, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c90, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c98, 0x08, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c94, 0x07, 0xd1, CSIPHY_DEFAULT_PARAMS}, + {0x0c30, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c00, 0x8e, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c38, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c2c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c34, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c1c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c14, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c3c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c04, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c20, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS}, + {0x0c08, 0x19, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {0x0c10, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS}, +}; + /* 4nm 2PH v 2.1.2 2p5Gbps 4 lane DPHY mode */ static const struct csiphy_lane_regs lane_regs_x1e80100[] = { @@ -714,13 +811,17 @@ static void csiphy_hw_version_read(struct csiphy_device *csiphy, CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->offset, 6)); hw_version = readl_relaxed(csiphy->base + - CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, 12)); + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, + regs->common_status_offset, 12)); hw_version |= readl_relaxed(csiphy->base + - CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, 13)) << 8; + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, + regs->common_status_offset, 13)) << 8; hw_version |= readl_relaxed(csiphy->base + - CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, 14)) << 16; + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, + regs->common_status_offset, 14)) << 16; hw_version |= readl_relaxed(csiphy->base + - CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, 15)) << 24; + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, + regs->common_status_offset, 15)) << 24; dev_dbg(dev, "CSIPHY 3PH HW Version = 0x%08x\n", hw_version); } @@ -749,7 +850,8 @@ static irqreturn_t csiphy_isr(int irq, void *dev) for (i = 0; i < 11; i++) { int c = i + 22; u8 val = readl_relaxed(csiphy->base + - CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, i)); + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->offset, + regs->common_status_offset, i)); writel_relaxed(val, csiphy->base + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->offset, c)); @@ -908,14 +1010,17 @@ static bool csiphy_is_gen2(u32 version) switch (version) { case CAMSS_2290: + case CAMSS_6150: case CAMSS_7280: case CAMSS_8250: case CAMSS_8280XP: case CAMSS_8300: case CAMSS_845: case CAMSS_8550: + case CAMSS_8650: case CAMSS_8775P: case CAMSS_X1E80100: + case CAMSS_X1P42100: ret = true; break; } @@ -989,6 +1094,7 @@ static int csiphy_init(struct csiphy_device *csiphy) csiphy->regs = regs; regs->offset = 0x800; + regs->common_status_offset = 0xb0; switch (csiphy->camss->res->version) { case CAMSS_845: @@ -996,6 +1102,7 @@ static int csiphy_init(struct csiphy_device *csiphy) regs->lane_array_size = ARRAY_SIZE(lane_regs_sdm845); break; case CAMSS_2290: + case CAMSS_6150: regs->lane_regs = &lane_regs_qcm2290[0]; regs->lane_array_size = ARRAY_SIZE(lane_regs_qcm2290); break; @@ -1009,6 +1116,7 @@ static int csiphy_init(struct csiphy_device *csiphy) regs->lane_array_size = ARRAY_SIZE(lane_regs_sc8280xp); break; case CAMSS_X1E80100: + case CAMSS_X1P42100: regs->lane_regs = &lane_regs_x1e80100[0]; regs->lane_array_size = ARRAY_SIZE(lane_regs_x1e80100); regs->offset = 0x1000; @@ -1018,6 +1126,11 @@ static int csiphy_init(struct csiphy_device *csiphy) regs->lane_array_size = ARRAY_SIZE(lane_regs_sm8550); regs->offset = 0x1000; break; + case CAMSS_8650: + regs->lane_regs = &lane_regs_sm8650[0]; + regs->lane_array_size = ARRAY_SIZE(lane_regs_sm8650); + regs->offset = 0x1000; + break; case CAMSS_8300: case CAMSS_8775P: regs->lane_regs = &lane_regs_sa8775p[0]; diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.c b/drivers/media/platform/qcom/camss/camss-csiphy.c index 2de97f58f9ae4..b9f7d6573a0f0 100644 --- a/drivers/media/platform/qcom/camss/camss-csiphy.c +++ b/drivers/media/platform/qcom/camss/camss-csiphy.c @@ -7,12 +7,14 @@ * Copyright (c) 2011-2015, The Linux Foundation. All rights reserved. * Copyright (C) 2016-2018 Linaro Ltd. */ +#include #include #include #include #include #include #include +#include #include #include #include @@ -131,10 +133,10 @@ static u8 csiphy_get_bpp(const struct csiphy_format_info *formats, } /* - * csiphy_set_clock_rates - Calculate and set clock rates on CSIPHY module + * csiphy_set_clock_rates_legacy - Calculate and set clock rates on CSIPHY module * @csiphy: CSIPHY device */ -static int csiphy_set_clock_rates(struct csiphy_device *csiphy) +static int csiphy_set_clock_rates_legacy(struct csiphy_device *csiphy) { struct device *dev = csiphy->camss->dev; s64 link_freq; @@ -200,7 +202,7 @@ static int csiphy_set_clock_rates(struct csiphy_device *csiphy) * * Return 0 on success or a negative error code otherwise */ -static int csiphy_set_power(struct v4l2_subdev *sd, int on) +static int csiphy_set_power_legacy(struct v4l2_subdev *sd, int on) { struct csiphy_device *csiphy = v4l2_get_subdevdata(sd); struct device *dev = csiphy->camss->dev; @@ -219,7 +221,7 @@ static int csiphy_set_power(struct v4l2_subdev *sd, int on) return ret; } - ret = csiphy_set_clock_rates(csiphy); + ret = csiphy_set_clock_rates_legacy(csiphy); if (ret < 0) { regulator_bulk_disable(csiphy->num_supplies, csiphy->supplies); @@ -254,7 +256,7 @@ static int csiphy_set_power(struct v4l2_subdev *sd, int on) } /* - * csiphy_stream_on - Enable streaming on CSIPHY module + * csiphy_stream_on_legacy - Enable streaming on CSIPHY module * @csiphy: CSIPHY device * * Helper function to enable streaming on CSIPHY module. @@ -262,7 +264,7 @@ static int csiphy_set_power(struct v4l2_subdev *sd, int on) * * Return 0 on success or a negative error code otherwise */ -static int csiphy_stream_on(struct csiphy_device *csiphy) +static int csiphy_stream_on_legacy(struct csiphy_device *csiphy) { struct csiphy_config *cfg = &csiphy->cfg; s64 link_freq; @@ -306,11 +308,99 @@ static int csiphy_stream_on(struct csiphy_device *csiphy) * * Helper function to disable streaming on CSIPHY module */ -static void csiphy_stream_off(struct csiphy_device *csiphy) +static void csiphy_stream_off_legacy(struct csiphy_device *csiphy) { csiphy->res->hw_ops->lanes_disable(csiphy, &csiphy->cfg); } +/* + * csiphy_stream_on - Enable streaming on CSIPHY module + * @csiphy: CSIPHY device + * + * Helper function to enable streaming on CSIPHY module. + * Main configuration of CSIPHY module is also done here. + * + * Return 0 on success or a negative error code otherwise + */ +static int csiphy_stream_on(struct csiphy_device *csiphy) +{ + u8 bpp = csiphy_get_bpp(csiphy->res->formats->formats, csiphy->res->formats->nformats, + csiphy->fmt[MSM_CSIPHY_PAD_SINK].code); + struct csiphy_lanes_cfg *lncfg = &csiphy->cfg.csi2->lane_cfg; + struct phy_configure_opts_mipi_dphy *dphy_cfg; + union phy_configure_opts dphy_opts = { 0 }; + struct device *dev = csiphy->camss->dev; + u8 num_lanes = lncfg->num_data; + s64 link_freq; + int i; + int ret; + + dphy_cfg = &dphy_opts.mipi_dphy; + + link_freq = camss_get_link_freq(&csiphy->subdev.entity, bpp, num_lanes); + + if (link_freq < 0) { + dev_err(dev, + "Cannot get CSI2 transmitter's link frequency\n"); + return -EINVAL; + } + + phy_mipi_dphy_get_default_config_for_hsclk(link_freq, num_lanes, dphy_cfg); + + /* Set clock lane id and polarity */ + dphy_cfg->clock_lane_position = lncfg->clk.pos; + dphy_cfg->clock_lane_polarity = lncfg->clk.pol; + + /* Set data lane_mask and lane_polarities */ + for (i = 0; i < num_lanes; i++) { + dphy_cfg->lane_positions[i] = lncfg->data[i].pos; + dphy_cfg->lane_polarities[i] = lncfg->data[i].pol; + } + + phy_set_mode(csiphy->phy, PHY_MODE_MIPI_DPHY); + + ret = phy_configure(csiphy->phy, &dphy_opts); + if (ret) { + dev_err(dev, "failed to configure MIPI D-PHY\n"); + goto error; + } + + return phy_power_on(csiphy->phy); + +error: + return ret; +} + +/* + * csiphy_stream_off - Disable streaming on CSIPHY module + * @csiphy: CSIPHY device + * + * Helper function to disable streaming on CSIPHY module + */ +static void csiphy_stream_off(struct csiphy_device *csiphy) +{ + phy_power_off(csiphy->phy); +} + +/* + * csiphy_set_stream - Enable/disable streaming on CSIPHY module + * @sd: CSIPHY V4L2 subdevice + * @enable: Requested streaming state + * + * Return 0 on success or a negative error code otherwise + */ +static int csiphy_set_stream_legacy(struct v4l2_subdev *sd, int enable) +{ + struct csiphy_device *csiphy = v4l2_get_subdevdata(sd); + int ret = 0; + + if (enable) + ret = csiphy_stream_on_legacy(csiphy); + else + csiphy_stream_off_legacy(csiphy); + + return ret; +} /* * csiphy_set_stream - Enable/disable streaming on CSIPHY module @@ -568,16 +658,16 @@ static bool csiphy_match_clock_name(const char *clock_name, const char *format, } /* - * msm_csiphy_subdev_init - Initialize CSIPHY device structure and resources + * msm_csiphy_subdev_init_legacy - Initialize CSIPHY device structure and resources * @csiphy: CSIPHY device * @res: CSIPHY module resources table * @id: CSIPHY module id * * Return 0 on success or a negative error code otherwise */ -int msm_csiphy_subdev_init(struct camss *camss, - struct csiphy_device *csiphy, - const struct camss_subdev_resources *res, u8 id) +int msm_csiphy_subdev_init_legacy(struct camss *camss, + struct csiphy_device *csiphy, + const struct camss_subdev_resources *res, u8 id) { struct device *dev = camss->dev; struct platform_device *pdev = to_platform_device(dev); @@ -600,6 +690,7 @@ int msm_csiphy_subdev_init(struct camss *camss, return PTR_ERR(csiphy->base); if (camss->res->version == CAMSS_8x16 || + camss->res->version == CAMSS_8x39 || camss->res->version == CAMSS_8x53 || camss->res->version == CAMSS_8x96) { csiphy->base_clk_mux = @@ -694,24 +785,67 @@ int msm_csiphy_subdev_init(struct camss *camss, /* CSIPHY supplies */ for (i = 0; i < ARRAY_SIZE(res->regulators); i++) { - if (res->regulators[i]) + if (res->regulators[i].supply) csiphy->num_supplies++; } - if (csiphy->num_supplies) { - csiphy->supplies = devm_kmalloc_array(camss->dev, - csiphy->num_supplies, - sizeof(*csiphy->supplies), - GFP_KERNEL); - if (!csiphy->supplies) - return -ENOMEM; + if (csiphy->num_supplies > 0) + ret = devm_regulator_bulk_get_const(camss->dev, csiphy->num_supplies, + res->regulators, &csiphy->supplies); + return ret; +} + +/* + * msm_csiphy_subdev_init - Initialize CSIPHY device structure and resources + * @csiphy: CSIPHY device + * @res: CSIPHY module resources table + * @id: CSIPHY module id + * + * Return 0 on success or a negative error code otherwise + */ +int msm_csiphy_subdev_init(struct camss *camss, + struct csiphy_device *csiphy, + const struct camss_subdev_resources *res, u8 id) +{ + struct device *dev = camss->dev; + struct of_phandle_args args; + int idx; + int ret; + + snprintf(csiphy->name, ARRAY_SIZE(csiphy->name), "csiphy%d", id); + + idx = of_property_match_string(dev->of_node, "phy-names", csiphy->name); + if (idx < 0) { + dev_err(dev, "%s not found\n", csiphy->name); + return idx; } - for (i = 0; i < csiphy->num_supplies; i++) - csiphy->supplies[i].supply = res->regulators[i]; + ret = of_parse_phandle_with_args(dev->of_node, "phys", "#phy-cells", idx, &args); + if (ret < 0) { + dev_err(dev, "unable to parse phys args %s\n", csiphy->name); + return ret; + } + + if (!of_device_is_available(args.np)) + goto put_np; + + csiphy->phy = devm_phy_get(dev, csiphy->name); + if (IS_ERR(csiphy->phy)) { + ret = PTR_ERR(csiphy->phy); + goto put_np; + } + + csiphy->camss = camss; + csiphy->id = id; + csiphy->res = &res->csiphy; + + ret = phy_init(csiphy->phy); + if (ret) + dev_err(dev, "phy %s init fail %d\n", csiphy->name, ret); + +put_np: + of_node_put(args.np); - ret = devm_regulator_bulk_get(camss->dev, csiphy->num_supplies, - csiphy->supplies); return ret; } @@ -749,8 +883,12 @@ static int csiphy_link_setup(struct media_entity *entity, return 0; } -static const struct v4l2_subdev_core_ops csiphy_core_ops = { - .s_power = csiphy_set_power, +static const struct v4l2_subdev_core_ops csiphy_core_ops_legacy = { + .s_power = csiphy_set_power_legacy, +}; + +static const struct v4l2_subdev_video_ops csiphy_video_ops_legacy = { + .s_stream = csiphy_set_stream_legacy, }; static const struct v4l2_subdev_video_ops csiphy_video_ops = { @@ -764,8 +902,13 @@ static const struct v4l2_subdev_pad_ops csiphy_pad_ops = { .set_fmt = csiphy_set_format, }; +static const struct v4l2_subdev_ops csiphy_v4l2_ops_legacy = { + .core = &csiphy_core_ops_legacy, + .video = &csiphy_video_ops_legacy, + .pad = &csiphy_pad_ops, +}; + static const struct v4l2_subdev_ops csiphy_v4l2_ops = { - .core = &csiphy_core_ops, .video = &csiphy_video_ops, .pad = &csiphy_pad_ops, }; @@ -794,11 +937,16 @@ int msm_csiphy_register_entity(struct csiphy_device *csiphy, struct device *dev = csiphy->camss->dev; int ret; - v4l2_subdev_init(sd, &csiphy_v4l2_ops); + if (IS_ERR(csiphy->phy)) + v4l2_subdev_init(sd, &csiphy_v4l2_ops_legacy); + else + v4l2_subdev_init(sd, &csiphy_v4l2_ops); + sd->internal_ops = &csiphy_v4l2_internal_ops; sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d", MSM_CSIPHY_NAME, csiphy->id); + sd->grp_id = CSIPHY_GRP_ID; v4l2_set_subdevdata(sd, csiphy); ret = csiphy_init_formats(sd, NULL); @@ -833,6 +981,8 @@ int msm_csiphy_register_entity(struct csiphy_device *csiphy, */ void msm_csiphy_unregister_entity(struct csiphy_device *csiphy) { + if (!IS_ERR(csiphy->phy)) + phy_exit(csiphy->phy); v4l2_device_unregister_subdev(&csiphy->subdev); media_entity_cleanup(&csiphy->subdev.entity); } diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.h b/drivers/media/platform/qcom/camss/camss-csiphy.h index 895f80003c441..1879826034aac 100644 --- a/drivers/media/platform/qcom/camss/camss-csiphy.h +++ b/drivers/media/platform/qcom/camss/camss-csiphy.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -21,6 +22,8 @@ #define MSM_CSIPHY_PAD_SRC 1 #define MSM_CSIPHY_PADS_NUM 2 +#define CSIPHY_GRP_ID 1 + struct csiphy_lane { u8 pos; u8 pol; @@ -90,10 +93,12 @@ struct csiphy_device_regs { const struct csiphy_lane_regs *lane_regs; int lane_array_size; u32 offset; + u32 common_status_offset; }; struct csiphy_device { struct camss *camss; + struct phy *phy; u8 id; struct v4l2_subdev subdev; struct media_pad pads[MSM_CSIPHY_PADS_NUM]; @@ -101,6 +106,7 @@ struct csiphy_device { void __iomem *base_clk_mux; u32 irq; char irq_name[30]; + char name[16]; struct camss_clock *clock; bool *rate_set; int nclocks; @@ -115,6 +121,10 @@ struct csiphy_device { struct camss_subdev_resources; +int msm_csiphy_subdev_init_legacy(struct camss *camss, + struct csiphy_device *csiphy, + const struct camss_subdev_resources *res, u8 id); + int msm_csiphy_subdev_init(struct camss *camss, struct csiphy_device *csiphy, const struct camss_subdev_resources *res, u8 id); diff --git a/drivers/media/platform/qcom/camss/camss-ispif.c b/drivers/media/platform/qcom/camss/camss-ispif.c index 2dc585c6123dd..aaf3caa42d33d 100644 --- a/drivers/media/platform/qcom/camss/camss-ispif.c +++ b/drivers/media/platform/qcom/camss/camss-ispif.c @@ -1112,6 +1112,8 @@ int msm_ispif_subdev_init(struct camss *camss, /* Number of ISPIF lines - same as number of CSID hardware modules */ if (camss->res->version == CAMSS_8x16) ispif->line_num = 2; + else if (camss->res->version == CAMSS_8x39) + ispif->line_num = 3; else if (camss->res->version == CAMSS_8x96 || camss->res->version == CAMSS_8x53 || camss->res->version == CAMSS_660) @@ -1128,7 +1130,8 @@ int msm_ispif_subdev_init(struct camss *camss, ispif->line[i].ispif = ispif; ispif->line[i].id = i; - if (camss->res->version == CAMSS_8x16) { + if (camss->res->version == CAMSS_8x16 || + camss->res->version == CAMSS_8x39) { ispif->line[i].formats = ispif_formats_8x16; ispif->line[i].nformats = ARRAY_SIZE(ispif_formats_8x16); @@ -1162,7 +1165,8 @@ int msm_ispif_subdev_init(struct camss *camss, ispif->irq = ret; snprintf(ispif->irq_name, sizeof(ispif->irq_name), "%s_%s", dev_name(dev), MSM_ISPIF_NAME); - if (camss->res->version == CAMSS_8x16) + if (camss->res->version == CAMSS_8x16 || + camss->res->version == CAMSS_8x39) ret = devm_request_irq(dev, ispif->irq, ispif_isr_8x16, IRQF_TRIGGER_RISING, ispif->irq_name, ispif); else if (camss->res->version == CAMSS_8x96 || diff --git a/drivers/media/platform/qcom/camss/camss-tpg-gen1.c b/drivers/media/platform/qcom/camss/camss-tpg-gen1.c new file mode 100644 index 0000000000000..d29de5f93c18e --- /dev/null +++ b/drivers/media/platform/qcom/camss/camss-tpg-gen1.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Qualcomm MSM Camera Subsystem - TPG (Test Pattern Generator) Module + * + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#include +#include + +#include "camss-tpg.h" +#include "camss.h" + +/* TPG global registers */ +#define TPG_HW_VERSION 0x0 +# define HW_VERSION_STEPPING GENMASK(15, 0) +# define HW_VERSION_REVISION GENMASK(27, 16) +# define HW_VERSION_GENERATION GENMASK(31, 28) + +#define TPG_HW_VER(gen, rev, step) \ + (((u32)(gen) << 28) | ((u32)(rev) << 16) | (u32)(step)) + +#define TPG_HW_VER_2_0_0 TPG_HW_VER(2, 0, 0) +#define TPG_HW_VER_2_1_0 TPG_HW_VER(2, 1, 0) + +#define TPG_HW_STATUS 0x4 + +#define TPG_CTRL 0x64 +# define TPG_CTRL_TEST_EN BIT(0) +# define TPG_CTRL_PHY_SEL BIT(3) +# define TPG_CTRL_NUM_ACTIVE_LANES GENMASK(5, 4) +# define TPG_CTRL_VC_DT_PATTERN_ID GENMASK(8, 6) +# define TPG_CTRL_OVERLAP_SHDR_EN BIT(10) +# define TPG_CTRL_NUM_ACTIVE_VC GENMASK(31, 30) + +#define TPG_CLEAR 0x1F4 + +/* TPG VC-based registers */ +#define TPG_VC_n_GAIN_CFG(n) (0x60 + (n) * 0x60) + +#define TPG_VC_n_CFG0(n) (0x68 + (n) * 0x60) +# define TPG_VC_n_CFG0_VC_NUM GENMASK(4, 0) +# define TPG_VC_n_CFG0_NUM_ACTIVE_DT GENMASK(9, 8) +# define TPG_VC_n_CFG0_NUM_BATCH GENMASK(15, 12) +# define TPG_VC_n_CFG0_NUM_FRAMES GENMASK(31, 16) + +#define TPG_VC_n_LSFR_SEED(n) (0x6C + (n) * 0x60) +#define TPG_VC_n_HBI_CFG(n) (0x70 + (n) * 0x60) +#define TPG_VC_n_VBI_CFG(n) (0x74 + (n) * 0x60) + +#define TPG_VC_n_COLOR_BARS_CFG(n) (0x78 + (n) * 0x60) +# define TPG_VC_n_COLOR_BARS_CFG_PIX_PATTERN GENMASK(2, 0) +# define TPG_VC_n_COLOR_BARS_CFG_QCFA_EN BIT(3) +# define TPG_VC_n_COLOR_BARS_CFG_SPLIT_EN BIT(4) +# define TPG_VC_n_COLOR_BARS_CFG_NOISE_EN BIT(5) +# define TPG_VC_n_COLOR_BARS_CFG_ROTATE_PERIOD GENMASK(13, 8) +# define TPG_VC_n_COLOR_BARS_CFG_XCFA_EN BIT(16) +# define TPG_VC_n_COLOR_BARS_CFG_SIZE_X GENMASK(26, 24) +# define TPG_VC_n_COLOR_BARS_CFG_SIZE_Y GENMASK(30, 28) + +/* TPG DT-based registers */ +#define TPG_VC_m_DT_n_CFG_0(m, n) (0x7C + (m) * 0x60 + (n) * 0xC) +# define TPG_VC_m_DT_n_CFG_0_FRAME_HEIGHT GENMASK(15, 0) +# define TPG_VC_m_DT_n_CFG_0_FRAME_WIDTH GENMASK(31, 16) + +#define TPG_VC_m_DT_n_CFG_1(m, n) (0x80 + (m) * 0x60 + (n) * 0xC) +# define TPG_VC_m_DT_n_CFG_1_DATA_TYPE GENMASK(5, 0) +# define TPG_VC_m_DT_n_CFG_1_ECC_XOR_MASK GENMASK(13, 8) +# define TPG_VC_m_DT_n_CFG_1_CRC_XOR_MASK GENMASK(31, 16) + +#define TPG_VC_m_DT_n_CFG_2(m, n) (0x84 + (m) * 0x60 + (n) * 0xC) +# define TPG_VC_m_DT_n_CFG_2_PAYLOAD_MODE GENMASK(3, 0) +/* v2.0.0: USER[19:4], ENC[23:20] */ +# define TPG_V2_0_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD GENMASK(19, 4) +# define TPG_V2_0_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT GENMASK(23, 20) +/* v2.1.0: USER[27:4], ENC[31:28] */ +# define TPG_V2_1_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD GENMASK(27, 4) +# define TPG_V2_1_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT GENMASK(31, 28) + +#define TPG_HBI_PCT_DEFAULT 545 /* 545% */ +#define TPG_VBI_PCT_DEFAULT 10 /* 10% */ +#define PERCENT_BASE 100 + +/* Default user-specified payload for TPG test generator. + * Keep consistent with CSID TPG default: 0xBE. + */ +#define TPG_USER_SPECIFIED_PAYLOAD_DEFAULT 0xBE +#define TPG_LFSR_SEED_DEFAULT 0x12345678 +#define TPG_COLOR_BARS_CFG_STANDARD \ + FIELD_PREP(TPG_VC_n_COLOR_BARS_CFG_ROTATE_PERIOD, 0xA) + +static const char * const testgen_payload_modes[] = { + [TPG_PAYLOAD_MODE_DISABLED] = "Disabled", + [TPG_PAYLOAD_MODE_INCREMENTING] = "Incrementing", + [TPG_PAYLOAD_MODE_ALTERNATING_55_AA] = "Alternating 0x55/0xAA", + [TPG_PAYLOAD_MODE_RANDOM] = "Pseudo-random Data", + [TPG_PAYLOAD_MODE_USER_SPECIFIED] = "User Specified", + [TPG_PAYLOAD_MODE_COLOR_BARS] = "Color bars", +}; + +static int tpg_stream_on(struct tpg_device *tpg) +{ + struct tpg_testgen_config *tg = &tpg->testgen; + struct v4l2_mbus_framefmt *input_format; + const struct tpg_format_info *format; + u8 payload_mode = (tg->mode > TPG_PAYLOAD_MODE_DISABLED) ? + tg->mode - 1 : 0; + u8 lane_cnt = tpg->res->lane_cnt; + u8 vc, dt, last_vc = 0; + u32 val; + + for (vc = 0; vc <= MSM_TPG_ACTIVE_VC; vc++) { + last_vc = vc; + + input_format = &tpg->fmt; + format = tpg_get_fmt_entry(tpg->res->formats->formats, + tpg->res->formats->nformats, + input_format->code); + if (IS_ERR(format)) + return -EINVAL; + + /* VC configuration */ + val = FIELD_PREP(TPG_VC_n_CFG0_NUM_ACTIVE_DT, MSM_TPG_ACTIVE_DT) | + FIELD_PREP(TPG_VC_n_CFG0_NUM_FRAMES, 0); + writel(val, tpg->base + TPG_VC_n_CFG0(vc)); + + writel(TPG_LFSR_SEED_DEFAULT, tpg->base + TPG_VC_n_LSFR_SEED(vc)); + + val = DIV_ROUND_UP(input_format->width * format->bpp * TPG_HBI_PCT_DEFAULT, + BITS_PER_BYTE * lane_cnt * PERCENT_BASE); + writel(val, tpg->base + TPG_VC_n_HBI_CFG(vc)); + + val = input_format->height * TPG_VBI_PCT_DEFAULT / PERCENT_BASE; + writel(val, tpg->base + TPG_VC_n_VBI_CFG(vc)); + + writel(TPG_COLOR_BARS_CFG_STANDARD, tpg->base + TPG_VC_n_COLOR_BARS_CFG(vc)); + + /* DT configuration */ + for (dt = 0; dt <= MSM_TPG_ACTIVE_DT; dt++) { + val = FIELD_PREP(TPG_VC_m_DT_n_CFG_0_FRAME_HEIGHT, + input_format->height & 0xffff) | + FIELD_PREP(TPG_VC_m_DT_n_CFG_0_FRAME_WIDTH, + input_format->width & 0xffff); + writel(val, tpg->base + TPG_VC_m_DT_n_CFG_0(vc, dt)); + + val = FIELD_PREP(TPG_VC_m_DT_n_CFG_1_DATA_TYPE, format->data_type); + writel(val, tpg->base + TPG_VC_m_DT_n_CFG_1(vc, dt)); + + if (tpg->hw_version == TPG_HW_VER_2_0_0) { + val = FIELD_PREP(TPG_VC_m_DT_n_CFG_2_PAYLOAD_MODE, payload_mode) | + FIELD_PREP(TPG_V2_0_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD, + TPG_USER_SPECIFIED_PAYLOAD_DEFAULT) | + FIELD_PREP(TPG_V2_0_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT, + format->encode_format); + } else if (tpg->hw_version >= TPG_HW_VER_2_1_0) { + val = FIELD_PREP(TPG_VC_m_DT_n_CFG_2_PAYLOAD_MODE, payload_mode) | + FIELD_PREP(TPG_V2_1_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD, + TPG_USER_SPECIFIED_PAYLOAD_DEFAULT) | + FIELD_PREP(TPG_V2_1_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT, + format->encode_format); + } + writel(val, tpg->base + TPG_VC_m_DT_n_CFG_2(vc, dt)); + } + } + + /* Global TPG control */ + val = FIELD_PREP(TPG_CTRL_TEST_EN, 1) | + FIELD_PREP(TPG_CTRL_NUM_ACTIVE_LANES, lane_cnt - 1) | + FIELD_PREP(TPG_CTRL_NUM_ACTIVE_VC, last_vc); + writel(val, tpg->base + TPG_CTRL); + + return 0; +} + +static int tpg_reset(struct tpg_device *tpg) +{ + writel(0, tpg->base + TPG_CTRL); + writel(1, tpg->base + TPG_CLEAR); + + return 0; +} + +static void tpg_stream_off(struct tpg_device *tpg) +{ + tpg_reset(tpg); +} + +static int tpg_configure_stream(struct tpg_device *tpg, u8 enable) +{ + if (enable) + return tpg_stream_on(tpg); + + tpg_stream_off(tpg); + + return 0; +} + +static int tpg_configure_testgen_pattern(struct tpg_device *tpg, s32 val) +{ + if (val >= 0 && val <= TPG_PAYLOAD_MODE_COLOR_BARS) + tpg->testgen.mode = val; + + return 0; +} + +static u32 tpg_hw_version(struct tpg_device *tpg) +{ + u32 hw_version = readl(tpg->base + TPG_HW_VERSION); + + tpg->hw_version = hw_version; + dev_dbg(tpg->camss->dev, "tpg HW Version = %u.%u.%u\n", + (u32)FIELD_GET(HW_VERSION_GENERATION, hw_version), + (u32)FIELD_GET(HW_VERSION_REVISION, hw_version), + (u32)FIELD_GET(HW_VERSION_STEPPING, hw_version)); + + return hw_version; +} + +static void tpg_subdev_init(struct tpg_device *tpg) +{ + tpg->testgen.modes = testgen_payload_modes; + tpg->testgen.nmodes = TPG_PAYLOAD_MODE_NUM_SUPPORTED_GEN1; +} + +const struct tpg_hw_ops tpg_ops_gen1 = { + .configure_stream = tpg_configure_stream, + .configure_testgen_pattern = tpg_configure_testgen_pattern, + .hw_version = tpg_hw_version, + .reset = tpg_reset, + .subdev_init = tpg_subdev_init, +}; diff --git a/drivers/media/platform/qcom/camss/camss-tpg.c b/drivers/media/platform/qcom/camss/camss-tpg.c new file mode 100644 index 0000000000000..c5b75132add44 --- /dev/null +++ b/drivers/media/platform/qcom/camss/camss-tpg.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Qualcomm MSM Camera Subsystem - TPG Module + * + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "camss-tpg.h" +#include "camss.h" + +static const struct tpg_format_info formats_gen1[] = { + { + MEDIA_BUS_FMT_SBGGR8_1X8, + MIPI_CSI2_DT_RAW8, + ENCODE_FORMAT_UNCOMPRESSED_8_BIT, + 8, + }, + { + MEDIA_BUS_FMT_SGBRG8_1X8, + MIPI_CSI2_DT_RAW8, + ENCODE_FORMAT_UNCOMPRESSED_8_BIT, + 8, + }, + { + MEDIA_BUS_FMT_SGRBG8_1X8, + MIPI_CSI2_DT_RAW8, + ENCODE_FORMAT_UNCOMPRESSED_8_BIT, + 8, + }, + { + MEDIA_BUS_FMT_SRGGB8_1X8, + MIPI_CSI2_DT_RAW8, + ENCODE_FORMAT_UNCOMPRESSED_8_BIT, + 8, + }, + { + MEDIA_BUS_FMT_SBGGR10_1X10, + MIPI_CSI2_DT_RAW10, + ENCODE_FORMAT_UNCOMPRESSED_10_BIT, + 10, + }, + { + MEDIA_BUS_FMT_SGBRG10_1X10, + MIPI_CSI2_DT_RAW10, + ENCODE_FORMAT_UNCOMPRESSED_10_BIT, + 10, + }, + { + MEDIA_BUS_FMT_SGRBG10_1X10, + MIPI_CSI2_DT_RAW10, + ENCODE_FORMAT_UNCOMPRESSED_10_BIT, + 10, + }, + { + MEDIA_BUS_FMT_SRGGB10_1X10, + MIPI_CSI2_DT_RAW10, + ENCODE_FORMAT_UNCOMPRESSED_10_BIT, + 10, + }, + { + MEDIA_BUS_FMT_SBGGR12_1X12, + MIPI_CSI2_DT_RAW12, + ENCODE_FORMAT_UNCOMPRESSED_12_BIT, + 12, + }, + { + MEDIA_BUS_FMT_SGBRG12_1X12, + MIPI_CSI2_DT_RAW12, + ENCODE_FORMAT_UNCOMPRESSED_12_BIT, + 12, + }, + { + MEDIA_BUS_FMT_SGRBG12_1X12, + MIPI_CSI2_DT_RAW12, + ENCODE_FORMAT_UNCOMPRESSED_12_BIT, + 12, + }, + { + MEDIA_BUS_FMT_SRGGB12_1X12, + MIPI_CSI2_DT_RAW12, + ENCODE_FORMAT_UNCOMPRESSED_12_BIT, + 12, + }, + { + MEDIA_BUS_FMT_Y8_1X8, + MIPI_CSI2_DT_RAW8, + ENCODE_FORMAT_UNCOMPRESSED_8_BIT, + 8, + }, + { + MEDIA_BUS_FMT_Y10_1X10, + MIPI_CSI2_DT_RAW10, + ENCODE_FORMAT_UNCOMPRESSED_10_BIT, + 10, + }, +}; + +const struct tpg_formats tpg_formats_gen1 = { + .nformats = ARRAY_SIZE(formats_gen1), + .formats = formats_gen1 +}; + +const struct tpg_format_info *tpg_get_fmt_entry(const struct tpg_format_info *formats, + unsigned int nformats, + u32 code) +{ + unsigned int i; + + for (i = 0; i < nformats; i++) + if (code == formats[i].code) + return &formats[i]; + + return ERR_PTR(-EINVAL); +} + +static int tpg_set_clock_rates(struct tpg_device *tpg) +{ + struct device *dev = tpg->camss->dev; + int i, ret; + + for (i = 0; i < tpg->nclocks; i++) { + struct camss_clock *clock = &tpg->clock[i]; + long round_rate; + + if (clock->freq) { + round_rate = clk_round_rate(clock->clk, clock->freq[0]); + if (round_rate < 0) { + dev_err(dev, "clk round rate failed: %ld\n", + round_rate); + return -EINVAL; + } + + ret = clk_set_rate(clock->clk, round_rate); + if (ret < 0) { + dev_err(dev, "clk set rate failed: %d\n", ret); + return ret; + } + } + } + + return 0; +} + +static int tpg_set_power(struct v4l2_subdev *sd, int on) +{ + struct tpg_device *tpg = v4l2_get_subdevdata(sd); + struct device *dev = tpg->camss->dev; + + if (on) { + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; + + ret = tpg_set_clock_rates(tpg); + if (ret < 0) { + pm_runtime_put_sync(dev); + return ret; + } + + ret = camss_enable_clocks(tpg->nclocks, tpg->clock, dev); + if (ret < 0) { + pm_runtime_put_sync(dev); + return ret; + } + + tpg->res->hw_ops->reset(tpg); + + tpg->res->hw_ops->hw_version(tpg); + } else { + camss_disable_clocks(tpg->nclocks, tpg->clock); + + pm_runtime_put_sync(dev); + } + + return 0; +} + +static int tpg_set_stream(struct v4l2_subdev *sd, int enable) +{ + struct tpg_device *tpg = v4l2_get_subdevdata(sd); + int ret; + + if (enable) { + ret = v4l2_ctrl_handler_setup(&tpg->ctrls); + if (ret < 0) { + dev_err(tpg->camss->dev, + "could not sync v4l2 controls: %d\n", ret); + return ret; + } + } + + return tpg->res->hw_ops->configure_stream(tpg, enable); +} + +static struct v4l2_mbus_framefmt * +__tpg_get_format(struct tpg_device *tpg, + struct v4l2_subdev_state *sd_state, + unsigned int pad, + enum v4l2_subdev_format_whence which) +{ + if (which == V4L2_SUBDEV_FORMAT_TRY) + return v4l2_subdev_state_get_format(sd_state, + pad); + + return &tpg->fmt; +} + +static void tpg_try_format(struct tpg_device *tpg, + struct v4l2_mbus_framefmt *fmt) +{ + unsigned int i; + + for (i = 0; i < tpg->res->formats->nformats; i++) + if (tpg->res->formats->formats[i].code == fmt->code) + break; + + if (i >= tpg->res->formats->nformats) + fmt->code = MEDIA_BUS_FMT_SBGGR8_1X8; + + fmt->width = clamp_t(u32, fmt->width, TPG_MIN_WIDTH, TPG_MAX_WIDTH); + fmt->height = clamp_t(u32, fmt->height, TPG_MIN_HEIGHT, TPG_MAX_HEIGHT); + fmt->field = V4L2_FIELD_NONE; + fmt->colorspace = V4L2_COLORSPACE_SRGB; +} + +static int tpg_enum_mbus_code(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_mbus_code_enum *code) +{ + struct tpg_device *tpg = v4l2_get_subdevdata(sd); + + if (code->index >= tpg->res->formats->nformats) + return -EINVAL; + + code->code = tpg->res->formats->formats[code->index].code; + + return 0; +} + +static int tpg_enum_frame_size(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_frame_size_enum *fse) +{ + struct tpg_device *tpg = v4l2_get_subdevdata(sd); + unsigned int i; + + if (fse->index != 0) + return -EINVAL; + + for (i = 0; i < tpg->res->formats->nformats; i++) + if (tpg->res->formats->formats[i].code == fse->code) + break; + + if (i >= tpg->res->formats->nformats) + return -EINVAL; + + fse->min_width = TPG_MIN_WIDTH; + fse->min_height = TPG_MIN_HEIGHT; + fse->max_width = TPG_MAX_WIDTH; + fse->max_height = TPG_MAX_HEIGHT; + + return 0; +} + +static int tpg_get_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) +{ + struct tpg_device *tpg = v4l2_get_subdevdata(sd); + struct v4l2_mbus_framefmt *format; + + format = __tpg_get_format(tpg, sd_state, fmt->pad, fmt->which); + if (!format) + return -EINVAL; + + fmt->format = *format; + + return 0; +} + +static int tpg_set_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) +{ + struct tpg_device *tpg = v4l2_get_subdevdata(sd); + struct v4l2_mbus_framefmt *format; + + format = __tpg_get_format(tpg, sd_state, fmt->pad, fmt->which); + if (!format) + return -EINVAL; + + tpg_try_format(tpg, &fmt->format); + *format = fmt->format; + + return 0; +} + +static int tpg_init_formats(struct v4l2_subdev *sd, + struct v4l2_subdev_fh *fh) +{ + struct v4l2_subdev_format format = { + .pad = MSM_TPG_PAD_SRC, + .which = fh ? V4L2_SUBDEV_FORMAT_TRY : + V4L2_SUBDEV_FORMAT_ACTIVE, + .format = { + .code = MEDIA_BUS_FMT_SBGGR8_1X8, + .width = 1920, + .height = 1080, + } + }; + + return tpg_set_format(sd, fh ? fh->state : NULL, &format); +} + +static int tpg_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct tpg_device *tpg = container_of(ctrl->handler, + struct tpg_device, ctrls); + int ret = -EINVAL; + + switch (ctrl->id) { + case V4L2_CID_TEST_PATTERN: + ret = tpg->res->hw_ops->configure_testgen_pattern(tpg, ctrl->val); + break; + } + + return ret; +} + +static const struct v4l2_ctrl_ops tpg_ctrl_ops = { + .s_ctrl = tpg_s_ctrl, +}; + +int msm_tpg_subdev_init(struct camss *camss, + struct tpg_device *tpg, + const struct camss_subdev_resources *res, u8 id) +{ + struct platform_device *pdev; + struct device *dev; + int i, j; + + dev = camss->dev; + pdev = to_platform_device(dev); + + tpg->camss = camss; + tpg->id = id; + tpg->res = &res->tpg; + tpg->res->hw_ops->subdev_init(tpg); + + tpg->base = devm_platform_ioremap_resource_byname(pdev, res->reg[0]); + if (IS_ERR(tpg->base)) + return PTR_ERR(tpg->base); + + tpg->nclocks = 0; + while (res->clock[tpg->nclocks]) + tpg->nclocks++; + + if (!tpg->nclocks) + return 0; + + tpg->clock = devm_kcalloc(dev, tpg->nclocks, + sizeof(*tpg->clock), GFP_KERNEL); + if (!tpg->clock) + return -ENOMEM; + + for (i = 0; i < tpg->nclocks; i++) { + struct camss_clock *clock = &tpg->clock[i]; + + clock->clk = devm_clk_get(dev, res->clock[i]); + if (IS_ERR(clock->clk)) + return PTR_ERR(clock->clk); + + clock->name = res->clock[i]; + + clock->nfreqs = 0; + while (res->clock_rate[i][clock->nfreqs]) + clock->nfreqs++; + + if (!clock->nfreqs) { + clock->freq = NULL; + continue; + } + + clock->freq = devm_kcalloc(dev, clock->nfreqs, + sizeof(*clock->freq), GFP_KERNEL); + if (!clock->freq) + return -ENOMEM; + + for (j = 0; j < clock->nfreqs; j++) + clock->freq[j] = res->clock_rate[i][j]; + } + + return 0; +} + +static int tpg_link_setup(struct media_entity *entity, + const struct media_pad *local, + const struct media_pad *remote, u32 flags) +{ + if (flags & MEDIA_LNK_FL_ENABLED) + if (media_pad_remote_pad_first(local)) + return -EBUSY; + + return 0; +} + +static const struct v4l2_subdev_core_ops tpg_core_ops = { + .s_power = tpg_set_power, +}; + +static const struct v4l2_subdev_video_ops tpg_video_ops = { + .s_stream = tpg_set_stream, +}; + +static const struct v4l2_subdev_pad_ops tpg_pad_ops = { + .enum_mbus_code = tpg_enum_mbus_code, + .enum_frame_size = tpg_enum_frame_size, + .get_fmt = tpg_get_format, + .set_fmt = tpg_set_format, +}; + +static const struct v4l2_subdev_ops tpg_v4l2_ops = { + .core = &tpg_core_ops, + .video = &tpg_video_ops, + .pad = &tpg_pad_ops, +}; + +static const struct v4l2_subdev_internal_ops tpg_v4l2_internal_ops = { + .open = tpg_init_formats, +}; + +static const struct media_entity_operations tpg_media_ops = { + .link_setup = tpg_link_setup, + .link_validate = v4l2_subdev_link_validate, +}; + +int msm_tpg_register_entity(struct tpg_device *tpg, + struct v4l2_device *v4l2_dev) +{ + struct v4l2_subdev *sd = &tpg->subdev; + struct device *dev = tpg->camss->dev; + int ret; + + v4l2_subdev_init(sd, &tpg_v4l2_ops); + sd->internal_ops = &tpg_v4l2_internal_ops; + sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | + V4L2_SUBDEV_FL_HAS_EVENTS; + snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d", + "msm_tpg", tpg->id); + sd->grp_id = TPG_GRP_ID; + v4l2_set_subdevdata(sd, tpg); + + ret = v4l2_ctrl_handler_init(&tpg->ctrls, 1); + if (ret < 0) { + dev_err(dev, "Failed to init ctrl handler: %d\n", ret); + return ret; + } + + tpg->testgen_mode = v4l2_ctrl_new_std_menu_items(&tpg->ctrls, + &tpg_ctrl_ops, V4L2_CID_TEST_PATTERN, + tpg->testgen.nmodes, 0, 0, + tpg->testgen.modes); + if (tpg->ctrls.error) { + dev_err(dev, "Failed to init ctrl: %d\n", tpg->ctrls.error); + ret = tpg->ctrls.error; + goto free_ctrl; + } + + tpg->subdev.ctrl_handler = &tpg->ctrls; + + ret = tpg_init_formats(sd, NULL); + if (ret < 0) { + dev_err(dev, "Failed to init format: %d\n", ret); + goto free_ctrl; + } + + tpg->pad.flags = MEDIA_PAD_FL_SOURCE; + + sd->entity.ops = &tpg_media_ops; + ret = media_entity_pads_init(&sd->entity, 1, &tpg->pad); + if (ret < 0) { + dev_err(dev, "Failed to init media entity: %d\n", ret); + goto free_ctrl; + } + + ret = v4l2_device_register_subdev(v4l2_dev, sd); + if (ret < 0) { + dev_err(dev, "Failed to register subdev: %d\n", ret); + media_entity_cleanup(&sd->entity); + goto free_ctrl; + } + + return 0; + +free_ctrl: + v4l2_ctrl_handler_free(&tpg->ctrls); + + return ret; +} + +void msm_tpg_unregister_entity(struct tpg_device *tpg) +{ + v4l2_device_unregister_subdev(&tpg->subdev); + media_entity_cleanup(&tpg->subdev.entity); + v4l2_ctrl_handler_free(&tpg->ctrls); +} diff --git a/drivers/media/platform/qcom/camss/camss-tpg.h b/drivers/media/platform/qcom/camss/camss-tpg.h new file mode 100644 index 0000000000000..7fb35a97dd068 --- /dev/null +++ b/drivers/media/platform/qcom/camss/camss-tpg.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * camss-tpg.h + * + * Qualcomm MSM Camera Subsystem - TPG Module + * + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ +#ifndef QC_MSM_CAMSS_TPG_H +#define QC_MSM_CAMSS_TPG_H + +#include +#include +#include +#include +#include +#include +#include + +#define ENCODE_FORMAT_UNCOMPRESSED_8_BIT 0x1 +#define ENCODE_FORMAT_UNCOMPRESSED_10_BIT 0x2 +#define ENCODE_FORMAT_UNCOMPRESSED_12_BIT 0x3 +#define ENCODE_FORMAT_UNCOMPRESSED_14_BIT 0x4 +#define ENCODE_FORMAT_UNCOMPRESSED_16_BIT 0x5 +#define ENCODE_FORMAT_UNCOMPRESSED_20_BIT 0x6 +#define ENCODE_FORMAT_UNCOMPRESSED_24_BIT 0x7 + +#define MSM_TPG_PAD_SRC 0 +#define MSM_TPG_ACTIVE_VC 0 +#define MSM_TPG_ACTIVE_DT 0 + +#define TPG_MIN_WIDTH 1 +#define TPG_MIN_HEIGHT 1 +#define TPG_MAX_WIDTH 8191 +#define TPG_MAX_HEIGHT 8191 + +#define TPG_GRP_ID 0 + +enum tpg_testgen_mode { + TPG_PAYLOAD_MODE_DISABLED = 0, + TPG_PAYLOAD_MODE_INCREMENTING = 1, + TPG_PAYLOAD_MODE_ALTERNATING_55_AA = 2, + TPG_PAYLOAD_MODE_RANDOM = 5, + TPG_PAYLOAD_MODE_USER_SPECIFIED = 6, + TPG_PAYLOAD_MODE_COLOR_BARS = 9, + TPG_PAYLOAD_MODE_NUM_SUPPORTED_GEN1 = 9, +}; + +struct tpg_testgen_config { + enum tpg_testgen_mode mode; + const char * const*modes; + u8 nmodes; +}; + +struct tpg_format_info { + u32 code; + u8 data_type; + u8 encode_format; + u8 bpp; +}; + +struct tpg_formats { + unsigned int nformats; + const struct tpg_format_info *formats; +}; + +struct tpg_device; + +struct tpg_hw_ops { + int (*configure_stream)(struct tpg_device *tpg, u8 enable); + int (*configure_testgen_pattern)(struct tpg_device *tpg, s32 val); + u32 (*hw_version)(struct tpg_device *tpg); + int (*reset)(struct tpg_device *tpg); + void (*subdev_init)(struct tpg_device *tpg); +}; + +struct tpg_subdev_resources { + u8 lane_cnt; + const struct tpg_formats *formats; + const struct tpg_hw_ops *hw_ops; +}; + +struct tpg_device { + struct camss *camss; + u8 id; + struct v4l2_subdev subdev; + struct media_pad pad; + void __iomem *base; + struct camss_clock *clock; + int nclocks; + struct tpg_testgen_config testgen; + struct v4l2_mbus_framefmt fmt; + struct v4l2_ctrl_handler ctrls; + struct v4l2_ctrl *testgen_mode; + const struct tpg_subdev_resources *res; + u32 hw_version; +}; + +struct camss_subdev_resources; + +const struct tpg_format_info *tpg_get_fmt_entry(const struct tpg_format_info *formats, + unsigned int nformats, + u32 code); + +int msm_tpg_subdev_init(struct camss *camss, + struct tpg_device *tpg, + const struct camss_subdev_resources *res, u8 id); + +int msm_tpg_register_entity(struct tpg_device *tpg, + struct v4l2_device *v4l2_dev); + +void msm_tpg_unregister_entity(struct tpg_device *tpg); + +extern const struct tpg_formats tpg_formats_gen1; + +extern const struct tpg_hw_ops tpg_ops_gen1; + +#endif /* QC_MSM_CAMSS_TPG_H */ diff --git a/drivers/media/platform/qcom/camss/camss-vfe-4-1.c b/drivers/media/platform/qcom/camss/camss-vfe-4-1.c index 901677293d971..9cf1ccdb2fe7c 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe-4-1.c +++ b/drivers/media/platform/qcom/camss/camss-vfe-4-1.c @@ -15,6 +15,7 @@ #include "camss.h" #include "camss-vfe.h" #include "camss-vfe-gen1.h" +#include "camss-vfe-vbif.h" #define VFE_0_HW_VERSION 0x000 @@ -733,6 +734,7 @@ static void vfe_set_qos(struct vfe_device *vfe) { u32 val = VFE_0_BUS_BDG_QOS_CFG_0_CFG; u32 val7 = VFE_0_BUS_BDG_QOS_CFG_7_CFG; + int ret; writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_0); writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_1); @@ -742,6 +744,16 @@ static void vfe_set_qos(struct vfe_device *vfe) writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_5); writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_6); writel_relaxed(val7, vfe->base + VFE_0_BUS_BDG_QOS_CFG_7); + + /* SoC-specific VBIF settings */ + if (vfe->res->has_vbif) { + ret = vfe_vbif_apply_settings(vfe); + if (ret < 0) { + dev_err_ratelimited(vfe->camss->dev, + "VFE: VBIF error %d\n", + ret); + } + } } static void vfe_set_ds(struct vfe_device *vfe) diff --git a/drivers/media/platform/qcom/camss/camss-vfe-480.c b/drivers/media/platform/qcom/camss/camss-vfe-480.c index 4feea590a47bc..d73f733fde045 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe-480.c +++ b/drivers/media/platform/qcom/camss/camss-vfe-480.c @@ -202,11 +202,13 @@ static irqreturn_t vfe_isr(int irq, void *dev) writel_relaxed(status, vfe->base + VFE_BUS_IRQ_CLEAR(0)); writel_relaxed(1, vfe->base + VFE_BUS_IRQ_CLEAR_GLOBAL); - /* Loop through all WMs IRQs */ - for (i = 0; i < MSM_VFE_IMAGE_MASTERS_NUM; i++) { + for (i = 0; i < MAX_VFE_OUTPUT_LINES; i++) { if (status & BUS_IRQ_MASK_0_RDI_RUP(vfe, i)) vfe_isr_reg_update(vfe, i); + } + /* Loop through all WMs IRQs */ + for (i = 0; i < MSM_VFE_IMAGE_MASTERS_NUM; i++) { if (status & BUS_IRQ_MASK_0_COMP_DONE(vfe, RDI_COMP_GROUP(i))) vfe_buf_done(vfe, i); } diff --git a/drivers/media/platform/qcom/camss/camss-vfe-vbif.c b/drivers/media/platform/qcom/camss/camss-vfe-vbif.c new file mode 100644 index 0000000000000..911f8da02f1fb --- /dev/null +++ b/drivers/media/platform/qcom/camss/camss-vfe-vbif.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * camss-vfe-vbif.c + * + * Qualcomm MSM Camera Subsystem - VFE VBIF Module + * + * Copyright (c) 2025, The Linux Foundation. All rights reserved. + * + */ + +#include + +#include "camss.h" +#include "camss-vfe.h" +#include "camss-vfe-vbif.h" + +#define VBIF_FIXED_SORT_EN 0x30 +#define VBIF_FIXED_SORT_SEL0 0x34 + +void vfe_vbif_write_reg(struct vfe_device *vfe, u32 reg, u32 val) +{ + writel_relaxed(val, vfe->vbif_base + reg); +} + +int vfe_vbif_apply_settings(struct vfe_device *vfe) +{ + vfe_vbif_write_reg(vfe, VBIF_FIXED_SORT_EN, 0xfff); + vfe_vbif_write_reg(vfe, VBIF_FIXED_SORT_SEL0, 0x555000); + + return 0; +} diff --git a/drivers/media/platform/qcom/camss/camss-vfe-vbif.h b/drivers/media/platform/qcom/camss/camss-vfe-vbif.h new file mode 100644 index 0000000000000..502db629e961f --- /dev/null +++ b/drivers/media/platform/qcom/camss/camss-vfe-vbif.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * camss-vfe-vbif.h + * + * Qualcomm MSM Camera Subsystem - VFE VBIF Module + * + * Copyright (c) 2025, The Linux Foundation. All rights reserved. + * + */ +#ifndef QC_MSM_CAMSS_VFE_VBIF_H +#define QC_MSM_CAMSS_VFE_VBIF_H + +#include "camss-vfe.h" + +void vfe_vbif_write_reg(struct vfe_device *vfe, u32 reg, u32 val); + +int vfe_vbif_apply_settings(struct vfe_device *vfe); + +#endif /* QC_MSM_CAMSS_VFE_VBIF_H */ diff --git a/drivers/media/platform/qcom/camss/camss-vfe.c b/drivers/media/platform/qcom/camss/camss-vfe.c index dff8d0a1e8c22..b48dfad5a8a73 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe.c +++ b/drivers/media/platform/qcom/camss/camss-vfe.c @@ -290,6 +290,7 @@ static u32 vfe_src_pad_code(struct vfe_line *line, u32 sink_code, switch (vfe->camss->res->version) { case CAMSS_8x16: + case CAMSS_8x39: case CAMSS_8x53: switch (sink_code) { case MEDIA_BUS_FMT_YUYV8_1X16: @@ -341,6 +342,7 @@ static u32 vfe_src_pad_code(struct vfe_line *line, u32 sink_code, break; case CAMSS_660: case CAMSS_2290: + case CAMSS_6150: case CAMSS_7280: case CAMSS_8x96: case CAMSS_8250: @@ -348,8 +350,10 @@ static u32 vfe_src_pad_code(struct vfe_line *line, u32 sink_code, case CAMSS_8300: case CAMSS_845: case CAMSS_8550: + case CAMSS_8650: case CAMSS_8775P: case CAMSS_X1E80100: + case CAMSS_X1P42100: switch (sink_code) { case MEDIA_BUS_FMT_YUYV8_1X16: { @@ -541,7 +545,7 @@ int vfe_enable_output_v2(struct vfe_line *line) ops->vfe_wm_start(vfe, output->wm_idx[0], line); - for (i = 0; i < 2; i++) { + for (i = 0; i < CAMSS_INIT_BUF_COUNT; i++) { output->buf[i] = vfe_buf_get_pending(output); if (!output->buf[i]) break; @@ -914,7 +918,8 @@ static int vfe_match_clock_names(struct vfe_device *vfe, return (!strcmp(clock->name, vfe_name) || !strcmp(clock->name, vfe_lite_name) || !strcmp(clock->name, "vfe_lite") || - !strcmp(clock->name, "camnoc_axi")); + !strcmp(clock->name, "camnoc_axi") || + !strcmp(clock->name, "camnoc_rt_axi")); } /* @@ -1827,6 +1832,15 @@ int msm_vfe_subdev_init(struct camss *camss, struct vfe_device *vfe, return PTR_ERR(vfe->base); } + if (vfe->res->has_vbif) { + vfe->vbif_base = devm_platform_ioremap_resource_byname(pdev, + vfe->res->vbif_name); + if (IS_ERR(vfe->vbif_base)) { + dev_err(dev, "could not map vbif memory\n"); + return PTR_ERR(vfe->vbif_base); + } + } + /* Interrupt */ ret = platform_get_irq_byname(pdev, res->interrupt[0]); @@ -1989,14 +2003,17 @@ static int vfe_bpl_align(struct vfe_device *vfe) int ret = 8; switch (vfe->camss->res->version) { + case CAMSS_6150: case CAMSS_7280: case CAMSS_8250: case CAMSS_8280XP: case CAMSS_8300: case CAMSS_845: case CAMSS_8550: + case CAMSS_8650: case CAMSS_8775P: case CAMSS_X1E80100: + case CAMSS_X1P42100: ret = 16; break; default: diff --git a/drivers/media/platform/qcom/camss/camss-vfe.h b/drivers/media/platform/qcom/camss/camss-vfe.h index 0300efdb1c46f..ae9dad353a378 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe.h +++ b/drivers/media/platform/qcom/camss/camss-vfe.h @@ -136,6 +136,8 @@ struct vfe_subdev_resources { u8 line_num; bool has_pd; char *pd_name; + bool has_vbif; + char *vbif_name; const struct vfe_hw_ops *hw_ops; const struct camss_formats *formats_rdi; const struct camss_formats *formats_pix; @@ -145,6 +147,7 @@ struct vfe_device { struct camss *camss; u8 id; void __iomem *base; + void __iomem *vbif_base; u32 irq; char irq_name[30]; struct camss_clock *clock; diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c index 2fbcd0e343aac..cab82648134c5 100644 --- a/drivers/media/platform/qcom/camss/camss.c +++ b/drivers/media/platform/qcom/camss/camss.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -73,7 +74,9 @@ static const struct camss_subdev_resources csiphy_res_8x16[] = { static const struct camss_subdev_resources csid_res_8x16[] = { /* CSID0 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 40000 } + }, .clock = { "top_ahb", "ispif_ahb", "csi0_ahb", "ahb", "csi0", "csi0_phy", "csi0_pix", "csi0_rdi" }, .clock_rate = { { 0 }, @@ -95,7 +98,9 @@ static const struct camss_subdev_resources csid_res_8x16[] = { /* CSID1 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 40000 } + }, .clock = { "top_ahb", "ispif_ahb", "csi1_ahb", "ahb", "csi1", "csi1_phy", "csi1_pix", "csi1_rdi" }, .clock_rate = { { 0 }, @@ -154,10 +159,159 @@ static const struct camss_subdev_resources vfe_res_8x16[] = { } }; +static const struct camss_subdev_resources csiphy_res_8x39[] = { + /* CSIPHY0 */ + { + .regulators = { + { .supply = "vdda", .init_load_uA = 40000 } + }, + .clock = { "top_ahb", "ispif_ahb", "ahb", "csiphy0_timer" }, + .clock_rate = { { 0 }, + { 40000000, 80000000 }, + { 0 }, + { 100000000, 200000000 } }, + .reg = { "csiphy0", "csiphy0_clk_mux" }, + .interrupt = { "csiphy0" }, + .csiphy = { + .id = 0, + .hw_ops = &csiphy_ops_2ph_1_0, + .formats = &csiphy_formats_8x16 + } + }, + + /* CSIPHY1 */ + { + .regulators = { + { .supply = "vdda", .init_load_uA = 40000 } + }, + .clock = { "top_ahb", "ispif_ahb", "ahb", "csiphy1_timer" }, + .clock_rate = { { 0 }, + { 40000000, 80000000 }, + { 0 }, + { 100000000, 200000000 } }, + .reg = { "csiphy1", "csiphy1_clk_mux" }, + .interrupt = { "csiphy1" }, + .csiphy = { + .id = 1, + .hw_ops = &csiphy_ops_2ph_1_0, + .formats = &csiphy_formats_8x16 + } + } +}; + +static const struct camss_subdev_resources csid_res_8x39[] = { + /* CSID0 */ + { + .regulators = {}, + .clock = { "top_ahb", "ispif_ahb", "csi0_ahb", "ahb", + "csi0", "csi0_phy", "csi0_pix", "csi0_rdi" }, + .clock_rate = { { 0 }, + { 40000000, 80000000 }, + { 0 }, + { 0 }, + { 100000000, 200000000 }, + { 0 }, + { 0 }, + { 0 } }, + .reg = { "csid0" }, + .interrupt = { "csid0" }, + .csid = { + .hw_ops = &csid_ops_4_1, + .parent_dev_ops = &vfe_parent_dev_ops, + .formats = &csid_formats_4_1 + } + }, + + /* CSID1 */ + { + .regulators = {}, + .clock = { "top_ahb", "ispif_ahb", "csi1_ahb", "ahb", + "csi1", "csi1_phy", "csi1_pix", "csi1_rdi" }, + .clock_rate = { { 0 }, + { 40000000, 80000000 }, + { 0 }, + { 0 }, + { 100000000, 200000000 }, + { 0 }, + { 0 }, + { 0 } }, + .reg = { "csid1" }, + .interrupt = { "csid1" }, + .csid = { + .hw_ops = &csid_ops_4_1, + .parent_dev_ops = &vfe_parent_dev_ops, + .formats = &csid_formats_4_1 + } + }, + + /* CSID2 */ + { + .regulators = {}, + .clock = { "top_ahb", "ispif_ahb", "csi2_ahb", "ahb", + "csi2", "csi2_phy", "csi2_pix", "csi2_rdi" }, + .clock_rate = { { 0 }, + { 40000000, 80000000 }, + { 0 }, + { 0 }, + { 100000000, 200000000 }, + { 0 }, + { 0 }, + { 0 } }, + .reg = { "csid2" }, + .interrupt = { "csid2" }, + .csid = { + .hw_ops = &csid_ops_4_1, + .parent_dev_ops = &vfe_parent_dev_ops, + .formats = &csid_formats_4_1 + } + }, +}; + +static const struct camss_subdev_resources ispif_res_8x39 = { + /* ISPIF */ + .clock = { "top_ahb", "ispif_ahb", "ahb", + "csi0", "csi0_pix", "csi0_rdi", + "csi1", "csi1_pix", "csi1_rdi", + "csi2", "csi2_pix", "csi2_rdi" }, + .clock_for_reset = { "vfe0", "csi_vfe0" }, + .reg = { "ispif", "csi_clk_mux" }, + .interrupt = { "ispif" }, +}; + +static const struct camss_subdev_resources vfe_res_8x39[] = { + /* VFE0 */ + { + .regulators = {}, + .clock = { "top_ahb", "ispif_ahb", "vfe0", "csi_vfe0", + "vfe_ahb", "vfe_axi", "ahb" }, + .clock_rate = { { 0 }, + { 40000000, 80000000 }, + { 50000000, 80000000, 100000000, 160000000, + 177780000, 200000000, 266670000, 320000000, + 400000000, 465000000, 480000000, 600000000 }, + { 0 }, + { 0 }, + { 0 }, + { 0 } }, + .reg = { "vfe0" }, + .interrupt = { "vfe0" }, + .vfe = { + .line_num = 3, + .has_vbif = true, + .vbif_name = "vfe0_vbif", + .hw_ops = &vfe_ops_4_1, + .formats_rdi = &vfe_formats_rdi_8x16, + .formats_pix = &vfe_formats_pix_8x16 + } + } +}; + static const struct camss_subdev_resources csid_res_8x53[] = { /* CSID0 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 9900 } + }, .clock = { "top_ahb", "ispif_ahb", "csi0_ahb", "ahb", "csi0", "csi0_phy", "csi0_pix", "csi0_rdi" }, .clock_rate = { { 0 }, @@ -180,7 +334,9 @@ static const struct camss_subdev_resources csid_res_8x53[] = { /* CSID1 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 9900 } + }, .clock = { "top_ahb", "ispif_ahb", "csi1_ahb", "ahb", "csi1", "csi1_phy", "csi1_pix", "csi1_rdi" }, .clock_rate = { { 0 }, @@ -203,7 +359,9 @@ static const struct camss_subdev_resources csid_res_8x53[] = { /* CSID2 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 9900 } + }, .clock = { "top_ahb", "ispif_ahb", "csi2_ahb", "ahb", "csi2", "csi2_phy", "csi2_pix", "csi2_rdi" }, .clock_rate = { { 0 }, @@ -364,7 +522,9 @@ static const struct camss_subdev_resources csiphy_res_8x96[] = { static const struct camss_subdev_resources csid_res_8x96[] = { /* CSID0 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 80160 } + }, .clock = { "top_ahb", "ispif_ahb", "csi0_ahb", "ahb", "csi0", "csi0_phy", "csi0_pix", "csi0_rdi" }, .clock_rate = { { 0 }, @@ -386,7 +546,9 @@ static const struct camss_subdev_resources csid_res_8x96[] = { /* CSID1 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 80160 } + }, .clock = { "top_ahb", "ispif_ahb", "csi1_ahb", "ahb", "csi1", "csi1_phy", "csi1_pix", "csi1_rdi" }, .clock_rate = { { 0 }, @@ -408,7 +570,9 @@ static const struct camss_subdev_resources csid_res_8x96[] = { /* CSID2 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 80160 } + }, .clock = { "top_ahb", "ispif_ahb", "csi2_ahb", "ahb", "csi2", "csi2_phy", "csi2_pix", "csi2_rdi" }, .clock_rate = { { 0 }, @@ -430,7 +594,9 @@ static const struct camss_subdev_resources csid_res_8x96[] = { /* CSID3 */ { - .regulators = { "vdda" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 80160 } + }, .clock = { "top_ahb", "ispif_ahb", "csi3_ahb", "ahb", "csi3", "csi3_phy", "csi3_pix", "csi3_rdi" }, .clock_rate = { { 0 }, @@ -518,7 +684,10 @@ static const struct camss_subdev_resources vfe_res_8x96[] = { static const struct camss_subdev_resources csiphy_res_2290[] = { /* CSIPHY0 */ { - .regulators = { "vdd-csiphy-1p2", "vdd-csiphy-1p8" }, + .regulators = { + { .supply = "vdd-csiphy-1p2", .init_load_uA = 26700 }, + { .supply = "vdd-csiphy-1p8", .init_load_uA = 2600 } + }, .clock = { "top_ahb", "ahb", "csiphy0", "csiphy0_timer" }, .clock_rate = { { 0 }, { 0 }, @@ -535,7 +704,10 @@ static const struct camss_subdev_resources csiphy_res_2290[] = { /* CSIPHY1 */ { - .regulators = { "vdd-csiphy-1p2", "vdd-csiphy-1p8" }, + .regulators = { + { .supply = "vdd-csiphy-1p2", .init_load_uA = 26700 }, + { .supply = "vdd-csiphy-1p8", .init_load_uA = 2600 } + }, .clock = { "top_ahb", "ahb", "csiphy1", "csiphy1_timer" }, .clock_rate = { { 0 }, { 0 }, @@ -711,7 +883,10 @@ static const struct camss_subdev_resources csiphy_res_660[] = { static const struct camss_subdev_resources csid_res_660[] = { /* CSID0 */ { - .regulators = { "vdda", "vdd_sec" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 0 }, + { .supply = "vdd_sec", .init_load_uA = 0 } + }, .clock = { "top_ahb", "ispif_ahb", "csi0_ahb", "ahb", "csi0", "csi0_phy", "csi0_pix", "csi0_rdi", "cphy_csid0" }, @@ -736,7 +911,10 @@ static const struct camss_subdev_resources csid_res_660[] = { /* CSID1 */ { - .regulators = { "vdda", "vdd_sec" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 0 }, + { .supply = "vdd_sec", .init_load_uA = 0 } + }, .clock = { "top_ahb", "ispif_ahb", "csi1_ahb", "ahb", "csi1", "csi1_phy", "csi1_pix", "csi1_rdi", "cphy_csid1" }, @@ -761,7 +939,10 @@ static const struct camss_subdev_resources csid_res_660[] = { /* CSID2 */ { - .regulators = { "vdda", "vdd_sec" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 0 }, + { .supply = "vdd_sec", .init_load_uA = 0 } + }, .clock = { "top_ahb", "ispif_ahb", "csi2_ahb", "ahb", "csi2", "csi2_phy", "csi2_pix", "csi2_rdi", "cphy_csid2" }, @@ -786,7 +967,10 @@ static const struct camss_subdev_resources csid_res_660[] = { /* CSID3 */ { - .regulators = { "vdda", "vdd_sec" }, + .regulators = { + { .supply = "vdda", .init_load_uA = 0 }, + { .supply = "vdd_sec", .init_load_uA = 0 } + }, .clock = { "top_ahb", "ispif_ahb", "csi3_ahb", "ahb", "csi3", "csi3_phy", "csi3_pix", "csi3_rdi", "cphy_csid3" }, @@ -883,7 +1067,10 @@ static const struct camss_subdev_resources vfe_res_660[] = { static const struct camss_subdev_resources csiphy_res_670[] = { /* CSIPHY0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 42800 }, + { .supply = "vdda-pll", .init_load_uA = 13900 } + }, .clock = { "soc_ahb", "cpas_ahb", "csiphy0", "csiphy0_timer" }, .clock_rate = { { 0 }, @@ -901,7 +1088,10 @@ static const struct camss_subdev_resources csiphy_res_670[] = { /* CSIPHY1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 42800 }, + { .supply = "vdda-pll", .init_load_uA = 13900 } + }, .clock = { "soc_ahb", "cpas_ahb", "csiphy1", "csiphy1_timer" }, .clock_rate = { { 0 }, @@ -919,7 +1109,10 @@ static const struct camss_subdev_resources csiphy_res_670[] = { /* CSIPHY2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 42800 }, + { .supply = "vdda-pll", .init_load_uA = 13900 } + }, .clock = { "soc_ahb", "cpas_ahb", "csiphy2", "csiphy2_timer" }, .clock_rate = { { 0 }, @@ -1159,7 +1352,10 @@ static const struct camss_subdev_resources csiphy_res_845[] = { static const struct camss_subdev_resources csid_res_845[] = { /* CSID0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "cpas_ahb", "cphy_rx_src", "slow_ahb_src", "soc_ahb", "vfe0", "vfe0_src", "vfe0_cphy_rx", "csi0", @@ -1184,7 +1380,10 @@ static const struct camss_subdev_resources csid_res_845[] = { /* CSID1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "cpas_ahb", "cphy_rx_src", "slow_ahb_src", "soc_ahb", "vfe1", "vfe1_src", "vfe1_cphy_rx", "csi1", @@ -1209,7 +1408,10 @@ static const struct camss_subdev_resources csid_res_845[] = { /* CSID2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "cpas_ahb", "cphy_rx_src", "slow_ahb_src", "soc_ahb", "vfe_lite", "vfe_lite_src", "vfe_lite_cphy_rx", "csi2", @@ -1318,13 +1520,16 @@ static const struct camss_subdev_resources vfe_res_845[] = { } }; -static const struct camss_subdev_resources csiphy_res_8250[] = { +static const struct camss_subdev_resources csiphy_res_sm6150[] = { /* CSIPHY0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdd-csiphy-1p2", .init_load_uA = 35000 }, + { .supply = "vdd-csiphy-1p8", .init_load_uA = 5000 } + }, .clock = { "csiphy0", "csiphy0_timer" }, - .clock_rate = { { 400000000 }, - { 300000000 } }, + .clock_rate = { { 269333333, 384000000 }, + { 269333333 } }, .reg = { "csiphy0" }, .interrupt = { "csiphy0" }, .csiphy = { @@ -1335,10 +1540,13 @@ static const struct camss_subdev_resources csiphy_res_8250[] = { }, /* CSIPHY1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdd-csiphy-1p2", .init_load_uA = 35000 }, + { .supply = "vdd-csiphy-1p8", .init_load_uA = 5000 } + }, .clock = { "csiphy1", "csiphy1_timer" }, - .clock_rate = { { 400000000 }, - { 300000000 } }, + .clock_rate = { { 269333333, 384000000 }, + { 269333333 } }, .reg = { "csiphy1" }, .interrupt = { "csiphy1" }, .csiphy = { @@ -1349,10 +1557,13 @@ static const struct camss_subdev_resources csiphy_res_8250[] = { }, /* CSIPHY2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdd-csiphy-1p2", .init_load_uA = 35000 }, + { .supply = "vdd-csiphy-1p8", .init_load_uA = 5000 } + }, .clock = { "csiphy2", "csiphy2_timer" }, - .clock_rate = { { 400000000 }, - { 300000000 } }, + .clock_rate = { { 269333333, 384000000 }, + { 269333333 } }, .reg = { "csiphy2" }, .interrupt = { "csiphy2" }, .csiphy = { @@ -1361,63 +1572,19 @@ static const struct camss_subdev_resources csiphy_res_8250[] = { .formats = &csiphy_formats_sdm845 } }, - /* CSIPHY3 */ - { - .regulators = { "vdda-phy", "vdda-pll" }, - .clock = { "csiphy3", "csiphy3_timer" }, - .clock_rate = { { 400000000 }, - { 300000000 } }, - .reg = { "csiphy3" }, - .interrupt = { "csiphy3" }, - .csiphy = { - .id = 3, - .hw_ops = &csiphy_ops_3ph_1_0, - .formats = &csiphy_formats_sdm845 - } - }, - /* CSIPHY4 */ - { - .regulators = { "vdda-phy", "vdda-pll" }, - .clock = { "csiphy4", "csiphy4_timer" }, - .clock_rate = { { 400000000 }, - { 300000000 } }, - .reg = { "csiphy4" }, - .interrupt = { "csiphy4" }, - .csiphy = { - .id = 4, - .hw_ops = &csiphy_ops_3ph_1_0, - .formats = &csiphy_formats_sdm845 - } - }, - /* CSIPHY5 */ - { - .regulators = { "vdda-phy", "vdda-pll" }, - .clock = { "csiphy5", "csiphy5_timer" }, - .clock_rate = { { 400000000 }, - { 300000000 } }, - .reg = { "csiphy5" }, - .interrupt = { "csiphy5" }, - .csiphy = { - .id = 5, - .hw_ops = &csiphy_ops_3ph_1_0, - .formats = &csiphy_formats_sdm845 - } - } }; -static const struct camss_subdev_resources csid_res_8250[] = { +static const struct camss_subdev_resources csid_res_sm6150[] = { /* CSID0 */ { .regulators = {}, - .clock = { "vfe0_csid", "vfe0_cphy_rx", "vfe0", "vfe0_areg", "vfe0_ahb" }, - .clock_rate = { { 400000000 }, - { 400000000 }, - { 350000000, 475000000, 576000000, 720000000 }, - { 100000000, 200000000, 300000000, 400000000 }, - { 0 } }, + .clock = { "vfe0_cphy_rx", "vfe0_csid" }, + .clock_rate = { { 269333333, 384000000 }, + { 320000000, 540000000 } }, .reg = { "csid0" }, .interrupt = { "csid0" }, .csid = { + .is_lite = false, .hw_ops = &csid_ops_gen2, .parent_dev_ops = &vfe_parent_dev_ops, .formats = &csid_formats_gen2 @@ -1426,15 +1593,13 @@ static const struct camss_subdev_resources csid_res_8250[] = { /* CSID1 */ { .regulators = {}, - .clock = { "vfe1_csid", "vfe1_cphy_rx", "vfe1", "vfe1_areg", "vfe1_ahb" }, - .clock_rate = { { 400000000 }, - { 400000000 }, - { 350000000, 475000000, 576000000, 720000000 }, - { 100000000, 200000000, 300000000, 400000000 }, - { 0 } }, + .clock = { "vfe1_cphy_rx", "vfe1_csid" }, + .clock_rate = { { 269333333, 384000000 }, + { 320000000, 540000000 } }, .reg = { "csid1" }, .interrupt = { "csid1" }, .csid = { + .is_lite = false, .hw_ops = &csid_ops_gen2, .parent_dev_ops = &vfe_parent_dev_ops, .formats = &csid_formats_gen2 @@ -1443,13 +1608,11 @@ static const struct camss_subdev_resources csid_res_8250[] = { /* CSID2 */ { .regulators = {}, - .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx", "vfe_lite", "vfe_lite_ahb" }, - .clock_rate = { { 400000000 }, - { 400000000 }, - { 400000000, 480000000 }, - { 0 } }, - .reg = { "csid2" }, - .interrupt = { "csid2" }, + .clock = { "vfe_lite_cphy_rx", "vfe_lite_csid" }, + .clock_rate = { { 269333333, 384000000 }, + { 320000000, 540000000 } }, + .reg = { "csid_lite" }, + .interrupt = { "csid_lite" }, .csid = { .is_lite = true, .hw_ops = &csid_ops_gen2, @@ -1457,48 +1620,28 @@ static const struct camss_subdev_resources csid_res_8250[] = { .formats = &csid_formats_gen2 } }, - /* CSID3 */ - { - .regulators = {}, - .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx", "vfe_lite", "vfe_lite_ahb" }, - .clock_rate = { { 400000000 }, - { 400000000 }, - { 400000000, 480000000 }, - { 0 } }, - .reg = { "csid3" }, - .interrupt = { "csid3" }, - .csid = { - .is_lite = true, - .hw_ops = &csid_ops_gen2, - .parent_dev_ops = &vfe_parent_dev_ops, - .formats = &csid_formats_gen2 - } - } }; -static const struct camss_subdev_resources vfe_res_8250[] = { +static const struct camss_subdev_resources vfe_res_sm6150[] = { /* VFE0 */ { .regulators = {}, - .clock = { "camnoc_axi_src", "slow_ahb_src", "cpas_ahb", - "camnoc_axi", "vfe0_ahb", "vfe0_areg", "vfe0", - "vfe0_axi", "cam_hf_axi" }, - .clock_rate = { { 19200000, 300000000, 400000000, 480000000 }, - { 19200000, 80000000 }, - { 19200000 }, - { 0 }, - { 0 }, - { 100000000, 200000000, 300000000, 400000000 }, - { 350000000, 475000000, 576000000, 720000000 }, + .clock = { "gcc_axi_hf", "camnoc_axi", "cpas_ahb", "soc_ahb", + "vfe0", "vfe0_axi"}, + .clock_rate = { { 0 }, { 0 }, - { 0 } }, + { 80000000 }, + { 37500000, 40000000 }, + { 360000000, 432000000, 540000000, 600000000 }, + { 265000000, 426000000 } }, .reg = { "vfe0" }, .interrupt = { "vfe0" }, .vfe = { .line_num = 3, + .is_lite = false, .has_pd = true, .pd_name = "ife0", - .hw_ops = &vfe_ops_480, + .hw_ops = &vfe_ops_170, .formats_rdi = &vfe_formats_rdi_845, .formats_pix = &vfe_formats_pix_845 } @@ -1506,8 +1649,269 @@ static const struct camss_subdev_resources vfe_res_8250[] = { /* VFE1 */ { .regulators = {}, - .clock = { "camnoc_axi_src", "slow_ahb_src", "cpas_ahb", - "camnoc_axi", "vfe1_ahb", "vfe1_areg", "vfe1", + .clock = { "gcc_axi_hf", "camnoc_axi", "cpas_ahb", "soc_ahb", + "vfe1", "vfe1_axi"}, + .clock_rate = { { 0 }, + { 0 }, + { 80000000 }, + { 37500000, 40000000 }, + { 360000000, 432000000, 540000000, 600000000 }, + { 265000000, 426000000 } }, + .reg = { "vfe1" }, + .interrupt = { "vfe1" }, + .vfe = { + .line_num = 3, + .is_lite = false, + .has_pd = true, + .pd_name = "ife1", + .hw_ops = &vfe_ops_170, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + } + }, + /* VFE2 */ + { + .regulators = {}, + .clock = { "gcc_axi_hf", "camnoc_axi", "cpas_ahb", "soc_ahb", + "vfe_lite" }, + .clock_rate = { { 0 }, + { 0 }, + { 80000000 }, + { 37500000, 40000000 }, + { 360000000, 432000000, 540000000, 600000000 } }, + .reg = { "vfe_lite" }, + .interrupt = { "vfe_lite" }, + .vfe = { + .line_num = 4, + .is_lite = true, + .hw_ops = &vfe_ops_170, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + } + }, +}; + +static const struct resources_icc icc_res_sm6150[] = { + { + .name = "ahb", + .icc_bw_tbl.avg = 38400, + .icc_bw_tbl.peak = 76800, + }, + { + .name = "hf_0", + .icc_bw_tbl.avg = 2097152, + .icc_bw_tbl.peak = 2097152, + }, +}; + +static const struct camss_subdev_resources csiphy_res_8250[] = { + /* CSIPHY0 */ + { + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 17500 }, + { .supply = "vdda-pll", .init_load_uA = 10000 } + }, + .clock = { "csiphy0", "csiphy0_timer" }, + .clock_rate = { { 400000000 }, + { 300000000 } }, + .reg = { "csiphy0" }, + .interrupt = { "csiphy0" }, + .csiphy = { + .id = 0, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + }, + /* CSIPHY1 */ + { + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 17500 }, + { .supply = "vdda-pll", .init_load_uA = 10000 } + }, + .clock = { "csiphy1", "csiphy1_timer" }, + .clock_rate = { { 400000000 }, + { 300000000 } }, + .reg = { "csiphy1" }, + .interrupt = { "csiphy1" }, + .csiphy = { + .id = 1, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + }, + /* CSIPHY2 */ + { + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 17500 }, + { .supply = "vdda-pll", .init_load_uA = 10000 } + }, + .clock = { "csiphy2", "csiphy2_timer" }, + .clock_rate = { { 400000000 }, + { 300000000 } }, + .reg = { "csiphy2" }, + .interrupt = { "csiphy2" }, + .csiphy = { + .id = 2, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + }, + /* CSIPHY3 */ + { + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 17500 }, + { .supply = "vdda-pll", .init_load_uA = 10000 } + }, + .clock = { "csiphy3", "csiphy3_timer" }, + .clock_rate = { { 400000000 }, + { 300000000 } }, + .reg = { "csiphy3" }, + .interrupt = { "csiphy3" }, + .csiphy = { + .id = 3, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + }, + /* CSIPHY4 */ + { + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 17500 }, + { .supply = "vdda-pll", .init_load_uA = 10000 } + }, + .clock = { "csiphy4", "csiphy4_timer" }, + .clock_rate = { { 400000000 }, + { 300000000 } }, + .reg = { "csiphy4" }, + .interrupt = { "csiphy4" }, + .csiphy = { + .id = 4, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + }, + /* CSIPHY5 */ + { + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 17500 }, + { .supply = "vdda-pll", .init_load_uA = 10000 } + }, + .clock = { "csiphy5", "csiphy5_timer" }, + .clock_rate = { { 400000000 }, + { 300000000 } }, + .reg = { "csiphy5" }, + .interrupt = { "csiphy5" }, + .csiphy = { + .id = 5, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + } +}; + +static const struct camss_subdev_resources csid_res_8250[] = { + /* CSID0 */ + { + .regulators = {}, + .clock = { "vfe0_csid", "vfe0_cphy_rx", "vfe0", "vfe0_areg", "vfe0_ahb" }, + .clock_rate = { { 400000000 }, + { 400000000 }, + { 350000000, 475000000, 576000000, 720000000 }, + { 100000000, 200000000, 300000000, 400000000 }, + { 0 } }, + .reg = { "csid0" }, + .interrupt = { "csid0" }, + .csid = { + .hw_ops = &csid_ops_gen2, + .parent_dev_ops = &vfe_parent_dev_ops, + .formats = &csid_formats_gen2 + } + }, + /* CSID1 */ + { + .regulators = {}, + .clock = { "vfe1_csid", "vfe1_cphy_rx", "vfe1", "vfe1_areg", "vfe1_ahb" }, + .clock_rate = { { 400000000 }, + { 400000000 }, + { 350000000, 475000000, 576000000, 720000000 }, + { 100000000, 200000000, 300000000, 400000000 }, + { 0 } }, + .reg = { "csid1" }, + .interrupt = { "csid1" }, + .csid = { + .hw_ops = &csid_ops_gen2, + .parent_dev_ops = &vfe_parent_dev_ops, + .formats = &csid_formats_gen2 + } + }, + /* CSID2 */ + { + .regulators = {}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx", "vfe_lite", "vfe_lite_ahb" }, + .clock_rate = { { 400000000 }, + { 400000000 }, + { 400000000, 480000000 }, + { 0 } }, + .reg = { "csid2" }, + .interrupt = { "csid2" }, + .csid = { + .is_lite = true, + .hw_ops = &csid_ops_gen2, + .parent_dev_ops = &vfe_parent_dev_ops, + .formats = &csid_formats_gen2 + } + }, + /* CSID3 */ + { + .regulators = {}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx", "vfe_lite", "vfe_lite_ahb" }, + .clock_rate = { { 400000000 }, + { 400000000 }, + { 400000000, 480000000 }, + { 0 } }, + .reg = { "csid3" }, + .interrupt = { "csid3" }, + .csid = { + .is_lite = true, + .hw_ops = &csid_ops_gen2, + .parent_dev_ops = &vfe_parent_dev_ops, + .formats = &csid_formats_gen2 + } + } +}; + +static const struct camss_subdev_resources vfe_res_8250[] = { + /* VFE0 */ + { + .regulators = {}, + .clock = { "camnoc_axi_src", "slow_ahb_src", "cpas_ahb", + "camnoc_axi", "vfe0_ahb", "vfe0_areg", "vfe0", + "vfe0_axi", "cam_hf_axi" }, + .clock_rate = { { 19200000, 300000000, 400000000, 480000000 }, + { 19200000, 80000000 }, + { 19200000 }, + { 0 }, + { 0 }, + { 100000000, 200000000, 300000000, 400000000 }, + { 350000000, 475000000, 576000000, 720000000 }, + { 0 }, + { 0 } }, + .reg = { "vfe0" }, + .interrupt = { "vfe0" }, + .vfe = { + .line_num = 3, + .has_pd = true, + .pd_name = "ife0", + .hw_ops = &vfe_ops_480, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + } + }, + /* VFE1 */ + { + .regulators = {}, + .clock = { "camnoc_axi_src", "slow_ahb_src", "cpas_ahb", + "camnoc_axi", "vfe1_ahb", "vfe1_areg", "vfe1", "vfe1_axi", "cam_hf_axi" }, .clock_rate = { { 19200000, 300000000, 400000000, 480000000 }, { 19200000, 80000000 }, @@ -1605,7 +2009,10 @@ static const struct resources_icc icc_res_sm8250[] = { static const struct camss_subdev_resources csiphy_res_7280[] = { /* CSIPHY0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 16100 }, + { .supply = "vdda-pll", .init_load_uA = 9000 } + }, .clock = { "csiphy0", "csiphy0_timer" }, .clock_rate = { { 300000000, 400000000 }, @@ -1620,7 +2027,10 @@ static const struct camss_subdev_resources csiphy_res_7280[] = { }, /* CSIPHY1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 16100 }, + { .supply = "vdda-pll", .init_load_uA = 9000 } + }, .clock = { "csiphy1", "csiphy1_timer" }, .clock_rate = { { 300000000, 400000000 }, @@ -1635,7 +2045,10 @@ static const struct camss_subdev_resources csiphy_res_7280[] = { }, /* CSIPHY2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 16100 }, + { .supply = "vdda-pll", .init_load_uA = 9000 } + }, .clock = { "csiphy2", "csiphy2_timer" }, .clock_rate = { { 300000000, 400000000 }, @@ -1650,7 +2063,10 @@ static const struct camss_subdev_resources csiphy_res_7280[] = { }, /* CSIPHY3 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 16100 }, + { .supply = "vdda-pll", .init_load_uA = 9000 } + }, .clock = { "csiphy3", "csiphy3_timer" }, .clock_rate = { { 300000000, 400000000 }, @@ -1665,7 +2081,10 @@ static const struct camss_subdev_resources csiphy_res_7280[] = { }, /* CSIPHY4 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 16100 }, + { .supply = "vdda-pll", .init_load_uA = 9000 } + }, .clock = { "csiphy4", "csiphy4_timer" }, .clock_rate = { { 300000000, 400000000 }, @@ -1978,7 +2397,10 @@ static const struct camss_subdev_resources csiphy_res_sc8280xp[] = { static const struct camss_subdev_resources csid_res_sc8280xp[] = { /* CSID0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe0_csid", "vfe0_cphy_rx", "vfe0", "vfe0_axi" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -1994,7 +2416,10 @@ static const struct camss_subdev_resources csid_res_sc8280xp[] = { }, /* CSID1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe1_csid", "vfe1_cphy_rx", "vfe1", "vfe1_axi" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -2010,7 +2435,10 @@ static const struct camss_subdev_resources csid_res_sc8280xp[] = { }, /* CSID2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe2_csid", "vfe2_cphy_rx", "vfe2", "vfe2_axi" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -2026,7 +2454,10 @@ static const struct camss_subdev_resources csid_res_sc8280xp[] = { }, /* CSID3 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe3_csid", "vfe3_cphy_rx", "vfe3", "vfe3_axi" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -2042,7 +2473,10 @@ static const struct camss_subdev_resources csid_res_sc8280xp[] = { }, /* CSID_LITE0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe_lite0_csid", "vfe_lite0_cphy_rx", "vfe_lite0" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -2058,7 +2492,10 @@ static const struct camss_subdev_resources csid_res_sc8280xp[] = { }, /* CSID_LITE1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe_lite1_csid", "vfe_lite1_cphy_rx", "vfe_lite1" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -2074,7 +2511,10 @@ static const struct camss_subdev_resources csid_res_sc8280xp[] = { }, /* CSID_LITE2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe_lite2_csid", "vfe_lite2_cphy_rx", "vfe_lite2" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -2090,7 +2530,10 @@ static const struct camss_subdev_resources csid_res_sc8280xp[] = { }, /* CSID_LITE3 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 0 }, + { .supply = "vdda-pll", .init_load_uA = 0 } + }, .clock = { "vfe_lite3_csid", "vfe_lite3_cphy_rx", "vfe_lite3" }, .clock_rate = { { 400000000, 480000000, 600000000 }, { 0 }, @@ -2291,7 +2734,10 @@ static const struct resources_icc icc_res_sc8280xp[] = { static const struct camss_subdev_resources csiphy_res_8550[] = { /* CSIPHY0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 32200 }, + { .supply = "vdda-pll", .init_load_uA = 18000 } + }, .clock = { "csiphy0", "csiphy0_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2305,7 +2751,10 @@ static const struct camss_subdev_resources csiphy_res_8550[] = { }, /* CSIPHY1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 32200 }, + { .supply = "vdda-pll", .init_load_uA = 18000 } + }, .clock = { "csiphy1", "csiphy1_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2319,7 +2768,10 @@ static const struct camss_subdev_resources csiphy_res_8550[] = { }, /* CSIPHY2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 32200 }, + { .supply = "vdda-pll", .init_load_uA = 18000 } + }, .clock = { "csiphy2", "csiphy2_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2333,7 +2785,10 @@ static const struct camss_subdev_resources csiphy_res_8550[] = { }, /* CSIPHY3 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 32200 }, + { .supply = "vdda-pll", .init_load_uA = 18000 } + }, .clock = { "csiphy3", "csiphy3_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2347,7 +2802,10 @@ static const struct camss_subdev_resources csiphy_res_8550[] = { }, /* CSIPHY4 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 37900 }, + { .supply = "vdda-pll", .init_load_uA = 18600 } + }, .clock = { "csiphy4", "csiphy4_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2361,7 +2819,10 @@ static const struct camss_subdev_resources csiphy_res_8550[] = { }, /* CSIPHY5 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 32200 }, + { .supply = "vdda-pll", .init_load_uA = 18000 } + }, .clock = { "csiphy5", "csiphy5_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2375,7 +2836,10 @@ static const struct camss_subdev_resources csiphy_res_8550[] = { }, /* CSIPHY6 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 37900 }, + { .supply = "vdda-pll", .init_load_uA = 18600 } + }, .clock = { "csiphy6", "csiphy6_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2389,7 +2853,10 @@ static const struct camss_subdev_resources csiphy_res_8550[] = { }, /* CSIPHY7 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 32200 }, + { .supply = "vdda-pll", .init_load_uA = 18000 } + }, .clock = { "csiphy7", "csiphy7_timer" }, .clock_rate = { { 400000000, 480000000 }, { 400000000 } }, @@ -2561,12 +3028,11 @@ static const struct camss_subdev_resources vfe_res_8550[] = { /* VFE3 lite */ { .regulators = {}, - .clock = { "gcc_axi_hf", "cpas_ahb", "cpas_fast_ahb_clk", "vfe_lite_ahb", + .clock = { "gcc_axi_hf", "cpas_ahb", "vfe_lite_ahb", "vfe_lite", "cpas_ife_lite", "camnoc_axi" }, .clock_rate = { { 0 }, { 80000000 }, { 300000000, 400000000 }, - { 300000000, 400000000 }, { 400000000, 480000000 }, { 300000000, 400000000 }, { 300000000, 400000000 } }, @@ -2583,12 +3049,11 @@ static const struct camss_subdev_resources vfe_res_8550[] = { /* VFE4 lite */ { .regulators = {}, - .clock = { "gcc_axi_hf", "cpas_ahb", "cpas_fast_ahb_clk", "vfe_lite_ahb", + .clock = { "gcc_axi_hf", "cpas_ahb", "vfe_lite_ahb", "vfe_lite", "cpas_ife_lite", "camnoc_axi" }, .clock_rate = { { 0 }, { 80000000 }, { 300000000, 400000000 }, - { 300000000, 400000000 }, { 400000000, 480000000 }, { 300000000, 400000000 }, { 300000000, 400000000 } }, @@ -2617,10 +3082,342 @@ static const struct resources_icc icc_res_sm8550[] = { }, }; +static const struct camss_subdev_resources csiphy_res_sm8650[] = { + /* CSIPHY0 */ + { + .regulators = { + { .supply = "vdd-csiphy01-0p9", .init_load_uA = 88000 }, + { .supply = "vdd-csiphy01-1p2", .init_load_uA = 17800 }, + }, + .clock = { "csiphy0", "csiphy0_timer" }, + .clock_rate = { { 400000000 }, + { 400000000 } }, + .reg = { "csiphy0" }, + .interrupt = { "csiphy0" }, + .csiphy = { + .id = 0, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845, + }, + }, + /* CSIPHY1 */ + { + .regulators = { + { .supply = "vdd-csiphy01-0p9", .init_load_uA = 88000 }, + { .supply = "vdd-csiphy01-1p2", .init_load_uA = 17800 }, + }, + .clock = { "csiphy1", "csiphy1_timer" }, + .clock_rate = { { 400000000 }, + { 400000000 } }, + .reg = { "csiphy1" }, + .interrupt = { "csiphy1" }, + .csiphy = { + .id = 1, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845, + }, + }, + /* CSIPHY2 */ + { + .regulators = { + { .supply = "vdd-csiphy24-0p9", .init_load_uA = 147000 }, + { .supply = "vdd-csiphy24-1p2", .init_load_uA = 24400 }, + }, + .clock = { "csiphy2", "csiphy2_timer" }, + .clock_rate = { { 400000000 }, + { 400000000 } }, + .reg = { "csiphy2" }, + .interrupt = { "csiphy2" }, + .csiphy = { + .id = 2, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845, + }, + }, + /* CSIPHY3 */ + { + .regulators = { + { .supply = "vdd-csiphy35-0p9", .init_load_uA = 88000 }, + { .supply = "vdd-csiphy35-1p2", .init_load_uA = 17800 }, + }, + .clock = { "csiphy3", "csiphy3_timer" }, + .clock_rate = { { 400000000 }, + { 400000000 } }, + .reg = { "csiphy3" }, + .interrupt = { "csiphy3" }, + .csiphy = { + .id = 3, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845, + }, + }, + /* CSIPHY4 */ + { + .regulators = { + { .supply = "vdd-csiphy24-0p9", .init_load_uA = 147000 }, + { .supply = "vdd-csiphy24-1p2", .init_load_uA = 24400 }, + }, + .clock = { "csiphy4", "csiphy4_timer" }, + .clock_rate = { { 400000000 }, + { 400000000 } }, + .reg = { "csiphy4" }, + .interrupt = { "csiphy4" }, + .csiphy = { + .id = 4, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845, + }, + }, + /* CSIPHY5 */ + { + .regulators = { + { .supply = "vdd-csiphy35-0p9", .init_load_uA = 88000 }, + { .supply = "vdd-csiphy35-1p2", .init_load_uA = 17800 }, + }, + .clock = { "csiphy5", "csiphy5_timer" }, + .clock_rate = { { 400000000 }, + { 400000000 } }, + .reg = { "csiphy5" }, + .interrupt = { "csiphy5" }, + .csiphy = { + .id = 5, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845, + }, + }, +}; + +static const struct camss_subdev_resources csid_res_sm8650[] = { + /* CSID0 */ + { + .regulators = { }, + .clock = { "csid", "csiphy_rx" }, + .clock_rate = { { 400000000 }, + { 400000000, 480000000 } }, + .reg = { "csid0" }, + .interrupt = { "csid0" }, + .csid = { + .parent_dev_ops = &vfe_parent_dev_ops, + .hw_ops = &csid_ops_gen3, + .formats = &csid_formats_gen2, + }, + }, + /* CSID1 */ + { + .regulators = { }, + .clock = { "csid", "csiphy_rx" }, + .clock_rate = { { 400000000 }, + { 400000000, 480000000 } }, + .reg = { "csid1" }, + .interrupt = { "csid1" }, + .csid = { + .parent_dev_ops = &vfe_parent_dev_ops, + .hw_ops = &csid_ops_gen3, + .formats = &csid_formats_gen2, + }, + }, + /* CSID2 */ + { + .regulators = { }, + .clock = { "csid", "csiphy_rx" }, + .clock_rate = { { 400000000 }, + { 400000000, 480000000 } }, + .reg = { "csid2" }, + .interrupt = { "csid2" }, + .csid = { + .parent_dev_ops = &vfe_parent_dev_ops, + .hw_ops = &csid_ops_gen3, + .formats = &csid_formats_gen2, + }, + }, + /* CSID3 lite */ + { + .regulators = { }, + .clock = { "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx" }, + .clock_rate = { { 0 }, + { 400000000, 480000000 }, + { 0 } }, + .reg = { "csid_lite0" }, + .interrupt = { "csid_lite0" }, + .csid = { + .is_lite = true, + .parent_dev_ops = &vfe_parent_dev_ops, + .hw_ops = &csid_ops_gen3, + .formats = &csid_formats_gen2, + }, + }, + /* CSID4 lite */ + { + .regulators = { }, + .clock = { "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx" }, + .clock_rate = { { 0 }, + { 400000000, 480000000 }, + { 0 } }, + .reg = { "csid_lite1" }, + .interrupt = { "csid_lite1" }, + .csid = { + .is_lite = true, + .parent_dev_ops = &vfe_parent_dev_ops, + .hw_ops = &csid_ops_gen3, + .formats = &csid_formats_gen2, + }, + }, +}; + +static const struct camss_subdev_resources vfe_res_sm8650[] = { + /* VFE0 */ + { + .regulators = { }, + .clock = { "gcc_axi_hf", "cpas_ahb", "cpas_fast_ahb", + "camnoc_axi", "vfe0_fast_ahb", "vfe0", "cpas_vfe0", + "qdss_debug_xo", + }, + .clock_rate = { { 0 }, + { 80000000 }, + { 300000000, 400000000 }, + { 300000000, 400000000 }, + { 0 }, + { 466000000, 594000000, 675000000, 785000000 }, + { 0 }, + { 0 }, + }, + .reg = { "vfe0" }, + .interrupt = { "vfe0" }, + .vfe = { + .line_num = 3, + .has_pd = true, + .pd_name = "ife0", + .hw_ops = &vfe_ops_gen3, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, + /* VFE1 */ + { + .regulators = { }, + .clock = { "gcc_axi_hf", "cpas_ahb", "cpas_fast_ahb", + "camnoc_axi", "vfe1_fast_ahb", "vfe1", "cpas_vfe1", + "qdss_debug_xo", + }, + .clock_rate = { { 0 }, + { 80000000 }, + { 300000000, 400000000 }, + { 300000000, 400000000 }, + { 0 }, + { 466000000, 594000000, 675000000, 785000000 }, + { 0 }, + { 0 }, + }, + .reg = { "vfe1" }, + .interrupt = { "vfe1" }, + .vfe = { + .line_num = 3, + .has_pd = true, + .pd_name = "ife1", + .hw_ops = &vfe_ops_gen3, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, + /* VFE2 */ + { + .regulators = { }, + .clock = { "gcc_axi_hf", "cpas_ahb", "cpas_fast_ahb", + "camnoc_axi", "vfe2_fast_ahb", "vfe2", "cpas_vfe2", + "qdss_debug_xo", + }, + .clock_rate = { { 0 }, + { 80000000 }, + { 300000000, 400000000 }, + { 300000000, 400000000 }, + { 0 }, + { 466000000, 594000000, 675000000, 785000000 }, + { 0 }, + { 0 }, + }, + .reg = { "vfe2" }, + .interrupt = { "vfe2" }, + .vfe = { + .line_num = 3, + .has_pd = true, + .pd_name = "ife2", + .hw_ops = &vfe_ops_gen3, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, + /* VFE3 lite */ + { + .regulators = { }, + .clock = { "gcc_axi_hf", "cpas_ahb", "camnoc_axi", + "vfe_lite_ahb", "vfe_lite", "cpas_vfe_lite", + "qdss_debug_xo", + }, + .clock_rate = { { 0 }, + { 80000000 }, + { 300000000, 400000000 }, + { 0 }, + { 400000000, 480000000 }, + { 0 }, + { 0 }, + }, + .reg = { "vfe_lite0" }, + .interrupt = { "vfe_lite0" }, + .vfe = { + .line_num = 4, + .is_lite = true, + .hw_ops = &vfe_ops_gen3, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, + /* VFE4 lite */ + { + .regulators = { }, + .clock = { "gcc_axi_hf", "cpas_ahb", "camnoc_axi", + "vfe_lite_ahb", "vfe_lite", "cpas_vfe_lite", + "qdss_debug_xo", + }, + .clock_rate = { { 0 }, + { 80000000 }, + { 300000000, 400000000 }, + { 0 }, + { 400000000, 480000000 }, + { 0 }, + { 0 }, + }, + .reg = { "vfe_lite1" }, + .interrupt = { "vfe_lite1" }, + .vfe = { + .line_num = 4, + .is_lite = true, + .hw_ops = &vfe_ops_gen3, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, +}; + +static const struct resources_icc icc_res_sm8650[] = { + { + .name = "ahb", + .icc_bw_tbl.avg = 38400, + .icc_bw_tbl.peak = 76800, + }, + { + .name = "hf_mnoc", + .icc_bw_tbl.avg = 2097152, + .icc_bw_tbl.peak = 2097152, + }, +}; + static const struct camss_subdev_resources csiphy_res_8300[] = { /* CSIPHY0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 15900 }, + { .supply = "vdda-pll", .init_load_uA = 8900 } + }, .clock = { "csiphy_rx", "csiphy0", "csiphy0_timer" }, .clock_rate = { @@ -2638,7 +3435,10 @@ static const struct camss_subdev_resources csiphy_res_8300[] = { }, /* CSIPHY1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 15900 }, + { .supply = "vdda-pll", .init_load_uA = 8900 } + }, .clock = { "csiphy_rx", "csiphy1", "csiphy1_timer" }, .clock_rate = { @@ -2656,7 +3456,10 @@ static const struct camss_subdev_resources csiphy_res_8300[] = { }, /* CSIPHY2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 15900 }, + { .supply = "vdda-pll", .init_load_uA = 8900 } + }, .clock = { "csiphy_rx", "csiphy2", "csiphy2_timer" }, .clock_rate = { @@ -2677,7 +3480,10 @@ static const struct camss_subdev_resources csiphy_res_8300[] = { static const struct camss_subdev_resources csiphy_res_8775p[] = { /* CSIPHY0 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 15900 }, + { .supply = "vdda-pll", .init_load_uA = 8900 } + }, .clock = { "csiphy_rx", "csiphy0", "csiphy0_timer"}, .clock_rate = { { 400000000 }, @@ -2694,7 +3500,10 @@ static const struct camss_subdev_resources csiphy_res_8775p[] = { }, /* CSIPHY1 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 15900 }, + { .supply = "vdda-pll", .init_load_uA = 8900 } + }, .clock = { "csiphy_rx", "csiphy1", "csiphy1_timer"}, .clock_rate = { { 400000000 }, @@ -2711,36 +3520,90 @@ static const struct camss_subdev_resources csiphy_res_8775p[] = { }, /* CSIPHY2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 15900 }, + { .supply = "vdda-pll", .init_load_uA = 8900 } + }, .clock = { "csiphy_rx", "csiphy2", "csiphy2_timer"}, .clock_rate = { { 400000000 }, { 0 }, { 400000000 }, }, - .reg = { "csiphy2" }, - .interrupt = { "csiphy2" }, - .csiphy = { - .id = 2, - .hw_ops = &csiphy_ops_3ph_1_0, - .formats = &csiphy_formats_sdm845 + .reg = { "csiphy2" }, + .interrupt = { "csiphy2" }, + .csiphy = { + .id = 2, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + }, + /* CSIPHY3 */ + { + .regulators = { + { .supply = "vdda-phy", .init_load_uA = 15900 }, + { .supply = "vdda-pll", .init_load_uA = 8900 } + }, + .clock = { "csiphy_rx", "csiphy3", "csiphy3_timer"}, + .clock_rate = { + { 400000000 }, + { 0 }, + { 400000000 }, + }, + .reg = { "csiphy3" }, + .interrupt = { "csiphy3" }, + .csiphy = { + .id = 3, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + } + }, +}; + +static const struct camss_subdev_resources tpg_res_8775p[] = { + /* TPG0 */ + { + .regulators = {}, + .clock = { "cpas_ahb", "csiphy_rx" }, + .clock_rate = { + { 0 }, + { 400000000 }, + }, + .reg = { "tpg0" }, + .tpg = { + .lane_cnt = 4, + .formats = &tpg_formats_gen1, + .hw_ops = &tpg_ops_gen1 + } + }, + /* TPG1 */ + { + .regulators = {}, + .clock = { "cpas_ahb", "csiphy_rx" }, + .clock_rate = { + { 0 }, + { 400000000 }, + }, + .reg = { "tpg1" }, + .tpg = { + .lane_cnt = 4, + .formats = &tpg_formats_gen1, + .hw_ops = &tpg_ops_gen1 } }, - /* CSIPHY3 */ + /* TPG2 */ { - .regulators = { "vdda-phy", "vdda-pll" }, - .clock = { "csiphy_rx", "csiphy3", "csiphy3_timer"}, + .regulators = {}, + .clock = { "cpas_ahb", "csiphy_rx" }, .clock_rate = { - { 400000000 }, { 0 }, { 400000000 }, }, - .reg = { "csiphy3" }, - .interrupt = { "csiphy3" }, - .csiphy = { - .id = 3, - .hw_ops = &csiphy_ops_3ph_1_0, - .formats = &csiphy_formats_sdm845 + .reg = { "tpg2" }, + .tpg = { + .lane_cnt = 4, + .formats = &tpg_formats_gen1, + .hw_ops = &tpg_ops_gen1 } }, }; @@ -2784,12 +3647,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID2 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite0" }, .interrupt = { "csid_lite0" }, @@ -2803,12 +3664,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID3 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite1" }, .interrupt = { "csid_lite1" }, @@ -2822,12 +3681,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID4 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite2" }, .interrupt = { "csid_lite2" }, @@ -2841,12 +3698,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID5 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite3" }, .interrupt = { "csid_lite3" }, @@ -2860,12 +3715,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID6 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite4" }, .interrupt = { "csid_lite4" }, @@ -2938,15 +3791,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE2 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite0" }, .interrupt = { "vfe_lite0" }, @@ -2961,15 +3816,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE3 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite1" }, .interrupt = { "vfe_lite1" }, @@ -2984,15 +3841,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE4 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite2" }, .interrupt = { "vfe_lite2" }, @@ -3007,15 +3866,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE5 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite3" }, .interrupt = { "vfe_lite3" }, @@ -3030,15 +3891,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE6 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite4" }, .interrupt = { "vfe_lite4" }, @@ -3081,13 +3944,6 @@ static const struct resources_icc icc_res_sa8775p[] = { static const struct camss_subdev_resources csiphy_res_x1e80100[] = { /* CSIPHY0 */ { - .regulators = { "vdd-csiphy-0p8", - "vdd-csiphy-1p2" }, - .clock = { "csiphy0", "csiphy0_timer" }, - .clock_rate = { { 300000000, 400000000, 480000000 }, - { 266666667, 400000000 } }, - .reg = { "csiphy0" }, - .interrupt = { "csiphy0" }, .csiphy = { .id = 0, .hw_ops = &csiphy_ops_3ph_1_0, @@ -3096,13 +3952,6 @@ static const struct camss_subdev_resources csiphy_res_x1e80100[] = { }, /* CSIPHY1 */ { - .regulators = { "vdd-csiphy-0p8", - "vdd-csiphy-1p2" }, - .clock = { "csiphy1", "csiphy1_timer" }, - .clock_rate = { { 300000000, 400000000, 480000000 }, - { 266666667, 400000000 } }, - .reg = { "csiphy1" }, - .interrupt = { "csiphy1" }, .csiphy = { .id = 1, .hw_ops = &csiphy_ops_3ph_1_0, @@ -3111,13 +3960,6 @@ static const struct camss_subdev_resources csiphy_res_x1e80100[] = { }, /* CSIPHY2 */ { - .regulators = { "vdd-csiphy-0p8", - "vdd-csiphy-1p2" }, - .clock = { "csiphy2", "csiphy2_timer" }, - .clock_rate = { { 300000000, 400000000, 480000000 }, - { 266666667, 400000000 } }, - .reg = { "csiphy2" }, - .interrupt = { "csiphy2" }, .csiphy = { .id = 2, .hw_ops = &csiphy_ops_3ph_1_0, @@ -3126,13 +3968,6 @@ static const struct camss_subdev_resources csiphy_res_x1e80100[] = { }, /* CSIPHY4 */ { - .regulators = { "vdd-csiphy-0p8", - "vdd-csiphy-1p2" }, - .clock = { "csiphy4", "csiphy4_timer" }, - .clock_rate = { { 300000000, 400000000, 480000000 }, - { 266666667, 400000000 } }, - .reg = { "csiphy4" }, - .interrupt = { "csiphy4" }, .csiphy = { .id = 4, .hw_ops = &csiphy_ops_3ph_1_0, @@ -3141,6 +3976,54 @@ static const struct camss_subdev_resources csiphy_res_x1e80100[] = { }, }; +static const struct camss_subdev_resources tpg_res_x1e80100[] = { + /* TPG0 */ + { + .regulators = {}, + .clock = { "cpas_ahb", "csid_csiphy_rx" }, + .clock_rate = { + { 0 }, + { 400000000 }, + }, + .reg = { "csitpg0" }, + .tpg = { + .lane_cnt = 4, + .formats = &tpg_formats_gen1, + .hw_ops = &tpg_ops_gen1 + } + }, + /* TPG1 */ + { + .regulators = {}, + .clock = { "cpas_ahb", "csid_csiphy_rx" }, + .clock_rate = { + { 0 }, + { 400000000 }, + }, + .reg = { "csitpg1" }, + .tpg = { + .lane_cnt = 4, + .formats = &tpg_formats_gen1, + .hw_ops = &tpg_ops_gen1 + } + }, + /* TPG2 */ + { + .regulators = {}, + .clock = { "cpas_ahb", "csid_csiphy_rx" }, + .clock_rate = { + { 0 }, + { 400000000 }, + }, + .reg = { "csitpg2" }, + .tpg = { + .lane_cnt = 4, + .formats = &tpg_formats_gen1, + .hw_ops = &tpg_ops_gen1 + } + }, +}; + static const struct camss_subdev_resources csid_res_x1e80100[] = { /* CSID0 */ { @@ -3254,7 +4137,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = { .clock = {"camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb", "cpas_fast_ahb", "cpas_vfe0", "vfe0_fast_ahb", "vfe0" }, - .clock_rate = { { 0 }, + .clock_rate = { { 400000000 }, { 0 }, { 0 }, { 0 }, @@ -3278,7 +4161,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = { .clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb", "cpas_fast_ahb", "cpas_vfe1", "vfe1_fast_ahb", "vfe1" }, - .clock_rate = { { 0 }, + .clock_rate = { { 400000000 }, { 0 }, { 0 }, { 0 }, @@ -3302,7 +4185,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = { .clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb", "vfe_lite_ahb", "cpas_vfe_lite", "vfe_lite", "vfe_lite_csid" }, - .clock_rate = { { 0 }, + .clock_rate = { { 400000000 }, { 0 }, { 0 }, { 0 }, @@ -3325,7 +4208,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = { .clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb", "vfe_lite_ahb", "cpas_vfe_lite", "vfe_lite", "vfe_lite_csid" }, - .clock_rate = { { 0 }, + .clock_rate = { { 400000000 }, { 0 }, { 0 }, { 0 }, @@ -3371,6 +4254,98 @@ static const struct resources_wrapper csid_wrapper_res_x1e80100 = { .reg = "csid_wrapper", }; +static const struct camss_subdev_resources csiphy_res_x1p42100[] = { + /* CSIPHY0 */ + { + .csiphy = { + .id = 0, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + }, + }, + /* CSIPHY4 */ + { + .csiphy = { + .id = 4, + .hw_ops = &csiphy_ops_3ph_1_0, + .formats = &csiphy_formats_sdm845 + }, + }, +}; + +static const struct camss_subdev_resources vfe_res_x1p42100[] = { + /* IFE0 */ + { + .regulators = {}, + .clock = {"camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb", + "cpas_fast_ahb", "cpas_vfe0", "vfe0_fast_ahb", + "vfe0" }, + .clock_rate = { { 400000000 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 345600000, 432000000, 594000000, 675000000, + 727000000 }, }, + .reg = { "vfe0" }, + .interrupt = { "vfe0" }, + .vfe = { + .line_num = 4, + .pd_name = "ife0", + .hw_ops = &vfe_ops_680, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, + /* IFE_LITE_0 */ + { + .regulators = {}, + .clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb", + "vfe_lite_ahb", "cpas_vfe_lite", "vfe_lite", + "vfe_lite_csid" }, + .clock_rate = { { 400000000 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 266666667, 400000000, 480000000 }, + { 266666667, 400000000, 480000000 }, }, + .reg = { "vfe_lite0" }, + .interrupt = { "vfe_lite0" }, + .vfe = { + .is_lite = true, + .line_num = 4, + .hw_ops = &vfe_ops_680, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, + /* IFE_LITE_1 */ + { + .regulators = {}, + .clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb", + "vfe_lite_ahb", "cpas_vfe_lite", "vfe_lite", + "vfe_lite_csid" }, + .clock_rate = { { 400000000 }, + { 0 }, + { 0 }, + { 0 }, + { 0 }, + { 266666667, 400000000, 480000000 }, + { 266666667, 400000000, 480000000 }, }, + .reg = { "vfe_lite1" }, + .interrupt = { "vfe_lite1" }, + .vfe = { + .is_lite = true, + .line_num = 4, + .hw_ops = &vfe_ops_680, + .formats_rdi = &vfe_formats_rdi_845, + .formats_pix = &vfe_formats_pix_845 + }, + }, +}; + /* * camss_add_clock_margin - Add margin to clock frequency rate * @rate: Clock frequency rate @@ -3568,16 +4543,16 @@ static const struct parent_dev_ops vfe_parent_dev_ops = { }; /* - * camss_of_parse_endpoint_node - Parse port endpoint node - * @dev: Device - * @node: Device node to be parsed + * camss_parse_endpoint_node - Parse port endpoint node + * @dev: CAMSS device + * @ep: Device endpoint to be parsed * @csd: Parsed data from port endpoint node * * Return 0 on success or a negative error code on failure */ -static int camss_of_parse_endpoint_node(struct device *dev, - struct device_node *node, - struct camss_async_subdev *csd) +static int camss_parse_endpoint_node(struct device *dev, + struct fwnode_handle *ep, + struct camss_async_subdev *csd) { struct csiphy_lanes_cfg *lncfg = &csd->interface.csi2.lane_cfg; struct v4l2_mbus_config_mipi_csi2 *mipi_csi2; @@ -3585,7 +4560,7 @@ static int camss_of_parse_endpoint_node(struct device *dev, unsigned int i; int ret; - ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(node), &vep); + ret = v4l2_fwnode_endpoint_parse(ep, &vep); if (ret) return ret; @@ -3620,49 +4595,58 @@ static int camss_of_parse_endpoint_node(struct device *dev, } /* - * camss_of_parse_ports - Parse ports node - * @dev: Device - * @notifier: v4l2_device notifier data + * camss_parse_ports - Parse ports node + * @dev: CAMSS device * - * Return number of "port" nodes found in "ports" node + * Return 0 on success or a negative error code on failure */ -static int camss_of_parse_ports(struct camss *camss) +static int camss_parse_ports(struct camss *camss) { struct device *dev = camss->dev; - struct device_node *node = NULL; - struct device_node *remote = NULL; - int ret, num_subdevs = 0; + const struct camss_resources *res = camss->res; + struct fwnode_handle *fwnode = dev_fwnode(dev), *ep; + int ret; - for_each_endpoint_of_node(dev->of_node, node) { + fwnode_graph_for_each_endpoint(fwnode, ep) { struct camss_async_subdev *csd; + struct fwnode_handle *remote; - remote = of_graph_get_remote_port_parent(node); - if (!remote) { - dev_err(dev, "Cannot get remote parent\n"); - ret = -EINVAL; - goto err_cleanup; + if (!fwnode_device_is_available(ep)) + continue; + + if (res->legacy_phy) { + csd = v4l2_async_nf_add_fwnode_remote(&camss->notifier, ep, + typeof(*csd)); + } else { + /* + * For non-legacy PHY, the CSIPHY is a separate device. + * Register the remote endpoint (CSIPHY's endpoint) as + * the async subdev, not the remote port parent. + */ + remote = fwnode_graph_get_remote_endpoint(ep); + if (!remote) + continue; + + csd = v4l2_async_nf_add_fwnode(&camss->notifier, remote, + struct camss_async_subdev); + fwnode_handle_put(remote); } - csd = v4l2_async_nf_add_fwnode(&camss->notifier, - of_fwnode_handle(remote), - struct camss_async_subdev); - of_node_put(remote); if (IS_ERR(csd)) { ret = PTR_ERR(csd); goto err_cleanup; } - ret = camss_of_parse_endpoint_node(dev, node, csd); + ret = camss_parse_endpoint_node(dev, ep, csd); if (ret < 0) goto err_cleanup; - - num_subdevs++; } - return num_subdevs; + return 0; err_cleanup: - of_node_put(node); + fwnode_handle_put(ep); + return ret; } @@ -3679,15 +4663,39 @@ static int camss_init_subdevices(struct camss *camss) unsigned int i; int ret; - for (i = 0; i < camss->res->csiphy_num; i++) { - ret = msm_csiphy_subdev_init(camss, &camss->csiphy[i], - &res->csiphy_res[i], - res->csiphy_res[i].csiphy.id); - if (ret < 0) { - dev_err(camss->dev, - "Failed to init csiphy%d sub-device: %d\n", - i, ret); - return ret; + if (!res->legacy_phy) { + for (i = 0; i < camss->res->csiphy_num; i++) { + ret = msm_csiphy_subdev_init(camss, &camss->csiphy[i], + &res->csiphy_res[i], + res->csiphy_res[i].csiphy.id); + if (ret < 0) + return ret; + } + } else { + for (i = 0; i < camss->res->csiphy_num; i++) { + ret = msm_csiphy_subdev_init_legacy(camss, &camss->csiphy[i], + &res->csiphy_res[i], + res->csiphy_res[i].csiphy.id); + if (ret < 0) { + dev_err(camss->dev, + "Failed to init csiphy%d sub-device: %d\n", + i, ret); + return ret; + } + camss->csiphy[i].phy = ERR_PTR(-ENODEV); + } + } + + if (camss->tpg) { + for (i = 0; i < camss->res->tpg_num; i++) { + ret = msm_tpg_subdev_init(camss, &camss->tpg[i], + &res->tpg_res[i], i); + if (ret < 0) { + dev_err(camss->dev, + "Failed to init tpg%d sub-device: %d\n", + i, ret); + return ret; + } } } @@ -3764,6 +4772,9 @@ static int camss_link_entities(struct camss *camss) for (i = 0; i < camss->res->csiphy_num; i++) { for (j = 0; j < camss->res->csid_num; j++) { + if (!camss->csiphy[i].phy) + continue; + ret = media_create_pad_link(&camss->csiphy[i].subdev.entity, MSM_CSIPHY_PAD_SRC, &camss->csid[j].subdev.entity, @@ -3779,6 +4790,23 @@ static int camss_link_entities(struct camss *camss) } } + for (i = 0; i < camss->res->tpg_num; i++) { + for (j = 0; j < camss->res->csid_num; j++) { + ret = media_create_pad_link(&camss->tpg[i].subdev.entity, + MSM_TPG_PAD_SRC, + &camss->csid[j].subdev.entity, + MSM_CSID_PAD_SINK, + 0); + if (ret < 0) { + camss_link_err(camss, + camss->tpg[i].subdev.entity.name, + camss->csid[j].subdev.entity.name, + ret); + return ret; + } + } + } + if (camss->ispif) { for (i = 0; i < camss->res->csid_num; i++) { for (j = 0; j < camss->ispif->line_num; j++) { @@ -3873,6 +4901,9 @@ static int camss_register_entities(struct camss *camss) int ret; for (i = 0; i < camss->res->csiphy_num; i++) { + if (!camss->csiphy[i].phy) + continue; + ret = msm_csiphy_register_entity(&camss->csiphy[i], &camss->v4l2_dev); if (ret < 0) { @@ -3883,6 +4914,19 @@ static int camss_register_entities(struct camss *camss) } } + if (camss->tpg) { + for (i = 0; i < camss->res->tpg_num; i++) { + ret = msm_tpg_register_entity(&camss->tpg[i], + &camss->v4l2_dev); + if (ret < 0) { + dev_err(camss->dev, + "Failed to register tpg%d entity: %d\n", + i, ret); + goto err_reg_tpg; + } + } + } + for (i = 0; i < camss->res->csid_num; i++) { ret = msm_csid_register_entity(&camss->csid[i], &camss->v4l2_dev); @@ -3926,10 +4970,19 @@ static int camss_register_entities(struct camss *camss) for (i--; i >= 0; i--) msm_csid_unregister_entity(&camss->csid[i]); + i = camss->res->tpg_num; +err_reg_tpg: + if (camss->tpg) { + for (i--; i >= 0; i--) + msm_tpg_unregister_entity(&camss->tpg[i]); + } + i = camss->res->csiphy_num; err_reg_csiphy: - for (i--; i >= 0; i--) - msm_csiphy_unregister_entity(&camss->csiphy[i]); + for (i--; i >= 0; i--) { + if (camss->csiphy[i].phy) + msm_csiphy_unregister_entity(&camss->csiphy[i]); + } return ret; } @@ -3944,8 +4997,15 @@ static void camss_unregister_entities(struct camss *camss) { unsigned int i; - for (i = 0; i < camss->res->csiphy_num; i++) - msm_csiphy_unregister_entity(&camss->csiphy[i]); + for (i = 0; i < camss->res->csiphy_num; i++) { + if (camss->csiphy[i].phy) + msm_csiphy_unregister_entity(&camss->csiphy[i]); + } + + if (camss->tpg) { + for (i = 0; i < camss->res->tpg_num; i++) + msm_tpg_unregister_entity(&camss->tpg[i]); + } for (i = 0; i < camss->res->csid_num; i++) msm_csid_unregister_entity(&camss->csid[i]); @@ -4154,6 +5214,8 @@ static int camss_probe(struct platform_device *pdev) if (!camss) return -ENOMEM; + devm_of_platform_populate(dev); + camss->res = of_device_get_match_data(dev); atomic_set(&camss->ref_count, 0); @@ -4165,12 +5227,20 @@ static int camss_probe(struct platform_device *pdev) if (!camss->csiphy) return -ENOMEM; + if (camss->res->tpg_num > 0) { + camss->tpg = devm_kcalloc(dev, camss->res->tpg_num, + sizeof(*camss->tpg), GFP_KERNEL); + if (!camss->tpg) + return -ENOMEM; + } + camss->csid = devm_kcalloc(dev, camss->res->csid_num, sizeof(*camss->csid), GFP_KERNEL); if (!camss->csid) return -ENOMEM; if (camss->res->version == CAMSS_8x16 || + camss->res->version == CAMSS_8x39 || camss->res->version == CAMSS_8x53 || camss->res->version == CAMSS_8x96) { camss->ispif = devm_kcalloc(dev, 1, sizeof(*camss->ispif), GFP_KERNEL); @@ -4218,7 +5288,7 @@ static int camss_probe(struct platform_device *pdev) pm_runtime_enable(dev); - ret = camss_of_parse_ports(camss); + ret = camss_parse_ports(camss); if (ret < 0) goto err_v4l2_device_unregister; @@ -4293,6 +5363,7 @@ static void camss_remove(struct platform_device *pdev) static const struct camss_resources msm8916_resources = { .version = CAMSS_8x16, + .legacy_phy = true, .csiphy_res = csiphy_res_8x16, .csid_res = csid_res_8x16, .ispif_res = &ispif_res_8x16, @@ -4302,8 +5373,21 @@ static const struct camss_resources msm8916_resources = { .vfe_num = ARRAY_SIZE(vfe_res_8x16), }; +static const struct camss_resources msm8939_resources = { + .version = CAMSS_8x39, + .legacy_phy = true, + .csiphy_res = csiphy_res_8x39, + .csid_res = csid_res_8x39, + .ispif_res = &ispif_res_8x39, + .vfe_res = vfe_res_8x39, + .csiphy_num = ARRAY_SIZE(csiphy_res_8x39), + .csid_num = ARRAY_SIZE(csid_res_8x39), + .vfe_num = ARRAY_SIZE(vfe_res_8x39), +}; + static const struct camss_resources msm8953_resources = { .version = CAMSS_8x53, + .legacy_phy = true, .icc_res = icc_res_8x53, .icc_path_num = ARRAY_SIZE(icc_res_8x53), .csiphy_res = csiphy_res_8x96, @@ -4317,6 +5401,7 @@ static const struct camss_resources msm8953_resources = { static const struct camss_resources msm8996_resources = { .version = CAMSS_8x96, + .legacy_phy = true, .csiphy_res = csiphy_res_8x96, .csid_res = csid_res_8x96, .ispif_res = &ispif_res_8x96, @@ -4328,6 +5413,7 @@ static const struct camss_resources msm8996_resources = { static const struct camss_resources qcm2290_resources = { .version = CAMSS_2290, + .legacy_phy = true, .csiphy_res = csiphy_res_2290, .csid_res = csid_res_2290, .vfe_res = vfe_res_2290, @@ -4341,12 +5427,15 @@ static const struct camss_resources qcm2290_resources = { static const struct camss_resources qcs8300_resources = { .version = CAMSS_8300, .pd_name = "top", + .legacy_phy = true, .csiphy_res = csiphy_res_8300, + .tpg_res = tpg_res_8775p, .csid_res = csid_res_8775p, .csid_wrapper_res = &csid_wrapper_res_sm8550, .vfe_res = vfe_res_8775p, .icc_res = icc_res_qcs8300, .csiphy_num = ARRAY_SIZE(csiphy_res_8300), + .tpg_num = ARRAY_SIZE(tpg_res_8775p), .csid_num = ARRAY_SIZE(csid_res_8775p), .vfe_num = ARRAY_SIZE(vfe_res_8775p), .icc_path_num = ARRAY_SIZE(icc_res_qcs8300), @@ -4355,12 +5444,15 @@ static const struct camss_resources qcs8300_resources = { static const struct camss_resources sa8775p_resources = { .version = CAMSS_8775P, .pd_name = "top", + .legacy_phy = true, .csiphy_res = csiphy_res_8775p, + .tpg_res = tpg_res_8775p, .csid_res = csid_res_8775p, .csid_wrapper_res = &csid_wrapper_res_sm8550, .vfe_res = vfe_res_8775p, .icc_res = icc_res_sa8775p, .csiphy_num = ARRAY_SIZE(csiphy_res_8775p), + .tpg_num = ARRAY_SIZE(tpg_res_8775p), .csid_num = ARRAY_SIZE(csid_res_8775p), .vfe_num = ARRAY_SIZE(vfe_res_8775p), .icc_path_num = ARRAY_SIZE(icc_res_sa8775p), @@ -4368,6 +5460,7 @@ static const struct camss_resources sa8775p_resources = { static const struct camss_resources sdm660_resources = { .version = CAMSS_660, + .legacy_phy = true, .csiphy_res = csiphy_res_660, .csid_res = csid_res_660, .ispif_res = &ispif_res_660, @@ -4379,6 +5472,7 @@ static const struct camss_resources sdm660_resources = { static const struct camss_resources sdm670_resources = { .version = CAMSS_845, + .legacy_phy = true, .csiphy_res = csiphy_res_670, .csid_res = csid_res_670, .vfe_res = vfe_res_670, @@ -4390,6 +5484,7 @@ static const struct camss_resources sdm670_resources = { static const struct camss_resources sdm845_resources = { .version = CAMSS_845, .pd_name = "top", + .legacy_phy = true, .csiphy_res = csiphy_res_845, .csid_res = csid_res_845, .vfe_res = vfe_res_845, @@ -4398,9 +5493,24 @@ static const struct camss_resources sdm845_resources = { .vfe_num = ARRAY_SIZE(vfe_res_845), }; +static const struct camss_resources sm6150_resources = { + .version = CAMSS_6150, + .pd_name = "top", + .legacy_phy = true, + .csiphy_res = csiphy_res_sm6150, + .csid_res = csid_res_sm6150, + .vfe_res = vfe_res_sm6150, + .icc_res = icc_res_sm6150, + .icc_path_num = ARRAY_SIZE(icc_res_sm6150), + .csiphy_num = ARRAY_SIZE(csiphy_res_sm6150), + .csid_num = ARRAY_SIZE(csid_res_sm6150), + .vfe_num = ARRAY_SIZE(vfe_res_sm6150), +}; + static const struct camss_resources sm8250_resources = { .version = CAMSS_8250, .pd_name = "top", + .legacy_phy = true, .csiphy_res = csiphy_res_8250, .csid_res = csid_res_8250, .vfe_res = vfe_res_8250, @@ -4414,6 +5524,7 @@ static const struct camss_resources sm8250_resources = { static const struct camss_resources sc8280xp_resources = { .version = CAMSS_8280XP, .pd_name = "top", + .legacy_phy = true, .csiphy_res = csiphy_res_sc8280xp, .csid_res = csid_res_sc8280xp, .ispif_res = NULL, @@ -4428,6 +5539,7 @@ static const struct camss_resources sc8280xp_resources = { static const struct camss_resources sc7280_resources = { .version = CAMSS_7280, .pd_name = "top", + .legacy_phy = true, .csiphy_res = csiphy_res_7280, .csid_res = csid_res_7280, .vfe_res = vfe_res_7280, @@ -4441,6 +5553,7 @@ static const struct camss_resources sc7280_resources = { static const struct camss_resources sm8550_resources = { .version = CAMSS_8550, .pd_name = "top", + .legacy_phy = true, .csiphy_res = csiphy_res_8550, .csid_res = csid_res_8550, .vfe_res = vfe_res_8550, @@ -4452,22 +5565,56 @@ static const struct camss_resources sm8550_resources = { .vfe_num = ARRAY_SIZE(vfe_res_8550), }; +static const struct camss_resources sm8650_resources = { + .version = CAMSS_8650, + .pd_name = "top", + .legacy_phy = true, + .csiphy_res = csiphy_res_sm8650, + .csid_res = csid_res_sm8650, + .csid_wrapper_res = &csid_wrapper_res_sm8550, + .vfe_res = vfe_res_sm8650, + .icc_res = icc_res_sm8650, + .icc_path_num = ARRAY_SIZE(icc_res_sm8650), + .csiphy_num = ARRAY_SIZE(csiphy_res_sm8650), + .csid_num = ARRAY_SIZE(csid_res_sm8650), + .vfe_num = ARRAY_SIZE(vfe_res_sm8650), +}; + static const struct camss_resources x1e80100_resources = { .version = CAMSS_X1E80100, .pd_name = "top", .csiphy_res = csiphy_res_x1e80100, + .tpg_res = tpg_res_x1e80100, .csid_res = csid_res_x1e80100, .vfe_res = vfe_res_x1e80100, .csid_wrapper_res = &csid_wrapper_res_x1e80100, .icc_res = icc_res_x1e80100, .icc_path_num = ARRAY_SIZE(icc_res_x1e80100), .csiphy_num = ARRAY_SIZE(csiphy_res_x1e80100), + .tpg_num = ARRAY_SIZE(tpg_res_x1e80100), .csid_num = ARRAY_SIZE(csid_res_x1e80100), .vfe_num = ARRAY_SIZE(vfe_res_x1e80100), }; +static const struct camss_resources x1p42100_resources = { + .version = CAMSS_X1P42100, + .pd_name = "top", + .csiphy_res = csiphy_res_x1p42100, + .tpg_res = tpg_res_x1e80100, + .csid_res = csid_res_x1e80100, + .vfe_res = vfe_res_x1p42100, + .csid_wrapper_res = &csid_wrapper_res_x1e80100, + .icc_res = icc_res_x1e80100, + .icc_path_num = ARRAY_SIZE(icc_res_x1e80100), + .csiphy_num = ARRAY_SIZE(csiphy_res_x1p42100), + .tpg_num = ARRAY_SIZE(tpg_res_x1e80100), + .csid_num = ARRAY_SIZE(csid_res_x1e80100), + .vfe_num = ARRAY_SIZE(vfe_res_x1p42100), +}; + static const struct of_device_id camss_dt_match[] = { { .compatible = "qcom,msm8916-camss", .data = &msm8916_resources }, + { .compatible = "qcom,msm8939-camss", .data = &msm8939_resources }, { .compatible = "qcom,msm8953-camss", .data = &msm8953_resources }, { .compatible = "qcom,msm8996-camss", .data = &msm8996_resources }, { .compatible = "qcom,qcm2290-camss", .data = &qcm2290_resources }, @@ -4478,9 +5625,12 @@ static const struct of_device_id camss_dt_match[] = { { .compatible = "qcom,sdm660-camss", .data = &sdm660_resources }, { .compatible = "qcom,sdm670-camss", .data = &sdm670_resources }, { .compatible = "qcom,sdm845-camss", .data = &sdm845_resources }, + { .compatible = "qcom,sm6150-camss", .data = &sm6150_resources }, { .compatible = "qcom,sm8250-camss", .data = &sm8250_resources }, { .compatible = "qcom,sm8550-camss", .data = &sm8550_resources }, + { .compatible = "qcom,sm8650-camss", .data = &sm8650_resources }, { .compatible = "qcom,x1e80100-camss", .data = &x1e80100_resources }, + { .compatible = "qcom,x1p42100-camss", .data = &x1p42100_resources }, { } }; diff --git a/drivers/media/platform/qcom/camss/camss.h b/drivers/media/platform/qcom/camss/camss.h index a70fbc78ccc30..541b5c8bf1926 100644 --- a/drivers/media/platform/qcom/camss/camss.h +++ b/drivers/media/platform/qcom/camss/camss.h @@ -21,6 +21,7 @@ #include "camss-csid.h" #include "camss-csiphy.h" #include "camss-ispif.h" +#include "camss-tpg.h" #include "camss-vfe.h" #include "camss-format.h" @@ -41,9 +42,10 @@ (to_camss_index(ptr_module, index)->dev) #define CAMSS_RES_MAX 17 +#define CAMSS_INIT_BUF_COUNT 2 struct camss_subdev_resources { - char *regulators[CAMSS_RES_MAX]; + struct regulator_bulk_data regulators[CAMSS_RES_MAX]; char *clock[CAMSS_RES_MAX]; char *clock_for_reset[CAMSS_RES_MAX]; u32 clock_rate[CAMSS_RES_MAX][CAMSS_RES_MAX]; @@ -51,6 +53,7 @@ struct camss_subdev_resources { char *interrupt[CAMSS_RES_MAX]; union { struct csiphy_subdev_resources csiphy; + struct tpg_subdev_resources tpg; struct csid_subdev_resources csid; struct vfe_subdev_resources vfe; }; @@ -79,8 +82,10 @@ enum pm_domain { enum camss_version { CAMSS_660, CAMSS_2290, + CAMSS_6150, CAMSS_7280, CAMSS_8x16, + CAMSS_8x39, CAMSS_8x53, CAMSS_8x96, CAMSS_8250, @@ -88,8 +93,10 @@ enum camss_version { CAMSS_8300, CAMSS_845, CAMSS_8550, + CAMSS_8650, CAMSS_8775P, CAMSS_X1E80100, + CAMSS_X1P42100, }; enum icc_count { @@ -100,7 +107,9 @@ enum icc_count { struct camss_resources { enum camss_version version; const char *pd_name; + const bool legacy_phy; const struct camss_subdev_resources *csiphy_res; + const struct camss_subdev_resources *tpg_res; const struct camss_subdev_resources *csid_res; const struct camss_subdev_resources *ispif_res; const struct camss_subdev_resources *vfe_res; @@ -108,6 +117,7 @@ struct camss_resources { const struct resources_icc *icc_res; const unsigned int icc_path_num; const unsigned int csiphy_num; + const unsigned int tpg_num; const unsigned int csid_num; const unsigned int vfe_num; }; @@ -118,6 +128,7 @@ struct camss { struct media_device media_dev; struct device *dev; struct csiphy_device *csiphy; + struct tpg_device *tpg; struct csid_device *csid; struct ispif_device *ispif; struct vfe_device *vfe; diff --git a/drivers/media/platform/qcom/iris/Makefile b/drivers/media/platform/qcom/iris/Makefile index 13270cd6d8998..2fde45f817276 100644 --- a/drivers/media/platform/qcom/iris/Makefile +++ b/drivers/media/platform/qcom/iris/Makefile @@ -10,6 +10,7 @@ qcom-iris-objs += iris_buffer.o \ iris_hfi_gen2_packet.o \ iris_hfi_gen2_response.o \ iris_hfi_queue.o \ + iris_platform_gen1.o \ iris_platform_gen2.o \ iris_power.o \ iris_probe.o \ @@ -22,11 +23,8 @@ qcom-iris-objs += iris_buffer.o \ iris_venc.o \ iris_vpu2.o \ iris_vpu3x.o \ + iris_vpu4x.o \ iris_vpu_buffer.o \ iris_vpu_common.o \ -ifeq ($(CONFIG_VIDEO_QCOM_VENUS),) -qcom-iris-objs += iris_platform_sm8250.o -endif - obj-$(CONFIG_VIDEO_QCOM_IRIS) += qcom-iris.o diff --git a/drivers/media/platform/qcom/iris/iris_buffer.c b/drivers/media/platform/qcom/iris/iris_buffer.c index c0900038e7def..0db94a5e38789 100644 --- a/drivers/media/platform/qcom/iris/iris_buffer.c +++ b/drivers/media/platform/qcom/iris/iris_buffer.c @@ -171,9 +171,14 @@ static u32 iris_yuv_buffer_size_nv12(struct iris_inst *inst) static u32 iris_yuv_buffer_size_qc08c(struct iris_inst *inst) { u32 y_plane, uv_plane, y_stride, uv_stride; - struct v4l2_format *f = inst->fmt_dst; u32 uv_meta_stride, uv_meta_plane; u32 y_meta_stride, y_meta_plane; + struct v4l2_format *f = NULL; + + if (inst->domain == DECODER) + f = inst->fmt_dst; + else + f = inst->fmt_src; y_meta_stride = ALIGN(DIV_ROUND_UP(f->fmt.pix_mp.width, META_STRIDE_ALIGNED >> 1), META_STRIDE_ALIGNED); @@ -261,7 +266,10 @@ int iris_get_buffer_size(struct iris_inst *inst, case BUF_INPUT: return iris_dec_bitstream_buffer_size(inst); case BUF_OUTPUT: - return iris_yuv_buffer_size_nv12(inst); + if (inst->fmt_dst->fmt.pix_mp.pixelformat == V4L2_PIX_FMT_QC08C) + return iris_yuv_buffer_size_qc08c(inst); + else + return iris_yuv_buffer_size_nv12(inst); case BUF_DPB: return iris_yuv_buffer_size_qc08c(inst); default: @@ -270,7 +278,10 @@ int iris_get_buffer_size(struct iris_inst *inst, } else { switch (buffer_type) { case BUF_INPUT: - return iris_yuv_buffer_size_nv12(inst); + if (inst->fmt_src->fmt.pix_mp.pixelformat == V4L2_PIX_FMT_QC08C) + return iris_yuv_buffer_size_qc08c(inst); + else + return iris_yuv_buffer_size_nv12(inst); case BUF_OUTPUT: return iris_enc_bitstream_buffer_size(inst); default: @@ -325,7 +336,7 @@ static int iris_create_internal_buffer(struct iris_inst *inst, enum iris_buffer_type buffer_type, u32 index) { struct iris_buffers *buffers = &inst->buffers[buffer_type]; - struct iris_core *core = inst->core; + struct device *dev = iris_get_cb_dev(inst, buffer_type); struct iris_buffer *buffer; if (!buffers->size) @@ -340,12 +351,15 @@ static int iris_create_internal_buffer(struct iris_inst *inst, buffer->index = index; buffer->buffer_size = buffers->size; buffer->dma_attrs = DMA_ATTR_WRITE_COMBINE | DMA_ATTR_NO_KERNEL_MAPPING; - list_add_tail(&buffer->list, &buffers->list); - buffer->kvaddr = dma_alloc_attrs(core->dev, buffer->buffer_size, + buffer->kvaddr = dma_alloc_attrs(dev, buffer->buffer_size, &buffer->device_addr, GFP_KERNEL, buffer->dma_attrs); - if (!buffer->kvaddr) + if (!buffer->kvaddr) { + kfree(buffer); return -ENOMEM; + } + + list_add_tail(&buffer->list, &buffers->list); return 0; } @@ -475,10 +489,10 @@ int iris_queue_internal_buffers(struct iris_inst *inst, u32 plane) int iris_destroy_internal_buffer(struct iris_inst *inst, struct iris_buffer *buffer) { - struct iris_core *core = inst->core; + struct device *dev = iris_get_cb_dev(inst, buffer->type); list_del(&buffer->list); - dma_free_attrs(core->dev, buffer->buffer_size, buffer->kvaddr, + dma_free_attrs(dev, buffer->buffer_size, buffer->kvaddr, buffer->device_addr, buffer->dma_attrs); kfree(buffer); diff --git a/drivers/media/platform/qcom/iris/iris_buffer.h b/drivers/media/platform/qcom/iris/iris_buffer.h index 325d30fce5c99..75bb767761824 100644 --- a/drivers/media/platform/qcom/iris/iris_buffer.h +++ b/drivers/media/platform/qcom/iris/iris_buffer.h @@ -27,6 +27,7 @@ struct iris_inst; * @BUF_SCRATCH_1: buffer to store decoding/encoding context data for HW * @BUF_SCRATCH_2: buffer to store encoding context data for HW * @BUF_VPSS: buffer to store VPSS context data for HW + * @BUF_PARTIAL: buffer for AV1 IBC data * @BUF_TYPE_MAX: max buffer types */ enum iris_buffer_type { @@ -42,6 +43,7 @@ enum iris_buffer_type { BUF_SCRATCH_1, BUF_SCRATCH_2, BUF_VPSS, + BUF_PARTIAL, BUF_TYPE_MAX, }; diff --git a/drivers/media/platform/qcom/iris/iris_common.c b/drivers/media/platform/qcom/iris/iris_common.c index 9fc663bdaf3fc..7f1c7fe144f70 100644 --- a/drivers/media/platform/qcom/iris/iris_common.c +++ b/drivers/media/platform/qcom/iris/iris_common.c @@ -91,12 +91,14 @@ int iris_process_streamon_input(struct iris_inst *inst) int iris_process_streamon_output(struct iris_inst *inst) { const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops; - bool drain_active = false, drc_active = false; enum iris_inst_sub_state clear_sub_state = 0; + bool drain_active, drc_active, first_ipsc; int ret = 0; iris_scale_power(inst); + first_ipsc = inst->sub_state & IRIS_INST_SUB_FIRST_IPSC; + drain_active = inst->sub_state & IRIS_INST_SUB_DRAIN && inst->sub_state & IRIS_INST_SUB_DRAIN_LAST; @@ -108,7 +110,8 @@ int iris_process_streamon_output(struct iris_inst *inst) else if (drain_active) clear_sub_state = IRIS_INST_SUB_DRAIN | IRIS_INST_SUB_DRAIN_LAST; - if (inst->domain == DECODER && inst->sub_state & IRIS_INST_SUB_INPUT_PAUSE) { + /* Input internal buffer reconfiguration required in case of resolution change */ + if (first_ipsc || drc_active) { ret = iris_alloc_and_queue_input_int_bufs(inst); if (ret) return ret; diff --git a/drivers/media/platform/qcom/iris/iris_core.c b/drivers/media/platform/qcom/iris/iris_core.c index 8406c48d635b6..62a9e15b47372 100644 --- a/drivers/media/platform/qcom/iris/iris_core.c +++ b/drivers/media/platform/qcom/iris/iris_core.c @@ -18,6 +18,7 @@ void iris_core_deinit(struct iris_core *core) if (core->state != IRIS_CORE_DEINIT) { iris_fw_unload(core); iris_vpu_power_off(core); + iris_fw_deinit(core); iris_hfi_queues_deinit(core); core->state = IRIS_CORE_DEINIT; } @@ -67,10 +68,14 @@ int iris_core_init(struct iris_core *core) if (ret) goto error_queue_deinit; - ret = iris_fw_load(core); + ret = iris_fw_init(core); if (ret) goto error_power_off; + ret = iris_fw_load(core); + if (ret) + goto error_firmware_deinit; + ret = iris_vpu_boot_firmware(core); if (ret) goto error_unload_fw; @@ -85,6 +90,8 @@ int iris_core_init(struct iris_core *core) error_unload_fw: iris_fw_unload(core); +error_firmware_deinit: + iris_fw_deinit(core); error_power_off: iris_vpu_power_off(core); error_queue_deinit: diff --git a/drivers/media/platform/qcom/iris/iris_core.h b/drivers/media/platform/qcom/iris/iris_core.h index fb194c967ad4f..c85fd7f292467 100644 --- a/drivers/media/platform/qcom/iris/iris_core.h +++ b/drivers/media/platform/qcom/iris/iris_core.h @@ -34,6 +34,9 @@ enum domain_type { * struct iris_core - holds core parameters valid for all instances * * @dev: reference to device structure + * @dev_np: reference to non-pixel context bank device structure + * @dev_p: reference to pixel context bank device structure + * @dev_bs: reference to bitstream context bank device structure * @reg_base: IO memory base address * @irq: iris irq * @v4l2_dev: a holder for v4l2 device structure @@ -77,11 +80,19 @@ enum domain_type { struct iris_core { struct device *dev; + struct device *dev_np; + struct device *dev_p; + struct device *dev_bs; void __iomem *reg_base; int irq; struct v4l2_device v4l2_dev; struct video_device *vdev_dec; struct video_device *vdev_enc; + struct video_firmware { + struct device *dev; + struct qcom_scm_pas_context *ctx; + struct iommu_domain *iommu_domain; + } fw; const struct v4l2_file_operations *iris_v4l2_file_ops; const struct v4l2_ioctl_ops *iris_v4l2_ioctl_ops_dec; const struct v4l2_ioctl_ops *iris_v4l2_ioctl_ops_enc; diff --git a/drivers/media/platform/qcom/iris/iris_ctrls.c b/drivers/media/platform/qcom/iris/iris_ctrls.c index 754a5ad718bc3..3cec957580f5e 100644 --- a/drivers/media/platform/qcom/iris/iris_ctrls.c +++ b/drivers/media/platform/qcom/iris/iris_ctrls.c @@ -98,6 +98,20 @@ static enum platform_inst_fw_cap_type iris_get_cap_id(u32 id) return B_FRAME_QP_H264; case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_QP: return B_FRAME_QP_HEVC; + case V4L2_CID_MPEG_VIDEO_AV1_PROFILE: + return PROFILE_AV1; + case V4L2_CID_MPEG_VIDEO_AV1_LEVEL: + return LEVEL_AV1; + case V4L2_CID_ROTATE: + return ROTATION; + case V4L2_CID_HFLIP: + return HFLIP; + case V4L2_CID_VFLIP: + return VFLIP; + case V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE: + return IR_TYPE; + case V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD: + return IR_PERIOD; default: return INST_FW_CAP_MAX; } @@ -185,6 +199,20 @@ static u32 iris_get_v4l2_id(enum platform_inst_fw_cap_type cap_id) return V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP; case B_FRAME_QP_HEVC: return V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_QP; + case PROFILE_AV1: + return V4L2_CID_MPEG_VIDEO_AV1_PROFILE; + case LEVEL_AV1: + return V4L2_CID_MPEG_VIDEO_AV1_LEVEL; + case ROTATION: + return V4L2_CID_ROTATE; + case HFLIP: + return V4L2_CID_HFLIP; + case VFLIP: + return V4L2_CID_VFLIP; + case IR_TYPE: + return V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE; + case IR_PERIOD: + return V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD; default: return 0; } @@ -301,7 +329,7 @@ int iris_ctrls_init(struct iris_inst *inst) void iris_session_init_caps(struct iris_core *core) { - struct platform_inst_fw_cap *caps; + const struct platform_inst_fw_cap *caps; u32 i, num_cap, cap_id; caps = core->iris_platform_data->inst_fw_caps_dec; @@ -313,13 +341,23 @@ void iris_session_init_caps(struct iris_core *core) continue; core->inst_fw_caps_dec[cap_id].cap_id = caps[i].cap_id; - core->inst_fw_caps_dec[cap_id].min = caps[i].min; - core->inst_fw_caps_dec[cap_id].max = caps[i].max; core->inst_fw_caps_dec[cap_id].step_or_mask = caps[i].step_or_mask; - core->inst_fw_caps_dec[cap_id].value = caps[i].value; core->inst_fw_caps_dec[cap_id].flags = caps[i].flags; core->inst_fw_caps_dec[cap_id].hfi_id = caps[i].hfi_id; core->inst_fw_caps_dec[cap_id].set = caps[i].set; + + if (cap_id == PIPE) { + core->inst_fw_caps_dec[cap_id].value = + core->iris_platform_data->num_vpp_pipe; + core->inst_fw_caps_dec[cap_id].min = + core->iris_platform_data->num_vpp_pipe; + core->inst_fw_caps_dec[cap_id].max = + core->iris_platform_data->num_vpp_pipe; + } else { + core->inst_fw_caps_dec[cap_id].min = caps[i].min; + core->inst_fw_caps_dec[cap_id].max = caps[i].max; + core->inst_fw_caps_dec[cap_id].value = caps[i].value; + } } caps = core->iris_platform_data->inst_fw_caps_enc; @@ -883,6 +921,81 @@ int iris_set_qp_range(struct iris_inst *inst, enum platform_inst_fw_cap_type cap &range, sizeof(range)); } +int iris_set_rotation(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id) +{ + const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops; + u32 hfi_id = inst->fw_caps[cap_id].hfi_id; + u32 hfi_val; + + switch (inst->fw_caps[cap_id].value) { + case 0: + hfi_val = HFI_ROTATION_NONE; + return 0; + case 90: + hfi_val = HFI_ROTATION_90; + break; + case 180: + hfi_val = HFI_ROTATION_180; + break; + case 270: + hfi_val = HFI_ROTATION_270; + break; + default: + return -EINVAL; + } + + return hfi_ops->session_set_property(inst, hfi_id, + HFI_HOST_FLAGS_NONE, + iris_get_port_info(inst, cap_id), + HFI_PAYLOAD_U32, + &hfi_val, sizeof(u32)); +} + +int iris_set_flip(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id) +{ + const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops; + u32 hfi_id = inst->fw_caps[cap_id].hfi_id; + u32 hfi_val = HFI_DISABLE_FLIP; + + if (inst->fw_caps[HFLIP].value) + hfi_val |= HFI_HORIZONTAL_FLIP; + + if (inst->fw_caps[VFLIP].value) + hfi_val |= HFI_VERTICAL_FLIP; + + return hfi_ops->session_set_property(inst, hfi_id, + HFI_HOST_FLAGS_NONE, + iris_get_port_info(inst, cap_id), + HFI_PAYLOAD_U32_ENUM, + &hfi_val, sizeof(u32)); +} + +int iris_set_ir_period(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id) +{ + const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops; + struct vb2_queue *q = v4l2_m2m_get_dst_vq(inst->m2m_ctx); + u32 ir_period = inst->fw_caps[cap_id].value; + u32 ir_type = 0; + + if (inst->fw_caps[IR_TYPE].value == + V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM) { + if (vb2_is_streaming(q)) + return 0; + ir_type = HFI_PROP_IR_RANDOM_PERIOD; + } else if (inst->fw_caps[IR_TYPE].value == + V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC) { + ir_type = HFI_PROP_IR_CYCLIC_PERIOD; + } else { + return -EINVAL; + } + + return hfi_ops->session_set_property(inst, ir_type, + HFI_HOST_FLAGS_NONE, + iris_get_port_info(inst, cap_id), + HFI_PAYLOAD_U32, + &ir_period, sizeof(u32)); +} + int iris_set_properties(struct iris_inst *inst, u32 plane) { const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops; diff --git a/drivers/media/platform/qcom/iris/iris_ctrls.h b/drivers/media/platform/qcom/iris/iris_ctrls.h index 30af333cc4941..9518803577bc3 100644 --- a/drivers/media/platform/qcom/iris/iris_ctrls.h +++ b/drivers/media/platform/qcom/iris/iris_ctrls.h @@ -32,6 +32,9 @@ int iris_set_min_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_i int iris_set_max_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id); int iris_set_frame_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id); int iris_set_qp_range(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id); +int iris_set_rotation(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id); +int iris_set_flip(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id); +int iris_set_ir_period(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id); int iris_set_properties(struct iris_inst *inst, u32 plane); #endif diff --git a/drivers/media/platform/qcom/iris/iris_firmware.c b/drivers/media/platform/qcom/iris/iris_firmware.c index 9ab499fad9464..2a5d6c8170212 100644 --- a/drivers/media/platform/qcom/iris/iris_firmware.c +++ b/drivers/media/platform/qcom/iris/iris_firmware.c @@ -3,10 +3,15 @@ * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. */ +#include #include #include +#include +#include #include +#include #include +#include #include #include "iris_core.h" @@ -17,30 +22,32 @@ static int iris_load_fw_to_memory(struct iris_core *core, const char *fw_name) { u32 pas_id = core->iris_platform_data->pas_id; + struct qcom_scm_pas_context *ctx; const struct firmware *firmware = NULL; struct device *dev = core->dev; - struct reserved_mem *rmem; - struct device_node *node; + struct resource res; phys_addr_t mem_phys; size_t res_size; ssize_t fw_size; - void *mem_virt; int ret; if (strlen(fw_name) >= MAX_FIRMWARE_NAME_SIZE - 4) return -EINVAL; - node = of_parse_phandle(dev->of_node, "memory-region", 0); - if (!node) - return -EINVAL; + ret = of_reserved_mem_region_to_resource(dev->of_node, 0, &res); + if (ret) + return ret; - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) - return -EINVAL; + mem_phys = res.start; + res_size = resource_size(&res); + + dev = core->fw.dev ? : core->dev; + + ctx = devm_qcom_scm_pas_context_alloc(dev, pas_id, mem_phys, res_size); + if (!ctx) + return -ENOMEM; - mem_phys = rmem->base; - res_size = rmem->size; + ctx->use_tzmem = core->fw.dev; ret = request_firmware(&firmware, fw_name, dev); if (ret) @@ -52,16 +59,28 @@ static int iris_load_fw_to_memory(struct iris_core *core, const char *fw_name) goto err_release_fw; } - mem_virt = memremap(mem_phys, res_size, MEMREMAP_WC); - if (!mem_virt) { - ret = -ENOMEM; + ret = qcom_mdt_pas_load(ctx, firmware, fw_name, NULL); + qcom_scm_pas_metadata_release(ctx); + if (ret) goto err_release_fw; + + if (core->fw.iommu_domain) { + ret = iommu_map(core->fw.iommu_domain, 0, mem_phys, res_size, + IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV, GFP_KERNEL); + if (ret) + goto err_release_fw; } - ret = qcom_mdt_load(dev, firmware, fw_name, - pas_id, mem_virt, mem_phys, res_size, NULL); + ret = qcom_scm_pas_prepare_and_auth_reset(ctx); + if (ret) + goto err_iommu_unmap; + + core->fw.ctx = ctx; - memunmap(mem_virt); + return ret; + +err_iommu_unmap: + iommu_unmap(core->fw.iommu_domain, 0, res_size); err_release_fw: release_firmware(firmware); @@ -70,9 +89,9 @@ static int iris_load_fw_to_memory(struct iris_core *core, const char *fw_name) int iris_fw_load(struct iris_core *core) { - struct tz_cp_config *cp_config = core->iris_platform_data->tz_cp_config_data; + const struct tz_cp_config *cp_config; const char *fwpath = NULL; - int ret; + int i, ret; ret = of_property_read_string_index(core->dev->of_node, "firmware-name", 0, &fwpath); @@ -85,20 +104,17 @@ int iris_fw_load(struct iris_core *core) return -ENOMEM; } - ret = qcom_scm_pas_auth_and_reset(core->iris_platform_data->pas_id); - if (ret) { - dev_err(core->dev, "auth and reset failed: %d\n", ret); - return ret; - } - - ret = qcom_scm_mem_protect_video_var(cp_config->cp_start, - cp_config->cp_size, - cp_config->cp_nonpixel_start, - cp_config->cp_nonpixel_size); - if (ret) { - dev_err(core->dev, "protect memory failed\n"); - qcom_scm_pas_shutdown(core->iris_platform_data->pas_id); - return ret; + for (i = 0; i < core->iris_platform_data->tz_cp_config_data_size; i++) { + cp_config = &core->iris_platform_data->tz_cp_config_data[i]; + ret = qcom_scm_mem_protect_video_var(cp_config->cp_start, + cp_config->cp_size, + cp_config->cp_nonpixel_start, + cp_config->cp_nonpixel_size); + if (ret) { + dev_err(core->dev, "qcom_scm_mem_protect_video_var failed: %d\n", ret); + iris_fw_unload(core); + return ret; + } } return ret; @@ -106,10 +122,91 @@ int iris_fw_load(struct iris_core *core) int iris_fw_unload(struct iris_core *core) { - return qcom_scm_pas_shutdown(core->iris_platform_data->pas_id); + struct qcom_scm_pas_context *ctx = core->fw.ctx; + int ret; + + if (!ctx) + return -EINVAL; + + ret = qcom_scm_pas_shutdown(ctx->pas_id); + if (core->fw.iommu_domain) + iommu_unmap(core->fw.iommu_domain, 0, ctx->mem_size); + + core->fw.ctx = NULL; + return ret; } int iris_set_hw_state(struct iris_core *core, bool resume) { return qcom_scm_set_remote_state(resume, 0); } + +int iris_fw_init(struct iris_core *core) +{ + struct platform_device_info info; + struct iommu_domain *iommu_dom; + struct platform_device *pdev; + struct device_node *np; + int ret; + + np = of_get_child_by_name(core->dev->of_node, "video-firmware"); + if (!np) + return 0; + + memset(&info, 0, sizeof(info)); + info.fwnode = &np->fwnode; + info.parent = core->dev; + info.name = np->name; + info.dma_mask = DMA_BIT_MASK(32); + + pdev = platform_device_register_full(&info); + if (IS_ERR(pdev)) { + of_node_put(np); + return PTR_ERR(pdev); + } + + pdev->dev.of_node = np; + + ret = of_dma_configure(&pdev->dev, np, true); + if (ret) + goto err_unregister; + + core->fw.dev = &pdev->dev; + + iommu_dom = iommu_get_domain_for_dev(core->fw.dev); + if (!iommu_dom) { + ret = -EINVAL; + goto err_unset_fw_dev; + } + + ret = iommu_attach_device(iommu_dom, core->fw.dev); + if (ret) + goto err_unset_fw_dev; + + core->fw.iommu_domain = iommu_dom; + + of_node_put(np); + + return 0; + +err_unset_fw_dev: + core->fw.dev = NULL; +err_unregister: + platform_device_unregister(pdev); + of_node_put(np); + return ret; +} + +void iris_fw_deinit(struct iris_core *core) +{ + if (!core->fw.dev) + return; + + if (core->fw.iommu_domain) { + iommu_detach_device(core->fw.iommu_domain, core->fw.dev); + core->fw.iommu_domain = NULL; + } + + platform_device_unregister(to_platform_device(core->fw.dev)); + core->fw.dev = NULL; +} diff --git a/drivers/media/platform/qcom/iris/iris_firmware.h b/drivers/media/platform/qcom/iris/iris_firmware.h index e833ecd348871..adde461099667 100644 --- a/drivers/media/platform/qcom/iris/iris_firmware.h +++ b/drivers/media/platform/qcom/iris/iris_firmware.h @@ -11,5 +11,7 @@ struct iris_core; int iris_fw_load(struct iris_core *core); int iris_fw_unload(struct iris_core *core); int iris_set_hw_state(struct iris_core *core, bool resume); +int iris_fw_init(struct iris_core *core); +void iris_fw_deinit(struct iris_core *core); #endif diff --git a/drivers/media/platform/qcom/iris/iris_hfi_common.h b/drivers/media/platform/qcom/iris/iris_hfi_common.h index b51471fb32c70..3edb5ae582b49 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_common.h +++ b/drivers/media/platform/qcom/iris/iris_hfi_common.h @@ -141,6 +141,9 @@ struct hfi_subscription_params { u32 profile; u32 level; u32 tier; + u32 drap; + u32 film_grain; + u32 super_block; }; u32 iris_hfi_get_v4l2_color_primaries(u32 hfi_primaries); diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c index e1788c266bb10..11815f6f5bacd 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c @@ -282,7 +282,7 @@ static int iris_hfi_gen1_queue_input_buffer(struct iris_inst *inst, struct iris_ com_ip_pkt.shdr.session_id = inst->session_id; com_ip_pkt.time_stamp_hi = upper_32_bits(buf->timestamp); com_ip_pkt.time_stamp_lo = lower_32_bits(buf->timestamp); - com_ip_pkt.flags = buf->flags; + com_ip_pkt.flags = 0; com_ip_pkt.mark_target = 0; com_ip_pkt.mark_data = 0; com_ip_pkt.offset = buf->data_offset; @@ -441,6 +441,8 @@ static int iris_hfi_gen1_session_unset_buffers(struct iris_inst *inst, struct ir goto exit; ret = iris_wait_for_session_response(inst, false); + if (!ret) + ret = iris_destroy_internal_buffer(inst, buf); exit: kfree(pkt); @@ -733,7 +735,7 @@ static int iris_hfi_gen1_set_resolution(struct iris_inst *inst, u32 plane) struct hfi_framesize fs; int ret; - if (!iris_drc_pending(inst)) { + if (!iris_drc_pending(inst) && !(inst->sub_state & IRIS_INST_SUB_FIRST_IPSC)) { fs.buffer_type = HFI_BUFFER_INPUT; fs.width = inst->fmt_src->fmt.pix_mp.width; fs.height = inst->fmt_src->fmt.pix_mp.height; @@ -774,27 +776,29 @@ static int iris_hfi_gen1_set_raw_format(struct iris_inst *inst, u32 plane) pixelformat = inst->fmt_dst->fmt.pix_mp.pixelformat; if (iris_split_mode_enabled(inst)) { fmt.buffer_type = HFI_BUFFER_OUTPUT; - fmt.format = pixelformat == V4L2_PIX_FMT_NV12 ? - HFI_COLOR_FORMAT_NV12_UBWC : 0; + fmt.format = HFI_COLOR_FORMAT_NV12_UBWC; ret = hfi_gen1_set_property(inst, ptype, &fmt, sizeof(fmt)); if (ret) return ret; fmt.buffer_type = HFI_BUFFER_OUTPUT2; - fmt.format = pixelformat == V4L2_PIX_FMT_NV12 ? HFI_COLOR_FORMAT_NV12 : 0; + fmt.format = pixelformat == V4L2_PIX_FMT_NV12 ? + HFI_COLOR_FORMAT_NV12 : HFI_COLOR_FORMAT_NV12_UBWC; ret = hfi_gen1_set_property(inst, ptype, &fmt, sizeof(fmt)); } else { fmt.buffer_type = HFI_BUFFER_OUTPUT; - fmt.format = pixelformat == V4L2_PIX_FMT_NV12 ? HFI_COLOR_FORMAT_NV12 : 0; + fmt.format = pixelformat == V4L2_PIX_FMT_NV12 ? + HFI_COLOR_FORMAT_NV12 : HFI_COLOR_FORMAT_NV12_UBWC; ret = hfi_gen1_set_property(inst, ptype, &fmt, sizeof(fmt)); } } else { pixelformat = inst->fmt_src->fmt.pix_mp.pixelformat; fmt.buffer_type = HFI_BUFFER_INPUT; - fmt.format = pixelformat == V4L2_PIX_FMT_NV12 ? HFI_COLOR_FORMAT_NV12 : 0; + fmt.format = pixelformat == V4L2_PIX_FMT_NV12 ? + HFI_COLOR_FORMAT_NV12 : HFI_COLOR_FORMAT_NV12_UBWC; ret = hfi_gen1_set_property(inst, ptype, &fmt, sizeof(fmt)); } @@ -806,6 +810,9 @@ static int iris_hfi_gen1_set_format_constraints(struct iris_inst *inst, u32 plan const u32 ptype = HFI_PROPERTY_PARAM_UNCOMPRESSED_PLANE_ACTUAL_CONSTRAINTS_INFO; struct hfi_uncompressed_plane_actual_constraints_info pconstraint; + if (inst->fmt_dst->fmt.pix_mp.pixelformat == V4L2_PIX_FMT_QC08C) + return 0; + pconstraint.buffer_type = HFI_BUFFER_OUTPUT2; pconstraint.num_planes = 2; pconstraint.plane_format[0].stride_multiples = 128; diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c index 4ce71a1425083..715ec9575b90f 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c @@ -10,6 +10,7 @@ #define UNSPECIFIED_COLOR_FORMAT 5 #define NUM_SYS_INIT_PACKETS 8 +#define NUM_COMV_AV1 18 #define SYS_INIT_PKT_SIZE (sizeof(struct iris_hfi_header) + \ NUM_SYS_INIT_PACKETS * (sizeof(struct iris_hfi_packet) + sizeof(u32))) @@ -121,6 +122,7 @@ static u32 iris_hfi_gen2_get_port_from_buf_type(struct iris_inst *inst, case BUF_COMV: case BUF_NON_COMV: case BUF_LINE: + case BUF_PARTIAL: return HFI_PORT_BITSTREAM; case BUF_OUTPUT: case BUF_DPB: @@ -168,8 +170,7 @@ static int iris_hfi_gen2_session_set_property(struct iris_inst *inst, u32 packet static int iris_hfi_gen2_set_raw_resolution(struct iris_inst *inst, u32 plane) { - u32 resolution = inst->fmt_src->fmt.pix_mp.width << 16 | - inst->fmt_src->fmt.pix_mp.height; + u32 resolution = inst->enc_raw_width << 16 | inst->enc_raw_height; u32 port = iris_hfi_gen2_get_port(inst, plane); return iris_hfi_gen2_session_set_property(inst, @@ -181,22 +182,36 @@ static int iris_hfi_gen2_set_raw_resolution(struct iris_inst *inst, u32 plane) sizeof(u32)); } +static inline u32 iris_hfi_get_aligned_resolution(struct iris_inst *inst, u32 width, u32 height) +{ + u32 codec_align = inst->codec == V4L2_PIX_FMT_HEVC ? 32 : 16; + + return (ALIGN(width, codec_align) << 16 | ALIGN(height, codec_align)); +} + static int iris_hfi_gen2_set_bitstream_resolution(struct iris_inst *inst, u32 plane) { struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); u32 port = iris_hfi_gen2_get_port(inst, plane); enum hfi_packet_payload_info payload_type; - u32 resolution, codec_align; + u32 width, height; + u32 resolution; if (inst->domain == DECODER) { - resolution = inst->fmt_src->fmt.pix_mp.width << 16 | - inst->fmt_src->fmt.pix_mp.height; + width = inst->fmt_src->fmt.pix_mp.width; + height = inst->fmt_src->fmt.pix_mp.height; + resolution = iris_hfi_get_aligned_resolution(inst, width, height); inst_hfi_gen2->src_subcr_params.bitstream_resolution = resolution; payload_type = HFI_PAYLOAD_U32; } else { - codec_align = inst->codec == V4L2_PIX_FMT_HEVC ? 32 : 16; - resolution = ALIGN(inst->fmt_dst->fmt.pix_mp.width, codec_align) << 16 | - ALIGN(inst->fmt_dst->fmt.pix_mp.height, codec_align); + if (is_rotation_90_or_270(inst)) { + width = inst->enc_scale_height; + height = inst->enc_scale_width; + } else { + width = inst->enc_scale_width; + height = inst->enc_scale_height; + } + resolution = iris_hfi_get_aligned_resolution(inst, width, height); inst_hfi_gen2->dst_subcr_params.bitstream_resolution = resolution; payload_type = HFI_PAYLOAD_32_PACKED; } @@ -216,7 +231,7 @@ static int iris_hfi_gen2_set_crop_offsets(struct iris_inst *inst, u32 plane) u32 port = iris_hfi_gen2_get_port(inst, plane); u32 bottom_offset, right_offset; u32 left_offset, top_offset; - u32 payload[2]; + u32 payload[2], codec_align; if (inst->domain == DECODER) { if (V4L2_TYPE_IS_OUTPUT(plane)) { @@ -231,10 +246,27 @@ static int iris_hfi_gen2_set_crop_offsets(struct iris_inst *inst, u32 plane) top_offset = inst->compose.top; } } else { - bottom_offset = (inst->fmt_src->fmt.pix_mp.height - inst->crop.height); - right_offset = (inst->fmt_src->fmt.pix_mp.width - inst->crop.width); - left_offset = inst->crop.left; - top_offset = inst->crop.top; + codec_align = inst->codec == V4L2_PIX_FMT_HEVC ? 32 : 16; + if (V4L2_TYPE_IS_OUTPUT(plane)) { + bottom_offset = (inst->enc_raw_height - inst->crop.height); + right_offset = (inst->enc_raw_width - inst->crop.width); + left_offset = inst->crop.left; + top_offset = inst->crop.top; + } else { + if (is_rotation_90_or_270(inst)) { + bottom_offset = (ALIGN(inst->enc_scale_width, codec_align) - + inst->enc_scale_width); + right_offset = (ALIGN(inst->enc_scale_height, codec_align) - + inst->enc_scale_height); + } else { + bottom_offset = (ALIGN(inst->enc_scale_height, codec_align) - + inst->enc_scale_height); + right_offset = (ALIGN(inst->enc_scale_width, codec_align) - + inst->enc_scale_width); + } + left_offset = 0; + top_offset = 0; + } } payload[0] = FIELD_PREP(GENMASK(31, 16), left_offset) | top_offset; @@ -380,6 +412,9 @@ static int iris_hfi_gen2_set_profile(struct iris_inst *inst, u32 plane) case V4L2_PIX_FMT_H264: profile = inst->fw_caps[PROFILE_H264].value; break; + case V4L2_PIX_FMT_AV1: + profile = inst->fw_caps[PROFILE_AV1].value; + break; } inst_hfi_gen2->src_subcr_params.profile = profile; @@ -409,6 +444,9 @@ static int iris_hfi_gen2_set_level(struct iris_inst *inst, u32 plane) case V4L2_PIX_FMT_H264: level = inst->fw_caps[LEVEL_H264].value; break; + case V4L2_PIX_FMT_AV1: + level = inst->fw_caps[LEVEL_AV1].value; + break; } inst_hfi_gen2->src_subcr_params.level = level; @@ -422,6 +460,20 @@ static int iris_hfi_gen2_set_level(struct iris_inst *inst, u32 plane) sizeof(u32)); } +static int iris_hfi_gen2_set_opb_enable(struct iris_inst *inst, u32 plane) +{ + u32 port = iris_hfi_gen2_get_port(inst, plane); + u32 opb_enable = iris_split_mode_enabled(inst); + + return iris_hfi_gen2_session_set_property(inst, + HFI_PROP_OPB_ENABLE, + HFI_HOST_FLAGS_NONE, + port, + HFI_PAYLOAD_U32, + &opb_enable, + sizeof(u32)); +} + static int iris_hfi_gen2_set_colorformat(struct iris_inst *inst, u32 plane) { u32 port = iris_hfi_gen2_get_port(inst, plane); @@ -429,10 +481,12 @@ static int iris_hfi_gen2_set_colorformat(struct iris_inst *inst, u32 plane) if (inst->domain == DECODER) { pixelformat = inst->fmt_dst->fmt.pix_mp.pixelformat; - hfi_colorformat = pixelformat == V4L2_PIX_FMT_NV12 ? HFI_COLOR_FMT_NV12 : 0; + hfi_colorformat = pixelformat == V4L2_PIX_FMT_NV12 ? + HFI_COLOR_FMT_NV12 : HFI_COLOR_FMT_NV12_UBWC; } else { pixelformat = inst->fmt_src->fmt.pix_mp.pixelformat; - hfi_colorformat = pixelformat == V4L2_PIX_FMT_NV12 ? HFI_COLOR_FMT_NV12 : 0; + hfi_colorformat = pixelformat == V4L2_PIX_FMT_NV12 ? + HFI_COLOR_FMT_NV12 : HFI_COLOR_FMT_NV12_UBWC; } return iris_hfi_gen2_session_set_property(inst, @@ -480,10 +534,12 @@ static int iris_hfi_gen2_set_linear_stride_scanline(struct iris_inst *inst, u32 static int iris_hfi_gen2_set_tier(struct iris_inst *inst, u32 plane) { - struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); u32 port = iris_hfi_gen2_get_port(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); u32 tier = inst->fw_caps[TIER].value; + tier = (inst->codec == V4L2_PIX_FMT_AV1) ? inst->fw_caps[TIER_AV1].value : + inst->fw_caps[TIER].value; inst_hfi_gen2->src_subcr_params.tier = tier; return iris_hfi_gen2_session_set_property(inst, @@ -509,6 +565,40 @@ static int iris_hfi_gen2_set_frame_rate(struct iris_inst *inst, u32 plane) sizeof(u32)); } +static int iris_hfi_gen2_set_film_grain(struct iris_inst *inst, u32 plane) +{ + u32 port = iris_hfi_gen2_get_port(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); + u32 film_grain = inst->fw_caps[FILM_GRAIN].value; + + inst_hfi_gen2->src_subcr_params.film_grain = film_grain; + + return iris_hfi_gen2_session_set_property(inst, + HFI_PROP_AV1_FILM_GRAIN_PRESENT, + HFI_HOST_FLAGS_NONE, + port, + HFI_PAYLOAD_U32_ENUM, + &film_grain, + sizeof(u32)); +} + +static int iris_hfi_gen2_set_super_block(struct iris_inst *inst, u32 plane) +{ + u32 port = iris_hfi_gen2_get_port(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); + u32 super_block = inst->fw_caps[SUPER_BLOCK].value; + + inst_hfi_gen2->src_subcr_params.super_block = super_block; + + return iris_hfi_gen2_session_set_property(inst, + HFI_PROP_AV1_SUPER_BLOCK_ENABLED, + HFI_HOST_FLAGS_NONE, + port, + HFI_PAYLOAD_U32_ENUM, + &super_block, + sizeof(u32)); +} + static int iris_hfi_gen2_session_set_config_params(struct iris_inst *inst, u32 plane) { const struct iris_platform_data *pdata = inst->core->iris_platform_data; @@ -527,10 +617,14 @@ static int iris_hfi_gen2_session_set_config_params(struct iris_inst *inst, u32 p {HFI_PROP_SIGNAL_COLOR_INFO, iris_hfi_gen2_set_colorspace }, {HFI_PROP_PROFILE, iris_hfi_gen2_set_profile }, {HFI_PROP_LEVEL, iris_hfi_gen2_set_level }, + {HFI_PROP_OPB_ENABLE, iris_hfi_gen2_set_opb_enable }, {HFI_PROP_COLOR_FORMAT, iris_hfi_gen2_set_colorformat }, {HFI_PROP_LINEAR_STRIDE_SCANLINE, iris_hfi_gen2_set_linear_stride_scanline }, {HFI_PROP_TIER, iris_hfi_gen2_set_tier }, {HFI_PROP_FRAME_RATE, iris_hfi_gen2_set_frame_rate }, + {HFI_PROP_AV1_FILM_GRAIN_PRESENT, iris_hfi_gen2_set_film_grain }, + {HFI_PROP_AV1_SUPER_BLOCK_ENABLED, iris_hfi_gen2_set_super_block }, + {HFI_PROP_OPB_ENABLE, iris_hfi_gen2_set_opb_enable }, }; if (inst->domain == DECODER) { @@ -544,6 +638,9 @@ static int iris_hfi_gen2_session_set_config_params(struct iris_inst *inst, u32 p } else if (inst->codec == V4L2_PIX_FMT_VP9) { config_params = pdata->dec_input_config_params_vp9; config_params_size = pdata->dec_input_config_params_vp9_size; + } else if (inst->codec == V4L2_PIX_FMT_AV1) { + config_params = pdata->dec_input_config_params_av1; + config_params_size = pdata->dec_input_config_params_av1_size; } else { return -EINVAL; } @@ -598,6 +695,9 @@ static int iris_hfi_gen2_session_set_codec(struct iris_inst *inst) break; case V4L2_PIX_FMT_VP9: codec = HFI_CODEC_DECODE_VP9; + break; + case V4L2_PIX_FMT_AV1: + codec = HFI_CODEC_DECODE_AV1; } iris_hfi_gen2_packet_session_property(inst, @@ -743,7 +843,7 @@ static int iris_hfi_gen2_subscribe_change_param(struct iris_inst *inst, u32 plan if ((V4L2_TYPE_IS_OUTPUT(plane) && inst_hfi_gen2->ipsc_properties_set) || (V4L2_TYPE_IS_CAPTURE(plane) && inst_hfi_gen2->opsc_properties_set)) { - dev_err(core->dev, "invalid plane\n"); + dev_dbg(core->dev, "%cPSC already set\n", V4L2_TYPE_IS_OUTPUT(plane) ? 'I' : 'O'); return 0; } @@ -763,6 +863,11 @@ static int iris_hfi_gen2_subscribe_change_param(struct iris_inst *inst, u32 plan change_param_size = core->iris_platform_data->dec_input_config_params_vp9_size; break; + case V4L2_PIX_FMT_AV1: + change_param = core->iris_platform_data->dec_input_config_params_av1; + change_param_size = + core->iris_platform_data->dec_input_config_params_av1_size; + break; } payload[0] = HFI_MODE_PORT_SETTINGS_CHANGE; @@ -845,6 +950,16 @@ static int iris_hfi_gen2_subscribe_change_param(struct iris_inst *inst, u32 plan payload_size = sizeof(u32); payload_type = HFI_PAYLOAD_U32; break; + case HFI_PROP_AV1_FILM_GRAIN_PRESENT: + payload[0] = subsc_params.film_grain; + payload_size = sizeof(u32); + payload_type = HFI_PAYLOAD_U32; + break; + case HFI_PROP_AV1_SUPER_BLOCK_ENABLED: + payload[0] = subsc_params.super_block; + payload_size = sizeof(u32); + payload_type = HFI_PAYLOAD_U32; + break; default: prop_type = 0; ret = -EINVAL; @@ -900,6 +1015,11 @@ static int iris_hfi_gen2_subscribe_property(struct iris_inst *inst, u32 plane) subscribe_prop_size = core->iris_platform_data->dec_output_prop_vp9_size; break; + case V4L2_PIX_FMT_AV1: + subcribe_prop = core->iris_platform_data->dec_output_prop_av1; + subscribe_prop_size = + core->iris_platform_data->dec_output_prop_av1_size; + break; } } @@ -946,6 +1066,9 @@ static int iris_hfi_gen2_session_stop(struct iris_inst *inst, u32 plane) struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); int ret = 0; + if (!inst_hfi_gen2->packet) + return -EINVAL; + reinit_completion(&inst->completion); iris_hfi_gen2_packet_session_command(inst, @@ -1075,6 +1198,8 @@ static u32 iris_hfi_gen2_buf_type_from_driver(u32 domain, enum iris_buffer_type return HFI_BUFFER_ARP; case BUF_VPSS: return HFI_BUFFER_VPSS; + case BUF_PARTIAL: + return HFI_BUFFER_PARTIAL_DATA; default: return 0; } @@ -1087,7 +1212,13 @@ static int iris_set_num_comv(struct iris_inst *inst) u32 num_comv; caps = core->iris_platform_data->inst_caps; - num_comv = caps->num_comv; + + /* + * AV1 needs more comv buffers than other codecs. + * Update accordingly. + */ + num_comv = (inst->codec == V4L2_PIX_FMT_AV1) ? + NUM_COMV_AV1 : caps->num_comv; return core->hfi_ops->session_set_property(inst, HFI_PROP_COMV_BUFFER_COUNT, @@ -1195,5 +1326,10 @@ void iris_hfi_gen2_command_ops_init(struct iris_core *core) struct iris_inst *iris_hfi_gen2_get_instance(void) { - return kzalloc(sizeof(struct iris_inst_hfi_gen2), GFP_KERNEL); + struct iris_inst_hfi_gen2 *out; + + /* The allocation is intentionally larger than struct iris_inst. */ + out = kzalloc(sizeof(*out), GFP_KERNEL); + + return &out->inst; } diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_defines.h b/drivers/media/platform/qcom/iris/iris_hfi_gen2_defines.h index aa1f795f5626c..cecf771c55dd3 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_defines.h +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_defines.h @@ -70,6 +70,7 @@ enum hfi_rate_control { #define HFI_PROP_QP_PACKED 0x0300012e #define HFI_PROP_MIN_QP_PACKED 0x0300012f #define HFI_PROP_MAX_QP_PACKED 0x03000130 +#define HFI_PROP_IR_RANDOM_PERIOD 0x03000131 #define HFI_PROP_TOTAL_BITRATE 0x0300013b #define HFI_PROP_MAX_GOP_FRAMES 0x03000146 #define HFI_PROP_MAX_B_FRAMES 0x03000147 @@ -83,15 +84,42 @@ enum hfi_seq_header_mode { }; #define HFI_PROP_SEQ_HEADER_MODE 0x03000149 + +enum hfi_rotation { + HFI_ROTATION_NONE = 0x00000000, + HFI_ROTATION_90 = 0x00000001, + HFI_ROTATION_180 = 0x00000002, + HFI_ROTATION_270 = 0x00000003, +}; + +#define HFI_PROP_ROTATION 0x0300014b + +enum hfi_flip { + HFI_DISABLE_FLIP = 0x00000000, + HFI_HORIZONTAL_FLIP = 0x00000001, + HFI_VERTICAL_FLIP = 0x00000002, +}; + +#define HFI_PROP_FLIP 0x0300014c #define HFI_PROP_SIGNAL_COLOR_INFO 0x03000155 #define HFI_PROP_PICTURE_TYPE 0x03000162 #define HFI_PROP_DEC_DEFAULT_HEADER 0x03000168 #define HFI_PROP_DEC_START_FROM_RAP_FRAME 0x03000169 #define HFI_PROP_NO_OUTPUT 0x0300016a #define HFI_PROP_BUFFER_MARK 0x0300016c +#define HFI_PROP_WORST_COMPRESSION_RATIO 0x03000174 +#define HFI_PROP_WORST_COMPLEXITY_FACTOR 0x03000175 #define HFI_PROP_RAW_RESOLUTION 0x03000178 #define HFI_PROP_TOTAL_PEAK_BITRATE 0x0300017C +#define HFI_PROP_AV1_FILM_GRAIN_PRESENT 0x03000180 +#define HFI_PROP_AV1_SUPER_BLOCK_ENABLED 0x03000181 +#define HFI_PROP_AV1_OP_POINT 0x03000182 +#define HFI_PROP_IR_CYCLIC_PERIOD 0x0300017E +#define HFI_PROP_OPB_ENABLE 0x03000184 +#define HFI_PROP_AV1_TILE_ROWS_COLUMNS 0x03000187 +#define HFI_PROP_AV1_DRAP_CONFIG 0x03000189 #define HFI_PROP_COMV_BUFFER_COUNT 0x03000193 +#define HFI_PROP_AV1_UNIFORM_TILE_SPACING 0x03000197 #define HFI_PROP_END 0x03FFFFFF #define HFI_SESSION_ERROR_BEGIN 0x04000000 @@ -138,6 +166,7 @@ enum hfi_codec_type { HFI_CODEC_DECODE_HEVC = 3, HFI_CODEC_ENCODE_HEVC = 4, HFI_CODEC_DECODE_VP9 = 5, + HFI_CODEC_DECODE_AV1 = 7, }; enum hfi_picture_type { diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c index 2f1f118eae4f6..8e19f61bbbf9e 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c @@ -54,6 +54,10 @@ static u32 iris_hfi_gen2_buf_type_to_driver(struct iris_inst *inst, return BUF_SCRATCH_2; case HFI_BUFFER_PERSIST: return BUF_PERSIST; + case HFI_BUFFER_PARTIAL_DATA: + return BUF_PARTIAL; + case HFI_BUFFER_VPSS: + return BUF_VPSS; default: return 0; } @@ -72,6 +76,7 @@ static bool iris_hfi_gen2_is_valid_hfi_buffer_type(u32 buffer_type) case HFI_BUFFER_DPB: case HFI_BUFFER_PERSIST: case HFI_BUFFER_VPSS: + case HFI_BUFFER_PARTIAL_DATA: return true; default: return false; @@ -596,6 +601,10 @@ static void iris_hfi_gen2_read_input_subcr_params(struct iris_inst *inst) inst->fw_caps[PROFILE_H264].value = subsc_params.profile; inst->fw_caps[LEVEL_H264].value = subsc_params.level; break; + case V4L2_PIX_FMT_AV1: + inst->fw_caps[PROFILE_AV1].value = subsc_params.profile; + inst->fw_caps[LEVEL_AV1].value = subsc_params.level; + break; } inst->fw_caps[POC].value = subsc_params.pic_order_cnt; @@ -608,6 +617,11 @@ static void iris_hfi_gen2_read_input_subcr_params(struct iris_inst *inst) iris_inst_change_state(inst, IRIS_INST_ERROR); } + if (inst->codec == V4L2_PIX_FMT_AV1) { + inst->fw_caps[FILM_GRAIN].value = subsc_params.film_grain; + inst->fw_caps[SUPER_BLOCK].value = subsc_params.super_block; + } + inst->fw_min_count = subsc_params.fw_min_count; inst->buffers[BUF_OUTPUT].min_count = iris_vpu_buf_count(inst, BUF_OUTPUT); inst->buffers[BUF_OUTPUT].size = pixmp_op->plane_fmt[0].sizeimage; @@ -711,6 +725,12 @@ static int iris_hfi_gen2_handle_session_property(struct iris_inst *inst, case HFI_PROP_NO_OUTPUT: inst_hfi_gen2->hfi_frame_info.no_output = 1; break; + case HFI_PROP_AV1_FILM_GRAIN_PRESENT: + inst_hfi_gen2->src_subcr_params.film_grain = pkt->payload[0]; + break; + case HFI_PROP_AV1_SUPER_BLOCK_ENABLED: + inst_hfi_gen2->src_subcr_params.super_block = pkt->payload[0]; + break; case HFI_PROP_QUALITY_MODE: case HFI_PROP_STAGE: case HFI_PROP_PIPE: @@ -841,6 +861,10 @@ static void iris_hfi_gen2_init_src_change_param(struct iris_inst *inst) subsc_params->profile = inst->fw_caps[PROFILE_H264].value; subsc_params->level = inst->fw_caps[LEVEL_H264].value; break; + case V4L2_PIX_FMT_AV1: + subsc_params->profile = inst->fw_caps[PROFILE_AV1].value; + subsc_params->level = inst->fw_caps[LEVEL_AV1].value; + break; } subsc_params->pic_order_cnt = inst->fw_caps[POC].value; diff --git a/drivers/media/platform/qcom/iris/iris_hfi_queue.c b/drivers/media/platform/qcom/iris/iris_hfi_queue.c index b3ed06297953b..f465ff00a9ba3 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_queue.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_queue.c @@ -245,25 +245,26 @@ static void iris_hfi_queue_deinit(struct iris_iface_q_info *iface_q) int iris_hfi_queues_init(struct iris_core *core) { + struct device *dev = core->dev_np ? core->dev_np : core->dev; struct iris_hfi_queue_table_header *q_tbl_hdr; u32 queue_size; /* Iris hardware requires 4K queue alignment */ queue_size = ALIGN((sizeof(*q_tbl_hdr) + (IFACEQ_QUEUE_SIZE * IFACEQ_NUMQ)), SZ_4K); - core->iface_q_table_vaddr = dma_alloc_attrs(core->dev, queue_size, + core->iface_q_table_vaddr = dma_alloc_attrs(dev, queue_size, &core->iface_q_table_daddr, GFP_KERNEL, DMA_ATTR_WRITE_COMBINE); if (!core->iface_q_table_vaddr) { - dev_err(core->dev, "queues alloc and map failed\n"); + dev_err(dev, "queues alloc and map failed\n"); return -ENOMEM; } - core->sfr_vaddr = dma_alloc_attrs(core->dev, SFR_SIZE, + core->sfr_vaddr = dma_alloc_attrs(dev, SFR_SIZE, &core->sfr_daddr, GFP_KERNEL, DMA_ATTR_WRITE_COMBINE); if (!core->sfr_vaddr) { - dev_err(core->dev, "sfr alloc and map failed\n"); - dma_free_attrs(core->dev, sizeof(*q_tbl_hdr), core->iface_q_table_vaddr, + dev_err(dev, "sfr alloc and map failed\n"); + dma_free_attrs(dev, sizeof(*q_tbl_hdr), core->iface_q_table_vaddr, core->iface_q_table_daddr, DMA_ATTR_WRITE_COMBINE); return -ENOMEM; } @@ -291,6 +292,7 @@ int iris_hfi_queues_init(struct iris_core *core) void iris_hfi_queues_deinit(struct iris_core *core) { + struct device *dev = core->dev_np ? core->dev_np : core->dev; u32 queue_size; if (!core->iface_q_table_vaddr) @@ -300,7 +302,7 @@ void iris_hfi_queues_deinit(struct iris_core *core) iris_hfi_queue_deinit(&core->message_queue); iris_hfi_queue_deinit(&core->command_queue); - dma_free_attrs(core->dev, SFR_SIZE, core->sfr_vaddr, + dma_free_attrs(dev, SFR_SIZE, core->sfr_vaddr, core->sfr_daddr, DMA_ATTR_WRITE_COMBINE); core->sfr_vaddr = NULL; @@ -309,7 +311,7 @@ void iris_hfi_queues_deinit(struct iris_core *core) queue_size = ALIGN(sizeof(struct iris_hfi_queue_table_header) + (IFACEQ_QUEUE_SIZE * IFACEQ_NUMQ), SZ_4K); - dma_free_attrs(core->dev, queue_size, core->iface_q_table_vaddr, + dma_free_attrs(dev, queue_size, core->iface_q_table_vaddr, core->iface_q_table_daddr, DMA_ATTR_WRITE_COMBINE); core->iface_q_table_vaddr = NULL; diff --git a/drivers/media/platform/qcom/iris/iris_instance.h b/drivers/media/platform/qcom/iris/iris_instance.h index 5982d7adefeab..63dd889c9992a 100644 --- a/drivers/media/platform/qcom/iris/iris_instance.h +++ b/drivers/media/platform/qcom/iris/iris_instance.h @@ -15,10 +15,16 @@ #define DEFAULT_WIDTH 320 #define DEFAULT_HEIGHT 240 -enum iris_fmt_type { +enum iris_fmt_type_out { IRIS_FMT_H264, IRIS_FMT_HEVC, IRIS_FMT_VP9, + IRIS_FMT_AV1, +}; + +enum iris_fmt_type_cap { + IRIS_FMT_NV12, + IRIS_FMT_QC08C, }; struct iris_fmt { @@ -61,9 +67,15 @@ struct iris_fmt { * @metadata_idx: index for metadata buffer * @codec: codec type * @last_buffer_dequeued: a flag to indicate that last buffer is sent by driver + * @last_buf_ns: start time of received input buffer for current one second FPS window + * @frame_counter: input buffer counter for current one second FPS window * @frame_rate: frame rate of current instance * @operating_rate: operating rate of current instance * @hfi_rc_type: rate control type + * @enc_raw_width: source image width for encoder instance + * @enc_raw_height: source image height for encoder instance + * @enc_scale_width: scale width for encoder instance + * @enc_scale_height: scale height for encoder instance */ struct iris_inst { @@ -99,9 +111,15 @@ struct iris_inst { u32 metadata_idx; u32 codec; bool last_buffer_dequeued; + u64 last_buf_ns; + u32 frame_counter; u32 frame_rate; u32 operating_rate; u32 hfi_rc_type; + u32 enc_raw_width; + u32 enc_raw_height; + u32 enc_scale_width; + u32 enc_scale_height; }; #endif diff --git a/drivers/media/platform/qcom/iris/iris_platform_common.h b/drivers/media/platform/qcom/iris/iris_platform_common.h index 58d05e0a112ee..57eb702ab4973 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_common.h +++ b/drivers/media/platform/qcom/iris/iris_platform_common.h @@ -41,19 +41,27 @@ enum pipe_type { PIPE_4 = 4, }; -extern struct iris_platform_data qcs8300_data; -extern struct iris_platform_data sm8250_data; -extern struct iris_platform_data sm8550_data; -extern struct iris_platform_data sm8650_data; -extern struct iris_platform_data sm8750_data; +extern const struct iris_platform_data qcs8300_data; +extern const struct iris_platform_data sc7280_data; +extern const struct iris_platform_data sm8250_data; +extern const struct iris_platform_data sm8550_data; +extern const struct iris_platform_data sm8650_data; +extern const struct iris_platform_data sm8750_data; +extern const struct iris_platform_data x1p42100_data; enum platform_clk_type { IRIS_AXI_CLK, /* AXI0 in case of platforms with multiple AXI clocks */ IRIS_CTRL_CLK, + IRIS_AHB_CLK, IRIS_HW_CLK, + IRIS_HW_AHB_CLK, IRIS_AXI1_CLK, IRIS_CTRL_FREERUN_CLK, IRIS_HW_FREERUN_CLK, + IRIS_BSE_HW_CLK, + IRIS_VPP0_HW_CLK, + IRIS_VPP1_HW_CLK, + IRIS_APV_HW_CLK, }; struct platform_clk_data { @@ -100,6 +108,13 @@ enum platform_inst_fw_cap_type { LEVEL_H264, LEVEL_HEVC, LEVEL_VP9, + PROFILE_AV1, + LEVEL_AV1, + TIER_AV1, + DRAP, + FILM_GRAIN, + SUPER_BLOCK, + ENH_LAYER_COUNT, INPUT_BUF_HOST_MAX_COUNT, OUTPUT_BUF_HOST_MAX_COUNT, STAGE, @@ -140,6 +155,11 @@ enum platform_inst_fw_cap_type { P_FRAME_QP_HEVC, B_FRAME_QP_H264, B_FRAME_QP_HEVC, + ROTATION, + HFLIP, + VFLIP, + IR_TYPE, + IR_PERIOD, INST_FW_CAP_MAX, }; @@ -188,6 +208,9 @@ struct icc_vote_data { enum platform_pm_domain_type { IRIS_CTRL_POWER_DOMAIN, IRIS_HW_POWER_DOMAIN, + IRIS_VPP0_HW_POWER_DOMAIN, + IRIS_VPP1_HW_POWER_DOMAIN, + IRIS_APV_HW_POWER_DOMAIN, }; struct iris_platform_data { @@ -197,6 +220,8 @@ struct iris_platform_data { u32 (*get_vpu_buffer_size)(struct iris_inst *inst, enum iris_buffer_type buffer_type); const struct vpu_ops *vpu_ops; void (*set_preset_registers)(struct iris_core *core); + int (*init_cb_devs)(struct iris_core *core); + void (*deinit_cb_devs)(struct iris_core *core); const struct icc_info *icc_tbl; unsigned int icc_tbl_size; const struct bw_info *bw_tbl_dec; @@ -206,6 +231,7 @@ struct iris_platform_data { const char * const *opp_pd_tbl; unsigned int opp_pd_tbl_size; const struct platform_clk_data *clk_tbl; + const char * const *opp_clk_tbl; unsigned int clk_tbl_size; const char * const *clk_rst_tbl; unsigned int clk_rst_tbl_size; @@ -214,16 +240,20 @@ struct iris_platform_data { u64 dma_mask; const char *fwname; u32 pas_id; + struct iris_fmt *inst_iris_fmts; + u32 inst_iris_fmts_size; struct platform_inst_caps *inst_caps; - struct platform_inst_fw_cap *inst_fw_caps_dec; + const struct platform_inst_fw_cap *inst_fw_caps_dec; u32 inst_fw_caps_dec_size; - struct platform_inst_fw_cap *inst_fw_caps_enc; + const struct platform_inst_fw_cap *inst_fw_caps_enc; u32 inst_fw_caps_enc_size; - struct tz_cp_config *tz_cp_config_data; + const struct tz_cp_config *tz_cp_config_data; + u32 tz_cp_config_data_size; u32 core_arch; u32 hw_response_timeout; struct ubwc_config_data *ubwc_config; u32 num_vpp_pipe; + bool no_aon; u32 max_session_count; /* max number of macroblocks per frame supported */ u32 max_core_mbpf; @@ -235,6 +265,8 @@ struct iris_platform_data { unsigned int dec_input_config_params_hevc_size; const u32 *dec_input_config_params_vp9; unsigned int dec_input_config_params_vp9_size; + const u32 *dec_input_config_params_av1; + unsigned int dec_input_config_params_av1_size; const u32 *dec_output_config_params; unsigned int dec_output_config_params_size; const u32 *enc_input_config_params; @@ -249,6 +281,8 @@ struct iris_platform_data { unsigned int dec_output_prop_hevc_size; const u32 *dec_output_prop_vp9; unsigned int dec_output_prop_vp9_size; + const u32 *dec_output_prop_av1; + unsigned int dec_output_prop_av1_size; const u32 *dec_ip_int_buf_tbl; unsigned int dec_ip_int_buf_tbl_size; const u32 *dec_op_int_buf_tbl; diff --git a/drivers/media/platform/qcom/iris/iris_platform_sm8250.c b/drivers/media/platform/qcom/iris/iris_platform_gen1.c similarity index 76% rename from drivers/media/platform/qcom/iris/iris_platform_sm8250.c rename to drivers/media/platform/qcom/iris/iris_platform_gen1.c index 16486284f8acc..df8e6bf9430ed 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_sm8250.c +++ b/drivers/media/platform/qcom/iris/iris_platform_gen1.c @@ -11,19 +11,35 @@ #include "iris_hfi_gen1_defines.h" #include "iris_vpu_buffer.h" #include "iris_vpu_common.h" +#include "iris_instance.h" + +#include "iris_platform_sc7280.h" #define BITRATE_MIN 32000 #define BITRATE_MAX 160000000 #define BITRATE_PEAK_DEFAULT (BITRATE_DEFAULT * 2) #define BITRATE_STEP 100 +static struct iris_fmt platform_fmts_sm8250_dec[] = { + [IRIS_FMT_H264] = { + .pixfmt = V4L2_PIX_FMT_H264, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, + [IRIS_FMT_HEVC] = { + .pixfmt = V4L2_PIX_FMT_HEVC, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, + [IRIS_FMT_VP9] = { + .pixfmt = V4L2_PIX_FMT_VP9, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, +}; + static struct platform_inst_fw_cap inst_fw_cap_sm8250_dec[] = { { .cap_id = PIPE, - .min = PIPE_1, - .max = PIPE_4, + /* .max, .min and .value are set via platform data */ .step_or_mask = 1, - .value = PIPE_4, .hfi_id = HFI_PROPERTY_PARAM_WORK_ROUTE, .set = iris_set_pipe, }, @@ -38,7 +54,7 @@ static struct platform_inst_fw_cap inst_fw_cap_sm8250_dec[] = { }, }; -static struct platform_inst_fw_cap inst_fw_cap_sm8250_enc[] = { +static const struct platform_inst_fw_cap inst_fw_cap_sm8250_enc[] = { { .cap_id = STAGE, .min = STAGE_1, @@ -273,11 +289,18 @@ static const struct platform_clk_data sm8250_clk_table[] = { {IRIS_HW_CLK, "vcodec0_core" }, }; -static struct tz_cp_config tz_cp_config_sm8250 = { - .cp_start = 0, - .cp_size = 0x25800000, - .cp_nonpixel_start = 0x01000000, - .cp_nonpixel_size = 0x24800000, +static const char * const sm8250_opp_clk_table[] = { + "vcodec0_core", + NULL, +}; + +static const struct tz_cp_config tz_cp_config_sm8250[] = { + { + .cp_start = 0, + .cp_size = 0x25800000, + .cp_nonpixel_start = 0x01000000, + .cp_nonpixel_size = 0x24800000, + }, }; static const u32 sm8250_vdec_input_config_param_default[] = { @@ -314,7 +337,7 @@ static const u32 sm8250_enc_ip_int_buf_tbl[] = { BUF_SCRATCH_2, }; -struct iris_platform_data sm8250_data = { +const struct iris_platform_data sm8250_data = { .get_instance = iris_hfi_gen1_get_instance, .init_hfi_command_ops = &iris_hfi_gen1_command_ops_init, .init_hfi_response_ops = iris_hfi_gen1_response_ops_init, @@ -333,16 +356,20 @@ struct iris_platform_data sm8250_data = { .opp_pd_tbl_size = ARRAY_SIZE(sm8250_opp_pd_table), .clk_tbl = sm8250_clk_table, .clk_tbl_size = ARRAY_SIZE(sm8250_clk_table), + .opp_clk_tbl = sm8250_opp_clk_table, /* Upper bound of DMA address range */ .dma_mask = 0xe0000000 - 1, .fwname = "qcom/vpu-1.0/venus.mbn", .pas_id = IRIS_PAS_ID, + .inst_iris_fmts = platform_fmts_sm8250_dec, + .inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8250_dec), .inst_caps = &platform_inst_cap_sm8250, .inst_fw_caps_dec = inst_fw_cap_sm8250_dec, .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8250_dec), .inst_fw_caps_enc = inst_fw_cap_sm8250_enc, .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8250_enc), - .tz_cp_config_data = &tz_cp_config_sm8250, + .tz_cp_config_data = tz_cp_config_sm8250, + .tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8250), .hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE, .num_vpp_pipe = 4, .max_session_count = 16, @@ -364,3 +391,58 @@ struct iris_platform_data sm8250_data = { .enc_ip_int_buf_tbl = sm8250_enc_ip_int_buf_tbl, .enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_enc_ip_int_buf_tbl), }; + +const struct iris_platform_data sc7280_data = { + .get_instance = iris_hfi_gen1_get_instance, + .init_hfi_command_ops = &iris_hfi_gen1_command_ops_init, + .init_hfi_response_ops = iris_hfi_gen1_response_ops_init, + .get_vpu_buffer_size = iris_vpu_buf_size, + .vpu_ops = &iris_vpu2_ops, + .set_preset_registers = iris_set_sm8250_preset_registers, + .icc_tbl = sm8250_icc_table, + .icc_tbl_size = ARRAY_SIZE(sm8250_icc_table), + .bw_tbl_dec = sc7280_bw_table_dec, + .bw_tbl_dec_size = ARRAY_SIZE(sc7280_bw_table_dec), + .pmdomain_tbl = sm8250_pmdomain_table, + .pmdomain_tbl_size = ARRAY_SIZE(sm8250_pmdomain_table), + .opp_pd_tbl = sc7280_opp_pd_table, + .opp_pd_tbl_size = ARRAY_SIZE(sc7280_opp_pd_table), + .clk_tbl = sc7280_clk_table, + .clk_tbl_size = ARRAY_SIZE(sc7280_clk_table), + .opp_clk_tbl = sc7280_opp_clk_table, + /* Upper bound of DMA address range */ + .dma_mask = 0xe0000000 - 1, + .fwname = "qcom/vpu/vpu20_p1.mbn", + .pas_id = IRIS_PAS_ID, + .inst_iris_fmts = platform_fmts_sm8250_dec, + .inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8250_dec), + .inst_caps = &platform_inst_cap_sm8250, + .inst_fw_caps_dec = inst_fw_cap_sm8250_dec, + .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8250_dec), + .inst_fw_caps_enc = inst_fw_cap_sm8250_enc, + .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8250_enc), + .tz_cp_config_data = tz_cp_config_sm8250, + .tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8250), + .hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE, + .num_vpp_pipe = 1, + .no_aon = true, + .max_session_count = 16, + .max_core_mbpf = 4096 * 2176 / 256 * 2 + 1920 * 1088 / 256, + /* max spec for SC7280 is 4096x2176@60fps */ + .max_core_mbps = 4096 * 2176 / 256 * 60, + .dec_input_config_params_default = + sm8250_vdec_input_config_param_default, + .dec_input_config_params_default_size = + ARRAY_SIZE(sm8250_vdec_input_config_param_default), + .enc_input_config_params = sm8250_venc_input_config_param, + .enc_input_config_params_size = + ARRAY_SIZE(sm8250_venc_input_config_param), + + .dec_ip_int_buf_tbl = sm8250_dec_ip_int_buf_tbl, + .dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_ip_int_buf_tbl), + .dec_op_int_buf_tbl = sm8250_dec_op_int_buf_tbl, + .dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_op_int_buf_tbl), + + .enc_ip_int_buf_tbl = sm8250_enc_ip_int_buf_tbl, + .enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_enc_ip_int_buf_tbl), +}; diff --git a/drivers/media/platform/qcom/iris/iris_platform_gen2.c b/drivers/media/platform/qcom/iris/iris_platform_gen2.c index 36d69cc73986b..d42ca57824666 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_gen2.c +++ b/drivers/media/platform/qcom/iris/iris_platform_gen2.c @@ -15,11 +15,31 @@ #include "iris_platform_qcs8300.h" #include "iris_platform_sm8650.h" #include "iris_platform_sm8750.h" +#include "iris_platform_x1p42100.h" #define VIDEO_ARCH_LX 1 #define BITRATE_MAX 245000000 -static struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = { +static struct iris_fmt platform_fmts_sm8550_dec[] = { + [IRIS_FMT_H264] = { + .pixfmt = V4L2_PIX_FMT_H264, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, + [IRIS_FMT_HEVC] = { + .pixfmt = V4L2_PIX_FMT_HEVC, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, + [IRIS_FMT_VP9] = { + .pixfmt = V4L2_PIX_FMT_VP9, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, + [IRIS_FMT_AV1] = { + .pixfmt = V4L2_PIX_FMT_AV1, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, +}; + +static const struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = { { .cap_id = PROFILE_H264, .min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE, @@ -45,6 +65,16 @@ static struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = { .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, .set = iris_set_u32_enum, }, + { + .cap_id = PROFILE_AV1, + .min = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN, + .max = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN, + .step_or_mask = BIT(V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN), + .value = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN, + .hfi_id = HFI_PROP_PROFILE, + .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, + .set = iris_set_u32_enum, + }, { .cap_id = PROFILE_VP9, .min = V4L2_MPEG_VIDEO_VP9_PROFILE_0, @@ -128,6 +158,33 @@ static struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = { .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, .set = iris_set_u32_enum, }, + { + .cap_id = LEVEL_AV1, + .min = V4L2_MPEG_VIDEO_AV1_LEVEL_2_0, + .max = V4L2_MPEG_VIDEO_AV1_LEVEL_6_1, + .step_or_mask = BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_0) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_1) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_2) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_3) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_0) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_1) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_2) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_3) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_0) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_1) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_2) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_3) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_0) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_1) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_2) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_3) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_6_0) | + BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_6_1), + .value = V4L2_MPEG_VIDEO_AV1_LEVEL_6_1, + .hfi_id = HFI_PROP_LEVEL, + .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, + .set = iris_set_u32_enum, + }, { .cap_id = TIER, .min = V4L2_MPEG_VIDEO_HEVC_TIER_MAIN, @@ -139,6 +196,53 @@ static struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = { .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, .set = iris_set_u32_enum, }, + { + .cap_id = TIER_AV1, + .min = 0, + .max = 1, + .step_or_mask = 1, + .value = 0, + .hfi_id = HFI_PROP_TIER, + .flags = CAP_FLAG_INPUT_PORT, + .set = iris_set_u32, + }, + { + .cap_id = DRAP, + .min = 0, + .max = 1, + .step_or_mask = 1, + .value = 0, + .hfi_id = HFI_PROP_AV1_DRAP_CONFIG, + .flags = CAP_FLAG_INPUT_PORT, + .set = iris_set_u32, + }, + { + .cap_id = FILM_GRAIN, + .min = 0, + .max = 1, + .step_or_mask = 1, + .value = 0, + .hfi_id = HFI_PROP_AV1_FILM_GRAIN_PRESENT, + .flags = CAP_FLAG_VOLATILE, + }, + { + .cap_id = SUPER_BLOCK, + .min = 0, + .max = 1, + .step_or_mask = 1, + .value = 0, + .hfi_id = HFI_PROP_AV1_SUPER_BLOCK_ENABLED, + }, + { + .cap_id = ENH_LAYER_COUNT, + .min = 0, + .max = 1, + .step_or_mask = 1, + .value = 0, + .hfi_id = HFI_PROP_AV1_OP_POINT, + .flags = CAP_FLAG_INPUT_PORT, + .set = iris_set_u32, + }, { .cap_id = INPUT_BUF_HOST_MAX_COUNT, .min = DEFAULT_MAX_HOST_BUF_COUNT, @@ -160,10 +264,8 @@ static struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = { }, { .cap_id = PIPE, - .min = PIPE_1, - .max = PIPE_4, + /* .max, .min and .value are set via platform data */ .step_or_mask = 1, - .value = PIPE_4, .hfi_id = HFI_PROP_PIPE, .set = iris_set_pipe, }, @@ -203,7 +305,7 @@ static struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = { }, }; -static struct platform_inst_fw_cap inst_fw_cap_sm8550_enc[] = { +static const struct platform_inst_fw_cap inst_fw_cap_sm8550_enc[] = { { .cap_id = PROFILE_H264, .min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE, @@ -588,6 +690,57 @@ static struct platform_inst_fw_cap inst_fw_cap_sm8550_enc[] = { .flags = CAP_FLAG_OUTPUT_PORT, .set = iris_set_u32, }, + { + .cap_id = ROTATION, + .min = 0, + .max = 270, + .step_or_mask = 90, + .value = 0, + .hfi_id = HFI_PROP_ROTATION, + .flags = CAP_FLAG_OUTPUT_PORT, + .set = iris_set_rotation, + }, + { + .cap_id = HFLIP, + .min = 0, + .max = 1, + .step_or_mask = 1, + .value = 0, + .hfi_id = HFI_PROP_FLIP, + .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | + CAP_FLAG_DYNAMIC_ALLOWED, + .set = iris_set_flip, + }, + { + .cap_id = VFLIP, + .min = 0, + .max = 1, + .step_or_mask = 1, + .value = 0, + .hfi_id = HFI_PROP_FLIP, + .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | + CAP_FLAG_DYNAMIC_ALLOWED, + .set = iris_set_flip, + }, + { + .cap_id = IR_TYPE, + .min = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM, + .max = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC, + .step_or_mask = BIT(V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM) | + BIT(V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC), + .value = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM, + .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, + }, + { + .cap_id = IR_PERIOD, + .min = 0, + .max = INT_MAX, + .step_or_mask = 1, + .value = 0, + .flags = CAP_FLAG_OUTPUT_PORT | + CAP_FLAG_DYNAMIC_ALLOWED, + .set = iris_set_ir_period, + }, }; static struct platform_inst_caps platform_inst_cap_sm8550 = { @@ -609,6 +762,47 @@ static void iris_set_sm8550_preset_registers(struct iris_core *core) writel(0x0, core->reg_base + 0xB0088); } +static int sm8550_init_cb_devs(struct iris_core *core) +{ + const u32 f_id_np = 0; /* IRIS_NON_PIXEL_VCODEC */ + const u32 f_id_p = 1; /* IRIS_PIXEL */ + struct device *dev; + + dev = iris_create_cb_dev(core, "iris_non_pixel", &f_id_np); + if (IS_ERR(dev)) + return PTR_ERR(dev); + + core->dev_np = dev; + core->dev_bs = core->dev_np; + + dev = iris_create_cb_dev(core, "iris_pixel", &f_id_p); + if (IS_ERR(dev)) + goto err_unreg_dev_np; + + core->dev_p = dev; + + return 0; + +err_unreg_dev_np: + platform_device_unregister(to_platform_device(core->dev_np)); + core->dev_np = NULL; + core->dev_bs = NULL; + + return PTR_ERR(dev); +} + +static void sm8550_deinit_cb_devs(struct iris_core *core) +{ + if (core->dev_np) + platform_device_unregister(to_platform_device(core->dev_np)); + if (core->dev_p) + platform_device_unregister(to_platform_device(core->dev_p)); + + core->dev_np = NULL; + core->dev_bs = NULL; + core->dev_p = NULL; +} + static const struct icc_info sm8550_icc_table[] = { { "cpu-cfg", 1000, 1000 }, { "video-mem", 1000, 15000000 }, @@ -633,6 +827,11 @@ static const struct platform_clk_data sm8550_clk_table[] = { {IRIS_HW_CLK, "vcodec0_core" }, }; +static const char * const sm8550_opp_clk_table[] = { + "vcodec0_core", + NULL, +}; + static struct ubwc_config_data ubwc_config_sm8550 = { .max_channels = 8, .mal_length = 32, @@ -643,11 +842,13 @@ static struct ubwc_config_data ubwc_config_sm8550 = { .bank_spreading = 1, }; -static struct tz_cp_config tz_cp_config_sm8550 = { - .cp_start = 0, - .cp_size = 0x25800000, - .cp_nonpixel_start = 0x01000000, - .cp_nonpixel_size = 0x24800000, +static const struct tz_cp_config tz_cp_config_sm8550[] = { + { + .cp_start = 0, + .cp_size = 0x25800000, + .cp_nonpixel_start = 0x01000000, + .cp_nonpixel_size = 0x24800000, + }, }; static const u32 sm8550_vdec_input_config_params_default[] = { @@ -682,6 +883,19 @@ static const u32 sm8550_vdec_input_config_param_vp9[] = { HFI_PROP_LEVEL, }; +static const u32 sm8550_vdec_input_config_param_av1[] = { + HFI_PROP_BITSTREAM_RESOLUTION, + HFI_PROP_CROP_OFFSETS, + HFI_PROP_LUMA_CHROMA_BIT_DEPTH, + HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT, + HFI_PROP_PROFILE, + HFI_PROP_LEVEL, + HFI_PROP_TIER, + HFI_PROP_AV1_FILM_GRAIN_PRESENT, + HFI_PROP_AV1_SUPER_BLOCK_ENABLED, + HFI_PROP_SIGNAL_COLOR_INFO, +}; + static const u32 sm8550_venc_input_config_params[] = { HFI_PROP_COLOR_FORMAT, HFI_PROP_RAW_RESOLUTION, @@ -691,6 +905,7 @@ static const u32 sm8550_venc_input_config_params[] = { }; static const u32 sm8550_vdec_output_config_params[] = { + HFI_PROP_OPB_ENABLE, HFI_PROP_COLOR_FORMAT, HFI_PROP_LINEAR_STRIDE_SCANLINE, }; @@ -718,17 +933,28 @@ static const u32 sm8550_vdec_subscribe_output_properties_vp9[] = { HFI_PROP_PICTURE_TYPE, }; +static const u32 sm8550_vdec_subscribe_output_properties_av1[] = { + HFI_PROP_PICTURE_TYPE, + HFI_PROP_WORST_COMPRESSION_RATIO, + HFI_PROP_WORST_COMPLEXITY_FACTOR, +}; + static const u32 sm8550_dec_ip_int_buf_tbl[] = { BUF_BIN, BUF_COMV, BUF_NON_COMV, BUF_LINE, + BUF_PARTIAL, }; static const u32 sm8550_dec_op_int_buf_tbl[] = { BUF_DPB, }; +static const u32 sm8550_enc_ip_int_buf_tbl[] = { + BUF_VPSS, +}; + static const u32 sm8550_enc_op_int_buf_tbl[] = { BUF_BIN, BUF_COMV, @@ -737,13 +963,15 @@ static const u32 sm8550_enc_op_int_buf_tbl[] = { BUF_SCRATCH_2, }; -struct iris_platform_data sm8550_data = { +const struct iris_platform_data sm8550_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, .get_vpu_buffer_size = iris_vpu_buf_size, .vpu_ops = &iris_vpu3_ops, .set_preset_registers = iris_set_sm8550_preset_registers, + .init_cb_devs = sm8550_init_cb_devs, + .deinit_cb_devs = sm8550_deinit_cb_devs, .icc_tbl = sm8550_icc_table, .icc_tbl_size = ARRAY_SIZE(sm8550_icc_table), .clk_rst_tbl = sm8550_clk_reset_table, @@ -756,16 +984,20 @@ struct iris_platform_data sm8550_data = { .opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table), .clk_tbl = sm8550_clk_table, .clk_tbl_size = ARRAY_SIZE(sm8550_clk_table), + .opp_clk_tbl = sm8550_opp_clk_table, /* Upper bound of DMA address range */ .dma_mask = 0xe0000000 - 1, .fwname = "qcom/vpu/vpu30_p4.mbn", .pas_id = IRIS_PAS_ID, + .inst_iris_fmts = platform_fmts_sm8550_dec, + .inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec), .inst_caps = &platform_inst_cap_sm8550, .inst_fw_caps_dec = inst_fw_cap_sm8550_dec, .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec), .inst_fw_caps_enc = inst_fw_cap_sm8550_enc, .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc), - .tz_cp_config_data = &tz_cp_config_sm8550, + .tz_cp_config_data = tz_cp_config_sm8550, + .tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550), .core_arch = VIDEO_ARCH_LX, .hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE, .ubwc_config = &ubwc_config_sm8550, @@ -785,6 +1017,10 @@ struct iris_platform_data sm8550_data = { sm8550_vdec_input_config_param_vp9, .dec_input_config_params_vp9_size = ARRAY_SIZE(sm8550_vdec_input_config_param_vp9), + .dec_input_config_params_av1 = + sm8550_vdec_input_config_param_av1, + .dec_input_config_params_av1_size = + ARRAY_SIZE(sm8550_vdec_input_config_param_av1), .dec_output_config_params = sm8550_vdec_output_config_params, .dec_output_config_params_size = @@ -810,12 +1046,17 @@ struct iris_platform_data sm8550_data = { .dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9, .dec_output_prop_vp9_size = ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9), + .dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1, + .dec_output_prop_av1_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1), .dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl, .dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl), .dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl, .dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl), + .enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl, + .enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl), .enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl, .enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl), }; @@ -827,7 +1068,7 @@ struct iris_platform_data sm8550_data = { * - controller_rst_tbl to sm8650_controller_reset_table * - fwname to "qcom/vpu/vpu33_p4.mbn" */ -struct iris_platform_data sm8650_data = { +const struct iris_platform_data sm8650_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, @@ -848,16 +1089,20 @@ struct iris_platform_data sm8650_data = { .opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table), .clk_tbl = sm8550_clk_table, .clk_tbl_size = ARRAY_SIZE(sm8550_clk_table), + .opp_clk_tbl = sm8550_opp_clk_table, /* Upper bound of DMA address range */ .dma_mask = 0xe0000000 - 1, .fwname = "qcom/vpu/vpu33_p4.mbn", .pas_id = IRIS_PAS_ID, + .inst_iris_fmts = platform_fmts_sm8550_dec, + .inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec), .inst_caps = &platform_inst_cap_sm8550, .inst_fw_caps_dec = inst_fw_cap_sm8550_dec, .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec), .inst_fw_caps_enc = inst_fw_cap_sm8550_enc, .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc), - .tz_cp_config_data = &tz_cp_config_sm8550, + .tz_cp_config_data = tz_cp_config_sm8550, + .tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550), .core_arch = VIDEO_ARCH_LX, .hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE, .ubwc_config = &ubwc_config_sm8550, @@ -877,6 +1122,10 @@ struct iris_platform_data sm8650_data = { sm8550_vdec_input_config_param_vp9, .dec_input_config_params_vp9_size = ARRAY_SIZE(sm8550_vdec_input_config_param_vp9), + .dec_input_config_params_av1 = + sm8550_vdec_input_config_param_av1, + .dec_input_config_params_av1_size = + ARRAY_SIZE(sm8550_vdec_input_config_param_av1), .dec_output_config_params = sm8550_vdec_output_config_params, .dec_output_config_params_size = @@ -902,20 +1151,26 @@ struct iris_platform_data sm8650_data = { .dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9, .dec_output_prop_vp9_size = ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9), + .dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1, + .dec_output_prop_av1_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1), .dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl, .dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl), .dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl, .dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl), + .enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl, + .enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl), .enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl, .enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl), }; -struct iris_platform_data sm8750_data = { +const struct iris_platform_data sm8750_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, + .get_vpu_buffer_size = iris_vpu33_buf_size, .vpu_ops = &iris_vpu35_ops, .set_preset_registers = iris_set_sm8550_preset_registers, .icc_tbl = sm8550_icc_table, @@ -930,22 +1185,27 @@ struct iris_platform_data sm8750_data = { .opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table), .clk_tbl = sm8750_clk_table, .clk_tbl_size = ARRAY_SIZE(sm8750_clk_table), + .opp_clk_tbl = sm8550_opp_clk_table, /* Upper bound of DMA address range */ .dma_mask = 0xe0000000 - 1, .fwname = "qcom/vpu/vpu35_p4.mbn", .pas_id = IRIS_PAS_ID, + .inst_iris_fmts = platform_fmts_sm8550_dec, + .inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec), .inst_caps = &platform_inst_cap_sm8550, .inst_fw_caps_dec = inst_fw_cap_sm8550_dec, .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec), .inst_fw_caps_enc = inst_fw_cap_sm8550_enc, .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc), - .tz_cp_config_data = &tz_cp_config_sm8550, + .tz_cp_config_data = tz_cp_config_sm8550, + .tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550), .core_arch = VIDEO_ARCH_LX, .hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE, .ubwc_config = &ubwc_config_sm8550, .num_vpp_pipe = 4, .max_session_count = 16, .max_core_mbpf = NUM_MBS_8K * 2, + .max_core_mbps = ((7680 * 4320) / 256) * 60, .dec_input_config_params_default = sm8550_vdec_input_config_params_default, .dec_input_config_params_default_size = @@ -958,6 +1218,10 @@ struct iris_platform_data sm8750_data = { sm8550_vdec_input_config_param_vp9, .dec_input_config_params_vp9_size = ARRAY_SIZE(sm8550_vdec_input_config_param_vp9), + .dec_input_config_params_av1 = + sm8550_vdec_input_config_param_av1, + .dec_input_config_params_av1_size = + ARRAY_SIZE(sm8550_vdec_input_config_param_av1), .dec_output_config_params = sm8550_vdec_output_config_params, .dec_output_config_params_size = @@ -983,12 +1247,17 @@ struct iris_platform_data sm8750_data = { .dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9, .dec_output_prop_vp9_size = ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9), + .dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1, + .dec_output_prop_av1_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1), .dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl, .dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl), .dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl, .dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl), + .enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl, + .enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl), .enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl, .enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl), }; @@ -996,9 +1265,8 @@ struct iris_platform_data sm8750_data = { /* * Shares most of SM8550 data except: * - inst_caps to platform_inst_cap_qcs8300 - * - inst_fw_caps to inst_fw_cap_qcs8300 */ -struct iris_platform_data qcs8300_data = { +const struct iris_platform_data qcs8300_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, @@ -1017,16 +1285,20 @@ struct iris_platform_data qcs8300_data = { .opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table), .clk_tbl = sm8550_clk_table, .clk_tbl_size = ARRAY_SIZE(sm8550_clk_table), + .opp_clk_tbl = sm8550_opp_clk_table, /* Upper bound of DMA address range */ .dma_mask = 0xe0000000 - 1, .fwname = "qcom/vpu/vpu30_p4_s6.mbn", .pas_id = IRIS_PAS_ID, + .inst_iris_fmts = platform_fmts_sm8550_dec, + .inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec), .inst_caps = &platform_inst_cap_qcs8300, - .inst_fw_caps_dec = inst_fw_cap_qcs8300_dec, - .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_qcs8300_dec), - .inst_fw_caps_enc = inst_fw_cap_qcs8300_enc, - .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_qcs8300_enc), - .tz_cp_config_data = &tz_cp_config_sm8550, + .inst_fw_caps_dec = inst_fw_cap_sm8550_dec, + .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec), + .inst_fw_caps_enc = inst_fw_cap_sm8550_enc, + .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc), + .tz_cp_config_data = tz_cp_config_sm8550, + .tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550), .core_arch = VIDEO_ARCH_LX, .hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE, .ubwc_config = &ubwc_config_sm8550, @@ -1046,6 +1318,114 @@ struct iris_platform_data qcs8300_data = { sm8550_vdec_input_config_param_vp9, .dec_input_config_params_vp9_size = ARRAY_SIZE(sm8550_vdec_input_config_param_vp9), + .dec_input_config_params_av1 = + sm8550_vdec_input_config_param_av1, + .dec_input_config_params_av1_size = + ARRAY_SIZE(sm8550_vdec_input_config_param_av1), + .dec_output_config_params = + sm8550_vdec_output_config_params, + .dec_output_config_params_size = + ARRAY_SIZE(sm8550_vdec_output_config_params), + + .enc_input_config_params = + sm8550_venc_input_config_params, + .enc_input_config_params_size = + ARRAY_SIZE(sm8550_venc_input_config_params), + .enc_output_config_params = + sm8550_venc_output_config_params, + .enc_output_config_params_size = + ARRAY_SIZE(sm8550_venc_output_config_params), + + .dec_input_prop = sm8550_vdec_subscribe_input_properties, + .dec_input_prop_size = ARRAY_SIZE(sm8550_vdec_subscribe_input_properties), + .dec_output_prop_avc = sm8550_vdec_subscribe_output_properties_avc, + .dec_output_prop_avc_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_avc), + .dec_output_prop_hevc = sm8550_vdec_subscribe_output_properties_hevc, + .dec_output_prop_hevc_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_hevc), + .dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9, + .dec_output_prop_vp9_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9), + .dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1, + .dec_output_prop_av1_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1), + + .dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl, + .dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl), + .dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl, + .dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl), + + .enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl, + .enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl), + .enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl, + .enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl), +}; + +/* + * Shares most of SM8550 data except: + * - clk_tbl and opp_clk_tbl for x1p42100 + * - different firmware + * - different num_vpp_pipe + */ +const struct iris_platform_data x1p42100_data = { + .get_instance = iris_hfi_gen2_get_instance, + .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, + .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, + .get_vpu_buffer_size = iris_vpu_buf_size, + .vpu_ops = &iris_vpu3_ops, + .set_preset_registers = iris_set_sm8550_preset_registers, + .init_cb_devs = sm8550_init_cb_devs, + .deinit_cb_devs = sm8550_deinit_cb_devs, + .icc_tbl = sm8550_icc_table, + .icc_tbl_size = ARRAY_SIZE(sm8550_icc_table), + .clk_rst_tbl = sm8550_clk_reset_table, + .clk_rst_tbl_size = ARRAY_SIZE(sm8550_clk_reset_table), + .bw_tbl_dec = sm8550_bw_table_dec, + .bw_tbl_dec_size = ARRAY_SIZE(sm8550_bw_table_dec), + .pmdomain_tbl = sm8550_pmdomain_table, + .pmdomain_tbl_size = ARRAY_SIZE(sm8550_pmdomain_table), + .opp_pd_tbl = sm8550_opp_pd_table, + .opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table), + .clk_tbl = x1p42100_clk_table, + .clk_tbl_size = ARRAY_SIZE(x1p42100_clk_table), + .opp_clk_tbl = x1p42100_opp_clk_table, + /* Upper bound of DMA address range */ + .dma_mask = 0xe0000000 - 1, + .fwname = "qcom/vpu/vpu30_1v.mbn", + .pas_id = IRIS_PAS_ID, + .inst_iris_fmts = platform_fmts_sm8550_dec, + .inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec), + .inst_caps = &platform_inst_cap_sm8550, + .inst_fw_caps_dec = inst_fw_cap_sm8550_dec, + .inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec), + .inst_fw_caps_enc = inst_fw_cap_sm8550_enc, + .inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc), + .tz_cp_config_data = tz_cp_config_sm8550, + .tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550), + .core_arch = VIDEO_ARCH_LX, + .hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE, + .ubwc_config = &ubwc_config_sm8550, + .num_vpp_pipe = 1, + .max_session_count = 16, + .max_core_mbpf = NUM_MBS_8K * 2, + .max_core_mbps = ((7680 * 4320) / 256) * 60, + .dec_input_config_params_default = + sm8550_vdec_input_config_params_default, + .dec_input_config_params_default_size = + ARRAY_SIZE(sm8550_vdec_input_config_params_default), + .dec_input_config_params_hevc = + sm8550_vdec_input_config_param_hevc, + .dec_input_config_params_hevc_size = + ARRAY_SIZE(sm8550_vdec_input_config_param_hevc), + .dec_input_config_params_vp9 = + sm8550_vdec_input_config_param_vp9, + .dec_input_config_params_vp9_size = + ARRAY_SIZE(sm8550_vdec_input_config_param_vp9), + .dec_input_config_params_av1 = + sm8550_vdec_input_config_param_av1, + .dec_input_config_params_av1_size = + ARRAY_SIZE(sm8550_vdec_input_config_param_av1), .dec_output_config_params = sm8550_vdec_output_config_params, .dec_output_config_params_size = @@ -1071,12 +1451,17 @@ struct iris_platform_data qcs8300_data = { .dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9, .dec_output_prop_vp9_size = ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9), + .dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1, + .dec_output_prop_av1_size = + ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1), .dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl, .dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl), .dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl, .dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl), + .enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl, + .enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl), .enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl, .enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl), }; diff --git a/drivers/media/platform/qcom/iris/iris_platform_qcs8300.h b/drivers/media/platform/qcom/iris/iris_platform_qcs8300.h index 35ea0efade73c..61025f1e965b8 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_qcs8300.h +++ b/drivers/media/platform/qcom/iris/iris_platform_qcs8300.h @@ -3,537 +3,8 @@ * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved. */ -#define BITRATE_MAX 245000000 - -static struct platform_inst_fw_cap inst_fw_cap_qcs8300_dec[] = { - { - .cap_id = PROFILE_H264, - .min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE, - .max = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH), - .value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH, - .hfi_id = HFI_PROP_PROFILE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - .set = iris_set_u32_enum, - }, - { - .cap_id = PROFILE_HEVC, - .min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, - .max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) | - BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE), - .value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, - .hfi_id = HFI_PROP_PROFILE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - .set = iris_set_u32_enum, - }, - { - .cap_id = PROFILE_VP9, - .min = V4L2_MPEG_VIDEO_VP9_PROFILE_0, - .max = V4L2_MPEG_VIDEO_VP9_PROFILE_2, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_VP9_PROFILE_0) | - BIT(V4L2_MPEG_VIDEO_VP9_PROFILE_2), - .value = V4L2_MPEG_VIDEO_VP9_PROFILE_0, - .hfi_id = HFI_PROP_PROFILE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - .set = iris_set_u32_enum, - }, - { - .cap_id = LEVEL_H264, - .min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0, - .max = V4L2_MPEG_VIDEO_H264_LEVEL_6_2, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_2), - .value = V4L2_MPEG_VIDEO_H264_LEVEL_6_1, - .hfi_id = HFI_PROP_LEVEL, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - .set = iris_set_u32_enum, - }, - { - .cap_id = LEVEL_HEVC, - .min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1, - .max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2), - .value = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1, - .hfi_id = HFI_PROP_LEVEL, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - .set = iris_set_u32_enum, - }, - { - .cap_id = LEVEL_VP9, - .min = V4L2_MPEG_VIDEO_VP9_LEVEL_1_0, - .max = V4L2_MPEG_VIDEO_VP9_LEVEL_6_0, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_1_0) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_1_1) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_2_0) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_2_1) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_3_0) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_3_1) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_4_0) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_4_1) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_0) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_1) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_2) | - BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_6_0), - .value = V4L2_MPEG_VIDEO_VP9_LEVEL_6_0, - .hfi_id = HFI_PROP_LEVEL, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - .set = iris_set_u32_enum, - }, - { - .cap_id = TIER, - .min = V4L2_MPEG_VIDEO_HEVC_TIER_MAIN, - .max = V4L2_MPEG_VIDEO_HEVC_TIER_HIGH, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_TIER_MAIN) | - BIT(V4L2_MPEG_VIDEO_HEVC_TIER_HIGH), - .value = V4L2_MPEG_VIDEO_HEVC_TIER_HIGH, - .hfi_id = HFI_PROP_TIER, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - .set = iris_set_u32_enum, - }, - { - .cap_id = INPUT_BUF_HOST_MAX_COUNT, - .min = DEFAULT_MAX_HOST_BUF_COUNT, - .max = DEFAULT_MAX_HOST_BURST_BUF_COUNT, - .step_or_mask = 1, - .value = DEFAULT_MAX_HOST_BUF_COUNT, - .hfi_id = HFI_PROP_BUFFER_HOST_MAX_COUNT, - .flags = CAP_FLAG_INPUT_PORT, - .set = iris_set_u32, - }, - { - .cap_id = STAGE, - .min = STAGE_1, - .max = STAGE_2, - .step_or_mask = 1, - .value = STAGE_2, - .hfi_id = HFI_PROP_STAGE, - .set = iris_set_stage, - }, - { - .cap_id = PIPE, - .min = PIPE_1, - .max = PIPE_2, - .step_or_mask = 1, - .value = PIPE_2, - .hfi_id = HFI_PROP_PIPE, - .set = iris_set_pipe, - }, - { - .cap_id = POC, - .min = 0, - .max = 2, - .step_or_mask = 1, - .value = 1, - .hfi_id = HFI_PROP_PIC_ORDER_CNT_TYPE, - }, - { - .cap_id = CODED_FRAMES, - .min = CODED_FRAMES_PROGRESSIVE, - .max = CODED_FRAMES_PROGRESSIVE, - .step_or_mask = 0, - .value = CODED_FRAMES_PROGRESSIVE, - .hfi_id = HFI_PROP_CODED_FRAMES, - }, - { - .cap_id = BIT_DEPTH, - .min = BIT_DEPTH_8, - .max = BIT_DEPTH_8, - .step_or_mask = 1, - .value = BIT_DEPTH_8, - .hfi_id = HFI_PROP_LUMA_CHROMA_BIT_DEPTH, - }, - { - .cap_id = RAP_FRAME, - .min = 0, - .max = 1, - .step_or_mask = 1, - .value = 1, - .hfi_id = HFI_PROP_DEC_START_FROM_RAP_FRAME, - .flags = CAP_FLAG_INPUT_PORT, - .set = iris_set_u32, - }, -}; - -static struct platform_inst_fw_cap inst_fw_cap_qcs8300_enc[] = { - { - .cap_id = PROFILE_H264, - .min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE, - .max = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) | - BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH), - .value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH, - .hfi_id = HFI_PROP_PROFILE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = PROFILE_HEVC, - .min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, - .max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) | - BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE) | - BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10), - .value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, - .hfi_id = HFI_PROP_PROFILE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = LEVEL_H264, - .min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0, - .max = V4L2_MPEG_VIDEO_H264_LEVEL_6_0, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_2) | - BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_0), - .value = V4L2_MPEG_VIDEO_H264_LEVEL_5_0, - .hfi_id = HFI_PROP_LEVEL, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = LEVEL_HEVC, - .min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1, - .max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) | - BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2), - .value = V4L2_MPEG_VIDEO_HEVC_LEVEL_5, - .hfi_id = HFI_PROP_LEVEL, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = STAGE, - .min = STAGE_1, - .max = STAGE_2, - .step_or_mask = 1, - .value = STAGE_2, - .hfi_id = HFI_PROP_STAGE, - }, - { - .cap_id = HEADER_MODE, - .min = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE, - .max = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE) | - BIT(V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME), - .value = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, - .hfi_id = HFI_PROP_SEQ_HEADER_MODE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = PREPEND_SPSPPS_TO_IDR, - .min = 0, - .max = 1, - .step_or_mask = 1, - .value = 0, - }, - { - .cap_id = BITRATE, - .min = 1, - .max = BITRATE_MAX, - .step_or_mask = 1, - .value = BITRATE_DEFAULT, - .hfi_id = HFI_PROP_TOTAL_BITRATE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = BITRATE_PEAK, - .min = 1, - .max = BITRATE_MAX, - .step_or_mask = 1, - .value = BITRATE_DEFAULT, - .hfi_id = HFI_PROP_TOTAL_PEAK_BITRATE, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = BITRATE_MODE, - .min = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR, - .max = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) | - BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR), - .value = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR, - .hfi_id = HFI_PROP_RATE_CONTROL, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = FRAME_SKIP_MODE, - .min = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED, - .max = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED) | - BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_LEVEL_LIMIT) | - BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT), - .value = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = FRAME_RC_ENABLE, - .min = 0, - .max = 1, - .step_or_mask = 1, - .value = 1, - }, - { - .cap_id = GOP_SIZE, - .min = 0, - .max = INT_MAX, - .step_or_mask = 1, - .value = 2 * DEFAULT_FPS - 1, - .hfi_id = HFI_PROP_MAX_GOP_FRAMES, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = ENTROPY_MODE, - .min = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC, - .max = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC, - .step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC) | - BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC), - .value = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC, - .hfi_id = HFI_PROP_CABAC_SESSION, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU, - }, - { - .cap_id = MIN_FRAME_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - .hfi_id = HFI_PROP_MIN_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT, - }, - { - .cap_id = MIN_FRAME_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - .hfi_id = HFI_PROP_MIN_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT, - }, - { - .cap_id = MAX_FRAME_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - .hfi_id = HFI_PROP_MAX_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT, - }, - { - .cap_id = MAX_FRAME_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - .hfi_id = HFI_PROP_MAX_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT, - }, - { - .cap_id = I_FRAME_MIN_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - }, - { - .cap_id = I_FRAME_MIN_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - }, - { - .cap_id = P_FRAME_MIN_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - }, - { - .cap_id = P_FRAME_MIN_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - }, - { - .cap_id = B_FRAME_MIN_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - }, - { - .cap_id = B_FRAME_MIN_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MIN_QP_8BIT, - }, - { - .cap_id = I_FRAME_MAX_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - }, - { - .cap_id = I_FRAME_MAX_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - }, - { - .cap_id = P_FRAME_MAX_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - }, - { - .cap_id = P_FRAME_MAX_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - }, - { - .cap_id = B_FRAME_MAX_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - }, - { - .cap_id = B_FRAME_MAX_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = MAX_QP, - }, - { - .cap_id = I_FRAME_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = DEFAULT_QP, - .hfi_id = HFI_PROP_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = I_FRAME_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = DEFAULT_QP, - .hfi_id = HFI_PROP_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = P_FRAME_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = DEFAULT_QP, - .hfi_id = HFI_PROP_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = P_FRAME_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = DEFAULT_QP, - .hfi_id = HFI_PROP_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = B_FRAME_QP_H264, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = DEFAULT_QP, - .hfi_id = HFI_PROP_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, - { - .cap_id = B_FRAME_QP_HEVC, - .min = MIN_QP_8BIT, - .max = MAX_QP, - .step_or_mask = 1, - .value = DEFAULT_QP, - .hfi_id = HFI_PROP_QP_PACKED, - .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT | - CAP_FLAG_DYNAMIC_ALLOWED, - }, -}; +#ifndef __IRIS_PLATFORM_QCS8300_H__ +#define __IRIS_PLATFORM_QCS8300_H__ static struct platform_inst_caps platform_inst_cap_qcs8300 = { .min_frame_width = 96, @@ -548,3 +19,5 @@ static struct platform_inst_caps platform_inst_cap_qcs8300 = { .max_frame_rate = MAXIMUM_FPS, .max_operating_rate = MAXIMUM_FPS, }; + +#endif diff --git a/drivers/media/platform/qcom/iris/iris_platform_sc7280.h b/drivers/media/platform/qcom/iris/iris_platform_sc7280.h new file mode 100644 index 0000000000000..0ec8f334df670 --- /dev/null +++ b/drivers/media/platform/qcom/iris/iris_platform_sc7280.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __IRIS_PLATFORM_SC7280_H__ +#define __IRIS_PLATFORM_SC7280_H__ + +static const struct bw_info sc7280_bw_table_dec[] = { + { ((3840 * 2160) / 256) * 60, 1896000, }, + { ((3840 * 2160) / 256) * 30, 968000, }, + { ((1920 * 1080) / 256) * 60, 618000, }, + { ((1920 * 1080) / 256) * 30, 318000, }, +}; + +static const char * const sc7280_opp_pd_table[] = { "cx" }; + +static const struct platform_clk_data sc7280_clk_table[] = { + {IRIS_CTRL_CLK, "core" }, + {IRIS_AXI_CLK, "iface" }, + {IRIS_AHB_CLK, "bus" }, + {IRIS_HW_CLK, "vcodec_core" }, + {IRIS_HW_AHB_CLK, "vcodec_bus" }, +}; + +static const char * const sc7280_opp_clk_table[] = { + "vcodec_core", + NULL, +}; + +#endif diff --git a/drivers/media/platform/qcom/iris/iris_platform_x1p42100.h b/drivers/media/platform/qcom/iris/iris_platform_x1p42100.h new file mode 100644 index 0000000000000..d89acfbc1233d --- /dev/null +++ b/drivers/media/platform/qcom/iris/iris_platform_x1p42100.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __IRIS_PLATFORM_X1P42100_H__ +#define __IRIS_PLATFORM_X1P42100_H__ + +static const struct platform_clk_data x1p42100_clk_table[] = { + {IRIS_AXI_CLK, "iface" }, + {IRIS_CTRL_CLK, "core" }, + {IRIS_HW_CLK, "vcodec0_core" }, + {IRIS_BSE_HW_CLK, "vcodec0_bse" }, +}; + +static const char *const x1p42100_opp_clk_table[] = { + "vcodec0_core", + "vcodec0_bse", + NULL, +}; + +#endif diff --git a/drivers/media/platform/qcom/iris/iris_power.c b/drivers/media/platform/qcom/iris/iris_power.c index dbca42df0910f..91aa21d4070eb 100644 --- a/drivers/media/platform/qcom/iris/iris_power.c +++ b/drivers/media/platform/qcom/iris/iris_power.c @@ -91,7 +91,7 @@ static int iris_set_clocks(struct iris_inst *inst) } core->power.clk_freq = freq; - ret = dev_pm_opp_set_rate(core->dev, freq); + ret = iris_opp_set_rate(core->dev, freq); mutex_unlock(&core->lock); return ret; diff --git a/drivers/media/platform/qcom/iris/iris_probe.c b/drivers/media/platform/qcom/iris/iris_probe.c index 00e99be16e087..25368cea89cc3 100644 --- a/drivers/media/platform/qcom/iris/iris_probe.c +++ b/drivers/media/platform/qcom/iris/iris_probe.c @@ -40,8 +40,6 @@ static int iris_init_icc(struct iris_core *core) static int iris_init_power_domains(struct iris_core *core) { - const struct platform_clk_data *clk_tbl; - u32 clk_cnt, i; int ret; struct dev_pm_domain_attach_data iris_pd_data = { @@ -56,6 +54,11 @@ static int iris_init_power_domains(struct iris_core *core) .pd_flags = PD_FLAG_DEV_LINK_ON | PD_FLAG_REQUIRED_OPP, }; + struct dev_pm_opp_config iris_opp_clk_data = { + .clk_names = core->iris_platform_data->opp_clk_tbl, + .config_clks = dev_pm_opp_config_clks_simple, + }; + ret = devm_pm_domain_attach_list(core->dev, &iris_pd_data, &core->pmdomain_tbl); if (ret < 0) return ret; @@ -64,16 +67,9 @@ static int iris_init_power_domains(struct iris_core *core) if (ret < 0) return ret; - clk_tbl = core->iris_platform_data->clk_tbl; - clk_cnt = core->iris_platform_data->clk_tbl_size; - - for (i = 0; i < clk_cnt; i++) { - if (clk_tbl[i].clk_type == IRIS_HW_CLK) { - ret = devm_pm_opp_set_clkname(core->dev, clk_tbl[i].clk_name); - if (ret) - return ret; - } - } + ret = devm_pm_opp_set_config(core->dev, &iris_opp_clk_data); + if (ret) + return ret; return devm_pm_opp_of_add_table(core->dev); } @@ -127,6 +123,20 @@ static int iris_init_resets(struct iris_core *core) core->iris_platform_data->controller_rst_tbl_size); } +static int iris_init_cb_devs(struct iris_core *core) +{ + if (core->iris_platform_data->init_cb_devs) + return core->iris_platform_data->init_cb_devs(core); + + return 0; +} + +static void iris_deinit_cb_devs(struct iris_core *core) +{ + if (core->iris_platform_data->deinit_cb_devs) + core->iris_platform_data->deinit_cb_devs(core); +} + static int iris_init_resources(struct iris_core *core) { int ret; @@ -197,6 +207,7 @@ static void iris_remove(struct platform_device *pdev) return; iris_core_deinit(core); + iris_deinit_cb_devs(core); video_unregister_device(core->vdev_dec); video_unregister_device(core->vdev_enc); @@ -263,11 +274,15 @@ static int iris_probe(struct platform_device *pdev) if (ret) return ret; + ret = iris_init_cb_devs(core); + if (ret) + return ret; + iris_session_init_caps(core); ret = v4l2_device_register(dev, &core->v4l2_dev); if (ret) - return ret; + goto err_deinit_cb; ret = iris_register_video_device(core, DECODER); if (ret) @@ -281,9 +296,11 @@ static int iris_probe(struct platform_device *pdev) dma_mask = core->iris_platform_data->dma_mask; - ret = dma_set_mask_and_coherent(dev, dma_mask); - if (ret) - goto err_vdev_unreg_enc; + if (device_iommu_mapped(core->dev)) { + ret = dma_set_mask_and_coherent(core->dev, dma_mask); + if (ret) + goto err_vdev_unreg_enc; + } dma_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); dma_set_seg_boundary(&pdev->dev, DMA_BIT_MASK(32)); @@ -302,6 +319,8 @@ static int iris_probe(struct platform_device *pdev) video_unregister_device(core->vdev_dec); err_v4l2_unreg: v4l2_device_unregister(&core->v4l2_dev); +err_deinit_cb: + iris_deinit_cb_devs(core); return ret; } @@ -356,12 +375,14 @@ static const struct of_device_id iris_dt_match[] = { .compatible = "qcom,qcs8300-iris", .data = &qcs8300_data, }, -#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_VENUS)) + { + .compatible = "qcom,sc7280-venus", + .data = &sc7280_data, + }, { .compatible = "qcom,sm8250-venus", .data = &sm8250_data, }, -#endif { .compatible = "qcom,sm8550-iris", .data = &sm8550_data, @@ -374,6 +395,10 @@ static const struct of_device_id iris_dt_match[] = { .compatible = "qcom,sm8750-iris", .data = &sm8750_data, }, + { + .compatible = "qcom,x1p42100-iris", + .data = &x1p42100_data, + }, { }, }; MODULE_DEVICE_TABLE(of, iris_dt_match); diff --git a/drivers/media/platform/qcom/iris/iris_resources.c b/drivers/media/platform/qcom/iris/iris_resources.c index cf32f268b703c..b10884eb3ac64 100644 --- a/drivers/media/platform/qcom/iris/iris_resources.c +++ b/drivers/media/platform/qcom/iris/iris_resources.c @@ -4,13 +4,16 @@ */ #include +#include #include +#include #include #include #include #include #include "iris_core.h" +#include "iris_instance.h" #include "iris_resources.h" #define BW_THRESHOLD 50000 @@ -58,11 +61,22 @@ int iris_unset_icc_bw(struct iris_core *core) return icc_bulk_set_bw(core->icc_count, core->icc_tbl); } +int iris_opp_set_rate(struct device *dev, unsigned long freq) +{ + struct dev_pm_opp *opp __free(put_opp); + + opp = devfreq_recommended_opp(dev, &freq, 0); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + return dev_pm_opp_set_opp(dev, opp); +} + int iris_enable_power_domains(struct iris_core *core, struct device *pd_dev) { int ret; - ret = dev_pm_opp_set_rate(core->dev, ULONG_MAX); + ret = iris_opp_set_rate(core->dev, ULONG_MAX); if (ret) return ret; @@ -77,7 +91,7 @@ int iris_disable_power_domains(struct iris_core *core, struct device *pd_dev) { int ret; - ret = dev_pm_opp_set_rate(core->dev, 0); + ret = iris_opp_set_rate(core->dev, 0); if (ret) return ret; @@ -112,7 +126,7 @@ int iris_prepare_enable_clock(struct iris_core *core, enum platform_clk_type clk clock = iris_get_clk_by_type(core, clk_type); if (!clock) - return -EINVAL; + return -ENOENT; return clk_prepare_enable(clock); } @@ -129,3 +143,77 @@ int iris_disable_unprepare_clock(struct iris_core *core, enum platform_clk_type return 0; } + +struct device *iris_create_cb_dev(struct iris_core *core, const char *name, const u32 *f_id) +{ + struct platform_device *pdev; + int ret; + + pdev = platform_device_alloc(name, 0); + if (!pdev) + return ERR_PTR(-ENOMEM); + + pdev->dev.parent = core->dev; + + ret = platform_device_add(pdev); + if (ret) { + platform_device_put(pdev); + return ERR_PTR(ret); + } + + ret = of_dma_configure_id(&pdev->dev, core->dev->of_node, true, f_id); + if (ret) + goto error_unregister; + + ret = dma_set_mask_and_coherent(&pdev->dev, core->iris_platform_data->dma_mask); + if (ret) + goto error_unregister; + + return &pdev->dev; + +error_unregister: + platform_device_unregister(to_platform_device(&pdev->dev)); + + return ERR_PTR(ret); +} + +struct device *iris_get_cb_dev(struct iris_inst *inst, enum iris_buffer_type buffer_type) +{ + struct iris_core *core = inst->core; + struct device *dev = NULL; + + switch (buffer_type) { + case BUF_INPUT: + if (inst->domain == DECODER) + dev = core->dev_bs; + else + dev = core->dev_p; + break; + case BUF_OUTPUT: + if (inst->domain == DECODER) + dev = core->dev_p; + else + dev = core->dev_bs; + break; + case BUF_BIN: + dev = core->dev_bs; + break; + case BUF_DPB: + case BUF_PARTIAL: + case BUF_SCRATCH_2: + case BUF_VPSS: + dev = core->dev_p; + break; + case BUF_ARP: + case BUF_COMV: + case BUF_LINE: + case BUF_NON_COMV: + case BUF_PERSIST: + dev = core->dev_np; + break; + default: + dev_err(core->dev, "invalid buffer type: %d\n", buffer_type); + } + + return dev ? dev : core->dev; +} diff --git a/drivers/media/platform/qcom/iris/iris_resources.h b/drivers/media/platform/qcom/iris/iris_resources.h index f723dfe5bd81a..e6a6dc24a7f12 100644 --- a/drivers/media/platform/qcom/iris/iris_resources.h +++ b/drivers/media/platform/qcom/iris/iris_resources.h @@ -8,11 +8,14 @@ struct iris_core; +int iris_opp_set_rate(struct device *dev, unsigned long freq); int iris_enable_power_domains(struct iris_core *core, struct device *pd_dev); int iris_disable_power_domains(struct iris_core *core, struct device *pd_dev); int iris_unset_icc_bw(struct iris_core *core); int iris_set_icc_bw(struct iris_core *core, unsigned long icc_bw); int iris_disable_unprepare_clock(struct iris_core *core, enum platform_clk_type clk_type); int iris_prepare_enable_clock(struct iris_core *core, enum platform_clk_type clk_type); +struct device *iris_create_cb_dev(struct iris_core *core, const char *name, const u32 *f_id); +struct device *iris_get_cb_dev(struct iris_inst *inst, enum iris_buffer_type buffer_type); #endif diff --git a/drivers/media/platform/qcom/iris/iris_utils.c b/drivers/media/platform/qcom/iris/iris_utils.c index 85c70a62b1fd2..cfc5b576ec56b 100644 --- a/drivers/media/platform/qcom/iris/iris_utils.c +++ b/drivers/media/platform/qcom/iris/iris_utils.c @@ -34,7 +34,8 @@ int iris_get_mbpf(struct iris_inst *inst) bool iris_split_mode_enabled(struct iris_inst *inst) { - return inst->fmt_dst->fmt.pix_mp.pixelformat == V4L2_PIX_FMT_NV12; + return inst->fmt_dst->fmt.pix_mp.pixelformat == V4L2_PIX_FMT_NV12 || + inst->fmt_dst->fmt.pix_mp.pixelformat == V4L2_PIX_FMT_QC08C; } void iris_helper_buffers_done(struct iris_inst *inst, unsigned int type, @@ -124,3 +125,9 @@ int iris_check_core_mbps(struct iris_inst *inst) return 0; } + +bool is_rotation_90_or_270(struct iris_inst *inst) +{ + return inst->fw_caps[ROTATION].value == 90 || + inst->fw_caps[ROTATION].value == 270; +} diff --git a/drivers/media/platform/qcom/iris/iris_utils.h b/drivers/media/platform/qcom/iris/iris_utils.h index 75740181122f5..b5705d156431a 100644 --- a/drivers/media/platform/qcom/iris/iris_utils.h +++ b/drivers/media/platform/qcom/iris/iris_utils.h @@ -51,5 +51,6 @@ void iris_helper_buffers_done(struct iris_inst *inst, unsigned int type, int iris_wait_for_session_response(struct iris_inst *inst, bool is_flush); int iris_check_core_mbpf(struct iris_inst *inst); int iris_check_core_mbps(struct iris_inst *inst); +bool is_rotation_90_or_270(struct iris_inst *inst); #endif diff --git a/drivers/media/platform/qcom/iris/iris_vb2.c b/drivers/media/platform/qcom/iris/iris_vb2.c index 139b821f7952f..bf0b8400996ec 100644 --- a/drivers/media/platform/qcom/iris/iris_vb2.c +++ b/drivers/media/platform/qcom/iris/iris_vb2.c @@ -193,10 +193,14 @@ int iris_vb2_start_streaming(struct vb2_queue *q, unsigned int count) buf_type = iris_v4l2_type_to_driver(q->type); if (inst->domain == DECODER) { - if (inst->state == IRIS_INST_STREAMING) + if (buf_type == BUF_INPUT) + ret = iris_queue_deferred_buffers(inst, BUF_INPUT); + + if (!ret && inst->state == IRIS_INST_STREAMING) { ret = iris_queue_internal_deferred_buffers(inst, BUF_DPB); - if (!ret) - ret = iris_queue_deferred_buffers(inst, buf_type); + if (!ret) + ret = iris_queue_deferred_buffers(inst, BUF_OUTPUT); + } } else { if (inst->state == IRIS_INST_STREAMING) { ret = iris_queue_deferred_buffers(inst, BUF_INPUT); diff --git a/drivers/media/platform/qcom/iris/iris_vdec.c b/drivers/media/platform/qcom/iris/iris_vdec.c index ae13c3e1b426b..91c885f62a82a 100644 --- a/drivers/media/platform/qcom/iris/iris_vdec.c +++ b/drivers/media/platform/qcom/iris/iris_vdec.c @@ -54,6 +54,7 @@ int iris_vdec_inst_init(struct iris_inst *inst) f->fmt.pix_mp.quantization = V4L2_QUANTIZATION_DEFAULT; inst->buffers[BUF_OUTPUT].min_count = iris_vpu_buf_count(inst, BUF_OUTPUT); inst->buffers[BUF_OUTPUT].size = f->fmt.pix_mp.plane_fmt[0].sizeimage; + inst->frame_rate = MAXIMUM_FPS; memcpy(&inst->fw_caps[0], &core->inst_fw_caps_dec[0], INST_FW_CAP_MAX * sizeof(struct platform_inst_fw_cap)); @@ -67,27 +68,35 @@ void iris_vdec_inst_deinit(struct iris_inst *inst) kfree(inst->fmt_src); } -static const struct iris_fmt iris_vdec_formats[] = { - [IRIS_FMT_H264] = { - .pixfmt = V4L2_PIX_FMT_H264, - .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, +static const struct iris_fmt iris_vdec_formats_cap[] = { + [IRIS_FMT_NV12] = { + .pixfmt = V4L2_PIX_FMT_NV12, + .type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, }, - [IRIS_FMT_HEVC] = { - .pixfmt = V4L2_PIX_FMT_HEVC, - .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, - }, - [IRIS_FMT_VP9] = { - .pixfmt = V4L2_PIX_FMT_VP9, - .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + [IRIS_FMT_QC08C] = { + .pixfmt = V4L2_PIX_FMT_QC08C, + .type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, }, }; static const struct iris_fmt * find_format(struct iris_inst *inst, u32 pixfmt, u32 type) { - unsigned int size = ARRAY_SIZE(iris_vdec_formats); - const struct iris_fmt *fmt = iris_vdec_formats; + const struct iris_fmt *fmt = NULL; + unsigned int size = 0; unsigned int i; + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + fmt = inst->core->iris_platform_data->inst_iris_fmts; + size = inst->core->iris_platform_data->inst_iris_fmts_size; + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + fmt = iris_vdec_formats_cap; + size = ARRAY_SIZE(iris_vdec_formats_cap); + break; + default: + return NULL; + } for (i = 0; i < size; i++) { if (fmt[i].pixfmt == pixfmt) @@ -103,8 +112,21 @@ find_format(struct iris_inst *inst, u32 pixfmt, u32 type) static const struct iris_fmt * find_format_by_index(struct iris_inst *inst, u32 index, u32 type) { - const struct iris_fmt *fmt = iris_vdec_formats; - unsigned int size = ARRAY_SIZE(iris_vdec_formats); + const struct iris_fmt *fmt = NULL; + unsigned int size = 0; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + fmt = inst->core->iris_platform_data->inst_iris_fmts; + size = inst->core->iris_platform_data->inst_iris_fmts_size; + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + fmt = iris_vdec_formats_cap; + size = ARRAY_SIZE(iris_vdec_formats_cap); + break; + default: + return NULL; + } if (index >= size || fmt[index].type != type) return NULL; @@ -126,9 +148,10 @@ int iris_vdec_enum_fmt(struct iris_inst *inst, struct v4l2_fmtdesc *f) f->flags = V4L2_FMT_FLAG_COMPRESSED | V4L2_FMT_FLAG_DYN_RESOLUTION; break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: - if (f->index) + fmt = find_format_by_index(inst, f->index, f->type); + if (!fmt) return -EINVAL; - f->pixelformat = V4L2_PIX_FMT_NV12; + f->pixelformat = fmt->pixfmt; break; default: return -EINVAL; @@ -157,7 +180,7 @@ int iris_vdec_try_fmt(struct iris_inst *inst, struct v4l2_format *f) } break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: - if (f->fmt.pix_mp.pixelformat != V4L2_PIX_FMT_NV12) { + if (!fmt) { f_inst = inst->fmt_dst; f->fmt.pix_mp.pixelformat = f_inst->fmt.pix_mp.pixelformat; f->fmt.pix_mp.width = f_inst->fmt.pix_mp.width; @@ -196,6 +219,14 @@ int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f) if (vb2_is_busy(q)) return -EBUSY; + /* Width and height are optional, so fall back to a valid placeholder + * resolution until the real one is decoded from the bitstream. + */ + if (f->fmt.pix_mp.width == 0 && f->fmt.pix_mp.height == 0) { + f->fmt.pix_mp.width = DEFAULT_WIDTH; + f->fmt.pix_mp.height = DEFAULT_HEIGHT; + } + iris_vdec_try_fmt(inst, f); switch (f->type) { @@ -238,10 +269,11 @@ int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f) inst->crop.height = f->fmt.pix_mp.height; break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + if (!(find_format(inst, f->fmt.pix_mp.pixelformat, f->type))) + return -EINVAL; + fmt = inst->fmt_dst; fmt->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - if (fmt->fmt.pix_mp.pixelformat != V4L2_PIX_FMT_NV12) - return -EINVAL; fmt->fmt.pix_mp.pixelformat = f->fmt.pix_mp.pixelformat; fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, 128); fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, 32); @@ -268,7 +300,8 @@ int iris_vdec_validate_format(struct iris_inst *inst, u32 pixelformat) { const struct iris_fmt *fmt = NULL; - if (pixelformat != V4L2_PIX_FMT_NV12) { + fmt = find_format(inst, pixelformat, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + if (!fmt) { fmt = find_format(inst, pixelformat, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); if (!fmt) return -EINVAL; @@ -339,6 +372,8 @@ int iris_vdec_streamon_input(struct iris_inst *inst) if (ret) return ret; + inst->frame_counter = 0; + return iris_process_streamon_input(inst); } @@ -381,6 +416,7 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf) { struct iris_buffer *buf = to_iris_buffer(vbuf); struct vb2_buffer *vb2 = &vbuf->vb2_buf; + u64 cur_buf_ns, delta_ns; struct vb2_queue *q; int ret; @@ -397,6 +433,22 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf) return 0; } + if (buf->type == BUF_INPUT) { + cur_buf_ns = ktime_get_ns(); + + if (!inst->frame_counter) + inst->last_buf_ns = cur_buf_ns; + + inst->frame_counter++; + delta_ns = cur_buf_ns - inst->last_buf_ns; + + if (delta_ns >= NSEC_PER_SEC) { + inst->frame_rate = clamp_t(u32, inst->frame_counter, DEFAULT_FPS, + MAXIMUM_FPS); + inst->frame_counter = 0; + } + } + iris_scale_power(inst); return iris_queue_buffer(inst, buf); diff --git a/drivers/media/platform/qcom/iris/iris_venc.c b/drivers/media/platform/qcom/iris/iris_venc.c index 099bd5ed4ae02..13394f87b833e 100644 --- a/drivers/media/platform/qcom/iris/iris_venc.c +++ b/drivers/media/platform/qcom/iris/iris_venc.c @@ -62,12 +62,17 @@ int iris_venc_inst_init(struct iris_inst *inst) inst->crop.left = 0; inst->crop.top = 0; - inst->crop.width = f->fmt.pix_mp.width; - inst->crop.height = f->fmt.pix_mp.height; + inst->crop.width = DEFAULT_WIDTH; + inst->crop.height = DEFAULT_HEIGHT; inst->operating_rate = DEFAULT_FPS; inst->frame_rate = DEFAULT_FPS; + inst->enc_raw_width = DEFAULT_WIDTH; + inst->enc_raw_height = DEFAULT_HEIGHT; + inst->enc_scale_width = DEFAULT_WIDTH; + inst->enc_scale_height = DEFAULT_HEIGHT; + memcpy(&inst->fw_caps[0], &core->inst_fw_caps_enc[0], INST_FW_CAP_MAX * sizeof(struct platform_inst_fw_cap)); @@ -80,7 +85,7 @@ void iris_venc_inst_deinit(struct iris_inst *inst) kfree(inst->fmt_src); } -static const struct iris_fmt iris_venc_formats[] = { +static const struct iris_fmt iris_venc_formats_cap[] = { [IRIS_FMT_H264] = { .pixfmt = V4L2_PIX_FMT_H264, .type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, @@ -91,12 +96,35 @@ static const struct iris_fmt iris_venc_formats[] = { }, }; +static const struct iris_fmt iris_venc_formats_out[] = { + [IRIS_FMT_NV12] = { + .pixfmt = V4L2_PIX_FMT_NV12, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, + [IRIS_FMT_QC08C] = { + .pixfmt = V4L2_PIX_FMT_QC08C, + .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + }, +}; + static const struct iris_fmt * find_format(struct iris_inst *inst, u32 pixfmt, u32 type) { - const struct iris_fmt *fmt = iris_venc_formats; - unsigned int size = ARRAY_SIZE(iris_venc_formats); + const struct iris_fmt *fmt = NULL; + unsigned int size = 0; unsigned int i; + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + fmt = iris_venc_formats_out; + size = ARRAY_SIZE(iris_venc_formats_out); + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + fmt = iris_venc_formats_cap; + size = ARRAY_SIZE(iris_venc_formats_cap); + break; + default: + return NULL; + } for (i = 0; i < size; i++) { if (fmt[i].pixfmt == pixfmt) @@ -112,8 +140,21 @@ find_format(struct iris_inst *inst, u32 pixfmt, u32 type) static const struct iris_fmt * find_format_by_index(struct iris_inst *inst, u32 index, u32 type) { - const struct iris_fmt *fmt = iris_venc_formats; - unsigned int size = ARRAY_SIZE(iris_venc_formats); + const struct iris_fmt *fmt = NULL; + unsigned int size = 0; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + fmt = iris_venc_formats_out; + size = ARRAY_SIZE(iris_venc_formats_out); + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + fmt = iris_venc_formats_cap; + size = ARRAY_SIZE(iris_venc_formats_cap); + break; + default: + return NULL; + } if (index >= size || fmt[index].type != type) return NULL; @@ -127,9 +168,11 @@ int iris_venc_enum_fmt(struct iris_inst *inst, struct v4l2_fmtdesc *f) switch (f->type) { case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: - if (f->index) + fmt = find_format_by_index(inst, f->index, f->type); + if (!fmt) return -EINVAL; - f->pixelformat = V4L2_PIX_FMT_NV12; + + f->pixelformat = fmt->pixfmt; break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: fmt = find_format_by_index(inst, f->index, f->type); @@ -156,7 +199,7 @@ int iris_venc_try_fmt(struct iris_inst *inst, struct v4l2_format *f) fmt = find_format(inst, pixmp->pixelformat, f->type); switch (f->type) { case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: - if (f->fmt.pix_mp.pixelformat != V4L2_PIX_FMT_NV12) { + if (!fmt) { f_inst = inst->fmt_src; f->fmt.pix_mp.width = f_inst->fmt.pix_mp.width; f->fmt.pix_mp.height = f_inst->fmt.pix_mp.height; @@ -185,15 +228,32 @@ int iris_venc_try_fmt(struct iris_inst *inst, struct v4l2_format *f) static int iris_venc_s_fmt_output(struct iris_inst *inst, struct v4l2_format *f) { + const struct iris_fmt *venc_fmt; struct v4l2_format *fmt; + u32 codec_align; iris_venc_try_fmt(inst, f); - if (!(find_format(inst, f->fmt.pix_mp.pixelformat, f->type))) + venc_fmt = find_format(inst, f->fmt.pix_mp.pixelformat, f->type); + if (!venc_fmt) return -EINVAL; + codec_align = venc_fmt->pixfmt == V4L2_PIX_FMT_HEVC ? 32 : 16; + fmt = inst->fmt_dst; fmt->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + /* + * If output format size != input format size, + * it is considered a scaling case, + * and the scaled size needs to be saved. + */ + if (f->fmt.pix_mp.width != inst->fmt_src->fmt.pix_mp.width || + f->fmt.pix_mp.height != inst->fmt_src->fmt.pix_mp.height) { + inst->enc_scale_width = f->fmt.pix_mp.width; + inst->enc_scale_height = f->fmt.pix_mp.height; + fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, codec_align); + fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, codec_align); + } fmt->fmt.pix_mp.num_planes = 1; fmt->fmt.pix_mp.plane_fmt[0].bytesperline = 0; fmt->fmt.pix_mp.plane_fmt[0].sizeimage = iris_get_buffer_size(inst, BUF_OUTPUT); @@ -221,7 +281,7 @@ static int iris_venc_s_fmt_input(struct iris_inst *inst, struct v4l2_format *f) iris_venc_try_fmt(inst, f); - if (f->fmt.pix_mp.pixelformat != V4L2_PIX_FMT_NV12) + if (!(find_format(inst, f->fmt.pix_mp.pixelformat, f->type))) return -EINVAL; fmt = inst->fmt_src; @@ -249,6 +309,11 @@ static int iris_venc_s_fmt_input(struct iris_inst *inst, struct v4l2_format *f) inst->buffers[BUF_INPUT].min_count = iris_vpu_buf_count(inst, BUF_INPUT); inst->buffers[BUF_INPUT].size = fmt->fmt.pix_mp.plane_fmt[0].sizeimage; + inst->enc_raw_width = f->fmt.pix_mp.width; + inst->enc_raw_height = f->fmt.pix_mp.height; + inst->enc_scale_width = f->fmt.pix_mp.width; + inst->enc_scale_height = f->fmt.pix_mp.height; + if (f->fmt.pix_mp.width != inst->crop.width || f->fmt.pix_mp.height != inst->crop.height) { inst->crop.top = 0; @@ -289,7 +354,8 @@ int iris_venc_validate_format(struct iris_inst *inst, u32 pixelformat) { const struct iris_fmt *fmt = NULL; - if (pixelformat != V4L2_PIX_FMT_NV12) { + fmt = find_format(inst, pixelformat, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + if (!fmt) { fmt = find_format(inst, pixelformat, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); if (!fmt) return -EINVAL; @@ -345,8 +411,7 @@ int iris_venc_s_param(struct iris_inst *inst, struct v4l2_streamparm *s_parm) struct v4l2_fract *timeperframe = NULL; u32 default_rate = DEFAULT_FPS; bool is_frame_rate = false; - u64 us_per_frame, fps; - u32 max_rate; + u32 fps, max_rate; int ret = 0; @@ -368,23 +433,19 @@ int iris_venc_s_param(struct iris_inst *inst, struct v4l2_streamparm *s_parm) timeperframe->denominator = default_rate; } - us_per_frame = timeperframe->numerator * (u64)USEC_PER_SEC; - do_div(us_per_frame, timeperframe->denominator); - - if (!us_per_frame) + fps = timeperframe->denominator / timeperframe->numerator; + if (!fps) return -EINVAL; - fps = (u64)USEC_PER_SEC; - do_div(fps, us_per_frame); if (fps > max_rate) { ret = -ENOMEM; goto reset_rate; } if (is_frame_rate) - inst->frame_rate = (u32)fps; + inst->frame_rate = fps; else - inst->operating_rate = (u32)fps; + inst->operating_rate = fps; if ((s_parm->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && vb2_is_streaming(src_q)) || (s_parm->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE && vb2_is_streaming(dst_q))) { diff --git a/drivers/media/platform/qcom/iris/iris_vidc.c b/drivers/media/platform/qcom/iris/iris_vidc.c index d38d0f6961cd5..8ed6520f5ae3a 100644 --- a/drivers/media/platform/qcom/iris/iris_vidc.c +++ b/drivers/media/platform/qcom/iris/iris_vidc.c @@ -107,7 +107,7 @@ iris_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_ src_vq->drv_priv = inst; src_vq->buf_struct_size = sizeof(struct iris_buffer); src_vq->min_reqbufs_allocation = MIN_BUFFERS; - src_vq->dev = inst->core->dev; + src_vq->dev = iris_get_cb_dev(inst, BUF_INPUT); src_vq->lock = &inst->ctx_q_lock; ret = vb2_queue_init(src_vq); if (ret) @@ -121,7 +121,7 @@ iris_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_ dst_vq->drv_priv = inst; dst_vq->buf_struct_size = sizeof(struct iris_buffer); dst_vq->min_reqbufs_allocation = MIN_BUFFERS; - dst_vq->dev = inst->core->dev; + dst_vq->dev = iris_get_cb_dev(inst, BUF_OUTPUT); dst_vq->lock = &inst->ctx_q_lock; return vb2_queue_init(dst_vq); @@ -178,6 +178,7 @@ int iris_open(struct file *filp) INIT_LIST_HEAD(&inst->buffers[BUF_SCRATCH_1].list); INIT_LIST_HEAD(&inst->buffers[BUF_SCRATCH_2].list); INIT_LIST_HEAD(&inst->buffers[BUF_VPSS].list); + INIT_LIST_HEAD(&inst->buffers[BUF_PARTIAL].list); init_completion(&inst->completion); init_completion(&inst->flush_completion); @@ -572,9 +573,10 @@ static int iris_dec_cmd(struct file *filp, void *fh, mutex_lock(&inst->lock); - ret = v4l2_m2m_ioctl_decoder_cmd(filp, fh, dec); - if (ret) + if (dec->cmd != V4L2_DEC_CMD_STOP && dec->cmd != V4L2_DEC_CMD_START) { + ret = -EINVAL; goto unlock; + } if (inst->state == IRIS_INST_DEINIT) goto unlock; @@ -605,9 +607,10 @@ static int iris_enc_cmd(struct file *filp, void *fh, mutex_lock(&inst->lock); - ret = v4l2_m2m_ioctl_encoder_cmd(filp, fh, enc); - if (ret) + if (enc->cmd != V4L2_ENC_CMD_STOP && enc->cmd != V4L2_ENC_CMD_START) { + ret = -EINVAL; goto unlock; + } if (inst->state == IRIS_INST_DEINIT) goto unlock; @@ -630,7 +633,7 @@ static int iris_enc_cmd(struct file *filp, void *fh, return ret; } -static struct v4l2_file_operations iris_v4l2_file_ops = { +static const struct v4l2_file_operations iris_v4l2_file_ops = { .owner = THIS_MODULE, .open = iris_open, .release = iris_close, diff --git a/drivers/media/platform/qcom/iris/iris_vpu2.c b/drivers/media/platform/qcom/iris/iris_vpu2.c index de7d142316d2d..73c201f4f3382 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu2.c +++ b/drivers/media/platform/qcom/iris/iris_vpu2.c @@ -3,16 +3,22 @@ * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. */ +#include +#include +#include + #include "iris_instance.h" #include "iris_vpu_common.h" +#include "iris_vpu_register_defines.h" + static u64 iris_vpu2_calc_freq(struct iris_inst *inst, size_t data_size) { struct platform_inst_caps *caps = inst->core->iris_platform_data->inst_caps; struct v4l2_format *inp_f = inst->fmt_src; u32 mbs_per_second, mbpf, height, width; unsigned long vpp_freq, vsp_freq; - u32 fps = DEFAULT_FPS; + u32 fps = inst->frame_rate; width = max(inp_f->fmt.pix_mp.width, inst->crop.width); height = max(inp_f->fmt.pix_mp.height, inst->crop.height); diff --git a/drivers/media/platform/qcom/iris/iris_vpu3x.c b/drivers/media/platform/qcom/iris/iris_vpu3x.c index 339776a0b4672..fe4423b951b1e 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu3x.c +++ b/drivers/media/platform/qcom/iris/iris_vpu3x.c @@ -11,55 +11,11 @@ #include "iris_vpu_common.h" #include "iris_vpu_register_defines.h" -#define WRAPPER_TZ_BASE_OFFS 0x000C0000 -#define AON_BASE_OFFS 0x000E0000 -#define AON_MVP_NOC_RESET 0x0001F000 - -#define WRAPPER_DEBUG_BRIDGE_LPI_CONTROL (WRAPPER_BASE_OFFS + 0x54) -#define WRAPPER_DEBUG_BRIDGE_LPI_STATUS (WRAPPER_BASE_OFFS + 0x58) -#define WRAPPER_IRIS_CPU_NOC_LPI_CONTROL (WRAPPER_BASE_OFFS + 0x5C) -#define REQ_POWER_DOWN_PREP BIT(0) -#define WRAPPER_IRIS_CPU_NOC_LPI_STATUS (WRAPPER_BASE_OFFS + 0x60) -#define NOC_LPI_STATUS_DONE BIT(0) /* Indicates the NOC handshake is complete */ -#define NOC_LPI_STATUS_DENY BIT(1) /* Indicates the NOC handshake is denied */ -#define NOC_LPI_STATUS_ACTIVE BIT(2) /* Indicates the NOC is active */ -#define WRAPPER_CORE_CLOCK_CONFIG (WRAPPER_BASE_OFFS + 0x88) -#define CORE_CLK_RUN 0x0 -/* VPU v3.5 */ -#define WRAPPER_IRIS_VCODEC_VPU_WRAPPER_SPARE_0 (WRAPPER_BASE_OFFS + 0x78) - -#define WRAPPER_TZ_CTL_AXI_CLOCK_CONFIG (WRAPPER_TZ_BASE_OFFS + 0x14) -#define CTL_AXI_CLK_HALT BIT(0) -#define CTL_CLK_HALT BIT(1) - -#define WRAPPER_TZ_QNS4PDXFIFO_RESET (WRAPPER_TZ_BASE_OFFS + 0x18) -#define RESET_HIGH BIT(0) - -#define CPU_CS_AHB_BRIDGE_SYNC_RESET (CPU_CS_BASE_OFFS + 0x160) -#define CORE_BRIDGE_SW_RESET BIT(0) -#define CORE_BRIDGE_HW_RESET_DISABLE BIT(1) - -#define CPU_CS_X2RPMH (CPU_CS_BASE_OFFS + 0x168) -#define MSK_SIGNAL_FROM_TENSILICA BIT(0) -#define MSK_CORE_POWER_ON BIT(1) - -#define AON_WRAPPER_MVP_NOC_RESET_REQ (AON_MVP_NOC_RESET + 0x000) -#define VIDEO_NOC_RESET_REQ (BIT(0) | BIT(1)) - -#define AON_WRAPPER_MVP_NOC_RESET_ACK (AON_MVP_NOC_RESET + 0x004) - -#define VCODEC_SS_IDLE_STATUSN (VCODEC_BASE_OFFS + 0x70) - -#define AON_WRAPPER_MVP_NOC_LPI_CONTROL (AON_BASE_OFFS) -#define AON_WRAPPER_MVP_NOC_LPI_STATUS (AON_BASE_OFFS + 0x4) - #define AON_WRAPPER_MVP_NOC_CORE_SW_RESET (AON_BASE_OFFS + 0x18) #define SW_RESET BIT(0) #define AON_WRAPPER_MVP_NOC_CORE_CLK_CONTROL (AON_BASE_OFFS + 0x20) #define NOC_HALT BIT(0) #define AON_WRAPPER_SPARE (AON_BASE_OFFS + 0x28) -#define AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL (AON_BASE_OFFS + 0x2C) -#define AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_STATUS (AON_BASE_OFFS + 0x30) static bool iris_vpu3x_hw_power_collapsed(struct iris_core *core) { @@ -304,155 +260,12 @@ static void iris_vpu35_power_off_hw(struct iris_core *core) iris_disable_unprepare_clock(core, IRIS_AXI_CLK); } -static int iris_vpu35_power_off_controller(struct iris_core *core) -{ - u32 clk_rst_tbl_size = core->iris_platform_data->clk_rst_tbl_size; - unsigned int count = 0; - u32 val = 0; - bool handshake_done, handshake_busy; - int ret; - - writel(MSK_SIGNAL_FROM_TENSILICA | MSK_CORE_POWER_ON, core->reg_base + CPU_CS_X2RPMH); - - writel(REQ_POWER_DOWN_PREP, core->reg_base + WRAPPER_IRIS_CPU_NOC_LPI_CONTROL); - - ret = readl_poll_timeout(core->reg_base + WRAPPER_IRIS_CPU_NOC_LPI_STATUS, - val, val & BIT(0), 200, 2000); - if (ret) - goto disable_power; - - writel(0, core->reg_base + WRAPPER_IRIS_CPU_NOC_LPI_CONTROL); - - /* Retry up to 1000 times as recommended by hardware documentation */ - do { - /* set MNoC to low power */ - writel(REQ_POWER_DOWN_PREP, core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL); - - udelay(15); - - val = readl(core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_STATUS); - - handshake_done = val & NOC_LPI_STATUS_DONE; - handshake_busy = val & (NOC_LPI_STATUS_DENY | NOC_LPI_STATUS_ACTIVE); - - if (handshake_done || !handshake_busy) - break; - - writel(0, core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL); - - udelay(15); - - } while (++count < 1000); - - if (!handshake_done && handshake_busy) - dev_err(core->dev, "LPI handshake timeout\n"); - - ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_STATUS, - val, val & BIT(0), 200, 2000); - if (ret) - goto disable_power; - - writel(0, core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL); - - writel(0, core->reg_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL); - - ret = readl_poll_timeout(core->reg_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS, - val, val == 0, 200, 2000); - if (ret) - goto disable_power; - -disable_power: - iris_disable_unprepare_clock(core, IRIS_CTRL_CLK); - iris_disable_unprepare_clock(core, IRIS_CTRL_FREERUN_CLK); - iris_disable_unprepare_clock(core, IRIS_AXI1_CLK); - - iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); - - reset_control_bulk_reset(clk_rst_tbl_size, core->resets); - - return 0; -} - -static int iris_vpu35_power_on_controller(struct iris_core *core) -{ - int ret; - - ret = iris_enable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); - if (ret) - return ret; - - ret = iris_prepare_enable_clock(core, IRIS_AXI1_CLK); - if (ret) - goto err_disable_power; - - ret = iris_prepare_enable_clock(core, IRIS_CTRL_FREERUN_CLK); - if (ret) - goto err_disable_axi1_clk; - - ret = iris_prepare_enable_clock(core, IRIS_CTRL_CLK); - if (ret) - goto err_disable_ctrl_free_clk; - - return 0; - -err_disable_ctrl_free_clk: - iris_disable_unprepare_clock(core, IRIS_CTRL_FREERUN_CLK); -err_disable_axi1_clk: - iris_disable_unprepare_clock(core, IRIS_AXI1_CLK); -err_disable_power: - iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); - - return ret; -} - -static void iris_vpu35_program_bootup_registers(struct iris_core *core) -{ - writel(0x1, core->reg_base + WRAPPER_IRIS_VCODEC_VPU_WRAPPER_SPARE_0); -} - -static u64 iris_vpu3x_calculate_frequency(struct iris_inst *inst, size_t data_size) -{ - struct platform_inst_caps *caps = inst->core->iris_platform_data->inst_caps; - struct v4l2_format *inp_f = inst->fmt_src; - u32 height, width, mbs_per_second, mbpf; - u64 fw_cycles, fw_vpp_cycles; - u64 vsp_cycles, vpp_cycles; - u32 fps = DEFAULT_FPS; - - width = max(inp_f->fmt.pix_mp.width, inst->crop.width); - height = max(inp_f->fmt.pix_mp.height, inst->crop.height); - - mbpf = NUM_MBS_PER_FRAME(height, width); - mbs_per_second = mbpf * fps; - - fw_cycles = fps * caps->mb_cycles_fw; - fw_vpp_cycles = fps * caps->mb_cycles_fw_vpp; - - vpp_cycles = mult_frac(mbs_per_second, caps->mb_cycles_vpp, (u32)inst->fw_caps[PIPE].value); - /* 21 / 20 is minimum overhead factor */ - vpp_cycles += max(div_u64(vpp_cycles, 20), fw_vpp_cycles); - - /* 1.059 is multi-pipe overhead */ - if (inst->fw_caps[PIPE].value > 1) - vpp_cycles += div_u64(vpp_cycles * 59, 1000); - - vsp_cycles = fps * data_size * 8; - vsp_cycles = div_u64(vsp_cycles, 2); - /* VSP FW overhead 1.05 */ - vsp_cycles = div_u64(vsp_cycles * 21, 20); - - if (inst->fw_caps[STAGE].value == STAGE_1) - vsp_cycles = vsp_cycles * 3; - - return max3(vpp_cycles, vsp_cycles, fw_cycles); -} - const struct vpu_ops iris_vpu3_ops = { .power_off_hw = iris_vpu3_power_off_hardware, .power_on_hw = iris_vpu_power_on_hw, .power_off_controller = iris_vpu_power_off_controller, .power_on_controller = iris_vpu_power_on_controller, - .calc_freq = iris_vpu3x_calculate_frequency, + .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, }; const struct vpu_ops iris_vpu33_ops = { @@ -460,14 +273,14 @@ const struct vpu_ops iris_vpu33_ops = { .power_on_hw = iris_vpu_power_on_hw, .power_off_controller = iris_vpu33_power_off_controller, .power_on_controller = iris_vpu_power_on_controller, - .calc_freq = iris_vpu3x_calculate_frequency, + .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, }; const struct vpu_ops iris_vpu35_ops = { .power_off_hw = iris_vpu35_power_off_hw, .power_on_hw = iris_vpu35_power_on_hw, - .power_off_controller = iris_vpu35_power_off_controller, - .power_on_controller = iris_vpu35_power_on_controller, - .program_bootup_registers = iris_vpu35_program_bootup_registers, - .calc_freq = iris_vpu3x_calculate_frequency, + .power_off_controller = iris_vpu35_vpu4x_power_off_controller, + .power_on_controller = iris_vpu35_vpu4x_power_on_controller, + .program_bootup_registers = iris_vpu35_vpu4x_program_bootup_registers, + .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, }; diff --git a/drivers/media/platform/qcom/iris/iris_vpu4x.c b/drivers/media/platform/qcom/iris/iris_vpu4x.c new file mode 100644 index 0000000000000..a8db02ce5c5ec --- /dev/null +++ b/drivers/media/platform/qcom/iris/iris_vpu4x.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include + +#include "iris_instance.h" +#include "iris_vpu_common.h" +#include "iris_vpu_register_defines.h" + +#define AON_WRAPPER_MVP_NOC_RESET_SYNCRST (AON_MVP_NOC_RESET + 0x08) +#define CPU_CS_APV_BRIDGE_SYNC_RESET (CPU_BASE_OFFS + 0x174) +#define MVP_NOC_RESET_REQ_MASK 0x70103 +#define VPU_IDLE_BITS 0x7103 +#define WRAPPER_EFUSE_MONITOR (WRAPPER_BASE_OFFS + 0x08) + +#define APV_CLK_HALT BIT(1) +#define CORE_CLK_HALT BIT(0) +#define CORE_PWR_ON BIT(1) +#define DISABLE_VIDEO_APV_BIT BIT(27) +#define DISABLE_VIDEO_VPP1_BIT BIT(28) +#define DISABLE_VIDEO_VPP0_BIT BIT(29) + +static int iris_vpu4x_genpd_set_hwmode(struct iris_core *core, bool hw_mode, u32 efuse_value) +{ + int ret; + + ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], hw_mode); + if (ret) + return ret; + + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) { + ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs + [IRIS_VPP0_HW_POWER_DOMAIN], hw_mode); + if (ret) + goto restore_hw_domain_mode; + } + + if (!(efuse_value & DISABLE_VIDEO_VPP1_BIT)) { + ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs + [IRIS_VPP1_HW_POWER_DOMAIN], hw_mode); + if (ret) + goto restore_vpp0_domain_mode; + } + + if (!(efuse_value & DISABLE_VIDEO_APV_BIT)) { + ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs + [IRIS_APV_HW_POWER_DOMAIN], hw_mode); + if (ret) + goto restore_vpp1_domain_mode; + } + + return 0; + +restore_vpp1_domain_mode: + if (!(efuse_value & DISABLE_VIDEO_VPP1_BIT)) + dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_VPP1_HW_POWER_DOMAIN], + !hw_mode); +restore_vpp0_domain_mode: + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) + dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_VPP0_HW_POWER_DOMAIN], + !hw_mode); +restore_hw_domain_mode: + dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], !hw_mode); + + return ret; +} + +static int iris_vpu4x_power_on_apv(struct iris_core *core) +{ + int ret; + + ret = iris_enable_power_domains(core, + core->pmdomain_tbl->pd_devs[IRIS_APV_HW_POWER_DOMAIN]); + if (ret) + return ret; + + ret = iris_prepare_enable_clock(core, IRIS_APV_HW_CLK); + if (ret) + goto disable_apv_hw_power_domain; + + return 0; + +disable_apv_hw_power_domain: + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_APV_HW_POWER_DOMAIN]); + + return ret; +} + +static void iris_vpu4x_power_off_apv(struct iris_core *core) +{ + bool handshake_done, handshake_busy; + u32 value, count = 0; + int ret; + + value = readl(core->reg_base + WRAPPER_CORE_CLOCK_CONFIG); + + if (value & APV_CLK_HALT) + writel(0x0, core->reg_base + WRAPPER_CORE_CLOCK_CONFIG); + + do { + writel(REQ_POWER_DOWN_PREP, core->reg_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); + usleep_range(10, 20); + value = readl(core->reg_base + AON_WRAPPER_MVP_NOC_LPI_STATUS); + + handshake_done = value & NOC_LPI_STATUS_DONE; + handshake_busy = value & (NOC_LPI_STATUS_DENY | NOC_LPI_STATUS_ACTIVE); + + if (handshake_done || !handshake_busy) + break; + + writel(0x0, core->reg_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); + usleep_range(10, 20); + + } while (++count < 1000); + + if (!handshake_done && handshake_busy) + dev_err(core->dev, "LPI handshake timeout\n"); + + writel(0x080200, core->reg_base + AON_WRAPPER_MVP_NOC_RESET_REQ); + ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_NOC_RESET_ACK, + value, value & 0x080200, 200, 2000); + if (ret) + goto disable_clocks_and_power; + + writel(0x0, core->reg_base + AON_WRAPPER_MVP_NOC_RESET_SYNCRST); + writel(0x0, core->reg_base + AON_WRAPPER_MVP_NOC_RESET_REQ); + ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_NOC_RESET_ACK, + value, value == 0x0, 200, 2000); + if (ret) + goto disable_clocks_and_power; + + writel(CORE_BRIDGE_SW_RESET | CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + + CPU_CS_APV_BRIDGE_SYNC_RESET); + writel(CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + CPU_CS_APV_BRIDGE_SYNC_RESET); + writel(0x0, core->reg_base + CPU_CS_APV_BRIDGE_SYNC_RESET); + +disable_clocks_and_power: + iris_disable_unprepare_clock(core, IRIS_APV_HW_CLK); + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_APV_HW_POWER_DOMAIN]); +} + +static void iris_vpu4x_ahb_sync_reset_apv(struct iris_core *core) +{ + writel(CORE_BRIDGE_SW_RESET | CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + + CPU_CS_APV_BRIDGE_SYNC_RESET); + writel(CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + CPU_CS_APV_BRIDGE_SYNC_RESET); + writel(0x0, core->reg_base + CPU_CS_APV_BRIDGE_SYNC_RESET); +} + +static void iris_vpu4x_ahb_sync_reset_hardware(struct iris_core *core) +{ + writel(CORE_BRIDGE_SW_RESET | CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + + CPU_CS_AHB_BRIDGE_SYNC_RESET); + writel(CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + CPU_CS_AHB_BRIDGE_SYNC_RESET); + writel(0x0, core->reg_base + CPU_CS_AHB_BRIDGE_SYNC_RESET); +} + +static int iris_vpu4x_enable_hardware_clocks(struct iris_core *core, u32 efuse_value) +{ + int ret; + + ret = iris_prepare_enable_clock(core, IRIS_AXI_CLK); + if (ret) + return ret; + + ret = iris_prepare_enable_clock(core, IRIS_HW_FREERUN_CLK); + if (ret) + goto disable_axi_clock; + + ret = iris_prepare_enable_clock(core, IRIS_HW_CLK); + if (ret) + goto disable_hw_free_run_clock; + + ret = iris_prepare_enable_clock(core, IRIS_BSE_HW_CLK); + if (ret) + goto disable_hw_clock; + + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) { + ret = iris_prepare_enable_clock(core, IRIS_VPP0_HW_CLK); + if (ret) + goto disable_bse_hw_clock; + } + + if (!(efuse_value & DISABLE_VIDEO_VPP1_BIT)) { + ret = iris_prepare_enable_clock(core, IRIS_VPP1_HW_CLK); + if (ret) + goto disable_vpp0_hw_clock; + } + + return 0; + +disable_vpp0_hw_clock: + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) + iris_disable_unprepare_clock(core, IRIS_VPP0_HW_CLK); +disable_bse_hw_clock: + iris_disable_unprepare_clock(core, IRIS_BSE_HW_CLK); +disable_hw_clock: + iris_disable_unprepare_clock(core, IRIS_HW_CLK); +disable_hw_free_run_clock: + iris_disable_unprepare_clock(core, IRIS_HW_FREERUN_CLK); +disable_axi_clock: + iris_disable_unprepare_clock(core, IRIS_AXI_CLK); + + return ret; +} + +static void iris_vpu4x_disable_hardware_clocks(struct iris_core *core, u32 efuse_value) +{ + if (!(efuse_value & DISABLE_VIDEO_VPP1_BIT)) + iris_disable_unprepare_clock(core, IRIS_VPP1_HW_CLK); + + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) + iris_disable_unprepare_clock(core, IRIS_VPP0_HW_CLK); + + iris_disable_unprepare_clock(core, IRIS_BSE_HW_CLK); + iris_disable_unprepare_clock(core, IRIS_HW_CLK); + iris_disable_unprepare_clock(core, IRIS_HW_FREERUN_CLK); + iris_disable_unprepare_clock(core, IRIS_AXI_CLK); +} + +static int iris_vpu4x_power_on_hardware(struct iris_core *core) +{ + u32 efuse_value = readl(core->reg_base + WRAPPER_EFUSE_MONITOR); + int ret; + + ret = iris_enable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN]); + if (ret) + return ret; + + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) { + ret = iris_enable_power_domains(core, core->pmdomain_tbl->pd_devs + [IRIS_VPP0_HW_POWER_DOMAIN]); + if (ret) + goto disable_hw_power_domain; + } + + if (!(efuse_value & DISABLE_VIDEO_VPP1_BIT)) { + ret = iris_enable_power_domains(core, core->pmdomain_tbl->pd_devs + [IRIS_VPP1_HW_POWER_DOMAIN]); + if (ret) + goto disable_vpp0_power_domain; + } + + ret = iris_vpu4x_enable_hardware_clocks(core, efuse_value); + if (ret) + goto disable_vpp1_power_domain; + + if (!(efuse_value & DISABLE_VIDEO_APV_BIT)) { + ret = iris_vpu4x_power_on_apv(core); + if (ret) + goto disable_hw_clocks; + + iris_vpu4x_ahb_sync_reset_apv(core); + } + + iris_vpu4x_ahb_sync_reset_hardware(core); + + ret = iris_vpu4x_genpd_set_hwmode(core, true, efuse_value); + if (ret) + goto disable_apv_power_domain; + + return 0; + +disable_apv_power_domain: + if (!(efuse_value & DISABLE_VIDEO_APV_BIT)) + iris_vpu4x_power_off_apv(core); +disable_hw_clocks: + iris_vpu4x_disable_hardware_clocks(core, efuse_value); +disable_vpp1_power_domain: + if (!(efuse_value & DISABLE_VIDEO_VPP1_BIT)) + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs + [IRIS_VPP1_HW_POWER_DOMAIN]); +disable_vpp0_power_domain: + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs + [IRIS_VPP0_HW_POWER_DOMAIN]); +disable_hw_power_domain: + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN]); + + return ret; +} + +static void iris_vpu4x_power_off_hardware(struct iris_core *core) +{ + u32 efuse_value = readl(core->reg_base + WRAPPER_EFUSE_MONITOR); + bool handshake_done, handshake_busy; + u32 value, count = 0; + int ret; + + iris_vpu4x_genpd_set_hwmode(core, false, efuse_value); + + if (!(efuse_value & DISABLE_VIDEO_APV_BIT)) + iris_vpu4x_power_off_apv(core); + + value = readl(core->reg_base + WRAPPER_CORE_POWER_STATUS); + + if (!(value & CORE_PWR_ON)) + goto disable_clocks_and_power; + + value = readl(core->reg_base + WRAPPER_CORE_CLOCK_CONFIG); + + if (value & CORE_CLK_HALT) + writel(0x0, core->reg_base + WRAPPER_CORE_CLOCK_CONFIG); + + readl_poll_timeout(core->reg_base + VCODEC_SS_IDLE_STATUSN, value, + value & VPU_IDLE_BITS, 2000, 20000); + + do { + writel(REQ_POWER_DOWN_PREP, core->reg_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); + usleep_range(10, 20); + value = readl(core->reg_base + AON_WRAPPER_MVP_NOC_LPI_STATUS); + + handshake_done = value & NOC_LPI_STATUS_DONE; + handshake_busy = value & (NOC_LPI_STATUS_DENY | NOC_LPI_STATUS_ACTIVE); + + if (handshake_done || !handshake_busy) + break; + + writel(0x0, core->reg_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); + usleep_range(10, 20); + + } while (++count < 1000); + + if (!handshake_done && handshake_busy) + dev_err(core->dev, "LPI handshake timeout\n"); + + writel(MVP_NOC_RESET_REQ_MASK, core->reg_base + AON_WRAPPER_MVP_NOC_RESET_REQ); + ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_NOC_RESET_ACK, + value, value & MVP_NOC_RESET_REQ_MASK, 200, 2000); + if (ret) + goto disable_clocks_and_power; + + writel(0x0, core->reg_base + AON_WRAPPER_MVP_NOC_RESET_SYNCRST); + writel(0x0, core->reg_base + AON_WRAPPER_MVP_NOC_RESET_REQ); + ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_NOC_RESET_ACK, + value, value == 0x0, 200, 2000); + if (ret) + goto disable_clocks_and_power; + + writel(CORE_BRIDGE_SW_RESET | CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + + CPU_CS_AHB_BRIDGE_SYNC_RESET); + writel(CORE_BRIDGE_HW_RESET_DISABLE, core->reg_base + CPU_CS_AHB_BRIDGE_SYNC_RESET); + writel(0x0, core->reg_base + CPU_CS_AHB_BRIDGE_SYNC_RESET); + +disable_clocks_and_power: + iris_vpu4x_disable_hardware_clocks(core, efuse_value); + + if (!(efuse_value & DISABLE_VIDEO_VPP1_BIT)) + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs + [IRIS_VPP1_HW_POWER_DOMAIN]); + + if (!(efuse_value & DISABLE_VIDEO_VPP0_BIT)) + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs + [IRIS_VPP0_HW_POWER_DOMAIN]); + + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN]); +} + +const struct vpu_ops iris_vpu4x_ops = { + .power_off_hw = iris_vpu4x_power_off_hardware, + .power_on_hw = iris_vpu4x_power_on_hardware, + .power_off_controller = iris_vpu35_vpu4x_power_off_controller, + .power_on_controller = iris_vpu35_vpu4x_power_on_controller, + .program_bootup_registers = iris_vpu35_vpu4x_program_bootup_registers, + .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, +}; diff --git a/drivers/media/platform/qcom/iris/iris_vpu_buffer.c b/drivers/media/platform/qcom/iris/iris_vpu_buffer.c index 4463be05ce165..9270422c16019 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_buffer.c +++ b/drivers/media/platform/qcom/iris/iris_vpu_buffer.c @@ -9,6 +9,17 @@ #include "iris_hfi_gen2_defines.h" #define HFI_MAX_COL_FRAME 6 +#define HFI_COLOR_FORMAT_YUV420_NV12_UBWC_Y_TILE_HEIGHT (8) +#define HFI_COLOR_FORMAT_YUV420_NV12_UBWC_Y_TILE_WIDTH (32) +#define HFI_COLOR_FORMAT_YUV420_NV12_UBWC_UV_TILE_HEIGHT (8) +#define HFI_COLOR_FORMAT_YUV420_NV12_UBWC_UV_TILE_WIDTH (16) +#define HFI_COLOR_FORMAT_YUV420_TP10_UBWC_Y_TILE_HEIGHT (4) +#define HFI_COLOR_FORMAT_YUV420_TP10_UBWC_Y_TILE_WIDTH (48) +#define HFI_COLOR_FORMAT_YUV420_TP10_UBWC_UV_TILE_HEIGHT (4) +#define HFI_COLOR_FORMAT_YUV420_TP10_UBWC_UV_TILE_WIDTH (24) +#define AV1D_SIZE_BSE_COL_MV_64x64 512 +#define AV1D_SIZE_BSE_COL_MV_128x128 2816 +#define UBWC_TILE_SIZE 256 #ifndef SYSTEM_LAL_TILE10 #define SYSTEM_LAL_TILE10 192 @@ -39,6 +50,31 @@ static u32 hfi_buffer_bin_h264d(u32 frame_width, u32 frame_height, u32 num_vpp_p return size_h264d_hw_bin_buffer(n_aligned_w, n_aligned_h, num_vpp_pipes); } +static u32 size_av1d_hw_bin_buffer(u32 frame_width, u32 frame_height, u32 num_vpp_pipes) +{ + u32 size_yuv, size_bin_hdr, size_bin_res; + + size_yuv = ((frame_width * frame_height) <= BIN_BUFFER_THRESHOLD) ? + ((BIN_BUFFER_THRESHOLD * 3) >> 1) : + ((frame_width * frame_height * 3) >> 1); + size_bin_hdr = size_yuv * AV1_CABAC_HDR_RATIO_HD_TOT; + size_bin_res = size_yuv * AV1_CABAC_RES_RATIO_HD_TOT; + size_bin_hdr = ALIGN(size_bin_hdr / num_vpp_pipes, + DMA_ALIGNMENT) * num_vpp_pipes; + size_bin_res = ALIGN(size_bin_res / num_vpp_pipes, + DMA_ALIGNMENT) * num_vpp_pipes; + + return size_bin_hdr + size_bin_res; +} + +static u32 hfi_buffer_bin_av1d(u32 frame_width, u32 frame_height, u32 num_vpp_pipes) +{ + u32 n_aligned_h = ALIGN(frame_height, 16); + u32 n_aligned_w = ALIGN(frame_width, 16); + + return size_av1d_hw_bin_buffer(n_aligned_w, n_aligned_h, num_vpp_pipes); +} + static u32 size_h265d_hw_bin_buffer(u32 frame_width, u32 frame_height, u32 num_vpp_pipes) { u32 product = frame_width * frame_height; @@ -110,6 +146,26 @@ static u32 hfi_buffer_comv_h265d(u32 frame_width, u32 frame_height, u32 _comv_bu return (_size * (_comv_bufcount)) + 512; } +static u32 num_lcu(u32 frame_width, u32 frame_height, u32 lcu_size) +{ + return ((frame_width + lcu_size - 1) / lcu_size) * + ((frame_height + lcu_size - 1) / lcu_size); +} + +static u32 hfi_buffer_comv_av1d(u32 frame_width, u32 frame_height, u32 comv_bufcount) +{ + u32 size; + + size = 2 * ALIGN(max(num_lcu(frame_width, frame_height, 64) * + AV1D_SIZE_BSE_COL_MV_64x64, + num_lcu(frame_width, frame_height, 128) * + AV1D_SIZE_BSE_COL_MV_128x128), + DMA_ALIGNMENT); + size *= comv_bufcount; + + return size; +} + static u32 size_h264d_bse_cmd_buf(u32 frame_height) { u32 height = ALIGN(frame_height, 32); @@ -122,7 +178,7 @@ static u32 size_h265d_bse_cmd_buf(u32 frame_width, u32 frame_height) { u32 _size = ALIGN(((ALIGN(frame_width, LCU_MAX_SIZE_PELS) / LCU_MIN_SIZE_PELS) * (ALIGN(frame_height, LCU_MAX_SIZE_PELS) / LCU_MIN_SIZE_PELS)) * - NUM_HW_PIC_BUF, DMA_ALIGNMENT); + NUM_HW_PIC_BUF, DMA_ALIGNMENT); _size = min_t(u32, _size, H265D_MAX_SLICE + 1); _size = 2 * _size * SIZE_H265D_BSE_CMD_PER_BUF; @@ -174,6 +230,20 @@ static u32 hfi_buffer_persist_h264d(void) DMA_ALIGNMENT); } +static u32 hfi_buffer_persist_av1d(u32 max_width, u32 max_height, u32 total_ref_count) +{ + u32 comv_size, size; + + comv_size = hfi_buffer_comv_av1d(max_width, max_height, total_ref_count); + size = ALIGN((SIZE_AV1D_SEQUENCE_HEADER * 2 + SIZE_AV1D_METADATA + + AV1D_NUM_HW_PIC_BUF * (SIZE_AV1D_TILE_OFFSET + SIZE_AV1D_QM) + + AV1D_NUM_FRAME_HEADERS * (SIZE_AV1D_FRAME_HEADER + + 2 * SIZE_AV1D_PROB_TABLE) + comv_size + HDR10_HIST_EXTRADATA_SIZE + + SIZE_AV1D_METADATA * AV1D_NUM_HW_PIC_BUF), DMA_ALIGNMENT); + + return ALIGN(size, DMA_ALIGNMENT); +} + static u32 hfi_buffer_non_comv_h264d(u32 frame_width, u32 frame_height, u32 num_vpp_pipes) { u32 size_bse = size_h264d_bse_cmd_buf(frame_height); @@ -459,6 +529,182 @@ static u32 hfi_buffer_line_h264d(u32 frame_width, u32 frame_height, return ALIGN((size + vpss_lb_size), DMA_ALIGNMENT); } +static u32 size_av1d_lb_opb_wr1_nv12_ubwc(u32 frame_width, u32 frame_height) +{ + u32 size, y_width, y_width_a = 128; + + y_width = ALIGN(frame_width, y_width_a); + + size = ((y_width + HFI_COLOR_FORMAT_YUV420_NV12_UBWC_Y_TILE_WIDTH - 1) / + HFI_COLOR_FORMAT_YUV420_NV12_UBWC_Y_TILE_WIDTH + + (AV1D_MAX_TILE_COLS - 1)); + return size * UBWC_TILE_SIZE; +} + +static u32 size_av1d_lb_opb_wr1_tp10_ubwc(u32 frame_width, u32 frame_height) +{ + u32 size, y_width, y_width_a = 256; + + y_width = ALIGN(frame_width, y_width_a); + + size = ((y_width + HFI_COLOR_FORMAT_YUV420_TP10_UBWC_Y_TILE_WIDTH - 1) / + HFI_COLOR_FORMAT_YUV420_TP10_UBWC_Y_TILE_WIDTH + + (AV1D_MAX_TILE_COLS - 1)); + + return size * UBWC_TILE_SIZE; +} + +static u32 hfi_buffer_line_av1d(u32 frame_width, u32 frame_height, + bool is_opb, u32 num_vpp_pipes) +{ + u32 size, vpss_lb_size, opbwrbufsize, opbwr8, opbwr10; + + size = ALIGN(size_av1d_lb_fe_top_data(frame_width, frame_height), + DMA_ALIGNMENT) + + ALIGN(size_av1d_lb_fe_top_ctrl(frame_width, frame_height), + DMA_ALIGNMENT) + + ALIGN(size_av1d_lb_fe_left_data(frame_width, frame_height), + DMA_ALIGNMENT) * num_vpp_pipes + + ALIGN(size_av1d_lb_fe_left_ctrl(frame_width, frame_height), + DMA_ALIGNMENT) * num_vpp_pipes + + ALIGN(size_av1d_lb_se_left_ctrl(frame_width, frame_height), + DMA_ALIGNMENT) * num_vpp_pipes + + ALIGN(size_av1d_lb_se_top_ctrl(frame_width, frame_height), + DMA_ALIGNMENT) + + ALIGN(size_av1d_lb_pe_top_data(frame_width, frame_height), + DMA_ALIGNMENT) + + ALIGN(size_av1d_lb_vsp_top(frame_width, frame_height), + DMA_ALIGNMENT) + + ALIGN(size_av1d_lb_recon_dma_metadata_wr + (frame_width, frame_height), DMA_ALIGNMENT) * 2 + + ALIGN(size_av1d_qp(frame_width, frame_height), DMA_ALIGNMENT); + opbwr8 = size_av1d_lb_opb_wr1_nv12_ubwc(frame_width, frame_height); + opbwr10 = size_av1d_lb_opb_wr1_tp10_ubwc(frame_width, frame_height); + opbwrbufsize = opbwr8 >= opbwr10 ? opbwr8 : opbwr10; + size = ALIGN((size + opbwrbufsize), DMA_ALIGNMENT); + if (is_opb) { + vpss_lb_size = size_vpss_lb(frame_width, frame_height); + size = ALIGN((size + vpss_lb_size) * 2, DMA_ALIGNMENT); + } + + return size; +} + +static u32 size_av1d_ibc_nv12_ubwc(u32 frame_width, u32 frame_height) +{ + u32 size; + u32 y_width_a = 128, y_height_a = 32; + u32 uv_width_a = 128, uv_height_a = 32; + u32 ybufsize, uvbufsize, y_width, y_height, uv_width, uv_height; + u32 y_meta_width_a = 64, y_meta_height_a = 16; + u32 uv_meta_width_a = 64, uv_meta_height_a = 16; + u32 meta_height, meta_stride, meta_size; + u32 tile_width_y = HFI_COLOR_FORMAT_YUV420_NV12_UBWC_Y_TILE_WIDTH; + u32 tile_height_y = HFI_COLOR_FORMAT_YUV420_NV12_UBWC_Y_TILE_HEIGHT; + u32 tile_width_uv = HFI_COLOR_FORMAT_YUV420_NV12_UBWC_UV_TILE_WIDTH; + u32 tile_height_uv = HFI_COLOR_FORMAT_YUV420_NV12_UBWC_UV_TILE_HEIGHT; + + y_width = ALIGN(frame_width, y_width_a); + y_height = ALIGN(frame_height, y_height_a); + uv_width = ALIGN(frame_width, uv_width_a); + uv_height = ALIGN(((frame_height + 1) >> 1), uv_height_a); + ybufsize = ALIGN((y_width * y_height), HFI_ALIGNMENT_4096); + uvbufsize = ALIGN(uv_width * uv_height, HFI_ALIGNMENT_4096); + size = ybufsize + uvbufsize; + meta_stride = ALIGN(((frame_width + (tile_width_y - 1)) / tile_width_y), + y_meta_width_a); + meta_height = ALIGN(((frame_height + (tile_height_y - 1)) / tile_height_y), + y_meta_height_a); + meta_size = ALIGN(meta_stride * meta_height, HFI_ALIGNMENT_4096); + size += meta_size; + meta_stride = ALIGN(((((frame_width + 1) >> 1) + (tile_width_uv - 1)) / + tile_width_uv), uv_meta_width_a); + meta_height = ALIGN(((((frame_height + 1) >> 1) + (tile_height_uv - 1)) / + tile_height_uv), uv_meta_height_a); + meta_size = ALIGN(meta_stride * meta_height, HFI_ALIGNMENT_4096); + size += meta_size; + + return size; +} + +static u32 hfi_yuv420_tp10_calc_y_stride(u32 frame_width, u32 stride_multiple) +{ + u32 stride; + + stride = ALIGN(frame_width, 192); + stride = ALIGN(stride * 4 / 3, stride_multiple); + + return stride; +} + +static u32 hfi_yuv420_tp10_calc_y_bufheight(u32 frame_height, u32 min_buf_height_multiple) +{ + return ALIGN(frame_height, min_buf_height_multiple); +} + +static u32 hfi_yuv420_tp10_calc_uv_stride(u32 frame_width, u32 stride_multiple) +{ + u32 stride; + + stride = ALIGN(frame_width, 192); + stride = ALIGN(stride * 4 / 3, stride_multiple); + + return stride; +} + +static u32 hfi_yuv420_tp10_calc_uv_bufheight(u32 frame_height, u32 min_buf_height_multiple) +{ + return ALIGN(((frame_height + 1) >> 1), min_buf_height_multiple); +} + +static u32 size_av1d_ibc_tp10_ubwc(u32 frame_width, u32 frame_height) +{ + u32 size; + u32 y_width_a = 256, y_height_a = 16, + uv_width_a = 256, uv_height_a = 16; + u32 ybufsize, uvbufsize, y_width, y_height, uv_width, uv_height; + u32 y_meta_width_a = 64, y_meta_height_a = 16, + uv_meta_width_a = 64, uv_meta_height_a = 16; + u32 meta_height, meta_stride, meta_size; + u32 tile_width_y = HFI_COLOR_FORMAT_YUV420_TP10_UBWC_Y_TILE_WIDTH; + u32 tile_height_y = HFI_COLOR_FORMAT_YUV420_TP10_UBWC_Y_TILE_HEIGHT; + u32 tile_width_uv = HFI_COLOR_FORMAT_YUV420_TP10_UBWC_UV_TILE_WIDTH; + u32 tile_height_uv = HFI_COLOR_FORMAT_YUV420_TP10_UBWC_UV_TILE_HEIGHT; + + y_width = hfi_yuv420_tp10_calc_y_stride(frame_width, y_width_a); + y_height = hfi_yuv420_tp10_calc_y_bufheight(frame_height, y_height_a); + uv_width = hfi_yuv420_tp10_calc_uv_stride(frame_width, uv_width_a); + uv_height = hfi_yuv420_tp10_calc_uv_bufheight(frame_height, uv_height_a); + ybufsize = ALIGN(y_width * y_height, HFI_ALIGNMENT_4096); + uvbufsize = ALIGN(uv_width * uv_height, HFI_ALIGNMENT_4096); + size = ybufsize + uvbufsize; + meta_stride = ALIGN(((frame_width + (tile_width_y - 1)) / tile_width_y), + y_meta_width_a); + meta_height = ALIGN(((frame_height + (tile_height_y - 1)) / tile_height_y), + y_meta_height_a); + meta_size = ALIGN(meta_stride * meta_height, HFI_ALIGNMENT_4096); + size += meta_size; + meta_stride = ALIGN(((((frame_width + 1) >> 1) + (tile_width_uv - 1)) / + tile_width_uv), uv_meta_width_a); + meta_height = ALIGN(((((frame_height + 1) >> 1) + (tile_height_uv - 1)) / + tile_height_uv), uv_meta_height_a); + meta_size = ALIGN(meta_stride * meta_height, HFI_ALIGNMENT_4096); + size += meta_size; + + return size; +} + +static u32 hfi_buffer_ibc_av1d(u32 frame_width, u32 frame_height) +{ + u32 size, ibc8, ibc10; + + ibc8 = size_av1d_ibc_nv12_ubwc(frame_width, frame_height); + ibc10 = size_av1d_ibc_tp10_ubwc(frame_width, frame_height); + size = ibc8 >= ibc10 ? ibc8 : ibc10; + + return ALIGN(size, DMA_ALIGNMENT); +} + static u32 iris_vpu_dec_bin_size(struct iris_inst *inst) { u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; @@ -472,6 +718,8 @@ static u32 iris_vpu_dec_bin_size(struct iris_inst *inst) return hfi_buffer_bin_h265d(width, height, num_vpp_pipes); else if (inst->codec == V4L2_PIX_FMT_VP9) return hfi_buffer_bin_vp9d(width, height, num_vpp_pipes); + else if (inst->codec == V4L2_PIX_FMT_AV1) + return hfi_buffer_bin_av1d(width, height, num_vpp_pipes); return 0; } @@ -487,18 +735,34 @@ static u32 iris_vpu_dec_comv_size(struct iris_inst *inst) return hfi_buffer_comv_h264d(width, height, num_comv); else if (inst->codec == V4L2_PIX_FMT_HEVC) return hfi_buffer_comv_h265d(width, height, num_comv); + else if (inst->codec == V4L2_PIX_FMT_AV1) { + if (inst->fw_caps[DRAP].value) + return 0; + else + return hfi_buffer_comv_av1d(width, height, num_comv); + } return 0; } static u32 iris_vpu_dec_persist_size(struct iris_inst *inst) { + struct platform_inst_caps *caps; + if (inst->codec == V4L2_PIX_FMT_H264) return hfi_buffer_persist_h264d(); else if (inst->codec == V4L2_PIX_FMT_HEVC) return hfi_buffer_persist_h265d(0); else if (inst->codec == V4L2_PIX_FMT_VP9) return hfi_buffer_persist_vp9d(); + else if (inst->codec == V4L2_PIX_FMT_AV1) { + caps = inst->core->iris_platform_data->inst_caps; + if (inst->fw_caps[DRAP].value) + return hfi_buffer_persist_av1d(caps->max_frame_width, + caps->max_frame_height, 16); + else + return hfi_buffer_persist_av1d(0, 0, 0); + } return 0; } @@ -545,6 +809,8 @@ static u32 iris_vpu_dec_line_size(struct iris_inst *inst) else if (inst->codec == V4L2_PIX_FMT_VP9) return hfi_buffer_line_vp9d(width, height, out_min_count, is_opb, num_vpp_pipes); + else if (inst->codec == V4L2_PIX_FMT_AV1) + return hfi_buffer_line_av1d(width, height, is_opb, num_vpp_pipes); return 0; } @@ -556,6 +822,22 @@ static u32 iris_vpu_dec_scratch1_size(struct iris_inst *inst) iris_vpu_dec_line_size(inst); } +static inline u32 iris_vpu_enc_get_bitstream_width(struct iris_inst *inst) +{ + if (is_rotation_90_or_270(inst)) + return inst->fmt_dst->fmt.pix_mp.height; + else + return inst->fmt_dst->fmt.pix_mp.width; +} + +static inline u32 iris_vpu_enc_get_bitstream_height(struct iris_inst *inst) +{ + if (is_rotation_90_or_270(inst)) + return inst->fmt_dst->fmt.pix_mp.width; + else + return inst->fmt_dst->fmt.pix_mp.height; +} + static inline u32 size_bin_bitstream_enc(u32 width, u32 height, u32 rc_type) { @@ -638,10 +920,9 @@ static inline u32 hfi_buffer_bin_enc(u32 width, u32 height, static u32 iris_vpu_enc_bin_size(struct iris_inst *inst) { u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; + u32 height = iris_vpu_enc_get_bitstream_height(inst); + u32 width = iris_vpu_enc_get_bitstream_width(inst); u32 stage = inst->fw_caps[STAGE].value; - struct v4l2_format *f = inst->fmt_dst; - u32 height = f->fmt.pix_mp.height; - u32 width = f->fmt.pix_mp.width; u32 lcu_size; if (inst->codec == V4L2_PIX_FMT_HEVC) @@ -653,6 +934,15 @@ static u32 iris_vpu_enc_bin_size(struct iris_inst *inst) num_vpp_pipes, inst->hfi_rc_type); } +static u32 iris_vpu_dec_partial_size(struct iris_inst *inst) +{ + struct v4l2_format *f = inst->fmt_src; + u32 height = f->fmt.pix_mp.height; + u32 width = f->fmt.pix_mp.width; + + return hfi_buffer_ibc_av1d(width, height); +} + static inline u32 hfi_buffer_comv_enc(u32 frame_width, u32 frame_height, u32 lcu_size, u32 num_recon, u32 standard) @@ -676,9 +966,8 @@ u32 hfi_buffer_comv_enc(u32 frame_width, u32 frame_height, u32 lcu_size, static u32 iris_vpu_enc_comv_size(struct iris_inst *inst) { - struct v4l2_format *f = inst->fmt_dst; - u32 height = f->fmt.pix_mp.height; - u32 width = f->fmt.pix_mp.width; + u32 height = iris_vpu_enc_get_bitstream_height(inst); + u32 width = iris_vpu_enc_get_bitstream_width(inst); u32 num_recon = 1; u32 lcu_size = 16; @@ -958,9 +1247,8 @@ u32 hfi_buffer_non_comv_enc(u32 frame_width, u32 frame_height, static u32 iris_vpu_enc_non_comv_size(struct iris_inst *inst) { u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; - struct v4l2_format *f = inst->fmt_dst; - u32 height = f->fmt.pix_mp.height; - u32 width = f->fmt.pix_mp.width; + u32 height = iris_vpu_enc_get_bitstream_height(inst); + u32 width = iris_vpu_enc_get_bitstream_width(inst); u32 lcu_size = 16; if (inst->codec == V4L2_PIX_FMT_HEVC) { @@ -1051,9 +1339,8 @@ u32 hfi_buffer_line_enc_vpu33(u32 frame_width, u32 frame_height, bool is_ten_bit static u32 iris_vpu_enc_line_size(struct iris_inst *inst) { u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; - struct v4l2_format *f = inst->fmt_dst; - u32 height = f->fmt.pix_mp.height; - u32 width = f->fmt.pix_mp.width; + u32 height = iris_vpu_enc_get_bitstream_height(inst); + u32 width = iris_vpu_enc_get_bitstream_width(inst); u32 lcu_size = 16; if (inst->codec == V4L2_PIX_FMT_HEVC) { @@ -1069,9 +1356,8 @@ static u32 iris_vpu_enc_line_size(struct iris_inst *inst) static u32 iris_vpu33_enc_line_size(struct iris_inst *inst) { u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; - struct v4l2_format *f = inst->fmt_dst; - u32 height = f->fmt.pix_mp.height; - u32 width = f->fmt.pix_mp.width; + u32 height = iris_vpu_enc_get_bitstream_height(inst); + u32 width = iris_vpu_enc_get_bitstream_width(inst); u32 lcu_size = 16; if (inst->codec == V4L2_PIX_FMT_HEVC) { @@ -1131,10 +1417,11 @@ static u32 iris_vpu_enc_arp_size(struct iris_inst *inst) inline bool is_scaling_enabled(struct iris_inst *inst) { - return inst->crop.left != inst->compose.left || - inst->crop.top != inst->compose.top || - inst->crop.width != inst->compose.width || - inst->crop.height != inst->compose.height; + struct v4l2_pix_format_mplane *dst_fmt = &inst->fmt_dst->fmt.pix_mp; + struct v4l2_pix_format_mplane *src_fmt = &inst->fmt_src->fmt.pix_mp; + + return dst_fmt->width != src_fmt->width || + dst_fmt->height != src_fmt->height; } static inline @@ -1291,9 +1578,8 @@ static inline u32 hfi_buffer_scratch1_enc(u32 frame_width, u32 frame_height, static u32 iris_vpu_enc_scratch1_size(struct iris_inst *inst) { u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; - struct v4l2_format *f = inst->fmt_dst; - u32 frame_height = f->fmt.pix_mp.height; - u32 frame_width = f->fmt.pix_mp.width; + u32 frame_height = iris_vpu_enc_get_bitstream_height(inst); + u32 frame_width = iris_vpu_enc_get_bitstream_width(inst); u32 num_ref = 1; u32 lcu_size; bool is_h265; @@ -1389,9 +1675,8 @@ static inline u32 hfi_buffer_scratch2_enc(u32 frame_width, u32 frame_height, static u32 iris_vpu_enc_scratch2_size(struct iris_inst *inst) { - struct v4l2_format *f = inst->fmt_dst; - u32 frame_width = f->fmt.pix_mp.width; - u32 frame_height = f->fmt.pix_mp.height; + u32 frame_height = iris_vpu_enc_get_bitstream_height(inst); + u32 frame_width = iris_vpu_enc_get_bitstream_width(inst); u32 num_ref = 1; return hfi_buffer_scratch2_enc(frame_width, frame_height, num_ref, @@ -1408,13 +1693,313 @@ static u32 iris_vpu_enc_vpss_size(struct iris_inst *inst) return hfi_buffer_vpss_enc(width, height, ds_enable, 0, 0); } +static inline u32 size_dpb_opb(u32 height, u32 lcu_size) +{ + u32 max_tile_height = ((height + lcu_size - 1) / lcu_size) * lcu_size + 8; + u32 dpb_opb = 3 * ((max_tile_height >> 3) * DMA_ALIGNMENT); + u32 num_luma_chrome_plane = 2; + + return ALIGN(dpb_opb, DMA_ALIGNMENT) * num_luma_chrome_plane; +} + +static u32 hfi_vpu4x_vp9d_lb_size(u32 frame_width, u32 frame_height, u32 num_vpp_pipes) +{ + u32 vp9_top_lb, vp9_fe_left_lb, vp9_se_left_lb, dpb_opb, vp9d_qp, num_lcu_per_pipe; + u32 lcu_size = 64; + + vp9_top_lb = ALIGN(size_vp9d_lb_vsp_top(frame_width, frame_height), DMA_ALIGNMENT); + vp9_top_lb += ALIGN(size_vpxd_lb_se_top_ctrl(frame_width, frame_height), DMA_ALIGNMENT); + vp9_top_lb += max3(DIV_ROUND_UP(frame_width, BUFFER_ALIGNMENT_16_BYTES) * + MAX_PE_NBR_DATA_LCU16_LINE_BUFFER_SIZE, + DIV_ROUND_UP(frame_width, BUFFER_ALIGNMENT_32_BYTES) * + MAX_PE_NBR_DATA_LCU32_LINE_BUFFER_SIZE, + DIV_ROUND_UP(frame_width, BUFFER_ALIGNMENT_64_BYTES) * + MAX_PE_NBR_DATA_LCU64_LINE_BUFFER_SIZE); + vp9_top_lb = ALIGN(vp9_top_lb, DMA_ALIGNMENT); + vp9_top_lb += ALIGN((DMA_ALIGNMENT * DIV_ROUND_UP(frame_width, lcu_size)), + DMA_ALIGNMENT) * FE_TOP_CTRL_LINE_NUMBERS; + vp9_top_lb += ALIGN(DMA_ALIGNMENT * 8 * DIV_ROUND_UP(frame_width, lcu_size), + DMA_ALIGNMENT) * (FE_TOP_DATA_LUMA_LINE_NUMBERS + + FE_TOP_DATA_CHROMA_LINE_NUMBERS); + + num_lcu_per_pipe = (DIV_ROUND_UP(frame_height, lcu_size) / num_vpp_pipes) + + (DIV_ROUND_UP(frame_height, lcu_size) % num_vpp_pipes); + vp9_fe_left_lb = ALIGN((DMA_ALIGNMENT * num_lcu_per_pipe), DMA_ALIGNMENT) * + FE_LFT_CTRL_LINE_NUMBERS; + vp9_fe_left_lb += ((ALIGN((DMA_ALIGNMENT * 8 * num_lcu_per_pipe), DMA_ALIGNMENT) * + FE_LFT_DB_DATA_LINE_NUMBERS) + + ALIGN((DMA_ALIGNMENT * 3 * num_lcu_per_pipe), DMA_ALIGNMENT) + + ALIGN((DMA_ALIGNMENT * 4 * num_lcu_per_pipe), DMA_ALIGNMENT) + + (ALIGN((DMA_ALIGNMENT * 24 * num_lcu_per_pipe), DMA_ALIGNMENT) * + FE_LFT_LR_DATA_LINE_NUMBERS)); + vp9_fe_left_lb = vp9_fe_left_lb * num_vpp_pipes; + + vp9_se_left_lb = ALIGN(size_vpxd_lb_se_left_ctrl(frame_width, frame_height), + DMA_ALIGNMENT); + dpb_opb = size_dpb_opb(frame_height, lcu_size); + vp9d_qp = ALIGN(size_vp9d_qp(frame_width, frame_height), DMA_ALIGNMENT); + + return vp9_top_lb + vp9_fe_left_lb + (vp9_se_left_lb * num_vpp_pipes) + + (dpb_opb * num_vpp_pipes) + vp9d_qp; +} + +static u32 hfi_vpu4x_buffer_line_vp9d(u32 frame_width, u32 frame_height, u32 _yuv_bufcount_min, + bool is_opb, u32 num_vpp_pipes) +{ + u32 lb_size = hfi_vpu4x_vp9d_lb_size(frame_width, frame_height, num_vpp_pipes); + u32 dpb_obp_size = 0, lcu_size = 64; + + if (is_opb) + dpb_obp_size = size_dpb_opb(frame_height, lcu_size) * num_vpp_pipes; + + return lb_size + dpb_obp_size; +} + +static u32 iris_vpu4x_dec_line_size(struct iris_inst *inst) +{ + u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; + u32 out_min_count = inst->buffers[BUF_OUTPUT].min_count; + struct v4l2_format *f = inst->fmt_src; + u32 height = f->fmt.pix_mp.height; + u32 width = f->fmt.pix_mp.width; + bool is_opb = false; + + if (iris_split_mode_enabled(inst)) + is_opb = true; + + if (inst->codec == V4L2_PIX_FMT_H264) + return hfi_buffer_line_h264d(width, height, is_opb, num_vpp_pipes); + else if (inst->codec == V4L2_PIX_FMT_HEVC) + return hfi_buffer_line_h265d(width, height, is_opb, num_vpp_pipes); + else if (inst->codec == V4L2_PIX_FMT_VP9) + return hfi_vpu4x_buffer_line_vp9d(width, height, out_min_count, is_opb, + num_vpp_pipes); + + return 0; +} + +static u32 hfi_vpu4x_buffer_persist_h265d(u32 rpu_enabled) +{ + return ALIGN((SIZE_SLIST_BUF_H265 * NUM_SLIST_BUF_H265 + H265_NUM_FRM_INFO * + H265_DISPLAY_BUF_SIZE + (H265_NUM_TILE * sizeof(u32)) + (NUM_HW_PIC_BUF * + (SIZE_SEI_USERDATA + SIZE_H265D_ARP + SIZE_THREE_DIMENSION_USERDATA)) + + rpu_enabled * NUM_HW_PIC_BUF * SIZE_DOLBY_RPU_METADATA), DMA_ALIGNMENT); +} + +static u32 hfi_vpu4x_buffer_persist_vp9d(void) +{ + return ALIGN(VP9_NUM_PROBABILITY_TABLE_BUF * VP9_PROB_TABLE_SIZE, DMA_ALIGNMENT) + + (ALIGN(hfi_iris3_vp9d_comv_size(), DMA_ALIGNMENT) * 2) + + ALIGN(MAX_SUPERFRAME_HEADER_LEN, DMA_ALIGNMENT) + + ALIGN(VP9_UDC_HEADER_BUF_SIZE, DMA_ALIGNMENT) + + ALIGN(VP9_NUM_FRAME_INFO_BUF * CCE_TILE_OFFSET_SIZE, DMA_ALIGNMENT) + + ALIGN(VP9_NUM_FRAME_INFO_BUF * VP9_FRAME_INFO_BUF_SIZE_VPU4X, DMA_ALIGNMENT) + + HDR10_HIST_EXTRADATA_SIZE; +} + +static u32 iris_vpu4x_dec_persist_size(struct iris_inst *inst) +{ + if (inst->codec == V4L2_PIX_FMT_H264) + return hfi_buffer_persist_h264d(); + else if (inst->codec == V4L2_PIX_FMT_HEVC) + return hfi_vpu4x_buffer_persist_h265d(0); + else if (inst->codec == V4L2_PIX_FMT_VP9) + return hfi_vpu4x_buffer_persist_vp9d(); + + return 0; +} + +static u32 size_se_lb(u32 standard, u32 num_vpp_pipes_enc, + u32 frame_width_coded, u32 frame_height_coded) +{ + u32 se_tlb_size = ALIGN(frame_width_coded, DMA_ALIGNMENT); + u32 se_llb_size = (standard == HFI_CODEC_ENCODE_HEVC) ? + ((frame_height_coded + BUFFER_ALIGNMENT_32_BYTES - 1) / + BUFFER_ALIGNMENT_32_BYTES) * LOG2_16 * LLB_UNIT_SIZE : + ((frame_height_coded + BUFFER_ALIGNMENT_16_BYTES - 1) / + BUFFER_ALIGNMENT_16_BYTES) * LOG2_32 * LLB_UNIT_SIZE; + + se_llb_size = ALIGN(se_llb_size, BUFFER_ALIGNMENT_32_BYTES); + + if (num_vpp_pipes_enc > 1) + se_llb_size = ALIGN(se_llb_size + BUFFER_ALIGNMENT_512_BYTES, + DMA_ALIGNMENT) * num_vpp_pipes_enc; + + return ALIGN(se_tlb_size + se_llb_size, DMA_ALIGNMENT); +} + +static u32 size_te_lb(bool is_ten_bit, u32 num_vpp_pipes_enc, u32 width_in_lcus, + u32 frame_height_coded, u32 frame_width_coded) +{ + u32 num_pixel_10_bit = 3, num_pixel_8_bit = 2, num_pixel_te_llb = 3; + u32 te_llb_col_rc_size = ALIGN(32 * width_in_lcus / num_vpp_pipes_enc, + DMA_ALIGNMENT) * num_vpp_pipes_enc; + u32 te_tlb_recon_data_size = ALIGN((is_ten_bit ? num_pixel_10_bit : num_pixel_8_bit) * + frame_width_coded, DMA_ALIGNMENT); + u32 te_llb_recon_data_size = ((1 + is_ten_bit) * num_pixel_te_llb * frame_height_coded + + num_vpp_pipes_enc - 1) / num_vpp_pipes_enc; + te_llb_recon_data_size = ALIGN(te_llb_recon_data_size, DMA_ALIGNMENT) * num_vpp_pipes_enc; + + return ALIGN(te_llb_recon_data_size + te_llb_col_rc_size + te_tlb_recon_data_size, + DMA_ALIGNMENT); +} + +static inline u32 calc_fe_tlb_size(u32 size_per_lcu, bool is_ten_bit) +{ + u32 num_pixels_fe_tlb_10_bit = 128, num_pixels_fe_tlb_8_bit = 64; + + return is_ten_bit ? (num_pixels_fe_tlb_10_bit * (size_per_lcu + 1)) : + (size_per_lcu * num_pixels_fe_tlb_8_bit); +} + +static u32 size_fe_lb(bool is_ten_bit, u32 standard, u32 num_vpp_pipes_enc, + u32 frame_height_coded, u32 frame_width_coded) +{ + u32 log2_lcu_size, num_cu_in_height_pipe, num_cu_in_width, + fb_llb_db_ctrl_size, fb_llb_db_luma_size, fb_llb_db_chroma_size, + fb_tlb_db_ctrl_size, fb_tlb_db_luma_size, fb_tlb_db_chroma_size, + fb_llb_sao_ctrl_size, fb_llb_sao_luma_size, fb_llb_sao_chroma_size, + fb_tlb_sao_ctrl_size, fb_tlb_sao_luma_size, fb_tlb_sao_chroma_size, + fb_lb_top_sdc_size, fb_lb_se_ctrl_size, fe_tlb_size, size_per_lcu; + + log2_lcu_size = (standard == HFI_CODEC_ENCODE_HEVC) ? 5 : 4; + num_cu_in_height_pipe = ((frame_height_coded >> log2_lcu_size) + num_vpp_pipes_enc - 1) / + num_vpp_pipes_enc; + num_cu_in_width = frame_width_coded >> log2_lcu_size; + + size_per_lcu = 2; + fe_tlb_size = calc_fe_tlb_size(size_per_lcu, 1); + fb_llb_db_ctrl_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_height_pipe; + fb_llb_db_ctrl_size = ALIGN(fb_llb_db_ctrl_size, DMA_ALIGNMENT) * num_vpp_pipes_enc; + + size_per_lcu = (1 << (log2_lcu_size - 3)); + fe_tlb_size = calc_fe_tlb_size(size_per_lcu, is_ten_bit); + fb_llb_db_luma_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_height_pipe; + fb_llb_db_luma_size = ALIGN(fb_llb_db_luma_size, DMA_ALIGNMENT) * num_vpp_pipes_enc; + + size_per_lcu = ((1 << (log2_lcu_size - 4)) * 2); + fe_tlb_size = calc_fe_tlb_size(size_per_lcu, is_ten_bit); + fb_llb_db_chroma_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_height_pipe; + fb_llb_db_chroma_size = ALIGN(fb_llb_db_chroma_size, DMA_ALIGNMENT) * num_vpp_pipes_enc; + + size_per_lcu = 1; + fe_tlb_size = calc_fe_tlb_size(size_per_lcu, 1); + fb_tlb_db_ctrl_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_width; + fb_llb_sao_ctrl_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_height_pipe; + fb_llb_sao_ctrl_size = fb_llb_sao_ctrl_size * num_vpp_pipes_enc; + fb_tlb_sao_ctrl_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_width; + + size_per_lcu = ((1 << (log2_lcu_size - 3)) + 1); + fe_tlb_size = calc_fe_tlb_size(size_per_lcu, is_ten_bit); + fb_tlb_db_luma_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_width; + + size_per_lcu = (2 * ((1 << (log2_lcu_size - 4)) + 1)); + fe_tlb_size = calc_fe_tlb_size(size_per_lcu, is_ten_bit); + fb_tlb_db_chroma_size = ALIGN(fe_tlb_size, DMA_ALIGNMENT) * num_cu_in_width; + + fb_llb_sao_luma_size = BUFFER_ALIGNMENT_256_BYTES * num_vpp_pipes_enc; + fb_llb_sao_chroma_size = BUFFER_ALIGNMENT_256_BYTES * num_vpp_pipes_enc; + fb_tlb_sao_luma_size = BUFFER_ALIGNMENT_256_BYTES; + fb_tlb_sao_chroma_size = BUFFER_ALIGNMENT_256_BYTES; + fb_lb_top_sdc_size = ALIGN((FE_SDC_DATA_PER_BLOCK * (frame_width_coded >> 5)), + DMA_ALIGNMENT); + fb_lb_se_ctrl_size = ALIGN((SE_CTRL_DATA_PER_BLOCK * (frame_width_coded >> 5)), + DMA_ALIGNMENT); + + return fb_llb_db_ctrl_size + fb_llb_db_luma_size + fb_llb_db_chroma_size + + fb_tlb_db_ctrl_size + fb_tlb_db_luma_size + fb_tlb_db_chroma_size + + fb_llb_sao_ctrl_size + fb_llb_sao_luma_size + fb_llb_sao_chroma_size + + fb_tlb_sao_ctrl_size + fb_tlb_sao_luma_size + fb_tlb_sao_chroma_size + + fb_lb_top_sdc_size + fb_lb_se_ctrl_size; +} + +static u32 size_md_lb(u32 standard, u32 frame_width_coded, + u32 frame_height_coded, u32 num_vpp_pipes_enc) +{ + u32 md_tlb_size = ALIGN(frame_width_coded, DMA_ALIGNMENT); + u32 md_llb_size = (standard == HFI_CODEC_ENCODE_HEVC) ? + ((frame_height_coded + BUFFER_ALIGNMENT_32_BYTES - 1) / + BUFFER_ALIGNMENT_32_BYTES) * LOG2_16 * LLB_UNIT_SIZE : + ((frame_height_coded + BUFFER_ALIGNMENT_16_BYTES - 1) / + BUFFER_ALIGNMENT_16_BYTES) * LOG2_32 * LLB_UNIT_SIZE; + + md_llb_size = ALIGN(md_llb_size, BUFFER_ALIGNMENT_32_BYTES); + + if (num_vpp_pipes_enc > 1) + md_llb_size = ALIGN(md_llb_size + BUFFER_ALIGNMENT_512_BYTES, + DMA_ALIGNMENT) * num_vpp_pipes_enc; + + md_llb_size = ALIGN(md_llb_size, DMA_ALIGNMENT); + + return ALIGN(md_tlb_size + md_llb_size, DMA_ALIGNMENT); +} + +static u32 size_dma_opb_lb(u32 num_vpp_pipes_enc, u32 frame_width_coded, + u32 frame_height_coded) +{ + u32 opb_packet_bytes = 128, opb_bpp = 128, opb_size_per_row = 6; + u32 dma_opb_wr_tlb_y_size = DIV_ROUND_UP(frame_width_coded, 16) * opb_packet_bytes; + u32 dma_opb_wr_tlb_uv_size = DIV_ROUND_UP(frame_width_coded, 16) * opb_packet_bytes; + u32 dma_opb_wr2_tlb_y_size = ALIGN((opb_bpp * opb_size_per_row * frame_height_coded / 8), + DMA_ALIGNMENT) * num_vpp_pipes_enc; + u32 dma_opb_wr2_tlb_uv_size = ALIGN((opb_bpp * opb_size_per_row * frame_height_coded / 8), + DMA_ALIGNMENT) * num_vpp_pipes_enc; + + dma_opb_wr2_tlb_y_size = max(dma_opb_wr2_tlb_y_size, dma_opb_wr_tlb_y_size << 1); + dma_opb_wr2_tlb_uv_size = max(dma_opb_wr2_tlb_uv_size, dma_opb_wr_tlb_uv_size << 1); + + return ALIGN(dma_opb_wr_tlb_y_size + dma_opb_wr_tlb_uv_size + dma_opb_wr2_tlb_y_size + + dma_opb_wr2_tlb_uv_size, DMA_ALIGNMENT); +} + +static u32 hfi_vpu4x_buffer_line_enc(u32 frame_width, u32 frame_height, + bool is_ten_bit, u32 num_vpp_pipes_enc, + u32 lcu_size, u32 standard) +{ + u32 width_in_lcus = (frame_width + lcu_size - 1) / lcu_size; + u32 height_in_lcus = (frame_height + lcu_size - 1) / lcu_size; + u32 frame_width_coded = width_in_lcus * lcu_size; + u32 frame_height_coded = height_in_lcus * lcu_size; + + u32 se_lb_size = size_se_lb(standard, num_vpp_pipes_enc, frame_width_coded, + frame_height_coded); + u32 te_lb_size = size_te_lb(is_ten_bit, num_vpp_pipes_enc, width_in_lcus, + frame_height_coded, frame_width_coded); + u32 fe_lb_size = size_fe_lb(is_ten_bit, standard, num_vpp_pipes_enc, frame_height_coded, + frame_width_coded); + u32 md_lb_size = size_md_lb(standard, frame_width_coded, frame_height_coded, + num_vpp_pipes_enc); + u32 dma_opb_lb_size = size_dma_opb_lb(num_vpp_pipes_enc, frame_width_coded, + frame_height_coded); + u32 dse_lb_size = ALIGN((256 + (16 * (frame_width_coded >> 4))), DMA_ALIGNMENT); + u32 size_vpss_lb_enc = size_vpss_line_buf_vpu33(num_vpp_pipes_enc, frame_width_coded, + frame_height_coded); + + return se_lb_size + te_lb_size + fe_lb_size + md_lb_size + dma_opb_lb_size + + dse_lb_size + size_vpss_lb_enc; +} + +static u32 iris_vpu4x_enc_line_size(struct iris_inst *inst) +{ + u32 num_vpp_pipes = inst->core->iris_platform_data->num_vpp_pipe; + u32 lcu_size = inst->codec == V4L2_PIX_FMT_HEVC ? 32 : 16; + struct v4l2_format *f = inst->fmt_dst; + u32 height = f->fmt.pix_mp.height; + u32 width = f->fmt.pix_mp.width; + + return hfi_vpu4x_buffer_line_enc(width, height, 0, num_vpp_pipes, + lcu_size, inst->codec); +} + static int output_min_count(struct iris_inst *inst) { int output_min_count = 4; /* fw_min_count > 0 indicates reconfig event has already arrived */ if (inst->fw_min_count) { - if (iris_split_mode_enabled(inst) && inst->codec == V4L2_PIX_FMT_VP9) + if (iris_split_mode_enabled(inst) && + (inst->codec == V4L2_PIX_FMT_VP9 || + inst->codec == V4L2_PIX_FMT_AV1)) return min_t(u32, 4, inst->fw_min_count); else return inst->fw_min_count; @@ -1422,6 +2007,8 @@ static int output_min_count(struct iris_inst *inst) if (inst->codec == V4L2_PIX_FMT_VP9) output_min_count = 9; + else if (inst->codec == V4L2_PIX_FMT_AV1) + output_min_count = 11; return output_min_count; } @@ -1444,6 +2031,7 @@ u32 iris_vpu_buf_size(struct iris_inst *inst, enum iris_buffer_type buffer_type) {BUF_PERSIST, iris_vpu_dec_persist_size }, {BUF_DPB, iris_vpu_dec_dpb_size }, {BUF_SCRATCH_1, iris_vpu_dec_scratch1_size }, + {BUF_PARTIAL, iris_vpu_dec_partial_size }, }; static const struct iris_vpu_buf_type_handle enc_internal_buf_type_handle[] = { @@ -1503,6 +2091,50 @@ u32 iris_vpu33_buf_size(struct iris_inst *inst, enum iris_buffer_type buffer_typ return size; } +u32 iris_vpu4x_buf_size(struct iris_inst *inst, enum iris_buffer_type buffer_type) +{ + const struct iris_vpu_buf_type_handle *buf_type_handle_arr = NULL; + u32 size = 0, buf_type_handle_size = 0, i; + + static const struct iris_vpu_buf_type_handle dec_internal_buf_type_handle[] = { + {BUF_BIN, iris_vpu_dec_bin_size }, + {BUF_COMV, iris_vpu_dec_comv_size }, + {BUF_NON_COMV, iris_vpu_dec_non_comv_size }, + {BUF_LINE, iris_vpu4x_dec_line_size }, + {BUF_PERSIST, iris_vpu4x_dec_persist_size }, + {BUF_DPB, iris_vpu_dec_dpb_size }, + {BUF_SCRATCH_1, iris_vpu_dec_scratch1_size }, + }; + + static const struct iris_vpu_buf_type_handle enc_internal_buf_type_handle[] = { + {BUF_BIN, iris_vpu_enc_bin_size }, + {BUF_COMV, iris_vpu_enc_comv_size }, + {BUF_NON_COMV, iris_vpu_enc_non_comv_size }, + {BUF_LINE, iris_vpu4x_enc_line_size }, + {BUF_ARP, iris_vpu_enc_arp_size }, + {BUF_VPSS, iris_vpu_enc_vpss_size }, + {BUF_SCRATCH_1, iris_vpu_enc_scratch1_size }, + {BUF_SCRATCH_2, iris_vpu_enc_scratch2_size }, + }; + + if (inst->domain == DECODER) { + buf_type_handle_size = ARRAY_SIZE(dec_internal_buf_type_handle); + buf_type_handle_arr = dec_internal_buf_type_handle; + } else if (inst->domain == ENCODER) { + buf_type_handle_size = ARRAY_SIZE(enc_internal_buf_type_handle); + buf_type_handle_arr = enc_internal_buf_type_handle; + } + + for (i = 0; i < buf_type_handle_size; i++) { + if (buf_type_handle_arr[i].type == buffer_type) { + size = buf_type_handle_arr[i].handle(inst); + break; + } + } + + return size; +} + static u32 internal_buffer_count(struct iris_inst *inst, enum iris_buffer_type buffer_type) { @@ -1510,14 +2142,20 @@ static u32 internal_buffer_count(struct iris_inst *inst, buffer_type == BUF_PERSIST) { return 1; } else if (buffer_type == BUF_COMV || buffer_type == BUF_NON_COMV) { - if (inst->codec == V4L2_PIX_FMT_H264 || inst->codec == V4L2_PIX_FMT_HEVC) + if (inst->codec == V4L2_PIX_FMT_H264 || + inst->codec == V4L2_PIX_FMT_HEVC || + inst->codec == V4L2_PIX_FMT_AV1) return 1; } + return 0; } static inline int iris_vpu_dpb_count(struct iris_inst *inst) { + if (inst->codec == V4L2_PIX_FMT_AV1) + return 11; + if (iris_split_mode_enabled(inst)) { return inst->fw_min_count ? inst->fw_min_count : inst->buffers[BUF_OUTPUT].min_count; @@ -1536,9 +2174,13 @@ int iris_vpu_buf_count(struct iris_inst *inst, enum iris_buffer_type buffer_type return MIN_BUFFERS; else return output_min_count(inst); + case BUF_NON_COMV: + if (inst->codec == V4L2_PIX_FMT_AV1) + return 0; + else + return 1; case BUF_BIN: case BUF_COMV: - case BUF_NON_COMV: case BUF_LINE: case BUF_PERSIST: return internal_buffer_count(inst, buffer_type); @@ -1546,6 +2188,7 @@ int iris_vpu_buf_count(struct iris_inst *inst, enum iris_buffer_type buffer_type case BUF_SCRATCH_2: case BUF_VPSS: case BUF_ARP: + case BUF_PARTIAL: return 1; /* internal buffer count needed by firmware is 1 */ case BUF_DPB: return iris_vpu_dpb_count(inst); diff --git a/drivers/media/platform/qcom/iris/iris_vpu_buffer.h b/drivers/media/platform/qcom/iris/iris_vpu_buffer.h index 04f0b7400a1e4..8c0d6b7b5de85 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_buffer.h +++ b/drivers/media/platform/qcom/iris/iris_vpu_buffer.h @@ -11,6 +11,7 @@ struct iris_inst; #define MIN_BUFFERS 4 #define DMA_ALIGNMENT 256 +#define HFI_ALIGNMENT_4096 4096 #define NUM_HW_PIC_BUF 32 #define LCU_MAX_SIZE_PELS 64 @@ -47,7 +48,12 @@ struct iris_inst; #define VP9_NUM_PROBABILITY_TABLE_BUF (VP9_NUM_FRAME_INFO_BUF + 4) #define VP9_PROB_TABLE_SIZE (3840) #define VP9_FRAME_INFO_BUF_SIZE (6144) +#define VP9_FRAME_INFO_BUF_SIZE_VPU4X (6400) +#define BUFFER_ALIGNMENT_16_BYTES 16 #define BUFFER_ALIGNMENT_32_BYTES 32 +#define BUFFER_ALIGNMENT_64_BYTES 64 +#define BUFFER_ALIGNMENT_256_BYTES 256 +#define BUFFER_ALIGNMENT_512_BYTES 512 #define CCE_TILE_OFFSET_SIZE ALIGN(32 * 4 * 4, BUFFER_ALIGNMENT_32_BYTES) #define MAX_SUPERFRAME_HEADER_LEN (34) #define MAX_FE_NBR_CTRL_LCU64_LINE_BUFFER_SIZE 64 @@ -61,11 +67,13 @@ struct iris_inst; #define SIZE_DOLBY_RPU_METADATA (41 * 1024) #define H264_CABAC_HDR_RATIO_HD_TOT 1 #define H264_CABAC_RES_RATIO_HD_TOT 3 -#define H265D_MAX_SLICE 1200 +#define H265D_MAX_SLICE 3600 #define SIZE_H265D_HW_PIC_T SIZE_H264D_HW_PIC_T #define H265_CABAC_HDR_RATIO_HD_TOT 2 #define H265_CABAC_RES_RATIO_HD_TOT 2 #define SIZE_H265D_VPP_CMD_PER_BUF (256) +#define SIZE_THREE_DIMENSION_USERDATA 768 +#define SIZE_H265D_ARP 9728 #define VPX_DECODER_FRAME_CONCURENCY_LVL (2) #define VPX_DECODER_FRAME_BIN_HDR_BUDGET 1 @@ -76,11 +84,39 @@ struct iris_inst; #define SIZE_H264D_HW_PIC_T (BIT(11)) +#define FE_LFT_CTRL_LINE_NUMBERS 4 +#define FE_LFT_DB_DATA_LINE_NUMBERS 2 +#define FE_LFT_LR_DATA_LINE_NUMBERS 4 +#define FE_TOP_CTRL_LINE_NUMBERS 3 +#define FE_TOP_DATA_LUMA_LINE_NUMBERS 2 +#define FE_TOP_DATA_CHROMA_LINE_NUMBERS 3 +#define FE_SDC_DATA_PER_BLOCK 16 +#define SE_CTRL_DATA_PER_BLOCK 2020 + +#define MAX_PE_NBR_DATA_LCU16_LINE_BUFFER_SIZE 96 +#define MAX_PE_NBR_DATA_LCU32_LINE_BUFFER_SIZE 192 + #define MAX_FE_NBR_CTRL_LCU64_LINE_BUFFER_SIZE 64 #define MAX_SE_NBR_CTRL_LCU64_LINE_BUFFER_SIZE 16 #define MAX_PE_NBR_DATA_LCU64_LINE_BUFFER_SIZE 384 #define MAX_FE_NBR_DATA_LUMA_LINE_BUFFER_SIZE 640 +#define AV1_CABAC_HDR_RATIO_HD_TOT 2 +#define AV1_CABAC_RES_RATIO_HD_TOT 2 +#define AV1D_LCU_MAX_SIZE_PELS 128 +#define AV1D_LCU_MIN_SIZE_PELS 64 +#define AV1D_MAX_TILE_COLS 64 +#define MAX_PE_NBR_DATA_LCU32_LINE_BUFFER_SIZE 192 +#define MAX_PE_NBR_DATA_LCU16_LINE_BUFFER_SIZE 96 +#define AV1D_NUM_HW_PIC_BUF 16 +#define AV1D_NUM_FRAME_HEADERS 16 +#define SIZE_AV1D_SEQUENCE_HEADER 768 +#define SIZE_AV1D_METADATA 512 +#define SIZE_AV1D_FRAME_HEADER 1280 +#define SIZE_AV1D_TILE_OFFSET 65536 +#define SIZE_AV1D_QM 3328 +#define SIZE_AV1D_PROB_TABLE 22784 + #define SIZE_SLICE_CMD_BUFFER (ALIGN(20480, 256)) #define SIZE_SPS_PPS_SLICE_HDR (2048 + 4096) #define SIZE_BSE_SLICE_CMD_BUF ((((8192 << 2) + 7) & (~7)) * 3) @@ -96,11 +132,24 @@ struct iris_inst; #define HFI_BUFFER_ARP_ENC 204800 +#define LOG2_16 4 +#define LOG2_32 5 +#define LLB_UNIT_SIZE 16 + #define MAX_WIDTH 4096 #define MAX_HEIGHT 2304 #define NUM_MBS_4K (DIV_ROUND_UP(MAX_WIDTH, 16) * DIV_ROUND_UP(MAX_HEIGHT, 16)) #define NUM_MBS_720P (((ALIGN(1280, 16)) >> 4) * ((ALIGN(736, 16)) >> 4)) +#define BITS_PER_PIX 16 +#define NUM_LINES_LUMA 10 +#define NUM_LINES_CHROMA 6 +#define AV1D_LCU_MAX_SIZE_PELS 128 +#define AV1D_LCU_MIN_SIZE_PELS 64 +#define AV1D_MAX_TILE_COLS 64 +#define BITS_PER_CTRL_PACK 128 +#define NUM_CTRL_PACK_LCU 10 + static inline u32 size_h264d_lb_fe_top_data(u32 frame_width) { return MAX_FE_NBR_DATA_LUMA_LINE_BUFFER_SIZE * ALIGN(frame_width, 16) * 3; @@ -146,8 +195,99 @@ static inline u32 size_h264d_qp(u32 frame_width, u32 frame_height) return DIV_ROUND_UP(frame_width, 64) * DIV_ROUND_UP(frame_height, 64) * 128; } +static inline u32 size_av1d_lb_fe_top_data(u32 frame_width, u32 frame_height) +{ + return (ALIGN(frame_width, AV1D_LCU_MAX_SIZE_PELS) * + ((BITS_PER_PIX * NUM_LINES_LUMA) >> 3) + + ALIGN(frame_width, AV1D_LCU_MAX_SIZE_PELS) / 2 * + ((BITS_PER_PIX * NUM_LINES_CHROMA) >> 3) * 2); +} + +static inline u32 size_av1d_lb_fe_left_data(u32 frame_width, u32 frame_height) +{ + return (32 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) + + ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS * 16) + + 16 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / 2 + + ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS * 8) * 2 + + 24 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) + + ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS * 16) + + 24 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / 2 + + ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS * 12) * 2 + + 24 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) + + ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS * 16) + + 16 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) + + ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS * 16) + + 16 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / 2 + + ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS * 12) * 2); +} + +static inline u32 size_av1d_lb_fe_top_ctrl(u32 frame_width, u32 frame_height) +{ + return (NUM_CTRL_PACK_LCU * ((frame_width + AV1D_LCU_MIN_SIZE_PELS - 1) / + AV1D_LCU_MIN_SIZE_PELS) * BITS_PER_CTRL_PACK / 8); +} + +static inline u32 size_av1d_lb_fe_left_ctrl(u32 frame_width, u32 frame_height) +{ + return (16 * ((ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / 16) + + (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS)) + + 3 * 16 * (ALIGN(frame_height, AV1D_LCU_MAX_SIZE_PELS) / + AV1D_LCU_MIN_SIZE_PELS)); +} + +static inline u32 size_av1d_lb_se_top_ctrl(u32 frame_width, u32 frame_height) +{ + return (((frame_width + 7) / 8) * MAX_SE_NBR_CTRL_LCU16_LINE_BUFFER_SIZE); +} + +static inline u32 size_av1d_lb_se_left_ctrl(u32 frame_width, u32 frame_height) +{ + return (max(((frame_height + 15) / 16) * + MAX_SE_NBR_CTRL_LCU16_LINE_BUFFER_SIZE, + max(((frame_height + 31) / 32) * + MAX_SE_NBR_CTRL_LCU32_LINE_BUFFER_SIZE, + ((frame_height + 63) / 64) * + MAX_SE_NBR_CTRL_LCU64_LINE_BUFFER_SIZE))); +} + +static inline u32 size_av1d_lb_pe_top_data(u32 frame_width, u32 frame_height) +{ + return (max(((frame_width + 15) / 16) * + MAX_PE_NBR_DATA_LCU16_LINE_BUFFER_SIZE, + max(((frame_width + 31) / 32) * + MAX_PE_NBR_DATA_LCU32_LINE_BUFFER_SIZE, + ((frame_width + 63) / 64) * + MAX_PE_NBR_DATA_LCU64_LINE_BUFFER_SIZE))); +} + +static inline u32 size_av1d_lb_vsp_top(u32 frame_width, u32 frame_height) +{ + return (max(((frame_width + 63) / 64) * 1280, + ((frame_width + 127) / 128) * MAX_HEIGHT)); +} + +static inline u32 size_av1d_lb_recon_dma_metadata_wr(u32 frame_width, + u32 frame_height) +{ + return ((ALIGN(frame_height, 8) / (4 / 2)) * 64); +} + +static inline u32 size_av1d_qp(u32 frame_width, u32 frame_height) +{ + return size_h264d_qp(frame_width, frame_height); +} + u32 iris_vpu_buf_size(struct iris_inst *inst, enum iris_buffer_type buffer_type); u32 iris_vpu33_buf_size(struct iris_inst *inst, enum iris_buffer_type buffer_type); +u32 iris_vpu4x_buf_size(struct iris_inst *inst, enum iris_buffer_type buffer_type); int iris_vpu_buf_count(struct iris_inst *inst, enum iris_buffer_type buffer_type); #endif diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.c b/drivers/media/platform/qcom/iris/iris_vpu_common.c index bb98950e018fa..2559f758b250f 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_common.c +++ b/drivers/media/platform/qcom/iris/iris_vpu_common.c @@ -8,16 +8,12 @@ #include #include "iris_core.h" +#include "iris_instance.h" #include "iris_vpu_common.h" #include "iris_vpu_register_defines.h" -#define WRAPPER_TZ_BASE_OFFS 0x000C0000 -#define AON_BASE_OFFS 0x000E0000 - -#define CPU_IC_BASE_OFFS (CPU_BASE_OFFS) - -#define CPU_CS_A2HSOFTINTCLR (CPU_CS_BASE_OFFS + 0x1C) -#define CLEAR_XTENSA2HOST_INTR BIT(0) +#define AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL (AON_BASE_OFFS + 0x2C) +#define AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_STATUS (AON_BASE_OFFS + 0x30) #define CTRL_INIT (CPU_CS_BASE_OFFS + 0x48) #define CTRL_STATUS (CPU_CS_BASE_OFFS + 0x4C) @@ -35,42 +31,6 @@ #define UC_REGION_ADDR (CPU_CS_BASE_OFFS + 0x64) #define UC_REGION_SIZE (CPU_CS_BASE_OFFS + 0x68) -#define CPU_CS_H2XSOFTINTEN (CPU_CS_BASE_OFFS + 0x148) -#define HOST2XTENSA_INTR_ENABLE BIT(0) - -#define CPU_CS_X2RPMH (CPU_CS_BASE_OFFS + 0x168) -#define MSK_SIGNAL_FROM_TENSILICA BIT(0) -#define MSK_CORE_POWER_ON BIT(1) - -#define CPU_IC_SOFTINT (CPU_IC_BASE_OFFS + 0x150) -#define CPU_IC_SOFTINT_H2A_SHFT 0x0 - -#define WRAPPER_INTR_STATUS (WRAPPER_BASE_OFFS + 0x0C) -#define WRAPPER_INTR_STATUS_A2HWD_BMSK BIT(3) -#define WRAPPER_INTR_STATUS_A2H_BMSK BIT(2) - -#define WRAPPER_INTR_MASK (WRAPPER_BASE_OFFS + 0x10) -#define WRAPPER_INTR_MASK_A2HWD_BMSK BIT(3) -#define WRAPPER_INTR_MASK_A2HCPU_BMSK BIT(2) - -#define WRAPPER_DEBUG_BRIDGE_LPI_CONTROL (WRAPPER_BASE_OFFS + 0x54) -#define WRAPPER_DEBUG_BRIDGE_LPI_STATUS (WRAPPER_BASE_OFFS + 0x58) -#define WRAPPER_IRIS_CPU_NOC_LPI_CONTROL (WRAPPER_BASE_OFFS + 0x5C) -#define WRAPPER_IRIS_CPU_NOC_LPI_STATUS (WRAPPER_BASE_OFFS + 0x60) - -#define WRAPPER_TZ_CPU_STATUS (WRAPPER_TZ_BASE_OFFS + 0x10) -#define WRAPPER_TZ_CTL_AXI_CLOCK_CONFIG (WRAPPER_TZ_BASE_OFFS + 0x14) -#define CTL_AXI_CLK_HALT BIT(0) -#define CTL_CLK_HALT BIT(1) - -#define WRAPPER_TZ_QNS4PDXFIFO_RESET (WRAPPER_TZ_BASE_OFFS + 0x18) -#define RESET_HIGH BIT(0) - -#define AON_WRAPPER_MVP_NOC_LPI_CONTROL (AON_BASE_OFFS) -#define REQ_POWER_DOWN_PREP BIT(0) - -#define AON_WRAPPER_MVP_NOC_LPI_STATUS (AON_BASE_OFFS + 0x4) - static void iris_vpu_interrupt_init(struct iris_core *core) { u32 mask_val; @@ -222,12 +182,14 @@ int iris_vpu_power_off_controller(struct iris_core *core) writel(MSK_SIGNAL_FROM_TENSILICA | MSK_CORE_POWER_ON, core->reg_base + CPU_CS_X2RPMH); - writel(REQ_POWER_DOWN_PREP, core->reg_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); + if (!core->iris_platform_data->no_aon) { + writel(REQ_POWER_DOWN_PREP, core->reg_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); - ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_NOC_LPI_STATUS, - val, val & BIT(0), 200, 2000); - if (ret) - goto disable_power; + ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_NOC_LPI_STATUS, + val, val & BIT(0), 200, 2000); + if (ret) + goto disable_power; + } writel(REQ_POWER_DOWN_PREP, core->reg_base + WRAPPER_IRIS_CPU_NOC_LPI_CONTROL); @@ -250,6 +212,7 @@ int iris_vpu_power_off_controller(struct iris_core *core) writel(0x0, core->reg_base + WRAPPER_TZ_CTL_AXI_CLOCK_CONFIG); disable_power: + iris_disable_unprepare_clock(core, IRIS_AHB_CLK); iris_disable_unprepare_clock(core, IRIS_CTRL_CLK); iris_disable_unprepare_clock(core, IRIS_AXI_CLK); iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); @@ -261,12 +224,14 @@ void iris_vpu_power_off_hw(struct iris_core *core) { dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], false); iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN]); + iris_disable_unprepare_clock(core, IRIS_BSE_HW_CLK); + iris_disable_unprepare_clock(core, IRIS_HW_AHB_CLK); iris_disable_unprepare_clock(core, IRIS_HW_CLK); } void iris_vpu_power_off(struct iris_core *core) { - dev_pm_opp_set_rate(core->dev, 0); + iris_opp_set_rate(core->dev, 0); core->iris_platform_data->vpu_ops->power_off_hw(core); core->iris_platform_data->vpu_ops->power_off_controller(core); iris_unset_icc_bw(core); @@ -294,11 +259,17 @@ int iris_vpu_power_on_controller(struct iris_core *core) ret = iris_prepare_enable_clock(core, IRIS_CTRL_CLK); if (ret) - goto err_disable_clock; + goto err_disable_axi_clock; + + ret = iris_prepare_enable_clock(core, IRIS_AHB_CLK); + if (ret && ret != -ENOENT) + goto err_disable_ctrl_clock; return 0; -err_disable_clock: +err_disable_ctrl_clock: + iris_disable_unprepare_clock(core, IRIS_CTRL_CLK); +err_disable_axi_clock: iris_disable_unprepare_clock(core, IRIS_AXI_CLK); err_disable_power: iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); @@ -318,13 +289,25 @@ int iris_vpu_power_on_hw(struct iris_core *core) if (ret) goto err_disable_power; + ret = iris_prepare_enable_clock(core, IRIS_HW_AHB_CLK); + if (ret && ret != -ENOENT) + goto err_disable_hw_clock; + + ret = iris_prepare_enable_clock(core, IRIS_BSE_HW_CLK); + if (ret && ret != -ENOENT) + goto err_disable_hw_ahb_clock; + ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], true); if (ret) - goto err_disable_clock; + goto err_disable_bse_hw_clock; return 0; -err_disable_clock: +err_disable_bse_hw_clock: + iris_disable_unprepare_clock(core, IRIS_BSE_HW_CLK); +err_disable_hw_ahb_clock: + iris_disable_unprepare_clock(core, IRIS_HW_AHB_CLK); +err_disable_hw_clock: iris_disable_unprepare_clock(core, IRIS_HW_CLK); err_disable_power: iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN]); @@ -332,6 +315,148 @@ int iris_vpu_power_on_hw(struct iris_core *core) return ret; } +int iris_vpu35_vpu4x_power_off_controller(struct iris_core *core) +{ + u32 clk_rst_tbl_size = core->iris_platform_data->clk_rst_tbl_size; + bool handshake_done, handshake_busy; + u32 count = 0, val = 0; + int ret; + + writel(MSK_SIGNAL_FROM_TENSILICA | MSK_CORE_POWER_ON, core->reg_base + CPU_CS_X2RPMH); + + writel(REQ_POWER_DOWN_PREP, core->reg_base + WRAPPER_IRIS_CPU_NOC_LPI_CONTROL); + + ret = readl_poll_timeout(core->reg_base + WRAPPER_IRIS_CPU_NOC_LPI_STATUS, + val, val & BIT(0), 200, 2000); + if (ret) + goto disable_power; + + writel(0, core->reg_base + WRAPPER_IRIS_CPU_NOC_LPI_CONTROL); + + /* Retry up to 1000 times as recommended by hardware documentation */ + do { + /* set MNoC to low power */ + writel(REQ_POWER_DOWN_PREP, core->reg_base + + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL); + usleep_range(10, 20); + val = readl(core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_STATUS); + + handshake_done = val & NOC_LPI_STATUS_DONE; + handshake_busy = val & (NOC_LPI_STATUS_DENY | NOC_LPI_STATUS_ACTIVE); + + if (handshake_done || !handshake_busy) + break; + + writel(0, core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL); + usleep_range(10, 20); + + } while (++count < 1000); + + if (!handshake_done && handshake_busy) + dev_err(core->dev, "LPI handshake timeout\n"); + + ret = readl_poll_timeout(core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_STATUS, + val, val & BIT(0), 200, 2000); + if (ret) + goto disable_power; + + writel(0, core->reg_base + AON_WRAPPER_MVP_VIDEO_CTL_NOC_LPI_CONTROL); + + writel(0, core->reg_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL); + + readl_poll_timeout(core->reg_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS, + val, val == 0, 200, 2000); + +disable_power: + iris_disable_unprepare_clock(core, IRIS_CTRL_CLK); + iris_disable_unprepare_clock(core, IRIS_CTRL_FREERUN_CLK); + iris_disable_unprepare_clock(core, IRIS_AXI1_CLK); + + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); + + reset_control_bulk_reset(clk_rst_tbl_size, core->resets); + + return 0; +} + +int iris_vpu35_vpu4x_power_on_controller(struct iris_core *core) +{ + int ret; + + ret = iris_enable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); + if (ret) + return ret; + + ret = iris_prepare_enable_clock(core, IRIS_AXI1_CLK); + if (ret) + goto err_disable_power; + + ret = iris_prepare_enable_clock(core, IRIS_CTRL_FREERUN_CLK); + if (ret) + goto err_disable_axi1_clk; + + ret = iris_prepare_enable_clock(core, IRIS_CTRL_CLK); + if (ret) + goto err_disable_ctrl_free_clk; + + return 0; + +err_disable_ctrl_free_clk: + iris_disable_unprepare_clock(core, IRIS_CTRL_FREERUN_CLK); +err_disable_axi1_clk: + iris_disable_unprepare_clock(core, IRIS_AXI1_CLK); +err_disable_power: + iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_CTRL_POWER_DOMAIN]); + + return ret; +} + +void iris_vpu35_vpu4x_program_bootup_registers(struct iris_core *core) +{ + writel(0x1, core->reg_base + WRAPPER_IRIS_VCODEC_VPU_WRAPPER_SPARE_0); +} + +u64 iris_vpu3x_vpu4x_calculate_frequency(struct iris_inst *inst, size_t data_size) +{ + struct platform_inst_caps *caps = inst->core->iris_platform_data->inst_caps; + struct v4l2_format *inp_f = inst->fmt_src; + u32 height, width, mbs_per_second, mbpf; + u64 fw_cycles, fw_vpp_cycles; + u64 vsp_cycles, vpp_cycles; + u32 fps = inst->frame_rate; + + width = max(inp_f->fmt.pix_mp.width, inst->crop.width); + height = max(inp_f->fmt.pix_mp.height, inst->crop.height); + + mbpf = NUM_MBS_PER_FRAME(height, width); + mbs_per_second = mbpf * fps; + + fw_cycles = fps * caps->mb_cycles_fw; + fw_vpp_cycles = fps * caps->mb_cycles_fw_vpp; + + vpp_cycles = mult_frac(mbs_per_second, caps->mb_cycles_vpp, (u32)inst->fw_caps[PIPE].value); + /* 21 / 20 is minimum overhead factor */ + vpp_cycles += max(div_u64(vpp_cycles, 20), fw_vpp_cycles); + + /* 1.059 is multi-pipe overhead */ + if (inst->fw_caps[PIPE].value > 1) + vpp_cycles += div_u64(vpp_cycles * 59, 1000); + + /* 1.05 is VPP FW overhead */ + if (inst->fw_caps[STAGE].value == STAGE_2) + vpp_cycles += mult_frac(vpp_cycles, 5, 100); + + vsp_cycles = fps * data_size * 8; + vsp_cycles = div_u64(vsp_cycles, 2); + /* VSP FW overhead 1.05 */ + vsp_cycles = div_u64(vsp_cycles * 21, 20); + + if (inst->fw_caps[STAGE].value == STAGE_1) + vsp_cycles = vsp_cycles * 3; + + return max3(vpp_cycles, vsp_cycles, fw_cycles); +} + int iris_vpu_power_on(struct iris_core *core) { u32 freq; @@ -352,7 +477,7 @@ int iris_vpu_power_on(struct iris_core *core) freq = core->power.clk_freq ? core->power.clk_freq : (u32)ULONG_MAX; - dev_pm_opp_set_rate(core->dev, freq); + iris_opp_set_rate(core->dev, freq); core->iris_platform_data->set_preset_registers(core); diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.h b/drivers/media/platform/qcom/iris/iris_vpu_common.h index d636e287457ad..f6dffc613b822 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_common.h +++ b/drivers/media/platform/qcom/iris/iris_vpu_common.h @@ -12,6 +12,7 @@ extern const struct vpu_ops iris_vpu2_ops; extern const struct vpu_ops iris_vpu3_ops; extern const struct vpu_ops iris_vpu33_ops; extern const struct vpu_ops iris_vpu35_ops; +extern const struct vpu_ops iris_vpu4x_ops; struct vpu_ops { void (*power_off_hw)(struct iris_core *core); @@ -33,5 +34,9 @@ int iris_vpu_power_on(struct iris_core *core); int iris_vpu_power_off_controller(struct iris_core *core); void iris_vpu_power_off_hw(struct iris_core *core); void iris_vpu_power_off(struct iris_core *core); +int iris_vpu35_vpu4x_power_off_controller(struct iris_core *core); +int iris_vpu35_vpu4x_power_on_controller(struct iris_core *core); +void iris_vpu35_vpu4x_program_bootup_registers(struct iris_core *core); +u64 iris_vpu3x_vpu4x_calculate_frequency(struct iris_inst *inst, size_t data_size); #endif diff --git a/drivers/media/platform/qcom/iris/iris_vpu_register_defines.h b/drivers/media/platform/qcom/iris/iris_vpu_register_defines.h index fe8a39e5e5a3f..72168b9ffa738 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_register_defines.h +++ b/drivers/media/platform/qcom/iris/iris_vpu_register_defines.h @@ -7,11 +7,72 @@ #define __IRIS_VPU_REGISTER_DEFINES_H__ #define VCODEC_BASE_OFFS 0x00000000 +#define AON_MVP_NOC_RESET 0x0001F000 #define CPU_BASE_OFFS 0x000A0000 #define WRAPPER_BASE_OFFS 0x000B0000 +#define WRAPPER_TZ_BASE_OFFS 0x000C0000 +#define AON_BASE_OFFS 0x000E0000 + +#define VCODEC_SS_IDLE_STATUSN (VCODEC_BASE_OFFS + 0x70) + +#define AON_WRAPPER_MVP_NOC_RESET_REQ (AON_MVP_NOC_RESET + 0x000) +#define VIDEO_NOC_RESET_REQ (BIT(0) | BIT(1)) + +#define AON_WRAPPER_MVP_NOC_RESET_ACK (AON_MVP_NOC_RESET + 0x004) #define CPU_CS_BASE_OFFS (CPU_BASE_OFFS) +#define CPU_IC_BASE_OFFS (CPU_BASE_OFFS) + +#define CPU_CS_A2HSOFTINTCLR (CPU_CS_BASE_OFFS + 0x1C) +#define CLEAR_XTENSA2HOST_INTR BIT(0) + +#define CPU_CS_H2XSOFTINTEN (CPU_CS_BASE_OFFS + 0x148) +#define HOST2XTENSA_INTR_ENABLE BIT(0) + +#define CPU_IC_SOFTINT (CPU_IC_BASE_OFFS + 0x150) +#define CPU_IC_SOFTINT_H2A_SHFT 0x0 + +#define CPU_CS_AHB_BRIDGE_SYNC_RESET (CPU_CS_BASE_OFFS + 0x160) +#define CORE_BRIDGE_SW_RESET BIT(0) +#define CORE_BRIDGE_HW_RESET_DISABLE BIT(1) + +#define CPU_CS_X2RPMH (CPU_CS_BASE_OFFS + 0x168) +#define MSK_SIGNAL_FROM_TENSILICA BIT(0) +#define MSK_CORE_POWER_ON BIT(1) +#define WRAPPER_INTR_STATUS (WRAPPER_BASE_OFFS + 0x0C) +#define WRAPPER_INTR_STATUS_A2HWD_BMSK BIT(3) +#define WRAPPER_INTR_STATUS_A2H_BMSK BIT(2) + +#define WRAPPER_INTR_MASK (WRAPPER_BASE_OFFS + 0x10) +#define WRAPPER_INTR_MASK_A2HWD_BMSK BIT(3) +#define WRAPPER_INTR_MASK_A2HCPU_BMSK BIT(2) + +#define WRAPPER_DEBUG_BRIDGE_LPI_CONTROL (WRAPPER_BASE_OFFS + 0x54) +#define WRAPPER_DEBUG_BRIDGE_LPI_STATUS (WRAPPER_BASE_OFFS + 0x58) +#define WRAPPER_IRIS_CPU_NOC_LPI_CONTROL (WRAPPER_BASE_OFFS + 0x5C) +#define REQ_POWER_DOWN_PREP BIT(0) + +#define WRAPPER_IRIS_CPU_NOC_LPI_STATUS (WRAPPER_BASE_OFFS + 0x60) +#define NOC_LPI_STATUS_DONE BIT(0) /* Indicates the NOC handshake is complete */ +#define NOC_LPI_STATUS_DENY BIT(1) /* Indicates the NOC handshake is denied */ +#define NOC_LPI_STATUS_ACTIVE BIT(2) /* Indicates the NOC is active */ + +#define WRAPPER_IRIS_VCODEC_VPU_WRAPPER_SPARE_0 (WRAPPER_BASE_OFFS + 0x78) #define WRAPPER_CORE_POWER_STATUS (WRAPPER_BASE_OFFS + 0x80) +#define WRAPPER_CORE_CLOCK_CONFIG (WRAPPER_BASE_OFFS + 0x88) +#define CORE_CLK_RUN 0x0 + +#define WRAPPER_TZ_CPU_STATUS (WRAPPER_TZ_BASE_OFFS + 0x10) + +#define WRAPPER_TZ_CTL_AXI_CLOCK_CONFIG (WRAPPER_TZ_BASE_OFFS + 0x14) +#define CTL_AXI_CLK_HALT BIT(0) +#define CTL_CLK_HALT BIT(1) + +#define WRAPPER_TZ_QNS4PDXFIFO_RESET (WRAPPER_TZ_BASE_OFFS + 0x18) +#define RESET_HIGH BIT(0) + +#define AON_WRAPPER_MVP_NOC_LPI_CONTROL (AON_BASE_OFFS) +#define AON_WRAPPER_MVP_NOC_LPI_STATUS (AON_BASE_OFFS + 0x4) #endif diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index abf959b8f3a67..646dae3407b4d 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -949,6 +949,7 @@ static const struct venus_resources sc7180_res = { .enc_nodename = "video-encoder", }; +#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS)) static const struct freq_tbl sm8250_freq_table[] = { { 0, 444000000 }, { 0, 366000000 }, @@ -1069,6 +1070,7 @@ static const struct venus_resources sc7280_res = { .dec_nodename = "video-decoder", .enc_nodename = "video-encoder", }; +#endif static const struct bw_tbl qcm2290_bw_table_dec[] = { { 352800, 597000, 0, 746000, 0 }, /* 1080p@30 + 720p@30 */ @@ -1125,11 +1127,15 @@ static const struct of_device_id venus_dt_match[] = { { .compatible = "qcom,msm8998-venus", .data = &msm8998_res, }, { .compatible = "qcom,qcm2290-venus", .data = &qcm2290_res, }, { .compatible = "qcom,sc7180-venus", .data = &sc7180_res, }, +#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS)) { .compatible = "qcom,sc7280-venus", .data = &sc7280_res, }, +#endif { .compatible = "qcom,sdm660-venus", .data = &sdm660_res, }, { .compatible = "qcom,sdm845-venus", .data = &sdm845_res, }, { .compatible = "qcom,sdm845-venus-v2", .data = &sdm845_res_v2, }, +#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS)) { .compatible = "qcom,sm8250-venus", .data = &sm8250_res, }, +#endif { } }; MODULE_DEVICE_TABLE(of, venus_dt_match); @@ -1146,6 +1152,5 @@ static struct platform_driver qcom_venus_driver = { }; module_platform_driver(qcom_venus_driver); -MODULE_ALIAS("platform:qcom-venus"); MODULE_DESCRIPTION("Qualcomm Venus video encoder and decoder driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h index 7506f5d0f609a..c345736346c8b 100644 --- a/drivers/media/platform/qcom/venus/core.h +++ b/drivers/media/platform/qcom/venus/core.h @@ -54,8 +54,10 @@ enum vpu_version { VPU_VERSION_AR50, VPU_VERSION_AR50_LITE, VPU_VERSION_IRIS1, +#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS)) VPU_VERSION_IRIS2, VPU_VERSION_IRIS2_1, +#endif }; struct firmware_version { @@ -215,6 +217,8 @@ struct venus_core { size_t mapped_mem_size; phys_addr_t mem_phys; size_t mem_size; + struct qcom_scm_pas_context *ctx; + bool iommu_domain_owned; } fw; struct mutex lock; struct list_head instances; @@ -525,13 +529,22 @@ struct venus_inst { #define IS_V1(core) ((core)->res->hfi_version == HFI_VERSION_1XX) #define IS_V3(core) ((core)->res->hfi_version == HFI_VERSION_3XX) #define IS_V4(core) ((core)->res->hfi_version == HFI_VERSION_4XX) +#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS)) #define IS_V6(core) ((core)->res->hfi_version == HFI_VERSION_6XX) +#else +#define IS_V6(core) (0) +#endif #define IS_AR50(core) ((core)->res->vpu_version == VPU_VERSION_AR50) #define IS_AR50_LITE(core) ((core)->res->vpu_version == VPU_VERSION_AR50_LITE) #define IS_IRIS1(core) ((core)->res->vpu_version == VPU_VERSION_IRIS1) +#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS)) #define IS_IRIS2(core) ((core)->res->vpu_version == VPU_VERSION_IRIS2) #define IS_IRIS2_1(core) ((core)->res->vpu_version == VPU_VERSION_IRIS2_1) +#else +#define IS_IRIS2(core) (0) +#define IS_IRIS2_1(core) (0) +#endif static inline bool is_lite(struct venus_core *core) { diff --git a/drivers/media/platform/qcom/venus/firmware.c b/drivers/media/platform/qcom/venus/firmware.c index af0ac40bec9b9..dbfb322a99978 100644 --- a/drivers/media/platform/qcom/venus/firmware.c +++ b/drivers/media/platform/qcom/venus/firmware.c @@ -9,13 +9,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include "core.h" #include "firmware.h" @@ -79,95 +79,125 @@ int venus_set_hw_state(struct venus_core *core, bool resume) return 0; } -static int venus_load_fw(struct venus_core *core, const char *fwname, - phys_addr_t *mem_phys, size_t *mem_size) +static int venus_load_fw_prepare(struct venus_core *core, const char *fwname, + phys_addr_t *mem_phys, size_t *res_size, + const struct firmware **mdt) { - const struct firmware *mdt; - struct reserved_mem *rmem; - struct device_node *node; - struct device *dev; + struct resource res; ssize_t fw_size; - void *mem_va; int ret; - *mem_phys = 0; - *mem_size = 0; - - dev = core->dev; - node = of_parse_phandle(dev->of_node, "memory-region", 0); - if (!node) { - dev_err(dev, "no memory-region specified\n"); + ret = of_reserved_mem_region_to_resource(core->dev->of_node, 0, &res); + if (ret) { + dev_err(core->dev, "failed to lookup reserved memory-region\n"); return -EINVAL; } - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { - dev_err(dev, "failed to lookup reserved memory-region\n"); - return -EINVAL; - } + *mem_phys = res.start; + *res_size = resource_size(&res); - ret = request_firmware(&mdt, fwname, dev); - if (ret < 0) + ret = request_firmware(mdt, fwname, core->dev); + if (ret < 0) { + dev_err(core->dev, "%s: request_firmware: %d\n", __func__, ret); return ret; + } - fw_size = qcom_mdt_get_size(mdt); + fw_size = qcom_mdt_get_size(*mdt); if (fw_size < 0) { ret = fw_size; - goto err_release_fw; + goto err_release; } - *mem_phys = rmem->base; - *mem_size = rmem->size; - - if (*mem_size < fw_size || fw_size > VENUS_FW_MEM_SIZE) { + if (*res_size < fw_size || fw_size > VENUS_FW_MEM_SIZE) { ret = -EINVAL; - goto err_release_fw; - } - - mem_va = memremap(*mem_phys, *mem_size, MEMREMAP_WC); - if (!mem_va) { - dev_err(dev, "unable to map memory region %pa size %#zx\n", mem_phys, *mem_size); - ret = -ENOMEM; - goto err_release_fw; + goto err_release; } - if (core->use_tz) - ret = qcom_mdt_load(dev, mdt, fwname, VENUS_PAS_ID, - mem_va, *mem_phys, *mem_size, NULL); - else - ret = qcom_mdt_load_no_init(dev, mdt, fwname, mem_va, - *mem_phys, *mem_size, NULL); + return 0; - memunmap(mem_va); -err_release_fw: - release_firmware(mdt); +err_release: + release_firmware(*mdt); return ret; } -static int venus_boot_no_tz(struct venus_core *core, phys_addr_t mem_phys, - size_t mem_size) +static int venus_load_fw(struct venus_core *core, + const struct firmware *mdt, const char *fwname, + phys_addr_t mem_phys, size_t res_size) { - struct iommu_domain *iommu; + struct qcom_scm_pas_context *ctx; struct device *dev; int ret; - dev = core->fw.dev; - if (!dev) - return -EPROBE_DEFER; + dev = core->fw.dev ? core->fw.dev : core->dev; + ctx = devm_qcom_scm_pas_context_alloc(dev, VENUS_PAS_ID, mem_phys, res_size); + if (!ctx) { + dev_err(core->dev, "%s: ctx is null\n", __func__); + return -ENOMEM; + } - iommu = core->fw.iommu_domain; - core->fw.mapped_mem_size = mem_size; + ctx->use_tzmem = !!core->fw.dev; - ret = iommu_map(iommu, VENUS_FW_START_ADDR, mem_phys, mem_size, - IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV, GFP_KERNEL); + ret = qcom_mdt_pas_load(ctx, mdt, fwname, NULL); + qcom_scm_pas_metadata_release(ctx); if (ret) { - dev_err(dev, "could not map video firmware region\n"); + dev_err(core->dev, "%s: qcom_mdt_pas_load: %d\n", __func__, ret); return ret; } - venus_reset_cpu(core); + if (core->fw.iommu_domain) { + ret = iommu_map(core->fw.iommu_domain, 0, mem_phys, res_size, + IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV, GFP_KERNEL); + if (ret) { + dev_err(core->dev, "%s: iommu_map: %d\n", __func__, ret); + return ret; + } + } + + core->fw.mapped_mem_size = res_size; + + ret = qcom_scm_pas_prepare_and_auth_reset(ctx); + if (ret) { + dev_err(core->dev, "%s: qcom_scm_pas_prepare_and_auth_reset: %d\n", __func__, ret); + if (core->fw.iommu_domain) + iommu_unmap(core->fw.iommu_domain, 0, res_size); + core->fw.mapped_mem_size = 0; + return ret; + } + core->fw.ctx = ctx; + return 0; +} + +static int venus_load_fw_no_tz(struct venus_core *core, + const struct firmware *mdt, const char *fwname, + phys_addr_t mem_phys, size_t res_size) +{ + void *mem_va; + int ret; + + mem_va = memremap(mem_phys, res_size, MEMREMAP_WC); + if (!mem_va) { + dev_err(core->dev, + "unable to map memory region %pa size %#zx\n", &mem_phys, res_size); + return -ENOMEM; + } + + ret = qcom_mdt_load_no_init(core->fw.dev, mdt, fwname, mem_va, mem_phys, res_size, NULL); + memunmap(mem_va); + if (ret) { + dev_err(core->dev, "%s: qcom_mdt_load_no_init: %d\n", __func__, ret); + return ret; + } + + ret = iommu_map(core->fw.iommu_domain, VENUS_FW_START_ADDR, mem_phys, + res_size, IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV, GFP_KERNEL); + if (ret) { + dev_err(core->dev, "could not map video firmware region\n"); + return ret; + } + + core->fw.mapped_mem_size = res_size; + venus_reset_cpu(core); return 0; } @@ -221,36 +251,35 @@ int venus_boot(struct venus_core *core) { struct device *dev = core->dev; const struct venus_resources *res = core->res; + const struct firmware *mdt; const char *fwpath = NULL; phys_addr_t mem_phys; - size_t mem_size; + size_t res_size; int ret; if (!IS_ENABLED(CONFIG_QCOM_MDT_LOADER) || - (core->use_tz && !qcom_scm_is_available())) - return -EPROBE_DEFER; + (!core->use_tz && !core->fw.dev)) + return driver_deferred_probe_check_state(core->dev); - ret = of_property_read_string_index(dev->of_node, "firmware-name", 0, - &fwpath); + ret = of_property_read_string_index(dev->of_node, "firmware-name", 0, &fwpath); if (ret) fwpath = core->res->fwname; - ret = venus_load_fw(core, fwpath, &mem_phys, &mem_size); - if (ret) { - dev_err(dev, "fail to load video firmware\n"); - return -EINVAL; - } - - core->fw.mem_size = mem_size; - core->fw.mem_phys = mem_phys; + ret = venus_load_fw_prepare(core, fwpath, &mem_phys, &res_size, &mdt); + if (ret) + return ret; if (core->use_tz) - ret = qcom_scm_pas_auth_and_reset(VENUS_PAS_ID); + ret = venus_load_fw(core, mdt, fwpath, mem_phys, res_size); else - ret = venus_boot_no_tz(core, mem_phys, mem_size); + ret = venus_load_fw_no_tz(core, mdt, fwpath, mem_phys, res_size); - if (ret) + release_firmware(mdt); + + if (ret) { + dev_err(dev, "fail to load video firmware\n"); return ret; + } if (core->use_tz && res->cp_size) { /* @@ -268,24 +297,29 @@ int venus_boot(struct venus_core *core) res->cp_nonpixel_start, res->cp_nonpixel_size); if (ret) { - qcom_scm_pas_shutdown(VENUS_PAS_ID); - dev_err(dev, "set virtual address ranges fail (%d)\n", - ret); + venus_shutdown(core); + dev_err(dev, "set virtual address ranges fail (%d)\n", ret); return ret; } } - return 0; + return ret; } int venus_shutdown(struct venus_core *core) { int ret; - if (core->use_tz) + if (core->use_tz) { ret = qcom_scm_pas_shutdown(VENUS_PAS_ID); - else + if (core->fw.iommu_domain && core->fw.mapped_mem_size) { + iommu_unmap(core->fw.iommu_domain, 0, core->fw.mapped_mem_size); + core->fw.mapped_mem_size = 0; + } + core->fw.ctx = NULL; + } else { ret = venus_shutdown_no_tz(core); + } return ret; } @@ -310,6 +344,94 @@ int venus_firmware_check(struct venus_core *core) return -EINVAL; } +static struct device *venus_firmware_alloc_platform_dev(struct venus_core *core, + const char *name, const u32 *f_id) +{ + struct platform_device *pdev; + int ret; + + pdev = platform_device_alloc(name, 0); + if (!pdev) { + dev_err(core->dev, "%s: platform_device_alloc err\n", __func__); + return ERR_PTR(-ENOMEM); + } + + pdev->dev.parent = core->dev; + + ret = platform_device_add(pdev); + if (ret) { + dev_err(core->dev, "%s: platform_device_add err(%d)\n", __func__, ret); + platform_device_put(pdev); + return ERR_PTR(ret); + } + + ret = of_dma_configure_id(&pdev->dev, core->dev->of_node, true, f_id); + if (ret) { + dev_err(core->dev, "%s: of_dma_configure_id err(%d)\n", __func__, ret); + platform_device_unregister(to_platform_device(&pdev->dev)); + return ERR_PTR(ret); + } + + return &pdev->dev; +} + +static int venus_firmware_setup_iommu_dev(struct venus_core *core) +{ + const u32 f_id = VENUS_FIRMWARE; + struct device *dev; + int ret = 0; + + dev = venus_firmware_alloc_platform_dev(core, "video_firmware", &f_id); + if (IS_ERR(dev)) { + dev_err(core->dev, "%s: err\n", __func__); + return PTR_ERR(dev); + } + + if (!device_iommu_mapped(dev)) { + device_unregister(dev); + return -ENODEV; + } + + ret = dma_set_mask_and_coherent(dev, core->res->dma_mask); + if (ret) { + device_unregister(dev); + return ret; + } + + core->fw.dev = dev; + core->fw.iommu_domain = iommu_get_domain_for_dev(core->fw.dev); + core->fw.iommu_domain_owned = false; + + return 0; +} + +static int venus_firmware_init_auto_detect(struct venus_core *core) +{ + int ret; + + core->use_tz = false; + if (qcom_scm_is_available()) { + if (qcom_scm_pas_supported(VENUS_PAS_ID)) + core->use_tz = true; + } else { + ret = driver_deferred_probe_check_state(core->dev); + if (ret == -EPROBE_DEFER) + return ret; + } + + /* + * 1. use_tz is false: No authentication is performed. + * 2. use_tz is true: TZ perform authentication. + * a. device_iommu_mapped true: Linux config smmu + * b. device_iommu_mapped false: TZ config smmu + */ + ret = venus_firmware_setup_iommu_dev(core); + if (ret == -ENODEV && core->use_tz) + ret = 0; + + return ret; +} + int venus_firmware_init(struct venus_core *core) { struct platform_device_info info; @@ -320,8 +442,8 @@ int venus_firmware_init(struct venus_core *core) np = of_get_child_by_name(core->dev->of_node, "video-firmware"); if (!np) { - core->use_tz = true; - return 0; + ret = venus_firmware_init_auto_detect(core); + return ret; } memset(&info, 0, sizeof(info)); @@ -360,6 +482,7 @@ int venus_firmware_init(struct venus_core *core) } core->fw.iommu_domain = iommu_dom; + core->fw.iommu_domain_owned = true; of_node_put(np); @@ -368,6 +491,7 @@ int venus_firmware_init(struct venus_core *core) err_iommu_free: iommu_domain_free(iommu_dom); err_unregister: + core->fw.dev = NULL; platform_device_unregister(pdev); of_node_put(np); return ret; @@ -380,14 +504,17 @@ void venus_firmware_deinit(struct venus_core *core) if (!core->fw.dev) return; - iommu = core->fw.iommu_domain; - - iommu_detach_device(iommu, core->fw.dev); + if (!core->use_tz && core->fw.iommu_domain_owned) { + iommu = core->fw.iommu_domain; - if (core->fw.iommu_domain) { - iommu_domain_free(iommu); - core->fw.iommu_domain = NULL; + if (iommu) { + iommu_detach_device(iommu, core->fw.dev); + iommu_domain_free(iommu); + } } - platform_device_unregister(to_platform_device(core->fw.dev)); + core->fw.dev = NULL; + core->fw.ctx = NULL; + core->fw.iommu_domain = NULL; + core->fw.iommu_domain_owned = false; } diff --git a/drivers/media/platform/qcom/venus/helpers.c b/drivers/media/platform/qcom/venus/helpers.c index 2e4363f822317..7f370caacd9f8 100644 --- a/drivers/media/platform/qcom/venus/helpers.c +++ b/drivers/media/platform/qcom/venus/helpers.c @@ -954,8 +954,8 @@ static u32 get_framesize_raw_nv12(u32 width, u32 height) uv_sclines = ALIGN(((height + 1) >> 1), 16); y_plane = y_stride * y_sclines; - uv_plane = uv_stride * uv_sclines + SZ_4K; - size = y_plane + uv_plane + SZ_8K; + uv_plane = uv_stride * uv_sclines; + size = y_plane + uv_plane; return ALIGN(size, SZ_4K); } diff --git a/drivers/media/platform/qcom/venus/hfi_platform_v4.c b/drivers/media/platform/qcom/venus/hfi_platform_v4.c index cda888b56b5d4..e0b3652bb4409 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform_v4.c +++ b/drivers/media/platform/qcom/venus/hfi_platform_v4.c @@ -136,8 +136,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_H264, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 1, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 120000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -173,8 +173,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_HEVC, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 1, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 120000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -195,8 +195,8 @@ static const struct hfi_plat_caps caps[] = { .caps[19] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, .caps[20] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[21] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, - .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .num_caps = 24, .pl[0] = {HFI_HEVC_PROFILE_MAIN, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, .pl[1] = {HFI_HEVC_PROFILE_MAIN10, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, @@ -210,8 +210,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_VP8, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 96, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 96, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 1, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 120000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -229,8 +229,8 @@ static const struct hfi_plat_caps caps[] = { .caps[16] = {HFI_CAPABILITY_P_FRAME_QP, 0, 127, 1}, .caps[17] = {HFI_CAPABILITY_MAX_WORKMODES, 1, 2, 1}, .caps[18] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, - .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .caps[21] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[22] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, .num_caps = 23, diff --git a/drivers/media/platform/qcom/venus/hfi_platform_v6.c b/drivers/media/platform/qcom/venus/hfi_platform_v6.c index d8568c08cc361..fb8d10ab34043 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform_v6.c +++ b/drivers/media/platform/qcom/venus/hfi_platform_v6.c @@ -173,8 +173,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_HEVC, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 8192, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 8192, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 8192, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 8192, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 64, 138240, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 160000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -195,8 +195,8 @@ static const struct hfi_plat_caps caps[] = { .caps[19] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, .caps[20] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[21] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, - .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[22] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[23] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .num_caps = 24, .pl[0] = {HFI_HEVC_PROFILE_MAIN, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, .pl[1] = {HFI_HEVC_PROFILE_MAIN10, HFI_HEVC_LEVEL_6 | HFI_HEVC_TIER_HIGH0}, @@ -210,8 +210,8 @@ static const struct hfi_plat_caps caps[] = { .codec = HFI_VIDEO_CODEC_VP8, .domain = VIDC_SESSION_TYPE_ENC, .cap_bufs_mode_dynamic = true, - .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 4096, 16}, - .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 4096, 16}, + .caps[0] = {HFI_CAPABILITY_FRAME_WIDTH, 128, 4096, 1}, + .caps[1] = {HFI_CAPABILITY_FRAME_HEIGHT, 128, 4096, 1}, .caps[2] = {HFI_CAPABILITY_MBS_PER_FRAME, 64, 36864, 1}, .caps[3] = {HFI_CAPABILITY_BITRATE, 1, 74000000, 1}, .caps[4] = {HFI_CAPABILITY_SCALE_X, 8192, 65536, 1}, @@ -229,8 +229,8 @@ static const struct hfi_plat_caps caps[] = { .caps[16] = {HFI_CAPABILITY_P_FRAME_QP, 0, 127, 1}, .caps[17] = {HFI_CAPABILITY_MAX_WORKMODES, 1, 2, 1}, .caps[18] = {HFI_CAPABILITY_RATE_CONTROL_MODES, 0x1000001, 0x1000005, 1}, - .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 16}, - .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 16}, + .caps[19] = {HFI_CAPABILITY_BLUR_WIDTH, 96, 4096, 1}, + .caps[20] = {HFI_CAPABILITY_BLUR_HEIGHT, 96, 4096, 1}, .caps[21] = {HFI_CAPABILITY_COLOR_SPACE_CONVERSION, 0, 2, 1}, .caps[22] = {HFI_CAPABILITY_ROTATION, 1, 4, 90}, .num_caps = 23, diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c index 55c27345b7d89..d2d46f97bbc32 100644 --- a/drivers/media/platform/qcom/venus/vdec.c +++ b/drivers/media/platform/qcom/venus/vdec.c @@ -435,9 +435,13 @@ vdec_g_selection(struct file *file, void *fh, struct v4l2_selection *s) static int vdec_querycap(struct file *file, void *fh, struct v4l2_capability *cap) { + struct venus_inst *inst = to_inst(file); + struct venus_core *core = inst->core; + strscpy(cap->driver, "qcom-venus", sizeof(cap->driver)); strscpy(cap->card, "Qualcomm Venus video decoder", sizeof(cap->card)); - strscpy(cap->bus_info, "platform:qcom-venus", sizeof(cap->bus_info)); + snprintf(cap->bus_info, sizeof(cap->bus_info), + "plat:%s:dec", dev_name(core->dev)); return 0; } @@ -567,7 +571,13 @@ vdec_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder_cmd *cmd) fdata.buffer_type = HFI_BUFFER_INPUT; fdata.flags |= HFI_BUFFERFLAG_EOS; - if (IS_V6(inst->core) && is_fw_rev_or_older(inst->core, 1, 0, 87)) + + /* Send NULL EOS addr for only IRIS2 (SM8250),for firmware <= 1.0.87. + * SC7280 also reports "1.0." parsed as 1.0.0; restricting to IRIS2 + * avoids misapplying this quirk and breaking VP9 decode on SC7280. + */ + + if (IS_IRIS2(inst->core) && is_fw_rev_or_older(inst->core, 1, 0, 87)) fdata.device_addr = 0; else fdata.device_addr = 0xdeadb000; @@ -1442,10 +1452,10 @@ static void vdec_buf_done(struct venus_inst *inst, unsigned int buf_type, inst->drain_active = false; inst->codec_state = VENUS_DEC_STATE_STOPPED; } + } else { + if (!bytesused) + state = VB2_BUF_STATE_ERROR; } - - if (!bytesused) - state = VB2_BUF_STATE_ERROR; } else { vbuf->sequence = inst->sequence_out++; } @@ -1778,12 +1788,9 @@ static int vdec_probe(struct platform_device *pdev) struct venus_core *core; int ret; - if (!dev->parent) - return -EPROBE_DEFER; - core = dev_get_drvdata(dev->parent); if (!core) - return -EPROBE_DEFER; + return -EINVAL; platform_set_drvdata(pdev, core); @@ -1882,6 +1889,5 @@ static struct platform_driver qcom_venus_dec_driver = { }; module_platform_driver(qcom_venus_dec_driver); -MODULE_ALIAS("platform:qcom-venus-decoder"); MODULE_DESCRIPTION("Qualcomm Venus video decoder driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c index fba07557a3996..689d39dae6315 100644 --- a/drivers/media/platform/qcom/venus/venc.c +++ b/drivers/media/platform/qcom/venus/venc.c @@ -144,9 +144,13 @@ static int venc_v4l2_to_hfi(int id, int value) static int venc_querycap(struct file *file, void *fh, struct v4l2_capability *cap) { + struct venus_inst *inst = to_inst(file); + struct venus_core *core = inst->core; + strscpy(cap->driver, "qcom-venus", sizeof(cap->driver)); strscpy(cap->card, "Qualcomm Venus video encoder", sizeof(cap->card)); - strscpy(cap->bus_info, "platform:qcom-venus", sizeof(cap->bus_info)); + snprintf(cap->bus_info, sizeof(cap->bus_info), + "plat:%s:enc", dev_name(core->dev)); return 0; } @@ -1560,12 +1564,9 @@ static int venc_probe(struct platform_device *pdev) struct venus_core *core; int ret; - if (!dev->parent) - return -EPROBE_DEFER; - core = dev_get_drvdata(dev->parent); if (!core) - return -EPROBE_DEFER; + return -EINVAL; platform_set_drvdata(pdev, core); @@ -1664,6 +1665,5 @@ static struct platform_driver qcom_venus_enc_driver = { }; module_platform_driver(qcom_venus_enc_driver); -MODULE_ALIAS("platform:qcom-venus-encoder"); MODULE_DESCRIPTION("Qualcomm Venus video encoder driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/platform/renesas/rcar_drif.c b/drivers/media/platform/renesas/rcar_drif.c index 11bf47fb8266d..0844934f7aa68 100644 --- a/drivers/media/platform/renesas/rcar_drif.c +++ b/drivers/media/platform/renesas/rcar_drif.c @@ -1246,6 +1246,7 @@ static struct device_node *rcar_drif_bond_enabled(struct platform_device *p) if (np && of_device_is_available(np)) return np; + of_node_put(np); return NULL; } diff --git a/drivers/media/platform/rockchip/rga/rga-buf.c b/drivers/media/platform/rockchip/rga/rga-buf.c index 730bdf98565a5..bb575873f2b24 100644 --- a/drivers/media/platform/rockchip/rga/rga-buf.c +++ b/drivers/media/platform/rockchip/rga/rga-buf.c @@ -80,6 +80,9 @@ static int rga_buf_init(struct vb2_buffer *vb) struct rga_frame *f = rga_get_frame(ctx, vb->vb2_queue->type); size_t n_desc = 0; + if (IS_ERR(f)) + return PTR_ERR(f); + n_desc = DIV_ROUND_UP(f->size, PAGE_SIZE); rbuf->n_desc = n_desc; diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c index f1585f8fa0f47..1c90b1810fcb9 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c @@ -409,12 +409,6 @@ static void rkisp1_flt_config(struct rkisp1_params *params, rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_FILT_LUM_WEIGHT, arg->lum_weight); - rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_FILT_MODE, - (arg->mode ? RKISP1_CIF_ISP_FLT_MODE_DNR : 0) | - RKISP1_CIF_ISP_FLT_CHROMA_V_MODE(arg->chr_v_mode) | - RKISP1_CIF_ISP_FLT_CHROMA_H_MODE(arg->chr_h_mode) | - RKISP1_CIF_ISP_FLT_GREEN_STAGE1(arg->grn_stage1)); - /* avoid to override the old enable value */ filt_mode = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_FILT_MODE); filt_mode &= RKISP1_CIF_ISP_FLT_ENA; diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c index 0e7e16f20eeb0..bc74d2d824ef2 100644 --- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c +++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c @@ -923,7 +923,8 @@ static void rkvdec_vp9_done(struct rkvdec_ctx *ctx, update_ctx_last_info(vp9_ctx); } -static void rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx) +static noinline_for_stack void +rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx) { struct rkvdec_vp9_ctx *vp9_ctx = ctx->priv; struct rkvdec_vp9_intra_frame_symbol_counts *intra_cnts = vp9_ctx->count_tbl.cpu; diff --git a/drivers/media/platform/samsung/exynos4-is/media-dev.c b/drivers/media/platform/samsung/exynos4-is/media-dev.c index c781853586fd2..e9d7875cf01ec 100644 --- a/drivers/media/platform/samsung/exynos4-is/media-dev.c +++ b/drivers/media/platform/samsung/exynos4-is/media-dev.c @@ -1399,12 +1399,14 @@ static int subdev_notifier_complete(struct v4l2_async_notifier *notifier) mutex_lock(&fmd->media_dev.graph_mutex); ret = fimc_md_create_links(fmd); - if (ret < 0) - goto unlock; + if (ret < 0) { + mutex_unlock(&fmd->media_dev.graph_mutex); + return ret; + } - ret = v4l2_device_register_subdev_nodes(&fmd->v4l2_dev); -unlock: mutex_unlock(&fmd->media_dev.graph_mutex); + + ret = v4l2_device_register_subdev_nodes(&fmd->v4l2_dev); if (ret < 0) return ret; diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c index 1c1b6b48918ee..b18e273ef4a3e 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c @@ -512,6 +512,9 @@ static void dcmipp_bytecap_stop_streaming(struct vb2_queue *vq) /* Disable pipe */ reg_clear(vcap, DCMIPP_P0FSCR, DCMIPP_P0FSCR_PIPEN); + /* Clear any pending interrupts */ + reg_write(vcap, DCMIPP_CMFCR, DCMIPP_CMIER_P0ALL); + spin_lock_irq(&vcap->irqlock); /* Return all queued buffers to vb2 in ERROR state */ diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c index db76a02a1848a..ec1d773d5ad12 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c @@ -130,11 +130,8 @@ static void dcmipp_byteproc_adjust_compose(struct v4l2_rect *r, r->left = 0; /* Compose is not possible for JPEG or Bayer formats */ - if (fmt->code == MEDIA_BUS_FMT_JPEG_1X8 || - fmt->code == MEDIA_BUS_FMT_SBGGR8_1X8 || - fmt->code == MEDIA_BUS_FMT_SGBRG8_1X8 || - fmt->code == MEDIA_BUS_FMT_SGRBG8_1X8 || - fmt->code == MEDIA_BUS_FMT_SRGGB8_1X8) { + if (fmt->code >= MEDIA_BUS_FMT_SBGGR8_1X8 && + fmt->code <= MEDIA_BUS_FMT_JPEG_1X8) { r->width = fmt->width; r->height = fmt->height; return; diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c index 1b7bae3266c8d..49398d0777646 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c @@ -526,7 +526,12 @@ static int dcmipp_probe(struct platform_device *pdev) return ret; } - kclk = devm_clk_get(&pdev->dev, "kclk"); + /* + * In case of the DCMIPP has only 1 clock (such as on MP13), the + * clock might not be named. + */ + kclk = devm_clk_get(&pdev->dev, + dcmipp->pipe_cfg->needs_mclk ? "kclk" : NULL); if (IS_ERR(kclk)) return dev_err_probe(&pdev->dev, PTR_ERR(kclk), "Unable to get kclk\n"); diff --git a/drivers/media/platform/ti/davinci/vpif_capture.c b/drivers/media/platform/ti/davinci/vpif_capture.c index d053972888d1b..243c6196b0247 100644 --- a/drivers/media/platform/ti/davinci/vpif_capture.c +++ b/drivers/media/platform/ti/davinci/vpif_capture.c @@ -1600,7 +1600,7 @@ vpif_capture_get_pdata(struct platform_device *pdev, * This creates device entries by register itself to the V4L2 driver and * initializes fields of each channel objects */ -static __init int vpif_probe(struct platform_device *pdev) +static int vpif_probe(struct platform_device *pdev) { struct vpif_subdev_info *subdevdata; struct i2c_adapter *i2c_adap; @@ -1807,7 +1807,7 @@ static int vpif_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(vpif_pm_ops, vpif_suspend, vpif_resume); -static __refdata struct platform_driver vpif_driver = { +static struct platform_driver vpif_driver = { .driver = { .name = VPIF_DRIVER_NAME, .pm = &vpif_pm_ops, diff --git a/drivers/media/platform/ti/davinci/vpif_display.c b/drivers/media/platform/ti/davinci/vpif_display.c index 70c89549f4b61..1e7815e9f8e07 100644 --- a/drivers/media/platform/ti/davinci/vpif_display.c +++ b/drivers/media/platform/ti/davinci/vpif_display.c @@ -1214,7 +1214,7 @@ static int vpif_probe_complete(void) * vpif_probe: This function creates device entries by register itself to the * V4L2 driver and initializes fields of each channel objects */ -static __init int vpif_probe(struct platform_device *pdev) +static int vpif_probe(struct platform_device *pdev) { struct vpif_subdev_info *subdevdata; struct i2c_adapter *i2c_adap; @@ -1390,7 +1390,7 @@ static int vpif_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(vpif_pm_ops, vpif_suspend, vpif_resume); -static __refdata struct platform_driver vpif_driver = { +static struct platform_driver vpif_driver = { .driver = { .name = VPIF_DRIVER_NAME, .pm = &vpif_pm_ops, diff --git a/drivers/media/platform/ti/omap3isp/isppreview.c b/drivers/media/platform/ti/omap3isp/isppreview.c index e383a57654de8..5c492b31b5160 100644 --- a/drivers/media/platform/ti/omap3isp/isppreview.c +++ b/drivers/media/platform/ti/omap3isp/isppreview.c @@ -1742,22 +1742,17 @@ static void preview_try_format(struct isp_prev_device *prev, switch (pad) { case PREV_PAD_SINK: - /* When reading data from the CCDC, the input size has already - * been mangled by the CCDC output pad so it can be accepted - * as-is. - * - * When reading data from memory, clamp the requested width and - * height. The TRM doesn't specify a minimum input height, make + /* + * Clamp the requested width and height. + * The TRM doesn't specify a minimum input height, make * sure we got enough lines to enable the noise filter and color * filter array interpolation. */ - if (prev->input == PREVIEW_INPUT_MEMORY) { - fmt->width = clamp_t(u32, fmt->width, PREV_MIN_IN_WIDTH, - preview_max_out_width(prev)); - fmt->height = clamp_t(u32, fmt->height, - PREV_MIN_IN_HEIGHT, - PREV_MAX_IN_HEIGHT); - } + fmt->width = clamp_t(u32, fmt->width, PREV_MIN_IN_WIDTH, + preview_max_out_width(prev)); + fmt->height = clamp_t(u32, fmt->height, + PREV_MIN_IN_HEIGHT, + PREV_MAX_IN_HEIGHT); fmt->colorspace = V4L2_COLORSPACE_SRGB; diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c index 0e7f0bf2b3463..eb33a776f27c9 100644 --- a/drivers/media/platform/ti/omap3isp/ispvideo.c +++ b/drivers/media/platform/ti/omap3isp/ispvideo.c @@ -148,12 +148,12 @@ static unsigned int isp_video_mbus_to_pix(const struct isp_video *video, pix->width = mbus->width; pix->height = mbus->height; - for (i = 0; i < ARRAY_SIZE(formats); ++i) { + for (i = 0; i < ARRAY_SIZE(formats) - 1; ++i) { if (formats[i].code == mbus->code) break; } - if (WARN_ON(i == ARRAY_SIZE(formats))) + if (WARN_ON(i == ARRAY_SIZE(formats) - 1)) return 0; min_bpl = pix->width * formats[i].bpp; @@ -191,7 +191,7 @@ static void isp_video_pix_to_mbus(const struct v4l2_pix_format *pix, /* Skip the last format in the loop so that it will be selected if no * match is found. */ - for (i = 0; i < ARRAY_SIZE(formats) - 1; ++i) { + for (i = 0; i < ARRAY_SIZE(formats) - 2; ++i) { if (formats[i].pixelformat == pix->pixelformat) break; } @@ -1288,6 +1288,7 @@ static const struct v4l2_ioctl_ops isp_video_ioctl_ops = { static int isp_video_open(struct file *file) { struct isp_video *video = video_drvdata(file); + struct v4l2_mbus_framefmt fmt; struct isp_video_fh *handle; struct vb2_queue *queue; int ret = 0; @@ -1330,6 +1331,13 @@ static int isp_video_open(struct file *file) memset(&handle->format, 0, sizeof(handle->format)); handle->format.type = video->type; + handle->format.fmt.pix.width = 720; + handle->format.fmt.pix.height = 480; + handle->format.fmt.pix.pixelformat = V4L2_PIX_FMT_UYVY; + handle->format.fmt.pix.field = V4L2_FIELD_NONE; + handle->format.fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + isp_video_pix_to_mbus(&handle->format.fmt.pix, &fmt); + isp_video_mbus_to_pix(video, &fmt, &handle->format.fmt.pix); handle->timeperframe.denominator = 1; handle->video = video; diff --git a/drivers/media/platform/verisilicon/hantro_g2.c b/drivers/media/platform/verisilicon/hantro_g2.c index aae0b562fabb1..318673b66da88 100644 --- a/drivers/media/platform/verisilicon/hantro_g2.c +++ b/drivers/media/platform/verisilicon/hantro_g2.c @@ -5,43 +5,93 @@ * Copyright (C) 2021 Collabora Ltd, Andrzej Pietrasiewicz */ +#include #include "hantro_hw.h" #include "hantro_g2_regs.h" #define G2_ALIGN 16 -void hantro_g2_check_idle(struct hantro_dev *vpu) +static bool hantro_g2_active(struct hantro_ctx *ctx) { - int i; - - for (i = 0; i < 3; i++) { - u32 status; - - /* Make sure the VPU is idle */ - status = vdpu_read(vpu, G2_REG_INTERRUPT); - if (status & G2_REG_INTERRUPT_DEC_E) { - dev_warn(vpu->dev, "device still running, aborting"); - status |= G2_REG_INTERRUPT_DEC_ABORT_E | G2_REG_INTERRUPT_DEC_IRQ_DIS; - vdpu_write(vpu, status, G2_REG_INTERRUPT); - } + struct hantro_dev *vpu = ctx->dev; + u32 status; + + status = vdpu_read(vpu, G2_REG_INTERRUPT); + + return (status & G2_REG_INTERRUPT_DEC_E); +} + +/** + * hantro_g2_reset: + * @ctx: the hantro context + * + * Emulates a reset using Hantro abort function. Failing this procedure would + * results in programming a running IP which leads to CPU hang. + * + * Using a hard reset procedure instead is prefferred. + */ +void hantro_g2_reset(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + u32 status; + + status = vdpu_read(vpu, G2_REG_INTERRUPT); + if (status & G2_REG_INTERRUPT_DEC_E) { + dev_warn_ratelimited(vpu->dev, "device still running, aborting"); + status |= G2_REG_INTERRUPT_DEC_ABORT_E | G2_REG_INTERRUPT_DEC_IRQ_DIS; + vdpu_write(vpu, status, G2_REG_INTERRUPT); + + do { + mdelay(1); + } while (hantro_g2_active(ctx)); } } irqreturn_t hantro_g2_irq(int irq, void *dev_id) { struct hantro_dev *vpu = dev_id; - enum vb2_buffer_state state; u32 status; status = vdpu_read(vpu, G2_REG_INTERRUPT); - state = (status & G2_REG_INTERRUPT_DEC_RDY_INT) ? - VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; - vdpu_write(vpu, 0, G2_REG_INTERRUPT); - vdpu_write(vpu, G2_REG_CONFIG_DEC_CLK_GATE_E, G2_REG_CONFIG); + if (!(status & G2_REG_INTERRUPT_DEC_IRQ)) + return IRQ_NONE; + + hantro_reg_write(vpu, &g2_dec_irq, 0); + hantro_reg_write(vpu, &g2_dec_int_stat, 0); + hantro_reg_write(vpu, &g2_clk_gate_e, 1); + + if (status & G2_REG_INTERRUPT_DEC_RDY_INT) { + hantro_irq_done(vpu, VB2_BUF_STATE_DONE); + return IRQ_HANDLED; + } + + if (status & G2_REG_INTERRUPT_DEC_ABORT_INT) { + /* disabled on abort, though lets be safe and handle it */ + dev_warn_ratelimited(vpu->dev, "decode operation aborted."); + return IRQ_HANDLED; + } + + if (status & G2_REG_INTERRUPT_DEC_LAST_SLICE_INT) + dev_warn_ratelimited(vpu->dev, "not all macroblocks were decoded."); + + if (status & G2_REG_INTERRUPT_DEC_BUS_INT) + dev_warn_ratelimited(vpu->dev, "bus error detected."); + + if (status & G2_REG_INTERRUPT_DEC_ERROR_INT) + dev_warn_ratelimited(vpu->dev, "decode error detected."); + + if (status & G2_REG_INTERRUPT_DEC_TIMEOUT) + dev_warn_ratelimited(vpu->dev, "frame decode timed out."); - hantro_irq_done(vpu, state); + /** + * If the decoding haven't stopped, let it continue. The hardware timeout + * will trigger if it is trully stuck. + */ + if (status & G2_REG_INTERRUPT_DEC_E) + return IRQ_HANDLED; + hantro_irq_done(vpu, VB2_BUF_STATE_ERROR); return IRQ_HANDLED; } diff --git a/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c b/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c index 0e212198dd65b..e8c2e83379def 100644 --- a/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c +++ b/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c @@ -283,6 +283,15 @@ static void set_params(struct hantro_ctx *ctx) hantro_reg_write(vpu, &g2_apf_threshold, 8); } +static u32 get_dpb_index(const struct v4l2_ctrl_hevc_decode_params *decode_params, + const u32 index) +{ + if (index > decode_params->num_active_dpb_entries) + return 0; + + return index; +} + static void set_ref_pic_list(struct hantro_ctx *ctx) { const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; @@ -355,8 +364,10 @@ static void set_ref_pic_list(struct hantro_ctx *ctx) list1[j++] = list1[i++]; for (i = 0; i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) { - hantro_reg_write(vpu, &ref_pic_regs0[i], list0[i]); - hantro_reg_write(vpu, &ref_pic_regs1[i], list1[i]); + hantro_reg_write(vpu, &ref_pic_regs0[i], + get_dpb_index(decode_params, list0[i])); + hantro_reg_write(vpu, &ref_pic_regs1[i], + get_dpb_index(decode_params, list1[i])); } } @@ -582,8 +593,6 @@ int hantro_g2_hevc_dec_run(struct hantro_ctx *ctx) struct hantro_dev *vpu = ctx->dev; int ret; - hantro_g2_check_idle(vpu); - /* Prepare HEVC decoder context. */ ret = hantro_hevc_dec_prepare_run(ctx); if (ret) diff --git a/drivers/media/platform/verisilicon/hantro_g2_regs.h b/drivers/media/platform/verisilicon/hantro_g2_regs.h index b943b1816db7f..c614951121c79 100644 --- a/drivers/media/platform/verisilicon/hantro_g2_regs.h +++ b/drivers/media/platform/verisilicon/hantro_g2_regs.h @@ -22,7 +22,14 @@ #define G2_REG_VERSION G2_SWREG(0) #define G2_REG_INTERRUPT G2_SWREG(1) +#define G2_REG_INTERRUPT_DEC_LAST_SLICE_INT BIT(19) +#define G2_REG_INTERRUPT_DEC_TIMEOUT BIT(18) +#define G2_REG_INTERRUPT_DEC_ERROR_INT BIT(16) +#define G2_REG_INTERRUPT_DEC_BUF_INT BIT(14) +#define G2_REG_INTERRUPT_DEC_BUS_INT BIT(13) #define G2_REG_INTERRUPT_DEC_RDY_INT BIT(12) +#define G2_REG_INTERRUPT_DEC_ABORT_INT BIT(11) +#define G2_REG_INTERRUPT_DEC_IRQ BIT(8) #define G2_REG_INTERRUPT_DEC_ABORT_E BIT(5) #define G2_REG_INTERRUPT_DEC_IRQ_DIS BIT(4) #define G2_REG_INTERRUPT_DEC_E BIT(0) @@ -35,6 +42,9 @@ #define BUS_WIDTH_128 2 #define BUS_WIDTH_256 3 +#define g2_dec_int_stat G2_DEC_REG(1, 11, 0xf) +#define g2_dec_irq G2_DEC_REG(1, 8, 0x1) + #define g2_strm_swap G2_DEC_REG(2, 28, 0xf) #define g2_strm_swap_old G2_DEC_REG(2, 27, 0x1f) #define g2_pic_swap G2_DEC_REG(2, 22, 0x1f) @@ -225,6 +235,9 @@ #define vp9_filt_level_seg5 G2_DEC_REG(19, 8, 0x3f) #define vp9_quant_seg5 G2_DEC_REG(19, 0, 0xff) +#define g2_timemout_override_e G2_DEC_REG(45, 31, 0x1) +#define g2_timemout_cycles G2_DEC_REG(45, 0, 0x7fffffff) + #define hevc_cur_poc_00 G2_DEC_REG(46, 24, 0xff) #define hevc_cur_poc_01 G2_DEC_REG(46, 16, 0xff) #define hevc_cur_poc_02 G2_DEC_REG(46, 8, 0xff) diff --git a/drivers/media/platform/verisilicon/hantro_g2_vp9_dec.c b/drivers/media/platform/verisilicon/hantro_g2_vp9_dec.c index 82a478ac645e1..56c79e339030e 100644 --- a/drivers/media/platform/verisilicon/hantro_g2_vp9_dec.c +++ b/drivers/media/platform/verisilicon/hantro_g2_vp9_dec.c @@ -893,8 +893,6 @@ int hantro_g2_vp9_dec_run(struct hantro_ctx *ctx) struct vb2_v4l2_buffer *dst; int ret; - hantro_g2_check_idle(ctx->dev); - ret = start_prepare_run(ctx, &decode_params); if (ret) { hantro_end_prepare_run(ctx); diff --git a/drivers/media/platform/verisilicon/hantro_hw.h b/drivers/media/platform/verisilicon/hantro_hw.h index c9b6556f8b2b7..5f2011529f02d 100644 --- a/drivers/media/platform/verisilicon/hantro_hw.h +++ b/drivers/media/platform/verisilicon/hantro_hw.h @@ -583,6 +583,7 @@ void hantro_g2_vp9_dec_done(struct hantro_ctx *ctx); int hantro_vp9_dec_init(struct hantro_ctx *ctx); void hantro_vp9_dec_exit(struct hantro_ctx *ctx); void hantro_g2_check_idle(struct hantro_dev *vpu); +void hantro_g2_reset(struct hantro_ctx *ctx); irqreturn_t hantro_g2_irq(int irq, void *dev_id); #endif /* HANTRO_HW_H_ */ diff --git a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c index f9f276385c117..5be0e2e76882f 100644 --- a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c +++ b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c @@ -294,11 +294,13 @@ static const struct hantro_codec_ops imx8mq_vpu_g1_codec_ops[] = { static const struct hantro_codec_ops imx8mq_vpu_g2_codec_ops[] = { [HANTRO_MODE_HEVC_DEC] = { .run = hantro_g2_hevc_dec_run, + .reset = hantro_g2_reset, .init = hantro_hevc_dec_init, .exit = hantro_hevc_dec_exit, }, [HANTRO_MODE_VP9_DEC] = { .run = hantro_g2_vp9_dec_run, + .reset = hantro_g2_reset, .done = hantro_g2_vp9_dec_done, .init = hantro_vp9_dec_init, .exit = hantro_vp9_dec_exit, diff --git a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c index e4703bb6be7c1..e4e21ad373233 100644 --- a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c +++ b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c @@ -72,6 +72,14 @@ : AV1_DIV_ROUND_UP_POW2((_value_), (_n_))); \ }) +enum rockchip_av1_tx_mode { + ROCKCHIP_AV1_TX_MODE_ONLY_4X4 = 0, + ROCKCHIP_AV1_TX_MODE_8X8 = 1, + ROCKCHIP_AV1_TX_MODE_16x16 = 2, + ROCKCHIP_AV1_TX_MODE_32x32 = 3, + ROCKCHIP_AV1_TX_MODE_SELECT = 4, +}; + struct rockchip_av1_film_grain { u8 scaling_lut_y[256]; u8 scaling_lut_cb[256]; @@ -373,12 +381,12 @@ int rockchip_vpu981_av1_dec_init(struct hantro_ctx *ctx) return -ENOMEM; av1_dec->global_model.size = GLOBAL_MODEL_SIZE; - av1_dec->tile_info.cpu = dma_alloc_coherent(vpu->dev, AV1_MAX_TILES, + av1_dec->tile_info.cpu = dma_alloc_coherent(vpu->dev, AV1_TILE_INFO_SIZE, &av1_dec->tile_info.dma, GFP_KERNEL); if (!av1_dec->tile_info.cpu) return -ENOMEM; - av1_dec->tile_info.size = AV1_MAX_TILES; + av1_dec->tile_info.size = AV1_TILE_INFO_SIZE; av1_dec->film_grain.cpu = dma_alloc_coherent(vpu->dev, ALIGN(sizeof(struct rockchip_av1_film_grain), 2048), @@ -1396,8 +1404,16 @@ static void rockchip_vpu981_av1_dec_set_cdef(struct hantro_ctx *ctx) u16 luma_sec_strength = 0; u32 chroma_pri_strength = 0; u16 chroma_sec_strength = 0; + bool enable_cdef; int i; + enable_cdef = !(cdef->bits == 0 && + cdef->damping_minus_3 == 0 && + cdef->y_pri_strength[0] == 0 && + cdef->y_sec_strength[0] == 0 && + cdef->uv_pri_strength[0] == 0 && + cdef->uv_sec_strength[0] == 0); + hantro_reg_write(vpu, &av1_enable_cdef, enable_cdef); hantro_reg_write(vpu, &av1_cdef_bits, cdef->bits); hantro_reg_write(vpu, &av1_cdef_damping, cdef->damping_minus_3); @@ -1927,11 +1943,26 @@ static void rockchip_vpu981_av1_dec_set_reference_frames(struct hantro_ctx *ctx) rockchip_vpu981_av1_dec_set_other_frames(ctx); } +static int rockchip_vpu981_av1_get_hardware_tx_mode(enum v4l2_av1_tx_mode tx_mode) +{ + switch (tx_mode) { + case V4L2_AV1_TX_MODE_ONLY_4X4: + return ROCKCHIP_AV1_TX_MODE_ONLY_4X4; + case V4L2_AV1_TX_MODE_LARGEST: + return ROCKCHIP_AV1_TX_MODE_32x32; + case V4L2_AV1_TX_MODE_SELECT: + return ROCKCHIP_AV1_TX_MODE_SELECT; + } + + return ROCKCHIP_AV1_TX_MODE_32x32; +} + static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; struct hantro_av1_dec_hw_ctx *av1_dec = &ctx->av1_dec; struct hantro_av1_dec_ctrls *ctrls = &av1_dec->ctrls; + int tx_mode; hantro_reg_write(vpu, &av1_skip_mode, !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_SKIP_MODE_PRESENT)); @@ -1953,8 +1984,6 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_SHOW_FRAME)); hantro_reg_write(vpu, &av1_switchable_motion_mode, !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_IS_MOTION_MODE_SWITCHABLE)); - hantro_reg_write(vpu, &av1_enable_cdef, - !!(ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF)); hantro_reg_write(vpu, &av1_allow_masked_compound, !!(ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_MASKED_COMPOUND)); @@ -1989,7 +2018,7 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->quantization.flags & V4L2_AV1_QUANTIZATION_FLAG_DELTA_Q_PRESENT)); - hantro_reg_write(vpu, &av1_idr_pic_e, !ctrls->frame->frame_type); + hantro_reg_write(vpu, &av1_idr_pic_e, IS_INTRA(ctrls->frame->frame_type)); hantro_reg_write(vpu, &av1_quant_base_qindex, ctrls->frame->quantization.base_q_idx); hantro_reg_write(vpu, &av1_bit_depth_y_minus8, ctx->bit_depth - 8); hantro_reg_write(vpu, &av1_bit_depth_c_minus8, ctx->bit_depth - 8); @@ -1999,7 +2028,9 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV)); hantro_reg_write(vpu, &av1_comp_pred_mode, (ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_REFERENCE_SELECT) ? 2 : 0); - hantro_reg_write(vpu, &av1_transform_mode, (ctrls->frame->tx_mode == 1) ? 3 : 4); + + tx_mode = rockchip_vpu981_av1_get_hardware_tx_mode(ctrls->frame->tx_mode); + hantro_reg_write(vpu, &av1_transform_mode, tx_mode); hantro_reg_write(vpu, &av1_max_cb_size, (ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK) ? 7 : 6); diff --git a/drivers/media/radio/radio-keene.c b/drivers/media/radio/radio-keene.c index f3b57f0cb1ec4..c133305fd0194 100644 --- a/drivers/media/radio/radio-keene.c +++ b/drivers/media/radio/radio-keene.c @@ -338,7 +338,6 @@ static int usb_keene_probe(struct usb_interface *intf, if (hdl->error) { retval = hdl->error; - v4l2_ctrl_handler_free(hdl); goto err_v4l2; } retval = v4l2_device_register(&intf->dev, &radio->v4l2_dev); @@ -384,6 +383,7 @@ static int usb_keene_probe(struct usb_interface *intf, err_vdev: v4l2_device_unregister(&radio->v4l2_dev); err_v4l2: + v4l2_ctrl_handler_free(&radio->hdl); kfree(radio->buffer); kfree(radio); err: diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c index 6539fa0a6e790..6b70bac5f45d6 100644 --- a/drivers/media/rc/st_rc.c +++ b/drivers/media/rc/st_rc.c @@ -284,7 +284,7 @@ static int st_rc_probe(struct platform_device *pdev) else rc_dev->rx_base = rc_dev->base; - rc_dev->rstc = reset_control_get_optional_exclusive(dev, NULL); + rc_dev->rstc = devm_reset_control_get_optional_exclusive(dev, NULL); if (IS_ERR(rc_dev->rstc)) { ret = PTR_ERR(rc_dev->rstc); goto err; diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c index 438483c62facc..52b2abe16dcf5 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c +++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c @@ -237,8 +237,10 @@ static int vidtv_start_feed(struct dvb_demux_feed *feed) if (dvb->nfeeds == 1) { ret = vidtv_start_streaming(dvb); - if (ret < 0) + if (ret < 0) { + dvb->nfeeds--; rc = ret; + } } mutex_unlock(&dvb->feed_lock); diff --git a/drivers/media/test-drivers/vidtv/vidtv_channel.c b/drivers/media/test-drivers/vidtv/vidtv_channel.c index f3023e91b3ebc..aa177cf96b6ac 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_channel.c +++ b/drivers/media/test-drivers/vidtv/vidtv_channel.c @@ -341,6 +341,10 @@ vidtv_channel_pmt_match_sections(struct vidtv_channel *channels, tail = vidtv_psi_pmt_stream_init(tail, s->type, e_pid); + if (!tail) { + vidtv_psi_pmt_stream_destroy(head); + return; + } if (!head) head = tail; @@ -461,12 +465,15 @@ int vidtv_channel_si_init(struct vidtv_mux *m) /* assemble all programs and assign to PAT */ vidtv_psi_pat_program_assign(m->si.pat, programs); + programs = NULL; /* assemble all services and assign to SDT */ vidtv_psi_sdt_service_assign(m->si.sdt, services); + services = NULL; /* assemble all events and assign to EIT */ vidtv_psi_eit_event_assign(m->si.eit, events); + events = NULL; m->si.pmt_secs = vidtv_psi_pmt_create_sec_for_each_pat_entry(m->si.pat, m->pcr_pid); diff --git a/drivers/media/test-drivers/vidtv/vidtv_mux.c b/drivers/media/test-drivers/vidtv/vidtv_mux.c index f99878eff7ace..7dad97881fdb0 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_mux.c +++ b/drivers/media/test-drivers/vidtv/vidtv_mux.c @@ -233,7 +233,7 @@ static u32 vidtv_mux_push_pcr(struct vidtv_mux *m) /* the 27Mhz clock will feed both parts of the PCR bitfield */ args.pcr = m->timing.clk; - nbytes += vidtv_ts_pcr_write_into(args); + nbytes += vidtv_ts_pcr_write_into(&args); m->mux_buf_offset += nbytes; m->num_streamed_pcr++; @@ -363,7 +363,7 @@ static u32 vidtv_mux_pad_with_nulls(struct vidtv_mux *m, u32 npkts) args.continuity_counter = &ctx->cc; for (i = 0; i < npkts; ++i) { - m->mux_buf_offset += vidtv_ts_null_write_into(args); + m->mux_buf_offset += vidtv_ts_null_write_into(&args); args.dest_offset = m->mux_buf_offset; } diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.c b/drivers/media/test-drivers/vidtv/vidtv_ts.c index ca4bb9c40b78e..cbe9aff9ffb56 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_ts.c +++ b/drivers/media/test-drivers/vidtv/vidtv_ts.c @@ -48,7 +48,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter) *continuity_counter = 0; } -u32 vidtv_ts_null_write_into(struct null_packet_write_args args) +u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args) { u32 nbytes = 0; struct vidtv_mpeg_ts ts_header = {}; @@ -56,21 +56,21 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args) ts_header.sync_byte = TS_SYNC_BYTE; ts_header.bitfield = cpu_to_be16(TS_NULL_PACKET_PID); ts_header.payload = 1; - ts_header.continuity_counter = *args.continuity_counter; + ts_header.continuity_counter = *args->continuity_counter; /* copy TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_header, sizeof(ts_header)); - vidtv_ts_inc_cc(args.continuity_counter); + vidtv_ts_inc_cc(args->continuity_counter); /* fill the rest with empty data */ - nbytes += vidtv_memset(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memset(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, TS_FILL_BYTE, TS_PACKET_LEN - nbytes); @@ -83,17 +83,17 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args) return nbytes; } -u32 vidtv_ts_pcr_write_into(struct pcr_write_args args) +u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args) { u32 nbytes = 0; struct vidtv_mpeg_ts ts_header = {}; struct vidtv_mpeg_ts_adaption ts_adap = {}; ts_header.sync_byte = TS_SYNC_BYTE; - ts_header.bitfield = cpu_to_be16(args.pid); + ts_header.bitfield = cpu_to_be16(args->pid); ts_header.scrambling = 0; /* cc is not incremented, but it is needed. see 13818-1 clause 2.4.3.3 */ - ts_header.continuity_counter = *args.continuity_counter; + ts_header.continuity_counter = *args->continuity_counter; ts_header.payload = 0; ts_header.adaptation_field = 1; @@ -102,27 +102,27 @@ u32 vidtv_ts_pcr_write_into(struct pcr_write_args args) ts_adap.PCR = 1; /* copy TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_header, sizeof(ts_header)); /* write the adap after the TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_adap, sizeof(ts_adap)); /* write the PCR optional */ - nbytes += vidtv_ts_write_pcr_bits(args.dest_buf, - args.dest_offset + nbytes, - args.pcr); + nbytes += vidtv_ts_write_pcr_bits(args->dest_buf, + args->dest_offset + nbytes, + args->pcr); - nbytes += vidtv_memset(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memset(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, TS_FILL_BYTE, TS_PACKET_LEN - nbytes); diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.h b/drivers/media/test-drivers/vidtv/vidtv_ts.h index 09b4ffd028292..3606398e160d9 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_ts.h +++ b/drivers/media/test-drivers/vidtv/vidtv_ts.h @@ -90,7 +90,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter); * * Return: The number of bytes written into the buffer. */ -u32 vidtv_ts_null_write_into(struct null_packet_write_args args); +u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args); /** * vidtv_ts_pcr_write_into - Write a PCR packet into a buffer. @@ -101,6 +101,6 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args); * * Return: The number of bytes written into the buffer. */ -u32 vidtv_ts_pcr_write_into(struct pcr_write_args args); +u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args); #endif //VIDTV_TS_H diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c index e0ef66a522e23..44565f0297cdb 100644 --- a/drivers/media/usb/as102/as102_usb_drv.c +++ b/drivers/media/usb/as102/as102_usb_drv.c @@ -403,7 +403,9 @@ static int as102_usb_probe(struct usb_interface *intf, failed_dvb: as102_free_usb_stream_buffer(as102_dev); failed_stream: + usb_set_intfdata(intf, NULL); usb_deregister_dev(intf, &as102_usb_class_driver); + return ret; failed: usb_put_dev(as102_dev->bus_adap.usb_dev); usb_set_intfdata(intf, NULL); diff --git a/drivers/media/usb/dvb-usb/dtv5100.c b/drivers/media/usb/dvb-usb/dtv5100.c index 3d85c6f7f6ecf..c448e2ebda1ac 100644 --- a/drivers/media/usb/dvb-usb/dtv5100.c +++ b/drivers/media/usb/dvb-usb/dtv5100.c @@ -55,6 +55,11 @@ static int dtv5100_i2c_msg(struct dvb_usb_device *d, u8 addr, } index = (addr << 8) + wbuf[0]; + if (rlen > sizeof(st->data)) { + warn("rlen = %x is too big!\n", rlen); + return -EINVAL; + } + memcpy(st->data, rbuf, rlen); msleep(1); /* avoid I2C errors */ return usb_control_msg(d->udev, pipe, request, diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c index 2dfa3242a7ab5..14c35995cd952 100644 --- a/drivers/media/usb/em28xx/em28xx-video.c +++ b/drivers/media/usb/em28xx/em28xx-video.c @@ -2126,7 +2126,7 @@ static int em28xx_v4l2_open(struct file *filp) { struct video_device *vdev = video_devdata(filp); struct em28xx *dev = video_drvdata(filp); - struct em28xx_v4l2 *v4l2 = dev->v4l2; + struct em28xx_v4l2 *v4l2; enum v4l2_buf_type fh_type = 0; int ret; @@ -2143,13 +2143,19 @@ static int em28xx_v4l2_open(struct file *filp) return -EINVAL; } + if (mutex_lock_interruptible(&dev->lock)) + return -ERESTARTSYS; + + v4l2 = dev->v4l2; + if (!v4l2) { + mutex_unlock(&dev->lock); + return -ENODEV; + } + em28xx_videodbg("open dev=%s type=%s users=%d\n", video_device_node_name(vdev), v4l2_type_names[fh_type], v4l2->users); - if (mutex_lock_interruptible(&dev->lock)) - return -ERESTARTSYS; - ret = v4l2_fh_open(filp); if (ret) { dev_err(&dev->intf->dev, diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c index 0b50de8775a38..c3c4247194d19 100644 --- a/drivers/media/usb/hackrf/hackrf.c +++ b/drivers/media/usb/hackrf/hackrf.c @@ -1485,7 +1485,7 @@ static int hackrf_probe(struct usb_interface *intf, if (ret) { dev_err(dev->dev, "Failed to register as video device (%d)\n", ret); - goto err_v4l2_device_unregister; + goto err_v4l2_device_put; } dev_info(dev->dev, "Registered as %s\n", video_device_node_name(&dev->rx_vdev)); @@ -1513,8 +1513,9 @@ static int hackrf_probe(struct usb_interface *intf, return 0; err_video_unregister_device_rx: video_unregister_device(&dev->rx_vdev); -err_v4l2_device_unregister: - v4l2_device_unregister(&dev->v4l2_dev); +err_v4l2_device_put: + v4l2_device_put(&dev->v4l2_dev); + return ret; err_v4l2_ctrl_handler_free_tx: v4l2_ctrl_handler_free(&dev->tx_ctrl_handler); err_v4l2_ctrl_handler_free_rx: diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c index f21c2806eb9fe..5807734ae26c6 100644 --- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c +++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c @@ -3622,7 +3622,7 @@ static int pvr2_send_request_ex(struct pvr2_hdw *hdw, pvr2_trace( PVR2_TRACE_ERROR_LEGS, "Attempted to execute %d byte control-read transfer (limit=%d)", - write_len,PVR2_CTL_BUFFSIZE); + read_len, PVR2_CTL_BUFFSIZE); return -EINVAL; } if ((!write_len) && (!read_len)) { @@ -3709,6 +3709,11 @@ status); "Failed to submit read-control URB status=%d", status); hdw->ctl_read_pend_flag = 0; + if (hdw->ctl_write_pend_flag) { + usb_unlink_urb(hdw->ctl_write_urb); + while (hdw->ctl_write_pend_flag) + wait_for_completion(&hdw->ctl_done); + } goto done; } } diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c index 2905505c240c0..2738ef74c7373 100644 --- a/drivers/media/usb/uvc/uvc_ctrl.c +++ b/drivers/media/usb/uvc/uvc_ctrl.c @@ -1432,7 +1432,7 @@ static bool uvc_ctrl_is_readable(u32 which, struct uvc_control *ctrl, * auto_exposure=1, exposure_time_absolute=251. */ int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, - const struct v4l2_ext_controls *ctrls, + const struct v4l2_ext_controls *ctrls, u32 which, unsigned long ioctl) { struct uvc_control_mapping *master_map = NULL; @@ -1442,14 +1442,24 @@ int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, s32 val; int ret; int i; + /* + * There is no need to check the ioctl, all the ioctls except + * VIDIOC_G_EXT_CTRLS use which=V4L2_CTRL_WHICH_CUR_VAL. + */ + bool is_which_min_max = which == V4L2_CTRL_WHICH_MIN_VAL || + which == V4L2_CTRL_WHICH_MAX_VAL; if (__uvc_query_v4l2_class(chain, v4l2_id, 0) >= 0) - return -EACCES; + return is_which_min_max ? -EINVAL : -EACCES; ctrl = uvc_find_control(chain, v4l2_id, &mapping); if (!ctrl) return -EINVAL; + if ((!(ctrl->info.flags & UVC_CTRL_FLAG_GET_MIN) || + !(ctrl->info.flags & UVC_CTRL_FLAG_GET_MAX)) && is_which_min_max) + return -EINVAL; + if (ioctl == VIDIOC_G_EXT_CTRLS) return uvc_ctrl_is_readable(ctrls->which, ctrl, mapping); diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c index ee4f54d683496..aa3e8d295e0f5 100644 --- a/drivers/media/usb/uvc/uvc_driver.c +++ b/drivers/media/usb/uvc/uvc_driver.c @@ -165,28 +165,17 @@ static struct uvc_entity *uvc_entity_by_reference(struct uvc_device *dev, return NULL; } -static struct uvc_streaming *uvc_stream_by_id(struct uvc_device *dev, int id) +static struct uvc_streaming *uvc_stream_for_terminal(struct uvc_device *dev, + struct uvc_entity *term) { - struct uvc_streaming *stream, *last_stream; - unsigned int count = 0; + u16 id = UVC_HARDWARE_ENTITY_ID(term->id); + struct uvc_streaming *stream; list_for_each_entry(stream, &dev->streams, list) { - count += 1; - last_stream = stream; if (stream->header.bTerminalLink == id) return stream; } - /* - * If the streaming entity is referenced by an invalid ID, notify the - * user and use heuristics to guess the correct entity. - */ - if (count == 1 && id == UVC_INVALID_ENTITY_ID) { - dev_warn(&dev->intf->dev, - "UVC non compliance: Invalid USB header. The streaming entity has an invalid ID, guessing the correct one."); - return last_stream; - } - return NULL; } @@ -823,10 +812,12 @@ static struct uvc_entity *uvc_alloc_new_entity(struct uvc_device *dev, u16 type, } /* Per UVC 1.1+ spec 3.7.2, the ID is unique. */ - if (uvc_entity_by_id(dev, id)) { - dev_err(&dev->intf->dev, "Found multiple Units with ID %u\n", id); + if (uvc_entity_by_id(dev, UVC_HARDWARE_ENTITY_ID(id))) + dev_err(&dev->intf->dev, "Found multiple Units with ID %u\n", + UVC_HARDWARE_ENTITY_ID(id)); + + if (uvc_entity_by_id(dev, id)) id = UVC_INVALID_ENTITY_ID; - } extra_size = roundup(extra_size, sizeof(*entity->pads)); if (num_pads) @@ -982,6 +973,7 @@ static int uvc_parse_standard_control(struct uvc_device *dev, struct usb_host_interface *alts = dev->intf->cur_altsetting; unsigned int i, n, p, len; const char *type_name; + unsigned int id; u16 type; switch (buffer[2]) { @@ -1120,8 +1112,28 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return 0; } + id = buffer[3]; + + /* + * Some devices, such as the Grandstream GUV3100, exhibit entity + * ID collisions between units and streaming output terminals. + * Move streaming output terminals to their own ID namespace by + * setting bit UVC_TERM_OUTPUT (15), above the ID's 8-bit value. + * The bit is ignored in uvc_stream_for_terminal() when looking + * up the streaming interface for the terminal. + * + * This hack is safe to enable unconditionally, as the ID is not + * used for any other purpose (streaming output terminals have + * no controls and are never referenced as sources in UVC + * descriptors). Other types output terminals can have controls, + * so limit usage of this separate namespace to streaming output + * terminals. + */ + if (type & UVC_TT_STREAMING) + id |= UVC_TERM_OUTPUT; + term = uvc_alloc_new_entity(dev, type | UVC_TERM_OUTPUT, - buffer[3], 1, 0); + id, 1, 0); if (IS_ERR(term)) return PTR_ERR(term); @@ -2118,8 +2130,8 @@ static int uvc_register_terms(struct uvc_device *dev, if (UVC_ENTITY_TYPE(term) != UVC_TT_STREAMING) continue; - stream = uvc_stream_by_id(dev, term->id); - if (stream == NULL) { + stream = uvc_stream_for_terminal(dev, term); + if (!stream) { dev_info(&dev->intf->dev, "No streaming interface found for terminal %u.", term->id); diff --git a/drivers/media/usb/uvc/uvc_queue.c b/drivers/media/usb/uvc/uvc_queue.c index 790184c9843d2..e838c6c1893a6 100644 --- a/drivers/media/usb/uvc/uvc_queue.c +++ b/drivers/media/usb/uvc/uvc_queue.c @@ -177,18 +177,20 @@ static int uvc_start_streaming_video(struct vb2_queue *vq, unsigned int count) ret = uvc_pm_get(stream->dev); if (ret) - return ret; + goto err_buffers; queue->buf_used = 0; ret = uvc_video_start_streaming(stream); - if (ret == 0) - return 0; + if (ret) + goto err_pm; - uvc_pm_put(stream->dev); + return 0; +err_pm: + uvc_pm_put(stream->dev); +err_buffers: uvc_queue_return_buffers(queue, UVC_BUF_STATE_QUEUED); - return ret; } diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c index 9e4a251eca880..30c160daed8cb 100644 --- a/drivers/media/usb/uvc/uvc_v4l2.c +++ b/drivers/media/usb/uvc/uvc_v4l2.c @@ -765,14 +765,15 @@ static int uvc_ioctl_query_ext_ctrl(struct file *file, void *priv, static int uvc_ctrl_check_access(struct uvc_video_chain *chain, struct v4l2_ext_controls *ctrls, - unsigned long ioctl) + u32 which, unsigned long ioctl) { struct v4l2_ext_control *ctrl = ctrls->controls; unsigned int i; int ret = 0; for (i = 0; i < ctrls->count; ++ctrl, ++i) { - ret = uvc_ctrl_is_accessible(chain, ctrl->id, ctrls, ioctl); + ret = uvc_ctrl_is_accessible(chain, ctrl->id, ctrls, which, + ioctl); if (ret) break; } @@ -806,7 +807,7 @@ static int uvc_ioctl_g_ext_ctrls(struct file *file, void *priv, which = V4L2_CTRL_WHICH_CUR_VAL; } - ret = uvc_ctrl_check_access(chain, ctrls, VIDIOC_G_EXT_CTRLS); + ret = uvc_ctrl_check_access(chain, ctrls, which, VIDIOC_G_EXT_CTRLS); if (ret < 0) return ret; @@ -840,7 +841,8 @@ static int uvc_ioctl_s_try_ext_ctrls(struct uvc_fh *handle, if (!ctrls->count) return 0; - ret = uvc_ctrl_check_access(chain, ctrls, ioctl); + ret = uvc_ctrl_check_access(chain, ctrls, V4L2_CTRL_WHICH_CUR_VAL, + ioctl); if (ret < 0) return ret; diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 2094e059d7d39..ec76595f3c4be 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -1812,7 +1812,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, npackets = UVC_MAX_PACKETS; /* Retry allocations until one succeed. */ - for (; npackets > 1; npackets /= 2) { + for (; npackets > 0; npackets /= 2) { stream->urb_size = psize * npackets; for (i = 0; i < UVC_URBS; ++i) { @@ -1837,6 +1837,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, uvc_dbg(stream->dev, VIDEO, "Failed to allocate URB buffers (%u bytes per packet)\n", psize); + stream->urb_size = 0; return 0; } diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h index ed7bad31f75ca..8480d65ecb85e 100644 --- a/drivers/media/usb/uvc/uvcvideo.h +++ b/drivers/media/usb/uvc/uvcvideo.h @@ -41,7 +41,8 @@ #define UVC_EXT_GPIO_UNIT 0x7ffe #define UVC_EXT_GPIO_UNIT_ID 0x100 -#define UVC_INVALID_ENTITY_ID 0xffff +#define UVC_HARDWARE_ENTITY_ID(id) ((id) & 0xff) +#define UVC_INVALID_ENTITY_ID 0xffff /* ------------------------------------------------------------------------ * Driver specific constants. @@ -786,7 +787,7 @@ int uvc_ctrl_get(struct uvc_video_chain *chain, u32 which, struct v4l2_ext_control *xctrl); int uvc_ctrl_set(struct uvc_fh *handle, struct v4l2_ext_control *xctrl); int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, - const struct v4l2_ext_controls *ctrls, + const struct v4l2_ext_controls *ctrls, u32 which, unsigned long ioctl); int uvc_xu_ctrl_query(struct uvc_video_chain *chain, diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c index ee884a8221fbd..1c08bba9ecb91 100644 --- a/drivers/media/v4l2-core/v4l2-async.c +++ b/drivers/media/v4l2-core/v4l2-async.c @@ -343,7 +343,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, struct v4l2_subdev *sd, struct v4l2_async_connection *asc) { - struct v4l2_async_notifier *subdev_notifier; bool registered = false; int ret; @@ -389,6 +388,25 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, dev_dbg(notifier_dev(notifier), "v4l2-async: %s bound (ret %d)\n", dev_name(sd->dev), ret); + return 0; + +err_call_unbind: + v4l2_async_nf_call_unbind(notifier, sd, asc); + list_del(&asc->asc_subdev_entry); + +err_unregister_subdev: + if (registered) + v4l2_device_unregister_subdev(sd); + + return ret; +} + +static int +v4l2_async_nf_try_subdev_notifier(struct v4l2_async_notifier *notifier, + struct v4l2_subdev *sd) +{ + struct v4l2_async_notifier *subdev_notifier; + /* * See if the sub-device has a notifier. If not, return here. */ @@ -404,16 +422,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, subdev_notifier->parent = notifier; return v4l2_async_nf_try_all_subdevs(subdev_notifier); - -err_call_unbind: - v4l2_async_nf_call_unbind(notifier, sd, asc); - list_del(&asc->asc_subdev_entry); - -err_unregister_subdev: - if (registered) - v4l2_device_unregister_subdev(sd); - - return ret; } /* Test all async sub-devices in a notifier for a match. */ @@ -445,6 +453,10 @@ v4l2_async_nf_try_all_subdevs(struct v4l2_async_notifier *notifier) if (ret < 0) return ret; + ret = v4l2_async_nf_try_subdev_notifier(notifier, sd); + if (ret < 0) + return ret; + /* * v4l2_async_match_notify() may lead to registering a * new notifier and thus changing the async subdevs @@ -829,7 +841,11 @@ int __v4l2_async_register_subdev(struct v4l2_subdev *sd, struct module *module) ret = v4l2_async_match_notify(notifier, v4l2_dev, sd, asc); if (ret) - goto err_unbind; + goto err_unlock; + + ret = v4l2_async_nf_try_subdev_notifier(notifier, sd); + if (ret) + goto err_unbind_one; ret = v4l2_async_nf_try_complete(notifier); if (ret) @@ -853,9 +869,10 @@ int __v4l2_async_register_subdev(struct v4l2_subdev *sd, struct module *module) if (subdev_notifier) v4l2_async_nf_unbind_all_subdevs(subdev_notifier); - if (asc) - v4l2_async_unbind_subdev_one(notifier, asc); +err_unbind_one: + v4l2_async_unbind_subdev_one(notifier, asc); +err_unlock: mutex_unlock(&list_lock); sd->owner = NULL; diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c index 01cf52c3ea33e..3acfb501080ba 100644 --- a/drivers/media/v4l2-core/v4l2-ioctl.c +++ b/drivers/media/v4l2-core/v4l2-ioctl.c @@ -1542,6 +1542,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt) case V4L2_PIX_FMT_QC10C: descr = "QCOM Compressed 10-bit Format"; break; case V4L2_PIX_FMT_AJPG: descr = "Aspeed JPEG"; break; case V4L2_PIX_FMT_AV1_FRAME: descr = "AV1 Frame"; break; + case V4L2_PIX_FMT_AV1: descr = "AV1 OBU Stream"; break; case V4L2_PIX_FMT_MT2110T: descr = "Mediatek 10bit Tile Mode"; break; case V4L2_PIX_FMT_MT2110R: descr = "Mediatek 10bit Raster Mode"; break; case V4L2_PIX_FMT_HEXTILE: descr = "Hextile Compressed Format"; break; @@ -3079,13 +3080,14 @@ static long __video_do_ioctl(struct file *file, } /* - * We need to serialize streamon/off with queueing new requests. + * We need to serialize streamon/off/reqbufs with queueing new requests. * These ioctls may trigger the cancellation of a streaming * operation, and that should not be mixed with queueing a new * request at the same time. */ if (v4l2_device_supports_requests(vfd->v4l2_dev) && - (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF)) { + (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF || + cmd == VIDIOC_REQBUFS)) { req_queue_lock = &vfd->v4l2_dev->mdev->req_queue_mutex; if (mutex_lock_interruptible(req_queue_lock)) diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index 733e22f695ab7..3609bfd3c64be 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -674,6 +674,7 @@ static int mtk_smi_larb_probe(struct platform_device *pdev) err_pm_disable: pm_runtime_disable(dev); device_link_remove(dev, larb->smi_common_dev); + put_device(larb->smi_common_dev); return ret; } @@ -684,6 +685,7 @@ static void mtk_smi_larb_remove(struct platform_device *pdev) device_link_remove(&pdev->dev, larb->smi_common_dev); pm_runtime_disable(&pdev->dev); component_del(&pdev->dev, &mtk_smi_larb_component_ops); + put_device(larb->smi_common_dev); } static int __maybe_unused mtk_smi_larb_resume(struct device *dev) @@ -917,6 +919,7 @@ static void mtk_smi_common_remove(struct platform_device *pdev) if (common->plat->type == MTK_SMI_GEN2_SUB_COMM) device_link_remove(&pdev->dev, common->smi_common_dev); pm_runtime_disable(&pdev->dev); + put_device(common->smi_common_dev); } static int __maybe_unused mtk_smi_common_resume(struct device *dev) diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 6cec1858947bf..55a9fea95195b 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -406,6 +406,17 @@ config MFD_CS47L92 help Support for Cirrus Logic CS42L92, CS47L92 and CS47L93 Smart Codecs +config MFD_TN48M_CPLD + tristate "Delta Networks TN48M switch CPLD driver" + depends on I2C + depends on ARCH_MVEBU || COMPILE_TEST + select MFD_SIMPLE_MFD_I2C + help + Select this option to enable support for Delta Networks TN48M switch + CPLD. It consists of reset and GPIO drivers. CPLD provides GPIOS-s + for the SFP slots as well as power supply related information. + SFP support depends on the GPIO driver being selected. + config PMIC_DA903X bool "Dialog Semiconductor DA9030/DA9034 PMIC Support" depends on I2C=y diff --git a/drivers/mfd/altera-sysmgr.c b/drivers/mfd/altera-sysmgr.c index fb5f988e61f37..90c6902d537da 100644 --- a/drivers/mfd/altera-sysmgr.c +++ b/drivers/mfd/altera-sysmgr.c @@ -117,6 +117,8 @@ struct regmap *altr_sysmgr_regmap_lookup_by_phandle(struct device_node *np, sysmgr = dev_get_drvdata(dev); + put_device(dev); + return sysmgr->regmap; } EXPORT_SYMBOL_GPL(altr_sysmgr_regmap_lookup_by_phandle); diff --git a/drivers/mfd/arizona-core.c b/drivers/mfd/arizona-core.c index 85ff8717d8504..91975536d14d2 100644 --- a/drivers/mfd/arizona-core.c +++ b/drivers/mfd/arizona-core.c @@ -1100,7 +1100,7 @@ int arizona_dev_init(struct arizona *arizona) } else if (val & 0x01) { ret = wm5102_clear_write_sequencer(arizona); if (ret) - return ret; + goto err_reset; } break; default: diff --git a/drivers/mfd/da9052-spi.c b/drivers/mfd/da9052-spi.c index 80fc5c0cac2fb..be5f2b34e18ae 100644 --- a/drivers/mfd/da9052-spi.c +++ b/drivers/mfd/da9052-spi.c @@ -37,7 +37,7 @@ static int da9052_spi_probe(struct spi_device *spi) spi_set_drvdata(spi, da9052); config = da9052_regmap_config; - config.write_flag_mask = 1; + config.read_flag_mask = 1; config.reg_bits = 7; config.pad_bits = 1; config.val_bits = 8; diff --git a/drivers/mfd/da9055-core.c b/drivers/mfd/da9055-core.c index 1f727ef60d638..8c989b74f924e 100644 --- a/drivers/mfd/da9055-core.c +++ b/drivers/mfd/da9055-core.c @@ -388,6 +388,7 @@ int da9055_device_init(struct da9055 *da9055) err: mfd_remove_devices(da9055->dev); + regmap_del_irq_chip(da9055->chip_irq, da9055->irq_data); return ret; } diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index 8d92c895d3aef..713a5bfb1a3c2 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -437,6 +437,19 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x5ac4), (kernel_ulong_t)&bxt_spi_info }, { PCI_VDEVICE(INTEL, 0x5ac6), (kernel_ulong_t)&bxt_spi_info }, { PCI_VDEVICE(INTEL, 0x5aee), (kernel_ulong_t)&bxt_uart_info }, + /* NVL-S */ + { PCI_VDEVICE(INTEL, 0x6e28), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e29), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e2a), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e2b), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e4c), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4d), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4e), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4f), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e5c), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e5e), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e7a), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e7b), (kernel_ulong_t)&ehl_i2c_info }, /* ARL-H */ { PCI_VDEVICE(INTEL, 0x7725), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x7726), (kernel_ulong_t)&bxt_uart_info }, diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e6cdae221f1d4..3228e79c86eb5 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -413,6 +413,7 @@ static int apple_smc_probe(struct platform_device *pdev) if (!smc) return -ENOMEM; + mutex_init(&smc->mutex); smc->dev = &pdev->dev; smc->sram_base = devm_platform_get_and_ioremap_resource(pdev, 1, &smc->sram); if (IS_ERR(smc->sram_base)) diff --git a/drivers/mfd/max77620.c b/drivers/mfd/max77620.c index 21d2ab3db2542..3af2974b30232 100644 --- a/drivers/mfd/max77620.c +++ b/drivers/mfd/max77620.c @@ -254,7 +254,7 @@ static int max77620_irq_global_unmask(void *irq_drv_data) return ret; } -static struct regmap_irq_chip max77620_top_irq_chip = { +static const struct regmap_irq_chip max77620_top_irq_chip = { .name = "max77620-top", .irqs = max77620_top_irqs, .num_irqs = ARRAY_SIZE(max77620_top_irqs), @@ -498,6 +498,7 @@ static int max77620_probe(struct i2c_client *client) const struct i2c_device_id *id = i2c_client_get_device_id(client); const struct regmap_config *rmap_config; struct max77620_chip *chip; + struct regmap_irq_chip *chip_desc; const struct mfd_cell *mfd_cells; int n_mfd_cells; bool pm_off; @@ -508,6 +509,14 @@ static int max77620_probe(struct i2c_client *client) return -ENOMEM; i2c_set_clientdata(client, chip); + + chip_desc = devm_kmemdup(&client->dev, &max77620_top_irq_chip, + sizeof(max77620_top_irq_chip), + GFP_KERNEL); + if (!chip_desc) + return -ENOMEM; + chip_desc->irq_drv_data = chip; + chip->dev = &client->dev; chip->chip_irq = client->irq; chip->chip_id = (enum max77620_chip_id)id->driver_data; @@ -544,11 +553,9 @@ static int max77620_probe(struct i2c_client *client) if (ret < 0) return ret; - max77620_top_irq_chip.irq_drv_data = chip; ret = devm_regmap_add_irq_chip(chip->dev, chip->rmap, client->irq, IRQF_ONESHOT | IRQF_SHARED, 0, - &max77620_top_irq_chip, - &chip->top_irq_data); + chip_desc, &chip->top_irq_data); if (ret < 0) { dev_err(chip->dev, "Failed to add regmap irq: %d\n", ret); return ret; diff --git a/drivers/mfd/mfd-core.c b/drivers/mfd/mfd-core.c index 7d14a1e7631ee..c55223ce4327a 100644 --- a/drivers/mfd/mfd-core.c +++ b/drivers/mfd/mfd-core.c @@ -22,6 +22,7 @@ #include static LIST_HEAD(mfd_of_node_list); +static DEFINE_MUTEX(mfd_of_node_mutex); struct mfd_of_node_entry { struct list_head list; @@ -105,9 +106,11 @@ static int mfd_match_of_node_to_dev(struct platform_device *pdev, u64 of_node_addr; /* Skip if OF node has previously been allocated to a device */ - list_for_each_entry(of_entry, &mfd_of_node_list, list) - if (of_entry->np == np) - return -EAGAIN; + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry(of_entry, &mfd_of_node_list, list) + if (of_entry->np == np) + return -EAGAIN; + } if (!cell->use_of_reg) /* No of_reg defined - allocate first free compatible match */ @@ -129,7 +132,8 @@ static int mfd_match_of_node_to_dev(struct platform_device *pdev, of_entry->dev = &pdev->dev; of_entry->np = np; - list_add_tail(&of_entry->list, &mfd_of_node_list); + scoped_guard(mutex, &mfd_of_node_mutex) + list_add_tail(&of_entry->list, &mfd_of_node_list); of_node_get(np); device_set_node(&pdev->dev, of_fwnode_handle(np)); @@ -286,11 +290,13 @@ static int mfd_add_device(struct device *parent, int id, if (cell->swnode) device_remove_software_node(&pdev->dev); fail_of_entry: - list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) - if (of_entry->dev == &pdev->dev) { - list_del(&of_entry->list); - kfree(of_entry); - } + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) + if (of_entry->dev == &pdev->dev) { + list_del(&of_entry->list); + kfree(of_entry); + } + } fail_alias: regulator_bulk_unregister_supply_alias(&pdev->dev, cell->parent_supplies, @@ -360,11 +366,13 @@ static int mfd_remove_devices_fn(struct device *dev, void *data) if (cell->swnode) device_remove_software_node(&pdev->dev); - list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) - if (of_entry->dev == &pdev->dev) { - list_del(&of_entry->list); - kfree(of_entry); - } + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) + if (of_entry->dev == &pdev->dev) { + list_del(&of_entry->list); + kfree(of_entry); + } + } regulator_bulk_unregister_supply_alias(dev, cell->parent_supplies, cell->num_parent_supplies); diff --git a/drivers/mfd/mt6358-irq.c b/drivers/mfd/mt6358-irq.c index f467b00d23660..74cf208430440 100644 --- a/drivers/mfd/mt6358-irq.c +++ b/drivers/mfd/mt6358-irq.c @@ -285,6 +285,7 @@ int mt6358_irq_init(struct mt6397_chip *chip) if (ret) { dev_err(chip->dev, "Failed to register IRQ=%d, ret=%d\n", chip->irq, ret); + irq_domain_remove(chip->irq_domain); return ret; } diff --git a/drivers/mfd/mt6397-irq.c b/drivers/mfd/mt6397-irq.c index 0e463026c5a91..5d2e5459f7444 100644 --- a/drivers/mfd/mt6397-irq.c +++ b/drivers/mfd/mt6397-irq.c @@ -229,6 +229,7 @@ int mt6397_irq_init(struct mt6397_chip *chip) if (ret) { dev_err(chip->dev, "failed to register irq=%d; err: %d\n", chip->irq, ret); + irq_domain_remove(chip->irq_domain); return ret; } diff --git a/drivers/mfd/omap-usb-host.c b/drivers/mfd/omap-usb-host.c index a77b6fc790f2e..4d29a6e2ed87a 100644 --- a/drivers/mfd/omap-usb-host.c +++ b/drivers/mfd/omap-usb-host.c @@ -819,8 +819,10 @@ static void usbhs_omap_remove(struct platform_device *pdev) { pm_runtime_disable(&pdev->dev); - /* remove children */ - device_for_each_child(&pdev->dev, NULL, usbhs_omap_remove_child); + if (pdev->dev.of_node) + of_platform_depopulate(&pdev->dev); + else + device_for_each_child(&pdev->dev, NULL, usbhs_omap_remove_child); } static const struct dev_pm_ops usbhsomap_dev_pm_ops = { diff --git a/drivers/mfd/qcom-pm8xxx.c b/drivers/mfd/qcom-pm8xxx.c index 1149f7102a365..0cf374c015ce7 100644 --- a/drivers/mfd/qcom-pm8xxx.c +++ b/drivers/mfd/qcom-pm8xxx.c @@ -577,17 +577,11 @@ static int pm8xxx_probe(struct platform_device *pdev) return rc; } -static int pm8xxx_remove_child(struct device *dev, void *unused) -{ - platform_device_unregister(to_platform_device(dev)); - return 0; -} - static void pm8xxx_remove(struct platform_device *pdev) { struct pm_irq_chip *chip = platform_get_drvdata(pdev); - device_for_each_child(&pdev->dev, NULL, pm8xxx_remove_child); + of_platform_depopulate(&pdev->dev); irq_domain_remove(chip->irqdomain); } diff --git a/drivers/mfd/simple-mfd-i2c.c b/drivers/mfd/simple-mfd-i2c.c index 0a607a1e3ca1d..9f911afafc25e 100644 --- a/drivers/mfd/simple-mfd-i2c.c +++ b/drivers/mfd/simple-mfd-i2c.c @@ -110,6 +110,7 @@ static const struct simple_mfd_data spacemit_p1 = { }; static const struct of_device_id simple_mfd_i2c_of_match[] = { + { .compatible = "delta,tn48m-cpld" }, { .compatible = "fsl,ls1028aqds-fpga" }, { .compatible = "fsl,lx2160aqds-fpga" }, { .compatible = "fsl,lx2160ardb-fpga" }, diff --git a/drivers/mfd/tps65219.c b/drivers/mfd/tps65219.c index 65a952555218d..7275dcdb7c44f 100644 --- a/drivers/mfd/tps65219.c +++ b/drivers/mfd/tps65219.c @@ -498,6 +498,15 @@ static int tps65219_probe(struct i2c_client *client) return ret; } + if (chip_id == TPS65214) { + ret = i2c_smbus_write_byte_data(client, TPS65214_REG_LOCK, + TPS65214_LOCK_ACCESS_CMD); + if (ret) { + dev_err(tps->dev, "Failed to unlock registers %d\n", ret); + return ret; + } + } + ret = devm_regmap_add_irq_chip(tps->dev, tps->regmap, client->irq, IRQF_ONESHOT, 0, pmic->irq_chip, &tps->irq_data); diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index b9c11f67315f0..7b19dd2f28e64 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -310,6 +310,20 @@ config QCOM_FASTRPC applications DSP processor. Say M if you want to enable this module. +config QCOM_DCC + tristate "Qualcomm Data Capture and Compare (DCC) engine driver" + depends on ARCH_QCOM || COMPILE_TEST + help + This option enables the driver for the Data Capture and Compare engine. DCC + driver provides interfaces to configure DCC block and read back the captured + data from the DCC's internal SRAM. The module name for this is qcom-dcc. + +config QCOM_DCC_DEV + tristate "Qualcomm Data Capture and Compare (DCC) engine device instance" + depends on QCOM_DCC + help + This is the device instance of the QCOM DCC driver. + config SGI_GRU tristate "SGI GRU driver" depends on X86_UV && SMP diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index b32a2597d2467..812746940d628 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -75,3 +75,5 @@ obj-$(CONFIG_MCHP_LAN966X_PCI) += lan966x-pci.o obj-y += keba/ obj-y += amd-sbi/ obj-$(CONFIG_MISC_RP1) += rp1/ +obj-$(CONFIG_QCOM_DCC) += qcom-dcc.o +obj-$(CONFIG_QCOM_DCC_DEV) += qcom-dcc-dev.o diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index 1f42d1d5a630a..665a3888708ac 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -1010,6 +1010,9 @@ ssize_t bcm_vk_read(struct file *p_file, struct device *dev = &vk->pdev->dev; struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; struct bcm_vk_wkent *entry = NULL, *iter; + struct vk_msg_blk tmp_msg; + u32 tmp_usr_msg_id; + u32 tmp_blks; u32 q_num; u32 rsp_length; @@ -1034,6 +1037,9 @@ ssize_t bcm_vk_read(struct file *p_file, entry = iter; } else { /* buffer not big enough */ + tmp_msg = iter->to_h_msg[0]; + tmp_usr_msg_id = iter->usr_msg_id; + tmp_blks = iter->to_h_blks; rc = -EMSGSIZE; } goto read_loop_exit; @@ -1052,14 +1058,12 @@ ssize_t bcm_vk_read(struct file *p_file, bcm_vk_free_wkent(dev, entry); } else if (rc == -EMSGSIZE) { - struct vk_msg_blk tmp_msg = entry->to_h_msg[0]; - /* * in this case, return just the first block, so * that app knows what size it is looking for. */ - set_msg_id(&tmp_msg, entry->usr_msg_id); - tmp_msg.size = entry->to_h_blks - 1; + set_msg_id(&tmp_msg, tmp_usr_msg_id); + tmp_msg.size = tmp_blks - 1; if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) { dev_err(dev, "Error return 1st block in -EMSGSIZE\n"); rc = -EFAULT; diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 9cae6f530679b..5230e910a1d11 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -45,6 +45,7 @@ struct eeprom_93xx46_platform_data { #define OP_START 0x4 #define OP_WRITE (OP_START | 0x1) #define OP_READ (OP_START | 0x2) +/* The following addresses are offset for the 1K EEPROM variant in 16-bit mode */ #define ADDR_EWDS 0x00 #define ADDR_ERAL 0x20 #define ADDR_EWEN 0x30 @@ -191,10 +192,7 @@ static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on) bits = edev->addrlen + 3; cmd_addr = OP_START << edev->addrlen; - if (edev->pdata->flags & EE_ADDR8) - cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << 1; - else - cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS); + cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << (edev->addrlen - 6); if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; @@ -328,10 +326,7 @@ static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev) bits = edev->addrlen + 3; cmd_addr = OP_START << edev->addrlen; - if (edev->pdata->flags & EE_ADDR8) - cmd_addr |= ADDR_ERAL << 1; - else - cmd_addr |= ADDR_ERAL; + cmd_addr |= ADDR_ERAL << (edev->addrlen - 6); if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index ee652ef01534a..92966dd3bb998 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #define ADSP_DOMAIN_ID (0) #define MDSP_DOMAIN_ID (1) @@ -37,7 +39,8 @@ #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_DSP_UTILITIES_HANDLE 2 -#define FASTRPC_CTXID_MASK (0xFF0) +#define FASTRPC_MAX_STATIC_HANDLE (20) +#define FASTRPC_CTXID_MASK (0xFF00) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define INIT_FILE_NAMELEN_MAX (128) #define FASTRPC_DEVICE_NAME "fastrpc" @@ -105,6 +108,20 @@ #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) +/* Poll response number from remote processor for call completion */ +#define FASTRPC_POLL_RESPONSE (0xdecaf) + +/* Polling mode timeout limit */ +#define FASTRPC_POLL_MAX_TIMEOUT_US (10000) + +/* Response types supported for RPC calls */ +enum fastrpc_response_flags { + /* normal job completion glink response */ + NORMAL_RESPONSE = 0, + /* process updates poll memory instead of glink response */ + POLL_MODE = 1, +}; + struct fastrpc_phy_page { u64 addr; /* physical address */ u64 size; /* size of contiguous region */ @@ -233,9 +250,16 @@ struct fastrpc_invoke_ctx { int pid; int client_id; u32 sc; + u64 *fdlist; u32 *crc; + /* Poll memory that DSP updates */ + u32 *poll; u64 ctxid; u64 msg_sz; + /* work done status flag */ + bool is_work_done; + /* response flags from remote processor */ + enum fastrpc_response_flags rsp_flags; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; @@ -270,6 +294,8 @@ struct fastrpc_channel_ctx { struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; + /* Flag if audio PD init mem was allocated */ + bool audio_init_mem; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; @@ -303,6 +329,8 @@ struct fastrpc_user { spinlock_t lock; /* lock for allocations */ struct mutex mutex; + /* Reference count */ + struct kref refcount; }; static void fastrpc_free_map(struct kref *ref) @@ -388,6 +416,9 @@ static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, static void fastrpc_buf_free(struct fastrpc_buf *buf) { + if (!buf) + return; + dma_free_coherent(buf->dev, buf->size, buf->virt, FASTRPC_PHYS(buf->phys)); kfree(buf); @@ -471,30 +502,52 @@ static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx) kref_put(&cctx->refcount, fastrpc_channel_ctx_free); } +static void fastrpc_user_free(struct kref *ref) +{ + struct fastrpc_user *fl = container_of(ref, struct fastrpc_user, refcount); + + fastrpc_channel_ctx_put(fl->cctx); + mutex_destroy(&fl->mutex); + kfree(fl); +} + +static void fastrpc_user_get(struct fastrpc_user *fl) +{ + kref_get(&fl->refcount); +} + +static void fastrpc_user_put(struct fastrpc_user *fl) +{ + kref_put(&fl->refcount, fastrpc_user_free); +} + static void fastrpc_context_free(struct kref *ref) { struct fastrpc_invoke_ctx *ctx; struct fastrpc_channel_ctx *cctx; + struct fastrpc_user *fl; unsigned long flags; int i; ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount); cctx = ctx->cctx; + fl = ctx->fl; for (i = 0; i < ctx->nbufs; i++) fastrpc_map_put(ctx->maps[i]); - if (ctx->buf) - fastrpc_buf_free(ctx->buf); + fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); - idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); + idr_remove(&cctx->ctx_idr, ctx->ctxid >> 8); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps); kfree(ctx->olaps); kfree(ctx); + /* Release the reference taken in fastrpc_context_alloc() */ + fastrpc_user_put(fl); fastrpc_channel_ctx_put(cctx); } @@ -604,6 +657,8 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( /* Released in fastrpc_context_put() */ fastrpc_channel_ctx_get(cctx); + /* Take a reference to user, released in fastrpc_context_free() */ + fastrpc_user_get(user); ctx->sc = sc; ctx->retval = -1; @@ -624,7 +679,7 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } - ctx->ctxid = ret << 4; + ctx->ctxid = ret << 8; spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&ctx->refcount); @@ -634,6 +689,7 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( spin_lock(&user->lock); list_del(&ctx->node); spin_unlock(&user->lock); + fastrpc_user_put(user); fastrpc_channel_ctx_put(cctx); kfree(ctx->maps); kfree(ctx->olaps); @@ -892,7 +948,8 @@ static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx) sizeof(struct fastrpc_invoke_buf) + sizeof(struct fastrpc_phy_page)) * ctx->nscalars + sizeof(u64) * FASTRPC_MAX_FDLIST + - sizeof(u32) * FASTRPC_MAX_CRCLIST; + sizeof(u32) * FASTRPC_MAX_CRCLIST + + sizeof(u32); return size; } @@ -987,6 +1044,9 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) rpra = ctx->buf->virt; list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); pages = fastrpc_phy_page_start(list, ctx->nscalars); + ctx->fdlist = (u64 *)(pages + ctx->nscalars); + ctx->crc = (u32 *)(ctx->fdlist + FASTRPC_MAX_FDLIST); + ctx->poll = (u32 *)(ctx->crc + FASTRPC_MAX_CRCLIST); args = (uintptr_t)ctx->buf->virt + metalen; rlen = pkt_size - metalen; ctx->rpra = rpra; @@ -1089,18 +1149,10 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, union fastrpc_remote_arg *rpra = ctx->rpra; struct fastrpc_user *fl = ctx->fl; struct fastrpc_map *mmap = NULL; - struct fastrpc_invoke_buf *list; - struct fastrpc_phy_page *pages; - u64 *fdlist; - int i, inbufs, outbufs, handles; + int i, inbufs; int ret = 0; inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); - outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc); - handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc); - list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); - pages = fastrpc_phy_page_start(list, ctx->nscalars); - fdlist = (uint64_t *)(pages + inbufs + outbufs + handles); for (i = inbufs; i < ctx->nbufs; ++i) { if (!ctx->maps[i]) { @@ -1122,9 +1174,9 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, cleanup_fdlist: /* Clean up fdlist which is updated by DSP */ for (i = 0; i < FASTRPC_MAX_FDLIST; i++) { - if (!fdlist[i]) + if (!ctx->fdlist[i]) break; - if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap)) + if (!fastrpc_map_lookup(fl, (int)ctx->fdlist[i], &mmap)) fastrpc_map_put(mmap); } @@ -1163,6 +1215,83 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, } +static inline u32 fastrpc_poll_op(void *p) +{ + struct fastrpc_invoke_ctx *ctx = p; + + dma_rmb(); + return READ_ONCE(*ctx->poll); +} + +static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx) +{ + u32 val; + int ret; + + /* + * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll + * or until another path marks the work done. + */ + ret = read_poll_timeout_atomic(fastrpc_poll_op, val, + (val == FASTRPC_POLL_RESPONSE) || + ctx->is_work_done, 1, + FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx); + + if (!ret && val == FASTRPC_POLL_RESPONSE) { + ctx->is_work_done = true; + ctx->retval = 0; + } + + if (ret == -ETIMEDOUT) + ret = -EIO; + + return ret; +} + +static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err = 0; + + if (kernel) { + if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) + err = -ETIMEDOUT; + } else { + err = wait_for_completion_interruptible(&ctx->work); + } + + return err; +} + +static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err; + + do { + switch (ctx->rsp_flags) { + case NORMAL_RESPONSE: + err = fastrpc_wait_for_response(ctx, kernel); + if (err || ctx->is_work_done) + return err; + break; + case POLL_MODE: + err = poll_for_remote_response(ctx); + /* If polling timed out, move to normal response mode */ + if (err) + ctx->rsp_flags = NORMAL_RESPONSE; + break; + default: + err = -EBADR; + dev_dbg(ctx->fl->sctx->dev, + "unsupported response type:0x%x\n", ctx->rsp_flags); + break; + } + } while (!ctx->is_work_done); + + return err; +} + static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, u32 handle, u32 sc, struct fastrpc_invoke_args *args) @@ -1198,16 +1327,19 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, if (err) goto bail; - if (kernel) { - if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) - err = -ETIMEDOUT; - } else { - err = wait_for_completion_interruptible(&ctx->work); - } + if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD) + ctx->rsp_flags = POLL_MODE; + err = fastrpc_wait_for_completion(ctx, kernel); if (err) goto bail; + if (!ctx->is_work_done) { + err = -ETIMEDOUT; + dev_dbg(fl->sctx->dev, "Invalid workdone state for handle 0x%x, sc 0x%x\n", + handle, sc); + goto bail; + } /* make sure that all memory writes by DSP are seen by CPU */ dma_rmb(); /* populate all the output buffers with results */ @@ -1266,15 +1398,16 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, struct fastrpc_init_create_static init; struct fastrpc_invoke_args *args; struct fastrpc_phy_page pages[1]; + struct fastrpc_channel_ctx *cctx = fl->cctx; char *name; int err; - bool scm_done = false; struct { int client_id; u32 namelen; u32 pageslen; } inbuf; u32 sc; + unsigned long flags; args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, sizeof(*args), GFP_KERNEL); if (!args) @@ -1295,33 +1428,10 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, err = PTR_ERR(name); goto err; } - - if (!fl->cctx->remote_heap) { - err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen, - &fl->cctx->remote_heap); - if (err) - goto err_name; - - /* Map if we have any heap VMIDs associated with this ADSP Static Process. */ - if (fl->cctx->vmcount) { - u64 src_perms = BIT(QCOM_SCM_VMID_HLOS); - - err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys, - (u64)fl->cctx->remote_heap->size, - &src_perms, - fl->cctx->vmperms, fl->cctx->vmcount); - if (err) { - dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n", - fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); - goto err_map; - } - scm_done = true; - } - } - inbuf.client_id = fl->client_id; inbuf.namelen = init.namelen; inbuf.pageslen = 0; + fl->pd = USER_PD; args[0].ptr = (u64)(uintptr_t)&inbuf; @@ -1332,8 +1442,25 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, args[1].length = inbuf.namelen; args[1].fd = -1; - pages[0].addr = fl->cctx->remote_heap->phys; - pages[0].size = fl->cctx->remote_heap->size; + spin_lock_irqsave(&cctx->lock, flags); + if (!fl->cctx->audio_init_mem) { + if (!fl->cctx->remote_heap || + !fl->cctx->remote_heap->phys || + !fl->cctx->remote_heap->size) { + spin_unlock_irqrestore(&cctx->lock, flags); + err = -ENOMEM; + goto err; + } + + pages[0].addr = fl->cctx->remote_heap->phys; + pages[0].size = fl->cctx->remote_heap->size; + fl->cctx->audio_init_mem = true; + inbuf.pageslen = 1; + } else { + pages[0].addr = 0; + pages[0].size = 0; + } + spin_unlock_irqrestore(&cctx->lock, flags); args[2].ptr = (u64)(uintptr_t) pages; args[2].length = sizeof(*pages); @@ -1351,26 +1478,7 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, return 0; err_invoke: - if (fl->cctx->vmcount && scm_done) { - u64 src_perms = 0; - struct qcom_scm_vmperm dst_perms; - u32 i; - - for (i = 0; i < fl->cctx->vmcount; i++) - src_perms |= BIT(fl->cctx->vmperms[i].vmid); - - dst_perms.vmid = QCOM_SCM_VMID_HLOS; - dst_perms.perm = QCOM_SCM_PERM_RWX; - err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys, - (u64)fl->cctx->remote_heap->size, - &src_perms, &dst_perms, 1); - if (err) - dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n", - fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); - } -err_map: - fastrpc_buf_free(fl->cctx->remote_heap); -err_name: + fl->cctx->audio_init_mem = false; kfree(name); err: kfree(args); @@ -1558,8 +1666,7 @@ static int fastrpc_device_release(struct inode *inode, struct file *file) list_del(&fl->user); spin_unlock_irqrestore(&cctx->lock, flags); - if (fl->init_mem) - fastrpc_buf_free(fl->init_mem); + fastrpc_buf_free(fl->init_mem); list_for_each_entry_safe(ctx, n, &fl->pending, node) { list_del(&ctx->node); @@ -1575,11 +1682,9 @@ static int fastrpc_device_release(struct inode *inode, struct file *file) } fastrpc_session_free(cctx, fl->sctx); - fastrpc_channel_ctx_put(cctx); - - mutex_destroy(&fl->mutex); - kfree(fl); file->private_data = NULL; + /* Release the reference taken in fastrpc_device_open */ + fastrpc_user_put(fl); return 0; } @@ -1623,6 +1728,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) spin_lock_irqsave(&cctx->lock, flags); list_add_tail(&fl->user, &cctx->users); spin_unlock_irqrestore(&cctx->lock, flags); + kref_init(&fl->refcount); return 0; } @@ -1831,9 +1937,6 @@ static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf * &args[0]); if (!err) { dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr); - spin_lock(&fl->lock); - list_del(&buf->node); - spin_unlock(&fl->lock); fastrpc_buf_free(buf); } else { dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr); @@ -1847,6 +1950,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) struct fastrpc_buf *buf = NULL, *iter, *b; struct fastrpc_req_munmap req; struct device *dev = fl->sctx->dev; + int err; if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT; @@ -1854,6 +1958,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) spin_lock(&fl->lock); list_for_each_entry_safe(iter, b, &fl->mmaps, node) { if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) { + list_del(&iter->node); buf = iter; break; } @@ -1866,7 +1971,14 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) return -EINVAL; } - return fastrpc_req_munmap_impl(fl, buf); + err = fastrpc_req_munmap_impl(fl, buf); + if (err) { + spin_lock(&fl->lock); + list_add_tail(&buf->node, &fl->mmaps); + spin_unlock(&fl->lock); + } + + return err; } static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) @@ -1956,14 +2068,17 @@ static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) if (copy_to_user((void __user *)argp, &req, sizeof(req))) { err = -EFAULT; - goto err_assign; + goto err_copy; } dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n", buf->raddr, buf->size); return 0; - +err_copy: + spin_lock(&fl->lock); + list_del(&buf->node); + spin_unlock(&fl->lock); err_assign: fastrpc_req_munmap_impl(fl, buf); @@ -2329,7 +2444,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) } } - if (domain_id == SDSP_DOMAIN_ID) { + if (domain_id == SDSP_DOMAIN_ID || domain_id == ADSP_DOMAIN_ID) { struct resource res; u64 src_perms; @@ -2337,10 +2452,20 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) if (!err) { src_perms = BIT(QCOM_SCM_VMID_HLOS); - qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms, + err = qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms, data->vmperms, data->vmcount); + if (err) + goto err_free_data; + } + if (domain_id == ADSP_DOMAIN_ID) { + data->remote_heap = + kzalloc(sizeof(*data->remote_heap), GFP_KERNEL); + if (!data->remote_heap) + return -ENOMEM; + + data->remote_heap->phys = res.start; + data->remote_heap->size = resource_size(&res); } - } secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); @@ -2420,6 +2545,7 @@ static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev) struct fastrpc_buf *buf, *b; struct fastrpc_user *user; unsigned long flags; + int err; /* No invocations past this point */ spin_lock_irqsave(&cctx->lock, flags); @@ -2437,8 +2563,22 @@ static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev) list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node) list_del(&buf->node); - if (cctx->remote_heap) - fastrpc_buf_free(cctx->remote_heap); + if (cctx->remote_heap && cctx->vmcount) { + u64 src_perms = 0; + struct qcom_scm_vmperm dst_perms; + + for (u32 i = 0; i < cctx->vmcount; i++) + src_perms |= BIT(cctx->vmperms[i].vmid); + + dst_perms.vmid = QCOM_SCM_VMID_HLOS; + dst_perms.perm = QCOM_SCM_PERM_RWX; + + err = qcom_scm_assign_mem(cctx->remote_heap->phys, + cctx->remote_heap->size, &src_perms, + &dst_perms, 1); + if (!err) + fastrpc_buf_free(cctx->remote_heap); + } of_platform_depopulate(&rpdev->dev); @@ -2457,7 +2597,7 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, if (len < sizeof(*rsp)) return -EINVAL; - ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4); + ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 8); spin_lock_irqsave(&cctx->lock, flags); ctx = idr_find(&cctx->ctx_idr, ctxid); @@ -2470,6 +2610,7 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, ctx->retval = rsp->retval; complete(&ctx->work); + ctx->is_work_done = true; /* * The DMA buffer associated with the context cannot be freed in diff --git a/drivers/misc/mei/Kconfig b/drivers/misc/mei/Kconfig index f8b04e49e4baf..f4eb307cd35ed 100644 --- a/drivers/misc/mei/Kconfig +++ b/drivers/misc/mei/Kconfig @@ -49,7 +49,7 @@ config INTEL_MEI_TXE config INTEL_MEI_GSC tristate "Intel MEI GSC embedded device" depends on INTEL_MEI_ME - depends on DRM_I915 + depends on DRM_I915 || DRM_XE help Intel auxiliary driver for GSC devices embedded in Intel graphics devices. diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index a4f75dc369292..fa30899a5fa26 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -122,6 +122,8 @@ #define MEI_DEV_ID_WCL_P 0x4D70 /* Wildcat Lake P */ +#define MEI_DEV_ID_NVL_S 0x6E68 /* Nova Lake Point S */ + /* * MEI HW Section */ diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c index d4612c6597844..1e4a41ac428fd 100644 --- a/drivers/misc/mei/hw-me.c +++ b/drivers/misc/mei/hw-me.c @@ -1337,19 +1337,13 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id) /* check if we need to start the dev */ if (!mei_host_is_ready(dev)) { if (mei_hw_is_ready(dev)) { - /* synchronized by dev mutex */ - if (waitqueue_active(&dev->wait_hw_ready)) { - dev_dbg(&dev->dev, "we need to start the dev.\n"); - dev->recvd_hw_ready = true; - wake_up(&dev->wait_hw_ready); - } else if (dev->dev_state != MEI_DEV_UNINITIALIZED && - dev->dev_state != MEI_DEV_POWERING_DOWN && - dev->dev_state != MEI_DEV_POWER_DOWN) { + if (dev->dev_state == MEI_DEV_ENABLED) { dev_dbg(&dev->dev, "Force link reset.\n"); schedule_work(&dev->reset_work); } else { - dev_dbg(&dev->dev, "Ignore this interrupt in state = %d\n", - dev->dev_state); + dev_dbg(&dev->dev, "we need to start the dev.\n"); + dev->recvd_hw_ready = true; + wake_up(&dev->wait_hw_ready); } } else { dev_dbg(&dev->dev, "Spurious Interrupt\n"); diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 86a73684a3732..6f26d51607883 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -1307,6 +1307,7 @@ int mei_register(struct mei_device *dev, struct device *parent) err_del_cdev: cdev_del(dev->cdev); err: + put_device(&dev->dev); mei_minor_free(minor); return ret; } diff --git a/drivers/misc/mei/mei-trace.h b/drivers/misc/mei/mei-trace.h index 5312edbf5190d..24fa321d88bd8 100644 --- a/drivers/misc/mei/mei-trace.h +++ b/drivers/misc/mei/mei-trace.h @@ -21,18 +21,18 @@ TRACE_EVENT(mei_reg_read, TP_ARGS(dev, reg, offs, val), TP_STRUCT__entry( __string(dev, dev_name(dev)) - __field(const char *, reg) + __string(reg, reg) __field(u32, offs) __field(u32, val) ), TP_fast_assign( __assign_str(dev); - __entry->reg = reg; + __assign_str(reg); __entry->offs = offs; __entry->val = val; ), TP_printk("[%s] read %s:[%#x] = %#x", - __get_str(dev), __entry->reg, __entry->offs, __entry->val) + __get_str(dev), __get_str(reg), __entry->offs, __entry->val) ); TRACE_EVENT(mei_reg_write, @@ -40,18 +40,18 @@ TRACE_EVENT(mei_reg_write, TP_ARGS(dev, reg, offs, val), TP_STRUCT__entry( __string(dev, dev_name(dev)) - __field(const char *, reg) + __string(reg, reg) __field(u32, offs) __field(u32, val) ), TP_fast_assign( __assign_str(dev); - __entry->reg = reg; + __assign_str(reg); __entry->offs = offs; __entry->val = val; ), TP_printk("[%s] write %s[%#x] = %#x", - __get_str(dev), __entry->reg, __entry->offs, __entry->val) + __get_str(dev), __get_str(reg), __entry->offs, __entry->val) ); TRACE_EVENT(mei_pci_cfg_read, @@ -59,18 +59,18 @@ TRACE_EVENT(mei_pci_cfg_read, TP_ARGS(dev, reg, offs, val), TP_STRUCT__entry( __string(dev, dev_name(dev)) - __field(const char *, reg) + __string(reg, reg) __field(u32, offs) __field(u32, val) ), TP_fast_assign( __assign_str(dev); - __entry->reg = reg; + __assign_str(reg); __entry->offs = offs; __entry->val = val; ), TP_printk("[%s] pci cfg read %s:[%#x] = %#x", - __get_str(dev), __entry->reg, __entry->offs, __entry->val) + __get_str(dev), __get_str(reg), __entry->offs, __entry->val) ); #endif /* _MEI_TRACE_H_ */ diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 73cad914be9f9..2a6e569558b94 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -129,6 +129,8 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_WCL_P, MEI_ME_PCH15_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_NVL_S, MEI_ME_PCH15_CFG)}, + /* required last entry */ {0, } }; diff --git a/drivers/misc/qcom-dcc-dev.c b/drivers/misc/qcom-dcc-dev.c new file mode 100644 index 0000000000000..6f9f1004587ae --- /dev/null +++ b/drivers/misc/qcom-dcc-dev.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include "qcom-dcc.h" +#include "qcom-dcc-talos-config.h" +#include "qcom-dcc-lemans-config.h" +#include "qcom-dcc-kodiak-config.h" +#include "qcom-dcc-pakala-config.h" + +#define DEV_NAME "qcom-dcc" + +static struct platform_device *dcc_pdev; + +static const struct dcc_pdata kaanapali_pdata = { + .base = 0x100ff000, + .size = 0x1000, + .ram_base = 0x10080000, + .ram_size = 0x8000, + .dcc_offset = 0x0, + .map_ver = 0x3, +}; + +static const struct dcc_pdata hamoa_pdata = { + .base = 0x100ff000, + .size = 0x1000, + .ram_base = 0x10080000, + .ram_size = 0x18000, + .dcc_offset = 0x0, + .map_ver = 0x3, +}; + +static int __init dcc_dev_init(void) +{ + int ret; + u32 soc_id; + + dcc_pdev = platform_device_alloc(DEV_NAME, -1); + if (!dcc_pdev) + return -ENOMEM; + + ret = qcom_smem_get_soc_id(&soc_id); + if (ret) + goto fail; + + switch (soc_id) { + case 475: + case 497: + case 498: + case 515: + ret = platform_device_add_data(dcc_pdev, &kodiak_pdata, sizeof(kodiak_pdata)); + if (ret) + goto fail; + + break; + /* lemans IDs */ + case 534: + case 667: + case 676: + /* monaco IDs */ + case 606: + case 674: + case 675: + ret = platform_device_add_data(dcc_pdev, &lemans_pdata, sizeof(lemans_pdata)); + if (ret) + goto fail; + + break; + case 377: + case 380: + case 384: + case 401: + case 406: + case 680: + ret = platform_device_add_data(dcc_pdev, &talos_pdata, sizeof(talos_pdata)); + if (ret) + goto fail; + + break; + case 618: + case 639: + case 705: + case 706: + ret = platform_device_add_data(dcc_pdev, &pakala_pdata, sizeof(pakala_pdata)); + if (ret) + goto fail; + + break; + case 660: + case 661: + case 704: + case 722: + case 723: + case 730: + case 743: + ret = platform_device_add_data(dcc_pdev, &kaanapali_pdata, sizeof(kaanapali_pdata)); + if (ret) + goto fail; + + break; + case 555: + case 615: + case 616: + case 709: + case 710: + ret = platform_device_add_data(dcc_pdev, &hamoa_pdata, sizeof(hamoa_pdata)); + if (ret) + goto fail; + + break; + default: + pr_err("DCC: Invalid SoC ID\n"); + ret = -EINVAL; + goto fail; + } + + ret = platform_device_add(dcc_pdev); + if (ret) + goto fail; + + pr_info("DCC platform device has registered\n"); + + return 0; + +fail: + pr_err("Failed to register DCC platform device\n"); + platform_device_put(dcc_pdev); + + return ret; +} + +static void __exit dcc_dev_exit(void) +{ + platform_device_unregister(dcc_pdev); +} + +late_initcall(dcc_dev_init); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Qualcomm Technologies Inc. DCC driver, device stub"); diff --git a/drivers/misc/qcom-dcc-kodiak-config.h b/drivers/misc/qcom-dcc-kodiak-config.h new file mode 100644 index 0000000000000..7184252734917 --- /dev/null +++ b/drivers/misc/qcom-dcc-kodiak-config.h @@ -0,0 +1,1188 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _QCOM_DCC_KODIAK_CONFIG_H +#define _QCOM_DCC_KODIAK_CONFIG_H + +#include "qcom-dcc.h" + +static const struct dcc_register_entry kodiak_dcc_entries_ll6[] = { + /* pcu */ + { "R 0x18000010" }, + { "R 0x18000024" }, + { "R 0x18000038 6" }, + { "R 0x18010010" }, + { "R 0x18010024" }, + { "R 0x18010038 6" }, + { "R 0x18020010" }, + { "R 0x18020024" }, + { "R 0x18020038 6" }, + { "R 0x18030010" }, + { "R 0x18030024" }, + { "R 0x18030038 6" }, + { "R 0x18040010" }, + { "R 0x18040024" }, + { "R 0x18040038 6" }, + { "R 0x18050010" }, + { "R 0x18050024" }, + { "R 0x18050038 6" }, + { "R 0x18060010" }, + { "R 0x18060024" }, + { "R 0x18060038 6" }, + { "R 0x18070010" }, + { "R 0x18070024" }, + { "R 0x18070038 6" }, + { "R 0x18080010" }, + { "R 0x18080024" }, + { "R 0x18080038 6" }, + { "R 0x1808006c 5" }, + { "R 0x18080084" }, + { "R 0x180800f4 18" }, + { "R 0x18080158 5" }, + { "R 0x18080170 2" }, + { "R 0x18080188 5" }, + { "R 0x180801ac 6" }, + { "R 0x180801c8" }, + { "R 0x180801f0" }, + /* epss */ + { "R 0x18598020" }, + { "R 0x1859001c 2" }, + { "R 0x1859002c" }, + { "R 0x18590064 6" }, + { "R 0x1859008c" }, + { "R 0x185900dc" }, + { "R 0x185900e8 3" }, + { "R 0x18590300" }, + { "R 0x1859030c" }, + { "R 0x18590320" }, + { "R 0x1859034c" }, + { "R 0x185903bc 2" }, + { "R 0x1859101c 2" }, + { "R 0x1859102c" }, + { "R 0x18591064 6" }, + { "R 0x1859108c" }, + { "R 0x185910dc" }, + { "R 0x185910e8 3" }, + { "R 0x18591300" }, + { "R 0x1859130c" }, + { "R 0x18591320" }, + { "R 0x1859134c" }, + { "R 0x185913bc 2" }, + { "R 0x1859201c 2" }, + { "R 0x1859202c" }, + { "R 0x18592064 6" }, + { "R 0x1859208c" }, + { "R 0x185920dc" }, + { "R 0x185920e8 3" }, + { "R 0x18592300" }, + { "R 0x1859230c" }, + { "R 0x18592320" }, + { "R 0x1859234c" }, + { "R 0x185923bc 2" }, + { "R 0x1859301c 2" }, + { "R 0x18593064 6" }, + { "R 0x1859308c" }, + { "R 0x185930dc" }, + { "R 0x185930e8 3" }, + { "R 0x18593300" }, + { "R 0x1859330c" }, + { "R 0x18593320" }, + { "R 0x1859302c" }, + { "R 0x1859334c" }, + { "R 0x185933bc 2" }, + { "R 0x18300000" }, + { "R 0x1830000c" }, + { "R 0x18300018" }, + { "R 0x17c21000 2" }, + { "R 0x18393a84 2" }, + { "R 0x183a3a84 2" }, + { "R 0x18280000 2" }, + { "R 0x18282000 2" }, + { "R 0x18284000 2" }, + { "R 0x18286000 2" }, + { "R 0x18300000" }, + { "R 0x18200400 3" }, + { "R 0x18200038" }, + { "R 0x18200040" }, + { "R 0x18200048" }, + { "R 0x18220038" }, + { "R 0x18220040" }, + { "R 0x182200d0" }, + { "R 0x18200030" }, + { "R 0x18200010" }, + /* pimem */ + { "R 0x610100 11" }, + /* core */ + { "R 0x18000058 4" }, + { "R 0x1800006c" }, + { "R 0x180000f0 2" }, + { "R 0x18010058 4" }, + { "R 0x1801006c" }, + { "R 0x180100f0 2" }, + { "R 0x18020058 4" }, + { "R 0x1802006c" }, + { "R 0x180200f0 2" }, + { "R 0x18030058 4" }, + { "R 0x1803006c" }, + { "R 0x180300f0 2" }, + { "R 0x18040058 4" }, + { "R 0x1804006c" }, + { "R 0x180400f0 2" }, + { "R 0x18050058 4" }, + { "R 0x1805006c" }, + { "R 0x180500f0 2" }, + { "R 0x18060058 4" }, + { "R 0x1806006c" }, + { "R 0x180600f0 2" }, + { "R 0x18070058 4" }, + { "R 0x1807006c" }, + { "R 0x180700f0 2" }, + { "R 0x18101908" }, + { "R 0x18101c18" }, + { "R 0x18390810" }, + { "R 0x18390c50" }, + { "R 0x18390814" }, + { "R 0x18390c54" }, + { "R 0x18390818" }, + { "R 0x18390c58" }, + { "R 0x18393a84 2" }, + { "R 0x18100908" }, + { "R 0x18100c18" }, + { "R 0x183a0810" }, + { "R 0x183a0c50" }, + { "R 0x183a0814" }, + { "R 0x183a0c54" }, + { "R 0x183a0818" }, + { "R 0x183a0c58" }, + { "R 0x183a3a84 2" }, + { "R 0x18393500" }, + { "R 0x18393580" }, + { "R 0x183a3500" }, + { "R 0x183a3580" }, + { "R 0x18282000 4" }, + { "R 0x18282028" }, + { "R 0x18282038" }, + { "R 0x18282080 5" }, + { "R 0x18286000 4" }, + { "R 0x18286028" }, + { "R 0x18286038" }, + { "R 0x18286080 5" }, + { "R 0xc201244" }, + { "R 0xc202244" }, + { "R 0x18300000" }, + { "R 0x1829208c" }, + { "R 0x18292098" }, + { "R 0x18292098" }, + { "R 0x1829608c" }, + { "R 0x18296098" }, + { "R 0x18296098" }, + { "R 0x784184" }, + /* gemnoc */ + { "R 0x9103008" }, + { "R 0x9103408" }, + { "R 0x9143008" }, + { "R 0x9143408" }, + { "R 0x91b0008" }, + { "R 0x91b1008" }, + { "R 0x9101808 2" }, + { "R 0x9141808 2" }, + { "R 0x91a8008 2" }, + { "R 0x91a8808 2" }, + { "R 0x9100000" }, + { "R 0x9100008 2" }, + { "R 0x9140000" }, + { "R 0x9140008 2" }, + { "R 0x9180000" }, + { "R 0x9180008 2" }, + { "R 0x9180404 3" }, + { "R 0x9181010" }, + { "R 0x9181020 8" }, + { "R 0x91e1048" }, + { "R 0x9121010" }, + { "R 0x9122010" }, + { "R 0x9123010" }, + { "R 0x9125010" }, + { "R 0x9161010" }, + { "R 0x9162010" }, + { "R 0x9163010" }, + { "R 0x9165010" }, + { "R 0x91cf010" }, + { "R 0x91d0010" }, + { "R 0x91d1010" }, + { "R 0x91d2010" }, + { "R 0x91d3010" }, + { "R 0x91d4010" }, + { "R 0x91d5010" }, + { "R 0x91d6010" }, + { "R 0x91d7010" }, + { "R 0x9101408" }, + { "R 0x9141410" }, + { "R 0x9100810" }, + { "R 0x9140810" }, + { "R 0x9100820" }, + { "R 0x9140820" }, + { "R 0x9100828 2" }, + { "R 0x9140828 2" }, + /* tsens */ + { "R 0xc222004" }, + { "R 0xc263014" }, + { "R 0xc2630e0" }, + { "R 0xc2630ec" }, + { "R 0xc2630a0 16" }, + { "R 0xc2630e8" }, + { "R 0xc26313c" }, + { "R 0xc223004" }, + { "R 0xc265014" }, + { "R 0xc2650e0" }, + { "R 0xc2650ec" }, + { "R 0xc2650a0 16" }, + { "R 0xc2650e8" }, + { "R 0xc26513c" }, + /* spmi */ + { "R 0xc410000" }, + { "R 0xc40af04" }, + { "R 0xc40af10" }, + { "R 0xc40a000" }, + { "R 0xc40a018" }, + { "R 0xc40a028 2" }, + { "R 0xc40a100" }, + { "R 0xc2a22fc 3" }, + { "R 0xc440200 2" }, + { "R 0xc442200 4" }, + /* gpu */ + { "R 0x3d9100c 3" }, + { "R 0x3d9106c 3" }, + { "R 0x3d91004" }, + { "R 0x3d91054 5" }, + { "R 0x3d91070 2" }, + { "R 0x3d91080 3" }, + { "R 0x3d91078 2" }, + { "R 0x3d9108c 2" }, + { "R 0x3d91098 2" }, + { "R 0x3d910a4 2" }, + { "R 0x3d910f0 2" }, + { "R 0x3d91100" }, + { "R 0x3d91118" }, + { "R 0x3d91164 2" }, + { "R 0x3d91170" }, + { "R 0x3d91178" }, + { "R 0x3d91204" }, + { "R 0x3d9120c" }, + { "R 0x3d98024" }, + { "R 0x3d9802c 2" }, + { "R 0x3d92000 2" }, + { "R 0x3d93000 2" }, + { "R 0x3d95000 2" }, + { "R 0x3d96000 2" }, + { "R 0x3d97000 2" }, + { "R 0x119000" }, + { "R 0x11903c" }, + { "R 0x171004 3" }, + { "R 0x171014 2" }, + { "R 0x171154 3" }, + { "R 0x17a04c" }, + { "R 0x17b000" }, + { "R 0x17b03c" }, + { "R 0x17c000" }, + { "R 0x17c03c" }, + { "R 0x17d000" }, + { "R 0x17d03c" }, + { "R 0x17e000" }, + { "R 0x17e03c" }, + { "R 0x187000" }, + { "R 0x18703c" }, + { "R 0x3d91534" }, + { "R 0x3d002b4" }, + { "R 0x3d00410 2" }, + { "R 0x3d00818" }, + { "R 0x3d7e220 2" }, + /* sysnoc */ + { "R 0x1680000" }, + { "R 0x1680008" }, + { "R 0x1680010" }, + { "R 0x1680020 8" }, + { "R 0x1680248" }, + { "R 0x1680b00 6" }, + /* aggrenoc */ + { "R 0x16e4008" }, + { "R 0x1706208" }, + { "R 0x16e0000" }, + { "R 0x16e0010" }, + { "R 0x16e0008" }, + { "R 0x16e0020 8" }, + { "R 0x16e5048" }, + { "R 0x16e5248" }, + { "R 0x16e5448" }, + { "R 0x16e5100 5" }, + { "R 0x16e5300 2" }, + { "R 0x16e5500 2" }, + { "R 0x1700000" }, + { "R 0x1700008" }, + { "R 0x1700010" }, + { "R 0x1700020 8" }, + { "R 0x170b100 5" }, + { "R 0x170b048" }, + /* gcc */ + { "R 0x100000 15" }, + { "R 0x101000 15" }, + { "R 0x176000 15" }, + { "R 0x174000 15" }, + { "R 0x113000 15" }, + { "R 0x11a000 15" }, + { "R 0x11c000 15" }, + { "R 0x11c048 3" }, + { "R 0x11e000 15" }, + { "R 0x10401c" }, + { "R 0x105074" }, + { "R 0x183024" }, + { "R 0x109050" }, + { "R 0x123020" }, + { "R 0x117024" }, + { "R 0x117154" }, + { "R 0x117284" }, + { "R 0x1173b4" }, + { "R 0x1174e4" }, + { "R 0x117614" }, + { "R 0x117744" }, + { "R 0x117874" }, + { "R 0x118024" }, + { "R 0x118154" }, + { "R 0x118284" }, + { "R 0x1183b4" }, + { "R 0x1184e4" }, + { "R 0x118614" }, + { "R 0x118744" }, + { "R 0x118874" }, + { "R 0x129020" }, + { "R 0x11d020" }, + { "R 0x134024" }, + { "R 0x141024" }, + { "R 0x14415c" }, + { "R 0x14504c" }, + { "R 0x18903c" }, + { "R 0x151000" }, + { "R 0x151008" }, + { "R 0x151010" }, + { "R 0x152000" }, + { "R 0x152008" }, + { "R 0x152010" }, + { "R 0x153020" }, + { "R 0x153028" }, + { "R 0x153030" }, + { "R 0x155000" }, + { "R 0x155008" }, + { "R 0x155010" }, + { "R 0x15b000" }, + { "R 0x15b008" }, + { "R 0x15b010" }, + { "R 0x157000" }, + { "R 0x157008" }, + { "R 0x157010" }, + { "R 0x135020" }, + { "R 0x135028" }, + { "R 0x135030" }, + { "R 0x156000" }, + { "R 0x156008" }, + { "R 0x156010" }, + { "R 0x15a000" }, + { "R 0x15a008" }, + { "R 0x15a010" }, + { "R 0x190004" }, + { "R 0x109008" }, + { "R 0x190010" }, + { "R 0x190020" }, + { "R 0x190028" }, + { "R 0x109010" }, + { "R 0x109018" }, + { "R 0x109018" }, + { "R 0x109020" }, + { "R 0x18d080" }, + { "R 0x145014" }, + { "R 0x14501c" }, + { "R 0x183004 2" }, + { "R 0x183140" }, + { "R 0x171158 2" }, + { "R 0x109004 3" }, + { "R 0x109160" }, + { "R 0x109468" }, + { "R 0x10f004 3" }, + { "R 0x145000 3" }, + { "R 0x16b004 3" }, + { "R 0x18d004 3" }, + { "R 0x177004 3" }, + { "R 0x189004 3" }, + { "R 0x153000 9" }, + { "R 0x135000 9" }, + { "R 0x106100" }, + { "R 0x147004" }, + { "R 0x17b000 12" }, + { "R 0x17b03c 12" }, + { "R 0x17c000 12" }, + { "R 0x17c03c 12" }, + { "R 0x153124" }, + { "R 0x156124" }, + { "R 0x1453a4" }, + { "R 0x182884" }, + { "R 0x145384" }, + { "R 0xc2a0000 15" }, + { "R 0xc2a1000 15" }, + { "R 0x17101c 2" }, + { "R 0x14401c 2" }, + { "R 0x183010 2" }, + { "R 0x18a160 2" }, + { "R 0x18a004" }, + { "R 0x18a01c 3" }, + { "R 0x19d004 2" }, + { "R 0x196100" }, + { "R 0x10003c" }, + { "R 0x10103c" }, + { "R 0x10203c" }, + { "R 0x10303c" }, + { "R 0x11303c" }, + { "R 0x11a03c" }, + { "R 0x11c03c" }, + { "R 0x17403c" }, + { "R 0x17603c" }, + { "R 0x11e03c" }, + { "R 0xbbf0004 12" }, + { "R 0xbbf0800 12" }, + { "R 0xbbf0004 12" }, + { "R 0xbbf0800 12" }, +}; + +static const struct dcc_register_entry kodiak_dcc_entries_ll4[] = { + /* confignoc */ + { "R 0x1510008" }, + { "R 0x151d208" }, + { "R 0x1507008" }, + { "R 0x151d308" }, + { "R 0x1509008" }, + { "R 0x1514008" }, + { "R 0x1500000" }, + { "R 0x1500010" }, + { "R 0x1510010" }, + { "R 0x1500020 8" }, + { "R 0x1510020 8" }, + { "R 0x1511048" }, + { "R 0x1501048 2" }, + { "R 0x1501058" }, + { "R 0x1501248" }, + { "R 0x1511248" }, + { "R 0x1501248" }, + { "R 0x1511b00" }, + { "R 0x151e100" }, + { "R 0x150b100" }, + /* limits */ + { "R 0xec80004" }, + { "R 0xec80058" }, + { "R 0xec80060 8" }, + { "R 0xec800a0 40" }, + { "R 0x634008" }, + { "R 0x634f00 8" }, + { "R 0x635560" }, + { "R 0x635570" }, + { "R 0x635580" }, + { "R 0x635590" }, + { "R 0x6355a0 4" }, + { "R 0x635600" }, + { "R 0x635610" }, + { "R 0x636008" }, + { "R 0x636f00 8" }, + { "R 0x637560" }, + { "R 0x637570" }, + { "R 0x637580" }, + { "R 0x637590" }, + { "R 0x6375a0 4" }, + { "R 0x637600" }, + { "R 0x637610" }, + { "R 0x18370220 2" }, + { "R 0x183702a0 2" }, + { "R 0x183704a0 12" }, + { "R 0x18370520" }, + { "R 0x18370588" }, + { "R 0x18370d10 12" }, + { "R 0x18370f90 10" }, + { "R 0x18371010 10" }, + { "R 0x18371a10 8" }, + { "R 0x183784a0 12" }, + { "R 0x18378520" }, + { "R 0x18378588" }, + { "R 0x18378d10 8" }, + { "R 0x18378f90 6" }, + { "R 0x18379010 6" }, + { "R 0x18379a10 4" }, + { "R 0xa310220 3" }, + { "R 0xa3102a0 3" }, + { "R 0xa3104a0 6" }, + { "R 0xa310520" }, + { "R 0xa310588" }, + { "R 0xa310d10 8" }, + { "R 0xa310f90 6" }, + { "R 0xa311010 6" }, + { "R 0xa311a10 3" }, + /* gic */ + { "R 0x17a00104 29" }, + { "R 0x17a00204 29" }, + /* mmssnoc */ + { "R 0x1741008" }, + { "R 0x1740000" }, + { "R 0x1740008" }, + { "R 0x1740010" }, + { "R 0x1740020 8" }, + { "R 0x174b048" }, + { "R 0x174b100 8" }, + { "R 0x90e0000" }, + { "R 0x90e0008" }, + { "R 0x90e0010" }, + { "R 0x90e0020 8" }, + { "R 0x90e0248" }, + { "R 0x90e3100" }, + { "R 0x90e4100 7" }, + { "R 0x1750010" }, + { "R 0x1750190" }, + { "R 0x1751010" }, + { "R 0x1752010" }, + { "R 0x1754010" }, + { "R 0x1755010" }, + { "R 0x1756010" }, + { "R 0x1758010" }, + { "R 0x1758090" }, + { "R 0x1759010" }, + { "R 0x175a010" }, + { "R 0x175c010" }, + { "R 0x175d010" }, + { "R 0x175e010" }, + /* apps_rsc */ + { "R 0xb201020 2" }, + { "R 0xb200010 4" }, + { "R 0xb220010 4" }, + { "R 0xb200900 4" }, + { "R 0xb220900 4" }, + { "R 0xb201030" }, + { "R 0xb201204" }, + { "R 0xb201218" }, + { "R 0xb20122c" }, + { "R 0xb201240" }, + { "R 0xb201254" }, + { "R 0xb201208" }, + { "R 0xb20121c" }, + { "R 0xb201230" }, + { "R 0xb201244" }, + { "R 0xb201258" }, + { "R 0xb204510 2" }, + { "R 0xb204520" }, + { "R 0xb211024" }, + { "R 0xb221024" }, + { "R 0xb231024" }, + { "R 0x18220010" }, + { "R 0x18220030" }, + { "R 0x182200d0" }, + { "R 0x18220408" }, + { "R 0x18230408" }, + /* misc */ + { "R 0x17e00434" }, + { "R 0x17e0043c 2" }, + { "R 0x17c00038 2" }, + { "R 0x17c00438" }, + { "R 0x17e0041c 2" }, + { "R 0x17e00404" }, + { "R 0xc220000 2" }, + { "R 0xc230000 6" }, + { "R 0xc260008" }, + { "R 0x18598014" }, + { "R 0x4d8634" }, + { "R 0x4d8834" }, + { "R 0x418620" }, + { "R 0x418820" }, + /* ddr */ + { "R 0x9084208" }, + { "R 0x9084204" }, + { "R 0x9084108" }, + { "R 0x90841c0" }, + { "R 0x10c034 2" }, + { "R 0x144018" }, + { "R 0x9050078" }, + { "R 0x9050110 8" }, + { "R 0x9080058 2" }, + { "R 0x90800c8" }, + { "R 0x90800d4" }, + { "R 0x90800e0" }, + { "R 0x90800fc" }, + { "R 0x9084030" }, + { "R 0x9084038 2" }, + { "R 0x90840e4" }, + { "R 0x90840f4" }, + { "R 0x9084104 2" }, + { "R 0x9084198" }, + { "R 0x9084804" }, + { "R 0x908480c" }, + { "R 0x9084844" }, + { "R 0x9084850 2" }, + { "R 0x9084860 3" }, + { "R 0x9084888 2" }, + { "R 0x908409c 2" }, + { "R 0x908426c" }, + { "R 0x908439c" }, + { "R 0x9085124" }, + { "R 0x9085134 2" }, + { "R 0x9084840" }, + { "R 0x9084834" }, + { "R 0x9085124" }, + { "R 0x90ba280" }, + { "R 0x90ba288 7" }, + { "R 0x9258610 4" }, + { "R 0x92d8610 4" }, + { "R 0x9220344 8" }, + { "R 0x9220370 6" }, + { "R 0x9220480" }, + { "R 0x9222400" }, + { "R 0x922240c" }, + { "R 0x9223214 2" }, + { "R 0x9223220 3" }, + { "R 0x9223308" }, + { "R 0x9223318" }, + { "R 0x9232100" }, + { "R 0x9236040 6" }, + { "R 0x92360b0" }, + { "R 0x923a004 4" }, + { "R 0x923e030 2" }, + { "R 0x9241000" }, + { "R 0x9242028" }, + { "R 0x9242044 3" }, + { "R 0x9242070" }, + { "R 0x9248030" }, + { "R 0x9248048 8" }, + { "R 0x9238030" }, + { "R 0x9238060 2" }, + { "R 0x9238074" }, + { "R 0x9238088" }, + { "R 0x92380a0" }, + { "R 0x92380b0" }, + { "R 0x92a0344 8" }, + { "R 0x92a0370 6" }, + { "R 0x92a0480" }, + { "R 0x92a2400" }, + { "R 0x92a240c" }, + { "R 0x92a3214 2" }, + { "R 0x92a3220 3" }, + { "R 0x92a3308" }, + { "R 0x92a3318" }, + { "R 0x92b2100" }, + { "R 0x92b6040 6" }, + { "R 0x92b60b0" }, + { "R 0x92ba004 4" }, + { "R 0x92be030 2" }, + { "R 0x92c1000" }, + { "R 0x92c2028" }, + { "R 0x92c2044 3" }, + { "R 0x92c2070" }, + { "R 0x92c8030" }, + { "R 0x92c8048 8" }, + { "R 0x92b8030" }, + { "R 0x92b8060 2" }, + { "R 0x92b8074" }, + { "R 0x92b8088" }, + { "R 0x92b80a0" }, + { "R 0x92b80b0" }, + { "R 0x92c8064" }, + { "R 0x9270080" }, + { "R 0x9270310" }, + { "R 0x9270400" }, + { "R 0x9270410 6" }, + { "R 0x9270430" }, + { "R 0x9270440" }, + { "R 0x9270448" }, + { "R 0x92704a0" }, + { "R 0x92704b0" }, + { "R 0x92704b8 2" }, + { "R 0x92704d0 2" }, + { "R 0x9271400" }, + { "R 0x9271408" }, + { "R 0x927341c 2" }, + { "R 0x92753b0" }, + { "R 0x9275804" }, + { "R 0x9275c18 2" }, + { "R 0x9275c2c" }, + { "R 0x9275c38" }, + { "R 0x9276418 2" }, + { "R 0x9279100" }, + { "R 0x9279110" }, + { "R 0x9279120" }, + { "R 0x9279180 2" }, + { "R 0x92f0080" }, + { "R 0x92f0310" }, + { "R 0x92f0400" }, + { "R 0x92f0410 6" }, + { "R 0x92f0430" }, + { "R 0x92f0440" }, + { "R 0x92f0448" }, + { "R 0x92f04a0" }, + { "R 0x92f04b0" }, + { "R 0x92f04b8 2" }, + { "R 0x92f04d0 2" }, + { "R 0x92f1400" }, + { "R 0x92f1408" }, + { "R 0x92f341c 2" }, + { "R 0x92f53b0" }, + { "R 0x92f5804" }, + { "R 0x92f5c18 2" }, + { "R 0x92f5c2c" }, + { "R 0x92f5c38" }, + { "R 0x92f6418 2" }, + { "R 0x92f9100" }, + { "R 0x92f9110" }, + { "R 0x92f9120" }, + { "R 0x92f9180 2" }, + { "R 0x9260080" }, + { "R 0x9260400" }, + { "R 0x9260410 3" }, + { "R 0x9260420 2" }, + { "R 0x9260430" }, + { "R 0x9260440" }, + { "R 0x9260448" }, + { "R 0x92604a0" }, + { "R 0x92604b0" }, + { "R 0x92604b8 2" }, + { "R 0x92604d0 2" }, + { "R 0x9261400" }, + { "R 0x9263410" }, + { "R 0x92653b0" }, + { "R 0x9265804" }, + { "R 0x9265b1c" }, + { "R 0x9265b2c" }, + { "R 0x9265b38" }, + { "R 0x9269100" }, + { "R 0x9269108" }, + { "R 0x9269110" }, + { "R 0x9269118" }, + { "R 0x9269120" }, + { "R 0x9269180 2" }, + { "R 0x92e0080" }, + { "R 0x92e0400" }, + { "R 0x92e0410 3" }, + { "R 0x92e0420 2" }, + { "R 0x92e0430" }, + { "R 0x92e0440" }, + { "R 0x92e0448" }, + { "R 0x92e04a0" }, + { "R 0x92e04b0" }, + { "R 0x92e04b8 2" }, + { "R 0x92e04d0 2" }, + { "R 0x92e1400" }, + { "R 0x92e3410" }, + { "R 0x92e53b0" }, + { "R 0x92e5804" }, + { "R 0x92e5b1c" }, + { "R 0x92e5b2c" }, + { "R 0x92e5b38" }, + { "R 0x92e9100" }, + { "R 0x92e9108" }, + { "R 0x92e9110" }, + { "R 0x92e9118" }, + { "R 0x92e9120" }, + { "R 0x92e9180 2" }, + { "R 0x96b0868" }, + { "R 0x96b0870" }, + { "R 0x96b1004" }, + { "R 0x96b100c" }, + { "R 0x96b1014" }, + { "R 0x96b1204" }, + { "R 0x96b120c" }, + { "R 0x96b1214" }, + { "R 0x96b1504" }, + { "R 0x96b150c" }, + { "R 0x96b1514" }, + { "R 0x96b1604" }, + { "R 0x96b8100" }, + { "R 0x96b813c" }, + { "R 0x96b8500" }, + { "R 0x96b853c" }, + { "R 0x96b8a04" }, + { "R 0x96b8a18" }, + { "R 0x96b8ea8" }, + { "R 0x96b9044" }, + { "R 0x96b904c" }, + { "R 0x96b9054" }, + { "R 0x96b905c" }, + { "R 0x96b910c 2" }, + { "R 0x96b9204" }, + { "R 0x96b920c" }, + { "R 0x96b9238" }, + { "R 0x96b9240" }, + { "R 0x96b926c" }, + { "R 0x96b9394" }, + { "R 0x96b939c" }, + { "R 0x96b9704" }, + { "R 0x96b970c" }, + { "R 0x96f0868" }, + { "R 0x96f0870" }, + { "R 0x96f1004" }, + { "R 0x96f100c" }, + { "R 0x96f1014" }, + { "R 0x96f1204" }, + { "R 0x96f120c" }, + { "R 0x96f1214" }, + { "R 0x96f1504" }, + { "R 0x96f150c" }, + { "R 0x96f1514" }, + { "R 0x96f1604" }, + { "R 0x96f8100" }, + { "R 0x96f813c" }, + { "R 0x96f8500" }, + { "R 0x96f853c" }, + { "R 0x96f8a04" }, + { "R 0x96f8a18" }, + { "R 0x96f8ea8" }, + { "R 0x96f9044" }, + { "R 0x96f904c" }, + { "R 0x96f9054" }, + { "R 0x96f905c" }, + { "R 0x96f910c 2" }, + { "R 0x96f9204" }, + { "R 0x96f920c" }, + { "R 0x96f9238" }, + { "R 0x96f9240" }, + { "R 0x96f926c" }, + { "R 0x96f9394" }, + { "R 0x96f939c" }, + { "R 0x96f9704" }, + { "R 0x96f970c" }, + { "R 0x9130100 3" }, + { "R 0x9170100 3" }, + { "R 0x91dd100 4" }, + { "R 0x91df100" }, + { "R 0x610110 5" }, + { "R 0x9230010" }, + { "R 0x9230020" }, + { "R 0x9230030" }, + { "R 0x9230040" }, + { "R 0x92b0010" }, + { "R 0x92b0020" }, + { "R 0x92b0030" }, + { "R 0x92b0040" }, + { "R 0x9232050" }, + { "R 0x923605c 2" }, + { "R 0x92360a0" }, + { "R 0x923a018 2" }, + { "R 0x92b2050" }, + { "R 0x92b605c 2" }, + { "R 0x92b60a0" }, + { "R 0x92ba018 2" }, + { "R 0x9222404 2" }, + { "R 0x9222410" }, + { "R 0x9238004" }, + { "R 0x9238014" }, + { "R 0x923805c" }, + { "R 0x923a014" }, + { "R 0x92a2404 2" }, + { "R 0x92a2410" }, + { "R 0x92b8004" }, + { "R 0x92b8014" }, + { "R 0x92b805c" }, + { "R 0x92ba014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x6e0a014" }, + { "R 0x9067e00 124" }, + { "R 0x905000c" }, + { "R 0x9050948" }, + { "R 0x9050078" }, + { "R 0x9050008" }, +}; + +static const struct dcc_register_entry kodiak_dcc_entries_ll3[] = { + /* lpass_rsc */ + { "R 0xb254520" }, + { "R 0xb251020 2" }, + { "R 0xb251030" }, + { "R 0xb251200" }, + { "R 0xb251214" }, + { "R 0xb251228" }, + { "R 0xb25123c" }, + { "R 0xb251250" }, + { "R 0xb251204" }, + { "R 0xb251218" }, + { "R 0xb25122c" }, + { "R 0xb251240" }, + { "R 0xb251254" }, + { "R 0xb251208" }, + { "R 0xb25121c" }, + { "R 0xb251230" }, + { "R 0xb251244" }, + { "R 0xb251258" }, + { "R 0xb254510 2" }, + { "R 0xb250010 2" }, + { "R 0xb250900 2" }, + { "R 0x3500010 3" }, + { "R 0x3500030" }, + { "R 0x3500038" }, + { "R 0x3500040" }, + { "R 0x3500048" }, + { "R 0x35000d0" }, + { "R 0x3500210" }, + { "R 0x3500230" }, + { "R 0x3500250" }, + { "R 0x3500270" }, + { "R 0x3500290" }, + { "R 0x35002b0" }, + { "R 0x3500208" }, + { "R 0x3500228" }, + { "R 0x3500248" }, + { "R 0x3500268" }, + { "R 0x3500288" }, + { "R 0x35002a8" }, + { "R 0x350020c" }, + { "R 0x350022c" }, + { "R 0x350024c" }, + { "R 0x350026c" }, + { "R 0x350028c" }, + { "R 0x35002ac" }, + { "R 0x3500400 3" }, + { "R 0x3500d04" }, + { "R 0x30b0010 3" }, + { "R 0x30b0210" }, + { "R 0x30b0230" }, + { "R 0x30b0250" }, + { "R 0x30b0270" }, + { "R 0x30b0290" }, + { "R 0x30b02b0" }, + { "R 0x30b0208" }, + { "R 0x30b0228" }, + { "R 0x30b0248" }, + { "R 0x30b0268" }, + { "R 0x30b0288" }, + { "R 0x30b02a8" }, + { "R 0x30b020c" }, + { "R 0x30b022c" }, + { "R 0x30b024c" }, + { "R 0x30b026c" }, + { "R 0x30b028c" }, + { "R 0x30b02ac" }, + { "R 0x30b0400 3" }, + /* nsp_rsc */ + { "R 0xb2b4520" }, + { "R 0xb2b1020 2" }, + { "R 0xb2b1030" }, + { "R 0xb2b1200" }, + { "R 0xb2b1214" }, + { "R 0xb2b1228" }, + { "R 0xb2b123c" }, + { "R 0xb2b1250" }, + { "R 0xb2b1204" }, + { "R 0xb2b1218" }, + { "R 0xb2b122c" }, + { "R 0xb2b1240" }, + { "R 0xb2b1254" }, + { "R 0xb2b1208" }, + { "R 0xb2b121c" }, + { "R 0xb2b1230" }, + { "R 0xb2b1244" }, + { "R 0xb2b1258" }, + { "R 0xb2b4510 2" }, + { "R 0xb2b0010" }, + { "R 0xb2b0900" }, + { "R 0xa302028" }, + { "R 0xa0a4010 3" }, + { "R 0xa0a4030" }, + { "R 0xa0a4038" }, + { "R 0xa0a4040 2" }, + { "R 0xa0a40d0" }, + { "R 0xa0a4210" }, + { "R 0xa0a4230" }, + { "R 0xa0a4250" }, + { "R 0xa0a4270" }, + { "R 0xa0a4290" }, + { "R 0xa0a42b0" }, + { "R 0xa0a4208" }, + { "R 0xa0a4228" }, + { "R 0xa0a4248" }, + { "R 0xa0a4268" }, + { "R 0xa0a4288" }, + { "R 0xa0a42a8" }, + { "R 0xa0a420c" }, + { "R 0xa0a422c" }, + { "R 0xa0a424c" }, + { "R 0xa0a426c" }, + { "R 0xa0a428c" }, + { "R 0xa0a42ac" }, + { "R 0xa0a4400 2" }, + { "R 0xa0a4d04" }, + { "R 0xa3b0010 3" }, + { "R 0xa3b0210" }, + { "R 0xa3b0230" }, + { "R 0xa3b0250" }, + { "R 0xa3b0270" }, + { "R 0xa3b0290" }, + { "R 0xa3b02b0" }, + { "R 0xa3b0208" }, + { "R 0xa3b0228" }, + { "R 0xa3b0248" }, + { "R 0xa3b0268" }, + { "R 0xa3b0288" }, + { "R 0xa3b02a8" }, + { "R 0xa3b020c" }, + { "R 0xa3b022c" }, + { "R 0xa3b024c" }, + { "R 0xa3b026c" }, + { "R 0xa3b028c" }, + { "R 0xa3b02ac" }, + { "R 0xa3b0400 3" }, + /* sdi_debug */ + { "R 0x10413c" }, + { "R 0x105024" }, + { "R 0x108008" }, + { "R 0x108004" }, + { "R 0x10500c" }, + { "R 0x105658" }, + { "R 0x105640 2" }, + { "R 0x105010" }, + { "R 0x10500c" }, + { "R 0x105024" }, + /* lpass_cdsp_tunning */ + { "R 0x3500d00 3" }, + { "R 0x3500d10 4" }, + { "R 0x3500fb0 4" }, + { "R 0x3501250 4" }, + { "R 0x35014f0 4" }, + { "R 0x3501790 4" }, + { "R 0x3501a30 4" }, + { "R 0x3503d44 4" }, + { "R 0x35000d0 3" }, + { "R 0x3500100" }, + { "R 0x3500d3c" }, + { "R 0xa0a40d0 3" }, + { "R 0xa0a4100" }, + { "R 0xa0a4d3c" }, + { "R 0xa0a7d44 4" }, + { "R 0xa0a4d00 3" }, + /* wpss_rsc */ + { "R 0x8a02028" }, + { "R 0x8b00000 2" }, + { "R 0x8b00010 15" }, + { "R 0x8b000d0" }, + { "R 0x8b000d8" }, + { "R 0x8b00100 3" }, + { "R 0x8b00200 2" }, + { "R 0x8b00224" }, + { "R 0x8b00244" }, + { "R 0x8b00264" }, + { "R 0x8b00284" }, + { "R 0x8b002a4" }, + { "R 0x8b00208" }, + { "R 0x8b00228" }, + { "R 0x8b00248" }, + { "R 0x8b00268" }, + { "R 0x8b00288" }, + { "R 0x8b002a8" }, + { "R 0x8b0020c" }, + { "R 0x8b0022c" }, + { "R 0x8b0024c" }, + { "R 0x8b0026c" }, + { "R 0x8b0028c" }, + { "R 0x8b002ac" }, + { "R 0x8b00210" }, + { "R 0x8b00230" }, + { "R 0x8b00250" }, + { "R 0x8b00270" }, + { "R 0x8b00290" }, + { "R 0x8b002b0" }, + { "R 0x8b00400 3" }, + { "R 0x8b00460 2" }, + { "R 0x8b004a0 7" }, + { "R 0x8ab0000 2" }, + { "R 0x8ab0010 3" }, + { "R 0x8ab00d0" }, + { "R 0x8ab00d8" }, + { "R 0x8ab0100 3" }, + { "R 0x8ab0200 2" }, + { "R 0x8ab0224" }, + { "R 0x8ab0244" }, + { "R 0x8ab0264" }, + { "R 0x8ab0284" }, + { "R 0x8ab02a4" }, + { "R 0x8ab0208" }, + { "R 0x8ab0228" }, + { "R 0x8ab0248" }, + { "R 0x8ab0268" }, + { "R 0x8ab0288" }, + { "R 0x8ab02a8" }, + { "R 0x8ab020c" }, + { "R 0x8ab022c" }, + { "R 0x8ab024c" }, + { "R 0x8ab026c" }, + { "R 0x8ab028c" }, + { "R 0x8ab02ac" }, + { "R 0x8ab0210" }, + { "R 0x8ab0230" }, + { "R 0x8ab0250" }, + { "R 0x8ab0270" }, + { "R 0x8ab0290" }, + { "R 0x8ab02b0" }, + { "R 0x8ab0400 3" }, + { "R 0x8ab0460 2" }, + { "R 0x8ab04a0 7" }, + /* video_noc */ + { "R 0xaa10504 2" }, + { "R 0xaa10510" }, + { "R 0xaa10520 8" }, + { "R 0xaa10300" }, + { "R 0xaa10010" }, + { "R 0xaa10020" }, +}; + +static const struct dcc_link_config kodiak_link_configs[] = { + { + .link_list = 6, + .entries = kodiak_dcc_entries_ll6, + .num_entries = ARRAY_SIZE(kodiak_dcc_entries_ll6), + }, + { + .link_list = 4, + .entries = kodiak_dcc_entries_ll4, + .num_entries = ARRAY_SIZE(kodiak_dcc_entries_ll4), + }, + { + .link_list = 3, + .entries = kodiak_dcc_entries_ll3, + .num_entries = ARRAY_SIZE(kodiak_dcc_entries_ll3), + }, +}; + +static const struct dcc_config kodiak_config = { + .lists = kodiak_link_configs, + .num_lists = ARRAY_SIZE(kodiak_link_configs), +}; + +static const struct dcc_pdata kodiak_pdata = { + .base = 0x0117f000, + .size = 0x00001000, + .ram_base = 0x01112000, + .ram_size = 0x00006000, + .dcc_offset = 0x12000, + .map_ver = 0x2, + .config = &kodiak_config, +}; + +#endif /* _QCOM_DCC_KODIAK_CONFIG_H */ diff --git a/drivers/misc/qcom-dcc-lemans-config.h b/drivers/misc/qcom-dcc-lemans-config.h new file mode 100644 index 0000000000000..01327a67eac91 --- /dev/null +++ b/drivers/misc/qcom-dcc-lemans-config.h @@ -0,0 +1,1630 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _QCOM_DCC_LEMANS_CONFIG_H +#define _QCOM_DCC_LEMANS_CONFIG_H + +#include "qcom-dcc.h" + +static const struct dcc_register_entry lemans_dcc_entries[] = { + /* lpm */ + { "R 0x18000040" }, + { "R 0x18000024" }, + { "R 0x18010024" }, + { "R 0x18010040" }, + { "R 0x18020024" }, + { "R 0x18020040" }, + { "R 0x18030024" }, + { "R 0x18030040" }, + { "R 0x18040024" }, + { "R 0x18040040" }, + { "R 0x18050024" }, + { "R 0x18050040" }, + { "R 0x18060024" }, + { "R 0x18060040" }, + { "R 0x18070024" }, + { "R 0x18070040" }, + { "R 0x18080024" }, + { "R 0x18080040" }, + { "R 0x18090024" }, + { "R 0x18090040" }, + { "R 0x18090104" }, + { "R 0x180900f8" }, + { "R 0x180800f4" }, + { "R 0x18080104" }, + { "R 0x180800f8" }, + { "R 0x180900f4" }, + { "R 0x18210404 2" }, + { "R 0xb201024" }, + { "R 0xb201020" }, + { "R 0xb211020" }, + { "R 0xb221020" }, + { "R 0xb232020" }, + /* tsens */ + { "R 0xc222004" }, + { "R 0xc263014" }, + { "R 0xc2630e0" }, + { "R 0xc2630ec" }, + { "R 0xc2630a0 16" }, + { "R 0xc2630e8" }, + { "R 0xc26313c" }, + { "R 0xc223004" }, + { "R 0xc265014" }, + { "R 0xc2650e0" }, + { "R 0xc2650ec" }, + { "R 0xc2650a0 16" }, + { "R 0xc2650e8" }, + { "R 0xc26513c" }, + { "R 0xc224004" }, + { "R 0xc251014" }, + { "R 0xc2510e0" }, + { "R 0xc2510ec" }, + { "R 0xc2510a0 16" }, + { "R 0xc2510e8" }, + { "R 0xc25113c" }, + { "R 0xc225004" }, + { "R 0xc252014" }, + { "R 0xc2520e0" }, + { "R 0xc2520ec" }, + { "R 0xc2520a0 16" }, + { "R 0xc2520e8" }, + { "R 0xc25213c" }, + /* pll */ + { "R 0x182b0000 12" }, + { "R 0x182b003c" }, + { "R 0x182b4000 12" }, + { "R 0x182b403c" }, + { "R 0x18280044" }, + { "R 0x1828004c" }, + { "R 0x18280060" }, + { "R 0x18284044" }, + { "R 0x1828404c" }, + { "R 0x18284060" }, + { "R 0x182b2000 12" }, + { "R 0x182b203c" }, + { "R 0x182b6000 12" }, + { "R 0x182b603c" }, + { "R 0x18282044" }, + { "R 0x1828204c" }, + { "R 0x18282060" }, + { "R 0x18286044" }, + { "R 0x1828604c" }, + { "R 0x18286060" }, + /* limits */ + { "R 0xec80010" }, + { "R 0xec81000" }, + { "R 0xec81010 64" }, + { "R 0x18378220 2" }, + { "R 0x183782a0 2" }, + { "R 0x183784a0 12" }, + { "R 0x18378520" }, + { "R 0x18378588" }, + { "R 0x18378d90 12" }, + { "R 0x18379010 10" }, + { "R 0x18379090 10" }, + { "R 0x18379a90 8" }, + { "R 0x18370220 2" }, + { "R 0x183702a0 2" }, + { "R 0x183704a0 12" }, + { "R 0x18370520" }, + { "R 0x18370588" }, + { "R 0x18370d90 12" }, + { "R 0x18371010 10" }, + { "R 0x18371090 10" }, + { "R 0x18371a90 8" }, + { "R 0x26310220 3" }, + { "R 0x263102a0 3" }, + { "R 0x263104a0 6" }, + { "R 0x26310520" }, + { "R 0x26310588" }, + { "R 0x26310d90 8" }, + { "R 0x26311010 6" }, + { "R 0x26311090 6" }, + { "R 0x26311a90 3" }, + { "R 0x2a310220 3" }, + { "R 0x2a3102a0 3" }, + { "R 0x2a3104a0 6" }, + { "R 0x2a310520" }, + { "R 0x2a310588" }, + { "R 0x2a310d90 8" }, + { "R 0x2a311010 6" }, + { "R 0x2a311090 6" }, + { "R 0x2a311a90 3" }, + /* ddr */ + { "R 0x610110" }, + { "R 0x9050008" }, + { "R 0x905001c" }, + { "R 0x9050030" }, + { "R 0x9050050" }, + { "R 0x9050070" }, + { "R 0x9050948" }, + { "R 0x905104c" }, + { "R 0x9050078" }, + { "R 0x9050114" }, + { "R 0x90a0008 2" }, + { "R 0x90a0080" }, + { "R 0x90a1008 2" }, + { "R 0x90a80e4 2" }, + { "R 0x90a816c 2" }, + { "R 0x90a80f8" }, + { "R 0x90a8180 2" }, + { "R 0x90a81a0 2" }, + { "R 0x90a81c0 2" }, + { "R 0x90a81e4" }, + { "R 0x90a81f4" }, + { "R 0x90a8204" }, + { "R 0x90a8218" }, + { "R 0x90a8228" }, + { "R 0x90a8264 2" }, + { "R 0x90a8270" }, + { "R 0x90a80fc 4" }, + { "R 0x90a8148 2" }, + { "R 0x90a8158 2" }, + { "R 0x90a8190 2" }, + { "R 0x90a81b0 2" }, + { "R 0x90a81d0 2" }, + { "R 0x90a84f4" }, + { "R 0x90a8138" }, + { "R 0x90a8150" }, + { "R 0x90a8160" }, + { "R 0x90a8198" }, + { "R 0x90a81b8" }, + { "R 0x90a81d8" }, + { "R 0x90a8280" }, + { "R 0x90a819c" }, + { "R 0x90a81bc" }, + { "R 0x90a81dc" }, + { "R 0x90a821c" }, + { "R 0x90a822c" }, + { "R 0x90a84f8" }, + { "R 0x90a9204" }, + { "R 0x90a880c" }, + { "R 0x90a8834" }, + { "R 0x90a8840 2" }, + { "R 0x90a8850 2" }, + { "R 0x90a8860 3" }, + { "R 0x90a8878" }, + { "R 0x90a8884" }, + { "R 0x90a9260" }, + { "R 0x90a8900" }, + { "R 0x90a9134 2" }, + { "R 0x90a8880 3" }, + { "R 0x90e0000" }, + { "R 0x90e0008" }, + { "R 0x90e0010" }, + { "R 0x90e0020 8" }, + { "R 0x90e0248" }, + { "R 0x90e3100" }, + { "R 0x90e4100 7" }, + { "R 0x90e5008" }, + { "R 0x90e5010 3" }, + { "R 0x9080024" }, + { "R 0x908002c" }, + { "R 0x9080034" }, + { "R 0x908003c" }, + { "R 0x9080044" }, + { "R 0x908004c" }, + { "R 0x9080054 3" }, + { "R 0x90800b8 2" }, + { "R 0x90800c8" }, + { "R 0x90800d4" }, + { "R 0x90800e0" }, + { "R 0x90800ec" }, + { "R 0x90800f8" }, + { "R 0x9080104 2" }, + { "R 0x9080114" }, + { "R 0x9080144" }, + { "R 0x908014c" }, + { "R 0x9080168" }, + { "R 0x9080174" }, + { "R 0x90801ac" }, + { "R 0x90801b4" }, + { "R 0x90801bc" }, + { "R 0x90801c4" }, + { "R 0x90801cc" }, + { "R 0x90801d4" }, + { "R 0x90801dc" }, + { "R 0x90801e4" }, + { "R 0x90801ec" }, + { "R 0x90801f4 2" }, + { "R 0x90ba000" }, + { "R 0x90ba050" }, + { "R 0x90ba280" }, + { "R 0x90ba288" }, + { "R 0x9280610 4" }, + { "R 0x9280680 4" }, + { "R 0x9380610" }, + { "R 0x9280614" }, + { "R 0x9380618 2" }, + { "R 0x9380680 4" }, + { "R 0x9480610 4" }, + { "R 0x9480680 4" }, + { "R 0x9580610 4" }, + { "R 0x9580680 4" }, + { "R 0x9680610 4" }, + { "R 0x9680680 4" }, + { "R 0x9380610" }, + { "R 0x9280614" }, + { "R 0x9380618 2" }, + { "R 0x9380680 4" }, + { "R 0x9100008" }, + { "R 0x9100010" }, + { "R 0x9100408" }, + { "R 0x9100410" }, + { "R 0x9100808" }, + { "R 0x9100810" }, + { "R 0x9100c08" }, + { "R 0x9100c10" }, + { "R 0x9101008" }, + { "R 0x9101010" }, + { "R 0x9101408" }, + { "R 0x9101410" }, + { "R 0x9101808" }, + { "R 0x9101810" }, + { "R 0x9101c08" }, + { "R 0x9101c10" }, + { "R 0x9102008" }, + { "R 0x9102010" }, + { "R 0x9102408" }, + { "R 0x9102410" }, + { "R 0x9102808" }, + { "R 0x9102810" }, + { "R 0x9102c08" }, + { "R 0x9102c10" }, + { "R 0x9103000 3" }, + { "R 0x9103010 3" }, + { "R 0x9103080 3" }, + { "R 0x9103090 3" }, + { "R 0x9104008" }, + { "R 0x9105008" }, + { "R 0x9106008" }, + { "R 0x9107008" }, + { "R 0x9108008" }, + { "R 0x9108808" }, + { "R 0x9109008" }, + { "R 0x9109808" }, + { "R 0x9122008" }, + { "R 0x9122408" }, + { "R 0x9122808" }, + { "R 0x9122c08" }, + { "R 0x9123008" }, + { "R 0x9123408" }, + { "R 0x9123808" }, + { "R 0x9123c08" }, + { "R 0x9124008" }, + { "R 0x9124408" }, + { "R 0x9124808" }, + { "R 0x9124c08" }, + { "R 0x9138008" }, + { "R 0x9138088" }, + { "R 0x9138108" }, + { "R 0x9138188" }, + { "R 0x9138208" }, + { "R 0x9138288" }, + { "R 0x9138308" }, + { "R 0x9138388" }, + { "R 0x9138408" }, + { "R 0x9138488" }, + { "R 0x9138508" }, + { "R 0x9138588" }, + { "R 0x9138608" }, + { "R 0x9138688" }, + { "R 0x9140000 5" }, + { "R 0x9140018" }, + { "R 0x9140020 3" }, + { "R 0x9140030 3" }, + { "R 0x9140040 3" }, + { "R 0x9140400 5" }, + { "R 0x9140418" }, + { "R 0x9140420 3" }, + { "R 0x9140430 3" }, + { "R 0x9140440 3" }, + { "R 0x9140800 5" }, + { "R 0x9140818" }, + { "R 0x9140820 3" }, + { "R 0x9140830 3" }, + { "R 0x9140840 3" }, + { "R 0x9140c00 5" }, + { "R 0x9140c18" }, + { "R 0x9140c20 3" }, + { "R 0x9140c30 3" }, + { "R 0x9140c40 3" }, + { "R 0x9141000 5" }, + { "R 0x9141018" }, + { "R 0x9141020 3" }, + { "R 0x9141030 3" }, + { "R 0x9141040 3" }, + { "R 0x9141400 5" }, + { "R 0x9141418" }, + { "R 0x9141420 3" }, + { "R 0x9141430 3" }, + { "R 0x9141440 3" }, + { "R 0x9141800 3" }, + { "R 0x9141810 3" }, + { "R 0x9141880 3" }, + { "R 0x9141890 3" }, + { "R 0x9142008" }, + { "R 0x9143008" }, + { "R 0x9144008" }, + { "R 0x9145008" }, + { "R 0x9146008" }, + { "R 0x9147008" }, + { "R 0x9148008" }, + { "R 0x9149008" }, + { "R 0x914a008" }, + { "R 0x914b008" }, + { "R 0x914c008" }, + { "R 0x914d008" }, + { "R 0x914e008" }, + { "R 0x914e208" }, + { "R 0x914e408" }, + { "R 0x914e608" }, + { "R 0x914e808" }, + { "R 0x914ea08" }, + { "R 0x914f008" }, + { "R 0x914f808" }, + { "R 0x9150008" }, + { "R 0x9150808" }, + { "R 0x9151008" }, + { "R 0x9151808" }, + { "R 0x9152008" }, + { "R 0x9153008" }, + { "R 0x9154008" }, + { "R 0x9155008" }, + { "R 0x9156008" }, + { "R 0x9157008" }, + { "R 0x9163008" }, + { "R 0x9163010" }, + { "R 0x9165008" }, + { "R 0x9165010" }, + { "R 0x9167008" }, + { "R 0x9167010" }, + { "R 0x9169008" }, + { "R 0x9169010" }, + { "R 0x916b008" }, + { "R 0x916b010" }, + { "R 0x916d008" }, + { "R 0x916d010" }, + { "R 0x9170000 3" }, + { "R 0x9170010" }, + { "R 0x9170018" }, + { "R 0x9170020 6" }, + { "R 0x9170400 3" }, + { "R 0x9170410" }, + { "R 0x9170418" }, + { "R 0x9170420 6" }, + { "R 0x9170800 3" }, + { "R 0x9170810" }, + { "R 0x9170818" }, + { "R 0x9170820 6" }, + { "R 0x9170c00 3" }, + { "R 0x9170c10" }, + { "R 0x9170c18" }, + { "R 0x9170c20 6" }, + { "R 0x9171000 3" }, + { "R 0x9171010" }, + { "R 0x9171018" }, + { "R 0x9171020 6" }, + { "R 0x9171400 3" }, + { "R 0x9171410" }, + { "R 0x9171418" }, + { "R 0x9171420 6" }, + { "R 0x9178008" }, + { "R 0x9178088" }, + { "R 0x9178108" }, + { "R 0x9178188" }, + { "R 0x9178208" }, + { "R 0x9178288" }, + { "R 0x9178308" }, + { "R 0x9178388" }, + { "R 0x9178408" }, + { "R 0x9178488" }, + { "R 0x9178508" }, + { "R 0x9180000 5" }, + { "R 0x9180018" }, + { "R 0x9180020 3" }, + { "R 0x9180030 3" }, + { "R 0x9180040 3" }, + { "R 0x9180400 5" }, + { "R 0x9180418" }, + { "R 0x9180420 3" }, + { "R 0x9180430 3" }, + { "R 0x9180440 3" }, + { "R 0x9180800 3" }, + { "R 0x9180810 3" }, + { "R 0x9180880 3" }, + { "R 0x9180890 3" }, + { "R 0x9180900 3" }, + { "R 0x9180910 3" }, + { "R 0x9180a08" }, + { "R 0x9181008" }, + { "R 0x9181808" }, + { "R 0x9182208" }, + { "R 0x9182408" }, + { "R 0x9182608" }, + { "R 0x9182808" }, + { "R 0x9182a08" }, + { "R 0x9182c08" }, + { "R 0x9182e08" }, + { "R 0x9183008" }, + { "R 0x9183808" }, + { "R 0x9184008" }, + { "R 0x9185008" }, + { "R 0x9186008" }, + { "R 0x9187008" }, + { "R 0x9188008" }, + { "R 0x9189008" }, + { "R 0x918a008" }, + { "R 0x918b008" }, + { "R 0x9190000 3" }, + { "R 0x9190010" }, + { "R 0x9190018" }, + { "R 0x9190020 6" }, + { "R 0x9190400 3" }, + { "R 0x9190410" }, + { "R 0x9190418" }, + { "R 0x9190420 6" }, + { "R 0x9197008" }, + { "R 0x9197088" }, + { "R 0x9198008" }, + { "R 0x9198088" }, + { "R 0x9198108" }, + { "R 0x9198188" }, + { "R 0x9198208" }, + { "R 0x9198288" }, + { "R 0x9198308" }, + { "R 0x9198388" }, + { "R 0x9198408" }, + { "R 0x9198508" }, + { "R 0x9198588" }, + { "R 0x9198608" }, + { "R 0x9198688" }, + { "R 0x9198708" }, + { "R 0x9198788" }, + { "R 0x9198808" }, + { "R 0x9198888" }, + { "R 0x9198908" }, + { "R 0x9198988" }, + { "R 0x9198a08" }, + { "R 0x9198a88" }, + { "R 0x9198b08" }, + { "R 0x9198b88" }, + { "R 0x9198c08" }, + { "R 0x9198e40" }, + { "R 0x9198e48" }, + { "R 0x91a0008" }, + { "R 0x91a1008" }, + { "R 0x91a2008" }, + { "R 0x91a6040" }, + { "R 0x91a6048" }, + { "R 0x91a8100 4" }, + { "R 0x91ab100" }, + { "R 0x91ac040" }, + { "R 0x91ac048" }, + { "R 0x91ad008" }, + { "R 0x91ad010" }, + { "R 0x91ae008" }, + { "R 0x91ae010" }, + { "R 0x91af008" }, + { "R 0x91af010" }, + { "R 0x91b0008" }, + { "R 0x91b0010" }, + { "R 0x91b1008" }, + { "R 0x91b1010" }, + { "R 0x91b2008" }, + { "R 0x91b2010" }, + { "R 0x91b3008" }, + { "R 0x91b3010" }, + { "R 0x91b4008" }, + { "R 0x91b4010" }, + { "R 0x91b5008" }, + { "R 0x91b5010" }, + { "R 0x91b6008" }, + { "R 0x91b6010" }, + { "R 0x91b7008" }, + { "R 0x91b7010" }, + { "R 0x91b8008" }, + { "R 0x91b8010" }, + { "R 0x91b9008" }, + { "R 0x91b9010" }, + { "R 0x91ba008" }, + { "R 0x91ba010" }, + { "R 0x91c0008" }, + { "R 0x91c0208" }, + { "R 0x91c0408" }, + { "R 0x91c0608" }, + { "R 0x91c0808" }, + { "R 0x91c0a08" }, + { "R 0x91c0c08" }, + { "R 0x91c0e08" }, + { "R 0x91c1008" }, + { "R 0x91c1208" }, + { "R 0x91c1408" }, + { "R 0x91c1608" }, + { "R 0x91c1808" }, + { "R 0x91c1a08" }, + { "R 0x91c1c08" }, + { "R 0x91c1e08" }, + { "R 0x91d8008" }, + { "R 0x91d8088" }, + { "R 0x91d8108" }, + { "R 0x91d8188" }, + { "R 0x91d8208" }, + { "R 0x91d8288" }, + { "R 0x91d8308" }, + { "R 0x91d8388" }, + { "R 0x91d8408" }, + { "R 0x91d8488" }, + { "R 0x91d8508" }, + { "R 0x91d8588" }, + { "R 0x91d8608" }, + { "R 0x91d8688" }, + { "R 0x91d8708" }, + { "R 0x91d8788" }, + { "R 0x91e1008" }, + { "R 0x91e1088" }, + { "R 0x91e1108" }, + { "R 0x91e3008" }, + { "R 0x91e3010" }, + { "R 0x91e4008" }, + { "R 0x91e4010" }, + { "R 0x91e5008" }, + { "R 0x91e5010" }, + { "R 0x91e6008" }, + { "R 0x91e6010" }, + { "R 0x91e7008" }, + { "R 0x91e7010" }, + { "R 0x91e8008" }, + { "R 0x91e8010" }, + { "R 0x91e9008" }, + { "R 0x91e9010" }, + { "R 0x91ea008" }, + { "R 0x91ea010" }, + { "R 0x91eb008" }, + { "R 0x91eb010" }, + { "R 0x91ec008" }, + { "R 0x91ec010" }, + { "R 0x91ed008" }, + { "R 0x91ed010" }, + { "R 0x91ee008" }, + { "R 0x91ee010" }, + { "R 0x91ef008" }, + { "R 0x91ef010" }, + { "R 0x91f0008" }, + { "R 0x91f0010" }, + { "R 0x91f1008" }, + { "R 0x91f1010" }, + { "R 0x91f2008" }, + { "R 0x91f2010" }, + { "R 0x91f3008" }, + { "R 0x91f3010" }, + { "R 0x91f4008" }, + { "R 0x91f4010" }, + { "R 0x91f5008" }, + { "R 0x91f5010" }, + { "R 0x91f6008" }, + { "R 0x91f6010" }, + { "R 0x9220344 9" }, + { "R 0x9220370 7" }, + { "R 0x9220480" }, + { "R 0x9222400 26" }, + { "R 0x9222470 5" }, + { "R 0x922320c" }, + { "R 0x9223214 2" }, + { "R 0x9223220 4" }, + { "R 0x9223308" }, + { "R 0x9223318" }, + { "R 0x922358c" }, + { "R 0x9234010" }, + { "R 0x923801c 8" }, + { "R 0x9238050" }, + { "R 0x9238100 7" }, + { "R 0x923c004" }, + { "R 0x923c014" }, + { "R 0x923c020" }, + { "R 0x923c030" }, + { "R 0x923c05c 3" }, + { "R 0x923c074" }, + { "R 0x923c088" }, + { "R 0x923c0a0" }, + { "R 0x923c0b0" }, + { "R 0x923c0c0" }, + { "R 0x923c0d0" }, + { "R 0x923c0e0" }, + { "R 0x923c0f0" }, + { "R 0x923c100" }, + { "R 0x923d064" }, + { "R 0x9240008 6" }, + { "R 0x9240028" }, + { "R 0x924203c 6" }, + { "R 0x9242058 7" }, + { "R 0x924208c" }, + { "R 0x92420b0" }, + { "R 0x92420b8 3" }, + { "R 0x92420f4" }, + { "R 0x92420fc 7" }, + { "R 0x9242324 14" }, + { "R 0x9242410" }, + { "R 0x92430a8" }, + { "R 0x9248004 7" }, + { "R 0x9248024" }, + { "R 0x9248040" }, + { "R 0x9248048" }, + { "R 0x9249064" }, + { "R 0x924c000" }, + { "R 0x924c030 3" }, + { "R 0x924c040 3" }, + { "R 0x924c054 2" }, + { "R 0x924c078" }, + { "R 0x924c108" }, + { "R 0x924c110" }, + { "R 0x9250020" }, + { "R 0x9251054" }, + { "R 0x9252014 3" }, + { "R 0x9252028 17" }, + { "R 0x9252070 8" }, + { "R 0x9252098" }, + { "R 0x92520a0" }, + { "R 0x92520b4" }, + { "R 0x92520c0 4" }, + { "R 0x92520f4 10" }, + { "R 0x9252120 12" }, + { "R 0x925802c" }, + { "R 0x925809c 2" }, + { "R 0x92580a8 3" }, + { "R 0x92580b8" }, + { "R 0x92580c0 7" }, + { "R 0x92580e0" }, + { "R 0x92580e8" }, + { "R 0x92580f0" }, + { "R 0x92580f8" }, + { "R 0x9258100" }, + { "R 0x9258108" }, + { "R 0x9258110" }, + { "R 0x9258118" }, + { "R 0x9258120" }, + { "R 0x9258128" }, + { "R 0x9258210 3" }, + { "R 0x9259010" }, + { "R 0x9259070" }, + { "R 0x925b004" }, + { "R 0x926004c 10" }, + { "R 0x9260078" }, + { "R 0x926020c" }, + { "R 0x9260214" }, + { "R 0x9261084" }, + { "R 0x9262020" }, + { "R 0x9263020" }, + { "R 0x9264020" }, + { "R 0x9265020" }, + { "R 0x9320344 9" }, + { "R 0x9320370 7" }, + { "R 0x9320480" }, + { "R 0x9322400 26" }, + { "R 0x9322470 5" }, + { "R 0x932320c" }, + { "R 0x9323214 2" }, + { "R 0x9323220 4" }, + { "R 0x9323308" }, + { "R 0x9323318" }, + { "R 0x932358c" }, + { "R 0x9334010" }, + { "R 0x933801c 8" }, + { "R 0x9338050" }, + { "R 0x9338100 7" }, + { "R 0x933c004" }, + { "R 0x933c014" }, + { "R 0x933c020" }, + { "R 0x933c030" }, + { "R 0x933c05c 3" }, + { "R 0x933c074" }, + { "R 0x933c088" }, + { "R 0x933c0a0" }, + { "R 0x933c0b0" }, + { "R 0x933c0c0" }, + { "R 0x933c0d0" }, + { "R 0x933c0e0" }, + { "R 0x933c0f0" }, + { "R 0x933c100" }, + { "R 0x933d064" }, + { "R 0x9340008 6" }, + { "R 0x9340028" }, + { "R 0x934203c 6" }, + { "R 0x9342058 7" }, + { "R 0x934208c" }, + { "R 0x93420b0" }, + { "R 0x93420b8 3" }, + { "R 0x93420f4" }, + { "R 0x93420fc 7" }, + { "R 0x9342324 14" }, + { "R 0x9342410" }, + { "R 0x93430a8" }, + { "R 0x9348004 7" }, + { "R 0x9348024" }, + { "R 0x9348040" }, + { "R 0x9348048" }, + { "R 0x9349064" }, + { "R 0x934c000" }, + { "R 0x934c030 3" }, + { "R 0x934c040 3" }, + { "R 0x934c054 2" }, + { "R 0x934c078" }, + { "R 0x934c108" }, + { "R 0x934c110" }, + { "R 0x9350020" }, + { "R 0x9351054" }, + { "R 0x9352014 3" }, + { "R 0x9352028 17" }, + { "R 0x9352070 8" }, + { "R 0x9352098" }, + { "R 0x93520a0" }, + { "R 0x93520b4" }, + { "R 0x93520c0 4" }, + { "R 0x93520f4 10" }, + { "R 0x9352120 12" }, + { "R 0x935802c" }, + { "R 0x935809c 2" }, + { "R 0x93580a8 3" }, + { "R 0x93580b8" }, + { "R 0x93580c0 7" }, + { "R 0x93580e0" }, + { "R 0x93580e8" }, + { "R 0x93580f0" }, + { "R 0x93580f8" }, + { "R 0x9358100" }, + { "R 0x9358108" }, + { "R 0x9358110" }, + { "R 0x9358118" }, + { "R 0x9358120" }, + { "R 0x9358128" }, + { "R 0x9358210 3" }, + { "R 0x9359010" }, + { "R 0x9359070" }, + { "R 0x935b004" }, + { "R 0x936004c 10" }, + { "R 0x9360078" }, + { "R 0x936020c" }, + { "R 0x9360214" }, + { "R 0x9361084" }, + { "R 0x9362020" }, + { "R 0x9363020" }, + { "R 0x9364020" }, + { "R 0x9365020" }, + { "R 0x9420344 9" }, + { "R 0x9420370 7" }, + { "R 0x9420480" }, + { "R 0x9422400 26" }, + { "R 0x9422470 5" }, + { "R 0x942320c" }, + { "R 0x9423214 2" }, + { "R 0x9423220 4" }, + { "R 0x9423308" }, + { "R 0x9423318" }, + { "R 0x942358c" }, + { "R 0x9434010" }, + { "R 0x943801c 8" }, + { "R 0x9438050" }, + { "R 0x9438100 7" }, + { "R 0x943c004" }, + { "R 0x943c014" }, + { "R 0x943c020" }, + { "R 0x943c030" }, + { "R 0x943c05c 3" }, + { "R 0x943c074" }, + { "R 0x943c088" }, + { "R 0x943c0a0" }, + { "R 0x943c0b0" }, + { "R 0x943c0c0" }, + { "R 0x943c0d0" }, + { "R 0x943c0e0" }, + { "R 0x943c0f0" }, + { "R 0x943c100" }, + { "R 0x943d064" }, + { "R 0x9440008 6" }, + { "R 0x9440028" }, + { "R 0x944203c 6" }, + { "R 0x9442058 7" }, + { "R 0x944208c" }, + { "R 0x94420b0" }, + { "R 0x94420b8 3" }, + { "R 0x94420f4" }, + { "R 0x94420fc 7" }, + { "R 0x9442324 14" }, + { "R 0x9442410" }, + { "R 0x94430a8" }, + { "R 0x9448004 7" }, + { "R 0x9448024" }, + { "R 0x9448040" }, + { "R 0x9448048" }, + { "R 0x9449064" }, + { "R 0x944c000" }, + { "R 0x944c030 3" }, + { "R 0x944c040 3" }, + { "R 0x944c054 2" }, + { "R 0x944c078" }, + { "R 0x944c108" }, + { "R 0x944c110" }, + { "R 0x9450020" }, + { "R 0x9451054" }, + { "R 0x9452014 3" }, + { "R 0x9452028 17" }, + { "R 0x9452070 8" }, + { "R 0x9452098" }, + { "R 0x94520a0" }, + { "R 0x94520b4" }, + { "R 0x94520c0 4" }, + { "R 0x94520f4 10" }, + { "R 0x9452120 12" }, + { "R 0x945802c" }, + { "R 0x945809c 2" }, + { "R 0x94580a8 3" }, + { "R 0x94580b8" }, + { "R 0x94580c0 7" }, + { "R 0x94580e0" }, + { "R 0x94580e8" }, + { "R 0x94580f0" }, + { "R 0x94580f8" }, + { "R 0x9458100" }, + { "R 0x9458108" }, + { "R 0x9458110" }, + { "R 0x9458118" }, + { "R 0x9458120" }, + { "R 0x9458128" }, + { "R 0x9458210 3" }, + { "R 0x9459010" }, + { "R 0x9459070" }, + { "R 0x945b004" }, + { "R 0x946004c 10" }, + { "R 0x9460078" }, + { "R 0x946020c" }, + { "R 0x9460214" }, + { "R 0x9461084" }, + { "R 0x9462020" }, + { "R 0x9463020" }, + { "R 0x9464020" }, + { "R 0x9465020" }, + { "R 0x9520344 9" }, + { "R 0x9520370 7" }, + { "R 0x9520480" }, + { "R 0x9522400 26" }, + { "R 0x9522470 5" }, + { "R 0x952320c" }, + { "R 0x9523214 2" }, + { "R 0x9523220 4" }, + { "R 0x9523308" }, + { "R 0x9523318" }, + { "R 0x952358c" }, + { "R 0x9534010" }, + { "R 0x953801c 8" }, + { "R 0x9538050" }, + { "R 0x9538100 7" }, + { "R 0x953c004" }, + { "R 0x953c014" }, + { "R 0x953c020" }, + { "R 0x953c030" }, + { "R 0x953c05c 3" }, + { "R 0x953c074" }, + { "R 0x953c088" }, + { "R 0x953c0a0" }, + { "R 0x953c0b0" }, + { "R 0x953c0c0" }, + { "R 0x953c0d0" }, + { "R 0x953c0e0" }, + { "R 0x953c0f0" }, + { "R 0x953c100" }, + { "R 0x953d064" }, + { "R 0x9540008 6" }, + { "R 0x9540028" }, + { "R 0x954203c 6" }, + { "R 0x9542058 7" }, + { "R 0x954208c" }, + { "R 0x95420b0" }, + { "R 0x95420b8 3" }, + { "R 0x95420f4" }, + { "R 0x95420fc 7" }, + { "R 0x9542324 14" }, + { "R 0x9542410" }, + { "R 0x95430a8" }, + { "R 0x9548004 7" }, + { "R 0x9548024" }, + { "R 0x9548040" }, + { "R 0x9548048" }, + { "R 0x9549064" }, + { "R 0x954c000" }, + { "R 0x954c030 3" }, + { "R 0x954c040 3" }, + { "R 0x954c054 2" }, + { "R 0x954c078" }, + { "R 0x954c108" }, + { "R 0x954c110" }, + { "R 0x9550020" }, + { "R 0x9551054" }, + { "R 0x9552014 3" }, + { "R 0x9552028 17" }, + { "R 0x9552070 8" }, + { "R 0x9552098" }, + { "R 0x95520a0" }, + { "R 0x95520b4" }, + { "R 0x95520c0 4" }, + { "R 0x95520f4 10" }, + { "R 0x9552120 12" }, + { "R 0x955802c" }, + { "R 0x955809c 2" }, + { "R 0x95580a8 3" }, + { "R 0x95580b8" }, + { "R 0x95580c0 7" }, + { "R 0x95580e0" }, + { "R 0x95580e8" }, + { "R 0x95580f0" }, + { "R 0x95580f8" }, + { "R 0x9558100" }, + { "R 0x9558108" }, + { "R 0x9558110" }, + { "R 0x9558118" }, + { "R 0x9558120" }, + { "R 0x9558128" }, + { "R 0x9558210 3" }, + { "R 0x9559010" }, + { "R 0x9559070" }, + { "R 0x955b004" }, + { "R 0x956004c 10" }, + { "R 0x9560078" }, + { "R 0x956020c" }, + { "R 0x9560214" }, + { "R 0x9561084" }, + { "R 0x9562020" }, + { "R 0x9563020" }, + { "R 0x9564020" }, + { "R 0x9565020" }, + { "R 0x9620344 9" }, + { "R 0x9620370 7" }, + { "R 0x9620480" }, + { "R 0x9622400 26" }, + { "R 0x9622470 5" }, + { "R 0x962320c" }, + { "R 0x9623214 2" }, + { "R 0x9623220 4" }, + { "R 0x9623308" }, + { "R 0x9623318" }, + { "R 0x962358c" }, + { "R 0x9634010" }, + { "R 0x963801c 8" }, + { "R 0x9638050" }, + { "R 0x9638100 7" }, + { "R 0x963c004" }, + { "R 0x963c014" }, + { "R 0x963c020" }, + { "R 0x963c030" }, + { "R 0x963c05c 3" }, + { "R 0x963c074" }, + { "R 0x963c088" }, + { "R 0x963c0a0" }, + { "R 0x963c0b0" }, + { "R 0x963c0c0" }, + { "R 0x963c0d0" }, + { "R 0x963c0e0" }, + { "R 0x963c0f0" }, + { "R 0x963c100" }, + { "R 0x963d064" }, + { "R 0x9640008 6" }, + { "R 0x9640028" }, + { "R 0x964203c 6" }, + { "R 0x9642058 7" }, + { "R 0x964208c" }, + { "R 0x96420b0" }, + { "R 0x96420b8 3" }, + { "R 0x96420f4" }, + { "R 0x96420fc 7" }, + { "R 0x9642324 14" }, + { "R 0x9642410" }, + { "R 0x96430a8" }, + { "R 0x9648004 7" }, + { "R 0x9648024" }, + { "R 0x9648040" }, + { "R 0x9648048" }, + { "R 0x9649064" }, + { "R 0x964c000" }, + { "R 0x964c030 3" }, + { "R 0x964c040 3" }, + { "R 0x964c054 2" }, + { "R 0x964c078" }, + { "R 0x964c108" }, + { "R 0x964c110" }, + { "R 0x9650020" }, + { "R 0x9651054" }, + { "R 0x9652014 3" }, + { "R 0x9652028 17" }, + { "R 0x9652070 8" }, + { "R 0x9652098" }, + { "R 0x96520a0" }, + { "R 0x96520b4" }, + { "R 0x96520c0 4" }, + { "R 0x96520f4 10" }, + { "R 0x9652120 12" }, + { "R 0x965802c" }, + { "R 0x965809c 2" }, + { "R 0x96580a8 3" }, + { "R 0x96580b8" }, + { "R 0x96580c0 7" }, + { "R 0x96580e0" }, + { "R 0x96580e8" }, + { "R 0x96580f0" }, + { "R 0x96580f8" }, + { "R 0x9658100" }, + { "R 0x9658108" }, + { "R 0x9658110" }, + { "R 0x9658118" }, + { "R 0x9658120" }, + { "R 0x9658128" }, + { "R 0x9658210 3" }, + { "R 0x9659010" }, + { "R 0x9659070" }, + { "R 0x965b004" }, + { "R 0x966004c 10" }, + { "R 0x9660078" }, + { "R 0x966020c" }, + { "R 0x9660214" }, + { "R 0x9661084" }, + { "R 0x9662020" }, + { "R 0x9663020" }, + { "R 0x9664020" }, + { "R 0x9665020" }, + { "R 0x9720344 9" }, + { "R 0x9720370 7" }, + { "R 0x9720480" }, + { "R 0x9722400 26" }, + { "R 0x9722470 5" }, + { "R 0x972320c" }, + { "R 0x9723214 2" }, + { "R 0x9723220 4" }, + { "R 0x9723308" }, + { "R 0x9723318" }, + { "R 0x972358c" }, + { "R 0x9734010" }, + { "R 0x973801c 8" }, + { "R 0x9738050" }, + { "R 0x9738100 7" }, + { "R 0x973c004" }, + { "R 0x973c014" }, + { "R 0x973c020" }, + { "R 0x973c030" }, + { "R 0x973c05c 3" }, + { "R 0x973c074" }, + { "R 0x973c088" }, + { "R 0x973c0a0" }, + { "R 0x973c0b0" }, + { "R 0x973c0c0" }, + { "R 0x973c0d0" }, + { "R 0x973c0e0" }, + { "R 0x973c0f0" }, + { "R 0x973c100" }, + { "R 0x973d064" }, + { "R 0x9740008 6" }, + { "R 0x9740028" }, + { "R 0x974203c 6" }, + { "R 0x9742058 7" }, + { "R 0x974208c" }, + { "R 0x97420b0" }, + { "R 0x97420b8 3" }, + { "R 0x97420f4" }, + { "R 0x97420fc 7" }, + { "R 0x9742324 14" }, + { "R 0x9742410" }, + { "R 0x97430a8" }, + { "R 0x9748004 7" }, + { "R 0x9748024" }, + { "R 0x9748040" }, + { "R 0x9748048" }, + { "R 0x9749064" }, + { "R 0x974c000" }, + { "R 0x974c030 3" }, + { "R 0x974c040 3" }, + { "R 0x974c054 2" }, + { "R 0x974c078" }, + { "R 0x974c108" }, + { "R 0x974c110" }, + { "R 0x9750020" }, + { "R 0x9751054" }, + { "R 0x9752014 3" }, + { "R 0x9752028 17" }, + { "R 0x9752070 8" }, + { "R 0x9752098" }, + { "R 0x97520a0" }, + { "R 0x97520b4" }, + { "R 0x97520c0 4" }, + { "R 0x97520f4 10" }, + { "R 0x9752120 12" }, + { "R 0x975802c" }, + { "R 0x975809c 2" }, + { "R 0x97580a8 3" }, + { "R 0x97580b8" }, + { "R 0x97580c0 7" }, + { "R 0x97580e0" }, + { "R 0x97580e8" }, + { "R 0x97580f0" }, + { "R 0x97580f8" }, + { "R 0x9758100" }, + { "R 0x9758108" }, + { "R 0x9758110" }, + { "R 0x9758118" }, + { "R 0x9758120" }, + { "R 0x9758128" }, + { "R 0x9758210 3" }, + { "R 0x9759010" }, + { "R 0x9759070" }, + { "R 0x975b004" }, + { "R 0x976004c 10" }, + { "R 0x9760078" }, + { "R 0x976020c" }, + { "R 0x9760214" }, + { "R 0x9761084" }, + { "R 0x9762020" }, + { "R 0x9763020" }, + { "R 0x9764020" }, + { "R 0x9765020" }, + { "R 0x92a0304" }, + { "R 0x92a0080" }, + { "R 0x92a0310" }, + { "R 0x92a0400 2" }, + { "R 0x92a0410 6" }, + { "R 0x92a0430" }, + { "R 0x92a0440" }, + { "R 0x92a0448" }, + { "R 0x92a04a0" }, + { "R 0x92a04b0 4" }, + { "R 0x92a04d0 2" }, + { "R 0x92a1400" }, + { "R 0x92a1408" }, + { "R 0x92a2400 2" }, + { "R 0x92a2438 2" }, + { "R 0x92a2454" }, + { "R 0x92a3400 4" }, + { "R 0x92a3418 3" }, + { "R 0x92a4700" }, + { "R 0x92a53b0" }, + { "R 0x92a5804" }, + { "R 0x92a590c" }, + { "R 0x92a5a14" }, + { "R 0x92a5c0c" }, + { "R 0x92a5c18 2" }, + { "R 0x92a5c2c 2" }, + { "R 0x92a5c38" }, + { "R 0x92a5c4c" }, + { "R 0x92a5ca4" }, + { "R 0x92a5cac 3" }, + { "R 0x92a6400" }, + { "R 0x92a6418 2" }, + { "R 0x92a9100" }, + { "R 0x92a9110" }, + { "R 0x92a9120" }, + { "R 0x92a9180 2" }, + { "R 0x92a91a0" }, + { "R 0x92a91b0" }, + { "R 0x92a91c0 2" }, + { "R 0x92a91e0" }, + { "R 0x93a0304" }, + { "R 0x93a0080" }, + { "R 0x93a0310" }, + { "R 0x93a0400 2" }, + { "R 0x93a0410 6" }, + { "R 0x93a0430" }, + { "R 0x93a0440" }, + { "R 0x93a0448" }, + { "R 0x93a04a0" }, + { "R 0x93a04b0 4" }, + { "R 0x93a04d0 2" }, + { "R 0x93a1400" }, + { "R 0x93a1408" }, + { "R 0x93a2400 2" }, + { "R 0x93a2438 2" }, + { "R 0x93a2454" }, + { "R 0x93a3400 4" }, + { "R 0x93a3418 3" }, + { "R 0x93a4700" }, + { "R 0x93a53b0" }, + { "R 0x93a5804" }, + { "R 0x93a590c" }, + { "R 0x93a5a14" }, + { "R 0x93a5c0c" }, + { "R 0x93a5c18 2" }, + { "R 0x93a5c2c 2" }, + { "R 0x93a5c38" }, + { "R 0x93a5c4c" }, + { "R 0x93a5ca4" }, + { "R 0x93a5cac 3" }, + { "R 0x93a6400" }, + { "R 0x93a6418 2" }, + { "R 0x93a9100" }, + { "R 0x93a9110" }, + { "R 0x93a9120" }, + { "R 0x93a9180 2" }, + { "R 0x93a91a0" }, + { "R 0x93a91b0" }, + { "R 0x93a91c0 2" }, + { "R 0x93a91e0" }, + { "R 0x94a0304" }, + { "R 0x94a0080" }, + { "R 0x94a0310" }, + { "R 0x94a0400 2" }, + { "R 0x94a0410 6" }, + { "R 0x94a0430" }, + { "R 0x94a0440" }, + { "R 0x94a0448" }, + { "R 0x94a04a0" }, + { "R 0x94a04b0 4" }, + { "R 0x94a04d0 2" }, + { "R 0x94a1400" }, + { "R 0x94a1408" }, + { "R 0x94a2400 2" }, + { "R 0x94a2438 2" }, + { "R 0x94a2454" }, + { "R 0x94a3400 4" }, + { "R 0x94a3418 3" }, + { "R 0x94a4700" }, + { "R 0x94a53b0" }, + { "R 0x94a5804" }, + { "R 0x94a590c" }, + { "R 0x94a5a14" }, + { "R 0x94a5c0c" }, + { "R 0x94a5c18 2" }, + { "R 0x94a5c2c 2" }, + { "R 0x94a5c38" }, + { "R 0x94a5c4c" }, + { "R 0x94a5ca4" }, + { "R 0x94a5cac 3" }, + { "R 0x94a6400" }, + { "R 0x94a6418 2" }, + { "R 0x94a9100" }, + { "R 0x94a9110" }, + { "R 0x94a9120" }, + { "R 0x94a9180 2" }, + { "R 0x94a91a0" }, + { "R 0x94a91b0" }, + { "R 0x94a91c0 2" }, + { "R 0x94a91e0" }, + { "R 0x95a0304" }, + { "R 0x95a0080" }, + { "R 0x95a0310" }, + { "R 0x95a0400 2" }, + { "R 0x95a0410 6" }, + { "R 0x95a0430" }, + { "R 0x95a0440" }, + { "R 0x95a0448" }, + { "R 0x95a04a0" }, + { "R 0x95a04b0 4" }, + { "R 0x95a04d0 2" }, + { "R 0x95a1400" }, + { "R 0x95a1408" }, + { "R 0x95a2400 2" }, + { "R 0x95a2438 2" }, + { "R 0x95a2454" }, + { "R 0x95a3400 4" }, + { "R 0x95a3418 3" }, + { "R 0x95a4700" }, + { "R 0x95a53b0" }, + { "R 0x95a5804" }, + { "R 0x95a590c" }, + { "R 0x95a5a14" }, + { "R 0x95a5c0c" }, + { "R 0x95a5c18 2" }, + { "R 0x95a5c2c 2" }, + { "R 0x95a5c38" }, + { "R 0x95a5c4c" }, + { "R 0x95a5ca4" }, + { "R 0x95a5cac 3" }, + { "R 0x95a6400" }, + { "R 0x95a6418 2" }, + { "R 0x95a9100" }, + { "R 0x95a9110" }, + { "R 0x95a9120" }, + { "R 0x95a9180 2" }, + { "R 0x95a91a0" }, + { "R 0x95a91b0" }, + { "R 0x95a91c0 2" }, + { "R 0x95a91e0" }, + { "R 0x96a0304" }, + { "R 0x96a0080" }, + { "R 0x96a0310" }, + { "R 0x96a0400 2" }, + { "R 0x96a0410 6" }, + { "R 0x96a0430" }, + { "R 0x96a0440" }, + { "R 0x96a0448" }, + { "R 0x96a04a0" }, + { "R 0x96a04b0 4" }, + { "R 0x96a04d0 2" }, + { "R 0x96a1400" }, + { "R 0x96a1408" }, + { "R 0x96a2400 2" }, + { "R 0x96a2438 2" }, + { "R 0x96a2454" }, + { "R 0x96a3400 4" }, + { "R 0x96a3418 3" }, + { "R 0x96a4700" }, + { "R 0x96a53b0" }, + { "R 0x96a5804" }, + { "R 0x96a590c" }, + { "R 0x96a5a14" }, + { "R 0x96a5c0c" }, + { "R 0x96a5c18 2" }, + { "R 0x96a5c2c 2" }, + { "R 0x96a5c38" }, + { "R 0x96a5c4c" }, + { "R 0x96a5ca4" }, + { "R 0x96a5cac 3" }, + { "R 0x96a6400" }, + { "R 0x96a6418 2" }, + { "R 0x96a9100" }, + { "R 0x96a9110" }, + { "R 0x96a9120" }, + { "R 0x96a9180 2" }, + { "R 0x96a91a0" }, + { "R 0x96a91b0" }, + { "R 0x96a91c0 2" }, + { "R 0x96a91e0" }, + { "R 0x97a0304" }, + { "R 0x97a0080" }, + { "R 0x97a0310" }, + { "R 0x97a0400 2" }, + { "R 0x97a0410 6" }, + { "R 0x97a0430" }, + { "R 0x97a0440" }, + { "R 0x97a0448" }, + { "R 0x97a04a0" }, + { "R 0x97a04b0 4" }, + { "R 0x97a04d0 2" }, + { "R 0x97a1400" }, + { "R 0x97a1408" }, + { "R 0x97a2400 2" }, + { "R 0x97a2438 2" }, + { "R 0x97a2454" }, + { "R 0x97a3400 4" }, + { "R 0x97a3418 3" }, + { "R 0x97a4700" }, + { "R 0x97a53b0" }, + { "R 0x97a5804" }, + { "R 0x97a590c" }, + { "R 0x97a5a14" }, + { "R 0x97a5c0c" }, + { "R 0x97a5c18 2" }, + { "R 0x97a5c2c 2" }, + { "R 0x97a5c38" }, + { "R 0x97a5c4c" }, + { "R 0x97a5ca4" }, + { "R 0x97a5cac 3" }, + { "R 0x97a6400" }, + { "R 0x97a6418 2" }, + { "R 0x97a9100" }, + { "R 0x97a9110" }, + { "R 0x97a9120" }, + { "R 0x97a9180 2" }, + { "R 0x97a91a0" }, + { "R 0x97a91b0" }, + { "R 0x97a91c0 2" }, + { "R 0x97a91e0" }, + { "R 0x9b01e64" }, + { "R 0x9b01ea0" }, + { "R 0x9b01f30 2" }, + { "R 0x9b03e64" }, + { "R 0x9b03ea0" }, + { "R 0x9b03f30 2" }, + { "R 0x9b0527c" }, + { "R 0x9b05290" }, + { "R 0x9b054ec" }, + { "R 0x9b054f4" }, + { "R 0x9b05514" }, + { "R 0x9b0551c" }, + { "R 0x9b05524" }, + { "R 0x9b05548" }, + { "R 0x9b05550" }, + { "R 0x9b05558" }, + { "R 0x9b055b8" }, + { "R 0x9b055c0" }, + { "R 0x9b055ec" }, + { "R 0x9b05860" }, + { "R 0x9b05870" }, + { "R 0x9b058a0" }, + { "R 0x9b058a8" }, + { "R 0x9b058b0" }, + { "R 0x9b058b8" }, + { "R 0x9b058d8 2" }, + { "R 0x9b058f4" }, + { "R 0x9b058fc" }, + { "R 0x9b05920" }, + { "R 0x9b05928" }, + { "R 0x9b05944" }, + { "R 0x9b06604" }, + { "R 0x9b0660c" }, + { "R 0x9b41e64" }, + { "R 0x9b41ea0" }, + { "R 0x9b41f30 2" }, + { "R 0x9b43e64" }, + { "R 0x9b43ea0" }, + { "R 0x9b43f30 2" }, + { "R 0x9b4527c" }, + { "R 0x9b45290" }, + { "R 0x9b454ec" }, + { "R 0x9b454f4" }, + { "R 0x9b45514" }, + { "R 0x9b4551c" }, + { "R 0x9b45524" }, + { "R 0x9b45548" }, + { "R 0x9b45550" }, + { "R 0x9b45558" }, + { "R 0x9b455b8" }, + { "R 0x9b455c0" }, + { "R 0x9b455ec" }, + { "R 0x9b45860" }, + { "R 0x9b45870" }, + { "R 0x9b458a0" }, + { "R 0x9b458a8" }, + { "R 0x9b458b0" }, + { "R 0x9b458b8" }, + { "R 0x9b458d8 2" }, + { "R 0x9b458f4" }, + { "R 0x9b458fc" }, + { "R 0x9b45920" }, + { "R 0x9b45928" }, + { "R 0x9b45944" }, + { "R 0x9b46604" }, + { "R 0x9b4660c" }, + { "R 0x9b81e64" }, + { "R 0x9b81ea0" }, + { "R 0x9b81f30 2" }, + { "R 0x9b83e64" }, + { "R 0x9b83ea0" }, + { "R 0x9b83f30 2" }, + { "R 0x9b8527c" }, + { "R 0x9b85290" }, + { "R 0x9b854ec" }, + { "R 0x9b854f4" }, + { "R 0x9b85514" }, + { "R 0x9b8551c" }, + { "R 0x9b85524" }, + { "R 0x9b85548" }, + { "R 0x9b85550" }, + { "R 0x9b85558" }, + { "R 0x9b855b8" }, + { "R 0x9b855c0" }, + { "R 0x9b855ec" }, + { "R 0x9b85860" }, + { "R 0x9b85870" }, + { "R 0x9b858a0" }, + { "R 0x9b858a8" }, + { "R 0x9b858b0" }, + { "R 0x9b858b8" }, + { "R 0x9b858d8 2" }, + { "R 0x9b858f4" }, + { "R 0x9b858fc" }, + { "R 0x9b85920" }, + { "R 0x9b85928" }, + { "R 0x9b85944" }, + { "R 0x9b86604" }, + { "R 0x9b8660c" }, + { "R 0x9bc1e64" }, + { "R 0x9bc1ea0" }, + { "R 0x9bc1f30 2" }, + { "R 0x9bc3e64" }, + { "R 0x9bc3ea0" }, + { "R 0x9bc3f30 2" }, + { "R 0x9bc527c" }, + { "R 0x9bc5290" }, + { "R 0x9bc54ec" }, + { "R 0x9bc54f4" }, + { "R 0x9bc5514" }, + { "R 0x9bc551c" }, + { "R 0x9bc5524" }, + { "R 0x9bc5548" }, + { "R 0x9bc5550" }, + { "R 0x9bc5558" }, + { "R 0x9bc55b8" }, + { "R 0x9bc55c0" }, + { "R 0x9bc55ec" }, + { "R 0x9bc5860" }, + { "R 0x9bc5870" }, + { "R 0x9bc58a0" }, + { "R 0x9bc58a8" }, + { "R 0x9bc58b0" }, + { "R 0x9bc58b8" }, + { "R 0x9bc58d8 2" }, + { "R 0x9bc58f4" }, + { "R 0x9bc58fc" }, + { "R 0x9bc5920" }, + { "R 0x9bc5928" }, + { "R 0x9bc5944" }, + { "R 0x9bc6604" }, + { "R 0x9bc660c" }, + { "R 0x9c01e64" }, + { "R 0x9c01ea0" }, + { "R 0x9c01f30 2" }, + { "R 0x9c03e64" }, + { "R 0x9c03ea0" }, + { "R 0x9c03f30 2" }, + { "R 0x9c0527c" }, + { "R 0x9c05290" }, + { "R 0x9c054ec" }, + { "R 0x9c054f4" }, + { "R 0x9c05514" }, + { "R 0x9c0551c" }, + { "R 0x9c05524" }, + { "R 0x9c05548" }, + { "R 0x9c05550" }, + { "R 0x9c05558" }, + { "R 0x9c055b8" }, + { "R 0x9c055c0" }, + { "R 0x9c055ec" }, + { "R 0x9c05860" }, + { "R 0x9c05870" }, + { "R 0x9c058a0" }, + { "R 0x9c058a8" }, + { "R 0x9c058b0" }, + { "R 0x9c058b8" }, + { "R 0x9c058d8 2" }, + { "R 0x9c058f4" }, + { "R 0x9c058fc" }, + { "R 0x9c05920" }, + { "R 0x9c05928" }, + { "R 0x9c05944" }, + { "R 0x9c06604" }, + { "R 0x9c0660c" }, + { "R 0x9c41e64" }, + { "R 0x9c41ea0" }, + { "R 0x9c41f30 2" }, + { "R 0x9c43e64" }, + { "R 0x9c43ea0" }, + { "R 0x9c43f30 2" }, + { "R 0x9c4527c" }, + { "R 0x9c45290" }, + { "R 0x9c454ec" }, + { "R 0x9c454f4" }, + { "R 0x9c45514" }, + { "R 0x9c4551c" }, + { "R 0x9c45524" }, + { "R 0x9c45548" }, + { "R 0x9c45550" }, + { "R 0x9c45558" }, + { "R 0x9c455b8" }, + { "R 0x9c455c0" }, + { "R 0x9c455ec" }, + { "R 0x9c45860" }, + { "R 0x9c45870" }, + { "R 0x9c458a0" }, + { "R 0x9c458a8" }, + { "R 0x9c458b0" }, + { "R 0x9c458b8" }, + { "R 0x9c458d8 2" }, + { "R 0x9c458f4" }, + { "R 0x9c458fc" }, + { "R 0x9c45920" }, + { "R 0x9c45928" }, + { "R 0x9c45944" }, + { "R 0x9c46604" }, + { "R 0x9c4660c" }, +}; + +static const struct dcc_link_config lemans_link_configs[] = { + { + .link_list = 6, + .entries = lemans_dcc_entries, + .num_entries = ARRAY_SIZE(lemans_dcc_entries), + }, +}; + +static const struct dcc_config lemans_config = { + .lists = lemans_link_configs, + .num_lists = ARRAY_SIZE(lemans_link_configs), +}; + +static const struct dcc_pdata lemans_pdata = { + .base = 0x040ff000, + .size = 0x00001000, + .ram_base = 0x040b8800, + .ram_size = 0x00006000, + .dcc_offset = 0x38800, + .map_ver = 0x3, + .config = &lemans_config, +}; + +#endif /* _QCOM_DCC_LEMANS_CONFIG_H */ diff --git a/drivers/misc/qcom-dcc-pakala-config.h b/drivers/misc/qcom-dcc-pakala-config.h new file mode 100644 index 0000000000000..25e6221805d27 --- /dev/null +++ b/drivers/misc/qcom-dcc-pakala-config.h @@ -0,0 +1,1167 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _QCOM_DCC_PAKALA_CONFIG_H +#define _QCOM_DCC_PAKALA_CONFIG_H + +#include "qcom-dcc.h" + +static const struct dcc_register_entry pakala_dcc_entries_ll6[] = { + { "R 0x16801000 2" }, + { "R 0x18880258 10" }, + { "R 0x18880288 6" }, + { "R 0x188802a8 10" }, + { "R 0x18880328 6" }, + { "R 0x188803d8 10" }, + { "R 0x18880408 2" }, + { "R 0x19880258 10" }, + { "R 0x19880288 6" }, + { "R 0x198802a8 10" }, + { "R 0x19880328 6" }, + { "R 0x198803d8 10" }, + { "R 0x19880408 2" }, + { "R 0x188b0000 4" }, + { "R 0x188b0050 2" }, + { "R 0x188b00e8 36" }, + { "R 0x188b0788 2" }, + { "R 0x188b0c18 2" }, + { "R 0x198b0000 4" }, + { "R 0x198b0050 2" }, + { "R 0x198b00e8 36" }, + { "R 0x198b0788 2" }, + { "R 0x198b0c18 2" }, + { "R 0x17988814 2" }, + { "R 0x17998814 2" }, + { "R 0x179d2000 3" }, + { "R 0x179d2020 3" }, + { "R 0x179d2040 2" }, + { "R 0x179d2060 2" }, + { "R 0x179d2080 2" }, + { "R 0x179d20a0" }, + { "R 0x179d20b0" }, + { "R 0x179d20c0" }, + { "R 0x179d2200 2" }, + { "R 0x179d2220 2" }, + { "R 0x179d2280 2" }, + { "R 0x179d22f0" }, + { "R 0x179d2304" }, + { "R 0x179d2310" }, + { "R 0x179d2400 4" }, + { "R 0x179d2420" }, + { "R 0x179d2428 2" }, + { "R 0x179d2440 2" }, + { "R 0x179d2520 2" }, + { "R 0x179d2600 2" }, + { "R 0x179d2710 2" }, + { "R 0x179d2720 2" }, + { "R 0x179d2740 2" }, + { "R 0x179d3080 2" }, + { "R 0x17846018 2" }, + { "R 0x17846060" }, + { "R 0x17846100 2" }, + { "R 0x17846110" }, + { "R 0x17847030 2" }, + { "R 0x17847040 2" }, + { "R 0x17847050 2" }, + { "R 0x17847060 2" }, + { "R 0x17847070 2" }, + { "R 0x17847080 2" }, + { "R 0x17847090 2" }, + { "R 0x178470a0 2" }, + { "R 0x178470b0 2" }, + { "R 0x178470c0 2" }, + { "R 0x17850000 2" }, + { "R 0x17850010 2" }, + { "R 0x17850030 2" }, + { "R 0x17850040" }, + { "R 0x17854000" }, + { "R 0x17854008 4" }, + { "R 0x179c8814 2" }, + { "R 0x179d0104" }, + { "R 0x179d0118 2" }, + { "R 0x179d0148 2" }, + { "R 0x179d1600" }, + { "R 0x179d1678" }, + { "R 0x179d1688 2" }, + { "R 0x179d1694 3" }, + { "R 0x179d1820 3" }, + { "R 0x17b70000" }, + { "R 0x17b70008 2" }, + { "R 0x17b71000" }, + { "R 0x17b71008 2" }, + { "R 0x164807f8" }, + { "R 0x16480810 3" }, + { "R 0x16483000 40" }, + { "R 0x16483a00 2" }, + { "R 0x16488908" }, + { "R 0x16488c18" }, + { "R 0x164a8908" }, + { "R 0x164a8c18" }, + { "R 0x164a07f8" }, + { "R 0x164a0810 3" }, + { "R 0x164a3000 40" }, + { "R 0x164a3a00 2" }, + { "R 0x16493000 40" }, + { "R 0x16493a00 2" }, + { "R 0x16498c18" }, + { "R 0x164b8c18" }, + { "R 0x164b3a00 2" }, + { "R 0x16440000 2" }, + { "R 0x16440020 3" }, + { "R 0x16440030" }, + { "R 0x1644003c" }, + { "R 0x16440044 3" }, + { "R 0x16440438" }, + { "R 0x16440500 5" }, + { "R 0x16562000 2" }, + { "R 0x16565004" }, + { "R 0x17000bd0" }, + { "R 0x170404a0" }, + { "R 0x170a0590" }, + { "R 0x170c0330" }, + { "R 0x170c0338" }, + { "R 0x170c0340" }, + { "R 0x170c0518" }, + { "R 0x170c0528" }, + { "R 0x170c0538" }, + { "R 0x170c0560 4" }, + { "R 0x17200bd0" }, + { "R 0x172404a0" }, + { "R 0x172a0590" }, + { "R 0x172c0330" }, + { "R 0x172c0338" }, + { "R 0x172c0340" }, + { "R 0x172c0518" }, + { "R 0x172c0528" }, + { "R 0x172c0538" }, + { "R 0x172c0560 4" }, + { "R 0x1641000c" }, + { "R 0x1641400c" }, + { "R 0x18830320 2" }, + { "R 0x19830320 2" }, + { "R 0x18040010 6" }, + { "R 0x18040040 10" }, + { "R 0x18040090 6" }, + { "R 0x18850020 2" }, + { "R 0x18850060 2" }, + { "R 0x188500a0 2" }, + { "R 0x188500e0 2" }, + { "R 0x18850120 2" }, + { "R 0x18850160 2" }, + { "R 0x189c1000" }, + { "R 0x199c1000" }, + { "R 0x18a30000 9" }, + { "R 0x18a30030" }, + { "R 0x18a3003c 2" }, + { "R 0x19a30000 9" }, + { "R 0x19a30030" }, + { "R 0x19a3003c 2" }, + { "R 0x24201040" }, + { "R 0x24201048" }, + { "R 0x24100010" }, + { "R 0x24100020 6" }, + { "R 0x24180010" }, + { "R 0x24180020 6" }, + { "R 0x24200010" }, + { "R 0x24200020 6" }, + { "R 0x24200410" }, + { "R 0x24200420 6" }, + { "R 0x24102010" }, + { "R 0x24102038" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 3" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 3" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 3" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102008 2" }, + { "R 0x24181010" }, + { "R 0x24181038" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 3" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 3" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 3" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181008 2" }, + { "R 0x24203010" }, + { "R 0x24203038" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 3" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 3" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 3" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203030 2" }, + { "R 0x24203008 2" }, + { "R 0x24203410" }, + { "R 0x24203438" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 3" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 3" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 3" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203430 2" }, + { "R 0x24203408 2" }, + { "R 0x24104018" }, + { "R 0x24104008" }, + { "R 0x24104010 2" }, + { "R 0x24104010 2" }, + { "R 0x24104010 2" }, + { "R 0x24104010 2" }, + { "R 0x24104098" }, + { "R 0x24104088" }, + { "R 0x24104090 2" }, + { "R 0x24104090 2" }, + { "R 0x24104090 2" }, + { "R 0x24104090 2" }, + { "R 0x24104090 2" }, + { "R 0x24182018" }, + { "R 0x24182008" }, + { "R 0x24182010 2" }, + { "R 0x24182010 2" }, + { "R 0x24182010 2" }, + { "R 0x24182010 2" }, + { "R 0x24182098" }, + { "R 0x24182088" }, + { "R 0x24182090 2" }, + { "R 0x24182090 2" }, + { "R 0x24182090 2" }, + { "R 0x24182090 2" }, + { "R 0x24182090 2" }, + { "R 0x24204018" }, + { "R 0x24204008" }, + { "R 0x24204010 2" }, + { "R 0x24204010 2" }, + { "R 0x24204010 2" }, + { "R 0x24204098" }, + { "R 0x24204088" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24204090 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24102030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x24181030 2" }, + { "R 0x16801000 2" }, + { "R 0x240ba050" }, + { "R 0x240ba164" }, + { "R 0x240ba280" }, + { "R 0x240ba288 8" }, + { "R 0x240ba2c0 2" }, + { "R 0x24bc0610 11" }, + { "R 0x24bc0640 2" }, + { "R 0x24bc06a0" }, + { "R 0x24fc0610 11" }, + { "R 0x24fc0640 2" }, + { "R 0x24fc06a0" }, + { "R 0x25bc0610 11" }, + { "R 0x25bc0640 2" }, + { "R 0x25bc06a0" }, + { "R 0x25fc0610 11" }, + { "R 0x25fc0640 2" }, + { "R 0x25fc06a0" }, + { "R 0x240a801c 2" }, + { "R 0x240a8038 3" }, + { "R 0x240a8058 5" }, + { "R 0x240a8074 4" }, + { "R 0x240a80a8" }, + { "R 0x240a80b8" }, + { "R 0x240a80ec 5" }, + { "R 0x240a8140" }, + { "R 0x240a8164 2" }, + { "R 0x240a81c4 2" }, + { "R 0x240a81dc 2" }, + { "R 0x240a844c" }, + { "R 0x240a845c" }, + { "R 0x240a84c8 27" }, + { "R 0x240a9000 2" }, + { "R 0x240a9010 2" }, + { "R 0x240a9020 3" }, + { "R 0x240a9034 2" }, + { "R 0x240a9130 3" }, + { "R 0x240a9140 18" }, + { "R 0x240a91ac 6" }, + { "R 0x240a91c8" }, + { "R 0x240a920c" }, + { "R 0x240a9220 2" }, + { "R 0x240a9264" }, + { "R 0x240a9294 7" }, + { "R 0x240a92b8" }, + { "R 0x240a92f8" }, + { "R 0x24800004" }, + { "R 0x24801004" }, + { "R 0x24802004" }, + { "R 0x24803004" }, + { "R 0x24804004" }, + { "R 0x24805004" }, + { "R 0x24806004" }, + { "R 0x24807004" }, + { "R 0x24808004" }, + { "R 0x24809004" }, + { "R 0x2480a004" }, + { "R 0x2480b004" }, + { "R 0x2480c004" }, + { "R 0x2480d004" }, + { "R 0x2480e004" }, + { "R 0x2480f004" }, + { "R 0x24810004" }, + { "R 0x24811004" }, + { "R 0x24812004" }, + { "R 0x24813004" }, + { "R 0x24814004" }, + { "R 0x24815004" }, + { "R 0x24816004" }, + { "R 0x24817004" }, + { "R 0x24818004" }, + { "R 0x24819004" }, + { "R 0x2481a004" }, + { "R 0x2481b004" }, + { "R 0x2481c004" }, + { "R 0x2481d004" }, + { "R 0x2481e004" }, + { "R 0x2481f004" }, + { "R 0x24820004" }, + { "R 0x24821004" }, + { "R 0x24822004" }, + { "R 0x24823004" }, + { "R 0x24843350 2" }, + { "R 0x248433f4 11" }, + { "R 0x248434c0 6" }, + { "R 0x24843600" }, + { "R 0x2484360c" }, + { "R 0x24845040" }, + { "R 0x24845048" }, + { "R 0x24845050" }, + { "R 0x24845058" }, + { "R 0x24845060" }, + { "R 0x24845068" }, + { "R 0x24845070 4" }, + { "R 0x24845090 14" }, + { "R 0x248450cc 2" }, + { "R 0x24845240" }, + { "R 0x248452a0" }, + { "R 0x24845300" }, + { "R 0x24845310" }, + { "R 0x24845320" }, + { "R 0x24845330 3" }, + { "R 0x24847404" }, + { "R 0x2484740c 3" }, + { "R 0x24847448" }, + { "R 0x24847450" }, + { "R 0x24847458 2" }, + { "R 0x24847600" }, + { "R 0x24849000" }, + { "R 0x24849010" }, + { "R 0x2487c000" }, + { "R 0x2487c01c 3" }, + { "R 0x2487c030 8" }, + { "R 0x2487c054 2" }, + { "R 0x2487c078" }, + { "R 0x2487c108 9" }, + { "R 0x2487c20c 3" }, + { "R 0x248a600c 4" }, + { "R 0x248a6020 3" }, + { "R 0x248a6034 2" }, + { "R 0x248a6040" }, + { "R 0x248a6050" }, + { "R 0x248a6058" }, + { "R 0x248a6060 4" }, + { "R 0x248a7020" }, + { "R 0x248a7030 2" }, + { "R 0x248a7078 4" }, + { "R 0x248a708c 6" }, + { "R 0x248e002c" }, + { "R 0x248e009c 2" }, + { "R 0x248e00a8 3" }, + { "R 0x248e00b8" }, + { "R 0x248e00c0 7" }, + { "R 0x248e00e0" }, + { "R 0x248e00e8" }, + { "R 0x248e00f0" }, + { "R 0x248e00f8" }, + { "R 0x248e0100" }, + { "R 0x248e0108" }, + { "R 0x248e0110" }, + { "R 0x248e0118" }, + { "R 0x248e0120" }, + { "R 0x248e0128" }, + { "R 0x248e0150 4" }, + { "R 0x248e0164" }, + { "R 0x248e01e8" }, + { "R 0x248e1010" }, + { "R 0x248e1060 3" }, + { "R 0x248e1070" }, + { "R 0x248e1084 3" }, + { "R 0x248e3004" }, + { "R 0x248e300c" }, + { "R 0x248e4004" }, + { "R 0x248e4010 2" }, + { "R 0x248e4024 2" }, + { "R 0x248e4038 2" }, + { "R 0x248e600c 5" }, + { "R 0x248e700c 5" }, + { "R 0x248e9004" }, + { "R 0x248e9010 3" }, + { "R 0x248e9020 3" }, + { "R 0x248e9030 3" }, + { "R 0x248e9040 3" }, + { "R 0x248e9050 3" }, + { "R 0x248ea004" }, + { "R 0x248ea010 3" }, + { "R 0x248ea020 3" }, + { "R 0x248ea030 3" }, + { "R 0x248ea040 3" }, + { "R 0x248ea050 3" }, + { "R 0x248f001c" }, + { "R 0x248f0050" }, + { "R 0x248f0058" }, + { "R 0x24c00004" }, + { "R 0x24c01004" }, + { "R 0x24c02004" }, + { "R 0x24c03004" }, + { "R 0x24c04004" }, + { "R 0x24c05004" }, + { "R 0x24c06004" }, + { "R 0x24c07004" }, + { "R 0x24c08004" }, + { "R 0x24c09004" }, + { "R 0x24c0a004" }, + { "R 0x24c0b004" }, + { "R 0x24c0c004" }, + { "R 0x24c0d004" }, + { "R 0x24c0e004" }, + { "R 0x24c0f004" }, + { "R 0x24c10004" }, + { "R 0x24c11004" }, + { "R 0x24c12004" }, + { "R 0x24c13004" }, + { "R 0x24c14004" }, + { "R 0x24c15004" }, + { "R 0x24c16004" }, + { "R 0x24c17004" }, + { "R 0x24c18004" }, + { "R 0x24c19004" }, + { "R 0x24c1a004" }, + { "R 0x24c1b004" }, + { "R 0x24c1c004" }, + { "R 0x24c1d004" }, + { "R 0x24c1e004" }, + { "R 0x24c1f004" }, + { "R 0x24c20004" }, + { "R 0x24c21004" }, + { "R 0x24c22004" }, + { "R 0x24c23004" }, + { "R 0x24c43350 2" }, + { "R 0x24c433f4 11" }, + { "R 0x24c434c0 6" }, + { "R 0x24c43600" }, + { "R 0x24c4360c" }, + { "R 0x24c45040" }, + { "R 0x24c45048" }, + { "R 0x24c45050" }, + { "R 0x24c45058" }, + { "R 0x24c45060" }, + { "R 0x24c45068" }, + { "R 0x24c45070 4" }, + { "R 0x24c45090 14" }, + { "R 0x24c450cc 2" }, + { "R 0x24c45240" }, + { "R 0x24c452a0" }, + { "R 0x24c45300" }, + { "R 0x24c45310" }, + { "R 0x24c45320" }, + { "R 0x24c45330 3" }, + { "R 0x24c47404" }, + { "R 0x24c4740c 3" }, + { "R 0x24c47448" }, + { "R 0x24c47450" }, + { "R 0x24c47458 2" }, + { "R 0x24c47600" }, + { "R 0x24c49000" }, + { "R 0x24c49010" }, + { "R 0x24c7c000" }, + { "R 0x24c7c01c 3" }, + { "R 0x24c7c030 8" }, + { "R 0x24c7c054 2" }, + { "R 0x24c7c078" }, + { "R 0x24c7c108 9" }, + { "R 0x24c7c20c 3" }, + { "R 0x24ca600c 4" }, + { "R 0x24ca6020 3" }, + { "R 0x24ca6034 2" }, + { "R 0x24ca6040" }, + { "R 0x24ca6050" }, + { "R 0x24ca6058" }, + { "R 0x24ca6060 4" }, + { "R 0x24ca7020" }, + { "R 0x24ca7030 2" }, + { "R 0x24ca7078 4" }, + { "R 0x24ca708c 6" }, + { "R 0x24ce002c" }, + { "R 0x24ce009c 2" }, + { "R 0x24ce00a8 3" }, + { "R 0x24ce00b8" }, + { "R 0x24ce00c0 7" }, + { "R 0x24ce00e0" }, + { "R 0x24ce00e8" }, + { "R 0x24ce00f0" }, + { "R 0x24ce00f8" }, + { "R 0x24ce0100" }, + { "R 0x24ce0108" }, + { "R 0x24ce0110" }, + { "R 0x24ce0118" }, + { "R 0x24ce0120" }, + { "R 0x24ce0128" }, + { "R 0x24ce0150 4" }, + { "R 0x24ce0164" }, + { "R 0x24ce01e8" }, + { "R 0x24ce1010" }, + { "R 0x24ce1060 3" }, + { "R 0x24ce1070" }, + { "R 0x24ce1084 3" }, + { "R 0x24ce3004" }, + { "R 0x24ce300c" }, + { "R 0x24ce4004" }, + { "R 0x24ce4010 2" }, + { "R 0x24ce4024 2" }, + { "R 0x24ce4038 2" }, + { "R 0x24ce600c 5" }, + { "R 0x24ce700c 5" }, + { "R 0x24ce9004" }, + { "R 0x24ce9010 3" }, + { "R 0x24ce9020 3" }, + { "R 0x24ce9030 3" }, + { "R 0x24ce9040 3" }, + { "R 0x24ce9050 3" }, + { "R 0x24cea004" }, + { "R 0x24cea010 3" }, + { "R 0x24cea020 3" }, + { "R 0x24cea030 3" }, + { "R 0x24cea040 3" }, + { "R 0x24cea050 3" }, + { "R 0x24cf001c" }, + { "R 0x24cf0050" }, + { "R 0x24cf0058" }, + { "R 0x25800004" }, + { "R 0x25801004" }, + { "R 0x25802004" }, + { "R 0x25803004" }, + { "R 0x25804004" }, + { "R 0x25805004" }, + { "R 0x25806004" }, + { "R 0x25807004" }, + { "R 0x25808004" }, + { "R 0x25809004" }, + { "R 0x2580a004" }, + { "R 0x2580b004" }, + { "R 0x2580c004" }, + { "R 0x2580d004" }, + { "R 0x2580e004" }, + { "R 0x2580f004" }, + { "R 0x25810004" }, + { "R 0x25811004" }, + { "R 0x25812004" }, + { "R 0x25813004" }, + { "R 0x25814004" }, + { "R 0x25815004" }, + { "R 0x25816004" }, + { "R 0x25817004" }, + { "R 0x25818004" }, + { "R 0x25819004" }, + { "R 0x2581a004" }, + { "R 0x2581b004" }, + { "R 0x2581c004" }, + { "R 0x2581d004" }, + { "R 0x2581e004" }, + { "R 0x2581f004" }, + { "R 0x25820004" }, + { "R 0x25821004" }, + { "R 0x25822004" }, + { "R 0x25823004" }, + { "R 0x25843350 2" }, + { "R 0x258433f4 11" }, + { "R 0x258434c0 6" }, + { "R 0x25843600" }, + { "R 0x2584360c" }, + { "R 0x25845040" }, + { "R 0x25845048" }, + { "R 0x25845050" }, + { "R 0x25845058" }, + { "R 0x25845060" }, + { "R 0x25845068" }, + { "R 0x25845070 4" }, + { "R 0x25845090 14" }, + { "R 0x258450cc 2" }, + { "R 0x25845240" }, + { "R 0x258452a0" }, + { "R 0x25845300" }, + { "R 0x25845310" }, + { "R 0x25845320" }, + { "R 0x25845330 3" }, + { "R 0x25847404" }, + { "R 0x2584740c 3" }, + { "R 0x25847448" }, + { "R 0x25847450" }, + { "R 0x25847458 2" }, + { "R 0x25847600" }, + { "R 0x25849000" }, + { "R 0x25849010" }, + { "R 0x2587c000" }, + { "R 0x2587c01c 3" }, + { "R 0x2587c030 8" }, + { "R 0x2587c054 2" }, + { "R 0x2587c078" }, + { "R 0x2587c108 9" }, + { "R 0x2587c20c 3" }, + { "R 0x258a600c 4" }, + { "R 0x258a6020 3" }, + { "R 0x258a6034 2" }, + { "R 0x258a6040" }, + { "R 0x258a6050" }, + { "R 0x258a6058" }, + { "R 0x258a6060 4" }, + { "R 0x258a7020" }, + { "R 0x258a7030 2" }, + { "R 0x258a7078 4" }, + { "R 0x258a708c 6" }, + { "R 0x258e002c" }, + { "R 0x258e009c 2" }, + { "R 0x258e00a8 3" }, + { "R 0x258e00b8" }, + { "R 0x258e00c0 7" }, + { "R 0x258e00e0" }, + { "R 0x258e00e8" }, + { "R 0x258e00f0" }, + { "R 0x258e00f8" }, + { "R 0x258e0100" }, + { "R 0x258e0108" }, + { "R 0x258e0110" }, + { "R 0x258e0118" }, + { "R 0x258e0120" }, + { "R 0x258e0128" }, + { "R 0x258e0150 4" }, + { "R 0x258e0164" }, + { "R 0x258e01e8" }, + { "R 0x258e1010" }, + { "R 0x258e1060 3" }, + { "R 0x258e1070" }, + { "R 0x258e1084 3" }, + { "R 0x258e3004" }, + { "R 0x258e300c" }, + { "R 0x258e4004" }, + { "R 0x258e4010 2" }, + { "R 0x258e4024 2" }, + { "R 0x258e4038 2" }, + { "R 0x258e600c 5" }, + { "R 0x258e700c 5" }, + { "R 0x258e9004" }, + { "R 0x258e9010 3" }, + { "R 0x258e9020 3" }, + { "R 0x258e9030 3" }, + { "R 0x258e9040 3" }, + { "R 0x258e9050 3" }, + { "R 0x258ea004" }, + { "R 0x258ea010 3" }, + { "R 0x258ea020 3" }, + { "R 0x258ea030 3" }, + { "R 0x258ea040 3" }, + { "R 0x258ea050 3" }, + { "R 0x258f001c" }, + { "R 0x258f0050" }, + { "R 0x258f0058" }, + { "R 0x25c00004" }, + { "R 0x25c01004" }, + { "R 0x25c02004" }, + { "R 0x25c03004" }, + { "R 0x25c04004" }, + { "R 0x25c05004" }, + { "R 0x25c06004" }, + { "R 0x25c07004" }, + { "R 0x25c08004" }, + { "R 0x25c09004" }, + { "R 0x25c0a004" }, + { "R 0x25c0b004" }, + { "R 0x25c0c004" }, + { "R 0x25c0d004" }, + { "R 0x25c0e004" }, + { "R 0x25c0f004" }, + { "R 0x25c10004" }, + { "R 0x25c11004" }, + { "R 0x25c12004" }, + { "R 0x25c13004" }, + { "R 0x25c14004" }, + { "R 0x25c15004" }, + { "R 0x25c16004" }, + { "R 0x25c17004" }, + { "R 0x25c18004" }, + { "R 0x25c19004" }, + { "R 0x25c1a004" }, + { "R 0x25c1b004" }, + { "R 0x25c1c004" }, + { "R 0x25c1d004" }, + { "R 0x25c1e004" }, + { "R 0x25c1f004" }, + { "R 0x25c20004" }, + { "R 0x25c21004" }, + { "R 0x25c22004" }, + { "R 0x25c23004" }, + { "R 0x25c43350 2" }, + { "R 0x25c433f4 11" }, + { "R 0x25c434c0 6" }, + { "R 0x25c43600" }, + { "R 0x25c4360c" }, + { "R 0x25c45040" }, + { "R 0x25c45048" }, + { "R 0x25c45050" }, + { "R 0x25c45058" }, + { "R 0x25c45060" }, + { "R 0x25c45068" }, + { "R 0x25c45070 4" }, + { "R 0x25c45090 14" }, + { "R 0x25c450cc 2" }, + { "R 0x25c45240" }, + { "R 0x25c452a0" }, + { "R 0x25c45300" }, + { "R 0x25c45310" }, + { "R 0x25c45320" }, + { "R 0x25c45330 3" }, + { "R 0x25c47404" }, + { "R 0x25c4740c 3" }, + { "R 0x25c47448" }, + { "R 0x25c47450" }, + { "R 0x25c47458 2" }, + { "R 0x25c47600" }, + { "R 0x25c49000" }, + { "R 0x25c49010" }, + { "R 0x25c7c000" }, + { "R 0x25c7c01c 3" }, + { "R 0x25c7c030 8" }, + { "R 0x25c7c054 2" }, + { "R 0x25c7c078" }, + { "R 0x25c7c108 9" }, + { "R 0x25c7c20c 3" }, + { "R 0x25ca600c 4" }, + { "R 0x25ca6020 3" }, + { "R 0x25ca6034 2" }, + { "R 0x25ca6040" }, + { "R 0x25ca6050" }, + { "R 0x25ca6058" }, + { "R 0x25ca6060 4" }, + { "R 0x25ca7020" }, + { "R 0x25ca7030 2" }, + { "R 0x25ca7078 4" }, + { "R 0x25ca708c 6" }, + { "R 0x25ce002c" }, + { "R 0x25ce009c 2" }, + { "R 0x25ce00a8 3" }, + { "R 0x25ce00b8" }, + { "R 0x25ce00c0 7" }, + { "R 0x25ce00e0" }, + { "R 0x25ce00e8" }, + { "R 0x25ce00f0" }, + { "R 0x25ce00f8" }, + { "R 0x25ce0100" }, + { "R 0x25ce0108" }, + { "R 0x25ce0110" }, + { "R 0x25ce0118" }, + { "R 0x25ce0120" }, + { "R 0x25ce0128" }, + { "R 0x25ce0150 4" }, + { "R 0x25ce0164" }, + { "R 0x25ce01e8" }, + { "R 0x25ce1010" }, + { "R 0x25ce1060 3" }, + { "R 0x25ce1070" }, + { "R 0x25ce1084 3" }, + { "R 0x25ce3004" }, + { "R 0x25ce300c" }, + { "R 0x25ce4004" }, + { "R 0x25ce4010 2" }, + { "R 0x25ce4024 2" }, + { "R 0x25ce4038 2" }, + { "R 0x25ce600c 5" }, + { "R 0x25ce700c 5" }, + { "R 0x25ce9004" }, + { "R 0x25ce9010 3" }, + { "R 0x25ce9020 3" }, + { "R 0x25ce9030 3" }, + { "R 0x25ce9040 3" }, + { "R 0x25ce9050 3" }, + { "R 0x25cea004" }, + { "R 0x25cea010 3" }, + { "R 0x25cea020 3" }, + { "R 0x25cea030 3" }, + { "R 0x25cea040 3" }, + { "R 0x25cea050 3" }, + { "R 0x25cf001c" }, + { "R 0x25cf0050" }, + { "R 0x25cf0058" }, + { "R 0x240a0008 2" }, + { "R 0x240a1008 2" }, + { "R 0x248e0070 7" }, + { "R 0x248e0094 2" }, + { "R 0x248e012c 9" }, + { "R 0x248e0160" }, + { "R 0x248e0364 2" }, + { "R 0x24ce0070 7" }, + { "R 0x24ce0094 2" }, + { "R 0x24ce012c 9" }, + { "R 0x24ce0160" }, + { "R 0x24ce0364 2" }, + { "R 0x258e0070 7" }, + { "R 0x258e0094 2" }, + { "R 0x258e012c 9" }, + { "R 0x258e0160" }, + { "R 0x258e0364 2" }, + { "R 0x25ce0070 7" }, + { "R 0x25ce0094 2" }, + { "R 0x25ce012c 9" }, + { "R 0x25ce0160" }, + { "R 0x25ce0364 2" }, + { "R 0x248e0358" }, + { "R 0x24ce0358" }, + { "R 0x258e0358" }, + { "R 0x25ce0358" }, + { "R 0x248e0008" }, + { "R 0x24ce0008" }, + { "R 0x258e0008" }, + { "R 0x25ce0008" }, + { "R 0x24076b00 2" }, + { "R 0x24076b0c 2" }, + { "R 0x24076b18 2" }, + { "R 0x24076b24 2" }, + { "R 0x24076b30 2" }, + { "R 0x24076b3c 2" }, + { "R 0x24076b48 2" }, + { "R 0x24076b54 2" }, + { "R 0x24076b60 2" }, + { "R 0x24076b6c 2" }, + { "R 0x24076b78 2" }, + { "R 0x24076b84 2" }, + { "R 0x24076b90 2" }, + { "R 0x24076b9c 2" }, + { "R 0x24076ba8 2" }, + { "R 0x24076bb4 2" }, + { "R 0x24076bc0 2" }, + { "R 0x24076bcc 2" }, + { "R 0x24076bd8 2" }, + { "R 0x24076be4 2" }, + { "R 0x24076bf0 2" }, + { "R 0x24076bfc 2" }, + { "R 0x24076c08 2" }, + { "R 0x24076c14 2" }, + { "R 0x24076c20 2" }, + { "R 0x24076c2c 2" }, + { "R 0x24076c38 2" }, + { "R 0x24076c44 2" }, + { "R 0x24076c50 2" }, + { "R 0x24076c5c 2" }, + { "R 0x24076c68 2" }, + { "R 0x24076c74 2" }, + { "R 0x24076950" }, + { "R 0x240760a4" }, + { "R 0x248720b0" }, + { "R 0x258720b0" }, + { "R 0x24c720b0" }, + { "R 0x25c720b0" }, +}; + +static const struct dcc_register_entry pakala_dcc_entries_ll4[] = { + { "R 0x16801000 2" }, + { "R 0x240e0010" }, + { "R 0x240e0020 8" }, + { "R 0x240e0248" }, + { "R 0x24330010" }, + { "R 0x24330020 8" }, + { "R 0x24330248" }, + { "R 0x240e1018" }, + { "R 0x240e1008" }, + { "R 0x240e1010 2" }, + { "R 0x24331018" }, + { "R 0x24331008" }, + { "R 0x24331010 2" }, + { "R 0x1780010" }, + { "R 0x1780020 8" }, + { "R 0x1780248" }, + { "R 0x1782018" }, + { "R 0x1782008" }, + { "R 0x1782010 2" }, + { "R 0x1783018" }, + { "R 0x1783008" }, + { "R 0x1783010 2" }, + { "R 0x1680010" }, + { "R 0x1680020 8" }, + { "R 0x1681048" }, + { "R 0x1682018" }, + { "R 0x1682008" }, + { "R 0x1682010 2" }, + { "R 0x16e0010" }, + { "R 0x16e0020 8" }, + { "R 0x16e0248" }, + { "R 0x16e1018" }, + { "R 0x16e1008" }, + { "R 0x16e1010 2" }, + { "R 0x16e1098" }, + { "R 0x16e1088" }, + { "R 0x16e1090 2" }, + { "R 0x16e1118" }, + { "R 0x16e1108" }, + { "R 0x16e1110 2" }, + { "R 0x1700010" }, + { "R 0x1700020 8" }, + { "R 0x1700248" }, + { "R 0x1702018" }, + { "R 0x1702008" }, + { "R 0x1702010 2" }, + { "R 0x1702218" }, + { "R 0x1702208" }, + { "R 0x1702210 2" }, + { "R 0x1702118" }, + { "R 0x1702108" }, + { "R 0x1702110 2" }, + { "R 0x1600010" }, + { "R 0x1600020 8" }, + { "R 0x1600248 2" }, + { "R 0x1600258" }, + { "R 0x1602018" }, + { "R 0x1602008" }, + { "R 0x1602010 2" }, + { "R 0x1602098" }, + { "R 0x1602088" }, + { "R 0x1602090 2" }, + { "R 0x1602118" }, + { "R 0x1602108" }, + { "R 0x1602110 2" }, + { "R 0x1602198" }, + { "R 0x1602188" }, + { "R 0x1602190 2" }, + { "R 0x1602218" }, + { "R 0x1602208" }, + { "R 0x1602210 2" }, + { "R 0x1602098" }, + { "R 0x1602088" }, + { "R 0x1602090 2" }, + { "R 0x1500010" }, + { "R 0x1500020 8" }, + { "R 0x1500248" }, + { "R 0x1500448" }, + { "R 0x1502018" }, + { "R 0x1502008" }, + { "R 0x1502010 2" }, + { "R 0x1502098" }, + { "R 0x1502088" }, + { "R 0x1502090 2" }, + { "R 0x16e00010" }, + { "R 0x16e00020 8" }, + { "R 0x16e00248" }, + { "R 0x16e01018" }, + { "R 0x16e01008" }, + { "R 0x16e01010 2" }, + { "R 0x1b600010" }, + { "R 0x1b600020 8" }, + { "R 0x1b600248" }, + { "R 0x1b601018" }, + { "R 0x1b601008" }, + { "R 0x1b601010 2" }, + { "R 0x16000104 30" }, + { "R 0x16000204 29" }, + { "R 0x16000384 30" }, + { "R 0xb291024" }, + { "R 0xc201244" }, + { "R 0xc202244" }, + { "R 0xbde1034 2" }, + { "R 0xb201020 2" }, + { "R 0xb211020 2" }, + { "R 0xb221020 2" }, + { "R 0xb231020 2" }, + { "R 0xb204520" }, + { "R 0xb200000" }, + { "R 0xb210000" }, + { "R 0xb220000" }, + { "R 0xb230000" }, + { "R 0x16500010" }, + { "R 0x16510010" }, + { "R 0x16520010" }, + { "R 0x16530010" }, + { "R 0x16500030" }, + { "R 0x16510030" }, + { "R 0x16520030" }, + { "R 0x16530030" }, + { "R 0x16500038" }, + { "R 0x16510038" }, + { "R 0x16520038" }, + { "R 0x16530038" }, + { "R 0x16500040" }, + { "R 0x16510040" }, + { "R 0x16520040" }, + { "R 0x16530040" }, + { "R 0x16500048" }, + { "R 0x16500400 3" }, + { "R 0x16510400 3" }, + { "R 0x16520400 3" }, + { "R 0x16530400 3" }, + { "R 0x16510d3c" }, + { "R 0x16510d54" }, + { "R 0x16510d6c" }, + { "R 0x16510d84" }, + { "R 0x16510d9c" }, + { "R 0x16510db4" }, + { "R 0x16510dcc" }, + { "R 0x16510de4" }, + { "R 0x16510dfc" }, + { "R 0x16510e14" }, + { "R 0x16510e2c" }, + { "R 0x16510e44" }, + { "R 0x16510e5c" }, + { "R 0x16510e74" }, + { "R 0x16510e8c" }, + { "R 0x16510ea4" }, + { "R 0x16510fdc" }, + { "R 0x16510ff4" }, + { "R 0x1651100c" }, + { "R 0x16511024" }, + { "R 0x1651103c" }, + { "R 0x16511054" }, + { "R 0x1651106c" }, + { "R 0x16511084" }, + { "R 0x1651109c" }, + { "R 0x165110b4" }, + { "R 0x165110cc" }, + { "R 0x165110e4" }, + { "R 0x165110fc" }, + { "R 0x16511114" }, + { "R 0x1651112c" }, + { "R 0x16511144" }, + { "R 0x1651127c" }, + { "R 0x16511294" }, + { "R 0x165112ac" }, + { "R 0x165112c4" }, + { "R 0x165112dc" }, + { "R 0x165112f4" }, + { "R 0x1651130c" }, + { "R 0x16511324" }, + { "R 0x1651133c" }, + { "R 0x16511354" }, + { "R 0x1651136c" }, + { "R 0x16511384" }, + { "R 0x1651139c" }, + { "R 0x165113b4" }, + { "R 0x165113cc" }, + { "R 0x165113e4" }, + { "R 0x1651151c" }, + { "R 0x16511534" }, + { "R 0x1651154c" }, + { "R 0x16511564" }, + { "R 0x1651157c" }, + { "R 0x16511594" }, + { "R 0x165115ac" }, + { "R 0x165115c4" }, + { "R 0x165115dc" }, + { "R 0x165115f4" }, + { "R 0x1651160c" }, + { "R 0x16511624" }, + { "R 0x1651163c" }, + { "R 0x16511654" }, + { "R 0x1651166c" }, + { "R 0x16511684" }, + { "R 0x110004 2" }, + { "R 0x11003c 3" }, + { "R 0x176040" }, + { "R 0x10c0000 4" }, + { "R 0x10c1000 2" }, + { "R 0x10c1010 7" }, + { "R 0x10c1100 3" }, + { "R 0x10c1110 5" }, + { "R 0x10c1130 2" }, + { "R 0x10c113c 2" }, + { "R 0x10c1148 3" }, + { "R 0x10c1800 11" }, + { "R 0x10c2000" }, + { "R 0x10cf004" }, + { "R 0x16801000 2" }, +}; + +static const struct dcc_link_config pakala_link_configs[] = { + { + .link_list = 6, + .entries = pakala_dcc_entries_ll6, + .num_entries = ARRAY_SIZE(pakala_dcc_entries_ll6), + }, + { + .link_list = 4, + .entries = pakala_dcc_entries_ll4, + .num_entries = ARRAY_SIZE(pakala_dcc_entries_ll4), + }, +}; + +static const struct dcc_config pakala_config = { + .lists = pakala_link_configs, + .num_lists = ARRAY_SIZE(pakala_link_configs), +}; + +static const struct dcc_pdata pakala_pdata = { + .base = 0x100ff000, + .size = 0x00001000, + .ram_base = 0x10084000, + .ram_size = 0x4000, + .dcc_offset = 0x4000, + .map_ver = 0x3, + .config = &pakala_config, +}; + +#endif /* _QCOM_DCC_PAKALA_CONFIG_H */ diff --git a/drivers/misc/qcom-dcc-talos-config.h b/drivers/misc/qcom-dcc-talos-config.h new file mode 100644 index 0000000000000..2c2316ae37853 --- /dev/null +++ b/drivers/misc/qcom-dcc-talos-config.h @@ -0,0 +1,792 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _QCOM_DCC_TALOS_CONFIG_H +#define _QCOM_DCC_TALOS_CONFIG_H + +#include "qcom-dcc.h" + +static const struct dcc_register_entry talos_dcc_entries[] = { + { "R 0x9680000" }, + { "R 0x9680004" }, + { "R 0x9681000" }, + { "R 0x9681004" }, + { "R 0x9681008" }, + { "R 0x968100c" }, + { "R 0x9681010" }, + { "R 0x9681014" }, + { "R 0x968101c" }, + { "R 0x9681020" }, + { "R 0x9681024" }, + { "R 0x9681028" }, + { "R 0x968102c" }, + { "R 0x9681030" }, + { "R 0x9681034" }, + { "R 0x968103c" }, + { "R 0x9698100" }, + { "R 0x9698104" }, + { "R 0x9698108" }, + { "R 0x9698110" }, + { "R 0x9698120" }, + { "R 0x9698124" }, + { "R 0x9698128" }, + { "R 0x969812c" }, + { "R 0x9698130" }, + { "R 0x9698134" }, + { "R 0x9698138" }, + { "R 0x969813c" }, + { "R 0x9698500" }, + { "R 0x9698504" }, + { "R 0x9698508" }, + { "R 0x969850c" }, + { "R 0x9698510" }, + { "R 0x9698514" }, + { "R 0x9698518" }, + { "R 0x969851c" }, + { "R 0x9698700" }, + { "R 0x9698704" }, + { "R 0x9698708" }, + { "R 0x969870c" }, + { "R 0x9698714" }, + { "R 0x9698718" }, + { "R 0x969871c" }, + { "R 0x1620204" }, + { "R 0x1620240" }, + { "R 0x1620248" }, + { "R 0x1620288" }, + { "R 0x162028C" }, + { "R 0x1620290" }, + { "R 0x1620294" }, + { "R 0x16202A8" }, + { "R 0x16202AC" }, + { "R 0x16202B0" }, + { "R 0x16202B4" }, + { "R 0x1620300" }, + { "R 0x1700204" }, + { "R 0x1700240" }, + { "R 0x1700248" }, + { "R 0x1700288" }, + { "R 0x1700290" }, + { "R 0x1700300" }, + { "R 0x1700304" }, + { "R 0x1700308" }, + { "R 0x170030C" }, + { "R 0x1700310" }, + { "R 0x1700314" }, + { "R 0x1700C08" }, + { "R 0x1700C10" }, + { "R 0x1700C20" }, + { "R 0x1700C24" }, + { "R 0x1700C28" }, + { "R 0x1700C2C" }, + { "R 0x1700C30" }, + { "R 0x1700C34" }, + { "R 0x1700C38" }, + { "R 0x1700C3C" }, + { "R 0x1740240" }, + { "R 0x1740248" }, + { "R 0x1740288" }, + { "R 0x1740290" }, + { "R 0x1740300" }, + { "R 0x1740304" }, + { "R 0x1740308" }, + { "R 0x174030C" }, + { "R 0x1740310" }, + { "R 0x1740314" }, + { "R 0x1740004" }, + { "R 0x1740008" }, + { "R 0x1740010" }, + { "R 0x1740020" }, + { "R 0x1740024" }, + { "R 0x1740028" }, + { "R 0x174002C" }, + { "R 0x1740030" }, + { "R 0x1740034" }, + { "R 0x1740038" }, + { "R 0x174003C" }, + { "R 0x9698204" }, + { "R 0x9698240" }, + { "R 0x9698244" }, + { "R 0x9698248" }, + { "R 0x969824C" }, + { "R 0x9681010" }, + { "R 0x9681014" }, + { "R 0x9681018" }, + { "R 0x968101C" }, + { "R 0x9681020" }, + { "R 0x9681024" }, + { "R 0x9681028" }, + { "R 0x968102C" }, + { "R 0x9681030" }, + { "R 0x9681034" }, + { "R 0x968103C" }, + { "R 0x9698100" }, + { "R 0x9698104" }, + { "R 0x9698108" }, + { "R 0x9698110" }, + { "R 0x9698120" }, + { "R 0x9698124" }, + { "R 0x9698128" }, + { "R 0x969812C" }, + { "R 0x9698130" }, + { "R 0x9698134" }, + { "R 0x9698138" }, + { "R 0x969813C" }, + { "R 0x62BE2004" }, + { "R 0x62BE2040" }, + { "R 0x62BE2048" }, + { "R 0x62BE2088" }, + { "R 0x62BE2090" }, + { "R 0x62BE2100" }, + { "R 0x62BE2104" }, + { "R 0x62BE2108" }, + { "R 0x62BE210C" }, + { "R 0x62BE2110" }, + { "R 0x62BE2114" }, + { "R 0x62BE2118" }, + { "R 0x62BE0010" }, + { "R 0x62BE0020" }, + { "R 0x62BE0024" }, + { "R 0x62BE0028" }, + { "R 0x62BE002C" }, + { "R 0x62BE0030" }, + { "R 0x62BE0034" }, + { "R 0x62BE0038" }, + { "R 0x62BE003C" }, + { "R 0x9160204" }, + { "R 0x9160240" }, + { "R 0x9160248" }, + { "R 0x9160288" }, + { "R 0x9160290" }, + { "R 0x9160300" }, + { "R 0x9160304" }, + { "R 0x9160308" }, + { "R 0x916030C" }, + { "R 0x9160310" }, + { "R 0x9160314" }, + { "R 0x9160318" }, + { "R 0x9160008" }, + { "R 0x9160010" }, + { "R 0x9160020" }, + { "R 0x9160024" }, + { "R 0x9160028" }, + { "R 0x916002C" }, + { "R 0x9160030" }, + { "R 0x9160034" }, + { "R 0x9160038" }, + { "R 0x916003C" }, + { "R 0x1620500 4" }, + { "R 0x1620700 4" }, + { "R 0x1620300" }, + { "R 0x1620F00 2" }, + { "R 0x1620B00 2" }, + { "R 0x1700B00 2" }, + { "R 0x1700700 3" }, + { "R 0x9163100" }, + { "R 0x96AA100" }, + { "R 0x9050008" }, + { "R 0x9050068" }, + { "R 0x9050078" }, + { "R 0x18200400" }, + { "R 0x18200404" }, + { "R 0x18200408" }, + { "R 0x18200038" }, + { "R 0x18200040" }, + { "R 0x18200048" }, + { "R 0x18220038" }, + { "R 0x18220040" }, + { "R 0x182200D0" }, + { "R 0x18200030" }, + { "R 0x18200010" }, + { "R 0x1822000c" }, + { "R 0x18220d14" }, + { "R 0x18220fb4" }, + { "R 0x18221254" }, + { "R 0x182214f4" }, + { "R 0x18221794" }, + { "R 0x18221a34" }, + { "R 0x18221cd4" }, + { "R 0x18221f74" }, + { "R 0x18220d18" }, + { "R 0x18220fb8" }, + { "R 0x18221258" }, + { "R 0x182214f8" }, + { "R 0x18221798" }, + { "R 0x18221a38" }, + { "R 0x18221cd8" }, + { "R 0x18221f78" }, + { "R 0x18220d00" }, + { "R 0x18220d04" }, + { "R 0x18220d1c" }, + { "R 0x18220fbc" }, + { "R 0x1822125c" }, + { "R 0x182214fc" }, + { "R 0x1822179c" }, + { "R 0x18221a3c" }, + { "R 0x18221cdc" }, + { "R 0x18221f7c" }, + { "R 0x18221274" }, + { "R 0x18221288" }, + { "R 0x1822129c" }, + { "R 0x182212b0" }, + { "R 0x182212c4" }, + { "R 0x182212d8" }, + { "R 0x182212ec" }, + { "R 0x18221300" }, + { "R 0x18221314" }, + { "R 0x18221328" }, + { "R 0x1822133c" }, + { "R 0x18221350" }, + { "R 0x18221364" }, + { "R 0x18221378" }, + { "R 0x1822138c" }, + { "R 0x182213a0" }, + { "R 0x18221514" }, + { "R 0x18221528" }, + { "R 0x1822153c" }, + { "R 0x18221550" }, + { "R 0x18221564" }, + { "R 0x18221578" }, + { "R 0x1822158c" }, + { "R 0x182215a0" }, + { "R 0x182215b4" }, + { "R 0x182215c8" }, + { "R 0x182215dc" }, + { "R 0x182215f0" }, + { "R 0x18221604" }, + { "R 0x18221618" }, + { "R 0x1822162c" }, + { "R 0x18221640" }, + { "R 0x182217b4" }, + { "R 0x182217c8" }, + { "R 0x182217dc" }, + { "R 0x182217f0" }, + { "R 0x18221804" }, + { "R 0x18221818" }, + { "R 0x1822182c" }, + { "R 0x18221840" }, + { "R 0x18221854" }, + { "R 0x18221868" }, + { "R 0x1822187c" }, + { "R 0x18221890" }, + { "R 0x182218a4" }, + { "R 0x182218b8" }, + { "R 0x182218cc" }, + { "R 0x182218e0" }, + { "R 0x18221a54" }, + { "R 0x18221a68" }, + { "R 0x18221a7c" }, + { "R 0x18221a90" }, + { "R 0x18221aa4" }, + { "R 0x18221ab8" }, + { "R 0x18221acc" }, + { "R 0x18221ae0" }, + { "R 0x18221af4" }, + { "R 0x18221b08" }, + { "R 0x18221b1c" }, + { "R 0x18221b30" }, + { "R 0x18221b44" }, + { "R 0x18221b58" }, + { "R 0x18221b6c" }, + { "R 0x18221b80" }, + { "R 0x18221cf4" }, + { "R 0x18221d08" }, + { "R 0x18221d1c" }, + { "R 0x18221d30" }, + { "R 0x18221d44" }, + { "R 0x18221d58" }, + { "R 0x18221d6c" }, + { "R 0x18221d80" }, + { "R 0x18221d94" }, + { "R 0x18221da8" }, + { "R 0x18221dbc" }, + { "R 0x18221dd0" }, + { "R 0x18221de4" }, + { "R 0x18221df8" }, + { "R 0x18221e0c" }, + { "R 0x18221e20" }, + { "R 0x18221f94" }, + { "R 0x18221fa8" }, + { "R 0x18221fbc" }, + { "R 0x18221fd0" }, + { "R 0x18221fe4" }, + { "R 0x18221ff8" }, + { "R 0x1822200c" }, + { "R 0x18222020" }, + { "R 0x18222034" }, + { "R 0x18222048" }, + { "R 0x1822205c" }, + { "R 0x18222070" }, + { "R 0x18222084" }, + { "R 0x18222098" }, + { "R 0x182220ac" }, + { "R 0x182220c0" }, + { "R 0x18221278" }, + { "R 0x1822128c" }, + { "R 0x182212a0" }, + { "R 0x182212b4" }, + { "R 0x182212c8" }, + { "R 0x182212dc" }, + { "R 0x182212f0" }, + { "R 0x18221304" }, + { "R 0x18221318" }, + { "R 0x1822132c" }, + { "R 0x18221340" }, + { "R 0x18221354" }, + { "R 0x18221368" }, + { "R 0x1822137c" }, + { "R 0x18221390" }, + { "R 0x182213a4" }, + { "R 0x18221518" }, + { "R 0x1822152c" }, + { "R 0x18221540" }, + { "R 0x18221554" }, + { "R 0x18221568" }, + { "R 0x1822157c" }, + { "R 0x18221590" }, + { "R 0x182215a4" }, + { "R 0x182215b8" }, + { "R 0x182215cc" }, + { "R 0x182215e0" }, + { "R 0x182215f4" }, + { "R 0x18221608" }, + { "R 0x1822161c" }, + { "R 0x18221630" }, + { "R 0x18221644" }, + { "R 0x182217b8" }, + { "R 0x182217cc" }, + { "R 0x182217e0" }, + { "R 0x182217f4" }, + { "R 0x18221808" }, + { "R 0x1822181c" }, + { "R 0x18221830" }, + { "R 0x18221844" }, + { "R 0x18221858" }, + { "R 0x1822186c" }, + { "R 0x18221880" }, + { "R 0x18221894" }, + { "R 0x182218a8" }, + { "R 0x182218bc" }, + { "R 0x182218d0" }, + { "R 0x182218e4" }, + { "R 0x18221a58" }, + { "R 0x18221a6c" }, + { "R 0x18221a80" }, + { "R 0x18221a94" }, + { "R 0x18221aa8" }, + { "R 0x18221abc" }, + { "R 0x18221ad0" }, + { "R 0x18221ae4" }, + { "R 0x18221af8" }, + { "R 0x18221b0c" }, + { "R 0x18221b20" }, + { "R 0x18221b34" }, + { "R 0x18221b48" }, + { "R 0x18221b5c" }, + { "R 0x18221b70" }, + { "R 0x18221b84" }, + { "R 0x18221cf8" }, + { "R 0x18221d0c" }, + { "R 0x18221d20" }, + { "R 0x18221d34" }, + { "R 0x18221d48" }, + { "R 0x18221d5c" }, + { "R 0x18221d70" }, + { "R 0x18221d84" }, + { "R 0x18221d98" }, + { "R 0x18221dac" }, + { "R 0x18221dc0" }, + { "R 0x18221dd4" }, + { "R 0x18221de8" }, + { "R 0x18221dfc" }, + { "R 0x18221e10" }, + { "R 0x18221e24" }, + { "R 0x18221f98" }, + { "R 0x18221fac" }, + { "R 0x18221fc0" }, + { "R 0x18221fd4" }, + { "R 0x18221fe8" }, + { "R 0x18221ffc" }, + { "R 0x18222010" }, + { "R 0x18222024" }, + { "R 0x18222038" }, + { "R 0x1822204c" }, + { "R 0x18222060" }, + { "R 0x18222074" }, + { "R 0x18222088" }, + { "R 0x1822209c" }, + { "R 0x182220b0" }, + { "R 0x182220c4" }, + { "R 0x18000024" }, + { "R 0x18000040" }, + { "R 0x18010024" }, + { "R 0x18010040" }, + { "R 0x18020024" }, + { "R 0x18020040" }, + { "R 0x18030024" }, + { "R 0x18030040" }, + { "R 0x18040024" }, + { "R 0x18040040" }, + { "R 0x18050024" }, + { "R 0x18050040" }, + { "R 0x18060024" }, + { "R 0x18060040" }, + { "R 0x18070024" }, + { "R 0x18070040" }, + { "R 0x18080024" }, + { "R 0x18080040" }, + { "R 0x180800F8" }, + { "R 0x18080104" }, + { "R 0x1808011C" }, + { "R 0x18080128" }, + { "R 0x90b0280" }, + { "R 0x90b0288" }, + { "R 0x90b028c" }, + { "R 0x90b0290" }, + { "R 0x90b0294" }, + { "R 0x90b0298" }, + { "R 0x90b029c" }, + { "R 0x90b02a0" }, + { "R 0x18321700" }, + { "R 0x18322C18" }, + { "R 0x18323700" }, + { "R 0x18324C18" }, + { "R 0x18325F00" }, + { "R 0x18327418" }, + { "R 0x9236028" }, + { "R 0x923602C" }, + { "R 0x9236030" }, + { "R 0x9236034" }, + { "R 0x9236038" }, + { "R 0x9232100" }, + { "R 0x92360b0" }, + { "R 0x9236044" }, + { "R 0x9236048" }, + { "R 0x923604c" }, + { "R 0x9236050" }, + { "R 0x923e030" }, + { "R 0x923e034" }, + { "R 0x9241000" }, + { "R 0x9248058" }, + { "R 0x924805c" }, + { "R 0x9248060" }, + { "R 0x9248064" }, + { "R 0x9260410" }, + { "R 0x92e0410" }, + { "R 0x9260414" }, + { "R 0x92e0414" }, + { "R 0x9260418" }, + { "R 0x92e0418" }, + { "R 0x9260420" }, + { "R 0x9260424" }, + { "R 0x9260430" }, + { "R 0x9260440" }, + { "R 0x9260448" }, + { "R 0x92604a0" }, + { "R 0x92e0420" }, + { "R 0x92e0424" }, + { "R 0x92e0430" }, + { "R 0x92e0440" }, + { "R 0x92e0448" }, + { "R 0x92e04a0" }, + { "R 0x9600000" }, + { "R 0x9601000" }, + { "R 0x9602000" }, + { "R 0x9603000" }, + { "R 0x9604000" }, + { "R 0x9605000" }, + { "R 0x9606000" }, + { "R 0x9607000" }, + { "R 0x9608000" }, + { "R 0x9609000" }, + { "R 0x960a000" }, + { "R 0x960b000" }, + { "R 0x960c000" }, + { "R 0x960d000" }, + { "R 0x960e000" }, + { "R 0x960f000" }, + { "R 0x9610000" }, + { "R 0x9611000" }, + { "R 0x9612000" }, + { "R 0x9613000" }, + { "R 0x9614000" }, + { "R 0x9615000" }, + { "R 0x9616000" }, + { "R 0x9617000" }, + { "R 0x9618000" }, + { "R 0x9619000" }, + { "R 0x961a000" }, + { "R 0x961b000" }, + { "R 0x961c000" }, + { "R 0x961d000" }, + { "R 0x961e000" }, + { "R 0x961f000" }, + { "R 0x9600004" }, + { "R 0x9601004" }, + { "R 0x9602004" }, + { "R 0x9603004" }, + { "R 0x9604004" }, + { "R 0x9605004" }, + { "R 0x9606004" }, + { "R 0x9607004" }, + { "R 0x9608004" }, + { "R 0x9609004" }, + { "R 0x960a004" }, + { "R 0x960b004" }, + { "R 0x960c004" }, + { "R 0x960d004" }, + { "R 0x960e004" }, + { "R 0x960f004" }, + { "R 0x9610004" }, + { "R 0x9611004" }, + { "R 0x9612004" }, + { "R 0x9613004" }, + { "R 0x9614004" }, + { "R 0x9615004" }, + { "R 0x9616004" }, + { "R 0x9617004" }, + { "R 0x9618004" }, + { "R 0x9619004" }, + { "R 0x961a004" }, + { "R 0x961b004" }, + { "R 0x961c004" }, + { "R 0x961d004" }, + { "R 0x961e004" }, + { "R 0x961f004" }, + { "R 0x9266418" }, + { "R 0x92e6418" }, + { "R 0x9265804" }, + { "R 0x92e5804" }, + { "R 0x92604b8" }, + { "R 0x92e04b8" }, + { "R 0x0C201244 1" }, + { "R 0x0C202244 1" }, + { "R 0x18100C18 1" }, + { "R 0x18101C18 1" }, + { "R 0x18300000 1" }, + { "R 0x183A3A84 2" }, + { "R 0x18393A84 2" }, + { "R 0x00100000" }, + { "R 0x00100004" }, + { "R 0x00100008" }, + { "R 0x0010000C" }, + { "R 0x00100010" }, + { "R 0x00100014" }, + { "R 0x00100018" }, + { "R 0x0010001C" }, + { "R 0x00100020" }, + { "R 0x00100024" }, + { "R 0x00100028" }, + { "R 0x0010002C" }, + { "R 0x00100030" }, + { "R 0x00100034" }, + { "R 0x00100100" }, + { "R 0x00100104" }, + { "R 0x00100108" }, + { "R 0x0010010C" }, + { "R 0x00101000" }, + { "R 0x00101004" }, + { "R 0x00101008" }, + { "R 0x0010100C" }, + { "R 0x00101010" }, + { "R 0x00101014" }, + { "R 0x00101018" }, + { "R 0x0010101C" }, + { "R 0x00101020" }, + { "R 0x00101024" }, + { "R 0x00101028" }, + { "R 0x0010102C" }, + { "R 0x00101030" }, + { "R 0x00101034" }, + { "R 0x00102000" }, + { "R 0x00102004" }, + { "R 0x00102008" }, + { "R 0x0010200C" }, + { "R 0x00102010" }, + { "R 0x00102014" }, + { "R 0x00102018" }, + { "R 0x0010201C" }, + { "R 0x00102020" }, + { "R 0x00102024" }, + { "R 0x00102028" }, + { "R 0x0010202C" }, + { "R 0x00102030" }, + { "R 0x00102034" }, + { "R 0x00103000" }, + { "R 0x00103004" }, + { "R 0x00103008" }, + { "R 0x0010300C" }, + { "R 0x00103010" }, + { "R 0x00103014" }, + { "R 0x00103018" }, + { "R 0x0010301C" }, + { "R 0x00103020" }, + { "R 0x00103024" }, + { "R 0x00103028" }, + { "R 0x0010302C" }, + { "R 0x00103030" }, + { "R 0x00103034" }, + { "R 0x00113000" }, + { "R 0x00113004" }, + { "R 0x00113008" }, + { "R 0x0011300C" }, + { "R 0x00113010" }, + { "R 0x00113014" }, + { "R 0x00113018" }, + { "R 0x0011301C" }, + { "R 0x00113020" }, + { "R 0x00113024" }, + { "R 0x00113028" }, + { "R 0x0011302C" }, + { "R 0x00113030" }, + { "R 0x00113034" }, + { "R 0x0011A000" }, + { "R 0x0011A004" }, + { "R 0x0011A008" }, + { "R 0x0011A00C" }, + { "R 0x0011A010" }, + { "R 0x0011A014" }, + { "R 0x0011A018" }, + { "R 0x0011A01C" }, + { "R 0x0011A020" }, + { "R 0x0011A024" }, + { "R 0x0011A028" }, + { "R 0x0011A02C" }, + { "R 0x0011A030" }, + { "R 0x0011A034" }, + { "R 0x0011B000" }, + { "R 0x0011B004" }, + { "R 0x0011B008" }, + { "R 0x0011B00C" }, + { "R 0x0011B010" }, + { "R 0x0011B014" }, + { "R 0x0011B018" }, + { "R 0x0011B01C" }, + { "R 0x0011B020" }, + { "R 0x0011B024" }, + { "R 0x0011B028" }, + { "R 0x0011B02C" }, + { "R 0x0011B030" }, + { "R 0x0011B034" }, + { "R 0x00174000" }, + { "R 0x00174004" }, + { "R 0x00174008" }, + { "R 0x0017400C" }, + { "R 0x00174010" }, + { "R 0x00174014" }, + { "R 0x00174018" }, + { "R 0x0017401C" }, + { "R 0x00174020" }, + { "R 0x00174024" }, + { "R 0x00174028" }, + { "R 0x0017402C" }, + { "R 0x00174030" }, + { "R 0x00174034" }, + { "R 0x00176000" }, + { "R 0x00176004" }, + { "R 0x00176008" }, + { "R 0x0017600C" }, + { "R 0x00176010" }, + { "R 0x00176014" }, + { "R 0x00176018" }, + { "R 0x0017601C" }, + { "R 0x00176020" }, + { "R 0x00176024" }, + { "R 0x00176028" }, + { "R 0x0017602C" }, + { "R 0x00176030" }, + { "R 0x00176034" }, + { "R 0x0010401C" }, + { "R 0x00183024" }, + { "R 0x00144168" }, + { "R 0x0011702C" }, + { "R 0x0010904C" }, + { "R 0x00189038" }, + { "R 0x001443E8" }, + { "R 0x001442B8" }, + { "R 0x00105060" }, + { "R 0x00141024" }, + { "R 0x00145038" }, + { "R 0x00109004" }, + { "R 0x00189004" }, + { "R 0x00190004" }, + { "R 0x0C2A0000" }, + { "R 0x0C2A0004" }, + { "R 0x0C2A0008" }, + { "R 0x0C2A000C" }, + { "R 0x0C2A0010" }, + { "R 0x0C2A0014" }, + { "R 0x0C2A0018" }, + { "R 0x0C2A001C" }, + { "R 0x0C2A0020" }, + { "R 0x0C2A0024" }, + { "R 0x0C2A0028" }, + { "R 0x0C2A002C" }, + { "R 0x0C2A0030" }, + { "R 0x0C2A0034" }, + { "R 0x0C2A1000" }, + { "R 0x0C2A1004" }, + { "R 0x0C2A1008" }, + { "R 0x0C2A100C" }, + { "R 0x0C2A1010" }, + { "R 0x0C2A1014" }, + { "R 0x0C2A1018" }, + { "R 0x0C2A101C" }, + { "R 0x0C2A1020" }, + { "R 0x0C2A1024" }, + { "R 0x0C2A1028" }, + { "R 0x0C2A102C" }, + { "R 0x0C2A1030" }, + { "R 0x0C2A2260" }, + { "R 0x0C2A2264" }, + { "R 0x0C2A3008" }, + { "R 0x0C2A300C" }, + { "R 0x0C2A3010" }, + { "R 0x0C2A3014" }, + { "R 0x0C2A3024" }, + { "R 0x0C2A2034" }, + { "R 0x0C2A214C" }, + { "R 0x0C2A2150" }, + { "R 0x0C2A2154" }, + { "R 0x0C2630A0 4" }, + { "R 0x0C2630B0 4" }, + { "R 0x0C2630C0 4" }, + { "R 0x0C2630D0 4" }, + { "R 0x1800005C 1" }, + { "R 0x1801005C 1" }, + { "R 0x1802005C 1" }, + { "R 0x1803005C 1" }, + { "R 0x1804005C 1" }, + { "R 0x1805005C 1" }, + { "R 0x1806005C 1" }, + { "R 0x1807005C 1" }, + { "R 0x3d96000" }, + { "R 0x3d96004" }, +}; + +static const struct dcc_link_config talos_link_configs[] = { + { + .link_list = 3, + .entries = talos_dcc_entries, + .num_entries = ARRAY_SIZE(talos_dcc_entries), + }, +}; + +static const struct dcc_config talos_config = { + .lists = talos_link_configs, + .num_lists = ARRAY_SIZE(talos_link_configs), +}; + +static const struct dcc_pdata talos_pdata = { + .base = 0x010a2000, + .size = 0x00001000, + .ram_base = 0x010ae000, + .ram_size = 0x00002000, + .dcc_offset = 0x6000, + .map_ver = 0x1, + .config = &talos_config, +}; + +#endif /* _QCOM_DCC_TALOS_CONFIG_H */ diff --git a/drivers/misc/qcom-dcc.c b/drivers/misc/qcom-dcc.c new file mode 100644 index 0000000000000..d38c5f4a680b6 --- /dev/null +++ b/drivers/misc/qcom-dcc.c @@ -0,0 +1,1614 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +/* + * DCC(Data Capture and Compare) is a DMA engine designed for debugging + * purposes. + * In case of a system crash or manual software triggers by the user the + * DCC hardware stores the value at the register addresses which can be + * used for debugging purposes. + * The DCC driver provides the user with debugfs interface to configure the + * register addresses. The options that the DCC hardware provides include + * reading from registers, writing to registers, first reading and then + * writing to registers and looping through the values of the same + * register. + * + * In certain cases a register write needs to be executed for accessing the + * rest of the registers, also the user might want to record the changing + * values of a register with time for which he has the option to use the + * loop feature. + * + * The options mentioned above are exposed to the user by debugfs files + * once the driver is probed. The details and usage of this debugfs files + * are documented in Documentation/ABI/testing/debugfs-driver-dcc. + * + * As an example let us consider a couple of debug scenarios where DCC has + * been proved to be effective for debugging purposes:- + * + * i)TimeStamp Related Issue + * + * On SC7180, there was a coresight timestamp issue where it would + * occasionally be all 0 instead of proper timestamp values. + * + * Proper timestamp: + * Idx:3373; ID:10; I_TIMESTAMP : Timestamp.; Updated val = + * 0x13004d8f5b7aa; CC=0x9e + * + * Zero timestamp: + * Idx:3387; ID:10; I_TIMESTAMP : Timestamp.; Updated val = 0x0; CC=0xa2 + * + * Now this is a non-fatal issue and doesn't need a system reset, but still + * needs to be rootcaused and fixed for those who do care about coresight + * etm traces. + * Since this is a timestamp issue, we would be looking for any timestamp + * related clocks and such. + * + * We get all the clk register details from IP documentation and configure + * it via DCC config debugfs node. Before that we set the current + * linked list. + * + * Program the linked list with the addresses + * echo R 0x10c004 > /sys/kernel/debug/qcom-dcc/../3/config + * echo R 0x10c008 > /sys/kernel/debug/qcom-dcc/../3/config + * echo R 0x10c00c > /sys/kernel/debug/qcom-dcc/../3/config + * echo R 0x10c010 > /sys/kernel/debug/qcom-dcc/../3/config + * ..... and so on for other timestamp related clk registers + * + * Other way of specifying is in "addr len" pair, in below case it + * specifies to capture 4 words starting 0x10C004 + * + * echo R 0x10C004 4 > /sys/kernel/debug/qcom-dcc/../3/config + * + * Configuration can be saved to a file and reuse it later. + * cat /sys/kernel/debug/qcom-dcc/../3/config > config_3 + * Post reboot, write the file to config. + * echo config_3 > /sys/kernel/debug/qcom-dcc/../3/config + * + * Enable DCC + * echo 1 > /sys/kernel/debug/qcom-dcc/../3/enable + * + * Run the timestamp test for working case + * + * Send SW trigger + * echo 1 > /sys/kernel/debug/qcom-dcc/../trigger + * + * Read SRAM + * cat /dev/dcc_sram > dcc_sram1.bin + * + * Run the timestamp test for non-working case + * + * Send SW trigger + * echo 1 > /sys/kernel/debug/qcom-dcc/../trigger + * + * Read SRAM + * cat /dev/dcc_sram > dcc_sram2.bin + * + * Get the parser from + * https://git.codelinaro.org/clo/le/platform/vendor/qcom-opensource/tools/-/tree/opensource-tools.lnx.1.0.r176-rel/dcc_parser + * + * Parse the SRAM bin + * python dcc_parser.py -s dcc_sram1.bin --v2 -o output/ python + * dcc_parser.py -s dcc_sram2.bin --v2 -o output/ + * + * Sample parsed output of dcc_sram1.bin: + * + * + * 03/14/21 + * Linux DCC Parser + * + * + * + * + * + * + * next_ll_offset : 0x1c + * + * ii)NOC register errors + * + * A particular class of registers called NOC which are functional + * registers was reporting errors while logging the values.To trace these + * errors the DCC has been used effectively. + * The steps followed were similar to the ones mentioned above. + * In addition to NOC registers a few other dependent registers were + * configured in DCC to monitor it's values during a crash. A look at the + * dependent register values revealed that the crash was happening due to a + * secured access to one of these dependent registers. + * All these debugging activity and finding the root cause was achieved + * using DCC. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom-dcc.h" + +#define STATUS_READY_TIMEOUT 5000 /* microseconds */ + +/* DCC registers */ +#define DCC_HW_INFO 0x04 +#define DCC_LL_NUM_INFO 0x10 +#define DCC_LL_LOCK 0x00 +#define DCC_LL_CFG 0x04 +#define DCC_LL_BASE 0x08 +#define DCC_FD_BASE 0x0c +#define DCC_LL_OFFSET 0x80 +#define DCC_LL_TIMEOUT 0x10 +#define DCC_LL_INT_ENABLE 0x18 +#define DCC_LL_INT_STATUS 0x1c +#define DCC_LL_SW_TRIGGER 0x2c +#define DCC_LL_BUS_ACCESS_STATUS 0x30 + +/* Default value used if a bit 6 in the HW_INFO register is set. */ +#define DCC_FIX_LOOP_OFFSET 16 + +/* Mask to find version info from HW_Info register */ +#define DCC_VER_INFO_MASK BIT(9) + +#define MAX_DCC_OFFSET GENMASK(9, 2) +#define MAX_DCC_LEN GENMASK(6, 0) +#define MAX_LOOP_CNT GENMASK(7, 0) +#define MAX_LOOP_ADDR 10 + +#define DCC_ADDR_DESCRIPTOR 0x00 +#define DCC_ADDR_LIMIT 27 +#define DCC_WORD_SIZE sizeof(u32) +#define DCC_ADDR_RANGE_MASK GENMASK(31, 4) +#define DCC_LOOP_DESCRIPTOR BIT(30) +#define DCC_RD_MOD_WR_DESCRIPTOR BIT(31) +#define DCC_LINK_DESCRIPTOR GENMASK(31, 30) +#define DCC_STATUS_MASK GENMASK(1, 0) +#define DCC_LOCK_MASK BIT(0) +#define DCC_LOOP_OFFSET_MASK BIT(6) +#define DCC_TRIGGER_MASK BIT(9) + +#define DCC_WRITE_MASK BIT(15) +#define DCC_WRITE_OFF_MASK GENMASK(7, 0) +#define DCC_WRITE_LEN_MASK GENMASK(14, 8) + +#define DCC_READ_IND 0x00 +#define DCC_WRITE_IND (BIT(28)) + +#define DCC_AHB_IND 0x00 +#define DCC_APB_IND BIT(29) + +#define DCC_MAX_LINK_LIST 8 + +#define DCC_VER_MASK2 GENMASK(5, 0) + +#define DCC_SRAM_WORD_LENGTH 4 + +#define DCC_RD_MOD_WR_ADDR 0xC105E + +#define MEM_MAP_VER1 0x1 +#define MEM_MAP_VER2 0x2 +#define MEM_MAP_VER3 0x3 + +#define LINE_BUFFER_MAX_SZ 50 +enum dcc_descriptor_type { + DCC_READ_TYPE, + DCC_LOOP_TYPE, + DCC_READ_WRITE_TYPE, + DCC_WRITE_TYPE +}; + +/** + * struct dcc_config_entry - configuration information related to each dcc instruction + * @base: Base address of the register to be configured in dcc + * @offset: Offset to the base address to be configured in dcc + * @len: Length of the address in words of 4 bytes to be configured in dcc + * @loop_cnt: The number of times to loop on the register address in case + of loop instructions + * @write_val: The value to be written on the register address in case of + write instructions + * @mask: Mask corresponding to the value to be written in case of + write instructions + * @apb_bus: Type of bus to be used for the instruction, can be either + 'apb' if 1 or 'ahb' if 0 + * @desc_type: Stores the type of dcc instruction + * @list: This is used to append this instruction to the list of + instructions + */ +struct dcc_config_entry { + u32 base; + u32 offset; + u32 len; + u32 loop_cnt; + u32 write_val; + u32 mask; + bool apb_bus; + enum dcc_descriptor_type desc_type; + struct list_head list; +}; + +/** + * struct dcc_drvdata - configuration information related to a dcc device + * @base: Base Address of the dcc device + * @dev: The device attached to the driver data + * @mutex: Lock to protect access and manipulation of dcc_drvdata + * @ram_base: Base address for the SRAM dedicated for the dcc device + * @ram_size: Total size of the SRAM dedicated for the dcc device + * @ram_offset: Offset to the SRAM dedicated for dcc device + * @ram_cfg: Used for address limit calculation for dcc + * @ram_start: Starting address of DCC SRAM + * @mem_map_ver: Memory map version of DCC hardware + * @sram_dev: Miscellaneous device equivalent of dcc SRAM + * @cfg_head: Points to the head of the linked list of addresses + * @dbg_dir: The dcc debugfs directory under which all the debugfs files are placed + * @max_link_list: Total number of linkedlists supported by the DCC configuration + * @loop_shift: Loop offset bits range for the addresses + * @enable_bitmap: Bitmap to capture the enabled status of each linked list of addresses + */ +struct dcc_drvdata { + void __iomem *base; + void __iomem *ram_base; + struct device *dev; + /* Lock to protect access and manipulation of dcc_drvdata */ + struct mutex mutex; + size_t ram_size; + u32 ram_offset; + unsigned int ram_cfg; + unsigned int ram_start; + u64 mem_map_ver; + struct miscdevice sram_dev; + struct list_head *cfg_head; + struct dentry *dbg_dir; + size_t max_link_list; + u8 loop_shift; + unsigned long *enable_bitmap; + char **temp_buff_ptr; +}; + +struct dcc_cfg_attr { + u32 addr; + u32 prev_addr; + u32 prev_off; + u32 link; + u32 sram_offset; +}; + +struct dcc_cfg_loop_attr { + u32 loop_cnt; + u32 loop_len; + u32 loop_off; + bool loop_start; +}; + +static inline u32 dcc_status(int version) +{ + return version == 1 ? 0x0c : 0x1c; +} + +static inline u32 dcc_list_offset(int version) +{ + if (version == 1) + return 0x1c; + else if (version == 2) + return 0x2c; + else + return 0x34; +} + +static inline void dcc_list_writel(struct dcc_drvdata *drvdata, + u32 val, u32 ll, u32 off) +{ + u32 offset = dcc_list_offset(drvdata->mem_map_ver) + off; + + writel(val, drvdata->base + ll * DCC_LL_OFFSET + offset); +} + +static inline u32 dcc_list_readl(struct dcc_drvdata *drvdata, u32 ll, u32 off) +{ + u32 offset = dcc_list_offset(drvdata->mem_map_ver) + off; + + return readl(drvdata->base + ll * DCC_LL_OFFSET + offset); +} + +static void dcc_sram_write_auto(struct dcc_drvdata *drvdata, + u32 val, u32 *off) +{ + /* If the overflow condition is met increment the offset + * and return to indicate that overflow has occurred + */ + if (*off > drvdata->ram_size - 4) { + *off += 4; + return; + } + + writel(val, drvdata->ram_base + *off); + + *off += 4; +} + +static int dcc_sw_trigger(struct dcc_drvdata *drvdata) +{ + void __iomem *addr; + int i; + u32 status; + u32 ll_cfg; + u32 tmp_ll_cfg; + u32 val; + int ret = 0; + + mutex_lock(&drvdata->mutex); + + for (i = 0; i < drvdata->max_link_list; i++) { + if (!test_bit(i, drvdata->enable_bitmap)) + continue; + ll_cfg = dcc_list_readl(drvdata, i, DCC_LL_CFG); + if (drvdata->mem_map_ver == MEM_MAP_VER3) + tmp_ll_cfg = ll_cfg & ~BIT(8); + else + tmp_ll_cfg = ll_cfg & ~DCC_TRIGGER_MASK; + + dcc_list_writel(drvdata, tmp_ll_cfg, i, DCC_LL_CFG); + dcc_list_writel(drvdata, 1, i, DCC_LL_SW_TRIGGER); + dcc_list_writel(drvdata, ll_cfg, i, DCC_LL_CFG); + } + + addr = drvdata->base + dcc_status(drvdata->mem_map_ver); + if (readl_poll_timeout(addr, val, !FIELD_GET(DCC_STATUS_MASK, val), + 1, STATUS_READY_TIMEOUT)) { + dev_err(drvdata->dev, "DCC is busy after receiving sw trigger\n"); + ret = -EBUSY; + goto out_unlock; + } + + for (i = 0; i < drvdata->max_link_list; i++) { + if (!test_bit(i, drvdata->enable_bitmap)) + continue; + + status = dcc_list_readl(drvdata, i, DCC_LL_BUS_ACCESS_STATUS); + if (!status) + continue; + + dev_err(drvdata->dev, "Read access error for list %d err: 0x%x\n", + i, status); + ll_cfg = dcc_list_readl(drvdata, i, DCC_LL_CFG); + if (drvdata->mem_map_ver == MEM_MAP_VER3) + tmp_ll_cfg = ll_cfg & ~BIT(8); + else + tmp_ll_cfg = ll_cfg & ~DCC_TRIGGER_MASK; + + dcc_list_writel(drvdata, tmp_ll_cfg, i, DCC_LL_CFG); + dcc_list_writel(drvdata, DCC_STATUS_MASK, i, DCC_LL_BUS_ACCESS_STATUS); + dcc_list_writel(drvdata, ll_cfg, i, DCC_LL_CFG); + ret = -ENODATA; + break; + } + +out_unlock: + mutex_unlock(&drvdata->mutex); + return ret; +} + +static void dcc_ll_cfg_reset_link(struct dcc_cfg_attr *cfg) +{ + cfg->addr = 0x00; + cfg->link = 0; + cfg->prev_off = 0; + cfg->prev_addr = cfg->addr; +} + +static void dcc_emit_read_write(struct dcc_drvdata *drvdata, + struct dcc_config_entry *entry, + struct dcc_cfg_attr *cfg) +{ + if (cfg->link) { + /* + * write new offset = 1 to continue + * processing the list + */ + + dcc_sram_write_auto(drvdata, cfg->link, &cfg->sram_offset); + + /* Reset link and prev_off */ + dcc_ll_cfg_reset_link(cfg); + } + + cfg->addr = DCC_RD_MOD_WR_DESCRIPTOR; + dcc_sram_write_auto(drvdata, cfg->addr, &cfg->sram_offset); + + dcc_sram_write_auto(drvdata, entry->mask, &cfg->sram_offset); + + dcc_sram_write_auto(drvdata, entry->write_val, &cfg->sram_offset); + + cfg->addr = 0; +} + +static void dcc_emit_loop(struct dcc_drvdata *drvdata, struct dcc_config_entry *entry, + struct dcc_cfg_attr *cfg, + struct dcc_cfg_loop_attr *cfg_loop, + u32 *total_len) +{ + int loop; + + /* Check if we need to write link of prev entry */ + if (cfg->link) + dcc_sram_write_auto(drvdata, cfg->link, &cfg->sram_offset); + + if (cfg_loop->loop_start) { + loop = (cfg->sram_offset - cfg_loop->loop_off) / 4; + loop |= (cfg_loop->loop_cnt << drvdata->loop_shift) & + GENMASK(DCC_ADDR_LIMIT, drvdata->loop_shift); + loop |= DCC_LOOP_DESCRIPTOR; + *total_len += (*total_len - cfg_loop->loop_len) * cfg_loop->loop_cnt; + + dcc_sram_write_auto(drvdata, loop, &cfg->sram_offset); + + cfg_loop->loop_start = false; + cfg_loop->loop_len = 0; + cfg_loop->loop_off = 0; + } else { + cfg_loop->loop_start = true; + cfg_loop->loop_cnt = entry->loop_cnt - 1; + cfg_loop->loop_len = *total_len; + cfg_loop->loop_off = cfg->sram_offset; + } + + /* Reset link and prev_off */ + dcc_ll_cfg_reset_link(cfg); +} + +static void dcc_emit_write(struct dcc_drvdata *drvdata, + struct dcc_config_entry *entry, + struct dcc_cfg_attr *cfg) +{ + u32 off; + + if (cfg->link) { + /* + * write new offset = 1 to continue + * processing the list + */ + dcc_sram_write_auto(drvdata, cfg->link, &cfg->sram_offset); + + /* Reset link and prev_off */ + cfg->addr = 0x00; + cfg->prev_off = 0; + cfg->prev_addr = cfg->addr; + } + + off = entry->offset / 4; + /* write new offset-length pair to correct position */ + cfg->link |= ((off & DCC_WRITE_OFF_MASK) | DCC_WRITE_MASK | + FIELD_PREP(DCC_WRITE_LEN_MASK, entry->len)); + cfg->link |= DCC_LINK_DESCRIPTOR; + + /* Address type */ + cfg->addr = (entry->base >> 4) & GENMASK(DCC_ADDR_LIMIT, 0); + if (entry->apb_bus) + cfg->addr |= DCC_ADDR_DESCRIPTOR | DCC_WRITE_IND | DCC_APB_IND; + else + cfg->addr |= DCC_ADDR_DESCRIPTOR | DCC_WRITE_IND | DCC_AHB_IND; + dcc_sram_write_auto(drvdata, cfg->addr, &cfg->sram_offset); + + dcc_sram_write_auto(drvdata, cfg->link, &cfg->sram_offset); + + dcc_sram_write_auto(drvdata, entry->write_val, &cfg->sram_offset); + + cfg->addr = 0x00; + cfg->link = 0; +} + +static int dcc_emit_read(struct dcc_drvdata *drvdata, + struct dcc_config_entry *entry, + struct dcc_cfg_attr *cfg, + u32 *pos, u32 *total_len) +{ + u32 off; + u32 temp_off; + + cfg->addr = (entry->base >> 4) & GENMASK(27, 0); + + if (entry->apb_bus) + cfg->addr |= DCC_ADDR_DESCRIPTOR | DCC_READ_IND | DCC_APB_IND; + else + cfg->addr |= DCC_ADDR_DESCRIPTOR | DCC_READ_IND | DCC_AHB_IND; + + off = entry->offset / 4; + + *total_len += entry->len * 4; + + if (!cfg->prev_addr || cfg->prev_addr != cfg->addr || cfg->prev_off > off) { + /* Check if we need to write prev link entry */ + if (cfg->link) + dcc_sram_write_auto(drvdata, cfg->link, &cfg->sram_offset); + dev_dbg(drvdata->dev, "DCC: sram address 0x%x\n", cfg->sram_offset); + + /* Write address */ + dcc_sram_write_auto(drvdata, cfg->addr, &cfg->sram_offset); + + /* Reset link and prev_off */ + cfg->link = 0; + cfg->prev_off = 0; + } + + if ((off - cfg->prev_off) > 0xff || entry->len > MAX_DCC_LEN) { + dev_err(drvdata->dev, "DCC: Programming error Base: 0x%x, offset 0x%x\n", + entry->base, entry->offset); + return -EINVAL; + } + + if (cfg->link) { + /* + * link already has one offset-length so new + * offset-length needs to be placed at + * bits [29:15] + */ + *pos = 15; + + /* Clear bits [31:16] */ + cfg->link &= GENMASK(14, 0); + } else { + /* + * link is empty, so new offset-length needs + * to be placed at bits [15:0] + */ + *pos = 0; + cfg->link = 1 << 15; + } + + /* write new offset-length pair to correct position */ + temp_off = (off - cfg->prev_off) & GENMASK(7, 0); + cfg->link |= (temp_off | ((entry->len << 8) & GENMASK(14, 8))) << *pos; + + cfg->link |= DCC_LINK_DESCRIPTOR; + + if (*pos) { + dcc_sram_write_auto(drvdata, cfg->link, &cfg->sram_offset); + cfg->link = 0; + } + + cfg->prev_off = off + entry->len - 1; + cfg->prev_addr = cfg->addr; + return 0; +} + +static int dcc_emit_config(struct dcc_drvdata *drvdata, unsigned int curr_list) +{ + int ret; + u32 total_len, pos; + struct dcc_config_entry *entry; + struct dcc_cfg_attr cfg = {0}; + struct dcc_cfg_loop_attr cfg_loop = {0}; + + cfg.sram_offset = drvdata->ram_cfg * 4; + total_len = 0; + + list_for_each_entry(entry, &drvdata->cfg_head[curr_list], list) { + switch (entry->desc_type) { + case DCC_READ_WRITE_TYPE: + dcc_emit_read_write(drvdata, entry, &cfg); + break; + + case DCC_LOOP_TYPE: + dcc_emit_loop(drvdata, entry, &cfg, &cfg_loop, &total_len); + break; + + case DCC_WRITE_TYPE: + dcc_emit_write(drvdata, entry, &cfg); + break; + + case DCC_READ_TYPE: + ret = dcc_emit_read(drvdata, entry, &cfg, &pos, &total_len); + if (ret) + goto err; + break; + } + } + + if (cfg.link) + dcc_sram_write_auto(drvdata, cfg.link, &cfg.sram_offset); + + if (cfg_loop.loop_start) { + dev_err(drvdata->dev, "DCC: Programming error: Loop unterminated\n"); + ret = -EINVAL; + goto err; + } + + /* Handling special case of list ending with a rd_mod_wr */ + if (cfg.addr == DCC_RD_MOD_WR_DESCRIPTOR) { + cfg.addr = (DCC_RD_MOD_WR_ADDR) & GENMASK(27, 0); + cfg.addr |= DCC_ADDR_DESCRIPTOR; + dcc_sram_write_auto(drvdata, cfg.addr, &cfg.sram_offset); + } + + /* Setting zero to indicate end of the list */ + cfg.link = DCC_LINK_DESCRIPTOR; + dcc_sram_write_auto(drvdata, cfg.link, &cfg.sram_offset); + + /* Check if sram offset exceeds the ram size */ + if (cfg.sram_offset > drvdata->ram_size) + goto overstep; + + /* Update ram_cfg and check if the data will overstep */ + drvdata->ram_cfg = (cfg.sram_offset + total_len) / 4; + + if (cfg.sram_offset + total_len > drvdata->ram_size) { + cfg.sram_offset += total_len; + goto overstep; + } + + drvdata->ram_start = cfg.sram_offset / 4; + return 0; +overstep: + ret = -EINVAL; + memset_io(drvdata->ram_base, 0, drvdata->ram_size); + +err: + return ret; +} + +static bool dcc_valid_list(struct dcc_drvdata *drvdata, unsigned int curr_list) +{ + u32 lock_reg; + + if (list_empty(&drvdata->cfg_head[curr_list])) + return false; + + if (test_bit(curr_list, drvdata->enable_bitmap)) { + dev_err(drvdata->dev, "List %d is already enabled\n", curr_list); + return false; + } + + lock_reg = dcc_list_readl(drvdata, curr_list, DCC_LL_LOCK); + if (lock_reg & DCC_LOCK_MASK) { + dev_err(drvdata->dev, "List %d is already locked\n", curr_list); + return false; + } + + return true; +} + +static bool is_dcc_enabled(struct dcc_drvdata *drvdata) +{ + int list; + + for (list = 0; list < drvdata->max_link_list; list++) + if (test_bit(list, drvdata->enable_bitmap)) + return true; + + return false; +} + +static int dcc_enable(struct dcc_drvdata *drvdata, unsigned int curr_list) +{ + int ret; + u32 ram_cfg_base; + + mutex_lock(&drvdata->mutex); + + if (!dcc_valid_list(drvdata, curr_list)) { + ret = -EINVAL; + goto out_unlock; + } + + /* Fill dcc sram with the poison value. + * This helps in understanding bus + * hang from registers returning a zero + */ + if (!is_dcc_enabled(drvdata)) + memset_io(drvdata->ram_base, 0xde, drvdata->ram_size); + + /* 1. Take ownership of the list */ + dcc_list_writel(drvdata, DCC_LOCK_MASK, curr_list, DCC_LL_LOCK); + + /* 2. Program linked-list in the SRAM */ + ram_cfg_base = drvdata->ram_cfg; + ret = dcc_emit_config(drvdata, curr_list); + if (ret) { + dcc_list_writel(drvdata, 0, curr_list, DCC_LL_LOCK); + goto out_unlock; + } + + /* 3. Program DCC_RAM_CFG reg */ + dcc_list_writel(drvdata, ram_cfg_base + + drvdata->ram_offset / 4, curr_list, DCC_LL_BASE); + dcc_list_writel(drvdata, drvdata->ram_start + + drvdata->ram_offset / 4, curr_list, DCC_FD_BASE); + dcc_list_writel(drvdata, 0xFFF, curr_list, DCC_LL_TIMEOUT); + + /* 4. Clears interrupt status register */ + dcc_list_writel(drvdata, 0, curr_list, DCC_LL_INT_ENABLE); + dcc_list_writel(drvdata, (BIT(0) | BIT(1) | BIT(2)), + curr_list, DCC_LL_INT_STATUS); + + set_bit(curr_list, drvdata->enable_bitmap); + + /* 5. Configure trigger */ + if (drvdata->mem_map_ver == MEM_MAP_VER3) + dcc_list_writel(drvdata, BIT(8), + curr_list, DCC_LL_CFG); + else + dcc_list_writel(drvdata, DCC_TRIGGER_MASK, + curr_list, DCC_LL_CFG); + +out_unlock: + mutex_unlock(&drvdata->mutex); + return ret; +} + +static void dcc_disable(struct dcc_drvdata *drvdata, int curr_list) +{ + mutex_lock(&drvdata->mutex); + + if (!test_bit(curr_list, drvdata->enable_bitmap)) + goto out_unlock; + dcc_list_writel(drvdata, 0, curr_list, DCC_LL_CFG); + dcc_list_writel(drvdata, 0, curr_list, DCC_LL_BASE); + dcc_list_writel(drvdata, 0, curr_list, DCC_FD_BASE); + dcc_list_writel(drvdata, 0, curr_list, DCC_LL_LOCK); + clear_bit(curr_list, drvdata->enable_bitmap); +out_unlock: + mutex_unlock(&drvdata->mutex); +} + +static u32 dcc_filp_curr_list(const struct file *filp) +{ + struct dentry *dentry = file_dentry(filp); + int curr_list, ret; + + ret = kstrtoint(dentry->d_parent->d_name.name, 0, &curr_list); + if (ret) + return ret; + + return curr_list; +} + +static ssize_t enable_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *ppos) +{ + char *buf; + int curr_list = dcc_filp_curr_list(filp); + struct dcc_drvdata *drvdata = filp->private_data; + + if (curr_list < 0) + return curr_list; + + mutex_lock(&drvdata->mutex); + if (test_bit(curr_list, drvdata->enable_bitmap)) + buf = "Y\n"; + else + buf = "N\n"; + mutex_unlock(&drvdata->mutex); + + return simple_read_from_buffer(userbuf, count, ppos, buf, strlen(buf)); +} + +static ssize_t enable_write(struct file *filp, const char __user *userbuf, + size_t count, loff_t *ppos) +{ + int ret = 0, curr_list; + bool val; + struct dcc_drvdata *drvdata = filp->private_data; + + curr_list = dcc_filp_curr_list(filp); + if (curr_list < 0) + return curr_list; + + ret = kstrtobool_from_user(userbuf, count, &val); + if (ret < 0) + return ret; + + if (val) { + ret = dcc_enable(drvdata, curr_list); + if (ret) + return ret; + } else { + dcc_disable(drvdata, curr_list); + } + + return count; +} + +static const struct file_operations enable_fops = { + .read = enable_read, + .write = enable_write, + .open = simple_open, + .llseek = generic_file_llseek, +}; + +static ssize_t trigger_write(struct file *filp, + const char __user *user_buf, size_t count, + loff_t *ppos) +{ + int ret; + unsigned int val; + struct dcc_drvdata *drvdata = filp->private_data; + + ret = kstrtouint_from_user(user_buf, count, 0, &val); + if (ret < 0) + return ret; + + if (val != 1) + return -EINVAL; + + ret = dcc_sw_trigger(drvdata); + if (ret < 0) + return ret; + + return count; +} + +static const struct file_operations trigger_fops = { + .write = trigger_write, + .open = simple_open, + .llseek = generic_file_llseek, +}; + +static int dcc_config_add(struct dcc_drvdata *drvdata, unsigned int addr, + unsigned int len, bool apb_bus, int curr_list) +{ + int ret = 0; + struct dcc_config_entry *entry, *pentry; + unsigned int base, offset; + + mutex_lock(&drvdata->mutex); + + if (!len || len > drvdata->ram_size / DCC_WORD_SIZE) { + dev_err(drvdata->dev, "DCC: Invalid length\n"); + ret = -EINVAL; + goto out_unlock; + } + + base = addr & DCC_ADDR_RANGE_MASK; + + if (!list_empty(&drvdata->cfg_head[curr_list])) { + pentry = list_last_entry(&drvdata->cfg_head[curr_list], + struct dcc_config_entry, list); + + if (pentry->desc_type == DCC_READ_TYPE && + addr >= (pentry->base + pentry->offset) && + addr <= (pentry->base + pentry->offset + MAX_DCC_OFFSET)) { + /* Re-use base address from last entry */ + base = pentry->base; + + if ((pentry->len * 4 + pentry->base + pentry->offset) + == addr) { + len += pentry->len; + + if (len > MAX_DCC_LEN) + pentry->len = MAX_DCC_LEN; + else + pentry->len = len; + + addr = pentry->base + pentry->offset + + pentry->len * 4; + len -= pentry->len; + } + } + } + + offset = addr - base; + + while (len) { + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + ret = -ENOMEM; + goto out_unlock; + } + + entry->base = base; + entry->offset = offset; + entry->len = min_t(u32, len, MAX_DCC_LEN); + entry->desc_type = DCC_READ_TYPE; + entry->apb_bus = apb_bus; + INIT_LIST_HEAD(&entry->list); + list_add_tail(&entry->list, + &drvdata->cfg_head[curr_list]); + + len -= entry->len; + offset += MAX_DCC_LEN * 4; + } + +out_unlock: + mutex_unlock(&drvdata->mutex); + return ret; +} + +static ssize_t dcc_config_add_read(struct dcc_drvdata *drvdata, char *buf, int curr_list) +{ + bool bus; + int len, nval; + unsigned int base; + char apb_bus[4]; + + nval = sscanf(buf, "%x %i %3s", &base, &len, apb_bus); + if (nval <= 0 || nval > 3) + return -EINVAL; + + if (nval == 1) { + len = 1; + bus = false; + } else if (nval == 2) { + bus = false; + } else if (!strcmp("apb", apb_bus)) { + bus = true; + } else if (!strcmp("ahb", apb_bus)) { + bus = false; + } else { + return -EINVAL; + } + + return dcc_config_add(drvdata, base, len, bus, curr_list); +} + +static void dcc_config_reset(struct dcc_drvdata *drvdata) +{ + struct dcc_config_entry *entry, *temp; + int curr_list; + + mutex_lock(&drvdata->mutex); + + for (curr_list = 0; curr_list < drvdata->max_link_list; curr_list++) { + list_for_each_entry_safe(entry, temp, + &drvdata->cfg_head[curr_list], list) { + list_del(&entry->list); + } + } + drvdata->ram_start = 0; + drvdata->ram_cfg = 0; + mutex_unlock(&drvdata->mutex); +} + +static ssize_t config_reset_write(struct file *filp, + const char __user *user_buf, size_t count, + loff_t *ppos) +{ + unsigned int val; + int ret; + struct dcc_drvdata *drvdata = filp->private_data; + + ret = kstrtouint_from_user(user_buf, count, 0, &val); + if (ret < 0) + return ret; + + if (val) + dcc_config_reset(drvdata); + + return count; +} + +static const struct file_operations config_reset_fops = { + .write = config_reset_write, + .open = simple_open, + .llseek = generic_file_llseek, +}; + +static ssize_t ready_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *ppos) +{ + char *buf; + struct dcc_drvdata *drvdata = filp->private_data; + + if (!is_dcc_enabled(drvdata)) + return -EINVAL; + + if (!FIELD_GET(BIT(1), readl(drvdata->base + dcc_status(drvdata->mem_map_ver)))) + buf = "Y\n"; + else + buf = "N\n"; + + return simple_read_from_buffer(userbuf, count, ppos, buf, strlen(buf) + 1); +} + +static const struct file_operations ready_fops = { + .read = ready_read, + .open = simple_open, + .llseek = generic_file_llseek, +}; + +static ssize_t loop_offset_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *ppos) +{ + char buf[4]; + struct dcc_drvdata *drvdata = filp->private_data; + + snprintf(buf, sizeof(buf), "%d", drvdata->loop_shift); + return simple_read_from_buffer(userbuf, count, ppos, buf, strlen(buf) + 1); +} + +static const struct file_operations loop_offset_fops = { + .read = loop_offset_read, + .open = simple_open, + .llseek = generic_file_llseek, +}; +static int dcc_add_loop(struct dcc_drvdata *drvdata, unsigned long loop_cnt, int curr_list) +{ + struct dcc_config_entry *entry; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + entry->loop_cnt = min_t(u32, loop_cnt, MAX_LOOP_CNT); + entry->desc_type = DCC_LOOP_TYPE; + INIT_LIST_HEAD(&entry->list); + list_add_tail(&entry->list, &drvdata->cfg_head[curr_list]); + + return 0; +} + +static ssize_t dcc_config_add_loop(struct dcc_drvdata *drvdata, char *buf, int curr_list) +{ + int ret, i = 0; + char *token, *input; + char delim[2] = " "; + unsigned int val[MAX_LOOP_ADDR]; + + input = buf; + + while ((token = strsep(&input, delim)) && i < MAX_LOOP_ADDR) { + ret = kstrtouint(token, 0, &val[i++]); + if (ret) + return ret; + } + + if (token) { + dev_err(drvdata->dev, "Max limit %u of loop address exceeded\n", + MAX_LOOP_ADDR); + return -EINVAL; + } + + if (val[1] < 1 || val[1] > 8 || val[1] > (i - 2)) + return -EINVAL; + + ret = dcc_add_loop(drvdata, val[0], curr_list); + if (ret) + return ret; + + for (i = 0; i < val[1]; i++) + dcc_config_add(drvdata, val[i + 2], 1, false, curr_list); + + return dcc_add_loop(drvdata, 1, curr_list); +} + +static int dcc_rd_mod_wr_add(struct dcc_drvdata *drvdata, unsigned int mask, + unsigned int val, int curr_list) +{ + int ret = 0; + struct dcc_config_entry *entry; + + mutex_lock(&drvdata->mutex); + + if (list_empty(&drvdata->cfg_head[curr_list])) { + dev_err(drvdata->dev, "DCC: No read address programmed\n"); + ret = -EPERM; + goto out_unlock; + } + + entry = devm_kzalloc(drvdata->dev, sizeof(*entry), GFP_KERNEL); + if (!entry) { + ret = -ENOMEM; + goto out_unlock; + } + + entry->desc_type = DCC_READ_WRITE_TYPE; + entry->mask = mask; + entry->write_val = val; + list_add_tail(&entry->list, &drvdata->cfg_head[curr_list]); +out_unlock: + mutex_unlock(&drvdata->mutex); + return ret; +} + +static ssize_t dcc_config_add_read_write(struct dcc_drvdata *drvdata, char *buf, int curr_list) +{ + int ret; + int nval; + unsigned int addr, mask, val; + + nval = sscanf(buf, "%x %x %x", &addr, &mask, &val); + + if (nval <= 1 || nval > 3) + return -EINVAL; + + ret = dcc_config_add(drvdata, addr, 1, false, curr_list); + if (ret) + return ret; + + return dcc_rd_mod_wr_add(drvdata, mask, val, curr_list); +} + +static int dcc_add_write(struct dcc_drvdata *drvdata, unsigned int addr, + unsigned int write_val, int apb_bus, int curr_list) +{ + struct dcc_config_entry *entry; + + entry = devm_kzalloc(drvdata->dev, sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + entry->desc_type = DCC_WRITE_TYPE; + entry->base = addr & GENMASK(31, 4); + entry->offset = addr - entry->base; + entry->write_val = write_val; + entry->len = 1; + entry->apb_bus = apb_bus; + list_add_tail(&entry->list, &drvdata->cfg_head[curr_list]); + + return 0; +} + +static ssize_t dcc_config_add_write(struct dcc_drvdata *drvdata, char *buf, int curr_list) +{ + bool bus; + int nval; + unsigned int addr, write_val; + char apb_bus[4]; + + nval = sscanf(buf, "%x %x %3s", &addr, &write_val, apb_bus); + + if (nval <= 1 || nval > 3) + return -EINVAL; + + if (nval == 2) + bus = false; + + if (nval == 3) { + if (!strcmp("apb", apb_bus)) + bus = true; + else if (!strcmp("ahb", apb_bus)) + bus = false; + else + return -EINVAL; + } + + return dcc_add_write(drvdata, addr, write_val, bus, curr_list); +} + +static int config_show(struct seq_file *m, void *data) +{ + struct dcc_drvdata *drvdata = m->private; + struct dcc_config_entry *entry, *next_entry, *prev_entry, *loop_entry; + int index = 0, curr_list, i; + unsigned int loop_val[MAX_LOOP_ADDR]; + + curr_list = dcc_filp_curr_list(m->file); + if (curr_list < 0) + return curr_list; + + mutex_lock(&drvdata->mutex); + + list_for_each_entry(entry, &drvdata->cfg_head[curr_list], list) { + index++; + switch (entry->desc_type) { + case DCC_READ_WRITE_TYPE: + prev_entry = list_prev_entry(entry, list); + seq_printf(m, "RW 0x%x 0x%x 0x%x\n", + prev_entry->base + prev_entry->offset, + entry->mask, + entry->write_val); + break; + case DCC_LOOP_TYPE: + loop_entry = entry; + loop_val[0] = loop_entry->loop_cnt; + loop_entry = list_next_entry(loop_entry, list); + for (i = 0; i < (MAX_LOOP_ADDR-2); + i++, loop_entry = list_next_entry(loop_entry, list)) { + if (loop_entry->desc_type == DCC_READ_TYPE) { + loop_val[i+2] = loop_entry->base + loop_entry->offset; + } else if (loop_entry->desc_type == DCC_LOOP_TYPE) { + loop_val[i+2] = loop_entry->loop_cnt; + loop_val[1] = i; + entry = loop_entry; + break; + } + } + seq_printf(m, "L 0x%x 0x%x", loop_val[0], loop_val[1]); + for (i = 0; i < loop_val[1]; i++) + seq_printf(m, " 0x%x", loop_val[i+2]); + seq_puts(m, "\n"); + break; + case DCC_WRITE_TYPE: + seq_printf(m, "W 0x%x 0x%x %s\n", + entry->base + entry->offset, + entry->write_val, + entry->apb_bus ? "apb":"ahb"); + break; + case DCC_READ_TYPE: + if (entry->len == 1) { + next_entry = list_next_entry(entry, list); + if (next_entry && next_entry->desc_type == DCC_READ_WRITE_TYPE) + continue; + } + seq_printf(m, "R 0x%x 0x%x %s\n", + entry->base + entry->offset, + entry->len, + entry->apb_bus ? "apb":"ahb"); + } + } + mutex_unlock(&drvdata->mutex); + return 0; +} + +static int config_open(struct inode *inode, struct file *file) +{ + struct dcc_drvdata *drvdata = inode->i_private; + + return single_open(file, config_show, drvdata); +} + +static ssize_t config_write(struct file *filp, + const char __user *user_buf, size_t count, + loff_t *ppos) +{ + int ret, curr_list; + char *token, *line; + char *buf, *bufp, *temp_buff; + char *delim = " "; + struct dcc_drvdata *drvdata = filp->f_inode->i_private; + ssize_t processed_len = 0; + + if (count == 0) + return -EINVAL; + buf = kzalloc(count+1, GFP_KERNEL); + if (buf) + bufp = buf; + else + return -ENOMEM; + + ret = copy_from_user(buf, user_buf, count); + if (ret) + goto err; + + curr_list = dcc_filp_curr_list(filp); + if (curr_list < 0) { + ret = curr_list; + goto err; + } + + while (bufp[0] != '\0') { + /* Parse line by line */ + line = strsep(&bufp, "\n"); + /* When one complete line could be parsed */ + if (line && bufp) { + processed_len += strlen(line) + 1; + if (drvdata->temp_buff_ptr && drvdata->temp_buff_ptr[curr_list]) { + temp_buff = drvdata->temp_buff_ptr[curr_list]; + /* Size of combined string must not be greater than + * allowed line size. + */ + if (strlen(line) + strlen(temp_buff) + 1 > LINE_BUFFER_MAX_SZ) { + dev_err(drvdata->dev, "Invalid input\n"); + ret = -EINVAL; + goto err; + } + strlcat(temp_buff, line, PAGE_SIZE); + line = temp_buff; + kfree(temp_buff); + drvdata->temp_buff_ptr[curr_list] = NULL; + } + + token = strsep(&line, delim); + + if (!strcmp("R", token)) { + ret = dcc_config_add_read(drvdata, line, curr_list); + } else if (!strcmp("W", token)) { + ret = dcc_config_add_write(drvdata, line, curr_list); + } else if (!strcmp("RW", token)) { + ret = dcc_config_add_read_write(drvdata, line, curr_list); + } else if (!strcmp("L", token)) { + ret = dcc_config_add_loop(drvdata, line, curr_list); + } else { + dev_err(drvdata->dev, "%s is not a correct input\n", token); + ret = -EINVAL; + } + + if (ret) + goto err; + } else { + /* Save the incomplete line to a temporary buffer and rejoin it later */ + if (!drvdata->temp_buff_ptr) { + drvdata->temp_buff_ptr = devm_kcalloc(drvdata->dev, + drvdata->max_link_list, + sizeof(char *), + GFP_KERNEL); + if (!drvdata->temp_buff_ptr) { + ret = -ENOMEM; + goto err; + } + } + drvdata->temp_buff_ptr[curr_list] = kzalloc(LINE_BUFFER_MAX_SZ, + GFP_KERNEL); + temp_buff = drvdata->temp_buff_ptr[curr_list]; + if (!temp_buff) { + ret = -ENOMEM; + goto err; + } + if ((count - processed_len) >= LINE_BUFFER_MAX_SZ) { + dev_err(drvdata->dev, "Invalid input\n"); + ret = -EINVAL; + goto err; + } + memcpy(temp_buff, line, count - processed_len); + temp_buff[count - processed_len + 1] = '\0'; + processed_len += (strlen(temp_buff) + 1); + break; + } + } + + kfree(buf); + return processed_len; + +err: + kfree(buf); + if (drvdata->temp_buff_ptr && drvdata->temp_buff_ptr[curr_list]) { + kfree(drvdata->temp_buff_ptr[curr_list]); + drvdata->temp_buff_ptr[curr_list] = NULL; + } + return ret; +} + +static const struct file_operations config_fops = { + .open = config_open, + .read = seq_read, + .write = config_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static void dcc_delete_debug_dir(struct dcc_drvdata *drvdata) +{ + debugfs_remove_recursive(drvdata->dbg_dir); +}; + +static void dcc_create_debug_dir(struct dcc_drvdata *drvdata) +{ + int i; + char list_num[10]; + struct dentry *dcc_dev, *list; + struct device *dev = drvdata->dev; + + drvdata->dbg_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + dcc_dev = debugfs_create_dir(dev_name(dev), drvdata->dbg_dir); + + for (i = 0; i < drvdata->max_link_list; i++) { + snprintf(list_num, sizeof(list_num), "%d", i); + list = debugfs_create_dir(list_num, dcc_dev); + debugfs_create_file("enable", 0600, list, drvdata, &enable_fops); + debugfs_create_file("config", 0600, list, drvdata, &config_fops); + } + + debugfs_create_file("trigger", 0200, drvdata->dbg_dir, drvdata, &trigger_fops); + debugfs_create_file("ready", 0400, drvdata->dbg_dir, drvdata, &ready_fops); + debugfs_create_file("config_reset", 0200, drvdata->dbg_dir, drvdata, &config_reset_fops); + debugfs_create_file("loop_offset", 0400, drvdata->dbg_dir, drvdata, &loop_offset_fops); +} + +static ssize_t dcc_sram_read(struct file *file, char __user *data, + size_t len, loff_t *ppos) +{ + unsigned char *buf; + struct dcc_drvdata *drvdata; + + drvdata = container_of(file->private_data, struct dcc_drvdata, + sram_dev); + + /* EOF check */ + if (*ppos >= drvdata->ram_size) + return 0; + + if ((*ppos + len) > drvdata->ram_size) + len = (drvdata->ram_size - *ppos); + + buf = kzalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + memcpy_fromio(buf, drvdata->ram_base + *ppos, len); + + if (copy_to_user(data, buf, len)) { + kfree(buf); + return -EFAULT; + } + + *ppos += len; + + kfree(buf); + + return len; +} + +static void dcc_configure_list(struct dcc_drvdata *drvdata, + const struct dcc_config *config) +{ + const struct dcc_link_config *link; + const struct dcc_register_entry *entry; + char *token, *bufp, *buf_orig; + char *delim = " "; + int ret, i, j, configured; + size_t len; + + if (!config || !config->num_lists) + return; + + for (i = 0; i < config->num_lists; i++) { + link = &config->lists[i]; + + if (link->link_list >= drvdata->max_link_list) { + dev_err(drvdata->dev, "Invalid link list index %d\n", link->link_list); + continue; + } + + configured = 0; + + for (j = 0; j < link->num_entries; j++) { + entry = &link->entries[j]; + + len = strlen(entry->config); + buf_orig = kzalloc(len + 1, GFP_KERNEL); + if (!buf_orig) + return; + + strscpy(buf_orig, entry->config, len + 1); + bufp = buf_orig; + + token = strsep(&bufp, delim); + if (!bufp) { + dev_err(drvdata->dev, "Malformed entry: \"%s\"\n", + entry->config); + kfree(buf_orig); + continue; + } + + if (!strcmp("R", token)) { + ret = dcc_config_add_read(drvdata, bufp, link->link_list); + } else if (!strcmp("W", token)) { + ret = dcc_config_add_write(drvdata, bufp, link->link_list); + } else if (!strcmp("RW", token)) { + ret = dcc_config_add_read_write(drvdata, bufp, link->link_list); + } else if (!strcmp("L", token)) { + ret = dcc_config_add_loop(drvdata, bufp, link->link_list); + } else { + dev_err(drvdata->dev, "%s is not a correct input\n", token); + ret = -EINVAL; + } + + if (ret >= 0) + configured++; + + kfree(buf_orig); + } + + if (configured) + dcc_enable(drvdata, link->link_list); + } +} + +static const struct file_operations dcc_sram_fops = { + .owner = THIS_MODULE, + .read = dcc_sram_read, +}; + +static int dcc_sram_dev_init(struct dcc_drvdata *drvdata) +{ + drvdata->sram_dev.minor = MISC_DYNAMIC_MINOR; + drvdata->sram_dev.name = "dcc_sram"; + drvdata->sram_dev.fops = &dcc_sram_fops; + + return misc_register(&drvdata->sram_dev); +} + +static void dcc_sram_dev_exit(struct dcc_drvdata *drvdata) +{ + misc_deregister(&drvdata->sram_dev); +} + +static int dcc_probe(struct platform_device *pdev) +{ + u32 val; + int ret = 0, i; + struct device *dev = &pdev->dev; + struct dcc_drvdata *drvdata; + const struct dcc_pdata *pdata = dev_get_platdata(dev); + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + drvdata->dev = &pdev->dev; + platform_set_drvdata(pdev, drvdata); + + drvdata->base = devm_ioremap(dev, pdata->base, pdata->size); + if (IS_ERR(drvdata->base)) + return PTR_ERR(drvdata->base); + + drvdata->ram_base = devm_ioremap(dev, pdata->ram_base, pdata->ram_size); + if (IS_ERR(drvdata->ram_base)) + return PTR_ERR(drvdata->ram_base); + + drvdata->ram_size = pdata->ram_size; + drvdata->ram_offset = pdata->dcc_offset; + drvdata->mem_map_ver = pdata->map_ver; + + switch (drvdata->mem_map_ver) { + case MEM_MAP_VER3: + case MEM_MAP_VER2: + drvdata->max_link_list = readl(drvdata->base + DCC_LL_NUM_INFO); + if (!drvdata->max_link_list) + return -EINVAL; + break; + case MEM_MAP_VER1: + drvdata->max_link_list = DCC_MAX_LINK_LIST; + break; + default: + dev_err(drvdata->dev, "Unsupported memory map version.\n"); + return -EINVAL; + } + + val = readl(drvdata->base + DCC_HW_INFO); + /* Either set the fixed loop offset or calculate + * it from the total number of words in dcc_sram. + * Max consecutive addresses dcc can loop is + * equivalent to the words in dcc_sram. + */ + if (val & DCC_LOOP_OFFSET_MASK) + drvdata->loop_shift = DCC_FIX_LOOP_OFFSET; + else + drvdata->loop_shift = get_bitmask_order((drvdata->ram_offset + + drvdata->ram_size) / DCC_SRAM_WORD_LENGTH - 1); + + mutex_init(&drvdata->mutex); + + drvdata->enable_bitmap = devm_kcalloc(dev, BITS_TO_LONGS(drvdata->max_link_list), + sizeof(*drvdata->enable_bitmap), GFP_KERNEL); + if (!drvdata->enable_bitmap) + return -ENOMEM; + + drvdata->cfg_head = devm_kcalloc(dev, drvdata->max_link_list, + sizeof(*drvdata->cfg_head), GFP_KERNEL); + if (!drvdata->cfg_head) + return -ENOMEM; + + for (i = 0; i < drvdata->max_link_list; i++) + INIT_LIST_HEAD(&drvdata->cfg_head[i]); + + ret = dcc_sram_dev_init(drvdata); + if (ret) { + dev_err(drvdata->dev, "DCC: sram node not registered.\n"); + return ret; + } + + dcc_create_debug_dir(drvdata); + dcc_configure_list(drvdata, pdata->config); + + return 0; +} + +static void dcc_remove(struct platform_device *pdev) +{ + struct dcc_drvdata *drvdata = platform_get_drvdata(pdev); + + dcc_delete_debug_dir(drvdata); + dcc_sram_dev_exit(drvdata); + dcc_config_reset(drvdata); +} + +static struct platform_driver dcc_driver = { + .probe = dcc_probe, + .remove = dcc_remove, + .driver = { + .name = "qcom-dcc", + }, +}; + +module_platform_driver(dcc_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Qualcomm Technologies Inc. DCC driver"); diff --git a/drivers/misc/qcom-dcc.h b/drivers/misc/qcom-dcc.h new file mode 100644 index 0000000000000..ce8551141062d --- /dev/null +++ b/drivers/misc/qcom-dcc.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _QCOM_DCC_H +#define _QCOM_DCC_H + +#include + +struct dcc_register_entry { + const char *config; +}; + +struct dcc_link_config { + int link_list; + const struct dcc_register_entry *entries; + int num_entries; +}; + +struct dcc_config { + const struct dcc_link_config *lists; + int num_lists; +}; + +struct dcc_pdata { + phys_addr_t base; + resource_size_t size; + phys_addr_t ram_base; + resource_size_t ram_size; + u32 dcc_offset; + u8 map_ver; + const struct dcc_config *config; +}; + +#endif diff --git a/drivers/misc/rp1/rp1_pci.c b/drivers/misc/rp1/rp1_pci.c index 803832006ec87..a342bcc6164bb 100644 --- a/drivers/misc/rp1/rp1_pci.c +++ b/drivers/misc/rp1/rp1_pci.c @@ -289,6 +289,9 @@ static int rp1_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_unload_overlay; } + if (skip_ovl) + of_node_put(rp1_node); + return 0; err_unload_overlay: diff --git a/drivers/misc/ti_fpc202.c b/drivers/misc/ti_fpc202.c index 7964e46c74482..8eb2b5ac98506 100644 --- a/drivers/misc/ti_fpc202.c +++ b/drivers/misc/ti_fpc202.c @@ -309,7 +309,6 @@ static void fpc202_remove_port(struct fpc202_priv *priv, int port_id) static int fpc202_probe(struct i2c_client *client) { struct device *dev = &client->dev; - struct device_node *i2c_handle; struct fpc202_priv *priv; int ret, port_id; @@ -357,7 +356,7 @@ static int fpc202_probe(struct i2c_client *client) bitmap_zero(priv->probed_ports, FPC202_NUM_PORTS); - for_each_child_of_node(dev->of_node, i2c_handle) { + for_each_child_of_node_scoped(dev->of_node, i2c_handle) { ret = of_property_read_u32(i2c_handle, "reg", &port_id); if (ret) { if (ret == -EINVAL) diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c index 42e7d2a2a90c6..6d71355528d34 100644 --- a/drivers/misc/uacce/uacce.c +++ b/drivers/misc/uacce/uacce.c @@ -40,20 +40,34 @@ static int uacce_start_queue(struct uacce_queue *q) return 0; } -static int uacce_put_queue(struct uacce_queue *q) +static int uacce_stop_queue(struct uacce_queue *q) { struct uacce_device *uacce = q->uacce; - if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue) + if (q->state != UACCE_Q_STARTED) + return 0; + + if (uacce->ops->stop_queue) uacce->ops->stop_queue(q); - if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) && - uacce->ops->put_queue) + q->state = UACCE_Q_INIT; + + return 0; +} + +static void uacce_put_queue(struct uacce_queue *q) +{ + struct uacce_device *uacce = q->uacce; + + uacce_stop_queue(q); + + if (q->state != UACCE_Q_INIT) + return; + + if (uacce->ops->put_queue) uacce->ops->put_queue(q); q->state = UACCE_Q_ZOMBIE; - - return 0; } static long uacce_fops_unl_ioctl(struct file *filep, @@ -80,7 +94,7 @@ static long uacce_fops_unl_ioctl(struct file *filep, ret = uacce_start_queue(q); break; case UACCE_CMD_PUT_Q: - ret = uacce_put_queue(q); + ret = uacce_stop_queue(q); break; default: if (uacce->ops->ioctl) @@ -214,8 +228,14 @@ static void uacce_vma_close(struct vm_area_struct *vma) } } +static int uacce_vma_mremap(struct vm_area_struct *area) +{ + return -EPERM; +} + static const struct vm_operations_struct uacce_vm_ops = { .close = uacce_vma_close, + .mremap = uacce_vma_mremap, }; static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) @@ -382,6 +402,9 @@ static ssize_t isolate_strategy_show(struct device *dev, struct device_attribute struct uacce_device *uacce = to_uacce_device(dev); u32 val; + if (!uacce->ops->isolate_err_threshold_read) + return -ENOENT; + val = uacce->ops->isolate_err_threshold_read(uacce); return sysfs_emit(buf, "%u\n", val); @@ -394,6 +417,9 @@ static ssize_t isolate_strategy_store(struct device *dev, struct device_attribut unsigned long val; int ret; + if (!uacce->ops->isolate_err_threshold_write) + return -ENOENT; + if (kstrtoul(buf, 0, &val) < 0) return -EINVAL; @@ -519,6 +545,8 @@ EXPORT_SYMBOL_GPL(uacce_alloc); */ int uacce_register(struct uacce_device *uacce) { + int ret; + if (!uacce) return -ENODEV; @@ -529,7 +557,11 @@ int uacce_register(struct uacce_device *uacce) uacce->cdev->ops = &uacce_fops; uacce->cdev->owner = THIS_MODULE; - return cdev_device_add(uacce->cdev, &uacce->dev); + ret = cdev_device_add(uacce->cdev, &uacce->dev); + if (ret) + uacce->cdev = NULL; + + return ret; } EXPORT_SYMBOL_GPL(uacce_register); diff --git a/drivers/mmc/core/crypto.c b/drivers/mmc/core/crypto.c index fec4fbf16a5b6..a5a90bfc634ee 100644 --- a/drivers/mmc/core/crypto.c +++ b/drivers/mmc/core/crypto.c @@ -15,7 +15,7 @@ void mmc_crypto_set_initial_state(struct mmc_host *host) { /* Reset might clear all keys, so reprogram all the keys. */ - if (host->caps2 & MMC_CAP2_CRYPTO) + if ((host->caps2 & MMC_CAP2_CRYPTO) && !(host->caps2 & MMC_CAP2_CRYPTO_NO_REPROG)) blk_crypto_reprogram_all_keys(&host->crypto_profile); } diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 10d0ef58ef493..c94ae4794545d 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -315,14 +315,14 @@ config MMC_SDHCI_ESDHC_MCF config MMC_SDHCI_ESDHC_IMX tristate "SDHCI support for the Freescale eSDHC/uSDHC i.MX controller" - depends on ARCH_MXC || COMPILE_TEST + depends on ARCH_MXC || ARCH_S32 || COMPILE_TEST depends on MMC_SDHCI_PLTFM depends on OF select MMC_SDHCI_IO_ACCESSORS select MMC_CQHCI help This selects the Freescale eSDHC/uSDHC controller support - found on i.MX25, i.MX35 i.MX5x and i.MX6x. + found on i.MX25, i.MX35, i.MX5x, i.MX6x, and S32G. If you have a controller with this interface, say Y or M here. diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c index 681354942e974..ff6a52d85e520 100644 --- a/drivers/mmc/host/dw_mmc-rockchip.c +++ b/drivers/mmc/host/dw_mmc-rockchip.c @@ -19,6 +19,8 @@ #define RK3288_CLKGEN_DIV 2 #define SDMMC_TIMING_CON0 0x130 #define SDMMC_TIMING_CON1 0x134 +#define SDMMC_MISC_CON 0x138 +#define MEM_CLK_AUTOGATE_ENABLE BIT(5) #define ROCKCHIP_MMC_DELAY_SEL BIT(10) #define ROCKCHIP_MMC_DEGREE_MASK 0x3 #define ROCKCHIP_MMC_DEGREE_OFFSET 1 @@ -34,6 +36,8 @@ struct dw_mci_rockchip_priv_data { int default_sample_phase; int num_phases; bool internal_phase; + int sample_phase; + int drv_phase; }; /* @@ -470,6 +474,7 @@ static int dw_mci_rk3576_parse_dt(struct dw_mci *host) static int dw_mci_rockchip_init(struct dw_mci *host) { + struct dw_mci_rockchip_priv_data *priv = host->priv; int ret, i; /* It is slot 8 on Rockchip SoCs */ @@ -494,6 +499,9 @@ static int dw_mci_rockchip_init(struct dw_mci *host) dev_warn(host->dev, "no valid minimum freq: %d\n", ret); } + if (priv->internal_phase) + mci_writel(host, MISC_CON, MEM_CLK_AUTOGATE_ENABLE); + return 0; } @@ -568,9 +576,43 @@ static void dw_mci_rockchip_remove(struct platform_device *pdev) dw_mci_pltfm_remove(pdev); } +static int dw_mci_rockchip_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct dw_mci *host = platform_get_drvdata(pdev); + struct dw_mci_rockchip_priv_data *priv = host->priv; + + if (priv->internal_phase) { + priv->sample_phase = rockchip_mmc_get_phase(host, true); + priv->drv_phase = rockchip_mmc_get_phase(host, false); + } + + return dw_mci_runtime_suspend(dev); +} + +static int dw_mci_rockchip_runtime_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct dw_mci *host = platform_get_drvdata(pdev); + struct dw_mci_rockchip_priv_data *priv = host->priv; + int ret; + + ret = dw_mci_runtime_resume(dev); + if (ret) + return ret; + + if (priv->internal_phase) { + rockchip_mmc_set_phase(host, true, priv->sample_phase); + rockchip_mmc_set_phase(host, false, priv->drv_phase); + mci_writel(host, MISC_CON, MEM_CLK_AUTOGATE_ENABLE); + } + + return ret; +} + static const struct dev_pm_ops dw_mci_rockchip_dev_pm_ops = { SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) - RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL) + RUNTIME_PM_OPS(dw_mci_rockchip_runtime_suspend, dw_mci_rockchip_runtime_resume, NULL) }; static struct platform_driver dw_mci_rockchip_pltfm_driver = { diff --git a/drivers/mmc/host/mmci_qcom_dml.c b/drivers/mmc/host/mmci_qcom_dml.c index 3da6112fbe39d..67371389cc331 100644 --- a/drivers/mmc/host/mmci_qcom_dml.c +++ b/drivers/mmc/host/mmci_qcom_dml.c @@ -109,6 +109,7 @@ static int of_get_dml_pipe_index(struct device_node *np, const char *name) &dma_spec)) return -ENODEV; + of_node_put(dma_spec.np); if (dma_spec.args_count) return dma_spec.args[0]; diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c index dc2587ff8519f..4db3328f46dfb 100644 --- a/drivers/mmc/host/rtsx_pci_sdmmc.c +++ b/drivers/mmc/host/rtsx_pci_sdmmc.c @@ -1306,6 +1306,46 @@ static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios) return err; } +static int sdmmc_card_busy(struct mmc_host *mmc) +{ + struct realtek_pci_sdmmc *host = mmc_priv(mmc); + struct rtsx_pcr *pcr = host->pcr; + int err; + u8 stat; + u8 mask = SD_DAT3_STATUS | SD_DAT2_STATUS | SD_DAT1_STATUS + | SD_DAT0_STATUS; + + mutex_lock(&pcr->pcr_mutex); + + rtsx_pci_start_run(pcr); + + err = rtsx_pci_write_register(pcr, SD_BUS_STAT, + SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, + SD_CLK_TOGGLE_EN); + if (err) + goto out; + + mdelay(1); + + err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat); + if (err) + goto out; + + err = rtsx_pci_write_register(pcr, SD_BUS_STAT, + SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0); +out: + mutex_unlock(&pcr->pcr_mutex); + + if (err) + return err; + + /* check if any pin between dat[0:3] is low */ + if ((stat & mask) != mask) + return 1; + else + return 0; +} + static int sdmmc_execute_tuning(struct mmc_host *mmc, u32 opcode) { struct realtek_pci_sdmmc *host = mmc_priv(mmc); @@ -1418,6 +1458,7 @@ static const struct mmc_host_ops realtek_pci_sdmmc_ops = { .get_ro = sdmmc_get_ro, .get_cd = sdmmc_get_cd, .start_signal_voltage_switch = sdmmc_switch_voltage, + .card_busy = sdmmc_card_busy, .execute_tuning = sdmmc_execute_tuning, .init_sd_express = sdmmc_init_sd_express, }; diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c index 4e5edbf2fc9b6..111a437244d6b 100644 --- a/drivers/mmc/host/sdhci-msm.c +++ b/drivers/mmc/host/sdhci-msm.c @@ -344,41 +344,43 @@ static void sdhci_msm_v5_variant_writel_relaxed(u32 val, writel_relaxed(val, host->ioaddr + offset); } -static unsigned int msm_get_clock_mult_for_bus_mode(struct sdhci_host *host) +static unsigned int msm_get_clock_mult_for_bus_mode(struct sdhci_host *host, + unsigned int clock, + unsigned int timing) { - struct mmc_ios ios = host->mmc->ios; /* * The SDHC requires internal clock frequency to be double the * actual clock that will be set for DDR mode. The controller * uses the faster clock(100/400MHz) for some of its parts and * send the actual required clock (50/200MHz) to the card. */ - if (ios.timing == MMC_TIMING_UHS_DDR50 || - ios.timing == MMC_TIMING_MMC_DDR52 || - ios.timing == MMC_TIMING_MMC_HS400 || + if (timing == MMC_TIMING_UHS_DDR50 || + timing == MMC_TIMING_MMC_DDR52 || + (timing == MMC_TIMING_MMC_HS400 && + clock == MMC_HS200_MAX_DTR) || host->flags & SDHCI_HS400_TUNING) return 2; return 1; } static void msm_set_clock_rate_for_bus_mode(struct sdhci_host *host, - unsigned int clock) + unsigned int clock, + unsigned int timing) { struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); - struct mmc_ios curr_ios = host->mmc->ios; struct clk *core_clk = msm_host->bulk_clks[0].clk; unsigned long achieved_rate; unsigned int desired_rate; unsigned int mult; int rc; - mult = msm_get_clock_mult_for_bus_mode(host); + mult = msm_get_clock_mult_for_bus_mode(host, clock, timing); desired_rate = clock * mult; rc = dev_pm_opp_set_rate(mmc_dev(host->mmc), desired_rate); if (rc) { pr_err("%s: Failed to set clock at rate %u at timing %d\n", - mmc_hostname(host->mmc), desired_rate, curr_ios.timing); + mmc_hostname(host->mmc), desired_rate, timing); return; } @@ -397,7 +399,7 @@ static void msm_set_clock_rate_for_bus_mode(struct sdhci_host *host, msm_host->clk_rate = desired_rate; pr_debug("%s: Setting clock at rate %lu at timing %d\n", - mmc_hostname(host->mmc), achieved_rate, curr_ios.timing); + mmc_hostname(host->mmc), achieved_rate, timing); } /* Platform specific tuning */ @@ -1239,7 +1241,7 @@ static int sdhci_msm_execute_tuning(struct mmc_host *mmc, u32 opcode) */ if (host->flags & SDHCI_HS400_TUNING) { sdhci_msm_hc_select_mode(host); - msm_set_clock_rate_for_bus_mode(host, ios.clock); + msm_set_clock_rate_for_bus_mode(host, ios.clock, ios.timing); host->flags &= ~SDHCI_HS400_TUNING; } @@ -1864,6 +1866,7 @@ static void sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock) { struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); + struct mmc_ios ios = host->mmc->ios; if (!clock) { host->mmc->actual_clock = msm_host->clk_rate = 0; @@ -1872,7 +1875,7 @@ static void sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock) sdhci_msm_hc_select_mode(host); - msm_set_clock_rate_for_bus_mode(host, clock); + msm_set_clock_rate_for_bus_mode(host, ios.clock, ios.timing); out: __sdhci_msm_set_clock(host, clock); } @@ -1886,6 +1889,8 @@ static void sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock) #ifdef CONFIG_MMC_CRYPTO static const struct blk_crypto_ll_ops sdhci_msm_crypto_ops; /* forward decl */ +static int sdhci_msm_ice_scale_clk(struct sdhci_msm_host *msm_host, unsigned long target_freq, + bool round_ceil); /* forward decl */ static int sdhci_msm_ice_init(struct sdhci_msm_host *msm_host, struct cqhci_host *cq_host) @@ -1911,11 +1916,6 @@ static int sdhci_msm_ice_init(struct sdhci_msm_host *msm_host, if (IS_ERR_OR_NULL(ice)) return PTR_ERR_OR_ZERO(ice); - if (qcom_ice_get_supported_key_type(ice) != BLK_CRYPTO_KEY_TYPE_RAW) { - dev_warn(dev, "Wrapped keys not supported. Disabling inline encryption support.\n"); - return 0; - } - msm_host->ice = ice; /* Initialize the blk_crypto_profile */ @@ -1929,7 +1929,7 @@ static int sdhci_msm_ice_init(struct sdhci_msm_host *msm_host, profile->ll_ops = sdhci_msm_crypto_ops; profile->max_dun_bytes_supported = 4; - profile->key_types_supported = BLK_CRYPTO_KEY_TYPE_RAW; + profile->key_types_supported = qcom_ice_get_supported_key_type(ice); profile->dev = dev; /* @@ -1949,6 +1949,12 @@ static int sdhci_msm_ice_init(struct sdhci_msm_host *msm_host, } mmc->caps2 |= MMC_CAP2_CRYPTO; + mmc->caps2 |= MMC_CAP2_CRYPTO_NO_REPROG; + + err = sdhci_msm_ice_scale_clk(msm_host, INT_MAX, false); + if (err && err != -EOPNOTSUPP) + dev_warn(dev, "Unable to boost ICE clock to TURBO\n"); + return 0; } @@ -1974,6 +1980,16 @@ static int sdhci_msm_ice_suspend(struct sdhci_msm_host *msm_host) return 0; } +static int sdhci_msm_ice_scale_clk(struct sdhci_msm_host *msm_host, + unsigned long target_freq, + bool round_ceil) +{ + if (msm_host->mmc->caps2 & MMC_CAP2_CRYPTO) + return qcom_ice_scale_clk(msm_host->ice, target_freq, round_ceil); + + return 0; +} + static inline struct sdhci_msm_host * sdhci_msm_host_from_crypto_profile(struct blk_crypto_profile *profile) { @@ -2009,9 +2025,49 @@ static int sdhci_msm_ice_keyslot_evict(struct blk_crypto_profile *profile, return qcom_ice_evict_key(msm_host->ice, slot); } +static int sdhci_msm_ice_derive_sw_secret(struct blk_crypto_profile *profile, + const u8 *eph_key, size_t eph_key_size, + u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]) +{ + struct sdhci_msm_host *msm_host = sdhci_msm_host_from_crypto_profile(profile); + + return qcom_ice_derive_sw_secret(msm_host->ice, eph_key, eph_key_size, + sw_secret); +} + +static int sdhci_msm_ice_import_key(struct blk_crypto_profile *profile, + const u8 *raw_key, size_t raw_key_size, + u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) +{ + struct sdhci_msm_host *msm_host = sdhci_msm_host_from_crypto_profile(profile); + + return qcom_ice_import_key(msm_host->ice, raw_key, raw_key_size, lt_key); +} + +static int sdhci_msm_ice_generate_key(struct blk_crypto_profile *profile, + u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) +{ + struct sdhci_msm_host *msm_host = sdhci_msm_host_from_crypto_profile(profile); + + return qcom_ice_generate_key(msm_host->ice, lt_key); +} + +static int sdhci_msm_ice_prepare_key(struct blk_crypto_profile *profile, + const u8 *lt_key, size_t lt_key_size, + u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) +{ + struct sdhci_msm_host *msm_host = sdhci_msm_host_from_crypto_profile(profile); + + return qcom_ice_prepare_key(msm_host->ice, lt_key, lt_key_size, eph_key); +} + static const struct blk_crypto_ll_ops sdhci_msm_crypto_ops = { .keyslot_program = sdhci_msm_ice_keyslot_program, .keyslot_evict = sdhci_msm_ice_keyslot_evict, + .derive_sw_secret = sdhci_msm_ice_derive_sw_secret, + .import_key = sdhci_msm_ice_import_key, + .generate_key = sdhci_msm_ice_generate_key, + .prepare_key = sdhci_msm_ice_prepare_key, }; #else /* CONFIG_MMC_CRYPTO */ @@ -2037,6 +2093,13 @@ sdhci_msm_ice_suspend(struct sdhci_msm_host *msm_host) { return 0; } + +static inline int +sdhci_msm_ice_scale_clk(struct sdhci_msm_host *msm_host, unsigned long target_freq, + bool round_ceil) +{ + return 0; +} #endif /* !CONFIG_MMC_CRYPTO */ /*****************************************************************************\ @@ -2526,6 +2589,11 @@ static int sdhci_msm_gcc_reset(struct device *dev, struct sdhci_host *host) usleep_range(200, 210); reset_control_put(reset); +#ifdef CONFIG_MMC_CRYPTO + if (host->mmc->caps2 & MMC_CAP2_CRYPTO) + blk_crypto_reprogram_all_keys(&host->mmc->crypto_profile); +#endif + return ret; } diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c index c6f09b53325d2..0a4b4696bc01c 100644 --- a/drivers/mmc/host/sdhci-of-arasan.c +++ b/drivers/mmc/host/sdhci-of-arasan.c @@ -99,7 +99,7 @@ #define HIWORD_UPDATE(val, mask, shift) \ ((val) << (shift) | (mask) << ((shift) + 16)) -#define CD_STABLE_TIMEOUT_US 1000000 +#define CD_STABLE_TIMEOUT_US 2000000 #define CD_STABLE_MAX_SLEEP_US 10 /** diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index 4e256673a0983..5ae4e11b1e90c 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -608,6 +608,13 @@ static void dwcmshc_rk3568_set_clock(struct sdhci_host *host, unsigned int clock sdhci_writel(host, extra, reg); if (clock <= 52000000) { + if (host->mmc->ios.timing == MMC_TIMING_MMC_HS200 || + host->mmc->ios.timing == MMC_TIMING_MMC_HS400) { + dev_err(mmc_dev(host->mmc), + "Can't reduce the clock below 52MHz in HS200/HS400 mode"); + return; + } + /* * Disable DLL and reset both of sample and drive clock. * The bypass bit and start bit need to be set if DLL is not locked. diff --git a/drivers/mmc/host/sdhci-pci-gli.c b/drivers/mmc/host/sdhci-pci-gli.c index b0f91cc9e40e4..6e4084407662a 100644 --- a/drivers/mmc/host/sdhci-pci-gli.c +++ b/drivers/mmc/host/sdhci-pci-gli.c @@ -68,6 +68,9 @@ #define GLI_9750_MISC_TX1_DLY_VALUE 0x5 #define SDHCI_GLI_9750_MISC_SSC_OFF BIT(26) +#define SDHCI_GLI_9750_GM_BURST_SIZE 0x510 +#define SDHCI_GLI_9750_GM_BURST_SIZE_R_OSRC_LMT GENMASK(17, 16) + #define SDHCI_GLI_9750_TUNING_CONTROL 0x540 #define SDHCI_GLI_9750_TUNING_CONTROL_EN BIT(4) #define GLI_9750_TUNING_CONTROL_EN_ON 0x1 @@ -345,10 +348,16 @@ static void gli_set_9750(struct sdhci_host *host) u32 misc_value; u32 parameter_value; u32 control_value; + u32 burst_value; u16 ctrl2; gl9750_wt_on(host); + /* clear R_OSRC_Lmt to avoid DMA write corruption */ + burst_value = sdhci_readl(host, SDHCI_GLI_9750_GM_BURST_SIZE); + burst_value &= ~SDHCI_GLI_9750_GM_BURST_SIZE_R_OSRC_LMT; + sdhci_writel(host, burst_value, SDHCI_GLI_9750_GM_BURST_SIZE); + driving_value = sdhci_readl(host, SDHCI_GLI_9750_DRIVING); pll_value = sdhci_readl(host, SDHCI_GLI_9750_PLL); sw_ctrl_value = sdhci_readl(host, SDHCI_GLI_9750_SW_CTRL); diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index ac7e11f37af71..fec9329e1edbe 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -4532,8 +4532,15 @@ int sdhci_setup_host(struct sdhci_host *host) * their platform code before calling sdhci_add_host(), and we * won't assume 8-bit width for hosts without that CAP. */ - if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA)) + if (host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA) { + host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 | SDHCI_SUPPORT_DDR50); + if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400) + host->caps1 &= ~SDHCI_SUPPORT_HS400; + mmc->caps2 &= ~(MMC_CAP2_HS200 | MMC_CAP2_HS400 | MMC_CAP2_HS400_ES); + mmc->caps &= ~(MMC_CAP_DDR | MMC_CAP_UHS); + } else { mmc->caps |= MMC_CAP_4_BIT_DATA; + } if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23) mmc->caps &= ~MMC_CAP_CMD23; diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c index ff49d0770506f..3c9df27f9fa74 100644 --- a/drivers/mmc/host/vub300.c +++ b/drivers/mmc/host/vub300.c @@ -369,11 +369,14 @@ struct vub300_mmc_host { static void vub300_delete(struct kref *kref) { /* kref callback - softirq */ struct vub300_mmc_host *vub300 = kref_to_vub300_mmc_host(kref); + struct mmc_host *mmc = vub300->mmc; + usb_free_urb(vub300->command_out_urb); vub300->command_out_urb = NULL; usb_free_urb(vub300->command_res_urb); vub300->command_res_urb = NULL; usb_put_dev(vub300->udev); + mmc_free_host(mmc); /* * and hence also frees vub300 * which is contained at the end of struct mmc @@ -2112,7 +2115,7 @@ static int vub300_probe(struct usb_interface *interface, goto error1; } /* this also allocates memory for our VUB300 mmc host device */ - mmc = devm_mmc_alloc_host(&udev->dev, sizeof(*vub300)); + mmc = mmc_alloc_host(sizeof(*vub300), &udev->dev); if (!mmc) { retval = -ENOMEM; dev_err(&udev->dev, "not enough memory for the mmc_host\n"); @@ -2269,7 +2272,7 @@ static int vub300_probe(struct usb_interface *interface, dev_err(&vub300->udev->dev, "Could not find two sets of bulk-in/out endpoint pairs\n"); retval = -EINVAL; - goto error4; + goto err_free_host; } retval = usb_control_msg(vub300->udev, usb_rcvctrlpipe(vub300->udev, 0), @@ -2278,14 +2281,14 @@ static int vub300_probe(struct usb_interface *interface, 0x0000, 0x0000, &vub300->hc_info, sizeof(vub300->hc_info), 1000); if (retval < 0) - goto error4; + goto err_free_host; retval = usb_control_msg(vub300->udev, usb_sndctrlpipe(vub300->udev, 0), SET_ROM_WAIT_STATES, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, firmware_rom_wait_states, 0x0000, NULL, 0, 1000); if (retval < 0) - goto error4; + goto err_free_host; dev_info(&vub300->udev->dev, "operating_mode = %s %s %d MHz %s %d byte USB packets\n", (mmc->caps & MMC_CAP_SDIO_IRQ) ? "IRQs" : "POLL", @@ -2300,7 +2303,7 @@ static int vub300_probe(struct usb_interface *interface, 0x0000, 0x0000, &vub300->system_port_status, sizeof(vub300->system_port_status), 1000); if (retval < 0) { - goto error4; + goto err_free_host; } else if (sizeof(vub300->system_port_status) == retval) { vub300->card_present = (0x0001 & vub300->system_port_status.port_flags) ? 1 : 0; @@ -2308,7 +2311,7 @@ static int vub300_probe(struct usb_interface *interface, (0x0010 & vub300->system_port_status.port_flags) ? 1 : 0; } else { retval = -EINVAL; - goto error4; + goto err_free_host; } usb_set_intfdata(interface, vub300); INIT_DELAYED_WORK(&vub300->pollwork, vub300_pollwork_thread); @@ -2338,6 +2341,8 @@ static int vub300_probe(struct usb_interface *interface, return 0; error6: timer_delete_sync(&vub300->inactivity_timer); +err_free_host: + mmc_free_host(mmc); /* * and hence also frees vub300 * which is contained at the end of struct mmc @@ -2365,8 +2370,8 @@ static void vub300_disconnect(struct usb_interface *interface) usb_set_intfdata(interface, NULL); /* prevent more I/O from starting */ vub300->interface = NULL; - kref_put(&vub300->kref, vub300_delete); mmc_remove_host(mmc); + kref_put(&vub300->kref, vub300_delete); pr_info("USB vub300 remote SDIO host controller[%d]" " now disconnected", ifnum); return; diff --git a/drivers/most/core.c b/drivers/most/core.c index da319d108ea1d..40d63e38fef54 100644 --- a/drivers/most/core.c +++ b/drivers/most/core.c @@ -1282,19 +1282,28 @@ int most_register_interface(struct most_interface *iface) int id; struct most_channel *c; - if (!iface || !iface->enqueue || !iface->configure || - !iface->poison_channel || (iface->num_channels > MAX_CHANNELS)) + if (!iface) return -EINVAL; + device_initialize(iface->dev); + + if (!iface->enqueue || !iface->configure || !iface->poison_channel || + (iface->num_channels > MAX_CHANNELS)) { + put_device(iface->dev); + return -EINVAL; + } + id = ida_alloc(&mdev_id, GFP_KERNEL); if (id < 0) { dev_err(iface->dev, "Failed to allocate device ID\n"); + put_device(iface->dev); return id; } iface->p = kzalloc(sizeof(*iface->p), GFP_KERNEL); if (!iface->p) { ida_free(&mdev_id, id); + put_device(iface->dev); return -ENOMEM; } @@ -1304,7 +1313,7 @@ int most_register_interface(struct most_interface *iface) iface->dev->bus = &mostbus; iface->dev->groups = interface_attr_groups; dev_set_drvdata(iface->dev, iface); - if (device_register(iface->dev)) { + if (device_add(iface->dev)) { dev_err(iface->dev, "Failed to register interface device\n"); kfree(iface->p); put_device(iface->dev); diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index b438ee5aacc34..114e69135b8d9 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -738,6 +738,7 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, kref_init(&nvm->refcnt); mutex_init(&nvm->lock); + nvm->nregions = nregions; for (n = 0, i = 0; i < INTEL_DG_NVM_REGIONS; i++) { if (!invm->regions[i].name) @@ -745,13 +746,15 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, char *name = kasprintf(GFP_KERNEL, "%s.%s", dev_name(&aux_dev->dev), invm->regions[i].name); - if (!name) - continue; + if (!name) { + ret = -ENOMEM; + goto err; + } + nvm->regions[n].name = name; nvm->regions[n].id = i; n++; } - nvm->nregions = n; /* in case where kasprintf fail */ nvm->base = devm_ioremap_resource(device, &invm->bar); if (IS_ERR(nvm->base)) { diff --git a/drivers/mtd/lpddr/lpddr_cmds.c b/drivers/mtd/lpddr/lpddr_cmds.c index 290fd0119e984..cd37d58abacb7 100644 --- a/drivers/mtd/lpddr/lpddr_cmds.c +++ b/drivers/mtd/lpddr/lpddr_cmds.c @@ -79,7 +79,7 @@ struct mtd_info *lpddr_cmdset(struct map_info *map) mutex_init(&shared[i].lock); for (j = 0; j < lpddr->qinfo->HWPartsNum; j++) { *chip = lpddr->chips[i]; - chip->start += j << lpddr->chipshift; + chip->start += (unsigned long)j << lpddr->chipshift; chip->oldstate = chip->state = FL_READY; chip->priv = &shared[i]; /* those should be reset too since @@ -559,7 +559,7 @@ static int lpddr_point(struct mtd_info *mtd, loff_t adr, size_t len, break; if ((len + ofs - 1) >> lpddr->chipshift) - thislen = (1<chipshift) - ofs; + thislen = (1UL << lpddr->chipshift) - ofs; else thislen = len; /* get the chip */ @@ -575,7 +575,7 @@ static int lpddr_point(struct mtd_info *mtd, loff_t adr, size_t len, len -= thislen; ofs = 0; - last_end += 1 << lpddr->chipshift; + last_end += 1UL << lpddr->chipshift; chipnum++; chip = &lpddr->chips[chipnum]; } @@ -601,7 +601,7 @@ static int lpddr_unpoint (struct mtd_info *mtd, loff_t adr, size_t len) break; if ((len + ofs - 1) >> lpddr->chipshift) - thislen = (1<chipshift) - ofs; + thislen = (1UL << lpddr->chipshift) - ofs; else thislen = len; diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c index 994e8c51e6743..2876501a78145 100644 --- a/drivers/mtd/mtdpart.c +++ b/drivers/mtd/mtdpart.c @@ -425,9 +425,12 @@ int add_mtd_partitions(struct mtd_info *parent, mtd_add_partition_attrs(child); - /* Look for subpartitions */ + /* Look for subpartitions (skip if no maching parser found) */ ret = parse_mtd_partitions(child, parts[i].types, NULL); - if (ret < 0) { + if (ret < 0 && ret == -ENOENT) { + pr_debug("Skip parsing subpartitions: %d\n", ret); + continue; + } else if (ret < 0) { pr_err("Failed to parse subpartitions: %d\n", ret); goto err_del_partitions; } diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c index 835653bdd5abc..8f4d001377a1c 100644 --- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c +++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c @@ -2350,14 +2350,12 @@ static int brcmnand_write(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0; i < ctrl->max_oob; i += 4) oob_reg_write(ctrl, i, 0xffffffff); - if (mtd->oops_panic_write) + if (mtd->oops_panic_write) { /* switch to interrupt polling and PIO mode */ disable_ctrl_irqs(ctrl); - - if (use_dma(ctrl) && (has_edu(ctrl) || !oob) && flash_dma_buf_ok(buf)) { + } else if (use_dma(ctrl) && (has_edu(ctrl) || !oob) && flash_dma_buf_ok(buf)) { if (ctrl->dma_trans(host, addr, (u32 *)buf, oob, mtd->writesize, CMD_PROGRAM_PAGE)) - ret = -EIO; goto out; diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index 32ed38b893947..54ed84dfc0be1 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -1015,7 +1015,7 @@ static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl, } /* Send SDMA command and wait for finish. */ -static u32 +static int cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl, u8 thread) { @@ -2878,7 +2878,7 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl) sizeof(*cdns_ctrl->cdma_desc), &cdns_ctrl->dma_cdma_desc, GFP_KERNEL); - if (!cdns_ctrl->dma_cdma_desc) + if (!cdns_ctrl->cdma_desc) return -ENOMEM; cdns_ctrl->buf_size = SZ_16K; diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c index b54d76547ffb2..fea3705a21386 100644 --- a/drivers/mtd/nand/raw/lpc32xx_slc.c +++ b/drivers/mtd/nand/raw/lpc32xx_slc.c @@ -937,6 +937,7 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) dma_release_channel(host->dma_chan); enable_wp: lpc32xx_wp_enable(host); + gpiod_put(host->wp_gpio); return res; } @@ -962,6 +963,7 @@ static void lpc32xx_nand_remove(struct platform_device *pdev) writel(tmp, SLC_CTRL(host->io_base)); lpc32xx_wp_enable(host); + gpiod_put(host->wp_gpio); } static int lpc32xx_nand_resume(struct platform_device *pdev) diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c index 303b3016a070b..38b7eb5b992c8 100644 --- a/drivers/mtd/nand/raw/marvell_nand.c +++ b/drivers/mtd/nand/raw/marvell_nand.c @@ -290,13 +290,16 @@ static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = { MARVELL_LAYOUT( 2048, 512, 4, 1, 1, 2048, 32, 30, 0, 0, 0), MARVELL_LAYOUT( 2048, 512, 8, 2, 1, 1024, 0, 30,1024,32, 30), MARVELL_LAYOUT( 2048, 512, 8, 2, 1, 1024, 0, 30,1024,64, 30), - MARVELL_LAYOUT( 2048, 512, 16, 4, 4, 512, 0, 30, 0, 32, 30), + MARVELL_LAYOUT( 2048, 512, 12, 3, 2, 704, 0, 30,640, 0, 30), + MARVELL_LAYOUT( 2048, 512, 16, 5, 4, 512, 0, 30, 0, 32, 30), MARVELL_LAYOUT( 4096, 512, 4, 2, 2, 2048, 32, 30, 0, 0, 0), - MARVELL_LAYOUT( 4096, 512, 8, 4, 4, 1024, 0, 30, 0, 64, 30), - MARVELL_LAYOUT( 4096, 512, 16, 8, 8, 512, 0, 30, 0, 32, 30), + MARVELL_LAYOUT( 4096, 512, 8, 5, 4, 1024, 0, 30, 0, 64, 30), + MARVELL_LAYOUT( 4096, 512, 12, 6, 5, 704, 0, 30,576, 32, 30), + MARVELL_LAYOUT( 4096, 512, 16, 9, 8, 512, 0, 30, 0, 32, 30), MARVELL_LAYOUT( 8192, 512, 4, 4, 4, 2048, 0, 30, 0, 0, 0), - MARVELL_LAYOUT( 8192, 512, 8, 8, 8, 1024, 0, 30, 0, 160, 30), - MARVELL_LAYOUT( 8192, 512, 16, 16, 16, 512, 0, 30, 0, 32, 30), + MARVELL_LAYOUT( 8192, 512, 8, 9, 8, 1024, 0, 30, 0, 160, 30), + MARVELL_LAYOUT( 8192, 512, 12, 12, 11, 704, 0, 30,448, 64, 30), + MARVELL_LAYOUT( 8192, 512, 16, 17, 16, 512, 0, 30, 0, 32, 30), }; /** diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index c7d9501f646b3..b4bd73d7cd05c 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -4737,11 +4737,16 @@ static void nand_shutdown(struct mtd_info *mtd) static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { struct nand_chip *chip = mtd_to_nand(mtd); + int ret; if (!chip->ops.lock_area) return -ENOTSUPP; - return chip->ops.lock_area(chip, ofs, len); + nand_get_device(chip); + ret = chip->ops.lock_area(chip, ofs, len); + nand_release_device(chip); + + return ret; } /** @@ -4753,11 +4758,16 @@ static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { struct nand_chip *chip = mtd_to_nand(mtd); + int ret; if (!chip->ops.unlock_area) return -ENOTSUPP; - return chip->ops.unlock_area(chip, ofs, len); + nand_get_device(chip); + ret = chip->ops.unlock_area(chip, ofs, len); + nand_release_device(chip); + + return ret; } /* Set default functions */ @@ -6338,11 +6348,14 @@ static int nand_scan_tail(struct nand_chip *chip) ecc->steps = mtd->writesize / ecc->size; if (!base->ecc.ctx.nsteps) base->ecc.ctx.nsteps = ecc->steps; - if (ecc->steps * ecc->size != mtd->writesize) { - WARN(1, "Invalid ECC parameters\n"); - ret = -EINVAL; - goto err_nand_manuf_cleanup; - } + + /* + * Validity check: Warn if ECC parameters are not compatible with page size. + * Due to the custom handling of ECC blocks in certain controllers the check + * may result in an expected failure. + */ + if (ecc->steps * ecc->size != mtd->writesize) + pr_warn("ECC parameters may be invalid in reference to underlying NAND chip\n"); if (!ecc->total) { ecc->total = ecc->steps * ecc->bytes; diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c index 11bd90e3f18cb..50d4305729f4a 100644 --- a/drivers/mtd/nand/raw/pl35x-nand-controller.c +++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c @@ -862,6 +862,9 @@ static int pl35x_nfc_setup_interface(struct nand_chip *chip, int cs, PL35X_SMC_NAND_TAR_CYCLES(tmgs.t_ar) | PL35X_SMC_NAND_TRR_CYCLES(tmgs.t_rr); + writel(plnand->timings, nfc->conf_regs + PL35X_SMC_CYCLES); + pl35x_smc_update_regs(nfc); + return 0; } @@ -976,6 +979,7 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) fallthrough; case NAND_ECC_ENGINE_TYPE_NONE: case NAND_ECC_ENGINE_TYPE_SOFT: + chip->ecc.write_page_raw = nand_monolithic_write_page_raw; break; case NAND_ECC_ENGINE_TYPE_ON_HOST: ret = pl35x_nand_init_hw_ecc_controller(nfc, chip); diff --git a/drivers/mtd/nand/raw/renesas-nand-controller.c b/drivers/mtd/nand/raw/renesas-nand-controller.c index ac8c1b80d7be9..201dd62b99905 100644 --- a/drivers/mtd/nand/raw/renesas-nand-controller.c +++ b/drivers/mtd/nand/raw/renesas-nand-controller.c @@ -1336,7 +1336,10 @@ static int rnandc_probe(struct platform_device *pdev) if (IS_ERR(rnandc->regs)) return PTR_ERR(rnandc->regs); - devm_pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index f92133b8e1a60..697877584a285 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -859,6 +859,14 @@ static void spinand_cont_read_init(struct spinand_device *spinand) (engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE || engine_type == NAND_ECC_ENGINE_TYPE_NONE)) { spinand->cont_read_possible = true; + + /* + * Ensure continuous read is disabled on probe. + * Some devices retain this state across soft reset, + * which leaves the OOB area inaccessible and results + * in false positive returns from spinand_isbad(). + */ + spinand_cont_read_enable(spinand, false); } } diff --git a/drivers/mtd/parsers/ofpart_core.c b/drivers/mtd/parsers/ofpart_core.c index abfa687989182..09961c6f39496 100644 --- a/drivers/mtd/parsers/ofpart_core.c +++ b/drivers/mtd/parsers/ofpart_core.c @@ -77,6 +77,7 @@ static int parse_fixed_partitions(struct mtd_info *master, of_id = of_match_node(parse_ofpart_match_table, ofpart_node); if (dedicated && !of_id) { /* The 'partitions' subnode might be used by another parser */ + of_node_put(ofpart_node); return 0; } @@ -91,12 +92,18 @@ static int parse_fixed_partitions(struct mtd_info *master, nr_parts++; } - if (nr_parts == 0) + if (nr_parts == 0) { + if (dedicated) + of_node_put(ofpart_node); return 0; + } parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); - if (!parts) + if (!parts) { + if (dedicated) + of_node_put(ofpart_node); return -ENOMEM; + } i = 0; for_each_child_of_node(ofpart_node, pp) { @@ -175,6 +182,9 @@ static int parse_fixed_partitions(struct mtd_info *master, if (quirks && quirks->post_parse) quirks->post_parse(master, parts, nr_parts); + if (dedicated) + of_node_put(ofpart_node); + *pparts = parts; return nr_parts; @@ -183,6 +193,8 @@ static int parse_fixed_partitions(struct mtd_info *master, master->name, pp, mtd_node); ret = -EINVAL; ofpart_none: + if (dedicated) + of_node_put(ofpart_node); of_node_put(pp); kfree(parts); return ret; diff --git a/drivers/mtd/parsers/redboot.c b/drivers/mtd/parsers/redboot.c index 3b55b676ca6b9..c06ba7a2a34b4 100644 --- a/drivers/mtd/parsers/redboot.c +++ b/drivers/mtd/parsers/redboot.c @@ -270,9 +270,9 @@ static int parse_redboot_partitions(struct mtd_info *master, strcpy(names, fl->img->name); #ifdef CONFIG_MTD_REDBOOT_PARTS_READONLY - if (!memcmp(names, "RedBoot", 8) || - !memcmp(names, "RedBoot config", 15) || - !memcmp(names, "FIS directory", 14)) { + if (!strcmp(names, "RedBoot") || + !strcmp(names, "RedBoot config") || + !strcmp(names, "FIS directory")) { parts[i].mask_flags = MTD_WRITEABLE; } #endif diff --git a/drivers/mtd/parsers/tplink_safeloader.c b/drivers/mtd/parsers/tplink_safeloader.c index e358a029dc70c..4fcaf92d22e4f 100644 --- a/drivers/mtd/parsers/tplink_safeloader.c +++ b/drivers/mtd/parsers/tplink_safeloader.c @@ -116,6 +116,7 @@ static int mtd_parser_tplink_safeloader_parse(struct mtd_info *mtd, return idx; err_free: + kfree(buf); for (idx -= 1; idx >= 0; idx--) kfree(parts[idx].name); err_free_parts: diff --git a/drivers/mtd/spi-nor/winbond.c b/drivers/mtd/spi-nor/winbond.c index 63a93c9eb9174..fb855fe44733d 100644 --- a/drivers/mtd/spi-nor/winbond.c +++ b/drivers/mtd/spi-nor/winbond.c @@ -343,6 +343,30 @@ static const struct flash_info winbond_nor_parts[] = { .id = SNOR_ID(0xef, 0x80, 0x20), .name = "w25q512nwm", .otp = SNOR_OTP(256, 3, 0x1000, 0x1000), + }, { + /* W25Q01NWxxIQ */ + .id = SNOR_ID(0xef, 0x60, 0x21), + .flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP, + }, { + /* W25Q01NWxxIM */ + .id = SNOR_ID(0xef, 0x80, 0x21), + .flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP, + }, { + /* W25Q02NWxxIM */ + .id = SNOR_ID(0xef, 0x80, 0x22), + .flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP, + }, { + /* W25H512NWxxAM */ + .id = SNOR_ID(0xef, 0xa0, 0x20), + .flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP, + }, { + /* W25H01NWxxAM */ + .id = SNOR_ID(0xef, 0xa0, 0x21), + .flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP, + }, { + /* W25H02NWxxAM */ + .id = SNOR_ID(0xef, 0xa0, 0x22), + .flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP, }, }; diff --git a/drivers/mux/mmio.c b/drivers/mux/mmio.c index 9993ce38a818d..5b0171d19d430 100644 --- a/drivers/mux/mmio.c +++ b/drivers/mux/mmio.c @@ -58,7 +58,7 @@ static int mux_mmio_probe(struct platform_device *pdev) if (IS_ERR(base)) regmap = ERR_PTR(-ENODEV); else - regmap = regmap_init_mmio(dev, base, &mux_mmio_regmap_cfg); + regmap = devm_regmap_init_mmio(dev, base, &mux_mmio_regmap_cfg); /* Fallback to checking the parent node on "real" errors. */ if (IS_ERR(regmap) && regmap != ERR_PTR(-EPROBE_DEFER)) { regmap = dev_get_regmap(dev->parent, NULL); diff --git a/drivers/net/arcnet/com20020-pci.c b/drivers/net/arcnet/com20020-pci.c index 0472bcdff1307..b5729d6c0b47c 100644 --- a/drivers/net/arcnet/com20020-pci.c +++ b/drivers/net/arcnet/com20020-pci.c @@ -115,6 +115,8 @@ static const struct attribute_group com20020_state_group = { .attrs = com20020_state_attrs, }; +static struct com20020_pci_card_info card_info_2p5mbit; + static void com20020pci_remove(struct pci_dev *pdev); static int com20020pci_probe(struct pci_dev *pdev, @@ -140,7 +142,7 @@ static int com20020pci_probe(struct pci_dev *pdev, ci = (struct com20020_pci_card_info *)id->driver_data; if (!ci) - return -EINVAL; + ci = &card_info_2p5mbit; priv->ci = ci; mm = &ci->misc_map; @@ -347,6 +349,18 @@ static struct com20020_pci_card_info card_info_5mbit = { .flags = ARC_IS_5MBIT, }; +static struct com20020_pci_card_info card_info_2p5mbit = { + .name = "ARC-PCI", + .devcount = 1, + .chan_map_tbl = { + { + .bar = 2, + .offset = 0x00, + .size = 0x08, + }, + }, +}; + static struct com20020_pci_card_info card_info_sohard = { .name = "SOHARD SH ARC-PCI", .devcount = 1, diff --git a/drivers/net/bonding/bond_debugfs.c b/drivers/net/bonding/bond_debugfs.c index 8adbec7c5084a..8967b65f6d840 100644 --- a/drivers/net/bonding/bond_debugfs.c +++ b/drivers/net/bonding/bond_debugfs.c @@ -34,11 +34,17 @@ static int bond_debug_rlb_hash_show(struct seq_file *m, void *v) for (; hash_index != RLB_NULL_INDEX; hash_index = client_info->used_next) { client_info = &(bond_info->rx_hashtbl[hash_index]); - seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n", - &client_info->ip_src, - &client_info->ip_dst, - &client_info->mac_dst, - client_info->slave->dev->name); + if (client_info->slave) + seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n", + &client_info->ip_src, + &client_info->ip_dst, + &client_info->mac_dst, + client_info->slave->dev->name); + else + seq_printf(m, "%-15pI4 %-15pI4 %-17pM (none)\n", + &client_info->ip_src, + &client_info->ip_dst, + &client_info->mac_dst); } spin_unlock_bh(&bond->mode_lock); diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 5abef8a3b7758..1d84e348f2cc7 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -324,7 +324,7 @@ static bool bond_sk_check(struct bonding *bond) } } -bool bond_xdp_check(struct bonding *bond, int mode) +bool __bond_xdp_check(int mode, int xmit_policy) { switch (mode) { case BOND_MODE_ROUNDROBIN: @@ -335,7 +335,7 @@ bool bond_xdp_check(struct bonding *bond, int mode) /* vlan+srcmac is not supported with XDP as in most cases the 802.1q * payload is not in the packet due to hardware offload. */ - if (bond->params.xmit_policy != BOND_XMIT_POLICY_VLAN_SRCMAC) + if (xmit_policy != BOND_XMIT_POLICY_VLAN_SRCMAC) return true; fallthrough; default: @@ -343,6 +343,11 @@ bool bond_xdp_check(struct bonding *bond, int mode) } } +bool bond_xdp_check(struct bonding *bond, int mode) +{ + return __bond_xdp_check(mode, bond->params.xmit_policy); +} + /*---------------------------------- VLAN -----------------------------------*/ /* In the following 2 functions, bond_vlan_rx_add_vid and bond_vlan_rx_kill_vid, @@ -791,26 +796,29 @@ static int bond_update_speed_duplex(struct slave *slave) struct ethtool_link_ksettings ecmd; int res; - slave->speed = SPEED_UNKNOWN; - slave->duplex = DUPLEX_UNKNOWN; - res = __ethtool_get_link_ksettings(slave_dev, &ecmd); if (res < 0) - return 1; + goto speed_duplex_unknown; if (ecmd.base.speed == 0 || ecmd.base.speed == ((__u32)-1)) - return 1; + goto speed_duplex_unknown; switch (ecmd.base.duplex) { case DUPLEX_FULL: case DUPLEX_HALF: break; default: - return 1; + goto speed_duplex_unknown; } slave->speed = ecmd.base.speed; slave->duplex = ecmd.base.duplex; return 0; + +speed_duplex_unknown: + slave->speed = SPEED_UNKNOWN; + slave->duplex = DUPLEX_UNKNOWN; + + return 1; } const char *bond_slave_link_status(s8 link) @@ -1468,97 +1476,52 @@ static netdev_features_t bond_fix_features(struct net_device *dev, return features; } -#define BOND_VLAN_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ - NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE | \ - NETIF_F_GSO_ENCAP_ALL | \ - NETIF_F_HIGHDMA | NETIF_F_LRO) - -#define BOND_ENC_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ - NETIF_F_RXCSUM | NETIF_F_GSO_SOFTWARE | \ - NETIF_F_GSO_PARTIAL) - -#define BOND_MPLS_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ - NETIF_F_GSO_SOFTWARE) - -#define BOND_GSO_PARTIAL_FEATURES (NETIF_F_GSO_ESP) - - -static void bond_compute_features(struct bonding *bond) +static int bond_header_create(struct sk_buff *skb, struct net_device *bond_dev, + unsigned short type, const void *daddr, + const void *saddr, unsigned int len) { - netdev_features_t gso_partial_features = BOND_GSO_PARTIAL_FEATURES; - unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | - IFF_XMIT_DST_RELEASE_PERM; - netdev_features_t vlan_features = BOND_VLAN_FEATURES; - netdev_features_t enc_features = BOND_ENC_FEATURES; -#ifdef CONFIG_XFRM_OFFLOAD - netdev_features_t xfrm_features = BOND_XFRM_FEATURES; -#endif /* CONFIG_XFRM_OFFLOAD */ - netdev_features_t mpls_features = BOND_MPLS_FEATURES; - struct net_device *bond_dev = bond->dev; - struct list_head *iter; + struct bonding *bond = netdev_priv(bond_dev); + const struct header_ops *slave_ops; struct slave *slave; - unsigned short max_hard_header_len = ETH_HLEN; - unsigned int tso_max_size = TSO_MAX_SIZE; - u16 tso_max_segs = TSO_MAX_SEGS; - - if (!bond_has_slaves(bond)) - goto done; - - vlan_features = netdev_base_features(vlan_features); - mpls_features = netdev_base_features(mpls_features); - - bond_for_each_slave(bond, slave, iter) { - vlan_features = netdev_increment_features(vlan_features, - slave->dev->vlan_features, BOND_VLAN_FEATURES); - - enc_features = netdev_increment_features(enc_features, - slave->dev->hw_enc_features, - BOND_ENC_FEATURES); - -#ifdef CONFIG_XFRM_OFFLOAD - xfrm_features = netdev_increment_features(xfrm_features, - slave->dev->hw_enc_features, - BOND_XFRM_FEATURES); -#endif /* CONFIG_XFRM_OFFLOAD */ - - gso_partial_features = netdev_increment_features(gso_partial_features, - slave->dev->gso_partial_features, - BOND_GSO_PARTIAL_FEATURES); - - mpls_features = netdev_increment_features(mpls_features, - slave->dev->mpls_features, - BOND_MPLS_FEATURES); - - dst_release_flag &= slave->dev->priv_flags; - if (slave->dev->hard_header_len > max_hard_header_len) - max_hard_header_len = slave->dev->hard_header_len; + int ret = 0; - tso_max_size = min(tso_max_size, slave->dev->tso_max_size); - tso_max_segs = min(tso_max_segs, slave->dev->tso_max_segs); + rcu_read_lock(); + slave = rcu_dereference(bond->curr_active_slave); + if (slave) { + slave_ops = READ_ONCE(slave->dev->header_ops); + if (slave_ops && slave_ops->create) + ret = slave_ops->create(skb, slave->dev, + type, daddr, saddr, len); } - bond_dev->hard_header_len = max_hard_header_len; - -done: - bond_dev->gso_partial_features = gso_partial_features; - bond_dev->vlan_features = vlan_features; - bond_dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_HW_VLAN_STAG_TX; -#ifdef CONFIG_XFRM_OFFLOAD - bond_dev->hw_enc_features |= xfrm_features; -#endif /* CONFIG_XFRM_OFFLOAD */ - bond_dev->mpls_features = mpls_features; - netif_set_tso_max_segs(bond_dev, tso_max_segs); - netif_set_tso_max_size(bond_dev, tso_max_size); + rcu_read_unlock(); + return ret; +} - bond_dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; - if ((bond_dev->priv_flags & IFF_XMIT_DST_RELEASE_PERM) && - dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM)) - bond_dev->priv_flags |= IFF_XMIT_DST_RELEASE; +static int bond_header_parse(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr) +{ + struct bonding *bond = netdev_priv(dev); + const struct header_ops *slave_ops; + struct slave *slave; + int ret = 0; - netdev_change_features(bond_dev); + rcu_read_lock(); + slave = rcu_dereference(bond->curr_active_slave); + if (slave) { + slave_ops = READ_ONCE(slave->dev->header_ops); + if (slave_ops && slave_ops->parse) + ret = slave_ops->parse(skb, slave->dev, haddr); + } + rcu_read_unlock(); + return ret; } +static const struct header_ops bond_header_ops = { + .create = bond_header_create, + .parse = bond_header_parse, +}; + static void bond_setup_by_slave(struct net_device *bond_dev, struct net_device *slave_dev) { @@ -1566,7 +1529,8 @@ static void bond_setup_by_slave(struct net_device *bond_dev, dev_close(bond_dev); - bond_dev->header_ops = slave_dev->header_ops; + bond_dev->header_ops = slave_dev->header_ops ? + &bond_header_ops : NULL; bond_dev->type = slave_dev->type; bond_dev->hard_header_len = slave_dev->hard_header_len; @@ -1953,6 +1917,12 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, */ if (!bond_has_slaves(bond)) { if (bond_dev->type != slave_dev->type) { + if (slave_dev->type != ARPHRD_ETHER && + BOND_MODE(bond) == BOND_MODE_8023AD) { + SLAVE_NL_ERR(bond_dev, slave_dev, extack, + "8023AD mode requires Ethernet devices"); + return -EINVAL; + } slave_dbg(bond_dev, slave_dev, "change device type from %d to %d\n", bond_dev->type, slave_dev->type); @@ -2273,7 +2243,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, } bond->slave_cnt++; - bond_compute_features(bond); + netdev_compute_master_upper_features(bond->dev, true); bond_set_carrier(bond); /* Needs to be called before bond_select_active_slave(), which will @@ -2287,11 +2257,6 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, unblock_netpoll_tx(); } - /* broadcast mode uses the all_slaves to loop through slaves. */ - if (bond_mode_can_use_xmit_hash(bond) || - BOND_MODE(bond) == BOND_MODE_BROADCAST) - bond_update_slave_arr(bond, NULL); - if (!slave_dev->netdev_ops->ndo_bpf || !slave_dev->netdev_ops->ndo_xdp_xmit) { if (bond->xdp_prog) { @@ -2325,6 +2290,11 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, bpf_prog_inc(bond->xdp_prog); } + /* broadcast mode uses the all_slaves to loop through slaves. */ + if (bond_mode_can_use_xmit_hash(bond) || + BOND_MODE(bond) == BOND_MODE_BROADCAST) + bond_update_slave_arr(bond, NULL); + bond_xdp_set_features(bond_dev); slave_info(bond_dev, slave_dev, "Enslaving as %s interface with %s link\n", @@ -2528,7 +2498,7 @@ static int __bond_release_one(struct net_device *bond_dev, call_netdevice_notifiers(NETDEV_RELEASE, bond->dev); } - bond_compute_features(bond); + netdev_compute_master_upper_features(bond->dev, true); if (!(bond_dev->features & NETIF_F_VLAN_CHALLENGED) && (old_features & NETIF_F_VLAN_CHALLENGED)) slave_info(bond_dev, slave_dev, "last VLAN challenged slave left bond - VLAN blocking is removed\n"); @@ -2846,8 +2816,14 @@ static void bond_miimon_commit(struct bonding *bond) continue; + case BOND_LINK_FAIL: + case BOND_LINK_BACK: + slave_dbg(bond->dev, slave->dev, "link_new_state %d on slave\n", + slave->link_new_state); + continue; + default: - slave_err(bond->dev, slave->dev, "invalid new link %d on slave\n", + slave_err(bond->dev, slave->dev, "invalid link_new_state %d on slave\n", slave->link_new_state); bond_propose_link_state(slave, BOND_LINK_NOCHANGE); @@ -3132,8 +3108,8 @@ static void bond_validate_arp(struct bonding *bond, struct slave *slave, __be32 __func__, &sip); return; } - slave->last_rx = jiffies; - slave->target_last_arp_rx[i] = jiffies; + WRITE_ONCE(slave->last_rx, jiffies); + WRITE_ONCE(slave->target_last_arp_rx[i], jiffies); } static int bond_arp_rcv(const struct sk_buff *skb, struct bonding *bond, @@ -3352,8 +3328,8 @@ static void bond_validate_na(struct bonding *bond, struct slave *slave, __func__, saddr); return; } - slave->last_rx = jiffies; - slave->target_last_arp_rx[i] = jiffies; + WRITE_ONCE(slave->last_rx, jiffies); + WRITE_ONCE(slave->target_last_arp_rx[i], jiffies); } static int bond_na_rcv(const struct sk_buff *skb, struct bonding *bond, @@ -3423,12 +3399,12 @@ int bond_rcv_validate(const struct sk_buff *skb, struct bonding *bond, (slave_do_arp_validate_only(bond) && is_ipv6) || #endif !slave_do_arp_validate_only(bond)) - slave->last_rx = jiffies; + WRITE_ONCE(slave->last_rx, jiffies); return RX_HANDLER_ANOTHER; } else if (is_arp) { return bond_arp_rcv(skb, bond, slave); #if IS_ENABLED(CONFIG_IPV6) - } else if (is_ipv6) { + } else if (is_ipv6 && likely(ipv6_mod_enabled())) { return bond_na_rcv(skb, bond, slave); #endif } else { @@ -3491,7 +3467,7 @@ static void bond_loadbalance_arp_mon(struct bonding *bond) if (slave->link != BOND_LINK_UP) { if (bond_time_in_interval(bond, last_tx, 1) && - bond_time_in_interval(bond, slave->last_rx, 1)) { + bond_time_in_interval(bond, READ_ONCE(slave->last_rx), 1)) { bond_propose_link_state(slave, BOND_LINK_UP); slave_state_changed = 1; @@ -3515,8 +3491,10 @@ static void bond_loadbalance_arp_mon(struct bonding *bond) * when the source ip is 0, so don't take the link down * if we don't know our ip yet */ - if (!bond_time_in_interval(bond, last_tx, bond->params.missed_max) || - !bond_time_in_interval(bond, slave->last_rx, bond->params.missed_max)) { + if (!bond_time_in_interval(bond, last_tx, + bond->params.missed_max) || + !bond_time_in_interval(bond, READ_ONCE(slave->last_rx), + bond->params.missed_max)) { bond_propose_link_state(slave, BOND_LINK_DOWN); slave_state_changed = 1; @@ -4028,7 +4006,7 @@ static int bond_slave_netdev_event(unsigned long event, case NETDEV_FEAT_CHANGE: if (!bond->notifier_ctx) { bond->notifier_ctx = true; - bond_compute_features(bond); + netdev_compute_master_upper_features(bond->dev, true); bond->notifier_ctx = false; } break; @@ -4181,8 +4159,9 @@ static bool bond_flow_dissect(struct bonding *bond, struct sk_buff *skb, const v case BOND_XMIT_POLICY_ENCAP23: case BOND_XMIT_POLICY_ENCAP34: memset(fk, 0, sizeof(*fk)); - return __skb_flow_dissect(NULL, skb, &flow_keys_bonding, - fk, data, l2_proto, nhoff, hlen, 0); + return __skb_flow_dissect(dev_net(bond->dev), skb, + &flow_keys_bonding, fk, data, + l2_proto, nhoff, hlen, 0); default: break; } @@ -4393,9 +4372,13 @@ static int bond_close(struct net_device *bond_dev) bond_work_cancel_all(bond); bond->send_peer_notif = 0; + WRITE_ONCE(bond->recv_probe, NULL); + + /* Wait for any in-flight RX handlers */ + synchronize_net(); + if (bond_is_lb(bond)) bond_alb_deinitialize(bond); - bond->recv_probe = NULL; if (BOND_MODE(bond) == BOND_MODE_8023AD && bond->params.broadcast_neighbor) @@ -5111,13 +5094,18 @@ static void bond_set_slave_arr(struct bonding *bond, { struct bond_up_slave *usable, *all; - usable = rtnl_dereference(bond->usable_slaves); - rcu_assign_pointer(bond->usable_slaves, usable_slaves); - kfree_rcu(usable, rcu); - all = rtnl_dereference(bond->all_slaves); rcu_assign_pointer(bond->all_slaves, all_slaves); kfree_rcu(all, rcu); + + if (BOND_MODE(bond) == BOND_MODE_BROADCAST) { + kfree_rcu(usable_slaves, rcu); + return; + } + + usable = rtnl_dereference(bond->usable_slaves); + rcu_assign_pointer(bond->usable_slaves, usable_slaves); + kfree_rcu(usable, rcu); } static void bond_reset_slave_arr(struct bonding *bond) @@ -5312,7 +5300,7 @@ static netdev_tx_t bond_xmit_broadcast(struct sk_buff *skb, if (!(bond_slave_is_up(slave) && slave->link == BOND_LINK_UP)) continue; - if (bond_is_last_slave(bond, slave)) { + if (i + 1 == slaves_count) { skb2 = skb; skb_used = true; } else { @@ -6011,7 +5999,7 @@ void bond_setup(struct net_device *bond_dev) * capable */ - bond_dev->hw_features = BOND_VLAN_FEATURES | + bond_dev->hw_features = MASTER_UPPER_DEV_VLAN_FEATURES | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_RX | diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index 384499c869b8d..adc216df43459 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -1152,7 +1152,7 @@ static void _bond_options_arp_ip_target_set(struct bonding *bond, int slot, if (slot >= 0 && slot < BOND_MAX_ARP_TARGETS) { bond_for_each_slave(bond, slave, iter) - slave->target_last_arp_rx[slot] = last_rx; + WRITE_ONCE(slave->target_last_arp_rx[slot], last_rx); targets[slot] = target; } } @@ -1221,8 +1221,8 @@ static int bond_option_arp_ip_target_rem(struct bonding *bond, __be32 target) bond_for_each_slave(bond, slave, iter) { targets_rx = slave->target_last_arp_rx; for (i = ind; (i < BOND_MAX_ARP_TARGETS-1) && targets[i+1]; i++) - targets_rx[i] = targets_rx[i+1]; - targets_rx[i] = 0; + WRITE_ONCE(targets_rx[i], READ_ONCE(targets_rx[i+1])); + WRITE_ONCE(targets_rx[i], 0); } for (i = ind; (i < BOND_MAX_ARP_TARGETS-1) && targets[i+1]; i++) targets[i] = targets[i+1]; @@ -1377,7 +1377,7 @@ static void _bond_options_ns_ip6_target_set(struct bonding *bond, int slot, if (slot >= 0 && slot < BOND_MAX_NS_TARGETS) { bond_for_each_slave(bond, slave, iter) { - slave->target_last_arp_rx[slot] = last_rx; + WRITE_ONCE(slave->target_last_arp_rx[slot], last_rx); slave_set_ns_maddr(bond, slave, target, &targets[slot]); } targets[slot] = *target; @@ -1574,6 +1574,8 @@ static int bond_option_fail_over_mac_set(struct bonding *bond, static int bond_option_xmit_hash_policy_set(struct bonding *bond, const struct bond_opt_value *newval) { + if (bond->xdp_prog && !__bond_xdp_check(BOND_MODE(bond), newval->value)) + return -EOPNOTSUPP; netdev_dbg(bond->dev, "Setting xmit hash policy to %s (%llu)\n", newval->string, newval->value); bond->params.xmit_policy = newval->value; diff --git a/drivers/net/caif/caif_serial.c b/drivers/net/caif/caif_serial.c index c398ac42eae90..1873d8287bb9b 100644 --- a/drivers/net/caif/caif_serial.c +++ b/drivers/net/caif/caif_serial.c @@ -284,6 +284,7 @@ static void ser_release(struct work_struct *work) { struct list_head list; struct ser_device *ser, *tmp; + struct tty_struct *tty; spin_lock(&ser_lock); list_replace_init(&ser_release_list, &list); @@ -292,9 +293,12 @@ static void ser_release(struct work_struct *work) if (!list_empty(&list)) { rtnl_lock(); list_for_each_entry_safe(ser, tmp, &list, node) { + tty = ser->tty; dev_close(ser->dev); unregister_netdevice(ser->dev); debugfs_deinit(ser); + tty_kref_put(tty->link); + tty_kref_put(tty); } rtnl_unlock(); } @@ -328,6 +332,7 @@ static int ldisc_open(struct tty_struct *tty) ser = netdev_priv(dev); ser->tty = tty_kref_get(tty); + tty_kref_get(tty->link); ser->dev = dev; debugfs_init(ser, tty); tty->receive_room = 4096; @@ -336,6 +341,7 @@ static int ldisc_open(struct tty_struct *tty) rtnl_lock(); result = register_netdevice(dev); if (result) { + tty_kref_put(tty->link); tty_kref_put(tty); rtnl_unlock(); free_netdev(dev); @@ -355,8 +361,6 @@ static void ldisc_close(struct tty_struct *tty) { struct ser_device *ser = tty->disc_data; - tty_kref_put(ser->tty); - spin_lock(&ser_lock); list_move(&ser->node, &ser_release_list); spin_unlock(&ser_lock); diff --git a/drivers/net/can/at91_can.c b/drivers/net/can/at91_can.c index 191707d7e3dac..d6dcb2be56342 100644 --- a/drivers/net/can/at91_can.c +++ b/drivers/net/can/at91_can.c @@ -1100,7 +1100,7 @@ static int at91_can_probe(struct platform_device *pdev) if (IS_ERR(transceiver)) { err = PTR_ERR(transceiver); dev_err_probe(&pdev->dev, err, "failed to get phy\n"); - goto exit_iounmap; + goto exit_free; } dev->netdev_ops = &at91_netdev_ops; diff --git a/drivers/net/can/ctucanfd/ctucanfd_base.c b/drivers/net/can/ctucanfd/ctucanfd_base.c index 8bd3f0fc385c3..33a5ec4346278 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_base.c +++ b/drivers/net/can/ctucanfd/ctucanfd_base.c @@ -310,7 +310,7 @@ static int ctucan_set_secondary_sample_point(struct net_device *ndev) } ssp_cfg = FIELD_PREP(REG_TRV_DELAY_SSP_OFFSET, ssp_offset); - ssp_cfg |= FIELD_PREP(REG_TRV_DELAY_SSP_SRC, 0x1); + ssp_cfg |= FIELD_PREP(REG_TRV_DELAY_SSP_SRC, 0x0); } ctucan_write32(priv, CTUCANFD_TRV_DELAY, ssp_cfg); diff --git a/drivers/net/can/dev/netlink.c b/drivers/net/can/dev/netlink.c index 6f83b87d54fcb..fcfff221d1cc2 100644 --- a/drivers/net/can/dev/netlink.c +++ b/drivers/net/can/dev/netlink.c @@ -407,7 +407,9 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[], /* We need synchronization with dev->stop() */ ASSERT_RTNL(); - can_ctrlmode_changelink(dev, data, extack); + err = can_ctrlmode_changelink(dev, data, extack); + if (err) + return err; if (data[IFLA_CAN_BITTIMING]) { struct can_bittiming bt; diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index 6d4b643e135fd..5f5a7e7e547eb 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -755,7 +755,9 @@ static int hi3110_open(struct net_device *net) return ret; mutex_lock(&priv->hi3110_lock); - hi3110_power_enable(priv->transceiver, 1); + ret = hi3110_power_enable(priv->transceiver, 1); + if (ret) + goto out_close_candev; priv->force_quit = 0; priv->tx_skb = NULL; @@ -790,6 +792,7 @@ static int hi3110_open(struct net_device *net) hi3110_hw_sleep(spi); out_close: hi3110_power_enable(priv->transceiver, 0); + out_close_candev: close_candev(net); mutex_unlock(&priv->hi3110_lock); return ret; diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index b797e08499d70..5a7aa02092c7e 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -1214,6 +1214,7 @@ static int mcp251x_open(struct net_device *net) { struct mcp251x_priv *priv = netdev_priv(net); struct spi_device *spi = priv->spi; + bool release_irq = false; unsigned long flags = 0; int ret; @@ -1224,7 +1225,11 @@ static int mcp251x_open(struct net_device *net) } mutex_lock(&priv->mcp_lock); - mcp251x_power_enable(priv->transceiver, 1); + ret = mcp251x_power_enable(priv->transceiver, 1); + if (ret) { + dev_err(&spi->dev, "failed to enable transceiver power: %pe\n", ERR_PTR(ret)); + goto out_close_candev; + } priv->force_quit = 0; priv->tx_skb = NULL; @@ -1257,12 +1262,25 @@ static int mcp251x_open(struct net_device *net) return 0; out_free_irq: - free_irq(spi->irq, priv); + /* The IRQ handler might be running, and if so it will be waiting + * for the lock. But free_irq() must wait for the handler to finish + * so calling it here would deadlock. + * + * Setting priv->force_quit will let the handler exit right away + * without any access to the hardware. This make it safe to call + * free_irq() after the lock is released. + */ + priv->force_quit = 1; + release_irq = true; + mcp251x_hw_sleep(spi); out_close: mcp251x_power_enable(priv->transceiver, 0); +out_close_candev: close_candev(net); mutex_unlock(&priv->mcp_lock); + if (release_irq) + free_irq(spi->irq, priv); return ret; } @@ -1495,11 +1513,25 @@ static int __maybe_unused mcp251x_can_resume(struct device *dev) { struct spi_device *spi = to_spi_device(dev); struct mcp251x_priv *priv = spi_get_drvdata(spi); + int ret = 0; - if (priv->after_suspend & AFTER_SUSPEND_POWER) - mcp251x_power_enable(priv->power, 1); - if (priv->after_suspend & AFTER_SUSPEND_UP) - mcp251x_power_enable(priv->transceiver, 1); + if (priv->after_suspend & AFTER_SUSPEND_POWER) { + ret = mcp251x_power_enable(priv->power, 1); + if (ret) { + dev_err(dev, "failed to restore power: %pe\n", ERR_PTR(ret)); + return ret; + } + } + + if (priv->after_suspend & AFTER_SUSPEND_UP) { + ret = mcp251x_power_enable(priv->transceiver, 1); + if (ret) { + dev_err(dev, "failed to restore transceiver power: %pe\n", ERR_PTR(ret)); + if (priv->after_suspend & AFTER_SUSPEND_POWER) + mcp251x_power_enable(priv->power, 0); + return ret; + } + } if (priv->after_suspend & (AFTER_SUSPEND_POWER | AFTER_SUSPEND_UP)) queue_work(priv->wq, &priv->restart_work); diff --git a/drivers/net/can/spi/mcp251xfd/Kconfig b/drivers/net/can/spi/mcp251xfd/Kconfig index 877e4356010dd..7c29846e60517 100644 --- a/drivers/net/can/spi/mcp251xfd/Kconfig +++ b/drivers/net/can/spi/mcp251xfd/Kconfig @@ -5,6 +5,7 @@ config CAN_MCP251XFD select CAN_RX_OFFLOAD select REGMAP select WANT_DEV_COREDUMP + select GPIOLIB help Driver for the Microchip MCP251XFD SPI FD-CAN controller family. diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c index 7450ea42c1ea5..586336d9e421e 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c @@ -608,23 +608,21 @@ static int mcp251xfd_set_bittiming(const struct mcp251xfd_priv *priv) static int mcp251xfd_chip_rx_int_enable(const struct mcp251xfd_priv *priv) { - u32 val; + u32 val, mask; if (!priv->rx_int) return 0; - /* Configure GPIOs: - * - PIN0: GPIO Input - * - PIN1: GPIO Input/RX Interrupt + /* Configure PIN1 as RX Interrupt: * * PIN1 must be Input, otherwise there is a glitch on the * rx-INT line. It happens between setting the PIN as output * (in the first byte of the SPI transfer) and configuring the * PIN as interrupt (in the last byte of the SPI transfer). */ - val = MCP251XFD_REG_IOCON_PM0 | MCP251XFD_REG_IOCON_TRIS1 | - MCP251XFD_REG_IOCON_TRIS0; - return regmap_write(priv->map_reg, MCP251XFD_REG_IOCON, val); + val = MCP251XFD_REG_IOCON_TRIS(1); + mask = MCP251XFD_REG_IOCON_TRIS(1) | MCP251XFD_REG_IOCON_PM(1); + return regmap_update_bits(priv->map_reg, MCP251XFD_REG_IOCON, mask, val); } static int mcp251xfd_chip_rx_int_disable(const struct mcp251xfd_priv *priv) @@ -634,13 +632,9 @@ static int mcp251xfd_chip_rx_int_disable(const struct mcp251xfd_priv *priv) if (!priv->rx_int) return 0; - /* Configure GPIOs: - * - PIN0: GPIO Input - * - PIN1: GPIO Input - */ - val = MCP251XFD_REG_IOCON_PM1 | MCP251XFD_REG_IOCON_PM0 | - MCP251XFD_REG_IOCON_TRIS1 | MCP251XFD_REG_IOCON_TRIS0; - return regmap_write(priv->map_reg, MCP251XFD_REG_IOCON, val); + /* Configure PIN1 as GPIO Input */ + val = MCP251XFD_REG_IOCON_PM(1) | MCP251XFD_REG_IOCON_TRIS(1); + return regmap_update_bits(priv->map_reg, MCP251XFD_REG_IOCON, val, val); } static int mcp251xfd_chip_ecc_init(struct mcp251xfd_priv *priv) @@ -767,21 +761,13 @@ static void mcp251xfd_chip_stop(struct mcp251xfd_priv *priv, mcp251xfd_chip_interrupts_disable(priv); mcp251xfd_chip_rx_int_disable(priv); mcp251xfd_timestamp_stop(priv); - mcp251xfd_chip_sleep(priv); + mcp251xfd_chip_set_mode(priv, MCP251XFD_REG_CON_MODE_CONFIG); } static int mcp251xfd_chip_start(struct mcp251xfd_priv *priv) { int err; - err = mcp251xfd_chip_softreset(priv); - if (err) - goto out_chip_stop; - - err = mcp251xfd_chip_clock_init(priv); - if (err) - goto out_chip_stop; - err = mcp251xfd_chip_timestamp_init(priv); if (err) goto out_chip_stop; @@ -1625,8 +1611,11 @@ static int mcp251xfd_open(struct net_device *ndev) return err; err = pm_runtime_resume_and_get(ndev->dev.parent); - if (err) + if (err) { + if (err == -ETIMEDOUT || err == -ENODEV) + pm_runtime_set_suspended(ndev->dev.parent); goto out_close_candev; + } err = mcp251xfd_ring_alloc(priv); if (err) @@ -1808,6 +1797,160 @@ static int mcp251xfd_register_check_rx_int(struct mcp251xfd_priv *priv) return 0; } +static const char * const mcp251xfd_gpio_names[] = { "GPIO0", "GPIO1" }; + +static int mcp251xfd_gpio_request(struct gpio_chip *chip, unsigned int offset) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 pin_mask = MCP251XFD_REG_IOCON_PM(offset); + int ret; + + if (priv->rx_int && offset == 1) { + netdev_err(priv->ndev, "Can't use GPIO 1 with RX-INT!\n"); + return -EINVAL; + } + + ret = pm_runtime_resume_and_get(priv->ndev->dev.parent); + if (ret) + return ret; + + return regmap_update_bits(priv->map_reg, MCP251XFD_REG_IOCON, pin_mask, pin_mask); +} + +static void mcp251xfd_gpio_free(struct gpio_chip *chip, unsigned int offset) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + + pm_runtime_put(priv->ndev->dev.parent); +} + +static int mcp251xfd_gpio_get_direction(struct gpio_chip *chip, + unsigned int offset) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 mask = MCP251XFD_REG_IOCON_TRIS(offset); + u32 val; + int ret; + + ret = regmap_read(priv->map_reg, MCP251XFD_REG_IOCON, &val); + if (ret) + return ret; + + if (mask & val) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; +} + +static int mcp251xfd_gpio_get(struct gpio_chip *chip, unsigned int offset) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 mask = MCP251XFD_REG_IOCON_GPIO(offset); + u32 val; + int ret; + + ret = regmap_read(priv->map_reg, MCP251XFD_REG_IOCON, &val); + if (ret) + return ret; + + return !!(mask & val); +} + +static int mcp251xfd_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask, + unsigned long *bit) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 val; + int ret; + + ret = regmap_read(priv->map_reg, MCP251XFD_REG_IOCON, &val); + if (ret) + return ret; + + *bit = FIELD_GET(MCP251XFD_REG_IOCON_GPIO_MASK, val) & *mask; + + return 0; +} + +static int mcp251xfd_gpio_direction_output(struct gpio_chip *chip, + unsigned int offset, int value) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 dir_mask = MCP251XFD_REG_IOCON_TRIS(offset); + u32 val_mask = MCP251XFD_REG_IOCON_LAT(offset); + u32 val; + + if (value) + val = val_mask; + else + val = 0; + + return regmap_update_bits(priv->map_reg, MCP251XFD_REG_IOCON, + dir_mask | val_mask, val); +} + +static int mcp251xfd_gpio_direction_input(struct gpio_chip *chip, + unsigned int offset) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 dir_mask = MCP251XFD_REG_IOCON_TRIS(offset); + + return regmap_update_bits(priv->map_reg, MCP251XFD_REG_IOCON, dir_mask, dir_mask); +} + +static int mcp251xfd_gpio_set(struct gpio_chip *chip, unsigned int offset, int value) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 val_mask = MCP251XFD_REG_IOCON_LAT(offset); + u32 val; + + if (value) + val = val_mask; + else + val = 0; + + return regmap_update_bits(priv->map_reg, MCP251XFD_REG_IOCON, val_mask, val); +} + +static int mcp251xfd_gpio_set_multiple(struct gpio_chip *chip, unsigned long *mask, + unsigned long *bits) +{ + struct mcp251xfd_priv *priv = gpiochip_get_data(chip); + u32 val; + + val = FIELD_PREP(MCP251XFD_REG_IOCON_LAT_MASK, *bits); + + return regmap_update_bits(priv->map_reg, MCP251XFD_REG_IOCON, + MCP251XFD_REG_IOCON_LAT_MASK, val); +} + +static int mcp251fdx_gpio_setup(struct mcp251xfd_priv *priv) +{ + struct gpio_chip *gc = &priv->gc; + + if (!device_property_present(&priv->spi->dev, "gpio-controller")) + return 0; + + gc->label = dev_name(&priv->spi->dev); + gc->parent = &priv->spi->dev; + gc->owner = THIS_MODULE; + gc->request = mcp251xfd_gpio_request; + gc->free = mcp251xfd_gpio_free; + gc->get_direction = mcp251xfd_gpio_get_direction; + gc->direction_output = mcp251xfd_gpio_direction_output; + gc->direction_input = mcp251xfd_gpio_direction_input; + gc->get = mcp251xfd_gpio_get; + gc->get_multiple = mcp251xfd_gpio_get_multiple; + gc->set = mcp251xfd_gpio_set; + gc->set_multiple = mcp251xfd_gpio_set_multiple; + gc->base = -1; + gc->can_sleep = true; + gc->ngpio = ARRAY_SIZE(mcp251xfd_gpio_names); + gc->names = mcp251xfd_gpio_names; + + return devm_gpiochip_add_data(&priv->spi->dev, gc, priv); +} + static int mcp251xfd_register_get_dev_id(const struct mcp251xfd_priv *priv, u32 *dev_id, u32 *effective_speed_hz_slow, @@ -1907,53 +2050,59 @@ static int mcp251xfd_register(struct mcp251xfd_priv *priv) struct net_device *ndev = priv->ndev; int err; + mcp251xfd_register_quirks(priv); + err = mcp251xfd_clks_and_vdd_enable(priv); if (err) return err; - pm_runtime_get_noresume(ndev->dev.parent); - err = pm_runtime_set_active(ndev->dev.parent); - if (err) - goto out_runtime_put_noidle; - pm_runtime_enable(ndev->dev.parent); - - mcp251xfd_register_quirks(priv); - err = mcp251xfd_chip_softreset(priv); if (err == -ENODEV) - goto out_runtime_disable; + goto out_clks_and_vdd_disable; if (err) goto out_chip_sleep; err = mcp251xfd_chip_clock_init(priv); if (err == -ENODEV) - goto out_runtime_disable; + goto out_clks_and_vdd_disable; if (err) goto out_chip_sleep; + pm_runtime_get_noresume(ndev->dev.parent); + err = pm_runtime_set_active(ndev->dev.parent); + if (err) + goto out_runtime_put_noidle; + pm_runtime_enable(ndev->dev.parent); + err = mcp251xfd_register_chip_detect(priv); if (err) - goto out_chip_sleep; + goto out_runtime_disable; err = mcp251xfd_register_check_rx_int(priv); if (err) - goto out_chip_sleep; + goto out_runtime_disable; mcp251xfd_ethtool_init(priv); + err = mcp251fdx_gpio_setup(priv); + if (err) { + dev_err_probe(&priv->spi->dev, err, "Failed to register gpio-controller.\n"); + goto out_runtime_disable; + } + err = register_candev(ndev); if (err) - goto out_chip_sleep; + goto out_runtime_disable; err = mcp251xfd_register_done(priv); if (err) goto out_unregister_candev; - /* Put controller into sleep mode and let pm_runtime_put() - * disable the clocks and vdd. If CONFIG_PM is not enabled, - * the clocks and vdd will stay powered. + /* Put controller into Config mode and let pm_runtime_put() + * put in sleep mode, disable the clocks and vdd. If CONFIG_PM + * is not enabled, the clocks and vdd will stay powered. */ - err = mcp251xfd_chip_sleep(priv); + err = mcp251xfd_chip_set_mode(priv, MCP251XFD_REG_CON_MODE_CONFIG); if (err) goto out_unregister_candev; @@ -1963,12 +2112,13 @@ static int mcp251xfd_register(struct mcp251xfd_priv *priv) out_unregister_candev: unregister_candev(ndev); -out_chip_sleep: - mcp251xfd_chip_sleep(priv); out_runtime_disable: pm_runtime_disable(ndev->dev.parent); out_runtime_put_noidle: pm_runtime_put_noidle(ndev->dev.parent); +out_chip_sleep: + mcp251xfd_chip_sleep(priv); +out_clks_and_vdd_disable: mcp251xfd_clks_and_vdd_disable(priv); return err; @@ -1980,10 +2130,12 @@ static inline void mcp251xfd_unregister(struct mcp251xfd_priv *priv) unregister_candev(ndev); - if (pm_runtime_enabled(ndev->dev.parent)) + if (pm_runtime_enabled(ndev->dev.parent)) { pm_runtime_disable(ndev->dev.parent); - else + } else { + mcp251xfd_chip_sleep(priv); mcp251xfd_clks_and_vdd_disable(priv); + } } static const struct of_device_id mcp251xfd_of_match[] = { @@ -2206,16 +2358,41 @@ static void mcp251xfd_remove(struct spi_device *spi) static int __maybe_unused mcp251xfd_runtime_suspend(struct device *device) { - const struct mcp251xfd_priv *priv = dev_get_drvdata(device); + struct mcp251xfd_priv *priv = dev_get_drvdata(device); + mcp251xfd_chip_sleep(priv); return mcp251xfd_clks_and_vdd_disable(priv); } static int __maybe_unused mcp251xfd_runtime_resume(struct device *device) { - const struct mcp251xfd_priv *priv = dev_get_drvdata(device); + struct mcp251xfd_priv *priv = dev_get_drvdata(device); + int err; + + err = mcp251xfd_clks_and_vdd_enable(priv); + if (err) + return err; - return mcp251xfd_clks_and_vdd_enable(priv); + err = mcp251xfd_chip_softreset(priv); + if (err == -ENODEV) + goto out_clks_and_vdd_disable; + if (err) + goto out_chip_sleep; + + err = mcp251xfd_chip_clock_init(priv); + if (err == -ENODEV) + goto out_clks_and_vdd_disable; + if (err) + goto out_chip_sleep; + + return 0; + +out_chip_sleep: + mcp251xfd_chip_sleep(priv); +out_clks_and_vdd_disable: + mcp251xfd_clks_and_vdd_disable(priv); + + return err; } static const struct dev_pm_ops mcp251xfd_pm_ops = { diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-regmap.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-regmap.c index 8c5be8d1c5196..70d5ff0ae7acb 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-regmap.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-regmap.c @@ -13,17 +13,9 @@ static const struct regmap_config mcp251xfd_regmap_crc; static int -mcp251xfd_regmap_nocrc_write(void *context, const void *data, size_t count) -{ - struct spi_device *spi = context; - - return spi_write(spi, data, count); -} - -static int -mcp251xfd_regmap_nocrc_gather_write(void *context, - const void *reg, size_t reg_len, - const void *val, size_t val_len) +_mcp251xfd_regmap_nocrc_gather_write(void *context, + const void *reg, size_t reg_len, + const void *val, size_t val_len) { struct spi_device *spi = context; struct mcp251xfd_priv *priv = spi_get_drvdata(spi); @@ -47,6 +39,54 @@ mcp251xfd_regmap_nocrc_gather_write(void *context, return spi_sync_transfer(spi, xfer, ARRAY_SIZE(xfer)); } +static int +mcp251xfd_regmap_nocrc_gather_write(void *context, + const void *reg_p, size_t reg_len, + const void *val, size_t val_len) +{ + const u16 byte_exclude = MCP251XFD_REG_IOCON + + mcp251xfd_first_byte_set(MCP251XFD_REG_IOCON_GPIO_MASK); + u16 reg = be16_to_cpu(*(__be16 *)reg_p) & MCP251XFD_SPI_ADDRESS_MASK; + int ret; + + /* Never write to bits 16..23 of IOCON register to avoid clearing of LAT0/LAT1 + * + * According to MCP2518FD Errata DS80000789E 5 writing IOCON register using one + * SPI write command clears LAT0/LAT1. + * + * Errata Fix/Work Around suggests to write registers with single byte + * write instructions. However, it seems that the byte at 0xe06(IOCON[23:16]) + * is for read-only access and writing to it causes the clearing of LAT0/LAT1. + */ + if (reg <= byte_exclude && reg + val_len > byte_exclude) { + size_t len = byte_exclude - reg; + + /* Write up to 0xe05 */ + ret = _mcp251xfd_regmap_nocrc_gather_write(context, reg_p, reg_len, val, len); + if (ret) + return ret; + + /* Write from 0xe07 on */ + reg += len + 1; + reg = (__force unsigned short)cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_WRITE | reg); + return _mcp251xfd_regmap_nocrc_gather_write(context, ®, reg_len, + val + len + 1, + val_len - len - 1); + } + + return _mcp251xfd_regmap_nocrc_gather_write(context, reg_p, reg_len, + val, val_len); +} + +static int +mcp251xfd_regmap_nocrc_write(void *context, const void *data, size_t count) +{ + const size_t data_offset = sizeof(__be16); + + return mcp251xfd_regmap_nocrc_gather_write(context, data, data_offset, + data + data_offset, count - data_offset); +} + static inline bool mcp251xfd_update_bits_read_reg(const struct mcp251xfd_priv *priv, unsigned int reg) @@ -64,6 +104,7 @@ mcp251xfd_update_bits_read_reg(const struct mcp251xfd_priv *priv, case MCP251XFD_REG_CON: case MCP251XFD_REG_OSC: case MCP251XFD_REG_ECCCON: + case MCP251XFD_REG_IOCON: return true; default: mcp251xfd_for_each_rx_ring(priv, ring, n) { @@ -139,10 +180,9 @@ mcp251xfd_regmap_nocrc_update_bits(void *context, unsigned int reg, tmp_le32 = orig_le32 & ~mask_le32; tmp_le32 |= val_le32 & mask_le32; - mcp251xfd_spi_cmd_write_nocrc(&buf_tx->cmd, reg + first_byte); - memcpy(buf_tx->data, &tmp_le32, len); - - return spi_write(spi, buf_tx, sizeof(buf_tx->cmd) + len); + reg += first_byte; + mcp251xfd_spi_cmd_write_nocrc(&buf_tx->cmd, reg); + return mcp251xfd_regmap_nocrc_gather_write(context, &buf_tx->cmd, 2, &tmp_le32, len); } static int @@ -196,9 +236,9 @@ mcp251xfd_regmap_nocrc_read(void *context, } static int -mcp251xfd_regmap_crc_gather_write(void *context, - const void *reg_p, size_t reg_len, - const void *val, size_t val_len) +_mcp251xfd_regmap_crc_gather_write(void *context, + const void *reg_p, size_t reg_len, + const void *val, size_t val_len) { struct spi_device *spi = context; struct mcp251xfd_priv *priv = spi_get_drvdata(spi); @@ -229,6 +269,44 @@ mcp251xfd_regmap_crc_gather_write(void *context, return spi_sync_transfer(spi, xfer, ARRAY_SIZE(xfer)); } +static int +mcp251xfd_regmap_crc_gather_write(void *context, + const void *reg_p, size_t reg_len, + const void *val, size_t val_len) +{ + const u16 byte_exclude = MCP251XFD_REG_IOCON + + mcp251xfd_first_byte_set(MCP251XFD_REG_IOCON_GPIO_MASK); + u16 reg = *(u16 *)reg_p; + int ret; + + /* Never write to bits 16..23 of IOCON register to avoid clearing of LAT0/LAT1 + * + * According to MCP2518FD Errata DS80000789E 5 writing IOCON register using one + * SPI write command clears LAT0/LAT1. + * + * Errata Fix/Work Around suggests to write registers with single byte + * write instructions. However, it seems that the byte at 0xe06(IOCON[23:16]) + * is for read-only access and writing to it causes the clearing of LAT0/LAT1. + */ + if (reg <= byte_exclude && reg + val_len > byte_exclude) { + size_t len = byte_exclude - reg; + + /* Write up to 0xe05 */ + ret = _mcp251xfd_regmap_crc_gather_write(context, ®, reg_len, val, len); + if (ret) + return ret; + + /* Write from 0xe07 on */ + reg += len + 1; + return _mcp251xfd_regmap_crc_gather_write(context, ®, reg_len, + val + len + 1, + val_len - len - 1); + } + + return _mcp251xfd_regmap_crc_gather_write(context, reg_p, reg_len, + val, val_len); +} + static int mcp251xfd_regmap_crc_write(void *context, const void *data, size_t count) diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd.h b/drivers/net/can/spi/mcp251xfd/mcp251xfd.h index dcbbd2b2fae82..085d7101e595e 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd.h +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -335,13 +336,19 @@ #define MCP251XFD_REG_IOCON_TXCANOD BIT(28) #define MCP251XFD_REG_IOCON_PM1 BIT(25) #define MCP251XFD_REG_IOCON_PM0 BIT(24) +#define MCP251XFD_REG_IOCON_PM(n) (MCP251XFD_REG_IOCON_PM0 << (n)) #define MCP251XFD_REG_IOCON_GPIO1 BIT(17) #define MCP251XFD_REG_IOCON_GPIO0 BIT(16) +#define MCP251XFD_REG_IOCON_GPIO(n) (MCP251XFD_REG_IOCON_GPIO0 << (n)) +#define MCP251XFD_REG_IOCON_GPIO_MASK GENMASK(17, 16) #define MCP251XFD_REG_IOCON_LAT1 BIT(9) #define MCP251XFD_REG_IOCON_LAT0 BIT(8) +#define MCP251XFD_REG_IOCON_LAT(n) (MCP251XFD_REG_IOCON_LAT0 << (n)) +#define MCP251XFD_REG_IOCON_LAT_MASK GENMASK(9, 8) #define MCP251XFD_REG_IOCON_XSTBYEN BIT(6) #define MCP251XFD_REG_IOCON_TRIS1 BIT(1) #define MCP251XFD_REG_IOCON_TRIS0 BIT(0) +#define MCP251XFD_REG_IOCON_TRIS(n) (MCP251XFD_REG_IOCON_TRIS0 << (n)) #define MCP251XFD_REG_CRC 0xe08 #define MCP251XFD_REG_CRC_FERRIE BIT(25) @@ -670,6 +677,7 @@ struct mcp251xfd_priv { struct mcp251xfd_devtype_data devtype_data; struct can_berr_counter bec; + struct gpio_chip gc; }; #define MCP251XFD_IS(_model) \ diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index 5355bac4dccbe..d8c881130e900 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -445,6 +445,11 @@ static void ems_usb_read_bulk_callback(struct urb *urb) start = CPC_HEADER_SIZE; while (msg_count) { + if (start + CPC_MSG_HEADER_LEN > urb->actual_length) { + netdev_err(netdev, "format error\n"); + break; + } + msg = (struct ems_cpc_msg *)&ibuf[start]; switch (msg->type) { @@ -474,7 +479,7 @@ static void ems_usb_read_bulk_callback(struct urb *urb) start += CPC_MSG_HEADER_LEN + msg->length; msg_count--; - if (start > urb->transfer_buffer_length) { + if (start > urb->actual_length) { netdev_err(netdev, "format error\n"); break; } @@ -486,11 +491,17 @@ static void ems_usb_read_bulk_callback(struct urb *urb) urb->transfer_buffer, RX_BUFFER_SIZE, ems_usb_read_bulk_callback, dev); + usb_anchor_urb(urb, &dev->rx_submitted); + retval = usb_submit_urb(urb, GFP_ATOMIC); + if (!retval) + return; + + usb_unanchor_urb(urb); if (retval == -ENODEV) netif_device_detach(netdev); - else if (retval) + else netdev_err(netdev, "failed resubmitting read bulk urb: %d\n", retval); } diff --git a/drivers/net/can/usb/esd_usb.c b/drivers/net/can/usb/esd_usb.c index 9bc1824d7be6a..e58f0bfa84a4c 100644 --- a/drivers/net/can/usb/esd_usb.c +++ b/drivers/net/can/usb/esd_usb.c @@ -541,13 +541,20 @@ static void esd_usb_read_bulk_callback(struct urb *urb) urb->transfer_buffer, ESD_USB_RX_BUFFER_SIZE, esd_usb_read_bulk_callback, dev); + usb_anchor_urb(urb, &dev->rx_submitted); + err = usb_submit_urb(urb, GFP_ATOMIC); + if (!err) + return; + + usb_unanchor_urb(urb); + if (err == -ENODEV) { for (i = 0; i < dev->net_count; i++) { if (dev->nets[i]) netif_device_detach(dev->nets[i]->netdev); } - } else if (err) { + } else { dev_err(dev->udev->dev.parent, "failed resubmitting read bulk urb: %pe\n", ERR_PTR(err)); } diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c index adc91873c083f..1e44bd1dfee9b 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.c +++ b/drivers/net/can/usb/etas_es58x/es58x_core.c @@ -1461,12 +1461,18 @@ static void es58x_read_bulk_callback(struct urb *urb) } resubmit_urb: + usb_anchor_urb(urb, &es58x_dev->rx_urbs); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + + usb_unanchor_urb(urb); + if (ret == -ENODEV) { for (i = 0; i < es58x_dev->num_can_ch; i++) if (es58x_dev->netdev[i]) netif_device_detach(es58x_dev->netdev[i]); - } else if (ret) + } else dev_err_ratelimited(dev, "Failed resubmitting read bulk urb: %pe\n", ERR_PTR(ret)); @@ -1736,7 +1742,7 @@ static int es58x_alloc_rx_urbs(struct es58x_device *es58x_dev) dev_dbg(dev, "%s: Allocated %d rx URBs each of size %u\n", __func__, i, rx_buf_len); - return ret; + return 0; } /** diff --git a/drivers/net/can/usb/f81604.c b/drivers/net/can/usb/f81604.c index e0cfa1460b0b8..6b8b2795c0188 100644 --- a/drivers/net/can/usb/f81604.c +++ b/drivers/net/can/usb/f81604.c @@ -413,6 +413,7 @@ static void f81604_read_bulk_callback(struct urb *urb) { struct f81604_can_frame *frame = urb->transfer_buffer; struct net_device *netdev = urb->context; + struct f81604_port_priv *priv = netdev_priv(netdev); int ret; if (!netif_device_present(netdev)) @@ -445,10 +446,15 @@ static void f81604_read_bulk_callback(struct urb *urb) f81604_process_rx_packet(netdev, frame); resubmit_urb: + usb_anchor_urb(urb, &priv->urbs_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + usb_unanchor_urb(urb); + if (ret == -ENODEV) netif_device_detach(netdev); - else if (ret) + else netdev_err(netdev, "%s: failed to resubmit read bulk urb: %pe\n", __func__, ERR_PTR(ret)); @@ -620,6 +626,12 @@ static void f81604_read_int_callback(struct urb *urb) netdev_info(netdev, "%s: Int URB aborted: %pe\n", __func__, ERR_PTR(urb->status)); + if (urb->actual_length < sizeof(*data)) { + netdev_warn(netdev, "%s: short int URB: %u < %zu\n", + __func__, urb->actual_length, sizeof(*data)); + goto resubmit_urb; + } + switch (urb->status) { case 0: /* success */ break; @@ -646,10 +658,15 @@ static void f81604_read_int_callback(struct urb *urb) f81604_handle_tx(priv, data); resubmit_urb: + usb_anchor_urb(urb, &priv->urbs_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + usb_unanchor_urb(urb); + if (ret == -ENODEV) netif_device_detach(netdev); - else if (ret) + else netdev_err(netdev, "%s: failed to resubmit int urb: %pe\n", __func__, ERR_PTR(ret)); } @@ -874,9 +891,27 @@ static void f81604_write_bulk_callback(struct urb *urb) if (!netif_device_present(netdev)) return; - if (urb->status) - netdev_info(netdev, "%s: Tx URB error: %pe\n", __func__, - ERR_PTR(urb->status)); + if (!urb->status) + return; + + switch (urb->status) { + case -ENOENT: + case -ECONNRESET: + case -ESHUTDOWN: + return; + default: + break; + } + + if (net_ratelimit()) + netdev_err(netdev, "%s: Tx URB error: %pe\n", __func__, + ERR_PTR(urb->status)); + + can_free_echo_skb(netdev, 0, NULL); + netdev->stats.tx_dropped++; + netdev->stats.tx_errors++; + + netif_wake_queue(netdev); } static void f81604_clear_reg_work(struct work_struct *work) diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index 8d8a610f91441..e336703b941ef 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -610,7 +610,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) { struct gs_usb *parent = urb->context; struct gs_can *dev; - struct net_device *netdev; + struct net_device *netdev = NULL; int rc; struct net_device_stats *stats; struct gs_host_frame *hf = urb->transfer_buffer; @@ -751,7 +751,13 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) hf, parent->hf_size_rx, gs_usb_receive_bulk_callback, parent); + usb_anchor_urb(urb, &parent->rx_submitted); + rc = usb_submit_urb(urb, GFP_ATOMIC); + if (!rc) + return; + + usb_unanchor_urb(urb); /* USB failure take down all interfaces */ if (rc == -ENODEV) { @@ -760,12 +766,14 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) if (parent->canch[rc]) netif_device_detach(parent->canch[rc]->netdev); } + } else if (rc != -ESHUTDOWN && net_ratelimit()) { + netdev_info(netdev, "failed to re-submit IN URB: %pe\n", + ERR_PTR(rc)); } } -static int gs_usb_set_bittiming(struct net_device *netdev) +static int gs_usb_set_bittiming(struct gs_can *dev) { - struct gs_can *dev = netdev_priv(netdev); struct can_bittiming *bt = &dev->can.bittiming; struct gs_device_bittiming dbt = { .prop_seg = cpu_to_le32(bt->prop_seg), @@ -782,9 +790,8 @@ static int gs_usb_set_bittiming(struct net_device *netdev) GFP_KERNEL); } -static int gs_usb_set_data_bittiming(struct net_device *netdev) +static int gs_usb_set_data_bittiming(struct gs_can *dev) { - struct gs_can *dev = netdev_priv(netdev); struct can_bittiming *bt = &dev->can.fd.data_bittiming; struct gs_device_bittiming dbt = { .prop_seg = cpu_to_le32(bt->prop_seg), @@ -1048,6 +1055,20 @@ static int gs_can_open(struct net_device *netdev) if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) flags |= GS_CAN_MODE_HW_TIMESTAMP; + rc = gs_usb_set_bittiming(dev); + if (rc) { + netdev_err(netdev, "failed to set bittiming: %pe\n", ERR_PTR(rc)); + goto out_usb_kill_anchored_urbs; + } + + if (ctrlmode & CAN_CTRLMODE_FD) { + rc = gs_usb_set_data_bittiming(dev); + if (rc) { + netdev_err(netdev, "failed to set data bittiming: %pe\n", ERR_PTR(rc)); + goto out_usb_kill_anchored_urbs; + } + } + /* finally start device */ dev->can.state = CAN_STATE_ERROR_ACTIVE; dm.flags = cpu_to_le32(flags); @@ -1074,7 +1095,7 @@ static int gs_can_open(struct net_device *netdev) usb_free_urb(urb); out_usb_kill_anchored_urbs: if (!parent->active_channels) { - usb_kill_anchored_urbs(&dev->tx_submitted); + usb_kill_anchored_urbs(&parent->rx_submitted); if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) gs_usb_timestamp_stop(parent); @@ -1348,7 +1369,6 @@ static struct gs_can *gs_make_candev(unsigned int channel, dev->can.state = CAN_STATE_STOPPED; dev->can.clock.freq = le32_to_cpu(bt_const.fclk_can); dev->can.bittiming_const = &dev->bt_const; - dev->can.do_set_bittiming = gs_usb_set_bittiming; dev->can.ctrlmode_supported = CAN_CTRLMODE_CC_LEN8_DLC; @@ -1372,7 +1392,6 @@ static struct gs_can *gs_make_candev(unsigned int channel, * GS_CAN_FEATURE_BT_CONST_EXT is set. */ dev->can.fd.data_bittiming_const = &dev->bt_const; - dev->can.fd.do_set_data_bittiming = gs_usb_set_data_bittiming; } if (feature & GS_CAN_FEATURE_TERMINATION) { diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c index 90e77fa0ff4a5..74aa171268672 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c @@ -361,7 +361,14 @@ static void kvaser_usb_read_bulk_callback(struct urb *urb) urb->transfer_buffer, KVASER_USB_RX_BUFFER_SIZE, kvaser_usb_read_bulk_callback, dev); + usb_anchor_urb(urb, &dev->rx_submitted); + err = usb_submit_urb(urb, GFP_ATOMIC); + if (!err) + return; + + usb_unanchor_urb(urb); + if (err == -ENODEV) { for (i = 0; i < dev->nchannels; i++) { struct kvaser_usb_net_priv *priv; @@ -372,7 +379,7 @@ static void kvaser_usb_read_bulk_callback(struct urb *urb) netif_device_detach(priv->netdev); } - } else if (err) { + } else { dev_err(&dev->intf->dev, "Failed resubmitting read bulk urb: %d\n", err); } diff --git a/drivers/net/can/usb/mcba_usb.c b/drivers/net/can/usb/mcba_usb.c index 1f9b915094e64..40cc158c1a672 100644 --- a/drivers/net/can/usb/mcba_usb.c +++ b/drivers/net/can/usb/mcba_usb.c @@ -608,11 +608,17 @@ static void mcba_usb_read_bulk_callback(struct urb *urb) urb->transfer_buffer, MCBA_USB_RX_BUFF_SIZE, mcba_usb_read_bulk_callback, priv); + usb_anchor_urb(urb, &priv->rx_submitted); + retval = usb_submit_urb(urb, GFP_ATOMIC); + if (!retval) + return; + + usb_unanchor_urb(urb); if (retval == -ENODEV) netif_device_detach(netdev); - else if (retval) + else netdev_err(netdev, "failed resubmitting read bulk urb: %d\n", retval); } diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index 07406daf7c88e..6c90b4a7d955a 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -749,7 +749,7 @@ static void ucan_read_bulk_callback(struct urb *urb) len = le16_to_cpu(m->len); /* check sanity (length of content) */ - if (urb->actual_length - pos < len) { + if ((len == 0) || (urb->actual_length - pos < len)) { netdev_warn(up->netdev, "invalid message (short; no data; l:%d)\n", urb->actual_length); diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c index 8a5596ce4e463..0fedfc287f1f6 100644 --- a/drivers/net/can/usb/usb_8dev.c +++ b/drivers/net/can/usb/usb_8dev.c @@ -541,11 +541,17 @@ static void usb_8dev_read_bulk_callback(struct urb *urb) urb->transfer_buffer, RX_BUFFER_SIZE, usb_8dev_read_bulk_callback, priv); + usb_anchor_urb(urb, &priv->rx_submitted); + retval = usb_submit_urb(urb, GFP_ATOMIC); + if (!retval) + return; + + usb_unanchor_urb(urb); if (retval == -ENODEV) netif_device_detach(netdev); - else if (retval) + else netdev_err(netdev, "failed resubmitting read bulk urb: %d\n", retval); } diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index eb767edc4c135..7d6ec2eb7c75e 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -1830,49 +1830,83 @@ static int b53_arl_rw_op(struct b53_device *dev, unsigned int op) return b53_arl_op_wait(dev); } -static int b53_arl_read(struct b53_device *dev, u64 mac, - u16 vid, struct b53_arl_entry *ent, u8 *idx) +static void b53_arl_read_entry_25(struct b53_device *dev, + struct b53_arl_entry *ent, u8 idx) { - DECLARE_BITMAP(free_bins, B53_ARLTBL_MAX_BIN_ENTRIES); - unsigned int i; - int ret; + u8 vid_entry; + u64 mac_vid; - ret = b53_arl_op_wait(dev); - if (ret) - return ret; + b53_read8(dev, B53_ARLIO_PAGE, B53_ARLTBL_VID_ENTRY_25(idx), + &vid_entry); + b53_read64(dev, B53_ARLIO_PAGE, B53_ARLTBL_MAC_VID_ENTRY(idx), + &mac_vid); + b53_arl_to_entry_25(ent, mac_vid, vid_entry); +} - bitmap_zero(free_bins, dev->num_arl_bins); +static void b53_arl_write_entry_25(struct b53_device *dev, + const struct b53_arl_entry *ent, u8 idx) +{ + u8 vid_entry; + u64 mac_vid; - /* Read the bins */ - for (i = 0; i < dev->num_arl_bins; i++) { - u64 mac_vid; - u32 fwd_entry; + b53_arl_from_entry_25(&mac_vid, &vid_entry, ent); + b53_write8(dev, B53_ARLIO_PAGE, B53_ARLTBL_VID_ENTRY_25(idx), vid_entry); + b53_write64(dev, B53_ARLIO_PAGE, B53_ARLTBL_MAC_VID_ENTRY(idx), + mac_vid); +} + +static void b53_arl_read_entry_89(struct b53_device *dev, + struct b53_arl_entry *ent, u8 idx) +{ + u64 mac_vid; + u16 fwd_entry; - b53_read64(dev, B53_ARLIO_PAGE, - B53_ARLTBL_MAC_VID_ENTRY(i), &mac_vid); - b53_read32(dev, B53_ARLIO_PAGE, - B53_ARLTBL_DATA_ENTRY(i), &fwd_entry); - b53_arl_to_entry(ent, mac_vid, fwd_entry); + b53_read64(dev, B53_ARLIO_PAGE, B53_ARLTBL_MAC_VID_ENTRY(idx), + &mac_vid); + b53_read16(dev, B53_ARLIO_PAGE, B53_ARLTBL_DATA_ENTRY(idx), &fwd_entry); + b53_arl_to_entry_89(ent, mac_vid, fwd_entry); +} - if (!(fwd_entry & ARLTBL_VALID)) { - set_bit(i, free_bins); - continue; - } - if ((mac_vid & ARLTBL_MAC_MASK) != mac) - continue; - if (dev->vlan_enabled && - ((mac_vid >> ARLTBL_VID_S) & ARLTBL_VID_MASK) != vid) - continue; - *idx = i; - return 0; - } +static void b53_arl_write_entry_89(struct b53_device *dev, + const struct b53_arl_entry *ent, u8 idx) +{ + u32 fwd_entry; + u64 mac_vid; - *idx = find_first_bit(free_bins, dev->num_arl_bins); - return *idx >= dev->num_arl_bins ? -ENOSPC : -ENOENT; + b53_arl_from_entry_89(&mac_vid, &fwd_entry, ent); + b53_write64(dev, B53_ARLIO_PAGE, + B53_ARLTBL_MAC_VID_ENTRY(idx), mac_vid); + b53_write16(dev, B53_ARLIO_PAGE, + B53_ARLTBL_DATA_ENTRY(idx), fwd_entry); +} + +static void b53_arl_read_entry_95(struct b53_device *dev, + struct b53_arl_entry *ent, u8 idx) +{ + u32 fwd_entry; + u64 mac_vid; + + b53_read64(dev, B53_ARLIO_PAGE, B53_ARLTBL_MAC_VID_ENTRY(idx), + &mac_vid); + b53_read32(dev, B53_ARLIO_PAGE, B53_ARLTBL_DATA_ENTRY(idx), &fwd_entry); + b53_arl_to_entry(ent, mac_vid, fwd_entry); } -static int b53_arl_read_25(struct b53_device *dev, u64 mac, - u16 vid, struct b53_arl_entry *ent, u8 *idx) +static void b53_arl_write_entry_95(struct b53_device *dev, + const struct b53_arl_entry *ent, u8 idx) +{ + u32 fwd_entry; + u64 mac_vid; + + b53_arl_from_entry(&mac_vid, &fwd_entry, ent); + b53_write64(dev, B53_ARLIO_PAGE, B53_ARLTBL_MAC_VID_ENTRY(idx), + mac_vid); + b53_write32(dev, B53_ARLIO_PAGE, B53_ARLTBL_DATA_ENTRY(idx), + fwd_entry); +} + +static int b53_arl_read(struct b53_device *dev, const u8 *mac, + u16 vid, struct b53_arl_entry *ent, u8 *idx) { DECLARE_BITMAP(free_bins, B53_ARLTBL_MAX_BIN_ENTRIES); unsigned int i; @@ -1886,21 +1920,15 @@ static int b53_arl_read_25(struct b53_device *dev, u64 mac, /* Read the bins */ for (i = 0; i < dev->num_arl_bins; i++) { - u64 mac_vid; - - b53_read64(dev, B53_ARLIO_PAGE, - B53_ARLTBL_MAC_VID_ENTRY(i), &mac_vid); + b53_arl_read_entry(dev, ent, i); - b53_arl_to_entry_25(ent, mac_vid); - - if (!(mac_vid & ARLTBL_VALID_25)) { + if (!ent->is_valid) { set_bit(i, free_bins); continue; } - if ((mac_vid & ARLTBL_MAC_MASK) != mac) + if (!ether_addr_equal(ent->mac, mac)) continue; - if (dev->vlan_enabled && - ((mac_vid >> ARLTBL_VID_S_65) & ARLTBL_VID_MASK_25) != vid) + if (dev->vlan_enabled && ent->vid != vid) continue; *idx = i; return 0; @@ -1914,9 +1942,8 @@ static int b53_arl_op(struct b53_device *dev, int op, int port, const unsigned char *addr, u16 vid, bool is_valid) { struct b53_arl_entry ent; - u32 fwd_entry; - u64 mac, mac_vid = 0; u8 idx = 0; + u64 mac; int ret; /* Convert the array into a 64-bit MAC */ @@ -1924,18 +1951,19 @@ static int b53_arl_op(struct b53_device *dev, int op, int port, /* Perform a read for the given MAC and VID */ b53_write48(dev, B53_ARLIO_PAGE, B53_MAC_ADDR_IDX, mac); - if (!is5325m(dev)) - b53_write16(dev, B53_ARLIO_PAGE, B53_VLAN_ID_IDX, vid); + if (!is5325m(dev)) { + if (is5325(dev) || is5365(dev)) + b53_write8(dev, B53_ARLIO_PAGE, B53_VLAN_ID_IDX, vid); + else + b53_write16(dev, B53_ARLIO_PAGE, B53_VLAN_ID_IDX, vid); + } /* Issue a read operation for this MAC */ ret = b53_arl_rw_op(dev, 1); if (ret) return ret; - if (is5325(dev) || is5365(dev)) - ret = b53_arl_read_25(dev, mac, vid, &ent, &idx); - else - ret = b53_arl_read(dev, mac, vid, &ent, &idx); + ret = b53_arl_read(dev, addr, vid, &ent, &idx); /* If this is a read, just finish now */ if (op) @@ -1952,7 +1980,6 @@ static int b53_arl_op(struct b53_device *dev, int op, int port, /* We could not find a matching MAC, so reset to a new entry */ dev_dbg(dev->dev, "{%pM,%.4d} not found, using idx: %d\n", addr, vid, idx); - fwd_entry = 0; break; default: dev_dbg(dev->dev, "{%pM,%.4d} found, using idx: %d\n", @@ -1979,17 +2006,7 @@ static int b53_arl_op(struct b53_device *dev, int op, int port, ent.is_static = true; ent.is_age = false; memcpy(ent.mac, addr, ETH_ALEN); - if (is5325(dev) || is5365(dev)) - b53_arl_from_entry_25(&mac_vid, &ent); - else - b53_arl_from_entry(&mac_vid, &fwd_entry, &ent); - - b53_write64(dev, B53_ARLIO_PAGE, - B53_ARLTBL_MAC_VID_ENTRY(idx), mac_vid); - - if (!is5325(dev) && !is5365(dev)) - b53_write32(dev, B53_ARLIO_PAGE, - B53_ARLTBL_DATA_ENTRY(idx), fwd_entry); + b53_arl_write_entry(dev, &ent, idx); return b53_arl_rw_op(dev, 0); } @@ -2024,18 +2041,53 @@ int b53_fdb_del(struct dsa_switch *ds, int port, } EXPORT_SYMBOL(b53_fdb_del); -static int b53_arl_search_wait(struct b53_device *dev) +static void b53_read_arl_srch_ctl(struct b53_device *dev, u8 *val) { - unsigned int timeout = 1000; - u8 reg, offset; + u8 offset; + + if (is5325(dev) || is5365(dev)) + offset = B53_ARL_SRCH_CTL_25; + else if (dev->chip_id == BCM5389_DEVICE_ID || is5397_98(dev) || + is63xx(dev)) + offset = B53_ARL_SRCH_CTL_89; + else + offset = B53_ARL_SRCH_CTL; + + if (is63xx(dev)) { + u16 val16; + + b53_read16(dev, B53_ARLIO_PAGE, offset, &val16); + *val = val16 & 0xff; + } else { + b53_read8(dev, B53_ARLIO_PAGE, offset, val); + } +} + +static void b53_write_arl_srch_ctl(struct b53_device *dev, u8 val) +{ + u8 offset; if (is5325(dev) || is5365(dev)) offset = B53_ARL_SRCH_CTL_25; + else if (dev->chip_id == BCM5389_DEVICE_ID || is5397_98(dev) || + is63xx(dev)) + offset = B53_ARL_SRCH_CTL_89; else offset = B53_ARL_SRCH_CTL; + if (is63xx(dev)) + b53_write16(dev, B53_ARLIO_PAGE, offset, val); + else + b53_write8(dev, B53_ARLIO_PAGE, offset, val); +} + +static int b53_arl_search_wait(struct b53_device *dev) +{ + unsigned int timeout = 1000; + u8 reg; + do { - b53_read8(dev, B53_ARLIO_PAGE, offset, ®); + b53_read_arl_srch_ctl(dev, ®); if (!(reg & ARL_SRCH_STDN)) return -ENOENT; @@ -2048,28 +2100,53 @@ static int b53_arl_search_wait(struct b53_device *dev) return -ETIMEDOUT; } -static void b53_arl_search_rd(struct b53_device *dev, u8 idx, - struct b53_arl_entry *ent) +static void b53_arl_search_read_25(struct b53_device *dev, u8 idx, + struct b53_arl_entry *ent) { u64 mac_vid; + u8 ext; - if (is5325(dev)) { - b53_read64(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSTL_0_MACVID_25, - &mac_vid); - b53_arl_to_entry_25(ent, mac_vid); - } else if (is5365(dev)) { - b53_read64(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSTL_0_MACVID_65, - &mac_vid); - b53_arl_to_entry_25(ent, mac_vid); - } else { - u32 fwd_entry; + b53_read8(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSLT_EXT_25, &ext); + b53_read64(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSTL_0_MACVID_25, + &mac_vid); + b53_arl_search_to_entry_25(ent, mac_vid, ext); +} - b53_read64(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSTL_MACVID(idx), - &mac_vid); - b53_read32(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSTL(idx), - &fwd_entry); - b53_arl_to_entry(ent, mac_vid, fwd_entry); - } +static void b53_arl_search_read_89(struct b53_device *dev, u8 idx, + struct b53_arl_entry *ent) +{ + u16 fwd_entry; + u64 mac_vid; + + b53_read64(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSLT_MACVID_89, + &mac_vid); + b53_read16(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSLT_89, &fwd_entry); + b53_arl_to_entry_89(ent, mac_vid, fwd_entry); +} + +static void b53_arl_search_read_63xx(struct b53_device *dev, u8 idx, + struct b53_arl_entry *ent) +{ + u16 fwd_entry; + u64 mac_vid; + + b53_read64(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSLT_MACVID_63XX, + &mac_vid); + b53_read16(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSLT_63XX, &fwd_entry); + b53_arl_search_to_entry_63xx(ent, mac_vid, fwd_entry); +} + +static void b53_arl_search_read_95(struct b53_device *dev, u8 idx, + struct b53_arl_entry *ent) +{ + u32 fwd_entry; + u64 mac_vid; + + b53_read64(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSTL_MACVID(idx), + &mac_vid); + b53_read32(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_RSTL(idx), + &fwd_entry); + b53_arl_to_entry(ent, mac_vid, fwd_entry); } static int b53_fdb_copy(int port, const struct b53_arl_entry *ent, @@ -2078,6 +2155,9 @@ static int b53_fdb_copy(int port, const struct b53_arl_entry *ent, if (!ent->is_valid) return 0; + if (is_multicast_ether_addr(ent->mac)) + return 0; + if (port != ent->port) return 0; @@ -2090,36 +2170,28 @@ int b53_fdb_dump(struct dsa_switch *ds, int port, unsigned int count = 0, results_per_hit = 1; struct b53_device *priv = ds->priv; struct b53_arl_entry results[2]; - u8 offset; int ret; - u8 reg; if (priv->num_arl_bins > 2) results_per_hit = 2; mutex_lock(&priv->arl_mutex); - if (is5325(priv) || is5365(priv)) - offset = B53_ARL_SRCH_CTL_25; - else - offset = B53_ARL_SRCH_CTL; - /* Start search operation */ - reg = ARL_SRCH_STDN; - b53_write8(priv, B53_ARLIO_PAGE, offset, reg); + b53_write_arl_srch_ctl(priv, ARL_SRCH_STDN); do { ret = b53_arl_search_wait(priv); if (ret) break; - b53_arl_search_rd(priv, 0, &results[0]); + b53_arl_search_read(priv, 0, &results[0]); ret = b53_fdb_copy(port, &results[0], cb, data); if (ret) break; if (results_per_hit == 2) { - b53_arl_search_rd(priv, 1, &results[1]); + b53_arl_search_read(priv, 1, &results[1]); ret = b53_fdb_copy(port, &results[1], cb, data); if (ret) break; @@ -2645,6 +2717,30 @@ static const struct dsa_switch_ops b53_switch_ops = { .port_change_mtu = b53_change_mtu, }; +static const struct b53_arl_ops b53_arl_ops_25 = { + .arl_read_entry = b53_arl_read_entry_25, + .arl_write_entry = b53_arl_write_entry_25, + .arl_search_read = b53_arl_search_read_25, +}; + +static const struct b53_arl_ops b53_arl_ops_89 = { + .arl_read_entry = b53_arl_read_entry_89, + .arl_write_entry = b53_arl_write_entry_89, + .arl_search_read = b53_arl_search_read_89, +}; + +static const struct b53_arl_ops b53_arl_ops_63xx = { + .arl_read_entry = b53_arl_read_entry_89, + .arl_write_entry = b53_arl_write_entry_89, + .arl_search_read = b53_arl_search_read_63xx, +}; + +static const struct b53_arl_ops b53_arl_ops_95 = { + .arl_read_entry = b53_arl_read_entry_95, + .arl_write_entry = b53_arl_write_entry_95, + .arl_search_read = b53_arl_search_read_95, +}; + struct b53_chip_data { u32 chip_id; const char *dev_name; @@ -2658,6 +2754,7 @@ struct b53_chip_data { u8 duplex_reg; u8 jumbo_pm_reg; u8 jumbo_size_reg; + const struct b53_arl_ops *arl_ops; }; #define B53_VTA_REGS \ @@ -2677,6 +2774,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .arl_buckets = 1024, .imp_port = 5, .duplex_reg = B53_DUPLEX_STAT_FE, + .arl_ops = &b53_arl_ops_25, }, { .chip_id = BCM5365_DEVICE_ID, @@ -2687,6 +2785,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .arl_buckets = 1024, .imp_port = 5, .duplex_reg = B53_DUPLEX_STAT_FE, + .arl_ops = &b53_arl_ops_25, }, { .chip_id = BCM5389_DEVICE_ID, @@ -2700,6 +2799,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_89, }, { .chip_id = BCM5395_DEVICE_ID, @@ -2713,6 +2813,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM5397_DEVICE_ID, @@ -2726,6 +2827,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_89, }, { .chip_id = BCM5398_DEVICE_ID, @@ -2739,6 +2841,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_89, }, { .chip_id = BCM53101_DEVICE_ID, @@ -2752,6 +2855,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53115_DEVICE_ID, @@ -2765,6 +2869,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53125_DEVICE_ID, @@ -2778,6 +2883,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53128_DEVICE_ID, @@ -2791,19 +2897,21 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM63XX_DEVICE_ID, .dev_name = "BCM63xx", .vlans = 4096, .enabled_ports = 0, /* pdata must provide them */ - .arl_bins = 4, - .arl_buckets = 1024, + .arl_bins = 1, + .arl_buckets = 4096, .imp_port = 8, .vta_regs = B53_VTA_REGS_63XX, .duplex_reg = B53_DUPLEX_STAT_63XX, .jumbo_pm_reg = B53_JUMBO_PORT_MASK_63XX, .jumbo_size_reg = B53_JUMBO_MAX_SIZE_63XX, + .arl_ops = &b53_arl_ops_63xx, }, { .chip_id = BCM53010_DEVICE_ID, @@ -2817,6 +2925,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53011_DEVICE_ID, @@ -2830,6 +2939,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53012_DEVICE_ID, @@ -2843,6 +2953,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53018_DEVICE_ID, @@ -2856,6 +2967,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53019_DEVICE_ID, @@ -2869,6 +2981,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM58XX_DEVICE_ID, @@ -2882,6 +2995,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM583XX_DEVICE_ID, @@ -2895,6 +3009,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, /* Starfighter 2 */ { @@ -2909,6 +3024,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM7445_DEVICE_ID, @@ -2922,6 +3038,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM7278_DEVICE_ID, @@ -2935,6 +3052,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, { .chip_id = BCM53134_DEVICE_ID, @@ -2949,6 +3067,7 @@ static const struct b53_chip_data b53_switch_chips[] = { .duplex_reg = B53_DUPLEX_STAT_GE, .jumbo_pm_reg = B53_JUMBO_PORT_MASK, .jumbo_size_reg = B53_JUMBO_MAX_SIZE, + .arl_ops = &b53_arl_ops_95, }, }; @@ -2977,6 +3096,7 @@ static int b53_switch_init(struct b53_device *dev) dev->num_vlans = chip->vlans; dev->num_arl_bins = chip->arl_bins; dev->num_arl_buckets = chip->arl_buckets; + dev->arl_ops = chip->arl_ops; break; } } diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h index 458775f951643..bd6849e5bb939 100644 --- a/drivers/net/dsa/b53/b53_priv.h +++ b/drivers/net/dsa/b53/b53_priv.h @@ -58,6 +58,17 @@ struct b53_io_ops { bool link_up); }; +struct b53_arl_entry; + +struct b53_arl_ops { + void (*arl_read_entry)(struct b53_device *dev, + struct b53_arl_entry *ent, u8 idx); + void (*arl_write_entry)(struct b53_device *dev, + const struct b53_arl_entry *ent, u8 idx); + void (*arl_search_read)(struct b53_device *dev, u8 idx, + struct b53_arl_entry *ent); +}; + #define B53_INVALID_LANE 0xff enum { @@ -127,6 +138,7 @@ struct b53_device { struct mutex stats_mutex; struct mutex arl_mutex; const struct b53_io_ops *ops; + const struct b53_arl_ops *arl_ops; /* chip specific data */ u32 chip_id; @@ -329,16 +341,30 @@ static inline void b53_arl_to_entry(struct b53_arl_entry *ent, } static inline void b53_arl_to_entry_25(struct b53_arl_entry *ent, - u64 mac_vid) + u64 mac_vid, u8 vid_entry) { memset(ent, 0, sizeof(*ent)); - ent->port = (mac_vid >> ARLTBL_DATA_PORT_ID_S_25) & - ARLTBL_DATA_PORT_ID_MASK_25; ent->is_valid = !!(mac_vid & ARLTBL_VALID_25); ent->is_age = !!(mac_vid & ARLTBL_AGE_25); ent->is_static = !!(mac_vid & ARLTBL_STATIC_25); u64_to_ether_addr(mac_vid, ent->mac); - ent->vid = mac_vid >> ARLTBL_VID_S_65; + ent->port = (mac_vid & ARLTBL_DATA_PORT_ID_MASK_25) >> + ARLTBL_DATA_PORT_ID_S_25; + if (is_unicast_ether_addr(ent->mac) && ent->port == B53_CPU_PORT) + ent->port = B53_CPU_PORT_25; + ent->vid = vid_entry; +} + +static inline void b53_arl_to_entry_89(struct b53_arl_entry *ent, + u64 mac_vid, u16 fwd_entry) +{ + memset(ent, 0, sizeof(*ent)); + ent->port = fwd_entry & ARLTBL_DATA_PORT_ID_MASK_89; + ent->is_valid = !!(fwd_entry & ARLTBL_VALID_89); + ent->is_age = !!(fwd_entry & ARLTBL_AGE_89); + ent->is_static = !!(fwd_entry & ARLTBL_STATIC_89); + u64_to_ether_addr(mac_vid, ent->mac); + ent->vid = mac_vid >> ARLTBL_VID_S; } static inline void b53_arl_from_entry(u64 *mac_vid, u32 *fwd_entry, @@ -355,20 +381,87 @@ static inline void b53_arl_from_entry(u64 *mac_vid, u32 *fwd_entry, *fwd_entry |= ARLTBL_AGE; } -static inline void b53_arl_from_entry_25(u64 *mac_vid, +static inline void b53_arl_from_entry_25(u64 *mac_vid, u8 *vid_entry, const struct b53_arl_entry *ent) { *mac_vid = ether_addr_to_u64(ent->mac); - *mac_vid |= (u64)(ent->port & ARLTBL_DATA_PORT_ID_MASK_25) << - ARLTBL_DATA_PORT_ID_S_25; - *mac_vid |= (u64)(ent->vid & ARLTBL_VID_MASK_25) << - ARLTBL_VID_S_65; + if (is_unicast_ether_addr(ent->mac) && ent->port == B53_CPU_PORT_25) + *mac_vid |= (u64)B53_CPU_PORT << ARLTBL_DATA_PORT_ID_S_25; + else + *mac_vid |= ((u64)ent->port << ARLTBL_DATA_PORT_ID_S_25) & + ARLTBL_DATA_PORT_ID_MASK_25; if (ent->is_valid) *mac_vid |= ARLTBL_VALID_25; if (ent->is_static) *mac_vid |= ARLTBL_STATIC_25; if (ent->is_age) *mac_vid |= ARLTBL_AGE_25; + *vid_entry = ent->vid; +} + +static inline void b53_arl_from_entry_89(u64 *mac_vid, u32 *fwd_entry, + const struct b53_arl_entry *ent) +{ + *mac_vid = ether_addr_to_u64(ent->mac); + *mac_vid |= (u64)(ent->vid & ARLTBL_VID_MASK) << ARLTBL_VID_S; + *fwd_entry = ent->port & ARLTBL_DATA_PORT_ID_MASK_89; + if (ent->is_valid) + *fwd_entry |= ARLTBL_VALID_89; + if (ent->is_static) + *fwd_entry |= ARLTBL_STATIC_89; + if (ent->is_age) + *fwd_entry |= ARLTBL_AGE_89; +} + +static inline void b53_arl_search_to_entry_25(struct b53_arl_entry *ent, + u64 mac_vid, u8 ext) +{ + memset(ent, 0, sizeof(*ent)); + ent->is_valid = !!(mac_vid & ARLTBL_VALID_25); + ent->is_age = !!(mac_vid & ARLTBL_AGE_25); + ent->is_static = !!(mac_vid & ARLTBL_STATIC_25); + u64_to_ether_addr(mac_vid, ent->mac); + ent->vid = (mac_vid & ARL_SRCH_RSLT_VID_MASK_25) >> + ARL_SRCH_RSLT_VID_S_25; + ent->port = (mac_vid & ARL_SRCH_RSLT_PORT_ID_MASK_25) >> + ARL_SRCH_RSLT_PORT_ID_S_25; + if (is_multicast_ether_addr(ent->mac) && (ext & ARL_SRCH_RSLT_EXT_MC_MII)) + ent->port |= BIT(B53_CPU_PORT_25); + else if (!is_multicast_ether_addr(ent->mac) && ent->port == B53_CPU_PORT) + ent->port = B53_CPU_PORT_25; +} + +static inline void b53_arl_search_to_entry_63xx(struct b53_arl_entry *ent, + u64 mac_vid, u16 fwd_entry) +{ + memset(ent, 0, sizeof(*ent)); + u64_to_ether_addr(mac_vid, ent->mac); + ent->vid = mac_vid >> ARLTBL_VID_S; + + ent->port = fwd_entry & ARL_SRST_PORT_ID_MASK_63XX; + ent->port >>= 1; + + ent->is_age = !!(fwd_entry & ARL_SRST_AGE_63XX); + ent->is_static = !!(fwd_entry & ARL_SRST_STATIC_63XX); + ent->is_valid = 1; +} + +static inline void b53_arl_read_entry(struct b53_device *dev, + struct b53_arl_entry *ent, u8 idx) +{ + dev->arl_ops->arl_read_entry(dev, ent, idx); +} + +static inline void b53_arl_write_entry(struct b53_device *dev, + const struct b53_arl_entry *ent, u8 idx) +{ + dev->arl_ops->arl_write_entry(dev, ent, idx); +} + +static inline void b53_arl_search_read(struct b53_device *dev, u8 idx, + struct b53_arl_entry *ent) +{ + dev->arl_ops->arl_search_read(dev, idx, ent); } #ifdef CONFIG_BCM47XX diff --git a/drivers/net/dsa/b53/b53_regs.h b/drivers/net/dsa/b53/b53_regs.h index 8ce1ce72e9385..b6fe7d207a2c1 100644 --- a/drivers/net/dsa/b53/b53_regs.h +++ b/drivers/net/dsa/b53/b53_regs.h @@ -325,11 +325,9 @@ #define B53_ARLTBL_MAC_VID_ENTRY(n) ((0x10 * (n)) + 0x10) #define ARLTBL_MAC_MASK 0xffffffffffffULL #define ARLTBL_VID_S 48 -#define ARLTBL_VID_MASK_25 0xff #define ARLTBL_VID_MASK 0xfff #define ARLTBL_DATA_PORT_ID_S_25 48 -#define ARLTBL_DATA_PORT_ID_MASK_25 0xf -#define ARLTBL_VID_S_65 53 +#define ARLTBL_DATA_PORT_ID_MASK_25 GENMASK_ULL(53, 48) #define ARLTBL_AGE_25 BIT_ULL(61) #define ARLTBL_STATIC_25 BIT_ULL(62) #define ARLTBL_VALID_25 BIT_ULL(63) @@ -342,12 +340,23 @@ #define ARLTBL_STATIC BIT(15) #define ARLTBL_VALID BIT(16) +/* BCM5389 ARL Table Data Entry N Register format (16 bit) */ +#define ARLTBL_DATA_PORT_ID_MASK_89 GENMASK(8, 0) +#define ARLTBL_TC_MASK_89 GENMASK(12, 10) +#define ARLTBL_AGE_89 BIT(13) +#define ARLTBL_STATIC_89 BIT(14) +#define ARLTBL_VALID_89 BIT(15) + +/* BCM5325/BCM565 ARL Table VID Entry N Registers (8 bit) */ +#define B53_ARLTBL_VID_ENTRY_25(n) ((0x2 * (n)) + 0x30) + /* Maximum number of bin entries in the ARL for all switches */ #define B53_ARLTBL_MAX_BIN_ENTRIES 4 /* ARL Search Control Register (8 bit) */ #define B53_ARL_SRCH_CTL 0x50 #define B53_ARL_SRCH_CTL_25 0x20 +#define B53_ARL_SRCH_CTL_89 0x30 #define ARL_SRCH_VLID BIT(0) #define ARL_SRCH_STDN BIT(7) @@ -355,22 +364,42 @@ #define B53_ARL_SRCH_ADDR 0x51 #define B53_ARL_SRCH_ADDR_25 0x22 #define B53_ARL_SRCH_ADDR_65 0x24 +#define B53_ARL_SRCH_ADDR_89 0x31 +#define B53_ARL_SRCH_ADDR_63XX 0x32 #define ARL_ADDR_MASK GENMASK(14, 0) /* ARL Search MAC/VID Result (64 bit) */ #define B53_ARL_SRCH_RSTL_0_MACVID 0x60 +#define B53_ARL_SRCH_RSLT_MACVID_89 0x33 +#define B53_ARL_SRCH_RSLT_MACVID_63XX 0x34 -/* Single register search result on 5325 */ +/* Single register search result on 5325/5365 */ #define B53_ARL_SRCH_RSTL_0_MACVID_25 0x24 -/* Single register search result on 5365 */ -#define B53_ARL_SRCH_RSTL_0_MACVID_65 0x30 +#define ARL_SRCH_RSLT_PORT_ID_S_25 48 +#define ARL_SRCH_RSLT_PORT_ID_MASK_25 GENMASK_ULL(52, 48) +#define ARL_SRCH_RSLT_VID_S_25 53 +#define ARL_SRCH_RSLT_VID_MASK_25 GENMASK_ULL(60, 53) + +/* BCM5325/5365 Search result extend register (8 bit) */ +#define B53_ARL_SRCH_RSLT_EXT_25 0x2c +#define ARL_SRCH_RSLT_EXT_MC_MII BIT(2) /* ARL Search Data Result (32 bit) */ #define B53_ARL_SRCH_RSTL_0 0x68 +/* BCM5389 ARL Search Data Result (16 bit) */ +#define B53_ARL_SRCH_RSLT_89 0x3b + #define B53_ARL_SRCH_RSTL_MACVID(x) (B53_ARL_SRCH_RSTL_0_MACVID + ((x) * 0x10)) #define B53_ARL_SRCH_RSTL(x) (B53_ARL_SRCH_RSTL_0 + ((x) * 0x10)) +/* 63XX ARL Search Data Result (16 bit) */ +#define B53_ARL_SRCH_RSLT_63XX 0x3c +#define ARL_SRST_PORT_ID_MASK_63XX GENMASK(9, 1) +#define ARL_SRST_TC_MASK_63XX GENMASK(13, 11) +#define ARL_SRST_AGE_63XX BIT(14) +#define ARL_SRST_STATIC_63XX BIT(15) + /************************************************************************* * IEEE 802.1X Registers *************************************************************************/ diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index 960685596093b..de3efa3ce9a75 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -980,15 +980,19 @@ static int bcm_sf2_sw_resume(struct dsa_switch *ds) ret = bcm_sf2_sw_rst(priv); if (ret) { pr_err("%s: failed to software reset switch\n", __func__); + if (!priv->wol_ports_mask) + clk_disable_unprepare(priv->clk); return ret; } bcm_sf2_crossbar_setup(priv); ret = bcm_sf2_cfp_resume(ds); - if (ret) + if (ret) { + if (!priv->wol_ports_mask) + clk_disable_unprepare(priv->clk); return ret; - + } if (priv->hw_params.num_gphy == 1) bcm_sf2_gphy_enable_set(ds, true); diff --git a/drivers/net/dsa/microchip/ksz_ptp.c b/drivers/net/dsa/microchip/ksz_ptp.c index 997e4a76d0a68..318ab38ac49b2 100644 --- a/drivers/net/dsa/microchip/ksz_ptp.c +++ b/drivers/net/dsa/microchip/ksz_ptp.c @@ -1095,6 +1095,7 @@ static int ksz_ptp_msg_irq_setup(struct ksz_port *port, u8 n) const struct ksz_dev_ops *ops = port->ksz_dev->dev_ops; struct ksz_irq *ptpirq = &port->ptpirq; struct ksz_ptp_irq *ptpmsg_irq; + int ret; ptpmsg_irq = &port->ptpmsg_irq[n]; ptpmsg_irq->num = irq_create_mapping(ptpirq->domain, n); @@ -1106,9 +1107,13 @@ static int ksz_ptp_msg_irq_setup(struct ksz_port *port, u8 n) strscpy(ptpmsg_irq->name, name[n]); - return request_threaded_irq(ptpmsg_irq->num, NULL, - ksz_ptp_msg_thread_fn, IRQF_ONESHOT, - ptpmsg_irq->name, ptpmsg_irq); + ret = request_threaded_irq(ptpmsg_irq->num, NULL, + ksz_ptp_msg_thread_fn, IRQF_ONESHOT, + ptpmsg_irq->name, ptpmsg_irq); + if (ret) + irq_dispose_mapping(ptpmsg_irq->num); + + return ret; } int ksz_ptp_irq_setup(struct dsa_switch *ds, u8 p) diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index b4d48997bf467..09002c853b78e 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -3364,13 +3364,10 @@ static int mv88e6xxx_setup_upstream_port(struct mv88e6xxx_chip *chip, int port) static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port) { - struct device_node *phy_handle = NULL; struct fwnode_handle *ports_fwnode; struct fwnode_handle *port_fwnode; struct dsa_switch *ds = chip->ds; struct mv88e6xxx_port *p; - struct dsa_port *dp; - int tx_amp; int err; u16 reg; u32 val; @@ -3582,23 +3579,6 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port) return err; } - if (chip->info->ops->serdes_set_tx_amplitude) { - dp = dsa_to_port(ds, port); - if (dp) - phy_handle = of_parse_phandle(dp->dn, "phy-handle", 0); - - if (phy_handle && !of_property_read_u32(phy_handle, - "tx-p2p-microvolt", - &tx_amp)) - err = chip->info->ops->serdes_set_tx_amplitude(chip, - port, tx_amp); - if (phy_handle) { - of_node_put(phy_handle); - if (err) - return err; - } - } - /* Port based VLAN map: give each port the same default address * database, and allow bidirectional communication between the * CPU and DSA port(s), and the other ports. @@ -4768,7 +4748,6 @@ static const struct mv88e6xxx_ops mv88e6176_ops = { .serdes_irq_mapping = mv88e6352_serdes_irq_mapping, .serdes_get_regs_len = mv88e6352_serdes_get_regs_len, .serdes_get_regs = mv88e6352_serdes_get_regs, - .serdes_set_tx_amplitude = mv88e6352_serdes_set_tx_amplitude, .gpio_ops = &mv88e6352_gpio_ops, .phylink_get_caps = mv88e6352_phylink_get_caps, .pcs_ops = &mv88e6352_pcs_ops, @@ -5044,7 +5023,6 @@ static const struct mv88e6xxx_ops mv88e6240_ops = { .serdes_irq_mapping = mv88e6352_serdes_irq_mapping, .serdes_get_regs_len = mv88e6352_serdes_get_regs_len, .serdes_get_regs = mv88e6352_serdes_get_regs, - .serdes_set_tx_amplitude = mv88e6352_serdes_set_tx_amplitude, .gpio_ops = &mv88e6352_gpio_ops, .avb_ops = &mv88e6352_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, @@ -5481,7 +5459,6 @@ static const struct mv88e6xxx_ops mv88e6352_ops = { .serdes_get_stats = mv88e6352_serdes_get_stats, .serdes_get_regs_len = mv88e6352_serdes_get_regs_len, .serdes_get_regs = mv88e6352_serdes_get_regs, - .serdes_set_tx_amplitude = mv88e6352_serdes_set_tx_amplitude, .phylink_get_caps = mv88e6352_phylink_get_caps, .pcs_ops = &mv88e6352_pcs_ops, }; diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h index 2f211e55cb47b..e073446ee7d02 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.h +++ b/drivers/net/dsa/mv88e6xxx/chip.h @@ -642,10 +642,6 @@ struct mv88e6xxx_ops { void (*serdes_get_regs)(struct mv88e6xxx_chip *chip, int port, void *_p); - /* SERDES SGMII/Fiber Output Amplitude */ - int (*serdes_set_tx_amplitude)(struct mv88e6xxx_chip *chip, int port, - int val); - /* Address Translation Unit operations */ int (*atu_get_hash)(struct mv88e6xxx_chip *chip, u8 *hash); int (*atu_set_hash)(struct mv88e6xxx_chip *chip, u8 hash); diff --git a/drivers/net/dsa/mv88e6xxx/serdes.c b/drivers/net/dsa/mv88e6xxx/serdes.c index b3330211edbca..a936ee80ce006 100644 --- a/drivers/net/dsa/mv88e6xxx/serdes.c +++ b/drivers/net/dsa/mv88e6xxx/serdes.c @@ -25,14 +25,6 @@ static int mv88e6352_serdes_read(struct mv88e6xxx_chip *chip, int reg, reg, val); } -static int mv88e6352_serdes_write(struct mv88e6xxx_chip *chip, int reg, - u16 val) -{ - return mv88e6xxx_phy_page_write(chip, MV88E6352_ADDR_SERDES, - MV88E6352_SERDES_PAGE_FIBER, - reg, val); -} - static int mv88e6390_serdes_read(struct mv88e6xxx_chip *chip, int lane, int device, int reg, u16 *val) { @@ -506,41 +498,3 @@ void mv88e6390_serdes_get_regs(struct mv88e6xxx_chip *chip, int port, void *_p) p[i] = reg; } } - -static const int mv88e6352_serdes_p2p_to_reg[] = { - /* Index of value in microvolts corresponds to the register value */ - 14000, 112000, 210000, 308000, 406000, 504000, 602000, 700000, -}; - -int mv88e6352_serdes_set_tx_amplitude(struct mv88e6xxx_chip *chip, int port, - int val) -{ - bool found = false; - u16 ctrl, reg; - int err; - int i; - - err = mv88e6352_g2_scratch_port_has_serdes(chip, port); - if (err <= 0) - return err; - - for (i = 0; i < ARRAY_SIZE(mv88e6352_serdes_p2p_to_reg); ++i) { - if (mv88e6352_serdes_p2p_to_reg[i] == val) { - reg = i; - found = true; - break; - } - } - - if (!found) - return -EINVAL; - - err = mv88e6352_serdes_read(chip, MV88E6352_SERDES_SPEC_CTRL2, &ctrl); - if (err) - return err; - - ctrl &= ~MV88E6352_SERDES_OUT_AMP_MASK; - ctrl |= reg; - - return mv88e6352_serdes_write(chip, MV88E6352_SERDES_SPEC_CTRL2, ctrl); -} diff --git a/drivers/net/dsa/mv88e6xxx/serdes.h b/drivers/net/dsa/mv88e6xxx/serdes.h index ad887d8601bcb..17a3e85fabaa3 100644 --- a/drivers/net/dsa/mv88e6xxx/serdes.h +++ b/drivers/net/dsa/mv88e6xxx/serdes.h @@ -29,8 +29,6 @@ struct phylink_link_state; #define MV88E6352_SERDES_INT_FIBRE_ENERGY BIT(4) #define MV88E6352_SERDES_INT_STATUS 0x13 -#define MV88E6352_SERDES_SPEC_CTRL2 0x1a -#define MV88E6352_SERDES_OUT_AMP_MASK 0x0007 #define MV88E6341_PORT5_LANE 0x15 @@ -140,9 +138,6 @@ void mv88e6352_serdes_get_regs(struct mv88e6xxx_chip *chip, int port, void *_p); int mv88e6390_serdes_get_regs_len(struct mv88e6xxx_chip *chip, int port); void mv88e6390_serdes_get_regs(struct mv88e6xxx_chip *chip, int port, void *_p); -int mv88e6352_serdes_set_tx_amplitude(struct mv88e6xxx_chip *chip, int port, - int val); - /* Return the (first) SERDES lane address a port is using, -errno otherwise. */ static inline int mv88e6xxx_serdes_get_lane(struct mv88e6xxx_chip *chip, int port) diff --git a/drivers/net/dsa/realtek/rtl8365mb.c b/drivers/net/dsa/realtek/rtl8365mb.c index 964a56ee16cc9..3a48db295e7e4 100644 --- a/drivers/net/dsa/realtek/rtl8365mb.c +++ b/drivers/net/dsa/realtek/rtl8365mb.c @@ -769,7 +769,7 @@ static int rtl8365mb_phy_ocp_write(struct realtek_priv *priv, int phy, out: rtl83xx_unlock(priv); - return 0; + return ret; } static int rtl8365mb_phy_read(struct realtek_priv *priv, int phy, int regnum) @@ -1480,8 +1480,7 @@ static void rtl8365mb_stats_update(struct realtek_priv *priv, int port) stats->rx_packets = cnt[RTL8365MB_MIB_ifInUcastPkts] + cnt[RTL8365MB_MIB_ifInMulticastPkts] + - cnt[RTL8365MB_MIB_ifInBroadcastPkts] - - cnt[RTL8365MB_MIB_ifOutDiscards]; + cnt[RTL8365MB_MIB_ifInBroadcastPkts]; stats->tx_packets = cnt[RTL8365MB_MIB_ifOutUcastPkts] + cnt[RTL8365MB_MIB_ifOutMulticastPkts] + diff --git a/drivers/net/dsa/realtek/rtl8366rb-leds.c b/drivers/net/dsa/realtek/rtl8366rb-leds.c index 99c890681ae60..509ffd3f8db5c 100644 --- a/drivers/net/dsa/realtek/rtl8366rb-leds.c +++ b/drivers/net/dsa/realtek/rtl8366rb-leds.c @@ -12,11 +12,11 @@ static inline u32 rtl8366rb_led_group_port_mask(u8 led_group, u8 port) case 0: return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); case 1: - return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); + return FIELD_PREP(RTL8366RB_LED_X_1_CTRL_MASK, BIT(port)); case 2: - return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); + return FIELD_PREP(RTL8366RB_LED_2_X_CTRL_MASK, BIT(port)); case 3: - return FIELD_PREP(RTL8366RB_LED_0_X_CTRL_MASK, BIT(port)); + return FIELD_PREP(RTL8366RB_LED_X_3_CTRL_MASK, BIT(port)); default: return 0; } diff --git a/drivers/net/dsa/xrs700x/xrs700x.c b/drivers/net/dsa/xrs700x/xrs700x.c index 4dbcc49a9e526..0a05f4156ef4d 100644 --- a/drivers/net/dsa/xrs700x/xrs700x.c +++ b/drivers/net/dsa/xrs700x/xrs700x.c @@ -566,6 +566,7 @@ static int xrs700x_hsr_join(struct dsa_switch *ds, int port, struct xrs700x *priv = ds->priv; struct net_device *user; int ret, i, hsr_pair[2]; + enum hsr_port_type type; enum hsr_version ver; bool fwd = false; @@ -589,6 +590,16 @@ static int xrs700x_hsr_join(struct dsa_switch *ds, int port, return -EOPNOTSUPP; } + ret = hsr_get_port_type(hsr, dsa_to_port(ds, port)->user, &type); + if (ret) + return ret; + + if (type != HSR_PT_SLAVE_A && type != HSR_PT_SLAVE_B) { + NL_SET_ERR_MSG_MOD(extack, + "Only HSR slave ports can be offloaded"); + return -EOPNOTSUPP; + } + dsa_hsr_foreach_port(dp, ds, hsr) { if (dp->index != port) { partner = dp; diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c index 8c9cc97efd4ee..4fe4efdb3737f 100644 --- a/drivers/net/ethernet/3com/3c59x.c +++ b/drivers/net/ethernet/3com/3c59x.c @@ -1473,7 +1473,7 @@ static int vortex_probe1(struct device *gendev, void __iomem *ioaddr, int irq, return 0; free_ring: - dma_free_coherent(&pdev->dev, + dma_free_coherent(gendev, sizeof(struct boom_rx_desc) * RX_RING_SIZE + sizeof(struct boom_tx_desc) * TX_RING_SIZE, vp->rx_ring, vp->rx_ring_dma); diff --git a/drivers/net/ethernet/adi/adin1110.c b/drivers/net/ethernet/adi/adin1110.c index 30f9d271e5953..71a2397edf2bb 100644 --- a/drivers/net/ethernet/adi/adin1110.c +++ b/drivers/net/ethernet/adi/adin1110.c @@ -1089,6 +1089,9 @@ static int adin1110_check_spi(struct adin1110_priv *priv) reset_gpio = devm_gpiod_get_optional(&priv->spidev->dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(reset_gpio)) + return dev_err_probe(&priv->spidev->dev, PTR_ERR(reset_gpio), + "failed to get reset gpio\n"); if (reset_gpio) { /* MISO pin is used for internal configuration, can't have * anyone else disturbing the SDO line. diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 433a646e98317..bdf600fea9508 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -709,9 +709,8 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) if (q->skb) { dev_kfree_skb(q->skb); q->skb = NULL; - } else { - page_pool_put_full_page(q->page_pool, page, true); } + page_pool_put_full_page(q->page_pool, page, true); } airoha_qdma_fill_rx_queue(q); @@ -806,18 +805,34 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q, static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q) { - struct airoha_eth *eth = q->qdma->eth; + struct airoha_qdma *qdma = q->qdma; + struct airoha_eth *eth = qdma->eth; + int qid = q - &qdma->q_rx[0]; while (q->queued) { struct airoha_queue_entry *e = &q->entry[q->tail]; + struct airoha_qdma_desc *desc = &q->desc[q->tail]; struct page *page = virt_to_head_page(e->buf); dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, page_pool_get_dma_dir(q->page_pool)); page_pool_put_full_page(q->page_pool, page, false); + /* Reset DMA descriptor */ + WRITE_ONCE(desc->ctrl, 0); + WRITE_ONCE(desc->addr, 0); + WRITE_ONCE(desc->data, 0); + WRITE_ONCE(desc->msg0, 0); + WRITE_ONCE(desc->msg1, 0); + WRITE_ONCE(desc->msg2, 0); + WRITE_ONCE(desc->msg3, 0); + q->tail = (q->tail + 1) % q->ndesc; q->queued--; } + + q->head = q->tail; + airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, + FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail)); } static int airoha_qdma_init_rx(struct airoha_qdma *qdma) @@ -2900,19 +2915,26 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth, port->id = id; eth->ports[p] = port; - err = airoha_metadata_dst_alloc(port); - if (err) - return err; + return airoha_metadata_dst_alloc(port); +} - err = register_netdev(dev); - if (err) - goto free_metadata_dst; +static int airoha_register_gdm_devices(struct airoha_eth *eth) +{ + int i; - return 0; + for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { + struct airoha_gdm_port *port = eth->ports[i]; + int err; -free_metadata_dst: - airoha_metadata_dst_free(port); - return err; + if (!port) + continue; + + err = register_netdev(port->dev); + if (err) + return err; + } + + return 0; } static int airoha_probe(struct platform_device *pdev) @@ -2993,6 +3015,10 @@ static int airoha_probe(struct platform_device *pdev) } } + err = airoha_register_gdm_devices(eth); + if (err) + goto error_napi_stop; + return 0; error_napi_stop: @@ -3006,10 +3032,12 @@ static int airoha_probe(struct platform_device *pdev) for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { struct airoha_gdm_port *port = eth->ports[i]; - if (port && port->dev->reg_state == NETREG_REGISTERED) { + if (!port) + continue; + + if (port->dev->reg_state == NETREG_REGISTERED) unregister_netdev(port->dev); - airoha_metadata_dst_free(port); - } + airoha_metadata_dst_free(port); } free_netdev(eth->napi_dev); platform_set_drvdata(pdev, NULL); @@ -3033,7 +3061,6 @@ static void airoha_remove(struct platform_device *pdev) if (!port) continue; - airoha_dev_stop(port->dev); unregister_netdev(port->dev); airoha_metadata_dst_free(port); } diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index c0e17035db18e..b3632f964a580 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -201,7 +201,9 @@ static int airoha_ppe_get_wdma_info(struct net_device *dev, const u8 *addr, if (!dev) return -ENODEV; + rcu_read_lock(); err = dev_fill_forward_path(dev, addr, &stack); + rcu_read_unlock(); if (err) return err; @@ -1466,13 +1468,16 @@ void airoha_ppe_deinit(struct airoha_eth *eth) { struct airoha_npu *npu; - rcu_read_lock(); - npu = rcu_dereference(eth->npu); + mutex_lock(&flow_offload_mutex); + + npu = rcu_replace_pointer(eth->npu, NULL, + lockdep_is_held(&flow_offload_mutex)); if (npu) { npu->ops.ppe_deinit(npu); airoha_npu_put(npu); } - rcu_read_unlock(); + + mutex_unlock(&flow_offload_mutex); rhashtable_destroy(ð->ppe->l2_flows); rhashtable_destroy(ð->flow_table); diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c index 3f6204de9e6b8..a5d61a9a225aa 100644 --- a/drivers/net/ethernet/altera/altera_tse_main.c +++ b/drivers/net/ethernet/altera/altera_tse_main.c @@ -572,6 +572,7 @@ static netdev_tx_t tse_start_xmit(struct sk_buff *skb, struct net_device *dev) DMA_TO_DEVICE); if (dma_mapping_error(priv->device, dma_addr)) { netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); + dev_kfree_skb_any(skb); ret = NETDEV_TX_OK; goto out; } diff --git a/drivers/net/ethernet/amazon/ena/ena_devlink.c b/drivers/net/ethernet/amazon/ena/ena_devlink.c index ac81c24016dd4..4772185e669d2 100644 --- a/drivers/net/ethernet/amazon/ena/ena_devlink.c +++ b/drivers/net/ethernet/amazon/ena/ena_devlink.c @@ -53,10 +53,12 @@ void ena_devlink_disable_phc_param(struct devlink *devlink) { union devlink_param_value value; + devl_lock(devlink); value.vbool = false; devl_param_driverinit_value_set(devlink, DEVLINK_PARAM_GENERIC_ID_ENABLE_PHC, value); + devl_unlock(devlink); } static void ena_devlink_port_register(struct devlink *devlink) @@ -145,10 +147,12 @@ static int ena_devlink_configure_params(struct devlink *devlink) return rc; } + devl_lock(devlink); value.vbool = ena_phc_is_enabled(adapter); devl_param_driverinit_value_set(devlink, DEVLINK_PARAM_GENERIC_ID_ENABLE_PHC, value); + devl_unlock(devlink); return 0; } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h index 62b01de93db49..826c5caa70d71 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h @@ -431,7 +431,7 @@ #define MAC_SSIR_SSINC_INDEX 16 #define MAC_SSIR_SSINC_WIDTH 8 #define MAC_TCR_SS_INDEX 29 -#define MAC_TCR_SS_WIDTH 2 +#define MAC_TCR_SS_WIDTH 3 #define MAC_TCR_TE_INDEX 0 #define MAC_TCR_TE_WIDTH 1 #define MAC_TCR_VNE_INDEX 24 diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 4dc631af79332..fe4233fef3089 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -1089,7 +1089,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - unsigned long flags; DBGPR("-->xgbe_powerdown\n"); @@ -1100,8 +1099,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) return -EINVAL; } - spin_lock_irqsave(&pdata->lock, flags); - if (caller == XGMAC_DRIVER_CONTEXT) netif_device_detach(netdev); @@ -1117,8 +1114,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) pdata->power_down = 1; - spin_unlock_irqrestore(&pdata->lock, flags); - DBGPR("<--xgbe_powerdown\n"); return 0; @@ -1128,7 +1123,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - unsigned long flags; DBGPR("-->xgbe_powerup\n"); @@ -1139,8 +1133,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) return -EINVAL; } - spin_lock_irqsave(&pdata->lock, flags); - pdata->power_down = 0; xgbe_napi_enable(pdata, 0); @@ -1155,8 +1147,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) xgbe_start_timers(pdata); - spin_unlock_irqrestore(&pdata->lock, flags); - DBGPR("<--xgbe_powerup\n"); return 0; @@ -1256,6 +1246,10 @@ static int xgbe_start(struct xgbe_prv_data *pdata) hw_if->enable_tx(pdata); hw_if->enable_rx(pdata); + /* Synchronize flag with hardware state after enabling TX/RX. + * This prevents stale state after device restart cycles. + */ + pdata->data_path_stopped = false; udp_tunnel_nic_reset_ntf(netdev); @@ -1823,7 +1817,7 @@ static void xgbe_get_stats64(struct net_device *netdev, s->multicast = pstats->rxmulticastframes_g; s->rx_length_errors = pstats->rxlengtherror; s->rx_crc_errors = pstats->rxcrcerror; - s->rx_fifo_errors = pstats->rxfifooverflow; + s->rx_over_errors = pstats->rxfifooverflow; s->tx_packets = pstats->txframecount_gb; s->tx_bytes = pstats->txoctetcount_gb; @@ -2277,9 +2271,6 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) goto read_again; if (error || packet->errors) { - if (packet->errors) - netif_err(pdata, rx_err, netdev, - "error in received packet\n"); dev_kfree_skb(skb); goto next_packet; } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c index d1f0419edb234..7d45ea22a02e2 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c @@ -76,7 +76,6 @@ struct xgbe_prv_data *xgbe_alloc_pdata(struct device *dev) pdata->netdev = netdev; pdata->dev = dev; - spin_lock_init(&pdata->lock); spin_lock_init(&pdata->xpcs_lock); mutex_init(&pdata->rss_mutex); spin_lock_init(&pdata->tstamp_lock); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index a56efc1bee339..20d19d5a4effd 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -1927,6 +1927,7 @@ static void xgbe_set_rx_adap_mode(struct xgbe_prv_data *pdata, { if (pdata->rx_adapt_retries++ >= MAX_RX_ADAPT_RETRIES) { pdata->rx_adapt_retries = 0; + pdata->mode_set = false; return; } @@ -1940,7 +1941,7 @@ static void xgbe_set_rx_adap_mode(struct xgbe_prv_data *pdata, static void xgbe_rx_adaptation(struct xgbe_prv_data *pdata) { struct xgbe_phy_data *phy_data = pdata->phy_data; - unsigned int reg; + int reg; /* step 2: force PCS to send RX_ADAPT Req to PHY */ XMDIO_WRITE_BITS(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_RX_EQ_CTRL4, @@ -1962,17 +1963,27 @@ static void xgbe_rx_adaptation(struct xgbe_prv_data *pdata) /* Step 4: Check for Block lock */ - /* Link status is latched low, so read once to clear - * and then read again to get current state - */ - reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); + if (reg < 0) + goto set_mode; + + /* Link status is latched low so that momentary link drops + * can be detected. If link was already down read again + * to get the latest state. + */ + if (!pdata->phy.link && !(reg & MDIO_STAT1_LSTATUS)) { + reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); + if (reg < 0) + goto set_mode; + } + if (reg & MDIO_STAT1_LSTATUS) { /* If the block lock is found, update the helpers * and declare the link up */ netif_dbg(pdata, link, pdata->netdev, "Block_lock done"); pdata->rx_adapt_done = true; + pdata->rx_adapt_retries = 0; pdata->mode_set = false; return; } @@ -2005,6 +2016,48 @@ static void xgbe_phy_rx_adaptation(struct xgbe_prv_data *pdata) xgbe_rx_adaptation(pdata); } +/* + * xgbe_phy_stop_data_path - Stop TX/RX to prevent packet corruption + * @pdata: driver private data + * + * This function stops the data path (TX and RX) to prevent packet + * corruption during critical PHY operations like RX adaptation. + * Must be called before initiating RX adaptation when link goes down. + */ +static void xgbe_phy_stop_data_path(struct xgbe_prv_data *pdata) +{ + if (pdata->data_path_stopped) + return; + + /* Stop TX/RX to prevent packet corruption during RX adaptation */ + pdata->hw_if.disable_tx(pdata); + pdata->hw_if.disable_rx(pdata); + pdata->data_path_stopped = true; + + netif_dbg(pdata, link, pdata->netdev, + "stopping data path for RX adaptation\n"); +} + +/* + * xgbe_phy_start_data_path - Re-enable TX/RX after RX adaptation + * @pdata: driver private data + * + * This function re-enables the data path (TX and RX) after RX adaptation + * has completed successfully. Only called when link is confirmed up. + */ +static void xgbe_phy_start_data_path(struct xgbe_prv_data *pdata) +{ + if (!pdata->data_path_stopped) + return; + + pdata->hw_if.enable_rx(pdata); + pdata->hw_if.enable_tx(pdata); + pdata->data_path_stopped = false; + + netif_dbg(pdata, link, pdata->netdev, + "restarting data path after RX adaptation\n"); +} + static void xgbe_phy_rx_reset(struct xgbe_prv_data *pdata) { int reg; @@ -2798,13 +2851,27 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) if (pdata->en_rx_adap) { /* if the link is available and adaptation is done, * declare link up + * + * Note: When link is up and adaptation is done, we can + * safely re-enable the data path if it was stopped + * for adaptation. */ - if ((reg & MDIO_STAT1_LSTATUS) && pdata->rx_adapt_done) + if ((reg & MDIO_STAT1_LSTATUS) && pdata->rx_adapt_done) { + xgbe_phy_start_data_path(pdata); return 1; + } /* If either link is not available or adaptation is not done, * retrigger the adaptation logic. (if the mode is not set, * then issue mailbox command first) */ + + /* CRITICAL: Stop data path BEFORE triggering RX adaptation + * to prevent CRC errors from packets corrupted during + * the adaptation process. This is especially important + * when AN is OFF in 10G KR mode. + */ + xgbe_phy_stop_data_path(pdata); + if (pdata->mode_set) { xgbe_phy_rx_adaptation(pdata); } else { @@ -2812,8 +2879,11 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) xgbe_phy_set_mode(pdata, phy_data->cur_mode); } - if (pdata->rx_adapt_done) + if (pdata->rx_adapt_done) { + /* Adaptation complete, safe to re-enable data path */ + xgbe_phy_start_data_path(pdata); return 1; + } } else if (reg & MDIO_STAT1_LSTATUS) return 1; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index e8bbb68059013..ac0ba3d899dfd 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -1003,9 +1003,6 @@ struct xgbe_prv_data { unsigned int pp3; unsigned int pp4; - /* Overall device lock */ - spinlock_t lock; - /* XPCS indirect addressing lock */ spinlock_t xpcs_lock; unsigned int xpcs_window_def_reg; @@ -1245,6 +1242,10 @@ struct xgbe_prv_data { bool en_rx_adap; int rx_adapt_retries; bool rx_adapt_done; + /* Flag to track if data path (TX/RX) was stopped for RX adaptation. + * This prevents packet corruption during the adaptation window. + */ + bool data_path_stopped; bool mode_set; }; diff --git a/drivers/net/ethernet/arc/emac_main.c b/drivers/net/ethernet/arc/emac_main.c index 8283aeee35fb6..dde4046cbf010 100644 --- a/drivers/net/ethernet/arc/emac_main.c +++ b/drivers/net/ethernet/arc/emac_main.c @@ -934,6 +934,17 @@ int arc_emac_probe(struct net_device *ndev, int interface) /* Set poll rate so that it polls every 1 ms */ arc_reg_set(priv, R_POLLRATE, clock_frequency / 1000000); + /* + * Put the device into a known quiescent state before requesting + * the IRQ. Clear only EMAC interrupt status bits here; leave the + * MDIO completion bit alone and avoid writing TXPL_MASK, which is + * used to force TX polling rather than acknowledge interrupts. + */ + arc_reg_set(priv, R_ENABLE, 0); + arc_reg_set(priv, R_STATUS, RXINT_MASK | TXINT_MASK | ERR_MASK | + TXCH_MASK | MSER_MASK | RXCR_MASK | + RXFR_MASK | RXFL_MASK); + ndev->irq = irq; dev_info(dev, "IRQ is %d\n", ndev->irq); diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index 9fdef874f5ca3..fe15d684990fc 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -254,14 +254,14 @@ config BNXT_HWMON devices, via the hwmon sysfs interface. config BNGE - tristate "Broadcom Ethernet device support" + tristate "Broadcom ThorUltra Ethernet device support" depends on PCI select NET_DEVLINK select PAGE_POOL help - This driver supports Broadcom 50/100/200/400/800 gigabit Ethernet cards. - The module will be called bng_en. To compile this driver as a module, - choose M here. + This driver supports Broadcom ThorUltra 50/100/200/400/800 gigabit + Ethernet cards. The module will be called bng_en. To compile this + driver as a module, choose M here. config BCMASP tristate "Broadcom ASP 2.0 Ethernet support" diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.c b/drivers/net/ethernet/broadcom/asp2/bcmasp.c index fd35f4b4dc50b..1fdf0822c8a02 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.c @@ -156,7 +156,7 @@ static void bcmasp_netfilt_hw_en_wake(struct bcmasp_priv *priv, ASP_RX_FILTER_NET_OFFSET_L4(32), ASP_RX_FILTER_NET_OFFSET(nfilt->hw_index + 1)); - rx_filter_core_wl(priv, ASP_RX_FILTER_NET_CFG_CH(nfilt->port + 8) | + rx_filter_core_wl(priv, ASP_RX_FILTER_NET_CFG_CH(nfilt->ch) | ASP_RX_FILTER_NET_CFG_EN | ASP_RX_FILTER_NET_CFG_L2_EN | ASP_RX_FILTER_NET_CFG_L3_EN | @@ -166,7 +166,7 @@ static void bcmasp_netfilt_hw_en_wake(struct bcmasp_priv *priv, ASP_RX_FILTER_NET_CFG_UMC(nfilt->port), ASP_RX_FILTER_NET_CFG(nfilt->hw_index)); - rx_filter_core_wl(priv, ASP_RX_FILTER_NET_CFG_CH(nfilt->port + 8) | + rx_filter_core_wl(priv, ASP_RX_FILTER_NET_CFG_CH(nfilt->ch) | ASP_RX_FILTER_NET_CFG_EN | ASP_RX_FILTER_NET_CFG_L2_EN | ASP_RX_FILTER_NET_CFG_L3_EN | @@ -714,6 +714,7 @@ struct bcmasp_net_filter *bcmasp_netfilt_get_init(struct bcmasp_intf *intf, nfilter = &priv->net_filters[open_index]; nfilter->claimed = true; nfilter->port = intf->port; + nfilter->ch = intf->channel + priv->tx_chan_offset; nfilter->hw_index = open_index; } @@ -1156,12 +1157,6 @@ void bcmasp_enable_wol(struct bcmasp_intf *intf, bool en) } } -static void bcmasp_wol_irq_destroy(struct bcmasp_priv *priv) -{ - if (priv->wol_irq > 0) - free_irq(priv->wol_irq, priv); -} - static void bcmasp_eee_fixup(struct bcmasp_intf *intf, bool en) { u32 reg, phy_lpi_overwrite; @@ -1259,7 +1254,7 @@ static int bcmasp_probe(struct platform_device *pdev) if (priv->irq <= 0) return -EINVAL; - priv->clk = devm_clk_get_optional_enabled(dev, "sw_asp"); + priv->clk = devm_clk_get_optional(dev, "sw_asp"); if (IS_ERR(priv->clk)) return dev_err_probe(dev, PTR_ERR(priv->clk), "failed to request clock\n"); @@ -1287,6 +1282,10 @@ static int bcmasp_probe(struct platform_device *pdev) bcmasp_set_pdata(priv, pdata); + ret = clk_prepare_enable(priv->clk); + if (ret) + return dev_err_probe(dev, ret, "failed to start clock\n"); + /* Enable all clocks to ensure successful probing */ bcmasp_core_clock_set(priv, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE, 0); @@ -1298,8 +1297,10 @@ static int bcmasp_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, priv->irq, bcmasp_isr, 0, pdev->name, priv); - if (ret) - return dev_err_probe(dev, ret, "failed to request ASP interrupt: %d", ret); + if (ret) { + dev_err(dev, "Failed to request ASP interrupt: %d", ret); + goto err_clock_disable; + } /* Register mdio child nodes */ of_platform_populate(dev->of_node, bcmasp_mdio_of_match, NULL, dev); @@ -1311,20 +1312,27 @@ static int bcmasp_probe(struct platform_device *pdev) priv->mda_filters = devm_kcalloc(dev, priv->num_mda_filters, sizeof(*priv->mda_filters), GFP_KERNEL); - if (!priv->mda_filters) - return -ENOMEM; + if (!priv->mda_filters) { + ret = -ENOMEM; + goto err_clock_disable; + } priv->net_filters = devm_kcalloc(dev, priv->num_net_filters, sizeof(*priv->net_filters), GFP_KERNEL); - if (!priv->net_filters) - return -ENOMEM; + if (!priv->net_filters) { + ret = -ENOMEM; + goto err_clock_disable; + } bcmasp_core_init_filters(priv); + bcmasp_init_wol(priv); + ports_node = of_find_node_by_name(dev->of_node, "ethernet-ports"); if (!ports_node) { dev_warn(dev, "No ports found\n"); - return -EINVAL; + ret = -EINVAL; + goto err_clock_disable; } i = 0; @@ -1332,43 +1340,43 @@ static int bcmasp_probe(struct platform_device *pdev) intf = bcmasp_interface_create(priv, intf_node, i); if (!intf) { dev_err(dev, "Cannot create eth interface %d\n", i); - bcmasp_remove_intfs(priv); - ret = -ENOMEM; - goto of_put_exit; + of_node_put(ports_node); + ret = -EINVAL; + goto err_cleanup; } list_add_tail(&intf->list, &priv->intfs); i++; } - - /* Check and enable WoL */ - bcmasp_init_wol(priv); + of_node_put(ports_node); /* Drop the clock reference count now and let ndo_open()/ndo_close() * manage it for us from now on. */ bcmasp_core_clock_set(priv, 0, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE); - clk_disable_unprepare(priv->clk); - /* Now do the registration of the network ports which will take care * of managing the clock properly. */ list_for_each_entry(intf, &priv->intfs, list) { ret = register_netdev(intf->ndev); if (ret) { - netdev_err(intf->ndev, - "failed to register net_device: %d\n", ret); - bcmasp_wol_irq_destroy(priv); - bcmasp_remove_intfs(priv); - goto of_put_exit; + dev_err(dev, "failed to register net_device: %d\n", ret); + goto err_cleanup; } count++; } + clk_disable_unprepare(priv->clk); + dev_info(dev, "Initialized %d port(s)\n", count); -of_put_exit: - of_node_put(ports_node); + return ret; + +err_cleanup: + bcmasp_remove_intfs(priv); +err_clock_disable: + clk_disable_unprepare(priv->clk); + return ret; } @@ -1379,7 +1387,6 @@ static void bcmasp_remove(struct platform_device *pdev) if (!priv) return; - bcmasp_wol_irq_destroy(priv); bcmasp_remove_intfs(priv); } diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.h b/drivers/net/ethernet/broadcom/asp2/bcmasp.h index 74adfdb50e11d..e238507be40af 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp.h +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.h @@ -348,6 +348,7 @@ struct bcmasp_net_filter { bool wake_filter; int port; + int ch; unsigned int hw_index; }; diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c index b9973956c4809..ceb6c11431dd9 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c @@ -1261,7 +1261,7 @@ struct bcmasp_intf *bcmasp_interface_create(struct bcmasp_priv *priv, netdev_err(intf->ndev, "invalid PHY mode: %s for port %d\n", phy_modes(intf->phy_interface), intf->port); ret = -EINVAL; - goto err_free_netdev; + goto err_deregister_fixed_link; } ret = of_get_ethdev_address(ndev_dn, ndev); @@ -1286,6 +1286,9 @@ struct bcmasp_intf *bcmasp_interface_create(struct bcmasp_priv *priv, return intf; +err_deregister_fixed_link: + if (of_phy_is_fixed_link(ndev_dn)) + of_phy_deregister_fixed_link(ndev_dn); err_free_netdev: free_netdev(ndev); err: diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c index 0353359c3fe96..073d7d490d4b6 100644 --- a/drivers/net/ethernet/broadcom/b44.c +++ b/drivers/net/ethernet/broadcom/b44.c @@ -1789,6 +1789,9 @@ static int b44_nway_reset(struct net_device *dev) u32 bmcr; int r; + if (bp->flags & B44_FLAG_EXTERNAL_PHY) + return phy_ethtool_nway_reset(dev); + spin_lock_irq(&bp->lock); b44_readphy(bp, MII_BMCR, &bmcr); b44_readphy(bp, MII_BMCR, &bmcr); diff --git a/drivers/net/ethernet/broadcom/bnge/bnge.h b/drivers/net/ethernet/broadcom/bnge/bnge.h index 7aed5f81cd51e..0c154995d9ab5 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge.h +++ b/drivers/net/ethernet/broadcom/bnge/bnge.h @@ -5,7 +5,7 @@ #define _BNGE_H_ #define DRV_NAME "bng_en" -#define DRV_SUMMARY "Broadcom 800G Ethernet Linux Driver" +#define DRV_SUMMARY "Broadcom ThorUltra NIC Ethernet Driver" #include #include diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_core.c b/drivers/net/ethernet/broadcom/bnge/bnge_core.c index 2c72dd34d50d0..312a9db4d75d1 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_core.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_core.c @@ -19,7 +19,7 @@ char bnge_driver_name[] = DRV_NAME; static const struct { char *name; } board_info[] = { - [BCM57708] = { "Broadcom BCM57708 50Gb/100Gb/200Gb/400Gb/800Gb Ethernet" }, + [BCM57708] = { "Broadcom BCM57708 ThorUltra 50Gb/100Gb/200Gb/400Gb/800Gb Ethernet" }, }; static const struct pci_device_id bnge_pci_tbl[] = { diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c index 198f49b40dbf0..2994f10446a63 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c @@ -442,7 +442,7 @@ __bnge_hwrm_reserve_pf_rings(struct bnge_dev *bd, struct bnge_hw_rings *hwr) struct hwrm_func_cfg_input *req; u32 enables = 0; - if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_QCFG)) + if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_CFG)) return NULL; req->fid = cpu_to_le16(0xffff); diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index cb1011f6fd307..805daae9dd362 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -6444,7 +6444,6 @@ bnx2_reset_task(struct work_struct *work) if (!(pcicmd & PCI_COMMAND_MEMORY)) { /* in case PCI block has reset */ pci_restore_state(bp->pdev); - pci_save_state(bp->pdev); } rc = bnx2_init_nic(bp, 1); if (rc) { @@ -8718,7 +8717,6 @@ static pci_ers_result_t bnx2_io_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); if (netif_running(dev)) err = bnx2_init_nic(bp, 1); diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index f0f05d7315ac4..8e6eec828d482 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -14216,7 +14216,6 @@ static pci_ers_result_t bnx2x_io_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); if (netif_running(dev)) bnx2x_set_power_state(bp, PCI_D0); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index a625e7c311dd7..d02ccf79e3b6f 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -1482,9 +1482,11 @@ static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; u16 idx = agg_id & MAX_TPA_P5_MASK; - if (test_bit(idx, map->agg_idx_bmap)) - idx = find_first_zero_bit(map->agg_idx_bmap, - BNXT_AGG_IDX_BMAP_SIZE); + if (test_bit(idx, map->agg_idx_bmap)) { + idx = find_first_zero_bit(map->agg_idx_bmap, MAX_TPA_P5); + if (idx >= MAX_TPA_P5) + return INVALID_HW_RING_ID; + } __set_bit(idx, map->agg_idx_bmap); map->agg_id_tbl[agg_id] = idx; return idx; @@ -1548,6 +1550,13 @@ static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { agg_id = TPA_START_AGG_ID_P5(tpa_start); agg_id = bnxt_alloc_agg_idx(rxr, agg_id); + if (unlikely(agg_id == INVALID_HW_RING_ID)) { + netdev_warn(bp->dev, "Unable to allocate agg ID for ring %d, agg 0x%x\n", + rxr->bnapi->index, + TPA_START_AGG_ID_P5(tpa_start)); + bnxt_sched_reset_rxr(bp, rxr); + return; + } } else { agg_id = TPA_START_AGG_ID(tpa_start); } @@ -2918,6 +2927,8 @@ static int bnxt_async_event_process(struct bnxt *bp, u16 type = (u16)BNXT_EVENT_BUF_PRODUCER_TYPE(data1); u32 offset = BNXT_EVENT_BUF_PRODUCER_OFFSET(data2); + if (type >= ARRAY_SIZE(bp->bs_trace)) + goto async_event_process_exit; bnxt_bs_trace_check_wrap(&bp->bs_trace[type], offset); goto async_event_process_exit; } @@ -6192,6 +6203,9 @@ int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, int rc; set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state); + if (!test_bit(BNXT_STATE_OPEN, &bp->state)) + return 0; + rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); if (rc) return rc; @@ -7977,6 +7991,8 @@ static int __bnxt_reserve_rings(struct bnxt *bp) ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want); if (!ulp_msix) bnxt_set_ulp_stat_ctxs(bp, 0); + else + bnxt_set_dflt_ulp_stat_ctxs(bp); if (ulp_msix > bp->ulp_num_msix_want) ulp_msix = bp->ulp_num_msix_want; @@ -8593,7 +8609,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) struct hwrm_func_backing_store_qcaps_v2_output *resp; struct hwrm_func_backing_store_qcaps_v2_input *req; struct bnxt_ctx_mem_info *ctx = bp->ctx; - u16 type; + u16 type, next_type = 0; int rc; rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2); @@ -8609,7 +8625,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) resp = hwrm_req_hold(bp, req); - for (type = 0; type < BNXT_CTX_V2_MAX; ) { + for (type = 0; type < BNXT_CTX_V2_MAX; type = next_type) { struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; u8 init_val, init_off, i; u32 max_entries; @@ -8622,7 +8638,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) if (rc) goto ctx_done; flags = le32_to_cpu(resp->flags); - type = le16_to_cpu(resp->next_valid_type); + next_type = le16_to_cpu(resp->next_valid_type); if (!(flags & BNXT_CTX_MEM_TYPE_VALID)) { bnxt_free_one_ctx_mem(bp, ctxm, true); continue; @@ -8637,7 +8653,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) else continue; } - ctxm->type = le16_to_cpu(resp->type); + ctxm->type = type; ctxm->entry_size = entry_size; ctxm->flags = flags; ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map); @@ -10800,12 +10816,10 @@ void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx, struct bnxt_ntuple_filter *ntp_fltr; int i; - if (netif_running(bp->dev)) { - bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); - for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { - if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) - bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); - } + bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); + for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { + if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) + bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); } if (!all) return; @@ -16856,12 +16870,12 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) init_err_pci_clean: bnxt_hwrm_func_drv_unrgtr(bp); - bnxt_free_hwrm_resources(bp); - bnxt_hwmon_uninit(bp); - bnxt_ethtool_free(bp); bnxt_ptp_clear(bp); kfree(bp->ptp_cfg); bp->ptp_cfg = NULL; + bnxt_free_hwrm_resources(bp); + bnxt_hwmon_uninit(bp); + bnxt_ethtool_free(bp); kfree(bp->fw_health); bp->fw_health = NULL; bnxt_cleanup_pci(bp); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 3613a172483a1..c80603704838d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -1079,11 +1079,9 @@ struct bnxt_tpa_info { struct rx_agg_cmp *agg_arr; }; -#define BNXT_AGG_IDX_BMAP_SIZE (MAX_TPA_P5 / BITS_PER_LONG) - struct bnxt_tpa_idx_map { u16 agg_id_tbl[1024]; - unsigned long agg_idx_bmap[BNXT_AGG_IDX_BMAP_SIZE]; + DECLARE_BITMAP(agg_idx_bmap, MAX_TPA_P5); }; struct bnxt_rx_ring_info { @@ -2136,7 +2134,7 @@ enum board_idx { }; #define BNXT_TRACE_BUF_MAGIC_BYTE ((u8)0xbc) -#define BNXT_TRACE_MAX 11 +#define BNXT_TRACE_MAX (DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ERR_QPC_TRACE + 1) struct bnxt_bs_trace_info { u8 *magic_byte; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 41686a6f84b58..3237515f0e7ec 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -973,8 +973,8 @@ static int bnxt_set_channels(struct net_device *dev, if (bnxt_get_nr_rss_ctxs(bp, req_rx_rings) != bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) && - netif_is_rxfh_configured(dev)) { - netdev_warn(dev, "RSS table size change required, RSS table entries must be default to proceed\n"); + (netif_is_rxfh_configured(dev) || bp->num_rss_ctx)) { + netdev_warn(dev, "RSS table size change required, RSS table entries must be default (with no additional RSS contexts present) to proceed\n"); return -EINVAL; } @@ -1340,16 +1340,17 @@ static int bnxt_add_ntuple_cls_rule(struct bnxt *bp, struct bnxt_l2_filter *l2_fltr; struct bnxt_flow_masks *fmasks; struct flow_keys *fkeys; - u32 idx, ring; + u32 idx; int rc; - u8 vf; if (!bp->vnic_info) return -EAGAIN; - vf = ethtool_get_flow_spec_ring_vf(fs->ring_cookie); - ring = ethtool_get_flow_spec_ring(fs->ring_cookie); - if ((fs->flow_type & (FLOW_MAC_EXT | FLOW_EXT)) || vf) + if (fs->flow_type & (FLOW_MAC_EXT | FLOW_EXT)) + return -EOPNOTSUPP; + + if (fs->ring_cookie != RX_CLS_FLOW_DISC && + ethtool_get_flow_spec_ring_vf(fs->ring_cookie)) return -EOPNOTSUPP; if (flow_type == IP_USER_FLOW) { @@ -1475,7 +1476,7 @@ static int bnxt_add_ntuple_cls_rule(struct bnxt *bp, if (fs->ring_cookie == RX_CLS_FLOW_DISC) new_fltr->base.flags |= BNXT_ACT_DROP; else - new_fltr->base.rxq = ring; + new_fltr->base.rxq = ethtool_get_flow_spec_ring(fs->ring_cookie); __set_bit(BNXT_FLTR_VALID, &new_fltr->base.state); rc = bnxt_insert_ntp_filter(bp, new_fltr, idx); if (!rc) { diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 3e77a96e5a3e3..c94a391b1ba5b 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -268,13 +268,11 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, case XDP_TX: rx_buf = &rxr->rx_buf_ring[cons]; mapping = rx_buf->mapping - bp->rx_dma_offset; - *event &= BNXT_TX_CMP_EVENT; if (unlikely(xdp_buff_has_frags(xdp))) { struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); tx_needed += sinfo->nr_frags; - *event = BNXT_AGG_EVENT; } if (tx_avail < tx_needed) { @@ -287,6 +285,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, dma_sync_single_for_device(&pdev->dev, mapping + offset, *len, bp->rx_dir); + *event &= ~BNXT_RX_EVENT; *event |= BNXT_TX_EVENT; __bnxt_xmit_xdp(bp, txr, mapping + offset, *len, NEXT_RX(rxr->rx_prod), xdp); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 98971ae4f87df..e142939d87cbe 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -1343,8 +1343,7 @@ static void bcmgenet_get_ethtool_stats(struct net_device *dev, } } -void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, - bool tx_lpi_enabled) +void bcmgenet_eee_enable_set(struct net_device *dev, bool enable) { struct bcmgenet_priv *priv = netdev_priv(dev); u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL; @@ -1364,7 +1363,7 @@ void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, /* Enable EEE and switch to a 27Mhz clock automatically */ reg = bcmgenet_readl(priv->base + off); - if (tx_lpi_enabled) + if (enable) reg |= TBUF_EEE_EN | TBUF_PM_EN; else reg &= ~(TBUF_EEE_EN | TBUF_PM_EN); @@ -1383,14 +1382,12 @@ void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, priv->clk_eee_enabled = false; } - priv->eee.eee_enabled = enable; - priv->eee.tx_lpi_enabled = tx_lpi_enabled; } static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmgenet_priv *priv = netdev_priv(dev); - struct ethtool_keee *p = &priv->eee; + int ret; if (GENET_IS_V1(priv)) return -EOPNOTSUPP; @@ -1398,17 +1395,21 @@ static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e) if (!dev->phydev) return -ENODEV; - e->tx_lpi_enabled = p->tx_lpi_enabled; + ret = phy_ethtool_get_eee(dev->phydev, e); + if (ret) + return ret; + + /* tx_lpi_timer is maintained by the MAC hardware register; the + * PHY-level eee_cfg timer is not set for GENET. + */ e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER); - return phy_ethtool_get_eee(dev->phydev, e); + return 0; } static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmgenet_priv *priv = netdev_priv(dev); - struct ethtool_keee *p = &priv->eee; - bool active; if (GENET_IS_V1(priv)) return -EOPNOTSUPP; @@ -1416,15 +1417,7 @@ static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e) if (!dev->phydev) return -ENODEV; - p->eee_enabled = e->eee_enabled; - - if (!p->eee_enabled) { - bcmgenet_eee_enable_set(dev, false, false); - } else { - active = phy_init_eee(dev->phydev, false) >= 0; - bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER); - bcmgenet_eee_enable_set(dev, active, e->tx_lpi_enabled); - } + bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER); return phy_ethtool_set_eee(dev->phydev, e); } diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 5ec3979779ece..9e4110c7fdf6f 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -665,8 +665,6 @@ struct bcmgenet_priv { u8 sopass[SOPASS_MAX]; struct bcmgenet_mib_counters mib; - - struct ethtool_keee eee; }; static inline bool bcmgenet_has_40bits(struct bcmgenet_priv *priv) @@ -749,7 +747,6 @@ int bcmgenet_wol_power_down_cfg(struct bcmgenet_priv *priv, int bcmgenet_wol_power_up_cfg(struct bcmgenet_priv *priv, enum bcmgenet_power_mode mode); -void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, - bool tx_lpi_enabled); +void bcmgenet_eee_enable_set(struct net_device *dev, bool enable); #endif /* __BCMGENET_H__ */ diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c index 8fb5512882980..96d5d4f7f51fe 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c @@ -123,7 +123,7 @@ static int bcmgenet_poll_wol_status(struct bcmgenet_priv *priv) while (!(bcmgenet_rbuf_readl(priv, RBUF_STATUS) & RBUF_STATUS_WOL)) { retries++; - if (retries > 5) { + if (retries > 50) { netdev_crit(dev, "polling wol mode timeout\n"); return -ETIMEDOUT; } diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index 573e8b279e52f..33e3eec31cc9e 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -30,7 +30,6 @@ static void bcmgenet_mac_config(struct net_device *dev) struct bcmgenet_priv *priv = netdev_priv(dev); struct phy_device *phydev = dev->phydev; u32 reg, cmd_bits = 0; - bool active; /* speed */ if (phydev->speed == SPEED_1000) @@ -91,10 +90,6 @@ static void bcmgenet_mac_config(struct net_device *dev) bcmgenet_umac_writel(priv, reg, UMAC_CMD); spin_unlock_bh(&priv->reg_lock); - active = phy_init_eee(phydev, 0) >= 0; - bcmgenet_eee_enable_set(dev, - priv->eee.eee_enabled && active, - priv->eee.tx_lpi_enabled); } /* setup netdev link state when PHY link status change and @@ -114,6 +109,8 @@ void bcmgenet_mii_setup(struct net_device *dev) bcmgenet_ext_writel(priv, reg, EXT_RGMII_OOB_CTRL); } + bcmgenet_eee_enable_set(dev, phydev->enable_tx_lpi); + phy_print_status(phydev); } @@ -413,6 +410,9 @@ int bcmgenet_mii_probe(struct net_device *dev) /* Indicate that the MAC is responsible for PHY PM */ dev->phydev->mac_managed_pm = true; + if (!GENET_IS_V1(priv)) + phy_support_eee(dev->phydev); + return 0; } diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index d78cafdb20949..af21952373ea1 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -12300,7 +12300,7 @@ static int tg3_get_link_ksettings(struct net_device *dev, ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, advertising); - if (netif_running(dev) && tp->link_up) { + if (netif_running(dev) && netif_carrier_ok(dev)) { cmd->base.speed = tp->link_config.active_speed; cmd->base.duplex = tp->link_config.active_duplex; ethtool_convert_legacy_u32_to_link_mode( @@ -17042,6 +17042,13 @@ static int tg3_get_invariants(struct tg3 *tp, const struct pci_device_id *ent) return err; } +static int tg3_is_default_mac_address(u8 *addr) +{ + static const u8 default_mac_address[ETH_ALEN] = { 0x00, 0x10, 0x18, 0x00, 0x00, 0x00 }; + + return ether_addr_equal(default_mac_address, addr); +} + static int tg3_get_device_address(struct tg3 *tp, u8 *addr) { u32 hi, lo, mac_offset; @@ -17115,6 +17122,10 @@ static int tg3_get_device_address(struct tg3 *tp, u8 *addr) if (!is_valid_ether_addr(addr)) return -EINVAL; + + if (tg3_is_default_mac_address(addr)) + return device_get_mac_address(&tp->pdev->dev, addr); + return 0; } @@ -18349,7 +18360,6 @@ static pci_ers_result_t tg3_io_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); if (!netdev || !netif_running(netdev)) { rc = PCI_ERS_RESULT_RECOVERED; diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index ca2386b834737..17d4a3e039458 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -6,36 +6,38 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include +#include #include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include #include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include #include #include #include -#include -#include -#include -#include #include +#include +#include #include #include #include -#include -#include +#include +#include +#include +#include +#include #include #include "macb.h" @@ -704,6 +706,97 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, netif_tx_stop_all_queues(ndev); } +/* Use juggling algorithm to left rotate tx ring and tx skb array */ +static void gem_shuffle_tx_one_ring(struct macb_queue *queue) +{ + unsigned int head, tail, count, ring_size, desc_size; + struct macb_tx_skb tx_skb, *skb_curr, *skb_next; + struct macb_dma_desc *desc_curr, *desc_next; + unsigned int i, cycles, shift, curr, next; + struct macb *bp = queue->bp; + unsigned char desc[24]; + unsigned long flags; + + desc_size = macb_dma_desc_get_size(bp); + + if (WARN_ON_ONCE(desc_size > ARRAY_SIZE(desc))) + return; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + head = queue->tx_head; + tail = queue->tx_tail; + ring_size = bp->tx_ring_size; + count = CIRC_CNT(head, tail, ring_size); + + if (!(tail % ring_size)) + goto unlock; + + if (!count) { + queue->tx_head = 0; + queue->tx_tail = 0; + goto unlock; + } + + shift = tail % ring_size; + cycles = gcd(ring_size, shift); + + for (i = 0; i < cycles; i++) { + memcpy(&desc, macb_tx_desc(queue, i), desc_size); + memcpy(&tx_skb, macb_tx_skb(queue, i), + sizeof(struct macb_tx_skb)); + + curr = i; + next = (curr + shift) % ring_size; + + while (next != i) { + desc_curr = macb_tx_desc(queue, curr); + desc_next = macb_tx_desc(queue, next); + + memcpy(desc_curr, desc_next, desc_size); + + if (next == ring_size - 1) + desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); + if (curr == ring_size - 1) + desc_curr->ctrl |= MACB_BIT(TX_WRAP); + + skb_curr = macb_tx_skb(queue, curr); + skb_next = macb_tx_skb(queue, next); + memcpy(skb_curr, skb_next, sizeof(struct macb_tx_skb)); + + curr = next; + next = (curr + shift) % ring_size; + } + + desc_curr = macb_tx_desc(queue, curr); + memcpy(desc_curr, &desc, desc_size); + if (i == ring_size - 1) + desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); + if (curr == ring_size - 1) + desc_curr->ctrl |= MACB_BIT(TX_WRAP); + memcpy(macb_tx_skb(queue, curr), &tx_skb, + sizeof(struct macb_tx_skb)); + } + + queue->tx_head = count; + queue->tx_tail = 0; + + /* Make descriptor updates visible to hardware */ + wmb(); + +unlock: + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); +} + +/* Rotate the queue so that the tail is at index 0 */ +static void gem_shuffle_tx_rings(struct macb *bp) +{ + struct macb_queue *queue; + int q; + + for (q = 0, queue = bp->queues; q < bp->num_queues; q++, queue++) + gem_shuffle_tx_one_ring(queue); +} + static void macb_mac_link_up(struct phylink_config *config, struct phy_device *phy, unsigned int mode, phy_interface_t interface, @@ -741,15 +834,10 @@ static void macb_mac_link_up(struct phylink_config *config, if (rx_pause) ctrl |= MACB_BIT(PAE); - /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down - * cleared the pipeline and control registers. - */ - bp->macbgem_ops.mog_init_rings(bp); - macb_init_buffers(bp); - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { queue_writel(queue, IER, bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); + } } macb_or_gem_writel(bp, NCFGR, ctrl); @@ -760,8 +848,10 @@ static void macb_mac_link_up(struct phylink_config *config, spin_unlock_irqrestore(&bp->lock, flags); - if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { macb_set_tx_clk(bp, speed); + gem_shuffle_tx_rings(bp); + } /* Enable Rx and Tx; Enable PTP unicast */ ctrl = macb_readl(bp, NCR); @@ -1018,7 +1108,7 @@ static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budge } if (tx_skb->skb) { - napi_consume_skb(tx_skb->skb, budget); + dev_consume_skb_any(tx_skb->skb); tx_skb->skb = NULL; } } @@ -2617,6 +2707,14 @@ static void macb_init_tieoff(struct macb *bp) desc->ctrl = 0; } +static void gem_init_rx_ring(struct macb_queue *queue) +{ + queue->rx_tail = 0; + queue->rx_prepared_head = 0; + + gem_rx_refill(queue); +} + static void gem_init_rings(struct macb *bp) { struct macb_queue *queue; @@ -2634,10 +2732,7 @@ static void gem_init_rings(struct macb *bp) queue->tx_head = 0; queue->tx_tail = 0; - queue->rx_tail = 0; - queue->rx_prepared_head = 0; - - gem_rx_refill(queue); + gem_init_rx_ring(queue); } macb_init_tieoff(bp); @@ -2991,6 +3086,9 @@ static int macb_open(struct net_device *dev) goto pm_exit; } + bp->macbgem_ops.mog_init_rings(bp); + macb_init_buffers(bp); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { napi_enable(&queue->napi_rx); napi_enable(&queue->napi_tx); @@ -3160,7 +3258,7 @@ static void gem_get_ethtool_stats(struct net_device *dev, spin_lock_irq(&bp->stats_lock); gem_update_stats(bp); memcpy(data, &bp->ethtool_stats, sizeof(u64) - * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); + * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); spin_unlock_irq(&bp->stats_lock); } @@ -3915,6 +4013,9 @@ static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) struct macb *bp = netdev_priv(netdev); int ret; + if (!(netdev->hw_features & NETIF_F_NTUPLE)) + return -EOPNOTSUPP; + switch (cmd->cmd) { case ETHTOOL_SRXCLSRLINS: if ((cmd->fs.location >= bp->max_tuples) @@ -5672,9 +5773,9 @@ static int __maybe_unused macb_suspend(struct device *dev) struct macb_queue *queue; struct in_device *idev; unsigned long flags; + u32 tmp, ifa_local; unsigned int q; int err; - u32 tmp; if (!device_may_wakeup(&bp->dev->dev)) phy_exit(bp->sgmii_phy); @@ -5683,14 +5784,21 @@ static int __maybe_unused macb_suspend(struct device *dev) return 0; if (bp->wol & MACB_WOL_ENABLED) { - /* Check for IP address in WOL ARP mode */ - idev = __in_dev_get_rcu(bp->dev); - if (idev) - ifa = rcu_dereference(idev->ifa_list); - if ((bp->wolopts & WAKE_ARP) && !ifa) { - netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); - return -EOPNOTSUPP; + if (bp->wolopts & WAKE_ARP) { + /* Check for IP address in WOL ARP mode */ + rcu_read_lock(); + idev = __in_dev_get_rcu(bp->dev); + if (idev) + ifa = rcu_dereference(idev->ifa_list); + if (!ifa) { + rcu_read_unlock(); + netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); + return -EOPNOTSUPP; + } + ifa_local = be32_to_cpu(ifa->ifa_local); + rcu_read_unlock(); } + spin_lock_irqsave(&bp->lock, flags); /* Disable Tx and Rx engines before disabling the queues, @@ -5729,8 +5837,9 @@ static int __maybe_unused macb_suspend(struct device *dev) if (bp->wolopts & WAKE_ARP) { tmp |= MACB_BIT(ARP); /* write IP address into register */ - tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local)); + tmp |= MACB_BFEXT(IP, ifa_local); } + spin_unlock_irqrestore(&bp->lock, flags); /* Change interrupt handler and * Enable WoL IRQ on queue 0 @@ -5743,11 +5852,12 @@ static int __maybe_unused macb_suspend(struct device *dev) dev_err(dev, "Unable to request IRQ %d (error %d)\n", bp->queues[0].irq, err); - spin_unlock_irqrestore(&bp->lock, flags); return err; } + spin_lock_irqsave(&bp->lock, flags); queue_writel(bp->queues, IER, GEM_BIT(WOL)); gem_writel(bp, WOL, tmp); + spin_unlock_irqrestore(&bp->lock, flags); } else { err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, IRQF_SHARED, netdev->name, bp->queues); @@ -5755,13 +5865,13 @@ static int __maybe_unused macb_suspend(struct device *dev) dev_err(dev, "Unable to request IRQ %d (error %d)\n", bp->queues[0].irq, err); - spin_unlock_irqrestore(&bp->lock, flags); return err; } + spin_lock_irqsave(&bp->lock, flags); queue_writel(bp->queues, IER, MACB_BIT(WOL)); macb_writel(bp, WOL, tmp); + spin_unlock_irqrestore(&bp->lock, flags); } - spin_unlock_irqrestore(&bp->lock, flags); enable_irq_wake(bp->queues[0].irq); } @@ -5828,6 +5938,8 @@ static int __maybe_unused macb_resume(struct device *dev) queue_readl(bp->queues, ISR); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) queue_writel(bp->queues, ISR, -1); + spin_unlock_irqrestore(&bp->lock, flags); + /* Replace interrupt handler on queue 0 */ devm_free_irq(dev, bp->queues[0].irq, bp->queues); err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, @@ -5836,10 +5948,8 @@ static int __maybe_unused macb_resume(struct device *dev) dev_err(dev, "Unable to request IRQ %d (error %d)\n", bp->queues[0].irq, err); - spin_unlock_irqrestore(&bp->lock, flags); return err; } - spin_unlock_irqrestore(&bp->lock, flags); disable_irq_wake(bp->queues[0].irq); @@ -5851,8 +5961,18 @@ static int __maybe_unused macb_resume(struct device *dev) rtnl_unlock(); } + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + macb_init_buffers(bp); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { + if (macb_is_gem(bp)) + gem_init_rx_ring(queue); + else + macb_init_rx_ring(queue); + } + napi_enable(&queue->napi_rx); napi_enable(&queue->napi_tx); } diff --git a/drivers/net/ethernet/cadence/macb_pci.c b/drivers/net/ethernet/cadence/macb_pci.c index fc4f5aee6ab3f..b79dec17e6b09 100644 --- a/drivers/net/ethernet/cadence/macb_pci.c +++ b/drivers/net/ethernet/cadence/macb_pci.c @@ -96,10 +96,10 @@ static int macb_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err_plat_dev_register: - clk_unregister(plat_data.hclk); + clk_unregister_fixed_rate(plat_data.hclk); err_hclk_register: - clk_unregister(plat_data.pclk); + clk_unregister_fixed_rate(plat_data.pclk); err_pclk_register: return err; @@ -109,10 +109,12 @@ static void macb_remove(struct pci_dev *pdev) { struct platform_device *plat_dev = pci_get_drvdata(pdev); struct macb_platform_data *plat_data = dev_get_platdata(&plat_dev->dev); + struct clk *pclk = plat_data->pclk; + struct clk *hclk = plat_data->hclk; - clk_unregister(plat_data->pclk); - clk_unregister(plat_data->hclk); platform_device_unregister(plat_dev); + clk_unregister_fixed_rate(pclk); + clk_unregister_fixed_rate(hclk); } static const struct pci_device_id dev_id_table[] = { diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c index a63bf29c4fa81..f2b09100f710e 100644 --- a/drivers/net/ethernet/cadence/macb_ptp.c +++ b/drivers/net/ethernet/cadence/macb_ptp.c @@ -355,8 +355,10 @@ void gem_ptp_remove(struct net_device *ndev) { struct macb *bp = netdev_priv(ndev); - if (bp->ptp_clock) + if (bp->ptp_clock) { ptp_clock_unregister(bp->ptp_clock); + bp->ptp_clock = NULL; + } gem_ptp_clear_timer(bp); diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index 8e2fcec26ea13..eb620e8544cf1 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -3515,6 +3515,23 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) */ netdev->netdev_ops = &lionetdevops; + lio = GET_LIO(netdev); + + memset(lio, 0, sizeof(struct lio)); + + lio->ifidx = ifidx_or_pfnum; + + props = &octeon_dev->props[i]; + props->gmxport = resp->cfg_info.linfo.gmxport; + props->netdev = netdev; + + /* Point to the properties for octeon device to which this + * interface belongs. + */ + lio->oct_dev = octeon_dev; + lio->octprops = props; + lio->netdev = netdev; + retval = netif_set_real_num_rx_queues(netdev, num_oqueues); if (retval) { dev_err(&octeon_dev->pci_dev->dev, @@ -3531,16 +3548,6 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) goto setup_nic_dev_free; } - lio = GET_LIO(netdev); - - memset(lio, 0, sizeof(struct lio)); - - lio->ifidx = ifidx_or_pfnum; - - props = &octeon_dev->props[i]; - props->gmxport = resp->cfg_info.linfo.gmxport; - props->netdev = netdev; - lio->linfo.num_rxpciq = num_oqueues; lio->linfo.num_txpciq = num_iqueues; for (j = 0; j < num_oqueues; j++) { @@ -3606,13 +3613,6 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) netdev->min_mtu = LIO_MIN_MTU_SIZE; netdev->max_mtu = LIO_MAX_MTU_SIZE; - /* Point to the properties for octeon device to which this - * interface belongs. - */ - lio->oct_dev = octeon_dev; - lio->octprops = props; - lio->netdev = netdev; - dev_dbg(&octeon_dev->pci_dev->dev, "if%d gmx: %d hw_addr: 0x%llx\n", i, lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr)); @@ -3760,6 +3760,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) if (!devlink) { device_unlock(&octeon_dev->pci_dev->dev); dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n"); + i--; goto setup_nic_dev_free; } @@ -3775,11 +3776,11 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) setup_nic_dev_free: - while (i--) { + do { dev_err(&octeon_dev->pci_dev->dev, "NIC ifidx:%d Setup failed\n", i); liquidio_destroy_nic_device(octeon_dev, i); - } + } while (i--); setup_nic_dev_done: diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c index 3230dff5ba056..5c177146b35b1 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c @@ -2222,11 +2222,11 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) setup_nic_dev_free: - while (i--) { + do { dev_err(&octeon_dev->pci_dev->dev, "NIC ifidx:%d Setup failed\n", i); liquidio_destroy_nic_device(octeon_dev, i); - } + } while (i--); setup_nic_dev_done: diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index f92a3550e4800..3b1321c8ed143 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -2933,7 +2933,6 @@ static int t3_reenable_adapter(struct adapter *adapter) } pci_set_master(adapter->pdev); pci_restore_state(adapter->pdev); - pci_save_state(adapter->pdev); /* Free sge resources */ t3_free_sge_resources(adapter); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index 392723ef14e51..1ce2091cdc01a 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -5456,7 +5456,6 @@ static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) if (!adap) { pci_restore_state(pdev); - pci_save_state(pdev); return PCI_ERS_RESULT_RECOVERED; } @@ -5471,7 +5470,6 @@ static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); if (t4_wait_dev_ready(adap->regs) < 0) return PCI_ERS_RESULT_DISCONNECT; diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c index 4ee970f3bad6e..ee0154337a9c5 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c @@ -1199,12 +1199,12 @@ static struct sock *chtls_recv_sock(struct sock *lsk, struct ipv6_pinfo *newnp = inet6_sk(newsk); struct ipv6_pinfo *np = inet6_sk(lsk); - inet_sk(newsk)->pinet6 = &newtcp6sk->inet6; + newinet->pinet6 = &newtcp6sk->inet6; + newinet->ipv6_fl_list = NULL; memcpy(newnp, np, sizeof(struct ipv6_pinfo)); newsk->sk_v6_daddr = treq->ir_v6_rmt_addr; newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr; inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr; - newnp->ipv6_fl_list = NULL; newnp->pktoptions = NULL; newsk->sk_bound_dev_if = treq->ir_iif; newinet->inet_opt = NULL; diff --git a/drivers/net/ethernet/ec_bhf.c b/drivers/net/ethernet/ec_bhf.c index 67275aa4f65b2..0c86cbb0313c3 100644 --- a/drivers/net/ethernet/ec_bhf.c +++ b/drivers/net/ethernet/ec_bhf.c @@ -423,7 +423,7 @@ static int ec_bhf_open(struct net_device *net_dev) error_rx_free: dma_free_coherent(dev, priv->rx_buf.alloc_len, priv->rx_buf.alloc, - priv->rx_buf.alloc_len); + priv->rx_buf.alloc_phys); out: return err; } diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c index bb5d2fa157365..8ed45bceb5379 100644 --- a/drivers/net/ethernet/emulex/benet/be_cmds.c +++ b/drivers/net/ethernet/emulex/benet/be_cmds.c @@ -3801,6 +3801,7 @@ int be_cmd_get_perm_mac(struct be_adapter *adapter, u8 *mac) { int status; bool pmac_valid = false; + u32 pmac_id; eth_zero_addr(mac); @@ -3813,7 +3814,7 @@ int be_cmd_get_perm_mac(struct be_adapter *adapter, u8 *mac) adapter->if_handle, 0); } else { status = be_cmd_get_mac_from_list(adapter, mac, &pmac_valid, - NULL, adapter->if_handle, 0); + &pmac_id, adapter->if_handle, 0); } return status; diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index 5bb31c8fab391..995c159003d79 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -2141,7 +2141,7 @@ static int be_get_new_eqd(struct be_eq_obj *eqo) struct be_aic_obj *aic; struct be_rx_obj *rxo; struct be_tx_obj *txo; - u64 rx_pkts = 0, tx_pkts = 0; + u64 rx_pkts = 0, tx_pkts = 0, pkts; ulong now; u32 pps, delta; int i; @@ -2157,15 +2157,17 @@ static int be_get_new_eqd(struct be_eq_obj *eqo) for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { do { start = u64_stats_fetch_begin(&rxo->stats.sync); - rx_pkts += rxo->stats.rx_pkts; + pkts = rxo->stats.rx_pkts; } while (u64_stats_fetch_retry(&rxo->stats.sync, start)); + rx_pkts += pkts; } for_all_tx_queues_on_eq(adapter, eqo, txo, i) { do { start = u64_stats_fetch_begin(&txo->stats.sync); - tx_pkts += txo->stats.tx_reqs; + pkts = txo->stats.tx_reqs; } while (u64_stats_fetch_retry(&txo->stats.sync, start)); + tx_pkts += pkts; } /* Skip, if wrapped around or first calculation */ diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index a863f78412104..06ee7af7fdb9a 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -964,19 +964,19 @@ static int ftgmac100_alloc_rings(struct ftgmac100 *priv) priv->tx_skbs = kcalloc(MAX_TX_QUEUE_ENTRIES, sizeof(void *), GFP_KERNEL); if (!priv->tx_skbs) - return -ENOMEM; + goto err_free_rx_skbs; /* Allocate descriptors */ priv->rxdes = dma_alloc_coherent(priv->dev, MAX_RX_QUEUE_ENTRIES * sizeof(struct ftgmac100_rxdes), &priv->rxdes_dma, GFP_KERNEL); if (!priv->rxdes) - return -ENOMEM; + goto err_free_tx_skbs; priv->txdes = dma_alloc_coherent(priv->dev, MAX_TX_QUEUE_ENTRIES * sizeof(struct ftgmac100_txdes), &priv->txdes_dma, GFP_KERNEL); if (!priv->txdes) - return -ENOMEM; + goto err_free_rxdes; /* Allocate scratch packet buffer */ priv->rx_scratch = dma_alloc_coherent(priv->dev, @@ -984,9 +984,29 @@ static int ftgmac100_alloc_rings(struct ftgmac100 *priv) &priv->rx_scratch_dma, GFP_KERNEL); if (!priv->rx_scratch) - return -ENOMEM; + goto err_free_txdes; return 0; + +err_free_txdes: + dma_free_coherent(priv->dev, + MAX_TX_QUEUE_ENTRIES * + sizeof(struct ftgmac100_txdes), + priv->txdes, priv->txdes_dma); + priv->txdes = NULL; +err_free_rxdes: + dma_free_coherent(priv->dev, + MAX_RX_QUEUE_ENTRIES * + sizeof(struct ftgmac100_rxdes), + priv->rxdes, priv->rxdes_dma); + priv->rxdes = NULL; +err_free_tx_skbs: + kfree(priv->tx_skbs); + priv->tx_skbs = NULL; +err_free_rx_skbs: + kfree(priv->rx_skbs); + priv->rx_skbs = NULL; + return -ENOMEM; } static void ftgmac100_init_rings(struct ftgmac100 *priv) diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index b1e1ad9e4b48e..e212a014c8d41 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -1531,6 +1531,10 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) } if_id = (status & 0xFFFF0000) >> 16; + if (if_id >= ethsw->sw_attr.num_ifs) { + dev_err(dev, "Invalid if_id %d in IRQ status\n", if_id); + goto out_clear; + } port_priv = ethsw->ports[if_id]; if (status & DPSW_IRQ_EVENT_LINK_CHANGED) @@ -1549,6 +1553,7 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) dpaa2_switch_port_connect_mac(port_priv); } +out_clear: err = dpsw_clear_irq_status(ethsw->mc_io, 0, ethsw->dpsw_handle, DPSW_IRQ_INDEX_IF, status); if (err) @@ -3024,6 +3029,19 @@ static int dpaa2_switch_init(struct fsl_mc_device *sw_dev) goto err_close; } + if (!ethsw->sw_attr.num_ifs) { + dev_err(dev, "DPSW device has no interfaces\n"); + err = -ENODEV; + goto err_close; + } + + if (ethsw->sw_attr.num_ifs >= DPSW_MAX_IF) { + dev_err(dev, "DPSW num_ifs %u exceeds max %u\n", + ethsw->sw_attr.num_ifs, DPSW_MAX_IF); + err = -EINVAL; + goto err_close; + } + err = dpsw_get_api_version(ethsw->mc_io, 0, ðsw->major, ðsw->minor); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 0535e92404e3c..b5dd49337513d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -1778,7 +1778,8 @@ int enetc_xdp_xmit(struct net_device *ndev, int num_frames, int xdp_tx_bd_cnt, i, k; int xdp_tx_frm_cnt = 0; - if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) + if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags) || + !netif_carrier_ok(ndev))) return -ENETDOWN; enetc_lock_mdio(); @@ -2502,10 +2503,13 @@ int enetc_configure_si(struct enetc_ndev_priv *priv) struct enetc_hw *hw = &si->hw; int err; - /* set SI cache attributes */ - enetc_wr(hw, ENETC_SICAR0, - ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); - enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); + if (is_enetc_rev1(si)) { + /* set SI cache attributes */ + enetc_wr(hw, ENETC_SICAR0, + ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); + enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); + } + /* enable SI */ enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); @@ -2566,6 +2570,7 @@ EXPORT_SYMBOL_GPL(enetc_free_si_resources); static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) { + struct enetc_si *si = container_of(hw, struct enetc_si, hw); int idx = tx_ring->index; u32 tbmr; @@ -2579,10 +2584,20 @@ static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) enetc_txbdr_wr(hw, idx, ENETC_TBLENR, ENETC_RTBLENR_LEN(tx_ring->bd_count)); - /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ + /* For ENETC v1, clearing PI/CI registers for Tx not supported, + * adjust sw indexes + */ tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); + if (tx_ring->next_to_use != tx_ring->next_to_clean && + !is_enetc_rev1(si)) { + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; + enetc_txbdr_wr(hw, idx, ENETC_TBPIR, 0); + enetc_txbdr_wr(hw, idx, ENETC_TBCIR, 0); + } + /* enable Tx ints by setting pkt thr to 1 */ enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); @@ -3454,7 +3469,7 @@ static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, priv->rx_ring[i] = bdr; err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0, - ENETC_RXB_DMA_SIZE_XDP); + ENETC_RXB_TRUESIZE); if (err) goto free_vector; diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index f279fa597991e..60c7205ea9ff5 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -79,9 +79,9 @@ struct enetc_lso_t { #define ENETC_RXB_TRUESIZE (PAGE_SIZE >> 1) #define ENETC_RXB_PAD NET_SKB_PAD /* add extra space if needed */ #define ENETC_RXB_DMA_SIZE \ - (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - ENETC_RXB_PAD) + min(SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - ENETC_RXB_PAD, 0xffff) #define ENETC_RXB_DMA_SIZE_XDP \ - (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - XDP_PACKET_HEADROOM) + min(SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - XDP_PACKET_HEADROOM, 0xffff) struct enetc_rx_swbd { dma_addr_t dma; diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c index 82c443b28b154..7dbfbc6fbdcb0 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c @@ -49,10 +49,10 @@ static void enetc4_pf_set_si_primary_mac(struct enetc_hw *hw, int si, if (si != 0) { __raw_writel(upper, hw->port + ENETC4_PSIPMAR0(si)); - __raw_writew(lower, hw->port + ENETC4_PSIPMAR1(si)); + __raw_writel(lower, hw->port + ENETC4_PSIPMAR1(si)); } else { __raw_writel(upper, hw->port + ENETC4_PMAR0); - __raw_writew(lower, hw->port + ENETC4_PMAR1); + __raw_writel(lower, hw->port + ENETC4_PMAR1); } } @@ -63,7 +63,7 @@ static void enetc4_pf_get_si_primary_mac(struct enetc_hw *hw, int si, u16 lower; upper = __raw_readl(hw->port + ENETC4_PSIPMAR0(si)); - lower = __raw_readw(hw->port + ENETC4_PSIPMAR1(si)); + lower = __raw_readl(hw->port + ENETC4_PSIPMAR1(si)); put_unaligned_le32(upper, addr); put_unaligned_le16(lower, addr + 4); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_cbdr.c b/drivers/net/ethernet/freescale/enetc/enetc_cbdr.c index 3d5f31879d5c6..a635bfdc30afc 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_cbdr.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_cbdr.c @@ -74,10 +74,6 @@ int enetc4_setup_cbdr(struct enetc_si *si) if (!user->ring) return -ENOMEM; - /* set CBDR cache attributes */ - enetc_wr(hw, ENETC_SICAR2, - ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); - regs.pir = hw->reg + ENETC_SICBDRPIR; regs.cir = hw->reg + ENETC_SICBDRCIR; regs.mr = hw->reg + ENETC_SICBDRMR; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index 71d052de669ab..5166f16f196ff 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -770,9 +770,17 @@ static int enetc_set_rxfh(struct net_device *ndev, struct enetc_si *si = priv->si; int err = 0; + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + /* set hash key, if PF */ - if (rxfh->key && enetc_si_is_pf(si)) + if (rxfh->key) { + if (!enetc_si_is_pf(si)) + return -EOPNOTSUPP; + enetc_set_rss_key(si, rxfh->key); + } /* set RSS table */ if (rxfh->indir) @@ -788,6 +796,8 @@ static void enetc_get_ringparam(struct net_device *ndev, { struct enetc_ndev_priv *priv = netdev_priv(ndev); + ring->rx_max_pending = priv->rx_bd_count; + ring->tx_max_pending = priv->tx_bd_count; ring->rx_pending = priv->rx_bd_count; ring->tx_pending = priv->tx_bd_count; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h index 377c963258147..d382220ef2f0d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h @@ -707,13 +707,24 @@ struct enetc_cmd_rfse { #define ENETC_RFSE_EN BIT(15) #define ENETC_RFSE_MODE_BD 2 +static inline void enetc_get_primary_mac_addr(struct enetc_hw *hw, u8 *addr) +{ + u32 upper; + u16 lower; + + upper = __raw_readl(hw->reg + ENETC_SIPMAR0); + lower = __raw_readl(hw->reg + ENETC_SIPMAR1); + + put_unaligned_le32(upper, addr); + put_unaligned_le16(lower, addr + 4); +} + static inline void enetc_load_primary_mac_addr(struct enetc_hw *hw, struct net_device *ndev) { - u8 addr[ETH_ALEN] __aligned(4); + u8 addr[ETH_ALEN]; - *(u32 *)addr = __raw_readl(hw->reg + ENETC_SIPMAR0); - *(u16 *)(addr + 4) = __raw_readw(hw->reg + ENETC_SIPMAR1); + enetc_get_primary_mac_addr(hw, addr); eth_hw_addr_set(ndev, addr); } diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 3222359ac15b7..f30e8fabfaded 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -1151,7 +1151,7 @@ fec_restart(struct net_device *ndev) u32 rcntl = FEC_RCR_MII; if (OPT_ARCH_HAS_MAX_FL) - rcntl |= (fep->netdev->mtu + ETH_HLEN + ETH_FCS_LEN) << 16; + rcntl |= (fep->netdev->mtu + VLAN_ETH_HLEN + ETH_FCS_LEN) << 16; if (fep->bufdesc_ex) fec_ptp_save_state(fep); @@ -1286,12 +1286,13 @@ fec_restart(struct net_device *ndev) /* When Jumbo Frame is enabled, the FIFO may not be large enough * to hold an entire frame. In such cases, if the MTU exceeds - * (PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN), configure the interface - * to operate in cut-through mode, triggered by the FIFO threshold. + * (PKT_MAXBUF_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN), configure + * the interface to operate in cut-through mode, triggered by + * the FIFO threshold. * Otherwise, enable the ENET store-and-forward mode. */ if ((fep->quirks & FEC_QUIRK_JUMBO_FRAME) && - (ndev->mtu > (PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN))) + (ndev->mtu > (PKT_MAXBUF_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN))) writel(0xF, fep->hwp + FEC_X_WMRK); else writel(FEC_TXWMRK_STRFWD, fep->hwp + FEC_X_WMRK); @@ -3948,7 +3949,12 @@ static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep, txq->bd.cur = bdp; /* Trigger transmission start */ - writel(0, txq->bd.reg_desc_active); + if (!(fep->quirks & FEC_QUIRK_ERR007885) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active)) + writel(0, txq->bd.reg_desc_active); return 0; } @@ -4047,7 +4053,7 @@ static int fec_change_mtu(struct net_device *ndev, int new_mtu) if (netif_running(ndev)) return -EBUSY; - order = get_order(new_mtu + ETH_HLEN + ETH_FCS_LEN + order = get_order(new_mtu + VLAN_ETH_HLEN + ETH_FCS_LEN + FEC_DRV_RESERVE_SPACE); fep->rx_frame_size = (PAGE_SIZE << order) - FEC_DRV_RESERVE_SPACE; fep->pagepool_order = order; @@ -4604,7 +4610,7 @@ fec_probe(struct platform_device *pdev) else fep->max_buf_size = PKT_MAXBUF_SIZE; - ndev->max_mtu = fep->max_buf_size - ETH_HLEN - ETH_FCS_LEN; + ndev->max_mtu = fep->max_buf_size - VLAN_ETH_HLEN - ETH_FCS_LEN; ret = register_netdev(ndev); if (ret) diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c index 4b7bad9a485df..56801c2009d59 100644 --- a/drivers/net/ethernet/freescale/fec_ptp.c +++ b/drivers/net/ethernet/freescale/fec_ptp.c @@ -545,9 +545,6 @@ static int fec_ptp_enable(struct ptp_clock_info *ptp, if (rq->perout.flags) return -EOPNOTSUPP; - if (rq->perout.index != fep->pps_channel) - return -EOPNOTSUPP; - period.tv_sec = rq->perout.period.sec; period.tv_nsec = rq->perout.period.nsec; period_ns = timespec64_to_ns(&period); diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c index affd5a6c44e7b..131d1210dc4a8 100644 --- a/drivers/net/ethernet/freescale/ucc_geth.c +++ b/drivers/net/ethernet/freescale/ucc_geth.c @@ -1602,8 +1602,10 @@ static void ugeth_mac_config(struct phylink_config *config, unsigned int mode, pr_warn("TBI mode requires that the device tree specify a tbi-handle\n"); tbiphy = of_phy_find_device(ug_info->tbi_node); - if (!tbiphy) + if (!tbiphy) { pr_warn("Could not get TBI device\n"); + return; + } value = phy_read(tbiphy, ENET_TBI_MII_CR); value &= ~0x1000; /* Turn off autonegotiation */ diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c index d0a223250845b..edc0c718f71e9 100644 --- a/drivers/net/ethernet/google/gve/gve_ethtool.c +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c @@ -152,11 +152,13 @@ gve_get_ethtool_stats(struct net_device *netdev, u64 tmp_rx_pkts, tmp_rx_hsplit_pkt, tmp_rx_bytes, tmp_rx_hsplit_bytes, tmp_rx_skb_alloc_fail, tmp_rx_buf_alloc_fail, tmp_rx_desc_err_dropped_pkt, tmp_rx_hsplit_unsplit_pkt, - tmp_tx_pkts, tmp_tx_bytes; + tmp_tx_pkts, tmp_tx_bytes, + tmp_xdp_tx_errors, tmp_xdp_redirect_errors; u64 rx_buf_alloc_fail, rx_desc_err_dropped_pkt, rx_hsplit_unsplit_pkt, rx_pkts, rx_hsplit_pkt, rx_skb_alloc_fail, rx_bytes, tx_pkts, tx_bytes, - tx_dropped; - int stats_idx, base_stats_idx, max_stats_idx; + tx_dropped, xdp_tx_errors, xdp_redirect_errors; + int rx_base_stats_idx, max_rx_stats_idx, max_tx_stats_idx; + int stats_idx, stats_region_len, nic_stats_len; struct stats *report_stats; int *rx_qid_to_stats_idx; int *tx_qid_to_stats_idx; @@ -198,6 +200,7 @@ gve_get_ethtool_stats(struct net_device *netdev, for (rx_pkts = 0, rx_bytes = 0, rx_hsplit_pkt = 0, rx_skb_alloc_fail = 0, rx_buf_alloc_fail = 0, rx_desc_err_dropped_pkt = 0, rx_hsplit_unsplit_pkt = 0, + xdp_tx_errors = 0, xdp_redirect_errors = 0, ring = 0; ring < priv->rx_cfg.num_queues; ring++) { if (priv->rx) { @@ -215,6 +218,9 @@ gve_get_ethtool_stats(struct net_device *netdev, rx->rx_desc_err_dropped_pkt; tmp_rx_hsplit_unsplit_pkt = rx->rx_hsplit_unsplit_pkt; + tmp_xdp_tx_errors = rx->xdp_tx_errors; + tmp_xdp_redirect_errors = + rx->xdp_redirect_errors; } while (u64_stats_fetch_retry(&priv->rx[ring].statss, start)); rx_pkts += tmp_rx_pkts; @@ -224,6 +230,8 @@ gve_get_ethtool_stats(struct net_device *netdev, rx_buf_alloc_fail += tmp_rx_buf_alloc_fail; rx_desc_err_dropped_pkt += tmp_rx_desc_err_dropped_pkt; rx_hsplit_unsplit_pkt += tmp_rx_hsplit_unsplit_pkt; + xdp_tx_errors += tmp_xdp_tx_errors; + xdp_redirect_errors += tmp_xdp_redirect_errors; } } for (tx_pkts = 0, tx_bytes = 0, tx_dropped = 0, ring = 0; @@ -249,8 +257,8 @@ gve_get_ethtool_stats(struct net_device *netdev, data[i++] = rx_bytes; data[i++] = tx_bytes; /* total rx dropped packets */ - data[i++] = rx_skb_alloc_fail + rx_buf_alloc_fail + - rx_desc_err_dropped_pkt; + data[i++] = rx_skb_alloc_fail + rx_desc_err_dropped_pkt + + xdp_tx_errors + xdp_redirect_errors; data[i++] = tx_dropped; data[i++] = priv->tx_timeo_cnt; data[i++] = rx_skb_alloc_fail; @@ -265,20 +273,38 @@ gve_get_ethtool_stats(struct net_device *netdev, data[i++] = priv->stats_report_trigger_cnt; i = GVE_MAIN_STATS_LEN; - /* For rx cross-reporting stats, start from nic rx stats in report */ - base_stats_idx = GVE_TX_STATS_REPORT_NUM * num_tx_queues + - GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues; - /* The boundary between driver stats and NIC stats shifts if there are - * stopped queues. - */ - base_stats_idx += NIC_RX_STATS_REPORT_NUM * num_stopped_rxqs + - NIC_TX_STATS_REPORT_NUM * num_stopped_txqs; - max_stats_idx = NIC_RX_STATS_REPORT_NUM * - (priv->rx_cfg.num_queues - num_stopped_rxqs) + - base_stats_idx; + rx_base_stats_idx = 0; + max_rx_stats_idx = 0; + max_tx_stats_idx = 0; + stats_region_len = priv->stats_report_len - + sizeof(struct gve_stats_report); + nic_stats_len = (NIC_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues + + NIC_TX_STATS_REPORT_NUM * num_tx_queues) * sizeof(struct stats); + if (unlikely((stats_region_len - + nic_stats_len) % sizeof(struct stats))) { + net_err_ratelimited("Starting index of NIC stats should be multiple of stats size"); + } else { + /* For rx cross-reporting stats, + * start from nic rx stats in report + */ + rx_base_stats_idx = (stats_region_len - nic_stats_len) / + sizeof(struct stats); + /* The boundary between driver stats and NIC stats + * shifts if there are stopped queues + */ + rx_base_stats_idx += NIC_RX_STATS_REPORT_NUM * + num_stopped_rxqs + NIC_TX_STATS_REPORT_NUM * + num_stopped_txqs; + max_rx_stats_idx = NIC_RX_STATS_REPORT_NUM * + (priv->rx_cfg.num_queues - num_stopped_rxqs) + + rx_base_stats_idx; + max_tx_stats_idx = NIC_TX_STATS_REPORT_NUM * + (num_tx_queues - num_stopped_txqs) + + max_rx_stats_idx; + } /* Preprocess the stats report for rx, map queue id to start index */ skip_nic_stats = false; - for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; + for (stats_idx = rx_base_stats_idx; stats_idx < max_rx_stats_idx; stats_idx += NIC_RX_STATS_REPORT_NUM) { u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); @@ -311,6 +337,9 @@ gve_get_ethtool_stats(struct net_device *netdev, tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; tmp_rx_desc_err_dropped_pkt = rx->rx_desc_err_dropped_pkt; + tmp_xdp_tx_errors = rx->xdp_tx_errors; + tmp_xdp_redirect_errors = + rx->xdp_redirect_errors; } while (u64_stats_fetch_retry(&priv->rx[ring].statss, start)); data[i++] = tmp_rx_bytes; @@ -321,8 +350,9 @@ gve_get_ethtool_stats(struct net_device *netdev, data[i++] = rx->rx_frag_alloc_cnt; /* rx dropped packets */ data[i++] = tmp_rx_skb_alloc_fail + - tmp_rx_buf_alloc_fail + - tmp_rx_desc_err_dropped_pkt; + tmp_rx_desc_err_dropped_pkt + + tmp_xdp_tx_errors + + tmp_xdp_redirect_errors; data[i++] = rx->rx_copybreak_pkt; data[i++] = rx->rx_copied_pkt; /* stats from NIC */ @@ -354,14 +384,9 @@ gve_get_ethtool_stats(struct net_device *netdev, i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS; } - /* For tx cross-reporting stats, start from nic tx stats in report */ - base_stats_idx = max_stats_idx; - max_stats_idx = NIC_TX_STATS_REPORT_NUM * - (num_tx_queues - num_stopped_txqs) + - max_stats_idx; - /* Preprocess the stats report for tx, map queue id to start index */ skip_nic_stats = false; - for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; + /* NIC TX stats start right after NIC RX stats */ + for (stats_idx = max_rx_stats_idx; stats_idx < max_tx_stats_idx; stats_idx += NIC_TX_STATS_REPORT_NUM) { u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 1be1b1ef31ee3..030800776ead1 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -283,9 +283,9 @@ static int gve_alloc_stats_report(struct gve_priv *priv) int tx_stats_num, rx_stats_num; tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) * - gve_num_tx_queues(priv); + priv->tx_cfg.max_queues; rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) * - priv->rx_cfg.num_queues; + priv->rx_cfg.max_queues; priv->stats_report_len = struct_size(priv->stats_report, stats, size_add(tx_stats_num, rx_stats_num)); priv->stats_report = @@ -558,7 +558,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv) block->priv = priv; err = request_irq(priv->msix_vectors[msix_idx].vector, gve_is_gqi(priv) ? gve_intr : gve_intr_dqo, - 0, block->name, block); + IRQF_NO_AUTOEN, block->name, block); if (err) { dev_err(&priv->pdev->dev, "Failed to receive msix vector %d\n", i); diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 6f1d515673d25..5fbee184f0e7a 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -167,6 +167,25 @@ gve_free_pending_packet(struct gve_tx_ring *tx, } } +static void gve_unmap_packet(struct device *dev, + struct gve_tx_pending_packet_dqo *pkt) +{ + int i; + + if (!pkt->num_bufs) + return; + + /* SKB linear portion is guaranteed to be mapped */ + dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]), + dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE); + for (i = 1; i < pkt->num_bufs; i++) { + netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]), + dma_unmap_len(pkt, len[i]), + DMA_TO_DEVICE, 0); + } + pkt->num_bufs = 0; +} + /* gve_tx_free_desc - Cleans up all pending tx requests and buffers. */ static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) @@ -176,21 +195,12 @@ static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) for (i = 0; i < tx->dqo.num_pending_packets; i++) { struct gve_tx_pending_packet_dqo *cur_state = &tx->dqo.pending_packets[i]; - int j; - - for (j = 0; j < cur_state->num_bufs; j++) { - if (j == 0) { - dma_unmap_single(tx->dev, - dma_unmap_addr(cur_state, dma[j]), - dma_unmap_len(cur_state, len[j]), - DMA_TO_DEVICE); - } else { - dma_unmap_page(tx->dev, - dma_unmap_addr(cur_state, dma[j]), - dma_unmap_len(cur_state, len[j]), - DMA_TO_DEVICE); - } - } + + if (tx->dqo.qpl) + gve_free_tx_qpl_bufs(tx, cur_state); + else + gve_unmap_packet(tx->dev, cur_state); + if (cur_state->skb) { dev_consume_skb_any(cur_state->skb); cur_state->skb = NULL; @@ -1158,22 +1168,6 @@ static void remove_from_list(struct gve_tx_ring *tx, } } -static void gve_unmap_packet(struct device *dev, - struct gve_tx_pending_packet_dqo *pkt) -{ - int i; - - /* SKB linear portion is guaranteed to be mapped */ - dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]), - dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE); - for (i = 1; i < pkt->num_bufs; i++) { - netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]), - dma_unmap_len(pkt, len[i]), - DMA_TO_DEVICE, 0); - } - pkt->num_bufs = 0; -} - /* Completion types and expected behavior: * No Miss compl + Packet compl = Packet completed normally. * Miss compl + Re-inject compl = Packet completed normally. diff --git a/drivers/net/ethernet/google/gve/gve_utils.c b/drivers/net/ethernet/google/gve/gve_utils.c index ace9b8698021f..b53b7fcdcdaf1 100644 --- a/drivers/net/ethernet/google/gve/gve_utils.c +++ b/drivers/net/ethernet/google/gve/gve_utils.c @@ -112,11 +112,13 @@ void gve_add_napi(struct gve_priv *priv, int ntfy_idx, netif_napi_add_locked(priv->dev, &block->napi, gve_poll); netif_napi_set_irq_locked(&block->napi, block->irq); + enable_irq(block->irq); } void gve_remove_napi(struct gve_priv *priv, int ntfy_idx) { struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; + disable_irq(block->irq); netif_napi_del_locked(&block->napi); } diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_err.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_err.c index e11495b7ee98c..7234618e8e818 100644 --- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_err.c +++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_err.c @@ -160,7 +160,6 @@ static pci_ers_result_t hbg_pci_err_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); hbg_err_reset(priv); return PCI_ERS_RESULT_RECOVERED; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index bfa5568baa926..c8eba180250e7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -1048,13 +1048,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) int order; if (!alloc_size) - return; + goto not_init; order = get_order(alloc_size); if (order > MAX_PAGE_ORDER) { if (net_ratelimit()) dev_warn(ring_to_dev(ring), "failed to allocate tx spare buffer, exceed to max order\n"); - return; + goto not_init; } tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare), @@ -1092,6 +1092,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) devm_kfree(ring_to_dev(ring), tx_spare); devm_kzalloc_error: ring->tqp->handle->kinfo.tx_spare_buf_size = 0; +not_init: + /* When driver init or reset_init, the ring->tx_spare is always NULL; + * but when called from hns3_set_ringparam, it's usually not NULL, and + * will be restored if hns3_init_all_ring() failed. So it's safe to set + * ring->tx_spare to NULL here. + */ + ring->tx_spare = NULL; } /* Use hns3_tx_spare_space() to make sure there is enough buffer @@ -2500,44 +2507,47 @@ static netdev_features_t hns3_features_check(struct sk_buff *skb, static void hns3_fetch_stats(struct rtnl_link_stats64 *stats, struct hns3_enet_ring *ring, bool is_tx) { + struct ring_stats ring_stats; unsigned int start; do { start = u64_stats_fetch_begin(&ring->syncp); - if (is_tx) { - stats->tx_bytes += ring->stats.tx_bytes; - stats->tx_packets += ring->stats.tx_pkts; - stats->tx_dropped += ring->stats.sw_err_cnt; - stats->tx_dropped += ring->stats.tx_vlan_err; - stats->tx_dropped += ring->stats.tx_l4_proto_err; - stats->tx_dropped += ring->stats.tx_l2l3l4_err; - stats->tx_dropped += ring->stats.tx_tso_err; - stats->tx_dropped += ring->stats.over_max_recursion; - stats->tx_dropped += ring->stats.hw_limitation; - stats->tx_dropped += ring->stats.copy_bits_err; - stats->tx_dropped += ring->stats.skb2sgl_err; - stats->tx_dropped += ring->stats.map_sg_err; - stats->tx_errors += ring->stats.sw_err_cnt; - stats->tx_errors += ring->stats.tx_vlan_err; - stats->tx_errors += ring->stats.tx_l4_proto_err; - stats->tx_errors += ring->stats.tx_l2l3l4_err; - stats->tx_errors += ring->stats.tx_tso_err; - stats->tx_errors += ring->stats.over_max_recursion; - stats->tx_errors += ring->stats.hw_limitation; - stats->tx_errors += ring->stats.copy_bits_err; - stats->tx_errors += ring->stats.skb2sgl_err; - stats->tx_errors += ring->stats.map_sg_err; - } else { - stats->rx_bytes += ring->stats.rx_bytes; - stats->rx_packets += ring->stats.rx_pkts; - stats->rx_dropped += ring->stats.l2_err; - stats->rx_errors += ring->stats.l2_err; - stats->rx_errors += ring->stats.l3l4_csum_err; - stats->rx_crc_errors += ring->stats.l2_err; - stats->multicast += ring->stats.rx_multicast; - stats->rx_length_errors += ring->stats.err_pkt_len; - } + ring_stats = ring->stats; } while (u64_stats_fetch_retry(&ring->syncp, start)); + + if (is_tx) { + stats->tx_bytes += ring_stats.tx_bytes; + stats->tx_packets += ring_stats.tx_pkts; + stats->tx_dropped += ring_stats.sw_err_cnt; + stats->tx_dropped += ring_stats.tx_vlan_err; + stats->tx_dropped += ring_stats.tx_l4_proto_err; + stats->tx_dropped += ring_stats.tx_l2l3l4_err; + stats->tx_dropped += ring_stats.tx_tso_err; + stats->tx_dropped += ring_stats.over_max_recursion; + stats->tx_dropped += ring_stats.hw_limitation; + stats->tx_dropped += ring_stats.copy_bits_err; + stats->tx_dropped += ring_stats.skb2sgl_err; + stats->tx_dropped += ring_stats.map_sg_err; + stats->tx_errors += ring_stats.sw_err_cnt; + stats->tx_errors += ring_stats.tx_vlan_err; + stats->tx_errors += ring_stats.tx_l4_proto_err; + stats->tx_errors += ring_stats.tx_l2l3l4_err; + stats->tx_errors += ring_stats.tx_tso_err; + stats->tx_errors += ring_stats.over_max_recursion; + stats->tx_errors += ring_stats.hw_limitation; + stats->tx_errors += ring_stats.copy_bits_err; + stats->tx_errors += ring_stats.skb2sgl_err; + stats->tx_errors += ring_stats.map_sg_err; + } else { + stats->rx_bytes += ring_stats.rx_bytes; + stats->rx_packets += ring_stats.rx_pkts; + stats->rx_dropped += ring_stats.l2_err; + stats->rx_errors += ring_stats.l2_err; + stats->rx_errors += ring_stats.l3l4_csum_err; + stats->rx_crc_errors += ring_stats.l2_err; + stats->multicast += ring_stats.rx_multicast; + stats->rx_length_errors += ring_stats.err_pkt_len; + } } static void hns3_nic_get_stats64(struct net_device *netdev, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index 9bb708fa42f24..bc333d8710ac1 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -727,11 +727,11 @@ struct hclge_fd_tcam_config_3_cmd { #define HCLGE_FD_AD_DROP_B 0 #define HCLGE_FD_AD_DIRECT_QID_B 1 -#define HCLGE_FD_AD_QID_S 2 -#define HCLGE_FD_AD_QID_M GENMASK(11, 2) +#define HCLGE_FD_AD_QID_L_S 2 +#define HCLGE_FD_AD_QID_L_M GENMASK(11, 2) #define HCLGE_FD_AD_USE_COUNTER_B 12 #define HCLGE_FD_AD_COUNTER_NUM_S 13 -#define HCLGE_FD_AD_COUNTER_NUM_M GENMASK(20, 13) +#define HCLGE_FD_AD_COUNTER_NUM_M GENMASK(19, 13) #define HCLGE_FD_AD_NXT_STEP_B 20 #define HCLGE_FD_AD_NXT_KEY_S 21 #define HCLGE_FD_AD_NXT_KEY_M GENMASK(25, 21) @@ -741,6 +741,7 @@ struct hclge_fd_tcam_config_3_cmd { #define HCLGE_FD_AD_TC_OVRD_B 16 #define HCLGE_FD_AD_TC_SIZE_S 17 #define HCLGE_FD_AD_TC_SIZE_M GENMASK(20, 17) +#define HCLGE_FD_AD_QID_H_B 21 struct hclge_fd_ad_config_cmd { u8 stage; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 782bb48c9f3d7..54d0a9ba7879b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -5679,18 +5679,20 @@ static int hclge_fd_ad_config(struct hclge_dev *hdev, u8 stage, int loc, hnae3_set_field(ad_data, HCLGE_FD_AD_TC_SIZE_M, HCLGE_FD_AD_TC_SIZE_S, (u32)action->tc_size); } + hnae3_set_bit(ad_data, HCLGE_FD_AD_QID_H_B, + action->queue_id >= HCLGE_TQP_MAX_SIZE_DEV_V2 ? 1 : 0); ad_data <<= 32; hnae3_set_bit(ad_data, HCLGE_FD_AD_DROP_B, action->drop_packet); hnae3_set_bit(ad_data, HCLGE_FD_AD_DIRECT_QID_B, action->forward_to_direct_queue); - hnae3_set_field(ad_data, HCLGE_FD_AD_QID_M, HCLGE_FD_AD_QID_S, + hnae3_set_field(ad_data, HCLGE_FD_AD_QID_L_M, HCLGE_FD_AD_QID_L_S, action->queue_id); hnae3_set_bit(ad_data, HCLGE_FD_AD_USE_COUNTER_B, action->use_counter); hnae3_set_field(ad_data, HCLGE_FD_AD_COUNTER_NUM_M, HCLGE_FD_AD_COUNTER_NUM_S, action->counter_id); hnae3_set_bit(ad_data, HCLGE_FD_AD_NXT_STEP_B, action->use_next_stage); hnae3_set_field(ad_data, HCLGE_FD_AD_NXT_KEY_M, HCLGE_FD_AD_NXT_KEY_S, - action->counter_id); + action->next_input_key); req->ad_data = cpu_to_le64(ad_data); ret = hclge_cmd_send(&hdev->hw, &desc, 1); @@ -10562,6 +10564,9 @@ int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto, bool writen_to_tbl = false; int ret = 0; + if (vlan_id >= VLAN_N_VID) + return -EINVAL; + /* When device is resetting or reset failed, firmware is unable to * handle mailbox. Just record the vlan id, and remove it after * reset finished. diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c index c7ff12a6c0764..b7d4e06a55d40 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c @@ -193,10 +193,10 @@ static int hclge_get_ring_chain_from_mbx( return -EINVAL; for (i = 0; i < ring_num; i++) { - if (req->msg.param[i].tqp_index >= vport->nic.kinfo.rss_size) { + if (req->msg.param[i].tqp_index >= vport->nic.kinfo.num_tqps) { dev_err(&hdev->pdev->dev, "tqp index(%u) is out of range(0-%u)\n", req->msg.param[i].tqp_index, - vport->nic.kinfo.rss_size - 1U); + vport->nic.kinfo.num_tqps - 1U); return -EINVAL; } } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 8fcf220a120d2..70327a73dee32 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -368,12 +368,12 @@ static int hclgevf_knic_setup(struct hclgevf_dev *hdev) new_tqps = kinfo->rss_size * num_tc; kinfo->num_tqps = min(new_tqps, hdev->num_tqps); - kinfo->tqp = devm_kcalloc(&hdev->pdev->dev, kinfo->num_tqps, + kinfo->tqp = devm_kcalloc(&hdev->pdev->dev, hdev->num_tqps, sizeof(struct hnae3_queue *), GFP_KERNEL); if (!kinfo->tqp) return -ENOMEM; - for (i = 0; i < kinfo->num_tqps; i++) { + for (i = 0; i < hdev->num_tqps; i++) { hdev->htqp[i].q.handle = &hdev->nic; hdev->htqp[i].q.tqp_index = i; kinfo->tqp[i] = &hdev->htqp[i].q; diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_irq.c b/drivers/net/ethernet/huawei/hinic3/hinic3_irq.c index a69b361225e90..84bee5d6e638e 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_irq.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_irq.c @@ -43,21 +43,12 @@ static void qp_add_napi(struct hinic3_irq_cfg *irq_cfg) struct hinic3_nic_dev *nic_dev = netdev_priv(irq_cfg->netdev); netif_napi_add(nic_dev->netdev, &irq_cfg->napi, hinic3_poll); - netif_queue_set_napi(irq_cfg->netdev, irq_cfg->irq_id, - NETDEV_QUEUE_TYPE_RX, &irq_cfg->napi); - netif_queue_set_napi(irq_cfg->netdev, irq_cfg->irq_id, - NETDEV_QUEUE_TYPE_TX, &irq_cfg->napi); napi_enable(&irq_cfg->napi); } static void qp_del_napi(struct hinic3_irq_cfg *irq_cfg) { napi_disable(&irq_cfg->napi); - netif_queue_set_napi(irq_cfg->netdev, irq_cfg->irq_id, - NETDEV_QUEUE_TYPE_RX, NULL); - netif_queue_set_napi(irq_cfg->netdev, irq_cfg->irq_id, - NETDEV_QUEUE_TYPE_TX, NULL); - netif_stop_subqueue(irq_cfg->netdev, irq_cfg->irq_id); netif_napi_del(&irq_cfg->napi); } @@ -150,6 +141,11 @@ int hinic3_qps_irq_init(struct net_device *netdev) goto err_release_irqs; } + netif_queue_set_napi(irq_cfg->netdev, q_id, + NETDEV_QUEUE_TYPE_RX, &irq_cfg->napi); + netif_queue_set_napi(irq_cfg->netdev, q_id, + NETDEV_QUEUE_TYPE_TX, &irq_cfg->napi); + hinic3_set_msix_auto_mask_state(nic_dev->hwdev, irq_cfg->msix_entry_idx, HINIC3_SET_MSIX_AUTO_MASK); @@ -164,6 +160,10 @@ int hinic3_qps_irq_init(struct net_device *netdev) q_id--; irq_cfg = &nic_dev->q_params.irq_cfg[q_id]; qp_del_napi(irq_cfg); + netif_queue_set_napi(irq_cfg->netdev, q_id, + NETDEV_QUEUE_TYPE_RX, NULL); + netif_queue_set_napi(irq_cfg->netdev, q_id, + NETDEV_QUEUE_TYPE_TX, NULL); hinic3_set_msix_state(nic_dev->hwdev, irq_cfg->msix_entry_idx, HINIC3_MSIX_DISABLE); hinic3_set_msix_auto_mask_state(nic_dev->hwdev, @@ -184,6 +184,10 @@ void hinic3_qps_irq_uninit(struct net_device *netdev) for (q_id = 0; q_id < nic_dev->q_params.num_qps; q_id++) { irq_cfg = &nic_dev->q_params.irq_cfg[q_id]; qp_del_napi(irq_cfg); + netif_queue_set_napi(irq_cfg->netdev, q_id, + NETDEV_QUEUE_TYPE_RX, NULL); + netif_queue_set_napi(irq_cfg->netdev, q_id, + NETDEV_QUEUE_TYPE_TX, NULL); hinic3_set_msix_state(nic_dev->hwdev, irq_cfg->msix_entry_idx, HINIC3_MSIX_DISABLE); hinic3_set_msix_auto_mask_state(nic_dev->hwdev, diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c index 726365c567ef3..75d0bfa7530b4 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c +++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c @@ -496,14 +496,19 @@ static int e1000_set_eeprom(struct net_device *netdev, */ ret_val = e1000_read_eeprom(hw, first_word, 1, &eeprom_buff[0]); + if (ret_val) + goto out; + ptr++; } - if (((eeprom->offset + eeprom->len) & 1) && (ret_val == 0)) { + if ((eeprom->offset + eeprom->len) & 1) { /* need read/modify/write of last changed EEPROM word * only the first byte of the word is being modified */ ret_val = e1000_read_eeprom(hw, last_word, 1, &eeprom_buff[last_word - first_word]); + if (ret_val) + goto out; } /* Device's eeprom is always little-endian, word addressable */ @@ -522,6 +527,7 @@ static int e1000_set_eeprom(struct net_device *netdev, if ((ret_val == 0) && (first_word <= EEPROM_CHECKSUM_REG)) e1000_update_eeprom_checksum(hw); +out: kfree(eeprom_buff); return ret_val; } diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 292389aceb2d4..15160427c8b30 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -2952,8 +2952,6 @@ static int e1000_tx_map(struct e1000_adapter *adapter, dma_error: dev_err(&pdev->dev, "TX DMA map failed\n"); buffer_info->dma = 0; - if (count) - count--; while (count--) { if (i == 0) @@ -4094,7 +4092,15 @@ static bool e1000_tbi_should_accept(struct e1000_adapter *adapter, u32 length, const u8 *data) { struct e1000_hw *hw = &adapter->hw; - u8 last_byte = *(data + length - 1); + u8 last_byte; + + /* Guard against OOB on data[length - 1] */ + if (unlikely(!length)) + return false; + /* Upper bound: length must not exceed rx_buffer_len */ + if (unlikely(length > adapter->rx_buffer_len)) + return false; + last_byte = *(data + length - 1); if (TBI_ACCEPT(hw, status, errors, length, last_byte)) { unsigned long irq_flags; diff --git a/drivers/net/ethernet/intel/e1000e/defines.h b/drivers/net/ethernet/intel/e1000e/defines.h index ba331899d1861..d4a1041e456dc 100644 --- a/drivers/net/ethernet/intel/e1000e/defines.h +++ b/drivers/net/ethernet/intel/e1000e/defines.h @@ -33,6 +33,7 @@ /* Extended Device Control */ #define E1000_CTRL_EXT_LPCD 0x00000004 /* LCD Power Cycle Done */ +#define E1000_CTRL_EXT_DPG_EN 0x00000008 /* Dynamic Power Gating Enable */ #define E1000_CTRL_EXT_SDP3_DATA 0x00000080 /* Value of SW Definable Pin 3 */ #define E1000_CTRL_EXT_FORCE_SMBUS 0x00000800 /* Force SMBus mode */ #define E1000_CTRL_EXT_EE_RST 0x00002000 /* Reinitialize from EEPROM */ diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index df4e7d781cb1c..f9328caefe44b 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -4925,6 +4925,15 @@ static s32 e1000_reset_hw_ich8lan(struct e1000_hw *hw) reg |= E1000_KABGTXD_BGSQLBIAS; ew32(KABGTXD, reg); + /* The hardware reset value of the DPG_EN bit is 1. + * Clear DPG_EN to prevent unexpected autonomous power gating. + */ + if (hw->mac.type >= e1000_pch_ptp) { + reg = er32(CTRL_EXT); + reg &= ~E1000_CTRL_EXT_DPG_EN; + ew32(CTRL_EXT, reg); + } + return 0; } diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 201322dac2330..3e39032696100 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -5654,8 +5654,6 @@ static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb, dma_error: dev_err(&pdev->dev, "Tx DMA map failed\n"); buffer_info->dma = 0; - if (count) - count--; while (count--) { if (i == 0) @@ -7195,7 +7193,6 @@ static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev) "Cannot re-enable PCI device after reset.\n"); result = PCI_ERS_RESULT_DISCONNECT; } else { - pdev->state_saved = true; pci_restore_state(pdev); pci_set_master(pdev); diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c index ae5fe34659cfb..d75b8a50413d2 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c @@ -2423,12 +2423,6 @@ static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - - /* After second error pci->state_saved is false, this - * resets it so EEH doesn't break. - */ - pci_save_state(pdev); - pci_wake_from_d3(pdev, false); result = PCI_ERS_RESULT_RECOVERED; diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 801a57a925dad..feec9e1e13b36 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -1418,4 +1418,15 @@ static inline struct i40e_veb *i40e_pf_get_main_veb(struct i40e_pf *pf) return (pf->lan_veb != I40E_NO_VEB) ? pf->veb[pf->lan_veb] : NULL; } +static inline u32 i40e_get_max_num_descriptors(const struct i40e_pf *pf) +{ + const struct i40e_hw *hw = &pf->hw; + + switch (hw->mac.type) { + case I40E_MAC_XL710: + return I40E_MAX_NUM_DESCRIPTORS_XL710; + default: + return I40E_MAX_NUM_DESCRIPTORS; + } +} #endif /* _I40E_H_ */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 86c72596617a3..61c39e881b008 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -2013,18 +2013,6 @@ static void i40e_get_drvinfo(struct net_device *netdev, drvinfo->n_priv_flags += I40E_GL_PRIV_FLAGS_STR_LEN; } -static u32 i40e_get_max_num_descriptors(struct i40e_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - - switch (hw->mac.type) { - case I40E_MAC_XL710: - return I40E_MAX_NUM_DESCRIPTORS_XL710; - default: - return I40E_MAX_NUM_DESCRIPTORS; - } -} - static void i40e_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct kernel_ethtool_ringparam *kernel_ring, diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 50be0a60ae13b..598739220dfb9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -75,7 +75,13 @@ static const struct pci_device_id i40e_pci_tbl[] = { {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, - {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, + /* + * This ID conflicts with ipw2200, but the devices can be differentiated + * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 + * devices use PCI_CLASS_NETWORK_OTHER. + */ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, I40E_DEV_ID_10G_B), + PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, @@ -2234,6 +2240,7 @@ static void i40e_set_rx_mode(struct net_device *netdev) vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); } + i40e_service_event_schedule(vsi->back); } /** @@ -3562,6 +3569,7 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) u16 pf_q = vsi->base_queue + ring->queue_index; struct i40e_hw *hw = &vsi->back->hw; struct i40e_hmc_obj_rxq rx_ctx; + u32 xdp_frame_sz; int err = 0; bool ok; @@ -3571,49 +3579,47 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) memset(&rx_ctx, 0, sizeof(rx_ctx)); ring->rx_buf_len = vsi->rx_buf_len; + xdp_frame_sz = i40e_rx_pg_size(ring) / 2; /* XDP RX-queue info only needed for RX rings exposed to XDP */ if (ring->vsi->type != I40E_VSI_MAIN) goto skip; - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { - err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->queue_index, - ring->q_vector->napi.napi_id, - ring->rx_buf_len); - if (err) - return err; - } - ring->xsk_pool = i40e_xsk_pool(ring); if (ring->xsk_pool) { - xdp_rxq_info_unreg(&ring->xdp_rxq); + xdp_frame_sz = xsk_pool_get_rx_frag_step(ring->xsk_pool); ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->queue_index, ring->q_vector->napi.napi_id, - ring->rx_buf_len); + xdp_frame_sz); if (err) return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); if (err) - return err; + goto unreg_xdp; dev_info(&vsi->back->pdev->dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->queue_index); } else { + err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->queue_index, + ring->q_vector->napi.napi_id, + xdp_frame_sz); + if (err) + return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL); if (err) - return err; + goto unreg_xdp; } skip: - xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq); + xdp_init_buff(&ring->xdp, xdp_frame_sz, &ring->xdp_rxq); rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); @@ -3647,7 +3653,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) dev_info(&vsi->back->pdev->dev, "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", ring->queue_index, pf_q, err); - return -ENOMEM; + err = -ENOMEM; + goto unreg_xdp; } /* set the context in the HMC */ @@ -3656,7 +3663,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) dev_info(&vsi->back->pdev->dev, "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", ring->queue_index, pf_q, err); - return -ENOMEM; + err = -ENOMEM; + goto unreg_xdp; } /* configure Rx buffer alignment */ @@ -3664,7 +3672,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) if (I40E_2K_TOO_SMALL_WITH_PADDING) { dev_info(&vsi->back->pdev->dev, "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n"); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto unreg_xdp; } clear_ring_build_skb_enabled(ring); } else { @@ -3694,6 +3703,11 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) } return 0; +unreg_xdp: + if (ring->vsi->type == I40E_VSI_MAIN) + xdp_rxq_info_unreg(&ring->xdp_rxq); + + return err; } /** @@ -9029,7 +9043,6 @@ int i40e_open(struct net_device *netdev) TCP_FLAG_FIN | TCP_FLAG_CWR) >> 16); wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); - udp_tunnel_get_rx_info(netdev); return 0; } @@ -16455,7 +16468,6 @@ static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_wake_from_d3(pdev, false); reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); diff --git a/drivers/net/ethernet/intel/i40e/i40e_trace.h b/drivers/net/ethernet/intel/i40e/i40e_trace.h index 759f3d1c4c8f0..dde0ccd789ed1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_trace.h +++ b/drivers/net/ethernet/intel/i40e/i40e_trace.h @@ -88,7 +88,7 @@ TRACE_EVENT(i40e_napi_poll, __entry->rx_clean_complete = rx_clean_complete; __entry->tx_clean_complete = tx_clean_complete; __entry->irq_num = q->irq_num; - __entry->curr_cpu = get_cpu(); + __entry->curr_cpu = smp_processor_id(); __assign_str(qname); __assign_str(dev_name); __assign_bitmask(irq_affinity, cpumask_bits(&q->affinity_mask), diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index cc0b9efc2637a..816179c7e2712 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -1470,6 +1470,9 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) if (!rx_ring->rx_bi) return; + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + if (rx_ring->xsk_pool) { i40e_xsk_clean_rx_ring(rx_ring); goto skip_free; @@ -1527,8 +1530,6 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) void i40e_free_rx_resources(struct i40e_ring *rx_ring) { i40e_clean_rx_ring(rx_ring); - if (rx_ring->vsi->type == I40E_VSI_MAIN) - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); rx_ring->xdp_prog = NULL; kfree(rx_ring->rx_bi); rx_ring->rx_bi = NULL; diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index 081a4526a2f00..8351330930429 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -656,7 +656,7 @@ static int i40e_config_vsi_tx_queue(struct i40e_vf *vf, u16 vsi_id, /* ring_len has to be multiple of 8 */ if (!IS_ALIGNED(info->ring_len, 8) || - info->ring_len > I40E_MAX_NUM_DESCRIPTORS_XL710) { + info->ring_len > i40e_get_max_num_descriptors(pf)) { ret = -EINVAL; goto error_context; } @@ -726,7 +726,7 @@ static int i40e_config_vsi_rx_queue(struct i40e_vf *vf, u16 vsi_id, /* ring_len has to be multiple of 32 */ if (!IS_ALIGNED(info->ring_len, 32) || - info->ring_len > I40E_MAX_NUM_DESCRIPTORS_XL710) { + info->ring_len > i40e_get_max_num_descriptors(pf)) { ret = -EINVAL; goto error_param; } @@ -3818,10 +3818,10 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) cfilter.n_proto = ETH_P_IP; if (mask.dst_ip[0] & tcf.dst_ip[0]) memcpy(&cfilter.ip.v4.dst_ip, tcf.dst_ip, - ARRAY_SIZE(tcf.dst_ip)); - else if (mask.src_ip[0] & tcf.dst_ip[0]) + sizeof(cfilter.ip.v4.dst_ip)); + else if (mask.src_ip[0] & tcf.src_ip[0]) memcpy(&cfilter.ip.v4.src_ip, tcf.src_ip, - ARRAY_SIZE(tcf.dst_ip)); + sizeof(cfilter.ip.v4.src_ip)); break; case VIRTCHNL_TCP_V6_FLOW: cfilter.n_proto = ETH_P_IPV6; @@ -3876,7 +3876,7 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) /* for ipv6, mask is set for all sixteen bytes (4 words) */ if (cfilter.n_proto == ETH_P_IPV6 && mask.dst_ip[3]) if (memcmp(&cfilter.ip.v6.dst_ip6, &cf->ip.v6.dst_ip6, - sizeof(cfilter.ip.v6.src_ip6))) + sizeof(cfilter.ip.v6.dst_ip6))) continue; if (mask.vlan_id) if (cfilter.vlan_id != cf->vlan_id) @@ -3964,10 +3964,10 @@ static int i40e_vc_add_cloud_filter(struct i40e_vf *vf, u8 *msg) cfilter->n_proto = ETH_P_IP; if (mask.dst_ip[0] & tcf.dst_ip[0]) memcpy(&cfilter->ip.v4.dst_ip, tcf.dst_ip, - ARRAY_SIZE(tcf.dst_ip)); - else if (mask.src_ip[0] & tcf.dst_ip[0]) + sizeof(cfilter->ip.v4.dst_ip)); + else if (mask.src_ip[0] & tcf.src_ip[0]) memcpy(&cfilter->ip.v4.src_ip, tcf.src_ip, - ARRAY_SIZE(tcf.dst_ip)); + sizeof(cfilter->ip.v4.src_ip)); break; case VIRTCHNL_TCP_V6_FLOW: cfilter->n_proto = ETH_P_IPV6; diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index a87e0c6d4017a..e9fb0a0919e37 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -260,7 +260,6 @@ struct iavf_adapter { struct work_struct adminq_task; struct work_struct finish_config; wait_queue_head_t down_waitqueue; - wait_queue_head_t reset_waitqueue; wait_queue_head_t vc_waitqueue; struct iavf_q_vector *q_vectors; struct list_head vlan_filter_list; @@ -626,5 +625,5 @@ void iavf_add_adv_rss_cfg(struct iavf_adapter *adapter); void iavf_del_adv_rss_cfg(struct iavf_adapter *adapter); struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, const u8 *macaddr); -int iavf_wait_for_reset(struct iavf_adapter *adapter); +void iavf_reset_step(struct iavf_adapter *adapter); #endif /* _IAVF_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 05d72be3fe80c..9ed1fb9432fc7 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -313,14 +313,13 @@ static int iavf_get_sset_count(struct net_device *netdev, int sset) { /* Report the maximum number queues, even if not every queue is * currently configured. Since allocation of queues is in pairs, - * use netdev->real_num_tx_queues * 2. The real_num_tx_queues is set - * at device creation and never changes. + * use netdev->num_tx_queues * 2. The num_tx_queues is set at + * device creation and never changes. */ if (sset == ETH_SS_STATS) return IAVF_STATS_LEN + - (IAVF_QUEUE_STATS_LEN * 2 * - netdev->real_num_tx_queues); + (IAVF_QUEUE_STATS_LEN * 2 * netdev->num_tx_queues); else return -EINVAL; } @@ -345,19 +344,19 @@ static void iavf_get_ethtool_stats(struct net_device *netdev, iavf_add_ethtool_stats(&data, adapter, iavf_gstrings_stats); rcu_read_lock(); - /* As num_active_queues describe both tx and rx queues, we can use - * it to iterate over rings' stats. + /* Use num_tx_queues to report stats for the maximum number of queues. + * Queues beyond num_active_queues will report zero. */ - for (i = 0; i < adapter->num_active_queues; i++) { - struct iavf_ring *ring; + for (i = 0; i < netdev->num_tx_queues; i++) { + struct iavf_ring *tx_ring = NULL, *rx_ring = NULL; - /* Tx rings stats */ - ring = &adapter->tx_rings[i]; - iavf_add_queue_stats(&data, ring); + if (i < adapter->num_active_queues) { + tx_ring = &adapter->tx_rings[i]; + rx_ring = &adapter->rx_rings[i]; + } - /* Rx rings stats */ - ring = &adapter->rx_rings[i]; - iavf_add_queue_stats(&data, ring); + iavf_add_queue_stats(&data, tx_ring); + iavf_add_queue_stats(&data, rx_ring); } rcu_read_unlock(); } @@ -376,9 +375,9 @@ static void iavf_get_stat_strings(struct net_device *netdev, u8 *data) iavf_add_stat_strings(&data, iavf_gstrings_stats); /* Queues are always allocated in pairs, so we just use - * real_num_tx_queues for both Tx and Rx queues. + * num_tx_queues for both Tx and Rx queues. */ - for (i = 0; i < netdev->real_num_tx_queues; i++) { + for (i = 0; i < netdev->num_tx_queues; i++) { iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, "tx", i); iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, @@ -492,7 +491,6 @@ static int iavf_set_ringparam(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); u32 new_rx_count, new_tx_count; - int ret = 0; if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) return -EINVAL; @@ -537,13 +535,11 @@ static int iavf_set_ringparam(struct net_device *netdev, } if (netif_running(netdev)) { - iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); - ret = iavf_wait_for_reset(adapter); - if (ret) - netdev_warn(netdev, "Changing ring parameters timeout or interrupted waiting for reset"); + adapter->flags |= IAVF_FLAG_RESET_NEEDED; + iavf_reset_step(adapter); } - return ret; + return 0; } /** @@ -1625,7 +1621,6 @@ static int iavf_set_channels(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); u32 num_req = ch->combined_count; - int ret = 0; if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && adapter->num_tc) { @@ -1647,13 +1642,10 @@ static int iavf_set_channels(struct net_device *netdev, adapter->num_req_queues = num_req; adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED; - iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); - - ret = iavf_wait_for_reset(adapter); - if (ret) - netdev_warn(netdev, "Changing channel count timeout or interrupted waiting for reset"); + adapter->flags |= IAVF_FLAG_RESET_NEEDED; + iavf_reset_step(adapter); - return ret; + return 0; } /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index c2fbe443ef853..0a72d419782e5 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -185,31 +185,6 @@ static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter) return false; } -/** - * iavf_wait_for_reset - Wait for reset to finish. - * @adapter: board private structure - * - * Returns 0 if reset finished successfully, negative on timeout or interrupt. - */ -int iavf_wait_for_reset(struct iavf_adapter *adapter) -{ - int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue, - !iavf_is_reset_in_progress(adapter), - msecs_to_jiffies(5000)); - - /* If ret < 0 then it means wait was interrupted. - * If ret == 0 then it means we got a timeout while waiting - * for reset to finish. - * If ret > 0 it means reset has finished. - */ - if (ret > 0) - return 0; - else if (ret < 0) - return -EINTR; - else - return -EBUSY; -} - /** * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code * @hw: pointer to the HW structure @@ -782,10 +757,13 @@ iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, adapter->num_vlan_filters++; iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER); } else if (f->state == IAVF_VLAN_REMOVE) { - /* IAVF_VLAN_REMOVE means that VLAN wasn't yet removed. - * We can safely only change the state here. + /* Re-add the filter since we cannot tell whether the + * pending delete has already been processed by the PF. + * A duplicate add is harmless. */ - f->state = IAVF_VLAN_ACTIVE; + f->state = IAVF_VLAN_ADD; + iavf_schedule_aq_request(adapter, + IAVF_FLAG_AQ_ADD_VLAN_FILTER); } clearout: @@ -1726,11 +1704,11 @@ static int iavf_config_rss_reg(struct iavf_adapter *adapter) u16 i; dw = (u32 *)adapter->rss_key; - for (i = 0; i <= adapter->rss_key_size / 4; i++) + for (i = 0; i < adapter->rss_key_size / 4; i++) wr32(hw, IAVF_VFQF_HKEY(i), dw[i]); dw = (u32 *)adapter->rss_lut; - for (i = 0; i <= adapter->rss_lut_size / 4; i++) + for (i = 0; i < adapter->rss_lut_size / 4; i++) wr32(hw, IAVF_VFQF_HLUT(i), dw[i]); iavf_flush(hw); @@ -2797,7 +2775,22 @@ static void iavf_init_config_adapter(struct iavf_adapter *adapter) netdev->watchdog_timeo = 5 * HZ; netdev->min_mtu = ETH_MIN_MTU; - netdev->max_mtu = LIBIE_MAX_MTU; + + /* PF/VF API: vf_res->max_mtu is max frame size (not MTU). + * Convert to MTU. + */ + if (!adapter->vf_res->max_mtu) { + netdev->max_mtu = LIBIE_MAX_MTU; + } else if (adapter->vf_res->max_mtu < LIBETH_RX_LL_LEN + ETH_MIN_MTU || + adapter->vf_res->max_mtu > + LIBETH_RX_LL_LEN + LIBIE_MAX_MTU) { + netdev_warn_once(adapter->netdev, + "invalid max frame size %d from PF, using default MTU %d", + adapter->vf_res->max_mtu, LIBIE_MAX_MTU); + netdev->max_mtu = LIBIE_MAX_MTU; + } else { + netdev->max_mtu = adapter->vf_res->max_mtu - LIBETH_RX_LL_LEN; + } if (!is_valid_ether_addr(adapter->hw.mac.addr)) { dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", @@ -3025,6 +3018,8 @@ static void iavf_disable_vf(struct iavf_adapter *adapter) adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; + iavf_ptp_release(adapter); + /* We don't use netif_running() because it may be true prior to * ndo_open() returning, so we can't assume it means all our open * tasks have finished, since we're not holding the rtnl_lock here. @@ -3100,18 +3095,16 @@ static void iavf_reconfig_qs_bw(struct iavf_adapter *adapter) } /** - * iavf_reset_task - Call-back task to handle hardware reset - * @work: pointer to work_struct + * iavf_reset_step - Perform the VF reset sequence + * @adapter: board private structure * - * During reset we need to shut down and reinitialize the admin queue - * before we can use it to communicate with the PF again. We also clear - * and reinit the rings because that context is lost as well. - **/ -static void iavf_reset_task(struct work_struct *work) + * Requests a reset from PF, polls for completion, and reconfigures + * the driver. Caller must hold the netdev instance lock. + * + * This can sleep for several seconds while polling HW registers. + */ +void iavf_reset_step(struct iavf_adapter *adapter) { - struct iavf_adapter *adapter = container_of(work, - struct iavf_adapter, - reset_task); struct virtchnl_vf_resource *vfres = adapter->vf_res; struct net_device *netdev = adapter->netdev; struct iavf_hw *hw = &adapter->hw; @@ -3122,7 +3115,7 @@ static void iavf_reset_task(struct work_struct *work) int i = 0, err; bool running; - netdev_lock(netdev); + netdev_assert_locked(netdev); iavf_misc_irq_disable(adapter); if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { @@ -3167,7 +3160,6 @@ static void iavf_reset_task(struct work_struct *work) dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", reg_val); iavf_disable_vf(adapter); - netdev_unlock(netdev); return; /* Do not attempt to reinit. It's dead, Jim. */ } @@ -3179,7 +3171,6 @@ static void iavf_reset_task(struct work_struct *work) iavf_startup(adapter); queue_delayed_work(adapter->wq, &adapter->watchdog_task, msecs_to_jiffies(30)); - netdev_unlock(netdev); return; } @@ -3200,6 +3191,8 @@ static void iavf_reset_task(struct work_struct *work) iavf_change_state(adapter, __IAVF_RESETTING); adapter->flags &= ~IAVF_FLAG_RESET_PENDING; + iavf_ptp_release(adapter); + /* free the Tx/Rx rings and descriptors, might be better to just * re-use them sometime in the future */ @@ -3320,9 +3313,6 @@ static void iavf_reset_task(struct work_struct *work) adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; - wake_up(&adapter->reset_waitqueue); - netdev_unlock(netdev); - return; reset_err: if (running) { @@ -3331,10 +3321,21 @@ static void iavf_reset_task(struct work_struct *work) } iavf_disable_vf(adapter); - netdev_unlock(netdev); dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); } +static void iavf_reset_task(struct work_struct *work) +{ + struct iavf_adapter *adapter = container_of(work, + struct iavf_adapter, + reset_task); + struct net_device *netdev = adapter->netdev; + + netdev_lock(netdev); + iavf_reset_step(adapter); + netdev_unlock(netdev); +} + /** * iavf_adminq_task - worker thread to clean the admin queue * @work: pointer to work_struct containing our data @@ -4600,22 +4601,17 @@ static int iavf_close(struct net_device *netdev) static int iavf_change_mtu(struct net_device *netdev, int new_mtu) { struct iavf_adapter *adapter = netdev_priv(netdev); - int ret = 0; netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); WRITE_ONCE(netdev->mtu, new_mtu); if (netif_running(netdev)) { - iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); - ret = iavf_wait_for_reset(adapter); - if (ret < 0) - netdev_warn(netdev, "MTU change interrupted waiting for reset"); - else if (ret) - netdev_warn(netdev, "MTU change timed out waiting for reset"); + adapter->flags |= IAVF_FLAG_RESET_NEEDED; + iavf_reset_step(adapter); } - return ret; + return 0; } /** @@ -5420,9 +5416,6 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Setup the wait queue for indicating transition to down status */ init_waitqueue_head(&adapter->down_waitqueue); - /* Setup the wait queue for indicating transition to running state */ - init_waitqueue_head(&adapter->reset_waitqueue); - /* Setup the wait queue for indicating virtchannel events */ init_waitqueue_head(&adapter->vc_waitqueue); diff --git a/drivers/net/ethernet/intel/iavf/iavf_ptp.c b/drivers/net/ethernet/intel/iavf/iavf_ptp.c index b4d5eda2e84fc..9cbd8c1540318 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ptp.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ptp.c @@ -252,6 +252,12 @@ static int iavf_ptp_gettimex64(struct ptp_clock_info *info, return iavf_read_phc_indirect(adapter, ts, sts); } +static int iavf_ptp_settime64(struct ptp_clock_info *info, + const struct timespec64 *ts) +{ + return -EOPNOTSUPP; +} + /** * iavf_ptp_cache_phc_time - Cache PHC time for performing timestamp extension * @adapter: private adapter structure @@ -320,6 +326,7 @@ static int iavf_ptp_register_clock(struct iavf_adapter *adapter) KBUILD_MODNAME, dev_name(dev)); ptp_info->owner = THIS_MODULE; ptp_info->gettimex64 = iavf_ptp_gettimex64; + ptp_info->settime64 = iavf_ptp_settime64; ptp_info->do_aux_work = iavf_ptp_do_aux_work; clock = ptp_clock_register(ptp_info, dev); diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 34a422a4a29c1..291b21230b65f 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -2732,7 +2732,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, case VIRTCHNL_OP_ENABLE_QUEUES: /* enable transmits */ iavf_irq_enable(adapter, true); - wake_up(&adapter->reset_waitqueue); adapter->flags &= ~IAVF_FLAG_QUEUES_DISABLED; break; case VIRTCHNL_OP_DISABLE_QUEUES: diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c index fb2de521731ae..862ff1cdd46d6 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c @@ -459,6 +459,8 @@ static void ice_devlink_reinit_down(struct ice_pf *pf) rtnl_lock(); ice_vsi_decfg(ice_get_main_vsi(pf)); rtnl_unlock(); + ice_deinit_pf(pf); + ice_deinit_hw(&pf->hw); ice_deinit_dev(pf); } @@ -1231,11 +1233,13 @@ static void ice_set_min_max_msix(struct ice_pf *pf) static int ice_devlink_reinit_up(struct ice_pf *pf) { struct ice_vsi *vsi = ice_get_main_vsi(pf); + struct device *dev = ice_pf_to_dev(pf); + bool need_dev_deinit = false; int err; err = ice_init_hw(&pf->hw); if (err) { - dev_err(ice_pf_to_dev(pf), "ice_init_hw failed: %d\n", err); + dev_err(dev, "ice_init_hw failed: %d\n", err); return err; } @@ -1246,13 +1250,19 @@ static int ice_devlink_reinit_up(struct ice_pf *pf) if (err) goto unroll_hw_init; + err = ice_init_pf(pf); + if (err) { + dev_err(dev, "ice_init_pf failed: %d\n", err); + goto unroll_dev_init; + } + vsi->flags = ICE_VSI_FLAG_INIT; rtnl_lock(); err = ice_vsi_cfg(vsi); rtnl_unlock(); if (err) - goto err_vsi_cfg; + goto unroll_pf_init; /* No need to take devl_lock, it's already taken by devlink API */ err = ice_load(pf); @@ -1265,10 +1275,14 @@ static int ice_devlink_reinit_up(struct ice_pf *pf) rtnl_lock(); ice_vsi_decfg(vsi); rtnl_unlock(); -err_vsi_cfg: - ice_deinit_dev(pf); +unroll_pf_init: + ice_deinit_pf(pf); +unroll_dev_init: + need_dev_deinit = true; unroll_hw_init: ice_deinit_hw(&pf->hw); + if (need_dev_deinit) + ice_deinit_dev(pf); return err; } @@ -1343,7 +1357,7 @@ ice_devlink_enable_roce_get(struct devlink *devlink, u32 id, cdev = pf->cdev_info; if (!cdev) - return -ENODEV; + return -EOPNOTSUPP; ctx->val.vbool = !!(cdev->rdma_protocol & IIDC_RDMA_PROTOCOL_ROCEV2); @@ -1409,7 +1423,7 @@ ice_devlink_enable_iw_get(struct devlink *devlink, u32 id, cdev = pf->cdev_info; if (!cdev) - return -ENODEV; + return -EOPNOTSUPP; ctx->val.vbool = !!(cdev->rdma_protocol & IIDC_RDMA_PROTOCOL_IWARP); diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 22b8323ff0d0e..6886188043764 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -977,6 +977,7 @@ void ice_map_xdp_rings(struct ice_vsi *vsi); int ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags); +int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size); int ice_set_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size); int ice_get_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size); int ice_set_rss_key(struct ice_vsi *vsi, u8 *seed); @@ -987,6 +988,7 @@ int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset); void ice_print_link_msg(struct ice_vsi *vsi, bool isup); int ice_plug_aux_dev(struct ice_pf *pf); void ice_unplug_aux_dev(struct ice_pf *pf); +void ice_rdma_finalize_setup(struct ice_pf *pf); int ice_init_rdma(struct ice_pf *pf); void ice_deinit_rdma(struct ice_pf *pf); bool ice_is_wol_supported(struct ice_hw *hw); @@ -1029,11 +1031,15 @@ int ice_open(struct net_device *netdev); int ice_open_internal(struct net_device *netdev); int ice_stop(struct net_device *netdev); void ice_service_task_schedule(struct ice_pf *pf); +void ice_start_service_task(struct ice_pf *pf); int ice_load(struct ice_pf *pf); void ice_unload(struct ice_pf *pf); void ice_adv_lnk_speed_maps_init(void); +void ice_init_dev_hw(struct ice_pf *pf); int ice_init_dev(struct ice_pf *pf); void ice_deinit_dev(struct ice_pf *pf); +int ice_init_pf(struct ice_pf *pf); +void ice_deinit_pf(struct ice_pf *pf); int ice_change_mtu(struct net_device *netdev, int new_mtu); void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue); int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp); diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 2532b6f82e971..c23a31ec3c413 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -1161,6 +1161,9 @@ int ice_init_hw(struct ice_hw *hw) status = ice_init_hw_tbls(hw); if (status) goto err_unroll_fltr_mgmt_struct; + + ice_init_dev_hw(hw->back); + mutex_init(&hw->tnl_lock); ice_init_chk_recipe_reuse_support(hw); @@ -1851,6 +1854,7 @@ static bool ice_should_retry_sq_send_cmd(u16 opcode) case ice_aqc_opc_lldp_stop: case ice_aqc_opc_lldp_start: case ice_aqc_opc_lldp_filter_ctrl: + case ice_aqc_opc_sff_eeprom: return true; } @@ -1876,6 +1880,7 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq, { struct libie_aq_desc desc_cpy; bool is_cmd_for_retry; + u8 *buf_cpy = NULL; u8 idx = 0; u16 opcode; int status; @@ -1885,8 +1890,11 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq, memset(&desc_cpy, 0, sizeof(desc_cpy)); if (is_cmd_for_retry) { - /* All retryable cmds are direct, without buf. */ - WARN_ON(buf); + if (buf) { + buf_cpy = kmemdup(buf, buf_size, GFP_KERNEL); + if (!buf_cpy) + return -ENOMEM; + } memcpy(&desc_cpy, desc, sizeof(desc_cpy)); } @@ -1898,12 +1906,14 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq, hw->adminq.sq_last_status != LIBIE_AQ_RC_EBUSY) break; + if (buf_cpy) + memcpy(buf, buf_cpy, buf_size); memcpy(desc, &desc_cpy, sizeof(desc_cpy)); - msleep(ICE_SQ_SEND_DELAY_TIME_MS); } while (++idx < ICE_SQ_SEND_MAX_EXECUTE); + kfree(buf_cpy); return status; } @@ -2248,7 +2258,7 @@ void ice_release_res(struct ice_hw *hw, enum ice_aq_res_ids res) /* there are some rare cases when trying to release the resource * results in an admin queue timeout, so handle them correctly */ - timeout = jiffies + 10 * ICE_CTL_Q_SQ_CMD_TIMEOUT; + timeout = jiffies + 10 * usecs_to_jiffies(ICE_CTL_Q_SQ_CMD_TIMEOUT); do { status = ice_aq_release_res(hw, res, 0, NULL); if (status != -EIO) @@ -6425,7 +6435,7 @@ int ice_lldp_fltr_add_remove(struct ice_hw *hw, struct ice_vsi *vsi, bool add) struct ice_aqc_lldp_filter_ctrl *cmd; struct libie_aq_desc desc; - if (vsi->type != ICE_VSI_PF || !ice_fw_supports_lldp_fltr_ctrl(hw)) + if (!ice_fw_supports_lldp_fltr_ctrl(hw)) return -EOPNOTSUPP; cmd = libie_aq_raw(&desc); diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index dc131779d4267..912bcf9fce52d 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -1940,6 +1940,17 @@ __ice_get_ethtool_stats(struct net_device *netdev, int i = 0; char *p; + if (ice_is_port_repr_netdev(netdev)) { + ice_update_eth_stats(vsi); + + for (j = 0; j < ICE_VSI_STATS_LEN; j++) { + p = (char *)vsi + ice_gstrings_vsi_stats[j].stat_offset; + data[i++] = (ice_gstrings_vsi_stats[j].sizeof_stat == + sizeof(u64)) ? *(u64 *)p : *(u32 *)p; + } + return; + } + ice_update_pf_stats(pf); ice_update_vsi_stats(vsi); @@ -1949,9 +1960,6 @@ __ice_get_ethtool_stats(struct net_device *netdev, sizeof(u64)) ? *(u64 *)p : *(u32 *)p; } - if (ice_is_port_repr_netdev(netdev)) - return; - /* populate per queue stats */ rcu_read_lock(); @@ -3312,7 +3320,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, rx_rings = kcalloc(vsi->num_rxq, sizeof(*rx_rings), GFP_KERNEL); if (!rx_rings) { err = -ENOMEM; - goto done; + goto free_xdp; } ice_for_each_rxq(vsi, i) { @@ -3342,7 +3350,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, } kfree(rx_rings); err = -ENOMEM; - goto free_tx; + goto free_xdp; } } @@ -3393,6 +3401,13 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, } goto done; +free_xdp: + if (xdp_rings) { + ice_for_each_xdp_txq(vsi, i) + ice_free_tx_ring(&xdp_rings[i]); + kfree(xdp_rings); + } + free_tx: /* error cleanup if the Rx allocations failed after getting Tx */ if (tx_rings) { @@ -3621,11 +3636,7 @@ ice_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) if (!lut) return -ENOMEM; - err = ice_get_rss_key(vsi, rxfh->key); - if (err) - goto out; - - err = ice_get_rss_lut(vsi, lut, vsi->rss_table_size); + err = ice_get_rss(vsi, rxfh->key, lut, vsi->rss_table_size); if (err) goto out; @@ -4499,7 +4510,7 @@ ice_get_module_eeprom(struct net_device *netdev, struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; bool is_sfp = false; - unsigned int i, j; + unsigned int i; u16 offset = 0; u8 page = 0; int status; @@ -4541,26 +4552,19 @@ ice_get_module_eeprom(struct net_device *netdev, if (page == 0 || !(data[0x2] & 0x4)) { u32 copy_len; - /* If i2c bus is busy due to slow page change or - * link management access, call can fail. This is normal. - * So we retry this a few times. - */ - for (j = 0; j < 4; j++) { - status = ice_aq_sff_eeprom(hw, 0, addr, offset, page, - !is_sfp, value, - SFF_READ_BLOCK_SIZE, - 0, NULL); - netdev_dbg(netdev, "SFF %02X %02X %02X %X = %02X%02X%02X%02X.%02X%02X%02X%02X (%X)\n", - addr, offset, page, is_sfp, - value[0], value[1], value[2], value[3], - value[4], value[5], value[6], value[7], - status); - if (status) { - usleep_range(1500, 2500); - memset(value, 0, SFF_READ_BLOCK_SIZE); - continue; - } - break; + status = ice_aq_sff_eeprom(hw, 0, addr, offset, page, + !is_sfp, value, + SFF_READ_BLOCK_SIZE, + 0, NULL); + netdev_dbg(netdev, "SFF %02X %02X %02X %X = %02X%02X%02X%02X.%02X%02X%02X%02X (%pe)\n", + addr, offset, page, is_sfp, + value[0], value[1], value[2], value[3], + value[4], value[5], value[6], value[7], + ERR_PTR(status)); + if (status) { + netdev_err(netdev, "%s: error reading module EEPROM: status %pe\n", + __func__, ERR_PTR(status)); + return status; } /* Make sure we have enough room for the new block */ diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c index 420d45c2558b6..ded029aa71d7d 100644 --- a/drivers/net/ethernet/intel/ice/ice_idc.c +++ b/drivers/net/ethernet/intel/ice/ice_idc.c @@ -360,6 +360,39 @@ void ice_unplug_aux_dev(struct ice_pf *pf) auxiliary_device_uninit(adev); } +/** + * ice_rdma_finalize_setup - Complete RDMA setup after VSI is ready + * @pf: ptr to ice_pf + * + * Sets VSI-dependent information and plugs aux device. + * Must be called after ice_init_rdma(), ice_vsi_rebuild(), and + * ice_dcb_rebuild() complete. + */ +void ice_rdma_finalize_setup(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + struct iidc_rdma_priv_dev_info *privd; + int ret; + + if (!ice_is_rdma_ena(pf) || !pf->cdev_info) + return; + + privd = pf->cdev_info->iidc_priv; + if (!privd || !pf->vsi || !pf->vsi[0] || !pf->vsi[0]->netdev) + return; + + /* Assign VSI info now that VSI is valid */ + privd->netdev = pf->vsi[0]->netdev; + privd->vport_id = pf->vsi[0]->vsi_num; + + /* Update QoS info after DCB has been rebuilt */ + ice_setup_dcb_qos_info(pf, &privd->qos_info); + + ret = ice_plug_aux_dev(pf); + if (ret) + dev_warn(dev, "Failed to plug RDMA aux device: %d\n", ret); +} + /** * ice_init_rdma - initializes PF for RDMA use * @pf: ptr to ice_pf @@ -398,22 +431,14 @@ int ice_init_rdma(struct ice_pf *pf) } cdev->iidc_priv = privd; - privd->netdev = pf->vsi[0]->netdev; privd->hw_addr = (u8 __iomem *)pf->hw.hw_addr; cdev->pdev = pf->pdev; - privd->vport_id = pf->vsi[0]->vsi_num; pf->cdev_info->rdma_protocol |= IIDC_RDMA_PROTOCOL_ROCEV2; - ice_setup_dcb_qos_info(pf, &privd->qos_info); - ret = ice_plug_aux_dev(pf); - if (ret) - goto err_plug_aux_dev; + return 0; -err_plug_aux_dev: - pf->cdev_info->adev = NULL; - xa_erase(&ice_aux_id, pf->aux_idx); err_alloc_xa: kfree(privd); err_privd_alloc: @@ -432,7 +457,6 @@ void ice_deinit_rdma(struct ice_pf *pf) if (!ice_is_rdma_ena(pf)) return; - ice_unplug_aux_dev(pf); xa_erase(&ice_aux_id, pf->aux_idx); kfree(pf->cdev_info->iidc_priv); kfree(pf->cdev_info); diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 4479c824561e9..3d14932871c58 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -398,6 +398,8 @@ static int ice_vsi_alloc_ring_stats(struct ice_vsi *vsi) if (!ring_stats) goto err_out; + u64_stats_init(&ring_stats->syncp); + WRITE_ONCE(tx_ring_stats[i], ring_stats); } @@ -417,6 +419,8 @@ static int ice_vsi_alloc_ring_stats(struct ice_vsi *vsi) if (!ring_stats) goto err_out; + u64_stats_init(&ring_stats->syncp); + WRITE_ONCE(rx_ring_stats[i], ring_stats); } @@ -2780,12 +2784,14 @@ void ice_vsi_set_napi_queues(struct ice_vsi *vsi) return; ice_for_each_rxq(vsi, q_idx) - netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_RX, - &vsi->rx_rings[q_idx]->q_vector->napi); + if (vsi->rx_rings[q_idx] && vsi->rx_rings[q_idx]->q_vector) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_RX, + &vsi->rx_rings[q_idx]->q_vector->napi); ice_for_each_txq(vsi, q_idx) - netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_TX, - &vsi->tx_rings[q_idx]->q_vector->napi); + if (vsi->tx_rings[q_idx] && vsi->tx_rings[q_idx]->q_vector) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_TX, + &vsi->tx_rings[q_idx]->q_vector->napi); /* Also set the interrupt number for the NAPI */ ice_for_each_q_vector(vsi, v_idx) { struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; @@ -3806,22 +3812,31 @@ int ice_vsi_add_vlan_zero(struct ice_vsi *vsi) int ice_vsi_del_vlan_zero(struct ice_vsi *vsi) { struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); + struct ice_pf *pf = vsi->back; struct ice_vlan vlan; int err; - vlan = ICE_VLAN(0, 0, 0); - err = vlan_ops->del_vlan(vsi, &vlan); - if (err && err != -EEXIST) - return err; + if (pf->lag && pf->lag->primary) { + dev_dbg(ice_pf_to_dev(pf), "Interface is primary in aggregate - not deleting prune list\n"); + } else { + vlan = ICE_VLAN(0, 0, 0); + err = vlan_ops->del_vlan(vsi, &vlan); + if (err && err != -EEXIST) + return err; + } /* in SVM both VLAN 0 filters are identical */ if (!ice_is_dvm_ena(&vsi->back->hw)) return 0; - vlan = ICE_VLAN(ETH_P_8021Q, 0, 0); - err = vlan_ops->del_vlan(vsi, &vlan); - if (err && err != -EEXIST) - return err; + if (pf->lag && pf->lag->primary) { + dev_dbg(ice_pf_to_dev(pf), "Interface is primary in aggregate - not deleting QinQ prune list\n"); + } else { + vlan = ICE_VLAN(ETH_P_8021Q, 0, 0); + err = vlan_ops->del_vlan(vsi, &vlan); + if (err && err != -EEXIST) + return err; + } /* when deleting the last VLAN filter, make sure to disable the VLAN * promisc mode so the filter isn't left by accident diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 86f5859e88ef5..a4ae032f2161b 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -3323,18 +3323,20 @@ static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data) if (ice_is_reset_in_progress(pf->state)) goto skip_irq; - if (test_and_clear_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread)) { - /* Process outstanding Tx timestamps. If there is more work, - * re-arm the interrupt to trigger again. - */ - if (ice_ptp_process_ts(pf) == ICE_TX_TSTAMP_WORK_PENDING) { - wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M); - ice_flush(hw); - } - } + if (test_and_clear_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread)) + ice_ptp_process_ts(pf); skip_irq: ice_irq_dynamic_ena(hw, NULL, NULL); + ice_flush(hw); + + if (ice_ptp_tx_tstamps_pending(pf)) { + /* If any new Tx timestamps happened while in interrupt, + * re-arm the interrupt to trigger it again. + */ + wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M); + ice_flush(hw); + } return IRQ_HANDLED; } @@ -3949,9 +3951,10 @@ u16 ice_get_avail_rxq_count(struct ice_pf *pf) * ice_deinit_pf - Unrolls initialziations done by ice_init_pf * @pf: board private structure to initialize */ -static void ice_deinit_pf(struct ice_pf *pf) +void ice_deinit_pf(struct ice_pf *pf) { - ice_service_task_stop(pf); + /* note that we unroll also on ice_init_pf() failure here */ + mutex_destroy(&pf->lag_mutex); mutex_destroy(&pf->adev_mutex); mutex_destroy(&pf->sw_mutex); @@ -3977,6 +3980,9 @@ static void ice_deinit_pf(struct ice_pf *pf) if (pf->ptp.clock) ptp_clock_unregister(pf->ptp.clock); + if (!xa_empty(&pf->irq_tracker.entries)) + ice_free_irq_msix_misc(pf); + xa_destroy(&pf->dyn_ports); xa_destroy(&pf->sf_nums); } @@ -4030,13 +4036,25 @@ static void ice_set_pf_caps(struct ice_pf *pf) pf->max_pf_rxqs = func_caps->common_cap.num_rxq; } +void ice_start_service_task(struct ice_pf *pf) +{ + timer_setup(&pf->serv_tmr, ice_service_timer, 0); + pf->serv_tmr_period = HZ; + INIT_WORK(&pf->serv_task, ice_service_task); + clear_bit(ICE_SERVICE_SCHED, pf->state); +} + /** * ice_init_pf - Initialize general software structures (struct ice_pf) * @pf: board private structure to initialize + * Return: 0 on success, negative errno otherwise. */ -static int ice_init_pf(struct ice_pf *pf) +int ice_init_pf(struct ice_pf *pf) { - ice_set_pf_caps(pf); + struct udp_tunnel_nic_info *udp_tunnel_nic = &pf->hw.udp_tunnel_nic; + struct device *dev = ice_pf_to_dev(pf); + struct ice_hw *hw = &pf->hw; + int err = -ENOMEM; mutex_init(&pf->sw_mutex); mutex_init(&pf->tc_mutex); @@ -4049,32 +4067,7 @@ static int ice_init_pf(struct ice_pf *pf) init_waitqueue_head(&pf->reset_wait_queue); - /* setup service timer and periodic service task */ - timer_setup(&pf->serv_tmr, ice_service_timer, 0); - pf->serv_tmr_period = HZ; - INIT_WORK(&pf->serv_task, ice_service_task); - clear_bit(ICE_SERVICE_SCHED, pf->state); - mutex_init(&pf->avail_q_mutex); - pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); - if (!pf->avail_txqs) - return -ENOMEM; - - pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); - if (!pf->avail_rxqs) { - bitmap_free(pf->avail_txqs); - pf->avail_txqs = NULL; - return -ENOMEM; - } - - pf->txtime_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); - if (!pf->txtime_txqs) { - bitmap_free(pf->avail_txqs); - pf->avail_txqs = NULL; - bitmap_free(pf->avail_rxqs); - pf->avail_rxqs = NULL; - return -ENOMEM; - } mutex_init(&pf->vfs.table_lock); hash_init(pf->vfs.table); @@ -4087,7 +4080,36 @@ static int ice_init_pf(struct ice_pf *pf) xa_init(&pf->dyn_ports); xa_init(&pf->sf_nums); + pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); + pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); + pf->txtime_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); + if (!pf->avail_txqs || !pf->avail_rxqs || !pf->txtime_txqs) + goto undo_init; + + udp_tunnel_nic->set_port = ice_udp_tunnel_set_port; + udp_tunnel_nic->unset_port = ice_udp_tunnel_unset_port; + udp_tunnel_nic->shared = &hw->udp_tunnel_shared; + udp_tunnel_nic->tables[0].n_entries = hw->tnl.valid_count[TNL_VXLAN]; + udp_tunnel_nic->tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; + udp_tunnel_nic->tables[1].n_entries = hw->tnl.valid_count[TNL_GENEVE]; + udp_tunnel_nic->tables[1].tunnel_types = UDP_TUNNEL_TYPE_GENEVE; + + /* In case of MSIX we are going to setup the misc vector right here + * to handle admin queue events etc. In case of legacy and MSI + * the misc functionality and queue processing is combined in + * the same vector and that gets setup at open. + */ + err = ice_req_irq_msix_misc(pf); + if (err) { + dev_err(dev, "setup of misc vector failed: %d\n", err); + goto undo_init; + } + return 0; +undo_init: + /* deinit handles half-initialized pf just fine */ + ice_deinit_pf(pf); + return err; } /** @@ -4722,9 +4744,8 @@ static void ice_decfg_netdev(struct ice_vsi *vsi) vsi->netdev = NULL; } -int ice_init_dev(struct ice_pf *pf) +void ice_init_dev_hw(struct ice_pf *pf) { - struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; int err; @@ -4744,61 +4765,28 @@ int ice_init_dev(struct ice_pf *pf) */ ice_set_safe_mode_caps(hw); } +} - err = ice_init_pf(pf); - if (err) { - dev_err(dev, "ice_init_pf failed: %d\n", err); - return err; - } - - pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port; - pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port; - pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared; - if (pf->hw.tnl.valid_count[TNL_VXLAN]) { - pf->hw.udp_tunnel_nic.tables[0].n_entries = - pf->hw.tnl.valid_count[TNL_VXLAN]; - pf->hw.udp_tunnel_nic.tables[0].tunnel_types = - UDP_TUNNEL_TYPE_VXLAN; - } - if (pf->hw.tnl.valid_count[TNL_GENEVE]) { - pf->hw.udp_tunnel_nic.tables[1].n_entries = - pf->hw.tnl.valid_count[TNL_GENEVE]; - pf->hw.udp_tunnel_nic.tables[1].tunnel_types = - UDP_TUNNEL_TYPE_GENEVE; - } +int ice_init_dev(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + int err; + ice_set_pf_caps(pf); err = ice_init_interrupt_scheme(pf); if (err) { dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); - err = -EIO; - goto unroll_pf_init; + return -EIO; } - /* In case of MSIX we are going to setup the misc vector right here - * to handle admin queue events etc. In case of legacy and MSI - * the misc functionality and queue processing is combined in - * the same vector and that gets setup at open. - */ - err = ice_req_irq_msix_misc(pf); - if (err) { - dev_err(dev, "setup of misc vector failed: %d\n", err); - goto unroll_irq_scheme_init; - } + ice_start_service_task(pf); return 0; - -unroll_irq_scheme_init: - ice_clear_interrupt_scheme(pf); -unroll_pf_init: - ice_deinit_pf(pf); - return err; } void ice_deinit_dev(struct ice_pf *pf) { - ice_free_irq_msix_misc(pf); - ice_deinit_pf(pf); - ice_deinit_hw(&pf->hw); + ice_service_task_stop(pf); /* Service task is already stopped, so call reset directly. */ ice_reset(&pf->hw, ICE_RESET_PFR); @@ -4859,6 +4847,7 @@ static void ice_deinit_features(struct ice_pf *pf) ice_dpll_deinit(pf); if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) xa_destroy(&pf->eswitch.reprs); + ice_hwmon_exit(pf); } static void ice_init_wakeup(struct ice_pf *pf) @@ -5038,21 +5027,24 @@ static void ice_deinit_devlink(struct ice_pf *pf) static int ice_init(struct ice_pf *pf) { + struct device *dev = ice_pf_to_dev(pf); int err; - err = ice_init_dev(pf); - if (err) + err = ice_init_pf(pf); + if (err) { + dev_err(dev, "ice_init_pf failed: %d\n", err); return err; + } if (pf->hw.mac_type == ICE_MAC_E830) { err = pci_enable_ptm(pf->pdev, NULL); if (err) - dev_dbg(ice_pf_to_dev(pf), "PCIe PTM not supported by PCIe bus/controller\n"); + dev_dbg(dev, "PCIe PTM not supported by PCIe bus/controller\n"); } err = ice_alloc_vsis(pf); if (err) - goto err_alloc_vsis; + goto unroll_pf_init; err = ice_init_pf_sw(pf); if (err) @@ -5089,8 +5081,8 @@ static int ice_init(struct ice_pf *pf) ice_deinit_pf_sw(pf); err_init_pf_sw: ice_dealloc_vsis(pf); -err_alloc_vsis: - ice_deinit_dev(pf); +unroll_pf_init: + ice_deinit_pf(pf); return err; } @@ -5101,7 +5093,7 @@ static void ice_deinit(struct ice_pf *pf) ice_deinit_pf_sw(pf); ice_dealloc_vsis(pf); - ice_deinit_dev(pf); + ice_deinit_pf(pf); } /** @@ -5155,6 +5147,9 @@ int ice_load(struct ice_pf *pf) if (err) goto err_init_rdma; + /* Finalize RDMA: VSI already created, assign info and plug device */ + ice_rdma_finalize_setup(pf); + ice_service_task_restart(pf); clear_bit(ICE_DOWN, pf->state); @@ -5186,6 +5181,7 @@ void ice_unload(struct ice_pf *pf) devl_assert_locked(priv_to_devlink(pf)); + ice_unplug_aux_dev(pf); ice_deinit_rdma(pf); ice_deinit_features(pf); ice_tc_indir_block_unregister(vsi); @@ -5235,6 +5231,7 @@ static int ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) { struct device *dev = &pdev->dev; + bool need_dev_deinit = false; struct ice_adapter *adapter; struct ice_pf *pf; struct ice_hw *hw; @@ -5331,10 +5328,14 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) } pf->adapter = adapter; - err = ice_init(pf); + err = ice_init_dev(pf); if (err) goto unroll_adapter; + err = ice_init(pf); + if (err) + goto unroll_dev_init; + devl_lock(priv_to_devlink(pf)); err = ice_load(pf); if (err) @@ -5352,10 +5353,14 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) unroll_init: devl_unlock(priv_to_devlink(pf)); ice_deinit(pf); +unroll_dev_init: + need_dev_deinit = true; unroll_adapter: ice_adapter_put(pdev); unroll_hw_init: ice_deinit_hw(hw); + if (need_dev_deinit) + ice_deinit_dev(pf); return err; } @@ -5448,12 +5453,6 @@ static void ice_remove(struct pci_dev *pdev) ice_free_vfs(pf); } - ice_hwmon_exit(pf); - - ice_service_task_stop(pf); - ice_aq_cancel_waiting_tasks(pf); - set_bit(ICE_DOWN, pf->state); - if (!ice_is_safe_mode(pf)) ice_remove_arfs(pf); @@ -5471,6 +5470,11 @@ static void ice_remove(struct pci_dev *pdev) ice_set_wake(pf); ice_adapter_put(pdev); + ice_deinit_hw(&pf->hw); + + ice_deinit_dev(pf); + ice_aq_cancel_waiting_tasks(pf); + set_bit(ICE_DOWN, pf->state); } /** @@ -5604,6 +5608,7 @@ static int ice_suspend(struct device *dev) */ disabled = ice_service_task_stop(pf); + ice_unplug_aux_dev(pf); ice_deinit_rdma(pf); /* Already suspended?, then there is nothing to do */ @@ -5663,7 +5668,6 @@ static int ice_resume(struct device *dev) pci_set_power_state(pdev, PCI_D0); pci_restore_state(pdev); - pci_save_state(pdev); if (!pci_device_is_present(pdev)) return -ENODEV; @@ -5763,7 +5767,6 @@ static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_wake_from_d3(pdev, false); /* Check for life */ @@ -6995,7 +6998,6 @@ void ice_update_vsi_stats(struct ice_vsi *vsi) cur_ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes + pf->stats.rx_undersize + - pf->hw_csum_rx_error + pf->stats.rx_jabber + pf->stats.rx_fragments + pf->stats.rx_oversize; @@ -7812,12 +7814,15 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ice_health_clear(pf); - ice_plug_aux_dev(pf); + ice_rdma_finalize_setup(pf); if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) ice_lag_rebuild(pf); /* Restore timestamp mode settings after VSI rebuild */ ice_ptp_restore_timestamp_mode(pf); + + /* Start PTP periodic work after VSI is fully rebuilt */ + ice_ptp_queue_work(pf); return; err_vsi_rebuild: @@ -8003,6 +8008,34 @@ int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed) return status; } +/** + * ice_get_rss - Get RSS LUT and/or key + * @vsi: Pointer to VSI structure + * @seed: Buffer to store the key in + * @lut: Buffer to store the lookup table entries + * @lut_size: Size of buffer to store the lookup table entries + * + * Return: 0 on success, negative on failure + */ +int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) +{ + int err; + + if (seed) { + err = ice_get_rss_key(vsi, seed); + if (err) + return err; + } + + if (lut) { + err = ice_get_rss_lut(vsi, lut, lut_size); + if (err) + return err; + } + + return 0; +} + /** * ice_set_rss_hfunc - Set RSS HASH function * @vsi: Pointer to VSI structure @@ -9650,9 +9683,6 @@ int ice_open_internal(struct net_device *netdev) netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", vsi->vsi_num, vsi->vsw->sw_id); - /* Update existing tunnels information */ - udp_tunnel_get_rx_info(netdev); - return err; } diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index 8ec0f7d0fcebd..02517772fb5f4 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -569,6 +569,9 @@ static void ice_ptp_process_tx_tstamp(struct ice_ptp_tx *tx) pf = ptp_port_to_pf(ptp_port); hw = &pf->hw; + if (!tx->init) + return; + /* Read the Tx ready status first */ if (tx->has_ready_bitmap) { err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready); @@ -665,14 +668,9 @@ static void ice_ptp_process_tx_tstamp(struct ice_ptp_tx *tx) } } -/** - * ice_ptp_tx_tstamp_owner - Process Tx timestamps for all ports on the device - * @pf: Board private structure - */ -static enum ice_tx_tstamp_work ice_ptp_tx_tstamp_owner(struct ice_pf *pf) +static void ice_ptp_tx_tstamp_owner(struct ice_pf *pf) { struct ice_ptp_port *port; - unsigned int i; mutex_lock(&pf->adapter->ports.lock); list_for_each_entry(port, &pf->adapter->ports.ports, list_node) { @@ -684,49 +682,6 @@ static enum ice_tx_tstamp_work ice_ptp_tx_tstamp_owner(struct ice_pf *pf) ice_ptp_process_tx_tstamp(tx); } mutex_unlock(&pf->adapter->ports.lock); - - for (i = 0; i < ICE_GET_QUAD_NUM(pf->hw.ptp.num_lports); i++) { - u64 tstamp_ready; - int err; - - /* Read the Tx ready status first */ - err = ice_get_phy_tx_tstamp_ready(&pf->hw, i, &tstamp_ready); - if (err) - break; - else if (tstamp_ready) - return ICE_TX_TSTAMP_WORK_PENDING; - } - - return ICE_TX_TSTAMP_WORK_DONE; -} - -/** - * ice_ptp_tx_tstamp - Process Tx timestamps for this function. - * @tx: Tx tracking structure to initialize - * - * Returns: ICE_TX_TSTAMP_WORK_PENDING if there are any outstanding incomplete - * Tx timestamps, or ICE_TX_TSTAMP_WORK_DONE otherwise. - */ -static enum ice_tx_tstamp_work ice_ptp_tx_tstamp(struct ice_ptp_tx *tx) -{ - bool more_timestamps; - unsigned long flags; - - if (!tx->init) - return ICE_TX_TSTAMP_WORK_DONE; - - /* Process the Tx timestamp tracker */ - ice_ptp_process_tx_tstamp(tx); - - /* Check if there are outstanding Tx timestamps */ - spin_lock_irqsave(&tx->lock, flags); - more_timestamps = tx->init && !bitmap_empty(tx->in_use, tx->len); - spin_unlock_irqrestore(&tx->lock, flags); - - if (more_timestamps) - return ICE_TX_TSTAMP_WORK_PENDING; - - return ICE_TX_TSTAMP_WORK_DONE; } /** @@ -1338,9 +1293,12 @@ void ice_ptp_link_change(struct ice_pf *pf, bool linkup) /* Do not reconfigure E810 or E830 PHY */ return; case ICE_MAC_GENERIC: - case ICE_MAC_GENERIC_3K_E825: ice_ptp_port_phy_restart(ptp_port); return; + case ICE_MAC_GENERIC_3K_E825: + if (linkup) + ice_ptp_port_phy_restart(ptp_port); + return; default: dev_warn(ice_pf_to_dev(pf), "%s: Unknown PHY type\n", __func__); } @@ -2656,32 +2614,94 @@ s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) return idx + tx->offset; } -/** - * ice_ptp_process_ts - Process the PTP Tx timestamps - * @pf: Board private structure - * - * Returns: ICE_TX_TSTAMP_WORK_PENDING if there are any outstanding Tx - * timestamps that need processing, and ICE_TX_TSTAMP_WORK_DONE otherwise. - */ -enum ice_tx_tstamp_work ice_ptp_process_ts(struct ice_pf *pf) +void ice_ptp_process_ts(struct ice_pf *pf) { switch (pf->ptp.tx_interrupt_mode) { case ICE_PTP_TX_INTERRUPT_NONE: /* This device has the clock owner handle timestamps for it */ - return ICE_TX_TSTAMP_WORK_DONE; + return; case ICE_PTP_TX_INTERRUPT_SELF: /* This device handles its own timestamps */ - return ice_ptp_tx_tstamp(&pf->ptp.port.tx); + ice_ptp_process_tx_tstamp(&pf->ptp.port.tx); + return; case ICE_PTP_TX_INTERRUPT_ALL: /* This device handles timestamps for all ports */ - return ice_ptp_tx_tstamp_owner(pf); + ice_ptp_tx_tstamp_owner(pf); + return; default: WARN_ONCE(1, "Unexpected Tx timestamp interrupt mode %u\n", pf->ptp.tx_interrupt_mode); - return ICE_TX_TSTAMP_WORK_DONE; + return; } } +static bool ice_port_has_timestamps(struct ice_ptp_tx *tx) +{ + bool more_timestamps; + + scoped_guard(spinlock_irqsave, &tx->lock) { + if (!tx->init) + return false; + + more_timestamps = !bitmap_empty(tx->in_use, tx->len); + } + + return more_timestamps; +} + +static bool ice_any_port_has_timestamps(struct ice_pf *pf) +{ + struct ice_ptp_port *port; + + scoped_guard(mutex, &pf->adapter->ports.lock) { + list_for_each_entry(port, &pf->adapter->ports.ports, + list_node) { + struct ice_ptp_tx *tx = &port->tx; + + if (ice_port_has_timestamps(tx)) + return true; + } + } + + return false; +} + +bool ice_ptp_tx_tstamps_pending(struct ice_pf *pf) +{ + struct ice_hw *hw = &pf->hw; + unsigned int i; + + /* Check software indicator */ + switch (pf->ptp.tx_interrupt_mode) { + case ICE_PTP_TX_INTERRUPT_NONE: + return false; + case ICE_PTP_TX_INTERRUPT_SELF: + if (ice_port_has_timestamps(&pf->ptp.port.tx)) + return true; + break; + case ICE_PTP_TX_INTERRUPT_ALL: + if (ice_any_port_has_timestamps(pf)) + return true; + break; + default: + WARN_ONCE(1, "Unexpected Tx timestamp interrupt mode %u\n", + pf->ptp.tx_interrupt_mode); + break; + } + + /* Check hardware indicator */ + for (i = 0; i < ICE_GET_QUAD_NUM(hw->ptp.num_lports); i++) { + u64 tstamp_ready = 0; + int err; + + err = ice_get_phy_tx_tstamp_ready(&pf->hw, i, &tstamp_ready); + if (err || tstamp_ready) + return true; + } + + return false; +} + /** * ice_ptp_ts_irq - Process the PTP Tx timestamps in IRQ context * @pf: Board private structure @@ -2731,7 +2751,9 @@ irqreturn_t ice_ptp_ts_irq(struct ice_pf *pf) return IRQ_WAKE_THREAD; case ICE_MAC_E830: /* E830 can read timestamps in the top half using rd32() */ - if (ice_ptp_process_ts(pf) == ICE_TX_TSTAMP_WORK_PENDING) { + ice_ptp_process_ts(pf); + + if (ice_ptp_tx_tstamps_pending(pf)) { /* Process outstanding Tx timestamps. If there * is more work, re-arm the interrupt to trigger again. */ @@ -2810,6 +2832,20 @@ static void ice_ptp_periodic_work(struct kthread_work *work) msecs_to_jiffies(err ? 10 : 500)); } +/** + * ice_ptp_queue_work - Queue PTP periodic work for a PF + * @pf: Board private structure + * + * Helper function to queue PTP periodic work after VSI rebuild completes. + * This ensures that PTP work only runs when VSI structures are ready. + */ +void ice_ptp_queue_work(struct ice_pf *pf) +{ + if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags) && + pf->ptp.state == ICE_PTP_READY) + kthread_queue_delayed_work(pf->ptp.kworker, &pf->ptp.work, 0); +} + /** * ice_ptp_prepare_rebuild_sec - Prepare second NAC for PTP reset or rebuild * @pf: Board private structure @@ -2828,10 +2864,15 @@ static void ice_ptp_prepare_rebuild_sec(struct ice_pf *pf, bool rebuild, struct ice_pf *peer_pf = ptp_port_to_pf(port); if (!ice_is_primary(&peer_pf->hw)) { - if (rebuild) + if (rebuild) { + /* TODO: When implementing rebuild=true: + * 1. Ensure secondary PFs' VSIs are rebuilt + * 2. Call ice_ptp_queue_work(peer_pf) after VSI rebuild + */ ice_ptp_rebuild(peer_pf, reset_type); - else + } else { ice_ptp_prepare_for_reset(peer_pf, reset_type); + } } } } @@ -2977,9 +3018,6 @@ void ice_ptp_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ptp->state = ICE_PTP_READY; - /* Start periodic work going */ - kthread_queue_delayed_work(ptp->kworker, &ptp->work, 0); - dev_info(ice_pf_to_dev(pf), "PTP reset successful\n"); return; @@ -3003,7 +3041,13 @@ static int ice_ptp_setup_pf(struct ice_pf *pf) struct ice_ptp *ctrl_ptp = ice_get_ctrl_ptp(pf); struct ice_ptp *ptp = &pf->ptp; - if (WARN_ON(!ctrl_ptp) || pf->hw.mac_type == ICE_MAC_UNKNOWN) + if (!ctrl_ptp) { + dev_info(ice_pf_to_dev(pf), + "PTP unavailable: no controlling PF\n"); + return -EOPNOTSUPP; + } + + if (pf->hw.mac_type == ICE_MAC_UNKNOWN) return -ENODEV; INIT_LIST_HEAD(&ptp->port.list_node); @@ -3184,8 +3228,9 @@ static void ice_ptp_init_tx_interrupt_mode(struct ice_pf *pf) { switch (pf->hw.mac_type) { case ICE_MAC_GENERIC: - /* E822 based PHY has the clock owner process the interrupt - * for all ports. + case ICE_MAC_GENERIC_3K_E825: + /* E82x hardware has the clock owner process timestamps for + * all ports. */ if (ice_pf_src_tmr_owned(pf)) pf->ptp.tx_interrupt_mode = ICE_PTP_TX_INTERRUPT_ALL; diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h index 137f2070a2d99..4e02f922c1ff8 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp.h @@ -302,8 +302,9 @@ void ice_ptp_extts_event(struct ice_pf *pf); s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb); void ice_ptp_req_tx_single_tstamp(struct ice_ptp_tx *tx, u8 idx); void ice_ptp_complete_tx_single_tstamp(struct ice_ptp_tx *tx); -enum ice_tx_tstamp_work ice_ptp_process_ts(struct ice_pf *pf); +void ice_ptp_process_ts(struct ice_pf *pf); irqreturn_t ice_ptp_ts_irq(struct ice_pf *pf); +bool ice_ptp_tx_tstamps_pending(struct ice_pf *pf); u64 ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts); @@ -315,6 +316,7 @@ void ice_ptp_prepare_for_reset(struct ice_pf *pf, void ice_ptp_init(struct ice_pf *pf); void ice_ptp_release(struct ice_pf *pf); void ice_ptp_link_change(struct ice_pf *pf, bool linkup); +void ice_ptp_queue_work(struct ice_pf *pf); #else /* IS_ENABLED(CONFIG_PTP_1588_CLOCK) */ static inline int ice_ptp_hwtstamp_get(struct net_device *netdev, @@ -343,16 +345,18 @@ static inline void ice_ptp_req_tx_single_tstamp(struct ice_ptp_tx *tx, u8 idx) static inline void ice_ptp_complete_tx_single_tstamp(struct ice_ptp_tx *tx) { } -static inline bool ice_ptp_process_ts(struct ice_pf *pf) -{ - return true; -} +static inline void ice_ptp_process_ts(struct ice_pf *pf) { } static inline irqreturn_t ice_ptp_ts_irq(struct ice_pf *pf) { return IRQ_HANDLED; } +static inline bool ice_ptp_tx_tstamps_pending(struct ice_pf *pf) +{ + return false; +} + static inline u64 ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts) { @@ -381,6 +385,10 @@ static inline void ice_ptp_link_change(struct ice_pf *pf, bool linkup) { } +static inline void ice_ptp_queue_work(struct ice_pf *pf) +{ +} + static inline int ice_ptp_clock_index(struct ice_pf *pf) { return -1; diff --git a/drivers/net/ethernet/intel/ice/ice_repr.c b/drivers/net/ethernet/intel/ice/ice_repr.c index cb08746556a67..f1e82ba155cff 100644 --- a/drivers/net/ethernet/intel/ice/ice_repr.c +++ b/drivers/net/ethernet/intel/ice/ice_repr.c @@ -2,6 +2,7 @@ /* Copyright (C) 2019-2021, Intel Corporation. */ #include "ice.h" +#include "ice_lib.h" #include "ice_eswitch.h" #include "devlink/devlink.h" #include "devlink/port.h" @@ -67,7 +68,7 @@ ice_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) return; vsi = repr->src_vsi; - ice_update_vsi_stats(vsi); + ice_update_eth_stats(vsi); eth_stats = &vsi->eth_stats; stats->tx_packets = eth_stats->tx_unicast + eth_stats->tx_broadcast + @@ -315,7 +316,7 @@ ice_repr_reg_netdev(struct net_device *netdev, const struct net_device_ops *ops) static int ice_repr_ready_vf(struct ice_repr *repr) { - return !ice_check_vf_ready_for_cfg(repr->vf); + return ice_check_vf_ready_for_cfg(repr->vf); } static int ice_repr_ready_sf(struct ice_repr *repr) diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index ca4da0c899794..c74c47bc0b9b9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -131,14 +131,12 @@ enum idpf_cap_field { /** * enum idpf_vport_state - Current vport state - * @__IDPF_VPORT_DOWN: Vport is down - * @__IDPF_VPORT_UP: Vport is up - * @__IDPF_VPORT_STATE_LAST: Must be last, number of states + * @IDPF_VPORT_UP: Vport is up + * @IDPF_VPORT_STATE_NBITS: Must be last, number of states */ enum idpf_vport_state { - __IDPF_VPORT_DOWN, - __IDPF_VPORT_UP, - __IDPF_VPORT_STATE_LAST, + IDPF_VPORT_UP, + IDPF_VPORT_STATE_NBITS }; /** @@ -162,7 +160,7 @@ struct idpf_netdev_priv { u16 vport_idx; u16 max_tx_hdr_size; u16 tx_max_bufs; - enum idpf_vport_state state; + DECLARE_BITMAP(state, IDPF_VPORT_STATE_NBITS); struct rtnl_link_stats64 netstats; spinlock_t stats_lock; }; @@ -286,8 +284,7 @@ struct idpf_port_stats { struct idpf_fsteer_fltr { struct list_head list; - u32 loc; - u32 q_index; + struct ethtool_rx_flow_spec fs; }; /** @@ -426,14 +423,12 @@ enum idpf_user_flags { * @rss_key: RSS hash key * @rss_lut_size: Size of RSS lookup table * @rss_lut: RSS lookup table - * @cached_lut: Used to restore previously init RSS lut */ struct idpf_rss_data { u16 rss_key_size; u8 *rss_key; u16 rss_lut_size; u32 *rss_lut; - u32 *cached_lut; }; /** @@ -560,6 +555,7 @@ struct idpf_vector_lifo { * @max_q: Maximum possible queues * @req_qs_chunks: Queue chunk data for requested queues * @mac_filter_list_lock: Lock to protect mac filters + * @flow_steer_list_lock: Lock to protect fsteer filters * @flags: See enum idpf_vport_config_flags */ struct idpf_vport_config { @@ -567,6 +563,7 @@ struct idpf_vport_config { struct idpf_vport_max_q max_q; struct virtchnl2_add_queues *req_qs_chunks; spinlock_t mac_filter_list_lock; + spinlock_t flow_steer_list_lock; DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS); }; diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index a5a1eec9ade8b..6c0a9296eccc8 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -18,6 +18,7 @@ static int idpf_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, { struct idpf_netdev_priv *np = netdev_priv(netdev); struct idpf_vport_user_config_data *user_config; + struct idpf_vport_config *vport_config; struct idpf_fsteer_fltr *f; struct idpf_vport *vport; unsigned int cnt = 0; @@ -25,7 +26,8 @@ static int idpf_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); - user_config = &np->adapter->vport_config[np->vport_idx]->user_config; + vport_config = np->adapter->vport_config[np->vport_idx]; + user_config = &vport_config->user_config; switch (cmd->cmd) { case ETHTOOL_GRXRINGS: @@ -36,26 +38,34 @@ static int idpf_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, cmd->data = idpf_fsteer_max_rules(vport); break; case ETHTOOL_GRXCLSRULE: - err = -EINVAL; + err = -ENOENT; + spin_lock_bh(&vport_config->flow_steer_list_lock); list_for_each_entry(f, &user_config->flow_steer_list, list) - if (f->loc == cmd->fs.location) { - cmd->fs.ring_cookie = f->q_index; + if (f->fs.location == cmd->fs.location) { + /* Avoid infoleak from padding: zero first, + * then assign fields + */ + memset(&cmd->fs, 0, sizeof(cmd->fs)); + cmd->fs = f->fs; err = 0; break; } + spin_unlock_bh(&vport_config->flow_steer_list_lock); break; case ETHTOOL_GRXCLSRLALL: cmd->data = idpf_fsteer_max_rules(vport); + spin_lock_bh(&vport_config->flow_steer_list_lock); list_for_each_entry(f, &user_config->flow_steer_list, list) { if (cnt == cmd->rule_cnt) { err = -EMSGSIZE; break; } - rule_locs[cnt] = f->loc; + rule_locs[cnt] = f->fs.location; cnt++; } if (!err) cmd->rule_cnt = user_config->num_fsteer_fltrs; + spin_unlock_bh(&vport_config->flow_steer_list_lock); break; default: break; @@ -152,7 +162,7 @@ static int idpf_add_flow_steer(struct net_device *netdev, struct idpf_vport *vport; u32 flow_type, q_index; u16 num_rxq; - int err; + int err = 0; vport = idpf_netdev_to_vport(netdev); vport_config = vport->adapter->vport_config[np->vport_idx]; @@ -178,6 +188,29 @@ static int idpf_add_flow_steer(struct net_device *netdev, if (!rule) return -ENOMEM; + fltr = kzalloc(sizeof(*fltr), GFP_KERNEL); + if (!fltr) { + err = -ENOMEM; + goto out_free_rule; + } + + /* detect duplicate entry and reject before adding rules */ + spin_lock_bh(&vport_config->flow_steer_list_lock); + list_for_each_entry(f, &user_config->flow_steer_list, list) { + if (f->fs.location == fsp->location) { + err = -EEXIST; + break; + } + + if (f->fs.location > fsp->location) + break; + parent = f; + } + spin_unlock_bh(&vport_config->flow_steer_list_lock); + + if (err) + goto out; + rule->vport_id = cpu_to_le32(vport->vport_id); rule->count = cpu_to_le32(1); info = &rule->rule_info[0]; @@ -216,26 +249,20 @@ static int idpf_add_flow_steer(struct net_device *netdev, goto out; } - fltr = kzalloc(sizeof(*fltr), GFP_KERNEL); - if (!fltr) { - err = -ENOMEM; - goto out; - } - - fltr->loc = fsp->location; - fltr->q_index = q_index; - list_for_each_entry(f, &user_config->flow_steer_list, list) { - if (f->loc >= fltr->loc) - break; - parent = f; - } + /* Save a copy of the user's flow spec so ethtool can later retrieve it */ + fltr->fs = *fsp; + spin_lock_bh(&vport_config->flow_steer_list_lock); parent ? list_add(&fltr->list, &parent->list) : list_add(&fltr->list, &user_config->flow_steer_list); user_config->num_fsteer_fltrs++; + spin_unlock_bh(&vport_config->flow_steer_list_lock); + goto out_free_rule; out: + kfree(fltr); +out_free_rule: kfree(rule); return err; } @@ -264,9 +291,6 @@ static int idpf_del_flow_steer(struct net_device *netdev, vport_config = vport->adapter->vport_config[np->vport_idx]; user_config = &vport_config->user_config; - if (!idpf_sideband_action_ena(vport, fsp)) - return -EOPNOTSUPP; - rule = kzalloc(struct_size(rule, rule_info, 1), GFP_KERNEL); if (!rule) return -ENOMEM; @@ -286,17 +310,20 @@ static int idpf_del_flow_steer(struct net_device *netdev, goto out; } + spin_lock_bh(&vport_config->flow_steer_list_lock); list_for_each_entry_safe(f, iter, &user_config->flow_steer_list, list) { - if (f->loc == fsp->location) { + if (f->fs.location == fsp->location) { list_del(&f->list); kfree(f); user_config->num_fsteer_fltrs--; - goto out; + goto out_unlock; } } - err = -EINVAL; + err = -ENOENT; +out_unlock: + spin_unlock_bh(&vport_config->flow_steer_list_lock); out: kfree(rule); return err; @@ -365,7 +392,10 @@ static u32 idpf_get_rxfh_indir_size(struct net_device *netdev) * @netdev: network interface device structure * @rxfh: pointer to param struct (indir, key, hfunc) * - * Reads the indirection table directly from the hardware. Always returns 0. + * RSS LUT and Key information are read from driver's cached + * copy. When rxhash is off, rss lut will be displayed as zeros. + * + * Return: 0 on success, -errno otherwise. */ static int idpf_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) @@ -373,10 +403,13 @@ static int idpf_get_rxfh(struct net_device *netdev, struct idpf_netdev_priv *np = netdev_priv(netdev); struct idpf_rss_data *rss_data; struct idpf_adapter *adapter; + struct idpf_vport *vport; + bool rxhash_ena; int err = 0; u16 i; idpf_vport_ctrl_lock(netdev); + vport = idpf_netdev_to_vport(netdev); adapter = np->adapter; @@ -386,9 +419,8 @@ static int idpf_get_rxfh(struct net_device *netdev, } rss_data = &adapter->vport_config[np->vport_idx]->user_config.rss_data; - if (np->state != __IDPF_VPORT_UP) - goto unlock_mutex; + rxhash_ena = idpf_is_feature_ena(vport, NETIF_F_RXHASH); rxfh->hfunc = ETH_RSS_HASH_TOP; if (rxfh->key) @@ -396,7 +428,7 @@ static int idpf_get_rxfh(struct net_device *netdev, if (rxfh->indir) { for (i = 0; i < rss_data->rss_lut_size; i++) - rxfh->indir[i] = rss_data->rss_lut[i]; + rxfh->indir[i] = rxhash_ena ? rss_data->rss_lut[i] : 0; } unlock_mutex: @@ -436,8 +468,6 @@ static int idpf_set_rxfh(struct net_device *netdev, } rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data; - if (np->state != __IDPF_VPORT_UP) - goto unlock_mutex; if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && rxfh->hfunc != ETH_RSS_HASH_TOP) { @@ -453,7 +483,8 @@ static int idpf_set_rxfh(struct net_device *netdev, rss_data->rss_lut[lut] = rxfh->indir[lut]; } - err = idpf_config_rss(vport); + if (test_bit(IDPF_VPORT_UP, np->state)) + err = idpf_config_rss(vport); unlock_mutex: idpf_vport_ctrl_unlock(netdev); @@ -1167,7 +1198,7 @@ static void idpf_get_ethtool_stats(struct net_device *netdev, idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); - if (np->state != __IDPF_VPORT_UP) { + if (!test_bit(IDPF_VPORT_UP, np->state)) { idpf_vport_ctrl_unlock(netdev); return; @@ -1319,7 +1350,7 @@ static int idpf_get_q_coalesce(struct net_device *netdev, idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); - if (np->state != __IDPF_VPORT_UP) + if (!test_bit(IDPF_VPORT_UP, np->state)) goto unlock_mutex; if (q_num >= vport->num_rxq && q_num >= vport->num_txq) { @@ -1507,7 +1538,7 @@ static int idpf_set_coalesce(struct net_device *netdev, idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); - if (np->state != __IDPF_VPORT_UP) + if (!test_bit(IDPF_VPORT_UP, np->state)) goto unlock_mutex; for (i = 0; i < vport->num_txq; i++) { @@ -1710,7 +1741,7 @@ static void idpf_get_ts_stats(struct net_device *netdev, ts_stats->err = u64_stats_read(&vport->tstamp_stats.discarded); } while (u64_stats_fetch_retry(&vport->tstamp_stats.stats_sync, start)); - if (np->state != __IDPF_VPORT_UP) + if (!test_bit(IDPF_VPORT_UP, np->state)) goto exit; for (u16 i = 0; i < vport->num_txq_grp; i++) { diff --git a/drivers/net/ethernet/intel/idpf/idpf_idc.c b/drivers/net/ethernet/intel/idpf/idpf_idc.c index 7e20a07e98e53..6dad0593f7f22 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_idc.c +++ b/drivers/net/ethernet/intel/idpf/idpf_idc.c @@ -322,7 +322,7 @@ static void idpf_idc_vport_dev_down(struct idpf_adapter *adapter) for (i = 0; i < adapter->num_alloc_vports; i++) { struct idpf_vport *vport = adapter->vports[i]; - if (!vport) + if (!vport || !vport->vdev_info) continue; idpf_unplug_aux_dev(vport->vdev_info->adev); diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 8a941f0fb048b..131a8121839bd 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -442,6 +442,29 @@ int idpf_intr_req(struct idpf_adapter *adapter) return err; } +/** + * idpf_del_all_flow_steer_filters - Delete all flow steer filters in list + * @vport: main vport struct + * + * Takes flow_steer_list_lock spinlock. Deletes all filters + */ +static void idpf_del_all_flow_steer_filters(struct idpf_vport *vport) +{ + struct idpf_vport_config *vport_config; + struct idpf_fsteer_fltr *f, *ftmp; + + vport_config = vport->adapter->vport_config[vport->idx]; + + spin_lock_bh(&vport_config->flow_steer_list_lock); + list_for_each_entry_safe(f, ftmp, &vport_config->user_config.flow_steer_list, + list) { + list_del(&f->list); + kfree(f); + } + vport_config->user_config.num_fsteer_fltrs = 0; + spin_unlock_bh(&vport_config->flow_steer_list_lock); +} + /** * idpf_find_mac_filter - Search filter list for specific mac filter * @vconfig: Vport config structure @@ -519,7 +542,7 @@ static int idpf_del_mac_filter(struct idpf_vport *vport, } spin_unlock_bh(&vport_config->mac_filter_list_lock); - if (np->state == __IDPF_VPORT_UP) { + if (test_bit(IDPF_VPORT_UP, np->state)) { int err; err = idpf_add_del_mac_filters(vport, np, false, async); @@ -590,7 +613,7 @@ static int idpf_add_mac_filter(struct idpf_vport *vport, if (err) return err; - if (np->state == __IDPF_VPORT_UP) + if (test_bit(IDPF_VPORT_UP, np->state)) err = idpf_add_del_mac_filters(vport, np, true, async); return err; @@ -729,6 +752,65 @@ static int idpf_init_mac_addr(struct idpf_vport *vport, return 0; } +static void idpf_detach_and_close(struct idpf_adapter *adapter) +{ + int max_vports = adapter->max_vports; + + for (int i = 0; i < max_vports; i++) { + struct net_device *netdev = adapter->netdevs[i]; + + /* If the interface is in detached state, that means the + * previous reset was not handled successfully for this + * vport. + */ + if (!netif_device_present(netdev)) + continue; + + /* Hold RTNL to protect racing with callbacks */ + rtnl_lock(); + netif_device_detach(netdev); + if (netif_running(netdev)) { + set_bit(IDPF_VPORT_UP_REQUESTED, + adapter->vport_config[i]->flags); + dev_close(netdev); + } + rtnl_unlock(); + } +} + +static void idpf_attach_and_open(struct idpf_adapter *adapter) +{ + int max_vports = adapter->max_vports; + + for (int i = 0; i < max_vports; i++) { + struct idpf_vport *vport = adapter->vports[i]; + struct idpf_vport_config *vport_config; + struct net_device *netdev; + + /* In case of a critical error in the init task, the vport + * will be freed. Only continue to restore the netdevs + * if the vport is allocated. + */ + if (!vport) + continue; + + /* No need for RTNL on attach as this function is called + * following detach and dev_close(). We do take RTNL for + * dev_open() below as it can race with external callbacks + * following the call to netif_device_attach(). + */ + netdev = adapter->netdevs[i]; + netif_device_attach(netdev); + vport_config = adapter->vport_config[vport->idx]; + if (test_and_clear_bit(IDPF_VPORT_UP_REQUESTED, + vport_config->flags)) { + rtnl_lock(); + dev_open(netdev, NULL); + rtnl_unlock(); + } + } +} + /** * idpf_cfg_netdev - Allocate, configure and register a netdev * @vport: main vport structure @@ -894,7 +976,7 @@ static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); - if (np->state <= __IDPF_VPORT_DOWN) + if (!test_bit(IDPF_VPORT_UP, np->state)) return; if (rtnl) @@ -921,7 +1003,7 @@ static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl) idpf_xdp_rxq_info_deinit_all(vport); idpf_vport_queues_rel(vport); idpf_vport_intr_rel(vport); - np->state = __IDPF_VPORT_DOWN; + clear_bit(IDPF_VPORT_UP, np->state); if (rtnl) rtnl_unlock(); @@ -991,7 +1073,7 @@ static void idpf_vport_rel(struct idpf_vport *vport) u16 idx = vport->idx; vport_config = adapter->vport_config[vport->idx]; - idpf_deinit_rss(vport); + idpf_deinit_rss_lut(vport); rss_data = &vport_config->user_config.rss_data; kfree(rss_data->rss_key); rss_data->rss_key = NULL; @@ -1023,6 +1105,8 @@ static void idpf_vport_rel(struct idpf_vport *vport) kfree(adapter->vport_config[idx]->req_qs_chunks); adapter->vport_config[idx]->req_qs_chunks = NULL; } + kfree(vport->rx_ptype_lkup); + vport->rx_ptype_lkup = NULL; kfree(vport); adapter->num_alloc_vports--; } @@ -1041,12 +1125,15 @@ static void idpf_vport_dealloc(struct idpf_vport *vport) idpf_idc_deinit_vport_aux_device(vport->vdev_info); idpf_deinit_mac_addr(vport); - idpf_vport_stop(vport, true); - if (!test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags)) + if (!test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags)) { + idpf_vport_stop(vport, true); idpf_decfg_netdev(vport); - if (test_bit(IDPF_REMOVE_IN_PROG, adapter->flags)) + } + if (test_bit(IDPF_REMOVE_IN_PROG, adapter->flags)) { idpf_del_all_mac_filters(vport); + idpf_del_all_flow_steer_filters(vport); + } if (adapter->netdevs[i]) { struct idpf_netdev_priv *np = netdev_priv(adapter->netdevs[i]); @@ -1139,6 +1226,7 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, u16 idx = adapter->next_vport; struct idpf_vport *vport; u16 num_max_q; + int err; if (idx == IDPF_NO_FREE_SLOT) return NULL; @@ -1189,10 +1277,11 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, idpf_vport_init(vport, max_q); - /* This alloc is done separate from the LUT because it's not strictly - * dependent on how many queues we have. If we change number of queues - * and soft reset we'll need a new LUT but the key can remain the same - * for as long as the vport exists. + /* LUT and key are both initialized here. Key is not strictly dependent + * on how many queues we have. If we change number of queues and soft + * reset is initiated, LUT will be freed and a new LUT will be allocated + * as per the updated number of queues during vport bringup. However, + * the key remains the same for as long as the vport exists. */ rss_data = &adapter->vport_config[idx]->user_config.rss_data; rss_data->rss_key = kzalloc(rss_data->rss_key_size, GFP_KERNEL); @@ -1202,6 +1291,11 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, /* Initialize default rss key */ netdev_rss_key_fill((void *)rss_data->rss_key, rss_data->rss_key_size); + /* Initialize default rss LUT */ + err = idpf_init_rss_lut(vport); + if (err) + goto free_rss_key; + /* fill vport slot in the adapter struct */ adapter->vports[idx] = vport; adapter->vport_ids[idx] = idpf_get_vport_id(vport); @@ -1212,6 +1306,8 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, return vport; +free_rss_key: + kfree(rss_data->rss_key); free_vector_idxs: kfree(vport->q_vector_idxs); free_vport: @@ -1271,7 +1367,7 @@ void idpf_mbx_task(struct work_struct *work) idpf_mb_irq_enable(adapter); else queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, - msecs_to_jiffies(300)); + usecs_to_jiffies(300)); idpf_recv_mb_msg(adapter); } @@ -1345,7 +1441,7 @@ static int idpf_up_complete(struct idpf_vport *vport) netif_tx_start_all_queues(vport->netdev); } - np->state = __IDPF_VPORT_UP; + set_bit(IDPF_VPORT_UP, np->state); return 0; } @@ -1388,10 +1484,9 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); struct idpf_adapter *adapter = vport->adapter; - struct idpf_vport_config *vport_config; int err; - if (np->state != __IDPF_VPORT_DOWN) + if (test_bit(IDPF_VPORT_UP, np->state)) return -EBUSY; if (rtnl) @@ -1429,14 +1524,14 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", vport->vport_id, err); - goto queues_rel; + goto intr_deinit; } err = idpf_rx_bufs_init_all(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", vport->vport_id, err); - goto queues_rel; + goto intr_deinit; } idpf_rx_init_buf_tail(vport); @@ -1482,13 +1577,9 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) idpf_restore_features(vport); - vport_config = adapter->vport_config[vport->idx]; - if (vport_config->user_config.rss_data.rss_lut) - err = idpf_config_rss(vport); - else - err = idpf_init_rss(vport); + err = idpf_config_rss(vport); if (err) { - dev_err(&adapter->pdev->dev, "Failed to initialize RSS for vport %u: %d\n", + dev_err(&adapter->pdev->dev, "Failed to configure RSS for vport %u: %d\n", vport->vport_id, err); goto disable_vport; } @@ -1497,7 +1588,7 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) if (err) { dev_err(&adapter->pdev->dev, "Failed to complete interface up for vport %u: %d\n", vport->vport_id, err); - goto deinit_rss; + goto disable_vport; } if (rtnl) @@ -1505,8 +1596,6 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) return 0; -deinit_rss: - idpf_deinit_rss(vport); disable_vport: idpf_send_disable_vport_msg(vport); disable_queues: @@ -1544,7 +1633,6 @@ void idpf_init_task(struct work_struct *work) struct idpf_vport_config *vport_config; struct idpf_vport_max_q max_q; struct idpf_adapter *adapter; - struct idpf_netdev_priv *np; struct idpf_vport *vport; u16 num_default_vports; struct pci_dev *pdev; @@ -1579,10 +1667,15 @@ void idpf_init_task(struct work_struct *work) goto unwind_vports; } + err = idpf_send_get_rx_ptype_msg(vport); + if (err) + goto unwind_vports; + index = vport->idx; vport_config = adapter->vport_config[index]; spin_lock_init(&vport_config->mac_filter_list_lock); + spin_lock_init(&vport_config->flow_steer_list_lock); INIT_LIST_HEAD(&vport_config->user_config.mac_filter_list); INIT_LIST_HEAD(&vport_config->user_config.flow_steer_list); @@ -1590,21 +1683,11 @@ void idpf_init_task(struct work_struct *work) err = idpf_check_supported_desc_ids(vport); if (err) { dev_err(&pdev->dev, "failed to get required descriptor ids\n"); - goto cfg_netdev_err; + goto unwind_vports; } if (idpf_cfg_netdev(vport)) - goto cfg_netdev_err; - - err = idpf_send_get_rx_ptype_msg(vport); - if (err) - goto handle_err; - - /* Once state is put into DOWN, driver is ready for dev_open */ - np = netdev_priv(vport->netdev); - np->state = __IDPF_VPORT_DOWN; - if (test_and_clear_bit(IDPF_VPORT_UP_REQUESTED, vport_config->flags)) - idpf_vport_open(vport, true); + goto unwind_vports; /* Spawn and return 'idpf_init_task' work queue until all the * default vports are created @@ -1635,21 +1718,15 @@ void idpf_init_task(struct work_struct *work) set_bit(IDPF_VPORT_REG_NETDEV, vport_config->flags); } - /* As all the required vports are created, clear the reset flag - * unconditionally here in case we were in reset and the link was down. - */ + /* Clear the reset and load bits as all vports are created */ clear_bit(IDPF_HR_RESET_IN_PROG, adapter->flags); + clear_bit(IDPF_HR_DRV_LOAD, adapter->flags); /* Start the statistics task now */ queue_delayed_work(adapter->stats_wq, &adapter->stats_task, msecs_to_jiffies(10 * (pdev->devfn & 0x07))); return; -handle_err: - idpf_decfg_netdev(vport); -cfg_netdev_err: - idpf_vport_rel(vport); - adapter->vports[index] = NULL; unwind_vports: if (default_vport) { for (index = 0; index < adapter->max_vports; index++) { @@ -1657,6 +1734,15 @@ void idpf_init_task(struct work_struct *work) idpf_vport_dealloc(adapter->vports[index]); } } + /* Cleanup after vc_core_init, which has no way of knowing the + * init task failed on driver load. + */ + if (test_and_clear_bit(IDPF_HR_DRV_LOAD, adapter->flags)) { + cancel_delayed_work_sync(&adapter->serv_task); + cancel_delayed_work_sync(&adapter->mbx_task); + } + idpf_ptp_release(adapter); + clear_bit(IDPF_HR_RESET_IN_PROG, adapter->flags); } @@ -1786,27 +1872,6 @@ static int idpf_check_reset_complete(struct idpf_hw *hw, return -EBUSY; } -/** - * idpf_set_vport_state - Set the vport state to be after the reset - * @adapter: Driver specific private structure - */ -static void idpf_set_vport_state(struct idpf_adapter *adapter) -{ - u16 i; - - for (i = 0; i < adapter->max_vports; i++) { - struct idpf_netdev_priv *np; - - if (!adapter->netdevs[i]) - continue; - - np = netdev_priv(adapter->netdevs[i]); - if (np->state == __IDPF_VPORT_UP) - set_bit(IDPF_VPORT_UP_REQUESTED, - adapter->vport_config[i]->flags); - } -} - /** * idpf_init_hard_reset - Initiate a hardware reset * @adapter: Driver specific private structure @@ -1815,37 +1880,25 @@ static void idpf_set_vport_state(struct idpf_adapter *adapter) * reallocate. Also reinitialize the mailbox. Return 0 on success, * negative on failure. */ -static int idpf_init_hard_reset(struct idpf_adapter *adapter) +static void idpf_init_hard_reset(struct idpf_adapter *adapter) { struct idpf_reg_ops *reg_ops = &adapter->dev_ops.reg_ops; struct device *dev = &adapter->pdev->dev; - struct net_device *netdev; int err; - u16 i; + idpf_detach_and_close(adapter); mutex_lock(&adapter->vport_ctrl_lock); dev_info(dev, "Device HW Reset initiated\n"); - /* Avoid TX hangs on reset */ - for (i = 0; i < adapter->max_vports; i++) { - netdev = adapter->netdevs[i]; - if (!netdev) - continue; - - netif_carrier_off(netdev); - netif_tx_disable(netdev); - } - /* Prepare for reset */ - if (test_and_clear_bit(IDPF_HR_DRV_LOAD, adapter->flags)) { + if (test_bit(IDPF_HR_DRV_LOAD, adapter->flags)) { reg_ops->trigger_reset(adapter, IDPF_HR_DRV_LOAD); } else if (test_and_clear_bit(IDPF_HR_FUNC_RESET, adapter->flags)) { bool is_reset = idpf_is_reset_detected(adapter); idpf_idc_issue_reset_event(adapter->cdev_info); - idpf_set_vport_state(adapter); idpf_vc_core_deinit(adapter); if (!is_reset) reg_ops->trigger_reset(adapter, IDPF_HR_FUNC_RESET); @@ -1892,11 +1945,14 @@ static int idpf_init_hard_reset(struct idpf_adapter *adapter) unlock_mutex: mutex_unlock(&adapter->vport_ctrl_lock); - /* Wait until all vports are created to init RDMA CORE AUX */ - if (!err) - err = idpf_idc_init(adapter); - - return err; + /* Attempt to restore netdevs and initialize RDMA CORE AUX device, + * provided vc_core_init succeeded. It is still possible that + * vports are not allocated at this point if the init task failed. + */ + if (!err) { + idpf_attach_and_open(adapter); + idpf_idc_init(adapter); + } } /** @@ -1939,7 +1995,7 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, enum idpf_vport_reset_cause reset_cause) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); - enum idpf_vport_state current_state = np->state; + bool vport_is_up = test_bit(IDPF_VPORT_UP, np->state); struct idpf_adapter *adapter = vport->adapter; struct idpf_vport *new_vport; int err; @@ -1990,14 +2046,13 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, goto free_vport; } - if (current_state <= __IDPF_VPORT_DOWN) { + if (!vport_is_up) { idpf_send_delete_queues_msg(vport); } else { set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags); idpf_vport_stop(vport, false); } - idpf_deinit_rss(vport); /* We're passing in vport here because we need its wait_queue * to send a message and it should be getting all the vport * config data out of the adapter but we need to be careful not @@ -2023,7 +2078,11 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, if (err) goto err_open; - if (current_state == __IDPF_VPORT_UP) + if (reset_cause == IDPF_SR_Q_CHANGE && + !netif_is_rxfh_configured(vport->netdev)) + idpf_fill_dflt_rss_lut(vport); + + if (vport_is_up) err = idpf_vport_open(vport, false); goto free_vport; @@ -2033,7 +2092,7 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, vport->num_rxq, vport->num_bufq); err_open: - if (current_state == __IDPF_VPORT_UP) + if (vport_is_up) idpf_vport_open(vport, false); free_vport: @@ -2165,40 +2224,6 @@ static void idpf_set_rx_mode(struct net_device *netdev) dev_err(dev, "Failed to set promiscuous mode: %d\n", err); } -/** - * idpf_vport_manage_rss_lut - disable/enable RSS - * @vport: the vport being changed - * - * In the event of disable request for RSS, this function will zero out RSS - * LUT, while in the event of enable request for RSS, it will reconfigure RSS - * LUT with the default LUT configuration. - */ -static int idpf_vport_manage_rss_lut(struct idpf_vport *vport) -{ - bool ena = idpf_is_feature_ena(vport, NETIF_F_RXHASH); - struct idpf_rss_data *rss_data; - u16 idx = vport->idx; - int lut_size; - - rss_data = &vport->adapter->vport_config[idx]->user_config.rss_data; - lut_size = rss_data->rss_lut_size * sizeof(u32); - - if (ena) { - /* This will contain the default or user configured LUT */ - memcpy(rss_data->rss_lut, rss_data->cached_lut, lut_size); - } else { - /* Save a copy of the current LUT to be restored later if - * requested. - */ - memcpy(rss_data->cached_lut, rss_data->rss_lut, lut_size); - - /* Zero out the current LUT to disable */ - memset(rss_data->rss_lut, 0, lut_size); - } - - return idpf_config_rss(vport); -} - /** * idpf_set_features - set the netdev feature flags * @netdev: ptr to the netdev being adjusted @@ -2224,10 +2249,19 @@ static int idpf_set_features(struct net_device *netdev, } if (changed & NETIF_F_RXHASH) { + struct idpf_netdev_priv *np = netdev_priv(netdev); + netdev->features ^= NETIF_F_RXHASH; - err = idpf_vport_manage_rss_lut(vport); - if (err) - goto unlock_mutex; + + /* If the interface is not up when changing the rxhash, update + * to the HW is skipped. The updated LUT will be committed to + * the HW when the interface is brought up. + */ + if (test_bit(IDPF_VPORT_UP, np->state)) { + err = idpf_config_rss(vport); + if (err) + goto unlock_mutex; + } } if (changed & NETIF_F_GRO_HW) { diff --git a/drivers/net/ethernet/intel/idpf/idpf_ptp.c b/drivers/net/ethernet/intel/idpf/idpf_ptp.c index 3e1052d070cfd..0a8b50350b860 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ptp.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ptp.c @@ -108,11 +108,11 @@ static u64 idpf_ptp_read_src_clk_reg_direct(struct idpf_adapter *adapter, ptp_read_system_prets(sts); idpf_ptp_enable_shtime(adapter); + lo = readl(ptp->dev_clk_regs.dev_clk_ns_l); /* Read the system timestamp post PHC read */ ptp_read_system_postts(sts); - lo = readl(ptp->dev_clk_regs.dev_clk_ns_l); hi = readl(ptp->dev_clk_regs.dev_clk_ns_h); spin_unlock(&ptp->read_dev_clk_lock); diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index 61e6130661404..e3ddf18dcbf51 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -570,7 +570,7 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, np = netdev_priv(tx_q->netdev); nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - dont_wake = np->state != __IDPF_VPORT_UP || + dont_wake = !test_bit(IDPF_VPORT_UP, np->state) || !netif_carrier_ok(tx_q->netdev); __netif_txq_completed_wake(nq, ss.packets, ss.bytes, IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH, diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 828f7c444d309..c859665b2dc89 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -695,9 +695,10 @@ static int idpf_rx_buf_alloc_singleq(struct idpf_rx_queue *rxq) static int idpf_rx_bufs_init_singleq(struct idpf_rx_queue *rxq) { struct libeth_fq fq = { - .count = rxq->desc_count, - .type = LIBETH_FQE_MTU, - .nid = idpf_q_vector_to_mem(rxq->q_vector), + .count = rxq->desc_count, + .type = LIBETH_FQE_MTU, + .buf_len = IDPF_RX_MAX_BUF_SZ, + .nid = idpf_q_vector_to_mem(rxq->q_vector), }; int ret; @@ -754,6 +755,7 @@ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, .truesize = bufq->truesize, .count = bufq->desc_count, .type = type, + .buf_len = IDPF_RX_MAX_BUF_SZ, .hsplit = idpf_queue_has(HSPLIT_EN, bufq), .xdp = idpf_xdp_enabled(bufq->q_vector->vport), .nid = idpf_q_vector_to_mem(bufq->q_vector), @@ -2275,7 +2277,7 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, /* Update BQL */ nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - dont_wake = !complq_ok || np->state != __IDPF_VPORT_UP || + dont_wake = !complq_ok || !test_bit(IDPF_VPORT_UP, np->state) || !netif_carrier_ok(tx_q->netdev); /* Check if the TXQ needs to and can be restarted */ __netif_txq_completed_wake(nq, tx_q->cleaned_pkts, tx_q->cleaned_bytes, @@ -2324,7 +2326,7 @@ void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) do { struct idpf_splitq_4b_tx_compl_desc *tx_desc; - struct idpf_tx_queue *target; + struct idpf_tx_queue *target = NULL; u32 ctype_gen, id; tx_desc = flow ? &complq->comp[ntc].common : @@ -2344,14 +2346,14 @@ void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) target = complq->txq_grp->txqs[id]; idpf_queue_clear(SW_MARKER, target); - if (target == txq) - break; next: if (unlikely(++ntc == complq->desc_count)) { ntc = 0; gen_flag = !gen_flag; } + if (target == txq) + break; } while (time_before(jiffies, timeout)); idpf_queue_assign(GEN_CHK, complq, gen_flag); @@ -3939,7 +3941,7 @@ static void idpf_update_dim_sample(struct idpf_q_vector *q_vector, static void idpf_net_dim(struct idpf_q_vector *q_vector) { struct dim_sample dim_sample = { }; - u64 packets, bytes; + u64 packets, bytes, pkts, bts; u32 i; if (!IDPF_ITR_IS_DYNAMIC(q_vector->tx_intr_mode)) @@ -3951,9 +3953,12 @@ static void idpf_net_dim(struct idpf_q_vector *q_vector) do { start = u64_stats_fetch_begin(&txq->stats_sync); - packets += u64_stats_read(&txq->q_stats.packets); - bytes += u64_stats_read(&txq->q_stats.bytes); + pkts = u64_stats_read(&txq->q_stats.packets); + bts = u64_stats_read(&txq->q_stats.bytes); } while (u64_stats_fetch_retry(&txq->stats_sync, start)); + + packets += pkts; + bytes += bts; } idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->tx_dim, @@ -3970,9 +3975,12 @@ static void idpf_net_dim(struct idpf_q_vector *q_vector) do { start = u64_stats_fetch_begin(&rxq->stats_sync); - packets += u64_stats_read(&rxq->q_stats.packets); - bytes += u64_stats_read(&rxq->q_stats.bytes); + pkts = u64_stats_read(&rxq->q_stats.packets); + bts = u64_stats_read(&rxq->q_stats.bytes); } while (u64_stats_fetch_retry(&rxq->stats_sync, start)); + + packets += pkts; + bytes += bts; } idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->rx_dim, @@ -4030,7 +4038,7 @@ static int idpf_vport_intr_req_irq(struct idpf_vport *vport) continue; name = kasprintf(GFP_KERNEL, "%s-%s-%s-%d", drv_name, if_name, - vec_name, vidx); + vec_name, vector); err = request_irq(irq_num, idpf_vport_intr_clean_queues, 0, name, q_vector); @@ -4641,7 +4649,7 @@ int idpf_config_rss(struct idpf_vport *vport) * idpf_fill_dflt_rss_lut - Fill the indirection table with the default values * @vport: virtual port structure */ -static void idpf_fill_dflt_rss_lut(struct idpf_vport *vport) +void idpf_fill_dflt_rss_lut(struct idpf_vport *vport) { struct idpf_adapter *adapter = vport->adapter; u16 num_active_rxq = vport->num_rxq; @@ -4650,57 +4658,47 @@ static void idpf_fill_dflt_rss_lut(struct idpf_vport *vport) rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data; - for (i = 0; i < rss_data->rss_lut_size; i++) { + for (i = 0; i < rss_data->rss_lut_size; i++) rss_data->rss_lut[i] = i % num_active_rxq; - rss_data->cached_lut[i] = rss_data->rss_lut[i]; - } } /** - * idpf_init_rss - Allocate and initialize RSS resources + * idpf_init_rss_lut - Allocate and initialize RSS LUT * @vport: virtual port * - * Return 0 on success, negative on failure + * Return: 0 on success, negative on failure */ -int idpf_init_rss(struct idpf_vport *vport) +int idpf_init_rss_lut(struct idpf_vport *vport) { struct idpf_adapter *adapter = vport->adapter; struct idpf_rss_data *rss_data; - u32 lut_size; rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data; + if (!rss_data->rss_lut) { + u32 lut_size; - lut_size = rss_data->rss_lut_size * sizeof(u32); - rss_data->rss_lut = kzalloc(lut_size, GFP_KERNEL); - if (!rss_data->rss_lut) - return -ENOMEM; - - rss_data->cached_lut = kzalloc(lut_size, GFP_KERNEL); - if (!rss_data->cached_lut) { - kfree(rss_data->rss_lut); - rss_data->rss_lut = NULL; - - return -ENOMEM; + lut_size = rss_data->rss_lut_size * sizeof(u32); + rss_data->rss_lut = kzalloc(lut_size, GFP_KERNEL); + if (!rss_data->rss_lut) + return -ENOMEM; } /* Fill the default RSS lut values */ idpf_fill_dflt_rss_lut(vport); - return idpf_config_rss(vport); + return 0; } /** - * idpf_deinit_rss - Release RSS resources + * idpf_deinit_rss_lut - Release RSS LUT * @vport: virtual port */ -void idpf_deinit_rss(struct idpf_vport *vport) +void idpf_deinit_rss_lut(struct idpf_vport *vport) { struct idpf_adapter *adapter = vport->adapter; struct idpf_rss_data *rss_data; rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data; - kfree(rss_data->cached_lut); - rss_data->cached_lut = NULL; kfree(rss_data->rss_lut); rss_data->rss_lut = NULL; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 75b977094741f..423cc9486dce7 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -101,6 +101,7 @@ do { \ idx = 0; \ } while (0) +#define IDPF_RX_MAX_BUF_SZ (16384 - 128) #define IDPF_RX_BUF_STRIDE 32 #define IDPF_RX_BUF_POST_STRIDE 16 #define IDPF_LOW_WATERMARK 64 @@ -1085,9 +1086,10 @@ void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector); void idpf_vport_intr_deinit(struct idpf_vport *vport); int idpf_vport_intr_init(struct idpf_vport *vport); void idpf_vport_intr_ena(struct idpf_vport *vport); +void idpf_fill_dflt_rss_lut(struct idpf_vport *vport); int idpf_config_rss(struct idpf_vport *vport); -int idpf_init_rss(struct idpf_vport *vport); -void idpf_deinit_rss(struct idpf_vport *vport); +int idpf_init_rss_lut(struct idpf_vport *vport); +void idpf_deinit_rss_lut(struct idpf_vport *vport); int idpf_rx_bufs_init_all(struct idpf_vport *vport); struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index cbb5fa30f5a0e..3c8ceff0e669f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -68,7 +68,7 @@ static void idpf_handle_event_link(struct idpf_adapter *adapter, vport->link_up = v2e->link_status; - if (np->state != __IDPF_VPORT_UP) + if (!test_bit(IDPF_VPORT_UP, np->state)) return; if (vport->link_up) { @@ -284,26 +284,21 @@ int idpf_send_mb_msg(struct idpf_adapter *adapter, u32 op, return err; } -/* API for virtchnl "transaction" support ("xn" for short). - * - * We are reusing the completion lock to serialize the accesses to the - * transaction state for simplicity, but it could be its own separate synchro - * as well. For now, this API is only used from within a workqueue context; - * raw_spin_lock() is enough. - */ +/* API for virtchnl "transaction" support ("xn" for short). */ + /** * idpf_vc_xn_lock - Request exclusive access to vc transaction * @xn: struct idpf_vc_xn* to access */ #define idpf_vc_xn_lock(xn) \ - raw_spin_lock(&(xn)->completed.wait.lock) + spin_lock(&(xn)->lock) /** * idpf_vc_xn_unlock - Release exclusive access to vc transaction * @xn: struct idpf_vc_xn* to access */ #define idpf_vc_xn_unlock(xn) \ - raw_spin_unlock(&(xn)->completed.wait.lock) + spin_unlock(&(xn)->lock) /** * idpf_vc_xn_release_bufs - Release reference to reply buffer(s) and @@ -335,6 +330,7 @@ static void idpf_vc_xn_init(struct idpf_vc_xn_manager *vcxn_mngr) xn->state = IDPF_VC_XN_IDLE; xn->idx = i; idpf_vc_xn_release_bufs(xn); + spin_lock_init(&xn->lock); init_completion(&xn->completed); } @@ -403,7 +399,9 @@ static void idpf_vc_xn_push_free(struct idpf_vc_xn_manager *vcxn_mngr, struct idpf_vc_xn *xn) { idpf_vc_xn_release_bufs(xn); + spin_lock_bh(&vcxn_mngr->xn_bm_lock); set_bit(xn->idx, vcxn_mngr->free_xn_bm); + spin_unlock_bh(&vcxn_mngr->xn_bm_lock); } /** @@ -614,6 +612,10 @@ idpf_vc_xn_forward_reply(struct idpf_adapter *adapter, err = -ENXIO; goto out_unlock; case IDPF_VC_XN_ASYNC: + /* Set reply_sz from the actual payload so that async_handler + * can evaluate the response. + */ + xn->reply_sz = ctlq_msg->data_len; err = idpf_vc_xn_forward_async(adapter, xn, ctlq_msg); idpf_vc_xn_unlock(xn); return err; @@ -1016,6 +1018,9 @@ static int idpf_send_get_lan_memory_regions(struct idpf_adapter *adapter) struct idpf_vc_xn_params xn_params = { .vc_op = VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS, .recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN, + .send_buf.iov_len = + sizeof(struct virtchnl2_get_lan_memory_regions) + + sizeof(struct virtchnl2_mem_region), .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; int num_regions, size; @@ -1028,6 +1033,8 @@ static int idpf_send_get_lan_memory_regions(struct idpf_adapter *adapter) return -ENOMEM; xn_params.recv_buf.iov_base = rcvd_regions; + rcvd_regions->num_memory_regions = cpu_to_le16(1); + xn_params.send_buf.iov_base = rcvd_regions; reply_sz = idpf_vc_xn_exec(adapter, &xn_params); if (reply_sz < 0) return reply_sz; @@ -2755,7 +2762,7 @@ int idpf_send_get_stats_msg(struct idpf_vport *vport) /* Don't send get_stats message if the link is down */ - if (np->state <= __IDPF_VPORT_DOWN) + if (!test_bit(IDPF_VPORT_UP, np->state)) return 0; stats_msg.vport_id = cpu_to_le32(vport->vport_id); @@ -2799,6 +2806,10 @@ int idpf_send_get_stats_msg(struct idpf_vport *vport) * @vport: virtual port data structure * @get: flag to set or get rss look up table * + * When rxhash is disabled, RSS LUT will be configured with zeros. If rxhash + * is enabled, the LUT values stored in driver's soft copy will be used to setup + * the HW. + * * Returns 0 on success, negative on failure. */ int idpf_send_get_set_rss_lut_msg(struct idpf_vport *vport, bool get) @@ -2809,10 +2820,12 @@ int idpf_send_get_set_rss_lut_msg(struct idpf_vport *vport, bool get) struct idpf_rss_data *rss_data; int buf_size, lut_buf_size; ssize_t reply_sz; + bool rxhash_ena; int i; rss_data = &vport->adapter->vport_config[vport->idx]->user_config.rss_data; + rxhash_ena = idpf_is_feature_ena(vport, NETIF_F_RXHASH); buf_size = struct_size(rl, lut, rss_data->rss_lut_size); rl = kzalloc(buf_size, GFP_KERNEL); if (!rl) @@ -2834,7 +2847,8 @@ int idpf_send_get_set_rss_lut_msg(struct idpf_vport *vport, bool get) } else { rl->lut_entries = cpu_to_le16(rss_data->rss_lut_size); for (i = 0; i < rss_data->rss_lut_size; i++) - rl->lut[i] = cpu_to_le32(rss_data->rss_lut[i]); + rl->lut[i] = rxhash_ena ? + cpu_to_le32(rss_data->rss_lut[i]) : 0; xn_params.vc_op = VIRTCHNL2_OP_SET_RSS_LUT; } @@ -3565,6 +3579,7 @@ int idpf_vc_core_init(struct idpf_adapter *adapter) */ void idpf_vc_core_deinit(struct idpf_adapter *adapter) { + struct idpf_hw *hw = &adapter->hw; bool remove_in_prog; if (!test_bit(IDPF_VC_CORE_INIT, adapter->flags)) @@ -3588,6 +3603,9 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter) idpf_vport_params_buf_rel(adapter); + kfree(hw->lan_regs); + hw->lan_regs = NULL; + kfree(adapter->vports); adapter->vports = NULL; diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h index eac3d15daa42f..0a50a08eb9af7 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h @@ -42,8 +42,8 @@ typedef int (*async_vc_cb) (struct idpf_adapter *, struct idpf_vc_xn *, * struct idpf_vc_xn - Data structure representing virtchnl transactions * @completed: virtchnl event loop uses that to signal when a reply is * available, uses kernel completion API - * @state: virtchnl event loop stores the data below, protected by the - * completion's lock. + * @lock: protects the transaction state fields below + * @state: virtchnl event loop stores the data below, protected by @lock * @reply_sz: Original size of reply, may be > reply_buf.iov_len; it will be * truncated on its way to the receiver thread according to * reply_buf.iov_len. @@ -58,6 +58,7 @@ typedef int (*async_vc_cb) (struct idpf_adapter *, struct idpf_vc_xn *, */ struct idpf_vc_xn { struct completion completed; + spinlock_t lock; enum idpf_vc_xn_state state; size_t reply_sz; struct kvec reply; diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index 21ce25b0567f6..958d16f874248 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -418,7 +418,7 @@ static int idpf_xdp_setup_prog(struct idpf_vport *vport, if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) || !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) || !!vport->xdp_prog == !!prog) { - if (np->state == __IDPF_VPORT_UP) + if (test_bit(IDPF_VPORT_UP, np->state)) idpf_xdp_copy_prog_to_rqs(vport, prog); old = xchg(&vport->xdp_prog, prog); diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 85f9589cc568c..33c6d61ef5188 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -2203,9 +2203,8 @@ void igb_down(struct igb_adapter *adapter) for (i = 0; i < adapter->num_q_vectors; i++) { if (adapter->q_vector[i]) { - napi_synchronize(&adapter->q_vector[i]->napi); - igb_set_queue_napi(adapter, i, NULL); napi_disable(&adapter->q_vector[i]->napi); + igb_set_queue_napi(adapter, i, NULL); } } @@ -9599,7 +9598,6 @@ static int __igb_resume(struct device *dev, bool rpm) pci_set_power_state(pdev, PCI_D0); pci_restore_state(pdev); - pci_save_state(pdev); if (!pci_device_is_present(pdev)) return -ENODEV; @@ -9754,7 +9752,6 @@ static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_enable_wake(pdev, PCI_D3hot, 0); pci_enable_wake(pdev, PCI_D3cold, 0); diff --git a/drivers/net/ethernet/intel/igb/igb_xsk.c b/drivers/net/ethernet/intel/igb/igb_xsk.c index 30ce5fbb5b776..ce4a7b58cad2f 100644 --- a/drivers/net/ethernet/intel/igb/igb_xsk.c +++ b/drivers/net/ethernet/intel/igb/igb_xsk.c @@ -524,6 +524,16 @@ bool igb_xmit_zc(struct igb_ring *tx_ring, struct xsk_buff_pool *xsk_pool) return nb_pkts < budget; } +static u32 igb_sw_irq_prep(struct igb_q_vector *q_vector) +{ + u32 eics = 0; + + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + eics = q_vector->eims_value; + + return eics; +} + int igb_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) { struct igb_adapter *adapter = netdev_priv(dev); @@ -542,20 +552,32 @@ int igb_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) ring = adapter->tx_ring[qid]; - if (test_bit(IGB_RING_FLAG_TX_DISABLED, &ring->flags)) - return -ENETDOWN; - if (!READ_ONCE(ring->xsk_pool)) return -EINVAL; - if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) { + if (flags & XDP_WAKEUP_TX) { + if (test_bit(IGB_RING_FLAG_TX_DISABLED, &ring->flags)) + return -ENETDOWN; + + eics |= igb_sw_irq_prep(ring->q_vector); + } + + if (flags & XDP_WAKEUP_RX) { + /* If IGB_FLAG_QUEUE_PAIRS is active, the q_vector + * and NAPI is shared between RX and TX. + * If NAPI is already running it would be marked as missed + * from the TX path, making this RX call a NOP + */ + ring = adapter->rx_ring[qid]; + eics |= igb_sw_irq_prep(ring->q_vector); + } + + if (eics) { /* Cause software interrupt */ - if (adapter->flags & IGB_FLAG_HAS_MSIX) { - eics |= ring->q_vector->eims_value; + if (adapter->flags & IGB_FLAG_HAS_MSIX) wr32(E1000_EICS, eics); - } else { + else wr32(E1000_ICS, E1000_ICS_RXDMT0); - } } return 0; diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index a427f05814c1a..17236813965d3 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -781,6 +781,8 @@ int igc_ptp_hwtstamp_set(struct net_device *netdev, struct kernel_hwtstamp_config *config, struct netlink_ext_ack *extack); void igc_ptp_tx_hang(struct igc_adapter *adapter); +void igc_ptp_clear_xsk_tx_tstamp_queue(struct igc_adapter *adapter, + u16 queue_id); void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts); void igc_ptp_tx_tstamp_event(struct igc_adapter *adapter); diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 498ba1522ca4d..9482ab11f050f 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -443,9 +443,10 @@ #define IGC_TXPBSIZE_DEFAULT ( \ IGC_TXPB0SIZE(20) | IGC_TXPB1SIZE(0) | IGC_TXPB2SIZE(0) | \ IGC_TXPB3SIZE(0) | IGC_OS2BMCPBSIZE(4)) +/* TSN value following I225/I226 SW User Manual Section 7.5.4 */ #define IGC_TXPBSIZE_TSN ( \ - IGC_TXPB0SIZE(7) | IGC_TXPB1SIZE(7) | IGC_TXPB2SIZE(7) | \ - IGC_TXPB3SIZE(7) | IGC_OS2BMCPBSIZE(4)) + IGC_TXPB0SIZE(5) | IGC_TXPB1SIZE(5) | IGC_TXPB2SIZE(5) | \ + IGC_TXPB3SIZE(5) | IGC_OS2BMCPBSIZE(4)) #define IGC_DTXMXPKTSZ_TSN 0x19 /* 1600 bytes of max TX DMA packet size */ #define IGC_DTXMXPKTSZ_DEFAULT 0x98 /* 9728-byte Jumbo frames */ diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index bb783042d1af9..4b39329e9e32b 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -1561,8 +1561,8 @@ static int igc_ethtool_set_channels(struct net_device *netdev, if (ch->other_count != NON_Q_VECTORS) return -EINVAL; - /* Do not allow channel reconfiguration when mqprio is enabled */ - if (adapter->strict_priority_enable) + /* Do not allow channel reconfiguration when any TSN qdisc is enabled */ + if (adapter->flags & IGC_FLAG_TSN_ANY_ENABLED) return -EINVAL; /* Verify the number of channels doesn't exceed hw limits */ diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 728d7ca5338bf..104d6ab2ce5fa 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -264,6 +264,13 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) /* reset next_to_use and next_to_clean */ tx_ring->next_to_use = 0; tx_ring->next_to_clean = 0; + + /* Clear any lingering XSK TX timestamp requests */ + if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) { + struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); + + igc_ptp_clear_xsk_tx_tstamp_queue(adapter, tx_ring->queue_index); + } } /** @@ -1730,11 +1737,8 @@ static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, /* The minimum packet size with TCTL.PSP set is 17 so pad the skb * in order to meet this minimum size requirement. */ - if (skb->len < 17) { - if (skb_padto(skb, 17)) - return NETDEV_TX_OK; - skb->len = 17; - } + if (skb_put_padto(skb, 17)) + return NETDEV_TX_OK; return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); } @@ -7530,7 +7534,6 @@ static int __igc_resume(struct device *dev, bool rpm) pci_set_power_state(pdev, PCI_D0); pci_restore_state(pdev); - pci_save_state(pdev); if (!pci_device_is_present(pdev)) return -ENODEV; @@ -7667,7 +7670,6 @@ static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_enable_wake(pdev, PCI_D3hot, 0); pci_enable_wake(pdev, PCI_D3cold, 0); @@ -7761,6 +7763,11 @@ int igc_reinit_queues(struct igc_adapter *adapter) if (netif_running(netdev)) err = igc_open(netdev); + if (!err) { + /* Restore default IEEE 802.1Qbv schedule after queue reinit */ + igc_tsn_clear_schedule(adapter); + } + return err; } diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index b7b46d863bee4..98491346d21b8 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -576,6 +576,39 @@ static void igc_ptp_clear_tx_tstamp(struct igc_adapter *adapter) spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); } +/** + * igc_ptp_clear_xsk_tx_tstamp_queue - Clear pending XSK TX timestamps for a queue + * @adapter: Board private structure + * @queue_id: TX queue index to clear timestamps for + * + * Iterates over all TX timestamp registers and releases any pending + * timestamp requests associated with the given TX queue. This is + * called when an XDP pool is being disabled to ensure no stale + * timestamp references remain. + */ +void igc_ptp_clear_xsk_tx_tstamp_queue(struct igc_adapter *adapter, u16 queue_id) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&adapter->ptp_tx_lock, flags); + + for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) { + struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i]; + + if (tstamp->buffer_type != IGC_TX_BUFFER_TYPE_XSK) + continue; + if (tstamp->xsk_queue_index != queue_id) + continue; + if (!tstamp->xsk_tx_buffer) + continue; + + igc_ptp_free_tx_buffer(adapter, tstamp); + } + + spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); +} + static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter) { struct igc_hw *hw = &adapter->hw; @@ -774,36 +807,43 @@ static void igc_ptp_tx_reg_to_stamp(struct igc_adapter *adapter, static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter) { struct igc_hw *hw = &adapter->hw; + u32 txstmpl_old; u64 regval; u32 mask; int i; + /* Establish baseline of TXSTMPL_0 before checking TXTT_0. + * This baseline is used to detect if a new timestamp arrives in + * register 0 during the hardware bug workaround below. + */ + txstmpl_old = rd32(IGC_TXSTMPL); + mask = rd32(IGC_TSYNCTXCTL) & IGC_TSYNCTXCTL_TXTT_ANY; if (mask & IGC_TSYNCTXCTL_TXTT_0) { regval = rd32(IGC_TXSTMPL); regval |= (u64)rd32(IGC_TXSTMPH) << 32; } else { - /* There's a bug in the hardware that could cause - * missing interrupts for TX timestamping. The issue - * is that for new interrupts to be triggered, the - * IGC_TXSTMPH_0 register must be read. + /* TXTT_0 not set - register 0 has no new timestamp initially. + * + * Hardware bug: Future timestamp interrupts won't fire unless + * TXSTMPH_0 is read, even if the timestamp was captured in + * registers 1-3. * - * To avoid discarding a valid timestamp that just - * happened at the "wrong" time, we need to confirm - * that there was no timestamp captured, we do that by - * assuming that no two timestamps in sequence have - * the same nanosecond value. + * Workaround: Read TXSTMPH_0 here to enable future interrupts. + * However, this read clears TXTT_0. If a timestamp arrives in + * register 0 after checking TXTT_0 but before this read, it + * would be lost. * - * So, we read the "low" register, read the "high" - * register (to latch a new timestamp) and read the - * "low" register again, if "old" and "new" versions - * of the "low" register are different, a valid - * timestamp was captured, we can read the "high" - * register again. + * To detect this race: We saved a baseline read of TXSTMPL_0 + * before TXTT_0 check. After performing the workaround read of + * TXSTMPH_0, we read TXSTMPL_0 again. Since consecutive + * timestamps never share the same nanosecond value, a change + * between the baseline and new TXSTMPL_0 indicates a timestamp + * arrived during the race window. If so, read the complete + * timestamp. */ - u32 txstmpl_old, txstmpl_new; + u32 txstmpl_new; - txstmpl_old = rd32(IGC_TXSTMPL); rd32(IGC_TXSTMPH); txstmpl_new = rd32(IGC_TXSTMPL); @@ -818,7 +858,7 @@ static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter) done: /* Now that the problematic first register was handled, we can - * use retrieve the timestamps from the other registers + * retrieve the timestamps from the other registers * (starting from '1') with less complications. */ for (i = 1; i < IGC_MAX_TX_TSTAMP_REGS; i++) { diff --git a/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c b/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c index d227f4d2a2d17..f32e640ef4ac0 100644 --- a/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ixgbe/devlink/devlink.c @@ -474,7 +474,7 @@ static int ixgbe_devlink_reload_empr_finish(struct devlink *devlink, adapter->flags2 &= ~(IXGBE_FLAG2_API_MISMATCH | IXGBE_FLAG2_FW_ROLLBACK); - return 0; + return ixgbe_refresh_fw_version(adapter); } static const struct devlink_ops ixgbe_devlink_ops = { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index dce4936708eb4..047f04045585a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -973,7 +973,7 @@ int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter); bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, u16 subdevice_id); void ixgbe_set_fw_version_e610(struct ixgbe_adapter *adapter); -void ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter); +int ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter); #ifdef CONFIG_PCI_IOV void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter); #endif diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 2d660e9edb80a..0c8f310689776 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1153,12 +1153,17 @@ static int ixgbe_set_eeprom(struct net_device *netdev, return ret_val; } -void ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter) +int ixgbe_refresh_fw_version(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + int err; + + err = ixgbe_get_flash_data(hw); + if (err) + return err; - ixgbe_get_flash_data(hw); ixgbe_set_fw_version_e610(adapter); + return 0; } static void ixgbe_get_drvinfo(struct net_device *netdev, @@ -1166,10 +1171,6 @@ static void ixgbe_get_drvinfo(struct net_device *netdev, { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); - /* need to refresh info for e610 in case fw reloads in runtime */ - if (adapter->hw.mac.type == ixgbe_mac_e610) - ixgbe_refresh_fw_version(adapter); - strscpy(drvinfo->driver, ixgbe_driver_name, sizeof(drvinfo->driver)); strscpy(drvinfo->fw_version, adapter->eeprom_id, diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 3190ce7e44c74..240f7cc3f213f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -6289,6 +6289,16 @@ void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) msleep(2000); ixgbe_up(adapter); + + /* E610 has no FW event to notify all PFs of an EMPR reset, so + * refresh the FW version here to pick up any new FW version after + * a hardware reset (e.g. EMPR triggered by another PF's devlink + * reload). ixgbe_refresh_fw_version() updates both hw->flash and + * adapter->eeprom_id so ethtool -i reports the correct string. + */ + if (adapter->hw.mac.type == ixgbe_mac_e610) + (void)ixgbe_refresh_fw_version(adapter); + clear_bit(__IXGBE_RESETTING, &adapter->state); } @@ -11468,20 +11478,17 @@ static void ixgbe_set_fw_version(struct ixgbe_adapter *adapter) */ static int ixgbe_recovery_probe(struct ixgbe_adapter *adapter) { - struct net_device *netdev = adapter->netdev; struct pci_dev *pdev = adapter->pdev; struct ixgbe_hw *hw = &adapter->hw; - bool disable_dev; int err = -EIO; if (hw->mac.type != ixgbe_mac_e610) - goto clean_up_probe; + return err; ixgbe_get_hw_control(adapter); - mutex_init(&hw->aci.lock); err = ixgbe_get_flash_data(&adapter->hw); if (err) - goto shutdown_aci; + goto err_release_hw_control; timer_setup(&adapter->service_timer, ixgbe_service_timer, 0); INIT_WORK(&adapter->service_task, ixgbe_recovery_service_task); @@ -11504,16 +11511,8 @@ static int ixgbe_recovery_probe(struct ixgbe_adapter *adapter) devl_unlock(adapter->devlink); return 0; -shutdown_aci: - mutex_destroy(&adapter->hw.aci.lock); +err_release_hw_control: ixgbe_release_hw_control(adapter); -clean_up_probe: - disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state); - free_netdev(netdev); - devlink_free(adapter->devlink); - pci_release_mem_regions(pdev); - if (disable_dev) - pci_disable_device(pdev); return err; } @@ -11655,8 +11654,13 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) goto err_sw_init; - if (ixgbe_check_fw_error(adapter)) - return ixgbe_recovery_probe(adapter); + if (ixgbe_check_fw_error(adapter)) { + err = ixgbe_recovery_probe(adapter); + if (err) + goto err_sw_init; + + return 0; + } if (adapter->hw.mac.type == ixgbe_mac_e610) { err = ixgbe_get_caps(&adapter->hw); @@ -12298,7 +12302,6 @@ static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) adapter->hw.hw_addr = adapter->io_addr; pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_wake_from_d3(pdev, false); diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.c b/drivers/net/ethernet/intel/ixgbevf/vf.c index 74d320879513c..f6df86d124b9e 100644 --- a/drivers/net/ethernet/intel/ixgbevf/vf.c +++ b/drivers/net/ethernet/intel/ixgbevf/vf.c @@ -709,6 +709,12 @@ static int ixgbevf_negotiate_features_vf(struct ixgbe_hw *hw, u32 *pf_features) return err; } +static int ixgbevf_hv_negotiate_features_vf(struct ixgbe_hw *hw, + u32 *pf_features) +{ + return -EOPNOTSUPP; +} + /** * ixgbevf_set_vfta_vf - Set/Unset VLAN filter table address * @hw: pointer to the HW structure @@ -852,7 +858,8 @@ static s32 ixgbevf_check_mac_link_vf(struct ixgbe_hw *hw, if (!mac->get_link_status) goto out; - if (hw->mac.type == ixgbe_mac_e610_vf) { + if (hw->mac.type == ixgbe_mac_e610_vf && + hw->api_version >= ixgbe_mbox_api_16) { ret_val = ixgbevf_get_pf_link_state(hw, speed, link_up); if (ret_val) goto out; @@ -1141,6 +1148,7 @@ static const struct ixgbe_mac_operations ixgbevf_hv_mac_ops = { .setup_link = ixgbevf_setup_mac_link_vf, .check_link = ixgbevf_hv_check_mac_link_vf, .negotiate_api_version = ixgbevf_hv_negotiate_api_version_vf, + .negotiate_features = ixgbevf_hv_negotiate_features_vf, .set_rar = ixgbevf_hv_set_rar_vf, .update_mc_addr_list = ixgbevf_hv_update_mc_addr_list_vf, .update_xcast_mode = ixgbevf_hv_update_xcast_mode, diff --git a/drivers/net/ethernet/intel/libie/fwlog.c b/drivers/net/ethernet/intel/libie/fwlog.c index f39cc11cb7c56..3b32986c2978a 100644 --- a/drivers/net/ethernet/intel/libie/fwlog.c +++ b/drivers/net/ethernet/intel/libie/fwlog.c @@ -433,17 +433,21 @@ libie_debugfs_module_write(struct file *filp, const char __user *buf, module = libie_find_module_by_dentry(fwlog->debugfs_modules, dentry); if (module < 0) { dev_info(dev, "unknown module\n"); - return -EINVAL; + count = -EINVAL; + goto free_cmd_buf; } cnt = sscanf(cmd_buf, "%s", user_val); - if (cnt != 1) - return -EINVAL; + if (cnt != 1) { + count = -EINVAL; + goto free_cmd_buf; + } log_level = sysfs_match_string(libie_fwlog_level_string, user_val); if (log_level < 0) { dev_info(dev, "unknown log level '%s'\n", user_val); - return -EINVAL; + count = -EINVAL; + goto free_cmd_buf; } if (module != LIBIE_AQC_FW_LOG_ID_MAX) { @@ -458,6 +462,9 @@ libie_debugfs_module_write(struct file *filp, const char __user *buf, fwlog->cfg.module_entries[i].log_level = log_level; } +free_cmd_buf: + kfree(cmd_buf); + return count; } @@ -515,23 +522,31 @@ libie_debugfs_nr_messages_write(struct file *filp, const char __user *buf, return PTR_ERR(cmd_buf); ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; + if (ret != 1) { + count = -EINVAL; + goto free_cmd_buf; + } ret = kstrtos16(user_val, 0, &nr_messages); - if (ret) - return ret; + if (ret) { + count = ret; + goto free_cmd_buf; + } if (nr_messages < LIBIE_AQC_FW_LOG_MIN_RESOLUTION || nr_messages > LIBIE_AQC_FW_LOG_MAX_RESOLUTION) { dev_err(dev, "Invalid FW log number of messages %d, value must be between %d - %d\n", nr_messages, LIBIE_AQC_FW_LOG_MIN_RESOLUTION, LIBIE_AQC_FW_LOG_MAX_RESOLUTION); - return -EINVAL; + count = -EINVAL; + goto free_cmd_buf; } fwlog->cfg.log_resolution = nr_messages; +free_cmd_buf: + kfree(cmd_buf); + return count; } @@ -588,8 +603,10 @@ libie_debugfs_enable_write(struct file *filp, const char __user *buf, return PTR_ERR(cmd_buf); ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; + if (ret != 1) { + ret = -EINVAL; + goto free_cmd_buf; + } ret = kstrtobool(user_val, &enable); if (ret) @@ -624,6 +641,8 @@ libie_debugfs_enable_write(struct file *filp, const char __user *buf, */ if (WARN_ON(ret != (ssize_t)count && ret >= 0)) ret = -EIO; +free_cmd_buf: + kfree(cmd_buf); return ret; } @@ -682,8 +701,10 @@ libie_debugfs_log_size_write(struct file *filp, const char __user *buf, return PTR_ERR(cmd_buf); ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; + if (ret != 1) { + ret = -EINVAL; + goto free_cmd_buf; + } index = sysfs_match_string(libie_fwlog_log_size, user_val); if (index < 0) { @@ -712,6 +733,8 @@ libie_debugfs_log_size_write(struct file *filp, const char __user *buf, */ if (WARN_ON(ret != (ssize_t)count && ret >= 0)) ret = -EIO; +free_cmd_buf: + kfree(cmd_buf); return ret; } @@ -1051,6 +1074,10 @@ void libie_fwlog_deinit(struct libie_fwlog *fwlog) { int status; + /* if FW logging isn't supported it means no configuration was done */ + if (!libie_fwlog_supported(fwlog)) + return; + /* make sure FW logging is disabled to not put the FW in a weird state * for the next driver load */ diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_cls.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_cls.c index 44b201817d94c..c116da7d7f18c 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_cls.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_cls.c @@ -1389,7 +1389,7 @@ int mvpp2_ethtool_cls_rule_ins(struct mvpp2_port *port, efs->rule.flow_type = mvpp2_cls_ethtool_flow_to_type(info->fs.flow_type); if (efs->rule.flow_type < 0) { ret = efs->rule.flow_type; - goto clean_rule; + goto clean_eth_rule; } ret = mvpp2_cls_rfs_parse_rule(&efs->rule); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index ab0c99aa9f9a5..74d44510684bf 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5018,7 +5018,7 @@ static int mvpp2_bm_switch_buffers(struct mvpp2 *priv, bool percpu) if (priv->percpu_pools) numbufs = port->nrxqs * 2; - if (change_percpu) + if (change_percpu && priv->global_tx_fc) mvpp2_bm_pool_update_priv_fc(priv, false); for (i = 0; i < numbufs; i++) @@ -5043,7 +5043,7 @@ static int mvpp2_bm_switch_buffers(struct mvpp2 *priv, bool percpu) mvpp2_open(port->dev); } - if (change_percpu) + if (change_percpu && priv->global_tx_fc) mvpp2_bm_pool_update_priv_fc(priv, true); return 0; diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c index b5805969404fa..01e82d0b6b2cd 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c @@ -307,7 +307,7 @@ static void octep_setup_iq_regs_cn93_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) { u64 reg_val; u64 oq_ctl = 0ULL; @@ -355,6 +355,7 @@ static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -696,14 +697,26 @@ static void octep_enable_interrupts_cn93_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cn93_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c index 5de0b5ecbc5fd..09a3f1d0645b8 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "octep_config.h" #include "octep_main.h" @@ -327,12 +328,14 @@ static void octep_setup_iq_regs_cnxk_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) { - u64 reg_val; - u64 oq_ctl = 0ULL; - u32 time_threshold = 0; struct octep_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; + u32 time_threshold = 0; + u64 oq_ctl = 0ULL; + u64 reg_ba_val; + u64 reg_val; oq_no += CFG_GET_PORTS_PF_SRN(oct->conf); reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -343,6 +346,36 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_R_OUT_CTL_IDLE)); } + octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = + octep_read_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_R_OUT_CTL_ROR_P); @@ -356,10 +389,6 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val |= (CNXK_R_OUT_CTL_ES_P); octep_write_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), - oq->desc_ring_dma); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), - oq->max_count); oq_ctl = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -385,6 +414,7 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val &= ~0xFFFFFFFFULL; reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -720,14 +750,26 @@ static void octep_enable_interrupts_cnxk_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cnxk_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CNXK_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CNXK_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c index bcea3fc26a8c7..16f52d4b11e91 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c @@ -555,28 +555,43 @@ static void octep_clean_irqs(struct octep_device *oct) } /** - * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * octep_update_pkt() - Update IQ/OQ IN/OUT_CNT registers. * * @iq: Octeon Tx queue data structure. * @oq: Octeon Rx queue data structure. */ -static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) +static void octep_update_pkt(struct octep_iq *iq, struct octep_oq *oq) { - u32 pkts_pend = oq->pkts_pending; + u32 pkts_pend = READ_ONCE(oq->pkts_pending); + u32 last_pkt_count = READ_ONCE(oq->last_pkt_count); + u32 pkts_processed = READ_ONCE(iq->pkts_processed); + u32 pkt_in_done = READ_ONCE(iq->pkt_in_done); netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); - if (iq->pkts_processed) { - writel(iq->pkts_processed, iq->inst_cnt_reg); - iq->pkt_in_done -= iq->pkts_processed; - iq->pkts_processed = 0; + if (pkts_processed) { + writel(pkts_processed, iq->inst_cnt_reg); + readl(iq->inst_cnt_reg); + WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed)); + WRITE_ONCE(iq->pkts_processed, 0); } - if (oq->last_pkt_count - pkts_pend) { - writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); - oq->last_pkt_count = pkts_pend; + if (last_pkt_count - pkts_pend) { + writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg); + readl(oq->pkts_sent_reg); + WRITE_ONCE(oq->last_pkt_count, pkts_pend); } /* Flush the previous wrties before writing to RESEND bit */ - wmb(); + smp_wmb(); +} + +/** + * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * + * @iq: Octeon Tx queue data structure. + * @oq: Octeon Rx queue data structure. + */ +static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) +{ writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); } @@ -602,7 +617,8 @@ static int octep_napi_poll(struct napi_struct *napi, int budget) if (tx_pending || rx_done >= budget) return budget; - napi_complete(napi); + octep_update_pkt(ioq_vector->iq, ioq_vector->oq); + napi_complete_done(napi, rx_done); octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); return rx_done; } @@ -1338,7 +1354,7 @@ int octep_device_setup(struct octep_device *oct) ret = octep_ctrl_net_init(oct); if (ret) - return ret; + goto unsupported_dev; INIT_WORK(&oct->tx_timeout_task, octep_tx_timeout_task); INIT_WORK(&oct->ctrl_mbox_task, octep_ctrl_mbox_task); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h index 81ac4267811c8..35d0ff289a70d 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h @@ -77,7 +77,7 @@ struct octep_pci_win_regs { struct octep_hw_ops { void (*setup_iq_regs)(struct octep_device *oct, int q); - void (*setup_oq_regs)(struct octep_device *oct, int q); + int (*setup_oq_regs)(struct octep_device *oct, int q); void (*setup_mbox_regs)(struct octep_device *oct, int mbox); irqreturn_t (*mbox_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h index ca473502d7a02..95f1dfff90cce 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h @@ -386,5 +386,6 @@ #define CN93_PEM_BAR4_INDEX 7 #define CN93_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CN93_PEM_BAR4_INDEX_OFFSET (CN93_PEM_BAR4_INDEX * CN93_PEM_BAR4_INDEX_SIZE) +#define CN93_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CN9K_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h index e637d7c8224d4..4d172a552f80c 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h @@ -412,5 +412,6 @@ #define CNXK_PEM_BAR4_INDEX 7 #define CNXK_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CNXK_PEM_BAR4_INDEX_OFFSET (CNXK_PEM_BAR4_INDEX * CNXK_PEM_BAR4_INDEX_SIZE) +#define CNXK_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CNXK_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c index 82b6b19e76b47..74de19166488f 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -12,6 +12,8 @@ #include "octep_config.h" #include "octep_main.h" +static void octep_oq_free_ring_buffers(struct octep_oq *oq); + static void octep_oq_reset_indices(struct octep_oq *oq) { oq->host_read_idx = 0; @@ -170,11 +172,15 @@ static int octep_setup_oq(struct octep_device *oct, int q_no) goto oq_fill_buff_err; octep_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; @@ -318,10 +324,16 @@ static int octep_oq_check_hw_for_pkts(struct octep_device *oct, struct octep_oq *oq) { u32 pkt_count, new_pkts; + u32 last_pkt_count, pkts_pending; pkt_count = readl(oq->pkts_sent_reg); - new_pkts = pkt_count - oq->last_pkt_count; + last_pkt_count = READ_ONCE(oq->last_pkt_count); + new_pkts = pkt_count - last_pkt_count; + if (pkt_count < last_pkt_count) { + dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n", + oq->q_no, pkt_count, last_pkt_count); + } /* Clear the hardware packets counter register if the rx queue is * being processed continuously with-in a single interrupt and * reached half its max value. @@ -332,8 +344,9 @@ static int octep_oq_check_hw_for_pkts(struct octep_device *oct, pkt_count = readl(oq->pkts_sent_reg); new_pkts += pkt_count; } - oq->last_pkt_count = pkt_count; - oq->pkts_pending += new_pkts; + WRITE_ONCE(oq->last_pkt_count, pkt_count); + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts)); return new_pkts; } @@ -408,7 +421,7 @@ static int __octep_oq_process_rx(struct octep_device *oct, u16 rx_ol_flags; u32 read_idx; - read_idx = oq->host_read_idx; + read_idx = READ_ONCE(oq->host_read_idx); rx_bytes = 0; desc_used = 0; for (pkt = 0; pkt < pkts_to_process; pkt++) { @@ -493,7 +506,7 @@ static int __octep_oq_process_rx(struct octep_device *oct, napi_gro_receive(oq->napi, skb); } - oq->host_read_idx = read_idx; + WRITE_ONCE(oq->host_read_idx, read_idx); oq->refill_count += desc_used; oq->stats->packets += pkt; oq->stats->bytes += rx_bytes; @@ -516,22 +529,26 @@ int octep_oq_process_rx(struct octep_oq *oq, int budget) { u32 pkts_available, pkts_processed, total_pkts_processed; struct octep_device *oct = oq->octep_dev; + u32 pkts_pending; pkts_available = 0; pkts_processed = 0; total_pkts_processed = 0; while (total_pkts_processed < budget) { /* update pending count only when current one exhausted */ - if (oq->pkts_pending == 0) + pkts_pending = READ_ONCE(oq->pkts_pending); + if (pkts_pending == 0) octep_oq_check_hw_for_pkts(oct, oq); + pkts_pending = READ_ONCE(oq->pkts_pending); pkts_available = min(budget - total_pkts_processed, - oq->pkts_pending); + pkts_pending); if (!pkts_available) break; pkts_processed = __octep_oq_process_rx(oct, oq, pkts_available); - oq->pkts_pending -= pkts_processed; + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed)); total_pkts_processed += pkts_processed; } diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c index 88937fce75f14..4c769b27c2789 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c @@ -196,7 +196,7 @@ static void octep_vf_setup_iq_regs_cn93(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; u32 time_threshold = 0; @@ -239,6 +239,7 @@ static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) time_threshold = CFG_GET_OQ_INTR_TIME(oct->conf); reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_vf_write_csr64(oct, CN93_VF_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c index 1f79dfad42c62..a968b93a67943 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c @@ -199,11 +199,13 @@ static void octep_vf_setup_iq_regs_cnxk(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; u32 time_threshold = 0; u64 oq_ctl = ULL(0); + u64 reg_ba_val; u64 reg_val; reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); @@ -214,6 +216,38 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_VF_R_OUT_CTL_IDLE)); } + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), + oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no)); + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no), oq->desc_ring_dma); + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_RSIZE + (oq_no), oq->max_count); + reg_ba_val = + octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_VF_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_VF_R_OUT_CTL_ROR_P); @@ -227,8 +261,6 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val |= (CNXK_VF_R_OUT_CTL_ES_P); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), oq->desc_ring_dma); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), oq->max_count); oq_ctl = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); /* Clear the ISIZE and BSIZE (22-0) */ @@ -250,6 +282,7 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val &= ~GENMASK_ULL(31, 0); reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c index 420c3f4cf7417..a3c359124887e 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c @@ -218,7 +218,7 @@ static int octep_vf_request_irqs(struct octep_vf_device *oct) ioq_irq_err: while (i) { --i; - free_irq(oct->msix_entries[i].vector, oct); + free_irq(oct->msix_entries[i].vector, oct->ioq_vector[i]); } return -1; } @@ -286,28 +286,45 @@ static void octep_vf_clean_irqs(struct octep_vf_device *oct) } /** - * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * octep_vf_update_pkt() - Update IQ/OQ IN/OUT_CNT registers. * * @iq: Octeon Tx queue data structure. * @oq: Octeon Rx queue data structure. */ -static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, struct octep_vf_oq *oq) + +static void octep_vf_update_pkt(struct octep_vf_iq *iq, struct octep_vf_oq *oq) { - u32 pkts_pend = oq->pkts_pending; + u32 pkts_pend = READ_ONCE(oq->pkts_pending); + u32 last_pkt_count = READ_ONCE(oq->last_pkt_count); + u32 pkts_processed = READ_ONCE(iq->pkts_processed); + u32 pkt_in_done = READ_ONCE(iq->pkt_in_done); netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); - if (iq->pkts_processed) { - writel(iq->pkts_processed, iq->inst_cnt_reg); - iq->pkt_in_done -= iq->pkts_processed; - iq->pkts_processed = 0; + if (pkts_processed) { + writel(pkts_processed, iq->inst_cnt_reg); + readl(iq->inst_cnt_reg); + WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed)); + WRITE_ONCE(iq->pkts_processed, 0); } - if (oq->last_pkt_count - pkts_pend) { - writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); - oq->last_pkt_count = pkts_pend; + if (last_pkt_count - pkts_pend) { + writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg); + readl(oq->pkts_sent_reg); + WRITE_ONCE(oq->last_pkt_count, pkts_pend); } /* Flush the previous wrties before writing to RESEND bit */ smp_wmb(); +} + +/** + * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * + * @iq: Octeon Tx queue data structure. + * @oq: Octeon Rx queue data structure. + */ +static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, + struct octep_vf_oq *oq) +{ writeq(1UL << OCTEP_VF_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); writeq(1UL << OCTEP_VF_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); } @@ -333,6 +350,7 @@ static int octep_vf_napi_poll(struct napi_struct *napi, int budget) if (tx_pending || rx_done >= budget) return budget; + octep_vf_update_pkt(ioq_vector->iq, ioq_vector->oq); if (likely(napi_complete_done(napi, rx_done))) octep_vf_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h index b9f13506f4620..c74cd2369e90d 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h @@ -55,7 +55,7 @@ struct octep_vf_mmio { struct octep_vf_hw_ops { void (*setup_iq_regs)(struct octep_vf_device *oct, int q); - void (*setup_oq_regs)(struct octep_vf_device *oct, int q); + int (*setup_oq_regs)(struct octep_vf_device *oct, int q); void (*setup_mbox_regs)(struct octep_vf_device *oct, int mbox); irqreturn_t (*non_ioq_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c index d70c8be3cfc40..b579d5b545c46 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c @@ -12,6 +12,8 @@ #include "octep_vf_config.h" #include "octep_vf_main.h" +static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq); + static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq) { oq->host_read_idx = 0; @@ -171,11 +173,15 @@ static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no) goto oq_fill_buff_err; octep_vf_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_vf_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; @@ -319,9 +325,16 @@ static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, struct octep_vf_oq *oq) { u32 pkt_count, new_pkts; + u32 last_pkt_count, pkts_pending; pkt_count = readl(oq->pkts_sent_reg); - new_pkts = pkt_count - oq->last_pkt_count; + last_pkt_count = READ_ONCE(oq->last_pkt_count); + new_pkts = pkt_count - last_pkt_count; + + if (pkt_count < last_pkt_count) { + dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n", + oq->q_no, pkt_count, last_pkt_count); + } /* Clear the hardware packets counter register if the rx queue is * being processed continuously with-in a single interrupt and @@ -333,8 +346,9 @@ static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, pkt_count = readl(oq->pkts_sent_reg); new_pkts += pkt_count; } - oq->last_pkt_count = pkt_count; - oq->pkts_pending += new_pkts; + WRITE_ONCE(oq->last_pkt_count, pkt_count); + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts)); return new_pkts; } @@ -363,7 +377,7 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, struct sk_buff *skb; u32 read_idx; - read_idx = oq->host_read_idx; + read_idx = READ_ONCE(oq->host_read_idx); rx_bytes = 0; desc_used = 0; for (pkt = 0; pkt < pkts_to_process; pkt++) { @@ -457,7 +471,7 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, napi_gro_receive(oq->napi, skb); } - oq->host_read_idx = read_idx; + WRITE_ONCE(oq->host_read_idx, read_idx); oq->refill_count += desc_used; oq->stats->packets += pkt; oq->stats->bytes += rx_bytes; @@ -480,22 +494,26 @@ int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget) { u32 pkts_available, pkts_processed, total_pkts_processed; struct octep_vf_device *oct = oq->octep_vf_dev; + u32 pkts_pending; pkts_available = 0; pkts_processed = 0; total_pkts_processed = 0; while (total_pkts_processed < budget) { /* update pending count only when current one exhausted */ - if (oq->pkts_pending == 0) + pkts_pending = READ_ONCE(oq->pkts_pending); + if (pkts_pending == 0) octep_vf_oq_check_hw_for_pkts(oct, oq); + pkts_pending = READ_ONCE(oq->pkts_pending); pkts_available = min(budget - total_pkts_processed, - oq->pkts_pending); + pkts_pending); if (!pkts_available) break; pkts_processed = __octep_vf_oq_process_rx(oct, oq, pkts_available); - oq->pkts_pending -= pkts_processed; + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed)); total_pkts_processed += pkts_processed; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index ec0e11c77cbf2..81b55f1416e0d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -1823,6 +1823,8 @@ static int cgx_lmac_exit(struct cgx *cgx) cgx->mac_ops->mac_pause_frm_config(cgx, lmac->lmac_id, false); cgx_configure_interrupt(cgx, lmac, lmac->lmac_id, true); kfree(lmac->mac_to_index_bmap.bmap); + rvu_free_bitmap(&lmac->rx_fc_pfvf_bmap); + rvu_free_bitmap(&lmac->tx_fc_pfvf_bmap); kfree(lmac->name); kfree(lmac); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 2d78e08f985f0..8530df8b3fdaf 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -1551,8 +1551,8 @@ static int rvu_get_attach_blkaddr(struct rvu *rvu, int blktype, return -ENODEV; } -static void rvu_attach_block(struct rvu *rvu, int pcifunc, int blktype, - int num_lfs, struct rsrc_attach *attach) +static int rvu_attach_block(struct rvu *rvu, int pcifunc, int blktype, + int num_lfs, struct rsrc_attach *attach) { struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); struct rvu_hwinfo *hw = rvu->hw; @@ -1562,21 +1562,21 @@ static void rvu_attach_block(struct rvu *rvu, int pcifunc, int blktype, u64 cfg; if (!num_lfs) - return; + return -EINVAL; blkaddr = rvu_get_attach_blkaddr(rvu, blktype, pcifunc, attach); if (blkaddr < 0) - return; + return -EFAULT; block = &hw->block[blkaddr]; if (!block->lf.bmap) - return; + return -ESRCH; for (slot = 0; slot < num_lfs; slot++) { /* Allocate the resource */ lf = rvu_alloc_rsrc(&block->lf); if (lf < 0) - return; + return -EFAULT; cfg = (1ULL << 63) | (pcifunc << 8) | slot; rvu_write64(rvu, blkaddr, block->lfcfg_reg | @@ -1587,6 +1587,8 @@ static void rvu_attach_block(struct rvu *rvu, int pcifunc, int blktype, /* Set start MSIX vector for this LF within this PF/VF */ rvu_set_msix_offset(rvu, pfvf, block, lf); } + + return 0; } static int rvu_check_rsrc_availability(struct rvu *rvu, @@ -1724,22 +1726,31 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, int err; /* If first request, detach all existing attached resources */ - if (!attach->modify) - rvu_detach_rsrcs(rvu, NULL, pcifunc); + if (!attach->modify) { + err = rvu_detach_rsrcs(rvu, NULL, pcifunc); + if (err) + return err; + } mutex_lock(&rvu->rsrc_lock); /* Check if the request can be accommodated */ err = rvu_check_rsrc_availability(rvu, attach, pcifunc); if (err) - goto exit; + goto fail1; /* Now attach the requested resources */ - if (attach->npalf) - rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1, attach); + if (attach->npalf) { + err = rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1, attach); + if (err) + goto fail1; + } - if (attach->nixlf) - rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach); + if (attach->nixlf) { + err = rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach); + if (err) + goto fail2; + } if (attach->sso) { /* RVU func doesn't know which exact LF or slot is attached @@ -1749,33 +1760,64 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, */ if (attach->modify) rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO); - rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, - attach->sso, attach); + err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, + attach->sso, attach); + if (err) + goto fail3; } if (attach->ssow) { if (attach->modify) rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW); - rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, - attach->ssow, attach); + err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, + attach->ssow, attach); + if (err) + goto fail4; } if (attach->timlfs) { if (attach->modify) rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM); - rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, - attach->timlfs, attach); + err = rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, + attach->timlfs, attach); + if (err) + goto fail5; } if (attach->cptlfs) { if (attach->modify && rvu_attach_from_same_block(rvu, BLKTYPE_CPT, attach)) rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT); - rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, - attach->cptlfs, attach); + err = rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, + attach->cptlfs, attach); + if (err) + goto fail6; } -exit: + mutex_unlock(&rvu->rsrc_lock); + return 0; + +fail6: + if (attach->timlfs) + rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM); + +fail5: + if (attach->ssow) + rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW); + +fail4: + if (attach->sso) + rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO); + +fail3: + if (attach->nixlf) + rvu_detach_block(rvu, pcifunc, BLKTYPE_NIX); + +fail2: + if (attach->npalf) + rvu_detach_block(rvu, pcifunc, BLKTYPE_NPA); + +fail1: mutex_unlock(&rvu->rsrc_lock); return err; } @@ -3590,11 +3632,22 @@ static void rvu_remove(struct pci_dev *pdev) devm_kfree(&pdev->dev, rvu); } +static void rvu_shutdown(struct pci_dev *pdev) +{ + struct rvu *rvu = pci_get_drvdata(pdev); + + if (!rvu) + return; + + rvu_clear_rvum_blk_revid(rvu); +} + static struct pci_driver rvu_driver = { .name = DRV_NAME, .id_table = rvu_id_table, .probe = rvu_probe, .remove = rvu_remove, + .shutdown = rvu_shutdown, }; static int __init rvu_init_module(void) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c index 3abd750a4bd74..3d91a34f8b57b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c @@ -1222,6 +1222,9 @@ int rvu_mbox_handler_cgx_set_link_mode(struct rvu *rvu, u8 cgx_idx, lmac; void *cgxd; + if (!rvu->fwdata) + return LMAC_AF_ERR_FIRMWARE_DATA_NOT_MAPPED; + if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) return -EPERM; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index 3735372539bd9..5852a72b22306 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -327,10 +327,10 @@ static int rvu_nix_report_show(struct devlink_fmsg *fmsg, void *ctx, rvu_report_pair_end(fmsg); break; case NIX_AF_RVU_RAS: - intr_val = nix_event_context->nix_af_rvu_err; + intr_val = nix_event_context->nix_af_rvu_ras; rvu_report_pair_start(fmsg, "NIX_AF_RAS"); devlink_fmsg_u64_pair_put(fmsg, "\tNIX RAS Interrupt Reg ", - nix_event_context->nix_af_rvu_err); + nix_event_context->nix_af_rvu_ras); devlink_fmsg_string_put(fmsg, "\n\tPoison Data on:"); if (intr_val & BIT_ULL(34)) devlink_fmsg_string_put(fmsg, "\n\tNIX_AQ_INST_S"); @@ -475,7 +475,7 @@ static int rvu_hw_nix_ras_recover(struct devlink_health_reporter *reporter, if (blkaddr < 0) return blkaddr; - if (nix_event_ctx->nix_af_rvu_int) + if (nix_event_ctx->nix_af_rvu_ras) rvu_write64(rvu, blkaddr, NIX_AF_RAS_ENA_W1S, ~0ULL); return 0; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 828316211b245..c9c65ac69ead8 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -4932,12 +4932,18 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) /* Set chan/link to backpressure TL3 instead of TL2 */ rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01); - /* Disable SQ manager's sticky mode operation (set TM6 = 0) + /* Disable SQ manager's sticky mode operation (set TM6 = 0, TM11 = 0) * This sticky mode is known to cause SQ stalls when multiple - * SQs are mapped to same SMQ and transmitting pkts at a time. + * SQs are mapped to same SMQ and transmitting pkts simultaneously. + * NIX PSE may deadlock when there are any sticky to non-sticky + * transmission. Hence disable it (TM5 = 0). */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS); - cfg &= ~BIT_ULL(15); + cfg &= ~(BIT_ULL(15) | BIT_ULL(14) | BIT_ULL(23)); + /* NIX may drop credits when condition clocks are turned off. + * Hence enable control flow clk (set TM9 = 1). + */ + cfg |= BIT_ULL(21); rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg); ltdefs = rvu->kpu.lt_def; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index c7c70429eb6c1..8658cb2143dfc 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -1042,32 +1042,35 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); - /* update the VF flow rule action with the VF default entry action */ - if (mcam_index < 0) - npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, - *(u64 *)&action); - /* update the action change in default rule */ pfvf = rvu_get_pfvf(rvu, pcifunc); if (pfvf->def_ucast_rule) pfvf->def_ucast_rule->rx_action = action; - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_PROMISC_ENTRY); + if (mcam_index < 0) { + /* update the VF flow rule action with the VF default + * entry action + */ + npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, + *(u64 *)&action); - /* If PF's promiscuous entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_PROMISC_ENTRY); - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_ALLMULTI_ENTRY); - /* If PF's allmulti entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + /* If PF's promiscuous entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_ALLMULTI_ENTRY); + /* If PF's allmulti entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + } } void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_sdp.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_sdp.c index e4a5f9fa6fd46..bbfd8231aed5c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_sdp.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_sdp.c @@ -56,7 +56,7 @@ int rvu_sdp_init(struct rvu *rvu) struct rvu_pfvf *pfvf; u32 i = 0; - if (rvu->fwdata->channel_data.valid) { + if (rvu->fwdata && rvu->fwdata->channel_data.valid) { sdp_pf_num[0] = 0; pfvf = &rvu->pf[sdp_pf_num[0]]; pfvf->sdp_info = &rvu->fwdata->channel_data.info; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c index 4c7e0f345cb5b..060c715ebad0a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c @@ -328,7 +328,7 @@ static int cn10k_mcs_write_rx_flowid(struct otx2_nic *pfvf, req->data[0] = FIELD_PREP(MCS_TCAM0_MAC_DA_MASK, mac_da); req->mask[0] = ~0ULL; - req->mask[0] = ~MCS_TCAM0_MAC_DA_MASK; + req->mask[0] &= ~MCS_TCAM0_MAC_DA_MASK; req->data[1] = FIELD_PREP(MCS_TCAM1_ETYPE_MASK, ETH_P_MACSEC); req->mask[1] = ~0ULL; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 1c8a3c078a647..844ad55dd34b8 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -934,13 +934,8 @@ static inline dma_addr_t otx2_dma_map_page(struct otx2_nic *pfvf, size_t offset, size_t size, enum dma_data_direction dir) { - dma_addr_t iova; - - iova = dma_map_page_attrs(pfvf->dev, page, + return dma_map_page_attrs(pfvf->dev, page, offset, size, dir, DMA_ATTR_SKIP_CPU_SYNC); - if (unlikely(dma_mapping_error(pfvf->dev, iova))) - return (dma_addr_t)NULL; - return iova; } static inline void otx2_dma_unmap_page(struct otx2_nic *pfvf, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index b90e23dc49de9..b6449f0a9e7dd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -418,6 +418,14 @@ static int otx2_set_ringparam(struct net_device *netdev, */ if (rx_count < pfvf->hw.rq_skid) rx_count = pfvf->hw.rq_skid; + + if (ring->rx_pending < 16) { + netdev_err(netdev, + "rx ring size %u invalid, min is 16\n", + ring->rx_pending); + return -EINVAL; + } + rx_count = Q_COUNT(Q_SIZE(rx_count, 3)); /* Due pipelining impact minimum 2000 unused SQ CQE's diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index e808995703cfd..bbf25769f4994 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -3255,7 +3255,9 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->watchdog_timeo = OTX2_TX_TIMEOUT; netdev->netdev_ops = &otx2_netdev_ops; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | + NETDEV_XDP_ACT_NDO_XMIT | + NETDEV_XDP_ACT_XSK_ZEROCOPY; netdev->min_mtu = OTX2_MIN_MTU; netdev->max_mtu = otx2_get_max_mtu(pf); @@ -3319,6 +3321,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_sriov_cleannup: otx2_sriov_vfcfg_cleanup(pf); err_pf_sriov_init: + otx2_unregister_dl(pf); otx2_shutdown_tc(pf); err_mcam_flow_del: otx2_mcam_flow_del(pf); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_devlink.c b/drivers/net/ethernet/marvell/prestera/prestera_devlink.c index 2a4c9df4eb797..e63d95c1842f3 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_devlink.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_devlink.c @@ -387,6 +387,8 @@ struct prestera_switch *prestera_devlink_alloc(struct prestera_device *dev) dl = devlink_alloc(&prestera_dl_ops, sizeof(struct prestera_switch), dev->dev); + if (!dl) + return NULL; return devlink_priv(dl); } diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index 05349a0b2db1c..cf4e26d337bb5 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -78,7 +78,6 @@ static const struct pci_device_id skge_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, 0x4320) }, /* SK-98xx V2.0 */ { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4b01) }, /* D-Link DGE-530T (rev.B) */ { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4c00) }, /* D-Link DGE-530T */ - { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4302) }, /* D-Link DGE-530T Rev C1 */ { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0x4320) }, /* Marvell Yukon 88E8001/8003/8010 */ { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0x5005) }, /* Belkin */ { PCI_DEVICE(PCI_VENDOR_ID_CNET, 0x434E) }, /* CNet PowerG-2000 */ diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index e68997a29191b..0f676bd72832b 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -3567,12 +3567,23 @@ static int mtk_device_event(struct notifier_block *n, unsigned long event, void return NOTIFY_DONE; } +static int mtk_max_gmac_mtu(struct mtk_eth *eth) +{ + int i, max_mtu = ETH_DATA_LEN; + + for (i = 0; i < ARRAY_SIZE(eth->netdev); i++) + if (eth->netdev[i] && eth->netdev[i]->mtu > max_mtu) + max_mtu = eth->netdev[i]->mtu; + + return max_mtu; +} + static int mtk_open(struct net_device *dev) { struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; struct mtk_mac *target_mac; - int i, err, ppe_num; + int i, err, ppe_num, mtu; ppe_num = eth->soc->ppe_num; @@ -3619,6 +3630,10 @@ static int mtk_open(struct net_device *dev) mtk_gdm_config(eth, target_mac->id, gdm_config); } + mtu = mtk_max_gmac_mtu(eth); + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_update_mtu(eth->ppe[i], mtu); + napi_enable(ð->tx_napi); napi_enable(ð->rx_napi); mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); @@ -3749,12 +3764,21 @@ static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog, mtk_stop(dev); old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held()); + + if (netif_running(dev) && need_update) { + int err; + + err = mtk_open(dev); + if (err) { + rcu_assign_pointer(eth->prog, old_prog); + + return err; + } + } + if (old_prog) bpf_prog_put(old_prog); - if (netif_running(dev) && need_update) - return mtk_open(dev); - return 0; } @@ -4303,6 +4327,7 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu) int length = new_mtu + MTK_RX_ETH_HLEN; struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; + int max_mtu, i; if (rcu_access_pointer(eth->prog) && length > MTK_PP_MAX_BUF_SIZE) { @@ -4313,6 +4338,10 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu) mtk_set_mcr_max_rx(mac, length); WRITE_ONCE(dev->mtu, new_mtu); + max_mtu = mtk_max_gmac_mtu(eth); + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_update_mtu(eth->ppe[i], max_mtu); + return 0; } diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index ada852adc5f70..fa688a42a22f5 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -973,6 +973,36 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) } } +void mtk_ppe_update_mtu(struct mtk_ppe *ppe, int mtu) +{ + int base; + u32 val; + + if (!ppe) + return; + + /* The PPE checks output frame size against per-tag-layer MTU limits, + * treating PPPoE and DSA tags just like 802.1Q VLAN tags. The Linux + * device MTU already accounts for PPPoE (PPPOE_SES_HLEN) and DSA tag + * overhead, but 802.1Q VLAN tags are handled transparently without + * being reflected by the lower device MTU being increased by 4. + * Use the maximum MTU across all GMAC interfaces so that PPE output + * frame limits are sufficiently high regardless of which port a flow + * egresses through. + */ + base = ETH_HLEN + mtu; + + val = FIELD_PREP(MTK_PPE_VLAN_MTU0_NONE, base) | + FIELD_PREP(MTK_PPE_VLAN_MTU0_1TAG, base + VLAN_HLEN); + ppe_w32(ppe, MTK_PPE_VLAN_MTU0, val); + + val = FIELD_PREP(MTK_PPE_VLAN_MTU1_2TAG, + base + 2 * VLAN_HLEN) | + FIELD_PREP(MTK_PPE_VLAN_MTU1_3TAG, + base + 3 * VLAN_HLEN); + ppe_w32(ppe, MTK_PPE_VLAN_MTU1, val); +} + void mtk_ppe_start(struct mtk_ppe *ppe) { u32 val; diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h index 223f709e2704f..ba85e39a155bf 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe.h @@ -346,6 +346,7 @@ struct mtk_ppe { struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index); void mtk_ppe_deinit(struct mtk_eth *eth); +void mtk_ppe_update_mtu(struct mtk_ppe *ppe, int mtu); void mtk_ppe_start(struct mtk_ppe *ppe); int mtk_ppe_stop(struct mtk_ppe *ppe); int mtk_ppe_prepare_reset(struct mtk_ppe *ppe); diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index e9bd327419837..4894d4f187f70 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -244,6 +244,25 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe, return 0; } +static bool +mtk_flow_is_valid_idev(const struct mtk_eth *eth, const struct net_device *idev) +{ + size_t i; + + if (!idev) + return false; + + for (i = 0; i < ARRAY_SIZE(eth->netdev); i++) { + if (!eth->netdev[i]) + continue; + + if (idev->netdev_ops == eth->netdev[i]->netdev_ops) + return true; + } + + return false; +} + static int mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f, int ppe_index) @@ -270,7 +289,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f, flow_rule_match_meta(rule, &match); if (mtk_is_netsys_v2_or_greater(eth)) { idev = __dev_get_by_index(&init_net, match.key->ingress_ifindex); - if (idev && idev->netdev_ops == eth->netdev[0]->netdev_ops) { + if (mtk_flow_is_valid_idev(eth, idev)) { struct mtk_mac *mac = netdev_priv(idev); if (WARN_ON(mac->ppe_idx >= eth->soc->ppe_num)) diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 03d2fc7d9b09f..d1fbf37bdaf79 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -4366,7 +4366,6 @@ static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); return PCI_ERS_RESULT_RECOVERED; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c index 36806e813c33c..1301c56e20d65 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c @@ -613,3 +613,19 @@ void mlx5_debug_cq_remove(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) cq->dbg = NULL; } } + +static int vhca_id_show(struct seq_file *file, void *priv) +{ + struct mlx5_core_dev *dev = file->private; + + seq_printf(file, "0x%x\n", MLX5_CAP_GEN(dev, vhca_id)); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(vhca_id); + +void mlx5_vhca_debugfs_init(struct mlx5_core_dev *dev) +{ + debugfs_create_file("vhca_id", 0400, dev->priv.dbg.dbg_root, dev, + &vhca_id_fops); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 887adf4807d16..055ee020c56f4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -107,9 +107,7 @@ mlx5_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, if (err) return err; - err = mlx5_fw_version_query(dev, &running_fw, &stored_fw); - if (err) - return err; + mlx5_fw_version_query(dev, &running_fw, &stored_fw); snprintf(version_str, sizeof(version_str), "%d.%d.%04d", mlx5_fw_ver_major(running_fw), mlx5_fw_ver_minor(running_fw), @@ -197,6 +195,11 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, struct pci_dev *pdev = dev->pdev; int ret = 0; + if (mlx5_fw_reset_in_progress(dev)) { + NL_SET_ERR_MSG_MOD(extack, "Can't reload during firmware reset"); + return -EBUSY; + } + if (mlx5_dev_is_lightweight(dev)) { if (action != DEVLINK_RELOAD_ACTION_DRIVER_REINIT) return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c index 080e7eab52c7e..0b82a6a133d6c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c @@ -33,6 +33,7 @@ #include "lib/eq.h" #include "fw_tracer.h" #include "fw_tracer_tracepoint.h" +#include static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer) { @@ -358,6 +359,47 @@ static const char *VAL_PARM = "%llx"; static const char *REPLACE_64_VAL_PARM = "%x%x"; static const char *PARAM_CHAR = "%"; +static bool mlx5_is_valid_spec(const char *str) +{ + /* Parse format specifiers to find the actual type. + * Structure: %[flags][width][.precision][length]type + * Skip flags, width, precision & length. + */ + while (isdigit(*str) || *str == '#' || *str == '.' || *str == 'l') + str++; + + /* Check if it's a valid integer/hex specifier or %%: + * Valid formats: %x, %d, %i, %u, etc. + */ + if (*str != 'x' && *str != 'X' && *str != 'd' && *str != 'i' && + *str != 'u' && *str != 'c' && *str != '%') + return false; + + return true; +} + +static bool mlx5_tracer_validate_params(const char *str) +{ + const char *substr = str; + + if (!str) + return false; + + substr = strstr(substr, PARAM_CHAR); + while (substr) { + if (!mlx5_is_valid_spec(substr + 1)) + return false; + + if (*(substr + 1) == '%') + substr = strstr(substr + 2, PARAM_CHAR); + else + substr = strstr(substr + 1, PARAM_CHAR); + + } + + return true; +} + static int mlx5_tracer_message_hash(u32 message_id) { return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1); @@ -419,6 +461,10 @@ static int mlx5_tracer_get_num_of_params(char *str) char *substr, *pstr = str; int num_of_params = 0; + /* Validate that all parameters are valid before processing */ + if (!mlx5_tracer_validate_params(str)) + return -EINVAL; + /* replace %llx with %x%x */ substr = strstr(pstr, VAL_PARM); while (substr) { @@ -427,11 +473,15 @@ static int mlx5_tracer_get_num_of_params(char *str) substr = strstr(pstr, VAL_PARM); } - /* count all the % characters */ + /* count all the % characters, but skip %% (escaped percent) */ substr = strstr(str, PARAM_CHAR); while (substr) { - num_of_params += 1; - str = substr + 1; + if (*(substr + 1) != '%') { + num_of_params += 1; + str = substr + 1; + } else { + str = substr + 2; + } substr = strstr(str, PARAM_CHAR); } @@ -570,14 +620,17 @@ void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt, { char tmp[512]; - snprintf(tmp, sizeof(tmp), str_frmt->string, - str_frmt->params[0], - str_frmt->params[1], - str_frmt->params[2], - str_frmt->params[3], - str_frmt->params[4], - str_frmt->params[5], - str_frmt->params[6]); + if (str_frmt->invalid_string) + snprintf(tmp, sizeof(tmp), "BAD_FORMAT: %s", str_frmt->string); + else + snprintf(tmp, sizeof(tmp), str_frmt->string, + str_frmt->params[0], + str_frmt->params[1], + str_frmt->params[2], + str_frmt->params[3], + str_frmt->params[4], + str_frmt->params[5], + str_frmt->params[6]); trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost, str_frmt->event_id, tmp); @@ -609,6 +662,13 @@ static int mlx5_tracer_handle_raw_string(struct mlx5_fw_tracer *tracer, return 0; } +static void mlx5_tracer_handle_bad_format_string(struct mlx5_fw_tracer *tracer, + struct tracer_string_format *cur_string) +{ + cur_string->invalid_string = true; + list_add_tail(&cur_string->list, &tracer->ready_strings_list); +} + static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, struct tracer_event *tracer_event) { @@ -619,12 +679,18 @@ static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, if (!cur_string) return mlx5_tracer_handle_raw_string(tracer, tracer_event); - cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string); - cur_string->last_param_num = 0; cur_string->event_id = tracer_event->event_id; cur_string->tmsn = tracer_event->string_event.tmsn; cur_string->timestamp = tracer_event->string_event.timestamp; cur_string->lost = tracer_event->lost_event; + cur_string->last_param_num = 0; + cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string); + if (cur_string->num_of_params < 0) { + pr_debug("%s Invalid format string parameters\n", + __func__); + mlx5_tracer_handle_bad_format_string(tracer, cur_string); + return 0; + } if (cur_string->num_of_params == 0) /* trace with no params */ list_add_tail(&cur_string->list, &tracer->ready_strings_list); } else { @@ -634,6 +700,11 @@ static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, __func__, tracer_event->string_event.tmsn); return mlx5_tracer_handle_raw_string(tracer, tracer_event); } + if (cur_string->num_of_params < 0) { + pr_debug("%s string parameter of invalid string, dumping\n", + __func__); + return 0; + } cur_string->last_param_num += 1; if (cur_string->last_param_num > TRACER_MAX_PARAMS) { pr_debug("%s Number of params exceeds the max (%d)\n", diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h index 5c548bb74f07b..30d0bcba88479 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h @@ -125,6 +125,7 @@ struct tracer_string_format { struct list_head list; u32 timestamp; bool lost; + bool invalid_string; }; enum mlx5_fw_tracer_ownership_state { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index a163f81f07c13..5bced924a24fb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -69,7 +69,7 @@ struct page_pool; #define MLX5E_METADATA_ETHER_TYPE (0x8CE4) #define MLX5E_METADATA_ETHER_LEN 8 -#define MLX5E_ETH_HARD_MTU (ETH_HLEN + PSP_ENCAP_HLEN + PSP_TRL_SIZE + VLAN_HLEN + ETH_FCS_LEN) +#define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN) #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu)) #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu)) @@ -179,7 +179,8 @@ static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size) } /* Use this function to get max num channels (rxqs/txqs) only to create netdev */ -static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) +static inline unsigned int +mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) { return is_kdump_kernel() ? MLX5E_MIN_NUM_CHANNELS : @@ -963,7 +964,7 @@ struct mlx5e_priv { }; struct mlx5e_dev { - struct mlx5e_priv *priv; + struct net_device *netdev; struct devlink_port dl_port; }; @@ -1238,10 +1239,13 @@ struct net_device * mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile); int mlx5e_attach_netdev(struct mlx5e_priv *priv); void mlx5e_detach_netdev(struct mlx5e_priv *priv); -void mlx5e_destroy_netdev(struct mlx5e_priv *priv); -int mlx5e_netdev_change_profile(struct mlx5e_priv *priv, - const struct mlx5e_profile *new_profile, void *new_ppriv); -void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv); +void mlx5e_destroy_netdev(struct net_device *netdev); +int mlx5e_netdev_change_profile(struct net_device *netdev, + struct mlx5_core_dev *mdev, + const struct mlx5e_profile *new_profile, + void *new_ppriv); +void mlx5e_netdev_attach_nic_profile(struct net_device *netdev, + struct mlx5_core_dev *mdev); void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv); void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index c93ee969ea647..ec715b158a342 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -448,22 +448,8 @@ static void mlx5e_ptpsq_unhealthy_work(struct work_struct *work) { struct mlx5e_ptpsq *ptpsq = container_of(work, struct mlx5e_ptpsq, report_unhealthy_work); - struct mlx5e_txqsq *sq = &ptpsq->txqsq; - - /* Recovering the PTP SQ means re-enabling NAPI, which requires the - * netdev instance lock. However, SQ closing has to wait for this work - * task to finish while also holding the same lock. So either get the - * lock or find that the SQ is no longer enabled and thus this work is - * not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } mlx5e_reporter_tx_ptpsq_unhealthy(ptpsq); - netdev_unlock(sq->netdev); } static int mlx5e_ptp_open_txqsq(struct mlx5e_ptp *c, u32 tisn, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index b1415992ffa24..a09a7c05820d6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2019 Mellanox Technologies. +#include + #include "health.h" #include "params.h" #include "txrx.h" @@ -177,6 +179,16 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) rq = ctx; priv = rq->priv; + /* Acquire netdev instance lock to synchronize with channel close and + * reopen flows. Either successfully obtain the lock, or detect that + * channels are closing for another reason, making this work no longer + * necessary. + */ + while (!netdev_trylock(rq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); eq = rq->cq.mcq.eq; @@ -186,6 +198,7 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) clear_bit(MLX5E_SQ_STATE_ENABLED, &rq->icosq->state); mutex_unlock(&priv->state_lock); + netdev_unlock(rq->netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 9e2cf191ed308..d6ace2b6fc1df 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2019 Mellanox Technologies. */ +#include + #include "health.h" #include "en/ptp.h" #include "en/devlink.h" @@ -44,7 +46,6 @@ static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq) "SQ 0x%x: cc (0x%x) != pc (0x%x)\n", sq->sqn, sq->cc, sq->pc); sq->cc = 0; - sq->dma_fifo_cc = 0; sq->pc = 0; } @@ -78,6 +79,18 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) return 0; + /* Recovering queues means re-enabling NAPI, which requires the netdev + * instance lock. However, SQ closing flows have to wait for work tasks + * to finish while also holding the netdev instance lock. So either get + * the lock or find that the SQ is no longer enabled and thus this work + * is not relevant anymore. + */ + while (!netdev_trylock(dev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) + return 0; + msleep(20); + } + err = mlx5_core_query_sq_state(mdev, sq->sqn, &state); if (err) { netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n", @@ -113,9 +126,11 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) else mlx5e_trigger_napi_sched(sq->cq.napi); + netdev_unlock(dev); return 0; out: clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); + netdev_unlock(dev); return err; } @@ -136,10 +151,24 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) sq = to_ctx->sq; eq = sq->cq.mcq.eq; priv = sq->priv; + + /* Recovering the TX queues implies re-enabling NAPI, which requires + * the netdev instance lock. + * However, channel closing flows have to wait for this work to finish + * while holding the same lock. So either get the lock or find that + * channels are being closed for other reason and this work is not + * relevant anymore. + */ + while (!netdev_trylock(sq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) + return 0; + msleep(20); + } + err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats); if (!err) { to_ctx->status = 0; /* this sq recovered */ - return err; + goto out; } mutex_lock(&priv->state_lock); @@ -147,7 +176,7 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) mutex_unlock(&priv->state_lock); if (!err) { to_ctx->status = 1; /* all channels recovered */ - return err; + goto out; } to_ctx->status = err; @@ -155,7 +184,8 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) netdev_err(priv->netdev, "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", err); - +out: + netdev_unlock(sq->netdev); return err; } @@ -172,10 +202,22 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) return 0; priv = ptpsq->txqsq.priv; + netdev = priv->netdev; + + /* Recovering the PTP SQ means re-enabling NAPI, which requires the + * netdev instance lock. However, SQ closing has to wait for this work + * task to finish while also holding the same lock. So either get the + * lock or find that the SQ is no longer enabled and thus this work is + * not relevant anymore. + */ + while (!netdev_trylock(netdev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &ptpsq->txqsq.state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); chs = &priv->channels; - netdev = priv->netdev; carrier_ok = netif_carrier_ok(netdev); netif_carrier_off(netdev); @@ -192,6 +234,7 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) netif_carrier_on(netdev); mutex_unlock(&priv->state_lock); + netdev_unlock(netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 35d9530037a65..f03507a522b4f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -259,7 +259,6 @@ static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_accel_esp_xfrm_attrs *attrs) { - struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec_addr *addrs = &attrs->addrs; struct net_device *netdev = sa_entry->dev; struct xfrm_state *x = sa_entry->x; @@ -276,7 +275,7 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->type != XFRM_DEV_OFFLOAD_PACKET) return; - mlx5_query_mac_address(mdev, addr); + ether_addr_copy(addr, netdev->dev_addr); switch (attrs->dir) { case XFRM_DEV_OFFLOAD_IN: src = attrs->dmac; @@ -342,9 +341,8 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, rt_dst_entry = &rt->dst; break; case AF_INET6: - rt_dst_entry = ipv6_stub->ipv6_dst_lookup_flow( - dev_net(netdev), NULL, &fl6, NULL); - if (IS_ERR(rt_dst_entry)) + if (!IS_ENABLED(CONFIG_IPV6) || + ip6_dst_lookup(dev_net(netdev), NULL, &rt_dst_entry, &fl6)) goto neigh; break; default: @@ -359,6 +357,9 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, neigh_ha_snapshot(addr, n, netdev); ether_addr_copy(dst, addr); + if (attrs->dir == XFRM_DEV_OFFLOAD_OUT && + is_zero_ether_addr(addr)) + neigh_event_send(n, NULL); dst_release(rt_dst_entry); neigh_release(n); return; @@ -428,7 +429,8 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->replay_esn.esn = sa_entry->esn_state.esn; attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb; attrs->replay_esn.overlap = sa_entry->esn_state.overlap; - if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) + if (attrs->dir == XFRM_DEV_OFFLOAD_OUT || + x->xso.type != XFRM_DEV_OFFLOAD_PACKET) goto skip_replay_window; switch (x->replay_esn->replay_window) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index f8eaaf37963b1..abcbd38db9dbb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -287,6 +287,7 @@ struct mlx5e_ipsec_sa_entry { struct mlx5e_ipsec_dwork *dwork; struct mlx5e_ipsec_limits limits; u32 rx_mapped_id; + u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; }; struct mlx5_accel_pol_xfrm_attrs { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index feef86fff4bfd..91cfabc450325 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -2912,7 +2912,7 @@ void mlx5e_ipsec_disable_events(struct mlx5e_priv *priv) goto out; peer_priv = mlx5_devcom_get_next_peer_data(priv->devcom, &tmp); - if (peer_priv) + if (peer_priv && peer_priv->ipsec) complete_all(&peer_priv->ipsec->comp); mlx5_devcom_for_each_peer_end(priv->devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index ef7322d381af6..e0611fa827971 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -310,10 +310,11 @@ static void mlx5e_ipsec_aso_update(struct mlx5e_ipsec_sa_entry *sa_entry, mlx5e_ipsec_aso_query(sa_entry, data); } -static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, - u32 mode_param) +static void +mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, + u32 mode_param, + struct mlx5_accel_esp_xfrm_attrs *attrs) { - struct mlx5_accel_esp_xfrm_attrs attrs = {}; struct mlx5_wqe_aso_ctrl_seg data = {}; if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) { @@ -323,18 +324,7 @@ static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, sa_entry->esn_state.overlap = 1; } - mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs); - - /* It is safe to execute the modify below unlocked since the only flows - * that could affect this HW object, are create, destroy and this work. - * - * Creation flow can't co-exist with this modify work, the destruction - * flow would cancel this work, and this work is a single entity that - * can't conflict with it self. - */ - spin_unlock_bh(&sa_entry->x->lock); - mlx5_accel_esp_modify_xfrm(sa_entry, &attrs); - spin_lock_bh(&sa_entry->x->lock); + mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, attrs); data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET; @@ -370,20 +360,18 @@ static void mlx5e_ipsec_aso_update_soft(struct mlx5e_ipsec_sa_entry *sa_entry, static void mlx5e_ipsec_handle_limits(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - struct mlx5e_ipsec_aso *aso = ipsec->aso; bool soft_arm, hard_arm; u64 hard_cnt; lockdep_assert_held(&sa_entry->x->lock); - soft_arm = !MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm); - hard_arm = !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm); + soft_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, soft_lft_arm); + hard_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, hard_lft_arm); if (!soft_arm && !hard_arm) /* It is not lifetime event */ return; - hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt); + hard_cnt = MLX5_GET(ipsec_aso, sa_entry->ctx, remove_flow_pkt_cnt); if (!hard_cnt || hard_arm) { /* It is possible to see packet counter equal to zero without * hard limit event armed. Such situation can be if packet @@ -453,11 +441,11 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) struct mlx5e_ipsec_work *work = container_of(_work, struct mlx5e_ipsec_work, work); struct mlx5e_ipsec_sa_entry *sa_entry = work->data; + struct mlx5_accel_esp_xfrm_attrs tmp = {}; struct mlx5_accel_esp_xfrm_attrs *attrs; - struct mlx5e_ipsec_aso *aso; + bool need_modify = false; int ret; - aso = sa_entry->ipsec->aso; attrs = &sa_entry->attrs; spin_lock_bh(&sa_entry->x->lock); @@ -465,18 +453,22 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) if (ret) goto unlock; + if (attrs->lft.soft_packet_limit != XFRM_INF) + mlx5e_ipsec_handle_limits(sa_entry); + if (attrs->replay_esn.trigger && - !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) { - u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter); + !MLX5_GET(ipsec_aso, sa_entry->ctx, esn_event_arm)) { + u32 mode_param = MLX5_GET(ipsec_aso, sa_entry->ctx, + mode_parameter); - mlx5e_ipsec_update_esn_state(sa_entry, mode_param); + mlx5e_ipsec_update_esn_state(sa_entry, mode_param, &tmp); + need_modify = true; } - if (attrs->lft.soft_packet_limit != XFRM_INF) - mlx5e_ipsec_handle_limits(sa_entry); - unlock: spin_unlock_bh(&sa_entry->x->lock); + if (need_modify) + mlx5_accel_esp_modify_xfrm(sa_entry, &tmp); kfree(work); } @@ -629,6 +621,8 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, /* We are in atomic context */ udelay(10); } while (ret && time_is_after_jiffies(expires)); + if (!ret) + memcpy(sa_entry->ctx, aso->ctx, MLX5_ST_SZ_BYTES(ipsec_aso)); spin_unlock_bh(&aso->lock); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c index 8565cfe8d7dce..943d6fc6e7a04 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c @@ -41,6 +41,7 @@ struct mlx5e_accel_fs_psp_prot { struct mlx5_flow_table *ft; struct mlx5_flow_group *miss_group; struct mlx5_flow_handle *miss_rule; + struct mlx5_modify_hdr *rx_modify_hdr; struct mlx5_flow_destination default_dest; struct mlx5e_psp_rx_err rx_err; u32 refcnt; @@ -217,13 +218,19 @@ static int accel_psp_fs_rx_err_create_ft(struct mlx5e_psp_fs *fs, return err; } -static void accel_psp_fs_rx_fs_destroy(struct mlx5e_accel_fs_psp_prot *fs_prot) +static void accel_psp_fs_rx_fs_destroy(struct mlx5e_psp_fs *fs, + struct mlx5e_accel_fs_psp_prot *fs_prot) { if (fs_prot->def_rule) { mlx5_del_flow_rules(fs_prot->def_rule); fs_prot->def_rule = NULL; } + if (fs_prot->rx_modify_hdr) { + mlx5_modify_header_dealloc(fs->mdev, fs_prot->rx_modify_hdr); + fs_prot->rx_modify_hdr = NULL; + } + if (fs_prot->miss_rule) { mlx5_del_flow_rules(fs_prot->miss_rule); fs_prot->miss_rule = NULL; @@ -327,6 +334,7 @@ static int accel_psp_fs_rx_create_ft(struct mlx5e_psp_fs *fs, modify_hdr = NULL; goto out_err; } + fs_prot->rx_modify_hdr = modify_hdr; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | @@ -347,7 +355,7 @@ static int accel_psp_fs_rx_create_ft(struct mlx5e_psp_fs *fs, goto out; out_err: - accel_psp_fs_rx_fs_destroy(fs_prot); + accel_psp_fs_rx_fs_destroy(fs, fs_prot); out: kvfree(flow_group_in); kvfree(spec); @@ -364,7 +372,7 @@ static int accel_psp_fs_rx_destroy(struct mlx5e_psp_fs *fs, enum accel_fs_psp_ty /* The netdev unreg already happened, so all offloaded rule are already removed */ fs_prot = &accel_psp->fs_prot[type]; - accel_psp_fs_rx_fs_destroy(fs_prot); + accel_psp_fs_rx_fs_destroy(fs, fs_prot); accel_psp_fs_rx_err_destroy_ft(fs, &fs_prot->rx_err); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c index 828bff1137aff..fa98d0074531b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c @@ -177,8 +177,6 @@ bool mlx5e_psp_handle_tx_skb(struct net_device *netdev, { struct mlx5e_priv *priv = netdev_priv(netdev); struct net *net = sock_net(skb->sk); - const struct ipv6hdr *ip6; - struct tcphdr *th; if (!mlx5e_psp_set_state(priv, skb, psp_st)) return true; @@ -189,11 +187,18 @@ bool mlx5e_psp_handle_tx_skb(struct net_device *netdev, return false; } if (skb_is_gso(skb)) { - ip6 = ipv6_hdr(skb); - th = inner_tcp_hdr(skb); + int len = skb_shinfo(skb)->gso_size + inner_tcp_hdrlen(skb); + struct tcphdr *th = inner_tcp_hdr(skb); - th->check = ~tcp_v6_check(skb_shinfo(skb)->gso_size + inner_tcp_hdrlen(skb), &ip6->saddr, - &ip6->daddr, 0); + if (skb->protocol == htons(ETH_P_IP)) { + const struct iphdr *ip = ip_hdr(skb); + + th->check = ~tcp_v4_check(len, ip->saddr, ip->daddr, 0); + } else { + const struct ipv6hdr *ip6 = ipv6_hdr(skb); + + th->check = ~tcp_v6_check(len, &ip6->saddr, &ip6->daddr, 0); + } } return true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 5e17eae81f4b3..cb993ad2d9ad9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -688,19 +688,7 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) struct mlx5e_rq, rx_timeout_work); - /* Acquire netdev instance lock to synchronize with channel close and - * reopen flows. Either successfully obtain the lock, or detect that - * channels are closing for another reason, making this work no longer - * necessary. - */ - while (!netdev_trylock(rq->netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) - return; - msleep(20); - } - mlx5e_reporter_rx_timeout(rq); - netdev_unlock(rq->netdev); } static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) @@ -1997,20 +1985,7 @@ void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, recover_work); - /* Recovering queues means re-enabling NAPI, which requires the netdev - * instance lock. However, SQ closing flows have to wait for work tasks - * to finish while also holding the netdev instance lock. So either get - * the lock or find that the SQ is no longer enabled and thus this work - * is not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } - mlx5e_reporter_tx_err_cqe(sq); - netdev_unlock(sq->netdev); } static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) @@ -4033,6 +4008,8 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) mlx5e_queue_update_stats(priv); } + netdev_stats_to_stats64(stats, &dev->stats); + if (mlx5e_is_uplink_rep(priv)) { struct mlx5e_vport_stats *vstats = &priv->stats.vport; @@ -4049,21 +4026,21 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) mlx5e_fold_sw_stats64(priv, stats); } - stats->rx_missed_errors = priv->stats.qcnt.rx_out_of_buffer; - stats->rx_dropped = PPORT_2863_GET(pstats, if_in_discards); + stats->rx_missed_errors += priv->stats.qcnt.rx_out_of_buffer; + stats->rx_dropped += PPORT_2863_GET(pstats, if_in_discards); - stats->rx_length_errors = + stats->rx_length_errors += PPORT_802_3_GET(pstats, a_in_range_length_errors) + PPORT_802_3_GET(pstats, a_out_of_range_length_field) + PPORT_802_3_GET(pstats, a_frame_too_long_errors) + VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small); - stats->rx_crc_errors = + stats->rx_crc_errors += PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); - stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors); - stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards); - stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors + - stats->rx_frame_errors; - stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; + stats->rx_frame_errors += PPORT_802_3_GET(pstats, a_alignment_errors); + stats->tx_aborted_errors += PPORT_2863_GET(pstats, if_out_discards); + stats->rx_errors += stats->rx_length_errors + stats->rx_crc_errors + + stats->rx_frame_errors; + stats->tx_errors += stats->tx_aborted_errors + stats->tx_carrier_errors; } static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) @@ -5100,19 +5077,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) struct net_device *netdev = priv->netdev; int i; - /* Recovering the TX queues implies re-enabling NAPI, which requires - * the netdev instance lock. - * However, channel closing flows have to wait for this work to finish - * while holding the same lock. So either get the lock or find that - * channels are being closed for other reason and this work is not - * relevant anymore. - */ - while (!netdev_trylock(netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) - return; - msleep(20); - } - for (i = 0; i < netdev->real_num_tx_queues; i++) { struct netdev_queue *dev_queue = netdev_get_tx_queue(netdev, i); @@ -5125,8 +5089,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) /* break if tried to reopened channels */ break; } - - netdev_unlock(netdev); } static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) @@ -6305,6 +6267,7 @@ int mlx5e_priv_init(struct mlx5e_priv *priv, void mlx5e_priv_cleanup(struct mlx5e_priv *priv) { + bool destroying = test_bit(MLX5E_STATE_DESTROYING, &priv->state); int i; /* bail if change profile failed and also rollback failed */ @@ -6332,6 +6295,8 @@ void mlx5e_priv_cleanup(struct mlx5e_priv *priv) } memset(priv, 0, sizeof(*priv)); + if (destroying) /* restore destroying bit, to allow unload */ + set_bit(MLX5E_STATE_DESTROYING, &priv->state); } static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev, @@ -6564,19 +6529,28 @@ mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mde return err; } -int mlx5e_netdev_change_profile(struct mlx5e_priv *priv, - const struct mlx5e_profile *new_profile, void *new_ppriv) +int mlx5e_netdev_change_profile(struct net_device *netdev, + struct mlx5_core_dev *mdev, + const struct mlx5e_profile *new_profile, + void *new_ppriv) { - const struct mlx5e_profile *orig_profile = priv->profile; - struct net_device *netdev = priv->netdev; - struct mlx5_core_dev *mdev = priv->mdev; - void *orig_ppriv = priv->ppriv; + struct mlx5e_priv *priv = netdev_priv(netdev); + const struct mlx5e_profile *orig_profile; int err, rollback_err; + void *orig_ppriv; - /* cleanup old profile */ - mlx5e_detach_netdev(priv); - priv->profile->cleanup(priv); - mlx5e_priv_cleanup(priv); + orig_profile = priv->profile; + orig_ppriv = priv->ppriv; + + /* NULL could happen if previous change_profile failed to rollback */ + if (priv->profile) { + WARN_ON_ONCE(priv->mdev != mdev); + /* cleanup old profile */ + mlx5e_detach_netdev(priv); + priv->profile->cleanup(priv); + mlx5e_priv_cleanup(priv); + } + /* priv members are not valid from this point ... */ if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); @@ -6593,23 +6567,33 @@ int mlx5e_netdev_change_profile(struct mlx5e_priv *priv, return 0; rollback: + if (!orig_profile) { + netdev_warn(netdev, "no original profile to rollback to\n"); + priv->profile = NULL; + return err; + } + rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv); - if (rollback_err) - netdev_err(netdev, "%s: failed to rollback to orig profile, %d\n", - __func__, rollback_err); + if (rollback_err) { + netdev_err(netdev, "failed to rollback to orig profile, %d\n", + rollback_err); + priv->profile = NULL; + } return err; } -void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv) +void mlx5e_netdev_attach_nic_profile(struct net_device *netdev, + struct mlx5_core_dev *mdev) { - mlx5e_netdev_change_profile(priv, &mlx5e_nic_profile, NULL); + mlx5e_netdev_change_profile(netdev, mdev, &mlx5e_nic_profile, NULL); } -void mlx5e_destroy_netdev(struct mlx5e_priv *priv) +void mlx5e_destroy_netdev(struct net_device *netdev) { - struct net_device *netdev = priv->netdev; + struct mlx5e_priv *priv = netdev_priv(netdev); - mlx5e_priv_cleanup(priv); + if (priv->profile) + mlx5e_priv_cleanup(priv); free_netdev(netdev); } @@ -6617,8 +6601,8 @@ static int _mlx5e_resume(struct auxiliary_device *adev) { struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); - struct mlx5e_priv *priv = mlx5e_dev->priv; - struct net_device *netdev = priv->netdev; + struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); + struct net_device *netdev = mlx5e_dev->netdev; struct mlx5_core_dev *mdev = edev->mdev; struct mlx5_core_dev *pos, *to; int err, i; @@ -6664,10 +6648,11 @@ static int mlx5e_resume(struct auxiliary_device *adev) static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg) { + struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); - struct mlx5e_priv *priv = mlx5e_dev->priv; - struct net_device *netdev = priv->netdev; - struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); + struct net_device *netdev = mlx5e_dev->netdev; + struct mlx5_core_dev *mdev = edev->mdev; struct mlx5_core_dev *pos; int i; @@ -6728,11 +6713,11 @@ static int _mlx5e_probe(struct auxiliary_device *adev) goto err_devlink_port_unregister; } SET_NETDEV_DEVLINK_PORT(netdev, &mlx5e_dev->dl_port); + mlx5e_dev->netdev = netdev; mlx5e_build_nic_netdev(netdev); priv = netdev_priv(netdev); - mlx5e_dev->priv = priv; priv->profile = profile; priv->ppriv = NULL; @@ -6765,7 +6750,7 @@ static int _mlx5e_probe(struct auxiliary_device *adev) err_profile_cleanup: profile->cleanup(priv); err_destroy_netdev: - mlx5e_destroy_netdev(priv); + mlx5e_destroy_netdev(netdev); err_devlink_port_unregister: mlx5e_devlink_port_unregister(mlx5e_dev); err_devlink_unregister: @@ -6795,18 +6780,20 @@ static void _mlx5e_remove(struct auxiliary_device *adev) { struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); - struct mlx5e_priv *priv = mlx5e_dev->priv; + struct net_device *netdev = mlx5e_dev->netdev; + struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5_core_dev *mdev = edev->mdev; mlx5_core_uplink_netdev_set(mdev, NULL); - mlx5e_dcbnl_delete_app(priv); + + if (priv->profile) + mlx5e_dcbnl_delete_app(priv); /* When unload driver, the netdev is in registered state * if it's from legacy mode. If from switchdev mode, it * is already unregistered before changing to NIC profile. */ - if (priv->netdev->reg_state == NETREG_REGISTERED) { - mlx5e_psp_unregister(priv); - unregister_netdev(priv->netdev); + if (netdev->reg_state == NETREG_REGISTERED) { + unregister_netdev(netdev); _mlx5e_suspend(adev, false); } else { struct mlx5_core_dev *pos; @@ -6821,7 +6808,7 @@ static void _mlx5e_remove(struct auxiliary_device *adev) /* Avoid cleanup if profile rollback failed. */ if (priv->profile) priv->profile->cleanup(priv); - mlx5e_destroy_netdev(priv); + mlx5e_destroy_netdev(netdev); mlx5e_devlink_port_unregister(mlx5e_dev); mlx5e_destroy_devlink(mlx5e_dev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 0335ca8277efa..8b65441246244 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -1508,17 +1508,16 @@ mlx5e_vport_uplink_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep * { struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); struct net_device *netdev; - struct mlx5e_priv *priv; int err; netdev = mlx5_uplink_netdev_get(dev); if (!netdev) return 0; - priv = netdev_priv(netdev); - rpriv->netdev = priv->netdev; - err = mlx5e_netdev_change_profile(priv, &mlx5e_uplink_rep_profile, - rpriv); + /* must not use netdev_priv(netdev), it might not be initialized yet */ + rpriv->netdev = netdev; + err = mlx5e_netdev_change_profile(netdev, dev, + &mlx5e_uplink_rep_profile, rpriv); mlx5_uplink_netdev_put(dev, netdev); return err; } @@ -1546,7 +1545,7 @@ mlx5e_vport_uplink_rep_unload(struct mlx5e_rep_priv *rpriv) if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_SWITCH_LEGACY)) unregister_netdev(netdev); - mlx5e_netdev_attach_nic_profile(priv); + mlx5e_netdev_attach_nic_profile(netdev, priv->mdev); } static int @@ -1612,7 +1611,7 @@ mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) priv->profile->cleanup(priv); err_destroy_netdev: - mlx5e_destroy_netdev(netdev_priv(netdev)); + mlx5e_destroy_netdev(netdev); return err; } @@ -1667,7 +1666,7 @@ mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) mlx5e_rep_vnic_reporter_destroy(priv); mlx5e_detach_netdev(priv); priv->profile->cleanup(priv); - mlx5e_destroy_netdev(priv); + mlx5e_destroy_netdev(netdev); free_ppriv: kvfree(ppriv); /* mlx5e_rep_priv */ } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 687cf123211d7..2ffa4e6b8c373 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1759,6 +1759,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi struct skb_shared_info *sinfo; u32 frag_consumed_bytes; struct bpf_prog *prog; + u8 nr_frags_free = 0; struct sk_buff *skb; dma_addr_t addr; u32 truesize; @@ -1801,15 +1802,13 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi prog = rcu_dereference(rq->xdp_prog); if (prog) { - u8 nr_frags_free, old_nr_frags = sinfo->nr_frags; + u8 old_nr_frags = sinfo->nr_frags; if (mlx5e_xdp_handle(rq, prog, mxbuf)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { struct mlx5e_wqe_frag_info *pwi; - wi -= old_nr_frags - sinfo->nr_frags; - for (pwi = head_wi; pwi < wi; pwi++) pwi->frag_page->frags++; } @@ -1817,10 +1816,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi } nr_frags_free = old_nr_frags - sinfo->nr_frags; - if (unlikely(nr_frags_free)) { - wi -= nr_frags_free; + if (unlikely(nr_frags_free)) truesize -= nr_frags_free * frag_info->frag_stride; - } } skb = mlx5e_build_linear_skb( @@ -1836,7 +1833,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi if (xdp_buff_has_frags(&mxbuf->xdp)) { /* sinfo->nr_frags is reset by build_skb, calculate again. */ - xdp_update_skb_frags_info(skb, wi - head_wi - 1, + xdp_update_skb_frags_info(skb, wi - head_wi - nr_frags_free - 1, sinfo->xdp_frags_size, truesize, xdp_buff_get_skb_flags(&mxbuf->xdp)); @@ -2118,14 +2115,13 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w if (prog) { u8 nr_frags_free, old_nr_frags = sinfo->nr_frags; + u8 new_nr_frags; u32 len; if (mlx5e_xdp_handle(rq, prog, mxbuf)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { struct mlx5e_frag_page *pfp; - frag_page -= old_nr_frags - sinfo->nr_frags; - for (pfp = head_page; pfp < frag_page; pfp++) pfp->frags++; @@ -2136,13 +2132,12 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w return NULL; /* page/packet was consumed by XDP */ } - nr_frags_free = old_nr_frags - sinfo->nr_frags; - if (unlikely(nr_frags_free)) { - frag_page -= nr_frags_free; + new_nr_frags = sinfo->nr_frags; + nr_frags_free = old_nr_frags - new_nr_frags; + if (unlikely(nr_frags_free)) truesize -= (nr_frags_free - 1) * PAGE_SIZE + ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); - } len = mxbuf->xdp.data_end - mxbuf->xdp.data; @@ -2164,7 +2159,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w struct mlx5e_frag_page *pagep; /* sinfo->nr_frags is reset by build_skb, calculate again. */ - xdp_update_skb_frags_info(skb, frag_page - head_page, + xdp_update_skb_frags_info(skb, new_nr_frags, sinfo->xdp_frags_size, truesize, xdp_buff_get_skb_flags(&mxbuf->xdp)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index a2802cfc9b989..a8af84fc97638 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1608,12 +1608,13 @@ void mlx5e_stats_fec_get(struct mlx5e_priv *priv, { int mode = fec_active_mode(priv->mdev); - if (mode == MLX5E_FEC_NOFEC || - !MLX5_CAP_PCAM_FEATURE(priv->mdev, ppcnt_statistical_group)) + if (mode == MLX5E_FEC_NOFEC) return; - fec_set_corrected_bits_total(priv, fec_stats); - fec_set_block_stats(priv, mode, fec_stats); + if (MLX5_CAP_PCAM_FEATURE(priv->mdev, ppcnt_statistical_group)) { + fec_set_corrected_bits_total(priv, fec_stats); + fec_set_block_stats(priv, mode, fec_stats); + } if (MLX5_CAP_PCAM_REG(priv->mdev, pphcr)) fec_set_histograms_stats(priv, mode, hist); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 00c2763e57ca1..ebea43c235cc3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -2147,11 +2147,14 @@ static void mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow *flow, static void mlx5e_tc_del_fdb_peers_flow(struct mlx5e_tc_flow *flow) { + struct mlx5_devcom_comp_dev *devcom; + struct mlx5_devcom_comp_dev *pos; + struct mlx5_eswitch *peer_esw; int i; - for (i = 0; i < MLX5_MAX_PORTS; i++) { - if (i == mlx5_get_dev_index(flow->priv->mdev)) - continue; + devcom = flow->priv->mdev->priv.eswitch->devcom; + mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) { + i = mlx5_get_dev_index(peer_esw->dev); mlx5e_tc_del_fdb_peer_flow(flow, i); } } @@ -5511,12 +5514,16 @@ int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags) void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw) { + struct mlx5_devcom_comp_dev *devcom; + struct mlx5_devcom_comp_dev *pos; struct mlx5e_tc_flow *flow, *tmp; + struct mlx5_eswitch *peer_esw; int i; - for (i = 0; i < MLX5_MAX_PORTS; i++) { - if (i == mlx5_get_dev_index(esw->dev)) - continue; + devcom = esw->devcom; + + mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) { + i = mlx5_get_dev_index(peer_esw->dev); list_for_each_entry_safe(flow, tmp, &esw->offloads.peer_flows[i], peer[i]) mlx5e_tc_del_fdb_peers_flow(flow); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c index 1c37098e09ea5..49a637829c594 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c @@ -188,7 +188,7 @@ int esw_acl_ingress_lgcy_setup(struct mlx5_eswitch *esw, if (IS_ERR(vport->ingress.acl)) { err = PTR_ERR(vport->ingress.acl); vport->ingress.acl = NULL; - return err; + goto out; } err = esw_acl_ingress_lgcy_groups_create(esw, vport); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c index 56e6f54b1e2ed..af58ad72906ff 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c @@ -1497,24 +1497,24 @@ static int esw_qos_node_enable_tc_arbitration(struct mlx5_esw_sched_node *node, return err; } -static u32 mlx5_esw_qos_lag_link_speed_get_locked(struct mlx5_core_dev *mdev) +static u32 mlx5_esw_qos_lag_link_speed_get(struct mlx5_core_dev *mdev, + bool take_rtnl) { struct ethtool_link_ksettings lksettings; struct net_device *slave, *master; u32 speed = SPEED_UNKNOWN; - /* Lock ensures a stable reference to master and slave netdevice - * while port speed of master is queried. - */ - ASSERT_RTNL(); - slave = mlx5_uplink_netdev_get(mdev); if (!slave) goto out; + if (take_rtnl) + rtnl_lock(); master = netdev_master_upper_dev_get(slave); if (master && !__ethtool_get_link_ksettings(master, &lksettings)) speed = lksettings.base.speed; + if (take_rtnl) + rtnl_unlock(); out: mlx5_uplink_netdev_put(mdev, slave); @@ -1522,20 +1522,15 @@ static u32 mlx5_esw_qos_lag_link_speed_get_locked(struct mlx5_core_dev *mdev) } static int mlx5_esw_qos_max_link_speed_get(struct mlx5_core_dev *mdev, u32 *link_speed_max, - bool hold_rtnl_lock, struct netlink_ext_ack *extack) + bool take_rtnl, + struct netlink_ext_ack *extack) { int err; if (!mlx5_lag_is_active(mdev)) goto skip_lag; - if (hold_rtnl_lock) - rtnl_lock(); - - *link_speed_max = mlx5_esw_qos_lag_link_speed_get_locked(mdev); - - if (hold_rtnl_lock) - rtnl_unlock(); + *link_speed_max = mlx5_esw_qos_lag_link_speed_get(mdev, take_rtnl); if (*link_speed_max != (u32)SPEED_UNKNOWN) return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index e2ffb87b94cbe..49bc409d7dbb0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1081,10 +1081,11 @@ static void mlx5_eswitch_event_handler_register(struct mlx5_eswitch *esw) static void mlx5_eswitch_event_handler_unregister(struct mlx5_eswitch *esw) { - if (esw->mode == MLX5_ESWITCH_OFFLOADS && mlx5_eswitch_is_funcs_handler(esw->dev)) + if (esw->mode == MLX5_ESWITCH_OFFLOADS && + mlx5_eswitch_is_funcs_handler(esw->dev)) { mlx5_eq_notifier_unregister(esw->dev, &esw->esw_funcs.nb); - - flush_workqueue(esw->work_queue); + atomic_inc(&esw->esw_funcs.generation); + } } static void mlx5_eswitch_clear_vf_vports_info(struct mlx5_eswitch *esw) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 16eb99aba2a7e..558055d214e10 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -331,10 +331,12 @@ struct esw_mc_addr { /* SRIOV only */ struct mlx5_host_work { struct work_struct work; struct mlx5_eswitch *esw; + int work_gen; }; struct mlx5_esw_functions { struct mlx5_nb nb; + atomic_t generation; bool host_funcs_disabled; u16 num_vfs; u16 num_ec_vfs; @@ -1002,7 +1004,7 @@ mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev) static inline bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id) { - return -EOPNOTSUPP; + return false; } #endif /* CONFIG_MLX5_ESWITCH */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 44a142a041b2f..8be0961cb6c7e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -52,6 +52,7 @@ #include "devlink.h" #include "lag/lag.h" #include "en/tc/post_meter.h" +#include "fw_reset.h" /* There are two match-all miss flows, one for unicast dst mac and * one for multicast. @@ -1240,21 +1241,17 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, flows[peer_vport->index] = flow; } - if (mlx5_esw_host_functions_enabled(esw->dev)) { - mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, - mlx5_core_max_vfs(peer_dev)) { - esw_set_peer_miss_rule_source_port(esw, peer_esw, - spec, - peer_vport->vport); - - flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), - spec, &flow_act, &dest, 1); - if (IS_ERR(flow)) { - err = PTR_ERR(flow); - goto add_vf_flow_err; - } - flows[peer_vport->index] = flow; + mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_vfs(peer_dev)) { + esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, + peer_vport->vport); + flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), + spec, &flow_act, &dest, 1); + if (IS_ERR(flow)) { + err = PTR_ERR(flow); + goto add_vf_flow_err; } + flows[peer_vport->index] = flow; } if (mlx5_core_ec_sriov_enabled(peer_dev)) { @@ -1346,7 +1343,8 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, mlx5_del_flow_rules(flows[peer_vport->index]); } - if (mlx5_core_is_ecpf_esw_manager(peer_dev)) { + if (mlx5_core_is_ecpf_esw_manager(peer_dev) && + mlx5_esw_host_functions_enabled(peer_dev)) { peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); mlx5_del_flow_rules(flows[peer_vport->index]); } @@ -3450,22 +3448,28 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) } static void -esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) +esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, int work_gen, + const u32 *out) { struct devlink *devlink; bool host_pf_disabled; u16 new_num_vfs; + devlink = priv_to_devlink(esw->dev); + devl_lock(devlink); + + /* Stale work from one or more mode changes ago. Bail out. */ + if (work_gen != atomic_read(&esw->esw_funcs.generation)) + goto unlock; + new_num_vfs = MLX5_GET(query_esw_functions_out, out, host_params_context.host_num_of_vfs); host_pf_disabled = MLX5_GET(query_esw_functions_out, out, host_params_context.host_pf_disabled); if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) - return; + goto unlock; - devlink = priv_to_devlink(esw->dev); - devl_lock(devlink); /* Number of VFs can only change from "0 to x" or "x to 0". */ if (esw->esw_funcs.num_vfs > 0) { mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); @@ -3480,6 +3484,7 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) } } esw->esw_funcs.num_vfs = new_num_vfs; +unlock: devl_unlock(devlink); } @@ -3496,7 +3501,7 @@ static void esw_functions_changed_event_handler(struct work_struct *work) if (IS_ERR(out)) goto out; - esw_vfs_changed_event_handler(esw, out); + esw_vfs_changed_event_handler(esw, host_work->work_gen, out); kvfree(out); out: kfree(host_work); @@ -3516,6 +3521,7 @@ int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); host_work->esw = esw; + host_work->work_gen = atomic_read(&esw_funcs->generation); INIT_WORK(&host_work->work, esw_functions_changed_event_handler); queue_work(esw->work_queue, &host_work->work); @@ -3617,6 +3623,8 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) return 0; err_vports: + /* rollback to legacy, indicates don't unregister the uplink netdev */ + esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); err_uplink: esw_offloads_steering_cleanup(esw); @@ -3807,6 +3815,11 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (IS_ERR(esw)) return PTR_ERR(esw); + if (mlx5_fw_reset_in_progress(esw->dev)) { + NL_SET_ERR_MSG_MOD(extack, "Can't change eswitch mode during firmware reset"); + return -EBUSY; + } + if (esw_mode_from_devlink(mode, &mlx5_mode)) return -EINVAL; @@ -3848,6 +3861,8 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (mode == DEVLINK_ESWITCH_MODE_LEGACY) esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) + esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_eswitch_disable_locked(esw); if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 1af76da8b1320..b79544134e2a2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -1167,7 +1167,8 @@ int mlx5_fs_cmd_set_tx_flow_table_root(struct mlx5_core_dev *dev, u32 ft_id, boo u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; - if (disconnect && MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default)) + if (disconnect && + !MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default)) return -EOPNOTSUPP; MLX5_SET(set_flow_table_root_in, in, opcode, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index eeb4437975f20..c1f220e5fe185 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -822,48 +822,63 @@ mlx5_fw_image_pending(struct mlx5_core_dev *dev, return 0; } -int mlx5_fw_version_query(struct mlx5_core_dev *dev, - u32 *running_ver, u32 *pending_ver) +void mlx5_fw_version_query(struct mlx5_core_dev *dev, + u32 *running_ver, u32 *pending_ver) { u32 reg_mcqi_version[MLX5_ST_SZ_DW(mcqi_version)] = {}; bool pending_version_exists; int component_index; int err; + *running_ver = 0; + *pending_ver = 0; + if (!MLX5_CAP_GEN(dev, mcam_reg) || !MLX5_CAP_MCAM_REG(dev, mcqi) || !MLX5_CAP_MCAM_REG(dev, mcqs)) { mlx5_core_warn(dev, "fw query isn't supported by the FW\n"); - return -EOPNOTSUPP; + return; } component_index = mlx5_get_boot_img_component_index(dev); - if (component_index < 0) - return component_index; + if (component_index < 0) { + mlx5_core_warn(dev, "fw query failed to find boot img component index, err %d\n", + component_index); + return; + } + *running_ver = U32_MAX; /* indicate failure */ err = mlx5_reg_mcqi_version_query(dev, component_index, MCQI_FW_RUNNING_VERSION, reg_mcqi_version); - if (err) - return err; - - *running_ver = MLX5_GET(mcqi_version, reg_mcqi_version, version); - + if (!err) + *running_ver = MLX5_GET(mcqi_version, reg_mcqi_version, + version); + else + mlx5_core_warn(dev, "failed to query running version, err %d\n", + err); + + *pending_ver = U32_MAX; /* indicate failure */ err = mlx5_fw_image_pending(dev, component_index, &pending_version_exists); - if (err) - return err; + if (err) { + mlx5_core_warn(dev, "failed to query pending image, err %d\n", + err); + return; + } if (!pending_version_exists) { *pending_ver = 0; - return 0; + return; } err = mlx5_reg_mcqi_version_query(dev, component_index, MCQI_FW_STORED_VERSION, reg_mcqi_version); - if (err) - return err; - - *pending_ver = MLX5_GET(mcqi_version, reg_mcqi_version, version); - - return 0; + if (!err) + *pending_ver = MLX5_GET(mcqi_version, reg_mcqi_version, + version); + else + mlx5_core_warn(dev, "failed to query pending version, err %d\n", + err); + + return; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index 89e399606877b..4544f1968f73f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -15,6 +15,7 @@ enum { MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, + MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, }; struct mlx5_fw_reset { @@ -127,6 +128,16 @@ int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_ty return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL, NULL); } +bool mlx5_fw_reset_in_progress(struct mlx5_core_dev *dev) +{ + struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; + + if (!fw_reset) + return false; + + return test_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags); +} + static int mlx5_fw_reset_get_reset_method(struct mlx5_core_dev *dev, u8 *reset_method) { @@ -242,6 +253,8 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev) BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE)); devl_unlock(devlink); } + + clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags); } static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev) @@ -461,27 +474,48 @@ static void mlx5_sync_reset_request_event(struct work_struct *work) struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, reset_request_work); struct mlx5_core_dev *dev = fw_reset->dev; + bool nack_request = false; + struct devlink *devlink; int err; err = mlx5_fw_reset_get_reset_method(dev, &fw_reset->reset_method); - if (err) + if (err) { + nack_request = true; mlx5_core_warn(dev, "Failed reading MFRL, err %d\n", err); + } else if (!mlx5_is_reset_now_capable(dev, fw_reset->reset_method) || + test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, + &fw_reset->reset_flags)) { + nack_request = true; + } - if (err || test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags) || - !mlx5_is_reset_now_capable(dev, fw_reset->reset_method)) { + devlink = priv_to_devlink(dev); + /* For external resets, try to acquire devl_lock. Skip if devlink reset is + * pending (lock already held) + */ + if (nack_request || + (!test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, + &fw_reset->reset_flags) && + !devl_trylock(devlink))) { err = mlx5_fw_reset_set_reset_sync_nack(dev); mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s", err ? "Failed" : "Sent"); return; } + if (mlx5_sync_reset_set_reset_requested(dev)) - return; + goto unlock; + + set_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags); err = mlx5_fw_reset_set_reset_sync_ack(dev); if (err) mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err); else mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n"); + +unlock: + if (!test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) + devl_unlock(devlink); } static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev, u16 dev_id) @@ -721,6 +755,8 @@ static void mlx5_sync_reset_abort_event(struct work_struct *work) if (mlx5_sync_reset_clear_reset_requested(dev, true)) return; + + clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags); mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n"); } @@ -757,6 +793,7 @@ static void mlx5_sync_reset_timeout_work(struct work_struct *work) if (mlx5_sync_reset_clear_reset_requested(dev, true)) return; + clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags); mlx5_core_warn(dev, "PCI Sync FW Update Reset Timeout.\n"); } @@ -843,7 +880,8 @@ void mlx5_drain_fw_reset(struct mlx5_core_dev *dev) cancel_work_sync(&fw_reset->reset_reload_work); cancel_work_sync(&fw_reset->reset_now_work); cancel_work_sync(&fw_reset->reset_abort_work); - cancel_delayed_work(&fw_reset->reset_timeout_work); + if (test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) + mlx5_sync_reset_clear_reset_requested(dev, true); } static const struct devlink_param mlx5_fw_reset_devlink_params[] = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h index d5b28525c960d..2d96b2adc1cdf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h @@ -10,6 +10,7 @@ int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_ty int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel, struct netlink_ext_ack *extack); int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev); +bool mlx5_fw_reset_in_progress(struct mlx5_core_dev *dev); int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev); void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c index 62b6faa4276aa..b8d5f6a44d26a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c @@ -160,8 +160,11 @@ DEFINE_SHOW_ATTRIBUTE(members); void mlx5_ldev_add_debugfs(struct mlx5_core_dev *dev) { + struct mlx5_lag *ldev = mlx5_lag_dev(dev); struct dentry *dbg; + if (!ldev) + return; dbg = debugfs_create_dir("lag", mlx5_debugfs_get_dev_root(dev)); dev->priv.dbg.lag_debugfs = dbg; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 3db0387bf6dcb..2d6019a32669c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1413,6 +1413,7 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) static void mlx5_lag_unregister_hca_devcom_comp(struct mlx5_core_dev *dev) { mlx5_devcom_unregister_component(dev->priv.hca_devcom_comp); + dev->priv.hca_devcom_comp = NULL; } static int mlx5_lag_register_hca_devcom_comp(struct mlx5_core_dev *dev) @@ -1651,8 +1652,12 @@ void mlx5_lag_disable_change(struct mlx5_core_dev *dev) mutex_lock(&ldev->lock); ldev->mode_changes_in_progress++; - if (__mlx5_lag_is_active(ldev)) - mlx5_disable_lag(ldev); + if (__mlx5_lag_is_active(ldev)) { + if (ldev->mode == MLX5_LAG_MODE_MPESW) + mlx5_lag_disable_mpesw(ldev); + else + mlx5_disable_lag(ldev); + } mutex_unlock(&ldev->lock); mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c index aee17fcf3b36c..cdc99fe5c9568 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c @@ -173,10 +173,15 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event, } /* Handle multipath entry with lower priority value */ - if (mp->fib.mfi && mp->fib.mfi != fi && + if (mp->fib.mfi && (mp->fib.dst != fen_info->dst || mp->fib.dst_len != fen_info->dst_len) && - fi->fib_priority >= mp->fib.priority) + mp->fib.dst_len <= fen_info->dst_len && + !(mp->fib.dst_len == fen_info->dst_len && + fi->fib_priority < mp->fib.priority)) { + mlx5_core_dbg(ldev->pf[idx].dev, + "Multipath entry with lower priority was rejected\n"); return; + } nh_dev0 = mlx5_lag_get_next_fib_dev(ldev, fi, NULL); nh_dev1 = mlx5_lag_get_next_fib_dev(ldev, fi, nh_dev0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index aad52d3a90e68..c217998604fdb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -65,14 +65,21 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev) return err; } -static int enable_mpesw(struct mlx5_lag *ldev) +static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) { - int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_core_dev *dev0; int err; + int idx; int i; - if (idx < 0 || ldev->mode != MLX5_LAG_MODE_NONE) + if (ldev->mode == MLX5_LAG_MODE_MPESW) + return 0; + + if (ldev->mode != MLX5_LAG_MODE_NONE) + return -EINVAL; + + idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); + if (idx < 0) return -EINVAL; dev0 = ldev->pf[idx].dev; @@ -117,7 +124,7 @@ static int enable_mpesw(struct mlx5_lag *ldev) return err; } -static void disable_mpesw(struct mlx5_lag *ldev) +void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) { if (ldev->mode == MLX5_LAG_MODE_MPESW) { mlx5_mpesw_metadata_cleanup(ldev); @@ -143,9 +150,9 @@ static void mlx5_mpesw_work(struct work_struct *work) } if (mpesww->op == MLX5_MPESW_OP_ENABLE) - mpesww->result = enable_mpesw(ldev); + mpesww->result = mlx5_lag_enable_mpesw(ldev); else if (mpesww->op == MLX5_MPESW_OP_DISABLE) - disable_mpesw(ldev); + mlx5_lag_disable_mpesw(ldev); unlock: mutex_unlock(&ldev->lock); mlx5_devcom_comp_unlock(devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h index 02520f27a033c..46de93ed790de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h @@ -31,5 +31,10 @@ int mlx5_lag_mpesw_do_mirred(struct mlx5_core_dev *mdev, bool mlx5_lag_is_mpesw(struct mlx5_core_dev *dev); void mlx5_lag_mpesw_disable(struct mlx5_core_dev *dev); int mlx5_lag_mpesw_enable(struct mlx5_core_dev *dev); +#ifdef CONFIG_MLX5_ESWITCH +void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev); +#else +static inline void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) {} +#endif /* CONFIG_MLX5_ESWITCH */ #endif /* __MLX5_LAG_MPESW_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 70c156591b0ba..5903a4af9173b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -999,16 +999,10 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) goto err_irq_cleanup; } - err = mlx5_events_init(dev); - if (err) { - mlx5_core_err(dev, "failed to initialize events\n"); - goto err_eq_cleanup; - } - err = mlx5_fw_reset_init(dev); if (err) { mlx5_core_err(dev, "failed to initialize fw reset events\n"); - goto err_events_cleanup; + goto err_eq_cleanup; } mlx5_cq_debugfs_init(dev); @@ -1110,8 +1104,6 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) mlx5_cleanup_reserved_gids(dev); mlx5_cq_debugfs_cleanup(dev); mlx5_fw_reset_cleanup(dev); -err_events_cleanup: - mlx5_events_cleanup(dev); err_eq_cleanup: mlx5_eq_table_cleanup(dev); err_irq_cleanup: @@ -1144,7 +1136,6 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) mlx5_cleanup_reserved_gids(dev); mlx5_cq_debugfs_cleanup(dev); mlx5_fw_reset_cleanup(dev); - mlx5_events_cleanup(dev); mlx5_eq_table_cleanup(dev); mlx5_irq_table_cleanup(dev); mlx5_devcom_unregister_device(dev->priv.devc); @@ -1812,15 +1803,23 @@ static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev) return -ENOMEM; } -static int vhca_id_show(struct seq_file *file, void *priv) +static int mlx5_notifiers_init(struct mlx5_core_dev *dev) { - struct mlx5_core_dev *dev = file->private; + int err; + + err = mlx5_events_init(dev); + if (err) { + mlx5_core_err(dev, "failed to initialize events\n"); + return err; + } - seq_printf(file, "0x%x\n", MLX5_CAP_GEN(dev, vhca_id)); return 0; } -DEFINE_SHOW_ATTRIBUTE(vhca_id); +static void mlx5_notifiers_cleanup(struct mlx5_core_dev *dev) +{ + mlx5_events_cleanup(dev); +} int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) { @@ -1846,7 +1845,7 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev)); priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device), mlx5_debugfs_root); - debugfs_create_file("vhca_id", 0400, priv->dbg.dbg_root, dev, &vhca_id_fops); + INIT_LIST_HEAD(&priv->traps); err = mlx5_cmd_init(dev); @@ -1877,6 +1876,10 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) if (err) goto err_hca_caps; + err = mlx5_notifiers_init(dev); + if (err) + goto err_hca_caps; + /* The conjunction of sw_vhca_id with sw_owner_id will be a global * unique id per function which uses mlx5_core. * Those values are supplied to FW as part of the init HCA command to @@ -1919,6 +1922,7 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev) if (priv->sw_vhca_id > 0) ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id); + mlx5_notifiers_cleanup(dev); mlx5_hca_caps_free(dev); mlx5_adev_cleanup(dev); mlx5_pagealloc_cleanup(dev); @@ -1979,6 +1983,8 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) goto err_init_one; } + mlx5_vhca_debugfs_init(dev); + pci_save_state(pdev); return 0; @@ -2094,7 +2100,6 @@ static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); err = wait_vital(pdev); if (err) { @@ -2189,6 +2194,7 @@ static void shutdown(struct pci_dev *pdev) mlx5_core_info(dev, "Shutdown was called\n"); set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state); + mlx5_drain_fw_reset(dev); mlx5_drain_health_wq(dev); err = mlx5_try_fast_unload(dev); if (err) @@ -2232,6 +2238,7 @@ static const struct pci_device_id mlx5_core_pci_table[] = { { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */ { PCI_VDEVICE(MELLANOX, 0x1025) }, /* ConnectX-9 */ { PCI_VDEVICE(MELLANOX, 0x1027) }, /* ConnectX-10 */ + { PCI_VDEVICE(MELLANOX, 0x2101) }, /* ConnectX-10 NVLink-C2C */ { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */ { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */ { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 082259b56816c..09c544bdf70da 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -258,6 +258,7 @@ int mlx5_wait_for_pages(struct mlx5_core_dev *dev, int *pages); void mlx5_cmd_flush(struct mlx5_core_dev *dev); void mlx5_cq_debugfs_init(struct mlx5_core_dev *dev); void mlx5_cq_debugfs_cleanup(struct mlx5_core_dev *dev); +void mlx5_vhca_debugfs_init(struct mlx5_core_dev *dev); int mlx5_query_pcam_reg(struct mlx5_core_dev *dev, u32 *pcam, u8 feature_group, u8 access_reg_group); @@ -390,8 +391,8 @@ int mlx5_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed); int mlx5_firmware_flash(struct mlx5_core_dev *dev, const struct firmware *fw, struct netlink_ext_ack *extack); -int mlx5_fw_version_query(struct mlx5_core_dev *dev, - u32 *running_ver, u32 *stored_ver); +void mlx5_fw_version_query(struct mlx5_core_dev *dev, u32 *running_ver, + u32 *stored_ver); #ifdef CONFIG_MLX5_CORE_EN int mlx5e_init(void); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index aa9f2b0a77d36..876e648c91ba8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -430,7 +430,8 @@ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev, mlx5_qsfp_eeprom_params_set(&query.i2c_address, &query.page, &offset); break; default: - mlx5_core_err(dev, "Module ID not recognized: 0x%x\n", module_id); + mlx5_core_dbg(dev, "Module ID not recognized: 0x%x\n", + module_id); return -EINVAL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c index b706f1486504a..c45540fe7d9d9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c @@ -76,6 +76,7 @@ static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxilia goto init_one_err; } + mlx5_vhca_debugfs_init(mdev); return 0; init_one_err: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index a2fc937d54617..172862a70c70d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -193,7 +193,9 @@ static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) err = pci_enable_sriov(pdev, num_vfs); if (err) { mlx5_core_warn(dev, "pci_enable_sriov failed : %d\n", err); + devl_lock(devlink); mlx5_device_disable_sriov(dev, num_vfs, true, true); + devl_unlock(devlink); } return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c index 030a5776c9374..a4c19af1775f1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c @@ -1050,8 +1050,8 @@ static int dr_dump_domain_all(struct seq_file *file, struct mlx5dr_domain *dmn) struct mlx5dr_table *tbl; int ret; - mutex_lock(&dmn->dump_info.dbg_mutex); mlx5dr_domain_lock(dmn); + mutex_lock(&dmn->dump_info.dbg_mutex); ret = dr_dump_domain(file, dmn); if (ret < 0) @@ -1064,8 +1064,8 @@ static int dr_dump_domain_all(struct seq_file *file, struct mlx5dr_domain *dmn) } unlock_mutex: - mlx5dr_domain_unlock(dmn); mutex_unlock(&dmn->dump_info.dbg_mutex); + mlx5dr_domain_unlock(dmn); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c index 05e5fd777d4f4..8701b7b6a2d51 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c @@ -2,6 +2,7 @@ // Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. #include +#include #include #include "lib/clock.h" #include "mlx5_core.h" @@ -14,7 +15,7 @@ #define TEST_WC_NUM_WQES 255 #define TEST_WC_LOG_CQ_SZ (order_base_2(TEST_WC_NUM_WQES)) #define TEST_WC_SQ_LOG_WQ_SZ TEST_WC_LOG_CQ_SZ -#define TEST_WC_POLLING_MAX_TIME_JIFFIES msecs_to_jiffies(100) +#define TEST_WC_POLLING_MAX_TIME_USEC (100 * USEC_PER_MSEC) struct mlx5_wc_cq { /* data path - accessed per cqe */ @@ -358,7 +359,6 @@ static int mlx5_wc_poll_cq(struct mlx5_wc_sq *sq) static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) { unsigned int offset = 0; - unsigned long expires; struct mlx5_wc_sq *sq; int i, err; @@ -388,13 +388,9 @@ static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) mlx5_wc_post_nop(sq, &offset, true); - expires = jiffies + TEST_WC_POLLING_MAX_TIME_JIFFIES; - do { - err = mlx5_wc_poll_cq(sq); - if (err) - usleep_range(2, 10); - } while (mdev->wc_state == MLX5_WC_STATE_UNINITIALIZED && - time_is_after_jiffies(expires)); + poll_timeout_us(mlx5_wc_poll_cq(sq), + mdev->wc_state != MLX5_WC_STATE_UNINITIALIZED, 10, + TEST_WC_POLLING_MAX_TIME_USEC, false); mlx5_wc_destroy_sq(sq); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c index 5afe6b155ef0d..81935f87bfcd7 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c @@ -440,7 +440,9 @@ int mlxsw_sp_mr_route_add(struct mlxsw_sp_mr_table *mr_table, rhashtable_remove_fast(&mr_table->route_ht, &mr_orig_route->ht_node, mlxsw_sp_mr_route_ht_params); + mutex_lock(&mr_table->route_list_lock); list_del(&mr_orig_route->node); + mutex_unlock(&mr_table->route_list_lock); mlxsw_sp_mr_route_destroy(mr_table, mr_orig_route); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index a2033837182e8..2d0e89bd2fb9c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -2265,6 +2265,7 @@ mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, if (!neigh_entry) return NULL; + neigh_hold(n); neigh_entry->key.n = n; neigh_entry->rif = rif; INIT_LIST_HEAD(&neigh_entry->nexthop_list); @@ -2274,6 +2275,7 @@ mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) { + neigh_release(neigh_entry->key.n); kfree(neigh_entry); } @@ -2858,6 +2860,11 @@ static int mlxsw_sp_router_schedule_work(struct net *net, if (!net_work) return NOTIFY_BAD; + /* Take a reference to ensure the neighbour won't be destructed until + * we drop the reference in the work item. + */ + neigh_clone(n); + INIT_WORK(&net_work->work, cb); net_work->mlxsw_sp = router->mlxsw_sp; net_work->n = n; @@ -2881,11 +2888,6 @@ static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router, struct net *net; net = neigh_parms_net(n->parms); - - /* Take a reference to ensure the neighbour won't be destructed until we - * drop the reference in delayed work. - */ - neigh_clone(n); return mlxsw_sp_router_schedule_work(net, router, n, mlxsw_sp_router_neigh_event_work); } @@ -4320,6 +4322,8 @@ mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, if (err) goto err_neigh_entry_insert; + neigh_release(old_n); + read_lock_bh(&n->lock); nud_state = n->nud_state; dead = n->dead; @@ -4328,14 +4332,10 @@ mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, list_for_each_entry(nh, &neigh_entry->nexthop_list, neigh_list_node) { - neigh_release(old_n); - neigh_clone(n); __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); } - neigh_release(n); - return 0; err_neigh_entry_insert: @@ -4428,6 +4428,11 @@ static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, } } + /* Release the reference taken by neigh_lookup() / neigh_create() since + * neigh_entry already holds one. + */ + neigh_release(n); + /* If that is the first nexthop connected to that neigh, add to * nexthop_neighs_list */ @@ -4454,11 +4459,9 @@ static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop *nh) { struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; - struct neighbour *n; if (!neigh_entry) return; - n = neigh_entry->key.n; __mlxsw_sp_nexthop_neigh_update(nh, true); list_del(&nh->neigh_list_node); @@ -4472,8 +4475,6 @@ static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); - - neigh_release(n); } static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 95fac020eb93c..08aed4103323e 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -1142,6 +1142,9 @@ static int fbnic_set_cls_rule_ins(struct fbnic_net *fbn, return -EINVAL; } + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); + /* Write action table values */ act_tcam->dest = dest; act_tcam->rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, hash_idx); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c index 85a883dba385f..d8a9a7d7c2375 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c @@ -51,8 +51,6 @@ int fbnic_fw_log_init(struct fbnic_dev *fbd) log->data_start = data; log->data_end = data + FBNIC_FW_LOG_SIZE; - fbnic_fw_log_enable(fbd, true); - return 0; } @@ -63,7 +61,6 @@ void fbnic_fw_log_free(struct fbnic_dev *fbd) if (!fbnic_fw_log_ready(fbd)) return; - fbnic_fw_log_disable(fbd); INIT_LIST_HEAD(&log->entries); log->size = 0; vfree(log->data_start); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index e95be0e7bd9e0..cbedaa037cfa8 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -262,6 +262,23 @@ static int fbnic_set_mac(struct net_device *netdev, void *p) return 0; } +static int fbnic_change_mtu(struct net_device *dev, int new_mtu) +{ + struct fbnic_net *fbn = netdev_priv(dev); + + if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) { + dev_err(&dev->dev, + "MTU %d is larger than HDS threshold %d in XDP mode\n", + new_mtu, fbn->hds_thresh); + + return -EINVAL; + } + + WRITE_ONCE(dev->mtu, new_mtu); + + return 0; +} + void fbnic_clear_rx_mode(struct fbnic_dev *fbd) { struct net_device *netdev = fbd->netdev; @@ -533,6 +550,7 @@ static const struct net_device_ops fbnic_netdev_ops = { .ndo_start_xmit = fbnic_xmit_frame, .ndo_features_check = fbnic_features_check, .ndo_set_mac_address = fbnic_set_mac, + .ndo_change_mtu = fbnic_change_mtu, .ndo_set_rx_mode = fbnic_set_rx_mode, .ndo_get_stats64 = fbnic_get_stats64, .ndo_bpf = fbnic_bpf, @@ -790,6 +808,8 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) netdev->hw_enc_features |= netdev->features; netdev->features |= NETIF_F_NTUPLE; + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; + netdev->min_mtu = IPV6_MIN_MTU; netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h index b0a87c57910f2..e6ca23a9957d9 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h @@ -73,6 +73,8 @@ struct fbnic_net { /* Time stamping filter config */ struct kernel_hwtstamp_config hwtstamp_config; + + bool tx_pause; }; int __fbnic_open(struct fbnic_net *fbn); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index a7a6b4db8016f..698b8a85afb31 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -303,11 +303,17 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto free_irqs; } + err = fbnic_fw_log_init(fbd); + if (err) + dev_warn(fbd->dev, + "Unable to initialize firmware log buffer: %d\n", + err); + err = fbnic_fw_request_mbx(fbd); if (err) { dev_err(&pdev->dev, "Firmware mailbox initialization failure\n"); - goto free_irqs; + goto free_fw_log; } /* Send the request to enable the FW logging to host. Note if this @@ -315,11 +321,7 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * possible the FW is just too old to support the logging and needs * to be updated. */ - err = fbnic_fw_log_init(fbd); - if (err) - dev_warn(fbd->dev, - "Unable to initialize firmware log buffer: %d\n", - err); + fbnic_fw_log_enable(fbd, true); fbnic_devlink_register(fbd); fbnic_devlink_otp_check(fbd, "error detected during probe"); @@ -363,6 +365,8 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * firmware updates for fixes. */ return 0; +free_fw_log: + fbnic_fw_log_free(fbd); free_irqs: fbnic_free_irqs(fbd); err_destroy_health: @@ -397,8 +401,9 @@ static void fbnic_remove(struct pci_dev *pdev) fbnic_hwmon_unregister(fbd); fbnic_dbg_fbd_exit(fbd); fbnic_devlink_unregister(fbd); - fbnic_fw_log_free(fbd); + fbnic_fw_log_disable(fbd); fbnic_fw_free_mbx(fbd); + fbnic_fw_log_free(fbd); fbnic_free_irqs(fbd); fbnic_devlink_health_destroy(fbd); @@ -574,7 +579,6 @@ static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev) pci_set_power_state(pdev, PCI_D0); pci_restore_state(pdev); - pci_save_state(pdev); if (pci_enable_device_mem(pdev)) { dev_err(&pdev->dev, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c index 7ce3fdd252828..62701923cfe96 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c @@ -208,6 +208,9 @@ fbnic_phylink_mac_link_up(struct phylink_config *config, struct fbnic_net *fbn = netdev_priv(netdev); struct fbnic_dev *fbd = fbn->fbd; + fbn->tx_pause = tx_pause; + fbnic_config_drop_mode(fbn, tx_pause); + fbd->mac->link_up(fbd, tx_pause, rx_pause); } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index 7f31e890031c0..42a186db43ea9 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -338,9 +338,8 @@ void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) else if (tstamp_mask & (1u << flow_type)) dest |= FBNIC_RPC_ACT_TBL0_TS_ENA; - if (act1_value[flow_type] & FBNIC_RPC_TCAM_ACT1_L4_VALID) - dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, - FBNIC_RCD_HDR_AL_DMA_HINT_L4); + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, flow_type); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index b1e8ce89870f7..6e21f6c17c658 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -925,7 +925,7 @@ static void fbnic_fill_bdq(struct fbnic_ring *bdq) /* Force DMA writes to flush before writing to tail */ dma_wmb(); - writel(i, bdq->doorbell); + writel(i * FBNIC_BD_FRAG_COUNT, bdq->doorbell); } } @@ -2546,7 +2546,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) hpq->tail = 0; hpq->head = 0; - log_size = fls(hpq->size_mask); + log_size = fls(hpq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); /* Store descriptor ring address and size */ fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); @@ -2558,7 +2558,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) if (!ppq->size_mask) goto write_ctl; - log_size = fls(ppq->size_mask); + log_size = fls(ppq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); /* Add enabling of PPQ to BDQ control */ bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; @@ -2573,20 +2573,44 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) } static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, - struct fbnic_ring *rcq) + struct fbnic_ring *rcq, bool tx_pause, + bool hdr_split) { + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 drop_mode, rcq_ctl; - drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; + if (!tx_pause && fbn->num_rx_queues > 1) + drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; + else + drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_NEVER; /* Specify packet layout */ rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | - FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM); + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT, hdr_split); fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); } +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool txp) +{ + bool hds; + int i, t; + + hds = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + + for (i = 0; i < fbn->num_napi; i++) { + struct fbnic_napi_vector *nv = fbn->napi[i]; + + for (t = 0; t < nv->rxt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[nv->txt_count + t]; + + fbnic_config_drop_mode_rcq(nv, &qt->cmpl, txp, hds); + } + } +} + static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 rx_desc) { u32 threshold; @@ -2633,20 +2657,18 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 log_size = fls(rcq->size_mask); - u32 hds_thresh = fbn->hds_thresh; u32 rcq_ctl = 0; - - fbnic_config_drop_mode_rcq(nv, rcq); + bool hdr_split; + u32 hds_thresh; /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should * be split at L4. It would also result in the frames being split at * L2/L3 depending on the frame size. */ - if (fbn->hds_thresh < FBNIC_HDR_BYTES_MIN) { - rcq_ctl = FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT; - hds_thresh = FBNIC_HDR_BYTES_MIN; - } + hdr_split = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause, hdr_split); + hds_thresh = max(fbn->hds_thresh, FBNIC_HDR_BYTES_MIN); rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, @@ -2699,7 +2721,6 @@ static void __fbnic_nv_enable(struct fbnic_napi_vector *nv) &nv->napi); fbnic_enable_bdq(&qt->sub0, &qt->sub1); - fbnic_config_drop_mode_rcq(nv, &qt->cmpl); fbnic_enable_rcq(nv, &qt->cmpl); } } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index ca37da5a0b179..f2ee2cbf3486b 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -38,7 +38,7 @@ struct fbnic_net; #define FBNIC_MAX_XDPQS 128u /* These apply to TWQs, TCQ, RCQ */ -#define FBNIC_QUEUE_SIZE_MIN 16u +#define FBNIC_QUEUE_SIZE_MIN 64u #define FBNIC_QUEUE_SIZE_MAX SZ_64K #define FBNIC_TXQ_SIZE_DEFAULT 1024 @@ -66,7 +66,7 @@ struct fbnic_net; (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD) #define FBNIC_HDS_THRESH_DEFAULT \ (1536 - FBNIC_RX_PAD) -#define FBNIC_HDR_BYTES_MIN 128 +#define FBNIC_HDR_BYTES_MIN 256 struct fbnic_pkt_buff { struct xdp_buff buff; @@ -184,6 +184,7 @@ void fbnic_reset_netif_queues(struct fbnic_net *fbn); irqreturn_t fbnic_msix_clean_rings(int irq, void *data); void fbnic_napi_enable(struct fbnic_net *fbn); void fbnic_napi_disable(struct fbnic_net *fbn); +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool tx_pause); void fbnic_enable(struct fbnic_net *fbn); void fbnic_disable(struct fbnic_net *fbn); void fbnic_flush(struct fbnic_net *fbn); diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index 9d70b51ca91d6..09d255e78f6cd 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -3054,6 +3054,11 @@ static void lan743x_phylink_mac_link_up(struct phylink_config *config, else if (speed == SPEED_100) mac_cr |= MAC_CR_CFG_L_; + if (duplex == DUPLEX_FULL) + mac_cr |= MAC_CR_DPX_; + else + mac_cr &= ~MAC_CR_DPX_; + lan743x_csr_write(adapter, MAC_CR, mac_cr); lan743x_ptp_update_latency(adapter, speed); @@ -3915,7 +3920,6 @@ static int lan743x_pm_resume(struct device *dev) pci_set_power_state(pdev, PCI_D0); pci_restore_state(pdev); - pci_save_state(pdev); /* Restore HW_CFG that was saved during pm suspend */ if (adapter->is_pci11x1x) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index 5026707181042..646f3d65274e3 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -91,6 +91,8 @@ static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx) pp_params.dma_dir = DMA_BIDIRECTIONAL; rx->page_pool = page_pool_create(&pp_params); + if (unlikely(IS_ERR(rx->page_pool))) + return PTR_ERR(rx->page_pool); for (int i = 0; i < lan966x->num_phys_ports; ++i) { struct lan966x_port *port; @@ -117,8 +119,10 @@ static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) return PTR_ERR(rx->page_pool); err = fdma_alloc_coherent(lan966x->dev, fdma); - if (err) + if (err) { + page_pool_destroy(rx->page_pool); return err; + } fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size), FDMA_DCB_STATUS_INTR); @@ -809,9 +813,15 @@ static int lan966x_qsys_sw_status(struct lan966x *lan966x) static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) { + struct page *(*old_pages)[FDMA_RX_DCB_MAX_DBS]; struct page_pool *page_pool; struct fdma fdma_rx_old; - int err; + int err, i, j; + + old_pages = kmemdup(lan966x->rx.page, sizeof(lan966x->rx.page), + GFP_KERNEL); + if (!old_pages) + return -ENOMEM; /* Store these for later to free them */ memcpy(&fdma_rx_old, &lan966x->rx.fdma, sizeof(struct fdma)); @@ -822,7 +832,6 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) lan966x_fdma_stop_netdev(lan966x); lan966x_fdma_rx_disable(&lan966x->rx); - lan966x_fdma_rx_free_pages(&lan966x->rx); lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1; lan966x->rx.max_mtu = new_mtu; err = lan966x_fdma_rx_alloc(&lan966x->rx); @@ -830,6 +839,11 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) goto restore; lan966x_fdma_rx_start(&lan966x->rx); + for (i = 0; i < fdma_rx_old.n_dcbs; ++i) + for (j = 0; j < fdma_rx_old.n_dbs; ++j) + page_pool_put_full_page(page_pool, + old_pages[i][j], false); + fdma_free_coherent(lan966x->dev, &fdma_rx_old); page_pool_destroy(page_pool); @@ -837,12 +851,17 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) lan966x_fdma_wakeup_netdev(lan966x); napi_enable(&lan966x->napi); - return err; + kfree(old_pages); + return 0; restore: lan966x->rx.page_pool = page_pool; memcpy(&lan966x->rx.fdma, &fdma_rx_old, sizeof(struct fdma)); lan966x_fdma_rx_start(&lan966x->rx); + lan966x_fdma_wakeup_netdev(lan966x); + napi_enable(&lan966x->napi); + + kfree(old_pages); return err; } @@ -956,6 +975,7 @@ int lan966x_fdma_init(struct lan966x *lan966x) err = lan966x_fdma_tx_alloc(&lan966x->tx); if (err) { fdma_free_coherent(lan966x->dev, &lan966x->rx.fdma); + page_pool_destroy(lan966x->rx.page_pool); return err; } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index 2f168700f63c1..8b2e07821a950 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -576,7 +576,7 @@ static int sparx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) static struct ptp_clock_info sparx5_ptp_clock_info = { .owner = THIS_MODULE, .name = "sparx5 ptp", - .max_adj = 200000, + .max_adj = 10000000, .gettime64 = sparx5_ptp_gettime64, .settime64 = sparx5_ptp_settime64, .adjtime = sparx5_ptp_adjtime, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h index 1231a80335d7b..04f76f1e23f60 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h @@ -35,7 +35,7 @@ #define SPX5_SE_BURST_UNIT 4096 /* Dwrr */ -#define SPX5_DWRR_COST_MAX 63 +#define SPX5_DWRR_COST_MAX 31 struct sparx5_shaper { u32 mode; diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 43f034e180c41..962fdd29d6063 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -1854,6 +1854,7 @@ static int mana_gd_setup(struct pci_dev *pdev) mana_gd_remove_irqs(pdev); free_workqueue: destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; dev_err(&pdev->dev, "%s failed (error %d)\n", __func__, err); return err; } @@ -1866,7 +1867,10 @@ static void mana_gd_cleanup(struct pci_dev *pdev) mana_gd_remove_irqs(pdev); - destroy_workqueue(gc->service_wq); + if (gc->service_wq) { + destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; + } dev_dbg(&pdev->dev, "mana gdma cleanup successful\n"); } diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index ada6c78a2bef4..21cddafba5061 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -802,9 +802,6 @@ void mana_hwc_destroy_channel(struct gdma_context *gc) gc->max_num_cqs = 0; } - kfree(hwc->caller_ctx); - hwc->caller_ctx = NULL; - if (hwc->txq) mana_hwc_destroy_wq(hwc, hwc->txq); @@ -814,6 +811,9 @@ void mana_hwc_destroy_channel(struct gdma_context *gc) if (hwc->cq) mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); + kfree(hwc->caller_ctx); + hwc->caller_ctx = NULL; + mana_gd_free_res_map(&hwc->inflight_msg_res); hwc->num_inflight_msg = 0; diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 0142fd98392c2..729133c1e5e4f 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -660,6 +660,13 @@ static void mana_get_rxbuf_cfg(struct mana_port_context *apc, } *frag_count = 1; + + /* In the single-buffer path, napi_build_skb() must see the + * actual backing allocation size so skb->truesize reflects + * the full page (or higher-order page), not just the usable + * packet area. + */ + *alloc_size = PAGE_SIZE << get_order(*alloc_size); return; } @@ -1645,8 +1652,14 @@ static void mana_poll_tx_cq(struct mana_cq *cq) ndev = txq->ndev; apc = netdev_priv(ndev); + /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the + * doorbell can be rung in time for the hardware's requirement + * of at least one doorbell ring every 8 wraparounds. + */ comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, - CQE_POLLING_BUFFER); + min((cq->gdma_cq->queue_size / + COMP_ENTRY_SIZE) * 4, + CQE_POLLING_BUFFER)); if (comp_read < 1) return; @@ -2031,7 +2044,14 @@ static void mana_poll_rx_cq(struct mana_cq *cq) struct mana_rxq *rxq = cq->rxq; int comp_read, i; - comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER); + /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the + * doorbell can be rung in time for the hardware's requirement + * of at least one doorbell ring every 8 wraparounds. + */ + comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, + min((cq->gdma_cq->queue_size / + COMP_ENTRY_SIZE) * 4, + CQE_POLLING_BUFFER)); WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); rxq->xdp_flush = false; @@ -2076,11 +2096,11 @@ static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) mana_gd_ring_cq(gdma_queue, SET_ARM_BIT); cq->work_done_since_doorbell = 0; napi_complete_done(&cq->napi, w); - } else if (cq->work_done_since_doorbell > - cq->gdma_cq->queue_size / COMP_ENTRY_SIZE * 4) { + } else if (cq->work_done_since_doorbell >= + (cq->gdma_cq->queue_size / COMP_ENTRY_SIZE) * 4) { /* MANA hardware requires at least one doorbell ring every 8 * wraparounds of CQ even if there is no need to arm the CQ. - * This driver rings the doorbell as soon as we have exceeded + * This driver rings the doorbell as soon as it has processed * 4 wraparounds. */ mana_gd_ring_cq(gdma_queue, 0); @@ -3282,6 +3302,7 @@ static int add_adev(struct gdma_dev *gd, const char *name) struct auxiliary_device *adev; struct mana_adev *madev; int ret; + int id; madev = kzalloc(sizeof(*madev), GFP_KERNEL); if (!madev) @@ -3291,7 +3312,8 @@ static int add_adev(struct gdma_dev *gd, const char *name) ret = mana_adev_idx_alloc(); if (ret < 0) goto idx_fail; - adev->id = ret; + id = ret; + adev->id = id; adev->name = name; adev->dev.parent = gd->gdma_context->dev; @@ -3317,7 +3339,7 @@ static int add_adev(struct gdma_dev *gd, const char *name) auxiliary_device_uninit(adev); init_fail: - mana_adev_idx_free(adev->id); + mana_adev_idx_free(id); idx_fail: kfree(madev); @@ -3580,7 +3602,9 @@ void mana_rdma_remove(struct gdma_dev *gd) } WRITE_ONCE(gd->rdma_teardown, true); - flush_workqueue(gc->service_wq); + + if (gc->service_wq) + flush_workqueue(gc->service_wq); if (gd->adev) remove_adev(gd); diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 08bee56aea35f..c345d9b17c892 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -2307,14 +2307,16 @@ static void ocelot_set_aggr_pgids(struct ocelot *ocelot) /* Now, set PGIDs for each active LAG */ for (lag = 0; lag < ocelot->num_phys_ports; lag++) { - struct net_device *bond = ocelot->ports[lag]->bond; + struct ocelot_port *ocelot_port = ocelot->ports[lag]; int num_active_ports = 0; + struct net_device *bond; unsigned long bond_mask; u8 aggr_idx[16]; - if (!bond || (visited & BIT(lag))) + if (!ocelot_port || !ocelot_port->bond || (visited & BIT(lag))) continue; + bond = ocelot_port->bond; bond_mask = ocelot_get_bond_mask(ocelot, bond); for_each_set_bit(port, &bond_mask, ocelot->num_phys_ports) { diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 469784d3a1a67..1b82693204640 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -551,44 +551,81 @@ static int ocelot_port_stop(struct net_device *dev) return 0; } -static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +static bool ocelot_xmit_timestamp(struct ocelot *ocelot, int port, + struct sk_buff *skb, u32 *rew_op) { - struct ocelot_port_private *priv = netdev_priv(dev); - struct ocelot_port *ocelot_port = &priv->port; - struct ocelot *ocelot = ocelot_port->ocelot; - int port = priv->port.index; - u32 rew_op = 0; - - if (!static_branch_unlikely(&ocelot_fdma_enabled) && - !ocelot_can_inject(ocelot, 0)) - return NETDEV_TX_BUSY; - - /* Check if timestamping is needed */ if (ocelot->ptp && (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { struct sk_buff *clone = NULL; if (ocelot_port_txtstamp_request(ocelot, port, skb, &clone)) { kfree_skb(skb); - return NETDEV_TX_OK; + return false; } if (clone) OCELOT_SKB_CB(skb)->clone = clone; - rew_op = ocelot_ptp_rew_op(skb); + *rew_op = ocelot_ptp_rew_op(skb); } - if (static_branch_unlikely(&ocelot_fdma_enabled)) { - ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); - } else { - ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + return true; +} + +static netdev_tx_t ocelot_port_xmit_fdma(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) + return NETDEV_TX_OK; + + ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); + + return NETDEV_TX_OK; +} - consume_skb(skb); +static netdev_tx_t ocelot_port_xmit_inj(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + ocelot_lock_inj_grp(ocelot, 0); + + if (!ocelot_can_inject(ocelot, 0)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_BUSY; } + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_OK; + } + + ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + + ocelot_unlock_inj_grp(ocelot, 0); + + consume_skb(skb); + return NETDEV_TX_OK; } +static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +{ + if (static_branch_unlikely(&ocelot_fdma_enabled)) + return ocelot_port_xmit_fdma(skb, dev); + + return ocelot_port_xmit_inj(skb, dev); +} + enum ocelot_action_type { OCELOT_MACT_LEARN, OCELOT_MACT_FORGET, diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index e611ff7fa3fab..2f0cdbd4e2ac9 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -688,6 +688,9 @@ static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) /* probe for IPv6 TSO support */ mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; + cmd.data0 = 0, + cmd.data1 = 0, + cmd.data2 = 0, status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, &cmd, 0); if (status == 0) { @@ -806,6 +809,7 @@ static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, | (addr[2] << 8) | addr[3]); cmd.data1 = ((addr[4] << 8) | (addr[5])); + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); return status; @@ -817,6 +821,9 @@ static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) int status, ctl; ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; + cmd.data0 = 0, + cmd.data1 = 0, + cmd.data2 = 0, status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); if (status) { @@ -834,6 +841,9 @@ myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) int status, ctl; ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); if (status) netdev_err(mgp->dev, "Failed to set promisc mode\n"); @@ -1946,6 +1956,8 @@ static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) /* get ring sizes */ slice = ss - mgp->ss; cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); tx_ring_size = cmd.data0; cmd.data0 = slice; @@ -2238,12 +2250,16 @@ static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) status = 0; if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, &cmd, 0); ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) (mgp->sram + cmd.data0); } cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd, 0); ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) @@ -2312,6 +2328,7 @@ static int myri10ge_open(struct net_device *dev) if (mgp->num_slices > 1) { cmd.data0 = mgp->num_slices; cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; + cmd.data2 = 0; if (mgp->dev->real_num_tx_queues > 1) cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, @@ -2414,6 +2431,8 @@ static int myri10ge_open(struct net_device *dev) /* now give firmware buffers sizes, and MTU */ cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); cmd.data0 = mgp->small_bytes; status |= @@ -2472,7 +2491,6 @@ static int myri10ge_open(struct net_device *dev) static int myri10ge_close(struct net_device *dev) { struct myri10ge_priv *mgp = netdev_priv(dev); - struct myri10ge_cmd cmd; int status, old_down_cnt; int i; @@ -2491,8 +2509,13 @@ static int myri10ge_close(struct net_device *dev) netif_tx_stop_all_queues(dev); if (mgp->rebooted == 0) { + struct myri10ge_cmd cmd; + old_down_cnt = mgp->down_cnt; mb(); + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); if (status) @@ -2956,6 +2979,9 @@ static void myri10ge_set_multicast_list(struct net_device *dev) /* Disable multicast filtering */ + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); if (err != 0) { netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", @@ -3416,10 +3442,6 @@ static void myri10ge_watchdog(struct work_struct *work) * nic was resumed from power saving mode. */ pci_restore_state(mgp->pdev); - - /* save state again for accounting reasons */ - pci_save_state(mgp->pdev); - } else { /* if we get back -1's from our slot, perhaps somebody * powered off our card. Don't try to reset it in diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c index 5026b0263d430..1e55ccb4822b0 100644 --- a/drivers/net/ethernet/neterion/s2io.c +++ b/drivers/net/ethernet/neterion/s2io.c @@ -3425,7 +3425,6 @@ static void s2io_reset(struct s2io_nic *sp) /* Restore the PCI state saved during initialization. */ pci_restore_state(sp->pdev); - pci_save_state(sp->pdev); pci_read_config_word(sp->pdev, 0x2, &val16); if (check_pci_device_id(val16) != (u16)PCI_ANY_ID) break; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 2d9efadb5d2ae..347b0aff100b9 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -263,9 +263,10 @@ static int ionic_get_link_ksettings(struct net_device *netdev, /* This means there's no module plugged in */ break; default: - dev_info(lif->ionic->dev, "unknown xcvr type pid=%d / 0x%x\n", - idev->port_info->status.xcvr.pid, - idev->port_info->status.xcvr.pid); + dev_dbg_ratelimited(lif->ionic->dev, + "unknown xcvr type pid=%d / 0x%x\n", + idev->port_info->status.xcvr.pid, + idev->port_info->status.xcvr.pid); break; } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index b28966ae50c22..29a8a25a3ed08 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -1719,13 +1719,18 @@ static int ionic_set_mac_address(struct net_device *netdev, void *sa) if (ether_addr_equal(netdev->dev_addr, mac)) return 0; - err = ionic_program_mac(lif, mac); - if (err < 0) - return err; + /* Only program macs for virtual functions to avoid losing the permanent + * Mac across warm reset/reboot. + */ + if (lif->ionic->pdev->is_virtfn) { + err = ionic_program_mac(lif, mac); + if (err < 0) + return err; - if (err > 0) - netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n", - __func__); + if (err > 0) + netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n", + __func__); + } err = eth_prepare_mac_addr_change(netdev, addr); if (err) diff --git a/drivers/net/ethernet/qualcomm/qca_uart.c b/drivers/net/ethernet/qualcomm/qca_uart.c index 37efb1ea9fcd9..847a5f928e413 100644 --- a/drivers/net/ethernet/qualcomm/qca_uart.c +++ b/drivers/net/ethernet/qualcomm/qca_uart.c @@ -100,7 +100,7 @@ qca_tty_receive(struct serdev_device *serdev, const u8 *data, size_t count) if (!qca->rx_skb) { netdev_dbg(netdev, "recv: out of RX resources\n"); n_stats->rx_errors++; - return i; + return i + 1; } } } diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 853aabedb128d..d407b031cc2ae 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -2669,9 +2669,6 @@ static void rtl_wol_enable_rx(struct rtl8169_private *tp) static void rtl_prepare_power_down(struct rtl8169_private *tp) { - if (tp->dash_enabled) - return; - if (tp->mac_version == RTL_GIGA_MAC_VER_32 || tp->mac_version == RTL_GIGA_MAC_VER_33) rtl_ephy_write(tp, 0x19, 0xff64); @@ -4807,7 +4804,7 @@ static void rtl8169_down(struct rtl8169_private *tp) rtl_disable_exit_l1(tp); rtl_prepare_power_down(tp); - if (tp->dash_type != RTL_DASH_NONE) + if (tp->dash_type != RTL_DASH_NONE && !tp->saved_wolopts) rtl8168_driver_stop(tp); } diff --git a/drivers/net/ethernet/renesas/rswitch_l2.c b/drivers/net/ethernet/renesas/rswitch_l2.c index 4a69ec77d69c6..9433cd8adced9 100644 --- a/drivers/net/ethernet/renesas/rswitch_l2.c +++ b/drivers/net/ethernet/renesas/rswitch_l2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Renesas Ethernet Switch device driver * - * Copyright (C) 2025 Renesas Electronics Corporation + * Copyright (C) 2025 - 2026 Renesas Electronics Corporation */ #include @@ -60,6 +60,7 @@ static void rswitch_update_l2_hw_learning(struct rswitch_private *priv) static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) { struct rswitch_device *rdev; + bool new_forwarding_offload; unsigned int fwd_mask; /* calculate fwd_mask with zeroes in bits corresponding to ports that @@ -73,8 +74,9 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) } rswitch_for_all_ports(priv, rdev) { - if ((rdev_for_l2_offload(rdev) && rdev->forwarding_requested) || - rdev->forwarding_offloaded) { + new_forwarding_offload = (rdev_for_l2_offload(rdev) && rdev->forwarding_requested); + + if (new_forwarding_offload || rdev->forwarding_offloaded) { /* Update allowed offload destinations even for ports * with L2 offload enabled earlier. * @@ -84,13 +86,10 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) priv->addr + FWPC2(rdev->port)); } - if (rdev_for_l2_offload(rdev) && - rdev->forwarding_requested && - !rdev->forwarding_offloaded) { + if (new_forwarding_offload && !rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, true, false); - } else if (rdev->forwarding_offloaded) { + else if (!new_forwarding_offload && rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, false, false); - } } } diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c index 36af94a2e062a..2794f75df8fcb 100644 --- a/drivers/net/ethernet/rocker/rocker_main.c +++ b/drivers/net/ethernet/rocker/rocker_main.c @@ -1524,9 +1524,8 @@ static void rocker_world_port_post_fini(struct rocker_port *rocker_port) { struct rocker_world_ops *wops = rocker_port->rocker->wops; - if (!wops->port_post_fini) - return; - wops->port_post_fini(rocker_port); + if (wops->port_post_fini) + wops->port_post_fini(rocker_port); kfree(rocker_port->wpriv); } diff --git a/drivers/net/ethernet/sfc/mcdi_filters.c b/drivers/net/ethernet/sfc/mcdi_filters.c index 6ef96292909a2..3db589b90b68a 100644 --- a/drivers/net/ethernet/sfc/mcdi_filters.c +++ b/drivers/net/ethernet/sfc/mcdi_filters.c @@ -2182,12 +2182,7 @@ int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx, int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx) { - int rc; - - mutex_lock(&efx->net_dev->ethtool->rss_lock); - rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context); - mutex_unlock(&efx->net_dev->ethtool->rss_lock); - return rc; + return efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context); } void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx) diff --git a/drivers/net/ethernet/smsc/smc91x.c b/drivers/net/ethernet/smsc/smc91x.c index 9d1a83a5fa7e5..d16c178d10344 100644 --- a/drivers/net/ethernet/smsc/smc91x.c +++ b/drivers/net/ethernet/smsc/smc91x.c @@ -516,15 +516,7 @@ static inline void smc_rcv(struct net_device *dev) * any other concurrent access and C would always interrupt B. But life * isn't that easy in a SMP world... */ -#define smc_special_trylock(lock, flags) \ -({ \ - int __ret; \ - local_irq_save(flags); \ - __ret = spin_trylock(lock); \ - if (!__ret) \ - local_irq_restore(flags); \ - __ret; \ -}) +#define smc_special_trylock(lock, flags) spin_trylock_irqsave(lock, flags) #define smc_special_lock(lock, flags) spin_lock_irqsave(lock, flags) #define smc_special_unlock(lock, flags) spin_unlock_irqrestore(lock, flags) #else diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c index 220eb5ce75833..d64ca7bbda9ea 100644 --- a/drivers/net/ethernet/spacemit/k1_emac.c +++ b/drivers/net/ethernet/spacemit/k1_emac.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -38,7 +39,7 @@ #define EMAC_DEFAULT_BUFSIZE 1536 #define EMAC_RX_BUF_2K 2048 -#define EMAC_RX_BUF_4K 4096 +#define EMAC_RX_BUF_MAX FIELD_MAX(RX_DESC_1_BUFFER_SIZE_1_MASK) /* Tuning parameters from SpacemiT */ #define EMAC_TX_FRAMES 64 @@ -202,8 +203,7 @@ static void emac_init_hw(struct emac_priv *priv) { /* Destination address for 802.3x Ethernet flow control */ u8 fc_dest_addr[ETH_ALEN] = { 0x01, 0x80, 0xc2, 0x00, 0x00, 0x01 }; - - u32 rxirq = 0, dma = 0; + u32 rxirq = 0, dma = 0, frame_sz; regmap_set_bits(priv->regmap_apmu, priv->regmap_apmu_offset + APMU_EMAC_CTRL_REG, @@ -228,6 +228,15 @@ static void emac_init_hw(struct emac_priv *priv) DEFAULT_TX_THRESHOLD); emac_wr(priv, MAC_RECEIVE_PACKET_START_THRESHOLD, DEFAULT_RX_THRESHOLD); + /* Set maximum frame size and jabber size based on configured MTU, + * accounting for Ethernet header, double VLAN tags, and FCS. + */ + frame_sz = priv->ndev->mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; + + emac_wr(priv, MAC_MAXIMUM_FRAME_SIZE, frame_sz); + emac_wr(priv, MAC_TRANSMIT_JABBER_SIZE, frame_sz); + emac_wr(priv, MAC_RECEIVE_JABBER_SIZE, frame_sz); + /* Configure flow control (enabled in emac_adjust_link() later) */ emac_set_mac_addr_reg(priv, fc_dest_addr, MAC_FC_SOURCE_ADDRESS_HIGH); emac_wr(priv, MAC_FC_PAUSE_HIGH_THRESHOLD, DEFAULT_FC_FIFO_HIGH); @@ -573,7 +582,9 @@ static void emac_alloc_rx_desc_buffers(struct emac_priv *priv) DMA_FROM_DEVICE); if (dma_mapping_error(&priv->pdev->dev, rx_buf->dma_addr)) { dev_err_ratelimited(&ndev->dev, "Mapping skb failed\n"); - goto err_free_skb; + dev_kfree_skb_any(skb); + rx_buf->skb = NULL; + break; } rx_desc_addr = &((struct emac_desc *)rx_ring->desc_addr)[i]; @@ -598,10 +609,6 @@ static void emac_alloc_rx_desc_buffers(struct emac_priv *priv) rx_ring->head = i; return; - -err_free_skb: - dev_kfree_skb_any(skb); - rx_buf->skb = NULL; } /* Returns number of packets received */ @@ -743,7 +750,7 @@ static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb) struct emac_desc tx_desc, *tx_desc_addr; struct device *dev = &priv->pdev->dev; struct emac_tx_desc_buffer *tx_buf; - u32 head, old_head, frag_num, f; + u32 head, old_head, frag_num, f, i; bool buf_idx; frag_num = skb_shinfo(skb)->nr_frags; @@ -811,6 +818,15 @@ static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb) err_free_skb: dev_dstats_tx_dropped(priv->ndev); + + i = old_head; + while (i != head) { + emac_free_tx_buf(priv, i); + + if (++i == tx_ring->total_cnt) + i = 0; + } + dev_kfree_skb_any(skb); } @@ -924,14 +940,14 @@ static int emac_change_mtu(struct net_device *ndev, int mtu) return -EBUSY; } - frame_len = mtu + ETH_HLEN + ETH_FCS_LEN; + frame_len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; if (frame_len <= EMAC_DEFAULT_BUFSIZE) priv->dma_buf_sz = EMAC_DEFAULT_BUFSIZE; else if (frame_len <= EMAC_RX_BUF_2K) priv->dma_buf_sz = EMAC_RX_BUF_2K; else - priv->dma_buf_sz = EMAC_RX_BUF_4K; + priv->dma_buf_sz = EMAC_RX_BUF_MAX; ndev->mtu = mtu; @@ -1099,7 +1115,13 @@ static int emac_read_stat_cnt(struct emac_priv *priv, u8 cnt, u32 *res, 100, 10000); if (ret) { - netdev_err(priv->ndev, "Read stat timeout\n"); + /* + * This could be caused by the PHY stopping its refclk even when + * the link is up, for power saving. See also comments in + * emac_stats_update(). + */ + dev_err_ratelimited(&priv->ndev->dev, + "Read stat timeout. PHY clock stopped?\n"); return ret; } @@ -1147,17 +1169,25 @@ static void emac_stats_update(struct emac_priv *priv) assert_spin_locked(&priv->stats_lock); - if (!netif_running(priv->ndev) || !netif_device_present(priv->ndev)) { - /* Not up, don't try to update */ + /* + * We can't read statistics if the interface is not up. Also, some PHYs + * stop their reference clocks for link down power saving, which also + * causes reading statistics to time out. Don't update and don't + * reschedule in these cases. + */ + if (!netif_running(priv->ndev) || + !netif_carrier_ok(priv->ndev) || + !netif_device_present(priv->ndev)) { return; } for (i = 0; i < sizeof(priv->tx_stats) / sizeof(*tx_stats); i++) { /* - * If reading stats times out, everything is broken and there's - * nothing we can do. Reading statistics also can't return an - * error, so just return without updating and without - * rescheduling. + * If reading stats times out anyway, the stat registers will be + * stuck, and we can't really recover from that. + * + * Reading statistics also can't return an error, so just return + * without updating and without rescheduling. */ if (emac_tx_read_stat_cnt(priv, i, &res)) return; @@ -1636,6 +1666,12 @@ static void emac_adjust_link(struct net_device *dev) emac_wr(priv, MAC_GLOBAL_CONTROL, ctrl); emac_set_fc_autoneg(priv); + + /* + * Reschedule stats updates now that link is up. See comments in + * emac_stats_update(). + */ + mod_timer(&priv->stats_timer, jiffies); } phy_print_status(phydev); @@ -2005,7 +2041,7 @@ static int emac_probe(struct platform_device *pdev) ndev->hw_features = NETIF_F_SG; ndev->features |= ndev->hw_features; - ndev->max_mtu = EMAC_RX_BUF_4K - (ETH_HLEN + ETH_FCS_LEN); + ndev->max_mtu = EMAC_RX_BUF_MAX - (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN); ndev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; priv = netdev_priv(ndev); diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c index fb55efd52240d..1c01e3c640cee 100644 --- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c @@ -20,7 +20,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, unsigned int nopaged_len = skb_headlen(skb); struct stmmac_priv *priv = tx_q->priv_data; unsigned int entry = tx_q->cur_tx; - unsigned int bmax, des2; + unsigned int bmax, buf_len, des2; unsigned int i = 1, len; struct dma_desc *desc; @@ -31,17 +31,18 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, else bmax = BUF_SIZE_2KiB; - len = nopaged_len - bmax; + buf_len = min_t(unsigned int, nopaged_len, bmax); + len = nopaged_len - buf_len; des2 = dma_map_single(priv->device, skb->data, - bmax, DMA_TO_DEVICE); + buf_len, DMA_TO_DEVICE); desc->des2 = cpu_to_le32(des2); if (dma_mapping_error(priv->device, des2)) return -1; tx_q->tx_skbuff_dma[entry].buf = des2; - tx_q->tx_skbuff_dma[entry].len = bmax; + tx_q->tx_skbuff_dma[entry].len = buf_len; /* do not close the descriptor and do not set own bit */ - stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum, STMMAC_CHAIN_MODE, + stmmac_prepare_tx_desc(priv, desc, 1, buf_len, csum, STMMAC_CHAIN_MODE, 0, false, skb->len); while (len != 0) { diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index 8f34c9ad457f0..9e012720a69fa 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -43,6 +43,11 @@ #define DWXGMAC_ID 0x76 #define DWXLGMAC_ID 0x27 +static inline bool dwmac_is_xmac(enum dwmac_core_type core_type) +{ + return core_type == DWMAC_CORE_GMAC4 || core_type == DWMAC_CORE_XGMAC; +} + #define STMMAC_CHAN0 0 /* Always supported and default for all chips */ /* TX and RX Descriptor Length, these need to be power of two. @@ -376,7 +381,6 @@ enum request_irq_err { REQ_IRQ_ERR_SFTY, REQ_IRQ_ERR_SFTY_UE, REQ_IRQ_ERR_SFTY_CE, - REQ_IRQ_ERR_LPI, REQ_IRQ_ERR_WOL, REQ_IRQ_ERR_MAC, REQ_IRQ_ERR_NO, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c index e8539cad4602e..1d30f2fb984f1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c @@ -109,7 +109,7 @@ static int dwc_eth_dwmac_config_dt(struct platform_device *pdev, } /* dwc-qos needs GMAC4, AAL, TSO and PMT */ - plat_dat->has_gmac4 = 1; + plat_dat->core_type = DWMAC_CORE_GMAC4; plat_dat->dma_cfg->aal = 1; plat_dat->flags |= STMMAC_FLAG_TSO_EN; plat_dat->pmt = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index e74d00984b889..47fda982d6b15 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -565,7 +565,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat) { /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ plat->clk_csr = STMMAC_CSR_20_35M; - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; plat->mdio_bus_data->needs_reset = true; @@ -612,8 +612,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->pdev = pdev; plat->phy_addr = -1; plat->clk_csr = STMMAC_CSR_250_300M; - plat->has_gmac = 0; - plat->has_gmac4 = 1; + plat->core_type = DWMAC_CORE_GMAC4; plat->force_sf_dma_mode = 0; plat->flags |= (STMMAC_FLAG_TSO_EN | STMMAC_FLAG_SPH_DISABLE); @@ -760,7 +759,6 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, /* Setup MSI vector offset specific to Intel mGbE controller */ plat->msi_mac_vec = 29; - plat->msi_lpi_vec = 28; plat->msi_sfty_ce_vec = 27; plat->msi_sfty_ue_vec = 26; plat->msi_rx_base_vec = 0; @@ -1218,8 +1216,6 @@ static int stmmac_config_multi_msi(struct pci_dev *pdev, res->irq = pci_irq_vector(pdev, plat->msi_mac_vec); if (plat->msi_wol_vec < STMMAC_MSI_VEC_MAX) res->wol_irq = pci_irq_vector(pdev, plat->msi_wol_vec); - if (plat->msi_lpi_vec < STMMAC_MSI_VEC_MAX) - res->lpi_irq = pci_irq_vector(pdev, plat->msi_lpi_vec); if (plat->msi_sfty_ce_vec < STMMAC_MSI_VEC_MAX) res->sfty_ce_irq = pci_irq_vector(pdev, plat->msi_sfty_ce_vec); if (plat->msi_sfty_ue_vec < STMMAC_MSI_VEC_MAX) @@ -1335,7 +1331,6 @@ static int intel_eth_pci_probe(struct pci_dev *pdev, */ plat->msi_mac_vec = STMMAC_MSI_VEC_MAX; plat->msi_wol_vec = STMMAC_MSI_VEC_MAX; - plat->msi_lpi_vec = STMMAC_MSI_VEC_MAX; plat->msi_sfty_ce_vec = STMMAC_MSI_VEC_MAX; plat->msi_sfty_ue_vec = STMMAC_MSI_VEC_MAX; plat->msi_rx_base_vec = STMMAC_MSI_VEC_MAX; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c index ca4035cbb55b6..c05f85534f0ca 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c @@ -473,7 +473,7 @@ static int ipq806x_gmac_probe(struct platform_device *pdev) return err; } - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->bsp_priv = gmac; plat_dat->set_clk_tx_rate = ipq806x_gmac_set_clk_tx_rate; plat_dat->multicast_filter_bins = 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 592aa9d636e50..ab431bf9b25f2 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -90,9 +90,9 @@ static void loongson_default_data(struct pci_dev *pdev, /* Get bus_id, this can be overwritten later */ plat->bus_id = pci_dev_id(pdev); - /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ - plat->clk_csr = STMMAC_CSR_20_35M; - plat->has_gmac = 1; + /* clk_csr_i = 100-150MHz & MDC = clk_csr_i/62 */ + plat->clk_csr = STMMAC_CSR_100_150M; + plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; /* Set default value for multicast hash bins */ @@ -465,13 +465,6 @@ static int loongson_dwmac_dt_config(struct pci_dev *pdev, res->wol_irq = res->irq; } - res->lpi_irq = of_irq_get_byname(np, "eth_lpi"); - if (res->lpi_irq < 0) { - dev_err(&pdev->dev, "IRQ eth_lpi not found\n"); - ret = -ENODEV; - goto err_put_node; - } - ret = device_get_phy_mode(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "phy_mode not found\n"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c index 2562a6d036a2a..6fffc9dfbae55 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c @@ -41,7 +41,7 @@ static int lpc18xx_dwmac_probe(struct platform_device *pdev) if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; reg = syscon_regmap_lookup_by_compatible("nxp,lpc1850-creg"); if (IS_ERR(reg)) { diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index d8fd4d8f6ced7..594e7ff82f2f1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -96,7 +96,6 @@ struct ethqos_emac_driver_data { bool rgmii_config_loopback_en; bool has_emac_ge_3; const char *link_clk_name; - bool has_integrated_pcs; u32 dma_addr_width; struct dwmac4_addrs dwmac4_addrs; bool needs_sgmii_loopback; @@ -282,7 +281,6 @@ static const struct ethqos_emac_driver_data emac_v4_0_0_data = { .rgmii_config_loopback_en = false, .has_emac_ge_3 = true, .link_clk_name = "phyaux", - .has_integrated_pcs = true, .needs_sgmii_loopback = true, .dma_addr_width = 36, .dwmac4_addrs = { @@ -387,14 +385,11 @@ static int ethqos_dll_configure(struct qcom_ethqos *ethqos) static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed) { struct device *dev = ðqos->pdev->dev; - int phase_shift; + int phase_shift = 0; int loopback; /* Determine if the PHY adds a 2 ns TX delay or the MAC handles it */ - if (ethqos->phy_mode == PHY_INTERFACE_MODE_RGMII_ID || - ethqos->phy_mode == PHY_INTERFACE_MODE_RGMII_TXID) - phase_shift = 0; - else + if (ethqos->phy_mode == PHY_INTERFACE_MODE_RGMII_ID) phase_shift = RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN; /* Disable loopback mode */ @@ -789,7 +784,14 @@ static int qcom_ethqos_probe(struct platform_device *pdev) if (!ethqos) return -ENOMEM; + /* Qualcomm configures the MAC to introduce delay; instruct the + * PHY not to add additional delay. + */ + if (plat_dat->phy_interface == PHY_INTERFACE_MODE_RGMII_ID) + plat_dat->phy_interface = PHY_INTERFACE_MODE_RGMII; + ethqos->phy_mode = plat_dat->phy_interface; + switch (ethqos->phy_mode) { case PHY_INTERFACE_MODE_RGMII: case PHY_INTERFACE_MODE_RGMII_ID: @@ -848,7 +850,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->fix_mac_speed = ethqos_fix_mac_speed; plat_dat->dump_debug_regs = rgmii_dump; plat_dat->ptp_clk_freq_config = ethqos_ptp_clk_freq_config; - plat_dat->has_gmac4 = 1; + plat_dat->core_type = DWMAC_CORE_GMAC4; if (ethqos->has_emac_ge_3) plat_dat->dwmac4_addrs = &data->dwmac4_addrs; plat_dat->pmt = 1; @@ -856,8 +858,6 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->flags |= STMMAC_FLAG_TSO_EN; if (of_device_is_compatible(np, "qcom,qcs404-ethqos")) plat_dat->flags |= STMMAC_FLAG_RX_CLK_RUNS_IN_LPI; - if (data->has_integrated_pcs) - plat_dat->flags |= STMMAC_FLAG_HAS_INTEGRATED_PCS; if (data->dma_addr_width) plat_dat->host_dma_width = data->dma_addr_width; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index 0786816e05f04..643578266dfca 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -1751,8 +1751,8 @@ static int rk_gmac_probe(struct platform_device *pdev) /* If the stmmac is not already selected as gmac4, * then make sure we fallback to gmac. */ - if (!plat_dat->has_gmac4) { - plat_dat->has_gmac = true; + if (plat_dat->core_type != DWMAC_CORE_GMAC4) { + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->rx_fifo_size = 4096; plat_dat->tx_fifo_size = 2048; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c index 221539d760bc8..ee095ac132037 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c @@ -146,7 +146,7 @@ static int s32_dwmac_probe(struct platform_device *pdev) gmac->ioaddr = res.addr; /* S32CC core feature set */ - plat->has_gmac4 = true; + plat->core_type = DWMAC_CORE_GMAC4; plat->pmt = 1; plat->flags |= STMMAC_FLAG_SPH_DISABLE; plat->rx_fifo_size = 20480; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c index 354f01184e6cc..2ff5db6d41ca0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c @@ -497,7 +497,7 @@ static int socfpga_dwmac_probe(struct platform_device *pdev) plat_dat->pcs_init = socfpga_dwmac_pcs_init; plat_dat->pcs_exit = socfpga_dwmac_pcs_exit; plat_dat->select_pcs = socfpga_dwmac_select_pcs; - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->riwt_off = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c index 3b7947a7a7ba7..fcdda2401968b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c @@ -7,11 +7,16 @@ #include #include +#include #include #include #include "stmmac_platform.h" +struct sophgo_dwmac_data { + bool has_internal_rx_delay; +}; + static int sophgo_sg2044_dwmac_init(struct platform_device *pdev, struct plat_stmmacenet_data *plat_dat, struct stmmac_resources *stmmac_res) @@ -32,6 +37,7 @@ static int sophgo_sg2044_dwmac_init(struct platform_device *pdev, static int sophgo_dwmac_probe(struct platform_device *pdev) { struct plat_stmmacenet_data *plat_dat; + const struct sophgo_dwmac_data *data; struct stmmac_resources stmmac_res; struct device *dev = &pdev->dev; int ret; @@ -50,11 +56,23 @@ static int sophgo_dwmac_probe(struct platform_device *pdev) if (ret) return ret; + data = device_get_match_data(&pdev->dev); + if (data && data->has_internal_rx_delay) { + plat_dat->phy_interface = phy_fix_phy_mode_for_mac_delays(plat_dat->phy_interface, + false, true); + if (plat_dat->phy_interface == PHY_INTERFACE_MODE_NA) + return -EINVAL; + } + return stmmac_dvr_probe(dev, plat_dat, &stmmac_res); } +static const struct sophgo_dwmac_data sg2042_dwmac_data = { + .has_internal_rx_delay = true, +}; + static const struct of_device_id sophgo_dwmac_match[] = { - { .compatible = "sophgo,sg2042-dwmac" }, + { .compatible = "sophgo,sg2042-dwmac", .data = &sg2042_dwmac_data }, { .compatible = "sophgo,sg2044-dwmac" }, { /* sentinel */ } }; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c index 1eadcf5d1ad63..7f560d78209d1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c @@ -136,7 +136,7 @@ static int sun7i_gmac_probe(struct platform_device *pdev) /* platform data specifying hardware features and callbacks. * hardware features were copied from Allwinner drivers. */ plat_dat->tx_coe = 1; - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->bsp_priv = gmac; plat_dat->init = sun7i_gmac_init; plat_dat->exit = sun7i_gmac_exit; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c index dc903b846b1bf..21a0a11fc0118 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c @@ -9,7 +9,7 @@ #include "stmmac_platform.h" static const char *const mgbe_clks[] = { - "rx-pcs", "tx", "tx-pcs", "mac-divider", "mac", "mgbe", "ptp-ref", "mac" + "rx-pcs", "tx", "tx-pcs", "mac-divider", "mac", "mgbe", "ptp_ref", "mac" }; struct tegra_mgbe { @@ -215,6 +215,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev) { struct plat_stmmacenet_data *plat; struct stmmac_resources res; + bool use_legacy_ptp = false; struct tegra_mgbe *mgbe; int irq, err, i; u32 value; @@ -257,9 +258,23 @@ static int tegra_mgbe_probe(struct platform_device *pdev) if (!mgbe->clks) return -ENOMEM; - for (i = 0; i < ARRAY_SIZE(mgbe_clks); i++) + /* Older device-trees use 'ptp-ref' rather than 'ptp_ref'. + * Fall back when the legacy name is present. + */ + if (of_property_match_string(pdev->dev.of_node, "clock-names", + "ptp-ref") >= 0) + use_legacy_ptp = true; + + for (i = 0; i < ARRAY_SIZE(mgbe_clks); i++) { mgbe->clks[i].id = mgbe_clks[i]; + if (use_legacy_ptp && !strcmp(mgbe_clks[i], "ptp_ref")) { + dev_warn(mgbe->dev, + "Device-tree update needed for PTP clock!\n"); + mgbe->clks[i].id = "ptp-ref"; + } + } + err = devm_clk_bulk_get(mgbe->dev, ARRAY_SIZE(mgbe_clks), mgbe->clks); if (err < 0) return err; @@ -308,7 +323,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev) goto disable_clks; } - plat->has_xgmac = 1; + plat->core_type = DWMAC_CORE_XGMAC; plat->flags |= STMMAC_FLAG_TSO_EN; plat->pmt = 1; plat->bsp_priv = mgbe; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.c b/drivers/net/ethernet/stmicro/stmmac/hwif.c index 3f7c765dcb797..00083ce525492 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.c +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.c @@ -106,9 +106,7 @@ int stmmac_reset(struct stmmac_priv *priv, void __iomem *ioaddr) } static const struct stmmac_hwif_entry { - bool gmac; - bool gmac4; - bool xgmac; + enum dwmac_core_type core_type; u32 min_id; u32 dev_id; const struct stmmac_regs_off regs; @@ -127,9 +125,7 @@ static const struct stmmac_hwif_entry { } stmmac_hw[] = { /* NOTE: New HW versions shall go to the end of this table */ { - .gmac = false, - .gmac4 = false, - .xgmac = false, + .core_type = DWMAC_CORE_MAC100, .min_id = 0, .regs = { .ptp_off = PTP_GMAC3_X_OFFSET, @@ -146,9 +142,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac100_setup, .quirks = stmmac_dwmac1_quirks, }, { - .gmac = true, - .gmac4 = false, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC, .min_id = 0, .regs = { .ptp_off = PTP_GMAC3_X_OFFSET, @@ -165,9 +159,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac1000_setup, .quirks = stmmac_dwmac1_quirks, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = 0, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -187,9 +179,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = stmmac_dwmac4_quirks, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = DWMAC_CORE_4_00, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -210,9 +200,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = DWMAC_CORE_4_10, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -233,9 +221,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = DWMAC_CORE_5_10, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -256,9 +242,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = false, - .xgmac = true, + .core_type = DWMAC_CORE_XGMAC, .min_id = DWXGMAC_CORE_2_10, .dev_id = DWXGMAC_ID, .regs = { @@ -280,9 +264,7 @@ static const struct stmmac_hwif_entry { .setup = dwxgmac2_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = false, - .xgmac = true, + .core_type = DWMAC_CORE_XGMAC, .min_id = DWXLGMAC_CORE_2_00, .dev_id = DWXLGMAC_ID, .regs = { @@ -308,20 +290,18 @@ static const struct stmmac_hwif_entry { int stmmac_hwif_init(struct stmmac_priv *priv) { - bool needs_xgmac = priv->plat->has_xgmac; - bool needs_gmac4 = priv->plat->has_gmac4; - bool needs_gmac = priv->plat->has_gmac; + enum dwmac_core_type core_type = priv->plat->core_type; const struct stmmac_hwif_entry *entry; struct mac_device_info *mac; bool needs_setup = true; u32 id, dev_id = 0; int i, ret; - if (needs_gmac) { + if (core_type == DWMAC_CORE_GMAC) { id = stmmac_get_id(priv, GMAC_VERSION); - } else if (needs_gmac4 || needs_xgmac) { + } else if (dwmac_is_xmac(core_type)) { id = stmmac_get_id(priv, GMAC4_VERSION); - if (needs_xgmac) + if (core_type == DWMAC_CORE_XGMAC) dev_id = stmmac_get_dev_id(priv, GMAC4_VERSION); } else { id = 0; @@ -331,14 +311,16 @@ int stmmac_hwif_init(struct stmmac_priv *priv) priv->synopsys_id = id; /* Lets assume some safe values first */ - priv->ptpaddr = priv->ioaddr + - (needs_gmac4 ? PTP_GMAC4_OFFSET : PTP_GMAC3_X_OFFSET); - priv->mmcaddr = priv->ioaddr + - (needs_gmac4 ? MMC_GMAC4_OFFSET : MMC_GMAC3_X_OFFSET); - if (needs_gmac4) + if (core_type == DWMAC_CORE_GMAC4) { + priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET; + priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET; priv->estaddr = priv->ioaddr + EST_GMAC4_OFFSET; - else if (needs_xgmac) - priv->estaddr = priv->ioaddr + EST_XGMAC_OFFSET; + } else { + priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET; + priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET; + if (core_type == DWMAC_CORE_XGMAC) + priv->estaddr = priv->ioaddr + EST_XGMAC_OFFSET; + } /* Check for HW specific setup first */ if (priv->plat->setup) { @@ -355,16 +337,12 @@ int stmmac_hwif_init(struct stmmac_priv *priv) for (i = ARRAY_SIZE(stmmac_hw) - 1; i >= 0; i--) { entry = &stmmac_hw[i]; - if (needs_gmac ^ entry->gmac) - continue; - if (needs_gmac4 ^ entry->gmac4) - continue; - if (needs_xgmac ^ entry->xgmac) + if (core_type != entry->core_type) continue; /* Use synopsys_id var because some setups can override this */ if (priv->synopsys_id < entry->min_id) continue; - if (needs_xgmac && (dev_id ^ entry->dev_id)) + if (core_type == DWMAC_CORE_XGMAC && (dev_id ^ entry->dev_id)) continue; /* Only use generic HW helpers if needed */ @@ -400,6 +378,7 @@ int stmmac_hwif_init(struct stmmac_priv *priv) } dev_err(priv->device, "Failed to find HW IF (id=0x%x, gmac=%d/%d)\n", - id, needs_gmac, needs_gmac4); + id, core_type == DWMAC_CORE_GMAC, + core_type == DWMAC_CORE_GMAC4); return -EINVAL; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 7ca5477be390b..865531d6cd3b8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -29,7 +29,6 @@ struct stmmac_resources { void __iomem *addr; u8 mac[ETH_ALEN]; int wol_irq; - int lpi_irq; int irq; int sfty_irq; int sfty_ce_irq; @@ -291,7 +290,6 @@ struct stmmac_priv { int wol_irq; u32 gmii_address_bus_config; struct timer_list eee_ctrl_timer; - int lpi_irq; u32 tx_lpi_timer; bool tx_lpi_clk_stop; bool eee_enabled; @@ -318,6 +316,7 @@ struct stmmac_priv { void __iomem *ptpaddr; void __iomem *estaddr; unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + unsigned int num_double_vlans; int sfty_irq; int sfty_ce_irq; int sfty_ue_irq; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c index 4b513d27a9889..afc516059b897 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c @@ -53,7 +53,7 @@ static int est_configure(struct stmmac_priv *priv, struct stmmac_est *cfg, } ctrl = readl(est_addr + EST_CONTROL); - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { ctrl &= ~EST_XGMAC_PTOV; ctrl |= ((NSEC_PER_SEC / ptp_rate) * EST_XGMAC_PTOV_MUL) << EST_XGMAC_PTOV_SHIFT; @@ -148,7 +148,7 @@ static void est_irq_status(struct stmmac_priv *priv, struct net_device *dev, } if (status & EST_BTRE) { - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { btrl = FIELD_GET(EST_XGMAC_BTRL, status); btrl_max = FIELD_MAX(EST_XGMAC_BTRL); } else { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 39fa1ec92f82f..81d4039e1c082 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -303,9 +303,10 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev, { struct stmmac_priv *priv = netdev_priv(dev); - if (priv->plat->has_gmac || priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC || + priv->plat->core_type == DWMAC_CORE_GMAC4) strscpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver)); - else if (priv->plat->has_xgmac) + else if (priv->plat->core_type == DWMAC_CORE_XGMAC) strscpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver)); else strscpy(info->driver, MAC100_ETHTOOL_NAME, @@ -322,47 +323,6 @@ static int stmmac_ethtool_get_link_ksettings(struct net_device *dev, { struct stmmac_priv *priv = netdev_priv(dev); - if (!(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS) && - (priv->hw->pcs & STMMAC_PCS_RGMII || - priv->hw->pcs & STMMAC_PCS_SGMII)) { - u32 supported, advertising, lp_advertising; - - if (!priv->xstats.pcs_link) { - cmd->base.speed = SPEED_UNKNOWN; - cmd->base.duplex = DUPLEX_UNKNOWN; - return 0; - } - cmd->base.duplex = priv->xstats.pcs_duplex; - - cmd->base.speed = priv->xstats.pcs_speed; - - /* Encoding of PSE bits is defined in 802.3z, 37.2.1.4 */ - - ethtool_convert_link_mode_to_legacy_u32( - &supported, cmd->link_modes.supported); - ethtool_convert_link_mode_to_legacy_u32( - &advertising, cmd->link_modes.advertising); - ethtool_convert_link_mode_to_legacy_u32( - &lp_advertising, cmd->link_modes.lp_advertising); - - /* Reg49[3] always set because ANE is always supported */ - cmd->base.autoneg = ADVERTISED_Autoneg; - supported |= SUPPORTED_Autoneg; - advertising |= ADVERTISED_Autoneg; - lp_advertising |= ADVERTISED_Autoneg; - - cmd->base.port = PORT_OTHER; - - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.supported, supported); - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.advertising, advertising); - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.lp_advertising, lp_advertising); - - return 0; - } - return phylink_ethtool_ksettings_get(priv->phylink, cmd); } @@ -372,20 +332,6 @@ stmmac_ethtool_set_link_ksettings(struct net_device *dev, { struct stmmac_priv *priv = netdev_priv(dev); - if (!(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS) && - (priv->hw->pcs & STMMAC_PCS_RGMII || - priv->hw->pcs & STMMAC_PCS_SGMII)) { - /* Only support ANE */ - if (cmd->base.autoneg != AUTONEG_ENABLE) - return -EINVAL; - - mutex_lock(&priv->lock); - stmmac_pcs_ctrl_ane(priv, 1, priv->hw->ps, 0); - mutex_unlock(&priv->lock); - - return 0; - } - return phylink_ethtool_ksettings_set(priv->phylink, cmd); } @@ -406,9 +352,9 @@ static int stmmac_ethtool_get_regs_len(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) return XGMAC_REGSIZE * 4; - else if (priv->plat->has_gmac4) + else if (priv->plat->core_type == DWMAC_CORE_GMAC4) return GMAC4_REG_SPACE_SIZE; return REG_SPACE_SIZE; } @@ -423,12 +369,12 @@ static void stmmac_ethtool_gregs(struct net_device *dev, stmmac_dump_dma_regs(priv, priv->ioaddr, reg_space); /* Copy DMA registers to where ethtool expects them */ - if (priv->plat->has_gmac4) { + if (priv->plat->core_type == DWMAC_CORE_GMAC4) { /* GMAC4 dumps its DMA registers at its DMA_CHAN_BASE_ADDR */ memcpy(®_space[ETHTOOL_DMA_OFFSET], ®_space[GMAC4_DMA_CHAN_BASE_ADDR / 4], NUM_DWMAC4_DMA_REGS * 4); - } else if (!priv->plat->has_xgmac) { + } else if (priv->plat->core_type != DWMAC_CORE_XGMAC) { memcpy(®_space[ETHTOOL_DMA_OFFSET], ®_space[DMA_BUS_MODE / 4], NUM_DWMAC1000_DMA_REGS * 4); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 7b90ecd3a55e6..f8df0609e0e69 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -88,6 +88,7 @@ MODULE_PARM_DESC(phyaddr, "Physical device address"); #define STMMAC_XDP_CONSUMED BIT(0) #define STMMAC_XDP_TX BIT(1) #define STMMAC_XDP_REDIRECT BIT(2) +#define STMMAC_XSK_CONSUMED BIT(3) static int flow_ctrl = 0xdead; module_param(flow_ctrl, int, 0644); @@ -138,6 +139,7 @@ static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, u32 rxmode, u32 chan); +static void stmmac_vlan_restore(struct stmmac_priv *priv); #ifdef CONFIG_DEBUG_FS static const struct net_device_ops stmmac_netdev_ops; @@ -445,7 +447,7 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, if (!priv->hwts_rx_en) return; /* For GMAC4, the valid timestamp is from CTX next desc. */ - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) + if (dwmac_is_xmac(priv->plat->core_type)) desc = np; /* Check if timestamp is available */ @@ -696,7 +698,7 @@ static int stmmac_hwtstamp_get(struct net_device *dev, static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) { - bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + bool xmac = dwmac_is_xmac(priv->plat->core_type); struct timespec64 now; u32 sec_inc = 0; u64 temp = 0; @@ -745,7 +747,7 @@ static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, */ static int stmmac_init_timestamping(struct stmmac_priv *priv) { - bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + bool xmac = dwmac_is_xmac(priv->plat->core_type); int ret; if (priv->plat->ptp_clk_freq_config) @@ -2397,7 +2399,7 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) txfifosz = priv->dma_cap.tx_fifo_size; /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { + if (dwmac_is_xmac(priv->plat->core_type)) { rxfifosz /= rx_channels_count; txfifosz /= tx_channels_count; } @@ -3578,10 +3580,6 @@ static void stmmac_free_irq(struct net_device *dev, free_irq(priv->sfty_ce_irq, dev); fallthrough; case REQ_IRQ_ERR_SFTY_CE: - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) - free_irq(priv->lpi_irq, dev); - fallthrough; - case REQ_IRQ_ERR_LPI: if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) free_irq(priv->wol_irq, dev); fallthrough; @@ -3639,24 +3637,6 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) } } - /* Request the LPI IRQ in case of another line - * is used for LPI - */ - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { - int_name = priv->int_name_lpi; - sprintf(int_name, "%s:%s", dev->name, "lpi"); - ret = request_irq(priv->lpi_irq, - stmmac_mac_interrupt, - 0, int_name, dev); - if (unlikely(ret < 0)) { - netdev_err(priv->dev, - "%s: alloc lpi MSI %d (error: %d)\n", - __func__, priv->lpi_irq, ret); - irq_err = REQ_IRQ_ERR_LPI; - goto irq_error; - } - } - /* Request the common Safety Feature Correctible/Uncorrectible * Error line in case of another line is used */ @@ -3796,19 +3776,6 @@ static int stmmac_request_irq_single(struct net_device *dev) } } - /* Request the IRQ lines */ - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { - ret = request_irq(priv->lpi_irq, stmmac_interrupt, - IRQF_SHARED, dev->name, dev); - if (unlikely(ret < 0)) { - netdev_err(priv->dev, - "%s: ERROR: allocating the LPI IRQ %d (%d)\n", - __func__, priv->lpi_irq, ret); - irq_err = REQ_IRQ_ERR_LPI; - goto irq_error; - } - } - /* Request the common Safety Feature Correctible/Uncorrectible * Error line in case of another line is used */ @@ -3966,6 +3933,8 @@ static int __stmmac_open(struct net_device *dev, /* We may have called phylink_speed_down before */ phylink_speed_up(priv->phylink); + stmmac_vlan_restore(priv); + ret = stmmac_request_irq(dev); if (ret) goto irq_error; @@ -4513,7 +4482,8 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) if (skb_is_gso(skb) && priv->tso) { if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) return stmmac_tso_xmit(skb, dev); - if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) + if (priv->plat->core_type == DWMAC_CORE_GMAC4 && + (gso & SKB_GSO_UDP_L4)) return stmmac_tso_xmit(skb, dev); } @@ -4878,13 +4848,27 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, if (!priv->sph) return 0; - /* Not last descriptor */ - if (status & rx_not_ls) + /* For GMAC4, when split header is enabled, in some rare cases, the + * hardware does not fill buf2 of the first descriptor with payload. + * Thus we cannot assume buf2 is always fully filled if it is not + * the last descriptor. Otherwise, the length of buf2 of the second + * descriptor will be calculated wrong and cause an oops. + * + * If this is the last descriptor, 'plen' is the length of the + * received packet that was transferred to system memory. + * Otherwise, it is the accumulated number of bytes that have been + * transferred for the current packet. + * + * Thus 'plen - len' always gives the correct length of buf2. + */ + + /* Not GMAC4 and not last descriptor */ + if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) return priv->dma_conf.dma_buf_sz; + /* GMAC4 or last descriptor */ plen = stmmac_get_rx_frame_len(priv, p, coe); - /* Last descriptor */ return plen - len; } @@ -4988,6 +4972,7 @@ static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, struct xdp_buff *xdp) { + bool zc = !!(xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL); struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); int cpu = smp_processor_id(); struct netdev_queue *nq; @@ -5004,9 +4989,18 @@ static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, /* Avoids TX time-out as we are sharing with slow path */ txq_trans_cond_update(nq); - res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); - if (res == STMMAC_XDP_TX) + /* For zero copy XDP_TX action, dma_map is true */ + res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, zc); + if (res == STMMAC_XDP_TX) { stmmac_flush_tx_descriptors(priv, queue); + } else if (res == STMMAC_XDP_CONSUMED && zc) { + /* xdp has been freed by xdp_convert_buff_to_frame(), + * no need to call xsk_buff_free() again, so return + * STMMAC_XSK_CONSUMED. + */ + res = STMMAC_XSK_CONSUMED; + xdp_return_frame(xdpf); + } __netif_tx_unlock(nq); @@ -5258,10 +5252,10 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) len = 0; } +read_again: if (count >= limit) break; -read_again: buf1_len = 0; entry = next_entry; buf = &rx_q->buf_pool[entry]; @@ -5356,6 +5350,8 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) break; case STMMAC_XDP_CONSUMED: xsk_buff_free(buf->xdp); + fallthrough; + case STMMAC_XSK_CONSUMED: rx_dropped++; break; case STMMAC_XDP_TX: @@ -5971,7 +5967,7 @@ static void stmmac_common_interrupt(struct stmmac_priv *priv) u32 queue; bool xmac; - xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + xmac = dwmac_is_xmac(priv->plat->core_type); queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; if (priv->irq_wake) @@ -5985,7 +5981,7 @@ static void stmmac_common_interrupt(struct stmmac_priv *priv) stmmac_fpe_irq_status(priv); /* To handle GMAC own interrupts */ - if ((priv->plat->has_gmac) || xmac) { + if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) { int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); if (unlikely(status)) { @@ -5999,15 +5995,6 @@ static void stmmac_common_interrupt(struct stmmac_priv *priv) for (queue = 0; queue < queues_count; queue++) stmmac_host_mtl_irq_status(priv, priv->hw, queue); - /* PCS link status */ - if (priv->hw->pcs && - !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { - if (priv->xstats.pcs_link) - netif_carrier_on(priv->dev); - else - netif_carrier_off(priv->dev); - } - stmmac_timestamp_interrupt(priv, priv); } } @@ -6355,7 +6342,7 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) (priv->dma_cap.mbps_1000) ? "Y" : "N"); seq_printf(seq, "\tHalf duplex: %s\n", (priv->dma_cap.half_duplex) ? "Y" : "N"); - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { seq_printf(seq, "\tNumber of Additional MAC address registers: %d\n", priv->dma_cap.multi_addr); @@ -6379,7 +6366,7 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) (priv->dma_cap.time_stamp) ? "Y" : "N"); seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", (priv->dma_cap.atime_stamp) ? "Y" : "N"); - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) seq_printf(seq, "\tTimestamp System Time Source: %s\n", dwxgmac_timestamp_source[priv->dma_cap.tssrc]); seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", @@ -6388,7 +6375,7 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) seq_printf(seq, "\tChecksum Offload in TX: %s\n", (priv->dma_cap.tx_coe) ? "Y" : "N"); if (priv->synopsys_id >= DWMAC_CORE_4_00 || - priv->plat->has_xgmac) { + priv->plat->core_type == DWMAC_CORE_XGMAC) { seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", (priv->dma_cap.rx_coe) ? "Y" : "N"); } else { @@ -6591,6 +6578,9 @@ static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) hash = 0; } + if (!netif_running(priv->dev)) + return 0; + return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); } @@ -6600,6 +6590,7 @@ static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) { struct stmmac_priv *priv = netdev_priv(ndev); + unsigned int num_double_vlans; bool is_double = false; int ret; @@ -6611,7 +6602,8 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid is_double = true; set_bit(vid, priv->active_vlans); - ret = stmmac_vlan_update(priv, is_double); + num_double_vlans = priv->num_double_vlans + is_double; + ret = stmmac_vlan_update(priv, num_double_vlans); if (ret) { clear_bit(vid, priv->active_vlans); goto err_pm_put; @@ -6619,9 +6611,15 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid if (priv->hw->num_vlan) { ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); - if (ret) + if (ret) { + clear_bit(vid, priv->active_vlans); + stmmac_vlan_update(priv, priv->num_double_vlans); goto err_pm_put; + } } + + priv->num_double_vlans = num_double_vlans; + err_pm_put: pm_runtime_put(priv->device); @@ -6634,6 +6632,7 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) { struct stmmac_priv *priv = netdev_priv(ndev); + unsigned int num_double_vlans; bool is_double = false; int ret; @@ -6645,14 +6644,23 @@ static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vi is_double = true; clear_bit(vid, priv->active_vlans); + num_double_vlans = priv->num_double_vlans - is_double; + ret = stmmac_vlan_update(priv, num_double_vlans); + if (ret) { + set_bit(vid, priv->active_vlans); + goto del_vlan_error; + } if (priv->hw->num_vlan) { ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); - if (ret) + if (ret) { + set_bit(vid, priv->active_vlans); + stmmac_vlan_update(priv, priv->num_double_vlans); goto del_vlan_error; + } } - ret = stmmac_vlan_update(priv, is_double); + priv->num_double_vlans = num_double_vlans; del_vlan_error: pm_runtime_put(priv->device); @@ -6660,6 +6668,17 @@ static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vi return ret; } +static void stmmac_vlan_restore(struct stmmac_priv *priv) +{ + if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) + return; + + if (priv->hw->num_vlan) + stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); + + stmmac_vlan_update(priv, priv->num_double_vlans); +} + static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) { struct stmmac_priv *priv = netdev_priv(dev); @@ -7240,8 +7259,9 @@ static int stmmac_hw_init(struct stmmac_priv *priv) * has to be disable and this can be done by passing the * riwt_off field from the platform. */ - if (((priv->synopsys_id >= DWMAC_CORE_3_50) || - (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { + if ((priv->synopsys_id >= DWMAC_CORE_3_50 || + priv->plat->core_type == DWMAC_CORE_XGMAC) && + !priv->plat->riwt_off) { priv->use_riwt = 1; dev_info(priv->device, "Enable RX Mitigation via HW Watchdog Timer\n"); @@ -7355,7 +7375,7 @@ static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) return -ENODATA; /* For GMAC4, the valid timestamp is from CTX next desc. */ - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) + if (dwmac_is_xmac(priv->plat->core_type)) desc_contains_ts = ndesc; /* Check if timestamp is available */ @@ -7425,7 +7445,6 @@ int stmmac_dvr_probe(struct device *device, priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; - priv->lpi_irq = res->lpi_irq; priv->sfty_irq = res->sfty_irq; priv->sfty_ce_irq = res->sfty_ce_irq; priv->sfty_ue_irq = res->sfty_ue_irq; @@ -7511,7 +7530,7 @@ int stmmac_dvr_probe(struct device *device, if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) ndev->hw_features |= NETIF_F_GSO_UDP_L4; priv->tso = true; dev_info(priv->device, "TSO feature enabled\n"); @@ -7564,7 +7583,7 @@ int stmmac_dvr_probe(struct device *device, #ifdef STMMAC_VLAN_TAG_USED /* Both mac100 and gmac support receive VLAN tag detection */ ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { + if (dwmac_is_xmac(priv->plat->core_type)) { ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; priv->hw->hw_vlan_en = true; } @@ -7592,7 +7611,7 @@ int stmmac_dvr_probe(struct device *device, /* MTU range: 46 - hw-specific max */ ndev->min_mtu = ETH_ZLEN - ETH_HLEN; - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) ndev->max_mtu = XGMAC_JUMBO_LEN; else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) ndev->max_mtu = JUMBO_LEN; @@ -7893,10 +7912,10 @@ int stmmac_resume(struct device *dev) stmmac_init_coalesce(priv); phylink_rx_clk_stop_block(priv->phylink); stmmac_set_rx_mode(ndev); - - stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); phylink_rx_clk_stop_unblock(priv->phylink); + stmmac_vlan_restore(priv); + stmmac_enable_all_queues(priv); stmmac_enable_all_dma_irq(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c index f408737f6fc73..2b55b02de380d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c @@ -301,7 +301,7 @@ static int stmmac_mdio_read_c22(struct mii_bus *bus, int phyaddr, int phyreg) struct stmmac_priv *priv = netdev_priv(bus->priv); u32 cmd; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) cmd = MII_GMAC4_READ; else cmd = 0; @@ -344,7 +344,7 @@ static int stmmac_mdio_write_c22(struct mii_bus *bus, int phyaddr, int phyreg, struct stmmac_priv *priv = netdev_priv(bus->priv); u32 cmd; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) cmd = MII_GMAC4_WRITE; else cmd = MII_ADDR_GWRITE; @@ -417,7 +417,7 @@ int stmmac_mdio_reset(struct mii_bus *bus) * on MDC, so perform a dummy mdio read. To be updated for GMAC4 * if needed. */ - if (!priv->plat->has_gmac4) + if (priv->plat->core_type != DWMAC_CORE_GMAC4) writel(0, priv->ioaddr + mii_address); #endif return 0; @@ -528,7 +528,7 @@ static u32 stmmac_clk_csr_set(struct stmmac_priv *priv) value = 0; } - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { if (clk_rate > 400000000) value = 0x5; else if (clk_rate > 350000000) @@ -600,7 +600,7 @@ int stmmac_mdio_register(struct net_device *ndev) new_bus->name = "stmmac"; - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { new_bus->read = &stmmac_xgmac2_mdio_read_c22; new_bus->write = &stmmac_xgmac2_mdio_write_c22; new_bus->read_c45 = &stmmac_xgmac2_mdio_read_c45; @@ -621,7 +621,7 @@ int stmmac_mdio_register(struct net_device *ndev) } else { new_bus->read = &stmmac_mdio_read_c22; new_bus->write = &stmmac_mdio_write_c22; - if (priv->plat->has_gmac4) { + if (priv->plat->core_type == DWMAC_CORE_GMAC4) { new_bus->read_c45 = &stmmac_mdio_read_c45; new_bus->write_c45 = &stmmac_mdio_write_c45; } @@ -649,7 +649,7 @@ int stmmac_mdio_register(struct net_device *ndev) } /* Looks like we need a dummy read for XGMAC only and C45 PHYs */ - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) stmmac_xgmac2_mdio_read_c45(new_bus, 0, 0, 0); /* If fixed-link is set, skip PHY scanning */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c index 4e3aa611fda83..94b3a3b272706 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c @@ -23,7 +23,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat) { /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ plat->clk_csr = STMMAC_CSR_20_35M; - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; plat->mdio_bus_data->needs_reset = true; @@ -76,7 +76,7 @@ static int snps_gmac5_default_data(struct pci_dev *pdev, int i; plat->clk_csr = STMMAC_CSR_250_300M; - plat->has_gmac4 = 1; + plat->core_type = DWMAC_CORE_GMAC4; plat->force_sf_dma_mode = 1; plat->flags |= STMMAC_FLAG_TSO_EN; plat->pmt = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 27bcaae07a7f2..0cb51935c4056 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -552,12 +552,12 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) &pdev->dev, plat->unicast_filter_entries); plat->multicast_filter_bins = dwmac1000_validate_mcast_bins( &pdev->dev, plat->multicast_filter_bins); - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->pmt = 1; } if (of_device_is_compatible(np, "snps,dwmac-3.40a")) { - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->enh_desc = 1; plat->tx_coe = 1; plat->bugged_jumbo = 1; @@ -565,8 +565,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) } if (of_device_compatible_match(np, stmmac_gmac4_compats)) { - plat->has_gmac4 = 1; - plat->has_gmac = 0; + plat->core_type = DWMAC_CORE_GMAC4; plat->pmt = 1; if (of_property_read_bool(np, "snps,tso")) plat->flags |= STMMAC_FLAG_TSO_EN; @@ -580,7 +579,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) } if (of_device_is_compatible(np, "snps,dwxgmac")) { - plat->has_xgmac = 1; + plat->core_type = DWMAC_CORE_XGMAC; plat->pmt = 1; if (of_property_read_bool(np, "snps,tso")) plat->flags |= STMMAC_FLAG_TSO_EN; @@ -762,14 +761,6 @@ int stmmac_get_platform_resources(struct platform_device *pdev, stmmac_res->wol_irq = stmmac_res->irq; } - stmmac_res->lpi_irq = - platform_get_irq_byname_optional(pdev, "eth_lpi"); - if (stmmac_res->lpi_irq < 0) { - if (stmmac_res->lpi_irq == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_info(&pdev->dev, "IRQ eth_lpi not found\n"); - } - stmmac_res->sfty_irq = platform_get_irq_byname_optional(pdev, "sfty"); if (stmmac_res->sfty_irq < 0) { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 993ff4e87e557..3e30172fa1294 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -57,7 +57,7 @@ static int stmmac_adjust_time(struct ptp_clock_info *ptp, s64 delta) bool xmac, est_rst = false; int ret; - xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + xmac = dwmac_is_xmac(priv->plat->core_type); if (delta < 0) { neg_adj = 1; @@ -344,7 +344,7 @@ void stmmac_ptp_register(struct stmmac_priv *priv) /* Calculate the clock domain crossing (CDC) error if necessary */ priv->plat->cdc_error_adj = 0; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) priv->plat->cdc_error_adj = (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; /* Update the ptp clock parameters based on feature discovery, when diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c index ff02a79c00d4f..e24efe3bfedbe 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c @@ -76,7 +76,9 @@ static int vlan_add_hw_rx_fltr(struct net_device *dev, } hw->vlan_filter[0] = vid; - vlan_write_single(dev, vid); + + if (netif_running(dev)) + vlan_write_single(dev, vid); return 0; } @@ -97,12 +99,15 @@ static int vlan_add_hw_rx_fltr(struct net_device *dev, return -EPERM; } - ret = vlan_write_filter(dev, hw, index, val); + if (netif_running(dev)) { + ret = vlan_write_filter(dev, hw, index, val); + if (ret) + return ret; + } - if (!ret) - hw->vlan_filter[index] = val; + hw->vlan_filter[index] = val; - return ret; + return 0; } static int vlan_del_hw_rx_fltr(struct net_device *dev, @@ -115,33 +120,34 @@ static int vlan_del_hw_rx_fltr(struct net_device *dev, if (hw->num_vlan == 1) { if ((hw->vlan_filter[0] & VLAN_TAG_VID) == vid) { hw->vlan_filter[0] = 0; - vlan_write_single(dev, 0); + + if (netif_running(dev)) + vlan_write_single(dev, 0); } return 0; } /* Extended Rx VLAN Filter Enable */ for (i = 0; i < hw->num_vlan; i++) { - if ((hw->vlan_filter[i] & VLAN_TAG_DATA_VID) == vid) { - ret = vlan_write_filter(dev, hw, i, 0); + if ((hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) && + ((hw->vlan_filter[i] & VLAN_TAG_DATA_VID) == vid)) { + + if (netif_running(dev)) { + ret = vlan_write_filter(dev, hw, i, 0); + if (ret) + return ret; + } - if (!ret) - hw->vlan_filter[i] = 0; - else - return ret; + hw->vlan_filter[i] = 0; } } - return ret; + return 0; } static void vlan_restore_hw_rx_fltr(struct net_device *dev, struct mac_device_info *hw) { - void __iomem *ioaddr = hw->pcsr; - u32 value; - u32 hash; - u32 val; int i; /* Single Rx VLAN Filter */ @@ -151,19 +157,8 @@ static void vlan_restore_hw_rx_fltr(struct net_device *dev, } /* Extended Rx VLAN Filter Enable */ - for (i = 0; i < hw->num_vlan; i++) { - if (hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) { - val = hw->vlan_filter[i]; - vlan_write_filter(dev, hw, i, val); - } - } - - hash = readl(ioaddr + VLAN_HASH_TABLE); - if (hash & VLAN_VLHT) { - value = readl(ioaddr + VLAN_TAG); - value |= VLAN_VTHM; - writel(value, ioaddr + VLAN_TAG); - } + for (i = 0; i < hw->num_vlan; i++) + vlan_write_filter(dev, hw, i, hw->vlan_filter[i]); } static void vlan_update_hash(struct mac_device_info *hw, u32 hash, @@ -182,6 +177,10 @@ static void vlan_update_hash(struct mac_device_info *hw, u32 hash, value |= VLAN_EDVLP; value |= VLAN_ESVL; value |= VLAN_DOVLTC; + } else { + value &= ~VLAN_EDVLP; + value &= ~VLAN_ESVL; + value &= ~VLAN_DOVLTC; } writel(value, ioaddr + VLAN_TAG); @@ -192,6 +191,10 @@ static void vlan_update_hash(struct mac_device_info *hw, u32 hash, value |= VLAN_EDVLP; value |= VLAN_ESVL; value |= VLAN_DOVLTC; + } else { + value &= ~VLAN_EDVLP; + value &= ~VLAN_ESVL; + value &= ~VLAN_DOVLTC; } writel(value | perfect_match, ioaddr + VLAN_TAG); diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c index 48f0a96c0e9e3..6669980829980 100644 --- a/drivers/net/ethernet/sun/sunhme.c +++ b/drivers/net/ethernet/sun/sunhme.c @@ -2551,6 +2551,9 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe) goto err_out_clear_quattro; } + /* BIGMAC may have bogus sizes */ + if ((op->resource[3].end - op->resource[3].start) >= BMAC_REG_SIZE) + op->resource[3].end = op->resource[3].start + BMAC_REG_SIZE - 1; hp->bigmacregs = devm_platform_ioremap_resource(op, 3); if (IS_ERR(hp->bigmacregs)) { dev_err(&op->dev, "Cannot map BIGMAC registers.\n"); diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig index a54d71155263c..c60b04921c62c 100644 --- a/drivers/net/ethernet/ti/Kconfig +++ b/drivers/net/ethernet/ti/Kconfig @@ -192,6 +192,7 @@ config TI_ICSSG_PRUETH depends on NET_SWITCHDEV depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER depends on PTP_1588_CLOCK_OPTIONAL + depends on HSR || !HSR help Support dual Gigabit Ethernet ports over the ICSSG PRU Subsystem. This subsystem is available starting with the AM65 platform. @@ -209,6 +210,7 @@ config TI_ICSSG_PRUETH_SR1 depends on PRU_REMOTEPROC depends on NET_SWITCHDEV depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER + depends on PTP_1588_CLOCK_OPTIONAL help Support dual Gigabit Ethernet ports over the ICSSG PRU Subsystem. This subsystem is available on the AM65 SR1.0 platform. @@ -234,7 +236,7 @@ config TI_PRUETH depends on PRU_REMOTEPROC depends on NET_SWITCHDEV select TI_ICSS_IEP - imply PTP_1588_CLOCK + depends on PTP_1588_CLOCK_OPTIONAL help Some TI SoCs has Programmable Realtime Unit (PRU) cores which can support Single or Dual Ethernet ports with the help of firmware code diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 110eb2da8dbc1..31d436cdceb7c 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -391,7 +391,7 @@ static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev) cpsw_ale_set_allmulti(common->ale, ndev->flags & IFF_ALLMULTI, port->port_id); - port_mask = ALE_PORT_HOST; + port_mask = BIT(port->port_id) | ALE_PORT_HOST; /* Clear all mcast from ALE */ cpsw_ale_flush_multicast(common->ale, port_mask, -1); @@ -1351,7 +1351,7 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow, ndev_priv = netdev_priv(ndev); am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark); skb_put(skb, pkt_len); - if (port->rx_ts_enabled) + if (port->rx_ts_filter) am65_cpts_rx_timestamp(common->cpts, skb); skb_mark_for_recycle(skb); skb->protocol = eth_type_trans(skb, ndev); @@ -1788,34 +1788,37 @@ static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev, } static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, - struct ifreq *ifr) + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct am65_cpsw_port *port = am65_ndev_to_port(ndev); u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype; - struct hwtstamp_config cfg; - if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS)) + if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS)) { + NL_SET_ERR_MSG(extack, "Time stamping is not supported"); return -EOPNOTSUPP; - - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; + } /* TX HW timestamp */ - switch (cfg.tx_type) { + switch (cfg->tx_type) { case HWTSTAMP_TX_OFF: case HWTSTAMP_TX_ON: break; default: + NL_SET_ERR_MSG(extack, "TX mode is not supported"); return -ERANGE; } - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: - port->rx_ts_enabled = false; + port->rx_ts_filter = HWTSTAMP_FILTER_NONE; break; case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + port->rx_ts_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + break; case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: @@ -1825,18 +1828,20 @@ static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_SYNC: case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: - port->rx_ts_enabled = true; - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT | HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + port->rx_ts_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; break; case HWTSTAMP_FILTER_ALL: case HWTSTAMP_FILTER_SOME: case HWTSTAMP_FILTER_NTP_ALL: + NL_SET_ERR_MSG(extack, "RX filter is not supported"); return -EOPNOTSUPP; default: + NL_SET_ERR_MSG(extack, "RX filter is not supported"); return -ERANGE; } - port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON); + port->tx_ts_enabled = (cfg->tx_type == HWTSTAMP_TX_ON); /* cfg TX timestamp */ seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET << @@ -1861,7 +1866,7 @@ static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN | AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN; - if (port->rx_ts_enabled) + if (port->rx_ts_filter) ts_ctrl |= AM65_CPSW_TS_RX_ANX_ALL_EN | AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN; @@ -1872,25 +1877,23 @@ static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2); writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL); - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev, - struct ifreq *ifr) + struct kernel_hwtstamp_config *cfg) { struct am65_cpsw_port *port = am65_ndev_to_port(ndev); - struct hwtstamp_config cfg; if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS)) return -EOPNOTSUPP; - cfg.flags = 0; - cfg.tx_type = port->tx_ts_enabled ? + cfg->flags = 0; + cfg->tx_type = port->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; - cfg.rx_filter = port->rx_ts_enabled ? HWTSTAMP_FILTER_PTP_V2_EVENT | - HWTSTAMP_FILTER_PTP_V1_L4_EVENT : HWTSTAMP_FILTER_NONE; + cfg->rx_filter = port->rx_ts_filter; - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev, @@ -1901,13 +1904,6 @@ static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev, if (!netif_running(ndev)) return -EINVAL; - switch (cmd) { - case SIOCSHWTSTAMP: - return am65_cpsw_nuss_hwtstamp_set(ndev, req); - case SIOCGHWTSTAMP: - return am65_cpsw_nuss_hwtstamp_get(ndev, req); - } - return phylink_mii_ioctl(port->slave.phylink, req, cmd); } @@ -1991,6 +1987,8 @@ static const struct net_device_ops am65_cpsw_nuss_netdev_ops = { .ndo_set_tx_maxrate = am65_cpsw_qos_ndo_tx_p0_set_maxrate, .ndo_bpf = am65_cpsw_ndo_bpf, .ndo_xdp_xmit = am65_cpsw_ndo_xdp_xmit, + .ndo_hwtstamp_get = am65_cpsw_nuss_hwtstamp_get, + .ndo_hwtstamp_set = am65_cpsw_nuss_hwtstamp_set, }; static void am65_cpsw_disable_phy(struct phy *phy) diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.h b/drivers/net/ethernet/ti/am65-cpsw-nuss.h index 917c37e4e89bd..7750448e47468 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h @@ -52,7 +52,7 @@ struct am65_cpsw_port { bool disabled; struct am65_cpsw_slave_data slave; bool tx_ts_enabled; - bool rx_ts_enabled; + enum hwtstamp_rx_filters rx_ts_filter; struct am65_cpsw_qos qos; struct devlink_port devlink_port; struct bpf_prog *xdp_prog; diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index 54c24cd3d3be6..b0e18bdc2c851 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -305,12 +305,19 @@ static int cpsw_purge_all_mc(struct net_device *ndev, const u8 *addr, int num) return 0; } -static void cpsw_ndo_set_rx_mode(struct net_device *ndev) +static void cpsw_ndo_set_rx_mode_work(struct work_struct *work) { - struct cpsw_priv *priv = netdev_priv(ndev); + struct cpsw_priv *priv = container_of(work, struct cpsw_priv, rx_mode_work); struct cpsw_common *cpsw = priv->cpsw; + struct net_device *ndev = priv->ndev; int slave_port = -1; + rtnl_lock(); + if (!netif_running(ndev)) + goto unlock_rtnl; + + netif_addr_lock_bh(ndev); + if (cpsw->data.dual_emac) slave_port = priv->emac_port + 1; @@ -318,7 +325,7 @@ static void cpsw_ndo_set_rx_mode(struct net_device *ndev) /* Enable promiscuous mode */ cpsw_set_promiscious(ndev, true); cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI, slave_port); - return; + goto unlock_addr; } else { /* Disable promiscuous mode */ cpsw_set_promiscious(ndev, false); @@ -331,6 +338,18 @@ static void cpsw_ndo_set_rx_mode(struct net_device *ndev) /* add/remove mcast address either for real netdev or for vlan */ __hw_addr_ref_sync_dev(&ndev->mc, ndev, cpsw_add_mc_addr, cpsw_del_mc_addr); + +unlock_addr: + netif_addr_unlock_bh(ndev); +unlock_rtnl: + rtnl_unlock(); +} + +static void cpsw_ndo_set_rx_mode(struct net_device *ndev) +{ + struct cpsw_priv *priv = netdev_priv(ndev); + + schedule_work(&priv->rx_mode_work); } static unsigned int cpsw_rxbuf_total_len(unsigned int len) @@ -1472,6 +1491,7 @@ static int cpsw_probe_dual_emac(struct cpsw_priv *priv) priv_sl2->ndev = ndev; priv_sl2->dev = &ndev->dev; priv_sl2->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG); + INIT_WORK(&priv_sl2->rx_mode_work, cpsw_ndo_set_rx_mode_work); if (is_valid_ether_addr(data->slave_data[1].mac_addr)) { memcpy(priv_sl2->mac_addr, data->slave_data[1].mac_addr, @@ -1653,6 +1673,7 @@ static int cpsw_probe(struct platform_device *pdev) priv->dev = dev; priv->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG); priv->emac_port = 0; + INIT_WORK(&priv->rx_mode_work, cpsw_ndo_set_rx_mode_work); if (is_valid_ether_addr(data->slave_data[0].mac_addr)) { memcpy(priv->mac_addr, data->slave_data[0].mac_addr, ETH_ALEN); @@ -1758,6 +1779,8 @@ static int cpsw_probe(struct platform_device *pdev) static void cpsw_remove(struct platform_device *pdev) { struct cpsw_common *cpsw = platform_get_drvdata(pdev); + struct net_device *ndev; + struct cpsw_priv *priv; int i, ret; ret = pm_runtime_resume_and_get(&pdev->dev); @@ -1770,9 +1793,15 @@ static void cpsw_remove(struct platform_device *pdev) return; } - for (i = 0; i < cpsw->data.slaves; i++) - if (cpsw->slaves[i].ndev) - unregister_netdev(cpsw->slaves[i].ndev); + for (i = 0; i < cpsw->data.slaves; i++) { + ndev = cpsw->slaves[i].ndev; + if (!ndev) + continue; + + priv = netdev_priv(ndev); + unregister_netdev(ndev); + disable_work_sync(&priv->rx_mode_work); + } cpts_release(cpsw->cpts); cpdma_ctlr_destroy(cpsw->dma); diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c index fbe35af615a6f..9632ad3741de1 100644 --- a/drivers/net/ethernet/ti/cpsw_ale.c +++ b/drivers/net/ethernet/ti/cpsw_ale.c @@ -455,14 +455,13 @@ static void cpsw_ale_flush_mcast(struct cpsw_ale *ale, u32 *ale_entry, ale->port_mask_bits); if ((mask & port_mask) == 0) return; /* ports dont intersect, not interested */ - mask &= ~port_mask; + mask &= (~port_mask | ALE_PORT_HOST); - /* free if only remaining port is host port */ - if (mask) + if (mask == 0x0 || mask == ALE_PORT_HOST) + cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_FREE); + else cpsw_ale_set_port_mask(ale_entry, mask, ale->port_mask_bits); - else - cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_FREE); } int cpsw_ale_flush_multicast(struct cpsw_ale *ale, int port_mask, int vid) diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 8b9e2078c6025..fd3931d667021 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -248,16 +248,22 @@ static int cpsw_purge_all_mc(struct net_device *ndev, const u8 *addr, int num) return 0; } -static void cpsw_ndo_set_rx_mode(struct net_device *ndev) +static void cpsw_ndo_set_rx_mode_work(struct work_struct *work) { - struct cpsw_priv *priv = netdev_priv(ndev); + struct cpsw_priv *priv = container_of(work, struct cpsw_priv, rx_mode_work); struct cpsw_common *cpsw = priv->cpsw; + struct net_device *ndev = priv->ndev; + + rtnl_lock(); + if (!netif_running(ndev)) + goto unlock_rtnl; + netif_addr_lock_bh(ndev); if (ndev->flags & IFF_PROMISC) { /* Enable promiscuous mode */ cpsw_set_promiscious(ndev, true); cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI, priv->emac_port); - return; + goto unlock_addr; } /* Disable promiscuous mode */ @@ -270,6 +276,18 @@ static void cpsw_ndo_set_rx_mode(struct net_device *ndev) /* add/remove mcast address either for real netdev or for vlan */ __hw_addr_ref_sync_dev(&ndev->mc, ndev, cpsw_add_mc_addr, cpsw_del_mc_addr); + +unlock_addr: + netif_addr_unlock_bh(ndev); +unlock_rtnl: + rtnl_unlock(); +} + +static void cpsw_ndo_set_rx_mode(struct net_device *ndev) +{ + struct cpsw_priv *priv = netdev_priv(ndev); + + schedule_work(&priv->rx_mode_work); } static unsigned int cpsw_rxbuf_total_len(unsigned int len) @@ -1398,6 +1416,7 @@ static int cpsw_create_ports(struct cpsw_common *cpsw) priv->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG); priv->emac_port = i + 1; priv->tx_packet_min = CPSW_MIN_PACKET_SIZE; + INIT_WORK(&priv->rx_mode_work, cpsw_ndo_set_rx_mode_work); if (is_valid_ether_addr(slave_data->mac_addr)) { ether_addr_copy(priv->mac_addr, slave_data->mac_addr); @@ -1447,13 +1466,18 @@ static int cpsw_create_ports(struct cpsw_common *cpsw) static void cpsw_unregister_ports(struct cpsw_common *cpsw) { + struct net_device *ndev; + struct cpsw_priv *priv; int i = 0; for (i = 0; i < cpsw->data.slaves; i++) { - if (!cpsw->slaves[i].ndev) + ndev = cpsw->slaves[i].ndev; + if (!ndev || ndev->reg_state != NETREG_REGISTERED) continue; - unregister_netdev(cpsw->slaves[i].ndev); + priv = netdev_priv(ndev); + unregister_netdev(ndev); + disable_work_sync(&priv->rx_mode_work); } } @@ -1470,7 +1494,6 @@ static int cpsw_register_ports(struct cpsw_common *cpsw) if (ret) { dev_err(cpsw->dev, "cpsw: err registering net device%d\n", i); - cpsw->slaves[i].ndev = NULL; break; } } @@ -1977,7 +2000,7 @@ static int cpsw_probe(struct platform_device *pdev) /* setup netdevs */ ret = cpsw_create_ports(cpsw); if (ret) - goto clean_unregister_netdev; + goto clean_cpts; /* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and * MISC IRQs which are always kept disabled with this driver so @@ -1991,14 +2014,14 @@ static int cpsw_probe(struct platform_device *pdev) 0, dev_name(dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } ret = devm_request_irq(dev, cpsw->irqs_table[1], cpsw_tx_interrupt, 0, dev_name(dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } if (!cpsw->cpts) @@ -2008,7 +2031,7 @@ static int cpsw_probe(struct platform_device *pdev) 0, dev_name(&pdev->dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching misc irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } /* Enable misc CPTS evnt_pend IRQ */ @@ -2017,7 +2040,7 @@ static int cpsw_probe(struct platform_device *pdev) skip_cpts: ret = cpsw_register_notifiers(cpsw); if (ret) - goto clean_unregister_netdev; + goto clean_cpts; ret = cpsw_register_devlink(cpsw); if (ret) @@ -2039,8 +2062,6 @@ static int cpsw_probe(struct platform_device *pdev) clean_unregister_notifiers: cpsw_unregister_notifiers(cpsw); -clean_unregister_netdev: - cpsw_unregister_ports(cpsw); clean_cpts: cpts_release(cpsw->cpts); cpdma_ctlr_destroy(cpsw->dma); diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h index 91add8925e235..acb6181c5c9e1 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.h +++ b/drivers/net/ethernet/ti/cpsw_priv.h @@ -391,6 +391,7 @@ struct cpsw_priv { u32 tx_packet_min; struct cpsw_ale_ratelimit ale_bc_ratelimit; struct cpsw_ale_ratelimit ale_mc_ratelimit; + struct work_struct rx_mode_work; }; #define ndev_to_cpsw(ndev) (((struct cpsw_priv *)netdev_priv(ndev))->cpsw) diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.c b/drivers/net/ethernet/ti/icssg/icssg_prueth.c index e42d0fdefee12..07489564270b2 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_prueth.c +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.c @@ -270,6 +270,14 @@ static int prueth_emac_common_start(struct prueth *prueth) if (ret) goto disable_class; + /* Reset link state to force reconfiguration in + * emac_adjust_link(). Without this, if the link was already up + * before restart, emac_adjust_link() won't detect any state + * change and will skip critical configuration like writing + * speed to firmware. + */ + emac->link = 0; + mutex_lock(&emac->ndev->phydev->lock); emac_adjust_link(emac->ndev); mutex_unlock(&emac->ndev->phydev->lock); diff --git a/drivers/net/ethernet/wangxun/Kconfig b/drivers/net/ethernet/wangxun/Kconfig index d138dea7d208d..ec278f99d2955 100644 --- a/drivers/net/ethernet/wangxun/Kconfig +++ b/drivers/net/ethernet/wangxun/Kconfig @@ -21,6 +21,7 @@ config LIBWX depends on PTP_1588_CLOCK_OPTIONAL select PAGE_POOL select DIMLIB + select PHYLINK help Common library for Wangxun(R) Ethernet drivers. @@ -29,7 +30,6 @@ config NGBE depends on PCI depends on PTP_1588_CLOCK_OPTIONAL select LIBWX - select PHYLINK help This driver supports Wangxun(R) GbE PCI Express family of adapters. @@ -48,7 +48,6 @@ config TXGBE depends on PTP_1588_CLOCK_OPTIONAL select MARVELL_10G_PHY select REGMAP - select PHYLINK select HWMON if TXGBE=y select SFP select GPIOLIB @@ -71,7 +70,6 @@ config TXGBEVF depends on PCI_MSI depends on PTP_1588_CLOCK_OPTIONAL select LIBWX - select PHYLINK help This driver supports virtual functions for SP1000A, WX1820AL, WX5XXX, WX5XXXAL. diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c index dc87ccad96524..08b9b426f6484 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c @@ -65,7 +65,7 @@ int txgbe_test_hostif(struct wx *wx) buffer.hdr.cmd_or_resp.cmd_resv = FW_CEM_CMD_RESERVED; return wx_host_interface_command(wx, (u32 *)&buffer, sizeof(buffer), - WX_HI_COMMAND_TIMEOUT, true); + WX_HI_COMMAND_TIMEOUT, false); } static int txgbe_identify_sfp_hostif(struct wx *wx, struct txgbe_hic_i2c_read *buffer) @@ -103,7 +103,7 @@ static int txgbe_set_phy_link_hostif(struct wx *wx, int speed, int autoneg, int buffer.duplex = duplex; return wx_host_interface_command(wx, (u32 *)&buffer, sizeof(buffer), - WX_HI_COMMAND_TIMEOUT, true); + WX_HI_COMMAND_TIMEOUT, false); } static void txgbe_get_link_capabilities(struct wx *wx) diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h index 41915d7dd372a..be78f8f61a795 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h @@ -399,10 +399,10 @@ struct txgbe_nodes { char i2c_name[32]; char sfp_name[32]; char phylink_name[32]; - struct property_entry gpio_props[1]; - struct property_entry i2c_props[3]; - struct property_entry sfp_props[8]; - struct property_entry phylink_props[2]; + struct property_entry gpio_props[2]; + struct property_entry i2c_props[4]; + struct property_entry sfp_props[9]; + struct property_entry phylink_props[3]; struct software_node_ref_args i2c_ref[1]; struct software_node_ref_args gpio0_ref[1]; struct software_node_ref_args gpio1_ref[1]; diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h index 5ff742103beb9..fcd3aaef27fc3 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet.h +++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h @@ -105,7 +105,7 @@ #define XAXIDMA_BD_HAS_DRE_MASK 0xF00 /* Whether has DRE mask */ #define XAXIDMA_BD_WORDLEN_MASK 0xFF /* Whether has DRE mask */ -#define XAXIDMA_BD_CTRL_LENGTH_MASK 0x007FFFFF /* Requested len */ +#define XAXIDMA_BD_CTRL_LENGTH_MASK GENMASK(25, 0) /* Requested len */ #define XAXIDMA_BD_CTRL_TXSOF_MASK 0x08000000 /* First tx packet */ #define XAXIDMA_BD_CTRL_TXEOF_MASK 0x04000000 /* Last tx packet */ #define XAXIDMA_BD_CTRL_ALL_MASK 0x0C000000 /* All control bits */ @@ -130,7 +130,7 @@ #define XAXIDMA_BD_CTRL_TXEOF_MASK 0x04000000 /* Last tx packet */ #define XAXIDMA_BD_CTRL_ALL_MASK 0x0C000000 /* All control bits */ -#define XAXIDMA_BD_STS_ACTUAL_LEN_MASK 0x007FFFFF /* Actual len */ +#define XAXIDMA_BD_STS_ACTUAL_LEN_MASK GENMASK(25, 0) /* Actual len */ #define XAXIDMA_BD_STS_COMPLETE_MASK 0x80000000 /* Completed */ #define XAXIDMA_BD_STS_DEC_ERR_MASK 0x40000000 /* Decode error */ #define XAXIDMA_BD_STS_SLV_ERR_MASK 0x20000000 /* Slave error */ diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index 284031fb2e2c7..eefe54ce66852 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -770,8 +770,8 @@ static int axienet_device_reset(struct net_device *ndev) * @first_bd: Index of first descriptor to clean up * @nr_bds: Max number of descriptors to clean up * @force: Whether to clean descriptors even if not complete - * @sizep: Pointer to a u32 filled with the total sum of all bytes - * in all cleaned-up descriptors. Ignored if NULL. + * @sizep: Pointer to a u32 accumulating the total byte count of + * completed packets (using skb->len). Ignored if NULL. * @budget: NAPI budget (use 0 when not called from NAPI poll) * * Would either be called after a successful transmit operation, or after @@ -805,6 +805,8 @@ static int axienet_free_tx_chain(struct axienet_local *lp, u32 first_bd, DMA_TO_DEVICE); if (cur_p->skb && (status & XAXIDMA_BD_STS_COMPLETE_MASK)) { + if (sizep) + *sizep += cur_p->skb->len; napi_consume_skb(cur_p->skb, budget); packets++; } @@ -818,9 +820,6 @@ static int axienet_free_tx_chain(struct axienet_local *lp, u32 first_bd, wmb(); cur_p->cntrl = 0; cur_p->status = 0; - - if (sizep) - *sizep += status & XAXIDMA_BD_STS_ACTUAL_LEN_MASK; } if (!force) { diff --git a/drivers/net/ethernet/xscale/ixp4xx_eth.c b/drivers/net/ethernet/xscale/ixp4xx_eth.c index e1e7f65553e76..b0faa0f1780d0 100644 --- a/drivers/net/ethernet/xscale/ixp4xx_eth.c +++ b/drivers/net/ethernet/xscale/ixp4xx_eth.c @@ -403,15 +403,12 @@ static int ixp4xx_hwtstamp_set(struct net_device *netdev, int ret; int ch; - if (!cpu_is_ixp46x()) - return -EOPNOTSUPP; - if (!netif_running(netdev)) return -EINVAL; ret = ixp46x_ptp_find(&port->timesync_regs, &port->phc_index); if (ret) - return ret; + return -EOPNOTSUPP; ch = PORT2CHANNEL(port); regs = port->timesync_regs; diff --git a/drivers/net/ethernet/xscale/ptp_ixp46x.c b/drivers/net/ethernet/xscale/ptp_ixp46x.c index 94203eb46e6b0..93c64db22a696 100644 --- a/drivers/net/ethernet/xscale/ptp_ixp46x.c +++ b/drivers/net/ethernet/xscale/ptp_ixp46x.c @@ -232,6 +232,9 @@ static struct ixp_clock ixp_clock; int ixp46x_ptp_find(struct ixp46x_ts_regs *__iomem *regs, int *phc_index) { + if (!cpu_is_ixp46x()) + return -ENODEV; + *regs = ixp_clock.regs; *phc_index = ptp_clock_index(ixp_clock.ptp_clock); diff --git a/drivers/net/fjes/fjes_hw.c b/drivers/net/fjes/fjes_hw.c index b9b5554ea8620..5ad2673f213d6 100644 --- a/drivers/net/fjes/fjes_hw.c +++ b/drivers/net/fjes/fjes_hw.c @@ -334,7 +334,7 @@ int fjes_hw_init(struct fjes_hw *hw) ret = fjes_hw_reset(hw); if (ret) - return ret; + goto err_iounmap; fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, true); @@ -347,8 +347,10 @@ int fjes_hw_init(struct fjes_hw *hw) hw->max_epid = fjes_hw_get_max_epid(hw); hw->my_epid = fjes_hw_get_my_epid(hw); - if ((hw->max_epid == 0) || (hw->my_epid >= hw->max_epid)) - return -ENXIO; + if ((hw->max_epid == 0) || (hw->my_epid >= hw->max_epid)) { + ret = -ENXIO; + goto err_iounmap; + } ret = fjes_hw_setup(hw); @@ -356,6 +358,10 @@ int fjes_hw_init(struct fjes_hw *hw) hw->hw_info.trace_size = FJES_DEBUG_BUFFER_SIZE; return ret; + +err_iounmap: + fjes_hw_iounmap(hw); + return ret; } void fjes_hw_exit(struct fjes_hw *hw) diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 39c892e46cb01..25a358524a096 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -1757,6 +1757,9 @@ static int netvsc_set_rxfh(struct net_device *dev, rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; + if (!ndc->rx_table_sz) + return -EOPNOTSUPP; + rndis_dev = ndev->extension; if (rxfh->indir) { for (i = 0; i < ndc->rx_table_sz; i++) diff --git a/drivers/net/ipa/reg/gsi_reg-v5.0.c b/drivers/net/ipa/reg/gsi_reg-v5.0.c index 36d1e65df71bb..6c4a7fbe4de94 100644 --- a/drivers/net/ipa/reg/gsi_reg-v5.0.c +++ b/drivers/net/ipa/reg/gsi_reg-v5.0.c @@ -30,7 +30,7 @@ REG_STRIDE_FIELDS(CH_C_CNTXT_0, ch_c_cntxt_0, static const u32 reg_ch_c_cntxt_1_fmask[] = { [CH_R_LENGTH] = GENMASK(23, 0), - [ERINDEX] = GENMASK(31, 24), + [CH_ERINDEX] = GENMASK(31, 24), }; REG_STRIDE_FIELDS(CH_C_CNTXT_1, ch_c_cntxt_1, @@ -156,9 +156,10 @@ REG_FIELDS(EV_CH_CMD, ev_ch_cmd, 0x00025010 + 0x12000 * GSI_EE_AP); static const u32 reg_generic_cmd_fmask[] = { [GENERIC_OPCODE] = GENMASK(4, 0), - [GENERIC_CHID] = GENMASK(9, 5), - [GENERIC_EE] = GENMASK(13, 10), - /* Bits 14-31 reserved */ + [GENERIC_CHID] = GENMASK(12, 5), + [GENERIC_EE] = GENMASK(16, 13), + /* Bits 17-23 reserved */ + [GENERIC_PARAMS] = GENMASK(31, 24), }; REG_FIELDS(GENERIC_CMD, generic_cmd, 0x00025018 + 0x12000 * GSI_EE_AP); diff --git a/drivers/net/ipvlan/ipvlan.h b/drivers/net/ipvlan/ipvlan.h index 50de3ee204dbc..80f84fc87008b 100644 --- a/drivers/net/ipvlan/ipvlan.h +++ b/drivers/net/ipvlan/ipvlan.h @@ -69,7 +69,6 @@ struct ipvl_dev { DECLARE_BITMAP(mac_filters, IPVLAN_MAC_FILTER_SIZE); netdev_features_t sfeatures; u32 msg_enable; - spinlock_t addrs_lock; }; struct ipvl_addr { @@ -90,6 +89,7 @@ struct ipvl_port { struct net_device *dev; possible_net_t pnet; struct hlist_head hlhead[IPVLAN_HASH_SIZE]; + spinlock_t addrs_lock; /* guards hash-table and addrs */ struct list_head ipvlans; u16 mode; u16 flags; diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c index d7e3ddbcab6f4..6c6677ded82ef 100644 --- a/drivers/net/ipvlan/ipvlan_core.c +++ b/drivers/net/ipvlan/ipvlan_core.c @@ -107,17 +107,15 @@ void ipvlan_ht_addr_del(struct ipvl_addr *addr) struct ipvl_addr *ipvlan_find_addr(const struct ipvl_dev *ipvlan, const void *iaddr, bool is_v6) { - struct ipvl_addr *addr, *ret = NULL; + struct ipvl_addr *addr; - rcu_read_lock(); - list_for_each_entry_rcu(addr, &ipvlan->addrs, anode) { - if (addr_equal(is_v6, addr, iaddr)) { - ret = addr; - break; - } + assert_spin_locked(&ipvlan->port->addrs_lock); + + list_for_each_entry(addr, &ipvlan->addrs, anode) { + if (addr_equal(is_v6, addr, iaddr)) + return addr; } - rcu_read_unlock(); - return ret; + return NULL; } bool ipvlan_addr_busy(struct ipvl_port *port, void *iaddr, bool is_v6) @@ -737,6 +735,9 @@ static rx_handler_result_t ipvlan_handle_mode_l2(struct sk_buff **pskb, struct ethhdr *eth = eth_hdr(skb); rx_handler_result_t ret = RX_HANDLER_PASS; + if (unlikely(skb->pkt_type == PACKET_LOOPBACK)) + return RX_HANDLER_PASS; + if (is_multicast_ether_addr(eth->h_dest)) { if (ipvlan_external_frame(skb, port)) { struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index 660f3db117664..baccdad695fda 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -75,6 +75,7 @@ static int ipvlan_port_create(struct net_device *dev) for (idx = 0; idx < IPVLAN_HASH_SIZE; idx++) INIT_HLIST_HEAD(&port->hlhead[idx]); + spin_lock_init(&port->addrs_lock); skb_queue_head_init(&port->backlog); INIT_WORK(&port->wq, ipvlan_process_multicast); ida_init(&port->ida); @@ -181,6 +182,7 @@ static void ipvlan_uninit(struct net_device *dev) static int ipvlan_open(struct net_device *dev) { struct ipvl_dev *ipvlan = netdev_priv(dev); + struct ipvl_port *port = ipvlan->port; struct ipvl_addr *addr; if (ipvlan->port->mode == IPVLAN_MODE_L3 || @@ -189,10 +191,10 @@ static int ipvlan_open(struct net_device *dev) else dev->flags &= ~IFF_NOARP; - rcu_read_lock(); - list_for_each_entry_rcu(addr, &ipvlan->addrs, anode) + spin_lock_bh(&port->addrs_lock); + list_for_each_entry(addr, &ipvlan->addrs, anode) ipvlan_ht_addr_add(ipvlan, addr); - rcu_read_unlock(); + spin_unlock_bh(&port->addrs_lock); return 0; } @@ -206,10 +208,10 @@ static int ipvlan_stop(struct net_device *dev) dev_uc_unsync(phy_dev, dev); dev_mc_unsync(phy_dev, dev); - rcu_read_lock(); - list_for_each_entry_rcu(addr, &ipvlan->addrs, anode) + spin_lock_bh(&ipvlan->port->addrs_lock); + list_for_each_entry(addr, &ipvlan->addrs, anode) ipvlan_ht_addr_del(addr); - rcu_read_unlock(); + spin_unlock_bh(&ipvlan->port->addrs_lock); return 0; } @@ -579,7 +581,6 @@ int ipvlan_link_new(struct net_device *dev, struct rtnl_newlink_params *params, if (!tb[IFLA_MTU]) ipvlan_adjust_mtu(ipvlan, phy_dev); INIT_LIST_HEAD(&ipvlan->addrs); - spin_lock_init(&ipvlan->addrs_lock); /* TODO Probably put random address here to be presented to the * world but keep using the physical-dev address for the outgoing @@ -657,13 +658,13 @@ void ipvlan_link_delete(struct net_device *dev, struct list_head *head) struct ipvl_dev *ipvlan = netdev_priv(dev); struct ipvl_addr *addr, *next; - spin_lock_bh(&ipvlan->addrs_lock); + spin_lock_bh(&ipvlan->port->addrs_lock); list_for_each_entry_safe(addr, next, &ipvlan->addrs, anode) { ipvlan_ht_addr_del(addr); list_del_rcu(&addr->anode); kfree_rcu(addr, rcu); } - spin_unlock_bh(&ipvlan->addrs_lock); + spin_unlock_bh(&ipvlan->port->addrs_lock); ida_free(&ipvlan->port->ida, dev->dev_id); list_del_rcu(&ipvlan->pnode); @@ -817,6 +818,8 @@ static int ipvlan_add_addr(struct ipvl_dev *ipvlan, void *iaddr, bool is_v6) { struct ipvl_addr *addr; + assert_spin_locked(&ipvlan->port->addrs_lock); + addr = kzalloc(sizeof(struct ipvl_addr), GFP_ATOMIC); if (!addr) return -ENOMEM; @@ -847,16 +850,16 @@ static void ipvlan_del_addr(struct ipvl_dev *ipvlan, void *iaddr, bool is_v6) { struct ipvl_addr *addr; - spin_lock_bh(&ipvlan->addrs_lock); + spin_lock_bh(&ipvlan->port->addrs_lock); addr = ipvlan_find_addr(ipvlan, iaddr, is_v6); if (!addr) { - spin_unlock_bh(&ipvlan->addrs_lock); + spin_unlock_bh(&ipvlan->port->addrs_lock); return; } ipvlan_ht_addr_del(addr); list_del_rcu(&addr->anode); - spin_unlock_bh(&ipvlan->addrs_lock); + spin_unlock_bh(&ipvlan->port->addrs_lock); kfree_rcu(addr, rcu); } @@ -878,14 +881,14 @@ static int ipvlan_add_addr6(struct ipvl_dev *ipvlan, struct in6_addr *ip6_addr) { int ret = -EINVAL; - spin_lock_bh(&ipvlan->addrs_lock); + spin_lock_bh(&ipvlan->port->addrs_lock); if (ipvlan_addr_busy(ipvlan->port, ip6_addr, true)) netif_err(ipvlan, ifup, ipvlan->dev, "Failed to add IPv6=%pI6c addr for %s intf\n", ip6_addr, ipvlan->dev->name); else ret = ipvlan_add_addr(ipvlan, ip6_addr, true); - spin_unlock_bh(&ipvlan->addrs_lock); + spin_unlock_bh(&ipvlan->port->addrs_lock); return ret; } @@ -924,21 +927,24 @@ static int ipvlan_addr6_validator_event(struct notifier_block *unused, struct in6_validator_info *i6vi = (struct in6_validator_info *)ptr; struct net_device *dev = (struct net_device *)i6vi->i6vi_dev->dev; struct ipvl_dev *ipvlan = netdev_priv(dev); + int ret = NOTIFY_OK; if (!ipvlan_is_valid_dev(dev)) return NOTIFY_DONE; switch (event) { case NETDEV_UP: + spin_lock_bh(&ipvlan->port->addrs_lock); if (ipvlan_addr_busy(ipvlan->port, &i6vi->i6vi_addr, true)) { NL_SET_ERR_MSG(i6vi->extack, "Address already assigned to an ipvlan device"); - return notifier_from_errno(-EADDRINUSE); + ret = notifier_from_errno(-EADDRINUSE); } + spin_unlock_bh(&ipvlan->port->addrs_lock); break; } - return NOTIFY_OK; + return ret; } #endif @@ -946,14 +952,14 @@ static int ipvlan_add_addr4(struct ipvl_dev *ipvlan, struct in_addr *ip4_addr) { int ret = -EINVAL; - spin_lock_bh(&ipvlan->addrs_lock); + spin_lock_bh(&ipvlan->port->addrs_lock); if (ipvlan_addr_busy(ipvlan->port, ip4_addr, false)) netif_err(ipvlan, ifup, ipvlan->dev, "Failed to add IPv4=%pI4 on %s intf.\n", ip4_addr, ipvlan->dev->name); else ret = ipvlan_add_addr(ipvlan, ip4_addr, false); - spin_unlock_bh(&ipvlan->addrs_lock); + spin_unlock_bh(&ipvlan->port->addrs_lock); return ret; } @@ -995,21 +1001,24 @@ static int ipvlan_addr4_validator_event(struct notifier_block *unused, struct in_validator_info *ivi = (struct in_validator_info *)ptr; struct net_device *dev = (struct net_device *)ivi->ivi_dev->dev; struct ipvl_dev *ipvlan = netdev_priv(dev); + int ret = NOTIFY_OK; if (!ipvlan_is_valid_dev(dev)) return NOTIFY_DONE; switch (event) { case NETDEV_UP: + spin_lock_bh(&ipvlan->port->addrs_lock); if (ipvlan_addr_busy(ipvlan->port, &ivi->ivi_addr, false)) { NL_SET_ERR_MSG(ivi->extack, "Address already assigned to an ipvlan device"); - return notifier_from_errno(-EADDRINUSE); + ret = notifier_from_errno(-EADDRINUSE); } + spin_unlock_bh(&ipvlan->port->addrs_lock); break; } - return NOTIFY_OK; + return ret; } static struct notifier_block ipvlan_addr4_notifier_block __read_mostly = { diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 7966545512cfe..4433b8e95b6ac 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -59,7 +59,7 @@ struct macvlan_port { struct macvlan_source_entry { struct hlist_node hlist; - struct macvlan_dev *vlan; + struct macvlan_dev __rcu *vlan; unsigned char addr[6+2] __aligned(sizeof(u16)); struct rcu_head rcu; }; @@ -146,7 +146,7 @@ static struct macvlan_source_entry *macvlan_hash_lookup_source( hlist_for_each_entry_rcu(entry, h, hlist, lockdep_rtnl_is_held()) { if (ether_addr_equal_64bits(entry->addr, addr) && - entry->vlan == vlan) + rcu_access_pointer(entry->vlan) == vlan) return entry; } return NULL; @@ -168,7 +168,7 @@ static int macvlan_hash_add_source(struct macvlan_dev *vlan, return -ENOMEM; ether_addr_copy(entry->addr, addr); - entry->vlan = vlan; + RCU_INIT_POINTER(entry->vlan, vlan); h = &port->vlan_source_hash[macvlan_eth_hash(addr)]; hlist_add_head_rcu(&entry->hlist, h); vlan->macaddr_count++; @@ -187,6 +187,7 @@ static void macvlan_hash_add(struct macvlan_dev *vlan) static void macvlan_hash_del_source(struct macvlan_source_entry *entry) { + RCU_INIT_POINTER(entry->vlan, NULL); hlist_del_rcu(&entry->hlist); kfree_rcu(entry, rcu); } @@ -390,7 +391,7 @@ static void macvlan_flush_sources(struct macvlan_port *port, int i; hash_for_each_safe(port->vlan_source_hash, i, next, entry, hlist) - if (entry->vlan == vlan) + if (rcu_access_pointer(entry->vlan) == vlan) macvlan_hash_del_source(entry); vlan->macaddr_count = 0; @@ -433,9 +434,14 @@ static bool macvlan_forward_source(struct sk_buff *skb, hlist_for_each_entry_rcu(entry, h, hlist) { if (ether_addr_equal_64bits(entry->addr, addr)) { - if (entry->vlan->flags & MACVLAN_FLAG_NODST) + struct macvlan_dev *vlan = rcu_dereference(entry->vlan); + + if (!vlan) + continue; + + if (vlan->flags & MACVLAN_FLAG_NODST) consume = true; - macvlan_forward_source_one(skb, entry->vlan); + macvlan_forward_source_one(skb, vlan); } } @@ -1561,10 +1567,16 @@ int macvlan_common_newlink(struct net_device *dev, /* the macvlan port may be freed by macvlan_uninit when fail to register. * so we destroy the macvlan port only when it's valid. */ - if (create && macvlan_port_get_rtnl(lowerdev)) { + if (macvlan_port_get_rtnl(lowerdev)) { macvlan_flush_sources(port, vlan); - macvlan_port_destroy(port->dev); + if (create) + macvlan_port_destroy(port->dev); } + /* @dev might have been made visible before an error was detected. + * Make sure to observe an RCU grace period before our caller + * (rtnl_newlink()) frees it. + */ + synchronize_net(); return err; } EXPORT_SYMBOL_GPL(macvlan_common_newlink); @@ -1680,7 +1692,7 @@ static int macvlan_fill_info_macaddr(struct sk_buff *skb, struct macvlan_source_entry *entry; hlist_for_each_entry_rcu(entry, h, hlist, lockdep_rtnl_is_held()) { - if (entry->vlan != vlan) + if (rcu_access_pointer(entry->vlan) != vlan) continue; if (nla_put(skb, IFLA_MACVLAN_MACADDR, ETH_ALEN, entry->addr)) return 1; diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c index f782d93f826ef..f138b0251313e 100644 --- a/drivers/net/mctp/mctp-i2c.c +++ b/drivers/net/mctp/mctp-i2c.c @@ -242,6 +242,12 @@ static int mctp_i2c_slave_cb(struct i2c_client *client, return 0; switch (event) { + case I2C_SLAVE_READ_REQUESTED: + case I2C_SLAVE_READ_PROCESSED: + /* MCTP I2C transport only uses writes */ + midev->rx_pos = 0; + *val = 0xff; + break; case I2C_SLAVE_WRITE_RECEIVED: if (midev->rx_pos < MCTP_I2C_BUFSZ) { midev->rx_buffer[midev->rx_pos] = *val; @@ -279,6 +285,9 @@ static int mctp_i2c_recv(struct mctp_i2c_dev *midev) size_t recvlen; int status; + if (midev->rx_pos == 0) + return 0; + /* + 1 for the PEC */ if (midev->rx_pos < MCTP_I2C_MINLEN + 1) { ndev->stats.rx_length_errors++; @@ -334,6 +343,7 @@ static int mctp_i2c_recv(struct mctp_i2c_dev *midev) } else { status = NET_RX_DROP; spin_unlock_irqrestore(&midev->lock, flags); + kfree_skb(skb); } if (status == NET_RX_SUCCESS) { diff --git a/drivers/net/mctp/mctp-usb.c b/drivers/net/mctp/mctp-usb.c index ef860cfc629f6..3b5dff1441774 100644 --- a/drivers/net/mctp/mctp-usb.c +++ b/drivers/net/mctp/mctp-usb.c @@ -329,7 +329,7 @@ static int mctp_usb_probe(struct usb_interface *intf, SET_NETDEV_DEV(netdev, &intf->dev); dev = netdev_priv(netdev); dev->netdev = netdev; - dev->usbdev = usb_get_dev(interface_to_usbdev(intf)); + dev->usbdev = interface_to_usbdev(intf); dev->intf = intf; usb_set_intfdata(intf, dev); @@ -365,7 +365,6 @@ static void mctp_usb_disconnect(struct usb_interface *intf) mctp_unregister_netdev(dev->netdev); usb_free_urb(dev->tx_urb); usb_free_urb(dev->rx_urb); - usb_put_dev(dev->usbdev); free_netdev(dev->netdev); } diff --git a/drivers/net/mdio/mdio-aspeed.c b/drivers/net/mdio/mdio-aspeed.c index e55be6dc9ae70..d6b9004c61dc1 100644 --- a/drivers/net/mdio/mdio-aspeed.c +++ b/drivers/net/mdio/mdio-aspeed.c @@ -63,6 +63,13 @@ static int aspeed_mdio_op(struct mii_bus *bus, u8 st, u8 op, u8 phyad, u8 regad, iowrite32(ctrl, ctx->base + ASPEED_MDIO_CTRL); + /* Workaround for read-after-write issue. + * The controller may return stale data if a read follows immediately + * after a write. A dummy read forces the hardware to update its + * internal state, ensuring that the next real read returns correct data. + */ + ioread32(ctx->base + ASPEED_MDIO_CTRL); + return readl_poll_timeout(ctx->base + ASPEED_MDIO_CTRL, ctrl, !(ctrl & ASPEED_MDIO_CTRL_FIRE), ASPEED_MDIO_INTERVAL_US, diff --git a/drivers/net/mdio/mdio-realtek-rtl9300.c b/drivers/net/mdio/mdio-realtek-rtl9300.c index 33694c3ff9a71..8d5fb014ca06c 100644 --- a/drivers/net/mdio/mdio-realtek-rtl9300.c +++ b/drivers/net/mdio/mdio-realtek-rtl9300.c @@ -354,7 +354,6 @@ static int rtl9300_mdiobus_probe_one(struct device *dev, struct rtl9300_mdio_pri struct fwnode_handle *node) { struct rtl9300_mdio_chan *chan; - struct fwnode_handle *child; struct mii_bus *bus; u32 mdio_bus; int err; @@ -371,7 +370,7 @@ static int rtl9300_mdiobus_probe_one(struct device *dev, struct rtl9300_mdio_pri * compatible = "ethernet-phy-ieee802.3-c45". This does mean we can't * support both c45 and c22 on the same MDIO bus. */ - fwnode_for_each_child_node(node, child) + fwnode_for_each_child_node_scoped(node, child) if (fwnode_device_is_compatible(child, "ethernet-phy-ieee802.3-c45")) priv->smi_bus_is_c45[mdio_bus] = true; @@ -409,7 +408,6 @@ static int rtl9300_mdiobus_map_ports(struct device *dev) { struct rtl9300_mdio_priv *priv = dev_get_drvdata(dev); struct device *parent = dev->parent; - struct fwnode_handle *port; int err; struct fwnode_handle *ports __free(fwnode_handle) = @@ -418,7 +416,7 @@ static int rtl9300_mdiobus_map_ports(struct device *dev) return dev_err_probe(dev, -EINVAL, "%pfwP missing ethernet-ports\n", dev_fwnode(parent)); - fwnode_for_each_child_node(ports, port) { + fwnode_for_each_child_node_scoped(ports, port) { struct device_node *mdio_dn; u32 addr; u32 bus; @@ -468,7 +466,6 @@ static int rtl9300_mdiobus_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct rtl9300_mdio_priv *priv; - struct fwnode_handle *child; int err; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -489,7 +486,7 @@ static int rtl9300_mdiobus_probe(struct platform_device *pdev) if (err) return err; - device_for_each_child_node(dev, child) { + device_for_each_child_node_scoped(dev, child) { err = rtl9300_mdiobus_probe_one(dev, priv, child); if (err) return err; diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index bb6e03a929565..7288b25d6b8c4 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -503,7 +503,7 @@ static ssize_t sysdata_release_enabled_show(struct config_item *item, bool release_enabled; mutex_lock(&dynamic_netconsole_mutex); - release_enabled = !!(nt->sysdata_fields & SYSDATA_TASKNAME); + release_enabled = !!(nt->sysdata_fields & SYSDATA_RELEASE); mutex_unlock(&dynamic_netconsole_mutex); return sysfs_emit(buf, "%d\n", release_enabled); @@ -1537,7 +1537,8 @@ static void send_msg_no_fragmentation(struct netconsole_target *nt, if (release_len) { release = init_utsname()->release; - scnprintf(nt->buf, MAX_PRINT_CHUNK, "%s,%s", release, msg); + scnprintf(nt->buf, MAX_PRINT_CHUNK, "%s,%.*s", release, + msg_len, msg); msg_len += release_len; } else { memcpy(nt->buf, msg, msg_len); diff --git a/drivers/net/netdevsim/bpf.c b/drivers/net/netdevsim/bpf.c index 49537d3c41205..5f17f68f3c083 100644 --- a/drivers/net/netdevsim/bpf.c +++ b/drivers/net/netdevsim/bpf.c @@ -244,7 +244,9 @@ static int nsim_bpf_create_prog(struct nsim_dev *nsim_dev, &state->state, &nsim_bpf_string_fops); debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded); + mutex_lock(&nsim_dev->progs_list_lock); list_add_tail(&state->l, &nsim_dev->bpf_bound_progs); + mutex_unlock(&nsim_dev->progs_list_lock); prog->aux->offload->dev_priv = state; @@ -273,12 +275,16 @@ static int nsim_bpf_translate(struct bpf_prog *prog) static void nsim_bpf_destroy_prog(struct bpf_prog *prog) { struct nsim_bpf_bound_prog *state; + struct nsim_dev *nsim_dev; state = prog->aux->offload->dev_priv; + nsim_dev = state->nsim_dev; WARN(state->is_loaded, "offload state destroyed while program still bound"); debugfs_remove_recursive(state->ddir); + mutex_lock(&nsim_dev->progs_list_lock); list_del(&state->l); + mutex_unlock(&nsim_dev->progs_list_lock); kfree(state); } diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c index 70e8c38ddad6b..d16b95304aa7e 100644 --- a/drivers/net/netdevsim/bus.c +++ b/drivers/net/netdevsim/bus.c @@ -332,6 +332,11 @@ static ssize_t link_device_store(const struct bus_type *bus, const char *buf, si rcu_assign_pointer(nsim_a->peer, nsim_b); rcu_assign_pointer(nsim_b->peer, nsim_a); + if (netif_running(dev_a) && netif_running(dev_b)) { + netif_carrier_on(dev_a); + netif_carrier_on(dev_b); + } + out_err: put_net(ns_b); put_net(ns_a); @@ -381,6 +386,9 @@ static ssize_t unlink_device_store(const struct bus_type *bus, const char *buf, if (!peer) goto out_put_netns; + netif_carrier_off(dev); + netif_carrier_off(peer->netdev); + err = 0; RCU_INIT_POINTER(nsim->peer, NULL); RCU_INIT_POINTER(peer->peer, NULL); diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index 95f66c1f59db8..56a47c060f2e1 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -1591,6 +1591,7 @@ int nsim_drv_probe(struct nsim_bus_dev *nsim_bus_dev) nsim_dev->max_macs = NSIM_DEV_MAX_MACS_DEFAULT; nsim_dev->test1 = NSIM_DEV_TEST1_DEFAULT; spin_lock_init(&nsim_dev->fa_cookie_lock); + mutex_init(&nsim_dev->progs_list_lock); dev_set_drvdata(&nsim_bus_dev->dev, nsim_dev); @@ -1729,6 +1730,7 @@ void nsim_drv_remove(struct nsim_bus_dev *nsim_bus_dev) devl_unregister(devlink); kfree(nsim_dev->vfconfigs); kfree(nsim_dev->fa_cookie); + mutex_destroy(&nsim_dev->progs_list_lock); devl_unlock(devlink); devlink_free(devlink); dev_set_drvdata(&nsim_bus_dev->dev, NULL); diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index fa1d97885caaf..06446b03cd9bc 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -109,8 +109,11 @@ static int nsim_forward_skb(struct net_device *tx_dev, int ret; ret = __dev_forward_skb(rx_dev, skb); - if (ret) + if (ret) { + if (psp_ext) + __skb_ext_put(psp_ext); return ret; + } nsim_psp_handle_ext(skb, psp_ext); diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h index 02c1c97b70080..d91c0899e536e 100644 --- a/drivers/net/netdevsim/netdevsim.h +++ b/drivers/net/netdevsim/netdevsim.h @@ -319,6 +319,7 @@ struct nsim_dev { u32 prog_id_gen; struct list_head bpf_bound_progs; struct list_head bpf_bound_maps; + struct mutex progs_list_lock; struct netdev_phys_item_id switch_id; struct list_head port_list; bool fw_update_status; diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c index 3e9e7f8444b34..955c9a37e1f8d 100644 --- a/drivers/net/ovpn/io.c +++ b/drivers/net/ovpn/io.c @@ -355,6 +355,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) struct ovpn_priv *ovpn = netdev_priv(dev); struct sk_buff *segments, *curr, *next; struct sk_buff_head skb_list; + unsigned int tx_bytes = 0; struct ovpn_peer *peer; __be16 proto; int ret; @@ -365,7 +366,27 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) /* verify IP header size in network packet */ proto = ovpn_ip_check_protocol(skb); if (unlikely(!proto || skb->protocol != proto)) - goto drop; + goto drop_no_peer; + + /* retrieve peer serving the destination IP of this packet */ + peer = ovpn_peer_get_by_dst(ovpn, skb); + if (unlikely(!peer)) { + switch (skb->protocol) { + case htons(ETH_P_IP): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", + netdev_name(ovpn->dev), + &ip_hdr(skb)->daddr); + break; + case htons(ETH_P_IPV6): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", + netdev_name(ovpn->dev), + &ipv6_hdr(skb)->daddr); + break; + } + goto drop_no_peer; + } + /* dst was needed for peer selection - it can now be dropped */ + skb_dst_drop(skb); if (skb_is_gso(skb)) { segments = skb_gso_segment(skb, 0); @@ -394,36 +415,28 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) continue; } + /* only count what we actually send */ + tx_bytes += curr->len; __skb_queue_tail(&skb_list, curr); } - skb_list.prev->next = NULL; - /* retrieve peer serving the destination IP of this packet */ - peer = ovpn_peer_get_by_dst(ovpn, skb); - if (unlikely(!peer)) { - switch (skb->protocol) { - case htons(ETH_P_IP): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", - netdev_name(ovpn->dev), - &ip_hdr(skb)->daddr); - break; - case htons(ETH_P_IPV6): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", - netdev_name(ovpn->dev), - &ipv6_hdr(skb)->daddr); - break; - } - goto drop; + /* no segments survived: don't jump to 'drop' because we already + * incremented the counter for each failure in the loop + */ + if (unlikely(skb_queue_empty(&skb_list))) { + ovpn_peer_put(peer); + return NETDEV_TX_OK; } - /* dst was needed for peer selection - it can now be dropped */ - skb_dst_drop(skb); + skb_list.prev->next = NULL; - ovpn_peer_stats_increment_tx(&peer->vpn_stats, skb->len); + ovpn_peer_stats_increment_tx(&peer->vpn_stats, tx_bytes); ovpn_send(ovpn, skb_list.next, peer); return NETDEV_TX_OK; drop: + ovpn_peer_put(peer); +drop_no_peer: dev_dstats_tx_dropped(ovpn->dev); skb_tx_error(skb); kfree_skb_list(skb); diff --git a/drivers/net/ovpn/socket.c b/drivers/net/ovpn/socket.c index 9750871ab65ce..448cee3b3f9fa 100644 --- a/drivers/net/ovpn/socket.c +++ b/drivers/net/ovpn/socket.c @@ -200,6 +200,22 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ovpn_sock->sk = sk; kref_init(&ovpn_sock->refcount); + /* TCP sockets are per-peer, therefore they are linked to their unique + * peer + */ + if (sk->sk_protocol == IPPROTO_TCP) { + INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); + ovpn_sock->peer = peer; + ovpn_peer_hold(peer); + } else if (sk->sk_protocol == IPPROTO_UDP) { + /* in UDP we only link the ovpn instance since the socket is + * shared among multiple peers + */ + ovpn_sock->ovpn = peer->ovpn; + netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, + GFP_KERNEL); + } + /* the newly created ovpn_socket is holding reference to sk, * therefore we increase its refcounter. * @@ -212,29 +228,16 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ret = ovpn_socket_attach(ovpn_sock, sock, peer); if (ret < 0) { + if (sk->sk_protocol == IPPROTO_TCP) + ovpn_peer_put(peer); + else if (sk->sk_protocol == IPPROTO_UDP) + netdev_put(peer->ovpn->dev, &ovpn_sock->dev_tracker); + sock_put(sk); kfree(ovpn_sock); ovpn_sock = ERR_PTR(ret); - goto sock_release; - } - - /* TCP sockets are per-peer, therefore they are linked to their unique - * peer - */ - if (sk->sk_protocol == IPPROTO_TCP) { - INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); - ovpn_sock->peer = peer; - ovpn_peer_hold(peer); - } else if (sk->sk_protocol == IPPROTO_UDP) { - /* in UDP we only link the ovpn instance since the socket is - * shared among multiple peers - */ - ovpn_sock->ovpn = peer->ovpn; - netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, - GFP_KERNEL); } - rcu_assign_sk_user_data(sk, ovpn_sock); sock_release: release_sock(sk); return ovpn_sock; diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c index 0d7f30360d874..5499c1572f3e2 100644 --- a/drivers/net/ovpn/tcp.c +++ b/drivers/net/ovpn/tcp.c @@ -70,37 +70,56 @@ static void ovpn_tcp_to_userspace(struct ovpn_peer *peer, struct sock *sk, peer->tcp.sk_cb.sk_data_ready(sk); } -static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) +static struct sk_buff *ovpn_tcp_skb_packet(const struct ovpn_peer *peer, + struct sk_buff *orig_skb, + const int pkt_len, const int pkt_off) { - struct ovpn_peer *peer = container_of(strp, struct ovpn_peer, tcp.strp); - struct strp_msg *msg = strp_msg(skb); - size_t pkt_len = msg->full_len - 2; - size_t off = msg->offset + 2; - u8 opcode; + struct sk_buff *ovpn_skb; + int err; - /* ensure skb->data points to the beginning of the openvpn packet */ - if (!pskb_pull(skb, off)) { - net_warn_ratelimited("%s: packet too small for peer %u\n", - netdev_name(peer->ovpn->dev), peer->id); + /* create a new skb with only the content of the current packet */ + ovpn_skb = netdev_alloc_skb(peer->ovpn->dev, pkt_len); + if (unlikely(!ovpn_skb)) goto err; - } - /* strparser does not trim the skb for us, therefore we do it now */ - if (pskb_trim(skb, pkt_len) != 0) { - net_warn_ratelimited("%s: trimming skb failed for peer %u\n", + skb_copy_header(ovpn_skb, orig_skb); + err = skb_copy_bits(orig_skb, pkt_off, skb_put(ovpn_skb, pkt_len), + pkt_len); + if (unlikely(err)) { + net_warn_ratelimited("%s: skb_copy_bits failed for peer %u\n", netdev_name(peer->ovpn->dev), peer->id); + kfree_skb(ovpn_skb); goto err; } - /* we need the first 4 bytes of data to be accessible + consume_skb(orig_skb); + return ovpn_skb; +err: + kfree_skb(orig_skb); + return NULL; +} + +static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) +{ + struct ovpn_peer *peer = container_of(strp, struct ovpn_peer, tcp.strp); + struct strp_msg *msg = strp_msg(skb); + int pkt_len = msg->full_len - 2; + u8 opcode; + + /* we need at least 4 bytes of data in the packet * to extract the opcode and the key ID later on */ - if (!pskb_may_pull(skb, OVPN_OPCODE_SIZE)) { + if (unlikely(pkt_len < OVPN_OPCODE_SIZE)) { net_warn_ratelimited("%s: packet too small to fetch opcode for peer %u\n", netdev_name(peer->ovpn->dev), peer->id); goto err; } + /* extract the packet into a new skb */ + skb = ovpn_tcp_skb_packet(peer, skb, pkt_len, msg->offset + 2); + if (unlikely(!skb)) + goto err; + /* DATA_V2 packets are handled in kernel, the rest goes to user space */ opcode = ovpn_opcode_from_skb(skb, 0); if (unlikely(opcode != OVPN_DATA_V2)) { @@ -113,7 +132,7 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) /* The packet size header must be there when sending the packet * to userspace, therefore we put it back */ - skb_push(skb, 2); + *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(pkt_len); ovpn_tcp_to_userspace(peer, strp->sk, skb); return; } @@ -199,7 +218,19 @@ void ovpn_tcp_socket_detach(struct ovpn_socket *ovpn_sock) sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; sk->sk_prot = peer->tcp.sk_cb.prot; - sk->sk_socket->ops = peer->tcp.sk_cb.ops; + + /* tcp_close() may race this function and could set + * sk->sk_socket to NULL. It does so by invoking + * sock_orphan(), which holds sk_callback_lock before + * doing the assignment. + * + * For this reason we acquire the same lock to avoid + * sk_socket to disappear under our feet + */ + write_lock_bh(&sk->sk_callback_lock); + if (sk->sk_socket) + sk->sk_socket->ops = peer->tcp.sk_cb.ops; + write_unlock_bh(&sk->sk_callback_lock); rcu_assign_sk_user_data(sk, NULL); } @@ -487,6 +518,7 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, /* make sure no pre-existing encapsulation handler exists */ if (ovpn_sock->sk->sk_user_data) return -EBUSY; + rcu_assign_sk_user_data(ovpn_sock->sk, ovpn_sock); /* only a fully connected socket is expected. Connection should be * handled in userspace @@ -495,13 +527,14 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, net_err_ratelimited("%s: provided TCP socket is not in ESTABLISHED state: %d\n", netdev_name(peer->ovpn->dev), ovpn_sock->sk->sk_state); - return -EINVAL; + ret = -EINVAL; + goto err; } ret = strp_init(&peer->tcp.strp, ovpn_sock->sk, &cb); if (ret < 0) { DEBUG_NET_WARN_ON_ONCE(1); - return ret; + goto err; } INIT_WORK(&peer->tcp.defer_del_work, ovpn_tcp_peer_del_work); @@ -536,6 +569,9 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, strp_check_rcv(&peer->tcp.strp); return 0; +err: + rcu_assign_sk_user_data(ovpn_sock->sk, NULL); + return ret; } static void ovpn_tcp_close(struct sock *sk, long timeout) diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c index d6a0f7a0b75d7..272b535ecaad4 100644 --- a/drivers/net/ovpn/udp.c +++ b/drivers/net/ovpn/udp.c @@ -386,6 +386,7 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock, struct ovpn_priv *ovpn) { struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = ovpn_sock, .encap_type = UDP_ENCAP_OVPNINUDP, .encap_rcv = ovpn_udp_encap_recv, .encap_destroy = ovpn_udp_encap_destroy, diff --git a/drivers/net/pcs/pcs-mtk-lynxi.c b/drivers/net/pcs/pcs-mtk-lynxi.c index 149ddf51d7856..87df3a9dfc9bc 100644 --- a/drivers/net/pcs/pcs-mtk-lynxi.c +++ b/drivers/net/pcs/pcs-mtk-lynxi.c @@ -93,12 +93,10 @@ static unsigned int mtk_pcs_lynxi_inband_caps(struct phylink_pcs *pcs, { switch (interface) { case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: case PHY_INTERFACE_MODE_SGMII: return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE; - case PHY_INTERFACE_MODE_2500BASEX: - return LINK_INBAND_DISABLE; - default: return 0; } diff --git a/drivers/net/phy/adin1100.c b/drivers/net/phy/adin1100.c index bd7a47a903aca..10b796c2daee7 100644 --- a/drivers/net/phy/adin1100.c +++ b/drivers/net/phy/adin1100.c @@ -201,7 +201,7 @@ static int adin_set_powerdown_mode(struct phy_device *phydev, bool en) return ret; return phy_read_mmd_poll_timeout(phydev, MDIO_MMD_VEND1, ADIN_CRSM_STAT, ret, - (ret & ADIN_CRSM_SFT_PD_RDY) == val, + !!(ret & ADIN_CRSM_SFT_PD_RDY) == en, 1000, 30000, true); } diff --git a/drivers/net/phy/aquantia/aquantia_firmware.c b/drivers/net/phy/aquantia/aquantia_firmware.c index bbbcc9736b00e..569256152689f 100644 --- a/drivers/net/phy/aquantia/aquantia_firmware.c +++ b/drivers/net/phy/aquantia/aquantia_firmware.c @@ -369,7 +369,7 @@ int aqr_firmware_load(struct phy_device *phydev) * assume that, and load a new image. */ ret = aqr_firmware_load_nvmem(phydev); - if (!ret) + if (ret == -EPROBE_DEFER || !ret) return ret; ret = aqr_firmware_load_fs(phydev); diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c index 9766dd99afaa0..12ff4c1f285d2 100644 --- a/drivers/net/phy/intel-xway.c +++ b/drivers/net/phy/intel-xway.c @@ -277,7 +277,7 @@ static int xway_gphy_init_leds(struct phy_device *phydev) static int xway_gphy_config_init(struct phy_device *phydev) { - struct device_node *np = phydev->mdio.dev.of_node; + struct device_node *np; int err; /* Mask all interrupts */ @@ -286,7 +286,10 @@ static int xway_gphy_config_init(struct phy_device *phydev) return err; /* Use default LED configuration if 'leds' node isn't defined */ - if (!of_get_child_by_name(np, "leds")) + np = of_get_child_by_name(phydev->mdio.dev.of_node, "leds"); + if (np) + of_node_put(np); + else xway_gphy_init_leds(phydev); /* Clear all pending interrupts */ diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c index f3d83b04c9535..201dee1a16985 100644 --- a/drivers/net/phy/marvell-88q2xxx.c +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -698,7 +698,7 @@ static int mv88q2xxx_hwmon_write(struct device *dev, switch (attr) { case hwmon_temp_max: - clamp_val(val, -75000, 180000); + val = clamp(val, -75000, 180000); val = (val / 1000) + 75; val = FIELD_PREP(MDIO_MMD_PCS_MV_TEMP_SENSOR3_INT_THRESH_MASK, val); diff --git a/drivers/net/phy/mediatek/mtk-ge-soc.c b/drivers/net/phy/mediatek/mtk-ge-soc.c index cd09fbf92ef23..2c4bbc236202b 100644 --- a/drivers/net/phy/mediatek/mtk-ge-soc.c +++ b/drivers/net/phy/mediatek/mtk-ge-soc.c @@ -1167,9 +1167,9 @@ static int mt798x_phy_calibration(struct phy_device *phydev) } buf = (u32 *)nvmem_cell_read(cell, &len); + nvmem_cell_put(cell); if (IS_ERR(buf)) return PTR_ERR(buf); - nvmem_cell_put(cell); if (!buf[0] || !buf[1] || !buf[2] || !buf[3] || len < 4 * sizeof(u32)) { phydev_err(phydev, "invalid efuse data\n"); diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 01c87c9b77020..bc19880107ae4 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -2541,11 +2541,21 @@ static int kszphy_probe(struct phy_device *phydev) kszphy_parse_led_mode(phydev); - clk = devm_clk_get_optional_enabled(&phydev->mdio.dev, "rmii-ref"); + clk = devm_clk_get_optional(&phydev->mdio.dev, "rmii-ref"); /* NOTE: clk may be NULL if building without CONFIG_HAVE_CLK */ if (!IS_ERR_OR_NULL(clk)) { - unsigned long rate = clk_get_rate(clk); bool rmii_ref_clk_sel_25_mhz; + unsigned long rate; + int err; + + err = clk_prepare_enable(clk); + if (err) { + phydev_err(phydev, "Failed to enable rmii-ref clock\n"); + return err; + } + + rate = clk_get_rate(clk); + clk_disable_unprepare(clk); if (type) priv->rmii_ref_clk_sel = type->has_rmii_ref_clk_sel; @@ -2563,13 +2573,12 @@ static int kszphy_probe(struct phy_device *phydev) } } else if (!clk) { /* unnamed clock from the generic ethernet-phy binding */ - clk = devm_clk_get_optional_enabled(&phydev->mdio.dev, NULL); + clk = devm_clk_get_optional(&phydev->mdio.dev, NULL); } if (IS_ERR(clk)) return PTR_ERR(clk); - clk_disable_unprepare(clk); priv->clk = clk; if (ksz8041_fiber_mode(phydev)) diff --git a/drivers/net/phy/motorcomm.c b/drivers/net/phy/motorcomm.c index a3593e6630594..b49897500a592 100644 --- a/drivers/net/phy/motorcomm.c +++ b/drivers/net/phy/motorcomm.c @@ -1741,10 +1741,10 @@ static int yt8521_led_hw_control_set(struct phy_device *phydev, u8 index, val |= YT8521_LED_1000_ON_EN; if (test_bit(TRIGGER_NETDEV_FULL_DUPLEX, &rules)) - val |= YT8521_LED_HDX_ON_EN; + val |= YT8521_LED_FDX_ON_EN; if (test_bit(TRIGGER_NETDEV_HALF_DUPLEX, &rules)) - val |= YT8521_LED_FDX_ON_EN; + val |= YT8521_LED_HDX_ON_EN; if (test_bit(TRIGGER_NETDEV_TX, &rules) || test_bit(TRIGGER_NETDEV_RX, &rules)) diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c index ef0ef1570d392..48d43f60b8ff8 100644 --- a/drivers/net/phy/mscc/mscc_main.c +++ b/drivers/net/phy/mscc/mscc_main.c @@ -2625,7 +2625,7 @@ static struct phy_driver vsc85xx_driver[] = { .suspend = &genphy_suspend, .resume = &genphy_resume, .remove = &vsc85xx_remove, - .probe = &vsc8574_probe, + .probe = &vsc8584_probe, .set_wol = &vsc85xx_wol_set, .get_wol = &vsc85xx_wol_get, .get_tunable = &vsc85xx_get_tunable, @@ -2648,12 +2648,12 @@ static struct phy_driver vsc85xx_driver[] = { .config_aneg = &vsc85xx_config_aneg, .aneg_done = &genphy_aneg_done, .read_status = &vsc85xx_read_status, - .handle_interrupt = vsc85xx_handle_interrupt, + .handle_interrupt = vsc8584_handle_interrupt, .config_intr = &vsc85xx_config_intr, .suspend = &genphy_suspend, .resume = &genphy_resume, .remove = &vsc85xx_remove, - .probe = &vsc8574_probe, + .probe = &vsc8584_probe, .set_wol = &vsc85xx_wol_set, .get_wol = &vsc85xx_wol_get, .get_tunable = &vsc85xx_get_tunable, diff --git a/drivers/net/phy/mxl-86110.c b/drivers/net/phy/mxl-86110.c index e5d137a37a1d4..42a5fe3f115f4 100644 --- a/drivers/net/phy/mxl-86110.c +++ b/drivers/net/phy/mxl-86110.c @@ -938,6 +938,9 @@ static struct phy_driver mxl_phy_drvs[] = { PHY_ID_MATCH_EXACT(PHY_ID_MXL86110), .name = "MXL86110 Gigabit Ethernet", .config_init = mxl86110_config_init, + .suspend = genphy_suspend, + .resume = genphy_resume, + .soft_reset = genphy_soft_reset, .get_wol = mxl86110_get_wol, .set_wol = mxl86110_set_wol, .led_brightness_set = mxl86110_led_brightness_set, diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c index 605ca20ae192d..0c63e6ba2cb0c 100644 --- a/drivers/net/phy/phy-core.c +++ b/drivers/net/phy/phy-core.c @@ -101,6 +101,49 @@ const char *phy_rate_matching_to_str(int rate_matching) } EXPORT_SYMBOL_GPL(phy_rate_matching_to_str); +/** + * phy_fix_phy_mode_for_mac_delays - Convenience function for fixing PHY + * mode based on whether mac adds internal delay + * + * @interface: The current interface mode of the port + * @mac_txid: True if the mac adds internal tx delay + * @mac_rxid: True if the mac adds internal rx delay + * + * Return: fixed PHY mode, or PHY_INTERFACE_MODE_NA if the interface can + * not apply the internal delay + */ +phy_interface_t phy_fix_phy_mode_for_mac_delays(phy_interface_t interface, + bool mac_txid, bool mac_rxid) +{ + if (!phy_interface_mode_is_rgmii(interface)) + return interface; + + if (mac_txid && mac_rxid) { + if (interface == PHY_INTERFACE_MODE_RGMII_ID) + return PHY_INTERFACE_MODE_RGMII; + return PHY_INTERFACE_MODE_NA; + } + + if (mac_txid) { + if (interface == PHY_INTERFACE_MODE_RGMII_ID) + return PHY_INTERFACE_MODE_RGMII_RXID; + if (interface == PHY_INTERFACE_MODE_RGMII_TXID) + return PHY_INTERFACE_MODE_RGMII; + return PHY_INTERFACE_MODE_NA; + } + + if (mac_rxid) { + if (interface == PHY_INTERFACE_MODE_RGMII_ID) + return PHY_INTERFACE_MODE_RGMII_TXID; + if (interface == PHY_INTERFACE_MODE_RGMII_RXID) + return PHY_INTERFACE_MODE_RGMII; + return PHY_INTERFACE_MODE_NA; + } + + return interface; +} +EXPORT_SYMBOL_GPL(phy_fix_phy_mode_for_mac_delays); + /** * phy_interface_num_ports - Return the number of links that can be carried by * a given MAC-PHY physical link. Returns 0 if this is diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 7a67c900e79a5..2353d6eced68d 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1751,8 +1751,6 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, goto error; phy_resume(phydev); - if (!phydev->is_on_sfp_module) - phy_led_triggers_register(phydev); /** * If the external phy used by current mac interface is managed by @@ -1867,9 +1865,6 @@ void phy_detach(struct phy_device *phydev) phydev->phy_link_change = NULL; phydev->phylink = NULL; - if (!phydev->is_on_sfp_module) - phy_led_triggers_unregister(phydev); - if (phydev->mdio.dev.driver) module_put(phydev->mdio.dev.driver->owner); @@ -3500,16 +3495,27 @@ static int phy_probe(struct device *dev) /* Set the state to READY by default */ phydev->state = PHY_READY; + /* Register the PHY LED triggers */ + if (!phydev->is_on_sfp_module) + phy_led_triggers_register(phydev); + /* Get the LEDs from the device tree, and instantiate standard * LEDs for them. */ - if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) + if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) { err = of_phy_leds(phydev); + if (err) + goto out; + } + + return 0; out: + if (!phydev->is_on_sfp_module) + phy_led_triggers_unregister(phydev); + /* Re-assert the reset signal on error */ - if (err) - phy_device_reset(phydev, 1); + phy_device_reset(phydev, 1); return err; } @@ -3523,6 +3529,9 @@ static int phy_remove(struct device *dev) if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) phy_leds_unregister(phydev); + if (!phydev->is_on_sfp_module) + phy_led_triggers_unregister(phydev); + phydev->state = PHY_DOWN; sfp_bus_del_upstream(phydev->sfp_bus); diff --git a/drivers/net/phy/qcom/qca807x.c b/drivers/net/phy/qcom/qca807x.c index 1be8295a95cb5..511cde345e089 100644 --- a/drivers/net/phy/qcom/qca807x.c +++ b/drivers/net/phy/qcom/qca807x.c @@ -375,7 +375,7 @@ static int qca807x_gpio_get(struct gpio_chip *gc, unsigned int offset) reg = QCA807X_MMD7_LED_FORCE_CTRL(offset); val = phy_read_mmd(priv->phy, MDIO_MMD_AN, reg); - return FIELD_GET(QCA807X_GPIO_FORCE_MODE_MASK, val); + return !!FIELD_GET(QCA807X_GPIO_FORCE_MODE_MASK, val); } static int qca807x_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) diff --git a/drivers/net/phy/qcom/qca808x.c b/drivers/net/phy/qcom/qca808x.c index 8eb51b1a006c4..f22ebb1ac3328 100644 --- a/drivers/net/phy/qcom/qca808x.c +++ b/drivers/net/phy/qcom/qca808x.c @@ -643,6 +643,15 @@ static void qca808x_get_phy_stats(struct phy_device *phydev, qcom_phy_get_stats(stats, priv->hw_stats); } +static int qca808x_get_rate_matching(struct phy_device *phydev, + phy_interface_t iface) +{ + if (iface == PHY_INTERFACE_MODE_2500BASEX) + return RATE_MATCH_PAUSE; + + return RATE_MATCH_NONE; +} + static struct phy_driver qca808x_driver[] = { { /* Qualcomm QCA8081 */ @@ -674,6 +683,7 @@ static struct phy_driver qca808x_driver[] = { .led_polarity_set = qca808x_led_polarity_set, .update_stats = qca808x_update_stats, .get_phy_stats = qca808x_get_phy_stats, + .get_rate_matching = qca808x_get_rate_matching, }, }; module_phy_driver(qca808x_driver); diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c index 16a347084293e..7c3d277efaf07 100644 --- a/drivers/net/phy/realtek/realtek_main.c +++ b/drivers/net/phy/realtek/realtek_main.c @@ -89,6 +89,14 @@ #define RTL8211F_LEDCR_MASK GENMASK(4, 0) #define RTL8211F_LEDCR_SHIFT 5 +/* RTL8211F(D)(I)-VD-CG CLKOUT configuration is specified via magic values + * to undocumented register pages. The names here do not reflect the datasheet. + * Unlike other PHY models, CLKOUT configuration does not go through PHYCR2. + */ +#define RTL8211FVD_CLKOUT_PAGE 0xd05 +#define RTL8211FVD_CLKOUT_REG 0x11 +#define RTL8211FVD_CLKOUT_EN BIT(8) + /* RTL8211F RGMII configuration */ #define RTL8211F_RGMII_PAGE 0xd08 @@ -166,9 +174,8 @@ MODULE_AUTHOR("Johnson Leung"); MODULE_LICENSE("GPL"); struct rtl821x_priv { - u16 phycr1; - u16 phycr2; - bool has_phycr2; + bool enable_aldps; + bool disable_clk_out; struct clk *clk; /* rtl8211f */ u16 iner; @@ -218,8 +225,6 @@ static int rtl821x_probe(struct phy_device *phydev) { struct device *dev = &phydev->mdio.dev; struct rtl821x_priv *priv; - u32 phy_id = phydev->drv->phy_id; - int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) @@ -230,24 +235,10 @@ static int rtl821x_probe(struct phy_device *phydev) return dev_err_probe(dev, PTR_ERR(priv->clk), "failed to get phy clock\n"); - ret = phy_read_paged(phydev, RTL8211F_PHYCR_PAGE, RTL8211F_PHYCR1); - if (ret < 0) - return ret; - - priv->phycr1 = ret & (RTL8211F_ALDPS_PLL_OFF | RTL8211F_ALDPS_ENABLE | RTL8211F_ALDPS_XTAL_OFF); - if (of_property_read_bool(dev->of_node, "realtek,aldps-enable")) - priv->phycr1 |= RTL8211F_ALDPS_PLL_OFF | RTL8211F_ALDPS_ENABLE | RTL8211F_ALDPS_XTAL_OFF; - - priv->has_phycr2 = !(phy_id == RTL_8211FVD_PHYID); - if (priv->has_phycr2) { - ret = phy_read_paged(phydev, RTL8211F_PHYCR_PAGE, RTL8211F_PHYCR2); - if (ret < 0) - return ret; - - priv->phycr2 = ret & RTL8211F_CLKOUT_EN; - if (of_property_read_bool(dev->of_node, "realtek,clkout-disable")) - priv->phycr2 &= ~RTL8211F_CLKOUT_EN; - } + priv->enable_aldps = of_property_read_bool(dev->of_node, + "realtek,aldps-enable"); + priv->disable_clk_out = of_property_read_bool(dev->of_node, + "realtek,clkout-disable"); phydev->priv = priv; @@ -560,22 +551,11 @@ static int rtl8211c_config_init(struct phy_device *phydev) CTL1000_ENABLE_MASTER | CTL1000_AS_MASTER); } -static int rtl8211f_config_init(struct phy_device *phydev) +static int rtl8211f_config_rgmii_delay(struct phy_device *phydev) { - struct rtl821x_priv *priv = phydev->priv; - struct device *dev = &phydev->mdio.dev; u16 val_txdly, val_rxdly; int ret; - ret = phy_modify_paged_changed(phydev, RTL8211F_PHYCR_PAGE, RTL8211F_PHYCR1, - RTL8211F_ALDPS_PLL_OFF | RTL8211F_ALDPS_ENABLE | RTL8211F_ALDPS_XTAL_OFF, - priv->phycr1); - if (ret < 0) { - dev_err(dev, "aldps mode configuration failed: %pe\n", - ERR_PTR(ret)); - return ret; - } - switch (phydev->interface) { case PHY_INTERFACE_MODE_RGMII: val_txdly = 0; @@ -605,53 +585,114 @@ static int rtl8211f_config_init(struct phy_device *phydev) RTL8211F_TXCR, RTL8211F_TX_DELAY, val_txdly); if (ret < 0) { - dev_err(dev, "Failed to update the TX delay register\n"); + phydev_err(phydev, "Failed to update the TX delay register: %pe\n", + ERR_PTR(ret)); return ret; } else if (ret) { - dev_dbg(dev, - "%s 2ns TX delay (and changing the value from pin-strapping RXD1 or the bootloader)\n", - str_enable_disable(val_txdly)); + phydev_dbg(phydev, + "%s 2ns TX delay (and changing the value from pin-strapping RXD1 or the bootloader)\n", + str_enable_disable(val_txdly)); } else { - dev_dbg(dev, - "2ns TX delay was already %s (by pin-strapping RXD1 or bootloader configuration)\n", - str_enabled_disabled(val_txdly)); + phydev_dbg(phydev, + "2ns TX delay was already %s (by pin-strapping RXD1 or bootloader configuration)\n", + str_enabled_disabled(val_txdly)); } ret = phy_modify_paged_changed(phydev, RTL8211F_RGMII_PAGE, RTL8211F_RXCR, RTL8211F_RX_DELAY, val_rxdly); if (ret < 0) { - dev_err(dev, "Failed to update the RX delay register\n"); + phydev_err(phydev, "Failed to update the RX delay register: %pe\n", + ERR_PTR(ret)); return ret; } else if (ret) { - dev_dbg(dev, - "%s 2ns RX delay (and changing the value from pin-strapping RXD0 or the bootloader)\n", - str_enable_disable(val_rxdly)); + phydev_dbg(phydev, + "%s 2ns RX delay (and changing the value from pin-strapping RXD0 or the bootloader)\n", + str_enable_disable(val_rxdly)); } else { - dev_dbg(dev, - "2ns RX delay was already %s (by pin-strapping RXD0 or bootloader configuration)\n", - str_enabled_disabled(val_rxdly)); + phydev_dbg(phydev, + "2ns RX delay was already %s (by pin-strapping RXD0 or bootloader configuration)\n", + str_enabled_disabled(val_rxdly)); } - if (!priv->has_phycr2) + return 0; +} + +static int rtl8211f_config_clk_out(struct phy_device *phydev) +{ + struct rtl821x_priv *priv = phydev->priv; + int ret; + + /* The value is preserved if the device tree property is absent */ + if (!priv->disable_clk_out) + return 0; + + if (phydev->drv->phy_id == RTL_8211FVD_PHYID) + ret = phy_modify_paged(phydev, RTL8211FVD_CLKOUT_PAGE, + RTL8211FVD_CLKOUT_REG, + RTL8211FVD_CLKOUT_EN, 0); + else + ret = phy_modify_paged(phydev, RTL8211F_PHYCR_PAGE, + RTL8211F_PHYCR2, RTL8211F_CLKOUT_EN, 0); + if (ret) + return ret; + + return genphy_soft_reset(phydev); +} + +/* Advance Link Down Power Saving (ALDPS) mode changes crystal/clock behaviour, + * which causes the RXC clock signal to stop for tens to hundreds of + * milliseconds. + * + * Some MACs need the RXC clock to support their internal RX logic, so ALDPS is + * only enabled based on an opt-in device tree property. + */ +static int rtl8211f_config_aldps(struct phy_device *phydev) +{ + struct rtl821x_priv *priv = phydev->priv; + u16 mask = RTL8211F_ALDPS_PLL_OFF | + RTL8211F_ALDPS_ENABLE | + RTL8211F_ALDPS_XTAL_OFF; + + /* The value is preserved if the device tree property is absent */ + if (!priv->enable_aldps) return 0; + return phy_modify_paged(phydev, RTL8211F_PHYCR_PAGE, RTL8211F_PHYCR1, + mask, mask); +} + +static int rtl8211f_config_phy_eee(struct phy_device *phydev) +{ /* Disable PHY-mode EEE so LPI is passed to the MAC */ - ret = phy_modify_paged(phydev, RTL8211F_PHYCR_PAGE, RTL8211F_PHYCR2, - RTL8211F_PHYCR2_PHY_EEE_ENABLE, 0); + return phy_modify_paged(phydev, RTL8211F_PHYCR_PAGE, RTL8211F_PHYCR2, + RTL8211F_PHYCR2_PHY_EEE_ENABLE, 0); +} + +static int rtl8211f_config_init(struct phy_device *phydev) +{ + struct device *dev = &phydev->mdio.dev; + int ret; + + ret = rtl8211f_config_aldps(phydev); + if (ret) { + dev_err(dev, "aldps mode configuration failed: %pe\n", + ERR_PTR(ret)); + return ret; + } + + ret = rtl8211f_config_rgmii_delay(phydev); if (ret) return ret; - ret = phy_modify_paged(phydev, RTL8211F_PHYCR_PAGE, - RTL8211F_PHYCR2, RTL8211F_CLKOUT_EN, - priv->phycr2); - if (ret < 0) { + ret = rtl8211f_config_clk_out(phydev); + if (ret) { dev_err(dev, "clkout configuration failed: %pe\n", ERR_PTR(ret)); return ret; } - return genphy_soft_reset(phydev); + return rtl8211f_config_phy_eee(phydev); } static int rtl821x_suspend(struct phy_device *phydev) diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c index 0401fa6b24d25..c62e3f364ea73 100644 --- a/drivers/net/phy/sfp.c +++ b/drivers/net/phy/sfp.c @@ -367,6 +367,12 @@ static void sfp_fixup_ignore_tx_fault(struct sfp *sfp) sfp->state_ignore_mask |= SFP_F_TX_FAULT; } +static void sfp_fixup_ignore_tx_fault_and_los(struct sfp *sfp) +{ + sfp_fixup_ignore_tx_fault(sfp); + sfp_fixup_ignore_los(sfp); +} + static void sfp_fixup_ignore_hw(struct sfp *sfp, unsigned int mask) { sfp->state_hw_mask &= ~mask; @@ -474,11 +480,18 @@ static void sfp_quirk_ubnt_uf_instant(const struct sfp_eeprom_id *id, { /* Ubiquiti U-Fiber Instant module claims that support all transceiver * types including 10G Ethernet which is not truth. So clear all claimed - * modes and set only one mode which module supports: 1000baseX_Full. + * modes and set only one mode which module supports: 1000baseX_Full, + * along with the Autoneg and pause bits. */ linkmode_zero(caps->link_modes); linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseX_Full_BIT, caps->link_modes); + linkmode_set_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, caps->link_modes); + linkmode_set_bit(ETHTOOL_LINK_MODE_Pause_BIT, caps->link_modes); + linkmode_set_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, caps->link_modes); + + phy_interface_zero(caps->interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, caps->interfaces); } #define SFP_QUIRK(_v, _p, _s, _f) \ @@ -497,6 +510,8 @@ static const struct sfp_quirk sfp_quirks[] = { SFP_QUIRK("ALCATELLUCENT", "3FE46541AA", sfp_quirk_2500basex, sfp_fixup_nokia), + SFP_QUIRK_F("BIDB", "X-ONU-SFPP", sfp_fixup_potron), + // FLYPRO SFP-10GT-CS-30M uses Rollball protocol to talk to the PHY. SFP_QUIRK_F("FLYPRO", "SFP-10GT-CS-30M", sfp_fixup_rollball), @@ -517,6 +532,8 @@ static const struct sfp_quirk sfp_quirks[] = { SFP_QUIRK_F("HALNy", "HL-GSFP", sfp_fixup_halny_gsfp), + SFP_QUIRK_F("H-COM", "SPP425H-GAB4", sfp_fixup_potron), + // HG MXPD-483II-F 2.5G supports 2500Base-X, but incorrectly reports // 2600MBd in their EERPOM SFP_QUIRK_S("HG GENUINE", "MXPD-483II", sfp_quirk_2500basex), @@ -524,11 +541,31 @@ static const struct sfp_quirk sfp_quirks[] = { // Huawei MA5671A can operate at 2500base-X, but report 1.2GBd NRZ in // their EEPROM SFP_QUIRK("HUAWEI", "MA5671A", sfp_quirk_2500basex, + sfp_fixup_ignore_tx_fault_and_los), + + // Hisense LXT-010S-H is a GPON ONT SFP (sold as LEOX LXT-010S-H) that + // can operate at 2500base-X, but reports 1000BASE-LX / 1300MBd in its + // EEPROM + SFP_QUIRK("Hisense-Leox", "LXT-010S-H", sfp_quirk_2500basex, + sfp_fixup_ignore_tx_fault), + + // Hisense ZNID-GPON-2311NA can operate at 2500base-X, but reports + // 1000BASE-LX / 1300MBd in its EEPROM + SFP_QUIRK("Hisense", "ZNID-GPON-2311NA", sfp_quirk_2500basex, sfp_fixup_ignore_tx_fault), - // Lantech 8330-262D-E can operate at 2500base-X, but incorrectly report - // 2500MBd NRZ in their EEPROM + // HSGQ HSGQ-XPON-Stick can operate at 2500base-X, but reports + // 1000BASE-LX / 1300MBd in its EEPROM + SFP_QUIRK("HSGQ", "HSGQ-XPON-Stick", sfp_quirk_2500basex, + sfp_fixup_ignore_tx_fault), + + // Lantech 8330-262D-E and 8330-265D can operate at 2500base-X, but + // incorrectly report 2500MBd NRZ in their EEPROM. + // Some 8330-265D modules have inverted LOS, while all of them report + // normal LOS in EEPROM. Therefore we need to ignore LOS entirely. SFP_QUIRK_S("Lantech", "8330-262D-E", sfp_quirk_2500basex), + SFP_QUIRK("Lantech", "8330-265D", sfp_quirk_2500basex, + sfp_fixup_ignore_los), SFP_QUIRK_S("UBNT", "UF-INSTANT", sfp_quirk_ubnt_uf_instant), @@ -763,7 +800,7 @@ static int sfp_smbus_byte_write(struct sfp *sfp, bool a2, u8 dev_addr, dev_addr++; } - return 0; + return data - (u8 *)buf; } static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c) diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c index 25562b17debe1..a98f5e5061544 100644 --- a/drivers/net/team/team_core.c +++ b/drivers/net/team/team_core.c @@ -878,7 +878,7 @@ static void __team_queue_override_enabled_check(struct team *team) static void team_queue_override_port_prio_changed(struct team *team, struct team_port *port) { - if (!port->queue_id || team_port_enabled(port)) + if (!port->queue_id || !team_port_enabled(port)) return; __team_queue_override_port_del(team, port); __team_queue_override_port_add(team, port); @@ -982,63 +982,6 @@ static void team_port_disable(struct team *team, team_lower_state_changed(port); } -#define TEAM_VLAN_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ - NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE | \ - NETIF_F_HIGHDMA | NETIF_F_LRO | \ - NETIF_F_GSO_ENCAP_ALL) - -#define TEAM_ENC_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ - NETIF_F_RXCSUM | NETIF_F_GSO_SOFTWARE) - -static void __team_compute_features(struct team *team) -{ - struct team_port *port; - netdev_features_t vlan_features = TEAM_VLAN_FEATURES; - netdev_features_t enc_features = TEAM_ENC_FEATURES; - unsigned short max_hard_header_len = ETH_HLEN; - unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | - IFF_XMIT_DST_RELEASE_PERM; - - rcu_read_lock(); - if (list_empty(&team->port_list)) - goto done; - - vlan_features = netdev_base_features(vlan_features); - enc_features = netdev_base_features(enc_features); - - list_for_each_entry_rcu(port, &team->port_list, list) { - vlan_features = netdev_increment_features(vlan_features, - port->dev->vlan_features, - TEAM_VLAN_FEATURES); - enc_features = - netdev_increment_features(enc_features, - port->dev->hw_enc_features, - TEAM_ENC_FEATURES); - - dst_release_flag &= port->dev->priv_flags; - if (port->dev->hard_header_len > max_hard_header_len) - max_hard_header_len = port->dev->hard_header_len; - } -done: - rcu_read_unlock(); - - team->dev->vlan_features = vlan_features; - team->dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_HW_VLAN_STAG_TX; - team->dev->hard_header_len = max_hard_header_len; - - team->dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; - if (dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM)) - team->dev->priv_flags |= IFF_XMIT_DST_RELEASE; -} - -static void team_compute_features(struct team *team) -{ - __team_compute_features(team); - netdev_change_features(team->dev); -} - static int team_port_enter(struct team *team, struct team_port *port) { int err = 0; @@ -1306,7 +1249,7 @@ static int team_port_add(struct team *team, struct net_device *port_dev, port->index = -1; list_add_tail_rcu(&port->list, &team->port_list); team_port_enable(team, port); - __team_compute_features(team); + netdev_compute_master_upper_features(team->dev, true); __team_port_change_port_added(port, !!netif_oper_up(port_dev)); __team_options_change_check(team); @@ -1348,7 +1291,7 @@ static int team_port_add(struct team *team, struct net_device *port_dev, static void __team_port_change_port_removed(struct team_port *port); -static int team_port_del(struct team *team, struct net_device *port_dev) +static int team_port_del(struct team *team, struct net_device *port_dev, bool unregister) { struct net_device *dev = team->dev; struct team_port *port; @@ -1386,10 +1329,16 @@ static int team_port_del(struct team *team, struct net_device *port_dev) __team_port_change_port_removed(port); team_port_set_orig_dev_addr(port); - dev_set_mtu(port_dev, port->orig.mtu); + if (unregister) { + netdev_lock_ops(port_dev); + __netif_set_mtu(port_dev, port->orig.mtu); + netdev_unlock_ops(port_dev); + } else { + dev_set_mtu(port_dev, port->orig.mtu); + } kfree_rcu(port, rcu); netdev_info(dev, "Port device %s removed\n", portname); - __team_compute_features(team); + netdev_compute_master_upper_features(team->dev, true); return 0; } @@ -1690,7 +1639,7 @@ static void team_uninit(struct net_device *dev) ASSERT_RTNL(); list_for_each_entry_safe(port, tmp, &team->port_list, list) - team_port_del(team, port->dev); + team_port_del(team, port->dev, false); __team_change_mode(team, NULL); /* cleanup */ __team_options_unregister(team, team_options, ARRAY_SIZE(team_options)); @@ -1977,33 +1926,28 @@ static int team_add_slave(struct net_device *dev, struct net_device *port_dev, struct netlink_ext_ack *extack) { struct team *team = netdev_priv(dev); - int err; ASSERT_RTNL(); - err = team_port_add(team, port_dev, extack); - - if (!err) - netdev_change_features(dev); - - return err; + return team_port_add(team, port_dev, extack); } static int team_del_slave(struct net_device *dev, struct net_device *port_dev) { struct team *team = netdev_priv(dev); - int err; ASSERT_RTNL(); - err = team_port_del(team, port_dev); + return team_port_del(team, port_dev, false); +} - if (err) - return err; +static int team_del_slave_on_unregister(struct net_device *dev, struct net_device *port_dev) +{ + struct team *team = netdev_priv(dev); - netdev_change_features(dev); + ASSERT_RTNL(); - return err; + return team_port_del(team, port_dev, true); } static netdev_features_t team_fix_features(struct net_device *dev, @@ -2115,6 +2059,68 @@ static const struct ethtool_ops team_ethtool_ops = { * rt netlink interface ***********************/ +/* For tx path we need a linkup && enabled port and for parse any port + * suffices. + */ +static struct team_port *team_header_port_get_rcu(struct team *team, + bool txable) +{ + struct team_port *port; + + list_for_each_entry_rcu(port, &team->port_list, list) { + if (!txable || team_port_txable(port)) + return port; + } + + return NULL; +} + +static int team_header_create(struct sk_buff *skb, struct net_device *team_dev, + unsigned short type, const void *daddr, + const void *saddr, unsigned int len) +{ + struct team *team = netdev_priv(team_dev); + const struct header_ops *port_ops; + struct team_port *port; + int ret = 0; + + rcu_read_lock(); + port = team_header_port_get_rcu(team, true); + if (port) { + port_ops = READ_ONCE(port->dev->header_ops); + if (port_ops && port_ops->create) + ret = port_ops->create(skb, port->dev, + type, daddr, saddr, len); + } + rcu_read_unlock(); + return ret; +} + +static int team_header_parse(const struct sk_buff *skb, + const struct net_device *team_dev, + unsigned char *haddr) +{ + struct team *team = netdev_priv(team_dev); + const struct header_ops *port_ops; + struct team_port *port; + int ret = 0; + + rcu_read_lock(); + port = team_header_port_get_rcu(team, false); + if (port) { + port_ops = READ_ONCE(port->dev->header_ops); + if (port_ops && port_ops->parse) + ret = port_ops->parse(skb, port->dev, haddr); + } + rcu_read_unlock(); + return ret; +} + +static const struct header_ops team_header_ops = { + .create = team_header_create, + .parse = team_header_parse, +}; + static void team_setup_by_port(struct net_device *dev, struct net_device *port_dev) { @@ -2123,7 +2129,8 @@ static void team_setup_by_port(struct net_device *dev, if (port_dev->type == ARPHRD_ETHER) dev->header_ops = team->header_ops_cache; else - dev->header_ops = port_dev->header_ops; + dev->header_ops = port_dev->header_ops ? + &team_header_ops : NULL; dev->type = port_dev->type; dev->hard_header_len = port_dev->hard_header_len; dev->needed_headroom = port_dev->needed_headroom; @@ -2197,7 +2204,7 @@ static void team_setup(struct net_device *dev) dev->features |= NETIF_F_GRO; - dev->hw_features = TEAM_VLAN_FEATURES | + dev->hw_features = MASTER_UPPER_DEV_VLAN_FEATURES | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_RX | @@ -2996,12 +3003,12 @@ static int team_device_event(struct notifier_block *unused, !!netif_oper_up(port->dev)); break; case NETDEV_UNREGISTER: - team_del_slave(port->team->dev, dev); + team_del_slave_on_unregister(port->team->dev, dev); break; case NETDEV_FEAT_CHANGE: if (!port->team->notifier_ctx) { port->team->notifier_ctx = true; - team_compute_features(port->team); + netdev_compute_master_upper_features(port->team->dev, true); port->team->notifier_ctx = false; } break; @@ -3069,3 +3076,4 @@ MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Jiri Pirko "); MODULE_DESCRIPTION("Ethernet team device driver"); MODULE_ALIAS_RTNL_LINK(DRV_NAME); +MODULE_IMPORT_NS("NETDEV_INTERNAL"); diff --git a/drivers/net/tun_vnet.h b/drivers/net/tun_vnet.h index a5f93b6c4482c..fa5cab9d3e55c 100644 --- a/drivers/net/tun_vnet.h +++ b/drivers/net/tun_vnet.h @@ -244,7 +244,7 @@ tun_vnet_hdr_tnl_from_skb(unsigned int flags, if (virtio_net_hdr_tnl_from_skb(skb, tnl_hdr, has_tnl_offload, tun_vnet_is_little_endian(flags), - vlan_hlen, true)) { + vlan_hlen, true, false)) { struct virtio_net_hdr_v1 *hdr = &tnl_hdr->hash_hdr.hdr; struct skb_shared_info *sinfo = skb_shinfo(skb); diff --git a/drivers/net/usb/Kconfig b/drivers/net/usb/Kconfig index 856e648d804e0..da0f6a138f4fc 100644 --- a/drivers/net/usb/Kconfig +++ b/drivers/net/usb/Kconfig @@ -319,7 +319,6 @@ config USB_NET_DM9601 config USB_NET_SR9700 tristate "CoreChip-sz SR9700 based USB 1.1 10/100 ethernet devices" depends on USB_USBNET - select CRC32 help This option adds support for CoreChip-sz SR9700 based USB 1.1 10/100 Ethernet adapters. diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c index 9201ee10a13f7..d316aa66dbc23 100644 --- a/drivers/net/usb/aqc111.c +++ b/drivers/net/usb/aqc111.c @@ -1400,14 +1400,14 @@ static int aqc111_suspend(struct usb_interface *intf, pm_message_t message) aqc111_write16_cmd_nopm(dev, AQ_ACCESS_MAC, SFR_MEDIUM_STATUS_MODE, 2, ®16); - aqc111_write_cmd(dev, AQ_WOL_CFG, 0, 0, - WOL_CFG_SIZE, &wol_cfg); - aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0, - &aqc111_data->phy_cfg); + aqc111_write_cmd_nopm(dev, AQ_WOL_CFG, 0, 0, + WOL_CFG_SIZE, &wol_cfg); + aqc111_write32_cmd_nopm(dev, AQ_PHY_OPS, 0, 0, + &aqc111_data->phy_cfg); } else { aqc111_data->phy_cfg |= AQ_LOW_POWER; - aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0, - &aqc111_data->phy_cfg); + aqc111_write32_cmd_nopm(dev, AQ_PHY_OPS, 0, 0, + &aqc111_data->phy_cfg); /* Disable RX path */ aqc111_read16_cmd_nopm(dev, AQ_ACCESS_MAC, diff --git a/drivers/net/usb/asix_common.c b/drivers/net/usb/asix_common.c index 7fd763917ae2c..6ab3486072cb0 100644 --- a/drivers/net/usb/asix_common.c +++ b/drivers/net/usb/asix_common.c @@ -335,6 +335,11 @@ int asix_read_phy_addr(struct usbnet *dev, bool internal) offset = (internal ? 1 : 0); ret = buf[offset]; + if (ret >= PHY_MAX_ADDR) { + netdev_err(dev->net, "invalid PHY address: %d\n", ret); + return -ENODEV; + } + netdev_dbg(dev->net, "%s PHY address 0x%x\n", internal ? "internal" : "external", ret); diff --git a/drivers/net/usb/ax88172a.c b/drivers/net/usb/ax88172a.c index f613e4bc68c85..758a423a459b8 100644 --- a/drivers/net/usb/ax88172a.c +++ b/drivers/net/usb/ax88172a.c @@ -210,11 +210,7 @@ static int ax88172a_bind(struct usbnet *dev, struct usb_interface *intf) ret = asix_read_phy_addr(dev, priv->use_embdphy); if (ret < 0) goto free; - if (ret >= PHY_MAX_ADDR) { - netdev_err(dev->net, "Invalid PHY address %#x\n", ret); - ret = -ENODEV; - goto free; - } + priv->phy_addr = ret; ax88172a_reset_phy(dev, priv->use_embdphy); diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c index 6759388692f8e..3c824340ffb06 100644 --- a/drivers/net/usb/catc.c +++ b/drivers/net/usb/catc.c @@ -64,6 +64,16 @@ static const char driver_name[] = "catc"; #define CTRL_QUEUE 16 /* Max control requests in flight (power of two) */ #define RX_PKT_SZ 1600 /* Max size of receive packet for F5U011 */ +/* + * USB endpoints. + */ + +enum catc_usb_ep { + CATC_USB_EP_CONTROL = 0, + CATC_USB_EP_BULK = 1, + CATC_USB_EP_INT_IN = 2, +}; + /* * Control requests. */ @@ -772,6 +782,13 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id u8 broadcast[ETH_ALEN]; u8 *macbuf; int pktsz, ret = -ENOMEM; + static const u8 bulk_ep_addr[] = { + CATC_USB_EP_BULK | USB_DIR_OUT, + CATC_USB_EP_BULK | USB_DIR_IN, + 0}; + static const u8 int_ep_addr[] = { + CATC_USB_EP_INT_IN | USB_DIR_IN, + 0}; macbuf = kmalloc(ETH_ALEN, GFP_KERNEL); if (!macbuf) @@ -784,6 +801,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id goto fail_mem; } + /* Verify that all required endpoints are present */ + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(dev, "Missing or invalid endpoints\n"); + ret = -ENODEV; + goto fail_mem; + } + netdev = alloc_etherdev(sizeof(struct catc)); if (!netdev) goto fail_mem; @@ -828,14 +853,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id usb_fill_control_urb(catc->ctrl_urb, usbdev, usb_sndctrlpipe(usbdev, 0), NULL, NULL, 0, catc_ctrl_done, catc); - usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, 1), - NULL, 0, catc_tx_done, catc); + usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, CATC_USB_EP_BULK), + NULL, 0, catc_tx_done, catc); - usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, 1), - catc->rx_buf, pktsz, catc_rx_done, catc); + usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, CATC_USB_EP_BULK), + catc->rx_buf, pktsz, catc_rx_done, catc); - usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, 2), - catc->irq_buf, 2, catc_irq_done, catc, 1); + usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, CATC_USB_EP_INT_IN), + catc->irq_buf, 2, catc_irq_done, catc, 1); if (!catc->is_f5u011) { u32 *buf; diff --git a/drivers/net/usb/cdc-phonet.c b/drivers/net/usb/cdc-phonet.c index ad5121e9cf5d6..165650ecef649 100644 --- a/drivers/net/usb/cdc-phonet.c +++ b/drivers/net/usb/cdc-phonet.c @@ -157,11 +157,16 @@ static void rx_complete(struct urb *req) PAGE_SIZE); page = NULL; } - } else { + } else if (skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) { skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 0, req->actual_length, PAGE_SIZE); page = NULL; + } else { + dev_kfree_skb_any(skb); + pnd->rx_skb = NULL; + skb = NULL; + dev->stats.rx_length_errors++; } if (req->actual_length < PAGE_SIZE) pnd->rx_skb = NULL; /* Last fragment */ diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c index 5d123df0a866b..81d7e99fc0f09 100644 --- a/drivers/net/usb/cdc_ncm.c +++ b/drivers/net/usb/cdc_ncm.c @@ -1656,6 +1656,7 @@ int cdc_ncm_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset) struct usbnet *dev = netdev_priv(skb_in->dev); struct usb_cdc_ncm_ndp16 *ndp16; int ret = -EINVAL; + size_t ndp_len; if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp16)) > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "invalid NDP offset <%u>\n", @@ -1675,8 +1676,8 @@ int cdc_ncm_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset) sizeof(struct usb_cdc_ncm_dpe16)); ret--; /* we process NDP entries except for the last one */ - if ((sizeof(struct usb_cdc_ncm_ndp16) + - ret * (sizeof(struct usb_cdc_ncm_dpe16))) > skb_in->len) { + ndp_len = struct_size_t(struct usb_cdc_ncm_ndp16, dpe16, ret); + if (ndpoffset + ndp_len > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret); ret = -EINVAL; } @@ -1692,6 +1693,7 @@ int cdc_ncm_rx_verify_ndp32(struct sk_buff *skb_in, int ndpoffset) struct usbnet *dev = netdev_priv(skb_in->dev); struct usb_cdc_ncm_ndp32 *ndp32; int ret = -EINVAL; + size_t ndp_len; if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp32)) > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "invalid NDP offset <%u>\n", @@ -1711,8 +1713,8 @@ int cdc_ncm_rx_verify_ndp32(struct sk_buff *skb_in, int ndpoffset) sizeof(struct usb_cdc_ncm_dpe32)); ret--; /* we process NDP entries except for the last one */ - if ((sizeof(struct usb_cdc_ncm_ndp32) + - ret * (sizeof(struct usb_cdc_ncm_dpe32))) > skb_in->len) { + ndp_len = struct_size_t(struct usb_cdc_ncm_ndp32, dpe32, ret); + if (ndpoffset + ndp_len > skb_in->len) { netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret); ret = -EINVAL; } diff --git a/drivers/net/usb/dm9601.c b/drivers/net/usb/dm9601.c index 8b6d6a1b3c2ec..2b4716ccf0c5b 100644 --- a/drivers/net/usb/dm9601.c +++ b/drivers/net/usb/dm9601.c @@ -603,10 +603,6 @@ static const struct usb_device_id products[] = { USB_DEVICE(0x0fe6, 0x8101), /* DM9601 USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, - { - USB_DEVICE(0x0fe6, 0x9700), /* DM9601 USB to Fast Ethernet Adapter */ - .driver_info = (unsigned long)&dm9601_info, - }, { USB_DEVICE(0x0a46, 0x9000), /* DM9000E */ .driver_info = (unsigned long)&dm9601_info, diff --git a/drivers/net/usb/kalmia.c b/drivers/net/usb/kalmia.c index 613fc6910f148..ee9c48f7f68f9 100644 --- a/drivers/net/usb/kalmia.c +++ b/drivers/net/usb/kalmia.c @@ -132,11 +132,18 @@ kalmia_bind(struct usbnet *dev, struct usb_interface *intf) { int status; u8 ethernet_addr[ETH_ALEN]; + static const u8 ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; /* Don't bind to AT command interface */ if (intf->cur_altsetting->desc.bInterfaceClass != USB_CLASS_VENDOR_SPEC) return -EINVAL; + if (!usb_check_bulk_endpoints(intf, ep_addr)) + return -ENODEV; + dev->in = usb_rcvbulkpipe(dev->udev, 0x81 & USB_ENDPOINT_NUMBER_MASK); dev->out = usb_sndbulkpipe(dev->udev, 0x02 & USB_ENDPOINT_NUMBER_MASK); dev->status = NULL; diff --git a/drivers/net/usb/kaweth.c b/drivers/net/usb/kaweth.c index c9efb7df892ec..cb2472b59e104 100644 --- a/drivers/net/usb/kaweth.c +++ b/drivers/net/usb/kaweth.c @@ -765,7 +765,6 @@ static void kaweth_set_rx_mode(struct net_device *net) netdev_dbg(net, "Setting Rx mode to %d\n", packet_filter_bitmap); - netif_stop_queue(net); if (net->flags & IFF_PROMISC) { packet_filter_bitmap |= KAWETH_PACKET_FILTER_PROMISCUOUS; @@ -775,7 +774,6 @@ static void kaweth_set_rx_mode(struct net_device *net) } kaweth->packet_filter_bitmap = packet_filter_bitmap; - netif_wake_queue(net); } /**************************************************************** @@ -885,6 +883,13 @@ static int kaweth_probe( const eth_addr_t bcast_addr = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; int result = 0; int rv = -EIO; + static const u8 bulk_ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + 3 | USB_DIR_IN, + 0}; dev_dbg(dev, "Kawasaki Device Probe (Device number:%d): 0x%4.4x:0x%4.4x:0x%4.4x\n", @@ -898,6 +903,12 @@ static int kaweth_probe( (int)udev->descriptor.bLength, (int)udev->descriptor.bDescriptorType); + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(dev, "couldn't find required endpoints\n"); + return -ENODEV; + } + netdev = alloc_etherdev(sizeof(*kaweth)); if (!netdev) return -ENOMEM; diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index 00397a8073934..858a442d6996a 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -2094,8 +2094,6 @@ static int lan78xx_mdio_init(struct lan78xx_net *dev) dev->mdiobus->phy_mask = ~(1 << 1); break; case ID_REV_CHIP_ID_7801_: - /* scan thru PHYAD[2..0] */ - dev->mdiobus->phy_mask = ~(0xFF); break; } @@ -3121,6 +3119,10 @@ static int lan78xx_init_ltm(struct lan78xx_net *dev) int ret; u32 buf; + /* LAN7850 is USB 2.0 and does not support LTM */ + if (dev->chipid == ID_REV_CHIP_ID_7850_) + return 0; + ret = lan78xx_read_reg(dev, USB_CFG1, &buf); if (ret < 0) goto init_ltm_failed; @@ -3831,6 +3833,7 @@ static void lan78xx_rx_csum_offload(struct lan78xx_net *dev, */ if (!(dev->net->features & NETIF_F_RXCSUM) || unlikely(rx_cmd_a & RX_CMD_A_ICSM_) || + unlikely(rx_cmd_a & RX_CMD_A_CSE_MASK_) || ((rx_cmd_a & RX_CMD_A_FVTG_) && !(dev->net->features & NETIF_F_HW_VLAN_CTAG_RX))) { skb->ip_summed = CHECKSUM_NONE; @@ -3903,7 +3906,8 @@ static int lan78xx_rx(struct lan78xx_net *dev, struct sk_buff *skb, return 0; } - if (unlikely(rx_cmd_a & RX_CMD_A_RED_)) { + if (unlikely(rx_cmd_a & RX_CMD_A_RED_) && + (rx_cmd_a & RX_CMD_A_RX_HARD_ERRS_MASK_)) { netif_dbg(dev, rx_err, dev->net, "Error rx_cmd_a=0x%08x", rx_cmd_a); } else { @@ -4178,7 +4182,7 @@ static struct skb_data *lan78xx_tx_buf_fill(struct lan78xx_net *dev, } tx_data += len; - entry->length += len; + entry->length += max_t(unsigned int, len, ETH_ZLEN); entry->num_of_packet += skb_shinfo(skb)->gso_segs ?: 1; dev_kfree_skb_any(skb); @@ -4548,8 +4552,6 @@ static void lan78xx_disconnect(struct usb_interface *intf) phylink_disconnect_phy(dev->phylink); rtnl_unlock(); - netif_napi_del(&dev->napi); - unregister_netdev(net); timer_shutdown_sync(&dev->stat_monitor); diff --git a/drivers/net/usb/lan78xx.h b/drivers/net/usb/lan78xx.h index 968e5e5faee0a..17a934acff3d6 100644 --- a/drivers/net/usb/lan78xx.h +++ b/drivers/net/usb/lan78xx.h @@ -74,6 +74,9 @@ #define RX_CMD_A_ICSM_ (0x00004000) #define RX_CMD_A_LEN_MASK_ (0x00003FFF) +#define RX_CMD_A_RX_HARD_ERRS_MASK_ \ + (RX_CMD_A_RX_ERRS_MASK_ & ~RX_CMD_A_CSE_MASK_) + /* Rx Command B */ #define RX_CMD_B_CSUM_SHIFT_ (16) #define RX_CMD_B_CSUM_MASK_ (0xFFFF0000) diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c index 81ca64debc5b9..475b066081c7f 100644 --- a/drivers/net/usb/pegasus.c +++ b/drivers/net/usb/pegasus.c @@ -31,6 +31,17 @@ static const char driver_name[] = "pegasus"; BMSR_100FULL | BMSR_ANEGCAPABLE) #define CARRIER_CHECK_DELAY (2 * HZ) +/* + * USB endpoints. + */ + +enum pegasus_usb_ep { + PEGASUS_USB_EP_CONTROL = 0, + PEGASUS_USB_EP_BULK_IN = 1, + PEGASUS_USB_EP_BULK_OUT = 2, + PEGASUS_USB_EP_INT_IN = 3, +}; + static bool loopback; static bool mii_mode; static char *devid; @@ -168,6 +179,8 @@ static int update_eth_regs_async(pegasus_t *pegasus) netif_device_detach(pegasus->net); netif_err(pegasus, drv, pegasus->net, "%s returned %d\n", __func__, ret); + usb_free_urb(async_urb); + kfree(req); } return ret; } @@ -543,7 +556,7 @@ static void read_bulk_callback(struct urb *urb) goto tl_sched; goon: usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); rx_status = usb_submit_urb(pegasus->rx_urb, GFP_ATOMIC); @@ -583,7 +596,7 @@ static void rx_fixup(struct tasklet_struct *t) return; } usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); try_again: @@ -711,7 +724,7 @@ static netdev_tx_t pegasus_start_xmit(struct sk_buff *skb, ((__le16 *) pegasus->tx_buff)[0] = cpu_to_le16(l16); skb_copy_from_linear_data(skb, pegasus->tx_buff + 2, skb->len); usb_fill_bulk_urb(pegasus->tx_urb, pegasus->usb, - usb_sndbulkpipe(pegasus->usb, 2), + usb_sndbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_OUT), pegasus->tx_buff, count, write_bulk_callback, pegasus); if ((res = usb_submit_urb(pegasus->tx_urb, GFP_ATOMIC))) { @@ -802,8 +815,19 @@ static void unlink_all_urbs(pegasus_t *pegasus) static int alloc_urbs(pegasus_t *pegasus) { + static const u8 bulk_ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + 3 | USB_DIR_IN, + 0}; int res = -ENOMEM; + if (!usb_check_bulk_endpoints(pegasus->intf, bulk_ep_addr) || + !usb_check_int_endpoints(pegasus->intf, int_ep_addr)) + return -ENODEV; + pegasus->rx_urb = usb_alloc_urb(0, GFP_KERNEL); if (!pegasus->rx_urb) { return res; @@ -838,7 +862,7 @@ static int pegasus_open(struct net_device *net) set_registers(pegasus, EthID, 6, net->dev_addr); usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); if ((res = usb_submit_urb(pegasus->rx_urb, GFP_KERNEL))) { @@ -849,7 +873,7 @@ static int pegasus_open(struct net_device *net) } usb_fill_int_urb(pegasus->intr_urb, pegasus->usb, - usb_rcvintpipe(pegasus->usb, 3), + usb_rcvintpipe(pegasus->usb, PEGASUS_USB_EP_INT_IN), pegasus->intr_buff, sizeof(pegasus->intr_buff), intr_callback, pegasus, pegasus->intr_interval); if ((res = usb_submit_urb(pegasus->intr_urb, GFP_KERNEL))) { @@ -1134,16 +1158,31 @@ static int pegasus_probe(struct usb_interface *intf, pegasus_t *pegasus; int dev_index = id - pegasus_ids; int res = -ENOMEM; + static const u8 bulk_ep_addr[] = { + PEGASUS_USB_EP_BULK_IN | USB_DIR_IN, + PEGASUS_USB_EP_BULK_OUT | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + PEGASUS_USB_EP_INT_IN | USB_DIR_IN, + 0}; if (pegasus_blacklisted(dev)) return -ENODEV; + /* Verify that all required endpoints are present */ + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(&intf->dev, "Missing or invalid endpoints\n"); + return -ENODEV; + } + net = alloc_etherdev(sizeof(struct pegasus)); if (!net) goto out; pegasus = netdev_priv(net); pegasus->dev_index = dev_index; + pegasus->intf = intf; res = alloc_urbs(pegasus); if (res < 0) { @@ -1155,7 +1194,6 @@ static int pegasus_probe(struct usb_interface *intf, INIT_DELAYED_WORK(&pegasus->carrier_check, check_carrier); - pegasus->intf = intf; pegasus->usb = dev; pegasus->net = net; diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 3a4985b582cb1..05acac10cd2ba 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -928,7 +928,7 @@ static int qmi_wwan_resume(struct usb_interface *intf) static const struct driver_info qmi_wwan_info = { .description = "WWAN/QMI device", - .flags = FLAG_WWAN | FLAG_SEND_ZLP, + .flags = FLAG_WWAN | FLAG_NOMAXMTU | FLAG_SEND_ZLP, .bind = qmi_wwan_bind, .unbind = qmi_wwan_unbind, .manage_power = qmi_wwan_manage_power, @@ -937,7 +937,7 @@ static const struct driver_info qmi_wwan_info = { static const struct driver_info qmi_wwan_info_quirk_dtr = { .description = "WWAN/QMI device", - .flags = FLAG_WWAN | FLAG_SEND_ZLP, + .flags = FLAG_WWAN | FLAG_NOMAXMTU | FLAG_SEND_ZLP, .bind = qmi_wwan_bind, .unbind = qmi_wwan_unbind, .manage_power = qmi_wwan_manage_power, diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index a22d4bb2cf3b5..357f5c733d0b5 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -2449,6 +2449,8 @@ static int r8152_tx_agg_fill(struct r8152 *tp, struct tx_agg *agg) ret = usb_submit_urb(agg->urb, GFP_ATOMIC); if (ret < 0) usb_autopm_put_interface_async(tp->intf); + else + netif_trans_update(tp->netdev); out_tx_fill: return ret; @@ -8535,19 +8537,6 @@ static int rtl8152_system_resume(struct r8152 *tp) usb_submit_urb(tp->intr_urb, GFP_NOIO); } - /* If the device is RTL8152_INACCESSIBLE here then we should do a - * reset. This is important because the usb_lock_device_for_reset() - * that happens as a result of usb_queue_reset_device() will silently - * fail if the device was suspended or if too much time passed. - * - * NOTE: The device is locked here so we can directly do the reset. - * We don't need usb_lock_device_for_reset() because that's just a - * wrapper over device_lock() and device_resume() (which calls us) - * does that for us. - */ - if (test_bit(RTL8152_INACCESSIBLE, &tp->flags)) - usb_reset_device(tp->udev); - return 0; } @@ -8658,19 +8647,33 @@ static int rtl8152_suspend(struct usb_interface *intf, pm_message_t message) static int rtl8152_resume(struct usb_interface *intf) { struct r8152 *tp = usb_get_intfdata(intf); + bool runtime_resume = test_bit(SELECTIVE_SUSPEND, &tp->flags); int ret; mutex_lock(&tp->control); rtl_reset_ocp_base(tp); - if (test_bit(SELECTIVE_SUSPEND, &tp->flags)) + if (runtime_resume) ret = rtl8152_runtime_resume(tp); else ret = rtl8152_system_resume(tp); mutex_unlock(&tp->control); + /* If the device is RTL8152_INACCESSIBLE here then we should do a + * reset. This is important because the usb_lock_device_for_reset() + * that happens as a result of usb_queue_reset_device() will silently + * fail if the device was suspended or if too much time passed. + * + * NOTE: The device is locked here so we can directly do the reset. + * We don't need usb_lock_device_for_reset() because that's just a + * wrapper over device_lock() and device_resume() (which calls us) + * does that for us. + */ + if (!runtime_resume && test_bit(RTL8152_INACCESSIBLE, &tp->flags)) + usb_reset_device(tp->udev); + return ret; } @@ -10058,6 +10061,7 @@ static const struct usb_device_id rtl8152_table[] = { { USB_DEVICE(VENDOR_ID_DLINK, 0xb301) }, { USB_DEVICE(VENDOR_ID_DELL, 0xb097) }, { USB_DEVICE(VENDOR_ID_ASUS, 0x1976) }, + { USB_DEVICE(VENDOR_ID_TRENDNET, 0xe02b) }, {} }; diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c index 278e6cb6f4d99..e40b0669d9f4b 100644 --- a/drivers/net/usb/rtl8150.c +++ b/drivers/net/usb/rtl8150.c @@ -211,6 +211,8 @@ static int async_set_registers(rtl8150_t *dev, u16 indx, u16 size, u16 reg) if (res == -ENODEV) netif_device_detach(dev->netdev); dev_err(&dev->udev->dev, "%s failed with %d\n", __func__, res); + kfree(req); + usb_free_urb(async_urb); } return res; } diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c index 091bc2aca7e8e..a5d364fbc3639 100644 --- a/drivers/net/usb/sr9700.c +++ b/drivers/net/usb/sr9700.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include "sr9700.h" @@ -52,7 +51,7 @@ static int sr_read_reg(struct usbnet *dev, u8 reg, u8 *value) static int sr_write_reg(struct usbnet *dev, u8 reg, u8 value) { - return usbnet_write_cmd(dev, SR_WR_REGS, SR_REQ_WR_REG, + return usbnet_write_cmd(dev, SR_WR_REG, SR_REQ_WR_REG, value, reg, NULL, 0); } @@ -65,7 +64,7 @@ static void sr_write_async(struct usbnet *dev, u8 reg, u16 length, static void sr_write_reg_async(struct usbnet *dev, u8 reg, u8 value) { - usbnet_write_cmd_async(dev, SR_WR_REGS, SR_REQ_WR_REG, + usbnet_write_cmd_async(dev, SR_WR_REG, SR_REQ_WR_REG, value, reg, NULL, 0); } @@ -265,31 +264,15 @@ static const struct ethtool_ops sr9700_ethtool_ops = { static void sr9700_set_multicast(struct net_device *netdev) { struct usbnet *dev = netdev_priv(netdev); - /* We use the 20 byte dev->data for our 8 byte filter buffer - * to avoid allocating memory that is tricky to free later - */ - u8 *hashes = (u8 *)&dev->data; /* rx_ctl setting : enable, disable_long, disable_crc */ u8 rx_ctl = RCR_RXEN | RCR_DIS_CRC | RCR_DIS_LONG; - memset(hashes, 0x00, SR_MCAST_SIZE); - /* broadcast address */ - hashes[SR_MCAST_SIZE - 1] |= SR_MCAST_ADDR_FLAG; - if (netdev->flags & IFF_PROMISC) { + if (netdev->flags & IFF_PROMISC) rx_ctl |= RCR_PRMSC; - } else if (netdev->flags & IFF_ALLMULTI || - netdev_mc_count(netdev) > SR_MCAST_MAX) { - rx_ctl |= RCR_RUNT; - } else if (!netdev_mc_empty(netdev)) { - struct netdev_hw_addr *ha; - - netdev_for_each_mc_addr(ha, netdev) { - u32 crc = ether_crc(ETH_ALEN, ha->addr) >> 26; - hashes[crc >> 3] |= 1 << (crc & 0x7); - } - } + else if (netdev->flags & IFF_ALLMULTI || !netdev_mc_empty(netdev)) + /* The chip has no multicast filter */ + rx_ctl |= RCR_ALL; - sr_write_async(dev, SR_MAR, SR_MCAST_SIZE, hashes); sr_write_reg_async(dev, SR_RCR, rx_ctl); } @@ -539,6 +522,11 @@ static const struct usb_device_id products[] = { USB_DEVICE(0x0fe6, 0x9700), /* SR9700 device */ .driver_info = (unsigned long)&sr9700_driver_info, }, + { + /* SR9700 with virtual driver CD-ROM - interface 0 is the CD-ROM device */ + USB_DEVICE_INTERFACE_NUMBER(0x0fe6, 0x9702, 1), + .driver_info = (unsigned long)&sr9700_driver_info, + }, {}, /* END */ }; diff --git a/drivers/net/usb/sr9700.h b/drivers/net/usb/sr9700.h index ea2b4de621c86..c479908f7d823 100644 --- a/drivers/net/usb/sr9700.h +++ b/drivers/net/usb/sr9700.h @@ -104,9 +104,7 @@ #define WCR_LINKEN (1 << 5) /* Physical Address Reg */ #define SR_PAR 0x10 /* 0x10 ~ 0x15 6 bytes for PAR */ -/* Multicast Address Reg */ -#define SR_MAR 0x16 /* 0x16 ~ 0x1D 8 bytes for MAR */ -/* 0x1e unused */ +/* 0x16 --> 0x1E unused */ /* Phy Reset Reg */ #define SR_PRR 0x1F #define PRR_PHY_RST (1 << 0) @@ -161,9 +159,6 @@ /* parameters */ #define SR_SHARE_TIMEOUT 1000 #define SR_EEPROM_LEN 256 -#define SR_MCAST_SIZE 8 -#define SR_MCAST_ADDR_FLAG 0x80 -#define SR_MCAST_MAX 64 #define SR_TX_OVERHEAD 2 /* 2bytes header */ #define SR_RX_OVERHEAD 7 /* 3bytes header + 4crc tail */ diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index 697cd9d866d3d..1a50775abbdac 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -1803,9 +1803,13 @@ usbnet_probe (struct usb_interface *udev, const struct usb_device_id *prod) if ((dev->driver_info->flags & FLAG_NOARP) != 0) net->flags |= IFF_NOARP; - /* maybe the remote can't receive an Ethernet MTU */ + if ((dev->driver_info->flags & FLAG_NOMAXMTU) == 0 && + net->max_mtu > (dev->hard_mtu - net->hard_header_len)) + net->max_mtu = dev->hard_mtu - net->hard_header_len; + if (net->mtu > (dev->hard_mtu - net->hard_header_len)) net->mtu = dev->hard_mtu - net->hard_header_len; + } else if (!info->in || !info->out) status = usbnet_get_endpoints (dev, udev); else { diff --git a/drivers/net/veth.c b/drivers/net/veth.c index cc502bf022d55..b00613cb07cf0 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -228,16 +228,20 @@ static void veth_get_ethtool_stats(struct net_device *dev, const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; const void *base = (void *)&rq_stats->vs; unsigned int start, tx_idx = idx; + u64 buf[VETH_TQ_STATS_LEN]; size_t offset; - tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; do { start = u64_stats_fetch_begin(&rq_stats->syncp); for (j = 0; j < VETH_TQ_STATS_LEN; j++) { offset = veth_tq_stats_desc[j].offset; - data[tx_idx + j] += *(u64 *)(base + offset); + buf[j] = *(u64 *)(base + offset); } } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); + + tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; + for (j = 0; j < VETH_TQ_STATS_LEN; j++) + data[tx_idx + j] += buf[j]; } pp_idx = idx + dev->real_num_tx_queues * VETH_TQ_STATS_LEN; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 8e04adb57f52a..0cfe7ab59412c 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -381,8 +381,6 @@ struct receive_queue { struct xdp_buff **xsk_buffs; }; -#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 - /* Control VQ buffers: protected by the rtnl lock */ struct control_buf { struct virtio_net_ctrl_hdr hdr; @@ -425,9 +423,6 @@ struct virtnet_info { u16 rss_indir_table_size; u32 rss_hash_types_supported; u32 rss_hash_types_saved; - struct virtio_net_rss_config_hdr *rss_hdr; - struct virtio_net_rss_config_trailer rss_trailer; - u8 rss_hash_key_data[VIRTIO_NET_RSS_MAX_KEY_SIZE]; /* Has control virtqueue */ bool has_cvq; @@ -493,7 +488,16 @@ struct virtnet_info { struct failover *failover; u64 device_stats_cap; + + struct virtio_net_rss_config_hdr *rss_hdr; + + /* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct virtio_net_rss_config_trailer, rss_trailer, hash_key_data, + u8 rss_hash_key_data[NETDEV_RSS_KEY_LEN]; + ); }; +static_assert(offsetof(struct virtnet_info, rss_trailer.hash_key_data) == + offsetof(struct virtnet_info, rss_hash_key_data)); struct padded_vnet_hdr { struct virtio_net_hdr_v1_hash hdr; @@ -3031,16 +3035,16 @@ static int virtnet_receive(struct receive_queue *rq, int budget, else packets = virtnet_receive_packets(vi, rq, budget, xdp_xmit, &stats); + u64_stats_set(&stats.packets, packets); if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { - if (!try_fill_recv(vi, rq, GFP_ATOMIC)) { - spin_lock(&vi->refill_lock); - if (vi->refill_enabled) - schedule_delayed_work(&vi->refill, 0); - spin_unlock(&vi->refill_lock); - } + if (!try_fill_recv(vi, rq, GFP_ATOMIC)) + /* We need to retry refilling in the next NAPI poll so + * we must return budget to make sure the NAPI is + * repolled. + */ + packets = budget; } - u64_stats_set(&stats.packets, packets); u64_stats_update_begin(&rq->stats.syncp); for (i = 0; i < ARRAY_SIZE(virtnet_rq_stats_desc); i++) { size_t offset = virtnet_rq_stats_desc[i].offset; @@ -3220,9 +3224,10 @@ static int virtnet_open(struct net_device *dev) for (i = 0; i < vi->max_queue_pairs; i++) { if (i < vi->curr_queue_pairs) - /* Make sure we have some buffers: if oom use wq. */ - if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) - schedule_delayed_work(&vi->refill, 0); + /* Pre-fill rq agressively, to make sure we are ready to + * get packets immediately. + */ + try_fill_recv(vi, &vi->rq[i], GFP_KERNEL); err = virtnet_enable_queue_pair(vi, i); if (err < 0) @@ -3319,8 +3324,12 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) struct virtio_net_hdr_v1_hash_tunnel *hdr; int num_sg; unsigned hdr_len = vi->hdr_len; + bool feature_hdrlen; bool can_push; + feature_hdrlen = virtio_has_feature(vi->vdev, + VIRTIO_NET_F_GUEST_HDRLEN); + pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); /* Make sure it's safe to cast between formats */ @@ -3340,7 +3349,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) if (virtio_net_hdr_tnl_from_skb(skb, hdr, vi->tx_tnl, virtio_is_little_endian(vi->vdev), 0, - false)) + false, feature_hdrlen)) return -EPROTO; if (vi->mergeable_rx_bufs) @@ -3403,6 +3412,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) /* Don't wait up for transmitted skbs to be freed. */ if (!use_napi) { skb_orphan(skb); + skb_dst_drop(skb); nf_reset_ct(skb); } @@ -3467,16 +3477,15 @@ static void __virtnet_rx_resume(struct virtnet_info *vi, struct receive_queue *rq, bool refill) { - bool running = netif_running(vi->dev); - bool schedule_refill = false; + if (netif_running(vi->dev)) { + /* Pre-fill rq agressively, to make sure we are ready to get + * packets immediately. + */ + if (refill) + try_fill_recv(vi, rq, GFP_KERNEL); - if (refill && !try_fill_recv(vi, rq, GFP_KERNEL)) - schedule_refill = true; - if (running) virtnet_napi_enable(rq); - - if (schedule_refill) - schedule_delayed_work(&vi->refill, 0); + } } static void virtnet_rx_resume_all(struct virtnet_info *vi) @@ -3783,7 +3792,7 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) if (vi->has_rss && !netif_is_rxfh_configured(dev)) { old_rss_hdr = vi->rss_hdr; old_rss_trailer = vi->rss_trailer; - vi->rss_hdr = devm_kzalloc(&dev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); + vi->rss_hdr = devm_kzalloc(&vi->vdev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); if (!vi->rss_hdr) { vi->rss_hdr = old_rss_hdr; return -ENOMEM; @@ -3794,7 +3803,7 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) if (!virtnet_commit_rss_command(vi)) { /* restore ctrl_rss if commit_rss_command failed */ - devm_kfree(&dev->dev, vi->rss_hdr); + devm_kfree(&vi->vdev->dev, vi->rss_hdr); vi->rss_hdr = old_rss_hdr; vi->rss_trailer = old_rss_trailer; @@ -3802,7 +3811,7 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) queue_pairs); return -EINVAL; } - devm_kfree(&dev->dev, old_rss_hdr); + devm_kfree(&vi->vdev->dev, old_rss_hdr); goto succ; } @@ -3821,11 +3830,12 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) } succ: vi->curr_queue_pairs = queue_pairs; - /* virtnet_open() will refill when device is going to up. */ - spin_lock_bh(&vi->refill_lock); - if (dev->flags & IFF_UP && vi->refill_enabled) - schedule_delayed_work(&vi->refill, 0); - spin_unlock_bh(&vi->refill_lock); + if (dev->flags & IFF_UP) { + local_bh_disable(); + for (int i = 0; i < vi->curr_queue_pairs; ++i) + virtqueue_napi_schedule(&vi->rq[i].napi, vi->rq[i].vq); + local_bh_enable(); + } return 0; } @@ -6782,6 +6792,7 @@ static int virtnet_probe(struct virtio_device *vdev) struct virtnet_info *vi; u16 max_queue_pairs; int mtu = 0; + u16 key_sz; /* Find if host supports multiqueue/rss virtio_net device */ max_queue_pairs = 1; @@ -6917,14 +6928,13 @@ static int virtnet_probe(struct virtio_device *vdev) } if (vi->has_rss || vi->has_rss_hash_report) { - vi->rss_key_size = - virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); - if (vi->rss_key_size > VIRTIO_NET_RSS_MAX_KEY_SIZE) { - dev_err(&vdev->dev, "rss_max_key_size=%u exceeds the limit %u.\n", - vi->rss_key_size, VIRTIO_NET_RSS_MAX_KEY_SIZE); - err = -EINVAL; - goto free; - } + key_sz = virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); + + vi->rss_key_size = min_t(u16, key_sz, NETDEV_RSS_KEY_LEN); + if (key_sz > vi->rss_key_size) + dev_warn(&vdev->dev, + "rss_max_key_size=%u exceeds driver limit %u, clamping\n", + key_sz, vi->rss_key_size); vi->rss_hash_types_supported = virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index a5c55e7e4d795..d2d0e0bd43716 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -1965,12 +1965,14 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, ns_olen = request->len - skb_network_offset(request) - sizeof(struct ipv6hdr) - sizeof(*ns); for (i = 0; i < ns_olen-1; i += (ns->opt[i+1]<<3)) { - if (!ns->opt[i + 1]) { + if (!ns->opt[i + 1] || i + (ns->opt[i + 1] << 3) > ns_olen) { kfree_skb(reply); return NULL; } if (ns->opt[i] == ND_OPT_SOURCE_LL_ADDR) { - daddr = ns->opt + i + sizeof(struct nd_opt_hdr); + if ((ns->opt[i + 1] << 3) >= + sizeof(struct nd_opt_hdr) + ETH_ALEN) + daddr = ns->opt + i + sizeof(struct nd_opt_hdr); break; } } @@ -2130,6 +2132,11 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb) { struct ipv6hdr *pip6; + /* check if nd_tbl is not initiliazed due to + * ipv6.disable=1 set during boot + */ + if (!ipv6_stub->nd_tbl) + return false; if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) return false; pip6 = ipv6_hdr(skb); @@ -2349,7 +2356,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, int addr_family; __u8 tos, ttl; int ifindex; - int err; + int err = 0; u32 flags = vxlan->cfg.flags; bool use_cache; bool udp_sum = false; @@ -2454,12 +2461,18 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, rcu_read_lock(); if (addr_family == AF_INET) { - struct vxlan_sock *sock4 = rcu_dereference(vxlan->vn4_sock); + struct vxlan_sock *sock4; u16 ipcb_flags = 0; struct rtable *rt; __be16 df = 0; __be32 saddr; + sock4 = rcu_dereference(vxlan->vn4_sock); + if (unlikely(!sock4)) { + reason = SKB_DROP_REASON_DEV_READY; + goto tx_error; + } + if (!ifindex) ifindex = sock4->sock->sk->sk_bound_dev_if; @@ -2534,10 +2547,16 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, ipcb_flags); #if IS_ENABLED(CONFIG_IPV6) } else { - struct vxlan_sock *sock6 = rcu_dereference(vxlan->vn6_sock); + struct vxlan_sock *sock6; struct in6_addr saddr; u16 ip6cb_flags = 0; + sock6 = rcu_dereference(vxlan->vn6_sock); + if (unlikely(!sock6)) { + reason = SKB_DROP_REASON_DEV_READY; + goto tx_error; + } + if (!ifindex) ifindex = sock6->sock->sk->sk_bound_dev_if; diff --git a/drivers/net/wan/farsync.c b/drivers/net/wan/farsync.c index 5b01642ca44e0..6b2d1e63855e8 100644 --- a/drivers/net/wan/farsync.c +++ b/drivers/net/wan/farsync.c @@ -2550,6 +2550,8 @@ fst_remove_one(struct pci_dev *pdev) fst_disable_intr(card); free_irq(card->irq, card); + tasklet_kill(&fst_tx_task); + tasklet_kill(&fst_int_task); iounmap(card->ctlmem); iounmap(card->mem); diff --git a/drivers/net/wan/fsl_ucc_hdlc.c b/drivers/net/wan/fsl_ucc_hdlc.c index f999798a56127..dff84731343cc 100644 --- a/drivers/net/wan/fsl_ucc_hdlc.c +++ b/drivers/net/wan/fsl_ucc_hdlc.c @@ -790,18 +790,14 @@ static void uhdlc_memclean(struct ucc_hdlc_private *priv) if (priv->rx_buffer) { dma_free_coherent(priv->dev, - RX_BD_RING_LEN * MAX_RX_BUF_LENGTH, + (RX_BD_RING_LEN + TX_BD_RING_LEN) * MAX_RX_BUF_LENGTH, priv->rx_buffer, priv->dma_rx_addr); priv->rx_buffer = NULL; priv->dma_rx_addr = 0; - } - if (priv->tx_buffer) { - dma_free_coherent(priv->dev, - TX_BD_RING_LEN * MAX_RX_BUF_LENGTH, - priv->tx_buffer, priv->dma_tx_addr); priv->tx_buffer = NULL; priv->dma_tx_addr = 0; + } } diff --git a/drivers/net/wan/lapbether.c b/drivers/net/wan/lapbether.c index f357a7ac70ac4..9861c99ea56c4 100644 --- a/drivers/net/wan/lapbether.c +++ b/drivers/net/wan/lapbether.c @@ -446,33 +446,36 @@ static void lapbeth_free_device(struct lapbethdev *lapbeth) static int lapbeth_device_event(struct notifier_block *this, unsigned long event, void *ptr) { - struct lapbethdev *lapbeth; struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct lapbethdev *lapbeth; if (dev_net(dev) != &init_net) return NOTIFY_DONE; - if (!dev_is_ethdev(dev) && !lapbeth_get_x25_dev(dev)) + lapbeth = lapbeth_get_x25_dev(dev); + if (!dev_is_ethdev(dev) && !lapbeth) return NOTIFY_DONE; switch (event) { case NETDEV_UP: /* New ethernet device -> new LAPB interface */ - if (!lapbeth_get_x25_dev(dev)) + if (!lapbeth) lapbeth_new_device(dev); break; case NETDEV_GOING_DOWN: /* ethernet device closes -> close LAPB interface */ - lapbeth = lapbeth_get_x25_dev(dev); if (lapbeth) dev_close(lapbeth->axdev); break; case NETDEV_UNREGISTER: /* ethernet device disappears -> remove LAPB interface */ - lapbeth = lapbeth_get_x25_dev(dev); if (lapbeth) lapbeth_free_device(lapbeth); break; + case NETDEV_PRE_TYPE_CHANGE: + /* Our underlying device type must not change. */ + if (lapbeth) + return NOTIFY_BAD; } return NOTIFY_DONE; diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c index 46a71ec36af87..67b07ee2d6600 100644 --- a/drivers/net/wireguard/device.c +++ b/drivers/net/wireguard/device.c @@ -411,12 +411,11 @@ static struct rtnl_link_ops link_ops __read_mostly = { .newlink = wg_newlink, }; -static void wg_netns_pre_exit(struct net *net) +static void __net_exit wg_netns_exit_rtnl(struct net *net, struct list_head *dev_kill_list) { struct wg_device *wg; struct wg_peer *peer; - rtnl_lock(); list_for_each_entry(wg, &device_list, device_list) { if (rcu_access_pointer(wg->creating_net) == net) { pr_debug("%s: Creating namespace exiting\n", wg->dev->name); @@ -429,11 +428,10 @@ static void wg_netns_pre_exit(struct net *net) mutex_unlock(&wg->device_update_lock); } } - rtnl_unlock(); } -static struct pernet_operations pernet_ops = { - .pre_exit = wg_netns_pre_exit +static struct pernet_operations pernet_ops __read_mostly = { + .exit_rtnl = wg_netns_exit_rtnl }; int __init wg_device_init(void) diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c index 7bbda46cfd93c..82f120ee1c66d 100644 --- a/drivers/net/wireless/ath/ath10k/ce.c +++ b/drivers/net/wireless/ath/ath10k/ce.c @@ -1727,8 +1727,8 @@ static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) (ce_state->src_ring->nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN), - ce_state->src_ring->base_addr_owner_space, - ce_state->src_ring->base_addr_ce_space); + ce_state->src_ring->base_addr_owner_space_unaligned, + ce_state->src_ring->base_addr_ce_space_unaligned); kfree(ce_state->src_ring); } @@ -1737,8 +1737,8 @@ static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) (ce_state->dest_ring->nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN), - ce_state->dest_ring->base_addr_owner_space, - ce_state->dest_ring->base_addr_ce_space); + ce_state->dest_ring->base_addr_owner_space_unaligned, + ce_state->dest_ring->base_addr_ce_space_unaligned); kfree(ce_state->dest_ring); } @@ -1758,8 +1758,8 @@ static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) (ce_state->src_ring->nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN), - ce_state->src_ring->base_addr_owner_space, - ce_state->src_ring->base_addr_ce_space); + ce_state->src_ring->base_addr_owner_space_unaligned, + ce_state->src_ring->base_addr_ce_space_unaligned); kfree(ce_state->src_ring); } @@ -1768,8 +1768,8 @@ static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) (ce_state->dest_ring->nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN), - ce_state->dest_ring->base_addr_owner_space, - ce_state->dest_ring->base_addr_ce_space); + ce_state->dest_ring->base_addr_owner_space_unaligned, + ce_state->dest_ring->base_addr_ce_space_unaligned); kfree(ce_state->dest_ring); } diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c index 6f78f1752cd6f..9ae3595fb6986 100644 --- a/drivers/net/wireless/ath/ath10k/core.c +++ b/drivers/net/wireless/ath/ath10k/core.c @@ -3,7 +3,6 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -2493,8 +2492,9 @@ static int ath10k_init_hw_params(struct ath10k *ar) return 0; } -static bool ath10k_core_needs_recovery(struct ath10k *ar) +static void ath10k_core_recovery_check_work(struct work_struct *work) { + struct ath10k *ar = container_of(work, struct ath10k, recovery_check_work); long time_left; /* Sometimes the recovery will fail and then the next all recovery fail, @@ -2504,7 +2504,7 @@ static bool ath10k_core_needs_recovery(struct ath10k *ar) ath10k_err(ar, "consecutive fail %d times, will shutdown driver!", atomic_read(&ar->fail_cont_count)); ar->state = ATH10K_STATE_WEDGED; - return false; + return; } ath10k_dbg(ar, ATH10K_DBG_BOOT, "total recovery count: %d", ++ar->recovery_count); @@ -2518,27 +2518,24 @@ static bool ath10k_core_needs_recovery(struct ath10k *ar) ATH10K_RECOVERY_TIMEOUT_HZ); if (time_left) { ath10k_warn(ar, "previous recovery succeeded, skip this!\n"); - return false; + return; } /* Record the continuous recovery fail count when recovery failed. */ atomic_inc(&ar->fail_cont_count); /* Avoid having multiple recoveries at the same time. */ - return false; + return; } atomic_inc(&ar->pending_recovery); - - return true; + queue_work(ar->workqueue, &ar->restart_work); } void ath10k_core_start_recovery(struct ath10k *ar) { - if (!ath10k_core_needs_recovery(ar)) - return; - - queue_work(ar->workqueue, &ar->restart_work); + /* Use workqueue_aux to avoid blocking recovery tracking */ + queue_work(ar->workqueue_aux, &ar->recovery_check_work); } EXPORT_SYMBOL(ath10k_core_start_recovery); @@ -3734,6 +3731,7 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev, INIT_WORK(&ar->register_work, ath10k_core_register_work); INIT_WORK(&ar->restart_work, ath10k_core_restart); + INIT_WORK(&ar->recovery_check_work, ath10k_core_recovery_check_work); INIT_WORK(&ar->set_coverage_class_work, ath10k_core_set_coverage_class_work); diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index 8c72ed386edb7..859176fcb5a29 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -3,7 +3,6 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -1208,6 +1207,7 @@ struct ath10k { struct work_struct register_work; struct work_struct restart_work; + struct work_struct recovery_check_work; struct work_struct bundle_tx_work; struct work_struct tx_complete_work; diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c index 154ac7a709824..da6f7957a0ae7 100644 --- a/drivers/net/wireless/ath/ath10k/mac.c +++ b/drivers/net/wireless/ath/ath10k/mac.c @@ -3,7 +3,6 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -5428,6 +5427,7 @@ static void ath10k_stop(struct ieee80211_hw *hw, bool suspend) cancel_work_sync(&ar->set_coverage_class_work); cancel_delayed_work_sync(&ar->scan.timeout); cancel_work_sync(&ar->restart_work); + cancel_work_sync(&ar->recovery_check_work); } static int ath10k_config_ps(struct ath10k *ar) diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c index c06d50db40b81..00d0556dafefd 100644 --- a/drivers/net/wireless/ath/ath10k/sdio.c +++ b/drivers/net/wireless/ath/ath10k/sdio.c @@ -2487,7 +2487,11 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) if (fast_dump) ath10k_bmi_start(ar); + mutex_lock(&ar->dump_mutex); + + spin_lock_bh(&ar->data_lock); ar->stats.fw_crash_counter++; + spin_unlock_bh(&ar->data_lock); ath10k_sdio_disable_intrs(ar); @@ -2505,6 +2509,8 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) ath10k_sdio_enable_intrs(ar); + mutex_unlock(&ar->dump_mutex); + ath10k_core_start_recovery(ar); } diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c index b4aad6604d6d9..ce22141e5efd9 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.c +++ b/drivers/net/wireless/ath/ath10k/wmi.c @@ -5289,8 +5289,6 @@ ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) struct ath10k_sta *arsta; u8 peer_addr[ETH_ALEN]; - lockdep_assert_held(&ar->data_lock); - ev = (struct wmi_peer_sta_ps_state_chg_event *)skb->data; ether_addr_copy(peer_addr, ev->peer_macaddr.addr); @@ -5305,7 +5303,9 @@ ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) } arsta = (struct ath10k_sta *)sta->drv_priv; + spin_lock_bh(&ar->data_lock); arsta->peer_ps_state = __le32_to_cpu(ev->peer_ps_state); + spin_unlock_bh(&ar->data_lock); exit: rcu_read_unlock(); diff --git a/drivers/net/wireless/ath/ath11k/Kconfig b/drivers/net/wireless/ath/ath11k/Kconfig index 659ef134ef168..47dfd39caa89a 100644 --- a/drivers/net/wireless/ath/ath11k/Kconfig +++ b/drivers/net/wireless/ath/ath11k/Kconfig @@ -58,3 +58,14 @@ config ATH11K_SPECTRAL Enable ath11k spectral scan support Say Y to enable access to the FFT/spectral data via debugfs. + +config ATH11K_CFR + bool "ath11k channel frequency response support" + depends on ATH11K_DEBUGFS + depends on RELAY + help + Enable ath11k channel frequency response dump support. + This option exposes debugfs nodes that will allow the user + to enable, disable, and dump data. + + Say Y to enable CFR data dump collection via debugfs. diff --git a/drivers/net/wireless/ath/ath11k/Makefile b/drivers/net/wireless/ath/ath11k/Makefile index d9092414b362d..b1435fcf3e1bf 100644 --- a/drivers/net/wireless/ath/ath11k/Makefile +++ b/drivers/net/wireless/ath/ath11k/Makefile @@ -28,6 +28,7 @@ ath11k-$(CONFIG_THERMAL) += thermal.o ath11k-$(CONFIG_ATH11K_SPECTRAL) += spectral.o ath11k-$(CONFIG_PM) += wow.o ath11k-$(CONFIG_DEV_COREDUMP) += coredump.o +ath11k-$(CONFIG_ATH11K_CFR) += cfr.o obj-$(CONFIG_ATH11K_AHB) += ath11k_ahb.o ath11k_ahb-y += ahb.o diff --git a/drivers/net/wireless/ath/ath11k/cfr.c b/drivers/net/wireless/ath/ath11k/cfr.c new file mode 100644 index 0000000000000..6568edd5d600f --- /dev/null +++ b/drivers/net/wireless/ath/ath11k/cfr.c @@ -0,0 +1,1022 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include "core.h" +#include "debug.h" + +struct ath11k_dbring *ath11k_cfr_get_dbring(struct ath11k *ar) +{ + if (ar->cfr_enabled) + return &ar->cfr.rx_ring; + + return NULL; +} + +static int ath11k_cfr_calculate_tones_from_dma_hdr(struct ath11k_cfr_dma_hdr *hdr) +{ + u8 bw = FIELD_GET(CFIR_DMA_HDR_INFO1_UPLOAD_PKT_BW, hdr->info1); + u8 preamble = FIELD_GET(CFIR_DMA_HDR_INFO1_PREAMBLE_TYPE, hdr->info1); + + switch (preamble) { + case ATH11K_CFR_PREAMBLE_TYPE_LEGACY: + fallthrough; + case ATH11K_CFR_PREAMBLE_TYPE_VHT: + switch (bw) { + case 0: + return TONES_IN_20MHZ; + case 1: /* DUP40/VHT40 */ + return TONES_IN_40MHZ; + case 2: /* DUP80/VHT80 */ + return TONES_IN_80MHZ; + case 3: /* DUP160/VHT160 */ + return TONES_IN_160MHZ; + default: + return TONES_INVALID; + } + case ATH11K_CFR_PREAMBLE_TYPE_HT: + switch (bw) { + case 0: + return TONES_IN_20MHZ; + case 1: + return TONES_IN_40MHZ; + default: + return TONES_INVALID; + } + default: + return TONES_INVALID; + } +} + +void ath11k_cfr_release_lut_entry(struct ath11k_look_up_table *lut) +{ + memset(lut, 0, sizeof(*lut)); +} + +static void ath11k_cfr_rfs_write(struct ath11k *ar, const void *head, + u32 head_len, const void *data, u32 data_len, + const void *tail, int tail_data) +{ + struct ath11k_cfr *cfr = &ar->cfr; + + if (!cfr->rfs_cfr_capture) + return; + + relay_write(cfr->rfs_cfr_capture, head, head_len); + relay_write(cfr->rfs_cfr_capture, data, data_len); + relay_write(cfr->rfs_cfr_capture, tail, tail_data); + relay_flush(cfr->rfs_cfr_capture); +} + +static void ath11k_cfr_free_pending_dbr_events(struct ath11k *ar) +{ + struct ath11k_cfr *cfr = &ar->cfr; + struct ath11k_look_up_table *lut; + int i; + + if (!cfr->lut) + return; + + for (i = 0; i < cfr->lut_num; i++) { + lut = &cfr->lut[i]; + if (lut->dbr_recv && !lut->tx_recv && + lut->dbr_tstamp < cfr->last_success_tstamp) { + ath11k_dbring_bufs_replenish(ar, &cfr->rx_ring, lut->buff, + WMI_DIRECT_BUF_CFR); + ath11k_cfr_release_lut_entry(lut); + cfr->flush_dbr_cnt++; + } + } +} + +/** + * ath11k_cfr_correlate_and_relay() - Correlate and relay CFR events + * @ar: Pointer to ath11k structure + * @lut: Lookup table for correlation + * @event_type: Type of event received (TX or DBR) + * + * Correlates WMI_PDEV_DMA_RING_BUF_RELEASE_EVENT (DBR) and + * WMI_PEER_CFR_CAPTURE_EVENT (TX capture) by PPDU ID. If both events + * are present and the PPDU IDs match, returns CORRELATE_STATUS_RELEASE + * to relay thecorrelated data to userspace. Otherwise returns + * CORRELATE_STATUS_HOLD to wait for the other event. + * + * Also checks pending DBR events and clears them when no corresponding TX + * capture event is received for the PPDU. + * + * Return: CORRELATE_STATUS_RELEASE or CORRELATE_STATUS_HOLD + */ + +static enum ath11k_cfr_correlate_status +ath11k_cfr_correlate_and_relay(struct ath11k *ar, + struct ath11k_look_up_table *lut, + u8 event_type) +{ + enum ath11k_cfr_correlate_status status; + struct ath11k_cfr *cfr = &ar->cfr; + u64 diff; + + if (event_type == ATH11K_CORRELATE_TX_EVENT) { + if (lut->tx_recv) + cfr->cfr_dma_aborts++; + cfr->tx_evt_cnt++; + lut->tx_recv = true; + } else if (event_type == ATH11K_CORRELATE_DBR_EVENT) { + cfr->dbr_evt_cnt++; + lut->dbr_recv = true; + } + + if (lut->dbr_recv && lut->tx_recv) { + if (lut->dbr_ppdu_id == lut->tx_ppdu_id) { + /* + * 64-bit counters make wraparound highly improbable, + * wraparound handling is omitted. + */ + cfr->last_success_tstamp = lut->dbr_tstamp; + if (lut->dbr_tstamp > lut->txrx_tstamp) { + diff = lut->dbr_tstamp - lut->txrx_tstamp; + ath11k_dbg(ar->ab, ATH11K_DBG_CFR, + "txrx event -> dbr event delay = %u ms", + jiffies_to_msecs(diff)); + } else if (lut->txrx_tstamp > lut->dbr_tstamp) { + diff = lut->txrx_tstamp - lut->dbr_tstamp; + ath11k_dbg(ar->ab, ATH11K_DBG_CFR, + "dbr event -> txrx event delay = %u ms", + jiffies_to_msecs(diff)); + } + + ath11k_cfr_free_pending_dbr_events(ar); + + cfr->release_cnt++; + status = ATH11K_CORRELATE_STATUS_RELEASE; + } else { + /* + * Discard TXRX event on PPDU ID mismatch because multiple PPDUs + * may share the same DMA address due to ucode aborts. + */ + + ath11k_dbg(ar->ab, ATH11K_DBG_CFR, + "Received dbr event twice for the same lut entry"); + lut->tx_recv = false; + lut->tx_ppdu_id = 0; + cfr->clear_txrx_event++; + cfr->cfr_dma_aborts++; + status = ATH11K_CORRELATE_STATUS_HOLD; + } + } else { + status = ATH11K_CORRELATE_STATUS_HOLD; + } + + return status; +} + +static int ath11k_cfr_process_data(struct ath11k *ar, + struct ath11k_dbring_data *param) +{ + u32 end_magic = ATH11K_CFR_END_MAGIC; + struct ath11k_csi_cfr_header *header; + struct ath11k_cfr_dma_hdr *dma_hdr; + struct ath11k_cfr *cfr = &ar->cfr; + struct ath11k_look_up_table *lut; + struct ath11k_base *ab = ar->ab; + u32 buf_id, tones, length; + u8 num_chains; + int status; + u8 *data; + + data = param->data; + buf_id = param->buf_id; + + if (param->data_sz < sizeof(*dma_hdr)) + return -EINVAL; + + dma_hdr = (struct ath11k_cfr_dma_hdr *)data; + + tones = ath11k_cfr_calculate_tones_from_dma_hdr(dma_hdr); + if (tones == TONES_INVALID) { + ath11k_warn(ar->ab, "Number of tones received is invalid\n"); + return -EINVAL; + } + + num_chains = FIELD_GET(CFIR_DMA_HDR_INFO1_NUM_CHAINS, + dma_hdr->info1); + + length = sizeof(*dma_hdr); + length += tones * (num_chains + 1); + + spin_lock_bh(&cfr->lut_lock); + + if (!cfr->lut) { + spin_unlock_bh(&cfr->lut_lock); + return -EINVAL; + } + + lut = &cfr->lut[buf_id]; + + ath11k_dbg_dump(ab, ATH11K_DBG_CFR_DUMP, "data_from_buf_rel:", "", + data, length); + + lut->buff = param->buff; + lut->data = data; + lut->data_len = length; + lut->dbr_ppdu_id = dma_hdr->phy_ppdu_id; + lut->dbr_tstamp = jiffies; + + memcpy(&lut->hdr, dma_hdr, sizeof(*dma_hdr)); + + header = &lut->header; + header->meta_data.channel_bw = FIELD_GET(CFIR_DMA_HDR_INFO1_UPLOAD_PKT_BW, + dma_hdr->info1); + header->meta_data.length = length; + + status = ath11k_cfr_correlate_and_relay(ar, lut, + ATH11K_CORRELATE_DBR_EVENT); + if (status == ATH11K_CORRELATE_STATUS_RELEASE) { + ath11k_dbg(ab, ATH11K_DBG_CFR, + "releasing CFR data to user space"); + ath11k_cfr_rfs_write(ar, &lut->header, + sizeof(struct ath11k_csi_cfr_header), + lut->data, lut->data_len, + &end_magic, sizeof(u32)); + ath11k_cfr_release_lut_entry(lut); + } else if (status == ATH11K_CORRELATE_STATUS_HOLD) { + ath11k_dbg(ab, ATH11K_DBG_CFR, + "tx event is not yet received holding the buf"); + } + + spin_unlock_bh(&cfr->lut_lock); + + return status; +} + +static void ath11k_cfr_fill_hdr_info(struct ath11k *ar, + struct ath11k_csi_cfr_header *header, + struct ath11k_cfr_peer_tx_param *params) +{ + struct ath11k_cfr *cfr; + + cfr = &ar->cfr; + header->cfr_metadata_version = ATH11K_CFR_META_VERSION_4; + header->cfr_data_version = ATH11K_CFR_DATA_VERSION_1; + header->cfr_metadata_len = sizeof(struct cfr_metadata); + header->chip_type = ar->ab->hw_rev; + header->meta_data.status = FIELD_GET(WMI_CFR_PEER_CAPTURE_STATUS, + params->status); + header->meta_data.capture_bw = params->bandwidth; + + /* + * FW reports phymode will always be HE mode. + * Replace it with cached phy mode during peer assoc + */ + header->meta_data.phy_mode = cfr->phymode; + + header->meta_data.prim20_chan = params->primary_20mhz_chan; + header->meta_data.center_freq1 = params->band_center_freq1; + header->meta_data.center_freq2 = params->band_center_freq2; + + /* + * CFR capture is triggered by the ACK of a QoS Null frame: + * - 20 MHz: Legacy ACK + * - 40/80/160 MHz: DUP Legacy ACK + */ + header->meta_data.capture_mode = params->bandwidth ? + ATH11K_CFR_CAPTURE_DUP_LEGACY_ACK : ATH11K_CFR_CAPTURE_LEGACY_ACK; + header->meta_data.capture_type = params->capture_method; + header->meta_data.num_rx_chain = ar->num_rx_chains; + header->meta_data.sts_count = params->spatial_streams; + header->meta_data.timestamp = params->timestamp_us; + ether_addr_copy(header->meta_data.peer_addr, params->peer_mac_addr); + memcpy(header->meta_data.chain_rssi, params->chain_rssi, + sizeof(params->chain_rssi)); + memcpy(header->meta_data.chain_phase, params->chain_phase, + sizeof(params->chain_phase)); + memcpy(header->meta_data.agc_gain, params->agc_gain, + sizeof(params->agc_gain)); +} + +int ath11k_process_cfr_capture_event(struct ath11k_base *ab, + struct ath11k_cfr_peer_tx_param *params) +{ + struct ath11k_look_up_table *lut = NULL; + u32 end_magic = ATH11K_CFR_END_MAGIC; + struct ath11k_csi_cfr_header *header; + struct ath11k_dbring_element *buff; + struct ath11k_cfr *cfr; + dma_addr_t buf_addr; + struct ath11k *ar; + u8 tx_status; + int status; + int i; + + rcu_read_lock(); + ar = ath11k_mac_get_ar_by_vdev_id(ab, params->vdev_id); + if (!ar) { + rcu_read_unlock(); + ath11k_warn(ab, "Failed to get ar for vdev id %d\n", + params->vdev_id); + return -ENOENT; + } + + cfr = &ar->cfr; + rcu_read_unlock(); + + if (WMI_CFR_CAPTURE_STATUS_PEER_PS & params->status) { + ath11k_warn(ab, "CFR capture failed as peer %pM is in powersave", + params->peer_mac_addr); + return -EINVAL; + } + + if (!(WMI_CFR_PEER_CAPTURE_STATUS & params->status)) { + ath11k_warn(ab, "CFR capture failed for the peer : %pM", + params->peer_mac_addr); + cfr->tx_peer_status_cfr_fail++; + return -EINVAL; + } + + tx_status = FIELD_GET(WMI_CFR_FRAME_TX_STATUS, params->status); + if (tx_status != WMI_FRAME_TX_STATUS_OK) { + ath11k_warn(ab, "WMI tx status %d for the peer %pM", + tx_status, params->peer_mac_addr); + cfr->tx_evt_status_cfr_fail++; + return -EINVAL; + } + + buf_addr = (((u64)FIELD_GET(WMI_CFR_CORRELATION_INFO2_BUF_ADDR_HIGH, + params->correlation_info_2)) << 32) | + params->correlation_info_1; + + spin_lock_bh(&cfr->lut_lock); + + if (!cfr->lut) { + spin_unlock_bh(&cfr->lut_lock); + return -EINVAL; + } + + for (i = 0; i < cfr->lut_num; i++) { + struct ath11k_look_up_table *temp = &cfr->lut[i]; + + if (temp->dbr_address == buf_addr) { + lut = &cfr->lut[i]; + break; + } + } + + if (!lut) { + spin_unlock_bh(&cfr->lut_lock); + ath11k_warn(ab, "lut failure to process tx event\n"); + cfr->tx_dbr_lookup_fail++; + return -EINVAL; + } + + lut->tx_ppdu_id = FIELD_GET(WMI_CFR_CORRELATION_INFO2_PPDU_ID, + params->correlation_info_2); + lut->txrx_tstamp = jiffies; + + header = &lut->header; + header->start_magic_num = ATH11K_CFR_START_MAGIC; + header->vendorid = VENDOR_QCA; + header->platform_type = PLATFORM_TYPE_ARM; + + ath11k_cfr_fill_hdr_info(ar, header, params); + + status = ath11k_cfr_correlate_and_relay(ar, lut, + ATH11K_CORRELATE_TX_EVENT); + if (status == ATH11K_CORRELATE_STATUS_RELEASE) { + ath11k_dbg(ab, ATH11K_DBG_CFR, + "Releasing CFR data to user space"); + ath11k_cfr_rfs_write(ar, &lut->header, + sizeof(struct ath11k_csi_cfr_header), + lut->data, lut->data_len, + &end_magic, sizeof(u32)); + buff = lut->buff; + ath11k_cfr_release_lut_entry(lut); + + ath11k_dbring_bufs_replenish(ar, &cfr->rx_ring, buff, + WMI_DIRECT_BUF_CFR); + } else if (status == ATH11K_CORRELATE_STATUS_HOLD) { + ath11k_dbg(ab, ATH11K_DBG_CFR, + "dbr event is not yet received holding buf\n"); + } + + spin_unlock_bh(&cfr->lut_lock); + + return 0; +} + +/* Helper function to check whether the given peer mac address + * is in unassociated peer pool or not. + */ +bool ath11k_cfr_peer_is_in_cfr_unassoc_pool(struct ath11k *ar, const u8 *peer_mac) +{ + struct ath11k_cfr *cfr = &ar->cfr; + struct cfr_unassoc_pool_entry *entry; + int i; + + if (!ar->cfr_enabled) + return false; + + spin_lock_bh(&cfr->lock); + for (i = 0; i < ATH11K_MAX_CFR_ENABLED_CLIENTS; i++) { + entry = &cfr->unassoc_pool[i]; + if (!entry->is_valid) + continue; + + if (ether_addr_equal(peer_mac, entry->peer_mac)) { + spin_unlock_bh(&cfr->lock); + return true; + } + } + + spin_unlock_bh(&cfr->lock); + + return false; +} + +void ath11k_cfr_update_unassoc_pool_entry(struct ath11k *ar, + const u8 *peer_mac) +{ + struct ath11k_cfr *cfr = &ar->cfr; + struct cfr_unassoc_pool_entry *entry; + int i; + + spin_lock_bh(&cfr->lock); + for (i = 0; i < ATH11K_MAX_CFR_ENABLED_CLIENTS; i++) { + entry = &cfr->unassoc_pool[i]; + if (!entry->is_valid) + continue; + + if (ether_addr_equal(peer_mac, entry->peer_mac) && + entry->period == 0) { + memset(entry->peer_mac, 0, ETH_ALEN); + entry->is_valid = false; + cfr->cfr_enabled_peer_cnt--; + break; + } + } + + spin_unlock_bh(&cfr->lock); +} + +void ath11k_cfr_decrement_peer_count(struct ath11k *ar, + struct ath11k_sta *arsta) +{ + struct ath11k_cfr *cfr = &ar->cfr; + + spin_lock_bh(&cfr->lock); + + if (arsta->cfr_capture.cfr_enable) + cfr->cfr_enabled_peer_cnt--; + + spin_unlock_bh(&cfr->lock); +} + +static enum ath11k_wmi_cfr_capture_bw +ath11k_cfr_bw_to_fw_cfr_bw(enum ath11k_cfr_capture_bw bw) +{ + switch (bw) { + case ATH11K_CFR_CAPTURE_BW_20: + return WMI_PEER_CFR_CAPTURE_BW_20; + case ATH11K_CFR_CAPTURE_BW_40: + return WMI_PEER_CFR_CAPTURE_BW_40; + case ATH11K_CFR_CAPTURE_BW_80: + return WMI_PEER_CFR_CAPTURE_BW_80; + default: + return WMI_PEER_CFR_CAPTURE_BW_MAX; + } +} + +static enum ath11k_wmi_cfr_capture_method +ath11k_cfr_method_to_fw_cfr_method(enum ath11k_cfr_capture_method method) +{ + switch (method) { + case ATH11K_CFR_CAPTURE_METHOD_NULL_FRAME: + return WMI_CFR_CAPTURE_METHOD_NULL_FRAME; + case ATH11K_CFR_CAPTURE_METHOD_NULL_FRAME_WITH_PHASE: + return WMI_CFR_CAPTURE_METHOD_NULL_FRAME_WITH_PHASE; + case ATH11K_CFR_CAPTURE_METHOD_PROBE_RESP: + return WMI_CFR_CAPTURE_METHOD_PROBE_RESP; + default: + return WMI_CFR_CAPTURE_METHOD_MAX; + } +} + +int ath11k_cfr_send_peer_cfr_capture_cmd(struct ath11k *ar, + struct ath11k_sta *arsta, + struct ath11k_per_peer_cfr_capture *params, + const u8 *peer_mac) +{ + struct ath11k_cfr *cfr = &ar->cfr; + struct wmi_peer_cfr_capture_conf_arg arg; + enum ath11k_wmi_cfr_capture_bw bw; + enum ath11k_wmi_cfr_capture_method method; + int ret = 0; + + if (cfr->cfr_enabled_peer_cnt >= ATH11K_MAX_CFR_ENABLED_CLIENTS && + !arsta->cfr_capture.cfr_enable) { + ath11k_err(ar->ab, "CFR enable peer threshold reached %u\n", + cfr->cfr_enabled_peer_cnt); + return -ENOSPC; + } + + if (params->cfr_enable == arsta->cfr_capture.cfr_enable && + params->cfr_period == arsta->cfr_capture.cfr_period && + params->cfr_method == arsta->cfr_capture.cfr_method && + params->cfr_bw == arsta->cfr_capture.cfr_bw) + return ret; + + if (!params->cfr_enable && !arsta->cfr_capture.cfr_enable) + return ret; + + bw = ath11k_cfr_bw_to_fw_cfr_bw(params->cfr_bw); + if (bw >= WMI_PEER_CFR_CAPTURE_BW_MAX) { + ath11k_warn(ar->ab, "FW doesn't support configured bw %d\n", + params->cfr_bw); + return -EINVAL; + } + + method = ath11k_cfr_method_to_fw_cfr_method(params->cfr_method); + if (method >= WMI_CFR_CAPTURE_METHOD_MAX) { + ath11k_warn(ar->ab, "FW doesn't support configured method %d\n", + params->cfr_method); + return -EINVAL; + } + + arg.request = params->cfr_enable; + arg.periodicity = params->cfr_period; + arg.bw = bw; + arg.method = method; + + ret = ath11k_wmi_peer_set_cfr_capture_conf(ar, arsta->arvif->vdev_id, + peer_mac, &arg); + if (ret) { + ath11k_warn(ar->ab, + "failed to send cfr capture info: vdev_id %u peer %pM: %d\n", + arsta->arvif->vdev_id, peer_mac, ret); + return ret; + } + + spin_lock_bh(&cfr->lock); + + if (params->cfr_enable && + params->cfr_enable != arsta->cfr_capture.cfr_enable) + cfr->cfr_enabled_peer_cnt++; + else if (!params->cfr_enable) + cfr->cfr_enabled_peer_cnt--; + + spin_unlock_bh(&cfr->lock); + + arsta->cfr_capture.cfr_enable = params->cfr_enable; + arsta->cfr_capture.cfr_period = params->cfr_period; + arsta->cfr_capture.cfr_method = params->cfr_method; + arsta->cfr_capture.cfr_bw = params->cfr_bw; + + return ret; +} + +void ath11k_cfr_update_unassoc_pool(struct ath11k *ar, + struct ath11k_per_peer_cfr_capture *params, + u8 *peer_mac) +{ + struct ath11k_cfr *cfr = &ar->cfr; + struct cfr_unassoc_pool_entry *entry; + int available_idx = -1; + int i; + + guard(spinlock_bh)(&cfr->lock); + + if (!params->cfr_enable) { + for (i = 0; i < ATH11K_MAX_CFR_ENABLED_CLIENTS; i++) { + entry = &cfr->unassoc_pool[i]; + if (ether_addr_equal(peer_mac, entry->peer_mac)) { + memset(entry->peer_mac, 0, ETH_ALEN); + entry->is_valid = false; + cfr->cfr_enabled_peer_cnt--; + break; + } + } + return; + } + + if (cfr->cfr_enabled_peer_cnt >= ATH11K_MAX_CFR_ENABLED_CLIENTS) { + ath11k_info(ar->ab, "Max cfr peer threshold reached\n"); + return; + } + + for (i = 0; i < ATH11K_MAX_CFR_ENABLED_CLIENTS; i++) { + entry = &cfr->unassoc_pool[i]; + + if (ether_addr_equal(peer_mac, entry->peer_mac)) { + ath11k_info(ar->ab, + "peer entry already present updating params\n"); + entry->period = params->cfr_period; + available_idx = -1; + break; + } + + if (available_idx < 0 && !entry->is_valid) + available_idx = i; + } + + if (available_idx >= 0) { + entry = &cfr->unassoc_pool[available_idx]; + ether_addr_copy(entry->peer_mac, peer_mac); + entry->period = params->cfr_period; + entry->is_valid = true; + cfr->cfr_enabled_peer_cnt++; + } +} + +static ssize_t ath11k_read_file_enable_cfr(struct file *file, + char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + char buf[32] = {}; + size_t len; + + mutex_lock(&ar->conf_mutex); + len = scnprintf(buf, sizeof(buf), "%d\n", ar->cfr_enabled); + mutex_unlock(&ar->conf_mutex); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static ssize_t ath11k_write_file_enable_cfr(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + u32 enable_cfr; + int ret; + + if (kstrtouint_from_user(ubuf, count, 0, &enable_cfr)) + return -EINVAL; + + guard(mutex)(&ar->conf_mutex); + + if (ar->state != ATH11K_STATE_ON) + return -ENETDOWN; + + if (enable_cfr > 1) + return -EINVAL; + + if (ar->cfr_enabled == enable_cfr) + return count; + + ret = ath11k_wmi_pdev_set_param(ar, WMI_PDEV_PARAM_PER_PEER_CFR_ENABLE, + enable_cfr, ar->pdev->pdev_id); + if (ret) { + ath11k_warn(ar->ab, + "Failed to enable/disable per peer cfr %d\n", ret); + return ret; + } + + ar->cfr_enabled = enable_cfr; + + return count; +} + +static const struct file_operations fops_enable_cfr = { + .read = ath11k_read_file_enable_cfr, + .write = ath11k_write_file_enable_cfr, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static ssize_t ath11k_write_file_cfr_unassoc(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + struct ath11k_cfr *cfr = &ar->cfr; + struct cfr_unassoc_pool_entry *entry; + char buf[64] = {}; + u8 peer_mac[6]; + u32 cfr_capture_enable; + u32 cfr_capture_period; + int available_idx = -1; + int ret, i; + + simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, ubuf, count); + + guard(mutex)(&ar->conf_mutex); + guard(spinlock_bh)(&cfr->lock); + + if (ar->state != ATH11K_STATE_ON) + return -ENETDOWN; + + if (!ar->cfr_enabled) { + ath11k_err(ar->ab, "CFR is not enabled on this pdev %d\n", + ar->pdev_idx); + return -EINVAL; + } + + ret = sscanf(buf, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx %u %u", + &peer_mac[0], &peer_mac[1], &peer_mac[2], &peer_mac[3], + &peer_mac[4], &peer_mac[5], &cfr_capture_enable, + &cfr_capture_period); + + if (ret < 1) + return -EINVAL; + + if (cfr_capture_enable && ret != 8) + return -EINVAL; + + if (!cfr_capture_enable) { + for (i = 0; i < ATH11K_MAX_CFR_ENABLED_CLIENTS; i++) { + entry = &cfr->unassoc_pool[i]; + if (ether_addr_equal(peer_mac, entry->peer_mac)) { + memset(entry->peer_mac, 0, ETH_ALEN); + entry->is_valid = false; + cfr->cfr_enabled_peer_cnt--; + } + } + + return count; + } + + if (cfr->cfr_enabled_peer_cnt >= ATH11K_MAX_CFR_ENABLED_CLIENTS) { + ath11k_info(ar->ab, "Max cfr peer threshold reached\n"); + return count; + } + + for (i = 0; i < ATH11K_MAX_CFR_ENABLED_CLIENTS; i++) { + entry = &cfr->unassoc_pool[i]; + + if (available_idx < 0 && !entry->is_valid) + available_idx = i; + + if (ether_addr_equal(peer_mac, entry->peer_mac)) { + ath11k_info(ar->ab, + "peer entry already present updating params\n"); + entry->period = cfr_capture_period; + return count; + } + } + + if (available_idx >= 0) { + entry = &cfr->unassoc_pool[available_idx]; + ether_addr_copy(entry->peer_mac, peer_mac); + entry->period = cfr_capture_period; + entry->is_valid = true; + cfr->cfr_enabled_peer_cnt++; + } + + return count; +} + +static ssize_t ath11k_read_file_cfr_unassoc(struct file *file, + char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + struct ath11k_cfr *cfr = &ar->cfr; + struct cfr_unassoc_pool_entry *entry; + char buf[512] = {}; + int len = 0, i; + + spin_lock_bh(&cfr->lock); + + for (i = 0; i < ATH11K_MAX_CFR_ENABLED_CLIENTS; i++) { + entry = &cfr->unassoc_pool[i]; + if (entry->is_valid) + len += scnprintf(buf + len, sizeof(buf) - len, + "peer: %pM period: %u\n", + entry->peer_mac, entry->period); + } + + spin_unlock_bh(&cfr->lock); + + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations fops_configure_cfr_unassoc = { + .write = ath11k_write_file_cfr_unassoc, + .read = ath11k_read_file_cfr_unassoc, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static void ath11k_cfr_debug_unregister(struct ath11k *ar) +{ + debugfs_remove(ar->cfr.enable_cfr); + ar->cfr.enable_cfr = NULL; + debugfs_remove(ar->cfr.cfr_unassoc); + ar->cfr.cfr_unassoc = NULL; + + relay_close(ar->cfr.rfs_cfr_capture); + ar->cfr.rfs_cfr_capture = NULL; +} + +static struct dentry *ath11k_cfr_create_buf_file_handler(const char *filename, + struct dentry *parent, + umode_t mode, + struct rchan_buf *buf, + int *is_global) +{ + struct dentry *buf_file; + + buf_file = debugfs_create_file(filename, mode, parent, buf, + &relay_file_operations); + *is_global = 1; + return buf_file; +} + +static int ath11k_cfr_remove_buf_file_handler(struct dentry *dentry) +{ + debugfs_remove(dentry); + + return 0; +} + +static const struct rchan_callbacks rfs_cfr_capture_cb = { + .create_buf_file = ath11k_cfr_create_buf_file_handler, + .remove_buf_file = ath11k_cfr_remove_buf_file_handler, +}; + +static void ath11k_cfr_debug_register(struct ath11k *ar) +{ + ar->cfr.rfs_cfr_capture = relay_open("cfr_capture", + ar->debug.debugfs_pdev, + ar->ab->hw_params.cfr_stream_buf_size, + ar->ab->hw_params.cfr_num_stream_bufs, + &rfs_cfr_capture_cb, NULL); + + ar->cfr.enable_cfr = debugfs_create_file("enable_cfr", 0600, + ar->debug.debugfs_pdev, ar, + &fops_enable_cfr); + + ar->cfr.cfr_unassoc = debugfs_create_file("cfr_unassoc", 0600, + ar->debug.debugfs_pdev, ar, + &fops_configure_cfr_unassoc); +} + +void ath11k_cfr_lut_update_paddr(struct ath11k *ar, dma_addr_t paddr, + u32 buf_id) +{ + struct ath11k_cfr *cfr = &ar->cfr; + + if (cfr->lut) + cfr->lut[buf_id].dbr_address = paddr; +} + +void ath11k_cfr_update_phymode(struct ath11k *ar, enum wmi_phy_mode phymode) +{ + struct ath11k_cfr *cfr = &ar->cfr; + cfr->phymode = phymode; +} + +static void ath11k_cfr_ring_free(struct ath11k *ar) +{ + struct ath11k_cfr *cfr = &ar->cfr; + + ath11k_dbring_buf_cleanup(ar, &cfr->rx_ring); + ath11k_dbring_srng_cleanup(ar, &cfr->rx_ring); +} + +static int ath11k_cfr_ring_alloc(struct ath11k *ar, + struct ath11k_dbring_cap *db_cap) +{ + struct ath11k_cfr *cfr = &ar->cfr; + int ret; + + ret = ath11k_dbring_srng_setup(ar, &cfr->rx_ring, + ATH11K_CFR_NUM_RING_ENTRIES, + db_cap->min_elem); + if (ret) { + ath11k_warn(ar->ab, "failed to setup db ring: %d\n", ret); + return ret; + } + + ath11k_dbring_set_cfg(ar, &cfr->rx_ring, + ATH11K_CFR_NUM_RESP_PER_EVENT, + ATH11K_CFR_EVENT_TIMEOUT_MS, + ath11k_cfr_process_data); + + ret = ath11k_dbring_buf_setup(ar, &cfr->rx_ring, db_cap); + if (ret) { + ath11k_warn(ar->ab, "failed to setup db ring buffer: %d\n", ret); + goto srng_cleanup; + } + + ret = ath11k_dbring_wmi_cfg_setup(ar, &cfr->rx_ring, WMI_DIRECT_BUF_CFR); + if (ret) { + ath11k_warn(ar->ab, "failed to setup db ring cfg: %d\n", ret); + goto buffer_cleanup; + } + + return 0; + +buffer_cleanup: + ath11k_dbring_buf_cleanup(ar, &cfr->rx_ring); +srng_cleanup: + ath11k_dbring_srng_cleanup(ar, &cfr->rx_ring); + return ret; +} + +void ath11k_cfr_deinit(struct ath11k_base *ab) +{ + struct ath11k_cfr *cfr; + struct ath11k *ar; + int i; + + if (!test_bit(WMI_TLV_SERVICE_CFR_CAPTURE_SUPPORT, ab->wmi_ab.svc_map) || + !ab->hw_params.cfr_support) + return; + + for (i = 0; i < ab->num_radios; i++) { + ar = ab->pdevs[i].ar; + cfr = &ar->cfr; + + if (!cfr->enabled) + continue; + + ath11k_cfr_debug_unregister(ar); + ath11k_cfr_ring_free(ar); + + spin_lock_bh(&cfr->lut_lock); + kfree(cfr->lut); + cfr->lut = NULL; + cfr->enabled = false; + spin_unlock_bh(&cfr->lut_lock); + } +} + +int ath11k_cfr_init(struct ath11k_base *ab) +{ + struct ath11k_dbring_cap db_cap; + struct ath11k_cfr *cfr; + u32 num_lut_entries; + struct ath11k *ar; + int i, ret; + + if (!test_bit(WMI_TLV_SERVICE_CFR_CAPTURE_SUPPORT, ab->wmi_ab.svc_map) || + !ab->hw_params.cfr_support) + return 0; + + for (i = 0; i < ab->num_radios; i++) { + ar = ab->pdevs[i].ar; + cfr = &ar->cfr; + + ret = ath11k_dbring_get_cap(ar->ab, ar->pdev_idx, + WMI_DIRECT_BUF_CFR, &db_cap); + if (ret) + continue; + + idr_init(&cfr->rx_ring.bufs_idr); + spin_lock_init(&cfr->rx_ring.idr_lock); + spin_lock_init(&cfr->lock); + spin_lock_init(&cfr->lut_lock); + + num_lut_entries = min_t(u32, CFR_MAX_LUT_ENTRIES, db_cap.min_elem); + cfr->lut = kcalloc(num_lut_entries, sizeof(*cfr->lut), + GFP_KERNEL); + if (!cfr->lut) { + ret = -ENOMEM; + goto err; + } + + ret = ath11k_cfr_ring_alloc(ar, &db_cap); + if (ret) { + ath11k_warn(ab, "failed to init cfr ring for pdev %d: %d\n", + i, ret); + spin_lock_bh(&cfr->lut_lock); + kfree(cfr->lut); + cfr->lut = NULL; + cfr->enabled = false; + spin_unlock_bh(&cfr->lut_lock); + goto err; + } + + cfr->lut_num = num_lut_entries; + cfr->enabled = true; + + ath11k_cfr_debug_register(ar); + } + + return 0; + +err: + for (i = i - 1; i >= 0; i--) { + ar = ab->pdevs[i].ar; + cfr = &ar->cfr; + + if (!cfr->enabled) + continue; + + ath11k_cfr_debug_unregister(ar); + ath11k_cfr_ring_free(ar); + + spin_lock_bh(&cfr->lut_lock); + kfree(cfr->lut); + cfr->lut = NULL; + cfr->enabled = false; + spin_unlock_bh(&cfr->lut_lock); + } + return ret; +} diff --git a/drivers/net/wireless/ath/ath11k/cfr.h b/drivers/net/wireless/ath/ath11k/cfr.h new file mode 100644 index 0000000000000..94fcb706f2ef2 --- /dev/null +++ b/drivers/net/wireless/ath/ath11k/cfr.h @@ -0,0 +1,308 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH11K_CFR_H +#define ATH11K_CFR_H + +#include "dbring.h" +#include "wmi.h" + +#define ATH11K_CFR_NUM_RESP_PER_EVENT 1 +#define ATH11K_CFR_EVENT_TIMEOUT_MS 1 +#define ATH11K_CFR_NUM_RING_ENTRIES 1 + +#define ATH11K_MAX_CFR_ENABLED_CLIENTS 10 +#define CFR_MAX_LUT_ENTRIES 136 + +#define HOST_MAX_CHAINS 8 + +enum ath11k_cfr_correlate_event_type { + ATH11K_CORRELATE_DBR_EVENT, + ATH11K_CORRELATE_TX_EVENT, +}; + +struct ath11k_sta; +struct ath11k_per_peer_cfr_capture; + +#define ATH11K_CFR_START_MAGIC 0xDEADBEAF +#define ATH11K_CFR_END_MAGIC 0xBEAFDEAD + +#define VENDOR_QCA 0x8cfdf0 +#define PLATFORM_TYPE_ARM 2 + +enum ath11k_cfr_meta_version { + ATH11K_CFR_META_VERSION_NONE, + ATH11K_CFR_META_VERSION_1, + ATH11K_CFR_META_VERSION_2, + ATH11K_CFR_META_VERSION_3, + ATH11K_CFR_META_VERSION_4, + ATH11K_CFR_META_VERSION_MAX = 0xFF, +}; + +enum ath11k_cfr_data_version { + ATH11K_CFR_DATA_VERSION_NONE, + ATH11K_CFR_DATA_VERSION_1, + ATH11K_CFR_DATA_VERSION_MAX = 0xFF, +}; + +enum ath11k_cfr_capture_ack_mode { + ATH11K_CFR_CAPTURE_LEGACY_ACK, + ATH11K_CFR_CAPTURE_DUP_LEGACY_ACK, + ATH11K_CFR_CAPTURE_HT_ACK, + ATH11K_CFR_CAPTURE_VHT_ACK, + + /*Always keep this at last*/ + ATH11K_CFR_CAPTURE_INVALID_ACK +}; + +enum ath11k_cfr_correlate_status { + ATH11K_CORRELATE_STATUS_RELEASE, + ATH11K_CORRELATE_STATUS_HOLD, + ATH11K_CORRELATE_STATUS_ERR, +}; + +enum ath11k_cfr_preamble_type { + ATH11K_CFR_PREAMBLE_TYPE_LEGACY, + ATH11K_CFR_PREAMBLE_TYPE_HT, + ATH11K_CFR_PREAMBLE_TYPE_VHT, +}; + +struct ath11k_cfr_peer_tx_param { + u32 capture_method; + u32 vdev_id; + u8 peer_mac_addr[ETH_ALEN]; + u32 primary_20mhz_chan; + u32 bandwidth; + u32 phy_mode; + u32 band_center_freq1; + u32 band_center_freq2; + u32 spatial_streams; + u32 correlation_info_1; + u32 correlation_info_2; + u32 status; + u32 timestamp_us; + u32 counter; + u32 chain_rssi[WMI_MAX_CHAINS]; + u16 chain_phase[WMI_MAX_CHAINS]; + u32 cfo_measurement; + u8 agc_gain[HOST_MAX_CHAINS]; + u32 rx_start_ts; +}; + +struct cfr_metadata { + u8 peer_addr[ETH_ALEN]; + u8 status; + u8 capture_bw; + u8 channel_bw; + u8 phy_mode; + u16 prim20_chan; + u16 center_freq1; + u16 center_freq2; + u8 capture_mode; + u8 capture_type; + u8 sts_count; + u8 num_rx_chain; + u32 timestamp; + u32 length; + u32 chain_rssi[HOST_MAX_CHAINS]; + u16 chain_phase[HOST_MAX_CHAINS]; + u32 cfo_measurement; + u8 agc_gain[HOST_MAX_CHAINS]; + u32 rx_start_ts; +} __packed; + +struct ath11k_csi_cfr_header { + u32 start_magic_num; + u32 vendorid; + u8 cfr_metadata_version; + u8 cfr_data_version; + u8 chip_type; + u8 platform_type; + u32 cfr_metadata_len; + struct cfr_metadata meta_data; +} __packed; + +#define TONES_IN_20MHZ 256 +#define TONES_IN_40MHZ 512 +#define TONES_IN_80MHZ 1024 +#define TONES_IN_160MHZ 2048 /* 160 MHz isn't supported yet */ +#define TONES_INVALID 0 + +#define CFIR_DMA_HDR_INFO0_TAG GENMASK(7, 0) +#define CFIR_DMA_HDR_INFO0_LEN GENMASK(13, 8) + +#define CFIR_DMA_HDR_INFO1_UPLOAD_DONE GENMASK(0, 0) +#define CFIR_DMA_HDR_INFO1_CAPTURE_TYPE GENMASK(3, 1) +#define CFIR_DMA_HDR_INFO1_PREAMBLE_TYPE GENMASK(5, 4) +#define CFIR_DMA_HDR_INFO1_NSS GENMASK(8, 6) +#define CFIR_DMA_HDR_INFO1_NUM_CHAINS GENMASK(11, 9) +#define CFIR_DMA_HDR_INFO1_UPLOAD_PKT_BW GENMASK(14, 12) +#define CFIR_DMA_HDR_INFO1_SW_PEER_ID_VALID GENMASK(15, 15) + +struct ath11k_cfr_dma_hdr { + u16 info0; + u16 info1; + u16 sw_peer_id; + u16 phy_ppdu_id; +}; + +struct ath11k_look_up_table { + bool dbr_recv; + bool tx_recv; + u8 *data; + u32 data_len; + u16 dbr_ppdu_id; + u16 tx_ppdu_id; + dma_addr_t dbr_address; + struct ath11k_csi_cfr_header header; + struct ath11k_cfr_dma_hdr hdr; + u64 txrx_tstamp; + u64 dbr_tstamp; + u32 header_length; + u32 payload_length; + struct ath11k_dbring_element *buff; +}; + +struct cfr_unassoc_pool_entry { + u8 peer_mac[ETH_ALEN]; + u32 period; + bool is_valid; +}; + +struct ath11k_cfr { + struct ath11k_dbring rx_ring; + /* Protects cfr data */ + spinlock_t lock; + /* Protect for lut entries */ + spinlock_t lut_lock; + struct ath11k_look_up_table *lut; + struct dentry *enable_cfr; + struct dentry *cfr_unassoc; + struct rchan *rfs_cfr_capture; + u8 cfr_enabled_peer_cnt; + u32 lut_num; + u64 tx_evt_cnt; + u64 dbr_evt_cnt; + u64 release_cnt; + u64 tx_peer_status_cfr_fail; + u64 tx_evt_status_cfr_fail; + u64 tx_dbr_lookup_fail; + u64 last_success_tstamp; + u64 flush_dbr_cnt; + u64 clear_txrx_event; + u64 cfr_dma_aborts; + bool enabled; + enum wmi_phy_mode phymode; + struct cfr_unassoc_pool_entry unassoc_pool[ATH11K_MAX_CFR_ENABLED_CLIENTS]; +}; + +enum ath11k_cfr_capture_method { + ATH11K_CFR_CAPTURE_METHOD_NULL_FRAME, + ATH11K_CFR_CAPTURE_METHOD_NULL_FRAME_WITH_PHASE, + ATH11K_CFR_CAPTURE_METHOD_PROBE_RESP, + ATH11K_CFR_CAPTURE_METHOD_MAX, +}; + +enum ath11k_cfr_capture_bw { + ATH11K_CFR_CAPTURE_BW_20, + ATH11K_CFR_CAPTURE_BW_40, + ATH11K_CFR_CAPTURE_BW_80, + ATH11K_CFR_CAPTURE_BW_MAX, +}; + +#ifdef CONFIG_ATH11K_CFR +int ath11k_cfr_init(struct ath11k_base *ab); +void ath11k_cfr_deinit(struct ath11k_base *ab); +void ath11k_cfr_lut_update_paddr(struct ath11k *ar, dma_addr_t paddr, + u32 buf_id); +void ath11k_cfr_decrement_peer_count(struct ath11k *ar, + struct ath11k_sta *arsta); +void ath11k_cfr_update_unassoc_pool_entry(struct ath11k *ar, + const u8 *peer_mac); +bool ath11k_cfr_peer_is_in_cfr_unassoc_pool(struct ath11k *ar, + const u8 *peer_mac); +void ath11k_cfr_update_unassoc_pool(struct ath11k *ar, + struct ath11k_per_peer_cfr_capture *params, + u8 *peer_mac); +int ath11k_cfr_send_peer_cfr_capture_cmd(struct ath11k *ar, + struct ath11k_sta *arsta, + struct ath11k_per_peer_cfr_capture *params, + const u8 *peer_mac); +struct ath11k_dbring *ath11k_cfr_get_dbring(struct ath11k *ar); +void ath11k_cfr_release_lut_entry(struct ath11k_look_up_table *lut); +int ath11k_process_cfr_capture_event(struct ath11k_base *ab, + struct ath11k_cfr_peer_tx_param *params); +void ath11k_cfr_update_phymode(struct ath11k *ar, enum wmi_phy_mode phymode); +#else +static inline void ath11k_cfr_update_phymode(struct ath11k *ar, + enum wmi_phy_mode phymode) +{ +} + +static inline int ath11k_cfr_init(struct ath11k_base *ab) +{ + return 0; +} + +static inline void ath11k_cfr_deinit(struct ath11k_base *ab) +{ +} + +static inline void ath11k_cfr_lut_update_paddr(struct ath11k *ar, + dma_addr_t paddr, u32 buf_id) +{ +} + +static inline void ath11k_cfr_decrement_peer_count(struct ath11k *ar, + struct ath11k_sta *arsta) +{ +} + +static inline void ath11k_cfr_update_unassoc_pool_entry(struct ath11k *ar, + const u8 *peer_mac) +{ +} + +static inline bool +ath11k_cfr_peer_is_in_cfr_unassoc_pool(struct ath11k *ar, const u8 *peer_mac) +{ + return false; +} + +static inline void +ath11k_cfr_update_unassoc_pool(struct ath11k *ar, + struct ath11k_per_peer_cfr_capture *params, + u8 *peer_mac) +{ +} + +static inline int +ath11k_cfr_send_peer_cfr_capture_cmd(struct ath11k *ar, + struct ath11k_sta *arsta, + struct ath11k_per_peer_cfr_capture *params, + const u8 *peer_mac) +{ + return 0; +} + +static inline void ath11k_cfr_release_lut_entry(struct ath11k_look_up_table *lut) +{ +} + +static inline +struct ath11k_dbring *ath11k_cfr_get_dbring(struct ath11k *ar) +{ + return NULL; +} + +static inline +int ath11k_process_cfr_capture_event(struct ath11k_base *ab, + struct ath11k_cfr_peer_tx_param *params) +{ + return 0; +} +#endif /* CONFIG_ATH11K_CFR */ +#endif /* ATH11K_CFR_H */ diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index 812686173ac8a..b789c4abbd110 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -126,6 +125,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .smp2p_wow_exit = false, .support_dual_stations = false, .pdev_suspend = false, + .cfr_support = true, + .cfr_num_stream_bufs = 255, + .cfr_stream_buf_size = 8200, }, { .hw_rev = ATH11K_HW_IPQ6018_HW10, @@ -211,6 +213,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = false, .support_dual_stations = false, .pdev_suspend = false, + .cfr_support = false, + .cfr_num_stream_bufs = 0, + .cfr_stream_buf_size = 0, }, { .name = "qca6390 hw2.0", @@ -301,6 +306,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = true, .support_dual_stations = true, .pdev_suspend = false, + .cfr_support = false, + .cfr_num_stream_bufs = 0, + .cfr_stream_buf_size = 0, }, { .name = "qcn9074 hw1.0", @@ -385,6 +393,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = false, .support_dual_stations = false, .pdev_suspend = false, + .cfr_support = false, + .cfr_num_stream_bufs = 0, + .cfr_stream_buf_size = 0, }, { .name = "wcn6855 hw2.0", @@ -475,6 +486,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = true, .support_dual_stations = true, .pdev_suspend = false, + .cfr_support = false, + .cfr_num_stream_bufs = 0, + .cfr_stream_buf_size = 0, }, { .name = "wcn6855 hw2.1", @@ -563,6 +577,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = true, .support_dual_stations = true, .pdev_suspend = false, + .cfr_support = true, + .cfr_num_stream_bufs = 255, + .cfr_stream_buf_size = 8200, }, { .name = "wcn6750 hw1.0", @@ -646,6 +663,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = true, .support_dual_stations = false, .pdev_suspend = true, + .cfr_support = false, + .cfr_num_stream_bufs = 0, + .cfr_stream_buf_size = 0, }, { .hw_rev = ATH11K_HW_IPQ5018_HW10, @@ -729,6 +749,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = false, .support_dual_stations = false, .pdev_suspend = false, + .cfr_support = false, + .cfr_num_stream_bufs = 0, + .cfr_stream_buf_size = 0, }, { .name = "qca2066 hw2.1", @@ -818,6 +841,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .smp2p_wow_exit = false, .support_fw_mac_sequence = true, .support_dual_stations = true, + .cfr_support = false, + .cfr_num_stream_bufs = 0, + .cfr_stream_buf_size = 0, }, { .name = "qca6698aq hw2.1", @@ -906,6 +932,9 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .support_fw_mac_sequence = true, .support_dual_stations = true, .pdev_suspend = false, + .cfr_support = true, + .cfr_num_stream_bufs = 255, + .cfr_stream_buf_size = 8200, }, }; @@ -994,9 +1023,64 @@ static const struct dmi_system_id ath11k_pm_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "21F9"), }, }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z13 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D2"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z13 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D3"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z16 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D4"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z16 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D5"), + }, + }, {} }; +static const struct __ath11k_core_usecase_firmware_table { + u32 hw_rev; + const char *compatible; + const char *firmware_name; +} ath11k_core_usecase_firmware_table[] = { + { ATH11K_HW_WCN6855_HW21, "qcom,lemans-evk", "nfa765"}, + { ATH11K_HW_WCN6855_HW21, "qcom,monaco-evk", "nfa765"}, + { ATH11K_HW_WCN6855_HW21, "qcom,hamoa-iot-evk", "nfa765"}, + { /* Sentinel */ } +}; + +const char *ath11k_core_get_usecase_firmware(struct ath11k_base *ab) +{ + const struct __ath11k_core_usecase_firmware_table *entry = NULL; + + entry = ath11k_core_usecase_firmware_table; + while (entry->compatible) { + if (ab->hw_rev == entry->hw_rev && + of_machine_is_compatible(entry->compatible)) + return entry->firmware_name; + entry++; + } + + return NULL; +} +EXPORT_SYMBOL(ath11k_core_get_usecase_firmware); + void ath11k_fw_stats_pdevs_free(struct list_head *head) { struct ath11k_fw_stats_pdev *i, *tmp; @@ -1987,8 +2071,16 @@ static int ath11k_core_pdev_create(struct ath11k_base *ab) goto err_thermal_unregister; } + ret = ath11k_cfr_init(ab); + if (ret) { + ath11k_err(ab, "failed to init cfr %d\n", ret); + goto err_spectral_unregister; + } + return 0; +err_spectral_unregister: + ath11k_spectral_deinit(ab); err_thermal_unregister: ath11k_thermal_unregister(ab); err_mac_unregister: @@ -2038,6 +2130,7 @@ static void ath11k_core_pdev_suspend_target(struct ath11k_base *ab) static void ath11k_core_pdev_destroy(struct ath11k_base *ab) { + ath11k_cfr_deinit(ab); ath11k_spectral_deinit(ab); ath11k_thermal_unregister(ab); ath11k_mac_unregister(ab); @@ -2250,6 +2343,7 @@ static int ath11k_core_reconfigure_on_crash(struct ath11k_base *ab) mutex_lock(&ab->core_lock); ath11k_thermal_unregister(ab); ath11k_dp_pdev_free(ab); + ath11k_cfr_deinit(ab); ath11k_spectral_deinit(ab); ath11k_ce_cleanup_pipes(ab); ath11k_wmi_detach(ab); diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index e8780b05ce11e..c5b0356d0f404 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH11K_CORE_H @@ -35,6 +35,7 @@ #include "wow.h" #include "fw.h" #include "coredump.h" +#include "cfr.h" #define SM(_v, _f) (((_v) << _f##_LSB) & _f##_MASK) @@ -531,6 +532,13 @@ struct ath11k_per_ppdu_tx_stats { DECLARE_EWMA(avg_rssi, 10, 8) +struct ath11k_per_peer_cfr_capture { + enum ath11k_cfr_capture_method cfr_method; + enum ath11k_cfr_capture_bw cfr_bw; + u32 cfr_enable; + u32 cfr_period; +}; + struct ath11k_sta { struct ath11k_vif *arvif; @@ -571,6 +579,10 @@ struct ath11k_sta { bool peer_current_ps_valid; u32 bw_prev; + +#ifdef CONFIG_ATH11K_CFR + struct ath11k_per_peer_cfr_capture cfr_capture; +#endif }; #define ATH11K_MIN_5G_FREQ 4150 @@ -795,6 +807,11 @@ struct ath11k { bool ps_state_enable; bool ps_timekeeper_enable; s8 max_allowed_tx_power; + +#ifdef CONFIG_ATH11K_CFR + struct ath11k_cfr cfr; +#endif + bool cfr_enabled; }; struct ath11k_band_cap { @@ -1275,6 +1292,7 @@ bool ath11k_core_coldboot_cal_support(struct ath11k_base *ab); const struct firmware *ath11k_core_firmware_request(struct ath11k_base *ab, const char *filename); +const char *ath11k_core_get_usecase_firmware(struct ath11k_base *ab); static inline const char *ath11k_scan_state_str(enum ath11k_scan_state state) { @@ -1325,9 +1343,10 @@ static inline void ath11k_core_create_firmware_path(struct ath11k_base *ab, const char *filename, void *buf, size_t buf_len) { - const char *fw_name = NULL; + const char *fw_name = ath11k_core_get_usecase_firmware(ab); - of_property_read_string(ab->dev->of_node, "firmware-name", &fw_name); + if (!fw_name) + fw_name = ath11k_core_get_usecase_firmware(ab); if (fw_name && strncmp(filename, "board", 5)) snprintf(buf, buf_len, "%s/%s/%s/%s", ATH11K_FW_DIR, diff --git a/drivers/net/wireless/ath/ath11k/dbring.c b/drivers/net/wireless/ath/ath11k/dbring.c index 520d8b8662a28..d6994ce6ebff4 100644 --- a/drivers/net/wireless/ath/ath11k/dbring.c +++ b/drivers/net/wireless/ath/ath11k/dbring.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -37,10 +36,10 @@ static void ath11k_dbring_fill_magic_value(struct ath11k *ar, memset32(buffer, ATH11K_DB_MAGIC_VALUE, size); } -static int ath11k_dbring_bufs_replenish(struct ath11k *ar, - struct ath11k_dbring *ring, - struct ath11k_dbring_element *buff, - enum wmi_direct_buffer_module id) +int ath11k_dbring_bufs_replenish(struct ath11k *ar, + struct ath11k_dbring *ring, + struct ath11k_dbring_element *buff, + enum wmi_direct_buffer_module id) { struct ath11k_base *ab = ar->ab; struct hal_srng *srng; @@ -80,6 +79,9 @@ static int ath11k_dbring_bufs_replenish(struct ath11k *ar, goto err_idr_remove; } + if (id == WMI_DIRECT_BUF_CFR) + ath11k_cfr_lut_update_paddr(ar, paddr, buf_id); + buff->paddr = paddr; cookie = FIELD_PREP(DP_RXDMA_BUF_COOKIE_PDEV_ID, ar->pdev_idx) | @@ -155,12 +157,11 @@ int ath11k_dbring_wmi_cfg_setup(struct ath11k *ar, enum wmi_direct_buffer_module id) { struct ath11k_wmi_pdev_dma_ring_cfg_req_cmd param = {}; - int ret; + int ret, i; if (id >= WMI_DIRECT_BUF_MAX) return -EINVAL; - param.pdev_id = DP_SW2HW_MACID(ring->pdev_id); param.module_id = id; param.base_paddr_lo = lower_32_bits(ring->refill_srng.paddr); param.base_paddr_hi = upper_32_bits(ring->refill_srng.paddr); @@ -173,10 +174,23 @@ int ath11k_dbring_wmi_cfg_setup(struct ath11k *ar, param.num_resp_per_event = ring->num_resp_per_event; param.event_timeout_ms = ring->event_timeout_ms; - ret = ath11k_wmi_pdev_dma_ring_cfg(ar, ¶m); - if (ret) { - ath11k_warn(ar->ab, "failed to setup db ring cfg\n"); - return ret; + /* For single pdev, 2GHz and 5GHz use one DBR. */ + if (ar->ab->hw_params.single_pdev_only) { + for (i = 0; i < ar->ab->target_pdev_count; i++) { + param.pdev_id = ar->ab->target_pdev_ids[i].pdev_id; + ret = ath11k_wmi_pdev_dma_ring_cfg(ar, ¶m); + if (ret) { + ath11k_warn(ar->ab, "failed to setup db ring cfg\n"); + return ret; + } + } + } else { + param.pdev_id = DP_SW2HW_MACID(ring->pdev_id); + ret = ath11k_wmi_pdev_dma_ring_cfg(ar, ¶m); + if (ret) { + ath11k_warn(ar->ab, "failed to setup db ring cfg\n"); + return ret; + } } return 0; @@ -281,10 +295,15 @@ int ath11k_dbring_buffer_release_event(struct ath11k_base *ab, int size; dma_addr_t paddr; int ret = 0; + int status; pdev_idx = ev->fixed.pdev_id; module_id = ev->fixed.module_id; + if (ab->hw_params.single_pdev_only && + pdev_idx < ab->target_pdev_count) + pdev_idx = 0; + if (pdev_idx >= ab->num_radios) { ath11k_warn(ab, "Invalid pdev id %d\n", pdev_idx); return -EINVAL; @@ -310,6 +329,9 @@ int ath11k_dbring_buffer_release_event(struct ath11k_base *ab, case WMI_DIRECT_BUF_SPECTRAL: ring = ath11k_spectral_get_dbring(ar); break; + case WMI_DIRECT_BUF_CFR: + ring = ath11k_cfr_get_dbring(ar); + break; default: ring = NULL; ath11k_warn(ab, "Recv dma buffer release ev on unsupp module %d\n", @@ -360,8 +382,12 @@ int ath11k_dbring_buffer_release_event(struct ath11k_base *ab, handler_data.data = PTR_ALIGN(vaddr_unalign, ring->buf_align); handler_data.data_sz = ring->buf_sz; + handler_data.buff = buff; + handler_data.buf_id = buf_id; - ring->handler(ar, &handler_data); + status = ring->handler(ar, &handler_data); + if (status == ATH11K_CORRELATE_STATUS_HOLD) + continue; } buff->paddr = 0; diff --git a/drivers/net/wireless/ath/ath11k/dbring.h b/drivers/net/wireless/ath/ath11k/dbring.h index 2f93b78a50df0..e5f244dfa9631 100644 --- a/drivers/net/wireless/ath/ath11k/dbring.h +++ b/drivers/net/wireless/ath/ath11k/dbring.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH11K_DBRING_H @@ -21,6 +21,8 @@ struct ath11k_dbring_data { void *data; u32 data_sz; struct wmi_dma_buf_release_meta_data meta; + struct ath11k_dbring_element *buff; + u32 buf_id; }; struct ath11k_dbring_buf_release_event { @@ -61,6 +63,10 @@ int ath11k_dbring_set_cfg(struct ath11k *ar, u32 event_timeout_ms, int (*handler)(struct ath11k *, struct ath11k_dbring_data *)); +int ath11k_dbring_bufs_replenish(struct ath11k *ar, + struct ath11k_dbring *ring, + struct ath11k_dbring_element *buff, + enum wmi_direct_buffer_module id); int ath11k_dbring_wmi_cfg_setup(struct ath11k *ar, struct ath11k_dbring *ring, enum wmi_direct_buffer_module id); diff --git a/drivers/net/wireless/ath/ath11k/debug.h b/drivers/net/wireless/ath/ath11k/debug.h index cc8934d156977..aaa0034527a59 100644 --- a/drivers/net/wireless/ath/ath11k/debug.h +++ b/drivers/net/wireless/ath/ath11k/debug.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef _ATH11K_DEBUG_H_ @@ -27,6 +27,8 @@ enum ath11k_debug_mask { ATH11K_DBG_DP_TX = 0x00002000, ATH11K_DBG_DP_RX = 0x00004000, ATH11K_DBG_CE = 0x00008000, + ATH11K_DBG_CFR = 0x00010000, + ATH11K_DBG_CFR_DUMP = 0x00020000, }; static inline const char *ath11k_dbg_str(enum ath11k_debug_mask mask) @@ -64,6 +66,10 @@ static inline const char *ath11k_dbg_str(enum ath11k_debug_mask mask) return "dp_rx"; case ATH11K_DBG_CE: return "ce"; + case ATH11K_DBG_CFR: + return "cfr"; + case ATH11K_DBG_CFR_DUMP: + return "cfr_dump"; /* no default handler to allow compiler to check that the * enum is fully handled diff --git a/drivers/net/wireless/ath/ath11k/debugfs_sta.c b/drivers/net/wireless/ath/ath11k/debugfs_sta.c index d89d0f28d890b..621a8a8df4b86 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_sta.c +++ b/drivers/net/wireless/ath/ath11k/debugfs_sta.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -240,6 +239,140 @@ static const struct file_operations fops_tx_stats = { .llseek = default_llseek, }; +#ifdef CONFIG_ATH11K_CFR +static ssize_t ath11k_dbg_sta_write_cfr_capture(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ieee80211_sta *sta = file->private_data; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); + struct ath11k *ar = arsta->arvif->ar; + struct ath11k_cfr *cfr = &ar->cfr; + struct wmi_peer_cfr_capture_conf_arg arg; + u32 cfr_capture_enable = 0, cfr_capture_bw = 0; + u32 cfr_capture_method = 0, cfr_capture_period = 0; + char buf[64] = {}; + int ret; + + simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count); + + guard(mutex)(&ar->conf_mutex); + + if (ar->state != ATH11K_STATE_ON) + return -ENETDOWN; + + if (!ar->cfr_enabled) + return -EINVAL; + + ret = sscanf(buf, "%u %u %u %u", &cfr_capture_enable, &cfr_capture_bw, + &cfr_capture_period, &cfr_capture_method); + + if (ret < 1 || (cfr_capture_enable && ret != 4)) + return -EINVAL; + + if (cfr_capture_enable == arsta->cfr_capture.cfr_enable && + (cfr_capture_period && + cfr_capture_period == arsta->cfr_capture.cfr_period) && + cfr_capture_bw == arsta->cfr_capture.cfr_bw && + cfr_capture_method == arsta->cfr_capture.cfr_method) + return count; + + if (!cfr_capture_enable && + cfr_capture_enable == arsta->cfr_capture.cfr_enable) + return count; + + if (cfr_capture_enable > WMI_PEER_CFR_CAPTURE_ENABLE || + cfr_capture_bw > WMI_PEER_CFR_CAPTURE_BW_80 || + cfr_capture_method > ATH11K_CFR_CAPTURE_METHOD_NULL_FRAME_WITH_PHASE || + cfr_capture_period > WMI_PEER_CFR_PERIODICITY_MAX) + return -EINVAL; + + /* Target expects cfr period in multiple of 10 */ + if (cfr_capture_period % 10) { + ath11k_err(ar->ab, "periodicity should be 10x\n"); + return -EINVAL; + } + + if (ar->cfr.cfr_enabled_peer_cnt >= ATH11K_MAX_CFR_ENABLED_CLIENTS && + !arsta->cfr_capture.cfr_enable) { + ath11k_err(ar->ab, "CFR enable peer threshold reached %u\n", + ar->cfr.cfr_enabled_peer_cnt); + return -EINVAL; + } + + if (!cfr_capture_enable) { + cfr_capture_bw = arsta->cfr_capture.cfr_bw; + cfr_capture_period = arsta->cfr_capture.cfr_period; + cfr_capture_method = arsta->cfr_capture.cfr_method; + } + + arg.request = cfr_capture_enable; + arg.periodicity = cfr_capture_period; + arg.bw = cfr_capture_bw; + arg.method = cfr_capture_method; + + ret = ath11k_wmi_peer_set_cfr_capture_conf(ar, arsta->arvif->vdev_id, + sta->addr, &arg); + if (ret) { + ath11k_warn(ar->ab, + "failed to send cfr capture info: vdev_id %u peer %pM: %d\n", + arsta->arvif->vdev_id, sta->addr, ret); + return ret; + } + + spin_lock_bh(&ar->cfr.lock); + + if (cfr_capture_enable && + cfr_capture_enable != arsta->cfr_capture.cfr_enable) + cfr->cfr_enabled_peer_cnt++; + else if (!cfr_capture_enable) + cfr->cfr_enabled_peer_cnt--; + + spin_unlock_bh(&ar->cfr.lock); + + arsta->cfr_capture.cfr_enable = cfr_capture_enable; + arsta->cfr_capture.cfr_period = cfr_capture_period; + arsta->cfr_capture.cfr_bw = cfr_capture_bw; + arsta->cfr_capture.cfr_method = cfr_capture_method; + + return count; +} + +static ssize_t ath11k_dbg_sta_read_cfr_capture(struct file *file, + char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ieee80211_sta *sta = file->private_data; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); + struct ath11k *ar = arsta->arvif->ar; + char buf[512] = {}; + int len = 0; + + mutex_lock(&ar->conf_mutex); + + len += scnprintf(buf + len, sizeof(buf) - len, "cfr_enabled = %d\n", + arsta->cfr_capture.cfr_enable); + len += scnprintf(buf + len, sizeof(buf) - len, "bandwidth = %d\n", + arsta->cfr_capture.cfr_bw); + len += scnprintf(buf + len, sizeof(buf) - len, "period = %d\n", + arsta->cfr_capture.cfr_period); + len += scnprintf(buf + len, sizeof(buf) - len, "cfr_method = %d\n", + arsta->cfr_capture.cfr_method); + + mutex_unlock(&ar->conf_mutex); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static const struct file_operations fops_peer_cfr_capture = { + .write = ath11k_dbg_sta_write_cfr_capture, + .read = ath11k_dbg_sta_read_cfr_capture, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; +#endif /* CONFIG_ATH11K_CFR */ + static ssize_t ath11k_dbg_sta_dump_rx_stats(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) @@ -877,6 +1010,13 @@ void ath11k_debugfs_sta_op_add(struct ieee80211_hw *hw, struct ieee80211_vif *vi debugfs_create_file("htt_peer_stats_reset", 0600, dir, sta, &fops_htt_peer_stats_reset); +#ifdef CONFIG_ATH11K_CFR + if (test_bit(WMI_TLV_SERVICE_CFR_CAPTURE_SUPPORT, + ar->ab->wmi_ab.svc_map)) + debugfs_create_file("cfr_capture", 0600, dir, sta, + &fops_peer_cfr_capture); +#endif/* CONFIG_ATH11K_CFR */ + debugfs_create_file("peer_ps_state", 0400, dir, sta, &fops_peer_ps_state); diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index b9e976ddcbbf6..8525c03180335 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -1110,9 +1110,8 @@ int ath11k_dp_rx_ampdu_stop(struct ath11k *ar, struct ath11k_base *ab = ar->ab; struct ath11k_peer *peer; struct ath11k_sta *arsta = ath11k_sta_to_arsta(params->sta); + struct dp_rx_tid *rx_tid; int vdev_id = arsta->arvif->vdev_id; - dma_addr_t paddr; - bool active; int ret; spin_lock_bh(&ab->base_lock); @@ -1124,15 +1123,14 @@ int ath11k_dp_rx_ampdu_stop(struct ath11k *ar, return -ENOENT; } - paddr = peer->rx_tid[params->tid].paddr; - active = peer->rx_tid[params->tid].active; + rx_tid = &peer->rx_tid[params->tid]; - if (!active) { + if (!rx_tid->active) { spin_unlock_bh(&ab->base_lock); return 0; } - ret = ath11k_peer_rx_tid_reo_update(ar, peer, peer->rx_tid, 1, 0, false); + ret = ath11k_peer_rx_tid_reo_update(ar, peer, rx_tid, 1, 0, false); spin_unlock_bh(&ab->base_lock); if (ret) { ath11k_warn(ab, "failed to update reo for rx tid %d: %d\n", @@ -1141,7 +1139,8 @@ int ath11k_dp_rx_ampdu_stop(struct ath11k *ar, } ret = ath11k_wmi_peer_rx_reorder_queue_setup(ar, vdev_id, - params->sta->addr, paddr, + params->sta->addr, + rx_tid->paddr, params->tid, 1, 1); if (ret) ath11k_warn(ab, "failed to send wmi to delete rx tid %d\n", @@ -2503,6 +2502,29 @@ static void ath11k_dp_rx_deliver_msdu(struct ath11k *ar, struct napi_struct *nap ieee80211_rx_napi(ar->hw, pubsta, msdu, napi); } +static bool ath11k_dp_rx_check_nwifi_hdr_len_valid(struct ath11k_base *ab, + struct hal_rx_desc *rx_desc, + struct sk_buff *msdu) +{ + struct ieee80211_hdr *hdr; + u8 decap_type; + u32 hdr_len; + + decap_type = ath11k_dp_rx_h_msdu_start_decap_type(ab, rx_desc); + if (decap_type != DP_RX_DECAP_TYPE_NATIVE_WIFI) + return true; + + hdr = (struct ieee80211_hdr *)msdu->data; + hdr_len = ieee80211_hdrlen(hdr->frame_control); + + if ((likely(hdr_len <= DP_MAX_NWIFI_HDR_LEN))) + return true; + + ab->soc_stats.invalid_rbm++; + WARN_ON_ONCE(1); + return false; +} + static int ath11k_dp_rx_process_msdu(struct ath11k *ar, struct sk_buff *msdu, struct sk_buff_head *msdu_list, @@ -2573,6 +2595,11 @@ static int ath11k_dp_rx_process_msdu(struct ath11k *ar, } } + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ab, rx_desc, msdu))) { + ret = -EINVAL; + goto free_out; + } + ath11k_dp_rx_h_ppdu(ar, rx_desc, rx_status); ath11k_dp_rx_h_mpdu(ar, msdu, rx_desc, rx_status); @@ -3308,6 +3335,12 @@ static int ath11k_dp_rx_h_verify_tkip_mic(struct ath11k *ar, struct ath11k_peer RX_FLAG_IV_STRIPPED | RX_FLAG_DECRYPTED; skb_pull(msdu, hal_rx_desc_sz); + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ar->ab, rx_desc, + msdu))) { + dev_kfree_skb_any(msdu); + return -EINVAL; + } + ath11k_dp_rx_h_ppdu(ar, rx_desc, rxs); ath11k_dp_rx_h_undecap(ar, msdu, rx_desc, HAL_ENCRYPT_TYPE_TKIP_MIC, rxs, true); @@ -4000,6 +4033,10 @@ static int ath11k_dp_rx_h_null_q_desc(struct ath11k *ar, struct sk_buff *msdu, skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); } + + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ar->ab, desc, msdu))) + return -EINVAL; + ath11k_dp_rx_h_ppdu(ar, desc, status); ath11k_dp_rx_h_mpdu(ar, msdu, desc, status); @@ -4044,7 +4081,7 @@ static bool ath11k_dp_rx_h_reo_err(struct ath11k *ar, struct sk_buff *msdu, return drop; } -static void ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, +static bool ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, struct ieee80211_rx_status *status) { u16 msdu_len; @@ -4052,15 +4089,28 @@ static void ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, u8 l3pad_bytes; struct ath11k_skb_rxcb *rxcb = ATH11K_SKB_RXCB(msdu); u32 hal_rx_desc_sz = ar->ab->hw_params.hal_desc_sz; + struct ath11k_base *ab = ar->ab; rxcb->is_first_msdu = ath11k_dp_rx_h_msdu_end_first_msdu(ar->ab, desc); rxcb->is_last_msdu = ath11k_dp_rx_h_msdu_end_last_msdu(ar->ab, desc); l3pad_bytes = ath11k_dp_rx_h_msdu_end_l3pad(ar->ab, desc); msdu_len = ath11k_dp_rx_h_msdu_start_msdu_len(ar->ab, desc); + + if (unlikely(hal_rx_desc_sz + l3pad_bytes + msdu_len > DP_RX_BUFFER_SIZE)) { + ath11k_dbg(ab, ATH11K_DBG_DATA, + "invalid msdu len in tkip mic err %u\n", msdu_len); + ath11k_dbg_dump(ab, ATH11K_DBG_DATA, NULL, "", desc, + sizeof(*desc)); + return true; + } + skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ab, desc, msdu))) + return true; + ath11k_dp_rx_h_ppdu(ar, desc, status); status->flag |= (RX_FLAG_MMIC_STRIPPED | RX_FLAG_MMIC_ERROR | @@ -4068,19 +4118,21 @@ static void ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, ath11k_dp_rx_h_undecap(ar, msdu, desc, HAL_ENCRYPT_TYPE_TKIP_MIC, status, false); + + return false; } static bool ath11k_dp_rx_h_rxdma_err(struct ath11k *ar, struct sk_buff *msdu, struct ieee80211_rx_status *status) { struct ath11k_skb_rxcb *rxcb = ATH11K_SKB_RXCB(msdu); - bool drop = false; + bool drop; ar->ab->soc_stats.rxdma_error[rxcb->err_code]++; switch (rxcb->err_code) { case HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR: - ath11k_dp_rx_h_tkip_mic_err(ar, msdu, status); + drop = ath11k_dp_rx_h_tkip_mic_err(ar, msdu, status); break; default: /* TODO: Review other rxdma error code to check if anything is diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c index 0c797b8d0a276..e821e5a62c1c0 100644 --- a/drivers/net/wireless/ath/ath11k/hal.c +++ b/drivers/net/wireless/ath/ath11k/hal.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -184,7 +183,7 @@ static const struct hal_srng_config hw_srng_config_template[] = { }, { /* RXDMA DIR BUF */ .start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF, - .max_rings = 1, + .max_rings = 2, .entry_size = 8 >> 2, /* TODO: Define the struct */ .lmac_ring = true, .ring_dir = HAL_SRNG_DIR_SRC, diff --git a/drivers/net/wireless/ath/ath11k/hw.h b/drivers/net/wireless/ath/ath11k/hw.h index 52d9f4c13b136..e13ca02a9d057 100644 --- a/drivers/net/wireless/ath/ath11k/hw.h +++ b/drivers/net/wireless/ath/ath11k/hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH11K_HW_H @@ -228,6 +228,9 @@ struct ath11k_hw_params { bool support_fw_mac_sequence; bool support_dual_stations; bool pdev_suspend; + bool cfr_support; + u32 cfr_num_stream_bufs; + u32 cfr_stream_buf_size; }; struct ath11k_hw_ops { diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index 0e41b5a91d66d..861d83b976e08 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -2235,9 +2235,9 @@ static void ath11k_peer_assoc_h_vht(struct ath11k *ar, arg->peer_nss = min(sta->deflink.rx_nss, max_nss); arg->rx_max_rate = __le16_to_cpu(vht_cap->vht_mcs.rx_highest); arg->rx_mcs_set = __le16_to_cpu(vht_cap->vht_mcs.rx_mcs_map); + arg->rx_mcs_set = ath11k_peer_assoc_h_vht_limit(arg->rx_mcs_set, vht_mcs_mask); arg->tx_max_rate = __le16_to_cpu(vht_cap->vht_mcs.tx_highest); - arg->tx_mcs_set = ath11k_peer_assoc_h_vht_limit( - __le16_to_cpu(vht_cap->vht_mcs.tx_mcs_map), vht_mcs_mask); + arg->tx_mcs_set = __le16_to_cpu(vht_cap->vht_mcs.tx_mcs_map); /* In IPQ8074 platform, VHT mcs rate 10 and 11 is enabled by default. * VHT mcs rate 10 and 11 is not supported in 11ac standard. @@ -2522,10 +2522,10 @@ static void ath11k_peer_assoc_h_he(struct ath11k *ar, he_tx_mcs = v; } v = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160); + v = ath11k_peer_assoc_h_he_limit(v, he_mcs_mask); arg->peer_he_rx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_160] = v; v = le16_to_cpu(he_cap->he_mcs_nss_supp.tx_mcs_160); - v = ath11k_peer_assoc_h_he_limit(v, he_mcs_mask); arg->peer_he_tx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_160] = v; arg->peer_he_mcs_count++; @@ -2535,10 +2535,10 @@ static void ath11k_peer_assoc_h_he(struct ath11k *ar, default: v = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80); + v = ath11k_peer_assoc_h_he_limit(v, he_mcs_mask); arg->peer_he_rx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_80] = v; v = le16_to_cpu(he_cap->he_mcs_nss_supp.tx_mcs_80); - v = ath11k_peer_assoc_h_he_limit(v, he_mcs_mask); arg->peer_he_tx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_80] = v; arg->peer_he_mcs_count++; @@ -2911,6 +2911,8 @@ static void ath11k_peer_assoc_h_phymode(struct ath11k *ar, arg->peer_phymode = phymode; WARN_ON(phymode == MODE_UNKNOWN); + + ath11k_cfr_update_phymode(ar, phymode); } static void ath11k_peer_assoc_prepare(struct ath11k *ar, @@ -6042,6 +6044,8 @@ static int ath11k_mac_mgmt_tx_wmi(struct ath11k *ar, struct ath11k_vif *arvif, dma_addr_t paddr; int buf_id; int ret; + bool tx_params_valid = false; + bool peer_in_unassoc_pool; ATH11K_SKB_CB(skb)->ar = ar; @@ -6080,7 +6084,18 @@ static int ath11k_mac_mgmt_tx_wmi(struct ath11k *ar, struct ath11k_vif *arvif, ATH11K_SKB_CB(skb)->paddr = paddr; - ret = ath11k_wmi_mgmt_send(ar, arvif->vdev_id, buf_id, skb); + peer_in_unassoc_pool = ath11k_cfr_peer_is_in_cfr_unassoc_pool(ar, hdr->addr1); + + if (ar->cfr_enabled && + ieee80211_is_probe_resp(hdr->frame_control) && + peer_in_unassoc_pool) + tx_params_valid = true; + + if (peer_in_unassoc_pool) + ath11k_cfr_update_unassoc_pool_entry(ar, hdr->addr1); + + ret = ath11k_wmi_mgmt_send(ar, arvif->vdev_id, buf_id, skb, + tx_params_valid); if (ret) { ath11k_warn(ar->ab, "failed to send mgmt frame: %d\n", ret); goto err_unmap_buf; @@ -9804,6 +9819,8 @@ static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw, } spin_unlock_bh(&ar->ab->base_lock); mutex_unlock(&ar->ab->tbl_mtx_lock); + + ath11k_cfr_decrement_peer_count(ar, arsta); } else if (old_state == IEEE80211_STA_AUTH && new_state == IEEE80211_STA_ASSOC && (vif->type == NL80211_IFTYPE_AP || diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c index acd76e9392d31..d2c44f7f9b622 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.c +++ b/drivers/net/wireless/ath/ath11k/mhi.c @@ -34,7 +34,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qca6390[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -48,7 +47,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qca6390[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; @@ -99,7 +97,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qcn9074[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -113,7 +110,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qcn9074[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c index d8655badd96d0..7114eca8810db 100644 --- a/drivers/net/wireless/ath/ath11k/pci.c +++ b/drivers/net/wireless/ath/ath11k/pci.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -177,6 +177,19 @@ static inline void ath11k_pci_select_static_window(struct ath11k_pci *ab_pci) ab_pci->ab->mem + ATH11K_PCI_WINDOW_REG_ADDRESS); } +static void ath11k_pci_restore_window(struct ath11k_base *ab) +{ + struct ath11k_pci *ab_pci = ath11k_pci_priv(ab); + + spin_lock_bh(&ab_pci->window_lock); + + iowrite32(ATH11K_PCI_WINDOW_ENABLE_BIT | ab_pci->register_window, + ab->mem + ATH11K_PCI_WINDOW_REG_ADDRESS); + ioread32(ab->mem + ATH11K_PCI_WINDOW_REG_ADDRESS); + + spin_unlock_bh(&ab_pci->window_lock); +} + static void ath11k_pci_soc_global_reset(struct ath11k_base *ab) { u32 val, delay; @@ -201,6 +214,11 @@ static void ath11k_pci_soc_global_reset(struct ath11k_base *ab) val = ath11k_pcic_read32(ab, PCIE_SOC_GLOBAL_RESET); if (val == 0xffffffff) ath11k_warn(ab, "link down error during global reset\n"); + + /* Restore window register as its content is cleared during + * hardware global reset, such that it aligns with host cache. + */ + ath11k_pci_restore_window(ab); } static void ath11k_pci_clear_dbg_registers(struct ath11k_base *ab) diff --git a/drivers/net/wireless/ath/ath11k/reg.c b/drivers/net/wireless/ath/ath11k/reg.c index d62a2014315a0..49b79648752cf 100644 --- a/drivers/net/wireless/ath/ath11k/reg.c +++ b/drivers/net/wireless/ath/ath11k/reg.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -926,8 +926,11 @@ int ath11k_reg_handle_chan_list(struct ath11k_base *ab, */ if (ab->default_regd[pdev_idx] && !ab->new_regd[pdev_idx] && !memcmp((char *)ab->default_regd[pdev_idx]->alpha2, - (char *)reg_info->alpha2, 2)) - goto retfail; + (char *)reg_info->alpha2, 2) && + power_type == IEEE80211_REG_UNSET_AP) { + ath11k_reg_reset_info(reg_info); + return 0; + } /* Intersect new rules with default regd if a new country setting was * requested, i.e a default regd was already set during initialization diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index e3b444333deed..451cc4c719aec 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include @@ -651,11 +651,12 @@ static u32 ath11k_wmi_mgmt_get_freq(struct ath11k *ar, } int ath11k_wmi_mgmt_send(struct ath11k *ar, u32 vdev_id, u32 buf_id, - struct sk_buff *frame) + struct sk_buff *frame, bool tx_params_valid) { struct ath11k_pdev_wmi *wmi = ar->wmi; struct ieee80211_tx_info *info = IEEE80211_SKB_CB(frame); struct wmi_mgmt_send_cmd *cmd; + struct wmi_mgmt_send_params *params; struct wmi_tlv *frame_tlv; struct sk_buff *skb; u32 buf_len; @@ -665,6 +666,8 @@ int ath11k_wmi_mgmt_send(struct ath11k *ar, u32 vdev_id, u32 buf_id, frame->len : WMI_MGMT_SEND_DOWNLD_LEN; len = sizeof(*cmd) + sizeof(*frame_tlv) + roundup(buf_len, 4); + if (tx_params_valid) + len += sizeof(*params); skb = ath11k_wmi_alloc_skb(wmi->wmi_ab, len); if (!skb) @@ -680,7 +683,7 @@ int ath11k_wmi_mgmt_send(struct ath11k *ar, u32 vdev_id, u32 buf_id, cmd->paddr_hi = upper_32_bits(ATH11K_SKB_CB(frame)->paddr); cmd->frame_len = frame->len; cmd->buf_len = buf_len; - cmd->tx_params_valid = 0; + cmd->tx_params_valid = !!tx_params_valid; frame_tlv = (struct wmi_tlv *)(skb->data + sizeof(*cmd)); frame_tlv->header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_ARRAY_BYTE) | @@ -690,6 +693,15 @@ int ath11k_wmi_mgmt_send(struct ath11k *ar, u32 vdev_id, u32 buf_id, ath11k_ce_byte_swap(frame_tlv->value, buf_len); + if (tx_params_valid) { + params = + (struct wmi_mgmt_send_params *)(skb->data + (len - sizeof(*params))); + params->tlv_header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_TX_SEND_PARAMS) | + FIELD_PREP(WMI_TLV_LEN, + sizeof(*params) - TLV_HDR_SIZE); + params->tx_params_dword1 |= WMI_TX_PARAMS_DWORD1_CFR_CAPTURE; + } + ret = ath11k_wmi_cmd_send(wmi, skb, WMI_MGMT_TX_SEND_CMDID); if (ret) { ath11k_warn(ar->ab, @@ -2061,10 +2073,13 @@ int ath11k_wmi_send_peer_assoc_cmd(struct ath11k *ar, cmd->peer_bw_rxnss_override |= param->peer_bw_rxnss_override; if (param->vht_capable) { - mcs->rx_max_rate = param->rx_max_rate; - mcs->rx_mcs_set = param->rx_mcs_set; - mcs->tx_max_rate = param->tx_max_rate; - mcs->tx_mcs_set = param->tx_mcs_set; + /* firmware interprets mcs->tx_mcs_set field as peer's + * RX capability + */ + mcs->tx_max_rate = param->rx_max_rate; + mcs->tx_mcs_set = param->rx_mcs_set; + mcs->rx_max_rate = param->tx_max_rate; + mcs->rx_mcs_set = param->tx_mcs_set; } /* HE Rates */ @@ -2088,8 +2103,11 @@ int ath11k_wmi_send_peer_assoc_cmd(struct ath11k *ar, FIELD_PREP(WMI_TLV_LEN, sizeof(*he_mcs) - TLV_HDR_SIZE); - he_mcs->rx_mcs_set = param->peer_he_tx_mcs_set[i]; - he_mcs->tx_mcs_set = param->peer_he_rx_mcs_set[i]; + /* firmware interprets mcs->rx_mcs_set field as peer's + * RX capability + */ + he_mcs->rx_mcs_set = param->peer_he_rx_mcs_set[i]; + he_mcs->tx_mcs_set = param->peer_he_tx_mcs_set[i]; ptr += sizeof(*he_mcs); } @@ -3935,6 +3953,47 @@ int ath11k_wmi_fils_discovery_tmpl(struct ath11k *ar, u32 vdev_id, return 0; } +int ath11k_wmi_peer_set_cfr_capture_conf(struct ath11k *ar, + u32 vdev_id, const u8 *mac_addr, + struct wmi_peer_cfr_capture_conf_arg *arg) +{ + struct ath11k_pdev_wmi *wmi = ar->wmi; + struct wmi_peer_cfr_capture_cmd_fixed_param *cmd; + struct sk_buff *skb; + int ret; + + skb = ath11k_wmi_alloc_skb(wmi->wmi_ab, sizeof(*cmd)); + if (!skb) + return -ENOMEM; + + cmd = (struct wmi_peer_cfr_capture_cmd_fixed_param *)skb->data; + cmd->tlv_header = FIELD_PREP(WMI_TLV_TAG, + WMI_TAG_PEER_CFR_CAPTURE_CMD) | + FIELD_PREP(WMI_TLV_LEN, sizeof(*cmd) - TLV_HDR_SIZE); + + memcpy(&cmd->mac_addr, mac_addr, ETH_ALEN); + cmd->request = arg->request; + cmd->vdev_id = vdev_id; + cmd->periodicity = arg->periodicity; + cmd->bandwidth = arg->bw; + cmd->capture_method = arg->method; + + ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_PEER_CFR_CAPTURE_CMDID); + if (ret) { + ath11k_warn(ar->ab, + "WMI vdev %d failed to send peer cfr capture cmd: %d\n", + vdev_id, ret); + dev_kfree_skb(skb); + } + + ath11k_dbg(ar->ab, ATH11K_DBG_WMI, + "WMI peer CFR capture cmd req %u id %u period %u bw %u mode %u\n", + arg->request, vdev_id, arg->periodicity, + arg->bw, arg->method); + + return ret; +} + int ath11k_wmi_probe_resp_tmpl(struct ath11k *ar, u32 vdev_id, struct sk_buff *tmpl) { @@ -8746,6 +8805,93 @@ static void ath11k_wmi_p2p_noa_event(struct ath11k_base *ab, kfree(tb); } +static void ath11k_wmi_tlv_cfr_capture_event_fixed_param(const void *ptr, + void *data) +{ + struct ath11k_cfr_peer_tx_param *tx_params = data; + const struct ath11k_wmi_cfr_peer_tx_event_param *params = ptr; + + tx_params->capture_method = params->capture_method; + tx_params->vdev_id = params->vdev_id; + ether_addr_copy(tx_params->peer_mac_addr, params->mac_addr.addr); + tx_params->primary_20mhz_chan = params->chan_mhz; + tx_params->bandwidth = params->bandwidth; + tx_params->phy_mode = params->phy_mode; + tx_params->band_center_freq1 = params->band_center_freq1; + tx_params->band_center_freq2 = params->band_center_freq2; + tx_params->spatial_streams = params->sts_count; + tx_params->correlation_info_1 = params->correlation_info_1; + tx_params->correlation_info_2 = params->correlation_info_2; + tx_params->status = params->status; + tx_params->timestamp_us = params->timestamp_us; + tx_params->counter = params->counter; + tx_params->rx_start_ts = params->rx_start_ts; + + memcpy(tx_params->chain_rssi, params->chain_rssi, + sizeof(tx_params->chain_rssi)); + + if (WMI_CFR_CFO_MEASUREMENT_VALID & params->cfo_measurement) + tx_params->cfo_measurement = FIELD_GET(WMI_CFR_CFO_MEASUREMENT_RAW_DATA, + params->cfo_measurement); +} + +static void ath11k_wmi_tlv_cfr_capture_phase_fixed_param(const void *ptr, + void *data) +{ + struct ath11k_cfr_peer_tx_param *tx_params = data; + const struct ath11k_wmi_cfr_peer_tx_event_phase_param *params = ptr; + int i; + + for (i = 0; i < WMI_MAX_CHAINS; i++) { + tx_params->chain_phase[i] = params->chain_phase[i]; + tx_params->agc_gain[i] = params->agc_gain[i]; + } +} + +static int ath11k_wmi_tlv_cfr_capture_evt_parse(struct ath11k_base *ab, + u16 tag, u16 len, + const void *ptr, void *data) +{ + switch (tag) { + case WMI_TAG_PEER_CFR_CAPTURE_EVENT: + ath11k_wmi_tlv_cfr_capture_event_fixed_param(ptr, data); + break; + case WMI_TAG_CFR_CAPTURE_PHASE_PARAM: + ath11k_wmi_tlv_cfr_capture_phase_fixed_param(ptr, data); + break; + default: + ath11k_warn(ab, "Invalid tag received tag %d len %d\n", + tag, len); + return -EINVAL; + } + + return 0; +} + +static void ath11k_wmi_parse_cfr_capture_event(struct ath11k_base *ab, + struct sk_buff *skb) +{ + struct ath11k_cfr_peer_tx_param params = {}; + int ret; + + ath11k_dbg_dump(ab, ATH11K_DBG_CFR_DUMP, "cfr_dump:", "", + skb->data, skb->len); + + ret = ath11k_wmi_tlv_iter(ab, skb->data, skb->len, + ath11k_wmi_tlv_cfr_capture_evt_parse, + ¶ms); + if (ret) { + ath11k_warn(ab, "failed to parse cfr capture event tlv %d\n", + ret); + return; + } + + ret = ath11k_process_cfr_capture_event(ab, ¶ms); + if (ret) + ath11k_dbg(ab, ATH11K_DBG_CFR, + "failed to process cfr capture ret = %d\n", ret); +} + static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; @@ -8876,6 +9022,9 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) case WMI_P2P_NOA_EVENTID: ath11k_wmi_p2p_noa_event(ab, skb); break; + case WMI_PEER_CFR_CAPTURE_EVENTID: + ath11k_wmi_parse_cfr_capture_event(ab, skb); + break; default: ath11k_dbg(ab, ATH11K_DBG_WMI, "unsupported event id 0x%x\n", id); break; diff --git a/drivers/net/wireless/ath/ath11k/wmi.h b/drivers/net/wireless/ath/ath11k/wmi.h index 9fcffaa2f383c..24b8f867c60dc 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.h +++ b/drivers/net/wireless/ath/ath11k/wmi.h @@ -362,6 +362,10 @@ enum wmi_tlv_cmd_id { WMI_PEER_REORDER_QUEUE_REMOVE_CMDID, WMI_PEER_SET_RX_BLOCKSIZE_CMDID, WMI_PEER_ANTDIV_INFO_REQ_CMDID, + WMI_PEER_RESERVED0_CMDID, + WMI_PEER_TID_MSDUQ_QDEPTH_THRESH_UPDATE_CMDID, + WMI_PEER_TID_CONFIGURATIONS_CMDID, + WMI_PEER_CFR_CAPTURE_CMDID, WMI_BCN_TX_CMDID = WMI_TLV_CMD(WMI_GRP_MGMT), WMI_PDEV_SEND_BCN_CMDID, WMI_BCN_TMPL_CMDID, @@ -981,6 +985,7 @@ enum wmi_tlv_pdev_param { WMI_PDEV_PARAM_RADIO_CHAN_STATS_ENABLE, WMI_PDEV_PARAM_RADIO_DIAGNOSIS_ENABLE, WMI_PDEV_PARAM_MESH_MCAST_ENABLE, + WMI_PDEV_PARAM_PER_PEER_CFR_ENABLE = 0xa8, WMI_PDEV_PARAM_SET_CMD_OBSS_PD_THRESHOLD = 0xbc, WMI_PDEV_PARAM_SET_CMD_OBSS_PD_PER_AC = 0xbe, WMI_PDEV_PARAM_ENABLE_SR_PROHIBIT = 0xc6, @@ -1884,6 +1889,8 @@ enum wmi_tlv_tag { WMI_TAG_NDP_EVENT, WMI_TAG_PDEV_PEER_PKTLOG_FILTER_CMD = 0x301, WMI_TAG_PDEV_PEER_PKTLOG_FILTER_INFO, + WMI_TAG_PEER_CFR_CAPTURE_EVENT = 0x317, + WMI_TAG_CFR_CAPTURE_PHASE_PARAM = 0x33b, WMI_TAG_FILS_DISCOVERY_TMPL_CMD = 0x344, WMI_TAG_PDEV_SRG_BSS_COLOR_BITMAP_CMD = 0x37b, WMI_TAG_PDEV_SRG_PARTIAL_BSSID_BITMAP_CMD, @@ -3846,7 +3853,8 @@ struct wmi_scan_prob_req_oui_cmd { #define WMI_TX_PARAMS_DWORD1_BW_MASK GENMASK(14, 8) #define WMI_TX_PARAMS_DWORD1_PREAMBLE_TYPE GENMASK(19, 15) #define WMI_TX_PARAMS_DWORD1_FRAME_TYPE BIT(20) -#define WMI_TX_PARAMS_DWORD1_RSVD GENMASK(31, 21) +#define WMI_TX_PARAMS_DWORD1_CFR_CAPTURE BIT(21) +#define WMI_TX_PARAMS_DWORD1_RSVD GENMASK(31, 22) struct wmi_mgmt_send_params { u32 tlv_header; @@ -4133,8 +4141,10 @@ struct wmi_rate_set { struct wmi_vht_rate_set { u32 tlv_header; u32 rx_max_rate; + /* MCS at which the peer can transmit */ u32 rx_mcs_set; u32 tx_max_rate; + /* MCS at which the peer can receive */ u32 tx_mcs_set; u32 tx_max_mcs_nss; } __packed; @@ -4229,6 +4239,87 @@ enum cc_setting_code { */ }; +enum ath11k_wmi_cfr_capture_bw { + WMI_PEER_CFR_CAPTURE_BW_20, + WMI_PEER_CFR_CAPTURE_BW_40, + WMI_PEER_CFR_CAPTURE_BW_80, + WMI_PEER_CFR_CAPTURE_BW_MAX, +}; + +enum ath11k_wmi_cfr_capture_method { + WMI_CFR_CAPTURE_METHOD_NULL_FRAME, + WMI_CFR_CAPTURE_METHOD_NULL_FRAME_WITH_PHASE, + WMI_CFR_CAPTURE_METHOD_PROBE_RESP, + WMI_CFR_CAPTURE_METHOD_MAX, +}; + +#define WMI_CFR_FRAME_TX_STATUS GENMASK(1, 0) +#define WMI_CFR_CAPTURE_STATUS_PEER_PS BIT(30) +#define WMI_CFR_PEER_CAPTURE_STATUS BIT(31) + +#define WMI_CFR_CORRELATION_INFO2_BUF_ADDR_HIGH GENMASK(3, 0) +#define WMI_CFR_CORRELATION_INFO2_PPDU_ID GENMASK(31, 16) + +#define WMI_CFR_CFO_MEASUREMENT_VALID BIT(0) +#define WMI_CFR_CFO_MEASUREMENT_RAW_DATA GENMASK(14, 1) + +struct ath11k_wmi_cfr_peer_tx_event_param { + u32 capture_method; + u32 vdev_id; + struct wmi_mac_addr mac_addr; + u32 chan_mhz; + u32 bandwidth; + u32 phy_mode; + u32 band_center_freq1; + u32 band_center_freq2; + u32 sts_count; + u32 correlation_info_1; + u32 correlation_info_2; + u32 status; + u32 timestamp_us; + u32 counter; + u32 chain_rssi[WMI_MAX_CHAINS]; + u32 cfo_measurement; + u32 rx_start_ts; +} __packed; + +struct ath11k_wmi_cfr_peer_tx_event_phase_param { + u32 chain_phase[WMI_MAX_CHAINS]; + u8 agc_gain[WMI_MAX_CHAINS]; +} __packed; + +enum ath11k_wmi_frame_tx_status { + WMI_FRAME_TX_STATUS_OK, + WMI_FRAME_TX_STATUS_XRETRY, + WMI_FRAME_TX_STATUS_DROP, + WMI_FRAME_TX_STATUS_FILTERED, +}; + +struct wmi_peer_cfr_capture_conf_arg { + enum ath11k_wmi_cfr_capture_bw bw; + enum ath11k_wmi_cfr_capture_method method; + u32 request; + u32 periodicity; +}; + +struct wmi_peer_cfr_capture_cmd_fixed_param { + u32 tlv_header; + u32 request; + struct wmi_mac_addr mac_addr; + u32 vdev_id; + u32 periodicity; + /* BW of measurement - of type enum ath11k_wmi_cfr_capture_bw */ + u32 bandwidth; + /* Method used to capture CFR - of type enum ath11k_wmi_cfr_capture_method */ + u32 capture_method; +} __packed; + +#define WMI_PEER_CFR_CAPTURE_ENABLE 1 +#define WMI_PEER_CFR_CAPTURE_DISABLE 0 + +/*periodicity in ms */ +#define WMI_PEER_CFR_PERIODICITY_MAX 600000 + static inline enum cc_setting_code ath11k_wmi_cc_setting_code_to_reg(enum wmi_reg_cc_setting_code status_code) { @@ -6358,7 +6449,7 @@ int ath11k_wmi_cmd_send(struct ath11k_pdev_wmi *wmi, struct sk_buff *skb, u32 cmd_id); struct sk_buff *ath11k_wmi_alloc_skb(struct ath11k_wmi_base *wmi_sc, u32 len); int ath11k_wmi_mgmt_send(struct ath11k *ar, u32 vdev_id, u32 buf_id, - struct sk_buff *frame); + struct sk_buff *frame, bool tx_params_valid); int ath11k_wmi_p2p_go_bcn_ie(struct ath11k *ar, u32 vdev_id, const u8 *p2p_ie); int ath11k_wmi_bcn_tmpl(struct ath11k *ar, u32 vdev_id, @@ -6543,5 +6634,7 @@ bool ath11k_wmi_supports_6ghz_cc_ext(struct ath11k *ar); int ath11k_wmi_send_vdev_set_tpc_power(struct ath11k *ar, u32 vdev_id, struct ath11k_reg_tpc_power_info *param); - +int ath11k_wmi_peer_set_cfr_capture_conf(struct ath11k *ar, + u32 vdev_id, const u8 *mac, + struct wmi_peer_cfr_capture_conf_arg *arg); #endif diff --git a/drivers/net/wireless/ath/ath12k/Makefile b/drivers/net/wireless/ath/ath12k/Makefile index d95ee525a6cd0..3ba1236956cc8 100644 --- a/drivers/net/wireless/ath/ath12k/Makefile +++ b/drivers/net/wireless/ath/ath12k/Makefile @@ -2,8 +2,6 @@ obj-$(CONFIG_ATH12K) += ath12k.o ath12k-y += core.o \ hal.o \ - hal_tx.o \ - hal_rx.o \ wmi.o \ mac.o \ reg.o \ @@ -12,11 +10,12 @@ ath12k-y += core.o \ dp.o \ dp_tx.o \ dp_rx.o \ + dp_htt.o \ + dp_peer.o \ debug.o \ ce.o \ peer.o \ dbring.o \ - hw.o \ mhi.o \ pci.o \ dp_mon.o \ @@ -24,6 +23,9 @@ ath12k-y += core.o \ p2p.o ath12k-$(CONFIG_ATH12K_AHB) += ahb.o + +obj-$(CONFIG_ATH12K) += wifi7/ + ath12k-$(CONFIG_ATH12K_DEBUGFS) += debugfs.o debugfs_htt_stats.o debugfs_sta.o ath12k-$(CONFIG_ACPI) += acpi.o ath12k-$(CONFIG_ATH12K_TRACING) += trace.o diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c index b30527c402f6c..9a4d34e491044 100644 --- a/drivers/net/wireless/ath/ath12k/ahb.c +++ b/drivers/net/wireless/ath/ath12k/ahb.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -16,18 +16,11 @@ #include "debug.h" #include "hif.h" -static const struct of_device_id ath12k_ahb_of_match[] = { - { .compatible = "qcom,ipq5332-wifi", - .data = (void *)ATH12K_HW_IPQ5332_HW10, - }, - { } -}; - -MODULE_DEVICE_TABLE(of, ath12k_ahb_of_match); - #define ATH12K_IRQ_CE0_OFFSET 4 #define ATH12K_MAX_UPDS 1 #define ATH12K_UPD_IRQ_WRD_LEN 18 + +static struct ath12k_ahb_driver *ath12k_ahb_family_drivers[ATH12K_DEVICE_FAMILY_MAX]; static const char ath12k_userpd_irq[][9] = {"spawn", "ready", "stop-ack"}; @@ -130,7 +123,7 @@ enum ext_irq_num { static u32 ath12k_ahb_read32(struct ath12k_base *ab, u32 offset) { - if (ab->ce_remap && offset < HAL_SEQ_WCSS_CMEM_OFFSET) + if (ab->ce_remap && offset < ab->cmem_offset) return ioread32(ab->mem_ce + offset); return ioread32(ab->mem + offset); } @@ -138,7 +131,7 @@ static u32 ath12k_ahb_read32(struct ath12k_base *ab, u32 offset) static void ath12k_ahb_write32(struct ath12k_base *ab, u32 offset, u32 value) { - if (ab->ce_remap && offset < HAL_SEQ_WCSS_CMEM_OFFSET) + if (ab->ce_remap && offset < ab->cmem_offset) iowrite32(value, ab->mem_ce + offset); else iowrite32(value, ab->mem + offset); @@ -531,9 +524,10 @@ static int ath12k_ahb_ext_grp_napi_poll(struct napi_struct *napi, int budget) struct ath12k_ext_irq_grp, napi); struct ath12k_base *ab = irq_grp->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int work_done; - work_done = ath12k_dp_service_srng(ab, irq_grp, budget); + work_done = ath12k_dp_service_srng(dp, irq_grp, budget); if (work_done < budget) { napi_complete_done(napi, work_done); ath12k_ahb_ext_grp_enable(irq_grp); @@ -563,12 +557,10 @@ static int ath12k_ahb_config_ext_irq(struct ath12k_base *ab) { const struct ath12k_hw_ring_mask *ring_mask; struct ath12k_ext_irq_grp *irq_grp; - const struct hal_ops *hal_ops; int i, j, irq, irq_idx, ret; u32 num_irq; ring_mask = ab->hw_params->ring_mask; - hal_ops = ab->hw_params->hal_ops; for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) { irq_grp = &ab->ext_irq_grp[i]; num_irq = 0; @@ -588,7 +580,7 @@ static int ath12k_ahb_config_ext_irq(struct ath12k_base *ab) * tcl_to_wbm_rbm_map point to the same ring number. */ if (ring_mask->tx[i] & - BIT(hal_ops->tcl_to_wbm_rbm_map[j].wbm_ring_num)) { + BIT(ab->hal.tcl_to_wbm_rbm_map[j].wbm_ring_num)) { irq_grp->irqs[num_irq++] = wbm2host_tx_completions_ring1 - j; } @@ -698,7 +690,7 @@ static int ath12k_ahb_map_service_to_pipe(struct ath12k_base *ab, u16 service_id return 0; } -static const struct ath12k_hif_ops ath12k_ahb_hif_ops_ipq5332 = { +static const struct ath12k_hif_ops ath12k_ahb_hif_ops = { .start = ath12k_ahb_start, .stop = ath12k_ahb_stop, .read32 = ath12k_ahb_read32, @@ -935,7 +927,8 @@ static int ath12k_ahb_resource_init(struct ath12k_base *ab) goto err_mem_unmap; } ab->ce_remap = true; - ab->ce_remap_base_addr = HAL_IPQ5332_CE_WFSS_REG_BASE; + ab->cmem_offset = ce_remap->cmem_offset; + ab->ce_remap_base_addr = ce_remap->base; } ab_ahb->xo_clk = devm_clk_get(ab->dev, "xo"); @@ -988,13 +981,34 @@ static void ath12k_ahb_resource_deinit(struct ath12k_base *ab) ab_ahb->xo_clk = NULL; } +static enum ath12k_device_family +ath12k_ahb_get_device_family(const struct platform_device *pdev) +{ + enum ath12k_device_family device_family_id; + struct ath12k_ahb_driver *driver; + const struct of_device_id *of_id; + + for (device_family_id = ATH12K_DEVICE_FAMILY_START; + device_family_id < ATH12K_DEVICE_FAMILY_MAX; device_family_id++) { + driver = ath12k_ahb_family_drivers[device_family_id]; + if (driver) { + of_id = of_match_device(driver->id_table, &pdev->dev); + if (of_id) { + /* Found the driver */ + return device_family_id; + } + } + } + + return ATH12K_DEVICE_FAMILY_MAX; +} + static int ath12k_ahb_probe(struct platform_device *pdev) { - struct ath12k_base *ab; - const struct ath12k_hif_ops *hif_ops; + enum ath12k_device_family device_id; struct ath12k_ahb *ab_ahb; - enum ath12k_hw_rev hw_rev; - u32 addr, userpd_id; + struct ath12k_base *ab; + u32 addr; int ret; ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); @@ -1008,25 +1022,32 @@ static int ath12k_ahb_probe(struct platform_device *pdev) if (!ab) return -ENOMEM; - hw_rev = (enum ath12k_hw_rev)(kernel_ulong_t)of_device_get_match_data(&pdev->dev); - switch (hw_rev) { - case ATH12K_HW_IPQ5332_HW10: - hif_ops = &ath12k_ahb_hif_ops_ipq5332; - userpd_id = ATH12K_IPQ5332_USERPD_ID; - break; - default: - ret = -EOPNOTSUPP; + ab_ahb = ath12k_ab_to_ahb(ab); + ab_ahb->ab = ab; + ab->hif.ops = &ath12k_ahb_hif_ops; + ab->pdev = pdev; + platform_set_drvdata(pdev, ab); + + device_id = ath12k_ahb_get_device_family(pdev); + if (device_id >= ATH12K_DEVICE_FAMILY_MAX) { + ath12k_err(ab, "failed to get device family: %d\n", device_id); + ret = -EINVAL; goto err_core_free; } - ab->hif.ops = hif_ops; - ab->pdev = pdev; - ab->hw_rev = hw_rev; - ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT; - platform_set_drvdata(pdev, ab); - ab_ahb = ath12k_ab_to_ahb(ab); - ab_ahb->ab = ab; - ab_ahb->userpd_id = userpd_id; + ath12k_dbg(ab, ATH12K_DBG_AHB, "AHB device family id: %d\n", device_id); + + ab_ahb->device_family_ops = &ath12k_ahb_family_drivers[device_id]->ops; + + /* Call device specific probe. This is the callback that can + * be used to override any ops in future + * probe is validated for NULL during registration. + */ + ret = ab_ahb->device_family_ops->probe(pdev); + if (ret) { + ath12k_err(ab, "failed to probe device: %d\n", ret); + goto err_core_free; + } /* Set fixed_mem_region to true for platforms that support fixed memory * reservation from DT. If memory is reserved from DT for FW, ath12k driver @@ -1065,14 +1086,26 @@ static int ath12k_ahb_probe(struct platform_device *pdev) goto err_rproc_deconfigure; } + /* Invoke arch_init here so that arch-specific init operations + * can utilize already initialized ab fields, such as HAL SRNGs. + */ + ret = ab_ahb->device_family_ops->arch_init(ab); + if (ret) { + ath12k_err(ab, "AHB arch_init failed %d\n", ret); + goto err_rproc_deconfigure; + } + ret = ath12k_core_init(ab); if (ret) { ath12k_err(ab, "failed to init core: %d\n", ret); - goto err_rproc_deconfigure; + goto err_deinit_arch; } return 0; +err_deinit_arch: + ab_ahb->device_family_ops->arch_deinit(ab); + err_rproc_deconfigure: ath12k_ahb_deconfigure_rproc(ab); @@ -1111,11 +1144,13 @@ static void ath12k_ahb_remove_prepare(struct ath12k_base *ab) static void ath12k_ahb_free_resources(struct ath12k_base *ab) { struct platform_device *pdev = ab->pdev; + struct ath12k_ahb *ab_ahb = ath12k_ab_to_ahb(ab); ath12k_hal_srng_deinit(ab); ath12k_ce_free_pipes(ab); ath12k_ahb_resource_deinit(ab); ath12k_ahb_deconfigure_rproc(ab); + ab_ahb->device_family_ops->arch_deinit(ab); ath12k_core_free(ab); platform_set_drvdata(pdev, NULL); } @@ -1136,21 +1171,47 @@ static void ath12k_ahb_remove(struct platform_device *pdev) ath12k_ahb_free_resources(ab); } -static struct platform_driver ath12k_ahb_driver = { - .driver = { - .name = "ath12k_ahb", - .of_match_table = ath12k_ahb_of_match, - }, - .probe = ath12k_ahb_probe, - .remove = ath12k_ahb_remove, -}; - -int ath12k_ahb_init(void) +int ath12k_ahb_register_driver(const enum ath12k_device_family device_id, + struct ath12k_ahb_driver *driver) { - return platform_driver_register(&ath12k_ahb_driver); + struct platform_driver *ahb_driver; + + if (device_id >= ATH12K_DEVICE_FAMILY_MAX) + return -EINVAL; + + if (!driver || !driver->ops.probe || + !driver->ops.arch_init || !driver->ops.arch_deinit) + return -EINVAL; + + if (ath12k_ahb_family_drivers[device_id]) { + pr_err("Driver already registered for id %d\n", device_id); + return -EALREADY; + } + + ath12k_ahb_family_drivers[device_id] = driver; + + ahb_driver = &ath12k_ahb_family_drivers[device_id]->driver; + ahb_driver->driver.name = driver->name; + ahb_driver->driver.of_match_table = driver->id_table; + ahb_driver->probe = ath12k_ahb_probe; + ahb_driver->remove = ath12k_ahb_remove; + + return platform_driver_register(ahb_driver); } +EXPORT_SYMBOL(ath12k_ahb_register_driver); -void ath12k_ahb_exit(void) +void ath12k_ahb_unregister_driver(const enum ath12k_device_family device_id) { - platform_driver_unregister(&ath12k_ahb_driver); + struct platform_driver *ahb_driver; + + if (device_id >= ATH12K_DEVICE_FAMILY_MAX) + return; + + if (!ath12k_ahb_family_drivers[device_id]) + return; + + ahb_driver = &ath12k_ahb_family_drivers[device_id]->driver; + platform_driver_unregister(ahb_driver); + ath12k_ahb_family_drivers[device_id] = NULL; } +EXPORT_SYMBOL(ath12k_ahb_unregister_driver); diff --git a/drivers/net/wireless/ath/ath12k/ahb.h b/drivers/net/wireless/ath/ath12k/ahb.h index d56244b20a6a6..8a040d03d27a3 100644 --- a/drivers/net/wireless/ath/ath12k/ahb.h +++ b/drivers/net/wireless/ath/ath12k/ahb.h @@ -1,13 +1,14 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022-2025, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_AHB_H #define ATH12K_AHB_H #include #include +#include #include "core.h" #define ATH12K_AHB_RECOVERY_TIMEOUT (3 * HZ) @@ -43,6 +44,12 @@ enum ath12k_ahb_userpd_irq { struct ath12k_base; +struct ath12k_ahb_device_family_ops { + int (*probe)(struct platform_device *pdev); + int (*arch_init)(struct ath12k_base *ab); + void (*arch_deinit)(struct ath12k_base *ab); +}; + struct ath12k_ahb { struct ath12k_base *ab; struct rproc *tgt_rproc; @@ -59,6 +66,15 @@ struct ath12k_ahb { u32 spawn_bit; u32 stop_bit; int userpd_irq_num[ATH12K_USERPD_MAX_IRQ]; + const struct ath12k_ahb_ops *ahb_ops; + const struct ath12k_ahb_device_family_ops *device_family_ops; +}; + +struct ath12k_ahb_driver { + const char *name; + const struct of_device_id *id_table; + struct ath12k_ahb_device_family_ops ops; + struct platform_driver driver; }; static inline struct ath12k_ahb *ath12k_ab_to_ahb(struct ath12k_base *ab) @@ -66,15 +82,8 @@ static inline struct ath12k_ahb *ath12k_ab_to_ahb(struct ath12k_base *ab) return (struct ath12k_ahb *)ab->drv_priv; } -#ifdef CONFIG_ATH12K_AHB -int ath12k_ahb_init(void); -void ath12k_ahb_exit(void); -#else -static inline int ath12k_ahb_init(void) -{ - return 0; -} +int ath12k_ahb_register_driver(const enum ath12k_device_family device_id, + struct ath12k_ahb_driver *driver); +void ath12k_ahb_unregister_driver(const enum ath12k_device_family device_id); -static inline void ath12k_ahb_exit(void) {}; -#endif #endif diff --git a/drivers/net/wireless/ath/ath12k/ce.c b/drivers/net/wireless/ath/ath12k/ce.c index 9a63608838ace..f13b260c5c96e 100644 --- a/drivers/net/wireless/ath/ath12k/ce.c +++ b/drivers/net/wireless/ath/ath12k/ce.c @@ -1,314 +1,13 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "dp_rx.h" #include "debug.h" #include "hif.h" -const struct ce_attr ath12k_host_ce_config_qcn9274[] = { - /* CE0: host->target HTC control and raw streams */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 16, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - - /* CE1: target->host HTT + HTC control */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 512, - .recv_cb = ath12k_htc_rx_completion_handler, - }, - - /* CE2: target->host WMI */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 128, - .recv_cb = ath12k_htc_rx_completion_handler, - }, - - /* CE3: host->target WMI (mac0) */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 32, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - - /* CE4: host->target HTT */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 2048, - .src_sz_max = 256, - .dest_nentries = 0, - }, - - /* CE5: target->host pktlog */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 512, - .recv_cb = ath12k_dp_htt_htc_t2h_msg_handler, - }, - - /* CE6: target autonomous hif_memcpy */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE7: host->target WMI (mac1) */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 32, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - - /* CE8: target autonomous hif_memcpy */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE9: MHI */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE10: MHI */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE11: MHI */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE12: CV Prefetch */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE13: CV Prefetch */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE14: target->host dbg log */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 512, - .recv_cb = ath12k_htc_rx_completion_handler, - }, - - /* CE15: reserved for future use */ - { - .flags = (CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, -}; - -const struct ce_attr ath12k_host_ce_config_wcn7850[] = { - /* CE0: host->target HTC control and raw streams */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 16, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - - /* CE1: target->host HTT + HTC control */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 512, - .recv_cb = ath12k_htc_rx_completion_handler, - }, - - /* CE2: target->host WMI */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 64, - .recv_cb = ath12k_htc_rx_completion_handler, - }, - - /* CE3: host->target WMI (mac0) */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 32, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - - /* CE4: host->target HTT */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 2048, - .src_sz_max = 256, - .dest_nentries = 0, - }, - - /* CE5: target->host pktlog */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE6: target autonomous hif_memcpy */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - - /* CE7: host->target WMI (mac1) */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - - /* CE8: target autonomous hif_memcpy */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - -}; - -const struct ce_attr ath12k_host_ce_config_ipq5332[] = { - /* CE0: host->target HTC control and raw streams */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 16, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - /* CE1: target->host HTT + HTC control */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 512, - .recv_cb = ath12k_htc_rx_completion_handler, - }, - /* CE2: target->host WMI */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 128, - .recv_cb = ath12k_htc_rx_completion_handler, - }, - /* CE3: host->target WMI */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 32, - .src_sz_max = 2048, - .dest_nentries = 0, - }, - /* CE4: host->target HTT */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 2048, - .src_sz_max = 256, - .dest_nentries = 0, - }, - /* CE5: target -> host PKTLOG */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 512, - .recv_cb = ath12k_dp_htt_htc_t2h_msg_handler, - }, - /* CE6: Target autonomous HIF_memcpy */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - /* CE7: CV Prefetch */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - /* CE8: Target HIF memcpy (Generic HIF memcypy) */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - /* CE9: WMI logging/CFR/Spectral/Radar */ - { - .flags = CE_ATTR_FLAGS, - .src_nentries = 0, - .src_sz_max = 2048, - .dest_nentries = 128, - }, - /* CE10: Unused */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, - /* CE11: Unused */ - { - .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, - .src_nentries = 0, - .src_sz_max = 0, - .dest_nentries = 0, - }, -}; - static int ath12k_ce_rx_buf_enqueue_pipe(struct ath12k_ce_pipe *pipe, struct sk_buff *skb, dma_addr_t paddr) { @@ -341,7 +40,7 @@ static int ath12k_ce_rx_buf_enqueue_pipe(struct ath12k_ce_pipe *pipe, goto exit; } - ath12k_hal_ce_dst_set_desc(desc, paddr); + ath12k_hal_ce_dst_set_desc(&ab->hal, desc, paddr); ring->skb[write_index] = skb; write_index = CE_RING_IDX_INCR(nentries_mask, write_index); @@ -434,7 +133,7 @@ static int ath12k_ce_completed_recv_next(struct ath12k_ce_pipe *pipe, goto err; } - *nbytes = ath12k_hal_ce_dst_status_get_length(desc); + *nbytes = ath12k_hal_ce_dst_status_get_length(&ab->hal, desc); *skb = pipe->dest_ring->skb[sw_index]; pipe->dest_ring->skb[sw_index] = NULL; @@ -666,6 +365,7 @@ ath12k_ce_alloc_ring(struct ath12k_base *ab, int nentries, int desc_sz) static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id) { + struct ath12k_hal *hal = &ab->hal; struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id]; const struct ce_attr *attr = &ab->hw_params->host_ce_config[ce_id]; struct ath12k_ce_ring *ring; @@ -677,7 +377,7 @@ static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id) if (attr->src_nentries) { pipe->send_cb = ath12k_ce_send_done_cb; nentries = roundup_pow_of_two(attr->src_nentries); - desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_SRC); + desc_sz = ath12k_hal_ce_get_desc_size(hal, HAL_CE_DESC_SRC); ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz); if (IS_ERR(ring)) return PTR_ERR(ring); @@ -687,13 +387,13 @@ static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id) if (attr->dest_nentries) { pipe->recv_cb = attr->recv_cb; nentries = roundup_pow_of_two(attr->dest_nentries); - desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST); + desc_sz = ath12k_hal_ce_get_desc_size(hal, HAL_CE_DESC_DST); ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz); if (IS_ERR(ring)) return PTR_ERR(ring); pipe->dest_ring = ring; - desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS); + desc_sz = ath12k_hal_ce_get_desc_size(hal, HAL_CE_DESC_DST_STATUS); ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz); if (IS_ERR(ring)) return PTR_ERR(ring); @@ -786,7 +486,7 @@ int ath12k_ce_send(struct ath12k_base *ab, struct sk_buff *skb, u8 pipe_id, if (pipe->attr_flags & CE_ATTR_BYTE_SWAP_DATA) byte_swap_data = 1; - ath12k_hal_ce_src_set_desc(desc, ATH12K_SKB_CB(skb)->paddr, + ath12k_hal_ce_src_set_desc(&ab->hal, desc, ATH12K_SKB_CB(skb)->paddr, skb->len, transfer_id, byte_swap_data); pipe->src_ring->skb[write_index] = skb; @@ -972,6 +672,7 @@ int ath12k_ce_init_pipes(struct ath12k_base *ab) void ath12k_ce_free_pipes(struct ath12k_base *ab) { + struct ath12k_hal *hal = &ab->hal; struct ath12k_ce_pipe *pipe; int desc_sz; int i; @@ -980,35 +681,38 @@ void ath12k_ce_free_pipes(struct ath12k_base *ab) pipe = &ab->ce.ce_pipe[i]; if (pipe->src_ring) { - desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_SRC); + desc_sz = ath12k_hal_ce_get_desc_size(hal, + HAL_CE_DESC_SRC); dma_free_coherent(ab->dev, pipe->src_ring->nentries * desc_sz + CE_DESC_RING_ALIGN, - pipe->src_ring->base_addr_owner_space, - pipe->src_ring->base_addr_ce_space); + pipe->src_ring->base_addr_owner_space_unaligned, + pipe->src_ring->base_addr_ce_space_unaligned); kfree(pipe->src_ring); pipe->src_ring = NULL; } if (pipe->dest_ring) { - desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST); + desc_sz = ath12k_hal_ce_get_desc_size(hal, + HAL_CE_DESC_DST); dma_free_coherent(ab->dev, pipe->dest_ring->nentries * desc_sz + CE_DESC_RING_ALIGN, - pipe->dest_ring->base_addr_owner_space, - pipe->dest_ring->base_addr_ce_space); + pipe->dest_ring->base_addr_owner_space_unaligned, + pipe->dest_ring->base_addr_ce_space_unaligned); kfree(pipe->dest_ring); pipe->dest_ring = NULL; } if (pipe->status_ring) { desc_sz = - ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS); + ath12k_hal_ce_get_desc_size(hal, + HAL_CE_DESC_DST_STATUS); dma_free_coherent(ab->dev, pipe->status_ring->nentries * desc_sz + CE_DESC_RING_ALIGN, - pipe->status_ring->base_addr_owner_space, - pipe->status_ring->base_addr_ce_space); + pipe->status_ring->base_addr_owner_space_unaligned, + pipe->status_ring->base_addr_ce_space_unaligned); kfree(pipe->status_ring); pipe->status_ring = NULL; } diff --git a/drivers/net/wireless/ath/ath12k/ce.h b/drivers/net/wireless/ath/ath12k/ce.h index 57f75899ee03d..df4f2a4f84809 100644 --- a/drivers/net/wireless/ath/ath12k/ce.h +++ b/drivers/net/wireless/ath/ath12k/ce.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_CE_H @@ -85,6 +85,7 @@ struct ce_ie_addr { struct ce_remap { u32 base; u32 size; + u32 cmem_offset; }; struct ce_attr { @@ -173,10 +174,6 @@ struct ath12k_ce { struct ath12k_hp_update_timer hp_timer[CE_COUNT_MAX]; }; -extern const struct ce_attr ath12k_host_ce_config_qcn9274[]; -extern const struct ce_attr ath12k_host_ce_config_wcn7850[]; -extern const struct ce_attr ath12k_host_ce_config_ipq5332[]; - void ath12k_ce_cleanup_pipes(struct ath12k_base *ab); void ath12k_ce_rx_replenish_retry(struct timer_list *t); void ath12k_ce_per_engine_service(struct ath12k_base *ab, u16 ce_id); diff --git a/drivers/net/wireless/ath/ath12k/cmn_defs.h b/drivers/net/wireless/ath/ath12k/cmn_defs.h new file mode 100644 index 0000000000000..20208ffea1c9a --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/cmn_defs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_CMN_DEFS_H +#define ATH12K_CMN_DEFS_H + +#include + +#define MAX_RADIOS 2 +#define ATH12K_MAX_DEVICES 3 +#define ATH12K_GROUP_MAX_RADIO (ATH12K_MAX_DEVICES * MAX_RADIOS) + +#define ATH12K_SCAN_MAX_LINKS ATH12K_GROUP_MAX_RADIO +/* Define 1 scan link for each radio for parallel scan purposes */ +#define ATH12K_NUM_MAX_LINKS (IEEE80211_MLD_MAX_NUM_LINKS + ATH12K_SCAN_MAX_LINKS) + +#define MAX_MU_GROUP_ID 64 +#endif diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c index 5d494c5cdc0da..9d6c50a94e642 100644 --- a/drivers/net/wireless/ath/ath12k/core.c +++ b/drivers/net/wireless/ath/ath12k/core.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ @@ -22,15 +21,18 @@ #include "hif.h" #include "pci.h" #include "wow.h" +#include "dp_cmn.h" +#include "peer.h" -static int ahb_err, pci_err; unsigned int ath12k_debug_mask; module_param_named(debug_mask, ath12k_debug_mask, uint, 0644); MODULE_PARM_DESC(debug_mask, "Debugging mask"); +EXPORT_SYMBOL(ath12k_debug_mask); bool ath12k_ftm_mode; module_param_named(ftm_mode, ath12k_ftm_mode, bool, 0444); MODULE_PARM_DESC(ftm_mode, "Boots up in factory test mode"); +EXPORT_SYMBOL(ath12k_ftm_mode); /* protected with ath12k_hw_group_mutex */ static struct list_head ath12k_hw_group_list = LIST_HEAD_INIT(ath12k_hw_group_list); @@ -633,6 +635,7 @@ u32 ath12k_core_get_max_peers_per_radio(struct ath12k_base *ab) { return ath12k_core_get_max_station_per_radio(ab) + TARGET_NUM_VDEVS(ab); } +EXPORT_SYMBOL(ath12k_core_get_max_peers_per_radio); struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab, int index) @@ -701,6 +704,8 @@ void ath12k_core_to_group_ref_put(struct ath12k_base *ab) static void ath12k_core_stop(struct ath12k_base *ab) { + ath12k_link_sta_rhash_tbl_destroy(ab); + ath12k_core_to_group_ref_put(ab); if (!test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags)) @@ -711,7 +716,7 @@ static void ath12k_core_stop(struct ath12k_base *ab) ath12k_dp_rx_pdev_reo_cleanup(ab); ath12k_hif_stop(ab); ath12k_wmi_detach(ab); - ath12k_dp_free(ab); + ath12k_dp_cmn_device_deinit(ath12k_ab_to_dp(ab)); /* De-Init of components as needed */ } @@ -896,7 +901,7 @@ static int ath12k_core_start(struct ath12k_base *ab) goto err_hif_stop; } - ret = ath12k_dp_htt_connect(&ab->dp); + ret = ath12k_dp_htt_connect(ath12k_ab_to_dp(ab)); if (ret) { ath12k_err(ab, "failed to connect to HTT: %d\n", ret); goto err_hif_stop; @@ -921,7 +926,7 @@ static int ath12k_core_start(struct ath12k_base *ab) goto err_hif_stop; } - ath12k_dp_cc_config(ab); + ath12k_hal_cc_config(ab); ret = ath12k_dp_rx_pdev_reo_setup(ab); if (ret) { @@ -929,8 +934,6 @@ static int ath12k_core_start(struct ath12k_base *ab) goto err_hif_stop; } - ath12k_dp_hal_rx_desc_init(ab); - ret = ath12k_wmi_cmd_init(ab); if (ret) { ath12k_err(ab, "failed to send wmi init cmd: %d\n", ret); @@ -965,6 +968,12 @@ static int ath12k_core_start(struct ath12k_base *ab) /* Indicate the core start in the appropriate group */ ath12k_core_to_group_ref_get(ab); + ret = ath12k_link_sta_rhash_tbl_init(ab); + if (ret) { + ath12k_warn(ab, "failed to init peer addr rhash table %d\n", ret); + goto err_reo_cleanup; + } + return 0; err_reo_cleanup: @@ -1250,7 +1259,6 @@ void ath12k_fw_stats_reset(struct ath12k *ar) spin_lock_bh(&ar->data_lock); ath12k_fw_stats_free(&ar->fw_stats); ar->fw_stats.num_vdev_recvd = 0; - ar->fw_stats.num_bcn_recvd = 0; spin_unlock_bh(&ar->data_lock); } @@ -1290,7 +1298,7 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab) goto err_firmware_stop; } - ret = ath12k_dp_alloc(ab); + ret = ath12k_dp_cmn_device_init(ath12k_ab_to_dp(ab)); if (ret) { ath12k_err(ab, "failed to init DP: %d\n", ret); goto err_firmware_stop; @@ -1302,7 +1310,7 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab) ret = ath12k_core_start(ab); if (ret) { ath12k_err(ab, "failed to start core: %d\n", ret); - goto err_dp_free; + goto err_deinit; } mutex_unlock(&ab->core_lock); @@ -1335,8 +1343,8 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab) mutex_unlock(&ag->mutex); goto exit; -err_dp_free: - ath12k_dp_free(ab); +err_deinit: + ath12k_dp_cmn_device_deinit(ath12k_ab_to_dp(ab)); mutex_unlock(&ab->core_lock); mutex_unlock(&ag->mutex); @@ -1352,13 +1360,14 @@ static int ath12k_core_reconfigure_on_crash(struct ath12k_base *ab) int ret, total_vdev; mutex_lock(&ab->core_lock); + ath12k_link_sta_rhash_tbl_destroy(ab); ath12k_dp_pdev_free(ab); ath12k_ce_cleanup_pipes(ab); ath12k_wmi_detach(ab); ath12k_dp_rx_pdev_reo_cleanup(ab); mutex_unlock(&ab->core_lock); - ath12k_dp_free(ab); + ath12k_dp_cmn_device_deinit(ath12k_ab_to_dp(ab)); ath12k_hal_srng_deinit(ab); total_vdev = ab->num_radios * TARGET_NUM_VDEVS(ab); ab->free_vdev_map = (1LL << total_vdev) - 1; @@ -1567,6 +1576,7 @@ static void ath12k_core_post_reconfigure_recovery(struct ath12k_base *ab) ath12k_core_halt(ar); } + ath12k_mac_dp_peer_cleanup(ah); break; case ATH12K_HW_STATE_OFF: ath12k_warn(ab, @@ -1741,17 +1751,11 @@ enum ath12k_qmi_mem_mode ath12k_core_get_memory_mode(struct ath12k_base *ab) return ATH12K_QMI_MEMORY_MODE_DEFAULT; } +EXPORT_SYMBOL(ath12k_core_get_memory_mode); int ath12k_core_pre_init(struct ath12k_base *ab) { const struct ath12k_mem_profile_based_param *param; - int ret; - - ret = ath12k_hw_init(ab); - if (ret) { - ath12k_err(ab, "failed to init hw params: %d\n", ret); - return ret; - } param = &ath12k_mem_profile_based_param[ab->target_mem_mode]; ab->profile_param = param; @@ -1998,6 +2002,8 @@ static struct ath12k_hw_group *ath12k_core_hw_group_assign(struct ath12k_base *a ag->ab[ab->device_id] = ab; ab->ag = ag; + ath12k_dp_cmn_hw_group_assign(ath12k_ab_to_dp(ab), ag); + ath12k_dbg(ab, ATH12K_DBG_BOOT, "wsi group-id %d num-devices %d index %d", ag->id, ag->num_devices, wsi->index); @@ -2025,6 +2031,8 @@ void ath12k_core_hw_group_unassign(struct ath12k_base *ab) return; } + ath12k_dp_cmn_hw_group_unassign(ath12k_ab_to_dp(ab), ag); + ag->ab[device_id] = NULL; ab->ag = NULL; ab->device_id = ATH12K_INVALID_DEVICE_ID; @@ -2106,14 +2114,27 @@ static int ath12k_core_hw_group_create(struct ath12k_hw_group *ag) ret = ath12k_core_soc_create(ab); if (ret) { mutex_unlock(&ab->core_lock); - ath12k_err(ab, "failed to create soc core: %d\n", ret); - return ret; + ath12k_err(ab, "failed to create soc %d core: %d\n", i, ret); + goto destroy; } mutex_unlock(&ab->core_lock); } return 0; + +destroy: + for (i--; i >= 0; i--) { + ab = ag->ab[i]; + if (!ab) + continue; + + mutex_lock(&ab->core_lock); + ath12k_core_soc_destroy(ab); + mutex_unlock(&ab->core_lock); + } + + return ret; } void ath12k_core_hw_group_set_mlo_capable(struct ath12k_hw_group *ag) @@ -2188,7 +2209,7 @@ int ath12k_core_init(struct ath12k_base *ab) if (ret) { mutex_unlock(&ag->mutex); ath12k_warn(ab, "unable to create hw group\n"); - goto err_destroy_hw_group; + goto err_unassign_hw_group; } } @@ -2196,8 +2217,7 @@ int ath12k_core_init(struct ath12k_base *ab) return 0; -err_destroy_hw_group: - ath12k_core_hw_group_destroy(ab->ag); +err_unassign_hw_group: ath12k_core_hw_group_unassign(ab); err_unregister_notifier: ath12k_core_panic_notifier_unregister(ab); @@ -2243,7 +2263,6 @@ struct ath12k_base *ath12k_core_alloc(struct device *dev, size_t priv_size, spin_lock_init(&ab->base_lock); init_completion(&ab->reset_complete); - INIT_LIST_HEAD(&ab->peers); init_waitqueue_head(&ab->peer_mapping_wq); init_waitqueue_head(&ab->wmi_ab.tx_credits_wq); INIT_WORK(&ab->restart_work, ath12k_core_restart); @@ -2281,31 +2300,5 @@ struct ath12k_base *ath12k_core_alloc(struct device *dev, size_t priv_size, return NULL; } -static int ath12k_init(void) -{ - ahb_err = ath12k_ahb_init(); - if (ahb_err) - pr_warn("Failed to initialize ath12k AHB device: %d\n", ahb_err); - - pci_err = ath12k_pci_init(); - if (pci_err) - pr_warn("Failed to initialize ath12k PCI device: %d\n", pci_err); - - /* If both failed, return one of the failures (arbitrary) */ - return ahb_err && pci_err ? ahb_err : 0; -} - -static void ath12k_exit(void) -{ - if (!pci_err) - ath12k_pci_exit(); - - if (!ahb_err) - ath12k_ahb_exit(); -} - -module_init(ath12k_init); -module_exit(ath12k_exit); - -MODULE_DESCRIPTION("Driver support for Qualcomm Technologies 802.11be WLAN devices"); +MODULE_DESCRIPTION("Driver support for Qualcomm Technologies WLAN devices"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h index 3d1956966a485..990934ec92fca 100644 --- a/drivers/net/wireless/ath/ath12k/core.h +++ b/drivers/net/wireless/ath/ath12k/core.h @@ -18,6 +18,7 @@ #include #include #include +#include #include "qmi.h" #include "htc.h" #include "wmi.h" @@ -26,7 +27,6 @@ #include "ce.h" #include "mac.h" #include "hw.h" -#include "hal_rx.h" #include "reg.h" #include "dbring.h" #include "fw.h" @@ -34,6 +34,8 @@ #include "wow.h" #include "debugfs_htt_stats.h" #include "coredump.h" +#include "cmn_defs.h" +#include "dp_cmn.h" #define SM(_v, _f) (((_v) << _f##_LSB) & _f##_MASK) @@ -64,8 +66,6 @@ #define ATH12K_RECONFIGURE_TIMEOUT_HZ (10 * HZ) #define ATH12K_RECOVER_START_TIMEOUT_HZ (20 * HZ) -#define ATH12K_MAX_DEVICES 3 -#define ATH12K_GROUP_MAX_RADIO (ATH12K_MAX_DEVICES * MAX_RADIOS) #define ATH12K_INVALID_GROUP_ID 0xFF #define ATH12K_INVALID_DEVICE_ID 0xFF @@ -155,6 +155,7 @@ enum ath12k_hw_rev { ATH12K_HW_QCN9274_HW20, ATH12K_HW_WCN7850_HW20, ATH12K_HW_IPQ5332_HW10, + ATH12K_HW_QCC2072_HW10, }; enum ath12k_firmware_mode { @@ -310,16 +311,9 @@ struct ath12k_link_vif { u32 vdev_id; u32 beacon_interval; u32 dtim_period; - u16 ast_hash; - u16 ast_idx; - u16 tcl_metadata; - u8 hal_addr_search_flags; - u8 search_type; struct ath12k *ar; - int bank_id; - u8 vdev_id_check_en; bool beacon_prot; struct wmi_wmm_params_all_arg wmm_params; @@ -355,9 +349,13 @@ struct ath12k_link_vif { struct wmi_vdev_install_key_arg group_key; bool pairwise_key_done; u16 num_stations; + bool is_csa_in_progress; + struct wiphy_work bcn_tx_work; }; struct ath12k_vif { + struct ath12k_dp_vif dp_vif; + enum wmi_vdev_type vdev_type; enum wmi_vdev_subtype vdev_subtype; struct ieee80211_vif *vif; @@ -381,10 +379,7 @@ struct ath12k_vif { } u; u32 aid; - u32 key_cipher; - u8 tx_encap_type; bool ps; - atomic_t mcbc_gsn; struct ath12k_link_vif deflink; struct ath12k_link_vif __rcu *link[ATH12K_NUM_MAX_LINKS]; @@ -405,51 +400,8 @@ struct ath12k_vif_iter { struct ath12k_link_vif *arvif; }; -#define HAL_AST_IDX_INVALID 0xFFFF -#define HAL_RX_MAX_MCS 12 -#define HAL_RX_MAX_MCS_HT 31 -#define HAL_RX_MAX_MCS_VHT 9 -#define HAL_RX_MAX_MCS_HE 11 -#define HAL_RX_MAX_MCS_BE 15 -#define HAL_RX_MAX_NSS 8 -#define HAL_RX_MAX_NUM_LEGACY_RATES 12 - #define ATH12K_SCAN_TIMEOUT_HZ (20 * HZ) -struct ath12k_rx_peer_rate_stats { - u64 ht_mcs_count[HAL_RX_MAX_MCS_HT + 1]; - u64 vht_mcs_count[HAL_RX_MAX_MCS_VHT + 1]; - u64 he_mcs_count[HAL_RX_MAX_MCS_HE + 1]; - u64 be_mcs_count[HAL_RX_MAX_MCS_BE + 1]; - u64 nss_count[HAL_RX_MAX_NSS]; - u64 bw_count[HAL_RX_BW_MAX]; - u64 gi_count[HAL_RX_GI_MAX]; - u64 legacy_count[HAL_RX_MAX_NUM_LEGACY_RATES]; - u64 rx_rate[HAL_RX_BW_MAX][HAL_RX_GI_MAX][HAL_RX_MAX_NSS][HAL_RX_MAX_MCS_HT + 1]; -}; - -struct ath12k_rx_peer_stats { - u64 num_msdu; - u64 num_mpdu_fcs_ok; - u64 num_mpdu_fcs_err; - u64 tcp_msdu_count; - u64 udp_msdu_count; - u64 other_msdu_count; - u64 ampdu_msdu_count; - u64 non_ampdu_msdu_count; - u64 stbc_count; - u64 beamformed_count; - u64 coding_count[HAL_RX_SU_MU_CODING_MAX]; - u64 tid_count[IEEE80211_NUM_TIDS + 1]; - u64 pream_cnt[HAL_RX_PREAMBLE_MAX]; - u64 reception_type[HAL_RX_RECEPTION_TYPE_MAX]; - u64 rx_duration; - u64 dcm_count; - u64 ru_alloc_cnt[HAL_RX_RU_ALLOC_TYPE_MAX]; - struct ath12k_rx_peer_rate_stats pkt_stats; - struct ath12k_rx_peer_rate_stats byte_stats; -}; - #define ATH12K_HE_MCS_NUM 12 #define ATH12K_VHT_MCS_NUM 10 #define ATH12K_BW_NUM 5 @@ -531,12 +483,6 @@ struct ath12k_per_ppdu_tx_stats { u32 retry_bytes; }; -struct ath12k_wbm_tx_stats { - u64 wbm_tx_comp_stats[HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX]; -}; - -DECLARE_EWMA(avg_rssi, 10, 8) - struct ath12k_link_sta { struct ath12k_link_vif *arvif; struct ath12k_sta *ahsta; @@ -551,15 +497,7 @@ struct ath12k_link_sta { u32 smps; struct wiphy_work update_wk; - struct rate_info txrate; - struct rate_info last_txrate; - u64 rx_duration; - u64 tx_duration; - u8 rssi_comb; - struct ewma_avg_rssi avg_rssi; u8 link_id; - struct ath12k_rx_peer_stats *rx_stats; - struct ath12k_wbm_tx_stats *wbm_tx_stats; u32 bw_prev; u32 peer_nss; s8 rssi_beacon; @@ -570,14 +508,9 @@ struct ath12k_link_sta { /* for firmware use only */ u8 link_idx; - u32 tx_retry_failed; - u32 tx_retry_count; -}; -struct ath12k_reoq_buf { - void *vaddr; - dma_addr_t paddr_aligned; - u32 size; + /* peer addr based rhashtable list pointer */ + struct rhash_head rhash_addr; }; struct ath12k_sta { @@ -592,8 +525,6 @@ struct ath12k_sta { u8 num_peer; enum ieee80211_sta_state state; - - struct ath12k_reoq_buf reoq_bufs[IEEE80211_NUM_TIDS + 1]; }; #define ATH12K_HALF_20MHZ_BW 10 @@ -644,7 +575,6 @@ struct ath12k_fw_stats { struct list_head vdevs; struct list_head bcn; u32 num_vdev_recvd; - u32 num_bcn_recvd; }; struct ath12k_dbg_htt_stats { @@ -666,23 +596,6 @@ struct ath12k_debug { bool extd_rx_stats; }; -struct ath12k_per_peer_tx_stats { - u32 succ_bytes; - u32 retry_bytes; - u32 failed_bytes; - u32 duration; - u16 succ_pkts; - u16 retry_pkts; - u16 failed_pkts; - u16 ru_start; - u16 ru_tones; - u8 ba_fails; - u8 ppdu_type; - u32 mu_grpid; - u32 mu_pos; - bool is_ampdu; -}; - struct ath12k_pdev_rssi_offsets { s32 temp_offset; s8 min_nf_dbm; @@ -808,9 +721,6 @@ struct ath12k { struct ath12k_wow wow; struct completion target_suspend; bool target_suspend_ack; - struct ath12k_per_peer_tx_stats peer_tx_stats; - struct list_head ppdu_stats_info; - u32 ppdu_stat_list_depth; struct ath12k_per_peer_tx_stats cached_stats; u32 last_ppdu_id; @@ -865,8 +775,7 @@ struct ath12k_hw { DECLARE_BITMAP(free_ml_peer_id_map, ATH12K_MAX_MLO_PEERS); - /* protected by wiphy_lock() */ - struct list_head ml_peers; + struct ath12k_dp_hw dp_hw; /* Keep last */ struct ath12k radio[] __aligned(sizeof(void *)); @@ -940,31 +849,6 @@ struct ath12k_board_data { size_t len; }; -struct ath12k_device_dp_tx_err_stats { - /* TCL Ring Descriptor unavailable */ - u32 desc_na[DP_TCL_NUM_RING_MAX]; - /* Other failures during dp_tx due to mem allocation failure - * idr unavailable etc. - */ - atomic_t misc_fail; -}; - -struct ath12k_device_dp_stats { - u32 err_ring_pkts; - u32 invalid_rbm; - u32 rxdma_error[HAL_REO_ENTR_RING_RXDMA_ECODE_MAX]; - u32 reo_error[HAL_REO_DEST_RING_ERROR_CODE_MAX]; - u32 hal_reo_error[DP_REO_DST_RING_MAX]; - struct ath12k_device_dp_tx_err_stats tx_err; - u32 reo_rx[DP_REO_DST_RING_MAX][ATH12K_MAX_DEVICES]; - u32 rx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX][ATH12K_MAX_DEVICES]; - u32 tqm_rel_reason[MAX_TQM_RELEASE_REASON]; - u32 fw_tx_status[MAX_FW_TX_STATUS]; - u32 tx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX]; - u32 tx_enqueued[DP_TCL_NUM_RING_MAX]; - u32 tx_completed[DP_TCL_NUM_RING_MAX]; -}; - struct ath12k_reg_freq { u32 start_freq; u32 end_freq; @@ -985,6 +869,11 @@ struct ath12k_hw_link { * wiphy, protected with struct ath12k_hw_group::mutex. */ struct ath12k_hw_group { + /* Keep dp_hw_grp as the first member to allow efficient + * usage of cache lines for DP fields + */ + struct ath12k_dp_hw_group dp_hw_grp; + struct ath12k_hw_link hw_links[ATH12K_GROUP_MAX_RADIO]; struct list_head list; u8 id; u8 num_devices; @@ -1007,7 +896,6 @@ struct ath12k_hw_group { bool mlo_capable; struct device_node *wsi_node[ATH12K_MAX_DEVICES]; struct ath12k_mlo_memory mlo_mem; - struct ath12k_hw_link hw_links[ATH12K_GROUP_MAX_RADIO]; bool hw_link_id_init_done; }; @@ -1033,6 +921,12 @@ struct ath12k_mem_profile_based_param { struct ath12k_dp_profile_params dp_params; }; +enum ath12k_device_family { + ATH12K_DEVICE_FAMILY_START, + ATH12K_DEVICE_FAMILY_WIFI7 = ATH12K_DEVICE_FAMILY_START, + ATH12K_DEVICE_FAMILY_MAX, +}; + /* Master structure to hold the hw data which may be used in core module */ struct ath12k_base { enum ath12k_hw_rev hw_rev; @@ -1052,13 +946,14 @@ struct ath12k_base { struct ath12k_htc htc; - struct ath12k_dp dp; + struct ath12k_dp *dp; void __iomem *mem; unsigned long mem_len; void __iomem *mem_ce; u32 ce_remap_base_addr; + u32 cmem_offset; bool ce_remap; struct { @@ -1103,7 +998,6 @@ struct ath12k_base { struct ath12k_wmi_hal_reg_capabilities_ext_arg hal_reg_cap[MAX_RADIOS]; unsigned long long free_vdev_map; unsigned long long free_vdev_stats_id_map; - struct list_head peers; wait_queue_head_t peer_mapping_wq; u8 mac_addr[ETH_ALEN]; bool wmi_ready; @@ -1133,7 +1027,6 @@ struct ath12k_base { /* Current DFS Regulatory */ enum ath12k_dfs_region dfs_region; - struct ath12k_device_dp_stats device_stats; #ifdef CONFIG_ATH12K_DEBUGFS struct dentry *debugfs_soc; #endif @@ -1189,13 +1082,13 @@ struct ath12k_base { size_t amss_dualmac_len; const u8 *m3_data; size_t m3_len; + const u8 *aux_uc_data; + size_t aux_uc_len; DECLARE_BITMAP(fw_features, ATH12K_FW_FEATURE_COUNT); bool fw_features_valid; } fw; - const struct hal_rx_ops *hal_rx_ops; - struct completion restart_completed; #ifdef CONFIG_ACPI @@ -1239,6 +1132,14 @@ struct ath12k_base { const struct ath12k_mem_profile_based_param *profile_param; enum ath12k_qmi_mem_mode target_mem_mode; + /* FIXME: Define this field in a ag equivalent object available + * during the initial phase of probe later. + */ + const struct ieee80211_ops *ath12k_ops; + + struct rhashtable *rhead_sta_addr; + struct rhashtable_params rhash_sta_addr_param; + /* must be last */ u8 drv_priv[] __aligned(sizeof(void *)); }; @@ -1514,4 +1415,18 @@ static inline s32 ath12k_pdev_get_noise_floor(struct ath12k *ar) return ar->rssi_info.noise_floor; } +/* The @ab->dp NULL check or assertion is intentionally omitted because + * @ab->dp is guaranteed to be non-NULL after a successful probe and + * remains valid until teardown. Invoking this before allocation or + * after teardown is considered invalid usage. + */ +static inline struct ath12k_dp *ath12k_ab_to_dp(struct ath12k_base *ab) +{ + return ab->dp; +} + +static inline struct ath12k *ath12k_pdev_dp_to_ar(struct ath12k_pdev_dp *dp) +{ + return container_of(dp, struct ath12k, dp); +} #endif /* _CORE_H_ */ diff --git a/drivers/net/wireless/ath/ath12k/dbring.c b/drivers/net/wireless/ath/ath12k/dbring.c index 6604dacea2ae5..f71ec2a58469f 100644 --- a/drivers/net/wireless/ath/ath12k/dbring.c +++ b/drivers/net/wireless/ath/ath12k/dbring.c @@ -1,12 +1,12 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "core.h" #include "debug.h" +#include "hal.h" static int ath12k_dbring_bufs_replenish(struct ath12k *ar, struct ath12k_dbring *ring, @@ -55,7 +55,7 @@ static int ath12k_dbring_bufs_replenish(struct ath12k *ar, cookie = u32_encode_bits(ar->pdev_idx, DP_RXDMA_BUF_COOKIE_PDEV_ID) | u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID); - ath12k_hal_rx_buf_addr_info_set(desc, paddr, cookie, 0); + ath12k_hal_rx_buf_addr_info_set(&ab->hal, desc, paddr, cookie, 0); ath12k_hal_srng_access_end(ab, srng); @@ -298,7 +298,7 @@ int ath12k_dbring_buffer_release_event(struct ath12k_base *ab, num_buff_reaped++; - ath12k_hal_rx_buf_addr_info_get(&desc, &paddr, &cookie, &rbm); + ath12k_hal_rx_buf_addr_info_get(&ab->hal, &desc, &paddr, &cookie, &rbm); buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); diff --git a/drivers/net/wireless/ath/ath12k/debug.c b/drivers/net/wireless/ath/ath12k/debug.c index 5ce100cd9a9d1..34b3b2c920dc7 100644 --- a/drivers/net/wireless/ath/ath12k/debug.c +++ b/drivers/net/wireless/ath/ath12k/debug.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * */ #include @@ -21,6 +22,7 @@ void ath12k_info(struct ath12k_base *ab, const char *fmt, ...) /* TODO: Trace the log */ va_end(args); } +EXPORT_SYMBOL(ath12k_info); void ath12k_err(struct ath12k_base *ab, const char *fmt, ...) { @@ -35,6 +37,7 @@ void ath12k_err(struct ath12k_base *ab, const char *fmt, ...) /* TODO: Trace the log */ va_end(args); } +EXPORT_SYMBOL(ath12k_err); void __ath12k_warn(struct device *dev, const char *fmt, ...) { @@ -49,6 +52,7 @@ void __ath12k_warn(struct device *dev, const char *fmt, ...) /* TODO: Trace the log */ va_end(args); } +EXPORT_SYMBOL(__ath12k_warn); #ifdef CONFIG_ATH12K_DEBUG @@ -72,6 +76,7 @@ void __ath12k_dbg(struct ath12k_base *ab, enum ath12k_debug_mask mask, va_end(args); } +EXPORT_SYMBOL(__ath12k_dbg); void ath12k_dbg_dump(struct ath12k_base *ab, enum ath12k_debug_mask mask, @@ -100,5 +105,6 @@ void ath12k_dbg_dump(struct ath12k_base *ab, } } } +EXPORT_SYMBOL(ath12k_dbg_dump); #endif /* CONFIG_ATH12K_DEBUG */ diff --git a/drivers/net/wireless/ath/ath12k/debugfs.c b/drivers/net/wireless/ath/ath12k/debugfs.c index 16601a8c36448..358031fa14ebb 100644 --- a/drivers/net/wireless/ath/ath12k/debugfs.c +++ b/drivers/net/wireless/ath/ath12k/debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "core.h" @@ -967,7 +967,7 @@ static int ath12k_open_link_stats(struct inode *inode, struct file *file) "\nlink[%d] Tx Frame descriptor Encrypt Type = ", link_id); - for (i = 0; i < HAL_ENCRYPT_TYPE_MAX; i++) { + for (i = 0; i < DP_ENCRYPT_TYPE_MAX; i++) { len += scnprintf(buf + len, buf_len - len, " %d:%d", i, linkstat.tx_encrypt_type[i]); @@ -1020,13 +1020,15 @@ void ath12k_debugfs_op_vif_add(struct ieee80211_hw *hw, debugfs_create_file("link_stats", 0400, vif->debugfs_dir, ahvif, &ath12k_fops_link_stats); } +EXPORT_SYMBOL(ath12k_debugfs_op_vif_add); static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { struct ath12k_base *ab = file->private_data; - struct ath12k_device_dp_stats *device_stats = &ab->device_stats; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_device_dp_stats *device_stats = &dp->device_stats; int len = 0, i, j, ret; struct ath12k *ar; const int size = 4096; @@ -1155,6 +1157,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file, len += scnprintf(buf + len, size - len, "\n"); + rcu_read_lock(); for (i = 0; i < ab->num_radios; i++) { ar = ath12k_mac_get_ar_by_pdev_id(ab, DP_SW2HW_MACID(i)); if (ar) { @@ -1163,6 +1166,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file, atomic_read(&ar->dp.num_tx_pending)); } } + rcu_read_unlock(); len += scnprintf(buf + len, size - len, "\nREO Rx Received:\n"); @@ -1178,6 +1182,9 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file, len += scnprintf(buf + len, size - len, "\n"); } + len += scnprintf(buf + len, size - len, "\nREO excep MSDU buf type:%u\n", + device_stats->reo_excep_msdu_buf_type); + len += scnprintf(buf + len, size - len, "\nRx WBM REL SRC Errors:\n"); for (i = 0; i < HAL_WBM_REL_SRC_MODULE_MAX; i++) { @@ -1283,6 +1290,7 @@ static int ath12k_open_vdev_stats(struct inode *inode, struct file *file) ath12k_wmi_fw_stats_dump(ar, &ar->fw_stats, param.stats_id, buf); + ath12k_fw_stats_reset(ar); file->private_data = no_free_ptr(buf); @@ -1349,12 +1357,7 @@ static int ath12k_open_bcn_stats(struct inode *inode, struct file *file) ath12k_wmi_fw_stats_dump(ar, &ar->fw_stats, param.stats_id, buf); - /* since beacon stats request is looped for all active VDEVs, saved fw - * stats is not freed for each request until done for all active VDEVs - */ - spin_lock_bh(&ar->data_lock); - ath12k_fw_stats_bcn_free(&ar->fw_stats.bcn); - spin_unlock_bh(&ar->data_lock); + ath12k_fw_stats_reset(ar); file->private_data = no_free_ptr(buf); @@ -1415,6 +1418,7 @@ static int ath12k_open_pdev_stats(struct inode *inode, struct file *file) ath12k_wmi_fw_stats_dump(ar, &ar->fw_stats, param.stats_id, buf); + ath12k_fw_stats_reset(ar); file->private_data = no_free_ptr(buf); diff --git a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c index 48b010a1b7566..7f6ca07fb3359 100644 --- a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c +++ b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -29,8 +29,10 @@ print_array_to_buf_index(u8 *buf, u32 offset, const char *header, u32 stats_inde " %u:%u,", stats_index++, le32_to_cpu(array[i])); } /* To overwrite the last trailing comma */ - index--; - *(buf + offset + index) = '\0'; + if (array_len > 0) { + index--; + *(buf + offset + index) = '\0'; + } if (footer) { index += scnprintf(buf + offset + index, @@ -5537,6 +5539,189 @@ ath12k_htt_print_pdev_rtt_tbr_cmd_res_stats_tlv(const void *tag_buf, u16 tag_len stats_req->buf_len = len; } +static void +ath12k_htt_print_rx_pdev_fw_stats_tlv(const void *tag_buf, u16 tag_len, + struct debug_htt_stats_req *stats_req) +{ + const struct htt_rx_pdev_fw_stats_tlv *htt_stats_buf = tag_buf; + u32 buf_len = ATH12K_HTT_STATS_BUF_SIZE; + u32 len = stats_req->buf_len; + u8 *buf = stats_req->buf; + + if (tag_len < sizeof(*htt_stats_buf)) + return; + + len += scnprintf(buf + len, buf_len - len, "HTT_RX_PDEV_FW_STATS_TLV:\n"); + len += scnprintf(buf + len, buf_len - len, "mac_id = %u\n", + le32_to_cpu(htt_stats_buf->mac_id__word) & 0xFF); + len += scnprintf(buf + len, buf_len - len, "ppdu_recvd = %u\n", + le32_to_cpu(htt_stats_buf->ppdu_recvd)); + len += scnprintf(buf + len, buf_len - len, "mpdu_cnt_fcs_ok = %u\n", + le32_to_cpu(htt_stats_buf->mpdu_cnt_fcs_ok)); + len += scnprintf(buf + len, buf_len - len, "mpdu_cnt_fcs_err = %u\n", + le32_to_cpu(htt_stats_buf->mpdu_cnt_fcs_err)); + len += scnprintf(buf + len, buf_len - len, "tcp_msdu_cnt = %u\n", + le32_to_cpu(htt_stats_buf->tcp_msdu_cnt)); + len += scnprintf(buf + len, buf_len - len, "tcp_ack_msdu_cnt = %u\n", + le32_to_cpu(htt_stats_buf->tcp_ack_msdu_cnt)); + len += scnprintf(buf + len, buf_len - len, "udp_msdu_cnt = %u\n", + le32_to_cpu(htt_stats_buf->udp_msdu_cnt)); + len += scnprintf(buf + len, buf_len - len, "other_msdu_cnt = %u\n", + le32_to_cpu(htt_stats_buf->other_msdu_cnt)); + len += scnprintf(buf + len, buf_len - len, "fw_ring_mpdu_ind = %u\n", + le32_to_cpu(htt_stats_buf->fw_ring_mpdu_ind)); + len += print_array_to_buf(buf, len, "fw_ring_mgmt_subtype", + htt_stats_buf->fw_ring_mgmt_subtype, + ATH12K_HTT_STATS_SUBTYPE_MAX, "\n"); + len += print_array_to_buf(buf, len, "fw_ring_ctrl_subtype", + htt_stats_buf->fw_ring_ctrl_subtype, + ATH12K_HTT_STATS_SUBTYPE_MAX, "\n"); + len += scnprintf(buf + len, buf_len - len, "fw_ring_mcast_data_msdu = %u\n", + le32_to_cpu(htt_stats_buf->fw_ring_mcast_data_msdu)); + len += scnprintf(buf + len, buf_len - len, "fw_ring_bcast_data_msdu = %u\n", + le32_to_cpu(htt_stats_buf->fw_ring_bcast_data_msdu)); + len += scnprintf(buf + len, buf_len - len, "fw_ring_ucast_data_msdu = %u\n", + le32_to_cpu(htt_stats_buf->fw_ring_ucast_data_msdu)); + len += scnprintf(buf + len, buf_len - len, "fw_ring_null_data_msdu = %u\n", + le32_to_cpu(htt_stats_buf->fw_ring_null_data_msdu)); + len += scnprintf(buf + len, buf_len - len, "fw_ring_mpdu_drop = %u\n", + le32_to_cpu(htt_stats_buf->fw_ring_mpdu_drop)); + len += scnprintf(buf + len, buf_len - len, "ofld_local_data_ind_cnt = %u\n", + le32_to_cpu(htt_stats_buf->ofld_local_data_ind_cnt)); + len += scnprintf(buf + len, buf_len - len, + "ofld_local_data_buf_recycle_cnt = %u\n", + le32_to_cpu(htt_stats_buf->ofld_local_data_buf_recycle_cnt)); + len += scnprintf(buf + len, buf_len - len, "drx_local_data_ind_cnt = %u\n", + le32_to_cpu(htt_stats_buf->drx_local_data_ind_cnt)); + len += scnprintf(buf + len, buf_len - len, + "drx_local_data_buf_recycle_cnt = %u\n", + le32_to_cpu(htt_stats_buf->drx_local_data_buf_recycle_cnt)); + len += scnprintf(buf + len, buf_len - len, "local_nondata_ind_cnt = %u\n", + le32_to_cpu(htt_stats_buf->local_nondata_ind_cnt)); + len += scnprintf(buf + len, buf_len - len, "local_nondata_buf_recycle_cnt = %u\n", + le32_to_cpu(htt_stats_buf->local_nondata_buf_recycle_cnt)); + len += scnprintf(buf + len, buf_len - len, "fw_status_buf_ring_refill_cnt = %u\n", + le32_to_cpu(htt_stats_buf->fw_status_buf_ring_refill_cnt)); + len += scnprintf(buf + len, buf_len - len, "fw_status_buf_ring_empty_cnt = %u\n", + le32_to_cpu(htt_stats_buf->fw_status_buf_ring_empty_cnt)); + len += scnprintf(buf + len, buf_len - len, "fw_pkt_buf_ring_refill_cnt = %u\n", + le32_to_cpu(htt_stats_buf->fw_pkt_buf_ring_refill_cnt)); + len += scnprintf(buf + len, buf_len - len, "fw_pkt_buf_ring_empty_cnt = %u\n", + le32_to_cpu(htt_stats_buf->fw_pkt_buf_ring_empty_cnt)); + len += scnprintf(buf + len, buf_len - len, "fw_link_buf_ring_refill_cnt = %u\n", + le32_to_cpu(htt_stats_buf->fw_link_buf_ring_refill_cnt)); + len += scnprintf(buf + len, buf_len - len, "fw_link_buf_ring_empty_cnt = %u\n", + le32_to_cpu(htt_stats_buf->fw_link_buf_ring_empty_cnt)); + len += scnprintf(buf + len, buf_len - len, "host_pkt_buf_ring_refill_cnt = %u\n", + le32_to_cpu(htt_stats_buf->host_pkt_buf_ring_refill_cnt)); + len += scnprintf(buf + len, buf_len - len, "host_pkt_buf_ring_empty_cnt = %u\n", + le32_to_cpu(htt_stats_buf->host_pkt_buf_ring_empty_cnt)); + len += scnprintf(buf + len, buf_len - len, "mon_pkt_buf_ring_refill_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_pkt_buf_ring_refill_cnt)); + len += scnprintf(buf + len, buf_len - len, "mon_pkt_buf_ring_empty_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_pkt_buf_ring_empty_cnt)); + len += scnprintf(buf + len, buf_len - len, + "mon_status_buf_ring_refill_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_status_buf_ring_refill_cnt)); + len += scnprintf(buf + len, buf_len - len, "mon_status_buf_ring_empty_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_status_buf_ring_empty_cnt)); + len += scnprintf(buf + len, buf_len - len, "mon_desc_buf_ring_refill_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_desc_buf_ring_refill_cnt)); + len += scnprintf(buf + len, buf_len - len, "mon_desc_buf_ring_empty_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_desc_buf_ring_empty_cnt)); + len += scnprintf(buf + len, buf_len - len, "mon_dest_ring_update_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_dest_ring_update_cnt)); + len += scnprintf(buf + len, buf_len - len, "mon_dest_ring_full_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mon_dest_ring_full_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_suspend_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_suspend_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_suspend_fail_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_suspend_fail_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_resume_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_resume_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_resume_fail_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_resume_fail_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_ring_switch_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_ring_switch_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_ring_restore_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_ring_restore_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_flush_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_flush_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_recovery_reset_cnt = %u\n", + le32_to_cpu(htt_stats_buf->rx_recovery_reset_cnt)); + len += scnprintf(buf + len, buf_len - len, "rx_lwm_prom_filter_dis = %u\n", + le32_to_cpu(htt_stats_buf->rx_lwm_prom_filter_dis)); + len += scnprintf(buf + len, buf_len - len, "rx_hwm_prom_filter_en = %u\n", + le32_to_cpu(htt_stats_buf->rx_hwm_prom_filter_en)); + len += scnprintf(buf + len, buf_len - len, "bytes_received_low_32 = %u\n", + le32_to_cpu(htt_stats_buf->bytes_received_low_32)); + len += scnprintf(buf + len, buf_len - len, "bytes_received_high_32 = %u\n", + le32_to_cpu(htt_stats_buf->bytes_received_high_32)); + + stats_req->buf_len = len; +} + +static void +ath12k_htt_print_tx_hwq_stats_cmn_tlv(const void *tag_buf, u16 tag_len, + struct debug_htt_stats_req *stats_req) +{ + const struct htt_tx_hwq_stats_cmn_tlv *htt_stats_buf = tag_buf; + u32 buf_len = ATH12K_HTT_STATS_BUF_SIZE; + u32 len = stats_req->buf_len; + u8 *buf = stats_req->buf; + + if (tag_len < sizeof(*htt_stats_buf)) + return; + + len += scnprintf(buf + len, buf_len - len, "HTT_TX_HWQ_STATS_CMN_TLV:\n"); + len += scnprintf(buf + len, buf_len - len, "mac_id = %u\n", + le32_to_cpu(htt_stats_buf->mac_id__hwq_id__word) & 0xFF); + len += scnprintf(buf + len, buf_len - len, "hwq_id = %u\n", + (le32_to_cpu(htt_stats_buf->mac_id__hwq_id__word) & 0xFF00) >> 8); + len += scnprintf(buf + len, buf_len - len, "xretry = %u\n", + le32_to_cpu(htt_stats_buf->xretry)); + len += scnprintf(buf + len, buf_len - len, "underrun_cnt = %u\n", + le32_to_cpu(htt_stats_buf->underrun_cnt)); + len += scnprintf(buf + len, buf_len - len, "flush_cnt = %u\n", + le32_to_cpu(htt_stats_buf->flush_cnt)); + len += scnprintf(buf + len, buf_len - len, "filt_cnt = %u\n", + le32_to_cpu(htt_stats_buf->filt_cnt)); + len += scnprintf(buf + len, buf_len - len, "null_mpdu_bmap = %u\n", + le32_to_cpu(htt_stats_buf->null_mpdu_bmap)); + len += scnprintf(buf + len, buf_len - len, "user_ack_failure = %u\n", + le32_to_cpu(htt_stats_buf->user_ack_failure)); + len += scnprintf(buf + len, buf_len - len, "ack_tlv_proc = %u\n", + le32_to_cpu(htt_stats_buf->ack_tlv_proc)); + len += scnprintf(buf + len, buf_len - len, "sched_id_proc = %u\n", + le32_to_cpu(htt_stats_buf->sched_id_proc)); + len += scnprintf(buf + len, buf_len - len, "null_mpdu_tx_count = %u\n", + le32_to_cpu(htt_stats_buf->null_mpdu_tx_count)); + len += scnprintf(buf + len, buf_len - len, "mpdu_bmap_not_recvd = %u\n", + le32_to_cpu(htt_stats_buf->mpdu_bmap_not_recvd)); + len += scnprintf(buf + len, buf_len - len, "num_bar = %u\n", + le32_to_cpu(htt_stats_buf->num_bar)); + len += scnprintf(buf + len, buf_len - len, "rts = %u\n", + le32_to_cpu(htt_stats_buf->rts)); + len += scnprintf(buf + len, buf_len - len, "cts2self = %u\n", + le32_to_cpu(htt_stats_buf->cts2self)); + len += scnprintf(buf + len, buf_len - len, "qos_null = %u\n", + le32_to_cpu(htt_stats_buf->qos_null)); + len += scnprintf(buf + len, buf_len - len, "mpdu_tried_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mpdu_tried_cnt)); + len += scnprintf(buf + len, buf_len - len, "mpdu_queued_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mpdu_queued_cnt)); + len += scnprintf(buf + len, buf_len - len, "mpdu_ack_fail_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mpdu_ack_fail_cnt)); + len += scnprintf(buf + len, buf_len - len, "mpdu_filt_cnt = %u\n", + le32_to_cpu(htt_stats_buf->mpdu_filt_cnt)); + len += scnprintf(buf + len, buf_len - len, "false_mpdu_ack_count = %u\n", + le32_to_cpu(htt_stats_buf->false_mpdu_ack_count)); + len += scnprintf(buf + len, buf_len - len, "txq_timeout = %u\n", + le32_to_cpu(htt_stats_buf->txq_timeout)); + + stats_req->buf_len = len; +} + static int ath12k_dbg_htt_ext_stats_parse(struct ath12k_base *ab, u16 tag, u16 len, const void *tag_buf, void *user_data) @@ -5690,6 +5875,9 @@ static int ath12k_dbg_htt_ext_stats_parse(struct ath12k_base *ab, case HTT_STATS_SFM_CLIENT_USER_TAG: ath12k_htt_print_sfm_client_user_tlv(tag_buf, len, stats_req); break; + case HTT_STATS_RX_PDEV_FW_STATS_TAG: + ath12k_htt_print_rx_pdev_fw_stats_tlv(tag_buf, len, stats_req); + break; case HTT_STATS_TX_PDEV_MU_MIMO_STATS_TAG: ath12k_htt_print_tx_pdev_mu_mimo_sch_stats_tlv(tag_buf, len, stats_req); break; @@ -5833,6 +6021,9 @@ static int ath12k_dbg_htt_ext_stats_parse(struct ath12k_base *ab, case HTT_STATS_PDEV_RTT_TBR_CMD_RESULT_STATS_TAG: ath12k_htt_print_pdev_rtt_tbr_cmd_res_stats_tlv(tag_buf, len, stats_req); break; + case HTT_STATS_TX_HWQ_CMN_TAG: + ath12k_htt_print_tx_hwq_stats_cmn_tlv(tag_buf, len, stats_req); + break; default: break; } diff --git a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h index 9bd3a632b002d..bfabe6500d44d 100644 --- a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h +++ b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h @@ -1,12 +1,14 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef DEBUG_HTT_STATS_H #define DEBUG_HTT_STATS_H +#include "dp_htt.h" + #define ATH12K_HTT_STATS_BUF_SIZE (1024 * 512) #define ATH12K_HTT_STATS_COOKIE_LSB GENMASK_ULL(31, 0) #define ATH12K_HTT_STATS_COOKIE_MSB GENMASK_ULL(63, 32) @@ -125,6 +127,8 @@ struct ath12k_htt_extd_stats_msg { enum ath12k_dbg_htt_ext_stats_type { ATH12K_DBG_HTT_EXT_STATS_RESET = 0, ATH12K_DBG_HTT_EXT_STATS_PDEV_TX = 1, + ATH12K_DBG_HTT_EXT_STATS_PDEV_RX = 2, + ATH12K_DBG_HTT_EXT_STATS_PDEV_TX_HWQ = 3, ATH12K_DBG_HTT_EXT_STATS_PDEV_TX_SCHED = 4, ATH12K_DBG_HTT_EXT_STATS_PDEV_ERROR = 5, ATH12K_DBG_HTT_EXT_STATS_PDEV_TQM = 6, @@ -171,6 +175,7 @@ enum ath12k_dbg_htt_tlv_tag { HTT_STATS_TX_PDEV_SIFS_TAG = 2, HTT_STATS_TX_PDEV_FLUSH_TAG = 3, HTT_STATS_STRING_TAG = 5, + HTT_STATS_TX_HWQ_CMN_TAG = 6, HTT_STATS_TX_TQM_GEN_MPDU_TAG = 11, HTT_STATS_TX_TQM_LIST_MPDU_TAG = 12, HTT_STATS_TX_TQM_LIST_MPDU_CNT_TAG = 13, @@ -186,6 +191,7 @@ enum ath12k_dbg_htt_tlv_tag { HTT_STATS_TX_PDEV_MU_MIMO_STATS_TAG = 25, HTT_STATS_SFM_CMN_TAG = 26, HTT_STATS_SRING_STATS_TAG = 27, + HTT_STATS_RX_PDEV_FW_STATS_TAG = 28, HTT_STATS_TX_PDEV_RATE_STATS_TAG = 34, HTT_STATS_RX_PDEV_RATE_STATS_TAG = 35, HTT_STATS_TX_PDEV_SCHEDULER_TXQ_STATS_TAG = 36, @@ -2073,4 +2079,81 @@ struct ath12k_htt_stats_pdev_rtt_tbr_cmd_result_stats_tlv { __le32 mu_res[ATH12K_HTT_FTYPE_MAX][ATH12K_HTT_MAX_SCH_CMD_RESULT]; } __packed; +struct htt_rx_pdev_fw_stats_tlv { + __le32 mac_id__word; + __le32 ppdu_recvd; + __le32 mpdu_cnt_fcs_ok; + __le32 mpdu_cnt_fcs_err; + __le32 tcp_msdu_cnt; + __le32 tcp_ack_msdu_cnt; + __le32 udp_msdu_cnt; + __le32 other_msdu_cnt; + __le32 fw_ring_mpdu_ind; + __le32 fw_ring_mgmt_subtype[ATH12K_HTT_STATS_SUBTYPE_MAX]; + __le32 fw_ring_ctrl_subtype[ATH12K_HTT_STATS_SUBTYPE_MAX]; + __le32 fw_ring_mcast_data_msdu; + __le32 fw_ring_bcast_data_msdu; + __le32 fw_ring_ucast_data_msdu; + __le32 fw_ring_null_data_msdu; + __le32 fw_ring_mpdu_drop; + __le32 ofld_local_data_ind_cnt; + __le32 ofld_local_data_buf_recycle_cnt; + __le32 drx_local_data_ind_cnt; + __le32 drx_local_data_buf_recycle_cnt; + __le32 local_nondata_ind_cnt; + __le32 local_nondata_buf_recycle_cnt; + __le32 fw_status_buf_ring_refill_cnt; + __le32 fw_status_buf_ring_empty_cnt; + __le32 fw_pkt_buf_ring_refill_cnt; + __le32 fw_pkt_buf_ring_empty_cnt; + __le32 fw_link_buf_ring_refill_cnt; + __le32 fw_link_buf_ring_empty_cnt; + __le32 host_pkt_buf_ring_refill_cnt; + __le32 host_pkt_buf_ring_empty_cnt; + __le32 mon_pkt_buf_ring_refill_cnt; + __le32 mon_pkt_buf_ring_empty_cnt; + __le32 mon_status_buf_ring_refill_cnt; + __le32 mon_status_buf_ring_empty_cnt; + __le32 mon_desc_buf_ring_refill_cnt; + __le32 mon_desc_buf_ring_empty_cnt; + __le32 mon_dest_ring_update_cnt; + __le32 mon_dest_ring_full_cnt; + __le32 rx_suspend_cnt; + __le32 rx_suspend_fail_cnt; + __le32 rx_resume_cnt; + __le32 rx_resume_fail_cnt; + __le32 rx_ring_switch_cnt; + __le32 rx_ring_restore_cnt; + __le32 rx_flush_cnt; + __le32 rx_recovery_reset_cnt; + __le32 rx_lwm_prom_filter_dis; + __le32 rx_hwm_prom_filter_en; + __le32 bytes_received_low_32; + __le32 bytes_received_high_32; +} __packed; + +struct htt_tx_hwq_stats_cmn_tlv { + __le32 mac_id__hwq_id__word; + __le32 xretry; + __le32 underrun_cnt; + __le32 flush_cnt; + __le32 filt_cnt; + __le32 null_mpdu_bmap; + __le32 user_ack_failure; + __le32 ack_tlv_proc; + __le32 sched_id_proc; + __le32 null_mpdu_tx_count; + __le32 mpdu_bmap_not_recvd; + __le32 num_bar; + __le32 rts; + __le32 cts2self; + __le32 qos_null; + __le32 mpdu_tried_cnt; + __le32 mpdu_queued_cnt; + __le32 mpdu_ack_fail_cnt; + __le32 mpdu_filt_cnt; + __le32 false_mpdu_ack_count; + __le32 txq_timeout; +} __packed; + #endif diff --git a/drivers/net/wireless/ath/ath12k/debugfs_sta.c b/drivers/net/wireless/ath/ath12k/debugfs_sta.c index 5bd2bf4c9dac3..585c40bd29517 100644 --- a/drivers/net/wireless/ath/ath12k/debugfs_sta.c +++ b/drivers/net/wireless/ath/ath12k/debugfs_sta.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* - * Copyright (c) 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -11,6 +11,7 @@ #include "debug.h" #include "debugfs_htt_stats.h" #include "debugfs.h" +#include "dp_cmn.h" static u32 ath12k_dbg_sta_dump_rate_stats(u8 *buf, u32 offset, const int size, @@ -144,40 +145,40 @@ static ssize_t ath12k_dbg_sta_dump_rx_stats(struct file *file, const int size = ATH12K_STA_RX_STATS_BUF_SIZE; struct ath12k_hw *ah = ahsta->ahvif->ah; struct ath12k_rx_peer_stats *rx_stats; + struct ath12k_dp_link_peer *link_peer; struct ath12k_link_sta *arsta; u8 link_id = link_sta->link_id; int len = 0, i, ret = 0; + struct ath12k_dp *dp; bool he_rates_avail; struct ath12k *ar; - wiphy_lock(ah->hw->wiphy); + guard(wiphy)(ah->hw->wiphy); - if (!(BIT(link_id) & ahsta->links_map)) { - wiphy_unlock(ah->hw->wiphy); + if (!(BIT(link_id) & ahsta->links_map)) return -ENOENT; - } arsta = wiphy_dereference(ah->hw->wiphy, ahsta->link[link_id]); - if (!arsta || !arsta->arvif->ar) { - wiphy_unlock(ah->hw->wiphy); + if (!arsta || !arsta->arvif->ar) return -ENOENT; - } ar = arsta->arvif->ar; u8 *buf __free(kfree) = kzalloc(size, GFP_KERNEL); - if (!buf) { - ret = -ENOENT; - goto out; - } + if (!buf) + return -ENOMEM; - spin_lock_bh(&ar->ab->base_lock); + dp = ath12k_ab_to_dp(ar->ab); - rx_stats = arsta->rx_stats; - if (!rx_stats) { - ret = -ENOENT; - goto unlock; - } + guard(spinlock_bh)(&dp->dp_lock); + + link_peer = ath12k_dp_link_peer_find_by_addr(dp, arsta->addr); + if (!link_peer) + return -ENOENT; + + rx_stats = link_peer->peer_stats.rx_stats; + if (!rx_stats) + return -ENOENT; len += scnprintf(buf + len, size - len, "RX peer stats:\n\n"); len += scnprintf(buf + len, size - len, "Num of MSDUs: %llu\n", @@ -237,13 +238,8 @@ static ssize_t ath12k_dbg_sta_dump_rx_stats(struct file *file, len += ath12k_dbg_sta_dump_rate_stats(buf, len, size, he_rates_avail, &rx_stats->byte_stats); -unlock: - spin_unlock_bh(&ar->ab->base_lock); - if (len) ret = simple_read_from_buffer(user_buf, count, ppos, buf, len); -out: - wiphy_unlock(ah->hw->wiphy); return ret; } @@ -261,10 +257,9 @@ static ssize_t ath12k_dbg_sta_reset_rx_stats(struct file *file, struct ieee80211_link_sta *link_sta = file->private_data; struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(link_sta->sta); struct ath12k_hw *ah = ahsta->ahvif->ah; - struct ath12k_rx_peer_stats *rx_stats; - struct ath12k_link_sta *arsta; u8 link_id = link_sta->link_id; - struct ath12k *ar; + struct ath12k_link_sta *arsta; + struct ath12k_dp *dp; bool reset; int ret; @@ -288,19 +283,9 @@ static ssize_t ath12k_dbg_sta_reset_rx_stats(struct file *file, goto out; } - ar = arsta->arvif->ar; - - spin_lock_bh(&ar->ab->base_lock); - - rx_stats = arsta->rx_stats; - if (!rx_stats) { - spin_unlock_bh(&ar->ab->base_lock); - ret = -ENOENT; - goto out; - } + dp = ath12k_ab_to_dp(arsta->arvif->ar->ab); - memset(rx_stats, 0, sizeof(*rx_stats)); - spin_unlock_bh(&ar->ab->base_lock); + ath12k_dp_link_peer_reset_rx_stats(dp, arsta->addr); ret = count; out: @@ -335,3 +320,4 @@ void ath12k_debugfs_link_sta_op_add(struct ieee80211_hw *hw, &fops_reset_rx_stats); } } +EXPORT_SYMBOL(ath12k_debugfs_link_sta_op_add); diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c index 4a54b8c353111..ab54c8a84d3e0 100644 --- a/drivers/net/wireless/ath/ath12k/dp.c +++ b/drivers/net/wireless/ath/ath12k/dp.c @@ -1,63 +1,58 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include "core.h" #include "dp_tx.h" -#include "hal_tx.h" #include "hif.h" +#include "hal.h" #include "debug.h" -#include "dp_rx.h" #include "peer.h" -#include "dp_mon.h" +#include "dp_cmn.h" enum ath12k_dp_desc_type { ATH12K_DP_TX_DESC, ATH12K_DP_RX_DESC, }; -static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab, - struct sk_buff *skb) -{ - dev_kfree_skb_any(skb); -} - void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr) { struct ath12k_base *ab = ar->ab; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); /* TODO: Any other peer specific DP cleanup */ - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find(ab, vdev_id, addr); - if (!peer) { + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr); + if (!peer || !peer->dp_peer) { ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n", addr, vdev_id); - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return; } if (!peer->primary_link) { - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return; } ath12k_dp_rx_peer_tid_cleanup(ar, peer); - crypto_free_shash(peer->tfm_mmic); - peer->dp_setup_done = false; - spin_unlock_bh(&ab->base_lock); + crypto_free_shash(peer->dp_peer->tfm_mmic); + peer->dp_peer->dp_setup_done = false; + spin_unlock_bh(&dp->dp_lock); } int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) { struct ath12k_base *ab = ar->ab; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; u32 reo_dest; int ret = 0, tid; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); /* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */ reo_dest = ar->dp.mac_id + 1; @@ -92,19 +87,19 @@ int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) return 0; peer_clean: - spin_lock_bh(&ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ab, vdev_id, addr); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr); if (!peer) { ath12k_warn(ab, "failed to find the peer to del rx tid\n"); - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return -ENOENT; } for (tid--; tid >= 0; tid--) - ath12k_dp_rx_peer_tid_delete(ar, peer, tid); + ath12k_dp_arch_rx_peer_tid_delete(dp, peer, tid); - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return ret; } @@ -147,7 +142,7 @@ static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab, grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0]; ring_num = 0; } else { - map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map; + map = ab->hal.tcl_to_wbm_rbm_map; for (i = 0; i < ab->hw_params->max_tx_ring; i++) { if (ring_num == map[i].wbm_ring_num) { ring_num = i; @@ -338,50 +333,6 @@ int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring, return 0; } -static -u32 ath12k_dp_tx_get_vdev_bank_config(struct ath12k_base *ab, - struct ath12k_link_vif *arvif) -{ - u32 bank_config = 0; - struct ath12k_vif *ahvif = arvif->ahvif; - - /* Only valid for raw frames with HW crypto enabled. - * With SW crypto, mac80211 sets key per packet - */ - if (ahvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW && - test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags)) - bank_config |= - u32_encode_bits(ath12k_dp_tx_get_encrypt_type(ahvif->key_cipher), - HAL_TX_BANK_CONFIG_ENCRYPT_TYPE); - - bank_config |= u32_encode_bits(ahvif->tx_encap_type, - HAL_TX_BANK_CONFIG_ENCAP_TYPE); - bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) | - u32_encode_bits(0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) | - u32_encode_bits(0, HAL_TX_BANK_CONFIG_EPD); - - /* only valid if idx_lookup_override is not set in tcl_data_cmd */ - if (ahvif->vdev_type == WMI_VDEV_TYPE_STA) - bank_config |= u32_encode_bits(1, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN); - else - bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN); - - bank_config |= u32_encode_bits(arvif->hal_addr_search_flags & HAL_TX_ADDRX_EN, - HAL_TX_BANK_CONFIG_ADDRX_EN) | - u32_encode_bits(!!(arvif->hal_addr_search_flags & - HAL_TX_ADDRY_EN), - HAL_TX_BANK_CONFIG_ADDRY_EN); - - bank_config |= u32_encode_bits(ieee80211_vif_is_mesh(ahvif->vif) ? 3 : 0, - HAL_TX_BANK_CONFIG_MESH_EN) | - u32_encode_bits(arvif->vdev_id_check_en, - HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN); - - bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID); - - return bank_config; -} - static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, struct ath12k_link_vif *arvif, struct ath12k_dp *dp) @@ -392,7 +343,7 @@ static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, bool configure_register = false; /* convert vdev params into hal_tx_bank_config */ - bank_config = ath12k_dp_tx_get_vdev_bank_config(ab, arvif); + bank_config = ath12k_dp_arch_tx_get_vdev_bank_config(dp, arvif); spin_lock_bh(&dp->tx_bank_lock); /* TODO: implement using idr kernel framework*/ @@ -424,7 +375,8 @@ static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, spin_unlock_bh(&dp->tx_bank_lock); if (configure_register) - ath12k_hal_tx_configure_bank_register(ab, bank_config, bank_id); + ath12k_hal_tx_configure_bank_register(ab, + bank_config, bank_id); ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u", bank_id, bank_config, dp->bank_profiles[bank_id].bank_config, @@ -442,7 +394,7 @@ void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id) static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); kfree(dp->bank_profiles); dp->bank_profiles = NULL; @@ -450,7 +402,7 @@ static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab) static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); u32 num_tcl_banks = ab->hw_params->num_tcl_banks; int i; @@ -473,7 +425,7 @@ static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab) static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int i; ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring); @@ -490,7 +442,7 @@ static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab) static int ath12k_dp_srng_common_setup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); const struct ath12k_hal_tcl_to_wbm_rbm_map *map; struct hal_srng *srng; int i, ret, tx_comp_ring_num; @@ -506,7 +458,7 @@ static int ath12k_dp_srng_common_setup(struct ath12k_base *ab) } for (i = 0; i < ab->hw_params->max_tx_ring; i++) { - map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map; + map = ab->hal.tcl_to_wbm_rbm_map; tx_comp_ring_num = map[i].wbm_ring_num; ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring, @@ -596,7 +548,7 @@ static int ath12k_dp_srng_common_setup(struct ath12k_base *ab) static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; int i; @@ -616,7 +568,7 @@ static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab, u32 n_link_desc, u32 last_bank_sz) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks; struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; u32 n_entries_per_buf; @@ -659,7 +611,7 @@ static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab, paddr = link_desc_banks[i].paddr; while (n_entries) { cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); - ath12k_hal_set_link_desc_addr(scatter_buf, cookie, + ath12k_hal_set_link_desc_addr(dp->hal, scatter_buf, cookie, paddr, rbm); n_entries--; paddr += HAL_LINK_DESC_SIZE; @@ -710,7 +662,7 @@ static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab, int n_link_desc_bank, int last_bank_sz) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int i; int ret = 0; int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH; @@ -758,7 +710,7 @@ void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab, static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); u32 n_mpdu_link_desc, n_mpdu_queue_desc; u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc; int ret = 0; @@ -797,6 +749,7 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab, u32 ring_type, struct hal_srng *srng, u32 n_link_desc) { + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); u32 tot_mem_sz; u32 n_link_desc_bank, last_bank_sz; u32 entry_sz, align_bytes, n_entries; @@ -804,7 +757,7 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab, u32 paddr; int i, ret; u32 cookie; - enum hal_rx_buf_return_buf_manager rbm = ab->dp.idle_link_rbm; + enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm; tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE; tot_mem_sz += HAL_LINK_DESC_ALIGN; @@ -865,7 +818,8 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab, while (n_entries && (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) { cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); - ath12k_hal_set_link_desc_addr(desc, cookie, paddr, rbm); + ath12k_hal_set_link_desc_addr(dp->hal, desc, cookie, paddr, + rbm); n_entries--; paddr += HAL_LINK_DESC_SIZE; } @@ -883,133 +837,18 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab, return ret; } -int ath12k_dp_service_srng(struct ath12k_base *ab, - struct ath12k_ext_irq_grp *irq_grp, - int budget) +void ath12k_dp_pdev_free(struct ath12k_base *ab) { - struct napi_struct *napi = &irq_grp->napi; - int grp_id = irq_grp->grp_id; - int work_done = 0; - int i = 0, j; - int tot_work_done = 0; - enum dp_monitor_mode monitor_mode; - u8 ring_mask; - - if (ab->hw_params->ring_mask->tx[grp_id]) { - i = fls(ab->hw_params->ring_mask->tx[grp_id]) - 1; - ath12k_dp_tx_completion_handler(ab, i); - } - - if (ab->hw_params->ring_mask->rx_err[grp_id]) { - work_done = ath12k_dp_rx_process_err(ab, napi, budget); - budget -= work_done; - tot_work_done += work_done; - if (budget <= 0) - goto done; - } - - if (ab->hw_params->ring_mask->rx_wbm_rel[grp_id]) { - work_done = ath12k_dp_rx_process_wbm_err(ab, - napi, - budget); - budget -= work_done; - tot_work_done += work_done; - - if (budget <= 0) - goto done; - } - - if (ab->hw_params->ring_mask->rx[grp_id]) { - i = fls(ab->hw_params->ring_mask->rx[grp_id]) - 1; - work_done = ath12k_dp_rx_process(ab, i, napi, - budget); - budget -= work_done; - tot_work_done += work_done; - if (budget <= 0) - goto done; - } - - if (ab->hw_params->ring_mask->rx_mon_status[grp_id]) { - ring_mask = ab->hw_params->ring_mask->rx_mon_status[grp_id]; - for (i = 0; i < ab->num_radios; i++) { - for (j = 0; j < ab->hw_params->num_rxdma_per_pdev; j++) { - int id = i * ab->hw_params->num_rxdma_per_pdev + j; - - if (ring_mask & BIT(id)) { - work_done = - ath12k_dp_mon_process_ring(ab, id, napi, budget, - 0); - budget -= work_done; - tot_work_done += work_done; - if (budget <= 0) - goto done; - } - } - } - } - - if (ab->hw_params->ring_mask->rx_mon_dest[grp_id]) { - monitor_mode = ATH12K_DP_RX_MONITOR_MODE; - ring_mask = ab->hw_params->ring_mask->rx_mon_dest[grp_id]; - for (i = 0; i < ab->num_radios; i++) { - for (j = 0; j < ab->hw_params->num_rxdma_per_pdev; j++) { - int id = i * ab->hw_params->num_rxdma_per_pdev + j; - - if (ring_mask & BIT(id)) { - work_done = - ath12k_dp_mon_process_ring(ab, id, napi, budget, - monitor_mode); - budget -= work_done; - tot_work_done += work_done; - - if (budget <= 0) - goto done; - } - } - } - } - - if (ab->hw_params->ring_mask->tx_mon_dest[grp_id]) { - monitor_mode = ATH12K_DP_TX_MONITOR_MODE; - ring_mask = ab->hw_params->ring_mask->tx_mon_dest[grp_id]; - for (i = 0; i < ab->num_radios; i++) { - for (j = 0; j < ab->hw_params->num_rxdma_per_pdev; j++) { - int id = i * ab->hw_params->num_rxdma_per_pdev + j; - - if (ring_mask & BIT(id)) { - work_done = - ath12k_dp_mon_process_ring(ab, id, napi, budget, - monitor_mode); - budget -= work_done; - tot_work_done += work_done; - - if (budget <= 0) - goto done; - } - } - } - } - - if (ab->hw_params->ring_mask->reo_status[grp_id]) - ath12k_dp_rx_process_reo_status(ab); - - if (ab->hw_params->ring_mask->host2rxdma[grp_id]) { - struct ath12k_dp *dp = &ab->dp; - struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; - LIST_HEAD(list); + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k *ar; + int i; - ath12k_dp_rx_bufs_replenish(ab, rx_ring, &list, 0); + for (i = 0; i < ab->num_radios; i++) { + ar = ab->pdevs[i].ar; + rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], NULL); } - /* TODO: Implement handler for other interrupts */ - -done: - return tot_work_done; -} - -void ath12k_dp_pdev_free(struct ath12k_base *ab) -{ - int i; + synchronize_rcu(); for (i = 0; i < ab->num_radios; i++) ath12k_dp_rx_pdev_free(ab, i); @@ -1025,31 +864,10 @@ void ath12k_dp_pdev_pre_alloc(struct ath12k *ar) /* TODO: Add any RXDMA setup required per pdev */ } -bool ath12k_dp_wmask_compaction_rx_tlv_supported(struct ath12k_base *ab) -{ - if (test_bit(WMI_TLV_SERVICE_WMSK_COMPACTION_RX_TLVS, ab->wmi_ab.svc_map) && - ab->hw_params->hal_ops->rxdma_ring_wmask_rx_mpdu_start && - ab->hw_params->hal_ops->rxdma_ring_wmask_rx_msdu_end && - ab->hw_params->hal_ops->get_hal_rx_compact_ops) { - return true; - } - return false; -} - -void ath12k_dp_hal_rx_desc_init(struct ath12k_base *ab) -{ - if (ath12k_dp_wmask_compaction_rx_tlv_supported(ab)) { - /* RX TLVS compaction is supported, hence change the hal_rx_ops - * to compact hal_rx_ops. - */ - ab->hal_rx_ops = ab->hw_params->hal_ops->get_hal_rx_compact_ops(); - } - ab->hal.hal_desc_sz = - ab->hal_rx_ops->rx_desc_get_desc_size(); -} - int ath12k_dp_pdev_alloc(struct ath12k_base *ab) { + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_pdev_dp *dp_pdev; struct ath12k *ar; int ret; int i; @@ -1061,6 +879,14 @@ int ath12k_dp_pdev_alloc(struct ath12k_base *ab) /* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */ for (i = 0; i < ab->num_radios; i++) { ar = ab->pdevs[i].ar; + + dp_pdev = &ar->dp; + + dp_pdev->hw = ar->ah->hw; + dp_pdev->dp = dp; + dp_pdev->hw_link_id = ar->hw_link_id; + dp_pdev->dp_hw = &ar->ah->dp_hw; + ret = ath12k_dp_rx_pdev_alloc(ab, i); if (ret) { ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n", @@ -1074,6 +900,11 @@ int ath12k_dp_pdev_alloc(struct ath12k_base *ab) } } + for (i = 0; i < ab->num_radios; i++) { + ar = ab->pdevs[i].ar; + rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], &ar->dp); + } + return 0; err: ath12k_dp_pdev_free(ab); @@ -1081,40 +912,23 @@ int ath12k_dp_pdev_alloc(struct ath12k_base *ab) return ret; } -int ath12k_dp_htt_connect(struct ath12k_dp *dp) +static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif) { - struct ath12k_htc_svc_conn_req conn_req = {}; - struct ath12k_htc_svc_conn_resp conn_resp = {}; - int status; - - conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete; - conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler; - - /* connect to control service */ - conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG; - - status = ath12k_htc_connect_service(&dp->ab->htc, &conn_req, - &conn_resp); - - if (status) - return status; - - dp->eid = conn_resp.eid; + u8 link_id = arvif->link_id; + struct ath12k_vif *ahvif = arvif->ahvif; + struct ath12k_dp_link_vif *dp_link_vif; - return 0; -} + dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id); -static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif) -{ switch (arvif->ahvif->vdev_type) { case WMI_VDEV_TYPE_STA: - arvif->hal_addr_search_flags = HAL_TX_ADDRY_EN; - arvif->search_type = HAL_TX_ADDR_SEARCH_INDEX; + dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRY_EN; + dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; break; case WMI_VDEV_TYPE_AP: case WMI_VDEV_TYPE_IBSS: - arvif->hal_addr_search_flags = HAL_TX_ADDRX_EN; - arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; + dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRX_EN; + dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; break; case WMI_VDEV_TYPE_MONITOR: default: @@ -1125,22 +939,29 @@ static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif) void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif) { struct ath12k_base *ab = ar->ab; + struct ath12k_vif *ahvif = arvif->ahvif; + u8 link_id = arvif->link_id; + int bank_id; + struct ath12k_dp_link_vif *dp_link_vif; + + dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id); - arvif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | - u32_encode_bits(arvif->vdev_id, - HTT_TCL_META_DATA_VDEV_ID) | - u32_encode_bits(ar->pdev->pdev_id, - HTT_TCL_META_DATA_PDEV_ID); + dp_link_vif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | + u32_encode_bits(arvif->vdev_id, + HTT_TCL_META_DATA_VDEV_ID) | + u32_encode_bits(ar->pdev->pdev_id, + HTT_TCL_META_DATA_PDEV_ID); /* set HTT extension valid bit to 0 by default */ - arvif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT; + dp_link_vif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT; ath12k_dp_update_vdev_search(arvif); - arvif->vdev_id_check_en = true; - arvif->bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, &ab->dp); + dp_link_vif->vdev_id_check_en = true; + bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, ath12k_ab_to_dp(ab)); + dp_link_vif->bank_id = bank_id; /* TODO: error path for bank id failure */ - if (arvif->bank_id == DP_INVALID_BANK_ID) { + if (bank_id == DP_INVALID_BANK_ID) { ath12k_err(ar->ab, "Failed to initialize DP TX Banks"); return; } @@ -1150,7 +971,7 @@ static void ath12k_dp_cc_cleanup(struct ath12k_base *ab) { struct ath12k_rx_desc_info *desc_info; struct ath12k_tx_desc_info *tx_desc_info, *tmp1; - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct ath12k_skb_cb *skb_cb; struct sk_buff *skb; struct ath12k *ar; @@ -1273,15 +1094,13 @@ static void ath12k_dp_cc_cleanup(struct ath12k_base *ab) static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); if (!ab->hw_params->reoq_lut_support) return; if (dp->reoq_lut.vaddr_unaligned) { - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_REO_REG + - HAL_REO1_QDESC_LUT_BASE0(ab), 0); + ath12k_hal_write_reoq_lut_addr(ab, 0); dma_free_coherent(ab->dev, dp->reoq_lut.size, dp->reoq_lut.vaddr_unaligned, dp->reoq_lut.paddr_unaligned); @@ -1289,9 +1108,7 @@ static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab) } if (dp->ml_reoq_lut.vaddr_unaligned) { - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_REO_REG + - HAL_REO1_QDESC_LUT_BASE1(ab), 0); + ath12k_hal_write_ml_reoq_lut_addr(ab, 0); dma_free_coherent(ab->dev, dp->ml_reoq_lut.size, dp->ml_reoq_lut.vaddr_unaligned, dp->ml_reoq_lut.paddr_unaligned); @@ -1299,11 +1116,13 @@ static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab) } } -void ath12k_dp_free(struct ath12k_base *ab) +static void ath12k_dp_cleanup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int i; + ath12k_dp_link_peer_rhash_tbl_destroy(dp); + if (!dp->ab) return; @@ -1324,60 +1143,6 @@ void ath12k_dp_free(struct ath12k_base *ab) ath12k_dp_rx_free(ab); /* Deinit any SOC level resource */ - dp->ab = NULL; -} - -void ath12k_dp_cc_config(struct ath12k_base *ab) -{ - u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; - u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG; - u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG; - u32 val = 0; - - if (ath12k_ftm_mode) - return; - - ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG0(ab), cmem_base); - - val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB, - HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | - u32_encode_bits(ATH12K_CC_PPT_MSB, - HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) | - u32_encode_bits(ATH12K_CC_SPT_MSB, - HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) | - u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ALIGN) | - u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ENABLE) | - u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE); - - ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG1(ab), val); - - /* Enable HW CC for WBM */ - ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG0, cmem_base); - - val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB, - HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | - u32_encode_bits(ATH12K_CC_PPT_MSB, - HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) | - u32_encode_bits(ATH12K_CC_SPT_MSB, - HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) | - u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ALIGN); - - ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG1, val); - - /* Enable conversion complete indication */ - val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2); - val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) | - u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) | - u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN); - - ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2, val); - - /* Enable Cookie conversion for WBM2SW Rings */ - val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG); - val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) | - ab->hw_params->hal_params->wbm2sw_cc_enable; - - ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, val); } static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx) @@ -1385,26 +1150,23 @@ static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx) return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx; } -static inline void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_base *ab, - u16 ppt_idx, u16 spt_idx) +static void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp *dp, + u16 ppt_idx, u16 spt_idx) { - struct ath12k_dp *dp = &ab->dp; - return dp->spt_info[ppt_idx].vaddr + spt_idx; } -struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab, +struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp, u32 cookie) { - struct ath12k_dp *dp = &ab->dp; struct ath12k_rx_desc_info **desc_addr_ptr; u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx; ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT); - start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(ab); - end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(ab); + start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(dp->ab); + end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(dp->ab); if (ppt_idx < start_ppt_idx || ppt_idx >= end_ppt_idx || @@ -1412,12 +1174,13 @@ struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab, return NULL; ppt_idx = ppt_idx - dp->rx_ppt_base; - desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); + desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx); return *desc_addr_ptr; } +EXPORT_SYMBOL(ath12k_dp_get_rx_desc); -struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab, +struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp, u32 cookie) { struct ath12k_tx_desc_info **desc_addr_ptr; @@ -1428,21 +1191,22 @@ struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab, start_ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET; end_ppt_idx = start_ppt_idx + - (ATH12K_TX_SPT_PAGES_PER_POOL(ab) * ATH12K_HW_MAX_QUEUES); + (ATH12K_TX_SPT_PAGES_PER_POOL(dp->ab) * ATH12K_HW_MAX_QUEUES); if (ppt_idx < start_ppt_idx || ppt_idx >= end_ppt_idx || spt_idx > ATH12K_MAX_SPT_ENTRIES) return NULL; - desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); + desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx); return *desc_addr_ptr; } +EXPORT_SYMBOL(ath12k_dp_get_tx_desc); static int ath12k_dp_cc_desc_init(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr; struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr; u32 num_rx_spt_pages = ATH12K_NUM_RX_SPT_PAGES(ab); @@ -1482,7 +1246,7 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab) list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list); /* Update descriptor VA in SPT */ - rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, j); + rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j); *rx_desc_addr = &rx_descs[j]; } } @@ -1521,7 +1285,7 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab) /* Update descriptor VA in SPT */ tx_desc_addr = - ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, j); + ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j); *tx_desc_addr = &tx_descs[j]; } } @@ -1571,7 +1335,7 @@ void ath12k_dp_partner_cc_init(struct ath12k_base *ab) if (ag->ab[i] == ab) continue; - ath12k_dp_cmem_init(ab, &ag->ab[i]->dp, ATH12K_DP_RX_DESC); + ath12k_dp_cmem_init(ab, ath12k_ab_to_dp(ag->ab[i]), ATH12K_DP_RX_DESC); } } @@ -1582,7 +1346,7 @@ static u32 ath12k_dp_get_num_spt_pages(struct ath12k_base *ab) static int ath12k_dp_cc_init(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int i, ret = 0; INIT_LIST_HEAD(&dp->rx_desc_free_list); @@ -1668,8 +1432,7 @@ static int ath12k_dp_alloc_reoq_lut(struct ath12k_base *ab, static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; - u32 val; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int ret; if (!ab->hw_params->reoq_lut_support) @@ -1697,50 +1460,24 @@ static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab) * register only */ - ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), - dp->reoq_lut.paddr >> 8); - - ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE1(ab), - dp->ml_reoq_lut.paddr >> 8); - - val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_ADDR(ab)); - - ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_ADDR(ab), - val | HAL_REO_QDESC_ADDR_READ_LUT_ENABLE); - - ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_MAX_PEERID(ab), - HAL_REO_QDESC_MAX_PEERID); + ath12k_hal_write_reoq_lut_addr(ab, dp->reoq_lut.paddr >> 8); + ath12k_hal_write_ml_reoq_lut_addr(ab, dp->ml_reoq_lut.paddr >> 8); + ath12k_hal_reoq_lut_addr_read_enable(ab); + ath12k_hal_reoq_lut_set_max_peerid(ab); return 0; } -static enum hal_rx_buf_return_buf_manager -ath12k_dp_get_idle_link_rbm(struct ath12k_base *ab) +static int ath12k_dp_setup(struct ath12k_base *ab) { - switch (ab->device_id) { - case 0: - return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST; - case 1: - return HAL_RX_BUF_RBM_WBM_DEV1_IDLE_DESC_LIST; - case 2: - return HAL_RX_BUF_RBM_WBM_DEV2_IDLE_DESC_LIST; - default: - ath12k_warn(ab, "invalid %d device id, so choose default rbm\n", - ab->device_id); - WARN_ON(1); - return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST; - } -} - -int ath12k_dp_alloc(struct ath12k_base *ab) -{ - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp; struct hal_srng *srng = NULL; size_t size = 0; u32 n_link_desc = 0; int ret; int i; + dp = ath12k_ab_to_dp(ab); dp->ab = ab; INIT_LIST_HEAD(&dp->reo_cmd_list); @@ -1749,13 +1486,25 @@ int ath12k_dp_alloc(struct ath12k_base *ab) spin_lock_init(&dp->reo_cmd_lock); spin_lock_init(&dp->reo_rxq_flush_lock); + spin_lock_init(&dp->dp_lock); + INIT_LIST_HEAD(&dp->peers); + + mutex_init(&dp->link_peer_rhash_tbl_lock); + dp->reo_cmd_cache_flush_count = 0; - dp->idle_link_rbm = ath12k_dp_get_idle_link_rbm(ab); + dp->idle_link_rbm = + ath12k_hal_get_idle_link_rbm(&ab->hal, ab->device_id); + + ret = ath12k_dp_link_peer_rhash_tbl_init(dp); + if (ret) { + ath12k_warn(ab, "failed to init link_peer rhash table: %d\n", ret); + return ret; + } ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc); if (ret) { ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); - return ret; + goto rhash_destroy; } srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id]; @@ -1764,7 +1513,7 @@ int ath12k_dp_alloc(struct ath12k_base *ab) HAL_WBM_IDLE_LINK, srng, n_link_desc); if (ret) { ath12k_warn(ab, "failed to setup link desc: %d\n", ret); - return ret; + goto rhash_destroy; } ret = ath12k_dp_cc_init(ab); @@ -1783,7 +1532,7 @@ int ath12k_dp_alloc(struct ath12k_base *ab) if (ret) goto fail_dp_bank_profiles_cleanup; - size = sizeof(struct hal_wbm_release_ring_tx) * + size = ab->hal.hal_wbm_release_ring_tx_size * DP_TX_COMP_RING_SIZE(ab); ret = ath12k_dp_reoq_lut_setup(ab); @@ -1836,6 +1585,48 @@ int ath12k_dp_alloc(struct ath12k_base *ab) fail_link_desc_cleanup: ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); +rhash_destroy: + ath12k_dp_link_peer_rhash_tbl_destroy(dp); return ret; } + +void ath12k_dp_cmn_device_deinit(struct ath12k_dp *dp) +{ + ath12k_dp_cleanup(dp->ab); +} + +int ath12k_dp_cmn_device_init(struct ath12k_dp *dp) +{ + int ret; + + ret = ath12k_dp_setup(dp->ab); + if (ret) + return ret; + + return 0; +} + +void ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp *dp, + struct ath12k_hw_group *ag) +{ + struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; + + lockdep_assert_held(&ag->mutex); + + dp_hw_grp->dp[dp->device_id] = NULL; + + dp->ag = NULL; + dp->device_id = ATH12K_INVALID_DEVICE_ID; +} + +void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp, + struct ath12k_hw_group *ag) +{ + struct ath12k_base *ab = dp->ab; + struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; + + dp->ag = ag; + dp->device_id = ab->device_id; + dp_hw_grp->dp[dp->device_id] = dp; +} diff --git a/drivers/net/wireless/ath/ath12k/dp.h b/drivers/net/wireless/ath/ath12k/dp.h index 4ffec6ad7d8d9..f8cfc7bb29dd7 100644 --- a/drivers/net/wireless/ath/ath12k/dp.h +++ b/drivers/net/wireless/ath/ath12k/dp.h @@ -1,29 +1,35 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_DP_H #define ATH12K_DP_H -#include "hal_desc.h" -#include "hal_rx.h" #include "hw.h" +#include "dp_htt.h" +#include "dp_cmn.h" +#include #define MAX_RXDMA_PER_PDEV 2 struct ath12k_base; -struct ath12k_peer; +struct ath12k_dp_link_peer; struct ath12k_dp; struct ath12k_vif; struct ath12k_link_vif; -struct hal_tcl_status_ring; struct ath12k_ext_irq_grp; +struct ath12k_dp_rx_tid; +struct ath12k_dp_rx_tid_rxq; #define DP_MON_PURGE_TIMEOUT_MS 100 #define DP_MON_SERVICE_BUDGET 128 +#define DP_ENCAP_TYPE_MAX 4 +#define DP_ENCRYPT_TYPE_MAX 12 +#define DP_DESC_TYPE_MAX 2 + struct dp_srng { u32 *vaddr_unaligned; u32 *vaddr; @@ -149,6 +155,18 @@ struct ath12k_pdev_dp { u32 mac_id; atomic_t num_tx_pending; wait_queue_head_t tx_empty_waitq; + + struct ath12k_dp *dp; + struct ieee80211_hw *hw; + u8 hw_link_id; + struct ath12k_dp_hw *dp_hw; + + /* Protects ppdu stats */ + spinlock_t ppdu_list_lock; + struct ath12k_per_peer_tx_stats peer_tx_stats; + struct list_head ppdu_stats_info; + u32 ppdu_stat_list_depth; + struct dp_srng rxdma_mon_dst_ring[MAX_RXDMA_PER_PDEV]; struct dp_srng tx_mon_dst_ring[MAX_RXDMA_PER_PDEV]; @@ -360,9 +378,77 @@ struct ath12k_link_stats { u32 tx_completed; u32 tx_bcast_mcast; u32 tx_dropped; - u32 tx_encap_type[HAL_TCL_ENCAP_TYPE_MAX]; - u32 tx_encrypt_type[HAL_ENCRYPT_TYPE_MAX]; - u32 tx_desc_type[HAL_TCL_DESC_TYPE_MAX]; + u32 tx_encap_type[DP_ENCAP_TYPE_MAX]; + u32 tx_encrypt_type[DP_ENCRYPT_TYPE_MAX]; + u32 tx_desc_type[DP_DESC_TYPE_MAX]; +}; + +/* DP arch ops to communicate from common module + * to arch specific module + */ +struct ath12k_dp_arch_ops { + int (*service_srng)(struct ath12k_dp *dp, + struct ath12k_ext_irq_grp *irq_grp, + int budget); + u32 (*tx_get_vdev_bank_config)(struct ath12k_base *ab, + struct ath12k_link_vif *arvif); + int (*reo_cmd_send)(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid, + enum hal_reo_cmd_type type, + struct ath12k_hal_reo_cmd *cmd, + void (*cb)(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status)); + void (*setup_pn_check_reo_cmd)(struct ath12k_hal_reo_cmd *cmd, + struct ath12k_dp_rx_tid *rx_tid, + u32 cipher, enum set_key_cmd key_cmd); + void (*rx_peer_tid_delete)(struct ath12k_base *ab, + struct ath12k_dp_link_peer *peer, u8 tid); + int (*reo_cache_flush)(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid); + int (*rx_link_desc_return)(struct ath12k_dp *dp, + struct ath12k_buffer_addr *buf_addr_info, + enum hal_wbm_rel_bm_act action); + void (*rx_frags_cleanup)(struct ath12k_dp_rx_tid *rx_tid, + bool rel_link_desc); + int (*peer_rx_tid_reo_update)(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer, + struct ath12k_dp_rx_tid *rx_tid, + u32 ba_win_sz, u16 ssn, + bool update_ssn); + int (*rx_assign_reoq)(struct ath12k_base *ab, struct ath12k_dp_peer *dp_peer, + struct ath12k_dp_rx_tid *rx_tid, + u16 ssn, enum hal_pn_type pn_type); + void (*peer_rx_tid_qref_setup)(struct ath12k_base *ab, u16 peer_id, u16 tid, + dma_addr_t paddr); + void (*peer_rx_tid_qref_reset)(struct ath12k_base *ab, u16 peer_id, u16 tid); + int (*rx_tid_delete_handler)(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid); +}; + +struct ath12k_device_dp_tx_err_stats { + /* TCL Ring Descriptor unavailable */ + u32 desc_na[DP_TCL_NUM_RING_MAX]; + /* Other failures during dp_tx due to mem allocation failure + * idr unavailable etc. + */ + atomic_t misc_fail; +}; + +struct ath12k_device_dp_stats { + u32 err_ring_pkts; + u32 invalid_rbm; + u32 rxdma_error[HAL_REO_ENTR_RING_RXDMA_ECODE_MAX]; + u32 reo_error[HAL_REO_DEST_RING_ERROR_CODE_MAX]; + u32 hal_reo_error[DP_REO_DST_RING_MAX]; + struct ath12k_device_dp_tx_err_stats tx_err; + u32 reo_rx[DP_REO_DST_RING_MAX][ATH12K_MAX_DEVICES]; + u32 rx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX][ATH12K_MAX_DEVICES]; + u32 tqm_rel_reason[MAX_TQM_RELEASE_REASON]; + u32 fw_tx_status[MAX_FW_TX_STATUS]; + u32 tx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX]; + u32 tx_enqueued[DP_TCL_NUM_RING_MAX]; + u32 tx_completed[DP_TCL_NUM_RING_MAX]; + u32 reo_excep_msdu_buf_type; }; struct ath12k_dp { @@ -427,1512 +513,137 @@ struct ath12k_dp { struct dp_rxdma_mon_ring rx_mon_status_refill_ring[MAX_RXDMA_PER_PDEV]; struct ath12k_reo_q_addr_lut reoq_lut; struct ath12k_reo_q_addr_lut ml_reoq_lut; -}; - -/* HTT definitions */ -#define HTT_TAG_TCL_METADATA_VERSION 5 - -#define HTT_TCL_META_DATA_TYPE GENMASK(1, 0) -#define HTT_TCL_META_DATA_VALID_HTT BIT(2) - -/* vdev meta data */ -#define HTT_TCL_META_DATA_VDEV_ID GENMASK(10, 3) -#define HTT_TCL_META_DATA_PDEV_ID GENMASK(12, 11) -#define HTT_TCL_META_DATA_HOST_INSPECTED_MISSION BIT(13) - -/* peer meta data */ -#define HTT_TCL_META_DATA_PEER_ID GENMASK(15, 3) - -/* Global sequence number */ -#define HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM 3 -#define HTT_TCL_META_DATA_GLOBAL_SEQ_HOST_INSPECTED BIT(2) -#define HTT_TCL_META_DATA_GLOBAL_SEQ_NUM GENMASK(14, 3) -#define HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID 128 - -/* HTT tx completion is overlaid in wbm_release_ring */ -#define HTT_TX_WBM_COMP_INFO0_STATUS GENMASK(16, 13) -#define HTT_TX_WBM_COMP_INFO1_REINJECT_REASON GENMASK(3, 0) -#define HTT_TX_WBM_COMP_INFO1_EXCEPTION_FRAME BIT(4) - -#define HTT_TX_WBM_COMP_INFO2_ACK_RSSI GENMASK(31, 24) - -struct htt_tx_wbm_completion { - __le32 rsvd0[2]; - __le32 info0; - __le32 info1; - __le32 info2; - __le32 info3; - __le32 info4; - __le32 rsvd1; - -} __packed; - -enum htt_h2t_msg_type { - HTT_H2T_MSG_TYPE_VERSION_REQ = 0, - HTT_H2T_MSG_TYPE_SRING_SETUP = 0xb, - HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG = 0xc, - HTT_H2T_MSG_TYPE_EXT_STATS_CFG = 0x10, - HTT_H2T_MSG_TYPE_PPDU_STATS_CFG = 0x11, - HTT_H2T_MSG_TYPE_VDEV_TXRX_STATS_CFG = 0x1a, - HTT_H2T_MSG_TYPE_TX_MONITOR_CFG = 0x1b, -}; - -#define HTT_VER_REQ_INFO_MSG_ID GENMASK(7, 0) -#define HTT_OPTION_TCL_METADATA_VER_V1 1 -#define HTT_OPTION_TCL_METADATA_VER_V2 2 -#define HTT_OPTION_TAG GENMASK(7, 0) -#define HTT_OPTION_LEN GENMASK(15, 8) -#define HTT_OPTION_VALUE GENMASK(31, 16) -#define HTT_TCL_METADATA_VER_SZ 4 - -struct htt_ver_req_cmd { - __le32 ver_reg_info; - __le32 tcl_metadata_version; -} __packed; - -enum htt_srng_ring_type { - HTT_HW_TO_SW_RING, - HTT_SW_TO_HW_RING, - HTT_SW_TO_SW_RING, -}; - -enum htt_srng_ring_id { - HTT_RXDMA_HOST_BUF_RING, - HTT_RXDMA_MONITOR_STATUS_RING, - HTT_RXDMA_MONITOR_BUF_RING, - HTT_RXDMA_MONITOR_DESC_RING, - HTT_RXDMA_MONITOR_DEST_RING, - HTT_HOST1_TO_FW_RXBUF_RING, - HTT_HOST2_TO_FW_RXBUF_RING, - HTT_RXDMA_NON_MONITOR_DEST_RING, - HTT_RXDMA_HOST_BUF_RING2, - HTT_TX_MON_HOST2MON_BUF_RING, - HTT_TX_MON_MON2HOST_DEST_RING, - HTT_RX_MON_HOST2MON_BUF_RING, - HTT_RX_MON_MON2HOST_DEST_RING, -}; - -/* host -> target HTT_SRING_SETUP message - * - * After target is booted up, Host can send SRING setup message for - * each host facing LMAC SRING. Target setups up HW registers based - * on setup message and confirms back to Host if response_required is set. - * Host should wait for confirmation message before sending new SRING - * setup message - * - * The message would appear as follows: - * - * |31 24|23 20|19|18 16|15|14 8|7 0| - * |--------------- +-----------------+----------------+------------------| - * | ring_type | ring_id | pdev_id | msg_type | - * |----------------------------------------------------------------------| - * | ring_base_addr_lo | - * |----------------------------------------------------------------------| - * | ring_base_addr_hi | - * |----------------------------------------------------------------------| - * |ring_misc_cfg_flag|ring_entry_size| ring_size | - * |----------------------------------------------------------------------| - * | ring_head_offset32_remote_addr_lo | - * |----------------------------------------------------------------------| - * | ring_head_offset32_remote_addr_hi | - * |----------------------------------------------------------------------| - * | ring_tail_offset32_remote_addr_lo | - * |----------------------------------------------------------------------| - * | ring_tail_offset32_remote_addr_hi | - * |----------------------------------------------------------------------| - * | ring_msi_addr_lo | - * |----------------------------------------------------------------------| - * | ring_msi_addr_hi | - * |----------------------------------------------------------------------| - * | ring_msi_data | - * |----------------------------------------------------------------------| - * | intr_timer_th |IM| intr_batch_counter_th | - * |----------------------------------------------------------------------| - * | reserved |RR|PTCF| intr_low_threshold | - * |----------------------------------------------------------------------| - * Where - * IM = sw_intr_mode - * RR = response_required - * PTCF = prefetch_timer_cfg - * - * The message is interpreted as follows: - * dword0 - b'0:7 - msg_type: This will be set to - * HTT_H2T_MSG_TYPE_SRING_SETUP - * b'8:15 - pdev_id: - * 0 (for rings at SOC/UMAC level), - * 1/2/3 mac id (for rings at LMAC level) - * b'16:23 - ring_id: identify which ring is to setup, - * more details can be got from enum htt_srng_ring_id - * b'24:31 - ring_type: identify type of host rings, - * more details can be got from enum htt_srng_ring_type - * dword1 - b'0:31 - ring_base_addr_lo: Lower 32bits of ring base address - * dword2 - b'0:31 - ring_base_addr_hi: Upper 32bits of ring base address - * dword3 - b'0:15 - ring_size: size of the ring in unit of 4-bytes words - * b'16:23 - ring_entry_size: Size of each entry in 4-byte word units - * b'24:31 - ring_misc_cfg_flag: Valid only for HW_TO_SW_RING and - * SW_TO_HW_RING. - * Refer to HTT_SRING_SETUP_RING_MISC_CFG_RING defs. - * dword4 - b'0:31 - ring_head_off32_remote_addr_lo: - * Lower 32 bits of memory address of the remote variable - * storing the 4-byte word offset that identifies the head - * element within the ring. - * (The head offset variable has type u32.) - * Valid for HW_TO_SW and SW_TO_SW rings. - * dword5 - b'0:31 - ring_head_off32_remote_addr_hi: - * Upper 32 bits of memory address of the remote variable - * storing the 4-byte word offset that identifies the head - * element within the ring. - * (The head offset variable has type u32.) - * Valid for HW_TO_SW and SW_TO_SW rings. - * dword6 - b'0:31 - ring_tail_off32_remote_addr_lo: - * Lower 32 bits of memory address of the remote variable - * storing the 4-byte word offset that identifies the tail - * element within the ring. - * (The tail offset variable has type u32.) - * Valid for HW_TO_SW and SW_TO_SW rings. - * dword7 - b'0:31 - ring_tail_off32_remote_addr_hi: - * Upper 32 bits of memory address of the remote variable - * storing the 4-byte word offset that identifies the tail - * element within the ring. - * (The tail offset variable has type u32.) - * Valid for HW_TO_SW and SW_TO_SW rings. - * dword8 - b'0:31 - ring_msi_addr_lo: Lower 32bits of MSI cfg address - * valid only for HW_TO_SW_RING and SW_TO_HW_RING - * dword9 - b'0:31 - ring_msi_addr_hi: Upper 32bits of MSI cfg address - * valid only for HW_TO_SW_RING and SW_TO_HW_RING - * dword10 - b'0:31 - ring_msi_data: MSI data - * Refer to HTT_SRING_SETUP_RING_MSC_CFG_xxx defs - * valid only for HW_TO_SW_RING and SW_TO_HW_RING - * dword11 - b'0:14 - intr_batch_counter_th: - * batch counter threshold is in units of 4-byte words. - * HW internally maintains and increments batch count. - * (see SRING spec for detail description). - * When batch count reaches threshold value, an interrupt - * is generated by HW. - * b'15 - sw_intr_mode: - * This configuration shall be static. - * Only programmed at power up. - * 0: generate pulse style sw interrupts - * 1: generate level style sw interrupts - * b'16:31 - intr_timer_th: - * The timer init value when timer is idle or is - * initialized to start downcounting. - * In 8us units (to cover a range of 0 to 524 ms) - * dword12 - b'0:15 - intr_low_threshold: - * Used only by Consumer ring to generate ring_sw_int_p. - * Ring entries low threshold water mark, that is used - * in combination with the interrupt timer as well as - * the clearing of the level interrupt. - * b'16:18 - prefetch_timer_cfg: - * Used only by Consumer ring to set timer mode to - * support Application prefetch handling. - * The external tail offset/pointer will be updated - * at following intervals: - * 3'b000: (Prefetch feature disabled; used only for debug) - * 3'b001: 1 usec - * 3'b010: 4 usec - * 3'b011: 8 usec (default) - * 3'b100: 16 usec - * Others: Reserved - * b'19 - response_required: - * Host needs HTT_T2H_MSG_TYPE_SRING_SETUP_DONE as response - * b'20:31 - reserved: reserved for future use - */ - -#define HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE GENMASK(7, 0) -#define HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID GENMASK(15, 8) -#define HTT_SRNG_SETUP_CMD_INFO0_RING_ID GENMASK(23, 16) -#define HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE GENMASK(31, 24) - -#define HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE GENMASK(15, 0) -#define HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE GENMASK(23, 16) -#define HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS BIT(25) -#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP BIT(27) -#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP BIT(28) -#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP BIT(29) - -#define HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH GENMASK(14, 0) -#define HTT_SRNG_SETUP_CMD_INTR_INFO_SW_INTR_MODE BIT(15) -#define HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH GENMASK(31, 16) - -#define HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH GENMASK(15, 0) -#define HTT_SRNG_SETUP_CMD_INFO2_PRE_FETCH_TIMER_CFG GENMASK(18, 16) -#define HTT_SRNG_SETUP_CMD_INFO2_RESPONSE_REQUIRED BIT(19) - -struct htt_srng_setup_cmd { - __le32 info0; - __le32 ring_base_addr_lo; - __le32 ring_base_addr_hi; - __le32 info1; - __le32 ring_head_off32_remote_addr_lo; - __le32 ring_head_off32_remote_addr_hi; - __le32 ring_tail_off32_remote_addr_lo; - __le32 ring_tail_off32_remote_addr_hi; - __le32 ring_msi_addr_lo; - __le32 ring_msi_addr_hi; - __le32 msi_data; - __le32 intr_info; - __le32 info2; -} __packed; - -/* host -> target FW PPDU_STATS config message - * - * @details - * The following field definitions describe the format of the HTT host - * to target FW for PPDU_STATS_CFG msg. - * The message allows the host to configure the PPDU_STATS_IND messages - * produced by the target. - * - * |31 24|23 16|15 8|7 0| - * |-----------------------------------------------------------| - * | REQ bit mask | pdev_mask | msg type | - * |-----------------------------------------------------------| - * Header fields: - * - MSG_TYPE - * Bits 7:0 - * Purpose: identifies this is a req to configure ppdu_stats_ind from target - * Value: 0x11 - * - PDEV_MASK - * Bits 8:15 - * Purpose: identifies which pdevs this PPDU stats configuration applies to - * Value: This is a overloaded field, refer to usage and interpretation of - * PDEV in interface document. - * Bit 8 : Reserved for SOC stats - * Bit 9 - 15 : Indicates PDEV_MASK in DBDC - * Indicates MACID_MASK in DBS - * - REQ_TLV_BIT_MASK - * Bits 16:31 - * Purpose: each set bit indicates the corresponding PPDU stats TLV type - * needs to be included in the target's PPDU_STATS_IND messages. - * Value: refer htt_ppdu_stats_tlv_tag_t << 30 bits - * Refer to PKT_TYPE_ENABLE_FLAG0_xxx_MGMT_xxx defs - * dword3 - b'0:31 - packet_type_enable_flags_1: - * Enable MGMT packet from 0b1010 to 0b1111 - * bits from low to high: FP, MD, MO - 3 bits - * Refer to PKT_TYPE_ENABLE_FLAG1_xxx_MGMT_xxx defs - * dword4 - b'0:31 - packet_type_enable_flags_2: - * Enable CTRL packet from 0b0000 to 0b1001 - * bits from low to high: FP, MD, MO - 3 bits - * Refer to PKT_TYPE_ENABLE_FLAG2_xxx_CTRL_xxx defs - * dword5 - b'0:31 - packet_type_enable_flags_3: - * Enable CTRL packet from 0b1010 to 0b1111, - * MCAST_DATA, UCAST_DATA, NULL_DATA - * bits from low to high: FP, MD, MO - 3 bits - * Refer to PKT_TYPE_ENABLE_FLAG3_xxx_CTRL_xxx defs - * dword6 - b'0:31 - tlv_filter_in_flags: - * Filter in Attention/MPDU/PPDU/Header/User tlvs - * Refer to CFG_TLV_FILTER_IN_FLAG defs - */ - -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE GENMASK(7, 0) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID GENMASK(15, 8) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID GENMASK(23, 16) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS BIT(24) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS BIT(25) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID BIT(26) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL BIT(27) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON BIT(28) - -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE GENMASK(15, 0) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT GENMASK(18, 16) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL GENMASK(21, 19) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA GENMASK(24, 22) - -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD GENMASK(9, 0) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE BIT(17) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE BIT(18) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE BIT(19) - -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET BIT(0) -#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET GENMASK(14, 1) - -#define HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET GENMASK(15, 0) -#define HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET GENMASK(31, 16) -#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET GENMASK(15, 0) -#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET GENMASK(31, 16) -#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET GENMASK(15, 0) -#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET GENMASK(31, 16) -#define HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET GENMASK(15, 0) - -#define HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET BIT(23) -#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK GENMASK(15, 0) -#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK GENMASK(18, 16) -#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK GENMASK(16, 0) - -enum htt_rx_filter_tlv_flags { - HTT_RX_FILTER_TLV_FLAGS_MPDU_START = BIT(0), - HTT_RX_FILTER_TLV_FLAGS_MSDU_START = BIT(1), - HTT_RX_FILTER_TLV_FLAGS_RX_PACKET = BIT(2), - HTT_RX_FILTER_TLV_FLAGS_MSDU_END = BIT(3), - HTT_RX_FILTER_TLV_FLAGS_MPDU_END = BIT(4), - HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER = BIT(5), - HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER = BIT(6), - HTT_RX_FILTER_TLV_FLAGS_ATTENTION = BIT(7), - HTT_RX_FILTER_TLV_FLAGS_PPDU_START = BIT(8), - HTT_RX_FILTER_TLV_FLAGS_PPDU_END = BIT(9), - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS = BIT(10), - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT = BIT(11), - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE = BIT(12), - HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO = BIT(13), -}; - -enum htt_rx_mgmt_pkt_filter_tlv_flags0 { - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ = BIT(0), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ = BIT(1), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ = BIT(2), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP = BIT(3), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP = BIT(4), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP = BIT(5), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ = BIT(6), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ = BIT(7), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ = BIT(8), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP = BIT(9), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP = BIT(10), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP = BIT(11), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ = BIT(12), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ = BIT(13), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ = BIT(14), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP = BIT(15), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP = BIT(16), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP = BIT(17), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV = BIT(18), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV = BIT(19), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV = BIT(20), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7 = BIT(21), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7 = BIT(22), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7 = BIT(23), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON = BIT(24), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON = BIT(25), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON = BIT(26), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM = BIT(27), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM = BIT(28), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM = BIT(29), -}; - -enum htt_rx_mgmt_pkt_filter_tlv_flags1 { - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC = BIT(0), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC = BIT(1), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC = BIT(2), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH = BIT(3), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH = BIT(4), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH = BIT(5), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH = BIT(6), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH = BIT(7), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH = BIT(8), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION = BIT(9), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION = BIT(10), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION = BIT(11), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK = BIT(12), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK = BIT(13), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK = BIT(14), - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15 = BIT(15), - HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15 = BIT(16), - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15 = BIT(17), -}; - -enum htt_rx_ctrl_pkt_filter_tlv_flags2 { - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 = BIT(0), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 = BIT(1), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 = BIT(2), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 = BIT(3), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 = BIT(4), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 = BIT(5), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER = BIT(6), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER = BIT(7), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER = BIT(8), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 = BIT(9), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 = BIT(10), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 = BIT(11), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL = BIT(12), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL = BIT(13), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL = BIT(14), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP = BIT(15), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP = BIT(16), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP = BIT(17), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT = BIT(18), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT = BIT(19), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT = BIT(20), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER = BIT(21), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER = BIT(22), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER = BIT(23), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR = BIT(24), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR = BIT(25), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR = BIT(26), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA = BIT(27), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA = BIT(28), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA = BIT(29), -}; - -enum htt_rx_ctrl_pkt_filter_tlv_flags3 { - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL = BIT(0), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL = BIT(1), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL = BIT(2), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS = BIT(3), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS = BIT(4), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS = BIT(5), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS = BIT(6), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS = BIT(7), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS = BIT(8), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK = BIT(9), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK = BIT(10), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK = BIT(11), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND = BIT(12), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND = BIT(13), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND = BIT(14), - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK = BIT(15), - HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK = BIT(16), - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK = BIT(17), -}; - -enum htt_rx_data_pkt_filter_tlv_flasg3 { - HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST = BIT(18), - HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST = BIT(19), - HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST = BIT(20), - HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST = BIT(21), - HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST = BIT(22), - HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST = BIT(23), - HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA = BIT(24), - HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA = BIT(25), - HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA = BIT(26), -}; - -#define HTT_RX_FP_MGMT_FILTER_FLAGS0 \ - (HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM) - -#define HTT_RX_MD_MGMT_FILTER_FLAGS0 \ - (HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM) - -#define HTT_RX_MO_MGMT_FILTER_FLAGS0 \ - (HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM) - -#define HTT_RX_FP_MGMT_FILTER_FLAGS1 (HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \ - | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK) - -#define HTT_RX_MD_MGMT_FILTER_FLAGS1 (HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \ - | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK) - -#define HTT_RX_MO_MGMT_FILTER_FLAGS1 (HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \ - | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK) - -#define HTT_RX_FP_CTRL_FILTER_FLASG2 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \ - | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \ - | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA) - -#define HTT_RX_MD_CTRL_FILTER_FLASG2 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \ - | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \ - | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA) - -#define HTT_RX_MO_CTRL_FILTER_FLASG2 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \ - | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \ - | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA) - -#define HTT_RX_FP_CTRL_FILTER_FLASG3 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \ - | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \ - | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \ - | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \ - | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \ - | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK) - -#define HTT_RX_MD_CTRL_FILTER_FLASG3 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \ - | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \ - | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \ - | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \ - | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \ - | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK) - -#define HTT_RX_MO_CTRL_FILTER_FLASG3 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \ - | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \ - | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \ - | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \ - | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \ - | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK) - -#define HTT_RX_FP_DATA_FILTER_FLASG3 (HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST \ - | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST \ - | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA) - -#define HTT_RX_MD_DATA_FILTER_FLASG3 (HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST \ - | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST \ - | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA) - -#define HTT_RX_MO_DATA_FILTER_FLASG3 (HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST \ - | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST \ - | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA) - -#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 \ - (HTT_RX_FP_MGMT_FILTER_FLAGS0 | \ - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7) - -#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS0 \ - (HTT_RX_MO_MGMT_FILTER_FLAGS0 | \ - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7) - -#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 \ - (HTT_RX_FP_MGMT_FILTER_FLAGS1 | \ - HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15) - -#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS1 \ - (HTT_RX_MO_MGMT_FILTER_FLAGS1 | \ - HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15) - -#define HTT_RX_MON_FP_CTRL_FILTER_FLASG2 \ - (HTT_RX_FP_CTRL_FILTER_FLASG2 | \ - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \ - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \ - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \ - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \ - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \ - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \ - HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT) - -#define HTT_RX_MON_MO_CTRL_FILTER_FLASG2 \ - (HTT_RX_MO_CTRL_FILTER_FLASG2 | \ - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \ - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \ - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \ - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \ - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \ - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \ - HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT) - -#define HTT_RX_MON_FP_CTRL_FILTER_FLASG3 HTT_RX_FP_CTRL_FILTER_FLASG3 - -#define HTT_RX_MON_MO_CTRL_FILTER_FLASG3 HTT_RX_MO_CTRL_FILTER_FLASG3 - -#define HTT_RX_MON_FP_DATA_FILTER_FLASG3 HTT_RX_FP_DATA_FILTER_FLASG3 - -#define HTT_RX_MON_MO_DATA_FILTER_FLASG3 HTT_RX_MO_DATA_FILTER_FLASG3 - -#define HTT_RX_MON_FILTER_TLV_FLAGS \ - (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE) - -#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING \ - (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE) - -#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_BUF_RING \ - (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \ - HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \ - HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \ - HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \ - HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \ - HTT_RX_FILTER_TLV_FLAGS_ATTENTION) - -#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING \ - (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \ - HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \ - HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \ - HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \ - HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE | \ - HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO) - -/* msdu start. mpdu end, attention, rx hdr tlv's are not subscribed */ -#define HTT_RX_TLV_FLAGS_RXDMA_RING \ - (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ - HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \ - HTT_RX_FILTER_TLV_FLAGS_MSDU_END) - -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE GENMASK(7, 0) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID GENMASK(15, 8) - -struct htt_rx_ring_selection_cfg_cmd { - __le32 info0; - __le32 info1; - __le32 pkt_type_en_flags0; - __le32 pkt_type_en_flags1; - __le32 pkt_type_en_flags2; - __le32 pkt_type_en_flags3; - __le32 rx_filter_tlv; - __le32 rx_packet_offset; - __le32 rx_mpdu_offset; - __le32 rx_msdu_offset; - __le32 rx_attn_offset; - __le32 info2; - __le32 reserved[2]; - __le32 rx_mpdu_start_end_mask; - __le32 rx_msdu_end_word_mask; - __le32 info3; -} __packed; - -#define HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE 32 -#define HTT_RX_RING_DEFAULT_DMA_LENGTH 0x7 -#define HTT_RX_RING_PKT_TLV_OFFSET 0x1 - -struct htt_rx_ring_tlv_filter { - u32 rx_filter; /* see htt_rx_filter_tlv_flags */ - u32 pkt_filter_flags0; /* MGMT */ - u32 pkt_filter_flags1; /* MGMT */ - u32 pkt_filter_flags2; /* CTRL */ - u32 pkt_filter_flags3; /* DATA */ - bool offset_valid; - u16 rx_packet_offset; - u16 rx_header_offset; - u16 rx_mpdu_end_offset; - u16 rx_mpdu_start_offset; - u16 rx_msdu_end_offset; - u16 rx_msdu_start_offset; - u16 rx_attn_offset; - u16 rx_mpdu_start_wmask; - u16 rx_mpdu_end_wmask; - u32 rx_msdu_end_wmask; - u32 conf_len_ctrl; - u32 conf_len_mgmt; - u32 conf_len_data; - u16 rx_drop_threshold; - bool enable_log_mgmt_type; - bool enable_log_ctrl_type; - bool enable_log_data_type; - bool enable_rx_tlv_offset; - u16 rx_tlv_offset; - bool drop_threshold_valid; - bool rxmon_disable; -}; - -#define HTT_STATS_FRAME_CTRL_TYPE_MGMT 0x0 -#define HTT_STATS_FRAME_CTRL_TYPE_CTRL 0x1 -#define HTT_STATS_FRAME_CTRL_TYPE_DATA 0x2 -#define HTT_STATS_FRAME_CTRL_TYPE_RESV 0x3 - -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE GENMASK(7, 0) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID GENMASK(15, 8) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID GENMASK(23, 16) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS BIT(24) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS BIT(25) - -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE GENMASK(15, 0) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE GENMASK(18, 16) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT GENMASK(21, 19) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL GENMASK(24, 22) -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA GENMASK(27, 25) - -#define HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG GENMASK(2, 0) - -struct htt_tx_ring_selection_cfg_cmd { - __le32 info0; - __le32 info1; - __le32 info2; - __le32 tlv_filter_mask_in0; - __le32 tlv_filter_mask_in1; - __le32 tlv_filter_mask_in2; - __le32 tlv_filter_mask_in3; - __le32 reserved[3]; -} __packed; - -#define HTT_TX_RING_TLV_FILTER_MGMT_DMA_LEN GENMASK(3, 0) -#define HTT_TX_RING_TLV_FILTER_CTRL_DMA_LEN GENMASK(7, 4) -#define HTT_TX_RING_TLV_FILTER_DATA_DMA_LEN GENMASK(11, 8) - -#define HTT_TX_MON_FILTER_HYBRID_MODE \ - (HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS | \ - HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT | \ - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE | \ - HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO | \ - HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2) - -struct htt_tx_ring_tlv_filter { - u32 tx_mon_downstream_tlv_flags; - u32 tx_mon_upstream_tlv_flags0; - u32 tx_mon_upstream_tlv_flags1; - u32 tx_mon_upstream_tlv_flags2; - bool tx_mon_mgmt_filter; - bool tx_mon_data_filter; - bool tx_mon_ctrl_filter; - u16 tx_mon_pkt_dma_len; -} __packed; - -enum htt_tx_mon_upstream_tlv_flags0 { - HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS = BIT(1), - HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS = BIT(2), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START = BIT(3), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END = BIT(4), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU = BIT(5), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU = BIT(6), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA = BIT(7), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA = BIT(8), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT = BIT(9), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT = BIT(10), - HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE = BIT(11), - HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_BITMAP_ACK = BIT(12), - HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_1K_BITMAP_ACK = BIT(13), - HTT_TX_FILTER_TLV_FLAGS0_COEX_TX_STATUS = BIT(14), - HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO = BIT(15), - HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2 = BIT(16), -}; - -#define HTT_TX_FILTER_TLV_FLAGS2_TXPCU_PHYTX_OTHER_TRANSMIT_INFO32 BIT(11) - -/* HTT message target->host */ - -enum htt_t2h_msg_type { - HTT_T2H_MSG_TYPE_VERSION_CONF, - HTT_T2H_MSG_TYPE_PEER_MAP = 0x3, - HTT_T2H_MSG_TYPE_PEER_UNMAP = 0x4, - HTT_T2H_MSG_TYPE_RX_ADDBA = 0x5, - HTT_T2H_MSG_TYPE_PKTLOG = 0x8, - HTT_T2H_MSG_TYPE_SEC_IND = 0xb, - HTT_T2H_MSG_TYPE_PEER_MAP2 = 0x1e, - HTT_T2H_MSG_TYPE_PEER_UNMAP2 = 0x1f, - HTT_T2H_MSG_TYPE_PPDU_STATS_IND = 0x1d, - HTT_T2H_MSG_TYPE_EXT_STATS_CONF = 0x1c, - HTT_T2H_MSG_TYPE_BKPRESSURE_EVENT_IND = 0x24, - HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND = 0x28, - HTT_T2H_MSG_TYPE_PEER_MAP3 = 0x2b, - HTT_T2H_MSG_TYPE_VDEV_TXRX_STATS_PERIODIC_IND = 0x2c, -}; - -#define HTT_TARGET_VERSION_MAJOR 3 - -#define HTT_T2H_MSG_TYPE GENMASK(7, 0) -#define HTT_T2H_VERSION_CONF_MINOR GENMASK(15, 8) -#define HTT_T2H_VERSION_CONF_MAJOR GENMASK(23, 16) - -struct htt_t2h_version_conf_msg { - __le32 version; -} __packed; - -#define HTT_T2H_PEER_MAP_INFO_VDEV_ID GENMASK(15, 8) -#define HTT_T2H_PEER_MAP_INFO_PEER_ID GENMASK(31, 16) -#define HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16 GENMASK(15, 0) -#define HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID GENMASK(31, 16) -#define HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL GENMASK(15, 0) -#define HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID GENMASK(15, 0) -#define HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL GENMASK(31, 16) -#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M BIT(16) -#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S 16 - -struct htt_t2h_peer_map_event { - __le32 info; - __le32 mac_addr_l32; - __le32 info1; - __le32 info2; -} __packed; - -#define HTT_T2H_PEER_UNMAP_INFO_VDEV_ID HTT_T2H_PEER_MAP_INFO_VDEV_ID -#define HTT_T2H_PEER_UNMAP_INFO_PEER_ID HTT_T2H_PEER_MAP_INFO_PEER_ID -#define HTT_T2H_PEER_UNMAP_INFO1_MAC_ADDR_H16 \ - HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16 -#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_M HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M -#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_S HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S - -struct htt_t2h_peer_unmap_event { - __le32 info; - __le32 mac_addr_l32; - __le32 info1; -} __packed; - -struct htt_resp_msg { - union { - struct htt_t2h_version_conf_msg version_msg; - struct htt_t2h_peer_map_event peer_map_ev; - struct htt_t2h_peer_unmap_event peer_unmap_ev; - }; -} __packed; - -#define HTT_VDEV_GET_STATS_U64(msg_l32, msg_u32)\ - (((u64)__le32_to_cpu(msg_u32) << 32) | (__le32_to_cpu(msg_l32))) -#define HTT_T2H_VDEV_STATS_PERIODIC_MSG_TYPE GENMASK(7, 0) -#define HTT_T2H_VDEV_STATS_PERIODIC_PDEV_ID GENMASK(15, 8) -#define HTT_T2H_VDEV_STATS_PERIODIC_NUM_VDEV GENMASK(23, 16) -#define HTT_T2H_VDEV_STATS_PERIODIC_PAYLOAD_BYTES GENMASK(15, 0) -#define HTT_VDEV_TXRX_STATS_COMMON_TLV 0 -#define HTT_VDEV_TXRX_STATS_HW_STATS_TLV 1 - -struct htt_t2h_vdev_txrx_stats_ind { - __le32 vdev_id; - __le32 rx_msdu_byte_cnt_lo; - __le32 rx_msdu_byte_cnt_hi; - __le32 rx_msdu_cnt_lo; - __le32 rx_msdu_cnt_hi; - __le32 tx_msdu_byte_cnt_lo; - __le32 tx_msdu_byte_cnt_hi; - __le32 tx_msdu_cnt_lo; - __le32 tx_msdu_cnt_hi; - __le32 tx_retry_cnt_lo; - __le32 tx_retry_cnt_hi; - __le32 tx_retry_byte_cnt_lo; - __le32 tx_retry_byte_cnt_hi; - __le32 tx_drop_cnt_lo; - __le32 tx_drop_cnt_hi; - __le32 tx_drop_byte_cnt_lo; - __le32 tx_drop_byte_cnt_hi; - __le32 msdu_ttl_cnt_lo; - __le32 msdu_ttl_cnt_hi; - __le32 msdu_ttl_byte_cnt_lo; - __le32 msdu_ttl_byte_cnt_hi; -} __packed; - -struct htt_t2h_vdev_common_stats_tlv { - __le32 soc_drop_count_lo; - __le32 soc_drop_count_hi; -} __packed; - -/* ppdu stats - * - * @details - * The following field definitions describe the format of the HTT target - * to host ppdu stats indication message. - * - * - * |31 16|15 12|11 10|9 8|7 0 | - * |----------------------------------------------------------------------| - * | payload_size | rsvd |pdev_id|mac_id | msg type | - * |----------------------------------------------------------------------| - * | ppdu_id | - * |----------------------------------------------------------------------| - * | Timestamp in us | - * |----------------------------------------------------------------------| - * | reserved | - * |----------------------------------------------------------------------| - * | type-specific stats info | - * | (see htt_ppdu_stats.h) | - * |----------------------------------------------------------------------| - * Header fields: - * - MSG_TYPE - * Bits 7:0 - * Purpose: Identifies this is a PPDU STATS indication - * message. - * Value: 0x1d - * - mac_id - * Bits 9:8 - * Purpose: mac_id of this ppdu_id - * Value: 0-3 - * - pdev_id - * Bits 11:10 - * Purpose: pdev_id of this ppdu_id - * Value: 0-3 - * 0 (for rings at SOC level), - * 1/2/3 PDEV -> 0/1/2 - * - payload_size - * Bits 31:16 - * Purpose: total tlv size - * Value: payload_size in bytes - */ - -#define HTT_T2H_PPDU_STATS_INFO_PDEV_ID GENMASK(11, 10) -#define HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE GENMASK(31, 16) - -struct ath12k_htt_ppdu_stats_msg { - __le32 info; - __le32 ppdu_id; - __le32 timestamp; - __le32 rsvd; - u8 data[]; -} __packed; - -struct htt_tlv { - __le32 header; - u8 value[]; -} __packed; - -#define HTT_TLV_TAG GENMASK(11, 0) -#define HTT_TLV_LEN GENMASK(23, 12) - -enum HTT_PPDU_STATS_BW { - HTT_PPDU_STATS_BANDWIDTH_5MHZ = 0, - HTT_PPDU_STATS_BANDWIDTH_10MHZ = 1, - HTT_PPDU_STATS_BANDWIDTH_20MHZ = 2, - HTT_PPDU_STATS_BANDWIDTH_40MHZ = 3, - HTT_PPDU_STATS_BANDWIDTH_80MHZ = 4, - HTT_PPDU_STATS_BANDWIDTH_160MHZ = 5, /* includes 80+80 */ - HTT_PPDU_STATS_BANDWIDTH_DYN = 6, -}; - -#define HTT_PPDU_STATS_CMN_FLAGS_FRAME_TYPE_M GENMASK(7, 0) -#define HTT_PPDU_STATS_CMN_FLAGS_QUEUE_TYPE_M GENMASK(15, 8) -/* bw - HTT_PPDU_STATS_BW */ -#define HTT_PPDU_STATS_CMN_FLAGS_BW_M GENMASK(19, 16) - -struct htt_ppdu_stats_common { - __le32 ppdu_id; - __le16 sched_cmdid; - u8 ring_id; - u8 num_users; - __le32 flags; /* %HTT_PPDU_STATS_COMMON_FLAGS_*/ - __le32 chain_mask; - __le32 fes_duration_us; /* frame exchange sequence */ - __le32 ppdu_sch_eval_start_tstmp_us; - __le32 ppdu_sch_end_tstmp_us; - __le32 ppdu_start_tstmp_us; - /* BIT [15 : 0] - phy mode (WLAN_PHY_MODE) with which ppdu was transmitted - * BIT [31 : 16] - bandwidth (in MHz) with which ppdu was transmitted + const struct ath12k_hw_params *hw_params; + struct device *dev; + struct ath12k_hal *hal; + + /* RCU on dp_pdevs[] provides a teardown synchronization mechanism, + * ensuring in-flight data path readers complete before reclaim. Writers + * update internal fields under their own synchronization, while readers of + * internal fields may perform lockless read if occasional inconsistency + * is acceptable or use additional synchronization for a coherent view. + * + * RCU is used for dp_pdevs[] at this stage to align with + * ab->pdevs_active[]. However, if the teardown paths ensure quiescence, + * both dp_pdevs[] and pdevs_active[] can be converted to plain pointers, + * removing RCU synchronize overhead. + * + * TODO: evaluate removal of RCU from dp_pdevs in the future */ - __le16 phy_mode; - __le16 bw_mhz; -} __packed; + struct ath12k_pdev_dp __rcu *dp_pdevs[MAX_RADIOS]; -enum htt_ppdu_stats_gi { - HTT_PPDU_STATS_SGI_0_8_US, - HTT_PPDU_STATS_SGI_0_4_US, - HTT_PPDU_STATS_SGI_1_6_US, - HTT_PPDU_STATS_SGI_3_2_US, -}; + struct ath12k_hw_group *ag; + u8 device_id; -#define HTT_PPDU_STATS_USER_RATE_INFO0_USER_POS_M GENMASK(3, 0) -#define HTT_PPDU_STATS_USER_RATE_INFO0_MU_GROUP_ID_M GENMASK(11, 4) - -enum HTT_PPDU_STATS_PPDU_TYPE { - HTT_PPDU_STATS_PPDU_TYPE_SU, - HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO, - HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA, - HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA, - HTT_PPDU_STATS_PPDU_TYPE_UL_TRIG, - HTT_PPDU_STATS_PPDU_TYPE_BURST_BCN, - HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_RESP, - HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_TRIG, - HTT_PPDU_STATS_PPDU_TYPE_UL_RESP, - HTT_PPDU_STATS_PPDU_TYPE_MAX -}; + /* Lock for protection of peers and rhead_peer_addr */ + spinlock_t dp_lock; -#define HTT_PPDU_STATS_USER_RATE_INFO1_RESP_TYPE_VALD_M BIT(0) -#define HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M GENMASK(5, 1) - -#define HTT_PPDU_STATS_USER_RATE_FLAGS_LTF_SIZE_M GENMASK(1, 0) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_STBC_M BIT(2) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_HE_RE_M BIT(3) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_TXBF_M GENMASK(7, 4) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M GENMASK(11, 8) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M GENMASK(15, 12) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M GENMASK(19, 16) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M GENMASK(23, 20) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M GENMASK(27, 24) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M BIT(28) -#define HTT_PPDU_STATS_USER_RATE_FLAGS_LDPC_M BIT(29) - -#define HTT_USR_RATE_PPDU_TYPE(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M) -#define HTT_USR_RATE_PREAMBLE(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M) -#define HTT_USR_RATE_BW(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M) -#define HTT_USR_RATE_NSS(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M) -#define HTT_USR_RATE_MCS(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M) -#define HTT_USR_RATE_GI(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M) -#define HTT_USR_RATE_DCM(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M) - -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LTF_SIZE_M GENMASK(1, 0) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_STBC_M BIT(2) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_HE_RE_M BIT(3) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_TXBF_M GENMASK(7, 4) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_BW_M GENMASK(11, 8) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_NSS_M GENMASK(15, 12) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_MCS_M GENMASK(19, 16) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_PREAMBLE_M GENMASK(23, 20) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_GI_M GENMASK(27, 24) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_DCM_M BIT(28) -#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LDPC_M BIT(29) - -struct htt_ppdu_stats_user_rate { - u8 tid_num; - u8 reserved0; - __le16 sw_peer_id; - __le32 info0; /* %HTT_PPDU_STATS_USER_RATE_INFO0_*/ - __le16 ru_end; - __le16 ru_start; - __le16 resp_ru_end; - __le16 resp_ru_start; - __le32 info1; /* %HTT_PPDU_STATS_USER_RATE_INFO1_ */ - __le32 rate_flags; /* %HTT_PPDU_STATS_USER_RATE_FLAGS_ */ - /* Note: resp_rate_info is only valid for if resp_type is UL */ - __le32 resp_rate_flags; /* %HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_ */ -} __packed; + struct ath12k_dp_arch_ops *ops; -#define HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M GENMASK(7, 0) -#define HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M BIT(8) -#define HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M GENMASK(10, 9) -#define HTT_PPDU_STATS_TX_INFO_FLAGS_BW_M GENMASK(13, 11) -#define HTT_PPDU_STATS_TX_INFO_FLAGS_SGI_M BIT(14) -#define HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M GENMASK(31, 16) - -#define HTT_TX_INFO_IS_AMSDU(_flags) \ - u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M) -#define HTT_TX_INFO_BA_ACK_FAILED(_flags) \ - u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M) -#define HTT_TX_INFO_RATECODE(_flags) \ - u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M) -#define HTT_TX_INFO_PEERID(_flags) \ - u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M) - -enum htt_ppdu_stats_usr_compln_status { - HTT_PPDU_STATS_USER_STATUS_OK, - HTT_PPDU_STATS_USER_STATUS_FILTERED, - HTT_PPDU_STATS_USER_STATUS_RESP_TIMEOUT, - HTT_PPDU_STATS_USER_STATUS_RESP_MISMATCH, - HTT_PPDU_STATS_USER_STATUS_ABORT, -}; - -#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M GENMASK(3, 0) -#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M GENMASK(7, 4) -#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M BIT(8) -#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_RESP_TYPE_M GENMASK(12, 9) - -#define HTT_USR_CMPLTN_IS_AMPDU(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M) -#define HTT_USR_CMPLTN_LONG_RETRY(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M) -#define HTT_USR_CMPLTN_SHORT_RETRY(_val) \ - le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M) - -struct htt_ppdu_stats_usr_cmpltn_cmn { - u8 status; - u8 tid_num; - __le16 sw_peer_id; - /* RSSI value of last ack packet (units = dB above noise floor) */ - __le32 ack_rssi; - __le16 mpdu_tried; - __le16 mpdu_success; - __le32 flags; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRIES*/ -} __packed; + /* Linked list of struct ath12k_dp_link_peer */ + struct list_head peers; -#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MPDU_M GENMASK(8, 0) -#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M GENMASK(24, 9) -#define HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM GENMASK(31, 25) + /* For rhash table init and deinit protection */ + struct mutex link_peer_rhash_tbl_lock; -#define HTT_PPDU_STATS_NON_QOS_TID 16 - -struct htt_ppdu_stats_usr_cmpltn_ack_ba_status { - __le32 ppdu_id; - __le16 sw_peer_id; - __le16 reserved0; - __le32 info; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_INFO_ */ - __le16 current_seq; - __le16 start_seq; - __le32 success_bytes; -} __packed; - -struct htt_ppdu_user_stats { - u16 peer_id; - u16 delay_ba; - u32 tlv_flags; - bool is_valid_peer_id; - struct htt_ppdu_stats_user_rate rate; - struct htt_ppdu_stats_usr_cmpltn_cmn cmpltn_cmn; - struct htt_ppdu_stats_usr_cmpltn_ack_ba_status ack_ba; + /* The rhashtable containing struct ath12k_link_peer keyed by mac addr */ + struct rhashtable *rhead_peer_addr; + struct rhashtable_params rhash_peer_addr_param; + struct ath12k_device_dp_stats device_stats; }; -#define HTT_PPDU_STATS_MAX_USERS 8 -#define HTT_PPDU_DESC_MAX_DEPTH 16 +static inline u32 ath12k_dp_arch_tx_get_vdev_bank_config(struct ath12k_dp *dp, + struct ath12k_link_vif *arvif) +{ + return dp->ops->tx_get_vdev_bank_config(dp->ab, arvif); +} -struct htt_ppdu_stats { - struct htt_ppdu_stats_common common; - struct htt_ppdu_user_stats user_stats[HTT_PPDU_STATS_MAX_USERS]; -}; +static inline int ath12k_dp_arch_reo_cmd_send(struct ath12k_dp *dp, + struct ath12k_dp_rx_tid_rxq *rx_tid, + enum hal_reo_cmd_type type, + struct ath12k_hal_reo_cmd *cmd, + void (*cb)(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status)) +{ + return dp->ops->reo_cmd_send(dp->ab, rx_tid, type, cmd, cb); +} -struct htt_ppdu_stats_info { - u32 tlv_bitmap; - u32 ppdu_id; - u32 frame_type; - u32 frame_ctrl; - u32 delay_ba; - u32 bar_num_users; - struct htt_ppdu_stats ppdu_stats; - struct list_head list; -}; +static inline +void ath12k_dp_arch_setup_pn_check_reo_cmd(struct ath12k_dp *dp, + struct ath12k_hal_reo_cmd *cmd, + struct ath12k_dp_rx_tid *rx_tid, + u32 cipher, + enum set_key_cmd key_cmd) +{ + dp->ops->setup_pn_check_reo_cmd(cmd, rx_tid, cipher, key_cmd); +} -/* @brief target -> host MLO offset indiciation message - * - * @details - * The following field definitions describe the format of the HTT target - * to host mlo offset indication message. - * - * - * |31 29|28 |26|25 22|21 16|15 13|12 10 |9 8|7 0| - * |---------------------------------------------------------------------| - * | rsvd1 | mac_freq |chip_id |pdev_id|msgtype| - * |---------------------------------------------------------------------| - * | sync_timestamp_lo_us | - * |---------------------------------------------------------------------| - * | sync_timestamp_hi_us | - * |---------------------------------------------------------------------| - * | mlo_offset_lo | - * |---------------------------------------------------------------------| - * | mlo_offset_hi | - * |---------------------------------------------------------------------| - * | mlo_offset_clcks | - * |---------------------------------------------------------------------| - * | rsvd2 | mlo_comp_clks |mlo_comp_us | - * |---------------------------------------------------------------------| - * | rsvd3 |mlo_comp_timer | - * |---------------------------------------------------------------------| - * Header fields - * - MSG_TYPE - * Bits 7:0 - * Purpose: Identifies this is a MLO offset indication msg - * - PDEV_ID - * Bits 9:8 - * Purpose: Pdev of this MLO offset - * - CHIP_ID - * Bits 12:10 - * Purpose: chip_id of this MLO offset - * - MAC_FREQ - * Bits 28:13 - * - SYNC_TIMESTAMP_LO_US - * Purpose: clock frequency of the mac HW block in MHz - * Bits: 31:0 - * Purpose: lower 32 bits of the WLAN global time stamp at which - * last sync interrupt was received - * - SYNC_TIMESTAMP_HI_US - * Bits: 31:0 - * Purpose: upper 32 bits of WLAN global time stamp at which - * last sync interrupt was received - * - MLO_OFFSET_LO - * Bits: 31:0 - * Purpose: lower 32 bits of the MLO offset in us - * - MLO_OFFSET_HI - * Bits: 31:0 - * Purpose: upper 32 bits of the MLO offset in us - * - MLO_COMP_US - * Bits: 15:0 - * Purpose: MLO time stamp compensation applied in us - * - MLO_COMP_CLCKS - * Bits: 25:16 - * Purpose: MLO time stamp compensation applied in clock ticks - * - MLO_COMP_TIMER - * Bits: 21:0 - * Purpose: Periodic timer at which compensation is applied - */ +static inline void ath12k_dp_arch_rx_peer_tid_delete(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer, + u8 tid) +{ + dp->ops->rx_peer_tid_delete(dp->ab, peer, tid); +} -#define HTT_T2H_MLO_OFFSET_INFO_MSG_TYPE GENMASK(7, 0) -#define HTT_T2H_MLO_OFFSET_INFO_PDEV_ID GENMASK(9, 8) - -struct ath12k_htt_mlo_offset_msg { - __le32 info; - __le32 sync_timestamp_lo_us; - __le32 sync_timestamp_hi_us; - __le32 mlo_offset_hi; - __le32 mlo_offset_lo; - __le32 mlo_offset_clks; - __le32 mlo_comp_clks; - __le32 mlo_comp_timer; -} __packed; +static inline int ath12k_dp_arch_reo_cache_flush(struct ath12k_dp *dp, + struct ath12k_dp_rx_tid_rxq *rx_tid) +{ + return dp->ops->reo_cache_flush(dp->ab, rx_tid); +} -/* @brief host -> target FW extended statistics retrieve - * - * @details - * The following field definitions describe the format of the HTT host - * to target FW extended stats retrieve message. - * The message specifies the type of stats the host wants to retrieve. - * - * |31 24|23 16|15 8|7 0| - * |-----------------------------------------------------------| - * | reserved | stats type | pdev_mask | msg type | - * |-----------------------------------------------------------| - * | config param [0] | - * |-----------------------------------------------------------| - * | config param [1] | - * |-----------------------------------------------------------| - * | config param [2] | - * |-----------------------------------------------------------| - * | config param [3] | - * |-----------------------------------------------------------| - * | reserved | - * |-----------------------------------------------------------| - * | cookie LSBs | - * |-----------------------------------------------------------| - * | cookie MSBs | - * |-----------------------------------------------------------| - * Header fields: - * - MSG_TYPE - * Bits 7:0 - * Purpose: identifies this is a extended stats upload request message - * Value: 0x10 - * - PDEV_MASK - * Bits 8:15 - * Purpose: identifies the mask of PDEVs to retrieve stats from - * Value: This is a overloaded field, refer to usage and interpretation of - * PDEV in interface document. - * Bit 8 : Reserved for SOC stats - * Bit 9 - 15 : Indicates PDEV_MASK in DBDC - * Indicates MACID_MASK in DBS - * - STATS_TYPE - * Bits 23:16 - * Purpose: identifies which FW statistics to upload - * Value: Defined by htt_dbg_ext_stats_type (see htt_stats.h) - * - Reserved - * Bits 31:24 - * - CONFIG_PARAM [0] - * Bits 31:0 - * Purpose: give an opaque configuration value to the specified stats type - * Value: stats-type specific configuration value - * Refer to htt_stats.h for interpretation for each stats sub_type - * - CONFIG_PARAM [1] - * Bits 31:0 - * Purpose: give an opaque configuration value to the specified stats type - * Value: stats-type specific configuration value - * Refer to htt_stats.h for interpretation for each stats sub_type - * - CONFIG_PARAM [2] - * Bits 31:0 - * Purpose: give an opaque configuration value to the specified stats type - * Value: stats-type specific configuration value - * Refer to htt_stats.h for interpretation for each stats sub_type - * - CONFIG_PARAM [3] - * Bits 31:0 - * Purpose: give an opaque configuration value to the specified stats type - * Value: stats-type specific configuration value - * Refer to htt_stats.h for interpretation for each stats sub_type - * - Reserved [31:0] for future use. - * - COOKIE_LSBS - * Bits 31:0 - * Purpose: Provide a mechanism to match a target->host stats confirmation - * message with its preceding host->target stats request message. - * Value: LSBs of the opaque cookie specified by the host-side requestor - * - COOKIE_MSBS - * Bits 31:0 - * Purpose: Provide a mechanism to match a target->host stats confirmation - * message with its preceding host->target stats request message. - * Value: MSBs of the opaque cookie specified by the host-side requestor - */ +static inline +int ath12k_dp_arch_rx_link_desc_return(struct ath12k_dp *dp, + struct ath12k_buffer_addr *buf_addr_info, + enum hal_wbm_rel_bm_act action) +{ + return dp->ops->rx_link_desc_return(dp, buf_addr_info, action); +} -struct htt_ext_stats_cfg_hdr { - u8 msg_type; - u8 pdev_mask; - u8 stats_type; - u8 reserved; -} __packed; +static inline +void ath12k_dp_arch_rx_frags_cleanup(struct ath12k_dp *dp, + struct ath12k_dp_rx_tid *rx_tid, + bool rel_link_desc) +{ + dp->ops->rx_frags_cleanup(rx_tid, rel_link_desc); +} -struct htt_ext_stats_cfg_cmd { - struct htt_ext_stats_cfg_hdr hdr; - __le32 cfg_param0; - __le32 cfg_param1; - __le32 cfg_param2; - __le32 cfg_param3; - __le32 reserved; - __le32 cookie_lsb; - __le32 cookie_msb; -} __packed; +static inline int ath12k_dp_arch_peer_rx_tid_reo_update(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer, + struct ath12k_dp_rx_tid *rx_tid, + u32 ba_win_sz, u16 ssn, + bool update_ssn) +{ + return dp->ops->peer_rx_tid_reo_update(dp, peer, rx_tid, + ba_win_sz, ssn, update_ssn); +} -/* htt stats config default params */ -#define HTT_STAT_DEFAULT_RESET_START_OFFSET 0 -#define HTT_STAT_DEFAULT_CFG0_ALL_HWQS 0xffffffff -#define HTT_STAT_DEFAULT_CFG0_ALL_TXQS 0xffffffff -#define HTT_STAT_DEFAULT_CFG0_ALL_CMDQS 0xffff -#define HTT_STAT_DEFAULT_CFG0_ALL_RINGS 0xffff -#define HTT_STAT_DEFAULT_CFG0_ACTIVE_PEERS 0xff -#define HTT_STAT_DEFAULT_CFG0_CCA_CUMULATIVE 0x00 -#define HTT_STAT_DEFAULT_CFG0_ACTIVE_VDEVS 0x00 - -/* HTT_DBG_EXT_STATS_PEER_INFO - * PARAMS: - * @config_param0: - * [Bit0] - [0] for sw_peer_id, [1] for mac_addr based request - * [Bit15 : Bit 1] htt_peer_stats_req_mode_t - * [Bit31 : Bit16] sw_peer_id - * @config_param1: - * peer_stats_req_type_mask:32 (enum htt_peer_stats_tlv_enum) - * 0 bit htt_peer_stats_cmn_tlv - * 1 bit htt_peer_details_tlv - * 2 bit htt_tx_peer_rate_stats_tlv - * 3 bit htt_rx_peer_rate_stats_tlv - * 4 bit htt_tx_tid_stats_tlv/htt_tx_tid_stats_v1_tlv - * 5 bit htt_rx_tid_stats_tlv - * 6 bit htt_msdu_flow_stats_tlv - * @config_param2: [Bit31 : Bit0] mac_addr31to0 - * @config_param3: [Bit15 : Bit0] mac_addr47to32 - * [Bit31 : Bit16] reserved - */ -#define HTT_STAT_PEER_INFO_MAC_ADDR BIT(0) -#define HTT_STAT_DEFAULT_PEER_REQ_TYPE 0x7f - -/* Used to set different configs to the specified stats type.*/ -struct htt_ext_stats_cfg_params { - u32 cfg0; - u32 cfg1; - u32 cfg2; - u32 cfg3; -}; +static inline int ath12k_dp_arch_rx_assign_reoq(struct ath12k_dp *dp, + struct ath12k_dp_peer *dp_peer, + struct ath12k_dp_rx_tid *rx_tid, + u16 ssn, enum hal_pn_type pn_type) +{ + return dp->ops->rx_assign_reoq(dp->ab, dp_peer, rx_tid, ssn, pn_type); +} -enum vdev_stats_offload_timer_duration { - ATH12K_STATS_TIMER_DUR_500MS = 1, - ATH12K_STATS_TIMER_DUR_1SEC = 2, - ATH12K_STATS_TIMER_DUR_2SEC = 3, -}; +static inline void ath12k_dp_arch_peer_rx_tid_qref_setup(struct ath12k_dp *dp, + u16 peer_id, u16 tid, + dma_addr_t paddr) +{ + dp->ops->peer_rx_tid_qref_setup(dp->ab, peer_id, tid, paddr); +} -#define ATH12K_HTT_MAC_ADDR_L32_0 GENMASK(7, 0) -#define ATH12K_HTT_MAC_ADDR_L32_1 GENMASK(15, 8) -#define ATH12K_HTT_MAC_ADDR_L32_2 GENMASK(23, 16) -#define ATH12K_HTT_MAC_ADDR_L32_3 GENMASK(31, 24) -#define ATH12K_HTT_MAC_ADDR_H16_0 GENMASK(7, 0) -#define ATH12K_HTT_MAC_ADDR_H16_1 GENMASK(15, 8) +static inline void ath12k_dp_arch_peer_rx_tid_qref_reset(struct ath12k_dp *dp, + u16 peer_id, u16 tid) +{ + dp->ops->peer_rx_tid_qref_reset(dp->ab, peer_id, tid); +} -struct htt_mac_addr { - __le32 mac_addr_l32; - __le32 mac_addr_h16; -} __packed; +static inline +int ath12k_dp_arch_rx_tid_delete_handler(struct ath12k_dp *dp, + struct ath12k_dp_rx_tid_rxq *rx_tid) +{ + return dp->ops->rx_tid_delete_handler(dp->ab, rx_tid); +} static inline void ath12k_dp_get_mac_addr(u32 addr_l32, u16 addr_h16, u8 *addr) { @@ -1940,20 +651,39 @@ static inline void ath12k_dp_get_mac_addr(u32 addr_l32, u16 addr_h16, u8 *addr) memcpy(addr + 4, &addr_h16, ETH_ALEN - 4); } -int ath12k_dp_service_srng(struct ath12k_base *ab, - struct ath12k_ext_irq_grp *irq_grp, - int budget); -int ath12k_dp_htt_connect(struct ath12k_dp *dp); +static inline struct ath12k_dp * +ath12k_dp_hw_grp_to_dp(struct ath12k_dp_hw_group *dp_hw_grp, u8 device_id) +{ + return dp_hw_grp->dp[device_id]; +} + +static inline int +ath12k_dp_service_srng(struct ath12k_dp *dp, struct ath12k_ext_irq_grp *irq_grp, + int budget) +{ + return dp->ops->service_srng(dp, irq_grp, budget); +} + +static inline struct ieee80211_hw * +ath12k_pdev_dp_to_hw(struct ath12k_pdev_dp *pdev) +{ + return pdev->hw; +} + +static inline struct ath12k_pdev_dp * +ath12k_dp_to_pdev_dp(struct ath12k_dp *dp, u8 pdev_idx) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "ath12k dp to dp pdev called without rcu lock"); + + return rcu_dereference(dp->dp_pdevs[pdev_idx]); +} + void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif); -void ath12k_dp_free(struct ath12k_base *ab); -int ath12k_dp_alloc(struct ath12k_base *ab); -void ath12k_dp_cc_config(struct ath12k_base *ab); void ath12k_dp_partner_cc_init(struct ath12k_base *ab); int ath12k_dp_pdev_alloc(struct ath12k_base *ab); void ath12k_dp_pdev_pre_alloc(struct ath12k *ar); void ath12k_dp_pdev_free(struct ath12k_base *ab); -int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id, - int mac_id, enum hal_ring_type ring_type); int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr); void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr); void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring); @@ -1967,10 +697,8 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab, struct dp_link_desc_bank *link_desc_banks, u32 ring_type, struct hal_srng *srng, u32 n_link_desc); -struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab, +struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp, u32 cookie); -struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab, +struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp, u32 desc_id); -bool ath12k_dp_wmask_compaction_rx_tlv_supported(struct ath12k_base *ab); -void ath12k_dp_hal_rx_desc_init(struct ath12k_base *ab); #endif diff --git a/drivers/net/wireless/ath/ath12k/dp_cmn.h b/drivers/net/wireless/ath/ath12k/dp_cmn.h new file mode 100644 index 0000000000000..e17f044ff8122 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/dp_cmn.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_DP_CMN_H +#define ATH12K_DP_CMN_H + +#include "cmn_defs.h" + +struct ath12k_hw_group; + +/* + * ML Peer IDs start from 8192, assuming max SLO clients count 1536, + * then max peer id shall be 9728, therefore rounding the peer table size + * to the nearest next power of 2 i.e 16384. + */ +#define MAX_DP_PEER_LIST_SIZE 16384 + +struct ath12k_dp_hw { + struct ath12k_dp_peer __rcu *dp_peers[MAX_DP_PEER_LIST_SIZE]; + + /* Lock for protection of dp_peer_list and peers */ + spinlock_t peer_lock; + struct list_head dp_peers_list; +}; + +struct ath12k_dp_hw_group { + struct ath12k_dp *dp[ATH12K_MAX_DEVICES]; +}; + +struct ath12k_dp_link_vif { + u32 vdev_id; + u8 search_type; + u8 hal_addr_search_flags; + u8 pdev_idx; + u8 lmac_id; + u16 ast_idx; + u16 ast_hash; + u16 tcl_metadata; + u8 vdev_id_check_en; + int bank_id; +}; + +struct ath12k_dp_vif { + u8 tx_encap_type; + u32 key_cipher; + atomic_t mcbc_gsn; + struct ath12k_dp_link_vif dp_link_vif[ATH12K_NUM_MAX_LINKS]; +}; + +/* TODO: Move this to a separate dp_stats file */ +struct ath12k_per_peer_tx_stats { + u32 succ_bytes; + u32 retry_bytes; + u32 failed_bytes; + u32 duration; + u16 succ_pkts; + u16 retry_pkts; + u16 failed_pkts; + u16 ru_start; + u16 ru_tones; + u8 ba_fails; + u8 ppdu_type; + u32 mu_grpid; + u32 mu_pos; + bool is_ampdu; +}; + +struct ath12k_dp_peer_create_params { + struct ieee80211_sta *sta; + bool is_mlo; + u16 peer_id; + bool ucast_ra_only; +}; + +struct ath12k_dp_link_peer_rate_info { + struct rate_info txrate; + u64 rx_duration; + u64 tx_duration; + u8 rssi_comb; + s8 signal_avg; +}; + +static inline struct ath12k_dp_link_vif * +ath12k_dp_vif_to_dp_link_vif(struct ath12k_dp_vif *dp_vif, u8 link_id) +{ + return &dp_vif->dp_link_vif[link_id]; +} + +void ath12k_dp_cmn_device_deinit(struct ath12k_dp *dp); +int ath12k_dp_cmn_device_init(struct ath12k_dp *dp); +void ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp *dp, + struct ath12k_hw_group *ag); +void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp, + struct ath12k_hw_group *ag); +int ath12k_dp_link_peer_assign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw, + u8 vdev_id, struct ieee80211_sta *sta, u8 *addr, + u8 link_id, u32 hw_link_id); +void ath12k_dp_link_peer_unassign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw, + u8 vdev_id, u8 *addr, u32 hw_link_id); +void +ath12k_dp_link_peer_get_sta_rate_info_stats(struct ath12k_dp *dp, const u8 *addr, + struct ath12k_dp_link_peer_rate_info *info); +void ath12k_dp_link_peer_reset_rx_stats(struct ath12k_dp *dp, const u8 *addr); +#endif diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.c b/drivers/net/wireless/ath/ath12k/dp_htt.c new file mode 100644 index 0000000000000..cc71c5c5de5a1 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/dp_htt.c @@ -0,0 +1,1353 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "core.h" +#include "peer.h" +#include "htc.h" +#include "dp_htt.h" +#include "debugfs_htt_stats.h" +#include "debugfs.h" + +static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab, + struct sk_buff *skb) +{ + dev_kfree_skb_any(skb); +} + +int ath12k_dp_htt_connect(struct ath12k_dp *dp) +{ + struct ath12k_htc_svc_conn_req conn_req = {}; + struct ath12k_htc_svc_conn_resp conn_resp = {}; + int status; + + conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete; + conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler; + + /* connect to control service */ + conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG; + + status = ath12k_htc_connect_service(&dp->ab->htc, &conn_req, + &conn_resp); + + if (status) + return status; + + dp->eid = conn_resp.eid; + + return 0; +} + +static int ath12k_get_ppdu_user_index(struct htt_ppdu_stats *ppdu_stats, + u16 peer_id) +{ + int i; + + for (i = 0; i < HTT_PPDU_STATS_MAX_USERS - 1; i++) { + if (ppdu_stats->user_stats[i].is_valid_peer_id) { + if (peer_id == ppdu_stats->user_stats[i].peer_id) + return i; + } else { + return i; + } + } + + return -EINVAL; +} + +static int ath12k_htt_tlv_ppdu_stats_parse(struct ath12k_base *ab, + u16 tag, u16 len, const void *ptr, + void *data) +{ + const struct htt_ppdu_stats_usr_cmpltn_ack_ba_status *ba_status; + const struct htt_ppdu_stats_usr_cmpltn_cmn *cmplt_cmn; + const struct htt_ppdu_stats_user_rate *user_rate; + struct htt_ppdu_stats_info *ppdu_info; + struct htt_ppdu_user_stats *user_stats; + int cur_user; + u16 peer_id; + + ppdu_info = data; + + switch (tag) { + case HTT_PPDU_STATS_TAG_COMMON: + if (len < sizeof(struct htt_ppdu_stats_common)) { + ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", + len, tag); + return -EINVAL; + } + memcpy(&ppdu_info->ppdu_stats.common, ptr, + sizeof(struct htt_ppdu_stats_common)); + break; + case HTT_PPDU_STATS_TAG_USR_RATE: + if (len < sizeof(struct htt_ppdu_stats_user_rate)) { + ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", + len, tag); + return -EINVAL; + } + user_rate = ptr; + peer_id = le16_to_cpu(user_rate->sw_peer_id); + cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats, + peer_id); + if (cur_user < 0) + return -EINVAL; + user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user]; + user_stats->peer_id = peer_id; + user_stats->is_valid_peer_id = true; + memcpy(&user_stats->rate, ptr, + sizeof(struct htt_ppdu_stats_user_rate)); + user_stats->tlv_flags |= BIT(tag); + break; + case HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON: + if (len < sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn)) { + ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", + len, tag); + return -EINVAL; + } + + cmplt_cmn = ptr; + peer_id = le16_to_cpu(cmplt_cmn->sw_peer_id); + cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats, + peer_id); + if (cur_user < 0) + return -EINVAL; + user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user]; + user_stats->peer_id = peer_id; + user_stats->is_valid_peer_id = true; + memcpy(&user_stats->cmpltn_cmn, ptr, + sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn)); + user_stats->tlv_flags |= BIT(tag); + break; + case HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS: + if (len < + sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status)) { + ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", + len, tag); + return -EINVAL; + } + + ba_status = ptr; + peer_id = le16_to_cpu(ba_status->sw_peer_id); + cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats, + peer_id); + if (cur_user < 0) + return -EINVAL; + user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user]; + user_stats->peer_id = peer_id; + user_stats->is_valid_peer_id = true; + memcpy(&user_stats->ack_ba, ptr, + sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status)); + user_stats->tlv_flags |= BIT(tag); + break; + } + return 0; +} + +int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len, + int (*iter)(struct ath12k_base *ar, u16 tag, u16 len, + const void *ptr, void *data), + void *data) +{ + const struct htt_tlv *tlv; + const void *begin = ptr; + u16 tlv_tag, tlv_len; + int ret = -EINVAL; + + while (len > 0) { + if (len < sizeof(*tlv)) { + ath12k_err(ab, "htt tlv parse failure at byte %zd (%zu bytes left, %zu expected)\n", + ptr - begin, len, sizeof(*tlv)); + return -EINVAL; + } + tlv = (struct htt_tlv *)ptr; + tlv_tag = le32_get_bits(tlv->header, HTT_TLV_TAG); + tlv_len = le32_get_bits(tlv->header, HTT_TLV_LEN); + ptr += sizeof(*tlv); + len -= sizeof(*tlv); + + if (tlv_len > len) { + ath12k_err(ab, "htt tlv parse failure of tag %u at byte %zd (%zu bytes left, %u expected)\n", + tlv_tag, ptr - begin, len, tlv_len); + return -EINVAL; + } + ret = iter(ab, tlv_tag, tlv_len, ptr, data); + if (ret == -ENOMEM) + return ret; + + ptr += tlv_len; + len -= tlv_len; + } + return 0; +} + +static void +ath12k_update_per_peer_tx_stats(struct ath12k_pdev_dp *dp_pdev, + struct htt_ppdu_stats *ppdu_stats, u8 user) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; + struct ath12k_dp_link_peer *peer; + struct htt_ppdu_stats_user_rate *user_rate; + struct ath12k_per_peer_tx_stats *peer_stats = &dp_pdev->peer_tx_stats; + struct htt_ppdu_user_stats *usr_stats = &ppdu_stats->user_stats[user]; + struct htt_ppdu_stats_common *common = &ppdu_stats->common; + int ret; + u8 flags, mcs, nss, bw, sgi, dcm, ppdu_type, rate_idx = 0; + u32 v, succ_bytes = 0; + u16 tones, rate = 0, succ_pkts = 0; + u32 tx_duration = 0; + u8 tid = HTT_PPDU_STATS_NON_QOS_TID; + u16 tx_retry_failed = 0, tx_retry_count = 0; + bool is_ampdu = false, is_ofdma; + + if (!(usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_RATE))) + return; + + if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) { + is_ampdu = + HTT_USR_CMPLTN_IS_AMPDU(usr_stats->cmpltn_cmn.flags); + tx_retry_failed = + __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) - + __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success); + tx_retry_count = + HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) + + HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags); + } + + if (usr_stats->tlv_flags & + BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS)) { + succ_bytes = le32_to_cpu(usr_stats->ack_ba.success_bytes); + succ_pkts = le32_get_bits(usr_stats->ack_ba.info, + HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M); + tid = le32_get_bits(usr_stats->ack_ba.info, + HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM); + } + + if (common->fes_duration_us) + tx_duration = le32_to_cpu(common->fes_duration_us); + + user_rate = &usr_stats->rate; + flags = HTT_USR_RATE_PREAMBLE(user_rate->rate_flags); + bw = HTT_USR_RATE_BW(user_rate->rate_flags) - 2; + nss = HTT_USR_RATE_NSS(user_rate->rate_flags) + 1; + mcs = HTT_USR_RATE_MCS(user_rate->rate_flags); + sgi = HTT_USR_RATE_GI(user_rate->rate_flags); + dcm = HTT_USR_RATE_DCM(user_rate->rate_flags); + + ppdu_type = HTT_USR_RATE_PPDU_TYPE(user_rate->info1); + is_ofdma = (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA) || + (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA); + + /* Note: If host configured fixed rates and in some other special + * cases, the broadcast/management frames are sent in different rates. + * Firmware rate's control to be skipped for this? + */ + + if (flags == WMI_RATE_PREAMBLE_HE && mcs > ATH12K_HE_MCS_MAX) { + ath12k_warn(ab, "Invalid HE mcs %d peer stats", mcs); + return; + } + + if (flags == WMI_RATE_PREAMBLE_VHT && mcs > ATH12K_VHT_MCS_MAX) { + ath12k_warn(ab, "Invalid VHT mcs %d peer stats", mcs); + return; + } + + if (flags == WMI_RATE_PREAMBLE_HT && (mcs > ATH12K_HT_MCS_MAX || nss < 1)) { + ath12k_warn(ab, "Invalid HT mcs %d nss %d peer stats", + mcs, nss); + return; + } + + if (flags == WMI_RATE_PREAMBLE_CCK || flags == WMI_RATE_PREAMBLE_OFDM) { + ret = ath12k_mac_hw_ratecode_to_legacy_rate(mcs, + flags, + &rate_idx, + &rate); + if (ret < 0) + return; + } + + rcu_read_lock(); + peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, usr_stats->peer_id); + + if (!peer || !peer->sta) { + rcu_read_unlock(); + return; + } + + spin_lock_bh(&dp->dp_lock); + + memset(&peer->txrate, 0, sizeof(peer->txrate)); + + peer->txrate.bw = ath12k_mac_bw_to_mac80211_bw(bw); + + switch (flags) { + case WMI_RATE_PREAMBLE_OFDM: + peer->txrate.legacy = rate; + break; + case WMI_RATE_PREAMBLE_CCK: + peer->txrate.legacy = rate; + break; + case WMI_RATE_PREAMBLE_HT: + peer->txrate.mcs = mcs + 8 * (nss - 1); + peer->txrate.flags = RATE_INFO_FLAGS_MCS; + if (sgi) + peer->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; + break; + case WMI_RATE_PREAMBLE_VHT: + peer->txrate.mcs = mcs; + peer->txrate.flags = RATE_INFO_FLAGS_VHT_MCS; + if (sgi) + peer->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; + break; + case WMI_RATE_PREAMBLE_HE: + peer->txrate.mcs = mcs; + peer->txrate.flags = RATE_INFO_FLAGS_HE_MCS; + peer->txrate.he_dcm = dcm; + peer->txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(sgi); + tones = le16_to_cpu(user_rate->ru_end) - + le16_to_cpu(user_rate->ru_start) + 1; + v = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(tones); + peer->txrate.he_ru_alloc = v; + if (is_ofdma) + peer->txrate.bw = RATE_INFO_BW_HE_RU; + break; + case WMI_RATE_PREAMBLE_EHT: + peer->txrate.mcs = mcs; + peer->txrate.flags = RATE_INFO_FLAGS_EHT_MCS; + peer->txrate.he_dcm = dcm; + peer->txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(sgi); + tones = le16_to_cpu(user_rate->ru_end) - + le16_to_cpu(user_rate->ru_start) + 1; + v = ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(tones); + peer->txrate.eht_ru_alloc = v; + if (is_ofdma) + peer->txrate.bw = RATE_INFO_BW_EHT_RU; + break; + } + + peer->tx_retry_failed += tx_retry_failed; + peer->tx_retry_count += tx_retry_count; + peer->txrate.nss = nss; + peer->tx_duration += tx_duration; + memcpy(&peer->last_txrate, &peer->txrate, sizeof(struct rate_info)); + + spin_unlock_bh(&dp->dp_lock); + + /* PPDU stats reported for mgmt packet doesn't have valid tx bytes. + * So skip peer stats update for mgmt packets. + */ + if (tid < HTT_PPDU_STATS_NON_QOS_TID) { + memset(peer_stats, 0, sizeof(*peer_stats)); + peer_stats->succ_pkts = succ_pkts; + peer_stats->succ_bytes = succ_bytes; + peer_stats->is_ampdu = is_ampdu; + peer_stats->duration = tx_duration; + peer_stats->ba_fails = + HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) + + HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags); + } + + rcu_read_unlock(); +} + +static void ath12k_htt_update_ppdu_stats(struct ath12k_pdev_dp *dp_pdev, + struct htt_ppdu_stats *ppdu_stats) +{ + u8 user; + + for (user = 0; user < HTT_PPDU_STATS_MAX_USERS - 1; user++) + ath12k_update_per_peer_tx_stats(dp_pdev, ppdu_stats, user); +} + +static +struct htt_ppdu_stats_info *ath12k_dp_htt_get_ppdu_desc(struct ath12k_pdev_dp *dp_pdev, + u32 ppdu_id) +{ + struct htt_ppdu_stats_info *ppdu_info; + + lockdep_assert_held(&dp_pdev->ppdu_list_lock); + if (!list_empty(&dp_pdev->ppdu_stats_info)) { + list_for_each_entry(ppdu_info, &dp_pdev->ppdu_stats_info, list) { + if (ppdu_info->ppdu_id == ppdu_id) + return ppdu_info; + } + + if (dp_pdev->ppdu_stat_list_depth > HTT_PPDU_DESC_MAX_DEPTH) { + ppdu_info = list_first_entry(&dp_pdev->ppdu_stats_info, + typeof(*ppdu_info), list); + list_del(&ppdu_info->list); + dp_pdev->ppdu_stat_list_depth--; + ath12k_htt_update_ppdu_stats(dp_pdev, &ppdu_info->ppdu_stats); + kfree(ppdu_info); + } + } + + ppdu_info = kzalloc(sizeof(*ppdu_info), GFP_ATOMIC); + if (!ppdu_info) + return NULL; + + list_add_tail(&ppdu_info->list, &dp_pdev->ppdu_stats_info); + dp_pdev->ppdu_stat_list_depth++; + + return ppdu_info; +} + +static void ath12k_copy_to_delay_stats(struct ath12k_dp_link_peer *peer, + struct htt_ppdu_user_stats *usr_stats) +{ + peer->ppdu_stats_delayba.sw_peer_id = le16_to_cpu(usr_stats->rate.sw_peer_id); + peer->ppdu_stats_delayba.info0 = le32_to_cpu(usr_stats->rate.info0); + peer->ppdu_stats_delayba.ru_end = le16_to_cpu(usr_stats->rate.ru_end); + peer->ppdu_stats_delayba.ru_start = le16_to_cpu(usr_stats->rate.ru_start); + peer->ppdu_stats_delayba.info1 = le32_to_cpu(usr_stats->rate.info1); + peer->ppdu_stats_delayba.rate_flags = le32_to_cpu(usr_stats->rate.rate_flags); + peer->ppdu_stats_delayba.resp_rate_flags = + le32_to_cpu(usr_stats->rate.resp_rate_flags); + + peer->delayba_flag = true; +} + +static void ath12k_copy_to_bar(struct ath12k_dp_link_peer *peer, + struct htt_ppdu_user_stats *usr_stats) +{ + usr_stats->rate.sw_peer_id = cpu_to_le16(peer->ppdu_stats_delayba.sw_peer_id); + usr_stats->rate.info0 = cpu_to_le32(peer->ppdu_stats_delayba.info0); + usr_stats->rate.ru_end = cpu_to_le16(peer->ppdu_stats_delayba.ru_end); + usr_stats->rate.ru_start = cpu_to_le16(peer->ppdu_stats_delayba.ru_start); + usr_stats->rate.info1 = cpu_to_le32(peer->ppdu_stats_delayba.info1); + usr_stats->rate.rate_flags = cpu_to_le32(peer->ppdu_stats_delayba.rate_flags); + usr_stats->rate.resp_rate_flags = + cpu_to_le32(peer->ppdu_stats_delayba.resp_rate_flags); + + peer->delayba_flag = false; +} + +static int ath12k_htt_pull_ppdu_stats(struct ath12k_base *ab, + struct sk_buff *skb) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_htt_ppdu_stats_msg *msg; + struct htt_ppdu_stats_info *ppdu_info; + struct ath12k_dp_link_peer *peer = NULL; + struct htt_ppdu_user_stats *usr_stats = NULL; + u32 peer_id = 0; + struct ath12k_pdev_dp *dp_pdev; + int ret, i; + u8 pdev_id, pdev_idx; + u32 ppdu_id, len; + + msg = (struct ath12k_htt_ppdu_stats_msg *)skb->data; + len = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE); + if (len > (skb->len - struct_size(msg, data, 0))) { + ath12k_warn(ab, + "HTT PPDU STATS event has unexpected payload size %u, should be smaller than %u\n", + len, skb->len); + return -EINVAL; + } + + pdev_id = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PDEV_ID); + ppdu_id = le32_to_cpu(msg->ppdu_id); + + pdev_idx = DP_HW2SW_MACID(pdev_id); + if (pdev_idx >= MAX_RADIOS) { + ath12k_warn(ab, "HTT PPDU STATS invalid pdev id %u", pdev_id); + return -EINVAL; + } + + rcu_read_lock(); + + dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx); + if (!dp_pdev) { + ret = -EINVAL; + goto exit; + } + + spin_lock_bh(&dp_pdev->ppdu_list_lock); + ppdu_info = ath12k_dp_htt_get_ppdu_desc(dp_pdev, ppdu_id); + if (!ppdu_info) { + spin_unlock_bh(&dp_pdev->ppdu_list_lock); + ret = -EINVAL; + goto exit; + } + + ppdu_info->ppdu_id = ppdu_id; + ret = ath12k_dp_htt_tlv_iter(ab, msg->data, len, + ath12k_htt_tlv_ppdu_stats_parse, + (void *)ppdu_info); + if (ret) { + spin_unlock_bh(&dp_pdev->ppdu_list_lock); + ath12k_warn(ab, "Failed to parse tlv %d\n", ret); + goto exit; + } + + if (ppdu_info->ppdu_stats.common.num_users >= HTT_PPDU_STATS_MAX_USERS) { + spin_unlock_bh(&dp_pdev->ppdu_list_lock); + ath12k_warn(ab, + "HTT PPDU STATS event has unexpected num_users %u, should be smaller than %u\n", + ppdu_info->ppdu_stats.common.num_users, + HTT_PPDU_STATS_MAX_USERS); + ret = -EINVAL; + goto exit; + } + + /* back up data rate tlv for all peers */ + if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_DATA && + (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON)) && + ppdu_info->delay_ba) { + for (i = 0; i < ppdu_info->ppdu_stats.common.num_users; i++) { + peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id; + peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, peer_id); + if (!peer) + continue; + + usr_stats = &ppdu_info->ppdu_stats.user_stats[i]; + if (usr_stats->delay_ba) + ath12k_copy_to_delay_stats(peer, usr_stats); + } + } + + /* restore all peers' data rate tlv to mu-bar tlv */ + if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_BAR && + (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON))) { + for (i = 0; i < ppdu_info->bar_num_users; i++) { + peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id; + peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, peer_id); + if (!peer) + continue; + + usr_stats = &ppdu_info->ppdu_stats.user_stats[i]; + if (peer->delayba_flag) + ath12k_copy_to_bar(peer, usr_stats); + } + } + + spin_unlock_bh(&dp_pdev->ppdu_list_lock); + +exit: + rcu_read_unlock(); + + return ret; +} + +static void ath12k_htt_mlo_offset_event_handler(struct ath12k_base *ab, + struct sk_buff *skb) +{ + struct ath12k_htt_mlo_offset_msg *msg; + struct ath12k_pdev *pdev; + struct ath12k *ar; + u8 pdev_id; + + msg = (struct ath12k_htt_mlo_offset_msg *)skb->data; + pdev_id = u32_get_bits(__le32_to_cpu(msg->info), + HTT_T2H_MLO_OFFSET_INFO_PDEV_ID); + + rcu_read_lock(); + ar = ath12k_mac_get_ar_by_pdev_id(ab, pdev_id); + if (!ar) { + /* It is possible that the ar is not yet active (started). + * The above function will only look for the active pdev + * and hence %NULL return is possible. Just silently + * discard this message + */ + goto exit; + } + + spin_lock_bh(&ar->data_lock); + pdev = ar->pdev; + + pdev->timestamp.info = __le32_to_cpu(msg->info); + pdev->timestamp.sync_timestamp_lo_us = __le32_to_cpu(msg->sync_timestamp_lo_us); + pdev->timestamp.sync_timestamp_hi_us = __le32_to_cpu(msg->sync_timestamp_hi_us); + pdev->timestamp.mlo_offset_lo = __le32_to_cpu(msg->mlo_offset_lo); + pdev->timestamp.mlo_offset_hi = __le32_to_cpu(msg->mlo_offset_hi); + pdev->timestamp.mlo_offset_clks = __le32_to_cpu(msg->mlo_offset_clks); + pdev->timestamp.mlo_comp_clks = __le32_to_cpu(msg->mlo_comp_clks); + pdev->timestamp.mlo_comp_timer = __le32_to_cpu(msg->mlo_comp_timer); + + spin_unlock_bh(&ar->data_lock); +exit: + rcu_read_unlock(); +} + +void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab, + struct sk_buff *skb) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_resp_msg *resp = (struct htt_resp_msg *)skb->data; + enum htt_t2h_msg_type type; + u16 peer_id; + u8 vdev_id; + u8 mac_addr[ETH_ALEN]; + u16 peer_mac_h16; + u16 ast_hash = 0; + u16 hw_peer_id; + + type = le32_get_bits(resp->version_msg.version, HTT_T2H_MSG_TYPE); + + ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt rx msg type :0x%0x\n", type); + + switch (type) { + case HTT_T2H_MSG_TYPE_VERSION_CONF: + dp->htt_tgt_ver_major = le32_get_bits(resp->version_msg.version, + HTT_T2H_VERSION_CONF_MAJOR); + dp->htt_tgt_ver_minor = le32_get_bits(resp->version_msg.version, + HTT_T2H_VERSION_CONF_MINOR); + complete(&dp->htt_tgt_version_received); + break; + /* TODO: remove unused peer map versions after testing */ + case HTT_T2H_MSG_TYPE_PEER_MAP: + vdev_id = le32_get_bits(resp->peer_map_ev.info, + HTT_T2H_PEER_MAP_INFO_VDEV_ID); + peer_id = le32_get_bits(resp->peer_map_ev.info, + HTT_T2H_PEER_MAP_INFO_PEER_ID); + peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1, + HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16); + ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32), + peer_mac_h16, mac_addr); + ath12k_dp_link_peer_map_event(ab, vdev_id, peer_id, mac_addr, 0, 0); + break; + case HTT_T2H_MSG_TYPE_PEER_MAP2: + vdev_id = le32_get_bits(resp->peer_map_ev.info, + HTT_T2H_PEER_MAP_INFO_VDEV_ID); + peer_id = le32_get_bits(resp->peer_map_ev.info, + HTT_T2H_PEER_MAP_INFO_PEER_ID); + peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1, + HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16); + ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32), + peer_mac_h16, mac_addr); + ast_hash = le32_get_bits(resp->peer_map_ev.info2, + HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL); + hw_peer_id = le32_get_bits(resp->peer_map_ev.info1, + HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID); + ath12k_dp_link_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash, + hw_peer_id); + break; + case HTT_T2H_MSG_TYPE_PEER_MAP3: + vdev_id = le32_get_bits(resp->peer_map_ev.info, + HTT_T2H_PEER_MAP_INFO_VDEV_ID); + peer_id = le32_get_bits(resp->peer_map_ev.info, + HTT_T2H_PEER_MAP_INFO_PEER_ID); + peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1, + HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16); + ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32), + peer_mac_h16, mac_addr); + ast_hash = le32_get_bits(resp->peer_map_ev.info2, + HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL); + hw_peer_id = le32_get_bits(resp->peer_map_ev.info2, + HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID); + ath12k_dp_link_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash, + hw_peer_id); + break; + case HTT_T2H_MSG_TYPE_PEER_UNMAP: + case HTT_T2H_MSG_TYPE_PEER_UNMAP2: + peer_id = le32_get_bits(resp->peer_unmap_ev.info, + HTT_T2H_PEER_UNMAP_INFO_PEER_ID); + ath12k_dp_link_peer_unmap_event(ab, peer_id); + break; + case HTT_T2H_MSG_TYPE_PPDU_STATS_IND: + ath12k_htt_pull_ppdu_stats(ab, skb); + break; + case HTT_T2H_MSG_TYPE_EXT_STATS_CONF: + ath12k_debugfs_htt_ext_stats_handler(ab, skb); + break; + case HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND: + ath12k_htt_mlo_offset_event_handler(ab, skb); + break; + default: + ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt event %d not handled\n", + type); + break; + } + + dev_kfree_skb_any(skb); +} +EXPORT_SYMBOL(ath12k_dp_htt_htc_t2h_msg_handler); + +static int +ath12k_dp_tx_get_ring_id_type(struct ath12k_base *ab, + int mac_id, u32 ring_id, + enum hal_ring_type ring_type, + enum htt_srng_ring_type *htt_ring_type, + enum htt_srng_ring_id *htt_ring_id) +{ + int ret = 0; + + switch (ring_type) { + case HAL_RXDMA_BUF: + /* for some targets, host fills rx buffer to fw and fw fills to + * rxbuf ring for each rxdma + */ + if (!ab->hw_params->rx_mac_buf_ring) { + if (!(ring_id == HAL_SRNG_SW2RXDMA_BUF0 || + ring_id == HAL_SRNG_SW2RXDMA_BUF1)) { + ret = -EINVAL; + } + *htt_ring_id = HTT_RXDMA_HOST_BUF_RING; + *htt_ring_type = HTT_SW_TO_HW_RING; + } else { + if (ring_id == HAL_SRNG_SW2RXDMA_BUF0) { + *htt_ring_id = HTT_HOST1_TO_FW_RXBUF_RING; + *htt_ring_type = HTT_SW_TO_SW_RING; + } else { + *htt_ring_id = HTT_RXDMA_HOST_BUF_RING; + *htt_ring_type = HTT_SW_TO_HW_RING; + } + } + break; + case HAL_RXDMA_DST: + *htt_ring_id = HTT_RXDMA_NON_MONITOR_DEST_RING; + *htt_ring_type = HTT_HW_TO_SW_RING; + break; + case HAL_RXDMA_MONITOR_BUF: + *htt_ring_id = HTT_RX_MON_HOST2MON_BUF_RING; + *htt_ring_type = HTT_SW_TO_HW_RING; + break; + case HAL_RXDMA_MONITOR_STATUS: + *htt_ring_id = HTT_RXDMA_MONITOR_STATUS_RING; + *htt_ring_type = HTT_SW_TO_HW_RING; + break; + case HAL_RXDMA_MONITOR_DST: + *htt_ring_id = HTT_RX_MON_MON2HOST_DEST_RING; + *htt_ring_type = HTT_HW_TO_SW_RING; + break; + case HAL_RXDMA_MONITOR_DESC: + *htt_ring_id = HTT_RXDMA_MONITOR_DESC_RING; + *htt_ring_type = HTT_SW_TO_HW_RING; + break; + default: + ath12k_warn(ab, "Unsupported ring type in DP :%d\n", ring_type); + ret = -EINVAL; + } + return ret; +} + +int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id, + int mac_id, enum hal_ring_type ring_type) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_srng_setup_cmd *cmd; + struct hal_srng *srng = &ab->hal.srng_list[ring_id]; + struct hal_srng_params params; + struct sk_buff *skb; + u32 ring_entry_sz; + int len = sizeof(*cmd); + dma_addr_t hp_addr, tp_addr; + enum htt_srng_ring_type htt_ring_type; + enum htt_srng_ring_id htt_ring_id; + int ret; + + skb = ath12k_htc_alloc_skb(ab, len); + if (!skb) + return -ENOMEM; + + memset(¶ms, 0, sizeof(params)); + ath12k_hal_srng_get_params(ab, srng, ¶ms); + + hp_addr = ath12k_hal_srng_get_hp_addr(ab, srng); + tp_addr = ath12k_hal_srng_get_tp_addr(ab, srng); + + ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id, + ring_type, &htt_ring_type, + &htt_ring_id); + if (ret) + goto err_free; + + skb_put(skb, len); + cmd = (struct htt_srng_setup_cmd *)skb->data; + cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_SRING_SETUP, + HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE); + if (htt_ring_type == HTT_SW_TO_HW_RING || + htt_ring_type == HTT_HW_TO_SW_RING) + cmd->info0 |= le32_encode_bits(DP_SW2HW_MACID(mac_id), + HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID); + else + cmd->info0 |= le32_encode_bits(mac_id, + HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID); + cmd->info0 |= le32_encode_bits(htt_ring_type, + HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE); + cmd->info0 |= le32_encode_bits(htt_ring_id, + HTT_SRNG_SETUP_CMD_INFO0_RING_ID); + + cmd->ring_base_addr_lo = cpu_to_le32(params.ring_base_paddr & + HAL_ADDR_LSB_REG_MASK); + + cmd->ring_base_addr_hi = cpu_to_le32((u64)params.ring_base_paddr >> + HAL_ADDR_MSB_REG_SHIFT); + + ret = ath12k_hal_srng_get_entrysize(ab, ring_type); + if (ret < 0) + goto err_free; + + ring_entry_sz = ret; + + ring_entry_sz >>= 2; + cmd->info1 = le32_encode_bits(ring_entry_sz, + HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE); + cmd->info1 |= le32_encode_bits(params.num_entries * ring_entry_sz, + HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE); + cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP), + HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP); + cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP), + HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP); + cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_RING_PTR_SWAP), + HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP); + if (htt_ring_type == HTT_SW_TO_HW_RING) + cmd->info1 |= cpu_to_le32(HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS); + + cmd->ring_head_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(hp_addr)); + cmd->ring_head_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(hp_addr)); + + cmd->ring_tail_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(tp_addr)); + cmd->ring_tail_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(tp_addr)); + + cmd->ring_msi_addr_lo = cpu_to_le32(lower_32_bits(params.msi_addr)); + cmd->ring_msi_addr_hi = cpu_to_le32(upper_32_bits(params.msi_addr)); + cmd->msi_data = cpu_to_le32(params.msi_data); + + cmd->intr_info = + le32_encode_bits(params.intr_batch_cntr_thres_entries * ring_entry_sz, + HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH); + cmd->intr_info |= + le32_encode_bits(params.intr_timer_thres_us >> 3, + HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH); + + cmd->info2 = 0; + if (params.flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) { + cmd->info2 = le32_encode_bits(params.low_threshold, + HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH); + } + + ath12k_dbg(ab, ATH12K_DBG_HAL, + "%s msi_addr_lo:0x%x, msi_addr_hi:0x%x, msi_data:0x%x\n", + __func__, cmd->ring_msi_addr_lo, cmd->ring_msi_addr_hi, + cmd->msi_data); + + ath12k_dbg(ab, ATH12K_DBG_HAL, + "ring_id:%d, ring_type:%d, intr_info:0x%x, flags:0x%x\n", + ring_id, ring_type, cmd->intr_info, cmd->info2); + + ret = ath12k_htc_send(&ab->htc, dp->eid, skb); + if (ret) + goto err_free; + + return 0; + +err_free: + dev_kfree_skb_any(skb); + + return ret; +} + +int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct sk_buff *skb; + struct htt_ver_req_cmd *cmd; + int len = sizeof(*cmd); + u32 metadata_version; + int ret; + + init_completion(&dp->htt_tgt_version_received); + + skb = ath12k_htc_alloc_skb(ab, len); + if (!skb) + return -ENOMEM; + + skb_put(skb, len); + cmd = (struct htt_ver_req_cmd *)skb->data; + cmd->ver_reg_info = le32_encode_bits(HTT_H2T_MSG_TYPE_VERSION_REQ, + HTT_OPTION_TAG); + metadata_version = ath12k_ftm_mode ? HTT_OPTION_TCL_METADATA_VER_V1 : + HTT_OPTION_TCL_METADATA_VER_V2; + + cmd->tcl_metadata_version = le32_encode_bits(HTT_TAG_TCL_METADATA_VERSION, + HTT_OPTION_TAG) | + le32_encode_bits(HTT_TCL_METADATA_VER_SZ, + HTT_OPTION_LEN) | + le32_encode_bits(metadata_version, + HTT_OPTION_VALUE); + + ret = ath12k_htc_send(&ab->htc, dp->eid, skb); + if (ret) { + dev_kfree_skb_any(skb); + return ret; + } + + ret = wait_for_completion_timeout(&dp->htt_tgt_version_received, + HTT_TARGET_VERSION_TIMEOUT_HZ); + if (ret == 0) { + ath12k_warn(ab, "htt target version request timed out\n"); + return -ETIMEDOUT; + } + + if (dp->htt_tgt_ver_major != HTT_TARGET_VERSION_MAJOR) { + ath12k_err(ab, "unsupported htt major version %d supported version is %d\n", + dp->htt_tgt_ver_major, HTT_TARGET_VERSION_MAJOR); + return -EOPNOTSUPP; + } + + return 0; +} + +int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask) +{ + struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct sk_buff *skb; + struct htt_ppdu_stats_cfg_cmd *cmd; + int len = sizeof(*cmd); + u8 pdev_mask; + int ret; + int i; + + for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { + skb = ath12k_htc_alloc_skb(ab, len); + if (!skb) + return -ENOMEM; + + skb_put(skb, len); + cmd = (struct htt_ppdu_stats_cfg_cmd *)skb->data; + cmd->msg = le32_encode_bits(HTT_H2T_MSG_TYPE_PPDU_STATS_CFG, + HTT_PPDU_STATS_CFG_MSG_TYPE); + + pdev_mask = 1 << (i + ar->pdev_idx); + cmd->msg |= le32_encode_bits(pdev_mask, HTT_PPDU_STATS_CFG_PDEV_ID); + cmd->msg |= le32_encode_bits(mask, HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK); + + ret = ath12k_htc_send(&ab->htc, dp->eid, skb); + if (ret) { + dev_kfree_skb_any(skb); + return ret; + } + } + + return 0; +} + +int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id, + int mac_id, enum hal_ring_type ring_type, + int rx_buf_size, + struct htt_rx_ring_tlv_filter *tlv_filter) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_rx_ring_selection_cfg_cmd *cmd; + struct hal_srng *srng = &ab->hal.srng_list[ring_id]; + struct hal_srng_params params; + struct sk_buff *skb; + int len = sizeof(*cmd); + enum htt_srng_ring_type htt_ring_type; + enum htt_srng_ring_id htt_ring_id; + int ret; + + skb = ath12k_htc_alloc_skb(ab, len); + if (!skb) + return -ENOMEM; + + memset(¶ms, 0, sizeof(params)); + ath12k_hal_srng_get_params(ab, srng, ¶ms); + + ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id, + ring_type, &htt_ring_type, + &htt_ring_id); + if (ret) + goto err_free; + + skb_put(skb, len); + cmd = (struct htt_rx_ring_selection_cfg_cmd *)skb->data; + cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG, + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE); + if (htt_ring_type == HTT_SW_TO_HW_RING || + htt_ring_type == HTT_HW_TO_SW_RING) + cmd->info0 |= + le32_encode_bits(DP_SW2HW_MACID(mac_id), + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); + else + cmd->info0 |= + le32_encode_bits(mac_id, + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); + cmd->info0 |= le32_encode_bits(htt_ring_id, + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID); + cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP), + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS); + cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP), + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS); + cmd->info0 |= le32_encode_bits(tlv_filter->offset_valid, + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID); + cmd->info0 |= + le32_encode_bits(tlv_filter->drop_threshold_valid, + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL); + cmd->info0 |= le32_encode_bits(!tlv_filter->rxmon_disable, + HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON); + + cmd->info1 = le32_encode_bits(rx_buf_size, + HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE); + cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_mgmt, + HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT); + cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_ctrl, + HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL); + cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_data, + HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA); + cmd->pkt_type_en_flags0 = cpu_to_le32(tlv_filter->pkt_filter_flags0); + cmd->pkt_type_en_flags1 = cpu_to_le32(tlv_filter->pkt_filter_flags1); + cmd->pkt_type_en_flags2 = cpu_to_le32(tlv_filter->pkt_filter_flags2); + cmd->pkt_type_en_flags3 = cpu_to_le32(tlv_filter->pkt_filter_flags3); + cmd->rx_filter_tlv = cpu_to_le32(tlv_filter->rx_filter); + + cmd->info2 = le32_encode_bits(tlv_filter->rx_drop_threshold, + HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD); + cmd->info2 |= + le32_encode_bits(tlv_filter->enable_log_mgmt_type, + HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE); + cmd->info2 |= + le32_encode_bits(tlv_filter->enable_log_ctrl_type, + HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE); + cmd->info2 |= + le32_encode_bits(tlv_filter->enable_log_data_type, + HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE); + + cmd->info3 = + le32_encode_bits(tlv_filter->enable_rx_tlv_offset, + HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET); + cmd->info3 |= + le32_encode_bits(tlv_filter->rx_tlv_offset, + HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET); + + if (tlv_filter->offset_valid) { + cmd->rx_packet_offset = + le32_encode_bits(tlv_filter->rx_packet_offset, + HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET); + + cmd->rx_packet_offset |= + le32_encode_bits(tlv_filter->rx_header_offset, + HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET); + + cmd->rx_mpdu_offset = + le32_encode_bits(tlv_filter->rx_mpdu_end_offset, + HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET); + + cmd->rx_mpdu_offset |= + le32_encode_bits(tlv_filter->rx_mpdu_start_offset, + HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET); + + cmd->rx_msdu_offset = + le32_encode_bits(tlv_filter->rx_msdu_end_offset, + HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET); + + cmd->rx_msdu_offset |= + le32_encode_bits(tlv_filter->rx_msdu_start_offset, + HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET); + + cmd->rx_attn_offset = + le32_encode_bits(tlv_filter->rx_attn_offset, + HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET); + } + + if (tlv_filter->rx_mpdu_start_wmask > 0 && + tlv_filter->rx_msdu_end_wmask > 0) { + cmd->info2 |= + le32_encode_bits(true, + HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET); + cmd->rx_mpdu_start_end_mask = + le32_encode_bits(tlv_filter->rx_mpdu_start_wmask, + HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK); + /* mpdu_end is not used for any hardwares so far + * please assign it in future if any chip is + * using through hal ops + */ + cmd->rx_mpdu_start_end_mask |= + le32_encode_bits(tlv_filter->rx_mpdu_end_wmask, + HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK); + cmd->rx_msdu_end_word_mask = + le32_encode_bits(tlv_filter->rx_msdu_end_wmask, + HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK); + } + + ret = ath12k_htc_send(&ab->htc, dp->eid, skb); + if (ret) + goto err_free; + + return 0; + +err_free: + dev_kfree_skb_any(skb); + + return ret; +} +EXPORT_SYMBOL(ath12k_dp_tx_htt_rx_filter_setup); + +int +ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type, + struct htt_ext_stats_cfg_params *cfg_params, + u64 cookie) +{ + struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct sk_buff *skb; + struct htt_ext_stats_cfg_cmd *cmd; + int len = sizeof(*cmd); + int ret; + u32 pdev_id; + + skb = ath12k_htc_alloc_skb(ab, len); + if (!skb) + return -ENOMEM; + + skb_put(skb, len); + + cmd = (struct htt_ext_stats_cfg_cmd *)skb->data; + memset(cmd, 0, sizeof(*cmd)); + cmd->hdr.msg_type = HTT_H2T_MSG_TYPE_EXT_STATS_CFG; + + pdev_id = ath12k_mac_get_target_pdev_id(ar); + cmd->hdr.pdev_mask = 1 << pdev_id; + + cmd->hdr.stats_type = type; + cmd->cfg_param0 = cpu_to_le32(cfg_params->cfg0); + cmd->cfg_param1 = cpu_to_le32(cfg_params->cfg1); + cmd->cfg_param2 = cpu_to_le32(cfg_params->cfg2); + cmd->cfg_param3 = cpu_to_le32(cfg_params->cfg3); + cmd->cookie_lsb = cpu_to_le32(lower_32_bits(cookie)); + cmd->cookie_msb = cpu_to_le32(upper_32_bits(cookie)); + + ret = ath12k_htc_send(&ab->htc, dp->eid, skb); + if (ret) { + ath12k_warn(ab, "failed to send htt type stats request: %d", + ret); + dev_kfree_skb_any(skb); + return ret; + } + + return 0; +} + +int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset) +{ + struct ath12k_base *ab = ar->ab; + int ret; + + ret = ath12k_dp_tx_htt_rx_monitor_mode_ring_config(ar, reset); + if (ret) { + ath12k_err(ab, "failed to setup rx monitor filter %d\n", ret); + return ret; + } + + return 0; +} + +int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset) +{ + struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_rx_ring_tlv_filter tlv_filter = {}; + int ret, ring_id, i; + + tlv_filter.offset_valid = false; + + if (!reset) { + tlv_filter.rx_filter = HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING; + + tlv_filter.drop_threshold_valid = true; + tlv_filter.rx_drop_threshold = HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE; + + tlv_filter.enable_log_mgmt_type = true; + tlv_filter.enable_log_ctrl_type = true; + tlv_filter.enable_log_data_type = true; + + tlv_filter.conf_len_ctrl = HTT_RX_RING_DEFAULT_DMA_LENGTH; + tlv_filter.conf_len_mgmt = HTT_RX_RING_DEFAULT_DMA_LENGTH; + tlv_filter.conf_len_data = HTT_RX_RING_DEFAULT_DMA_LENGTH; + + tlv_filter.enable_rx_tlv_offset = true; + tlv_filter.rx_tlv_offset = HTT_RX_RING_PKT_TLV_OFFSET; + + tlv_filter.pkt_filter_flags0 = + HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 | + HTT_RX_MON_MO_MGMT_FILTER_FLAGS0; + tlv_filter.pkt_filter_flags1 = + HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 | + HTT_RX_MON_MO_MGMT_FILTER_FLAGS1; + tlv_filter.pkt_filter_flags2 = + HTT_RX_MON_FP_CTRL_FILTER_FLASG2 | + HTT_RX_MON_MO_CTRL_FILTER_FLASG2; + tlv_filter.pkt_filter_flags3 = + HTT_RX_MON_FP_CTRL_FILTER_FLASG3 | + HTT_RX_MON_MO_CTRL_FILTER_FLASG3 | + HTT_RX_MON_FP_DATA_FILTER_FLASG3 | + HTT_RX_MON_MO_DATA_FILTER_FLASG3; + } else { + tlv_filter = ath12k_mac_mon_status_filter_default; + + if (ath12k_debugfs_is_extd_rx_stats_enabled(ar)) + tlv_filter.rx_filter = ath12k_debugfs_rx_filter(ar); + } + + if (ab->hw_params->rxdma1_enable) { + for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { + ring_id = ar->dp.rxdma_mon_dst_ring[i].ring_id; + ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id, + ar->dp.mac_id + i, + HAL_RXDMA_MONITOR_DST, + DP_RXDMA_REFILL_RING_SIZE, + &tlv_filter); + if (ret) { + ath12k_err(ab, + "failed to setup filter for monitor buf %d\n", + ret); + return ret; + } + } + return 0; + } + + if (!reset) { + for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { + ring_id = dp->rx_mac_buf_ring[i].ring_id; + ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id, + i, + HAL_RXDMA_BUF, + DP_RXDMA_REFILL_RING_SIZE, + &tlv_filter); + if (ret) { + ath12k_err(ab, + "failed to setup filter for mon rx buf %d\n", + ret); + return ret; + } + } + } + + for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { + ring_id = dp->rx_mon_status_refill_ring[i].refill_buf_ring.ring_id; + if (!reset) { + tlv_filter.rx_filter = + HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING; + } + + ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, + i, + HAL_RXDMA_MONITOR_STATUS, + RX_MON_STATUS_BUF_SIZE, + &tlv_filter); + if (ret) { + ath12k_err(ab, + "failed to setup filter for mon status buf %d\n", + ret); + return ret; + } + } + + return 0; +} + +int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id, + int mac_id, enum hal_ring_type ring_type, + int tx_buf_size, + struct htt_tx_ring_tlv_filter *htt_tlv_filter) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_tx_ring_selection_cfg_cmd *cmd; + struct hal_srng *srng = &ab->hal.srng_list[ring_id]; + struct hal_srng_params params; + struct sk_buff *skb; + int len = sizeof(*cmd); + enum htt_srng_ring_type htt_ring_type; + enum htt_srng_ring_id htt_ring_id; + int ret; + + skb = ath12k_htc_alloc_skb(ab, len); + if (!skb) + return -ENOMEM; + + memset(¶ms, 0, sizeof(params)); + ath12k_hal_srng_get_params(ab, srng, ¶ms); + + ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id, + ring_type, &htt_ring_type, + &htt_ring_id); + + if (ret) + goto err_free; + + skb_put(skb, len); + cmd = (struct htt_tx_ring_selection_cfg_cmd *)skb->data; + cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_TX_MONITOR_CFG, + HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE); + if (htt_ring_type == HTT_SW_TO_HW_RING || + htt_ring_type == HTT_HW_TO_SW_RING) + cmd->info0 |= + le32_encode_bits(DP_SW2HW_MACID(mac_id), + HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); + else + cmd->info0 |= + le32_encode_bits(mac_id, + HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); + cmd->info0 |= le32_encode_bits(htt_ring_id, + HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID); + cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP), + HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS); + cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP), + HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS); + + cmd->info1 |= + le32_encode_bits(tx_buf_size, + HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE); + + if (htt_tlv_filter->tx_mon_mgmt_filter) { + cmd->info1 |= + le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT, + HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE); + cmd->info1 |= + le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len, + HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT); + cmd->info2 |= + le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT, + HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG); + } + + if (htt_tlv_filter->tx_mon_data_filter) { + cmd->info1 |= + le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL, + HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE); + cmd->info1 |= + le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len, + HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL); + cmd->info2 |= + le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL, + HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG); + } + + if (htt_tlv_filter->tx_mon_ctrl_filter) { + cmd->info1 |= + le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA, + HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE); + cmd->info1 |= + le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len, + HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA); + cmd->info2 |= + le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA, + HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG); + } + + cmd->tlv_filter_mask_in0 = + cpu_to_le32(htt_tlv_filter->tx_mon_downstream_tlv_flags); + cmd->tlv_filter_mask_in1 = + cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags0); + cmd->tlv_filter_mask_in2 = + cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags1); + cmd->tlv_filter_mask_in3 = + cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags2); + + ret = ath12k_htc_send(&ab->htc, dp->eid, skb); + if (ret) + goto err_free; + + return 0; + +err_free: + dev_kfree_skb_any(skb); + return ret; +} diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.h b/drivers/net/wireless/ath/ath12k/dp_htt.h new file mode 100644 index 0000000000000..6020e632f74ec --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/dp_htt.h @@ -0,0 +1,1546 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_DP_HTT_H +#define ATH12K_DP_HTT_H + +struct ath12k_dp; + +/* HTT definitions */ +#define HTT_TAG_TCL_METADATA_VERSION 5 + +#define HTT_TCL_META_DATA_TYPE GENMASK(1, 0) +#define HTT_TCL_META_DATA_VALID_HTT BIT(2) + +/* vdev meta data */ +#define HTT_TCL_META_DATA_VDEV_ID GENMASK(10, 3) +#define HTT_TCL_META_DATA_PDEV_ID GENMASK(12, 11) +#define HTT_TCL_META_DATA_HOST_INSPECTED_MISSION BIT(13) + +/* peer meta data */ +#define HTT_TCL_META_DATA_PEER_ID GENMASK(15, 3) + +/* Global sequence number */ +#define HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM 3 +#define HTT_TCL_META_DATA_GLOBAL_SEQ_HOST_INSPECTED BIT(2) +#define HTT_TCL_META_DATA_GLOBAL_SEQ_NUM GENMASK(14, 3) +#define HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID 128 + +/* HTT tx completion is overlaid in wbm_release_ring */ +#define HTT_TX_WBM_COMP_INFO0_STATUS GENMASK(16, 13) +#define HTT_TX_WBM_COMP_INFO1_REINJECT_REASON GENMASK(3, 0) +#define HTT_TX_WBM_COMP_INFO1_EXCEPTION_FRAME BIT(4) + +#define HTT_TX_WBM_COMP_INFO2_ACK_RSSI GENMASK(31, 24) + +#define HTT_TARGET_VERSION_TIMEOUT_HZ (3 * HZ) + +struct htt_tx_wbm_completion { + __le32 rsvd0[2]; + __le32 info0; + __le32 info1; + __le32 info2; + __le32 info3; + __le32 info4; + __le32 rsvd1; + +} __packed; + +enum htt_h2t_msg_type { + HTT_H2T_MSG_TYPE_VERSION_REQ = 0, + HTT_H2T_MSG_TYPE_SRING_SETUP = 0xb, + HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG = 0xc, + HTT_H2T_MSG_TYPE_EXT_STATS_CFG = 0x10, + HTT_H2T_MSG_TYPE_PPDU_STATS_CFG = 0x11, + HTT_H2T_MSG_TYPE_VDEV_TXRX_STATS_CFG = 0x1a, + HTT_H2T_MSG_TYPE_TX_MONITOR_CFG = 0x1b, +}; + +#define HTT_VER_REQ_INFO_MSG_ID GENMASK(7, 0) +#define HTT_OPTION_TCL_METADATA_VER_V1 1 +#define HTT_OPTION_TCL_METADATA_VER_V2 2 +#define HTT_OPTION_TAG GENMASK(7, 0) +#define HTT_OPTION_LEN GENMASK(15, 8) +#define HTT_OPTION_VALUE GENMASK(31, 16) +#define HTT_TCL_METADATA_VER_SZ 4 + +struct htt_ver_req_cmd { + __le32 ver_reg_info; + __le32 tcl_metadata_version; +} __packed; + +enum htt_srng_ring_type { + HTT_HW_TO_SW_RING, + HTT_SW_TO_HW_RING, + HTT_SW_TO_SW_RING, +}; + +enum htt_srng_ring_id { + HTT_RXDMA_HOST_BUF_RING, + HTT_RXDMA_MONITOR_STATUS_RING, + HTT_RXDMA_MONITOR_BUF_RING, + HTT_RXDMA_MONITOR_DESC_RING, + HTT_RXDMA_MONITOR_DEST_RING, + HTT_HOST1_TO_FW_RXBUF_RING, + HTT_HOST2_TO_FW_RXBUF_RING, + HTT_RXDMA_NON_MONITOR_DEST_RING, + HTT_RXDMA_HOST_BUF_RING2, + HTT_TX_MON_HOST2MON_BUF_RING, + HTT_TX_MON_MON2HOST_DEST_RING, + HTT_RX_MON_HOST2MON_BUF_RING, + HTT_RX_MON_MON2HOST_DEST_RING, +}; + +/* host -> target HTT_SRING_SETUP message + * + * After target is booted up, Host can send SRING setup message for + * each host facing LMAC SRING. Target setups up HW registers based + * on setup message and confirms back to Host if response_required is set. + * Host should wait for confirmation message before sending new SRING + * setup message + * + * The message would appear as follows: + * + * |31 24|23 20|19|18 16|15|14 8|7 0| + * |--------------- +-----------------+----------------+------------------| + * | ring_type | ring_id | pdev_id | msg_type | + * |----------------------------------------------------------------------| + * | ring_base_addr_lo | + * |----------------------------------------------------------------------| + * | ring_base_addr_hi | + * |----------------------------------------------------------------------| + * |ring_misc_cfg_flag|ring_entry_size| ring_size | + * |----------------------------------------------------------------------| + * | ring_head_offset32_remote_addr_lo | + * |----------------------------------------------------------------------| + * | ring_head_offset32_remote_addr_hi | + * |----------------------------------------------------------------------| + * | ring_tail_offset32_remote_addr_lo | + * |----------------------------------------------------------------------| + * | ring_tail_offset32_remote_addr_hi | + * |----------------------------------------------------------------------| + * | ring_msi_addr_lo | + * |----------------------------------------------------------------------| + * | ring_msi_addr_hi | + * |----------------------------------------------------------------------| + * | ring_msi_data | + * |----------------------------------------------------------------------| + * | intr_timer_th |IM| intr_batch_counter_th | + * |----------------------------------------------------------------------| + * | reserved |RR|PTCF| intr_low_threshold | + * |----------------------------------------------------------------------| + * Where + * IM = sw_intr_mode + * RR = response_required + * PTCF = prefetch_timer_cfg + * + * The message is interpreted as follows: + * dword0 - b'0:7 - msg_type: This will be set to + * HTT_H2T_MSG_TYPE_SRING_SETUP + * b'8:15 - pdev_id: + * 0 (for rings at SOC/UMAC level), + * 1/2/3 mac id (for rings at LMAC level) + * b'16:23 - ring_id: identify which ring is to setup, + * more details can be got from enum htt_srng_ring_id + * b'24:31 - ring_type: identify type of host rings, + * more details can be got from enum htt_srng_ring_type + * dword1 - b'0:31 - ring_base_addr_lo: Lower 32bits of ring base address + * dword2 - b'0:31 - ring_base_addr_hi: Upper 32bits of ring base address + * dword3 - b'0:15 - ring_size: size of the ring in unit of 4-bytes words + * b'16:23 - ring_entry_size: Size of each entry in 4-byte word units + * b'24:31 - ring_misc_cfg_flag: Valid only for HW_TO_SW_RING and + * SW_TO_HW_RING. + * Refer to HTT_SRING_SETUP_RING_MISC_CFG_RING defs. + * dword4 - b'0:31 - ring_head_off32_remote_addr_lo: + * Lower 32 bits of memory address of the remote variable + * storing the 4-byte word offset that identifies the head + * element within the ring. + * (The head offset variable has type u32.) + * Valid for HW_TO_SW and SW_TO_SW rings. + * dword5 - b'0:31 - ring_head_off32_remote_addr_hi: + * Upper 32 bits of memory address of the remote variable + * storing the 4-byte word offset that identifies the head + * element within the ring. + * (The head offset variable has type u32.) + * Valid for HW_TO_SW and SW_TO_SW rings. + * dword6 - b'0:31 - ring_tail_off32_remote_addr_lo: + * Lower 32 bits of memory address of the remote variable + * storing the 4-byte word offset that identifies the tail + * element within the ring. + * (The tail offset variable has type u32.) + * Valid for HW_TO_SW and SW_TO_SW rings. + * dword7 - b'0:31 - ring_tail_off32_remote_addr_hi: + * Upper 32 bits of memory address of the remote variable + * storing the 4-byte word offset that identifies the tail + * element within the ring. + * (The tail offset variable has type u32.) + * Valid for HW_TO_SW and SW_TO_SW rings. + * dword8 - b'0:31 - ring_msi_addr_lo: Lower 32bits of MSI cfg address + * valid only for HW_TO_SW_RING and SW_TO_HW_RING + * dword9 - b'0:31 - ring_msi_addr_hi: Upper 32bits of MSI cfg address + * valid only for HW_TO_SW_RING and SW_TO_HW_RING + * dword10 - b'0:31 - ring_msi_data: MSI data + * Refer to HTT_SRING_SETUP_RING_MSC_CFG_xxx defs + * valid only for HW_TO_SW_RING and SW_TO_HW_RING + * dword11 - b'0:14 - intr_batch_counter_th: + * batch counter threshold is in units of 4-byte words. + * HW internally maintains and increments batch count. + * (see SRING spec for detail description). + * When batch count reaches threshold value, an interrupt + * is generated by HW. + * b'15 - sw_intr_mode: + * This configuration shall be static. + * Only programmed at power up. + * 0: generate pulse style sw interrupts + * 1: generate level style sw interrupts + * b'16:31 - intr_timer_th: + * The timer init value when timer is idle or is + * initialized to start downcounting. + * In 8us units (to cover a range of 0 to 524 ms) + * dword12 - b'0:15 - intr_low_threshold: + * Used only by Consumer ring to generate ring_sw_int_p. + * Ring entries low threshold water mark, that is used + * in combination with the interrupt timer as well as + * the clearing of the level interrupt. + * b'16:18 - prefetch_timer_cfg: + * Used only by Consumer ring to set timer mode to + * support Application prefetch handling. + * The external tail offset/pointer will be updated + * at following intervals: + * 3'b000: (Prefetch feature disabled; used only for debug) + * 3'b001: 1 usec + * 3'b010: 4 usec + * 3'b011: 8 usec (default) + * 3'b100: 16 usec + * Others: Reserved + * b'19 - response_required: + * Host needs HTT_T2H_MSG_TYPE_SRING_SETUP_DONE as response + * b'20:31 - reserved: reserved for future use + */ + +#define HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE GENMASK(7, 0) +#define HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID GENMASK(15, 8) +#define HTT_SRNG_SETUP_CMD_INFO0_RING_ID GENMASK(23, 16) +#define HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE GENMASK(31, 24) + +#define HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE GENMASK(15, 0) +#define HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE GENMASK(23, 16) +#define HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS BIT(25) +#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP BIT(27) +#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP BIT(28) +#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP BIT(29) + +#define HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH GENMASK(14, 0) +#define HTT_SRNG_SETUP_CMD_INTR_INFO_SW_INTR_MODE BIT(15) +#define HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH GENMASK(31, 16) + +#define HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH GENMASK(15, 0) +#define HTT_SRNG_SETUP_CMD_INFO2_PRE_FETCH_TIMER_CFG GENMASK(18, 16) +#define HTT_SRNG_SETUP_CMD_INFO2_RESPONSE_REQUIRED BIT(19) + +struct htt_srng_setup_cmd { + __le32 info0; + __le32 ring_base_addr_lo; + __le32 ring_base_addr_hi; + __le32 info1; + __le32 ring_head_off32_remote_addr_lo; + __le32 ring_head_off32_remote_addr_hi; + __le32 ring_tail_off32_remote_addr_lo; + __le32 ring_tail_off32_remote_addr_hi; + __le32 ring_msi_addr_lo; + __le32 ring_msi_addr_hi; + __le32 msi_data; + __le32 intr_info; + __le32 info2; +} __packed; + +/* host -> target FW PPDU_STATS config message + * + * @details + * The following field definitions describe the format of the HTT host + * to target FW for PPDU_STATS_CFG msg. + * The message allows the host to configure the PPDU_STATS_IND messages + * produced by the target. + * + * |31 24|23 16|15 8|7 0| + * |-----------------------------------------------------------| + * | REQ bit mask | pdev_mask | msg type | + * |-----------------------------------------------------------| + * Header fields: + * - MSG_TYPE + * Bits 7:0 + * Purpose: identifies this is a req to configure ppdu_stats_ind from target + * Value: 0x11 + * - PDEV_MASK + * Bits 8:15 + * Purpose: identifies which pdevs this PPDU stats configuration applies to + * Value: This is a overloaded field, refer to usage and interpretation of + * PDEV in interface document. + * Bit 8 : Reserved for SOC stats + * Bit 9 - 15 : Indicates PDEV_MASK in DBDC + * Indicates MACID_MASK in DBS + * - REQ_TLV_BIT_MASK + * Bits 16:31 + * Purpose: each set bit indicates the corresponding PPDU stats TLV type + * needs to be included in the target's PPDU_STATS_IND messages. + * Value: refer htt_ppdu_stats_tlv_tag_t << 30 bits + * Refer to PKT_TYPE_ENABLE_FLAG0_xxx_MGMT_xxx defs + * dword3 - b'0:31 - packet_type_enable_flags_1: + * Enable MGMT packet from 0b1010 to 0b1111 + * bits from low to high: FP, MD, MO - 3 bits + * Refer to PKT_TYPE_ENABLE_FLAG1_xxx_MGMT_xxx defs + * dword4 - b'0:31 - packet_type_enable_flags_2: + * Enable CTRL packet from 0b0000 to 0b1001 + * bits from low to high: FP, MD, MO - 3 bits + * Refer to PKT_TYPE_ENABLE_FLAG2_xxx_CTRL_xxx defs + * dword5 - b'0:31 - packet_type_enable_flags_3: + * Enable CTRL packet from 0b1010 to 0b1111, + * MCAST_DATA, UCAST_DATA, NULL_DATA + * bits from low to high: FP, MD, MO - 3 bits + * Refer to PKT_TYPE_ENABLE_FLAG3_xxx_CTRL_xxx defs + * dword6 - b'0:31 - tlv_filter_in_flags: + * Filter in Attention/MPDU/PPDU/Header/User tlvs + * Refer to CFG_TLV_FILTER_IN_FLAG defs + */ + +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE GENMASK(7, 0) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID GENMASK(15, 8) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID GENMASK(23, 16) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS BIT(24) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS BIT(25) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID BIT(26) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL BIT(27) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON BIT(28) + +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE GENMASK(15, 0) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT GENMASK(18, 16) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL GENMASK(21, 19) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA GENMASK(24, 22) + +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD GENMASK(9, 0) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE BIT(17) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE BIT(18) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE BIT(19) + +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET BIT(0) +#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET GENMASK(14, 1) + +#define HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET GENMASK(15, 0) +#define HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET GENMASK(31, 16) +#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET GENMASK(15, 0) +#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET GENMASK(31, 16) +#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET GENMASK(15, 0) +#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET GENMASK(31, 16) +#define HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET GENMASK(15, 0) + +#define HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET BIT(23) +#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK GENMASK(15, 0) +#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK GENMASK(18, 16) +#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK GENMASK(16, 0) + +enum htt_rx_filter_tlv_flags { + HTT_RX_FILTER_TLV_FLAGS_MPDU_START = BIT(0), + HTT_RX_FILTER_TLV_FLAGS_MSDU_START = BIT(1), + HTT_RX_FILTER_TLV_FLAGS_RX_PACKET = BIT(2), + HTT_RX_FILTER_TLV_FLAGS_MSDU_END = BIT(3), + HTT_RX_FILTER_TLV_FLAGS_MPDU_END = BIT(4), + HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER = BIT(5), + HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER = BIT(6), + HTT_RX_FILTER_TLV_FLAGS_ATTENTION = BIT(7), + HTT_RX_FILTER_TLV_FLAGS_PPDU_START = BIT(8), + HTT_RX_FILTER_TLV_FLAGS_PPDU_END = BIT(9), + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS = BIT(10), + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT = BIT(11), + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE = BIT(12), + HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO = BIT(13), +}; + +enum htt_rx_mgmt_pkt_filter_tlv_flags0 { + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ = BIT(0), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ = BIT(1), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ = BIT(2), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP = BIT(3), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP = BIT(4), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP = BIT(5), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ = BIT(6), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ = BIT(7), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ = BIT(8), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP = BIT(9), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP = BIT(10), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP = BIT(11), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ = BIT(12), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ = BIT(13), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ = BIT(14), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP = BIT(15), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP = BIT(16), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP = BIT(17), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV = BIT(18), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV = BIT(19), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV = BIT(20), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7 = BIT(21), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7 = BIT(22), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7 = BIT(23), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON = BIT(24), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON = BIT(25), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON = BIT(26), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM = BIT(27), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM = BIT(28), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM = BIT(29), +}; + +enum htt_rx_mgmt_pkt_filter_tlv_flags1 { + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC = BIT(0), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC = BIT(1), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC = BIT(2), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH = BIT(3), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH = BIT(4), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH = BIT(5), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH = BIT(6), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH = BIT(7), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH = BIT(8), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION = BIT(9), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION = BIT(10), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION = BIT(11), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK = BIT(12), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK = BIT(13), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK = BIT(14), + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15 = BIT(15), + HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15 = BIT(16), + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15 = BIT(17), +}; + +enum htt_rx_ctrl_pkt_filter_tlv_flags2 { + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 = BIT(0), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 = BIT(1), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 = BIT(2), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 = BIT(3), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 = BIT(4), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 = BIT(5), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER = BIT(6), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER = BIT(7), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER = BIT(8), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 = BIT(9), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 = BIT(10), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 = BIT(11), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL = BIT(12), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL = BIT(13), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL = BIT(14), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP = BIT(15), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP = BIT(16), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP = BIT(17), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT = BIT(18), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT = BIT(19), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT = BIT(20), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER = BIT(21), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER = BIT(22), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER = BIT(23), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR = BIT(24), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR = BIT(25), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR = BIT(26), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA = BIT(27), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA = BIT(28), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA = BIT(29), +}; + +enum htt_rx_ctrl_pkt_filter_tlv_flags3 { + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL = BIT(0), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL = BIT(1), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL = BIT(2), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS = BIT(3), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS = BIT(4), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS = BIT(5), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS = BIT(6), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS = BIT(7), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS = BIT(8), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK = BIT(9), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK = BIT(10), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK = BIT(11), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND = BIT(12), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND = BIT(13), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND = BIT(14), + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK = BIT(15), + HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK = BIT(16), + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK = BIT(17), +}; + +enum htt_rx_data_pkt_filter_tlv_flasg3 { + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST = BIT(18), + HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST = BIT(19), + HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST = BIT(20), + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST = BIT(21), + HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST = BIT(22), + HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST = BIT(23), + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA = BIT(24), + HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA = BIT(25), + HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA = BIT(26), +}; + +#define HTT_RX_FP_MGMT_FILTER_FLAGS0 \ + (HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM) + +#define HTT_RX_MD_MGMT_FILTER_FLAGS0 \ + (HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM) + +#define HTT_RX_MO_MGMT_FILTER_FLAGS0 \ + (HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM) + +#define HTT_RX_FP_MGMT_FILTER_FLAGS1 (HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \ + | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK) + +#define HTT_RX_MD_MGMT_FILTER_FLAGS1 (HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \ + | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK) + +#define HTT_RX_MO_MGMT_FILTER_FLAGS1 (HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \ + | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK) + +#define HTT_RX_FP_CTRL_FILTER_FLASG2 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \ + | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \ + | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA) + +#define HTT_RX_MD_CTRL_FILTER_FLASG2 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \ + | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \ + | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA) + +#define HTT_RX_MO_CTRL_FILTER_FLASG2 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \ + | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \ + | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA) + +#define HTT_RX_FP_CTRL_FILTER_FLASG3 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \ + | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \ + | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \ + | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \ + | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \ + | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK) + +#define HTT_RX_MD_CTRL_FILTER_FLASG3 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \ + | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \ + | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \ + | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \ + | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \ + | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK) + +#define HTT_RX_MO_CTRL_FILTER_FLASG3 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \ + | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \ + | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \ + | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \ + | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \ + | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK) + +#define HTT_RX_FP_DATA_FILTER_FLASG3 (HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST \ + | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST \ + | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA) + +#define HTT_RX_MD_DATA_FILTER_FLASG3 (HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST \ + | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST \ + | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA) + +#define HTT_RX_MO_DATA_FILTER_FLASG3 (HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST \ + | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST \ + | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA) + +#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 \ + (HTT_RX_FP_MGMT_FILTER_FLAGS0 | \ + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7) + +#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS0 \ + (HTT_RX_MO_MGMT_FILTER_FLAGS0 | \ + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7) + +#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 \ + (HTT_RX_FP_MGMT_FILTER_FLAGS1 | \ + HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15) + +#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS1 \ + (HTT_RX_MO_MGMT_FILTER_FLAGS1 | \ + HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15) + +#define HTT_RX_MON_FP_CTRL_FILTER_FLASG2 \ + (HTT_RX_FP_CTRL_FILTER_FLASG2 | \ + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \ + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \ + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \ + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \ + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \ + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \ + HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT) + +#define HTT_RX_MON_MO_CTRL_FILTER_FLASG2 \ + (HTT_RX_MO_CTRL_FILTER_FLASG2 | \ + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \ + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \ + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \ + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \ + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \ + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \ + HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT) + +#define HTT_RX_MON_FP_CTRL_FILTER_FLASG3 HTT_RX_FP_CTRL_FILTER_FLASG3 + +#define HTT_RX_MON_MO_CTRL_FILTER_FLASG3 HTT_RX_MO_CTRL_FILTER_FLASG3 + +#define HTT_RX_MON_FP_DATA_FILTER_FLASG3 HTT_RX_FP_DATA_FILTER_FLASG3 + +#define HTT_RX_MON_MO_DATA_FILTER_FLASG3 HTT_RX_MO_DATA_FILTER_FLASG3 + +#define HTT_RX_MON_FILTER_TLV_FLAGS \ + (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE) + +#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING \ + (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE) + +#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_BUF_RING \ + (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \ + HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \ + HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \ + HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \ + HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \ + HTT_RX_FILTER_TLV_FLAGS_ATTENTION) + +#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING \ + (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \ + HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \ + HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \ + HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \ + HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE | \ + HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO) + +/* msdu start. mpdu end, attention, rx hdr tlv's are not subscribed */ +#define HTT_RX_TLV_FLAGS_RXDMA_RING \ + (HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \ + HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \ + HTT_RX_FILTER_TLV_FLAGS_MSDU_END) + +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE GENMASK(7, 0) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID GENMASK(15, 8) + +struct htt_rx_ring_selection_cfg_cmd { + __le32 info0; + __le32 info1; + __le32 pkt_type_en_flags0; + __le32 pkt_type_en_flags1; + __le32 pkt_type_en_flags2; + __le32 pkt_type_en_flags3; + __le32 rx_filter_tlv; + __le32 rx_packet_offset; + __le32 rx_mpdu_offset; + __le32 rx_msdu_offset; + __le32 rx_attn_offset; + __le32 info2; + __le32 reserved[2]; + __le32 rx_mpdu_start_end_mask; + __le32 rx_msdu_end_word_mask; + __le32 info3; +} __packed; + +#define HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE 32 +#define HTT_RX_RING_DEFAULT_DMA_LENGTH 0x7 +#define HTT_RX_RING_PKT_TLV_OFFSET 0x1 + +struct htt_rx_ring_tlv_filter { + u32 rx_filter; /* see htt_rx_filter_tlv_flags */ + u32 pkt_filter_flags0; /* MGMT */ + u32 pkt_filter_flags1; /* MGMT */ + u32 pkt_filter_flags2; /* CTRL */ + u32 pkt_filter_flags3; /* DATA */ + bool offset_valid; + u16 rx_packet_offset; + u16 rx_header_offset; + u16 rx_mpdu_end_offset; + u16 rx_mpdu_start_offset; + u16 rx_msdu_end_offset; + u16 rx_msdu_start_offset; + u16 rx_attn_offset; + u16 rx_mpdu_start_wmask; + u16 rx_mpdu_end_wmask; + u32 rx_msdu_end_wmask; + u32 conf_len_ctrl; + u32 conf_len_mgmt; + u32 conf_len_data; + u16 rx_drop_threshold; + bool enable_log_mgmt_type; + bool enable_log_ctrl_type; + bool enable_log_data_type; + bool enable_rx_tlv_offset; + u16 rx_tlv_offset; + bool drop_threshold_valid; + bool rxmon_disable; +}; + +#define HTT_STATS_FRAME_CTRL_TYPE_MGMT 0x0 +#define HTT_STATS_FRAME_CTRL_TYPE_CTRL 0x1 +#define HTT_STATS_FRAME_CTRL_TYPE_DATA 0x2 +#define HTT_STATS_FRAME_CTRL_TYPE_RESV 0x3 + +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE GENMASK(7, 0) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID GENMASK(15, 8) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID GENMASK(23, 16) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS BIT(24) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS BIT(25) + +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE GENMASK(15, 0) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE GENMASK(18, 16) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT GENMASK(21, 19) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL GENMASK(24, 22) +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA GENMASK(27, 25) + +#define HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG GENMASK(2, 0) + +struct htt_tx_ring_selection_cfg_cmd { + __le32 info0; + __le32 info1; + __le32 info2; + __le32 tlv_filter_mask_in0; + __le32 tlv_filter_mask_in1; + __le32 tlv_filter_mask_in2; + __le32 tlv_filter_mask_in3; + __le32 reserved[3]; +} __packed; + +#define HTT_TX_RING_TLV_FILTER_MGMT_DMA_LEN GENMASK(3, 0) +#define HTT_TX_RING_TLV_FILTER_CTRL_DMA_LEN GENMASK(7, 4) +#define HTT_TX_RING_TLV_FILTER_DATA_DMA_LEN GENMASK(11, 8) + +#define HTT_TX_MON_FILTER_HYBRID_MODE \ + (HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS | \ + HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT | \ + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE | \ + HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO | \ + HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2) + +struct htt_tx_ring_tlv_filter { + u32 tx_mon_downstream_tlv_flags; + u32 tx_mon_upstream_tlv_flags0; + u32 tx_mon_upstream_tlv_flags1; + u32 tx_mon_upstream_tlv_flags2; + bool tx_mon_mgmt_filter; + bool tx_mon_data_filter; + bool tx_mon_ctrl_filter; + u16 tx_mon_pkt_dma_len; +} __packed; + +enum htt_tx_mon_upstream_tlv_flags0 { + HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS = BIT(1), + HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS = BIT(2), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START = BIT(3), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END = BIT(4), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU = BIT(5), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU = BIT(6), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA = BIT(7), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA = BIT(8), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT = BIT(9), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT = BIT(10), + HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE = BIT(11), + HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_BITMAP_ACK = BIT(12), + HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_1K_BITMAP_ACK = BIT(13), + HTT_TX_FILTER_TLV_FLAGS0_COEX_TX_STATUS = BIT(14), + HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO = BIT(15), + HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2 = BIT(16), +}; + +#define HTT_TX_FILTER_TLV_FLAGS2_TXPCU_PHYTX_OTHER_TRANSMIT_INFO32 BIT(11) + +/* HTT message target->host */ + +enum htt_t2h_msg_type { + HTT_T2H_MSG_TYPE_VERSION_CONF, + HTT_T2H_MSG_TYPE_PEER_MAP = 0x3, + HTT_T2H_MSG_TYPE_PEER_UNMAP = 0x4, + HTT_T2H_MSG_TYPE_RX_ADDBA = 0x5, + HTT_T2H_MSG_TYPE_PKTLOG = 0x8, + HTT_T2H_MSG_TYPE_SEC_IND = 0xb, + HTT_T2H_MSG_TYPE_PEER_MAP2 = 0x1e, + HTT_T2H_MSG_TYPE_PEER_UNMAP2 = 0x1f, + HTT_T2H_MSG_TYPE_PPDU_STATS_IND = 0x1d, + HTT_T2H_MSG_TYPE_EXT_STATS_CONF = 0x1c, + HTT_T2H_MSG_TYPE_BKPRESSURE_EVENT_IND = 0x24, + HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND = 0x28, + HTT_T2H_MSG_TYPE_PEER_MAP3 = 0x2b, + HTT_T2H_MSG_TYPE_VDEV_TXRX_STATS_PERIODIC_IND = 0x2c, +}; + +#define HTT_TARGET_VERSION_MAJOR 3 + +#define HTT_T2H_MSG_TYPE GENMASK(7, 0) +#define HTT_T2H_VERSION_CONF_MINOR GENMASK(15, 8) +#define HTT_T2H_VERSION_CONF_MAJOR GENMASK(23, 16) + +struct htt_t2h_version_conf_msg { + __le32 version; +} __packed; + +#define HTT_T2H_PEER_MAP_INFO_VDEV_ID GENMASK(15, 8) +#define HTT_T2H_PEER_MAP_INFO_PEER_ID GENMASK(31, 16) +#define HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16 GENMASK(15, 0) +#define HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID GENMASK(31, 16) +#define HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL GENMASK(15, 0) +#define HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID GENMASK(15, 0) +#define HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL GENMASK(31, 16) +#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M BIT(16) +#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S 16 + +struct htt_t2h_peer_map_event { + __le32 info; + __le32 mac_addr_l32; + __le32 info1; + __le32 info2; +} __packed; + +#define HTT_T2H_PEER_UNMAP_INFO_VDEV_ID HTT_T2H_PEER_MAP_INFO_VDEV_ID +#define HTT_T2H_PEER_UNMAP_INFO_PEER_ID HTT_T2H_PEER_MAP_INFO_PEER_ID +#define HTT_T2H_PEER_UNMAP_INFO1_MAC_ADDR_H16 \ + HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16 +#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_M HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M +#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_S HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S + +struct htt_t2h_peer_unmap_event { + __le32 info; + __le32 mac_addr_l32; + __le32 info1; +} __packed; + +struct htt_resp_msg { + union { + struct htt_t2h_version_conf_msg version_msg; + struct htt_t2h_peer_map_event peer_map_ev; + struct htt_t2h_peer_unmap_event peer_unmap_ev; + }; +} __packed; + +#define HTT_VDEV_GET_STATS_U64(msg_l32, msg_u32)\ + (((u64)__le32_to_cpu(msg_u32) << 32) | (__le32_to_cpu(msg_l32))) +#define HTT_T2H_VDEV_STATS_PERIODIC_MSG_TYPE GENMASK(7, 0) +#define HTT_T2H_VDEV_STATS_PERIODIC_PDEV_ID GENMASK(15, 8) +#define HTT_T2H_VDEV_STATS_PERIODIC_NUM_VDEV GENMASK(23, 16) +#define HTT_T2H_VDEV_STATS_PERIODIC_PAYLOAD_BYTES GENMASK(15, 0) +#define HTT_VDEV_TXRX_STATS_COMMON_TLV 0 +#define HTT_VDEV_TXRX_STATS_HW_STATS_TLV 1 + +struct htt_t2h_vdev_txrx_stats_ind { + __le32 vdev_id; + __le32 rx_msdu_byte_cnt_lo; + __le32 rx_msdu_byte_cnt_hi; + __le32 rx_msdu_cnt_lo; + __le32 rx_msdu_cnt_hi; + __le32 tx_msdu_byte_cnt_lo; + __le32 tx_msdu_byte_cnt_hi; + __le32 tx_msdu_cnt_lo; + __le32 tx_msdu_cnt_hi; + __le32 tx_retry_cnt_lo; + __le32 tx_retry_cnt_hi; + __le32 tx_retry_byte_cnt_lo; + __le32 tx_retry_byte_cnt_hi; + __le32 tx_drop_cnt_lo; + __le32 tx_drop_cnt_hi; + __le32 tx_drop_byte_cnt_lo; + __le32 tx_drop_byte_cnt_hi; + __le32 msdu_ttl_cnt_lo; + __le32 msdu_ttl_cnt_hi; + __le32 msdu_ttl_byte_cnt_lo; + __le32 msdu_ttl_byte_cnt_hi; +} __packed; + +struct htt_t2h_vdev_common_stats_tlv { + __le32 soc_drop_count_lo; + __le32 soc_drop_count_hi; +} __packed; + +/* ppdu stats + * + * @details + * The following field definitions describe the format of the HTT target + * to host ppdu stats indication message. + * + * + * |31 16|15 12|11 10|9 8|7 0 | + * |----------------------------------------------------------------------| + * | payload_size | rsvd |pdev_id|mac_id | msg type | + * |----------------------------------------------------------------------| + * | ppdu_id | + * |----------------------------------------------------------------------| + * | Timestamp in us | + * |----------------------------------------------------------------------| + * | reserved | + * |----------------------------------------------------------------------| + * | type-specific stats info | + * | (see htt_ppdu_stats.h) | + * |----------------------------------------------------------------------| + * Header fields: + * - MSG_TYPE + * Bits 7:0 + * Purpose: Identifies this is a PPDU STATS indication + * message. + * Value: 0x1d + * - mac_id + * Bits 9:8 + * Purpose: mac_id of this ppdu_id + * Value: 0-3 + * - pdev_id + * Bits 11:10 + * Purpose: pdev_id of this ppdu_id + * Value: 0-3 + * 0 (for rings at SOC level), + * 1/2/3 PDEV -> 0/1/2 + * - payload_size + * Bits 31:16 + * Purpose: total tlv size + * Value: payload_size in bytes + */ + +#define HTT_T2H_PPDU_STATS_INFO_PDEV_ID GENMASK(11, 10) +#define HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE GENMASK(31, 16) + +struct ath12k_htt_ppdu_stats_msg { + __le32 info; + __le32 ppdu_id; + __le32 timestamp; + __le32 rsvd; + u8 data[]; +} __packed; + +struct htt_tlv { + __le32 header; + u8 value[]; +} __packed; + +#define HTT_TLV_TAG GENMASK(11, 0) +#define HTT_TLV_LEN GENMASK(23, 12) + +enum HTT_PPDU_STATS_BW { + HTT_PPDU_STATS_BANDWIDTH_5MHZ = 0, + HTT_PPDU_STATS_BANDWIDTH_10MHZ = 1, + HTT_PPDU_STATS_BANDWIDTH_20MHZ = 2, + HTT_PPDU_STATS_BANDWIDTH_40MHZ = 3, + HTT_PPDU_STATS_BANDWIDTH_80MHZ = 4, + HTT_PPDU_STATS_BANDWIDTH_160MHZ = 5, /* includes 80+80 */ + HTT_PPDU_STATS_BANDWIDTH_DYN = 6, +}; + +#define HTT_PPDU_STATS_CMN_FLAGS_FRAME_TYPE_M GENMASK(7, 0) +#define HTT_PPDU_STATS_CMN_FLAGS_QUEUE_TYPE_M GENMASK(15, 8) +/* bw - HTT_PPDU_STATS_BW */ +#define HTT_PPDU_STATS_CMN_FLAGS_BW_M GENMASK(19, 16) + +struct htt_ppdu_stats_common { + __le32 ppdu_id; + __le16 sched_cmdid; + u8 ring_id; + u8 num_users; + __le32 flags; /* %HTT_PPDU_STATS_COMMON_FLAGS_*/ + __le32 chain_mask; + __le32 fes_duration_us; /* frame exchange sequence */ + __le32 ppdu_sch_eval_start_tstmp_us; + __le32 ppdu_sch_end_tstmp_us; + __le32 ppdu_start_tstmp_us; + /* BIT [15 : 0] - phy mode (WLAN_PHY_MODE) with which ppdu was transmitted + * BIT [31 : 16] - bandwidth (in MHz) with which ppdu was transmitted + */ + __le16 phy_mode; + __le16 bw_mhz; +} __packed; + +enum htt_ppdu_stats_gi { + HTT_PPDU_STATS_SGI_0_8_US, + HTT_PPDU_STATS_SGI_0_4_US, + HTT_PPDU_STATS_SGI_1_6_US, + HTT_PPDU_STATS_SGI_3_2_US, +}; + +#define HTT_PPDU_STATS_USER_RATE_INFO0_USER_POS_M GENMASK(3, 0) +#define HTT_PPDU_STATS_USER_RATE_INFO0_MU_GROUP_ID_M GENMASK(11, 4) + +enum HTT_PPDU_STATS_PPDU_TYPE { + HTT_PPDU_STATS_PPDU_TYPE_SU, + HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO, + HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA, + HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA, + HTT_PPDU_STATS_PPDU_TYPE_UL_TRIG, + HTT_PPDU_STATS_PPDU_TYPE_BURST_BCN, + HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_RESP, + HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_TRIG, + HTT_PPDU_STATS_PPDU_TYPE_UL_RESP, + HTT_PPDU_STATS_PPDU_TYPE_MAX +}; + +#define HTT_PPDU_STATS_USER_RATE_INFO1_RESP_TYPE_VALD_M BIT(0) +#define HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M GENMASK(5, 1) + +#define HTT_PPDU_STATS_USER_RATE_FLAGS_LTF_SIZE_M GENMASK(1, 0) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_STBC_M BIT(2) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_HE_RE_M BIT(3) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_TXBF_M GENMASK(7, 4) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M GENMASK(11, 8) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M GENMASK(15, 12) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M GENMASK(19, 16) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M GENMASK(23, 20) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M GENMASK(27, 24) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M BIT(28) +#define HTT_PPDU_STATS_USER_RATE_FLAGS_LDPC_M BIT(29) + +#define HTT_USR_RATE_PPDU_TYPE(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M) +#define HTT_USR_RATE_PREAMBLE(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M) +#define HTT_USR_RATE_BW(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M) +#define HTT_USR_RATE_NSS(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M) +#define HTT_USR_RATE_MCS(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M) +#define HTT_USR_RATE_GI(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M) +#define HTT_USR_RATE_DCM(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M) + +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LTF_SIZE_M GENMASK(1, 0) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_STBC_M BIT(2) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_HE_RE_M BIT(3) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_TXBF_M GENMASK(7, 4) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_BW_M GENMASK(11, 8) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_NSS_M GENMASK(15, 12) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_MCS_M GENMASK(19, 16) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_PREAMBLE_M GENMASK(23, 20) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_GI_M GENMASK(27, 24) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_DCM_M BIT(28) +#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LDPC_M BIT(29) + +struct htt_ppdu_stats_user_rate { + u8 tid_num; + u8 reserved0; + __le16 sw_peer_id; + __le32 info0; /* %HTT_PPDU_STATS_USER_RATE_INFO0_*/ + __le16 ru_end; + __le16 ru_start; + __le16 resp_ru_end; + __le16 resp_ru_start; + __le32 info1; /* %HTT_PPDU_STATS_USER_RATE_INFO1_ */ + __le32 rate_flags; /* %HTT_PPDU_STATS_USER_RATE_FLAGS_ */ + /* Note: resp_rate_info is only valid for if resp_type is UL */ + __le32 resp_rate_flags; /* %HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_ */ +} __packed; + +#define HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M GENMASK(7, 0) +#define HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M BIT(8) +#define HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M GENMASK(10, 9) +#define HTT_PPDU_STATS_TX_INFO_FLAGS_BW_M GENMASK(13, 11) +#define HTT_PPDU_STATS_TX_INFO_FLAGS_SGI_M BIT(14) +#define HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M GENMASK(31, 16) + +#define HTT_TX_INFO_IS_AMSDU(_flags) \ + u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M) +#define HTT_TX_INFO_BA_ACK_FAILED(_flags) \ + u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M) +#define HTT_TX_INFO_RATECODE(_flags) \ + u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M) +#define HTT_TX_INFO_PEERID(_flags) \ + u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M) + +enum htt_ppdu_stats_usr_compln_status { + HTT_PPDU_STATS_USER_STATUS_OK, + HTT_PPDU_STATS_USER_STATUS_FILTERED, + HTT_PPDU_STATS_USER_STATUS_RESP_TIMEOUT, + HTT_PPDU_STATS_USER_STATUS_RESP_MISMATCH, + HTT_PPDU_STATS_USER_STATUS_ABORT, +}; + +#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M GENMASK(3, 0) +#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M GENMASK(7, 4) +#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M BIT(8) +#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_RESP_TYPE_M GENMASK(12, 9) + +#define HTT_USR_CMPLTN_IS_AMPDU(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M) +#define HTT_USR_CMPLTN_LONG_RETRY(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M) +#define HTT_USR_CMPLTN_SHORT_RETRY(_val) \ + le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M) + +struct htt_ppdu_stats_usr_cmpltn_cmn { + u8 status; + u8 tid_num; + __le16 sw_peer_id; + /* RSSI value of last ack packet (units = dB above noise floor) */ + __le32 ack_rssi; + __le16 mpdu_tried; + __le16 mpdu_success; + __le32 flags; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRIES*/ +} __packed; + +#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MPDU_M GENMASK(8, 0) +#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M GENMASK(24, 9) +#define HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM GENMASK(31, 25) + +#define HTT_PPDU_STATS_NON_QOS_TID 16 + +struct htt_ppdu_stats_usr_cmpltn_ack_ba_status { + __le32 ppdu_id; + __le16 sw_peer_id; + __le16 reserved0; + __le32 info; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_INFO_ */ + __le16 current_seq; + __le16 start_seq; + __le32 success_bytes; +} __packed; + +struct htt_ppdu_user_stats { + u16 peer_id; + u16 delay_ba; + u32 tlv_flags; + bool is_valid_peer_id; + struct htt_ppdu_stats_user_rate rate; + struct htt_ppdu_stats_usr_cmpltn_cmn cmpltn_cmn; + struct htt_ppdu_stats_usr_cmpltn_ack_ba_status ack_ba; +}; + +#define HTT_PPDU_STATS_MAX_USERS 8 +#define HTT_PPDU_DESC_MAX_DEPTH 16 + +struct htt_ppdu_stats { + struct htt_ppdu_stats_common common; + struct htt_ppdu_user_stats user_stats[HTT_PPDU_STATS_MAX_USERS]; +}; + +struct htt_ppdu_stats_info { + u32 tlv_bitmap; + u32 ppdu_id; + u32 frame_type; + u32 frame_ctrl; + u32 delay_ba; + u32 bar_num_users; + struct htt_ppdu_stats ppdu_stats; + struct list_head list; +}; + +/* @brief target -> host MLO offset indiciation message + * + * @details + * The following field definitions describe the format of the HTT target + * to host mlo offset indication message. + * + * + * |31 29|28 |26|25 22|21 16|15 13|12 10 |9 8|7 0| + * |---------------------------------------------------------------------| + * | rsvd1 | mac_freq |chip_id |pdev_id|msgtype| + * |---------------------------------------------------------------------| + * | sync_timestamp_lo_us | + * |---------------------------------------------------------------------| + * | sync_timestamp_hi_us | + * |---------------------------------------------------------------------| + * | mlo_offset_lo | + * |---------------------------------------------------------------------| + * | mlo_offset_hi | + * |---------------------------------------------------------------------| + * | mlo_offset_clcks | + * |---------------------------------------------------------------------| + * | rsvd2 | mlo_comp_clks |mlo_comp_us | + * |---------------------------------------------------------------------| + * | rsvd3 |mlo_comp_timer | + * |---------------------------------------------------------------------| + * Header fields + * - MSG_TYPE + * Bits 7:0 + * Purpose: Identifies this is a MLO offset indication msg + * - PDEV_ID + * Bits 9:8 + * Purpose: Pdev of this MLO offset + * - CHIP_ID + * Bits 12:10 + * Purpose: chip_id of this MLO offset + * - MAC_FREQ + * Bits 28:13 + * - SYNC_TIMESTAMP_LO_US + * Purpose: clock frequency of the mac HW block in MHz + * Bits: 31:0 + * Purpose: lower 32 bits of the WLAN global time stamp at which + * last sync interrupt was received + * - SYNC_TIMESTAMP_HI_US + * Bits: 31:0 + * Purpose: upper 32 bits of WLAN global time stamp at which + * last sync interrupt was received + * - MLO_OFFSET_LO + * Bits: 31:0 + * Purpose: lower 32 bits of the MLO offset in us + * - MLO_OFFSET_HI + * Bits: 31:0 + * Purpose: upper 32 bits of the MLO offset in us + * - MLO_COMP_US + * Bits: 15:0 + * Purpose: MLO time stamp compensation applied in us + * - MLO_COMP_CLCKS + * Bits: 25:16 + * Purpose: MLO time stamp compensation applied in clock ticks + * - MLO_COMP_TIMER + * Bits: 21:0 + * Purpose: Periodic timer at which compensation is applied + */ + +#define HTT_T2H_MLO_OFFSET_INFO_MSG_TYPE GENMASK(7, 0) +#define HTT_T2H_MLO_OFFSET_INFO_PDEV_ID GENMASK(9, 8) + +struct ath12k_htt_mlo_offset_msg { + __le32 info; + __le32 sync_timestamp_lo_us; + __le32 sync_timestamp_hi_us; + __le32 mlo_offset_hi; + __le32 mlo_offset_lo; + __le32 mlo_offset_clks; + __le32 mlo_comp_clks; + __le32 mlo_comp_timer; +} __packed; + +/* @brief host -> target FW extended statistics retrieve + * + * @details + * The following field definitions describe the format of the HTT host + * to target FW extended stats retrieve message. + * The message specifies the type of stats the host wants to retrieve. + * + * |31 24|23 16|15 8|7 0| + * |-----------------------------------------------------------| + * | reserved | stats type | pdev_mask | msg type | + * |-----------------------------------------------------------| + * | config param [0] | + * |-----------------------------------------------------------| + * | config param [1] | + * |-----------------------------------------------------------| + * | config param [2] | + * |-----------------------------------------------------------| + * | config param [3] | + * |-----------------------------------------------------------| + * | reserved | + * |-----------------------------------------------------------| + * | cookie LSBs | + * |-----------------------------------------------------------| + * | cookie MSBs | + * |-----------------------------------------------------------| + * Header fields: + * - MSG_TYPE + * Bits 7:0 + * Purpose: identifies this is a extended stats upload request message + * Value: 0x10 + * - PDEV_MASK + * Bits 8:15 + * Purpose: identifies the mask of PDEVs to retrieve stats from + * Value: This is a overloaded field, refer to usage and interpretation of + * PDEV in interface document. + * Bit 8 : Reserved for SOC stats + * Bit 9 - 15 : Indicates PDEV_MASK in DBDC + * Indicates MACID_MASK in DBS + * - STATS_TYPE + * Bits 23:16 + * Purpose: identifies which FW statistics to upload + * Value: Defined by htt_dbg_ext_stats_type (see htt_stats.h) + * - Reserved + * Bits 31:24 + * - CONFIG_PARAM [0] + * Bits 31:0 + * Purpose: give an opaque configuration value to the specified stats type + * Value: stats-type specific configuration value + * Refer to htt_stats.h for interpretation for each stats sub_type + * - CONFIG_PARAM [1] + * Bits 31:0 + * Purpose: give an opaque configuration value to the specified stats type + * Value: stats-type specific configuration value + * Refer to htt_stats.h for interpretation for each stats sub_type + * - CONFIG_PARAM [2] + * Bits 31:0 + * Purpose: give an opaque configuration value to the specified stats type + * Value: stats-type specific configuration value + * Refer to htt_stats.h for interpretation for each stats sub_type + * - CONFIG_PARAM [3] + * Bits 31:0 + * Purpose: give an opaque configuration value to the specified stats type + * Value: stats-type specific configuration value + * Refer to htt_stats.h for interpretation for each stats sub_type + * - Reserved [31:0] for future use. + * - COOKIE_LSBS + * Bits 31:0 + * Purpose: Provide a mechanism to match a target->host stats confirmation + * message with its preceding host->target stats request message. + * Value: LSBs of the opaque cookie specified by the host-side requestor + * - COOKIE_MSBS + * Bits 31:0 + * Purpose: Provide a mechanism to match a target->host stats confirmation + * message with its preceding host->target stats request message. + * Value: MSBs of the opaque cookie specified by the host-side requestor + */ + +struct htt_ext_stats_cfg_hdr { + u8 msg_type; + u8 pdev_mask; + u8 stats_type; + u8 reserved; +} __packed; + +struct htt_ext_stats_cfg_cmd { + struct htt_ext_stats_cfg_hdr hdr; + __le32 cfg_param0; + __le32 cfg_param1; + __le32 cfg_param2; + __le32 cfg_param3; + __le32 reserved; + __le32 cookie_lsb; + __le32 cookie_msb; +} __packed; + +/* htt stats config default params */ +#define HTT_STAT_DEFAULT_RESET_START_OFFSET 0 +#define HTT_STAT_DEFAULT_CFG0_ALL_HWQS 0xffffffff +#define HTT_STAT_DEFAULT_CFG0_ALL_TXQS 0xffffffff +#define HTT_STAT_DEFAULT_CFG0_ALL_CMDQS 0xffff +#define HTT_STAT_DEFAULT_CFG0_ALL_RINGS 0xffff +#define HTT_STAT_DEFAULT_CFG0_ACTIVE_PEERS 0xff +#define HTT_STAT_DEFAULT_CFG0_CCA_CUMULATIVE 0x00 +#define HTT_STAT_DEFAULT_CFG0_ACTIVE_VDEVS 0x00 + +/* HTT_DBG_EXT_STATS_PEER_INFO + * PARAMS: + * @config_param0: + * [Bit0] - [0] for sw_peer_id, [1] for mac_addr based request + * [Bit15 : Bit 1] htt_peer_stats_req_mode_t + * [Bit31 : Bit16] sw_peer_id + * @config_param1: + * peer_stats_req_type_mask:32 (enum htt_peer_stats_tlv_enum) + * 0 bit htt_peer_stats_cmn_tlv + * 1 bit htt_peer_details_tlv + * 2 bit htt_tx_peer_rate_stats_tlv + * 3 bit htt_rx_peer_rate_stats_tlv + * 4 bit htt_tx_tid_stats_tlv/htt_tx_tid_stats_v1_tlv + * 5 bit htt_rx_tid_stats_tlv + * 6 bit htt_msdu_flow_stats_tlv + * @config_param2: [Bit31 : Bit0] mac_addr31to0 + * @config_param3: [Bit15 : Bit0] mac_addr47to32 + * [Bit31 : Bit16] reserved + */ +#define HTT_STAT_PEER_INFO_MAC_ADDR BIT(0) +#define HTT_STAT_DEFAULT_PEER_REQ_TYPE 0x7f + +/* Used to set different configs to the specified stats type.*/ +struct htt_ext_stats_cfg_params { + u32 cfg0; + u32 cfg1; + u32 cfg2; + u32 cfg3; +}; + +enum vdev_stats_offload_timer_duration { + ATH12K_STATS_TIMER_DUR_500MS = 1, + ATH12K_STATS_TIMER_DUR_1SEC = 2, + ATH12K_STATS_TIMER_DUR_2SEC = 3, +}; + +#define ATH12K_HTT_MAC_ADDR_L32_0 GENMASK(7, 0) +#define ATH12K_HTT_MAC_ADDR_L32_1 GENMASK(15, 8) +#define ATH12K_HTT_MAC_ADDR_L32_2 GENMASK(23, 16) +#define ATH12K_HTT_MAC_ADDR_L32_3 GENMASK(31, 24) +#define ATH12K_HTT_MAC_ADDR_H16_0 GENMASK(7, 0) +#define ATH12K_HTT_MAC_ADDR_H16_1 GENMASK(15, 8) + +struct htt_mac_addr { + __le32 mac_addr_l32; + __le32 mac_addr_h16; +} __packed; + +int ath12k_dp_htt_connect(struct ath12k_dp *dp); +int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id, + int mac_id, enum hal_ring_type ring_type); + +void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab, + struct sk_buff *skb); +int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len, + int (*iter)(struct ath12k_base *ar, u16 tag, u16 len, + const void *ptr, void *data), + void *data); +int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab); +int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask); +int +ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type, + struct htt_ext_stats_cfg_params *cfg_params, + u64 cookie); +int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset); + +int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id, + int mac_id, enum hal_ring_type ring_type, + int rx_buf_size, + struct htt_rx_ring_tlv_filter *tlv_filter); +int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id, + int mac_id, enum hal_ring_type ring_type, + int tx_buf_size, + struct htt_tx_ring_tlv_filter *htt_tlv_filter); +int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset); +#endif diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c index 009c495021489..737287a9aa462 100644 --- a/drivers/net/wireless/ath/ath12k/dp_mon.c +++ b/drivers/net/wireless/ath/ath12k/dp_mon.c @@ -1,1756 +1,16 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "dp_mon.h" #include "debug.h" -#include "dp_rx.h" #include "dp_tx.h" #include "peer.h" -#define ATH12K_LE32_DEC_ENC(value, dec_bits, enc_bits) \ - u32_encode_bits(le32_get_bits(value, dec_bits), enc_bits) - -#define ATH12K_LE64_DEC_ENC(value, dec_bits, enc_bits) \ - u32_encode_bits(le64_get_bits(value, dec_bits), enc_bits) - -static void -ath12k_dp_mon_rx_handle_ofdma_info(const struct hal_rx_ppdu_end_user_stats *ppdu_end_user, - struct hal_rx_user_status *rx_user_status) -{ - rx_user_status->ul_ofdma_user_v0_word0 = - __le32_to_cpu(ppdu_end_user->usr_resp_ref); - rx_user_status->ul_ofdma_user_v0_word1 = - __le32_to_cpu(ppdu_end_user->usr_resp_ref_ext); -} - -static void -ath12k_dp_mon_rx_populate_byte_count(const struct hal_rx_ppdu_end_user_stats *stats, - void *ppduinfo, - struct hal_rx_user_status *rx_user_status) -{ - rx_user_status->mpdu_ok_byte_count = - le32_get_bits(stats->info7, - HAL_RX_PPDU_END_USER_STATS_INFO7_MPDU_OK_BYTE_COUNT); - rx_user_status->mpdu_err_byte_count = - le32_get_bits(stats->info8, - HAL_RX_PPDU_END_USER_STATS_INFO8_MPDU_ERR_BYTE_COUNT); -} - -static void -ath12k_dp_mon_rx_populate_mu_user_info(const struct hal_rx_ppdu_end_user_stats *rx_tlv, - struct hal_rx_mon_ppdu_info *ppdu_info, - struct hal_rx_user_status *rx_user_status) -{ - rx_user_status->ast_index = ppdu_info->ast_index; - rx_user_status->tid = ppdu_info->tid; - rx_user_status->tcp_ack_msdu_count = - ppdu_info->tcp_ack_msdu_count; - rx_user_status->tcp_msdu_count = - ppdu_info->tcp_msdu_count; - rx_user_status->udp_msdu_count = - ppdu_info->udp_msdu_count; - rx_user_status->other_msdu_count = - ppdu_info->other_msdu_count; - rx_user_status->frame_control = ppdu_info->frame_control; - rx_user_status->frame_control_info_valid = - ppdu_info->frame_control_info_valid; - rx_user_status->data_sequence_control_info_valid = - ppdu_info->data_sequence_control_info_valid; - rx_user_status->first_data_seq_ctrl = - ppdu_info->first_data_seq_ctrl; - rx_user_status->preamble_type = ppdu_info->preamble_type; - rx_user_status->ht_flags = ppdu_info->ht_flags; - rx_user_status->vht_flags = ppdu_info->vht_flags; - rx_user_status->he_flags = ppdu_info->he_flags; - rx_user_status->rs_flags = ppdu_info->rs_flags; - - rx_user_status->mpdu_cnt_fcs_ok = - ppdu_info->num_mpdu_fcs_ok; - rx_user_status->mpdu_cnt_fcs_err = - ppdu_info->num_mpdu_fcs_err; - memcpy(&rx_user_status->mpdu_fcs_ok_bitmap[0], &ppdu_info->mpdu_fcs_ok_bitmap[0], - HAL_RX_NUM_WORDS_PER_PPDU_BITMAP * - sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0])); - - ath12k_dp_mon_rx_populate_byte_count(rx_tlv, ppdu_info, rx_user_status); -} - -static void ath12k_dp_mon_parse_vht_sig_a(const struct hal_rx_vht_sig_a_info *vht_sig, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 nsts, info0, info1; - u8 gi_setting; - - info0 = __le32_to_cpu(vht_sig->info0); - info1 = __le32_to_cpu(vht_sig->info1); - - ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING); - ppdu_info->mcs = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_MCS); - gi_setting = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_GI_SETTING); - switch (gi_setting) { - case HAL_RX_VHT_SIG_A_NORMAL_GI: - ppdu_info->gi = HAL_RX_GI_0_8_US; - break; - case HAL_RX_VHT_SIG_A_SHORT_GI: - case HAL_RX_VHT_SIG_A_SHORT_GI_AMBIGUITY: - ppdu_info->gi = HAL_RX_GI_0_4_US; - break; - } - - ppdu_info->is_stbc = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_STBC); - nsts = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_NSTS); - if (ppdu_info->is_stbc && nsts > 0) - nsts = ((nsts + 1) >> 1) - 1; - - ppdu_info->nss = u32_get_bits(nsts, VHT_SIG_SU_NSS_MASK); - ppdu_info->bw = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_BW); - ppdu_info->beamformed = u32_get_bits(info1, - HAL_RX_VHT_SIG_A_INFO_INFO1_BEAMFORMED); - ppdu_info->vht_flag_values5 = u32_get_bits(info0, - HAL_RX_VHT_SIG_A_INFO_INFO0_GROUP_ID); - ppdu_info->vht_flag_values3[0] = (((ppdu_info->mcs) << 4) | - ppdu_info->nss); - ppdu_info->vht_flag_values2 = ppdu_info->bw; - ppdu_info->vht_flag_values4 = - u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING); -} - -static void ath12k_dp_mon_parse_ht_sig(const struct hal_rx_ht_sig_info *ht_sig, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0 = __le32_to_cpu(ht_sig->info0); - u32 info1 = __le32_to_cpu(ht_sig->info1); - - ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_MCS); - ppdu_info->bw = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_BW); - ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_STBC); - ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_FEC_CODING); - ppdu_info->gi = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_GI); - ppdu_info->nss = (ppdu_info->mcs >> 3); -} - -static void ath12k_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0 = __le32_to_cpu(lsigb->info0); - u8 rate; - - rate = u32_get_bits(info0, HAL_RX_LSIG_B_INFO_INFO0_RATE); - switch (rate) { - case 1: - rate = HAL_RX_LEGACY_RATE_1_MBPS; - break; - case 2: - case 5: - rate = HAL_RX_LEGACY_RATE_2_MBPS; - break; - case 3: - case 6: - rate = HAL_RX_LEGACY_RATE_5_5_MBPS; - break; - case 4: - case 7: - rate = HAL_RX_LEGACY_RATE_11_MBPS; - break; - default: - rate = HAL_RX_LEGACY_RATE_INVALID; - } - - ppdu_info->rate = rate; - ppdu_info->cck_flag = 1; -} - -static void ath12k_dp_mon_parse_l_sig_a(const struct hal_rx_lsig_a_info *lsiga, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0 = __le32_to_cpu(lsiga->info0); - u8 rate; - - rate = u32_get_bits(info0, HAL_RX_LSIG_A_INFO_INFO0_RATE); - switch (rate) { - case 8: - rate = HAL_RX_LEGACY_RATE_48_MBPS; - break; - case 9: - rate = HAL_RX_LEGACY_RATE_24_MBPS; - break; - case 10: - rate = HAL_RX_LEGACY_RATE_12_MBPS; - break; - case 11: - rate = HAL_RX_LEGACY_RATE_6_MBPS; - break; - case 12: - rate = HAL_RX_LEGACY_RATE_54_MBPS; - break; - case 13: - rate = HAL_RX_LEGACY_RATE_36_MBPS; - break; - case 14: - rate = HAL_RX_LEGACY_RATE_18_MBPS; - break; - case 15: - rate = HAL_RX_LEGACY_RATE_9_MBPS; - break; - default: - rate = HAL_RX_LEGACY_RATE_INVALID; - } - - ppdu_info->rate = rate; -} - -static void -ath12k_dp_mon_parse_he_sig_b2_ofdma(const struct hal_rx_he_sig_b2_ofdma_info *ofdma, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0, value; - - info0 = __le32_to_cpu(ofdma->info0); - - ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_DCM_KNOWN | HE_CODING_KNOWN; - - /* HE-data2 */ - ppdu_info->he_data2 |= HE_TXBF_KNOWN; - - ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_MCS); - value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_DCM); - value = value << HE_DCM_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_CODING); - ppdu_info->ldpc = value; - value = value << HE_CODING_SHIFT; - ppdu_info->he_data3 |= value; - - /* HE-data4 */ - value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_ID); - value = value << HE_STA_ID_SHIFT; - ppdu_info->he_data4 |= value; - - ppdu_info->nss = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_NSTS); - ppdu_info->beamformed = u32_get_bits(info0, - HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_TXBF); -} - -static void -ath12k_dp_mon_parse_he_sig_b2_mu(const struct hal_rx_he_sig_b2_mu_info *he_sig_b2_mu, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0, value; - - info0 = __le32_to_cpu(he_sig_b2_mu->info0); - - ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_CODING_KNOWN; - - ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_MCS); - value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_CODING); - ppdu_info->ldpc = value; - value = value << HE_CODING_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_ID); - value = value << HE_STA_ID_SHIFT; - ppdu_info->he_data4 |= value; - - ppdu_info->nss = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_NSTS); -} - -static void -ath12k_dp_mon_parse_he_sig_b1_mu(const struct hal_rx_he_sig_b1_mu_info *he_sig_b1_mu, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0 = __le32_to_cpu(he_sig_b1_mu->info0); - u16 ru_tones; - - ru_tones = u32_get_bits(info0, - HAL_RX_HE_SIG_B1_MU_INFO_INFO0_RU_ALLOCATION); - ppdu_info->ru_alloc = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones); - ppdu_info->he_RU[0] = ru_tones; -} - -static void -ath12k_dp_mon_parse_he_sig_mu(const struct hal_rx_he_sig_a_mu_dl_info *he_sig_a_mu_dl, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0, info1, value; - u16 he_gi = 0, he_ltf = 0; - - info0 = __le32_to_cpu(he_sig_a_mu_dl->info0); - info1 = __le32_to_cpu(he_sig_a_mu_dl->info1); - - ppdu_info->he_mu_flags = 1; - - ppdu_info->he_data1 = HE_MU_FORMAT_TYPE; - ppdu_info->he_data1 |= - HE_BSS_COLOR_KNOWN | - HE_DL_UL_KNOWN | - HE_LDPC_EXTRA_SYMBOL_KNOWN | - HE_STBC_KNOWN | - HE_DATA_BW_RU_KNOWN | - HE_DOPPLER_KNOWN; - - ppdu_info->he_data2 = - HE_GI_KNOWN | - HE_LTF_SYMBOLS_KNOWN | - HE_PRE_FEC_PADDING_KNOWN | - HE_PE_DISAMBIGUITY_KNOWN | - HE_TXOP_KNOWN | - HE_MIDABLE_PERIODICITY_KNOWN; - - /* data3 */ - ppdu_info->he_data3 = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_BSS_COLOR); - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_UL_FLAG); - value = value << HE_DL_UL_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_LDPC_EXTRA); - value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC); - value = value << HE_STBC_SHIFT; - ppdu_info->he_data3 |= value; - - /* data4 */ - ppdu_info->he_data4 = u32_get_bits(info0, - HAL_RX_HE_SIG_A_MU_DL_INFO0_SPATIAL_REUSE); - ppdu_info->he_data4 = value; - - /* data5 */ - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW); - ppdu_info->he_data5 = value; - ppdu_info->bw = value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_CP_LTF_SIZE); - switch (value) { - case 0: - he_gi = HE_GI_0_8; - he_ltf = HE_LTF_4_X; - break; - case 1: - he_gi = HE_GI_0_8; - he_ltf = HE_LTF_2_X; - break; - case 2: - he_gi = HE_GI_1_6; - he_ltf = HE_LTF_2_X; - break; - case 3: - he_gi = HE_GI_3_2; - he_ltf = HE_LTF_4_X; - break; - } - - ppdu_info->gi = he_gi; - value = he_gi << HE_GI_SHIFT; - ppdu_info->he_data5 |= value; - - value = he_ltf << HE_LTF_SIZE_SHIFT; - ppdu_info->he_data5 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_NUM_LTF_SYMB); - value = (value << HE_LTF_SYM_SHIFT); - ppdu_info->he_data5 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_FACTOR); - value = value << HE_PRE_FEC_PAD_SHIFT; - ppdu_info->he_data5 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_PE_DISAM); - value = value << HE_PE_DISAMBIGUITY_SHIFT; - ppdu_info->he_data5 |= value; - - /*data6*/ - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DOPPLER_INDICATION); - value = value << HE_DOPPLER_SHIFT; - ppdu_info->he_data6 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_TXOP_DURATION); - value = value << HE_TXOP_SHIFT; - ppdu_info->he_data6 |= value; - - /* HE-MU Flags */ - /* HE-MU-flags1 */ - ppdu_info->he_flags1 = - HE_SIG_B_MCS_KNOWN | - HE_SIG_B_DCM_KNOWN | - HE_SIG_B_COMPRESSION_FLAG_1_KNOWN | - HE_SIG_B_SYM_NUM_KNOWN | - HE_RU_0_KNOWN; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_MCS_OF_SIGB); - ppdu_info->he_flags1 |= value; - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DCM_OF_SIGB); - value = value << HE_DCM_FLAG_1_SHIFT; - ppdu_info->he_flags1 |= value; - - /* HE-MU-flags2 */ - ppdu_info->he_flags2 = HE_BW_KNOWN; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW); - ppdu_info->he_flags2 |= value; - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_COMP_MODE_SIGB); - value = value << HE_SIG_B_COMPRESSION_FLAG_2_SHIFT; - ppdu_info->he_flags2 |= value; - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_NUM_SIGB_SYMB); - value = value - 1; - value = value << HE_NUM_SIG_B_SYMBOLS_SHIFT; - ppdu_info->he_flags2 |= value; - - ppdu_info->is_stbc = info1 & - HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC; -} - -static void ath12k_dp_mon_parse_he_sig_su(const struct hal_rx_he_sig_a_su_info *he_sig_a, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 info0, info1, value; - u32 dcm; - u8 he_dcm = 0, he_stbc = 0; - u16 he_gi = 0, he_ltf = 0; - - ppdu_info->he_flags = 1; - - info0 = __le32_to_cpu(he_sig_a->info0); - info1 = __le32_to_cpu(he_sig_a->info1); - - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_FORMAT_IND); - if (value == 0) - ppdu_info->he_data1 = HE_TRIG_FORMAT_TYPE; - else - ppdu_info->he_data1 = HE_SU_FORMAT_TYPE; - - ppdu_info->he_data1 |= - HE_BSS_COLOR_KNOWN | - HE_BEAM_CHANGE_KNOWN | - HE_DL_UL_KNOWN | - HE_MCS_KNOWN | - HE_DCM_KNOWN | - HE_CODING_KNOWN | - HE_LDPC_EXTRA_SYMBOL_KNOWN | - HE_STBC_KNOWN | - HE_DATA_BW_RU_KNOWN | - HE_DOPPLER_KNOWN; - - ppdu_info->he_data2 |= - HE_GI_KNOWN | - HE_TXBF_KNOWN | - HE_PE_DISAMBIGUITY_KNOWN | - HE_TXOP_KNOWN | - HE_LTF_SYMBOLS_KNOWN | - HE_PRE_FEC_PADDING_KNOWN | - HE_MIDABLE_PERIODICITY_KNOWN; - - ppdu_info->he_data3 = u32_get_bits(info0, - HAL_RX_HE_SIG_A_SU_INFO_INFO0_BSS_COLOR); - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_BEAM_CHANGE); - value = value << HE_BEAM_CHANGE_SHIFT; - ppdu_info->he_data3 |= value; - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DL_UL_FLAG); - value = value << HE_DL_UL_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS); - ppdu_info->mcs = value; - value = value << HE_TRANSMIT_MCS_SHIFT; - ppdu_info->he_data3 |= value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM); - he_dcm = value; - value = value << HE_DCM_SHIFT; - ppdu_info->he_data3 |= value; - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING); - value = value << HE_CODING_SHIFT; - ppdu_info->he_data3 |= value; - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_LDPC_EXTRA); - value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT; - ppdu_info->he_data3 |= value; - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC); - he_stbc = value; - value = value << HE_STBC_SHIFT; - ppdu_info->he_data3 |= value; - - /* data4 */ - ppdu_info->he_data4 = u32_get_bits(info0, - HAL_RX_HE_SIG_A_SU_INFO_INFO0_SPATIAL_REUSE); - - /* data5 */ - value = u32_get_bits(info0, - HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW); - ppdu_info->he_data5 = value; - ppdu_info->bw = value; - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_CP_LTF_SIZE); - switch (value) { - case 0: - he_gi = HE_GI_0_8; - he_ltf = HE_LTF_1_X; - break; - case 1: - he_gi = HE_GI_0_8; - he_ltf = HE_LTF_2_X; - break; - case 2: - he_gi = HE_GI_1_6; - he_ltf = HE_LTF_2_X; - break; - case 3: - if (he_dcm && he_stbc) { - he_gi = HE_GI_0_8; - he_ltf = HE_LTF_4_X; - } else { - he_gi = HE_GI_3_2; - he_ltf = HE_LTF_4_X; - } - break; - } - ppdu_info->gi = he_gi; - value = he_gi << HE_GI_SHIFT; - ppdu_info->he_data5 |= value; - value = he_ltf << HE_LTF_SIZE_SHIFT; - ppdu_info->ltf_size = he_ltf; - ppdu_info->he_data5 |= value; - - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS); - value = (value << HE_LTF_SYM_SHIFT); - ppdu_info->he_data5 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_FACTOR); - value = value << HE_PRE_FEC_PAD_SHIFT; - ppdu_info->he_data5 |= value; - - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF); - value = value << HE_TXBF_SHIFT; - ppdu_info->he_data5 |= value; - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_PE_DISAM); - value = value << HE_PE_DISAMBIGUITY_SHIFT; - ppdu_info->he_data5 |= value; - - /* data6 */ - value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS); - value++; - ppdu_info->he_data6 = value; - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_DOPPLER_IND); - value = value << HE_DOPPLER_SHIFT; - ppdu_info->he_data6 |= value; - value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXOP_DURATION); - value = value << HE_TXOP_SHIFT; - ppdu_info->he_data6 |= value; - - ppdu_info->mcs = - u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS); - ppdu_info->bw = - u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW); - ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING); - ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC); - ppdu_info->beamformed = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF); - dcm = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM); - ppdu_info->nss = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS); - ppdu_info->dcm = dcm; -} - -static void -ath12k_dp_mon_hal_rx_parse_u_sig_cmn(const struct hal_mon_usig_cmn *cmn, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u32 common; - - ppdu_info->u_sig_info.bw = le32_get_bits(cmn->info0, - HAL_RX_USIG_CMN_INFO0_BW); - ppdu_info->u_sig_info.ul_dl = le32_get_bits(cmn->info0, - HAL_RX_USIG_CMN_INFO0_UL_DL); - - common = __le32_to_cpu(ppdu_info->u_sig_info.usig.common); - common |= IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER_KNOWN | - IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW_KNOWN | - IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL_KNOWN | - IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR_KNOWN | - IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP_KNOWN | - ATH12K_LE32_DEC_ENC(cmn->info0, - HAL_RX_USIG_CMN_INFO0_PHY_VERSION, - IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER) | - u32_encode_bits(ppdu_info->u_sig_info.bw, - IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW) | - u32_encode_bits(ppdu_info->u_sig_info.ul_dl, - IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL) | - ATH12K_LE32_DEC_ENC(cmn->info0, - HAL_RX_USIG_CMN_INFO0_BSS_COLOR, - IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR) | - ATH12K_LE32_DEC_ENC(cmn->info0, - HAL_RX_USIG_CMN_INFO0_TXOP, - IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP); - ppdu_info->u_sig_info.usig.common = cpu_to_le32(common); - - switch (ppdu_info->u_sig_info.bw) { - default: - fallthrough; - case HAL_EHT_BW_20: - ppdu_info->bw = HAL_RX_BW_20MHZ; - break; - case HAL_EHT_BW_40: - ppdu_info->bw = HAL_RX_BW_40MHZ; - break; - case HAL_EHT_BW_80: - ppdu_info->bw = HAL_RX_BW_80MHZ; - break; - case HAL_EHT_BW_160: - ppdu_info->bw = HAL_RX_BW_160MHZ; - break; - case HAL_EHT_BW_320_1: - case HAL_EHT_BW_320_2: - ppdu_info->bw = HAL_RX_BW_320MHZ; - break; - } -} - -static void -ath12k_dp_mon_hal_rx_parse_u_sig_tb(const struct hal_mon_usig_tb *usig_tb, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig; - enum ieee80211_radiotap_eht_usig_tb spatial_reuse1, spatial_reuse2; - u32 common, value, mask; - - spatial_reuse1 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1; - spatial_reuse2 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2; - - common = __le32_to_cpu(usig->common); - value = __le32_to_cpu(usig->value); - mask = __le32_to_cpu(usig->mask); - - ppdu_info->u_sig_info.ppdu_type_comp_mode = - le32_get_bits(usig_tb->info0, - HAL_RX_USIG_TB_INFO0_PPDU_TYPE_COMP_MODE); - - common |= ATH12K_LE32_DEC_ENC(usig_tb->info0, - HAL_RX_USIG_TB_INFO0_RX_INTEG_CHECK_PASS, - IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC); - - value |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD | - u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode, - IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE) | - IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE | - ATH12K_LE32_DEC_ENC(usig_tb->info0, - HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_1, - spatial_reuse1) | - ATH12K_LE32_DEC_ENC(usig_tb->info0, - HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_2, - spatial_reuse2) | - IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD | - ATH12K_LE32_DEC_ENC(usig_tb->info0, - HAL_RX_USIG_TB_INFO0_CRC, - IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC) | - ATH12K_LE32_DEC_ENC(usig_tb->info0, - HAL_RX_USIG_TB_INFO0_TAIL, - IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL); - - mask |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD | - IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE | - IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE | - spatial_reuse1 | spatial_reuse2 | - IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD | - IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC | - IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL; - - usig->common = cpu_to_le32(common); - usig->value = cpu_to_le32(value); - usig->mask = cpu_to_le32(mask); -} - -static void -ath12k_dp_mon_hal_rx_parse_u_sig_mu(const struct hal_mon_usig_mu *usig_mu, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig; - enum ieee80211_radiotap_eht_usig_mu sig_symb, punc; - u32 common, value, mask; - - sig_symb = IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS; - punc = IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO; - - common = __le32_to_cpu(usig->common); - value = __le32_to_cpu(usig->value); - mask = __le32_to_cpu(usig->mask); - - ppdu_info->u_sig_info.ppdu_type_comp_mode = - le32_get_bits(usig_mu->info0, - HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE); - ppdu_info->u_sig_info.eht_sig_mcs = - le32_get_bits(usig_mu->info0, - HAL_RX_USIG_MU_INFO0_EHT_SIG_MCS); - ppdu_info->u_sig_info.num_eht_sig_sym = - le32_get_bits(usig_mu->info0, - HAL_RX_USIG_MU_INFO0_NUM_EHT_SIG_SYM); - - common |= ATH12K_LE32_DEC_ENC(usig_mu->info0, - HAL_RX_USIG_MU_INFO0_RX_INTEG_CHECK_PASS, - IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC); - - value |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD | - IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE | - u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode, - IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE) | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE | - ATH12K_LE32_DEC_ENC(usig_mu->info0, - HAL_RX_USIG_MU_INFO0_PUNC_CH_INFO, - punc) | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE | - u32_encode_bits(ppdu_info->u_sig_info.eht_sig_mcs, - IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS) | - u32_encode_bits(ppdu_info->u_sig_info.num_eht_sig_sym, - sig_symb) | - ATH12K_LE32_DEC_ENC(usig_mu->info0, - HAL_RX_USIG_MU_INFO0_CRC, - IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC) | - ATH12K_LE32_DEC_ENC(usig_mu->info0, - HAL_RX_USIG_MU_INFO0_TAIL, - IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL); - - mask |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD | - IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE | - punc | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS | - sig_symb | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC | - IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL; - - usig->common = cpu_to_le32(common); - usig->value = cpu_to_le32(value); - usig->mask = cpu_to_le32(mask); -} - static void -ath12k_dp_mon_hal_rx_parse_u_sig_hdr(const struct hal_mon_usig_hdr *usig, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - u8 comp_mode; - - ppdu_info->eht_usig = true; - - ath12k_dp_mon_hal_rx_parse_u_sig_cmn(&usig->cmn, ppdu_info); - - comp_mode = le32_get_bits(usig->non_cmn.mu.info0, - HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE); - - if (comp_mode == 0 && ppdu_info->u_sig_info.ul_dl) - ath12k_dp_mon_hal_rx_parse_u_sig_tb(&usig->non_cmn.tb, ppdu_info); - else - ath12k_dp_mon_hal_rx_parse_u_sig_mu(&usig->non_cmn.mu, ppdu_info); -} - -static void -ath12k_dp_mon_hal_aggr_tlv(struct hal_rx_mon_ppdu_info *ppdu_info, - u16 tlv_len, const void *tlv_data) -{ - if (tlv_len <= HAL_RX_MON_MAX_AGGR_SIZE - ppdu_info->tlv_aggr.cur_len) { - memcpy(ppdu_info->tlv_aggr.buf + ppdu_info->tlv_aggr.cur_len, - tlv_data, tlv_len); - ppdu_info->tlv_aggr.cur_len += tlv_len; - } -} - -static inline bool -ath12k_dp_mon_hal_rx_is_frame_type_ndp(const struct hal_rx_u_sig_info *usig_info) -{ - if (usig_info->ppdu_type_comp_mode == 1 && - usig_info->eht_sig_mcs == 0 && - usig_info->num_eht_sig_sym == 0) - return true; - - return false; -} - -static inline bool -ath12k_dp_mon_hal_rx_is_non_ofdma(const struct hal_rx_u_sig_info *usig_info) -{ - u32 ppdu_type_comp_mode = usig_info->ppdu_type_comp_mode; - u32 ul_dl = usig_info->ul_dl; - - if ((ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO && ul_dl == 0) || - (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_OFDMA && ul_dl == 0) || - (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO && ul_dl == 1)) - return true; - - return false; -} - -static inline bool -ath12k_dp_mon_hal_rx_is_ofdma(const struct hal_rx_u_sig_info *usig_info) -{ - if (usig_info->ppdu_type_comp_mode == 0 && usig_info->ul_dl == 0) - return true; - - return false; -} - -static void -ath12k_dp_mon_hal_rx_parse_eht_sig_ndp(const struct hal_eht_sig_ndp_cmn_eb *eht_sig_ndp, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; - u32 known, data; - - known = __le32_to_cpu(eht->known); - known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE | - IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF | - IEEE80211_RADIOTAP_EHT_KNOWN_NSS_S | - IEEE80211_RADIOTAP_EHT_KNOWN_BEAMFORMED_S | - IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_S | - IEEE80211_RADIOTAP_EHT_KNOWN_CRC1 | - IEEE80211_RADIOTAP_EHT_KNOWN_TAIL1; - eht->known = cpu_to_le32(known); - - data = __le32_to_cpu(eht->data[0]); - data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, - HAL_RX_EHT_SIG_NDP_CMN_INFO0_SPATIAL_REUSE, - IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE); - /* GI and LTF size are separately indicated in radiotap header - * and hence will be parsed from other TLV - */ - data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, - HAL_RX_EHT_SIG_NDP_CMN_INFO0_NUM_LTF_SYM, - IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF); - - data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, - HAL_RX_EHT_SIG_NDP_CMN_INFO0_CRC, - IEEE80211_RADIOTAP_EHT_DATA0_CRC1_O); - - data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, - HAL_RX_EHT_SIG_NDP_CMN_INFO0_DISREGARD, - IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_S); - eht->data[0] = cpu_to_le32(data); - - data = __le32_to_cpu(eht->data[7]); - data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, - HAL_RX_EHT_SIG_NDP_CMN_INFO0_NSS, - IEEE80211_RADIOTAP_EHT_DATA7_NSS_S); - - data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, - HAL_RX_EHT_SIG_NDP_CMN_INFO0_BEAMFORMED, - IEEE80211_RADIOTAP_EHT_DATA7_BEAMFORMED_S); - eht->data[7] = cpu_to_le32(data); -} - -static void -ath12k_dp_mon_hal_rx_parse_usig_overflow(const struct hal_eht_sig_usig_overflow *ovflow, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; - u32 known, data; - - known = __le32_to_cpu(eht->known); - known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE | - IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF | - IEEE80211_RADIOTAP_EHT_KNOWN_LDPC_EXTRA_SYM_OM | - IEEE80211_RADIOTAP_EHT_KNOWN_PRE_PADD_FACOR_OM | - IEEE80211_RADIOTAP_EHT_KNOWN_PE_DISAMBIGUITY_OM | - IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_O; - eht->known = cpu_to_le32(known); - - data = __le32_to_cpu(eht->data[0]); - data |= ATH12K_LE32_DEC_ENC(ovflow->info0, - HAL_RX_EHT_SIG_OVERFLOW_INFO0_SPATIAL_REUSE, - IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE); - - /* GI and LTF size are separately indicated in radiotap header - * and hence will be parsed from other TLV - */ - data |= ATH12K_LE32_DEC_ENC(ovflow->info0, - HAL_RX_EHT_SIG_OVERFLOW_INFO0_NUM_LTF_SYM, - IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF); - - data |= ATH12K_LE32_DEC_ENC(ovflow->info0, - HAL_RX_EHT_SIG_OVERFLOW_INFO0_LDPC_EXTA_SYM, - IEEE80211_RADIOTAP_EHT_DATA0_LDPC_EXTRA_SYM_OM); - - data |= ATH12K_LE32_DEC_ENC(ovflow->info0, - HAL_RX_EHT_SIG_OVERFLOW_INFO0_PRE_FEC_PAD_FACTOR, - IEEE80211_RADIOTAP_EHT_DATA0_PRE_PADD_FACOR_OM); - - data |= ATH12K_LE32_DEC_ENC(ovflow->info0, - HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISAMBIGUITY, - IEEE80211_RADIOTAP_EHT_DATA0_PE_DISAMBIGUITY_OM); - - data |= ATH12K_LE32_DEC_ENC(ovflow->info0, - HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISREGARD, - IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_O); - eht->data[0] = cpu_to_le32(data); -} - -static void -ath12k_dp_mon_hal_rx_parse_non_ofdma_users(const struct hal_eht_sig_non_ofdma_cmn_eb *eb, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; - u32 known, data; - - known = __le32_to_cpu(eht->known); - known |= IEEE80211_RADIOTAP_EHT_KNOWN_NR_NON_OFDMA_USERS_M; - eht->known = cpu_to_le32(known); - - data = __le32_to_cpu(eht->data[7]); - data |= ATH12K_LE32_DEC_ENC(eb->info0, - HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_USERS, - IEEE80211_RADIOTAP_EHT_DATA7_NUM_OF_NON_OFDMA_USERS); - eht->data[7] = cpu_to_le32(data); -} - -static void -ath12k_dp_mon_hal_rx_parse_eht_mumimo_user(const struct hal_eht_sig_mu_mimo *user, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info; - u32 user_idx; - - if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info)) - return; - - user_idx = eht_info->num_user_info++; - - eht_info->user_info[user_idx] |= - IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN | - IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN | - IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN | - IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_KNOWN_M | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_STA_ID, - IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CODING, - IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS, - IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_SPATIAL_CODING, - IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_M); - - ppdu_info->mcs = le32_get_bits(user->info0, - HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS); -} - -static void -ath12k_dp_mon_hal_rx_parse_eht_non_mumimo_user(const struct hal_eht_sig_non_mu_mimo *user, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info; - u32 user_idx; - - if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info)) - return; - - user_idx = eht_info->num_user_info++; - - eht_info->user_info[user_idx] |= - IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN | - IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN | - IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN | - IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_KNOWN_O | - IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_KNOWN_O | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_STA_ID, - IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CODING, - IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS, - IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS, - IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_O) | - ATH12K_LE32_DEC_ENC(user->info0, - HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_BEAMFORMED, - IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_O); - - ppdu_info->mcs = le32_get_bits(user->info0, - HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS); - - ppdu_info->nss = le32_get_bits(user->info0, - HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS) + 1; -} - -static inline bool -ath12k_dp_mon_hal_rx_is_mu_mimo_user(const struct hal_rx_u_sig_info *usig_info) -{ - if (usig_info->ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_SU && - usig_info->ul_dl == 1) - return true; - - return false; -} - -static void -ath12k_dp_mon_hal_rx_parse_eht_sig_non_ofdma(const void *tlv, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - const struct hal_eht_sig_non_ofdma_cmn_eb *eb = tlv; - - ath12k_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info); - ath12k_dp_mon_hal_rx_parse_non_ofdma_users(eb, ppdu_info); - - if (ath12k_dp_mon_hal_rx_is_mu_mimo_user(&ppdu_info->u_sig_info)) - ath12k_dp_mon_hal_rx_parse_eht_mumimo_user(&eb->user_field.mu_mimo, - ppdu_info); - else - ath12k_dp_mon_hal_rx_parse_eht_non_mumimo_user(&eb->user_field.n_mu_mimo, - ppdu_info); -} - -static void -ath12k_dp_mon_hal_rx_parse_ru_allocation(const struct hal_eht_sig_ofdma_cmn_eb *eb, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - const struct hal_eht_sig_ofdma_cmn_eb1 *ofdma_cmn_eb1 = &eb->eb1; - const struct hal_eht_sig_ofdma_cmn_eb2 *ofdma_cmn_eb2 = &eb->eb2; - struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; - enum ieee80211_radiotap_eht_data ru_123, ru_124, ru_125, ru_126; - enum ieee80211_radiotap_eht_data ru_121, ru_122, ru_112, ru_111; - u32 data; - - ru_123 = IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3; - ru_124 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4; - ru_125 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5; - ru_126 = IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6; - ru_121 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1; - ru_122 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2; - ru_112 = IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2; - ru_111 = IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1; - - switch (ppdu_info->u_sig_info.bw) { - case HAL_EHT_BW_320_2: - case HAL_EHT_BW_320_1: - data = __le32_to_cpu(eht->data[4]); - /* CC1 2::3 */ - data |= IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, - HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_3, - ru_123); - eht->data[4] = cpu_to_le32(data); - - data = __le32_to_cpu(eht->data[5]); - /* CC1 2::4 */ - data |= IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, - HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_4, - ru_124); - - /* CC1 2::5 */ - data |= IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, - HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_5, - ru_125); - eht->data[5] = cpu_to_le32(data); - - data = __le32_to_cpu(eht->data[6]); - /* CC1 2::6 */ - data |= IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, - HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_6, - ru_126); - eht->data[6] = cpu_to_le32(data); - - fallthrough; - case HAL_EHT_BW_160: - data = __le32_to_cpu(eht->data[3]); - /* CC1 2::1 */ - data |= IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, - HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_1, - ru_121); - /* CC1 2::2 */ - data |= IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, - HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_2, - ru_122); - eht->data[3] = cpu_to_le32(data); - - fallthrough; - case HAL_EHT_BW_80: - data = __le32_to_cpu(eht->data[2]); - /* CC1 1::2 */ - data |= IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0, - HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_2, - ru_112); - eht->data[2] = cpu_to_le32(data); - - fallthrough; - case HAL_EHT_BW_40: - fallthrough; - case HAL_EHT_BW_20: - data = __le32_to_cpu(eht->data[1]); - /* CC1 1::1 */ - data |= IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1_KNOWN | - ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0, - HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_1, - ru_111); - eht->data[1] = cpu_to_le32(data); - break; - default: - break; - } -} - -static void -ath12k_dp_mon_hal_rx_parse_eht_sig_ofdma(const void *tlv, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - const struct hal_eht_sig_ofdma_cmn_eb *ofdma = tlv; - - ath12k_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info); - ath12k_dp_mon_hal_rx_parse_ru_allocation(ofdma, ppdu_info); - - ath12k_dp_mon_hal_rx_parse_eht_non_mumimo_user(&ofdma->user_field.n_mu_mimo, - ppdu_info); -} - -static void -ath12k_dp_mon_parse_eht_sig_hdr(struct hal_rx_mon_ppdu_info *ppdu_info, - const void *tlv_data) -{ - ppdu_info->is_eht = true; - - if (ath12k_dp_mon_hal_rx_is_frame_type_ndp(&ppdu_info->u_sig_info)) - ath12k_dp_mon_hal_rx_parse_eht_sig_ndp(tlv_data, ppdu_info); - else if (ath12k_dp_mon_hal_rx_is_non_ofdma(&ppdu_info->u_sig_info)) - ath12k_dp_mon_hal_rx_parse_eht_sig_non_ofdma(tlv_data, ppdu_info); - else if (ath12k_dp_mon_hal_rx_is_ofdma(&ppdu_info->u_sig_info)) - ath12k_dp_mon_hal_rx_parse_eht_sig_ofdma(tlv_data, ppdu_info); -} - -static inline enum ath12k_eht_ru_size -hal_rx_mon_hal_ru_size_to_ath12k_ru_size(u32 hal_ru_size) -{ - switch (hal_ru_size) { - case HAL_EHT_RU_26: - return ATH12K_EHT_RU_26; - case HAL_EHT_RU_52: - return ATH12K_EHT_RU_52; - case HAL_EHT_RU_78: - return ATH12K_EHT_RU_52_26; - case HAL_EHT_RU_106: - return ATH12K_EHT_RU_106; - case HAL_EHT_RU_132: - return ATH12K_EHT_RU_106_26; - case HAL_EHT_RU_242: - return ATH12K_EHT_RU_242; - case HAL_EHT_RU_484: - return ATH12K_EHT_RU_484; - case HAL_EHT_RU_726: - return ATH12K_EHT_RU_484_242; - case HAL_EHT_RU_996: - return ATH12K_EHT_RU_996; - case HAL_EHT_RU_996x2: - return ATH12K_EHT_RU_996x2; - case HAL_EHT_RU_996x3: - return ATH12K_EHT_RU_996x3; - case HAL_EHT_RU_996x4: - return ATH12K_EHT_RU_996x4; - case HAL_EHT_RU_NONE: - return ATH12K_EHT_RU_INVALID; - case HAL_EHT_RU_996_484: - return ATH12K_EHT_RU_996_484; - case HAL_EHT_RU_996x2_484: - return ATH12K_EHT_RU_996x2_484; - case HAL_EHT_RU_996x3_484: - return ATH12K_EHT_RU_996x3_484; - case HAL_EHT_RU_996_484_242: - return ATH12K_EHT_RU_996_484_242; - default: - return ATH12K_EHT_RU_INVALID; - } -} - -static inline u32 -hal_rx_ul_ofdma_ru_size_to_width(enum ath12k_eht_ru_size ru_size) -{ - switch (ru_size) { - case ATH12K_EHT_RU_26: - return RU_26; - case ATH12K_EHT_RU_52: - return RU_52; - case ATH12K_EHT_RU_52_26: - return RU_52_26; - case ATH12K_EHT_RU_106: - return RU_106; - case ATH12K_EHT_RU_106_26: - return RU_106_26; - case ATH12K_EHT_RU_242: - return RU_242; - case ATH12K_EHT_RU_484: - return RU_484; - case ATH12K_EHT_RU_484_242: - return RU_484_242; - case ATH12K_EHT_RU_996: - return RU_996; - case ATH12K_EHT_RU_996_484: - return RU_996_484; - case ATH12K_EHT_RU_996_484_242: - return RU_996_484_242; - case ATH12K_EHT_RU_996x2: - return RU_2X996; - case ATH12K_EHT_RU_996x2_484: - return RU_2X996_484; - case ATH12K_EHT_RU_996x3: - return RU_3X996; - case ATH12K_EHT_RU_996x3_484: - return RU_3X996_484; - case ATH12K_EHT_RU_996x4: - return RU_4X996; - default: - return RU_INVALID; - } -} - -static void -ath12k_dp_mon_hal_rx_parse_user_info(const struct hal_receive_user_info *rx_usr_info, - u16 user_id, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct hal_rx_user_status *mon_rx_user_status = NULL; - struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; - enum ath12k_eht_ru_size rtap_ru_size = ATH12K_EHT_RU_INVALID; - u32 ru_width, reception_type, ru_index = HAL_EHT_RU_INVALID; - u32 ru_type_80_0, ru_start_index_80_0; - u32 ru_type_80_1, ru_start_index_80_1; - u32 ru_type_80_2, ru_start_index_80_2; - u32 ru_type_80_3, ru_start_index_80_3; - u32 ru_size = 0, num_80mhz_with_ru = 0; - u64 ru_index_320mhz = 0; - u32 ru_index_per80mhz; - - reception_type = le32_get_bits(rx_usr_info->info0, - HAL_RX_USR_INFO0_RECEPTION_TYPE); - - switch (reception_type) { - case HAL_RECEPTION_TYPE_SU: - ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_SU; - break; - case HAL_RECEPTION_TYPE_DL_MU_MIMO: - case HAL_RECEPTION_TYPE_UL_MU_MIMO: - ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_MIMO; - break; - case HAL_RECEPTION_TYPE_DL_MU_OFMA: - case HAL_RECEPTION_TYPE_UL_MU_OFDMA: - ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA; - break; - case HAL_RECEPTION_TYPE_DL_MU_OFDMA_MIMO: - case HAL_RECEPTION_TYPE_UL_MU_OFDMA_MIMO: - ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO; - } - - ppdu_info->is_stbc = le32_get_bits(rx_usr_info->info0, HAL_RX_USR_INFO0_STBC); - ppdu_info->ldpc = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_LDPC); - ppdu_info->dcm = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_STA_DCM); - ppdu_info->bw = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_RX_BW); - ppdu_info->mcs = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_MCS); - ppdu_info->nss = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_NSS) + 1; - - if (user_id < HAL_MAX_UL_MU_USERS) { - mon_rx_user_status = &ppdu_info->userstats[user_id]; - mon_rx_user_status->mcs = ppdu_info->mcs; - mon_rx_user_status->nss = ppdu_info->nss; - } - - if (!(ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_MIMO || - ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA || - ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO)) - return; - - /* RU allocation present only for OFDMA reception */ - ru_type_80_0 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_0); - ru_start_index_80_0 = le32_get_bits(rx_usr_info->info3, - HAL_RX_USR_INFO3_RU_START_IDX_80_0); - if (ru_type_80_0 != HAL_EHT_RU_NONE) { - ru_size += ru_type_80_0; - ru_index_per80mhz = ru_start_index_80_0; - ru_index = ru_index_per80mhz; - ru_index_320mhz |= HAL_RU_PER80(ru_type_80_0, 0, ru_index_per80mhz); - num_80mhz_with_ru++; - } - - ru_type_80_1 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_1); - ru_start_index_80_1 = le32_get_bits(rx_usr_info->info3, - HAL_RX_USR_INFO3_RU_START_IDX_80_1); - if (ru_type_80_1 != HAL_EHT_RU_NONE) { - ru_size += ru_type_80_1; - ru_index_per80mhz = ru_start_index_80_1; - ru_index = ru_index_per80mhz; - ru_index_320mhz |= HAL_RU_PER80(ru_type_80_1, 1, ru_index_per80mhz); - num_80mhz_with_ru++; - } - - ru_type_80_2 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_2); - ru_start_index_80_2 = le32_get_bits(rx_usr_info->info3, - HAL_RX_USR_INFO3_RU_START_IDX_80_2); - if (ru_type_80_2 != HAL_EHT_RU_NONE) { - ru_size += ru_type_80_2; - ru_index_per80mhz = ru_start_index_80_2; - ru_index = ru_index_per80mhz; - ru_index_320mhz |= HAL_RU_PER80(ru_type_80_2, 2, ru_index_per80mhz); - num_80mhz_with_ru++; - } - - ru_type_80_3 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_3); - ru_start_index_80_3 = le32_get_bits(rx_usr_info->info2, - HAL_RX_USR_INFO3_RU_START_IDX_80_3); - if (ru_type_80_3 != HAL_EHT_RU_NONE) { - ru_size += ru_type_80_3; - ru_index_per80mhz = ru_start_index_80_3; - ru_index = ru_index_per80mhz; - ru_index_320mhz |= HAL_RU_PER80(ru_type_80_3, 3, ru_index_per80mhz); - num_80mhz_with_ru++; - } - - if (num_80mhz_with_ru > 1) { - /* Calculate the MRU index */ - switch (ru_index_320mhz) { - case HAL_EHT_RU_996_484_0: - case HAL_EHT_RU_996x2_484_0: - case HAL_EHT_RU_996x3_484_0: - ru_index = 0; - break; - case HAL_EHT_RU_996_484_1: - case HAL_EHT_RU_996x2_484_1: - case HAL_EHT_RU_996x3_484_1: - ru_index = 1; - break; - case HAL_EHT_RU_996_484_2: - case HAL_EHT_RU_996x2_484_2: - case HAL_EHT_RU_996x3_484_2: - ru_index = 2; - break; - case HAL_EHT_RU_996_484_3: - case HAL_EHT_RU_996x2_484_3: - case HAL_EHT_RU_996x3_484_3: - ru_index = 3; - break; - case HAL_EHT_RU_996_484_4: - case HAL_EHT_RU_996x2_484_4: - case HAL_EHT_RU_996x3_484_4: - ru_index = 4; - break; - case HAL_EHT_RU_996_484_5: - case HAL_EHT_RU_996x2_484_5: - case HAL_EHT_RU_996x3_484_5: - ru_index = 5; - break; - case HAL_EHT_RU_996_484_6: - case HAL_EHT_RU_996x2_484_6: - case HAL_EHT_RU_996x3_484_6: - ru_index = 6; - break; - case HAL_EHT_RU_996_484_7: - case HAL_EHT_RU_996x2_484_7: - case HAL_EHT_RU_996x3_484_7: - ru_index = 7; - break; - case HAL_EHT_RU_996x2_484_8: - ru_index = 8; - break; - case HAL_EHT_RU_996x2_484_9: - ru_index = 9; - break; - case HAL_EHT_RU_996x2_484_10: - ru_index = 10; - break; - case HAL_EHT_RU_996x2_484_11: - ru_index = 11; - break; - default: - ru_index = HAL_EHT_RU_INVALID; - break; - } - - ru_size += 4; - } - - rtap_ru_size = hal_rx_mon_hal_ru_size_to_ath12k_ru_size(ru_size); - if (rtap_ru_size != ATH12K_EHT_RU_INVALID) { - u32 known, data; - - known = __le32_to_cpu(eht->known); - known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_SIZE_OM; - eht->known = cpu_to_le32(known); - - data = __le32_to_cpu(eht->data[1]); - data |= u32_encode_bits(rtap_ru_size, - IEEE80211_RADIOTAP_EHT_DATA1_RU_SIZE); - eht->data[1] = cpu_to_le32(data); - } - - if (ru_index != HAL_EHT_RU_INVALID) { - u32 known, data; - - known = __le32_to_cpu(eht->known); - known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_INDEX_OM; - eht->known = cpu_to_le32(known); - - data = __le32_to_cpu(eht->data[1]); - data |= u32_encode_bits(rtap_ru_size, - IEEE80211_RADIOTAP_EHT_DATA1_RU_INDEX); - eht->data[1] = cpu_to_le32(data); - } - - if (mon_rx_user_status && ru_index != HAL_EHT_RU_INVALID && - rtap_ru_size != ATH12K_EHT_RU_INVALID) { - mon_rx_user_status->ul_ofdma_ru_start_index = ru_index; - mon_rx_user_status->ul_ofdma_ru_size = rtap_ru_size; - - ru_width = hal_rx_ul_ofdma_ru_size_to_width(rtap_ru_size); - - mon_rx_user_status->ul_ofdma_ru_width = ru_width; - mon_rx_user_status->ofdma_info_valid = 1; - } -} - -static void ath12k_dp_mon_parse_rx_msdu_end_err(u32 info, u32 *errmap) -{ - if (info & RX_MSDU_END_INFO13_FCS_ERR) - *errmap |= HAL_RX_MPDU_ERR_FCS; - - if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) - *errmap |= HAL_RX_MPDU_ERR_DECRYPT; - - if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) - *errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; - - if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) - *errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; - - if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) - *errmap |= HAL_RX_MPDU_ERR_OVERFLOW; - - if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) - *errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; - - if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) - *errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; -} - -static void -ath12k_parse_cmn_usr_info(const struct hal_phyrx_common_user_info *cmn_usr_info, - struct hal_rx_mon_ppdu_info *ppdu_info) -{ - struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; - u32 known, data, cp_setting, ltf_size; - - known = __le32_to_cpu(eht->known); - known |= IEEE80211_RADIOTAP_EHT_KNOWN_GI | - IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF; - eht->known = cpu_to_le32(known); - - cp_setting = le32_get_bits(cmn_usr_info->info0, - HAL_RX_CMN_USR_INFO0_CP_SETTING); - ltf_size = le32_get_bits(cmn_usr_info->info0, - HAL_RX_CMN_USR_INFO0_LTF_SIZE); - - data = __le32_to_cpu(eht->data[0]); - data |= u32_encode_bits(cp_setting, IEEE80211_RADIOTAP_EHT_DATA0_GI); - data |= u32_encode_bits(ltf_size, IEEE80211_RADIOTAP_EHT_DATA0_LTF); - eht->data[0] = cpu_to_le32(data); - - if (!ppdu_info->ltf_size) - ppdu_info->ltf_size = ltf_size; - if (!ppdu_info->gi) - ppdu_info->gi = cp_setting; -} - -static void -ath12k_dp_mon_parse_status_msdu_end(struct ath12k_mon_data *pmon, - const struct hal_rx_msdu_end *msdu_end) -{ - ath12k_dp_mon_parse_rx_msdu_end_err(__le32_to_cpu(msdu_end->info2), - &pmon->err_bitmap); - pmon->decap_format = le32_get_bits(msdu_end->info1, - RX_MSDU_END_INFO11_DECAP_FORMAT); -} - -static enum hal_rx_mon_status -ath12k_dp_mon_rx_parse_status_tlv(struct ath12k *ar, - struct ath12k_mon_data *pmon, - const struct hal_tlv_64_hdr *tlv) -{ - struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; - const void *tlv_data = tlv->value; - u32 info[7], userid; - u16 tlv_tag, tlv_len; - - tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG); - tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN); - userid = le64_get_bits(tlv->tl, HAL_TLV_64_USR_ID); - - if (ppdu_info->tlv_aggr.in_progress && ppdu_info->tlv_aggr.tlv_tag != tlv_tag) { - ath12k_dp_mon_parse_eht_sig_hdr(ppdu_info, ppdu_info->tlv_aggr.buf); - - ppdu_info->tlv_aggr.in_progress = false; - ppdu_info->tlv_aggr.cur_len = 0; - } - - switch (tlv_tag) { - case HAL_RX_PPDU_START: { - const struct hal_rx_ppdu_start *ppdu_start = tlv_data; - - u64 ppdu_ts = ath12k_le32hilo_to_u64(ppdu_start->ppdu_start_ts_63_32, - ppdu_start->ppdu_start_ts_31_0); - - info[0] = __le32_to_cpu(ppdu_start->info0); - - ppdu_info->ppdu_id = u32_get_bits(info[0], - HAL_RX_PPDU_START_INFO0_PPDU_ID); - - info[1] = __le32_to_cpu(ppdu_start->info1); - ppdu_info->chan_num = u32_get_bits(info[1], - HAL_RX_PPDU_START_INFO1_CHAN_NUM); - ppdu_info->freq = u32_get_bits(info[1], - HAL_RX_PPDU_START_INFO1_CHAN_FREQ); - ppdu_info->ppdu_ts = ppdu_ts; - - if (ppdu_info->ppdu_id != ppdu_info->last_ppdu_id) { - ppdu_info->last_ppdu_id = ppdu_info->ppdu_id; - ppdu_info->num_users = 0; - memset(&ppdu_info->mpdu_fcs_ok_bitmap, 0, - HAL_RX_NUM_WORDS_PER_PPDU_BITMAP * - sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0])); - } - break; - } - case HAL_RX_PPDU_END_USER_STATS: { - const struct hal_rx_ppdu_end_user_stats *eu_stats = tlv_data; - u32 tid_bitmap; - - info[0] = __le32_to_cpu(eu_stats->info0); - info[1] = __le32_to_cpu(eu_stats->info1); - info[2] = __le32_to_cpu(eu_stats->info2); - info[4] = __le32_to_cpu(eu_stats->info4); - info[5] = __le32_to_cpu(eu_stats->info5); - info[6] = __le32_to_cpu(eu_stats->info6); - - ppdu_info->ast_index = - u32_get_bits(info[2], HAL_RX_PPDU_END_USER_STATS_INFO2_AST_INDEX); - ppdu_info->fc_valid = - u32_get_bits(info[1], HAL_RX_PPDU_END_USER_STATS_INFO1_FC_VALID); - tid_bitmap = u32_get_bits(info[6], - HAL_RX_PPDU_END_USER_STATS_INFO6_TID_BITMAP); - ppdu_info->tid = ffs(tid_bitmap) - 1; - ppdu_info->tcp_msdu_count = - u32_get_bits(info[4], - HAL_RX_PPDU_END_USER_STATS_INFO4_TCP_MSDU_CNT); - ppdu_info->udp_msdu_count = - u32_get_bits(info[4], - HAL_RX_PPDU_END_USER_STATS_INFO4_UDP_MSDU_CNT); - ppdu_info->other_msdu_count = - u32_get_bits(info[5], - HAL_RX_PPDU_END_USER_STATS_INFO5_OTHER_MSDU_CNT); - ppdu_info->tcp_ack_msdu_count = - u32_get_bits(info[5], - HAL_RX_PPDU_END_USER_STATS_INFO5_TCP_ACK_MSDU_CNT); - ppdu_info->preamble_type = - u32_get_bits(info[1], - HAL_RX_PPDU_END_USER_STATS_INFO1_PKT_TYPE); - ppdu_info->num_mpdu_fcs_ok = - u32_get_bits(info[1], - HAL_RX_PPDU_END_USER_STATS_INFO1_MPDU_CNT_FCS_OK); - ppdu_info->num_mpdu_fcs_err = - u32_get_bits(info[0], - HAL_RX_PPDU_END_USER_STATS_INFO0_MPDU_CNT_FCS_ERR); - ppdu_info->peer_id = - u32_get_bits(info[0], HAL_RX_PPDU_END_USER_STATS_INFO0_PEER_ID); - - switch (ppdu_info->preamble_type) { - case HAL_RX_PREAMBLE_11N: - ppdu_info->ht_flags = 1; - break; - case HAL_RX_PREAMBLE_11AC: - ppdu_info->vht_flags = 1; - break; - case HAL_RX_PREAMBLE_11AX: - ppdu_info->he_flags = 1; - break; - case HAL_RX_PREAMBLE_11BE: - ppdu_info->is_eht = true; - break; - default: - break; - } - - if (userid < HAL_MAX_UL_MU_USERS) { - struct hal_rx_user_status *rxuser_stats = - &ppdu_info->userstats[userid]; - - if (ppdu_info->num_mpdu_fcs_ok > 1 || - ppdu_info->num_mpdu_fcs_err > 1) - ppdu_info->userstats[userid].ampdu_present = true; - - ppdu_info->num_users += 1; - - ath12k_dp_mon_rx_handle_ofdma_info(eu_stats, rxuser_stats); - ath12k_dp_mon_rx_populate_mu_user_info(eu_stats, ppdu_info, - rxuser_stats); - } - ppdu_info->mpdu_fcs_ok_bitmap[0] = __le32_to_cpu(eu_stats->rsvd1[0]); - ppdu_info->mpdu_fcs_ok_bitmap[1] = __le32_to_cpu(eu_stats->rsvd1[1]); - break; - } - case HAL_RX_PPDU_END_USER_STATS_EXT: { - const struct hal_rx_ppdu_end_user_stats_ext *eu_stats = tlv_data; - - ppdu_info->mpdu_fcs_ok_bitmap[2] = __le32_to_cpu(eu_stats->info1); - ppdu_info->mpdu_fcs_ok_bitmap[3] = __le32_to_cpu(eu_stats->info2); - ppdu_info->mpdu_fcs_ok_bitmap[4] = __le32_to_cpu(eu_stats->info3); - ppdu_info->mpdu_fcs_ok_bitmap[5] = __le32_to_cpu(eu_stats->info4); - ppdu_info->mpdu_fcs_ok_bitmap[6] = __le32_to_cpu(eu_stats->info5); - ppdu_info->mpdu_fcs_ok_bitmap[7] = __le32_to_cpu(eu_stats->info6); - break; - } - case HAL_PHYRX_HT_SIG: - ath12k_dp_mon_parse_ht_sig(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_L_SIG_B: - ath12k_dp_mon_parse_l_sig_b(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_L_SIG_A: - ath12k_dp_mon_parse_l_sig_a(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_VHT_SIG_A: - ath12k_dp_mon_parse_vht_sig_a(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_HE_SIG_A_SU: - ath12k_dp_mon_parse_he_sig_su(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_HE_SIG_A_MU_DL: - ath12k_dp_mon_parse_he_sig_mu(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_HE_SIG_B1_MU: - ath12k_dp_mon_parse_he_sig_b1_mu(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_HE_SIG_B2_MU: - ath12k_dp_mon_parse_he_sig_b2_mu(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_HE_SIG_B2_OFDMA: - ath12k_dp_mon_parse_he_sig_b2_ofdma(tlv_data, ppdu_info); - break; - - case HAL_PHYRX_RSSI_LEGACY: { - const struct hal_rx_phyrx_rssi_legacy_info *rssi = tlv_data; - - info[0] = __le32_to_cpu(rssi->info0); - info[2] = __le32_to_cpu(rssi->info2); - - /* TODO: Please note that the combined rssi will not be accurate - * in MU case. Rssi in MU needs to be retrieved from - * PHYRX_OTHER_RECEIVE_INFO TLV. - */ - ppdu_info->rssi_comb = - u32_get_bits(info[2], - HAL_RX_RSSI_LEGACY_INFO_INFO2_RSSI_COMB_PPDU); - - ppdu_info->bw = u32_get_bits(info[0], - HAL_RX_RSSI_LEGACY_INFO_INFO0_RX_BW); - break; - } - case HAL_PHYRX_COMMON_USER_INFO: { - ath12k_parse_cmn_usr_info(tlv_data, ppdu_info); - break; - } - case HAL_RX_PPDU_START_USER_INFO: - ath12k_dp_mon_hal_rx_parse_user_info(tlv_data, userid, ppdu_info); - break; - - case HAL_RXPCU_PPDU_END_INFO: { - const struct hal_rx_ppdu_end_duration *ppdu_rx_duration = tlv_data; - - info[0] = __le32_to_cpu(ppdu_rx_duration->info0); - ppdu_info->rx_duration = - u32_get_bits(info[0], HAL_RX_PPDU_END_DURATION); - ppdu_info->tsft = __le32_to_cpu(ppdu_rx_duration->rsvd0[1]); - ppdu_info->tsft = (ppdu_info->tsft << 32) | - __le32_to_cpu(ppdu_rx_duration->rsvd0[0]); - break; - } - case HAL_RX_MPDU_START: { - const struct hal_rx_mpdu_start *mpdu_start = tlv_data; - u16 peer_id; - - info[1] = __le32_to_cpu(mpdu_start->info1); - peer_id = u32_get_bits(info[1], HAL_RX_MPDU_START_INFO1_PEERID); - if (peer_id) - ppdu_info->peer_id = peer_id; - - ppdu_info->mpdu_len += u32_get_bits(info[1], - HAL_RX_MPDU_START_INFO2_MPDU_LEN); - if (userid < HAL_MAX_UL_MU_USERS) { - info[0] = __le32_to_cpu(mpdu_start->info0); - ppdu_info->userid = userid; - ppdu_info->userstats[userid].ampdu_id = - u32_get_bits(info[0], HAL_RX_MPDU_START_INFO0_PPDU_ID); - } - - return HAL_RX_MON_STATUS_MPDU_START; - } - case HAL_RX_MSDU_START: - /* TODO: add msdu start parsing logic */ - break; - case HAL_MON_BUF_ADDR: - return HAL_RX_MON_STATUS_BUF_ADDR; - case HAL_RX_MSDU_END: - ath12k_dp_mon_parse_status_msdu_end(pmon, tlv_data); - return HAL_RX_MON_STATUS_MSDU_END; - case HAL_RX_MPDU_END: - return HAL_RX_MON_STATUS_MPDU_END; - case HAL_PHYRX_GENERIC_U_SIG: - ath12k_dp_mon_hal_rx_parse_u_sig_hdr(tlv_data, ppdu_info); - break; - case HAL_PHYRX_GENERIC_EHT_SIG: - /* Handle the case where aggregation is in progress - * or the current TLV is one of the TLVs which should be - * aggregated - */ - if (!ppdu_info->tlv_aggr.in_progress) { - ppdu_info->tlv_aggr.in_progress = true; - ppdu_info->tlv_aggr.tlv_tag = tlv_tag; - ppdu_info->tlv_aggr.cur_len = 0; - } - - ppdu_info->is_eht = true; - - ath12k_dp_mon_hal_aggr_tlv(ppdu_info, tlv_len, tlv_data); - break; - case HAL_DUMMY: - return HAL_RX_MON_STATUS_BUF_DONE; - case HAL_RX_PPDU_END_STATUS_DONE: - case 0: - return HAL_RX_MON_STATUS_PPDU_DONE; - default: - break; - } - - return HAL_RX_MON_STATUS_PPDU_NOT_DONE; -} - -static void -ath12k_dp_mon_fill_rx_stats_info(struct ath12k *ar, - struct hal_rx_mon_ppdu_info *ppdu_info, +ath12k_dp_mon_fill_rx_stats_info(struct hal_rx_mon_ppdu_info *ppdu_info, struct ieee80211_rx_status *rx_status) { u32 center_freq = ppdu_info->freq; @@ -1776,7 +36,7 @@ ath12k_dp_mon_fill_rx_stats_info(struct ath12k *ar, } } -static struct sk_buff +struct sk_buff *ath12k_dp_rx_alloc_mon_status_buf(struct ath12k_base *ab, struct dp_rxdma_mon_ring *rx_ring, int *buf_id) @@ -1819,48 +79,9 @@ static struct sk_buff fail_alloc_skb: return NULL; } +EXPORT_SYMBOL(ath12k_dp_rx_alloc_mon_status_buf); -static enum dp_mon_status_buf_state -ath12k_dp_rx_mon_buf_done(struct ath12k_base *ab, struct hal_srng *srng, - struct dp_rxdma_mon_ring *rx_ring) -{ - struct ath12k_skb_rxcb *rxcb; - struct hal_tlv_64_hdr *tlv; - struct sk_buff *skb; - void *status_desc; - dma_addr_t paddr; - u32 cookie; - int buf_id; - u8 rbm; - - status_desc = ath12k_hal_srng_src_next_peek(ab, srng); - if (!status_desc) - return DP_MON_STATUS_NO_DMA; - - ath12k_hal_rx_buf_addr_info_get(status_desc, &paddr, &cookie, &rbm); - - buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); - - spin_lock_bh(&rx_ring->idr_lock); - skb = idr_find(&rx_ring->bufs_idr, buf_id); - spin_unlock_bh(&rx_ring->idr_lock); - - if (!skb) - return DP_MON_STATUS_NO_DMA; - - rxcb = ATH12K_SKB_RXCB(skb); - dma_sync_single_for_cpu(ab->dev, rxcb->paddr, - skb->len + skb_tailroom(skb), - DMA_FROM_DEVICE); - - tlv = (struct hal_tlv_64_hdr *)skb->data; - if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) != HAL_RX_STATUS_BUFFER_DONE) - return DP_MON_STATUS_NO_DMA; - - return DP_MON_STATUS_REPLINISH; -} - -static u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id) +u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id) { u32 ret = 0; @@ -1879,26 +100,15 @@ static u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id) } return ret; } - -static -void ath12k_dp_mon_next_link_desc_get(struct hal_rx_msdu_link *msdu_link, - dma_addr_t *paddr, u32 *sw_cookie, u8 *rbm, - struct ath12k_buffer_addr **pp_buf_addr_info) -{ - struct ath12k_buffer_addr *buf_addr_info; - - buf_addr_info = &msdu_link->buf_addr_info; - - ath12k_hal_rx_buf_addr_info_get(buf_addr_info, paddr, sw_cookie, rbm); - - *pp_buf_addr_info = buf_addr_info; -} +EXPORT_SYMBOL(ath12k_dp_mon_comp_ppduid); static void -ath12k_dp_mon_fill_rx_rate(struct ath12k *ar, +ath12k_dp_mon_fill_rx_rate(struct ath12k_pdev_dp *dp_pdev, struct hal_rx_mon_ppdu_info *ppdu_info, struct ieee80211_rx_status *rx_status) { + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; struct ieee80211_supported_band *sband; enum rx_msdu_start_pkt_type pkt_type; u8 rate_mcs, nss, sgi; @@ -1914,6 +124,8 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar, case RX_MSDU_START_PKT_TYPE_11B: is_cck = (pkt_type == RX_MSDU_START_PKT_TYPE_11B); if (rx_status->band < NUM_NL80211_BANDS) { + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + sband = &ar->mac.sbands[rx_status->band]; rx_status->rate_idx = ath12k_mac_hw_rate_to_idx(sband, rate_mcs, is_cck); @@ -1922,7 +134,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar, case RX_MSDU_START_PKT_TYPE_11N: rx_status->encoding = RX_ENC_HT; if (rate_mcs > ATH12K_HT_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(ab, "Received with invalid mcs in HT mode %d\n", rate_mcs); break; @@ -1935,7 +147,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar, rx_status->encoding = RX_ENC_VHT; rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH12K_VHT_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(ab, "Received with invalid mcs in VHT mode %d\n", rate_mcs); break; @@ -1946,7 +158,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar, case RX_MSDU_START_PKT_TYPE_11AX: rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH12K_HE_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(ab, "Received with invalid mcs in HE mode %d\n", rate_mcs); break; @@ -1957,7 +169,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar, case RX_MSDU_START_PKT_TYPE_11BE: rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH12K_EHT_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(ab, "Received with invalid mcs in EHT mode %d\n", rate_mcs); break; @@ -1966,24 +178,24 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar, rx_status->he_gi = ath12k_he_gi_to_nl80211_he_gi(sgi); break; default: - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + ath12k_dbg(ab, ATH12K_DBG_DATA, "monitor receives invalid preamble type %d", pkt_type); break; } } -static void ath12k_dp_mon_rx_msdus_set_payload(struct ath12k *ar, +static void ath12k_dp_mon_rx_msdus_set_payload(struct ath12k_base *ab, struct sk_buff *head_msdu, struct sk_buff *tail_msdu) { u32 rx_pkt_offset, l2_hdr_offset, total_offset; - rx_pkt_offset = ar->ab->hal.hal_desc_sz; + rx_pkt_offset = ab->hal.hal_desc_sz; l2_hdr_offset = - ath12k_dp_rx_h_l3pad(ar->ab, (struct hal_rx_desc *)tail_msdu->data); + ath12k_dp_rx_h_l3pad(ab, (struct hal_rx_desc *)tail_msdu->data); - if (ar->ab->hw_params->rxdma1_enable) + if (ab->hw_params->rxdma1_enable) total_offset = ATH12K_MON_RX_PKT_OFFSET; else total_offset = rx_pkt_offset + l2_hdr_offset; @@ -1991,13 +203,14 @@ static void ath12k_dp_mon_rx_msdus_set_payload(struct ath12k *ar, skb_pull(head_msdu, total_offset); } -static struct sk_buff * -ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar, +struct sk_buff * +ath12k_dp_mon_rx_merg_msdus(struct ath12k_pdev_dp *dp_pdev, struct dp_mon_mpdu *mon_mpdu, struct hal_rx_mon_ppdu_info *ppdu_info, struct ieee80211_rx_status *rxs) { - struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; struct sk_buff *msdu, *mpdu_buf, *prev_buf, *head_frag_list; struct sk_buff *head_msdu, *tail_msdu; struct hal_rx_desc *rx_desc; @@ -2014,11 +227,13 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar, if (!head_msdu || !tail_msdu) goto err_merge_fail; - ath12k_dp_mon_fill_rx_stats_info(ar, ppdu_info, rxs); + ath12k_dp_mon_fill_rx_stats_info(ppdu_info, rxs); if (unlikely(rxs->band == NUM_NL80211_BANDS || - !ath12k_ar_to_hw(ar)->wiphy->bands[rxs->band])) { - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + !ath12k_pdev_dp_to_hw(dp_pdev)->wiphy->bands[rxs->band])) { + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + + ath12k_dbg(ab, ATH12K_DBG_DATA, "sband is NULL for status band %d channel_num %d center_freq %d pdev_id %d\n", rxs->band, channel_num, ppdu_info->freq, ar->pdev_idx); @@ -2036,17 +251,17 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar, rxs->freq = ieee80211_channel_to_frequency(channel_num, rxs->band); - ath12k_dp_mon_fill_rx_rate(ar, ppdu_info, rxs); + ath12k_dp_mon_fill_rx_rate(dp_pdev, ppdu_info, rxs); if (decap_format == DP_RX_DECAP_TYPE_RAW) { - ath12k_dp_mon_rx_msdus_set_payload(ar, head_msdu, tail_msdu); + ath12k_dp_mon_rx_msdus_set_payload(ab, head_msdu, tail_msdu); prev_buf = head_msdu; msdu = head_msdu->next; head_frag_list = NULL; while (msdu) { - ath12k_dp_mon_rx_msdus_set_payload(ar, head_msdu, tail_msdu); + ath12k_dp_mon_rx_msdus_set_payload(ab, head_msdu, tail_msdu); if (!head_frag_list) head_frag_list = msdu; @@ -2070,7 +285,7 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar, rx_desc = (struct hal_rx_desc *)head_msdu->data; hdr_desc = - ab->hal_rx_ops->rx_desc_get_msdu_payload(rx_desc); + ab->hal.ops->rx_desc_get_msdu_payload(rx_desc); /* Base size */ wh = (struct ieee80211_hdr_3addr *)hdr_desc; @@ -2081,7 +296,7 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar, msdu = head_msdu; while (msdu) { - ath12k_dp_mon_rx_msdus_set_payload(ar, head_msdu, tail_msdu); + ath12k_dp_mon_rx_msdus_set_payload(ab, head_msdu, tail_msdu); if (qos_pkt) { dest = skb_push(msdu, sizeof(__le16)); if (!dest) @@ -2116,6 +331,7 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar, } return NULL; } +EXPORT_SYMBOL(ath12k_dp_mon_rx_merg_msdus); static void ath12k_dp_mon_rx_update_radiotap_he(struct hal_rx_mon_ppdu_info *rx_status, @@ -2165,11 +381,12 @@ ath12k_dp_mon_rx_update_radiotap_he_mu(struct hal_rx_mon_ppdu_info *rx_status, rtap_buf[rtap_len] = rx_status->he_RU[3]; } -static void ath12k_dp_mon_update_radiotap(struct ath12k *ar, - struct hal_rx_mon_ppdu_info *ppduinfo, - struct sk_buff *mon_skb, - struct ieee80211_rx_status *rxs) +void ath12k_dp_mon_update_radiotap(struct ath12k_pdev_dp *dp_pdev, + struct hal_rx_mon_ppdu_info *ppduinfo, + struct sk_buff *mon_skb, + struct ieee80211_rx_status *rxs) { + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); struct ieee80211_supported_band *sband; s32 noise_floor; u8 *ptr = NULL; @@ -2179,7 +396,7 @@ static void ath12k_dp_mon_update_radiotap(struct ath12k *ar, spin_unlock_bh(&ar->data_lock); rxs->flag |= RX_FLAG_MACTIME_START; - rxs->nss = ppduinfo->nss + 1; + rxs->nss = ppduinfo->nss; if (test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, ar->ab->wmi_ab.svc_map)) rxs->signal = ppduinfo->rssi_comb; @@ -2262,6 +479,9 @@ static void ath12k_dp_mon_update_radiotap(struct ath12k *ar, rxs->encoding = RX_ENC_HT; rxs->rate_idx = ppduinfo->rate; } else { + struct ath12k *ar; + + ar = ath12k_pdev_dp_to_ar(dp_pdev); rxs->encoding = RX_ENC_LEGACY; sband = &ar->mac.sbands[rxs->band]; rxs->rate_idx = ath12k_mac_hw_rate_to_idx(sband, ppduinfo->rate, @@ -2270,13 +490,17 @@ static void ath12k_dp_mon_update_radiotap(struct ath12k *ar, rxs->mactime = ppduinfo->tsft; } +EXPORT_SYMBOL(ath12k_dp_mon_update_radiotap); -static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *napi, - struct sk_buff *msdu, - const struct hal_rx_mon_ppdu_info *ppduinfo, - struct ieee80211_rx_status *status, - u8 decap) +void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, + struct napi_struct *napi, + struct sk_buff *msdu, + const struct hal_rx_mon_ppdu_info *ppduinfo, + struct ieee80211_rx_status *status, + u8 decap) { + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; static const struct ieee80211_radiotap_he known = { .data1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN | IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN), @@ -2285,10 +509,13 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct struct ieee80211_rx_status *rx_status; struct ieee80211_radiotap_he *he = NULL; struct ieee80211_sta *pubsta = NULL; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); + struct hal_rx_desc_data rx_info; bool is_mcbc = rxcb->is_mcbc; bool is_eapol_tkip = rxcb->is_eapol; + struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data; + u8 addr[ETH_ALEN] = {}; status->link_valid = 0; @@ -2299,23 +526,28 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct status->flag |= RX_FLAG_RADIOTAP_HE; } - spin_lock_bh(&ar->ab->base_lock); - peer = ath12k_peer_find_by_id(ar->ab, ppduinfo->peer_id); + ath12k_dp_extract_rx_desc_data(dp->hal, &rx_info, rx_desc, rx_desc); + + rcu_read_lock(); + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_rx_h_find_link_peer(dp_pdev, msdu, &rx_info); if (peer && peer->sta) { pubsta = peer->sta; + memcpy(addr, peer->addr, ETH_ALEN); if (pubsta->valid_links) { status->link_valid = 1; status->link_id = peer->link_id; } } - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); + rcu_read_unlock(); - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + ath12k_dbg(ab, ATH12K_DBG_DATA, "rx skb %p len %u peer %pM %u %s %s%s%s%s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n", msdu, msdu->len, - peer ? peer->addr : NULL, + addr, rxcb->tid, (is_mcbc) ? "mcast" : "ucast", (status->encoding == RX_ENC_LEGACY) ? "legacy" : "", @@ -2335,7 +567,7 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct !!(status->flag & RX_FLAG_MMIC_ERROR), !!(status->flag & RX_FLAG_AMSDU_MORE)); - ath12k_dbg_dump(ar->ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ", + ath12k_dbg_dump(ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ", msdu->data, msdu->len); rx_status = IEEE80211_SKB_RXCB(msdu); *rx_status = *status; @@ -2351,66 +583,11 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct !(is_mcbc && rx_status->flag & RX_FLAG_DECRYPTED)) rx_status->flag |= RX_FLAG_8023; - ieee80211_rx_napi(ath12k_ar_to_hw(ar), pubsta, msdu, napi); + ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), pubsta, msdu, napi); } +EXPORT_SYMBOL(ath12k_dp_mon_rx_deliver_msdu); -static int ath12k_dp_mon_rx_deliver(struct ath12k *ar, - struct dp_mon_mpdu *mon_mpdu, - struct hal_rx_mon_ppdu_info *ppduinfo, - struct napi_struct *napi) -{ - struct ath12k_pdev_dp *dp = &ar->dp; - struct sk_buff *mon_skb, *skb_next, *header; - struct ieee80211_rx_status *rxs = &dp->rx_status; - u8 decap = DP_RX_DECAP_TYPE_RAW; - - mon_skb = ath12k_dp_mon_rx_merg_msdus(ar, mon_mpdu, ppduinfo, rxs); - if (!mon_skb) - goto mon_deliver_fail; - - header = mon_skb; - rxs->flag = 0; - - if (mon_mpdu->err_bitmap & HAL_RX_MPDU_ERR_FCS) - rxs->flag = RX_FLAG_FAILED_FCS_CRC; - - do { - skb_next = mon_skb->next; - if (!skb_next) - rxs->flag &= ~RX_FLAG_AMSDU_MORE; - else - rxs->flag |= RX_FLAG_AMSDU_MORE; - - if (mon_skb == header) { - header = NULL; - rxs->flag &= ~RX_FLAG_ALLOW_SAME_PN; - } else { - rxs->flag |= RX_FLAG_ALLOW_SAME_PN; - } - rxs->flag |= RX_FLAG_ONLY_MONITOR; - - if (!(rxs->flag & RX_FLAG_ONLY_MONITOR)) - decap = mon_mpdu->decap_format; - - ath12k_dp_mon_update_radiotap(ar, ppduinfo, mon_skb, rxs); - ath12k_dp_mon_rx_deliver_msdu(ar, napi, mon_skb, ppduinfo, rxs, decap); - mon_skb = skb_next; - } while (mon_skb); - rxs->flag = 0; - - return 0; - -mon_deliver_fail: - mon_skb = mon_mpdu->head; - while (mon_skb) { - skb_next = mon_skb->next; - dev_kfree_skb_any(mon_skb); - mon_skb = skb_next; - } - return -EINVAL; -} - -static int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len) +int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len) { if (skb->len > len) { skb_trim(skb, len); @@ -2427,192 +604,54 @@ static int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len) return 0; } +EXPORT_SYMBOL(ath12k_dp_pkt_set_pktlen); -/* Hardware fill buffer with 128 bytes aligned. So need to reap it - * with 128 bytes aligned. - */ -#define RXDMA_DATA_DMA_BLOCK_SIZE 128 - -static void -ath12k_dp_mon_get_buf_len(struct hal_rx_msdu_desc_info *info, - bool *is_frag, u32 *total_len, - u32 *frag_len, u32 *msdu_cnt) -{ - if (info->msdu_flags & RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) { - *is_frag = true; - *frag_len = (RX_MON_STATUS_BASE_BUF_SIZE - - sizeof(struct hal_rx_desc)) & - ~(RXDMA_DATA_DMA_BLOCK_SIZE - 1); - *total_len += *frag_len; - } else { - if (*is_frag) - *frag_len = info->msdu_len - *total_len; - else - *frag_len = info->msdu_len; - - *msdu_cnt -= 1; - } -} - -static int -ath12k_dp_mon_parse_status_buf(struct ath12k *ar, +int +ath12k_dp_mon_parse_status_buf(struct ath12k_pdev_dp *dp_pdev, struct ath12k_mon_data *pmon, const struct dp_mon_packet_info *packet_info) { - struct ath12k_base *ab = ar->ab; - struct dp_rxdma_mon_ring *buf_ring = &ab->dp.rxdma_mon_buf_ring; + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; + struct dp_rxdma_mon_ring *buf_ring = &dp->rxdma_mon_buf_ring; struct sk_buff *msdu; - int buf_id; - u32 offset; - - buf_id = u32_get_bits(packet_info->cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); - - spin_lock_bh(&buf_ring->idr_lock); - msdu = idr_remove(&buf_ring->bufs_idr, buf_id); - spin_unlock_bh(&buf_ring->idr_lock); - - if (unlikely(!msdu)) { - ath12k_warn(ab, "mon dest desc with inval buf_id %d\n", buf_id); - return 0; - } - - dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(msdu)->paddr, - msdu->len + skb_tailroom(msdu), - DMA_FROM_DEVICE); - - offset = packet_info->dma_length + ATH12K_MON_RX_DOT11_OFFSET; - if (ath12k_dp_pkt_set_pktlen(msdu, offset)) { - dev_kfree_skb_any(msdu); - goto dest_replenish; - } - - if (!pmon->mon_mpdu->head) - pmon->mon_mpdu->head = msdu; - else - pmon->mon_mpdu->tail->next = msdu; - - pmon->mon_mpdu->tail = msdu; - -dest_replenish: - ath12k_dp_mon_buf_replenish(ab, buf_ring, 1); - - return 0; -} - -static int -ath12k_dp_mon_parse_rx_dest_tlv(struct ath12k *ar, - struct ath12k_mon_data *pmon, - enum hal_rx_mon_status hal_status, - const void *tlv_data) -{ - switch (hal_status) { - case HAL_RX_MON_STATUS_MPDU_START: - if (WARN_ON_ONCE(pmon->mon_mpdu)) - break; - - pmon->mon_mpdu = kzalloc(sizeof(*pmon->mon_mpdu), GFP_ATOMIC); - if (!pmon->mon_mpdu) - return -ENOMEM; - break; - case HAL_RX_MON_STATUS_BUF_ADDR: - return ath12k_dp_mon_parse_status_buf(ar, pmon, tlv_data); - case HAL_RX_MON_STATUS_MPDU_END: - /* If no MSDU then free empty MPDU */ - if (pmon->mon_mpdu->tail) { - pmon->mon_mpdu->tail->next = NULL; - list_add_tail(&pmon->mon_mpdu->list, &pmon->dp_rx_mon_mpdu_list); - } else { - kfree(pmon->mon_mpdu); - } - pmon->mon_mpdu = NULL; - break; - case HAL_RX_MON_STATUS_MSDU_END: - pmon->mon_mpdu->decap_format = pmon->decap_format; - pmon->mon_mpdu->err_bitmap = pmon->err_bitmap; - break; - default: - break; - } - - return 0; -} - -static enum hal_rx_mon_status -ath12k_dp_mon_parse_rx_dest(struct ath12k *ar, struct ath12k_mon_data *pmon, - struct sk_buff *skb) -{ - struct hal_tlv_64_hdr *tlv; - struct ath12k_skb_rxcb *rxcb; - enum hal_rx_mon_status hal_status; - u16 tlv_tag, tlv_len; - u8 *ptr = skb->data; - - do { - tlv = (struct hal_tlv_64_hdr *)ptr; - tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG); - - /* The actual length of PPDU_END is the combined length of many PHY - * TLVs that follow. Skip the TLV header and - * rx_rxpcu_classification_overview that follows the header to get to - * next TLV. - */ - - if (tlv_tag == HAL_RX_PPDU_END) - tlv_len = sizeof(struct hal_rx_rxpcu_classification_overview); - else - tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN); - - hal_status = ath12k_dp_mon_rx_parse_status_tlv(ar, pmon, tlv); - - if (ar->monitor_started && ar->ab->hw_params->rxdma1_enable && - ath12k_dp_mon_parse_rx_dest_tlv(ar, pmon, hal_status, tlv->value)) - return HAL_RX_MON_STATUS_PPDU_DONE; - - ptr += sizeof(*tlv) + tlv_len; - ptr = PTR_ALIGN(ptr, HAL_TLV_64_ALIGN); - - if ((ptr - skb->data) > skb->len) - break; + int buf_id; + u32 offset; - } while ((hal_status == HAL_RX_MON_STATUS_PPDU_NOT_DONE) || - (hal_status == HAL_RX_MON_STATUS_BUF_ADDR) || - (hal_status == HAL_RX_MON_STATUS_MPDU_START) || - (hal_status == HAL_RX_MON_STATUS_MPDU_END) || - (hal_status == HAL_RX_MON_STATUS_MSDU_END)); + buf_id = u32_get_bits(packet_info->cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); - rxcb = ATH12K_SKB_RXCB(skb); - if (rxcb->is_end_of_ppdu) - hal_status = HAL_RX_MON_STATUS_PPDU_DONE; + spin_lock_bh(&buf_ring->idr_lock); + msdu = idr_remove(&buf_ring->bufs_idr, buf_id); + spin_unlock_bh(&buf_ring->idr_lock); - return hal_status; -} + if (unlikely(!msdu)) { + ath12k_warn(ab, "mon dest desc with inval buf_id %d\n", buf_id); + return 0; + } -enum hal_rx_mon_status -ath12k_dp_mon_rx_parse_mon_status(struct ath12k *ar, - struct ath12k_mon_data *pmon, - struct sk_buff *skb, - struct napi_struct *napi) -{ - struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; - struct dp_mon_mpdu *tmp; - struct dp_mon_mpdu *mon_mpdu = pmon->mon_mpdu; - enum hal_rx_mon_status hal_status; + dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(msdu)->paddr, + msdu->len + skb_tailroom(msdu), + DMA_FROM_DEVICE); - hal_status = ath12k_dp_mon_parse_rx_dest(ar, pmon, skb); - if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE) - return hal_status; + offset = packet_info->dma_length + ATH12K_MON_RX_DOT11_OFFSET; + if (ath12k_dp_pkt_set_pktlen(msdu, offset)) { + dev_kfree_skb_any(msdu); + goto dest_replenish; + } - list_for_each_entry_safe(mon_mpdu, tmp, &pmon->dp_rx_mon_mpdu_list, list) { - list_del(&mon_mpdu->list); + if (!pmon->mon_mpdu->head) + pmon->mon_mpdu->head = msdu; + else + pmon->mon_mpdu->tail->next = msdu; - if (mon_mpdu->head && mon_mpdu->tail) - ath12k_dp_mon_rx_deliver(ar, mon_mpdu, ppdu_info, napi); + pmon->mon_mpdu->tail = msdu; - kfree(mon_mpdu); - } +dest_replenish: + ath12k_dp_mon_buf_replenish(ab, buf_ring, 1); - return hal_status; + return 0; } +EXPORT_SYMBOL(ath12k_dp_mon_parse_status_buf); int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab, struct dp_rxdma_mon_ring *buf_ring, @@ -2688,763 +727,94 @@ int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab, spin_unlock_bh(&srng->lock); return -ENOMEM; } +EXPORT_SYMBOL(ath12k_dp_mon_buf_replenish); int ath12k_dp_mon_status_bufs_replenish(struct ath12k_base *ab, struct dp_rxdma_mon_ring *rx_ring, int req_entries) { - enum hal_rx_buf_return_buf_manager mgr = - ab->hw_params->hal_params->rx_buf_rbm; - int num_free, num_remain, buf_id; - struct ath12k_buffer_addr *desc; - struct hal_srng *srng; - struct sk_buff *skb; - dma_addr_t paddr; - u32 cookie; - - req_entries = min(req_entries, rx_ring->bufs_max); - - srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id]; - - spin_lock_bh(&srng->lock); - - ath12k_hal_srng_access_begin(ab, srng); - - num_free = ath12k_hal_srng_src_num_free(ab, srng, true); - if (!req_entries && (num_free > (rx_ring->bufs_max * 3) / 4)) - req_entries = num_free; - - req_entries = min(num_free, req_entries); - num_remain = req_entries; - - while (num_remain > 0) { - skb = dev_alloc_skb(RX_MON_STATUS_BUF_SIZE); - if (!skb) - break; - - if (!IS_ALIGNED((unsigned long)skb->data, - RX_MON_STATUS_BUF_ALIGN)) { - skb_pull(skb, - PTR_ALIGN(skb->data, RX_MON_STATUS_BUF_ALIGN) - - skb->data); - } - - paddr = dma_map_single(ab->dev, skb->data, - skb->len + skb_tailroom(skb), - DMA_FROM_DEVICE); - if (dma_mapping_error(ab->dev, paddr)) - goto fail_free_skb; - - spin_lock_bh(&rx_ring->idr_lock); - buf_id = idr_alloc(&rx_ring->bufs_idr, skb, 0, - rx_ring->bufs_max * 3, GFP_ATOMIC); - spin_unlock_bh(&rx_ring->idr_lock); - if (buf_id < 0) - goto fail_dma_unmap; - cookie = u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID); - - desc = ath12k_hal_srng_src_get_next_entry(ab, srng); - if (!desc) - goto fail_buf_unassign; - - ATH12K_SKB_RXCB(skb)->paddr = paddr; - - num_remain--; - - ath12k_hal_rx_buf_addr_info_set(desc, paddr, cookie, mgr); - } - - ath12k_hal_srng_access_end(ab, srng); - - spin_unlock_bh(&srng->lock); - - return req_entries - num_remain; - -fail_buf_unassign: - spin_lock_bh(&rx_ring->idr_lock); - idr_remove(&rx_ring->bufs_idr, buf_id); - spin_unlock_bh(&rx_ring->idr_lock); -fail_dma_unmap: - dma_unmap_single(ab->dev, paddr, skb->len + skb_tailroom(skb), - DMA_FROM_DEVICE); -fail_free_skb: - dev_kfree_skb_any(skb); - - ath12k_hal_srng_access_end(ab, srng); - - spin_unlock_bh(&srng->lock); - - return req_entries - num_remain; -} - -static struct dp_mon_tx_ppdu_info * -ath12k_dp_mon_tx_get_ppdu_info(struct ath12k_mon_data *pmon, - unsigned int ppdu_id, - enum dp_mon_tx_ppdu_info_type type) -{ - struct dp_mon_tx_ppdu_info *tx_ppdu_info; - - if (type == DP_MON_TX_PROT_PPDU_INFO) { - tx_ppdu_info = pmon->tx_prot_ppdu_info; - - if (tx_ppdu_info && !tx_ppdu_info->is_used) - return tx_ppdu_info; - kfree(tx_ppdu_info); - } else { - tx_ppdu_info = pmon->tx_data_ppdu_info; - - if (tx_ppdu_info && !tx_ppdu_info->is_used) - return tx_ppdu_info; - kfree(tx_ppdu_info); - } - - /* allocate new tx_ppdu_info */ - tx_ppdu_info = kzalloc(sizeof(*tx_ppdu_info), GFP_ATOMIC); - if (!tx_ppdu_info) - return NULL; - - tx_ppdu_info->is_used = 0; - tx_ppdu_info->ppdu_id = ppdu_id; - - if (type == DP_MON_TX_PROT_PPDU_INFO) - pmon->tx_prot_ppdu_info = tx_ppdu_info; - else - pmon->tx_data_ppdu_info = tx_ppdu_info; - - return tx_ppdu_info; -} - -static struct dp_mon_tx_ppdu_info * -ath12k_dp_mon_hal_tx_ppdu_info(struct ath12k_mon_data *pmon, - u16 tlv_tag) -{ - switch (tlv_tag) { - case HAL_TX_FES_SETUP: - case HAL_TX_FLUSH: - case HAL_PCU_PPDU_SETUP_INIT: - case HAL_TX_PEER_ENTRY: - case HAL_TX_QUEUE_EXTENSION: - case HAL_TX_MPDU_START: - case HAL_TX_MSDU_START: - case HAL_TX_DATA: - case HAL_MON_BUF_ADDR: - case HAL_TX_MPDU_END: - case HAL_TX_LAST_MPDU_FETCHED: - case HAL_TX_LAST_MPDU_END: - case HAL_COEX_TX_REQ: - case HAL_TX_RAW_OR_NATIVE_FRAME_SETUP: - case HAL_SCH_CRITICAL_TLV_REFERENCE: - case HAL_TX_FES_SETUP_COMPLETE: - case HAL_TQM_MPDU_GLOBAL_START: - case HAL_SCHEDULER_END: - case HAL_TX_FES_STATUS_USER_PPDU: - break; - case HAL_TX_FES_STATUS_PROT: { - if (!pmon->tx_prot_ppdu_info->is_used) - pmon->tx_prot_ppdu_info->is_used = true; - - return pmon->tx_prot_ppdu_info; - } - } - - if (!pmon->tx_data_ppdu_info->is_used) - pmon->tx_data_ppdu_info->is_used = true; - - return pmon->tx_data_ppdu_info; -} - -#define MAX_MONITOR_HEADER 512 -#define MAX_DUMMY_FRM_BODY 128 - -struct sk_buff *ath12k_dp_mon_tx_alloc_skb(void) -{ - struct sk_buff *skb; - - skb = dev_alloc_skb(MAX_MONITOR_HEADER + MAX_DUMMY_FRM_BODY); - if (!skb) - return NULL; - - skb_reserve(skb, MAX_MONITOR_HEADER); - - if (!IS_ALIGNED((unsigned long)skb->data, 4)) - skb_pull(skb, PTR_ALIGN(skb->data, 4) - skb->data); - - return skb; -} - -static int -ath12k_dp_mon_tx_gen_cts2self_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) -{ - struct sk_buff *skb; - struct ieee80211_cts *cts; - - skb = ath12k_dp_mon_tx_alloc_skb(); - if (!skb) - return -ENOMEM; - - cts = (struct ieee80211_cts *)skb->data; - memset(cts, 0, MAX_DUMMY_FRM_BODY); - cts->frame_control = - cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTS); - cts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); - memcpy(cts->ra, tx_ppdu_info->rx_status.addr1, sizeof(cts->ra)); - - skb_put(skb, sizeof(*cts)); - tx_ppdu_info->tx_mon_mpdu->head = skb; - tx_ppdu_info->tx_mon_mpdu->tail = NULL; - list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, - &tx_ppdu_info->dp_tx_mon_mpdu_list); - - return 0; -} - -static int -ath12k_dp_mon_tx_gen_rts_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) -{ - struct sk_buff *skb; - struct ieee80211_rts *rts; - - skb = ath12k_dp_mon_tx_alloc_skb(); - if (!skb) - return -ENOMEM; - - rts = (struct ieee80211_rts *)skb->data; - memset(rts, 0, MAX_DUMMY_FRM_BODY); - rts->frame_control = - cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_RTS); - rts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); - memcpy(rts->ra, tx_ppdu_info->rx_status.addr1, sizeof(rts->ra)); - memcpy(rts->ta, tx_ppdu_info->rx_status.addr2, sizeof(rts->ta)); - - skb_put(skb, sizeof(*rts)); - tx_ppdu_info->tx_mon_mpdu->head = skb; - tx_ppdu_info->tx_mon_mpdu->tail = NULL; - list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, - &tx_ppdu_info->dp_tx_mon_mpdu_list); - - return 0; -} - -static int -ath12k_dp_mon_tx_gen_3addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) -{ - struct sk_buff *skb; - struct ieee80211_qos_hdr *qhdr; - - skb = ath12k_dp_mon_tx_alloc_skb(); - if (!skb) - return -ENOMEM; - - qhdr = (struct ieee80211_qos_hdr *)skb->data; - memset(qhdr, 0, MAX_DUMMY_FRM_BODY); - qhdr->frame_control = - cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC); - qhdr->duration_id = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); - memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN); - memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN); - memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN); - - skb_put(skb, sizeof(*qhdr)); - tx_ppdu_info->tx_mon_mpdu->head = skb; - tx_ppdu_info->tx_mon_mpdu->tail = NULL; - list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, - &tx_ppdu_info->dp_tx_mon_mpdu_list); - - return 0; -} - -static int -ath12k_dp_mon_tx_gen_4addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) -{ - struct sk_buff *skb; - struct dp_mon_qosframe_addr4 *qhdr; - - skb = ath12k_dp_mon_tx_alloc_skb(); - if (!skb) - return -ENOMEM; - - qhdr = (struct dp_mon_qosframe_addr4 *)skb->data; - memset(qhdr, 0, MAX_DUMMY_FRM_BODY); - qhdr->frame_control = - cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC); - qhdr->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); - memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN); - memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN); - memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN); - memcpy(qhdr->addr4, tx_ppdu_info->rx_status.addr4, ETH_ALEN); - - skb_put(skb, sizeof(*qhdr)); - tx_ppdu_info->tx_mon_mpdu->head = skb; - tx_ppdu_info->tx_mon_mpdu->tail = NULL; - list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, - &tx_ppdu_info->dp_tx_mon_mpdu_list); - - return 0; -} - -static int -ath12k_dp_mon_tx_gen_ack_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) -{ - struct sk_buff *skb; - struct dp_mon_frame_min_one *fbmhdr; - - skb = ath12k_dp_mon_tx_alloc_skb(); - if (!skb) - return -ENOMEM; - - fbmhdr = (struct dp_mon_frame_min_one *)skb->data; - memset(fbmhdr, 0, MAX_DUMMY_FRM_BODY); - fbmhdr->frame_control = - cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_CFACK); - memcpy(fbmhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN); - - /* set duration zero for ack frame */ - fbmhdr->duration = 0; - - skb_put(skb, sizeof(*fbmhdr)); - tx_ppdu_info->tx_mon_mpdu->head = skb; - tx_ppdu_info->tx_mon_mpdu->tail = NULL; - list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, - &tx_ppdu_info->dp_tx_mon_mpdu_list); - - return 0; -} - -static int -ath12k_dp_mon_tx_gen_prot_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) -{ - int ret = 0; - - switch (tx_ppdu_info->rx_status.medium_prot_type) { - case DP_MON_TX_MEDIUM_RTS_LEGACY: - case DP_MON_TX_MEDIUM_RTS_11AC_STATIC_BW: - case DP_MON_TX_MEDIUM_RTS_11AC_DYNAMIC_BW: - ret = ath12k_dp_mon_tx_gen_rts_frame(tx_ppdu_info); - break; - case DP_MON_TX_MEDIUM_CTS2SELF: - ret = ath12k_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info); - break; - case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_3ADDR: - ret = ath12k_dp_mon_tx_gen_3addr_qos_null_frame(tx_ppdu_info); - break; - case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_4ADDR: - ret = ath12k_dp_mon_tx_gen_4addr_qos_null_frame(tx_ppdu_info); - break; - } - - return ret; -} - -static enum dp_mon_tx_tlv_status -ath12k_dp_mon_tx_parse_status_tlv(struct ath12k_base *ab, - struct ath12k_mon_data *pmon, - u16 tlv_tag, const void *tlv_data, u32 userid) -{ - struct dp_mon_tx_ppdu_info *tx_ppdu_info; - enum dp_mon_tx_tlv_status status = DP_MON_TX_STATUS_PPDU_NOT_DONE; - u32 info[7]; - - tx_ppdu_info = ath12k_dp_mon_hal_tx_ppdu_info(pmon, tlv_tag); - - switch (tlv_tag) { - case HAL_TX_FES_SETUP: { - const struct hal_tx_fes_setup *tx_fes_setup = tlv_data; - - info[0] = __le32_to_cpu(tx_fes_setup->info0); - tx_ppdu_info->ppdu_id = __le32_to_cpu(tx_fes_setup->schedule_id); - tx_ppdu_info->num_users = - u32_get_bits(info[0], HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS); - status = DP_MON_TX_FES_SETUP; - break; - } - - case HAL_TX_FES_STATUS_END: { - const struct hal_tx_fes_status_end *tx_fes_status_end = tlv_data; - u32 tst_15_0, tst_31_16; - - info[0] = __le32_to_cpu(tx_fes_status_end->info0); - tst_15_0 = - u32_get_bits(info[0], - HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_15_0); - tst_31_16 = - u32_get_bits(info[0], - HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_31_16); - - tx_ppdu_info->rx_status.ppdu_ts = (tst_15_0 | (tst_31_16 << 16)); - status = DP_MON_TX_FES_STATUS_END; - break; - } - - case HAL_RX_RESPONSE_REQUIRED_INFO: { - const struct hal_rx_resp_req_info *rx_resp_req_info = tlv_data; - u32 addr_32; - u16 addr_16; - - info[0] = __le32_to_cpu(rx_resp_req_info->info0); - info[1] = __le32_to_cpu(rx_resp_req_info->info1); - info[2] = __le32_to_cpu(rx_resp_req_info->info2); - info[3] = __le32_to_cpu(rx_resp_req_info->info3); - info[4] = __le32_to_cpu(rx_resp_req_info->info4); - info[5] = __le32_to_cpu(rx_resp_req_info->info5); - - tx_ppdu_info->rx_status.ppdu_id = - u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_PPDU_ID); - tx_ppdu_info->rx_status.reception_type = - u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_RECEPTION_TYPE); - tx_ppdu_info->rx_status.rx_duration = - u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_DURATION); - tx_ppdu_info->rx_status.mcs = - u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_RATE_MCS); - tx_ppdu_info->rx_status.sgi = - u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_SGI); - tx_ppdu_info->rx_status.is_stbc = - u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_STBC); - tx_ppdu_info->rx_status.ldpc = - u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_LDPC); - tx_ppdu_info->rx_status.is_ampdu = - u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_IS_AMPDU); - tx_ppdu_info->rx_status.num_users = - u32_get_bits(info[2], HAL_RX_RESP_REQ_INFO2_NUM_USER); - - addr_32 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO3_ADDR1_31_0); - addr_16 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO4_ADDR1_47_32); - ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1); - - addr_16 = u32_get_bits(info[4], HAL_RX_RESP_REQ_INFO4_ADDR1_15_0); - addr_32 = u32_get_bits(info[5], HAL_RX_RESP_REQ_INFO5_ADDR1_47_16); - ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2); - - if (tx_ppdu_info->rx_status.reception_type == 0) - ath12k_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info); - status = DP_MON_RX_RESPONSE_REQUIRED_INFO; - break; - } - - case HAL_PCU_PPDU_SETUP_INIT: { - const struct hal_tx_pcu_ppdu_setup_init *ppdu_setup = tlv_data; - u32 addr_32; - u16 addr_16; - - info[0] = __le32_to_cpu(ppdu_setup->info0); - info[1] = __le32_to_cpu(ppdu_setup->info1); - info[2] = __le32_to_cpu(ppdu_setup->info2); - info[3] = __le32_to_cpu(ppdu_setup->info3); - info[4] = __le32_to_cpu(ppdu_setup->info4); - info[5] = __le32_to_cpu(ppdu_setup->info5); - info[6] = __le32_to_cpu(ppdu_setup->info6); - - /* protection frame address 1 */ - addr_32 = u32_get_bits(info[1], - HAL_TX_PPDU_SETUP_INFO1_PROT_FRAME_ADDR1_31_0); - addr_16 = u32_get_bits(info[2], - HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR1_47_32); - ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1); - - /* protection frame address 2 */ - addr_16 = u32_get_bits(info[2], - HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR2_15_0); - addr_32 = u32_get_bits(info[3], - HAL_TX_PPDU_SETUP_INFO3_PROT_FRAME_ADDR2_47_16); - ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2); - - /* protection frame address 3 */ - addr_32 = u32_get_bits(info[4], - HAL_TX_PPDU_SETUP_INFO4_PROT_FRAME_ADDR3_31_0); - addr_16 = u32_get_bits(info[5], - HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR3_47_32); - ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr3); - - /* protection frame address 4 */ - addr_16 = u32_get_bits(info[5], - HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR4_15_0); - addr_32 = u32_get_bits(info[6], - HAL_TX_PPDU_SETUP_INFO6_PROT_FRAME_ADDR4_47_16); - ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr4); - - status = u32_get_bits(info[0], - HAL_TX_PPDU_SETUP_INFO0_MEDIUM_PROT_TYPE); - break; - } - - case HAL_TX_QUEUE_EXTENSION: { - const struct hal_tx_queue_exten *tx_q_exten = tlv_data; - - info[0] = __le32_to_cpu(tx_q_exten->info0); - - tx_ppdu_info->rx_status.frame_control = - u32_get_bits(info[0], - HAL_TX_Q_EXT_INFO0_FRAME_CTRL); - tx_ppdu_info->rx_status.fc_valid = true; - break; - } - - case HAL_TX_FES_STATUS_START: { - const struct hal_tx_fes_status_start *tx_fes_start = tlv_data; - - info[0] = __le32_to_cpu(tx_fes_start->info0); - - tx_ppdu_info->rx_status.medium_prot_type = - u32_get_bits(info[0], - HAL_TX_FES_STATUS_START_INFO0_MEDIUM_PROT_TYPE); - break; - } - - case HAL_TX_FES_STATUS_PROT: { - const struct hal_tx_fes_status_prot *tx_fes_status = tlv_data; - u32 start_timestamp; - u32 end_timestamp; - - info[0] = __le32_to_cpu(tx_fes_status->info0); - info[1] = __le32_to_cpu(tx_fes_status->info1); - - start_timestamp = - u32_get_bits(info[0], - HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_15_0); - start_timestamp |= - u32_get_bits(info[0], - HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_31_16) << 15; - end_timestamp = - u32_get_bits(info[1], - HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_15_0); - end_timestamp |= - u32_get_bits(info[1], - HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_31_16) << 15; - tx_ppdu_info->rx_status.rx_duration = end_timestamp - start_timestamp; - - ath12k_dp_mon_tx_gen_prot_frame(tx_ppdu_info); - break; - } - - case HAL_TX_FES_STATUS_START_PPDU: - case HAL_TX_FES_STATUS_START_PROT: { - const struct hal_tx_fes_status_start_prot *tx_fes_stat_start = tlv_data; - u64 ppdu_ts; - - info[0] = __le32_to_cpu(tx_fes_stat_start->info0); - - tx_ppdu_info->rx_status.ppdu_ts = - u32_get_bits(info[0], - HAL_TX_FES_STAT_STRT_INFO0_PROT_TS_LOWER_32); - ppdu_ts = (u32_get_bits(info[1], - HAL_TX_FES_STAT_STRT_INFO1_PROT_TS_UPPER_32)); - tx_ppdu_info->rx_status.ppdu_ts |= ppdu_ts << 32; - break; - } - - case HAL_TX_FES_STATUS_USER_PPDU: { - const struct hal_tx_fes_status_user_ppdu *tx_fes_usr_ppdu = tlv_data; - - info[0] = __le32_to_cpu(tx_fes_usr_ppdu->info0); - - tx_ppdu_info->rx_status.rx_duration = - u32_get_bits(info[0], - HAL_TX_FES_STAT_USR_PPDU_INFO0_DURATION); - break; - } - - case HAL_MACTX_HE_SIG_A_SU: - ath12k_dp_mon_parse_he_sig_su(tlv_data, &tx_ppdu_info->rx_status); - break; - - case HAL_MACTX_HE_SIG_A_MU_DL: - ath12k_dp_mon_parse_he_sig_mu(tlv_data, &tx_ppdu_info->rx_status); - break; - - case HAL_MACTX_HE_SIG_B1_MU: - ath12k_dp_mon_parse_he_sig_b1_mu(tlv_data, &tx_ppdu_info->rx_status); - break; - - case HAL_MACTX_HE_SIG_B2_MU: - ath12k_dp_mon_parse_he_sig_b2_mu(tlv_data, &tx_ppdu_info->rx_status); - break; - - case HAL_MACTX_HE_SIG_B2_OFDMA: - ath12k_dp_mon_parse_he_sig_b2_ofdma(tlv_data, &tx_ppdu_info->rx_status); - break; - - case HAL_MACTX_VHT_SIG_A: - ath12k_dp_mon_parse_vht_sig_a(tlv_data, &tx_ppdu_info->rx_status); - break; - - case HAL_MACTX_L_SIG_A: - ath12k_dp_mon_parse_l_sig_a(tlv_data, &tx_ppdu_info->rx_status); - break; - - case HAL_MACTX_L_SIG_B: - ath12k_dp_mon_parse_l_sig_b(tlv_data, &tx_ppdu_info->rx_status); - break; + enum hal_rx_buf_return_buf_manager mgr = + ab->hal.hal_params->rx_buf_rbm; + int num_free, num_remain, buf_id; + struct ath12k_buffer_addr *desc; + struct hal_srng *srng; + struct sk_buff *skb; + dma_addr_t paddr; + u32 cookie; - case HAL_RX_FRAME_BITMAP_ACK: { - const struct hal_rx_frame_bitmap_ack *fbm_ack = tlv_data; - u32 addr_32; - u16 addr_16; + req_entries = min(req_entries, rx_ring->bufs_max); - info[0] = __le32_to_cpu(fbm_ack->info0); - info[1] = __le32_to_cpu(fbm_ack->info1); + srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id]; - addr_32 = u32_get_bits(info[0], - HAL_RX_FBM_ACK_INFO0_ADDR1_31_0); - addr_16 = u32_get_bits(info[1], - HAL_RX_FBM_ACK_INFO1_ADDR1_47_32); - ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1); + spin_lock_bh(&srng->lock); - ath12k_dp_mon_tx_gen_ack_frame(tx_ppdu_info); - break; - } + ath12k_hal_srng_access_begin(ab, srng); - case HAL_MACTX_PHY_DESC: { - const struct hal_tx_phy_desc *tx_phy_desc = tlv_data; - - info[0] = __le32_to_cpu(tx_phy_desc->info0); - info[1] = __le32_to_cpu(tx_phy_desc->info1); - info[2] = __le32_to_cpu(tx_phy_desc->info2); - info[3] = __le32_to_cpu(tx_phy_desc->info3); - - tx_ppdu_info->rx_status.beamformed = - u32_get_bits(info[0], - HAL_TX_PHY_DESC_INFO0_BF_TYPE); - tx_ppdu_info->rx_status.preamble_type = - u32_get_bits(info[0], - HAL_TX_PHY_DESC_INFO0_PREAMBLE_11B); - tx_ppdu_info->rx_status.mcs = - u32_get_bits(info[1], - HAL_TX_PHY_DESC_INFO1_MCS); - tx_ppdu_info->rx_status.ltf_size = - u32_get_bits(info[3], - HAL_TX_PHY_DESC_INFO3_LTF_SIZE); - tx_ppdu_info->rx_status.nss = - u32_get_bits(info[2], - HAL_TX_PHY_DESC_INFO2_NSS); - tx_ppdu_info->rx_status.chan_num = - u32_get_bits(info[3], - HAL_TX_PHY_DESC_INFO3_ACTIVE_CHANNEL); - tx_ppdu_info->rx_status.bw = - u32_get_bits(info[0], - HAL_TX_PHY_DESC_INFO0_BANDWIDTH); - break; - } + num_free = ath12k_hal_srng_src_num_free(ab, srng, true); + if (!req_entries && (num_free > (rx_ring->bufs_max * 3) / 4)) + req_entries = num_free; - case HAL_TX_MPDU_START: { - struct dp_mon_mpdu *mon_mpdu = tx_ppdu_info->tx_mon_mpdu; + req_entries = min(num_free, req_entries); + num_remain = req_entries; - mon_mpdu = kzalloc(sizeof(*mon_mpdu), GFP_ATOMIC); - if (!mon_mpdu) - return DP_MON_TX_STATUS_PPDU_NOT_DONE; - status = DP_MON_TX_MPDU_START; - break; - } + while (num_remain > 0) { + skb = dev_alloc_skb(RX_MON_STATUS_BUF_SIZE); + if (!skb) + break; - case HAL_TX_MPDU_END: - list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, - &tx_ppdu_info->dp_tx_mon_mpdu_list); - break; - } + if (!IS_ALIGNED((unsigned long)skb->data, + RX_MON_STATUS_BUF_ALIGN)) { + skb_pull(skb, + PTR_ALIGN(skb->data, RX_MON_STATUS_BUF_ALIGN) - + skb->data); + } - return status; -} + paddr = dma_map_single(ab->dev, skb->data, + skb->len + skb_tailroom(skb), + DMA_FROM_DEVICE); + if (dma_mapping_error(ab->dev, paddr)) + goto fail_free_skb; -enum dp_mon_tx_tlv_status -ath12k_dp_mon_tx_status_get_num_user(u16 tlv_tag, - struct hal_tlv_hdr *tx_tlv, - u8 *num_users) -{ - u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE; - u32 info0; + spin_lock_bh(&rx_ring->idr_lock); + buf_id = idr_alloc(&rx_ring->bufs_idr, skb, 0, + rx_ring->bufs_max * 3, GFP_ATOMIC); + spin_unlock_bh(&rx_ring->idr_lock); + if (buf_id < 0) + goto fail_dma_unmap; + cookie = u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID); - switch (tlv_tag) { - case HAL_TX_FES_SETUP: { - struct hal_tx_fes_setup *tx_fes_setup = - (struct hal_tx_fes_setup *)tx_tlv; + desc = ath12k_hal_srng_src_get_next_entry(ab, srng); + if (!desc) + goto fail_buf_unassign; - info0 = __le32_to_cpu(tx_fes_setup->info0); + ATH12K_SKB_RXCB(skb)->paddr = paddr; - *num_users = u32_get_bits(info0, HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS); - tlv_status = DP_MON_TX_FES_SETUP; - break; - } + num_remain--; - case HAL_RX_RESPONSE_REQUIRED_INFO: { - /* TODO: need to update *num_users */ - tlv_status = DP_MON_RX_RESPONSE_REQUIRED_INFO; - break; + ath12k_hal_rx_buf_addr_info_set(&ab->hal, desc, paddr, cookie, mgr); } - } - - return tlv_status; -} -static void -ath12k_dp_mon_tx_process_ppdu_info(struct ath12k *ar, - struct napi_struct *napi, - struct dp_mon_tx_ppdu_info *tx_ppdu_info) -{ - struct dp_mon_mpdu *tmp, *mon_mpdu; + ath12k_hal_srng_access_end(ab, srng); - list_for_each_entry_safe(mon_mpdu, tmp, - &tx_ppdu_info->dp_tx_mon_mpdu_list, list) { - list_del(&mon_mpdu->list); + spin_unlock_bh(&srng->lock); - if (mon_mpdu->head) - ath12k_dp_mon_rx_deliver(ar, mon_mpdu, - &tx_ppdu_info->rx_status, napi); + return req_entries - num_remain; - kfree(mon_mpdu); - } -} +fail_buf_unassign: + spin_lock_bh(&rx_ring->idr_lock); + idr_remove(&rx_ring->bufs_idr, buf_id); + spin_unlock_bh(&rx_ring->idr_lock); +fail_dma_unmap: + dma_unmap_single(ab->dev, paddr, skb->len + skb_tailroom(skb), + DMA_FROM_DEVICE); +fail_free_skb: + dev_kfree_skb_any(skb); -enum hal_rx_mon_status -ath12k_dp_mon_tx_parse_mon_status(struct ath12k *ar, - struct ath12k_mon_data *pmon, - struct sk_buff *skb, - struct napi_struct *napi, - u32 ppdu_id) -{ - struct ath12k_base *ab = ar->ab; - struct dp_mon_tx_ppdu_info *tx_prot_ppdu_info, *tx_data_ppdu_info; - struct hal_tlv_hdr *tlv; - u8 *ptr = skb->data; - u16 tlv_tag; - u16 tlv_len; - u32 tlv_userid = 0; - u8 num_user; - u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE; - - tx_prot_ppdu_info = ath12k_dp_mon_tx_get_ppdu_info(pmon, ppdu_id, - DP_MON_TX_PROT_PPDU_INFO); - if (!tx_prot_ppdu_info) - return -ENOMEM; - - tlv = (struct hal_tlv_hdr *)ptr; - tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG); - - tlv_status = ath12k_dp_mon_tx_status_get_num_user(tlv_tag, tlv, &num_user); - if (tlv_status == DP_MON_TX_STATUS_PPDU_NOT_DONE || !num_user) - return -EINVAL; - - tx_data_ppdu_info = ath12k_dp_mon_tx_get_ppdu_info(pmon, ppdu_id, - DP_MON_TX_DATA_PPDU_INFO); - if (!tx_data_ppdu_info) - return -ENOMEM; - - do { - tlv = (struct hal_tlv_hdr *)ptr; - tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG); - tlv_len = le32_get_bits(tlv->tl, HAL_TLV_HDR_LEN); - tlv_userid = le32_get_bits(tlv->tl, HAL_TLV_USR_ID); - - tlv_status = ath12k_dp_mon_tx_parse_status_tlv(ab, pmon, - tlv_tag, ptr, - tlv_userid); - ptr += tlv_len; - ptr = PTR_ALIGN(ptr, HAL_TLV_ALIGN); - if ((ptr - skb->data) >= DP_TX_MONITOR_BUF_SIZE) - break; - } while (tlv_status != DP_MON_TX_FES_STATUS_END); + ath12k_hal_srng_access_end(ab, srng); - ath12k_dp_mon_tx_process_ppdu_info(ar, napi, tx_data_ppdu_info); - ath12k_dp_mon_tx_process_ppdu_info(ar, napi, tx_prot_ppdu_info); + spin_unlock_bh(&srng->lock); - return tlv_status; + return req_entries - num_remain; } static void @@ -3460,6 +830,9 @@ ath12k_dp_mon_rx_update_peer_rate_table_stats(struct ath12k_rx_peer_stats *rx_st u32 gi_idx = ppdu_info->gi; u32 len; + if (!rx_stats) + return; + if (mcs_idx > HAL_RX_MAX_MCS_HT || nss_idx >= HAL_RX_MAX_NSS || bw_idx >= HAL_RX_BW_MAX || gi_idx >= HAL_RX_GI_MAX) { return; @@ -3480,15 +853,14 @@ ath12k_dp_mon_rx_update_peer_rate_table_stats(struct ath12k_rx_peer_stats *rx_st stats->rx_rate[bw_idx][gi_idx][nss_idx][mcs_idx] += len; } -static void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k *ar, - struct ath12k_link_sta *arsta, - struct hal_rx_mon_ppdu_info *ppdu_info) +void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k_dp_link_peer *peer, + struct hal_rx_mon_ppdu_info *ppdu_info) { - struct ath12k_rx_peer_stats *rx_stats = arsta->rx_stats; + struct ath12k_rx_peer_stats *rx_stats = peer->peer_stats.rx_stats; u32 num_msdu; - arsta->rssi_comb = ppdu_info->rssi_comb; - ewma_avg_rssi_add(&arsta->avg_rssi, ppdu_info->rssi_comb); + peer->rssi_comb = ppdu_info->rssi_comb; + ewma_avg_rssi_add(&peer->avg_rssi, ppdu_info->rssi_comb); if (!rx_stats) return; @@ -3536,7 +908,7 @@ static void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k *ar, rx_stats->dcm_count += ppdu_info->dcm; rx_stats->rx_duration += ppdu_info->rx_duration; - arsta->rx_duration = rx_stats->rx_duration; + peer->rx_duration = rx_stats->rx_duration; if (ppdu_info->nss > 0 && ppdu_info->nss <= HAL_RX_MAX_NSS) { rx_stats->pkt_stats.nss_count[ppdu_info->nss - 1] += num_msdu; @@ -3589,6 +961,7 @@ static void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k *ar, ath12k_dp_mon_rx_update_peer_rate_table_stats(rx_stats, ppdu_info, NULL, num_msdu); } +EXPORT_SYMBOL(ath12k_dp_mon_rx_update_peer_su_stats); void ath12k_dp_mon_rx_process_ulofdma(struct hal_rx_mon_ppdu_info *ppdu_info) { @@ -3637,39 +1010,33 @@ void ath12k_dp_mon_rx_process_ulofdma(struct hal_rx_mon_ppdu_info *ppdu_info) } ppdu_info->ldpc = 1; } +EXPORT_SYMBOL(ath12k_dp_mon_rx_process_ulofdma); static void -ath12k_dp_mon_rx_update_user_stats(struct ath12k *ar, +ath12k_dp_mon_rx_update_user_stats(struct ath12k_base *ab, struct hal_rx_mon_ppdu_info *ppdu_info, u32 uid) { - struct ath12k_link_sta *arsta; struct ath12k_rx_peer_stats *rx_stats = NULL; struct hal_rx_user_status *user_stats = &ppdu_info->userstats[uid]; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; u32 num_msdu; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); if (user_stats->ast_index == 0 || user_stats->ast_index == 0xFFFF) return; - peer = ath12k_peer_find_by_ast(ar->ab, user_stats->ast_index); + peer = ath12k_dp_link_peer_find_by_ast(dp, user_stats->ast_index); if (!peer) { - ath12k_warn(ar->ab, "peer ast idx %d can't be found\n", + ath12k_warn(ab, "peer ast idx %d can't be found\n", user_stats->ast_index); return; } - arsta = ath12k_peer_get_link_sta(ar->ab, peer); - if (!arsta) { - ath12k_warn(ar->ab, "link sta not found on peer %pM id %d\n", - peer->addr, peer->peer_id); - return; - } - - arsta->rssi_comb = ppdu_info->rssi_comb; - ewma_avg_rssi_add(&arsta->avg_rssi, ppdu_info->rssi_comb); - rx_stats = arsta->rx_stats; + peer->rssi_comb = ppdu_info->rssi_comb; + ewma_avg_rssi_add(&peer->avg_rssi, ppdu_info->rssi_comb); + rx_stats = peer->peer_stats.rx_stats; if (!rx_stats) return; @@ -3713,7 +1080,7 @@ ath12k_dp_mon_rx_update_user_stats(struct ath12k *ar, rx_stats->ru_alloc_cnt[user_stats->ul_ofdma_ru_size] += num_msdu; rx_stats->rx_duration += ppdu_info->rx_duration; - arsta->rx_duration = rx_stats->rx_duration; + peer->rx_duration = rx_stats->rx_duration; if (user_stats->nss > 0 && user_stats->nss <= HAL_RX_MAX_NSS) { rx_stats->pkt_stats.nss_count[user_stats->nss - 1] += num_msdu; @@ -3744,8 +1111,8 @@ ath12k_dp_mon_rx_update_user_stats(struct ath12k *ar, user_stats, num_msdu); } -static void -ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k *ar, +void +ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k_base *ab, struct hal_rx_mon_ppdu_info *ppdu_info) { u32 num_users, i; @@ -3755,679 +1122,6 @@ ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k *ar, num_users = HAL_MAX_UL_MU_USERS; for (i = 0; i < num_users; i++) - ath12k_dp_mon_rx_update_user_stats(ar, ppdu_info, i); -} - -static void -ath12k_dp_mon_rx_memset_ppdu_info(struct hal_rx_mon_ppdu_info *ppdu_info) -{ - memset(ppdu_info, 0, sizeof(*ppdu_info)); - ppdu_info->peer_id = HAL_INVALID_PEERID; -} - -int ath12k_dp_mon_srng_process(struct ath12k *ar, int *budget, - struct napi_struct *napi) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_pdev_dp *pdev_dp = &ar->dp; - struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&pdev_dp->mon_data; - struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; - struct ath12k_dp *dp = &ab->dp; - struct hal_mon_dest_desc *mon_dst_desc; - struct sk_buff *skb; - struct ath12k_skb_rxcb *rxcb; - struct dp_srng *mon_dst_ring; - struct hal_srng *srng; - struct dp_rxdma_mon_ring *buf_ring; - struct ath12k_link_sta *arsta; - struct ath12k_peer *peer; - struct sk_buff_head skb_list; - u64 cookie; - int num_buffs_reaped = 0, srng_id, buf_id; - u32 hal_status, end_offset, info0, end_reason; - u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, ar->pdev_idx); - - __skb_queue_head_init(&skb_list); - srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, pdev_idx); - mon_dst_ring = &pdev_dp->rxdma_mon_dst_ring[srng_id]; - buf_ring = &dp->rxdma_mon_buf_ring; - - srng = &ab->hal.srng_list[mon_dst_ring->ring_id]; - spin_lock_bh(&srng->lock); - ath12k_hal_srng_access_begin(ab, srng); - - while (likely(*budget)) { - mon_dst_desc = ath12k_hal_srng_dst_peek(ab, srng); - if (unlikely(!mon_dst_desc)) - break; - - /* In case of empty descriptor, the cookie in the ring descriptor - * is invalid. Therefore, this entry is skipped, and ring processing - * continues. - */ - info0 = le32_to_cpu(mon_dst_desc->info0); - if (u32_get_bits(info0, HAL_MON_DEST_INFO0_EMPTY_DESC)) - goto move_next; - - cookie = le32_to_cpu(mon_dst_desc->cookie); - buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); - - spin_lock_bh(&buf_ring->idr_lock); - skb = idr_remove(&buf_ring->bufs_idr, buf_id); - spin_unlock_bh(&buf_ring->idr_lock); - - if (unlikely(!skb)) { - ath12k_warn(ab, "monitor destination with invalid buf_id %d\n", - buf_id); - goto move_next; - } - - rxcb = ATH12K_SKB_RXCB(skb); - dma_unmap_single(ab->dev, rxcb->paddr, - skb->len + skb_tailroom(skb), - DMA_FROM_DEVICE); - - end_reason = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_REASON); - - /* HAL_MON_FLUSH_DETECTED implies that an rx flush received at the end of - * rx PPDU and HAL_MON_PPDU_TRUNCATED implies that the PPDU got - * truncated due to a system level error. In both the cases, buffer data - * can be discarded - */ - if ((end_reason == HAL_MON_FLUSH_DETECTED) || - (end_reason == HAL_MON_PPDU_TRUNCATED)) { - ath12k_dbg(ab, ATH12K_DBG_DATA, - "Monitor dest descriptor end reason %d", end_reason); - dev_kfree_skb_any(skb); - goto move_next; - } - - /* Calculate the budget when the ring descriptor with the - * HAL_MON_END_OF_PPDU to ensure that one PPDU worth of data is always - * reaped. This helps to efficiently utilize the NAPI budget. - */ - if (end_reason == HAL_MON_END_OF_PPDU) { - *budget -= 1; - rxcb->is_end_of_ppdu = true; - } - - end_offset = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_OFFSET); - if (likely(end_offset <= DP_RX_BUFFER_SIZE)) { - skb_put(skb, end_offset); - } else { - ath12k_warn(ab, - "invalid offset on mon stats destination %u\n", - end_offset); - skb_put(skb, DP_RX_BUFFER_SIZE); - } - - __skb_queue_tail(&skb_list, skb); - -move_next: - ath12k_dp_mon_buf_replenish(ab, buf_ring, 1); - ath12k_hal_srng_dst_get_next_entry(ab, srng); - num_buffs_reaped++; - } - - ath12k_hal_srng_access_end(ab, srng); - spin_unlock_bh(&srng->lock); - - if (!num_buffs_reaped) - return 0; - - /* In some cases, one PPDU worth of data can be spread across multiple NAPI - * schedules, To avoid losing existing parsed ppdu_info information, skip - * the memset of the ppdu_info structure and continue processing it. - */ - if (!ppdu_info->ppdu_continuation) - ath12k_dp_mon_rx_memset_ppdu_info(ppdu_info); - - while ((skb = __skb_dequeue(&skb_list))) { - hal_status = ath12k_dp_mon_rx_parse_mon_status(ar, pmon, skb, napi); - if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE) { - ppdu_info->ppdu_continuation = true; - dev_kfree_skb_any(skb); - continue; - } - - if (ppdu_info->peer_id == HAL_INVALID_PEERID) - goto free_skb; - - rcu_read_lock(); - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, ppdu_info->peer_id); - if (!peer || !peer->sta) { - ath12k_dbg(ab, ATH12K_DBG_DATA, - "failed to find the peer with monitor peer_id %d\n", - ppdu_info->peer_id); - goto next_skb; - } - - if (ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_SU) { - arsta = ath12k_peer_get_link_sta(ar->ab, peer); - if (!arsta) { - ath12k_warn(ar->ab, "link sta not found on peer %pM id %d\n", - peer->addr, peer->peer_id); - spin_unlock_bh(&ab->base_lock); - rcu_read_unlock(); - dev_kfree_skb_any(skb); - continue; - } - ath12k_dp_mon_rx_update_peer_su_stats(ar, arsta, - ppdu_info); - } else if ((ppdu_info->fc_valid) && - (ppdu_info->ast_index != HAL_AST_IDX_INVALID)) { - ath12k_dp_mon_rx_process_ulofdma(ppdu_info); - ath12k_dp_mon_rx_update_peer_mu_stats(ar, ppdu_info); - } - -next_skb: - spin_unlock_bh(&ab->base_lock); - rcu_read_unlock(); -free_skb: - dev_kfree_skb_any(skb); - ath12k_dp_mon_rx_memset_ppdu_info(ppdu_info); - } - - return num_buffs_reaped; -} - -static int ath12k_dp_rx_reap_mon_status_ring(struct ath12k_base *ab, int mac_id, - int *budget, struct sk_buff_head *skb_list) -{ - const struct ath12k_hw_hal_params *hal_params; - int buf_id, srng_id, num_buffs_reaped = 0; - enum dp_mon_status_buf_state reap_status; - struct dp_rxdma_mon_ring *rx_ring; - struct ath12k_mon_data *pmon; - struct ath12k_skb_rxcb *rxcb; - struct hal_tlv_64_hdr *tlv; - void *rx_mon_status_desc; - struct hal_srng *srng; - struct ath12k_dp *dp; - struct sk_buff *skb; - struct ath12k *ar; - dma_addr_t paddr; - u32 cookie; - u8 rbm; - - ar = ab->pdevs[ath12k_hw_mac_id_to_pdev_id(ab->hw_params, mac_id)].ar; - dp = &ab->dp; - pmon = &ar->dp.mon_data; - srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, mac_id); - rx_ring = &dp->rx_mon_status_refill_ring[srng_id]; - - srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id]; - - spin_lock_bh(&srng->lock); - - ath12k_hal_srng_access_begin(ab, srng); - - while (*budget) { - *budget -= 1; - rx_mon_status_desc = ath12k_hal_srng_src_peek(ab, srng); - if (!rx_mon_status_desc) { - pmon->buf_state = DP_MON_STATUS_REPLINISH; - break; - } - ath12k_hal_rx_buf_addr_info_get(rx_mon_status_desc, &paddr, - &cookie, &rbm); - if (paddr) { - buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); - - spin_lock_bh(&rx_ring->idr_lock); - skb = idr_find(&rx_ring->bufs_idr, buf_id); - spin_unlock_bh(&rx_ring->idr_lock); - - if (!skb) { - ath12k_warn(ab, "rx monitor status with invalid buf_id %d\n", - buf_id); - pmon->buf_state = DP_MON_STATUS_REPLINISH; - goto move_next; - } - - rxcb = ATH12K_SKB_RXCB(skb); - - dma_sync_single_for_cpu(ab->dev, rxcb->paddr, - skb->len + skb_tailroom(skb), - DMA_FROM_DEVICE); - - tlv = (struct hal_tlv_64_hdr *)skb->data; - if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) != - HAL_RX_STATUS_BUFFER_DONE) { - pmon->buf_state = DP_MON_STATUS_NO_DMA; - ath12k_warn(ab, - "mon status DONE not set %llx, buf_id %d\n", - le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG), - buf_id); - /* RxDMA status done bit might not be set even - * though tp is moved by HW. - */ - - /* If done status is missing: - * 1. As per MAC team's suggestion, - * when HP + 1 entry is peeked and if DMA - * is not done and if HP + 2 entry's DMA done - * is set. skip HP + 1 entry and - * start processing in next interrupt. - * 2. If HP + 2 entry's DMA done is not set, - * poll onto HP + 1 entry DMA done to be set. - * Check status for same buffer for next time - * dp_rx_mon_status_srng_process - */ - reap_status = ath12k_dp_rx_mon_buf_done(ab, srng, - rx_ring); - if (reap_status == DP_MON_STATUS_NO_DMA) - continue; - - spin_lock_bh(&rx_ring->idr_lock); - idr_remove(&rx_ring->bufs_idr, buf_id); - spin_unlock_bh(&rx_ring->idr_lock); - - dma_unmap_single(ab->dev, rxcb->paddr, - skb->len + skb_tailroom(skb), - DMA_FROM_DEVICE); - - dev_kfree_skb_any(skb); - pmon->buf_state = DP_MON_STATUS_REPLINISH; - goto move_next; - } - - spin_lock_bh(&rx_ring->idr_lock); - idr_remove(&rx_ring->bufs_idr, buf_id); - spin_unlock_bh(&rx_ring->idr_lock); - - dma_unmap_single(ab->dev, rxcb->paddr, - skb->len + skb_tailroom(skb), - DMA_FROM_DEVICE); - - if (ath12k_dp_pkt_set_pktlen(skb, RX_MON_STATUS_BUF_SIZE)) { - dev_kfree_skb_any(skb); - goto move_next; - } - __skb_queue_tail(skb_list, skb); - } else { - pmon->buf_state = DP_MON_STATUS_REPLINISH; - } -move_next: - skb = ath12k_dp_rx_alloc_mon_status_buf(ab, rx_ring, - &buf_id); - - if (!skb) { - ath12k_warn(ab, "failed to alloc buffer for status ring\n"); - hal_params = ab->hw_params->hal_params; - ath12k_hal_rx_buf_addr_info_set(rx_mon_status_desc, 0, 0, - hal_params->rx_buf_rbm); - num_buffs_reaped++; - break; - } - rxcb = ATH12K_SKB_RXCB(skb); - - cookie = u32_encode_bits(mac_id, DP_RXDMA_BUF_COOKIE_PDEV_ID) | - u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID); - - ath12k_hal_rx_buf_addr_info_set(rx_mon_status_desc, rxcb->paddr, - cookie, - ab->hw_params->hal_params->rx_buf_rbm); - ath12k_hal_srng_src_get_next_entry(ab, srng); - num_buffs_reaped++; - } - ath12k_hal_srng_access_end(ab, srng); - spin_unlock_bh(&srng->lock); - - return num_buffs_reaped; -} - -static u32 -ath12k_dp_rx_mon_mpdu_pop(struct ath12k *ar, int mac_id, - void *ring_entry, struct sk_buff **head_msdu, - struct sk_buff **tail_msdu, - struct list_head *used_list, - u32 *npackets, u32 *ppdu_id) -{ - struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data; - struct ath12k_buffer_addr *p_buf_addr_info, *p_last_buf_addr_info; - u32 msdu_ppdu_id = 0, msdu_cnt = 0, total_len = 0, frag_len = 0; - u32 rx_buf_size, rx_pkt_offset, sw_cookie; - bool is_frag, is_first_msdu, drop_mpdu = false; - struct hal_reo_entrance_ring *ent_desc = - (struct hal_reo_entrance_ring *)ring_entry; - u32 rx_bufs_used = 0, i = 0, desc_bank = 0; - struct hal_rx_desc *rx_desc, *tail_rx_desc; - struct hal_rx_msdu_link *msdu_link_desc; - struct sk_buff *msdu = NULL, *last = NULL; - struct ath12k_rx_desc_info *desc_info; - struct ath12k_buffer_addr buf_info; - struct hal_rx_msdu_list msdu_list; - struct ath12k_skb_rxcb *rxcb; - u16 num_msdus = 0; - dma_addr_t paddr; - u8 rbm; - - ath12k_hal_rx_reo_ent_buf_paddr_get(ring_entry, &paddr, - &sw_cookie, - &p_last_buf_addr_info, &rbm, - &msdu_cnt); - - spin_lock_bh(&pmon->mon_lock); - - if (le32_get_bits(ent_desc->info1, - HAL_REO_ENTR_RING_INFO1_RXDMA_PUSH_REASON) == - HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) { - u8 rxdma_err = le32_get_bits(ent_desc->info1, - HAL_REO_ENTR_RING_INFO1_RXDMA_ERROR_CODE); - if (rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR || - rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR || - rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR) { - drop_mpdu = true; - pmon->rx_mon_stats.dest_mpdu_drop++; - } - } - - is_frag = false; - is_first_msdu = true; - rx_pkt_offset = sizeof(struct hal_rx_desc); - - do { - if (pmon->mon_last_linkdesc_paddr == paddr) { - pmon->rx_mon_stats.dup_mon_linkdesc_cnt++; - spin_unlock_bh(&pmon->mon_lock); - return rx_bufs_used; - } - - desc_bank = u32_get_bits(sw_cookie, DP_LINK_DESC_BANK_MASK); - msdu_link_desc = - ar->ab->dp.link_desc_banks[desc_bank].vaddr + - (paddr - ar->ab->dp.link_desc_banks[desc_bank].paddr); - - ath12k_hal_rx_msdu_list_get(ar, msdu_link_desc, &msdu_list, - &num_msdus); - desc_info = ath12k_dp_get_rx_desc(ar->ab, - msdu_list.sw_cookie[num_msdus - 1]); - tail_rx_desc = (struct hal_rx_desc *)(desc_info->skb)->data; - - for (i = 0; i < num_msdus; i++) { - u32 l2_hdr_offset; - - if (pmon->mon_last_buf_cookie == msdu_list.sw_cookie[i]) { - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, - "i %d last_cookie %d is same\n", - i, pmon->mon_last_buf_cookie); - drop_mpdu = true; - pmon->rx_mon_stats.dup_mon_buf_cnt++; - continue; - } - - desc_info = - ath12k_dp_get_rx_desc(ar->ab, msdu_list.sw_cookie[i]); - msdu = desc_info->skb; - - if (!msdu) { - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, - "msdu_pop: invalid msdu (%d/%d)\n", - i + 1, num_msdus); - goto next_msdu; - } - rxcb = ATH12K_SKB_RXCB(msdu); - if (rxcb->paddr != msdu_list.paddr[i]) { - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, - "i %d paddr %lx != %lx\n", - i, (unsigned long)rxcb->paddr, - (unsigned long)msdu_list.paddr[i]); - drop_mpdu = true; - continue; - } - if (!rxcb->unmapped) { - dma_unmap_single(ar->ab->dev, rxcb->paddr, - msdu->len + - skb_tailroom(msdu), - DMA_FROM_DEVICE); - rxcb->unmapped = 1; - } - if (drop_mpdu) { - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, - "i %d drop msdu %p *ppdu_id %x\n", - i, msdu, *ppdu_id); - dev_kfree_skb_any(msdu); - msdu = NULL; - goto next_msdu; - } - - rx_desc = (struct hal_rx_desc *)msdu->data; - l2_hdr_offset = ath12k_dp_rx_h_l3pad(ar->ab, tail_rx_desc); - if (is_first_msdu) { - if (!ath12k_dp_rxdesc_mpdu_valid(ar->ab, rx_desc)) { - drop_mpdu = true; - dev_kfree_skb_any(msdu); - msdu = NULL; - pmon->mon_last_linkdesc_paddr = paddr; - goto next_msdu; - } - msdu_ppdu_id = - ath12k_dp_rxdesc_get_ppduid(ar->ab, rx_desc); - - if (ath12k_dp_mon_comp_ppduid(msdu_ppdu_id, - ppdu_id)) { - spin_unlock_bh(&pmon->mon_lock); - return rx_bufs_used; - } - pmon->mon_last_linkdesc_paddr = paddr; - is_first_msdu = false; - } - ath12k_dp_mon_get_buf_len(&msdu_list.msdu_info[i], - &is_frag, &total_len, - &frag_len, &msdu_cnt); - rx_buf_size = rx_pkt_offset + l2_hdr_offset + frag_len; - - if (ath12k_dp_pkt_set_pktlen(msdu, rx_buf_size)) { - dev_kfree_skb_any(msdu); - goto next_msdu; - } - - if (!(*head_msdu)) - *head_msdu = msdu; - else if (last) - last->next = msdu; - - last = msdu; -next_msdu: - pmon->mon_last_buf_cookie = msdu_list.sw_cookie[i]; - rx_bufs_used++; - desc_info->skb = NULL; - list_add_tail(&desc_info->list, used_list); - } - - ath12k_hal_rx_buf_addr_info_set(&buf_info, paddr, sw_cookie, rbm); - - ath12k_dp_mon_next_link_desc_get(msdu_link_desc, &paddr, - &sw_cookie, &rbm, - &p_buf_addr_info); - - ath12k_dp_rx_link_desc_return(ar->ab, &buf_info, - HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); - - p_last_buf_addr_info = p_buf_addr_info; - - } while (paddr && msdu_cnt); - - spin_unlock_bh(&pmon->mon_lock); - - if (last) - last->next = NULL; - - *tail_msdu = msdu; - - if (msdu_cnt == 0) - *npackets = 1; - - return rx_bufs_used; -} - -/* The destination ring processing is stuck if the destination is not - * moving while status ring moves 16 PPDU. The destination ring processing - * skips this destination ring PPDU as a workaround. - */ -#define MON_DEST_RING_STUCK_MAX_CNT 16 - -static void ath12k_dp_rx_mon_dest_process(struct ath12k *ar, int mac_id, - u32 quota, struct napi_struct *napi) -{ - struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data; - struct ath12k_pdev_mon_stats *rx_mon_stats; - u32 ppdu_id, rx_bufs_used = 0, ring_id; - u32 mpdu_rx_bufs_used, npackets = 0; - struct ath12k_dp *dp = &ar->ab->dp; - struct ath12k_base *ab = ar->ab; - void *ring_entry, *mon_dst_srng; - struct dp_mon_mpdu *tmp_mpdu; - LIST_HEAD(rx_desc_used_list); - struct hal_srng *srng; - - ring_id = dp->rxdma_err_dst_ring[mac_id].ring_id; - srng = &ab->hal.srng_list[ring_id]; - - mon_dst_srng = &ab->hal.srng_list[ring_id]; - - spin_lock_bh(&srng->lock); - - ath12k_hal_srng_access_begin(ab, mon_dst_srng); - - ppdu_id = pmon->mon_ppdu_info.ppdu_id; - rx_mon_stats = &pmon->rx_mon_stats; - - while ((ring_entry = ath12k_hal_srng_dst_peek(ar->ab, mon_dst_srng))) { - struct sk_buff *head_msdu, *tail_msdu; - - head_msdu = NULL; - tail_msdu = NULL; - - mpdu_rx_bufs_used = ath12k_dp_rx_mon_mpdu_pop(ar, mac_id, ring_entry, - &head_msdu, &tail_msdu, - &rx_desc_used_list, - &npackets, &ppdu_id); - - rx_bufs_used += mpdu_rx_bufs_used; - - if (mpdu_rx_bufs_used) { - dp->mon_dest_ring_stuck_cnt = 0; - } else { - dp->mon_dest_ring_stuck_cnt++; - rx_mon_stats->dest_mon_not_reaped++; - } - - if (dp->mon_dest_ring_stuck_cnt > MON_DEST_RING_STUCK_MAX_CNT) { - rx_mon_stats->dest_mon_stuck++; - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, - "status ring ppdu_id=%d dest ring ppdu_id=%d mon_dest_ring_stuck_cnt=%d dest_mon_not_reaped=%u dest_mon_stuck=%u\n", - pmon->mon_ppdu_info.ppdu_id, ppdu_id, - dp->mon_dest_ring_stuck_cnt, - rx_mon_stats->dest_mon_not_reaped, - rx_mon_stats->dest_mon_stuck); - spin_lock_bh(&pmon->mon_lock); - pmon->mon_ppdu_info.ppdu_id = ppdu_id; - spin_unlock_bh(&pmon->mon_lock); - continue; - } - - if (ppdu_id != pmon->mon_ppdu_info.ppdu_id) { - spin_lock_bh(&pmon->mon_lock); - pmon->mon_ppdu_status = DP_PPDU_STATUS_START; - spin_unlock_bh(&pmon->mon_lock); - ath12k_dbg(ar->ab, ATH12K_DBG_DATA, - "dest_rx: new ppdu_id %x != status ppdu_id %x dest_mon_not_reaped = %u dest_mon_stuck = %u\n", - ppdu_id, pmon->mon_ppdu_info.ppdu_id, - rx_mon_stats->dest_mon_not_reaped, - rx_mon_stats->dest_mon_stuck); - break; - } - - if (head_msdu && tail_msdu) { - tmp_mpdu = kzalloc(sizeof(*tmp_mpdu), GFP_ATOMIC); - if (!tmp_mpdu) - break; - - tmp_mpdu->head = head_msdu; - tmp_mpdu->tail = tail_msdu; - tmp_mpdu->err_bitmap = pmon->err_bitmap; - tmp_mpdu->decap_format = pmon->decap_format; - ath12k_dp_mon_rx_deliver(ar, tmp_mpdu, - &pmon->mon_ppdu_info, napi); - rx_mon_stats->dest_mpdu_done++; - kfree(tmp_mpdu); - } - - ring_entry = ath12k_hal_srng_dst_get_next_entry(ar->ab, - mon_dst_srng); - } - ath12k_hal_srng_access_end(ar->ab, mon_dst_srng); - - spin_unlock_bh(&srng->lock); - - if (rx_bufs_used) { - rx_mon_stats->dest_ppdu_done++; - ath12k_dp_rx_bufs_replenish(ar->ab, - &dp->rx_refill_buf_ring, - &rx_desc_used_list, - rx_bufs_used); - } -} - -static int -__ath12k_dp_mon_process_ring(struct ath12k *ar, int mac_id, - struct napi_struct *napi, int *budget) -{ - struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data; - struct ath12k_pdev_mon_stats *rx_mon_stats = &pmon->rx_mon_stats; - struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; - enum hal_rx_mon_status hal_status; - struct sk_buff_head skb_list; - int num_buffs_reaped; - struct sk_buff *skb; - - __skb_queue_head_init(&skb_list); - - num_buffs_reaped = ath12k_dp_rx_reap_mon_status_ring(ar->ab, mac_id, - budget, &skb_list); - if (!num_buffs_reaped) - goto exit; - - while ((skb = __skb_dequeue(&skb_list))) { - memset(ppdu_info, 0, sizeof(*ppdu_info)); - ppdu_info->peer_id = HAL_INVALID_PEERID; - - hal_status = ath12k_dp_mon_parse_rx_dest(ar, pmon, skb); - - if (ar->monitor_started && - pmon->mon_ppdu_status == DP_PPDU_STATUS_START && - hal_status == HAL_TLV_STATUS_PPDU_DONE) { - rx_mon_stats->status_ppdu_done++; - pmon->mon_ppdu_status = DP_PPDU_STATUS_DONE; - ath12k_dp_rx_mon_dest_process(ar, mac_id, *budget, napi); - pmon->mon_ppdu_status = DP_PPDU_STATUS_START; - } - - dev_kfree_skb_any(skb); - } - -exit: - return num_buffs_reaped; -} - -int ath12k_dp_mon_process_ring(struct ath12k_base *ab, int mac_id, - struct napi_struct *napi, int budget, - enum dp_monitor_mode monitor_mode) -{ - struct ath12k *ar = ath12k_ab_to_ar(ab, mac_id); - int num_buffs_reaped = 0; - - if (ab->hw_params->rxdma1_enable) { - if (monitor_mode == ATH12K_DP_RX_MONITOR_MODE) - num_buffs_reaped = ath12k_dp_mon_srng_process(ar, &budget, napi); - } else { - if (ar->monitor_started) - num_buffs_reaped = - __ath12k_dp_mon_process_ring(ar, mac_id, napi, &budget); - } - - return num_buffs_reaped; + ath12k_dp_mon_rx_update_user_stats(ab, ppdu_info, i); } +EXPORT_SYMBOL(ath12k_dp_mon_rx_update_peer_mu_stats); diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.h b/drivers/net/wireless/ath/ath12k/dp_mon.h index e25595cbdcf37..167028d27513b 100644 --- a/drivers/net/wireless/ath/ath12k/dp_mon.h +++ b/drivers/net/wireless/ath/ath12k/dp_mon.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_DP_MON_H @@ -12,6 +12,12 @@ #define ATH12K_MON_RX_DOT11_OFFSET 5 #define ATH12K_MON_RX_PKT_OFFSET 8 +#define ATH12K_LE32_DEC_ENC(value, dec_bits, enc_bits) \ + u32_encode_bits(le32_get_bits(value, dec_bits), enc_bits) + +#define ATH12K_LE64_DEC_ENC(value, dec_bits, enc_bits) \ + u32_encode_bits(le64_get_bits(value, dec_bits), enc_bits) + enum dp_monitor_mode { ATH12K_DP_TX_MONITOR_MODE, ATH12K_DP_RX_MONITOR_MODE @@ -77,31 +83,41 @@ struct dp_mon_tx_ppdu_info { struct dp_mon_mpdu *tx_mon_mpdu; }; -enum hal_rx_mon_status -ath12k_dp_mon_rx_parse_mon_status(struct ath12k *ar, - struct ath12k_mon_data *pmon, - struct sk_buff *skb, - struct napi_struct *napi); int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab, struct dp_rxdma_mon_ring *buf_ring, int req_entries); int ath12k_dp_mon_status_bufs_replenish(struct ath12k_base *ab, struct dp_rxdma_mon_ring *rx_ring, int req_entries); -int ath12k_dp_mon_process_ring(struct ath12k_base *ab, int mac_id, - struct napi_struct *napi, int budget, - enum dp_monitor_mode monitor_mode); -struct sk_buff *ath12k_dp_mon_tx_alloc_skb(void); -enum dp_mon_tx_tlv_status -ath12k_dp_mon_tx_status_get_num_user(u16 tlv_tag, - struct hal_tlv_hdr *tx_tlv, - u8 *num_users); -enum hal_rx_mon_status -ath12k_dp_mon_tx_parse_mon_status(struct ath12k *ar, - struct ath12k_mon_data *pmon, - struct sk_buff *skb, - struct napi_struct *napi, - u32 ppdu_id); void ath12k_dp_mon_rx_process_ulofdma(struct hal_rx_mon_ppdu_info *ppdu_info); -int ath12k_dp_mon_srng_process(struct ath12k *ar, int *budget, struct napi_struct *napi); +void +ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k_base *ab, + struct hal_rx_mon_ppdu_info *ppdu_info); +void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k_dp_link_peer *peer, + struct hal_rx_mon_ppdu_info *ppdu_info); +int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len); +struct sk_buff +*ath12k_dp_rx_alloc_mon_status_buf(struct ath12k_base *ab, + struct dp_rxdma_mon_ring *rx_ring, + int *buf_id); +u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id); +int +ath12k_dp_mon_parse_status_buf(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_mon_data *pmon, + const struct dp_mon_packet_info *packet_info); +void ath12k_dp_mon_update_radiotap(struct ath12k_pdev_dp *dp_pdev, + struct hal_rx_mon_ppdu_info *ppduinfo, + struct sk_buff *mon_skb, + struct ieee80211_rx_status *rxs); +void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, + struct napi_struct *napi, + struct sk_buff *msdu, + const struct hal_rx_mon_ppdu_info *ppduinfo, + struct ieee80211_rx_status *status, + u8 decap); +struct sk_buff * +ath12k_dp_mon_rx_merg_msdus(struct ath12k_pdev_dp *dp_pdev, + struct dp_mon_mpdu *mon_mpdu, + struct hal_rx_mon_ppdu_info *ppdu_info, + struct ieee80211_rx_status *rxs); #endif diff --git a/drivers/net/wireless/ath/ath12k/dp_peer.c b/drivers/net/wireless/ath/ath12k/dp_peer.c new file mode 100644 index 0000000000000..caa56942f0334 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/dp_peer.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "core.h" +#include "dp_peer.h" +#include "debug.h" +#include "debugfs.h" + +void ath12k_dp_link_peer_free(struct ath12k_dp_link_peer *peer) +{ + list_del(&peer->list); + + kfree(peer->peer_stats.rx_stats); + kfree(peer); +} + +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_vdev_and_addr(struct ath12k_dp *dp, + int vdev_id, const u8 *addr) +{ + struct ath12k_dp_link_peer *peer; + + lockdep_assert_held(&dp->dp_lock); + + list_for_each_entry(peer, &dp->peers, list) { + if (peer->vdev_id != vdev_id) + continue; + if (!ether_addr_equal(peer->addr, addr)) + continue; + + return peer; + } + + return NULL; +} + +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_pdev_and_addr(struct ath12k_dp *dp, u8 pdev_idx, + const u8 *addr) +{ + struct ath12k_dp_link_peer *peer; + + lockdep_assert_held(&dp->dp_lock); + + list_for_each_entry(peer, &dp->peers, list) { + if (peer->pdev_idx != pdev_idx) + continue; + if (!ether_addr_equal(peer->addr, addr)) + continue; + + return peer; + } + + return NULL; +} + +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_addr(struct ath12k_dp *dp, const u8 *addr) +{ + lockdep_assert_held(&dp->dp_lock); + + return rhashtable_lookup_fast(dp->rhead_peer_addr, addr, + dp->rhash_peer_addr_param); +} +EXPORT_SYMBOL(ath12k_dp_link_peer_find_by_addr); + +static struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_ml_id(struct ath12k_dp *dp, int ml_peer_id) +{ + struct ath12k_dp_link_peer *peer; + + lockdep_assert_held(&dp->dp_lock); + + list_for_each_entry(peer, &dp->peers, list) + if (ml_peer_id == peer->ml_id) + return peer; + + return NULL; +} + +static struct ath12k_dp_link_peer * +ath12k_dp_link_peer_search_by_id(struct ath12k_dp *dp, int peer_id) +{ + struct ath12k_dp_link_peer *peer; + + lockdep_assert_held(&dp->dp_lock); + + if (peer_id == HAL_INVALID_PEERID) + return NULL; + + if (peer_id & ATH12K_PEER_ML_ID_VALID) + return ath12k_dp_link_peer_find_by_ml_id(dp, peer_id); + + list_for_each_entry(peer, &dp->peers, list) + if (peer_id == peer->peer_id) + return peer; + + return NULL; +} + +bool ath12k_dp_link_peer_exist_by_vdev_id(struct ath12k_dp *dp, int vdev_id) +{ + struct ath12k_dp_link_peer *peer; + + spin_lock_bh(&dp->dp_lock); + + list_for_each_entry(peer, &dp->peers, list) { + if (vdev_id == peer->vdev_id) { + spin_unlock_bh(&dp->dp_lock); + return true; + } + } + spin_unlock_bh(&dp->dp_lock); + return false; +} + +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_ast(struct ath12k_dp *dp, int ast_hash) +{ + struct ath12k_dp_link_peer *peer; + + lockdep_assert_held(&dp->dp_lock); + + list_for_each_entry(peer, &dp->peers, list) + if (ast_hash == peer->ast_hash) + return peer; + + return NULL; +} + +void ath12k_dp_link_peer_unmap_event(struct ath12k_base *ab, u16 peer_id) +{ + struct ath12k_dp_link_peer *peer; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + + spin_lock_bh(&dp->dp_lock); + + peer = ath12k_dp_link_peer_search_by_id(dp, peer_id); + if (!peer) { + ath12k_warn(ab, "peer-unmap-event: unknown peer id %d\n", + peer_id); + goto exit; + } + + ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer unmap vdev %d peer %pM id %d\n", + peer->vdev_id, peer->addr, peer_id); + + ath12k_dp_link_peer_free(peer); + wake_up(&ab->peer_mapping_wq); + +exit: + spin_unlock_bh(&dp->dp_lock); +} + +void ath12k_dp_link_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id, + u8 *mac_addr, u16 ast_hash, u16 hw_peer_id) +{ + struct ath12k_dp_link_peer *peer; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k *ar; + + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, mac_addr); + if (!peer) { + peer = kzalloc(sizeof(*peer), GFP_ATOMIC); + if (!peer) + goto exit; + + peer->vdev_id = vdev_id; + peer->peer_id = peer_id; + peer->ast_hash = ast_hash; + peer->hw_peer_id = hw_peer_id; + ether_addr_copy(peer->addr, mac_addr); + + rcu_read_lock(); + ar = ath12k_mac_get_ar_by_vdev_id(ab, vdev_id); + if (ar && ath12k_debugfs_is_extd_rx_stats_enabled(ar) && + !peer->peer_stats.rx_stats) { + peer->peer_stats.rx_stats = + kzalloc(sizeof(*peer->peer_stats.rx_stats), GFP_ATOMIC); + } + rcu_read_unlock(); + + list_add(&peer->list, &dp->peers); + wake_up(&ab->peer_mapping_wq); + ewma_avg_rssi_init(&peer->avg_rssi); + } + ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer map vdev %d peer %pM id %d\n", + vdev_id, mac_addr, peer_id); + +exit: + spin_unlock_bh(&dp->dp_lock); +} + +struct ath12k_link_sta *ath12k_dp_link_peer_to_link_sta(struct ath12k_base *ab, + struct ath12k_dp_link_peer *peer) +{ + struct ath12k_sta *ahsta; + struct ath12k_link_sta *arsta; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "ath12k_dp_link_peer to ath12k_link_sta called without rcu lock"); + + if (!peer->sta) + return NULL; + + ahsta = ath12k_sta_to_ahsta(peer->sta); + if (peer->ml_id & ATH12K_PEER_ML_ID_VALID) { + if (!(ahsta->links_map & BIT(peer->link_id))) { + ath12k_warn(ab, "peer %pM id %d link_id %d can't found in STA link_map 0x%x\n", + peer->addr, peer->peer_id, peer->link_id, + ahsta->links_map); + return NULL; + } + arsta = rcu_dereference(ahsta->link[peer->link_id]); + if (!arsta) + return NULL; + } else { + arsta = &ahsta->deflink; + } + return arsta; +} + +static int ath12k_dp_link_peer_rhash_addr_tbl_init(struct ath12k_dp *dp) +{ + struct ath12k_base *ab = dp->ab; + struct rhashtable_params *param; + struct rhashtable *rhash_addr_tbl; + int ret; + + lockdep_assert_held(&dp->link_peer_rhash_tbl_lock); + + rhash_addr_tbl = kzalloc(sizeof(*dp->rhead_peer_addr), GFP_KERNEL); + if (!rhash_addr_tbl) + return -ENOMEM; + + param = &dp->rhash_peer_addr_param; + + param->key_offset = offsetof(struct ath12k_dp_link_peer, addr); + param->head_offset = offsetof(struct ath12k_dp_link_peer, rhash_addr); + param->key_len = sizeof_field(struct ath12k_dp_link_peer, addr); + param->automatic_shrinking = true; + param->nelem_hint = ab->num_radios * ath12k_core_get_max_peers_per_radio(ab); + + ret = rhashtable_init(rhash_addr_tbl, param); + if (ret) { + ath12k_warn(ab, "failed to init peer addr rhash table %d\n", ret); + goto err_free; + } + + dp->rhead_peer_addr = rhash_addr_tbl; + + return 0; + +err_free: + kfree(rhash_addr_tbl); + + return ret; +} + +int ath12k_dp_link_peer_rhash_tbl_init(struct ath12k_dp *dp) +{ + int ret; + + mutex_lock(&dp->link_peer_rhash_tbl_lock); + ret = ath12k_dp_link_peer_rhash_addr_tbl_init(dp); + mutex_unlock(&dp->link_peer_rhash_tbl_lock); + + return ret; +} + +void ath12k_dp_link_peer_rhash_tbl_destroy(struct ath12k_dp *dp) +{ + mutex_lock(&dp->link_peer_rhash_tbl_lock); + rhashtable_destroy(dp->rhead_peer_addr); + kfree(dp->rhead_peer_addr); + dp->rhead_peer_addr = NULL; + mutex_unlock(&dp->link_peer_rhash_tbl_lock); +} + +static int ath12k_dp_link_peer_rhash_insert(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer) +{ + struct ath12k_dp_link_peer *tmp; + + lockdep_assert_held(&dp->dp_lock); + + tmp = rhashtable_lookup_get_insert_fast(dp->rhead_peer_addr, &peer->rhash_addr, + dp->rhash_peer_addr_param); + if (!tmp) + return 0; + else if (IS_ERR(tmp)) + return PTR_ERR(tmp); + else + return -EEXIST; +} + +static int ath12k_dp_link_peer_rhash_remove(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer) +{ + int ret; + + lockdep_assert_held(&dp->dp_lock); + + ret = rhashtable_remove_fast(dp->rhead_peer_addr, &peer->rhash_addr, + dp->rhash_peer_addr_param); + if (ret && ret != -ENOENT) + return ret; + + return 0; +} + +int ath12k_dp_link_peer_rhash_add(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer) +{ + int ret; + + lockdep_assert_held(&dp->dp_lock); + + ret = ath12k_dp_link_peer_rhash_insert(dp, peer); + if (ret) + ath12k_warn(dp, "failed to add peer %pM with id %d in rhash_addr ret %d\n", + peer->addr, peer->peer_id, ret); + + return ret; +} + +void ath12k_dp_link_peer_rhash_delete(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer) +{ + /* No failure handling and hence return type is void */ + int ret; + + lockdep_assert_held(&dp->dp_lock); + + ret = ath12k_dp_link_peer_rhash_remove(dp, peer); + if (ret) + ath12k_warn(dp, "failed to remove peer %pM with id %d in rhash_addr ret %d\n", + peer->addr, peer->peer_id, ret); +} + +struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr(struct ath12k_dp_hw *dp_hw, u8 *addr) +{ + struct ath12k_dp_peer *peer; + + lockdep_assert_held(&dp_hw->peer_lock); + + list_for_each_entry(peer, &dp_hw->dp_peers_list, list) { + if (ether_addr_equal(peer->addr, addr)) + return peer; + } + + return NULL; +} +EXPORT_SYMBOL(ath12k_dp_peer_find_by_addr); + +struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr_and_sta(struct ath12k_dp_hw *dp_hw, + u8 *addr, + struct ieee80211_sta *sta) +{ + struct ath12k_dp_peer *dp_peer; + + lockdep_assert_held(&dp_hw->peer_lock); + + list_for_each_entry(dp_peer, &dp_hw->dp_peers_list, list) { + if (ether_addr_equal(dp_peer->addr, addr) && (dp_peer->sta == sta)) + return dp_peer; + } + + return NULL; +} + +static struct ath12k_dp_peer *ath12k_dp_peer_create_find(struct ath12k_dp_hw *dp_hw, + u8 *addr, + struct ieee80211_sta *sta, + bool mlo_peer) +{ + struct ath12k_dp_peer *dp_peer; + + lockdep_assert_held(&dp_hw->peer_lock); + + list_for_each_entry(dp_peer, &dp_hw->dp_peers_list, list) { + if (ether_addr_equal(dp_peer->addr, addr)) { + if (!sta || mlo_peer || dp_peer->is_mlo || + dp_peer->sta == sta) + return dp_peer; + } + } + + return NULL; +} + +/* + * Index of ath12k_dp_peer for MLO client is same as peer id of ath12k_dp_peer, + * while for ath12k_dp_link_peer(mlo and non-mlo) and ath12k_dp_peer for + * Non-MLO client it is derived as ((DEVICE_ID << 10) | (10 bits of peer id)). + * + * This is done because ml_peer_id and peer_id_table are at hw granularity, + * while link_peer_id is at device granularity, hence in order to avoid + * conflict this approach is followed. + */ +#define ATH12K_DP_PEER_TABLE_DEVICE_ID_SHIFT 10 + +u16 ath12k_dp_peer_get_peerid_index(struct ath12k_dp *dp, u16 peer_id) +{ + return (peer_id & ATH12K_PEER_ML_ID_VALID) ? peer_id : + ((dp->device_id << ATH12K_DP_PEER_TABLE_DEVICE_ID_SHIFT) | peer_id); +} + +struct ath12k_dp_peer *ath12k_dp_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev, + u16 peer_id) +{ + u16 index; + struct ath12k_dp *dp = dp_pdev->dp; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "ath12k dp peer find by peerid index called without rcu lock"); + + if (peer_id >= ATH12K_DP_PEER_ID_INVALID) + return NULL; + + index = ath12k_dp_peer_get_peerid_index(dp, peer_id); + + return rcu_dereference(dp_pdev->dp_hw->dp_peers[index]); +} +EXPORT_SYMBOL(ath12k_dp_peer_find_by_peerid); + +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev, u16 peer_id) +{ + struct ath12k_dp_peer *dp_peer = NULL; + u8 link_id; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "ath12k dp link peer find by peerid index called without rcu lock"); + + if (dp_pdev->hw_link_id >= ATH12K_GROUP_MAX_RADIO) + return NULL; + + dp_peer = ath12k_dp_peer_find_by_peerid(dp_pdev, peer_id); + if (!dp_peer) + return NULL; + + link_id = dp_peer->hw_links[dp_pdev->hw_link_id]; + + return rcu_dereference(dp_peer->link_peers[link_id]); +} +EXPORT_SYMBOL(ath12k_dp_link_peer_find_by_peerid); + +int ath12k_dp_peer_create(struct ath12k_dp_hw *dp_hw, u8 *addr, + struct ath12k_dp_peer_create_params *params) +{ + struct ath12k_dp_peer *dp_peer; + + spin_lock_bh(&dp_hw->peer_lock); + dp_peer = ath12k_dp_peer_create_find(dp_hw, addr, params->sta, params->is_mlo); + if (dp_peer) { + spin_unlock_bh(&dp_hw->peer_lock); + return -EEXIST; + } + spin_unlock_bh(&dp_hw->peer_lock); + + dp_peer = kzalloc(sizeof(*dp_peer), GFP_ATOMIC); + if (!dp_peer) + return -ENOMEM; + + ether_addr_copy(dp_peer->addr, addr); + dp_peer->sta = params->sta; + dp_peer->is_mlo = params->is_mlo; + + /* + * For MLO client, the host assigns the ML peer ID, so set peer_id in dp_peer + * For non-MLO client, host gets link peer ID from firmware and will be + * assigned at the time of link peer creation + */ + dp_peer->peer_id = params->is_mlo ? params->peer_id : ATH12K_DP_PEER_ID_INVALID; + dp_peer->ucast_ra_only = params->ucast_ra_only; + + dp_peer->sec_type = HAL_ENCRYPT_TYPE_OPEN; + dp_peer->sec_type_grp = HAL_ENCRYPT_TYPE_OPEN; + dp_peer->ucast_ra_only = params->ucast_ra_only; + + spin_lock_bh(&dp_hw->peer_lock); + + list_add(&dp_peer->list, &dp_hw->dp_peers_list); + + /* + * For MLO client, the peer_id for ath12k_dp_peer is allocated by host + * and that peer_id is known at this point, and hence this ath12k_dp_peer + * can be added to the RCU table using the peer_id. + * For non-MLO client, this addition to RCU table shall be done at the + * time of assignment of ath12k_dp_link_peer to ath12k_dp_peer. + */ + if (dp_peer->is_mlo) + rcu_assign_pointer(dp_hw->dp_peers[dp_peer->peer_id], dp_peer); + + spin_unlock_bh(&dp_hw->peer_lock); + + return 0; +} + +void ath12k_dp_peer_delete(struct ath12k_dp_hw *dp_hw, u8 *addr, + struct ieee80211_sta *sta) +{ + struct ath12k_dp_peer *dp_peer; + + spin_lock_bh(&dp_hw->peer_lock); + + dp_peer = ath12k_dp_peer_find_by_addr_and_sta(dp_hw, addr, sta); + if (!dp_peer) { + spin_unlock_bh(&dp_hw->peer_lock); + return; + } + + if (dp_peer->is_mlo) + rcu_assign_pointer(dp_hw->dp_peers[dp_peer->peer_id], NULL); + + list_del(&dp_peer->list); + + spin_unlock_bh(&dp_hw->peer_lock); + + synchronize_rcu(); + kfree(dp_peer); +} + +int ath12k_dp_link_peer_assign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw, + u8 vdev_id, struct ieee80211_sta *sta, u8 *addr, + u8 link_id, u32 hw_link_id) +{ + struct ath12k_dp_peer *dp_peer; + struct ath12k_dp_link_peer *peer, *temp_peer; + u16 peerid_index; + int ret = -EINVAL; + u8 *dp_peer_mac = !sta ? addr : sta->addr; + + spin_lock_bh(&dp->dp_lock); + + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr); + if (!peer) { + ath12k_warn(dp, "failed to find dp_link_peer with mac %pM on vdev %u\n", + addr, vdev_id); + ret = -ENOENT; + goto err_peer; + } + + spin_lock_bh(&dp_hw->peer_lock); + + dp_peer = ath12k_dp_peer_find_by_addr_and_sta(dp_hw, dp_peer_mac, sta); + if (!dp_peer) { + ath12k_warn(dp, "failed to find dp_peer with mac %pM\n", dp_peer_mac); + ret = -ENOENT; + goto err_dp_peer; + } + + /* + * Set peer_id in dp_peer for non-mlo client, peer_id for mlo client is + * set during dp_peer create + */ + if (!dp_peer->is_mlo) + dp_peer->peer_id = peer->peer_id; + + peer->dp_peer = dp_peer; + peer->hw_link_id = hw_link_id; + + dp_peer->hw_links[peer->hw_link_id] = link_id; + + peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id); + + rcu_assign_pointer(dp_peer->link_peers[peer->link_id], peer); + + rcu_assign_pointer(dp_hw->dp_peers[peerid_index], dp_peer); + + spin_unlock_bh(&dp_hw->peer_lock); + + /* + * In case of Split PHY and roaming scenario, pdev idx + * might differ but both the pdev will share same rhash + * table. In that case update the rhash table if link_peer is + * already present + */ + temp_peer = ath12k_dp_link_peer_find_by_addr(dp, addr); + if (temp_peer && temp_peer->hw_link_id != hw_link_id) + ath12k_dp_link_peer_rhash_delete(dp, temp_peer); + + ret = ath12k_dp_link_peer_rhash_add(dp, peer); + if (ret) { + /* + * If new entry addition failed, add back old entry + * If old entry addition also fails, then nothing + * can be done, simply proceed + */ + if (temp_peer) + ath12k_dp_link_peer_rhash_add(dp, temp_peer); + } + + spin_unlock_bh(&dp->dp_lock); + + return ret; + +err_dp_peer: + spin_unlock_bh(&dp_hw->peer_lock); + +err_peer: + spin_unlock_bh(&dp->dp_lock); + + return ret; +} + +void ath12k_dp_link_peer_unassign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw, + u8 vdev_id, u8 *addr, u32 hw_link_id) +{ + struct ath12k_dp_peer *dp_peer; + struct ath12k_dp_link_peer *peer, *temp_peer; + u16 peerid_index; + + spin_lock_bh(&dp->dp_lock); + + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&dp->dp_lock); + return; + } + + spin_lock_bh(&dp_hw->peer_lock); + + dp_peer = peer->dp_peer; + dp_peer->hw_links[peer->hw_link_id] = 0; + + peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id); + + rcu_assign_pointer(dp_peer->link_peers[peer->link_id], NULL); + + rcu_assign_pointer(dp_hw->dp_peers[peerid_index], NULL); + + spin_unlock_bh(&dp_hw->peer_lock); + + /* To handle roaming and split phy scenario */ + temp_peer = ath12k_dp_link_peer_find_by_addr(dp, addr); + if (temp_peer && temp_peer->hw_link_id == hw_link_id) + ath12k_dp_link_peer_rhash_delete(dp, peer); + + spin_unlock_bh(&dp->dp_lock); + + synchronize_rcu(); +} + +void +ath12k_dp_link_peer_get_sta_rate_info_stats(struct ath12k_dp *dp, const u8 *addr, + struct ath12k_dp_link_peer_rate_info *info) +{ + struct ath12k_dp_link_peer *link_peer; + + guard(spinlock_bh)(&dp->dp_lock); + + link_peer = ath12k_dp_link_peer_find_by_addr(dp, addr); + if (!link_peer) + return; + + info->rx_duration = link_peer->rx_duration; + info->tx_duration = link_peer->tx_duration; + info->txrate.legacy = link_peer->txrate.legacy; + info->txrate.mcs = link_peer->txrate.mcs; + info->txrate.nss = link_peer->txrate.nss; + info->txrate.bw = link_peer->txrate.bw; + info->txrate.he_gi = link_peer->txrate.he_gi; + info->txrate.he_dcm = link_peer->txrate.he_dcm; + info->txrate.he_ru_alloc = link_peer->txrate.he_ru_alloc; + info->txrate.flags = link_peer->txrate.flags; + info->rssi_comb = link_peer->rssi_comb; + info->signal_avg = ewma_avg_rssi_read(&link_peer->avg_rssi); +} + +void ath12k_dp_link_peer_reset_rx_stats(struct ath12k_dp *dp, const u8 *addr) +{ + struct ath12k_rx_peer_stats *rx_stats; + struct ath12k_dp_link_peer *link_peer; + + guard(spinlock_bh)(&dp->dp_lock); + + link_peer = ath12k_dp_link_peer_find_by_addr(dp, addr); + if (!link_peer || !link_peer->peer_stats.rx_stats) + return; + + rx_stats = link_peer->peer_stats.rx_stats; + if (rx_stats) + memset(rx_stats, 0, sizeof(*rx_stats)); +} diff --git a/drivers/net/wireless/ath/ath12k/dp_peer.h b/drivers/net/wireless/ath/ath12k/dp_peer.h new file mode 100644 index 0000000000000..20294ff095131 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/dp_peer.h @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_DP_PEER_H +#define ATH12K_DP_PEER_H + +#include "dp_rx.h" + +#define ATH12K_DP_PEER_ID_INVALID 0x3FFF + +struct ppdu_user_delayba { + u16 sw_peer_id; + u32 info0; + u16 ru_end; + u16 ru_start; + u32 info1; + u32 rate_flags; + u32 resp_rate_flags; +}; + +#define ATH12K_PEER_ML_ID_VALID BIT(13) + +struct ath12k_rx_peer_rate_stats { + u64 ht_mcs_count[HAL_RX_MAX_MCS_HT + 1]; + u64 vht_mcs_count[HAL_RX_MAX_MCS_VHT + 1]; + u64 he_mcs_count[HAL_RX_MAX_MCS_HE + 1]; + u64 be_mcs_count[HAL_RX_MAX_MCS_BE + 1]; + u64 nss_count[HAL_RX_MAX_NSS]; + u64 bw_count[HAL_RX_BW_MAX]; + u64 gi_count[HAL_RX_GI_MAX]; + u64 legacy_count[HAL_RX_MAX_NUM_LEGACY_RATES]; + u64 rx_rate[HAL_RX_BW_MAX][HAL_RX_GI_MAX][HAL_RX_MAX_NSS][HAL_RX_MAX_MCS_HT + 1]; +}; + +struct ath12k_rx_peer_stats { + u64 num_msdu; + u64 num_mpdu_fcs_ok; + u64 num_mpdu_fcs_err; + u64 tcp_msdu_count; + u64 udp_msdu_count; + u64 other_msdu_count; + u64 ampdu_msdu_count; + u64 non_ampdu_msdu_count; + u64 stbc_count; + u64 beamformed_count; + u64 coding_count[HAL_RX_SU_MU_CODING_MAX]; + u64 tid_count[IEEE80211_NUM_TIDS + 1]; + u64 pream_cnt[HAL_RX_PREAMBLE_MAX]; + u64 reception_type[HAL_RX_RECEPTION_TYPE_MAX]; + u64 rx_duration; + u64 dcm_count; + u64 ru_alloc_cnt[HAL_RX_RU_ALLOC_TYPE_MAX]; + struct ath12k_rx_peer_rate_stats pkt_stats; + struct ath12k_rx_peer_rate_stats byte_stats; +}; + +struct ath12k_wbm_tx_stats { + u64 wbm_tx_comp_stats[HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX]; +}; + +struct ath12k_dp_peer_stats { + struct ath12k_rx_peer_stats *rx_stats; + struct ath12k_wbm_tx_stats *wbm_tx_stats; +}; + +DECLARE_EWMA(avg_rssi, 10, 8) + +struct ath12k_dp_link_peer { + struct list_head list; + struct ieee80211_sta *sta; + struct ath12k_dp_peer *dp_peer; + int vdev_id; + u8 addr[ETH_ALEN]; + int peer_id; + u16 ast_hash; + u8 pdev_idx; + u16 hw_peer_id; + + struct ppdu_user_delayba ppdu_stats_delayba; + bool delayba_flag; + bool is_authorized; + bool mlo; + /* protected by ab->data_lock */ + + u16 ml_id; + + /* any other ML info common for all partners can be added + * here and would be same for all partner peers. + */ + u8 ml_addr[ETH_ALEN]; + + /* To ensure only certain work related to dp is done once */ + bool primary_link; + + /* for reference to ath12k_link_sta */ + u8 link_id; + + /* peer addr based rhashtable list pointer */ + struct rhash_head rhash_addr; + + u8 hw_link_id; + u32 rx_tid_active_bitmask; + + /* link stats */ + struct rate_info txrate; + struct rate_info last_txrate; + u64 rx_duration; + u64 tx_duration; + u8 rssi_comb; + struct ewma_avg_rssi avg_rssi; + struct ath12k_dp_peer_stats peer_stats; + u32 tx_retry_failed; + u32 tx_retry_count; +}; + +void ath12k_dp_link_peer_unmap_event(struct ath12k_base *ab, u16 peer_id); +void ath12k_dp_link_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id, + u8 *mac_addr, u16 ast_hash, u16 hw_peer_id); + +struct ath12k_dp_peer { + struct list_head list; + bool is_mlo; + bool dp_setup_done; + + u8 ucast_keyidx; + u8 addr[ETH_ALEN]; + + u8 mcast_keyidx; + bool ucast_ra_only; + int peer_id; + struct ieee80211_sta *sta; + + u8 hw_links[ATH12K_GROUP_MAX_RADIO]; + + u16 sec_type_grp; + u16 sec_type; + + /* Info used in MMIC verification of * RX fragments */ + struct crypto_shash *tfm_mmic; + struct ieee80211_key_conf *keys[WMI_MAX_KEY_INDEX + 1]; + struct ath12k_dp_link_peer __rcu *link_peers[ATH12K_NUM_MAX_LINKS]; + struct ath12k_reoq_buf reoq_bufs[IEEE80211_NUM_TIDS + 1]; + struct ath12k_dp_rx_tid rx_tid[IEEE80211_NUM_TIDS + 1]; +}; + +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_vdev_and_addr(struct ath12k_dp *dp, + int vdev_id, const u8 *addr); +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_addr(struct ath12k_dp *dp, const u8 *addr); +bool ath12k_dp_link_peer_exist_by_vdev_id(struct ath12k_dp *dp, int vdev_id); +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_ast(struct ath12k_dp *dp, int ast_hash); +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_pdev_and_addr(struct ath12k_dp *dp, u8 pdev_idx, + const u8 *addr); +struct ath12k_link_sta *ath12k_dp_link_peer_to_link_sta(struct ath12k_base *ab, + struct ath12k_dp_link_peer *peer); +int ath12k_dp_link_peer_rhash_tbl_init(struct ath12k_dp *dp); +void ath12k_dp_link_peer_rhash_tbl_destroy(struct ath12k_dp *dp); +int ath12k_dp_link_peer_rhash_add(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer); +void ath12k_dp_link_peer_rhash_delete(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer); +int ath12k_dp_peer_create(struct ath12k_dp_hw *dp_hw, u8 *addr, + struct ath12k_dp_peer_create_params *params); +void ath12k_dp_peer_delete(struct ath12k_dp_hw *dp_hw, u8 *addr, + struct ieee80211_sta *sta); +struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr(struct ath12k_dp_hw *dp_hw, u8 *addr); +struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr_and_sta(struct ath12k_dp_hw *dp_hw, + u8 *addr, + struct ieee80211_sta *sta); +u16 ath12k_dp_peer_get_peerid_index(struct ath12k_dp *dp, u16 peer_id); +struct ath12k_dp_peer *ath12k_dp_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev, + u16 peer_id); +struct ath12k_dp_link_peer * +ath12k_dp_link_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev, u16 peer_id); +void ath12k_dp_link_peer_free(struct ath12k_dp_link_peer *peer); +#endif diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c index 5e5c14a70316d..30888b2f2bac4 100644 --- a/drivers/net/wireless/ath/ath12k/dp_rx.c +++ b/drivers/net/wireless/ath/ath12k/dp_rx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -10,257 +10,16 @@ #include #include "core.h" #include "debug.h" -#include "hal_desc.h" #include "hw.h" #include "dp_rx.h" -#include "hal_rx.h" #include "dp_tx.h" #include "peer.h" #include "dp_mon.h" #include "debugfs_htt_stats.h" -#define ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS (2 * HZ) - static int ath12k_dp_rx_tid_delete_handler(struct ath12k_base *ab, struct ath12k_dp_rx_tid_rxq *rx_tid); -static enum hal_encrypt_type ath12k_dp_rx_h_enctype(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - if (!ab->hal_rx_ops->rx_desc_encrypt_valid(desc)) - return HAL_ENCRYPT_TYPE_OPEN; - - return ab->hal_rx_ops->rx_desc_get_encrypt_type(desc); -} - -u8 ath12k_dp_rx_h_decap_type(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_decap_type(desc); -} - -static u8 ath12k_dp_rx_h_mesh_ctl_present(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_mesh_ctl(desc); -} - -static bool ath12k_dp_rx_h_seq_ctrl_valid(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_mpdu_seq_ctl_vld(desc); -} - -static bool ath12k_dp_rx_h_fc_valid(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_mpdu_fc_valid(desc); -} - -static bool ath12k_dp_rx_h_more_frags(struct ath12k_base *ab, - struct sk_buff *skb) -{ - struct ieee80211_hdr *hdr; - - hdr = (struct ieee80211_hdr *)(skb->data + ab->hal.hal_desc_sz); - return ieee80211_has_morefrags(hdr->frame_control); -} - -static u16 ath12k_dp_rx_h_frag_no(struct ath12k_base *ab, - struct sk_buff *skb) -{ - struct ieee80211_hdr *hdr; - - hdr = (struct ieee80211_hdr *)(skb->data + ab->hal.hal_desc_sz); - return le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_FRAG; -} - -static u16 ath12k_dp_rx_h_seq_no(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_mpdu_start_seq_no(desc); -} - -static bool ath12k_dp_rx_h_msdu_done(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->dp_rx_h_msdu_done(desc); -} - -static bool ath12k_dp_rx_h_l4_cksum_fail(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->dp_rx_h_l4_cksum_fail(desc); -} - -static bool ath12k_dp_rx_h_ip_cksum_fail(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->dp_rx_h_ip_cksum_fail(desc); -} - -static bool ath12k_dp_rx_h_is_decrypted(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->dp_rx_h_is_decrypted(desc); -} - -u32 ath12k_dp_rx_h_mpdu_err(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->dp_rx_h_mpdu_err(desc); -} - -static u16 ath12k_dp_rx_h_msdu_len(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_msdu_len(desc); -} - -static u8 ath12k_dp_rx_h_sgi(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_msdu_sgi(desc); -} - -static u8 ath12k_dp_rx_h_rate_mcs(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_msdu_rate_mcs(desc); -} - -static u8 ath12k_dp_rx_h_rx_bw(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_msdu_rx_bw(desc); -} - -static u32 ath12k_dp_rx_h_freq(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_msdu_freq(desc); -} - -static u8 ath12k_dp_rx_h_pkt_type(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_msdu_pkt_type(desc); -} - -static u8 ath12k_dp_rx_h_nss(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return hweight8(ab->hal_rx_ops->rx_desc_get_msdu_nss(desc)); -} - -static u8 ath12k_dp_rx_h_tid(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_mpdu_tid(desc); -} - -static u16 ath12k_dp_rx_h_peer_id(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_mpdu_peer_id(desc); -} - -u8 ath12k_dp_rx_h_l3pad(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_l3_pad_bytes(desc); -} - -static bool ath12k_dp_rx_h_first_msdu(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_first_msdu(desc); -} - -static bool ath12k_dp_rx_h_last_msdu(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_last_msdu(desc); -} - -static void ath12k_dp_rx_desc_end_tlv_copy(struct ath12k_base *ab, - struct hal_rx_desc *fdesc, - struct hal_rx_desc *ldesc) -{ - ab->hal_rx_ops->rx_desc_copy_end_tlv(fdesc, ldesc); -} - -static void ath12k_dp_rxdesc_set_msdu_len(struct ath12k_base *ab, - struct hal_rx_desc *desc, - u16 len) -{ - ab->hal_rx_ops->rx_desc_set_msdu_len(desc, len); -} - -u32 ath12k_dp_rxdesc_get_ppduid(struct ath12k_base *ab, - struct hal_rx_desc *rx_desc) -{ - return ab->hal_rx_ops->rx_desc_get_mpdu_ppdu_id(rx_desc); -} - -bool ath12k_dp_rxdesc_mpdu_valid(struct ath12k_base *ab, - struct hal_rx_desc *rx_desc) -{ - u32 tlv_tag; - - tlv_tag = ab->hal_rx_ops->rx_desc_get_mpdu_start_tag(rx_desc); - - return tlv_tag == HAL_RX_MPDU_START; -} - -static bool ath12k_dp_rx_h_is_da_mcbc(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return (ath12k_dp_rx_h_first_msdu(ab, desc) && - ab->hal_rx_ops->rx_desc_is_da_mcbc(desc)); -} - -static bool ath12k_dp_rxdesc_mac_addr2_valid(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_mac_addr2_valid(desc); -} - -static u8 *ath12k_dp_rxdesc_get_mpdu_start_addr2(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_mpdu_start_addr2(desc); -} - -static void ath12k_dp_rx_desc_get_dot11_hdr(struct ath12k_base *ab, - struct hal_rx_desc *desc, - struct ieee80211_hdr *hdr) -{ - ab->hal_rx_ops->rx_desc_get_dot11_hdr(desc, hdr); -} - -static void ath12k_dp_rx_desc_get_crypto_header(struct ath12k_base *ab, - struct hal_rx_desc *desc, - u8 *crypto_hdr, - enum hal_encrypt_type enctype) -{ - ab->hal_rx_ops->rx_desc_get_crypto_header(desc, crypto_hdr, enctype); -} - -static inline u8 ath12k_dp_rx_get_msdu_src_link(struct ath12k_base *ab, - struct hal_rx_desc *desc) -{ - return ab->hal_rx_ops->rx_desc_get_msdu_src_link_id(desc); -} - -static void ath12k_dp_clean_up_skb_list(struct sk_buff_head *skb_list) -{ - struct sk_buff *skb; - - while ((skb = __skb_dequeue(skb_list))) - dev_kfree_skb_any(skb); -} - static size_t ath12k_dp_list_cut_nodes(struct list_head *list, struct list_head *head, size_t count) @@ -305,11 +64,12 @@ static void ath12k_dp_rx_enqueue_free(struct ath12k_dp *dp, } /* Returns number of Rx buffers replenished */ -int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, +int ath12k_dp_rx_bufs_replenish(struct ath12k_dp *dp, struct dp_rxdma_ring *rx_ring, struct list_head *used_list, int req_entries) { + struct ath12k_base *ab = dp->ab; struct ath12k_buffer_addr *desc; struct hal_srng *srng; struct sk_buff *skb; @@ -317,13 +77,12 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, int num_remain; u32 cookie; dma_addr_t paddr; - struct ath12k_dp *dp = &ab->dp; struct ath12k_rx_desc_info *rx_desc; - enum hal_rx_buf_return_buf_manager mgr = ab->hw_params->hal_params->rx_buf_rbm; + enum hal_rx_buf_return_buf_manager mgr = dp->hal->hal_params->rx_buf_rbm; req_entries = min(req_entries, rx_ring->bufs_max); - srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id]; + srng = &dp->hal->srng_list[rx_ring->refill_buf_ring.ring_id]; spin_lock_bh(&srng->lock); @@ -362,10 +121,10 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, skb->data); } - paddr = dma_map_single(ab->dev, skb->data, + paddr = dma_map_single(dp->dev, skb->data, skb->len + skb_tailroom(skb), DMA_FROM_DEVICE); - if (dma_mapping_error(ab->dev, paddr)) + if (dma_mapping_error(dp->dev, paddr)) goto fail_free_skb; rx_desc = list_first_entry_or_null(used_list, @@ -386,13 +145,14 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, num_remain--; - ath12k_hal_rx_buf_addr_info_set(desc, paddr, cookie, mgr); + ath12k_hal_rx_buf_addr_info_set(dp->hal, desc, paddr, cookie, + mgr); } goto out; fail_dma_unmap: - dma_unmap_single(ab->dev, paddr, skb->len + skb_tailroom(skb), + dma_unmap_single(dp->dev, paddr, skb->len + skb_tailroom(skb), DMA_FROM_DEVICE); fail_free_skb: dev_kfree_skb_any(skb); @@ -406,6 +166,7 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, return req_entries - num_remain; } +EXPORT_SYMBOL(ath12k_dp_rx_bufs_replenish); static int ath12k_dp_rxdma_mon_buf_ring_free(struct ath12k_base *ab, struct dp_rxdma_mon_ring *rx_ring) @@ -432,7 +193,7 @@ static int ath12k_dp_rxdma_mon_buf_ring_free(struct ath12k_base *ab, static int ath12k_dp_rxdma_buf_free(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int i; ath12k_dp_rxdma_mon_buf_ring_free(ab, &dp->rxdma_mon_buf_ring); @@ -475,14 +236,14 @@ static int ath12k_dp_rxdma_ring_buf_setup(struct ath12k_base *ab, rx_ring->bufs_max = rx_ring->refill_buf_ring.size / ath12k_hal_srng_get_entrysize(ab, HAL_RXDMA_BUF); - ath12k_dp_rx_bufs_replenish(ab, rx_ring, &list, 0); + ath12k_dp_rx_bufs_replenish(ath12k_ab_to_dp(ab), rx_ring, &list, 0); return 0; } static int ath12k_dp_rxdma_buf_setup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct dp_rxdma_mon_ring *mon_ring; int ret, i; @@ -529,7 +290,7 @@ static void ath12k_dp_rx_pdev_srng_free(struct ath12k *ar) void ath12k_dp_rx_pdev_reo_cleanup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int i; for (i = 0; i < DP_REO_DST_RING_MAX; i++) @@ -538,7 +299,7 @@ void ath12k_dp_rx_pdev_reo_cleanup(struct ath12k_base *ab) int ath12k_dp_rx_pdev_reo_setup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int ret; int i; @@ -584,13 +345,15 @@ static int ath12k_dp_rx_pdev_srng_alloc(struct ath12k *ar) return 0; } -static void ath12k_dp_init_rx_tid_rxq(struct ath12k_dp_rx_tid_rxq *rx_tid_rxq, - struct ath12k_dp_rx_tid *rx_tid) +void ath12k_dp_init_rx_tid_rxq(struct ath12k_dp_rx_tid_rxq *rx_tid_rxq, + struct ath12k_dp_rx_tid *rx_tid, + bool active) { rx_tid_rxq->tid = rx_tid->tid; - rx_tid_rxq->active = rx_tid->active; + rx_tid_rxq->active = active; rx_tid_rxq->qbuf = rx_tid->qbuf; } +EXPORT_SYMBOL(ath12k_dp_init_rx_tid_rxq); static void ath12k_dp_rx_tid_cleanup(struct ath12k_base *ab, struct ath12k_reoq_buf *tid_qbuf) @@ -605,7 +368,7 @@ static void ath12k_dp_rx_tid_cleanup(struct ath12k_base *ab, void ath12k_dp_rx_reo_cmd_list_cleanup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct ath12k_dp_rx_reo_cmd *cmd, *tmp; struct ath12k_dp_rx_reo_cache_flush_elem *cmd_cache, *tmp_cache; struct dp_reo_update_rx_queue_elem *cmd_queue, *tmp_queue; @@ -635,8 +398,8 @@ void ath12k_dp_rx_reo_cmd_list_cleanup(struct ath12k_base *ab) spin_unlock_bh(&dp->reo_cmd_lock); } -static void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx, - enum hal_reo_cmd_status status) +void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status) { struct ath12k_dp_rx_tid_rxq *rx_tid = ctx; @@ -646,109 +409,9 @@ static void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx, ath12k_dp_rx_tid_cleanup(dp->ab, &rx_tid->qbuf); } +EXPORT_SYMBOL(ath12k_dp_reo_cmd_free); -static int ath12k_dp_reo_cmd_send(struct ath12k_base *ab, - struct ath12k_dp_rx_tid_rxq *rx_tid, - enum hal_reo_cmd_type type, - struct ath12k_hal_reo_cmd *cmd, - void (*cb)(struct ath12k_dp *dp, void *ctx, - enum hal_reo_cmd_status status)) -{ - struct ath12k_dp *dp = &ab->dp; - struct ath12k_dp_rx_reo_cmd *dp_cmd; - struct hal_srng *cmd_ring; - int cmd_num; - - cmd_ring = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id]; - cmd_num = ath12k_hal_reo_cmd_send(ab, cmd_ring, type, cmd); - - /* cmd_num should start from 1, during failure return the error code */ - if (cmd_num < 0) - return cmd_num; - - /* reo cmd ring descriptors has cmd_num starting from 1 */ - if (cmd_num == 0) - return -EINVAL; - - if (!cb) - return 0; - - /* Can this be optimized so that we keep the pending command list only - * for tid delete command to free up the resource on the command status - * indication? - */ - dp_cmd = kzalloc(sizeof(*dp_cmd), GFP_ATOMIC); - - if (!dp_cmd) - return -ENOMEM; - - memcpy(&dp_cmd->data, rx_tid, sizeof(*rx_tid)); - dp_cmd->cmd_num = cmd_num; - dp_cmd->handler = cb; - - spin_lock_bh(&dp->reo_cmd_lock); - list_add_tail(&dp_cmd->list, &dp->reo_cmd_list); - spin_unlock_bh(&dp->reo_cmd_lock); - - return 0; -} - -static int ath12k_dp_reo_cache_flush(struct ath12k_base *ab, - struct ath12k_dp_rx_tid_rxq *rx_tid) -{ - struct ath12k_hal_reo_cmd cmd = {}; - int ret; - - cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned); - cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned); - /* HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS - all pending MPDUs - *in the bitmap will be forwarded/flushed to REO output rings - */ - cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS | - HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS; - - /* For all QoS TIDs (except NON_QOS), the driver allocates a maximum - * window size of 1024. In such cases, the driver can issue a single - * 1KB descriptor flush command instead of sending multiple 128-byte - * flush commands for each QoS TID, improving efficiency. - */ - - if (rx_tid->tid != HAL_DESC_REO_NON_QOS_TID) - cmd.flag |= HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC; - - ret = ath12k_dp_reo_cmd_send(ab, rx_tid, - HAL_REO_CMD_FLUSH_CACHE, - &cmd, ath12k_dp_reo_cmd_free); - return ret; -} - -static void ath12k_peer_rx_tid_qref_reset(struct ath12k_base *ab, u16 peer_id, u16 tid) -{ - struct ath12k_reo_queue_ref *qref; - struct ath12k_dp *dp = &ab->dp; - bool ml_peer = false; - - if (!ab->hw_params->reoq_lut_support) - return; - - if (peer_id & ATH12K_PEER_ML_ID_VALID) { - peer_id &= ~ATH12K_PEER_ML_ID_VALID; - ml_peer = true; - } - - if (ml_peer) - qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr + - (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); - else - qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr + - (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); - - qref->info0 = u32_encode_bits(0, BUFFER_ADDR_INFO0_ADDR); - qref->info1 = u32_encode_bits(0, BUFFER_ADDR_INFO1_ADDR) | - u32_encode_bits(tid, DP_REO_QREF_NUM); -} - -static void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *dp) +void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *dp) { struct ath12k_base *ab = dp->ab; struct dp_reo_update_rx_queue_elem *elem, *tmp; @@ -762,10 +425,10 @@ static void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp * if (ath12k_dp_rx_tid_delete_handler(ab, &elem->rx_tid)) break; - ath12k_peer_rx_tid_qref_reset(ab, - elem->is_ml_peer ? elem->ml_peer_id : - elem->peer_id, - elem->rx_tid.tid); + ath12k_dp_arch_peer_rx_tid_qref_reset(dp, + elem->is_ml_peer ? + elem->ml_peer_id : elem->peer_id, + elem->rx_tid.tid); if (ab->hw_params->reoq_lut_support) ath12k_hal_reo_shared_qaddr_cache_clear(ab); @@ -776,9 +439,10 @@ static void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp * spin_unlock_bh(&dp->reo_rxq_flush_lock); } +EXPORT_SYMBOL(ath12k_dp_rx_process_reo_cmd_update_rx_queue_list); -static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx, - enum hal_reo_cmd_status status) +void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status) { struct ath12k_base *ab = dp->ab; struct ath12k_dp_rx_tid_rxq *rx_tid = ctx; @@ -796,9 +460,9 @@ static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx, /* Retry the HAL_REO_CMD_UPDATE_RX_QUEUE command for entries * in the pending queue list marked TID as inactive */ - spin_lock_bh(&dp->ab->base_lock); + spin_lock_bh(&dp->dp_lock); ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(dp); - spin_unlock_bh(&dp->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); elem = kzalloc(sizeof(*elem), GFP_ATOMIC); if (!elem) @@ -825,11 +489,12 @@ static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx, * will be called during core destroy. */ - if (ath12k_dp_reo_cache_flush(ab, &elem->data)) + if (ath12k_dp_arch_reo_cache_flush(dp, &elem->data)) break; list_del(&elem->list); dp->reo_cmd_cache_flush_count--; + kfree(elem); } } @@ -839,55 +504,17 @@ static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx, free_desc: ath12k_dp_rx_tid_cleanup(ab, &rx_tid->qbuf); } +EXPORT_SYMBOL(ath12k_dp_rx_tid_del_func); static int ath12k_dp_rx_tid_delete_handler(struct ath12k_base *ab, struct ath12k_dp_rx_tid_rxq *rx_tid) { - struct ath12k_hal_reo_cmd cmd = {}; - - cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS; - cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned); - cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned); - cmd.upd0 |= HAL_REO_CMD_UPD0_VLD; - /* Observed flush cache failure, to avoid that set vld bit during delete */ - cmd.upd1 |= HAL_REO_CMD_UPD1_VLD; - - return ath12k_dp_reo_cmd_send(ab, rx_tid, - HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd, - ath12k_dp_rx_tid_del_func); -} - -static void ath12k_peer_rx_tid_qref_setup(struct ath12k_base *ab, u16 peer_id, u16 tid, - dma_addr_t paddr) -{ - struct ath12k_reo_queue_ref *qref; - struct ath12k_dp *dp = &ab->dp; - bool ml_peer = false; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); - if (!ab->hw_params->reoq_lut_support) - return; - - if (peer_id & ATH12K_PEER_ML_ID_VALID) { - peer_id &= ~ATH12K_PEER_ML_ID_VALID; - ml_peer = true; - } - - if (ml_peer) - qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr + - (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); - else - qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr + - (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); - - qref->info0 = u32_encode_bits(lower_32_bits(paddr), - BUFFER_ADDR_INFO0_ADDR); - qref->info1 = u32_encode_bits(upper_32_bits(paddr), - BUFFER_ADDR_INFO1_ADDR) | - u32_encode_bits(tid, DP_REO_QREF_NUM); - ath12k_hal_reo_shared_qaddr_cache_clear(ab); + return ath12k_dp_arch_rx_tid_delete_handler(dp, rx_tid); } -static void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8 tid) +void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8 tid) { struct dp_reo_update_rx_queue_elem *elem; struct ath12k_dp_rx_tid_rxq *rx_tid; @@ -904,191 +531,43 @@ static void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8 } spin_unlock_bh(&dp->reo_rxq_flush_lock); } +EXPORT_SYMBOL(ath12k_dp_mark_tid_as_inactive); -void ath12k_dp_rx_peer_tid_delete(struct ath12k *ar, - struct ath12k_peer *peer, u8 tid) -{ - struct ath12k_dp_rx_tid *rx_tid = &peer->rx_tid[tid]; - struct ath12k_base *ab = ar->ab; - struct ath12k_dp *dp = &ab->dp; - - if (!rx_tid->active) - return; - - rx_tid->active = false; - - ath12k_dp_mark_tid_as_inactive(dp, peer->peer_id, tid); - ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(dp); -} - -int ath12k_dp_rx_link_desc_return(struct ath12k_base *ab, - struct ath12k_buffer_addr *buf_addr_info, - enum hal_wbm_rel_bm_act action) -{ - struct hal_wbm_release_ring *desc; - struct ath12k_dp *dp = &ab->dp; - struct hal_srng *srng; - int ret = 0; - - srng = &ab->hal.srng_list[dp->wbm_desc_rel_ring.ring_id]; - - spin_lock_bh(&srng->lock); - - ath12k_hal_srng_access_begin(ab, srng); - - desc = ath12k_hal_srng_src_get_next_entry(ab, srng); - if (!desc) { - ret = -ENOBUFS; - goto exit; - } - - ath12k_hal_rx_msdu_link_desc_set(ab, desc, buf_addr_info, action); - -exit: - ath12k_hal_srng_access_end(ab, srng); - - spin_unlock_bh(&srng->lock); - - return ret; -} - -static void ath12k_dp_rx_frags_cleanup(struct ath12k_dp_rx_tid *rx_tid, - bool rel_link_desc) -{ - struct ath12k_buffer_addr *buf_addr_info; - struct ath12k_base *ab = rx_tid->ab; - - lockdep_assert_held(&ab->base_lock); - - if (rx_tid->dst_ring_desc) { - if (rel_link_desc) { - buf_addr_info = &rx_tid->dst_ring_desc->buf_addr_info; - ath12k_dp_rx_link_desc_return(ab, buf_addr_info, - HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); - } - kfree(rx_tid->dst_ring_desc); - rx_tid->dst_ring_desc = NULL; - } - - rx_tid->cur_sn = 0; - rx_tid->last_frag_no = 0; - rx_tid->rx_frag_bitmap = 0; - __skb_queue_purge(&rx_tid->rx_frags); -} - -void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_peer *peer) +void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_dp_link_peer *peer) { struct ath12k_dp_rx_tid *rx_tid; int i; + struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); - lockdep_assert_held(&ar->ab->base_lock); + lockdep_assert_held(&dp->dp_lock); + + if (!peer->primary_link) + return; for (i = 0; i <= IEEE80211_NUM_TIDS; i++) { - rx_tid = &peer->rx_tid[i]; + rx_tid = &peer->dp_peer->rx_tid[i]; - ath12k_dp_rx_peer_tid_delete(ar, peer, i); - ath12k_dp_rx_frags_cleanup(rx_tid, true); + ath12k_dp_arch_rx_peer_tid_delete(dp, peer, i); + ath12k_dp_arch_rx_frags_cleanup(dp, rx_tid, true); - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); timer_delete_sync(&rx_tid->frag_timer); - spin_lock_bh(&ar->ab->base_lock); - } -} - -static int ath12k_peer_rx_tid_reo_update(struct ath12k *ar, - struct ath12k_peer *peer, - struct ath12k_dp_rx_tid *rx_tid, - u32 ba_win_sz, u16 ssn, - bool update_ssn) -{ - struct ath12k_hal_reo_cmd cmd = {}; - int ret; - struct ath12k_dp_rx_tid_rxq rx_tid_rxq; - - ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid); - - cmd.addr_lo = lower_32_bits(rx_tid_rxq.qbuf.paddr_aligned); - cmd.addr_hi = upper_32_bits(rx_tid_rxq.qbuf.paddr_aligned); - cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS; - cmd.upd0 = HAL_REO_CMD_UPD0_BA_WINDOW_SIZE; - cmd.ba_window_size = ba_win_sz; - - if (update_ssn) { - cmd.upd0 |= HAL_REO_CMD_UPD0_SSN; - cmd.upd2 = u32_encode_bits(ssn, HAL_REO_CMD_UPD2_SSN); - } - - ret = ath12k_dp_reo_cmd_send(ar->ab, &rx_tid_rxq, - HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd, - NULL); - if (ret) { - ath12k_warn(ar->ab, "failed to update rx tid queue, tid %d (%d)\n", - rx_tid_rxq.tid, ret); - return ret; - } - - rx_tid->ba_win_sz = ba_win_sz; - - return 0; -} - -static int ath12k_dp_rx_assign_reoq(struct ath12k_base *ab, - struct ath12k_sta *ahsta, - struct ath12k_dp_rx_tid *rx_tid, - u16 ssn, enum hal_pn_type pn_type) -{ - u32 ba_win_sz = rx_tid->ba_win_sz; - struct ath12k_reoq_buf *buf; - void *vaddr, *vaddr_aligned; - dma_addr_t paddr_aligned; - u8 tid = rx_tid->tid; - u32 hw_desc_sz; - int ret; - - buf = &ahsta->reoq_bufs[tid]; - if (!buf->vaddr) { - /* TODO: Optimize the memory allocation for qos tid based on - * the actual BA window size in REO tid update path. - */ - if (tid == HAL_DESC_REO_NON_QOS_TID) - hw_desc_sz = ath12k_hal_reo_qdesc_size(ba_win_sz, tid); - else - hw_desc_sz = ath12k_hal_reo_qdesc_size(DP_BA_WIN_SZ_MAX, tid); - - vaddr = kzalloc(hw_desc_sz + HAL_LINK_DESC_ALIGN - 1, GFP_ATOMIC); - if (!vaddr) - return -ENOMEM; - - vaddr_aligned = PTR_ALIGN(vaddr, HAL_LINK_DESC_ALIGN); - - ath12k_hal_reo_qdesc_setup(vaddr_aligned, tid, ba_win_sz, - ssn, pn_type); - - paddr_aligned = dma_map_single(ab->dev, vaddr_aligned, hw_desc_sz, - DMA_BIDIRECTIONAL); - ret = dma_mapping_error(ab->dev, paddr_aligned); - if (ret) { - kfree(vaddr); - return ret; - } - - buf->vaddr = vaddr; - buf->paddr_aligned = paddr_aligned; - buf->size = hw_desc_sz; + spin_lock_bh(&dp->dp_lock); } - - rx_tid->qbuf = *buf; - rx_tid->active = true; - - return 0; } static int ath12k_dp_prepare_reo_update_elem(struct ath12k_dp *dp, - struct ath12k_peer *peer, + struct ath12k_dp_link_peer *peer, struct ath12k_dp_rx_tid *rx_tid) { struct dp_reo_update_rx_queue_elem *elem; + lockdep_assert_held(&dp->dp_lock); + + if (!peer->primary_link) + return 0; + elem = kzalloc(sizeof(*elem), GFP_ATOMIC); if (!elem) return -ENOMEM; @@ -1097,7 +576,8 @@ static int ath12k_dp_prepare_reo_update_elem(struct ath12k_dp *dp, elem->is_ml_peer = peer->mlo; elem->ml_peer_id = peer->ml_id; - ath12k_dp_init_rx_tid_rxq(&elem->rx_tid, rx_tid); + ath12k_dp_init_rx_tid_rxq(&elem->rx_tid, rx_tid, + (peer->rx_tid_active_bitmask & (1 << rx_tid->tid))); spin_lock_bh(&dp->reo_rxq_flush_lock); list_add_tail(&elem->list, &dp->reo_cmd_update_rx_queue_list); @@ -1111,31 +591,30 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_ enum hal_pn_type pn_type) { struct ath12k_base *ab = ar->ab; - struct ath12k_dp *dp = &ab->dp; - struct ath12k_peer *peer; - struct ath12k_sta *ahsta; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_dp_link_peer *peer; struct ath12k_dp_rx_tid *rx_tid; dma_addr_t paddr_aligned; int ret; - spin_lock_bh(&ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ab, vdev_id, peer_mac); - if (!peer) { - spin_unlock_bh(&ab->base_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, peer_mac); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ab, "failed to find the peer to set up rx tid\n"); return -ENOENT; } if (ab->hw_params->dp_primary_link_only && !peer->primary_link) { - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return 0; } if (ab->hw_params->reoq_lut_support && (!dp->reoq_lut.vaddr || !dp->ml_reoq_lut.vaddr)) { - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ab, "reo qref table is not setup\n"); return -EINVAL; } @@ -1143,16 +622,16 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_ if (peer->peer_id > DP_MAX_PEER_ID || tid > IEEE80211_NUM_TIDS) { ath12k_warn(ab, "peer id of peer %d or tid %d doesn't allow reoq setup\n", peer->peer_id, tid); - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return -EINVAL; } - rx_tid = &peer->rx_tid[tid]; + rx_tid = &peer->dp_peer->rx_tid[tid]; /* Update the tid queue if it is already setup */ - if (rx_tid->active) { - ret = ath12k_peer_rx_tid_reo_update(ar, peer, rx_tid, - ba_win_sz, ssn, true); - spin_unlock_bh(&ab->base_lock); + if (peer->rx_tid_active_bitmask & (1 << tid)) { + ret = ath12k_dp_arch_peer_rx_tid_reo_update(dp, peer, rx_tid, + ba_win_sz, ssn, true); + spin_unlock_bh(&dp->dp_lock); if (ret) { ath12k_warn(ab, "failed to update reo for rx tid %d\n", tid); return ret; @@ -1178,14 +657,15 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_ rx_tid->ba_win_sz = ba_win_sz; - ahsta = ath12k_sta_to_ahsta(peer->sta); - ret = ath12k_dp_rx_assign_reoq(ab, ahsta, rx_tid, ssn, pn_type); + ret = ath12k_dp_arch_rx_assign_reoq(dp, peer->dp_peer, rx_tid, ssn, pn_type); if (ret) { - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ab, "failed to assign reoq buf for rx tid %u\n", tid); return ret; } + peer->rx_tid_active_bitmask |= (1 << tid); + /* Pre-allocate the update_rxq_list for the corresponding tid * This will be used during the tid delete. The reason we are not * allocating during tid delete is that, if any alloc fail in update_rxq_list @@ -1195,7 +675,7 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_ if (ret) { ath12k_warn(ab, "failed to alloc update_rxq_list for rx tid %u\n", tid); ath12k_dp_rx_tid_cleanup(ab, &rx_tid->qbuf); - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return ret; } @@ -1205,15 +685,15 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_ * and tid with qaddr. */ if (peer->mlo) - ath12k_peer_rx_tid_qref_setup(ab, peer->ml_id, tid, - paddr_aligned); + ath12k_dp_arch_peer_rx_tid_qref_setup(dp, peer->ml_id, tid, + paddr_aligned); else - ath12k_peer_rx_tid_qref_setup(ab, peer->peer_id, tid, - paddr_aligned); + ath12k_dp_arch_peer_rx_tid_qref_setup(dp, peer->peer_id, tid, + paddr_aligned); - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); } else { - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); ret = ath12k_wmi_peer_rx_reorder_queue_setup(ar, vdev_id, peer_mac, paddr_aligned, tid, 1, ba_win_sz); @@ -1255,7 +735,8 @@ int ath12k_dp_rx_ampdu_stop(struct ath12k *ar, u8 link_id) { struct ath12k_base *ab = ar->ab; - struct ath12k_peer *peer; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_dp_link_peer *peer; struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(params->sta); struct ath12k_link_sta *arsta; int vdev_id; @@ -1271,24 +752,30 @@ int ath12k_dp_rx_ampdu_stop(struct ath12k *ar, vdev_id = arsta->arvif->vdev_id; - spin_lock_bh(&ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ab, vdev_id, arsta->addr); - if (!peer) { - spin_unlock_bh(&ab->base_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, arsta->addr); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ab, "failed to find the peer to stop rx aggregation\n"); return -ENOENT; } - active = peer->rx_tid[params->tid].active; - - if (!active) { - spin_unlock_bh(&ab->base_lock); + if (ab->hw_params->dp_primary_link_only && + !peer->primary_link) { + spin_unlock_bh(&dp->dp_lock); + return 0; + } + + active = peer->rx_tid_active_bitmask & (1 << params->tid); + if (!active) { + spin_unlock_bh(&dp->dp_lock); return 0; } - ret = ath12k_peer_rx_tid_reo_update(ar, peer, peer->rx_tid, 1, 0, false); - spin_unlock_bh(&ab->base_lock); + ret = ath12k_dp_arch_peer_rx_tid_reo_update(dp, peer, peer->dp_peer->rx_tid, + 1, 0, false); + spin_unlock_bh(&dp->dp_lock); if (ret) { ath12k_warn(ab, "failed to update reo for rx tid %d: %d\n", params->tid, ret); @@ -1305,8 +792,9 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif, { struct ath12k *ar = arvif->ar; struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct ath12k_hal_reo_cmd cmd = {}; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; struct ath12k_dp_rx_tid *rx_tid; struct ath12k_dp_rx_tid_rxq rx_tid_rxq; u8 tid; @@ -1319,49 +807,29 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif, if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) return 0; - cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS; - cmd.upd0 = HAL_REO_CMD_UPD0_PN | - HAL_REO_CMD_UPD0_PN_SIZE | - HAL_REO_CMD_UPD0_PN_VALID | - HAL_REO_CMD_UPD0_PN_CHECK | - HAL_REO_CMD_UPD0_SVLD; - - switch (key->cipher) { - case WLAN_CIPHER_SUITE_TKIP: - case WLAN_CIPHER_SUITE_CCMP: - case WLAN_CIPHER_SUITE_CCMP_256: - case WLAN_CIPHER_SUITE_GCMP: - case WLAN_CIPHER_SUITE_GCMP_256: - if (key_cmd == SET_KEY) { - cmd.upd1 |= HAL_REO_CMD_UPD1_PN_CHECK; - cmd.pn_size = 48; - } - break; - default: - break; - } - - spin_lock_bh(&ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ab, arvif->vdev_id, peer_addr); - if (!peer) { - spin_unlock_bh(&ab->base_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + peer_addr); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ab, "failed to find the peer %pM to configure pn replay detection\n", peer_addr); return -ENOENT; } for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) { - rx_tid = &peer->rx_tid[tid]; - if (!rx_tid->active) + if (!(peer->rx_tid_active_bitmask & (1 << tid))) continue; - ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid); - cmd.addr_lo = lower_32_bits(rx_tid_rxq.qbuf.paddr_aligned); - cmd.addr_hi = upper_32_bits(rx_tid_rxq.qbuf.paddr_aligned); - ret = ath12k_dp_reo_cmd_send(ab, &rx_tid_rxq, - HAL_REO_CMD_UPDATE_RX_QUEUE, - &cmd, NULL); + rx_tid = &peer->dp_peer->rx_tid[tid]; + ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid, + (peer->rx_tid_active_bitmask & (1 << tid))); + ath12k_dp_arch_setup_pn_check_reo_cmd(dp, &cmd, rx_tid, key->cipher, + key_cmd); + ret = ath12k_dp_arch_reo_cmd_send(dp, &rx_tid_rxq, + HAL_REO_CMD_UPDATE_RX_QUEUE, + &cmd, NULL); if (ret) { ath12k_warn(ab, "failed to configure rx tid %d queue of peer %pM for pn replay detection %d\n", tid, peer_addr, ret); @@ -1369,732 +837,14 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif, } } - spin_unlock_bh(&ab->base_lock); - - return ret; -} - -static int ath12k_get_ppdu_user_index(struct htt_ppdu_stats *ppdu_stats, - u16 peer_id) -{ - int i; - - for (i = 0; i < HTT_PPDU_STATS_MAX_USERS - 1; i++) { - if (ppdu_stats->user_stats[i].is_valid_peer_id) { - if (peer_id == ppdu_stats->user_stats[i].peer_id) - return i; - } else { - return i; - } - } - - return -EINVAL; -} - -static int ath12k_htt_tlv_ppdu_stats_parse(struct ath12k_base *ab, - u16 tag, u16 len, const void *ptr, - void *data) -{ - const struct htt_ppdu_stats_usr_cmpltn_ack_ba_status *ba_status; - const struct htt_ppdu_stats_usr_cmpltn_cmn *cmplt_cmn; - const struct htt_ppdu_stats_user_rate *user_rate; - struct htt_ppdu_stats_info *ppdu_info; - struct htt_ppdu_user_stats *user_stats; - int cur_user; - u16 peer_id; - - ppdu_info = data; - - switch (tag) { - case HTT_PPDU_STATS_TAG_COMMON: - if (len < sizeof(struct htt_ppdu_stats_common)) { - ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", - len, tag); - return -EINVAL; - } - memcpy(&ppdu_info->ppdu_stats.common, ptr, - sizeof(struct htt_ppdu_stats_common)); - break; - case HTT_PPDU_STATS_TAG_USR_RATE: - if (len < sizeof(struct htt_ppdu_stats_user_rate)) { - ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", - len, tag); - return -EINVAL; - } - user_rate = ptr; - peer_id = le16_to_cpu(user_rate->sw_peer_id); - cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats, - peer_id); - if (cur_user < 0) - return -EINVAL; - user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user]; - user_stats->peer_id = peer_id; - user_stats->is_valid_peer_id = true; - memcpy(&user_stats->rate, ptr, - sizeof(struct htt_ppdu_stats_user_rate)); - user_stats->tlv_flags |= BIT(tag); - break; - case HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON: - if (len < sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn)) { - ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", - len, tag); - return -EINVAL; - } - - cmplt_cmn = ptr; - peer_id = le16_to_cpu(cmplt_cmn->sw_peer_id); - cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats, - peer_id); - if (cur_user < 0) - return -EINVAL; - user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user]; - user_stats->peer_id = peer_id; - user_stats->is_valid_peer_id = true; - memcpy(&user_stats->cmpltn_cmn, ptr, - sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn)); - user_stats->tlv_flags |= BIT(tag); - break; - case HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS: - if (len < - sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status)) { - ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n", - len, tag); - return -EINVAL; - } - - ba_status = ptr; - peer_id = le16_to_cpu(ba_status->sw_peer_id); - cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats, - peer_id); - if (cur_user < 0) - return -EINVAL; - user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user]; - user_stats->peer_id = peer_id; - user_stats->is_valid_peer_id = true; - memcpy(&user_stats->ack_ba, ptr, - sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status)); - user_stats->tlv_flags |= BIT(tag); - break; - } - return 0; -} - -int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len, - int (*iter)(struct ath12k_base *ar, u16 tag, u16 len, - const void *ptr, void *data), - void *data) -{ - const struct htt_tlv *tlv; - const void *begin = ptr; - u16 tlv_tag, tlv_len; - int ret = -EINVAL; - - while (len > 0) { - if (len < sizeof(*tlv)) { - ath12k_err(ab, "htt tlv parse failure at byte %zd (%zu bytes left, %zu expected)\n", - ptr - begin, len, sizeof(*tlv)); - return -EINVAL; - } - tlv = (struct htt_tlv *)ptr; - tlv_tag = le32_get_bits(tlv->header, HTT_TLV_TAG); - tlv_len = le32_get_bits(tlv->header, HTT_TLV_LEN); - ptr += sizeof(*tlv); - len -= sizeof(*tlv); - - if (tlv_len > len) { - ath12k_err(ab, "htt tlv parse failure of tag %u at byte %zd (%zu bytes left, %u expected)\n", - tlv_tag, ptr - begin, len, tlv_len); - return -EINVAL; - } - ret = iter(ab, tlv_tag, tlv_len, ptr, data); - if (ret == -ENOMEM) - return ret; - - ptr += tlv_len; - len -= tlv_len; - } - return 0; -} - -static void -ath12k_update_per_peer_tx_stats(struct ath12k *ar, - struct htt_ppdu_stats *ppdu_stats, u8 user) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_peer *peer; - struct ath12k_link_sta *arsta; - struct htt_ppdu_stats_user_rate *user_rate; - struct ath12k_per_peer_tx_stats *peer_stats = &ar->peer_tx_stats; - struct htt_ppdu_user_stats *usr_stats = &ppdu_stats->user_stats[user]; - struct htt_ppdu_stats_common *common = &ppdu_stats->common; - int ret; - u8 flags, mcs, nss, bw, sgi, dcm, ppdu_type, rate_idx = 0; - u32 v, succ_bytes = 0; - u16 tones, rate = 0, succ_pkts = 0; - u32 tx_duration = 0; - u8 tid = HTT_PPDU_STATS_NON_QOS_TID; - u16 tx_retry_failed = 0, tx_retry_count = 0; - bool is_ampdu = false, is_ofdma; - - if (!(usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_RATE))) - return; - - if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) { - is_ampdu = - HTT_USR_CMPLTN_IS_AMPDU(usr_stats->cmpltn_cmn.flags); - tx_retry_failed = - __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) - - __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success); - tx_retry_count = - HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) + - HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags); - } - - if (usr_stats->tlv_flags & - BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS)) { - succ_bytes = le32_to_cpu(usr_stats->ack_ba.success_bytes); - succ_pkts = le32_get_bits(usr_stats->ack_ba.info, - HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M); - tid = le32_get_bits(usr_stats->ack_ba.info, - HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM); - } - - if (common->fes_duration_us) - tx_duration = le32_to_cpu(common->fes_duration_us); - - user_rate = &usr_stats->rate; - flags = HTT_USR_RATE_PREAMBLE(user_rate->rate_flags); - bw = HTT_USR_RATE_BW(user_rate->rate_flags) - 2; - nss = HTT_USR_RATE_NSS(user_rate->rate_flags) + 1; - mcs = HTT_USR_RATE_MCS(user_rate->rate_flags); - sgi = HTT_USR_RATE_GI(user_rate->rate_flags); - dcm = HTT_USR_RATE_DCM(user_rate->rate_flags); - - ppdu_type = HTT_USR_RATE_PPDU_TYPE(user_rate->info1); - is_ofdma = (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA) || - (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA); - - /* Note: If host configured fixed rates and in some other special - * cases, the broadcast/management frames are sent in different rates. - * Firmware rate's control to be skipped for this? - */ - - if (flags == WMI_RATE_PREAMBLE_HE && mcs > ATH12K_HE_MCS_MAX) { - ath12k_warn(ab, "Invalid HE mcs %d peer stats", mcs); - return; - } - - if (flags == WMI_RATE_PREAMBLE_VHT && mcs > ATH12K_VHT_MCS_MAX) { - ath12k_warn(ab, "Invalid VHT mcs %d peer stats", mcs); - return; - } - - if (flags == WMI_RATE_PREAMBLE_HT && (mcs > ATH12K_HT_MCS_MAX || nss < 1)) { - ath12k_warn(ab, "Invalid HT mcs %d nss %d peer stats", - mcs, nss); - return; - } - - if (flags == WMI_RATE_PREAMBLE_CCK || flags == WMI_RATE_PREAMBLE_OFDM) { - ret = ath12k_mac_hw_ratecode_to_legacy_rate(mcs, - flags, - &rate_idx, - &rate); - if (ret < 0) - return; - } - - rcu_read_lock(); - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, usr_stats->peer_id); - - if (!peer || !peer->sta) { - spin_unlock_bh(&ab->base_lock); - rcu_read_unlock(); - return; - } - - arsta = ath12k_peer_get_link_sta(ab, peer); - if (!arsta) { - spin_unlock_bh(&ab->base_lock); - rcu_read_unlock(); - return; - } - - memset(&arsta->txrate, 0, sizeof(arsta->txrate)); - - arsta->txrate.bw = ath12k_mac_bw_to_mac80211_bw(bw); - - switch (flags) { - case WMI_RATE_PREAMBLE_OFDM: - arsta->txrate.legacy = rate; - break; - case WMI_RATE_PREAMBLE_CCK: - arsta->txrate.legacy = rate; - break; - case WMI_RATE_PREAMBLE_HT: - arsta->txrate.mcs = mcs + 8 * (nss - 1); - arsta->txrate.flags = RATE_INFO_FLAGS_MCS; - if (sgi) - arsta->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; - break; - case WMI_RATE_PREAMBLE_VHT: - arsta->txrate.mcs = mcs; - arsta->txrate.flags = RATE_INFO_FLAGS_VHT_MCS; - if (sgi) - arsta->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; - break; - case WMI_RATE_PREAMBLE_HE: - arsta->txrate.mcs = mcs; - arsta->txrate.flags = RATE_INFO_FLAGS_HE_MCS; - arsta->txrate.he_dcm = dcm; - arsta->txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(sgi); - tones = le16_to_cpu(user_rate->ru_end) - - le16_to_cpu(user_rate->ru_start) + 1; - v = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(tones); - arsta->txrate.he_ru_alloc = v; - if (is_ofdma) - arsta->txrate.bw = RATE_INFO_BW_HE_RU; - break; - case WMI_RATE_PREAMBLE_EHT: - arsta->txrate.mcs = mcs; - arsta->txrate.flags = RATE_INFO_FLAGS_EHT_MCS; - arsta->txrate.he_dcm = dcm; - arsta->txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(sgi); - tones = le16_to_cpu(user_rate->ru_end) - - le16_to_cpu(user_rate->ru_start) + 1; - v = ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(tones); - arsta->txrate.eht_ru_alloc = v; - if (is_ofdma) - arsta->txrate.bw = RATE_INFO_BW_EHT_RU; - break; - } - - arsta->tx_retry_failed += tx_retry_failed; - arsta->tx_retry_count += tx_retry_count; - arsta->txrate.nss = nss; - arsta->tx_duration += tx_duration; - memcpy(&arsta->last_txrate, &arsta->txrate, sizeof(struct rate_info)); - - /* PPDU stats reported for mgmt packet doesn't have valid tx bytes. - * So skip peer stats update for mgmt packets. - */ - if (tid < HTT_PPDU_STATS_NON_QOS_TID) { - memset(peer_stats, 0, sizeof(*peer_stats)); - peer_stats->succ_pkts = succ_pkts; - peer_stats->succ_bytes = succ_bytes; - peer_stats->is_ampdu = is_ampdu; - peer_stats->duration = tx_duration; - peer_stats->ba_fails = - HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) + - HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags); - } - - spin_unlock_bh(&ab->base_lock); - rcu_read_unlock(); -} - -static void ath12k_htt_update_ppdu_stats(struct ath12k *ar, - struct htt_ppdu_stats *ppdu_stats) -{ - u8 user; - - for (user = 0; user < HTT_PPDU_STATS_MAX_USERS - 1; user++) - ath12k_update_per_peer_tx_stats(ar, ppdu_stats, user); -} - -static -struct htt_ppdu_stats_info *ath12k_dp_htt_get_ppdu_desc(struct ath12k *ar, - u32 ppdu_id) -{ - struct htt_ppdu_stats_info *ppdu_info; - - lockdep_assert_held(&ar->data_lock); - if (!list_empty(&ar->ppdu_stats_info)) { - list_for_each_entry(ppdu_info, &ar->ppdu_stats_info, list) { - if (ppdu_info->ppdu_id == ppdu_id) - return ppdu_info; - } - - if (ar->ppdu_stat_list_depth > HTT_PPDU_DESC_MAX_DEPTH) { - ppdu_info = list_first_entry(&ar->ppdu_stats_info, - typeof(*ppdu_info), list); - list_del(&ppdu_info->list); - ar->ppdu_stat_list_depth--; - ath12k_htt_update_ppdu_stats(ar, &ppdu_info->ppdu_stats); - kfree(ppdu_info); - } - } - - ppdu_info = kzalloc(sizeof(*ppdu_info), GFP_ATOMIC); - if (!ppdu_info) - return NULL; - - list_add_tail(&ppdu_info->list, &ar->ppdu_stats_info); - ar->ppdu_stat_list_depth++; - - return ppdu_info; -} - -static void ath12k_copy_to_delay_stats(struct ath12k_peer *peer, - struct htt_ppdu_user_stats *usr_stats) -{ - peer->ppdu_stats_delayba.sw_peer_id = le16_to_cpu(usr_stats->rate.sw_peer_id); - peer->ppdu_stats_delayba.info0 = le32_to_cpu(usr_stats->rate.info0); - peer->ppdu_stats_delayba.ru_end = le16_to_cpu(usr_stats->rate.ru_end); - peer->ppdu_stats_delayba.ru_start = le16_to_cpu(usr_stats->rate.ru_start); - peer->ppdu_stats_delayba.info1 = le32_to_cpu(usr_stats->rate.info1); - peer->ppdu_stats_delayba.rate_flags = le32_to_cpu(usr_stats->rate.rate_flags); - peer->ppdu_stats_delayba.resp_rate_flags = - le32_to_cpu(usr_stats->rate.resp_rate_flags); - - peer->delayba_flag = true; -} - -static void ath12k_copy_to_bar(struct ath12k_peer *peer, - struct htt_ppdu_user_stats *usr_stats) -{ - usr_stats->rate.sw_peer_id = cpu_to_le16(peer->ppdu_stats_delayba.sw_peer_id); - usr_stats->rate.info0 = cpu_to_le32(peer->ppdu_stats_delayba.info0); - usr_stats->rate.ru_end = cpu_to_le16(peer->ppdu_stats_delayba.ru_end); - usr_stats->rate.ru_start = cpu_to_le16(peer->ppdu_stats_delayba.ru_start); - usr_stats->rate.info1 = cpu_to_le32(peer->ppdu_stats_delayba.info1); - usr_stats->rate.rate_flags = cpu_to_le32(peer->ppdu_stats_delayba.rate_flags); - usr_stats->rate.resp_rate_flags = - cpu_to_le32(peer->ppdu_stats_delayba.resp_rate_flags); - - peer->delayba_flag = false; -} - -static int ath12k_htt_pull_ppdu_stats(struct ath12k_base *ab, - struct sk_buff *skb) -{ - struct ath12k_htt_ppdu_stats_msg *msg; - struct htt_ppdu_stats_info *ppdu_info; - struct ath12k_peer *peer = NULL; - struct htt_ppdu_user_stats *usr_stats = NULL; - u32 peer_id = 0; - struct ath12k *ar; - int ret, i; - u8 pdev_id; - u32 ppdu_id, len; - - msg = (struct ath12k_htt_ppdu_stats_msg *)skb->data; - len = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE); - if (len > (skb->len - struct_size(msg, data, 0))) { - ath12k_warn(ab, - "HTT PPDU STATS event has unexpected payload size %u, should be smaller than %u\n", - len, skb->len); - return -EINVAL; - } - - pdev_id = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PDEV_ID); - ppdu_id = le32_to_cpu(msg->ppdu_id); - - rcu_read_lock(); - ar = ath12k_mac_get_ar_by_pdev_id(ab, pdev_id); - if (!ar) { - ret = -EINVAL; - goto exit; - } - - spin_lock_bh(&ar->data_lock); - ppdu_info = ath12k_dp_htt_get_ppdu_desc(ar, ppdu_id); - if (!ppdu_info) { - spin_unlock_bh(&ar->data_lock); - ret = -EINVAL; - goto exit; - } - - ppdu_info->ppdu_id = ppdu_id; - ret = ath12k_dp_htt_tlv_iter(ab, msg->data, len, - ath12k_htt_tlv_ppdu_stats_parse, - (void *)ppdu_info); - if (ret) { - spin_unlock_bh(&ar->data_lock); - ath12k_warn(ab, "Failed to parse tlv %d\n", ret); - goto exit; - } - - if (ppdu_info->ppdu_stats.common.num_users >= HTT_PPDU_STATS_MAX_USERS) { - spin_unlock_bh(&ar->data_lock); - ath12k_warn(ab, - "HTT PPDU STATS event has unexpected num_users %u, should be smaller than %u\n", - ppdu_info->ppdu_stats.common.num_users, - HTT_PPDU_STATS_MAX_USERS); - ret = -EINVAL; - goto exit; - } - - /* back up data rate tlv for all peers */ - if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_DATA && - (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON)) && - ppdu_info->delay_ba) { - for (i = 0; i < ppdu_info->ppdu_stats.common.num_users; i++) { - peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id; - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, peer_id); - if (!peer) { - spin_unlock_bh(&ab->base_lock); - continue; - } - - usr_stats = &ppdu_info->ppdu_stats.user_stats[i]; - if (usr_stats->delay_ba) - ath12k_copy_to_delay_stats(peer, usr_stats); - spin_unlock_bh(&ab->base_lock); - } - } - - /* restore all peers' data rate tlv to mu-bar tlv */ - if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_BAR && - (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON))) { - for (i = 0; i < ppdu_info->bar_num_users; i++) { - peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id; - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, peer_id); - if (!peer) { - spin_unlock_bh(&ab->base_lock); - continue; - } - - usr_stats = &ppdu_info->ppdu_stats.user_stats[i]; - if (peer->delayba_flag) - ath12k_copy_to_bar(peer, usr_stats); - spin_unlock_bh(&ab->base_lock); - } - } - - spin_unlock_bh(&ar->data_lock); - -exit: - rcu_read_unlock(); + spin_unlock_bh(&dp->dp_lock); return ret; } +EXPORT_SYMBOL(ath12k_dp_rx_get_msdu_last_buf); -static void ath12k_htt_mlo_offset_event_handler(struct ath12k_base *ab, - struct sk_buff *skb) -{ - struct ath12k_htt_mlo_offset_msg *msg; - struct ath12k_pdev *pdev; - struct ath12k *ar; - u8 pdev_id; - - msg = (struct ath12k_htt_mlo_offset_msg *)skb->data; - pdev_id = u32_get_bits(__le32_to_cpu(msg->info), - HTT_T2H_MLO_OFFSET_INFO_PDEV_ID); - - rcu_read_lock(); - ar = ath12k_mac_get_ar_by_pdev_id(ab, pdev_id); - if (!ar) { - /* It is possible that the ar is not yet active (started). - * The above function will only look for the active pdev - * and hence %NULL return is possible. Just silently - * discard this message - */ - goto exit; - } - - spin_lock_bh(&ar->data_lock); - pdev = ar->pdev; - - pdev->timestamp.info = __le32_to_cpu(msg->info); - pdev->timestamp.sync_timestamp_lo_us = __le32_to_cpu(msg->sync_timestamp_lo_us); - pdev->timestamp.sync_timestamp_hi_us = __le32_to_cpu(msg->sync_timestamp_hi_us); - pdev->timestamp.mlo_offset_lo = __le32_to_cpu(msg->mlo_offset_lo); - pdev->timestamp.mlo_offset_hi = __le32_to_cpu(msg->mlo_offset_hi); - pdev->timestamp.mlo_offset_clks = __le32_to_cpu(msg->mlo_offset_clks); - pdev->timestamp.mlo_comp_clks = __le32_to_cpu(msg->mlo_comp_clks); - pdev->timestamp.mlo_comp_timer = __le32_to_cpu(msg->mlo_comp_timer); - - spin_unlock_bh(&ar->data_lock); -exit: - rcu_read_unlock(); -} - -void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab, - struct sk_buff *skb) -{ - struct ath12k_dp *dp = &ab->dp; - struct htt_resp_msg *resp = (struct htt_resp_msg *)skb->data; - enum htt_t2h_msg_type type; - u16 peer_id; - u8 vdev_id; - u8 mac_addr[ETH_ALEN]; - u16 peer_mac_h16; - u16 ast_hash = 0; - u16 hw_peer_id; - - type = le32_get_bits(resp->version_msg.version, HTT_T2H_MSG_TYPE); - - ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt rx msg type :0x%0x\n", type); - - switch (type) { - case HTT_T2H_MSG_TYPE_VERSION_CONF: - dp->htt_tgt_ver_major = le32_get_bits(resp->version_msg.version, - HTT_T2H_VERSION_CONF_MAJOR); - dp->htt_tgt_ver_minor = le32_get_bits(resp->version_msg.version, - HTT_T2H_VERSION_CONF_MINOR); - complete(&dp->htt_tgt_version_received); - break; - /* TODO: remove unused peer map versions after testing */ - case HTT_T2H_MSG_TYPE_PEER_MAP: - vdev_id = le32_get_bits(resp->peer_map_ev.info, - HTT_T2H_PEER_MAP_INFO_VDEV_ID); - peer_id = le32_get_bits(resp->peer_map_ev.info, - HTT_T2H_PEER_MAP_INFO_PEER_ID); - peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1, - HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16); - ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32), - peer_mac_h16, mac_addr); - ath12k_peer_map_event(ab, vdev_id, peer_id, mac_addr, 0, 0); - break; - case HTT_T2H_MSG_TYPE_PEER_MAP2: - vdev_id = le32_get_bits(resp->peer_map_ev.info, - HTT_T2H_PEER_MAP_INFO_VDEV_ID); - peer_id = le32_get_bits(resp->peer_map_ev.info, - HTT_T2H_PEER_MAP_INFO_PEER_ID); - peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1, - HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16); - ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32), - peer_mac_h16, mac_addr); - ast_hash = le32_get_bits(resp->peer_map_ev.info2, - HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL); - hw_peer_id = le32_get_bits(resp->peer_map_ev.info1, - HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID); - ath12k_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash, - hw_peer_id); - break; - case HTT_T2H_MSG_TYPE_PEER_MAP3: - vdev_id = le32_get_bits(resp->peer_map_ev.info, - HTT_T2H_PEER_MAP_INFO_VDEV_ID); - peer_id = le32_get_bits(resp->peer_map_ev.info, - HTT_T2H_PEER_MAP_INFO_PEER_ID); - peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1, - HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16); - ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32), - peer_mac_h16, mac_addr); - ast_hash = le32_get_bits(resp->peer_map_ev.info2, - HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL); - hw_peer_id = le32_get_bits(resp->peer_map_ev.info2, - HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID); - ath12k_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash, - hw_peer_id); - break; - case HTT_T2H_MSG_TYPE_PEER_UNMAP: - case HTT_T2H_MSG_TYPE_PEER_UNMAP2: - peer_id = le32_get_bits(resp->peer_unmap_ev.info, - HTT_T2H_PEER_UNMAP_INFO_PEER_ID); - ath12k_peer_unmap_event(ab, peer_id); - break; - case HTT_T2H_MSG_TYPE_PPDU_STATS_IND: - ath12k_htt_pull_ppdu_stats(ab, skb); - break; - case HTT_T2H_MSG_TYPE_EXT_STATS_CONF: - ath12k_debugfs_htt_ext_stats_handler(ab, skb); - break; - case HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND: - ath12k_htt_mlo_offset_event_handler(ab, skb); - break; - default: - ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt event %d not handled\n", - type); - break; - } - - dev_kfree_skb_any(skb); -} - -static int ath12k_dp_rx_msdu_coalesce(struct ath12k *ar, - struct sk_buff_head *msdu_list, - struct sk_buff *first, struct sk_buff *last, - u8 l3pad_bytes, int msdu_len) -{ - struct ath12k_base *ab = ar->ab; - struct sk_buff *skb; - struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(first); - int buf_first_hdr_len, buf_first_len; - struct hal_rx_desc *ldesc; - int space_extra, rem_len, buf_len; - u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; - bool is_continuation; - - /* As the msdu is spread across multiple rx buffers, - * find the offset to the start of msdu for computing - * the length of the msdu in the first buffer. - */ - buf_first_hdr_len = hal_rx_desc_sz + l3pad_bytes; - buf_first_len = DP_RX_BUFFER_SIZE - buf_first_hdr_len; - - if (WARN_ON_ONCE(msdu_len <= buf_first_len)) { - skb_put(first, buf_first_hdr_len + msdu_len); - skb_pull(first, buf_first_hdr_len); - return 0; - } - - ldesc = (struct hal_rx_desc *)last->data; - rxcb->is_first_msdu = ath12k_dp_rx_h_first_msdu(ab, ldesc); - rxcb->is_last_msdu = ath12k_dp_rx_h_last_msdu(ab, ldesc); - - /* MSDU spans over multiple buffers because the length of the MSDU - * exceeds DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE. So assume the data - * in the first buf is of length DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE. - */ - skb_put(first, DP_RX_BUFFER_SIZE); - skb_pull(first, buf_first_hdr_len); - - /* When an MSDU spread over multiple buffers MSDU_END - * tlvs are valid only in the last buffer. Copy those tlvs. - */ - ath12k_dp_rx_desc_end_tlv_copy(ab, rxcb->rx_desc, ldesc); - - space_extra = msdu_len - (buf_first_len + skb_tailroom(first)); - if (space_extra > 0 && - (pskb_expand_head(first, 0, space_extra, GFP_ATOMIC) < 0)) { - /* Free up all buffers of the MSDU */ - while ((skb = __skb_dequeue(msdu_list)) != NULL) { - rxcb = ATH12K_SKB_RXCB(skb); - if (!rxcb->is_continuation) { - dev_kfree_skb_any(skb); - break; - } - dev_kfree_skb_any(skb); - } - return -ENOMEM; - } - - rem_len = msdu_len - buf_first_len; - while ((skb = __skb_dequeue(msdu_list)) != NULL && rem_len > 0) { - rxcb = ATH12K_SKB_RXCB(skb); - is_continuation = rxcb->is_continuation; - if (is_continuation) - buf_len = DP_RX_BUFFER_SIZE - hal_rx_desc_sz; - else - buf_len = rem_len; - - if (buf_len > (DP_RX_BUFFER_SIZE - hal_rx_desc_sz)) { - WARN_ON_ONCE(1); - dev_kfree_skb_any(skb); - return -EINVAL; - } - - skb_put(skb, buf_len + hal_rx_desc_sz); - skb_pull(skb, hal_rx_desc_sz); - skb_copy_from_linear_data(skb, skb_put(first, buf_len), - buf_len); - dev_kfree_skb_any(skb); - - rem_len -= buf_len; - if (!is_continuation) - break; - } - - return 0; -} - -static struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_list, - struct sk_buff *first) +struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_list, + struct sk_buff *first) { struct sk_buff *skb; struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(first); @@ -2111,14 +861,7 @@ static struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_ return NULL; } -static void ath12k_dp_rx_h_csum_offload(struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info) -{ - msdu->ip_summed = (rx_info->ip_csum_fail || rx_info->l4_csum_fail) ? - CHECKSUM_NONE : CHECKSUM_UNNECESSARY; -} - -int ath12k_dp_rx_crypto_mic_len(struct ath12k *ar, enum hal_encrypt_type enctype) +int ath12k_dp_rx_crypto_mic_len(struct ath12k_dp *dp, enum hal_encrypt_type enctype) { switch (enctype) { case HAL_ENCRYPT_TYPE_OPEN: @@ -2140,11 +883,11 @@ int ath12k_dp_rx_crypto_mic_len(struct ath12k *ar, enum hal_encrypt_type enctype break; } - ath12k_warn(ar->ab, "unsupported encryption type %d for mic len\n", enctype); + ath12k_warn(dp->ab, "unsupported encryption type %d for mic len\n", enctype); return 0; } -static int ath12k_dp_rx_crypto_param_len(struct ath12k *ar, +static int ath12k_dp_rx_crypto_param_len(struct ath12k_pdev_dp *dp_pdev, enum hal_encrypt_type enctype) { switch (enctype) { @@ -2168,11 +911,11 @@ static int ath12k_dp_rx_crypto_param_len(struct ath12k *ar, break; } - ath12k_warn(ar->ab, "unsupported encryption type %d\n", enctype); + ath12k_warn(dp_pdev->dp->ab, "unsupported encryption type %d\n", enctype); return 0; } -static int ath12k_dp_rx_crypto_icv_len(struct ath12k *ar, +static int ath12k_dp_rx_crypto_icv_len(struct ath12k_pdev_dp *dp_pdev, enum hal_encrypt_type enctype) { switch (enctype) { @@ -2193,16 +936,15 @@ static int ath12k_dp_rx_crypto_icv_len(struct ath12k *ar, break; } - ath12k_warn(ar->ab, "unsupported encryption type %d\n", enctype); + ath12k_warn(dp_pdev->dp->ab, "unsupported encryption type %d\n", enctype); return 0; } -static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar, +static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, enum hal_encrypt_type enctype, - struct ieee80211_rx_status *status) + struct hal_rx_desc_data *rx_info) { - struct ath12k_base *ab = ar->ab; struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); u8 decap_hdr[DP_MAX_NWIFI_HDR_LEN]; struct ieee80211_hdr *hdr; @@ -2223,7 +965,7 @@ static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar, qos_ctl = rxcb->tid; - if (ath12k_dp_rx_h_mesh_ctl_present(ab, rxcb->rx_desc)) + if (rx_info->mesh_ctrl_present) qos_ctl |= IEEE80211_QOS_CTL_MESH_CONTROL_PRESENT; /* TODO: Add other QoS ctl fields when required */ @@ -2232,9 +974,10 @@ static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar, memcpy(decap_hdr, hdr, hdr_len); /* Rebuild crypto header for mac80211 use */ - if (!(status->flag & RX_FLAG_IV_STRIPPED)) { - crypto_hdr = skb_push(msdu, ath12k_dp_rx_crypto_param_len(ar, enctype)); - ath12k_dp_rx_desc_get_crypto_header(ar->ab, + if (!(rx_info->rx_status->flag & RX_FLAG_IV_STRIPPED)) { + crypto_hdr = skb_push(msdu, + ath12k_dp_rx_crypto_param_len(dp_pdev, enctype)); + ath12k_dp_rx_desc_get_crypto_header(dp_pdev->dp->hal, rxcb->rx_desc, crypto_hdr, enctype); } @@ -2245,11 +988,13 @@ static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar, memcpy(skb_push(msdu, hdr_len), decap_hdr, hdr_len); } -static void ath12k_dp_rx_h_undecap_raw(struct ath12k *ar, struct sk_buff *msdu, +static void ath12k_dp_rx_h_undecap_raw(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, enum hal_encrypt_type enctype, struct ieee80211_rx_status *status, bool decrypted) { + struct ath12k_dp *dp = dp_pdev->dp; struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); struct ieee80211_hdr *hdr; size_t hdr_len; @@ -2271,20 +1016,20 @@ static void ath12k_dp_rx_h_undecap_raw(struct ath12k *ar, struct sk_buff *msdu, /* Tail */ if (status->flag & RX_FLAG_IV_STRIPPED) { skb_trim(msdu, msdu->len - - ath12k_dp_rx_crypto_mic_len(ar, enctype)); + ath12k_dp_rx_crypto_mic_len(dp, enctype)); skb_trim(msdu, msdu->len - - ath12k_dp_rx_crypto_icv_len(ar, enctype)); + ath12k_dp_rx_crypto_icv_len(dp_pdev, enctype)); } else { /* MIC */ if (status->flag & RX_FLAG_MIC_STRIPPED) skb_trim(msdu, msdu->len - - ath12k_dp_rx_crypto_mic_len(ar, enctype)); + ath12k_dp_rx_crypto_mic_len(dp, enctype)); /* ICV */ if (status->flag & RX_FLAG_ICV_STRIPPED) skb_trim(msdu, msdu->len - - ath12k_dp_rx_crypto_icv_len(ar, enctype)); + ath12k_dp_rx_crypto_icv_len(dp_pdev, enctype)); } /* MMIC */ @@ -2296,58 +1041,59 @@ static void ath12k_dp_rx_h_undecap_raw(struct ath12k *ar, struct sk_buff *msdu, /* Head */ if (status->flag & RX_FLAG_IV_STRIPPED) { hdr_len = ieee80211_hdrlen(hdr->frame_control); - crypto_len = ath12k_dp_rx_crypto_param_len(ar, enctype); + crypto_len = ath12k_dp_rx_crypto_param_len(dp_pdev, enctype); memmove(msdu->data + crypto_len, msdu->data, hdr_len); skb_pull(msdu, crypto_len); } } -static void ath12k_get_dot11_hdr_from_rx_desc(struct ath12k *ar, +static void ath12k_get_dot11_hdr_from_rx_desc(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, struct ath12k_skb_rxcb *rxcb, - struct ieee80211_rx_status *status, - enum hal_encrypt_type enctype) + enum hal_encrypt_type enctype, + struct hal_rx_desc_data *rx_info) { struct hal_rx_desc *rx_desc = rxcb->rx_desc; - struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_hal *hal = dp->hal; size_t hdr_len, crypto_len; struct ieee80211_hdr hdr; __le16 qos_ctl; - u8 *crypto_hdr, mesh_ctrl; + u8 *crypto_hdr; - ath12k_dp_rx_desc_get_dot11_hdr(ab, rx_desc, &hdr); + ath12k_dp_rx_desc_get_dot11_hdr(hal, rx_desc, &hdr); hdr_len = ieee80211_hdrlen(hdr.frame_control); - mesh_ctrl = ath12k_dp_rx_h_mesh_ctl_present(ab, rx_desc); - if (!(status->flag & RX_FLAG_IV_STRIPPED)) { - crypto_len = ath12k_dp_rx_crypto_param_len(ar, enctype); + if (!(rx_info->rx_status->flag & RX_FLAG_IV_STRIPPED)) { + crypto_len = ath12k_dp_rx_crypto_param_len(dp_pdev, enctype); crypto_hdr = skb_push(msdu, crypto_len); - ath12k_dp_rx_desc_get_crypto_header(ab, rx_desc, crypto_hdr, enctype); + ath12k_dp_rx_desc_get_crypto_header(dp->hal, rx_desc, crypto_hdr, + enctype); } skb_push(msdu, hdr_len); memcpy(msdu->data, &hdr, min(hdr_len, sizeof(hdr))); if (rxcb->is_mcbc) - status->flag &= ~RX_FLAG_PN_VALIDATED; + rx_info->rx_status->flag &= ~RX_FLAG_PN_VALIDATED; /* Add QOS header */ if (ieee80211_is_data_qos(hdr.frame_control)) { struct ieee80211_hdr *qos_ptr = (struct ieee80211_hdr *)msdu->data; qos_ctl = cpu_to_le16(rxcb->tid & IEEE80211_QOS_CTL_TID_MASK); - if (mesh_ctrl) + if (rx_info->mesh_ctrl_present) qos_ctl |= cpu_to_le16(IEEE80211_QOS_CTL_MESH_CONTROL_PRESENT); memcpy(ieee80211_get_qos_ctl(qos_ptr), &qos_ctl, IEEE80211_QOS_CTL_LEN); } } -static void ath12k_dp_rx_h_undecap_eth(struct ath12k *ar, +static void ath12k_dp_rx_h_undecap_eth(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, enum hal_encrypt_type enctype, - struct ieee80211_rx_status *status) + struct hal_rx_desc_data *rx_info) { struct ieee80211_hdr *hdr; struct ethhdr *eth; @@ -2363,7 +1109,7 @@ static void ath12k_dp_rx_h_undecap_eth(struct ath12k *ar, skb_pull(msdu, sizeof(*eth)); memcpy(skb_push(msdu, sizeof(rfc)), &rfc, sizeof(rfc)); - ath12k_get_dot11_hdr_from_rx_desc(ar, msdu, rxcb, status, enctype); + ath12k_get_dot11_hdr_from_rx_desc(dp_pdev, msdu, rxcb, enctype, rx_info); /* original 802.11 header has a different DA and in * case of 4addr it may also have different SA @@ -2373,24 +1119,20 @@ static void ath12k_dp_rx_h_undecap_eth(struct ath12k *ar, ether_addr_copy(ieee80211_get_SA(hdr), sa); } -static void ath12k_dp_rx_h_undecap(struct ath12k *ar, struct sk_buff *msdu, - struct hal_rx_desc *rx_desc, - enum hal_encrypt_type enctype, - struct ieee80211_rx_status *status, - bool decrypted) +void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, + struct hal_rx_desc *rx_desc, + enum hal_encrypt_type enctype, + bool decrypted, + struct hal_rx_desc_data *rx_info) { - struct ath12k_base *ab = ar->ab; - u8 decap; struct ethhdr *ehdr; - decap = ath12k_dp_rx_h_decap_type(ab, rx_desc); - - switch (decap) { + switch (rx_info->decap_type) { case DP_RX_DECAP_TYPE_NATIVE_WIFI: - ath12k_dp_rx_h_undecap_nwifi(ar, msdu, enctype, status); + ath12k_dp_rx_h_undecap_nwifi(dp_pdev, msdu, enctype, rx_info); break; case DP_RX_DECAP_TYPE_RAW: - ath12k_dp_rx_h_undecap_raw(ar, msdu, enctype, status, + ath12k_dp_rx_h_undecap_raw(dp_pdev, msdu, enctype, rx_info->rx_status, decrypted); break; case DP_RX_DECAP_TYPE_ETHERNET2_DIX: @@ -2399,7 +1141,7 @@ static void ath12k_dp_rx_h_undecap(struct ath12k *ar, struct sk_buff *msdu, /* mac80211 allows fast path only for authorized STA */ if (ehdr->h_proto == cpu_to_be16(ETH_P_PAE)) { ATH12K_SKB_RXCB(msdu)->is_eapol = true; - ath12k_dp_rx_h_undecap_eth(ar, msdu, enctype, status); + ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info); break; } @@ -2407,125 +1149,52 @@ static void ath12k_dp_rx_h_undecap(struct ath12k *ar, struct sk_buff *msdu, * remove eth header and add 802.11 header. */ if (ATH12K_SKB_RXCB(msdu)->is_mcbc && decrypted) - ath12k_dp_rx_h_undecap_eth(ar, msdu, enctype, status); + ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info); break; case DP_RX_DECAP_TYPE_8023: /* TODO: Handle undecap for these formats */ break; } } +EXPORT_SYMBOL(ath12k_dp_rx_h_undecap); -struct ath12k_peer * -ath12k_dp_rx_h_find_peer(struct ath12k_base *ab, struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info) +struct ath12k_dp_link_peer * +ath12k_dp_rx_h_find_link_peer(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info) { struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); - struct ath12k_peer *peer = NULL; + struct ath12k_dp_link_peer *peer = NULL; + struct ath12k_dp *dp = dp_pdev->dp; - lockdep_assert_held(&ab->base_lock); + lockdep_assert_held(&dp->dp_lock); - if (rxcb->peer_id) - peer = ath12k_peer_find_by_id(ab, rxcb->peer_id); + peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, rxcb->peer_id); if (peer) return peer; if (rx_info->addr2_present) - peer = ath12k_peer_find_by_addr(ab, rx_info->addr2); + peer = ath12k_dp_link_peer_find_by_addr(dp, rx_info->addr2); return peer; } -static void ath12k_dp_rx_h_mpdu(struct ath12k *ar, - struct sk_buff *msdu, - struct hal_rx_desc *rx_desc, - struct ath12k_dp_rx_info *rx_info) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_skb_rxcb *rxcb; - enum hal_encrypt_type enctype; - bool is_decrypted = false; - struct ieee80211_hdr *hdr; - struct ath12k_peer *peer; - struct ieee80211_rx_status *rx_status = rx_info->rx_status; - u32 err_bitmap; - - /* PN for multicast packets will be checked in mac80211 */ - rxcb = ATH12K_SKB_RXCB(msdu); - rxcb->is_mcbc = rx_info->is_mcbc; - - if (rxcb->is_mcbc) - rxcb->peer_id = rx_info->peer_id; - - spin_lock_bh(&ar->ab->base_lock); - peer = ath12k_dp_rx_h_find_peer(ar->ab, msdu, rx_info); - if (peer) { - /* resetting mcbc bit because mcbc packets are unicast - * packets only for AP as STA sends unicast packets. - */ - rxcb->is_mcbc = rxcb->is_mcbc && !peer->ucast_ra_only; - - if (rxcb->is_mcbc) - enctype = peer->sec_type_grp; - else - enctype = peer->sec_type; - } else { - enctype = HAL_ENCRYPT_TYPE_OPEN; - } - spin_unlock_bh(&ar->ab->base_lock); - - err_bitmap = ath12k_dp_rx_h_mpdu_err(ab, rx_desc); - if (enctype != HAL_ENCRYPT_TYPE_OPEN && !err_bitmap) - is_decrypted = ath12k_dp_rx_h_is_decrypted(ab, rx_desc); - - /* Clear per-MPDU flags while leaving per-PPDU flags intact */ - rx_status->flag &= ~(RX_FLAG_FAILED_FCS_CRC | - RX_FLAG_MMIC_ERROR | - RX_FLAG_DECRYPTED | - RX_FLAG_IV_STRIPPED | - RX_FLAG_MMIC_STRIPPED); - - if (err_bitmap & HAL_RX_MPDU_ERR_FCS) - rx_status->flag |= RX_FLAG_FAILED_FCS_CRC; - if (err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC) - rx_status->flag |= RX_FLAG_MMIC_ERROR; - - if (is_decrypted) { - rx_status->flag |= RX_FLAG_DECRYPTED | RX_FLAG_MMIC_STRIPPED; - - if (rx_info->is_mcbc) - rx_status->flag |= RX_FLAG_MIC_STRIPPED | - RX_FLAG_ICV_STRIPPED; - else - rx_status->flag |= RX_FLAG_IV_STRIPPED | - RX_FLAG_PN_VALIDATED; - } - - ath12k_dp_rx_h_csum_offload(msdu, rx_info); - ath12k_dp_rx_h_undecap(ar, msdu, rx_desc, - enctype, rx_status, is_decrypted); - - if (!is_decrypted || rx_info->is_mcbc) - return; - - if (rx_info->decap_type != DP_RX_DECAP_TYPE_ETHERNET2_DIX) { - hdr = (void *)msdu->data; - hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_PROTECTED); - } -} - -static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info) +static void ath12k_dp_rx_h_rate(struct ath12k_pdev_dp *dp_pdev, + struct hal_rx_desc_data *rx_info) { + struct ath12k_dp *dp = dp_pdev->dp; struct ieee80211_supported_band *sband; struct ieee80211_rx_status *rx_status = rx_info->rx_status; enum rx_msdu_start_pkt_type pkt_type = rx_info->pkt_type; u8 bw = rx_info->bw, sgi = rx_info->sgi; u8 rate_mcs = rx_info->rate_mcs, nss = rx_info->nss; bool is_cck; + struct ath12k *ar; switch (pkt_type) { case RX_MSDU_START_PKT_TYPE_11A: case RX_MSDU_START_PKT_TYPE_11B: + ar = ath12k_pdev_dp_to_ar(dp_pdev); is_cck = (pkt_type == RX_MSDU_START_PKT_TYPE_11B); sband = &ar->mac.sbands[rx_status->band]; rx_status->rate_idx = ath12k_mac_hw_rate_to_idx(sband, rate_mcs, @@ -2534,7 +1203,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_ case RX_MSDU_START_PKT_TYPE_11N: rx_status->encoding = RX_ENC_HT; if (rate_mcs > ATH12K_HT_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(dp->ab, "Received with invalid mcs in HT mode %d\n", rate_mcs); break; @@ -2548,7 +1217,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_ rx_status->encoding = RX_ENC_VHT; rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH12K_VHT_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(dp->ab, "Received with invalid mcs in VHT mode %d\n", rate_mcs); break; @@ -2561,7 +1230,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_ case RX_MSDU_START_PKT_TYPE_11AX: rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH12K_HE_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(dp->ab, "Received with invalid mcs in HE mode %d\n", rate_mcs); break; @@ -2575,7 +1244,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_ rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH12K_EHT_MCS_MAX) { - ath12k_warn(ar->ab, + ath12k_warn(dp->ab, "Received with invalid mcs in EHT mode %d\n", rate_mcs); break; @@ -2591,33 +1260,8 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_ } } -void ath12k_dp_rx_h_fetch_info(struct ath12k_base *ab, struct hal_rx_desc *rx_desc, - struct ath12k_dp_rx_info *rx_info) -{ - rx_info->ip_csum_fail = ath12k_dp_rx_h_ip_cksum_fail(ab, rx_desc); - rx_info->l4_csum_fail = ath12k_dp_rx_h_l4_cksum_fail(ab, rx_desc); - rx_info->is_mcbc = ath12k_dp_rx_h_is_da_mcbc(ab, rx_desc); - rx_info->decap_type = ath12k_dp_rx_h_decap_type(ab, rx_desc); - rx_info->pkt_type = ath12k_dp_rx_h_pkt_type(ab, rx_desc); - rx_info->sgi = ath12k_dp_rx_h_sgi(ab, rx_desc); - rx_info->rate_mcs = ath12k_dp_rx_h_rate_mcs(ab, rx_desc); - rx_info->bw = ath12k_dp_rx_h_rx_bw(ab, rx_desc); - rx_info->nss = ath12k_dp_rx_h_nss(ab, rx_desc); - rx_info->tid = ath12k_dp_rx_h_tid(ab, rx_desc); - rx_info->peer_id = ath12k_dp_rx_h_peer_id(ab, rx_desc); - rx_info->phy_meta_data = ath12k_dp_rx_h_freq(ab, rx_desc); - - if (ath12k_dp_rxdesc_mac_addr2_valid(ab, rx_desc)) { - ether_addr_copy(rx_info->addr2, - ath12k_dp_rxdesc_get_mpdu_start_addr2(ab, rx_desc)); - rx_info->addr2_present = true; - } - - ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "rx_desc: ", - rx_desc, sizeof(*rx_desc)); -} - -void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info) +void ath12k_dp_rx_h_ppdu(struct ath12k_pdev_dp *dp_pdev, + struct hal_rx_desc_data *rx_info) { struct ieee80211_rx_status *rx_status = rx_info->rx_status; u8 channel_num; @@ -2650,7 +1294,9 @@ void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info) } if (unlikely(rx_status->band == NUM_NL80211_BANDS || - !ath12k_ar_to_hw(ar)->wiphy->bands[rx_status->band])) { + !ath12k_pdev_dp_to_hw(dp_pdev)->wiphy->bands[rx_status->band])) { + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + ath12k_warn(ar->ab, "sband is NULL for status band %d channel_num %d center_freq %d pdev_id %d\n", rx_status->band, channel_num, center_freq, ar->pdev_idx); @@ -2674,43 +1320,41 @@ void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info) rx_status->band); h_rate: - ath12k_dp_rx_h_rate(ar, rx_info); + ath12k_dp_rx_h_rate(dp_pdev, rx_info); } +EXPORT_SYMBOL(ath12k_dp_rx_h_ppdu); -static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *napi, - struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info) +void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struct *napi, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info) { - struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = dp_pdev->dp; struct ieee80211_rx_status *rx_status; struct ieee80211_sta *pubsta; - struct ath12k_peer *peer; + struct ath12k_dp_peer *peer; struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); struct ieee80211_rx_status *status = rx_info->rx_status; u8 decap = rx_info->decap_type; bool is_mcbc = rxcb->is_mcbc; bool is_eapol = rxcb->is_eapol; - spin_lock_bh(&ab->base_lock); - peer = ath12k_dp_rx_h_find_peer(ab, msdu, rx_info); + peer = ath12k_dp_peer_find_by_peerid(dp_pdev, rx_info->peer_id); pubsta = peer ? peer->sta : NULL; if (pubsta && pubsta->valid_links) { status->link_valid = 1; - status->link_id = peer->link_id; + status->link_id = peer->hw_links[rxcb->hw_link_id]; } - spin_unlock_bh(&ab->base_lock); - - ath12k_dbg(ab, ATH12K_DBG_DATA, + ath12k_dbg(dp->ab, ATH12K_DBG_DATA, "rx skb %p len %u peer %pM %d %s sn %u %s%s%s%s%s%s%s%s%s%s rate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n", msdu, msdu->len, peer ? peer->addr : NULL, rxcb->tid, is_mcbc ? "mcast" : "ucast", - ath12k_dp_rx_h_seq_no(ab, rxcb->rx_desc), + rx_info->seq_no, (status->encoding == RX_ENC_LEGACY) ? "legacy" : "", (status->encoding == RX_ENC_HT) ? "ht" : "", (status->encoding == RX_ENC_VHT) ? "vht" : "", @@ -2729,7 +1373,7 @@ static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *nap !!(status->flag & RX_FLAG_MMIC_ERROR), !!(status->flag & RX_FLAG_AMSDU_MORE)); - ath12k_dbg_dump(ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ", + ath12k_dbg_dump(dp->ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ", msdu->data, msdu->len); rx_status = IEEE80211_SKB_RXCB(msdu); @@ -2746,19 +1390,19 @@ static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *nap !(is_mcbc && rx_status->flag & RX_FLAG_DECRYPTED)) rx_status->flag |= RX_FLAG_8023; - ieee80211_rx_napi(ath12k_ar_to_hw(ar), pubsta, msdu, napi); + ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), pubsta, msdu, napi); } +EXPORT_SYMBOL(ath12k_dp_rx_deliver_msdu); -static bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_base *ab, - struct hal_rx_desc *rx_desc, - struct sk_buff *msdu) +bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_dp *dp, + struct hal_rx_desc *rx_desc, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info) { struct ieee80211_hdr *hdr; - u8 decap_type; u32 hdr_len; - decap_type = ath12k_dp_rx_h_decap_type(ab, rx_desc); - if (decap_type != DP_RX_DECAP_TYPE_NATIVE_WIFI) + if (rx_info->decap_type != DP_RX_DECAP_TYPE_NATIVE_WIFI) return true; hdr = (struct ieee80211_hdr *)msdu->data; @@ -2767,385 +1411,73 @@ static bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_base *ab, if ((likely(hdr_len <= DP_MAX_NWIFI_HDR_LEN))) return true; - ab->device_stats.invalid_rbm++; + dp->device_stats.invalid_rbm++; WARN_ON_ONCE(1); return false; } +EXPORT_SYMBOL(ath12k_dp_rx_check_nwifi_hdr_len_valid); -static int ath12k_dp_rx_process_msdu(struct ath12k *ar, - struct sk_buff *msdu, - struct sk_buff_head *msdu_list, - struct ath12k_dp_rx_info *rx_info) -{ - struct ath12k_base *ab = ar->ab; - struct hal_rx_desc *rx_desc, *lrx_desc; - struct ath12k_skb_rxcb *rxcb; - struct sk_buff *last_buf; - u8 l3_pad_bytes; - u16 msdu_len; - int ret; - u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; - - last_buf = ath12k_dp_rx_get_msdu_last_buf(msdu_list, msdu); - if (!last_buf) { - ath12k_warn(ab, - "No valid Rx buffer to access MSDU_END tlv\n"); - ret = -EIO; - goto free_out; - } - - rx_desc = (struct hal_rx_desc *)msdu->data; - lrx_desc = (struct hal_rx_desc *)last_buf->data; - if (!ath12k_dp_rx_h_msdu_done(ab, lrx_desc)) { - ath12k_warn(ab, "msdu_done bit in msdu_end is not set\n"); - ret = -EIO; - goto free_out; - } - - rxcb = ATH12K_SKB_RXCB(msdu); - rxcb->rx_desc = rx_desc; - msdu_len = ath12k_dp_rx_h_msdu_len(ab, lrx_desc); - l3_pad_bytes = ath12k_dp_rx_h_l3pad(ab, lrx_desc); - - if (rxcb->is_frag) { - skb_pull(msdu, hal_rx_desc_sz); - } else if (!rxcb->is_continuation) { - if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) { - ret = -EINVAL; - ath12k_warn(ab, "invalid msdu len %u\n", msdu_len); - ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "", rx_desc, - sizeof(*rx_desc)); - goto free_out; - } - skb_put(msdu, hal_rx_desc_sz + l3_pad_bytes + msdu_len); - skb_pull(msdu, hal_rx_desc_sz + l3_pad_bytes); - } else { - ret = ath12k_dp_rx_msdu_coalesce(ar, msdu_list, - msdu, last_buf, - l3_pad_bytes, msdu_len); - if (ret) { - ath12k_warn(ab, - "failed to coalesce msdu rx buffer%d\n", ret); - goto free_out; - } - } - - if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, rx_desc, msdu))) { - ret = -EINVAL; - goto free_out; - } - - ath12k_dp_rx_h_fetch_info(ab, rx_desc, rx_info); - ath12k_dp_rx_h_ppdu(ar, rx_info); - ath12k_dp_rx_h_mpdu(ar, msdu, rx_desc, rx_info); - - rx_info->rx_status->flag |= RX_FLAG_SKIP_MONITOR | RX_FLAG_DUP_VALIDATED; - - return 0; - -free_out: - return ret; -} - -static void ath12k_dp_rx_process_received_packets(struct ath12k_base *ab, - struct napi_struct *napi, - struct sk_buff_head *msdu_list, - int ring_id) -{ - struct ath12k_hw_group *ag = ab->ag; - struct ieee80211_rx_status rx_status = {}; - struct ath12k_skb_rxcb *rxcb; - struct sk_buff *msdu; - struct ath12k *ar; - struct ath12k_hw_link *hw_links = ag->hw_links; - struct ath12k_base *partner_ab; - struct ath12k_dp_rx_info rx_info; - u8 hw_link_id, pdev_id; - int ret; - - if (skb_queue_empty(msdu_list)) - return; - - rx_info.addr2_present = false; - rx_info.rx_status = &rx_status; - - rcu_read_lock(); - - while ((msdu = __skb_dequeue(msdu_list))) { - rxcb = ATH12K_SKB_RXCB(msdu); - hw_link_id = rxcb->hw_link_id; - partner_ab = ath12k_ag_to_ab(ag, - hw_links[hw_link_id].device_id); - pdev_id = ath12k_hw_mac_id_to_pdev_id(partner_ab->hw_params, - hw_links[hw_link_id].pdev_idx); - ar = partner_ab->pdevs[pdev_id].ar; - if (!rcu_dereference(partner_ab->pdevs_active[pdev_id])) { - dev_kfree_skb_any(msdu); - continue; - } - - if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) { - dev_kfree_skb_any(msdu); - continue; - } - - ret = ath12k_dp_rx_process_msdu(ar, msdu, msdu_list, &rx_info); - if (ret) { - ath12k_dbg(ab, ATH12K_DBG_DATA, - "Unable to process msdu %d", ret); - dev_kfree_skb_any(msdu); - continue; - } - - ath12k_dp_rx_deliver_msdu(ar, napi, msdu, &rx_info); - } - - rcu_read_unlock(); -} - -static u16 ath12k_dp_rx_get_peer_id(struct ath12k_base *ab, - enum ath12k_peer_metadata_version ver, - __le32 peer_metadata) -{ - switch (ver) { - default: - ath12k_warn(ab, "Unknown peer metadata version: %d", ver); - fallthrough; - case ATH12K_PEER_METADATA_V0: - return le32_get_bits(peer_metadata, - RX_MPDU_DESC_META_DATA_V0_PEER_ID); - case ATH12K_PEER_METADATA_V1: - return le32_get_bits(peer_metadata, - RX_MPDU_DESC_META_DATA_V1_PEER_ID); - case ATH12K_PEER_METADATA_V1A: - return le32_get_bits(peer_metadata, - RX_MPDU_DESC_META_DATA_V1A_PEER_ID); - case ATH12K_PEER_METADATA_V1B: - return le32_get_bits(peer_metadata, - RX_MPDU_DESC_META_DATA_V1B_PEER_ID); - } -} - -int ath12k_dp_rx_process(struct ath12k_base *ab, int ring_id, - struct napi_struct *napi, int budget) -{ - struct ath12k_hw_group *ag = ab->ag; - struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES]; - struct ath12k_hw_link *hw_links = ag->hw_links; - int num_buffs_reaped[ATH12K_MAX_DEVICES] = {}; - struct ath12k_rx_desc_info *desc_info; - struct ath12k_dp *dp = &ab->dp; - struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; - struct hal_reo_dest_ring *desc; - struct ath12k_base *partner_ab; - struct sk_buff_head msdu_list; - struct ath12k_skb_rxcb *rxcb; - int total_msdu_reaped = 0; - u8 hw_link_id, device_id; - struct hal_srng *srng; - struct sk_buff *msdu; - bool done = false; - u64 desc_va; - - __skb_queue_head_init(&msdu_list); - - for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) - INIT_LIST_HEAD(&rx_desc_used_list[device_id]); - - srng = &ab->hal.srng_list[dp->reo_dst_ring[ring_id].ring_id]; - - spin_lock_bh(&srng->lock); - -try_again: - ath12k_hal_srng_access_begin(ab, srng); - - while ((desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) { - struct rx_mpdu_desc *mpdu_info; - struct rx_msdu_desc *msdu_info; - enum hal_reo_dest_ring_push_reason push_reason; - u32 cookie; - - cookie = le32_get_bits(desc->buf_addr_info.info1, - BUFFER_ADDR_INFO1_SW_COOKIE); - - hw_link_id = le32_get_bits(desc->info0, - HAL_REO_DEST_RING_INFO0_SRC_LINK_ID); - - desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 | - le32_to_cpu(desc->buf_va_lo)); - desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va); - - device_id = hw_links[hw_link_id].device_id; - partner_ab = ath12k_ag_to_ab(ag, device_id); - if (unlikely(!partner_ab)) { - if (desc_info->skb) { - dev_kfree_skb_any(desc_info->skb); - desc_info->skb = NULL; - } - - continue; - } - - /* retry manual desc retrieval */ - if (!desc_info) { - desc_info = ath12k_dp_get_rx_desc(partner_ab, cookie); - if (!desc_info) { - ath12k_warn(partner_ab, "Invalid cookie in manual descriptor retrieval: 0x%x\n", - cookie); - continue; - } - } - - if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) - ath12k_warn(ab, "Check HW CC implementation"); - - msdu = desc_info->skb; - desc_info->skb = NULL; - - list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]); - - rxcb = ATH12K_SKB_RXCB(msdu); - dma_unmap_single(partner_ab->dev, rxcb->paddr, - msdu->len + skb_tailroom(msdu), - DMA_FROM_DEVICE); - - num_buffs_reaped[device_id]++; - ab->device_stats.reo_rx[ring_id][ab->device_id]++; - - push_reason = le32_get_bits(desc->info0, - HAL_REO_DEST_RING_INFO0_PUSH_REASON); - if (push_reason != - HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) { - dev_kfree_skb_any(msdu); - ab->device_stats.hal_reo_error[ring_id]++; - continue; - } - - msdu_info = &desc->rx_msdu_info; - mpdu_info = &desc->rx_mpdu_info; - - rxcb->is_first_msdu = !!(le32_to_cpu(msdu_info->info0) & - RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU); - rxcb->is_last_msdu = !!(le32_to_cpu(msdu_info->info0) & - RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU); - rxcb->is_continuation = !!(le32_to_cpu(msdu_info->info0) & - RX_MSDU_DESC_INFO0_MSDU_CONTINUATION); - rxcb->hw_link_id = hw_link_id; - rxcb->peer_id = ath12k_dp_rx_get_peer_id(ab, dp->peer_metadata_ver, - mpdu_info->peer_meta_data); - rxcb->tid = le32_get_bits(mpdu_info->info0, - RX_MPDU_DESC_INFO0_TID); - - __skb_queue_tail(&msdu_list, msdu); - - if (!rxcb->is_continuation) { - total_msdu_reaped++; - done = true; - } else { - done = false; - } - - if (total_msdu_reaped >= budget) - break; - } - - /* Hw might have updated the head pointer after we cached it. - * In this case, even though there are entries in the ring we'll - * get rx_desc NULL. Give the read another try with updated cached - * head pointer so that we can reap complete MPDU in the current - * rx processing. - */ - if (!done && ath12k_hal_srng_dst_num_free(ab, srng, true)) { - ath12k_hal_srng_access_end(ab, srng); - goto try_again; - } - - ath12k_hal_srng_access_end(ab, srng); - - spin_unlock_bh(&srng->lock); - - if (!total_msdu_reaped) - goto exit; - - for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) { - if (!num_buffs_reaped[device_id]) - continue; - - partner_ab = ath12k_ag_to_ab(ag, device_id); - rx_ring = &partner_ab->dp.rx_refill_buf_ring; - - ath12k_dp_rx_bufs_replenish(partner_ab, rx_ring, - &rx_desc_used_list[device_id], - num_buffs_reaped[device_id]); - } - - ath12k_dp_rx_process_received_packets(ab, napi, &msdu_list, - ring_id); - -exit: - return total_msdu_reaped; -} - -static void ath12k_dp_rx_frag_timer(struct timer_list *timer) +static void ath12k_dp_rx_frag_timer(struct timer_list *timer) { struct ath12k_dp_rx_tid *rx_tid = timer_container_of(rx_tid, timer, frag_timer); - spin_lock_bh(&rx_tid->ab->base_lock); + spin_lock_bh(&rx_tid->dp->dp_lock); if (rx_tid->last_frag_no && rx_tid->rx_frag_bitmap == GENMASK(rx_tid->last_frag_no, 0)) { - spin_unlock_bh(&rx_tid->ab->base_lock); + spin_unlock_bh(&rx_tid->dp->dp_lock); return; } - ath12k_dp_rx_frags_cleanup(rx_tid, true); - spin_unlock_bh(&rx_tid->ab->base_lock); + ath12k_dp_arch_rx_frags_cleanup(rx_tid->dp, rx_tid, true); + spin_unlock_bh(&rx_tid->dp->dp_lock); } int ath12k_dp_rx_peer_frag_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_id) { struct ath12k_base *ab = ar->ab; struct crypto_shash *tfm; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; struct ath12k_dp_rx_tid *rx_tid; int i; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); tfm = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(tfm)) return PTR_ERR(tfm); - spin_lock_bh(&ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ab, vdev_id, peer_mac); - if (!peer) { - spin_unlock_bh(&ab->base_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, peer_mac); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&dp->dp_lock); crypto_free_shash(tfm); ath12k_warn(ab, "failed to find the peer to set up fragment info\n"); return -ENOENT; } if (!peer->primary_link) { - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); crypto_free_shash(tfm); return 0; } for (i = 0; i <= IEEE80211_NUM_TIDS; i++) { - rx_tid = &peer->rx_tid[i]; - rx_tid->ab = ab; + rx_tid = &peer->dp_peer->rx_tid[i]; + rx_tid->dp = dp; timer_setup(&rx_tid->frag_timer, ath12k_dp_rx_frag_timer, 0); skb_queue_head_init(&rx_tid->rx_frags); } - peer->tfm_mmic = tfm; - peer->dp_setup_done = true; - spin_unlock_bh(&ab->base_lock); + peer->dp_peer->tfm_mmic = tfm; + peer->dp_peer->dp_setup_done = true; + spin_unlock_bh(&dp->dp_lock); return 0; } -static int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key, - struct ieee80211_hdr *hdr, u8 *data, - size_t data_len, u8 *mic) +int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key, + struct ieee80211_hdr *hdr, u8 *data, + size_t data_len, u8 *mic) { SHASH_DESC_ON_STACK(desc, tfm); u8 mic_hdr[16] = {}; @@ -3183,78 +1515,16 @@ static int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key, shash_desc_zero(desc); return ret; } +EXPORT_SYMBOL(ath12k_dp_rx_h_michael_mic); -static int ath12k_dp_rx_h_verify_tkip_mic(struct ath12k *ar, struct ath12k_peer *peer, - struct sk_buff *msdu) -{ - struct ath12k_base *ab = ar->ab; - struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data; - struct ieee80211_rx_status *rxs = IEEE80211_SKB_RXCB(msdu); - struct ieee80211_key_conf *key_conf; - struct ieee80211_hdr *hdr; - struct ath12k_dp_rx_info rx_info; - u8 mic[IEEE80211_CCMP_MIC_LEN]; - int head_len, tail_len, ret; - size_t data_len; - u32 hdr_len, hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; - u8 *key, *data; - u8 key_idx; - - if (ath12k_dp_rx_h_enctype(ab, rx_desc) != HAL_ENCRYPT_TYPE_TKIP_MIC) - return 0; - - rx_info.addr2_present = false; - rx_info.rx_status = rxs; - - hdr = (struct ieee80211_hdr *)(msdu->data + hal_rx_desc_sz); - hdr_len = ieee80211_hdrlen(hdr->frame_control); - head_len = hdr_len + hal_rx_desc_sz + IEEE80211_TKIP_IV_LEN; - tail_len = IEEE80211_CCMP_MIC_LEN + IEEE80211_TKIP_ICV_LEN + FCS_LEN; - - if (!is_multicast_ether_addr(hdr->addr1)) - key_idx = peer->ucast_keyidx; - else - key_idx = peer->mcast_keyidx; - - key_conf = peer->keys[key_idx]; - - data = msdu->data + head_len; - data_len = msdu->len - head_len - tail_len; - key = &key_conf->key[NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY]; - - ret = ath12k_dp_rx_h_michael_mic(peer->tfm_mmic, key, hdr, data, data_len, mic); - if (ret || memcmp(mic, data + data_len, IEEE80211_CCMP_MIC_LEN)) - goto mic_fail; - - return 0; - -mic_fail: - (ATH12K_SKB_RXCB(msdu))->is_first_msdu = true; - (ATH12K_SKB_RXCB(msdu))->is_last_msdu = true; - - ath12k_dp_rx_h_fetch_info(ab, rx_desc, &rx_info); - - rxs->flag |= RX_FLAG_MMIC_ERROR | RX_FLAG_MMIC_STRIPPED | - RX_FLAG_IV_STRIPPED | RX_FLAG_DECRYPTED; - skb_pull(msdu, hal_rx_desc_sz); - - if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, rx_desc, msdu))) - return -EINVAL; - - ath12k_dp_rx_h_ppdu(ar, &rx_info); - ath12k_dp_rx_h_undecap(ar, msdu, rx_desc, - HAL_ENCRYPT_TYPE_TKIP_MIC, rxs, true); - ieee80211_rx(ath12k_ar_to_hw(ar), msdu); - return -EINVAL; -} - -static void ath12k_dp_rx_h_undecap_frag(struct ath12k *ar, struct sk_buff *msdu, - enum hal_encrypt_type enctype, u32 flags) +void ath12k_dp_rx_h_undecap_frag(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, + enum hal_encrypt_type enctype, u32 flags) { + struct ath12k_dp *dp = dp_pdev->dp; struct ieee80211_hdr *hdr; size_t hdr_len; size_t crypto_len; - u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; + u32 hal_rx_desc_sz = dp->ab->hal.hal_desc_sz; if (!flags) return; @@ -3263,258 +1533,43 @@ static void ath12k_dp_rx_h_undecap_frag(struct ath12k *ar, struct sk_buff *msdu, if (flags & RX_FLAG_MIC_STRIPPED) skb_trim(msdu, msdu->len - - ath12k_dp_rx_crypto_mic_len(ar, enctype)); + ath12k_dp_rx_crypto_mic_len(dp, enctype)); if (flags & RX_FLAG_ICV_STRIPPED) skb_trim(msdu, msdu->len - - ath12k_dp_rx_crypto_icv_len(ar, enctype)); + ath12k_dp_rx_crypto_icv_len(dp_pdev, enctype)); if (flags & RX_FLAG_IV_STRIPPED) { hdr_len = ieee80211_hdrlen(hdr->frame_control); - crypto_len = ath12k_dp_rx_crypto_param_len(ar, enctype); + crypto_len = ath12k_dp_rx_crypto_param_len(dp_pdev, enctype); memmove(msdu->data + hal_rx_desc_sz + crypto_len, msdu->data + hal_rx_desc_sz, hdr_len); skb_pull(msdu, crypto_len); } } +EXPORT_SYMBOL(ath12k_dp_rx_h_undecap_frag); -static int ath12k_dp_rx_h_defrag(struct ath12k *ar, - struct ath12k_peer *peer, - struct ath12k_dp_rx_tid *rx_tid, - struct sk_buff **defrag_skb) -{ - struct ath12k_base *ab = ar->ab; - struct hal_rx_desc *rx_desc; - struct sk_buff *skb, *first_frag, *last_frag; - struct ieee80211_hdr *hdr; - enum hal_encrypt_type enctype; - bool is_decrypted = false; - int msdu_len = 0; - int extra_space; - u32 flags, hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; - - first_frag = skb_peek(&rx_tid->rx_frags); - last_frag = skb_peek_tail(&rx_tid->rx_frags); - - skb_queue_walk(&rx_tid->rx_frags, skb) { - flags = 0; - rx_desc = (struct hal_rx_desc *)skb->data; - hdr = (struct ieee80211_hdr *)(skb->data + hal_rx_desc_sz); - - enctype = ath12k_dp_rx_h_enctype(ab, rx_desc); - if (enctype != HAL_ENCRYPT_TYPE_OPEN) - is_decrypted = ath12k_dp_rx_h_is_decrypted(ab, - rx_desc); - - if (is_decrypted) { - if (skb != first_frag) - flags |= RX_FLAG_IV_STRIPPED; - if (skb != last_frag) - flags |= RX_FLAG_ICV_STRIPPED | - RX_FLAG_MIC_STRIPPED; - } - - /* RX fragments are always raw packets */ - if (skb != last_frag) - skb_trim(skb, skb->len - FCS_LEN); - ath12k_dp_rx_h_undecap_frag(ar, skb, enctype, flags); - - if (skb != first_frag) - skb_pull(skb, hal_rx_desc_sz + - ieee80211_hdrlen(hdr->frame_control)); - msdu_len += skb->len; - } - - extra_space = msdu_len - (DP_RX_BUFFER_SIZE + skb_tailroom(first_frag)); - if (extra_space > 0 && - (pskb_expand_head(first_frag, 0, extra_space, GFP_ATOMIC) < 0)) - return -ENOMEM; - - __skb_unlink(first_frag, &rx_tid->rx_frags); - while ((skb = __skb_dequeue(&rx_tid->rx_frags))) { - skb_put_data(first_frag, skb->data, skb->len); - dev_kfree_skb_any(skb); - } - - hdr = (struct ieee80211_hdr *)(first_frag->data + hal_rx_desc_sz); - hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_MOREFRAGS); - ATH12K_SKB_RXCB(first_frag)->is_frag = 1; - - if (ath12k_dp_rx_h_verify_tkip_mic(ar, peer, first_frag)) - first_frag = NULL; - - *defrag_skb = first_frag; - return 0; -} - -static int ath12k_dp_rx_h_defrag_reo_reinject(struct ath12k *ar, - struct ath12k_dp_rx_tid *rx_tid, - struct sk_buff *defrag_skb) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_dp *dp = &ab->dp; - struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)defrag_skb->data; - struct hal_reo_entrance_ring *reo_ent_ring; - struct hal_reo_dest_ring *reo_dest_ring; - struct dp_link_desc_bank *link_desc_banks; - struct hal_rx_msdu_link *msdu_link; - struct hal_rx_msdu_details *msdu0; - struct hal_srng *srng; - dma_addr_t link_paddr, buf_paddr; - u32 desc_bank, msdu_info, msdu_ext_info, mpdu_info; - u32 cookie, hal_rx_desc_sz, dest_ring_info0, queue_addr_hi; - int ret; - struct ath12k_rx_desc_info *desc_info; - enum hal_rx_buf_return_buf_manager idle_link_rbm = dp->idle_link_rbm; - u8 dst_ind; - - hal_rx_desc_sz = ab->hal.hal_desc_sz; - link_desc_banks = dp->link_desc_banks; - reo_dest_ring = rx_tid->dst_ring_desc; - - ath12k_hal_rx_reo_ent_paddr_get(ab, &reo_dest_ring->buf_addr_info, - &link_paddr, &cookie); - desc_bank = u32_get_bits(cookie, DP_LINK_DESC_BANK_MASK); - - msdu_link = (struct hal_rx_msdu_link *)(link_desc_banks[desc_bank].vaddr + - (link_paddr - link_desc_banks[desc_bank].paddr)); - msdu0 = &msdu_link->msdu_link[0]; - msdu_ext_info = le32_to_cpu(msdu0->rx_msdu_ext_info.info0); - dst_ind = u32_get_bits(msdu_ext_info, RX_MSDU_EXT_DESC_INFO0_REO_DEST_IND); - - memset(msdu0, 0, sizeof(*msdu0)); - - msdu_info = u32_encode_bits(1, RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU) | - u32_encode_bits(1, RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU) | - u32_encode_bits(0, RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) | - u32_encode_bits(defrag_skb->len - hal_rx_desc_sz, - RX_MSDU_DESC_INFO0_MSDU_LENGTH) | - u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_SA) | - u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_DA); - msdu0->rx_msdu_info.info0 = cpu_to_le32(msdu_info); - msdu0->rx_msdu_ext_info.info0 = cpu_to_le32(msdu_ext_info); - - /* change msdu len in hal rx desc */ - ath12k_dp_rxdesc_set_msdu_len(ab, rx_desc, defrag_skb->len - hal_rx_desc_sz); - - buf_paddr = dma_map_single(ab->dev, defrag_skb->data, - defrag_skb->len + skb_tailroom(defrag_skb), - DMA_TO_DEVICE); - if (dma_mapping_error(ab->dev, buf_paddr)) - return -ENOMEM; - - spin_lock_bh(&dp->rx_desc_lock); - desc_info = list_first_entry_or_null(&dp->rx_desc_free_list, - struct ath12k_rx_desc_info, - list); - if (!desc_info) { - spin_unlock_bh(&dp->rx_desc_lock); - ath12k_warn(ab, "failed to find rx desc for reinject\n"); - ret = -ENOMEM; - goto err_unmap_dma; - } - - desc_info->skb = defrag_skb; - desc_info->in_use = true; - - list_del(&desc_info->list); - spin_unlock_bh(&dp->rx_desc_lock); - - ATH12K_SKB_RXCB(defrag_skb)->paddr = buf_paddr; - - ath12k_hal_rx_buf_addr_info_set(&msdu0->buf_addr_info, buf_paddr, - desc_info->cookie, - HAL_RX_BUF_RBM_SW3_BM); - - /* Fill mpdu details into reo entrance ring */ - srng = &ab->hal.srng_list[dp->reo_reinject_ring.ring_id]; - - spin_lock_bh(&srng->lock); - ath12k_hal_srng_access_begin(ab, srng); - - reo_ent_ring = ath12k_hal_srng_src_get_next_entry(ab, srng); - if (!reo_ent_ring) { - ath12k_hal_srng_access_end(ab, srng); - spin_unlock_bh(&srng->lock); - ret = -ENOSPC; - goto err_free_desc; - } - memset(reo_ent_ring, 0, sizeof(*reo_ent_ring)); - - ath12k_hal_rx_buf_addr_info_set(&reo_ent_ring->buf_addr_info, link_paddr, - cookie, - idle_link_rbm); - - mpdu_info = u32_encode_bits(1, RX_MPDU_DESC_INFO0_MSDU_COUNT) | - u32_encode_bits(0, RX_MPDU_DESC_INFO0_FRAG_FLAG) | - u32_encode_bits(1, RX_MPDU_DESC_INFO0_RAW_MPDU) | - u32_encode_bits(1, RX_MPDU_DESC_INFO0_VALID_PN) | - u32_encode_bits(rx_tid->tid, RX_MPDU_DESC_INFO0_TID); - - reo_ent_ring->rx_mpdu_info.info0 = cpu_to_le32(mpdu_info); - reo_ent_ring->rx_mpdu_info.peer_meta_data = - reo_dest_ring->rx_mpdu_info.peer_meta_data; - - if (ab->hw_params->reoq_lut_support) { - reo_ent_ring->queue_addr_lo = reo_dest_ring->rx_mpdu_info.peer_meta_data; - queue_addr_hi = 0; - } else { - reo_ent_ring->queue_addr_lo = - cpu_to_le32(lower_32_bits(rx_tid->qbuf.paddr_aligned)); - queue_addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned); - } - - reo_ent_ring->info0 = le32_encode_bits(queue_addr_hi, - HAL_REO_ENTR_RING_INFO0_QUEUE_ADDR_HI) | - le32_encode_bits(dst_ind, - HAL_REO_ENTR_RING_INFO0_DEST_IND); - - reo_ent_ring->info1 = le32_encode_bits(rx_tid->cur_sn, - HAL_REO_ENTR_RING_INFO1_MPDU_SEQ_NUM); - dest_ring_info0 = le32_get_bits(reo_dest_ring->info0, - HAL_REO_DEST_RING_INFO0_SRC_LINK_ID); - reo_ent_ring->info2 = - cpu_to_le32(u32_get_bits(dest_ring_info0, - HAL_REO_ENTR_RING_INFO2_SRC_LINK_ID)); - - ath12k_hal_srng_access_end(ab, srng); - spin_unlock_bh(&srng->lock); - - return 0; - -err_free_desc: - spin_lock_bh(&dp->rx_desc_lock); - desc_info->in_use = false; - desc_info->skb = NULL; - list_add_tail(&desc_info->list, &dp->rx_desc_free_list); - spin_unlock_bh(&dp->rx_desc_lock); -err_unmap_dma: - dma_unmap_single(ab->dev, buf_paddr, defrag_skb->len + skb_tailroom(defrag_skb), - DMA_TO_DEVICE); - return ret; -} - -static int ath12k_dp_rx_h_cmp_frags(struct ath12k_base *ab, +static int ath12k_dp_rx_h_cmp_frags(struct ath12k_hal *hal, struct sk_buff *a, struct sk_buff *b) { int frag1, frag2; - frag1 = ath12k_dp_rx_h_frag_no(ab, a); - frag2 = ath12k_dp_rx_h_frag_no(ab, b); + frag1 = ath12k_dp_rx_h_frag_no(hal, a); + frag2 = ath12k_dp_rx_h_frag_no(hal, b); return frag1 - frag2; } -static void ath12k_dp_rx_h_sort_frags(struct ath12k_base *ab, - struct sk_buff_head *frag_list, - struct sk_buff *cur_frag) +void ath12k_dp_rx_h_sort_frags(struct ath12k_hal *hal, + struct sk_buff_head *frag_list, + struct sk_buff *cur_frag) { struct sk_buff *skb; int cmp; skb_queue_walk(frag_list, skb) { - cmp = ath12k_dp_rx_h_cmp_frags(ab, skb, cur_frag); + cmp = ath12k_dp_rx_h_cmp_frags(hal, skb, cur_frag); if (cmp < 0) continue; __skb_queue_before(frag_list, skb, cur_frag); @@ -3522,13 +1577,14 @@ static void ath12k_dp_rx_h_sort_frags(struct ath12k_base *ab, } __skb_queue_tail(frag_list, cur_frag); } +EXPORT_SYMBOL(ath12k_dp_rx_h_sort_frags); -static u64 ath12k_dp_rx_h_get_pn(struct ath12k *ar, struct sk_buff *skb) +u64 ath12k_dp_rx_h_get_pn(struct ath12k_dp *dp, struct sk_buff *skb) { struct ieee80211_hdr *hdr; u64 pn = 0; u8 *ehdr; - u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; + u32 hal_rx_desc_sz = dp->ab->hal.hal_desc_sz; hdr = (struct ieee80211_hdr *)(skb->data + hal_rx_desc_sz); ehdr = skb->data + hal_rx_desc_sz + ieee80211_hdrlen(hdr->frame_control); @@ -3542,1011 +1598,43 @@ static u64 ath12k_dp_rx_h_get_pn(struct ath12k *ar, struct sk_buff *skb) return pn; } +EXPORT_SYMBOL(ath12k_dp_rx_h_get_pn); -static bool -ath12k_dp_rx_h_defrag_validate_incr_pn(struct ath12k *ar, struct ath12k_dp_rx_tid *rx_tid) -{ - struct ath12k_base *ab = ar->ab; - enum hal_encrypt_type encrypt_type; - struct sk_buff *first_frag, *skb; - struct hal_rx_desc *desc; - u64 last_pn; - u64 cur_pn; - - first_frag = skb_peek(&rx_tid->rx_frags); - desc = (struct hal_rx_desc *)first_frag->data; - - encrypt_type = ath12k_dp_rx_h_enctype(ab, desc); - if (encrypt_type != HAL_ENCRYPT_TYPE_CCMP_128 && - encrypt_type != HAL_ENCRYPT_TYPE_CCMP_256 && - encrypt_type != HAL_ENCRYPT_TYPE_GCMP_128 && - encrypt_type != HAL_ENCRYPT_TYPE_AES_GCMP_256) - return true; - - last_pn = ath12k_dp_rx_h_get_pn(ar, first_frag); - skb_queue_walk(&rx_tid->rx_frags, skb) { - if (skb == first_frag) - continue; - - cur_pn = ath12k_dp_rx_h_get_pn(ar, skb); - if (cur_pn != last_pn + 1) - return false; - last_pn = cur_pn; - } - return true; -} - -static int ath12k_dp_rx_frag_h_mpdu(struct ath12k *ar, - struct sk_buff *msdu, - struct hal_reo_dest_ring *ring_desc) +void ath12k_dp_rx_free(struct ath12k_base *ab) { - struct ath12k_base *ab = ar->ab; - struct hal_rx_desc *rx_desc; - struct ath12k_peer *peer; - struct ath12k_dp_rx_tid *rx_tid; - struct sk_buff *defrag_skb = NULL; - u32 peer_id; - u16 seqno, frag_no; - u8 tid; - int ret = 0; - bool more_frags; - - rx_desc = (struct hal_rx_desc *)msdu->data; - peer_id = ath12k_dp_rx_h_peer_id(ab, rx_desc); - tid = ath12k_dp_rx_h_tid(ab, rx_desc); - seqno = ath12k_dp_rx_h_seq_no(ab, rx_desc); - frag_no = ath12k_dp_rx_h_frag_no(ab, msdu); - more_frags = ath12k_dp_rx_h_more_frags(ab, msdu); - - if (!ath12k_dp_rx_h_seq_ctrl_valid(ab, rx_desc) || - !ath12k_dp_rx_h_fc_valid(ab, rx_desc) || - tid > IEEE80211_NUM_TIDS) - return -EINVAL; - - /* received unfragmented packet in reo - * exception ring, this shouldn't happen - * as these packets typically come from - * reo2sw srngs. - */ - if (WARN_ON_ONCE(!frag_no && !more_frags)) - return -EINVAL; - - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, peer_id); - if (!peer) { - ath12k_warn(ab, "failed to find the peer to de-fragment received fragment peer_id %d\n", - peer_id); - ret = -ENOENT; - goto out_unlock; - } - - if (!peer->dp_setup_done) { - ath12k_warn(ab, "The peer %pM [%d] has uninitialized datapath\n", - peer->addr, peer_id); - ret = -ENOENT; - goto out_unlock; - } - - rx_tid = &peer->rx_tid[tid]; - - if ((!skb_queue_empty(&rx_tid->rx_frags) && seqno != rx_tid->cur_sn) || - skb_queue_empty(&rx_tid->rx_frags)) { - /* Flush stored fragments and start a new sequence */ - ath12k_dp_rx_frags_cleanup(rx_tid, true); - rx_tid->cur_sn = seqno; - } + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct dp_srng *srng; + int i; - if (rx_tid->rx_frag_bitmap & BIT(frag_no)) { - /* Fragment already present */ - ret = -EINVAL; - goto out_unlock; - } + ath12k_dp_srng_cleanup(ab, &dp->rx_refill_buf_ring.refill_buf_ring); - if ((!rx_tid->rx_frag_bitmap || frag_no > __fls(rx_tid->rx_frag_bitmap))) - __skb_queue_tail(&rx_tid->rx_frags, msdu); - else - ath12k_dp_rx_h_sort_frags(ab, &rx_tid->rx_frags, msdu); - - rx_tid->rx_frag_bitmap |= BIT(frag_no); - if (!more_frags) - rx_tid->last_frag_no = frag_no; - - if (frag_no == 0) { - rx_tid->dst_ring_desc = kmemdup(ring_desc, - sizeof(*rx_tid->dst_ring_desc), - GFP_ATOMIC); - if (!rx_tid->dst_ring_desc) { - ret = -ENOMEM; - goto out_unlock; + for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { + if (ab->hw_params->rx_mac_buf_ring) + ath12k_dp_srng_cleanup(ab, &dp->rx_mac_buf_ring[i]); + if (!ab->hw_params->rxdma1_enable) { + srng = &dp->rx_mon_status_refill_ring[i].refill_buf_ring; + ath12k_dp_srng_cleanup(ab, srng); } - } else { - ath12k_dp_rx_link_desc_return(ab, &ring_desc->buf_addr_info, - HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); } - if (!rx_tid->last_frag_no || - rx_tid->rx_frag_bitmap != GENMASK(rx_tid->last_frag_no, 0)) { - mod_timer(&rx_tid->frag_timer, jiffies + - ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS); - goto out_unlock; - } - - spin_unlock_bh(&ab->base_lock); - timer_delete_sync(&rx_tid->frag_timer); - spin_lock_bh(&ab->base_lock); - - peer = ath12k_peer_find_by_id(ab, peer_id); - if (!peer) - goto err_frags_cleanup; - - if (!ath12k_dp_rx_h_defrag_validate_incr_pn(ar, rx_tid)) - goto err_frags_cleanup; - - if (ath12k_dp_rx_h_defrag(ar, peer, rx_tid, &defrag_skb)) - goto err_frags_cleanup; - - if (!defrag_skb) - goto err_frags_cleanup; - - if (ath12k_dp_rx_h_defrag_reo_reinject(ar, rx_tid, defrag_skb)) - goto err_frags_cleanup; + for (i = 0; i < ab->hw_params->num_rxdma_dst_ring; i++) + ath12k_dp_srng_cleanup(ab, &dp->rxdma_err_dst_ring[i]); - ath12k_dp_rx_frags_cleanup(rx_tid, false); - goto out_unlock; + ath12k_dp_srng_cleanup(ab, &dp->rxdma_mon_buf_ring.refill_buf_ring); -err_frags_cleanup: - dev_kfree_skb_any(defrag_skb); - ath12k_dp_rx_frags_cleanup(rx_tid, true); -out_unlock: - spin_unlock_bh(&ab->base_lock); - return ret; + ath12k_dp_rxdma_buf_free(ab); } -static int -ath12k_dp_process_rx_err_buf(struct ath12k *ar, struct hal_reo_dest_ring *desc, - struct list_head *used_list, - bool drop, u32 cookie) +void ath12k_dp_rx_pdev_free(struct ath12k_base *ab, int mac_id) { - struct ath12k_base *ab = ar->ab; - struct sk_buff *msdu; - struct ath12k_skb_rxcb *rxcb; - struct hal_rx_desc *rx_desc; - u16 msdu_len; - u32 hal_rx_desc_sz = ab->hal.hal_desc_sz; - struct ath12k_rx_desc_info *desc_info; - u64 desc_va; - - desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 | - le32_to_cpu(desc->buf_va_lo)); - desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va); - - /* retry manual desc retrieval */ - if (!desc_info) { - desc_info = ath12k_dp_get_rx_desc(ab, cookie); - if (!desc_info) { - ath12k_warn(ab, "Invalid cookie in DP rx error descriptor retrieval: 0x%x\n", - cookie); - return -EINVAL; - } - } - - if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) - ath12k_warn(ab, " RX Exception, Check HW CC implementation"); - - msdu = desc_info->skb; - desc_info->skb = NULL; - - list_add_tail(&desc_info->list, used_list); - - rxcb = ATH12K_SKB_RXCB(msdu); - dma_unmap_single(ar->ab->dev, rxcb->paddr, - msdu->len + skb_tailroom(msdu), - DMA_FROM_DEVICE); - - if (drop) { - dev_kfree_skb_any(msdu); - return 0; - } - - rcu_read_lock(); - if (!rcu_dereference(ar->ab->pdevs_active[ar->pdev_idx])) { - dev_kfree_skb_any(msdu); - goto exit; - } - - if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) { - dev_kfree_skb_any(msdu); - goto exit; - } - - rx_desc = (struct hal_rx_desc *)msdu->data; - msdu_len = ath12k_dp_rx_h_msdu_len(ar->ab, rx_desc); - if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) { - ath12k_warn(ar->ab, "invalid msdu leng %u", msdu_len); - ath12k_dbg_dump(ar->ab, ATH12K_DBG_DATA, NULL, "", rx_desc, - sizeof(*rx_desc)); - dev_kfree_skb_any(msdu); - goto exit; - } - - skb_put(msdu, hal_rx_desc_sz + msdu_len); - - if (ath12k_dp_rx_frag_h_mpdu(ar, msdu, desc)) { - dev_kfree_skb_any(msdu); - ath12k_dp_rx_link_desc_return(ar->ab, &desc->buf_addr_info, - HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); - } -exit: - rcu_read_unlock(); - return 0; -} - -int ath12k_dp_rx_process_err(struct ath12k_base *ab, struct napi_struct *napi, - int budget) -{ - struct ath12k_hw_group *ag = ab->ag; - struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES]; - u32 msdu_cookies[HAL_NUM_RX_MSDUS_PER_LINK_DESC]; - int num_buffs_reaped[ATH12K_MAX_DEVICES] = {}; - struct dp_link_desc_bank *link_desc_banks; - enum hal_rx_buf_return_buf_manager rbm; - struct hal_rx_msdu_link *link_desc_va; - int tot_n_bufs_reaped, quota, ret, i; - struct hal_reo_dest_ring *reo_desc; - struct dp_rxdma_ring *rx_ring; - struct dp_srng *reo_except; - struct ath12k_hw_link *hw_links = ag->hw_links; - struct ath12k_base *partner_ab; - u8 hw_link_id, device_id; - u32 desc_bank, num_msdus; - struct hal_srng *srng; - struct ath12k *ar; - dma_addr_t paddr; - bool is_frag; - bool drop; - int pdev_id; - - tot_n_bufs_reaped = 0; - quota = budget; - - for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) - INIT_LIST_HEAD(&rx_desc_used_list[device_id]); - - reo_except = &ab->dp.reo_except_ring; - - srng = &ab->hal.srng_list[reo_except->ring_id]; - - spin_lock_bh(&srng->lock); - - ath12k_hal_srng_access_begin(ab, srng); - - while (budget && - (reo_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) { - drop = false; - ab->device_stats.err_ring_pkts++; - - ret = ath12k_hal_desc_reo_parse_err(ab, reo_desc, &paddr, - &desc_bank); - if (ret) { - ath12k_warn(ab, "failed to parse error reo desc %d\n", - ret); - continue; - } - - hw_link_id = le32_get_bits(reo_desc->info0, - HAL_REO_DEST_RING_INFO0_SRC_LINK_ID); - device_id = hw_links[hw_link_id].device_id; - partner_ab = ath12k_ag_to_ab(ag, device_id); - - pdev_id = ath12k_hw_mac_id_to_pdev_id(partner_ab->hw_params, - hw_links[hw_link_id].pdev_idx); - ar = partner_ab->pdevs[pdev_id].ar; - - link_desc_banks = partner_ab->dp.link_desc_banks; - link_desc_va = link_desc_banks[desc_bank].vaddr + - (paddr - link_desc_banks[desc_bank].paddr); - ath12k_hal_rx_msdu_link_info_get(link_desc_va, &num_msdus, msdu_cookies, - &rbm); - if (rbm != partner_ab->dp.idle_link_rbm && - rbm != HAL_RX_BUF_RBM_SW3_BM && - rbm != partner_ab->hw_params->hal_params->rx_buf_rbm) { - ab->device_stats.invalid_rbm++; - ath12k_warn(ab, "invalid return buffer manager %d\n", rbm); - ath12k_dp_rx_link_desc_return(partner_ab, - &reo_desc->buf_addr_info, - HAL_WBM_REL_BM_ACT_REL_MSDU); - continue; - } - - is_frag = !!(le32_to_cpu(reo_desc->rx_mpdu_info.info0) & - RX_MPDU_DESC_INFO0_FRAG_FLAG); - - /* Process only rx fragments with one msdu per link desc below, and drop - * msdu's indicated due to error reasons. - * Dynamic fragmentation not supported in Multi-link client, so drop the - * partner device buffers. - */ - if (!is_frag || num_msdus > 1 || - partner_ab->device_id != ab->device_id) { - drop = true; - - /* Return the link desc back to wbm idle list */ - ath12k_dp_rx_link_desc_return(partner_ab, - &reo_desc->buf_addr_info, - HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); - } - - for (i = 0; i < num_msdus; i++) { - if (!ath12k_dp_process_rx_err_buf(ar, reo_desc, - &rx_desc_used_list[device_id], - drop, - msdu_cookies[i])) { - num_buffs_reaped[device_id]++; - tot_n_bufs_reaped++; - } - } - - if (tot_n_bufs_reaped >= quota) { - tot_n_bufs_reaped = quota; - goto exit; - } - - budget = quota - tot_n_bufs_reaped; - } - -exit: - ath12k_hal_srng_access_end(ab, srng); - - spin_unlock_bh(&srng->lock); - - for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) { - if (!num_buffs_reaped[device_id]) - continue; - - partner_ab = ath12k_ag_to_ab(ag, device_id); - rx_ring = &partner_ab->dp.rx_refill_buf_ring; - - ath12k_dp_rx_bufs_replenish(partner_ab, rx_ring, - &rx_desc_used_list[device_id], - num_buffs_reaped[device_id]); - } - - return tot_n_bufs_reaped; -} - -static void ath12k_dp_rx_null_q_desc_sg_drop(struct ath12k *ar, - int msdu_len, - struct sk_buff_head *msdu_list) -{ - struct sk_buff *skb, *tmp; - struct ath12k_skb_rxcb *rxcb; - int n_buffs; - - n_buffs = DIV_ROUND_UP(msdu_len, - (DP_RX_BUFFER_SIZE - ar->ab->hal.hal_desc_sz)); - - skb_queue_walk_safe(msdu_list, skb, tmp) { - rxcb = ATH12K_SKB_RXCB(skb); - if (rxcb->err_rel_src == HAL_WBM_REL_SRC_MODULE_REO && - rxcb->err_code == HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO) { - if (!n_buffs) - break; - __skb_unlink(skb, msdu_list); - dev_kfree_skb_any(skb); - n_buffs--; - } - } -} - -static int ath12k_dp_rx_h_null_q_desc(struct ath12k *ar, struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info, - struct sk_buff_head *msdu_list) -{ - struct ath12k_base *ab = ar->ab; - u16 msdu_len; - struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data; - u8 l3pad_bytes; - struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); - u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; - - msdu_len = ath12k_dp_rx_h_msdu_len(ab, desc); - - if (!rxcb->is_frag && ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE)) { - /* First buffer will be freed by the caller, so deduct it's length */ - msdu_len = msdu_len - (DP_RX_BUFFER_SIZE - hal_rx_desc_sz); - ath12k_dp_rx_null_q_desc_sg_drop(ar, msdu_len, msdu_list); - return -EINVAL; - } - - /* Even after cleaning up the sg buffers in the msdu list with above check - * any msdu received with continuation flag needs to be dropped as invalid. - * This protects against some random err frame with continuation flag. - */ - if (rxcb->is_continuation) - return -EINVAL; - - if (!ath12k_dp_rx_h_msdu_done(ab, desc)) { - ath12k_warn(ar->ab, - "msdu_done bit not set in null_q_des processing\n"); - __skb_queue_purge(msdu_list); - return -EIO; - } - - /* Handle NULL queue descriptor violations arising out a missing - * REO queue for a given peer or a given TID. This typically - * may happen if a packet is received on a QOS enabled TID before the - * ADDBA negotiation for that TID, when the TID queue is setup. Or - * it may also happen for MC/BC frames if they are not routed to the - * non-QOS TID queue, in the absence of any other default TID queue. - * This error can show up both in a REO destination or WBM release ring. - */ - - if (rxcb->is_frag) { - skb_pull(msdu, hal_rx_desc_sz); - } else { - l3pad_bytes = ath12k_dp_rx_h_l3pad(ab, desc); - - if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE) - return -EINVAL; - - skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); - skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); - } - if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, desc, msdu))) - return -EINVAL; - - ath12k_dp_rx_h_fetch_info(ab, desc, rx_info); - ath12k_dp_rx_h_ppdu(ar, rx_info); - ath12k_dp_rx_h_mpdu(ar, msdu, desc, rx_info); - - rxcb->tid = rx_info->tid; - - /* Please note that caller will having the access to msdu and completing - * rx with mac80211. Need not worry about cleaning up amsdu_list. - */ - - return 0; -} - -static bool ath12k_dp_rx_h_reo_err(struct ath12k *ar, struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info, - struct sk_buff_head *msdu_list) -{ - struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); - bool drop = false; - - ar->ab->device_stats.reo_error[rxcb->err_code]++; - - switch (rxcb->err_code) { - case HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO: - if (ath12k_dp_rx_h_null_q_desc(ar, msdu, rx_info, msdu_list)) - drop = true; - break; - case HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED: - /* TODO: Do not drop PN failed packets in the driver; - * instead, it is good to drop such packets in mac80211 - * after incrementing the replay counters. - */ - fallthrough; - default: - /* TODO: Review other errors and process them to mac80211 - * as appropriate. - */ - drop = true; - break; - } - - return drop; -} - -static bool ath12k_dp_rx_h_tkip_mic_err(struct ath12k *ar, struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info) -{ - struct ath12k_base *ab = ar->ab; - u16 msdu_len; - struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data; - u8 l3pad_bytes; - struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); - u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz; - - rxcb->is_first_msdu = ath12k_dp_rx_h_first_msdu(ab, desc); - rxcb->is_last_msdu = ath12k_dp_rx_h_last_msdu(ab, desc); - - l3pad_bytes = ath12k_dp_rx_h_l3pad(ab, desc); - msdu_len = ath12k_dp_rx_h_msdu_len(ab, desc); - - if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE) { - ath12k_dbg(ab, ATH12K_DBG_DATA, - "invalid msdu len in tkip mic err %u\n", msdu_len); - ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "", desc, - sizeof(*desc)); - return true; - } - - skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); - skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); - - if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, desc, msdu))) - return true; - - ath12k_dp_rx_h_ppdu(ar, rx_info); - - rx_info->rx_status->flag |= (RX_FLAG_MMIC_STRIPPED | RX_FLAG_MMIC_ERROR | - RX_FLAG_DECRYPTED); - - ath12k_dp_rx_h_undecap(ar, msdu, desc, - HAL_ENCRYPT_TYPE_TKIP_MIC, rx_info->rx_status, false); - return false; -} - -static bool ath12k_dp_rx_h_rxdma_err(struct ath12k *ar, struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); - struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data; - bool drop = false; - u32 err_bitmap; - - ar->ab->device_stats.rxdma_error[rxcb->err_code]++; - - switch (rxcb->err_code) { - case HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR: - case HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR: - err_bitmap = ath12k_dp_rx_h_mpdu_err(ab, rx_desc); - if (err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC) { - ath12k_dp_rx_h_fetch_info(ab, rx_desc, rx_info); - drop = ath12k_dp_rx_h_tkip_mic_err(ar, msdu, rx_info); - break; - } - fallthrough; - default: - /* TODO: Review other rxdma error code to check if anything is - * worth reporting to mac80211 - */ - drop = true; - break; - } - - return drop; -} - -static void ath12k_dp_rx_wbm_err(struct ath12k *ar, - struct napi_struct *napi, - struct sk_buff *msdu, - struct sk_buff_head *msdu_list) -{ - struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); - struct ieee80211_rx_status rxs = {}; - struct ath12k_dp_rx_info rx_info; - bool drop = true; - - rx_info.addr2_present = false; - rx_info.rx_status = &rxs; - - switch (rxcb->err_rel_src) { - case HAL_WBM_REL_SRC_MODULE_REO: - drop = ath12k_dp_rx_h_reo_err(ar, msdu, &rx_info, msdu_list); - break; - case HAL_WBM_REL_SRC_MODULE_RXDMA: - drop = ath12k_dp_rx_h_rxdma_err(ar, msdu, &rx_info); - break; - default: - /* msdu will get freed */ - break; - } - - if (drop) { - dev_kfree_skb_any(msdu); - return; - } - - rx_info.rx_status->flag |= RX_FLAG_SKIP_MONITOR; - - ath12k_dp_rx_deliver_msdu(ar, napi, msdu, &rx_info); -} - -int ath12k_dp_rx_process_wbm_err(struct ath12k_base *ab, - struct napi_struct *napi, int budget) -{ - struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES]; - struct ath12k_hw_group *ag = ab->ag; - struct ath12k *ar; - struct ath12k_dp *dp = &ab->dp; - struct dp_rxdma_ring *rx_ring; - struct hal_rx_wbm_rel_info err_info; - struct hal_srng *srng; - struct sk_buff *msdu; - struct sk_buff_head msdu_list, scatter_msdu_list; - struct ath12k_skb_rxcb *rxcb; - void *rx_desc; - int num_buffs_reaped[ATH12K_MAX_DEVICES] = {}; - int total_num_buffs_reaped = 0; - struct ath12k_rx_desc_info *desc_info; - struct ath12k_device_dp_stats *device_stats = &ab->device_stats; - struct ath12k_hw_link *hw_links = ag->hw_links; - struct ath12k_base *partner_ab; - u8 hw_link_id, device_id; - int ret, pdev_id; - struct hal_rx_desc *msdu_data; - - __skb_queue_head_init(&msdu_list); - __skb_queue_head_init(&scatter_msdu_list); - - for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) - INIT_LIST_HEAD(&rx_desc_used_list[device_id]); - - srng = &ab->hal.srng_list[dp->rx_rel_ring.ring_id]; - spin_lock_bh(&srng->lock); - - ath12k_hal_srng_access_begin(ab, srng); - - while (budget) { - rx_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng); - if (!rx_desc) - break; - - ret = ath12k_hal_wbm_desc_parse_err(ab, rx_desc, &err_info); - if (ret) { - ath12k_warn(ab, - "failed to parse rx error in wbm_rel ring desc %d\n", - ret); - continue; - } - - desc_info = err_info.rx_desc; - - /* retry manual desc retrieval if hw cc is not done */ - if (!desc_info) { - desc_info = ath12k_dp_get_rx_desc(ab, err_info.cookie); - if (!desc_info) { - ath12k_warn(ab, "Invalid cookie in DP WBM rx error descriptor retrieval: 0x%x\n", - err_info.cookie); - continue; - } - } - - if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) - ath12k_warn(ab, "WBM RX err, Check HW CC implementation"); - - msdu = desc_info->skb; - desc_info->skb = NULL; - - device_id = desc_info->device_id; - partner_ab = ath12k_ag_to_ab(ag, device_id); - if (unlikely(!partner_ab)) { - dev_kfree_skb_any(msdu); - - /* In any case continuation bit is set - * in the previous record, cleanup scatter_msdu_list - */ - ath12k_dp_clean_up_skb_list(&scatter_msdu_list); - continue; - } - - list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]); - - rxcb = ATH12K_SKB_RXCB(msdu); - dma_unmap_single(partner_ab->dev, rxcb->paddr, - msdu->len + skb_tailroom(msdu), - DMA_FROM_DEVICE); - - num_buffs_reaped[device_id]++; - total_num_buffs_reaped++; - - if (!err_info.continuation) - budget--; - - if (err_info.push_reason != - HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) { - dev_kfree_skb_any(msdu); - continue; - } - - msdu_data = (struct hal_rx_desc *)msdu->data; - rxcb->err_rel_src = err_info.err_rel_src; - rxcb->err_code = err_info.err_code; - rxcb->is_first_msdu = err_info.first_msdu; - rxcb->is_last_msdu = err_info.last_msdu; - rxcb->is_continuation = err_info.continuation; - rxcb->rx_desc = msdu_data; - - if (err_info.continuation) { - __skb_queue_tail(&scatter_msdu_list, msdu); - continue; - } - - hw_link_id = ath12k_dp_rx_get_msdu_src_link(partner_ab, - msdu_data); - if (hw_link_id >= ATH12K_GROUP_MAX_RADIO) { - dev_kfree_skb_any(msdu); - - /* In any case continuation bit is set - * in the previous record, cleanup scatter_msdu_list - */ - ath12k_dp_clean_up_skb_list(&scatter_msdu_list); - continue; - } - - if (!skb_queue_empty(&scatter_msdu_list)) { - struct sk_buff *msdu; - - skb_queue_walk(&scatter_msdu_list, msdu) { - rxcb = ATH12K_SKB_RXCB(msdu); - rxcb->hw_link_id = hw_link_id; - } - - skb_queue_splice_tail_init(&scatter_msdu_list, - &msdu_list); - } - - rxcb = ATH12K_SKB_RXCB(msdu); - rxcb->hw_link_id = hw_link_id; - __skb_queue_tail(&msdu_list, msdu); - } - - /* In any case continuation bit is set in the - * last record, cleanup scatter_msdu_list - */ - ath12k_dp_clean_up_skb_list(&scatter_msdu_list); - - ath12k_hal_srng_access_end(ab, srng); - - spin_unlock_bh(&srng->lock); - - if (!total_num_buffs_reaped) - goto done; - - for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) { - if (!num_buffs_reaped[device_id]) - continue; - - partner_ab = ath12k_ag_to_ab(ag, device_id); - rx_ring = &partner_ab->dp.rx_refill_buf_ring; - - ath12k_dp_rx_bufs_replenish(ab, rx_ring, - &rx_desc_used_list[device_id], - num_buffs_reaped[device_id]); - } - - rcu_read_lock(); - while ((msdu = __skb_dequeue(&msdu_list))) { - rxcb = ATH12K_SKB_RXCB(msdu); - hw_link_id = rxcb->hw_link_id; - - device_id = hw_links[hw_link_id].device_id; - partner_ab = ath12k_ag_to_ab(ag, device_id); - if (unlikely(!partner_ab)) { - ath12k_dbg(ab, ATH12K_DBG_DATA, - "Unable to process WBM error msdu due to invalid hw link id %d device id %d\n", - hw_link_id, device_id); - dev_kfree_skb_any(msdu); - continue; - } - - pdev_id = ath12k_hw_mac_id_to_pdev_id(partner_ab->hw_params, - hw_links[hw_link_id].pdev_idx); - ar = partner_ab->pdevs[pdev_id].ar; - - if (!ar || !rcu_dereference(ar->ab->pdevs_active[pdev_id])) { - dev_kfree_skb_any(msdu); - continue; - } - - if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) { - dev_kfree_skb_any(msdu); - continue; - } - - if (rxcb->err_rel_src < HAL_WBM_REL_SRC_MODULE_MAX) { - device_id = ar->ab->device_id; - device_stats->rx_wbm_rel_source[rxcb->err_rel_src][device_id]++; - } - - ath12k_dp_rx_wbm_err(ar, napi, msdu, &msdu_list); - } - rcu_read_unlock(); -done: - return total_num_buffs_reaped; -} - -void ath12k_dp_rx_process_reo_status(struct ath12k_base *ab) -{ - struct ath12k_dp *dp = &ab->dp; - struct hal_tlv_64_hdr *hdr; - struct hal_srng *srng; - struct ath12k_dp_rx_reo_cmd *cmd, *tmp; - bool found = false; - u16 tag; - struct hal_reo_status reo_status; - - srng = &ab->hal.srng_list[dp->reo_status_ring.ring_id]; - - memset(&reo_status, 0, sizeof(reo_status)); - - spin_lock_bh(&srng->lock); - - ath12k_hal_srng_access_begin(ab, srng); - - while ((hdr = ath12k_hal_srng_dst_get_next_entry(ab, srng))) { - tag = le64_get_bits(hdr->tl, HAL_SRNG_TLV_HDR_TAG); - - switch (tag) { - case HAL_REO_GET_QUEUE_STATS_STATUS: - ath12k_hal_reo_status_queue_stats(ab, hdr, - &reo_status); - break; - case HAL_REO_FLUSH_QUEUE_STATUS: - ath12k_hal_reo_flush_queue_status(ab, hdr, - &reo_status); - break; - case HAL_REO_FLUSH_CACHE_STATUS: - ath12k_hal_reo_flush_cache_status(ab, hdr, - &reo_status); - break; - case HAL_REO_UNBLOCK_CACHE_STATUS: - ath12k_hal_reo_unblk_cache_status(ab, hdr, - &reo_status); - break; - case HAL_REO_FLUSH_TIMEOUT_LIST_STATUS: - ath12k_hal_reo_flush_timeout_list_status(ab, hdr, - &reo_status); - break; - case HAL_REO_DESCRIPTOR_THRESHOLD_REACHED_STATUS: - ath12k_hal_reo_desc_thresh_reached_status(ab, hdr, - &reo_status); - break; - case HAL_REO_UPDATE_RX_REO_QUEUE_STATUS: - ath12k_hal_reo_update_rx_reo_queue_status(ab, hdr, - &reo_status); - break; - default: - ath12k_warn(ab, "Unknown reo status type %d\n", tag); - continue; - } - - spin_lock_bh(&dp->reo_cmd_lock); - list_for_each_entry_safe(cmd, tmp, &dp->reo_cmd_list, list) { - if (reo_status.uniform_hdr.cmd_num == cmd->cmd_num) { - found = true; - list_del(&cmd->list); - break; - } - } - spin_unlock_bh(&dp->reo_cmd_lock); - - if (found) { - cmd->handler(dp, (void *)&cmd->data, - reo_status.uniform_hdr.cmd_status); - kfree(cmd); - } - - found = false; - } - - ath12k_hal_srng_access_end(ab, srng); - - spin_unlock_bh(&srng->lock); -} - -void ath12k_dp_rx_free(struct ath12k_base *ab) -{ - struct ath12k_dp *dp = &ab->dp; - struct dp_srng *srng; - int i; - - ath12k_dp_srng_cleanup(ab, &dp->rx_refill_buf_ring.refill_buf_ring); - - for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { - if (ab->hw_params->rx_mac_buf_ring) - ath12k_dp_srng_cleanup(ab, &dp->rx_mac_buf_ring[i]); - if (!ab->hw_params->rxdma1_enable) { - srng = &dp->rx_mon_status_refill_ring[i].refill_buf_ring; - ath12k_dp_srng_cleanup(ab, srng); - } - } - - for (i = 0; i < ab->hw_params->num_rxdma_dst_ring; i++) - ath12k_dp_srng_cleanup(ab, &dp->rxdma_err_dst_ring[i]); - - ath12k_dp_srng_cleanup(ab, &dp->rxdma_mon_buf_ring.refill_buf_ring); - - ath12k_dp_rxdma_buf_free(ab); -} - -void ath12k_dp_rx_pdev_free(struct ath12k_base *ab, int mac_id) -{ - struct ath12k *ar = ab->pdevs[mac_id].ar; + struct ath12k *ar = ab->pdevs[mac_id].ar; ath12k_dp_rx_pdev_srng_free(ar); } -int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab) -{ - struct ath12k_dp *dp = &ab->dp; - struct htt_rx_ring_tlv_filter tlv_filter = {}; - u32 ring_id; - int ret; - u32 hal_rx_desc_sz = ab->hal.hal_desc_sz; - - ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id; - - tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING; - tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR; - tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST | - HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST | - HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA; - tlv_filter.offset_valid = true; - tlv_filter.rx_packet_offset = hal_rx_desc_sz; - - tlv_filter.rx_mpdu_start_offset = - ab->hal_rx_ops->rx_desc_get_mpdu_start_offset(); - tlv_filter.rx_msdu_end_offset = - ab->hal_rx_ops->rx_desc_get_msdu_end_offset(); - - if (ath12k_dp_wmask_compaction_rx_tlv_supported(ab)) { - tlv_filter.rx_mpdu_start_wmask = - ab->hw_params->hal_ops->rxdma_ring_wmask_rx_mpdu_start(); - tlv_filter.rx_msdu_end_wmask = - ab->hw_params->hal_ops->rxdma_ring_wmask_rx_msdu_end(); - ath12k_dbg(ab, ATH12K_DBG_DATA, - "Configuring compact tlv masks rx_mpdu_start_wmask 0x%x rx_msdu_end_wmask 0x%x\n", - tlv_filter.rx_mpdu_start_wmask, tlv_filter.rx_msdu_end_wmask); - } - - ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, 0, - HAL_RXDMA_BUF, - DP_RXDMA_REFILL_RING_SIZE, - &tlv_filter); - - return ret; -} - -int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab) -{ - struct ath12k_dp *dp = &ab->dp; - struct htt_rx_ring_tlv_filter tlv_filter = {}; - u32 ring_id; - int ret = 0; - u32 hal_rx_desc_sz = ab->hal.hal_desc_sz; - int i; - - ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id; - - tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING; - tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR; - tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST | - HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST | - HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA; - tlv_filter.offset_valid = true; - tlv_filter.rx_packet_offset = hal_rx_desc_sz; - - tlv_filter.rx_header_offset = offsetof(struct hal_rx_desc_wcn7850, pkt_hdr_tlv); - - tlv_filter.rx_mpdu_start_offset = - ab->hal_rx_ops->rx_desc_get_mpdu_start_offset(); - tlv_filter.rx_msdu_end_offset = - ab->hal_rx_ops->rx_desc_get_msdu_end_offset(); - - /* TODO: Selectively subscribe to required qwords within msdu_end - * and mpdu_start and setup the mask in below msg - * and modify the rx_desc struct - */ - - for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { - ring_id = dp->rx_mac_buf_ring[i].ring_id; - ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, i, - HAL_RXDMA_BUF, - DP_RXDMA_REFILL_RING_SIZE, - &tlv_filter); - } - - return ret; -} - int ath12k_dp_rx_htt_setup(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); u32 ring_id; int i, ret; @@ -4618,7 +1706,7 @@ int ath12k_dp_rx_htt_setup(struct ath12k_base *ab) int ath12k_dp_rx_alloc(struct ath12k_base *ab) { - struct ath12k_dp *dp = &ab->dp; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct dp_srng *srng; int i, ret; diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.h b/drivers/net/wireless/ath/ath12k/dp_rx.h index 69d0a36a91d88..1ec5382f59955 100644 --- a/drivers/net/wireless/ath/ath12k/dp_rx.h +++ b/drivers/net/wireless/ath/ath12k/dp_rx.h @@ -1,21 +1,26 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_DP_RX_H #define ATH12K_DP_RX_H +#include #include "core.h" -#include "rx_desc.h" #include "debug.h" #define DP_MAX_NWIFI_HDR_LEN 30 +struct ath12k_reoq_buf { + void *vaddr; + dma_addr_t paddr_aligned; + u32 size; +}; + struct ath12k_dp_rx_tid { u8 tid; u32 ba_win_sz; - bool active; struct ath12k_reoq_buf qbuf; /* Info related to rx fragments */ @@ -28,7 +33,7 @@ struct ath12k_dp_rx_tid { /* Timer info related to fragments */ struct timer_list frag_timer; - struct ath12k_base *ab; + struct ath12k_dp *dp; }; struct ath12k_dp_rx_tid_rxq { @@ -59,6 +64,8 @@ struct ath12k_dp_rx_reo_cmd { enum hal_reo_cmd_status status); }; +#define ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS (2 * HZ) + #define ATH12K_DP_RX_REO_DESC_FREE_THRES 64 #define ATH12K_DP_RX_REO_DESC_FREE_TIMEOUT_MS 1000 @@ -77,24 +84,6 @@ struct ath12k_dp_rx_rfc1042_hdr { __be16 snap_type; } __packed; -struct ath12k_dp_rx_info { - struct ieee80211_rx_status *rx_status; - u32 phy_meta_data; - u16 peer_id; - u8 decap_type; - u8 pkt_type; - u8 sgi; - u8 rate_mcs; - u8 bw; - u8 nss; - u8 addr2[ETH_ALEN]; - u8 tid; - bool ip_csum_fail; - bool l4_csum_fail; - bool is_mcbc; - bool addr2_present; -}; - static inline u32 ath12k_he_gi_to_nl80211_he_gi(u8 sgi) { u32 ret = 0; @@ -117,6 +106,109 @@ static inline u32 ath12k_he_gi_to_nl80211_he_gi(u8 sgi) return ret; } +static inline bool ath12k_dp_rx_h_more_frags(struct ath12k_hal *hal, + struct sk_buff *skb) +{ + struct ieee80211_hdr *hdr; + + hdr = (struct ieee80211_hdr *)(skb->data + hal->hal_desc_sz); + return ieee80211_has_morefrags(hdr->frame_control); +} + +static inline u16 ath12k_dp_rx_h_frag_no(struct ath12k_hal *hal, + struct sk_buff *skb) +{ + struct ieee80211_hdr *hdr; + + hdr = (struct ieee80211_hdr *)(skb->data + hal->hal_desc_sz); + return le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_FRAG; +} + +static inline u8 ath12k_dp_rx_h_l3pad(struct ath12k_base *ab, + struct hal_rx_desc *desc) +{ + return ab->hal.ops->rx_desc_get_l3_pad_bytes(desc); +} + +static inline void ath12k_dp_rx_desc_end_tlv_copy(struct ath12k_hal *hal, + struct hal_rx_desc *fdesc, + struct hal_rx_desc *ldesc) +{ + hal->ops->rx_desc_copy_end_tlv(fdesc, ldesc); +} + +static inline void ath12k_dp_rxdesc_set_msdu_len(struct ath12k_hal *hal, + struct hal_rx_desc *desc, + u16 len) +{ + hal->ops->rx_desc_set_msdu_len(desc, len); +} + +static inline u32 ath12k_dp_rxdesc_get_ppduid(struct ath12k_base *ab, + struct hal_rx_desc *rx_desc) +{ + return ab->hal.ops->rx_desc_get_mpdu_ppdu_id(rx_desc); +} + +static inline void ath12k_dp_rx_desc_get_dot11_hdr(struct ath12k_hal *hal, + struct hal_rx_desc *desc, + struct ieee80211_hdr *hdr) +{ + hal->ops->rx_desc_get_dot11_hdr(desc, hdr); +} + +static inline void ath12k_dp_rx_desc_get_crypto_header(struct ath12k_hal *hal, + struct hal_rx_desc *desc, + u8 *crypto_hdr, + enum hal_encrypt_type enctype) +{ + hal->ops->rx_desc_get_crypto_header(desc, crypto_hdr, enctype); +} + +static inline u8 ath12k_dp_rx_get_msdu_src_link(struct ath12k_hal *hal, + struct hal_rx_desc *desc) +{ + return hal->ops->rx_desc_get_msdu_src_link_id(desc); +} + +static inline void ath12k_dp_clean_up_skb_list(struct sk_buff_head *skb_list) +{ + struct sk_buff *skb; + + while ((skb = __skb_dequeue(skb_list))) + dev_kfree_skb_any(skb); +} + +static inline +void ath12k_dp_extract_rx_desc_data(struct ath12k_hal *hal, + struct hal_rx_desc_data *rx_info, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc *ldesc) +{ + hal->ops->extract_rx_desc_data(rx_info, rx_desc, ldesc); +} + +void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, + struct hal_rx_desc *rx_desc, + enum hal_encrypt_type enctype, + bool decrypted, + struct hal_rx_desc_data *rx_info); +void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struct *napi, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info); +bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_dp *dp, + struct hal_rx_desc *rx_desc, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info); +u64 ath12k_dp_rx_h_get_pn(struct ath12k_dp *dp, struct sk_buff *skb); +void ath12k_dp_rx_h_sort_frags(struct ath12k_hal *hal, + struct sk_buff_head *frag_list, + struct sk_buff *cur_frag); +void ath12k_dp_rx_h_undecap_frag(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, + enum hal_encrypt_type enctype, u32 flags); +int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key, + struct ieee80211_hdr *hdr, u8 *data, + size_t data_len, u8 *mic); int ath12k_dp_rx_ampdu_start(struct ath12k *ar, struct ieee80211_ampdu_params *params, u8 link_id); @@ -127,14 +219,12 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif, const u8 *peer_addr, enum set_key_cmd key_cmd, struct ieee80211_key_conf *key); -void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_peer *peer); +void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_dp_link_peer *peer); void ath12k_dp_rx_peer_tid_delete(struct ath12k *ar, - struct ath12k_peer *peer, u8 tid); + struct ath12k_dp_link_peer *peer, u8 tid); int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_id, u8 tid, u32 ba_win_sz, u16 ssn, enum hal_pn_type pn_type); -void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab, - struct sk_buff *skb); int ath12k_dp_rx_pdev_reo_setup(struct ath12k_base *ab); void ath12k_dp_rx_pdev_reo_cleanup(struct ath12k_base *ab); int ath12k_dp_rx_htt_setup(struct ath12k_base *ab); @@ -143,15 +233,7 @@ void ath12k_dp_rx_free(struct ath12k_base *ab); int ath12k_dp_rx_pdev_alloc(struct ath12k_base *ab, int pdev_idx); void ath12k_dp_rx_pdev_free(struct ath12k_base *ab, int pdev_idx); void ath12k_dp_rx_reo_cmd_list_cleanup(struct ath12k_base *ab); -void ath12k_dp_rx_process_reo_status(struct ath12k_base *ab); -int ath12k_dp_rx_process_wbm_err(struct ath12k_base *ab, - struct napi_struct *napi, int budget); -int ath12k_dp_rx_process_err(struct ath12k_base *ab, struct napi_struct *napi, - int budget); -int ath12k_dp_rx_process(struct ath12k_base *ab, int mac_id, - struct napi_struct *napi, - int budget); -int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, +int ath12k_dp_rx_bufs_replenish(struct ath12k_dp *dp, struct dp_rxdma_ring *rx_ring, struct list_head *used_list, int req_entries); @@ -160,32 +242,27 @@ int ath12k_dp_rx_peer_frag_setup(struct ath12k *ar, const u8 *peer_mac, int vdev u8 ath12k_dp_rx_h_l3pad(struct ath12k_base *ab, struct hal_rx_desc *desc); -struct ath12k_peer * -ath12k_dp_rx_h_find_peer(struct ath12k_base *ab, struct sk_buff *msdu, - struct ath12k_dp_rx_info *rx_info); +struct ath12k_dp_link_peer * +ath12k_dp_rx_h_find_link_peer(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info); u8 ath12k_dp_rx_h_decap_type(struct ath12k_base *ab, struct hal_rx_desc *desc); u32 ath12k_dp_rx_h_mpdu_err(struct ath12k_base *ab, struct hal_rx_desc *desc); -void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info); -int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab); -int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab); - -int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len, - int (*iter)(struct ath12k_base *ar, u16 tag, u16 len, - const void *ptr, void *data), - void *data); -void ath12k_dp_rx_h_fetch_info(struct ath12k_base *ab, struct hal_rx_desc *rx_desc, - struct ath12k_dp_rx_info *rx_info); - -int ath12k_dp_rx_crypto_mic_len(struct ath12k *ar, enum hal_encrypt_type enctype); +int ath12k_dp_rx_crypto_mic_len(struct ath12k_dp *dp, enum hal_encrypt_type enctype); u32 ath12k_dp_rxdesc_get_ppduid(struct ath12k_base *ab, struct hal_rx_desc *rx_desc); -bool ath12k_dp_rxdesc_mpdu_valid(struct ath12k_base *ab, - struct hal_rx_desc *rx_desc); -int ath12k_dp_rx_link_desc_return(struct ath12k_base *ab, - struct ath12k_buffer_addr *buf_addr_info, - enum hal_wbm_rel_bm_act action); -bool ath12k_dp_rxdesc_mpdu_valid(struct ath12k_base *ab, - struct hal_rx_desc *rx_desc); +void ath12k_dp_rx_h_ppdu(struct ath12k_pdev_dp *dp_pdev, + struct hal_rx_desc_data *rx_info); +struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_list, + struct sk_buff *first); +void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status); +void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status); +void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *dp); +void ath12k_dp_init_rx_tid_rxq(struct ath12k_dp_rx_tid_rxq *rx_tid_rxq, + struct ath12k_dp_rx_tid *rx_tid, + bool active); +void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8 tid); #endif /* ATH12K_DP_RX_H */ diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.c b/drivers/net/wireless/ath/ath12k/dp_tx.c index abc84ca8467a2..c10da6195c9c3 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.c +++ b/drivers/net/wireless/ath/ath12k/dp_tx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "core.h" @@ -12,7 +12,7 @@ #include "peer.h" #include "mac.h" -static enum hal_tcl_encap_type +enum hal_tcl_encap_type ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb) { struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb); @@ -25,8 +25,9 @@ ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb) return HAL_TCL_ENCAP_TYPE_NATIVE_WIFI; } +EXPORT_SYMBOL(ath12k_dp_tx_get_encap_type); -static void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb) +void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb) { struct ieee80211_hdr *hdr = (void *)skb->data; u8 *qos_ctl; @@ -42,8 +43,9 @@ static void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb) hdr = (void *)skb->data; hdr->frame_control &= ~__cpu_to_le16(IEEE80211_STYPE_QOS_DATA); } +EXPORT_SYMBOL(ath12k_dp_tx_encap_nwifi); -static u8 ath12k_dp_tx_get_tid(struct sk_buff *skb) +u8 ath12k_dp_tx_get_tid(struct sk_buff *skb) { struct ieee80211_hdr *hdr = (void *)skb->data; struct ath12k_skb_cb *cb = ATH12K_SKB_CB(skb); @@ -55,6 +57,7 @@ static u8 ath12k_dp_tx_get_tid(struct sk_buff *skb) else return skb->priority & IEEE80211_QOS_CTL_TID_MASK; } +EXPORT_SYMBOL(ath12k_dp_tx_get_tid); enum hal_encrypt_type ath12k_dp_tx_get_encrypt_type(u32 cipher) { @@ -77,19 +80,21 @@ enum hal_encrypt_type ath12k_dp_tx_get_encrypt_type(u32 cipher) return HAL_ENCRYPT_TYPE_OPEN; } } +EXPORT_SYMBOL(ath12k_dp_tx_get_encrypt_type); -static void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp, - struct ath12k_tx_desc_info *tx_desc, - u8 pool_id) +void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp, + struct ath12k_tx_desc_info *tx_desc, + u8 pool_id) { spin_lock_bh(&dp->tx_desc_lock[pool_id]); tx_desc->skb_ext_desc = NULL; list_move_tail(&tx_desc->list, &dp->tx_desc_free_list[pool_id]); spin_unlock_bh(&dp->tx_desc_lock[pool_id]); } +EXPORT_SYMBOL(ath12k_dp_tx_release_txbuf); -static struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp *dp, - u8 pool_id) +struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp *dp, + u8 pool_id) { struct ath12k_tx_desc_info *desc; @@ -108,28 +113,9 @@ static struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp * return desc; } +EXPORT_SYMBOL(ath12k_dp_tx_assign_buffer); -static void ath12k_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab, - struct hal_tx_msdu_ext_desc *tcl_ext_cmd, - struct hal_tx_info *ti) -{ - tcl_ext_cmd->info0 = le32_encode_bits(ti->paddr, - HAL_TX_MSDU_EXT_INFO0_BUF_PTR_LO); - tcl_ext_cmd->info1 = le32_encode_bits(0x0, - HAL_TX_MSDU_EXT_INFO1_BUF_PTR_HI) | - le32_encode_bits(ti->data_len, - HAL_TX_MSDU_EXT_INFO1_BUF_LEN); - - tcl_ext_cmd->info1 |= le32_encode_bits(1, HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE) | - le32_encode_bits(ti->encap_type, - HAL_TX_MSDU_EXT_INFO1_ENCAP_TYPE) | - le32_encode_bits(ti->encrypt_type, - HAL_TX_MSDU_EXT_INFO1_ENCRYPT_TYPE); -} - -#define HTT_META_DATA_ALIGNMENT 0x8 - -static void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len) +void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len) { struct sk_buff *tail; void *metadata; @@ -141,29 +127,7 @@ static void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len) memset(metadata, 0, tail_len); return metadata; } - -/* Preparing HTT Metadata when utilized with ext MSDU */ -static int ath12k_dp_prepare_htt_metadata(struct sk_buff *skb) -{ - struct hal_tx_msdu_metadata *desc_ext; - u8 htt_desc_size; - /* Size rounded of multiple of 8 bytes */ - u8 htt_desc_size_aligned; - - htt_desc_size = sizeof(struct hal_tx_msdu_metadata); - htt_desc_size_aligned = ALIGN(htt_desc_size, HTT_META_DATA_ALIGNMENT); - - desc_ext = ath12k_dp_metadata_align_skb(skb, htt_desc_size_aligned); - if (!desc_ext) - return -ENOMEM; - - desc_ext->info0 = le32_encode_bits(1, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_FLAG) | - le32_encode_bits(0, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_TYPE) | - le32_encode_bits(1, - HAL_TX_MSDU_METADATA_INFO0_HOST_TX_DESC_POOL); - - return 0; -} +EXPORT_SYMBOL(ath12k_dp_metadata_align_skb); static void ath12k_dp_tx_move_payload(struct sk_buff *skb, unsigned long delta, @@ -182,10 +146,9 @@ static void ath12k_dp_tx_move_payload(struct sk_buff *skb, } } -static int ath12k_dp_tx_align_payload(struct ath12k_base *ab, - struct sk_buff **pskb) +int ath12k_dp_tx_align_payload(struct ath12k_dp *dp, struct sk_buff **pskb) { - u32 iova_mask = ab->hw_params->iova_mask; + u32 iova_mask = dp->hw_params->iova_mask; unsigned long offset, delta1, delta2; struct sk_buff *skb2, *skb = *pskb; unsigned int headroom = skb_headroom(skb); @@ -218,1564 +181,33 @@ static int ath12k_dp_tx_align_payload(struct ath12k_base *ab, out: return ret; } +EXPORT_SYMBOL(ath12k_dp_tx_align_payload); -int ath12k_dp_tx(struct ath12k *ar, struct ath12k_link_vif *arvif, - struct sk_buff *skb, bool gsn_valid, int mcbc_gsn, - bool is_mcast) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_dp *dp = &ab->dp; - struct hal_tx_info ti = {}; - struct ath12k_tx_desc_info *tx_desc; - struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); - struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb); - struct hal_tcl_data_cmd *hal_tcl_desc; - struct hal_tx_msdu_ext_desc *msg; - struct sk_buff *skb_ext_desc = NULL; - struct hal_srng *tcl_ring; - struct ieee80211_hdr *hdr = (void *)skb->data; - struct ath12k_vif *ahvif = arvif->ahvif; - struct dp_tx_ring *tx_ring; - u8 pool_id; - u8 hal_ring_id; - int ret; - u8 ring_selector, ring_map = 0; - bool tcl_ring_retry; - bool msdu_ext_desc = false; - bool add_htt_metadata = false; - u32 iova_mask = ab->hw_params->iova_mask; - bool is_diff_encap = false; - bool is_null_frame = false; - - if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ar->ab->dev_flags)) - return -ESHUTDOWN; - - if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && - !ieee80211_is_data(hdr->frame_control)) - return -EOPNOTSUPP; - - pool_id = skb_get_queue_mapping(skb) & (ATH12K_HW_MAX_QUEUES - 1); - - /* Let the default ring selection be based on current processor - * number, where one of the 3 tcl rings are selected based on - * the smp_processor_id(). In case that ring - * is full/busy, we resort to other available rings. - * If all rings are full, we drop the packet. - * TODO: Add throttling logic when all rings are full - */ - ring_selector = ab->hw_params->hw_ops->get_ring_selector(skb); - -tcl_ring_sel: - tcl_ring_retry = false; - ti.ring_id = ring_selector % ab->hw_params->max_tx_ring; - - ring_map |= BIT(ti.ring_id); - ti.rbm_id = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map[ti.ring_id].rbm_id; - - tx_ring = &dp->tx_ring[ti.ring_id]; - - tx_desc = ath12k_dp_tx_assign_buffer(dp, pool_id); - if (!tx_desc) - return -ENOMEM; - - ti.bank_id = arvif->bank_id; - ti.meta_data_flags = arvif->tcl_metadata; - - if (ahvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW && - test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags)) { - if (skb_cb->flags & ATH12K_SKB_CIPHER_SET) { - ti.encrypt_type = - ath12k_dp_tx_get_encrypt_type(skb_cb->cipher); - - if (ieee80211_has_protected(hdr->frame_control)) - skb_put(skb, IEEE80211_CCMP_MIC_LEN); - } else { - ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN; - } - - msdu_ext_desc = true; - } - - if (gsn_valid) { - /* Reset and Initialize meta_data_flags with Global Sequence - * Number (GSN) info. - */ - ti.meta_data_flags = - u32_encode_bits(HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM, - HTT_TCL_META_DATA_TYPE) | - u32_encode_bits(mcbc_gsn, HTT_TCL_META_DATA_GLOBAL_SEQ_NUM); - } - - ti.encap_type = ath12k_dp_tx_get_encap_type(ab, skb); - ti.addr_search_flags = arvif->hal_addr_search_flags; - ti.search_type = arvif->search_type; - ti.type = HAL_TCL_DESC_TYPE_BUFFER; - ti.pkt_offset = 0; - ti.lmac_id = ar->lmac_id; - - ti.vdev_id = arvif->vdev_id; - if (gsn_valid) - ti.vdev_id += HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID; - - ti.bss_ast_hash = arvif->ast_hash; - ti.bss_ast_idx = arvif->ast_idx; - ti.dscp_tid_tbl_idx = 0; - - if (skb->ip_summed == CHECKSUM_PARTIAL && - ti.encap_type != HAL_TCL_ENCAP_TYPE_RAW) { - ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_IP4_CKSUM_EN) | - u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP4_CKSUM_EN) | - u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP6_CKSUM_EN) | - u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP4_CKSUM_EN) | - u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP6_CKSUM_EN); - } - - ti.flags1 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO3_TID_OVERWRITE); - - ti.tid = ath12k_dp_tx_get_tid(skb); - - switch (ti.encap_type) { - case HAL_TCL_ENCAP_TYPE_NATIVE_WIFI: - is_null_frame = ieee80211_is_nullfunc(hdr->frame_control); - if (ahvif->vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) { - if (skb->protocol == cpu_to_be16(ETH_P_PAE) || is_null_frame) - is_diff_encap = true; - - /* Firmware expects msdu ext descriptor for nwifi/raw packets - * received in ETH mode. Without this, observed tx fail for - * Multicast packets in ETH mode. - */ - msdu_ext_desc = true; - } else { - ath12k_dp_tx_encap_nwifi(skb); - } - break; - case HAL_TCL_ENCAP_TYPE_RAW: - if (!test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) { - ret = -EINVAL; - goto fail_remove_tx_buf; - } - break; - case HAL_TCL_ENCAP_TYPE_ETHERNET: - /* no need to encap */ - break; - case HAL_TCL_ENCAP_TYPE_802_3: - default: - /* TODO: Take care of other encap modes as well */ - ret = -EINVAL; - atomic_inc(&ab->device_stats.tx_err.misc_fail); - goto fail_remove_tx_buf; - } - - if (iova_mask && - (unsigned long)skb->data & iova_mask) { - ret = ath12k_dp_tx_align_payload(ab, &skb); - if (ret) { - ath12k_warn(ab, "failed to align TX buffer %d\n", ret); - /* don't bail out, give original buffer - * a chance even unaligned. - */ - goto map; - } - - /* hdr is pointing to a wrong place after alignment, - * so refresh it for later use. - */ - hdr = (void *)skb->data; - } -map: - ti.paddr = dma_map_single(ab->dev, skb->data, skb->len, DMA_TO_DEVICE); - if (dma_mapping_error(ab->dev, ti.paddr)) { - atomic_inc(&ab->device_stats.tx_err.misc_fail); - ath12k_warn(ab, "failed to DMA map data Tx buffer\n"); - ret = -ENOMEM; - goto fail_remove_tx_buf; - } - - if ((!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags) && - !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) && - !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) && - ieee80211_has_protected(hdr->frame_control)) || - is_diff_encap) { - /* Firmware is not expecting meta data for qos null - * nwifi packet received in ETH encap mode. - */ - if (is_null_frame && msdu_ext_desc) - goto skip_htt_meta; - - /* Add metadata for sw encrypted vlan group traffic - * and EAPOL nwifi packet received in ETH encap mode. - */ - add_htt_metadata = true; - msdu_ext_desc = true; - ti.meta_data_flags |= HTT_TCL_META_DATA_VALID_HTT; -skip_htt_meta: - ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TO_FW); - ti.encap_type = HAL_TCL_ENCAP_TYPE_RAW; - ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN; - } - - tx_desc->skb = skb; - tx_desc->mac_id = ar->pdev_idx; - ti.desc_id = tx_desc->desc_id; - ti.data_len = skb->len; - skb_cb->paddr = ti.paddr; - skb_cb->vif = ahvif->vif; - skb_cb->ar = ar; - - if (msdu_ext_desc) { - skb_ext_desc = dev_alloc_skb(sizeof(struct hal_tx_msdu_ext_desc)); - if (!skb_ext_desc) { - ret = -ENOMEM; - goto fail_unmap_dma; - } - - skb_put(skb_ext_desc, sizeof(struct hal_tx_msdu_ext_desc)); - memset(skb_ext_desc->data, 0, skb_ext_desc->len); - - msg = (struct hal_tx_msdu_ext_desc *)skb_ext_desc->data; - ath12k_hal_tx_cmd_ext_desc_setup(ab, msg, &ti); - - if (add_htt_metadata) { - ret = ath12k_dp_prepare_htt_metadata(skb_ext_desc); - if (ret < 0) { - ath12k_dbg(ab, ATH12K_DBG_DP_TX, - "Failed to add HTT meta data, dropping packet\n"); - goto fail_free_ext_skb; - } - } - - ti.paddr = dma_map_single(ab->dev, skb_ext_desc->data, - skb_ext_desc->len, DMA_TO_DEVICE); - ret = dma_mapping_error(ab->dev, ti.paddr); - if (ret) - goto fail_free_ext_skb; - - ti.data_len = skb_ext_desc->len; - ti.type = HAL_TCL_DESC_TYPE_EXT_DESC; - - skb_cb->paddr_ext_desc = ti.paddr; - tx_desc->skb_ext_desc = skb_ext_desc; - } - - hal_ring_id = tx_ring->tcl_data_ring.ring_id; - tcl_ring = &ab->hal.srng_list[hal_ring_id]; - - spin_lock_bh(&tcl_ring->lock); - - ath12k_hal_srng_access_begin(ab, tcl_ring); - - hal_tcl_desc = ath12k_hal_srng_src_get_next_entry(ab, tcl_ring); - if (!hal_tcl_desc) { - /* NOTE: It is highly unlikely we'll be running out of tcl_ring - * desc because the desc is directly enqueued onto hw queue. - */ - ath12k_hal_srng_access_end(ab, tcl_ring); - ab->device_stats.tx_err.desc_na[ti.ring_id]++; - spin_unlock_bh(&tcl_ring->lock); - ret = -ENOMEM; - - /* Checking for available tcl descriptors in another ring in - * case of failure due to full tcl ring now, is better than - * checking this ring earlier for each pkt tx. - * Restart ring selection if some rings are not checked yet. - */ - if (ring_map != (BIT(ab->hw_params->max_tx_ring) - 1) && - ab->hw_params->tcl_ring_retry) { - tcl_ring_retry = true; - ring_selector++; - } - - goto fail_unmap_dma_ext; - } - - spin_lock_bh(&arvif->link_stats_lock); - arvif->link_stats.tx_encap_type[ti.encap_type]++; - arvif->link_stats.tx_encrypt_type[ti.encrypt_type]++; - arvif->link_stats.tx_desc_type[ti.type]++; - - if (is_mcast) - arvif->link_stats.tx_bcast_mcast++; - else - arvif->link_stats.tx_enqueued++; - spin_unlock_bh(&arvif->link_stats_lock); - - ab->device_stats.tx_enqueued[ti.ring_id]++; - - ath12k_hal_tx_cmd_desc_setup(ab, hal_tcl_desc, &ti); - - ath12k_hal_srng_access_end(ab, tcl_ring); - - spin_unlock_bh(&tcl_ring->lock); - - ath12k_dbg_dump(ab, ATH12K_DBG_DP_TX, NULL, "dp tx msdu: ", - skb->data, skb->len); - - atomic_inc(&ar->dp.num_tx_pending); - - return 0; - -fail_unmap_dma_ext: - if (skb_cb->paddr_ext_desc) - dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc, - skb_ext_desc->len, - DMA_TO_DEVICE); -fail_free_ext_skb: - kfree_skb(skb_ext_desc); - -fail_unmap_dma: - dma_unmap_single(ab->dev, ti.paddr, ti.data_len, DMA_TO_DEVICE); - -fail_remove_tx_buf: - ath12k_dp_tx_release_txbuf(dp, tx_desc, pool_id); - - spin_lock_bh(&arvif->link_stats_lock); - arvif->link_stats.tx_dropped++; - spin_unlock_bh(&arvif->link_stats_lock); - - if (tcl_ring_retry) - goto tcl_ring_sel; - - return ret; -} - -static void ath12k_dp_tx_free_txbuf(struct ath12k_base *ab, - struct dp_tx_ring *tx_ring, - struct ath12k_tx_desc_params *desc_params) +void ath12k_dp_tx_free_txbuf(struct ath12k_dp *dp, + struct dp_tx_ring *tx_ring, + struct ath12k_tx_desc_params *desc_params) { - struct ath12k *ar; + struct ath12k_pdev_dp *dp_pdev; struct sk_buff *msdu = desc_params->skb; struct ath12k_skb_cb *skb_cb; - u8 pdev_id = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, desc_params->mac_id); + u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, desc_params->mac_id); skb_cb = ATH12K_SKB_CB(msdu); - ar = ab->pdevs[pdev_id].ar; - dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); + dma_unmap_single(dp->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); if (skb_cb->paddr_ext_desc) { - dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc, + dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc, desc_params->skb_ext_desc->len, DMA_TO_DEVICE); dev_kfree_skb_any(desc_params->skb_ext_desc); } - ieee80211_free_txskb(ar->ah->hw, msdu); - - if (atomic_dec_and_test(&ar->dp.num_tx_pending)) - wake_up(&ar->dp.tx_empty_waitq); -} - -static void -ath12k_dp_tx_htt_tx_complete_buf(struct ath12k_base *ab, - struct ath12k_tx_desc_params *desc_params, - struct dp_tx_ring *tx_ring, - struct ath12k_dp_htt_wbm_tx_status *ts, - u16 peer_id) -{ - struct ieee80211_tx_info *info; - struct ath12k_link_vif *arvif; - struct ath12k_skb_cb *skb_cb; - struct ieee80211_vif *vif; - struct ath12k_vif *ahvif; - struct ath12k *ar; - struct sk_buff *msdu = desc_params->skb; - s32 noise_floor; - struct ieee80211_tx_status status = {}; - struct ath12k_peer *peer; - - skb_cb = ATH12K_SKB_CB(msdu); - info = IEEE80211_SKB_CB(msdu); - - ar = skb_cb->ar; - ab->device_stats.tx_completed[tx_ring->tcl_data_ring_id]++; - - if (atomic_dec_and_test(&ar->dp.num_tx_pending)) - wake_up(&ar->dp.tx_empty_waitq); - - dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); - if (skb_cb->paddr_ext_desc) { - dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc, - desc_params->skb_ext_desc->len, DMA_TO_DEVICE); - dev_kfree_skb_any(desc_params->skb_ext_desc); - } - - vif = skb_cb->vif; - if (vif) { - ahvif = ath12k_vif_to_ahvif(vif); - rcu_read_lock(); - arvif = rcu_dereference(ahvif->link[skb_cb->link_id]); - if (arvif) { - spin_lock_bh(&arvif->link_stats_lock); - arvif->link_stats.tx_completed++; - spin_unlock_bh(&arvif->link_stats_lock); - } - rcu_read_unlock(); - } - - memset(&info->status, 0, sizeof(info->status)); - - if (ts->acked) { - if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) { - info->flags |= IEEE80211_TX_STAT_ACK; - info->status.ack_signal = ts->ack_rssi; - - if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, - ab->wmi_ab.svc_map)) { - spin_lock_bh(&ar->data_lock); - noise_floor = ath12k_pdev_get_noise_floor(ar); - spin_unlock_bh(&ar->data_lock); - - info->status.ack_signal += noise_floor; - } - - info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID; - } else { - info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; - } - } - rcu_read_lock(); - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, peer_id); - if (!peer || !peer->sta) { - ath12k_dbg(ab, ATH12K_DBG_DATA, - "dp_tx: failed to find the peer with peer_id %d\n", peer_id); - spin_unlock_bh(&ab->base_lock); - ieee80211_free_txskb(ath12k_ar_to_hw(ar), msdu); - goto exit; - } else { - status.sta = peer->sta; - } - spin_unlock_bh(&ab->base_lock); - - status.info = info; - status.skb = msdu; - ieee80211_tx_status_ext(ath12k_ar_to_hw(ar), &status); -exit: - rcu_read_unlock(); -} - -static void -ath12k_dp_tx_process_htt_tx_complete(struct ath12k_base *ab, void *desc, - struct dp_tx_ring *tx_ring, - struct ath12k_tx_desc_params *desc_params) -{ - struct htt_tx_wbm_completion *status_desc; - struct ath12k_dp_htt_wbm_tx_status ts = {}; - enum hal_wbm_htt_tx_comp_status wbm_status; - u16 peer_id; - - status_desc = desc; - - wbm_status = le32_get_bits(status_desc->info0, - HTT_TX_WBM_COMP_INFO0_STATUS); - ab->device_stats.fw_tx_status[wbm_status]++; - - switch (wbm_status) { - case HAL_WBM_REL_HTT_TX_COMP_STATUS_OK: - ts.acked = (wbm_status == HAL_WBM_REL_HTT_TX_COMP_STATUS_OK); - ts.ack_rssi = le32_get_bits(status_desc->info2, - HTT_TX_WBM_COMP_INFO2_ACK_RSSI); - - peer_id = le32_get_bits(((struct hal_wbm_completion_ring_tx *)desc)-> - info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID); - - ath12k_dp_tx_htt_tx_complete_buf(ab, desc_params, tx_ring, &ts, peer_id); - break; - case HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP: - case HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL: - case HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ: - case HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT: - case HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH: - ath12k_dp_tx_free_txbuf(ab, tx_ring, desc_params); - break; - case HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY: - /* This event is to be handled only when the driver decides to - * use WDS offload functionality. - */ - break; - default: - ath12k_warn(ab, "Unknown htt wbm tx status %d\n", wbm_status); - break; - } -} - -static void ath12k_dp_tx_update_txcompl(struct ath12k *ar, struct hal_tx_status *ts) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_peer *peer; - struct ieee80211_sta *sta; - struct ath12k_sta *ahsta; - struct ath12k_link_sta *arsta; - struct rate_info txrate = {}; - u16 rate, ru_tones; - u8 rate_idx = 0; - int ret; - - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, ts->peer_id); - if (!peer || !peer->sta) { - ath12k_dbg(ab, ATH12K_DBG_DP_TX, - "failed to find the peer by id %u\n", ts->peer_id); - spin_unlock_bh(&ab->base_lock); - return; - } - sta = peer->sta; - ahsta = ath12k_sta_to_ahsta(sta); - arsta = &ahsta->deflink; - - /* This is to prefer choose the real NSS value arsta->last_txrate.nss, - * if it is invalid, then choose the NSS value while assoc. - */ - if (arsta->last_txrate.nss) - txrate.nss = arsta->last_txrate.nss; - else - txrate.nss = arsta->peer_nss; - spin_unlock_bh(&ab->base_lock); - - switch (ts->pkt_type) { - case HAL_TX_RATE_STATS_PKT_TYPE_11A: - case HAL_TX_RATE_STATS_PKT_TYPE_11B: - ret = ath12k_mac_hw_ratecode_to_legacy_rate(ts->mcs, - ts->pkt_type, - &rate_idx, - &rate); - if (ret < 0) { - ath12k_warn(ab, "Invalid tx legacy rate %d\n", ret); - return; - } - - txrate.legacy = rate; - break; - case HAL_TX_RATE_STATS_PKT_TYPE_11N: - if (ts->mcs > ATH12K_HT_MCS_MAX) { - ath12k_warn(ab, "Invalid HT mcs index %d\n", ts->mcs); - return; - } - - if (txrate.nss != 0) - txrate.mcs = ts->mcs + 8 * (txrate.nss - 1); - - txrate.flags = RATE_INFO_FLAGS_MCS; - - if (ts->sgi) - txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; - break; - case HAL_TX_RATE_STATS_PKT_TYPE_11AC: - if (ts->mcs > ATH12K_VHT_MCS_MAX) { - ath12k_warn(ab, "Invalid VHT mcs index %d\n", ts->mcs); - return; - } - - txrate.mcs = ts->mcs; - txrate.flags = RATE_INFO_FLAGS_VHT_MCS; - - if (ts->sgi) - txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; - break; - case HAL_TX_RATE_STATS_PKT_TYPE_11AX: - if (ts->mcs > ATH12K_HE_MCS_MAX) { - ath12k_warn(ab, "Invalid HE mcs index %d\n", ts->mcs); - return; - } - - txrate.mcs = ts->mcs; - txrate.flags = RATE_INFO_FLAGS_HE_MCS; - txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(ts->sgi); - break; - case HAL_TX_RATE_STATS_PKT_TYPE_11BE: - if (ts->mcs > ATH12K_EHT_MCS_MAX) { - ath12k_warn(ab, "Invalid EHT mcs index %d\n", ts->mcs); - return; - } - - txrate.mcs = ts->mcs; - txrate.flags = RATE_INFO_FLAGS_EHT_MCS; - txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(ts->sgi); - break; - default: - ath12k_warn(ab, "Invalid tx pkt type: %d\n", ts->pkt_type); - return; - } - - txrate.bw = ath12k_mac_bw_to_mac80211_bw(ts->bw); - - if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11AX) { - txrate.bw = RATE_INFO_BW_HE_RU; - ru_tones = ath12k_mac_he_convert_tones_to_ru_tones(ts->tones); - txrate.he_ru_alloc = - ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones); - } - - if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11BE) { - txrate.bw = RATE_INFO_BW_EHT_RU; - txrate.eht_ru_alloc = - ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(ts->tones); - } - - spin_lock_bh(&ab->base_lock); - arsta->txrate = txrate; - spin_unlock_bh(&ab->base_lock); -} - -static void ath12k_dp_tx_complete_msdu(struct ath12k *ar, - struct ath12k_tx_desc_params *desc_params, - struct hal_tx_status *ts, - int ring) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_hw *ah = ar->ah; - struct ieee80211_tx_info *info; - struct ath12k_link_vif *arvif; - struct ath12k_skb_cb *skb_cb; - struct ieee80211_vif *vif; - struct ath12k_vif *ahvif; - struct sk_buff *msdu = desc_params->skb; - s32 noise_floor; - struct ieee80211_tx_status status = {}; - struct ieee80211_rate_status status_rate = {}; - struct ath12k_peer *peer; - struct ath12k_link_sta *arsta; - struct ath12k_sta *ahsta; - struct rate_info rate; - - if (WARN_ON_ONCE(ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM)) { - /* Must not happen */ - return; - } - - skb_cb = ATH12K_SKB_CB(msdu); - ab->device_stats.tx_completed[ring]++; - - dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); - if (skb_cb->paddr_ext_desc) { - dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc, - desc_params->skb_ext_desc->len, DMA_TO_DEVICE); - dev_kfree_skb_any(desc_params->skb_ext_desc); - } - - rcu_read_lock(); - - if (!rcu_dereference(ab->pdevs_active[ar->pdev_idx])) { - ieee80211_free_txskb(ah->hw, msdu); - goto exit; - } - - if (!skb_cb->vif) { - ieee80211_free_txskb(ah->hw, msdu); - goto exit; - } - - vif = skb_cb->vif; - if (vif) { - ahvif = ath12k_vif_to_ahvif(vif); - arvif = rcu_dereference(ahvif->link[skb_cb->link_id]); - if (arvif) { - spin_lock_bh(&arvif->link_stats_lock); - arvif->link_stats.tx_completed++; - spin_unlock_bh(&arvif->link_stats_lock); - } - } - - info = IEEE80211_SKB_CB(msdu); - memset(&info->status, 0, sizeof(info->status)); - - /* skip tx rate update from ieee80211_status*/ - info->status.rates[0].idx = -1; - - switch (ts->status) { - case HAL_WBM_TQM_REL_REASON_FRAME_ACKED: - if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) { - info->flags |= IEEE80211_TX_STAT_ACK; - info->status.ack_signal = ts->ack_rssi; - - if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, - ab->wmi_ab.svc_map)) { - spin_lock_bh(&ar->data_lock); - noise_floor = ath12k_pdev_get_noise_floor(ar); - spin_unlock_bh(&ar->data_lock); - - info->status.ack_signal += noise_floor; - } - - info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID; - } - break; - case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX: - if (info->flags & IEEE80211_TX_CTL_NO_ACK) { - info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; - break; - } - fallthrough; - case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU: - case HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD: - case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES: - /* The failure status is due to internal firmware tx failure - * hence drop the frame; do not update the status of frame to - * the upper layer - */ - ieee80211_free_txskb(ah->hw, msdu); - goto exit; - default: - ath12k_dbg(ab, ATH12K_DBG_DP_TX, "tx frame is not acked status %d\n", - ts->status); - break; - } - - /* NOTE: Tx rate status reporting. Tx completion status does not have - * necessary information (for example nss) to build the tx rate. - * Might end up reporting it out-of-band from HTT stats. - */ - - ath12k_dp_tx_update_txcompl(ar, ts); - - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_id(ab, ts->peer_id); - if (!peer || !peer->sta) { - ath12k_err(ab, - "dp_tx: failed to find the peer with peer_id %d\n", - ts->peer_id); - spin_unlock_bh(&ab->base_lock); - ieee80211_free_txskb(ath12k_ar_to_hw(ar), msdu); - goto exit; - } - ahsta = ath12k_sta_to_ahsta(peer->sta); - arsta = &ahsta->deflink; - - spin_unlock_bh(&ab->base_lock); - - status.sta = peer->sta; - status.info = info; - status.skb = msdu; - rate = arsta->last_txrate; - - status_rate.rate_idx = rate; - status_rate.try_count = 1; - - status.rates = &status_rate; - status.n_rates = 1; - ieee80211_tx_status_ext(ath12k_ar_to_hw(ar), &status); - -exit: - rcu_read_unlock(); -} - -static void ath12k_dp_tx_status_parse(struct ath12k_base *ab, - struct hal_wbm_completion_ring_tx *desc, - struct hal_tx_status *ts) -{ - u32 info0 = le32_to_cpu(desc->rate_stats.info0); - - ts->buf_rel_source = - le32_get_bits(desc->info0, HAL_WBM_COMPL_TX_INFO0_REL_SRC_MODULE); - if (ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_FW && - ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM) - return; - - if (ts->buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) - return; - - ts->status = le32_get_bits(desc->info0, - HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON); - - ts->ppdu_id = le32_get_bits(desc->info1, - HAL_WBM_COMPL_TX_INFO1_TQM_STATUS_NUMBER); - - ts->peer_id = le32_get_bits(desc->info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID); - - ts->ack_rssi = le32_get_bits(desc->info2, - HAL_WBM_COMPL_TX_INFO2_ACK_FRAME_RSSI); - - if (info0 & HAL_TX_RATE_STATS_INFO0_VALID) { - ts->pkt_type = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_PKT_TYPE); - ts->mcs = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_MCS); - ts->sgi = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_SGI); - ts->bw = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_BW); - ts->tones = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_TONES_IN_RU); - ts->ofdma = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_OFDMA_TX); - } -} - -void ath12k_dp_tx_completion_handler(struct ath12k_base *ab, int ring_id) -{ - struct ath12k *ar; - struct ath12k_dp *dp = &ab->dp; - int hal_ring_id = dp->tx_ring[ring_id].tcl_comp_ring.ring_id; - struct hal_srng *status_ring = &ab->hal.srng_list[hal_ring_id]; - struct ath12k_tx_desc_info *tx_desc = NULL; - struct hal_tx_status ts = {}; - struct ath12k_tx_desc_params desc_params; - struct dp_tx_ring *tx_ring = &dp->tx_ring[ring_id]; - struct hal_wbm_release_ring *desc; - u8 pdev_id; - u64 desc_va; - enum hal_wbm_rel_src_module buf_rel_source; - enum hal_wbm_tqm_rel_reason rel_status; - - spin_lock_bh(&status_ring->lock); - - ath12k_hal_srng_access_begin(ab, status_ring); - - while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) != - tx_ring->tx_status_tail) { - desc = ath12k_hal_srng_dst_get_next_entry(ab, status_ring); - if (!desc) - break; - - memcpy(&tx_ring->tx_status[tx_ring->tx_status_head], - desc, sizeof(*desc)); - tx_ring->tx_status_head = - ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head); - } - - if (ath12k_hal_srng_dst_peek(ab, status_ring) && - (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) == - tx_ring->tx_status_tail)) { - /* TODO: Process pending tx_status messages when kfifo_is_full() */ - ath12k_warn(ab, "Unable to process some of the tx_status ring desc because status_fifo is full\n"); - } - - ath12k_hal_srng_access_end(ab, status_ring); - - spin_unlock_bh(&status_ring->lock); - - while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail) != - tx_ring->tx_status_head) { - struct hal_wbm_completion_ring_tx *tx_status; - u32 desc_id; - - tx_ring->tx_status_tail = - ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail); - tx_status = &tx_ring->tx_status[tx_ring->tx_status_tail]; - ath12k_dp_tx_status_parse(ab, tx_status, &ts); - - if (le32_get_bits(tx_status->info0, HAL_WBM_COMPL_TX_INFO0_CC_DONE)) { - /* HW done cookie conversion */ - desc_va = ((u64)le32_to_cpu(tx_status->buf_va_hi) << 32 | - le32_to_cpu(tx_status->buf_va_lo)); - tx_desc = (struct ath12k_tx_desc_info *)((unsigned long)desc_va); - } else { - /* SW does cookie conversion to VA */ - desc_id = le32_get_bits(tx_status->buf_va_hi, - BUFFER_ADDR_INFO1_SW_COOKIE); - - tx_desc = ath12k_dp_get_tx_desc(ab, desc_id); - } - if (!tx_desc) { - ath12k_warn(ab, "unable to retrieve tx_desc!"); - continue; - } - - desc_params.mac_id = tx_desc->mac_id; - desc_params.skb = tx_desc->skb; - desc_params.skb_ext_desc = tx_desc->skb_ext_desc; - - /* Find the HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE value */ - buf_rel_source = le32_get_bits(tx_status->info0, - HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE); - ab->device_stats.tx_wbm_rel_source[buf_rel_source]++; - - rel_status = le32_get_bits(tx_status->info0, - HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON); - ab->device_stats.tqm_rel_reason[rel_status]++; - - /* Release descriptor as soon as extracting necessary info - * to reduce contention - */ - ath12k_dp_tx_release_txbuf(dp, tx_desc, tx_desc->pool_id); - if (ts.buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) { - ath12k_dp_tx_process_htt_tx_complete(ab, (void *)tx_status, - tx_ring, &desc_params); - continue; - } - - pdev_id = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, desc_params.mac_id); - ar = ab->pdevs[pdev_id].ar; - - if (atomic_dec_and_test(&ar->dp.num_tx_pending)) - wake_up(&ar->dp.tx_empty_waitq); - - ath12k_dp_tx_complete_msdu(ar, &desc_params, &ts, - tx_ring->tcl_data_ring_id); - } -} - -static int -ath12k_dp_tx_get_ring_id_type(struct ath12k_base *ab, - int mac_id, u32 ring_id, - enum hal_ring_type ring_type, - enum htt_srng_ring_type *htt_ring_type, - enum htt_srng_ring_id *htt_ring_id) -{ - int ret = 0; - - switch (ring_type) { - case HAL_RXDMA_BUF: - /* for some targets, host fills rx buffer to fw and fw fills to - * rxbuf ring for each rxdma - */ - if (!ab->hw_params->rx_mac_buf_ring) { - if (!(ring_id == HAL_SRNG_SW2RXDMA_BUF0 || - ring_id == HAL_SRNG_SW2RXDMA_BUF1)) { - ret = -EINVAL; - } - *htt_ring_id = HTT_RXDMA_HOST_BUF_RING; - *htt_ring_type = HTT_SW_TO_HW_RING; - } else { - if (ring_id == HAL_SRNG_SW2RXDMA_BUF0) { - *htt_ring_id = HTT_HOST1_TO_FW_RXBUF_RING; - *htt_ring_type = HTT_SW_TO_SW_RING; - } else { - *htt_ring_id = HTT_RXDMA_HOST_BUF_RING; - *htt_ring_type = HTT_SW_TO_HW_RING; - } - } - break; - case HAL_RXDMA_DST: - *htt_ring_id = HTT_RXDMA_NON_MONITOR_DEST_RING; - *htt_ring_type = HTT_HW_TO_SW_RING; - break; - case HAL_RXDMA_MONITOR_BUF: - *htt_ring_id = HTT_RX_MON_HOST2MON_BUF_RING; - *htt_ring_type = HTT_SW_TO_HW_RING; - break; - case HAL_RXDMA_MONITOR_STATUS: - *htt_ring_id = HTT_RXDMA_MONITOR_STATUS_RING; - *htt_ring_type = HTT_SW_TO_HW_RING; - break; - case HAL_RXDMA_MONITOR_DST: - *htt_ring_id = HTT_RX_MON_MON2HOST_DEST_RING; - *htt_ring_type = HTT_HW_TO_SW_RING; - break; - case HAL_RXDMA_MONITOR_DESC: - *htt_ring_id = HTT_RXDMA_MONITOR_DESC_RING; - *htt_ring_type = HTT_SW_TO_HW_RING; - break; - default: - ath12k_warn(ab, "Unsupported ring type in DP :%d\n", ring_type); - ret = -EINVAL; - } - return ret; -} - -int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id, - int mac_id, enum hal_ring_type ring_type) -{ - struct htt_srng_setup_cmd *cmd; - struct hal_srng *srng = &ab->hal.srng_list[ring_id]; - struct hal_srng_params params; - struct sk_buff *skb; - u32 ring_entry_sz; - int len = sizeof(*cmd); - dma_addr_t hp_addr, tp_addr; - enum htt_srng_ring_type htt_ring_type; - enum htt_srng_ring_id htt_ring_id; - int ret; - - skb = ath12k_htc_alloc_skb(ab, len); - if (!skb) - return -ENOMEM; - - memset(¶ms, 0, sizeof(params)); - ath12k_hal_srng_get_params(ab, srng, ¶ms); - - hp_addr = ath12k_hal_srng_get_hp_addr(ab, srng); - tp_addr = ath12k_hal_srng_get_tp_addr(ab, srng); - - ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id, - ring_type, &htt_ring_type, - &htt_ring_id); - if (ret) - goto err_free; + guard(rcu)(); - skb_put(skb, len); - cmd = (struct htt_srng_setup_cmd *)skb->data; - cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_SRING_SETUP, - HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE); - if (htt_ring_type == HTT_SW_TO_HW_RING || - htt_ring_type == HTT_HW_TO_SW_RING) - cmd->info0 |= le32_encode_bits(DP_SW2HW_MACID(mac_id), - HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID); - else - cmd->info0 |= le32_encode_bits(mac_id, - HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID); - cmd->info0 |= le32_encode_bits(htt_ring_type, - HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE); - cmd->info0 |= le32_encode_bits(htt_ring_id, - HTT_SRNG_SETUP_CMD_INFO0_RING_ID); - - cmd->ring_base_addr_lo = cpu_to_le32(params.ring_base_paddr & - HAL_ADDR_LSB_REG_MASK); - - cmd->ring_base_addr_hi = cpu_to_le32((u64)params.ring_base_paddr >> - HAL_ADDR_MSB_REG_SHIFT); - - ret = ath12k_hal_srng_get_entrysize(ab, ring_type); - if (ret < 0) - goto err_free; - - ring_entry_sz = ret; - - ring_entry_sz >>= 2; - cmd->info1 = le32_encode_bits(ring_entry_sz, - HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE); - cmd->info1 |= le32_encode_bits(params.num_entries * ring_entry_sz, - HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE); - cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP), - HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP); - cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP), - HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP); - cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_RING_PTR_SWAP), - HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP); - if (htt_ring_type == HTT_SW_TO_HW_RING) - cmd->info1 |= cpu_to_le32(HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS); - - cmd->ring_head_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(hp_addr)); - cmd->ring_head_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(hp_addr)); - - cmd->ring_tail_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(tp_addr)); - cmd->ring_tail_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(tp_addr)); - - cmd->ring_msi_addr_lo = cpu_to_le32(lower_32_bits(params.msi_addr)); - cmd->ring_msi_addr_hi = cpu_to_le32(upper_32_bits(params.msi_addr)); - cmd->msi_data = cpu_to_le32(params.msi_data); - - cmd->intr_info = - le32_encode_bits(params.intr_batch_cntr_thres_entries * ring_entry_sz, - HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH); - cmd->intr_info |= - le32_encode_bits(params.intr_timer_thres_us >> 3, - HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH); - - cmd->info2 = 0; - if (params.flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) { - cmd->info2 = le32_encode_bits(params.low_threshold, - HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH); - } + dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx); - ath12k_dbg(ab, ATH12K_DBG_HAL, - "%s msi_addr_lo:0x%x, msi_addr_hi:0x%x, msi_data:0x%x\n", - __func__, cmd->ring_msi_addr_lo, cmd->ring_msi_addr_hi, - cmd->msi_data); + ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu); - ath12k_dbg(ab, ATH12K_DBG_HAL, - "ring_id:%d, ring_type:%d, intr_info:0x%x, flags:0x%x\n", - ring_id, ring_type, cmd->intr_info, cmd->info2); - - ret = ath12k_htc_send(&ab->htc, ab->dp.eid, skb); - if (ret) - goto err_free; - - return 0; - -err_free: - dev_kfree_skb_any(skb); - - return ret; -} - -#define HTT_TARGET_VERSION_TIMEOUT_HZ (3 * HZ) - -int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab) -{ - struct ath12k_dp *dp = &ab->dp; - struct sk_buff *skb; - struct htt_ver_req_cmd *cmd; - int len = sizeof(*cmd); - u32 metadata_version; - int ret; - - init_completion(&dp->htt_tgt_version_received); - - skb = ath12k_htc_alloc_skb(ab, len); - if (!skb) - return -ENOMEM; - - skb_put(skb, len); - cmd = (struct htt_ver_req_cmd *)skb->data; - cmd->ver_reg_info = le32_encode_bits(HTT_H2T_MSG_TYPE_VERSION_REQ, - HTT_OPTION_TAG); - metadata_version = ath12k_ftm_mode ? HTT_OPTION_TCL_METADATA_VER_V1 : - HTT_OPTION_TCL_METADATA_VER_V2; - - cmd->tcl_metadata_version = le32_encode_bits(HTT_TAG_TCL_METADATA_VERSION, - HTT_OPTION_TAG) | - le32_encode_bits(HTT_TCL_METADATA_VER_SZ, - HTT_OPTION_LEN) | - le32_encode_bits(metadata_version, - HTT_OPTION_VALUE); - - ret = ath12k_htc_send(&ab->htc, dp->eid, skb); - if (ret) { - dev_kfree_skb_any(skb); - return ret; - } - - ret = wait_for_completion_timeout(&dp->htt_tgt_version_received, - HTT_TARGET_VERSION_TIMEOUT_HZ); - if (ret == 0) { - ath12k_warn(ab, "htt target version request timed out\n"); - return -ETIMEDOUT; - } - - if (dp->htt_tgt_ver_major != HTT_TARGET_VERSION_MAJOR) { - ath12k_err(ab, "unsupported htt major version %d supported version is %d\n", - dp->htt_tgt_ver_major, HTT_TARGET_VERSION_MAJOR); - return -EOPNOTSUPP; - } - - return 0; -} - -int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_dp *dp = &ab->dp; - struct sk_buff *skb; - struct htt_ppdu_stats_cfg_cmd *cmd; - int len = sizeof(*cmd); - u8 pdev_mask; - int ret; - int i; - - for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { - skb = ath12k_htc_alloc_skb(ab, len); - if (!skb) - return -ENOMEM; - - skb_put(skb, len); - cmd = (struct htt_ppdu_stats_cfg_cmd *)skb->data; - cmd->msg = le32_encode_bits(HTT_H2T_MSG_TYPE_PPDU_STATS_CFG, - HTT_PPDU_STATS_CFG_MSG_TYPE); - - pdev_mask = 1 << (i + ar->pdev_idx); - cmd->msg |= le32_encode_bits(pdev_mask, HTT_PPDU_STATS_CFG_PDEV_ID); - cmd->msg |= le32_encode_bits(mask, HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK); - - ret = ath12k_htc_send(&ab->htc, dp->eid, skb); - if (ret) { - dev_kfree_skb_any(skb); - return ret; - } - } - - return 0; -} - -int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id, - int mac_id, enum hal_ring_type ring_type, - int rx_buf_size, - struct htt_rx_ring_tlv_filter *tlv_filter) -{ - struct htt_rx_ring_selection_cfg_cmd *cmd; - struct hal_srng *srng = &ab->hal.srng_list[ring_id]; - struct hal_srng_params params; - struct sk_buff *skb; - int len = sizeof(*cmd); - enum htt_srng_ring_type htt_ring_type; - enum htt_srng_ring_id htt_ring_id; - int ret; - - skb = ath12k_htc_alloc_skb(ab, len); - if (!skb) - return -ENOMEM; - - memset(¶ms, 0, sizeof(params)); - ath12k_hal_srng_get_params(ab, srng, ¶ms); - - ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id, - ring_type, &htt_ring_type, - &htt_ring_id); - if (ret) - goto err_free; - - skb_put(skb, len); - cmd = (struct htt_rx_ring_selection_cfg_cmd *)skb->data; - cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG, - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE); - if (htt_ring_type == HTT_SW_TO_HW_RING || - htt_ring_type == HTT_HW_TO_SW_RING) - cmd->info0 |= - le32_encode_bits(DP_SW2HW_MACID(mac_id), - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); - else - cmd->info0 |= - le32_encode_bits(mac_id, - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); - cmd->info0 |= le32_encode_bits(htt_ring_id, - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID); - cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP), - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS); - cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP), - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS); - cmd->info0 |= le32_encode_bits(tlv_filter->offset_valid, - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID); - cmd->info0 |= - le32_encode_bits(tlv_filter->drop_threshold_valid, - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL); - cmd->info0 |= le32_encode_bits(!tlv_filter->rxmon_disable, - HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON); - - cmd->info1 = le32_encode_bits(rx_buf_size, - HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE); - cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_mgmt, - HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT); - cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_ctrl, - HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL); - cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_data, - HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA); - cmd->pkt_type_en_flags0 = cpu_to_le32(tlv_filter->pkt_filter_flags0); - cmd->pkt_type_en_flags1 = cpu_to_le32(tlv_filter->pkt_filter_flags1); - cmd->pkt_type_en_flags2 = cpu_to_le32(tlv_filter->pkt_filter_flags2); - cmd->pkt_type_en_flags3 = cpu_to_le32(tlv_filter->pkt_filter_flags3); - cmd->rx_filter_tlv = cpu_to_le32(tlv_filter->rx_filter); - - cmd->info2 = le32_encode_bits(tlv_filter->rx_drop_threshold, - HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD); - cmd->info2 |= - le32_encode_bits(tlv_filter->enable_log_mgmt_type, - HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE); - cmd->info2 |= - le32_encode_bits(tlv_filter->enable_log_ctrl_type, - HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE); - cmd->info2 |= - le32_encode_bits(tlv_filter->enable_log_data_type, - HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE); - - cmd->info3 = - le32_encode_bits(tlv_filter->enable_rx_tlv_offset, - HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET); - cmd->info3 |= - le32_encode_bits(tlv_filter->rx_tlv_offset, - HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET); - - if (tlv_filter->offset_valid) { - cmd->rx_packet_offset = - le32_encode_bits(tlv_filter->rx_packet_offset, - HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET); - - cmd->rx_packet_offset |= - le32_encode_bits(tlv_filter->rx_header_offset, - HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET); - - cmd->rx_mpdu_offset = - le32_encode_bits(tlv_filter->rx_mpdu_end_offset, - HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET); - - cmd->rx_mpdu_offset |= - le32_encode_bits(tlv_filter->rx_mpdu_start_offset, - HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET); - - cmd->rx_msdu_offset = - le32_encode_bits(tlv_filter->rx_msdu_end_offset, - HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET); - - cmd->rx_msdu_offset |= - le32_encode_bits(tlv_filter->rx_msdu_start_offset, - HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET); - - cmd->rx_attn_offset = - le32_encode_bits(tlv_filter->rx_attn_offset, - HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET); - } - - if (tlv_filter->rx_mpdu_start_wmask > 0 && - tlv_filter->rx_msdu_end_wmask > 0) { - cmd->info2 |= - le32_encode_bits(true, - HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET); - cmd->rx_mpdu_start_end_mask = - le32_encode_bits(tlv_filter->rx_mpdu_start_wmask, - HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK); - /* mpdu_end is not used for any hardwares so far - * please assign it in future if any chip is - * using through hal ops - */ - cmd->rx_mpdu_start_end_mask |= - le32_encode_bits(tlv_filter->rx_mpdu_end_wmask, - HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK); - cmd->rx_msdu_end_word_mask = - le32_encode_bits(tlv_filter->rx_msdu_end_wmask, - HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK); - } - - ret = ath12k_htc_send(&ab->htc, ab->dp.eid, skb); - if (ret) - goto err_free; - - return 0; - -err_free: - dev_kfree_skb_any(skb); - - return ret; -} - -int -ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type, - struct htt_ext_stats_cfg_params *cfg_params, - u64 cookie) -{ - struct ath12k_base *ab = ar->ab; - struct ath12k_dp *dp = &ab->dp; - struct sk_buff *skb; - struct htt_ext_stats_cfg_cmd *cmd; - int len = sizeof(*cmd); - int ret; - u32 pdev_id; - - skb = ath12k_htc_alloc_skb(ab, len); - if (!skb) - return -ENOMEM; - - skb_put(skb, len); - - cmd = (struct htt_ext_stats_cfg_cmd *)skb->data; - memset(cmd, 0, sizeof(*cmd)); - cmd->hdr.msg_type = HTT_H2T_MSG_TYPE_EXT_STATS_CFG; - - pdev_id = ath12k_mac_get_target_pdev_id(ar); - cmd->hdr.pdev_mask = 1 << pdev_id; - - cmd->hdr.stats_type = type; - cmd->cfg_param0 = cpu_to_le32(cfg_params->cfg0); - cmd->cfg_param1 = cpu_to_le32(cfg_params->cfg1); - cmd->cfg_param2 = cpu_to_le32(cfg_params->cfg2); - cmd->cfg_param3 = cpu_to_le32(cfg_params->cfg3); - cmd->cookie_lsb = cpu_to_le32(lower_32_bits(cookie)); - cmd->cookie_msb = cpu_to_le32(upper_32_bits(cookie)); - - ret = ath12k_htc_send(&ab->htc, dp->eid, skb); - if (ret) { - ath12k_warn(ab, "failed to send htt type stats request: %d", - ret); - dev_kfree_skb_any(skb); - return ret; - } - - return 0; -} - -int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset) -{ - struct ath12k_base *ab = ar->ab; - int ret; - - ret = ath12k_dp_tx_htt_rx_monitor_mode_ring_config(ar, reset); - if (ret) { - ath12k_err(ab, "failed to setup rx monitor filter %d\n", ret); - return ret; - } - - return 0; -} - -int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset) -{ - struct ath12k_base *ab = ar->ab; - struct htt_rx_ring_tlv_filter tlv_filter = {}; - int ret, ring_id, i; - - tlv_filter.offset_valid = false; - - if (!reset) { - tlv_filter.rx_filter = HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING; - - tlv_filter.drop_threshold_valid = true; - tlv_filter.rx_drop_threshold = HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE; - - tlv_filter.enable_log_mgmt_type = true; - tlv_filter.enable_log_ctrl_type = true; - tlv_filter.enable_log_data_type = true; - - tlv_filter.conf_len_ctrl = HTT_RX_RING_DEFAULT_DMA_LENGTH; - tlv_filter.conf_len_mgmt = HTT_RX_RING_DEFAULT_DMA_LENGTH; - tlv_filter.conf_len_data = HTT_RX_RING_DEFAULT_DMA_LENGTH; - - tlv_filter.enable_rx_tlv_offset = true; - tlv_filter.rx_tlv_offset = HTT_RX_RING_PKT_TLV_OFFSET; - - tlv_filter.pkt_filter_flags0 = - HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 | - HTT_RX_MON_MO_MGMT_FILTER_FLAGS0; - tlv_filter.pkt_filter_flags1 = - HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 | - HTT_RX_MON_MO_MGMT_FILTER_FLAGS1; - tlv_filter.pkt_filter_flags2 = - HTT_RX_MON_FP_CTRL_FILTER_FLASG2 | - HTT_RX_MON_MO_CTRL_FILTER_FLASG2; - tlv_filter.pkt_filter_flags3 = - HTT_RX_MON_FP_CTRL_FILTER_FLASG3 | - HTT_RX_MON_MO_CTRL_FILTER_FLASG3 | - HTT_RX_MON_FP_DATA_FILTER_FLASG3 | - HTT_RX_MON_MO_DATA_FILTER_FLASG3; - } else { - tlv_filter = ath12k_mac_mon_status_filter_default; - - if (ath12k_debugfs_is_extd_rx_stats_enabled(ar)) - tlv_filter.rx_filter = ath12k_debugfs_rx_filter(ar); - } - - if (ab->hw_params->rxdma1_enable) { - for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { - ring_id = ar->dp.rxdma_mon_dst_ring[i].ring_id; - ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id, - ar->dp.mac_id + i, - HAL_RXDMA_MONITOR_DST, - DP_RXDMA_REFILL_RING_SIZE, - &tlv_filter); - if (ret) { - ath12k_err(ab, - "failed to setup filter for monitor buf %d\n", - ret); - return ret; - } - } - return 0; - } - - if (!reset) { - for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { - ring_id = ab->dp.rx_mac_buf_ring[i].ring_id; - ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id, - i, - HAL_RXDMA_BUF, - DP_RXDMA_REFILL_RING_SIZE, - &tlv_filter); - if (ret) { - ath12k_err(ab, - "failed to setup filter for mon rx buf %d\n", - ret); - return ret; - } - } - } - - for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { - ring_id = ab->dp.rx_mon_status_refill_ring[i].refill_buf_ring.ring_id; - if (!reset) { - tlv_filter.rx_filter = - HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING; - } - - ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, - i, - HAL_RXDMA_MONITOR_STATUS, - RX_MON_STATUS_BUF_SIZE, - &tlv_filter); - if (ret) { - ath12k_err(ab, - "failed to setup filter for mon status buf %d\n", - ret); - return ret; - } - } - - return 0; -} - -int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id, - int mac_id, enum hal_ring_type ring_type, - int tx_buf_size, - struct htt_tx_ring_tlv_filter *htt_tlv_filter) -{ - struct htt_tx_ring_selection_cfg_cmd *cmd; - struct hal_srng *srng = &ab->hal.srng_list[ring_id]; - struct hal_srng_params params; - struct sk_buff *skb; - int len = sizeof(*cmd); - enum htt_srng_ring_type htt_ring_type; - enum htt_srng_ring_id htt_ring_id; - int ret; - - skb = ath12k_htc_alloc_skb(ab, len); - if (!skb) - return -ENOMEM; - - memset(¶ms, 0, sizeof(params)); - ath12k_hal_srng_get_params(ab, srng, ¶ms); - - ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id, - ring_type, &htt_ring_type, - &htt_ring_id); - - if (ret) - goto err_free; - - skb_put(skb, len); - cmd = (struct htt_tx_ring_selection_cfg_cmd *)skb->data; - cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_TX_MONITOR_CFG, - HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE); - if (htt_ring_type == HTT_SW_TO_HW_RING || - htt_ring_type == HTT_HW_TO_SW_RING) - cmd->info0 |= - le32_encode_bits(DP_SW2HW_MACID(mac_id), - HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); - else - cmd->info0 |= - le32_encode_bits(mac_id, - HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID); - cmd->info0 |= le32_encode_bits(htt_ring_id, - HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID); - cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP), - HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS); - cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP), - HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS); - - cmd->info1 |= - le32_encode_bits(tx_buf_size, - HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE); - - if (htt_tlv_filter->tx_mon_mgmt_filter) { - cmd->info1 |= - le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT, - HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE); - cmd->info1 |= - le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len, - HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT); - cmd->info2 |= - le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT, - HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG); - } - - if (htt_tlv_filter->tx_mon_data_filter) { - cmd->info1 |= - le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL, - HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE); - cmd->info1 |= - le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len, - HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL); - cmd->info2 |= - le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL, - HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG); - } - - if (htt_tlv_filter->tx_mon_ctrl_filter) { - cmd->info1 |= - le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA, - HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE); - cmd->info1 |= - le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len, - HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA); - cmd->info2 |= - le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA, - HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG); - } - - cmd->tlv_filter_mask_in0 = - cpu_to_le32(htt_tlv_filter->tx_mon_downstream_tlv_flags); - cmd->tlv_filter_mask_in1 = - cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags0); - cmd->tlv_filter_mask_in2 = - cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags1); - cmd->tlv_filter_mask_in3 = - cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags2); - - ret = ath12k_htc_send(&ab->htc, ab->dp.eid, skb); - if (ret) - goto err_free; - - return 0; - -err_free: - dev_kfree_skb_any(skb); - return ret; + if (atomic_dec_and_test(&dp_pdev->num_tx_pending)) + wake_up(&dp_pdev->tx_empty_waitq); } +EXPORT_SYMBOL(ath12k_dp_tx_free_txbuf); diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.h b/drivers/net/wireless/ath/ath12k/dp_tx.h index 10acdcf1fa8f7..7cef20540179f 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.h +++ b/drivers/net/wireless/ath/ath12k/dp_tx.h @@ -1,41 +1,32 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_DP_TX_H #define ATH12K_DP_TX_H #include "core.h" -#include "hal_tx.h" struct ath12k_dp_htt_wbm_tx_status { bool acked; s8 ack_rssi; }; -int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab); -int ath12k_dp_tx(struct ath12k *ar, struct ath12k_link_vif *arvif, - struct sk_buff *skb, bool gsn_valid, int mcbc_gsn, - bool is_mcast); -void ath12k_dp_tx_completion_handler(struct ath12k_base *ab, int ring_id); - -int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask); -int -ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type, - struct htt_ext_stats_cfg_params *cfg_params, - u64 cookie); -int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset); - -int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id, - int mac_id, enum hal_ring_type ring_type, - int rx_buf_size, - struct htt_rx_ring_tlv_filter *tlv_filter); void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id); -int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id, - int mac_id, enum hal_ring_type ring_type, - int tx_buf_size, - struct htt_tx_ring_tlv_filter *htt_tlv_filter); -int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset); +enum hal_tcl_encap_type +ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb); +void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb); +u8 ath12k_dp_tx_get_tid(struct sk_buff *skb); +void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len); +int ath12k_dp_tx_align_payload(struct ath12k_dp *dp, struct sk_buff **pskb); +void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp, + struct ath12k_tx_desc_info *tx_desc, + u8 pool_id); +struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp *dp, + u8 pool_id); +void ath12k_dp_tx_free_txbuf(struct ath12k_dp *dp, + struct dp_tx_ring *tx_ring, + struct ath12k_tx_desc_params *desc_params); #endif diff --git a/drivers/net/wireless/ath/ath12k/fw.c b/drivers/net/wireless/ath/ath12k/fw.c index 5ac497f80cad8..22074653cbb82 100644 --- a/drivers/net/wireless/ath/ath12k/fw.c +++ b/drivers/net/wireless/ath/ath12k/fw.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* - * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "core.h" @@ -121,6 +121,14 @@ static int ath12k_fw_request_firmware_api_n(struct ath12k_base *ab, ab->fw.m3_data = data; ab->fw.m3_len = ie_len; break; + case ATH12K_FW_IE_AUX_UC_IMAGE: + ath12k_dbg(ab, ATH12K_DBG_BOOT, + "found aux_uc image ie (%zd B)\n", + ie_len); + + ab->fw.aux_uc_data = data; + ab->fw.aux_uc_len = ie_len; + break; case ATH12K_FW_IE_AMSS_DUALMAC_IMAGE: ath12k_dbg(ab, ATH12K_DBG_BOOT, "found dualmac fw image ie (%zd B)\n", diff --git a/drivers/net/wireless/ath/ath12k/fw.h b/drivers/net/wireless/ath/ath12k/fw.h index 7afaefed5086f..e146d24dfea4d 100644 --- a/drivers/net/wireless/ath/ath12k/fw.h +++ b/drivers/net/wireless/ath/ath12k/fw.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* - * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_FW_H @@ -15,6 +15,7 @@ enum ath12k_fw_ie_type { ATH12K_FW_IE_AMSS_IMAGE = 2, ATH12K_FW_IE_M3_IMAGE = 3, ATH12K_FW_IE_AMSS_DUALMAC_IMAGE = 4, + ATH12K_FW_IE_AUX_UC_IMAGE = 5, }; enum ath12k_fw_features { diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c index 6406fcf5d69fd..c0c3d2f047ef0 100644 --- a/drivers/net/wireless/ath/ath12k/hal.c +++ b/drivers/net/wireless/ath/ath12k/hal.c @@ -1,1573 +1,162 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include -#include "hal_tx.h" -#include "hal_rx.h" #include "debug.h" -#include "hal_desc.h" #include "hif.h" -static const struct hal_srng_config hw_srng_config_template[] = { - /* TODO: max_rings can populated by querying HW capabilities */ - [HAL_REO_DST] = { - .start_ring_id = HAL_SRNG_RING_ID_REO2SW1, - .max_rings = 8, - .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE, - }, - [HAL_REO_EXCEPTION] = { - /* Designating REO2SW0 ring as exception ring. - * Any of theREO2SW rings can be used as exception ring. - */ - .start_ring_id = HAL_SRNG_RING_ID_REO2SW0, - .max_rings = 1, - .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE, - }, - [HAL_REO_REINJECT] = { - .start_ring_id = HAL_SRNG_RING_ID_SW2REO, - .max_rings = 4, - .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE, - }, - [HAL_REO_CMD] = { - .start_ring_id = HAL_SRNG_RING_ID_REO_CMD, - .max_rings = 1, - .entry_size = (sizeof(struct hal_tlv_64_hdr) + - sizeof(struct hal_reo_get_queue_stats)) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE, - }, - [HAL_REO_STATUS] = { - .start_ring_id = HAL_SRNG_RING_ID_REO_STATUS, - .max_rings = 1, - .entry_size = (sizeof(struct hal_tlv_64_hdr) + - sizeof(struct hal_reo_get_queue_stats_status)) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE, - }, - [HAL_TCL_DATA] = { - .start_ring_id = HAL_SRNG_RING_ID_SW2TCL1, - .max_rings = 6, - .entry_size = sizeof(struct hal_tcl_data_cmd) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE, - }, - [HAL_TCL_CMD] = { - .start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD, - .max_rings = 1, - .entry_size = sizeof(struct hal_tcl_gse_cmd) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE, - }, - [HAL_TCL_STATUS] = { - .start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS, - .max_rings = 1, - .entry_size = (sizeof(struct hal_tlv_hdr) + - sizeof(struct hal_tcl_status_ring)) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE, - }, - [HAL_CE_SRC] = { - .start_ring_id = HAL_SRNG_RING_ID_CE0_SRC, - .max_rings = 16, - .entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE, - }, - [HAL_CE_DST] = { - .start_ring_id = HAL_SRNG_RING_ID_CE0_DST, - .max_rings = 16, - .entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE, - }, - [HAL_CE_DST_STATUS] = { - .start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS, - .max_rings = 16, - .entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE, - }, - [HAL_WBM_IDLE_LINK] = { - .start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK, - .max_rings = 1, - .entry_size = sizeof(struct hal_wbm_link_desc) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE, - }, - [HAL_SW2WBM_RELEASE] = { - .start_ring_id = HAL_SRNG_RING_ID_WBM_SW0_RELEASE, - .max_rings = 2, - .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE, - }, - [HAL_WBM2SW_RELEASE] = { - .start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE, - .max_rings = 8, - .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_UMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE, - }, - [HAL_RXDMA_BUF] = { - .start_ring_id = HAL_SRNG_SW2RXDMA_BUF0, - .max_rings = 1, - .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_DMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - }, - [HAL_RXDMA_DST] = { - .start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0, - .max_rings = 0, - .entry_size = 0, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - }, - [HAL_RXDMA_MONITOR_BUF] = { - .start_ring_id = HAL_SRNG_SW2RXMON_BUF0, - .max_rings = 1, - .entry_size = sizeof(struct hal_mon_buf_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - }, - [HAL_RXDMA_MONITOR_STATUS] = { - .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF, - .max_rings = 1, - .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - }, - [HAL_RXDMA_MONITOR_DESC] = { 0, }, - [HAL_RXDMA_DIR_BUF] = { - .start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF, - .max_rings = 2, - .entry_size = 8 >> 2, /* TODO: Define the struct */ - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - }, - [HAL_PPE2TCL] = { - .start_ring_id = HAL_SRNG_RING_ID_PPE2TCL1, - .max_rings = 1, - .entry_size = sizeof(struct hal_tcl_entrance_from_ppe_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE, - }, - [HAL_PPE_RELEASE] = { - .start_ring_id = HAL_SRNG_RING_ID_WBM_PPE_RELEASE, - .max_rings = 1, - .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE, - }, - [HAL_TX_MONITOR_BUF] = { - .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2TXMON_BUF0, - .max_rings = 1, - .entry_size = sizeof(struct hal_mon_buf_ring) >> 2, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_SRC, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - }, - [HAL_RXDMA_MONITOR_DST] = { - .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXMON_BUF0, - .max_rings = 1, - .entry_size = sizeof(struct hal_mon_dest_desc) >> 2, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - }, - [HAL_TX_MONITOR_DST] = { - .start_ring_id = HAL_SRNG_RING_ID_WMAC1_TXMON2SW0_BUF0, - .max_rings = 1, - .entry_size = sizeof(struct hal_mon_dest_desc) >> 2, - .mac_type = ATH12K_HAL_SRNG_PMAC, - .ring_dir = HAL_SRNG_DIR_DST, - .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, - } -}; - -static const struct ath12k_hal_tcl_to_wbm_rbm_map -ath12k_hal_qcn9274_tcl_to_wbm_rbm_map[DP_TCL_NUM_RING_MAX] = { - { - .wbm_ring_num = 0, - .rbm_id = HAL_RX_BUF_RBM_SW0_BM, - }, - { - .wbm_ring_num = 1, - .rbm_id = HAL_RX_BUF_RBM_SW1_BM, - }, - { - .wbm_ring_num = 2, - .rbm_id = HAL_RX_BUF_RBM_SW2_BM, - }, - { - .wbm_ring_num = 4, - .rbm_id = HAL_RX_BUF_RBM_SW4_BM, - } -}; - -static const struct ath12k_hal_tcl_to_wbm_rbm_map -ath12k_hal_wcn7850_tcl_to_wbm_rbm_map[DP_TCL_NUM_RING_MAX] = { - { - .wbm_ring_num = 0, - .rbm_id = HAL_RX_BUF_RBM_SW0_BM, - }, - { - .wbm_ring_num = 2, - .rbm_id = HAL_RX_BUF_RBM_SW2_BM, - }, - { - .wbm_ring_num = 4, - .rbm_id = HAL_RX_BUF_RBM_SW4_BM, - }, -}; - -static unsigned int ath12k_hal_reo1_ring_id_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_ID(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_msi1_base_lsb_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_MSI1_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_msi1_base_msb_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_MSI1_BASE_MSB(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_msi1_data_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_MSI1_DATA(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_base_msb_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_BASE_MSB(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_producer_int_setup_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_PRODUCER_INT_SETUP(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_hp_addr_lsb_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_HP_ADDR_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_hp_addr_msb_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_HP_ADDR_MSB(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static unsigned int ath12k_hal_reo1_ring_misc_offset(struct ath12k_base *ab) -{ - return HAL_REO1_RING_MISC(ab) - HAL_REO1_RING_BASE_LSB(ab); -} - -static bool ath12k_hw_qcn9274_rx_desc_get_first_msdu(struct hal_rx_desc *desc) -{ - return !!le16_get_bits(desc->u.qcn9274.msdu_end.info5, - RX_MSDU_END_INFO5_FIRST_MSDU); -} - -static bool ath12k_hw_qcn9274_rx_desc_get_last_msdu(struct hal_rx_desc *desc) -{ - return !!le16_get_bits(desc->u.qcn9274.msdu_end.info5, - RX_MSDU_END_INFO5_LAST_MSDU); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_l3_pad_bytes(struct hal_rx_desc *desc) -{ - return le16_get_bits(desc->u.qcn9274.msdu_end.info5, - RX_MSDU_END_INFO5_L3_HDR_PADDING); -} - -static bool ath12k_hw_qcn9274_rx_desc_encrypt_valid(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274.mpdu_start.info4, - RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID); -} - -static u32 ath12k_hw_qcn9274_rx_desc_get_encrypt_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.mpdu_start.info2, - RX_MPDU_START_INFO2_ENC_TYPE); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_decap_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info11, - RX_MSDU_END_INFO11_DECAP_FORMAT); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_mesh_ctl(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info11, - RX_MSDU_END_INFO11_MESH_CTRL_PRESENT); -} - -static bool ath12k_hw_qcn9274_rx_desc_get_mpdu_seq_ctl_vld(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID); -} - -static bool ath12k_hw_qcn9274_rx_desc_get_mpdu_fc_valid(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_FCTRL_VALID); -} - -static u16 ath12k_hw_qcn9274_rx_desc_get_mpdu_start_seq_no(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_SEQ_NUM); -} - -static u16 ath12k_hw_qcn9274_rx_desc_get_msdu_len(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info10, - RX_MSDU_END_INFO10_MSDU_LENGTH); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_sgi(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info12, - RX_MSDU_END_INFO12_SGI); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_rate_mcs(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info12, - RX_MSDU_END_INFO12_RATE_MCS); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_rx_bw(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info12, - RX_MSDU_END_INFO12_RECV_BW); -} - -static u32 ath12k_hw_qcn9274_rx_desc_get_msdu_freq(struct hal_rx_desc *desc) -{ - return __le32_to_cpu(desc->u.qcn9274.msdu_end.phy_meta_data); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_pkt_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info12, - RX_MSDU_END_INFO12_PKT_TYPE); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_nss(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info12, - RX_MSDU_END_INFO12_MIMO_SS_BITMAP); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_mpdu_tid(struct hal_rx_desc *desc) -{ - return le16_get_bits(desc->u.qcn9274.msdu_end.info5, - RX_MSDU_END_INFO5_TID); -} - -static u16 ath12k_hw_qcn9274_rx_desc_get_mpdu_peer_id(struct hal_rx_desc *desc) -{ - return __le16_to_cpu(desc->u.qcn9274.mpdu_start.sw_peer_id); -} - -static void ath12k_hw_qcn9274_rx_desc_copy_end_tlv(struct hal_rx_desc *fdesc, - struct hal_rx_desc *ldesc) -{ - memcpy(&fdesc->u.qcn9274.msdu_end, &ldesc->u.qcn9274.msdu_end, - sizeof(struct rx_msdu_end_qcn9274)); -} - -static u32 ath12k_hw_qcn9274_rx_desc_get_mpdu_ppdu_id(struct hal_rx_desc *desc) -{ - return __le16_to_cpu(desc->u.qcn9274.mpdu_start.phy_ppdu_id); -} - -static void ath12k_hw_qcn9274_rx_desc_set_msdu_len(struct hal_rx_desc *desc, u16 len) -{ - u32 info = __le32_to_cpu(desc->u.qcn9274.msdu_end.info10); - - info &= ~RX_MSDU_END_INFO10_MSDU_LENGTH; - info |= u32_encode_bits(len, RX_MSDU_END_INFO10_MSDU_LENGTH); - - desc->u.qcn9274.msdu_end.info10 = __cpu_to_le32(info); -} - -static u8 *ath12k_hw_qcn9274_rx_desc_get_msdu_payload(struct hal_rx_desc *desc) -{ - return &desc->u.qcn9274.msdu_payload[0]; -} - -static u32 ath12k_hw_qcn9274_rx_desc_get_mpdu_start_offset(void) -{ - return offsetof(struct hal_rx_desc_qcn9274, mpdu_start); -} - -static u32 ath12k_hw_qcn9274_rx_desc_get_msdu_end_offset(void) -{ - return offsetof(struct hal_rx_desc_qcn9274, msdu_end); -} - -static bool ath12k_hw_qcn9274_rx_desc_mac_addr2_valid(struct hal_rx_desc *desc) -{ - return __le32_to_cpu(desc->u.qcn9274.mpdu_start.info4) & - RX_MPDU_START_INFO4_MAC_ADDR2_VALID; -} - -static u8 *ath12k_hw_qcn9274_rx_desc_mpdu_start_addr2(struct hal_rx_desc *desc) -{ - return desc->u.qcn9274.mpdu_start.addr2; -} - -static bool ath12k_hw_qcn9274_rx_desc_is_da_mcbc(struct hal_rx_desc *desc) -{ - return __le16_to_cpu(desc->u.qcn9274.msdu_end.info5) & - RX_MSDU_END_INFO5_DA_IS_MCBC; -} - -static void ath12k_hw_qcn9274_rx_desc_get_dot11_hdr(struct hal_rx_desc *desc, - struct ieee80211_hdr *hdr) -{ - hdr->frame_control = desc->u.qcn9274.mpdu_start.frame_ctrl; - hdr->duration_id = desc->u.qcn9274.mpdu_start.duration; - ether_addr_copy(hdr->addr1, desc->u.qcn9274.mpdu_start.addr1); - ether_addr_copy(hdr->addr2, desc->u.qcn9274.mpdu_start.addr2); - ether_addr_copy(hdr->addr3, desc->u.qcn9274.mpdu_start.addr3); - if (__le32_to_cpu(desc->u.qcn9274.mpdu_start.info4) & - RX_MPDU_START_INFO4_MAC_ADDR4_VALID) { - ether_addr_copy(hdr->addr4, desc->u.qcn9274.mpdu_start.addr4); - } - hdr->seq_ctrl = desc->u.qcn9274.mpdu_start.seq_ctrl; -} - -static void ath12k_hw_qcn9274_rx_desc_get_crypto_hdr(struct hal_rx_desc *desc, - u8 *crypto_hdr, - enum hal_encrypt_type enctype) -{ - unsigned int key_id; - - switch (enctype) { - case HAL_ENCRYPT_TYPE_OPEN: - return; - case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: - case HAL_ENCRYPT_TYPE_TKIP_MIC: - crypto_hdr[0] = - HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274.mpdu_start.pn[0]); - crypto_hdr[1] = 0; - crypto_hdr[2] = - HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274.mpdu_start.pn[0]); - break; - case HAL_ENCRYPT_TYPE_CCMP_128: - case HAL_ENCRYPT_TYPE_CCMP_256: - case HAL_ENCRYPT_TYPE_GCMP_128: - case HAL_ENCRYPT_TYPE_AES_GCMP_256: - crypto_hdr[0] = - HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274.mpdu_start.pn[0]); - crypto_hdr[1] = - HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274.mpdu_start.pn[0]); - crypto_hdr[2] = 0; - break; - case HAL_ENCRYPT_TYPE_WEP_40: - case HAL_ENCRYPT_TYPE_WEP_104: - case HAL_ENCRYPT_TYPE_WEP_128: - case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4: - case HAL_ENCRYPT_TYPE_WAPI: - return; - } - key_id = le32_get_bits(desc->u.qcn9274.mpdu_start.info5, - RX_MPDU_START_INFO5_KEY_ID); - crypto_hdr[3] = 0x20 | (key_id << 6); - crypto_hdr[4] = HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.qcn9274.mpdu_start.pn[0]); - crypto_hdr[5] = HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.qcn9274.mpdu_start.pn[0]); - crypto_hdr[6] = HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274.mpdu_start.pn[1]); - crypto_hdr[7] = HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274.mpdu_start.pn[1]); -} - -static int ath12k_hal_srng_create_config_qcn9274(struct ath12k_base *ab) -{ - struct ath12k_hal *hal = &ab->hal; - struct hal_srng_config *s; - - hal->srng_config = kmemdup(hw_srng_config_template, - sizeof(hw_srng_config_template), - GFP_KERNEL); - if (!hal->srng_config) - return -ENOMEM; - - s = &hal->srng_config[HAL_REO_DST]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP; - s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP; - - s = &hal->srng_config[HAL_REO_EXCEPTION]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP; - - s = &hal->srng_config[HAL_REO_REINJECT]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP; - s->reg_size[0] = HAL_SW2REO1_RING_BASE_LSB(ab) - HAL_SW2REO_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_SW2REO1_RING_HP - HAL_SW2REO_RING_HP; - - s = &hal->srng_config[HAL_REO_CMD]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP; - - s = &hal->srng_config[HAL_REO_STATUS]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP; - - s = &hal->srng_config[HAL_TCL_DATA]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP; - s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP; - - s = &hal->srng_config[HAL_TCL_CMD]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP; - - s = &hal->srng_config[HAL_TCL_STATUS]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP; - - s = &hal->srng_config[HAL_CE_SRC]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB; - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP; - s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab); - s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab); - - s = &hal->srng_config[HAL_CE_DST]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB; - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP; - s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - - s = &hal->srng_config[HAL_CE_DST_STATUS]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + - HAL_CE_DST_STATUS_RING_BASE_LSB; - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP; - s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - - s = &hal->srng_config[HAL_WBM_IDLE_LINK]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP; - - s = &hal->srng_config[HAL_SW2WBM_RELEASE]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP; - s->reg_size[0] = HAL_WBM_SW1_RELEASE_RING_BASE_LSB(ab) - - HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_WBM_SW1_RELEASE_RING_HP - HAL_WBM_SW_RELEASE_RING_HP; - - s = &hal->srng_config[HAL_WBM2SW_RELEASE]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP; - s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) - - HAL_WBM0_RELEASE_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP; - - /* Some LMAC rings are not accessed from the host: - * RXDMA_BUG, RXDMA_DST, RXDMA_MONITOR_BUF, RXDMA_MONITOR_STATUS, - * RXDMA_MONITOR_DST, RXDMA_MONITOR_DESC, RXDMA_DIR_BUF_SRC, - * RXDMA_RX_MONITOR_BUF, TX_MONITOR_BUF, TX_MONITOR_DST, SW2RXDMA - */ - s = &hal->srng_config[HAL_PPE2TCL]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_BASE_LSB; - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_HP; - - s = &hal->srng_config[HAL_PPE_RELEASE]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_PPE_RELEASE_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_PPE_RELEASE_RING_HP; - - return 0; -} - -static u16 ath12k_hal_qcn9274_rx_mpdu_start_wmask_get(void) -{ - return QCN9274_MPDU_START_WMASK; -} - -static u32 ath12k_hal_qcn9274_rx_msdu_end_wmask_get(void) -{ - return QCN9274_MSDU_END_WMASK; -} - -static const struct hal_rx_ops *ath12k_hal_qcn9274_get_hal_rx_compact_ops(void) -{ - return &hal_rx_qcn9274_compact_ops; -} - -static bool ath12k_hw_qcn9274_dp_rx_h_msdu_done(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274.msdu_end.info14, - RX_MSDU_END_INFO14_MSDU_DONE); -} - -static bool ath12k_hw_qcn9274_dp_rx_h_l4_cksum_fail(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274.msdu_end.info13, - RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL); -} - -static bool ath12k_hw_qcn9274_dp_rx_h_ip_cksum_fail(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274.msdu_end.info13, - RX_MSDU_END_INFO13_IP_CKSUM_FAIL); -} - -static bool ath12k_hw_qcn9274_dp_rx_h_is_decrypted(struct hal_rx_desc *desc) -{ - return (le32_get_bits(desc->u.qcn9274.msdu_end.info14, - RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) == - RX_DESC_DECRYPT_STATUS_CODE_OK); -} - -static u32 ath12k_hw_qcn9274_dp_rx_h_mpdu_err(struct hal_rx_desc *desc) -{ - u32 info = __le32_to_cpu(desc->u.qcn9274.msdu_end.info13); - u32 errmap = 0; - - if (info & RX_MSDU_END_INFO13_FCS_ERR) - errmap |= HAL_RX_MPDU_ERR_FCS; - - if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) - errmap |= HAL_RX_MPDU_ERR_DECRYPT; - - if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) - errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; - - if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) - errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; - - if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) - errmap |= HAL_RX_MPDU_ERR_OVERFLOW; - - if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) - errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; - - if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) - errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; - - return errmap; -} - -static u32 ath12k_hw_qcn9274_get_rx_desc_size(void) -{ - return sizeof(struct hal_rx_desc_qcn9274); -} - -static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_src_link(struct hal_rx_desc *desc) -{ - return 0; -} - -const struct hal_rx_ops hal_rx_qcn9274_ops = { - .rx_desc_get_first_msdu = ath12k_hw_qcn9274_rx_desc_get_first_msdu, - .rx_desc_get_last_msdu = ath12k_hw_qcn9274_rx_desc_get_last_msdu, - .rx_desc_get_l3_pad_bytes = ath12k_hw_qcn9274_rx_desc_get_l3_pad_bytes, - .rx_desc_encrypt_valid = ath12k_hw_qcn9274_rx_desc_encrypt_valid, - .rx_desc_get_encrypt_type = ath12k_hw_qcn9274_rx_desc_get_encrypt_type, - .rx_desc_get_decap_type = ath12k_hw_qcn9274_rx_desc_get_decap_type, - .rx_desc_get_mesh_ctl = ath12k_hw_qcn9274_rx_desc_get_mesh_ctl, - .rx_desc_get_mpdu_seq_ctl_vld = ath12k_hw_qcn9274_rx_desc_get_mpdu_seq_ctl_vld, - .rx_desc_get_mpdu_fc_valid = ath12k_hw_qcn9274_rx_desc_get_mpdu_fc_valid, - .rx_desc_get_mpdu_start_seq_no = ath12k_hw_qcn9274_rx_desc_get_mpdu_start_seq_no, - .rx_desc_get_msdu_len = ath12k_hw_qcn9274_rx_desc_get_msdu_len, - .rx_desc_get_msdu_sgi = ath12k_hw_qcn9274_rx_desc_get_msdu_sgi, - .rx_desc_get_msdu_rate_mcs = ath12k_hw_qcn9274_rx_desc_get_msdu_rate_mcs, - .rx_desc_get_msdu_rx_bw = ath12k_hw_qcn9274_rx_desc_get_msdu_rx_bw, - .rx_desc_get_msdu_freq = ath12k_hw_qcn9274_rx_desc_get_msdu_freq, - .rx_desc_get_msdu_pkt_type = ath12k_hw_qcn9274_rx_desc_get_msdu_pkt_type, - .rx_desc_get_msdu_nss = ath12k_hw_qcn9274_rx_desc_get_msdu_nss, - .rx_desc_get_mpdu_tid = ath12k_hw_qcn9274_rx_desc_get_mpdu_tid, - .rx_desc_get_mpdu_peer_id = ath12k_hw_qcn9274_rx_desc_get_mpdu_peer_id, - .rx_desc_copy_end_tlv = ath12k_hw_qcn9274_rx_desc_copy_end_tlv, - .rx_desc_get_mpdu_ppdu_id = ath12k_hw_qcn9274_rx_desc_get_mpdu_ppdu_id, - .rx_desc_set_msdu_len = ath12k_hw_qcn9274_rx_desc_set_msdu_len, - .rx_desc_get_msdu_payload = ath12k_hw_qcn9274_rx_desc_get_msdu_payload, - .rx_desc_get_mpdu_start_offset = ath12k_hw_qcn9274_rx_desc_get_mpdu_start_offset, - .rx_desc_get_msdu_end_offset = ath12k_hw_qcn9274_rx_desc_get_msdu_end_offset, - .rx_desc_mac_addr2_valid = ath12k_hw_qcn9274_rx_desc_mac_addr2_valid, - .rx_desc_mpdu_start_addr2 = ath12k_hw_qcn9274_rx_desc_mpdu_start_addr2, - .rx_desc_is_da_mcbc = ath12k_hw_qcn9274_rx_desc_is_da_mcbc, - .rx_desc_get_dot11_hdr = ath12k_hw_qcn9274_rx_desc_get_dot11_hdr, - .rx_desc_get_crypto_header = ath12k_hw_qcn9274_rx_desc_get_crypto_hdr, - .dp_rx_h_msdu_done = ath12k_hw_qcn9274_dp_rx_h_msdu_done, - .dp_rx_h_l4_cksum_fail = ath12k_hw_qcn9274_dp_rx_h_l4_cksum_fail, - .dp_rx_h_ip_cksum_fail = ath12k_hw_qcn9274_dp_rx_h_ip_cksum_fail, - .dp_rx_h_is_decrypted = ath12k_hw_qcn9274_dp_rx_h_is_decrypted, - .dp_rx_h_mpdu_err = ath12k_hw_qcn9274_dp_rx_h_mpdu_err, - .rx_desc_get_desc_size = ath12k_hw_qcn9274_get_rx_desc_size, - .rx_desc_get_msdu_src_link_id = ath12k_hw_qcn9274_rx_desc_get_msdu_src_link, -}; - -static bool ath12k_hw_qcn9274_compact_rx_desc_get_first_msdu(struct hal_rx_desc *desc) -{ - return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, - RX_MSDU_END_INFO5_FIRST_MSDU); -} - -static bool ath12k_hw_qcn9274_compact_rx_desc_get_last_msdu(struct hal_rx_desc *desc) -{ - return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, - RX_MSDU_END_INFO5_LAST_MSDU); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_l3_pad_bytes(struct hal_rx_desc *desc) -{ - return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, - RX_MSDU_END_INFO5_L3_HDR_PADDING); -} - -static bool ath12k_hw_qcn9274_compact_rx_desc_encrypt_valid(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, - RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID); -} - -static u32 ath12k_hw_qcn9274_compact_rx_desc_get_encrypt_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info2, - RX_MPDU_START_INFO2_ENC_TYPE); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_decap_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info11, - RX_MSDU_END_INFO11_DECAP_FORMAT); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_mesh_ctl(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274.msdu_end.info11, - RX_MSDU_END_INFO11_MESH_CTRL_PRESENT); -} - -static bool -ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_seq_ctl_vld(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID); -} - -static bool ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_fc_valid(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_FCTRL_VALID); -} - -static u16 -ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_seq_no(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_SEQ_NUM); -} - -static u16 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_len(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info10, - RX_MSDU_END_INFO10_MSDU_LENGTH); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_sgi(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, - RX_MSDU_END_INFO12_SGI); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rate_mcs(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, - RX_MSDU_END_INFO12_RATE_MCS); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rx_bw(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, - RX_MSDU_END_INFO12_RECV_BW); -} - -static u32 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_freq(struct hal_rx_desc *desc) -{ - return __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.phy_meta_data); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_pkt_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, - RX_MSDU_END_INFO12_PKT_TYPE); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_nss(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, - RX_MSDU_END_INFO12_MIMO_SS_BITMAP); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_tid(struct hal_rx_desc *desc) -{ - return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, - RX_MSDU_END_INFO5_TID); -} - -static u16 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_peer_id(struct hal_rx_desc *desc) -{ - return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.sw_peer_id); -} - -static void ath12k_hw_qcn9274_compact_rx_desc_copy_end_tlv(struct hal_rx_desc *fdesc, - struct hal_rx_desc *ldesc) -{ - fdesc->u.qcn9274_compact.msdu_end = ldesc->u.qcn9274_compact.msdu_end; -} - -static u32 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_ppdu_id(struct hal_rx_desc *desc) -{ - return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.phy_ppdu_id); -} - -static void -ath12k_hw_qcn9274_compact_rx_desc_set_msdu_len(struct hal_rx_desc *desc, u16 len) -{ - u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info10); - - info = u32_replace_bits(info, len, RX_MSDU_END_INFO10_MSDU_LENGTH); - desc->u.qcn9274_compact.msdu_end.info10 = __cpu_to_le32(info); -} - -static u8 *ath12k_hw_qcn9274_compact_rx_desc_get_msdu_payload(struct hal_rx_desc *desc) -{ - return &desc->u.qcn9274_compact.msdu_payload[0]; -} - -static u32 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_offset(void) -{ - return offsetof(struct hal_rx_desc_qcn9274_compact, mpdu_start); -} - -static u32 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_end_offset(void) -{ - return offsetof(struct hal_rx_desc_qcn9274_compact, msdu_end); -} - -static bool ath12k_hw_qcn9274_compact_rx_desc_mac_addr2_valid(struct hal_rx_desc *desc) -{ - return __le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) & - RX_MPDU_START_INFO4_MAC_ADDR2_VALID; -} - -static u8 *ath12k_hw_qcn9274_compact_rx_desc_mpdu_start_addr2(struct hal_rx_desc *desc) -{ - return desc->u.qcn9274_compact.mpdu_start.addr2; -} - -static bool ath12k_hw_qcn9274_compact_rx_desc_is_da_mcbc(struct hal_rx_desc *desc) -{ - return __le16_to_cpu(desc->u.qcn9274_compact.msdu_end.info5) & - RX_MSDU_END_INFO5_DA_IS_MCBC; -} - -static void ath12k_hw_qcn9274_compact_rx_desc_get_dot11_hdr(struct hal_rx_desc *desc, - struct ieee80211_hdr *hdr) -{ - hdr->frame_control = desc->u.qcn9274_compact.mpdu_start.frame_ctrl; - hdr->duration_id = desc->u.qcn9274_compact.mpdu_start.duration; - ether_addr_copy(hdr->addr1, desc->u.qcn9274_compact.mpdu_start.addr1); - ether_addr_copy(hdr->addr2, desc->u.qcn9274_compact.mpdu_start.addr2); - ether_addr_copy(hdr->addr3, desc->u.qcn9274_compact.mpdu_start.addr3); - if (__le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) & - RX_MPDU_START_INFO4_MAC_ADDR4_VALID) { - ether_addr_copy(hdr->addr4, desc->u.qcn9274_compact.mpdu_start.addr4); - } - hdr->seq_ctrl = desc->u.qcn9274_compact.mpdu_start.seq_ctrl; -} - -static void -ath12k_hw_qcn9274_compact_rx_desc_get_crypto_hdr(struct hal_rx_desc *desc, - u8 *crypto_hdr, - enum hal_encrypt_type enctype) -{ - unsigned int key_id; - - switch (enctype) { - case HAL_ENCRYPT_TYPE_OPEN: - return; - case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: - case HAL_ENCRYPT_TYPE_TKIP_MIC: - crypto_hdr[0] = - HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]); - crypto_hdr[1] = 0; - crypto_hdr[2] = - HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]); - break; - case HAL_ENCRYPT_TYPE_CCMP_128: - case HAL_ENCRYPT_TYPE_CCMP_256: - case HAL_ENCRYPT_TYPE_GCMP_128: - case HAL_ENCRYPT_TYPE_AES_GCMP_256: - crypto_hdr[0] = - HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]); - crypto_hdr[1] = - HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]); - crypto_hdr[2] = 0; - break; - case HAL_ENCRYPT_TYPE_WEP_40: - case HAL_ENCRYPT_TYPE_WEP_104: - case HAL_ENCRYPT_TYPE_WEP_128: - case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4: - case HAL_ENCRYPT_TYPE_WAPI: - return; - } - key_id = le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info5, - RX_MPDU_START_INFO5_KEY_ID); - crypto_hdr[3] = 0x20 | (key_id << 6); - crypto_hdr[4] = - HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.qcn9274_compact.mpdu_start.pn[0]); - crypto_hdr[5] = - HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.qcn9274_compact.mpdu_start.pn[0]); - crypto_hdr[6] = - HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[1]); - crypto_hdr[7] = - HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[1]); -} - -static bool ath12k_hw_qcn9274_compact_dp_rx_h_msdu_done(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14, - RX_MSDU_END_INFO14_MSDU_DONE); -} - -static bool ath12k_hw_qcn9274_compact_dp_rx_h_l4_cksum_fail(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13, - RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL); -} - -static bool ath12k_hw_qcn9274_compact_dp_rx_h_ip_cksum_fail(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13, - RX_MSDU_END_INFO13_IP_CKSUM_FAIL); -} - -static bool ath12k_hw_qcn9274_compact_dp_rx_h_is_decrypted(struct hal_rx_desc *desc) -{ - return (le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14, - RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) == - RX_DESC_DECRYPT_STATUS_CODE_OK); -} - -static u32 ath12k_hw_qcn9274_compact_dp_rx_h_mpdu_err(struct hal_rx_desc *desc) -{ - u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info13); - u32 errmap = 0; - - if (info & RX_MSDU_END_INFO13_FCS_ERR) - errmap |= HAL_RX_MPDU_ERR_FCS; - - if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) - errmap |= HAL_RX_MPDU_ERR_DECRYPT; - - if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) - errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; - - if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) - errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; - - if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) - errmap |= HAL_RX_MPDU_ERR_OVERFLOW; - - if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) - errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; - - if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) - errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; - - return errmap; -} - -static u32 ath12k_hw_qcn9274_compact_get_rx_desc_size(void) -{ - return sizeof(struct hal_rx_desc_qcn9274_compact); -} - -static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_src_link(struct hal_rx_desc *desc) -{ - return le64_get_bits(desc->u.qcn9274_compact.msdu_end.msdu_end_tag, - RX_MSDU_END_64_TLV_SRC_LINK_ID); -} - -const struct hal_rx_ops hal_rx_qcn9274_compact_ops = { - .rx_desc_get_first_msdu = ath12k_hw_qcn9274_compact_rx_desc_get_first_msdu, - .rx_desc_get_last_msdu = ath12k_hw_qcn9274_compact_rx_desc_get_last_msdu, - .rx_desc_get_l3_pad_bytes = ath12k_hw_qcn9274_compact_rx_desc_get_l3_pad_bytes, - .rx_desc_encrypt_valid = ath12k_hw_qcn9274_compact_rx_desc_encrypt_valid, - .rx_desc_get_encrypt_type = ath12k_hw_qcn9274_compact_rx_desc_get_encrypt_type, - .rx_desc_get_decap_type = ath12k_hw_qcn9274_compact_rx_desc_get_decap_type, - .rx_desc_get_mesh_ctl = ath12k_hw_qcn9274_compact_rx_desc_get_mesh_ctl, - .rx_desc_get_mpdu_seq_ctl_vld = - ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_seq_ctl_vld, - .rx_desc_get_mpdu_fc_valid = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_fc_valid, - .rx_desc_get_mpdu_start_seq_no = - ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_seq_no, - .rx_desc_get_msdu_len = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_len, - .rx_desc_get_msdu_sgi = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_sgi, - .rx_desc_get_msdu_rate_mcs = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rate_mcs, - .rx_desc_get_msdu_rx_bw = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rx_bw, - .rx_desc_get_msdu_freq = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_freq, - .rx_desc_get_msdu_pkt_type = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_pkt_type, - .rx_desc_get_msdu_nss = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_nss, - .rx_desc_get_mpdu_tid = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_tid, - .rx_desc_get_mpdu_peer_id = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_peer_id, - .rx_desc_copy_end_tlv = ath12k_hw_qcn9274_compact_rx_desc_copy_end_tlv, - .rx_desc_get_mpdu_ppdu_id = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_ppdu_id, - .rx_desc_set_msdu_len = ath12k_hw_qcn9274_compact_rx_desc_set_msdu_len, - .rx_desc_get_msdu_payload = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_payload, - .rx_desc_get_mpdu_start_offset = - ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_offset, - .rx_desc_get_msdu_end_offset = - ath12k_hw_qcn9274_compact_rx_desc_get_msdu_end_offset, - .rx_desc_mac_addr2_valid = ath12k_hw_qcn9274_compact_rx_desc_mac_addr2_valid, - .rx_desc_mpdu_start_addr2 = ath12k_hw_qcn9274_compact_rx_desc_mpdu_start_addr2, - .rx_desc_is_da_mcbc = ath12k_hw_qcn9274_compact_rx_desc_is_da_mcbc, - .rx_desc_get_dot11_hdr = ath12k_hw_qcn9274_compact_rx_desc_get_dot11_hdr, - .rx_desc_get_crypto_header = ath12k_hw_qcn9274_compact_rx_desc_get_crypto_hdr, - .dp_rx_h_msdu_done = ath12k_hw_qcn9274_compact_dp_rx_h_msdu_done, - .dp_rx_h_l4_cksum_fail = ath12k_hw_qcn9274_compact_dp_rx_h_l4_cksum_fail, - .dp_rx_h_ip_cksum_fail = ath12k_hw_qcn9274_compact_dp_rx_h_ip_cksum_fail, - .dp_rx_h_is_decrypted = ath12k_hw_qcn9274_compact_dp_rx_h_is_decrypted, - .dp_rx_h_mpdu_err = ath12k_hw_qcn9274_compact_dp_rx_h_mpdu_err, - .rx_desc_get_desc_size = ath12k_hw_qcn9274_compact_get_rx_desc_size, - .rx_desc_get_msdu_src_link_id = - ath12k_hw_qcn9274_compact_rx_desc_get_msdu_src_link, -}; - -const struct hal_ops hal_qcn9274_ops = { - .create_srng_config = ath12k_hal_srng_create_config_qcn9274, - .tcl_to_wbm_rbm_map = ath12k_hal_qcn9274_tcl_to_wbm_rbm_map, - .rxdma_ring_wmask_rx_mpdu_start = ath12k_hal_qcn9274_rx_mpdu_start_wmask_get, - .rxdma_ring_wmask_rx_msdu_end = ath12k_hal_qcn9274_rx_msdu_end_wmask_get, - .get_hal_rx_compact_ops = ath12k_hal_qcn9274_get_hal_rx_compact_ops, -}; - -static bool ath12k_hw_wcn7850_rx_desc_get_first_msdu(struct hal_rx_desc *desc) -{ - return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5, - RX_MSDU_END_INFO5_FIRST_MSDU); -} - -static bool ath12k_hw_wcn7850_rx_desc_get_last_msdu(struct hal_rx_desc *desc) -{ - return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5, - RX_MSDU_END_INFO5_LAST_MSDU); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_l3_pad_bytes(struct hal_rx_desc *desc) -{ - return le16_get_bits(desc->u.wcn7850.msdu_end.info5, - RX_MSDU_END_INFO5_L3_HDR_PADDING); -} - -static bool ath12k_hw_wcn7850_rx_desc_encrypt_valid(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4, - RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID); -} - -static u32 ath12k_hw_wcn7850_rx_desc_get_encrypt_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.mpdu_start.info2, - RX_MPDU_START_INFO2_ENC_TYPE); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_decap_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.msdu_end.info11, - RX_MSDU_END_INFO11_DECAP_FORMAT); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_mesh_ctl(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.msdu_end.info11, - RX_MSDU_END_INFO11_MESH_CTRL_PRESENT); -} - -static bool ath12k_hw_wcn7850_rx_desc_get_mpdu_seq_ctl_vld(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID); -} - -static bool ath12k_hw_wcn7850_rx_desc_get_mpdu_fc_valid(struct hal_rx_desc *desc) -{ - return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_FCTRL_VALID); -} - -static u16 ath12k_hw_wcn7850_rx_desc_get_mpdu_start_seq_no(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.mpdu_start.info4, - RX_MPDU_START_INFO4_MPDU_SEQ_NUM); -} - -static u16 ath12k_hw_wcn7850_rx_desc_get_msdu_len(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.msdu_end.info10, - RX_MSDU_END_INFO10_MSDU_LENGTH); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_sgi(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.msdu_end.info12, - RX_MSDU_END_INFO12_SGI); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_rate_mcs(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.msdu_end.info12, - RX_MSDU_END_INFO12_RATE_MCS); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_rx_bw(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.msdu_end.info12, - RX_MSDU_END_INFO12_RECV_BW); -} - -static u32 ath12k_hw_wcn7850_rx_desc_get_msdu_freq(struct hal_rx_desc *desc) -{ - return __le32_to_cpu(desc->u.wcn7850.msdu_end.phy_meta_data); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_pkt_type(struct hal_rx_desc *desc) -{ - return le32_get_bits(desc->u.wcn7850.msdu_end.info12, - RX_MSDU_END_INFO12_PKT_TYPE); -} - -static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_nss(struct hal_rx_desc *desc) +static void ath12k_hal_ce_dst_setup(struct ath12k_base *ab, + struct hal_srng *srng, int ring_num) { - return le32_get_bits(desc->u.wcn7850.msdu_end.info12, - RX_MSDU_END_INFO12_MIMO_SS_BITMAP); + ab->hal.ops->ce_dst_setup(ab, srng, ring_num); } -static u8 ath12k_hw_wcn7850_rx_desc_get_mpdu_tid(struct hal_rx_desc *desc) +static void ath12k_hal_srng_src_hw_init(struct ath12k_base *ab, + struct hal_srng *srng) { - return le32_get_bits(desc->u.wcn7850.mpdu_start.info2, - RX_MPDU_START_INFO2_TID); + ab->hal.ops->srng_src_hw_init(ab, srng); } -static u16 ath12k_hw_wcn7850_rx_desc_get_mpdu_peer_id(struct hal_rx_desc *desc) +static void ath12k_hal_srng_dst_hw_init(struct ath12k_base *ab, + struct hal_srng *srng) { - return __le16_to_cpu(desc->u.wcn7850.mpdu_start.sw_peer_id); + ab->hal.ops->srng_dst_hw_init(ab, srng); } -static void ath12k_hw_wcn7850_rx_desc_copy_end_tlv(struct hal_rx_desc *fdesc, - struct hal_rx_desc *ldesc) +static void ath12k_hal_set_umac_srng_ptr_addr(struct ath12k_base *ab, + struct hal_srng *srng) { - memcpy(&fdesc->u.wcn7850.msdu_end, &ldesc->u.wcn7850.msdu_end, - sizeof(struct rx_msdu_end_qcn9274)); + ab->hal.ops->set_umac_srng_ptr_addr(ab, srng); } -static u32 ath12k_hw_wcn7850_rx_desc_get_mpdu_start_tag(struct hal_rx_desc *desc) +static int ath12k_hal_srng_get_ring_id(struct ath12k_hal *hal, + enum hal_ring_type type, + int ring_num, int mac_id) { - return le64_get_bits(desc->u.wcn7850.mpdu_start_tag, - HAL_TLV_HDR_TAG); + return hal->ops->srng_get_ring_id(hal, type, ring_num, mac_id); } -static u32 ath12k_hw_wcn7850_rx_desc_get_mpdu_ppdu_id(struct hal_rx_desc *desc) +int ath12k_hal_srng_update_shadow_config(struct ath12k_base *ab, + enum hal_ring_type ring_type, + int ring_num) { - return __le16_to_cpu(desc->u.wcn7850.mpdu_start.phy_ppdu_id); + return ab->hal.ops->srng_update_shadow_config(ab, ring_type, + ring_num); } -static void ath12k_hw_wcn7850_rx_desc_set_msdu_len(struct hal_rx_desc *desc, u16 len) +u32 ath12k_hal_ce_get_desc_size(struct ath12k_hal *hal, enum hal_ce_desc type) { - u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info10); - - info &= ~RX_MSDU_END_INFO10_MSDU_LENGTH; - info |= u32_encode_bits(len, RX_MSDU_END_INFO10_MSDU_LENGTH); - - desc->u.wcn7850.msdu_end.info10 = __cpu_to_le32(info); + return hal->ops->ce_get_desc_size(type); } -static u8 *ath12k_hw_wcn7850_rx_desc_get_msdu_payload(struct hal_rx_desc *desc) +void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id) { - return &desc->u.wcn7850.msdu_payload[0]; + ab->hal.ops->tx_set_dscp_tid_map(ab, id); } -static u32 ath12k_hw_wcn7850_rx_desc_get_mpdu_start_offset(void) +void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab, + u32 bank_config, u8 bank_id) { - return offsetof(struct hal_rx_desc_wcn7850, mpdu_start_tag); + ab->hal.ops->tx_configure_bank_register(ab, bank_config, bank_id); } -static u32 ath12k_hw_wcn7850_rx_desc_get_msdu_end_offset(void) +void ath12k_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab) { - return offsetof(struct hal_rx_desc_wcn7850, msdu_end_tag); + ab->hal.ops->reoq_lut_addr_read_enable(ab); } -static bool ath12k_hw_wcn7850_rx_desc_mac_addr2_valid(struct hal_rx_desc *desc) +void ath12k_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab) { - return __le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) & - RX_MPDU_START_INFO4_MAC_ADDR2_VALID; + ab->hal.ops->reoq_lut_set_max_peerid(ab); } -static u8 *ath12k_hw_wcn7850_rx_desc_mpdu_start_addr2(struct hal_rx_desc *desc) +void ath12k_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr) { - return desc->u.wcn7850.mpdu_start.addr2; + ab->hal.ops->write_ml_reoq_lut_addr(ab, paddr); } -static bool ath12k_hw_wcn7850_rx_desc_is_da_mcbc(struct hal_rx_desc *desc) +void ath12k_hal_write_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr) { - return __le32_to_cpu(desc->u.wcn7850.msdu_end.info13) & - RX_MSDU_END_INFO13_MCAST_BCAST; + ab->hal.ops->write_reoq_lut_addr(ab, paddr); } -static void ath12k_hw_wcn7850_rx_desc_get_dot11_hdr(struct hal_rx_desc *desc, - struct ieee80211_hdr *hdr) +void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab, + struct hal_wbm_idle_scatter_list *sbuf, + u32 nsbufs, u32 tot_link_desc, + u32 end_offset) { - hdr->frame_control = desc->u.wcn7850.mpdu_start.frame_ctrl; - hdr->duration_id = desc->u.wcn7850.mpdu_start.duration; - ether_addr_copy(hdr->addr1, desc->u.wcn7850.mpdu_start.addr1); - ether_addr_copy(hdr->addr2, desc->u.wcn7850.mpdu_start.addr2); - ether_addr_copy(hdr->addr3, desc->u.wcn7850.mpdu_start.addr3); - if (__le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) & - RX_MPDU_START_INFO4_MAC_ADDR4_VALID) { - ether_addr_copy(hdr->addr4, desc->u.wcn7850.mpdu_start.addr4); - } - hdr->seq_ctrl = desc->u.wcn7850.mpdu_start.seq_ctrl; + ab->hal.ops->setup_link_idle_list(ab, sbuf, nsbufs, tot_link_desc, + end_offset); } -static void ath12k_hw_wcn7850_rx_desc_get_crypto_hdr(struct hal_rx_desc *desc, - u8 *crypto_hdr, - enum hal_encrypt_type enctype) +void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map) { - unsigned int key_id; - - switch (enctype) { - case HAL_ENCRYPT_TYPE_OPEN: - return; - case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: - case HAL_ENCRYPT_TYPE_TKIP_MIC: - crypto_hdr[0] = - HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]); - crypto_hdr[1] = 0; - crypto_hdr[2] = - HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]); - break; - case HAL_ENCRYPT_TYPE_CCMP_128: - case HAL_ENCRYPT_TYPE_CCMP_256: - case HAL_ENCRYPT_TYPE_GCMP_128: - case HAL_ENCRYPT_TYPE_AES_GCMP_256: - crypto_hdr[0] = - HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]); - crypto_hdr[1] = - HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]); - crypto_hdr[2] = 0; - break; - case HAL_ENCRYPT_TYPE_WEP_40: - case HAL_ENCRYPT_TYPE_WEP_104: - case HAL_ENCRYPT_TYPE_WEP_128: - case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4: - case HAL_ENCRYPT_TYPE_WAPI: - return; - } - key_id = u32_get_bits(__le32_to_cpu(desc->u.wcn7850.mpdu_start.info5), - RX_MPDU_START_INFO5_KEY_ID); - crypto_hdr[3] = 0x20 | (key_id << 6); - crypto_hdr[4] = HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.wcn7850.mpdu_start.pn[0]); - crypto_hdr[5] = HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.wcn7850.mpdu_start.pn[0]); - crypto_hdr[6] = HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[1]); - crypto_hdr[7] = HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[1]); + ab->hal.ops->reo_hw_setup(ab, ring_hash_map); } -static int ath12k_hal_srng_create_config_wcn7850(struct ath12k_base *ab) +void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, struct hal_srng *srng) { - struct ath12k_hal *hal = &ab->hal; - struct hal_srng_config *s; - - hal->srng_config = kmemdup(hw_srng_config_template, - sizeof(hw_srng_config_template), - GFP_KERNEL); - if (!hal->srng_config) - return -ENOMEM; - - s = &hal->srng_config[HAL_REO_DST]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP; - s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP; - - s = &hal->srng_config[HAL_REO_EXCEPTION]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP; - - s = &hal->srng_config[HAL_REO_REINJECT]; - s->max_rings = 1; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP; - - s = &hal->srng_config[HAL_REO_CMD]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP; - - s = &hal->srng_config[HAL_REO_STATUS]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP; - - s = &hal->srng_config[HAL_TCL_DATA]; - s->max_rings = 5; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP; - s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP; - - s = &hal->srng_config[HAL_TCL_CMD]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP; - - s = &hal->srng_config[HAL_TCL_STATUS]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP; - - s = &hal->srng_config[HAL_CE_SRC]; - s->max_rings = 12; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB; - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP; - s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab); - s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab); - - s = &hal->srng_config[HAL_CE_DST]; - s->max_rings = 12; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB; - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP; - s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - - s = &hal->srng_config[HAL_CE_DST_STATUS]; - s->max_rings = 12; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + - HAL_CE_DST_STATUS_RING_BASE_LSB; - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP; - s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) - - HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab); - - s = &hal->srng_config[HAL_WBM_IDLE_LINK]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP; - - s = &hal->srng_config[HAL_SW2WBM_RELEASE]; - s->max_rings = 1; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP; - - s = &hal->srng_config[HAL_WBM2SW_RELEASE]; - s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab); - s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP; - s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) - - HAL_WBM0_RELEASE_RING_BASE_LSB(ab); - s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP; - - s = &hal->srng_config[HAL_RXDMA_BUF]; - s->max_rings = 2; - s->mac_type = ATH12K_HAL_SRNG_PMAC; - - s = &hal->srng_config[HAL_RXDMA_DST]; - s->max_rings = 1; - s->entry_size = sizeof(struct hal_reo_entrance_ring) >> 2; - - /* below rings are not used */ - s = &hal->srng_config[HAL_RXDMA_DIR_BUF]; - s->max_rings = 0; - - s = &hal->srng_config[HAL_PPE2TCL]; - s->max_rings = 0; - - s = &hal->srng_config[HAL_PPE_RELEASE]; - s->max_rings = 0; - - s = &hal->srng_config[HAL_TX_MONITOR_BUF]; - s->max_rings = 0; - - s = &hal->srng_config[HAL_TX_MONITOR_DST]; - s->max_rings = 0; - - s = &hal->srng_config[HAL_PPE2TCL]; - s->max_rings = 0; - - return 0; + ab->hal.ops->reo_init_cmd_ring(ab, srng); } -static bool ath12k_hw_wcn7850_dp_rx_h_msdu_done(struct hal_rx_desc *desc) +void ath12k_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab) { - return !!le32_get_bits(desc->u.wcn7850.msdu_end.info14, - RX_MSDU_END_INFO14_MSDU_DONE); + ab->hal.ops->reo_shared_qaddr_cache_clear(ab); } +EXPORT_SYMBOL(ath12k_hal_reo_shared_qaddr_cache_clear); -static bool ath12k_hw_wcn7850_dp_rx_h_l4_cksum_fail(struct hal_rx_desc *desc) +void ath12k_hal_rx_buf_addr_info_set(struct ath12k_hal *hal, + struct ath12k_buffer_addr *binfo, + dma_addr_t paddr, u32 cookie, u8 manager) { - return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13, - RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL); + hal->ops->rx_buf_addr_info_set(binfo, paddr, cookie, manager); } -static bool ath12k_hw_wcn7850_dp_rx_h_ip_cksum_fail(struct hal_rx_desc *desc) +void ath12k_hal_rx_buf_addr_info_get(struct ath12k_hal *hal, + struct ath12k_buffer_addr *binfo, + dma_addr_t *paddr, u32 *msdu_cookies, + u8 *rbm) { - return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13, - RX_MSDU_END_INFO13_IP_CKSUM_FAIL); + hal->ops->rx_buf_addr_info_get(binfo, paddr, msdu_cookies, rbm); } -static bool ath12k_hw_wcn7850_dp_rx_h_is_decrypted(struct hal_rx_desc *desc) +void ath12k_hal_rx_msdu_list_get(struct ath12k_hal *hal, struct ath12k *ar, + void *link_desc, + void *msdu_list, + u16 *num_msdus) { - return (le32_get_bits(desc->u.wcn7850.msdu_end.info14, - RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) == - RX_DESC_DECRYPT_STATUS_CODE_OK); + hal->ops->rx_msdu_list_get(ar, link_desc, msdu_list, num_msdus); } -static u32 ath12k_hw_wcn7850_dp_rx_h_mpdu_err(struct hal_rx_desc *desc) +void ath12k_hal_rx_reo_ent_buf_paddr_get(struct ath12k_hal *hal, void *rx_desc, + dma_addr_t *paddr, + u32 *sw_cookie, + struct ath12k_buffer_addr **pp_buf_addr, + u8 *rbm, u32 *msdu_cnt) { - u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info13); - u32 errmap = 0; - - if (info & RX_MSDU_END_INFO13_FCS_ERR) - errmap |= HAL_RX_MPDU_ERR_FCS; - - if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) - errmap |= HAL_RX_MPDU_ERR_DECRYPT; - - if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) - errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; - - if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) - errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; - - if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) - errmap |= HAL_RX_MPDU_ERR_OVERFLOW; - - if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) - errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; - - if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) - errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; - - return errmap; + hal->ops->rx_reo_ent_buf_paddr_get(rx_desc, paddr, sw_cookie, + pp_buf_addr, rbm, msdu_cnt); } -static u32 ath12k_hw_wcn7850_get_rx_desc_size(void) +void ath12k_hal_cc_config(struct ath12k_base *ab) { - return sizeof(struct hal_rx_desc_wcn7850); + ab->hal.ops->cc_config(ab); } -static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_src_link(struct hal_rx_desc *desc) +enum hal_rx_buf_return_buf_manager +ath12k_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id) { - return 0; + return hal->ops->get_idle_link_rbm(hal, device_id); } -const struct hal_rx_ops hal_rx_wcn7850_ops = { - .rx_desc_get_first_msdu = ath12k_hw_wcn7850_rx_desc_get_first_msdu, - .rx_desc_get_last_msdu = ath12k_hw_wcn7850_rx_desc_get_last_msdu, - .rx_desc_get_l3_pad_bytes = ath12k_hw_wcn7850_rx_desc_get_l3_pad_bytes, - .rx_desc_encrypt_valid = ath12k_hw_wcn7850_rx_desc_encrypt_valid, - .rx_desc_get_encrypt_type = ath12k_hw_wcn7850_rx_desc_get_encrypt_type, - .rx_desc_get_decap_type = ath12k_hw_wcn7850_rx_desc_get_decap_type, - .rx_desc_get_mesh_ctl = ath12k_hw_wcn7850_rx_desc_get_mesh_ctl, - .rx_desc_get_mpdu_seq_ctl_vld = ath12k_hw_wcn7850_rx_desc_get_mpdu_seq_ctl_vld, - .rx_desc_get_mpdu_fc_valid = ath12k_hw_wcn7850_rx_desc_get_mpdu_fc_valid, - .rx_desc_get_mpdu_start_seq_no = ath12k_hw_wcn7850_rx_desc_get_mpdu_start_seq_no, - .rx_desc_get_msdu_len = ath12k_hw_wcn7850_rx_desc_get_msdu_len, - .rx_desc_get_msdu_sgi = ath12k_hw_wcn7850_rx_desc_get_msdu_sgi, - .rx_desc_get_msdu_rate_mcs = ath12k_hw_wcn7850_rx_desc_get_msdu_rate_mcs, - .rx_desc_get_msdu_rx_bw = ath12k_hw_wcn7850_rx_desc_get_msdu_rx_bw, - .rx_desc_get_msdu_freq = ath12k_hw_wcn7850_rx_desc_get_msdu_freq, - .rx_desc_get_msdu_pkt_type = ath12k_hw_wcn7850_rx_desc_get_msdu_pkt_type, - .rx_desc_get_msdu_nss = ath12k_hw_wcn7850_rx_desc_get_msdu_nss, - .rx_desc_get_mpdu_tid = ath12k_hw_wcn7850_rx_desc_get_mpdu_tid, - .rx_desc_get_mpdu_peer_id = ath12k_hw_wcn7850_rx_desc_get_mpdu_peer_id, - .rx_desc_copy_end_tlv = ath12k_hw_wcn7850_rx_desc_copy_end_tlv, - .rx_desc_get_mpdu_start_tag = ath12k_hw_wcn7850_rx_desc_get_mpdu_start_tag, - .rx_desc_get_mpdu_ppdu_id = ath12k_hw_wcn7850_rx_desc_get_mpdu_ppdu_id, - .rx_desc_set_msdu_len = ath12k_hw_wcn7850_rx_desc_set_msdu_len, - .rx_desc_get_msdu_payload = ath12k_hw_wcn7850_rx_desc_get_msdu_payload, - .rx_desc_get_mpdu_start_offset = ath12k_hw_wcn7850_rx_desc_get_mpdu_start_offset, - .rx_desc_get_msdu_end_offset = ath12k_hw_wcn7850_rx_desc_get_msdu_end_offset, - .rx_desc_mac_addr2_valid = ath12k_hw_wcn7850_rx_desc_mac_addr2_valid, - .rx_desc_mpdu_start_addr2 = ath12k_hw_wcn7850_rx_desc_mpdu_start_addr2, - .rx_desc_is_da_mcbc = ath12k_hw_wcn7850_rx_desc_is_da_mcbc, - .rx_desc_get_dot11_hdr = ath12k_hw_wcn7850_rx_desc_get_dot11_hdr, - .rx_desc_get_crypto_header = ath12k_hw_wcn7850_rx_desc_get_crypto_hdr, - .dp_rx_h_msdu_done = ath12k_hw_wcn7850_dp_rx_h_msdu_done, - .dp_rx_h_l4_cksum_fail = ath12k_hw_wcn7850_dp_rx_h_l4_cksum_fail, - .dp_rx_h_ip_cksum_fail = ath12k_hw_wcn7850_dp_rx_h_ip_cksum_fail, - .dp_rx_h_is_decrypted = ath12k_hw_wcn7850_dp_rx_h_is_decrypted, - .dp_rx_h_mpdu_err = ath12k_hw_wcn7850_dp_rx_h_mpdu_err, - .rx_desc_get_desc_size = ath12k_hw_wcn7850_get_rx_desc_size, - .rx_desc_get_msdu_src_link_id = ath12k_hw_wcn7850_rx_desc_get_msdu_src_link, -}; - -const struct hal_ops hal_wcn7850_ops = { - .create_srng_config = ath12k_hal_srng_create_config_wcn7850, - .tcl_to_wbm_rbm_map = ath12k_hal_wcn7850_tcl_to_wbm_rbm_map, - .rxdma_ring_wmask_rx_mpdu_start = NULL, - .rxdma_ring_wmask_rx_msdu_end = NULL, - .get_hal_rx_compact_ops = NULL, -}; - -static int ath12k_hal_alloc_cont_rdp(struct ath12k_base *ab) +static int ath12k_hal_alloc_cont_rdp(struct ath12k_hal *hal) { - struct ath12k_hal *hal = &ab->hal; size_t size; size = sizeof(u32) * HAL_SRNG_RING_ID_MAX; - hal->rdp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->rdp.paddr, + hal->rdp.vaddr = dma_alloc_coherent(hal->dev, size, &hal->rdp.paddr, GFP_KERNEL); if (!hal->rdp.vaddr) return -ENOMEM; @@ -1575,27 +164,25 @@ static int ath12k_hal_alloc_cont_rdp(struct ath12k_base *ab) return 0; } -static void ath12k_hal_free_cont_rdp(struct ath12k_base *ab) +static void ath12k_hal_free_cont_rdp(struct ath12k_hal *hal) { - struct ath12k_hal *hal = &ab->hal; size_t size; if (!hal->rdp.vaddr) return; size = sizeof(u32) * HAL_SRNG_RING_ID_MAX; - dma_free_coherent(ab->dev, size, + dma_free_coherent(hal->dev, size, hal->rdp.vaddr, hal->rdp.paddr); hal->rdp.vaddr = NULL; } -static int ath12k_hal_alloc_cont_wrp(struct ath12k_base *ab) +static int ath12k_hal_alloc_cont_wrp(struct ath12k_hal *hal) { - struct ath12k_hal *hal = &ab->hal; size_t size; size = sizeof(u32) * (HAL_SRNG_NUM_PMAC_RINGS + HAL_SRNG_NUM_DMAC_RINGS); - hal->wrp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->wrp.paddr, + hal->wrp.vaddr = dma_alloc_coherent(hal->dev, size, &hal->wrp.paddr, GFP_KERNEL); if (!hal->wrp.vaddr) return -ENOMEM; @@ -1603,209 +190,19 @@ static int ath12k_hal_alloc_cont_wrp(struct ath12k_base *ab) return 0; } -static void ath12k_hal_free_cont_wrp(struct ath12k_base *ab) +static void ath12k_hal_free_cont_wrp(struct ath12k_hal *hal) { - struct ath12k_hal *hal = &ab->hal; size_t size; if (!hal->wrp.vaddr) return; size = sizeof(u32) * (HAL_SRNG_NUM_PMAC_RINGS + HAL_SRNG_NUM_DMAC_RINGS); - dma_free_coherent(ab->dev, size, + dma_free_coherent(hal->dev, size, hal->wrp.vaddr, hal->wrp.paddr); hal->wrp.vaddr = NULL; } -static void ath12k_hal_ce_dst_setup(struct ath12k_base *ab, - struct hal_srng *srng, int ring_num) -{ - struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST]; - u32 addr; - u32 val; - - addr = HAL_CE_DST_RING_CTRL + - srng_config->reg_start[HAL_SRNG_REG_GRP_R0] + - ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0]; - - val = ath12k_hif_read32(ab, addr); - val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN; - val |= u32_encode_bits(srng->u.dst_ring.max_buffer_length, - HAL_CE_DST_R0_DEST_CTRL_MAX_LEN); - ath12k_hif_write32(ab, addr, val); -} - -static void ath12k_hal_srng_dst_hw_init(struct ath12k_base *ab, - struct hal_srng *srng) -{ - struct ath12k_hal *hal = &ab->hal; - u32 val; - u64 hp_addr; - u32 reg_base; - - reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; - - if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) { - ath12k_hif_write32(ab, reg_base + - ath12k_hal_reo1_ring_msi1_base_lsb_offset(ab), - srng->msi_addr); - - val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_REO1_RING_MSI1_BASE_MSB_ADDR) | - HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE; - ath12k_hif_write32(ab, reg_base + - ath12k_hal_reo1_ring_msi1_base_msb_offset(ab), val); - - ath12k_hif_write32(ab, - reg_base + ath12k_hal_reo1_ring_msi1_data_offset(ab), - srng->msi_data); - } - - ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr); - - val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB) | - u32_encode_bits((srng->entry_size * srng->num_entries), - HAL_REO1_RING_BASE_MSB_RING_SIZE); - ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_base_msb_offset(ab), val); - - val = u32_encode_bits(srng->ring_id, HAL_REO1_RING_ID_RING_ID) | - u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE); - ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_id_offset(ab), val); - - /* interrupt setup */ - val = u32_encode_bits((srng->intr_timer_thres_us >> 3), - HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD); - - val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size), - HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD); - - ath12k_hif_write32(ab, - reg_base + ath12k_hal_reo1_ring_producer_int_setup_offset(ab), - val); - - hp_addr = hal->rdp.paddr + - ((unsigned long)srng->u.dst_ring.hp_addr - - (unsigned long)hal->rdp.vaddr); - ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_hp_addr_lsb_offset(ab), - hp_addr & HAL_ADDR_LSB_REG_MASK); - ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_hp_addr_msb_offset(ab), - hp_addr >> HAL_ADDR_MSB_REG_SHIFT); - - /* Initialize head and tail pointers to indicate ring is empty */ - reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; - ath12k_hif_write32(ab, reg_base, 0); - ath12k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET, 0); - *srng->u.dst_ring.hp_addr = 0; - - reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; - val = 0; - if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP) - val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP; - if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP) - val |= HAL_REO1_RING_MISC_HOST_FW_SWAP; - if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP) - val |= HAL_REO1_RING_MISC_MSI_SWAP; - val |= HAL_REO1_RING_MISC_SRNG_ENABLE; - - ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_misc_offset(ab), val); -} - -static void ath12k_hal_srng_src_hw_init(struct ath12k_base *ab, - struct hal_srng *srng) -{ - struct ath12k_hal *hal = &ab->hal; - u32 val; - u64 tp_addr; - u32 reg_base; - - reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; - - if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) { - ath12k_hif_write32(ab, reg_base + - HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab), - srng->msi_addr); - - val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_TCL1_RING_MSI1_BASE_MSB_ADDR) | - HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE; - ath12k_hif_write32(ab, reg_base + - HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab), - val); - - ath12k_hif_write32(ab, reg_base + - HAL_TCL1_RING_MSI1_DATA_OFFSET(ab), - srng->msi_data); - } - - ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr); - - val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB) | - u32_encode_bits((srng->entry_size * srng->num_entries), - HAL_TCL1_RING_BASE_MSB_RING_SIZE); - ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val); - - val = u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE); - ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(ab), val); - - val = u32_encode_bits(srng->intr_timer_thres_us, - HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD); - - val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size), - HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD); - - ath12k_hif_write32(ab, - reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab), - val); - - val = 0; - if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) { - val |= u32_encode_bits(srng->u.src_ring.low_threshold, - HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD); - } - ath12k_hif_write32(ab, - reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab), - val); - - if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) { - tp_addr = hal->rdp.paddr + - ((unsigned long)srng->u.src_ring.tp_addr - - (unsigned long)hal->rdp.vaddr); - ath12k_hif_write32(ab, - reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab), - tp_addr & HAL_ADDR_LSB_REG_MASK); - ath12k_hif_write32(ab, - reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab), - tp_addr >> HAL_ADDR_MSB_REG_SHIFT); - } - - /* Initialize head and tail pointers to indicate ring is empty */ - reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; - ath12k_hif_write32(ab, reg_base, 0); - ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0); - *srng->u.src_ring.tp_addr = 0; - - reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; - val = 0; - if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP) - val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP; - if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP) - val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP; - if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP) - val |= HAL_TCL1_RING_MISC_MSI_SWAP; - - /* Loop count is not used for SRC rings */ - val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE; - - val |= HAL_TCL1_RING_MISC_SRNG_ENABLE; - - if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK) - val |= HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE; - - ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(ab), val); -} - static void ath12k_hal_srng_hw_init(struct ath12k_base *ab, struct hal_srng *srng) { @@ -1815,28 +212,6 @@ static void ath12k_hal_srng_hw_init(struct ath12k_base *ab, ath12k_hal_srng_dst_hw_init(ab, srng); } -static int ath12k_hal_srng_get_ring_id(struct ath12k_base *ab, - enum hal_ring_type type, - int ring_num, int mac_id) -{ - struct hal_srng_config *srng_config = &ab->hal.srng_config[type]; - int ring_id; - - if (ring_num >= srng_config->max_rings) { - ath12k_warn(ab, "invalid ring number :%d\n", ring_num); - return -EINVAL; - } - - ring_id = srng_config->start_ring_id + ring_num; - if (srng_config->mac_type == ATH12K_HAL_SRNG_PMAC) - ring_id += mac_id * HAL_SRNG_RINGS_PER_PMAC; - - if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX)) - return -EINVAL; - - return ring_id; -} - int ath12k_hal_srng_get_entrysize(struct ath12k_base *ab, u32 ring_type) { struct hal_srng_config *srng_config; @@ -1848,6 +223,7 @@ int ath12k_hal_srng_get_entrysize(struct ath12k_base *ab, u32 ring_type) return (srng_config->entry_size << 2); } +EXPORT_SYMBOL(ath12k_hal_srng_get_entrysize); int ath12k_hal_srng_get_max_entries(struct ath12k_base *ab, u32 ring_type) { @@ -1877,6 +253,7 @@ void ath12k_hal_srng_get_params(struct ath12k_base *ab, struct hal_srng *srng, params->msi2_data = srng->msi2_data; params->flags = srng->flags; } +EXPORT_SYMBOL(ath12k_hal_srng_get_params); dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab, struct hal_srng *srng) @@ -1910,63 +287,32 @@ dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab, (unsigned long)ab->hal.wrp.vaddr); } -u32 ath12k_hal_ce_get_desc_size(enum hal_ce_desc type) -{ - switch (type) { - case HAL_CE_DESC_SRC: - return sizeof(struct hal_ce_srng_src_desc); - case HAL_CE_DESC_DST: - return sizeof(struct hal_ce_srng_dest_desc); - case HAL_CE_DESC_DST_STATUS: - return sizeof(struct hal_ce_srng_dst_status_desc); - } - - return 0; -} - -void ath12k_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc, dma_addr_t paddr, - u32 len, u32 id, u8 byte_swap_data) +void ath12k_hal_ce_src_set_desc(struct ath12k_hal *hal, + struct hal_ce_srng_src_desc *desc, + dma_addr_t paddr, u32 len, u32 id, + u8 byte_swap_data) { - desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK); - desc->buffer_addr_info = - le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI) | - le32_encode_bits(byte_swap_data, - HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP) | - le32_encode_bits(0, HAL_CE_SRC_DESC_ADDR_INFO_GATHER) | - le32_encode_bits(len, HAL_CE_SRC_DESC_ADDR_INFO_LEN); - desc->meta_info = le32_encode_bits(id, HAL_CE_SRC_DESC_META_INFO_DATA); + hal->ops->ce_src_set_desc(desc, paddr, len, id, byte_swap_data); } -void ath12k_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc, dma_addr_t paddr) +void ath12k_hal_ce_dst_set_desc(struct ath12k_hal *hal, + struct hal_ce_srng_dest_desc *desc, + dma_addr_t paddr) { - desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK); - desc->buffer_addr_info = - le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI); + hal->ops->ce_dst_set_desc(desc, paddr); } -u32 ath12k_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc) +u32 ath12k_hal_ce_dst_status_get_length(struct ath12k_hal *hal, + struct hal_ce_srng_dst_status_desc *desc) { - u32 len; - - len = le32_get_bits(desc->flags, HAL_CE_DST_STATUS_DESC_FLAGS_LEN); - desc->flags &= ~cpu_to_le32(HAL_CE_DST_STATUS_DESC_FLAGS_LEN); - - return len; + return hal->ops->ce_dst_status_get_length(desc); } -void ath12k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie, - dma_addr_t paddr, - enum hal_rx_buf_return_buf_manager rbm) +void ath12k_hal_set_link_desc_addr(struct ath12k_hal *hal, + struct hal_wbm_link_desc *desc, u32 cookie, + dma_addr_t paddr, int rbm) { - desc->buf_addr_info.info0 = le32_encode_bits((paddr & HAL_ADDR_LSB_REG_MASK), - BUFFER_ADDR_INFO0_ADDR); - desc->buf_addr_info.info1 = - le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT), - BUFFER_ADDR_INFO1_ADDR) | - le32_encode_bits(rbm, BUFFER_ADDR_INFO1_RET_BUF_MGR) | - le32_encode_bits(cookie, BUFFER_ADDR_INFO1_SW_COOKIE); + hal->ops->set_link_desc_addr(desc, cookie, paddr, rbm); } void *ath12k_hal_srng_dst_peek(struct ath12k_base *ab, struct hal_srng *srng) @@ -1978,6 +324,7 @@ void *ath12k_hal_srng_dst_peek(struct ath12k_base *ab, struct hal_srng *srng) return NULL; } +EXPORT_SYMBOL(ath12k_hal_srng_dst_peek); void *ath12k_hal_srng_dst_get_next_entry(struct ath12k_base *ab, struct hal_srng *srng) @@ -1996,6 +343,7 @@ void *ath12k_hal_srng_dst_get_next_entry(struct ath12k_base *ab, return desc; } +EXPORT_SYMBOL(ath12k_hal_srng_dst_get_next_entry); int ath12k_hal_srng_dst_num_free(struct ath12k_base *ab, struct hal_srng *srng, bool sync_hw_ptr) @@ -2018,6 +366,7 @@ int ath12k_hal_srng_dst_num_free(struct ath12k_base *ab, struct hal_srng *srng, else return (srng->ring_size - tp + hp) / srng->entry_size; } +EXPORT_SYMBOL(ath12k_hal_srng_dst_num_free); /* Returns number of available entries in src ring */ int ath12k_hal_srng_src_num_free(struct ath12k_base *ab, struct hal_srng *srng, @@ -2059,6 +408,7 @@ void *ath12k_hal_srng_src_next_peek(struct ath12k_base *ab, return desc; } +EXPORT_SYMBOL(ath12k_hal_srng_src_next_peek); void *ath12k_hal_srng_src_get_next_entry(struct ath12k_base *ab, struct hal_srng *srng) @@ -2092,6 +442,7 @@ void *ath12k_hal_srng_src_get_next_entry(struct ath12k_base *ab, return desc; } +EXPORT_SYMBOL(ath12k_hal_srng_src_get_next_entry); void *ath12k_hal_srng_src_peek(struct ath12k_base *ab, struct hal_srng *srng) { @@ -2103,6 +454,7 @@ void *ath12k_hal_srng_src_peek(struct ath12k_base *ab, struct hal_srng *srng) return srng->ring_base_vaddr + srng->u.src_ring.hp; } +EXPORT_SYMBOL(ath12k_hal_srng_src_peek); void *ath12k_hal_srng_src_reap_next(struct ath12k_base *ab, struct hal_srng *srng) @@ -2162,6 +514,7 @@ void ath12k_hal_srng_access_begin(struct ath12k_base *ab, struct hal_srng *srng) } } } +EXPORT_SYMBOL(ath12k_hal_srng_access_begin); /* Update cached ring head/tail pointers to HW. ath12k_hal_srng_access_begin() * should have been called before this. @@ -2217,112 +570,7 @@ void ath12k_hal_srng_access_end(struct ath12k_base *ab, struct hal_srng *srng) srng->timestamp = jiffies; } - -void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab, - struct hal_wbm_idle_scatter_list *sbuf, - u32 nsbufs, u32 tot_link_desc, - u32 end_offset) -{ - struct ath12k_buffer_addr *link_addr; - int i; - u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64; - u32 val; - - link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE; - - for (i = 1; i < nsbufs; i++) { - link_addr->info0 = cpu_to_le32(sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK); - - link_addr->info1 = - le32_encode_bits((u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT, - HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) | - le32_encode_bits(BASE_ADDR_MATCH_TAG_VAL, - HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG); - - link_addr = (void *)sbuf[i].vaddr + - HAL_WBM_IDLE_SCATTER_BUF_SIZE; - } - - val = u32_encode_bits(reg_scatter_buf_sz, HAL_WBM_SCATTER_BUFFER_SIZE) | - u32_encode_bits(0x1, HAL_WBM_LINK_DESC_IDLE_LIST_MODE); - - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(ab), - val); - - val = u32_encode_bits(reg_scatter_buf_sz * nsbufs, - HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(ab), - val); - - val = u32_encode_bits(sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK, - BUFFER_ADDR_INFO0_ADDR); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_RING_BASE_LSB(ab), - val); - - val = u32_encode_bits(BASE_ADDR_MATCH_TAG_VAL, - HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG) | - u32_encode_bits((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT, - HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_RING_BASE_MSB(ab), - val); - - /* Setup head and tail pointers for the idle list */ - val = u32_encode_bits(sbuf[nsbufs - 1].paddr, BUFFER_ADDR_INFO0_ADDR); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(ab), - val); - - val = u32_encode_bits(((u64)sbuf[nsbufs - 1].paddr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) | - u32_encode_bits((end_offset >> 2), - HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(ab), - val); - - val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(ab), - val); - - val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(ab), - val); - - val = u32_encode_bits(((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT), - HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) | - u32_encode_bits(0, HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(ab), - val); - - val = 2 * tot_link_desc; - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(ab), - val); - - /* Enable the SRNG */ - val = u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE) | - u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE); - ath12k_hif_write32(ab, - HAL_SEQ_WCSS_UMAC_WBM_REG + - HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab), - val); -} +EXPORT_SYMBOL(ath12k_hal_srng_access_end); int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type, int ring_num, int mac_id, @@ -2334,9 +582,8 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type, int ring_id; u32 idx; int i; - u32 reg_base; - ring_id = ath12k_hal_srng_get_ring_id(ab, type, ring_num, mac_id); + ring_id = ath12k_hal_srng_get_ring_id(hal, type, ring_num, mac_id); if (ring_id < 0) return ring_id; @@ -2369,8 +616,6 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type, memset(srng->ring_base_vaddr, 0, (srng->entry_size * srng->num_entries) << 2); - reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; - if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.hp = 0; srng->u.src_ring.cached_tp = 0; @@ -2379,16 +624,7 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type, srng->u.src_ring.low_threshold = params->low_threshold * srng->entry_size; if (srng_config->mac_type == ATH12K_HAL_SRNG_UMAC) { - if (!ab->hw_params->supports_shadow_regs) - srng->u.src_ring.hp_addr = - (u32 *)((unsigned long)ab->mem + reg_base); - else - ath12k_dbg(ab, ATH12K_DBG_HAL, - "hal type %d ring_num %d reg_base 0x%x shadow 0x%lx\n", - type, ring_num, - reg_base, - (unsigned long)srng->u.src_ring.hp_addr - - (unsigned long)ab->mem); + ath12k_hal_set_umac_srng_ptr_addr(ab, srng); } else { idx = ring_id - HAL_SRNG_RING_ID_DMAC_CMN_ID_START; srng->u.src_ring.hp_addr = (void *)(hal->wrp.vaddr + @@ -2409,17 +645,7 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type, srng->u.dst_ring.cached_hp = 0; srng->u.dst_ring.hp_addr = (void *)(hal->rdp.vaddr + ring_id); if (srng_config->mac_type == ATH12K_HAL_SRNG_UMAC) { - if (!ab->hw_params->supports_shadow_regs) - srng->u.dst_ring.tp_addr = - (u32 *)((unsigned long)ab->mem + reg_base + - (HAL_REO1_RING_TP - HAL_REO1_RING_HP)); - else - ath12k_dbg(ab, ATH12K_DBG_HAL, - "type %d ring_num %d target_reg 0x%x shadow 0x%lx\n", - type, ring_num, - reg_base + HAL_REO1_RING_TP - HAL_REO1_RING_HP, - (unsigned long)srng->u.dst_ring.tp_addr - - (unsigned long)ab->mem); + ath12k_hal_set_umac_srng_ptr_addr(ab, srng); } else { /* For PMAC & DMAC rings, tail pointer updates will be done * through FW by writing to a shared memory location @@ -2444,68 +670,6 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type, return ring_id; } -static void ath12k_hal_srng_update_hp_tp_addr(struct ath12k_base *ab, - int shadow_cfg_idx, - enum hal_ring_type ring_type, - int ring_num) -{ - struct hal_srng *srng; - struct ath12k_hal *hal = &ab->hal; - int ring_id; - struct hal_srng_config *srng_config = &hal->srng_config[ring_type]; - - ring_id = ath12k_hal_srng_get_ring_id(ab, ring_type, ring_num, 0); - if (ring_id < 0) - return; - - srng = &hal->srng_list[ring_id]; - - if (srng_config->ring_dir == HAL_SRNG_DIR_DST) - srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) + - (unsigned long)ab->mem); - else - srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) + - (unsigned long)ab->mem); -} - -int ath12k_hal_srng_update_shadow_config(struct ath12k_base *ab, - enum hal_ring_type ring_type, - int ring_num) -{ - struct ath12k_hal *hal = &ab->hal; - struct hal_srng_config *srng_config = &hal->srng_config[ring_type]; - int shadow_cfg_idx = hal->num_shadow_reg_configured; - u32 target_reg; - - if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS) - return -EINVAL; - - hal->num_shadow_reg_configured++; - - target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START]; - target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] * - ring_num; - - /* For destination ring, shadow the TP */ - if (srng_config->ring_dir == HAL_SRNG_DIR_DST) - target_reg += HAL_OFFSET_FROM_HP_TO_TP; - - hal->shadow_reg_addr[shadow_cfg_idx] = target_reg; - - /* update hp/tp addr to hal structure*/ - ath12k_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type, - ring_num); - - ath12k_dbg(ab, ATH12K_DBG_HAL, - "target_reg %x, shadow reg 0x%x shadow_idx 0x%x, ring_type %d, ring num %d", - target_reg, - HAL_SHADOW_REG(shadow_cfg_idx), - shadow_cfg_idx, - ring_type, ring_num); - - return 0; -} - void ath12k_hal_srng_shadow_config(struct ath12k_base *ab) { struct ath12k_hal *hal = &ab->hal; @@ -2551,18 +715,16 @@ void ath12k_hal_srng_shadow_update_hp_tp(struct ath12k_base *ab, ath12k_hal_srng_access_end(ab, srng); } -static void ath12k_hal_register_srng_lock_keys(struct ath12k_base *ab) +static void ath12k_hal_register_srng_lock_keys(struct ath12k_hal *hal) { - struct ath12k_hal *hal = &ab->hal; u32 ring_id; for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++) lockdep_register_key(&hal->srng_list[ring_id].lock_key); } -static void ath12k_hal_unregister_srng_lock_keys(struct ath12k_base *ab) +static void ath12k_hal_unregister_srng_lock_keys(struct ath12k_hal *hal) { - struct ath12k_hal *hal = &ab->hal; u32 ring_id; for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++) @@ -2574,26 +736,26 @@ int ath12k_hal_srng_init(struct ath12k_base *ab) struct ath12k_hal *hal = &ab->hal; int ret; - memset(hal, 0, sizeof(*hal)); - - ret = ab->hw_params->hal_ops->create_srng_config(ab); + ret = hal->ops->create_srng_config(hal); if (ret) goto err_hal; - ret = ath12k_hal_alloc_cont_rdp(ab); + hal->dev = ab->dev; + + ret = ath12k_hal_alloc_cont_rdp(hal); if (ret) goto err_hal; - ret = ath12k_hal_alloc_cont_wrp(ab); + ret = ath12k_hal_alloc_cont_wrp(hal); if (ret) goto err_free_cont_rdp; - ath12k_hal_register_srng_lock_keys(ab); + ath12k_hal_register_srng_lock_keys(hal); return 0; err_free_cont_rdp: - ath12k_hal_free_cont_rdp(ab); + ath12k_hal_free_cont_rdp(hal); err_hal: return ret; @@ -2603,9 +765,9 @@ void ath12k_hal_srng_deinit(struct ath12k_base *ab) { struct ath12k_hal *hal = &ab->hal; - ath12k_hal_unregister_srng_lock_keys(ab); - ath12k_hal_free_cont_rdp(ab); - ath12k_hal_free_cont_wrp(ab); + ath12k_hal_unregister_srng_lock_keys(hal); + ath12k_hal_free_cont_rdp(hal); + ath12k_hal_free_cont_wrp(hal); kfree(hal->srng_config); hal->srng_config = NULL; } @@ -2661,3 +823,67 @@ void ath12k_hal_dump_srng_stats(struct ath12k_base *ab) jiffies_to_msecs(jiffies - srng->timestamp)); } } + +void *ath12k_hal_encode_tlv64_hdr(void *tlv, u64 tag, u64 len) +{ + struct hal_tlv_64_hdr *tlv64 = tlv; + + tlv64->tl = le64_encode_bits(tag, HAL_TLV_64_HDR_TAG) | + le64_encode_bits(len, HAL_TLV_64_HDR_LEN); + + return tlv64->value; +} +EXPORT_SYMBOL(ath12k_hal_encode_tlv64_hdr); + +void *ath12k_hal_encode_tlv32_hdr(void *tlv, u64 tag, u64 len) +{ + struct hal_tlv_hdr *tlv32 = tlv; + + tlv32->tl = le32_encode_bits(tag, HAL_TLV_HDR_TAG) | + le32_encode_bits(len, HAL_TLV_HDR_LEN); + + return tlv32->value; +} +EXPORT_SYMBOL(ath12k_hal_encode_tlv32_hdr); + +void *ath12k_hal_decode_tlv64_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid) +{ + struct hal_tlv_64_hdr *tlv64 = tlv; + + if (tag) + *tag = le64_get_bits(tlv64->tl, HAL_TLV_64_HDR_TAG); + if (len) + *len = le64_get_bits(tlv64->tl, HAL_TLV_64_HDR_LEN); + if (usrid) + *usrid = le64_get_bits(tlv64->tl, HAL_TLV_64_USR_ID); + + return tlv64->value; +} +EXPORT_SYMBOL(ath12k_hal_decode_tlv64_hdr); + +void *ath12k_hal_decode_tlv32_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid) +{ + struct hal_tlv_hdr *tlv32 = tlv; + + if (tag) + *tag = le32_get_bits(tlv32->tl, HAL_TLV_HDR_TAG); + if (len) + *len = le32_get_bits(tlv32->tl, HAL_TLV_HDR_LEN); + if (usrid) + *usrid = le32_get_bits(tlv32->tl, HAL_TLV_USR_ID); + + return tlv32->value; +} +EXPORT_SYMBOL(ath12k_hal_decode_tlv32_hdr); + +u32 ath12k_hal_get_tlv64_hdr_align(void) +{ + return HAL_TLV_64_ALIGN; +} +EXPORT_SYMBOL(ath12k_hal_get_tlv64_hdr_align); + +u32 ath12k_hal_get_tlv32_hdr_align(void) +{ + return HAL_TLV_ALIGN; +} +EXPORT_SYMBOL(ath12k_hal_get_tlv32_hdr_align); diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h index efe00e1679986..c104ab3a60567 100644 --- a/drivers/net/wireless/ath/ath12k/hal.h +++ b/drivers/net/wireless/ath/ath12k/hal.h @@ -1,16 +1,43 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_HAL_H #define ATH12K_HAL_H -#include "hal_desc.h" -#include "rx_desc.h" +#include "hw.h" struct ath12k_base; + +#define HAL_DESC_REO_NON_QOS_TID 16 + +#define HAL_INVALID_PEERID 0x3fff +#define VHT_SIG_SU_NSS_MASK 0x7 + +#define HAL_TX_ADDRX_EN 1 +#define HAL_TX_ADDRY_EN 2 + +#define HAL_TX_ADDR_SEARCH_DEFAULT 0 +#define HAL_TX_ADDR_SEARCH_INDEX 1 + +#define HAL_RX_MAX_MPDU 256 +#define HAL_RX_NUM_WORDS_PER_PPDU_BITMAP (HAL_RX_MAX_MPDU >> 5) + +/* TODO: 16 entries per radio times MAX_VAPS_SUPPORTED */ +#define HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX 32 +#define HAL_DSCP_TID_TBL_SIZE 24 + +#define EHT_MAX_USER_INFO 4 +#define HAL_RX_MON_MAX_AGGR_SIZE 128 +#define HAL_MAX_UL_MU_USERS 37 + +#define MAX_USER_POS 8 +#define MAX_MU_GROUP_ID 64 +#define MAX_MU_GROUP_SHOW 16 +#define MAX_MU_GROUP_LENGTH (6 * MAX_MU_GROUP_SHOW) + #define HAL_CE_REMAP_REG_BASE (ab->ce_remap_base_addr) #define HAL_LINK_DESC_SIZE (32 << 2) @@ -24,364 +51,37 @@ struct ath12k_base; #define HAL_RING_BASE_ALIGN 8 #define HAL_REO_QLUT_ADDR_ALIGN 256 +#define HAL_ADDR_LSB_REG_MASK 0xffffffff +#define HAL_ADDR_MSB_REG_SHIFT 32 + +#define HAL_WBM2SW_REL_ERR_RING_NUM 3 + +#define HAL_SHADOW_NUM_REGS_MAX 40 + #define HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX 32704 /* TODO: Check with hw team on the supported scatter buf size */ #define HAL_WBM_IDLE_SCATTER_NEXT_PTR_SIZE 8 #define HAL_WBM_IDLE_SCATTER_BUF_SIZE (HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX - \ HAL_WBM_IDLE_SCATTER_NEXT_PTR_SIZE) -/* TODO: 16 entries per radio times MAX_VAPS_SUPPORTED */ -#define HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX 32 -#define HAL_DSCP_TID_TBL_SIZE 24 - -/* calculate the register address from bar0 of shadow register x */ -#define HAL_SHADOW_BASE_ADDR 0x000008fc -#define HAL_SHADOW_NUM_REGS 40 -#define HAL_HP_OFFSET_IN_REG_START 1 -#define HAL_OFFSET_FROM_HP_TO_TP 4 - -#define HAL_SHADOW_REG(x) (HAL_SHADOW_BASE_ADDR + (4 * (x))) -#define HAL_REO_QDESC_MAX_PEERID 8191 - -/* WCSS Relative address */ -#define HAL_SEQ_WCSS_CMEM_OFFSET 0x00100000 -#define HAL_SEQ_WCSS_UMAC_OFFSET 0x00a00000 -#define HAL_SEQ_WCSS_UMAC_REO_REG 0x00a38000 -#define HAL_SEQ_WCSS_UMAC_TCL_REG 0x00a44000 -#define HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) \ - ((ab)->hw_params->regs->hal_umac_ce0_src_reg_base) -#define HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) \ - ((ab)->hw_params->regs->hal_umac_ce0_dest_reg_base) -#define HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) \ - ((ab)->hw_params->regs->hal_umac_ce1_src_reg_base) -#define HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) \ - ((ab)->hw_params->regs->hal_umac_ce1_dest_reg_base) -#define HAL_SEQ_WCSS_UMAC_WBM_REG 0x00a34000 - -#define HAL_CE_WFSS_CE_REG_BASE 0x01b80000 - -#define HAL_TCL_SW_CONFIG_BANK_ADDR 0x00a4408c - -/* SW2TCL(x) R0 ring configuration address */ -#define HAL_TCL1_RING_CMN_CTRL_REG 0x00000020 -#define HAL_TCL1_RING_DSCP_TID_MAP 0x00000240 -#define HAL_TCL1_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_base_lsb) -#define HAL_TCL1_RING_BASE_MSB(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_base_msb) -#define HAL_TCL1_RING_ID(ab) ((ab)->hw_params->regs->hal_tcl1_ring_id) -#define HAL_TCL1_RING_MISC(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_misc) -#define HAL_TCL1_RING_TP_ADDR_LSB(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_tp_addr_lsb) -#define HAL_TCL1_RING_TP_ADDR_MSB(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_tp_addr_msb) -#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_consumer_int_setup_ix0) -#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_consumer_int_setup_ix1) -#define HAL_TCL1_RING_MSI1_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_msi1_base_lsb) -#define HAL_TCL1_RING_MSI1_BASE_MSB(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_msi1_base_msb) -#define HAL_TCL1_RING_MSI1_DATA(ab) \ - ((ab)->hw_params->regs->hal_tcl1_ring_msi1_data) -#define HAL_TCL2_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_tcl2_ring_base_lsb) -#define HAL_TCL_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_tcl_ring_base_lsb) - -#define HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_MSI1_BASE_LSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_MSI1_BASE_MSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_MSI1_DATA_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_MSI1_DATA(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_BASE_MSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_BASE_MSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_ID_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_ID(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_TP_ADDR_LSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_TP_ADDR_MSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) -#define HAL_TCL1_RING_MISC_OFFSET(ab) ({ typeof(ab) _ab = (ab); \ - (HAL_TCL1_RING_MISC(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); }) - -/* SW2TCL(x) R2 ring pointers (head/tail) address */ -#define HAL_TCL1_RING_HP 0x00002000 -#define HAL_TCL1_RING_TP 0x00002004 -#define HAL_TCL2_RING_HP 0x00002008 -#define HAL_TCL_RING_HP 0x00002028 - -#define HAL_TCL1_RING_TP_OFFSET \ - (HAL_TCL1_RING_TP - HAL_TCL1_RING_HP) - -/* TCL STATUS ring address */ -#define HAL_TCL_STATUS_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_tcl_status_ring_base_lsb) -#define HAL_TCL_STATUS_RING_HP 0x00002048 - -/* PPE2TCL1 Ring address */ -#define HAL_TCL_PPE2TCL1_RING_BASE_LSB 0x00000c48 -#define HAL_TCL_PPE2TCL1_RING_HP 0x00002038 - -/* WBM PPE Release Ring address */ -#define HAL_WBM_PPE_RELEASE_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_ppe_rel_ring_base) -#define HAL_WBM_PPE_RELEASE_RING_HP 0x00003020 - -/* REO2SW(x) R0 ring configuration address */ -#define HAL_REO1_GEN_ENABLE 0x00000000 -#define HAL_REO1_MISC_CTRL_ADDR(ab) \ - ((ab)->hw_params->regs->hal_reo1_misc_ctrl_addr) -#define HAL_REO1_DEST_RING_CTRL_IX_0 0x00000004 -#define HAL_REO1_DEST_RING_CTRL_IX_1 0x00000008 -#define HAL_REO1_DEST_RING_CTRL_IX_2 0x0000000c -#define HAL_REO1_DEST_RING_CTRL_IX_3 0x00000010 -#define HAL_REO1_QDESC_ADDR(ab) ((ab)->hw_params->regs->hal_reo1_qdesc_addr) -#define HAL_REO1_QDESC_MAX_PEERID(ab) ((ab)->hw_params->regs->hal_reo1_qdesc_max_peerid) -#define HAL_REO1_SW_COOKIE_CFG0(ab) ((ab)->hw_params->regs->hal_reo1_sw_cookie_cfg0) -#define HAL_REO1_SW_COOKIE_CFG1(ab) ((ab)->hw_params->regs->hal_reo1_sw_cookie_cfg1) -#define HAL_REO1_QDESC_LUT_BASE0(ab) ((ab)->hw_params->regs->hal_reo1_qdesc_lut_base0) -#define HAL_REO1_QDESC_LUT_BASE1(ab) ((ab)->hw_params->regs->hal_reo1_qdesc_lut_base1) -#define HAL_REO1_RING_BASE_LSB(ab) ((ab)->hw_params->regs->hal_reo1_ring_base_lsb) -#define HAL_REO1_RING_BASE_MSB(ab) ((ab)->hw_params->regs->hal_reo1_ring_base_msb) -#define HAL_REO1_RING_ID(ab) ((ab)->hw_params->regs->hal_reo1_ring_id) -#define HAL_REO1_RING_MISC(ab) ((ab)->hw_params->regs->hal_reo1_ring_misc) -#define HAL_REO1_RING_HP_ADDR_LSB(ab) ((ab)->hw_params->regs->hal_reo1_ring_hp_addr_lsb) -#define HAL_REO1_RING_HP_ADDR_MSB(ab) ((ab)->hw_params->regs->hal_reo1_ring_hp_addr_msb) -#define HAL_REO1_RING_PRODUCER_INT_SETUP(ab) \ - ((ab)->hw_params->regs->hal_reo1_ring_producer_int_setup) -#define HAL_REO1_RING_MSI1_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_reo1_ring_msi1_base_lsb) -#define HAL_REO1_RING_MSI1_BASE_MSB(ab) \ - ((ab)->hw_params->regs->hal_reo1_ring_msi1_base_msb) -#define HAL_REO1_RING_MSI1_DATA(ab) ((ab)->hw_params->regs->hal_reo1_ring_msi1_data) -#define HAL_REO2_RING_BASE_LSB(ab) ((ab)->hw_params->regs->hal_reo2_ring_base) -#define HAL_REO1_AGING_THRESH_IX_0(ab) ((ab)->hw_params->regs->hal_reo1_aging_thres_ix0) -#define HAL_REO1_AGING_THRESH_IX_1(ab) ((ab)->hw_params->regs->hal_reo1_aging_thres_ix1) -#define HAL_REO1_AGING_THRESH_IX_2(ab) ((ab)->hw_params->regs->hal_reo1_aging_thres_ix2) -#define HAL_REO1_AGING_THRESH_IX_3(ab) ((ab)->hw_params->regs->hal_reo1_aging_thres_ix3) - -/* REO2SW(x) R2 ring pointers (head/tail) address */ -#define HAL_REO1_RING_HP 0x00003048 -#define HAL_REO1_RING_TP 0x0000304c -#define HAL_REO2_RING_HP 0x00003050 - -#define HAL_REO1_RING_TP_OFFSET (HAL_REO1_RING_TP - HAL_REO1_RING_HP) - -/* REO2SW0 ring configuration address */ -#define HAL_REO_SW0_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_reo2_sw0_ring_base) - -/* REO2SW0 R2 ring pointer (head/tail) address */ -#define HAL_REO_SW0_RING_HP 0x00003088 - -/* REO CMD R0 address */ -#define HAL_REO_CMD_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_reo_cmd_ring_base) - -/* REO CMD R2 address */ -#define HAL_REO_CMD_HP 0x00003020 - -/* SW2REO R0 address */ -#define HAL_SW2REO_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_sw2reo_ring_base) -#define HAL_SW2REO1_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_sw2reo1_ring_base) - -/* SW2REO R2 address */ -#define HAL_SW2REO_RING_HP 0x00003028 -#define HAL_SW2REO1_RING_HP 0x00003030 - -/* CE ring R0 address */ -#define HAL_CE_SRC_RING_BASE_LSB 0x00000000 -#define HAL_CE_DST_RING_BASE_LSB 0x00000000 -#define HAL_CE_DST_STATUS_RING_BASE_LSB 0x00000058 -#define HAL_CE_DST_RING_CTRL 0x000000b0 - -/* CE ring R2 address */ -#define HAL_CE_DST_RING_HP 0x00000400 -#define HAL_CE_DST_STATUS_RING_HP 0x00000408 - -/* REO status address */ -#define HAL_REO_STATUS_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_reo_status_ring_base) -#define HAL_REO_STATUS_HP 0x000030a8 - -/* WBM Idle R0 address */ -#define HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_wbm_idle_ring_base_lsb) -#define HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab) \ - ((ab)->hw_params->regs->hal_wbm_idle_ring_misc_addr) -#define HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(ab) \ - ((ab)->hw_params->regs->hal_wbm_r0_idle_list_cntl_addr) -#define HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(ab) \ - ((ab)->hw_params->regs->hal_wbm_r0_idle_list_size_addr) -#define HAL_WBM_SCATTERED_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_wbm_scattered_ring_base_lsb) -#define HAL_WBM_SCATTERED_RING_BASE_MSB(ab) \ - ((ab)->hw_params->regs->hal_wbm_scattered_ring_base_msb) -#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(ab) \ - ((ab)->hw_params->regs->hal_wbm_scattered_desc_head_info_ix0) -#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(ab) \ - ((ab)->hw_params->regs->hal_wbm_scattered_desc_head_info_ix1) -#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(ab) \ - ((ab)->hw_params->regs->hal_wbm_scattered_desc_tail_info_ix0) -#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(ab) \ - ((ab)->hw_params->regs->hal_wbm_scattered_desc_tail_info_ix1) -#define HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(ab) \ - ((ab)->hw_params->regs->hal_wbm_scattered_desc_ptr_hp_addr) - -/* WBM Idle R2 address */ -#define HAL_WBM_IDLE_LINK_RING_HP 0x000030b8 - -/* SW2WBM R0 release address */ -#define HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_wbm_sw_release_ring_base_lsb) -#define HAL_WBM_SW1_RELEASE_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_wbm_sw1_release_ring_base_lsb) - -/* SW2WBM R2 release address */ -#define HAL_WBM_SW_RELEASE_RING_HP 0x00003010 -#define HAL_WBM_SW1_RELEASE_RING_HP 0x00003018 - -/* WBM2SW R0 release address */ -#define HAL_WBM0_RELEASE_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_wbm0_release_ring_base_lsb) - -#define HAL_WBM1_RELEASE_RING_BASE_LSB(ab) \ - ((ab)->hw_params->regs->hal_wbm1_release_ring_base_lsb) - -/* WBM2SW R2 release address */ -#define HAL_WBM0_RELEASE_RING_HP 0x000030c8 -#define HAL_WBM1_RELEASE_RING_HP 0x000030d0 - -/* WBM cookie config address and mask */ -#define HAL_WBM_SW_COOKIE_CFG0 0x00000040 -#define HAL_WBM_SW_COOKIE_CFG1 0x00000044 -#define HAL_WBM_SW_COOKIE_CFG2 0x00000090 -#define HAL_WBM_SW_COOKIE_CONVERT_CFG 0x00000094 - -#define HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB GENMASK(7, 0) -#define HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB GENMASK(12, 8) -#define HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB GENMASK(17, 13) -#define HAL_WBM_SW_COOKIE_CFG_ALIGN BIT(18) -#define HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN BIT(0) -#define HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN BIT(1) -#define HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN BIT(3) - -#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN BIT(1) -#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN BIT(2) -#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN BIT(3) -#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN BIT(4) -#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN BIT(5) -#define HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN BIT(8) - -/* TCL ring field mask and offset */ -#define HAL_TCL1_RING_BASE_MSB_RING_SIZE GENMASK(27, 8) -#define HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB GENMASK(7, 0) -#define HAL_TCL1_RING_ID_ENTRY_SIZE GENMASK(7, 0) -#define HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE BIT(0) -#define HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE BIT(1) -#define HAL_TCL1_RING_MISC_MSI_SWAP BIT(3) -#define HAL_TCL1_RING_MISC_HOST_FW_SWAP BIT(4) -#define HAL_TCL1_RING_MISC_DATA_TLV_SWAP BIT(5) -#define HAL_TCL1_RING_MISC_SRNG_ENABLE BIT(6) -#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD GENMASK(31, 16) -#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD GENMASK(14, 0) -#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD GENMASK(15, 0) -#define HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE BIT(8) -#define HAL_TCL1_RING_MSI1_BASE_MSB_ADDR GENMASK(7, 0) -#define HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN BIT(23) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP GENMASK(31, 0) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP0 GENMASK(2, 0) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP1 GENMASK(5, 3) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP2 GENMASK(8, 6) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP3 GENMASK(11, 9) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP4 GENMASK(14, 12) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP5 GENMASK(17, 15) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP6 GENMASK(20, 18) -#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP7 GENMASK(23, 21) - -/* REO ring field mask and offset */ -#define HAL_REO1_RING_BASE_MSB_RING_SIZE GENMASK(27, 8) -#define HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB GENMASK(7, 0) -#define HAL_REO1_RING_ID_RING_ID GENMASK(15, 8) -#define HAL_REO1_RING_ID_ENTRY_SIZE GENMASK(7, 0) -#define HAL_REO1_RING_MISC_MSI_SWAP BIT(3) -#define HAL_REO1_RING_MISC_HOST_FW_SWAP BIT(4) -#define HAL_REO1_RING_MISC_DATA_TLV_SWAP BIT(5) -#define HAL_REO1_RING_MISC_SRNG_ENABLE BIT(6) -#define HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD GENMASK(31, 16) -#define HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD GENMASK(14, 0) -#define HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE BIT(8) -#define HAL_REO1_RING_MSI1_BASE_MSB_ADDR GENMASK(7, 0) -#define HAL_REO1_MISC_CTL_FRAG_DST_RING GENMASK(20, 17) -#define HAL_REO1_MISC_CTL_BAR_DST_RING GENMASK(24, 21) -#define HAL_REO1_GEN_ENABLE_AGING_LIST_ENABLE BIT(2) -#define HAL_REO1_GEN_ENABLE_AGING_FLUSH_ENABLE BIT(3) -#define HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB GENMASK(7, 0) -#define HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB GENMASK(12, 8) -#define HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB GENMASK(17, 13) -#define HAL_REO1_SW_COOKIE_CFG_ALIGN BIT(18) -#define HAL_REO1_SW_COOKIE_CFG_ENABLE BIT(19) -#define HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE BIT(20) -#define HAL_REO_QDESC_ADDR_READ_LUT_ENABLE BIT(7) -#define HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY BIT(6) - -/* CE ring bit field mask and shift */ -#define HAL_CE_DST_R0_DEST_CTRL_MAX_LEN GENMASK(15, 0) - -#define HAL_ADDR_LSB_REG_MASK 0xffffffff - -#define HAL_ADDR_MSB_REG_SHIFT 32 - -/* WBM ring bit field mask and shift */ -#define HAL_WBM_LINK_DESC_IDLE_LIST_MODE BIT(1) -#define HAL_WBM_SCATTER_BUFFER_SIZE GENMASK(10, 2) -#define HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST GENMASK(31, 16) -#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32 GENMASK(7, 0) -#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG GENMASK(31, 8) - -#define HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1 GENMASK(20, 8) -#define HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1 GENMASK(20, 8) - -#define HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE BIT(6) -#define HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE BIT(0) - -#define BASE_ADDR_MATCH_TAG_VAL 0x5 - -#define HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE 0x000fffff -#define HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE 0x000fffff -#define HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_REO_CMD_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE 0x000fffff -#define HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE 0x000fffff -#define HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_CE_SRC_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_CE_DST_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE 0x000fffff -#define HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE 0x0000ffff -#define HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE 0x000fffff -#define HAL_RXDMA_RING_MAX_SIZE 0x0000ffff -#define HAL_RXDMA_RING_MAX_SIZE_BE 0x000fffff -#define HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE 0x000fffff - -#define HAL_WBM2SW_REL_ERR_RING_NUM 3 -/* Add any other errors here and return them in - * ath12k_hal_rx_desc_get_err(). - */ - -#define HAL_IPQ5332_CE_WFSS_REG_BASE 0x740000 -#define HAL_IPQ5332_CE_SIZE 0x100000 +#define HAL_AST_IDX_INVALID 0xFFFF +#define HAL_RX_MAX_MCS 12 +#define HAL_RX_MAX_MCS_HT 31 +#define HAL_RX_MAX_MCS_VHT 9 +#define HAL_RX_MAX_MCS_HE 11 +#define HAL_RX_MAX_MCS_BE 15 +#define HAL_RX_MAX_NSS 8 +#define HAL_RX_MAX_NUM_LEGACY_RATES 12 + +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VALID BIT(30) +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VER BIT(31) +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_NSS GENMASK(2, 0) +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_MCS GENMASK(6, 3) +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_LDPC BIT(7) +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_DCM BIT(8) +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_START GENMASK(15, 9) +#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_SIZE GENMASK(18, 16) +#define HAL_RX_FCS_LEN 4 enum hal_srng_ring_id { HAL_SRNG_RING_ID_REO2SW0 = 0, @@ -525,6 +225,64 @@ enum hal_srng_ring_id { #define HAL_SRNG_RING_ID_MAX (HAL_SRNG_RING_ID_DMAC_CMN_ID_END + \ HAL_SRNG_NUM_PMAC_RINGS) +enum hal_rx_su_mu_coding { + HAL_RX_SU_MU_CODING_BCC, + HAL_RX_SU_MU_CODING_LDPC, + HAL_RX_SU_MU_CODING_MAX, +}; + +enum hal_rx_gi { + HAL_RX_GI_0_8_US, + HAL_RX_GI_0_4_US, + HAL_RX_GI_1_6_US, + HAL_RX_GI_3_2_US, + HAL_RX_GI_MAX, +}; + +enum hal_rx_bw { + HAL_RX_BW_20MHZ, + HAL_RX_BW_40MHZ, + HAL_RX_BW_80MHZ, + HAL_RX_BW_160MHZ, + HAL_RX_BW_320MHZ, + HAL_RX_BW_MAX, +}; + +enum hal_rx_preamble { + HAL_RX_PREAMBLE_11A, + HAL_RX_PREAMBLE_11B, + HAL_RX_PREAMBLE_11N, + HAL_RX_PREAMBLE_11AC, + HAL_RX_PREAMBLE_11AX, + HAL_RX_PREAMBLE_11BA, + HAL_RX_PREAMBLE_11BE, + HAL_RX_PREAMBLE_MAX, +}; + +enum hal_rx_reception_type { + HAL_RX_RECEPTION_TYPE_SU, + HAL_RX_RECEPTION_TYPE_MU_MIMO, + HAL_RX_RECEPTION_TYPE_MU_OFDMA, + HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO, + HAL_RX_RECEPTION_TYPE_MAX, +}; + +enum hal_rx_legacy_rate { + HAL_RX_LEGACY_RATE_1_MBPS, + HAL_RX_LEGACY_RATE_2_MBPS, + HAL_RX_LEGACY_RATE_5_5_MBPS, + HAL_RX_LEGACY_RATE_6_MBPS, + HAL_RX_LEGACY_RATE_9_MBPS, + HAL_RX_LEGACY_RATE_11_MBPS, + HAL_RX_LEGACY_RATE_12_MBPS, + HAL_RX_LEGACY_RATE_18_MBPS, + HAL_RX_LEGACY_RATE_24_MBPS, + HAL_RX_LEGACY_RATE_36_MBPS, + HAL_RX_LEGACY_RATE_48_MBPS, + HAL_RX_LEGACY_RATE_54_MBPS, + HAL_RX_LEGACY_RATE_INVALID, +}; + enum hal_ring_type { HAL_REO_DST, HAL_REO_EXCEPTION, @@ -554,11 +312,6 @@ enum hal_ring_type { HAL_MAX_RING_TYPES, }; -#define HAL_RX_MAX_BA_WINDOW 256 - -#define HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC (100 * 1000) -#define HAL_DEFAULT_VO_REO_TIMEOUT_USEC (40 * 1000) - /** * enum hal_reo_cmd_type: Enum for REO command type * @HAL_REO_CMD_GET_QUEUE_STATS: Get REO queue status/stats @@ -597,6 +350,128 @@ enum hal_reo_cmd_status { HAL_REO_CMD_DRAIN = 0xff, }; +enum hal_tcl_encap_type { + HAL_TCL_ENCAP_TYPE_RAW, + HAL_TCL_ENCAP_TYPE_NATIVE_WIFI, + HAL_TCL_ENCAP_TYPE_ETHERNET, + HAL_TCL_ENCAP_TYPE_802_3 = 3, + HAL_TCL_ENCAP_TYPE_MAX +}; + +enum hal_tcl_desc_type { + HAL_TCL_DESC_TYPE_BUFFER, + HAL_TCL_DESC_TYPE_EXT_DESC, + HAL_TCL_DESC_TYPE_MAX, +}; + +enum hal_reo_dest_ring_buffer_type { + HAL_REO_DEST_RING_BUFFER_TYPE_MSDU, + HAL_REO_DEST_RING_BUFFER_TYPE_LINK_DESC, +}; + +enum hal_reo_dest_ring_push_reason { + HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED, + HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION, +}; + +enum hal_reo_entr_rxdma_push_reason { + HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ERR_DETECTED, + HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ROUTING_INSTRUCTION, + HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_RX_FLUSH, +}; + +enum hal_reo_dest_ring_error_code { + HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO, + HAL_REO_DEST_RING_ERROR_CODE_DESC_INVALID, + HAL_REO_DEST_RING_ERROR_CODE_AMPDU_IN_NON_BA, + HAL_REO_DEST_RING_ERROR_CODE_NON_BA_DUPLICATE, + HAL_REO_DEST_RING_ERROR_CODE_BA_DUPLICATE, + HAL_REO_DEST_RING_ERROR_CODE_FRAME_2K_JUMP, + HAL_REO_DEST_RING_ERROR_CODE_BAR_2K_JUMP, + HAL_REO_DEST_RING_ERROR_CODE_FRAME_OOR, + HAL_REO_DEST_RING_ERROR_CODE_BAR_OOR, + HAL_REO_DEST_RING_ERROR_CODE_NO_BA_SESSION, + HAL_REO_DEST_RING_ERROR_CODE_FRAME_SN_EQUALS_SSN, + HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED, + HAL_REO_DEST_RING_ERROR_CODE_2K_ERR_FLAG_SET, + HAL_REO_DEST_RING_ERROR_CODE_PN_ERR_FLAG_SET, + HAL_REO_DEST_RING_ERROR_CODE_DESC_BLOCKED, + HAL_REO_DEST_RING_ERROR_CODE_MAX, +}; + +enum hal_reo_entr_rxdma_ecode { + HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_FCS_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_UNECRYPTED_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LEN_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LIMIT_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_WIFI_PARSE_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_PARSE_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_SA_TIMEOUT_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_DA_TIMEOUT_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_FLOW_TIMEOUT_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_FRAG_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_MULTICAST_ECHO_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_MISMATCH_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_UNAUTH_WDS_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_GRPCAST_AMSDU_WDS_ERR, + HAL_REO_ENTR_RING_RXDMA_ECODE_MAX, +}; + +enum hal_wbm_htt_tx_comp_status { + HAL_WBM_REL_HTT_TX_COMP_STATUS_OK, + HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP, + HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL, + HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ, + HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT, + HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY, + HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH, + HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX, +}; + +enum hal_encrypt_type { + HAL_ENCRYPT_TYPE_WEP_40, + HAL_ENCRYPT_TYPE_WEP_104, + HAL_ENCRYPT_TYPE_TKIP_NO_MIC, + HAL_ENCRYPT_TYPE_WEP_128, + HAL_ENCRYPT_TYPE_TKIP_MIC, + HAL_ENCRYPT_TYPE_WAPI, + HAL_ENCRYPT_TYPE_CCMP_128, + HAL_ENCRYPT_TYPE_OPEN, + HAL_ENCRYPT_TYPE_CCMP_256, + HAL_ENCRYPT_TYPE_GCMP_128, + HAL_ENCRYPT_TYPE_AES_GCMP_256, + HAL_ENCRYPT_TYPE_WAPI_GCM_SM4, +}; + +enum hal_tx_rate_stats_bw { + HAL_TX_RATE_STATS_BW_20, + HAL_TX_RATE_STATS_BW_40, + HAL_TX_RATE_STATS_BW_80, + HAL_TX_RATE_STATS_BW_160, +}; + +enum hal_tx_rate_stats_pkt_type { + HAL_TX_RATE_STATS_PKT_TYPE_11A, + HAL_TX_RATE_STATS_PKT_TYPE_11B, + HAL_TX_RATE_STATS_PKT_TYPE_11N, + HAL_TX_RATE_STATS_PKT_TYPE_11AC, + HAL_TX_RATE_STATS_PKT_TYPE_11AX, + HAL_TX_RATE_STATS_PKT_TYPE_11BA, + HAL_TX_RATE_STATS_PKT_TYPE_11BE, +}; + +enum hal_tx_rate_stats_sgi { + HAL_TX_RATE_STATS_SGI_08US, + HAL_TX_RATE_STATS_SGI_04US, + HAL_TX_RATE_STATS_SGI_16US, + HAL_TX_RATE_STATS_SGI_32US, +}; + struct hal_wbm_idle_scatter_list { dma_addr_t paddr; struct hal_wbm_link_desc *vaddr; @@ -625,6 +500,339 @@ enum hal_srng_dir { HAL_SRNG_DIR_DST }; +enum rx_msdu_start_pkt_type { + RX_MSDU_START_PKT_TYPE_11A, + RX_MSDU_START_PKT_TYPE_11B, + RX_MSDU_START_PKT_TYPE_11N, + RX_MSDU_START_PKT_TYPE_11AC, + RX_MSDU_START_PKT_TYPE_11AX, + RX_MSDU_START_PKT_TYPE_11BA, + RX_MSDU_START_PKT_TYPE_11BE, +}; + +enum rx_msdu_start_sgi { + RX_MSDU_START_SGI_0_8_US, + RX_MSDU_START_SGI_0_4_US, + RX_MSDU_START_SGI_1_6_US, + RX_MSDU_START_SGI_3_2_US, +}; + +enum rx_msdu_start_recv_bw { + RX_MSDU_START_RECV_BW_20MHZ, + RX_MSDU_START_RECV_BW_40MHZ, + RX_MSDU_START_RECV_BW_80MHZ, + RX_MSDU_START_RECV_BW_160MHZ, +}; + +enum rx_msdu_start_reception_type { + RX_MSDU_START_RECEPTION_TYPE_SU, + RX_MSDU_START_RECEPTION_TYPE_DL_MU_MIMO, + RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA, + RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA_MIMO, + RX_MSDU_START_RECEPTION_TYPE_UL_MU_MIMO, + RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA, + RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA_MIMO, +}; + +enum rx_desc_decap_type { + RX_DESC_DECAP_TYPE_RAW, + RX_DESC_DECAP_TYPE_NATIVE_WIFI, + RX_DESC_DECAP_TYPE_ETHERNET2_DIX, + RX_DESC_DECAP_TYPE_8023, +}; + +struct hal_rx_user_status { + u32 mcs:4, + nss:3, + ofdma_info_valid:1, + ul_ofdma_ru_start_index:7, + ul_ofdma_ru_width:7, + ul_ofdma_ru_size:8; + u32 ul_ofdma_user_v0_word0; + u32 ul_ofdma_user_v0_word1; + u32 ast_index; + u32 tid; + u16 tcp_msdu_count; + u16 tcp_ack_msdu_count; + u16 udp_msdu_count; + u16 other_msdu_count; + u16 frame_control; + u8 frame_control_info_valid; + u8 data_sequence_control_info_valid; + u16 first_data_seq_ctrl; + u32 preamble_type; + u16 ht_flags; + u16 vht_flags; + u16 he_flags; + u8 rs_flags; + u8 ldpc; + u32 mpdu_cnt_fcs_ok; + u32 mpdu_cnt_fcs_err; + u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP]; + u32 mpdu_ok_byte_count; + u32 mpdu_err_byte_count; + bool ampdu_present; + u16 ampdu_id; +}; + +struct hal_rx_u_sig_info { + bool ul_dl; + u8 bw; + u8 ppdu_type_comp_mode; + u8 eht_sig_mcs; + u8 num_eht_sig_sym; + struct ieee80211_radiotap_eht_usig usig; +}; + +struct hal_rx_tlv_aggr_info { + bool in_progress; + u16 cur_len; + u16 tlv_tag; + u8 buf[HAL_RX_MON_MAX_AGGR_SIZE]; +}; + +struct hal_rx_radiotap_eht { + __le32 known; + __le32 data[9]; +}; + +struct hal_rx_eht_info { + u8 num_user_info; + struct hal_rx_radiotap_eht eht; + u32 user_info[EHT_MAX_USER_INFO]; +}; + +struct hal_rx_msdu_desc_info { + u32 msdu_flags; + u16 msdu_len; /* 14 bits for length */ +}; + +/* hal_mon_buf_ring + * Producer : SW + * Consumer : Monitor + * + * paddr_lo + * Lower 32-bit physical address of the buffer pointer from the source ring. + * paddr_hi + * bit range 7-0 : upper 8 bit of the physical address. + * bit range 31-8 : reserved. + * cookie + * Consumer: RxMon/TxMon 64 bit cookie of the buffers. + */ +struct hal_mon_buf_ring { + __le32 paddr_lo; + __le32 paddr_hi; + __le64 cookie; +}; + +struct hal_rx_mon_ppdu_info { + u32 ppdu_id; + u32 last_ppdu_id; + u64 ppdu_ts; + u32 num_mpdu_fcs_ok; + u32 num_mpdu_fcs_err; + u32 preamble_type; + u32 mpdu_len; + u16 chan_num; + u16 freq; + u16 tcp_msdu_count; + u16 tcp_ack_msdu_count; + u16 udp_msdu_count; + u16 other_msdu_count; + u16 peer_id; + u8 rate; + u8 mcs; + u8 nss; + u8 bw; + u8 vht_flag_values1; + u8 vht_flag_values2; + u8 vht_flag_values3[4]; + u8 vht_flag_values4; + u8 vht_flag_values5; + u16 vht_flag_values6; + u8 is_stbc; + u8 gi; + u8 sgi; + u8 ldpc; + u8 beamformed; + u8 rssi_comb; + u16 tid; + u8 fc_valid; + u16 ht_flags; + u16 vht_flags; + u16 he_flags; + u16 he_mu_flags; + u8 dcm; + u8 ru_alloc; + u8 reception_type; + u64 tsft; + u64 rx_duration; + u16 frame_control; + u32 ast_index; + u8 rs_fcs_err; + u8 rs_flags; + u8 cck_flag; + u8 ofdm_flag; + u8 ulofdma_flag; + u8 frame_control_info_valid; + u16 he_per_user_1; + u16 he_per_user_2; + u8 he_per_user_position; + u8 he_per_user_known; + u16 he_flags1; + u16 he_flags2; + u8 he_RU[4]; + u16 he_data1; + u16 he_data2; + u16 he_data3; + u16 he_data4; + u16 he_data5; + u16 he_data6; + u32 ppdu_len; + u32 prev_ppdu_id; + u32 device_id; + u16 first_data_seq_ctrl; + u8 monitor_direct_used; + u8 data_sequence_control_info_valid; + u8 ltf_size; + u8 rxpcu_filter_pass; + s8 rssi_chain[8][8]; + u32 num_users; + u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP]; + u8 addr1[ETH_ALEN]; + u8 addr2[ETH_ALEN]; + u8 addr3[ETH_ALEN]; + u8 addr4[ETH_ALEN]; + struct hal_rx_user_status userstats[HAL_MAX_UL_MU_USERS]; + u8 userid; + bool first_msdu_in_mpdu; + bool is_ampdu; + u8 medium_prot_type; + bool ppdu_continuation; + bool eht_usig; + struct hal_rx_u_sig_info u_sig_info; + bool is_eht; + struct hal_rx_eht_info eht_info; + struct hal_rx_tlv_aggr_info tlv_aggr; +}; + +struct hal_rx_desc_data { + struct ieee80211_rx_status *rx_status; + u32 phy_meta_data; + u32 err_bitmap; + u32 enctype; + u32 msdu_done:1, + is_decrypted:1, + ip_csum_fail:1, + l4_csum_fail:1, + is_first_msdu:1, + is_last_msdu:1, + mesh_ctrl_present:1, + addr2_present:1, + is_mcbc:1, + seq_ctl_valid:1, + fc_valid:1; + u16 msdu_len; + u16 peer_id; + u16 seq_no; + u8 *addr2; + u8 pkt_type; + u8 l3_pad_bytes; + u8 decap_type; + u8 bw; + u8 rate_mcs; + u8 nss; + u8 sgi; + u8 tid; +}; + +#define BUFFER_ADDR_INFO0_ADDR GENMASK(31, 0) + +#define BUFFER_ADDR_INFO1_ADDR GENMASK(7, 0) +#define BUFFER_ADDR_INFO1_RET_BUF_MGR GENMASK(11, 8) +#define BUFFER_ADDR_INFO1_SW_COOKIE GENMASK(31, 12) + +struct ath12k_buffer_addr { + __le32 info0; + __le32 info1; +} __packed; + +/* ath12k_buffer_addr + * + * buffer_addr_31_0 + * Address (lower 32 bits) of the MSDU buffer or MSDU_EXTENSION + * descriptor or Link descriptor + * + * buffer_addr_39_32 + * Address (upper 8 bits) of the MSDU buffer or MSDU_EXTENSION + * descriptor or Link descriptor + * + * return_buffer_manager (RBM) + * Consumer: WBM + * Producer: SW/FW + * Indicates to which buffer manager the buffer or MSDU_EXTENSION + * descriptor or link descriptor that is being pointed to shall be + * returned after the frame has been processed. It is used by WBM + * for routing purposes. + * + * Values are defined in enum %HAL_RX_BUF_RBM_ + * + * sw_buffer_cookie + * Cookie field exclusively used by SW. HW ignores the contents, + * accept that it passes the programmed value on to other + * descriptors together with the physical address. + * + * Field can be used by SW to for example associate the buffers + * physical address with the virtual address. + * + * NOTE1: + * The three most significant bits can have a special meaning + * in case this struct is embedded in a TX_MPDU_DETAILS STRUCT, + * and field transmit_bw_restriction is set + * + * In case of NON punctured transmission: + * Sw_buffer_cookie[19:17] = 3'b000: 20 MHz TX only + * Sw_buffer_cookie[19:17] = 3'b001: 40 MHz TX only + * Sw_buffer_cookie[19:17] = 3'b010: 80 MHz TX only + * Sw_buffer_cookie[19:17] = 3'b011: 160 MHz TX only + * Sw_buffer_cookie[19:17] = 3'b101: 240 MHz TX only + * Sw_buffer_cookie[19:17] = 3'b100: 320 MHz TX only + * Sw_buffer_cookie[19:18] = 2'b11: reserved + * + * In case of punctured transmission: + * Sw_buffer_cookie[19:16] = 4'b0000: pattern 0 only + * Sw_buffer_cookie[19:16] = 4'b0001: pattern 1 only + * Sw_buffer_cookie[19:16] = 4'b0010: pattern 2 only + * Sw_buffer_cookie[19:16] = 4'b0011: pattern 3 only + * Sw_buffer_cookie[19:16] = 4'b0100: pattern 4 only + * Sw_buffer_cookie[19:16] = 4'b0101: pattern 5 only + * Sw_buffer_cookie[19:16] = 4'b0110: pattern 6 only + * Sw_buffer_cookie[19:16] = 4'b0111: pattern 7 only + * Sw_buffer_cookie[19:16] = 4'b1000: pattern 8 only + * Sw_buffer_cookie[19:16] = 4'b1001: pattern 9 only + * Sw_buffer_cookie[19:16] = 4'b1010: pattern 10 only + * Sw_buffer_cookie[19:16] = 4'b1011: pattern 11 only + * Sw_buffer_cookie[19:18] = 2'b11: reserved + * + * Note: a punctured transmission is indicated by the presence + * of TLV TX_PUNCTURE_SETUP embedded in the scheduler TLV + * + * Sw_buffer_cookie[20:17]: Tid: The TID field in the QoS control + * field + * + * Sw_buffer_cookie[16]: Mpdu_qos_control_valid: This field + * indicates MPDUs with a QoS control field. + * + */ + +struct hal_ce_srng_dest_desc; +struct hal_ce_srng_dst_status_desc; +struct hal_ce_srng_src_desc; + +struct hal_wbm_link_desc { + struct ath12k_buffer_addr buf_addr_info; +} __packed; + /* srng flags */ #define HAL_SRNG_FLAGS_MSI_SWAP 0x00000008 #define HAL_SRNG_FLAGS_RING_PTR_SWAP 0x00000010 @@ -634,9 +842,6 @@ enum hal_srng_dir { #define HAL_SRNG_FLAGS_HIGH_THRESH_INTR_EN 0x00080000 #define HAL_SRNG_FLAGS_LMAC_RING 0x80000000 -#define HAL_SRNG_TLV_HDR_TAG GENMASK(9, 1) -#define HAL_SRNG_TLV_HDR_LEN GENMASK(25, 10) - /* Common SRNG ring structure for source and destination rings */ struct hal_srng { /* Unique SRNG ring ID */ @@ -758,6 +963,51 @@ struct hal_srng { } u; }; +/* hal_wbm_link_desc + * + * Producer: WBM + * Consumer: WBM + * + * buf_addr_info + * Details of the physical address of a buffer or MSDU + * link descriptor. + */ + +enum hal_wbm_rel_src_module { + HAL_WBM_REL_SRC_MODULE_TQM, + HAL_WBM_REL_SRC_MODULE_RXDMA, + HAL_WBM_REL_SRC_MODULE_REO, + HAL_WBM_REL_SRC_MODULE_FW, + HAL_WBM_REL_SRC_MODULE_SW, + HAL_WBM_REL_SRC_MODULE_MAX, +}; + +/* hal_wbm_rel_desc_type + * + * msdu_buffer + * The address points to an MSDU buffer + * + * msdu_link_descriptor + * The address points to an Tx MSDU link descriptor + * + * mpdu_link_descriptor + * The address points to an MPDU link descriptor + * + * msdu_ext_descriptor + * The address points to an MSDU extension descriptor + * + * queue_ext_descriptor + * The address points to an TQM queue extension descriptor. WBM should + * treat this is the same way as a link descriptor. + */ +enum hal_wbm_rel_desc_type { + HAL_WBM_REL_DESC_TYPE_REL_MSDU, + HAL_WBM_REL_DESC_TYPE_MSDU_LINK, + HAL_WBM_REL_DESC_TYPE_MPDU_LINK, + HAL_WBM_REL_DESC_TYPE_MSDU_EXT, + HAL_WBM_REL_DESC_TYPE_QUEUE_EXT, +}; + /* Interrupt mitigation - Batch threshold in terms of number of frames */ #define HAL_SRNG_INT_BATCH_THRESHOLD_TX 256 #define HAL_SRNG_INT_BATCH_THRESHOLD_RX 128 @@ -821,66 +1071,6 @@ enum hal_rx_buf_return_buf_manager { HAL_RX_BUF_RBM_SW6_BM, }; -#define HAL_SRNG_DESC_LOOP_CNT 0xf0000000 - -#define HAL_REO_CMD_FLG_NEED_STATUS BIT(0) -#define HAL_REO_CMD_FLG_STATS_CLEAR BIT(1) -#define HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER BIT(2) -#define HAL_REO_CMD_FLG_FLUSH_RELEASE_BLOCKING BIT(3) -#define HAL_REO_CMD_FLG_FLUSH_NO_INVAL BIT(4) -#define HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS BIT(5) -#define HAL_REO_CMD_FLG_FLUSH_ALL BIT(6) -#define HAL_REO_CMD_FLG_UNBLK_RESOURCE BIT(7) -#define HAL_REO_CMD_FLG_UNBLK_CACHE BIT(8) -#define HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC BIT(9) - -/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO0_UPD_* fields */ -#define HAL_REO_CMD_UPD0_RX_QUEUE_NUM BIT(8) -#define HAL_REO_CMD_UPD0_VLD BIT(9) -#define HAL_REO_CMD_UPD0_ALDC BIT(10) -#define HAL_REO_CMD_UPD0_DIS_DUP_DETECTION BIT(11) -#define HAL_REO_CMD_UPD0_SOFT_REORDER_EN BIT(12) -#define HAL_REO_CMD_UPD0_AC BIT(13) -#define HAL_REO_CMD_UPD0_BAR BIT(14) -#define HAL_REO_CMD_UPD0_RETRY BIT(15) -#define HAL_REO_CMD_UPD0_CHECK_2K_MODE BIT(16) -#define HAL_REO_CMD_UPD0_OOR_MODE BIT(17) -#define HAL_REO_CMD_UPD0_BA_WINDOW_SIZE BIT(18) -#define HAL_REO_CMD_UPD0_PN_CHECK BIT(19) -#define HAL_REO_CMD_UPD0_EVEN_PN BIT(20) -#define HAL_REO_CMD_UPD0_UNEVEN_PN BIT(21) -#define HAL_REO_CMD_UPD0_PN_HANDLE_ENABLE BIT(22) -#define HAL_REO_CMD_UPD0_PN_SIZE BIT(23) -#define HAL_REO_CMD_UPD0_IGNORE_AMPDU_FLG BIT(24) -#define HAL_REO_CMD_UPD0_SVLD BIT(25) -#define HAL_REO_CMD_UPD0_SSN BIT(26) -#define HAL_REO_CMD_UPD0_SEQ_2K_ERR BIT(27) -#define HAL_REO_CMD_UPD0_PN_ERR BIT(28) -#define HAL_REO_CMD_UPD0_PN_VALID BIT(29) -#define HAL_REO_CMD_UPD0_PN BIT(30) - -/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO1_* fields */ -#define HAL_REO_CMD_UPD1_VLD BIT(16) -#define HAL_REO_CMD_UPD1_ALDC GENMASK(18, 17) -#define HAL_REO_CMD_UPD1_DIS_DUP_DETECTION BIT(19) -#define HAL_REO_CMD_UPD1_SOFT_REORDER_EN BIT(20) -#define HAL_REO_CMD_UPD1_AC GENMASK(22, 21) -#define HAL_REO_CMD_UPD1_BAR BIT(23) -#define HAL_REO_CMD_UPD1_RETRY BIT(24) -#define HAL_REO_CMD_UPD1_CHECK_2K_MODE BIT(25) -#define HAL_REO_CMD_UPD1_OOR_MODE BIT(26) -#define HAL_REO_CMD_UPD1_PN_CHECK BIT(27) -#define HAL_REO_CMD_UPD1_EVEN_PN BIT(28) -#define HAL_REO_CMD_UPD1_UNEVEN_PN BIT(29) -#define HAL_REO_CMD_UPD1_PN_HANDLE_ENABLE BIT(30) -#define HAL_REO_CMD_UPD1_IGNORE_AMPDU_FLG BIT(31) - -/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO2_* fields */ -#define HAL_REO_CMD_UPD2_SVLD BIT(10) -#define HAL_REO_CMD_UPD2_SSN GENMASK(22, 11) -#define HAL_REO_CMD_UPD2_SEQ_2K_ERR BIT(23) -#define HAL_REO_CMD_UPD2_PN_ERR BIT(24) - struct ath12k_hal_reo_cmd { u32 addr_lo; u32 flag; @@ -926,91 +1116,93 @@ struct hal_reo_status_header { u32 timestamp; }; -struct hal_reo_status_queue_stats { - u16 ssn; - u16 curr_idx; - u32 pn[4]; - u32 last_rx_queue_ts; - u32 last_rx_dequeue_ts; - u32 rx_bitmap[8]; /* Bitmap from 0-255 */ - u32 curr_mpdu_cnt; - u32 curr_msdu_cnt; - u16 fwd_due_to_bar_cnt; - u16 dup_cnt; - u32 frames_in_order_cnt; - u32 num_mpdu_processed_cnt; - u32 num_msdu_processed_cnt; - u32 total_num_processed_byte_cnt; - u32 late_rx_mpdu_cnt; - u32 reorder_hole_cnt; - u8 timeout_cnt; - u8 bar_rx_cnt; - u8 num_window_2k_jump_cnt; +struct ath12k_hw_hal_params { + enum hal_rx_buf_return_buf_manager rx_buf_rbm; + u32 wbm2sw_cc_enable; }; -struct hal_reo_status_flush_queue { - bool err_detected; -}; - -enum hal_reo_status_flush_cache_err_code { - HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_SUCCESS, - HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_IN_USE, - HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_NOT_FOUND, -}; - -struct hal_reo_status_flush_cache { - bool err_detected; - enum hal_reo_status_flush_cache_err_code err_code; - bool cache_controller_flush_status_hit; - u8 cache_controller_flush_status_desc_type; - u8 cache_controller_flush_status_client_id; - u8 cache_controller_flush_status_err; - u8 cache_controller_flush_status_cnt; -}; - -enum hal_reo_status_unblock_cache_type { - HAL_REO_STATUS_UNBLOCK_BLOCKING_RESOURCE, - HAL_REO_STATUS_UNBLOCK_ENTIRE_CACHE_USAGE, -}; - -struct hal_reo_status_unblock_cache { - bool err_detected; - enum hal_reo_status_unblock_cache_type unblock_type; -}; - -struct hal_reo_status_flush_timeout_list { - bool err_detected; - bool list_empty; - u16 release_desc_cnt; - u16 fwd_buf_cnt; -}; - -enum hal_reo_threshold_idx { - HAL_REO_THRESHOLD_IDX_DESC_COUNTER0, - HAL_REO_THRESHOLD_IDX_DESC_COUNTER1, - HAL_REO_THRESHOLD_IDX_DESC_COUNTER2, - HAL_REO_THRESHOLD_IDX_DESC_COUNTER_SUM, -}; - -struct hal_reo_status_desc_thresh_reached { - enum hal_reo_threshold_idx threshold_idx; - u32 link_desc_counter0; - u32 link_desc_counter1; - u32 link_desc_counter2; - u32 link_desc_counter_sum; -}; - -struct hal_reo_status { - struct hal_reo_status_header uniform_hdr; - u8 loop_cnt; - union { - struct hal_reo_status_queue_stats queue_stats; - struct hal_reo_status_flush_queue flush_queue; - struct hal_reo_status_flush_cache flush_cache; - struct hal_reo_status_unblock_cache unblock_cache; - struct hal_reo_status_flush_timeout_list timeout_list; - struct hal_reo_status_desc_thresh_reached desc_thresh_reached; - } u; +#define ATH12K_HW_REG_UNDEFINED 0xdeadbeaf + +struct ath12k_hw_regs { + u32 tcl1_ring_id; + u32 tcl1_ring_misc; + u32 tcl1_ring_tp_addr_lsb; + u32 tcl1_ring_tp_addr_msb; + u32 tcl1_ring_consumer_int_setup_ix0; + u32 tcl1_ring_consumer_int_setup_ix1; + u32 tcl1_ring_msi1_base_lsb; + u32 tcl1_ring_msi1_base_msb; + u32 tcl1_ring_msi1_data; + u32 tcl_ring_base_lsb; + u32 tcl1_ring_base_lsb; + u32 tcl1_ring_base_msb; + u32 tcl2_ring_base_lsb; + + u32 tcl_status_ring_base_lsb; + + u32 reo1_qdesc_addr; + u32 reo1_qdesc_max_peerid; + + u32 wbm_idle_ring_base_lsb; + u32 wbm_idle_ring_misc_addr; + u32 wbm_r0_idle_list_cntl_addr; + u32 wbm_r0_idle_list_size_addr; + u32 wbm_scattered_ring_base_lsb; + u32 wbm_scattered_ring_base_msb; + u32 wbm_scattered_desc_head_info_ix0; + u32 wbm_scattered_desc_head_info_ix1; + u32 wbm_scattered_desc_tail_info_ix0; + u32 wbm_scattered_desc_tail_info_ix1; + u32 wbm_scattered_desc_ptr_hp_addr; + + u32 wbm_sw_release_ring_base_lsb; + u32 wbm_sw1_release_ring_base_lsb; + u32 wbm0_release_ring_base_lsb; + u32 wbm1_release_ring_base_lsb; + + u32 pcie_qserdes_sysclk_en_sel; + u32 pcie_pcs_osc_dtct_config_base; + + u32 umac_ce0_src_reg_base; + u32 umac_ce0_dest_reg_base; + u32 umac_ce1_src_reg_base; + u32 umac_ce1_dest_reg_base; + + u32 ppe_rel_ring_base; + + u32 reo2_ring_base; + u32 reo1_misc_ctrl_addr; + u32 reo1_sw_cookie_cfg0; + u32 reo1_sw_cookie_cfg1; + u32 reo1_qdesc_lut_base0; + u32 reo1_qdesc_lut_base1; + u32 reo1_ring_base_lsb; + u32 reo1_ring_base_msb; + u32 reo1_ring_id; + u32 reo1_ring_misc; + u32 reo1_ring_hp_addr_lsb; + u32 reo1_ring_hp_addr_msb; + u32 reo1_ring_producer_int_setup; + u32 reo1_ring_msi1_base_lsb; + u32 reo1_ring_msi1_base_msb; + u32 reo1_ring_msi1_data; + u32 reo1_aging_thres_ix0; + u32 reo1_aging_thres_ix1; + u32 reo1_aging_thres_ix2; + u32 reo1_aging_thres_ix3; + + u32 reo2_sw0_ring_base; + + u32 sw2reo_ring_base; + u32 sw2reo1_ring_base; + + u32 reo_cmd_ring_base; + + u32 reo_status_ring_base; + + u32 gcc_gcc_pcie_hot_rst; + + u32 qrtr_node_id; }; /* HAL context to be used to access SRNG APIs (currently used by data path @@ -1036,16 +1228,23 @@ struct ath12k_hal { dma_addr_t paddr; } wrp; + struct device *dev; + const struct hal_ops *ops; + const struct ath12k_hw_regs *regs; + const struct ath12k_hw_hal_params *hal_params; /* Available REO blocking resources bitmap */ u8 avail_blk_resource; u8 current_blk_index; /* shadow register configuration */ - u32 shadow_reg_addr[HAL_SHADOW_NUM_REGS]; + u32 shadow_reg_addr[HAL_SHADOW_NUM_REGS_MAX]; int num_shadow_reg_configured; u32 hal_desc_sz; + u32 hal_wbm_release_ring_tx_size; + + const struct ath12k_hal_tcl_to_wbm_rbm_map *tcl_to_wbm_rbm_map; }; /* Maps WBM ring number and Return Buffer Manager Id per TCL ring */ @@ -1054,92 +1253,222 @@ struct ath12k_hal_tcl_to_wbm_rbm_map { u8 rbm_id; }; -struct hal_rx_ops { - bool (*rx_desc_get_first_msdu)(struct hal_rx_desc *desc); - bool (*rx_desc_get_last_msdu)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_l3_pad_bytes)(struct hal_rx_desc *desc); - u8 *(*rx_desc_get_hdr_status)(struct hal_rx_desc *desc); - bool (*rx_desc_encrypt_valid)(struct hal_rx_desc *desc); - u32 (*rx_desc_get_encrypt_type)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_decap_type)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_mesh_ctl)(struct hal_rx_desc *desc); - bool (*rx_desc_get_mpdu_seq_ctl_vld)(struct hal_rx_desc *desc); - bool (*rx_desc_get_mpdu_fc_valid)(struct hal_rx_desc *desc); - u16 (*rx_desc_get_mpdu_start_seq_no)(struct hal_rx_desc *desc); - u16 (*rx_desc_get_msdu_len)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_msdu_sgi)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_msdu_rate_mcs)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_msdu_rx_bw)(struct hal_rx_desc *desc); - u32 (*rx_desc_get_msdu_freq)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_msdu_pkt_type)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_msdu_nss)(struct hal_rx_desc *desc); - u8 (*rx_desc_get_mpdu_tid)(struct hal_rx_desc *desc); - u16 (*rx_desc_get_mpdu_peer_id)(struct hal_rx_desc *desc); - void (*rx_desc_copy_end_tlv)(struct hal_rx_desc *fdesc, - struct hal_rx_desc *ldesc); - u32 (*rx_desc_get_mpdu_start_tag)(struct hal_rx_desc *desc); - u32 (*rx_desc_get_mpdu_ppdu_id)(struct hal_rx_desc *desc); +enum hal_wbm_rel_bm_act { + HAL_WBM_REL_BM_ACT_PUT_IN_IDLE, + HAL_WBM_REL_BM_ACT_REL_MSDU, +}; + +/* hal_wbm_rel_bm_act + * + * put_in_idle_list + * Put the buffer or descriptor back in the idle list. In case of MSDU or + * MDPU link descriptor, BM does not need to check to release any + * individual MSDU buffers. + * + * release_msdu_list + * This BM action can only be used in combination with desc_type being + * msdu_link_descriptor. Field first_msdu_index points out which MSDU + * pointer in the MSDU link descriptor is the first of an MPDU that is + * released. BM shall release all the MSDU buffers linked to this first + * MSDU buffer pointer. All related MSDU buffer pointer entries shall be + * set to value 0, which represents the 'NULL' pointer. When all MSDU + * buffer pointers in the MSDU link descriptor are 'NULL', the MSDU link + * descriptor itself shall also be released. + */ + +#define RU_INVALID 0 +#define RU_26 1 +#define RU_52 2 +#define RU_106 4 +#define RU_242 9 +#define RU_484 18 +#define RU_996 37 +#define RU_2X996 74 +#define RU_3X996 111 +#define RU_4X996 148 +#define RU_52_26 (RU_52 + RU_26) +#define RU_106_26 (RU_106 + RU_26) +#define RU_484_242 (RU_484 + RU_242) +#define RU_996_484 (RU_996 + RU_484) +#define RU_996_484_242 (RU_996 + RU_484_242) +#define RU_2X996_484 (RU_2X996 + RU_484) +#define RU_3X996_484 (RU_3X996 + RU_484) + +enum ath12k_eht_ru_size { + ATH12K_EHT_RU_26, + ATH12K_EHT_RU_52, + ATH12K_EHT_RU_106, + ATH12K_EHT_RU_242, + ATH12K_EHT_RU_484, + ATH12K_EHT_RU_996, + ATH12K_EHT_RU_996x2, + ATH12K_EHT_RU_996x4, + ATH12K_EHT_RU_52_26, + ATH12K_EHT_RU_106_26, + ATH12K_EHT_RU_484_242, + ATH12K_EHT_RU_996_484, + ATH12K_EHT_RU_996_484_242, + ATH12K_EHT_RU_996x2_484, + ATH12K_EHT_RU_996x3, + ATH12K_EHT_RU_996x3_484, + + /* Keep last */ + ATH12K_EHT_RU_INVALID, +}; + +#define HAL_RX_RU_ALLOC_TYPE_MAX ATH12K_EHT_RU_INVALID + +static inline +enum nl80211_he_ru_alloc ath12k_he_ru_tones_to_nl80211_he_ru_alloc(u16 ru_tones) +{ + enum nl80211_he_ru_alloc ret; + + switch (ru_tones) { + case RU_52: + ret = NL80211_RATE_INFO_HE_RU_ALLOC_52; + break; + case RU_106: + ret = NL80211_RATE_INFO_HE_RU_ALLOC_106; + break; + case RU_242: + ret = NL80211_RATE_INFO_HE_RU_ALLOC_242; + break; + case RU_484: + ret = NL80211_RATE_INFO_HE_RU_ALLOC_484; + break; + case RU_996: + ret = NL80211_RATE_INFO_HE_RU_ALLOC_996; + break; + case RU_2X996: + ret = NL80211_RATE_INFO_HE_RU_ALLOC_2x996; + break; + case RU_26: + fallthrough; + default: + ret = NL80211_RATE_INFO_HE_RU_ALLOC_26; + break; + } + return ret; +} + +struct ath12k_hw_version_map { + const struct hal_ops *hal_ops; + u32 hal_desc_sz; + const struct ath12k_hal_tcl_to_wbm_rbm_map *tcl_to_wbm_rbm_map; + const struct ath12k_hw_hal_params *hal_params; + const struct ath12k_hw_regs *hw_regs; +}; + +struct hal_ops { + int (*create_srng_config)(struct ath12k_hal *hal); void (*rx_desc_set_msdu_len)(struct hal_rx_desc *desc, u16 len); - struct rx_attention *(*rx_desc_get_attention)(struct hal_rx_desc *desc); - u8 *(*rx_desc_get_msdu_payload)(struct hal_rx_desc *desc); - u32 (*rx_desc_get_mpdu_start_offset)(void); - u32 (*rx_desc_get_msdu_end_offset)(void); - bool (*rx_desc_mac_addr2_valid)(struct hal_rx_desc *desc); - u8* (*rx_desc_mpdu_start_addr2)(struct hal_rx_desc *desc); - bool (*rx_desc_is_da_mcbc)(struct hal_rx_desc *desc); void (*rx_desc_get_dot11_hdr)(struct hal_rx_desc *desc, struct ieee80211_hdr *hdr); void (*rx_desc_get_crypto_header)(struct hal_rx_desc *desc, u8 *crypto_hdr, enum hal_encrypt_type enctype); - bool (*dp_rx_h_msdu_done)(struct hal_rx_desc *desc); - bool (*dp_rx_h_l4_cksum_fail)(struct hal_rx_desc *desc); - bool (*dp_rx_h_ip_cksum_fail)(struct hal_rx_desc *desc); - bool (*dp_rx_h_is_decrypted)(struct hal_rx_desc *desc); - u32 (*dp_rx_h_mpdu_err)(struct hal_rx_desc *desc); - u32 (*rx_desc_get_desc_size)(void); + void (*rx_desc_copy_end_tlv)(struct hal_rx_desc *fdesc, + struct hal_rx_desc *ldesc); u8 (*rx_desc_get_msdu_src_link_id)(struct hal_rx_desc *desc); + void (*extract_rx_desc_data)(struct hal_rx_desc_data *rx_desc_data, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc *ldesc); + u32 (*rx_desc_get_mpdu_start_tag)(struct hal_rx_desc *desc); + u32 (*rx_desc_get_mpdu_ppdu_id)(struct hal_rx_desc *desc); + u8 (*rx_desc_get_l3_pad_bytes)(struct hal_rx_desc *desc); + u8 *(*rx_desc_get_msdu_payload)(struct hal_rx_desc *desc); + void (*ce_dst_setup)(struct ath12k_base *ab, + struct hal_srng *srng, int ring_num); + void (*set_umac_srng_ptr_addr)(struct ath12k_base *ab, + struct hal_srng *srng); + void (*srng_src_hw_init)(struct ath12k_base *ab, struct hal_srng *srng); + void (*srng_dst_hw_init)(struct ath12k_base *ab, struct hal_srng *srng); + int (*srng_update_shadow_config)(struct ath12k_base *ab, + enum hal_ring_type ring_type, + int ring_num); + int (*srng_get_ring_id)(struct ath12k_hal *hal, enum hal_ring_type type, + int ring_num, int mac_id); + u32 (*ce_get_desc_size)(enum hal_ce_desc type); + void (*ce_src_set_desc)(struct hal_ce_srng_src_desc *desc, + dma_addr_t paddr, u32 len, u32 id, + u8 byte_swap_data); + void (*ce_dst_set_desc)(struct hal_ce_srng_dest_desc *desc, + dma_addr_t paddr); + u32 (*ce_dst_status_get_length)(struct hal_ce_srng_dst_status_desc *desc); + void (*set_link_desc_addr)(struct hal_wbm_link_desc *desc, u32 cookie, + dma_addr_t paddr, + enum hal_rx_buf_return_buf_manager rbm); + void (*tx_set_dscp_tid_map)(struct ath12k_base *ab, int id); + void (*tx_configure_bank_register)(struct ath12k_base *ab, + u32 bank_config, u8 bank_id); + void (*reoq_lut_addr_read_enable)(struct ath12k_base *ab); + void (*reoq_lut_set_max_peerid)(struct ath12k_base *ab); + void (*write_ml_reoq_lut_addr)(struct ath12k_base *ab, + dma_addr_t paddr); + void (*write_reoq_lut_addr)(struct ath12k_base *ab, dma_addr_t paddr); + void (*setup_link_idle_list)(struct ath12k_base *ab, + struct hal_wbm_idle_scatter_list *sbuf, + u32 nsbufs, u32 tot_link_desc, + u32 end_offset); + void (*reo_init_cmd_ring)(struct ath12k_base *ab, + struct hal_srng *srng); + void (*reo_shared_qaddr_cache_clear)(struct ath12k_base *ab); + void (*reo_hw_setup)(struct ath12k_base *ab, u32 ring_hash_map); + void (*rx_buf_addr_info_set)(struct ath12k_buffer_addr *binfo, + dma_addr_t paddr, u32 cookie, u8 manager); + void (*rx_buf_addr_info_get)(struct ath12k_buffer_addr *binfo, + dma_addr_t *paddr, u32 *msdu_cookies, + u8 *rbm); + void (*cc_config)(struct ath12k_base *ab); + enum hal_rx_buf_return_buf_manager + (*get_idle_link_rbm)(struct ath12k_hal *hal, u8 device_id); + void (*rx_msdu_list_get)(struct ath12k *ar, + void *link_desc, + void *msdu_list, + u16 *num_msdus); + void (*rx_reo_ent_buf_paddr_get)(void *rx_desc, dma_addr_t *paddr, + u32 *sw_cookie, + struct ath12k_buffer_addr **pp_buf_addr, + u8 *rbm, u32 *msdu_cnt); + void *(*reo_cmd_enc_tlv_hdr)(void *tlv, u64 tag, u64 len); + u16 (*reo_status_dec_tlv_hdr)(void *tlv, void **desc); + void *(*mon_rx_status_dec_tlv_hdr)(void *tlv, u16 *tag, u16 *len, u16 *usrid); + u32 (*get_tlv_hdr_align)(void); }; -struct hal_ops { - int (*create_srng_config)(struct ath12k_base *ab); - u16 (*rxdma_ring_wmask_rx_mpdu_start)(void); - u32 (*rxdma_ring_wmask_rx_msdu_end)(void); - const struct hal_rx_ops *(*get_hal_rx_compact_ops)(void); - const struct ath12k_hal_tcl_to_wbm_rbm_map *tcl_to_wbm_rbm_map; -}; +#define HAL_TLV_HDR_TAG GENMASK(9, 1) +#define HAL_TLV_HDR_LEN GENMASK(21, 10) +#define HAL_TLV_USR_ID GENMASK(31, 26) -extern const struct hal_ops hal_qcn9274_ops; -extern const struct hal_ops hal_wcn7850_ops; +#define HAL_TLV_ALIGN 4 -extern const struct hal_rx_ops hal_rx_qcn9274_ops; -extern const struct hal_rx_ops hal_rx_qcn9274_compact_ops; -extern const struct hal_rx_ops hal_rx_wcn7850_ops; +struct hal_tlv_hdr { + __le32 tl; + u8 value[]; +} __packed; -u32 ath12k_hal_reo_qdesc_size(u32 ba_window_size, u8 tid); -void ath12k_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc, - int tid, u32 ba_window_size, - u32 start_seq, enum hal_pn_type type); -void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, - struct hal_srng *srng); -void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map); -void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab, - struct hal_wbm_idle_scatter_list *sbuf, - u32 nsbufs, u32 tot_link_desc, - u32 end_offset); +#define HAL_TLV_64_HDR_TAG GENMASK(9, 1) +#define HAL_TLV_64_HDR_LEN GENMASK(21, 10) +#define HAL_TLV_64_USR_ID GENMASK(31, 26) +#define HAL_TLV_64_ALIGN 8 + +struct hal_tlv_64_hdr { + __le64 tl; + u8 value[]; +} __packed; dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab, struct hal_srng *srng); dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab, struct hal_srng *srng); -void ath12k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie, - dma_addr_t paddr, - enum hal_rx_buf_return_buf_manager rbm); -u32 ath12k_hal_ce_get_desc_size(enum hal_ce_desc type); -void ath12k_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc, dma_addr_t paddr, - u32 len, u32 id, u8 byte_swap_data); -void ath12k_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc, dma_addr_t paddr); -u32 ath12k_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc); +u32 ath12k_hal_ce_get_desc_size(struct ath12k_hal *hal, enum hal_ce_desc type); +void ath12k_hal_ce_dst_set_desc(struct ath12k_hal *hal, + struct hal_ce_srng_dest_desc *desc, + dma_addr_t paddr); +void ath12k_hal_ce_src_set_desc(struct ath12k_hal *hal, + struct hal_ce_srng_src_desc *desc, + dma_addr_t paddr, u32 len, u32 id, + u8 byte_swap_data); int ath12k_hal_srng_get_entrysize(struct ath12k_base *ab, u32 ring_type); int ath12k_hal_srng_get_max_entries(struct ath12k_base *ab, u32 ring_type); void ath12k_hal_srng_get_params(struct ath12k_base *ab, struct hal_srng *srng, @@ -1178,4 +1507,47 @@ void ath12k_hal_srng_shadow_config(struct ath12k_base *ab); void ath12k_hal_srng_shadow_update_hp_tp(struct ath12k_base *ab, struct hal_srng *srng); void ath12k_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab); +void ath12k_hal_set_link_desc_addr(struct ath12k_hal *hal, + struct hal_wbm_link_desc *desc, u32 cookie, + dma_addr_t paddr, int rbm); +void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab, + struct hal_wbm_idle_scatter_list *sbuf, + u32 nsbufs, u32 tot_link_desc, + u32 end_offset); +u32 +ath12k_hal_ce_dst_status_get_length(struct ath12k_hal *hal, + struct hal_ce_srng_dst_status_desc *desc); +void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id); +void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab, + u32 bank_config, u8 bank_id); +void ath12k_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab); +void ath12k_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab); +void ath12k_hal_write_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr); +void +ath12k_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr); +void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, struct hal_srng *srng); +void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map); +void ath12k_hal_rx_buf_addr_info_set(struct ath12k_hal *hal, + struct ath12k_buffer_addr *binfo, + dma_addr_t paddr, u32 cookie, u8 manager); +void ath12k_hal_rx_buf_addr_info_get(struct ath12k_hal *hal, + struct ath12k_buffer_addr *binfo, + dma_addr_t *paddr, u32 *msdu_cookies, + u8 *rbm); +void ath12k_hal_cc_config(struct ath12k_base *ab); +enum hal_rx_buf_return_buf_manager +ath12k_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id); +void ath12k_hal_rx_msdu_list_get(struct ath12k_hal *hal, struct ath12k *ar, + void *link_desc, void *msdu_list, + u16 *num_msdus); +void ath12k_hal_rx_reo_ent_buf_paddr_get(struct ath12k_hal *hal, void *rx_desc, + dma_addr_t *paddr, u32 *sw_cookie, + struct ath12k_buffer_addr **pp_buf_addr, + u8 *rbm, u32 *msdu_cnt); +void *ath12k_hal_encode_tlv64_hdr(void *tlv, u64 tag, u64 len); +void *ath12k_hal_encode_tlv32_hdr(void *tlv, u64 tag, u64 len); +void *ath12k_hal_decode_tlv64_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid); +void *ath12k_hal_decode_tlv32_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid); +u32 ath12k_hal_get_tlv64_hdr_align(void); +u32 ath12k_hal_get_tlv32_hdr_align(void); #endif diff --git a/drivers/net/wireless/ath/ath12k/htc.c b/drivers/net/wireless/ath/ath12k/htc.c index d13616bf07f43..92138caa2a82f 100644 --- a/drivers/net/wireless/ath/ath12k/htc.c +++ b/drivers/net/wireless/ath/ath12k/htc.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include @@ -376,6 +376,7 @@ void ath12k_htc_rx_completion_handler(struct ath12k_base *ab, out: kfree_skb(skb); } +EXPORT_SYMBOL(ath12k_htc_rx_completion_handler); static void ath12k_htc_control_rx_complete(struct ath12k_base *ab, struct sk_buff *skb) diff --git a/drivers/net/wireless/ath/ath12k/hw.c b/drivers/net/wireless/ath/ath12k/hw.c deleted file mode 100644 index 6791ae1d64e50..0000000000000 --- a/drivers/net/wireless/ath/ath12k/hw.c +++ /dev/null @@ -1,1680 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause-Clear -/* - * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. - */ - -#include -#include -#include - -#include "debug.h" -#include "core.h" -#include "ce.h" -#include "hw.h" -#include "mhi.h" -#include "dp_rx.h" -#include "peer.h" - -static const guid_t wcn7850_uuid = GUID_INIT(0xf634f534, 0x6147, 0x11ec, - 0x90, 0xd6, 0x02, 0x42, - 0xac, 0x12, 0x00, 0x03); - -static u8 ath12k_hw_qcn9274_mac_from_pdev_id(int pdev_idx) -{ - return pdev_idx; -} - -static int ath12k_hw_mac_id_to_pdev_id_qcn9274(const struct ath12k_hw_params *hw, - int mac_id) -{ - return mac_id; -} - -static int ath12k_hw_mac_id_to_srng_id_qcn9274(const struct ath12k_hw_params *hw, - int mac_id) -{ - return 0; -} - -static u8 ath12k_hw_get_ring_selector_qcn9274(struct sk_buff *skb) -{ - return smp_processor_id(); -} - -static bool ath12k_dp_srng_is_comp_ring_qcn9274(int ring_num) -{ - if (ring_num < 3 || ring_num == 4) - return true; - - return false; -} - -static bool ath12k_is_frame_link_agnostic_qcn9274(struct ath12k_link_vif *arvif, - struct ieee80211_mgmt *mgmt) -{ - return ieee80211_is_action(mgmt->frame_control); -} - -static int ath12k_hw_mac_id_to_pdev_id_wcn7850(const struct ath12k_hw_params *hw, - int mac_id) -{ - return 0; -} - -static int ath12k_hw_mac_id_to_srng_id_wcn7850(const struct ath12k_hw_params *hw, - int mac_id) -{ - return mac_id; -} - -static u8 ath12k_hw_get_ring_selector_wcn7850(struct sk_buff *skb) -{ - return skb_get_queue_mapping(skb); -} - -static bool ath12k_dp_srng_is_comp_ring_wcn7850(int ring_num) -{ - if (ring_num == 0 || ring_num == 2 || ring_num == 4) - return true; - - return false; -} - -static bool ath12k_is_addba_resp_action_code(struct ieee80211_mgmt *mgmt) -{ - if (!ieee80211_is_action(mgmt->frame_control)) - return false; - - if (mgmt->u.action.category != WLAN_CATEGORY_BACK) - return false; - - if (mgmt->u.action.u.addba_resp.action_code != WLAN_ACTION_ADDBA_RESP) - return false; - - return true; -} - -static bool ath12k_is_frame_link_agnostic_wcn7850(struct ath12k_link_vif *arvif, - struct ieee80211_mgmt *mgmt) -{ - struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif); - struct ath12k_hw *ah = ath12k_ar_to_ah(arvif->ar); - struct ath12k_base *ab = arvif->ar->ab; - __le16 fc = mgmt->frame_control; - - spin_lock_bh(&ab->base_lock); - if (!ath12k_peer_find_by_addr(ab, mgmt->da) && - !ath12k_peer_ml_find(ah, mgmt->da)) { - spin_unlock_bh(&ab->base_lock); - return false; - } - spin_unlock_bh(&ab->base_lock); - - if (vif->type == NL80211_IFTYPE_STATION) - return arvif->is_up && - (vif->valid_links == vif->active_links) && - !ieee80211_is_probe_req(fc) && - !ieee80211_is_auth(fc) && - !ieee80211_is_deauth(fc) && - !ath12k_is_addba_resp_action_code(mgmt); - - if (vif->type == NL80211_IFTYPE_AP) - return !(ieee80211_is_probe_resp(fc) || ieee80211_is_auth(fc) || - ieee80211_is_assoc_resp(fc) || ieee80211_is_reassoc_resp(fc) || - ath12k_is_addba_resp_action_code(mgmt)); - - return false; -} - -static const struct ath12k_hw_ops qcn9274_ops = { - .get_hw_mac_from_pdev_id = ath12k_hw_qcn9274_mac_from_pdev_id, - .mac_id_to_pdev_id = ath12k_hw_mac_id_to_pdev_id_qcn9274, - .mac_id_to_srng_id = ath12k_hw_mac_id_to_srng_id_qcn9274, - .rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_qcn9274, - .get_ring_selector = ath12k_hw_get_ring_selector_qcn9274, - .dp_srng_is_tx_comp_ring = ath12k_dp_srng_is_comp_ring_qcn9274, - .is_frame_link_agnostic = ath12k_is_frame_link_agnostic_qcn9274, -}; - -static const struct ath12k_hw_ops wcn7850_ops = { - .get_hw_mac_from_pdev_id = ath12k_hw_qcn9274_mac_from_pdev_id, - .mac_id_to_pdev_id = ath12k_hw_mac_id_to_pdev_id_wcn7850, - .mac_id_to_srng_id = ath12k_hw_mac_id_to_srng_id_wcn7850, - .rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_wcn7850, - .get_ring_selector = ath12k_hw_get_ring_selector_wcn7850, - .dp_srng_is_tx_comp_ring = ath12k_dp_srng_is_comp_ring_wcn7850, - .is_frame_link_agnostic = ath12k_is_frame_link_agnostic_wcn7850, -}; - -#define ATH12K_TX_RING_MASK_0 0x1 -#define ATH12K_TX_RING_MASK_1 0x2 -#define ATH12K_TX_RING_MASK_2 0x4 -#define ATH12K_TX_RING_MASK_3 0x8 -#define ATH12K_TX_RING_MASK_4 0x10 - -#define ATH12K_RX_RING_MASK_0 0x1 -#define ATH12K_RX_RING_MASK_1 0x2 -#define ATH12K_RX_RING_MASK_2 0x4 -#define ATH12K_RX_RING_MASK_3 0x8 - -#define ATH12K_RX_ERR_RING_MASK_0 0x1 - -#define ATH12K_RX_WBM_REL_RING_MASK_0 0x1 - -#define ATH12K_REO_STATUS_RING_MASK_0 0x1 - -#define ATH12K_HOST2RXDMA_RING_MASK_0 0x1 - -#define ATH12K_RX_MON_RING_MASK_0 0x1 -#define ATH12K_RX_MON_RING_MASK_1 0x2 -#define ATH12K_RX_MON_RING_MASK_2 0x4 - -#define ATH12K_TX_MON_RING_MASK_0 0x1 -#define ATH12K_TX_MON_RING_MASK_1 0x2 - -#define ATH12K_RX_MON_STATUS_RING_MASK_0 0x1 -#define ATH12K_RX_MON_STATUS_RING_MASK_1 0x2 -#define ATH12K_RX_MON_STATUS_RING_MASK_2 0x4 - -/* Target firmware's Copy Engine configuration. */ -static const struct ce_pipe_config ath12k_target_ce_config_wlan_qcn9274[] = { - /* CE0: host->target HTC control and raw streams */ - { - .pipenum = __cpu_to_le32(0), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE1: target->host HTT + HTC control */ - { - .pipenum = __cpu_to_le32(1), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE2: target->host WMI */ - { - .pipenum = __cpu_to_le32(2), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE3: host->target WMI (mac0) */ - { - .pipenum = __cpu_to_le32(3), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE4: host->target HTT */ - { - .pipenum = __cpu_to_le32(4), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(256), - .nbytes_max = __cpu_to_le32(256), - .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), - .reserved = __cpu_to_le32(0), - }, - - /* CE5: target->host Pktlog */ - { - .pipenum = __cpu_to_le32(5), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE6: Reserved for target autonomous hif_memcpy */ - { - .pipenum = __cpu_to_le32(6), - .pipedir = __cpu_to_le32(PIPEDIR_INOUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(16384), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE7: host->target WMI (mac1) */ - { - .pipenum = __cpu_to_le32(7), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE8: Reserved for target autonomous hif_memcpy */ - { - .pipenum = __cpu_to_le32(8), - .pipedir = __cpu_to_le32(PIPEDIR_INOUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(16384), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE9, 10 and 11: Reserved for MHI */ - - /* CE12: Target CV prefetch */ - { - .pipenum = __cpu_to_le32(12), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE13: Target CV prefetch */ - { - .pipenum = __cpu_to_le32(13), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE14: WMI logging/CFR/Spectral/Radar */ - { - .pipenum = __cpu_to_le32(14), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE15: Reserved */ -}; - -/* Target firmware's Copy Engine configuration. */ -static const struct ce_pipe_config ath12k_target_ce_config_wlan_wcn7850[] = { - /* CE0: host->target HTC control and raw streams */ - { - .pipenum = __cpu_to_le32(0), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE1: target->host HTT + HTC control */ - { - .pipenum = __cpu_to_le32(1), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE2: target->host WMI */ - { - .pipenum = __cpu_to_le32(2), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE3: host->target WMI */ - { - .pipenum = __cpu_to_le32(3), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE4: host->target HTT */ - { - .pipenum = __cpu_to_le32(4), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(256), - .nbytes_max = __cpu_to_le32(256), - .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), - .reserved = __cpu_to_le32(0), - }, - - /* CE5: target->host Pktlog */ - { - .pipenum = __cpu_to_le32(5), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE6: Reserved for target autonomous hif_memcpy */ - { - .pipenum = __cpu_to_le32(6), - .pipedir = __cpu_to_le32(PIPEDIR_INOUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(16384), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - - /* CE7 used only by Host */ - { - .pipenum = __cpu_to_le32(7), - .pipedir = __cpu_to_le32(PIPEDIR_INOUT_H2H), - .nentries = __cpu_to_le32(0), - .nbytes_max = __cpu_to_le32(0), - .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), - .reserved = __cpu_to_le32(0), - }, - - /* CE8 target->host used only by IPA */ - { - .pipenum = __cpu_to_le32(8), - .pipedir = __cpu_to_le32(PIPEDIR_INOUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(16384), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* CE 9, 10, 11 are used by MHI driver */ -}; - -/* Map from service/endpoint to Copy Engine. - * This table is derived from the CE_PCI TABLE, above. - * It is passed to the Target at startup for use by firmware. - */ -static const struct service_to_pipe ath12k_target_service_to_ce_map_wlan_qcn9274[] = { - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(0), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(1), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(0), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(1), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(4), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(1), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(7), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(5), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(14), - }, - - /* (Additions here) */ - - { /* must be last */ - __cpu_to_le32(0), - __cpu_to_le32(0), - __cpu_to_le32(0), - }, -}; - -static const struct service_to_pipe ath12k_target_service_to_ce_map_wlan_wcn7850[] = { - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(0), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), - __cpu_to_le32(PIPEDIR_OUT), /* out = UL = host -> target */ - __cpu_to_le32(4), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), - __cpu_to_le32(PIPEDIR_IN), /* in = DL = target -> host */ - __cpu_to_le32(1), - }, - - /* (Additions here) */ - - { /* must be last */ - __cpu_to_le32(0), - __cpu_to_le32(0), - __cpu_to_le32(0), - }, -}; - -static const struct ce_pipe_config ath12k_target_ce_config_wlan_ipq5332[] = { - /* host->target HTC control and raw streams */ - { - .pipenum = __cpu_to_le32(0), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* target->host HTT */ - { - .pipenum = __cpu_to_le32(1), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* target->host WMI + HTC control */ - { - .pipenum = __cpu_to_le32(2), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* host->target WMI */ - { - .pipenum = __cpu_to_le32(3), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* host->target HTT */ - { - .pipenum = __cpu_to_le32(4), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(256), - .nbytes_max = __cpu_to_le32(256), - .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), - .reserved = __cpu_to_le32(0), - }, - /* Target -> host PKTLOG */ - { - .pipenum = __cpu_to_le32(5), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* Reserved for target autonomous HIF_memcpy */ - { - .pipenum = __cpu_to_le32(6), - .pipedir = __cpu_to_le32(PIPEDIR_INOUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(16384), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* CE7 Reserved for CV Prefetch */ - { - .pipenum = __cpu_to_le32(7), - .pipedir = __cpu_to_le32(PIPEDIR_OUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* CE8 Reserved for target generic HIF memcpy */ - { - .pipenum = __cpu_to_le32(8), - .pipedir = __cpu_to_le32(PIPEDIR_INOUT), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(16384), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* CE9 WMI logging/CFR/Spectral/Radar/ */ - { - .pipenum = __cpu_to_le32(9), - .pipedir = __cpu_to_le32(PIPEDIR_IN), - .nentries = __cpu_to_le32(32), - .nbytes_max = __cpu_to_le32(2048), - .flags = __cpu_to_le32(CE_ATTR_FLAGS), - .reserved = __cpu_to_le32(0), - }, - /* Unused TBD */ - { - .pipenum = __cpu_to_le32(10), - .pipedir = __cpu_to_le32(PIPEDIR_NONE), - .nentries = __cpu_to_le32(0), - .nbytes_max = __cpu_to_le32(0), - .flags = __cpu_to_le32(0), - .reserved = __cpu_to_le32(0), - }, - /* Unused TBD */ - { - .pipenum = __cpu_to_le32(11), - .pipedir = __cpu_to_le32(PIPEDIR_NONE), - .nentries = __cpu_to_le32(0), - .nbytes_max = __cpu_to_le32(0), - .flags = __cpu_to_le32(0), - .reserved = __cpu_to_le32(0), - }, -}; - -static const struct service_to_pipe ath12k_target_service_to_ce_map_wlan_ipq5332[] = { - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(3), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(2), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(0), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(1), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(0), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(1), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), - __cpu_to_le32(PIPEDIR_OUT), - __cpu_to_le32(4), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(1), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(5), - }, - { - __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG), - __cpu_to_le32(PIPEDIR_IN), - __cpu_to_le32(9), - }, - /* (Additions here) */ - - { /* must be last */ - __cpu_to_le32(0), - __cpu_to_le32(0), - __cpu_to_le32(0), - }, -}; - -static const struct ath12k_hw_ring_mask ath12k_hw_ring_mask_qcn9274 = { - .tx = { - ATH12K_TX_RING_MASK_0, - ATH12K_TX_RING_MASK_1, - ATH12K_TX_RING_MASK_2, - ATH12K_TX_RING_MASK_3, - }, - .rx_mon_dest = { - 0, 0, 0, 0, - 0, 0, 0, 0, - ATH12K_RX_MON_RING_MASK_0, - ATH12K_RX_MON_RING_MASK_1, - ATH12K_RX_MON_RING_MASK_2, - }, - .rx = { - 0, 0, 0, 0, - ATH12K_RX_RING_MASK_0, - ATH12K_RX_RING_MASK_1, - ATH12K_RX_RING_MASK_2, - ATH12K_RX_RING_MASK_3, - }, - .rx_err = { - 0, 0, 0, - ATH12K_RX_ERR_RING_MASK_0, - }, - .rx_wbm_rel = { - 0, 0, 0, - ATH12K_RX_WBM_REL_RING_MASK_0, - }, - .reo_status = { - 0, 0, 0, - ATH12K_REO_STATUS_RING_MASK_0, - }, - .host2rxdma = { - 0, 0, 0, - ATH12K_HOST2RXDMA_RING_MASK_0, - }, - .tx_mon_dest = { - 0, 0, 0, - }, -}; - -static const struct ath12k_hw_ring_mask ath12k_hw_ring_mask_ipq5332 = { - .tx = { - ATH12K_TX_RING_MASK_0, - ATH12K_TX_RING_MASK_1, - ATH12K_TX_RING_MASK_2, - ATH12K_TX_RING_MASK_3, - }, - .rx_mon_dest = { - 0, 0, 0, 0, 0, 0, 0, 0, - ATH12K_RX_MON_RING_MASK_0, - }, - .rx = { - 0, 0, 0, 0, - ATH12K_RX_RING_MASK_0, - ATH12K_RX_RING_MASK_1, - ATH12K_RX_RING_MASK_2, - ATH12K_RX_RING_MASK_3, - }, - .rx_err = { - 0, 0, 0, - ATH12K_RX_ERR_RING_MASK_0, - }, - .rx_wbm_rel = { - 0, 0, 0, - ATH12K_RX_WBM_REL_RING_MASK_0, - }, - .reo_status = { - 0, 0, 0, - ATH12K_REO_STATUS_RING_MASK_0, - }, - .host2rxdma = { - 0, 0, 0, - ATH12K_HOST2RXDMA_RING_MASK_0, - }, - .tx_mon_dest = { - ATH12K_TX_MON_RING_MASK_0, - ATH12K_TX_MON_RING_MASK_1, - }, -}; - -static const struct ath12k_hw_ring_mask ath12k_hw_ring_mask_wcn7850 = { - .tx = { - ATH12K_TX_RING_MASK_0, - ATH12K_TX_RING_MASK_1, - ATH12K_TX_RING_MASK_2, - }, - .rx_mon_dest = { - }, - .rx_mon_status = { - 0, 0, 0, 0, - ATH12K_RX_MON_STATUS_RING_MASK_0, - ATH12K_RX_MON_STATUS_RING_MASK_1, - ATH12K_RX_MON_STATUS_RING_MASK_2, - }, - .rx = { - 0, 0, 0, - ATH12K_RX_RING_MASK_0, - ATH12K_RX_RING_MASK_1, - ATH12K_RX_RING_MASK_2, - ATH12K_RX_RING_MASK_3, - }, - .rx_err = { - ATH12K_RX_ERR_RING_MASK_0, - }, - .rx_wbm_rel = { - ATH12K_RX_WBM_REL_RING_MASK_0, - }, - .reo_status = { - ATH12K_REO_STATUS_RING_MASK_0, - }, - .host2rxdma = { - }, - .tx_mon_dest = { - }, -}; - -static const struct ath12k_hw_regs qcn9274_v1_regs = { - /* SW2TCL(x) R0 ring configuration address */ - .hal_tcl1_ring_id = 0x00000908, - .hal_tcl1_ring_misc = 0x00000910, - .hal_tcl1_ring_tp_addr_lsb = 0x0000091c, - .hal_tcl1_ring_tp_addr_msb = 0x00000920, - .hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000930, - .hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000934, - .hal_tcl1_ring_msi1_base_lsb = 0x00000948, - .hal_tcl1_ring_msi1_base_msb = 0x0000094c, - .hal_tcl1_ring_msi1_data = 0x00000950, - .hal_tcl_ring_base_lsb = 0x00000b58, - .hal_tcl1_ring_base_lsb = 0x00000900, - .hal_tcl1_ring_base_msb = 0x00000904, - .hal_tcl2_ring_base_lsb = 0x00000978, - - /* TCL STATUS ring address */ - .hal_tcl_status_ring_base_lsb = 0x00000d38, - - .hal_wbm_idle_ring_base_lsb = 0x00000d0c, - .hal_wbm_idle_ring_misc_addr = 0x00000d1c, - .hal_wbm_r0_idle_list_cntl_addr = 0x00000210, - .hal_wbm_r0_idle_list_size_addr = 0x00000214, - .hal_wbm_scattered_ring_base_lsb = 0x00000220, - .hal_wbm_scattered_ring_base_msb = 0x00000224, - .hal_wbm_scattered_desc_head_info_ix0 = 0x00000230, - .hal_wbm_scattered_desc_head_info_ix1 = 0x00000234, - .hal_wbm_scattered_desc_tail_info_ix0 = 0x00000240, - .hal_wbm_scattered_desc_tail_info_ix1 = 0x00000244, - .hal_wbm_scattered_desc_ptr_hp_addr = 0x0000024c, - - .hal_wbm_sw_release_ring_base_lsb = 0x0000034c, - .hal_wbm_sw1_release_ring_base_lsb = 0x000003c4, - .hal_wbm0_release_ring_base_lsb = 0x00000dd8, - .hal_wbm1_release_ring_base_lsb = 0x00000e50, - - /* PCIe base address */ - .pcie_qserdes_sysclk_en_sel = 0x01e0c0a8, - .pcie_pcs_osc_dtct_config_base = 0x01e0d45c, - - /* PPE release ring address */ - .hal_ppe_rel_ring_base = 0x0000043c, - - /* REO DEST ring address */ - .hal_reo2_ring_base = 0x0000055c, - .hal_reo1_misc_ctrl_addr = 0x00000b7c, - .hal_reo1_sw_cookie_cfg0 = 0x00000050, - .hal_reo1_sw_cookie_cfg1 = 0x00000054, - .hal_reo1_qdesc_lut_base0 = 0x00000058, - .hal_reo1_qdesc_lut_base1 = 0x0000005c, - .hal_reo1_ring_base_lsb = 0x000004e4, - .hal_reo1_ring_base_msb = 0x000004e8, - .hal_reo1_ring_id = 0x000004ec, - .hal_reo1_ring_misc = 0x000004f4, - .hal_reo1_ring_hp_addr_lsb = 0x000004f8, - .hal_reo1_ring_hp_addr_msb = 0x000004fc, - .hal_reo1_ring_producer_int_setup = 0x00000508, - .hal_reo1_ring_msi1_base_lsb = 0x0000052C, - .hal_reo1_ring_msi1_base_msb = 0x00000530, - .hal_reo1_ring_msi1_data = 0x00000534, - .hal_reo1_aging_thres_ix0 = 0x00000b08, - .hal_reo1_aging_thres_ix1 = 0x00000b0c, - .hal_reo1_aging_thres_ix2 = 0x00000b10, - .hal_reo1_aging_thres_ix3 = 0x00000b14, - - /* REO Exception ring address */ - .hal_reo2_sw0_ring_base = 0x000008a4, - - /* REO Reinject ring address */ - .hal_sw2reo_ring_base = 0x00000304, - .hal_sw2reo1_ring_base = 0x0000037c, - - /* REO cmd ring address */ - .hal_reo_cmd_ring_base = 0x0000028c, - - /* REO status ring address */ - .hal_reo_status_ring_base = 0x00000a84, - - /* CE base address */ - .hal_umac_ce0_src_reg_base = 0x01b80000, - .hal_umac_ce0_dest_reg_base = 0x01b81000, - .hal_umac_ce1_src_reg_base = 0x01b82000, - .hal_umac_ce1_dest_reg_base = 0x01b83000, - - .gcc_gcc_pcie_hot_rst = 0x1e38338, -}; - -static const struct ath12k_hw_regs qcn9274_v2_regs = { - /* SW2TCL(x) R0 ring configuration address */ - .hal_tcl1_ring_id = 0x00000908, - .hal_tcl1_ring_misc = 0x00000910, - .hal_tcl1_ring_tp_addr_lsb = 0x0000091c, - .hal_tcl1_ring_tp_addr_msb = 0x00000920, - .hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000930, - .hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000934, - .hal_tcl1_ring_msi1_base_lsb = 0x00000948, - .hal_tcl1_ring_msi1_base_msb = 0x0000094c, - .hal_tcl1_ring_msi1_data = 0x00000950, - .hal_tcl_ring_base_lsb = 0x00000b58, - .hal_tcl1_ring_base_lsb = 0x00000900, - .hal_tcl1_ring_base_msb = 0x00000904, - .hal_tcl2_ring_base_lsb = 0x00000978, - - /* TCL STATUS ring address */ - .hal_tcl_status_ring_base_lsb = 0x00000d38, - - /* WBM idle link ring address */ - .hal_wbm_idle_ring_base_lsb = 0x00000d3c, - .hal_wbm_idle_ring_misc_addr = 0x00000d4c, - .hal_wbm_r0_idle_list_cntl_addr = 0x00000240, - .hal_wbm_r0_idle_list_size_addr = 0x00000244, - .hal_wbm_scattered_ring_base_lsb = 0x00000250, - .hal_wbm_scattered_ring_base_msb = 0x00000254, - .hal_wbm_scattered_desc_head_info_ix0 = 0x00000260, - .hal_wbm_scattered_desc_head_info_ix1 = 0x00000264, - .hal_wbm_scattered_desc_tail_info_ix0 = 0x00000270, - .hal_wbm_scattered_desc_tail_info_ix1 = 0x00000274, - .hal_wbm_scattered_desc_ptr_hp_addr = 0x0000027c, - - /* SW2WBM release ring address */ - .hal_wbm_sw_release_ring_base_lsb = 0x0000037c, - .hal_wbm_sw1_release_ring_base_lsb = 0x000003f4, - - /* WBM2SW release ring address */ - .hal_wbm0_release_ring_base_lsb = 0x00000e08, - .hal_wbm1_release_ring_base_lsb = 0x00000e80, - - /* PCIe base address */ - .pcie_qserdes_sysclk_en_sel = 0x01e0c0a8, - .pcie_pcs_osc_dtct_config_base = 0x01e0d45c, - - /* PPE release ring address */ - .hal_ppe_rel_ring_base = 0x0000046c, - - /* REO DEST ring address */ - .hal_reo2_ring_base = 0x00000578, - .hal_reo1_misc_ctrl_addr = 0x00000b9c, - .hal_reo1_sw_cookie_cfg0 = 0x0000006c, - .hal_reo1_sw_cookie_cfg1 = 0x00000070, - .hal_reo1_qdesc_lut_base0 = 0x00000074, - .hal_reo1_qdesc_lut_base1 = 0x00000078, - .hal_reo1_qdesc_addr = 0x0000007c, - .hal_reo1_qdesc_max_peerid = 0x00000088, - .hal_reo1_ring_base_lsb = 0x00000500, - .hal_reo1_ring_base_msb = 0x00000504, - .hal_reo1_ring_id = 0x00000508, - .hal_reo1_ring_misc = 0x00000510, - .hal_reo1_ring_hp_addr_lsb = 0x00000514, - .hal_reo1_ring_hp_addr_msb = 0x00000518, - .hal_reo1_ring_producer_int_setup = 0x00000524, - .hal_reo1_ring_msi1_base_lsb = 0x00000548, - .hal_reo1_ring_msi1_base_msb = 0x0000054C, - .hal_reo1_ring_msi1_data = 0x00000550, - .hal_reo1_aging_thres_ix0 = 0x00000B28, - .hal_reo1_aging_thres_ix1 = 0x00000B2C, - .hal_reo1_aging_thres_ix2 = 0x00000B30, - .hal_reo1_aging_thres_ix3 = 0x00000B34, - - /* REO Exception ring address */ - .hal_reo2_sw0_ring_base = 0x000008c0, - - /* REO Reinject ring address */ - .hal_sw2reo_ring_base = 0x00000320, - .hal_sw2reo1_ring_base = 0x00000398, - - /* REO cmd ring address */ - .hal_reo_cmd_ring_base = 0x000002A8, - - /* REO status ring address */ - .hal_reo_status_ring_base = 0x00000aa0, - - /* CE base address */ - .hal_umac_ce0_src_reg_base = 0x01b80000, - .hal_umac_ce0_dest_reg_base = 0x01b81000, - .hal_umac_ce1_src_reg_base = 0x01b82000, - .hal_umac_ce1_dest_reg_base = 0x01b83000, - - .gcc_gcc_pcie_hot_rst = 0x1e38338, -}; - -static const struct ath12k_hw_regs ipq5332_regs = { - /* SW2TCL(x) R0 ring configuration address */ - .hal_tcl1_ring_id = 0x00000918, - .hal_tcl1_ring_misc = 0x00000920, - .hal_tcl1_ring_tp_addr_lsb = 0x0000092c, - .hal_tcl1_ring_tp_addr_msb = 0x00000930, - .hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000940, - .hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000944, - .hal_tcl1_ring_msi1_base_lsb = 0x00000958, - .hal_tcl1_ring_msi1_base_msb = 0x0000095c, - .hal_tcl1_ring_base_lsb = 0x00000910, - .hal_tcl1_ring_base_msb = 0x00000914, - .hal_tcl1_ring_msi1_data = 0x00000960, - .hal_tcl2_ring_base_lsb = 0x00000988, - .hal_tcl_ring_base_lsb = 0x00000b68, - - /* TCL STATUS ring address */ - .hal_tcl_status_ring_base_lsb = 0x00000d48, - - /* REO DEST ring address */ - .hal_reo2_ring_base = 0x00000578, - .hal_reo1_misc_ctrl_addr = 0x00000b9c, - .hal_reo1_sw_cookie_cfg0 = 0x0000006c, - .hal_reo1_sw_cookie_cfg1 = 0x00000070, - .hal_reo1_qdesc_lut_base0 = 0x00000074, - .hal_reo1_qdesc_lut_base1 = 0x00000078, - .hal_reo1_ring_base_lsb = 0x00000500, - .hal_reo1_ring_base_msb = 0x00000504, - .hal_reo1_ring_id = 0x00000508, - .hal_reo1_ring_misc = 0x00000510, - .hal_reo1_ring_hp_addr_lsb = 0x00000514, - .hal_reo1_ring_hp_addr_msb = 0x00000518, - .hal_reo1_ring_producer_int_setup = 0x00000524, - .hal_reo1_ring_msi1_base_lsb = 0x00000548, - .hal_reo1_ring_msi1_base_msb = 0x0000054C, - .hal_reo1_ring_msi1_data = 0x00000550, - .hal_reo1_aging_thres_ix0 = 0x00000B28, - .hal_reo1_aging_thres_ix1 = 0x00000B2C, - .hal_reo1_aging_thres_ix2 = 0x00000B30, - .hal_reo1_aging_thres_ix3 = 0x00000B34, - - /* REO Exception ring address */ - .hal_reo2_sw0_ring_base = 0x000008c0, - - /* REO Reinject ring address */ - .hal_sw2reo_ring_base = 0x00000320, - .hal_sw2reo1_ring_base = 0x00000398, - - /* REO cmd ring address */ - .hal_reo_cmd_ring_base = 0x000002A8, - - /* REO status ring address */ - .hal_reo_status_ring_base = 0x00000aa0, - - /* WBM idle link ring address */ - .hal_wbm_idle_ring_base_lsb = 0x00000d3c, - .hal_wbm_idle_ring_misc_addr = 0x00000d4c, - .hal_wbm_r0_idle_list_cntl_addr = 0x00000240, - .hal_wbm_r0_idle_list_size_addr = 0x00000244, - .hal_wbm_scattered_ring_base_lsb = 0x00000250, - .hal_wbm_scattered_ring_base_msb = 0x00000254, - .hal_wbm_scattered_desc_head_info_ix0 = 0x00000260, - .hal_wbm_scattered_desc_head_info_ix1 = 0x00000264, - .hal_wbm_scattered_desc_tail_info_ix0 = 0x00000270, - .hal_wbm_scattered_desc_tail_info_ix1 = 0x00000274, - .hal_wbm_scattered_desc_ptr_hp_addr = 0x0000027c, - - /* SW2WBM release ring address */ - .hal_wbm_sw_release_ring_base_lsb = 0x0000037c, - - /* WBM2SW release ring address */ - .hal_wbm0_release_ring_base_lsb = 0x00000e08, - .hal_wbm1_release_ring_base_lsb = 0x00000e80, - - /* PPE release ring address */ - .hal_ppe_rel_ring_base = 0x0000046c, - - /* CE address */ - .hal_umac_ce0_src_reg_base = 0x00740000 - - HAL_IPQ5332_CE_WFSS_REG_BASE, - .hal_umac_ce0_dest_reg_base = 0x00741000 - - HAL_IPQ5332_CE_WFSS_REG_BASE, - .hal_umac_ce1_src_reg_base = 0x00742000 - - HAL_IPQ5332_CE_WFSS_REG_BASE, - .hal_umac_ce1_dest_reg_base = 0x00743000 - - HAL_IPQ5332_CE_WFSS_REG_BASE, -}; - -static const struct ath12k_hw_regs wcn7850_regs = { - /* SW2TCL(x) R0 ring configuration address */ - .hal_tcl1_ring_id = 0x00000908, - .hal_tcl1_ring_misc = 0x00000910, - .hal_tcl1_ring_tp_addr_lsb = 0x0000091c, - .hal_tcl1_ring_tp_addr_msb = 0x00000920, - .hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000930, - .hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000934, - .hal_tcl1_ring_msi1_base_lsb = 0x00000948, - .hal_tcl1_ring_msi1_base_msb = 0x0000094c, - .hal_tcl1_ring_msi1_data = 0x00000950, - .hal_tcl_ring_base_lsb = 0x00000b58, - .hal_tcl1_ring_base_lsb = 0x00000900, - .hal_tcl1_ring_base_msb = 0x00000904, - .hal_tcl2_ring_base_lsb = 0x00000978, - - /* TCL STATUS ring address */ - .hal_tcl_status_ring_base_lsb = 0x00000d38, - - .hal_wbm_idle_ring_base_lsb = 0x00000d3c, - .hal_wbm_idle_ring_misc_addr = 0x00000d4c, - .hal_wbm_r0_idle_list_cntl_addr = 0x00000240, - .hal_wbm_r0_idle_list_size_addr = 0x00000244, - .hal_wbm_scattered_ring_base_lsb = 0x00000250, - .hal_wbm_scattered_ring_base_msb = 0x00000254, - .hal_wbm_scattered_desc_head_info_ix0 = 0x00000260, - .hal_wbm_scattered_desc_head_info_ix1 = 0x00000264, - .hal_wbm_scattered_desc_tail_info_ix0 = 0x00000270, - .hal_wbm_scattered_desc_tail_info_ix1 = 0x00000274, - .hal_wbm_scattered_desc_ptr_hp_addr = 0x00000027c, - - .hal_wbm_sw_release_ring_base_lsb = 0x0000037c, - .hal_wbm_sw1_release_ring_base_lsb = 0x00000284, - .hal_wbm0_release_ring_base_lsb = 0x00000e08, - .hal_wbm1_release_ring_base_lsb = 0x00000e80, - - /* PCIe base address */ - .pcie_qserdes_sysclk_en_sel = 0x01e0e0a8, - .pcie_pcs_osc_dtct_config_base = 0x01e0f45c, - - /* PPE release ring address */ - .hal_ppe_rel_ring_base = 0x0000043c, - - /* REO DEST ring address */ - .hal_reo2_ring_base = 0x0000055c, - .hal_reo1_misc_ctrl_addr = 0x00000b7c, - .hal_reo1_sw_cookie_cfg0 = 0x00000050, - .hal_reo1_sw_cookie_cfg1 = 0x00000054, - .hal_reo1_qdesc_lut_base0 = 0x00000058, - .hal_reo1_qdesc_lut_base1 = 0x0000005c, - .hal_reo1_ring_base_lsb = 0x000004e4, - .hal_reo1_ring_base_msb = 0x000004e8, - .hal_reo1_ring_id = 0x000004ec, - .hal_reo1_ring_misc = 0x000004f4, - .hal_reo1_ring_hp_addr_lsb = 0x000004f8, - .hal_reo1_ring_hp_addr_msb = 0x000004fc, - .hal_reo1_ring_producer_int_setup = 0x00000508, - .hal_reo1_ring_msi1_base_lsb = 0x0000052C, - .hal_reo1_ring_msi1_base_msb = 0x00000530, - .hal_reo1_ring_msi1_data = 0x00000534, - .hal_reo1_aging_thres_ix0 = 0x00000b08, - .hal_reo1_aging_thres_ix1 = 0x00000b0c, - .hal_reo1_aging_thres_ix2 = 0x00000b10, - .hal_reo1_aging_thres_ix3 = 0x00000b14, - - /* REO Exception ring address */ - .hal_reo2_sw0_ring_base = 0x000008a4, - - /* REO Reinject ring address */ - .hal_sw2reo_ring_base = 0x00000304, - .hal_sw2reo1_ring_base = 0x0000037c, - - /* REO cmd ring address */ - .hal_reo_cmd_ring_base = 0x0000028c, - - /* REO status ring address */ - .hal_reo_status_ring_base = 0x00000a84, - - /* CE base address */ - .hal_umac_ce0_src_reg_base = 0x01b80000, - .hal_umac_ce0_dest_reg_base = 0x01b81000, - .hal_umac_ce1_src_reg_base = 0x01b82000, - .hal_umac_ce1_dest_reg_base = 0x01b83000, - - .gcc_gcc_pcie_hot_rst = 0x1e40304, -}; - -static const struct ath12k_hw_hal_params ath12k_hw_hal_params_qcn9274 = { - .rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM, - .wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN, -}; - -static const struct ath12k_hw_hal_params ath12k_hw_hal_params_wcn7850 = { - .rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM, - .wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN, -}; - -static const struct ath12k_hw_hal_params ath12k_hw_hal_params_ipq5332 = { - .rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM, - .wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN | - HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN, -}; - -static const struct ce_ie_addr ath12k_ce_ie_addr_ipq5332 = { - .ie1_reg_addr = CE_HOST_IE_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE, - .ie2_reg_addr = CE_HOST_IE_2_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE, - .ie3_reg_addr = CE_HOST_IE_3_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE, -}; - -static const struct ce_remap ath12k_ce_remap_ipq5332 = { - .base = HAL_IPQ5332_CE_WFSS_REG_BASE, - .size = HAL_IPQ5332_CE_SIZE, -}; - -static const struct ath12k_hw_params ath12k_hw_params[] = { - { - .name = "qcn9274 hw1.0", - .hw_rev = ATH12K_HW_QCN9274_HW10, - .fw = { - .dir = "QCN9274/hw1.0", - .board_size = 256 * 1024, - .cal_offset = 128 * 1024, - .m3_loader = ath12k_m3_fw_loader_driver, - }, - .max_radios = 1, - .single_pdev_only = false, - .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274, - .internal_sleep_clock = false, - - .hw_ops = &qcn9274_ops, - .ring_mask = &ath12k_hw_ring_mask_qcn9274, - .regs = &qcn9274_v1_regs, - - .host_ce_config = ath12k_host_ce_config_qcn9274, - .ce_count = 16, - .target_ce_config = ath12k_target_ce_config_wlan_qcn9274, - .target_ce_count = 12, - .svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_qcn9274, - .svc_to_ce_map_len = 18, - - .hal_params = &ath12k_hw_hal_params_qcn9274, - - .rxdma1_enable = false, - .num_rxdma_per_pdev = 1, - .num_rxdma_dst_ring = 0, - .rx_mac_buf_ring = false, - .vdev_start_delay = false, - - .interface_modes = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP) | - BIT(NL80211_IFTYPE_MESH_POINT) | - BIT(NL80211_IFTYPE_AP_VLAN), - .supports_monitor = false, - - .idle_ps = false, - .download_calib = true, - .supports_suspend = false, - .tcl_ring_retry = true, - .reoq_lut_support = true, - .supports_shadow_regs = false, - - .num_tcl_banks = 48, - .max_tx_ring = 4, - - .mhi_config = &ath12k_mhi_config_qcn9274, - - .wmi_init = ath12k_wmi_init_qcn9274, - - .hal_ops = &hal_qcn9274_ops, - - .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01), - - .rfkill_pin = 0, - .rfkill_cfg = 0, - .rfkill_on_level = 0, - - .rddm_size = 0x600000, - - .def_num_link = 0, - .max_mlo_peer = 256, - - .otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB, - - .supports_sta_ps = false, - - .acpi_guid = NULL, - .supports_dynamic_smps_6ghz = true, - - .iova_mask = 0, - - .supports_aspm = false, - - .ce_ie_addr = NULL, - .ce_remap = NULL, - .bdf_addr_offset = 0, - - .current_cc_support = false, - - .dp_primary_link_only = true, - }, - { - .name = "wcn7850 hw2.0", - .hw_rev = ATH12K_HW_WCN7850_HW20, - - .fw = { - .dir = "WCN7850/hw2.0", - .board_size = 256 * 1024, - .cal_offset = 256 * 1024, - .m3_loader = ath12k_m3_fw_loader_driver, - }, - - .max_radios = 1, - .single_pdev_only = true, - .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850, - .internal_sleep_clock = true, - - .hw_ops = &wcn7850_ops, - .ring_mask = &ath12k_hw_ring_mask_wcn7850, - .regs = &wcn7850_regs, - - .host_ce_config = ath12k_host_ce_config_wcn7850, - .ce_count = 9, - .target_ce_config = ath12k_target_ce_config_wlan_wcn7850, - .target_ce_count = 9, - .svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_wcn7850, - .svc_to_ce_map_len = 14, - - .hal_params = &ath12k_hw_hal_params_wcn7850, - - .rxdma1_enable = false, - .num_rxdma_per_pdev = 2, - .num_rxdma_dst_ring = 1, - .rx_mac_buf_ring = true, - .vdev_start_delay = true, - - .interface_modes = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP) | - BIT(NL80211_IFTYPE_P2P_DEVICE) | - BIT(NL80211_IFTYPE_P2P_CLIENT) | - BIT(NL80211_IFTYPE_P2P_GO), - .supports_monitor = true, - - .idle_ps = true, - .download_calib = false, - .supports_suspend = true, - .tcl_ring_retry = false, - .reoq_lut_support = false, - .supports_shadow_regs = true, - - .num_tcl_banks = 7, - .max_tx_ring = 3, - - .mhi_config = &ath12k_mhi_config_wcn7850, - - .wmi_init = ath12k_wmi_init_wcn7850, - - .hal_ops = &hal_wcn7850_ops, - - .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01) | - BIT(CNSS_PCIE_PERST_NO_PULL_V01), - - .rfkill_pin = 48, - .rfkill_cfg = 0, - .rfkill_on_level = 1, - - .rddm_size = 0x780000, - - .def_num_link = 2, - .max_mlo_peer = 32, - - .otp_board_id_register = 0, - - .supports_sta_ps = true, - - .acpi_guid = &wcn7850_uuid, - .supports_dynamic_smps_6ghz = false, - - .iova_mask = ATH12K_PCIE_MAX_PAYLOAD_SIZE - 1, - - .supports_aspm = true, - - .ce_ie_addr = NULL, - .ce_remap = NULL, - .bdf_addr_offset = 0, - - .current_cc_support = true, - - .dp_primary_link_only = false, - }, - { - .name = "qcn9274 hw2.0", - .hw_rev = ATH12K_HW_QCN9274_HW20, - .fw = { - .dir = "QCN9274/hw2.0", - .board_size = 256 * 1024, - .cal_offset = 128 * 1024, - .m3_loader = ath12k_m3_fw_loader_driver, - }, - .max_radios = 2, - .single_pdev_only = false, - .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274, - .internal_sleep_clock = false, - - .hw_ops = &qcn9274_ops, - .ring_mask = &ath12k_hw_ring_mask_qcn9274, - .regs = &qcn9274_v2_regs, - - .host_ce_config = ath12k_host_ce_config_qcn9274, - .ce_count = 16, - .target_ce_config = ath12k_target_ce_config_wlan_qcn9274, - .target_ce_count = 12, - .svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_qcn9274, - .svc_to_ce_map_len = 18, - - .hal_params = &ath12k_hw_hal_params_qcn9274, - - .rxdma1_enable = true, - .num_rxdma_per_pdev = 1, - .num_rxdma_dst_ring = 0, - .rx_mac_buf_ring = false, - .vdev_start_delay = false, - - .interface_modes = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP) | - BIT(NL80211_IFTYPE_MESH_POINT) | - BIT(NL80211_IFTYPE_AP_VLAN), - .supports_monitor = true, - - .idle_ps = false, - .download_calib = true, - .supports_suspend = false, - .tcl_ring_retry = true, - .reoq_lut_support = true, - .supports_shadow_regs = false, - - .num_tcl_banks = 48, - .max_tx_ring = 4, - - .mhi_config = &ath12k_mhi_config_qcn9274, - - .wmi_init = ath12k_wmi_init_qcn9274, - - .hal_ops = &hal_qcn9274_ops, - - .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01), - - .rfkill_pin = 0, - .rfkill_cfg = 0, - .rfkill_on_level = 0, - - .rddm_size = 0x600000, - - .def_num_link = 0, - .max_mlo_peer = 256, - - .otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB, - - .supports_sta_ps = false, - - .acpi_guid = NULL, - .supports_dynamic_smps_6ghz = true, - - .iova_mask = 0, - - .supports_aspm = false, - - .ce_ie_addr = NULL, - .ce_remap = NULL, - .bdf_addr_offset = 0, - - .current_cc_support = false, - - .dp_primary_link_only = true, - }, - { - .name = "ipq5332 hw1.0", - .hw_rev = ATH12K_HW_IPQ5332_HW10, - .fw = { - .dir = "IPQ5332/hw1.0", - .board_size = 256 * 1024, - .cal_offset = 128 * 1024, - .m3_loader = ath12k_m3_fw_loader_remoteproc, - }, - .max_radios = 1, - .single_pdev_only = false, - .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_IPQ5332, - .internal_sleep_clock = false, - - .hw_ops = &qcn9274_ops, - .regs = &ipq5332_regs, - .ring_mask = &ath12k_hw_ring_mask_ipq5332, - - .host_ce_config = ath12k_host_ce_config_ipq5332, - .ce_count = 12, - .target_ce_config = ath12k_target_ce_config_wlan_ipq5332, - .target_ce_count = 12, - .svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_ipq5332, - .svc_to_ce_map_len = 18, - - .hal_params = &ath12k_hw_hal_params_ipq5332, - - .rxdma1_enable = false, - .num_rxdma_per_pdev = 1, - .num_rxdma_dst_ring = 0, - .rx_mac_buf_ring = false, - .vdev_start_delay = false, - - .interface_modes = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP) | - BIT(NL80211_IFTYPE_MESH_POINT), - .supports_monitor = false, - - .idle_ps = false, - .download_calib = true, - .supports_suspend = false, - .tcl_ring_retry = true, - .reoq_lut_support = false, - .supports_shadow_regs = false, - - .num_tcl_banks = 48, - .max_tx_ring = 4, - - .wmi_init = &ath12k_wmi_init_qcn9274, - - .hal_ops = &hal_qcn9274_ops, - - .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01), - - .rfkill_pin = 0, - .rfkill_cfg = 0, - .rfkill_on_level = 0, - - .rddm_size = 0, - - .def_num_link = 0, - .max_mlo_peer = 256, - - .otp_board_id_register = 0, - - .supports_sta_ps = false, - - .acpi_guid = NULL, - .supports_dynamic_smps_6ghz = false, - .iova_mask = 0, - .supports_aspm = false, - - .ce_ie_addr = &ath12k_ce_ie_addr_ipq5332, - .ce_remap = &ath12k_ce_remap_ipq5332, - .bdf_addr_offset = 0xC00000, - - .dp_primary_link_only = true, - }, -}; - -int ath12k_hw_init(struct ath12k_base *ab) -{ - const struct ath12k_hw_params *hw_params = NULL; - int i; - - for (i = 0; i < ARRAY_SIZE(ath12k_hw_params); i++) { - hw_params = &ath12k_hw_params[i]; - - if (hw_params->hw_rev == ab->hw_rev) - break; - } - - if (i == ARRAY_SIZE(ath12k_hw_params)) { - ath12k_err(ab, "Unsupported hardware version: 0x%x\n", ab->hw_rev); - return -EINVAL; - } - - ab->hw_params = hw_params; - - ath12k_info(ab, "Hardware name: %s\n", ab->hw_params->name); - - return 0; -} diff --git a/drivers/net/wireless/ath/ath12k/hw.h b/drivers/net/wireless/ath/ath12k/hw.h index 8ce11c3e6d5c2..a9888e0521a1d 100644 --- a/drivers/net/wireless/ath/ath12k/hw.h +++ b/drivers/net/wireless/ath/ath12k/hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_HW_H @@ -78,6 +78,7 @@ #define ATH12K_DEFAULT_CAL_FILE "caldata.bin" #define ATH12K_AMSS_FILE "amss.bin" #define ATH12K_M3_FILE "m3.bin" +#define ATH12K_AUX_UC_FILE "aux_ucode.bin" #define ATH12K_REGDB_FILE_NAME "regdb.bin" #define ATH12K_PCIE_MAX_PAYLOAD_SIZE 128 @@ -128,11 +129,6 @@ struct ath12k_hw_ring_mask { u8 tx_mon_dest[ATH12K_EXT_IRQ_GRP_NUM_MAX]; }; -struct ath12k_hw_hal_params { - enum hal_rx_buf_return_buf_manager rx_buf_rbm; - u32 wbm2sw_cc_enable; -}; - enum ath12k_m3_fw_loaders { ath12k_m3_fw_loader_driver, ath12k_m3_fw_loader_remoteproc, @@ -147,6 +143,7 @@ struct ath12k_hw_params { size_t board_size; size_t cal_offset; enum ath12k_m3_fw_loaders m3_loader; + bool download_aux_ucode:1; } fw; u8 max_radios; @@ -156,7 +153,6 @@ struct ath12k_hw_params { const struct ath12k_hw_ops *hw_ops; const struct ath12k_hw_ring_mask *ring_mask; - const struct ath12k_hw_regs *regs; const struct ce_attr *host_ce_config; u32 ce_count; @@ -165,8 +161,6 @@ struct ath12k_hw_params { const struct service_to_pipe *svc_to_ce_map; u32 svc_to_ce_map_len; - const struct ath12k_hw_hal_params *hal_params; - bool rxdma1_enable:1; int num_rxdma_per_pdev; int num_rxdma_dst_ring; @@ -193,8 +187,6 @@ struct ath12k_hw_params { void (*wmi_init)(struct ath12k_base *ab, struct ath12k_wmi_resource_config_arg *config); - const struct hal_ops *hal_ops; - u64 qmi_cnss_feature_bitmap; u32 rfkill_pin; @@ -285,86 +277,6 @@ enum ath12k_bd_ie_type { ATH12K_BD_IE_REGDB = 1, }; -struct ath12k_hw_regs { - u32 hal_tcl1_ring_id; - u32 hal_tcl1_ring_misc; - u32 hal_tcl1_ring_tp_addr_lsb; - u32 hal_tcl1_ring_tp_addr_msb; - u32 hal_tcl1_ring_consumer_int_setup_ix0; - u32 hal_tcl1_ring_consumer_int_setup_ix1; - u32 hal_tcl1_ring_msi1_base_lsb; - u32 hal_tcl1_ring_msi1_base_msb; - u32 hal_tcl1_ring_msi1_data; - u32 hal_tcl_ring_base_lsb; - u32 hal_tcl1_ring_base_lsb; - u32 hal_tcl1_ring_base_msb; - u32 hal_tcl2_ring_base_lsb; - - u32 hal_tcl_status_ring_base_lsb; - - u32 hal_reo1_qdesc_addr; - u32 hal_reo1_qdesc_max_peerid; - - u32 hal_wbm_idle_ring_base_lsb; - u32 hal_wbm_idle_ring_misc_addr; - u32 hal_wbm_r0_idle_list_cntl_addr; - u32 hal_wbm_r0_idle_list_size_addr; - u32 hal_wbm_scattered_ring_base_lsb; - u32 hal_wbm_scattered_ring_base_msb; - u32 hal_wbm_scattered_desc_head_info_ix0; - u32 hal_wbm_scattered_desc_head_info_ix1; - u32 hal_wbm_scattered_desc_tail_info_ix0; - u32 hal_wbm_scattered_desc_tail_info_ix1; - u32 hal_wbm_scattered_desc_ptr_hp_addr; - - u32 hal_wbm_sw_release_ring_base_lsb; - u32 hal_wbm_sw1_release_ring_base_lsb; - u32 hal_wbm0_release_ring_base_lsb; - u32 hal_wbm1_release_ring_base_lsb; - - u32 pcie_qserdes_sysclk_en_sel; - u32 pcie_pcs_osc_dtct_config_base; - - u32 hal_umac_ce0_src_reg_base; - u32 hal_umac_ce0_dest_reg_base; - u32 hal_umac_ce1_src_reg_base; - u32 hal_umac_ce1_dest_reg_base; - - u32 hal_ppe_rel_ring_base; - - u32 hal_reo2_ring_base; - u32 hal_reo1_misc_ctrl_addr; - u32 hal_reo1_sw_cookie_cfg0; - u32 hal_reo1_sw_cookie_cfg1; - u32 hal_reo1_qdesc_lut_base0; - u32 hal_reo1_qdesc_lut_base1; - u32 hal_reo1_ring_base_lsb; - u32 hal_reo1_ring_base_msb; - u32 hal_reo1_ring_id; - u32 hal_reo1_ring_misc; - u32 hal_reo1_ring_hp_addr_lsb; - u32 hal_reo1_ring_hp_addr_msb; - u32 hal_reo1_ring_producer_int_setup; - u32 hal_reo1_ring_msi1_base_lsb; - u32 hal_reo1_ring_msi1_base_msb; - u32 hal_reo1_ring_msi1_data; - u32 hal_reo1_aging_thres_ix0; - u32 hal_reo1_aging_thres_ix1; - u32 hal_reo1_aging_thres_ix2; - u32 hal_reo1_aging_thres_ix3; - - u32 hal_reo2_sw0_ring_base; - - u32 hal_sw2reo_ring_base; - u32 hal_sw2reo1_ring_base; - - u32 hal_reo_cmd_ring_base; - - u32 hal_reo_status_ring_base; - - u32 gcc_gcc_pcie_hot_rst; -}; - static inline const char *ath12k_bd_ie_type_str(enum ath12k_bd_ie_type type) { switch (type) { @@ -377,6 +289,4 @@ static inline const char *ath12k_bd_ie_type_str(enum ath12k_bd_ie_type type) return "unknown"; } -int ath12k_hw_init(struct ath12k_base *ab); - #endif diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index db351c9220181..9cffab4c1920a 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -21,6 +21,8 @@ #include "hif.h" #include "wow.h" #include "debugfs_sta.h" +#include "dp.h" +#include "dp_cmn.h" #define CHAN2G(_channel, _freq, _flags) { \ .band = NL80211_BAND_2GHZ, \ @@ -361,6 +363,7 @@ u16 ath12k_mac_he_convert_tones_to_ru_tones(u16 tones) return RU_26; } } +EXPORT_SYMBOL(ath12k_mac_he_convert_tones_to_ru_tones); enum nl80211_eht_gi ath12k_mac_eht_gi_to_nl80211_eht_gi(u8 sgi) { @@ -375,6 +378,7 @@ enum nl80211_eht_gi ath12k_mac_eht_gi_to_nl80211_eht_gi(u8 sgi) return NL80211_RATE_INFO_EHT_GI_0_8; } } +EXPORT_SYMBOL(ath12k_mac_eht_gi_to_nl80211_eht_gi); enum nl80211_eht_ru_alloc ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(u16 ru_tones) { @@ -415,6 +419,7 @@ enum nl80211_eht_ru_alloc ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(u16 ru return NL80211_RATE_INFO_EHT_RU_ALLOC_26; } } +EXPORT_SYMBOL(ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc); enum rate_info_bw ath12k_mac_bw_to_mac80211_bw(enum ath12k_supported_bw bw) @@ -441,6 +446,7 @@ ath12k_mac_bw_to_mac80211_bw(enum ath12k_supported_bw bw) return ret; } +EXPORT_SYMBOL(ath12k_mac_bw_to_mac80211_bw); enum ath12k_supported_bw ath12k_mac_mac80211_bw_to_ath12k_bw(enum rate_info_bw bw) { @@ -484,6 +490,7 @@ int ath12k_mac_hw_ratecode_to_legacy_rate(u8 hw_rc, u8 preamble, u8 *rateidx, return -EINVAL; } +EXPORT_SYMBOL(ath12k_mac_hw_ratecode_to_legacy_rate); u8 ath12k_mac_bitrate_to_idx(const struct ieee80211_supported_band *sband, u32 bitrate) @@ -533,6 +540,30 @@ ath12k_mac_max_he_nss(const u16 he_mcs_mask[NL80211_HE_NSS_MAX]) return 1; } +static u32 +ath12k_mac_max_eht_nss(const u16 eht_mcs_mask[NL80211_EHT_NSS_MAX]) +{ + int nss; + + for (nss = NL80211_EHT_NSS_MAX - 1; nss >= 0; nss--) + if (eht_mcs_mask[nss]) + return nss + 1; + + return 1; +} + +static u32 +ath12k_mac_max_eht_mcs_nss(const u8 *eht_mcs, int eht_mcs_set_size) +{ + int i; + u8 nss = 0; + + for (i = 0; i < eht_mcs_set_size; i++) + nss = max(nss, u8_get_bits(eht_mcs[i], IEEE80211_EHT_MCS_NSS_RX)); + + return nss; +} + static u8 ath12k_parse_mpdudensity(u8 mpdudensity) { /* From IEEE Std 802.11-2020 defined values for "Minimum MPDU Start Spacing": @@ -1143,26 +1174,119 @@ static int ath12k_mac_set_kickout(struct ath12k_link_vif *arvif) return 0; } +static void ath12k_mac_link_sta_rhash_cleanup(void *data, struct ieee80211_sta *sta) +{ + u8 link_id; + unsigned long links_map; + struct ath12k_sta *ahsta; + struct ath12k *ar = data; + struct ath12k_link_sta *arsta; + struct ath12k_link_vif *arvif; + struct ath12k_base *ab = ar->ab; + + ahsta = ath12k_sta_to_ahsta(sta); + links_map = ahsta->links_map; + + rcu_read_lock(); + for_each_set_bit(link_id, &links_map, IEEE80211_MLD_MAX_NUM_LINKS) { + arsta = rcu_dereference(ahsta->link[link_id]); + if (!arsta) + continue; + arvif = arsta->arvif; + if (!(arvif->ar == ar)) + continue; + + spin_lock_bh(&ab->base_lock); + ath12k_link_sta_rhash_delete(ab, arsta); + spin_unlock_bh(&ab->base_lock); + } + rcu_read_unlock(); +} + void ath12k_mac_peer_cleanup_all(struct ath12k *ar) { - struct ath12k_peer *peer, *tmp; + struct ath12k_dp_link_peer *peer, *tmp; struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_link_vif *arvif, *tmp_vif; + struct ath12k_dp_hw *dp_hw = &ar->ah->dp_hw; + struct ath12k_dp_peer *dp_peer = NULL; + u16 peerid_index; + struct list_head peers; + + INIT_LIST_HEAD(&peers); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); - spin_lock_bh(&ab->base_lock); - list_for_each_entry_safe(peer, tmp, &ab->peers, list) { + spin_lock_bh(&dp->dp_lock); + list_for_each_entry_safe(peer, tmp, &dp->peers, list) { /* Skip Rx TID cleanup for self peer */ - if (peer->sta) + if (peer->sta && peer->dp_peer) ath12k_dp_rx_peer_tid_cleanup(ar, peer); - list_del(&peer->list); - kfree(peer); + /* cleanup dp peer */ + spin_lock_bh(&dp_hw->peer_lock); + dp_peer = peer->dp_peer; + peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id); + rcu_assign_pointer(dp_peer->link_peers[peer->link_id], NULL); + rcu_assign_pointer(dp_hw->dp_peers[peerid_index], NULL); + spin_unlock_bh(&dp_hw->peer_lock); + + ath12k_dp_link_peer_rhash_delete(dp, peer); + + list_move(&peer->list, &peers); + } + spin_unlock_bh(&dp->dp_lock); + + synchronize_rcu(); + + list_for_each_entry_safe(peer, tmp, &peers, list) { + ath12k_dp_link_peer_free(peer); } - spin_unlock_bh(&ab->base_lock); ar->num_peers = 0; ar->num_stations = 0; + + /* Cleanup rhash table maintained for arsta by iterating over sta */ + ieee80211_iterate_stations_mtx(ar->ah->hw, ath12k_mac_link_sta_rhash_cleanup, + ar); + + /* Delete all the self dp_peers on asserted radio */ + list_for_each_entry_safe_reverse(arvif, tmp_vif, &ar->arvifs, list) { + if ((arvif->ahvif->vdev_type == WMI_VDEV_TYPE_AP) && + (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS)) { + ath12k_dp_peer_delete(dp_hw, arvif->bssid, NULL); + arvif->num_stations = 0; + } + } +} + +void ath12k_mac_dp_peer_cleanup(struct ath12k_hw *ah) +{ + struct list_head peers; + struct ath12k_dp_peer *dp_peer, *tmp; + struct ath12k_dp_hw *dp_hw = &ah->dp_hw; + + INIT_LIST_HEAD(&peers); + + spin_lock_bh(&dp_hw->peer_lock); + list_for_each_entry_safe(dp_peer, tmp, &dp_hw->dp_peers_list, list) { + if (dp_peer->is_mlo) { + rcu_assign_pointer(dp_hw->dp_peers[dp_peer->peer_id], NULL); + clear_bit(dp_peer->peer_id, ah->free_ml_peer_id_map); + } + + list_move(&dp_peer->list, &peers); + } + + spin_unlock_bh(&dp_hw->peer_lock); + + synchronize_rcu(); + + list_for_each_entry_safe(dp_peer, tmp, &peers, list) { + list_del(&dp_peer->list); + kfree(dp_peer); + } } static int ath12k_mac_vdev_setup_sync(struct ath12k *ar) @@ -1434,10 +1558,11 @@ int ath12k_mac_vdev_stop(struct ath12k_link_vif *arvif) return ret; } -static int ath12k_mac_op_config(struct ieee80211_hw *hw, int radio_idx, u32 changed) +int ath12k_mac_op_config(struct ieee80211_hw *hw, int radio_idx, u32 changed) { return 0; } +EXPORT_SYMBOL(ath12k_mac_op_config); static int ath12k_mac_setup_bcn_p2p_ie(struct ath12k_link_vif *arvif, struct sk_buff *bcn) @@ -2249,7 +2374,6 @@ static void ath12k_peer_assoc_h_vht(struct ath12k *ar, struct cfg80211_chan_def def; enum nl80211_band band; u16 *vht_mcs_mask; - u16 tx_mcs_map; u8 ampdu_factor; u8 max_nss, vht_mcs; int i, vht_nss, nss_idx; @@ -2340,10 +2464,10 @@ static void ath12k_peer_assoc_h_vht(struct ath12k *ar, arg->peer_nss = min(link_sta->rx_nss, max_nss); arg->rx_max_rate = __le16_to_cpu(vht_cap->vht_mcs.rx_highest); arg->rx_mcs_set = __le16_to_cpu(vht_cap->vht_mcs.rx_mcs_map); - arg->tx_max_rate = __le16_to_cpu(vht_cap->vht_mcs.tx_highest); + arg->rx_mcs_set = ath12k_peer_assoc_h_vht_limit(arg->rx_mcs_set, vht_mcs_mask); - tx_mcs_map = __le16_to_cpu(vht_cap->vht_mcs.tx_mcs_map); - arg->tx_mcs_set = ath12k_peer_assoc_h_vht_limit(tx_mcs_map, vht_mcs_mask); + arg->tx_max_rate = __le16_to_cpu(vht_cap->vht_mcs.tx_highest); + arg->tx_mcs_set = __le16_to_cpu(vht_cap->vht_mcs.tx_mcs_map); /* In QCN9274 platform, VHT MCS rate 10 and 11 is enabled by default. * VHT MCS rate 10 and 11 is not supported in 11ac standard. @@ -2625,9 +2749,10 @@ static void ath12k_peer_assoc_h_he(struct ath12k *ar, switch (link_sta->bandwidth) { case IEEE80211_STA_RX_BW_160: v = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160); + v = ath12k_peer_assoc_h_he_limit(v, he_mcs_mask); arg->peer_he_rx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_160] = v; - v = ath12k_peer_assoc_h_he_limit(v, he_mcs_mask); + v = le16_to_cpu(he_cap->he_mcs_nss_supp.tx_mcs_160); arg->peer_he_tx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_160] = v; arg->peer_he_mcs_count++; @@ -2637,10 +2762,10 @@ static void ath12k_peer_assoc_h_he(struct ath12k *ar, default: v = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80); + v = ath12k_peer_assoc_h_he_limit(v, he_mcs_mask); arg->peer_he_rx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_80] = v; v = le16_to_cpu(he_cap->he_mcs_nss_supp.tx_mcs_80); - v = ath12k_peer_assoc_h_he_limit(v, he_mcs_mask); arg->peer_he_tx_mcs_set[WMI_HECAP_TXRX_MCS_NSS_IDX_80] = v; arg->peer_he_mcs_count++; @@ -3004,6 +3129,18 @@ static enum wmi_phy_mode ath12k_mac_get_phymode_eht(struct ath12k *ar, return MODE_UNKNOWN; } +static bool +ath12k_peer_assoc_h_eht_masked(const u16 eht_mcs_mask[NL80211_EHT_NSS_MAX]) +{ + int nss; + + for (nss = 0; nss < NL80211_EHT_NSS_MAX; nss++) + if (eht_mcs_mask[nss]) + return false; + + return true; +} + static void ath12k_peer_assoc_h_phymode(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ath12k_link_sta *arsta, @@ -3015,6 +3152,7 @@ static void ath12k_peer_assoc_h_phymode(struct ath12k *ar, const u8 *ht_mcs_mask; const u16 *vht_mcs_mask; const u16 *he_mcs_mask; + const u16 *eht_mcs_mask; enum wmi_phy_mode phymode = MODE_UNKNOWN; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -3029,6 +3167,7 @@ static void ath12k_peer_assoc_h_phymode(struct ath12k *ar, ht_mcs_mask = arvif->bitrate_mask.control[band].ht_mcs; vht_mcs_mask = arvif->bitrate_mask.control[band].vht_mcs; he_mcs_mask = arvif->bitrate_mask.control[band].he_mcs; + eht_mcs_mask = arvif->bitrate_mask.control[band].eht_mcs; link_sta = ath12k_mac_get_link_sta(arsta); if (!link_sta) { @@ -3039,7 +3178,8 @@ static void ath12k_peer_assoc_h_phymode(struct ath12k *ar, switch (band) { case NL80211_BAND_2GHZ: - if (link_sta->eht_cap.has_eht) { + if (link_sta->eht_cap.has_eht && + !ath12k_peer_assoc_h_eht_masked(eht_mcs_mask)) { if (link_sta->bandwidth == IEEE80211_STA_RX_BW_40) phymode = MODE_11BE_EHT40_2G; else @@ -3102,37 +3242,50 @@ static void ath12k_peer_assoc_h_phymode(struct ath12k *ar, WARN_ON(phymode == MODE_UNKNOWN); } +#define ATH12K_EHT_MCS_7_ENABLED 0x00FF +#define ATH12K_EHT_MCS_9_ENABLED 0x0300 +#define ATH12K_EHT_MCS_11_ENABLED 0x0C00 +#define ATH12K_EHT_MCS_13_ENABLED 0x3000 + static void ath12k_mac_set_eht_mcs(u8 rx_tx_mcs7, u8 rx_tx_mcs9, u8 rx_tx_mcs11, u8 rx_tx_mcs13, - u32 *rx_mcs, u32 *tx_mcs) -{ - *rx_mcs = 0; - u32p_replace_bits(rx_mcs, - u8_get_bits(rx_tx_mcs7, IEEE80211_EHT_MCS_NSS_RX), - WMI_EHT_MCS_NSS_0_7); - u32p_replace_bits(rx_mcs, - u8_get_bits(rx_tx_mcs9, IEEE80211_EHT_MCS_NSS_RX), - WMI_EHT_MCS_NSS_8_9); - u32p_replace_bits(rx_mcs, - u8_get_bits(rx_tx_mcs11, IEEE80211_EHT_MCS_NSS_RX), - WMI_EHT_MCS_NSS_10_11); - u32p_replace_bits(rx_mcs, - u8_get_bits(rx_tx_mcs13, IEEE80211_EHT_MCS_NSS_RX), - WMI_EHT_MCS_NSS_12_13); - - *tx_mcs = 0; - u32p_replace_bits(tx_mcs, - u8_get_bits(rx_tx_mcs7, IEEE80211_EHT_MCS_NSS_TX), - WMI_EHT_MCS_NSS_0_7); - u32p_replace_bits(tx_mcs, - u8_get_bits(rx_tx_mcs9, IEEE80211_EHT_MCS_NSS_TX), - WMI_EHT_MCS_NSS_8_9); - u32p_replace_bits(tx_mcs, - u8_get_bits(rx_tx_mcs11, IEEE80211_EHT_MCS_NSS_TX), - WMI_EHT_MCS_NSS_10_11); - u32p_replace_bits(tx_mcs, - u8_get_bits(rx_tx_mcs13, IEEE80211_EHT_MCS_NSS_TX), - WMI_EHT_MCS_NSS_12_13); + u32 *rx_mcs, u32 *tx_mcs, + const u16 eht_mcs_limit[NL80211_EHT_NSS_MAX]) +{ + int nss; + u8 mcs_7 = 0, mcs_9 = 0, mcs_11 = 0, mcs_13 = 0; + u8 peer_mcs_7, peer_mcs_9, peer_mcs_11, peer_mcs_13; + + for (nss = 0; nss < NL80211_EHT_NSS_MAX; nss++) { + if (eht_mcs_limit[nss] & ATH12K_EHT_MCS_7_ENABLED) + mcs_7++; + if (eht_mcs_limit[nss] & ATH12K_EHT_MCS_9_ENABLED) + mcs_9++; + if (eht_mcs_limit[nss] & ATH12K_EHT_MCS_11_ENABLED) + mcs_11++; + if (eht_mcs_limit[nss] & ATH12K_EHT_MCS_13_ENABLED) + mcs_13++; + } + + peer_mcs_7 = u8_get_bits(rx_tx_mcs7, IEEE80211_EHT_MCS_NSS_RX); + peer_mcs_9 = u8_get_bits(rx_tx_mcs9, IEEE80211_EHT_MCS_NSS_RX); + peer_mcs_11 = u8_get_bits(rx_tx_mcs11, IEEE80211_EHT_MCS_NSS_RX); + peer_mcs_13 = u8_get_bits(rx_tx_mcs13, IEEE80211_EHT_MCS_NSS_RX); + + *rx_mcs = u32_encode_bits(min(peer_mcs_7, mcs_7), WMI_EHT_MCS_NSS_0_7) | + u32_encode_bits(min(peer_mcs_9, mcs_9), WMI_EHT_MCS_NSS_8_9) | + u32_encode_bits(min(peer_mcs_11, mcs_11), WMI_EHT_MCS_NSS_10_11) | + u32_encode_bits(min(peer_mcs_13, mcs_13), WMI_EHT_MCS_NSS_12_13); + + peer_mcs_7 = u8_get_bits(rx_tx_mcs7, IEEE80211_EHT_MCS_NSS_TX); + peer_mcs_9 = u8_get_bits(rx_tx_mcs9, IEEE80211_EHT_MCS_NSS_TX); + peer_mcs_11 = u8_get_bits(rx_tx_mcs11, IEEE80211_EHT_MCS_NSS_TX); + peer_mcs_13 = u8_get_bits(rx_tx_mcs13, IEEE80211_EHT_MCS_NSS_TX); + + *tx_mcs = u32_encode_bits(min(peer_mcs_7, mcs_7), WMI_EHT_MCS_NSS_0_7) | + u32_encode_bits(min(peer_mcs_9, mcs_9), WMI_EHT_MCS_NSS_8_9) | + u32_encode_bits(min(peer_mcs_11, mcs_11), WMI_EHT_MCS_NSS_10_11) | + u32_encode_bits(min(peer_mcs_13, mcs_13), WMI_EHT_MCS_NSS_12_13); } static void ath12k_mac_set_eht_ppe_threshold(const u8 *ppe_thres, @@ -3171,13 +3324,22 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, struct ath12k_wmi_peer_assoc_arg *arg) { struct ieee80211_sta *sta = ath12k_ahsta_to_sta(arsta->ahsta); + struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif); + const struct ieee80211_eht_mcs_nss_supp *own_eht_mcs_nss_supp; const struct ieee80211_eht_mcs_nss_supp_20mhz_only *bw_20; + const struct ieee80211_sta_eht_cap *eht_cap, *own_eht_cap; + const struct ieee80211_sband_iftype_data *iftd; const struct ieee80211_eht_mcs_nss_supp_bw *bw; - const struct ieee80211_sta_eht_cap *eht_cap; const struct ieee80211_sta_he_cap *he_cap; struct ieee80211_link_sta *link_sta; struct ieee80211_bss_conf *link_conf; + struct cfg80211_chan_def def; + bool user_rate_valid = true; + enum nl80211_band band; + int eht_nss, nss_idx; u32 *rx_mcs, *tx_mcs; + u16 *eht_mcs_mask; + u8 max_nss = 0; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -3199,6 +3361,22 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, if (!he_cap->has_he || !eht_cap->has_eht) return; + if (WARN_ON(ath12k_mac_vif_link_chan(vif, arvif->link_id, &def))) + return; + + band = def.chan->band; + eht_mcs_mask = arvif->bitrate_mask.control[band].eht_mcs; + + iftd = ieee80211_get_sband_iftype_data(&ar->mac.sbands[band], vif->type); + if (!iftd) { + ath12k_warn(ar->ab, + "unable to access iftype_data in struct ieee80211_supported_band\n"); + return; + } + + own_eht_cap = &iftd->eht_cap; + own_eht_mcs_nss_supp = &own_eht_cap->eht_mcs_nss_supp; + arg->eht_flag = true; if ((eht_cap->eht_cap_elem.phy_cap_info[5] & @@ -3215,6 +3393,28 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, rx_mcs = arg->peer_eht_rx_mcs_set; tx_mcs = arg->peer_eht_tx_mcs_set; + eht_nss = ath12k_mac_max_eht_mcs_nss((void *)own_eht_mcs_nss_supp, + sizeof(*own_eht_mcs_nss_supp)); + if (eht_nss > link_sta->rx_nss) { + user_rate_valid = false; + for (nss_idx = (link_sta->rx_nss - 1); nss_idx >= 0; nss_idx--) { + if (eht_mcs_mask[nss_idx]) { + user_rate_valid = true; + break; + } + } + } + + if (!user_rate_valid) { + ath12k_dbg(ar->ab, ATH12K_DBG_MAC, + "Setting eht range MCS value to peer supported nss %d for peer %pM\n", + link_sta->rx_nss, arsta->addr); + eht_mcs_mask[link_sta->rx_nss - 1] = eht_mcs_mask[eht_nss - 1]; + } + + bw_20 = &eht_cap->eht_mcs_nss_supp.only_20mhz; + bw = &eht_cap->eht_mcs_nss_supp.bw._80; + switch (link_sta->bandwidth) { case IEEE80211_STA_RX_BW_320: bw = &eht_cap->eht_mcs_nss_supp.bw._320; @@ -3223,7 +3423,8 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, bw->rx_tx_mcs11_max_nss, bw->rx_tx_mcs13_max_nss, &rx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_320], - &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_320]); + &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_320], + eht_mcs_mask); arg->peer_eht_mcs_count++; fallthrough; case IEEE80211_STA_RX_BW_160: @@ -3233,15 +3434,13 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, bw->rx_tx_mcs11_max_nss, bw->rx_tx_mcs13_max_nss, &rx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_160], - &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_160]); + &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_160], + eht_mcs_mask); arg->peer_eht_mcs_count++; fallthrough; default: - if ((he_cap->he_cap_elem.phy_cap_info[0] & - (IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G)) == 0) { + if (!(link_sta->he_cap.he_cap_elem.phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) { bw_20 = &eht_cap->eht_mcs_nss_supp.only_20mhz; ath12k_mac_set_eht_mcs(bw_20->rx_tx_mcs7_max_nss, @@ -3249,7 +3448,8 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, bw_20->rx_tx_mcs11_max_nss, bw_20->rx_tx_mcs13_max_nss, &rx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_80], - &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_80]); + &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_80], + eht_mcs_mask); } else { bw = &eht_cap->eht_mcs_nss_supp.bw._80; ath12k_mac_set_eht_mcs(bw->rx_tx_mcs9_max_nss, @@ -3257,7 +3457,8 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, bw->rx_tx_mcs11_max_nss, bw->rx_tx_mcs13_max_nss, &rx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_80], - &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_80]); + &tx_mcs[WMI_EHTCAP_TXRX_MCS_NSS_IDX_80], + eht_mcs_mask); } arg->peer_eht_mcs_count++; @@ -3266,6 +3467,41 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar, arg->punct_bitmap = ~arvif->punct_bitmap; arg->eht_disable_mcs15 = link_conf->eht_disable_mcs15; + + if (!(link_sta->he_cap.he_cap_elem.phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) { + if (bw_20->rx_tx_mcs13_max_nss) + max_nss = max(max_nss, u8_get_bits(bw_20->rx_tx_mcs13_max_nss, + IEEE80211_EHT_MCS_NSS_RX)); + if (bw_20->rx_tx_mcs11_max_nss) + max_nss = max(max_nss, u8_get_bits(bw_20->rx_tx_mcs11_max_nss, + IEEE80211_EHT_MCS_NSS_RX)); + if (bw_20->rx_tx_mcs9_max_nss) + max_nss = max(max_nss, u8_get_bits(bw_20->rx_tx_mcs9_max_nss, + IEEE80211_EHT_MCS_NSS_RX)); + if (bw_20->rx_tx_mcs7_max_nss) + max_nss = max(max_nss, u8_get_bits(bw_20->rx_tx_mcs7_max_nss, + IEEE80211_EHT_MCS_NSS_RX)); + } else { + if (bw->rx_tx_mcs13_max_nss) + max_nss = max(max_nss, u8_get_bits(bw->rx_tx_mcs13_max_nss, + IEEE80211_EHT_MCS_NSS_RX)); + if (bw->rx_tx_mcs11_max_nss) + max_nss = max(max_nss, u8_get_bits(bw->rx_tx_mcs11_max_nss, + IEEE80211_EHT_MCS_NSS_RX)); + if (bw->rx_tx_mcs9_max_nss) + max_nss = max(max_nss, u8_get_bits(bw->rx_tx_mcs9_max_nss, + IEEE80211_EHT_MCS_NSS_RX)); + } + + max_nss = min(max_nss, (uint8_t)eht_nss); + + arg->peer_nss = min(link_sta->rx_nss, max_nss); + + ath12k_dbg(ar->ab, ATH12K_DBG_MAC, + "mac eht peer %pM nss %d mcs cnt %d ru_punct_bitmap 0x%x\n", + arsta->addr, arg->peer_nss, arg->peer_eht_mcs_count, + arg->punct_bitmap); } static void ath12k_peer_assoc_h_mlo(struct ath12k_link_sta *arsta, @@ -3593,10 +3829,11 @@ static void ath12k_bss_assoc(struct ath12k *ar, struct ath12k_link_sta *arsta; struct ieee80211_sta *ap_sta; struct ath12k_sta *ahsta; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; bool is_auth = false; u32 hemode = 0; int ret; + struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -3708,13 +3945,14 @@ static void ath12k_bss_assoc(struct ath12k *ar, "mac vdev %d up (associated) bssid %pM aid %d\n", arvif->vdev_id, bss_conf->bssid, vif->cfg.aid); - spin_lock_bh(&ar->ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arvif->bssid); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + arvif->bssid); if (peer && peer->is_authorized) is_auth = true; - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); /* Authorize BSS Peer */ if (is_auth) { @@ -3834,6 +4072,38 @@ static void ath12k_recalculate_mgmt_rate(struct ath12k *ar, ath12k_warn(ar->ab, "failed to set beacon tx rate %d\n", ret); } +static void ath12k_mac_bcn_tx_event(struct ath12k_link_vif *arvif) +{ + struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif); + struct ieee80211_bss_conf *link_conf; + + link_conf = ath12k_mac_get_link_bss_conf(arvif); + if (!link_conf) { + ath12k_warn(arvif->ar->ab, "failed to get link conf for vdev %u\n", + arvif->vdev_id); + return; + } + + if (link_conf->color_change_active) { + if (ieee80211_beacon_cntdwn_is_complete(vif, arvif->link_id)) { + ieee80211_color_change_finish(vif, arvif->link_id); + return; + } + + ieee80211_beacon_update_cntdwn(vif, arvif->link_id); + ath12k_mac_setup_bcn_tmpl(arvif); + } +} + +static void ath12k_mac_bcn_tx_work(struct wiphy *wiphy, struct wiphy_work *work) +{ + struct ath12k_link_vif *arvif = container_of(work, struct ath12k_link_vif, + bcn_tx_work); + + lockdep_assert_wiphy(wiphy); + ath12k_mac_bcn_tx_event(arvif); +} + static void ath12k_mac_init_arvif(struct ath12k_vif *ahvif, struct ath12k_link_vif *arvif, int link_id) { @@ -3863,6 +4133,7 @@ static void ath12k_mac_init_arvif(struct ath12k_vif *ahvif, INIT_LIST_HEAD(&arvif->list); INIT_DELAYED_WORK(&arvif->connection_loss_work, ath12k_mac_vif_sta_connection_loss_work); + wiphy_work_init(&arvif->bcn_tx_work, ath12k_mac_bcn_tx_work); arvif->num_stations = 0; @@ -3875,6 +4146,8 @@ static void ath12k_mac_init_arvif(struct ath12k_vif *ahvif, sizeof(arvif->bitrate_mask.control[i].vht_mcs)); memset(arvif->bitrate_mask.control[i].he_mcs, 0xff, sizeof(arvif->bitrate_mask.control[i].he_mcs)); + memset(arvif->bitrate_mask.control[i].eht_mcs, 0xff, + sizeof(arvif->bitrate_mask.control[i].eht_mcs)); } /* Handle MLO related assignments */ @@ -3900,6 +4173,7 @@ static void ath12k_mac_remove_link_interface(struct ieee80211_hw *hw, lockdep_assert_wiphy(ah->hw->wiphy); cancel_delayed_work_sync(&arvif->connection_loss_work); + wiphy_work_cancel(ath12k_ar_to_hw(ar)->wiphy, &arvif->bcn_tx_work); ath12k_dbg(ar->ab, ATH12K_DBG_MAC, "mac remove link interface (vdev %d link id %d)", arvif->vdev_id, arvif->link_id); @@ -3914,6 +4188,9 @@ static void ath12k_mac_remove_link_interface(struct ieee80211_hw *hw, if (ret) ath12k_warn(ar->ab, "failed to submit AP self-peer removal on vdev %d link id %d: %d", arvif->vdev_id, arvif->link_id, ret); + + if (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS) + ath12k_dp_peer_delete(&ah->dp_hw, arvif->bssid, NULL); } ath12k_mac_vdev_delete(ar, arvif); } @@ -3967,7 +4244,7 @@ static void ath12k_mac_unassign_link_vif(struct ath12k_link_vif *arvif) memset(arvif, 0, sizeof(*arvif)); } -static int +int ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, u16 old_links, u16 new_links, @@ -4004,8 +4281,10 @@ ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw, if (WARN_ON(!arvif)) return -EINVAL; - if (!arvif->is_created) + if (!arvif->is_created) { + ath12k_mac_unassign_link_vif(arvif); continue; + } if (WARN_ON(!arvif->ar)) return -EINVAL; @@ -4016,6 +4295,7 @@ ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_change_vif_links); static int ath12k_mac_fils_discovery(struct ath12k_link_vif *arvif, struct ieee80211_bss_conf *info) @@ -4064,9 +4344,9 @@ static int ath12k_mac_fils_discovery(struct ath12k_link_vif *arvif, return ret; } -static void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - u64 changed) +void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + u64 changed) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); unsigned long links = ahvif->links_map; @@ -4134,6 +4414,7 @@ static void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw, } } } +EXPORT_SYMBOL(ath12k_mac_op_vif_cfg_changed); static void ath12k_mac_vif_setup_ps(struct ath12k_link_vif *arvif) { @@ -4202,6 +4483,190 @@ static bool ath12k_mac_supports_tpc(struct ath12k *ar, struct ath12k_vif *ahvif, chandef->chan->band == NL80211_BAND_6GHZ; } +static void ath12k_wmi_vdev_params_up(struct ath12k *ar, + struct ath12k_link_vif *arvif, + struct ath12k_link_vif *tx_arvif, + struct ieee80211_bss_conf *info, u16 aid) +{ + struct ath12k_wmi_vdev_up_params params = { + .vdev_id = arvif->vdev_id, + .aid = aid, + .bssid = arvif->bssid + }; + int ret; + + if (tx_arvif) { + params.tx_bssid = tx_arvif->bssid; + params.nontx_profile_idx = info->bssid_index; + params.nontx_profile_cnt = 1 << info->bssid_indicator; + } + + ret = ath12k_wmi_vdev_up(arvif->ar, ¶ms); + if (ret) + ath12k_warn(ar->ab, "failed to bring vdev up %d: %d\n", + arvif->vdev_id, ret); +} + +static int ath12k_mac_config_obss_pd(struct ath12k_link_vif *arvif, + const struct ieee80211_he_obss_pd *he_obss_pd) +{ + struct ath12k_wmi_obss_pd_arg obss_pd_arg = {}; + u32 srg_bitmap[2], non_srg_bitmap[2]; + struct ath12k *ar = arvif->ar; + u32 param_id, pdev_id; + u32 param_val; + int ret; + + if (ar->ab->hw_params->single_pdev_only) + pdev_id = ath12k_mac_get_target_pdev_id_from_vif(arvif); + else + pdev_id = ar->pdev->pdev_id; + + /* Set and enable SRG/non-SRG OBSS PD threshold */ + param_id = WMI_PDEV_PARAM_SET_CMD_OBSS_PD_THRESHOLD; + if (ar->monitor_started || !he_obss_pd->enable) { + ret = ath12k_wmi_pdev_set_param(ar, param_id, 0, pdev_id); + if (ret) + ath12k_warn(ar->ab, + "failed to set OBSS PD threshold for pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + /* + * This service flag indicates firmware support for SRG/SRP-based + * spatial reuse. It also specifies whether OBSS PD threshold values + * should be interpreted as dB (offset) or dBm (absolute) units. + */ + obss_pd_arg.srp_support = test_bit(WMI_TLV_SERVICE_SRG_SRP_SPATIAL_REUSE_SUPPORT, + ar->ab->wmi_ab.svc_map); + + if (!(he_obss_pd->sr_ctrl & + IEEE80211_HE_SPR_NON_SRG_OBSS_PD_SR_DISALLOWED)) { + if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT) + obss_pd_arg.non_srg_th = ATH12K_OBSS_PD_MAX_THRESHOLD + + he_obss_pd->non_srg_max_offset; + else + obss_pd_arg.non_srg_th = ATH12K_OBSS_PD_NON_SRG_MAX_THRESHOLD; + + if (!obss_pd_arg.srp_support) + obss_pd_arg.non_srg_th -= ATH12K_DEFAULT_NOISE_FLOOR; + + obss_pd_arg.non_srg_enabled = true; + } + + if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT) { + obss_pd_arg.srg_th = ATH12K_OBSS_PD_MAX_THRESHOLD + + he_obss_pd->max_offset; + obss_pd_arg.srg_enabled = true; + } + + ath12k_dbg(ar->ab, ATH12K_DBG_MAC, + "pdev %u OBSS PD sr_ctrl 0x%x srg_th %d dBm non_srg_th %d dBm\n", + pdev_id, he_obss_pd->sr_ctrl, + obss_pd_arg.srg_th, obss_pd_arg.non_srg_th); + + param_val = ath12k_wmi_build_obss_pd(&obss_pd_arg); + ret = ath12k_wmi_pdev_set_param(ar, param_id, param_val, pdev_id); + if (ret) { + ath12k_warn(ar->ab, + "failed to set OBSS PD threshold for pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + /* Enable OBSS PD for all access category */ + param_id = WMI_PDEV_PARAM_SET_CMD_OBSS_PD_PER_AC; + param_val = 0xf; + ret = ath12k_wmi_pdev_set_param(ar, param_id, param_val, pdev_id); + if (ret) { + ath12k_warn(ar->ab, + "failed to set OBSS PD per ac for pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + /* Set SR prohibit */ + param_id = WMI_PDEV_PARAM_ENABLE_SR_PROHIBIT; + param_val = !!(he_obss_pd->sr_ctrl & + IEEE80211_HE_SPR_HESIGA_SR_VAL15_ALLOWED); + ret = ath12k_wmi_pdev_set_param(ar, param_id, param_val, pdev_id); + if (ret) { + ath12k_warn(ar->ab, "failed to set SR prohibit for pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + if (!obss_pd_arg.srp_support) + return 0; + + memcpy(srg_bitmap, he_obss_pd->bss_color_bitmap, sizeof(srg_bitmap)); + /* Set SRG BSS color bitmap */ + ret = ath12k_wmi_pdev_set_srg_bss_color_bitmap(ar, pdev_id, srg_bitmap); + if (ret) { + ath12k_warn(ar->ab, + "failed to set SRG bss color bitmap for pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + /* Enable BSS colors for SRG */ + ret = ath12k_wmi_pdev_srg_obss_color_enable_bitmap(ar, pdev_id, srg_bitmap); + if (ret) { + ath12k_warn(ar->ab, + "failed to enable SRG bss color bitmap pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + memcpy(srg_bitmap, he_obss_pd->partial_bssid_bitmap, sizeof(srg_bitmap)); + /* Set SRG partial bssid bitmap */ + ret = ath12k_wmi_pdev_set_srg_partial_bssid_bitmap(ar, pdev_id, srg_bitmap); + if (ret) { + ath12k_warn(ar->ab, + "failed to set SRG partial bssid bitmap for pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + /* Enable partial bssid mask for SRG */ + ret = ath12k_wmi_pdev_srg_obss_bssid_enable_bitmap(ar, pdev_id, srg_bitmap); + if (ret) { + ath12k_warn(ar->ab, + "failed to enable SRG bssid bitmap pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + /* + * No explicit non-SRG bitmap from mac80211; enable all colors/bssids + * as non-SRG candidates. Actual SRG members are filtered by SRG bitmaps. + */ + memset(non_srg_bitmap, 0xff, sizeof(non_srg_bitmap)); + + /* Enable BSS colors for non-SRG */ + ret = ath12k_wmi_pdev_non_srg_obss_color_enable_bitmap(ar, pdev_id, + non_srg_bitmap); + if (ret) { + ath12k_warn(ar->ab, + "failed to enable non SRG color bitmap pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + /* Enable partial bssid mask for non-SRG */ + ret = ath12k_wmi_pdev_non_srg_obss_bssid_enable_bitmap(ar, pdev_id, + non_srg_bitmap); + if (ret) { + ath12k_warn(ar->ab, + "failed to enable non SRG bssid bitmap pdev %u: %d\n", + pdev_id, ret); + return ret; + } + + return 0; +} + static void ath12k_mac_bss_info_changed(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ieee80211_bss_conf *info, @@ -4210,6 +4675,7 @@ static void ath12k_mac_bss_info_changed(struct ath12k *ar, struct ath12k_vif *ahvif = arvif->ahvif; struct ieee80211_vif *vif = ath12k_ahvif_to_vif(ahvif); struct ieee80211_vif_cfg *vif_cfg = &vif->cfg; + struct ath12k_link_vif *tx_arvif; struct cfg80211_chan_def def; u32 param_id, param_value; enum nl80211_band band; @@ -4218,9 +4684,9 @@ static void ath12k_mac_bss_info_changed(struct ath12k *ar, u32 preamble; u16 hw_value; u16 bitrate; - int ret; u8 rateidx; u32 rate; + int ret; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -4253,12 +4719,41 @@ static void ath12k_mac_bss_info_changed(struct ath12k *ar, "Set burst beacon mode for VDEV: %d\n", arvif->vdev_id); + /* In MBSSID case, need to install transmitting VIF's template first */ + ret = ath12k_mac_setup_bcn_tmpl(arvif); if (ret) ath12k_warn(ar->ab, "failed to update bcn template: %d\n", ret); + + if (!arvif->is_csa_in_progress) + goto skip_vdev_up; + + tx_arvif = ath12k_mac_get_tx_arvif(arvif, info); + if (tx_arvif && arvif != tx_arvif && tx_arvif->is_csa_in_progress) + /* skip non tx vif's */ + goto skip_vdev_up; + + ath12k_wmi_vdev_params_up(ar, arvif, tx_arvif, info, ahvif->aid); + + arvif->is_csa_in_progress = false; + + if (tx_arvif && arvif == tx_arvif) { + struct ath12k_link_vif *arvif_itr; + + list_for_each_entry(arvif_itr, &ar->arvifs, list) { + if (!arvif_itr->is_csa_in_progress) + continue; + + ath12k_wmi_vdev_params_up(ar, arvif, tx_arvif, + info, ahvif->aid); + arvif_itr->is_csa_in_progress = false; + } + } } +skip_vdev_up: + if (changed & (BSS_CHANGED_BEACON_INFO | BSS_CHANGED_BEACON)) { arvif->dtim_period = info->dtim_period; @@ -4463,9 +4958,13 @@ static void ath12k_mac_bss_info_changed(struct ath12k *ar, ath12k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id); } - if (changed & BSS_CHANGED_HE_OBSS_PD) - ath12k_wmi_send_obss_spr_cmd(ar, arvif->vdev_id, - &info->he_obss_pd); + if (changed & BSS_CHANGED_HE_OBSS_PD) { + if (vif->type == NL80211_IFTYPE_AP) + ath12k_mac_config_obss_pd(arvif, &info->he_obss_pd); + else + ath12k_wmi_send_obss_spr_cmd(ar, arvif->vdev_id, + &info->he_obss_pd); + } if (changed & BSS_CHANGED_HE_BSS_COLOR) { if (vif->type == NL80211_IFTYPE_AP) { @@ -4475,8 +4974,25 @@ static void ath12k_mac_bss_info_changed(struct ath12k *ar, ATH12K_BSS_COLOR_AP_PERIODS, info->he_bss_color.enabled); if (ret) - ath12k_warn(ar->ab, "failed to set bss color collision on vdev %i: %d\n", + ath12k_warn(ar->ab, "failed to set bss color collision on vdev %u: %d\n", + arvif->vdev_id, ret); + + param_id = WMI_VDEV_PARAM_BSS_COLOR; + if (info->he_bss_color.enabled) + param_value = info->he_bss_color.color << + IEEE80211_HE_OPERATION_BSS_COLOR_OFFSET; + else + param_value = IEEE80211_HE_OPERATION_BSS_COLOR_DISABLED; + + ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, + param_id, + param_value); + if (ret) + ath12k_warn(ar->ab, "failed to set bss color param on vdev %u: %d\n", arvif->vdev_id, ret); + else + ath12k_dbg(ar->ab, ATH12K_DBG_MAC, "bss color param 0x%x set on vdev %u\n", + param_value, arvif->vdev_id); } else if (vif->type == NL80211_IFTYPE_STATION) { ret = ath12k_wmi_send_bss_color_change_enable_cmd(ar, arvif->vdev_id, @@ -4538,10 +5054,10 @@ static void ath12k_ahvif_put_link_cache(struct ath12k_vif *ahvif, u8 link_id) ahvif->cache[link_id] = NULL; } -static void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_bss_conf *info, - u64 changed) +void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *info, + u64 changed) { struct ath12k *ar; struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); @@ -4571,6 +5087,7 @@ static void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw, ath12k_mac_bss_info_changed(ar, arvif, info, changed); } +EXPORT_SYMBOL(ath12k_mac_op_link_info_changed); static struct ath12k* ath12k_mac_select_scan_device(struct ieee80211_hw *hw, @@ -4837,8 +5354,6 @@ int ath12k_mac_get_fw_stats(struct ath12k *ar, if (ah->state != ATH12K_HW_STATE_ON) return -ENETDOWN; - ath12k_fw_stats_reset(ar); - reinit_completion(&ar->fw_stats_complete); reinit_completion(&ar->fw_stats_done); @@ -4874,10 +5389,10 @@ int ath12k_mac_get_fw_stats(struct ath12k *ar, return 0; } -static int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - unsigned int link_id, - int *dbm) +int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + unsigned int link_id, + int *dbm) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_fw_stats_req_params params = {}; @@ -4936,6 +5451,7 @@ static int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw, ar->chan_tx_pwr = pdev->chan_tx_power / 2; spin_unlock_bh(&ar->data_lock); ar->last_tx_power_update = jiffies; + ath12k_fw_stats_reset(ar); send_tx_power: *dbm = ar->chan_tx_pwr; @@ -4950,6 +5466,7 @@ static int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw, *dbm); return 0; } +EXPORT_SYMBOL(ath12k_mac_op_get_txpower); static u8 ath12k_mac_find_link_id_by_ar(struct ath12k_vif *ahvif, struct ath12k *ar) @@ -5059,7 +5576,8 @@ static int ath12k_mac_initiate_hw_scan(struct ieee80211_hw *hw, ret = ath12k_mac_vdev_create(ar, arvif); if (ret) { ath12k_warn(ar->ab, "unable to create scan vdev %d\n", ret); - return -EINVAL; + ath12k_mac_unassign_link_vif(arvif); + return ret; } } @@ -5160,9 +5678,9 @@ static int ath12k_mac_initiate_hw_scan(struct ieee80211_hw *hw, return ret; } -static int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_scan_request *hw_req) +int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_scan_request *hw_req) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ieee80211_channel **chan_list, *chan; @@ -5240,9 +5758,10 @@ static int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw, kfree(chan_list); return ret; } +EXPORT_SYMBOL(ath12k_mac_op_hw_scan); -static void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw, - struct ieee80211_vif *vif) +void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); unsigned long link_id, links_map = ahvif->links_map; @@ -5253,7 +5772,8 @@ static void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw, for_each_set_bit(link_id, &links_map, ATH12K_NUM_MAX_LINKS) { arvif = wiphy_dereference(hw->wiphy, ahvif->link[link_id]); - if (!arvif || arvif->is_started) + if (!arvif || !arvif->is_created || + arvif->ar->scan.arvif != arvif) continue; ar = arvif->ar; @@ -5263,6 +5783,7 @@ static void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw, cancel_delayed_work_sync(&ar->scan.timeout); } } +EXPORT_SYMBOL(ath12k_mac_op_cancel_hw_scan); static int ath12k_install_key(struct ath12k_link_vif *arvif, struct ieee80211_key_conf *key, @@ -5370,7 +5891,7 @@ static int ath12k_install_key(struct ath12k_link_vif *arvif, return -ETIMEDOUT; if (ether_addr_equal(arg.macaddr, arvif->bssid)) - ahvif->key_cipher = arg.ieee80211_key_cipher; + ahvif->dp_vif.key_cipher = arg.ieee80211_key_cipher; if (ar->install_key_status) { ret = -EINVAL; @@ -5413,27 +5934,40 @@ static int ath12k_clear_peer_keys(struct ath12k_link_vif *arvif, { struct ath12k *ar = arvif->ar; struct ath12k_base *ab = ar->ab; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; int first_errno = 0; int ret; - int i; + int i, len; u32 flags = 0; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ieee80211_key_conf *keys[WMI_MAX_KEY_INDEX + 1] = {}; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find(ab, arvif->vdev_id, addr); - spin_unlock_bh(&ab->base_lock); - - if (!peer) + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, addr); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&dp->dp_lock); return -ENOENT; + } + + len = ARRAY_SIZE(peer->dp_peer->keys); + for (i = 0; i < len; i++) { + if (!peer->dp_peer->keys[i]) + continue; - for (i = 0; i < ARRAY_SIZE(peer->keys); i++) { - if (!peer->keys[i]) + keys[i] = peer->dp_peer->keys[i]; + peer->dp_peer->keys[i] = NULL; + } + + spin_unlock_bh(&dp->dp_lock); + + for (i = 0; i < len; i++) { + if (!keys[i]) continue; /* key flags are not required to delete the key */ - ret = ath12k_install_key(arvif, peer->keys[i], + ret = ath12k_install_key(arvif, keys[i], DISABLE_KEY, addr, flags); if (ret < 0 && first_errno == 0) first_errno = ret; @@ -5441,10 +5975,6 @@ static int ath12k_clear_peer_keys(struct ath12k_link_vif *arvif, if (ret < 0) ath12k_warn(ab, "failed to remove peer key %d: %d\n", i, ret); - - spin_lock_bh(&ab->base_lock); - peer->keys[i] = NULL; - spin_unlock_bh(&ab->base_lock); } return first_errno; @@ -5457,11 +5987,12 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd, { struct ieee80211_sta *sta = NULL; struct ath12k_base *ab = ar->ab; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; struct ath12k_sta *ahsta; const u8 *peer_addr; int ret; u32 flags = 0; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -5481,11 +6012,12 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd, /* the peer should not disappear in mid-way (unless FW goes awry) since * we already hold wiphy lock. we just make sure its there now. */ - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find(ab, arvif->vdev_id, peer_addr); - spin_unlock_bh(&ab->base_lock); + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + peer_addr); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&dp->dp_lock); - if (!peer) { if (cmd == SET_KEY) { ath12k_warn(ab, "cannot install key for non-existent peer %pM\n", peer_addr); @@ -5498,6 +6030,8 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd, return 0; } + spin_unlock_bh(&dp->dp_lock); + if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) flags = WMI_KEY_PAIRWISE; else @@ -5515,23 +6049,26 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd, return ret; } - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find(ab, arvif->vdev_id, peer_addr); - if (peer && cmd == SET_KEY) { - peer->keys[key->keyidx] = key; + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + peer_addr); + if (peer && peer->dp_peer && cmd == SET_KEY) { + peer->dp_peer->keys[key->keyidx] = key; if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) { - peer->ucast_keyidx = key->keyidx; - peer->sec_type = ath12k_dp_tx_get_encrypt_type(key->cipher); + peer->dp_peer->ucast_keyidx = key->keyidx; + peer->dp_peer->sec_type = + ath12k_dp_tx_get_encrypt_type(key->cipher); } else { - peer->mcast_keyidx = key->keyidx; - peer->sec_type_grp = ath12k_dp_tx_get_encrypt_type(key->cipher); + peer->dp_peer->mcast_keyidx = key->keyidx; + peer->dp_peer->sec_type_grp = + ath12k_dp_tx_get_encrypt_type(key->cipher); } - } else if (peer && cmd == DISABLE_KEY) { - peer->keys[key->keyidx] = NULL; + } else if (peer && peer->dp_peer && cmd == DISABLE_KEY) { + peer->dp_peer->keys[key->keyidx] = NULL; if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) - peer->ucast_keyidx = 0; + peer->dp_peer->ucast_keyidx = 0; else - peer->mcast_keyidx = 0; + peer->dp_peer->mcast_keyidx = 0; } else if (!peer) /* impossible unless FW goes crazy */ ath12k_warn(ab, "peer %pM disappeared!\n", peer_addr); @@ -5556,7 +6093,7 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd, } } - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return 0; } @@ -5600,9 +6137,9 @@ static int ath12k_mac_update_key_cache(struct ath12k_vif_cache *cache, return 0; } -static int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, - struct ieee80211_vif *vif, struct ieee80211_sta *sta, - struct ieee80211_key_conf *key) +int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, + struct ieee80211_vif *vif, struct ieee80211_sta *sta, + struct ieee80211_key_conf *key) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_link_vif *arvif; @@ -5689,6 +6226,7 @@ static int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_set_key); static int ath12k_mac_bitrate_mask_num_vht_rates(struct ath12k *ar, @@ -5718,6 +6256,20 @@ ath12k_mac_bitrate_mask_num_he_rates(struct ath12k *ar, return num_rates; } +static int +ath12k_mac_bitrate_mask_num_eht_rates(struct ath12k *ar, + enum nl80211_band band, + const struct cfg80211_bitrate_mask *mask) +{ + int num_rates = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(mask->control[band].eht_mcs); i++) + num_rates += hweight16(mask->control[band].eht_mcs[i]); + + return num_rates; +} + static int ath12k_mac_set_peer_vht_fixed_rate(struct ath12k_link_vif *arvif, struct ath12k_link_sta *arsta, @@ -5818,6 +6370,65 @@ ath12k_mac_set_peer_he_fixed_rate(struct ath12k_link_vif *arvif, return ret; } +static int +ath12k_mac_set_peer_eht_fixed_rate(struct ath12k_link_vif *arvif, + struct ath12k_link_sta *arsta, + const struct cfg80211_bitrate_mask *mask, + enum nl80211_band band) +{ + struct ath12k_sta *ahsta = arsta->ahsta; + struct ath12k *ar = arvif->ar; + struct ieee80211_sta *sta; + struct ieee80211_link_sta *link_sta; + u8 eht_rate, nss = 0; + u32 rate_code; + int ret, i; + + lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); + + sta = ath12k_ahsta_to_sta(ahsta); + + for (i = 0; i < ARRAY_SIZE(mask->control[band].eht_mcs); i++) { + if (hweight16(mask->control[band].eht_mcs[i]) == 1) { + nss = i + 1; + eht_rate = ffs(mask->control[band].eht_mcs[i]) - 1; + } + } + + if (!nss) { + ath12k_warn(ar->ab, "No single EHT Fixed rate found to set for %pM\n", + arsta->addr); + return -EINVAL; + } + + /* Avoid updating invalid nss as fixed rate*/ + link_sta = ath12k_mac_get_link_sta(arsta); + if (!link_sta || nss > link_sta->rx_nss) { + ath12k_warn(ar->ab, + "unable to access link sta for sta %pM link %u or fixed nss of %u is not supported by sta\n", + sta->addr, arsta->link_id, nss); + return -EINVAL; + } + + ath12k_dbg(ar->ab, ATH12K_DBG_MAC, + "Setting Fixed EHT Rate for peer %pM. Device will not switch to any other selected rates\n", + arsta->addr); + + rate_code = ATH12K_HW_RATE_CODE(eht_rate, nss - 1, + WMI_RATE_PREAMBLE_EHT); + + ret = ath12k_wmi_set_peer_param(ar, arsta->addr, + arvif->vdev_id, + WMI_PEER_PARAM_FIXED_RATE, + rate_code); + if (ret) + ath12k_warn(ar->ab, + "failed to update STA %pM Fixed Rate %d: %d\n", + arsta->addr, rate_code, ret); + + return ret; +} + static int ath12k_mac_station_assoc(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ath12k_link_sta *arsta, @@ -5830,7 +6441,7 @@ static int ath12k_mac_station_assoc(struct ath12k *ar, struct cfg80211_chan_def def; enum nl80211_band band; struct cfg80211_bitrate_mask *mask; - u8 num_vht_rates, num_he_rates; + u8 num_vht_rates, num_he_rates, num_eht_rates; u8 link_id = arvif->link_id; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -5873,10 +6484,11 @@ static int ath12k_mac_station_assoc(struct ath12k *ar, num_vht_rates = ath12k_mac_bitrate_mask_num_vht_rates(ar, band, mask); num_he_rates = ath12k_mac_bitrate_mask_num_he_rates(ar, band, mask); + num_eht_rates = ath12k_mac_bitrate_mask_num_eht_rates(ar, band, mask); - /* If single VHT/HE rate is configured (by set_bitrate_mask()), - * peer_assoc will disable VHT/HE. This is now enabled by a peer specific - * fixed param. + /* If single VHT/HE/EHT rate is configured (by set_bitrate_mask()), + * peer_assoc will disable VHT/HE/EHT. This is now enabled by a peer + * specific fixed param. * Note that all other rates and NSS will be disabled for this peer. */ link_sta = ath12k_mac_get_link_sta(arsta); @@ -5896,6 +6508,10 @@ static int ath12k_mac_station_assoc(struct ath12k *ar, ret = ath12k_mac_set_peer_he_fixed_rate(arvif, arsta, mask, band); if (ret) return ret; + } else if (link_sta->eht_cap.has_eht && num_eht_rates == 1) { + ret = ath12k_mac_set_peer_eht_fixed_rate(arvif, arsta, mask, band); + if (ret) + return ret; } /* Re-assoc is run only to update supported rates for given station. It @@ -5958,8 +6574,9 @@ static void ath12k_sta_rc_update_wk(struct wiphy *wiphy, struct wiphy_work *wk) const u8 *ht_mcs_mask; const u16 *vht_mcs_mask; const u16 *he_mcs_mask; + const u16 *eht_mcs_mask; u32 changed, bw, nss, mac_nss, smps, bw_prev; - int err, num_vht_rates, num_he_rates; + int err, num_vht_rates, num_he_rates, num_eht_rates; const struct cfg80211_bitrate_mask *mask; enum wmi_phy_mode peer_phymode; struct ath12k_link_sta *arsta; @@ -5980,6 +6597,7 @@ static void ath12k_sta_rc_update_wk(struct wiphy *wiphy, struct wiphy_work *wk) ht_mcs_mask = arvif->bitrate_mask.control[band].ht_mcs; vht_mcs_mask = arvif->bitrate_mask.control[band].vht_mcs; he_mcs_mask = arvif->bitrate_mask.control[band].he_mcs; + eht_mcs_mask = arvif->bitrate_mask.control[band].eht_mcs; spin_lock_bh(&ar->data_lock); @@ -5997,6 +6615,7 @@ static void ath12k_sta_rc_update_wk(struct wiphy *wiphy, struct wiphy_work *wk) mac_nss = max3(ath12k_mac_max_ht_nss(ht_mcs_mask), ath12k_mac_max_vht_nss(vht_mcs_mask), ath12k_mac_max_he_nss(he_mcs_mask)); + mac_nss = max(mac_nss, ath12k_mac_max_eht_nss(eht_mcs_mask)); nss = min(nss, mac_nss); struct ath12k_wmi_peer_assoc_arg *peer_arg __free(kfree) = @@ -6082,6 +6701,8 @@ static void ath12k_sta_rc_update_wk(struct wiphy *wiphy, struct wiphy_work *wk) mask); num_he_rates = ath12k_mac_bitrate_mask_num_he_rates(ar, band, mask); + num_eht_rates = ath12k_mac_bitrate_mask_num_eht_rates(ar, band, + mask); /* Peer_assoc_prepare will reject vht rates in * bitrate_mask if its not available in range format and @@ -6106,9 +6727,18 @@ static void ath12k_sta_rc_update_wk(struct wiphy *wiphy, struct wiphy_work *wk) band); } else if (link_sta->he_cap.has_he && num_he_rates == 1) { ath12k_mac_set_peer_he_fixed_rate(arvif, arsta, mask, band); + } else if (link_sta->eht_cap.has_eht && num_eht_rates == 1) { + err = ath12k_mac_set_peer_eht_fixed_rate(arvif, arsta, + mask, band); + if (err) { + ath12k_warn(ar->ab, + "failed to set peer EHT fixed rate for STA %pM ret %d\n", + arsta->addr, err); + return; + } } else { - /* If the peer is non-VHT/HE or no fixed VHT/HE rate - * is provided in the new bitrate mask we set the + /* If the peer is non-VHT/HE/EHT or no fixed VHT/HE/EHT + * rate is provided in the new bitrate mask we set the * other rates using peer_assoc command. Also clear * the peer fixed rate settings as it has higher proprity * than peer assoc @@ -6222,46 +6852,47 @@ static void ath12k_mac_station_post_remove(struct ath12k *ar, { struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif); struct ieee80211_sta *sta = ath12k_ahsta_to_sta(arsta->ahsta); - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; + struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); ath12k_mac_dec_num_stations(arvif, arsta); - spin_lock_bh(&ar->ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + arsta->addr); if (peer && peer->sta == sta) { ath12k_warn(ar->ab, "Found peer entry %pM n vdev %i after it was supposedly removed\n", vif->addr, arvif->vdev_id); peer->sta = NULL; - list_del(&peer->list); - kfree(peer); + + ath12k_dp_link_peer_free(peer); ar->num_peers--; } - spin_unlock_bh(&ar->ab->base_lock); - - kfree(arsta->rx_stats); - arsta->rx_stats = NULL; + spin_unlock_bh(&dp->dp_lock); } static int ath12k_mac_station_unauthorize(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ath12k_link_sta *arsta) { - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; int ret; + struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); - spin_lock_bh(&ar->ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + arsta->addr); if (peer) peer->is_authorized = false; - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); /* Driver must clear the keys during the state change from * IEEE80211_STA_AUTHORIZED to IEEE80211_STA_ASSOC, since after @@ -6283,19 +6914,21 @@ static int ath12k_mac_station_authorize(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ath12k_link_sta *arsta) { - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif); int ret; + struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); - spin_lock_bh(&ar->ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + arsta->addr); if (peer) peer->is_authorized = true; - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); if (vif->type == NL80211_IFTYPE_STATION && arvif->is_up) { ret = ath12k_wmi_set_peer_param(ar, arsta->addr, @@ -6319,6 +6952,7 @@ static int ath12k_mac_station_remove(struct ath12k *ar, struct ieee80211_sta *sta = ath12k_ahsta_to_sta(arsta->ahsta); struct ath12k_vif *ahvif = arvif->ahvif; int ret = 0; + struct ath12k_link_sta *temp_arsta; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -6347,6 +6981,15 @@ static int ath12k_mac_station_remove(struct ath12k *ar, ath12k_mac_station_post_remove(ar, arvif, arsta); + spin_lock_bh(&ar->ab->base_lock); + + /* To handle roaming and split phy scenario */ + temp_arsta = ath12k_link_sta_find_by_addr(ar->ab, arsta->addr); + if (temp_arsta && temp_arsta->arvif->ar == ar) + ath12k_link_sta_rhash_delete(ar->ab, arsta); + + spin_unlock_bh(&ar->ab->base_lock); + if (sta->valid_links) ath12k_mac_free_unassign_link_sta(ahvif->ah, arsta->ahsta, arsta->link_id); @@ -6363,6 +7006,7 @@ static int ath12k_mac_station_add(struct ath12k *ar, struct ieee80211_sta *sta = ath12k_ahsta_to_sta(arsta->ahsta); struct ath12k_wmi_peer_create_arg peer_param = {}; int ret; + struct ath12k_link_sta *temp_arsta; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -6373,12 +7017,24 @@ static int ath12k_mac_station_add(struct ath12k *ar, goto exit; } - if (ath12k_debugfs_is_extd_rx_stats_enabled(ar) && !arsta->rx_stats) { - arsta->rx_stats = kzalloc(sizeof(*arsta->rx_stats), GFP_KERNEL); - if (!arsta->rx_stats) { - ret = -ENOMEM; - goto dec_num_station; - } + spin_lock_bh(&ab->base_lock); + + /* + * In case of Split PHY and roaming scenario, pdev idx + * might differ but both the pdev will share same rhash + * table. In that case update the rhash table if link_sta is + * already present + */ + temp_arsta = ath12k_link_sta_find_by_addr(ab, arsta->addr); + if (temp_arsta && temp_arsta->arvif->ar != ar) + ath12k_link_sta_rhash_delete(ab, temp_arsta); + + ret = ath12k_link_sta_rhash_add(ab, arsta); + spin_unlock_bh(&ab->base_lock); + if (ret) { + ath12k_warn(ab, "Failed to add arsta: %pM to hash table, ret: %d", + arsta->addr, ret); + goto dec_num_station; } peer_param.vdev_id = arvif->vdev_id; @@ -6424,13 +7080,13 @@ static int ath12k_mac_station_add(struct ath12k *ar, } } - ewma_avg_rssi_init(&arsta->avg_rssi); return 0; free_peer: ath12k_peer_delete(ar, arvif->vdev_id, arsta->addr); - kfree(arsta->rx_stats); - arsta->rx_stats = NULL; + spin_lock_bh(&ab->base_lock); + ath12k_link_sta_rhash_delete(ab, arsta); + spin_unlock_bh(&ab->base_lock); dec_num_station: ath12k_mac_dec_num_stations(arvif, arsta); exit: @@ -6507,10 +7163,17 @@ static void ath12k_mac_ml_station_remove(struct ath12k_vif *ahvif, ath12k_mac_station_post_remove(ar, arvif, arsta); + spin_lock_bh(&ar->ab->base_lock); + ath12k_link_sta_rhash_delete(ar->ab, arsta); + spin_unlock_bh(&ar->ab->base_lock); + ath12k_mac_free_unassign_link_sta(ah, ahsta, link_id); } - ath12k_peer_ml_delete(ah, sta); + if (sta->mlo) { + clear_bit(ahsta->ml_peer_id, ah->free_ml_peer_id_map); + ahsta->ml_peer_id = ATH12K_MLO_PEER_ID_INVALID; + } } static int ath12k_mac_handle_link_sta_state(struct ieee80211_hw *hw, @@ -6933,11 +7596,11 @@ static int ath12k_mac_select_links(struct ath12k_base *ab, return 0; } -static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_sta *sta, - enum ieee80211_sta_state old_state, - enum ieee80211_sta_state new_state) +int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + enum ieee80211_sta_state old_state, + enum ieee80211_sta_state new_state) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta); @@ -6949,7 +7612,8 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, u16 selected_links = 0; u8 link_id = 0, i; struct ath12k *ar; - int ret; + int ret = -EINVAL; + struct ath12k_dp_peer_create_params dp_params = {}; lockdep_assert_wiphy(hw->wiphy); @@ -6972,12 +7636,28 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, /* ML sta */ if (sta->mlo && !ahsta->links_map && (hweight16(sta->valid_links) == 1)) { - ret = ath12k_peer_ml_create(ah, sta); - if (ret) { - ath12k_hw_warn(ah, "unable to create ML peer for sta %pM", + ahsta->ml_peer_id = ath12k_peer_ml_alloc(ah); + if (ahsta->ml_peer_id == ATH12K_MLO_PEER_ID_INVALID) { + ath12k_hw_warn(ah, "unable to allocate ML peer id for sta %pM", sta->addr); goto exit; } + + dp_params.is_mlo = true; + dp_params.peer_id = ahsta->ml_peer_id | ATH12K_PEER_ML_ID_VALID; + } + + dp_params.sta = sta; + + if (vif->type == NL80211_IFTYPE_AP) + dp_params.ucast_ra_only = true; + + ret = ath12k_dp_peer_create(&ah->dp_hw, sta->addr, &dp_params); + if (ret) { + ath12k_hw_warn(ah, "unable to create ath12k_dp_peer for sta %pM, ret: %d", + sta->addr, ret); + + goto ml_peer_id_clear; } ret = ath12k_mac_assign_link_sta(ah, ahsta, arsta, ahvif, @@ -6985,7 +7665,7 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, if (ret) { ath12k_hw_warn(ah, "unable assign link %d for sta %pM", link_id, sta->addr); - goto exit; + goto peer_delete; } /* above arsta will get memset, hence do this after assign @@ -7055,7 +7735,12 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, if (ret) { ath12k_hw_warn(ah, "unable to move link sta %d of sta %pM from state %d to %d", link_id, arsta->addr, old_state, new_state); - goto exit; + + if (old_state == IEEE80211_STA_NOTEXIST && + new_state == IEEE80211_STA_NONE) + goto peer_delete; + else + goto exit; } } @@ -7083,11 +7768,23 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, * handler below */ if (old_state == IEEE80211_STA_NONE && - new_state == IEEE80211_STA_NOTEXIST && sta->mlo) - ath12k_mac_ml_station_remove(ahvif, ahsta); + new_state == IEEE80211_STA_NOTEXIST) { + if (sta->mlo) + ath12k_mac_ml_station_remove(ahvif, ahsta); + + ath12k_dp_peer_delete(&ah->dp_hw, sta->addr, sta); + } ret = 0; + goto exit; +peer_delete: + ath12k_dp_peer_delete(&ah->dp_hw, sta->addr, sta); +ml_peer_id_clear: + if (sta->mlo) { + clear_bit(ahsta->ml_peer_id, ah->free_ml_peer_id_map); + ahsta->ml_peer_id = ATH12K_MLO_PEER_ID_INVALID; + } exit: /* update the state if everything went well */ if (!ret) @@ -7095,10 +7792,11 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, return ret; } +EXPORT_SYMBOL(ath12k_mac_op_sta_state); -static int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_sta *sta) +int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) { struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta); struct ath12k *ar; @@ -7145,11 +7843,12 @@ static int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw, out: return ret; } +EXPORT_SYMBOL(ath12k_mac_op_sta_set_txpwr); -static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_link_sta *link_sta, - u32 changed) +void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + u32 changed) { struct ieee80211_sta *sta = link_sta->sta; struct ath12k *ar; @@ -7158,8 +7857,9 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw, struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k_link_sta *arsta; struct ath12k_link_vif *arvif; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; u32 bw, smps; + struct ath12k_dp *dp; rcu_read_lock(); arvif = rcu_dereference(ahvif->link[link_sta->link_id]); @@ -7171,6 +7871,7 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw, } ar = arvif->ar; + dp = ath12k_ab_to_dp(ar->ab); arsta = rcu_dereference(ahsta->link[link_sta->link_id]); if (!arsta) { @@ -7179,18 +7880,19 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw, link_sta->link_id, sta->addr); return; } - spin_lock_bh(&ar->ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, + arsta->addr); if (!peer) { - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); rcu_read_unlock(); ath12k_warn(ar->ab, "mac sta rc update failed to find peer %pM on vdev %i\n", arsta->addr, arvif->vdev_id); return; } - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); if (arsta->link_id >= IEEE80211_MLD_MAX_NUM_LINKS) { rcu_read_unlock(); @@ -7253,6 +7955,7 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw, rcu_read_unlock(); } +EXPORT_SYMBOL(ath12k_mac_op_link_sta_rc_update); static struct ath12k_link_sta *ath12k_mac_alloc_assign_link_sta(struct ath12k_hw *ah, struct ath12k_sta *ahsta, @@ -7284,10 +7987,10 @@ static struct ath12k_link_sta *ath12k_mac_alloc_assign_link_sta(struct ath12k_hw return arsta; } -static int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_sta *sta, - u16 old_links, u16 new_links) +int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + u16 old_links, u16 new_links) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta); @@ -7326,16 +8029,16 @@ static int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw, continue; arvif = wiphy_dereference(hw->wiphy, ahvif->link[link_id]); - arsta = ath12k_mac_alloc_assign_link_sta(ah, ahsta, ahvif, link_id); + if (!arvif || !arvif->is_created) + continue; - if (!arvif || !arsta) { + arsta = ath12k_mac_alloc_assign_link_sta(ah, ahsta, ahvif, link_id); + if (!arsta) { ath12k_hw_warn(ah, "Failed to alloc/assign link sta"); continue; } ar = arvif->ar; - if (!ar) - continue; ret = ath12k_mac_station_add(ar, arvif, arsta); if (ret) { @@ -7348,15 +8051,17 @@ static int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_change_sta_links); -static bool ath12k_mac_op_can_activate_links(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - u16 active_links) +bool ath12k_mac_op_can_activate_links(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + u16 active_links) { /* TODO: Handle recovery case */ return true; } +EXPORT_SYMBOL(ath12k_mac_op_can_activate_links); static int ath12k_conf_tx_uapsd(struct ath12k_link_vif *arvif, u16 ac, bool enable) @@ -7468,10 +8173,10 @@ static int ath12k_mac_conf_tx(struct ath12k_link_vif *arvif, u16 ac, return ret; } -static int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - unsigned int link_id, u16 ac, - const struct ieee80211_tx_queue_params *params) +int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + unsigned int link_id, u16 ac, + const struct ieee80211_tx_queue_params *params) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_link_vif *arvif; @@ -7500,6 +8205,7 @@ static int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw, return ret; } +EXPORT_SYMBOL(ath12k_mac_op_conf_tx); static struct ieee80211_sta_ht_cap ath12k_create_ht_cap(struct ath12k *ar, u32 ar_ht_cap, u32 rate_cap_rx_chainmask) @@ -8351,7 +9057,7 @@ static int ath12k_mac_mgmt_tx_wmi(struct ath12k *ar, struct ath12k_link_vif *arv ieee80211_is_disassoc(hdr->frame_control)) && ieee80211_has_protected(hdr->frame_control)) { enctype = ath12k_dp_tx_get_encrypt_type(skb_cb->cipher); - mic_len = ath12k_dp_rx_crypto_mic_len(ar, enctype); + mic_len = ath12k_dp_rx_crypto_mic_len(ab->dp, enctype); skb_put(skb, mic_len); } } @@ -8622,8 +9328,8 @@ static void ath12k_mgmt_over_wmi_tx_work(struct wiphy *wiphy, struct wiphy_work } } -static int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb, - bool is_prb_rsp) +int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb, + bool is_prb_rsp) { struct sk_buff_head *q = &ar->wmi_mgmt_tx_queue; @@ -8653,11 +9359,12 @@ static int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb, return 0; } +EXPORT_SYMBOL(ath12k_mac_mgmt_tx); -static void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar, - struct ieee80211_vif *vif, - struct sk_buff *skb, - bool is_prb_rsp) +void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar, + struct ieee80211_vif *vif, + struct sk_buff *skb, + bool is_prb_rsp) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); @@ -8674,11 +9381,12 @@ static void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar, spin_unlock_bh(&ar->data_lock); } +EXPORT_SYMBOL(ath12k_mac_add_p2p_noa_ie); /* Note: called under rcu_read_lock() */ -static void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif, - u8 link_id, struct sk_buff *skb, - u32 info_flags) +void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif, + u8 link_id, struct sk_buff *skb, + u32 info_flags) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; struct ieee80211_bss_conf *bss_conf; @@ -8690,10 +9398,11 @@ static void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif, if (bss_conf) ether_addr_copy(hdr->addr2, bss_conf->addr); } +EXPORT_SYMBOL(ath12k_mlo_mcast_update_tx_link_address); /* Note: called under rcu_read_lock() */ -static u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif *vif, - u8 link, struct sk_buff *skb, u32 info_flags) +u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif *vif, + u8 link, struct sk_buff *skb, u32 info_flags) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); @@ -8736,242 +9445,59 @@ static u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif */ if (info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP || ieee80211_is_data(hdr->frame_control)) - return ahsta->assoc_link_id; - - /* 802.11 frame cases */ - if (link == IEEE80211_LINK_UNSPECIFIED) - link = ahsta->deflink.link_id; - - if (!ieee80211_is_mgmt(hdr->frame_control)) - return link; - - /* Perform address conversion for ML STA Tx */ - bss_conf = rcu_dereference(vif->link_conf[link]); - link_sta = rcu_dereference(sta->link[link]); - - if (bss_conf && link_sta) { - ether_addr_copy(hdr->addr1, link_sta->addr); - ether_addr_copy(hdr->addr2, bss_conf->addr); - - if (vif->type == NL80211_IFTYPE_STATION && bss_conf->bssid) - ether_addr_copy(hdr->addr3, bss_conf->bssid); - else if (vif->type == NL80211_IFTYPE_AP) - ether_addr_copy(hdr->addr3, bss_conf->addr); - - return link; - } - - if (bss_conf) { - /* In certain cases where a ML sta associated and added subset of - * links on which the ML AP is active, but now sends some frame - * (ex. Probe request) on a different link which is active in our - * MLD but was not added during previous association, we can - * still honor the Tx to that ML STA via the requested link. - * The control would reach here in such case only when that link - * address is same as the MLD address or in worst case clients - * used MLD address at TA wrongly which would have helped - * identify the ML sta object and pass it here. - * If the link address of that STA is different from MLD address, - * then the sta object would be NULL and control won't reach - * here but return at the start of the function itself with !sta - * check. Also this would not need any translation at hdr->addr1 - * from MLD to link address since the RA is the MLD address - * (same as that link address ideally) already. - */ - ether_addr_copy(hdr->addr2, bss_conf->addr); - - if (vif->type == NL80211_IFTYPE_STATION && bss_conf->bssid) - ether_addr_copy(hdr->addr3, bss_conf->bssid); - else if (vif->type == NL80211_IFTYPE_AP) - ether_addr_copy(hdr->addr3, bss_conf->addr); - } - - return link; -} - -/* Note: called under rcu_read_lock() */ -static void ath12k_mac_op_tx(struct ieee80211_hw *hw, - struct ieee80211_tx_control *control, - struct sk_buff *skb) -{ - struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb); - struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); - struct ieee80211_vif *vif = info->control.vif; - struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); - struct ath12k_link_vif *arvif = &ahvif->deflink; - struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; - struct ieee80211_key_conf *key = info->control.hw_key; - struct ieee80211_sta *sta = control->sta; - struct ath12k_link_vif *tmp_arvif; - u32 info_flags = info->flags; - struct sk_buff *msdu_copied; - struct ath12k *ar, *tmp_ar; - struct ath12k_peer *peer; - unsigned long links_map; - bool is_mcast = false; - bool is_dvlan = false; - struct ethhdr *eth; - bool is_prb_rsp; - u16 mcbc_gsn; - u8 link_id; - int ret; - - if (ahvif->vdev_type == WMI_VDEV_TYPE_MONITOR) { - ieee80211_free_txskb(hw, skb); - return; - } - - link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); - memset(skb_cb, 0, sizeof(*skb_cb)); - skb_cb->vif = vif; - - if (key) { - skb_cb->cipher = key->cipher; - skb_cb->flags |= ATH12K_SKB_CIPHER_SET; - } - - /* handle only for MLO case, use deflink for non MLO case */ - if (ieee80211_vif_is_mld(vif)) { - link_id = ath12k_mac_get_tx_link(sta, vif, link_id, skb, info_flags); - if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) { - ieee80211_free_txskb(hw, skb); - return; - } - } else { - link_id = 0; - } - - arvif = rcu_dereference(ahvif->link[link_id]); - if (!arvif || !arvif->ar) { - ath12k_warn(ahvif->ah, "failed to find arvif link id %u for frame transmission", - link_id); - ieee80211_free_txskb(hw, skb); - return; - } - - ar = arvif->ar; - skb_cb->link_id = link_id; - is_prb_rsp = ieee80211_is_probe_resp(hdr->frame_control); - - if (info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP) { - eth = (struct ethhdr *)skb->data; - is_mcast = is_multicast_ether_addr(eth->h_dest); - - skb_cb->flags |= ATH12K_SKB_HW_80211_ENCAP; - } else if (ieee80211_is_mgmt(hdr->frame_control)) { - if (sta && sta->mlo) - skb_cb->flags |= ATH12K_SKB_MLO_STA; - - ret = ath12k_mac_mgmt_tx(ar, skb, is_prb_rsp); - if (ret) { - ath12k_warn(ar->ab, "failed to queue management frame %d\n", - ret); - ieee80211_free_txskb(hw, skb); - } - return; - } - - if (!(info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP)) - is_mcast = is_multicast_ether_addr(hdr->addr1); - - /* This is case only for P2P_GO */ - if (vif->type == NL80211_IFTYPE_AP && vif->p2p) - ath12k_mac_add_p2p_noa_ie(ar, vif, skb, is_prb_rsp); + return ahsta->assoc_link_id; - /* Checking if it is a DVLAN frame */ - if (!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags) && - !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) && - !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) && - ieee80211_has_protected(hdr->frame_control)) - is_dvlan = true; + /* 802.11 frame cases */ + if (link == IEEE80211_LINK_UNSPECIFIED) + link = ahsta->deflink.link_id; - if (!vif->valid_links || !is_mcast || is_dvlan || - (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) || - test_bit(ATH12K_FLAG_RAW_MODE, &ar->ab->dev_flags)) { - ret = ath12k_dp_tx(ar, arvif, skb, false, 0, is_mcast); - if (unlikely(ret)) { - ath12k_warn(ar->ab, "failed to transmit frame %d\n", ret); - ieee80211_free_txskb(ar->ah->hw, skb); - return; - } - } else { - mcbc_gsn = atomic_inc_return(&ahvif->mcbc_gsn) & 0xfff; + if (!ieee80211_is_mgmt(hdr->frame_control)) + return link; - links_map = ahvif->links_map; - for_each_set_bit(link_id, &links_map, - IEEE80211_MLD_MAX_NUM_LINKS) { - tmp_arvif = rcu_dereference(ahvif->link[link_id]); - if (!tmp_arvif || !tmp_arvif->is_up) - continue; + /* Perform address conversion for ML STA Tx */ + bss_conf = rcu_dereference(vif->link_conf[link]); + link_sta = rcu_dereference(sta->link[link]); - tmp_ar = tmp_arvif->ar; - msdu_copied = skb_copy(skb, GFP_ATOMIC); - if (!msdu_copied) { - ath12k_err(ar->ab, - "skb copy failure link_id 0x%X vdevid 0x%X\n", - link_id, tmp_arvif->vdev_id); - continue; - } + if (bss_conf && link_sta) { + ether_addr_copy(hdr->addr1, link_sta->addr); + ether_addr_copy(hdr->addr2, bss_conf->addr); - ath12k_mlo_mcast_update_tx_link_address(vif, link_id, - msdu_copied, - info_flags); - - skb_cb = ATH12K_SKB_CB(msdu_copied); - skb_cb->link_id = link_id; - - /* For open mode, skip peer find logic */ - if (unlikely(!ahvif->key_cipher)) - goto skip_peer_find; - - spin_lock_bh(&tmp_ar->ab->base_lock); - peer = ath12k_peer_find_by_addr(tmp_ar->ab, tmp_arvif->bssid); - if (!peer) { - spin_unlock_bh(&tmp_ar->ab->base_lock); - ath12k_warn(tmp_ar->ab, - "failed to find peer for vdev_id 0x%X addr %pM link_map 0x%X\n", - tmp_arvif->vdev_id, tmp_arvif->bssid, - ahvif->links_map); - dev_kfree_skb_any(msdu_copied); - continue; - } + if (vif->type == NL80211_IFTYPE_STATION && bss_conf->bssid) + ether_addr_copy(hdr->addr3, bss_conf->bssid); + else if (vif->type == NL80211_IFTYPE_AP) + ether_addr_copy(hdr->addr3, bss_conf->addr); - key = peer->keys[peer->mcast_keyidx]; - if (key) { - skb_cb->cipher = key->cipher; - skb_cb->flags |= ATH12K_SKB_CIPHER_SET; + return link; + } - hdr = (struct ieee80211_hdr *)msdu_copied->data; - if (!ieee80211_has_protected(hdr->frame_control)) - hdr->frame_control |= - cpu_to_le16(IEEE80211_FCTL_PROTECTED); - } - spin_unlock_bh(&tmp_ar->ab->base_lock); - -skip_peer_find: - ret = ath12k_dp_tx(tmp_ar, tmp_arvif, - msdu_copied, true, mcbc_gsn, is_mcast); - if (unlikely(ret)) { - if (ret == -ENOMEM) { - /* Drops are expected during heavy multicast - * frame flood. Print with debug log - * level to avoid lot of console prints - */ - ath12k_dbg(ar->ab, ATH12K_DBG_MAC, - "failed to transmit frame %d\n", - ret); - } else { - ath12k_warn(ar->ab, - "failed to transmit frame %d\n", - ret); - } + if (bss_conf) { + /* In certain cases where a ML sta associated and added subset of + * links on which the ML AP is active, but now sends some frame + * (ex. Probe request) on a different link which is active in our + * MLD but was not added during previous association, we can + * still honor the Tx to that ML STA via the requested link. + * The control would reach here in such case only when that link + * address is same as the MLD address or in worst case clients + * used MLD address at TA wrongly which would have helped + * identify the ML sta object and pass it here. + * If the link address of that STA is different from MLD address, + * then the sta object would be NULL and control won't reach + * here but return at the start of the function itself with !sta + * check. Also this would not need any translation at hdr->addr1 + * from MLD to link address since the RA is the MLD address + * (same as that link address ideally) already. + */ + ether_addr_copy(hdr->addr2, bss_conf->addr); - dev_kfree_skb_any(msdu_copied); - } - } - ieee80211_free_txskb(ar->ah->hw, skb); + if (vif->type == NL80211_IFTYPE_STATION && bss_conf->bssid) + ether_addr_copy(hdr->addr3, bss_conf->bssid); + else if (vif->type == NL80211_IFTYPE_AP) + ether_addr_copy(hdr->addr3, bss_conf->addr); } + + return link; } +EXPORT_SYMBOL(ath12k_mac_get_tx_link); void ath12k_mac_drain_tx(struct ath12k *ar) { @@ -9146,7 +9672,7 @@ static void ath12k_drain_tx(struct ath12k_hw *ah) ath12k_mac_drain_tx(ar); } -static int ath12k_mac_op_start(struct ieee80211_hw *hw) +int ath12k_mac_op_start(struct ieee80211_hw *hw) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k *ar; @@ -9199,6 +9725,7 @@ static int ath12k_mac_op_start(struct ieee80211_hw *hw) return ret; } +EXPORT_SYMBOL(ath12k_mac_op_start); int ath12k_mac_rfkill_config(struct ath12k *ar) { @@ -9259,6 +9786,7 @@ int ath12k_mac_rfkill_enable_radio(struct ath12k *ar, bool enable) static void ath12k_mac_stop(struct ath12k *ar) { + struct ath12k_pdev_dp *dp_pdev = &ar->dp; struct ath12k_hw *ah = ar->ah; struct htt_ppdu_stats_info *ppdu_stats, *tmp; struct ath12k_wmi_scan_chan_list_arg *arg; @@ -9283,13 +9811,14 @@ static void ath12k_mac_stop(struct ath12k *ar) ar->state_11d = ATH12K_11D_IDLE; complete(&ar->completed_11d_scan); - spin_lock_bh(&ar->data_lock); - - list_for_each_entry_safe(ppdu_stats, tmp, &ar->ppdu_stats_info, list) { + spin_lock_bh(&dp_pdev->ppdu_list_lock); + list_for_each_entry_safe(ppdu_stats, tmp, &dp_pdev->ppdu_stats_info, list) { list_del(&ppdu_stats->list); kfree(ppdu_stats); } + spin_unlock_bh(&dp_pdev->ppdu_list_lock); + spin_lock_bh(&ar->data_lock); while ((arg = list_first_entry_or_null(&ar->regd_channel_update_queue, struct ath12k_wmi_scan_chan_list_arg, list))) { @@ -9305,7 +9834,7 @@ static void ath12k_mac_stop(struct ath12k *ar) atomic_set(&ar->num_pending_mgmt_tx, 0); } -static void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend) +void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k *ar; @@ -9324,6 +9853,7 @@ static void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend) mutex_unlock(&ah->hw_mutex); } +EXPORT_SYMBOL(ath12k_mac_op_stop); static u8 ath12k_mac_get_vdev_stats_id(struct ath12k_link_vif *arvif) @@ -9457,14 +9987,14 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif) IEEE80211_OFFLOAD_DECAP_ENABLED); if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) - ahvif->tx_encap_type = ATH12K_HW_TXRX_ETHERNET; + ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_ETHERNET; else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) - ahvif->tx_encap_type = ATH12K_HW_TXRX_RAW; + ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_RAW; else - ahvif->tx_encap_type = ATH12K_HW_TXRX_NATIVE_WIFI; + ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_NATIVE_WIFI; ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, - param_id, ahvif->tx_encap_type); + param_id, ahvif->dp_vif.tx_encap_type); if (ret) { ath12k_warn(ab, "failed to set vdev %d tx encap mode: %d\n", arvif->vdev_id, ret); @@ -9488,8 +10018,8 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif) } } -static void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw, - struct ieee80211_vif *vif) +void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_link_vif *arvif; @@ -9513,6 +10043,7 @@ static void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw, ath12k_mac_update_vif_offload(&ahvif->deflink); } +EXPORT_SYMBOL(ath12k_mac_op_update_vif_offload); static bool ath12k_mac_vif_ap_active_any(struct ath12k_base *ab) { @@ -9678,6 +10209,9 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) int i; int ret, vdev_id; u8 link_id; + struct ath12k_dp_link_vif *dp_link_vif = NULL; + struct ath12k_dp_peer_create_params params = {}; + bool dp_peer_created = false; lockdep_assert_wiphy(hw->wiphy); @@ -9687,6 +10221,12 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) if (vif->type == NL80211_IFTYPE_MONITOR && ar->monitor_vdev_created) return -EINVAL; + if (ar->num_created_vdevs >= TARGET_NUM_VDEVS(ab)) { + ath12k_warn(ab, "failed to create vdev, reached max vdev limit %d\n", + TARGET_NUM_VDEVS(ab)); + return -ENOSPC; + } + link_id = arvif->link_id; if (link_id < IEEE80211_MLD_MAX_NUM_LINKS) { @@ -9728,7 +10268,7 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) if (ret) { ath12k_warn(ab, "failed to create WMI vdev %d: %d\n", arvif->vdev_id, ret); - return ret; + goto err; } ar->num_created_vdevs++; @@ -9753,8 +10293,26 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) goto err_vdev_del; } + dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, arvif->link_id); + + dp_link_vif->vdev_id = arvif->vdev_id; + dp_link_vif->lmac_id = ar->lmac_id; + dp_link_vif->pdev_idx = ar->pdev_idx; + switch (ahvif->vdev_type) { case WMI_VDEV_TYPE_AP: + params.ucast_ra_only = true; + + if (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS) { + ret = ath12k_dp_peer_create(&ah->dp_hw, arvif->bssid, ¶ms); + if (ret) { + ath12k_warn(ab, "failed to vdev %d create dp_peer for AP: %d\n", + arvif->vdev_id, ret); + goto err_vdev_del; + } + dp_peer_created = true; + } + peer_param.vdev_id = arvif->vdev_id; peer_param.peer_addr = arvif->bssid; peer_param.peer_type = WMI_PEER_TYPE_DEFAULT; @@ -9762,7 +10320,7 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) if (ret) { ath12k_warn(ab, "failed to vdev %d create peer for AP: %d\n", arvif->vdev_id, ret); - goto err_vdev_del; + goto err_dp_peer_del; } ret = ath12k_mac_set_kickout(arvif); @@ -9857,17 +10415,21 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) if (ret) { ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n", arvif->vdev_id, arvif->bssid); - goto err; + goto err_dp_peer_del; } ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id, arvif->bssid); if (ret) - goto err_vdev_del; + goto err_dp_peer_del; ar->num_peers--; } +err_dp_peer_del: + if (dp_peer_created) + ath12k_dp_peer_delete(&ah->dp_hw, arvif->bssid, NULL); + err_vdev_del: if (ahvif->vdev_type == WMI_VDEV_TYPE_MONITOR) { ar->monitor_vdev_id = -1; @@ -9876,8 +10438,6 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) ath12k_wmi_vdev_delete(ar, arvif->vdev_id); ar->num_created_vdevs--; - arvif->is_created = false; - arvif->ar = NULL; ar->allocated_vdev_map &= ~(1LL << arvif->vdev_id); ab->free_vdev_map |= 1LL << arvif->vdev_id; ab->free_vdev_stats_id_map &= ~(1LL << arvif->vdev_stats_id); @@ -9886,6 +10446,7 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) spin_unlock_bh(&ar->data_lock); err: + arvif->is_created = false; arvif->ar = NULL; return ret; } @@ -10046,12 +10607,6 @@ static struct ath12k *ath12k_mac_assign_vif_to_vdev(struct ieee80211_hw *hw, if (arvif->is_created) goto flush; - if (ar->num_created_vdevs > (TARGET_NUM_VDEVS(ab) - 1)) { - ath12k_warn(ab, "failed to create vdev, reached max vdev limit %d\n", - TARGET_NUM_VDEVS(ab)); - goto unlock; - } - ret = ath12k_mac_vdev_create(ar, arvif); if (ret) { ath12k_warn(ab, "failed to create vdev %pM ret %d", vif->addr, ret); @@ -10068,8 +10623,8 @@ static struct ath12k *ath12k_mac_assign_vif_to_vdev(struct ieee80211_hw *hw, return arvif->ar; } -static int ath12k_mac_op_add_interface(struct ieee80211_hw *hw, - struct ieee80211_vif *vif) +int ath12k_mac_op_add_interface(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); @@ -10116,6 +10671,7 @@ static int ath12k_mac_op_add_interface(struct ieee80211_hw *hw, */ return 0; } +EXPORT_SYMBOL(ath12k_mac_op_add_interface); static void ath12k_mac_vif_unref(struct ath12k_dp *dp, struct ieee80211_vif *vif) { @@ -10146,6 +10702,7 @@ static int ath12k_mac_vdev_delete(struct ath12k *ar, struct ath12k_link_vif *arv { struct ath12k_vif *ahvif = arvif->ahvif; struct ieee80211_vif *vif = ath12k_ahvif_to_vif(ahvif); + struct ath12k_dp_link_vif *dp_link_vif; struct ath12k_base *ab = ar->ab; unsigned long time_left; int ret; @@ -10191,8 +10748,10 @@ static int ath12k_mac_vdev_delete(struct ath12k *ar, struct ath12k_link_vif *arv idr_for_each(&ar->txmgmt_idr, ath12k_mac_vif_txmgmt_idr_remove, vif); - ath12k_mac_vif_unref(&ab->dp, vif); - ath12k_dp_tx_put_bank_profile(&ab->dp, arvif->bank_id); + ath12k_mac_vif_unref(ath12k_ab_to_dp(ab), vif); + + dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, arvif->link_id); + ath12k_dp_tx_put_bank_profile(ath12k_ab_to_dp(ab), dp_link_vif->bank_id); /* Recalc txpower for remaining vdev */ ath12k_mac_txpower_recalc(ar); @@ -10204,8 +10763,8 @@ static int ath12k_mac_vdev_delete(struct ath12k *ar, struct ath12k_link_vif *arv return ret; } -static void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw, - struct ieee80211_vif *vif) +void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_link_vif *arvif; @@ -10252,6 +10811,7 @@ static void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw, ath12k_mac_unassign_link_vif(arvif); } } +EXPORT_SYMBOL(ath12k_mac_op_remove_interface); /* FIXME: Has to be verified. */ #define SUPPORTED_FILTERS \ @@ -10263,10 +10823,10 @@ static void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw, FIF_PROBE_REQ | \ FIF_FCSFAIL) -static void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw, - unsigned int changed_flags, - unsigned int *total_flags, - u64 multicast) +void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw, + unsigned int changed_flags, + unsigned int *total_flags, + u64 multicast) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k *ar; @@ -10278,9 +10838,10 @@ static void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw, *total_flags &= SUPPORTED_FILTERS; ar->filter_flags = *total_flags; } +EXPORT_SYMBOL(ath12k_mac_op_configure_filter); -static int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx, - u32 *tx_ant, u32 *rx_ant) +int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx, + u32 *tx_ant, u32 *rx_ant) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); int antennas_rx = 0, antennas_tx = 0; @@ -10299,9 +10860,10 @@ static int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_get_antenna); -static int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx, - u32 tx_ant, u32 rx_ant) +int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx, + u32 tx_ant, u32 rx_ant) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k *ar; @@ -10318,6 +10880,7 @@ static int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx, return ret; } +EXPORT_SYMBOL(ath12k_mac_op_set_antenna); static int ath12k_mac_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif, @@ -10359,9 +10922,9 @@ static int ath12k_mac_ampdu_action(struct ieee80211_hw *hw, return ret; } -static int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_ampdu_params *params) +int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_ampdu_params *params) { struct ieee80211_sta *sta = params->sta; struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta); @@ -10382,9 +10945,10 @@ static int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_ampdu_action); -static int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw, - struct ieee80211_chanctx_conf *ctx) +int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx) { struct ath12k *ar; struct ath12k_base *ab; @@ -10411,9 +10975,10 @@ static int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_add_chanctx); -static void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw, - struct ieee80211_chanctx_conf *ctx) +void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx) { struct ath12k *ar; struct ath12k_base *ab; @@ -10438,6 +11003,7 @@ static void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw, spin_unlock_bh(&ar->data_lock); ar->chan_tx_pwr = ATH12K_PDEV_TX_POWER_INVALID; } +EXPORT_SYMBOL(ath12k_mac_op_remove_chanctx); static enum wmi_phy_mode ath12k_mac_check_down_grade_phy_mode(struct ath12k *ar, @@ -10852,9 +11418,9 @@ ath12k_mac_update_vif_chan(struct ath12k *ar, int n_vifs) { struct ath12k_wmi_vdev_up_params params = {}; - struct ath12k_link_vif *arvif; struct ieee80211_bss_conf *link_conf; struct ath12k_base *ab = ar->ab; + struct ath12k_link_vif *arvif; struct ieee80211_vif *vif; struct ath12k_vif *ahvif; u8 link_id; @@ -10915,6 +11481,28 @@ ath12k_mac_update_vif_chan(struct ath12k *ar, continue; } + ret = ath12k_mac_update_peer_puncturing_width(arvif->ar, arvif, + vifs[i].new_ctx->def); + if (ret) { + ath12k_warn(ar->ab, + "failed to update puncturing bitmap %02x and width %d: %d\n", + vifs[i].new_ctx->def.punctured, + vifs[i].new_ctx->def.width, ret); + continue; + } + + /* Defer VDEV bring-up during CSA to avoid installing stale + * beacon templates. The beacon content is updated only + * after CSA finalize, so we mark CSA in progress and skip + * VDEV_UP for now. It will be handled later in + * bss_info_changed(). + */ + if (link_conf->csa_active && + arvif->ahvif->vdev_type == WMI_VDEV_TYPE_AP) { + arvif->is_csa_in_progress = true; + continue; + } + ret = ath12k_mac_setup_bcn_tmpl(arvif); if (ret) ath12k_warn(ab, "failed to update bcn tmpl during csa: %d\n", @@ -10935,16 +11523,6 @@ ath12k_mac_update_vif_chan(struct ath12k *ar, arvif->vdev_id, ret); continue; } - - ret = ath12k_mac_update_peer_puncturing_width(arvif->ar, arvif, - vifs[i].new_ctx->def); - if (ret) { - ath12k_warn(ar->ab, - "failed to update puncturing bitmap %02x and width %d: %d\n", - vifs[i].new_ctx->def.punctured, - vifs[i].new_ctx->def.width, ret); - continue; - } } /* Restart the internal monitor vdev on new channel */ @@ -10984,9 +11562,9 @@ ath12k_mac_update_active_vif_chan(struct ath12k *ar, kfree(arg.vifs); } -static void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw, - struct ieee80211_chanctx_conf *ctx, - u32 changed) +void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx, + u32 changed) { struct ath12k *ar; struct ath12k_base *ab; @@ -11016,6 +11594,7 @@ static void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw, /* TODO: Recalc radar detection */ } +EXPORT_SYMBOL(ath12k_mac_op_change_chanctx); static int ath12k_start_vdev_delay(struct ath12k *ar, struct ath12k_link_vif *arvif) @@ -11457,7 +12036,7 @@ static void ath12k_mac_parse_tx_pwr_env(struct ath12k *ar, } } -static int +int ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_bss_conf *link_conf, @@ -11506,7 +12085,7 @@ ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw, if (ab->hw_params->vdev_start_delay && ahvif->vdev_type != WMI_VDEV_TYPE_AP && ahvif->vdev_type != WMI_VDEV_TYPE_MONITOR && - !ath12k_peer_exist_by_vdev_id(ab, arvif->vdev_id)) { + !ath12k_dp_link_peer_exist_by_vdev_id(ath12k_ab_to_dp(ab), arvif->vdev_id)) { ret = 0; goto out; } @@ -11542,8 +12121,9 @@ ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw, out: return ret; } +EXPORT_SYMBOL(ath12k_mac_op_assign_vif_chanctx); -static void +void ath12k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_bss_conf *link_conf, @@ -11610,8 +12190,9 @@ ath12k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw, ar->scan.arvif = NULL; } } +EXPORT_SYMBOL(ath12k_mac_op_unassign_vif_chanctx); -static int +int ath12k_mac_op_switch_vif_chanctx(struct ieee80211_hw *hw, struct ieee80211_vif_chanctx_switch *vifs, int n_vifs, @@ -11636,6 +12217,7 @@ ath12k_mac_op_switch_vif_chanctx(struct ieee80211_hw *hw, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_switch_vif_chanctx); static int ath12k_set_vdev_param_to_all_vifs(struct ath12k *ar, int param, u32 value) @@ -11664,8 +12246,8 @@ ath12k_set_vdev_param_to_all_vifs(struct ath12k *ar, int param, u32 value) /* mac80211 stores device specific RTS/Fragmentation threshold value, * this is set interface specific to firmware from ath12k driver */ -static int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw, - int radio_idx, u32 value) +int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw, + int radio_idx, u32 value) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct wiphy *wiphy = hw->wiphy; @@ -11724,9 +12306,10 @@ static int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw, return ret; } +EXPORT_SYMBOL(ath12k_mac_op_set_rts_threshold); -static int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw, - int radio_idx, u32 value) +int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw, + int radio_idx, u32 value) { /* Even though there's a WMI vdev param for fragmentation threshold no * known firmware actually implements it. Moreover it is not possible to @@ -11743,6 +12326,7 @@ static int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw, return -EOPNOTSUPP; } +EXPORT_SYMBOL(ath12k_mac_op_set_frag_threshold); static int ath12k_mac_flush(struct ath12k *ar) { @@ -11780,8 +12364,8 @@ int ath12k_mac_wait_tx_complete(struct ath12k *ar) return ath12k_mac_flush(ar); } -static void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - u32 queues, bool drop) +void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + u32 queues, bool drop) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k_link_vif *arvif; @@ -11796,6 +12380,9 @@ static void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *v if (drop) return; + for_each_ar(ah, ar, i) + wiphy_work_flush(hw->wiphy, &ar->wmi_mgmt_tx_work); + /* vif can be NULL when flush() is considered for hw */ if (!vif) { for_each_ar(ah, ar, i) @@ -11803,9 +12390,6 @@ static void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *v return; } - for_each_ar(ah, ar, i) - wiphy_work_flush(hw->wiphy, &ar->wmi_mgmt_tx_work); - ahvif = ath12k_vif_to_ahvif(vif); links = ahvif->links_map; for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { @@ -11816,6 +12400,7 @@ static void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *v ath12k_mac_flush(arvif->ar); } } +EXPORT_SYMBOL(ath12k_mac_op_flush); static int ath12k_mac_bitrate_mask_num_ht_rates(struct ath12k *ar, @@ -11849,6 +12434,9 @@ ath12k_mac_has_single_legacy_rate(struct ath12k *ar, if (ath12k_mac_bitrate_mask_num_he_rates(ar, band, mask)) return false; + if (ath12k_mac_bitrate_mask_num_eht_rates(ar, band, mask)) + return false; + return num_rates == 1; } @@ -11871,11 +12459,15 @@ ath12k_mac_bitrate_mask_get_single_nss(struct ath12k *ar, { struct ieee80211_supported_band *sband = &ar->mac.sbands[band]; u16 vht_mcs_map = le16_to_cpu(sband->vht_cap.vht_mcs.tx_mcs_map); + const struct ieee80211_sband_iftype_data *data; const struct ieee80211_sta_he_cap *he_cap; u16 he_mcs_map = 0; + u16 eht_mcs_map = 0; u8 ht_nss_mask = 0; u8 vht_nss_mask = 0; u8 he_nss_mask = 0; + u8 eht_nss_mask = 0; + u8 mcs_nss_len; int i; /* No need to consider legacy here. Basic rates are always present @@ -11919,7 +12511,60 @@ ath12k_mac_bitrate_mask_get_single_nss(struct ath12k *ar, return false; } - if (ht_nss_mask != vht_nss_mask || ht_nss_mask != he_nss_mask) + data = ieee80211_get_sband_iftype_data(sband, vif->type); + + mcs_nss_len = ieee80211_eht_mcs_nss_size(&data->he_cap.he_cap_elem, + &data->eht_cap.eht_cap_elem, + false); + if (mcs_nss_len == 4) { + /* 20 MHz only STA case */ + const struct ieee80211_eht_mcs_nss_supp_20mhz_only *eht_mcs_nss = + &data->eht_cap.eht_mcs_nss_supp.only_20mhz; + if (eht_mcs_nss->rx_tx_mcs13_max_nss) + eht_mcs_map = 0x1fff; + else if (eht_mcs_nss->rx_tx_mcs11_max_nss) + eht_mcs_map = 0x07ff; + else if (eht_mcs_nss->rx_tx_mcs9_max_nss) + eht_mcs_map = 0x01ff; + else + eht_mcs_map = 0x007f; + } else { + const struct ieee80211_eht_mcs_nss_supp_bw *eht_mcs_nss; + + switch (mcs_nss_len) { + case 9: + eht_mcs_nss = &data->eht_cap.eht_mcs_nss_supp.bw._320; + break; + case 6: + eht_mcs_nss = &data->eht_cap.eht_mcs_nss_supp.bw._160; + break; + case 3: + eht_mcs_nss = &data->eht_cap.eht_mcs_nss_supp.bw._80; + break; + default: + return false; + } + + if (eht_mcs_nss->rx_tx_mcs13_max_nss) + eht_mcs_map = 0x1fff; + else if (eht_mcs_nss->rx_tx_mcs11_max_nss) + eht_mcs_map = 0x7ff; + else + eht_mcs_map = 0x1ff; + } + + for (i = 0; i < ARRAY_SIZE(mask->control[band].eht_mcs); i++) { + if (mask->control[band].eht_mcs[i] == 0) + continue; + + if (mask->control[band].eht_mcs[i] < eht_mcs_map) + eht_nss_mask |= BIT(i); + else + return false; + } + + if (ht_nss_mask != vht_nss_mask || ht_nss_mask != he_nss_mask || + ht_nss_mask != eht_nss_mask) return false; if (ht_nss_mask == 0) @@ -11967,7 +12612,8 @@ ath12k_mac_get_single_legacy_rate(struct ath12k *ar, } static int -ath12k_mac_set_fixed_rate_gi_ltf(struct ath12k_link_vif *arvif, u8 he_gi, u8 he_ltf) +ath12k_mac_set_fixed_rate_gi_ltf(struct ath12k_link_vif *arvif, u8 gi, u8 ltf, + u32 param) { struct ath12k *ar = arvif->ar; int ret; @@ -11975,47 +12621,54 @@ ath12k_mac_set_fixed_rate_gi_ltf(struct ath12k_link_vif *arvif, u8 he_gi, u8 he_ lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); /* 0.8 = 0, 1.6 = 2 and 3.2 = 3. */ - if (he_gi && he_gi != 0xFF) - he_gi += 1; + if (gi && gi != 0xFF) + gi += 1; ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, - WMI_VDEV_PARAM_SGI, he_gi); + WMI_VDEV_PARAM_SGI, gi); if (ret) { - ath12k_warn(ar->ab, "failed to set HE GI:%d, error:%d\n", - he_gi, ret); + ath12k_warn(ar->ab, "failed to set GI:%d, error:%d\n", + gi, ret); return ret; } - /* start from 1 */ - if (he_ltf != 0xFF) - he_ltf += 1; + + if (param == WMI_VDEV_PARAM_HE_LTF) { + /* HE values start from 1 */ + if (ltf != 0xFF) + ltf += 1; + } else { + /* EHT values start from 5 */ + if (ltf != 0xFF) + ltf += 4; + } ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, - WMI_VDEV_PARAM_HE_LTF, he_ltf); + param, ltf); if (ret) { - ath12k_warn(ar->ab, "failed to set HE LTF:%d, error:%d\n", - he_ltf, ret); + ath12k_warn(ar->ab, "failed to set LTF:%d, error:%d\n", + ltf, ret); return ret; } return 0; } static int -ath12k_mac_set_auto_rate_gi_ltf(struct ath12k_link_vif *arvif, u16 he_gi, u8 he_ltf) +ath12k_mac_set_auto_rate_gi_ltf(struct ath12k_link_vif *arvif, u16 gi, u8 ltf) { struct ath12k *ar = arvif->ar; int ret; - u32 he_ar_gi_ltf; + u32 ar_gi_ltf; - if (he_gi != 0xFF) { - switch (he_gi) { - case NL80211_RATE_INFO_HE_GI_0_8: - he_gi = WMI_AUTORATE_800NS_GI; + if (gi != 0xFF) { + switch (gi) { + case ATH12K_RATE_INFO_GI_0_8: + gi = WMI_AUTORATE_800NS_GI; break; - case NL80211_RATE_INFO_HE_GI_1_6: - he_gi = WMI_AUTORATE_1600NS_GI; + case ATH12K_RATE_INFO_GI_1_6: + gi = WMI_AUTORATE_1600NS_GI; break; - case NL80211_RATE_INFO_HE_GI_3_2: - he_gi = WMI_AUTORATE_3200NS_GI; + case ATH12K_RATE_INFO_GI_3_2: + gi = WMI_AUTORATE_3200NS_GI; break; default: ath12k_warn(ar->ab, "Invalid GI\n"); @@ -12023,16 +12676,16 @@ ath12k_mac_set_auto_rate_gi_ltf(struct ath12k_link_vif *arvif, u16 he_gi, u8 he_ } } - if (he_ltf != 0xFF) { - switch (he_ltf) { - case NL80211_RATE_INFO_HE_1XLTF: - he_ltf = WMI_HE_AUTORATE_LTF_1X; + if (ltf != 0xFF) { + switch (ltf) { + case ATH12K_RATE_INFO_1XLTF: + ltf = WMI_AUTORATE_LTF_1X; break; - case NL80211_RATE_INFO_HE_2XLTF: - he_ltf = WMI_HE_AUTORATE_LTF_2X; + case ATH12K_RATE_INFO_2XLTF: + ltf = WMI_AUTORATE_LTF_2X; break; - case NL80211_RATE_INFO_HE_4XLTF: - he_ltf = WMI_HE_AUTORATE_LTF_4X; + case ATH12K_RATE_INFO_4XLTF: + ltf = WMI_AUTORATE_LTF_4X; break; default: ath12k_warn(ar->ab, "Invalid LTF\n"); @@ -12040,15 +12693,15 @@ ath12k_mac_set_auto_rate_gi_ltf(struct ath12k_link_vif *arvif, u16 he_gi, u8 he_ } } - he_ar_gi_ltf = he_gi | he_ltf; + ar_gi_ltf = gi | ltf; ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, WMI_VDEV_PARAM_AUTORATE_MISC_CFG, - he_ar_gi_ltf); + ar_gi_ltf); if (ret) { ath12k_warn(ar->ab, - "failed to set HE autorate GI:%u, LTF:%u params, error:%d\n", - he_gi, he_ltf, ret); + "failed to set autorate GI:%u, LTF:%u params, error:%d\n", + gi, ltf, ret); return ret; } @@ -12069,14 +12722,16 @@ static u32 ath12k_mac_nlgi_to_wmigi(enum nl80211_txrate_gi gi) static int ath12k_mac_set_rate_params(struct ath12k_link_vif *arvif, u32 rate, u8 nss, u8 sgi, u8 ldpc, - u8 he_gi, u8 he_ltf, bool he_fixed_rate) + u8 he_gi, u8 he_ltf, bool he_fixed_rate, + u8 eht_gi, u8 eht_ltf, + bool eht_fixed_rate) { struct ieee80211_bss_conf *link_conf; struct ath12k *ar = arvif->ar; + bool he_support, eht_support, gi_ltf_set = false; u32 vdev_param; u32 param_value; int ret; - bool he_support; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -12085,6 +12740,7 @@ static int ath12k_mac_set_rate_params(struct ath12k_link_vif *arvif, return -EINVAL; he_support = link_conf->he_support; + eht_support = link_conf->eht_support; ath12k_dbg(ar->ab, ATH12K_DBG_MAC, "mac set rate params vdev %i rate 0x%02x nss 0x%02x sgi 0x%02x ldpc 0x%02x\n", @@ -12094,7 +12750,11 @@ static int ath12k_mac_set_rate_params(struct ath12k_link_vif *arvif, "he_gi 0x%02x he_ltf 0x%02x he_fixed_rate %d\n", he_gi, he_ltf, he_fixed_rate); - if (!he_support) { + ath12k_dbg(ar->ab, ATH12K_DBG_MAC, + "eht_gi 0x%02x eht_ltf 0x%02x eht_fixed_rate %d\n", + eht_gi, eht_ltf, eht_fixed_rate); + + if (!he_support && !eht_support) { vdev_param = WMI_VDEV_PARAM_FIXED_RATE; ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, vdev_param, rate); @@ -12123,14 +12783,34 @@ static int ath12k_mac_set_rate_params(struct ath12k_link_vif *arvif, return ret; } + if (eht_support) { + if (eht_fixed_rate) + ret = ath12k_mac_set_fixed_rate_gi_ltf(arvif, eht_gi, eht_ltf, + WMI_VDEV_PARAM_EHT_LTF); + else + ret = ath12k_mac_set_auto_rate_gi_ltf(arvif, eht_gi, eht_ltf); + + if (ret) { + ath12k_warn(ar->ab, + "failed to set EHT LTF/GI params %d/%d: %d\n", + eht_gi, eht_ltf, ret); + return ret; + } + gi_ltf_set = true; + } + if (he_support) { if (he_fixed_rate) - ret = ath12k_mac_set_fixed_rate_gi_ltf(arvif, he_gi, he_ltf); + ret = ath12k_mac_set_fixed_rate_gi_ltf(arvif, he_gi, he_ltf, + WMI_VDEV_PARAM_HE_LTF); else ret = ath12k_mac_set_auto_rate_gi_ltf(arvif, he_gi, he_ltf); if (ret) return ret; - } else { + gi_ltf_set = true; + } + + if (!gi_ltf_set) { vdev_param = WMI_VDEV_PARAM_SGI; param_value = ath12k_mac_nlgi_to_wmigi(sgi); ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, @@ -12195,6 +12875,38 @@ ath12k_mac_he_mcs_range_present(struct ath12k *ar, return true; } +static bool +ath12k_mac_eht_mcs_range_present(struct ath12k *ar, + enum nl80211_band band, + const struct cfg80211_bitrate_mask *mask) +{ + u16 eht_mcs; + int i; + + for (i = 0; i < NL80211_EHT_NSS_MAX; i++) { + eht_mcs = mask->control[band].eht_mcs[i]; + + switch (eht_mcs) { + case 0: + case BIT(8) - 1: + case BIT(10) - 1: + case BIT(12) - 1: + case BIT(14) - 1: + break; + case BIT(15) - 1: + case BIT(16) - 1: + case BIT(16) - BIT(14) - 1: + if (i != 0) + return false; + break; + default: + return false; + } + } + + return true; +} + static void ath12k_mac_set_bitrate_mask_iter(void *data, struct ieee80211_sta *sta) { @@ -12249,15 +12961,18 @@ ath12k_mac_validate_fixed_rate_settings(struct ath12k *ar, enum nl80211_band ban const struct cfg80211_bitrate_mask *mask, unsigned int link_id) { - bool he_fixed_rate = false, vht_fixed_rate = false; - const u16 *vht_mcs_mask, *he_mcs_mask; + bool eht_fixed_rate = false, he_fixed_rate = false, vht_fixed_rate = false; + const u16 *vht_mcs_mask, *he_mcs_mask, *eht_mcs_mask; struct ieee80211_link_sta *link_sta; - struct ath12k_peer *peer, *tmp; - u8 vht_nss, he_nss; + struct ath12k_dp_link_peer *peer, *tmp; + u8 vht_nss, he_nss, eht_nss; int ret = true; + struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); vht_mcs_mask = mask->control[band].vht_mcs; he_mcs_mask = mask->control[band].he_mcs; + eht_mcs_mask = mask->control[band].eht_mcs; if (ath12k_mac_bitrate_mask_num_vht_rates(ar, band, mask) == 1) vht_fixed_rate = true; @@ -12265,15 +12980,19 @@ ath12k_mac_validate_fixed_rate_settings(struct ath12k *ar, enum nl80211_band ban if (ath12k_mac_bitrate_mask_num_he_rates(ar, band, mask) == 1) he_fixed_rate = true; - if (!vht_fixed_rate && !he_fixed_rate) + if (ath12k_mac_bitrate_mask_num_eht_rates(ar, band, mask) == 1) + eht_fixed_rate = true; + + if (!vht_fixed_rate && !he_fixed_rate && !eht_fixed_rate) return true; vht_nss = ath12k_mac_max_vht_nss(vht_mcs_mask); he_nss = ath12k_mac_max_he_nss(he_mcs_mask); + eht_nss = ath12k_mac_max_eht_nss(eht_mcs_mask); rcu_read_lock(); - spin_lock_bh(&ar->ab->base_lock); - list_for_each_entry_safe(peer, tmp, &ar->ab->peers, list) { + spin_lock_bh(&dp->dp_lock); + list_for_each_entry_safe(peer, tmp, &dp->peers, list) { if (peer->sta) { link_sta = rcu_dereference(peer->sta->link[link_id]); if (!link_sta) { @@ -12291,15 +13010,20 @@ ath12k_mac_validate_fixed_rate_settings(struct ath12k *ar, enum nl80211_band ban ret = false; goto exit; } + if (eht_fixed_rate && (!link_sta->eht_cap.has_eht || + link_sta->rx_nss < eht_nss)) { + ret = false; + goto exit; + } } } exit: - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); rcu_read_unlock(); return ret; } -static int +int ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif, const struct cfg80211_bitrate_mask *mask) @@ -12312,8 +13036,10 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, const u8 *ht_mcs_mask; const u16 *vht_mcs_mask; const u16 *he_mcs_mask; + const u16 *eht_mcs_mask; u8 he_ltf = 0; u8 he_gi = 0; + u8 eht_ltf = 0, eht_gi = 0; u32 rate; u8 nss, mac_nss; u8 sgi; @@ -12322,6 +13048,7 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, int ret; int num_rates; bool he_fixed_rate = false; + bool eht_fixed_rate = false; lockdep_assert_wiphy(hw->wiphy); @@ -12337,6 +13064,7 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, ht_mcs_mask = mask->control[band].ht_mcs; vht_mcs_mask = mask->control[band].vht_mcs; he_mcs_mask = mask->control[band].he_mcs; + eht_mcs_mask = mask->control[band].eht_mcs; ldpc = !!(ar->ht_cap_info & WMI_HT_CAP_LDPC); sgi = mask->control[band].gi; @@ -12348,6 +13076,9 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, he_gi = mask->control[band].he_gi; he_ltf = mask->control[band].he_ltf; + eht_gi = mask->control[band].eht_gi; + eht_ltf = mask->control[band].eht_ltf; + /* mac80211 doesn't support sending a fixed HT/VHT MCS alone, rather it * requires passing at least one of used basic rates along with them. * Fixed rate setting across different preambles(legacy, HT, VHT) is @@ -12385,9 +13116,10 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, ath12k_warn(ar->ab, "failed to update fixed rate settings due to mcs/nss incompatibility\n"); - mac_nss = max3(ath12k_mac_max_ht_nss(ht_mcs_mask), - ath12k_mac_max_vht_nss(vht_mcs_mask), - ath12k_mac_max_he_nss(he_mcs_mask)); + mac_nss = max(max3(ath12k_mac_max_ht_nss(ht_mcs_mask), + ath12k_mac_max_vht_nss(vht_mcs_mask), + ath12k_mac_max_he_nss(he_mcs_mask)), + ath12k_mac_max_eht_nss(eht_mcs_mask)); nss = min_t(u32, ar->num_tx_chains, mac_nss); /* If multiple rates across different preambles are given @@ -12435,6 +13167,20 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, ret = -EINVAL; goto out; } + + num_rates = ath12k_mac_bitrate_mask_num_eht_rates(ar, band, + mask); + if (num_rates == 1) + eht_fixed_rate = true; + + if (!ath12k_mac_eht_mcs_range_present(ar, band, mask) && + num_rates > 1) { + ath12k_warn(ar->ab, + "Setting more than one EHT MCS Value in bitrate mask not supported\n"); + ret = -EINVAL; + goto out; + } + ieee80211_iterate_stations_mtx(hw, ath12k_mac_disable_peer_fixed_rate, arvif); @@ -12446,7 +13192,8 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, } ret = ath12k_mac_set_rate_params(arvif, rate, nss, sgi, ldpc, he_gi, - he_ltf, he_fixed_rate); + he_ltf, he_fixed_rate, eht_gi, eht_ltf, + eht_fixed_rate); if (ret) { ath12k_warn(ar->ab, "failed to set rate params on vdev %i: %d\n", arvif->vdev_id, ret); @@ -12455,8 +13202,9 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, out: return ret; } +EXPORT_SYMBOL(ath12k_mac_op_set_bitrate_mask); -static void +void ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw, enum ieee80211_reconfig_type reconfig_type) { @@ -12517,7 +13265,7 @@ ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw, ahvif = arvif->ahvif; ath12k_dbg(ab, ATH12K_DBG_BOOT, "reconfig cipher %d up %d vdev type %d\n", - ahvif->key_cipher, + ahvif->dp_vif.key_cipher, arvif->is_up, ahvif->vdev_type); @@ -12537,6 +13285,7 @@ ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw, } } } +EXPORT_SYMBOL(ath12k_mac_op_reconfig_complete); static void ath12k_mac_update_bss_chan_survey(struct ath12k *ar, @@ -12570,12 +13319,13 @@ ath12k_mac_update_bss_chan_survey(struct ath12k *ar, ath12k_warn(ar->ab, "bss channel survey timed out\n"); } -static int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx, - struct survey_info *survey) +int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx, + struct survey_info *survey) { struct ath12k *ar; struct ieee80211_supported_band *sband; struct survey_info *ar_survey; + int orig_idx = idx; lockdep_assert_wiphy(hw->wiphy); @@ -12610,7 +13360,7 @@ static int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx, return -ENOENT; } - ar_survey = &ar->survey[idx]; + ar_survey = &ar->survey[orig_idx]; ath12k_mac_update_bss_chan_survey(ar, &sband->channels[idx]); @@ -12625,6 +13375,7 @@ static int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_get_survey); static void ath12k_mac_put_chain_rssi(struct station_info *sinfo, struct ath12k_link_sta *arsta) @@ -12647,15 +13398,18 @@ static void ath12k_mac_put_chain_rssi(struct station_info *sinfo, } } -static void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_sta *sta, - struct station_info *sinfo) +void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + struct station_info *sinfo) { struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta); + struct ath12k_dp_link_peer_rate_info rate_info = {}; struct ath12k_fw_stats_req_params params = {}; + struct ath12k_dp_link_peer *peer; struct ath12k_link_sta *arsta; s8 signal, noise_floor; + struct ath12k_dp *dp; struct ath12k *ar; bool db2dbm; @@ -12666,34 +13420,37 @@ static void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw, if (!ar) return; + dp = ath12k_ab_to_dp(ar->ab); + ath12k_dp_link_peer_get_sta_rate_info_stats(dp, arsta->addr, &rate_info); + db2dbm = test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, ar->ab->wmi_ab.svc_map); - sinfo->rx_duration = arsta->rx_duration; + sinfo->rx_duration = rate_info.rx_duration; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_DURATION); - sinfo->tx_duration = arsta->tx_duration; + sinfo->tx_duration = rate_info.tx_duration; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION); - if (arsta->txrate.legacy || arsta->txrate.nss) { - if (arsta->txrate.legacy) { - sinfo->txrate.legacy = arsta->txrate.legacy; + if (rate_info.txrate.legacy || rate_info.txrate.nss) { + if (rate_info.txrate.legacy) { + sinfo->txrate.legacy = rate_info.txrate.legacy; } else { - sinfo->txrate.mcs = arsta->txrate.mcs; - sinfo->txrate.nss = arsta->txrate.nss; - sinfo->txrate.bw = arsta->txrate.bw; - sinfo->txrate.he_gi = arsta->txrate.he_gi; - sinfo->txrate.he_dcm = arsta->txrate.he_dcm; - sinfo->txrate.he_ru_alloc = arsta->txrate.he_ru_alloc; - sinfo->txrate.eht_gi = arsta->txrate.eht_gi; - sinfo->txrate.eht_ru_alloc = arsta->txrate.eht_ru_alloc; - } - sinfo->txrate.flags = arsta->txrate.flags; + sinfo->txrate.mcs = rate_info.txrate.mcs; + sinfo->txrate.nss = rate_info.txrate.nss; + sinfo->txrate.bw = rate_info.txrate.bw; + sinfo->txrate.he_gi = rate_info.txrate.he_gi; + sinfo->txrate.he_dcm = rate_info.txrate.he_dcm; + sinfo->txrate.he_ru_alloc = rate_info.txrate.he_ru_alloc; + sinfo->txrate.eht_gi = rate_info.txrate.eht_gi; + sinfo->txrate.eht_ru_alloc = rate_info.txrate.eht_ru_alloc; + } + sinfo->txrate.flags = rate_info.txrate.flags; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); } /* TODO: Use real NF instead of default one. */ - signal = arsta->rssi_comb; + signal = rate_info.rssi_comb; params.pdev_id = ar->pdev->pdev_id; params.vdev_id = 0; @@ -12701,14 +13458,18 @@ static void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw, if (!signal && ahsta->ahvif->vdev_type == WMI_VDEV_TYPE_STA && - !(ath12k_mac_get_fw_stats(ar, ¶ms))) + !(ath12k_mac_get_fw_stats(ar, ¶ms))) { signal = arsta->rssi_beacon; + ath12k_fw_stats_reset(ar); + } params.stats_id = WMI_REQUEST_RSSI_PER_CHAIN_STAT; if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL)) && ahsta->ahvif->vdev_type == WMI_VDEV_TYPE_STA && - !(ath12k_mac_get_fw_stats(ar, ¶ms))) + !(ath12k_mac_get_fw_stats(ar, ¶ms))) { ath12k_mac_put_chain_rssi(sinfo, arsta); + ath12k_fw_stats_reset(ar); + } spin_lock_bh(&ar->data_lock); noise_floor = ath12k_pdev_get_noise_floor(ar); @@ -12719,26 +13480,37 @@ static void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw, sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); } - sinfo->signal_avg = ewma_avg_rssi_read(&arsta->avg_rssi); + sinfo->signal_avg = rate_info.signal_avg; if (!db2dbm) sinfo->signal_avg += noise_floor; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG); - sinfo->tx_retries = arsta->tx_retry_count; - sinfo->tx_failed = arsta->tx_retry_failed; + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_addr(dp, arsta->addr); + if (!peer) { + spin_unlock_bh(&dp->dp_lock); + return; + } + + sinfo->tx_retries = peer->tx_retry_count; + sinfo->tx_failed = peer->tx_retry_failed; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES); sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); + + spin_unlock_bh(&dp->dp_lock); } +EXPORT_SYMBOL(ath12k_mac_op_sta_statistics); -static void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_link_sta *link_sta, - struct link_station_info *link_sinfo) +void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + struct link_station_info *link_sinfo) { struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(link_sta->sta); struct ath12k_fw_stats_req_params params = {}; + struct ath12k_dp_link_peer *peer; struct ath12k_link_sta *arsta; struct ath12k *ar; s8 signal; @@ -12758,64 +13530,76 @@ static void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw, db2dbm = test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, ar->ab->wmi_ab.svc_map); - link_sinfo->rx_duration = arsta->rx_duration; + spin_lock_bh(&ar->ab->dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_addr(ar->ab->dp, arsta->addr); + if (!peer) { + spin_unlock_bh(&ar->ab->dp->dp_lock); + return; + } + + link_sinfo->rx_duration = peer->rx_duration; link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_DURATION); - link_sinfo->tx_duration = arsta->tx_duration; + link_sinfo->tx_duration = peer->tx_duration; link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION); - if (arsta->txrate.legacy || arsta->txrate.nss) { - if (arsta->txrate.legacy) { - link_sinfo->txrate.legacy = arsta->txrate.legacy; + if (peer->txrate.legacy || peer->txrate.nss) { + if (peer->txrate.legacy) { + link_sinfo->txrate.legacy = peer->txrate.legacy; } else { - link_sinfo->txrate.mcs = arsta->txrate.mcs; - link_sinfo->txrate.nss = arsta->txrate.nss; - link_sinfo->txrate.bw = arsta->txrate.bw; - link_sinfo->txrate.he_gi = arsta->txrate.he_gi; - link_sinfo->txrate.he_dcm = arsta->txrate.he_dcm; + link_sinfo->txrate.mcs = peer->txrate.mcs; + link_sinfo->txrate.nss = peer->txrate.nss; + link_sinfo->txrate.bw = peer->txrate.bw; + link_sinfo->txrate.he_gi = peer->txrate.he_gi; + link_sinfo->txrate.he_dcm = peer->txrate.he_dcm; link_sinfo->txrate.he_ru_alloc = - arsta->txrate.he_ru_alloc; - link_sinfo->txrate.eht_gi = arsta->txrate.eht_gi; + peer->txrate.he_ru_alloc; + link_sinfo->txrate.eht_gi = peer->txrate.eht_gi; link_sinfo->txrate.eht_ru_alloc = - arsta->txrate.eht_ru_alloc; + peer->txrate.eht_ru_alloc; } - link_sinfo->txrate.flags = arsta->txrate.flags; + link_sinfo->txrate.flags = peer->txrate.flags; link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); } + link_sinfo->signal_avg = ewma_avg_rssi_read(&peer->avg_rssi); + + if (!db2dbm) + link_sinfo->signal_avg += ATH12K_DEFAULT_NOISE_FLOOR; + + link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG); + + link_sinfo->tx_retries = peer->tx_retry_count; + link_sinfo->tx_failed = peer->tx_retry_failed; + link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES); + link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); + /* TODO: Use real NF instead of default one. */ - signal = arsta->rssi_comb; + signal = peer->rssi_comb; - params.pdev_id = ar->pdev->pdev_id; - params.vdev_id = 0; - params.stats_id = WMI_REQUEST_VDEV_STAT; + spin_unlock_bh(&ar->ab->dp->dp_lock); - if (!signal && - ahsta->ahvif->vdev_type == WMI_VDEV_TYPE_STA && - !(ath12k_mac_get_fw_stats(ar, ¶ms))) - signal = arsta->rssi_beacon; + if (!signal && ahsta->ahvif->vdev_type == WMI_VDEV_TYPE_STA) { + params.pdev_id = ar->pdev->pdev_id; + params.vdev_id = 0; + params.stats_id = WMI_REQUEST_VDEV_STAT; + + if (!ath12k_mac_get_fw_stats(ar, ¶ms)) { + signal = arsta->rssi_beacon; + ath12k_fw_stats_reset(ar); + } + } if (signal) { link_sinfo->signal = db2dbm ? signal : signal + ATH12K_DEFAULT_NOISE_FLOOR; link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); } - - link_sinfo->signal_avg = ewma_avg_rssi_read(&arsta->avg_rssi); - - if (!db2dbm) - link_sinfo->signal_avg += ATH12K_DEFAULT_NOISE_FLOOR; - - link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG); - - link_sinfo->tx_retries = arsta->tx_retry_count; - link_sinfo->tx_failed = arsta->tx_retry_failed; - link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES); - link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); } +EXPORT_SYMBOL(ath12k_mac_op_link_sta_statistics); -static int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw, - struct ieee80211_vif *vif) +int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) { struct ath12k_hw *ah = ath12k_hw_to_ah(hw); struct ath12k *ar; @@ -12831,16 +13615,17 @@ static int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw, ath12k_scan_abort(ar); cancel_delayed_work_sync(&ar->scan.timeout); - wiphy_work_cancel(hw->wiphy, &ar->scan.vdev_clean_wk); + wiphy_work_flush(hw->wiphy, &ar->scan.vdev_clean_wk); return 0; } +EXPORT_SYMBOL(ath12k_mac_op_cancel_remain_on_channel); -static int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_channel *chan, - int duration, - enum ieee80211_roc_type type) +int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel *chan, + int duration, + enum ieee80211_roc_type type) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_hw *ah = ath12k_hw_to_ah(hw); @@ -12895,6 +13680,7 @@ static int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw, if (ret) { ath12k_warn(ar->ab, "unable to create scan vdev for roc: %d\n", ret); + ath12k_mac_unassign_link_vif(arvif); return ret; } } @@ -12974,10 +13760,11 @@ static int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw, return 0; } +EXPORT_SYMBOL(ath12k_mac_op_remain_on_channel); -static void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct cfg80211_gtk_rekey_data *data) +void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct cfg80211_gtk_rekey_data *data) { struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); struct ath12k_rekey_data *rekey_data; @@ -13010,63 +13797,7 @@ static void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw, ath12k_dbg_dump(ar->ab, ATH12K_DBG_MAC, "replay ctr", NULL, &rekey_data->replay_ctr, sizeof(rekey_data->replay_ctr)); } - -static const struct ieee80211_ops ath12k_ops = { - .tx = ath12k_mac_op_tx, - .wake_tx_queue = ieee80211_handle_wake_tx_queue, - .start = ath12k_mac_op_start, - .stop = ath12k_mac_op_stop, - .reconfig_complete = ath12k_mac_op_reconfig_complete, - .add_interface = ath12k_mac_op_add_interface, - .remove_interface = ath12k_mac_op_remove_interface, - .update_vif_offload = ath12k_mac_op_update_vif_offload, - .config = ath12k_mac_op_config, - .link_info_changed = ath12k_mac_op_link_info_changed, - .vif_cfg_changed = ath12k_mac_op_vif_cfg_changed, - .change_vif_links = ath12k_mac_op_change_vif_links, - .configure_filter = ath12k_mac_op_configure_filter, - .hw_scan = ath12k_mac_op_hw_scan, - .cancel_hw_scan = ath12k_mac_op_cancel_hw_scan, - .set_key = ath12k_mac_op_set_key, - .set_rekey_data = ath12k_mac_op_set_rekey_data, - .sta_state = ath12k_mac_op_sta_state, - .sta_set_txpwr = ath12k_mac_op_sta_set_txpwr, - .link_sta_rc_update = ath12k_mac_op_link_sta_rc_update, - .conf_tx = ath12k_mac_op_conf_tx, - .set_antenna = ath12k_mac_op_set_antenna, - .get_antenna = ath12k_mac_op_get_antenna, - .ampdu_action = ath12k_mac_op_ampdu_action, - .add_chanctx = ath12k_mac_op_add_chanctx, - .remove_chanctx = ath12k_mac_op_remove_chanctx, - .change_chanctx = ath12k_mac_op_change_chanctx, - .assign_vif_chanctx = ath12k_mac_op_assign_vif_chanctx, - .unassign_vif_chanctx = ath12k_mac_op_unassign_vif_chanctx, - .switch_vif_chanctx = ath12k_mac_op_switch_vif_chanctx, - .get_txpower = ath12k_mac_op_get_txpower, - .set_rts_threshold = ath12k_mac_op_set_rts_threshold, - .set_frag_threshold = ath12k_mac_op_set_frag_threshold, - .set_bitrate_mask = ath12k_mac_op_set_bitrate_mask, - .get_survey = ath12k_mac_op_get_survey, - .flush = ath12k_mac_op_flush, - .sta_statistics = ath12k_mac_op_sta_statistics, - .link_sta_statistics = ath12k_mac_op_link_sta_statistics, - .remain_on_channel = ath12k_mac_op_remain_on_channel, - .cancel_remain_on_channel = ath12k_mac_op_cancel_remain_on_channel, - .change_sta_links = ath12k_mac_op_change_sta_links, - .can_activate_links = ath12k_mac_op_can_activate_links, -#ifdef CONFIG_PM - .suspend = ath12k_wow_op_suspend, - .resume = ath12k_wow_op_resume, - .set_wakeup = ath12k_wow_op_set_wakeup, -#endif -#ifdef CONFIG_ATH12K_DEBUGFS - .vif_add_debugfs = ath12k_debugfs_op_vif_add, -#endif - CFG80211_TESTMODE_CMD(ath12k_tm_cmd) -#ifdef CONFIG_ATH12K_DEBUGFS - .link_sta_add_debugfs = ath12k_debugfs_link_sta_op_add, -#endif -}; +EXPORT_SYMBOL(ath12k_mac_op_set_rekey_data); void ath12k_mac_update_freq_range(struct ath12k *ar, u32 freq_low, u32 freq_high) @@ -13894,6 +14625,11 @@ static int ath12k_mac_hw_register(struct ath12k_hw *ah) wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_STA_TX_PWR); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT); + if (test_bit(WMI_TLV_SERVICE_BSS_COLOR_OFFLOAD, + ab->wmi_ab.svc_map)) { + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BSS_COLOR); + ieee80211_hw_set(hw, DETECTS_COLOR_COLLISION); + } wiphy->cipher_suites = cipher_suites; wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); @@ -14040,8 +14776,9 @@ static void ath12k_mac_setup(struct ath12k *ar) ar->vdev_id_11d_scan = ATH12K_11D_INVALID_VDEV_ID; spin_lock_init(&ar->data_lock); + spin_lock_init(&ar->dp.ppdu_list_lock); INIT_LIST_HEAD(&ar->arvifs); - INIT_LIST_HEAD(&ar->ppdu_stats_info); + INIT_LIST_HEAD(&ar->dp.ppdu_stats_info); init_completion(&ar->vdev_setup_done); init_completion(&ar->vdev_delete_done); @@ -14280,7 +15017,7 @@ static struct ath12k_hw *ath12k_mac_hw_allocate(struct ath12k_hw_group *ag, u8 pdev_idx; hw = ieee80211_alloc_hw(struct_size(ah, radio, num_pdev_map), - &ath12k_ops); + pdev_map->ab->ath12k_ops); if (!hw) return NULL; @@ -14289,7 +15026,9 @@ static struct ath12k_hw *ath12k_mac_hw_allocate(struct ath12k_hw_group *ag, ah->num_radio = num_pdev_map; mutex_init(&ah->hw_mutex); - INIT_LIST_HEAD(&ah->ml_peers); + + spin_lock_init(&ah->dp_hw.peer_lock); + INIT_LIST_HEAD(&ah->dp_hw.dp_peers_list); for (i = 0; i < num_pdev_map; i++) { ab = pdev_map[i].ab; diff --git a/drivers/net/wireless/ath/ath12k/mac.h b/drivers/net/wireless/ath/ath12k/mac.h index c05af40bd7a20..7b50c59763840 100644 --- a/drivers/net/wireless/ath/ath12k/mac.h +++ b/drivers/net/wireless/ath/ath12k/mac.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_MAC_H @@ -54,9 +54,6 @@ struct ath12k_generic_iter { * for driver usage purpose. */ #define ATH12K_FIRST_SCAN_LINK IEEE80211_MLD_MAX_NUM_LINKS -#define ATH12K_SCAN_MAX_LINKS ATH12K_GROUP_MAX_RADIO -/* Define 1 scan link for each radio for parallel scan purposes */ -#define ATH12K_NUM_MAX_LINKS (IEEE80211_MLD_MAX_NUM_LINKS + ATH12K_SCAN_MAX_LINKS) #define ATH12K_SCAN_LINKS_MASK GENMASK(ATH12K_NUM_MAX_LINKS, IEEE80211_MLD_MAX_NUM_LINKS) #define ATH12K_NUM_MAX_ACTIVE_LINKS_PER_DEVICE 2 @@ -84,6 +81,18 @@ enum ath12k_supported_bw { ATH12K_BW_320 = 4, }; +enum ath12k_gi { + ATH12K_RATE_INFO_GI_0_8, + ATH12K_RATE_INFO_GI_1_6, + ATH12K_RATE_INFO_GI_3_2, +}; + +enum ath12k_ltf { + ATH12K_RATE_INFO_1XLTF, + ATH12K_RATE_INFO_2XLTF, + ATH12K_RATE_INFO_4XLTF, +}; + struct ath12k_mac_get_any_chanctx_conf_arg { struct ath12k *ar; struct ieee80211_chanctx_conf *chanctx_conf; @@ -129,6 +138,9 @@ struct ath12k_reg_tpc_power_info { struct ath12k_chan_power_info chan_power_info[ATH12K_NUM_PWR_LEVELS]; }; +#define ATH12K_OBSS_PD_MAX_THRESHOLD -82 +#define ATH12K_OBSS_PD_NON_SRG_MAX_THRESHOLD -62 + extern const struct htt_rx_ring_tlv_filter ath12k_mac_mon_status_filter_default; #define ATH12K_SCAN_11D_INTERVAL 600000 @@ -160,6 +172,7 @@ struct ath12k *ath12k_mac_get_ar_by_pdev_id(struct ath12k_base *ab, u32 pdev_id) void ath12k_mac_drain_tx(struct ath12k *ar); void ath12k_mac_peer_cleanup_all(struct ath12k *ar); +void ath12k_mac_dp_peer_cleanup(struct ath12k_hw *ah); int ath12k_mac_tx_mgmt_pending_free(int buf_id, void *skb, void *ctx); enum rate_info_bw ath12k_mac_bw_to_mac80211_bw(enum ath12k_supported_bw bw); enum ath12k_supported_bw ath12k_mac_mac80211_bw_to_ath12k_bw(enum rate_info_bw bw); @@ -194,4 +207,139 @@ void ath12k_mac_update_freq_range(struct ath12k *ar, void ath12k_mac_fill_reg_tpc_info(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ieee80211_chanctx_conf *ctx); +int ath12k_mac_op_start(struct ieee80211_hw *hw); +void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend); +void +ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw, + enum ieee80211_reconfig_type reconfig_type); +int ath12k_mac_op_add_interface(struct ieee80211_hw *hw, + struct ieee80211_vif *vif); +void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw, + struct ieee80211_vif *vif); +void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw, + struct ieee80211_vif *vif); +int ath12k_mac_op_config(struct ieee80211_hw *hw, int radio_idx, u32 changed); +void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *info, + u64 changed); +void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + u64 changed); +int +ath12k_mac_op_change_vif_links + (struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + u16 old_links, u16 new_links, + struct ieee80211_bss_conf *ol[IEEE80211_MLD_MAX_NUM_LINKS]); +void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw, + unsigned int changed_flags, + unsigned int *total_flags, + u64 multicast); +int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_scan_request *hw_req); +void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw, + struct ieee80211_vif *vif); +int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, + struct ieee80211_vif *vif, struct ieee80211_sta *sta, + struct ieee80211_key_conf *key); +void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct cfg80211_gtk_rekey_data *data); +int ath12k_mac_op_sta_state(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + enum ieee80211_sta_state old_state, + enum ieee80211_sta_state new_state); +int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta); +void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + u32 changed); +int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + unsigned int link_id, u16 ac, + const struct ieee80211_tx_queue_params *params); +int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx, + u32 tx_ant, u32 rx_ant); +int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx, + u32 *tx_ant, u32 *rx_ant); +int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_ampdu_params *params); +int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx); +void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx); +void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx, + u32 changed); +int +ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx_conf *ctx); +void +ath12k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx_conf *ctx); +int +ath12k_mac_op_switch_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif_chanctx_switch *vifs, + int n_vifs, + enum ieee80211_chanctx_switch_mode mode); +int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw, + int radio_idx, u32 value); +int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw, + int radio_idx, u32 value); +int +ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + const struct cfg80211_bitrate_mask *mask); +int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx, + struct survey_info *survey); +void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + u32 queues, bool drop); +void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + struct station_info *sinfo); +void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + struct link_station_info *link_sinfo); +int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel *chan, + int duration, + enum ieee80211_roc_type type); +int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw, + struct ieee80211_vif *vif); +int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + u16 old_links, u16 new_links); +bool ath12k_mac_op_can_activate_links(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + u16 active_links); +int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + unsigned int link_id, + int *dbm); +int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb, + bool is_prb_rsp); +void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar, + struct ieee80211_vif *vif, + struct sk_buff *skb, + bool is_prb_rsp); +u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif *vif, + u8 link, struct sk_buff *skb, u32 info_flags); + +void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif, + u8 link_id, struct sk_buff *skb, + u32 info_flags); #endif diff --git a/drivers/net/wireless/ath/ath12k/mhi.c b/drivers/net/wireless/ath/ath12k/mhi.c index 08f44baf182a5..45c0f66dcc5ea 100644 --- a/drivers/net/wireless/ath/ath12k/mhi.c +++ b/drivers/net/wireless/ath/ath12k/mhi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -18,136 +18,6 @@ #define OTP_VALID_DUALMAC_BOARD_ID_MASK 0x1000 #define MHI_CB_INVALID 0xff -static const struct mhi_channel_config ath12k_mhi_channels_qcn9274[] = { - { - .num = 20, - .name = "IPCR", - .num_elements = 32, - .event_ring = 1, - .dir = DMA_TO_DEVICE, - .ee_mask = 0x4, - .pollcfg = 0, - .doorbell = MHI_DB_BRST_DISABLE, - .lpm_notify = false, - .offload_channel = false, - .doorbell_mode_switch = false, - .auto_queue = false, - }, - { - .num = 21, - .name = "IPCR", - .num_elements = 32, - .event_ring = 1, - .dir = DMA_FROM_DEVICE, - .ee_mask = 0x4, - .pollcfg = 0, - .doorbell = MHI_DB_BRST_DISABLE, - .lpm_notify = false, - .offload_channel = false, - .doorbell_mode_switch = false, - .auto_queue = true, - }, -}; - -static struct mhi_event_config ath12k_mhi_events_qcn9274[] = { - { - .num_elements = 32, - .irq_moderation_ms = 0, - .irq = 1, - .data_type = MHI_ER_CTRL, - .mode = MHI_DB_BRST_DISABLE, - .hardware_event = false, - .client_managed = false, - .offload_channel = false, - }, - { - .num_elements = 256, - .irq_moderation_ms = 1, - .irq = 2, - .mode = MHI_DB_BRST_DISABLE, - .priority = 1, - .hardware_event = false, - .client_managed = false, - .offload_channel = false, - }, -}; - -const struct mhi_controller_config ath12k_mhi_config_qcn9274 = { - .max_channels = 30, - .timeout_ms = 10000, - .use_bounce_buf = false, - .buf_len = 0, - .num_channels = ARRAY_SIZE(ath12k_mhi_channels_qcn9274), - .ch_cfg = ath12k_mhi_channels_qcn9274, - .num_events = ARRAY_SIZE(ath12k_mhi_events_qcn9274), - .event_cfg = ath12k_mhi_events_qcn9274, -}; - -static const struct mhi_channel_config ath12k_mhi_channels_wcn7850[] = { - { - .num = 20, - .name = "IPCR", - .num_elements = 64, - .event_ring = 1, - .dir = DMA_TO_DEVICE, - .ee_mask = 0x4, - .pollcfg = 0, - .doorbell = MHI_DB_BRST_DISABLE, - .lpm_notify = false, - .offload_channel = false, - .doorbell_mode_switch = false, - .auto_queue = false, - }, - { - .num = 21, - .name = "IPCR", - .num_elements = 64, - .event_ring = 1, - .dir = DMA_FROM_DEVICE, - .ee_mask = 0x4, - .pollcfg = 0, - .doorbell = MHI_DB_BRST_DISABLE, - .lpm_notify = false, - .offload_channel = false, - .doorbell_mode_switch = false, - .auto_queue = true, - }, -}; - -static struct mhi_event_config ath12k_mhi_events_wcn7850[] = { - { - .num_elements = 32, - .irq_moderation_ms = 0, - .irq = 1, - .mode = MHI_DB_BRST_DISABLE, - .data_type = MHI_ER_CTRL, - .hardware_event = false, - .client_managed = false, - .offload_channel = false, - }, - { - .num_elements = 256, - .irq_moderation_ms = 1, - .irq = 2, - .mode = MHI_DB_BRST_DISABLE, - .priority = 1, - .hardware_event = false, - .client_managed = false, - .offload_channel = false, - }, -}; - -const struct mhi_controller_config ath12k_mhi_config_wcn7850 = { - .max_channels = 128, - .timeout_ms = 2000, - .use_bounce_buf = false, - .buf_len = 8192, - .num_channels = ARRAY_SIZE(ath12k_mhi_channels_wcn7850), - .ch_cfg = ath12k_mhi_channels_wcn7850, - .num_events = ARRAY_SIZE(ath12k_mhi_events_wcn7850), - .event_cfg = ath12k_mhi_events_wcn7850, -}; - void ath12k_mhi_set_mhictrl_reset(struct ath12k_base *ab) { u32 val; diff --git a/drivers/net/wireless/ath/ath12k/mhi.h b/drivers/net/wireless/ath/ath12k/mhi.h index 7358b8477536a..3674326763858 100644 --- a/drivers/net/wireless/ath/ath12k/mhi.h +++ b/drivers/net/wireless/ath/ath12k/mhi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef _ATH12K_MHI_H #define _ATH12K_MHI_H @@ -31,9 +31,6 @@ enum ath12k_mhi_state { ATH12K_MHI_RDDM_DONE, }; -extern const struct mhi_controller_config ath12k_mhi_config_qcn9274; -extern const struct mhi_controller_config ath12k_mhi_config_wcn7850; - int ath12k_mhi_start(struct ath12k_pci *ar_pci); void ath12k_mhi_stop(struct ath12k_pci *ar_pci, bool is_suspend); int ath12k_mhi_register(struct ath12k_pci *ar_pci); diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c index c729d5526c753..375277ca2b892 100644 --- a/drivers/net/wireless/ath/ath12k/pci.c +++ b/drivers/net/wireless/ath/ath12k/pci.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -15,6 +15,7 @@ #include "hif.h" #include "mhi.h" #include "debug.h" +#include "hal.h" #define ATH12K_PCI_BAR_NUM 0 #define ATH12K_PCI_DMA_MASK 36 @@ -22,50 +23,18 @@ #define ATH12K_PCI_IRQ_CE0_OFFSET 3 #define WINDOW_ENABLE_BIT 0x40000000 -#define WINDOW_REG_ADDRESS 0x310c #define WINDOW_VALUE_MASK GENMASK(24, 19) #define WINDOW_START 0x80000 #define WINDOW_RANGE_MASK GENMASK(18, 0) #define WINDOW_STATIC_MASK GENMASK(31, 6) -#define TCSR_SOC_HW_VERSION 0x1B00000 -#define TCSR_SOC_HW_VERSION_MAJOR_MASK GENMASK(11, 8) -#define TCSR_SOC_HW_VERSION_MINOR_MASK GENMASK(7, 4) - /* BAR0 + 4k is always accessible, and no * need to force wakeup. * 4K - 32 = 0xFE0 */ #define ACCESS_ALWAYS_OFF 0xFE0 -#define QCN9274_DEVICE_ID 0x1109 -#define WCN7850_DEVICE_ID 0x1107 - -#define PCIE_LOCAL_REG_QRTR_NODE_ID 0x1E03164 -#define DOMAIN_NUMBER_MASK GENMASK(7, 4) -#define BUS_NUMBER_MASK GENMASK(3, 0) - -static const struct pci_device_id ath12k_pci_id_table[] = { - { PCI_VDEVICE(QCOM, QCN9274_DEVICE_ID) }, - { PCI_VDEVICE(QCOM, WCN7850_DEVICE_ID) }, - {} -}; - -MODULE_DEVICE_TABLE(pci, ath12k_pci_id_table); - -/* TODO: revisit IRQ mapping for new SRNG's */ -static const struct ath12k_msi_config ath12k_msi_config[] = { - { - .total_vectors = 16, - .total_users = 3, - .users = (struct ath12k_msi_user[]) { - { .name = "MHI", .num_vectors = 3, .base_vector = 0 }, - { .name = "CE", .num_vectors = 5, .base_vector = 3 }, - { .name = "DP", .num_vectors = 8, .base_vector = 8 }, - }, - }, -}; - +static struct ath12k_pci_driver *ath12k_pci_family_drivers[ATH12K_DEVICE_FAMILY_MAX]; static const struct ath12k_msi_config msi_config_one_msi = { .total_vectors = 1, .total_users = 4, @@ -136,30 +105,6 @@ static const char *irq_name[ATH12K_IRQ_NUM_MAX] = { "tcl2host-status-ring", }; -static int ath12k_pci_bus_wake_up(struct ath12k_base *ab) -{ - struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); - - return mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev); -} - -static void ath12k_pci_bus_release(struct ath12k_base *ab) -{ - struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); - - mhi_device_put(ab_pci->mhi_ctrl->mhi_dev); -} - -static const struct ath12k_pci_ops ath12k_pci_ops_qcn9274 = { - .wakeup = NULL, - .release = NULL, -}; - -static const struct ath12k_pci_ops ath12k_pci_ops_wcn7850 = { - .wakeup = ath12k_pci_bus_wake_up, - .release = ath12k_pci_bus_release, -}; - static void ath12k_pci_select_window(struct ath12k_pci *ab_pci, u32 offset) { struct ath12k_base *ab = ab_pci->ab; @@ -175,37 +120,40 @@ static void ath12k_pci_select_window(struct ath12k_pci *ab_pci, u32 offset) if (window != ab_pci->register_window) { iowrite32(WINDOW_ENABLE_BIT | window, - ab->mem + WINDOW_REG_ADDRESS); - ioread32(ab->mem + WINDOW_REG_ADDRESS); + ab->mem + ab_pci->window_reg_addr); + ioread32(ab->mem + ab_pci->window_reg_addr); ab_pci->register_window = window; } } static void ath12k_pci_select_static_window(struct ath12k_pci *ab_pci) { - u32 umac_window = u32_get_bits(HAL_SEQ_WCSS_UMAC_OFFSET, WINDOW_VALUE_MASK); - u32 ce_window = u32_get_bits(HAL_CE_WFSS_CE_REG_BASE, WINDOW_VALUE_MASK); + u32 umac_window; + u32 ce_window; u32 window; + umac_window = u32_get_bits(ab_pci->reg_base->umac_base, WINDOW_VALUE_MASK); + ce_window = u32_get_bits(ab_pci->reg_base->ce_reg_base, WINDOW_VALUE_MASK); window = (umac_window << 12) | (ce_window << 6); spin_lock_bh(&ab_pci->window_lock); ab_pci->register_window = window; spin_unlock_bh(&ab_pci->window_lock); - iowrite32(WINDOW_ENABLE_BIT | window, ab_pci->ab->mem + WINDOW_REG_ADDRESS); + iowrite32(WINDOW_ENABLE_BIT | window, ab_pci->ab->mem + ab_pci->window_reg_addr); } static u32 ath12k_pci_get_window_start(struct ath12k_base *ab, u32 offset) { + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); u32 window_start; /* If offset lies within DP register range, use 3rd window */ - if ((offset ^ HAL_SEQ_WCSS_UMAC_OFFSET) < WINDOW_RANGE_MASK) + if ((offset ^ ab_pci->reg_base->umac_base) < WINDOW_RANGE_MASK) window_start = 3 * WINDOW_START; /* If offset lies within CE register range, use 2nd window */ - else if ((offset ^ HAL_CE_WFSS_CE_REG_BASE) < WINDOW_RANGE_MASK) + else if ((offset ^ ab_pci->reg_base->ce_reg_base) < WINDOW_RANGE_MASK) window_start = 2 * WINDOW_START; else window_start = WINDOW_START; @@ -218,6 +166,19 @@ static inline bool ath12k_pci_is_offset_within_mhi_region(u32 offset) return (offset >= PCI_MHIREGLEN_REG && offset <= PCI_MHI_REGION_END); } +static void ath12k_pci_restore_window(struct ath12k_base *ab) +{ + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); + + spin_lock_bh(&ab_pci->window_lock); + + iowrite32(WINDOW_ENABLE_BIT | ab_pci->register_window, + ab->mem + ab_pci->window_reg_addr); + ioread32(ab->mem + ab_pci->window_reg_addr); + + spin_unlock_bh(&ab_pci->window_lock); +} + static void ath12k_pci_soc_global_reset(struct ath12k_base *ab) { u32 val, delay; @@ -242,6 +203,11 @@ static void ath12k_pci_soc_global_reset(struct ath12k_base *ab) val = ath12k_pci_read32(ab, PCIE_SOC_GLOBAL_RESET); if (val == 0xffffffff) ath12k_warn(ab, "link down error during global reset\n"); + + /* Restore window register as its content is cleared during + * hardware global reset, such that it aligns with host cache. + */ + ath12k_pci_restore_window(ab); } static void ath12k_pci_clear_dbg_registers(struct ath12k_base *ab) @@ -526,10 +492,11 @@ static int ath12k_pci_ext_grp_napi_poll(struct napi_struct *napi, int budget) struct ath12k_ext_irq_grp, napi); struct ath12k_base *ab = irq_grp->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); int work_done; int i; - work_done = ath12k_dp_service_srng(ab, irq_grp, budget); + work_done = ath12k_dp_service_srng(dp, irq_grp, budget); if (work_done < budget) { napi_complete_done(napi, work_done); for (i = 0; i < irq_grp->num_irq; i++) @@ -947,7 +914,7 @@ static void ath12k_pci_update_qrtr_node_id(struct ath12k_base *ab) * writes to the given register, it is available for firmware when the QMI service * is spawned. */ - reg = PCIE_LOCAL_REG_QRTR_NODE_ID & WINDOW_RANGE_MASK; + reg = PCIE_LOCAL_REG_QRTR_NODE_ID(ab) & WINDOW_RANGE_MASK; ath12k_pci_write32(ab, reg, ab_pci->qmi_instance); ath12k_dbg(ab, ATH12K_DBG_PCI, "pci reg 0x%x instance 0x%x read val 0x%x\n", @@ -1226,6 +1193,7 @@ u32 ath12k_pci_read32(struct ath12k_base *ab, u32 offset) ab_pci->pci_ops->release(ab); return val; } +EXPORT_SYMBOL(ath12k_pci_read32); void ath12k_pci_write32(struct ath12k_base *ab, u32 offset, u32 value) { @@ -1531,28 +1499,34 @@ static const struct ath12k_hif_ops ath12k_pci_hif_ops = { #endif }; -static -void ath12k_pci_read_hw_version(struct ath12k_base *ab, u32 *major, u32 *minor) +static enum ath12k_device_family +ath12k_get_device_family(const struct pci_device_id *pci_dev) { - u32 soc_hw_version; + enum ath12k_device_family device_family_id; + const struct pci_device_id *id; - soc_hw_version = ath12k_pci_read32(ab, TCSR_SOC_HW_VERSION); - *major = FIELD_GET(TCSR_SOC_HW_VERSION_MAJOR_MASK, - soc_hw_version); - *minor = FIELD_GET(TCSR_SOC_HW_VERSION_MINOR_MASK, - soc_hw_version); + for (device_family_id = ATH12K_DEVICE_FAMILY_START; + device_family_id < ATH12K_DEVICE_FAMILY_MAX; device_family_id++) { + if (!ath12k_pci_family_drivers[device_family_id]) + continue; - ath12k_dbg(ab, ATH12K_DBG_PCI, - "pci tcsr_soc_hw_version major %d minor %d\n", - *major, *minor); + id = ath12k_pci_family_drivers[device_family_id]->id_table; + while (id->device) { + if (id->device == pci_dev->device) + return device_family_id; + id += 1; + } + } + + return ATH12K_DEVICE_FAMILY_MAX; } static int ath12k_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pci_dev) { - struct ath12k_base *ab; + enum ath12k_device_family device_id; struct ath12k_pci *ab_pci; - u32 soc_hw_version_major, soc_hw_version_minor; + struct ath12k_base *ab; int ret; ab = ath12k_core_alloc(&pdev->dev, sizeof(*ab_pci), ATH12K_BUS_PCI); @@ -1562,7 +1536,6 @@ static int ath12k_pci_probe(struct pci_dev *pdev, } ab->dev = &pdev->dev; - pci_set_drvdata(pdev, ab); ab_pci = ath12k_pci_priv(ab); ab_pci->dev_id = pci_dev->device; ab_pci->ab = ab; @@ -1587,56 +1560,25 @@ static int ath12k_pci_probe(struct pci_dev *pdev, ab->id.subsystem_vendor = pdev->subsystem_vendor; ab->id.subsystem_device = pdev->subsystem_device; - switch (pci_dev->device) { - case QCN9274_DEVICE_ID: - ab_pci->msi_config = &ath12k_msi_config[0]; - ab->static_window_map = true; - ab_pci->pci_ops = &ath12k_pci_ops_qcn9274; - ab->hal_rx_ops = &hal_rx_qcn9274_ops; - ath12k_pci_read_hw_version(ab, &soc_hw_version_major, - &soc_hw_version_minor); - ab->target_mem_mode = ath12k_core_get_memory_mode(ab); - switch (soc_hw_version_major) { - case ATH12K_PCI_SOC_HW_VERSION_2: - ab->hw_rev = ATH12K_HW_QCN9274_HW20; - break; - case ATH12K_PCI_SOC_HW_VERSION_1: - ab->hw_rev = ATH12K_HW_QCN9274_HW10; - break; - default: - dev_err(&pdev->dev, - "Unknown hardware version found for QCN9274: 0x%x\n", - soc_hw_version_major); - ret = -EOPNOTSUPP; - goto err_pci_free_region; - } - break; - case WCN7850_DEVICE_ID: - ab->id.bdf_search = ATH12K_BDF_SEARCH_BUS_AND_BOARD; - ab_pci->msi_config = &ath12k_msi_config[0]; - ab->static_window_map = false; - ab_pci->pci_ops = &ath12k_pci_ops_wcn7850; - ab->hal_rx_ops = &hal_rx_wcn7850_ops; - ath12k_pci_read_hw_version(ab, &soc_hw_version_major, - &soc_hw_version_minor); - ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT; - switch (soc_hw_version_major) { - case ATH12K_PCI_SOC_HW_VERSION_2: - ab->hw_rev = ATH12K_HW_WCN7850_HW20; - break; - default: - dev_err(&pdev->dev, - "Unknown hardware version found for WCN7850: 0x%x\n", - soc_hw_version_major); - ret = -EOPNOTSUPP; - goto err_pci_free_region; - } - break; + device_id = ath12k_get_device_family(pci_dev); + if (device_id >= ATH12K_DEVICE_FAMILY_MAX) { + ath12k_err(ab, "failed to get device family id\n"); + ret = -EINVAL; + goto err_pci_free_region; + } - default: - dev_err(&pdev->dev, "Unknown PCI device found: 0x%x\n", - pci_dev->device); - ret = -EOPNOTSUPP; + ath12k_dbg(ab, ATH12K_DBG_PCI, "PCI device family id: %d\n", device_id); + + ab_pci->device_family_ops = &ath12k_pci_family_drivers[device_id]->ops; + ab_pci->reg_base = ath12k_pci_family_drivers[device_id]->reg_base; + + /* Call device specific probe. This is the callback that can + * be used to override any ops in future + * probe is validated for NULL during registration. + */ + ret = ab_pci->device_family_ops->probe(pdev, pci_dev); + if (ret) { + ath12k_err(ab, "failed to probe device: %d\n", ret); goto err_pci_free_region; } @@ -1691,13 +1633,25 @@ static int ath12k_pci_probe(struct pci_dev *pdev, goto err_free_irq; } + /* Invoke arch_init here so that arch-specific init operations + * can utilize already initialized ab fields, such as HAL SRNGs. + */ + ret = ab_pci->device_family_ops->arch_init(ab); + if (ret) { + ath12k_err(ab, "PCI arch_init failed %d\n", ret); + goto err_pci_msi_free; + } + ret = ath12k_core_init(ab); if (ret) { ath12k_err(ab, "failed to init core: %d\n", ret); - goto err_free_irq; + goto err_deinit_arch; } return 0; +err_deinit_arch: + ab_pci->device_family_ops->arch_deinit(ab); + err_free_irq: /* __free_irq() expects the caller to have cleared the affinity hint */ ath12k_pci_set_irq_affinity_hint(ab_pci, NULL); @@ -1756,6 +1710,9 @@ static void ath12k_pci_remove(struct pci_dev *pdev) ath12k_hal_srng_deinit(ab); ath12k_ce_free_pipes(ab); + + ab_pci->device_family_ops->arch_deinit(ab); + ath12k_core_free(ab); } @@ -1844,30 +1801,48 @@ static const struct dev_pm_ops __maybe_unused ath12k_pci_pm_ops = { ath12k_pci_pm_resume_early) }; -static struct pci_driver ath12k_pci_driver = { - .name = "ath12k_pci", - .id_table = ath12k_pci_id_table, - .probe = ath12k_pci_probe, - .remove = ath12k_pci_remove, - .shutdown = ath12k_pci_shutdown, - .driver.pm = &ath12k_pci_pm_ops, -}; - -int ath12k_pci_init(void) +int ath12k_pci_register_driver(const enum ath12k_device_family device_id, + struct ath12k_pci_driver *driver) { - int ret; + struct pci_driver *pci_driver; - ret = pci_register_driver(&ath12k_pci_driver); - if (ret) { - pr_err("failed to register ath12k pci driver: %d\n", - ret); - return ret; + if (device_id >= ATH12K_DEVICE_FAMILY_MAX) + return -EINVAL; + + if (!driver || !driver->ops.probe || + !driver->ops.arch_init || !driver->ops.arch_deinit) + return -EINVAL; + + if (ath12k_pci_family_drivers[device_id]) { + pr_err("Driver already registered for %d\n", device_id); + return -EALREADY; } - return 0; + ath12k_pci_family_drivers[device_id] = driver; + + pci_driver = &ath12k_pci_family_drivers[device_id]->driver; + pci_driver->name = driver->name; + pci_driver->id_table = driver->id_table; + pci_driver->probe = ath12k_pci_probe; + pci_driver->remove = ath12k_pci_remove; + pci_driver->shutdown = ath12k_pci_shutdown; + pci_driver->driver.pm = &ath12k_pci_pm_ops; + + return pci_register_driver(pci_driver); } +EXPORT_SYMBOL(ath12k_pci_register_driver); -void ath12k_pci_exit(void) +void ath12k_pci_unregister_driver(const enum ath12k_device_family device_id) { - pci_unregister_driver(&ath12k_pci_driver); + if (device_id >= ATH12K_DEVICE_FAMILY_MAX || + !ath12k_pci_family_drivers[device_id]) + return; + + pci_unregister_driver(&ath12k_pci_family_drivers[device_id]->driver); + ath12k_pci_family_drivers[device_id] = NULL; } +EXPORT_SYMBOL(ath12k_pci_unregister_driver); + +/* firmware files */ +MODULE_FIRMWARE(ATH12K_FW_DIR "/QCN9274/hw2.0/*"); +MODULE_FIRMWARE(ATH12K_FW_DIR "/WCN7850/hw2.0/*"); diff --git a/drivers/net/wireless/ath/ath12k/pci.h b/drivers/net/wireless/ath/ath12k/pci.h index d1ec8aad7f6c3..0e0e2020c6ae5 100644 --- a/drivers/net/wireless/ath/ath12k/pci.h +++ b/drivers/net/wireless/ath/ath12k/pci.h @@ -1,12 +1,13 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_PCI_H #define ATH12K_PCI_H #include +#include #include "core.h" @@ -29,7 +30,7 @@ #define PARM_LTSSM_VALUE 0x111 #define GCC_GCC_PCIE_HOT_RST(ab) \ - ((ab)->hw_params->regs->gcc_gcc_pcie_hot_rst) + ((ab)->hal.regs->gcc_gcc_pcie_hot_rst) #define GCC_GCC_PCIE_HOT_RST_VAL 0x10 @@ -38,17 +39,17 @@ #define PCIE_INT_CLEAR_ALL 0xffffffff #define PCIE_QSERDES_COM_SYSCLK_EN_SEL_REG(ab) \ - ((ab)->hw_params->regs->pcie_qserdes_sysclk_en_sel) + ((ab)->hal.regs->pcie_qserdes_sysclk_en_sel) #define PCIE_QSERDES_COM_SYSCLK_EN_SEL_VAL 0x10 #define PCIE_QSERDES_COM_SYSCLK_EN_SEL_MSK 0xffffffff #define PCIE_PCS_OSC_DTCT_CONFIG1_REG(ab) \ - ((ab)->hw_params->regs->pcie_pcs_osc_dtct_config_base) + ((ab)->hal.regs->pcie_pcs_osc_dtct_config_base) #define PCIE_PCS_OSC_DTCT_CONFIG1_VAL 0x02 #define PCIE_PCS_OSC_DTCT_CONFIG2_REG(ab) \ - ((ab)->hw_params->regs->pcie_pcs_osc_dtct_config_base + 0x4) + ((ab)->hal.regs->pcie_pcs_osc_dtct_config_base + 0x4) #define PCIE_PCS_OSC_DTCT_CONFIG2_VAL 0x52 #define PCIE_PCS_OSC_DTCT_CONFIG4_REG(ab) \ - ((ab)->hw_params->regs->pcie_pcs_osc_dtct_config_base + 0xc) + ((ab)->hal.regs->pcie_pcs_osc_dtct_config_base + 0xc) #define PCIE_PCS_OSC_DTCT_CONFIG4_VAL 0xff #define PCIE_PCS_OSC_DTCT_CONFIG_MSK 0x000000ff @@ -58,6 +59,11 @@ #define QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB 0x1E20338 #define OTP_BOARD_ID_MASK GENMASK(15, 0) +#define PCIE_LOCAL_REG_QRTR_NODE_ID(ab) \ + ((ab)->hal.regs->qrtr_node_id) +#define DOMAIN_NUMBER_MASK GENMASK(7, 4) +#define BUS_NUMBER_MASK GENMASK(3, 0) + #define PCI_BAR_WINDOW0_BASE 0x1E00000 #define PCI_BAR_WINDOW0_END 0x1E7FFFC #define PCI_SOC_RANGE_MASK 0x3FFF @@ -70,9 +76,6 @@ #define QRTR_PCI_DOMAIN_NR_MASK GENMASK(7, 4) #define QRTR_PCI_BUS_NUMBER_MASK GENMASK(3, 0) -#define ATH12K_PCI_SOC_HW_VERSION_1 1 -#define ATH12K_PCI_SOC_HW_VERSION_2 2 - struct ath12k_msi_user { const char *name; int num_vectors; @@ -97,6 +100,17 @@ struct ath12k_pci_ops { void (*release)(struct ath12k_base *ab); }; +struct ath12k_pci_device_family_ops { + int (*probe)(struct pci_dev *pdev, const struct pci_device_id *pci_dev); + int (*arch_init)(struct ath12k_base *ab); + void (*arch_deinit)(struct ath12k_base *ab); +}; + +struct ath12k_pci_reg_base { + u32 umac_base; + u32 ce_reg_base; +}; + struct ath12k_pci { struct pci_dev *pdev; struct ath12k_base *ab; @@ -119,6 +133,18 @@ struct ath12k_pci { const struct ath12k_pci_ops *pci_ops; u32 qmi_instance; u64 dma_mask; + const struct ath12k_pci_device_family_ops *device_family_ops; + const struct ath12k_pci_reg_base *reg_base; + + u32 window_reg_addr; +}; + +struct ath12k_pci_driver { + const char *name; + const struct pci_device_id *id_table; + struct ath12k_pci_device_family_ops ops; + struct pci_driver driver; + const struct ath12k_pci_reg_base *reg_base; }; static inline struct ath12k_pci *ath12k_pci_priv(struct ath12k_base *ab) @@ -148,6 +174,7 @@ void ath12k_pci_stop(struct ath12k_base *ab); int ath12k_pci_start(struct ath12k_base *ab); int ath12k_pci_power_up(struct ath12k_base *ab); void ath12k_pci_power_down(struct ath12k_base *ab, bool is_suspend); -int ath12k_pci_init(void); -void ath12k_pci_exit(void); +int ath12k_pci_register_driver(const enum ath12k_device_family device_id, + struct ath12k_pci_driver *driver); +void ath12k_pci_unregister_driver(const enum ath12k_device_family device_id); #endif /* ATH12K_PCI_H */ diff --git a/drivers/net/wireless/ath/ath12k/peer.c b/drivers/net/wireless/ath/ath12k/peer.c index f1ae9e5b5af72..5f3bd3b9a3e97 100644 --- a/drivers/net/wireless/ath/ath12k/peer.c +++ b/drivers/net/wireless/ath/ath12k/peer.c @@ -1,211 +1,28 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "core.h" #include "peer.h" #include "debug.h" +#include "debugfs.h" -struct ath12k_ml_peer *ath12k_peer_ml_find(struct ath12k_hw *ah, const u8 *addr) -{ - struct ath12k_ml_peer *ml_peer; - - lockdep_assert_wiphy(ah->hw->wiphy); - - list_for_each_entry(ml_peer, &ah->ml_peers, list) { - if (!ether_addr_equal(ml_peer->addr, addr)) - continue; - - return ml_peer; - } - - return NULL; -} - -struct ath12k_peer *ath12k_peer_find(struct ath12k_base *ab, int vdev_id, - const u8 *addr) -{ - struct ath12k_peer *peer; - - lockdep_assert_held(&ab->base_lock); - - list_for_each_entry(peer, &ab->peers, list) { - if (peer->vdev_id != vdev_id) - continue; - if (!ether_addr_equal(peer->addr, addr)) - continue; - - return peer; - } - - return NULL; -} - -static struct ath12k_peer *ath12k_peer_find_by_pdev_idx(struct ath12k_base *ab, - u8 pdev_idx, const u8 *addr) -{ - struct ath12k_peer *peer; - - lockdep_assert_held(&ab->base_lock); - - list_for_each_entry(peer, &ab->peers, list) { - if (peer->pdev_idx != pdev_idx) - continue; - if (!ether_addr_equal(peer->addr, addr)) - continue; - - return peer; - } - - return NULL; -} - -struct ath12k_peer *ath12k_peer_find_by_addr(struct ath12k_base *ab, - const u8 *addr) -{ - struct ath12k_peer *peer; - - lockdep_assert_held(&ab->base_lock); - - list_for_each_entry(peer, &ab->peers, list) { - if (!ether_addr_equal(peer->addr, addr)) - continue; - - return peer; - } - - return NULL; -} - -static struct ath12k_peer *ath12k_peer_find_by_ml_id(struct ath12k_base *ab, - int ml_peer_id) -{ - struct ath12k_peer *peer; - - lockdep_assert_held(&ab->base_lock); - - list_for_each_entry(peer, &ab->peers, list) - if (ml_peer_id == peer->ml_id) - return peer; - - return NULL; -} - -struct ath12k_peer *ath12k_peer_find_by_id(struct ath12k_base *ab, - int peer_id) -{ - struct ath12k_peer *peer; - - lockdep_assert_held(&ab->base_lock); - - if (peer_id == HAL_INVALID_PEERID) - return NULL; - - if (peer_id & ATH12K_PEER_ML_ID_VALID) - return ath12k_peer_find_by_ml_id(ab, peer_id); - - list_for_each_entry(peer, &ab->peers, list) - if (peer_id == peer->peer_id) - return peer; - - return NULL; -} - -bool ath12k_peer_exist_by_vdev_id(struct ath12k_base *ab, int vdev_id) -{ - struct ath12k_peer *peer; - - spin_lock_bh(&ab->base_lock); - - list_for_each_entry(peer, &ab->peers, list) { - if (vdev_id == peer->vdev_id) { - spin_unlock_bh(&ab->base_lock); - return true; - } - } - spin_unlock_bh(&ab->base_lock); - return false; -} - -struct ath12k_peer *ath12k_peer_find_by_ast(struct ath12k_base *ab, - int ast_hash) -{ - struct ath12k_peer *peer; - - lockdep_assert_held(&ab->base_lock); - - list_for_each_entry(peer, &ab->peers, list) - if (ast_hash == peer->ast_hash) - return peer; - - return NULL; -} - -void ath12k_peer_unmap_event(struct ath12k_base *ab, u16 peer_id) -{ - struct ath12k_peer *peer; - - spin_lock_bh(&ab->base_lock); - - peer = ath12k_peer_find_by_id(ab, peer_id); - if (!peer) { - ath12k_warn(ab, "peer-unmap-event: unknown peer id %d\n", - peer_id); - goto exit; - } - - ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer unmap vdev %d peer %pM id %d\n", - peer->vdev_id, peer->addr, peer_id); - - list_del(&peer->list); - kfree(peer); - wake_up(&ab->peer_mapping_wq); - -exit: - spin_unlock_bh(&ab->base_lock); -} - -void ath12k_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id, - u8 *mac_addr, u16 ast_hash, u16 hw_peer_id) -{ - struct ath12k_peer *peer; - - spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find(ab, vdev_id, mac_addr); - if (!peer) { - peer = kzalloc(sizeof(*peer), GFP_ATOMIC); - if (!peer) - goto exit; - - peer->vdev_id = vdev_id; - peer->peer_id = peer_id; - peer->ast_hash = ast_hash; - peer->hw_peer_id = hw_peer_id; - ether_addr_copy(peer->addr, mac_addr); - list_add(&peer->list, &ab->peers); - wake_up(&ab->peer_mapping_wq); - } - - ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer map vdev %d peer %pM id %d\n", - vdev_id, mac_addr, peer_id); - -exit: - spin_unlock_bh(&ab->base_lock); -} - -static int ath12k_wait_for_peer_common(struct ath12k_base *ab, int vdev_id, - const u8 *addr, bool expect_mapped) +static int ath12k_wait_for_dp_link_peer_common(struct ath12k_base *ab, int vdev_id, + const u8 *addr, bool expect_mapped) { int ret; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); ret = wait_event_timeout(ab->peer_mapping_wq, ({ bool mapped; - spin_lock_bh(&ab->base_lock); - mapped = !!ath12k_peer_find(ab, vdev_id, addr); - spin_unlock_bh(&ab->base_lock); + spin_lock_bh(&dp->dp_lock); + mapped = !!ath12k_dp_link_peer_find_by_vdev_and_addr(dp, + vdev_id, + addr); + spin_unlock_bh(&dp->dp_lock); (mapped == expect_mapped || test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags)); @@ -219,30 +36,30 @@ static int ath12k_wait_for_peer_common(struct ath12k_base *ab, int vdev_id, void ath12k_peer_cleanup(struct ath12k *ar, u32 vdev_id) { - struct ath12k_peer *peer, *tmp; + struct ath12k_dp_link_peer *peer, *tmp; struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); - spin_lock_bh(&ab->base_lock); - list_for_each_entry_safe(peer, tmp, &ab->peers, list) { + spin_lock_bh(&dp->dp_lock); + list_for_each_entry_safe(peer, tmp, &dp->peers, list) { if (peer->vdev_id != vdev_id) continue; ath12k_warn(ab, "removing stale peer %pM from vdev_id %d\n", peer->addr, vdev_id); - list_del(&peer->list); - kfree(peer); + ath12k_dp_link_peer_free(peer); ar->num_peers--; } - spin_unlock_bh(&ab->base_lock); + spin_unlock_bh(&dp->dp_lock); } static int ath12k_wait_for_peer_deleted(struct ath12k *ar, int vdev_id, const u8 *addr) { - return ath12k_wait_for_peer_common(ar->ab, vdev_id, addr, false); + return ath12k_wait_for_dp_link_peer_common(ar->ab, vdev_id, addr, false); } int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id, @@ -293,6 +110,10 @@ int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); + ath12k_dp_link_peer_unassign(ath12k_ab_to_dp(ar->ab), + &(ath12k_ar_to_ah(ar)->dp_hw), vdev_id, + addr, ar->hw_link_id); + ret = ath12k_peer_delete_send(ar, vdev_id, addr); if (ret) return ret; @@ -308,7 +129,7 @@ int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr) static int ath12k_wait_for_peer_created(struct ath12k *ar, int vdev_id, const u8 *addr) { - return ath12k_wait_for_peer_common(ar->ab, vdev_id, addr, true); + return ath12k_wait_for_dp_link_peer_common(ar->ab, vdev_id, addr, true); } int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, @@ -316,28 +137,34 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ath12k_wmi_peer_create_arg *arg) { struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif); + struct ath12k_vif *ahvif = arvif->ahvif; + struct ath12k_dp_link_vif *dp_link_vif; struct ath12k_link_sta *arsta; u8 link_id = arvif->link_id; - struct ath12k_peer *peer; + struct ath12k_dp_link_peer *peer; struct ath12k_sta *ahsta; u16 ml_peer_id; int ret; + struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab); lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); + dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id); + if (ar->num_peers > (ar->max_num_peers - 1)) { ath12k_warn(ar->ab, "failed to create peer due to insufficient peer entry resource in firmware\n"); return -ENOBUFS; } - spin_lock_bh(&ar->ab->base_lock); - peer = ath12k_peer_find_by_pdev_idx(ar->ab, ar->pdev_idx, arg->peer_addr); + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_pdev_and_addr(dp, ar->pdev_idx, + arg->peer_addr); if (peer) { - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); return -EINVAL; } - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); ret = ath12k_wmi_send_peer_create_cmd(ar, arg); if (ret) { @@ -352,11 +179,12 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, if (ret) return ret; - spin_lock_bh(&ar->ab->base_lock); + spin_lock_bh(&dp->dp_lock); - peer = ath12k_peer_find(ar->ab, arg->vdev_id, arg->peer_addr); + peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arg->vdev_id, + arg->peer_addr); if (!peer) { - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ar->ab, "failed to find peer %pM on vdev %i after creation\n", arg->peer_addr, arg->vdev_id); @@ -382,13 +210,10 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, peer->sta = sta; if (vif->type == NL80211_IFTYPE_STATION) { - arvif->ast_hash = peer->ast_hash; - arvif->ast_idx = peer->hw_peer_id; + dp_link_vif->ast_hash = peer->ast_hash; + dp_link_vif->ast_idx = peer->hw_peer_id; } - if (vif->type == NL80211_IFTYPE_AP) - peer->ucast_ra_only = true; - if (sta) { ahsta = ath12k_sta_to_ahsta(sta); arsta = wiphy_dereference(ath12k_ar_to_hw(ar)->wiphy, @@ -412,17 +237,22 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, } } - peer->sec_type = HAL_ENCRYPT_TYPE_OPEN; - peer->sec_type_grp = HAL_ENCRYPT_TYPE_OPEN; - ar->num_peers++; - spin_unlock_bh(&ar->ab->base_lock); + spin_unlock_bh(&dp->dp_lock); - return 0; + if (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS) { + ret = ath12k_dp_link_peer_assign(ath12k_ab_to_dp(ar->ab), + &(ath12k_ar_to_ah(ar)->dp_hw), + arvif->vdev_id, sta, + (u8 *)arg->peer_addr, link_id, + ar->hw_link_id); + } + + return ret; } -static u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah) +u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah) { u16 ml_peer_id; @@ -442,68 +272,6 @@ static u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah) return ml_peer_id; } -int ath12k_peer_ml_create(struct ath12k_hw *ah, struct ieee80211_sta *sta) -{ - struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta); - struct ath12k_ml_peer *ml_peer; - - lockdep_assert_wiphy(ah->hw->wiphy); - - if (!sta->mlo) - return -EINVAL; - - ml_peer = ath12k_peer_ml_find(ah, sta->addr); - if (ml_peer) { - ath12k_hw_warn(ah, "ML peer %d exists already, unable to add new entry for %pM", - ml_peer->id, sta->addr); - return -EEXIST; - } - - ml_peer = kzalloc(sizeof(*ml_peer), GFP_ATOMIC); - if (!ml_peer) - return -ENOMEM; - - ahsta->ml_peer_id = ath12k_peer_ml_alloc(ah); - - if (ahsta->ml_peer_id == ATH12K_MLO_PEER_ID_INVALID) { - ath12k_hw_warn(ah, "unable to allocate ML peer id for sta %pM", - sta->addr); - kfree(ml_peer); - return -ENOMEM; - } - - ether_addr_copy(ml_peer->addr, sta->addr); - ml_peer->id = ahsta->ml_peer_id; - list_add(&ml_peer->list, &ah->ml_peers); - - return 0; -} - -int ath12k_peer_ml_delete(struct ath12k_hw *ah, struct ieee80211_sta *sta) -{ - struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta); - struct ath12k_ml_peer *ml_peer; - - lockdep_assert_wiphy(ah->hw->wiphy); - - if (!sta->mlo) - return -EINVAL; - - clear_bit(ahsta->ml_peer_id, ah->free_ml_peer_id_map); - ahsta->ml_peer_id = ATH12K_MLO_PEER_ID_INVALID; - - ml_peer = ath12k_peer_ml_find(ah, sta->addr); - if (!ml_peer) { - ath12k_hw_warn(ah, "ML peer for %pM not found", sta->addr); - return -EINVAL; - } - - list_del(&ml_peer->list); - kfree(ml_peer); - - return 0; -} - int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta) { struct ieee80211_sta *sta = ath12k_ahsta_to_sta(ahsta); @@ -536,6 +304,11 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st ath12k_dp_peer_cleanup(ar, arvif->vdev_id, arsta->addr); + ath12k_dp_link_peer_unassign(ath12k_ab_to_dp(ar->ab), + &(ath12k_ar_to_ah(ar)->dp_hw), + arvif->vdev_id, arsta->addr, + ar->hw_link_id); + ret = ath12k_peer_delete_send(ar, arvif->vdev_id, arsta->addr); if (ret) { ath12k_warn(ar->ab, @@ -568,3 +341,119 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st return err_ret; } + +static int ath12k_link_sta_rhash_insert(struct ath12k_base *ab, + struct ath12k_link_sta *arsta) +{ + struct ath12k_link_sta *tmp; + + lockdep_assert_held(&ab->base_lock); + + tmp = rhashtable_lookup_get_insert_fast(ab->rhead_sta_addr, &arsta->rhash_addr, + ab->rhash_sta_addr_param); + if (!tmp) + return 0; + else if (IS_ERR(tmp)) + return PTR_ERR(tmp); + else + return -EEXIST; +} + +static int ath12k_link_sta_rhash_remove(struct ath12k_base *ab, + struct ath12k_link_sta *arsta) +{ + int ret; + + lockdep_assert_held(&ab->base_lock); + + ret = rhashtable_remove_fast(ab->rhead_sta_addr, &arsta->rhash_addr, + ab->rhash_sta_addr_param); + if (ret && ret != -ENOENT) + return ret; + + return 0; +} + +int ath12k_link_sta_rhash_add(struct ath12k_base *ab, + struct ath12k_link_sta *arsta) +{ + int ret; + + lockdep_assert_held(&ab->base_lock); + + ret = ath12k_link_sta_rhash_insert(ab, arsta); + if (ret) + ath12k_warn(ab, "failed to add arsta %pM in rhash_addr ret %d\n", + arsta->addr, ret); + + return ret; +} + +void ath12k_link_sta_rhash_delete(struct ath12k_base *ab, + struct ath12k_link_sta *arsta) +{ + /* + * Return type of this function is void since there is nothing to be + * done in failure case + */ + int ret; + + lockdep_assert_held(&ab->base_lock); + + ret = ath12k_link_sta_rhash_remove(ab, arsta); + if (ret) + ath12k_warn(ab, + "failed to remove arsta %pM in rhash_addr ret %d\n", + arsta->addr, ret); +} + +int ath12k_link_sta_rhash_tbl_init(struct ath12k_base *ab) +{ + struct rhashtable_params *param; + struct rhashtable *rhash_addr_tbl; + int ret; + + rhash_addr_tbl = kzalloc(sizeof(*ab->rhead_sta_addr), GFP_KERNEL); + if (!rhash_addr_tbl) + return -ENOMEM; + + param = &ab->rhash_sta_addr_param; + + param->key_offset = offsetof(struct ath12k_link_sta, addr); + param->head_offset = offsetof(struct ath12k_link_sta, rhash_addr); + param->key_len = sizeof_field(struct ath12k_link_sta, addr); + param->automatic_shrinking = true; + param->nelem_hint = ab->num_radios * ath12k_core_get_max_peers_per_radio(ab); + + ret = rhashtable_init(rhash_addr_tbl, param); + if (ret) { + ath12k_warn(ab, "failed to init peer addr rhash table %d\n", + ret); + goto err_free; + } + + ab->rhead_sta_addr = rhash_addr_tbl; + + return 0; + +err_free: + kfree(rhash_addr_tbl); + + return ret; +} + +void ath12k_link_sta_rhash_tbl_destroy(struct ath12k_base *ab) +{ + rhashtable_destroy(ab->rhead_sta_addr); + kfree(ab->rhead_sta_addr); + ab->rhead_sta_addr = NULL; +} + +struct ath12k_link_sta *ath12k_link_sta_find_by_addr(struct ath12k_base *ab, + const u8 *addr) +{ + lockdep_assert_held(&ab->base_lock); + + return rhashtable_lookup_fast(ab->rhead_sta_addr, addr, + ab->rhash_sta_addr_param); +} diff --git a/drivers/net/wireless/ath/ath12k/peer.h b/drivers/net/wireless/ath/ath12k/peer.h index 44afc0b7dd53e..49d89796bc46e 100644 --- a/drivers/net/wireless/ath/ath12k/peer.h +++ b/drivers/net/wireless/ath/ath12k/peer.h @@ -1,84 +1,14 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_PEER_H #define ATH12K_PEER_H -#include "dp_rx.h" +#include "dp_peer.h" -struct ppdu_user_delayba { - u16 sw_peer_id; - u32 info0; - u16 ru_end; - u16 ru_start; - u32 info1; - u32 rate_flags; - u32 resp_rate_flags; -}; - -#define ATH12K_PEER_ML_ID_VALID BIT(13) - -struct ath12k_peer { - struct list_head list; - struct ieee80211_sta *sta; - int vdev_id; - u8 addr[ETH_ALEN]; - int peer_id; - u16 ast_hash; - u8 pdev_idx; - u16 hw_peer_id; - - /* protected by ab->data_lock */ - struct ieee80211_key_conf *keys[WMI_MAX_KEY_INDEX + 1]; - struct ath12k_dp_rx_tid rx_tid[IEEE80211_NUM_TIDS + 1]; - - /* Info used in MMIC verification of - * RX fragments - */ - struct crypto_shash *tfm_mmic; - u8 mcast_keyidx; - u8 ucast_keyidx; - u16 sec_type; - u16 sec_type_grp; - struct ppdu_user_delayba ppdu_stats_delayba; - bool delayba_flag; - bool is_authorized; - bool mlo; - /* protected by ab->data_lock */ - bool dp_setup_done; - - u16 ml_id; - - /* any other ML info common for all partners can be added - * here and would be same for all partner peers. - */ - u8 ml_addr[ETH_ALEN]; - - /* To ensure only certain work related to dp is done once */ - bool primary_link; - - /* for reference to ath12k_link_sta */ - u8 link_id; - bool ucast_ra_only; -}; - -struct ath12k_ml_peer { - struct list_head list; - u8 addr[ETH_ALEN]; - u16 id; -}; - -void ath12k_peer_unmap_event(struct ath12k_base *ab, u16 peer_id); -void ath12k_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id, - u8 *mac_addr, u16 ast_hash, u16 hw_peer_id); -struct ath12k_peer *ath12k_peer_find(struct ath12k_base *ab, int vdev_id, - const u8 *addr); -struct ath12k_peer *ath12k_peer_find_by_addr(struct ath12k_base *ab, - const u8 *addr); -struct ath12k_peer *ath12k_peer_find_by_id(struct ath12k_base *ab, int peer_id); void ath12k_peer_cleanup(struct ath12k *ar, u32 vdev_id); int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr); int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, @@ -86,38 +16,14 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ath12k_wmi_peer_create_arg *arg); int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id, const u8 *addr); -bool ath12k_peer_exist_by_vdev_id(struct ath12k_base *ab, int vdev_id); -struct ath12k_peer *ath12k_peer_find_by_ast(struct ath12k_base *ab, int ast_hash); -int ath12k_peer_ml_create(struct ath12k_hw *ah, struct ieee80211_sta *sta); -int ath12k_peer_ml_delete(struct ath12k_hw *ah, struct ieee80211_sta *sta); int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta); struct ath12k_ml_peer *ath12k_peer_ml_find(struct ath12k_hw *ah, const u8 *addr); -static inline -struct ath12k_link_sta *ath12k_peer_get_link_sta(struct ath12k_base *ab, - struct ath12k_peer *peer) -{ - struct ath12k_sta *ahsta; - struct ath12k_link_sta *arsta; - - if (!peer->sta) - return NULL; - - ahsta = ath12k_sta_to_ahsta(peer->sta); - if (peer->ml_id & ATH12K_PEER_ML_ID_VALID) { - if (!(ahsta->links_map & BIT(peer->link_id))) { - ath12k_warn(ab, "peer %pM id %d link_id %d can't found in STA link_map 0x%x\n", - peer->addr, peer->peer_id, peer->link_id, - ahsta->links_map); - return NULL; - } - arsta = rcu_dereference(ahsta->link[peer->link_id]); - if (!arsta) - return NULL; - } else { - arsta = &ahsta->deflink; - } - return arsta; -} - +int ath12k_link_sta_rhash_tbl_init(struct ath12k_base *ab); +void ath12k_link_sta_rhash_tbl_destroy(struct ath12k_base *ab); +void ath12k_link_sta_rhash_delete(struct ath12k_base *ab, struct ath12k_link_sta *arsta); +int ath12k_link_sta_rhash_add(struct ath12k_base *ab, struct ath12k_link_sta *arsta); +struct ath12k_link_sta *ath12k_link_sta_find_by_addr(struct ath12k_base *ab, + const u8 *addr); +u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah); #endif /* _PEER_H_ */ diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c index 36325e62aa242..71786216d6473 100644 --- a/drivers/net/wireless/ath/ath12k/qmi.c +++ b/drivers/net/wireless/ath/ath12k/qmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -1623,6 +1623,47 @@ static const struct qmi_elem_info qmi_wlanfw_m3_info_resp_msg_v01_ei[] = { }, }; +static const struct qmi_elem_info qmi_wlanfw_aux_uc_info_req_msg_v01_ei[] = { + { + .data_type = QMI_UNSIGNED_8_BYTE, + .elem_len = 1, + .elem_size = sizeof(u64), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof(struct qmi_wlanfw_aux_uc_info_req_msg_v01, addr), + }, + { + .data_type = QMI_UNSIGNED_4_BYTE, + .elem_len = 1, + .elem_size = sizeof(u32), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct qmi_wlanfw_aux_uc_info_req_msg_v01, size), + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static const struct qmi_elem_info qmi_wlanfw_aux_uc_info_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct qmi_wlanfw_aux_uc_info_resp_msg_v01, resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + static const struct qmi_elem_info qmi_wlanfw_ce_tgt_pipe_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, @@ -2609,6 +2650,7 @@ static int ath12k_qmi_alloc_target_mem_chunk(struct ath12k_base *ab) case M3_DUMP_REGION_TYPE: case PAGEABLE_MEM_REGION_TYPE: case CALDB_MEM_REGION_TYPE: + case LPASS_SHARED_V01_REGION_TYPE: ret = ath12k_qmi_alloc_chunk(ab, chunk); if (ret) goto err; @@ -3114,9 +3156,10 @@ static void ath12k_qmi_m3_free(struct ath12k_base *ab) if (!m3_mem->vaddr) return; - dma_free_coherent(ab->dev, m3_mem->size, + dma_free_coherent(ab->dev, m3_mem->total_size, m3_mem->vaddr, m3_mem->paddr); m3_mem->vaddr = NULL; + m3_mem->total_size = 0; m3_mem->size = 0; } @@ -3152,7 +3195,7 @@ static int ath12k_qmi_m3_load(struct ath12k_base *ab) /* In recovery/resume cases, M3 buffer is not freed, try to reuse that */ if (m3_mem->vaddr) { - if (m3_mem->size >= m3_len) + if (m3_mem->total_size >= m3_len) goto skip_m3_alloc; /* Old buffer is too small, free and reallocate */ @@ -3164,11 +3207,13 @@ static int ath12k_qmi_m3_load(struct ath12k_base *ab) GFP_KERNEL); if (!m3_mem->vaddr) { ath12k_err(ab, "failed to allocate memory for M3 with size %zu\n", - fw->size); + m3_len); ret = -ENOMEM; goto out; } + m3_mem->total_size = m3_len; + skip_m3_alloc: memcpy(m3_mem->vaddr, m3_data, m3_len); m3_mem->size = m3_len; @@ -3233,6 +3278,131 @@ int ath12k_qmi_wlanfw_m3_info_send(struct ath12k_base *ab) return ret; } +static void ath12k_qmi_aux_uc_free(struct ath12k_base *ab) +{ + struct m3_mem_region *aux_uc_mem = &ab->qmi.aux_uc_mem; + + if (!aux_uc_mem->vaddr) + return; + + dma_free_coherent(ab->dev, aux_uc_mem->total_size, + aux_uc_mem->vaddr, aux_uc_mem->paddr); + aux_uc_mem->vaddr = NULL; + aux_uc_mem->total_size = 0; + aux_uc_mem->size = 0; +} + +static int ath12k_qmi_aux_uc_load(struct ath12k_base *ab) +{ + struct m3_mem_region *aux_uc_mem = &ab->qmi.aux_uc_mem; + const struct firmware *fw = NULL; + const void *aux_uc_data; + char path[100]; + size_t aux_uc_len; + int ret; + + if (ab->fw.aux_uc_data && ab->fw.aux_uc_len > 0) { + /* firmware-N.bin had a aux_uc firmware file so use that */ + aux_uc_data = ab->fw.aux_uc_data; + aux_uc_len = ab->fw.aux_uc_len; + } else { + /* + * No aux_uc file in firmware-N.bin so try to request old + * separate aux_ucode.bin. + */ + fw = ath12k_core_firmware_request(ab, ATH12K_AUX_UC_FILE); + if (IS_ERR(fw)) { + ret = PTR_ERR(fw); + ath12k_core_create_firmware_path(ab, ATH12K_AUX_UC_FILE, + path, sizeof(path)); + ath12k_err(ab, "failed to load %s: %d\n", path, ret); + return ret; + } + + aux_uc_data = fw->data; + aux_uc_len = fw->size; + } + + /* In recovery/resume cases, AUX_UC buffer is not freed, try to reuse that */ + if (aux_uc_mem->vaddr) { + if (aux_uc_mem->total_size >= aux_uc_len) + goto copy; + + /* Old buffer is too small, free and reallocate */ + ath12k_qmi_aux_uc_free(ab); + } + + aux_uc_mem->vaddr = dma_alloc_coherent(ab->dev, aux_uc_len, + &aux_uc_mem->paddr, GFP_KERNEL); + if (!aux_uc_mem->vaddr) { + ret = -ENOMEM; + goto out; + } + + aux_uc_mem->total_size = aux_uc_len; + +copy: + memcpy(aux_uc_mem->vaddr, aux_uc_data, aux_uc_len); + aux_uc_mem->size = aux_uc_len; + + ret = 0; + +out: + release_firmware(fw); + + return ret; +} + +static noinline_for_stack +int ath12k_qmi_wlanfw_aux_uc_info_send(struct ath12k_base *ab) +{ + struct m3_mem_region *aux_uc_mem = &ab->qmi.aux_uc_mem; + struct qmi_wlanfw_aux_uc_info_req_msg_v01 req = {}; + struct qmi_wlanfw_aux_uc_info_resp_msg_v01 resp = {}; + struct qmi_txn txn; + int ret = 0; + + ret = ath12k_qmi_aux_uc_load(ab); + if (ret) { + ath12k_err(ab, "failed to load aux_uc firmware: %d", ret); + return ret; + } + + req.addr = aux_uc_mem->paddr; + req.size = aux_uc_mem->size; + + ret = qmi_txn_init(&ab->qmi.handle, &txn, + qmi_wlanfw_aux_uc_info_resp_msg_v01_ei, &resp); + if (ret < 0) + goto out; + + ret = qmi_send_request(&ab->qmi.handle, NULL, &txn, + QMI_WLANFW_AUX_UC_INFO_REQ_V01, + QMI_WLANFW_AUX_UC_INFO_REQ_MSG_V01_MAX_MSG_LEN, + qmi_wlanfw_aux_uc_info_req_msg_v01_ei, &req); + if (ret < 0) { + qmi_txn_cancel(&txn); + ath12k_warn(ab, "qmi failed to send AUX_UC information request, err = %d\n", + ret); + goto out; + } + + ret = qmi_txn_wait(&txn, msecs_to_jiffies(ATH12K_QMI_WLANFW_TIMEOUT_MS)); + if (ret < 0) { + ath12k_warn(ab, "qmi failed AUX_UC information request %d\n", ret); + goto out; + } + + if (resp.resp.result != QMI_RESULT_SUCCESS_V01) { + ath12k_warn(ab, "qmi AUX_UC info request failed, result: %d, err: %d\n", + resp.resp.result, resp.resp.error); + ret = -EINVAL; + goto out; + } +out: + return ret; +} + static int ath12k_qmi_wlanfw_mode_send(struct ath12k_base *ab, u32 mode) { @@ -3597,6 +3767,7 @@ static noinline_for_stack int ath12k_qmi_event_load_bdf(struct ath12k_qmi *qmi) { struct ath12k_base *ab = qmi->ab; + const struct ath12k_hw_params *hw_params = ab->hw_params; int ret; ret = ath12k_qmi_request_target_cap(ab); @@ -3617,7 +3788,7 @@ int ath12k_qmi_event_load_bdf(struct ath12k_qmi *qmi) return ret; } - if (ab->hw_params->download_calib) { + if (hw_params->download_calib) { ret = ath12k_qmi_load_bdf_qmi(ab, ATH12K_QMI_BDF_TYPE_CALIBRATION); if (ret < 0) ath12k_warn(ab, "qmi failed to load calibrated data :%d\n", ret); @@ -3629,6 +3800,14 @@ int ath12k_qmi_event_load_bdf(struct ath12k_qmi *qmi) return ret; } + if (hw_params->fw.download_aux_ucode) { + ret = ath12k_qmi_wlanfw_aux_uc_info_send(ab); + if (ret < 0) { + ath12k_warn(ab, "qmi failed to send aux_uc info req: %d\n", ret); + return ret; + } + } + return ret; } @@ -3902,6 +4081,7 @@ void ath12k_qmi_deinit_service(struct ath12k_base *ab) qmi_handle_release(&ab->qmi.handle); cancel_work_sync(&ab->qmi.event_work); destroy_workqueue(ab->qmi.event_wq); + ath12k_qmi_aux_uc_free(ab); ath12k_qmi_m3_free(ab); ath12k_qmi_free_target_mem_chunk(ab); ab->qmi.ab = NULL; @@ -3910,5 +4090,6 @@ void ath12k_qmi_deinit_service(struct ath12k_base *ab) void ath12k_qmi_free_resource(struct ath12k_base *ab) { ath12k_qmi_free_target_mem_chunk(ab); + ath12k_qmi_aux_uc_free(ab); ath12k_qmi_m3_free(ab); } diff --git a/drivers/net/wireless/ath/ath12k/qmi.h b/drivers/net/wireless/ath/ath12k/qmi.h index 4767d9a2e309e..b5a4a01391cba 100644 --- a/drivers/net/wireless/ath/ath12k/qmi.h +++ b/drivers/net/wireless/ath/ath12k/qmi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_QMI_H @@ -120,6 +120,9 @@ struct target_info { }; struct m3_mem_region { + /* total memory allocated */ + u32 total_size; + /* actual memory being used */ u32 size; dma_addr_t paddr; void *vaddr; @@ -151,6 +154,7 @@ struct ath12k_qmi { u8 num_radios; struct target_info target; struct m3_mem_region m3_mem; + struct m3_mem_region aux_uc_mem; unsigned int service_ins_id; struct dev_mem_info dev_mem[ATH12K_QMI_WLFW_MAX_DEV_MEM_NUM_V01]; }; @@ -175,6 +179,7 @@ enum ath12k_qmi_target_mem { CALDB_MEM_REGION_TYPE = 0x4, MLO_GLOBAL_MEM_REGION_TYPE = 0x8, PAGEABLE_MEM_REGION_TYPE = 0x9, + LPASS_SHARED_V01_REGION_TYPE = 0xb, }; enum qmi_wlanfw_host_build_type { @@ -199,6 +204,7 @@ enum ath12k_qmi_cnss_feature { CNSS_FEATURE_MIN_ENUM_VAL_V01 = INT_MIN, CNSS_QDSS_CFG_MISS_V01 = 3, CNSS_PCIE_PERST_NO_PULL_V01 = 4, + CNSS_AUX_UC_SUPPORT_V01 = 6, CNSS_MAX_FEATURE_V01 = 64, CNSS_FEATURE_MAX_ENUM_VAL_V01 = INT_MAX, }; @@ -537,6 +543,19 @@ struct qmi_wlanfw_m3_info_resp_msg_v01 { struct qmi_response_type_v01 resp; }; +#define QMI_WLANFW_AUX_UC_INFO_REQ_MSG_V01_MAX_MSG_LEN 18 +#define QMI_WLANFW_AUX_UC_INFO_RESP_MSG_V01_MAX_MSG_LEN 7 +#define QMI_WLANFW_AUX_UC_INFO_REQ_V01 0x005A + +struct qmi_wlanfw_aux_uc_info_req_msg_v01 { + u64 addr; + u32 size; +}; + +struct qmi_wlanfw_aux_uc_info_resp_msg_v01 { + struct qmi_response_type_v01 resp; +}; + #define QMI_WLANFW_WLAN_MODE_REQ_MSG_V01_MAX_LEN 11 #define QMI_WLANFW_WLAN_MODE_RESP_MSG_V01_MAX_LEN 7 #define QMI_WLANFW_WLAN_CFG_REQ_MSG_V01_MAX_LEN 803 diff --git a/drivers/net/wireless/ath/ath12k/testmode.c b/drivers/net/wireless/ath/ath12k/testmode.c index fb6af7ccf71f4..05a65970c8624 100644 --- a/drivers/net/wireless/ath/ath12k/testmode.c +++ b/drivers/net/wireless/ath/ath12k/testmode.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "testmode.h" @@ -393,3 +393,4 @@ int ath12k_tm_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif, return -EOPNOTSUPP; } } +EXPORT_SYMBOL(ath12k_tm_cmd); diff --git a/drivers/net/wireless/ath/ath12k/wifi7/Makefile b/drivers/net/wireless/ath/ath12k/wifi7/Makefile new file mode 100644 index 0000000000000..45b561cdba4b1 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/Makefile @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: BSD-3-Clause-Clear +obj-$(CONFIG_ATH12K) += ath12k_wifi7.o +ath12k_wifi7-y += core.o \ + pci.o \ + wmi.o \ + mhi.o \ + ce.o \ + hw.o \ + hal_tx.o \ + hal_rx.o \ + dp_rx.o \ + dp_tx.o \ + dp.o \ + dp_mon.o \ + hal.o \ + hal_qcn9274.o \ + hal_wcn7850.o \ + hal_qcc2072.o + +ath12k_wifi7-$(CONFIG_ATH12K_AHB) += ahb.o diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ahb.c b/drivers/net/wireless/ath/ath12k/wifi7/ahb.c new file mode 100644 index 0000000000000..a6c5f7689edd1 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/ahb.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include "../ahb.h" +#include "ahb.h" +#include "../debug.h" +#include "../hif.h" +#include "hw.h" +#include "dp.h" +#include "core.h" + +static const struct of_device_id ath12k_wifi7_ahb_of_match[] = { + { .compatible = "qcom,ipq5332-wifi", + .data = (void *)ATH12K_HW_IPQ5332_HW10, + }, + { } +}; + +MODULE_DEVICE_TABLE(of, ath12k_wifi7_ahb_of_match); + +static int ath12k_wifi7_ahb_probe(struct platform_device *pdev) +{ + struct ath12k_ahb *ab_ahb; + enum ath12k_hw_rev hw_rev; + struct ath12k_base *ab; + int ret; + + ab = platform_get_drvdata(pdev); + ab_ahb = ath12k_ab_to_ahb(ab); + + hw_rev = (enum ath12k_hw_rev)(kernel_ulong_t)of_device_get_match_data(&pdev->dev); + switch (hw_rev) { + case ATH12K_HW_IPQ5332_HW10: + ab_ahb->userpd_id = ATH12K_IPQ5332_USERPD_ID; + break; + default: + return -EOPNOTSUPP; + } + + ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT; + ab->hw_rev = hw_rev; + + ret = ath12k_wifi7_hw_init(ab); + if (ret) { + ath12k_err(ab, "WiFi-7 hw_init for AHB failed: %d\n", ret); + return ret; + } + + return 0; +} + +static struct ath12k_ahb_driver ath12k_wifi7_ahb_driver = { + .name = "ath12k_wifi7_ahb", + .id_table = ath12k_wifi7_ahb_of_match, + .ops.probe = ath12k_wifi7_ahb_probe, + .ops.arch_init = ath12k_wifi7_arch_init, + .ops.arch_deinit = ath12k_wifi7_arch_deinit, +}; + +int ath12k_wifi7_ahb_init(void) +{ + return ath12k_ahb_register_driver(ATH12K_DEVICE_FAMILY_WIFI7, + &ath12k_wifi7_ahb_driver); +} + +void ath12k_wifi7_ahb_exit(void) +{ + ath12k_ahb_unregister_driver(ATH12K_DEVICE_FAMILY_WIFI7); +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ahb.h b/drivers/net/wireless/ath/ath12k/wifi7/ahb.h new file mode 100644 index 0000000000000..5974c7cad69ae --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/ahb.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#ifndef ATH12K_AHB_WIFI7_H +#define ATH12K_AHB_WIFI7_H + +#ifdef CONFIG_ATH12K_AHB +int ath12k_wifi7_ahb_init(void); +void ath12k_wifi7_ahb_exit(void); +#else +static inline int ath12k_wifi7_ahb_init(void) +{ + return 0; +} + +static inline void ath12k_wifi7_ahb_exit(void) {} +#endif +#endif /* ATH12K_AHB_WIFI7_H */ diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ce.c b/drivers/net/wireless/ath/ath12k/wifi7/ce.c new file mode 100644 index 0000000000000..952d6c39c3334 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/ce.c @@ -0,0 +1,973 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include + +#include "../core.h" +#include "../ce.h" +#include "ce.h" +#include "../dp_rx.h" + +/* Copy Engine (CE) configs for QCN9274 */ +/* Target firmware's Copy Engine configuration. */ +const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_qcn9274[] = { + /* CE0: host->target HTC control and raw streams */ + { + .pipenum = __cpu_to_le32(0), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE1: target->host HTT + HTC control */ + { + .pipenum = __cpu_to_le32(1), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE2: target->host WMI */ + { + .pipenum = __cpu_to_le32(2), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE3: host->target WMI (mac0) */ + { + .pipenum = __cpu_to_le32(3), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE4: host->target HTT */ + { + .pipenum = __cpu_to_le32(4), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(256), + .nbytes_max = __cpu_to_le32(256), + .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), + .reserved = __cpu_to_le32(0), + }, + + /* CE5: target->host Pktlog */ + { + .pipenum = __cpu_to_le32(5), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE6: Reserved for target autonomous hif_memcpy */ + { + .pipenum = __cpu_to_le32(6), + .pipedir = __cpu_to_le32(PIPEDIR_INOUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(16384), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE7: host->target WMI (mac1) */ + { + .pipenum = __cpu_to_le32(7), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE8: Reserved for target autonomous hif_memcpy */ + { + .pipenum = __cpu_to_le32(8), + .pipedir = __cpu_to_le32(PIPEDIR_INOUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(16384), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE9, 10 and 11: Reserved for MHI */ + + /* CE12: Target CV prefetch */ + { + .pipenum = __cpu_to_le32(12), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE13: Target CV prefetch */ + { + .pipenum = __cpu_to_le32(13), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE14: WMI logging/CFR/Spectral/Radar */ + { + .pipenum = __cpu_to_le32(14), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE15: Reserved */ +}; + +/* Map from service/endpoint to Copy Engine. + * This table is derived from the CE_PCI TABLE, above. + * It is passed to the Target at startup for use by firmware. + * Pipe direction: + * PIPEDIR_OUT = UL = host -> target + * PIPEDIR_IN = DL = target -> host + */ +const struct service_to_pipe +ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274[] = { + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(0), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(1), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(0), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(1), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(4), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(1), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(7), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(5), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(14), + }, + + /* (Additions here) */ + + { /* must be last */ + __cpu_to_le32(0), + __cpu_to_le32(0), + __cpu_to_le32(0), + }, +}; + +const struct ce_attr ath12k_wifi7_host_ce_config_qcn9274[] = { + /* CE0: host->target HTC control and raw streams */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 16, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE1: target->host HTT + HTC control */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 512, + .recv_cb = ath12k_htc_rx_completion_handler, + }, + + /* CE2: target->host WMI */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 128, + .recv_cb = ath12k_htc_rx_completion_handler, + }, + + /* CE3: host->target WMI (mac0) */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 32, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE4: host->target HTT */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 2048, + .src_sz_max = 256, + .dest_nentries = 0, + }, + + /* CE5: target->host pktlog */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 512, + .recv_cb = ath12k_dp_htt_htc_t2h_msg_handler, + }, + + /* CE6: target autonomous hif_memcpy */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE7: host->target WMI (mac1) */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 32, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE8: target autonomous hif_memcpy */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE9: MHI */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE10: MHI */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE11: MHI */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE12: CV Prefetch */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE13: CV Prefetch */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE14: target->host dbg log */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 512, + .recv_cb = ath12k_htc_rx_completion_handler, + }, + + /* CE15: reserved for future use */ + { + .flags = (CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, +}; + +/* Copy Engine (CE) configs for WCN7850 */ +/* Target firmware's Copy Engine configuration. */ +const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_wcn7850[] = { + /* CE0: host->target HTC control and raw streams */ + { + .pipenum = __cpu_to_le32(0), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE1: target->host HTT + HTC control */ + { + .pipenum = __cpu_to_le32(1), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE2: target->host WMI */ + { + .pipenum = __cpu_to_le32(2), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE3: host->target WMI */ + { + .pipenum = __cpu_to_le32(3), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE4: host->target HTT */ + { + .pipenum = __cpu_to_le32(4), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(256), + .nbytes_max = __cpu_to_le32(256), + .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), + .reserved = __cpu_to_le32(0), + }, + + /* CE5: target->host Pktlog */ + { + .pipenum = __cpu_to_le32(5), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE6: Reserved for target autonomous hif_memcpy */ + { + .pipenum = __cpu_to_le32(6), + .pipedir = __cpu_to_le32(PIPEDIR_INOUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(16384), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE7 used only by Host */ + { + .pipenum = __cpu_to_le32(7), + .pipedir = __cpu_to_le32(PIPEDIR_INOUT_H2H), + .nentries = __cpu_to_le32(0), + .nbytes_max = __cpu_to_le32(0), + .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), + .reserved = __cpu_to_le32(0), + }, + + /* CE8 target->host used only by IPA */ + { + .pipenum = __cpu_to_le32(8), + .pipedir = __cpu_to_le32(PIPEDIR_INOUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(16384), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + /* CE 9, 10, 11 are used by MHI driver */ +}; + +const struct service_to_pipe +ath12k_wifi7_target_service_to_ce_map_wlan_wcn7850[] = { + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(0), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(4), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(1), + }, + + /* (Additions here) */ + + { /* must be last */ + __cpu_to_le32(0), + __cpu_to_le32(0), + __cpu_to_le32(0), + }, +}; + +const struct ce_attr ath12k_wifi7_host_ce_config_wcn7850[] = { + /* CE0: host->target HTC control and raw streams */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 16, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE1: target->host HTT + HTC control */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 512, + .recv_cb = ath12k_htc_rx_completion_handler, + }, + + /* CE2: target->host WMI */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 64, + .recv_cb = ath12k_htc_rx_completion_handler, + }, + + /* CE3: host->target WMI (mac0) */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 32, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE4: host->target HTT */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 2048, + .src_sz_max = 256, + .dest_nentries = 0, + }, + + /* CE5: target->host pktlog */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE6: target autonomous hif_memcpy */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE7: host->target WMI (mac1) */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE8: target autonomous hif_memcpy */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, +}; + +/* Copy Engine (CE) configs for IPQ5332 */ +/* Target firmware's Copy Engine configuration. */ +const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_ipq5332[] = { + /* CE0: host->target HTC control and raw streams */ + { + .pipenum = __cpu_to_le32(0), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE1: target->host HTT */ + { + .pipenum = __cpu_to_le32(1), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE2: target->host WMI + HTC control */ + { + .pipenum = __cpu_to_le32(2), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE3: host->target WMI */ + { + .pipenum = __cpu_to_le32(3), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE4: host->target HTT */ + { + .pipenum = __cpu_to_le32(4), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(256), + .nbytes_max = __cpu_to_le32(256), + .flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR), + .reserved = __cpu_to_le32(0), + }, + + /* CE5: Target -> host PKTLOG */ + { + .pipenum = __cpu_to_le32(5), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE6: Reserved for target autonomous HIF_memcpy */ + { + .pipenum = __cpu_to_le32(6), + .pipedir = __cpu_to_le32(PIPEDIR_INOUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(16384), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE7: Reserved for CV Prefetch */ + { + .pipenum = __cpu_to_le32(7), + .pipedir = __cpu_to_le32(PIPEDIR_OUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE8: Reserved for target generic HIF memcpy */ + { + .pipenum = __cpu_to_le32(8), + .pipedir = __cpu_to_le32(PIPEDIR_INOUT), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(16384), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE9: WMI logging/CFR/Spectral/Radar/ */ + { + .pipenum = __cpu_to_le32(9), + .pipedir = __cpu_to_le32(PIPEDIR_IN), + .nentries = __cpu_to_le32(32), + .nbytes_max = __cpu_to_le32(2048), + .flags = __cpu_to_le32(CE_ATTR_FLAGS), + .reserved = __cpu_to_le32(0), + }, + + /* CE10: Unused TBD */ + { + .pipenum = __cpu_to_le32(10), + .pipedir = __cpu_to_le32(PIPEDIR_NONE), + .nentries = __cpu_to_le32(0), + .nbytes_max = __cpu_to_le32(0), + .flags = __cpu_to_le32(0), + .reserved = __cpu_to_le32(0), + }, + /* CE11: Unused TBD */ + { + .pipenum = __cpu_to_le32(11), + .pipedir = __cpu_to_le32(PIPEDIR_NONE), + .nentries = __cpu_to_le32(0), + .nbytes_max = __cpu_to_le32(0), + .flags = __cpu_to_le32(0), + .reserved = __cpu_to_le32(0), + }, +}; + +const struct service_to_pipe +ath12k_wifi7_target_service_to_ce_map_wlan_ipq5332[] = { + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(3), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(2), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(0), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(1), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(0), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(1), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), + __cpu_to_le32(PIPEDIR_OUT), + __cpu_to_le32(4), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(1), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(5), + }, + { + __cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG), + __cpu_to_le32(PIPEDIR_IN), + __cpu_to_le32(9), + }, + /* (Additions here) */ + + { /* must be last */ + __cpu_to_le32(0), + __cpu_to_le32(0), + __cpu_to_le32(0), + }, +}; + +const struct ce_attr ath12k_wifi7_host_ce_config_ipq5332[] = { + /* CE0: host->target HTC control and raw streams */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 16, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE1: target->host HTT + HTC control */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 512, + .recv_cb = ath12k_htc_rx_completion_handler, + }, + + /* CE2: target->host WMI */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 128, + .recv_cb = ath12k_htc_rx_completion_handler, + }, + + /* CE3: host->target WMI */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 32, + .src_sz_max = 2048, + .dest_nentries = 0, + }, + + /* CE4: host->target HTT */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 2048, + .src_sz_max = 256, + .dest_nentries = 0, + }, + + /* CE5: target -> host PKTLOG */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 512, + .recv_cb = ath12k_dp_htt_htc_t2h_msg_handler, + }, + + /* CE6: Target autonomous HIF_memcpy */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE7: CV Prefetch */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE8: Target HIF memcpy (Generic HIF memcypy) */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE9: WMI logging/CFR/Spectral/Radar */ + { + .flags = CE_ATTR_FLAGS, + .src_nentries = 0, + .src_sz_max = 2048, + .dest_nentries = 128, + }, + + /* CE10: Unused */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, + + /* CE11: Unused */ + { + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, + .src_nentries = 0, + .src_sz_max = 0, + .dest_nentries = 0, + }, +}; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ce.h b/drivers/net/wireless/ath/ath12k/wifi7/ce.h new file mode 100644 index 0000000000000..369a14472913b --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/ce.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_WIFI7_CE_H +#define ATH12K_WIFI7_CE_H + +extern const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_qcn9274[]; +extern const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_wcn7850[]; +extern const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_ipq5332[]; + +extern const struct service_to_pipe ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274[]; +extern const struct service_to_pipe ath12k_wifi7_target_service_to_ce_map_wlan_wcn7850[]; +extern const struct service_to_pipe ath12k_wifi7_target_service_to_ce_map_wlan_ipq5332[]; + +extern const struct ce_attr ath12k_wifi7_host_ce_config_qcn9274[]; +extern const struct ce_attr ath12k_wifi7_host_ce_config_wcn7850[]; +extern const struct ce_attr ath12k_wifi7_host_ce_config_ipq5332[]; + +#endif /* ATH12K_WIFI7_CE_H */ diff --git a/drivers/net/wireless/ath/ath12k/wifi7/core.c b/drivers/net/wireless/ath/ath12k/wifi7/core.c new file mode 100644 index 0000000000000..a02c57acf1374 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/core.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include "../ahb.h" +#include "../pci.h" +#include "pci.h" +#include "ahb.h" +#include "core.h" +#include "dp.h" +#include "../debug.h" + +static int ahb_err, pci_err; + +int ath12k_wifi7_arch_init(struct ath12k_base *ab) +{ + struct ath12k_dp *dp; + + dp = ath12k_wifi7_dp_device_alloc(ab); + if (!dp) { + ath12k_err(ab, "dp alloc failed"); + return -EINVAL; + } + + ab->dp = dp; + + return 0; +} + +void ath12k_wifi7_arch_deinit(struct ath12k_base *ab) +{ + ath12k_wifi7_dp_device_free(ab->dp); + ab->dp = NULL; +} + +static int ath12k_wifi7_init(void) +{ + ahb_err = ath12k_wifi7_ahb_init(); + if (ahb_err) + pr_warn("Failed to initialize ath12k Wi-Fi 7 AHB device: %d\n", + ahb_err); + + pci_err = ath12k_wifi7_pci_init(); + if (pci_err) + pr_warn("Failed to initialize ath12k Wi-Fi 7 PCI device: %d\n", + pci_err); + + /* If both failed, return one of the failures (arbitrary) */ + return ahb_err && pci_err ? ahb_err : 0; +} + +static void ath12k_wifi7_exit(void) +{ + if (!pci_err) + ath12k_wifi7_pci_exit(); + + if (!ahb_err) + ath12k_wifi7_ahb_exit(); +} + +module_init(ath12k_wifi7_init); +module_exit(ath12k_wifi7_exit); + +MODULE_DESCRIPTION("Driver support for Qualcomm Technologies 802.11be WLAN devices"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/ath/ath12k/wifi7/core.h b/drivers/net/wireless/ath/ath12k/wifi7/core.h new file mode 100644 index 0000000000000..7e9689d2ddd73 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/core.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#ifndef ATH12K_CORE_WIFI7_H +#define ATH12K_CORE_WIFI7_H + +int ath12k_wifi7_arch_init(struct ath12k_base *ab); +void ath12k_wifi7_arch_deinit(struct ath12k_base *ab); + +#endif diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp.c b/drivers/net/wireless/ath/ath12k/wifi7/dp.c new file mode 100644 index 0000000000000..2b194879ee80d --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#include "../core.h" +#include "../debug.h" +#include "../dp_rx.h" +#include "../dp_tx.h" +#include "hal_desc.h" +#include "../dp_mon.h" +#include "dp_mon.h" +#include "../dp_cmn.h" +#include "dp_rx.h" +#include "dp.h" +#include "dp_tx.h" +#include "hal.h" + +static int ath12k_wifi7_dp_service_srng(struct ath12k_dp *dp, + struct ath12k_ext_irq_grp *irq_grp, + int budget) +{ + struct napi_struct *napi = &irq_grp->napi; + int grp_id = irq_grp->grp_id; + int work_done = 0; + int i = 0, j; + int tot_work_done = 0; + enum dp_monitor_mode monitor_mode; + u8 ring_mask; + + if (dp->hw_params->ring_mask->tx[grp_id]) { + i = fls(dp->hw_params->ring_mask->tx[grp_id]) - 1; + ath12k_wifi7_dp_tx_completion_handler(dp, i); + } + + if (dp->hw_params->ring_mask->rx_err[grp_id]) { + work_done = ath12k_wifi7_dp_rx_process_err(dp, napi, budget); + budget -= work_done; + tot_work_done += work_done; + if (budget <= 0) + goto done; + } + + if (dp->hw_params->ring_mask->rx_wbm_rel[grp_id]) { + work_done = ath12k_wifi7_dp_rx_process_wbm_err(dp, napi, budget); + budget -= work_done; + tot_work_done += work_done; + + if (budget <= 0) + goto done; + } + + if (dp->hw_params->ring_mask->rx[grp_id]) { + i = fls(dp->hw_params->ring_mask->rx[grp_id]) - 1; + work_done = ath12k_wifi7_dp_rx_process(dp, i, napi, budget); + budget -= work_done; + tot_work_done += work_done; + if (budget <= 0) + goto done; + } + + if (dp->hw_params->ring_mask->rx_mon_status[grp_id]) { + ring_mask = dp->hw_params->ring_mask->rx_mon_status[grp_id]; + for (i = 0; i < dp->ab->num_radios; i++) { + for (j = 0; j < dp->hw_params->num_rxdma_per_pdev; j++) { + int id = i * dp->hw_params->num_rxdma_per_pdev + j; + + if (ring_mask & BIT(id)) { + work_done = + ath12k_wifi7_dp_mon_process_ring(dp, id, napi, + budget, + 0); + budget -= work_done; + tot_work_done += work_done; + if (budget <= 0) + goto done; + } + } + } + } + + if (dp->hw_params->ring_mask->rx_mon_dest[grp_id]) { + monitor_mode = ATH12K_DP_RX_MONITOR_MODE; + ring_mask = dp->hw_params->ring_mask->rx_mon_dest[grp_id]; + for (i = 0; i < dp->ab->num_radios; i++) { + for (j = 0; j < dp->hw_params->num_rxdma_per_pdev; j++) { + int id = i * dp->hw_params->num_rxdma_per_pdev + j; + + if (ring_mask & BIT(id)) { + work_done = + ath12k_wifi7_dp_mon_process_ring(dp, id, napi, + budget, + monitor_mode); + budget -= work_done; + tot_work_done += work_done; + + if (budget <= 0) + goto done; + } + } + } + } + + if (dp->hw_params->ring_mask->tx_mon_dest[grp_id]) { + monitor_mode = ATH12K_DP_TX_MONITOR_MODE; + ring_mask = dp->hw_params->ring_mask->tx_mon_dest[grp_id]; + for (i = 0; i < dp->ab->num_radios; i++) { + for (j = 0; j < dp->hw_params->num_rxdma_per_pdev; j++) { + int id = i * dp->hw_params->num_rxdma_per_pdev + j; + + if (ring_mask & BIT(id)) { + work_done = + ath12k_wifi7_dp_mon_process_ring(dp, id, + napi, budget, + monitor_mode); + budget -= work_done; + tot_work_done += work_done; + + if (budget <= 0) + goto done; + } + } + } + } + + if (dp->hw_params->ring_mask->reo_status[grp_id]) + ath12k_wifi7_dp_rx_process_reo_status(dp); + + if (dp->hw_params->ring_mask->host2rxdma[grp_id]) { + struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; + LIST_HEAD(list); + + ath12k_dp_rx_bufs_replenish(dp, rx_ring, &list, 0); + } + + /* TODO: Implement handler for other interrupts */ + +done: + return tot_work_done; +} + +static struct ath12k_dp_arch_ops ath12k_wifi7_dp_arch_ops = { + .service_srng = ath12k_wifi7_dp_service_srng, + .tx_get_vdev_bank_config = ath12k_wifi7_dp_tx_get_vdev_bank_config, + .reo_cmd_send = ath12k_wifi7_dp_reo_cmd_send, + .setup_pn_check_reo_cmd = ath12k_wifi7_dp_setup_pn_check_reo_cmd, + .rx_peer_tid_delete = ath12k_wifi7_dp_rx_peer_tid_delete, + .reo_cache_flush = ath12k_wifi7_dp_reo_cache_flush, + .rx_link_desc_return = ath12k_wifi7_dp_rx_link_desc_return, + .rx_frags_cleanup = ath12k_wifi7_dp_rx_frags_cleanup, + .peer_rx_tid_reo_update = ath12k_wifi7_peer_rx_tid_reo_update, + .rx_assign_reoq = ath12k_wifi7_dp_rx_assign_reoq, + .peer_rx_tid_qref_setup = ath12k_wifi7_peer_rx_tid_qref_setup, + .peer_rx_tid_qref_reset = ath12k_wifi7_peer_rx_tid_qref_reset, + .rx_tid_delete_handler = ath12k_wifi7_dp_rx_tid_delete_handler, +}; + +/* TODO: remove export once this file is built with wifi7 ko */ +struct ath12k_dp *ath12k_wifi7_dp_device_alloc(struct ath12k_base *ab) +{ + struct ath12k_dp *dp; + + /* TODO: align dp later if cache alignment becomes a bottleneck */ + dp = kzalloc(sizeof(*dp), GFP_KERNEL); + if (!dp) + return NULL; + + dp->ab = ab; + dp->dev = ab->dev; + dp->hw_params = ab->hw_params; + dp->hal = &ab->hal; + + dp->ops = &ath12k_wifi7_dp_arch_ops; + + return dp; +} + +void ath12k_wifi7_dp_device_free(struct ath12k_dp *dp) +{ + kfree(dp); +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp.h b/drivers/net/wireless/ath/ath12k/wifi7/dp.h new file mode 100644 index 0000000000000..a5f0941d34e2f --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_DP_WIFI7_H +#define ATH12K_DP_WIFI7_H + +#include "../dp_cmn.h" +#include "hw.h" + +struct ath12k_base; +struct ath12k_dp; +enum dp_monitor_mode; + +struct ath12k_dp *ath12k_wifi7_dp_device_alloc(struct ath12k_base *ab); +void ath12k_wifi7_dp_device_free(struct ath12k_dp *dp); + +#endif diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c new file mode 100644 index 0000000000000..2013b4146b1aa --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c @@ -0,0 +1,3385 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "hal_desc.h" +#include "../dp_mon.h" +#include "dp_mon.h" +#include "../debug.h" +#include "hal_qcn9274.h" +#include "dp_rx.h" +#include "../dp_tx.h" +#include "../peer.h" + +static void +ath12k_wifi7_dp_mon_hal_aggr_tlv(struct hal_rx_mon_ppdu_info *ppdu_info, + u16 tlv_len, const void *tlv_data) +{ + if (tlv_len <= HAL_RX_MON_MAX_AGGR_SIZE - ppdu_info->tlv_aggr.cur_len) { + memcpy(ppdu_info->tlv_aggr.buf + ppdu_info->tlv_aggr.cur_len, + tlv_data, tlv_len); + ppdu_info->tlv_aggr.cur_len += tlv_len; + } +} + +static void +ath12k_wifi7_dp_mon_rx_memset_ppdu_info(struct hal_rx_mon_ppdu_info *ppdu_info) +{ + memset(ppdu_info, 0, sizeof(*ppdu_info)); + ppdu_info->peer_id = HAL_INVALID_PEERID; +} + +/* Hardware fill buffer with 128 bytes aligned. So need to reap it + * with 128 bytes aligned. + */ +#define RXDMA_DATA_DMA_BLOCK_SIZE 128 + +static void +ath12k_wifi7_dp_mon_get_buf_len(struct hal_rx_msdu_desc_info *info, + bool *is_frag, u32 *total_len, + u32 *frag_len, u32 *msdu_cnt) +{ + if (info->msdu_flags & RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) { + *is_frag = true; + *frag_len = (RX_MON_STATUS_BASE_BUF_SIZE - + sizeof(struct hal_rx_desc)) & + ~(RXDMA_DATA_DMA_BLOCK_SIZE - 1); + *total_len += *frag_len; + } else { + if (*is_frag) + *frag_len = info->msdu_len - *total_len; + else + *frag_len = info->msdu_len; + + *msdu_cnt -= 1; + } +} + +static void +ath12k_wifi7_dp_mon_rx_handle_ofdma_info(const struct hal_rx_ppdu_end_user_stats *ppdu_end_user, + struct hal_rx_user_status *rx_user_status) +{ + rx_user_status->ul_ofdma_user_v0_word0 = + __le32_to_cpu(ppdu_end_user->usr_resp_ref); + rx_user_status->ul_ofdma_user_v0_word1 = + __le32_to_cpu(ppdu_end_user->usr_resp_ref_ext); +} + +static void +ath12k_wifi7_dp_mon_rx_populate_byte_count(const struct hal_rx_ppdu_end_user_stats *stats, + void *ppduinfo, + struct hal_rx_user_status *rx_user_status) +{ + rx_user_status->mpdu_ok_byte_count = + le32_get_bits(stats->info7, + HAL_RX_PPDU_END_USER_STATS_INFO7_MPDU_OK_BYTE_COUNT); + rx_user_status->mpdu_err_byte_count = + le32_get_bits(stats->info8, + HAL_RX_PPDU_END_USER_STATS_INFO8_MPDU_ERR_BYTE_COUNT); +} + +static void +ath12k_wifi7_dp_mon_rx_populate_mu_user_info(const struct hal_rx_ppdu_end_user_stats *rx_tlv, + struct hal_rx_mon_ppdu_info *ppdu_info, + struct hal_rx_user_status *rx_user_status) +{ + rx_user_status->ast_index = ppdu_info->ast_index; + rx_user_status->tid = ppdu_info->tid; + rx_user_status->tcp_ack_msdu_count = + ppdu_info->tcp_ack_msdu_count; + rx_user_status->tcp_msdu_count = + ppdu_info->tcp_msdu_count; + rx_user_status->udp_msdu_count = + ppdu_info->udp_msdu_count; + rx_user_status->other_msdu_count = + ppdu_info->other_msdu_count; + rx_user_status->frame_control = ppdu_info->frame_control; + rx_user_status->frame_control_info_valid = + ppdu_info->frame_control_info_valid; + rx_user_status->data_sequence_control_info_valid = + ppdu_info->data_sequence_control_info_valid; + rx_user_status->first_data_seq_ctrl = + ppdu_info->first_data_seq_ctrl; + rx_user_status->preamble_type = ppdu_info->preamble_type; + rx_user_status->ht_flags = ppdu_info->ht_flags; + rx_user_status->vht_flags = ppdu_info->vht_flags; + rx_user_status->he_flags = ppdu_info->he_flags; + rx_user_status->rs_flags = ppdu_info->rs_flags; + + rx_user_status->mpdu_cnt_fcs_ok = + ppdu_info->num_mpdu_fcs_ok; + rx_user_status->mpdu_cnt_fcs_err = + ppdu_info->num_mpdu_fcs_err; + memcpy(&rx_user_status->mpdu_fcs_ok_bitmap[0], &ppdu_info->mpdu_fcs_ok_bitmap[0], + HAL_RX_NUM_WORDS_PER_PPDU_BITMAP * + sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0])); + + ath12k_wifi7_dp_mon_rx_populate_byte_count(rx_tlv, ppdu_info, rx_user_status); +} + +static inline enum ath12k_eht_ru_size +hal_rx_mon_hal_ru_size_to_ath12k_ru_size(u32 hal_ru_size) +{ + switch (hal_ru_size) { + case HAL_EHT_RU_26: + return ATH12K_EHT_RU_26; + case HAL_EHT_RU_52: + return ATH12K_EHT_RU_52; + case HAL_EHT_RU_78: + return ATH12K_EHT_RU_52_26; + case HAL_EHT_RU_106: + return ATH12K_EHT_RU_106; + case HAL_EHT_RU_132: + return ATH12K_EHT_RU_106_26; + case HAL_EHT_RU_242: + return ATH12K_EHT_RU_242; + case HAL_EHT_RU_484: + return ATH12K_EHT_RU_484; + case HAL_EHT_RU_726: + return ATH12K_EHT_RU_484_242; + case HAL_EHT_RU_996: + return ATH12K_EHT_RU_996; + case HAL_EHT_RU_996x2: + return ATH12K_EHT_RU_996x2; + case HAL_EHT_RU_996x3: + return ATH12K_EHT_RU_996x3; + case HAL_EHT_RU_996x4: + return ATH12K_EHT_RU_996x4; + case HAL_EHT_RU_NONE: + return ATH12K_EHT_RU_INVALID; + case HAL_EHT_RU_996_484: + return ATH12K_EHT_RU_996_484; + case HAL_EHT_RU_996x2_484: + return ATH12K_EHT_RU_996x2_484; + case HAL_EHT_RU_996x3_484: + return ATH12K_EHT_RU_996x3_484; + case HAL_EHT_RU_996_484_242: + return ATH12K_EHT_RU_996_484_242; + default: + return ATH12K_EHT_RU_INVALID; + } +} + +static inline u32 +hal_rx_ul_ofdma_ru_size_to_width(enum ath12k_eht_ru_size ru_size) +{ + switch (ru_size) { + case ATH12K_EHT_RU_26: + return RU_26; + case ATH12K_EHT_RU_52: + return RU_52; + case ATH12K_EHT_RU_52_26: + return RU_52_26; + case ATH12K_EHT_RU_106: + return RU_106; + case ATH12K_EHT_RU_106_26: + return RU_106_26; + case ATH12K_EHT_RU_242: + return RU_242; + case ATH12K_EHT_RU_484: + return RU_484; + case ATH12K_EHT_RU_484_242: + return RU_484_242; + case ATH12K_EHT_RU_996: + return RU_996; + case ATH12K_EHT_RU_996_484: + return RU_996_484; + case ATH12K_EHT_RU_996_484_242: + return RU_996_484_242; + case ATH12K_EHT_RU_996x2: + return RU_2X996; + case ATH12K_EHT_RU_996x2_484: + return RU_2X996_484; + case ATH12K_EHT_RU_996x3: + return RU_3X996; + case ATH12K_EHT_RU_996x3_484: + return RU_3X996_484; + case ATH12K_EHT_RU_996x4: + return RU_4X996; + default: + return RU_INVALID; + } +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_user_info(const struct hal_receive_user_info *rx_usr_info, + u16 user_id, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct hal_rx_user_status *mon_rx_user_status = NULL; + struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; + enum ath12k_eht_ru_size rtap_ru_size = ATH12K_EHT_RU_INVALID; + u32 ru_width, reception_type, ru_index = HAL_EHT_RU_INVALID; + u32 ru_type_80_0, ru_start_index_80_0; + u32 ru_type_80_1, ru_start_index_80_1; + u32 ru_type_80_2, ru_start_index_80_2; + u32 ru_type_80_3, ru_start_index_80_3; + u32 ru_size = 0, num_80mhz_with_ru = 0; + u64 ru_index_320mhz = 0; + u32 ru_index_per80mhz; + + reception_type = le32_get_bits(rx_usr_info->info0, + HAL_RX_USR_INFO0_RECEPTION_TYPE); + + switch (reception_type) { + case HAL_RECEPTION_TYPE_SU: + ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_SU; + break; + case HAL_RECEPTION_TYPE_DL_MU_MIMO: + case HAL_RECEPTION_TYPE_UL_MU_MIMO: + ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_MIMO; + break; + case HAL_RECEPTION_TYPE_DL_MU_OFMA: + case HAL_RECEPTION_TYPE_UL_MU_OFDMA: + ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA; + break; + case HAL_RECEPTION_TYPE_DL_MU_OFDMA_MIMO: + case HAL_RECEPTION_TYPE_UL_MU_OFDMA_MIMO: + ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO; + } + + ppdu_info->is_stbc = le32_get_bits(rx_usr_info->info0, HAL_RX_USR_INFO0_STBC); + ppdu_info->ldpc = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_LDPC); + ppdu_info->dcm = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_STA_DCM); + ppdu_info->bw = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_RX_BW); + ppdu_info->mcs = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_MCS); + ppdu_info->nss = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_NSS) + 1; + + if (user_id < HAL_MAX_UL_MU_USERS) { + mon_rx_user_status = &ppdu_info->userstats[user_id]; + mon_rx_user_status->mcs = ppdu_info->mcs; + mon_rx_user_status->nss = ppdu_info->nss; + } + + if (!(ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_MIMO || + ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA || + ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO)) + return; + + /* RU allocation present only for OFDMA reception */ + ru_type_80_0 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_0); + ru_start_index_80_0 = le32_get_bits(rx_usr_info->info3, + HAL_RX_USR_INFO3_RU_START_IDX_80_0); + if (ru_type_80_0 != HAL_EHT_RU_NONE) { + ru_size += ru_type_80_0; + ru_index_per80mhz = ru_start_index_80_0; + ru_index = ru_index_per80mhz; + ru_index_320mhz |= HAL_RU_PER80(ru_type_80_0, 0, ru_index_per80mhz); + num_80mhz_with_ru++; + } + + ru_type_80_1 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_1); + ru_start_index_80_1 = le32_get_bits(rx_usr_info->info3, + HAL_RX_USR_INFO3_RU_START_IDX_80_1); + if (ru_type_80_1 != HAL_EHT_RU_NONE) { + ru_size += ru_type_80_1; + ru_index_per80mhz = ru_start_index_80_1; + ru_index = ru_index_per80mhz; + ru_index_320mhz |= HAL_RU_PER80(ru_type_80_1, 1, ru_index_per80mhz); + num_80mhz_with_ru++; + } + + ru_type_80_2 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_2); + ru_start_index_80_2 = le32_get_bits(rx_usr_info->info3, + HAL_RX_USR_INFO3_RU_START_IDX_80_2); + if (ru_type_80_2 != HAL_EHT_RU_NONE) { + ru_size += ru_type_80_2; + ru_index_per80mhz = ru_start_index_80_2; + ru_index = ru_index_per80mhz; + ru_index_320mhz |= HAL_RU_PER80(ru_type_80_2, 2, ru_index_per80mhz); + num_80mhz_with_ru++; + } + + ru_type_80_3 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_3); + ru_start_index_80_3 = le32_get_bits(rx_usr_info->info2, + HAL_RX_USR_INFO3_RU_START_IDX_80_3); + if (ru_type_80_3 != HAL_EHT_RU_NONE) { + ru_size += ru_type_80_3; + ru_index_per80mhz = ru_start_index_80_3; + ru_index = ru_index_per80mhz; + ru_index_320mhz |= HAL_RU_PER80(ru_type_80_3, 3, ru_index_per80mhz); + num_80mhz_with_ru++; + } + + if (num_80mhz_with_ru > 1) { + /* Calculate the MRU index */ + switch (ru_index_320mhz) { + case HAL_EHT_RU_996_484_0: + case HAL_EHT_RU_996x2_484_0: + case HAL_EHT_RU_996x3_484_0: + ru_index = 0; + break; + case HAL_EHT_RU_996_484_1: + case HAL_EHT_RU_996x2_484_1: + case HAL_EHT_RU_996x3_484_1: + ru_index = 1; + break; + case HAL_EHT_RU_996_484_2: + case HAL_EHT_RU_996x2_484_2: + case HAL_EHT_RU_996x3_484_2: + ru_index = 2; + break; + case HAL_EHT_RU_996_484_3: + case HAL_EHT_RU_996x2_484_3: + case HAL_EHT_RU_996x3_484_3: + ru_index = 3; + break; + case HAL_EHT_RU_996_484_4: + case HAL_EHT_RU_996x2_484_4: + case HAL_EHT_RU_996x3_484_4: + ru_index = 4; + break; + case HAL_EHT_RU_996_484_5: + case HAL_EHT_RU_996x2_484_5: + case HAL_EHT_RU_996x3_484_5: + ru_index = 5; + break; + case HAL_EHT_RU_996_484_6: + case HAL_EHT_RU_996x2_484_6: + case HAL_EHT_RU_996x3_484_6: + ru_index = 6; + break; + case HAL_EHT_RU_996_484_7: + case HAL_EHT_RU_996x2_484_7: + case HAL_EHT_RU_996x3_484_7: + ru_index = 7; + break; + case HAL_EHT_RU_996x2_484_8: + ru_index = 8; + break; + case HAL_EHT_RU_996x2_484_9: + ru_index = 9; + break; + case HAL_EHT_RU_996x2_484_10: + ru_index = 10; + break; + case HAL_EHT_RU_996x2_484_11: + ru_index = 11; + break; + default: + ru_index = HAL_EHT_RU_INVALID; + break; + } + + ru_size += 4; + } + + rtap_ru_size = hal_rx_mon_hal_ru_size_to_ath12k_ru_size(ru_size); + if (rtap_ru_size != ATH12K_EHT_RU_INVALID) { + u32 known, data; + + known = __le32_to_cpu(eht->known); + known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_SIZE_OM; + eht->known = cpu_to_le32(known); + + data = __le32_to_cpu(eht->data[1]); + data |= u32_encode_bits(rtap_ru_size, + IEEE80211_RADIOTAP_EHT_DATA1_RU_SIZE); + eht->data[1] = cpu_to_le32(data); + } + + if (ru_index != HAL_EHT_RU_INVALID) { + u32 known, data; + + known = __le32_to_cpu(eht->known); + known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_INDEX_OM; + eht->known = cpu_to_le32(known); + + data = __le32_to_cpu(eht->data[1]); + data |= u32_encode_bits(rtap_ru_size, + IEEE80211_RADIOTAP_EHT_DATA1_RU_INDEX); + eht->data[1] = cpu_to_le32(data); + } + + if (mon_rx_user_status && ru_index != HAL_EHT_RU_INVALID && + rtap_ru_size != ATH12K_EHT_RU_INVALID) { + mon_rx_user_status->ul_ofdma_ru_start_index = ru_index; + mon_rx_user_status->ul_ofdma_ru_size = rtap_ru_size; + + ru_width = hal_rx_ul_ofdma_ru_size_to_width(rtap_ru_size); + + mon_rx_user_status->ul_ofdma_ru_width = ru_width; + mon_rx_user_status->ofdma_info_valid = 1; + } +} + +static void +ath12k_wifi7_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0 = __le32_to_cpu(lsigb->info0); + u8 rate; + + rate = u32_get_bits(info0, HAL_RX_LSIG_B_INFO_INFO0_RATE); + switch (rate) { + case 1: + rate = HAL_RX_LEGACY_RATE_1_MBPS; + break; + case 2: + case 5: + rate = HAL_RX_LEGACY_RATE_2_MBPS; + break; + case 3: + case 6: + rate = HAL_RX_LEGACY_RATE_5_5_MBPS; + break; + case 4: + case 7: + rate = HAL_RX_LEGACY_RATE_11_MBPS; + break; + default: + rate = HAL_RX_LEGACY_RATE_INVALID; + } + + ppdu_info->rate = rate; + ppdu_info->cck_flag = 1; +} + +static void +ath12k_wifi7_dp_mon_parse_l_sig_a(const struct hal_rx_lsig_a_info *lsiga, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0 = __le32_to_cpu(lsiga->info0); + u8 rate; + + rate = u32_get_bits(info0, HAL_RX_LSIG_A_INFO_INFO0_RATE); + switch (rate) { + case 8: + rate = HAL_RX_LEGACY_RATE_48_MBPS; + break; + case 9: + rate = HAL_RX_LEGACY_RATE_24_MBPS; + break; + case 10: + rate = HAL_RX_LEGACY_RATE_12_MBPS; + break; + case 11: + rate = HAL_RX_LEGACY_RATE_6_MBPS; + break; + case 12: + rate = HAL_RX_LEGACY_RATE_54_MBPS; + break; + case 13: + rate = HAL_RX_LEGACY_RATE_36_MBPS; + break; + case 14: + rate = HAL_RX_LEGACY_RATE_18_MBPS; + break; + case 15: + rate = HAL_RX_LEGACY_RATE_9_MBPS; + break; + default: + rate = HAL_RX_LEGACY_RATE_INVALID; + } + + ppdu_info->rate = rate; +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_cmn(const struct hal_mon_usig_cmn *cmn, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 common; + + ppdu_info->u_sig_info.bw = le32_get_bits(cmn->info0, + HAL_RX_USIG_CMN_INFO0_BW); + ppdu_info->u_sig_info.ul_dl = le32_get_bits(cmn->info0, + HAL_RX_USIG_CMN_INFO0_UL_DL); + + common = __le32_to_cpu(ppdu_info->u_sig_info.usig.common); + common |= IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER_KNOWN | + IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW_KNOWN | + IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL_KNOWN | + IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR_KNOWN | + IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP_KNOWN | + ATH12K_LE32_DEC_ENC(cmn->info0, + HAL_RX_USIG_CMN_INFO0_PHY_VERSION, + IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER) | + u32_encode_bits(ppdu_info->u_sig_info.bw, + IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW) | + u32_encode_bits(ppdu_info->u_sig_info.ul_dl, + IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL) | + ATH12K_LE32_DEC_ENC(cmn->info0, + HAL_RX_USIG_CMN_INFO0_BSS_COLOR, + IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR) | + ATH12K_LE32_DEC_ENC(cmn->info0, + HAL_RX_USIG_CMN_INFO0_TXOP, + IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP); + ppdu_info->u_sig_info.usig.common = cpu_to_le32(common); + + switch (ppdu_info->u_sig_info.bw) { + default: + fallthrough; + case HAL_EHT_BW_20: + ppdu_info->bw = HAL_RX_BW_20MHZ; + break; + case HAL_EHT_BW_40: + ppdu_info->bw = HAL_RX_BW_40MHZ; + break; + case HAL_EHT_BW_80: + ppdu_info->bw = HAL_RX_BW_80MHZ; + break; + case HAL_EHT_BW_160: + ppdu_info->bw = HAL_RX_BW_160MHZ; + break; + case HAL_EHT_BW_320_1: + case HAL_EHT_BW_320_2: + ppdu_info->bw = HAL_RX_BW_320MHZ; + break; + } +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_tb(const struct hal_mon_usig_tb *usig_tb, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig; + enum ieee80211_radiotap_eht_usig_tb spatial_reuse1, spatial_reuse2; + u32 common, value, mask; + + spatial_reuse1 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1; + spatial_reuse2 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2; + + common = __le32_to_cpu(usig->common); + value = __le32_to_cpu(usig->value); + mask = __le32_to_cpu(usig->mask); + + ppdu_info->u_sig_info.ppdu_type_comp_mode = + le32_get_bits(usig_tb->info0, + HAL_RX_USIG_TB_INFO0_PPDU_TYPE_COMP_MODE); + + common |= ATH12K_LE32_DEC_ENC(usig_tb->info0, + HAL_RX_USIG_TB_INFO0_RX_INTEG_CHECK_PASS, + IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC); + + value |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD | + u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode, + IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE) | + IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE | + ATH12K_LE32_DEC_ENC(usig_tb->info0, + HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_1, + spatial_reuse1) | + ATH12K_LE32_DEC_ENC(usig_tb->info0, + HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_2, + spatial_reuse2) | + IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD | + ATH12K_LE32_DEC_ENC(usig_tb->info0, + HAL_RX_USIG_TB_INFO0_CRC, + IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC) | + ATH12K_LE32_DEC_ENC(usig_tb->info0, + HAL_RX_USIG_TB_INFO0_TAIL, + IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL); + + mask |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD | + IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE | + IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE | + spatial_reuse1 | spatial_reuse2 | + IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD | + IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC | + IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL; + + usig->common = cpu_to_le32(common); + usig->value = cpu_to_le32(value); + usig->mask = cpu_to_le32(mask); +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_mu(const struct hal_mon_usig_mu *usig_mu, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig; + enum ieee80211_radiotap_eht_usig_mu sig_symb, punc; + u32 common, value, mask; + + sig_symb = IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS; + punc = IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO; + + common = __le32_to_cpu(usig->common); + value = __le32_to_cpu(usig->value); + mask = __le32_to_cpu(usig->mask); + + ppdu_info->u_sig_info.ppdu_type_comp_mode = + le32_get_bits(usig_mu->info0, + HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE); + ppdu_info->u_sig_info.eht_sig_mcs = + le32_get_bits(usig_mu->info0, + HAL_RX_USIG_MU_INFO0_EHT_SIG_MCS); + ppdu_info->u_sig_info.num_eht_sig_sym = + le32_get_bits(usig_mu->info0, + HAL_RX_USIG_MU_INFO0_NUM_EHT_SIG_SYM); + + common |= ATH12K_LE32_DEC_ENC(usig_mu->info0, + HAL_RX_USIG_MU_INFO0_RX_INTEG_CHECK_PASS, + IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC); + + value |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD | + IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE | + u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode, + IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE) | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE | + ATH12K_LE32_DEC_ENC(usig_mu->info0, + HAL_RX_USIG_MU_INFO0_PUNC_CH_INFO, + punc) | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE | + u32_encode_bits(ppdu_info->u_sig_info.eht_sig_mcs, + IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS) | + u32_encode_bits(ppdu_info->u_sig_info.num_eht_sig_sym, + sig_symb) | + ATH12K_LE32_DEC_ENC(usig_mu->info0, + HAL_RX_USIG_MU_INFO0_CRC, + IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC) | + ATH12K_LE32_DEC_ENC(usig_mu->info0, + HAL_RX_USIG_MU_INFO0_TAIL, + IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL); + + mask |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD | + IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE | + punc | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS | + sig_symb | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC | + IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL; + + usig->common = cpu_to_le32(common); + usig->value = cpu_to_le32(value); + usig->mask = cpu_to_le32(mask); +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_hdr(const struct hal_mon_usig_hdr *usig, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u8 comp_mode; + + ppdu_info->eht_usig = true; + + ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_cmn(&usig->cmn, ppdu_info); + + comp_mode = le32_get_bits(usig->non_cmn.mu.info0, + HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE); + + if (comp_mode == 0 && ppdu_info->u_sig_info.ul_dl) + ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_tb(&usig->non_cmn.tb, ppdu_info); + else + ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_mu(&usig->non_cmn.mu, ppdu_info); +} + +static void +ath12k_wifi7_dp_mon_parse_vht_sig_a(const struct hal_rx_vht_sig_a_info *vht_sig, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 nsts, info0, info1; + u8 gi_setting; + + info0 = __le32_to_cpu(vht_sig->info0); + info1 = __le32_to_cpu(vht_sig->info1); + + ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING); + ppdu_info->mcs = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_MCS); + gi_setting = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_GI_SETTING); + switch (gi_setting) { + case HAL_RX_VHT_SIG_A_NORMAL_GI: + ppdu_info->gi = HAL_RX_GI_0_8_US; + break; + case HAL_RX_VHT_SIG_A_SHORT_GI: + case HAL_RX_VHT_SIG_A_SHORT_GI_AMBIGUITY: + ppdu_info->gi = HAL_RX_GI_0_4_US; + break; + } + + ppdu_info->is_stbc = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_STBC); + nsts = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_NSTS); + if (ppdu_info->is_stbc && nsts > 0) + nsts = ((nsts + 1) >> 1) - 1; + + ppdu_info->nss = u32_get_bits(nsts, VHT_SIG_SU_NSS_MASK) + 1; + ppdu_info->bw = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_BW); + ppdu_info->beamformed = u32_get_bits(info1, + HAL_RX_VHT_SIG_A_INFO_INFO1_BEAMFORMED); + ppdu_info->vht_flag_values5 = u32_get_bits(info0, + HAL_RX_VHT_SIG_A_INFO_INFO0_GROUP_ID); + ppdu_info->vht_flag_values3[0] = (((ppdu_info->mcs) << 4) | + ppdu_info->nss); + ppdu_info->vht_flag_values2 = ppdu_info->bw; + ppdu_info->vht_flag_values4 = + u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING); +} + +static void +ath12k_wifi7_dp_mon_parse_ht_sig(const struct hal_rx_ht_sig_info *ht_sig, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0 = __le32_to_cpu(ht_sig->info0); + u32 info1 = __le32_to_cpu(ht_sig->info1); + + ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_MCS); + ppdu_info->bw = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_BW); + ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_STBC); + ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_FEC_CODING); + ppdu_info->gi = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_GI); + ppdu_info->nss = (ppdu_info->mcs >> 3) + 1; +} + +static void +ath12k_wifi7_dp_mon_parse_he_sig_b2_ofdma(const struct hal_rx_he_sig_b2_ofdma_info *ofdma, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0, value; + + info0 = __le32_to_cpu(ofdma->info0); + + ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_DCM_KNOWN | HE_CODING_KNOWN; + + /* HE-data2 */ + ppdu_info->he_data2 |= HE_TXBF_KNOWN; + + ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_MCS); + value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_DCM); + value = value << HE_DCM_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_CODING); + ppdu_info->ldpc = value; + value = value << HE_CODING_SHIFT; + ppdu_info->he_data3 |= value; + + /* HE-data4 */ + value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_ID); + value = value << HE_STA_ID_SHIFT; + ppdu_info->he_data4 |= value; + + ppdu_info->nss = + u32_get_bits(info0, + HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_NSTS) + 1; + ppdu_info->beamformed = u32_get_bits(info0, + HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_TXBF); +} + +static void +ath12k_wifi7_dp_mon_parse_he_sig_b2_mu(const struct hal_rx_he_sig_b2_mu_info *he_sig_b2_mu, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0, value; + + info0 = __le32_to_cpu(he_sig_b2_mu->info0); + + ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_CODING_KNOWN; + + ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_MCS); + value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_CODING); + ppdu_info->ldpc = value; + value = value << HE_CODING_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_ID); + value = value << HE_STA_ID_SHIFT; + ppdu_info->he_data4 |= value; + + ppdu_info->nss = + u32_get_bits(info0, + HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_NSTS) + 1; +} + +static void +ath12k_wifi7_dp_mon_parse_he_sig_b1_mu(const struct hal_rx_he_sig_b1_mu_info *he_sig_b1_mu, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0 = __le32_to_cpu(he_sig_b1_mu->info0); + u16 ru_tones; + + ru_tones = u32_get_bits(info0, + HAL_RX_HE_SIG_B1_MU_INFO_INFO0_RU_ALLOCATION); + ppdu_info->ru_alloc = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones); + ppdu_info->he_RU[0] = ru_tones; +} + +static void +ath12k_wifi7_dp_mon_parse_he_sig_mu(const struct hal_rx_he_sig_a_mu_dl_info *he_sig_a_mu_dl, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0, info1, value; + u16 he_gi = 0, he_ltf = 0; + + info0 = __le32_to_cpu(he_sig_a_mu_dl->info0); + info1 = __le32_to_cpu(he_sig_a_mu_dl->info1); + + ppdu_info->he_mu_flags = 1; + + ppdu_info->he_data1 = HE_MU_FORMAT_TYPE; + ppdu_info->he_data1 |= + HE_BSS_COLOR_KNOWN | + HE_DL_UL_KNOWN | + HE_LDPC_EXTRA_SYMBOL_KNOWN | + HE_STBC_KNOWN | + HE_DATA_BW_RU_KNOWN | + HE_DOPPLER_KNOWN; + + ppdu_info->he_data2 = + HE_GI_KNOWN | + HE_LTF_SYMBOLS_KNOWN | + HE_PRE_FEC_PADDING_KNOWN | + HE_PE_DISAMBIGUITY_KNOWN | + HE_TXOP_KNOWN | + HE_MIDABLE_PERIODICITY_KNOWN; + + /* data3 */ + ppdu_info->he_data3 = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_BSS_COLOR); + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_UL_FLAG); + value = value << HE_DL_UL_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_LDPC_EXTRA); + value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC); + value = value << HE_STBC_SHIFT; + ppdu_info->he_data3 |= value; + + /* data4 */ + ppdu_info->he_data4 = u32_get_bits(info0, + HAL_RX_HE_SIG_A_MU_DL_INFO0_SPATIAL_REUSE); + ppdu_info->he_data4 = value; + + /* data5 */ + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW); + ppdu_info->he_data5 = value; + ppdu_info->bw = value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_CP_LTF_SIZE); + switch (value) { + case 0: + he_gi = HE_GI_0_8; + he_ltf = HE_LTF_4_X; + break; + case 1: + he_gi = HE_GI_0_8; + he_ltf = HE_LTF_2_X; + break; + case 2: + he_gi = HE_GI_1_6; + he_ltf = HE_LTF_2_X; + break; + case 3: + he_gi = HE_GI_3_2; + he_ltf = HE_LTF_4_X; + break; + } + + ppdu_info->gi = he_gi; + value = he_gi << HE_GI_SHIFT; + ppdu_info->he_data5 |= value; + + value = he_ltf << HE_LTF_SIZE_SHIFT; + ppdu_info->he_data5 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_NUM_LTF_SYMB); + value = (value << HE_LTF_SYM_SHIFT); + ppdu_info->he_data5 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_FACTOR); + value = value << HE_PRE_FEC_PAD_SHIFT; + ppdu_info->he_data5 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_PE_DISAM); + value = value << HE_PE_DISAMBIGUITY_SHIFT; + ppdu_info->he_data5 |= value; + + /*data6*/ + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DOPPLER_INDICATION); + value = value << HE_DOPPLER_SHIFT; + ppdu_info->he_data6 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_TXOP_DURATION); + value = value << HE_TXOP_SHIFT; + ppdu_info->he_data6 |= value; + + /* HE-MU Flags */ + /* HE-MU-flags1 */ + ppdu_info->he_flags1 = + HE_SIG_B_MCS_KNOWN | + HE_SIG_B_DCM_KNOWN | + HE_SIG_B_COMPRESSION_FLAG_1_KNOWN | + HE_SIG_B_SYM_NUM_KNOWN | + HE_RU_0_KNOWN; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_MCS_OF_SIGB); + ppdu_info->he_flags1 |= value; + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DCM_OF_SIGB); + value = value << HE_DCM_FLAG_1_SHIFT; + ppdu_info->he_flags1 |= value; + + /* HE-MU-flags2 */ + ppdu_info->he_flags2 = HE_BW_KNOWN; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW); + ppdu_info->he_flags2 |= value; + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_COMP_MODE_SIGB); + value = value << HE_SIG_B_COMPRESSION_FLAG_2_SHIFT; + ppdu_info->he_flags2 |= value; + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_NUM_SIGB_SYMB); + value = value - 1; + value = value << HE_NUM_SIG_B_SYMBOLS_SHIFT; + ppdu_info->he_flags2 |= value; + + ppdu_info->is_stbc = info1 & + HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC; +} + +static void +ath12k_wifi7_dp_mon_parse_he_sig_su(const struct hal_rx_he_sig_a_su_info *he_sig_a, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + u32 info0, info1, value; + u32 dcm; + u8 he_dcm = 0, he_stbc = 0; + u16 he_gi = 0, he_ltf = 0; + + ppdu_info->he_flags = 1; + + info0 = __le32_to_cpu(he_sig_a->info0); + info1 = __le32_to_cpu(he_sig_a->info1); + + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_FORMAT_IND); + if (value == 0) + ppdu_info->he_data1 = HE_TRIG_FORMAT_TYPE; + else + ppdu_info->he_data1 = HE_SU_FORMAT_TYPE; + + ppdu_info->he_data1 |= + HE_BSS_COLOR_KNOWN | + HE_BEAM_CHANGE_KNOWN | + HE_DL_UL_KNOWN | + HE_MCS_KNOWN | + HE_DCM_KNOWN | + HE_CODING_KNOWN | + HE_LDPC_EXTRA_SYMBOL_KNOWN | + HE_STBC_KNOWN | + HE_DATA_BW_RU_KNOWN | + HE_DOPPLER_KNOWN; + + ppdu_info->he_data2 |= + HE_GI_KNOWN | + HE_TXBF_KNOWN | + HE_PE_DISAMBIGUITY_KNOWN | + HE_TXOP_KNOWN | + HE_LTF_SYMBOLS_KNOWN | + HE_PRE_FEC_PADDING_KNOWN | + HE_MIDABLE_PERIODICITY_KNOWN; + + ppdu_info->he_data3 = u32_get_bits(info0, + HAL_RX_HE_SIG_A_SU_INFO_INFO0_BSS_COLOR); + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_BEAM_CHANGE); + value = value << HE_BEAM_CHANGE_SHIFT; + ppdu_info->he_data3 |= value; + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DL_UL_FLAG); + value = value << HE_DL_UL_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS); + ppdu_info->mcs = value; + value = value << HE_TRANSMIT_MCS_SHIFT; + ppdu_info->he_data3 |= value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM); + he_dcm = value; + value = value << HE_DCM_SHIFT; + ppdu_info->he_data3 |= value; + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING); + value = value << HE_CODING_SHIFT; + ppdu_info->he_data3 |= value; + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_LDPC_EXTRA); + value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT; + ppdu_info->he_data3 |= value; + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC); + he_stbc = value; + value = value << HE_STBC_SHIFT; + ppdu_info->he_data3 |= value; + + /* data4 */ + ppdu_info->he_data4 = u32_get_bits(info0, + HAL_RX_HE_SIG_A_SU_INFO_INFO0_SPATIAL_REUSE); + + /* data5 */ + value = u32_get_bits(info0, + HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW); + ppdu_info->he_data5 = value; + ppdu_info->bw = value; + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_CP_LTF_SIZE); + switch (value) { + case 0: + he_gi = HE_GI_0_8; + he_ltf = HE_LTF_1_X; + break; + case 1: + he_gi = HE_GI_0_8; + he_ltf = HE_LTF_2_X; + break; + case 2: + he_gi = HE_GI_1_6; + he_ltf = HE_LTF_2_X; + break; + case 3: + if (he_dcm && he_stbc) { + he_gi = HE_GI_0_8; + he_ltf = HE_LTF_4_X; + } else { + he_gi = HE_GI_3_2; + he_ltf = HE_LTF_4_X; + } + break; + } + ppdu_info->gi = he_gi; + value = he_gi << HE_GI_SHIFT; + ppdu_info->he_data5 |= value; + value = he_ltf << HE_LTF_SIZE_SHIFT; + ppdu_info->ltf_size = he_ltf; + ppdu_info->he_data5 |= value; + + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS); + value = (value << HE_LTF_SYM_SHIFT); + ppdu_info->he_data5 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_FACTOR); + value = value << HE_PRE_FEC_PAD_SHIFT; + ppdu_info->he_data5 |= value; + + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF); + value = value << HE_TXBF_SHIFT; + ppdu_info->he_data5 |= value; + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_PE_DISAM); + value = value << HE_PE_DISAMBIGUITY_SHIFT; + ppdu_info->he_data5 |= value; + + /* data6 */ + value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS); + value++; + ppdu_info->he_data6 = value; + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_DOPPLER_IND); + value = value << HE_DOPPLER_SHIFT; + ppdu_info->he_data6 |= value; + value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXOP_DURATION); + value = value << HE_TXOP_SHIFT; + ppdu_info->he_data6 |= value; + + ppdu_info->mcs = + u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS); + ppdu_info->bw = + u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW); + ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING); + ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC); + ppdu_info->beamformed = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF); + dcm = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM); + ppdu_info->nss = u32_get_bits(info0, + HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS) + 1; + ppdu_info->dcm = dcm; +} + +static inline bool +ath12k_wifi7_dp_mon_hal_rx_is_non_ofdma(const struct hal_rx_u_sig_info *usig_info) +{ + u32 ppdu_type_comp_mode = usig_info->ppdu_type_comp_mode; + u32 ul_dl = usig_info->ul_dl; + + if ((ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO && ul_dl == 0) || + (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_OFDMA && ul_dl == 0) || + (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO && ul_dl == 1)) + return true; + + return false; +} + +static inline bool +ath12k_wifi7_dp_mon_hal_rx_is_ofdma(const struct hal_rx_u_sig_info *usig_info) +{ + if (usig_info->ppdu_type_comp_mode == 0 && usig_info->ul_dl == 0) + return true; + + return false; +} + +static inline bool +ath12k_wifi7_dp_mon_hal_rx_is_frame_type_ndp(const struct hal_rx_u_sig_info *usig_info) +{ + if (usig_info->ppdu_type_comp_mode == 1 && + usig_info->eht_sig_mcs == 0 && + usig_info->num_eht_sig_sym == 0) + return true; + + return false; +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ndp(const struct hal_eht_sig_ndp_cmn_eb *eht_sig_ndp, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; + u32 known, data; + + known = __le32_to_cpu(eht->known); + known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE | + IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF | + IEEE80211_RADIOTAP_EHT_KNOWN_NSS_S | + IEEE80211_RADIOTAP_EHT_KNOWN_BEAMFORMED_S | + IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_S | + IEEE80211_RADIOTAP_EHT_KNOWN_CRC1 | + IEEE80211_RADIOTAP_EHT_KNOWN_TAIL1; + eht->known = cpu_to_le32(known); + + data = __le32_to_cpu(eht->data[0]); + data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, + HAL_RX_EHT_SIG_NDP_CMN_INFO0_SPATIAL_REUSE, + IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE); + /* GI and LTF size are separately indicated in radiotap header + * and hence will be parsed from other TLV + */ + data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, + HAL_RX_EHT_SIG_NDP_CMN_INFO0_NUM_LTF_SYM, + IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF); + + data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, + HAL_RX_EHT_SIG_NDP_CMN_INFO0_CRC, + IEEE80211_RADIOTAP_EHT_DATA0_CRC1_O); + + data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, + HAL_RX_EHT_SIG_NDP_CMN_INFO0_DISREGARD, + IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_S); + eht->data[0] = cpu_to_le32(data); + + data = __le32_to_cpu(eht->data[7]); + data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, + HAL_RX_EHT_SIG_NDP_CMN_INFO0_NSS, + IEEE80211_RADIOTAP_EHT_DATA7_NSS_S); + + data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0, + HAL_RX_EHT_SIG_NDP_CMN_INFO0_BEAMFORMED, + IEEE80211_RADIOTAP_EHT_DATA7_BEAMFORMED_S); + eht->data[7] = cpu_to_le32(data); +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_usig_overflow(const struct hal_eht_sig_usig_overflow *ovflow, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; + u32 known, data; + + known = __le32_to_cpu(eht->known); + known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE | + IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF | + IEEE80211_RADIOTAP_EHT_KNOWN_LDPC_EXTRA_SYM_OM | + IEEE80211_RADIOTAP_EHT_KNOWN_PRE_PADD_FACOR_OM | + IEEE80211_RADIOTAP_EHT_KNOWN_PE_DISAMBIGUITY_OM | + IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_O; + eht->known = cpu_to_le32(known); + + data = __le32_to_cpu(eht->data[0]); + data |= ATH12K_LE32_DEC_ENC(ovflow->info0, + HAL_RX_EHT_SIG_OVERFLOW_INFO0_SPATIAL_REUSE, + IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE); + + /* GI and LTF size are separately indicated in radiotap header + * and hence will be parsed from other TLV + */ + data |= ATH12K_LE32_DEC_ENC(ovflow->info0, + HAL_RX_EHT_SIG_OVERFLOW_INFO0_NUM_LTF_SYM, + IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF); + + data |= ATH12K_LE32_DEC_ENC(ovflow->info0, + HAL_RX_EHT_SIG_OVERFLOW_INFO0_LDPC_EXTA_SYM, + IEEE80211_RADIOTAP_EHT_DATA0_LDPC_EXTRA_SYM_OM); + + data |= ATH12K_LE32_DEC_ENC(ovflow->info0, + HAL_RX_EHT_SIG_OVERFLOW_INFO0_PRE_FEC_PAD_FACTOR, + IEEE80211_RADIOTAP_EHT_DATA0_PRE_PADD_FACOR_OM); + + data |= ATH12K_LE32_DEC_ENC(ovflow->info0, + HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISAMBIGUITY, + IEEE80211_RADIOTAP_EHT_DATA0_PE_DISAMBIGUITY_OM); + + data |= ATH12K_LE32_DEC_ENC(ovflow->info0, + HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISREGARD, + IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_O); + eht->data[0] = cpu_to_le32(data); +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_non_ofdma_users(const struct hal_eht_sig_non_ofdma_cmn_eb *eb, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; + u32 known, data; + + known = __le32_to_cpu(eht->known); + known |= IEEE80211_RADIOTAP_EHT_KNOWN_NR_NON_OFDMA_USERS_M; + eht->known = cpu_to_le32(known); + + data = __le32_to_cpu(eht->data[7]); + data |= ATH12K_LE32_DEC_ENC(eb->info0, + HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_USERS, + IEEE80211_RADIOTAP_EHT_DATA7_NUM_OF_NON_OFDMA_USERS); + eht->data[7] = cpu_to_le32(data); +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_eht_mumimo_user(const struct hal_eht_sig_mu_mimo *user, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info; + u32 user_idx; + + if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info)) + return; + + user_idx = eht_info->num_user_info++; + + eht_info->user_info[user_idx] |= + IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN | + IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN | + IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN | + IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_KNOWN_M | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_STA_ID, + IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CODING, + IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS, + IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_SPATIAL_CODING, + IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_M); + + ppdu_info->mcs = le32_get_bits(user->info0, + HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS); +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_eht_non_mumimo_user(const struct hal_eht_sig_non_mu_mimo *user, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info; + u32 user_idx; + + if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info)) + return; + + user_idx = eht_info->num_user_info++; + + eht_info->user_info[user_idx] |= + IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN | + IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN | + IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN | + IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_KNOWN_O | + IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_KNOWN_O | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_STA_ID, + IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CODING, + IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS, + IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS, + IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_O) | + ATH12K_LE32_DEC_ENC(user->info0, + HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_BEAMFORMED, + IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_O); + + ppdu_info->mcs = le32_get_bits(user->info0, + HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS); + + ppdu_info->nss = le32_get_bits(user->info0, + HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS) + 1; +} + +static inline bool +ath12k_wifi7_dp_mon_hal_rx_is_mu_mimo_user(const struct hal_rx_u_sig_info *usig_info) +{ + if (usig_info->ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_SU && + usig_info->ul_dl == 1) + return true; + + return false; +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_non_ofdma(const void *tlv, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + const struct hal_eht_sig_non_ofdma_cmn_eb *eb = tlv; + + ath12k_wifi7_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info); + ath12k_wifi7_dp_mon_hal_rx_parse_non_ofdma_users(eb, ppdu_info); + + if (ath12k_wifi7_dp_mon_hal_rx_is_mu_mimo_user(&ppdu_info->u_sig_info)) + ath12k_wifi7_dp_mon_hal_rx_parse_eht_mumimo_user(&eb->user_field.mu_mimo, + ppdu_info); + else + ath12k_wifi7_dp_mon_hal_rx_parse_eht_non_mumimo_user(&eb->user_field.n_mu_mimo, + ppdu_info); +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_ru_allocation(const struct hal_eht_sig_ofdma_cmn_eb *eb, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + const struct hal_eht_sig_ofdma_cmn_eb1 *ofdma_cmn_eb1 = &eb->eb1; + const struct hal_eht_sig_ofdma_cmn_eb2 *ofdma_cmn_eb2 = &eb->eb2; + struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; + enum ieee80211_radiotap_eht_data ru_123, ru_124, ru_125, ru_126; + enum ieee80211_radiotap_eht_data ru_121, ru_122, ru_112, ru_111; + u32 data; + + ru_123 = IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3; + ru_124 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4; + ru_125 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5; + ru_126 = IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6; + ru_121 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1; + ru_122 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2; + ru_112 = IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2; + ru_111 = IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1; + + switch (ppdu_info->u_sig_info.bw) { + case HAL_EHT_BW_320_2: + case HAL_EHT_BW_320_1: + data = __le32_to_cpu(eht->data[4]); + /* CC1 2::3 */ + data |= IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, + HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_3, + ru_123); + eht->data[4] = cpu_to_le32(data); + + data = __le32_to_cpu(eht->data[5]); + /* CC1 2::4 */ + data |= IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, + HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_4, + ru_124); + + /* CC1 2::5 */ + data |= IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, + HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_5, + ru_125); + eht->data[5] = cpu_to_le32(data); + + data = __le32_to_cpu(eht->data[6]); + /* CC1 2::6 */ + data |= IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, + HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_6, + ru_126); + eht->data[6] = cpu_to_le32(data); + + fallthrough; + case HAL_EHT_BW_160: + data = __le32_to_cpu(eht->data[3]); + /* CC1 2::1 */ + data |= IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, + HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_1, + ru_121); + /* CC1 2::2 */ + data |= IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0, + HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_2, + ru_122); + eht->data[3] = cpu_to_le32(data); + + fallthrough; + case HAL_EHT_BW_80: + data = __le32_to_cpu(eht->data[2]); + /* CC1 1::2 */ + data |= IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0, + HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_2, + ru_112); + eht->data[2] = cpu_to_le32(data); + + fallthrough; + case HAL_EHT_BW_40: + fallthrough; + case HAL_EHT_BW_20: + data = __le32_to_cpu(eht->data[1]); + /* CC1 1::1 */ + data |= IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1_KNOWN | + ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0, + HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_1, + ru_111); + eht->data[1] = cpu_to_le32(data); + break; + default: + break; + } +} + +static void +ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ofdma(const void *tlv, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + const struct hal_eht_sig_ofdma_cmn_eb *ofdma = tlv; + + ath12k_wifi7_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info); + ath12k_wifi7_dp_mon_hal_rx_parse_ru_allocation(ofdma, ppdu_info); + + ath12k_wifi7_dp_mon_hal_rx_parse_eht_non_mumimo_user(&ofdma->user_field.n_mu_mimo, + ppdu_info); +} + +static void +ath12k_wifi7_dp_mon_parse_eht_sig_hdr(struct hal_rx_mon_ppdu_info *ppdu_info, + const void *tlv_data) +{ + ppdu_info->is_eht = true; + + if (ath12k_wifi7_dp_mon_hal_rx_is_frame_type_ndp(&ppdu_info->u_sig_info)) + ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ndp(tlv_data, ppdu_info); + else if (ath12k_wifi7_dp_mon_hal_rx_is_non_ofdma(&ppdu_info->u_sig_info)) + ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_non_ofdma(tlv_data, ppdu_info); + else if (ath12k_wifi7_dp_mon_hal_rx_is_ofdma(&ppdu_info->u_sig_info)) + ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ofdma(tlv_data, ppdu_info); +} + +static void ath12k_wifi7_dp_mon_parse_rx_msdu_end_err(u32 info, u32 *errmap) +{ + if (info & RX_MSDU_END_INFO13_FCS_ERR) + *errmap |= HAL_RX_MPDU_ERR_FCS; + + if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) + *errmap |= HAL_RX_MPDU_ERR_DECRYPT; + + if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) + *errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; + + if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) + *errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; + + if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) + *errmap |= HAL_RX_MPDU_ERR_OVERFLOW; + + if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) + *errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; + + if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) + *errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; +} + +static void +ath12k_wifi7_parse_cmn_usr_info(const struct hal_phyrx_common_user_info *cmn_usr_info, + struct hal_rx_mon_ppdu_info *ppdu_info) +{ + struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht; + u32 known, data, cp_setting, ltf_size; + + known = __le32_to_cpu(eht->known); + known |= IEEE80211_RADIOTAP_EHT_KNOWN_GI | + IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF; + eht->known = cpu_to_le32(known); + + cp_setting = le32_get_bits(cmn_usr_info->info0, + HAL_RX_CMN_USR_INFO0_CP_SETTING); + ltf_size = le32_get_bits(cmn_usr_info->info0, + HAL_RX_CMN_USR_INFO0_LTF_SIZE); + + data = __le32_to_cpu(eht->data[0]); + data |= u32_encode_bits(cp_setting, IEEE80211_RADIOTAP_EHT_DATA0_GI); + data |= u32_encode_bits(ltf_size, IEEE80211_RADIOTAP_EHT_DATA0_LTF); + eht->data[0] = cpu_to_le32(data); + + if (!ppdu_info->ltf_size) + ppdu_info->ltf_size = ltf_size; + if (!ppdu_info->gi) + ppdu_info->gi = cp_setting; +} + +static void +ath12k_wifi7_dp_mon_parse_status_msdu_end(struct ath12k_mon_data *pmon, + const struct hal_rx_msdu_end *msdu_end) +{ + ath12k_wifi7_dp_mon_parse_rx_msdu_end_err(__le32_to_cpu(msdu_end->info2), + &pmon->err_bitmap); + pmon->decap_format = le32_get_bits(msdu_end->info1, + RX_MSDU_END_INFO11_DECAP_FORMAT); +} + +static enum hal_rx_mon_status +ath12k_wifi7_dp_mon_rx_parse_status_tlv(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_mon_data *pmon, + const void *tlv) +{ + struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + struct ath12k_hal *hal = &ar->ab->hal; + u16 tlv_tag, tlv_len, userid; + void *tlv_data; + u32 info[7]; + + tlv_data = hal->ops->mon_rx_status_dec_tlv_hdr((void *)tlv, &tlv_tag, + &tlv_len, &userid); + + if (ppdu_info->tlv_aggr.in_progress && ppdu_info->tlv_aggr.tlv_tag != tlv_tag) { + ath12k_wifi7_dp_mon_parse_eht_sig_hdr(ppdu_info, + ppdu_info->tlv_aggr.buf); + + ppdu_info->tlv_aggr.in_progress = false; + ppdu_info->tlv_aggr.cur_len = 0; + } + + switch (tlv_tag) { + case HAL_RX_PPDU_START: { + const struct hal_rx_ppdu_start *ppdu_start = tlv_data; + + u64 ppdu_ts = ath12k_le32hilo_to_u64(ppdu_start->ppdu_start_ts_63_32, + ppdu_start->ppdu_start_ts_31_0); + + info[0] = __le32_to_cpu(ppdu_start->info0); + + ppdu_info->ppdu_id = u32_get_bits(info[0], + HAL_RX_PPDU_START_INFO0_PPDU_ID); + + info[1] = __le32_to_cpu(ppdu_start->info1); + ppdu_info->chan_num = u32_get_bits(info[1], + HAL_RX_PPDU_START_INFO1_CHAN_NUM); + ppdu_info->freq = u32_get_bits(info[1], + HAL_RX_PPDU_START_INFO1_CHAN_FREQ); + ppdu_info->ppdu_ts = ppdu_ts; + + if (ppdu_info->ppdu_id != ppdu_info->last_ppdu_id) { + ppdu_info->last_ppdu_id = ppdu_info->ppdu_id; + ppdu_info->num_users = 0; + memset(&ppdu_info->mpdu_fcs_ok_bitmap, 0, + HAL_RX_NUM_WORDS_PER_PPDU_BITMAP * + sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0])); + } + break; + } + case HAL_RX_PPDU_END_USER_STATS: { + const struct hal_rx_ppdu_end_user_stats *eu_stats = tlv_data; + u32 tid_bitmap; + + info[0] = __le32_to_cpu(eu_stats->info0); + info[1] = __le32_to_cpu(eu_stats->info1); + info[2] = __le32_to_cpu(eu_stats->info2); + info[4] = __le32_to_cpu(eu_stats->info4); + info[5] = __le32_to_cpu(eu_stats->info5); + info[6] = __le32_to_cpu(eu_stats->info6); + + ppdu_info->ast_index = + u32_get_bits(info[2], HAL_RX_PPDU_END_USER_STATS_INFO2_AST_INDEX); + ppdu_info->fc_valid = + u32_get_bits(info[1], HAL_RX_PPDU_END_USER_STATS_INFO1_FC_VALID); + tid_bitmap = u32_get_bits(info[6], + HAL_RX_PPDU_END_USER_STATS_INFO6_TID_BITMAP); + ppdu_info->tid = ffs(tid_bitmap) - 1; + ppdu_info->tcp_msdu_count = + u32_get_bits(info[4], + HAL_RX_PPDU_END_USER_STATS_INFO4_TCP_MSDU_CNT); + ppdu_info->udp_msdu_count = + u32_get_bits(info[4], + HAL_RX_PPDU_END_USER_STATS_INFO4_UDP_MSDU_CNT); + ppdu_info->other_msdu_count = + u32_get_bits(info[5], + HAL_RX_PPDU_END_USER_STATS_INFO5_OTHER_MSDU_CNT); + ppdu_info->tcp_ack_msdu_count = + u32_get_bits(info[5], + HAL_RX_PPDU_END_USER_STATS_INFO5_TCP_ACK_MSDU_CNT); + ppdu_info->preamble_type = + u32_get_bits(info[1], + HAL_RX_PPDU_END_USER_STATS_INFO1_PKT_TYPE); + ppdu_info->num_mpdu_fcs_ok = + u32_get_bits(info[1], + HAL_RX_PPDU_END_USER_STATS_INFO1_MPDU_CNT_FCS_OK); + ppdu_info->num_mpdu_fcs_err = + u32_get_bits(info[0], + HAL_RX_PPDU_END_USER_STATS_INFO0_MPDU_CNT_FCS_ERR); + ppdu_info->peer_id = + u32_get_bits(info[0], HAL_RX_PPDU_END_USER_STATS_INFO0_PEER_ID); + + switch (ppdu_info->preamble_type) { + case HAL_RX_PREAMBLE_11N: + ppdu_info->ht_flags = 1; + break; + case HAL_RX_PREAMBLE_11AC: + ppdu_info->vht_flags = 1; + break; + case HAL_RX_PREAMBLE_11AX: + ppdu_info->he_flags = 1; + break; + case HAL_RX_PREAMBLE_11BE: + ppdu_info->is_eht = true; + break; + default: + break; + } + + if (userid < HAL_MAX_UL_MU_USERS) { + struct hal_rx_user_status *rxuser_stats = + &ppdu_info->userstats[userid]; + + if (ppdu_info->num_mpdu_fcs_ok > 1 || + ppdu_info->num_mpdu_fcs_err > 1) + ppdu_info->userstats[userid].ampdu_present = true; + + ppdu_info->num_users += 1; + + ath12k_wifi7_dp_mon_rx_handle_ofdma_info(eu_stats, rxuser_stats); + ath12k_wifi7_dp_mon_rx_populate_mu_user_info(eu_stats, ppdu_info, + rxuser_stats); + } + ppdu_info->mpdu_fcs_ok_bitmap[0] = __le32_to_cpu(eu_stats->rsvd1[0]); + ppdu_info->mpdu_fcs_ok_bitmap[1] = __le32_to_cpu(eu_stats->rsvd1[1]); + break; + } + case HAL_RX_PPDU_END_USER_STATS_EXT: { + const struct hal_rx_ppdu_end_user_stats_ext *eu_stats = tlv_data; + + ppdu_info->mpdu_fcs_ok_bitmap[2] = __le32_to_cpu(eu_stats->info1); + ppdu_info->mpdu_fcs_ok_bitmap[3] = __le32_to_cpu(eu_stats->info2); + ppdu_info->mpdu_fcs_ok_bitmap[4] = __le32_to_cpu(eu_stats->info3); + ppdu_info->mpdu_fcs_ok_bitmap[5] = __le32_to_cpu(eu_stats->info4); + ppdu_info->mpdu_fcs_ok_bitmap[6] = __le32_to_cpu(eu_stats->info5); + ppdu_info->mpdu_fcs_ok_bitmap[7] = __le32_to_cpu(eu_stats->info6); + break; + } + case HAL_PHYRX_HT_SIG: + ath12k_wifi7_dp_mon_parse_ht_sig(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_L_SIG_B: + ath12k_wifi7_dp_mon_parse_l_sig_b(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_L_SIG_A: + ath12k_wifi7_dp_mon_parse_l_sig_a(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_VHT_SIG_A: + ath12k_wifi7_dp_mon_parse_vht_sig_a(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_HE_SIG_A_SU: + ath12k_wifi7_dp_mon_parse_he_sig_su(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_HE_SIG_A_MU_DL: + ath12k_wifi7_dp_mon_parse_he_sig_mu(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_HE_SIG_B1_MU: + ath12k_wifi7_dp_mon_parse_he_sig_b1_mu(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_HE_SIG_B2_MU: + ath12k_wifi7_dp_mon_parse_he_sig_b2_mu(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_HE_SIG_B2_OFDMA: + ath12k_wifi7_dp_mon_parse_he_sig_b2_ofdma(tlv_data, ppdu_info); + break; + + case HAL_PHYRX_RSSI_LEGACY: { + const struct hal_rx_phyrx_rssi_legacy_info *rssi = tlv_data; + + info[0] = __le32_to_cpu(rssi->info0); + info[2] = __le32_to_cpu(rssi->info2); + + /* TODO: Please note that the combined rssi will not be accurate + * in MU case. Rssi in MU needs to be retrieved from + * PHYRX_OTHER_RECEIVE_INFO TLV. + */ + ppdu_info->rssi_comb = + u32_get_bits(info[2], + HAL_RX_RSSI_LEGACY_INFO_INFO2_RSSI_COMB_PPDU); + + ppdu_info->bw = u32_get_bits(info[0], + HAL_RX_RSSI_LEGACY_INFO_INFO0_RX_BW); + break; + } + case HAL_PHYRX_COMMON_USER_INFO: { + ath12k_wifi7_parse_cmn_usr_info(tlv_data, ppdu_info); + break; + } + case HAL_RX_PPDU_START_USER_INFO: + ath12k_wifi7_dp_mon_hal_rx_parse_user_info(tlv_data, userid, ppdu_info); + break; + + case HAL_RXPCU_PPDU_END_INFO: { + const struct hal_rx_ppdu_end_duration *ppdu_rx_duration = tlv_data; + + info[0] = __le32_to_cpu(ppdu_rx_duration->info0); + ppdu_info->rx_duration = + u32_get_bits(info[0], HAL_RX_PPDU_END_DURATION); + ppdu_info->tsft = __le32_to_cpu(ppdu_rx_duration->rsvd0[1]); + ppdu_info->tsft = (ppdu_info->tsft << 32) | + __le32_to_cpu(ppdu_rx_duration->rsvd0[0]); + break; + } + case HAL_RX_MPDU_START: { + const struct hal_rx_mpdu_start *mpdu_start = tlv_data; + u16 peer_id; + + info[1] = __le32_to_cpu(mpdu_start->info1); + peer_id = u32_get_bits(info[1], HAL_RX_MPDU_START_INFO1_PEERID); + ppdu_info->peer_id = peer_id; + + ppdu_info->mpdu_len += u32_get_bits(info[1], + HAL_RX_MPDU_START_INFO2_MPDU_LEN); + if (userid < HAL_MAX_UL_MU_USERS) { + info[0] = __le32_to_cpu(mpdu_start->info0); + ppdu_info->userid = userid; + ppdu_info->userstats[userid].ampdu_id = + u32_get_bits(info[0], HAL_RX_MPDU_START_INFO0_PPDU_ID); + } + + return HAL_RX_MON_STATUS_MPDU_START; + } + case HAL_RX_MSDU_START: + /* TODO: add msdu start parsing logic */ + break; + case HAL_MON_BUF_ADDR: + return HAL_RX_MON_STATUS_BUF_ADDR; + case HAL_RX_MSDU_END: + ath12k_wifi7_dp_mon_parse_status_msdu_end(pmon, tlv_data); + return HAL_RX_MON_STATUS_MSDU_END; + case HAL_RX_MPDU_END: + return HAL_RX_MON_STATUS_MPDU_END; + case HAL_PHYRX_GENERIC_U_SIG: + ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_hdr(tlv_data, ppdu_info); + break; + case HAL_PHYRX_GENERIC_EHT_SIG: + /* Handle the case where aggregation is in progress + * or the current TLV is one of the TLVs which should be + * aggregated + */ + if (!ppdu_info->tlv_aggr.in_progress) { + ppdu_info->tlv_aggr.in_progress = true; + ppdu_info->tlv_aggr.tlv_tag = tlv_tag; + ppdu_info->tlv_aggr.cur_len = 0; + } + + ppdu_info->is_eht = true; + + ath12k_wifi7_dp_mon_hal_aggr_tlv(ppdu_info, tlv_len, tlv_data); + break; + case HAL_DUMMY: + return HAL_RX_MON_STATUS_BUF_DONE; + case HAL_RX_PPDU_END_STATUS_DONE: + case 0: + return HAL_RX_MON_STATUS_PPDU_DONE; + default: + break; + } + + return HAL_RX_MON_STATUS_PPDU_NOT_DONE; +} + +static int +ath12k_wifi7_dp_mon_parse_rx_dest_tlv(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_mon_data *pmon, + enum hal_rx_mon_status hal_status, + const void *tlv_data) +{ + switch (hal_status) { + case HAL_RX_MON_STATUS_MPDU_START: + if (WARN_ON_ONCE(pmon->mon_mpdu)) + break; + + pmon->mon_mpdu = kzalloc(sizeof(*pmon->mon_mpdu), GFP_ATOMIC); + if (!pmon->mon_mpdu) + return -ENOMEM; + break; + case HAL_RX_MON_STATUS_BUF_ADDR: + return ath12k_dp_mon_parse_status_buf(dp_pdev, pmon, tlv_data); + case HAL_RX_MON_STATUS_MPDU_END: + /* If no MSDU then free empty MPDU */ + if (pmon->mon_mpdu->tail) { + pmon->mon_mpdu->tail->next = NULL; + list_add_tail(&pmon->mon_mpdu->list, &pmon->dp_rx_mon_mpdu_list); + } else { + kfree(pmon->mon_mpdu); + } + pmon->mon_mpdu = NULL; + break; + case HAL_RX_MON_STATUS_MSDU_END: + pmon->mon_mpdu->decap_format = pmon->decap_format; + pmon->mon_mpdu->err_bitmap = pmon->err_bitmap; + break; + default: + break; + } + + return 0; +} + +static struct dp_mon_tx_ppdu_info * +ath12k_wifi7_dp_mon_tx_get_ppdu_info(struct ath12k_mon_data *pmon, + unsigned int ppdu_id, + enum dp_mon_tx_ppdu_info_type type) +{ + struct dp_mon_tx_ppdu_info *tx_ppdu_info; + + if (type == DP_MON_TX_PROT_PPDU_INFO) { + tx_ppdu_info = pmon->tx_prot_ppdu_info; + + if (tx_ppdu_info && !tx_ppdu_info->is_used) + return tx_ppdu_info; + kfree(tx_ppdu_info); + } else { + tx_ppdu_info = pmon->tx_data_ppdu_info; + + if (tx_ppdu_info && !tx_ppdu_info->is_used) + return tx_ppdu_info; + kfree(tx_ppdu_info); + } + + /* allocate new tx_ppdu_info */ + tx_ppdu_info = kzalloc(sizeof(*tx_ppdu_info), GFP_ATOMIC); + if (!tx_ppdu_info) + return NULL; + + tx_ppdu_info->is_used = 0; + tx_ppdu_info->ppdu_id = ppdu_id; + + if (type == DP_MON_TX_PROT_PPDU_INFO) + pmon->tx_prot_ppdu_info = tx_ppdu_info; + else + pmon->tx_data_ppdu_info = tx_ppdu_info; + + return tx_ppdu_info; +} + +static struct dp_mon_tx_ppdu_info * +ath12k_wifi7_dp_mon_hal_tx_ppdu_info(struct ath12k_mon_data *pmon, + u16 tlv_tag) +{ + switch (tlv_tag) { + case HAL_TX_FES_SETUP: + case HAL_TX_FLUSH: + case HAL_PCU_PPDU_SETUP_INIT: + case HAL_TX_PEER_ENTRY: + case HAL_TX_QUEUE_EXTENSION: + case HAL_TX_MPDU_START: + case HAL_TX_MSDU_START: + case HAL_TX_DATA: + case HAL_MON_BUF_ADDR: + case HAL_TX_MPDU_END: + case HAL_TX_LAST_MPDU_FETCHED: + case HAL_TX_LAST_MPDU_END: + case HAL_COEX_TX_REQ: + case HAL_TX_RAW_OR_NATIVE_FRAME_SETUP: + case HAL_SCH_CRITICAL_TLV_REFERENCE: + case HAL_TX_FES_SETUP_COMPLETE: + case HAL_TQM_MPDU_GLOBAL_START: + case HAL_SCHEDULER_END: + case HAL_TX_FES_STATUS_USER_PPDU: + break; + case HAL_TX_FES_STATUS_PROT: { + if (!pmon->tx_prot_ppdu_info->is_used) + pmon->tx_prot_ppdu_info->is_used = true; + + return pmon->tx_prot_ppdu_info; + } + } + + if (!pmon->tx_data_ppdu_info->is_used) + pmon->tx_data_ppdu_info->is_used = true; + + return pmon->tx_data_ppdu_info; +} + +#define MAX_MONITOR_HEADER 512 +#define MAX_DUMMY_FRM_BODY 128 + +static struct +sk_buff *ath12k_wifi7_dp_mon_tx_alloc_skb(void) +{ + struct sk_buff *skb; + + skb = dev_alloc_skb(MAX_MONITOR_HEADER + MAX_DUMMY_FRM_BODY); + if (!skb) + return NULL; + + skb_reserve(skb, MAX_MONITOR_HEADER); + + if (!IS_ALIGNED((unsigned long)skb->data, 4)) + skb_pull(skb, PTR_ALIGN(skb->data, 4) - skb->data); + + return skb; +} + +static int +ath12k_wifi7_dp_mon_tx_gen_cts2self_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) +{ + struct sk_buff *skb; + struct ieee80211_cts *cts; + + skb = ath12k_wifi7_dp_mon_tx_alloc_skb(); + if (!skb) + return -ENOMEM; + + cts = (struct ieee80211_cts *)skb->data; + memset(cts, 0, MAX_DUMMY_FRM_BODY); + cts->frame_control = + cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTS); + cts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); + memcpy(cts->ra, tx_ppdu_info->rx_status.addr1, sizeof(cts->ra)); + + skb_put(skb, sizeof(*cts)); + tx_ppdu_info->tx_mon_mpdu->head = skb; + tx_ppdu_info->tx_mon_mpdu->tail = NULL; + list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, + &tx_ppdu_info->dp_tx_mon_mpdu_list); + + return 0; +} + +static int +ath12k_wifi7_dp_mon_tx_gen_rts_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) +{ + struct sk_buff *skb; + struct ieee80211_rts *rts; + + skb = ath12k_wifi7_dp_mon_tx_alloc_skb(); + if (!skb) + return -ENOMEM; + + rts = (struct ieee80211_rts *)skb->data; + memset(rts, 0, MAX_DUMMY_FRM_BODY); + rts->frame_control = + cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_RTS); + rts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); + memcpy(rts->ra, tx_ppdu_info->rx_status.addr1, sizeof(rts->ra)); + memcpy(rts->ta, tx_ppdu_info->rx_status.addr2, sizeof(rts->ta)); + + skb_put(skb, sizeof(*rts)); + tx_ppdu_info->tx_mon_mpdu->head = skb; + tx_ppdu_info->tx_mon_mpdu->tail = NULL; + list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, + &tx_ppdu_info->dp_tx_mon_mpdu_list); + + return 0; +} + +static int +ath12k_wifi7_dp_mon_tx_gen_3addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) +{ + struct sk_buff *skb; + struct ieee80211_qos_hdr *qhdr; + + skb = ath12k_wifi7_dp_mon_tx_alloc_skb(); + if (!skb) + return -ENOMEM; + + qhdr = (struct ieee80211_qos_hdr *)skb->data; + memset(qhdr, 0, MAX_DUMMY_FRM_BODY); + qhdr->frame_control = + cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC); + qhdr->duration_id = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); + memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN); + memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN); + memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN); + + skb_put(skb, sizeof(*qhdr)); + tx_ppdu_info->tx_mon_mpdu->head = skb; + tx_ppdu_info->tx_mon_mpdu->tail = NULL; + list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, + &tx_ppdu_info->dp_tx_mon_mpdu_list); + + return 0; +} + +static int +ath12k_wifi7_dp_mon_tx_gen_4addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) +{ + struct sk_buff *skb; + struct dp_mon_qosframe_addr4 *qhdr; + + skb = ath12k_wifi7_dp_mon_tx_alloc_skb(); + if (!skb) + return -ENOMEM; + + qhdr = (struct dp_mon_qosframe_addr4 *)skb->data; + memset(qhdr, 0, MAX_DUMMY_FRM_BODY); + qhdr->frame_control = + cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC); + qhdr->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration); + memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN); + memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN); + memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN); + memcpy(qhdr->addr4, tx_ppdu_info->rx_status.addr4, ETH_ALEN); + + skb_put(skb, sizeof(*qhdr)); + tx_ppdu_info->tx_mon_mpdu->head = skb; + tx_ppdu_info->tx_mon_mpdu->tail = NULL; + list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, + &tx_ppdu_info->dp_tx_mon_mpdu_list); + + return 0; +} + +static int +ath12k_wifi7_dp_mon_tx_gen_ack_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) +{ + struct sk_buff *skb; + struct dp_mon_frame_min_one *fbmhdr; + + skb = ath12k_wifi7_dp_mon_tx_alloc_skb(); + if (!skb) + return -ENOMEM; + + fbmhdr = (struct dp_mon_frame_min_one *)skb->data; + memset(fbmhdr, 0, MAX_DUMMY_FRM_BODY); + fbmhdr->frame_control = + cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_CFACK); + memcpy(fbmhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN); + + /* set duration zero for ack frame */ + fbmhdr->duration = 0; + + skb_put(skb, sizeof(*fbmhdr)); + tx_ppdu_info->tx_mon_mpdu->head = skb; + tx_ppdu_info->tx_mon_mpdu->tail = NULL; + list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, + &tx_ppdu_info->dp_tx_mon_mpdu_list); + + return 0; +} + +static int +ath12k_wifi7_dp_mon_tx_gen_prot_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info) +{ + int ret = 0; + + switch (tx_ppdu_info->rx_status.medium_prot_type) { + case DP_MON_TX_MEDIUM_RTS_LEGACY: + case DP_MON_TX_MEDIUM_RTS_11AC_STATIC_BW: + case DP_MON_TX_MEDIUM_RTS_11AC_DYNAMIC_BW: + ret = ath12k_wifi7_dp_mon_tx_gen_rts_frame(tx_ppdu_info); + break; + case DP_MON_TX_MEDIUM_CTS2SELF: + ret = ath12k_wifi7_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info); + break; + case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_3ADDR: + ret = ath12k_wifi7_dp_mon_tx_gen_3addr_qos_null_frame(tx_ppdu_info); + break; + case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_4ADDR: + ret = ath12k_wifi7_dp_mon_tx_gen_4addr_qos_null_frame(tx_ppdu_info); + break; + } + + return ret; +} + +static enum dp_mon_tx_tlv_status +ath12k_wifi7_dp_mon_tx_parse_status_tlv(struct ath12k_base *ab, + struct ath12k_mon_data *pmon, + u16 tlv_tag, const void *tlv_data, + u32 userid) +{ + struct dp_mon_tx_ppdu_info *tx_ppdu_info; + enum dp_mon_tx_tlv_status status = DP_MON_TX_STATUS_PPDU_NOT_DONE; + u32 info[7]; + + tx_ppdu_info = ath12k_wifi7_dp_mon_hal_tx_ppdu_info(pmon, tlv_tag); + + switch (tlv_tag) { + case HAL_TX_FES_SETUP: { + const struct hal_tx_fes_setup *tx_fes_setup = tlv_data; + + info[0] = __le32_to_cpu(tx_fes_setup->info0); + tx_ppdu_info->ppdu_id = __le32_to_cpu(tx_fes_setup->schedule_id); + tx_ppdu_info->num_users = + u32_get_bits(info[0], HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS); + status = DP_MON_TX_FES_SETUP; + break; + } + + case HAL_TX_FES_STATUS_END: { + const struct hal_tx_fes_status_end *tx_fes_status_end = tlv_data; + u32 tst_15_0, tst_31_16; + + info[0] = __le32_to_cpu(tx_fes_status_end->info0); + tst_15_0 = + u32_get_bits(info[0], + HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_15_0); + tst_31_16 = + u32_get_bits(info[0], + HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_31_16); + + tx_ppdu_info->rx_status.ppdu_ts = (tst_15_0 | (tst_31_16 << 16)); + status = DP_MON_TX_FES_STATUS_END; + break; + } + + case HAL_RX_RESPONSE_REQUIRED_INFO: { + const struct hal_rx_resp_req_info *rx_resp_req_info = tlv_data; + u32 addr_32; + u16 addr_16; + + info[0] = __le32_to_cpu(rx_resp_req_info->info0); + info[1] = __le32_to_cpu(rx_resp_req_info->info1); + info[2] = __le32_to_cpu(rx_resp_req_info->info2); + info[3] = __le32_to_cpu(rx_resp_req_info->info3); + info[4] = __le32_to_cpu(rx_resp_req_info->info4); + info[5] = __le32_to_cpu(rx_resp_req_info->info5); + + tx_ppdu_info->rx_status.ppdu_id = + u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_PPDU_ID); + tx_ppdu_info->rx_status.reception_type = + u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_RECEPTION_TYPE); + tx_ppdu_info->rx_status.rx_duration = + u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_DURATION); + tx_ppdu_info->rx_status.mcs = + u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_RATE_MCS); + tx_ppdu_info->rx_status.sgi = + u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_SGI); + tx_ppdu_info->rx_status.is_stbc = + u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_STBC); + tx_ppdu_info->rx_status.ldpc = + u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_LDPC); + tx_ppdu_info->rx_status.is_ampdu = + u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_IS_AMPDU); + tx_ppdu_info->rx_status.num_users = + u32_get_bits(info[2], HAL_RX_RESP_REQ_INFO2_NUM_USER); + + addr_32 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO3_ADDR1_31_0); + addr_16 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO4_ADDR1_47_32); + ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1); + + addr_16 = u32_get_bits(info[4], HAL_RX_RESP_REQ_INFO4_ADDR1_15_0); + addr_32 = u32_get_bits(info[5], HAL_RX_RESP_REQ_INFO5_ADDR1_47_16); + ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2); + + if (tx_ppdu_info->rx_status.reception_type == 0) + ath12k_wifi7_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info); + status = DP_MON_RX_RESPONSE_REQUIRED_INFO; + break; + } + + case HAL_PCU_PPDU_SETUP_INIT: { + const struct hal_tx_pcu_ppdu_setup_init *ppdu_setup = tlv_data; + u32 addr_32; + u16 addr_16; + + info[0] = __le32_to_cpu(ppdu_setup->info0); + info[1] = __le32_to_cpu(ppdu_setup->info1); + info[2] = __le32_to_cpu(ppdu_setup->info2); + info[3] = __le32_to_cpu(ppdu_setup->info3); + info[4] = __le32_to_cpu(ppdu_setup->info4); + info[5] = __le32_to_cpu(ppdu_setup->info5); + info[6] = __le32_to_cpu(ppdu_setup->info6); + + /* protection frame address 1 */ + addr_32 = u32_get_bits(info[1], + HAL_TX_PPDU_SETUP_INFO1_PROT_FRAME_ADDR1_31_0); + addr_16 = u32_get_bits(info[2], + HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR1_47_32); + ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1); + + /* protection frame address 2 */ + addr_16 = u32_get_bits(info[2], + HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR2_15_0); + addr_32 = u32_get_bits(info[3], + HAL_TX_PPDU_SETUP_INFO3_PROT_FRAME_ADDR2_47_16); + ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2); + + /* protection frame address 3 */ + addr_32 = u32_get_bits(info[4], + HAL_TX_PPDU_SETUP_INFO4_PROT_FRAME_ADDR3_31_0); + addr_16 = u32_get_bits(info[5], + HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR3_47_32); + ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr3); + + /* protection frame address 4 */ + addr_16 = u32_get_bits(info[5], + HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR4_15_0); + addr_32 = u32_get_bits(info[6], + HAL_TX_PPDU_SETUP_INFO6_PROT_FRAME_ADDR4_47_16); + ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr4); + + status = u32_get_bits(info[0], + HAL_TX_PPDU_SETUP_INFO0_MEDIUM_PROT_TYPE); + break; + } + + case HAL_TX_QUEUE_EXTENSION: { + const struct hal_tx_queue_exten *tx_q_exten = tlv_data; + + info[0] = __le32_to_cpu(tx_q_exten->info0); + + tx_ppdu_info->rx_status.frame_control = + u32_get_bits(info[0], + HAL_TX_Q_EXT_INFO0_FRAME_CTRL); + tx_ppdu_info->rx_status.fc_valid = true; + break; + } + + case HAL_TX_FES_STATUS_START: { + const struct hal_tx_fes_status_start *tx_fes_start = tlv_data; + + info[0] = __le32_to_cpu(tx_fes_start->info0); + + tx_ppdu_info->rx_status.medium_prot_type = + u32_get_bits(info[0], + HAL_TX_FES_STATUS_START_INFO0_MEDIUM_PROT_TYPE); + break; + } + + case HAL_TX_FES_STATUS_PROT: { + const struct hal_tx_fes_status_prot *tx_fes_status = tlv_data; + u32 start_timestamp; + u32 end_timestamp; + + info[0] = __le32_to_cpu(tx_fes_status->info0); + info[1] = __le32_to_cpu(tx_fes_status->info1); + + start_timestamp = + u32_get_bits(info[0], + HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_15_0); + start_timestamp |= + u32_get_bits(info[0], + HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_31_16) << 15; + end_timestamp = + u32_get_bits(info[1], + HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_15_0); + end_timestamp |= + u32_get_bits(info[1], + HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_31_16) << 15; + tx_ppdu_info->rx_status.rx_duration = end_timestamp - start_timestamp; + + ath12k_wifi7_dp_mon_tx_gen_prot_frame(tx_ppdu_info); + break; + } + + case HAL_TX_FES_STATUS_START_PPDU: + case HAL_TX_FES_STATUS_START_PROT: { + const struct hal_tx_fes_status_start_prot *tx_fes_stat_start = tlv_data; + u64 ppdu_ts; + + info[0] = __le32_to_cpu(tx_fes_stat_start->info0); + + tx_ppdu_info->rx_status.ppdu_ts = + u32_get_bits(info[0], + HAL_TX_FES_STAT_STRT_INFO0_PROT_TS_LOWER_32); + ppdu_ts = (u32_get_bits(info[1], + HAL_TX_FES_STAT_STRT_INFO1_PROT_TS_UPPER_32)); + tx_ppdu_info->rx_status.ppdu_ts |= ppdu_ts << 32; + break; + } + + case HAL_TX_FES_STATUS_USER_PPDU: { + const struct hal_tx_fes_status_user_ppdu *tx_fes_usr_ppdu = tlv_data; + + info[0] = __le32_to_cpu(tx_fes_usr_ppdu->info0); + + tx_ppdu_info->rx_status.rx_duration = + u32_get_bits(info[0], + HAL_TX_FES_STAT_USR_PPDU_INFO0_DURATION); + break; + } + + case HAL_MACTX_HE_SIG_A_SU: + ath12k_wifi7_dp_mon_parse_he_sig_su(tlv_data, + &tx_ppdu_info->rx_status); + break; + + case HAL_MACTX_HE_SIG_A_MU_DL: + ath12k_wifi7_dp_mon_parse_he_sig_mu(tlv_data, &tx_ppdu_info->rx_status); + break; + + case HAL_MACTX_HE_SIG_B1_MU: + ath12k_wifi7_dp_mon_parse_he_sig_b1_mu(tlv_data, + &tx_ppdu_info->rx_status); + break; + + case HAL_MACTX_HE_SIG_B2_MU: + ath12k_wifi7_dp_mon_parse_he_sig_b2_mu(tlv_data, + &tx_ppdu_info->rx_status); + break; + + case HAL_MACTX_HE_SIG_B2_OFDMA: + ath12k_wifi7_dp_mon_parse_he_sig_b2_ofdma(tlv_data, + &tx_ppdu_info->rx_status); + break; + + case HAL_MACTX_VHT_SIG_A: + ath12k_wifi7_dp_mon_parse_vht_sig_a(tlv_data, &tx_ppdu_info->rx_status); + break; + + case HAL_MACTX_L_SIG_A: + ath12k_wifi7_dp_mon_parse_l_sig_a(tlv_data, &tx_ppdu_info->rx_status); + break; + + case HAL_MACTX_L_SIG_B: + ath12k_wifi7_dp_mon_parse_l_sig_b(tlv_data, &tx_ppdu_info->rx_status); + break; + + case HAL_RX_FRAME_BITMAP_ACK: { + const struct hal_rx_frame_bitmap_ack *fbm_ack = tlv_data; + u32 addr_32; + u16 addr_16; + + info[0] = __le32_to_cpu(fbm_ack->info0); + info[1] = __le32_to_cpu(fbm_ack->info1); + + addr_32 = u32_get_bits(info[0], + HAL_RX_FBM_ACK_INFO0_ADDR1_31_0); + addr_16 = u32_get_bits(info[1], + HAL_RX_FBM_ACK_INFO1_ADDR1_47_32); + ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1); + + ath12k_wifi7_dp_mon_tx_gen_ack_frame(tx_ppdu_info); + break; + } + + case HAL_MACTX_PHY_DESC: { + const struct hal_tx_phy_desc *tx_phy_desc = tlv_data; + + info[0] = __le32_to_cpu(tx_phy_desc->info0); + info[1] = __le32_to_cpu(tx_phy_desc->info1); + info[2] = __le32_to_cpu(tx_phy_desc->info2); + info[3] = __le32_to_cpu(tx_phy_desc->info3); + + tx_ppdu_info->rx_status.beamformed = + u32_get_bits(info[0], + HAL_TX_PHY_DESC_INFO0_BF_TYPE); + tx_ppdu_info->rx_status.preamble_type = + u32_get_bits(info[0], + HAL_TX_PHY_DESC_INFO0_PREAMBLE_11B); + tx_ppdu_info->rx_status.mcs = + u32_get_bits(info[1], + HAL_TX_PHY_DESC_INFO1_MCS); + tx_ppdu_info->rx_status.ltf_size = + u32_get_bits(info[3], + HAL_TX_PHY_DESC_INFO3_LTF_SIZE); + tx_ppdu_info->rx_status.nss = + u32_get_bits(info[2], + HAL_TX_PHY_DESC_INFO2_NSS); + tx_ppdu_info->rx_status.chan_num = + u32_get_bits(info[3], + HAL_TX_PHY_DESC_INFO3_ACTIVE_CHANNEL); + tx_ppdu_info->rx_status.bw = + u32_get_bits(info[0], + HAL_TX_PHY_DESC_INFO0_BANDWIDTH); + break; + } + + case HAL_TX_MPDU_START: { + struct dp_mon_mpdu *mon_mpdu = tx_ppdu_info->tx_mon_mpdu; + + mon_mpdu = kzalloc(sizeof(*mon_mpdu), GFP_ATOMIC); + if (!mon_mpdu) + return DP_MON_TX_STATUS_PPDU_NOT_DONE; + status = DP_MON_TX_MPDU_START; + break; + } + + case HAL_TX_MPDU_END: + list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list, + &tx_ppdu_info->dp_tx_mon_mpdu_list); + break; + } + + return status; +} + +static enum dp_mon_tx_tlv_status +ath12k_wifi7_dp_mon_tx_status_get_num_user(u16 tlv_tag, + struct hal_tlv_hdr *tx_tlv, + u8 *num_users) +{ + u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE; + u32 info0; + + switch (tlv_tag) { + case HAL_TX_FES_SETUP: { + struct hal_tx_fes_setup *tx_fes_setup = + (struct hal_tx_fes_setup *)tx_tlv; + + info0 = __le32_to_cpu(tx_fes_setup->info0); + + *num_users = u32_get_bits(info0, HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS); + tlv_status = DP_MON_TX_FES_SETUP; + break; + } + + case HAL_RX_RESPONSE_REQUIRED_INFO: { + /* TODO: need to update *num_users */ + tlv_status = DP_MON_RX_RESPONSE_REQUIRED_INFO; + break; + } + } + + return tlv_status; +} + +static int +ath12k_wifi7_dp_mon_rx_deliver(struct ath12k_pdev_dp *dp_pdev, + struct dp_mon_mpdu *mon_mpdu, + struct hal_rx_mon_ppdu_info *ppduinfo, + struct napi_struct *napi) +{ + struct sk_buff *mon_skb, *skb_next, *header; + struct ieee80211_rx_status *rxs = &dp_pdev->rx_status; + u8 decap = DP_RX_DECAP_TYPE_RAW; + + mon_skb = ath12k_dp_mon_rx_merg_msdus(dp_pdev, mon_mpdu, ppduinfo, rxs); + if (!mon_skb) + goto mon_deliver_fail; + + header = mon_skb; + rxs->flag = 0; + + if (mon_mpdu->err_bitmap & HAL_RX_MPDU_ERR_FCS) + rxs->flag = RX_FLAG_FAILED_FCS_CRC; + + do { + skb_next = mon_skb->next; + if (!skb_next) + rxs->flag &= ~RX_FLAG_AMSDU_MORE; + else + rxs->flag |= RX_FLAG_AMSDU_MORE; + + if (mon_skb == header) { + header = NULL; + rxs->flag &= ~RX_FLAG_ALLOW_SAME_PN; + } else { + rxs->flag |= RX_FLAG_ALLOW_SAME_PN; + } + rxs->flag |= RX_FLAG_ONLY_MONITOR; + + if (!(rxs->flag & RX_FLAG_ONLY_MONITOR)) + decap = mon_mpdu->decap_format; + + ath12k_dp_mon_update_radiotap(dp_pdev, ppduinfo, mon_skb, rxs); + ath12k_dp_mon_rx_deliver_msdu(dp_pdev, napi, mon_skb, ppduinfo, + rxs, decap); + mon_skb = skb_next; + } while (mon_skb); + rxs->flag = 0; + + return 0; + +mon_deliver_fail: + mon_skb = mon_mpdu->head; + while (mon_skb) { + skb_next = mon_skb->next; + dev_kfree_skb_any(mon_skb); + mon_skb = skb_next; + } + return -EINVAL; +} + +static void +ath12k_wifi7_dp_mon_tx_process_ppdu_info(struct ath12k_pdev_dp *dp_pdev, + struct napi_struct *napi, + struct dp_mon_tx_ppdu_info *tx_ppdu_info) +{ + struct dp_mon_mpdu *tmp, *mon_mpdu; + + list_for_each_entry_safe(mon_mpdu, tmp, + &tx_ppdu_info->dp_tx_mon_mpdu_list, list) { + list_del(&mon_mpdu->list); + + if (mon_mpdu->head) + ath12k_wifi7_dp_mon_rx_deliver(dp_pdev, mon_mpdu, + &tx_ppdu_info->rx_status, napi); + + kfree(mon_mpdu); + } +} + +enum hal_rx_mon_status +ath12k_wifi7_dp_mon_tx_parse_mon_status(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_mon_data *pmon, + struct sk_buff *skb, + struct napi_struct *napi, + u32 ppdu_id) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; + struct dp_mon_tx_ppdu_info *tx_prot_ppdu_info, *tx_data_ppdu_info; + struct hal_tlv_hdr *tlv; + u8 *ptr = skb->data; + u16 tlv_tag; + u16 tlv_len; + u32 tlv_userid = 0; + u8 num_user; + u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE; + + tx_prot_ppdu_info = + ath12k_wifi7_dp_mon_tx_get_ppdu_info(pmon, ppdu_id, + DP_MON_TX_PROT_PPDU_INFO); + if (!tx_prot_ppdu_info) + return -ENOMEM; + + tlv = (struct hal_tlv_hdr *)ptr; + tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG); + + tlv_status = ath12k_wifi7_dp_mon_tx_status_get_num_user(tlv_tag, tlv, + &num_user); + if (tlv_status == DP_MON_TX_STATUS_PPDU_NOT_DONE || !num_user) + return -EINVAL; + + tx_data_ppdu_info = + ath12k_wifi7_dp_mon_tx_get_ppdu_info(pmon, ppdu_id, + DP_MON_TX_DATA_PPDU_INFO); + if (!tx_data_ppdu_info) + return -ENOMEM; + + do { + tlv = (struct hal_tlv_hdr *)ptr; + tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG); + tlv_len = le32_get_bits(tlv->tl, HAL_TLV_HDR_LEN); + tlv_userid = le32_get_bits(tlv->tl, HAL_TLV_USR_ID); + + tlv_status = ath12k_wifi7_dp_mon_tx_parse_status_tlv(ab, pmon, + tlv_tag, ptr, + tlv_userid); + ptr += tlv_len; + ptr = PTR_ALIGN(ptr, HAL_TLV_ALIGN); + if ((ptr - skb->data) >= DP_TX_MONITOR_BUF_SIZE) + break; + } while (tlv_status != DP_MON_TX_FES_STATUS_END); + + ath12k_wifi7_dp_mon_tx_process_ppdu_info(dp_pdev, napi, tx_data_ppdu_info); + ath12k_wifi7_dp_mon_tx_process_ppdu_info(dp_pdev, napi, tx_prot_ppdu_info); + + return tlv_status; +} + +static void +ath12k_wifi7_dp_mon_next_link_desc_get(struct ath12k_base *ab, + struct hal_rx_msdu_link *msdu_link, + dma_addr_t *paddr, u32 *sw_cookie, u8 *rbm, + struct ath12k_buffer_addr **pp_buf_addr_info) +{ + struct ath12k_buffer_addr *buf_addr_info; + + buf_addr_info = &msdu_link->buf_addr_info; + + ath12k_wifi7_hal_rx_buf_addr_info_get(buf_addr_info, paddr, sw_cookie, rbm); + + *pp_buf_addr_info = buf_addr_info; +} + +static u32 +ath12k_wifi7_dp_rx_mon_mpdu_pop(struct ath12k *ar, int mac_id, + void *ring_entry, struct sk_buff **head_msdu, + struct sk_buff **tail_msdu, + struct list_head *used_list, + u32 *npackets, u32 *ppdu_id) +{ + struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data; + struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_buffer_addr *p_buf_addr_info, *p_last_buf_addr_info; + u32 msdu_ppdu_id = 0, msdu_cnt = 0, total_len = 0, frag_len = 0; + u32 rx_buf_size, rx_pkt_offset, sw_cookie; + bool is_frag, is_first_msdu, drop_mpdu = false; + struct hal_reo_entrance_ring *ent_desc = + (struct hal_reo_entrance_ring *)ring_entry; + u32 rx_bufs_used = 0, i = 0, desc_bank = 0; + struct hal_rx_desc *rx_desc, *tail_rx_desc; + struct hal_rx_msdu_link *msdu_link_desc; + struct sk_buff *msdu = NULL, *last = NULL; + struct ath12k_rx_desc_info *desc_info; + struct ath12k_buffer_addr buf_info; + struct hal_rx_msdu_list msdu_list; + struct ath12k_skb_rxcb *rxcb; + u16 num_msdus = 0; + dma_addr_t paddr; + u8 rbm; + + ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get(ring_entry, &paddr, + &sw_cookie, + &p_last_buf_addr_info, + &rbm, + &msdu_cnt); + + spin_lock_bh(&pmon->mon_lock); + + if (le32_get_bits(ent_desc->info1, + HAL_REO_ENTR_RING_INFO1_RXDMA_PUSH_REASON) == + HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) { + u8 rxdma_err = le32_get_bits(ent_desc->info1, + HAL_REO_ENTR_RING_INFO1_RXDMA_ERROR_CODE); + if (rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR || + rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR || + rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR) { + drop_mpdu = true; + pmon->rx_mon_stats.dest_mpdu_drop++; + } + } + + is_frag = false; + is_first_msdu = true; + rx_pkt_offset = sizeof(struct hal_rx_desc); + + do { + if (pmon->mon_last_linkdesc_paddr == paddr) { + pmon->rx_mon_stats.dup_mon_linkdesc_cnt++; + spin_unlock_bh(&pmon->mon_lock); + return rx_bufs_used; + } + + desc_bank = u32_get_bits(sw_cookie, DP_LINK_DESC_BANK_MASK); + msdu_link_desc = + dp->link_desc_banks[desc_bank].vaddr + + (paddr - dp->link_desc_banks[desc_bank].paddr); + + ath12k_wifi7_hal_rx_msdu_list_get(ar, msdu_link_desc, &msdu_list, + &num_msdus); + desc_info = ath12k_dp_get_rx_desc(ar->ab->dp, + msdu_list.sw_cookie[num_msdus - 1]); + tail_rx_desc = (struct hal_rx_desc *)(desc_info->skb)->data; + + for (i = 0; i < num_msdus; i++) { + u32 l2_hdr_offset; + + if (pmon->mon_last_buf_cookie == msdu_list.sw_cookie[i]) { + ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + "i %d last_cookie %d is same\n", + i, pmon->mon_last_buf_cookie); + drop_mpdu = true; + pmon->rx_mon_stats.dup_mon_buf_cnt++; + continue; + } + + desc_info = + ath12k_dp_get_rx_desc(ar->ab->dp, msdu_list.sw_cookie[i]); + msdu = desc_info->skb; + + if (!msdu) { + ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + "msdu_pop: invalid msdu (%d/%d)\n", + i + 1, num_msdus); + goto next_msdu; + } + rxcb = ATH12K_SKB_RXCB(msdu); + if (rxcb->paddr != msdu_list.paddr[i]) { + ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + "i %d paddr %lx != %lx\n", + i, (unsigned long)rxcb->paddr, + (unsigned long)msdu_list.paddr[i]); + drop_mpdu = true; + continue; + } + if (!rxcb->unmapped) { + dma_unmap_single(ar->ab->dev, rxcb->paddr, + msdu->len + + skb_tailroom(msdu), + DMA_FROM_DEVICE); + rxcb->unmapped = 1; + } + if (drop_mpdu) { + ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + "i %d drop msdu %p *ppdu_id %x\n", + i, msdu, *ppdu_id); + dev_kfree_skb_any(msdu); + msdu = NULL; + goto next_msdu; + } + + rx_desc = (struct hal_rx_desc *)msdu->data; + l2_hdr_offset = ath12k_dp_rx_h_l3pad(ar->ab, tail_rx_desc); + if (is_first_msdu) { + if (!ath12k_wifi7_dp_rxdesc_mpdu_valid(ar->ab, + rx_desc)) { + drop_mpdu = true; + dev_kfree_skb_any(msdu); + msdu = NULL; + pmon->mon_last_linkdesc_paddr = paddr; + goto next_msdu; + } + msdu_ppdu_id = + ath12k_dp_rxdesc_get_ppduid(ar->ab, rx_desc); + + if (ath12k_dp_mon_comp_ppduid(msdu_ppdu_id, + ppdu_id)) { + spin_unlock_bh(&pmon->mon_lock); + return rx_bufs_used; + } + pmon->mon_last_linkdesc_paddr = paddr; + is_first_msdu = false; + } + ath12k_wifi7_dp_mon_get_buf_len(&msdu_list.msdu_info[i], + &is_frag, &total_len, + &frag_len, &msdu_cnt); + rx_buf_size = rx_pkt_offset + l2_hdr_offset + frag_len; + + if (ath12k_dp_pkt_set_pktlen(msdu, rx_buf_size)) { + dev_kfree_skb_any(msdu); + goto next_msdu; + } + + if (!(*head_msdu)) + *head_msdu = msdu; + else if (last) + last->next = msdu; + + last = msdu; +next_msdu: + pmon->mon_last_buf_cookie = msdu_list.sw_cookie[i]; + rx_bufs_used++; + desc_info->skb = NULL; + list_add_tail(&desc_info->list, used_list); + } + + ath12k_wifi7_hal_rx_buf_addr_info_set(&buf_info, paddr, + sw_cookie, rbm); + + ath12k_wifi7_dp_mon_next_link_desc_get(ab, + msdu_link_desc, &paddr, + &sw_cookie, &rbm, + &p_buf_addr_info); + + ath12k_dp_arch_rx_link_desc_return(ar->ab->dp, &buf_info, + HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); + + p_last_buf_addr_info = p_buf_addr_info; + + } while (paddr && msdu_cnt); + + spin_unlock_bh(&pmon->mon_lock); + + if (last) + last->next = NULL; + + *tail_msdu = msdu; + + if (msdu_cnt == 0) + *npackets = 1; + + return rx_bufs_used; +} + +/* The destination ring processing is stuck if the destination is not + * moving while status ring moves 16 PPDU. The destination ring processing + * skips this destination ring PPDU as a workaround. + */ +#define MON_DEST_RING_STUCK_MAX_CNT 16 + +static void +ath12k_wifi7_dp_rx_mon_dest_process(struct ath12k *ar, int mac_id, + u32 quota, struct napi_struct *napi) +{ + struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data; + struct ath12k_pdev_mon_stats *rx_mon_stats; + u32 ppdu_id, rx_bufs_used = 0, ring_id; + u32 mpdu_rx_bufs_used, npackets = 0; + struct ath12k_base *ab = ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + void *ring_entry, *mon_dst_srng; + struct dp_mon_mpdu *tmp_mpdu; + LIST_HEAD(rx_desc_used_list); + struct hal_srng *srng; + + ring_id = dp->rxdma_err_dst_ring[mac_id].ring_id; + srng = &ab->hal.srng_list[ring_id]; + + mon_dst_srng = &ab->hal.srng_list[ring_id]; + + spin_lock_bh(&srng->lock); + + ath12k_hal_srng_access_begin(ab, mon_dst_srng); + + ppdu_id = pmon->mon_ppdu_info.ppdu_id; + rx_mon_stats = &pmon->rx_mon_stats; + + while ((ring_entry = ath12k_hal_srng_dst_peek(ar->ab, mon_dst_srng))) { + struct sk_buff *head_msdu, *tail_msdu; + + head_msdu = NULL; + tail_msdu = NULL; + + mpdu_rx_bufs_used = ath12k_wifi7_dp_rx_mon_mpdu_pop(ar, mac_id, + ring_entry, + &head_msdu, + &tail_msdu, + &rx_desc_used_list, + &npackets, + &ppdu_id); + + rx_bufs_used += mpdu_rx_bufs_used; + + if (mpdu_rx_bufs_used) { + dp->mon_dest_ring_stuck_cnt = 0; + } else { + dp->mon_dest_ring_stuck_cnt++; + rx_mon_stats->dest_mon_not_reaped++; + } + + if (dp->mon_dest_ring_stuck_cnt > MON_DEST_RING_STUCK_MAX_CNT) { + rx_mon_stats->dest_mon_stuck++; + ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + "status ring ppdu_id=%d dest ring ppdu_id=%d mon_dest_ring_stuck_cnt=%d dest_mon_not_reaped=%u dest_mon_stuck=%u\n", + pmon->mon_ppdu_info.ppdu_id, ppdu_id, + dp->mon_dest_ring_stuck_cnt, + rx_mon_stats->dest_mon_not_reaped, + rx_mon_stats->dest_mon_stuck); + spin_lock_bh(&pmon->mon_lock); + pmon->mon_ppdu_info.ppdu_id = ppdu_id; + spin_unlock_bh(&pmon->mon_lock); + continue; + } + + if (ppdu_id != pmon->mon_ppdu_info.ppdu_id) { + spin_lock_bh(&pmon->mon_lock); + pmon->mon_ppdu_status = DP_PPDU_STATUS_START; + spin_unlock_bh(&pmon->mon_lock); + ath12k_dbg(ar->ab, ATH12K_DBG_DATA, + "dest_rx: new ppdu_id %x != status ppdu_id %x dest_mon_not_reaped = %u dest_mon_stuck = %u\n", + ppdu_id, pmon->mon_ppdu_info.ppdu_id, + rx_mon_stats->dest_mon_not_reaped, + rx_mon_stats->dest_mon_stuck); + break; + } + + if (head_msdu && tail_msdu) { + tmp_mpdu = kzalloc(sizeof(*tmp_mpdu), GFP_ATOMIC); + if (!tmp_mpdu) + break; + + tmp_mpdu->head = head_msdu; + tmp_mpdu->tail = tail_msdu; + tmp_mpdu->err_bitmap = pmon->err_bitmap; + tmp_mpdu->decap_format = pmon->decap_format; + ath12k_wifi7_dp_mon_rx_deliver(&ar->dp, tmp_mpdu, + &pmon->mon_ppdu_info, napi); + rx_mon_stats->dest_mpdu_done++; + kfree(tmp_mpdu); + } + + ring_entry = ath12k_hal_srng_dst_get_next_entry(ar->ab, + mon_dst_srng); + } + ath12k_hal_srng_access_end(ar->ab, mon_dst_srng); + + spin_unlock_bh(&srng->lock); + + if (rx_bufs_used) { + rx_mon_stats->dest_ppdu_done++; + ath12k_dp_rx_bufs_replenish(ar->ab->dp, + &dp->rx_refill_buf_ring, + &rx_desc_used_list, + rx_bufs_used); + } +} + +static enum dp_mon_status_buf_state +ath12k_wifi7_dp_rx_mon_buf_done(struct ath12k_base *ab, struct hal_srng *srng, + struct dp_rxdma_mon_ring *rx_ring) +{ + struct ath12k_hal *hal = &ab->hal; + struct ath12k_skb_rxcb *rxcb; + struct sk_buff *skb; + void *status_desc; + dma_addr_t paddr; + u16 tlv_tag; + u32 cookie; + int buf_id; + u8 rbm; + + status_desc = ath12k_hal_srng_src_next_peek(ab, srng); + if (!status_desc) + return DP_MON_STATUS_NO_DMA; + + ath12k_wifi7_hal_rx_buf_addr_info_get(status_desc, &paddr, &cookie, &rbm); + + buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); + + spin_lock_bh(&rx_ring->idr_lock); + skb = idr_find(&rx_ring->bufs_idr, buf_id); + spin_unlock_bh(&rx_ring->idr_lock); + + if (!skb) + return DP_MON_STATUS_NO_DMA; + + rxcb = ATH12K_SKB_RXCB(skb); + dma_sync_single_for_cpu(ab->dev, rxcb->paddr, + skb->len + skb_tailroom(skb), + DMA_FROM_DEVICE); + + hal->ops->mon_rx_status_dec_tlv_hdr(skb->data, &tlv_tag, NULL, NULL); + if (tlv_tag != HAL_RX_STATUS_BUFFER_DONE) + return DP_MON_STATUS_NO_DMA; + + return DP_MON_STATUS_REPLINISH; +} + +static enum hal_rx_mon_status +ath12k_wifi7_dp_mon_parse_rx_dest(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_mon_data *pmon, + struct sk_buff *skb) +{ + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + struct ath12k_hal *hal = &ar->ab->hal; + u8 *tlv_value, *tlv = skb->data; + struct ath12k_skb_rxcb *rxcb; + enum hal_rx_mon_status hal_status; + u16 tlv_tag, tlv_len; + u32 tlv_hdr_len; + + tlv_hdr_len = hal->ops->get_tlv_hdr_align(); + + do { + tlv_value = hal->ops->mon_rx_status_dec_tlv_hdr(tlv, &tlv_tag, + &tlv_len, NULL); + + /* The actual length of PPDU_END is the combined length of many PHY + * TLVs that follow. Skip the TLV header and + * rx_rxpcu_classification_overview that follows the header to get to + * next TLV. + */ + if (tlv_tag == HAL_RX_PPDU_END) + tlv_len = sizeof(struct hal_rx_rxpcu_classification_overview); + + hal_status = ath12k_wifi7_dp_mon_rx_parse_status_tlv(dp_pdev, pmon, + tlv); + + if (ar->monitor_started && ar->ab->hw_params->rxdma1_enable && + ath12k_wifi7_dp_mon_parse_rx_dest_tlv(dp_pdev, pmon, hal_status, + tlv_value)) + return HAL_RX_MON_STATUS_PPDU_DONE; + + tlv = PTR_ALIGN(tlv + tlv_len + tlv_hdr_len, tlv_hdr_len); + + if (unlikely(tlv - skb->data > skb->len || + skb->len - (tlv - skb->data) < tlv_hdr_len)) + break; + } while ((hal_status == HAL_RX_MON_STATUS_PPDU_NOT_DONE) || + (hal_status == HAL_RX_MON_STATUS_BUF_ADDR) || + (hal_status == HAL_RX_MON_STATUS_MPDU_START) || + (hal_status == HAL_RX_MON_STATUS_MPDU_END) || + (hal_status == HAL_RX_MON_STATUS_MSDU_END)); + + rxcb = ATH12K_SKB_RXCB(skb); + if (rxcb->is_end_of_ppdu) + hal_status = HAL_RX_MON_STATUS_PPDU_DONE; + + return hal_status; +} + +static enum hal_rx_mon_status +ath12k_wifi7_dp_mon_rx_parse_mon_status(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_mon_data *pmon, + struct sk_buff *skb, + struct napi_struct *napi) +{ + struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; + struct dp_mon_mpdu *tmp; + struct dp_mon_mpdu *mon_mpdu = pmon->mon_mpdu; + enum hal_rx_mon_status hal_status; + + hal_status = ath12k_wifi7_dp_mon_parse_rx_dest(dp_pdev, pmon, skb); + if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE) + return hal_status; + + list_for_each_entry_safe(mon_mpdu, tmp, &pmon->dp_rx_mon_mpdu_list, list) { + list_del(&mon_mpdu->list); + + if (mon_mpdu->head && mon_mpdu->tail) + ath12k_wifi7_dp_mon_rx_deliver(dp_pdev, mon_mpdu, + ppdu_info, napi); + + kfree(mon_mpdu); + } + + return hal_status; +} + +static int +ath12k_wifi7_dp_rx_reap_mon_status_ring(struct ath12k_base *ab, int mac_id, + int *budget, struct sk_buff_head *skb_list) +{ + const struct ath12k_hw_hal_params *hal_params; + int buf_id, srng_id, num_buffs_reaped = 0; + enum dp_mon_status_buf_state reap_status; + struct dp_rxdma_mon_ring *rx_ring; + struct ath12k_hal *hal = &ab->hal; + struct ath12k_mon_data *pmon; + struct ath12k_skb_rxcb *rxcb; + void *rx_mon_status_desc; + struct hal_srng *srng; + struct ath12k_dp *dp; + struct sk_buff *skb; + struct ath12k *ar; + dma_addr_t paddr; + u16 tlv_tag; + u32 cookie; + u8 rbm; + + ar = ab->pdevs[ath12k_hw_mac_id_to_pdev_id(ab->hw_params, mac_id)].ar; + dp = ath12k_ab_to_dp(ab); + pmon = &ar->dp.mon_data; + srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, mac_id); + rx_ring = &dp->rx_mon_status_refill_ring[srng_id]; + + srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id]; + + spin_lock_bh(&srng->lock); + + ath12k_hal_srng_access_begin(ab, srng); + + while (*budget) { + *budget -= 1; + rx_mon_status_desc = ath12k_hal_srng_src_peek(ab, srng); + if (!rx_mon_status_desc) { + pmon->buf_state = DP_MON_STATUS_REPLINISH; + break; + } + ath12k_wifi7_hal_rx_buf_addr_info_get(rx_mon_status_desc, &paddr, + &cookie, &rbm); + if (paddr) { + buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); + + spin_lock_bh(&rx_ring->idr_lock); + skb = idr_find(&rx_ring->bufs_idr, buf_id); + spin_unlock_bh(&rx_ring->idr_lock); + + if (!skb) { + ath12k_warn(ab, "rx monitor status with invalid buf_id %d\n", + buf_id); + pmon->buf_state = DP_MON_STATUS_REPLINISH; + goto move_next; + } + + rxcb = ATH12K_SKB_RXCB(skb); + + dma_sync_single_for_cpu(ab->dev, rxcb->paddr, + skb->len + skb_tailroom(skb), + DMA_FROM_DEVICE); + + hal->ops->mon_rx_status_dec_tlv_hdr(skb->data, &tlv_tag, + NULL, NULL); + if (tlv_tag != HAL_RX_STATUS_BUFFER_DONE) { + pmon->buf_state = DP_MON_STATUS_NO_DMA; + ath12k_warn(ab, + "mon status DONE not set %x, buf_id %d\n", + tlv_tag, buf_id); + /* RxDMA status done bit might not be set even + * though tp is moved by HW. + */ + + /* If done status is missing: + * 1. As per MAC team's suggestion, + * when HP + 1 entry is peeked and if DMA + * is not done and if HP + 2 entry's DMA done + * is set. skip HP + 1 entry and + * start processing in next interrupt. + * 2. If HP + 2 entry's DMA done is not set, + * poll onto HP + 1 entry DMA done to be set. + * Check status for same buffer for next time + * dp_rx_mon_status_srng_process + */ + reap_status = ath12k_wifi7_dp_rx_mon_buf_done(ab, srng, + rx_ring); + if (reap_status == DP_MON_STATUS_NO_DMA) + continue; + + spin_lock_bh(&rx_ring->idr_lock); + idr_remove(&rx_ring->bufs_idr, buf_id); + spin_unlock_bh(&rx_ring->idr_lock); + + dma_unmap_single(ab->dev, rxcb->paddr, + skb->len + skb_tailroom(skb), + DMA_FROM_DEVICE); + + dev_kfree_skb_any(skb); + pmon->buf_state = DP_MON_STATUS_REPLINISH; + goto move_next; + } + + spin_lock_bh(&rx_ring->idr_lock); + idr_remove(&rx_ring->bufs_idr, buf_id); + spin_unlock_bh(&rx_ring->idr_lock); + + dma_unmap_single(ab->dev, rxcb->paddr, + skb->len + skb_tailroom(skb), + DMA_FROM_DEVICE); + + if (ath12k_dp_pkt_set_pktlen(skb, RX_MON_STATUS_BUF_SIZE)) { + dev_kfree_skb_any(skb); + goto move_next; + } + __skb_queue_tail(skb_list, skb); + } else { + pmon->buf_state = DP_MON_STATUS_REPLINISH; + } +move_next: + skb = ath12k_dp_rx_alloc_mon_status_buf(ab, rx_ring, + &buf_id); + hal_params = ab->hal.hal_params; + + if (!skb) { + ath12k_warn(ab, "failed to alloc buffer for status ring\n"); + ath12k_wifi7_hal_rx_buf_addr_info_set(rx_mon_status_desc, + 0, 0, + hal_params->rx_buf_rbm); + num_buffs_reaped++; + break; + } + rxcb = ATH12K_SKB_RXCB(skb); + + cookie = u32_encode_bits(mac_id, DP_RXDMA_BUF_COOKIE_PDEV_ID) | + u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID); + + ath12k_wifi7_hal_rx_buf_addr_info_set(rx_mon_status_desc, rxcb->paddr, + cookie, hal_params->rx_buf_rbm); + ath12k_hal_srng_src_get_next_entry(ab, srng); + num_buffs_reaped++; + } + ath12k_hal_srng_access_end(ab, srng); + spin_unlock_bh(&srng->lock); + + return num_buffs_reaped; +} + +static int +__ath12k_wifi7_dp_mon_process_ring(struct ath12k *ar, int mac_id, + struct napi_struct *napi, int *budget) +{ + struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data; + struct ath12k_pdev_mon_stats *rx_mon_stats = &pmon->rx_mon_stats; + struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; + enum hal_rx_mon_status hal_status; + struct sk_buff_head skb_list; + int num_buffs_reaped; + struct sk_buff *skb; + + __skb_queue_head_init(&skb_list); + + num_buffs_reaped = ath12k_wifi7_dp_rx_reap_mon_status_ring(ar->ab, mac_id, + budget, &skb_list); + if (!num_buffs_reaped) + goto exit; + + while ((skb = __skb_dequeue(&skb_list))) { + memset(ppdu_info, 0, sizeof(*ppdu_info)); + ppdu_info->peer_id = HAL_INVALID_PEERID; + + hal_status = ath12k_wifi7_dp_mon_parse_rx_dest(&ar->dp, pmon, skb); + + if (ar->monitor_started && + pmon->mon_ppdu_status == DP_PPDU_STATUS_START && + hal_status == HAL_TLV_STATUS_PPDU_DONE) { + rx_mon_stats->status_ppdu_done++; + pmon->mon_ppdu_status = DP_PPDU_STATUS_DONE; + ath12k_wifi7_dp_rx_mon_dest_process(ar, mac_id, *budget, napi); + pmon->mon_ppdu_status = DP_PPDU_STATUS_START; + } + + dev_kfree_skb_any(skb); + } + +exit: + return num_buffs_reaped; +} + +static int +ath12k_wifi7_dp_mon_srng_process(struct ath12k_pdev_dp *pdev_dp, int *budget, + struct napi_struct *napi) +{ + struct ath12k_dp *dp = pdev_dp->dp; + struct ath12k_base *ab = dp->ab; + struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&pdev_dp->mon_data; + struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info; + struct hal_mon_dest_desc *mon_dst_desc; + struct sk_buff *skb; + struct ath12k_skb_rxcb *rxcb; + struct dp_srng *mon_dst_ring; + struct hal_srng *srng; + struct dp_rxdma_mon_ring *buf_ring; + struct ath12k_dp_link_peer *peer; + struct sk_buff_head skb_list; + u64 cookie; + int num_buffs_reaped = 0, srng_id, buf_id; + u32 hal_status, end_offset, info0, end_reason; + u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, pdev_dp->mac_id); + + __skb_queue_head_init(&skb_list); + srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, pdev_idx); + mon_dst_ring = &pdev_dp->rxdma_mon_dst_ring[srng_id]; + buf_ring = &dp->rxdma_mon_buf_ring; + + srng = &ab->hal.srng_list[mon_dst_ring->ring_id]; + spin_lock_bh(&srng->lock); + ath12k_hal_srng_access_begin(ab, srng); + + while (likely(*budget)) { + mon_dst_desc = ath12k_hal_srng_dst_peek(ab, srng); + if (unlikely(!mon_dst_desc)) + break; + + /* In case of empty descriptor, the cookie in the ring descriptor + * is invalid. Therefore, this entry is skipped, and ring processing + * continues. + */ + info0 = le32_to_cpu(mon_dst_desc->info0); + if (u32_get_bits(info0, HAL_MON_DEST_INFO0_EMPTY_DESC)) + goto move_next; + + cookie = le32_to_cpu(mon_dst_desc->cookie); + buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID); + + spin_lock_bh(&buf_ring->idr_lock); + skb = idr_remove(&buf_ring->bufs_idr, buf_id); + spin_unlock_bh(&buf_ring->idr_lock); + + if (unlikely(!skb)) { + ath12k_warn(ab, "monitor destination with invalid buf_id %d\n", + buf_id); + goto move_next; + } + + rxcb = ATH12K_SKB_RXCB(skb); + dma_unmap_single(ab->dev, rxcb->paddr, + skb->len + skb_tailroom(skb), + DMA_FROM_DEVICE); + + end_reason = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_REASON); + + /* HAL_MON_FLUSH_DETECTED implies that an rx flush received at the end of + * rx PPDU and HAL_MON_PPDU_TRUNCATED implies that the PPDU got + * truncated due to a system level error. In both the cases, buffer data + * can be discarded + */ + if ((end_reason == HAL_MON_FLUSH_DETECTED) || + (end_reason == HAL_MON_PPDU_TRUNCATED)) { + ath12k_dbg(ab, ATH12K_DBG_DATA, + "Monitor dest descriptor end reason %d", end_reason); + dev_kfree_skb_any(skb); + goto move_next; + } + + /* Calculate the budget when the ring descriptor with the + * HAL_MON_END_OF_PPDU to ensure that one PPDU worth of data is always + * reaped. This helps to efficiently utilize the NAPI budget. + */ + if (end_reason == HAL_MON_END_OF_PPDU) { + *budget -= 1; + rxcb->is_end_of_ppdu = true; + } + + end_offset = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_OFFSET); + if (likely(end_offset <= DP_RX_BUFFER_SIZE)) { + skb_put(skb, end_offset); + } else { + ath12k_warn(ab, + "invalid offset on mon stats destination %u\n", + end_offset); + skb_put(skb, DP_RX_BUFFER_SIZE); + } + + __skb_queue_tail(&skb_list, skb); + +move_next: + ath12k_dp_mon_buf_replenish(ab, buf_ring, 1); + ath12k_hal_srng_dst_get_next_entry(ab, srng); + num_buffs_reaped++; + } + + ath12k_hal_srng_access_end(ab, srng); + spin_unlock_bh(&srng->lock); + + if (!num_buffs_reaped) + return 0; + + /* In some cases, one PPDU worth of data can be spread across multiple NAPI + * schedules, To avoid losing existing parsed ppdu_info information, skip + * the memset of the ppdu_info structure and continue processing it. + */ + if (!ppdu_info->ppdu_continuation) + ath12k_wifi7_dp_mon_rx_memset_ppdu_info(ppdu_info); + + while ((skb = __skb_dequeue(&skb_list))) { + hal_status = ath12k_wifi7_dp_mon_rx_parse_mon_status(pdev_dp, pmon, + skb, napi); + if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE) { + ppdu_info->ppdu_continuation = true; + dev_kfree_skb_any(skb); + continue; + } + + if (ppdu_info->peer_id == HAL_INVALID_PEERID) + goto free_skb; + + rcu_read_lock(); + peer = ath12k_dp_link_peer_find_by_peerid(pdev_dp, ppdu_info->peer_id); + if (!peer || !peer->sta) { + ath12k_dbg(ab, ATH12K_DBG_DATA, + "failed to find the peer with monitor peer_id %d\n", + ppdu_info->peer_id); + goto next_skb; + } + + if (ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_SU) { + ath12k_dp_mon_rx_update_peer_su_stats(peer, ppdu_info); + } else if ((ppdu_info->fc_valid) && + (ppdu_info->ast_index != HAL_AST_IDX_INVALID)) { + ath12k_dp_mon_rx_process_ulofdma(ppdu_info); + ath12k_dp_mon_rx_update_peer_mu_stats(ab, ppdu_info); + } + +next_skb: + rcu_read_unlock(); +free_skb: + dev_kfree_skb_any(skb); + ath12k_wifi7_dp_mon_rx_memset_ppdu_info(ppdu_info); + } + + return num_buffs_reaped; +} + +int ath12k_wifi7_dp_mon_process_ring(struct ath12k_dp *dp, int mac_id, + struct napi_struct *napi, int budget, + enum dp_monitor_mode monitor_mode) +{ + u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, mac_id); + struct ath12k_pdev_dp *dp_pdev; + struct ath12k *ar; + int num_buffs_reaped = 0; + + rcu_read_lock(); + + dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx); + if (!dp_pdev) { + rcu_read_unlock(); + return 0; + } + + if (dp->hw_params->rxdma1_enable) { + if (monitor_mode == ATH12K_DP_RX_MONITOR_MODE) + num_buffs_reaped = ath12k_wifi7_dp_mon_srng_process(dp_pdev, + &budget, + napi); + } else { + ar = ath12k_pdev_dp_to_ar(dp_pdev); + + if (ar->monitor_started) + num_buffs_reaped = + __ath12k_wifi7_dp_mon_process_ring(ar, mac_id, napi, + &budget); + } + + rcu_read_unlock(); + + return num_buffs_reaped; +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h new file mode 100644 index 0000000000000..148d1e0b70fe8 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_DP_MON_WIFI7_H +#define ATH12K_DP_MON_WIFI7_H + +#include "hw.h" + +enum dp_monitor_mode; + +int ath12k_wifi7_dp_mon_process_ring(struct ath12k_dp *dp, int mac_id, + struct napi_struct *napi, int budget, + enum dp_monitor_mode monitor_mode); +enum hal_rx_mon_status +ath12k_wifi7_dp_mon_tx_parse_mon_status(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_mon_data *pmon, + struct sk_buff *skb, + struct napi_struct *napi, + u32 ppdu_id); +#endif diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c new file mode 100644 index 0000000000000..27b55fe181b96 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c @@ -0,0 +1,2248 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "dp_rx.h" +#include "../dp_tx.h" +#include "../peer.h" +#include "hal_qcn9274.h" +#include "hal_wcn7850.h" +#include "hal_qcc2072.h" + +static u16 ath12k_wifi7_dp_rx_get_peer_id(struct ath12k_dp *dp, + enum ath12k_peer_metadata_version ver, + __le32 peer_metadata) +{ + switch (ver) { + default: + ath12k_warn(dp->ab, "Unknown peer metadata version: %d", ver); + fallthrough; + case ATH12K_PEER_METADATA_V0: + return le32_get_bits(peer_metadata, + RX_MPDU_DESC_META_DATA_V0_PEER_ID); + case ATH12K_PEER_METADATA_V1: + return le32_get_bits(peer_metadata, + RX_MPDU_DESC_META_DATA_V1_PEER_ID); + case ATH12K_PEER_METADATA_V1A: + return le32_get_bits(peer_metadata, + RX_MPDU_DESC_META_DATA_V1A_PEER_ID); + case ATH12K_PEER_METADATA_V1B: + return le32_get_bits(peer_metadata, + RX_MPDU_DESC_META_DATA_V1B_PEER_ID); + } +} + +void ath12k_wifi7_peer_rx_tid_qref_setup(struct ath12k_base *ab, u16 peer_id, u16 tid, + dma_addr_t paddr) +{ + struct ath12k_reo_queue_ref *qref; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + bool ml_peer = false; + + if (!ab->hw_params->reoq_lut_support) + return; + + if (peer_id & ATH12K_PEER_ML_ID_VALID) { + peer_id &= ~ATH12K_PEER_ML_ID_VALID; + ml_peer = true; + } + + if (ml_peer) + qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr + + (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); + else + qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr + + (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); + + qref->info0 = u32_encode_bits(lower_32_bits(paddr), + BUFFER_ADDR_INFO0_ADDR); + qref->info1 = u32_encode_bits(upper_32_bits(paddr), + BUFFER_ADDR_INFO1_ADDR) | + u32_encode_bits(tid, DP_REO_QREF_NUM); + + ath12k_hal_reo_shared_qaddr_cache_clear(ab); +} + +void ath12k_wifi7_peer_rx_tid_qref_reset(struct ath12k_base *ab, + u16 peer_id, u16 tid) +{ + struct ath12k_reo_queue_ref *qref; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + bool ml_peer = false; + + if (!ab->hw_params->reoq_lut_support) + return; + + if (peer_id & ATH12K_PEER_ML_ID_VALID) { + peer_id &= ~ATH12K_PEER_ML_ID_VALID; + ml_peer = true; + } + + if (ml_peer) + qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr + + (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); + else + qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr + + (peer_id * (IEEE80211_NUM_TIDS + 1) + tid); + + qref->info0 = u32_encode_bits(0, BUFFER_ADDR_INFO0_ADDR); + qref->info1 = u32_encode_bits(0, BUFFER_ADDR_INFO1_ADDR) | + u32_encode_bits(tid, DP_REO_QREF_NUM); +} + +void ath12k_wifi7_dp_rx_peer_tid_delete(struct ath12k_base *ab, + struct ath12k_dp_link_peer *peer, u8 tid) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + + if (!(peer->rx_tid_active_bitmask & (1 << tid))) + return; + + ath12k_dp_mark_tid_as_inactive(dp, peer->peer_id, tid); + ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(dp); +} + +int ath12k_wifi7_dp_rx_link_desc_return(struct ath12k_dp *dp, + struct ath12k_buffer_addr *buf_addr_info, + enum hal_wbm_rel_bm_act action) +{ + struct ath12k_base *ab = dp->ab; + struct hal_wbm_release_ring *desc; + struct hal_srng *srng; + int ret = 0; + + srng = &dp->hal->srng_list[dp->wbm_desc_rel_ring.ring_id]; + + spin_lock_bh(&srng->lock); + + ath12k_hal_srng_access_begin(ab, srng); + + desc = ath12k_hal_srng_src_get_next_entry(ab, srng); + if (!desc) { + ret = -ENOBUFS; + goto exit; + } + + ath12k_wifi7_hal_rx_msdu_link_desc_set(ab, desc, buf_addr_info, action); + +exit: + ath12k_hal_srng_access_end(ab, srng); + + spin_unlock_bh(&srng->lock); + + return ret; +} + +int ath12k_wifi7_dp_reo_cmd_send(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid, + enum hal_reo_cmd_type type, + struct ath12k_hal_reo_cmd *cmd, + void (*cb)(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status)) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_dp_rx_reo_cmd *dp_cmd; + struct hal_srng *cmd_ring; + int cmd_num; + + cmd_ring = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id]; + cmd_num = ath12k_wifi7_hal_reo_cmd_send(ab, cmd_ring, type, cmd); + + /* cmd_num should start from 1, during failure return the error code */ + if (cmd_num < 0) + return cmd_num; + + /* reo cmd ring descriptors has cmd_num starting from 1 */ + if (cmd_num == 0) + return -EINVAL; + + if (!cb) + return 0; + + /* Can this be optimized so that we keep the pending command list only + * for tid delete command to free up the resource on the command status + * indication? + */ + dp_cmd = kzalloc(sizeof(*dp_cmd), GFP_ATOMIC); + + if (!dp_cmd) + return -ENOMEM; + + memcpy(&dp_cmd->data, rx_tid, sizeof(*rx_tid)); + dp_cmd->cmd_num = cmd_num; + dp_cmd->handler = cb; + + spin_lock_bh(&dp->reo_cmd_lock); + list_add_tail(&dp_cmd->list, &dp->reo_cmd_list); + spin_unlock_bh(&dp->reo_cmd_lock); + + return 0; +} + +int ath12k_wifi7_peer_rx_tid_reo_update(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer, + struct ath12k_dp_rx_tid *rx_tid, + u32 ba_win_sz, u16 ssn, + bool update_ssn) +{ + struct ath12k_hal_reo_cmd cmd = {}; + struct ath12k_base *ab = dp->ab; + int ret; + struct ath12k_dp_rx_tid_rxq rx_tid_rxq; + + ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid, + (peer->rx_tid_active_bitmask & (1 << rx_tid->tid))); + + cmd.addr_lo = lower_32_bits(rx_tid_rxq.qbuf.paddr_aligned); + cmd.addr_hi = upper_32_bits(rx_tid_rxq.qbuf.paddr_aligned); + cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS; + cmd.upd0 = HAL_REO_CMD_UPD0_BA_WINDOW_SIZE; + cmd.ba_window_size = ba_win_sz; + + if (update_ssn) { + cmd.upd0 |= HAL_REO_CMD_UPD0_SSN; + cmd.upd2 = u32_encode_bits(ssn, HAL_REO_CMD_UPD2_SSN); + } + + ret = ath12k_wifi7_dp_reo_cmd_send(ab, &rx_tid_rxq, + HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd, + NULL); + if (ret) { + ath12k_warn(ab, "failed to update rx tid queue, tid %d (%d)\n", + rx_tid_rxq.tid, ret); + return ret; + } + + rx_tid->ba_win_sz = ba_win_sz; + + return 0; +} + +int ath12k_wifi7_dp_reo_cache_flush(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid) +{ + struct ath12k_hal_reo_cmd cmd = {}; + int ret; + + cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned); + cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned); + /* HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS - all pending MPDUs + *in the bitmap will be forwarded/flushed to REO output rings + */ + cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS | + HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS; + + /* For all QoS TIDs (except NON_QOS), the driver allocates a maximum + * window size of 1024. In such cases, the driver can issue a single + * 1KB descriptor flush command instead of sending multiple 128-byte + * flush commands for each QoS TID, improving efficiency. + */ + + if (rx_tid->tid != HAL_DESC_REO_NON_QOS_TID) + cmd.flag |= HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC; + + ret = ath12k_wifi7_dp_reo_cmd_send(ab, rx_tid, + HAL_REO_CMD_FLUSH_CACHE, + &cmd, ath12k_dp_reo_cmd_free); + return ret; +} + +int ath12k_wifi7_dp_rx_assign_reoq(struct ath12k_base *ab, struct ath12k_dp_peer *dp_peer, + struct ath12k_dp_rx_tid *rx_tid, + u16 ssn, enum hal_pn_type pn_type) +{ + u32 ba_win_sz = rx_tid->ba_win_sz; + struct ath12k_reoq_buf *buf; + void *vaddr, *vaddr_aligned; + dma_addr_t paddr_aligned; + u8 tid = rx_tid->tid; + u32 hw_desc_sz; + int ret; + + buf = &dp_peer->reoq_bufs[tid]; + if (!buf->vaddr) { + /* TODO: Optimize the memory allocation for qos tid based on + * the actual BA window size in REO tid update path. + */ + if (tid == HAL_DESC_REO_NON_QOS_TID) + hw_desc_sz = ath12k_wifi7_hal_reo_qdesc_size(ba_win_sz, tid); + else + hw_desc_sz = ath12k_wifi7_hal_reo_qdesc_size(DP_BA_WIN_SZ_MAX, + tid); + + vaddr = kzalloc(hw_desc_sz + HAL_LINK_DESC_ALIGN - 1, GFP_ATOMIC); + if (!vaddr) + return -ENOMEM; + + vaddr_aligned = PTR_ALIGN(vaddr, HAL_LINK_DESC_ALIGN); + + ath12k_wifi7_hal_reo_qdesc_setup(vaddr_aligned, tid, ba_win_sz, + ssn, pn_type); + + paddr_aligned = dma_map_single(ab->dev, vaddr_aligned, hw_desc_sz, + DMA_BIDIRECTIONAL); + ret = dma_mapping_error(ab->dev, paddr_aligned); + if (ret) { + kfree(vaddr); + return ret; + } + + buf->vaddr = vaddr; + buf->paddr_aligned = paddr_aligned; + buf->size = hw_desc_sz; + } + + rx_tid->qbuf = *buf; + + return 0; +} + +int ath12k_wifi7_dp_rx_tid_delete_handler(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid) +{ + struct ath12k_hal_reo_cmd cmd = {}; + + cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS; + cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned); + cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned); + cmd.upd0 |= HAL_REO_CMD_UPD0_VLD; + /* Observed flush cache failure, to avoid that set vld bit during delete */ + cmd.upd1 |= HAL_REO_CMD_UPD1_VLD; + + return ath12k_wifi7_dp_reo_cmd_send(ab, rx_tid, + HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd, + ath12k_dp_rx_tid_del_func); +} + +static void ath12k_wifi7_dp_rx_h_csum_offload(struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info) +{ + msdu->ip_summed = (rx_info->ip_csum_fail || rx_info->l4_csum_fail) ? + CHECKSUM_NONE : CHECKSUM_UNNECESSARY; +} + +static void ath12k_wifi7_dp_rx_h_mpdu(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc_data *rx_info) +{ + struct ath12k_skb_rxcb *rxcb; + enum hal_encrypt_type enctype; + bool is_decrypted = false; + struct ieee80211_hdr *hdr; + struct ath12k_dp_peer *peer; + struct ieee80211_rx_status *rx_status = rx_info->rx_status; + u32 err_bitmap = rx_info->err_bitmap; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "dp_rx_h_mpdu called without rcu lock"); + + /* PN for multicast packets will be checked in mac80211 */ + rxcb = ATH12K_SKB_RXCB(msdu); + rxcb->is_mcbc = rx_info->is_mcbc; + + if (rxcb->is_mcbc) + rxcb->peer_id = rx_info->peer_id; + + peer = ath12k_dp_peer_find_by_peerid(dp_pdev, rxcb->peer_id); + if (peer) { + /* resetting mcbc bit because mcbc packets are unicast + * packets only for AP as STA sends unicast packets. + */ + rxcb->is_mcbc = rxcb->is_mcbc && !peer->ucast_ra_only; + + if (rxcb->is_mcbc) + enctype = peer->sec_type_grp; + else + enctype = peer->sec_type; + } else { + enctype = HAL_ENCRYPT_TYPE_OPEN; + } + + if (enctype != HAL_ENCRYPT_TYPE_OPEN && !err_bitmap) + is_decrypted = rx_info->is_decrypted; + + /* Clear per-MPDU flags while leaving per-PPDU flags intact */ + rx_status->flag &= ~(RX_FLAG_FAILED_FCS_CRC | + RX_FLAG_MMIC_ERROR | + RX_FLAG_DECRYPTED | + RX_FLAG_IV_STRIPPED | + RX_FLAG_MMIC_STRIPPED); + + if (err_bitmap & HAL_RX_MPDU_ERR_FCS) + rx_status->flag |= RX_FLAG_FAILED_FCS_CRC; + if (err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC) + rx_status->flag |= RX_FLAG_MMIC_ERROR; + + if (is_decrypted) { + rx_status->flag |= RX_FLAG_DECRYPTED | RX_FLAG_MMIC_STRIPPED; + + if (rx_info->is_mcbc) + rx_status->flag |= RX_FLAG_MIC_STRIPPED | + RX_FLAG_ICV_STRIPPED; + else + rx_status->flag |= RX_FLAG_IV_STRIPPED | + RX_FLAG_PN_VALIDATED; + } + + ath12k_wifi7_dp_rx_h_csum_offload(msdu, rx_info); + ath12k_dp_rx_h_undecap(dp_pdev, msdu, rx_desc, + enctype, is_decrypted, rx_info); + + if (!is_decrypted || rx_info->is_mcbc) + return; + + if (rx_info->decap_type != DP_RX_DECAP_TYPE_ETHERNET2_DIX) { + hdr = (void *)msdu->data; + hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_PROTECTED); + } +} + +static int ath12k_wifi7_dp_rx_msdu_coalesce(struct ath12k_hal *hal, + struct sk_buff_head *msdu_list, + struct sk_buff *first, struct sk_buff *last, + u8 l3pad_bytes, int msdu_len, + struct hal_rx_desc_data *rx_info) +{ + struct sk_buff *skb; + struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(first); + int buf_first_hdr_len, buf_first_len; + struct hal_rx_desc *ldesc; + int space_extra, rem_len, buf_len; + u32 hal_rx_desc_sz = hal->hal_desc_sz; + bool is_continuation; + + /* As the msdu is spread across multiple rx buffers, + * find the offset to the start of msdu for computing + * the length of the msdu in the first buffer. + */ + buf_first_hdr_len = hal_rx_desc_sz + l3pad_bytes; + buf_first_len = DP_RX_BUFFER_SIZE - buf_first_hdr_len; + + if (WARN_ON_ONCE(msdu_len <= buf_first_len)) { + skb_put(first, buf_first_hdr_len + msdu_len); + skb_pull(first, buf_first_hdr_len); + return 0; + } + + ldesc = (struct hal_rx_desc *)last->data; + rxcb->is_first_msdu = rx_info->is_first_msdu; + rxcb->is_last_msdu = rx_info->is_last_msdu; + + /* MSDU spans over multiple buffers because the length of the MSDU + * exceeds DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE. So assume the data + * in the first buf is of length DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE. + */ + skb_put(first, DP_RX_BUFFER_SIZE); + skb_pull(first, buf_first_hdr_len); + + /* When an MSDU spread over multiple buffers MSDU_END + * tlvs are valid only in the last buffer. Copy those tlvs. + */ + ath12k_dp_rx_desc_end_tlv_copy(hal, rxcb->rx_desc, ldesc); + + space_extra = msdu_len - (buf_first_len + skb_tailroom(first)); + if (space_extra > 0 && + (pskb_expand_head(first, 0, space_extra, GFP_ATOMIC) < 0)) { + /* Free up all buffers of the MSDU */ + while ((skb = __skb_dequeue(msdu_list)) != NULL) { + rxcb = ATH12K_SKB_RXCB(skb); + if (!rxcb->is_continuation) { + dev_kfree_skb_any(skb); + break; + } + dev_kfree_skb_any(skb); + } + return -ENOMEM; + } + + rem_len = msdu_len - buf_first_len; + while ((skb = __skb_dequeue(msdu_list)) != NULL && rem_len > 0) { + rxcb = ATH12K_SKB_RXCB(skb); + is_continuation = rxcb->is_continuation; + if (is_continuation) + buf_len = DP_RX_BUFFER_SIZE - hal_rx_desc_sz; + else + buf_len = rem_len; + + if (buf_len > (DP_RX_BUFFER_SIZE - hal_rx_desc_sz)) { + WARN_ON_ONCE(1); + dev_kfree_skb_any(skb); + return -EINVAL; + } + + skb_put(skb, buf_len + hal_rx_desc_sz); + skb_pull(skb, hal_rx_desc_sz); + skb_copy_from_linear_data(skb, skb_put(first, buf_len), + buf_len); + dev_kfree_skb_any(skb); + + rem_len -= buf_len; + if (!is_continuation) + break; + } + + return 0; +} + +static int ath12k_wifi7_dp_rx_process_msdu(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, + struct sk_buff_head *msdu_list, + struct hal_rx_desc_data *rx_info) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct hal_rx_desc *rx_desc, *lrx_desc; + struct ath12k_skb_rxcb *rxcb; + struct sk_buff *last_buf; + struct ath12k_hal *hal = dp->hal; + u8 l3_pad_bytes; + u16 msdu_len; + int ret; + u32 hal_rx_desc_sz = hal->hal_desc_sz; + + last_buf = ath12k_dp_rx_get_msdu_last_buf(msdu_list, msdu); + if (!last_buf) { + ath12k_warn(dp->ab, + "No valid Rx buffer to access MSDU_END tlv\n"); + ret = -EIO; + goto free_out; + } + + rx_desc = (struct hal_rx_desc *)msdu->data; + lrx_desc = (struct hal_rx_desc *)last_buf->data; + + ath12k_dp_extract_rx_desc_data(hal, rx_info, rx_desc, lrx_desc); + if (!rx_info->msdu_done) { + ath12k_warn(dp->ab, "msdu_done bit in msdu_end is not set\n"); + ret = -EIO; + goto free_out; + } + + rxcb = ATH12K_SKB_RXCB(msdu); + rxcb->rx_desc = rx_desc; + msdu_len = rx_info->msdu_len; + l3_pad_bytes = rx_info->l3_pad_bytes; + + if (rxcb->is_frag) { + skb_pull(msdu, hal_rx_desc_sz); + } else if (!rxcb->is_continuation) { + if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) { + ret = -EINVAL; + ath12k_warn(dp->ab, "invalid msdu len %u\n", msdu_len); + ath12k_dbg_dump(dp->ab, ATH12K_DBG_DATA, NULL, "", rx_desc, + sizeof(*rx_desc)); + goto free_out; + } + skb_put(msdu, hal_rx_desc_sz + l3_pad_bytes + msdu_len); + skb_pull(msdu, hal_rx_desc_sz + l3_pad_bytes); + } else { + ret = ath12k_wifi7_dp_rx_msdu_coalesce(hal, msdu_list, + msdu, last_buf, + l3_pad_bytes, msdu_len, + rx_info); + if (ret) { + ath12k_warn(dp->ab, + "failed to coalesce msdu rx buffer%d\n", ret); + goto free_out; + } + } + + if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, rx_desc, msdu, + rx_info))) { + ret = -EINVAL; + goto free_out; + } + + ath12k_dp_rx_h_ppdu(dp_pdev, rx_info); + ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, rx_desc, rx_info); + + rx_info->rx_status->flag |= RX_FLAG_SKIP_MONITOR | RX_FLAG_DUP_VALIDATED; + + return 0; + +free_out: + return ret; +} + +static void +ath12k_wifi7_dp_rx_process_received_packets(struct ath12k_dp *dp, + struct napi_struct *napi, + struct sk_buff_head *msdu_list, + int ring_id) +{ + struct ath12k_hw_group *ag = dp->ag; + struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; + struct ieee80211_rx_status rx_status = {}; + struct ath12k_skb_rxcb *rxcb; + struct sk_buff *msdu; + struct ath12k *ar; + struct ath12k_pdev_dp *dp_pdev; + struct ath12k_hw_link *hw_links = ag->hw_links; + struct ath12k_base *partner_ab; + struct hal_rx_desc_data rx_info; + struct ath12k_dp *partner_dp; + u8 hw_link_id, pdev_idx; + int ret; + + if (skb_queue_empty(msdu_list)) + return; + + rx_info.addr2_present = false; + rx_info.rx_status = &rx_status; + + rcu_read_lock(); + + while ((msdu = __skb_dequeue(msdu_list))) { + rxcb = ATH12K_SKB_RXCB(msdu); + hw_link_id = rxcb->hw_link_id; + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, + hw_links[hw_link_id].device_id); + pdev_idx = ath12k_hw_mac_id_to_pdev_id(partner_dp->hw_params, + hw_links[hw_link_id].pdev_idx); + partner_ab = partner_dp->ab; + ar = partner_ab->pdevs[pdev_idx].ar; + if (!rcu_dereference(partner_ab->pdevs_active[pdev_idx])) { + dev_kfree_skb_any(msdu); + continue; + } + + if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) { + dev_kfree_skb_any(msdu); + continue; + } + + dp_pdev = ath12k_dp_to_pdev_dp(partner_dp, pdev_idx); + if (!dp_pdev) { + dev_kfree_skb_any(msdu); + continue; + } + + ret = ath12k_wifi7_dp_rx_process_msdu(dp_pdev, msdu, msdu_list, &rx_info); + if (ret) { + ath12k_dbg(dp->ab, ATH12K_DBG_DATA, + "Unable to process msdu %d", ret); + dev_kfree_skb_any(msdu); + continue; + } + + ath12k_dp_rx_deliver_msdu(dp_pdev, napi, msdu, &rx_info); + } + + rcu_read_unlock(); +} + +int ath12k_wifi7_dp_rx_process(struct ath12k_dp *dp, int ring_id, + struct napi_struct *napi, int budget) +{ + struct ath12k_hw_group *ag = dp->ag; + struct ath12k_base *ab = dp->ab; + struct ath12k_hal *hal = dp->hal; + struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; + struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES]; + struct ath12k_hw_link *hw_links = ag->hw_links; + int num_buffs_reaped[ATH12K_MAX_DEVICES] = {}; + struct ath12k_rx_desc_info *desc_info; + struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; + struct hal_reo_dest_ring *desc; + struct ath12k_dp *partner_dp; + struct sk_buff_head msdu_list; + struct ath12k_skb_rxcb *rxcb; + int total_msdu_reaped = 0; + u8 hw_link_id, device_id; + struct hal_srng *srng; + struct sk_buff *msdu; + bool done = false; + u64 desc_va; + + __skb_queue_head_init(&msdu_list); + + for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) + INIT_LIST_HEAD(&rx_desc_used_list[device_id]); + + srng = &hal->srng_list[dp->reo_dst_ring[ring_id].ring_id]; + + spin_lock_bh(&srng->lock); + +try_again: + ath12k_hal_srng_access_begin(ab, srng); + + while ((desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) { + struct rx_mpdu_desc *mpdu_info; + struct rx_msdu_desc *msdu_info; + enum hal_reo_dest_ring_push_reason push_reason; + u32 cookie; + + cookie = le32_get_bits(desc->buf_addr_info.info1, + BUFFER_ADDR_INFO1_SW_COOKIE); + + hw_link_id = le32_get_bits(desc->info0, + HAL_REO_DEST_RING_INFO0_SRC_LINK_ID); + + desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 | + le32_to_cpu(desc->buf_va_lo)); + desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va); + + device_id = hw_links[hw_link_id].device_id; + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id); + if (unlikely(!partner_dp)) { + if (desc_info->skb) { + dev_kfree_skb_any(desc_info->skb); + desc_info->skb = NULL; + } + + continue; + } + + /* retry manual desc retrieval */ + if (!desc_info) { + desc_info = ath12k_dp_get_rx_desc(partner_dp, cookie); + if (!desc_info) { + ath12k_warn(partner_dp->ab, "Invalid cookie in manual descriptor retrieval: 0x%x\n", + cookie); + continue; + } + } + + if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) + ath12k_warn(ab, "Check HW CC implementation"); + + msdu = desc_info->skb; + desc_info->skb = NULL; + + list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]); + + rxcb = ATH12K_SKB_RXCB(msdu); + dma_unmap_single(partner_dp->dev, rxcb->paddr, + msdu->len + skb_tailroom(msdu), + DMA_FROM_DEVICE); + + num_buffs_reaped[device_id]++; + dp->device_stats.reo_rx[ring_id][dp->device_id]++; + + push_reason = le32_get_bits(desc->info0, + HAL_REO_DEST_RING_INFO0_PUSH_REASON); + if (push_reason != + HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) { + dev_kfree_skb_any(msdu); + dp->device_stats.hal_reo_error[ring_id]++; + continue; + } + + msdu_info = &desc->rx_msdu_info; + mpdu_info = &desc->rx_mpdu_info; + + rxcb->is_first_msdu = !!(le32_to_cpu(msdu_info->info0) & + RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU); + rxcb->is_last_msdu = !!(le32_to_cpu(msdu_info->info0) & + RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU); + rxcb->is_continuation = !!(le32_to_cpu(msdu_info->info0) & + RX_MSDU_DESC_INFO0_MSDU_CONTINUATION); + rxcb->hw_link_id = hw_link_id; + rxcb->peer_id = ath12k_wifi7_dp_rx_get_peer_id(dp, dp->peer_metadata_ver, + mpdu_info->peer_meta_data); + rxcb->tid = le32_get_bits(mpdu_info->info0, + RX_MPDU_DESC_INFO0_TID); + + __skb_queue_tail(&msdu_list, msdu); + + if (!rxcb->is_continuation) { + total_msdu_reaped++; + done = true; + } else { + done = false; + } + + if (total_msdu_reaped >= budget) + break; + } + + /* Hw might have updated the head pointer after we cached it. + * In this case, even though there are entries in the ring we'll + * get rx_desc NULL. Give the read another try with updated cached + * head pointer so that we can reap complete MPDU in the current + * rx processing. + */ + if (!done && ath12k_hal_srng_dst_num_free(ab, srng, true)) { + ath12k_hal_srng_access_end(ab, srng); + goto try_again; + } + + ath12k_hal_srng_access_end(ab, srng); + + spin_unlock_bh(&srng->lock); + + if (!total_msdu_reaped) + goto exit; + + for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) { + if (!num_buffs_reaped[device_id]) + continue; + + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id); + rx_ring = &partner_dp->rx_refill_buf_ring; + + ath12k_dp_rx_bufs_replenish(partner_dp, rx_ring, + &rx_desc_used_list[device_id], + num_buffs_reaped[device_id]); + } + + ath12k_wifi7_dp_rx_process_received_packets(dp, napi, &msdu_list, + ring_id); + +exit: + return total_msdu_reaped; +} + +static bool +ath12k_wifi7_dp_rx_h_defrag_validate_incr_pn(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_dp_rx_tid *rx_tid, + enum hal_encrypt_type encrypt_type) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct sk_buff *first_frag, *skb; + u64 last_pn; + u64 cur_pn; + + first_frag = skb_peek(&rx_tid->rx_frags); + + if (encrypt_type != HAL_ENCRYPT_TYPE_CCMP_128 && + encrypt_type != HAL_ENCRYPT_TYPE_CCMP_256 && + encrypt_type != HAL_ENCRYPT_TYPE_GCMP_128 && + encrypt_type != HAL_ENCRYPT_TYPE_AES_GCMP_256) + return true; + + last_pn = ath12k_dp_rx_h_get_pn(dp, first_frag); + skb_queue_walk(&rx_tid->rx_frags, skb) { + if (skb == first_frag) + continue; + + cur_pn = ath12k_dp_rx_h_get_pn(dp, skb); + if (cur_pn != last_pn + 1) + return false; + last_pn = cur_pn; + } + return true; +} + +static int ath12k_wifi7_dp_rx_h_defrag_reo_reinject(struct ath12k_dp *dp, + struct ath12k_dp_rx_tid *rx_tid, + struct sk_buff *defrag_skb) +{ + struct ath12k_base *ab = dp->ab; + struct ath12k_hal *hal = dp->hal; + struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)defrag_skb->data; + struct hal_reo_entrance_ring *reo_ent_ring; + struct hal_reo_dest_ring *reo_dest_ring; + struct dp_link_desc_bank *link_desc_banks; + struct hal_rx_msdu_link *msdu_link; + struct hal_rx_msdu_details *msdu0; + struct hal_srng *srng; + dma_addr_t link_paddr, buf_paddr; + u32 desc_bank, msdu_info, msdu_ext_info, mpdu_info; + u32 cookie, hal_rx_desc_sz, dest_ring_info0, queue_addr_hi; + int ret; + struct ath12k_rx_desc_info *desc_info; + enum hal_rx_buf_return_buf_manager idle_link_rbm = dp->idle_link_rbm; + u8 dst_ind; + + hal_rx_desc_sz = hal->hal_desc_sz; + link_desc_banks = dp->link_desc_banks; + reo_dest_ring = rx_tid->dst_ring_desc; + + ath12k_wifi7_hal_rx_reo_ent_paddr_get(&reo_dest_ring->buf_addr_info, + &link_paddr, &cookie); + desc_bank = u32_get_bits(cookie, DP_LINK_DESC_BANK_MASK); + + msdu_link = (struct hal_rx_msdu_link *)(link_desc_banks[desc_bank].vaddr + + (link_paddr - link_desc_banks[desc_bank].paddr)); + msdu0 = &msdu_link->msdu_link[0]; + msdu_ext_info = le32_to_cpu(msdu0->rx_msdu_ext_info.info0); + dst_ind = u32_get_bits(msdu_ext_info, RX_MSDU_EXT_DESC_INFO0_REO_DEST_IND); + + memset(msdu0, 0, sizeof(*msdu0)); + + msdu_info = u32_encode_bits(1, RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU) | + u32_encode_bits(1, RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU) | + u32_encode_bits(0, RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) | + u32_encode_bits(defrag_skb->len - hal_rx_desc_sz, + RX_MSDU_DESC_INFO0_MSDU_LENGTH) | + u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_SA) | + u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_DA); + msdu0->rx_msdu_info.info0 = cpu_to_le32(msdu_info); + msdu0->rx_msdu_ext_info.info0 = cpu_to_le32(msdu_ext_info); + + /* change msdu len in hal rx desc */ + ath12k_dp_rxdesc_set_msdu_len(hal, rx_desc, defrag_skb->len - hal_rx_desc_sz); + + buf_paddr = dma_map_single(dp->dev, defrag_skb->data, + defrag_skb->len + skb_tailroom(defrag_skb), + DMA_TO_DEVICE); + if (dma_mapping_error(dp->dev, buf_paddr)) + return -ENOMEM; + + spin_lock_bh(&dp->rx_desc_lock); + desc_info = list_first_entry_or_null(&dp->rx_desc_free_list, + struct ath12k_rx_desc_info, + list); + if (!desc_info) { + spin_unlock_bh(&dp->rx_desc_lock); + ath12k_warn(ab, "failed to find rx desc for reinject\n"); + ret = -ENOMEM; + goto err_unmap_dma; + } + + desc_info->skb = defrag_skb; + desc_info->in_use = true; + + list_del(&desc_info->list); + spin_unlock_bh(&dp->rx_desc_lock); + + ATH12K_SKB_RXCB(defrag_skb)->paddr = buf_paddr; + + ath12k_wifi7_hal_rx_buf_addr_info_set(&msdu0->buf_addr_info, buf_paddr, + desc_info->cookie, + HAL_RX_BUF_RBM_SW3_BM); + + /* Fill mpdu details into reo entrance ring */ + srng = &hal->srng_list[dp->reo_reinject_ring.ring_id]; + + spin_lock_bh(&srng->lock); + ath12k_hal_srng_access_begin(ab, srng); + + reo_ent_ring = ath12k_hal_srng_src_get_next_entry(ab, srng); + if (!reo_ent_ring) { + ath12k_hal_srng_access_end(ab, srng); + spin_unlock_bh(&srng->lock); + ret = -ENOSPC; + goto err_free_desc; + } + memset(reo_ent_ring, 0, sizeof(*reo_ent_ring)); + + ath12k_wifi7_hal_rx_buf_addr_info_set(&reo_ent_ring->buf_addr_info, link_paddr, + cookie, idle_link_rbm); + + mpdu_info = u32_encode_bits(1, RX_MPDU_DESC_INFO0_MSDU_COUNT) | + u32_encode_bits(0, RX_MPDU_DESC_INFO0_FRAG_FLAG) | + u32_encode_bits(1, RX_MPDU_DESC_INFO0_RAW_MPDU) | + u32_encode_bits(1, RX_MPDU_DESC_INFO0_VALID_PN) | + u32_encode_bits(rx_tid->tid, RX_MPDU_DESC_INFO0_TID); + + reo_ent_ring->rx_mpdu_info.info0 = cpu_to_le32(mpdu_info); + reo_ent_ring->rx_mpdu_info.peer_meta_data = + reo_dest_ring->rx_mpdu_info.peer_meta_data; + + if (dp->hw_params->reoq_lut_support) { + reo_ent_ring->queue_addr_lo = reo_dest_ring->rx_mpdu_info.peer_meta_data; + queue_addr_hi = 0; + } else { + reo_ent_ring->queue_addr_lo = + cpu_to_le32(lower_32_bits(rx_tid->qbuf.paddr_aligned)); + queue_addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned); + } + + reo_ent_ring->info0 = le32_encode_bits(queue_addr_hi, + HAL_REO_ENTR_RING_INFO0_QUEUE_ADDR_HI) | + le32_encode_bits(dst_ind, + HAL_REO_ENTR_RING_INFO0_DEST_IND); + + reo_ent_ring->info1 = le32_encode_bits(rx_tid->cur_sn, + HAL_REO_ENTR_RING_INFO1_MPDU_SEQ_NUM); + dest_ring_info0 = le32_get_bits(reo_dest_ring->info0, + HAL_REO_DEST_RING_INFO0_SRC_LINK_ID); + reo_ent_ring->info2 = + cpu_to_le32(u32_get_bits(dest_ring_info0, + HAL_REO_ENTR_RING_INFO2_SRC_LINK_ID)); + + ath12k_hal_srng_access_end(ab, srng); + spin_unlock_bh(&srng->lock); + + return 0; + +err_free_desc: + spin_lock_bh(&dp->rx_desc_lock); + desc_info->in_use = false; + desc_info->skb = NULL; + list_add_tail(&desc_info->list, &dp->rx_desc_free_list); + spin_unlock_bh(&dp->rx_desc_lock); +err_unmap_dma: + dma_unmap_single(dp->dev, buf_paddr, defrag_skb->len + skb_tailroom(defrag_skb), + DMA_TO_DEVICE); + return ret; +} + +static int ath12k_wifi7_dp_rx_h_verify_tkip_mic(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_dp_peer *peer, + enum hal_encrypt_type enctype, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_hal *hal = dp->hal; + struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data; + struct ieee80211_rx_status *rxs = IEEE80211_SKB_RXCB(msdu); + struct ieee80211_key_conf *key_conf; + struct ieee80211_hdr *hdr; + u8 mic[IEEE80211_CCMP_MIC_LEN]; + int head_len, tail_len, ret; + size_t data_len; + u32 hdr_len, hal_rx_desc_sz = hal->hal_desc_sz; + u8 *key, *data; + u8 key_idx; + + if (enctype != HAL_ENCRYPT_TYPE_TKIP_MIC) + return 0; + + rx_info->addr2_present = false; + rx_info->rx_status = rxs; + + hdr = (struct ieee80211_hdr *)(msdu->data + hal_rx_desc_sz); + hdr_len = ieee80211_hdrlen(hdr->frame_control); + head_len = hdr_len + hal_rx_desc_sz + IEEE80211_TKIP_IV_LEN; + tail_len = IEEE80211_CCMP_MIC_LEN + IEEE80211_TKIP_ICV_LEN + FCS_LEN; + + if (!is_multicast_ether_addr(hdr->addr1)) + key_idx = peer->ucast_keyidx; + else + key_idx = peer->mcast_keyidx; + + key_conf = peer->keys[key_idx]; + + data = msdu->data + head_len; + data_len = msdu->len - head_len - tail_len; + key = &key_conf->key[NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY]; + + ret = ath12k_dp_rx_h_michael_mic(peer->tfm_mmic, key, hdr, data, + data_len, mic); + if (ret || memcmp(mic, data + data_len, IEEE80211_CCMP_MIC_LEN)) + goto mic_fail; + + return 0; + +mic_fail: + (ATH12K_SKB_RXCB(msdu))->is_first_msdu = true; + (ATH12K_SKB_RXCB(msdu))->is_last_msdu = true; + + ath12k_dp_extract_rx_desc_data(hal, rx_info, rx_desc, rx_desc); + + rxs->flag |= RX_FLAG_MMIC_ERROR | RX_FLAG_MMIC_STRIPPED | + RX_FLAG_IV_STRIPPED | RX_FLAG_DECRYPTED; + skb_pull(msdu, hal_rx_desc_sz); + + if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, rx_desc, msdu, + rx_info))) { + dev_kfree_skb_any(msdu); + return -EINVAL; + } + + ath12k_dp_rx_h_ppdu(dp_pdev, rx_info); + ath12k_dp_rx_h_undecap(dp_pdev, msdu, rx_desc, + HAL_ENCRYPT_TYPE_TKIP_MIC, true, rx_info); + ieee80211_rx(ath12k_pdev_dp_to_hw(dp_pdev), msdu); + return -EINVAL; +} + +static int ath12k_wifi7_dp_rx_h_defrag(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_dp_peer *peer, + struct ath12k_dp_rx_tid *rx_tid, + struct sk_buff **defrag_skb, + enum hal_encrypt_type enctype, + struct hal_rx_desc_data *rx_info) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; + struct sk_buff *skb, *first_frag, *last_frag; + struct ieee80211_hdr *hdr; + bool is_decrypted = false; + int msdu_len = 0; + int extra_space; + u32 flags, hal_rx_desc_sz = ab->hal.hal_desc_sz; + + first_frag = skb_peek(&rx_tid->rx_frags); + last_frag = skb_peek_tail(&rx_tid->rx_frags); + + skb_queue_walk(&rx_tid->rx_frags, skb) { + flags = 0; + hdr = (struct ieee80211_hdr *)(skb->data + hal_rx_desc_sz); + + if (enctype != HAL_ENCRYPT_TYPE_OPEN) + is_decrypted = rx_info->is_decrypted; + + if (is_decrypted) { + if (skb != first_frag) + flags |= RX_FLAG_IV_STRIPPED; + if (skb != last_frag) + flags |= RX_FLAG_ICV_STRIPPED | + RX_FLAG_MIC_STRIPPED; + } + + /* RX fragments are always raw packets */ + if (skb != last_frag) + skb_trim(skb, skb->len - FCS_LEN); + ath12k_dp_rx_h_undecap_frag(dp_pdev, skb, enctype, flags); + + if (skb != first_frag) + skb_pull(skb, hal_rx_desc_sz + + ieee80211_hdrlen(hdr->frame_control)); + msdu_len += skb->len; + } + + extra_space = msdu_len - (DP_RX_BUFFER_SIZE + skb_tailroom(first_frag)); + if (extra_space > 0 && + (pskb_expand_head(first_frag, 0, extra_space, GFP_ATOMIC) < 0)) + return -ENOMEM; + + __skb_unlink(first_frag, &rx_tid->rx_frags); + while ((skb = __skb_dequeue(&rx_tid->rx_frags))) { + skb_put_data(first_frag, skb->data, skb->len); + dev_kfree_skb_any(skb); + } + + hdr = (struct ieee80211_hdr *)(first_frag->data + hal_rx_desc_sz); + hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_MOREFRAGS); + ATH12K_SKB_RXCB(first_frag)->is_frag = 1; + + if (ath12k_wifi7_dp_rx_h_verify_tkip_mic(dp_pdev, peer, enctype, first_frag, + rx_info)) + first_frag = NULL; + + *defrag_skb = first_frag; + return 0; +} + +void ath12k_wifi7_dp_rx_frags_cleanup(struct ath12k_dp_rx_tid *rx_tid, + bool rel_link_desc) +{ + enum hal_wbm_rel_bm_act act = HAL_WBM_REL_BM_ACT_PUT_IN_IDLE; + struct ath12k_buffer_addr *buf_addr_info; + struct ath12k_dp *dp = rx_tid->dp; + + lockdep_assert_held(&dp->dp_lock); + + if (rx_tid->dst_ring_desc) { + if (rel_link_desc) { + buf_addr_info = &rx_tid->dst_ring_desc->buf_addr_info; + ath12k_wifi7_dp_rx_link_desc_return(dp, buf_addr_info, act); + } + kfree(rx_tid->dst_ring_desc); + rx_tid->dst_ring_desc = NULL; + } + + rx_tid->cur_sn = 0; + rx_tid->last_frag_no = 0; + rx_tid->rx_frag_bitmap = 0; + __skb_queue_purge(&rx_tid->rx_frags); +} + +static int ath12k_wifi7_dp_rx_frag_h_mpdu(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, + struct hal_reo_dest_ring *ring_desc, + struct hal_rx_desc_data *rx_info) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_hal *hal = dp->hal; + struct ath12k_base *ab = dp->ab; + struct ath12k_dp_peer *peer; + struct ath12k_dp_rx_tid *rx_tid; + struct sk_buff *defrag_skb = NULL; + u32 peer_id = rx_info->peer_id; + u16 seqno, frag_no; + u8 tid = rx_info->tid; + int ret = 0; + bool more_frags; + enum hal_encrypt_type enctype = rx_info->enctype; + + frag_no = ath12k_dp_rx_h_frag_no(hal, msdu); + more_frags = ath12k_dp_rx_h_more_frags(hal, msdu); + seqno = rx_info->seq_no; + + if (!rx_info->seq_ctl_valid || !rx_info->fc_valid || + tid > IEEE80211_NUM_TIDS) + return -EINVAL; + + /* received unfragmented packet in reo + * exception ring, this shouldn't happen + * as these packets typically come from + * reo2sw srngs. + */ + if (WARN_ON_ONCE(!frag_no && !more_frags)) + return -EINVAL; + + spin_lock_bh(&dp->dp_lock); + peer = ath12k_dp_peer_find_by_peerid(dp_pdev, peer_id); + if (!peer) { + ath12k_warn(ab, "failed to find the peer to de-fragment received fragment peer_id %d\n", + peer_id); + ret = -ENOENT; + goto out_unlock; + } + + if (!peer->dp_setup_done) { + ath12k_warn(ab, "The peer %pM [%d] has uninitialized datapath\n", + peer->addr, peer_id); + ret = -ENOENT; + goto out_unlock; + } + + rx_tid = &peer->rx_tid[tid]; + + if ((!skb_queue_empty(&rx_tid->rx_frags) && seqno != rx_tid->cur_sn) || + skb_queue_empty(&rx_tid->rx_frags)) { + /* Flush stored fragments and start a new sequence */ + ath12k_wifi7_dp_rx_frags_cleanup(rx_tid, true); + rx_tid->cur_sn = seqno; + } + + if (rx_tid->rx_frag_bitmap & BIT(frag_no)) { + /* Fragment already present */ + ret = -EINVAL; + goto out_unlock; + } + + if ((!rx_tid->rx_frag_bitmap || frag_no > __fls(rx_tid->rx_frag_bitmap))) + __skb_queue_tail(&rx_tid->rx_frags, msdu); + else + ath12k_dp_rx_h_sort_frags(hal, &rx_tid->rx_frags, msdu); + + rx_tid->rx_frag_bitmap |= BIT(frag_no); + if (!more_frags) + rx_tid->last_frag_no = frag_no; + + if (frag_no == 0) { + rx_tid->dst_ring_desc = kmemdup(ring_desc, + sizeof(*rx_tid->dst_ring_desc), + GFP_ATOMIC); + if (!rx_tid->dst_ring_desc) { + ret = -ENOMEM; + goto out_unlock; + } + } else { + ath12k_wifi7_dp_rx_link_desc_return(dp, &ring_desc->buf_addr_info, + HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); + } + + if (!rx_tid->last_frag_no || + rx_tid->rx_frag_bitmap != GENMASK(rx_tid->last_frag_no, 0)) { + mod_timer(&rx_tid->frag_timer, jiffies + + ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS); + goto out_unlock; + } + + spin_unlock_bh(&dp->dp_lock); + timer_delete_sync(&rx_tid->frag_timer); + spin_lock_bh(&dp->dp_lock); + + peer = ath12k_dp_peer_find_by_peerid(dp_pdev, peer_id); + if (!peer) + goto err_frags_cleanup; + + if (!ath12k_wifi7_dp_rx_h_defrag_validate_incr_pn(dp_pdev, rx_tid, enctype)) + goto err_frags_cleanup; + + if (ath12k_wifi7_dp_rx_h_defrag(dp_pdev, peer, rx_tid, &defrag_skb, + enctype, rx_info)) + goto err_frags_cleanup; + + if (!defrag_skb) + goto err_frags_cleanup; + + if (ath12k_wifi7_dp_rx_h_defrag_reo_reinject(dp, rx_tid, defrag_skb)) + goto err_frags_cleanup; + + ath12k_wifi7_dp_rx_frags_cleanup(rx_tid, false); + goto out_unlock; + +err_frags_cleanup: + dev_kfree_skb_any(defrag_skb); + ath12k_wifi7_dp_rx_frags_cleanup(rx_tid, true); +out_unlock: + spin_unlock_bh(&dp->dp_lock); + return ret; +} + +static int +ath12k_wifi7_dp_process_rx_err_buf(struct ath12k_pdev_dp *dp_pdev, + struct hal_reo_dest_ring *desc, + struct list_head *used_list, + bool drop, u32 cookie) +{ + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_hal *hal = dp->hal; + struct sk_buff *msdu; + struct ath12k_skb_rxcb *rxcb; + struct hal_rx_desc_data rx_info; + struct hal_rx_desc *rx_desc; + u16 msdu_len; + u32 hal_rx_desc_sz = hal->hal_desc_sz; + struct ath12k_rx_desc_info *desc_info; + u64 desc_va; + + desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 | + le32_to_cpu(desc->buf_va_lo)); + desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va); + + /* retry manual desc retrieval */ + if (!desc_info) { + desc_info = ath12k_dp_get_rx_desc(dp, cookie); + if (!desc_info) { + ath12k_warn(dp->ab, + "Invalid cookie in DP rx error descriptor retrieval: 0x%x\n", + cookie); + return -EINVAL; + } + } + + if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) + ath12k_warn(dp->ab, "RX Exception, Check HW CC implementation"); + + msdu = desc_info->skb; + desc_info->skb = NULL; + + list_add_tail(&desc_info->list, used_list); + + rxcb = ATH12K_SKB_RXCB(msdu); + dma_unmap_single(dp->dev, rxcb->paddr, + msdu->len + skb_tailroom(msdu), + DMA_FROM_DEVICE); + + if (drop) { + dev_kfree_skb_any(msdu); + return 0; + } + + rcu_read_lock(); + if (!rcu_dereference(ar->ab->pdevs_active[ar->pdev_idx])) { + dev_kfree_skb_any(msdu); + goto exit; + } + + if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) { + dev_kfree_skb_any(msdu); + goto exit; + } + + rx_desc = (struct hal_rx_desc *)msdu->data; + ath12k_dp_extract_rx_desc_data(hal, &rx_info, rx_desc, rx_desc); + + msdu_len = rx_info.msdu_len; + if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) { + ath12k_warn(dp->ab, "invalid msdu leng %u", msdu_len); + ath12k_dbg_dump(dp->ab, ATH12K_DBG_DATA, NULL, "", rx_desc, + sizeof(*rx_desc)); + dev_kfree_skb_any(msdu); + goto exit; + } + + skb_put(msdu, hal_rx_desc_sz + msdu_len); + + if (ath12k_wifi7_dp_rx_frag_h_mpdu(dp_pdev, msdu, desc, &rx_info)) { + dev_kfree_skb_any(msdu); + ath12k_wifi7_dp_rx_link_desc_return(dp, &desc->buf_addr_info, + HAL_WBM_REL_BM_ACT_PUT_IN_IDLE); + } +exit: + rcu_read_unlock(); + return 0; +} + +static int ath12k_dp_h_msdu_buffer_type(struct ath12k_dp *dp, + struct list_head *list, + struct hal_reo_dest_ring *desc) +{ + struct ath12k_rx_desc_info *desc_info; + struct ath12k_skb_rxcb *rxcb; + struct sk_buff *msdu; + u64 desc_va; + + dp->device_stats.reo_excep_msdu_buf_type++; + + desc_va = (u64)le32_to_cpu(desc->buf_va_hi) << 32 | + le32_to_cpu(desc->buf_va_lo); + desc_info = (struct ath12k_rx_desc_info *)(uintptr_t)desc_va; + if (!desc_info) { + u32 cookie; + + cookie = le32_get_bits(desc->buf_addr_info.info1, + BUFFER_ADDR_INFO1_SW_COOKIE); + desc_info = ath12k_dp_get_rx_desc(dp, cookie); + if (!desc_info) { + ath12k_warn(dp->ab, "Invalid cookie in manual descriptor retrieval: 0x%x\n", + cookie); + return -EINVAL; + } + } + + if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) { + ath12k_warn(dp->ab, "rx exception, magic check failed with value: %u\n", + desc_info->magic); + return -EINVAL; + } + + msdu = desc_info->skb; + desc_info->skb = NULL; + list_add_tail(&desc_info->list, list); + rxcb = ATH12K_SKB_RXCB(msdu); + dma_unmap_single(dp->dev, rxcb->paddr, msdu->len + skb_tailroom(msdu), + DMA_FROM_DEVICE); + dev_kfree_skb_any(msdu); + + return 0; +} + +int ath12k_wifi7_dp_rx_process_err(struct ath12k_dp *dp, struct napi_struct *napi, + int budget) +{ + struct ath12k_base *ab = dp->ab; + struct ath12k_hal *hal = dp->hal; + struct ath12k_hw_group *ag = dp->ag; + struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; + struct ath12k_dp *partner_dp; + struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES]; + u32 msdu_cookies[HAL_NUM_RX_MSDUS_PER_LINK_DESC]; + int num_buffs_reaped[ATH12K_MAX_DEVICES] = {}; + struct dp_link_desc_bank *link_desc_banks; + enum hal_rx_buf_return_buf_manager rbm; + struct hal_rx_msdu_link *link_desc_va; + int tot_n_bufs_reaped, quota, ret, i; + struct hal_reo_dest_ring *reo_desc; + struct dp_rxdma_ring *rx_ring; + struct dp_srng *reo_except; + struct ath12k_hw_link *hw_links = ag->hw_links; + struct ath12k_pdev_dp *dp_pdev; + u8 hw_link_id, device_id; + u32 desc_bank, num_msdus; + struct hal_srng *srng; + dma_addr_t paddr; + bool is_frag; + bool drop; + int pdev_idx; + struct list_head *used_list; + enum hal_wbm_rel_bm_act act; + + tot_n_bufs_reaped = 0; + quota = budget; + + for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) + INIT_LIST_HEAD(&rx_desc_used_list[device_id]); + + reo_except = &dp->reo_except_ring; + + srng = &hal->srng_list[reo_except->ring_id]; + + spin_lock_bh(&srng->lock); + + ath12k_hal_srng_access_begin(ab, srng); + + while (budget && + (reo_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) { + drop = false; + dp->device_stats.err_ring_pkts++; + + hw_link_id = le32_get_bits(reo_desc->info0, + HAL_REO_DEST_RING_INFO0_SRC_LINK_ID); + device_id = hw_links[hw_link_id].device_id; + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id); + + /* Below case is added to handle data packet from un-associated clients. + * As it is expected that AST lookup will fail for + * un-associated station's data packets. + */ + if (le32_get_bits(reo_desc->info0, HAL_REO_DEST_RING_INFO0_BUFFER_TYPE) == + HAL_REO_DEST_RING_BUFFER_TYPE_MSDU) { + if (!ath12k_dp_h_msdu_buffer_type(partner_dp, + &rx_desc_used_list[device_id], + reo_desc)) { + num_buffs_reaped[device_id]++; + tot_n_bufs_reaped++; + } + goto next_desc; + } + + ret = ath12k_wifi7_hal_desc_reo_parse_err(dp, reo_desc, &paddr, + &desc_bank); + if (ret) { + ath12k_warn(ab, "failed to parse error reo desc %d\n", + ret); + continue; + } + + pdev_idx = ath12k_hw_mac_id_to_pdev_id(partner_dp->hw_params, + hw_links[hw_link_id].pdev_idx); + + link_desc_banks = partner_dp->link_desc_banks; + link_desc_va = link_desc_banks[desc_bank].vaddr + + (paddr - link_desc_banks[desc_bank].paddr); + ath12k_wifi7_hal_rx_msdu_link_info_get(link_desc_va, &num_msdus, + msdu_cookies, &rbm); + if (rbm != partner_dp->idle_link_rbm && + rbm != HAL_RX_BUF_RBM_SW3_BM && + rbm != partner_dp->hal->hal_params->rx_buf_rbm) { + act = HAL_WBM_REL_BM_ACT_REL_MSDU; + dp->device_stats.invalid_rbm++; + ath12k_warn(ab, "invalid return buffer manager %d\n", rbm); + ath12k_wifi7_dp_rx_link_desc_return(partner_dp, + &reo_desc->buf_addr_info, + act); + continue; + } + + is_frag = !!(le32_to_cpu(reo_desc->rx_mpdu_info.info0) & + RX_MPDU_DESC_INFO0_FRAG_FLAG); + + /* Process only rx fragments with one msdu per link desc below, and drop + * msdu's indicated due to error reasons. + * Dynamic fragmentation not supported in Multi-link client, so drop the + * partner device buffers. + */ + if (!is_frag || num_msdus > 1 || + partner_dp->device_id != dp->device_id) { + drop = true; + act = HAL_WBM_REL_BM_ACT_PUT_IN_IDLE; + + /* Return the link desc back to wbm idle list */ + ath12k_wifi7_dp_rx_link_desc_return(partner_dp, + &reo_desc->buf_addr_info, + act); + } + + rcu_read_lock(); + + dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx); + if (!dp_pdev) { + rcu_read_unlock(); + continue; + } + + for (i = 0; i < num_msdus; i++) { + used_list = &rx_desc_used_list[device_id]; + + if (!ath12k_wifi7_dp_process_rx_err_buf(dp_pdev, reo_desc, + used_list, + drop, + msdu_cookies[i])) { + num_buffs_reaped[device_id]++; + tot_n_bufs_reaped++; + } + } + + rcu_read_unlock(); + +next_desc: + if (tot_n_bufs_reaped >= quota) { + tot_n_bufs_reaped = quota; + goto exit; + } + + budget = quota - tot_n_bufs_reaped; + } + +exit: + ath12k_hal_srng_access_end(ab, srng); + + spin_unlock_bh(&srng->lock); + + for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) { + if (!num_buffs_reaped[device_id]) + continue; + + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id); + rx_ring = &partner_dp->rx_refill_buf_ring; + + ath12k_dp_rx_bufs_replenish(partner_dp, rx_ring, + &rx_desc_used_list[device_id], + num_buffs_reaped[device_id]); + } + + return tot_n_bufs_reaped; +} + +static void +ath12k_wifi7_dp_rx_null_q_desc_sg_drop(struct ath12k_dp *dp, int msdu_len, + struct sk_buff_head *msdu_list) +{ + struct sk_buff *skb, *tmp; + struct ath12k_skb_rxcb *rxcb; + int n_buffs; + + n_buffs = DIV_ROUND_UP(msdu_len, + (DP_RX_BUFFER_SIZE - dp->ab->hal.hal_desc_sz)); + + skb_queue_walk_safe(msdu_list, skb, tmp) { + rxcb = ATH12K_SKB_RXCB(skb); + if (rxcb->err_rel_src == HAL_WBM_REL_SRC_MODULE_REO && + rxcb->err_code == HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO) { + if (!n_buffs) + break; + __skb_unlink(skb, msdu_list); + dev_kfree_skb_any(skb); + n_buffs--; + } + } +} + +static int ath12k_wifi7_dp_rx_h_null_q_desc(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info, + struct sk_buff_head *msdu_list) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; + u16 msdu_len = rx_info->msdu_len; + struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data; + u8 l3pad_bytes = rx_info->l3_pad_bytes; + struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); + u32 hal_rx_desc_sz = dp->ab->hal.hal_desc_sz; + + if (!rxcb->is_frag && ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE)) { + /* First buffer will be freed by the caller, so deduct it's length */ + msdu_len = msdu_len - (DP_RX_BUFFER_SIZE - hal_rx_desc_sz); + ath12k_wifi7_dp_rx_null_q_desc_sg_drop(dp, msdu_len, msdu_list); + return -EINVAL; + } + + /* Even after cleaning up the sg buffers in the msdu list with above check + * any msdu received with continuation flag needs to be dropped as invalid. + * This protects against some random err frame with continuation flag. + */ + if (rxcb->is_continuation) + return -EINVAL; + + if (!rx_info->msdu_done) { + ath12k_warn(ab, + "msdu_done bit not set in null_q_des processing\n"); + __skb_queue_purge(msdu_list); + return -EIO; + } + + /* Handle NULL queue descriptor violations arising out a missing + * REO queue for a given peer or a given TID. This typically + * may happen if a packet is received on a QOS enabled TID before the + * ADDBA negotiation for that TID, when the TID queue is setup. Or + * it may also happen for MC/BC frames if they are not routed to the + * non-QOS TID queue, in the absence of any other default TID queue. + * This error can show up both in a REO destination or WBM release ring. + */ + + if (rxcb->is_frag) { + skb_pull(msdu, hal_rx_desc_sz); + } else { + if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE) + return -EINVAL; + + skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); + skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); + } + if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, desc, msdu, rx_info))) + return -EINVAL; + + ath12k_dp_rx_h_ppdu(dp_pdev, rx_info); + ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, desc, rx_info); + + rxcb->tid = rx_info->tid; + + /* Please note that caller will having the access to msdu and completing + * rx with mac80211. Need not worry about cleaning up amsdu_list. + */ + + return 0; +} + +static bool ath12k_wifi7_dp_rx_h_tkip_mic_err(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; + u16 msdu_len = rx_info->msdu_len; + struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data; + u8 l3pad_bytes = rx_info->l3_pad_bytes; + struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); + u32 hal_rx_desc_sz = ab->hal.hal_desc_sz; + + rxcb->is_first_msdu = rx_info->is_first_msdu; + rxcb->is_last_msdu = rx_info->is_last_msdu; + + if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE) { + ath12k_dbg(ab, ATH12K_DBG_DATA, + "invalid msdu len in tkip mic err %u\n", msdu_len); + ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "", desc, + sizeof(*desc)); + return true; + } + + skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); + skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); + + if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, desc, msdu, rx_info))) + return true; + + ath12k_dp_rx_h_ppdu(dp_pdev, rx_info); + + rx_info->rx_status->flag |= (RX_FLAG_MMIC_STRIPPED | RX_FLAG_MMIC_ERROR | + RX_FLAG_DECRYPTED); + + ath12k_dp_rx_h_undecap(dp_pdev, msdu, desc, + HAL_ENCRYPT_TYPE_TKIP_MIC, false, rx_info); + return false; +} + +static bool ath12k_wifi7_dp_rx_h_rxdma_err(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); + bool drop = false; + + dp->device_stats.rxdma_error[rxcb->err_code]++; + + switch (rxcb->err_code) { + case HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR: + case HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR: + if (rx_info->err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC) { + drop = ath12k_wifi7_dp_rx_h_tkip_mic_err(dp_pdev, msdu, rx_info); + break; + } + fallthrough; + default: + /* TODO: Review other rxdma error code to check if anything is + * worth reporting to mac80211 + */ + drop = true; + break; + } + + return drop; +} + +static bool ath12k_wifi7_dp_rx_h_reo_err(struct ath12k_pdev_dp *dp_pdev, + struct sk_buff *msdu, + struct hal_rx_desc_data *rx_info, + struct sk_buff_head *msdu_list) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); + bool drop = false; + + dp->device_stats.reo_error[rxcb->err_code]++; + + switch (rxcb->err_code) { + case HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO: + if (ath12k_wifi7_dp_rx_h_null_q_desc(dp_pdev, msdu, rx_info, msdu_list)) + drop = true; + break; + case HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED: + /* TODO: Do not drop PN failed packets in the driver; + * instead, it is good to drop such packets in mac80211 + * after incrementing the replay counters. + */ + fallthrough; + default: + /* TODO: Review other errors and process them to mac80211 + * as appropriate. + */ + drop = true; + break; + } + + return drop; +} + +static void ath12k_wifi7_dp_rx_wbm_err(struct ath12k_pdev_dp *dp_pdev, + struct napi_struct *napi, + struct sk_buff *msdu, + struct sk_buff_head *msdu_list) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data; + struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu); + struct ieee80211_rx_status rxs = {}; + struct hal_rx_desc_data rx_info; + bool drop = true; + + rx_info.addr2_present = false; + rx_info.rx_status = &rxs; + + ath12k_dp_extract_rx_desc_data(dp->hal, &rx_info, rx_desc, rx_desc); + + switch (rxcb->err_rel_src) { + case HAL_WBM_REL_SRC_MODULE_REO: + drop = ath12k_wifi7_dp_rx_h_reo_err(dp_pdev, msdu, &rx_info, msdu_list); + break; + case HAL_WBM_REL_SRC_MODULE_RXDMA: + drop = ath12k_wifi7_dp_rx_h_rxdma_err(dp_pdev, msdu, &rx_info); + break; + default: + /* msdu will get freed */ + break; + } + + if (drop) { + dev_kfree_skb_any(msdu); + return; + } + + rx_info.rx_status->flag |= RX_FLAG_SKIP_MONITOR; + + ath12k_dp_rx_deliver_msdu(dp_pdev, napi, msdu, &rx_info); +} + +void ath12k_wifi7_dp_setup_pn_check_reo_cmd(struct ath12k_hal_reo_cmd *cmd, + struct ath12k_dp_rx_tid *rx_tid, + u32 cipher, enum set_key_cmd key_cmd) +{ + cmd->flag = HAL_REO_CMD_FLG_NEED_STATUS; + cmd->upd0 = HAL_REO_CMD_UPD0_PN | + HAL_REO_CMD_UPD0_PN_SIZE | + HAL_REO_CMD_UPD0_PN_VALID | + HAL_REO_CMD_UPD0_PN_CHECK | + HAL_REO_CMD_UPD0_SVLD; + + switch (cipher) { + case WLAN_CIPHER_SUITE_TKIP: + case WLAN_CIPHER_SUITE_CCMP: + case WLAN_CIPHER_SUITE_CCMP_256: + case WLAN_CIPHER_SUITE_GCMP: + case WLAN_CIPHER_SUITE_GCMP_256: + if (key_cmd == SET_KEY) { + cmd->upd1 |= HAL_REO_CMD_UPD1_PN_CHECK; + cmd->pn_size = 48; + } + break; + default: + break; + } + + cmd->addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned); + cmd->addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned); +} + +int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp, + struct napi_struct *napi, int budget) +{ + struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES]; + struct ath12k_base *ab = dp->ab; + struct ath12k_hal *hal = dp->hal; + struct ath12k *ar; + struct ath12k_pdev_dp *dp_pdev; + struct ath12k_hw_group *ag = dp->ag; + struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; + struct ath12k_dp *partner_dp; + struct dp_rxdma_ring *rx_ring; + struct hal_rx_wbm_rel_info err_info; + struct hal_srng *srng; + struct sk_buff *msdu; + struct sk_buff_head msdu_list, scatter_msdu_list; + struct ath12k_skb_rxcb *rxcb; + void *rx_desc; + int num_buffs_reaped[ATH12K_MAX_DEVICES] = {}; + int total_num_buffs_reaped = 0; + struct ath12k_rx_desc_info *desc_info; + struct ath12k_device_dp_stats *device_stats = &dp->device_stats; + struct ath12k_hw_link *hw_links = ag->hw_links; + u8 hw_link_id, device_id; + int ret, pdev_idx; + struct hal_rx_desc *msdu_data; + + __skb_queue_head_init(&msdu_list); + __skb_queue_head_init(&scatter_msdu_list); + + for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) + INIT_LIST_HEAD(&rx_desc_used_list[device_id]); + + srng = &hal->srng_list[dp->rx_rel_ring.ring_id]; + spin_lock_bh(&srng->lock); + + ath12k_hal_srng_access_begin(ab, srng); + + while (budget) { + rx_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng); + if (!rx_desc) + break; + + ret = ath12k_wifi7_hal_wbm_desc_parse_err(dp, rx_desc, + &err_info); + if (ret) { + ath12k_warn(ab, "failed to parse rx error in wbm_rel ring desc %d\n", + ret); + continue; + } + + desc_info = err_info.rx_desc; + + /* retry manual desc retrieval if hw cc is not done */ + if (!desc_info) { + desc_info = ath12k_dp_get_rx_desc(dp, err_info.cookie); + if (!desc_info) { + ath12k_warn(ab, "Invalid cookie in DP WBM rx error descriptor retrieval: 0x%x\n", + err_info.cookie); + continue; + } + } + + if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) + ath12k_warn(ab, "WBM RX err, Check HW CC implementation"); + + msdu = desc_info->skb; + desc_info->skb = NULL; + + device_id = desc_info->device_id; + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id); + if (unlikely(!partner_dp)) { + dev_kfree_skb_any(msdu); + + /* In any case continuation bit is set + * in the previous record, cleanup scatter_msdu_list + */ + ath12k_dp_clean_up_skb_list(&scatter_msdu_list); + continue; + } + + list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]); + + rxcb = ATH12K_SKB_RXCB(msdu); + dma_unmap_single(partner_dp->dev, rxcb->paddr, + msdu->len + skb_tailroom(msdu), + DMA_FROM_DEVICE); + + num_buffs_reaped[device_id]++; + total_num_buffs_reaped++; + + if (!err_info.continuation) + budget--; + + if (err_info.push_reason != + HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) { + dev_kfree_skb_any(msdu); + continue; + } + + msdu_data = (struct hal_rx_desc *)msdu->data; + rxcb->err_rel_src = err_info.err_rel_src; + rxcb->err_code = err_info.err_code; + rxcb->is_first_msdu = err_info.first_msdu; + rxcb->is_last_msdu = err_info.last_msdu; + rxcb->is_continuation = err_info.continuation; + rxcb->rx_desc = msdu_data; + rxcb->peer_id = ath12k_wifi7_dp_rx_get_peer_id(dp, dp->peer_metadata_ver, + err_info.peer_metadata); + + if (err_info.continuation) { + __skb_queue_tail(&scatter_msdu_list, msdu); + continue; + } + + hw_link_id = ath12k_dp_rx_get_msdu_src_link(partner_dp->hal, + msdu_data); + if (hw_link_id >= ATH12K_GROUP_MAX_RADIO) { + dev_kfree_skb_any(msdu); + + /* In any case continuation bit is set + * in the previous record, cleanup scatter_msdu_list + */ + ath12k_dp_clean_up_skb_list(&scatter_msdu_list); + continue; + } + + if (!skb_queue_empty(&scatter_msdu_list)) { + struct sk_buff *msdu; + + skb_queue_walk(&scatter_msdu_list, msdu) { + rxcb = ATH12K_SKB_RXCB(msdu); + rxcb->hw_link_id = hw_link_id; + } + + skb_queue_splice_tail_init(&scatter_msdu_list, + &msdu_list); + } + + rxcb = ATH12K_SKB_RXCB(msdu); + rxcb->hw_link_id = hw_link_id; + __skb_queue_tail(&msdu_list, msdu); + } + + /* In any case continuation bit is set in the + * last record, cleanup scatter_msdu_list + */ + ath12k_dp_clean_up_skb_list(&scatter_msdu_list); + + ath12k_hal_srng_access_end(ab, srng); + + spin_unlock_bh(&srng->lock); + + if (!total_num_buffs_reaped) + goto done; + + for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) { + if (!num_buffs_reaped[device_id]) + continue; + + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id); + rx_ring = &partner_dp->rx_refill_buf_ring; + + ath12k_dp_rx_bufs_replenish(dp, rx_ring, + &rx_desc_used_list[device_id], + num_buffs_reaped[device_id]); + } + + rcu_read_lock(); + while ((msdu = __skb_dequeue(&msdu_list))) { + rxcb = ATH12K_SKB_RXCB(msdu); + hw_link_id = rxcb->hw_link_id; + + device_id = hw_links[hw_link_id].device_id; + partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id); + if (unlikely(!partner_dp)) { + ath12k_dbg(ab, ATH12K_DBG_DATA, + "Unable to process WBM error msdu due to invalid hw link id %d device id %d\n", + hw_link_id, device_id); + dev_kfree_skb_any(msdu); + continue; + } + + pdev_idx = ath12k_hw_mac_id_to_pdev_id(partner_dp->hw_params, + hw_links[hw_link_id].pdev_idx); + + dp_pdev = ath12k_dp_to_pdev_dp(partner_dp, pdev_idx); + if (!dp_pdev) { + dev_kfree_skb_any(msdu); + continue; + } + ar = ath12k_pdev_dp_to_ar(dp_pdev); + + if (!ar || !rcu_dereference(ar->ab->pdevs_active[pdev_idx])) { + dev_kfree_skb_any(msdu); + continue; + } + + if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) { + dev_kfree_skb_any(msdu); + continue; + } + + if (rxcb->err_rel_src < HAL_WBM_REL_SRC_MODULE_MAX) { + device_id = dp_pdev->dp->device_id; + device_stats->rx_wbm_rel_source[rxcb->err_rel_src][device_id]++; + } + + ath12k_wifi7_dp_rx_wbm_err(dp_pdev, napi, msdu, &msdu_list); + } + rcu_read_unlock(); +done: + return total_num_buffs_reaped; +} + +int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_rx_ring_tlv_filter tlv_filter = {}; + u32 ring_id; + int ret; + u32 hal_rx_desc_sz = ab->hal.hal_desc_sz; + + ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id; + + tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING; + tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR; + tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST | + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST | + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA; + tlv_filter.offset_valid = true; + tlv_filter.rx_packet_offset = hal_rx_desc_sz; + + tlv_filter.rx_mpdu_start_offset = + ath12k_hal_rx_desc_get_mpdu_start_offset_qcn9274(); + tlv_filter.rx_msdu_end_offset = + ath12k_hal_rx_desc_get_msdu_end_offset_qcn9274(); + + tlv_filter.rx_mpdu_start_wmask = ath12k_hal_rx_mpdu_start_wmask_get_qcn9274(); + tlv_filter.rx_msdu_end_wmask = ath12k_hal_rx_msdu_end_wmask_get_qcn9274(); + ath12k_dbg(ab, ATH12K_DBG_DATA, + "Configuring compact tlv masks rx_mpdu_start_wmask 0x%x rx_msdu_end_wmask 0x%x\n", + tlv_filter.rx_mpdu_start_wmask, tlv_filter.rx_msdu_end_wmask); + + ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, 0, + HAL_RXDMA_BUF, + DP_RXDMA_REFILL_RING_SIZE, + &tlv_filter); + + return ret; +} + +int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_rx_ring_tlv_filter tlv_filter = {}; + u32 ring_id; + int ret = 0; + u32 hal_rx_desc_sz = ab->hal.hal_desc_sz; + int i; + + ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id; + + tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING; + tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR; + tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST | + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST | + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA; + tlv_filter.offset_valid = true; + tlv_filter.rx_packet_offset = hal_rx_desc_sz; + + tlv_filter.rx_header_offset = offsetof(struct hal_rx_desc_wcn7850, pkt_hdr_tlv); + + tlv_filter.rx_mpdu_start_offset = + ath12k_hal_rx_desc_get_mpdu_start_offset_wcn7850(); + tlv_filter.rx_msdu_end_offset = + ath12k_hal_rx_desc_get_msdu_end_offset_wcn7850(); + + /* TODO: Selectively subscribe to required qwords within msdu_end + * and mpdu_start and setup the mask in below msg + * and modify the rx_desc struct + */ + + for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { + ring_id = dp->rx_mac_buf_ring[i].ring_id; + ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, i, + HAL_RXDMA_BUF, + DP_RXDMA_REFILL_RING_SIZE, + &tlv_filter); + } + + return ret; +} + +int ath12k_dp_rxdma_ring_sel_config_qcc2072(struct ath12k_base *ab) +{ + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct htt_rx_ring_tlv_filter tlv_filter = {}; + u32 ring_id; + int ret = 0; + u32 hal_rx_desc_sz = ab->hal.hal_desc_sz; + int i; + + ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id; + + tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING; + tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR; + tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST | + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST | + HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA; + tlv_filter.offset_valid = true; + tlv_filter.rx_packet_offset = hal_rx_desc_sz; + + tlv_filter.rx_header_offset = offsetof(struct hal_rx_desc_qcc2072, pkt_hdr_tlv); + + tlv_filter.rx_mpdu_start_offset = + ath12k_hal_rx_desc_get_mpdu_start_offset_qcc2072(); + tlv_filter.rx_msdu_end_offset = + ath12k_hal_rx_desc_get_msdu_end_offset_qcc2072(); + + /* + * TODO: Selectively subscribe to required qwords within msdu_end + * and mpdu_start and setup the mask in below msg + * and modify the rx_desc struct + */ + + for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) { + ring_id = dp->rx_mac_buf_ring[i].ring_id; + ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, i, + HAL_RXDMA_BUF, + DP_RXDMA_REFILL_RING_SIZE, + &tlv_filter); + } + + return ret; +} + +void ath12k_wifi7_dp_rx_process_reo_status(struct ath12k_dp *dp) +{ + struct ath12k_base *ab = dp->ab; + struct ath12k_hal *hal = dp->hal; + struct hal_srng *srng; + struct ath12k_dp_rx_reo_cmd *cmd, *tmp; + bool found = false; + u16 tag; + struct hal_reo_status reo_status; + void *hdr, *desc; + + srng = &hal->srng_list[dp->reo_status_ring.ring_id]; + + memset(&reo_status, 0, sizeof(reo_status)); + + spin_lock_bh(&srng->lock); + + ath12k_hal_srng_access_begin(ab, srng); + + while ((hdr = ath12k_hal_srng_dst_get_next_entry(ab, srng))) { + tag = hal->ops->reo_status_dec_tlv_hdr(hdr, &desc); + + switch (tag) { + case HAL_REO_GET_QUEUE_STATS_STATUS: + ath12k_wifi7_hal_reo_status_queue_stats(ab, desc, + &reo_status); + break; + case HAL_REO_FLUSH_QUEUE_STATUS: + ath12k_wifi7_hal_reo_flush_queue_status(ab, desc, + &reo_status); + break; + case HAL_REO_FLUSH_CACHE_STATUS: + ath12k_wifi7_hal_reo_flush_cache_status(ab, desc, + &reo_status); + break; + case HAL_REO_UNBLOCK_CACHE_STATUS: + ath12k_wifi7_hal_reo_unblk_cache_status(ab, desc, + &reo_status); + break; + case HAL_REO_FLUSH_TIMEOUT_LIST_STATUS: + ath12k_wifi7_hal_reo_flush_timeout_list_status(ab, desc, + &reo_status); + break; + case HAL_REO_DESCRIPTOR_THRESHOLD_REACHED_STATUS: + ath12k_wifi7_hal_reo_desc_thresh_reached_status(ab, desc, + &reo_status); + break; + case HAL_REO_UPDATE_RX_REO_QUEUE_STATUS: + ath12k_wifi7_hal_reo_update_rx_reo_queue_status(ab, desc, + &reo_status); + break; + default: + ath12k_warn(ab, "Unknown reo status type %d\n", tag); + continue; + } + + spin_lock_bh(&dp->reo_cmd_lock); + list_for_each_entry_safe(cmd, tmp, &dp->reo_cmd_list, list) { + if (reo_status.uniform_hdr.cmd_num == cmd->cmd_num) { + found = true; + list_del(&cmd->list); + break; + } + } + spin_unlock_bh(&dp->reo_cmd_lock); + + if (found) { + cmd->handler(dp, (void *)&cmd->data, + reo_status.uniform_hdr.cmd_status); + kfree(cmd); + } + + found = false; + } + + ath12k_hal_srng_access_end(ab, srng); + + spin_unlock_bh(&srng->lock); +} + +bool +ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab, + struct hal_rx_desc *rx_desc) +{ + u32 tlv_tag; + + tlv_tag = ab->hal.ops->rx_desc_get_mpdu_start_tag(rx_desc); + + return tlv_tag == HAL_RX_MPDU_START; +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h new file mode 100644 index 0000000000000..8aa79faf567fd --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#ifndef ATH12K_DP_RX_WIFI7_H +#define ATH12K_DP_RX_WIFI7_H + +#include "../core.h" +#include "../dp_rx.h" +#include "hal_rx_desc.h" + +struct ath12k_hal_reo_cmd; + +int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp, + struct napi_struct *napi, int budget); +int ath12k_wifi7_dp_rx_process_err(struct ath12k_dp *dp, struct napi_struct *napi, + int budget); +int ath12k_wifi7_dp_rx_process(struct ath12k_dp *dp, int mac_id, + struct napi_struct *napi, + int budget); +void ath12k_wifi7_dp_rx_process_reo_status(struct ath12k_dp *dp); +int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab); +int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab); +int ath12k_dp_rxdma_ring_sel_config_qcc2072(struct ath12k_base *ab); +void ath12k_wifi7_dp_setup_pn_check_reo_cmd(struct ath12k_hal_reo_cmd *cmd, + struct ath12k_dp_rx_tid *rx_tid, + u32 cipher, enum set_key_cmd key_cmd); +int ath12k_wifi7_dp_rx_assign_reoq(struct ath12k_base *ab, struct ath12k_dp_peer *dp_peer, + struct ath12k_dp_rx_tid *rx_tid, + u16 ssn, enum hal_pn_type pn_type); +int ath12k_wifi7_dp_rx_link_desc_return(struct ath12k_dp *dp, + struct ath12k_buffer_addr *buf_addr_info, + enum hal_wbm_rel_bm_act action); +void ath12k_wifi7_dp_rx_frags_cleanup(struct ath12k_dp_rx_tid *rx_tid, + bool rel_link_desc); +void ath12k_wifi7_peer_rx_tid_qref_setup(struct ath12k_base *ab, u16 peer_id, u16 tid, + dma_addr_t paddr); +void ath12k_wifi7_dp_rx_peer_tid_delete(struct ath12k_base *ab, + struct ath12k_dp_link_peer *peer, u8 tid); +int ath12k_wifi7_dp_reo_cmd_send(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid, + enum hal_reo_cmd_type type, + struct ath12k_hal_reo_cmd *cmd, + void (*cb)(struct ath12k_dp *dp, void *ctx, + enum hal_reo_cmd_status status)); +int ath12k_wifi7_dp_reo_cache_flush(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid); +int ath12k_wifi7_peer_rx_tid_reo_update(struct ath12k_dp *dp, + struct ath12k_dp_link_peer *peer, + struct ath12k_dp_rx_tid *rx_tid, + u32 ba_win_sz, u16 ssn, + bool update_ssn); +void ath12k_wifi7_peer_rx_tid_qref_reset(struct ath12k_base *ab, u16 peer_id, u16 tid); +bool +ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab, + struct hal_rx_desc *rx_desc); +int ath12k_wifi7_dp_rx_tid_delete_handler(struct ath12k_base *ab, + struct ath12k_dp_rx_tid_rxq *rx_tid); +#endif diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c new file mode 100644 index 0000000000000..629084aa36d85 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c @@ -0,0 +1,978 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "../core.h" +#include "../debug.h" +#include "../dp_tx.h" +#include "../peer.h" +#include "dp_tx.h" +#include "hal_desc.h" +#include "hal.h" +#include "hal_tx.h" + +static void +ath12k_wifi7_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab, + struct hal_tx_msdu_ext_desc *tcl_ext_cmd, + struct hal_tx_info *ti) +{ + tcl_ext_cmd->info0 = le32_encode_bits(ti->paddr, + HAL_TX_MSDU_EXT_INFO0_BUF_PTR_LO); + tcl_ext_cmd->info1 = le32_encode_bits(0x0, + HAL_TX_MSDU_EXT_INFO1_BUF_PTR_HI) | + le32_encode_bits(ti->data_len, + HAL_TX_MSDU_EXT_INFO1_BUF_LEN); + + tcl_ext_cmd->info1 |= le32_encode_bits(1, HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE) | + le32_encode_bits(ti->encap_type, + HAL_TX_MSDU_EXT_INFO1_ENCAP_TYPE) | + le32_encode_bits(ti->encrypt_type, + HAL_TX_MSDU_EXT_INFO1_ENCRYPT_TYPE); +} + +#define HTT_META_DATA_ALIGNMENT 0x8 + +/* Preparing HTT Metadata when utilized with ext MSDU */ +static int ath12k_wifi7_dp_prepare_htt_metadata(struct sk_buff *skb) +{ + struct hal_tx_msdu_metadata *desc_ext; + u8 htt_desc_size; + /* Size rounded of multiple of 8 bytes */ + u8 htt_desc_size_aligned; + + htt_desc_size = sizeof(struct hal_tx_msdu_metadata); + htt_desc_size_aligned = ALIGN(htt_desc_size, HTT_META_DATA_ALIGNMENT); + + desc_ext = ath12k_dp_metadata_align_skb(skb, htt_desc_size_aligned); + if (!desc_ext) + return -ENOMEM; + + desc_ext->info0 = le32_encode_bits(1, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_FLAG) | + le32_encode_bits(0, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_TYPE) | + le32_encode_bits(1, + HAL_TX_MSDU_METADATA_INFO0_HOST_TX_DESC_POOL); + + return 0; +} + +/* TODO: Remove the export once this file is built with wifi7 ko */ +int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *arvif, + struct sk_buff *skb, bool gsn_valid, int mcbc_gsn, + bool is_mcast) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_hal *hal = dp->hal; + struct ath12k_base *ab = dp->ab; + struct hal_tx_info ti = {}; + struct ath12k_tx_desc_info *tx_desc; + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb); + struct hal_tcl_data_cmd *hal_tcl_desc; + struct hal_tx_msdu_ext_desc *msg; + struct sk_buff *skb_ext_desc = NULL; + struct hal_srng *tcl_ring; + struct ieee80211_hdr *hdr = (void *)skb->data; + struct ath12k_vif *ahvif = arvif->ahvif; + struct ath12k_dp_vif *dp_vif = &ahvif->dp_vif; + struct ath12k_dp_link_vif *dp_link_vif; + struct dp_tx_ring *tx_ring; + u8 pool_id; + u8 hal_ring_id; + int ret; + u8 ring_selector, ring_map = 0; + bool tcl_ring_retry; + bool msdu_ext_desc = false; + bool add_htt_metadata = false; + u32 iova_mask = dp->hw_params->iova_mask; + bool is_diff_encap = false; + bool is_null_frame = false; + + if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags)) + return -ESHUTDOWN; + + if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && + !ieee80211_is_data(hdr->frame_control)) + return -EOPNOTSUPP; + + pool_id = skb_get_queue_mapping(skb) & (ATH12K_HW_MAX_QUEUES - 1); + + /* Let the default ring selection be based on current processor + * number, where one of the 3 tcl rings are selected based on + * the smp_processor_id(). In case that ring + * is full/busy, we resort to other available rings. + * If all rings are full, we drop the packet. + * TODO: Add throttling logic when all rings are full + */ + ring_selector = dp->hw_params->hw_ops->get_ring_selector(skb); + +tcl_ring_sel: + tcl_ring_retry = false; + ti.ring_id = ring_selector % dp->hw_params->max_tx_ring; + + ring_map |= BIT(ti.ring_id); + ti.rbm_id = hal->tcl_to_wbm_rbm_map[ti.ring_id].rbm_id; + + tx_ring = &dp->tx_ring[ti.ring_id]; + + tx_desc = ath12k_dp_tx_assign_buffer(dp, pool_id); + if (!tx_desc) + return -ENOMEM; + + dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, arvif->link_id); + + ti.bank_id = dp_link_vif->bank_id; + ti.meta_data_flags = dp_link_vif->tcl_metadata; + + if (dp_vif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW && + test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags)) { + if (skb_cb->flags & ATH12K_SKB_CIPHER_SET) { + ti.encrypt_type = + ath12k_dp_tx_get_encrypt_type(skb_cb->cipher); + + if (ieee80211_has_protected(hdr->frame_control)) + skb_put(skb, IEEE80211_CCMP_MIC_LEN); + } else { + ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN; + } + + msdu_ext_desc = true; + } + + if (gsn_valid) { + /* Reset and Initialize meta_data_flags with Global Sequence + * Number (GSN) info. + */ + ti.meta_data_flags = + u32_encode_bits(HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM, + HTT_TCL_META_DATA_TYPE) | + u32_encode_bits(mcbc_gsn, HTT_TCL_META_DATA_GLOBAL_SEQ_NUM); + } + + ti.encap_type = ath12k_dp_tx_get_encap_type(ab, skb); + ti.addr_search_flags = dp_link_vif->hal_addr_search_flags; + ti.search_type = dp_link_vif->search_type; + ti.type = HAL_TCL_DESC_TYPE_BUFFER; + ti.pkt_offset = 0; + ti.lmac_id = dp_link_vif->lmac_id; + + ti.vdev_id = dp_link_vif->vdev_id; + if (gsn_valid) + ti.vdev_id += HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID; + + ti.bss_ast_hash = dp_link_vif->ast_hash; + ti.bss_ast_idx = dp_link_vif->ast_idx; + ti.dscp_tid_tbl_idx = 0; + + if (skb->ip_summed == CHECKSUM_PARTIAL && + ti.encap_type != HAL_TCL_ENCAP_TYPE_RAW) { + ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_IP4_CKSUM_EN) | + u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP4_CKSUM_EN) | + u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP6_CKSUM_EN) | + u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP4_CKSUM_EN) | + u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP6_CKSUM_EN); + } + + ti.flags1 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO3_TID_OVERWRITE); + + ti.tid = ath12k_dp_tx_get_tid(skb); + + switch (ti.encap_type) { + case HAL_TCL_ENCAP_TYPE_NATIVE_WIFI: + is_null_frame = ieee80211_is_nullfunc(hdr->frame_control); + if (ahvif->vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) { + if (skb->protocol == cpu_to_be16(ETH_P_PAE) || is_null_frame) + is_diff_encap = true; + + /* Firmware expects msdu ext descriptor for nwifi/raw packets + * received in ETH mode. Without this, observed tx fail for + * Multicast packets in ETH mode. + */ + msdu_ext_desc = true; + } else { + ath12k_dp_tx_encap_nwifi(skb); + } + break; + case HAL_TCL_ENCAP_TYPE_RAW: + if (!test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) { + ret = -EINVAL; + goto fail_remove_tx_buf; + } + break; + case HAL_TCL_ENCAP_TYPE_ETHERNET: + /* no need to encap */ + break; + case HAL_TCL_ENCAP_TYPE_802_3: + default: + /* TODO: Take care of other encap modes as well */ + ret = -EINVAL; + atomic_inc(&dp->device_stats.tx_err.misc_fail); + goto fail_remove_tx_buf; + } + + if (iova_mask && + (unsigned long)skb->data & iova_mask) { + ret = ath12k_dp_tx_align_payload(dp, &skb); + if (ret) { + ath12k_warn(ab, "failed to align TX buffer %d\n", ret); + /* don't bail out, give original buffer + * a chance even unaligned. + */ + goto map; + } + + /* hdr is pointing to a wrong place after alignment, + * so refresh it for later use. + */ + hdr = (void *)skb->data; + } +map: + ti.paddr = dma_map_single(dp->dev, skb->data, skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(dp->dev, ti.paddr)) { + atomic_inc(&dp->device_stats.tx_err.misc_fail); + ath12k_warn(ab, "failed to DMA map data Tx buffer\n"); + ret = -ENOMEM; + goto fail_remove_tx_buf; + } + + if ((!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags) && + !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) && + !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) && + ieee80211_has_protected(hdr->frame_control)) || + is_diff_encap) { + /* Firmware is not expecting meta data for qos null + * nwifi packet received in ETH encap mode. + */ + if (is_null_frame && msdu_ext_desc) + goto skip_htt_meta; + + /* Add metadata for sw encrypted vlan group traffic + * and EAPOL nwifi packet received in ETH encap mode. + */ + add_htt_metadata = true; + msdu_ext_desc = true; + ti.meta_data_flags |= HTT_TCL_META_DATA_VALID_HTT; +skip_htt_meta: + ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TO_FW); + ti.encap_type = HAL_TCL_ENCAP_TYPE_RAW; + ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN; + } + + tx_desc->skb = skb; + tx_desc->mac_id = dp_link_vif->pdev_idx; + ti.desc_id = tx_desc->desc_id; + ti.data_len = skb->len; + skb_cb->paddr = ti.paddr; + + if (msdu_ext_desc) { + skb_ext_desc = dev_alloc_skb(sizeof(struct hal_tx_msdu_ext_desc)); + if (!skb_ext_desc) { + ret = -ENOMEM; + goto fail_unmap_dma; + } + + skb_put(skb_ext_desc, sizeof(struct hal_tx_msdu_ext_desc)); + memset(skb_ext_desc->data, 0, skb_ext_desc->len); + + msg = (struct hal_tx_msdu_ext_desc *)skb_ext_desc->data; + ath12k_wifi7_hal_tx_cmd_ext_desc_setup(ab, msg, &ti); + + if (add_htt_metadata) { + ret = ath12k_wifi7_dp_prepare_htt_metadata(skb_ext_desc); + if (ret < 0) { + ath12k_dbg(ab, ATH12K_DBG_DP_TX, + "Failed to add HTT meta data, dropping packet\n"); + goto fail_free_ext_skb; + } + } + + ti.paddr = dma_map_single(dp->dev, skb_ext_desc->data, + skb_ext_desc->len, DMA_TO_DEVICE); + ret = dma_mapping_error(dp->dev, ti.paddr); + if (ret) + goto fail_free_ext_skb; + + ti.data_len = skb_ext_desc->len; + ti.type = HAL_TCL_DESC_TYPE_EXT_DESC; + + skb_cb->paddr_ext_desc = ti.paddr; + tx_desc->skb_ext_desc = skb_ext_desc; + } + + hal_ring_id = tx_ring->tcl_data_ring.ring_id; + tcl_ring = &hal->srng_list[hal_ring_id]; + + spin_lock_bh(&tcl_ring->lock); + + ath12k_hal_srng_access_begin(ab, tcl_ring); + + hal_tcl_desc = ath12k_hal_srng_src_get_next_entry(ab, tcl_ring); + if (!hal_tcl_desc) { + /* NOTE: It is highly unlikely we'll be running out of tcl_ring + * desc because the desc is directly enqueued onto hw queue. + */ + ath12k_hal_srng_access_end(ab, tcl_ring); + dp->device_stats.tx_err.desc_na[ti.ring_id]++; + spin_unlock_bh(&tcl_ring->lock); + ret = -ENOMEM; + + /* Checking for available tcl descriptors in another ring in + * case of failure due to full tcl ring now, is better than + * checking this ring earlier for each pkt tx. + * Restart ring selection if some rings are not checked yet. + */ + if (ring_map != (BIT(dp->hw_params->max_tx_ring) - 1) && + dp->hw_params->tcl_ring_retry) { + tcl_ring_retry = true; + ring_selector++; + } + + goto fail_unmap_dma_ext; + } + + spin_lock_bh(&arvif->link_stats_lock); + arvif->link_stats.tx_encap_type[ti.encap_type]++; + arvif->link_stats.tx_encrypt_type[ti.encrypt_type]++; + arvif->link_stats.tx_desc_type[ti.type]++; + + if (is_mcast) + arvif->link_stats.tx_bcast_mcast++; + else + arvif->link_stats.tx_enqueued++; + spin_unlock_bh(&arvif->link_stats_lock); + + dp->device_stats.tx_enqueued[ti.ring_id]++; + + ath12k_wifi7_hal_tx_cmd_desc_setup(ab, hal_tcl_desc, &ti); + + ath12k_hal_srng_access_end(ab, tcl_ring); + + spin_unlock_bh(&tcl_ring->lock); + + ath12k_dbg_dump(ab, ATH12K_DBG_DP_TX, NULL, "dp tx msdu: ", + skb->data, skb->len); + + atomic_inc(&dp_pdev->num_tx_pending); + + return 0; + +fail_unmap_dma_ext: + if (skb_cb->paddr_ext_desc) + dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc, + skb_ext_desc->len, + DMA_TO_DEVICE); +fail_free_ext_skb: + kfree_skb(skb_ext_desc); + +fail_unmap_dma: + dma_unmap_single(dp->dev, ti.paddr, ti.data_len, DMA_TO_DEVICE); + +fail_remove_tx_buf: + ath12k_dp_tx_release_txbuf(dp, tx_desc, pool_id); + + spin_lock_bh(&arvif->link_stats_lock); + arvif->link_stats.tx_dropped++; + spin_unlock_bh(&arvif->link_stats_lock); + + if (tcl_ring_retry) + goto tcl_ring_sel; + + return ret; +} + +static void +ath12k_dp_tx_htt_tx_complete_buf(struct ath12k_dp *dp, + struct ath12k_tx_desc_params *desc_params, + struct dp_tx_ring *tx_ring, + struct ath12k_dp_htt_wbm_tx_status *ts, + u16 peer_id) +{ + struct ath12k_base *ab = dp->ab; + struct ieee80211_tx_info *info; + struct ath12k_link_vif *arvif; + struct ath12k_skb_cb *skb_cb; + struct ieee80211_vif *vif; + struct ath12k_vif *ahvif; + struct sk_buff *msdu = desc_params->skb; + s32 noise_floor; + struct ieee80211_tx_status status = {}; + struct ath12k_dp_link_peer *peer; + struct ath12k_pdev_dp *dp_pdev; + u8 pdev_id; + + skb_cb = ATH12K_SKB_CB(msdu); + info = IEEE80211_SKB_CB(msdu); + + pdev_id = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, desc_params->mac_id); + + rcu_read_lock(); + dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_id); + if (!dp_pdev) { + rcu_read_unlock(); + return; + } + + dp->device_stats.tx_completed[tx_ring->tcl_data_ring_id]++; + + if (atomic_dec_and_test(&dp_pdev->num_tx_pending)) + wake_up(&dp_pdev->tx_empty_waitq); + + dma_unmap_single(dp->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); + if (skb_cb->paddr_ext_desc) { + dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc, + desc_params->skb_ext_desc->len, DMA_TO_DEVICE); + dev_kfree_skb_any(desc_params->skb_ext_desc); + } + + vif = skb_cb->vif; + if (vif) { + ahvif = ath12k_vif_to_ahvif(vif); + arvif = rcu_dereference(ahvif->link[skb_cb->link_id]); + if (arvif) { + spin_lock_bh(&arvif->link_stats_lock); + arvif->link_stats.tx_completed++; + spin_unlock_bh(&arvif->link_stats_lock); + } + } + + memset(&info->status, 0, sizeof(info->status)); + + if (ts->acked) { + if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) { + info->flags |= IEEE80211_TX_STAT_ACK; + info->status.ack_signal = ts->ack_rssi; + + if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, + ab->wmi_ab.svc_map)) { + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + + spin_lock_bh(&ar->data_lock); + noise_floor = ath12k_pdev_get_noise_floor(ar); + spin_unlock_bh(&ar->data_lock); + + info->status.ack_signal += noise_floor; + } + + info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID; + } else { + info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; + } + } + + peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, peer_id); + if (!peer || !peer->sta) { + ath12k_dbg(ab, ATH12K_DBG_DATA, + "dp_tx: failed to find the peer with peer_id %d\n", peer_id); + ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu); + goto exit; + } else { + status.sta = peer->sta; + } + + status.info = info; + status.skb = msdu; + ieee80211_tx_status_ext(ath12k_pdev_dp_to_hw(dp_pdev), &status); +exit: + rcu_read_unlock(); +} + +static void +ath12k_dp_tx_process_htt_tx_complete(struct ath12k_dp *dp, void *desc, + struct dp_tx_ring *tx_ring, + struct ath12k_tx_desc_params *desc_params) +{ + struct htt_tx_wbm_completion *status_desc; + struct ath12k_dp_htt_wbm_tx_status ts = {}; + enum hal_wbm_htt_tx_comp_status wbm_status; + u16 peer_id; + + status_desc = desc; + + wbm_status = le32_get_bits(status_desc->info0, + HTT_TX_WBM_COMP_INFO0_STATUS); + dp->device_stats.fw_tx_status[wbm_status]++; + + switch (wbm_status) { + case HAL_WBM_REL_HTT_TX_COMP_STATUS_OK: + ts.acked = (wbm_status == HAL_WBM_REL_HTT_TX_COMP_STATUS_OK); + ts.ack_rssi = le32_get_bits(status_desc->info2, + HTT_TX_WBM_COMP_INFO2_ACK_RSSI); + + peer_id = le32_get_bits(((struct hal_wbm_completion_ring_tx *)desc)-> + info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID); + + ath12k_dp_tx_htt_tx_complete_buf(dp, desc_params, tx_ring, &ts, peer_id); + break; + case HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP: + case HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL: + case HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ: + case HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT: + case HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH: + ath12k_dp_tx_free_txbuf(dp, tx_ring, desc_params); + break; + case HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY: + /* This event is to be handled only when the driver decides to + * use WDS offload functionality. + */ + break; + default: + ath12k_warn(dp->ab, "Unknown htt wbm tx status %d\n", wbm_status); + break; + } +} + +static void ath12k_wifi7_dp_tx_update_txcompl(struct ath12k_pdev_dp *dp_pdev, + struct hal_tx_status *ts) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_dp_link_peer *peer; + struct ath12k_link_sta *arsta; + struct rate_info txrate = {}; + struct ieee80211_sta *sta; + struct ath12k_sta *ahsta; + u16 rate, ru_tones; + u8 rate_idx = 0; + int ret; + + peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, ts->peer_id); + if (!peer || !peer->sta) { + ath12k_dbg(dp->ab, ATH12K_DBG_DP_TX, + "failed to find the peer by id %u\n", ts->peer_id); + return; + } + + spin_lock_bh(&dp->dp_lock); + + sta = peer->sta; + ahsta = ath12k_sta_to_ahsta(sta); + arsta = &ahsta->deflink; + + spin_unlock_bh(&dp->dp_lock); + + /* This is to prefer choose the real NSS value arsta->last_txrate.nss, + * if it is invalid, then choose the NSS value while assoc. + */ + if (peer->last_txrate.nss) + txrate.nss = peer->last_txrate.nss; + else + txrate.nss = arsta->peer_nss; + + switch (ts->pkt_type) { + case HAL_TX_RATE_STATS_PKT_TYPE_11A: + case HAL_TX_RATE_STATS_PKT_TYPE_11B: + ret = ath12k_mac_hw_ratecode_to_legacy_rate(ts->mcs, + ts->pkt_type, + &rate_idx, + &rate); + if (ret < 0) { + ath12k_warn(dp->ab, "Invalid tx legacy rate %d\n", ret); + return; + } + + txrate.legacy = rate; + break; + case HAL_TX_RATE_STATS_PKT_TYPE_11N: + if (ts->mcs > ATH12K_HT_MCS_MAX) { + ath12k_warn(dp->ab, "Invalid HT mcs index %d\n", ts->mcs); + return; + } + + if (txrate.nss != 0) + txrate.mcs = ts->mcs + 8 * (txrate.nss - 1); + + txrate.flags = RATE_INFO_FLAGS_MCS; + + if (ts->sgi) + txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; + break; + case HAL_TX_RATE_STATS_PKT_TYPE_11AC: + if (ts->mcs > ATH12K_VHT_MCS_MAX) { + ath12k_warn(dp->ab, "Invalid VHT mcs index %d\n", ts->mcs); + return; + } + + txrate.mcs = ts->mcs; + txrate.flags = RATE_INFO_FLAGS_VHT_MCS; + + if (ts->sgi) + txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; + break; + case HAL_TX_RATE_STATS_PKT_TYPE_11AX: + if (ts->mcs > ATH12K_HE_MCS_MAX) { + ath12k_warn(dp->ab, "Invalid HE mcs index %d\n", ts->mcs); + return; + } + + txrate.mcs = ts->mcs; + txrate.flags = RATE_INFO_FLAGS_HE_MCS; + txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(ts->sgi); + break; + case HAL_TX_RATE_STATS_PKT_TYPE_11BE: + if (ts->mcs > ATH12K_EHT_MCS_MAX) { + ath12k_warn(dp->ab, "Invalid EHT mcs index %d\n", ts->mcs); + return; + } + + txrate.mcs = ts->mcs; + txrate.flags = RATE_INFO_FLAGS_EHT_MCS; + txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(ts->sgi); + break; + default: + ath12k_warn(dp->ab, "Invalid tx pkt type: %d\n", ts->pkt_type); + return; + } + + txrate.bw = ath12k_mac_bw_to_mac80211_bw(ts->bw); + + if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11AX) { + txrate.bw = RATE_INFO_BW_HE_RU; + ru_tones = ath12k_mac_he_convert_tones_to_ru_tones(ts->tones); + txrate.he_ru_alloc = + ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones); + } + + if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11BE) { + txrate.bw = RATE_INFO_BW_EHT_RU; + txrate.eht_ru_alloc = + ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(ts->tones); + } + + spin_lock_bh(&dp->dp_lock); + peer->txrate = txrate; + spin_unlock_bh(&dp->dp_lock); +} + +static void ath12k_wifi7_dp_tx_complete_msdu(struct ath12k_pdev_dp *dp_pdev, + struct ath12k_tx_desc_params *desc_params, + struct hal_tx_status *ts, + int ring) +{ + struct ath12k_dp *dp = dp_pdev->dp; + struct ath12k_base *ab = dp->ab; + struct ieee80211_tx_info *info; + struct ath12k_link_vif *arvif; + struct ath12k_skb_cb *skb_cb; + struct ieee80211_vif *vif; + struct ath12k_vif *ahvif; + struct sk_buff *msdu = desc_params->skb; + s32 noise_floor; + struct ieee80211_tx_status status = {}; + struct ieee80211_rate_status status_rate = {}; + struct ath12k_dp_link_peer *peer; + struct rate_info rate; + + if (WARN_ON_ONCE(ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM)) { + /* Must not happen */ + return; + } + + skb_cb = ATH12K_SKB_CB(msdu); + dp->device_stats.tx_completed[ring]++; + + dma_unmap_single(dp->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); + if (skb_cb->paddr_ext_desc) { + dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc, + desc_params->skb_ext_desc->len, DMA_TO_DEVICE); + dev_kfree_skb_any(desc_params->skb_ext_desc); + } + + rcu_read_lock(); + + if (!rcu_dereference(ab->pdevs_active[dp_pdev->mac_id])) { + ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu); + goto exit; + } + + if (!skb_cb->vif) { + ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu); + goto exit; + } + + vif = skb_cb->vif; + if (vif) { + ahvif = ath12k_vif_to_ahvif(vif); + arvif = rcu_dereference(ahvif->link[skb_cb->link_id]); + if (arvif) { + spin_lock_bh(&arvif->link_stats_lock); + arvif->link_stats.tx_completed++; + spin_unlock_bh(&arvif->link_stats_lock); + } + } + + info = IEEE80211_SKB_CB(msdu); + memset(&info->status, 0, sizeof(info->status)); + + /* skip tx rate update from ieee80211_status*/ + info->status.rates[0].idx = -1; + + switch (ts->status) { + case HAL_WBM_TQM_REL_REASON_FRAME_ACKED: + if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) { + info->flags |= IEEE80211_TX_STAT_ACK; + info->status.ack_signal = ts->ack_rssi; + + if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, + ab->wmi_ab.svc_map)) { + struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev); + + spin_lock_bh(&ar->data_lock); + noise_floor = ath12k_pdev_get_noise_floor(ar); + spin_unlock_bh(&ar->data_lock); + + info->status.ack_signal += noise_floor; + } + + info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID; + } + break; + case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX: + if (info->flags & IEEE80211_TX_CTL_NO_ACK) { + info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; + break; + } + fallthrough; + case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU: + case HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD: + case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES: + /* The failure status is due to internal firmware tx failure + * hence drop the frame; do not update the status of frame to + * the upper layer + */ + ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu); + goto exit; + default: + ath12k_dbg(ab, ATH12K_DBG_DP_TX, "tx frame is not acked status %d\n", + ts->status); + break; + } + + /* NOTE: Tx rate status reporting. Tx completion status does not have + * necessary information (for example nss) to build the tx rate. + * Might end up reporting it out-of-band from HTT stats. + */ + + ath12k_wifi7_dp_tx_update_txcompl(dp_pdev, ts); + + peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, ts->peer_id); + if (!peer || !peer->sta) { + ath12k_err(ab, + "dp_tx: failed to find the peer with peer_id %d\n", + ts->peer_id); + ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu); + goto exit; + } + + status.sta = peer->sta; + status.info = info; + status.skb = msdu; + rate = peer->last_txrate; + + status_rate.rate_idx = rate; + status_rate.try_count = 1; + + status.rates = &status_rate; + status.n_rates = 1; + ieee80211_tx_status_ext(ath12k_pdev_dp_to_hw(dp_pdev), &status); + +exit: + rcu_read_unlock(); +} + +static void +ath12k_wifi7_dp_tx_status_parse(struct ath12k_dp *dp, + struct hal_wbm_completion_ring_tx *desc, + struct hal_tx_status *ts) +{ + u32 info0 = le32_to_cpu(desc->rate_stats.info0); + + ts->buf_rel_source = + le32_get_bits(desc->info0, HAL_WBM_COMPL_TX_INFO0_REL_SRC_MODULE); + if (ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_FW && + ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM) + return; + + if (ts->buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) + return; + + ts->status = le32_get_bits(desc->info0, + HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON); + + ts->ppdu_id = le32_get_bits(desc->info1, + HAL_WBM_COMPL_TX_INFO1_TQM_STATUS_NUMBER); + + ts->peer_id = le32_get_bits(desc->info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID); + + ts->ack_rssi = le32_get_bits(desc->info2, + HAL_WBM_COMPL_TX_INFO2_ACK_FRAME_RSSI); + + if (info0 & HAL_TX_RATE_STATS_INFO0_VALID) { + ts->pkt_type = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_PKT_TYPE); + ts->mcs = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_MCS); + ts->sgi = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_SGI); + ts->bw = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_BW); + ts->tones = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_TONES_IN_RU); + ts->ofdma = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_OFDMA_TX); + } +} + +void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id) +{ + struct ath12k_base *ab = dp->ab; + struct ath12k_pdev_dp *dp_pdev; + int hal_ring_id = dp->tx_ring[ring_id].tcl_comp_ring.ring_id; + struct hal_srng *status_ring = &dp->hal->srng_list[hal_ring_id]; + struct ath12k_tx_desc_info *tx_desc = NULL; + struct hal_tx_status ts = {}; + struct ath12k_tx_desc_params desc_params; + struct dp_tx_ring *tx_ring = &dp->tx_ring[ring_id]; + struct hal_wbm_release_ring *desc; + u8 pdev_idx; + u64 desc_va; + enum hal_wbm_rel_src_module buf_rel_source; + enum hal_wbm_tqm_rel_reason rel_status; + + spin_lock_bh(&status_ring->lock); + + ath12k_hal_srng_access_begin(ab, status_ring); + + while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) != + tx_ring->tx_status_tail) { + desc = ath12k_hal_srng_dst_get_next_entry(ab, status_ring); + if (!desc) + break; + + memcpy(&tx_ring->tx_status[tx_ring->tx_status_head], + desc, sizeof(*desc)); + tx_ring->tx_status_head = + ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head); + } + + if (ath12k_hal_srng_dst_peek(ab, status_ring) && + (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) == + tx_ring->tx_status_tail)) { + /* TODO: Process pending tx_status messages when kfifo_is_full() */ + ath12k_warn(ab, "Unable to process some of the tx_status ring desc because status_fifo is full\n"); + } + + ath12k_hal_srng_access_end(ab, status_ring); + + spin_unlock_bh(&status_ring->lock); + + while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail) != + tx_ring->tx_status_head) { + struct hal_wbm_completion_ring_tx *tx_status; + u32 desc_id; + + tx_ring->tx_status_tail = + ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail); + tx_status = &tx_ring->tx_status[tx_ring->tx_status_tail]; + ath12k_wifi7_dp_tx_status_parse(dp, tx_status, &ts); + + if (le32_get_bits(tx_status->info0, HAL_WBM_COMPL_TX_INFO0_CC_DONE)) { + /* HW done cookie conversion */ + desc_va = ((u64)le32_to_cpu(tx_status->buf_va_hi) << 32 | + le32_to_cpu(tx_status->buf_va_lo)); + tx_desc = (struct ath12k_tx_desc_info *)((unsigned long)desc_va); + } else { + /* SW does cookie conversion to VA */ + desc_id = le32_get_bits(tx_status->buf_va_hi, + BUFFER_ADDR_INFO1_SW_COOKIE); + + tx_desc = ath12k_dp_get_tx_desc(dp, desc_id); + } + if (!tx_desc) { + ath12k_warn(ab, "unable to retrieve tx_desc!"); + continue; + } + + desc_params.mac_id = tx_desc->mac_id; + desc_params.skb = tx_desc->skb; + desc_params.skb_ext_desc = tx_desc->skb_ext_desc; + + /* Find the HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE value */ + buf_rel_source = le32_get_bits(tx_status->info0, + HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE); + dp->device_stats.tx_wbm_rel_source[buf_rel_source]++; + + rel_status = le32_get_bits(tx_status->info0, + HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON); + dp->device_stats.tqm_rel_reason[rel_status]++; + + /* Release descriptor as soon as extracting necessary info + * to reduce contention + */ + ath12k_dp_tx_release_txbuf(dp, tx_desc, tx_desc->pool_id); + if (ts.buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) { + ath12k_dp_tx_process_htt_tx_complete(dp, (void *)tx_status, + tx_ring, &desc_params); + continue; + } + + pdev_idx = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, desc_params.mac_id); + + rcu_read_lock(); + + dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx); + if (!dp_pdev) { + rcu_read_unlock(); + continue; + } + + if (atomic_dec_and_test(&dp_pdev->num_tx_pending)) + wake_up(&dp_pdev->tx_empty_waitq); + + ath12k_wifi7_dp_tx_complete_msdu(dp_pdev, &desc_params, &ts, + tx_ring->tcl_data_ring_id); + rcu_read_unlock(); + } +} + +u32 ath12k_wifi7_dp_tx_get_vdev_bank_config(struct ath12k_base *ab, + struct ath12k_link_vif *arvif) +{ + u32 bank_config = 0; + u8 link_id = arvif->link_id; + struct ath12k_vif *ahvif = arvif->ahvif; + struct ath12k_dp_vif *dp_vif = &ahvif->dp_vif; + struct ath12k_dp_link_vif *dp_link_vif; + + dp_link_vif = ath12k_dp_vif_to_dp_link_vif(dp_vif, link_id); + + /* Only valid for raw frames with HW crypto enabled. + * With SW crypto, mac80211 sets key per packet + */ + if (dp_vif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW && + test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags)) + bank_config |= + u32_encode_bits(ath12k_dp_tx_get_encrypt_type(dp_vif->key_cipher), + HAL_TX_BANK_CONFIG_ENCRYPT_TYPE); + + bank_config |= u32_encode_bits(dp_vif->tx_encap_type, + HAL_TX_BANK_CONFIG_ENCAP_TYPE); + bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) | + u32_encode_bits(0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) | + u32_encode_bits(0, HAL_TX_BANK_CONFIG_EPD); + + /* only valid if idx_lookup_override is not set in tcl_data_cmd */ + if (ahvif->vdev_type == WMI_VDEV_TYPE_STA) + bank_config |= u32_encode_bits(1, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN); + else + bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN); + + bank_config |= u32_encode_bits(dp_link_vif->hal_addr_search_flags & + HAL_TX_ADDRX_EN, + HAL_TX_BANK_CONFIG_ADDRX_EN) | + u32_encode_bits(!!(dp_link_vif->hal_addr_search_flags & + HAL_TX_ADDRY_EN), + HAL_TX_BANK_CONFIG_ADDRY_EN); + + bank_config |= u32_encode_bits(ieee80211_vif_is_mesh(ahvif->vif) ? 3 : 0, + HAL_TX_BANK_CONFIG_MESH_EN) | + u32_encode_bits(dp_link_vif->vdev_id_check_en, + HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN); + + bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID); + + return bank_config; +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h new file mode 100644 index 0000000000000..24cf7972d41ba --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_DP_TX_WIFI7_H +#define ATH12K_DP_TX_WIFI7_H + +int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *arvif, + struct sk_buff *skb, bool gsn_valid, int mcbc_gsn, + bool is_mcast); +void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id); +u32 ath12k_wifi7_dp_tx_get_vdev_bank_config(struct ath12k_base *ab, + struct ath12k_link_vif *arvif); +#endif diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal.c b/drivers/net/wireless/ath/ath12k/wifi7/hal.c new file mode 100644 index 0000000000000..bd1753ca0db67 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal.c @@ -0,0 +1,713 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#include "hw.h" +#include "hal_desc.h" +#include "../hal.h" +#include "hal.h" +#include "hal_tx.h" +#include "../debug.h" +#include "../hif.h" +#include "hal_qcn9274.h" +#include "hal_wcn7850.h" +#include "hal_qcc2072.h" + +static const struct ath12k_hw_version_map ath12k_wifi7_hw_ver_map[] = { + [ATH12K_HW_QCN9274_HW10] = { + .hal_ops = &hal_qcn9274_ops, + .hal_desc_sz = sizeof(struct hal_rx_desc_qcn9274_compact), + .tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_qcn9274, + .hal_params = &ath12k_hw_hal_params_qcn9274, + .hw_regs = &qcn9274_v1_regs, + }, + [ATH12K_HW_QCN9274_HW20] = { + .hal_ops = &hal_qcn9274_ops, + .hal_desc_sz = sizeof(struct hal_rx_desc_qcn9274_compact), + .tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_qcn9274, + .hal_params = &ath12k_hw_hal_params_qcn9274, + .hw_regs = &qcn9274_v2_regs, + }, + [ATH12K_HW_WCN7850_HW20] = { + .hal_ops = &hal_wcn7850_ops, + .hal_desc_sz = sizeof(struct hal_rx_desc_wcn7850), + .tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_wcn7850, + .hal_params = &ath12k_hw_hal_params_wcn7850, + .hw_regs = &wcn7850_regs, + }, + [ATH12K_HW_IPQ5332_HW10] = { + .hal_ops = &hal_qcn9274_ops, + .hal_desc_sz = sizeof(struct hal_rx_desc_qcn9274_compact), + .tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_qcn9274, + .hal_params = &ath12k_hw_hal_params_ipq5332, + .hw_regs = &ipq5332_regs, + }, + [ATH12K_HW_QCC2072_HW10] = { + .hal_ops = &hal_qcc2072_ops, + .hal_desc_sz = sizeof(struct hal_rx_desc_qcc2072), + .tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_wcn7850, + .hal_params = &ath12k_hw_hal_params_wcn7850, + .hw_regs = &qcc2072_regs, + }, +}; + +int ath12k_wifi7_hal_init(struct ath12k_base *ab) +{ + struct ath12k_hal *hal = &ab->hal; + + memset(hal, 0, sizeof(*hal)); + + hal->ops = ath12k_wifi7_hw_ver_map[ab->hw_rev].hal_ops; + hal->hal_desc_sz = ath12k_wifi7_hw_ver_map[ab->hw_rev].hal_desc_sz; + hal->tcl_to_wbm_rbm_map = ath12k_wifi7_hw_ver_map[ab->hw_rev].tcl_to_wbm_rbm_map; + hal->regs = ath12k_wifi7_hw_ver_map[ab->hw_rev].hw_regs; + hal->hal_params = ath12k_wifi7_hw_ver_map[ab->hw_rev].hal_params; + hal->hal_wbm_release_ring_tx_size = sizeof(struct hal_wbm_release_ring_tx); + + return 0; +} + +static unsigned int ath12k_wifi7_hal_reo1_ring_id_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_ID(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned +int ath12k_wifi7_hal_reo1_ring_msi1_base_lsb_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_MSI1_BASE_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned +int ath12k_wifi7_hal_reo1_ring_msi1_base_msb_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_MSI1_BASE_MSB(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned int ath12k_wifi7_hal_reo1_ring_msi1_data_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_MSI1_DATA(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned int ath12k_wifi7_hal_reo1_ring_base_msb_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_BASE_MSB(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned +int ath12k_wifi7_hal_reo1_ring_producer_int_setup_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_PRODUCER_INT_SETUP(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned int ath12k_wifi7_hal_reo1_ring_hp_addr_lsb_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_HP_ADDR_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned int ath12k_wifi7_hal_reo1_ring_hp_addr_msb_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_HP_ADDR_MSB(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +static unsigned int ath12k_wifi7_hal_reo1_ring_misc_offset(struct ath12k_hal *hal) +{ + return HAL_REO1_RING_MISC(hal) - HAL_REO1_RING_BASE_LSB(hal); +} + +void ath12k_wifi7_hal_ce_dst_setup(struct ath12k_base *ab, + struct hal_srng *srng, int ring_num) +{ + struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST]; + u32 addr; + u32 val; + + addr = HAL_CE_DST_RING_CTRL + + srng_config->reg_start[HAL_SRNG_REG_GRP_R0] + + ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0]; + + val = ath12k_hif_read32(ab, addr); + val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN; + val |= u32_encode_bits(srng->u.dst_ring.max_buffer_length, + HAL_CE_DST_R0_DEST_CTRL_MAX_LEN); + ath12k_hif_write32(ab, addr, val); +} + +void ath12k_wifi7_hal_srng_dst_hw_init(struct ath12k_base *ab, + struct hal_srng *srng) +{ + struct ath12k_hal *hal = &ab->hal; + u32 val; + u64 hp_addr; + u32 reg_base; + + reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; + + if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) { + ath12k_hif_write32(ab, reg_base + + ath12k_wifi7_hal_reo1_ring_msi1_base_lsb_offset(hal), + srng->msi_addr); + + val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_REO1_RING_MSI1_BASE_MSB_ADDR) | + HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE; + ath12k_hif_write32(ab, reg_base + + ath12k_wifi7_hal_reo1_ring_msi1_base_msb_offset(hal), + val); + + ath12k_hif_write32(ab, + reg_base + + ath12k_wifi7_hal_reo1_ring_msi1_data_offset(hal), + srng->msi_data); + } + + ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr); + + val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB) | + u32_encode_bits((srng->entry_size * srng->num_entries), + HAL_REO1_RING_BASE_MSB_RING_SIZE); + ath12k_hif_write32(ab, reg_base + ath12k_wifi7_hal_reo1_ring_base_msb_offset(hal), + val); + + val = u32_encode_bits(srng->ring_id, HAL_REO1_RING_ID_RING_ID) | + u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE); + ath12k_hif_write32(ab, reg_base + ath12k_wifi7_hal_reo1_ring_id_offset(hal), val); + + /* interrupt setup */ + val = u32_encode_bits((srng->intr_timer_thres_us >> 3), + HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD); + + val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size), + HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD); + + ath12k_hif_write32(ab, + reg_base + + ath12k_wifi7_hal_reo1_ring_producer_int_setup_offset(hal), + val); + + hp_addr = hal->rdp.paddr + + ((unsigned long)srng->u.dst_ring.hp_addr - + (unsigned long)hal->rdp.vaddr); + ath12k_hif_write32(ab, reg_base + + ath12k_wifi7_hal_reo1_ring_hp_addr_lsb_offset(hal), + hp_addr & HAL_ADDR_LSB_REG_MASK); + ath12k_hif_write32(ab, reg_base + + ath12k_wifi7_hal_reo1_ring_hp_addr_msb_offset(hal), + hp_addr >> HAL_ADDR_MSB_REG_SHIFT); + + /* Initialize head and tail pointers to indicate ring is empty */ + reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; + ath12k_hif_write32(ab, reg_base, 0); + ath12k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET, 0); + *srng->u.dst_ring.hp_addr = 0; + + reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; + val = 0; + if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP) + val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP; + if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP) + val |= HAL_REO1_RING_MISC_HOST_FW_SWAP; + if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP) + val |= HAL_REO1_RING_MISC_MSI_SWAP; + val |= HAL_REO1_RING_MISC_SRNG_ENABLE; + + ath12k_hif_write32(ab, reg_base + ath12k_wifi7_hal_reo1_ring_misc_offset(hal), + val); +} + +void ath12k_wifi7_hal_srng_src_hw_init(struct ath12k_base *ab, + struct hal_srng *srng) +{ + struct ath12k_hal *hal = &ab->hal; + u32 val; + u64 tp_addr; + u32 reg_base; + + reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; + + if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) { + ath12k_hif_write32(ab, reg_base + + HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(hal), + srng->msi_addr); + + val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_TCL1_RING_MSI1_BASE_MSB_ADDR) | + HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE; + ath12k_hif_write32(ab, reg_base + + HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(hal), + val); + + ath12k_hif_write32(ab, reg_base + + HAL_TCL1_RING_MSI1_DATA_OFFSET(hal), + srng->msi_data); + } + + ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr); + + val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB) | + u32_encode_bits((srng->entry_size * srng->num_entries), + HAL_TCL1_RING_BASE_MSB_RING_SIZE); + ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(hal), val); + + val = u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE); + ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(hal), val); + + val = u32_encode_bits(srng->intr_timer_thres_us, + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD); + + val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size), + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD); + + ath12k_hif_write32(ab, + reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(hal), + val); + + val = 0; + if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) { + val |= u32_encode_bits(srng->u.src_ring.low_threshold, + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD); + } + ath12k_hif_write32(ab, + reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(hal), + val); + + if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) { + tp_addr = hal->rdp.paddr + + ((unsigned long)srng->u.src_ring.tp_addr - + (unsigned long)hal->rdp.vaddr); + ath12k_hif_write32(ab, + reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(hal), + tp_addr & HAL_ADDR_LSB_REG_MASK); + ath12k_hif_write32(ab, + reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(hal), + tp_addr >> HAL_ADDR_MSB_REG_SHIFT); + } + + /* Initialize head and tail pointers to indicate ring is empty */ + reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; + ath12k_hif_write32(ab, reg_base, 0); + ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0); + *srng->u.src_ring.tp_addr = 0; + + reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0]; + val = 0; + if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP) + val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP; + if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP) + val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP; + if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP) + val |= HAL_TCL1_RING_MISC_MSI_SWAP; + + /* Loop count is not used for SRC rings */ + val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE; + + val |= HAL_TCL1_RING_MISC_SRNG_ENABLE; + + if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK) + val |= HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE; + + ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(hal), val); +} + +void ath12k_wifi7_hal_set_umac_srng_ptr_addr(struct ath12k_base *ab, + struct hal_srng *srng) +{ + u32 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2]; + + if (srng->ring_dir == HAL_SRNG_DIR_SRC) { + if (!ab->hw_params->supports_shadow_regs) { + srng->u.src_ring.hp_addr = + (u32 *)((unsigned long)ab->mem + reg_base); + } else { + ath12k_dbg(ab, ATH12K_DBG_HAL, + "hal reg_base 0x%x shadow 0x%lx\n", + reg_base, + (unsigned long)srng->u.src_ring.hp_addr - + (unsigned long)ab->mem); + } + } else { + if (!ab->hw_params->supports_shadow_regs) { + srng->u.dst_ring.tp_addr = + (u32 *)((unsigned long)ab->mem + reg_base + + (HAL_REO1_RING_TP - HAL_REO1_RING_HP)); + } else { + ath12k_dbg(ab, ATH12K_DBG_HAL, + "target_reg 0x%x shadow 0x%lx\n", + reg_base + HAL_REO1_RING_TP - HAL_REO1_RING_HP, + (unsigned long)srng->u.dst_ring.tp_addr - + (unsigned long)ab->mem); + } + } +} + +int ath12k_wifi7_hal_srng_get_ring_id(struct ath12k_hal *hal, + enum hal_ring_type type, + int ring_num, int mac_id) +{ + struct hal_srng_config *srng_config = &hal->srng_config[type]; + int ring_id; + + if (ring_num >= srng_config->max_rings) { + ath12k_warn(hal, "invalid ring number :%d\n", ring_num); + return -EINVAL; + } + + ring_id = srng_config->start_ring_id + ring_num; + if (srng_config->mac_type == ATH12K_HAL_SRNG_PMAC) + ring_id += mac_id * HAL_SRNG_RINGS_PER_PMAC; + + if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX)) + return -EINVAL; + + return ring_id; +} + +static +void ath12k_wifi7_hal_srng_update_hp_tp_addr(struct ath12k_base *ab, + int shadow_cfg_idx, + enum hal_ring_type ring_type, + int ring_num) +{ + struct hal_srng *srng; + struct ath12k_hal *hal = &ab->hal; + int ring_id; + struct hal_srng_config *srng_config = &hal->srng_config[ring_type]; + + ring_id = ath12k_wifi7_hal_srng_get_ring_id(hal, ring_type, ring_num, + 0); + if (ring_id < 0) + return; + + srng = &hal->srng_list[ring_id]; + + if (srng_config->ring_dir == HAL_SRNG_DIR_DST) + srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) + + (unsigned long)ab->mem); + else + srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) + + (unsigned long)ab->mem); +} + +u32 ath12k_wifi7_hal_ce_get_desc_size(enum hal_ce_desc type) +{ + switch (type) { + case HAL_CE_DESC_SRC: + return sizeof(struct hal_ce_srng_src_desc); + case HAL_CE_DESC_DST: + return sizeof(struct hal_ce_srng_dest_desc); + case HAL_CE_DESC_DST_STATUS: + return sizeof(struct hal_ce_srng_dst_status_desc); + } + + return 0; +} + +int ath12k_wifi7_hal_srng_update_shadow_config(struct ath12k_base *ab, + enum hal_ring_type ring_type, + int ring_num) +{ + struct ath12k_hal *hal = &ab->hal; + struct hal_srng_config *srng_config = &hal->srng_config[ring_type]; + int shadow_cfg_idx = hal->num_shadow_reg_configured; + u32 target_reg; + + if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS_MAX) + return -EINVAL; + + hal->num_shadow_reg_configured++; + + target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START]; + target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] * + ring_num; + + /* For destination ring, shadow the TP */ + if (srng_config->ring_dir == HAL_SRNG_DIR_DST) + target_reg += HAL_OFFSET_FROM_HP_TO_TP; + + hal->shadow_reg_addr[shadow_cfg_idx] = target_reg; + + /* update hp/tp addr to hal structure*/ + ath12k_wifi7_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type, + ring_num); + + ath12k_dbg(ab, ATH12K_DBG_HAL, + "target_reg %x, shadow reg 0x%x shadow_idx 0x%x, ring_type %d, ring num %d", + target_reg, + HAL_SHADOW_REG(shadow_cfg_idx), + shadow_cfg_idx, + ring_type, ring_num); + + return 0; +} + +void ath12k_wifi7_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc, + dma_addr_t paddr, + u32 len, u32 id, u8 byte_swap_data) +{ + desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK); + desc->buffer_addr_info = + le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI) | + le32_encode_bits(byte_swap_data, + HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP) | + le32_encode_bits(0, HAL_CE_SRC_DESC_ADDR_INFO_GATHER) | + le32_encode_bits(len, HAL_CE_SRC_DESC_ADDR_INFO_LEN); + desc->meta_info = le32_encode_bits(id, HAL_CE_SRC_DESC_META_INFO_DATA); +} + +void ath12k_wifi7_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc, + dma_addr_t paddr) +{ + desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK); + desc->buffer_addr_info = + le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI); +} + +void ath12k_wifi7_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, + u32 cookie, dma_addr_t paddr, + enum hal_rx_buf_return_buf_manager rbm) +{ + desc->buf_addr_info.info0 = le32_encode_bits((paddr & HAL_ADDR_LSB_REG_MASK), + BUFFER_ADDR_INFO0_ADDR); + desc->buf_addr_info.info1 = + le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT), + BUFFER_ADDR_INFO1_ADDR) | + le32_encode_bits(rbm, BUFFER_ADDR_INFO1_RET_BUF_MGR) | + le32_encode_bits(cookie, BUFFER_ADDR_INFO1_SW_COOKIE); +} + +u32 ath12k_wifi7_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc) +{ + u32 len; + + len = le32_get_bits(READ_ONCE(desc->flags), HAL_CE_DST_STATUS_DESC_FLAGS_LEN); + desc->flags &= ~cpu_to_le32(HAL_CE_DST_STATUS_DESC_FLAGS_LEN); + + return len; +} + +void +ath12k_wifi7_hal_setup_link_idle_list(struct ath12k_base *ab, + struct hal_wbm_idle_scatter_list *sbuf, + u32 nsbufs, u32 tot_link_desc, + u32 end_offset) +{ + struct ath12k_hal *hal = &ab->hal; + struct ath12k_buffer_addr *link_addr; + int i; + u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64; + u32 val; + + link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE; + + for (i = 1; i < nsbufs; i++) { + link_addr->info0 = cpu_to_le32(sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK); + + link_addr->info1 = + le32_encode_bits((u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT, + HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) | + le32_encode_bits(BASE_ADDR_MATCH_TAG_VAL, + HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG); + + link_addr = (void *)sbuf[i].vaddr + + HAL_WBM_IDLE_SCATTER_BUF_SIZE; + } + + val = u32_encode_bits(reg_scatter_buf_sz, HAL_WBM_SCATTER_BUFFER_SIZE) | + u32_encode_bits(0x1, HAL_WBM_LINK_DESC_IDLE_LIST_MODE); + + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(hal), + val); + + val = u32_encode_bits(reg_scatter_buf_sz * nsbufs, + HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(hal), + val); + + val = u32_encode_bits(sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK, + BUFFER_ADDR_INFO0_ADDR); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_RING_BASE_LSB(hal), + val); + + val = u32_encode_bits(BASE_ADDR_MATCH_TAG_VAL, + HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG) | + u32_encode_bits((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT, + HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_RING_BASE_MSB(hal), + val); + + /* Setup head and tail pointers for the idle list */ + val = u32_encode_bits(sbuf[nsbufs - 1].paddr, BUFFER_ADDR_INFO0_ADDR); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(hal), + val); + + val = u32_encode_bits(((u64)sbuf[nsbufs - 1].paddr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) | + u32_encode_bits((end_offset >> 2), + HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(hal), + val); + + val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(hal), + val); + + val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(hal), + val); + + val = u32_encode_bits(((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT), + HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) | + u32_encode_bits(0, HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(hal), + val); + + val = 2 * tot_link_desc; + ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(hal), + val); + + /* Enable the SRNG */ + val = u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE) | + u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE); + ath12k_hif_write32(ab, + HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_IDLE_LINK_RING_MISC_ADDR(hal), + val); +} + +void ath12k_wifi7_hal_tx_configure_bank_register(struct ath12k_base *ab, + u32 bank_config, + u8 bank_id) +{ + ath12k_hif_write32(ab, HAL_TCL_SW_CONFIG_BANK_ADDR + 4 * bank_id, + bank_config); +} + +void ath12k_wifi7_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab) +{ + struct ath12k_hal *hal = &ab->hal; + + u32 val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + + HAL_REO1_QDESC_ADDR(hal)); + + ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_ADDR(hal), + val | HAL_REO_QDESC_ADDR_READ_LUT_ENABLE); +} + +void ath12k_wifi7_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab) +{ + struct ath12k_hal *hal = &ab->hal; + + ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_MAX_PEERID(hal), + HAL_REO_QDESC_MAX_PEERID); +} + +void ath12k_wifi7_hal_write_reoq_lut_addr(struct ath12k_base *ab, + dma_addr_t paddr) +{ + ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + + HAL_REO1_QDESC_LUT_BASE0(&ab->hal), paddr); +} + +void ath12k_wifi7_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab, + dma_addr_t paddr) +{ + ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + + HAL_REO1_QDESC_LUT_BASE1(&ab->hal), paddr); +} + +void ath12k_wifi7_hal_cc_config(struct ath12k_base *ab) +{ + u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; + u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG; + u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG; + u32 val = 0; + struct ath12k_hal *hal = &ab->hal; + + if (ath12k_ftm_mode) + return; + + ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG0(hal), cmem_base); + + val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB, + HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | + u32_encode_bits(ATH12K_CC_PPT_MSB, + HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) | + u32_encode_bits(ATH12K_CC_SPT_MSB, + HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) | + u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ALIGN) | + u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ENABLE) | + u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE); + + ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG1(hal), val); + + /* Enable HW CC for WBM */ + ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG0, cmem_base); + + val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB, + HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | + u32_encode_bits(ATH12K_CC_PPT_MSB, + HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) | + u32_encode_bits(ATH12K_CC_SPT_MSB, + HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) | + u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ALIGN); + + ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG1, val); + + /* Enable conversion complete indication */ + val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2); + val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) | + u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) | + u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN); + + ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2, val); + + /* Enable Cookie conversion for WBM2SW Rings */ + val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG); + val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) | + hal->hal_params->wbm2sw_cc_enable; + + ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, val); +} + +enum hal_rx_buf_return_buf_manager +ath12k_wifi7_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id) +{ + switch (device_id) { + case 0: + return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST; + case 1: + return HAL_RX_BUF_RBM_WBM_DEV1_IDLE_DESC_LIST; + case 2: + return HAL_RX_BUF_RBM_WBM_DEV2_IDLE_DESC_LIST; + default: + ath12k_warn(hal, + "invalid %d device id, so choose default rbm\n", + device_id); + WARN_ON(1); + return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST; + } +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal.h b/drivers/net/wireless/ath/ath12k/wifi7/hal.h new file mode 100644 index 0000000000000..9337225a52533 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal.h @@ -0,0 +1,561 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_HAL_WIFI7_H +#define ATH12K_HAL_WIFI7_H + +#include "../core.h" +#include "../hal.h" +#include "hal_desc.h" +#include "hal_tx.h" +#include "hal_rx.h" +#include "hal_rx_desc.h" + +/* calculate the register address from bar0 of shadow register x */ +#define HAL_SHADOW_BASE_ADDR 0x000008fc +#define HAL_SHADOW_NUM_REGS 40 +#define HAL_HP_OFFSET_IN_REG_START 1 +#define HAL_OFFSET_FROM_HP_TO_TP 4 + +#define HAL_SHADOW_REG(x) (HAL_SHADOW_BASE_ADDR + (4 * (x))) +#define HAL_REO_QDESC_MAX_PEERID 8191 + +/* WCSS Relative address */ +#define HAL_SEQ_WCSS_CMEM_OFFSET 0x00100000 +#define HAL_SEQ_WCSS_UMAC_OFFSET 0x00a00000 +#define HAL_SEQ_WCSS_UMAC_REO_REG 0x00a38000 +#define HAL_SEQ_WCSS_UMAC_TCL_REG 0x00a44000 +#define HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) \ + ((hal)->regs->umac_ce0_src_reg_base) +#define HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) \ + ((hal)->regs->umac_ce0_dest_reg_base) +#define HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) \ + ((hal)->regs->umac_ce1_src_reg_base) +#define HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) \ + ((hal)->regs->umac_ce1_dest_reg_base) +#define HAL_SEQ_WCSS_UMAC_WBM_REG 0x00a34000 + +#define HAL_CE_WFSS_CE_REG_BASE 0x01b80000 + +#define HAL_TCL_SW_CONFIG_BANK_ADDR 0x00a4408c + +/* SW2TCL(x) R0 ring configuration address */ +#define HAL_TCL1_RING_CMN_CTRL_REG 0x00000020 +#define HAL_TCL1_RING_DSCP_TID_MAP 0x00000240 + +#define HAL_TCL1_RING_BASE_LSB(hal) \ + ((hal)->regs->tcl1_ring_base_lsb) +#define HAL_TCL1_RING_BASE_MSB(hal) \ + ((hal)->regs->tcl1_ring_base_msb) +#define HAL_TCL1_RING_ID(hal) ((hal)->regs->tcl1_ring_id) +#define HAL_TCL1_RING_MISC(hal) \ + ((hal)->regs->tcl1_ring_misc) +#define HAL_TCL1_RING_TP_ADDR_LSB(hal) \ + ((hal)->regs->tcl1_ring_tp_addr_lsb) +#define HAL_TCL1_RING_TP_ADDR_MSB(hal) \ + ((hal)->regs->tcl1_ring_tp_addr_msb) +#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(hal) \ + ((hal)->regs->tcl1_ring_consumer_int_setup_ix0) +#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(hal) \ + ((hal)->regs->tcl1_ring_consumer_int_setup_ix1) +#define HAL_TCL1_RING_MSI1_BASE_LSB(hal) \ + ((hal)->regs->tcl1_ring_msi1_base_lsb) +#define HAL_TCL1_RING_MSI1_BASE_MSB(hal) \ + ((hal)->regs->tcl1_ring_msi1_base_msb) +#define HAL_TCL1_RING_MSI1_DATA(hal) \ + ((hal)->regs->tcl1_ring_msi1_data) +#define HAL_TCL2_RING_BASE_LSB(hal) \ + ((hal)->regs->tcl2_ring_base_lsb) +#define HAL_TCL_RING_BASE_LSB(hal) \ + ((hal)->regs->tcl_ring_base_lsb) + +#define HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_MSI1_BASE_LSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_MSI1_BASE_MSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_MSI1_DATA_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_MSI1_DATA(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_BASE_MSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_BASE_MSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_ID_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_ID(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_TP_ADDR_LSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_TP_ADDR_MSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) +#define HAL_TCL1_RING_MISC_OFFSET(hal) ({ typeof(hal) _hal = (hal); \ + (HAL_TCL1_RING_MISC(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); }) + +/* SW2TCL(x) R2 ring pointers (head/tail) address */ +#define HAL_TCL1_RING_HP 0x00002000 +#define HAL_TCL1_RING_TP 0x00002004 +#define HAL_TCL2_RING_HP 0x00002008 +#define HAL_TCL_RING_HP 0x00002028 + +#define HAL_TCL1_RING_TP_OFFSET \ + (HAL_TCL1_RING_TP - HAL_TCL1_RING_HP) + +/* TCL STATUS ring address */ +#define HAL_TCL_STATUS_RING_BASE_LSB(hal) \ + ((hal)->regs->tcl_status_ring_base_lsb) +#define HAL_TCL_STATUS_RING_HP 0x00002048 + +/* PPE2TCL1 Ring address */ +#define HAL_TCL_PPE2TCL1_RING_BASE_LSB 0x00000c48 +#define HAL_TCL_PPE2TCL1_RING_HP 0x00002038 + +/* WBM PPE Release Ring address */ +#define HAL_WBM_PPE_RELEASE_RING_BASE_LSB(hal) \ + ((hal)->regs->ppe_rel_ring_base) +#define HAL_WBM_PPE_RELEASE_RING_HP 0x00003020 + +/* REO2SW(x) R0 ring configuration address */ +#define HAL_REO1_GEN_ENABLE 0x00000000 +#define HAL_REO1_MISC_CTRL_ADDR(hal) \ + ((hal)->regs->reo1_misc_ctrl_addr) +#define HAL_REO1_DEST_RING_CTRL_IX_0 0x00000004 +#define HAL_REO1_DEST_RING_CTRL_IX_1 0x00000008 +#define HAL_REO1_DEST_RING_CTRL_IX_2 0x0000000c +#define HAL_REO1_DEST_RING_CTRL_IX_3 0x00000010 +#define HAL_REO1_QDESC_ADDR(hal) ((hal)->regs->reo1_qdesc_addr) +#define HAL_REO1_QDESC_MAX_PEERID(hal) ((hal)->regs->reo1_qdesc_max_peerid) +#define HAL_REO1_SW_COOKIE_CFG0(hal) ((hal)->regs->reo1_sw_cookie_cfg0) +#define HAL_REO1_SW_COOKIE_CFG1(hal) ((hal)->regs->reo1_sw_cookie_cfg1) +#define HAL_REO1_QDESC_LUT_BASE0(hal) ((hal)->regs->reo1_qdesc_lut_base0) +#define HAL_REO1_QDESC_LUT_BASE1(hal) ((hal)->regs->reo1_qdesc_lut_base1) +#define HAL_REO1_RING_BASE_LSB(hal) ((hal)->regs->reo1_ring_base_lsb) +#define HAL_REO1_RING_BASE_MSB(hal) ((hal)->regs->reo1_ring_base_msb) +#define HAL_REO1_RING_ID(hal) ((hal)->regs->reo1_ring_id) +#define HAL_REO1_RING_MISC(hal) ((hal)->regs->reo1_ring_misc) +#define HAL_REO1_RING_HP_ADDR_LSB(hal) ((hal)->regs->reo1_ring_hp_addr_lsb) +#define HAL_REO1_RING_HP_ADDR_MSB(hal) ((hal)->regs->reo1_ring_hp_addr_msb) +#define HAL_REO1_RING_PRODUCER_INT_SETUP(hal) \ + ((hal)->regs->reo1_ring_producer_int_setup) +#define HAL_REO1_RING_MSI1_BASE_LSB(hal) \ + ((hal)->regs->reo1_ring_msi1_base_lsb) +#define HAL_REO1_RING_MSI1_BASE_MSB(hal) \ + ((hal)->regs->reo1_ring_msi1_base_msb) +#define HAL_REO1_RING_MSI1_DATA(hal) ((hal)->regs->reo1_ring_msi1_data) +#define HAL_REO2_RING_BASE_LSB(hal) ((hal)->regs->reo2_ring_base) +#define HAL_REO1_AGING_THRESH_IX_0(hal) ((hal)->regs->reo1_aging_thres_ix0) +#define HAL_REO1_AGING_THRESH_IX_1(hal) ((hal)->regs->reo1_aging_thres_ix1) +#define HAL_REO1_AGING_THRESH_IX_2(hal) ((hal)->regs->reo1_aging_thres_ix2) +#define HAL_REO1_AGING_THRESH_IX_3(hal) ((hal)->regs->reo1_aging_thres_ix3) + +/* REO2SW(x) R2 ring pointers (head/tail) address */ +#define HAL_REO1_RING_HP 0x00003048 +#define HAL_REO1_RING_TP 0x0000304c +#define HAL_REO2_RING_HP 0x00003050 + +#define HAL_REO1_RING_TP_OFFSET (HAL_REO1_RING_TP - HAL_REO1_RING_HP) + +/* REO2SW0 ring configuration address */ +#define HAL_REO_SW0_RING_BASE_LSB(hal) \ + ((hal)->regs->reo2_sw0_ring_base) + +/* REO2SW0 R2 ring pointer (head/tail) address */ +#define HAL_REO_SW0_RING_HP 0x00003088 + +/* REO CMD R0 address */ +#define HAL_REO_CMD_RING_BASE_LSB(hal) \ + ((hal)->regs->reo_cmd_ring_base) + +/* REO CMD R2 address */ +#define HAL_REO_CMD_HP 0x00003020 + +/* SW2REO R0 address */ +#define HAL_SW2REO_RING_BASE_LSB(hal) \ + ((hal)->regs->sw2reo_ring_base) +#define HAL_SW2REO1_RING_BASE_LSB(hal) \ + ((hal)->regs->sw2reo1_ring_base) + +/* SW2REO R2 address */ +#define HAL_SW2REO_RING_HP 0x00003028 +#define HAL_SW2REO1_RING_HP 0x00003030 + +/* CE ring R0 address */ +#define HAL_CE_SRC_RING_BASE_LSB 0x00000000 +#define HAL_CE_DST_RING_BASE_LSB 0x00000000 +#define HAL_CE_DST_STATUS_RING_BASE_LSB 0x00000058 +#define HAL_CE_DST_RING_CTRL 0x000000b0 + +/* CE ring R2 address */ +#define HAL_CE_DST_RING_HP 0x00000400 +#define HAL_CE_DST_STATUS_RING_HP 0x00000408 + +/* REO status address */ +#define HAL_REO_STATUS_RING_BASE_LSB(hal) \ + ((hal)->regs->reo_status_ring_base) +#define HAL_REO_STATUS_HP 0x000030a8 + +/* WBM Idle R0 address */ +#define HAL_WBM_IDLE_LINK_RING_BASE_LSB(hal) \ + ((hal)->regs->wbm_idle_ring_base_lsb) +#define HAL_WBM_IDLE_LINK_RING_MISC_ADDR(hal) \ + ((hal)->regs->wbm_idle_ring_misc_addr) +#define HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(hal) \ + ((hal)->regs->wbm_r0_idle_list_cntl_addr) +#define HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(hal) \ + ((hal)->regs->wbm_r0_idle_list_size_addr) +#define HAL_WBM_SCATTERED_RING_BASE_LSB(hal) \ + ((hal)->regs->wbm_scattered_ring_base_lsb) +#define HAL_WBM_SCATTERED_RING_BASE_MSB(hal) \ + ((hal)->regs->wbm_scattered_ring_base_msb) +#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(hal) \ + ((hal)->regs->wbm_scattered_desc_head_info_ix0) +#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(hal) \ + ((hal)->regs->wbm_scattered_desc_head_info_ix1) +#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(hal) \ + ((hal)->regs->wbm_scattered_desc_tail_info_ix0) +#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(hal) \ + ((hal)->regs->wbm_scattered_desc_tail_info_ix1) +#define HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(hal) \ + ((hal)->regs->wbm_scattered_desc_ptr_hp_addr) + +/* WBM Idle R2 address */ +#define HAL_WBM_IDLE_LINK_RING_HP 0x000030b8 + +/* SW2WBM R0 release address */ +#define HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal) \ + ((hal)->regs->wbm_sw_release_ring_base_lsb) +#define HAL_WBM_SW1_RELEASE_RING_BASE_LSB(hal) \ + ((hal)->regs->wbm_sw1_release_ring_base_lsb) + +/* SW2WBM R2 release address */ +#define HAL_WBM_SW_RELEASE_RING_HP 0x00003010 +#define HAL_WBM_SW1_RELEASE_RING_HP 0x00003018 + +/* WBM2SW R0 release address */ +#define HAL_WBM0_RELEASE_RING_BASE_LSB(hal) \ + ((hal)->regs->wbm0_release_ring_base_lsb) + +#define HAL_WBM1_RELEASE_RING_BASE_LSB(hal) \ + ((hal)->regs->wbm1_release_ring_base_lsb) + +/* WBM2SW R2 release address */ +#define HAL_WBM0_RELEASE_RING_HP 0x000030c8 +#define HAL_WBM1_RELEASE_RING_HP 0x000030d0 + +/* WBM cookie config address and mask */ +#define HAL_WBM_SW_COOKIE_CFG0 0x00000040 +#define HAL_WBM_SW_COOKIE_CFG1 0x00000044 +#define HAL_WBM_SW_COOKIE_CFG2 0x00000090 +#define HAL_WBM_SW_COOKIE_CONVERT_CFG 0x00000094 + +#define HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB GENMASK(7, 0) +#define HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB GENMASK(12, 8) +#define HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB GENMASK(17, 13) +#define HAL_WBM_SW_COOKIE_CFG_ALIGN BIT(18) +#define HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN BIT(0) +#define HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN BIT(1) +#define HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN BIT(3) + +#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN BIT(1) +#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN BIT(2) +#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN BIT(3) +#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN BIT(4) +#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN BIT(5) +#define HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN BIT(8) + +/* TCL ring field mask and offset */ +#define HAL_TCL1_RING_BASE_MSB_RING_SIZE GENMASK(27, 8) +#define HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB GENMASK(7, 0) +#define HAL_TCL1_RING_ID_ENTRY_SIZE GENMASK(7, 0) +#define HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE BIT(0) +#define HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE BIT(1) +#define HAL_TCL1_RING_MISC_MSI_SWAP BIT(3) +#define HAL_TCL1_RING_MISC_HOST_FW_SWAP BIT(4) +#define HAL_TCL1_RING_MISC_DATA_TLV_SWAP BIT(5) +#define HAL_TCL1_RING_MISC_SRNG_ENABLE BIT(6) +#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD GENMASK(31, 16) +#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD GENMASK(14, 0) +#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD GENMASK(15, 0) +#define HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE BIT(8) +#define HAL_TCL1_RING_MSI1_BASE_MSB_ADDR GENMASK(7, 0) +#define HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN BIT(23) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP GENMASK(31, 0) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP0 GENMASK(2, 0) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP1 GENMASK(5, 3) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP2 GENMASK(8, 6) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP3 GENMASK(11, 9) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP4 GENMASK(14, 12) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP5 GENMASK(17, 15) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP6 GENMASK(20, 18) +#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP7 GENMASK(23, 21) + +/* REO ring field mask and offset */ +#define HAL_REO1_RING_BASE_MSB_RING_SIZE GENMASK(27, 8) +#define HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB GENMASK(7, 0) +#define HAL_REO1_RING_ID_RING_ID GENMASK(15, 8) +#define HAL_REO1_RING_ID_ENTRY_SIZE GENMASK(7, 0) +#define HAL_REO1_RING_MISC_MSI_SWAP BIT(3) +#define HAL_REO1_RING_MISC_HOST_FW_SWAP BIT(4) +#define HAL_REO1_RING_MISC_DATA_TLV_SWAP BIT(5) +#define HAL_REO1_RING_MISC_SRNG_ENABLE BIT(6) +#define HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD GENMASK(31, 16) +#define HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD GENMASK(14, 0) +#define HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE BIT(8) +#define HAL_REO1_RING_MSI1_BASE_MSB_ADDR GENMASK(7, 0) +#define HAL_REO1_MISC_CTL_FRAG_DST_RING GENMASK(20, 17) +#define HAL_REO1_MISC_CTL_BAR_DST_RING GENMASK(24, 21) +#define HAL_REO1_GEN_ENABLE_AGING_LIST_ENABLE BIT(2) +#define HAL_REO1_GEN_ENABLE_AGING_FLUSH_ENABLE BIT(3) +#define HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB GENMASK(7, 0) +#define HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB GENMASK(12, 8) +#define HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB GENMASK(17, 13) +#define HAL_REO1_SW_COOKIE_CFG_ALIGN BIT(18) +#define HAL_REO1_SW_COOKIE_CFG_ENABLE BIT(19) +#define HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE BIT(20) +#define HAL_REO_QDESC_ADDR_READ_LUT_ENABLE BIT(7) +#define HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY BIT(6) + +/* CE ring bit field mask and shift */ +#define HAL_CE_DST_R0_DEST_CTRL_MAX_LEN GENMASK(15, 0) + +#define HAL_ADDR_LSB_REG_MASK 0xffffffff + +#define HAL_ADDR_MSB_REG_SHIFT 32 + +/* WBM ring bit field mask and shift */ +#define HAL_WBM_LINK_DESC_IDLE_LIST_MODE BIT(1) +#define HAL_WBM_SCATTER_BUFFER_SIZE GENMASK(10, 2) +#define HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST GENMASK(31, 16) +#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32 GENMASK(7, 0) +#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG GENMASK(31, 8) + +#define HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1 GENMASK(20, 8) +#define HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1 GENMASK(20, 8) + +#define HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE BIT(6) +#define HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE BIT(0) + +#define BASE_ADDR_MATCH_TAG_VAL 0x5 + +#define HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE 0x000fffff +#define HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE 0x000fffff +#define HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_REO_CMD_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE 0x000fffff +#define HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE 0x000fffff +#define HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_CE_SRC_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_CE_DST_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE 0x000fffff +#define HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE 0x0000ffff +#define HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE 0x000fffff +#define HAL_RXDMA_RING_MAX_SIZE 0x0000ffff +#define HAL_RXDMA_RING_MAX_SIZE_BE 0x000fffff +#define HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE 0x000fffff + +#define HAL_WBM2SW_REL_ERR_RING_NUM 3 +/* Add any other errors here and return them in + * ath12k_hal_rx_desc_get_err(). + */ + +#define HAL_IPQ5332_CE_WFSS_REG_BASE 0x740000 +#define HAL_IPQ5332_CE_SIZE 0x100000 + +#define HAL_RX_MAX_BA_WINDOW 256 + +#define HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC (100 * 1000) +#define HAL_DEFAULT_VO_REO_TIMEOUT_USEC (40 * 1000) + +#define HAL_SRNG_DESC_LOOP_CNT 0xf0000000 + +#define HAL_REO_CMD_FLG_NEED_STATUS BIT(0) +#define HAL_REO_CMD_FLG_STATS_CLEAR BIT(1) +#define HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER BIT(2) +#define HAL_REO_CMD_FLG_FLUSH_RELEASE_BLOCKING BIT(3) +#define HAL_REO_CMD_FLG_FLUSH_NO_INVAL BIT(4) +#define HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS BIT(5) +#define HAL_REO_CMD_FLG_FLUSH_ALL BIT(6) +#define HAL_REO_CMD_FLG_UNBLK_RESOURCE BIT(7) +#define HAL_REO_CMD_FLG_UNBLK_CACHE BIT(8) +#define HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC BIT(9) + +/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO0_UPD_* fields */ +#define HAL_REO_CMD_UPD0_RX_QUEUE_NUM BIT(8) +#define HAL_REO_CMD_UPD0_VLD BIT(9) +#define HAL_REO_CMD_UPD0_ALDC BIT(10) +#define HAL_REO_CMD_UPD0_DIS_DUP_DETECTION BIT(11) +#define HAL_REO_CMD_UPD0_SOFT_REORDER_EN BIT(12) +#define HAL_REO_CMD_UPD0_AC BIT(13) +#define HAL_REO_CMD_UPD0_BAR BIT(14) +#define HAL_REO_CMD_UPD0_RETRY BIT(15) +#define HAL_REO_CMD_UPD0_CHECK_2K_MODE BIT(16) +#define HAL_REO_CMD_UPD0_OOR_MODE BIT(17) +#define HAL_REO_CMD_UPD0_BA_WINDOW_SIZE BIT(18) +#define HAL_REO_CMD_UPD0_PN_CHECK BIT(19) +#define HAL_REO_CMD_UPD0_EVEN_PN BIT(20) +#define HAL_REO_CMD_UPD0_UNEVEN_PN BIT(21) +#define HAL_REO_CMD_UPD0_PN_HANDLE_ENABLE BIT(22) +#define HAL_REO_CMD_UPD0_PN_SIZE BIT(23) +#define HAL_REO_CMD_UPD0_IGNORE_AMPDU_FLG BIT(24) +#define HAL_REO_CMD_UPD0_SVLD BIT(25) +#define HAL_REO_CMD_UPD0_SSN BIT(26) +#define HAL_REO_CMD_UPD0_SEQ_2K_ERR BIT(27) +#define HAL_REO_CMD_UPD0_PN_ERR BIT(28) +#define HAL_REO_CMD_UPD0_PN_VALID BIT(29) +#define HAL_REO_CMD_UPD0_PN BIT(30) + +/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO1_* fields */ +#define HAL_REO_CMD_UPD1_VLD BIT(16) +#define HAL_REO_CMD_UPD1_ALDC GENMASK(18, 17) +#define HAL_REO_CMD_UPD1_DIS_DUP_DETECTION BIT(19) +#define HAL_REO_CMD_UPD1_SOFT_REORDER_EN BIT(20) +#define HAL_REO_CMD_UPD1_AC GENMASK(22, 21) +#define HAL_REO_CMD_UPD1_BAR BIT(23) +#define HAL_REO_CMD_UPD1_RETRY BIT(24) +#define HAL_REO_CMD_UPD1_CHECK_2K_MODE BIT(25) +#define HAL_REO_CMD_UPD1_OOR_MODE BIT(26) +#define HAL_REO_CMD_UPD1_PN_CHECK BIT(27) +#define HAL_REO_CMD_UPD1_EVEN_PN BIT(28) +#define HAL_REO_CMD_UPD1_UNEVEN_PN BIT(29) +#define HAL_REO_CMD_UPD1_PN_HANDLE_ENABLE BIT(30) +#define HAL_REO_CMD_UPD1_IGNORE_AMPDU_FLG BIT(31) + +/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO2_* fields */ +#define HAL_REO_CMD_UPD2_SVLD BIT(10) +#define HAL_REO_CMD_UPD2_SSN GENMASK(22, 11) +#define HAL_REO_CMD_UPD2_SEQ_2K_ERR BIT(23) +#define HAL_REO_CMD_UPD2_PN_ERR BIT(24) + +struct hal_reo_status_queue_stats { + u16 ssn; + u16 curr_idx; + u32 pn[4]; + u32 last_rx_queue_ts; + u32 last_rx_dequeue_ts; + u32 rx_bitmap[8]; /* Bitmap from 0-255 */ + u32 curr_mpdu_cnt; + u32 curr_msdu_cnt; + u16 fwd_due_to_bar_cnt; + u16 dup_cnt; + u32 frames_in_order_cnt; + u32 num_mpdu_processed_cnt; + u32 num_msdu_processed_cnt; + u32 total_num_processed_byte_cnt; + u32 late_rx_mpdu_cnt; + u32 reorder_hole_cnt; + u8 timeout_cnt; + u8 bar_rx_cnt; + u8 num_window_2k_jump_cnt; +}; + +struct hal_reo_status_flush_queue { + bool err_detected; +}; + +enum hal_reo_status_flush_cache_err_code { + HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_SUCCESS, + HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_IN_USE, + HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_NOT_FOUND, +}; + +struct hal_reo_status_flush_cache { + bool err_detected; + enum hal_reo_status_flush_cache_err_code err_code; + bool cache_controller_flush_status_hit; + u8 cache_controller_flush_status_desc_type; + u8 cache_controller_flush_status_client_id; + u8 cache_controller_flush_status_err; + u8 cache_controller_flush_status_cnt; +}; + +enum hal_reo_status_unblock_cache_type { + HAL_REO_STATUS_UNBLOCK_BLOCKING_RESOURCE, + HAL_REO_STATUS_UNBLOCK_ENTIRE_CACHE_USAGE, +}; + +struct hal_reo_status_unblock_cache { + bool err_detected; + enum hal_reo_status_unblock_cache_type unblock_type; +}; + +struct hal_reo_status_flush_timeout_list { + bool err_detected; + bool list_empty; + u16 release_desc_cnt; + u16 fwd_buf_cnt; +}; + +enum hal_reo_threshold_idx { + HAL_REO_THRESHOLD_IDX_DESC_COUNTER0, + HAL_REO_THRESHOLD_IDX_DESC_COUNTER1, + HAL_REO_THRESHOLD_IDX_DESC_COUNTER2, + HAL_REO_THRESHOLD_IDX_DESC_COUNTER_SUM, +}; + +struct hal_reo_status_desc_thresh_reached { + enum hal_reo_threshold_idx threshold_idx; + u32 link_desc_counter0; + u32 link_desc_counter1; + u32 link_desc_counter2; + u32 link_desc_counter_sum; +}; + +struct hal_reo_status { + struct hal_reo_status_header uniform_hdr; + u8 loop_cnt; + union { + struct hal_reo_status_queue_stats queue_stats; + struct hal_reo_status_flush_queue flush_queue; + struct hal_reo_status_flush_cache flush_cache; + struct hal_reo_status_unblock_cache unblock_cache; + struct hal_reo_status_flush_timeout_list timeout_list; + struct hal_reo_status_desc_thresh_reached desc_thresh_reached; + } u; +}; + +int ath12k_wifi7_hal_init(struct ath12k_base *ab); +void ath12k_wifi7_hal_ce_dst_setup(struct ath12k_base *ab, + struct hal_srng *srng, int ring_num); +void ath12k_wifi7_hal_srng_dst_hw_init(struct ath12k_base *ab, + struct hal_srng *srng); +void ath12k_wifi7_hal_srng_src_hw_init(struct ath12k_base *ab, + struct hal_srng *srng); +void ath12k_wifi7_hal_set_umac_srng_ptr_addr(struct ath12k_base *ab, + struct hal_srng *srng); +int ath12k_wifi7_hal_srng_update_shadow_config(struct ath12k_base *ab, + enum hal_ring_type ring_type, + int ring_num); +int ath12k_wifi7_hal_srng_get_ring_id(struct ath12k_hal *hal, + enum hal_ring_type type, + int ring_num, int mac_id); +u32 ath12k_wifi7_hal_ce_get_desc_size(enum hal_ce_desc type); +void ath12k_wifi7_hal_cc_config(struct ath12k_base *ab); +enum hal_rx_buf_return_buf_manager +ath12k_wifi7_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id); +void ath12k_wifi7_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc, + dma_addr_t paddr, + u32 len, u32 id, u8 byte_swap_data); +void ath12k_wifi7_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc, + dma_addr_t paddr); +void +ath12k_wifi7_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, + u32 cookie, dma_addr_t paddr, + enum hal_rx_buf_return_buf_manager rbm); +u32 +ath12k_wifi7_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc); +void +ath12k_wifi7_hal_setup_link_idle_list(struct ath12k_base *ab, + struct hal_wbm_idle_scatter_list *sbuf, + u32 nsbufs, u32 tot_link_desc, + u32 end_offset); +void ath12k_wifi7_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab); +void ath12k_wifi7_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab); +void ath12k_wifi7_hal_write_reoq_lut_addr(struct ath12k_base *ab, + dma_addr_t paddr); +void ath12k_wifi7_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab, + dma_addr_t paddr); +u32 ath12k_wifi7_hal_reo_qdesc_size(u32 ba_window_size, u8 tid); +#endif diff --git a/drivers/net/wireless/ath/ath12k/hal_desc.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_desc.h similarity index 91% rename from drivers/net/wireless/ath/ath12k/hal_desc.h rename to drivers/net/wireless/ath/ath12k/wifi7/hal_desc.h index 13ddac4a94125..e1ab47b44433d 100644 --- a/drivers/net/wireless/ath/ath12k/hal_desc.h +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_desc.h @@ -1,92 +1,13 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ -#include "core.h" +#include "../core.h" #ifndef ATH12K_HAL_DESC_H #define ATH12K_HAL_DESC_H -#define BUFFER_ADDR_INFO0_ADDR GENMASK(31, 0) - -#define BUFFER_ADDR_INFO1_ADDR GENMASK(7, 0) -#define BUFFER_ADDR_INFO1_RET_BUF_MGR GENMASK(11, 8) -#define BUFFER_ADDR_INFO1_SW_COOKIE GENMASK(31, 12) - -struct ath12k_buffer_addr { - __le32 info0; - __le32 info1; -} __packed; - -/* ath12k_buffer_addr - * - * buffer_addr_31_0 - * Address (lower 32 bits) of the MSDU buffer or MSDU_EXTENSION - * descriptor or Link descriptor - * - * buffer_addr_39_32 - * Address (upper 8 bits) of the MSDU buffer or MSDU_EXTENSION - * descriptor or Link descriptor - * - * return_buffer_manager (RBM) - * Consumer: WBM - * Producer: SW/FW - * Indicates to which buffer manager the buffer or MSDU_EXTENSION - * descriptor or link descriptor that is being pointed to shall be - * returned after the frame has been processed. It is used by WBM - * for routing purposes. - * - * Values are defined in enum %HAL_RX_BUF_RBM_ - * - * sw_buffer_cookie - * Cookie field exclusively used by SW. HW ignores the contents, - * accept that it passes the programmed value on to other - * descriptors together with the physical address. - * - * Field can be used by SW to for example associate the buffers - * physical address with the virtual address. - * - * NOTE1: - * The three most significant bits can have a special meaning - * in case this struct is embedded in a TX_MPDU_DETAILS STRUCT, - * and field transmit_bw_restriction is set - * - * In case of NON punctured transmission: - * Sw_buffer_cookie[19:17] = 3'b000: 20 MHz TX only - * Sw_buffer_cookie[19:17] = 3'b001: 40 MHz TX only - * Sw_buffer_cookie[19:17] = 3'b010: 80 MHz TX only - * Sw_buffer_cookie[19:17] = 3'b011: 160 MHz TX only - * Sw_buffer_cookie[19:17] = 3'b101: 240 MHz TX only - * Sw_buffer_cookie[19:17] = 3'b100: 320 MHz TX only - * Sw_buffer_cookie[19:18] = 2'b11: reserved - * - * In case of punctured transmission: - * Sw_buffer_cookie[19:16] = 4'b0000: pattern 0 only - * Sw_buffer_cookie[19:16] = 4'b0001: pattern 1 only - * Sw_buffer_cookie[19:16] = 4'b0010: pattern 2 only - * Sw_buffer_cookie[19:16] = 4'b0011: pattern 3 only - * Sw_buffer_cookie[19:16] = 4'b0100: pattern 4 only - * Sw_buffer_cookie[19:16] = 4'b0101: pattern 5 only - * Sw_buffer_cookie[19:16] = 4'b0110: pattern 6 only - * Sw_buffer_cookie[19:16] = 4'b0111: pattern 7 only - * Sw_buffer_cookie[19:16] = 4'b1000: pattern 8 only - * Sw_buffer_cookie[19:16] = 4'b1001: pattern 9 only - * Sw_buffer_cookie[19:16] = 4'b1010: pattern 10 only - * Sw_buffer_cookie[19:16] = 4'b1011: pattern 11 only - * Sw_buffer_cookie[19:18] = 2'b11: reserved - * - * Note: a punctured transmission is indicated by the presence - * of TLV TX_PUNCTURE_SETUP embedded in the scheduler TLV - * - * Sw_buffer_cookie[20:17]: Tid: The TID field in the QoS control - * field - * - * Sw_buffer_cookie[16]: Mpdu_qos_control_valid: This field - * indicates MPDUs with a QoS control field. - * - */ - enum hal_tlv_tag { HAL_MACTX_CBF_START = 0 /* 0x0 */, HAL_PHYRX_DATA = 1 /* 0x1 */, @@ -566,27 +487,6 @@ enum hal_tlv_tag { HAL_TLV_BASE = 511 /* 0x1ff */, }; -#define HAL_TLV_HDR_TAG GENMASK(9, 1) -#define HAL_TLV_HDR_LEN GENMASK(25, 10) -#define HAL_TLV_USR_ID GENMASK(31, 26) - -#define HAL_TLV_ALIGN 4 - -struct hal_tlv_hdr { - __le32 tl; - u8 value[]; -} __packed; - -#define HAL_TLV_64_HDR_TAG GENMASK(9, 1) -#define HAL_TLV_64_HDR_LEN GENMASK(21, 10) -#define HAL_TLV_64_USR_ID GENMASK(31, 26) -#define HAL_TLV_64_ALIGN 8 - -struct hal_tlv_64_hdr { - __le64 tl; - u8 value[]; -} __packed; - #define RX_MPDU_DESC_INFO0_MSDU_COUNT GENMASK(7, 0) #define RX_MPDU_DESC_INFO0_FRAG_FLAG BIT(8) #define RX_MPDU_DESC_INFO0_MPDU_RETRY BIT(9) @@ -820,35 +720,6 @@ struct rx_msdu_ext_desc { * Set to the link ID of the PMAC that received the frame */ -enum hal_reo_dest_ring_buffer_type { - HAL_REO_DEST_RING_BUFFER_TYPE_MSDU, - HAL_REO_DEST_RING_BUFFER_TYPE_LINK_DESC, -}; - -enum hal_reo_dest_ring_push_reason { - HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED, - HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION, -}; - -enum hal_reo_dest_ring_error_code { - HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO, - HAL_REO_DEST_RING_ERROR_CODE_DESC_INVALID, - HAL_REO_DEST_RING_ERROR_CODE_AMPDU_IN_NON_BA, - HAL_REO_DEST_RING_ERROR_CODE_NON_BA_DUPLICATE, - HAL_REO_DEST_RING_ERROR_CODE_BA_DUPLICATE, - HAL_REO_DEST_RING_ERROR_CODE_FRAME_2K_JUMP, - HAL_REO_DEST_RING_ERROR_CODE_BAR_2K_JUMP, - HAL_REO_DEST_RING_ERROR_CODE_FRAME_OOR, - HAL_REO_DEST_RING_ERROR_CODE_BAR_OOR, - HAL_REO_DEST_RING_ERROR_CODE_NO_BA_SESSION, - HAL_REO_DEST_RING_ERROR_CODE_FRAME_SN_EQUALS_SSN, - HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED, - HAL_REO_DEST_RING_ERROR_CODE_2K_ERR_FLAG_SET, - HAL_REO_DEST_RING_ERROR_CODE_PN_ERR_FLAG_SET, - HAL_REO_DEST_RING_ERROR_CODE_DESC_BLOCKED, - HAL_REO_DEST_RING_ERROR_CODE_MAX, -}; - #define HAL_REO_DEST_RING_INFO0_BUFFER_TYPE BIT(0) #define HAL_REO_DEST_RING_INFO0_PUSH_REASON GENMASK(2, 1) #define HAL_REO_DEST_RING_INFO0_ERROR_CODE GENMASK(7, 3) @@ -986,35 +857,6 @@ struct hal_reo_to_ppe_ring { * More Segments followed */ -enum hal_reo_entr_rxdma_push_reason { - HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ERR_DETECTED, - HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ROUTING_INSTRUCTION, - HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_RX_FLUSH, -}; - -enum hal_reo_entr_rxdma_ecode { - HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_FCS_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_UNECRYPTED_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LEN_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LIMIT_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_WIFI_PARSE_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_PARSE_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_SA_TIMEOUT_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_DA_TIMEOUT_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_FLOW_TIMEOUT_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_FRAG_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_MULTICAST_ECHO_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_MISMATCH_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_UNAUTH_WDS_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_GRPCAST_AMSDU_WDS_ERR, - HAL_REO_ENTR_RING_RXDMA_ECODE_MAX, -}; - enum hal_rx_reo_dest_ring { HAL_RX_REO_DEST_RING_TCL, HAL_RX_REO_DEST_RING_SW1, @@ -1207,6 +1049,13 @@ struct hal_reo_get_queue_stats { * Hole_count */ +struct hal_reo_get_queue_stats_qcc2072 { + struct hal_reo_cmd_hdr cmd; + __le32 queue_addr_lo; + __le32 info0; + __le32 rsvd0[6]; +} __packed; + #define HAL_REO_FLUSH_QUEUE_INFO0_DESC_ADDR_HI GENMASK(7, 0) #define HAL_REO_FLUSH_QUEUE_INFO0_BLOCK_DESC_ADDR BIT(8) #define HAL_REO_FLUSH_QUEUE_INFO0_BLOCK_RESRC_IDX GENMASK(10, 9) @@ -1269,46 +1118,6 @@ struct hal_reo_flush_cache { #define HAL_TCL_DATA_CMD_INFO5_RING_ID GENMASK(27, 20) #define HAL_TCL_DATA_CMD_INFO5_LOOPING_COUNT GENMASK(31, 28) -enum hal_encrypt_type { - HAL_ENCRYPT_TYPE_WEP_40, - HAL_ENCRYPT_TYPE_WEP_104, - HAL_ENCRYPT_TYPE_TKIP_NO_MIC, - HAL_ENCRYPT_TYPE_WEP_128, - HAL_ENCRYPT_TYPE_TKIP_MIC, - HAL_ENCRYPT_TYPE_WAPI, - HAL_ENCRYPT_TYPE_CCMP_128, - HAL_ENCRYPT_TYPE_OPEN, - HAL_ENCRYPT_TYPE_CCMP_256, - HAL_ENCRYPT_TYPE_GCMP_128, - HAL_ENCRYPT_TYPE_AES_GCMP_256, - HAL_ENCRYPT_TYPE_WAPI_GCM_SM4, -}; - -enum hal_tcl_encap_type { - HAL_TCL_ENCAP_TYPE_RAW, - HAL_TCL_ENCAP_TYPE_NATIVE_WIFI, - HAL_TCL_ENCAP_TYPE_ETHERNET, - HAL_TCL_ENCAP_TYPE_802_3 = 3, - HAL_TCL_ENCAP_TYPE_MAX -}; - -enum hal_tcl_desc_type { - HAL_TCL_DESC_TYPE_BUFFER, - HAL_TCL_DESC_TYPE_EXT_DESC, - HAL_TCL_DESC_TYPE_MAX, -}; - -enum hal_wbm_htt_tx_comp_status { - HAL_WBM_REL_HTT_TX_COMP_STATUS_OK, - HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP, - HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL, - HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ, - HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT, - HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY, - HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH, - HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX, -}; - struct hal_tcl_data_cmd { struct ath12k_buffer_addr buf_addr_info; __le32 info0; @@ -1765,107 +1574,11 @@ struct hal_ce_srng_dst_status_desc { #define HAL_TX_RATE_STATS_INFO0_OFDMA_TX BIT(16) #define HAL_TX_RATE_STATS_INFO0_TONES_IN_RU GENMASK(28, 17) -enum hal_tx_rate_stats_bw { - HAL_TX_RATE_STATS_BW_20, - HAL_TX_RATE_STATS_BW_40, - HAL_TX_RATE_STATS_BW_80, - HAL_TX_RATE_STATS_BW_160, -}; - -enum hal_tx_rate_stats_pkt_type { - HAL_TX_RATE_STATS_PKT_TYPE_11A, - HAL_TX_RATE_STATS_PKT_TYPE_11B, - HAL_TX_RATE_STATS_PKT_TYPE_11N, - HAL_TX_RATE_STATS_PKT_TYPE_11AC, - HAL_TX_RATE_STATS_PKT_TYPE_11AX, - HAL_TX_RATE_STATS_PKT_TYPE_11BA, - HAL_TX_RATE_STATS_PKT_TYPE_11BE, -}; - -enum hal_tx_rate_stats_sgi { - HAL_TX_RATE_STATS_SGI_08US, - HAL_TX_RATE_STATS_SGI_04US, - HAL_TX_RATE_STATS_SGI_16US, - HAL_TX_RATE_STATS_SGI_32US, -}; - struct hal_tx_rate_stats { __le32 info0; __le32 tsf; } __packed; -struct hal_wbm_link_desc { - struct ath12k_buffer_addr buf_addr_info; -} __packed; - -/* hal_wbm_link_desc - * - * Producer: WBM - * Consumer: WBM - * - * buf_addr_info - * Details of the physical address of a buffer or MSDU - * link descriptor. - */ - -enum hal_wbm_rel_src_module { - HAL_WBM_REL_SRC_MODULE_TQM, - HAL_WBM_REL_SRC_MODULE_RXDMA, - HAL_WBM_REL_SRC_MODULE_REO, - HAL_WBM_REL_SRC_MODULE_FW, - HAL_WBM_REL_SRC_MODULE_SW, - HAL_WBM_REL_SRC_MODULE_MAX, -}; - -enum hal_wbm_rel_desc_type { - HAL_WBM_REL_DESC_TYPE_REL_MSDU, - HAL_WBM_REL_DESC_TYPE_MSDU_LINK, - HAL_WBM_REL_DESC_TYPE_MPDU_LINK, - HAL_WBM_REL_DESC_TYPE_MSDU_EXT, - HAL_WBM_REL_DESC_TYPE_QUEUE_EXT, -}; - -/* hal_wbm_rel_desc_type - * - * msdu_buffer - * The address points to an MSDU buffer - * - * msdu_link_descriptor - * The address points to an Tx MSDU link descriptor - * - * mpdu_link_descriptor - * The address points to an MPDU link descriptor - * - * msdu_ext_descriptor - * The address points to an MSDU extension descriptor - * - * queue_ext_descriptor - * The address points to an TQM queue extension descriptor. WBM should - * treat this is the same way as a link descriptor. - */ - -enum hal_wbm_rel_bm_act { - HAL_WBM_REL_BM_ACT_PUT_IN_IDLE, - HAL_WBM_REL_BM_ACT_REL_MSDU, -}; - -/* hal_wbm_rel_bm_act - * - * put_in_idle_list - * Put the buffer or descriptor back in the idle list. In case of MSDU or - * MDPU link descriptor, BM does not need to check to release any - * individual MSDU buffers. - * - * release_msdu_list - * This BM action can only be used in combination with desc_type being - * msdu_link_descriptor. Field first_msdu_index points out which MSDU - * pointer in the MSDU link descriptor is the first of an MPDU that is - * released. BM shall release all the MSDU buffers linked to this first - * MSDU buffer pointer. All related MSDU buffer pointer entries shall be - * set to value 0, which represents the 'NULL' pointer. When all MSDU - * buffer pointers in the MSDU link descriptor are 'NULL', the MSDU link - * descriptor itself shall also be released. - */ #define HAL_WBM_COMPL_RX_INFO0_REL_SRC_MODULE GENMASK(2, 0) #define HAL_WBM_COMPL_RX_INFO0_BM_ACTION GENMASK(5, 3) #define HAL_WBM_COMPL_RX_INFO0_DESC_TYPE GENMASK(8, 6) @@ -2007,7 +1720,6 @@ struct hal_wbm_release_ring_cc_rx { #define HAL_WBM_RELEASE_INFO3_CONTINUATION BIT(2) #define HAL_WBM_RELEASE_INFO5_LOOPING_COUNT GENMASK(31, 28) -#define HAL_ENCRYPT_TYPE_MAX 12 struct hal_wbm_release_ring { struct ath12k_buffer_addr buf_addr_info; @@ -2331,7 +2043,6 @@ enum hal_desc_buf_type { #define HAL_DESC_REO_OWNED 4 #define HAL_DESC_REO_QUEUE_DESC 8 #define HAL_DESC_REO_QUEUE_EXT_DESC 9 -#define HAL_DESC_REO_NON_QOS_TID 16 #define HAL_DESC_HDR_INFO0_OWNER GENMASK(3, 0) #define HAL_DESC_HDR_INFO0_BUF_TYPE GENMASK(7, 4) @@ -2728,6 +2439,11 @@ struct hal_reo_get_queue_stats_status { * entries into this Ring has looped around the ring. */ +struct hal_reo_get_queue_stats_status_qcc2072 { + __le32 tlv32_padding; + struct hal_reo_get_queue_stats_status status; +} __packed; + #define HAL_REO_STATUS_LOOP_CNT GENMASK(31, 28) #define HAL_REO_FLUSH_QUEUE_INFO0_ERR_DETECTED BIT(0) @@ -2957,25 +2673,6 @@ struct hal_tcl_entrance_from_ppe_ring { __le32 info0; } __packed; -struct hal_mon_buf_ring { - __le32 paddr_lo; - __le32 paddr_hi; - __le64 cookie; -}; - -/* hal_mon_buf_ring - * Producer : SW - * Consumer : Monitor - * - * paddr_lo - * Lower 32-bit physical address of the buffer pointer from the source ring. - * paddr_hi - * bit range 7-0 : upper 8 bit of the physical address. - * bit range 31-8 : reserved. - * cookie - * Consumer: RxMon/TxMon 64 bit cookie of the buffers. - */ - #define HAL_MON_DEST_COOKIE_BUF_ID GENMASK(17, 0) #define HAL_MON_DEST_INFO0_END_OFFSET GENMASK(11, 0) diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.c new file mode 100644 index 0000000000000..80ffadc47d48c --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.c @@ -0,0 +1,505 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "hal_qcc2072.h" +#include "hal_wcn7850.h" + +const struct ath12k_hw_regs qcc2072_regs = { + /* SW2TCL(x) R0 ring configuration address */ + .tcl1_ring_id = 0x00000920, + .tcl1_ring_misc = 0x00000928, + .tcl1_ring_tp_addr_lsb = 0x00000934, + .tcl1_ring_tp_addr_msb = 0x00000938, + .tcl1_ring_consumer_int_setup_ix0 = 0x00000948, + .tcl1_ring_consumer_int_setup_ix1 = 0x0000094c, + .tcl1_ring_msi1_base_lsb = 0x00000960, + .tcl1_ring_msi1_base_msb = 0x00000964, + .tcl1_ring_msi1_data = 0x00000968, + .tcl_ring_base_lsb = 0x00000b70, + .tcl1_ring_base_lsb = 0x00000918, + .tcl1_ring_base_msb = 0x0000091c, + .tcl2_ring_base_lsb = 0x00000990, + + /* TCL STATUS ring address */ + .tcl_status_ring_base_lsb = 0x00000d50, + + .wbm_idle_ring_base_lsb = 0x00000d3c, + .wbm_idle_ring_misc_addr = 0x00000d4c, + .wbm_r0_idle_list_cntl_addr = 0x00000240, + .wbm_r0_idle_list_size_addr = 0x00000244, + .wbm_scattered_ring_base_lsb = 0x00000250, + .wbm_scattered_ring_base_msb = 0x00000254, + .wbm_scattered_desc_head_info_ix0 = 0x00000260, + .wbm_scattered_desc_head_info_ix1 = 0x00000264, + .wbm_scattered_desc_tail_info_ix0 = 0x00000270, + .wbm_scattered_desc_tail_info_ix1 = 0x00000274, + .wbm_scattered_desc_ptr_hp_addr = 0x00000027c, + + .wbm_sw_release_ring_base_lsb = 0x0000037c, + .wbm_sw1_release_ring_base_lsb = ATH12K_HW_REG_UNDEFINED, + .wbm0_release_ring_base_lsb = 0x00000e08, + .wbm1_release_ring_base_lsb = 0x00000e80, + + /* PCIe base address */ + .pcie_qserdes_sysclk_en_sel = 0x01e0c0ac, + .pcie_pcs_osc_dtct_config_base = 0x01e0cc58, + + /* PPE release ring address */ + .ppe_rel_ring_base = 0x0000046c, + + /* REO DEST ring address */ + .reo2_ring_base = 0x00000578, + .reo1_misc_ctrl_addr = 0x00000ba0, + .reo1_sw_cookie_cfg0 = 0x0000006c, + .reo1_sw_cookie_cfg1 = 0x00000070, + .reo1_qdesc_lut_base0 = ATH12K_HW_REG_UNDEFINED, + .reo1_qdesc_lut_base1 = ATH12K_HW_REG_UNDEFINED, + + .reo1_ring_base_lsb = 0x00000500, + .reo1_ring_base_msb = 0x00000504, + .reo1_ring_id = 0x00000508, + .reo1_ring_misc = 0x00000510, + .reo1_ring_hp_addr_lsb = 0x00000514, + .reo1_ring_hp_addr_msb = 0x00000518, + .reo1_ring_producer_int_setup = 0x00000524, + .reo1_ring_msi1_base_lsb = 0x00000548, + .reo1_ring_msi1_base_msb = 0x0000054c, + .reo1_ring_msi1_data = 0x00000550, + .reo1_aging_thres_ix0 = 0x00000b2c, + .reo1_aging_thres_ix1 = 0x00000b30, + .reo1_aging_thres_ix2 = 0x00000b34, + .reo1_aging_thres_ix3 = 0x00000b38, + + /* REO Exception ring address */ + .reo2_sw0_ring_base = 0x000008c0, + + /* REO Reinject ring address */ + .sw2reo_ring_base = 0x00000320, + .sw2reo1_ring_base = 0x00000398, + + /* REO cmd ring address */ + .reo_cmd_ring_base = 0x000002a8, + + /* REO status ring address */ + .reo_status_ring_base = 0x00000aa0, + + /* CE base address */ + .umac_ce0_src_reg_base = 0x01b80000, + .umac_ce0_dest_reg_base = 0x01b81000, + .umac_ce1_src_reg_base = 0x01b82000, + .umac_ce1_dest_reg_base = 0x01b83000, + + .gcc_gcc_pcie_hot_rst = 0x1e65304, + + .qrtr_node_id = 0x1e03300, +}; + +static void ath12k_hal_rx_desc_set_msdu_len_qcc2072(struct hal_rx_desc *desc, u16 len) +{ + u32 info = __le32_to_cpu(desc->u.qcc2072.msdu_end.info10); + + info &= ~RX_MSDU_END_INFO10_MSDU_LENGTH; + info |= u32_encode_bits(len, RX_MSDU_END_INFO10_MSDU_LENGTH); + + desc->u.qcc2072.msdu_end.info10 = __cpu_to_le32(info); +} + +static void ath12k_hal_rx_desc_get_dot11_hdr_qcc2072(struct hal_rx_desc *desc, + struct ieee80211_hdr *hdr) +{ + hdr->frame_control = desc->u.qcc2072.mpdu_start.frame_ctrl; + hdr->duration_id = desc->u.qcc2072.mpdu_start.duration; + ether_addr_copy(hdr->addr1, desc->u.qcc2072.mpdu_start.addr1); + ether_addr_copy(hdr->addr2, desc->u.qcc2072.mpdu_start.addr2); + ether_addr_copy(hdr->addr3, desc->u.qcc2072.mpdu_start.addr3); + + if (__le32_to_cpu(desc->u.qcc2072.mpdu_start.info4) & + RX_MPDU_START_INFO4_MAC_ADDR4_VALID) + ether_addr_copy(hdr->addr4, desc->u.qcc2072.mpdu_start.addr4); + + hdr->seq_ctrl = desc->u.qcc2072.mpdu_start.seq_ctrl; +} + +static void ath12k_hal_rx_desc_get_crypto_hdr_qcc2072(struct hal_rx_desc *desc, + u8 *crypto_hdr, + enum hal_encrypt_type enctype) +{ + unsigned int key_id; + + switch (enctype) { + case HAL_ENCRYPT_TYPE_OPEN: + return; + case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: + case HAL_ENCRYPT_TYPE_TKIP_MIC: + crypto_hdr[0] = + HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcc2072.mpdu_start.pn[0]); + crypto_hdr[1] = 0; + crypto_hdr[2] = + HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcc2072.mpdu_start.pn[0]); + break; + case HAL_ENCRYPT_TYPE_CCMP_128: + case HAL_ENCRYPT_TYPE_CCMP_256: + case HAL_ENCRYPT_TYPE_GCMP_128: + case HAL_ENCRYPT_TYPE_AES_GCMP_256: + crypto_hdr[0] = + HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcc2072.mpdu_start.pn[0]); + crypto_hdr[1] = + HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcc2072.mpdu_start.pn[0]); + crypto_hdr[2] = 0; + break; + case HAL_ENCRYPT_TYPE_WEP_40: + case HAL_ENCRYPT_TYPE_WEP_104: + case HAL_ENCRYPT_TYPE_WEP_128: + case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4: + case HAL_ENCRYPT_TYPE_WAPI: + return; + } + + key_id = u32_get_bits(__le32_to_cpu(desc->u.qcc2072.mpdu_start.info5), + RX_MPDU_START_INFO5_KEY_ID); + crypto_hdr[3] = 0x20 | (key_id << 6); + crypto_hdr[4] = HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.qcc2072.mpdu_start.pn[0]); + crypto_hdr[5] = HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.qcc2072.mpdu_start.pn[0]); + crypto_hdr[6] = HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcc2072.mpdu_start.pn[1]); + crypto_hdr[7] = HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcc2072.mpdu_start.pn[1]); +} + +static void ath12k_hal_rx_desc_copy_end_tlv_qcc2072(struct hal_rx_desc *fdesc, + struct hal_rx_desc *ldesc) +{ + memcpy(&fdesc->u.qcc2072.msdu_end, &ldesc->u.qcc2072.msdu_end, + sizeof(struct rx_msdu_end_qcn9274)); +} + +static u8 ath12k_hal_rx_desc_get_msdu_src_link_qcc2072(struct hal_rx_desc *desc) +{ + return 0; +} + +static u8 ath12k_hal_rx_desc_get_l3_pad_bytes_qcc2072(struct hal_rx_desc *desc) +{ + return le16_get_bits(desc->u.qcc2072.msdu_end.info5, + RX_MSDU_END_INFO5_L3_HDR_PADDING); +} + +static u32 ath12k_hal_rx_desc_get_mpdu_start_tag_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.mpdu_start_tag, + HAL_TLV_HDR_TAG); +} + +static u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcc2072(struct hal_rx_desc *desc) +{ + return __le16_to_cpu(desc->u.qcc2072.mpdu_start.phy_ppdu_id); +} + +static u8 *ath12k_hal_rx_desc_get_msdu_payload_qcc2072(struct hal_rx_desc *desc) +{ + return &desc->u.qcc2072.msdu_payload[0]; +} + +static bool ath12k_hal_rx_desc_get_first_msdu_qcc2072(struct hal_rx_desc *desc) +{ + return !!le16_get_bits(desc->u.qcc2072.msdu_end.info5, + RX_MSDU_END_INFO5_FIRST_MSDU); +} + +static bool ath12k_hal_rx_desc_get_last_msdu_qcc2072(struct hal_rx_desc *desc) +{ + return !!le16_get_bits(desc->u.qcc2072.msdu_end.info5, + RX_MSDU_END_INFO5_LAST_MSDU); +} + +static bool ath12k_hal_rx_desc_encrypt_valid_qcc2072(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcc2072.mpdu_start.info4, + RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID); +} + +static u32 ath12k_hal_rx_desc_get_encrypt_type_qcc2072(struct hal_rx_desc *desc) +{ + if (!ath12k_hal_rx_desc_encrypt_valid_qcc2072(desc)) + return HAL_ENCRYPT_TYPE_OPEN; + + return le32_get_bits(desc->u.qcc2072.mpdu_start.info2, + RX_MPDU_START_INFO2_ENC_TYPE); +} + +static u8 ath12k_hal_rx_desc_get_decap_type_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info11, + RX_MSDU_END_INFO11_DECAP_FORMAT); +} + +static u8 ath12k_hal_rx_desc_get_mesh_ctl_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info11, + RX_MSDU_END_INFO11_MESH_CTRL_PRESENT); +} + +static bool ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_qcc2072(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcc2072.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID); +} + +static bool ath12k_hal_rx_desc_get_mpdu_fc_valid_qcc2072(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcc2072.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_FCTRL_VALID); +} + +static u16 ath12k_hal_rx_desc_get_mpdu_start_seq_no_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_SEQ_NUM); +} + +static u16 ath12k_hal_rx_desc_get_msdu_len_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info10, + RX_MSDU_END_INFO10_MSDU_LENGTH); +} + +static u8 ath12k_hal_rx_desc_get_msdu_sgi_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info12, + RX_MSDU_END_INFO12_SGI); +} + +static u8 ath12k_hal_rx_desc_get_msdu_rate_mcs_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info12, + RX_MSDU_END_INFO12_RATE_MCS); +} + +static u8 ath12k_hal_rx_desc_get_msdu_rx_bw_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info12, + RX_MSDU_END_INFO12_RECV_BW); +} + +static u32 ath12k_hal_rx_desc_get_msdu_freq_qcc2072(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.qcc2072.msdu_end.phy_meta_data); +} + +static u8 ath12k_hal_rx_desc_get_msdu_pkt_type_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info12, + RX_MSDU_END_INFO12_PKT_TYPE); +} + +static u8 ath12k_hal_rx_desc_get_msdu_nss_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.msdu_end.info12, + RX_MSDU_END_INFO12_MIMO_SS_BITMAP); +} + +static u8 ath12k_hal_rx_desc_get_mpdu_tid_qcc2072(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcc2072.mpdu_start.info2, + RX_MPDU_START_INFO2_TID); +} + +static u16 ath12k_hal_rx_desc_get_mpdu_peer_id_qcc2072(struct hal_rx_desc *desc) +{ + return __le16_to_cpu(desc->u.qcc2072.mpdu_start.sw_peer_id); +} + +static bool ath12k_hal_rx_desc_mac_addr2_valid_qcc2072(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.qcc2072.mpdu_start.info4) & + RX_MPDU_START_INFO4_MAC_ADDR2_VALID; +} + +static u8 *ath12k_hal_rx_desc_mpdu_start_addr2_qcc2072(struct hal_rx_desc *desc) +{ + return desc->u.qcc2072.mpdu_start.addr2; +} + +static bool ath12k_hal_rx_desc_is_da_mcbc_qcc2072(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.qcc2072.msdu_end.info13) & + RX_MSDU_END_INFO13_MCAST_BCAST; +} + +static bool ath12k_hal_rx_h_msdu_done_qcc2072(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcc2072.msdu_end.info14, + RX_MSDU_END_INFO14_MSDU_DONE); +} + +static bool ath12k_hal_rx_h_l4_cksum_fail_qcc2072(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcc2072.msdu_end.info13, + RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL); +} + +static bool ath12k_hal_rx_h_ip_cksum_fail_qcc2072(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcc2072.msdu_end.info13, + RX_MSDU_END_INFO13_IP_CKSUM_FAIL); +} + +static bool ath12k_hal_rx_h_is_decrypted_qcc2072(struct hal_rx_desc *desc) +{ + return (le32_get_bits(desc->u.qcc2072.msdu_end.info14, + RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) == + RX_DESC_DECRYPT_STATUS_CODE_OK); +} + +static u32 ath12k_hal_rx_h_mpdu_err_qcc2072(struct hal_rx_desc *desc) +{ + u32 info = __le32_to_cpu(desc->u.qcc2072.msdu_end.info13); + u32 errmap = 0; + + if (info & RX_MSDU_END_INFO13_FCS_ERR) + errmap |= HAL_RX_MPDU_ERR_FCS; + + if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) + errmap |= HAL_RX_MPDU_ERR_DECRYPT; + + if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) + errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; + + if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) + errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; + + if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) + errmap |= HAL_RX_MPDU_ERR_OVERFLOW; + + if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) + errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; + + if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) + errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; + + return errmap; +} + +static void ath12k_hal_extract_rx_desc_data_qcc2072(struct hal_rx_desc_data *rx_desc_data, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc *ldesc) +{ + rx_desc_data->is_first_msdu = ath12k_hal_rx_desc_get_first_msdu_qcc2072(ldesc); + rx_desc_data->is_last_msdu = ath12k_hal_rx_desc_get_last_msdu_qcc2072(ldesc); + rx_desc_data->l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_qcc2072(ldesc); + rx_desc_data->enctype = ath12k_hal_rx_desc_get_encrypt_type_qcc2072(rx_desc); + rx_desc_data->decap_type = ath12k_hal_rx_desc_get_decap_type_qcc2072(rx_desc); + rx_desc_data->mesh_ctrl_present = + ath12k_hal_rx_desc_get_mesh_ctl_qcc2072(rx_desc); + rx_desc_data->seq_ctl_valid = + ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_qcc2072(rx_desc); + rx_desc_data->fc_valid = ath12k_hal_rx_desc_get_mpdu_fc_valid_qcc2072(rx_desc); + rx_desc_data->seq_no = ath12k_hal_rx_desc_get_mpdu_start_seq_no_qcc2072(rx_desc); + rx_desc_data->msdu_len = ath12k_hal_rx_desc_get_msdu_len_qcc2072(ldesc); + rx_desc_data->sgi = ath12k_hal_rx_desc_get_msdu_sgi_qcc2072(rx_desc); + rx_desc_data->rate_mcs = ath12k_hal_rx_desc_get_msdu_rate_mcs_qcc2072(rx_desc); + rx_desc_data->bw = ath12k_hal_rx_desc_get_msdu_rx_bw_qcc2072(rx_desc); + rx_desc_data->phy_meta_data = ath12k_hal_rx_desc_get_msdu_freq_qcc2072(rx_desc); + rx_desc_data->pkt_type = ath12k_hal_rx_desc_get_msdu_pkt_type_qcc2072(rx_desc); + rx_desc_data->nss = hweight8(ath12k_hal_rx_desc_get_msdu_nss_qcc2072(rx_desc)); + rx_desc_data->tid = ath12k_hal_rx_desc_get_mpdu_tid_qcc2072(rx_desc); + rx_desc_data->peer_id = ath12k_hal_rx_desc_get_mpdu_peer_id_qcc2072(rx_desc); + rx_desc_data->addr2_present = ath12k_hal_rx_desc_mac_addr2_valid_qcc2072(rx_desc); + rx_desc_data->addr2 = ath12k_hal_rx_desc_mpdu_start_addr2_qcc2072(rx_desc); + rx_desc_data->is_mcbc = ath12k_hal_rx_desc_is_da_mcbc_qcc2072(rx_desc); + rx_desc_data->msdu_done = ath12k_hal_rx_h_msdu_done_qcc2072(ldesc); + rx_desc_data->l4_csum_fail = ath12k_hal_rx_h_l4_cksum_fail_qcc2072(rx_desc); + rx_desc_data->ip_csum_fail = ath12k_hal_rx_h_ip_cksum_fail_qcc2072(rx_desc); + rx_desc_data->is_decrypted = ath12k_hal_rx_h_is_decrypted_qcc2072(rx_desc); + rx_desc_data->err_bitmap = ath12k_hal_rx_h_mpdu_err_qcc2072(rx_desc); +} + +static int ath12k_hal_srng_create_config_qcc2072(struct ath12k_hal *hal) +{ + struct hal_srng_config *s; + int ret; + + ret = ath12k_hal_srng_create_config_wcn7850(hal); + if (ret) + return ret; + + s = &hal->srng_config[HAL_REO_CMD]; + s->entry_size = (sizeof(struct hal_tlv_hdr) + + sizeof(struct hal_reo_get_queue_stats_qcc2072)) >> 2; + + s = &hal->srng_config[HAL_REO_STATUS]; + s->entry_size = (sizeof(struct hal_tlv_hdr) + + sizeof(struct hal_reo_get_queue_stats_status_qcc2072)) >> 2; + + return 0; +} + +static u16 ath12k_hal_reo_status_dec_tlv_hdr_qcc2072(void *tlv, void **desc) +{ + struct hal_reo_get_queue_stats_status_qcc2072 *status_tlv; + u16 tag; + + status_tlv = ath12k_hal_decode_tlv32_hdr(tlv, &tag, NULL, NULL); + /* + * actual desc of REO status entry starts after tlv32_padding, + * see hal_reo_get_queue_stats_status_qcc2072 + */ + *desc = &status_tlv->status; + + return tag; +} + +const struct hal_ops hal_qcc2072_ops = { + .create_srng_config = ath12k_hal_srng_create_config_qcc2072, + .rx_desc_set_msdu_len = ath12k_hal_rx_desc_set_msdu_len_qcc2072, + .rx_desc_get_dot11_hdr = ath12k_hal_rx_desc_get_dot11_hdr_qcc2072, + .rx_desc_get_crypto_header = ath12k_hal_rx_desc_get_crypto_hdr_qcc2072, + .rx_desc_copy_end_tlv = ath12k_hal_rx_desc_copy_end_tlv_qcc2072, + .rx_desc_get_msdu_src_link_id = ath12k_hal_rx_desc_get_msdu_src_link_qcc2072, + .extract_rx_desc_data = ath12k_hal_extract_rx_desc_data_qcc2072, + .rx_desc_get_l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_qcc2072, + .rx_desc_get_mpdu_start_tag = ath12k_hal_rx_desc_get_mpdu_start_tag_qcc2072, + .rx_desc_get_mpdu_ppdu_id = ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcc2072, + .rx_desc_get_msdu_payload = ath12k_hal_rx_desc_get_msdu_payload_qcc2072, + .ce_dst_setup = ath12k_wifi7_hal_ce_dst_setup, + .srng_src_hw_init = ath12k_wifi7_hal_srng_src_hw_init, + .srng_dst_hw_init = ath12k_wifi7_hal_srng_dst_hw_init, + .set_umac_srng_ptr_addr = ath12k_wifi7_hal_set_umac_srng_ptr_addr, + .srng_update_shadow_config = ath12k_wifi7_hal_srng_update_shadow_config, + .srng_get_ring_id = ath12k_wifi7_hal_srng_get_ring_id, + .ce_get_desc_size = ath12k_wifi7_hal_ce_get_desc_size, + .ce_src_set_desc = ath12k_wifi7_hal_ce_src_set_desc, + .ce_dst_set_desc = ath12k_wifi7_hal_ce_dst_set_desc, + .ce_dst_status_get_length = ath12k_wifi7_hal_ce_dst_status_get_length, + .set_link_desc_addr = ath12k_wifi7_hal_set_link_desc_addr, + .tx_set_dscp_tid_map = ath12k_wifi7_hal_tx_set_dscp_tid_map, + .tx_configure_bank_register = + ath12k_wifi7_hal_tx_configure_bank_register, + .reoq_lut_addr_read_enable = ath12k_wifi7_hal_reoq_lut_addr_read_enable, + .reoq_lut_set_max_peerid = ath12k_wifi7_hal_reoq_lut_set_max_peerid, + .write_reoq_lut_addr = ath12k_wifi7_hal_write_reoq_lut_addr, + .write_ml_reoq_lut_addr = ath12k_wifi7_hal_write_ml_reoq_lut_addr, + .setup_link_idle_list = ath12k_wifi7_hal_setup_link_idle_list, + .reo_init_cmd_ring = ath12k_wifi7_hal_reo_init_cmd_ring_tlv32, + .reo_hw_setup = ath12k_wifi7_hal_reo_hw_setup, + .rx_buf_addr_info_set = ath12k_wifi7_hal_rx_buf_addr_info_set, + .rx_buf_addr_info_get = ath12k_wifi7_hal_rx_buf_addr_info_get, + .cc_config = ath12k_wifi7_hal_cc_config, + .get_idle_link_rbm = ath12k_wifi7_hal_get_idle_link_rbm, + .rx_msdu_list_get = ath12k_wifi7_hal_rx_msdu_list_get, + .rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get, + .reo_cmd_enc_tlv_hdr = ath12k_hal_encode_tlv32_hdr, + .reo_status_dec_tlv_hdr = ath12k_hal_reo_status_dec_tlv_hdr_qcc2072, + .mon_rx_status_dec_tlv_hdr = ath12k_hal_decode_tlv32_hdr, + .get_tlv_hdr_align = ath12k_hal_get_tlv32_hdr_align, +}; + +u32 ath12k_hal_rx_desc_get_mpdu_start_offset_qcc2072(void) +{ + return offsetof(struct hal_rx_desc_qcc2072, mpdu_start_tag); +} + +u32 ath12k_hal_rx_desc_get_msdu_end_offset_qcc2072(void) +{ + return offsetof(struct hal_rx_desc_qcc2072, msdu_end_tag); +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.h new file mode 100644 index 0000000000000..6de943df77864 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "../hal.h" +#include "hal.h" + +extern const struct ath12k_hw_regs qcc2072_regs; +extern const struct hal_ops hal_qcc2072_ops; + +u32 ath12k_hal_rx_desc_get_mpdu_start_offset_qcc2072(void); +u32 ath12k_hal_rx_desc_get_msdu_end_offset_qcc2072(void); diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c new file mode 100644 index 0000000000000..e51b6f2afffc6 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c @@ -0,0 +1,1049 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#include "hal_desc.h" +#include "hal_qcn9274.h" +#include "hw.h" +#include "hal.h" +#include "hal_tx.h" + +static const struct hal_srng_config hw_srng_config_template[] = { + /* TODO: max_rings can populated by querying HW capabilities */ + [HAL_REO_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_REO2SW1, + .max_rings = 8, + .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_EXCEPTION] = { + /* Designating REO2SW0 ring as exception ring. + * Any of theREO2SW rings can be used as exception ring. + */ + .start_ring_id = HAL_SRNG_RING_ID_REO2SW0, + .max_rings = 1, + .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_REINJECT] = { + .start_ring_id = HAL_SRNG_RING_ID_SW2REO, + .max_rings = 4, + .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_CMD] = { + .start_ring_id = HAL_SRNG_RING_ID_REO_CMD, + .max_rings = 1, + .entry_size = (sizeof(struct hal_tlv_64_hdr) + + sizeof(struct hal_reo_get_queue_stats)) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_REO_STATUS, + .max_rings = 1, + .entry_size = (sizeof(struct hal_tlv_64_hdr) + + sizeof(struct hal_reo_get_queue_stats_status)) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE, + }, + [HAL_TCL_DATA] = { + .start_ring_id = HAL_SRNG_RING_ID_SW2TCL1, + .max_rings = 6, + .entry_size = sizeof(struct hal_tcl_data_cmd) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE, + }, + [HAL_TCL_CMD] = { + .start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD, + .max_rings = 1, + .entry_size = sizeof(struct hal_tcl_gse_cmd) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE, + }, + [HAL_TCL_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS, + .max_rings = 1, + .entry_size = (sizeof(struct hal_tlv_hdr) + + sizeof(struct hal_tcl_status_ring)) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE, + }, + [HAL_CE_SRC] = { + .start_ring_id = HAL_SRNG_RING_ID_CE0_SRC, + .max_rings = 16, + .entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE, + }, + [HAL_CE_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_CE0_DST, + .max_rings = 16, + .entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE, + }, + [HAL_CE_DST_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS, + .max_rings = 16, + .entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE, + }, + [HAL_WBM_IDLE_LINK] = { + .start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK, + .max_rings = 1, + .entry_size = sizeof(struct hal_wbm_link_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE, + }, + [HAL_SW2WBM_RELEASE] = { + .start_ring_id = HAL_SRNG_RING_ID_WBM_SW0_RELEASE, + .max_rings = 2, + .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE, + }, + [HAL_WBM2SW_RELEASE] = { + .start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE, + .max_rings = 8, + .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE, + }, + [HAL_RXDMA_BUF] = { + .start_ring_id = HAL_SRNG_SW2RXDMA_BUF0, + .max_rings = 1, + .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_DMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0, + .max_rings = 0, + .entry_size = 0, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_MONITOR_BUF] = { + .start_ring_id = HAL_SRNG_SW2RXMON_BUF0, + .max_rings = 1, + .entry_size = sizeof(struct hal_mon_buf_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_MONITOR_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF, + .max_rings = 1, + .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_MONITOR_DESC] = { 0, }, + [HAL_RXDMA_DIR_BUF] = { + .start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF, + .max_rings = 2, + .entry_size = 8 >> 2, /* TODO: Define the struct */ + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_PPE2TCL] = { + .start_ring_id = HAL_SRNG_RING_ID_PPE2TCL1, + .max_rings = 1, + .entry_size = sizeof(struct hal_tcl_entrance_from_ppe_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE, + }, + [HAL_PPE_RELEASE] = { + .start_ring_id = HAL_SRNG_RING_ID_WBM_PPE_RELEASE, + .max_rings = 1, + .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE, + }, + [HAL_TX_MONITOR_BUF] = { + .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2TXMON_BUF0, + .max_rings = 1, + .entry_size = sizeof(struct hal_mon_buf_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_MONITOR_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXMON_BUF0, + .max_rings = 1, + .entry_size = sizeof(struct hal_mon_dest_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_TX_MONITOR_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_WMAC1_TXMON2SW0_BUF0, + .max_rings = 1, + .entry_size = sizeof(struct hal_mon_dest_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + } +}; + +const struct ath12k_hw_regs qcn9274_v1_regs = { + /* SW2TCL(x) R0 ring configuration address */ + .tcl1_ring_id = 0x00000908, + .tcl1_ring_misc = 0x00000910, + .tcl1_ring_tp_addr_lsb = 0x0000091c, + .tcl1_ring_tp_addr_msb = 0x00000920, + .tcl1_ring_consumer_int_setup_ix0 = 0x00000930, + .tcl1_ring_consumer_int_setup_ix1 = 0x00000934, + .tcl1_ring_msi1_base_lsb = 0x00000948, + .tcl1_ring_msi1_base_msb = 0x0000094c, + .tcl1_ring_msi1_data = 0x00000950, + .tcl_ring_base_lsb = 0x00000b58, + .tcl1_ring_base_lsb = 0x00000900, + .tcl1_ring_base_msb = 0x00000904, + .tcl2_ring_base_lsb = 0x00000978, + + /* TCL STATUS ring address */ + .tcl_status_ring_base_lsb = 0x00000d38, + + .wbm_idle_ring_base_lsb = 0x00000d0c, + .wbm_idle_ring_misc_addr = 0x00000d1c, + .wbm_r0_idle_list_cntl_addr = 0x00000210, + .wbm_r0_idle_list_size_addr = 0x00000214, + .wbm_scattered_ring_base_lsb = 0x00000220, + .wbm_scattered_ring_base_msb = 0x00000224, + .wbm_scattered_desc_head_info_ix0 = 0x00000230, + .wbm_scattered_desc_head_info_ix1 = 0x00000234, + .wbm_scattered_desc_tail_info_ix0 = 0x00000240, + .wbm_scattered_desc_tail_info_ix1 = 0x00000244, + .wbm_scattered_desc_ptr_hp_addr = 0x0000024c, + + .wbm_sw_release_ring_base_lsb = 0x0000034c, + .wbm_sw1_release_ring_base_lsb = 0x000003c4, + .wbm0_release_ring_base_lsb = 0x00000dd8, + .wbm1_release_ring_base_lsb = 0x00000e50, + + /* PCIe base address */ + .pcie_qserdes_sysclk_en_sel = 0x01e0c0a8, + .pcie_pcs_osc_dtct_config_base = 0x01e0d45c, + + /* PPE release ring address */ + .ppe_rel_ring_base = 0x0000043c, + + /* REO DEST ring address */ + .reo2_ring_base = 0x0000055c, + .reo1_misc_ctrl_addr = 0x00000b7c, + .reo1_sw_cookie_cfg0 = 0x00000050, + .reo1_sw_cookie_cfg1 = 0x00000054, + .reo1_qdesc_lut_base0 = 0x00000058, + .reo1_qdesc_lut_base1 = 0x0000005c, + .reo1_ring_base_lsb = 0x000004e4, + .reo1_ring_base_msb = 0x000004e8, + .reo1_ring_id = 0x000004ec, + .reo1_ring_misc = 0x000004f4, + .reo1_ring_hp_addr_lsb = 0x000004f8, + .reo1_ring_hp_addr_msb = 0x000004fc, + .reo1_ring_producer_int_setup = 0x00000508, + .reo1_ring_msi1_base_lsb = 0x0000052C, + .reo1_ring_msi1_base_msb = 0x00000530, + .reo1_ring_msi1_data = 0x00000534, + .reo1_aging_thres_ix0 = 0x00000b08, + .reo1_aging_thres_ix1 = 0x00000b0c, + .reo1_aging_thres_ix2 = 0x00000b10, + .reo1_aging_thres_ix3 = 0x00000b14, + + /* REO Exception ring address */ + .reo2_sw0_ring_base = 0x000008a4, + + /* REO Reinject ring address */ + .sw2reo_ring_base = 0x00000304, + .sw2reo1_ring_base = 0x0000037c, + + /* REO cmd ring address */ + .reo_cmd_ring_base = 0x0000028c, + + /* REO status ring address */ + .reo_status_ring_base = 0x00000a84, + + /* CE base address */ + .umac_ce0_src_reg_base = 0x01b80000, + .umac_ce0_dest_reg_base = 0x01b81000, + .umac_ce1_src_reg_base = 0x01b82000, + .umac_ce1_dest_reg_base = 0x01b83000, + + .gcc_gcc_pcie_hot_rst = 0x1e38338, + + .qrtr_node_id = 0x1e03164, +}; + +const struct ath12k_hw_regs qcn9274_v2_regs = { + /* SW2TCL(x) R0 ring configuration address */ + .tcl1_ring_id = 0x00000908, + .tcl1_ring_misc = 0x00000910, + .tcl1_ring_tp_addr_lsb = 0x0000091c, + .tcl1_ring_tp_addr_msb = 0x00000920, + .tcl1_ring_consumer_int_setup_ix0 = 0x00000930, + .tcl1_ring_consumer_int_setup_ix1 = 0x00000934, + .tcl1_ring_msi1_base_lsb = 0x00000948, + .tcl1_ring_msi1_base_msb = 0x0000094c, + .tcl1_ring_msi1_data = 0x00000950, + .tcl_ring_base_lsb = 0x00000b58, + .tcl1_ring_base_lsb = 0x00000900, + .tcl1_ring_base_msb = 0x00000904, + .tcl2_ring_base_lsb = 0x00000978, + + /* TCL STATUS ring address */ + .tcl_status_ring_base_lsb = 0x00000d38, + + /* WBM idle link ring address */ + .wbm_idle_ring_base_lsb = 0x00000d3c, + .wbm_idle_ring_misc_addr = 0x00000d4c, + .wbm_r0_idle_list_cntl_addr = 0x00000240, + .wbm_r0_idle_list_size_addr = 0x00000244, + .wbm_scattered_ring_base_lsb = 0x00000250, + .wbm_scattered_ring_base_msb = 0x00000254, + .wbm_scattered_desc_head_info_ix0 = 0x00000260, + .wbm_scattered_desc_head_info_ix1 = 0x00000264, + .wbm_scattered_desc_tail_info_ix0 = 0x00000270, + .wbm_scattered_desc_tail_info_ix1 = 0x00000274, + .wbm_scattered_desc_ptr_hp_addr = 0x0000027c, + + /* SW2WBM release ring address */ + .wbm_sw_release_ring_base_lsb = 0x0000037c, + .wbm_sw1_release_ring_base_lsb = 0x000003f4, + + /* WBM2SW release ring address */ + .wbm0_release_ring_base_lsb = 0x00000e08, + .wbm1_release_ring_base_lsb = 0x00000e80, + + /* PCIe base address */ + .pcie_qserdes_sysclk_en_sel = 0x01e0c0a8, + .pcie_pcs_osc_dtct_config_base = 0x01e0d45c, + + /* PPE release ring address */ + .ppe_rel_ring_base = 0x0000046c, + + /* REO DEST ring address */ + .reo2_ring_base = 0x00000578, + .reo1_misc_ctrl_addr = 0x00000b9c, + .reo1_sw_cookie_cfg0 = 0x0000006c, + .reo1_sw_cookie_cfg1 = 0x00000070, + .reo1_qdesc_lut_base0 = 0x00000074, + .reo1_qdesc_lut_base1 = 0x00000078, + .reo1_qdesc_addr = 0x0000007c, + .reo1_qdesc_max_peerid = 0x00000088, + .reo1_ring_base_lsb = 0x00000500, + .reo1_ring_base_msb = 0x00000504, + .reo1_ring_id = 0x00000508, + .reo1_ring_misc = 0x00000510, + .reo1_ring_hp_addr_lsb = 0x00000514, + .reo1_ring_hp_addr_msb = 0x00000518, + .reo1_ring_producer_int_setup = 0x00000524, + .reo1_ring_msi1_base_lsb = 0x00000548, + .reo1_ring_msi1_base_msb = 0x0000054C, + .reo1_ring_msi1_data = 0x00000550, + .reo1_aging_thres_ix0 = 0x00000B28, + .reo1_aging_thres_ix1 = 0x00000B2C, + .reo1_aging_thres_ix2 = 0x00000B30, + .reo1_aging_thres_ix3 = 0x00000B34, + + /* REO Exception ring address */ + .reo2_sw0_ring_base = 0x000008c0, + + /* REO Reinject ring address */ + .sw2reo_ring_base = 0x00000320, + .sw2reo1_ring_base = 0x00000398, + + /* REO cmd ring address */ + .reo_cmd_ring_base = 0x000002A8, + + /* REO status ring address */ + .reo_status_ring_base = 0x00000aa0, + + /* CE base address */ + .umac_ce0_src_reg_base = 0x01b80000, + .umac_ce0_dest_reg_base = 0x01b81000, + .umac_ce1_src_reg_base = 0x01b82000, + .umac_ce1_dest_reg_base = 0x01b83000, + + .gcc_gcc_pcie_hot_rst = 0x1e38338, + + .qrtr_node_id = 0x1e03164, +}; + +const struct ath12k_hw_regs ipq5332_regs = { + /* SW2TCL(x) R0 ring configuration address */ + .tcl1_ring_id = 0x00000918, + .tcl1_ring_misc = 0x00000920, + .tcl1_ring_tp_addr_lsb = 0x0000092c, + .tcl1_ring_tp_addr_msb = 0x00000930, + .tcl1_ring_consumer_int_setup_ix0 = 0x00000940, + .tcl1_ring_consumer_int_setup_ix1 = 0x00000944, + .tcl1_ring_msi1_base_lsb = 0x00000958, + .tcl1_ring_msi1_base_msb = 0x0000095c, + .tcl1_ring_base_lsb = 0x00000910, + .tcl1_ring_base_msb = 0x00000914, + .tcl1_ring_msi1_data = 0x00000960, + .tcl2_ring_base_lsb = 0x00000988, + .tcl_ring_base_lsb = 0x00000b68, + + /* TCL STATUS ring address */ + .tcl_status_ring_base_lsb = 0x00000d48, + + /* REO DEST ring address */ + .reo2_ring_base = 0x00000578, + .reo1_misc_ctrl_addr = 0x00000b9c, + .reo1_sw_cookie_cfg0 = 0x0000006c, + .reo1_sw_cookie_cfg1 = 0x00000070, + .reo1_qdesc_lut_base0 = 0x00000074, + .reo1_qdesc_lut_base1 = 0x00000078, + .reo1_ring_base_lsb = 0x00000500, + .reo1_ring_base_msb = 0x00000504, + .reo1_ring_id = 0x00000508, + .reo1_ring_misc = 0x00000510, + .reo1_ring_hp_addr_lsb = 0x00000514, + .reo1_ring_hp_addr_msb = 0x00000518, + .reo1_ring_producer_int_setup = 0x00000524, + .reo1_ring_msi1_base_lsb = 0x00000548, + .reo1_ring_msi1_base_msb = 0x0000054C, + .reo1_ring_msi1_data = 0x00000550, + .reo1_aging_thres_ix0 = 0x00000B28, + .reo1_aging_thres_ix1 = 0x00000B2C, + .reo1_aging_thres_ix2 = 0x00000B30, + .reo1_aging_thres_ix3 = 0x00000B34, + + /* REO Exception ring address */ + .reo2_sw0_ring_base = 0x000008c0, + + /* REO Reinject ring address */ + .sw2reo_ring_base = 0x00000320, + .sw2reo1_ring_base = 0x00000398, + + /* REO cmd ring address */ + .reo_cmd_ring_base = 0x000002A8, + + /* REO status ring address */ + .reo_status_ring_base = 0x00000aa0, + + /* WBM idle link ring address */ + .wbm_idle_ring_base_lsb = 0x00000d3c, + .wbm_idle_ring_misc_addr = 0x00000d4c, + .wbm_r0_idle_list_cntl_addr = 0x00000240, + .wbm_r0_idle_list_size_addr = 0x00000244, + .wbm_scattered_ring_base_lsb = 0x00000250, + .wbm_scattered_ring_base_msb = 0x00000254, + .wbm_scattered_desc_head_info_ix0 = 0x00000260, + .wbm_scattered_desc_head_info_ix1 = 0x00000264, + .wbm_scattered_desc_tail_info_ix0 = 0x00000270, + .wbm_scattered_desc_tail_info_ix1 = 0x00000274, + .wbm_scattered_desc_ptr_hp_addr = 0x0000027c, + + /* SW2WBM release ring address */ + .wbm_sw_release_ring_base_lsb = 0x0000037c, + + /* WBM2SW release ring address */ + .wbm0_release_ring_base_lsb = 0x00000e08, + .wbm1_release_ring_base_lsb = 0x00000e80, + + /* PPE release ring address */ + .ppe_rel_ring_base = 0x0000046c, + + /* CE address */ + .umac_ce0_src_reg_base = 0x00740000 - + HAL_IPQ5332_CE_WFSS_REG_BASE, + .umac_ce0_dest_reg_base = 0x00741000 - + HAL_IPQ5332_CE_WFSS_REG_BASE, + .umac_ce1_src_reg_base = 0x00742000 - + HAL_IPQ5332_CE_WFSS_REG_BASE, + .umac_ce1_dest_reg_base = 0x00743000 - + HAL_IPQ5332_CE_WFSS_REG_BASE, +}; + +static inline +bool ath12k_hal_rx_desc_get_first_msdu_qcn9274(struct hal_rx_desc *desc) +{ + return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, + RX_MSDU_END_INFO5_FIRST_MSDU); +} + +static inline +bool ath12k_hal_rx_desc_get_last_msdu_qcn9274(struct hal_rx_desc *desc) +{ + return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, + RX_MSDU_END_INFO5_LAST_MSDU); +} + +u8 ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274(struct hal_rx_desc *desc) +{ + return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, + RX_MSDU_END_INFO5_L3_HDR_PADDING); +} + +static inline +bool ath12k_hal_rx_desc_encrypt_valid_qcn9274(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, + RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID); +} + +static inline +u32 ath12k_hal_rx_desc_get_encrypt_type_qcn9274(struct hal_rx_desc *desc) +{ + if (!ath12k_hal_rx_desc_encrypt_valid_qcn9274(desc)) + return HAL_ENCRYPT_TYPE_OPEN; + + return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info2, + RX_MPDU_START_INFO2_ENC_TYPE); +} + +static inline +u8 ath12k_hal_rx_desc_get_decap_type_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info11, + RX_MSDU_END_INFO11_DECAP_FORMAT); +} + +static inline +u8 ath12k_hal_rx_desc_get_mesh_ctl_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info11, + RX_MSDU_END_INFO11_MESH_CTRL_PRESENT); +} + +static inline +bool ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_qcn9274(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID); +} + +static inline +bool ath12k_hal_rx_desc_get_mpdu_fc_valid_qcn9274(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_FCTRL_VALID); +} + +static inline +u16 ath12k_hal_rx_desc_get_mpdu_start_seq_no_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_SEQ_NUM); +} + +static inline +u16 ath12k_hal_rx_desc_get_msdu_len_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info10, + RX_MSDU_END_INFO10_MSDU_LENGTH); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_sgi_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, + RX_MSDU_END_INFO12_SGI); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_rate_mcs_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, + RX_MSDU_END_INFO12_RATE_MCS); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_rx_bw_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, + RX_MSDU_END_INFO12_RECV_BW); +} + +static inline +u32 ath12k_hal_rx_desc_get_msdu_freq_qcn9274(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.phy_meta_data); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_pkt_type_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, + RX_MSDU_END_INFO12_PKT_TYPE); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_nss_qcn9274(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12, + RX_MSDU_END_INFO12_MIMO_SS_BITMAP); +} + +static inline +u8 ath12k_hal_rx_desc_get_mpdu_tid_qcn9274(struct hal_rx_desc *desc) +{ + return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5, + RX_MSDU_END_INFO5_TID); +} + +static inline +u16 ath12k_hal_rx_desc_get_mpdu_peer_id_qcn9274(struct hal_rx_desc *desc) +{ + return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.sw_peer_id); +} + +void ath12k_hal_rx_desc_copy_end_tlv_qcn9274(struct hal_rx_desc *fdesc, + struct hal_rx_desc *ldesc) +{ + fdesc->u.qcn9274_compact.msdu_end = ldesc->u.qcn9274_compact.msdu_end; +} + +u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcn9274(struct hal_rx_desc *desc) +{ + return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.phy_ppdu_id); +} + +void ath12k_hal_rx_desc_set_msdu_len_qcn9274(struct hal_rx_desc *desc, u16 len) +{ + u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info10); + + info = u32_replace_bits(info, len, RX_MSDU_END_INFO10_MSDU_LENGTH); + desc->u.qcn9274_compact.msdu_end.info10 = __cpu_to_le32(info); +} + +u8 *ath12k_hal_rx_desc_get_msdu_payload_qcn9274(struct hal_rx_desc *desc) +{ + return &desc->u.qcn9274_compact.msdu_payload[0]; +} + +u32 ath12k_hal_rx_desc_get_mpdu_start_offset_qcn9274(void) +{ + return offsetof(struct hal_rx_desc_qcn9274_compact, mpdu_start); +} + +u32 ath12k_hal_rx_desc_get_msdu_end_offset_qcn9274(void) +{ + return offsetof(struct hal_rx_desc_qcn9274_compact, msdu_end); +} + +static inline +bool ath12k_hal_rx_desc_mac_addr2_valid_qcn9274(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) & + RX_MPDU_START_INFO4_MAC_ADDR2_VALID; +} + +static inline +u8 *ath12k_hal_rx_desc_mpdu_start_addr2_qcn9274(struct hal_rx_desc *desc) +{ + return desc->u.qcn9274_compact.mpdu_start.addr2; +} + +static inline +bool ath12k_hal_rx_desc_is_da_mcbc_qcn9274(struct hal_rx_desc *desc) +{ + return __le16_to_cpu(desc->u.qcn9274_compact.msdu_end.info5) & + RX_MSDU_END_INFO5_DA_IS_MCBC; +} + +static inline +bool ath12k_hal_rx_h_msdu_done_qcn9274(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14, + RX_MSDU_END_INFO14_MSDU_DONE); +} + +static inline +bool ath12k_hal_rx_h_l4_cksum_fail_qcn9274(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13, + RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL); +} + +static inline +bool ath12k_hal_rx_h_ip_cksum_fail_qcn9274(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13, + RX_MSDU_END_INFO13_IP_CKSUM_FAIL); +} + +static inline +bool ath12k_hal_rx_h_is_decrypted_qcn9274(struct hal_rx_desc *desc) +{ + return (le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14, + RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) == + RX_DESC_DECRYPT_STATUS_CODE_OK); +} + +u32 ath12k_hal_get_rx_desc_size_qcn9274(void) +{ + return sizeof(struct hal_rx_desc_qcn9274_compact); +} + +u8 ath12k_hal_rx_desc_get_msdu_src_link_qcn9274(struct hal_rx_desc *desc) +{ + return le64_get_bits(desc->u.qcn9274_compact.msdu_end.msdu_end_tag, + RX_MSDU_END_64_TLV_SRC_LINK_ID); +} + +u16 ath12k_hal_rx_mpdu_start_wmask_get_qcn9274(void) +{ + return QCN9274_MPDU_START_WMASK; +} + +u32 ath12k_hal_rx_msdu_end_wmask_get_qcn9274(void) +{ + return QCN9274_MSDU_END_WMASK; +} + +static u32 ath12k_hal_rx_h_mpdu_err_qcn9274(struct hal_rx_desc *desc) +{ + u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info13); + u32 errmap = 0; + + if (info & RX_MSDU_END_INFO13_FCS_ERR) + errmap |= HAL_RX_MPDU_ERR_FCS; + + if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) + errmap |= HAL_RX_MPDU_ERR_DECRYPT; + + if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) + errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; + + if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) + errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; + + if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) + errmap |= HAL_RX_MPDU_ERR_OVERFLOW; + + if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) + errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; + + if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) + errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; + + return errmap; +} + +void ath12k_hal_rx_desc_get_crypto_hdr_qcn9274(struct hal_rx_desc *desc, + u8 *crypto_hdr, + enum hal_encrypt_type enctype) +{ + unsigned int key_id; + + switch (enctype) { + case HAL_ENCRYPT_TYPE_OPEN: + return; + case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: + case HAL_ENCRYPT_TYPE_TKIP_MIC: + crypto_hdr[0] = + HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]); + crypto_hdr[1] = 0; + crypto_hdr[2] = + HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]); + break; + case HAL_ENCRYPT_TYPE_CCMP_128: + case HAL_ENCRYPT_TYPE_CCMP_256: + case HAL_ENCRYPT_TYPE_GCMP_128: + case HAL_ENCRYPT_TYPE_AES_GCMP_256: + crypto_hdr[0] = + HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]); + crypto_hdr[1] = + HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]); + crypto_hdr[2] = 0; + break; + case HAL_ENCRYPT_TYPE_WEP_40: + case HAL_ENCRYPT_TYPE_WEP_104: + case HAL_ENCRYPT_TYPE_WEP_128: + case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4: + case HAL_ENCRYPT_TYPE_WAPI: + return; + } + key_id = le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info5, + RX_MPDU_START_INFO5_KEY_ID); + crypto_hdr[3] = 0x20 | (key_id << 6); + crypto_hdr[4] = + HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.qcn9274_compact.mpdu_start.pn[0]); + crypto_hdr[5] = + HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.qcn9274_compact.mpdu_start.pn[0]); + crypto_hdr[6] = + HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[1]); + crypto_hdr[7] = + HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[1]); +} + +void ath12k_hal_rx_desc_get_dot11_hdr_qcn9274(struct hal_rx_desc *desc, + struct ieee80211_hdr *hdr) +{ + hdr->frame_control = desc->u.qcn9274_compact.mpdu_start.frame_ctrl; + hdr->duration_id = desc->u.qcn9274_compact.mpdu_start.duration; + ether_addr_copy(hdr->addr1, desc->u.qcn9274_compact.mpdu_start.addr1); + ether_addr_copy(hdr->addr2, desc->u.qcn9274_compact.mpdu_start.addr2); + ether_addr_copy(hdr->addr3, desc->u.qcn9274_compact.mpdu_start.addr3); + if (__le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) & + RX_MPDU_START_INFO4_MAC_ADDR4_VALID) { + ether_addr_copy(hdr->addr4, desc->u.qcn9274_compact.mpdu_start.addr4); + } + hdr->seq_ctrl = desc->u.qcn9274_compact.mpdu_start.seq_ctrl; +} + +void ath12k_hal_extract_rx_desc_data_qcn9274(struct hal_rx_desc_data *rx_desc_data, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc *ldesc) +{ + rx_desc_data->is_first_msdu = ath12k_hal_rx_desc_get_first_msdu_qcn9274(ldesc); + rx_desc_data->is_last_msdu = ath12k_hal_rx_desc_get_last_msdu_qcn9274(ldesc); + rx_desc_data->l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274(ldesc); + rx_desc_data->enctype = ath12k_hal_rx_desc_get_encrypt_type_qcn9274(rx_desc); + rx_desc_data->decap_type = ath12k_hal_rx_desc_get_decap_type_qcn9274(rx_desc); + rx_desc_data->mesh_ctrl_present = + ath12k_hal_rx_desc_get_mesh_ctl_qcn9274(rx_desc); + rx_desc_data->seq_ctl_valid = + ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_qcn9274(rx_desc); + rx_desc_data->fc_valid = ath12k_hal_rx_desc_get_mpdu_fc_valid_qcn9274(rx_desc); + rx_desc_data->seq_no = ath12k_hal_rx_desc_get_mpdu_start_seq_no_qcn9274(rx_desc); + rx_desc_data->msdu_len = ath12k_hal_rx_desc_get_msdu_len_qcn9274(ldesc); + rx_desc_data->sgi = ath12k_hal_rx_desc_get_msdu_sgi_qcn9274(rx_desc); + rx_desc_data->rate_mcs = ath12k_hal_rx_desc_get_msdu_rate_mcs_qcn9274(rx_desc); + rx_desc_data->bw = ath12k_hal_rx_desc_get_msdu_rx_bw_qcn9274(rx_desc); + rx_desc_data->phy_meta_data = ath12k_hal_rx_desc_get_msdu_freq_qcn9274(rx_desc); + rx_desc_data->pkt_type = ath12k_hal_rx_desc_get_msdu_pkt_type_qcn9274(rx_desc); + rx_desc_data->nss = hweight8(ath12k_hal_rx_desc_get_msdu_nss_qcn9274(rx_desc)); + rx_desc_data->tid = ath12k_hal_rx_desc_get_mpdu_tid_qcn9274(rx_desc); + rx_desc_data->peer_id = ath12k_hal_rx_desc_get_mpdu_peer_id_qcn9274(rx_desc); + rx_desc_data->addr2_present = ath12k_hal_rx_desc_mac_addr2_valid_qcn9274(rx_desc); + rx_desc_data->addr2 = ath12k_hal_rx_desc_mpdu_start_addr2_qcn9274(rx_desc); + rx_desc_data->is_mcbc = ath12k_hal_rx_desc_is_da_mcbc_qcn9274(rx_desc); + rx_desc_data->msdu_done = ath12k_hal_rx_h_msdu_done_qcn9274(ldesc); + rx_desc_data->l4_csum_fail = ath12k_hal_rx_h_l4_cksum_fail_qcn9274(rx_desc); + rx_desc_data->ip_csum_fail = ath12k_hal_rx_h_ip_cksum_fail_qcn9274(rx_desc); + rx_desc_data->is_decrypted = ath12k_hal_rx_h_is_decrypted_qcn9274(rx_desc); + rx_desc_data->err_bitmap = ath12k_hal_rx_h_mpdu_err_qcn9274(rx_desc); +} + +static u16 ath12k_hal_reo_status_dec_tlv_hdr_qcn9274(void *tlv, void **desc) +{ + u16 tag; + + *desc = ath12k_hal_decode_tlv64_hdr(tlv, &tag, NULL, NULL); + + return tag; +} + +const struct ath12k_hw_hal_params ath12k_hw_hal_params_qcn9274 = { + .rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM, + .wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN, +}; + +const struct ath12k_hw_hal_params ath12k_hw_hal_params_ipq5332 = { + .rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM, + .wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN, +}; + +static int ath12k_hal_srng_create_config_qcn9274(struct ath12k_hal *hal) +{ + struct hal_srng_config *s; + + hal->srng_config = kmemdup(hw_srng_config_template, + sizeof(hw_srng_config_template), + GFP_KERNEL); + if (!hal->srng_config) + return -ENOMEM; + + s = &hal->srng_config[HAL_REO_DST]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP; + s->reg_size[0] = HAL_REO2_RING_BASE_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP; + + s = &hal->srng_config[HAL_REO_EXCEPTION]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP; + + s = &hal->srng_config[HAL_REO_REINJECT]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP; + s->reg_size[0] = HAL_SW2REO1_RING_BASE_LSB(hal) - HAL_SW2REO_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_SW2REO1_RING_HP - HAL_SW2REO_RING_HP; + + s = &hal->srng_config[HAL_REO_CMD]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP; + + s = &hal->srng_config[HAL_REO_STATUS]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP; + + s = &hal->srng_config[HAL_TCL_DATA]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP; + s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(hal) - HAL_TCL1_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP; + + s = &hal->srng_config[HAL_TCL_CMD]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP; + + s = &hal->srng_config[HAL_TCL_STATUS]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP; + + s = &hal->srng_config[HAL_CE_SRC]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_BASE_LSB; + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_HP; + s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal); + s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal); + + s = &hal->srng_config[HAL_CE_DST]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_BASE_LSB; + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_HP; + s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + + s = &hal->srng_config[HAL_CE_DST_STATUS]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + + HAL_CE_DST_STATUS_RING_BASE_LSB; + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_STATUS_RING_HP; + s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + + s = &hal->srng_config[HAL_WBM_IDLE_LINK]; + s->reg_start[0] = + HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP; + + s = &hal->srng_config[HAL_SW2WBM_RELEASE]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP; + s->reg_size[0] = HAL_WBM_SW1_RELEASE_RING_BASE_LSB(hal) - + HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_WBM_SW1_RELEASE_RING_HP - HAL_WBM_SW_RELEASE_RING_HP; + + s = &hal->srng_config[HAL_WBM2SW_RELEASE]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP; + s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(hal) - + HAL_WBM0_RELEASE_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP; + + /* Some LMAC rings are not accessed from the host: + * RXDMA_BUG, RXDMA_DST, RXDMA_MONITOR_BUF, RXDMA_MONITOR_STATUS, + * RXDMA_MONITOR_DST, RXDMA_MONITOR_DESC, RXDMA_DIR_BUF_SRC, + * RXDMA_RX_MONITOR_BUF, TX_MONITOR_BUF, TX_MONITOR_DST, SW2RXDMA + */ + s = &hal->srng_config[HAL_PPE2TCL]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_BASE_LSB; + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_HP; + + s = &hal->srng_config[HAL_PPE_RELEASE]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_PPE_RELEASE_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_PPE_RELEASE_RING_HP; + + return 0; +} + +const struct ath12k_hal_tcl_to_wbm_rbm_map +ath12k_hal_tcl_to_wbm_rbm_map_qcn9274[DP_TCL_NUM_RING_MAX] = { + { + .wbm_ring_num = 0, + .rbm_id = HAL_RX_BUF_RBM_SW0_BM, + }, + { + .wbm_ring_num = 1, + .rbm_id = HAL_RX_BUF_RBM_SW1_BM, + }, + { + .wbm_ring_num = 2, + .rbm_id = HAL_RX_BUF_RBM_SW2_BM, + }, + { + .wbm_ring_num = 4, + .rbm_id = HAL_RX_BUF_RBM_SW4_BM, + }, +}; + +const struct hal_ops hal_qcn9274_ops = { + .create_srng_config = ath12k_hal_srng_create_config_qcn9274, + .rx_desc_set_msdu_len = ath12k_hal_rx_desc_set_msdu_len_qcn9274, + .rx_desc_get_dot11_hdr = ath12k_hal_rx_desc_get_dot11_hdr_qcn9274, + .rx_desc_get_crypto_header = ath12k_hal_rx_desc_get_crypto_hdr_qcn9274, + .rx_desc_copy_end_tlv = ath12k_hal_rx_desc_copy_end_tlv_qcn9274, + .rx_desc_get_msdu_src_link_id = ath12k_hal_rx_desc_get_msdu_src_link_qcn9274, + .extract_rx_desc_data = ath12k_hal_extract_rx_desc_data_qcn9274, + .rx_desc_get_l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274, + .rx_desc_get_mpdu_ppdu_id = ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcn9274, + .rx_desc_get_msdu_payload = ath12k_hal_rx_desc_get_msdu_payload_qcn9274, + .ce_dst_setup = ath12k_wifi7_hal_ce_dst_setup, + .srng_src_hw_init = ath12k_wifi7_hal_srng_src_hw_init, + .srng_dst_hw_init = ath12k_wifi7_hal_srng_dst_hw_init, + .set_umac_srng_ptr_addr = ath12k_wifi7_hal_set_umac_srng_ptr_addr, + .srng_update_shadow_config = ath12k_wifi7_hal_srng_update_shadow_config, + .srng_get_ring_id = ath12k_wifi7_hal_srng_get_ring_id, + .ce_get_desc_size = ath12k_wifi7_hal_ce_get_desc_size, + .ce_src_set_desc = ath12k_wifi7_hal_ce_src_set_desc, + .ce_dst_set_desc = ath12k_wifi7_hal_ce_dst_set_desc, + .ce_dst_status_get_length = ath12k_wifi7_hal_ce_dst_status_get_length, + .set_link_desc_addr = ath12k_wifi7_hal_set_link_desc_addr, + .tx_set_dscp_tid_map = ath12k_wifi7_hal_tx_set_dscp_tid_map, + .tx_configure_bank_register = + ath12k_wifi7_hal_tx_configure_bank_register, + .reoq_lut_addr_read_enable = ath12k_wifi7_hal_reoq_lut_addr_read_enable, + .reoq_lut_set_max_peerid = ath12k_wifi7_hal_reoq_lut_set_max_peerid, + .write_reoq_lut_addr = ath12k_wifi7_hal_write_reoq_lut_addr, + .write_ml_reoq_lut_addr = ath12k_wifi7_hal_write_ml_reoq_lut_addr, + .setup_link_idle_list = ath12k_wifi7_hal_setup_link_idle_list, + .reo_init_cmd_ring = ath12k_wifi7_hal_reo_init_cmd_ring_tlv64, + .reo_hw_setup = ath12k_wifi7_hal_reo_hw_setup, + .reo_shared_qaddr_cache_clear = ath12k_wifi7_hal_reo_shared_qaddr_cache_clear, + .rx_buf_addr_info_set = ath12k_wifi7_hal_rx_buf_addr_info_set, + .rx_buf_addr_info_get = ath12k_wifi7_hal_rx_buf_addr_info_get, + .cc_config = ath12k_wifi7_hal_cc_config, + .get_idle_link_rbm = ath12k_wifi7_hal_get_idle_link_rbm, + .rx_msdu_list_get = ath12k_wifi7_hal_rx_msdu_list_get, + .rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get, + .reo_cmd_enc_tlv_hdr = ath12k_hal_encode_tlv64_hdr, + .reo_status_dec_tlv_hdr = ath12k_hal_reo_status_dec_tlv_hdr_qcn9274, + .mon_rx_status_dec_tlv_hdr = ath12k_hal_decode_tlv64_hdr, + .get_tlv_hdr_align = ath12k_hal_get_tlv64_hdr_align, +}; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.h new file mode 100644 index 0000000000000..08c0a04694741 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear*/ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_HAL_QCN9274_H +#define ATH12K_HAL_QCN9274_H + +#include +#include +#include "../hal.h" +#include "hal_rx.h" +#include "hal.h" + +extern const struct hal_ops hal_qcn9274_ops; +extern const struct ath12k_hw_regs qcn9274_v1_regs; +extern const struct ath12k_hw_regs qcn9274_v2_regs; +extern const struct ath12k_hw_regs ipq5332_regs; +extern const struct ath12k_hal_tcl_to_wbm_rbm_map +ath12k_hal_tcl_to_wbm_rbm_map_qcn9274[DP_TCL_NUM_RING_MAX]; +extern const struct ath12k_hw_hal_params ath12k_hw_hal_params_qcn9274; +extern const struct ath12k_hw_hal_params ath12k_hw_hal_params_ipq5332; + +u8 ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274(struct hal_rx_desc *desc); +void ath12k_hal_rx_desc_copy_end_tlv_qcn9274(struct hal_rx_desc *fdesc, + struct hal_rx_desc *ldesc); +u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcn9274(struct hal_rx_desc *desc); +void ath12k_hal_rx_desc_set_msdu_len_qcn9274(struct hal_rx_desc *desc, u16 len); +u8 *ath12k_hal_rx_desc_get_msdu_payload_qcn9274(struct hal_rx_desc *desc); +u32 ath12k_hal_rx_desc_get_mpdu_start_offset_qcn9274(void); +u32 ath12k_hal_rx_desc_get_msdu_end_offset_qcn9274(void); +u32 ath12k_hal_get_rx_desc_size_qcn9274(void); +u8 ath12k_hal_rx_desc_get_msdu_src_link_qcn9274(struct hal_rx_desc *desc); +u16 ath12k_hal_rx_mpdu_start_wmask_get_qcn9274(void); +u32 ath12k_hal_rx_msdu_end_wmask_get_qcn9274(void); +void ath12k_hal_rx_desc_get_crypto_hdr_qcn9274(struct hal_rx_desc *desc, + u8 *crypto_hdr, + enum hal_encrypt_type enctype); +void ath12k_hal_rx_desc_get_dot11_hdr_qcn9274(struct hal_rx_desc *desc, + struct ieee80211_hdr *hdr); +void ath12k_hal_extract_rx_desc_data_qcn9274(struct hal_rx_desc_data *rx_desc_data, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc *ldesc); +#endif diff --git a/drivers/net/wireless/ath/ath12k/hal_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c similarity index 80% rename from drivers/net/wireless/ath/ath12k/hal_rx.c rename to drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c index 669096278fdd4..49c6932897094 100644 --- a/drivers/net/wireless/ath/ath12k/hal_rx.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c @@ -1,18 +1,20 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ -#include "debug.h" -#include "hal.h" +#include "../debug.h" +#include "../hal.h" +#include "../hif.h" #include "hal_tx.h" #include "hal_rx.h" #include "hal_desc.h" -#include "hif.h" +#include "hal.h" -static void ath12k_hal_reo_set_desc_hdr(struct hal_desc_header *hdr, - u8 owner, u8 buffer_type, u32 magic) +static +void ath12k_wifi7_hal_reo_set_desc_hdr(struct hal_desc_header *hdr, + u8 owner, u8 buffer_type, u32 magic) { hdr->info0 = le32_encode_bits(owner, HAL_DESC_HDR_INFO0_OWNER) | le32_encode_bits(buffer_type, HAL_DESC_HDR_INFO0_BUF_TYPE); @@ -21,15 +23,13 @@ static void ath12k_hal_reo_set_desc_hdr(struct hal_desc_header *hdr, hdr->info0 |= le32_encode_bits(magic, HAL_DESC_HDR_INFO0_DBG_RESERVED); } -static int ath12k_hal_reo_cmd_queue_stats(struct hal_tlv_64_hdr *tlv, - struct ath12k_hal_reo_cmd *cmd) +static int ath12k_wifi7_hal_reo_cmd_queue_stats(struct ath12k_hal *hal, void *tlv, + struct ath12k_hal_reo_cmd *cmd) { struct hal_reo_get_queue_stats *desc; - tlv->tl = le64_encode_bits(HAL_REO_GET_QUEUE_STATS, HAL_TLV_HDR_TAG) | - le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN); - - desc = (struct hal_reo_get_queue_stats *)tlv->value; + desc = hal->ops->reo_cmd_enc_tlv_hdr(tlv, HAL_REO_GET_QUEUE_STATS, + sizeof(*desc)); memset_startat(desc, 0, queue_addr_lo); desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED); @@ -45,9 +45,8 @@ static int ath12k_hal_reo_cmd_queue_stats(struct hal_tlv_64_hdr *tlv, return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER); } -static int ath12k_hal_reo_cmd_flush_cache(struct ath12k_hal *hal, - struct hal_tlv_64_hdr *tlv, - struct ath12k_hal_reo_cmd *cmd) +static int ath12k_wifi7_hal_reo_cmd_flush_cache(struct ath12k_hal *hal, void *tlv, + struct ath12k_hal_reo_cmd *cmd) { struct hal_reo_flush_cache *desc; u8 avail_slot = ffz(hal->avail_blk_resource); @@ -59,10 +58,8 @@ static int ath12k_hal_reo_cmd_flush_cache(struct ath12k_hal *hal, hal->current_blk_index = avail_slot; } - tlv->tl = le64_encode_bits(HAL_REO_FLUSH_CACHE, HAL_TLV_HDR_TAG) | - le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN); - - desc = (struct hal_reo_flush_cache *)tlv->value; + desc = hal->ops->reo_cmd_enc_tlv_hdr(tlv, HAL_REO_FLUSH_CACHE, + sizeof(*desc)); memset_startat(desc, 0, cache_addr_lo); desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED); @@ -95,15 +92,14 @@ static int ath12k_hal_reo_cmd_flush_cache(struct ath12k_hal *hal, return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER); } -static int ath12k_hal_reo_cmd_update_rx_queue(struct hal_tlv_64_hdr *tlv, - struct ath12k_hal_reo_cmd *cmd) +static int +ath12k_wifi7_hal_reo_cmd_update_rx_queue(struct ath12k_hal *hal, void *tlv, + struct ath12k_hal_reo_cmd *cmd) { struct hal_reo_update_rx_queue *desc; - tlv->tl = le64_encode_bits(HAL_REO_UPDATE_RX_REO_QUEUE, HAL_TLV_HDR_TAG) | - le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN); - - desc = (struct hal_reo_update_rx_queue *)tlv->value; + desc = hal->ops->reo_cmd_enc_tlv_hdr(tlv, HAL_REO_UPDATE_RX_REO_QUEUE, + sizeof(*desc)); memset_startat(desc, 0, queue_addr_lo); desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED); @@ -220,11 +216,12 @@ static int ath12k_hal_reo_cmd_update_rx_queue(struct hal_tlv_64_hdr *tlv, return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER); } -int ath12k_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng, - enum hal_reo_cmd_type type, - struct ath12k_hal_reo_cmd *cmd) +int ath12k_wifi7_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng, + enum hal_reo_cmd_type type, + struct ath12k_hal_reo_cmd *cmd) { - struct hal_tlv_64_hdr *reo_desc; + struct ath12k_hal *hal = &ab->hal; + void *reo_desc; int ret; spin_lock_bh(&srng->lock); @@ -238,13 +235,13 @@ int ath12k_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng, switch (type) { case HAL_REO_CMD_GET_QUEUE_STATS: - ret = ath12k_hal_reo_cmd_queue_stats(reo_desc, cmd); + ret = ath12k_wifi7_hal_reo_cmd_queue_stats(hal, reo_desc, cmd); break; case HAL_REO_CMD_FLUSH_CACHE: - ret = ath12k_hal_reo_cmd_flush_cache(&ab->hal, reo_desc, cmd); + ret = ath12k_wifi7_hal_reo_cmd_flush_cache(hal, reo_desc, cmd); break; case HAL_REO_CMD_UPDATE_RX_QUEUE: - ret = ath12k_hal_reo_cmd_update_rx_queue(reo_desc, cmd); + ret = ath12k_wifi7_hal_reo_cmd_update_rx_queue(hal, reo_desc, cmd); break; case HAL_REO_CMD_FLUSH_QUEUE: case HAL_REO_CMD_UNBLOCK_CACHE: @@ -265,8 +262,9 @@ int ath12k_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng, return ret; } -void ath12k_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo, - dma_addr_t paddr, u32 cookie, u8 manager) +void ath12k_wifi7_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo, + dma_addr_t paddr, u32 cookie, + u8 manager) { u32 paddr_lo, paddr_hi; @@ -278,9 +276,9 @@ void ath12k_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo, le32_encode_bits(manager, BUFFER_ADDR_INFO1_RET_BUF_MGR); } -void ath12k_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo, - dma_addr_t *paddr, - u32 *cookie, u8 *rbm) +void ath12k_wifi7_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo, + dma_addr_t *paddr, + u32 *cookie, u8 *rbm) { *paddr = (((u64)le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_ADDR)) << 32) | le32_get_bits(binfo->info0, BUFFER_ADDR_INFO0_ADDR); @@ -288,9 +286,10 @@ void ath12k_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo, *rbm = le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_RET_BUF_MGR); } -void ath12k_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, u32 *num_msdus, - u32 *msdu_cookies, - enum hal_rx_buf_return_buf_manager *rbm) +void +ath12k_wifi7_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, + u32 *num_msdus, u32 *msdu_cookies, + enum hal_rx_buf_return_buf_manager *rbm) { struct hal_rx_msdu_details *msdu; u32 val; @@ -317,19 +316,20 @@ void ath12k_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, u32 *num_ms } } -int ath12k_hal_desc_reo_parse_err(struct ath12k_base *ab, - struct hal_reo_dest_ring *desc, - dma_addr_t *paddr, u32 *desc_bank) +int ath12k_wifi7_hal_desc_reo_parse_err(struct ath12k_dp *dp, + struct hal_reo_dest_ring *desc, + dma_addr_t *paddr, u32 *desc_bank) { + struct ath12k_base *ab = dp->ab; enum hal_reo_dest_ring_push_reason push_reason; enum hal_reo_dest_ring_error_code err_code; - u32 cookie, val; + u32 cookie; push_reason = le32_get_bits(desc->info0, HAL_REO_DEST_RING_INFO0_PUSH_REASON); err_code = le32_get_bits(desc->info0, HAL_REO_DEST_RING_INFO0_ERROR_CODE); - ab->device_stats.reo_error[err_code]++; + dp->device_stats.reo_error[err_code]++; if (push_reason != HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED && push_reason != HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) { @@ -338,20 +338,15 @@ int ath12k_hal_desc_reo_parse_err(struct ath12k_base *ab, return -EINVAL; } - val = le32_get_bits(desc->info0, HAL_REO_DEST_RING_INFO0_BUFFER_TYPE); - if (val != HAL_REO_DEST_RING_BUFFER_TYPE_LINK_DESC) { - ath12k_warn(ab, "expected buffer type link_desc"); - return -EINVAL; - } - - ath12k_hal_rx_reo_ent_paddr_get(ab, &desc->buf_addr_info, paddr, &cookie); + ath12k_wifi7_hal_rx_reo_ent_paddr_get(&desc->buf_addr_info, paddr, + &cookie); *desc_bank = u32_get_bits(cookie, DP_LINK_DESC_BANK_MASK); return 0; } -int ath12k_hal_wbm_desc_parse_err(struct ath12k_base *ab, void *desc, - struct hal_rx_wbm_rel_info *rel_info) +int ath12k_wifi7_hal_wbm_desc_parse_err(struct ath12k_dp *dp, void *desc, + struct hal_rx_wbm_rel_info *rel_info) { struct hal_wbm_release_ring *wbm_desc = desc; struct hal_wbm_release_ring_cc_rx *wbm_cc_desc = desc; @@ -384,7 +379,7 @@ int ath12k_hal_wbm_desc_parse_err(struct ath12k_base *ab, void *desc, val = le32_get_bits(wbm_desc->buf_addr_info.info1, BUFFER_ADDR_INFO1_RET_BUF_MGR); if (val != HAL_RX_BUF_RBM_SW3_BM) { - ab->device_stats.invalid_rbm++; + dp->device_stats.invalid_rbm++; return -EINVAL; } @@ -396,7 +391,7 @@ int ath12k_hal_wbm_desc_parse_err(struct ath12k_base *ab, void *desc, val = le32_get_bits(wbm_cc_desc->info0, HAL_WBM_RELEASE_RX_CC_INFO0_RBM); if (val != HAL_RX_BUF_RBM_SW3_BM) { - ab->device_stats.invalid_rbm++; + dp->device_stats.invalid_rbm++; return -EINVAL; } @@ -435,12 +430,13 @@ int ath12k_hal_wbm_desc_parse_err(struct ath12k_base *ab, void *desc, HAL_WBM_RELEASE_INFO0_RXDMA_ERROR_CODE); } + rel_info->peer_metadata = wbm_desc->info2; + return 0; } -void ath12k_hal_rx_reo_ent_paddr_get(struct ath12k_base *ab, - struct ath12k_buffer_addr *buff_addr, - dma_addr_t *paddr, u32 *cookie) +void ath12k_wifi7_hal_rx_reo_ent_paddr_get(struct ath12k_buffer_addr *buff_addr, + dma_addr_t *paddr, u32 *cookie) { *paddr = ((u64)(le32_get_bits(buff_addr->info1, BUFFER_ADDR_INFO1_ADDR)) << 32) | @@ -449,10 +445,10 @@ void ath12k_hal_rx_reo_ent_paddr_get(struct ath12k_base *ab, *cookie = le32_get_bits(buff_addr->info1, BUFFER_ADDR_INFO1_SW_COOKIE); } -void ath12k_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr, - u32 *sw_cookie, - struct ath12k_buffer_addr **pp_buf_addr, - u8 *rbm, u32 *msdu_cnt) +void ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr, + u32 *sw_cookie, + struct ath12k_buffer_addr **pp_buf_addr, + u8 *rbm, u32 *msdu_cnt) { struct hal_reo_entrance_ring *reo_ent_ring = (struct hal_reo_entrance_ring *)rx_desc; @@ -480,11 +476,14 @@ void ath12k_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr, *pp_buf_addr = (void *)buf_addr_info; } -void ath12k_hal_rx_msdu_list_get(struct ath12k *ar, - struct hal_rx_msdu_link *link_desc, - struct hal_rx_msdu_list *msdu_list, - u16 *num_msdus) +void ath12k_wifi7_hal_rx_msdu_list_get(struct ath12k *ar, + void *link_desc_opaque, + void *msdu_list_opaque, u16 *num_msdus) { + struct hal_rx_msdu_link *link_desc = + (struct hal_rx_msdu_link *)link_desc_opaque; + struct hal_rx_msdu_list *msdu_list = + (struct hal_rx_msdu_list *)msdu_list_opaque; struct hal_rx_msdu_details *msdu_details = NULL; struct rx_msdu_desc *msdu_desc_info = NULL; u32 last = 0, first = 0; @@ -529,10 +528,11 @@ void ath12k_hal_rx_msdu_list_get(struct ath12k *ar, *num_msdus = i; } -void ath12k_hal_rx_msdu_link_desc_set(struct ath12k_base *ab, - struct hal_wbm_release_ring *desc, - struct ath12k_buffer_addr *buf_addr_info, - enum hal_wbm_rel_bm_act action) +void +ath12k_wifi7_hal_rx_msdu_link_desc_set(struct ath12k_base *ab, + struct hal_wbm_release_ring *desc, + struct ath12k_buffer_addr *buf_addr_info, + enum hal_wbm_rel_bm_act action) { desc->buf_addr_info = *buf_addr_info; desc->info0 |= le32_encode_bits(HAL_WBM_REL_SRC_MODULE_SW, @@ -542,12 +542,10 @@ void ath12k_hal_rx_msdu_link_desc_set(struct ath12k_base *ab, HAL_WBM_RELEASE_INFO0_DESC_TYPE); } -void ath12k_hal_reo_status_queue_stats(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status) +void ath12k_wifi7_hal_reo_status_queue_stats(struct ath12k_base *ab, + struct hal_reo_get_queue_stats_status *desc, + struct hal_reo_status *status) { - struct hal_reo_get_queue_stats_status *desc = - (struct hal_reo_get_queue_stats_status *)tlv->value; - status->uniform_hdr.cmd_num = le32_get_bits(desc->hdr.info0, HAL_REO_STATUS_HDR_INFO0_STATUS_NUM); @@ -605,12 +603,10 @@ void ath12k_hal_reo_status_queue_stats(struct ath12k_base *ab, struct hal_tlv_64 HAL_REO_GET_QUEUE_STATS_STATUS_INFO5_LOOPING_CNT)); } -void ath12k_hal_reo_flush_queue_status(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status) +void ath12k_wifi7_hal_reo_flush_queue_status(struct ath12k_base *ab, + struct hal_reo_flush_queue_status *desc, + struct hal_reo_status *status) { - struct hal_reo_flush_queue_status *desc = - (struct hal_reo_flush_queue_status *)tlv->value; - status->uniform_hdr.cmd_num = le32_get_bits(desc->hdr.info0, HAL_REO_STATUS_HDR_INFO0_STATUS_NUM); @@ -622,12 +618,12 @@ void ath12k_hal_reo_flush_queue_status(struct ath12k_base *ab, struct hal_tlv_64 HAL_REO_FLUSH_QUEUE_INFO0_ERR_DETECTED); } -void ath12k_hal_reo_flush_cache_status(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status) +void +ath12k_wifi7_hal_reo_flush_cache_status(struct ath12k_base *ab, + struct hal_reo_flush_cache_status *desc, + struct hal_reo_status *status) { struct ath12k_hal *hal = &ab->hal; - struct hal_reo_flush_cache_status *desc = - (struct hal_reo_flush_cache_status *)tlv->value; status->uniform_hdr.cmd_num = le32_get_bits(desc->hdr.info0, @@ -663,12 +659,11 @@ void ath12k_hal_reo_flush_cache_status(struct ath12k_base *ab, struct hal_tlv_64 HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_COUNT); } -void ath12k_hal_reo_unblk_cache_status(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status) +void ath12k_wifi7_hal_reo_unblk_cache_status(struct ath12k_base *ab, + struct hal_reo_unblock_cache_status *desc, + struct hal_reo_status *status) { struct ath12k_hal *hal = &ab->hal; - struct hal_reo_unblock_cache_status *desc = - (struct hal_reo_unblock_cache_status *)tlv->value; status->uniform_hdr.cmd_num = le32_get_bits(desc->hdr.info0, @@ -690,13 +685,11 @@ void ath12k_hal_reo_unblk_cache_status(struct ath12k_base *ab, struct hal_tlv_64 hal->avail_blk_resource &= ~BIT(hal->current_blk_index); } -void ath12k_hal_reo_flush_timeout_list_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status) +void +ath12k_wifi7_hal_reo_flush_timeout_list_status(struct ath12k_base *ab, + struct hal_reo_flush_timeout_list_status *desc, + struct hal_reo_status *status) { - struct hal_reo_flush_timeout_list_status *desc = - (struct hal_reo_flush_timeout_list_status *)tlv->value; - status->uniform_hdr.cmd_num = le32_get_bits(desc->hdr.info0, HAL_REO_STATUS_HDR_INFO0_STATUS_NUM); @@ -719,13 +712,11 @@ void ath12k_hal_reo_flush_timeout_list_status(struct ath12k_base *ab, HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_FWD_BUF_COUNT); } -void ath12k_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status) +void +ath12k_wifi7_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab, + struct hal_reo_desc_thresh_reached_status *desc, + struct hal_reo_status *status) { - struct hal_reo_desc_thresh_reached_status *desc = - (struct hal_reo_desc_thresh_reached_status *)tlv->value; - status->uniform_hdr.cmd_num = le32_get_bits(desc->hdr.info0, HAL_REO_STATUS_HDR_INFO0_STATUS_NUM); @@ -754,13 +745,10 @@ void ath12k_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab, HAL_REO_DESC_THRESH_STATUS_INFO4_LINK_DESC_COUNTER_SUM); } -void ath12k_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status) +void ath12k_wifi7_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab, + struct hal_reo_status_hdr *desc, + struct hal_reo_status *status) { - struct hal_reo_status_hdr *desc = - (struct hal_reo_status_hdr *)tlv->value; - status->uniform_hdr.cmd_num = le32_get_bits(desc->info0, HAL_REO_STATUS_HDR_INFO0_STATUS_NUM); @@ -769,7 +757,7 @@ void ath12k_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab, HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS); } -u32 ath12k_hal_reo_qdesc_size(u32 ba_window_size, u8 tid) +u32 ath12k_wifi7_hal_reo_qdesc_size(u32 ba_window_size, u8 tid) { u32 num_ext_desc, num_1k_desc = 0; @@ -795,15 +783,15 @@ u32 ath12k_hal_reo_qdesc_size(u32 ba_window_size, u8 tid) (num_1k_desc * sizeof(struct hal_rx_reo_queue_1k)); } -void ath12k_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc, - int tid, u32 ba_window_size, - u32 start_seq, enum hal_pn_type type) +void ath12k_wifi7_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc, + int tid, u32 ba_window_size, + u32 start_seq, enum hal_pn_type type) { struct hal_rx_reo_queue_ext *ext_desc; - ath12k_hal_reo_set_desc_hdr(&qdesc->desc_hdr, HAL_DESC_REO_OWNED, - HAL_DESC_REO_QUEUE_DESC, - REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_0); + ath12k_wifi7_hal_reo_set_desc_hdr(&qdesc->desc_hdr, HAL_DESC_REO_OWNED, + HAL_DESC_REO_QUEUE_DESC, + REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_0); qdesc->rx_queue_num = le32_encode_bits(tid, HAL_RX_REO_QUEUE_RX_QUEUE_NUMBER); @@ -861,21 +849,24 @@ void ath12k_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc, * queue descriptor in Rx peer entry as part of dp_rx_tid_update. */ memset(ext_desc, 0, 3 * sizeof(*ext_desc)); - ath12k_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, HAL_DESC_REO_OWNED, - HAL_DESC_REO_QUEUE_EXT_DESC, - REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_1); + ath12k_wifi7_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, + HAL_DESC_REO_OWNED, + HAL_DESC_REO_QUEUE_EXT_DESC, + REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_1); ext_desc++; - ath12k_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, HAL_DESC_REO_OWNED, - HAL_DESC_REO_QUEUE_EXT_DESC, - REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_2); + ath12k_wifi7_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, + HAL_DESC_REO_OWNED, + HAL_DESC_REO_QUEUE_EXT_DESC, + REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_2); ext_desc++; - ath12k_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, HAL_DESC_REO_OWNED, - HAL_DESC_REO_QUEUE_EXT_DESC, - REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_3); + ath12k_wifi7_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, + HAL_DESC_REO_OWNED, + HAL_DESC_REO_QUEUE_EXT_DESC, + REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_3); } -void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, - struct hal_srng *srng) +void ath12k_wifi7_hal_reo_init_cmd_ring_tlv64(struct ath12k_base *ab, + struct hal_srng *srng) { struct hal_srng_params params; struct hal_tlv_64_hdr *tlv; @@ -899,8 +890,35 @@ void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, } } -void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map) +void ath12k_wifi7_hal_reo_init_cmd_ring_tlv32(struct ath12k_base *ab, + struct hal_srng *srng) +{ + struct hal_reo_get_queue_stats *desc; + struct hal_srng_params params; + struct hal_tlv_hdr *tlv; + int i, cmd_num = 1; + int entry_size; + u8 *entry; + + memset(¶ms, 0, sizeof(params)); + + entry_size = ath12k_hal_srng_get_entrysize(ab, HAL_REO_CMD); + ath12k_hal_srng_get_params(ab, srng, ¶ms); + entry = (u8 *)params.ring_base_vaddr; + + for (i = 0; i < params.num_entries; i++) { + tlv = (struct hal_tlv_hdr *)entry; + desc = (struct hal_reo_get_queue_stats *)tlv->value; + desc->cmd.info0 = le32_encode_bits(cmd_num++, + HAL_REO_CMD_HDR_INFO0_CMD_NUMBER); + entry += entry_size; + } +} + +void ath12k_wifi7_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map) { + struct ath12k_hal *hal = &ab->hal; + u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG; u32 val; @@ -910,7 +928,7 @@ void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map) u32_encode_bits(1, HAL_REO1_GEN_ENABLE_AGING_FLUSH_ENABLE); ath12k_hif_write32(ab, reo_base + HAL_REO1_GEN_ENABLE, val); - val = ath12k_hif_read32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(ab)); + val = ath12k_hif_read32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(hal)); val &= ~(HAL_REO1_MISC_CTL_FRAG_DST_RING | HAL_REO1_MISC_CTL_BAR_DST_RING); @@ -918,15 +936,15 @@ void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map) HAL_REO1_MISC_CTL_FRAG_DST_RING); val |= u32_encode_bits(HAL_SRNG_RING_ID_REO2SW0, HAL_REO1_MISC_CTL_BAR_DST_RING); - ath12k_hif_write32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(ab), val); + ath12k_hif_write32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(hal), val); - ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_0(ab), + ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_0(hal), HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC); - ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_1(ab), + ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_1(hal), HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC); - ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_2(ab), + ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_2(hal), HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC); - ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_3(ab), + ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_3(hal), HAL_DEFAULT_VO_REO_TIMEOUT_USEC); ath12k_hif_write32(ab, reo_base + HAL_REO1_DEST_RING_CTRL_IX_2, @@ -935,19 +953,21 @@ void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map) ring_hash_map); } -void ath12k_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab) +void ath12k_wifi7_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab) { u32 val; + struct ath12k_hal *hal = &ab->hal; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); - lockdep_assert_held(&ab->base_lock); + lockdep_assert_held(&dp->dp_lock); val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + - HAL_REO1_QDESC_ADDR(ab)); + HAL_REO1_QDESC_ADDR(hal)); val |= u32_encode_bits(1, HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY); ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + - HAL_REO1_QDESC_ADDR(ab), val); + HAL_REO1_QDESC_ADDR(hal), val); val &= ~HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY; ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + - HAL_REO1_QDESC_ADDR(ab), val); + HAL_REO1_QDESC_ADDR(hal), val); } diff --git a/drivers/net/wireless/ath/ath12k/hal_rx.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h similarity index 75% rename from drivers/net/wireless/ath/ath12k/hal_rx.h rename to drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h index d1ad7747b82c4..ac2a8ac03288f 100644 --- a/drivers/net/wireless/ath/ath12k/hal_rx.h +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h @@ -1,12 +1,16 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_HAL_RX_H #define ATH12K_HAL_RX_H +#include "hal_desc.h" + +struct hal_reo_status; + struct hal_rx_wbm_rel_info { u32 cookie; enum hal_wbm_rel_src_module err_rel_src; @@ -17,11 +21,9 @@ struct hal_rx_wbm_rel_info { bool continuation; void *rx_desc; bool hw_cc_done; + __le32 peer_metadata; }; -#define HAL_INVALID_PEERID 0x3fff -#define VHT_SIG_SU_NSS_MASK 0x7 - #define HAL_RX_MPDU_INFO_PN_GET_BYTE1(__val) \ le32_get_bits((__val), GENMASK(7, 0)) @@ -39,69 +41,10 @@ struct hal_rx_mon_status_tlv_hdr { u8 value[]; }; -enum hal_rx_su_mu_coding { - HAL_RX_SU_MU_CODING_BCC, - HAL_RX_SU_MU_CODING_LDPC, - HAL_RX_SU_MU_CODING_MAX, -}; - -enum hal_rx_gi { - HAL_RX_GI_0_8_US, - HAL_RX_GI_0_4_US, - HAL_RX_GI_1_6_US, - HAL_RX_GI_3_2_US, - HAL_RX_GI_MAX, -}; - -enum hal_rx_bw { - HAL_RX_BW_20MHZ, - HAL_RX_BW_40MHZ, - HAL_RX_BW_80MHZ, - HAL_RX_BW_160MHZ, - HAL_RX_BW_320MHZ, - HAL_RX_BW_MAX, -}; - -enum hal_rx_preamble { - HAL_RX_PREAMBLE_11A, - HAL_RX_PREAMBLE_11B, - HAL_RX_PREAMBLE_11N, - HAL_RX_PREAMBLE_11AC, - HAL_RX_PREAMBLE_11AX, - HAL_RX_PREAMBLE_11BA, - HAL_RX_PREAMBLE_11BE, - HAL_RX_PREAMBLE_MAX, -}; - -enum hal_rx_reception_type { - HAL_RX_RECEPTION_TYPE_SU, - HAL_RX_RECEPTION_TYPE_MU_MIMO, - HAL_RX_RECEPTION_TYPE_MU_OFDMA, - HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO, - HAL_RX_RECEPTION_TYPE_MAX, -}; - -enum hal_rx_legacy_rate { - HAL_RX_LEGACY_RATE_1_MBPS, - HAL_RX_LEGACY_RATE_2_MBPS, - HAL_RX_LEGACY_RATE_5_5_MBPS, - HAL_RX_LEGACY_RATE_6_MBPS, - HAL_RX_LEGACY_RATE_9_MBPS, - HAL_RX_LEGACY_RATE_11_MBPS, - HAL_RX_LEGACY_RATE_12_MBPS, - HAL_RX_LEGACY_RATE_18_MBPS, - HAL_RX_LEGACY_RATE_24_MBPS, - HAL_RX_LEGACY_RATE_36_MBPS, - HAL_RX_LEGACY_RATE_48_MBPS, - HAL_RX_LEGACY_RATE_54_MBPS, - HAL_RX_LEGACY_RATE_INVALID, -}; - #define HAL_TLV_STATUS_PPDU_NOT_DONE 0 #define HAL_TLV_STATUS_PPDU_DONE 1 #define HAL_TLV_STATUS_BUF_DONE 2 #define HAL_TLV_STATUS_PPDU_NON_STD_DONE 3 -#define HAL_RX_FCS_LEN 4 enum hal_rx_mon_status { HAL_RX_MON_STATUS_PPDU_NOT_DONE, @@ -113,167 +56,6 @@ enum hal_rx_mon_status { HAL_RX_MON_STATUS_MSDU_END, }; -#define HAL_RX_MAX_MPDU 1024 -#define HAL_RX_NUM_WORDS_PER_PPDU_BITMAP (HAL_RX_MAX_MPDU >> 5) - -struct hal_rx_user_status { - u32 mcs:4, - nss:3, - ofdma_info_valid:1, - ul_ofdma_ru_start_index:7, - ul_ofdma_ru_width:7, - ul_ofdma_ru_size:8; - u32 ul_ofdma_user_v0_word0; - u32 ul_ofdma_user_v0_word1; - u32 ast_index; - u32 tid; - u16 tcp_msdu_count; - u16 tcp_ack_msdu_count; - u16 udp_msdu_count; - u16 other_msdu_count; - u16 frame_control; - u8 frame_control_info_valid; - u8 data_sequence_control_info_valid; - u16 first_data_seq_ctrl; - u32 preamble_type; - u16 ht_flags; - u16 vht_flags; - u16 he_flags; - u8 rs_flags; - u8 ldpc; - u32 mpdu_cnt_fcs_ok; - u32 mpdu_cnt_fcs_err; - u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP]; - u32 mpdu_ok_byte_count; - u32 mpdu_err_byte_count; - bool ampdu_present; - u16 ampdu_id; -}; - -#define HAL_MAX_UL_MU_USERS 37 - -struct hal_rx_u_sig_info { - bool ul_dl; - u8 bw; - u8 ppdu_type_comp_mode; - u8 eht_sig_mcs; - u8 num_eht_sig_sym; - struct ieee80211_radiotap_eht_usig usig; -}; - -#define HAL_RX_MON_MAX_AGGR_SIZE 128 - -struct hal_rx_tlv_aggr_info { - bool in_progress; - u16 cur_len; - u16 tlv_tag; - u8 buf[HAL_RX_MON_MAX_AGGR_SIZE]; -}; - -struct hal_rx_radiotap_eht { - __le32 known; - __le32 data[9]; -}; - -#define EHT_MAX_USER_INFO 4 - -struct hal_rx_eht_info { - u8 num_user_info; - struct hal_rx_radiotap_eht eht; - u32 user_info[EHT_MAX_USER_INFO]; -}; - -struct hal_rx_mon_ppdu_info { - u32 ppdu_id; - u32 last_ppdu_id; - u64 ppdu_ts; - u32 num_mpdu_fcs_ok; - u32 num_mpdu_fcs_err; - u32 preamble_type; - u32 mpdu_len; - u16 chan_num; - u16 freq; - u16 tcp_msdu_count; - u16 tcp_ack_msdu_count; - u16 udp_msdu_count; - u16 other_msdu_count; - u16 peer_id; - u8 rate; - u8 mcs; - u8 nss; - u8 bw; - u8 vht_flag_values1; - u8 vht_flag_values2; - u8 vht_flag_values3[4]; - u8 vht_flag_values4; - u8 vht_flag_values5; - u16 vht_flag_values6; - u8 is_stbc; - u8 gi; - u8 sgi; - u8 ldpc; - u8 beamformed; - u8 rssi_comb; - u16 tid; - u8 fc_valid; - u16 ht_flags; - u16 vht_flags; - u16 he_flags; - u16 he_mu_flags; - u8 dcm; - u8 ru_alloc; - u8 reception_type; - u64 tsft; - u64 rx_duration; - u16 frame_control; - u32 ast_index; - u8 rs_fcs_err; - u8 rs_flags; - u8 cck_flag; - u8 ofdm_flag; - u8 ulofdma_flag; - u8 frame_control_info_valid; - u16 he_per_user_1; - u16 he_per_user_2; - u8 he_per_user_position; - u8 he_per_user_known; - u16 he_flags1; - u16 he_flags2; - u8 he_RU[4]; - u16 he_data1; - u16 he_data2; - u16 he_data3; - u16 he_data4; - u16 he_data5; - u16 he_data6; - u32 ppdu_len; - u32 prev_ppdu_id; - u32 device_id; - u16 first_data_seq_ctrl; - u8 monitor_direct_used; - u8 data_sequence_control_info_valid; - u8 ltf_size; - u8 rxpcu_filter_pass; - s8 rssi_chain[8][8]; - u32 num_users; - u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP]; - u8 addr1[ETH_ALEN]; - u8 addr2[ETH_ALEN]; - u8 addr3[ETH_ALEN]; - u8 addr4[ETH_ALEN]; - struct hal_rx_user_status userstats[HAL_MAX_UL_MU_USERS]; - u8 userid; - bool first_msdu_in_mpdu; - bool is_ampdu; - u8 medium_prot_type; - bool ppdu_continuation; - bool eht_usig; - struct hal_rx_u_sig_info u_sig_info; - bool is_eht; - struct hal_rx_eht_info eht_info; - struct hal_rx_tlv_aggr_info tlv_aggr; -}; - #define HAL_RX_PPDU_START_INFO0_PPDU_ID GENMASK(15, 0) #define HAL_RX_PPDU_START_INFO1_CHAN_NUM GENMASK(15, 0) #define HAL_RX_PPDU_START_INFO1_CHAN_FREQ GENMASK(31, 16) @@ -531,11 +313,6 @@ struct hal_rx_rxpcu_classification_overview { u32 rsvd0; } __packed; -struct hal_rx_msdu_desc_info { - u32 msdu_flags; - u16 msdu_len; /* 14 bits for length */ -}; - #define HAL_RX_NUM_MSDU_DESC 6 struct hal_rx_msdu_list { struct hal_rx_msdu_desc_info msdu_info[HAL_RX_NUM_MSDU_DESC]; @@ -588,15 +365,6 @@ struct hal_rx_resp_req_info { #define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_2 0xBDBEEF #define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_3 0xCDBEEF -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VALID BIT(30) -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VER BIT(31) -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_NSS GENMASK(2, 0) -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_MCS GENMASK(6, 3) -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_LDPC BIT(7) -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_DCM BIT(8) -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_START GENMASK(15, 9) -#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_SIZE GENMASK(18, 16) - /* HE Radiotap data1 Mask */ #define HE_SU_FORMAT_TYPE 0x0000 #define HE_EXT_SU_FORMAT_TYPE 0x0001 @@ -1044,128 +812,64 @@ enum hal_mon_reception_type { #define HAL_RU_PER80(ru_per80, num_80mhz, ru_idx_per80mhz) \ (HAL_RU(ru_per80, num_80mhz, ru_idx_per80mhz)) -#define RU_INVALID 0 -#define RU_26 1 -#define RU_52 2 -#define RU_106 4 -#define RU_242 9 -#define RU_484 18 -#define RU_996 37 -#define RU_2X996 74 -#define RU_3X996 111 -#define RU_4X996 148 -#define RU_52_26 (RU_52 + RU_26) -#define RU_106_26 (RU_106 + RU_26) -#define RU_484_242 (RU_484 + RU_242) -#define RU_996_484 (RU_996 + RU_484) -#define RU_996_484_242 (RU_996 + RU_484_242) -#define RU_2X996_484 (RU_2X996 + RU_484) -#define RU_3X996_484 (RU_3X996 + RU_484) - -enum ath12k_eht_ru_size { - ATH12K_EHT_RU_26, - ATH12K_EHT_RU_52, - ATH12K_EHT_RU_106, - ATH12K_EHT_RU_242, - ATH12K_EHT_RU_484, - ATH12K_EHT_RU_996, - ATH12K_EHT_RU_996x2, - ATH12K_EHT_RU_996x4, - ATH12K_EHT_RU_52_26, - ATH12K_EHT_RU_106_26, - ATH12K_EHT_RU_484_242, - ATH12K_EHT_RU_996_484, - ATH12K_EHT_RU_996_484_242, - ATH12K_EHT_RU_996x2_484, - ATH12K_EHT_RU_996x3, - ATH12K_EHT_RU_996x3_484, - - /* Keep last */ - ATH12K_EHT_RU_INVALID, -}; - -#define HAL_RX_RU_ALLOC_TYPE_MAX ATH12K_EHT_RU_INVALID - -static inline -enum nl80211_he_ru_alloc ath12k_he_ru_tones_to_nl80211_he_ru_alloc(u16 ru_tones) -{ - enum nl80211_he_ru_alloc ret; - - switch (ru_tones) { - case RU_52: - ret = NL80211_RATE_INFO_HE_RU_ALLOC_52; - break; - case RU_106: - ret = NL80211_RATE_INFO_HE_RU_ALLOC_106; - break; - case RU_242: - ret = NL80211_RATE_INFO_HE_RU_ALLOC_242; - break; - case RU_484: - ret = NL80211_RATE_INFO_HE_RU_ALLOC_484; - break; - case RU_996: - ret = NL80211_RATE_INFO_HE_RU_ALLOC_996; - break; - case RU_2X996: - ret = NL80211_RATE_INFO_HE_RU_ALLOC_2x996; - break; - case RU_26: - fallthrough; - default: - ret = NL80211_RATE_INFO_HE_RU_ALLOC_26; - break; - } - return ret; -} - -void ath12k_hal_reo_status_queue_stats(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status); -void ath12k_hal_reo_flush_queue_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status); -void ath12k_hal_reo_flush_cache_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status); -void ath12k_hal_reo_unblk_cache_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status); -void ath12k_hal_reo_flush_timeout_list_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status); -void ath12k_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, - struct hal_reo_status *status); -void ath12k_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab, - struct hal_tlv_64_hdr *tlv, +void ath12k_wifi7_hal_reo_status_queue_stats(struct ath12k_base *ab, + struct hal_reo_get_queue_stats_status *desc, + struct hal_reo_status *status); +void ath12k_wifi7_hal_reo_flush_queue_status(struct ath12k_base *ab, + struct hal_reo_flush_queue_status *desc, + struct hal_reo_status *status); +void ath12k_wifi7_hal_reo_flush_cache_status(struct ath12k_base *ab, + struct hal_reo_flush_cache_status *desc, + struct hal_reo_status *status); +void ath12k_wifi7_hal_reo_unblk_cache_status(struct ath12k_base *ab, + struct hal_reo_unblock_cache_status *desc, + struct hal_reo_status *status); +void +ath12k_wifi7_hal_reo_flush_timeout_list_status(struct ath12k_base *ab, + struct hal_reo_flush_timeout_list_status *desc, struct hal_reo_status *status); -void ath12k_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, u32 *num_msdus, - u32 *msdu_cookies, - enum hal_rx_buf_return_buf_manager *rbm); -void ath12k_hal_rx_msdu_link_desc_set(struct ath12k_base *ab, - struct hal_wbm_release_ring *desc, - struct ath12k_buffer_addr *buf_addr_info, - enum hal_wbm_rel_bm_act action); -void ath12k_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo, - dma_addr_t paddr, u32 cookie, u8 manager); -void ath12k_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo, - dma_addr_t *paddr, - u32 *cookie, u8 *rbm); -int ath12k_hal_desc_reo_parse_err(struct ath12k_base *ab, - struct hal_reo_dest_ring *desc, - dma_addr_t *paddr, u32 *desc_bank); -int ath12k_hal_wbm_desc_parse_err(struct ath12k_base *ab, void *desc, - struct hal_rx_wbm_rel_info *rel_info); -void ath12k_hal_rx_reo_ent_paddr_get(struct ath12k_base *ab, - struct ath12k_buffer_addr *buff_addr, - dma_addr_t *paddr, u32 *cookie); -void ath12k_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr, u32 *sw_cookie, - struct ath12k_buffer_addr **pp_buf_addr, - u8 *rbm, u32 *msdu_cnt); -void ath12k_hal_rx_msdu_list_get(struct ath12k *ar, - struct hal_rx_msdu_link *link_desc, - struct hal_rx_msdu_list *msdu_list, - u16 *num_msdus); +void +ath12k_wifi7_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab, + struct hal_reo_desc_thresh_reached_status *desc, + struct hal_reo_status *status); +void ath12k_wifi7_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab, + struct hal_reo_status_hdr *desc, + struct hal_reo_status *status); +void ath12k_wifi7_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, u32 *num_msdus, + u32 *msdu_cookies, + enum hal_rx_buf_return_buf_manager *rbm); +void ath12k_wifi7_hal_rx_msdu_link_desc_set(struct ath12k_base *ab, + struct hal_wbm_release_ring *desc, + struct ath12k_buffer_addr *buf_addr_info, + enum hal_wbm_rel_bm_act action); +void ath12k_wifi7_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo, + dma_addr_t paddr, u32 cookie, u8 manager); +void ath12k_wifi7_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo, + dma_addr_t *paddr, + u32 *cookie, u8 *rbm); +int ath12k_wifi7_hal_desc_reo_parse_err(struct ath12k_dp *dp, + struct hal_reo_dest_ring *desc, + dma_addr_t *paddr, u32 *desc_bank); +int ath12k_wifi7_hal_wbm_desc_parse_err(struct ath12k_dp *dp, void *desc, + struct hal_rx_wbm_rel_info *rel_info); +void ath12k_wifi7_hal_rx_reo_ent_paddr_get(struct ath12k_buffer_addr *buff_addr, + dma_addr_t *paddr, u32 *cookie); +void ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr, + u32 *sw_cookie, + struct ath12k_buffer_addr **pp_buf_addr, + u8 *rbm, u32 *msdu_cnt); +void ath12k_wifi7_hal_rx_msdu_list_get(struct ath12k *ar, + void *link_desc, + void *msdu_list_opaque, + u16 *num_msdus); +void ath12k_wifi7_hal_reo_init_cmd_ring_tlv64(struct ath12k_base *ab, + struct hal_srng *srng); +void ath12k_wifi7_hal_reo_init_cmd_ring_tlv32(struct ath12k_base *ab, + struct hal_srng *srng); +void ath12k_wifi7_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab); +void ath12k_wifi7_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map); +void ath12k_wifi7_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc, + int tid, u32 ba_window_size, + u32 start_seq, enum hal_pn_type type); #endif diff --git a/drivers/net/wireless/ath/ath12k/rx_desc.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h similarity index 97% rename from drivers/net/wireless/ath/ath12k/rx_desc.h rename to drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h index 6c600473b4021..0d19a9cbb68ce 100644 --- a/drivers/net/wireless/ath/ath12k/rx_desc.h +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h @@ -1,18 +1,11 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_RX_DESC_H #define ATH12K_RX_DESC_H -enum rx_desc_decap_type { - RX_DESC_DECAP_TYPE_RAW, - RX_DESC_DECAP_TYPE_NATIVE_WIFI, - RX_DESC_DECAP_TYPE_ETHERNET2_DIX, - RX_DESC_DECAP_TYPE_8023, -}; - enum rx_desc_decrypt_status_code { RX_DESC_DECRYPT_STATUS_CODE_OK, RX_DESC_DECRYPT_STATUS_CODE_UNPROTECTED_FRAME, @@ -631,40 +624,6 @@ struct rx_mpdu_start_qcn9274_compact { * */ -enum rx_msdu_start_pkt_type { - RX_MSDU_START_PKT_TYPE_11A, - RX_MSDU_START_PKT_TYPE_11B, - RX_MSDU_START_PKT_TYPE_11N, - RX_MSDU_START_PKT_TYPE_11AC, - RX_MSDU_START_PKT_TYPE_11AX, - RX_MSDU_START_PKT_TYPE_11BA, - RX_MSDU_START_PKT_TYPE_11BE, -}; - -enum rx_msdu_start_sgi { - RX_MSDU_START_SGI_0_8_US, - RX_MSDU_START_SGI_0_4_US, - RX_MSDU_START_SGI_1_6_US, - RX_MSDU_START_SGI_3_2_US, -}; - -enum rx_msdu_start_recv_bw { - RX_MSDU_START_RECV_BW_20MHZ, - RX_MSDU_START_RECV_BW_40MHZ, - RX_MSDU_START_RECV_BW_80MHZ, - RX_MSDU_START_RECV_BW_160MHZ, -}; - -enum rx_msdu_start_reception_type { - RX_MSDU_START_RECEPTION_TYPE_SU, - RX_MSDU_START_RECEPTION_TYPE_DL_MU_MIMO, - RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA, - RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA_MIMO, - RX_MSDU_START_RECEPTION_TYPE_UL_MU_MIMO, - RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA, - RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA_MIMO, -}; - #define RX_MSDU_END_64_TLV_SRC_LINK_ID GENMASK(24, 22) #define RX_MSDU_END_INFO0_RXPCU_MPDU_FITLER GENMASK(1, 0) @@ -1495,12 +1454,6 @@ struct rx_msdu_end_qcn9274_compact { * */ -struct hal_rx_desc_qcn9274 { - struct rx_msdu_end_qcn9274 msdu_end; - struct rx_mpdu_start_qcn9274 mpdu_start; - u8 msdu_payload[]; -} __packed; - struct hal_rx_desc_qcn9274_compact { struct rx_msdu_end_qcn9274_compact msdu_end; struct rx_mpdu_start_qcn9274_compact mpdu_start; @@ -1528,17 +1481,28 @@ struct hal_rx_desc_wcn7850 { u8 msdu_payload[]; }; +struct rx_pkt_hdr_tlv_qcc2072 { + __le32 tag; + __le64 phy_ppdu_id; + u8 rx_pkt_hdr[HAL_RX_BE_PKT_HDR_TLV_LEN]; +}; + +struct hal_rx_desc_qcc2072 { + __le32 msdu_end_tag; + struct rx_msdu_end_qcn9274 msdu_end; + u8 rx_padding0[RX_BE_PADDING0_BYTES]; + __le32 mpdu_start_tag; + struct rx_mpdu_start_qcn9274 mpdu_start; + struct rx_pkt_hdr_tlv_qcc2072 pkt_hdr_tlv; + u8 msdu_payload[]; +}; + struct hal_rx_desc { union { - struct hal_rx_desc_qcn9274 qcn9274; struct hal_rx_desc_qcn9274_compact qcn9274_compact; struct hal_rx_desc_wcn7850 wcn7850; + struct hal_rx_desc_qcc2072 qcc2072; } u; } __packed; -#define MAX_USER_POS 8 -#define MAX_MU_GROUP_ID 64 -#define MAX_MU_GROUP_SHOW 16 -#define MAX_MU_GROUP_LENGTH (6 * MAX_MU_GROUP_SHOW) - #endif /* ATH12K_RX_DESC_H */ diff --git a/drivers/net/wireless/ath/ath12k/hal_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c similarity index 87% rename from drivers/net/wireless/ath/ath12k/hal_tx.c rename to drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c index 869e07e406fef..02d3cadf03fea 100644 --- a/drivers/net/wireless/ath/ath12k/hal_tx.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c @@ -1,13 +1,13 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ -#include "hal_desc.h" -#include "hal.h" +#include "../hal.h" #include "hal_tx.h" -#include "hif.h" +#include "../hif.h" +#include "hal.h" #define DSCP_TID_MAP_TBL_ENTRY_SIZE 64 @@ -29,9 +29,9 @@ static inline u8 dscp2tid(u8 dscp) return dscp >> 3; } -void ath12k_hal_tx_cmd_desc_setup(struct ath12k_base *ab, - struct hal_tcl_data_cmd *tcl_cmd, - struct hal_tx_info *ti) +void ath12k_wifi7_hal_tx_cmd_desc_setup(struct ath12k_base *ab, + struct hal_tcl_data_cmd *tcl_cmd, + struct hal_tx_info *ti) { tcl_cmd->buf_addr_info.info0 = le32_encode_bits(ti->paddr, BUFFER_ADDR_INFO0_ADDR); @@ -66,7 +66,7 @@ void ath12k_hal_tx_cmd_desc_setup(struct ath12k_base *ab, tcl_cmd->info5 = 0; } -void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id) +void ath12k_wifi7_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id) { u32 ctrl_reg_val; u32 addr; @@ -136,10 +136,3 @@ void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id) HAL_TCL1_RING_CMN_CTRL_REG, ctrl_reg_val); } - -void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab, u32 bank_config, - u8 bank_id) -{ - ath12k_hif_write32(ab, HAL_TCL_SW_CONFIG_BANK_ADDR + 4 * bank_id, - bank_config); -} diff --git a/drivers/net/wireless/ath/ath12k/hal_tx.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.h similarity index 88% rename from drivers/net/wireless/ath/ath12k/hal_tx.h rename to drivers/net/wireless/ath/ath12k/wifi7/hal_tx.h index eb065a79f6c64..9d2b1552c2f51 100644 --- a/drivers/net/wireless/ath/ath12k/hal_tx.h +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.h @@ -1,21 +1,14 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. - * All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_HAL_TX_H #define ATH12K_HAL_TX_H +#include "../mac.h" #include "hal_desc.h" -#include "core.h" - -#define HAL_TX_ADDRX_EN 1 -#define HAL_TX_ADDRY_EN 2 - -#define HAL_TX_ADDR_SEARCH_DEFAULT 0 -#define HAL_TX_ADDR_SEARCH_INDEX 1 /* TODO: check all these data can be managed with struct ath12k_tx_desc_info for perf */ struct hal_tx_info { @@ -188,13 +181,14 @@ struct hal_tx_fes_status_end { /* STA mode will have MCAST_PKT_CTRL instead of DSCP_TID_MAP bitfield */ #define HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID GENMASK(22, 17) -void ath12k_hal_tx_cmd_desc_setup(struct ath12k_base *ab, - struct hal_tcl_data_cmd *tcl_cmd, - struct hal_tx_info *ti); -void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id); -int ath12k_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng, - enum hal_reo_cmd_type type, - struct ath12k_hal_reo_cmd *cmd); -void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab, u32 bank_config, - u8 bank_id); +void ath12k_wifi7_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id); +void ath12k_wifi7_hal_tx_cmd_desc_setup(struct ath12k_base *ab, + struct hal_tcl_data_cmd *tcl_cmd, + struct hal_tx_info *ti); +int ath12k_wifi7_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng, + enum hal_reo_cmd_type type, + struct ath12k_hal_reo_cmd *cmd); +void ath12k_wifi7_hal_tx_configure_bank_register(struct ath12k_base *ab, + u32 bank_config, + u8 bank_id); #endif diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c new file mode 100644 index 0000000000000..8819860755489 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c @@ -0,0 +1,820 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "hal_desc.h" +#include "hal_wcn7850.h" +#include "hw.h" +#include "hal.h" +#include "hal_tx.h" + +static const struct hal_srng_config hw_srng_config_template[] = { + /* TODO: max_rings can populated by querying HW capabilities */ + [HAL_REO_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_REO2SW1, + .max_rings = 8, + .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_EXCEPTION] = { + /* Designating REO2SW0 ring as exception ring. + * Any of theREO2SW rings can be used as exception ring. + */ + .start_ring_id = HAL_SRNG_RING_ID_REO2SW0, + .max_rings = 1, + .entry_size = sizeof(struct hal_reo_dest_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_REINJECT] = { + .start_ring_id = HAL_SRNG_RING_ID_SW2REO, + .max_rings = 4, + .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_CMD] = { + .start_ring_id = HAL_SRNG_RING_ID_REO_CMD, + .max_rings = 1, + .entry_size = (sizeof(struct hal_tlv_64_hdr) + + sizeof(struct hal_reo_get_queue_stats)) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE, + }, + [HAL_REO_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_REO_STATUS, + .max_rings = 1, + .entry_size = (sizeof(struct hal_tlv_64_hdr) + + sizeof(struct hal_reo_get_queue_stats_status)) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE, + }, + [HAL_TCL_DATA] = { + .start_ring_id = HAL_SRNG_RING_ID_SW2TCL1, + .max_rings = 6, + .entry_size = sizeof(struct hal_tcl_data_cmd) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE, + }, + [HAL_TCL_CMD] = { + .start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD, + .max_rings = 1, + .entry_size = sizeof(struct hal_tcl_gse_cmd) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE, + }, + [HAL_TCL_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS, + .max_rings = 1, + .entry_size = (sizeof(struct hal_tlv_hdr) + + sizeof(struct hal_tcl_status_ring)) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE, + }, + [HAL_CE_SRC] = { + .start_ring_id = HAL_SRNG_RING_ID_CE0_SRC, + .max_rings = 16, + .entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE, + }, + [HAL_CE_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_CE0_DST, + .max_rings = 16, + .entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE, + }, + [HAL_CE_DST_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS, + .max_rings = 16, + .entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE, + }, + [HAL_WBM_IDLE_LINK] = { + .start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK, + .max_rings = 1, + .entry_size = sizeof(struct hal_wbm_link_desc) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE, + }, + [HAL_SW2WBM_RELEASE] = { + .start_ring_id = HAL_SRNG_RING_ID_WBM_SW0_RELEASE, + .max_rings = 2, + .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE, + }, + [HAL_WBM2SW_RELEASE] = { + .start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE, + .max_rings = 8, + .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_UMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE, + }, + [HAL_RXDMA_BUF] = { + .start_ring_id = HAL_SRNG_SW2RXDMA_BUF0, + .max_rings = 1, + .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_DMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_DST] = { + .start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0, + .max_rings = 0, + .entry_size = 0, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_DST, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_MONITOR_BUF] = {}, + [HAL_RXDMA_MONITOR_STATUS] = { + .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF, + .max_rings = 1, + .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2, + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_RXDMA_MONITOR_DESC] = { 0, }, + [HAL_RXDMA_DIR_BUF] = { + .start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF, + .max_rings = 2, + .entry_size = 8 >> 2, /* TODO: Define the struct */ + .mac_type = ATH12K_HAL_SRNG_PMAC, + .ring_dir = HAL_SRNG_DIR_SRC, + .max_size = HAL_RXDMA_RING_MAX_SIZE_BE, + }, + [HAL_PPE2TCL] = {}, + [HAL_PPE_RELEASE] = {}, + [HAL_TX_MONITOR_BUF] = {}, + [HAL_RXDMA_MONITOR_DST] = {}, + [HAL_TX_MONITOR_DST] = {} +}; + +const struct ath12k_hw_regs wcn7850_regs = { + /* SW2TCL(x) R0 ring configuration address */ + .tcl1_ring_id = 0x00000908, + .tcl1_ring_misc = 0x00000910, + .tcl1_ring_tp_addr_lsb = 0x0000091c, + .tcl1_ring_tp_addr_msb = 0x00000920, + .tcl1_ring_consumer_int_setup_ix0 = 0x00000930, + .tcl1_ring_consumer_int_setup_ix1 = 0x00000934, + .tcl1_ring_msi1_base_lsb = 0x00000948, + .tcl1_ring_msi1_base_msb = 0x0000094c, + .tcl1_ring_msi1_data = 0x00000950, + .tcl_ring_base_lsb = 0x00000b58, + .tcl1_ring_base_lsb = 0x00000900, + .tcl1_ring_base_msb = 0x00000904, + .tcl2_ring_base_lsb = 0x00000978, + + /* TCL STATUS ring address */ + .tcl_status_ring_base_lsb = 0x00000d38, + + .wbm_idle_ring_base_lsb = 0x00000d3c, + .wbm_idle_ring_misc_addr = 0x00000d4c, + .wbm_r0_idle_list_cntl_addr = 0x00000240, + .wbm_r0_idle_list_size_addr = 0x00000244, + .wbm_scattered_ring_base_lsb = 0x00000250, + .wbm_scattered_ring_base_msb = 0x00000254, + .wbm_scattered_desc_head_info_ix0 = 0x00000260, + .wbm_scattered_desc_head_info_ix1 = 0x00000264, + .wbm_scattered_desc_tail_info_ix0 = 0x00000270, + .wbm_scattered_desc_tail_info_ix1 = 0x00000274, + .wbm_scattered_desc_ptr_hp_addr = 0x00000027c, + + .wbm_sw_release_ring_base_lsb = 0x0000037c, + .wbm_sw1_release_ring_base_lsb = 0x00000284, + .wbm0_release_ring_base_lsb = 0x00000e08, + .wbm1_release_ring_base_lsb = 0x00000e80, + + /* PCIe base address */ + .pcie_qserdes_sysclk_en_sel = 0x01e0e0a8, + .pcie_pcs_osc_dtct_config_base = 0x01e0f45c, + + /* PPE release ring address */ + .ppe_rel_ring_base = 0x0000043c, + + /* REO DEST ring address */ + .reo2_ring_base = 0x0000055c, + .reo1_misc_ctrl_addr = 0x00000b7c, + .reo1_sw_cookie_cfg0 = 0x00000050, + .reo1_sw_cookie_cfg1 = 0x00000054, + .reo1_qdesc_lut_base0 = 0x00000058, + .reo1_qdesc_lut_base1 = 0x0000005c, + .reo1_ring_base_lsb = 0x000004e4, + .reo1_ring_base_msb = 0x000004e8, + .reo1_ring_id = 0x000004ec, + .reo1_ring_misc = 0x000004f4, + .reo1_ring_hp_addr_lsb = 0x000004f8, + .reo1_ring_hp_addr_msb = 0x000004fc, + .reo1_ring_producer_int_setup = 0x00000508, + .reo1_ring_msi1_base_lsb = 0x0000052C, + .reo1_ring_msi1_base_msb = 0x00000530, + .reo1_ring_msi1_data = 0x00000534, + .reo1_aging_thres_ix0 = 0x00000b08, + .reo1_aging_thres_ix1 = 0x00000b0c, + .reo1_aging_thres_ix2 = 0x00000b10, + .reo1_aging_thres_ix3 = 0x00000b14, + + /* REO Exception ring address */ + .reo2_sw0_ring_base = 0x000008a4, + + /* REO Reinject ring address */ + .sw2reo_ring_base = 0x00000304, + .sw2reo1_ring_base = 0x0000037c, + + /* REO cmd ring address */ + .reo_cmd_ring_base = 0x0000028c, + + /* REO status ring address */ + .reo_status_ring_base = 0x00000a84, + + /* CE base address */ + .umac_ce0_src_reg_base = 0x01b80000, + .umac_ce0_dest_reg_base = 0x01b81000, + .umac_ce1_src_reg_base = 0x01b82000, + .umac_ce1_dest_reg_base = 0x01b83000, + + .gcc_gcc_pcie_hot_rst = 0x1e40304, + + .qrtr_node_id = 0x1e03164, +}; + +static inline +bool ath12k_hal_rx_desc_get_first_msdu_wcn7850(struct hal_rx_desc *desc) +{ + return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5, + RX_MSDU_END_INFO5_FIRST_MSDU); +} + +static inline +bool ath12k_hal_rx_desc_get_last_msdu_wcn7850(struct hal_rx_desc *desc) +{ + return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5, + RX_MSDU_END_INFO5_LAST_MSDU); +} + +u8 ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850(struct hal_rx_desc *desc) +{ + return le16_get_bits(desc->u.wcn7850.msdu_end.info5, + RX_MSDU_END_INFO5_L3_HDR_PADDING); +} + +static inline +bool ath12k_hal_rx_desc_encrypt_valid_wcn7850(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4, + RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID); +} + +static inline +u32 ath12k_hal_rx_desc_get_encrypt_type_wcn7850(struct hal_rx_desc *desc) +{ + if (!ath12k_hal_rx_desc_encrypt_valid_wcn7850(desc)) + return HAL_ENCRYPT_TYPE_OPEN; + + return le32_get_bits(desc->u.wcn7850.mpdu_start.info2, + RX_MPDU_START_INFO2_ENC_TYPE); +} + +static inline +u8 ath12k_hal_rx_desc_get_decap_type_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info11, + RX_MSDU_END_INFO11_DECAP_FORMAT); +} + +static inline +u8 ath12k_hal_rx_desc_get_mesh_ctl_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info11, + RX_MSDU_END_INFO11_MESH_CTRL_PRESENT); +} + +static inline +bool ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_wcn7850(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID); +} + +static inline +bool ath12k_hal_rx_desc_get_mpdu_fc_valid_wcn7850(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_FCTRL_VALID); +} + +static inline +u16 ath12k_hal_rx_desc_get_mpdu_start_seq_no_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.mpdu_start.info4, + RX_MPDU_START_INFO4_MPDU_SEQ_NUM); +} + +static inline +u16 ath12k_hal_rx_desc_get_msdu_len_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info10, + RX_MSDU_END_INFO10_MSDU_LENGTH); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_sgi_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info12, + RX_MSDU_END_INFO12_SGI); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_rate_mcs_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info12, + RX_MSDU_END_INFO12_RATE_MCS); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_rx_bw_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info12, + RX_MSDU_END_INFO12_RECV_BW); +} + +static inline +u32 ath12k_hal_rx_desc_get_msdu_freq_wcn7850(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.wcn7850.msdu_end.phy_meta_data); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_pkt_type_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info12, + RX_MSDU_END_INFO12_PKT_TYPE); +} + +static inline +u8 ath12k_hal_rx_desc_get_msdu_nss_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.msdu_end.info12, + RX_MSDU_END_INFO12_MIMO_SS_BITMAP); +} + +static inline +u8 ath12k_hal_rx_desc_get_mpdu_tid_wcn7850(struct hal_rx_desc *desc) +{ + return le32_get_bits(desc->u.wcn7850.mpdu_start.info2, + RX_MPDU_START_INFO2_TID); +} + +static inline +u16 ath12k_hal_rx_desc_get_mpdu_peer_id_wcn7850(struct hal_rx_desc *desc) +{ + return __le16_to_cpu(desc->u.wcn7850.mpdu_start.sw_peer_id); +} + +void ath12k_hal_rx_desc_copy_end_tlv_wcn7850(struct hal_rx_desc *fdesc, + struct hal_rx_desc *ldesc) +{ + memcpy(&fdesc->u.wcn7850.msdu_end, &ldesc->u.wcn7850.msdu_end, + sizeof(struct rx_msdu_end_qcn9274)); +} + +u32 ath12k_hal_rx_desc_get_mpdu_start_tag_wcn7850(struct hal_rx_desc *desc) +{ + return le64_get_bits(desc->u.wcn7850.mpdu_start_tag, + HAL_TLV_HDR_TAG); +} + +u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_wcn7850(struct hal_rx_desc *desc) +{ + return __le16_to_cpu(desc->u.wcn7850.mpdu_start.phy_ppdu_id); +} + +void ath12k_hal_rx_desc_set_msdu_len_wcn7850(struct hal_rx_desc *desc, u16 len) +{ + u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info10); + + info &= ~RX_MSDU_END_INFO10_MSDU_LENGTH; + info |= u32_encode_bits(len, RX_MSDU_END_INFO10_MSDU_LENGTH); + + desc->u.wcn7850.msdu_end.info10 = __cpu_to_le32(info); +} + +u8 *ath12k_hal_rx_desc_get_msdu_payload_wcn7850(struct hal_rx_desc *desc) +{ + return &desc->u.wcn7850.msdu_payload[0]; +} + +u32 ath12k_hal_rx_desc_get_mpdu_start_offset_wcn7850(void) +{ + return offsetof(struct hal_rx_desc_wcn7850, mpdu_start_tag); +} + +u32 ath12k_hal_rx_desc_get_msdu_end_offset_wcn7850(void) +{ + return offsetof(struct hal_rx_desc_wcn7850, msdu_end_tag); +} + +static inline +bool ath12k_hal_rx_desc_mac_addr2_valid_wcn7850(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) & + RX_MPDU_START_INFO4_MAC_ADDR2_VALID; +} + +static inline +u8 *ath12k_hal_rx_desc_mpdu_start_addr2_wcn7850(struct hal_rx_desc *desc) +{ + return desc->u.wcn7850.mpdu_start.addr2; +} + +static inline +bool ath12k_hal_rx_desc_is_da_mcbc_wcn7850(struct hal_rx_desc *desc) +{ + return __le32_to_cpu(desc->u.wcn7850.msdu_end.info13) & + RX_MSDU_END_INFO13_MCAST_BCAST; +} + +static inline +bool ath12k_hal_rx_h_msdu_done_wcn7850(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.wcn7850.msdu_end.info14, + RX_MSDU_END_INFO14_MSDU_DONE); +} + +static inline +bool ath12k_hal_rx_h_l4_cksum_fail_wcn7850(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13, + RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL); +} + +static inline +bool ath12k_hal_rx_h_ip_cksum_fail_wcn7850(struct hal_rx_desc *desc) +{ + return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13, + RX_MSDU_END_INFO13_IP_CKSUM_FAIL); +} + +static inline +bool ath12k_hal_rx_h_is_decrypted_wcn7850(struct hal_rx_desc *desc) +{ + return (le32_get_bits(desc->u.wcn7850.msdu_end.info14, + RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) == + RX_DESC_DECRYPT_STATUS_CODE_OK); +} + +u32 ath12k_hal_get_rx_desc_size_wcn7850(void) +{ + return sizeof(struct hal_rx_desc_wcn7850); +} + +u8 ath12k_hal_rx_desc_get_msdu_src_link_wcn7850(struct hal_rx_desc *desc) +{ + return 0; +} + +static u32 ath12k_hal_rx_h_mpdu_err_wcn7850(struct hal_rx_desc *desc) +{ + u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info13); + u32 errmap = 0; + + if (info & RX_MSDU_END_INFO13_FCS_ERR) + errmap |= HAL_RX_MPDU_ERR_FCS; + + if (info & RX_MSDU_END_INFO13_DECRYPT_ERR) + errmap |= HAL_RX_MPDU_ERR_DECRYPT; + + if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR) + errmap |= HAL_RX_MPDU_ERR_TKIP_MIC; + + if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR) + errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR; + + if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR) + errmap |= HAL_RX_MPDU_ERR_OVERFLOW; + + if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR) + errmap |= HAL_RX_MPDU_ERR_MSDU_LEN; + + if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR) + errmap |= HAL_RX_MPDU_ERR_MPDU_LEN; + + return errmap; +} + +void ath12k_hal_rx_desc_get_crypto_hdr_wcn7850(struct hal_rx_desc *desc, + u8 *crypto_hdr, + enum hal_encrypt_type enctype) +{ + unsigned int key_id; + + switch (enctype) { + case HAL_ENCRYPT_TYPE_OPEN: + return; + case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: + case HAL_ENCRYPT_TYPE_TKIP_MIC: + crypto_hdr[0] = + HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]); + crypto_hdr[1] = 0; + crypto_hdr[2] = + HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]); + break; + case HAL_ENCRYPT_TYPE_CCMP_128: + case HAL_ENCRYPT_TYPE_CCMP_256: + case HAL_ENCRYPT_TYPE_GCMP_128: + case HAL_ENCRYPT_TYPE_AES_GCMP_256: + crypto_hdr[0] = + HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]); + crypto_hdr[1] = + HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]); + crypto_hdr[2] = 0; + break; + case HAL_ENCRYPT_TYPE_WEP_40: + case HAL_ENCRYPT_TYPE_WEP_104: + case HAL_ENCRYPT_TYPE_WEP_128: + case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4: + case HAL_ENCRYPT_TYPE_WAPI: + return; + } + key_id = u32_get_bits(__le32_to_cpu(desc->u.wcn7850.mpdu_start.info5), + RX_MPDU_START_INFO5_KEY_ID); + crypto_hdr[3] = 0x20 | (key_id << 6); + crypto_hdr[4] = HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.wcn7850.mpdu_start.pn[0]); + crypto_hdr[5] = HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.wcn7850.mpdu_start.pn[0]); + crypto_hdr[6] = HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[1]); + crypto_hdr[7] = HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[1]); +} + +void ath12k_hal_rx_desc_get_dot11_hdr_wcn7850(struct hal_rx_desc *desc, + struct ieee80211_hdr *hdr) +{ + hdr->frame_control = desc->u.wcn7850.mpdu_start.frame_ctrl; + hdr->duration_id = desc->u.wcn7850.mpdu_start.duration; + ether_addr_copy(hdr->addr1, desc->u.wcn7850.mpdu_start.addr1); + ether_addr_copy(hdr->addr2, desc->u.wcn7850.mpdu_start.addr2); + ether_addr_copy(hdr->addr3, desc->u.wcn7850.mpdu_start.addr3); + if (__le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) & + RX_MPDU_START_INFO4_MAC_ADDR4_VALID) { + ether_addr_copy(hdr->addr4, desc->u.wcn7850.mpdu_start.addr4); + } + hdr->seq_ctrl = desc->u.wcn7850.mpdu_start.seq_ctrl; +} + +void ath12k_hal_extract_rx_desc_data_wcn7850(struct hal_rx_desc_data *rx_desc_data, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc *ldesc) +{ + rx_desc_data->is_first_msdu = ath12k_hal_rx_desc_get_first_msdu_wcn7850(ldesc); + rx_desc_data->is_last_msdu = ath12k_hal_rx_desc_get_last_msdu_wcn7850(ldesc); + rx_desc_data->l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850(ldesc); + rx_desc_data->enctype = ath12k_hal_rx_desc_get_encrypt_type_wcn7850(rx_desc); + rx_desc_data->decap_type = ath12k_hal_rx_desc_get_decap_type_wcn7850(rx_desc); + rx_desc_data->mesh_ctrl_present = + ath12k_hal_rx_desc_get_mesh_ctl_wcn7850(rx_desc); + rx_desc_data->seq_ctl_valid = + ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_wcn7850(rx_desc); + rx_desc_data->fc_valid = ath12k_hal_rx_desc_get_mpdu_fc_valid_wcn7850(rx_desc); + rx_desc_data->seq_no = ath12k_hal_rx_desc_get_mpdu_start_seq_no_wcn7850(rx_desc); + rx_desc_data->msdu_len = ath12k_hal_rx_desc_get_msdu_len_wcn7850(ldesc); + rx_desc_data->sgi = ath12k_hal_rx_desc_get_msdu_sgi_wcn7850(rx_desc); + rx_desc_data->rate_mcs = ath12k_hal_rx_desc_get_msdu_rate_mcs_wcn7850(rx_desc); + rx_desc_data->bw = ath12k_hal_rx_desc_get_msdu_rx_bw_wcn7850(rx_desc); + rx_desc_data->phy_meta_data = ath12k_hal_rx_desc_get_msdu_freq_wcn7850(rx_desc); + rx_desc_data->pkt_type = ath12k_hal_rx_desc_get_msdu_pkt_type_wcn7850(rx_desc); + rx_desc_data->nss = hweight8(ath12k_hal_rx_desc_get_msdu_nss_wcn7850(rx_desc)); + rx_desc_data->tid = ath12k_hal_rx_desc_get_mpdu_tid_wcn7850(rx_desc); + rx_desc_data->peer_id = ath12k_hal_rx_desc_get_mpdu_peer_id_wcn7850(rx_desc); + rx_desc_data->addr2_present = ath12k_hal_rx_desc_mac_addr2_valid_wcn7850(rx_desc); + rx_desc_data->addr2 = ath12k_hal_rx_desc_mpdu_start_addr2_wcn7850(rx_desc); + rx_desc_data->is_mcbc = ath12k_hal_rx_desc_is_da_mcbc_wcn7850(rx_desc); + rx_desc_data->msdu_done = ath12k_hal_rx_h_msdu_done_wcn7850(ldesc); + rx_desc_data->l4_csum_fail = ath12k_hal_rx_h_l4_cksum_fail_wcn7850(rx_desc); + rx_desc_data->ip_csum_fail = ath12k_hal_rx_h_ip_cksum_fail_wcn7850(rx_desc); + rx_desc_data->is_decrypted = ath12k_hal_rx_h_is_decrypted_wcn7850(rx_desc); + rx_desc_data->err_bitmap = ath12k_hal_rx_h_mpdu_err_wcn7850(rx_desc); +} + +int ath12k_hal_srng_create_config_wcn7850(struct ath12k_hal *hal) +{ + struct hal_srng_config *s; + + hal->srng_config = kmemdup(hw_srng_config_template, + sizeof(hw_srng_config_template), + GFP_KERNEL); + if (!hal->srng_config) + return -ENOMEM; + + s = &hal->srng_config[HAL_REO_DST]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP; + s->reg_size[0] = HAL_REO2_RING_BASE_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP; + + s = &hal->srng_config[HAL_REO_EXCEPTION]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP; + + s = &hal->srng_config[HAL_REO_REINJECT]; + s->max_rings = 1; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP; + + s = &hal->srng_config[HAL_REO_CMD]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP; + + s = &hal->srng_config[HAL_REO_STATUS]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP; + + s = &hal->srng_config[HAL_TCL_DATA]; + s->max_rings = 5; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP; + s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(hal) - HAL_TCL1_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP; + + s = &hal->srng_config[HAL_TCL_CMD]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP; + + s = &hal->srng_config[HAL_TCL_STATUS]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP; + + s = &hal->srng_config[HAL_CE_SRC]; + s->max_rings = 12; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_BASE_LSB; + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_HP; + s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal); + s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal); + + s = &hal->srng_config[HAL_CE_DST]; + s->max_rings = 12; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_BASE_LSB; + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_HP; + s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + + s = &hal->srng_config[HAL_CE_DST_STATUS]; + s->max_rings = 12; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + + HAL_CE_DST_STATUS_RING_BASE_LSB; + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_STATUS_RING_HP; + s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) - + HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal); + + s = &hal->srng_config[HAL_WBM_IDLE_LINK]; + s->reg_start[0] = + HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP; + + s = &hal->srng_config[HAL_SW2WBM_RELEASE]; + s->max_rings = 1; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + + HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP; + + s = &hal->srng_config[HAL_WBM2SW_RELEASE]; + s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(hal); + s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP; + s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(hal) - + HAL_WBM0_RELEASE_RING_BASE_LSB(hal); + s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP; + + s = &hal->srng_config[HAL_RXDMA_BUF]; + s->max_rings = 2; + s->mac_type = ATH12K_HAL_SRNG_PMAC; + + s = &hal->srng_config[HAL_RXDMA_DST]; + s->max_rings = 1; + s->entry_size = sizeof(struct hal_reo_entrance_ring) >> 2; + + /* below rings are not used */ + s = &hal->srng_config[HAL_RXDMA_DIR_BUF]; + s->max_rings = 0; + + s = &hal->srng_config[HAL_PPE2TCL]; + s->max_rings = 0; + + s = &hal->srng_config[HAL_PPE_RELEASE]; + s->max_rings = 0; + + s = &hal->srng_config[HAL_TX_MONITOR_BUF]; + s->max_rings = 0; + + s = &hal->srng_config[HAL_TX_MONITOR_DST]; + s->max_rings = 0; + + s = &hal->srng_config[HAL_PPE2TCL]; + s->max_rings = 0; + + return 0; +} + +static u16 ath12k_hal_reo_status_dec_tlv_hdr_wcn7850(void *tlv, void **desc) +{ + u16 tag; + + *desc = ath12k_hal_decode_tlv64_hdr(tlv, &tag, NULL, NULL); + + return tag; +} + +const struct ath12k_hal_tcl_to_wbm_rbm_map +ath12k_hal_tcl_to_wbm_rbm_map_wcn7850[DP_TCL_NUM_RING_MAX] = { + { + .wbm_ring_num = 0, + .rbm_id = HAL_RX_BUF_RBM_SW0_BM, + }, + { + .wbm_ring_num = 2, + .rbm_id = HAL_RX_BUF_RBM_SW2_BM, + }, + { + .wbm_ring_num = 4, + .rbm_id = HAL_RX_BUF_RBM_SW4_BM, + }, +}; + +const struct ath12k_hw_hal_params ath12k_hw_hal_params_wcn7850 = { + .rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM, + .wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN | + HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN, +}; + +const struct hal_ops hal_wcn7850_ops = { + .create_srng_config = ath12k_hal_srng_create_config_wcn7850, + .rx_desc_set_msdu_len = ath12k_hal_rx_desc_set_msdu_len_wcn7850, + .rx_desc_get_dot11_hdr = ath12k_hal_rx_desc_get_dot11_hdr_wcn7850, + .rx_desc_get_crypto_header = ath12k_hal_rx_desc_get_crypto_hdr_wcn7850, + .rx_desc_copy_end_tlv = ath12k_hal_rx_desc_copy_end_tlv_wcn7850, + .rx_desc_get_msdu_src_link_id = ath12k_hal_rx_desc_get_msdu_src_link_wcn7850, + .extract_rx_desc_data = ath12k_hal_extract_rx_desc_data_wcn7850, + .rx_desc_get_l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850, + .rx_desc_get_mpdu_start_tag = ath12k_hal_rx_desc_get_mpdu_start_tag_wcn7850, + .rx_desc_get_mpdu_ppdu_id = ath12k_hal_rx_desc_get_mpdu_ppdu_id_wcn7850, + .rx_desc_get_msdu_payload = ath12k_hal_rx_desc_get_msdu_payload_wcn7850, + .ce_dst_setup = ath12k_wifi7_hal_ce_dst_setup, + .srng_src_hw_init = ath12k_wifi7_hal_srng_src_hw_init, + .srng_dst_hw_init = ath12k_wifi7_hal_srng_dst_hw_init, + .set_umac_srng_ptr_addr = ath12k_wifi7_hal_set_umac_srng_ptr_addr, + .srng_update_shadow_config = ath12k_wifi7_hal_srng_update_shadow_config, + .srng_get_ring_id = ath12k_wifi7_hal_srng_get_ring_id, + .ce_get_desc_size = ath12k_wifi7_hal_ce_get_desc_size, + .ce_src_set_desc = ath12k_wifi7_hal_ce_src_set_desc, + .ce_dst_set_desc = ath12k_wifi7_hal_ce_dst_set_desc, + .ce_dst_status_get_length = ath12k_wifi7_hal_ce_dst_status_get_length, + .set_link_desc_addr = ath12k_wifi7_hal_set_link_desc_addr, + .tx_set_dscp_tid_map = ath12k_wifi7_hal_tx_set_dscp_tid_map, + .tx_configure_bank_register = + ath12k_wifi7_hal_tx_configure_bank_register, + .reoq_lut_addr_read_enable = ath12k_wifi7_hal_reoq_lut_addr_read_enable, + .reoq_lut_set_max_peerid = ath12k_wifi7_hal_reoq_lut_set_max_peerid, + .write_reoq_lut_addr = ath12k_wifi7_hal_write_reoq_lut_addr, + .write_ml_reoq_lut_addr = ath12k_wifi7_hal_write_ml_reoq_lut_addr, + .setup_link_idle_list = ath12k_wifi7_hal_setup_link_idle_list, + .reo_init_cmd_ring = ath12k_wifi7_hal_reo_init_cmd_ring_tlv64, + .reo_shared_qaddr_cache_clear = ath12k_wifi7_hal_reo_shared_qaddr_cache_clear, + .reo_hw_setup = ath12k_wifi7_hal_reo_hw_setup, + .rx_buf_addr_info_set = ath12k_wifi7_hal_rx_buf_addr_info_set, + .rx_buf_addr_info_get = ath12k_wifi7_hal_rx_buf_addr_info_get, + .cc_config = ath12k_wifi7_hal_cc_config, + .get_idle_link_rbm = ath12k_wifi7_hal_get_idle_link_rbm, + .rx_msdu_list_get = ath12k_wifi7_hal_rx_msdu_list_get, + .rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get, + .reo_cmd_enc_tlv_hdr = ath12k_hal_encode_tlv64_hdr, + .reo_status_dec_tlv_hdr = ath12k_hal_reo_status_dec_tlv_hdr_wcn7850, + .mon_rx_status_dec_tlv_hdr = ath12k_hal_decode_tlv64_hdr, + .get_tlv_hdr_align = ath12k_hal_get_tlv64_hdr_align, +}; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.h new file mode 100644 index 0000000000000..a56ca9fd3de4e --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_HAL_WCN7850_H +#define ATH12K_HAL_WCN7850_H + +#include "../hal.h" +#include "hal_rx.h" +#include "hal.h" + +extern const struct hal_ops hal_wcn7850_ops; +extern const struct ath12k_hw_regs wcn7850_regs; +extern const struct ath12k_hal_tcl_to_wbm_rbm_map +ath12k_hal_tcl_to_wbm_rbm_map_wcn7850[DP_TCL_NUM_RING_MAX]; +extern const struct ath12k_hw_hal_params ath12k_hw_hal_params_wcn7850; + +u8 ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850(struct hal_rx_desc *desc); +void ath12k_hal_rx_desc_copy_end_tlv_wcn7850(struct hal_rx_desc *fdesc, + struct hal_rx_desc *ldesc); +u32 ath12k_hal_rx_desc_get_mpdu_start_tag_wcn7850(struct hal_rx_desc *desc); +u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_wcn7850(struct hal_rx_desc *desc); +void ath12k_hal_rx_desc_set_msdu_len_wcn7850(struct hal_rx_desc *desc, u16 len); +u8 *ath12k_hal_rx_desc_get_msdu_payload_wcn7850(struct hal_rx_desc *desc); +u32 ath12k_hal_rx_desc_get_mpdu_start_offset_wcn7850(void); +u32 ath12k_hal_rx_desc_get_msdu_end_offset_wcn7850(void); +u32 ath12k_hal_get_rx_desc_size_wcn7850(void); +u8 ath12k_hal_rx_desc_get_msdu_src_link_wcn7850(struct hal_rx_desc *desc); +void ath12k_hal_rx_desc_get_crypto_hdr_wcn7850(struct hal_rx_desc *desc, + u8 *crypto_hdr, + enum hal_encrypt_type enctype); +void ath12k_hal_rx_desc_get_dot11_hdr_wcn7850(struct hal_rx_desc *desc, + struct ieee80211_hdr *hdr); +void ath12k_hal_extract_rx_desc_data_wcn7850(struct hal_rx_desc_data *rx_desc_data, + struct hal_rx_desc *rx_desc, + struct hal_rx_desc *ldesc); +int ath12k_hal_srng_create_config_wcn7850(struct ath12k_hal *hal); +#endif diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.c b/drivers/net/wireless/ath/ath12k/wifi7/hw.c new file mode 100644 index 0000000000000..df045ddf42da9 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.c @@ -0,0 +1,1049 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include + +#include "../debug.h" +#include "../core.h" +#include "../ce.h" +#include "ce.h" +#include "../hw.h" +#include "hw.h" +#include "../mhi.h" +#include "mhi.h" +#include "dp_rx.h" +#include "../peer.h" +#include "wmi.h" +#include "../wow.h" +#include "../debugfs.h" +#include "../debugfs_sta.h" +#include "../testmode.h" +#include "hal.h" +#include "dp_tx.h" + +static const guid_t wcn7850_uuid = GUID_INIT(0xf634f534, 0x6147, 0x11ec, + 0x90, 0xd6, 0x02, 0x42, + 0xac, 0x12, 0x00, 0x03); + +static u8 ath12k_wifi7_hw_qcn9274_mac_from_pdev_id(int pdev_idx) +{ + return pdev_idx; +} + +static int +ath12k_wifi7_hw_mac_id_to_pdev_id_qcn9274(const struct ath12k_hw_params *hw, + int mac_id) +{ + return mac_id; +} + +static int +ath12k_wifi7_hw_mac_id_to_srng_id_qcn9274(const struct ath12k_hw_params *hw, + int mac_id) +{ + return 0; +} + +static u8 ath12k_wifi7_hw_get_ring_selector_qcn9274(struct sk_buff *skb) +{ + return smp_processor_id(); +} + +static bool ath12k_wifi7_dp_srng_is_comp_ring_qcn9274(int ring_num) +{ + if (ring_num < 3 || ring_num == 4) + return true; + + return false; +} + +static bool +ath12k_wifi7_is_frame_link_agnostic_qcn9274(struct ath12k_link_vif *arvif, + struct ieee80211_mgmt *mgmt) +{ + return ieee80211_is_action(mgmt->frame_control); +} + +static int +ath12k_wifi7_hw_mac_id_to_pdev_id_wcn7850(const struct ath12k_hw_params *hw, + int mac_id) +{ + return 0; +} + +static int +ath12k_wifi7_hw_mac_id_to_srng_id_wcn7850(const struct ath12k_hw_params *hw, + int mac_id) +{ + return mac_id; +} + +static u8 ath12k_wifi7_hw_get_ring_selector_wcn7850(struct sk_buff *skb) +{ + return skb_get_queue_mapping(skb); +} + +static bool ath12k_wifi7_dp_srng_is_comp_ring_wcn7850(int ring_num) +{ + if (ring_num == 0 || ring_num == 2 || ring_num == 4) + return true; + + return false; +} + +static bool ath12k_is_addba_resp_action_code(struct ieee80211_mgmt *mgmt) +{ + if (!ieee80211_is_action(mgmt->frame_control)) + return false; + + if (mgmt->u.action.category != WLAN_CATEGORY_BACK) + return false; + + if (mgmt->u.action.u.addba_resp.action_code != WLAN_ACTION_ADDBA_RESP) + return false; + + return true; +} + +static bool +ath12k_wifi7_is_frame_link_agnostic_wcn7850(struct ath12k_link_vif *arvif, + struct ieee80211_mgmt *mgmt) +{ + struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif); + struct ath12k_hw *ah = ath12k_ar_to_ah(arvif->ar); + struct ath12k_base *ab = arvif->ar->ab; + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); + struct ath12k_dp_peer *peer; + __le16 fc = mgmt->frame_control; + + spin_lock_bh(&dp->dp_lock); + if (!ath12k_dp_link_peer_find_by_addr(dp, mgmt->da)) { + spin_lock_bh(&ah->dp_hw.peer_lock); + peer = ath12k_dp_peer_find_by_addr(&ah->dp_hw, mgmt->da); + if (!peer || (peer && !peer->is_mlo)) { + spin_unlock_bh(&ah->dp_hw.peer_lock); + spin_unlock_bh(&dp->dp_lock); + return false; + } + spin_unlock_bh(&ah->dp_hw.peer_lock); + } + spin_unlock_bh(&dp->dp_lock); + + if (vif->type == NL80211_IFTYPE_STATION) + return arvif->is_up && + (vif->valid_links == vif->active_links) && + !ieee80211_is_probe_req(fc) && + !ieee80211_is_auth(fc) && + !ieee80211_is_deauth(fc) && + !ath12k_is_addba_resp_action_code(mgmt); + + if (vif->type == NL80211_IFTYPE_AP) + return !(ieee80211_is_probe_resp(fc) || ieee80211_is_auth(fc) || + ieee80211_is_assoc_resp(fc) || ieee80211_is_reassoc_resp(fc) || + ath12k_is_addba_resp_action_code(mgmt)); + + return false; +} + +static const struct ath12k_hw_ops qcn9274_ops = { + .get_hw_mac_from_pdev_id = ath12k_wifi7_hw_qcn9274_mac_from_pdev_id, + .mac_id_to_pdev_id = ath12k_wifi7_hw_mac_id_to_pdev_id_qcn9274, + .mac_id_to_srng_id = ath12k_wifi7_hw_mac_id_to_srng_id_qcn9274, + .rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_qcn9274, + .get_ring_selector = ath12k_wifi7_hw_get_ring_selector_qcn9274, + .dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_qcn9274, + .is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_qcn9274, +}; + +static const struct ath12k_hw_ops wcn7850_ops = { + .get_hw_mac_from_pdev_id = ath12k_wifi7_hw_qcn9274_mac_from_pdev_id, + .mac_id_to_pdev_id = ath12k_wifi7_hw_mac_id_to_pdev_id_wcn7850, + .mac_id_to_srng_id = ath12k_wifi7_hw_mac_id_to_srng_id_wcn7850, + .rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_wcn7850, + .get_ring_selector = ath12k_wifi7_hw_get_ring_selector_wcn7850, + .dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_wcn7850, + .is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_wcn7850, +}; + +static const struct ath12k_hw_ops qcc2072_ops = { + .get_hw_mac_from_pdev_id = ath12k_wifi7_hw_qcn9274_mac_from_pdev_id, + .mac_id_to_pdev_id = ath12k_wifi7_hw_mac_id_to_pdev_id_wcn7850, + .mac_id_to_srng_id = ath12k_wifi7_hw_mac_id_to_srng_id_wcn7850, + .rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_qcc2072, + .get_ring_selector = ath12k_wifi7_hw_get_ring_selector_wcn7850, + .dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_wcn7850, + .is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_wcn7850, +}; + +#define ATH12K_TX_RING_MASK_0 0x1 +#define ATH12K_TX_RING_MASK_1 0x2 +#define ATH12K_TX_RING_MASK_2 0x4 +#define ATH12K_TX_RING_MASK_3 0x8 +#define ATH12K_TX_RING_MASK_4 0x10 + +#define ATH12K_RX_RING_MASK_0 0x1 +#define ATH12K_RX_RING_MASK_1 0x2 +#define ATH12K_RX_RING_MASK_2 0x4 +#define ATH12K_RX_RING_MASK_3 0x8 + +#define ATH12K_RX_ERR_RING_MASK_0 0x1 + +#define ATH12K_RX_WBM_REL_RING_MASK_0 0x1 + +#define ATH12K_REO_STATUS_RING_MASK_0 0x1 + +#define ATH12K_HOST2RXDMA_RING_MASK_0 0x1 + +#define ATH12K_RX_MON_RING_MASK_0 0x1 +#define ATH12K_RX_MON_RING_MASK_1 0x2 +#define ATH12K_RX_MON_RING_MASK_2 0x4 + +#define ATH12K_TX_MON_RING_MASK_0 0x1 +#define ATH12K_TX_MON_RING_MASK_1 0x2 + +#define ATH12K_RX_MON_STATUS_RING_MASK_0 0x1 +#define ATH12K_RX_MON_STATUS_RING_MASK_1 0x2 +#define ATH12K_RX_MON_STATUS_RING_MASK_2 0x4 + +static const struct ath12k_hw_ring_mask ath12k_wifi7_hw_ring_mask_qcn9274 = { + .tx = { + ATH12K_TX_RING_MASK_0, + ATH12K_TX_RING_MASK_1, + ATH12K_TX_RING_MASK_2, + ATH12K_TX_RING_MASK_3, + }, + .rx_mon_dest = { + 0, 0, 0, 0, + 0, 0, 0, 0, + ATH12K_RX_MON_RING_MASK_0, + ATH12K_RX_MON_RING_MASK_1, + ATH12K_RX_MON_RING_MASK_2, + }, + .rx = { + 0, 0, 0, 0, + ATH12K_RX_RING_MASK_0, + ATH12K_RX_RING_MASK_1, + ATH12K_RX_RING_MASK_2, + ATH12K_RX_RING_MASK_3, + }, + .rx_err = { + 0, 0, 0, + ATH12K_RX_ERR_RING_MASK_0, + }, + .rx_wbm_rel = { + 0, 0, 0, + ATH12K_RX_WBM_REL_RING_MASK_0, + }, + .reo_status = { + 0, 0, 0, + ATH12K_REO_STATUS_RING_MASK_0, + }, + .host2rxdma = { + 0, 0, 0, + ATH12K_HOST2RXDMA_RING_MASK_0, + }, + .tx_mon_dest = { + 0, 0, 0, + }, +}; + +static const struct ath12k_hw_ring_mask ath12k_wifi7_hw_ring_mask_ipq5332 = { + .tx = { + ATH12K_TX_RING_MASK_0, + ATH12K_TX_RING_MASK_1, + ATH12K_TX_RING_MASK_2, + ATH12K_TX_RING_MASK_3, + }, + .rx_mon_dest = { + 0, 0, 0, 0, 0, 0, 0, 0, + ATH12K_RX_MON_RING_MASK_0, + }, + .rx = { + 0, 0, 0, 0, + ATH12K_RX_RING_MASK_0, + ATH12K_RX_RING_MASK_1, + ATH12K_RX_RING_MASK_2, + ATH12K_RX_RING_MASK_3, + }, + .rx_err = { + 0, 0, 0, + ATH12K_RX_ERR_RING_MASK_0, + }, + .rx_wbm_rel = { + 0, 0, 0, + ATH12K_RX_WBM_REL_RING_MASK_0, + }, + .reo_status = { + 0, 0, 0, + ATH12K_REO_STATUS_RING_MASK_0, + }, + .host2rxdma = { + 0, 0, 0, + ATH12K_HOST2RXDMA_RING_MASK_0, + }, + .tx_mon_dest = { + ATH12K_TX_MON_RING_MASK_0, + ATH12K_TX_MON_RING_MASK_1, + }, +}; + +static const struct ath12k_hw_ring_mask ath12k_wifi7_hw_ring_mask_wcn7850 = { + .tx = { + ATH12K_TX_RING_MASK_0, + ATH12K_TX_RING_MASK_1, + ATH12K_TX_RING_MASK_2, + }, + .rx_mon_dest = { + }, + .rx_mon_status = { + 0, 0, 0, 0, + ATH12K_RX_MON_STATUS_RING_MASK_0, + ATH12K_RX_MON_STATUS_RING_MASK_1, + ATH12K_RX_MON_STATUS_RING_MASK_2, + }, + .rx = { + 0, 0, 0, + ATH12K_RX_RING_MASK_0, + ATH12K_RX_RING_MASK_1, + ATH12K_RX_RING_MASK_2, + ATH12K_RX_RING_MASK_3, + }, + .rx_err = { + ATH12K_RX_ERR_RING_MASK_0, + }, + .rx_wbm_rel = { + ATH12K_RX_WBM_REL_RING_MASK_0, + }, + .reo_status = { + ATH12K_REO_STATUS_RING_MASK_0, + }, + .host2rxdma = { + }, + .tx_mon_dest = { + }, +}; + +static const struct ce_ie_addr ath12k_wifi7_ce_ie_addr_ipq5332 = { + .ie1_reg_addr = CE_HOST_IE_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE, + .ie2_reg_addr = CE_HOST_IE_2_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE, + .ie3_reg_addr = CE_HOST_IE_3_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE, +}; + +static const struct ce_remap ath12k_wifi7_ce_remap_ipq5332 = { + .base = HAL_IPQ5332_CE_WFSS_REG_BASE, + .size = HAL_IPQ5332_CE_SIZE, + .cmem_offset = HAL_SEQ_WCSS_CMEM_OFFSET, +}; + +static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = { + { + .name = "qcn9274 hw1.0", + .hw_rev = ATH12K_HW_QCN9274_HW10, + .fw = { + .dir = "QCN9274/hw1.0", + .board_size = 256 * 1024, + .cal_offset = 128 * 1024, + .m3_loader = ath12k_m3_fw_loader_driver, + .download_aux_ucode = false, + }, + .max_radios = 1, + .single_pdev_only = false, + .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274, + .internal_sleep_clock = false, + + .hw_ops = &qcn9274_ops, + .ring_mask = &ath12k_wifi7_hw_ring_mask_qcn9274, + + .host_ce_config = ath12k_wifi7_host_ce_config_qcn9274, + .ce_count = 16, + .target_ce_config = ath12k_wifi7_target_ce_config_wlan_qcn9274, + .target_ce_count = 12, + .svc_to_ce_map = + ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274, + .svc_to_ce_map_len = 18, + + .rxdma1_enable = false, + .num_rxdma_per_pdev = 1, + .num_rxdma_dst_ring = 0, + .rx_mac_buf_ring = false, + .vdev_start_delay = false, + + .interface_modes = BIT(NL80211_IFTYPE_STATION) | + BIT(NL80211_IFTYPE_AP) | + BIT(NL80211_IFTYPE_MESH_POINT) | + BIT(NL80211_IFTYPE_AP_VLAN), + .supports_monitor = false, + + .idle_ps = false, + .download_calib = true, + .supports_suspend = false, + .tcl_ring_retry = true, + .reoq_lut_support = true, + .supports_shadow_regs = false, + + .num_tcl_banks = 48, + .max_tx_ring = 4, + + .mhi_config = &ath12k_wifi7_mhi_config_qcn9274, + + .wmi_init = ath12k_wifi7_wmi_init_qcn9274, + + .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01), + + .rfkill_pin = 0, + .rfkill_cfg = 0, + .rfkill_on_level = 0, + + .rddm_size = 0x600000, + + .def_num_link = 0, + .max_mlo_peer = 256, + + .otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB, + + .supports_sta_ps = false, + + .acpi_guid = NULL, + .supports_dynamic_smps_6ghz = true, + + .iova_mask = 0, + + .supports_aspm = false, + + .ce_ie_addr = NULL, + .ce_remap = NULL, + .bdf_addr_offset = 0, + + .current_cc_support = false, + + .dp_primary_link_only = true, + }, + { + .name = "wcn7850 hw2.0", + .hw_rev = ATH12K_HW_WCN7850_HW20, + + .fw = { + .dir = "WCN7850/hw2.0", + .board_size = 256 * 1024, + .cal_offset = 256 * 1024, + .m3_loader = ath12k_m3_fw_loader_driver, + .download_aux_ucode = false, + }, + + .max_radios = 1, + .single_pdev_only = true, + .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850, + .internal_sleep_clock = true, + + .hw_ops = &wcn7850_ops, + .ring_mask = &ath12k_wifi7_hw_ring_mask_wcn7850, + + .host_ce_config = ath12k_wifi7_host_ce_config_wcn7850, + .ce_count = 9, + .target_ce_config = ath12k_wifi7_target_ce_config_wlan_wcn7850, + .target_ce_count = 9, + .svc_to_ce_map = + ath12k_wifi7_target_service_to_ce_map_wlan_wcn7850, + .svc_to_ce_map_len = 14, + + .rxdma1_enable = false, + .num_rxdma_per_pdev = 2, + .num_rxdma_dst_ring = 1, + .rx_mac_buf_ring = true, + .vdev_start_delay = true, + + .interface_modes = BIT(NL80211_IFTYPE_STATION) | + BIT(NL80211_IFTYPE_AP) | + BIT(NL80211_IFTYPE_P2P_DEVICE) | + BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO), + .supports_monitor = true, + + .idle_ps = true, + .download_calib = false, + .supports_suspend = true, + .tcl_ring_retry = false, + .reoq_lut_support = false, + .supports_shadow_regs = true, + + .num_tcl_banks = 7, + .max_tx_ring = 3, + + .mhi_config = &ath12k_wifi7_mhi_config_wcn7850, + + .wmi_init = ath12k_wifi7_wmi_init_wcn7850, + + .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01) | + BIT(CNSS_PCIE_PERST_NO_PULL_V01), + + .rfkill_pin = 48, + .rfkill_cfg = 0, + .rfkill_on_level = 1, + + .rddm_size = 0x780000, + + .def_num_link = 2, + .max_mlo_peer = 32, + + .otp_board_id_register = 0, + + .supports_sta_ps = true, + + .acpi_guid = &wcn7850_uuid, + .supports_dynamic_smps_6ghz = false, + + .iova_mask = ATH12K_PCIE_MAX_PAYLOAD_SIZE - 1, + + .supports_aspm = true, + + .ce_ie_addr = NULL, + .ce_remap = NULL, + .bdf_addr_offset = 0, + + .current_cc_support = true, + + .dp_primary_link_only = false, + }, + { + .name = "qcn9274 hw2.0", + .hw_rev = ATH12K_HW_QCN9274_HW20, + .fw = { + .dir = "QCN9274/hw2.0", + .board_size = 256 * 1024, + .cal_offset = 128 * 1024, + .m3_loader = ath12k_m3_fw_loader_driver, + .download_aux_ucode = false, + }, + .max_radios = 2, + .single_pdev_only = false, + .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274, + .internal_sleep_clock = false, + + .hw_ops = &qcn9274_ops, + .ring_mask = &ath12k_wifi7_hw_ring_mask_qcn9274, + + .host_ce_config = ath12k_wifi7_host_ce_config_qcn9274, + .ce_count = 16, + .target_ce_config = ath12k_wifi7_target_ce_config_wlan_qcn9274, + .target_ce_count = 12, + .svc_to_ce_map = + ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274, + .svc_to_ce_map_len = 18, + + .rxdma1_enable = true, + .num_rxdma_per_pdev = 1, + .num_rxdma_dst_ring = 0, + .rx_mac_buf_ring = false, + .vdev_start_delay = false, + + .interface_modes = BIT(NL80211_IFTYPE_STATION) | + BIT(NL80211_IFTYPE_AP) | + BIT(NL80211_IFTYPE_MESH_POINT) | + BIT(NL80211_IFTYPE_AP_VLAN), + .supports_monitor = true, + + .idle_ps = false, + .download_calib = true, + .supports_suspend = false, + .tcl_ring_retry = true, + .reoq_lut_support = true, + .supports_shadow_regs = false, + + .num_tcl_banks = 48, + .max_tx_ring = 4, + + .mhi_config = &ath12k_wifi7_mhi_config_qcn9274, + + .wmi_init = ath12k_wifi7_wmi_init_qcn9274, + + .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01), + + .rfkill_pin = 0, + .rfkill_cfg = 0, + .rfkill_on_level = 0, + + .rddm_size = 0x600000, + + .def_num_link = 0, + .max_mlo_peer = 256, + + .otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB, + + .supports_sta_ps = false, + + .acpi_guid = NULL, + .supports_dynamic_smps_6ghz = true, + + .iova_mask = 0, + + .supports_aspm = false, + + .ce_ie_addr = NULL, + .ce_remap = NULL, + .bdf_addr_offset = 0, + + .current_cc_support = false, + + .dp_primary_link_only = true, + }, + { + .name = "ipq5332 hw1.0", + .hw_rev = ATH12K_HW_IPQ5332_HW10, + .fw = { + .dir = "IPQ5332/hw1.0", + .board_size = 256 * 1024, + .cal_offset = 128 * 1024, + .m3_loader = ath12k_m3_fw_loader_remoteproc, + .download_aux_ucode = false, + }, + .max_radios = 1, + .single_pdev_only = false, + .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_IPQ5332, + .internal_sleep_clock = false, + + .hw_ops = &qcn9274_ops, + .ring_mask = &ath12k_wifi7_hw_ring_mask_ipq5332, + + .host_ce_config = ath12k_wifi7_host_ce_config_ipq5332, + .ce_count = 12, + .target_ce_config = ath12k_wifi7_target_ce_config_wlan_ipq5332, + .target_ce_count = 12, + .svc_to_ce_map = + ath12k_wifi7_target_service_to_ce_map_wlan_ipq5332, + .svc_to_ce_map_len = 18, + + .rxdma1_enable = false, + .num_rxdma_per_pdev = 1, + .num_rxdma_dst_ring = 0, + .rx_mac_buf_ring = false, + .vdev_start_delay = false, + + .interface_modes = BIT(NL80211_IFTYPE_STATION) | + BIT(NL80211_IFTYPE_AP) | + BIT(NL80211_IFTYPE_MESH_POINT), + .supports_monitor = false, + + .idle_ps = false, + .download_calib = true, + .supports_suspend = false, + .tcl_ring_retry = true, + .reoq_lut_support = false, + .supports_shadow_regs = false, + + .num_tcl_banks = 48, + .max_tx_ring = 4, + + .wmi_init = &ath12k_wifi7_wmi_init_qcn9274, + + .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01), + + .rfkill_pin = 0, + .rfkill_cfg = 0, + .rfkill_on_level = 0, + + .rddm_size = 0, + + .def_num_link = 0, + .max_mlo_peer = 256, + + .otp_board_id_register = 0, + + .supports_sta_ps = false, + + .acpi_guid = NULL, + .supports_dynamic_smps_6ghz = false, + .iova_mask = 0, + .supports_aspm = false, + + .ce_ie_addr = &ath12k_wifi7_ce_ie_addr_ipq5332, + .ce_remap = &ath12k_wifi7_ce_remap_ipq5332, + .bdf_addr_offset = 0xC00000, + + .dp_primary_link_only = true, + }, + { + .name = "qcc2072 hw1.0", + .hw_rev = ATH12K_HW_QCC2072_HW10, + + .fw = { + .dir = "QCC2072/hw1.0", + .board_size = 256 * 1024, + .cal_offset = 256 * 1024, + .m3_loader = ath12k_m3_fw_loader_driver, + .download_aux_ucode = true, + }, + + .max_radios = 1, + .single_pdev_only = true, + .qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850, + .internal_sleep_clock = true, + + .hw_ops = &qcc2072_ops, + .ring_mask = &ath12k_wifi7_hw_ring_mask_wcn7850, + + .host_ce_config = ath12k_wifi7_host_ce_config_wcn7850, + .ce_count = 9, + .target_ce_config = ath12k_wifi7_target_ce_config_wlan_wcn7850, + .target_ce_count = 9, + .svc_to_ce_map = + ath12k_wifi7_target_service_to_ce_map_wlan_wcn7850, + .svc_to_ce_map_len = 14, + + .rxdma1_enable = false, + .num_rxdma_per_pdev = 2, + .num_rxdma_dst_ring = 1, + .rx_mac_buf_ring = true, + .vdev_start_delay = true, + + .interface_modes = BIT(NL80211_IFTYPE_STATION) | + BIT(NL80211_IFTYPE_AP) | + BIT(NL80211_IFTYPE_P2P_DEVICE) | + BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO), + .supports_monitor = true, + + .idle_ps = true, + .download_calib = false, + .supports_suspend = true, + .tcl_ring_retry = false, + .reoq_lut_support = false, + .supports_shadow_regs = true, + + .num_tcl_banks = 7, + .max_tx_ring = 3, + + .mhi_config = &ath12k_wifi7_mhi_config_wcn7850, + + .wmi_init = ath12k_wifi7_wmi_init_wcn7850, + + .qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01) | + BIT(CNSS_PCIE_PERST_NO_PULL_V01) | + BIT(CNSS_AUX_UC_SUPPORT_V01), + + .rfkill_pin = 0, + .rfkill_cfg = 0, + .rfkill_on_level = 0, + + .rddm_size = 0x780000, + + .def_num_link = 2, + .max_mlo_peer = 32, + + .otp_board_id_register = 0, + + .supports_sta_ps = true, + + .acpi_guid = &wcn7850_uuid, + .supports_dynamic_smps_6ghz = false, + + .iova_mask = 0, + + .supports_aspm = true, + + .ce_ie_addr = NULL, + .ce_remap = NULL, + .bdf_addr_offset = 0, + + .current_cc_support = true, + + .dp_primary_link_only = false, + }, +}; + +/* Note: called under rcu_read_lock() */ +static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, + struct ieee80211_tx_control *control, + struct sk_buff *skb) +{ + struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb); + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + struct ieee80211_vif *vif = info->control.vif; + struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif); + struct ath12k_link_vif *arvif = &ahvif->deflink; + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; + struct ieee80211_key_conf *key = info->control.hw_key; + struct ieee80211_sta *sta = control->sta; + struct ath12k_link_vif *tmp_arvif; + u32 info_flags = info->flags; + struct sk_buff *msdu_copied; + struct ath12k *ar, *tmp_ar; + struct ath12k_pdev_dp *dp_pdev, *tmp_dp_pdev; + struct ath12k_dp_link_peer *peer; + unsigned long links_map; + bool is_mcast = false; + bool is_dvlan = false; + struct ethhdr *eth; + bool is_prb_rsp; + u16 mcbc_gsn; + u8 link_id; + int ret; + struct ath12k_dp *tmp_dp; + + if (ahvif->vdev_type == WMI_VDEV_TYPE_MONITOR) { + ieee80211_free_txskb(hw, skb); + return; + } + + link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); + memset(skb_cb, 0, sizeof(*skb_cb)); + skb_cb->vif = vif; + + if (key) { + skb_cb->cipher = key->cipher; + skb_cb->flags |= ATH12K_SKB_CIPHER_SET; + } + + /* handle only for MLO case, use deflink for non MLO case */ + if (ieee80211_vif_is_mld(vif)) { + link_id = ath12k_mac_get_tx_link(sta, vif, link_id, skb, info_flags); + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) { + ieee80211_free_txskb(hw, skb); + return; + } + } else { + if (vif->type == NL80211_IFTYPE_P2P_DEVICE) + link_id = ATH12K_FIRST_SCAN_LINK; + else + link_id = 0; + } + + arvif = rcu_dereference(ahvif->link[link_id]); + if (!arvif || !arvif->ar) { + ath12k_warn(ahvif->ah, "failed to find arvif link id %u for frame transmission", + link_id); + ieee80211_free_txskb(hw, skb); + return; + } + + ar = arvif->ar; + skb_cb->link_id = link_id; + /* + * as skb_cb is common currently for dp and mgmt tx processing + * set this in the common mac op tx function. + */ + skb_cb->ar = ar; + is_prb_rsp = ieee80211_is_probe_resp(hdr->frame_control); + + if (info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP) { + eth = (struct ethhdr *)skb->data; + is_mcast = is_multicast_ether_addr(eth->h_dest); + + skb_cb->flags |= ATH12K_SKB_HW_80211_ENCAP; + } else if (ieee80211_is_mgmt(hdr->frame_control)) { + if (sta && sta->mlo) + skb_cb->flags |= ATH12K_SKB_MLO_STA; + + ret = ath12k_mac_mgmt_tx(ar, skb, is_prb_rsp); + if (ret) { + ath12k_warn(ar->ab, "failed to queue management frame %d\n", + ret); + ieee80211_free_txskb(hw, skb); + } + return; + } + + if (!(info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP)) + is_mcast = is_multicast_ether_addr(hdr->addr1); + + /* This is case only for P2P_GO */ + if (vif->type == NL80211_IFTYPE_AP && vif->p2p) + ath12k_mac_add_p2p_noa_ie(ar, vif, skb, is_prb_rsp); + + dp_pdev = ath12k_dp_to_pdev_dp(ar->ab->dp, ar->pdev_idx); + if (!dp_pdev) { + ieee80211_free_txskb(hw, skb); + return; + } + + /* Checking if it is a DVLAN frame */ + if (!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags) && + !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) && + !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) && + ieee80211_has_protected(hdr->frame_control)) + is_dvlan = true; + + if (!vif->valid_links || !is_mcast || is_dvlan || + (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) || + test_bit(ATH12K_FLAG_RAW_MODE, &ar->ab->dev_flags)) { + ret = ath12k_wifi7_dp_tx(dp_pdev, arvif, skb, false, 0, is_mcast); + if (unlikely(ret)) { + ath12k_warn(ar->ab, "failed to transmit frame %d\n", ret); + ieee80211_free_txskb(ar->ah->hw, skb); + return; + } + } else { + mcbc_gsn = atomic_inc_return(&ahvif->dp_vif.mcbc_gsn) & 0xfff; + + links_map = ahvif->links_map; + for_each_set_bit(link_id, &links_map, + IEEE80211_MLD_MAX_NUM_LINKS) { + tmp_arvif = rcu_dereference(ahvif->link[link_id]); + if (!tmp_arvif || !tmp_arvif->is_up) + continue; + + tmp_ar = tmp_arvif->ar; + tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_ar->ab->dp, + tmp_ar->pdev_idx); + if (!tmp_dp_pdev) + continue; + msdu_copied = skb_copy(skb, GFP_ATOMIC); + if (!msdu_copied) { + ath12k_err(ar->ab, + "skb copy failure link_id 0x%X vdevid 0x%X\n", + link_id, tmp_arvif->vdev_id); + continue; + } + + ath12k_mlo_mcast_update_tx_link_address(vif, link_id, + msdu_copied, + info_flags); + + skb_cb = ATH12K_SKB_CB(msdu_copied); + skb_cb->link_id = link_id; + skb_cb->vif = vif; + skb_cb->ar = tmp_ar; + + /* For open mode, skip peer find logic */ + if (unlikely(!ahvif->dp_vif.key_cipher)) + goto skip_peer_find; + + tmp_dp = ath12k_ab_to_dp(tmp_ar->ab); + spin_lock_bh(&tmp_dp->dp_lock); + peer = ath12k_dp_link_peer_find_by_addr(tmp_dp, + tmp_arvif->bssid); + if (!peer || !peer->dp_peer) { + spin_unlock_bh(&tmp_dp->dp_lock); + ath12k_warn(tmp_ar->ab, + "failed to find peer for vdev_id 0x%X addr %pM link_map 0x%X\n", + tmp_arvif->vdev_id, tmp_arvif->bssid, + ahvif->links_map); + dev_kfree_skb_any(msdu_copied); + continue; + } + + key = peer->dp_peer->keys[peer->dp_peer->mcast_keyidx]; + if (key) { + skb_cb->cipher = key->cipher; + skb_cb->flags |= ATH12K_SKB_CIPHER_SET; + + hdr = (struct ieee80211_hdr *)msdu_copied->data; + if (!ieee80211_has_protected(hdr->frame_control)) + hdr->frame_control |= + cpu_to_le16(IEEE80211_FCTL_PROTECTED); + } + spin_unlock_bh(&tmp_dp->dp_lock); + +skip_peer_find: + ret = ath12k_wifi7_dp_tx(tmp_dp_pdev, tmp_arvif, + msdu_copied, true, mcbc_gsn, is_mcast); + if (unlikely(ret)) { + if (ret == -ENOMEM) { + /* Drops are expected during heavy multicast + * frame flood. Print with debug log + * level to avoid lot of console prints + */ + ath12k_dbg(ar->ab, ATH12K_DBG_MAC, + "failed to transmit frame %d\n", + ret); + } else { + ath12k_warn(ar->ab, + "failed to transmit frame %d\n", + ret); + } + + dev_kfree_skb_any(msdu_copied); + } + } + ieee80211_free_txskb(ar->ah->hw, skb); + } +} + +static const struct ieee80211_ops ath12k_ops_wifi7 = { + .tx = ath12k_wifi7_mac_op_tx, + .wake_tx_queue = ieee80211_handle_wake_tx_queue, + .start = ath12k_mac_op_start, + .stop = ath12k_mac_op_stop, + .reconfig_complete = ath12k_mac_op_reconfig_complete, + .add_interface = ath12k_mac_op_add_interface, + .remove_interface = ath12k_mac_op_remove_interface, + .update_vif_offload = ath12k_mac_op_update_vif_offload, + .config = ath12k_mac_op_config, + .link_info_changed = ath12k_mac_op_link_info_changed, + .vif_cfg_changed = ath12k_mac_op_vif_cfg_changed, + .change_vif_links = ath12k_mac_op_change_vif_links, + .configure_filter = ath12k_mac_op_configure_filter, + .hw_scan = ath12k_mac_op_hw_scan, + .cancel_hw_scan = ath12k_mac_op_cancel_hw_scan, + .set_key = ath12k_mac_op_set_key, + .set_rekey_data = ath12k_mac_op_set_rekey_data, + .sta_state = ath12k_mac_op_sta_state, + .sta_set_txpwr = ath12k_mac_op_sta_set_txpwr, + .link_sta_rc_update = ath12k_mac_op_link_sta_rc_update, + .conf_tx = ath12k_mac_op_conf_tx, + .set_antenna = ath12k_mac_op_set_antenna, + .get_antenna = ath12k_mac_op_get_antenna, + .ampdu_action = ath12k_mac_op_ampdu_action, + .add_chanctx = ath12k_mac_op_add_chanctx, + .remove_chanctx = ath12k_mac_op_remove_chanctx, + .change_chanctx = ath12k_mac_op_change_chanctx, + .assign_vif_chanctx = ath12k_mac_op_assign_vif_chanctx, + .unassign_vif_chanctx = ath12k_mac_op_unassign_vif_chanctx, + .switch_vif_chanctx = ath12k_mac_op_switch_vif_chanctx, + .get_txpower = ath12k_mac_op_get_txpower, + .set_rts_threshold = ath12k_mac_op_set_rts_threshold, + .set_frag_threshold = ath12k_mac_op_set_frag_threshold, + .set_bitrate_mask = ath12k_mac_op_set_bitrate_mask, + .get_survey = ath12k_mac_op_get_survey, + .flush = ath12k_mac_op_flush, + .sta_statistics = ath12k_mac_op_sta_statistics, + .link_sta_statistics = ath12k_mac_op_link_sta_statistics, + .remain_on_channel = ath12k_mac_op_remain_on_channel, + .cancel_remain_on_channel = ath12k_mac_op_cancel_remain_on_channel, + .change_sta_links = ath12k_mac_op_change_sta_links, + .can_activate_links = ath12k_mac_op_can_activate_links, +#ifdef CONFIG_PM + .suspend = ath12k_wow_op_suspend, + .resume = ath12k_wow_op_resume, + .set_wakeup = ath12k_wow_op_set_wakeup, +#endif +#ifdef CONFIG_ATH12K_DEBUGFS + .vif_add_debugfs = ath12k_debugfs_op_vif_add, +#endif + CFG80211_TESTMODE_CMD(ath12k_tm_cmd) +#ifdef CONFIG_ATH12K_DEBUGFS + .link_sta_add_debugfs = ath12k_debugfs_link_sta_op_add, +#endif +}; + +int ath12k_wifi7_hw_init(struct ath12k_base *ab) +{ + const struct ath12k_hw_params *hw_params = NULL; + int i; + + for (i = 0; i < ARRAY_SIZE(ath12k_wifi7_hw_params); i++) { + hw_params = &ath12k_wifi7_hw_params[i]; + + if (hw_params->hw_rev == ab->hw_rev) + break; + } + + if (i == ARRAY_SIZE(ath12k_wifi7_hw_params)) { + ath12k_err(ab, "Unsupported Wi-Fi 7 hardware version: 0x%x\n", + ab->hw_rev); + return -EINVAL; + } + + ab->hw_params = hw_params; + ab->ath12k_ops = &ath12k_ops_wifi7; + + ath12k_wifi7_hal_init(ab); + + ath12k_info(ab, "Wi-Fi 7 Hardware name: %s\n", ab->hw_params->name); + + return 0; +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.h b/drivers/net/wireless/ath/ath12k/wifi7/hw.h new file mode 100644 index 0000000000000..643b6fdfdb66f --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_WIFI7_HW_H +#define ATH12K_WIFI7_HW_H + +struct ath12k_base; +int ath12k_wifi7_hw_init(struct ath12k_base *ab); + +#endif /* ATH12K_WIFI7_HW_H */ diff --git a/drivers/net/wireless/ath/ath12k/wifi7/mhi.c b/drivers/net/wireless/ath/ath12k/wifi7/mhi.c new file mode 100644 index 0000000000000..b8d972659314b --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/mhi.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "../mhi.h" +#include "mhi.h" + +static const struct mhi_channel_config ath12k_wifi7_mhi_channels_qcn9274[] = { + { + .num = 20, + .name = "IPCR", + .num_elements = 32, + .event_ring = 1, + .dir = DMA_TO_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + .auto_queue = false, + }, + { + .num = 21, + .name = "IPCR", + .num_elements = 32, + .event_ring = 1, + .dir = DMA_FROM_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + .auto_queue = true, + }, +}; + +static struct mhi_event_config ath12k_wifi7_mhi_events_qcn9274[] = { + { + .num_elements = 32, + .irq_moderation_ms = 0, + .irq = 1, + .data_type = MHI_ER_CTRL, + .mode = MHI_DB_BRST_DISABLE, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, + { + .num_elements = 256, + .irq_moderation_ms = 1, + .irq = 2, + .mode = MHI_DB_BRST_DISABLE, + .priority = 1, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, +}; + +const struct mhi_controller_config ath12k_wifi7_mhi_config_qcn9274 = { + .max_channels = 30, + .timeout_ms = 10000, + .use_bounce_buf = false, + .buf_len = 0, + .num_channels = ARRAY_SIZE(ath12k_wifi7_mhi_channels_qcn9274), + .ch_cfg = ath12k_wifi7_mhi_channels_qcn9274, + .num_events = ARRAY_SIZE(ath12k_wifi7_mhi_events_qcn9274), + .event_cfg = ath12k_wifi7_mhi_events_qcn9274, +}; + +static const struct mhi_channel_config ath12k_wifi7_mhi_channels_wcn7850[] = { + { + .num = 20, + .name = "IPCR", + .num_elements = 64, + .event_ring = 1, + .dir = DMA_TO_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + .auto_queue = false, + }, + { + .num = 21, + .name = "IPCR", + .num_elements = 64, + .event_ring = 1, + .dir = DMA_FROM_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + .auto_queue = true, + }, +}; + +static struct mhi_event_config ath12k_wifi7_mhi_events_wcn7850[] = { + { + .num_elements = 32, + .irq_moderation_ms = 0, + .irq = 1, + .mode = MHI_DB_BRST_DISABLE, + .data_type = MHI_ER_CTRL, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, + { + .num_elements = 256, + .irq_moderation_ms = 1, + .irq = 2, + .mode = MHI_DB_BRST_DISABLE, + .priority = 1, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, +}; + +const struct mhi_controller_config ath12k_wifi7_mhi_config_wcn7850 = { + .max_channels = 128, + .timeout_ms = 2000, + .use_bounce_buf = false, + .buf_len = 8192, + .num_channels = ARRAY_SIZE(ath12k_wifi7_mhi_channels_wcn7850), + .ch_cfg = ath12k_wifi7_mhi_channels_wcn7850, + .num_events = ARRAY_SIZE(ath12k_wifi7_mhi_events_wcn7850), + .event_cfg = ath12k_wifi7_mhi_events_wcn7850, +}; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/mhi.h b/drivers/net/wireless/ath/ath12k/wifi7/mhi.h new file mode 100644 index 0000000000000..2e2dd3503d830 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/mhi.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _ATH12K_WIFI7_MHI_H +#define _ATH12K_WIFI7_MHI_H +extern const struct mhi_controller_config ath12k_wifi7_mhi_config_qcn9274; +extern const struct mhi_controller_config ath12k_wifi7_mhi_config_wcn7850; +#endif /* _ATH12K_WIFI7_MHI_H */ diff --git a/drivers/net/wireless/ath/ath12k/wifi7/pci.c b/drivers/net/wireless/ath/ath12k/wifi7/pci.c new file mode 100644 index 0000000000000..6c96b52dec131 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/pci.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include + +#include "../pci.h" +#include "pci.h" +#include "../core.h" +#include "../hif.h" +#include "../mhi.h" +#include "hw.h" +#include "../hal.h" +#include "dp.h" +#include "core.h" +#include "hal.h" + +#define QCN9274_DEVICE_ID 0x1109 +#define WCN7850_DEVICE_ID 0x1107 +#define QCC2072_DEVICE_ID 0x1112 + +#define ATH12K_PCI_W7_SOC_HW_VERSION_1 1 +#define ATH12K_PCI_W7_SOC_HW_VERSION_2 2 + +#define TCSR_SOC_HW_VERSION 0x1B00000 +#define TCSR_SOC_HW_VERSION_MAJOR_MASK GENMASK(11, 8) +#define TCSR_SOC_HW_VERSION_MINOR_MASK GENMASK(7, 4) + +#define WINDOW_REG_ADDRESS 0x310c +#define WINDOW_REG_ADDRESS_QCC2072 0x3278 + +static const struct pci_device_id ath12k_wifi7_pci_id_table[] = { + { PCI_VDEVICE(QCOM, QCN9274_DEVICE_ID) }, + { PCI_VDEVICE(QCOM, WCN7850_DEVICE_ID) }, + { PCI_VDEVICE(QCOM, QCC2072_DEVICE_ID) }, + {} +}; + +MODULE_DEVICE_TABLE(pci, ath12k_wifi7_pci_id_table); + +/* TODO: revisit IRQ mapping for new SRNG's */ +static const struct ath12k_msi_config ath12k_wifi7_msi_config[] = { + { + .total_vectors = 16, + .total_users = 3, + .users = (struct ath12k_msi_user[]) { + { .name = "MHI", .num_vectors = 3, .base_vector = 0 }, + { .name = "CE", .num_vectors = 5, .base_vector = 3 }, + { .name = "DP", .num_vectors = 8, .base_vector = 8 }, + }, + }, +}; + +static const struct ath12k_pci_ops ath12k_wifi7_pci_ops_qcn9274 = { + .wakeup = NULL, + .release = NULL, +}; + +static int ath12k_wifi7_pci_bus_wake_up(struct ath12k_base *ab) +{ + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); + + return mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev); +} + +static void ath12k_wifi7_pci_bus_release(struct ath12k_base *ab) +{ + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); + + mhi_device_put(ab_pci->mhi_ctrl->mhi_dev); +} + +static const struct ath12k_pci_ops ath12k_wifi7_pci_ops_wcn7850 = { + .wakeup = ath12k_wifi7_pci_bus_wake_up, + .release = ath12k_wifi7_pci_bus_release, +}; + +static +void ath12k_wifi7_pci_read_hw_version(struct ath12k_base *ab, + u32 *major, u32 *minor) +{ + u32 soc_hw_version; + + soc_hw_version = ath12k_pci_read32(ab, TCSR_SOC_HW_VERSION); + *major = u32_get_bits(soc_hw_version, TCSR_SOC_HW_VERSION_MAJOR_MASK); + *minor = u32_get_bits(soc_hw_version, TCSR_SOC_HW_VERSION_MINOR_MASK); +} + +static int ath12k_wifi7_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *pci_dev) +{ + u32 soc_hw_version_major, soc_hw_version_minor; + struct ath12k_pci *ab_pci; + struct ath12k_base *ab; + int ret; + + ab = pci_get_drvdata(pdev); + if (!ab) + return -EINVAL; + + ab_pci = ath12k_pci_priv(ab); + if (!ab_pci) + return -EINVAL; + + switch (pci_dev->device) { + case QCN9274_DEVICE_ID: + ab_pci->msi_config = &ath12k_wifi7_msi_config[0]; + ab->static_window_map = true; + ab_pci->pci_ops = &ath12k_wifi7_pci_ops_qcn9274; + /* + * init window reg addr before reading hardware version + * as it will be used there + */ + ab_pci->window_reg_addr = WINDOW_REG_ADDRESS; + ath12k_wifi7_pci_read_hw_version(ab, &soc_hw_version_major, + &soc_hw_version_minor); + ab->target_mem_mode = ath12k_core_get_memory_mode(ab); + switch (soc_hw_version_major) { + case ATH12K_PCI_W7_SOC_HW_VERSION_2: + ab->hw_rev = ATH12K_HW_QCN9274_HW20; + break; + case ATH12K_PCI_W7_SOC_HW_VERSION_1: + ab->hw_rev = ATH12K_HW_QCN9274_HW10; + break; + default: + dev_err(&pdev->dev, + "Unknown hardware version found for QCN9274: 0x%x\n", + soc_hw_version_major); + return -EOPNOTSUPP; + } + break; + case WCN7850_DEVICE_ID: + ab->id.bdf_search = ATH12K_BDF_SEARCH_BUS_AND_BOARD; + ab_pci->msi_config = &ath12k_wifi7_msi_config[0]; + ab->static_window_map = false; + ab_pci->pci_ops = &ath12k_wifi7_pci_ops_wcn7850; + /* + * init window reg addr before reading hardware version + * as it will be used there + */ + ab_pci->window_reg_addr = WINDOW_REG_ADDRESS; + ath12k_wifi7_pci_read_hw_version(ab, &soc_hw_version_major, + &soc_hw_version_minor); + ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT; + switch (soc_hw_version_major) { + case ATH12K_PCI_W7_SOC_HW_VERSION_2: + ab->hw_rev = ATH12K_HW_WCN7850_HW20; + break; + default: + dev_err(&pdev->dev, + "Unknown hardware version found for WCN7850: 0x%x\n", + soc_hw_version_major); + return -EOPNOTSUPP; + } + break; + case QCC2072_DEVICE_ID: + ab->id.bdf_search = ATH12K_BDF_SEARCH_BUS_AND_BOARD; + ab_pci->msi_config = &ath12k_wifi7_msi_config[0]; + ab->static_window_map = false; + ab_pci->pci_ops = &ath12k_wifi7_pci_ops_wcn7850; + ab_pci->window_reg_addr = WINDOW_REG_ADDRESS_QCC2072; + ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT; + /* there is only one version till now */ + ab->hw_rev = ATH12K_HW_QCC2072_HW10; + break; + default: + dev_err(&pdev->dev, "Unknown Wi-Fi 7 PCI device found: 0x%x\n", + pci_dev->device); + return -EOPNOTSUPP; + } + + ret = ath12k_wifi7_hw_init(ab); + if (ret) { + dev_err(&pdev->dev, "WiFi-7 hw_init for PCI failed: %d\n", ret); + return ret; + } + + return 0; +} + +static const struct ath12k_pci_reg_base ath12k_wifi7_reg_base = { + .umac_base = HAL_SEQ_WCSS_UMAC_OFFSET, + .ce_reg_base = HAL_CE_WFSS_CE_REG_BASE, +}; + +static struct ath12k_pci_driver ath12k_wifi7_pci_driver = { + .name = "ath12k_wifi7_pci", + .id_table = ath12k_wifi7_pci_id_table, + .ops.probe = ath12k_wifi7_pci_probe, + .reg_base = &ath12k_wifi7_reg_base, + .ops.arch_init = ath12k_wifi7_arch_init, + .ops.arch_deinit = ath12k_wifi7_arch_deinit, +}; + +int ath12k_wifi7_pci_init(void) +{ + int ret; + + ret = ath12k_pci_register_driver(ATH12K_DEVICE_FAMILY_WIFI7, + &ath12k_wifi7_pci_driver); + if (ret) { + pr_err("Failed to register ath12k Wi-Fi 7 driver: %d\n", + ret); + return ret; + } + + return 0; +} + +void ath12k_wifi7_pci_exit(void) +{ + ath12k_pci_unregister_driver(ATH12K_DEVICE_FAMILY_WIFI7); +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/pci.h b/drivers/net/wireless/ath/ath12k/wifi7/pci.h new file mode 100644 index 0000000000000..662a8bab0ce7f --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/pci.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ +#ifndef ATH12K_PCI_WIFI7_H +#define ATH12K_PCI_WIFI7_H + +int ath12k_wifi7_pci_init(void); +void ath12k_wifi7_pci_exit(void); + +#endif /* ATH12K_PCI_WIFI7_H */ diff --git a/drivers/net/wireless/ath/ath12k/wifi7/wmi.c b/drivers/net/wireless/ath/ath12k/wifi7/wmi.c new file mode 100644 index 0000000000000..ed538d20d324a --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/wmi.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include "../core.h" +#include "wmi.h" + +void ath12k_wifi7_wmi_init_qcn9274(struct ath12k_base *ab, + struct ath12k_wmi_resource_config_arg *config) +{ + config->num_vdevs = ab->num_radios * TARGET_NUM_VDEVS(ab); + config->num_peers = ab->num_radios * + ath12k_core_get_max_peers_per_radio(ab); + config->num_offload_peers = TARGET_NUM_OFFLD_PEERS; + config->num_offload_reorder_buffs = TARGET_NUM_OFFLD_REORDER_BUFFS; + config->num_peer_keys = TARGET_NUM_PEER_KEYS; + config->ast_skid_limit = TARGET_AST_SKID_LIMIT; + config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; + config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; + config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI; + config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI; + config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI; + config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI; + + if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) + config->rx_decap_mode = TARGET_DECAP_MODE_RAW; + else + config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI; + + config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS; + config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV; + config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV; + config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES; + config->num_mcast_groups = TARGET_NUM_MCAST_GROUPS; + config->num_mcast_table_elems = TARGET_NUM_MCAST_TABLE_ELEMS; + config->mcast2ucast_mode = TARGET_MCAST2UCAST_MODE; + config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE; + config->num_wds_entries = TARGET_NUM_WDS_ENTRIES; + config->dma_burst_size = TARGET_DMA_BURST_SIZE; + config->rx_skip_defrag_timeout_dup_detection_check = + TARGET_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; + config->vow_config = TARGET_VOW_CONFIG; + config->gtk_offload_max_vdev = TARGET_GTK_OFFLOAD_MAX_VDEV; + config->num_msdu_desc = TARGET_NUM_MSDU_DESC; + config->beacon_tx_offload_max_vdev = ab->num_radios * TARGET_MAX_BCN_OFFLD; + config->rx_batchmode = TARGET_RX_BATCHMODE; + /* Indicates host supports peer map v3 and unmap v2 support */ + config->peer_map_unmap_version = 0x32; + config->twt_ap_pdev_count = ab->num_radios; + config->twt_ap_sta_count = 1000; + config->ema_max_vap_cnt = ab->num_radios; + config->ema_max_profile_period = TARGET_EMA_MAX_PROFILE_PERIOD; + config->beacon_tx_offload_max_vdev += config->ema_max_vap_cnt; + + if (test_bit(WMI_TLV_SERVICE_PEER_METADATA_V1A_V1B_SUPPORT, ab->wmi_ab.svc_map)) + config->peer_metadata_ver = ATH12K_PEER_METADATA_V1B; +} + +void ath12k_wifi7_wmi_init_wcn7850(struct ath12k_base *ab, + struct ath12k_wmi_resource_config_arg *config) +{ + config->num_vdevs = 4; + config->num_peers = 16; + config->num_tids = 32; + + config->num_offload_peers = 3; + config->num_offload_reorder_buffs = 3; + config->num_peer_keys = TARGET_NUM_PEER_KEYS; + config->ast_skid_limit = TARGET_AST_SKID_LIMIT; + config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; + config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; + config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI; + config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI; + config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI; + config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI; + config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI; + config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS; + config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV; + config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV; + config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES; + config->num_mcast_groups = 0; + config->num_mcast_table_elems = 0; + config->mcast2ucast_mode = 0; + config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE; + config->num_wds_entries = 0; + config->dma_burst_size = 0; + config->rx_skip_defrag_timeout_dup_detection_check = 0; + config->vow_config = TARGET_VOW_CONFIG; + config->gtk_offload_max_vdev = 2; + config->num_msdu_desc = 0x400; + config->beacon_tx_offload_max_vdev = 2; + config->rx_batchmode = TARGET_RX_BATCHMODE; + + config->peer_map_unmap_version = 0x1; + config->use_pdev_id = 1; + config->max_frag_entries = 0xa; + config->num_tdls_vdevs = 0x1; + config->num_tdls_conn_table_entries = 8; + config->beacon_tx_offload_max_vdev = 0x2; + config->num_multicast_filter_entries = 0x20; + config->num_wow_filters = 0x16; + config->num_keep_alive_pattern = 0; + + if (test_bit(WMI_TLV_SERVICE_PEER_METADATA_V1A_V1B_SUPPORT, ab->wmi_ab.svc_map)) + config->peer_metadata_ver = ATH12K_PEER_METADATA_V1A; + else + config->peer_metadata_ver = ab->wmi_ab.dp_peer_meta_data_ver; +} diff --git a/drivers/net/wireless/ath/ath12k/wifi7/wmi.h b/drivers/net/wireless/ath/ath12k/wifi7/wmi.h new file mode 100644 index 0000000000000..ae74e176fa2d3 --- /dev/null +++ b/drivers/net/wireless/ath/ath12k/wifi7/wmi.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* + * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef ATH12K_WMI_WIFI7_H +#define ATH12K_WMI_WIFI7_H + +void ath12k_wifi7_wmi_init_qcn9274(struct ath12k_base *ab, + struct ath12k_wmi_resource_config_arg *config); +void ath12k_wifi7_wmi_init_wcn7850(struct ath12k_base *ab, + struct ath12k_wmi_resource_config_arg *config); + +#endif diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c index ff6b3d4ea8208..7617fc3a2479c 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.c +++ b/drivers/net/wireless/ath/ath12k/wmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include @@ -14,6 +14,7 @@ #include #include #include +#include #include "core.h" #include "debugfs.h" #include "debug.h" @@ -125,6 +126,14 @@ struct wmi_tlv_mgmt_rx_parse { bool frame_buf_done; }; +struct wmi_pdev_set_obss_bitmap_arg { + u32 tlv_tag; + u32 pdev_id; + u32 cmd_id; + const u32 *bitmap; + const char *label; +}; + static const struct ath12k_wmi_tlv_policy ath12k_wmi_tlv_policies[] = { [WMI_TAG_ARRAY_BYTE] = { .min_len = 0 }, [WMI_TAG_ARRAY_UINT32] = { .min_len = 0 }, @@ -190,6 +199,8 @@ static const struct ath12k_wmi_tlv_policy ath12k_wmi_tlv_policies[] = { .min_len = sizeof(struct wmi_11d_new_cc_event) }, [WMI_TAG_PER_CHAIN_RSSI_STATS] = { .min_len = sizeof(struct wmi_per_chain_rssi_stat_params) }, + [WMI_TAG_OBSS_COLOR_COLLISION_EVT] = { + .min_len = sizeof(struct wmi_obss_color_collision_event) }, }; __le32 ath12k_wmi_tlv_hdr(u32 cmd, u32 len) @@ -203,103 +214,6 @@ static __le32 ath12k_wmi_tlv_cmd_hdr(u32 cmd, u32 len) return ath12k_wmi_tlv_hdr(cmd, len - TLV_HDR_SIZE); } -void ath12k_wmi_init_qcn9274(struct ath12k_base *ab, - struct ath12k_wmi_resource_config_arg *config) -{ - config->num_vdevs = ab->num_radios * TARGET_NUM_VDEVS(ab); - config->num_peers = ab->num_radios * - ath12k_core_get_max_peers_per_radio(ab); - config->num_offload_peers = TARGET_NUM_OFFLD_PEERS; - config->num_offload_reorder_buffs = TARGET_NUM_OFFLD_REORDER_BUFFS; - config->num_peer_keys = TARGET_NUM_PEER_KEYS; - config->ast_skid_limit = TARGET_AST_SKID_LIMIT; - config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; - config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; - config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI; - config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI; - config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI; - config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI; - - if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) - config->rx_decap_mode = TARGET_DECAP_MODE_RAW; - else - config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI; - - config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS; - config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV; - config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV; - config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES; - config->num_mcast_groups = TARGET_NUM_MCAST_GROUPS; - config->num_mcast_table_elems = TARGET_NUM_MCAST_TABLE_ELEMS; - config->mcast2ucast_mode = TARGET_MCAST2UCAST_MODE; - config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE; - config->num_wds_entries = TARGET_NUM_WDS_ENTRIES; - config->dma_burst_size = TARGET_DMA_BURST_SIZE; - config->rx_skip_defrag_timeout_dup_detection_check = - TARGET_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; - config->vow_config = TARGET_VOW_CONFIG; - config->gtk_offload_max_vdev = TARGET_GTK_OFFLOAD_MAX_VDEV; - config->num_msdu_desc = TARGET_NUM_MSDU_DESC; - config->beacon_tx_offload_max_vdev = ab->num_radios * TARGET_MAX_BCN_OFFLD; - config->rx_batchmode = TARGET_RX_BATCHMODE; - /* Indicates host supports peer map v3 and unmap v2 support */ - config->peer_map_unmap_version = 0x32; - config->twt_ap_pdev_count = ab->num_radios; - config->twt_ap_sta_count = 1000; - config->ema_max_vap_cnt = ab->num_radios; - config->ema_max_profile_period = TARGET_EMA_MAX_PROFILE_PERIOD; - config->beacon_tx_offload_max_vdev += config->ema_max_vap_cnt; - - if (test_bit(WMI_TLV_SERVICE_PEER_METADATA_V1A_V1B_SUPPORT, ab->wmi_ab.svc_map)) - config->peer_metadata_ver = ATH12K_PEER_METADATA_V1B; -} - -void ath12k_wmi_init_wcn7850(struct ath12k_base *ab, - struct ath12k_wmi_resource_config_arg *config) -{ - config->num_vdevs = 4; - config->num_peers = 16; - config->num_tids = 32; - - config->num_offload_peers = 3; - config->num_offload_reorder_buffs = 3; - config->num_peer_keys = TARGET_NUM_PEER_KEYS; - config->ast_skid_limit = TARGET_AST_SKID_LIMIT; - config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; - config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1; - config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI; - config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI; - config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI; - config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI; - config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI; - config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS; - config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV; - config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV; - config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES; - config->num_mcast_groups = 0; - config->num_mcast_table_elems = 0; - config->mcast2ucast_mode = 0; - config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE; - config->num_wds_entries = 0; - config->dma_burst_size = 0; - config->rx_skip_defrag_timeout_dup_detection_check = 0; - config->vow_config = TARGET_VOW_CONFIG; - config->gtk_offload_max_vdev = 2; - config->num_msdu_desc = 0x400; - config->beacon_tx_offload_max_vdev = 2; - config->rx_batchmode = TARGET_RX_BATCHMODE; - - config->peer_map_unmap_version = 0x1; - config->use_pdev_id = 1; - config->max_frag_entries = 0xa; - config->num_tdls_vdevs = 0x1; - config->num_tdls_conn_table_entries = 8; - config->beacon_tx_offload_max_vdev = 0x2; - config->num_multicast_filter_entries = 0x20; - config->num_wow_filters = 0x16; - config->num_keep_alive_pattern = 0; -} - #define PRIMAP(_hw_mode_) \ [_hw_mode_] = _hw_mode_##_PRI @@ -493,6 +407,7 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, struct ath12k_band_cap *cap_band; struct ath12k_pdev_cap *pdev_cap = &pdev->cap; struct ath12k_fw_pdev *fw_pdev; + u32 supported_bands; u32 phy_map; u32 hw_idx, phy_idx = 0; int i; @@ -516,14 +431,19 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, return -EINVAL; mac_caps = wmi_mac_phy_caps + phy_idx; + supported_bands = le32_to_cpu(mac_caps->supported_bands); + + if (!(supported_bands & WMI_HOST_WLAN_2GHZ_CAP) && + !(supported_bands & WMI_HOST_WLAN_5GHZ_CAP)) + return -EINVAL; pdev->pdev_id = ath12k_wmi_mac_phy_get_pdev_id(mac_caps); pdev->hw_link_id = ath12k_wmi_mac_phy_get_hw_link_id(mac_caps); - pdev_cap->supported_bands |= le32_to_cpu(mac_caps->supported_bands); + pdev_cap->supported_bands |= supported_bands; pdev_cap->ampdu_density = le32_to_cpu(mac_caps->ampdu_density); fw_pdev = &ab->fw_pdev[ab->fw_pdev_count]; - fw_pdev->supported_bands = le32_to_cpu(mac_caps->supported_bands); + fw_pdev->supported_bands = supported_bands; fw_pdev->pdev_id = ath12k_wmi_mac_phy_get_pdev_id(mac_caps); fw_pdev->phy_id = le32_to_cpu(mac_caps->phy_id); ab->fw_pdev_count++; @@ -532,10 +452,12 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, * band to band for a single radio, need to see how this should be * handled. */ - if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_2GHZ_CAP) { + if (supported_bands & WMI_HOST_WLAN_2GHZ_CAP) { pdev_cap->tx_chain_mask = le32_to_cpu(mac_caps->tx_chain_mask_2g); pdev_cap->rx_chain_mask = le32_to_cpu(mac_caps->rx_chain_mask_2g); - } else if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_5GHZ_CAP) { + } + + if (supported_bands & WMI_HOST_WLAN_5GHZ_CAP) { pdev_cap->vht_cap = le32_to_cpu(mac_caps->vht_cap_info_5g); pdev_cap->vht_mcs = le32_to_cpu(mac_caps->vht_supp_mcs_5g); pdev_cap->he_mcs = le32_to_cpu(mac_caps->he_supp_mcs_5g); @@ -545,8 +467,6 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, WMI_NSS_RATIO_EN_DIS_GET(mac_caps->nss_ratio); pdev_cap->nss_ratio_info = WMI_NSS_RATIO_INFO_GET(mac_caps->nss_ratio); - } else { - return -EINVAL; } /* tx/rx chainmask reported from fw depends on the actual hw chains used, @@ -562,7 +482,7 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, pdev_cap->rx_chain_mask_shift = find_first_bit((unsigned long *)&pdev_cap->rx_chain_mask, 32); - if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_2GHZ_CAP) { + if (supported_bands & WMI_HOST_WLAN_2GHZ_CAP) { cap_band = &pdev_cap->band[NL80211_BAND_2GHZ]; cap_band->phy_id = le32_to_cpu(mac_caps->phy_id); cap_band->max_bw_supported = le32_to_cpu(mac_caps->max_bw_supported_2g); @@ -582,7 +502,7 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle, le32_to_cpu(mac_caps->he_ppet2g.ppet16_ppet8_ru3_ru0[i]); } - if (le32_to_cpu(mac_caps->supported_bands) & WMI_HOST_WLAN_5GHZ_CAP) { + if (supported_bands & WMI_HOST_WLAN_5GHZ_CAP) { cap_band = &pdev_cap->band[NL80211_BAND_5GHZ]; cap_band->phy_id = le32_to_cpu(mac_caps->phy_id); cap_band->max_bw_supported = @@ -2367,10 +2287,13 @@ int ath12k_wmi_send_peer_assoc_cmd(struct ath12k *ar, cmd->peer_bw_rxnss_override |= cpu_to_le32(arg->peer_bw_rxnss_override); if (arg->vht_capable) { - mcs->rx_max_rate = cpu_to_le32(arg->rx_max_rate); - mcs->rx_mcs_set = cpu_to_le32(arg->rx_mcs_set); - mcs->tx_max_rate = cpu_to_le32(arg->tx_max_rate); - mcs->tx_mcs_set = cpu_to_le32(arg->tx_mcs_set); + /* Firmware interprets mcs->tx_mcs_set field as peer's + * RX capability + */ + mcs->rx_max_rate = cpu_to_le32(arg->tx_max_rate); + mcs->rx_mcs_set = cpu_to_le32(arg->tx_mcs_set); + mcs->tx_max_rate = cpu_to_le32(arg->rx_max_rate); + mcs->tx_mcs_set = cpu_to_le32(arg->rx_mcs_set); } /* HE Rates */ @@ -2891,7 +2814,8 @@ int ath12k_wmi_send_scan_chan_list_cmd(struct ath12k *ar, max_chan_limit = (wmi->wmi_ab->max_msg_len[ar->pdev_idx] - len) / sizeof(*chan_info); - num_send_chans = min(arg->nallchans, max_chan_limit); + num_send_chans = min3(arg->nallchans, max_chan_limit, + ATH12K_WMI_MAX_NUM_CHAN_PER_CMD); arg->nallchans -= num_send_chans; len += sizeof(*chan_info) * num_send_chans; @@ -3644,6 +3568,140 @@ ath12k_wmi_send_obss_spr_cmd(struct ath12k *ar, u32 vdev_id, return ret; } +u32 ath12k_wmi_build_obss_pd(const struct ath12k_wmi_obss_pd_arg *arg) +{ + u32 param_val = 0; + + param_val |= u32_encode_bits((u8)arg->srg_th, GENMASK(15, 8)); + param_val |= u32_encode_bits((u8)arg->non_srg_th, GENMASK(7, 0)); + + if (arg->srp_support) + param_val |= ATH12K_OBSS_PD_THRESHOLD_IN_DBM; + + if (arg->srg_enabled && arg->srp_support) + param_val |= ATH12K_OBSS_PD_SRG_EN; + + if (arg->non_srg_enabled) + param_val |= ATH12K_OBSS_PD_NON_SRG_EN; + + return param_val; +} + +static int ath12k_wmi_pdev_set_obss_bitmap(struct ath12k *ar, + const struct wmi_pdev_set_obss_bitmap_arg *arg) +{ + struct wmi_pdev_obss_pd_bitmap_cmd *cmd; + struct ath12k_wmi_pdev *wmi = ar->wmi; + const int len = sizeof(*cmd); + struct sk_buff *skb; + int ret; + + skb = ath12k_wmi_alloc_skb(wmi->wmi_ab, len); + if (!skb) + return -ENOMEM; + + cmd = (struct wmi_pdev_obss_pd_bitmap_cmd *)skb->data; + cmd->tlv_header = ath12k_wmi_tlv_cmd_hdr(arg->tlv_tag, len); + cmd->pdev_id = cpu_to_le32(arg->pdev_id); + memcpy(cmd->bitmap, arg->bitmap, sizeof(cmd->bitmap)); + + ath12k_dbg(ar->ab, ATH12K_DBG_WMI, + "wmi set pdev %u %s %08x %08x\n", + arg->pdev_id, arg->label, arg->bitmap[0], arg->bitmap[1]); + + ret = ath12k_wmi_cmd_send(wmi, skb, arg->cmd_id); + if (ret) { + ath12k_warn(ar->ab, "failed to send %s: %d\n", arg->label, ret); + dev_kfree_skb(skb); + } + + return ret; +} + +int ath12k_wmi_pdev_set_srg_bss_color_bitmap(struct ath12k *ar, + u32 pdev_id, const u32 *bitmap) +{ + struct wmi_pdev_set_obss_bitmap_arg arg = { + .tlv_tag = WMI_TAG_PDEV_SRG_BSS_COLOR_BITMAP_CMD, + .pdev_id = pdev_id, + .cmd_id = WMI_PDEV_SET_SRG_BSS_COLOR_BITMAP_CMDID, + .bitmap = bitmap, + .label = "SRG bss color bitmap", + }; + + return ath12k_wmi_pdev_set_obss_bitmap(ar, &arg); +} + +int ath12k_wmi_pdev_set_srg_partial_bssid_bitmap(struct ath12k *ar, + u32 pdev_id, const u32 *bitmap) +{ + struct wmi_pdev_set_obss_bitmap_arg arg = { + .tlv_tag = WMI_TAG_PDEV_SRG_PARTIAL_BSSID_BITMAP_CMD, + .pdev_id = pdev_id, + .cmd_id = WMI_PDEV_SET_SRG_PARTIAL_BSSID_BITMAP_CMDID, + .bitmap = bitmap, + .label = "SRG partial bssid bitmap", + }; + + return ath12k_wmi_pdev_set_obss_bitmap(ar, &arg); +} + +int ath12k_wmi_pdev_srg_obss_color_enable_bitmap(struct ath12k *ar, + u32 pdev_id, const u32 *bitmap) +{ + struct wmi_pdev_set_obss_bitmap_arg arg = { + .tlv_tag = WMI_TAG_PDEV_SRG_OBSS_COLOR_ENABLE_BITMAP_CMD, + .pdev_id = pdev_id, + .cmd_id = WMI_PDEV_SET_SRG_OBSS_COLOR_ENABLE_BITMAP_CMDID, + .bitmap = bitmap, + .label = "SRG obss color enable bitmap", + }; + + return ath12k_wmi_pdev_set_obss_bitmap(ar, &arg); +} + +int ath12k_wmi_pdev_srg_obss_bssid_enable_bitmap(struct ath12k *ar, + u32 pdev_id, const u32 *bitmap) +{ + struct wmi_pdev_set_obss_bitmap_arg arg = { + .tlv_tag = WMI_TAG_PDEV_SRG_OBSS_BSSID_ENABLE_BITMAP_CMD, + .pdev_id = pdev_id, + .cmd_id = WMI_PDEV_SET_SRG_OBSS_BSSID_ENABLE_BITMAP_CMDID, + .bitmap = bitmap, + .label = "SRG obss bssid enable bitmap", + }; + + return ath12k_wmi_pdev_set_obss_bitmap(ar, &arg); +} + +int ath12k_wmi_pdev_non_srg_obss_color_enable_bitmap(struct ath12k *ar, + u32 pdev_id, const u32 *bitmap) +{ + struct wmi_pdev_set_obss_bitmap_arg arg = { + .tlv_tag = WMI_TAG_PDEV_NON_SRG_OBSS_COLOR_ENABLE_BITMAP_CMD, + .pdev_id = pdev_id, + .cmd_id = WMI_PDEV_SET_NON_SRG_OBSS_COLOR_ENABLE_BITMAP_CMDID, + .bitmap = bitmap, + .label = "non SRG obss color enable bitmap", + }; + + return ath12k_wmi_pdev_set_obss_bitmap(ar, &arg); +} + +int ath12k_wmi_pdev_non_srg_obss_bssid_enable_bitmap(struct ath12k *ar, + u32 pdev_id, const u32 *bitmap) +{ + struct wmi_pdev_set_obss_bitmap_arg arg = { + .tlv_tag = WMI_TAG_PDEV_NON_SRG_OBSS_BSSID_ENABLE_BITMAP_CMD, + .pdev_id = pdev_id, + .cmd_id = WMI_PDEV_SET_NON_SRG_OBSS_BSSID_ENABLE_BITMAP_CMDID, + .bitmap = bitmap, + .label = "non SRG obss bssid enable bitmap", + }; + + return ath12k_wmi_pdev_set_obss_bitmap(ar, &arg); +} + int ath12k_wmi_obss_color_cfg_cmd(struct ath12k *ar, u32 vdev_id, u8 bss_color, u32 period, bool enable) @@ -3847,6 +3905,58 @@ int ath12k_wmi_fils_discovery(struct ath12k *ar, u32 vdev_id, u32 interval, return ret; } +static void +ath12k_wmi_obss_color_collision_event(struct ath12k_base *ab, struct sk_buff *skb) +{ + const struct wmi_obss_color_collision_event *ev; + struct ath12k_link_vif *arvif; + u32 vdev_id, evt_type; + u64 bitmap; + + const void **tb __free(kfree) = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC); + if (IS_ERR(tb)) { + ath12k_warn(ab, "failed to parse OBSS color collision tlv %ld\n", + PTR_ERR(tb)); + return; + } + + ev = tb[WMI_TAG_OBSS_COLOR_COLLISION_EVT]; + if (!ev) { + ath12k_warn(ab, "failed to fetch OBSS color collision event\n"); + return; + } + + vdev_id = le32_to_cpu(ev->vdev_id); + evt_type = le32_to_cpu(ev->evt_type); + bitmap = le64_to_cpu(ev->obss_color_bitmap); + + guard(rcu)(); + + arvif = ath12k_mac_get_arvif_by_vdev_id(ab, vdev_id); + if (!arvif) { + ath12k_warn(ab, "no arvif found for vdev %u in OBSS color collision event\n", + vdev_id); + return; + } + + switch (evt_type) { + case WMI_BSS_COLOR_COLLISION_DETECTION: + ieee80211_obss_color_collision_notify(arvif->ahvif->vif, + bitmap, + arvif->link_id); + ath12k_dbg(ab, ATH12K_DBG_WMI, + "obss color collision detected vdev %u event %d bitmap %016llx\n", + vdev_id, evt_type, bitmap); + break; + case WMI_BSS_COLOR_COLLISION_DISABLE: + case WMI_BSS_COLOR_FREE_SLOT_TIMER_EXPIRY: + case WMI_BSS_COLOR_FREE_SLOT_AVAILABLE: + break; + default: + ath12k_warn(ab, "unknown OBSS color collision event type %d\n", evt_type); + } +} + static void ath12k_fill_band_to_mac_param(struct ath12k_base *soc, struct ath12k_wmi_pdev_band_arg *arg) @@ -4159,6 +4269,7 @@ int ath12k_wmi_set_hw_mode(struct ath12k_base *ab, int ath12k_wmi_cmd_init(struct ath12k_base *ab) { + struct ath12k_dp *dp = ath12k_ab_to_dp(ab); struct ath12k_wmi_base *wmi_ab = &ab->wmi_ab; struct ath12k_wmi_init_cmd_arg arg = {}; @@ -4179,7 +4290,7 @@ int ath12k_wmi_cmd_init(struct ath12k_base *ab) arg.num_band_to_mac = ab->num_radios; ath12k_fill_band_to_mac_param(ab, arg.band_to_mac); - ab->dp.peer_metadata_ver = arg.res_cfg.peer_metadata_ver; + dp->peer_metadata_ver = arg.res_cfg.peer_metadata_ver; return ath12k_init_cmd_send(&wmi_ab->wmi[0], &arg); } @@ -4487,7 +4598,7 @@ static int ath12k_wmi_hw_mode_caps(struct ath12k_base *soc, pref = soc->wmi_ab.preferred_hw_mode; - if (ath12k_hw_mode_pri_map[mode] < ath12k_hw_mode_pri_map[pref]) { + if (ath12k_hw_mode_pri_map[mode] <= ath12k_hw_mode_pri_map[pref]) { svc_rdy_ext->pref_hw_mode_caps = *hw_mode_caps; soc->wmi_ab.preferred_hw_mode = mode; } @@ -4956,19 +5067,10 @@ ath12k_wmi_tlv_mac_phy_caps_ext_parse(struct ath12k_base *ab, const struct ath12k_wmi_caps_ext_params *caps, struct ath12k_pdev *pdev) { - struct ath12k_band_cap *cap_band; - u32 bands, support_320mhz; + u32 bands; int i; if (ab->hw_params->single_pdev_only) { - if (caps->hw_mode_id == WMI_HOST_HW_MODE_SINGLE) { - support_320mhz = le32_to_cpu(caps->eht_cap_phy_info_5ghz[0]) & - IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; - cap_band = &pdev->cap.band[NL80211_BAND_6GHZ]; - cap_band->eht_cap_phy_info[0] |= support_320mhz; - return 0; - } - for (i = 0; i < ab->fw_pdev_count; i++) { struct ath12k_fw_pdev *fw_pdev = &ab->fw_pdev[i]; @@ -5021,14 +5123,22 @@ static int ath12k_wmi_tlv_mac_phy_caps_ext(struct ath12k_base *ab, u16 tag, void *data) { const struct ath12k_wmi_caps_ext_params *caps = ptr; + struct ath12k_band_cap *cap_band; + u32 support_320mhz; int i = 0, ret; if (tag != WMI_TAG_MAC_PHY_CAPABILITIES_EXT) return -EPROTO; if (ab->hw_params->single_pdev_only) { - if (ab->wmi_ab.preferred_hw_mode != le32_to_cpu(caps->hw_mode_id) && - caps->hw_mode_id != WMI_HOST_HW_MODE_SINGLE) + if (caps->hw_mode_id == WMI_HOST_HW_MODE_SINGLE) { + support_320mhz = le32_to_cpu(caps->eht_cap_phy_info_5ghz[0]) & + IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + cap_band = &ab->pdevs[0].cap.band[NL80211_BAND_6GHZ]; + cap_band->eht_cap_phy_info[0] |= support_320mhz; + } + + if (ab->wmi_ab.preferred_hw_mode != le32_to_cpu(caps->hw_mode_id)) return 0; } else { for (i = 0; i < ab->num_radios; i++) { @@ -5509,6 +5619,10 @@ static int ath12k_wmi_svc_rdy_ext2_parse(struct ath12k_base *ab, ret); return ret; } + + ab->wmi_ab.dp_peer_meta_data_ver = + u32_get_bits(parse->arg.target_cap_flags, + WMI_TARGET_CAP_FLAGS_RX_PEER_METADATA_VERSION); break; case WMI_TAG_ARRAY_STRUCT: @@ -6517,16 +6631,9 @@ static int freq_to_idx(struct ath12k *ar, int freq) if (!sband) continue; - for (ch = 0; ch < sband->n_channels; ch++, idx++) { - if (sband->channels[ch].center_freq < - KHZ_TO_MHZ(ar->freq_range.start_freq) || - sband->channels[ch].center_freq > - KHZ_TO_MHZ(ar->freq_range.end_freq)) - continue; - + for (ch = 0; ch < sband->n_channels; ch++, idx++) if (sband->channels[ch].center_freq == freq) goto exit; - } } exit: @@ -7011,12 +7118,26 @@ static void ath12k_vdev_start_resp_event(struct ath12k_base *ab, struct sk_buff static void ath12k_bcn_tx_status_event(struct ath12k_base *ab, struct sk_buff *skb) { + struct ath12k_link_vif *arvif; + struct ath12k *ar; u32 vdev_id, tx_status; if (ath12k_pull_bcn_tx_status_ev(ab, skb, &vdev_id, &tx_status) != 0) { ath12k_warn(ab, "failed to extract bcn tx status"); return; } + + guard(rcu)(); + + arvif = ath12k_mac_get_arvif_by_vdev_id(ab, vdev_id); + if (!arvif) { + ath12k_warn(ab, "invalid vdev %u in bcn tx status\n", + vdev_id); + return; + } + + ar = arvif->ar; + wiphy_work_queue(ath12k_ar_to_hw(ar)->wiphy, &arvif->bcn_tx_work); } static void ath12k_vdev_stopped_event(struct ath12k_base *ab, struct sk_buff *skb) @@ -7307,8 +7428,8 @@ static void ath12k_peer_sta_kickout_event(struct ath12k_base *ab, struct sk_buff struct wmi_peer_sta_kickout_arg arg = {}; struct ath12k_link_vif *arvif; struct ieee80211_sta *sta; - struct ath12k_peer *peer; - unsigned int link_id; + struct ath12k_sta *ahsta; + struct ath12k_link_sta *arsta; struct ath12k *ar; if (ath12k_pull_peer_sta_kickout_ev(ab, skb, &arg) != 0) { @@ -7320,42 +7441,24 @@ static void ath12k_peer_sta_kickout_event(struct ath12k_base *ab, struct sk_buff spin_lock_bh(&ab->base_lock); - peer = ath12k_peer_find_by_addr(ab, arg.mac_addr); + arsta = ath12k_link_sta_find_by_addr(ab, arg.mac_addr); - if (!peer) { - ath12k_warn(ab, "peer not found %pM\n", + if (!arsta) { + ath12k_warn(ab, "arsta not found %pM\n", arg.mac_addr); goto exit; } - arvif = ath12k_mac_get_arvif_by_vdev_id(ab, peer->vdev_id); + arvif = arsta->arvif; if (!arvif) { - ath12k_warn(ab, "invalid vdev id in peer sta kickout ev %d", - peer->vdev_id); + ath12k_warn(ab, "invalid arvif in peer sta kickout ev for STA %pM", + arg.mac_addr); goto exit; } ar = arvif->ar; - - if (peer->mlo) { - sta = ieee80211_find_sta_by_link_addrs(ath12k_ar_to_hw(ar), - arg.mac_addr, - NULL, &link_id); - if (peer->link_id != link_id) { - ath12k_warn(ab, - "Spurious quick kickout for MLO STA %pM with invalid link_id, peer: %d, sta: %d\n", - arg.mac_addr, peer->link_id, link_id); - goto exit; - } - } else { - sta = ieee80211_find_sta_by_ifaddr(ath12k_ar_to_hw(ar), - arg.mac_addr, NULL); - } - if (!sta) { - ath12k_warn(ab, "Spurious quick kickout for %sSTA %pM\n", - peer->mlo ? "MLO " : "", arg.mac_addr); - goto exit; - } + ahsta = arsta->ahsta; + sta = ath12k_ahsta_to_sta(ahsta); ath12k_dbg(ab, ATH12K_DBG_WMI, "peer sta kickout event %pM reason: %d rssi: %d\n", @@ -8017,8 +8120,6 @@ void ath12k_wmi_fw_stats_dump(struct ath12k *ar, buf[len - 1] = 0; else buf[len] = 0; - - ath12k_fw_stats_reset(ar); } static void @@ -8415,18 +8516,10 @@ static void ath12k_wmi_fw_stats_process(struct ath12k *ar, ath12k_warn(ab, "empty beacon stats"); return; } - /* Mark end until we reached the count of all started VDEVs - * within the PDEV - */ - if (ar->num_started_vdevs) - is_end = ((++ar->fw_stats.num_bcn_recvd) == - ar->num_started_vdevs); list_splice_tail_init(&stats->bcn, &ar->fw_stats.bcn); - - if (is_end) - complete(&ar->fw_stats_done); + complete(&ar->fw_stats_done); } } @@ -9874,6 +9967,9 @@ static void ath12k_wmi_op_rx(struct ath12k_base *ab, struct sk_buff *skb) case WMI_PDEV_RSSI_DBM_CONVERSION_PARAMS_INFO_EVENTID: ath12k_wmi_rssi_dbm_conversion_params_info_event(ab, skb); break; + case WMI_OBSS_COLOR_COLLISION_DETECTION_EVENTID: + ath12k_wmi_obss_color_collision_event(ab, skb); + break; /* add Unsupported events (rare) here */ case WMI_TBTTOFFSET_EXT_UPDATE_EVENTID: case WMI_PEER_OPER_MODE_CHANGE_EVENTID: @@ -9884,7 +9980,6 @@ static void ath12k_wmi_op_rx(struct ath12k_base *ab, struct sk_buff *skb) /* add Unsupported events (frequent) here */ case WMI_PDEV_GET_HALPHY_CAL_STATUS_EVENTID: case WMI_MGMT_RX_FW_CONSUMED_EVENTID: - case WMI_OBSS_COLOR_COLLISION_DETECTION_EVENTID: /* debug might flood hence silently ignore (no-op) */ break; case WMI_PDEV_UTF_EVENTID: diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h index a8c3190e8ad95..0bf0a7941cd3c 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.h +++ b/drivers/net/wireless/ath/ath12k/wmi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef ATH12K_WMI_H @@ -9,6 +9,7 @@ #include #include "htc.h" +#include "cmn_defs.h" /* Naming conventions for structures: * @@ -223,15 +224,15 @@ enum WMI_HOST_WLAN_BAND { }; /* Parameters used for WMI_VDEV_PARAM_AUTORATE_MISC_CFG command. - * Used only for HE auto rate mode. + * Used for HE and EHT auto rate mode. */ enum { - /* HE LTF related configuration */ - WMI_HE_AUTORATE_LTF_1X = BIT(0), - WMI_HE_AUTORATE_LTF_2X = BIT(1), - WMI_HE_AUTORATE_LTF_4X = BIT(2), + /* LTF related configuration */ + WMI_AUTORATE_LTF_1X = BIT(0), + WMI_AUTORATE_LTF_2X = BIT(1), + WMI_AUTORATE_LTF_4X = BIT(2), - /* HE GI related configuration */ + /* GI related configuration */ WMI_AUTORATE_400NS_GI = BIT(8), WMI_AUTORATE_800NS_GI = BIT(9), WMI_AUTORATE_1600NS_GI = BIT(10), @@ -373,6 +374,12 @@ enum wmi_tlv_cmd_id { WMI_PDEV_DMA_RING_CFG_REQ_CMDID, WMI_PDEV_HE_TB_ACTION_FRM_CMDID, WMI_PDEV_PKTLOG_FILTER_CMDID, + WMI_PDEV_SET_SRG_BSS_COLOR_BITMAP_CMDID = 0x403b, + WMI_PDEV_SET_SRG_PARTIAL_BSSID_BITMAP_CMDID, + WMI_PDEV_SET_SRG_OBSS_COLOR_ENABLE_BITMAP_CMDID, + WMI_PDEV_SET_SRG_OBSS_BSSID_ENABLE_BITMAP_CMDID, + WMI_PDEV_SET_NON_SRG_OBSS_COLOR_ENABLE_BITMAP_CMDID, + WMI_PDEV_SET_NON_SRG_OBSS_BSSID_ENABLE_BITMAP_CMDID, WMI_PDEV_SET_BIOS_SAR_TABLE_CMDID = 0x4044, WMI_PDEV_SET_BIOS_GEO_TABLE_CMDID = 0x4045, WMI_PDEV_SET_BIOS_INTERFACE_CMDID = 0x404A, @@ -1075,6 +1082,9 @@ enum wmi_tlv_pdev_param { WMI_PDEV_PARAM_RADIO_CHAN_STATS_ENABLE, WMI_PDEV_PARAM_RADIO_DIAGNOSIS_ENABLE, WMI_PDEV_PARAM_MESH_MCAST_ENABLE, + WMI_PDEV_PARAM_SET_CMD_OBSS_PD_THRESHOLD = 0xbc, + WMI_PDEV_PARAM_SET_CMD_OBSS_PD_PER_AC = 0xbe, + WMI_PDEV_PARAM_ENABLE_SR_PROHIBIT = 0xc6, }; enum wmi_tlv_vdev_param { @@ -1197,6 +1207,7 @@ enum wmi_tlv_vdev_param { WMI_VDEV_PARAM_SET_HEMU_MODE, WMI_VDEV_PARAM_HEOPS_0_31 = 0x8003, WMI_VDEV_PARAM_SET_EHT_MU_MODE = 0x8005, + WMI_VDEV_PARAM_EHT_LTF, }; enum wmi_tlv_peer_flags { @@ -1985,6 +1996,12 @@ enum wmi_tlv_tag { WMI_TAG_SERVICE_READY_EXT2_EVENT = 0x334, WMI_TAG_FILS_DISCOVERY_TMPL_CMD = 0x344, WMI_TAG_MAC_PHY_CAPABILITIES_EXT = 0x36F, + WMI_TAG_PDEV_SRG_BSS_COLOR_BITMAP_CMD = 0x37b, + WMI_TAG_PDEV_SRG_PARTIAL_BSSID_BITMAP_CMD, + WMI_TAG_PDEV_SRG_OBSS_COLOR_ENABLE_BITMAP_CMD = 0x381, + WMI_TAG_PDEV_SRG_OBSS_BSSID_ENABLE_BITMAP_CMD, + WMI_TAG_PDEV_NON_SRG_OBSS_COLOR_ENABLE_BITMAP_CMD, + WMI_TAG_PDEV_NON_SRG_OBSS_BSSID_ENABLE_BITMAP_CMD, WMI_TAG_REGULATORY_RULE_EXT_STRUCT = 0x3A9, WMI_TAG_REG_CHAN_LIST_CC_EXT_EVENT, WMI_TAG_TPC_STATS_GET_CMD = 0x38B, @@ -2242,6 +2259,7 @@ enum wmi_tlv_service { WMI_TLV_SERVICE_FREQINFO_IN_METADATA = 219, WMI_TLV_SERVICE_EXT2_MSG = 220, WMI_TLV_SERVICE_BEACON_PROTECTION_SUPPORT = 244, + WMI_TLV_SERVICE_SRG_SRP_SPATIAL_REUSE_SUPPORT = 249, WMI_TLV_SERVICE_MBSS_PARAM_IN_VDEV_START_SUPPORT = 253, WMI_MAX_EXT_SERVICE = 256, @@ -2781,6 +2799,8 @@ enum wmi_channel_width { #define WMI_EHT_MCS_NSS_10_11 GENMASK(11, 8) #define WMI_EHT_MCS_NSS_12_13 GENMASK(15, 12) +#define WMI_TARGET_CAP_FLAGS_RX_PEER_METADATA_VERSION GENMASK(1, 0) + struct wmi_service_ready_ext2_event { __le32 reg_db_version; __le32 hw_min_max_tx_power_2ghz; @@ -3609,20 +3629,6 @@ struct ath12k_wmi_scan_cancel_arg { u32 pdev_id; }; -struct wmi_bcn_send_from_host_cmd { - __le32 tlv_header; - __le32 vdev_id; - __le32 data_len; - union { - __le32 frag_ptr; - __le32 frag_ptr_lo; - }; - __le32 frame_ctrl; - __le32 dtim_flag; - __le32 bcn_antenna; - __le32 frag_ptr_hi; -}; - #define WMI_CHAN_INFO_MODE GENMASK(5, 0) #define WMI_CHAN_INFO_HT40_PLUS BIT(6) #define WMI_CHAN_INFO_PASSIVE BIT(7) @@ -4218,8 +4224,10 @@ struct wmi_unit_test_cmd { struct ath12k_wmi_vht_rate_set_params { __le32 tlv_header; __le32 rx_max_rate; + /* MCS at which the peer can transmit */ __le32 rx_mcs_set; __le32 tx_max_rate; + /* MCS at which the peer can receive */ __le32 tx_mcs_set; __le32 tx_max_mcs_nss; } __packed; @@ -4933,6 +4941,12 @@ struct wmi_obss_spatial_reuse_params_cmd { __le32 vdev_id; } __packed; +struct wmi_pdev_obss_pd_bitmap_cmd { + __le32 tlv_header; + __le32 pdev_id; + __le32 bitmap[2]; +} __packed; + #define ATH12K_BSS_COLOR_COLLISION_SCAN_PERIOD_MS 200 #define ATH12K_OBSS_COLOR_COLLISION_DETECTION_DISABLE 0 #define ATH12K_OBSS_COLOR_COLLISION_DETECTION 1 @@ -4940,6 +4954,24 @@ struct wmi_obss_spatial_reuse_params_cmd { #define ATH12K_BSS_COLOR_STA_PERIODS 10000 #define ATH12K_BSS_COLOR_AP_PERIODS 5000 +/** + * enum wmi_bss_color_collision - Event types for BSS color collision handling + * @WMI_BSS_COLOR_COLLISION_DISABLE: Indicates that BSS color collision detection + * is disabled. + * @WMI_BSS_COLOR_COLLISION_DETECTION: Event triggered when a BSS color collision + * is detected. + * @WMI_BSS_COLOR_FREE_SLOT_TIMER_EXPIRY: Event indicating that the timer for waiting + * on a free BSS color slot has expired. + * @WMI_BSS_COLOR_FREE_SLOT_AVAILABLE: Event indicating that a free BSS color slot + * has become available. + */ +enum wmi_bss_color_collision { + WMI_BSS_COLOR_COLLISION_DISABLE = 0, + WMI_BSS_COLOR_COLLISION_DETECTION, + WMI_BSS_COLOR_FREE_SLOT_TIMER_EXPIRY, + WMI_BSS_COLOR_FREE_SLOT_AVAILABLE, +}; + struct wmi_obss_color_collision_cfg_params_cmd { __le32 tlv_header; __le32 vdev_id; @@ -4957,6 +4989,12 @@ struct wmi_bss_color_change_enable_params_cmd { __le32 enable; } __packed; +struct wmi_obss_color_collision_event { + __le32 vdev_id; + __le32 evt_type; + __le64 obss_color_bitmap; +} __packed; + #define ATH12K_IPV4_TH_SEED_SIZE 5 #define ATH12K_IPV6_TH_SEED_SIZE 11 @@ -5138,8 +5176,6 @@ struct wmi_probe_tmpl_cmd { __le32 buf_len; } __packed; -#define MAX_RADIOS 2 - #define WMI_MLO_CMD_TIMEOUT_HZ (5 * HZ) #define WMI_SERVICE_READY_TIMEOUT_HZ (5 * HZ) #define WMI_SEND_TIMEOUT_HZ (3 * HZ) @@ -5218,6 +5254,8 @@ struct ath12k_wmi_base { struct ath12k_svc_ext_info svc_ext_info; u32 sbs_lower_band_end_freq; struct ath12k_hw_mode_info hw_mode_info; + + u8 dp_peer_meta_data_ver; }; struct wmi_pdev_set_bios_interface_cmd { @@ -6310,10 +6348,21 @@ struct ath12k_wmi_rssi_dbm_conv_info_arg { s8 min_nf_dbm; }; -void ath12k_wmi_init_qcn9274(struct ath12k_base *ab, - struct ath12k_wmi_resource_config_arg *config); -void ath12k_wmi_init_wcn7850(struct ath12k_base *ab, - struct ath12k_wmi_resource_config_arg *config); +/* each WMI cmd can hold 58 channel entries at most */ +#define ATH12K_WMI_MAX_NUM_CHAN_PER_CMD 58 + +#define ATH12K_OBSS_PD_THRESHOLD_IN_DBM BIT(29) +#define ATH12K_OBSS_PD_SRG_EN BIT(30) +#define ATH12K_OBSS_PD_NON_SRG_EN BIT(31) + +struct ath12k_wmi_obss_pd_arg { + bool srp_support; + bool srg_enabled; + bool non_srg_enabled; + s8 srg_th; + s8 non_srg_th; +}; + int ath12k_wmi_cmd_send(struct ath12k_wmi_pdev *wmi, struct sk_buff *skb, u32 cmd_id); struct sk_buff *ath12k_wmi_alloc_skb(struct ath12k_wmi_base *wmi_sc, u32 len); @@ -6417,6 +6466,19 @@ int ath12k_wmi_send_twt_enable_cmd(struct ath12k *ar, u32 pdev_id); int ath12k_wmi_send_twt_disable_cmd(struct ath12k *ar, u32 pdev_id); int ath12k_wmi_send_obss_spr_cmd(struct ath12k *ar, u32 vdev_id, struct ieee80211_he_obss_pd *he_obss_pd); +u32 ath12k_wmi_build_obss_pd(const struct ath12k_wmi_obss_pd_arg *arg); +int ath12k_wmi_pdev_set_srg_bss_color_bitmap(struct ath12k *ar, u32 pdev_id, + const u32 *bitmap); +int ath12k_wmi_pdev_set_srg_partial_bssid_bitmap(struct ath12k *ar, u32 pdev_id, + const u32 *bitmap); +int ath12k_wmi_pdev_srg_obss_color_enable_bitmap(struct ath12k *ar, u32 pdev_id, + const u32 *bitmap); +int ath12k_wmi_pdev_srg_obss_bssid_enable_bitmap(struct ath12k *ar, u32 pdev_id, + const u32 *bitmap); +int ath12k_wmi_pdev_non_srg_obss_color_enable_bitmap(struct ath12k *ar, u32 pdev_id, + const u32 *bitmap); +int ath12k_wmi_pdev_non_srg_obss_bssid_enable_bitmap(struct ath12k *ar, u32 pdev_id, + const u32 *bitmap); int ath12k_wmi_obss_color_cfg_cmd(struct ath12k *ar, u32 vdev_id, u8 bss_color, u32 period, bool enable); diff --git a/drivers/net/wireless/ath/ath12k/wow.c b/drivers/net/wireless/ath/ath12k/wow.c index dce9bd0bcaefb..bb08e17405825 100644 --- a/drivers/net/wireless/ath/ath12k/wow.c +++ b/drivers/net/wireless/ath/ath12k/wow.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -135,6 +135,9 @@ static int ath12k_wow_cleanup(struct ath12k *ar) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + ret = ath12k_wow_vif_cleanup(arvif); if (ret) { ath12k_warn(ar->ab, "failed to clean wow wakeups on vdev %i: %d\n", @@ -479,8 +482,12 @@ static int ath12k_wow_set_wakeups(struct ath12k *ar, lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (ath12k_wow_is_p2p_vdev(arvif->ahvif)) continue; + ret = ath12k_wow_vif_set_wakeups(arvif, wowlan); if (ret) { ath12k_warn(ar->ab, "failed to set wow wakeups on vdev %i: %d\n", @@ -538,6 +545,9 @@ static int ath12k_wow_nlo_cleanup(struct ath12k *ar) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (ath12k_wow_is_p2p_vdev(arvif->ahvif)) continue; @@ -745,6 +755,9 @@ static int ath12k_wow_arp_ns_offload(struct ath12k *ar, bool enable) list_for_each_entry(arvif, &ar->arvifs, list) { ahvif = arvif->ahvif; + if (arvif != &ahvif->deflink) + continue; + if (ahvif->vdev_type != WMI_VDEV_TYPE_STA) continue; @@ -758,6 +771,7 @@ static int ath12k_wow_arp_ns_offload(struct ath12k *ar, bool enable) if (ret) { ath12k_warn(ar->ab, "failed to set arp ns offload vdev %i: enable %d, ret %d\n", arvif->vdev_id, enable, ret); + kfree(offload); return ret; } } @@ -775,6 +789,9 @@ static int ath12k_gtk_rekey_offload(struct ath12k *ar, bool enable) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (arvif->ahvif->vdev_type != WMI_VDEV_TYPE_STA || !arvif->is_up || !arvif->rekey_data.enable_offload) @@ -918,6 +935,7 @@ int ath12k_wow_op_suspend(struct ieee80211_hw *hw, exit: return ret ? 1 : 0; } +EXPORT_SYMBOL(ath12k_wow_op_suspend); void ath12k_wow_op_set_wakeup(struct ieee80211_hw *hw, bool enabled) { @@ -928,6 +946,7 @@ void ath12k_wow_op_set_wakeup(struct ieee80211_hw *hw, bool enabled) device_set_wakeup_enable(ar->ab->dev, enabled); } +EXPORT_SYMBOL(ath12k_wow_op_set_wakeup); int ath12k_wow_op_resume(struct ieee80211_hw *hw) { @@ -1000,6 +1019,7 @@ int ath12k_wow_op_resume(struct ieee80211_hw *hw) return ret; } +EXPORT_SYMBOL(ath12k_wow_op_resume); int ath12k_wow_init(struct ath12k *ar) { diff --git a/drivers/net/wireless/ath/ath9k/Kconfig b/drivers/net/wireless/ath/ath9k/Kconfig index 0c47be06c153b..47d570a5ca6a1 100644 --- a/drivers/net/wireless/ath/ath9k/Kconfig +++ b/drivers/net/wireless/ath/ath9k/Kconfig @@ -47,7 +47,7 @@ config ATH9K_PCI config ATH9K_AHB bool "Atheros ath9k AHB bus support" - depends on ATH9K + depends on ATH9K && OF default n help This option enables the AHB bus support in ath9k. diff --git a/drivers/net/wireless/ath/ath9k/channel.c b/drivers/net/wireless/ath/ath9k/channel.c index 121e51ce1bc0e..8b27d8cc086ab 100644 --- a/drivers/net/wireless/ath/ath9k/channel.c +++ b/drivers/net/wireless/ath/ath9k/channel.c @@ -1006,7 +1006,7 @@ static void ath_scan_send_probe(struct ath_softc *sc, skb_set_queue_mapping(skb, IEEE80211_AC_VO); if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, NULL)) - goto error; + return; txctl.txq = sc->tx.txq_map[IEEE80211_AC_VO]; if (ath_tx_start(sc->hw, skb, &txctl)) @@ -1119,10 +1119,8 @@ ath_chanctx_send_vif_ps_frame(struct ath_softc *sc, struct ath_vif *avp, skb->priority = 7; skb_set_queue_mapping(skb, IEEE80211_AC_VO); - if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, &sta)) { - dev_kfree_skb_any(skb); + if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, &sta)) return false; - } break; default: return false; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 6a3f187320fc4..13952dfeb3e30 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -951,11 +951,10 @@ int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) goto out; /* try to attach to the target device */ - sdiodev->bus = brcmf_sdio_probe(sdiodev); - if (IS_ERR(sdiodev->bus)) { - ret = PTR_ERR(sdiodev->bus); + ret = brcmf_sdio_probe(sdiodev); + if (ret) goto out; - } + brcmf_sdiod_host_fixup(sdiodev->func2->card->host); out: if (ret) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c index c3a602197662b..abe7f6501e5ed 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c @@ -24,6 +24,10 @@ static const struct brcmf_dmi_data acepc_t8_data = { BRCM_CC_4345_CHIP_ID, 6, "acepc-t8" }; +static const struct brcmf_dmi_data acer_a1_840_data = { + BRCM_CC_43340_CHIP_ID, 2, "acer-a1-840" +}; + /* The Chuwi Hi8 Pro uses the same Ampak AP6212 module as the Chuwi Vi8 Plus * and the nvram for the Vi8 Plus is already in linux-firmware, so use that. */ @@ -91,6 +95,16 @@ static const struct dmi_system_id dmi_platform_data[] = { }, .driver_data = (void *)&acepc_t8_data, }, + { + /* Acer Iconia One 8 A1-840 (non FHD version) */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Insyde"), + DMI_MATCH(DMI_PRODUCT_NAME, "BayTrail"), + /* Above strings are too generic also match BIOS date */ + DMI_MATCH(DMI_BIOS_DATE, "04/01/2014"), + }, + .driver_data = (void *)&acer_a1_840_data, + }, { /* Chuwi Hi8 Pro with D2D3_Hi8Pro.233 BIOS */ .matches = { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index c2d98ee6652f3..1d25dc9ebca8b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -153,6 +153,11 @@ static void brcmf_fweh_handle_if_event(struct brcmf_pub *drvr, bphy_err(drvr, "invalid interface index: %u\n", ifevent->ifidx); return; } + if (ifevent->bsscfgidx >= BRCMF_MAX_IFS) { + bphy_err(drvr, "invalid bsscfg index: %u\n", + ifevent->bsscfgidx); + return; + } ifp = drvr->iflist[ifevent->bsscfgidx]; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 8cf9d7e7c3f70..4e6ed02c15913 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4445,7 +4445,7 @@ brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus) return fwreq; } -struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) { int ret; struct brcmf_sdio *bus; @@ -4551,11 +4551,12 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) goto fail; } - return bus; + return 0; fail: brcmf_sdio_remove(bus); - return ERR_PTR(ret); + sdiodev->bus = NULL; + return ret; } /* Detach and free everything */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 0d18ed15b4032..80180d5c6c879 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -358,7 +358,7 @@ void brcmf_sdiod_freezer_uncount(struct brcmf_sdio_dev *sdiodev); int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev); int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); -struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); +int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); void brcmf_sdio_isr(struct brcmf_sdio *bus, bool in_isr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c index c739bf7463b31..13d0d6b68238d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c @@ -483,7 +483,7 @@ static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size, if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr & boundary)) { *alignbits = dma_align_sizetobits(size); - dma_free_coherent(di->dmadev, size, va, *descpa); + dma_free_coherent(di->dmadev, *alloced, va, *descpa); va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa); } diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c index 24a5624ef2071..3876010c2ce8f 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c @@ -11387,7 +11387,13 @@ static const struct pci_device_id card_ids[] = { {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2754, 0, 0, 0}, {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2761, 0, 0, 0}, {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2762, 0, 0, 0}, - {PCI_VDEVICE(INTEL, 0x104f), 0}, + /* + * This ID conflicts with i40e, but the devices can be differentiated + * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 + * devices use PCI_CLASS_NETWORK_OTHER. + */ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x104f), + PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, 0}, {PCI_VDEVICE(INTEL, 0x4220), 0}, /* BG */ {PCI_VDEVICE(INTEL, 0x4221), 0}, /* BG */ {PCI_VDEVICE(INTEL, 0x4223), 0}, /* ABG */ diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c index 104748fcdc33e..54991f31c52c5 100644 --- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c @@ -3224,7 +3224,9 @@ il3945_store_measurement(struct device *d, struct device_attribute *attr, D_INFO("Invoking measurement of type %d on " "channel %d (for '%s')\n", type, params.channel, buf); + mutex_lock(&il->mutex); il3945_get_measurement(il, ¶ms, type); + mutex_unlock(&il->mutex); return count; } diff --git a/drivers/net/wireless/intel/iwlegacy/4965-mac.c b/drivers/net/wireless/intel/iwlegacy/4965-mac.c index 3588dec75ebdd..57fa866efd9f8 100644 --- a/drivers/net/wireless/intel/iwlegacy/4965-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/4965-mac.c @@ -4606,7 +4606,9 @@ il4965_store_tx_power(struct device *d, struct device_attribute *attr, if (ret) IL_INFO("%s is not in decimal form.\n", buf); else { + mutex_lock(&il->mutex); ret = il_set_tx_power(il, val, false); + mutex_unlock(&il->mutex); if (ret) IL_ERR("failed setting tx power (0x%08x).\n", ret); else diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c index 456a666c8dfdf..fd82050e33a3c 100644 --- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c +++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c @@ -19,6 +19,7 @@ .non_shared_ant = ANT_B, \ .vht_mu_mimo_supported = true, \ .uhb_supported = true, \ + .eht_supported = true, \ .num_rbds = IWL_NUM_RBDS_EHT, \ .nvm_ver = IWL_FM_NVM_VERSION, \ .nvm_type = IWL_NVM_EXT diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c index 97735175cb0ea..b5803ea1eb782 100644 --- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c +++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c @@ -4,8 +4,31 @@ */ #include "iwl-config.h" +/* NVM versions */ +#define IWL_WH_NVM_VERSION 0x0a1d + +#define IWL_DEVICE_WH \ + .ht_params = { \ + .stbc = true, \ + .ldpc = true, \ + .ht40_bands = BIT(NL80211_BAND_2GHZ) | \ + BIT(NL80211_BAND_5GHZ), \ + }, \ + .led_mode = IWL_LED_RF_STATE, \ + .non_shared_ant = ANT_B, \ + .vht_mu_mimo_supported = true, \ + .uhb_supported = true, \ + .num_rbds = IWL_NUM_RBDS_EHT, \ + .nvm_ver = IWL_WH_NVM_VERSION, \ + .nvm_type = IWL_NVM_EXT + /* currently iwl_rf_wh/iwl_rf_wh_160mhz are just defines for the FM ones */ +const struct iwl_rf_cfg iwl_rf_wh_non_eht = { + IWL_DEVICE_WH, + .eht_supported = false, +}; + const char iwl_killer_be1775s_name[] = "Killer(R) Wi-Fi 7 BE1775s 320MHz Wireless Network Adapter (BE211D2W)"; const char iwl_killer_be1775i_name[] = @@ -13,3 +36,4 @@ const char iwl_killer_be1775i_name[] = const char iwl_be211_name[] = "Intel(R) Wi-Fi 7 BE211 320MHz"; const char iwl_be213_name[] = "Intel(R) Wi-Fi 7 BE213 160MHz"; +const char iwl_ax221_name[] = "Intel(R) Wi-Fi 6E AX221 160MHz"; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h index 20bc6671f4eb2..06cece4ea6d95 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h @@ -151,6 +151,7 @@ union acpi_object *iwl_acpi_get_dsm_object(struct device *dev, int rev, * @mcc: output buffer (3 bytes) that will get the MCC * * This function tries to read the current MCC from ACPI if available. + * Return: 0 on success, or a negative error code */ int iwl_acpi_get_mcc(struct iwl_fw_runtime *fwrt, char *mcc); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/cmdhdr.h b/drivers/net/wireless/intel/iwlwifi/fw/api/cmdhdr.h index d130d4f854447..073f003bdc5d2 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/cmdhdr.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/cmdhdr.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2005-2014 Intel Corporation + * Copyright (C) 2005-2014, 2025 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -98,7 +98,7 @@ struct iwl_cmd_header { } __packed; /** - * struct iwl_cmd_header_wide + * struct iwl_cmd_header_wide - wide command header * * This header format appears in the beginning of each command sent from the * driver, and each response/notification received from uCode. diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h b/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h index ddc84430d8959..616f00a8b6034 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * Copyright (C) 2013-2014, 2018-2019 Intel Corporation * Copyright (C) 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2017 Intel Deutschland GmbH @@ -52,7 +52,7 @@ struct iwl_bt_coex_cmd { } __packed; /* BT_COEX_CMD_API_S_VER_6 */ /** - * struct iwl_bt_coex_reduced_txp_update_cmd + * struct iwl_bt_coex_reduced_txp_update_cmd - reduced TX power command * @reduced_txp: bit BT_REDUCED_TX_POWER_BIT to enable / disable, rest of the * bits are the sta_id (value) */ diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h index 997b0c9ce9840..6b3e7c614e54c 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h @@ -296,6 +296,11 @@ enum iwl_legacy_cmds { */ SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6E, + /** + * @SCAN_START_NOTIFICATION_UMAC: uses &struct iwl_umac_scan_start + */ + SCAN_START_NOTIFICATION_UMAC = 0xb2, + /** * @MATCH_FOUND_NOTIFICATION: scan match found */ diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/dbg-tlv.h b/drivers/net/wireless/intel/iwlwifi/fw/api/dbg-tlv.h index 3173fa96cb489..b62f0687327a6 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/dbg-tlv.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/dbg-tlv.h @@ -16,7 +16,7 @@ #define IWL_FW_INI_PRESET_DISABLE 0xff /** - * struct iwl_fw_ini_hcmd + * struct iwl_fw_ini_hcmd - debug configuration host command * * @id: the debug configuration command type for instance: 0xf6 / 0xf5 / DHC * @group: the desired cmd group @@ -199,7 +199,7 @@ struct iwl_fw_ini_region_tlv { } __packed; /* FW_TLV_DEBUG_REGION_API_S_VER_1 */ /** - * struct iwl_fw_ini_debug_info_tlv + * struct iwl_fw_ini_debug_info_tlv - debug info TLV * * debug configuration name for a specific image * @@ -311,7 +311,7 @@ struct iwl_fw_ini_conf_set_tlv { } __packed; /* FW_TLV_DEBUG_CONFIG_SET_API_S_VER_1 */ /** - * enum iwl_fw_ini_config_set_type + * enum iwl_fw_ini_config_set_type - configuration set type * * @IWL_FW_INI_CONFIG_SET_TYPE_INVALID: invalid config set * @IWL_FW_INI_CONFIG_SET_TYPE_DEVICE_PERIPHERY_MAC: for PERIPHERY MAC configuration @@ -337,7 +337,7 @@ enum iwl_fw_ini_config_set_type { } __packed; /** - * enum iwl_fw_ini_allocation_id + * enum iwl_fw_ini_allocation_id - allocation ID * * @IWL_FW_INI_ALLOCATION_INVALID: invalid * @IWL_FW_INI_ALLOCATION_ID_DBGC1: allocation meant for DBGC1 configuration @@ -356,7 +356,7 @@ enum iwl_fw_ini_allocation_id { }; /* FW_DEBUG_TLV_ALLOCATION_ID_E_VER_1 */ /** - * enum iwl_fw_ini_buffer_location + * enum iwl_fw_ini_buffer_location - buffer location * * @IWL_FW_INI_LOCATION_INVALID: invalid * @IWL_FW_INI_LOCATION_SRAM_PATH: SRAM location @@ -373,7 +373,7 @@ enum iwl_fw_ini_buffer_location { }; /* FW_DEBUG_TLV_BUFFER_LOCATION_E_VER_1 */ /** - * enum iwl_fw_ini_region_type + * enum iwl_fw_ini_region_type - region type * * @IWL_FW_INI_REGION_INVALID: invalid * @IWL_FW_INI_REGION_TLV: uCode and debug TLVs @@ -437,7 +437,7 @@ enum iwl_fw_ini_region_device_memory_subtype { }; /* FW_TLV_DEBUG_REGION_DEVICE_MEMORY_SUBTYPE_API_E */ /** - * enum iwl_fw_ini_time_point + * enum iwl_fw_ini_time_point - time point type * * Hard coded time points in which the driver can send hcmd or perform dump * collection diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h index 0cf1e5124fba7..61a850de26fcc 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h @@ -421,7 +421,7 @@ struct iwl_dbgc1_info { } __packed; /* INIT_DRAM_FRAGS_ALLOCATIONS_S_VER_1 */ /** - * struct iwl_dbg_host_event_cfg_cmd + * struct iwl_dbg_host_event_cfg_cmd - host event config command * @enabled_severities: enabled severities */ struct iwl_dbg_host_event_cfg_cmd { diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/location.h b/drivers/net/wireless/intel/iwlwifi/fw/api/location.h index 33541f92c7c73..2ee3a48aa5dfe 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/location.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/location.h @@ -1092,7 +1092,7 @@ struct iwl_tof_range_req_ap_entry { } __packed; /* LOCATION_RANGE_REQ_AP_ENTRY_CMD_API_S_VER_9 */ /** - * enum iwl_tof_response_mode + * enum iwl_tof_response_mode - TOF response mode * @IWL_MVM_TOF_RESPONSE_ASAP: report each AP measurement separately as soon as * possible (not supported for this release) * @IWL_MVM_TOF_RESPONSE_TIMEOUT: report all AP measurements as a batch upon @@ -1108,7 +1108,7 @@ enum iwl_tof_response_mode { }; /** - * enum iwl_tof_initiator_flags + * enum iwl_tof_initiator_flags - TOF initiator flags * * @IWL_TOF_INITIATOR_FLAGS_FAST_ALGO_DISABLED: disable fast algo, meaning run * the algo on ant A+B, instead of only one of them. @@ -1409,7 +1409,7 @@ enum iwl_tof_range_request_status { }; /** - * enum iwl_tof_entry_status + * enum iwl_tof_entry_status - TOF entry status * * @IWL_TOF_ENTRY_SUCCESS: successful measurement. * @IWL_TOF_ENTRY_GENERAL_FAILURE: General failure. @@ -1856,7 +1856,7 @@ struct iwl_tof_mcsi_notif { } __packed; /** - * struct iwl_tof_range_abort_cmd + * struct iwl_tof_range_abort_cmd - TOF range abort command * @request_id: corresponds to a range request * @reserved: reserved */ diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h index 5eb8d10678fd4..535864e226260 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h @@ -620,7 +620,7 @@ struct iwl_sar_offset_mapping_cmd { } __packed; /*SAR_OFFSET_MAPPING_TABLE_CMD_API_S*/ /** - * struct iwl_beacon_filter_cmd + * struct iwl_beacon_filter_cmd - beacon filter command * REPLY_BEACON_FILTERING_CMD = 0xd2 (command) * @bf_energy_delta: Used for RSSI filtering, if in 'normal' state. Send beacon * to driver if delta in Energy values calculated for this and last @@ -762,7 +762,7 @@ enum iwl_6ghz_ap_type { }; /* PHY_AP_TYPE_API_E_VER_1 */ /** - * struct iwl_txpower_constraints_cmd + * struct iwl_txpower_constraints_cmd - TX power constraints command * AP_TX_POWER_CONSTRAINTS_CMD * Used for VLP/LPI/AFC Access Point power constraints for 6GHz channels * @link_id: linkId @@ -786,4 +786,5 @@ struct iwl_txpower_constraints_cmd { __s8 psd_pwr[IWL_MAX_TX_EIRP_PSD_PWR_MAX_SIZE]; u8 reserved[3]; } __packed; /* PHY_AP_TX_POWER_CONSTRAINTS_CMD_API_S_VER_1 */ + #endif /* __iwl_fw_api_power_h__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h index f486d624500be..46fcc32608e34 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2024 Intel Corporation + * Copyright (C) 2012-2014, 2018-2025 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -129,7 +129,7 @@ struct iwl_scan_offload_profile { } __packed; /** - * struct iwl_scan_offload_profile_cfg_data + * struct iwl_scan_offload_profile_cfg_data - scan offload profile configs * @blocklist_len: length of blocklist * @num_profiles: num of profiles in the list * @match_notify: clients waiting for match found notification @@ -159,7 +159,7 @@ struct iwl_scan_offload_profile_cfg_v1 { } __packed; /* SCAN_OFFLOAD_PROFILES_CFG_API_S_VER_1-2*/ /** - * struct iwl_scan_offload_profile_cfg + * struct iwl_scan_offload_profile_cfg - scan offload profile config * @profiles: profiles to search for match * @data: the rest of the data for profile_cfg */ @@ -507,7 +507,7 @@ enum iwl_uhb_chan_cfg_flags { IWL_UHB_CHAN_CFG_FLAG_FORCE_PASSIVE = BIT(26), }; /** - * struct iwl_scan_dwell + * struct iwl_scan_dwell - scan dwell configuration * @active: default dwell time for active scan * @passive: default dwell time for passive scan * @fragmented: default dwell time for fragmented scan @@ -728,7 +728,7 @@ enum iwl_umac_scan_general_params_flags2 { }; /** - * struct iwl_scan_channel_cfg_umac + * struct iwl_scan_channel_cfg_umac - scan channel config * @flags: bitmap - 0-19: directed scan to i'th ssid. * @channel_num: channel number 1-13 etc. * @v1: command version 1 @@ -774,7 +774,7 @@ struct iwl_scan_channel_cfg_umac { } __packed; /** - * struct iwl_scan_umac_schedule + * struct iwl_scan_umac_schedule - scan schedule parameters * @interval: interval in seconds between scan iterations * @iter_count: num of scan iterations for schedule plan, 0xff for infinite loop * @reserved: for alignment and future use @@ -815,7 +815,7 @@ struct iwl_scan_req_umac_tail_v2 { } __packed; /** - * struct iwl_scan_umac_chan_param + * struct iwl_scan_umac_chan_param - scan channel parameters * @flags: channel flags &enum iwl_scan_channel_flags * @count: num of channels in scan request * @reserved: for future use and alignment @@ -827,33 +827,37 @@ struct iwl_scan_umac_chan_param { } __packed; /*SCAN_CHANNEL_PARAMS_API_S_VER_1 */ /** - * struct iwl_scan_req_umac + * struct iwl_scan_req_umac - scan request command * @flags: &enum iwl_umac_scan_flags * @uid: scan id, &enum iwl_umac_scan_uid_offsets * @ooc_priority: out of channel priority - &enum iwl_scan_priority * @general_flags: &enum iwl_umac_scan_general_flags + * @reserved: reserved * @scan_start_mac_id: report the scan start TSF time according to this mac TSF - * @extended_dwell: dwell time for channels 1, 6 and 11 - * @active_dwell: dwell time for active scan per LMAC - * @passive_dwell: dwell time for passive scan per LMAC - * @fragmented_dwell: dwell time for fragmented passive scan - * @adwell_default_n_aps: for adaptive dwell the default number of APs + * @v1: version 1 command data + * @v6: version 6 command data + * @v7: version 7 command data + * @v8: version 8 command data + * @v9: version 9 command data + * @v1.extended_dwell: dwell time for channels 1, 6 and 11 + * @v1.active_dwell: dwell time for active scan per LMAC + * @v1.passive_dwell: dwell time for passive scan per LMAC + * @v1.fragmented_dwell: dwell time for fragmented passive scan + * @v7.adwell_default_n_aps: for adaptive dwell the default number of APs * per channel - * @adwell_default_n_aps_social: for adaptive dwell the default + * @v7.adwell_default_n_aps_social: for adaptive dwell the default * number of APs per social (1,6,11) channel - * @general_flags2: &enum iwl_umac_scan_general_flags2 - * @adwell_max_budget: for adaptive dwell the maximal budget of TU to be added - * to total scan time - * @max_out_time: max out of serving channel time, per LMAC - for CDB there - * are 2 LMACs - * @suspend_time: max suspend time, per LMAC - for CDB there are 2 LMACs - * @scan_priority: scan internal prioritization &enum iwl_scan_priority - * @num_of_fragments: Number of fragments needed for full coverage per band. + * @v8.general_flags2: &enum iwl_umac_scan_general_flags2 + * @v7.adwell_max_budget: for adaptive dwell the maximal budget of TU to be + * added to total scan time + * @v1.max_out_time: max out of serving channel time, per LMAC - for CDB + * there are 2 LMACs + * @v1.suspend_time: max suspend time, per LMAC - for CDB there are 2 LMACs + * @v1.scan_priority: scan internal prioritization &enum iwl_scan_priority + * @v8.num_of_fragments: Number of fragments needed for full coverage per band. * Relevant only for fragmented scan. - * @channel: &struct iwl_scan_umac_chan_param - * @reserved: for future use and alignment - * @reserved3: for future use and alignment - * @data: &struct iwl_scan_channel_cfg_umac and + * @v1.channel: &struct iwl_scan_umac_chan_param + * @v1.data: &struct iwl_scan_channel_cfg_umac and * &struct iwl_scan_req_umac_tail */ struct iwl_scan_req_umac { @@ -939,7 +943,7 @@ struct iwl_scan_req_umac { #define IWL_SCAN_REQ_UMAC_SIZE_V1 36 /** - * struct iwl_scan_probe_params_v3 + * struct iwl_scan_probe_params_v3 - scan probe parameters * @preq: scan probe request params * @ssid_num: number of valid SSIDs in direct scan array * @short_ssid_num: number of valid short SSIDs in short ssid array @@ -961,7 +965,7 @@ struct iwl_scan_probe_params_v3 { } __packed; /* SCAN_PROBE_PARAMS_API_S_VER_3 */ /** - * struct iwl_scan_probe_params_v4 + * struct iwl_scan_probe_params_v4 - scan probe parameters * @preq: scan probe request params * @short_ssid_num: number of valid short SSIDs in short ssid array * @bssid_num: number of valid bssid in bssids array @@ -983,7 +987,7 @@ struct iwl_scan_probe_params_v4 { #define SCAN_MAX_NUM_CHANS_V3 67 /** - * struct iwl_scan_channel_params_v4 + * struct iwl_scan_channel_params_v4 - channel params * @flags: channel flags &enum iwl_scan_channel_flags * @count: num of channels in scan request * @num_of_aps_override: override the number of APs the FW uses to calculate @@ -1006,7 +1010,7 @@ struct iwl_scan_channel_params_v4 { SCAN_CHANNEL_PARAMS_API_S_VER_5 */ /** - * struct iwl_scan_channel_params_v7 + * struct iwl_scan_channel_params_v7 - channel params * @flags: channel flags &enum iwl_scan_channel_flags * @count: num of channels in scan request * @n_aps_override: override the number of APs the FW uses to calculate dwell @@ -1024,7 +1028,7 @@ struct iwl_scan_channel_params_v7 { } __packed; /* SCAN_CHANNEL_PARAMS_API_S_VER_6 */ /** - * struct iwl_scan_general_params_v11 + * struct iwl_scan_general_params_v11 - channel params * @flags: &enum iwl_umac_scan_general_flags_v2 * @reserved: reserved for future * @scan_start_mac_or_link_id: report the scan start TSF time according to this @@ -1066,7 +1070,7 @@ struct iwl_scan_general_params_v11 { } __packed; /* SCAN_GENERAL_PARAMS_API_S_VER_12, *_VER_11 and *_VER_10 */ /** - * struct iwl_scan_periodic_parms_v1 + * struct iwl_scan_periodic_parms_v1 - periodicity parameters * @schedule: can scheduling parameter * @delay: initial delay of the periodic scan in seconds * @reserved: reserved for future @@ -1078,7 +1082,7 @@ struct iwl_scan_periodic_parms_v1 { } __packed; /* SCAN_PERIODIC_PARAMS_API_S_VER_1 */ /** - * struct iwl_scan_req_params_v12 + * struct iwl_scan_req_params_v12 - scan request parameters (v12) * @general_params: &struct iwl_scan_general_params_v11 * @channel_params: &struct iwl_scan_channel_params_v4 * @periodic_params: &struct iwl_scan_periodic_parms_v1 @@ -1106,7 +1110,7 @@ struct iwl_scan_req_params_v17 { } __packed; /* SCAN_REQUEST_PARAMS_API_S_VER_17 - 14 */ /** - * struct iwl_scan_req_umac_v12 + * struct iwl_scan_req_umac_v12 - scan request command (v12) * @uid: scan id, &enum iwl_umac_scan_uid_offsets * @ooc_priority: out of channel priority - &enum iwl_scan_priority * @scan_params: scan parameters @@ -1130,7 +1134,7 @@ struct iwl_scan_req_umac_v17 { } __packed; /* SCAN_REQUEST_CMD_UMAC_API_S_VER_17 - 14 */ /** - * struct iwl_umac_scan_abort + * struct iwl_umac_scan_abort - scan abort command * @uid: scan id, &enum iwl_umac_scan_uid_offsets * @flags: reserved */ @@ -1140,7 +1144,7 @@ struct iwl_umac_scan_abort { } __packed; /* SCAN_ABORT_CMD_UMAC_API_S_VER_1 */ /** - * enum iwl_umac_scan_abort_status + * enum iwl_umac_scan_abort_status - scan abort status * * @IWL_UMAC_SCAN_ABORT_STATUS_SUCCESS: scan was successfully aborted * @IWL_UMAC_SCAN_ABORT_STATUS_IN_PROGRESS: scan abort is in progress @@ -1153,7 +1157,17 @@ enum iwl_umac_scan_abort_status { }; /** - * struct iwl_umac_scan_complete + * struct iwl_umac_scan_start - scan start notification + * @uid: scan id, &enum iwl_umac_scan_uid_offsets + * @reserved: for future use + */ +struct iwl_umac_scan_start { + __le32 uid; + __le32 reserved; +} __packed; /* SCAN_START_UMAC_API_S_VER_1 */ + +/** + * struct iwl_umac_scan_complete - scan complete notification * @uid: scan id, &enum iwl_umac_scan_uid_offsets * @last_schedule: last scheduling line * @last_iter: last scan iteration number diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h b/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h index ecbcd5084cd8d..e6f9abdfa546d 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2021, 2023 Intel Corporation + * Copyright (C) 2012-2014, 2018-2021, 2023, 2025 Intel Corporation * Copyright (C) 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -428,7 +428,7 @@ struct iwl_mvm_rm_sta_cmd { } __packed; /* REMOVE_STA_CMD_API_S_VER_2 */ /** - * struct iwl_mvm_mgmt_mcast_key_cmd_v1 + * struct iwl_mvm_mgmt_mcast_key_cmd_v1 - IGTK command * ( MGMT_MCAST_KEY = 0x1f ) * @ctrl_flags: &enum iwl_sta_key_flag * @igtk: IGTK key material @@ -449,7 +449,7 @@ struct iwl_mvm_mgmt_mcast_key_cmd_v1 { } __packed; /* SEC_MGMT_MULTICAST_KEY_CMD_API_S_VER_1 */ /** - * struct iwl_mvm_mgmt_mcast_key_cmd + * struct iwl_mvm_mgmt_mcast_key_cmd - IGTK command * ( MGMT_MCAST_KEY = 0x1f ) * @ctrl_flags: &enum iwl_sta_key_flag * @igtk: IGTK master key diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h index 00713a9918792..8d9a5058d5a54 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h @@ -26,7 +26,7 @@ struct mvm_statistics_div { } __packed; /* STATISTICS_SLOW_DIV_API_S_VER_2 */ /** - * struct mvm_statistics_rx_non_phy + * struct mvm_statistics_rx_non_phy - non-PHY RX statistics * @bogus_cts: CTS received when not expecting CTS * @bogus_ack: ACK received when not expecting ACK * @non_channel_beacons: beacons with our bss id but not on our serving channel @@ -456,7 +456,7 @@ struct iwl_system_statistics_cmd { } __packed; /* STATISTICS_FW_CMD_API_S_VER_1 */ /** - * enum iwl_fw_statistics_type + * enum iwl_fw_statistics_type - statistics type * * @FW_STATISTICS_OPERATIONAL: operational statistics * @FW_STATISTICS_PHY: phy statistics @@ -478,7 +478,7 @@ enum iwl_fw_statistics_type { #define IWL_STATISTICS_TYPE_MSK 0x7f /** - * struct iwl_statistics_ntfy_hdr + * struct iwl_statistics_ntfy_hdr - statistics notification header * * @type: struct type * @version: version of the struct @@ -491,7 +491,7 @@ struct iwl_statistics_ntfy_hdr { }; /* STATISTICS_NTFY_HDR_API_S_VER_1 */ /** - * struct iwl_stats_ntfy_per_link + * struct iwl_stats_ntfy_per_link - per-link statistics * * @beacon_filter_average_energy: Average energy [-dBm] of the 2 * antennas. @@ -514,7 +514,7 @@ struct iwl_stats_ntfy_per_link { } __packed; /* STATISTICS_NTFY_PER_LINK_API_S_VER_1 */ /** - * struct iwl_stats_ntfy_part1_per_link + * struct iwl_stats_ntfy_part1_per_link - part1 per link statistics * * @rx_time: rx time * @tx_time: tx time @@ -533,7 +533,7 @@ struct iwl_stats_ntfy_part1_per_link { } __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_PART1_PER_LINK_API_S_VER_1 */ /** - * struct iwl_stats_ntfy_per_mac + * struct iwl_stats_ntfy_per_mac - per MAC statistics * * @beacon_filter_average_energy: Average energy [-dBm] of the 2 * antennas. @@ -556,7 +556,8 @@ struct iwl_stats_ntfy_per_mac { } __packed; /* STATISTICS_NTFY_PER_MAC_API_S_VER_1 */ #define IWL_STATS_MAX_BW_INDEX 5 -/** struct iwl_stats_ntfy_per_phy +/** + * struct iwl_stats_ntfy_per_phy - per PHY statistics * @channel_load: channel load * @channel_load_by_us: device contribution to MCLM * @channel_load_not_by_us: other devices' contribution to MCLM @@ -588,7 +589,7 @@ struct iwl_stats_ntfy_per_phy { #define IWL_STATS_UNKNOWN_CHANNEL_LOAD 0xffffffff /** - * struct iwl_stats_ntfy_per_sta + * struct iwl_stats_ntfy_per_sta - per STA statistics * * @average_energy: in fact it is minus the energy.. */ @@ -600,7 +601,7 @@ struct iwl_stats_ntfy_per_sta { #define IWL_STATS_MAX_FW_LINKS (IWL_FW_MAX_LINK_ID + 1) /** - * struct iwl_system_statistics_notif_oper + * struct iwl_system_statistics_notif_oper - statistics notification * * @time_stamp: time when the notification is sent from firmware * @per_link: per link statistics, &struct iwl_stats_ntfy_per_link @@ -615,7 +616,7 @@ struct iwl_system_statistics_notif_oper { } __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_3 */ /** - * struct iwl_system_statistics_part1_notif_oper + * struct iwl_system_statistics_part1_notif_oper - part1 stats notification * * @time_stamp: time when the notification is sent from firmware * @per_link: per link statistics &struct iwl_stats_ntfy_part1_per_link @@ -628,7 +629,7 @@ struct iwl_system_statistics_part1_notif_oper { } __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_PART1_API_S_VER_4 */ /** - * struct iwl_system_statistics_end_notif + * struct iwl_system_statistics_end_notif - statistics end notification * * @time_stamp: time when the notification is sent from firmware */ @@ -637,7 +638,7 @@ struct iwl_system_statistics_end_notif { } __packed; /* STATISTICS_FW_NTFY_END_API_S_VER_1 */ /** - * struct iwl_statistics_operational_ntfy + * struct iwl_statistics_operational_ntfy - operational stats notification * * @hdr: general statistics header * @flags: bitmap of possible notification structures @@ -662,7 +663,7 @@ struct iwl_statistics_operational_ntfy { } __packed; /* STATISTICS_OPERATIONAL_NTFY_API_S_VER_15 */ /** - * struct iwl_statistics_operational_ntfy_ver_14 + * struct iwl_statistics_operational_ntfy_ver_14 - operational stats notification * * @hdr: general statistics header * @flags: bitmap of possible notification structures @@ -707,7 +708,7 @@ struct iwl_statistics_operational_ntfy_ver_14 { } __packed; /* STATISTICS_OPERATIONAL_NTFY_API_S_VER_14 */ /** - * struct iwl_statistics_phy_ntfy + * struct iwl_statistics_phy_ntfy - PHY statistics notification * * @hdr: general statistics header * RX PHY related statistics @@ -808,7 +809,7 @@ struct iwl_statistics_phy_ntfy { } __packed; /* STATISTICS_PHY_NTFY_API_S_VER_1 */ /** - * struct iwl_statistics_mac_ntfy + * struct iwl_statistics_mac_ntfy - MAC statistics notification * * @hdr: general statistics header * @bcast_filter_passed_per_mac: bcast filter passed per mac @@ -827,7 +828,7 @@ struct iwl_statistics_mac_ntfy { } __packed; /* STATISTICS_MAC_NTFY_API_S_VER_1 */ /** - * struct iwl_statistics_rx_ntfy + * struct iwl_statistics_rx_ntfy - RX statistics notification * * @hdr: general statistics header * @rx_agg_mpdu_cnt: aggregation frame count (number of @@ -867,7 +868,7 @@ struct iwl_statistics_rx_ntfy { } __packed; /* STATISTICS_RX_NTFY_API_S_VER_1 */ /** - * struct iwl_statistics_tx_ntfy + * struct iwl_statistics_tx_ntfy - TX statistics notification * * @hdr: general statistics header * @cts_timeout: timeout when waiting for CTS @@ -976,7 +977,7 @@ struct iwl_statistics_tx_ntfy { } __packed; /* STATISTICS_TX_NTFY_API_S_VER_1 */ /** - * struct iwl_statistics_duration_ntfy + * struct iwl_statistics_duration_ntfy - burst/duration statistics * * @hdr: general statistics header * @cont_burst_chk_cnt: number of times continuation or @@ -995,7 +996,7 @@ struct iwl_statistics_duration_ntfy { } __packed; /* STATISTICS_DURATION_NTFY_API_S_VER_1 */ /** - * struct iwl_statistics_he_ntfy + * struct iwl_statistics_he_ntfy - HE statistics * * @hdr: general statistics header * received HE frames diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h index 26d2013905ed9..31d3336726b42 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h @@ -963,7 +963,7 @@ struct iwl_scd_txq_cfg_cmd { } __packed; /* SCD_QUEUE_CFG_CMD_API_S_VER_1 */ /** - * struct iwl_scd_txq_cfg_rsp + * struct iwl_scd_txq_cfg_rsp - scheduler TXQ configuration response * @token: taken from the command * @sta_id: station id from the command * @tid: tid from the command diff --git a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h index c2a73cc85eff4..525a82030daa4 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h @@ -266,7 +266,7 @@ struct iwl_fw_ini_error_dump_data { } __packed; /** - * struct iwl_fw_ini_dump_entry + * struct iwl_fw_ini_dump_entry - dump entry descriptor * @list: list of dump entries * @size: size of the data * @data: entry data @@ -305,7 +305,7 @@ struct iwl_fw_ini_fifo_hdr { * @dram_base_addr: base address of dram monitor range * @page_num: page number of memory range * @fifo_hdr: fifo header of memory range - * @fw_pkt: FW packet header of memory range + * @fw_pkt_hdr: FW packet header of memory range * @data: the actual memory */ struct iwl_fw_ini_error_dump_range { diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h index b7c1ab7a30061..b9e0b69c66803 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/file.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h @@ -222,7 +222,10 @@ typedef unsigned int __bitwise iwl_ucode_tlv_api_t; * @IWL_UCODE_TLV_API_STA_TYPE: This ucode supports station type assignement. * @IWL_UCODE_TLV_API_NAN2_VER2: This ucode supports NAN API version 2 * @IWL_UCODE_TLV_API_ADAPTIVE_DWELL: support for adaptive dwell in scanning + * @IWL_UCODE_TLV_API_OCE: support for OCE + * @IWL_UCODE_TLV_API_NEW_BEACON_TEMPLATE: new beacon template * @IWL_UCODE_TLV_API_NEW_RX_STATS: should new RX STATISTICS API be used + * @IWL_UCODE_TLV_API_WOWLAN_KEY_MATERIAL: WoWLAN key material support * @IWL_UCODE_TLV_API_QUOTA_LOW_LATENCY: Quota command includes a field * indicating low latency direction. * @IWL_UCODE_TLV_API_DEPRECATE_TTAK: RX status flag TTAK ok (bit 7) is @@ -245,6 +248,7 @@ typedef unsigned int __bitwise iwl_ucode_tlv_api_t; * SCAN_OFFLOAD_PROFILES_QUERY_RSP_S. * @IWL_UCODE_TLV_API_MBSSID_HE: This ucode supports v2 of * STA_CONTEXT_DOT11AX_API_S + * @IWL_UCODE_TLV_API_WOWLAN_TCP_SYN_WAKE: WoWLAN TCP-SYN wake support * @IWL_UCODE_TLV_API_FTM_RTT_ACCURACY: version 7 of the range response API * is supported by FW, this indicates the RTT confidence value * @IWL_UCODE_TLV_API_SAR_TABLE_VER: This ucode supports different sar @@ -253,6 +257,7 @@ typedef unsigned int __bitwise iwl_ucode_tlv_api_t; * SCAN_CONFIG_DB_CMD_API_S. * @IWL_UCODE_TLV_API_ADWELL_HB_DEF_N_AP: support for setting adaptive dwell * number of APs in the 5 GHz band + * @IWL_UCODE_TLV_API_SCAN_EXT_CHAN_VER: extended channel config in scan * @IWL_UCODE_TLV_API_BAND_IN_RX_DATA: FW reports band number in RX notification * @IWL_UCODE_TLV_API_NO_HOST_DISABLE_TX: Firmware offloaded the station disable tx * logic. @@ -352,16 +357,24 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t; * @IWL_UCODE_TLV_CAPA_SOC_LATENCY_SUPPORT: the firmware supports setting * stabilization latency for SoCs. * @IWL_UCODE_TLV_CAPA_STA_PM_NOTIF: firmware will send STA PM notification + * @IWL_UCODE_TLV_CAPA_BINDING_CDB_SUPPORT: binding CDB support + * @IWL_UCODE_TLV_CAPA_CDB_SUPPORT: CDB support + * @IWL_UCODE_TLV_CAPA_D0I3_END_FIRST: D0I3 end command comes first * @IWL_UCODE_TLV_CAPA_TLC_OFFLOAD: firmware implements rate scaling algorithm * @IWL_UCODE_TLV_CAPA_DYNAMIC_QUOTA: firmware implements quota related * @IWL_UCODE_TLV_CAPA_COEX_SCHEMA_2: firmware implements Coex Schema 2 - * IWL_UCODE_TLV_CAPA_CHANNEL_SWITCH_CMD: firmware supports CSA command + * @IWL_UCODE_TLV_CAPA_CHANNEL_SWITCH_CMD: firmware supports CSA command * @IWL_UCODE_TLV_CAPA_ULTRA_HB_CHANNELS: firmware supports ultra high band * (6 GHz). * @IWL_UCODE_TLV_CAPA_CS_MODIFY: firmware supports modify action CSA command + * @IWL_UCODE_TLV_CAPA_SET_LTR_GEN2: LTR gen2 support + * @IWL_UCODE_TLV_CAPA_TAS_CFG: TAS configuration support + * @IWL_UCODE_TLV_CAPA_SESSION_PROT_CMD: session protection command + * @IWL_UCODE_TLV_CAPA_SET_PPAG: PPAG support * @IWL_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE: extended DTS measurement * @IWL_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS: supports short PM timeouts * @IWL_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT: supports bt-coex Multi-priority LUT + * @IWL_UCODE_TLV_CAPA_MULTI_QUEUE_RX_SUPPORT: MQ RX support * @IWL_UCODE_TLV_CAPA_CSA_AND_TBTT_OFFLOAD: the firmware supports CSA * countdown offloading. Beacon notifications are not sent to the host. * The fw also offloads TBTT alignment. @@ -383,23 +396,46 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t; * command size (command version 4) that supports toggling ACK TX * power reduction. * @IWL_UCODE_TLV_CAPA_D3_DEBUG: supports debug recording during D3 + * @IWL_UCODE_TLV_CAPA_LED_CMD_SUPPORT: LED command support * @IWL_UCODE_TLV_CAPA_MCC_UPDATE_11AX_SUPPORT: MCC response support 11ax * capability. * @IWL_UCODE_TLV_CAPA_CSI_REPORTING: firmware is capable of being configured * to report the CSI information with (certain) RX frames + * @IWL_UCODE_TLV_CAPA_DBG_SUSPEND_RESUME_CMD_SUPP: suspend/resume command + * @IWL_UCODE_TLV_CAPA_DBG_BUF_ALLOC_CMD_SUPP: support for DBGC + * buffer allocation command * @IWL_UCODE_TLV_CAPA_FTM_CALIBRATED: has FTM calibrated and thus supports both * initiator and responder * @IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_UNII4_US_CA: supports (de)activating UNII-4 * for US/CA/WW from BIOS + * @IWL_UCODE_TLV_CAPA_PSC_CHAN_SUPPORT: supports PSC channels + * @IWL_UCODE_TLV_CAPA_BIGTK_SUPPORT: BIGTK support * @IWL_UCODE_TLV_CAPA_PROTECTED_TWT: Supports protection of TWT action frames * @IWL_UCODE_TLV_CAPA_FW_RESET_HANDSHAKE: Supports the firmware handshake in * reset flow * @IWL_UCODE_TLV_CAPA_PASSIVE_6GHZ_SCAN: Support for passive scan on 6GHz PSC * channels even when these are not enabled. + * @IWL_UCODE_TLV_CAPA_HIDDEN_6GHZ_SCAN: hidden SSID 6 GHz scan support + * @IWL_UCODE_TLV_CAPA_BROADCAST_TWT: broadcast TWT support + * @IWL_UCODE_TLV_CAPA_COEX_HIGH_PRIO: support for BT-coex high + * priority for 802.1X/4-way-HS + * @IWL_UCODE_TLV_CAPA_BAID_ML_SUPPORT: multi-link BAID support + * @IWL_UCODE_TLV_CAPA_SYNCED_TIME: synced time command support + * @IWL_UCODE_TLV_CAPA_TIME_SYNC_BOTH_FTM_TM: time sync support + * @IWL_UCODE_TLV_CAPA_BIGTK_TX_SUPPORT: BIGTK TX support + * @IWL_UCODE_TLV_CAPA_MLD_API_SUPPORT: MLD API support + * @IWL_UCODE_TLV_CAPA_SCAN_DONT_TOGGLE_ANT: fixed antenna scan support + * @IWL_UCODE_TLV_CAPA_PPAG_CHINA_BIOS_SUPPORT: PPAG China BIOS support + * @IWL_UCODE_TLV_CAPA_OFFLOAD_BTM_SUPPORT: BTM protocol offload support + * @IWL_UCODE_TLV_CAPA_STA_EXP_MFP_SUPPORT: STA command MFP support + * @IWL_UCODE_TLV_CAPA_SNIFF_VALIDATE_SUPPORT: sniffer validate bits support + * @IWL_UCODE_TLV_CAPA_CHINA_22_REG_SUPPORT: China 2022 regulator support * @IWL_UCODE_TLV_CAPA_DUMP_COMPLETE_SUPPORT: Support for indicating dump collection * complete to FW. * @IWL_UCODE_TLV_CAPA_SPP_AMSDU_SUPPORT: Support SPP (signaling and payload * protected) A-MSDU. + * @IWL_UCODE_TLV_CAPA_DRAM_FRAG_SUPPORT: support for DBGC fragmented + * DRAM buffers * @IWL_UCODE_TLV_CAPA_SECURE_LTF_SUPPORT: Support secure LTF measurement. * @IWL_UCODE_TLV_CAPA_MONITOR_PASSIVE_CHANS: Support monitor mode on otherwise * passive channels @@ -407,6 +443,8 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t; * for CA from BIOS. * @IWL_UCODE_TLV_CAPA_UHB_CANADA_TAS_SUPPORT: supports %TAS_UHB_ALLOWED_CANADA * @IWL_UCODE_TLV_CAPA_EXT_FSEQ_IMAGE_SUPPORT: external FSEQ image support + * @IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT: FW reset handshake is needed + * during assert handling even if the dump isn't split * @IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE: Firmware has capability of * handling raw DSM table data. * @@ -487,12 +525,7 @@ enum iwl_ucode_tlv_capa { /* set 3 */ IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_UNII4_US_CA = (__force iwl_ucode_tlv_capa_t)96, - - /* - * @IWL_UCODE_TLV_CAPA_PSC_CHAN_SUPPORT: supports PSC channels - */ IWL_UCODE_TLV_CAPA_PSC_CHAN_SUPPORT = (__force iwl_ucode_tlv_capa_t)98, - IWL_UCODE_TLV_CAPA_BIGTK_SUPPORT = (__force iwl_ucode_tlv_capa_t)100, IWL_UCODE_TLV_CAPA_SPP_AMSDU_SUPPORT = (__force iwl_ucode_tlv_capa_t)103, IWL_UCODE_TLV_CAPA_DRAM_FRAG_SUPPORT = (__force iwl_ucode_tlv_capa_t)104, @@ -514,11 +547,8 @@ enum iwl_ucode_tlv_capa { IWL_UCODE_TLV_CAPA_EXT_FSEQ_IMAGE_SUPPORT = (__force iwl_ucode_tlv_capa_t)125, /* set 4 */ - /** - * @IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT: FW reset handshake is needed - * during assert handling even if the dump isn't split - */ - IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT = (__force iwl_ucode_tlv_capa_t)(4 * 32 + 0), + + IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT = (__force iwl_ucode_tlv_capa_t)(4 * 32 + 0), IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE = (__force iwl_ucode_tlv_capa_t)(4 * 32 + 1), NUM_IWL_UCODE_TLV_CAPA /* @@ -852,6 +882,8 @@ struct iwl_fw_dbg_trigger_low_rssi { * @start_assoc_denied: number of denied association to start recording * @start_assoc_timeout: number of association timeout to start recording * @start_connection_loss: number of connection loss to start recording + * @reserved: reserved + * @reserved2: reserved */ struct iwl_fw_dbg_trigger_mlme { u8 stop_auth_denied; @@ -885,6 +917,7 @@ struct iwl_fw_dbg_trigger_mlme { * @p2p_device: timeout for the queues of a P2P device in ms * @ibss: timeout for the queues of an IBSS in ms * @tdls: timeout for the queues of a TDLS station in ms + * @reserved: reserved */ struct iwl_fw_dbg_trigger_txq_timer { __le32 command_queue; @@ -900,7 +933,7 @@ struct iwl_fw_dbg_trigger_txq_timer { /** * struct iwl_fw_dbg_trigger_time_event - configures a time event trigger - * time_Events: a list of tuples . The driver will issue a + * @time_events: a list of tuples . The driver will issue a * trigger each time a time event notification that relates to time event * id with one of the actions in the bitmap is received and * BIT(notif->status) is set in status_bitmap. @@ -916,19 +949,19 @@ struct iwl_fw_dbg_trigger_time_event { /** * struct iwl_fw_dbg_trigger_ba - configures BlockAck related trigger - * rx_ba_start: tid bitmap to configure on what tid the trigger should occur + * @rx_ba_start: tid bitmap to configure on what tid the trigger should occur * when an Rx BlockAck session is started. - * rx_ba_stop: tid bitmap to configure on what tid the trigger should occur + * @rx_ba_stop: tid bitmap to configure on what tid the trigger should occur * when an Rx BlockAck session is stopped. - * tx_ba_start: tid bitmap to configure on what tid the trigger should occur + * @tx_ba_start: tid bitmap to configure on what tid the trigger should occur * when a Tx BlockAck session is started. - * tx_ba_stop: tid bitmap to configure on what tid the trigger should occur + * @tx_ba_stop: tid bitmap to configure on what tid the trigger should occur * when a Tx BlockAck session is stopped. - * rx_bar: tid bitmap to configure on what tid the trigger should occur + * @rx_bar: tid bitmap to configure on what tid the trigger should occur * when a BAR is received (for a Tx BlockAck session). - * tx_bar: tid bitmap to configure on what tid the trigger should occur + * @tx_bar: tid bitmap to configure on what tid the trigger should occur * when a BAR is send (for an Rx BlocAck session). - * frame_timeout: tid bitmap to configure on what tid the trigger should occur + * @frame_timeout: tid bitmap to configure on what tid the trigger should occur * when a frame times out in the reordering buffer. */ struct iwl_fw_dbg_trigger_ba { @@ -946,6 +979,7 @@ struct iwl_fw_dbg_trigger_ba { * @action_bitmap: the TDLS action to trigger the collection upon * @peer_mode: trigger on specific peer or all * @peer: the TDLS peer to trigger the collection on + * @reserved: reserved */ struct iwl_fw_dbg_trigger_tdls { u8 action_bitmap; @@ -958,6 +992,7 @@ struct iwl_fw_dbg_trigger_tdls { * struct iwl_fw_dbg_trigger_tx_status - configures trigger for tx response * status. * @statuses: the list of statuses to trigger the collection on + * @reserved: reserved */ struct iwl_fw_dbg_trigger_tx_status { struct tx_status { @@ -971,6 +1006,7 @@ struct iwl_fw_dbg_trigger_tx_status { * struct iwl_fw_dbg_conf_tlv - a TLV that describes a debug configuration. * @id: conf id * @usniffer: should the uSniffer image be used + * @reserved: reserved * @num_of_hcmds: how many HCMDs to send are present here * @hcmd: a variable length host command to be sent to apply the configuration. * If there is more than one HCMD to send, they will appear one after the diff --git a/drivers/net/wireless/intel/iwlwifi/fw/img.h b/drivers/net/wireless/intel/iwlwifi/fw/img.h index 5256f20623e90..045a3e0094291 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/img.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/img.h @@ -14,14 +14,13 @@ #include "error-dump.h" /** - * enum iwl_ucode_type - * - * The type of ucode. + * enum iwl_ucode_type - type of ucode * * @IWL_UCODE_REGULAR: Normal runtime ucode * @IWL_UCODE_INIT: Initial ucode * @IWL_UCODE_WOWLAN: Wake on Wireless enabled ucode * @IWL_UCODE_REGULAR_USNIFFER: Normal runtime ucode when using usniffer image + * @IWL_UCODE_TYPE_MAX: (internal value) */ enum iwl_ucode_type { IWL_UCODE_REGULAR, @@ -122,7 +121,7 @@ struct fw_img { #define FW_ADDR_CACHE_CONTROL 0xC0000000UL /** - * struct iwl_fw_paging + * struct iwl_fw_paging - FW paging descriptor * @fw_paging_phys: page phy pointer * @fw_paging_block: pointer to the allocated block * @fw_paging_size: page size @@ -197,6 +196,11 @@ struct iwl_dump_exclude { * @dump_excl_wowlan: image dump exclusion areas for WoWLAN image * @pnvm_data: PNVM data embedded in the .ucode file, if any * @pnvm_size: size of the embedded PNVM data + * @dbg: debug data, see &struct iwl_fw_dbg + * @default_calib: default calibration data + * @phy_config: PHY configuration flags + * @valid_rx_ant: valid RX antenna bitmap + * @valid_tx_ant: valid TX antenna bitmap */ struct iwl_fw { u32 ucode_ver; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h index 806f9bcdf4f50..57570ff15622e 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h @@ -45,6 +45,8 @@ struct iwl_fwrt_shared_mem_cfg { * struct iwl_fwrt_dump_data - dump data * @trig: trigger the worker was scheduled upon * @fw_pkt: packet received from FW + * @desc: dump descriptor + * @monitor_only: only dump for monitor * * Note that the decision which part of the union is used * is based on iwl_trans_dbg_ini_valid(): the 'trig' part @@ -68,6 +70,7 @@ struct iwl_fwrt_dump_data { * struct iwl_fwrt_wk_data - dump worker data struct * @idx: index of the worker * @wk: worker + * @dump_data: dump data */ struct iwl_fwrt_wk_data { u8 idx; @@ -91,8 +94,8 @@ struct iwl_txf_iter_data { /** * struct iwl_fw_runtime - runtime data for firmware + * @trans: transport pointer * @fw: firmware image - * @cfg: NIC configuration * @dev: device pointer * @ops: user ops * @ops_ctx: user ops context @@ -117,6 +120,23 @@ struct iwl_txf_iter_data { * zero (default initialization) means it hasn't been read yet, * and BIT(0) is set when it has since function 0 also has this * bitmap and is always supported + * @geo_enabled: WGDS table is present + * @geo_num_profiles: number of geo profiles + * @geo_rev: geo profiles table revision + * @ppag_chains: PPAG table data + * @ppag_flags: PPAG flags + * @reduced_power_flags: reduced power flags + * @sanitize_ctx: context for dump sanitizer + * @sanitize_ops: dump sanitizer ops + * @sar_chain_a_profile: SAR chain A profile + * @sar_chain_b_profile: SAR chain B profile + * @sgom_enabled: SGOM enabled + * @sgom_table: SGOM table + * @timestamp: timestamp marker data + * @timestamp.wk: timestamp marking worker + * @timestamp.seq: timestamp marking sequence + * @timestamp.delay: timestamp marking worker delay + * @tpc_enabled: TPC enabled */ struct iwl_fw_runtime { struct iwl_trans *trans; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/smem.c b/drivers/net/wireless/intel/iwlwifi/fw/smem.c index 90fd69b4860c1..344ddde85b189 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/smem.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/smem.c @@ -6,6 +6,7 @@ */ #include "iwl-drv.h" #include "runtime.h" +#include "dbg.h" #include "fw/api/commands.h" static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, @@ -17,7 +18,9 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, u8 api_ver = iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP, SHARED_MEM_CFG_CMD, 0); - if (WARN_ON(lmac_num > ARRAY_SIZE(mem_cfg->lmac_smem))) + /* Note: notification has 3 entries, but we only expect 2 */ + if (IWL_FW_CHECK(fwrt, lmac_num > ARRAY_SIZE(fwrt->smem_cfg.lmac), + "FW advertises %d LMACs\n", lmac_num)) return; fwrt->smem_cfg.num_lmacs = lmac_num; @@ -26,7 +29,8 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, fwrt->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo2_size); if (api_ver >= 4 && - !WARN_ON_ONCE(iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg))) { + !IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg), + "bad shared mem notification size\n")) { fwrt->smem_cfg.rxfifo2_control_size = le32_to_cpu(mem_cfg->rxfifo2_control_size); } diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h index a607e7ab914ba..3b4f990a8d0bb 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h @@ -385,7 +385,7 @@ struct iwl_mac_cfg { #define IWL_NUM_RBDS_EHT (512 * 8) /** - * struct iwl_rf_cfg + * struct iwl_rf_cfg - RF/CRF configuration data * @fw_name_pre: Firmware filename prefix. The api version and extension * (.ucode) will be added to filename before loading from disk. The * filename is constructed as -.ucode. @@ -418,6 +418,7 @@ struct iwl_mac_cfg { * @vht_mu_mimo_supported: VHT MU-MIMO support * @nvm_type: see &enum iwl_nvm_type * @uhb_supported: ultra high band channels supported + * @eht_supported: EHT supported * @num_rbds: number of receive buffer descriptors to use * (only used for multi-queue capable devices) * @@ -450,7 +451,8 @@ struct iwl_rf_cfg { host_interrupt_operation_mode:1, lp_xtal_workaround:1, vht_mu_mimo_supported:1, - uhb_supported:1; + uhb_supported:1, + eht_supported:1; u8 valid_tx_ant; u8 valid_rx_ant; u8 non_shared_ant; @@ -688,6 +690,7 @@ extern const char iwl_killer_bn1850i_name[]; extern const char iwl_bn201_name[]; extern const char iwl_be221_name[]; extern const char iwl_be223_name[]; +extern const char iwl_ax221_name[]; #if IS_ENABLED(CONFIG_IWLDVM) extern const struct iwl_rf_cfg iwl5300_agn_cfg; extern const struct iwl_rf_cfg iwl5350_agn_cfg; @@ -743,6 +746,7 @@ extern const struct iwl_rf_cfg iwl_rf_fm; extern const struct iwl_rf_cfg iwl_rf_fm_160mhz; #define iwl_rf_wh iwl_rf_fm #define iwl_rf_wh_160mhz iwl_rf_fm_160mhz +extern const struct iwl_rf_cfg iwl_rf_wh_non_eht; #define iwl_rf_pe iwl_rf_fm #endif /* CONFIG_IWLMLD */ diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h index 7ed6329fd8ca1..fe4e46a0edbda 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2023, 2025 Intel Corporation */ #ifndef __iwl_dbg_tlv_h__ #define __iwl_dbg_tlv_h__ @@ -32,7 +32,7 @@ union iwl_dbg_tlv_tp_data { }; /** - * struct iwl_dbg_tlv_time_point_data + * struct iwl_dbg_tlv_time_point_data - debug time point data * @trig_list: list of triggers * @active_trig_list: list of active triggers * @hcmd_list: list of host commands diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c index 607fcea6f4efc..0f002ef261fcc 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c @@ -1599,7 +1599,7 @@ static void _iwl_op_mode_stop(struct iwl_drv *drv) */ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context) { - unsigned int min_core, max_core, loaded_core; + int min_core, max_core, loaded_core; struct iwl_drv *drv = context; struct iwl_fw *fw = &drv->fw; const struct iwl_ucode_header *ucode; @@ -1678,7 +1678,7 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context) if (loaded_core < min_core || loaded_core > max_core) { IWL_ERR(drv, "Driver unable to support your firmware API. " - "Driver supports FW core %u..%u, firmware is %u.\n", + "Driver supports FW core %d..%d, firmware is %d.\n", min_core, max_core, loaded_core); goto try_again; } diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.h b/drivers/net/wireless/intel/iwlwifi/iwl-drv.h index 595300a14639d..a0b67e8aba8da 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.h @@ -62,7 +62,8 @@ struct iwl_rf_cfg; * starts the driver: fetches the firmware. This should be called by bus * specific system flows implementations. For example, the bus specific probe * function should do bus related operations only, and then call to this - * function. It returns the driver object or %NULL if an error occurred. + * function. + * Return: the driver object or %NULL if an error occurred. */ struct iwl_drv *iwl_drv_start(struct iwl_trans *trans); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h b/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h index 21eabfc3ffc84..0476df7b7f179 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2005-2014, 2018-2022, 2024 Intel Corporation + * Copyright (C) 2005-2014, 2018-2022, 2024-2025 Intel Corporation */ #ifndef __iwl_modparams_h__ #define __iwl_modparams_h__ @@ -42,7 +42,7 @@ enum iwl_uapsd_disable { }; /** - * struct iwl_mod_params + * struct iwl_mod_params - module parameters for iwlwifi * * Holds the module parameters * diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c index 23465e4c4b399..e021fc57d85d2 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c @@ -2080,7 +2080,7 @@ struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans, !!(mac_flags & NVM_MAC_SKU_FLAGS_BAND_5_2_ENABLED); nvm->sku_cap_mimo_disabled = !!(mac_flags & NVM_MAC_SKU_FLAGS_MIMO_DISABLED); - if (CSR_HW_RFID_TYPE(trans->info.hw_rf_id) >= IWL_CFG_RF_TYPE_FM) + if (trans->cfg->eht_supported) nvm->sku_cap_11be_enable = true; /* Initialize PHY sku data */ diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h index cbc92abf9f87a..12f28bb0e859d 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h @@ -115,11 +115,12 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg, * iwl_parse_nvm_mcc_info - parse MCC (mobile country code) info coming from FW * * This function parses the regulatory channel data received as a - * MCC_UPDATE_CMD command. It returns a newly allocation regulatory domain, - * to be fed into the regulatory core. In case the geo_info is set handle - * accordingly. An ERR_PTR is returned on error. - * If not given to the regulatory core, the user is responsible for freeing - * the regdomain returned here with kfree. + * MCC_UPDATE_CMD command. + * + * Return: a newly allocation regulatory domain, to be given to the regulatory + * core. In case the geo_info is set handle accordingly. An ERR_PTR is + * returned on error. If not given to the regulatory core, the user is + * responsible for freeing the regdomain returned here with kfree(). * * @trans: the transport * @num_of_ch: the number of channels @@ -140,6 +141,8 @@ iwl_parse_nvm_mcc_info(struct iwl_trans *trans, * This struct holds an NVM section read from the NIC using NVM_ACCESS_CMD, * and saved for later use by the driver. Not all NVM sections are saved * this way, only the needed ones. + * @length: length of the section + * @data: section data */ struct iwl_nvm_section { u16 length; @@ -148,6 +151,10 @@ struct iwl_nvm_section { /** * iwl_read_external_nvm - Reads external NVM from a file into nvm_sections + * @trans: the transport + * @nvm_file_name: the filename to request + * @nvm_sections: sections data to fill + * Return: 0 on success or an error code */ int iwl_read_external_nvm(struct iwl_trans *trans, const char *nvm_file_name, diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h b/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h index a146d0e399f22..df6341dfc4a12 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h @@ -185,6 +185,7 @@ void iwl_opmode_deregister(const char *name); /** * struct iwl_op_mode - operational mode * @ops: pointer to its own ops + * @op_mode_specific: per-opmode data * * This holds an implementation of the mac80211 / fw API. */ diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h index a0cc5d7745e8f..a552669db6e2a 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h @@ -121,7 +121,7 @@ enum CMD_MODE { #define DEF_CMD_PAYLOAD_SIZE 320 /** - * struct iwl_device_cmd + * struct iwl_device_cmd - device command structure * * For allocation of the command and tx queues, this establishes the overall * size of the largest command we send to uCode, except for commands that @@ -516,7 +516,7 @@ enum iwl_trans_state { */ /** - * enum iwl_ini_cfg_state + * enum iwl_ini_cfg_state - debug config state * @IWL_INI_CFG_STATE_NOT_LOADED: no debug cfg was given * @IWL_INI_CFG_STATE_LOADED: debug cfg was found and loaded * @IWL_INI_CFG_STATE_CORRUPTED: debug cfg was found and some of the TLVs @@ -532,7 +532,7 @@ enum iwl_ini_cfg_state { #define IWL_TRANS_NMI_TIMEOUT (HZ / 4) /** - * struct iwl_dram_data + * struct iwl_dram_data - DRAM data descriptor * @physical: page phy pointer * @block: pointer to the allocated block/page * @size: size of the block/page diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.c b/drivers/net/wireless/intel/iwlwifi/mld/d3.c index 1d4282a21f09e..dd85be94433cc 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/d3.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.c @@ -1794,6 +1794,10 @@ iwl_mld_send_proto_offload(struct iwl_mld *mld, u32 enabled = 0; cmd = kzalloc(hcmd.len[0], GFP_KERNEL); + if (!cmd) { + IWL_DEBUG_WOWLAN(mld, "Failed to allocate proto offload cmd\n"); + return -ENOMEM; + } #if IS_ENABLED(CONFIG_IPV6) struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.c b/drivers/net/wireless/intel/iwlwifi/mld/iface.c index ed379825a9236..80bcd18930c57 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/iface.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.c @@ -55,8 +55,6 @@ void iwl_mld_cleanup_vif(void *data, u8 *mac, struct ieee80211_vif *vif) ieee80211_iter_keys(mld->hw, vif, iwl_mld_cleanup_keys_iter, NULL); - wiphy_delayed_work_cancel(mld->wiphy, &mld_vif->mlo_scan_start_wk); - CLEANUP_STRUCT(mld_vif); } @@ -113,14 +111,75 @@ static bool iwl_mld_is_nic_ack_enabled(struct iwl_mld *mld, IEEE80211_HE_MAC_CAP2_ACK_EN); } -static void iwl_mld_set_he_support(struct iwl_mld *mld, - struct ieee80211_vif *vif, - struct iwl_mac_config_cmd *cmd) +struct iwl_mld_mac_wifi_gen_sta_iter_data { + struct ieee80211_vif *vif; + struct iwl_mac_wifi_gen_support *support; +}; + +static void iwl_mld_mac_wifi_gen_sta_iter(void *_data, + struct ieee80211_sta *sta) +{ + struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta); + struct iwl_mld_mac_wifi_gen_sta_iter_data *data = _data; + struct ieee80211_link_sta *link_sta; + unsigned int link_id; + + if (mld_sta->vif != data->vif) + return; + + for_each_sta_active_link(data->vif, sta, link_sta, link_id) { + if (link_sta->he_cap.has_he) + data->support->he_support = 1; + if (link_sta->eht_cap.has_eht) + data->support->eht_support = 1; + } +} + +static void iwl_mld_set_wifi_gen(struct iwl_mld *mld, + struct ieee80211_vif *vif, + struct iwl_mac_wifi_gen_support *support) { - if (vif->type == NL80211_IFTYPE_AP) - cmd->wifi_gen.he_ap_support = 1; - else - cmd->wifi_gen.he_support = 1; + struct iwl_mld_mac_wifi_gen_sta_iter_data sta_iter_data = { + .vif = vif, + .support = support, + }; + struct ieee80211_bss_conf *link_conf; + unsigned int link_id; + + switch (vif->type) { + case NL80211_IFTYPE_MONITOR: + /* for sniffer, set to HW capabilities */ + support->he_support = 1; + support->eht_support = mld->trans->cfg->eht_supported; + break; + case NL80211_IFTYPE_AP: + /* for AP set according to the link configs */ + for_each_vif_active_link(vif, link_conf, link_id) { + support->he_ap_support |= link_conf->he_support; + support->eht_support |= link_conf->eht_support; + } + break; + default: + /* + * If we have MLO enabled, then the firmware needs to enable + * address translation for the station(s) we add. That depends + * on having EHT enabled in firmware, which in turn depends on + * mac80211 in the iteration below. + * However, mac80211 doesn't enable capabilities on the AP STA + * until it has parsed the association response successfully, + * so set EHT (and HE as a pre-requisite for EHT) when the vif + * is an MLD. + */ + if (ieee80211_vif_is_mld(vif)) { + support->he_support = 1; + support->eht_support = 1; + } + + ieee80211_iterate_stations_mtx(mld->hw, + iwl_mld_mac_wifi_gen_sta_iter, + &sta_iter_data); + break; + } } /* fill the common part for all interface types */ @@ -130,8 +189,6 @@ static void iwl_mld_mac_cmd_fill_common(struct iwl_mld *mld, u32 action) { struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); - struct ieee80211_bss_conf *link_conf; - unsigned int link_id; lockdep_assert_wiphy(mld->wiphy); @@ -149,29 +206,7 @@ static void iwl_mld_mac_cmd_fill_common(struct iwl_mld *mld, cmd->nic_not_ack_enabled = cpu_to_le32(!iwl_mld_is_nic_ack_enabled(mld, vif)); - /* If we have MLO enabled, then the firmware needs to enable - * address translation for the station(s) we add. That depends - * on having EHT enabled in firmware, which in turn depends on - * mac80211 in the code below. - * However, mac80211 doesn't enable HE/EHT until it has parsed - * the association response successfully, so just skip all that - * and enable both when we have MLO. - */ - if (ieee80211_vif_is_mld(vif)) { - iwl_mld_set_he_support(mld, vif, cmd); - cmd->wifi_gen.eht_support = 1; - return; - } - - for_each_vif_active_link(vif, link_conf, link_id) { - if (!link_conf->he_support) - continue; - - iwl_mld_set_he_support(mld, vif, cmd); - - /* EHT, if supported, was already set above */ - break; - } + iwl_mld_set_wifi_gen(mld, vif, &cmd->wifi_gen); } static void iwl_mld_fill_mac_cmd_sta(struct iwl_mld *mld, diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c index 5725104a53bf0..67b61765adf39 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c @@ -980,7 +980,9 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, { struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw); struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link); - unsigned int n_active = iwl_mld_count_active_links(mld, vif); + struct iwl_mld_link *temp_mld_link; + struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); + u16 final_active_links = 0; int ret; lockdep_assert_wiphy(mld->wiphy); @@ -988,10 +990,7 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, if (WARN_ON(!mld_link)) return -EINVAL; - /* if the assigned one was not counted yet, count it now */ if (!rcu_access_pointer(mld_link->chan_ctx)) { - n_active++; - /* Track addition of non-BSS link */ if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION) { ret = iwl_mld_emlsr_check_non_bss_block(mld, 1); @@ -1012,17 +1011,25 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, rcu_assign_pointer(mld_link->chan_ctx, ctx); - if (n_active > 1) { - struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); + /* We cannot rely on vif->active_links at this stage as it contains + * both the removed links and the newly added links. + * Therefore, we create our own bitmap of the final active links, + * which does not include the removed links. + */ + for_each_mld_vif_valid_link(mld_vif, temp_mld_link) { + if (rcu_access_pointer(temp_mld_link->chan_ctx)) + final_active_links |= BIT(link_id); + } + if (hweight16(final_active_links) > 1) { /* Indicate to mac80211 that EML is enabled */ vif->driver_flags |= IEEE80211_VIF_EML_ACTIVE; mld_vif->emlsr.last_entry_ts = jiffies; - if (vif->active_links & BIT(mld_vif->emlsr.selected_links)) + if (final_active_links == mld_vif->emlsr.selected_links) mld_vif->emlsr.primary = mld_vif->emlsr.selected_primary; else - mld_vif->emlsr.primary = __ffs(vif->active_links); + mld_vif->emlsr.primary = __ffs(final_active_links); iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_ESR_LINK_UP, NULL); @@ -1679,6 +1686,16 @@ static int iwl_mld_move_sta_state_up(struct iwl_mld *mld, if (vif->type == NL80211_IFTYPE_STATION) iwl_mld_link_set_2mhz_block(mld, vif, sta); + + if (sta->tdls) { + /* + * update MAC since wifi generation flags may change, + * we also update MAC on association to the AP via the + * vif assoc change + */ + iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY); + } + /* Now the link_sta's capabilities are set, update the FW */ iwl_mld_config_tlc(mld, vif, sta); @@ -1755,6 +1772,8 @@ static int iwl_mld_move_sta_state_down(struct iwl_mld *mld, wiphy_work_cancel(mld->wiphy, &mld_vif->emlsr.unblock_tpt_wk); wiphy_delayed_work_cancel(mld->wiphy, &mld_vif->emlsr.check_tpt_wk); + wiphy_delayed_work_cancel(mld->wiphy, + &mld_vif->mlo_scan_start_wk); iwl_mld_reset_cca_40mhz_workaround(mld, vif); iwl_mld_smps_workaround(mld, vif, true); @@ -1786,6 +1805,15 @@ static int iwl_mld_move_sta_state_down(struct iwl_mld *mld, /* just removed last TDLS STA, so enable PM */ iwl_mld_update_mac_power(mld, vif, false); } + + if (sta->tdls) { + /* + * update MAC since wifi generation flags may change, + * we also update MAC on disassociation to the AP via + * the vif assoc change + */ + iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY); + } } else { return -EINVAL; } diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.c b/drivers/net/wireless/intel/iwlwifi/mld/mld.c index a6962256bdd12..3cfe1bcb7d4e5 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mld.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.c @@ -171,6 +171,7 @@ static const struct iwl_hcmd_names iwl_mld_legacy_names[] = { HCMD_NAME(MISSED_BEACONS_NOTIFICATION), HCMD_NAME(MAC_PM_POWER_TABLE), HCMD_NAME(MFUART_LOAD_NOTIFICATION), + HCMD_NAME(SCAN_START_NOTIFICATION_UMAC), HCMD_NAME(RSS_CONFIG_CMD), HCMD_NAME(SCAN_ITERATION_COMPLETE_UMAC), HCMD_NAME(REPLY_RX_MPDU_CMD), diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c index 241a6271d13d6..836f15dc54b52 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c @@ -732,7 +732,7 @@ iwl_mld_set_link_sel_data(struct iwl_mld *mld, /* Ignore any BSS that was not seen in the last MLO scan */ if (ktime_before(link_conf->bss->ts_boottime, - mld->scan.last_mlo_scan_time)) + mld->scan.last_mlo_scan_start_time)) continue; data[n_data].link_id = link_id; @@ -851,9 +851,9 @@ iwl_mld_emlsr_pair_state(struct ieee80211_vif *vif, if (c_low->chan->center_freq > c_high->chan->center_freq) swap(c_low, c_high); - c_low_upper_edge = c_low->chan->center_freq + + c_low_upper_edge = c_low->center_freq1 + cfg80211_chandef_get_width(c_low) / 2; - c_high_lower_edge = c_high->chan->center_freq - + c_high_lower_edge = c_high->center_freq1 - cfg80211_chandef_get_width(c_high) / 2; if (a->chandef->chan->band == NL80211_BAND_5GHZ && @@ -939,7 +939,7 @@ static void _iwl_mld_select_links(struct iwl_mld *mld, if (!mld_vif->authorized || hweight16(usable_links) <= 1) return; - if (WARN(ktime_before(mld->scan.last_mlo_scan_time, + if (WARN(ktime_before(mld->scan.last_mlo_scan_start_time, ktime_sub_ns(ktime_get_boottime_ns(), 5ULL * NSEC_PER_SEC)), "Last MLO scan was too long ago, can't select links\n")) diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.c b/drivers/net/wireless/intel/iwlwifi/mld/notif.c index 884973d0b3440..a3fd0b7387d69 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/notif.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.c @@ -284,6 +284,8 @@ static void iwl_mld_handle_beacon_notification(struct iwl_mld *mld, * at least enough bytes to cover the structure listed in the CMD_VER_ENTRY. */ +CMD_VERSIONS(scan_start_notif, + CMD_VER_ENTRY(1, iwl_umac_scan_start)) CMD_VERSIONS(scan_complete_notif, CMD_VER_ENTRY(1, iwl_umac_scan_complete)) CMD_VERSIONS(scan_iter_complete_notif, @@ -355,6 +357,7 @@ DEFINE_SIMPLE_CANCELLATION(datapath_monitor, iwl_datapath_monitor_notif, link_id) DEFINE_SIMPLE_CANCELLATION(roc, iwl_roc_notif, activity) DEFINE_SIMPLE_CANCELLATION(scan_complete, iwl_umac_scan_complete, uid) +DEFINE_SIMPLE_CANCELLATION(scan_start, iwl_umac_scan_start, uid) DEFINE_SIMPLE_CANCELLATION(probe_resp_data, iwl_probe_resp_data_notif, mac_id) DEFINE_SIMPLE_CANCELLATION(uapsd_misbehaving_ap, iwl_uapsd_misbehaving_ap_notif, @@ -397,6 +400,8 @@ const struct iwl_rx_handler iwl_mld_rx_handlers[] = { RX_HANDLER_SYNC) RX_HANDLER_NO_OBJECT(LEGACY_GROUP, BA_NOTIF, compressed_ba_notif, RX_HANDLER_SYNC) + RX_HANDLER_OF_SCAN(LEGACY_GROUP, SCAN_START_NOTIFICATION_UMAC, + scan_start_notif) RX_HANDLER_OF_SCAN(LEGACY_GROUP, SCAN_COMPLETE_UMAC, scan_complete_notif) RX_HANDLER_NO_OBJECT(LEGACY_GROUP, SCAN_ITERATION_COMPLETE_UMAC, diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c index ffeb37a7f830e..231920425c066 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c @@ -121,6 +121,12 @@ static int iwl_mld_ptp_gettime(struct ptp_clock_info *ptp, return 0; } +static int iwl_mld_ptp_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + return -EOPNOTSUPP; +} + static int iwl_mld_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) { struct iwl_mld *mld = container_of(ptp, struct iwl_mld, @@ -279,6 +285,7 @@ void iwl_mld_ptp_init(struct iwl_mld *mld) mld->ptp_data.ptp_clock_info.owner = THIS_MODULE; mld->ptp_data.ptp_clock_info.gettime64 = iwl_mld_ptp_gettime; + mld->ptp_data.ptp_clock_info.settime64 = iwl_mld_ptp_settime; mld->ptp_data.ptp_clock_info.max_adj = 0x7fffffff; mld->ptp_data.ptp_clock_info.adjtime = iwl_mld_ptp_adjtime; mld->ptp_data.ptp_clock_info.adjfine = iwl_mld_ptp_adjfine; diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.c b/drivers/net/wireless/intel/iwlwifi/mld/scan.c index fd1022ddc9122..76ac6fd5f9ff3 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/scan.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.c @@ -473,6 +473,9 @@ iwl_mld_scan_get_cmd_gen_flags(struct iwl_mld *mld, params->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_TRIGGER_UHB_SCAN; + if (scan_status == IWL_MLD_SCAN_INT_MLO) + flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_NTF_START; + if (params->enable_6ghz_passive) flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_6GHZ_PASSIVE_SCAN; @@ -1817,9 +1820,6 @@ static void iwl_mld_int_mlo_scan_start(struct iwl_mld *mld, ret = _iwl_mld_single_scan_start(mld, vif, req, &ies, IWL_MLD_SCAN_INT_MLO); - if (!ret) - mld->scan.last_mlo_scan_time = ktime_get_boottime_ns(); - IWL_DEBUG_SCAN(mld, "Internal MLO scan: ret=%d\n", ret); } @@ -1904,6 +1904,30 @@ void iwl_mld_handle_match_found_notif(struct iwl_mld *mld, ieee80211_sched_scan_results(mld->hw); } +void iwl_mld_handle_scan_start_notif(struct iwl_mld *mld, + struct iwl_rx_packet *pkt) +{ + struct iwl_umac_scan_complete *notif = (void *)pkt->data; + u32 uid = le32_to_cpu(notif->uid); + + if (IWL_FW_CHECK(mld, uid >= ARRAY_SIZE(mld->scan.uid_status), + "FW reports out-of-range scan UID %d\n", uid)) + return; + + if (IWL_FW_CHECK(mld, !(mld->scan.uid_status[uid] & mld->scan.status), + "FW reports scan UID %d we didn't trigger\n", uid)) + return; + + IWL_DEBUG_SCAN(mld, "Scan started: uid=%u type=%u\n", uid, + mld->scan.uid_status[uid]); + if (IWL_FW_CHECK(mld, mld->scan.uid_status[uid] != IWL_MLD_SCAN_INT_MLO, + "FW reports scan start notification %d we didn't trigger\n", + mld->scan.uid_status[uid])) + return; + + mld->scan.last_mlo_scan_start_time = ktime_get_boottime_ns(); +} + void iwl_mld_handle_scan_complete_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt) { diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.h b/drivers/net/wireless/intel/iwlwifi/mld/scan.h index 69110f0cfc8e2..de5620e7f463b 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/scan.h +++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.h @@ -27,6 +27,9 @@ int iwl_mld_sched_scan_start(struct iwl_mld *mld, void iwl_mld_handle_match_found_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt); +void iwl_mld_handle_scan_start_notif(struct iwl_mld *mld, + struct iwl_rx_packet *pkt); + void iwl_mld_handle_scan_complete_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt); @@ -114,8 +117,8 @@ enum iwl_mld_traffic_load { * in jiffies. * @last_start_time_jiffies: stores the last start time in jiffies * (interface up/reset/resume). - * @last_mlo_scan_time: start time of the last MLO scan in nanoseconds since - * boot. + * @last_mlo_scan_start_time: start time of the last MLO scan in nanoseconds + * since boot. */ struct iwl_mld_scan { /* Add here fields that need clean up on restart */ @@ -136,7 +139,7 @@ struct iwl_mld_scan { void *cmd; unsigned long last_6ghz_passive_jiffies; unsigned long last_start_time_jiffies; - u64 last_mlo_scan_time; + u64 last_mlo_scan_start_time; }; /** diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c index 3b4b575aadaa5..e3fb4fc4f452e 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c @@ -345,6 +345,11 @@ u8 iwl_mld_get_lowest_rate(struct iwl_mld *mld, iwl_mld_get_basic_rates_and_band(mld, vif, info, &basic_rates, &band); + if (band >= NUM_NL80211_BANDS) { + WARN_ON(vif->type != NL80211_IFTYPE_NAN); + return IWL_FIRST_OFDM_RATE; + } + sband = mld->hw->wiphy->bands[band]; for_each_set_bit(i, &basic_rates, BITS_PER_LONG) { u16 hw = sband->bitrates[i].hw_value; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c index 07f1a84c274ef..11afe373961f3 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2012-2014, 2018-2025 Intel Corporation + * Copyright (C) 2012-2014, 2018-2026 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -2834,7 +2834,7 @@ static void iwl_mvm_nd_match_info_handler(struct iwl_mvm *mvm, if (IS_ERR_OR_NULL(vif)) return; - if (len < sizeof(struct iwl_scan_offload_match_info)) { + if (len < sizeof(struct iwl_scan_offload_match_info) + matches_len) { IWL_ERR(mvm, "Invalid scan match info notification\n"); return; } @@ -3239,6 +3239,8 @@ void iwl_mvm_fast_suspend(struct iwl_mvm *mvm) IWL_DEBUG_WOWLAN(mvm, "Starting fast suspend flow\n"); + iwl_mvm_pause_tcm(mvm, true); + mvm->fast_resume = true; set_bit(IWL_MVM_STATUS_IN_D3, &mvm->status); @@ -3295,6 +3297,8 @@ int iwl_mvm_fast_resume(struct iwl_mvm *mvm) mvm->trans->state = IWL_TRANS_NO_FW; } + iwl_mvm_resume_tcm(mvm); + out: clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status); mvm->fast_resume = false; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c index 865f973f677db..aa517978fc7a3 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c @@ -479,7 +479,8 @@ static void iwl_mvm_uats_init(struct iwl_mvm *mvm) .dataflags[0] = IWL_HCMD_DFL_NOCOPY, }; - if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) { + if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210 || + !mvm->trans->cfg->uhb_supported) { IWL_DEBUG_RADIO(mvm, "UATS feature is not supported\n"); return; } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c index 867807abde664..49ffc4ecee855 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c @@ -1761,6 +1761,20 @@ void iwl_mvm_probe_resp_data_notif(struct iwl_mvm *mvm, mvmvif = iwl_mvm_vif_from_mac80211(vif); + /* + * len_low should be 2 + n*13 (where n is the number of descriptors. + * 13 is the size of a NoA descriptor). We can have either one or two + * descriptors. + */ + if (IWL_FW_CHECK(mvm, notif->noa_active && + notif->noa_attr.len_low != 2 + + sizeof(struct ieee80211_p2p_noa_desc) && + notif->noa_attr.len_low != 2 + + sizeof(struct ieee80211_p2p_noa_desc) * 2, + "Invalid noa_attr.len_low (%d)\n", + notif->noa_attr.len_low)) + return; + new_data = kzalloc(sizeof(*new_data), GFP_KERNEL); if (!new_data) return; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c index 06a4c9f74797a..ad156b82eaa94 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c @@ -220,6 +220,12 @@ static int iwl_mvm_ptp_gettime(struct ptp_clock_info *ptp, return 0; } +static int iwl_mvm_ptp_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + return -EOPNOTSUPP; +} + static int iwl_mvm_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) { struct iwl_mvm *mvm = container_of(ptp, struct iwl_mvm, @@ -281,6 +287,7 @@ void iwl_mvm_ptp_init(struct iwl_mvm *mvm) mvm->ptp_data.ptp_clock_info.adjfine = iwl_mvm_ptp_adjfine; mvm->ptp_data.ptp_clock_info.adjtime = iwl_mvm_ptp_adjtime; mvm->ptp_data.ptp_clock_info.gettime64 = iwl_mvm_ptp_gettime; + mvm->ptp_data.ptp_clock_info.settime64 = iwl_mvm_ptp_settime; mvm->ptp_data.scaled_freq = SCALE_FACTOR; /* Give a short 'friendly name' to identify the PHC clock */ diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index b21a4d8eb1051..73001cdce13aa 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -1061,11 +1061,16 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info iwl_dev_info_table[] = { /* WH RF */ IWL_DEV_INFO(iwl_rf_wh, iwl_be211_name, RF_TYPE(WH)), + IWL_DEV_INFO(iwl_rf_wh_non_eht, iwl_ax221_name, RF_TYPE(WH), + SUBDEV(0x0514)), + IWL_DEV_INFO(iwl_rf_wh_non_eht, iwl_ax221_name, RF_TYPE(WH), + SUBDEV(0x4514)), IWL_DEV_INFO(iwl_rf_wh_160mhz, iwl_be213_name, RF_TYPE(WH), BW_LIMITED), /* PE RF */ IWL_DEV_INFO(iwl_rf_pe, iwl_bn201_name, RF_TYPE(PE)), IWL_DEV_INFO(iwl_rf_pe, iwl_be223_name, RF_TYPE(PE), SUBDEV(0x0524)), + IWL_DEV_INFO(iwl_rf_pe, iwl_be223_name, RF_TYPE(PE), SUBDEV(0x4524)), IWL_DEV_INFO(iwl_rf_pe, iwl_be221_name, RF_TYPE(PE), SUBDEV(0x0324)), /* Killer */ diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c index b3c4040257a67..924ab93b7b671 100644 --- a/drivers/net/wireless/marvell/libertas/if_usb.c +++ b/drivers/net/wireless/marvell/libertas/if_usb.c @@ -426,6 +426,8 @@ static int usb_tx_block(struct if_usb_card *cardp, uint8_t *payload, uint16_t nb goto tx_ret; } + usb_kill_urb(cardp->tx_urb); + usb_fill_bulk_urb(cardp->tx_urb, cardp->udev, usb_sndbulkpipe(cardp->udev, cardp->ep_out), diff --git a/drivers/net/wireless/marvell/libertas/main.c b/drivers/net/wireless/marvell/libertas/main.c index d44e02c6fe385..dd97f1b61f4d1 100644 --- a/drivers/net/wireless/marvell/libertas/main.c +++ b/drivers/net/wireless/marvell/libertas/main.c @@ -799,8 +799,8 @@ static void lbs_free_adapter(struct lbs_private *priv) { lbs_free_cmd_buffer(priv); kfifo_free(&priv->event_fifo); - timer_delete(&priv->command_timer); - timer_delete(&priv->tx_lockup_timer); + timer_delete_sync(&priv->command_timer); + timer_delete_sync(&priv->tx_lockup_timer); } static const struct net_device_ops lbs_netdev_ops = { diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c index 354c5ce660455..f3397dc6c422e 100644 --- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c +++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c @@ -825,7 +825,7 @@ void mwifiex_update_rxreor_flags(struct mwifiex_adapter *adapter, u8 flags) static void mwifiex_update_ampdu_rxwinsize(struct mwifiex_adapter *adapter, bool coex_flag) { - u8 i; + u8 i, j; u32 rx_win_size; struct mwifiex_private *priv; @@ -863,8 +863,8 @@ static void mwifiex_update_ampdu_rxwinsize(struct mwifiex_adapter *adapter, if (rx_win_size != priv->add_ba_param.rx_win_size) { if (!priv->media_connected) continue; - for (i = 0; i < MAX_NUM_TID; i++) - mwifiex_11n_delba(priv, i); + for (j = 0; j < MAX_NUM_TID; j++) + mwifiex_11n_delba(priv, j); } } } diff --git a/drivers/net/wireless/mediatek/mt76/eeprom.c b/drivers/net/wireless/mediatek/mt76/eeprom.c index a987c5e4eff6c..6ce8e4af18fe5 100644 --- a/drivers/net/wireless/mediatek/mt76/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/eeprom.c @@ -253,6 +253,19 @@ mt76_get_of_array(struct device_node *np, char *name, size_t *len, int min) return prop->value; } +static const s8 * +mt76_get_of_array_s8(struct device_node *np, char *name, size_t *len, int min) +{ + struct property *prop = of_find_property(np, name, NULL); + + if (!prop || !prop->value || prop->length < min) + return NULL; + + *len = prop->length; + + return prop->value; +} + struct device_node * mt76_find_channel_node(struct device_node *np, struct ieee80211_channel *chan) { @@ -294,7 +307,7 @@ mt76_get_txs_delta(struct device_node *np, u8 nss) } static void -mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const __be32 *data, +mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const s8 *data, s8 target_power, s8 nss_delta, s8 *max_power) { int i; @@ -303,15 +316,14 @@ mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const __be32 *data, return; for (i = 0; i < pwr_len; i++) { - pwr[i] = min_t(s8, target_power, - be32_to_cpu(data[i]) + nss_delta); + pwr[i] = min_t(s8, target_power, data[i] + nss_delta); *max_power = max(*max_power, pwr[i]); } } static void mt76_apply_multi_array_limit(s8 *pwr, size_t pwr_len, s8 pwr_num, - const __be32 *data, size_t len, s8 target_power, + const s8 *data, size_t len, s8 target_power, s8 nss_delta, s8 *max_power) { int i, cur; @@ -319,8 +331,7 @@ mt76_apply_multi_array_limit(s8 *pwr, size_t pwr_len, s8 pwr_num, if (!data) return; - len /= 4; - cur = be32_to_cpu(data[0]); + cur = data[0]; for (i = 0; i < pwr_num; i++) { if (len < pwr_len + 1) break; @@ -335,7 +346,7 @@ mt76_apply_multi_array_limit(s8 *pwr, size_t pwr_len, s8 pwr_num, if (!len) break; - cur = be32_to_cpu(data[0]); + cur = data[0]; } } @@ -346,7 +357,7 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, { struct mt76_dev *dev = phy->dev; struct device_node *np; - const __be32 *val; + const s8 *val; char name[16]; u32 mcs_rates = dev->drv->mcs_rates; u32 ru_rates = ARRAY_SIZE(dest->ru[0]); @@ -392,21 +403,21 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, txs_delta = mt76_get_txs_delta(np, hweight16(phy->chainmask)); - val = mt76_get_of_array(np, "rates-cck", &len, ARRAY_SIZE(dest->cck)); + val = mt76_get_of_array_s8(np, "rates-cck", &len, ARRAY_SIZE(dest->cck)); mt76_apply_array_limit(dest->cck, ARRAY_SIZE(dest->cck), val, target_power, txs_delta, &max_power); - val = mt76_get_of_array(np, "rates-ofdm", - &len, ARRAY_SIZE(dest->ofdm)); + val = mt76_get_of_array_s8(np, "rates-ofdm", + &len, ARRAY_SIZE(dest->ofdm)); mt76_apply_array_limit(dest->ofdm, ARRAY_SIZE(dest->ofdm), val, target_power, txs_delta, &max_power); - val = mt76_get_of_array(np, "rates-mcs", &len, mcs_rates + 1); + val = mt76_get_of_array_s8(np, "rates-mcs", &len, mcs_rates + 1); mt76_apply_multi_array_limit(dest->mcs[0], ARRAY_SIZE(dest->mcs[0]), ARRAY_SIZE(dest->mcs), val, len, target_power, txs_delta, &max_power); - val = mt76_get_of_array(np, "rates-ru", &len, ru_rates + 1); + val = mt76_get_of_array_s8(np, "rates-ru", &len, ru_rates + 1); mt76_apply_multi_array_limit(dest->ru[0], ARRAY_SIZE(dest->ru[0]), ARRAY_SIZE(dest->ru), val, len, target_power, txs_delta, &max_power); diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 5ceaf78c9ea06..5e75861bf6f99 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -847,8 +847,6 @@ void mt76_reset_device(struct mt76_dev *dev) } rcu_read_unlock(); - mt76_abort_scan(dev); - INIT_LIST_HEAD(&dev->wcid_list); INIT_LIST_HEAD(&dev->sta_poll_list); dev->vif_mask = 0; diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index e0d50b58cd012..7753afa3d883d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -1252,6 +1252,15 @@ static inline int mt76_wed_dma_setup(struct mt76_dev *dev, struct mt76_queue *q, #define mt76_dereference(p, dev) \ rcu_dereference_protected(p, lockdep_is_held(&(dev)->mutex)) +static inline struct mt76_dev *mt76_wed_to_dev(struct mtk_wed_device *wed) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + if (wed->wlan.hif2) + return container_of(wed, struct mt76_dev, mmio.wed_hif2); +#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ + return container_of(wed, struct mt76_dev, mmio.wed); +} + static inline struct mt76_wcid * __mt76_wcid_ptr(struct mt76_dev *dev, u16 idx) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c index 4064e193d4dec..08ee2e861c4e2 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c @@ -874,8 +874,10 @@ mt7615_mcu_wtbl_sta_add(struct mt7615_phy *phy, struct ieee80211_vif *vif, wtbl_hdr = mt76_connac_mcu_alloc_wtbl_req(&dev->mt76, &msta->wcid, WTBL_RESET_AND_SET, NULL, &wskb); - if (IS_ERR(wtbl_hdr)) + if (IS_ERR(wtbl_hdr)) { + dev_kfree_skb(sskb); return PTR_ERR(wtbl_hdr); + } if (enable) { mt76_connac_mcu_wtbl_generic_tlv(&dev->mt76, wskb, vif, sta, diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index 0db00efe88b0b..837bd0f136fa1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -411,6 +411,7 @@ mt76_connac2_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 + 1 + 2 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) { u16 capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index 1c0d310146d63..5caf818e82834 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -1451,6 +1451,8 @@ mt7915_mac_full_reset(struct mt7915_dev *dev) if (ext_phy) cancel_delayed_work_sync(&ext_phy->mac_work); + mt76_abort_scan(&dev->mt76); + mutex_lock(&dev->mt76.mutex); for (i = 0; i < 10; i++) { if (!mt7915_mac_restart(dev)) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index 86bd33b916a9d..edc1df3c071e5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -646,10 +646,10 @@ int mt7921_run_firmware(struct mt792x_dev *dev) if (err) return err; - set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); err = mt7921_load_clc(dev, mt792x_ram_name(dev)); if (err) return err; + set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); return mt7921_mcu_fw_log_2_host(dev, 1); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 1e44e96f034e9..e880f3820a1ad 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -667,6 +667,7 @@ mt7925_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) tid = MT_TX_ADDBA; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index 8eda407e4135e..c12b71b71cfc7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -1003,10 +1003,10 @@ int mt7925_run_firmware(struct mt792x_dev *dev) if (err) return err; - set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); err = mt7925_load_clc(dev, mt792x_ram_name(dev)); if (err) return err; + set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); return mt7925_mcu_fw_log_2_host(dev, 1); } diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_core.c b/drivers/net/wireless/mediatek/mt76/mt792x_core.c index c0e56541a9547..9cad572c34a38 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_core.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_core.c @@ -688,7 +688,6 @@ int mt792x_init_wiphy(struct ieee80211_hw *hw) ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS); ieee80211_hw_set(hw, SUPPORTS_VHT_EXT_NSS_BW); ieee80211_hw_set(hw, CONNECTION_MONITOR); - ieee80211_hw_set(hw, NO_VIRTUAL_MONITOR); ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); ieee80211_hw_set(hw, SUPPORTS_ONLY_HE_MULTI_BSSID); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c index 659015f93d323..7ed2f21b0e6db 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c @@ -512,12 +512,15 @@ int mt7996_dma_rro_init(struct mt7996_dev *dev) if (ret) return ret; - /* We need to set cpu idx pointer before resetting the EMI - * queues. - */ - mdev->q_rx[MT_RXQ_RRO_RXDMAD_C].emi_cpu_idx = - &dev->wed_rro.emi_rings_cpu.ptr->ring[0].idx; - mt76_queue_reset(dev, &mdev->q_rx[MT_RXQ_RRO_RXDMAD_C], true); + if (!mtk_wed_device_active(&mdev->mmio.wed)) { + /* We need to set cpu idx pointer before resetting the + * EMI queues. + */ + mdev->q_rx[MT_RXQ_RRO_RXDMAD_C].emi_cpu_idx = + &dev->wed_rro.emi_rings_cpu.ptr->ring[0].idx; + mt76_queue_reset(dev, &mdev->q_rx[MT_RXQ_RRO_RXDMAD_C], + true); + } goto start_hw_rro; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index 5e95a36b42d16..b136b9a669769 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -959,9 +959,10 @@ static int mt7996_wed_rro_init(struct mt7996_dev *dev) MT7996_RRO_MSDU_PG_SIZE_PER_CR); } - if (dev->mt76.hwrro_mode == MT76_HWRRO_V3_1) { + if (!mtk_wed_device_active(&dev->mt76.mmio.wed) && + dev->mt76.hwrro_mode == MT76_HWRRO_V3_1) { ptr = dmam_alloc_coherent(dev->mt76.dma_dev, - sizeof(dev->wed_rro.emi_rings_cpu.ptr), + sizeof(*dev->wed_rro.emi_rings_cpu.ptr), &dev->wed_rro.emi_rings_cpu.phy_addr, GFP_KERNEL); if (!ptr) @@ -970,7 +971,7 @@ static int mt7996_wed_rro_init(struct mt7996_dev *dev) dev->wed_rro.emi_rings_cpu.ptr = ptr; ptr = dmam_alloc_coherent(dev->mt76.dma_dev, - sizeof(dev->wed_rro.emi_rings_dma.ptr), + sizeof(*dev->wed_rro.emi_rings_dma.ptr), &dev->wed_rro.emi_rings_dma.phy_addr, GFP_KERNEL); if (!ptr) @@ -1036,6 +1037,18 @@ static void mt7996_wed_rro_free(struct mt7996_dev *dev) dev->wed_rro.msdu_pg[i].phy_addr); } + if (dev->wed_rro.emi_rings_cpu.ptr) + dmam_free_coherent(dev->mt76.dma_dev, + sizeof(*dev->wed_rro.emi_rings_cpu.ptr), + dev->wed_rro.emi_rings_cpu.ptr, + dev->wed_rro.emi_rings_cpu.phy_addr); + + if (dev->wed_rro.emi_rings_dma.ptr) + dmam_free_coherent(dev->mt76.dma_dev, + sizeof(*dev->wed_rro.emi_rings_dma.ptr), + dev->wed_rro.emi_rings_dma.ptr, + dev->wed_rro.emi_rings_dma.phy_addr); + if (!dev->wed_rro.session.ptr) return; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 9501def3e0e3e..0958961d2758e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -794,10 +794,12 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; __le16 fc = hdr->frame_control, sc = hdr->seq_ctrl; u16 seqno = le16_to_cpu(sc); + bool hw_bigtk = false; u8 fc_type, fc_stype; u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) { if (is_mt7990(&dev->mt76)) @@ -819,7 +821,11 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, info->flags & IEEE80211_TX_CTL_USE_MINRATE) val |= MT_TXD1_FIXED_RATE; - if (key && multicast && ieee80211_is_robust_mgmt_frame(skb)) { + if (is_mt7990(&dev->mt76) && ieee80211_is_beacon(fc) && + (wcid->hw_key_idx2 == 6 || wcid->hw_key_idx2 == 7)) + hw_bigtk = true; + + if ((key && multicast && ieee80211_is_robust_mgmt_frame(skb)) || hw_bigtk) { val |= MT_TXD1_BIP; txwi[3] &= ~cpu_to_le32(MT_TXD3_PROTECT_FRAME); } @@ -1681,8 +1687,7 @@ mt7996_msdu_page_get_from_cache(struct mt7996_dev *dev) if (!list_empty(&dev->wed_rro.page_cache)) { p = list_first_entry(&dev->wed_rro.page_cache, struct mt7996_msdu_page, list); - if (p) - list_del(&p->list); + list_del(&p->list); } spin_unlock(&dev->wed_rro.lock); @@ -2337,7 +2342,7 @@ mt7996_mac_restart(struct mt7996_dev *dev) if (!test_bit(MT76_STATE_RUNNING, &phy->mt76->state)) continue; - ret = mt7996_run(&dev->phy); + ret = mt7996_run(phy); if (ret) goto out; } @@ -2420,6 +2425,8 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) mt7996_for_each_phy(dev, phy) cancel_delayed_work_sync(&phy->mt76->mac_work); + mt76_abort_scan(&dev->mt76); + mutex_lock(&dev->mt76.mutex); for (i = 0; i < 10; i++) { if (!mt7996_mac_restart(dev)) @@ -2854,6 +2861,8 @@ void mt7996_mac_sta_rc_work(struct work_struct *work) LIST_HEAD(list); u32 changed; + mutex_lock(&dev->mt76.mutex); + spin_lock_bh(&dev->mt76.sta_poll_lock); list_splice_init(&dev->sta_rc_list, &list); @@ -2886,6 +2895,8 @@ void mt7996_mac_sta_rc_work(struct work_struct *work) } spin_unlock_bh(&dev->mt76.sta_poll_lock); + + mutex_unlock(&dev->mt76.mutex); } void mt7996_mac_work(struct work_struct *work) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index 581314368c5ba..2ad52ae2c5f55 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -90,9 +90,11 @@ static void mt7996_stop(struct ieee80211_hw *hw, bool suspend) { } -static inline int get_free_idx(u32 mask, u8 start, u8 end) +static inline int get_free_idx(u64 mask, u8 start, u8 end) { - return ffs(~mask & GENMASK(end, start)); + if (~mask & GENMASK_ULL(end, start)) + return __ffs64(~mask & GENMASK_ULL(end, start)) + 1; + return 0; } static int get_omac_idx(enum nl80211_iftype type, u64 mask) @@ -247,12 +249,13 @@ mt7996_set_hw_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, else if (idx == *wcid_keyidx) *wcid_keyidx = -1; - if (cmd != SET_KEY && sta) + /* only do remove key for BIGTK */ + if (cmd != SET_KEY && !is_bigtk) return 0; mt76_wcid_key_setup(&dev->mt76, &msta_link->wcid, key); - err = mt7996_mcu_add_key(&dev->mt76, vif, key, + err = mt7996_mcu_add_key(&dev->mt76, link, key, MCU_WMWA_UNI_CMD(STA_REC_UPDATE), &msta_link->wcid, cmd); @@ -308,12 +311,6 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, if (idx < 0) return -ENOSPC; - if (!dev->mld_idx_mask) { /* first link in the group */ - mvif->mld_group_idx = get_own_mld_idx(dev->mld_idx_mask, true); - mvif->mld_remap_idx = get_free_idx(dev->mld_remap_idx_mask, - 0, 15); - } - mld_idx = get_own_mld_idx(dev->mld_idx_mask, false); if (mld_idx < 0) return -ENOSPC; @@ -331,10 +328,6 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, return ret; dev->mt76.vif_mask |= BIT_ULL(mlink->idx); - if (!dev->mld_idx_mask) { - dev->mld_idx_mask |= BIT_ULL(mvif->mld_group_idx); - dev->mld_remap_idx_mask |= BIT_ULL(mvif->mld_remap_idx); - } dev->mld_idx_mask |= BIT_ULL(link->mld_idx); phy->omac_mask |= BIT_ULL(mlink->omac_idx); @@ -343,6 +336,7 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, INIT_LIST_HEAD(&msta_link->rc_list); msta_link->wcid.idx = idx; msta_link->wcid.link_id = link_conf->link_id; + msta_link->wcid.link_valid = ieee80211_vif_is_mld(vif); msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET; mt76_wcid_init(&msta_link->wcid, band_idx); @@ -376,7 +370,8 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); - if (mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) + if (!mlink->wcid->offchannel && + mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) mvif->mt76.deflink_id = link_conf->link_id; return 0; @@ -397,7 +392,8 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif, }; int idx = msta_link->wcid.idx; - ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); + if (!mlink->wcid->offchannel) + ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL, CONN_STATE_DISCONNECT, false); @@ -407,7 +403,8 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif, rcu_assign_pointer(dev->mt76.wcid[idx], NULL); - if (mvif->mt76.deflink_id == link_conf->link_id) { + if (!mlink->wcid->offchannel && + mvif->mt76.deflink_id == link_conf->link_id) { struct ieee80211_bss_conf *iter; unsigned int link_id; @@ -423,11 +420,6 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif, dev->mt76.vif_mask &= ~BIT_ULL(mlink->idx); dev->mld_idx_mask &= ~BIT_ULL(link->mld_idx); phy->omac_mask &= ~BIT_ULL(mlink->omac_idx); - if (!(dev->mld_idx_mask & ~BIT_ULL(mvif->mld_group_idx))) { - /* last link */ - dev->mld_idx_mask &= ~BIT_ULL(mvif->mld_group_idx); - dev->mld_remap_idx_mask &= ~BIT_ULL(mvif->mld_remap_idx); - } spin_lock_bh(&dev->mt76.sta_poll_lock); if (!list_empty(&msta_link->wcid.poll_list)) @@ -665,8 +657,8 @@ mt7996_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, unsigned int link_id, u16 queue, const struct ieee80211_tx_queue_params *params) { - struct mt7996_dev *dev = mt7996_hw_dev(hw); - struct mt7996_vif_link *mlink = mt7996_vif_link(dev, vif, link_id); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_vif_link_info *link_info = &mvif->link_info[link_id]; static const u8 mq_to_aci[] = { [IEEE80211_AC_VO] = 3, [IEEE80211_AC_VI] = 2, @@ -675,7 +667,7 @@ mt7996_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, }; /* firmware uses access class index */ - mlink->queue_params[mq_to_aci[queue]] = *params; + link_info->queue_params[mq_to_aci[queue]] = *params; /* no need to update right away, we'll get BSS_CHANGED_QOS */ return 0; @@ -984,6 +976,7 @@ mt7996_mac_sta_init_link(struct mt7996_dev *dev, msta_link->wcid.sta = 1; msta_link->wcid.idx = idx; msta_link->wcid.link_id = link_id; + msta_link->wcid.link_valid = !!sta->valid_links; msta_link->wcid.def_wcid = &msta->deflink.wcid; ewma_avg_signal_init(&msta_link->avg_ack_signal); @@ -1160,12 +1153,15 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, unsigned long links = sta->valid_links; struct ieee80211_link_sta *link_sta; unsigned int link_id; + int err = 0; + + mutex_lock(&dev->mt76.mutex); for_each_sta_active_link(vif, sta, link_sta, link_id) { struct ieee80211_bss_conf *link_conf; struct mt7996_sta_link *msta_link; struct mt7996_vif_link *link; - int i, err; + int i; link_conf = link_conf_dereference_protected(vif, link_id); if (!link_conf) @@ -1185,12 +1181,12 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, link, msta_link, CONN_STATE_CONNECT, true); if (err) - return err; + goto unlock; err = mt7996_mcu_add_rate_ctrl(dev, msta_link->sta, vif, link_id, false); if (err) - return err; + goto unlock; msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET; break; @@ -1199,28 +1195,30 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, link, msta_link, CONN_STATE_PORT_SECURE, false); if (err) - return err; + goto unlock; break; case MT76_STA_EVENT_DISASSOC: for (i = 0; i < ARRAY_SIZE(msta_link->twt.flow); i++) mt7996_mac_twt_teardown_flow(dev, link, msta_link, i); - if (sta->mlo && links == BIT(link_id)) /* last link */ - mt7996_mcu_teardown_mld_sta(dev, link, - msta_link); - else + if (!sta->mlo) mt7996_mcu_add_sta(dev, link_conf, link_sta, link, msta_link, CONN_STATE_DISCONNECT, false); + else if (sta->mlo && links == BIT(link_id)) /* last link */ + mt7996_mcu_teardown_mld_sta(dev, link, + msta_link); msta_link->wcid.sta_disabled = 1; msta_link->wcid.sta = 0; links = links & ~BIT(link_id); break; } } +unlock: + mutex_unlock(&dev->mt76.mutex); - return 0; + return err; } static void @@ -1339,12 +1337,10 @@ static void mt7996_tx(struct ieee80211_hw *hw, } if (mvif) { - struct mt76_vif_link *mlink = &mvif->deflink.mt76; - - if (link_id < IEEE80211_LINK_UNSPECIFIED) - mlink = rcu_dereference(mvif->mt76.link[link_id]); + struct mt76_vif_link *mlink; - if (mlink->wcid) + mlink = rcu_dereference(mvif->mt76.link[link_id]); + if (mlink && mlink->wcid) wcid = mlink->wcid; if (mvif->mt76.roc_phy && @@ -1352,7 +1348,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, mphy = mvif->mt76.roc_phy; if (mphy->roc_link) wcid = mphy->roc_link->wcid; - } else { + } else if (mlink) { mphy = mt76_vif_link_phy(mlink); } } @@ -1362,7 +1358,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, goto unlock; } - if (msta && link_id < IEEE80211_LINK_UNSPECIFIED) { + if (msta) { struct mt7996_sta_link *msta_link; msta_link = rcu_dereference(msta->link[link_id]); @@ -2217,7 +2213,42 @@ mt7996_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, u16 old_links, u16 new_links, struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]) { - return 0; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + int ret = 0; + + mutex_lock(&dev->mt76.mutex); + + if (!old_links) { + int idx; + + idx = get_own_mld_idx(dev->mld_idx_mask, true); + if (idx < 0) { + ret = -ENOSPC; + goto out; + } + mvif->mld_group_idx = idx; + dev->mld_idx_mask |= BIT_ULL(mvif->mld_group_idx); + + idx = get_free_idx(dev->mld_remap_idx_mask, 0, 15) - 1; + if (idx < 0) { + ret = -ENOSPC; + goto out; + } + mvif->mld_remap_idx = idx; + dev->mld_remap_idx_mask |= BIT_ULL(mvif->mld_remap_idx); + } + + if (new_links) + goto out; + + dev->mld_idx_mask &= ~BIT_ULL(mvif->mld_group_idx); + dev->mld_remap_idx_mask &= ~BIT_ULL(mvif->mld_remap_idx); + +out: + mutex_unlock(&dev->mt76.mutex); + + return ret; } static void diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index 0347ee0c2dd75..5bde9959bbb99 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -1034,7 +1034,6 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, struct mt76_connac_bss_basic_tlv *bss; u32 type = CONNECTION_INFRA_AP; u16 sta_wlan_idx = wlan_idx; - struct ieee80211_sta *sta; struct tlv *tlv; int idx; @@ -1045,14 +1044,18 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, break; case NL80211_IFTYPE_STATION: if (enable) { + struct ieee80211_sta *sta; + rcu_read_lock(); - sta = ieee80211_find_sta(vif, vif->bss_conf.bssid); - /* TODO: enable BSS_INFO_UAPSD & BSS_INFO_PM */ + sta = ieee80211_find_sta(vif, link_conf->bssid); if (sta) { - struct mt76_wcid *wcid; + struct mt7996_sta *msta = (void *)sta->drv_priv; + struct mt7996_sta_link *msta_link; + int link_id = link_conf->link_id; - wcid = (struct mt76_wcid *)sta->drv_priv; - sta_wlan_idx = wcid->idx; + msta_link = rcu_dereference(msta->link[link_id]); + if (msta_link) + sta_wlan_idx = msta_link->wcid.idx; } rcu_read_unlock(); } @@ -1069,8 +1072,6 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_BASIC, sizeof(*bss)); bss = (struct mt76_connac_bss_basic_tlv *)tlv; - bss->bcn_interval = cpu_to_le16(link_conf->beacon_int); - bss->dtim_period = link_conf->dtim_period; bss->bmc_tx_wlan_idx = cpu_to_le16(wlan_idx); bss->sta_idx = cpu_to_le16(sta_wlan_idx); bss->conn_type = cpu_to_le32(type); @@ -1090,10 +1091,10 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, memcpy(bss->bssid, link_conf->bssid, ETH_ALEN); bss->bcn_interval = cpu_to_le16(link_conf->beacon_int); - bss->dtim_period = vif->bss_conf.dtim_period; + bss->dtim_period = link_conf->dtim_period; bss->phymode = mt76_connac_get_phy_mode(phy, vif, chandef->chan->band, NULL); - bss->phymode_ext = mt76_connac_get_phy_mode_ext(phy, &vif->bss_conf, + bss->phymode_ext = mt76_connac_get_phy_mode_ext(phy, link_conf, chandef->chan->band); return 0; @@ -1822,8 +1823,8 @@ mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb, bf->ibf_nrow = tx_ant; if (link_sta->eht_cap.has_eht || link_sta->he_cap.has_he) - bf->ibf_timeout = is_mt7996(&dev->mt76) ? MT7996_IBF_TIMEOUT : - MT7992_IBF_TIMEOUT; + bf->ibf_timeout = is_mt7992(&dev->mt76) ? MT7992_IBF_TIMEOUT : + MT7996_IBF_TIMEOUT; else if (!ebf && link_sta->bandwidth <= IEEE80211_STA_RX_BW_40 && !bf->ncol) bf->ibf_timeout = MT7996_IBF_TIMEOUT_LEGACY; else @@ -2526,7 +2527,7 @@ int mt7996_mcu_teardown_mld_sta(struct mt7996_dev *dev, } static int -mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, +mt7996_mcu_sta_key_tlv(struct mt76_dev *dev, struct mt76_wcid *wcid, struct sk_buff *skb, struct ieee80211_key_conf *key, enum set_key_cmd cmd) @@ -2538,7 +2539,10 @@ mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_KEY_V2, sizeof(*sec)); sec = (struct sta_rec_sec_uni *)tlv; - sec->add = 0; + /* due to connac3 FW design, we only do remove key for BIGTK; even for + * removal, the field should be filled with SET_KEY + */ + sec->add = SET_KEY; sec->n_cipher = 1; sec_key = &sec->key[0]; sec_key->wlan_idx = cpu_to_le16(wcid->idx); @@ -2578,29 +2582,33 @@ mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, case WLAN_CIPHER_SUITE_BIP_GMAC_256: sec_key->cipher_id = MCU_CIPHER_BCN_PROT_GMAC_256; break; + case WLAN_CIPHER_SUITE_BIP_CMAC_256: + if (!is_mt7990(dev)) + return -EOPNOTSUPP; + sec_key->cipher_id = MCU_CIPHER_BCN_PROT_CMAC_256; + break; default: return -EOPNOTSUPP; } - sec_key->bcn_mode = BP_SW_MODE; + sec_key->bcn_mode = is_mt7990(dev) ? BP_HW_MODE : BP_SW_MODE; return 0; } -int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, +int mt7996_mcu_add_key(struct mt76_dev *dev, struct mt7996_vif_link *link, struct ieee80211_key_conf *key, int mcu_cmd, struct mt76_wcid *wcid, enum set_key_cmd cmd) { - struct mt76_vif_link *mvif = (struct mt76_vif_link *)vif->drv_priv; struct sk_buff *skb; int ret; - skb = __mt76_connac_mcu_alloc_sta_req(dev, mvif, wcid, - MT7996_STA_UPDATE_MAX_SIZE); + skb = __mt76_connac_mcu_alloc_sta_req(dev, (struct mt76_vif_link *)link, + wcid, MT7996_STA_UPDATE_MAX_SIZE); if (IS_ERR(skb)) return PTR_ERR(skb); - ret = mt7996_mcu_sta_key_tlv(wcid, skb, key, cmd); + ret = mt7996_mcu_sta_key_tlv(dev, wcid, skb, key, cmd); if (ret) { dev_kfree_skb(skb); return ret; @@ -2720,12 +2728,18 @@ mt7996_mcu_beacon_mbss(struct sk_buff *rskb, struct sk_buff *skb, static void mt7996_mcu_beacon_cont(struct mt7996_dev *dev, struct ieee80211_bss_conf *link_conf, + struct mt7996_vif_link *link, struct sk_buff *rskb, struct sk_buff *skb, struct bss_bcn_content_tlv *bcn, struct ieee80211_mutable_offsets *offs) { - struct mt76_wcid *wcid = &dev->mt76.global_wcid; - u8 *buf; + u8 *buf, keyidx = link->msta_link.wcid.hw_key_idx2; + struct mt76_wcid *wcid; + + if (is_mt7990(&dev->mt76) && (keyidx == 6 || keyidx == 7)) + wcid = &link->msta_link.wcid; + else + wcid = &dev->mt76.global_wcid; bcn->pkt_len = cpu_to_le16(MT_TXD_SIZE + skb->len); bcn->tim_ie_pos = cpu_to_le16(offs->tim_offset); @@ -2800,7 +2814,7 @@ int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, info = IEEE80211_SKB_CB(skb); info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, mlink->band_idx); - mt7996_mcu_beacon_cont(dev, link_conf, rskb, skb, bcn, &offs); + mt7996_mcu_beacon_cont(dev, link_conf, link, rskb, skb, bcn, &offs); if (link_conf->bssid_indicator) mt7996_mcu_beacon_mbss(rskb, skb, bcn, &offs); mt7996_mcu_beacon_cntdwn(rskb, skb, &offs, link_conf->csa_active); @@ -3414,6 +3428,9 @@ int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif, #define WMM_PARAM_SET (WMM_AIFS_SET | WMM_CW_MIN_SET | \ WMM_CW_MAX_SET | WMM_TXOP_SET) struct mt7996_vif_link *link = mt7996_vif_conf_link(dev, vif, link_conf); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + unsigned int link_id = link_conf->link_id; + struct mt7996_vif_link_info *link_info = &mvif->link_info[link_id]; struct { u8 bss_idx; u8 __rsv[3]; @@ -3431,7 +3448,7 @@ int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif, skb_put_data(skb, &hdr, sizeof(hdr)); for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) { - struct ieee80211_tx_queue_params *q = &link->queue_params[ac]; + struct ieee80211_tx_queue_params *q = &link_info->queue_params[ac]; struct edca *e; struct tlv *tlv; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c index d14b626ee5115..80db102ed809c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c @@ -595,6 +595,7 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.nbuf = MT7996_HW_TOKEN_SIZE; wed->wlan.token_start = MT7996_TOKEN_SIZE - wed->wlan.nbuf; + wed->wlan.hif2 = hif2; wed->wlan.amsdu_max_subframes = 8; wed->wlan.amsdu_max_len = 1536; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index 8ec2acdb33193..b942928c79e28 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -253,16 +253,21 @@ struct mt7996_vif_link { struct mt7996_sta_link msta_link; struct mt7996_phy *phy; - struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS]; struct cfg80211_bitrate_mask bitrate_mask; u8 mld_idx; }; +struct mt7996_vif_link_info { + struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS]; +}; + struct mt7996_vif { struct mt7996_vif_link deflink; /* must be first */ struct mt76_vif_data mt76; + struct mt7996_vif_link_info link_info[IEEE80211_MLD_MAX_NUM_LINKS]; + u8 mld_group_idx; u8 mld_remap_idx; }; @@ -781,7 +786,7 @@ void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset, static inline u16 mt7996_rx_chainmask(struct mt7996_phy *phy) { - int max_nss = hweight8(phy->mt76->hw->wiphy->available_antennas_tx); + int max_nss = hweight16(phy->orig_antenna_mask); int cur_nss = hweight8(phy->mt76->antenna_mask); u16 tx_chainmask = phy->mt76->chainmask; @@ -843,7 +848,7 @@ void mt7996_update_channel(struct mt76_phy *mphy); int mt7996_init_debugfs(struct mt7996_dev *dev); void mt7996_debugfs_rx_fw_monitor(struct mt7996_dev *dev, const void *data, int len); bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len); -int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, +int mt7996_mcu_add_key(struct mt76_dev *dev, struct mt7996_vif_link *link, struct ieee80211_key_conf *key, int mcu_cmd, struct mt76_wcid *wcid, enum set_key_cmd cmd); int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, diff --git a/drivers/net/wireless/mediatek/mt76/scan.c b/drivers/net/wireless/mediatek/mt76/scan.c index 5a875aac410fc..3d9cf6f5e137f 100644 --- a/drivers/net/wireless/mediatek/mt76/scan.c +++ b/drivers/net/wireless/mediatek/mt76/scan.c @@ -63,10 +63,8 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) rcu_read_lock(); - if (!ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL)) { - ieee80211_free_txskb(phy->hw, skb); + if (!ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL)) goto out; - } info = IEEE80211_SKB_CB(skb); if (req->no_cck) diff --git a/drivers/net/wireless/mediatek/mt76/wed.c b/drivers/net/wireless/mediatek/mt76/wed.c index 907a8e43e72ad..fbd7e59c73aaf 100644 --- a/drivers/net/wireless/mediatek/mt76/wed.c +++ b/drivers/net/wireless/mediatek/mt76/wed.c @@ -8,7 +8,7 @@ void mt76_wed_release_rx_buf(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); int i; for (i = 0; i < dev->rx_token_size; i++) { @@ -31,8 +31,8 @@ EXPORT_SYMBOL_GPL(mt76_wed_release_rx_buf); #ifdef CONFIG_NET_MEDIATEK_SOC_WED u32 mt76_wed_init_rx_buf(struct mtk_wed_device *wed, int size) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); struct mtk_wed_bm_desc *desc = wed->rx_buf_ring.desc; + struct mt76_dev *dev = mt76_wed_to_dev(wed); struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN]; struct mt76_txwi_cache *t = NULL; int i; @@ -80,7 +80,7 @@ EXPORT_SYMBOL_GPL(mt76_wed_init_rx_buf); int mt76_wed_offload_enable(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); spin_lock_bh(&dev->token_lock); dev->token_size = wed->wlan.token_start; @@ -164,7 +164,7 @@ EXPORT_SYMBOL_GPL(mt76_wed_dma_setup); void mt76_wed_offload_disable(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); spin_lock_bh(&dev->token_lock); dev->token_size = dev->drv->token_size; @@ -174,7 +174,7 @@ EXPORT_SYMBOL_GPL(mt76_wed_offload_disable); void mt76_wed_reset_complete(struct mtk_wed_device *wed) { - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt76_dev *dev = mt76_wed_to_dev(wed); complete(&dev->mmio.wed_reset_complete); } diff --git a/drivers/net/wireless/microchip/wilc1000/hif.c b/drivers/net/wireless/microchip/wilc1000/hif.c index a229c6cab3322..d795ca05729ab 100644 --- a/drivers/net/wireless/microchip/wilc1000/hif.c +++ b/drivers/net/wireless/microchip/wilc1000/hif.c @@ -163,7 +163,7 @@ int wilc_scan(struct wilc_vif *vif, u8 scan_source, u32 index = 0; u32 i, scan_timeout; u8 *buffer; - u8 valuesize = 0; + u32 valuesize = 0; u8 *search_ssid_vals = NULL; const u8 ch_list_len = request->n_channels; struct host_if_drv *hif_drv = vif->hif_drv; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c index a6d50149e0c3e..1275bd8232d21 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c @@ -828,7 +828,7 @@ int rt2x00usb_probe(struct usb_interface *usb_intf, if (retval) goto exit_free_device; - rt2x00dev->anchor = devm_kmalloc(&usb_dev->dev, + rt2x00dev->anchor = devm_kmalloc(&usb_intf->dev, sizeof(struct usb_anchor), GFP_KERNEL); if (!rt2x00dev->anchor) { diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c index 2905baea62390..070c0431c4821 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c @@ -1023,9 +1023,6 @@ static int rtl8180_init_rx_ring(struct ieee80211_hw *dev) dma_addr_t *mapping; entry = priv->rx_ring + priv->rx_ring_sz*i; if (!skb) { - dma_free_coherent(&priv->pdev->dev, - priv->rx_ring_sz * 32, - priv->rx_ring, priv->rx_ring_dma); wiphy_err(dev->wiphy, "Cannot allocate RX skb\n"); return -ENOMEM; } @@ -1037,9 +1034,7 @@ static int rtl8180_init_rx_ring(struct ieee80211_hw *dev) if (dma_mapping_error(&priv->pdev->dev, *mapping)) { kfree_skb(skb); - dma_free_coherent(&priv->pdev->dev, - priv->rx_ring_sz * 32, - priv->rx_ring, priv->rx_ring_dma); + priv->rx_buf[i] = NULL; wiphy_err(dev->wiphy, "Cannot map DMA for RX skb\n"); return -ENOMEM; } @@ -1130,7 +1125,7 @@ static int rtl8180_start(struct ieee80211_hw *dev) ret = rtl8180_init_rx_ring(dev); if (ret) - return ret; + goto err_free_rings; for (i = 0; i < (dev->queues + 1); i++) if ((ret = rtl8180_init_tx_ring(dev, i, 16))) diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c index 0c5c66401daa6..7aa2da0cd63cc 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c @@ -338,14 +338,16 @@ static void rtl8187_rx_cb(struct urb *urb) spin_unlock_irqrestore(&priv->rx_queue.lock, f); skb_put(skb, urb->actual_length); - if (unlikely(urb->status)) { - dev_kfree_skb_irq(skb); - return; - } + if (unlikely(urb->status)) + goto free_skb; if (!priv->is_rtl8187b) { - struct rtl8187_rx_hdr *hdr = - (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr)); + struct rtl8187_rx_hdr *hdr; + + if (skb->len < sizeof(struct rtl8187_rx_hdr)) + goto free_skb; + + hdr = (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr)); flags = le32_to_cpu(hdr->flags); /* As with the RTL8187B below, the AGC is used to calculate * signal strength. In this case, the scaling @@ -355,8 +357,12 @@ static void rtl8187_rx_cb(struct urb *urb) rx_status.antenna = (hdr->signal >> 7) & 1; rx_status.mactime = le64_to_cpu(hdr->mac_time); } else { - struct rtl8187b_rx_hdr *hdr = - (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr)); + struct rtl8187b_rx_hdr *hdr; + + if (skb->len < sizeof(struct rtl8187b_rx_hdr)) + goto free_skb; + + hdr = (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr)); /* The Realtek datasheet for the RTL8187B shows that the RX * header contains the following quantities: signal quality, * RSSI, AGC, the received power in dB, and the measured SNR. @@ -409,6 +415,11 @@ static void rtl8187_rx_cb(struct urb *urb) skb_unlink(skb, &priv->rx_queue); dev_kfree_skb_irq(skb); } + return; + +free_skb: + dev_kfree_skb_irq(skb); + return; } static int rtl8187_init_urbs(struct ieee80211_hw *dev) diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c index 3ded5952729fc..a17c1084931b0 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c @@ -1252,7 +1252,7 @@ void rtl8xxxu_gen1_config_channel(struct ieee80211_hw *hw) opmode &= ~BW_OPMODE_20MHZ; rtl8xxxu_write8(priv, REG_BW_OPMODE, opmode); rsr &= ~RSR_RSC_BANDWIDTH_40M; - if (sec_ch_above) + if (!sec_ch_above) rsr |= RSR_RSC_UPPER_SUB_CHANNEL; else rsr |= RSR_RSC_LOWER_SUB_CHANNEL; @@ -1321,9 +1321,8 @@ void rtl8xxxu_gen1_config_channel(struct ieee80211_hw *hw) for (i = RF_A; i < priv->rf_paths; i++) { val32 = rtl8xxxu_read_rfreg(priv, i, RF6052_REG_MODE_AG); - if (hw->conf.chandef.width == NL80211_CHAN_WIDTH_40) - val32 &= ~MODE_AG_CHANNEL_20MHZ; - else + val32 &= ~MODE_AG_BW_MASK; + if (hw->conf.chandef.width != NL80211_CHAN_WIDTH_40) val32 |= MODE_AG_CHANNEL_20MHZ; rtl8xxxu_write_rfreg(priv, i, RF6052_REG_MODE_AG, val32); } @@ -7928,6 +7927,7 @@ static int rtl8xxxu_probe(struct usb_interface *interface, goto err_set_intfdata; hw->vif_data_size = sizeof(struct rtl8xxxu_vif); + hw->sta_data_size = sizeof(struct rtl8xxxu_sta_info); hw->wiphy->max_scan_ssids = 1; hw->wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN; @@ -8136,6 +8136,9 @@ static const struct usb_device_id dev_table[] = { /* TP-Link TL-WN823N V2 */ {USB_DEVICE_AND_INTERFACE_INFO(0x2357, 0x0135, 0xff, 0xff, 0xff), .driver_info = (unsigned long)&rtl8192fu_fops}, +/* D-Link AN3U rev. A1 */ +{USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3328, 0xff, 0xff, 0xff), + .driver_info = (unsigned long)&rtl8192fu_fops}, #ifdef CONFIG_RTL8XXXU_UNTESTED /* Still supported by rtlwifi */ {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0x8176, 0xff, 0xff, 0xff), diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c index aa702ba7c9f54..d6c35e8d02a58 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c @@ -511,7 +511,8 @@ void rtl92cu_tx_fill_desc(struct ieee80211_hw *hw, if (sta) { sta_entry = (struct rtl_sta_info *)sta->drv_priv; tid = ieee80211_get_tid(hdr); - agg_state = sta_entry->tids[tid].agg.agg_state; + if (tid < MAX_TID_COUNT) + agg_state = sta_entry->tids[tid].agg.agg_state; ampdu_density = sta->deflink.ht_cap.ampdu_density; } diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index fa0ed39cb1992..dde2ea6a00e06 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -730,10 +730,10 @@ void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel) } EXPORT_SYMBOL(rtw_set_rx_freq_band); -void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period) +void rtw_set_dtim_period(struct rtw_dev *rtwdev, u8 dtim_period) { rtw_write32_set(rtwdev, REG_TCR, BIT_TCR_UPDATE_TIMIE); - rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period - 1); + rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period ? dtim_period - 1 : 0); } void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel, @@ -1658,14 +1658,41 @@ static u16 rtw_get_max_scan_ie_len(struct rtw_dev *rtwdev) return len; } +static struct ieee80211_supported_band * +rtw_sband_dup(struct rtw_dev *rtwdev, + const struct ieee80211_supported_band *sband) +{ + struct ieee80211_supported_band *dup; + + dup = devm_kmemdup(rtwdev->dev, sband, sizeof(*sband), GFP_KERNEL); + if (!dup) + return NULL; + + dup->channels = devm_kmemdup_array(rtwdev->dev, sband->channels, + sband->n_channels, + sizeof(*sband->channels), + GFP_KERNEL); + if (!dup->channels) + return NULL; + + dup->bitrates = devm_kmemdup_array(rtwdev->dev, sband->bitrates, + sband->n_bitrates, + sizeof(*sband->bitrates), + GFP_KERNEL); + if (!dup->bitrates) + return NULL; + + return dup; +} + static void rtw_set_supported_band(struct ieee80211_hw *hw, const struct rtw_chip_info *chip) { - struct rtw_dev *rtwdev = hw->priv; struct ieee80211_supported_band *sband; + struct rtw_dev *rtwdev = hw->priv; if (chip->band & RTW_BAND_2G) { - sband = kmemdup(&rtw_band_2ghz, sizeof(*sband), GFP_KERNEL); + sband = rtw_sband_dup(rtwdev, &rtw_band_2ghz); if (!sband) goto err_out; if (chip->ht_supported) @@ -1674,7 +1701,7 @@ static void rtw_set_supported_band(struct ieee80211_hw *hw, } if (chip->band & RTW_BAND_5G) { - sband = kmemdup(&rtw_band_5ghz, sizeof(*sband), GFP_KERNEL); + sband = rtw_sband_dup(rtwdev, &rtw_band_5ghz); if (!sband) goto err_out; if (chip->ht_supported) @@ -1690,13 +1717,6 @@ static void rtw_set_supported_band(struct ieee80211_hw *hw, rtw_err(rtwdev, "failed to set supported band\n"); } -static void rtw_unset_supported_band(struct ieee80211_hw *hw, - const struct rtw_chip_info *chip) -{ - kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]); - kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]); -} - static void rtw_vif_smps_iter(void *data, u8 *mac, struct ieee80211_vif *vif) { @@ -2320,10 +2340,7 @@ EXPORT_SYMBOL(rtw_register_hw); void rtw_unregister_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) { - const struct rtw_chip_info *chip = rtwdev->chip; - ieee80211_unregister_hw(hw); - rtw_unset_supported_band(hw, chip); rtw_debugfs_deinit(rtwdev); rtw_led_deinit(rtwdev); } @@ -2444,10 +2461,10 @@ void rtw_core_enable_beacon(struct rtw_dev *rtwdev, bool enable) if (enable) { rtw_write32_set(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION); - rtw_write32_clr(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); + rtw_write8_clr(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); } else { rtw_write32_clr(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION); - rtw_write32_set(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); + rtw_write8_set(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); } } diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h index 43ed6d6b42919..1ab70214ce36e 100644 --- a/drivers/net/wireless/realtek/rtw88/main.h +++ b/drivers/net/wireless/realtek/rtw88/main.h @@ -2226,7 +2226,7 @@ enum nl80211_band rtw_hw_to_nl80211_band(enum rtw_supported_band hw_band) } void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel); -void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period); +void rtw_set_dtim_period(struct rtw_dev *rtwdev, u8 dtim_period); void rtw_get_channel_params(struct cfg80211_chan_def *chandef, struct rtw_channel_params *ch_param); bool check_hw_ready(struct rtw_dev *rtwdev, u32 addr, u32 mask, u32 target); diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c index 7a0fffc359e25..8cd09d66655db 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c @@ -37,6 +37,8 @@ static const struct usb_device_id rtw_8821cu_id_table[] = { .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Edimax */ { USB_DEVICE_AND_INTERFACE_INFO(0x7392, 0xd811, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Edimax */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2c4e, 0x0105, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Mercusys */ {}, }; MODULE_DEVICE_TABLE(usb, rtw_8821cu_id_table); diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c index 89b6485b229a8..4d88cc2f41485 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c @@ -1005,7 +1005,8 @@ static int rtw8822b_set_antenna(struct rtw_dev *rtwdev, hal->antenna_tx = antenna_tx; hal->antenna_rx = antenna_rx; - rtw8822b_config_trx_mode(rtwdev, antenna_tx, antenna_rx, false); + if (test_bit(RTW_FLAG_POWERON, rtwdev->flags)) + rtw8822b_config_trx_mode(rtwdev, antenna_tx, antenna_rx, false); return 0; } diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cu.c b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c index 324fd5c8bfd44..755f76840b121 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822cu.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c @@ -21,6 +21,8 @@ static const struct usb_device_id rtw_8822cu_id_table[] = { .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, { USB_DEVICE_AND_INTERFACE_INFO(0x13b1, 0x0043, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, /* Alpha - Alpha */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3329, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, /* D-Link AC13U rev. A1 */ {}, }; MODULE_DEVICE_TABLE(usb, rtw_8822cu_id_table); diff --git a/drivers/net/wireless/realtek/rtw88/sdio.c b/drivers/net/wireless/realtek/rtw88/sdio.c index 99d7c629eac6f..e35de52d8eb43 100644 --- a/drivers/net/wireless/realtek/rtw88/sdio.c +++ b/drivers/net/wireless/realtek/rtw88/sdio.c @@ -144,8 +144,10 @@ static u32 rtw_sdio_to_io_address(struct rtw_dev *rtwdev, u32 addr, static bool rtw_sdio_use_direct_io(struct rtw_dev *rtwdev, u32 addr) { + bool might_indirect_under_power_off = rtwdev->chip->id == RTW_CHIP_TYPE_8822C; + if (!test_bit(RTW_FLAG_POWERON, rtwdev->flags) && - !rtw_sdio_is_bus_addr(addr)) + !rtw_sdio_is_bus_addr(addr) && might_indirect_under_power_off) return false; return !rtw_sdio_is_sdio30_supported(rtwdev) || diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c index 3b5126ffc81a1..6e841a11c752b 100644 --- a/drivers/net/wireless/realtek/rtw88/usb.c +++ b/drivers/net/wireless/realtek/rtw88/usb.c @@ -1040,7 +1040,7 @@ static int rtw_usb_intf_init(struct rtw_dev *rtwdev, struct usb_interface *intf) { struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); - struct usb_device *udev = usb_get_dev(interface_to_usbdev(intf)); + struct usb_device *udev = interface_to_usbdev(intf); int ret; rtwusb->udev = udev; @@ -1066,7 +1066,6 @@ static void rtw_usb_intf_deinit(struct rtw_dev *rtwdev, { struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); - usb_put_dev(rtwusb->udev); kfree(rtwusb->usb_data); usb_set_intfdata(intf, NULL); } diff --git a/drivers/net/wireless/realtek/rtw89/chan.c b/drivers/net/wireless/realtek/rtw89/chan.c index 86f1b39a967fe..8fe6a7ef738f7 100644 --- a/drivers/net/wireless/realtek/rtw89/chan.c +++ b/drivers/net/wireless/realtek/rtw89/chan.c @@ -2608,17 +2608,20 @@ bool rtw89_mcc_detect_go_bcn(struct rtw89_dev *rtwdev, static void rtw89_mcc_detect_connection(struct rtw89_dev *rtwdev, struct rtw89_mcc_role *role) { + struct rtw89_vif_link *rtwvif_link = role->rtwvif_link; struct ieee80211_vif *vif; bool start_detect; int ret; ret = rtw89_core_send_nullfunc(rtwdev, role->rtwvif_link, true, false, RTW89_MCC_PROBE_TIMEOUT); - if (ret) + if (ret && + READ_ONCE(rtwvif_link->sync_bcn_tsf) == rtwvif_link->last_sync_bcn_tsf) role->probe_count++; else role->probe_count = 0; + rtwvif_link->last_sync_bcn_tsf = READ_ONCE(rtwvif_link->sync_bcn_tsf); if (role->probe_count < RTW89_MCC_PROBE_MAX_TRIES) return; diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index 917b2adede61d..018857d3569a8 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -2655,7 +2655,7 @@ static void rtw89_core_bcn_track_assoc(struct rtw89_dev *rtwdev, rcu_read_lock(); bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true); - beacon_int = bss_conf->beacon_int; + beacon_int = bss_conf->beacon_int ?: 100; dtim = bss_conf->dtim_period; rcu_read_unlock(); @@ -2685,9 +2685,7 @@ static void rtw89_core_bcn_track_reset(struct rtw89_dev *rtwdev) memset(&rtwdev->bcn_track, 0, sizeof(rtwdev->bcn_track)); } -static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, - struct ieee80211_bss_conf *bss_conf, - struct sk_buff *skb) +static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, struct sk_buff *skb) { #define RTW89_APPEND_TSF_2GHZ 384 #define RTW89_APPEND_TSF_5GHZ 52 @@ -2696,7 +2694,7 @@ static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb); struct rtw89_beacon_stat *bcn_stat = &rtwdev->phystat.bcn_stat; struct rtw89_beacon_track_info *bcn_track = &rtwdev->bcn_track; - u32 bcn_intvl_us = ieee80211_tu_to_usec(bss_conf->beacon_int); + u32 bcn_intvl_us = ieee80211_tu_to_usec(bcn_track->beacon_int); u64 tsf = le64_to_cpu(mgmt->u.beacon.timestamp); u8 wp, num = bcn_stat->num; u16 append; @@ -2704,6 +2702,10 @@ static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, if (!RTW89_CHK_FW_FEATURE(BEACON_TRACKING, &rtwdev->fw)) return; + /* Skip if not yet associated */ + if (!bcn_intvl_us) + return; + switch (rx_status->band) { default: case NL80211_BAND_2GHZ: @@ -2791,7 +2793,7 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, pkt_stat->beacon_rate = desc_info->data_rate; pkt_stat->beacon_len = skb->len; - rtw89_vif_rx_bcn_stat(rtwdev, bss_conf, skb); + rtw89_vif_rx_bcn_stat(rtwdev, skb); } if (!ether_addr_equal(bss_conf->addr, hdr->addr1)) @@ -5109,7 +5111,7 @@ static void rtw89_init_eht_cap(struct rtw89_dev *rtwdev, u8 val, val_mcs13; int sts = 8; - if (chip->chip_gen == RTW89_CHIP_AX) + if (chip->chip_gen == RTW89_CHIP_AX || hal->no_eht) return; if (hal->no_mcs_12_13) diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index 928c8c84c9644..c3839d49f4420 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -4996,6 +4996,7 @@ struct rtw89_hal { bool support_cckpd; bool support_igi; bool no_mcs_12_13; + bool no_eht; atomic_t roc_chanctx_idx; u8 roc_link_index; diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 3dc7981c510fd..a82df3814069c 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -824,10 +824,6 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf s8 *bufp, tmp; int ret; - bufp = vzalloc(map->addr_to - map->addr_from + 4); - if (!bufp) - return -ENOMEM; - if (path_num == 1) max_valid_addr = map->addr_to_1ss; else @@ -836,6 +832,10 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf if (max_valid_addr == 0) return -EOPNOTSUPP; + bufp = vzalloc(map->addr_to - map->addr_from + 4); + if (!bufp) + return -ENOMEM; + for (addr = map->addr_from; addr <= max_valid_addr; addr += 4) { ret = rtw89_mac_txpwr_read32(rtwdev, RTW89_PHY_0, addr, &val); if (ret) diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 080c4f8a655a8..c9851aafe649e 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -8042,6 +8042,7 @@ int rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct cfg80211_scan_request *req = &scan_req->req; const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); + struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link); struct rtw89_vif *rtwvif = rtwvif_link->rtwvif; struct rtw89_chanctx_pause_parm pause_parm = { .rsn = RTW89_CHANCTX_PAUSE_REASON_HW_SCAN, @@ -8071,6 +8072,8 @@ int rtw89_hw_scan_start(struct rtw89_dev *rtwdev, if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) get_random_mask_addr(mac_addr, req->mac_addr, req->mac_addr_mask); + else if (ieee80211_vif_is_mld(vif)) + ether_addr_copy(mac_addr, vif->addr); else ether_addr_copy(mac_addr, rtwvif_link->mac_addr); diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index ddebf79720687..47e5cbec306d2 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -42,6 +42,10 @@ struct rtw89_c2hreg_phycap { #define RTW89_C2HREG_PHYCAP_W0_BW GENMASK(31, 24) #define RTW89_C2HREG_PHYCAP_W1_TX_NSS GENMASK(7, 0) #define RTW89_C2HREG_PHYCAP_W1_PROT GENMASK(15, 8) +#define RTW89_C2HREG_PHYCAP_W1_PROT_11N 1 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11AC 2 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11AX 3 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11BE 4 #define RTW89_C2HREG_PHYCAP_W1_NIC GENMASK(23, 16) #define RTW89_C2HREG_PHYCAP_W1_WL_FUNC GENMASK(31, 24) #define RTW89_C2HREG_PHYCAP_W2_HW_TYPE GENMASK(7, 0) diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index fd11b8fb3c899..71194ea68bcee 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -2999,6 +2999,7 @@ static int rtw89_mac_setup_phycap_part0(struct rtw89_dev *rtwdev) struct rtw89_efuse *efuse = &rtwdev->efuse; struct rtw89_mac_c2h_info c2h_info = {}; struct rtw89_hal *hal = &rtwdev->hal; + u8 protocol; u8 tx_nss; u8 rx_nss; u8 tx_ant; @@ -3046,6 +3047,10 @@ static int rtw89_mac_setup_phycap_part0(struct rtw89_dev *rtwdev) rtw89_debug(rtwdev, RTW89_DBG_FW, "TX path diversity=%d\n", hal->tx_path_diversity); rtw89_debug(rtwdev, RTW89_DBG_FW, "Antenna diversity=%d\n", hal->ant_diversity); + protocol = u32_get_bits(phycap->w1, RTW89_C2HREG_PHYCAP_W1_PROT); + if (protocol < RTW89_C2HREG_PHYCAP_W1_PROT_11BE) + hal->no_eht = true; + return 0; } @@ -4262,6 +4267,7 @@ static void rtw89_mac_bcn_drop(struct rtw89_dev *rtwdev, #define BCN_HOLD_DEF 200 #define BCN_MASK_DEF 0 #define TBTT_ERLY_DEF 5 +#define TBTT_AGG_DEF 1 #define BCN_SET_UNIT 32 #define BCN_ERLY_SET_DLY (10 * 2) @@ -4565,6 +4571,16 @@ static void rtw89_mac_port_cfg_tbtt_early(struct rtw89_dev *rtwdev, B_AX_TBTTERLY_MASK, TBTT_ERLY_DEF); } +static void rtw89_mac_port_cfg_tbtt_agg(struct rtw89_dev *rtwdev, + struct rtw89_vif_link *rtwvif_link) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + const struct rtw89_port_reg *p = mac->port_base; + + rtw89_write16_port_mask(rtwdev, rtwvif_link, p->tbtt_agg, + B_AX_TBTT_AGG_NUM_MASK, TBTT_AGG_DEF); +} + static void rtw89_mac_port_cfg_bss_color(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link) { @@ -4825,6 +4841,7 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvi rtw89_mac_port_cfg_bcn_hold_time(rtwdev, rtwvif_link); rtw89_mac_port_cfg_bcn_mask_area(rtwdev, rtwvif_link); rtw89_mac_port_cfg_tbtt_early(rtwdev, rtwvif_link); + rtw89_mac_port_cfg_tbtt_agg(rtwdev, rtwvif_link); rtw89_mac_port_cfg_bss_color(rtwdev, rtwvif_link); rtw89_mac_port_cfg_mbssid(rtwdev, rtwvif_link); rtw89_mac_port_cfg_func_en(rtwdev, rtwvif_link, true); @@ -7016,6 +7033,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = { .check_mac_en = rtw89_mac_check_mac_en_ax, .sys_init = sys_init_ax, .trx_init = trx_init_ax, + .err_imr_ctrl = err_imr_ctrl_ax, .hci_func_en = rtw89_mac_hci_func_en_ax, .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_ax, .dle_func_en = dle_func_en_ax, diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index 25fe5e5c8a979..51e37c183a35e 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -999,6 +999,7 @@ struct rtw89_mac_gen_def { enum rtw89_mac_hwmod_sel sel); int (*sys_init)(struct rtw89_dev *rtwdev); int (*trx_init)(struct rtw89_dev *rtwdev); + void (*err_imr_ctrl)(struct rtw89_dev *rtwdev, bool en); void (*hci_func_en)(struct rtw89_dev *rtwdev); void (*dmac_func_pre_en)(struct rtw89_dev *rtwdev); void (*dle_func_en)(struct rtw89_dev *rtwdev, bool enable); diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index 7b04183a3a5dd..474be7a5e49c7 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -127,6 +127,7 @@ static int __rtw89_ops_add_iface_link(struct rtw89_dev *rtwdev, rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT; rtwvif_link->rand_tsf_done = false; rtwvif_link->detect_bcn_count = 0; + rtwvif_link->last_sync_bcn_tsf = 0; rcu_read_lock(); diff --git a/drivers/net/wireless/realtek/rtw89/mac_be.c b/drivers/net/wireless/realtek/rtw89/mac_be.c index ef69672b6862e..e5a61c628b731 100644 --- a/drivers/net/wireless/realtek/rtw89/mac_be.c +++ b/drivers/net/wireless/realtek/rtw89/mac_be.c @@ -1168,7 +1168,7 @@ static int resp_pktctl_init_be(struct rtw89_dev *rtwdev, u8 mac_idx) reg = rtw89_mac_reg_by_idx(rtwdev, R_BE_RESP_CSI_RESERVED_PAGE, mac_idx); rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_START_PAGE_MASK, qt_cfg.pktid); - rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_PAGE_NUM_MASK, qt_cfg.pg_num); + rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_PAGE_NUM_MASK, qt_cfg.pg_num + 1); return 0; } @@ -2594,6 +2594,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = { .check_mac_en = rtw89_mac_check_mac_en_be, .sys_init = sys_init_be, .trx_init = trx_init_be, + .err_imr_ctrl = err_imr_ctrl_be, .hci_func_en = rtw89_mac_hci_func_en_be, .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_be, .dle_func_en = dle_func_en_be, diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 0ee5f85794476..6be1849b0c4d2 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -604,8 +604,15 @@ static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev, void *rpp) info->parse_rpp(rtwdev, rpp, &rpp_info); - if (rpp_info.txch == RTW89_TXCH_CH12) { - rtw89_warn(rtwdev, "should no fwcmd release report\n"); + if (unlikely(rpp_info.txch >= RTW89_TXCH_NUM || + info->tx_dma_ch_mask & BIT(rpp_info.txch))) { + rtw89_warn(rtwdev, "should no release report on txch %d\n", + rpp_info.txch); + return; + } + + if (unlikely(rpp_info.seq >= RTW89_PCI_TXWD_NUM_MAX)) { + rtw89_warn(rtwdev, "invalid seq %d\n", rpp_info.seq); return; } @@ -4584,6 +4591,7 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev) rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1); } + rtw89_pci_hci_ldo(rtwdev); rtw89_pci_l2_hci_ldo(rtwdev); rtw89_pci_basic_cfg(rtwdev, true); diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c index 58582f8d2b74c..8211d1277699c 100644 --- a/drivers/net/wireless/realtek/rtw89/regd.c +++ b/drivers/net/wireless/realtek/rtw89/regd.c @@ -1142,6 +1142,7 @@ static int rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev, } } else { rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT; + dflt = true; } rcu_read_unlock(); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852bu.c b/drivers/net/wireless/realtek/rtw89/rtw8852bu.c index 0694272f7ffae..add5987110b33 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852bu.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852bu.c @@ -30,6 +30,8 @@ static const struct usb_device_id rtw_8852bu_id_table[] = { .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0x6931, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, + { USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0xf0c8, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3327, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x3574, 0x6121, 0xff, 0xff, 0xff), diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922a.c b/drivers/net/wireless/realtek/rtw89/rtw8922a.c index 6aa19ad259aca..730e5d0d65750 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8922a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8922a.c @@ -636,16 +636,30 @@ static int rtw8922a_read_efuse_rf(struct rtw89_dev *rtwdev, u8 *log_map) static int rtw8922a_read_efuse(struct rtw89_dev *rtwdev, u8 *log_map, enum rtw89_efuse_block block) { + struct rtw89_efuse *efuse = &rtwdev->efuse; + int ret; + switch (block) { case RTW89_EFUSE_BLOCK_HCI_DIG_PCIE_SDIO: - return rtw8922a_read_efuse_pci_sdio(rtwdev, log_map); + ret = rtw8922a_read_efuse_pci_sdio(rtwdev, log_map); + break; case RTW89_EFUSE_BLOCK_HCI_DIG_USB: - return rtw8922a_read_efuse_usb(rtwdev, log_map); + ret = rtw8922a_read_efuse_usb(rtwdev, log_map); + break; case RTW89_EFUSE_BLOCK_RF: - return rtw8922a_read_efuse_rf(rtwdev, log_map); + ret = rtw8922a_read_efuse_rf(rtwdev, log_map); + break; default: - return 0; + ret = 0; + break; + } + + if (!ret && is_zero_ether_addr(efuse->addr)) { + rtw89_info(rtwdev, "efuse mac address is zero, using random mac\n"); + eth_random_addr(efuse->addr); } + + return ret; } #define THM_TRIM_POSITIVE_MASK BIT(6) @@ -1756,6 +1770,32 @@ static int rtw8922a_ctrl_rx_path_tmac(struct rtw89_dev *rtwdev, } #define DIGITAL_PWR_COMP_REG_NUM 22 +static const u32 rtw8922a_digital_pwr_comp_2g_s0_val[][DIGITAL_PWR_COMP_REG_NUM] = { + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010000, 0x00000101, + 0x01010101, 0x02020201, 0x02010000, 0x03030202, 0x00000303, + 0x03020101, 0x06060504, 0x01010000, 0x06050403, 0x01000606, + 0x05040202, 0x07070706}, + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010100, 0x00000101, + 0x01000000, 0x01010101, 0x01010000, 0x02020202, 0x00000404, + 0x03020101, 0x04040303, 0x02010000, 0x03030303, 0x00000505, + 0x03030201, 0x05050303}, +}; + +static const u32 rtw8922a_digital_pwr_comp_2g_s1_val[][DIGITAL_PWR_COMP_REG_NUM] = { + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x01010000, 0x01010101, 0x00000101, + 0x01010100, 0x01010101, 0x01010000, 0x02020202, 0x01000202, + 0x02020101, 0x03030202, 0x02010000, 0x05040403, 0x01000606, + 0x05040302, 0x07070605}, + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010100, 0x00000101, + 0x01010000, 0x02020201, 0x02010100, 0x03030202, 0x01000404, + 0x04030201, 0x05050404, 0x01010100, 0x04030303, 0x01000505, + 0x03030101, 0x05050404}, +}; + static const u32 rtw8922a_digital_pwr_comp_val[][DIGITAL_PWR_COMP_REG_NUM] = { {0x012C0096, 0x044C02BC, 0x00322710, 0x015E0096, 0x03C8028A, 0x0BB80708, 0x17701194, 0x02020100, 0x03030303, 0x01000303, @@ -1770,7 +1810,7 @@ static const u32 rtw8922a_digital_pwr_comp_val[][DIGITAL_PWR_COMP_REG_NUM] = { }; static void rtw8922a_set_digital_pwr_comp(struct rtw89_dev *rtwdev, - bool enable, u8 nss, + u8 band, u8 nss, enum rtw89_rf_path path) { static const u32 ltpc_t0[2] = {R_BE_LTPC_T0_PATH0, R_BE_LTPC_T0_PATH1}; @@ -1778,14 +1818,25 @@ static void rtw8922a_set_digital_pwr_comp(struct rtw89_dev *rtwdev, u32 addr, val; u32 i; - if (nss == 1) - digital_pwr_comp = rtw8922a_digital_pwr_comp_val[0]; - else - digital_pwr_comp = rtw8922a_digital_pwr_comp_val[1]; + if (nss == 1) { + if (band == RTW89_BAND_2G) + digital_pwr_comp = path == RF_PATH_A ? + rtw8922a_digital_pwr_comp_2g_s0_val[0] : + rtw8922a_digital_pwr_comp_2g_s1_val[0]; + else + digital_pwr_comp = rtw8922a_digital_pwr_comp_val[0]; + } else { + if (band == RTW89_BAND_2G) + digital_pwr_comp = path == RF_PATH_A ? + rtw8922a_digital_pwr_comp_2g_s0_val[1] : + rtw8922a_digital_pwr_comp_2g_s1_val[1]; + else + digital_pwr_comp = rtw8922a_digital_pwr_comp_val[1]; + } addr = ltpc_t0[path]; for (i = 0; i < DIGITAL_PWR_COMP_REG_NUM; i++, addr += 4) { - val = enable ? digital_pwr_comp[i] : 0; + val = digital_pwr_comp[i]; rtw89_phy_write32(rtwdev, addr, val); } } @@ -1794,7 +1845,7 @@ static void rtw8922a_digital_pwr_comp(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) { const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0); - bool enable = chan->band_type != RTW89_BAND_2G; + u8 band = chan->band_type; u8 path; if (rtwdev->mlo_dbcc_mode == MLO_1_PLUS_1_1RF) { @@ -1802,10 +1853,10 @@ static void rtw8922a_digital_pwr_comp(struct rtw89_dev *rtwdev, path = RF_PATH_A; else path = RF_PATH_B; - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 1, path); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 1, path); } else { - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 2, RF_PATH_A); - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 2, RF_PATH_B); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 2, RF_PATH_A); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 2, RF_PATH_B); } } diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index f99e179f7ff9f..7fdc69578da31 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -431,6 +431,14 @@ static void hal_send_m4_event(struct rtw89_ser *ser) rtw89_mac_set_err_status(rtwdev, MAC_AX_ERR_L1_RCVY_EN); } +static void hal_enable_err_imr(struct rtw89_ser *ser) +{ + struct rtw89_dev *rtwdev = container_of(ser, struct rtw89_dev, ser); + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + + mac->err_imr_ctrl(rtwdev, true); +} + /* state handler */ static void ser_idle_st_hdl(struct rtw89_ser *ser, u8 evt) { @@ -552,6 +560,8 @@ static void ser_do_hci_st_hdl(struct rtw89_ser *ser, u8 evt) break; case SER_EV_MAC_RESET_DONE: + hal_enable_err_imr(ser); + ser_state_goto(ser, SER_IDLE_ST); break; diff --git a/drivers/net/wireless/realtek/rtw89/usb.c b/drivers/net/wireless/realtek/rtw89/usb.c index 6cf89aee252ed..512a46dd9d06a 100644 --- a/drivers/net/wireless/realtek/rtw89/usb.c +++ b/drivers/net/wireless/realtek/rtw89/usb.c @@ -256,7 +256,7 @@ static int rtw89_usb_write_port(struct rtw89_dev *rtwdev, u8 ch_dma, int ret; if (test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags)) - return 0; + return -ENODEV; urb = usb_alloc_urb(0, GFP_ATOMIC); if (!urb) @@ -305,8 +305,9 @@ static void rtw89_usb_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch) ret = rtw89_usb_write_port(rtwdev, txch, skb->data, skb->len, txcb); if (ret) { - rtw89_err(rtwdev, "write port txch %d failed: %d\n", - txch, ret); + if (ret != -ENODEV) + rtw89_err(rtwdev, "write port txch %d failed: %d\n", + txch, ret); skb_dequeue(&txcb->tx_ack_queue); kfree(txcb); @@ -410,8 +411,7 @@ static void rtw89_usb_rx_handler(struct work_struct *work) if (skb_queue_len(&rtwusb->rx_queue) >= RTW89_USB_MAX_RXQ_LEN) { rtw89_warn(rtwdev, "rx_queue overflow\n"); - dev_kfree_skb_any(rx_skb); - continue; + goto free_or_reuse; } memset(&desc_info, 0, sizeof(desc_info)); @@ -422,7 +422,7 @@ static void rtw89_usb_rx_handler(struct work_struct *work) rtw89_debug(rtwdev, RTW89_DBG_HCI, "failed to allocate RX skb of size %u\n", desc_info.pkt_size); - continue; + goto free_or_reuse; } pkt_offset = desc_info.offset + desc_info.rxd_len; @@ -432,6 +432,7 @@ static void rtw89_usb_rx_handler(struct work_struct *work) rtw89_core_rx(rtwdev, &desc_info, skb); +free_or_reuse: if (skb_queue_len(&rtwusb->rx_free_queue) >= RTW89_USB_RX_SKB_NUM) dev_kfree_skb_any(rx_skb); else diff --git a/drivers/net/wireless/realtek/rtw89/wow.c b/drivers/net/wireless/realtek/rtw89/wow.c index 5faa51ad896a2..f34cd863d1009 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.c +++ b/drivers/net/wireless/realtek/rtw89/wow.c @@ -809,6 +809,10 @@ static void rtw89_wow_show_wakeup_reason(struct rtw89_dev *rtwdev) reason = rtw89_read8(rtwdev, wow_reason_reg); switch (reason) { + case RTW89_WOW_RSN_RX_DISASSOC: + wakeup.disconnect = true; + rtw89_debug(rtwdev, RTW89_DBG_WOW, "WOW: Rx disassoc\n"); + break; case RTW89_WOW_RSN_RX_DEAUTH: wakeup.disconnect = true; rtw89_debug(rtwdev, RTW89_DBG_WOW, "WOW: Rx deauth\n"); diff --git a/drivers/net/wireless/realtek/rtw89/wow.h b/drivers/net/wireless/realtek/rtw89/wow.h index d2ba6cebc2a6b..71e07f482174f 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.h +++ b/drivers/net/wireless/realtek/rtw89/wow.h @@ -33,6 +33,7 @@ enum rtw89_wake_reason { RTW89_WOW_RSN_RX_PTK_REKEY = 0x1, RTW89_WOW_RSN_RX_GTK_REKEY = 0x2, + RTW89_WOW_RSN_RX_DISASSOC = 0x4, RTW89_WOW_RSN_RX_DEAUTH = 0x8, RTW89_WOW_RSN_DISCONNECT = 0x10, RTW89_WOW_RSN_RX_MAGIC_PKT = 0x21, diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c index f3a853edfc11d..c7ae8031436ae 100644 --- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c +++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c @@ -668,7 +668,7 @@ static int rsi_mac80211_config(struct ieee80211_hw *hw, struct rsi_hw *adapter = hw->priv; struct rsi_common *common = adapter->priv; struct ieee80211_conf *conf = &hw->conf; - int status = -EOPNOTSUPP; + int status = 0; mutex_lock(&common->mutex); @@ -2035,6 +2035,7 @@ int rsi_mac80211_attach(struct rsi_common *common) hw->queues = MAX_HW_QUEUES; hw->extra_tx_headroom = RSI_NEEDED_HEADROOM; + hw->vif_data_size = sizeof(struct vif_priv); hw->max_rates = 1; hw->max_rate_tries = MAX_RETRIES; diff --git a/drivers/net/wireless/st/cw1200/bh.c b/drivers/net/wireless/st/cw1200/bh.c index 3b4ded2ac801c..37232ee220375 100644 --- a/drivers/net/wireless/st/cw1200/bh.c +++ b/drivers/net/wireless/st/cw1200/bh.c @@ -317,10 +317,12 @@ static int cw1200_bh_rx_helper(struct cw1200_common *priv, if (wsm_id & 0x0400) { int rc = wsm_release_tx_buffer(priv, 1); - if (WARN_ON(rc < 0)) + if (WARN_ON(rc < 0)) { + dev_kfree_skb(skb_rx); return rc; - else if (rc > 0) + } else if (rc > 0) { *tx = 1; + } } /* cw1200_wsm_rx takes care on SKB livetime */ diff --git a/drivers/net/wireless/st/cw1200/pm.c b/drivers/net/wireless/st/cw1200/pm.c index 2002e3f9fe45b..b656afe65db07 100644 --- a/drivers/net/wireless/st/cw1200/pm.c +++ b/drivers/net/wireless/st/cw1200/pm.c @@ -264,12 +264,14 @@ int cw1200_wow_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan) wiphy_err(priv->hw->wiphy, "PM request failed: %d. WoW is disabled.\n", ret); cw1200_wow_resume(hw); + mutex_unlock(&priv->conf_mutex); return -EBUSY; } /* Force resume if event is coming from the device. */ if (atomic_read(&priv->bh_rx)) { cw1200_wow_resume(hw); + mutex_unlock(&priv->conf_mutex); return -EAGAIN; } diff --git a/drivers/net/wireless/ti/wl1251/tx.c b/drivers/net/wireless/ti/wl1251/tx.c index adb4840b04893..c264d83e71d9c 100644 --- a/drivers/net/wireless/ti/wl1251/tx.c +++ b/drivers/net/wireless/ti/wl1251/tx.c @@ -402,12 +402,14 @@ static void wl1251_tx_packet_cb(struct wl1251 *wl, int hdrlen; u8 *frame; - skb = wl->tx_frames[result->id]; - if (skb == NULL) { - wl1251_error("SKB for packet %d is NULL", result->id); + if (unlikely(result->id >= ARRAY_SIZE(wl->tx_frames) || + wl->tx_frames[result->id] == NULL)) { + wl1251_error("invalid packet id %u", result->id); return; } + skb = wl->tx_frames[result->id]; + info = IEEE80211_SKB_CB(skb); if (!(info->flags & IEEE80211_TX_CTL_NO_ACK) && diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c index 6116a8522d960..bdb06584d7e45 100644 --- a/drivers/net/wireless/ti/wlcore/main.c +++ b/drivers/net/wireless/ti/wlcore/main.c @@ -1880,6 +1880,8 @@ static int __maybe_unused wl1271_op_resume(struct ieee80211_hw *hw) wl->wow_enabled); WARN_ON(!wl->wow_enabled); + mutex_lock(&wl->mutex); + ret = pm_runtime_force_resume(wl->dev); if (ret < 0) { wl1271_error("ELP wakeup failure!"); @@ -1896,8 +1898,6 @@ static int __maybe_unused wl1271_op_resume(struct ieee80211_hw *hw) run_irq_work = true; spin_unlock_irqrestore(&wl->wl_lock, flags); - mutex_lock(&wl->mutex); - /* test the recovery flag before calling any SDIO functions */ pending_recovery = test_bit(WL1271_FLAG_RECOVERY_IN_PROGRESS, &wl->flags); diff --git a/drivers/net/wireless/ti/wlcore/tx.c b/drivers/net/wireless/ti/wlcore/tx.c index 464587d16ab20..3c0f8f3ba2668 100644 --- a/drivers/net/wireless/ti/wlcore/tx.c +++ b/drivers/net/wireless/ti/wlcore/tx.c @@ -207,6 +207,11 @@ static int wl1271_tx_allocate(struct wl1271 *wl, struct wl12xx_vif *wlvif, total_blocks = wlcore_hw_calc_tx_blocks(wl, total_len, spare_blocks); if (total_blocks <= wl->tx_blocks_available) { + if (skb_headroom(skb) < (total_len - skb->len) && + pskb_expand_head(skb, (total_len - skb->len), 0, GFP_ATOMIC)) { + wl1271_free_tx_id(wl, id); + return -ENOMEM; + } desc = skb_push(skb, total_len - skb->len); wlcore_hw_set_tx_desc_blocks(wl, desc, total_blocks, diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c index 5903d82e1ab1e..20815fdc9d376 100644 --- a/drivers/net/wireless/virtual/mac80211_hwsim.c +++ b/drivers/net/wireless/virtual/mac80211_hwsim.c @@ -3021,7 +3021,6 @@ static void hw_scan_work(struct work_struct *work) hwsim->tmp_chan->band, NULL)) { rcu_read_unlock(); - kfree_skb(probe); continue; } @@ -4040,7 +4039,7 @@ mac80211_hwsim_nan_dw_start(struct hrtimer *timer) ieee80211_vif_to_wdev(data->nan_device_vif); if (data->nan_curr_dw_band == NL80211_BAND_5GHZ) - ch = ieee80211_get_channel(hw->wiphy, 5475); + ch = ieee80211_get_channel(hw->wiphy, 5745); else ch = ieee80211_get_channel(hw->wiphy, 2437); diff --git a/drivers/net/wireless/virtual/virt_wifi.c b/drivers/net/wireless/virtual/virt_wifi.c index 4eae89376feb5..cd6b66242bff2 100644 --- a/drivers/net/wireless/virtual/virt_wifi.c +++ b/drivers/net/wireless/virtual/virt_wifi.c @@ -557,7 +557,6 @@ static int virt_wifi_newlink(struct net_device *dev, eth_hw_addr_inherit(dev, priv->lowerdev); netif_stacked_transfer_operstate(priv->lowerdev, dev); - SET_NETDEV_DEV(dev, &priv->lowerdev->dev); dev->ieee80211_ptr = kzalloc(sizeof(*dev->ieee80211_ptr), GFP_KERNEL); if (!dev->ieee80211_ptr) { diff --git a/drivers/net/wwan/iosm/iosm_ipc_mux.c b/drivers/net/wwan/iosm/iosm_ipc_mux.c index fc928b298a984..b846889fcb099 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_mux.c +++ b/drivers/net/wwan/iosm/iosm_ipc_mux.c @@ -456,6 +456,7 @@ void ipc_mux_deinit(struct iosm_mux *ipc_mux) struct sk_buff_head *free_list; union mux_msg mux_msg; struct sk_buff *skb; + int i; if (!ipc_mux->initialized) return; @@ -479,5 +480,10 @@ void ipc_mux_deinit(struct iosm_mux *ipc_mux) ipc_mux->channel->dl_pipe.is_open = false; } + if (ipc_mux->protocol != MUX_LITE) { + for (i = 0; i < IPC_MEM_MUX_IP_SESSION_ENTRIES; i++) + kfree(ipc_mux->ul_adb.pp_qlt[i]); + } + kfree(ipc_mux); } diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c index f8bc9a39bfa30..1d7e3ad900c12 100644 --- a/drivers/net/wwan/mhi_wwan_mbim.c +++ b/drivers/net/wwan/mhi_wwan_mbim.c @@ -98,7 +98,8 @@ static struct mhi_mbim_link *mhi_mbim_get_link_rcu(struct mhi_mbim_context *mbim static int mhi_mbim_get_link_mux_id(struct mhi_controller *cntrl) { if (strcmp(cntrl->name, "foxconn-dw5934e") == 0 || - strcmp(cntrl->name, "foxconn-t99w640") == 0) + strcmp(cntrl->name, "foxconn-t99w640") == 0 || + strcmp(cntrl->name, "foxconn-t99w760") == 0) return WDS_BIND_MUX_DATA_PORT_MUX_ID; return 0; diff --git a/drivers/net/wwan/t7xx/t7xx_hif_dpmaif_rx.c b/drivers/net/wwan/t7xx/t7xx_hif_dpmaif_rx.c index 2310493203d3c..d9f10df03a5db 100644 --- a/drivers/net/wwan/t7xx/t7xx_hif_dpmaif_rx.c +++ b/drivers/net/wwan/t7xx/t7xx_hif_dpmaif_rx.c @@ -395,6 +395,7 @@ static int t7xx_dpmaif_set_frag_to_skb(const struct dpmaif_rx_queue *rxq, struct sk_buff *skb) { unsigned long long data_bus_addr, data_base_addr; + struct skb_shared_info *shinfo = skb_shinfo(skb); struct device *dev = rxq->dpmaif_ctrl->dev; struct dpmaif_bat_page *page_info; unsigned int data_len; @@ -402,18 +403,22 @@ static int t7xx_dpmaif_set_frag_to_skb(const struct dpmaif_rx_queue *rxq, page_info = rxq->bat_frag->bat_skb; page_info += t7xx_normal_pit_bid(pkt_info); - dma_unmap_page(dev, page_info->data_bus_addr, page_info->data_len, DMA_FROM_DEVICE); if (!page_info->page) return -EINVAL; + if (shinfo->nr_frags >= MAX_SKB_FRAGS) + return -EINVAL; + + dma_unmap_page(dev, page_info->data_bus_addr, page_info->data_len, DMA_FROM_DEVICE); + data_bus_addr = le32_to_cpu(pkt_info->pd.data_addr_h); data_bus_addr = (data_bus_addr << 32) + le32_to_cpu(pkt_info->pd.data_addr_l); data_base_addr = page_info->data_bus_addr; data_offset = data_bus_addr - data_base_addr; data_offset += page_info->offset; data_len = FIELD_GET(PD_PIT_DATA_LEN, le32_to_cpu(pkt_info->header)); - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page, + skb_add_rx_frag(skb, shinfo->nr_frags, page_info->page, data_offset, data_len, page_info->data_len); page_info->page = NULL; diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index a78a25b872409..61b547aab286a 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -735,10 +735,11 @@ static void connect(struct backend_info *be) */ requested_num_queues = xenbus_read_unsigned(dev->otherend, "multi-queue-num-queues", 1); - if (requested_num_queues > xenvif_max_queues) { + if (requested_num_queues > xenvif_max_queues || + requested_num_queues == 0) { /* buggy or malicious guest */ xenbus_dev_fatal(dev, -EINVAL, - "guest requested %u queues, exceeding the maximum of %u.", + "guest requested %u queues, but valid range is 1 - %u.", requested_num_queues, xenvif_max_queues); return; } diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c index 049662ffdf972..b3d34433bd14a 100644 --- a/drivers/nfc/nxp-nci/i2c.c +++ b/drivers/nfc/nxp-nci/i2c.c @@ -47,8 +47,8 @@ static int nxp_nci_i2c_set_mode(void *phy_id, { struct nxp_nci_i2c_phy *phy = (struct nxp_nci_i2c_phy *) phy_id; - gpiod_set_value(phy->gpiod_fw, (mode == NXP_NCI_MODE_FW) ? 1 : 0); - gpiod_set_value(phy->gpiod_en, (mode != NXP_NCI_MODE_COLD) ? 1 : 0); + gpiod_set_value_cansleep(phy->gpiod_fw, (mode == NXP_NCI_MODE_FW) ? 1 : 0); + gpiod_set_value_cansleep(phy->gpiod_en, (mode != NXP_NCI_MODE_COLD) ? 1 : 0); usleep_range(10000, 15000); if (mode == NXP_NCI_MODE_COLD) @@ -305,7 +305,7 @@ static int nxp_nci_i2c_probe(struct i2c_client *client) r = request_threaded_irq(client->irq, NULL, nxp_nci_i2c_irq_thread_fn, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_ONESHOT, NXP_NCI_I2C_DRIVER_NAME, phy); if (r < 0) nfc_err(&client->dev, "Unable to register IRQ handler\n"); diff --git a/drivers/nfc/pn533/uart.c b/drivers/nfc/pn533/uart.c index a081bce61c29f..3aacfc79fa72f 100644 --- a/drivers/nfc/pn533/uart.c +++ b/drivers/nfc/pn533/uart.c @@ -211,14 +211,22 @@ static size_t pn532_receive_buf(struct serdev_device *serdev, timer_delete(&dev->cmd_timeout); for (i = 0; i < count; i++) { + if (!dev->recv_skb) { + dev->recv_skb = alloc_skb(PN532_UART_SKB_BUFF_LEN, + GFP_KERNEL); + if (!dev->recv_skb) + return i; + } + + if (unlikely(!skb_tailroom(dev->recv_skb))) + skb_trim(dev->recv_skb, 0); + skb_put_u8(dev->recv_skb, *data++); if (!pn532_uart_rx_is_frame(dev->recv_skb)) continue; pn533_recv_frame(dev->priv, dev->recv_skb, 0); - dev->recv_skb = alloc_skb(PN532_UART_SKB_BUFF_LEN, GFP_KERNEL); - if (!dev->recv_skb) - return 0; + dev->recv_skb = NULL; } return i; diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c index ffd7367ce1194..0f12f86ebb023 100644 --- a/drivers/nfc/pn533/usb.c +++ b/drivers/nfc/pn533/usb.c @@ -406,7 +406,7 @@ static int pn533_acr122_poweron_rdr(struct pn533_usb_phy *phy) if (rc || (transferred != sizeof(cmd))) { nfc_err(&phy->udev->dev, "Reader power on cmd error %d\n", rc); - return rc; + return rc ?: -EINVAL; } rc = usb_submit_urb(phy->in_urb, GFP_KERNEL); @@ -628,6 +628,7 @@ static void pn533_usb_disconnect(struct usb_interface *interface) usb_free_urb(phy->out_urb); usb_free_urb(phy->ack_urb); kfree(phy->ack_buffer); + usb_put_dev(phy->udev); nfc_info(&interface->dev, "NXP PN533 NFC device disconnected\n"); } diff --git a/drivers/nfc/s3fwrn5/uart.c b/drivers/nfc/s3fwrn5/uart.c index 9c09c10c2a464..4ee481bd7e965 100644 --- a/drivers/nfc/s3fwrn5/uart.c +++ b/drivers/nfc/s3fwrn5/uart.c @@ -58,6 +58,12 @@ static size_t s3fwrn82_uart_read(struct serdev_device *serdev, size_t i; for (i = 0; i < count; i++) { + if (!phy->recv_skb) { + phy->recv_skb = alloc_skb(NCI_SKB_BUFF_LEN, GFP_KERNEL); + if (!phy->recv_skb) + return i; + } + skb_put_u8(phy->recv_skb, *data++); if (phy->recv_skb->len < S3FWRN82_NCI_HEADER) @@ -69,9 +75,7 @@ static size_t s3fwrn82_uart_read(struct serdev_device *serdev, s3fwrn5_recv_frame(phy->common.ndev, phy->recv_skb, phy->common.mode); - phy->recv_skb = alloc_skb(NCI_SKB_BUFF_LEN, GFP_KERNEL); - if (!phy->recv_skb) - return 0; + phy->recv_skb = NULL; } return i; diff --git a/drivers/nfc/virtual_ncidev.c b/drivers/nfc/virtual_ncidev.c index 9ef8ef2d4363a..b957fce83b7c8 100644 --- a/drivers/nfc/virtual_ncidev.c +++ b/drivers/nfc/virtual_ncidev.c @@ -125,10 +125,6 @@ static ssize_t virtual_ncidev_write(struct file *file, kfree_skb(skb); return -EFAULT; } - if (strnlen(skb->data, count) != count) { - kfree_skb(skb); - return -EINVAL; - } nci_recv_frame(vdev->ndev, skb); return count; diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index f851397b65d6e..0536521fa6ccc 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -1202,7 +1202,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) sndev->mmio_self_ctrl); sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); - sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); + if (sndev->nr_lut_mw) + sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", sndev->nr_direct_mw, sndev->nr_lut_mw); @@ -1212,7 +1213,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) sndev->peer_nr_lut_mw = ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); - sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); + if (sndev->peer_nr_lut_mw) + sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); @@ -1314,6 +1316,12 @@ static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) for (i = 0; i < sndev->nr_lut_mw; i++) { int idx = sndev->nr_direct_mw + i; + if (idx >= MAX_MWS) { + dev_err(&sndev->stdev->dev, + "Total number of MW cannot be bigger than %d", MAX_MWS); + break; + } + sndev->self_shared->mw_sizes[idx] = LUT_SIZE; } } diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index eb875e3db2e3b..2cee3c1729c35 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -1236,9 +1236,9 @@ static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, qp->tx_max_entry = tx_size / qp->tx_max_frame; if (nt->debugfs_node_dir) { - char debugfs_name[4]; + char debugfs_name[8]; - snprintf(debugfs_name, 4, "qp%d", qp_num); + snprintf(debugfs_name, sizeof(debugfs_name), "qp%d", qp_num); qp->debugfs_dir = debugfs_create_dir(debugfs_name, nt->debugfs_node_dir); @@ -1394,6 +1394,7 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) goto err2; } + mutex_init(&nt->link_event_lock); INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); @@ -1809,12 +1810,13 @@ static void ntb_tx_copy_callback(void *data, static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) { -#ifdef ARCH_HAS_NOCACHE_UACCESS +#ifdef copy_to_nontemporal /* * Using non-temporal mov to improve performance on non-cached - * writes, even though we aren't actually copying from user space. + * writes. This only works if __iomem is strictly memory-like, + * but that is the case on x86-64 */ - __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); + copy_to_nontemporal(offset, entry->buf, entry->len); #else memcpy_toio(offset, entry->buf, entry->len); #endif diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 87178a53ff9c2..8eb205459d4c0 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -486,14 +486,15 @@ EXPORT_SYMBOL_GPL(nd_synchronize); static void nd_async_device_register(void *d, async_cookie_t cookie) { struct device *dev = d; + struct device *parent = dev->parent; if (device_add(dev) != 0) { dev_err(dev, "%s: failed\n", __func__); put_device(dev); } put_device(dev); - if (dev->parent) - put_device(dev->parent); + if (parent) + put_device(parent); } static void nd_async_device_unregister(void *d, async_cookie_t cookie) diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c index c3f07be4aa22a..af82385be7c6a 100644 --- a/drivers/nvdimm/nd_virtio.c +++ b/drivers/nvdimm/nd_virtio.c @@ -44,6 +44,8 @@ static int virtio_pmem_flush(struct nd_region *nd_region) unsigned long flags; int err, err1; + guard(mutex)(&vpmem->flush_lock); + /* * Don't bother to submit the request to the device if the device is * not activated. @@ -53,7 +55,6 @@ static int virtio_pmem_flush(struct nd_region *nd_region) return -EIO; } - might_sleep(); req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); if (!req_data) return -ENOMEM; diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 2396d19ce5496..77b1966619059 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -64,6 +64,7 @@ static int virtio_pmem_probe(struct virtio_device *vdev) goto out_err; } + mutex_init(&vpmem->flush_lock); vpmem->vdev = vdev; vdev->priv = vpmem; err = init_vq(vpmem); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h index 0dddefe594c46..f72cf17f9518f 100644 --- a/drivers/nvdimm/virtio_pmem.h +++ b/drivers/nvdimm/virtio_pmem.h @@ -13,6 +13,7 @@ #include #include #include +#include #include struct virtio_pmem_request { @@ -35,6 +36,9 @@ struct virtio_pmem { /* Virtio pmem request queue */ struct virtqueue *req_vq; + /* Serialize flush requests to the device. */ + struct mutex flush_lock; + /* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus; struct nvdimm_bus_descriptor nd_desc; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f35d3f71d14f3..63819304290a1 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1703,6 +1703,7 @@ static const struct apple_nvme_hw apple_nvme_t8103_hw = { static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, + { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, }; diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c index a01178caf15bb..8f3ccb317e4de 100644 --- a/drivers/nvme/host/auth.c +++ b/drivers/nvme/host/auth.c @@ -1122,7 +1122,7 @@ void nvme_auth_free(struct nvme_ctrl *ctrl) if (ctrl->dhchap_ctxs) { for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]); - kfree(ctrl->dhchap_ctxs); + kvfree(ctrl->dhchap_ctxs); } if (ctrl->host_key) { nvme_auth_free_key(ctrl->host_key); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index f1f719351f3f2..2ba7244fdaf1c 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4865,6 +4865,13 @@ int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, if (ret) return ret; + /* + * If a previous admin queue exists (e.g., from before a reset), + * put it now before allocating a new one to avoid orphaning it. + */ + if (ctrl->admin_q) + blk_put_queue(ctrl->admin_q); + ctrl->admin_q = blk_mq_alloc_queue(set, &lim, NULL); if (IS_ERR(ctrl->admin_q)) { ret = PTR_ERR(ctrl->admin_q); diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 2e58a7ce10905..d37cb140d8323 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -592,7 +592,7 @@ bool nvmf_should_reconnect(struct nvme_ctrl *ctrl, int status) if (status > 0 && (status & NVME_STATUS_DNR)) return false; - if (status == -EKEYREJECTED) + if (status == -EKEYREJECTED || status == -ENOKEY) return false; if (ctrl->opts->max_reconnects == -1 || @@ -1290,8 +1290,8 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts) kfree(opts->subsysnqn); kfree(opts->host_traddr); kfree(opts->host_iface); - kfree(opts->dhchap_secret); - kfree(opts->dhchap_ctrl_secret); + kfree_sensitive(opts->dhchap_secret); + kfree_sensitive(opts->dhchap_ctrl_secret); kfree(opts); } EXPORT_SYMBOL_GPL(nvmf_free_options); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 2c903729b0b90..bf78faf1a4ffa 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -1468,14 +1468,14 @@ nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport, { struct fcnvme_ls_disconnect_assoc_rqst *rqst = &lsop->rqstbuf->rq_dis_assoc; - struct nvme_fc_ctrl *ctrl, *ret = NULL; + struct nvme_fc_ctrl *ctrl, *tmp, *ret = NULL; struct nvmefc_ls_rcv_op *oldls = NULL; u64 association_id = be64_to_cpu(rqst->associd.association_id); unsigned long flags; spin_lock_irqsave(&rport->lock, flags); - list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { + list_for_each_entry_safe(ctrl, tmp, &rport->ctrl_list, ctrl_list) { if (!nvme_fc_ctrl_get(ctrl)) continue; spin_lock(&ctrl->lock); @@ -1488,7 +1488,9 @@ nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport, if (ret) /* leave the ctrl get reference */ break; + spin_unlock_irqrestore(&rport->lock, flags); nvme_fc_ctrl_put(ctrl); + spin_lock_irqsave(&rport->lock, flags); } spin_unlock_irqrestore(&rport->lock, flags); @@ -3582,6 +3584,8 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, ctrl->ctrl.opts = NULL; + if (ctrl->ctrl.admin_tagset) + nvme_remove_admin_tag_set(&ctrl->ctrl); /* initiate nvme ctrl ref counting teardown */ nvme_uninit_ctrl(&ctrl->ctrl); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index e35eccacee8c8..81ccdd91f7790 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -1310,13 +1310,11 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head) if (!list_empty(&head->list)) goto out; - if (head->delayed_removal_secs) { - /* - * Ensure that no one could remove this module while the head - * remove work is pending. - */ - if (!try_module_get(THIS_MODULE)) - goto out; + /* + * Ensure that no one could remove this module while the head + * remove work is pending. + */ + if (head->delayed_removal_secs && try_module_get(THIS_MODULE)) { mod_delayed_work(nvme_wq, &head->remove_work, head->delayed_removal_secs * HZ); } else { diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 72fb675a696f4..a64b4b4a18a19 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -388,7 +388,7 @@ static void nvme_dbbuf_set(struct nvme_dev *dev) /* Free memory and continue on */ nvme_dbbuf_dma_free(dev); - for (i = 1; i <= dev->online_queues; i++) + for (i = 1; i < dev->online_queues; i++) nvme_dbbuf_free(&dev->queues[i]); } } @@ -771,6 +771,32 @@ static void nvme_unmap_data(struct request *req) nvme_free_descriptors(req); } +static bool nvme_pci_prp_save_mapping(struct request *req, + struct device *dma_dev, + struct blk_dma_iter *iter) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + + if (dma_use_iova(&iod->dma_state) || !dma_need_unmap(dma_dev)) + return true; + + if (!iod->nr_dma_vecs) { + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + + iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool, + GFP_ATOMIC); + if (!iod->dma_vecs) { + iter->status = BLK_STS_RESOURCE; + return false; + } + } + + iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr; + iod->dma_vecs[iod->nr_dma_vecs].len = iter->len; + iod->nr_dma_vecs++; + return true; +} + static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev, struct blk_dma_iter *iter) { @@ -780,12 +806,7 @@ static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev, return true; if (!blk_rq_dma_map_iter_next(req, dma_dev, &iod->dma_state, iter)) return false; - if (!dma_use_iova(&iod->dma_state) && dma_need_unmap(dma_dev)) { - iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr; - iod->dma_vecs[iod->nr_dma_vecs].len = iter->len; - iod->nr_dma_vecs++; - } - return true; + return nvme_pci_prp_save_mapping(req, dma_dev, iter); } static blk_status_t nvme_pci_setup_data_prp(struct request *req, @@ -798,15 +819,8 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req, unsigned int prp_len, i; __le64 *prp_list; - if (!dma_use_iova(&iod->dma_state) && dma_need_unmap(nvmeq->dev->dev)) { - iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool, - GFP_ATOMIC); - if (!iod->dma_vecs) - return BLK_STS_RESOURCE; - iod->dma_vecs[0].addr = iter->addr; - iod->dma_vecs[0].len = iter->len; - iod->nr_dma_vecs = 1; - } + if (!nvme_pci_prp_save_mapping(req, nvmeq->dev->dev, iter)) + return iter->status; /* * PRP1 always points to the start of the DMA transfers. @@ -1148,6 +1162,7 @@ static blk_status_t nvme_prep_rq(struct request *req) iod->nr_descriptors = 0; iod->total_len = 0; iod->meta_total_len = 0; + iod->nr_dma_vecs = 0; ret = nvme_setup_cmd(req->q->queuedata, req); if (ret) @@ -1398,14 +1413,16 @@ static irqreturn_t nvme_irq_check(int irq, void *data) static void nvme_poll_irqdisable(struct nvme_queue *nvmeq) { struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); + int irq; WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags)); - disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); + irq = pci_irq_vector(pdev, nvmeq->cq_vector); + disable_irq(irq); spin_lock(&nvmeq->cq_poll_lock); nvme_poll_cq(nvmeq, NULL); spin_unlock(&nvmeq->cq_poll_lock); - enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); + enable_irq(irq); } static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) @@ -1413,7 +1430,8 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) struct nvme_queue *nvmeq = hctx->driver_data; bool found; - if (!nvme_cqe_pending(nvmeq)) + if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) || + !nvme_cqe_pending(nvmeq)) return 0; spin_lock(&nvmeq->cq_poll_lock); @@ -1461,7 +1479,10 @@ static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl) } writel(NVME_SUBSYS_RESET, dev->bar + NVME_REG_NSSR); - nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE); + + if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING) || + !nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) + goto unlock; /* * Read controller status to flush the previous write and trigger a @@ -2687,7 +2708,13 @@ static int nvme_setup_io_queues(struct nvme_dev *dev) dev->nr_write_queues = write_queues; dev->nr_poll_queues = poll_queues; - nr_io_queues = dev->nr_allocated_queues - 1; + /* + * The initial number of allocated queue slots may be too large if the + * user reduced the special queue parameters. Cap the value to the + * number we need for this round. + */ + nr_io_queues = min(nvme_max_io_queues(dev), + dev->nr_allocated_queues - 1); result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); if (result < 0) return result; @@ -3917,6 +3944,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(0x1e49, 0x0041), /* ZHITAI TiPro7000 NVMe SSD */ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, + { PCI_DEVICE(0x1fa0, 0x2283), /* Wodposit WPBSNM8-256GTP */ + .driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, }, { PCI_DEVICE(0x025e, 0xf1ac), /* SOLIDIGM P44 pro SSDPFKKW020X7 */ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */ diff --git a/drivers/nvme/host/pr.c b/drivers/nvme/host/pr.c index ca6a74607b139..fe7dbe2648158 100644 --- a/drivers/nvme/host/pr.c +++ b/drivers/nvme/host/pr.c @@ -228,7 +228,8 @@ static int nvme_pr_resv_report(struct block_device *bdev, void *data, static int nvme_pr_read_keys(struct block_device *bdev, struct pr_keys *keys_info) { - u32 rse_len, num_keys = keys_info->num_keys; + size_t rse_len; + u32 num_keys = keys_info->num_keys; struct nvme_reservation_status_ext *rse; int ret, i; bool eds; @@ -238,7 +239,10 @@ static int nvme_pr_read_keys(struct block_device *bdev, * enough to get enough keys to fill the return keys buffer. */ rse_len = struct_size(rse, regctl_eds, num_keys); - rse = kzalloc(rse_len, GFP_KERNEL); + if (rse_len > U32_MAX) + return -EINVAL; + + rse = kvzalloc(rse_len, GFP_KERNEL); if (!rse) return -ENOMEM; @@ -263,7 +267,7 @@ static int nvme_pr_read_keys(struct block_device *bdev, } free_rse: - kfree(rse); + kvfree(rse); return ret; } diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 3e378153a7817..950b7f8e8ad56 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -1586,7 +1586,7 @@ void nvmet_execute_async_event(struct nvmet_req *req) ctrl->async_event_cmds[ctrl->nr_async_event_cmds++] = req; mutex_unlock(&ctrl->lock); - queue_work(nvmet_wq, &ctrl->async_event_work); + queue_work(nvmet_aen_wq, &ctrl->async_event_work); } void nvmet_execute_keep_alive(struct nvmet_req *req) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 5d7d483bfbe37..1c5b6bab47791 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -26,6 +26,8 @@ static DEFINE_IDA(cntlid_ida); struct workqueue_struct *nvmet_wq; EXPORT_SYMBOL_GPL(nvmet_wq); +struct workqueue_struct *nvmet_aen_wq; +EXPORT_SYMBOL_GPL(nvmet_aen_wq); /* * This read/write semaphore is used to synchronize access to configuration @@ -205,7 +207,7 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, list_add_tail(&aen->entry, &ctrl->async_events); mutex_unlock(&ctrl->lock); - queue_work(nvmet_wq, &ctrl->async_event_work); + queue_work(nvmet_aen_wq, &ctrl->async_event_work); } static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid) @@ -1957,9 +1959,14 @@ static int __init nvmet_init(void) if (!nvmet_wq) goto out_free_buffered_work_queue; + nvmet_aen_wq = alloc_workqueue("nvmet-aen-wq", + WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (!nvmet_aen_wq) + goto out_free_nvmet_work_queue; + error = nvmet_init_debugfs(); if (error) - goto out_free_nvmet_work_queue; + goto out_free_nvmet_aen_work_queue; error = nvmet_init_discovery(); if (error) @@ -1975,6 +1982,8 @@ static int __init nvmet_init(void) nvmet_exit_discovery(); out_exit_debugfs: nvmet_exit_debugfs(); +out_free_nvmet_aen_work_queue: + destroy_workqueue(nvmet_aen_wq); out_free_nvmet_work_queue: destroy_workqueue(nvmet_wq); out_free_buffered_work_queue: @@ -1992,6 +2001,7 @@ static void __exit nvmet_exit(void) nvmet_exit_discovery(); nvmet_exit_debugfs(); ida_destroy(&cntlid_ida); + destroy_workqueue(nvmet_aen_wq); destroy_workqueue(nvmet_wq); destroy_workqueue(buffered_io_wq); destroy_workqueue(zbd_wq); diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index 5dffcc5becae8..305ab7ee6e760 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -492,6 +492,7 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, struct fcloop_rport *rport = remoteport->private; struct nvmet_fc_target_port *targetport = rport->targetport; struct fcloop_tport *tport; + int ret = 0; if (!targetport) { /* @@ -501,12 +502,18 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, * We end up here from delete association exchange: * nvmet_fc_xmt_disconnect_assoc sends an async request. * - * Return success because this is what LLDDs do; silently - * drop the response. + * Return success when remoteport is still online because this + * is what LLDDs do and silently drop the response. Otherwise, + * return with error to signal upper layer to perform the lsrsp + * resource cleanup. */ - lsrsp->done(lsrsp); + if (remoteport->port_state == FC_OBJSTATE_ONLINE) + lsrsp->done(lsrsp); + else + ret = -ENODEV; + kmem_cache_free(lsreq_cache, tls_req); - return 0; + return ret; } memcpy(lsreq->rspaddr, lsrsp->rspbuf, diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 8d246b8ca604c..0103815542d49 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -180,9 +180,10 @@ u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts) static void nvmet_bio_done(struct bio *bio) { struct nvmet_req *req = bio->bi_private; + blk_status_t blk_status = bio->bi_status; - nvmet_req_complete(req, blk_to_nvme_status(req, bio->bi_status)); nvmet_req_bio_put(req, bio); + nvmet_req_complete(req, blk_to_nvme_status(req, blk_status)); } #ifdef CONFIG_BLK_DEV_INTEGRITY diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index f3b09f4099f08..059fd9f356c45 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -502,6 +502,7 @@ extern struct kmem_cache *nvmet_bvec_cache; extern struct workqueue_struct *buffered_io_wq; extern struct workqueue_struct *zbd_wq; extern struct workqueue_struct *nvmet_wq; +extern struct workqueue_struct *nvmet_aen_wq; static inline void nvmet_set_result(struct nvmet_req *req, u32 result) { diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 2e78397a7373a..9c5b0f78ce8df 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -2325,6 +2325,8 @@ static int nvmet_pci_epf_epc_init(struct pci_epf *epf) return ret; } + nvmet_pci_epf_init_dma(nvme_epf); + /* Set device ID, class, etc. */ epf->header->vendorid = ctrl->tctrl->subsys->vendor_id; epf->header->subsys_vendor_id = ctrl->tctrl->subsys->subsys_vendor_id; @@ -2422,8 +2424,6 @@ static int nvmet_pci_epf_bind(struct pci_epf *epf) if (ret) return ret; - nvmet_pci_epf_init_dma(nvme_epf); - return 0; } diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 0485e25ab797c..284ab759bbce3 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -2088,6 +2088,7 @@ static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data mutex_unlock(&nvmet_rdma_queue_mutex); flush_workqueue(nvmet_wq); + flush_workqueue(nvmet_aen_wq); } static struct ib_client nvmet_rdma_ib_client = { diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 470bf37e5a637..5c8d17bcc49bd 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -349,11 +349,14 @@ static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd) cmd->req.sg = NULL; } +static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue); + static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) { struct bio_vec *iov = cmd->iov; struct scatterlist *sg; u32 length, offset, sg_offset; + unsigned int sg_remaining; int nr_pages; length = cmd->pdu_len; @@ -361,9 +364,22 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) offset = cmd->rbytes_done; cmd->sg_idx = offset / PAGE_SIZE; sg_offset = offset % PAGE_SIZE; + if (!cmd->req.sg_cnt || cmd->sg_idx >= cmd->req.sg_cnt) { + nvmet_tcp_fatal_error(cmd->queue); + return; + } sg = &cmd->req.sg[cmd->sg_idx]; + sg_remaining = cmd->req.sg_cnt - cmd->sg_idx; while (length) { + if (!sg_remaining) { + nvmet_tcp_fatal_error(cmd->queue); + return; + } + if (!sg->length || sg->length <= sg_offset) { + nvmet_tcp_fatal_error(cmd->queue); + return; + } u32 iov_len = min_t(u32, length, sg->length - sg_offset); bvec_set_page(iov, sg_page(sg), iov_len, @@ -371,6 +387,7 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) length -= iov_len; sg = sg_next(sg); + sg_remaining--; iov++; sg_offset = 0; } @@ -982,6 +999,18 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) pr_err("H2CData PDU len %u is invalid\n", cmd->pdu_len); goto err_proto; } + /* + * Ensure command data structures are initialized. We must check both + * cmd->req.sg and cmd->iov because they can have different NULL states: + * - Uninitialized commands: both NULL + * - READ commands: cmd->req.sg allocated, cmd->iov NULL + * - WRITE commands: both allocated + */ + if (unlikely(!cmd->req.sg || !cmd->iov)) { + pr_err("queue %d: H2CData PDU received for invalid command state (ttag %u)\n", + queue->idx, data->ttag); + goto err_proto; + } cmd->pdu_recv = 0; nvmet_tcp_build_pdu_iovec(cmd); queue->cmd = cmd; @@ -1992,14 +2021,13 @@ static void nvmet_tcp_listen_data_ready(struct sock *sk) trace_sk_data_ready(sk); + if (sk->sk_state != TCP_LISTEN) + return; + read_lock_bh(&sk->sk_callback_lock); port = sk->sk_user_data; - if (!port) - goto out; - - if (sk->sk_state == TCP_LISTEN) + if (port) queue_work(nvmet_wq, &port->accept_work); -out: read_unlock_bh(&sk->sk_callback_lock); } diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index e0d88d3199c11..11b098705ec62 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -30,7 +30,7 @@ source "drivers/nvmem/layouts/Kconfig" config NVMEM_AN8855_EFUSE tristate "Airoha AN8855 eFuse support" - depends on MFD_AIROHA_AN8855 || COMPILE_TEST + depends on COMPILE_TEST help Say y here to enable support for reading eFuses on Airoha AN8855 Switch. These are e.g. used to store factory programmed diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 387c88c552595..ff68fd5ad3d6f 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -831,6 +831,7 @@ static int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_nod kfree(info.name); if (ret) { of_node_put(child); + of_node_put(info.np); return ret; } } diff --git a/drivers/nvmem/imx-ocotp-ele.c b/drivers/nvmem/imx-ocotp-ele.c index 7807ec0e2d18d..4585da51f7f47 100644 --- a/drivers/nvmem/imx-ocotp-ele.c +++ b/drivers/nvmem/imx-ocotp-ele.c @@ -131,6 +131,7 @@ static int imx_ocotp_cell_pp(void *context, const char *id, int index, static void imx_ocotp_fixup_dt_cell_info(struct nvmem_device *nvmem, struct nvmem_cell_info *cell) { + cell->raw_len = round_up(cell->bytes, 4); cell->read_post_process = imx_ocotp_cell_pp; } diff --git a/drivers/nvmem/imx-ocotp.c b/drivers/nvmem/imx-ocotp.c index 7bf7656d4f963..108d78d7f6cba 100644 --- a/drivers/nvmem/imx-ocotp.c +++ b/drivers/nvmem/imx-ocotp.c @@ -589,6 +589,7 @@ MODULE_DEVICE_TABLE(of, imx_ocotp_dt_ids); static void imx_ocotp_fixup_dt_cell_info(struct nvmem_device *nvmem, struct nvmem_cell_info *cell) { + cell->raw_len = round_up(cell->bytes, 4); cell->read_post_process = imx_ocotp_cell_pp; } diff --git a/drivers/nvmem/zynqmp_nvmem.c b/drivers/nvmem/zynqmp_nvmem.c index 7da717d6c7faf..d297ff150dc02 100644 --- a/drivers/nvmem/zynqmp_nvmem.c +++ b/drivers/nvmem/zynqmp_nvmem.c @@ -66,7 +66,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, dma_addr_t dma_buf; size_t words = bytes / WORD_INBYTES; int ret; - int value; + unsigned int value; char *data; if (bytes % WORD_INBYTES != 0) { @@ -80,7 +80,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, } if (pufflag == 1 && flag == EFUSE_WRITE) { - memcpy(&value, val, bytes); + memcpy(&value, val, sizeof(value)); if ((offset == EFUSE_PUF_START_OFFSET || offset == EFUSE_PUF_MID_OFFSET) && value & P_USER_0_64_UPPER_MASK) { @@ -100,7 +100,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, if (!efuse) return -ENOMEM; - data = dma_alloc_coherent(dev, sizeof(bytes), + data = dma_alloc_coherent(dev, bytes, &dma_buf, GFP_KERNEL); if (!data) { ret = -ENOMEM; @@ -134,7 +134,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset, if (flag == EFUSE_READ) memcpy(val, data, bytes); efuse_access_err: - dma_free_coherent(dev, sizeof(bytes), + dma_free_coherent(dev, bytes, data, dma_buf); efuse_data_fail: dma_free_coherent(dev, sizeof(struct xilinx_efuse), diff --git a/drivers/of/base.c b/drivers/of/base.c index 7043acd971a0f..43e8a91566ab8 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -1895,13 +1895,17 @@ void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align)) end--; len = end - start; - if (kstrtoint(end, 10, &id) < 0) + if (kstrtoint(end, 10, &id) < 0) { + of_node_put(np); continue; + } /* Allocate an alias_prop with enough space for the stem */ ap = dt_alloc(sizeof(*ap) + len + 1, __alignof__(*ap)); - if (!ap) + if (!ap) { + of_node_put(np); continue; + } memset(ap, 0, sizeof(*ap) + len + 1); ap->alias = start; of_alias_add(ap, np, id, start, len); @@ -2045,50 +2049,81 @@ int of_find_last_cache_level(unsigned int cpu) return cache_level; } +/* + * Some DTs have an iommu-map targeting a 2-cell IOMMU node while + * specifying only 1 cell. Fortunately they all consist of value '1' + * as the 2nd cell entry with the same target, so check for that pattern. + * + * Example: + * IOMMU node: + * #iommu-cells = <2>; + * + * Device node: + * iommu-map = <0x0000 &smmu 0x0000 0x1>, + * <0x0100 &smmu 0x0100 0x1>; + */ +static bool of_check_bad_map(const __be32 *map, int len) +{ + __be32 phandle = map[1]; + + if (len % 4) + return false; + for (int i = 0; i < len; i += 4) { + if (map[i + 1] != phandle || map[i + 3] != cpu_to_be32(1)) + return false; + } + return true; +} + /** * of_map_id - Translate an ID through a downstream mapping. * @np: root complex device node. * @id: device ID to map. * @map_name: property name of the map to use. + * @cells_name: property name of target specifier cells. * @map_mask_name: optional property name of the mask to use. - * @target: optional pointer to a target device node. - * @id_out: optional pointer to receive the translated ID. + * @filter_np: optional device node to filter matches by, or NULL to match any. + * If non-NULL, only map entries targeting this node will be matched. + * @arg: pointer to a &struct of_phandle_args for the result. On success, + * @arg->args_count will be set to the number of output specifier cells + * as defined by @cells_name in the target node, and + * @arg->args[0..args_count-1] will contain the translated output + * specifier values. If a map entry was matched, @arg->np will be set + * to the target node with a reference held that the caller must release + * with of_node_put(). * * Given a device ID, look up the appropriate implementation-defined * platform ID and/or the target device which receives transactions on that - * ID, as per the "iommu-map" and "msi-map" bindings. Either of @target or - * @id_out may be NULL if only the other is required. If @target points to - * a non-NULL device node pointer, only entries targeting that node will be - * matched; if it points to a NULL value, it will receive the device node of - * the first matching target phandle, with a reference held. + * ID, as per the "iommu-map" and "msi-map" bindings. * * Return: 0 on success or a standard error code on failure. */ int of_map_id(const struct device_node *np, u32 id, - const char *map_name, const char *map_mask_name, - struct device_node **target, u32 *id_out) + const char *map_name, const char *cells_name, + const char *map_mask_name, + const struct device_node *filter_np, struct of_phandle_args *arg) { u32 map_mask, masked_id; - int map_len; + int map_bytes, map_len, offset = 0; + bool bad_map = false; const __be32 *map = NULL; - if (!np || !map_name || (!target && !id_out)) + if (!np || !map_name || !arg) return -EINVAL; - map = of_get_property(np, map_name, &map_len); + map = of_get_property(np, map_name, &map_bytes); if (!map) { - if (target) + if (filter_np) return -ENODEV; /* Otherwise, no map implies no translation */ - *id_out = id; + arg->args[0] = id; + arg->args_count = 1; return 0; } - if (!map_len || map_len % (4 * sizeof(*map))) { - pr_err("%pOF: Error: Bad %s length: %d\n", np, - map_name, map_len); - return -EINVAL; - } + if (map_bytes % sizeof(*map)) + goto err_map_len; + map_len = map_bytes / sizeof(*map); /* The default is to select all bits. */ map_mask = 0xffffffff; @@ -2101,52 +2136,148 @@ int of_map_id(const struct device_node *np, u32 id, of_property_read_u32(np, map_mask_name, &map_mask); masked_id = map_mask & id; - for ( ; map_len > 0; map_len -= 4 * sizeof(*map), map += 4) { + + while (offset < map_len) { struct device_node *phandle_node; - u32 id_base = be32_to_cpup(map + 0); - u32 phandle = be32_to_cpup(map + 1); - u32 out_base = be32_to_cpup(map + 2); - u32 id_len = be32_to_cpup(map + 3); + u32 id_base, phandle, id_len, id_off, cells = 0; + const __be32 *out_base; + + if (map_len - offset < 2) + goto err_map_len; + + id_base = be32_to_cpup(map + offset); if (id_base & ~map_mask) { - pr_err("%pOF: Invalid %s translation - %s-mask (0x%x) ignores id-base (0x%x)\n", - np, map_name, map_name, - map_mask, id_base); + pr_err("%pOF: Invalid %s translation - %s (0x%x) ignores id-base (0x%x)\n", + np, map_name, map_mask_name, map_mask, id_base); return -EFAULT; } - if (masked_id < id_base || masked_id >= id_base + id_len) - continue; - + phandle = be32_to_cpup(map + offset + 1); phandle_node = of_find_node_by_phandle(phandle); if (!phandle_node) return -ENODEV; - if (target) { - if (*target) - of_node_put(phandle_node); - else - *target = phandle_node; + if (bad_map) { + cells = 1; + } else if (of_property_read_u32(phandle_node, cells_name, &cells)) { + pr_err("%pOF: missing %s property\n", phandle_node, cells_name); + of_node_put(phandle_node); + return -EINVAL; + } - if (*target != phandle_node) - continue; + if (map_len - offset < 3 + cells) { + of_node_put(phandle_node); + goto err_map_len; } - if (id_out) - *id_out = masked_id - id_base + out_base; + if (offset == 0 && cells == 2) { + bad_map = of_check_bad_map(map, map_len); + if (bad_map) { + pr_warn_once("%pOF: %s mismatches target %s, assuming extra cell of 0\n", + np, map_name, cells_name); + cells = 1; + } + } + + out_base = map + offset + 2; + offset += 3 + cells; + + id_len = be32_to_cpup(map + offset - 1); + if (id_len > 1 && cells > 1) { + /* + * With 1 output cell we reasonably assume its value + * has a linear relationship to the input; with more, + * we'd need help from the provider to know what to do. + */ + pr_err("%pOF: Unsupported %s - cannot handle %d-ID range with %d-cell output specifier\n", + np, map_name, id_len, cells); + of_node_put(phandle_node); + return -EINVAL; + } + id_off = masked_id - id_base; + if (masked_id < id_base || id_off >= id_len) { + of_node_put(phandle_node); + continue; + } + + if (filter_np && filter_np != phandle_node) { + of_node_put(phandle_node); + continue; + } + + arg->np = phandle_node; + for (int i = 0; i < cells; i++) + arg->args[i] = id_off + be32_to_cpu(out_base[i]); + arg->args_count = cells; pr_debug("%pOF: %s, using mask %08x, id-base: %08x, out-base: %08x, length: %08x, id: %08x -> %08x\n", - np, map_name, map_mask, id_base, out_base, - id_len, id, masked_id - id_base + out_base); + np, map_name, map_mask, id_base, be32_to_cpup(out_base), + id_len, id, id_off + be32_to_cpup(out_base)); return 0; } pr_info("%pOF: no %s translation for id 0x%x on %pOF\n", np, map_name, - id, target && *target ? *target : NULL); + id, filter_np); /* Bypasses translation */ - if (id_out) - *id_out = id; + arg->args[0] = id; + arg->args_count = 1; return 0; + +err_map_len: + pr_err("%pOF: Error: Bad %s length: %d\n", np, map_name, map_bytes); + return -EINVAL; } EXPORT_SYMBOL_GPL(of_map_id); + +/** + * of_map_iommu_id - Translate an ID using "iommu-map" bindings. + * @np: root complex device node. + * @id: Requester ID of the device (e.g. PCI RID/BDF or a platform + * stream/device ID) used as the lookup key in the iommu-map table. + * @arg: pointer to a &struct of_phandle_args for the result. On success, + * @arg->args_count will be set to the number of output specifier cells + * and @arg->args[0..args_count-1] will contain the translated output + * specifier values. If a map entry was matched, @arg->np holds a + * reference to the target node that the caller must release with + * of_node_put(). + * + * Convenience wrapper around of_map_id() using "iommu-map", "#iommu-cells", + * and "iommu-map-mask". + * + * Return: 0 on success or a standard error code on failure. + */ +int of_map_iommu_id(const struct device_node *np, u32 id, + struct of_phandle_args *arg) +{ + return of_map_id(np, id, "iommu-map", "#iommu-cells", "iommu-map-mask", NULL, arg); +} +EXPORT_SYMBOL_GPL(of_map_iommu_id); + +/** + * of_map_msi_id - Translate an ID using "msi-map" bindings. + * @np: root complex device node. + * @id: Requester ID of the device (e.g. PCI RID/BDF or a platform + * stream/device ID) used as the lookup key in the msi-map table. + * @filter_np: optional MSI controller node to filter matches by, or NULL + * to match any. If non-NULL, only map entries targeting this node will + * be matched. + * @arg: pointer to a &struct of_phandle_args for the result. On success, + * @arg->args_count will be set to the number of output specifier cells + * and @arg->args[0..args_count-1] will contain the translated output + * specifier values. If a map entry was matched, @arg->np holds a + * reference to the target node that the caller must release with + * of_node_put(). + * + * Convenience wrapper around of_map_id() using "msi-map", "#msi-cells", + * and "msi-map-mask". + * + * Return: 0 on success or a standard error code on failure. + */ +int of_map_msi_id(const struct device_node *np, u32 id, + const struct device_node *filter_np, struct of_phandle_args *arg) +{ + return of_map_id(np, id, "msi-map", "#msi-cells", "msi-map-mask", filter_np, arg); +} +EXPORT_SYMBOL_GPL(of_map_msi_id); diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index 0edd639898a63..3851ce2445858 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -503,8 +503,8 @@ void __init early_init_fdt_scan_reserved_mem(void) if (!initial_boot_params) return; - fdt_scan_reserved_mem(); fdt_reserve_elfcorehdr(); + fdt_scan_reserved_mem(); /* Process header /memreserve/ fields */ for (n = 0; ; n++) { @@ -625,6 +625,47 @@ const void *__init of_get_flat_dt_prop(unsigned long node, const char *name, return fdt_getprop(initial_boot_params, node, name, size); } +const __be32 *__init of_flat_dt_get_addr_size_prop(unsigned long node, + const char *name, + int *entries) +{ + const __be32 *prop; + int len, elen = (dt_root_addr_cells + dt_root_size_cells) * sizeof(__be32); + + prop = of_get_flat_dt_prop(node, name, &len); + if (!prop || len % elen) { + *entries = 0; + return NULL; + } + + *entries = len / elen; + return prop; +} + +bool __init of_flat_dt_get_addr_size(unsigned long node, const char *name, + u64 *addr, u64 *size) +{ + const __be32 *prop; + int entries; + + prop = of_flat_dt_get_addr_size_prop(node, name, &entries); + if (!prop || entries != 1) + return false; + + of_flat_dt_read_addr_size(prop, 0, addr, size); + return true; +} + +void __init of_flat_dt_read_addr_size(const __be32 *prop, int entry_index, + u64 *addr, u64 *size) +{ + int entry_cells = dt_root_addr_cells + dt_root_size_cells; + prop += entry_cells * entry_index; + + *addr = dt_mem_next_cell(dt_root_addr_cells, &prop); + *size = dt_mem_next_cell(dt_root_size_cells, &prop); +} + /** * of_fdt_is_compatible - Return true if given node from the given blob has * compat in its compatible list @@ -812,21 +853,15 @@ static void __init early_init_dt_check_for_initrd(unsigned long node) */ static void __init early_init_dt_check_for_elfcorehdr(unsigned long node) { - const __be32 *prop; - int len; - if (!IS_ENABLED(CONFIG_CRASH_DUMP)) return; pr_debug("Looking for elfcorehdr property... "); - prop = of_get_flat_dt_prop(node, "linux,elfcorehdr", &len); - if (!prop || (len < (dt_root_addr_cells + dt_root_size_cells))) + if (!of_flat_dt_get_addr_size(node, "linux,elfcorehdr", + &elfcorehdr_addr, &elfcorehdr_size)) return; - elfcorehdr_addr = dt_mem_next_cell(dt_root_addr_cells, &prop); - elfcorehdr_size = dt_mem_next_cell(dt_root_size_cells, &prop); - pr_debug("elfcorehdr_start=0x%llx elfcorehdr_size=0x%llx\n", elfcorehdr_addr, elfcorehdr_size); } @@ -849,8 +884,9 @@ static unsigned long chosen_node_offset = -FDT_ERR_NOTFOUND; void __init early_init_dt_check_for_usable_mem_range(void) { struct memblock_region rgn[MAX_USABLE_RANGES] = {0}; - const __be32 *prop, *endp; + const __be32 *prop; int len, i; + u64 base, size; unsigned long node = chosen_node_offset; if ((long)node < 0) @@ -858,14 +894,17 @@ void __init early_init_dt_check_for_usable_mem_range(void) pr_debug("Looking for usable-memory-range property... "); - prop = of_get_flat_dt_prop(node, "linux,usable-memory-range", &len); - if (!prop || (len % (dt_root_addr_cells + dt_root_size_cells))) + prop = of_flat_dt_get_addr_size_prop(node, "linux,usable-memory-range", + &len); + if (!prop) return; - endp = prop + (len / sizeof(__be32)); - for (i = 0; i < MAX_USABLE_RANGES && prop < endp; i++) { - rgn[i].base = dt_mem_next_cell(dt_root_addr_cells, &prop); - rgn[i].size = dt_mem_next_cell(dt_root_size_cells, &prop); + len = min(len, MAX_USABLE_RANGES); + + for (i = 0; i < len; i++) { + of_flat_dt_read_addr_size(prop, i, &base, &size); + rgn[i].base = base; + rgn[i].size = size; pr_debug("cap_mem_regions[%d]: base=%pa, size=%pa\n", i, &rgn[i].base, &rgn[i].size); @@ -883,26 +922,18 @@ static void __init early_init_dt_check_kho(void) { unsigned long node = chosen_node_offset; u64 fdt_start, fdt_size, scratch_start, scratch_size; - const __be32 *p; - int l; if (!IS_ENABLED(CONFIG_KEXEC_HANDOVER) || (long)node < 0) return; - p = of_get_flat_dt_prop(node, "linux,kho-fdt", &l); - if (l != (dt_root_addr_cells + dt_root_size_cells) * sizeof(__be32)) + if (!of_flat_dt_get_addr_size(node, "linux,kho-fdt", + &fdt_start, &fdt_size)) return; - fdt_start = dt_mem_next_cell(dt_root_addr_cells, &p); - fdt_size = dt_mem_next_cell(dt_root_addr_cells, &p); - - p = of_get_flat_dt_prop(node, "linux,kho-scratch", &l); - if (l != (dt_root_addr_cells + dt_root_size_cells) * sizeof(__be32)) + if (!of_flat_dt_get_addr_size(node, "linux,kho-scratch", + &scratch_start, &scratch_size)) return; - scratch_start = dt_mem_next_cell(dt_root_addr_cells, &p); - scratch_size = dt_mem_next_cell(dt_root_addr_cells, &p); - kho_populate(fdt_start, fdt_size, scratch_start, scratch_size); } diff --git a/drivers/of/irq.c b/drivers/of/irq.c index 1cd93549d0939..70d000a530e1c 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -704,19 +704,22 @@ static int of_check_msi_parent(struct device_node *dev_node, struct device_node /** * of_msi_xlate - map a MSI ID and find relevant MSI controller node * @dev: device for which the mapping is to be done. - * @msi_np: Pointer to target MSI controller node + * @msi_np: Pointer to target MSI controller node, or NULL if the caller + * only needs the translated ID without receiving the controller node. + * If non-NULL and pointing to a non-NULL node, only entries targeting + * that node will be matched. If non-NULL and pointing to NULL, it will + * receive the first matching target node with a reference held. * @id_in: Device ID. * * Walk up the device hierarchy looking for devices with a "msi-map" * or "msi-parent" property. If found, apply the mapping to @id_in. - * If @msi_np points to a non-NULL device node pointer, only entries targeting - * that node will be matched; if it points to a NULL value, it will receive the - * device node of the first matching target phandle, with a reference held. * * Returns: The mapped MSI id. */ u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in) { + struct device_node *local_np = NULL; + struct device_node **np = msi_np ?: &local_np; struct device *parent_dev; u32 id_out = id_in; @@ -725,12 +728,26 @@ u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in) * "msi-map" or an "msi-parent" property. */ for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent) { - if (!of_map_id(parent_dev->of_node, id_in, "msi-map", - "msi-map-mask", msi_np, &id_out)) + struct of_phandle_args msi_spec = {}; + + if (!of_map_msi_id(parent_dev->of_node, id_in, *np, &msi_spec)) { + /* + * Pass-through result: no msi-map on this node (or no + * matching entry). Keep walking up the hierarchy. + */ + if (!msi_spec.np) + continue; + id_out = msi_spec.args[0]; + if (!*np) + *np = msi_spec.np; + else + of_node_put(msi_spec.np); break; - if (!of_check_msi_parent(parent_dev->of_node, msi_np)) + } + if (!of_check_msi_parent(parent_dev->of_node, np)) break; } + of_node_put(local_np); return id_out; } EXPORT_SYMBOL_GPL(of_msi_xlate); diff --git a/drivers/of/kexec.c b/drivers/of/kexec.c index 1ee2d31816aeb..c4cf3552c0183 100644 --- a/drivers/of/kexec.c +++ b/drivers/of/kexec.c @@ -128,7 +128,6 @@ int __init ima_get_kexec_buffer(void **addr, size_t *size) { int ret, len; unsigned long tmp_addr; - unsigned long start_pfn, end_pfn; size_t tmp_size; const void *prop; @@ -144,17 +143,9 @@ int __init ima_get_kexec_buffer(void **addr, size_t *size) if (!tmp_size) return -ENOENT; - /* - * Calculate the PFNs for the buffer and ensure - * they are with in addressable memory. - */ - start_pfn = PHYS_PFN(tmp_addr); - end_pfn = PHYS_PFN(tmp_addr + tmp_size - 1); - if (!page_is_ram(start_pfn) || !page_is_ram(end_pfn)) { - pr_warn("IMA buffer at 0x%lx, size = 0x%zx beyond memory\n", - tmp_addr, tmp_size); - return -EINVAL; - } + ret = ima_validate_range(tmp_addr, tmp_size); + if (ret) + return ret; *addr = __va(tmp_addr); *size = tmp_size; diff --git a/drivers/of/of_kunit_helpers.c b/drivers/of/of_kunit_helpers.c index 7b3ed5a382aaa..f6ed1af8b62aa 100644 --- a/drivers/of/of_kunit_helpers.c +++ b/drivers/of/of_kunit_helpers.c @@ -18,8 +18,9 @@ */ void of_root_kunit_skip(struct kunit *test) { - if (IS_ENABLED(CONFIG_ARM64) && IS_ENABLED(CONFIG_ACPI) && !of_root) - kunit_skip(test, "arm64+acpi doesn't populate a root node"); + if ((IS_ENABLED(CONFIG_ARM64) || IS_ENABLED(CONFIG_RISCV)) && + IS_ENABLED(CONFIG_ACPI) && !of_root) + kunit_skip(test, "arm64/riscv+acpi doesn't populate a root node"); } EXPORT_SYMBOL_GPL(of_root_kunit_skip); diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 2e9ea751ed2df..fe111d1ea7397 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -158,7 +158,7 @@ static int __init __reserved_mem_reserve_reg(unsigned long node, phys_addr_t base, size; int len; const __be32 *prop; - bool nomap; + bool nomap, default_cma; prop = of_get_flat_dt_prop(node, "reg", &len); if (!prop) @@ -171,6 +171,12 @@ static int __init __reserved_mem_reserve_reg(unsigned long node, } nomap = of_get_flat_dt_prop(node, "no-map", NULL) != NULL; + default_cma = of_get_flat_dt_prop(node, "linux,cma-default", NULL); + + if (default_cma && cma_skip_dt_default_reserved_mem()) { + pr_err("Skipping dt linux,cma-default for \"cma=\" kernel param.\n"); + return -EINVAL; + } while (len >= t_len) { base = dt_mem_next_cell(dt_root_addr_cells, &prop); @@ -230,12 +236,9 @@ static void __init __rmem_check_for_overlap(void); */ void __init fdt_scan_reserved_mem_reg_nodes(void) { - int t_len = (dt_root_addr_cells + dt_root_size_cells) * sizeof(__be32); const void *fdt = initial_boot_params; phys_addr_t base, size; - const __be32 *prop; int node, child; - int len; if (!fdt) return; @@ -256,29 +259,24 @@ void __init fdt_scan_reserved_mem_reg_nodes(void) fdt_for_each_subnode(child, fdt, node) { const char *uname; + bool default_cma = of_get_flat_dt_prop(child, "linux,cma-default", NULL); + u64 b, s; - prop = of_get_flat_dt_prop(child, "reg", &len); - if (!prop) - continue; if (!of_fdt_device_is_available(fdt, child)) continue; - - uname = fdt_get_name(fdt, child, NULL); - if (len && len % t_len != 0) { - pr_err("Reserved memory: invalid reg property in '%s', skipping node.\n", - uname); + if (default_cma && cma_skip_dt_default_reserved_mem()) continue; - } - if (len > t_len) - pr_warn("%s() ignores %d regions in node '%s'\n", - __func__, len / t_len - 1, uname); + if (!of_flat_dt_get_addr_size(child, "reg", &b, &s)) + continue; - base = dt_mem_next_cell(dt_root_addr_cells, &prop); - size = dt_mem_next_cell(dt_root_size_cells, &prop); + base = b; + size = s; - if (size) + if (size) { + uname = fdt_get_name(fdt, child, NULL); fdt_reserved_mem_save_node(child, uname, base, size); + } } /* check for overlapping reserved regions */ @@ -406,7 +404,7 @@ static int __init __reserved_mem_alloc_size(unsigned long node, const char *unam phys_addr_t base = 0, align = 0, size; int len; const __be32 *prop; - bool nomap; + bool nomap, default_cma; int ret; prop = of_get_flat_dt_prop(node, "size", &len); @@ -430,6 +428,12 @@ static int __init __reserved_mem_alloc_size(unsigned long node, const char *unam } nomap = of_get_flat_dt_prop(node, "no-map", NULL) != NULL; + default_cma = of_get_flat_dt_prop(node, "linux,cma-default", NULL); + + if (default_cma && cma_skip_dt_default_reserved_mem()) { + pr_err("Skipping dt linux,cma-default for \"cma=\" kernel param.\n"); + return -EINVAL; + } /* Need adjust the alignment to satisfy the CMA requirement */ if (IS_ENABLED(CONFIG_CMA) diff --git a/drivers/of/platform.c b/drivers/of/platform.c index f77cb19973a5d..a6dca3a005aac 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -569,7 +569,7 @@ static int __init of_platform_default_populate_init(void) node = of_find_node_by_path("/firmware"); if (node) { - of_platform_populate(node, NULL, NULL, NULL); + of_platform_default_populate(node, NULL, NULL); of_node_put(node); } diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 388e9ec2cccf8..9c184e93f50c6 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -804,11 +804,13 @@ static void __init of_unittest_property_copy(void) new = __of_prop_dup(&p1, GFP_KERNEL); unittest(new && propcmp(&p1, new), "empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); new = __of_prop_dup(&p2, GFP_KERNEL); unittest(new && propcmp(&p2, new), "non-empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); #endif } @@ -1985,7 +1987,6 @@ static void attach_node_and_children(struct device_node *np) */ static int __init unittest_data_add(void) { - void *unittest_data; void *unittest_data_align; struct device_node *unittest_data_node = NULL, *np; /* @@ -2004,7 +2005,7 @@ static int __init unittest_data_add(void) } /* creating copy */ - unittest_data = kmalloc(size + FDT_ALIGN_SIZE, GFP_KERNEL); + void *unittest_data __free(kfree) = kmalloc(size + FDT_ALIGN_SIZE, GFP_KERNEL); if (!unittest_data) return -ENOMEM; @@ -2014,12 +2015,10 @@ static int __init unittest_data_add(void) ret = of_fdt_unflatten_tree(unittest_data_align, NULL, &unittest_data_node); if (!ret) { pr_warn("%s: unflatten testcases tree failed\n", __func__); - kfree(unittest_data); return -ENODATA; } if (!unittest_data_node) { pr_warn("%s: testcases tree is empty\n", __func__); - kfree(unittest_data); return -ENODATA; } @@ -2038,7 +2037,6 @@ static int __init unittest_data_add(void) /* attach the sub-tree to live tree */ if (!of_root) { pr_warn("%s: no live tree to attach sub-tree\n", __func__); - kfree(unittest_data); rc = -ENODEV; goto unlock; } @@ -2059,6 +2057,8 @@ static int __init unittest_data_add(void) EXPECT_END(KERN_INFO, "Duplicate name in testcase-data, renamed to \"duplicate-name#1\""); + retain_and_null_ptr(unittest_data); + unlock: of_overlay_mutex_unlock(); diff --git a/drivers/opp/core.c b/drivers/opp/core.c index bba4f7daff8cb..775d4a36f2f54 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -241,7 +241,7 @@ unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp) { if (IS_ERR_OR_NULL(opp) || !opp->available) { pr_err("%s: Invalid parameters\n", __func__); - return 0; + return U32_MAX; } return opp->level; diff --git a/drivers/parisc/gsc.c b/drivers/parisc/gsc.c index a0daaa548bc39..8ba778170447e 100644 --- a/drivers/parisc/gsc.c +++ b/drivers/parisc/gsc.c @@ -154,7 +154,9 @@ static int gsc_set_affinity_irq(struct irq_data *d, const struct cpumask *dest, gsc_dev->eim = ((u32) gsc_dev->gsc_irq.txn_addr) | gsc_dev->gsc_irq.txn_data; /* switch IRQ's for devices below LASI/WAX to other CPU */ - gsc_writel(gsc_dev->eim, gsc_dev->hpa + OFFSET_IAR); + /* ASP chip (svers 0x70) does not support reprogramming */ + if (gsc_dev->gsc->id.sversion != 0x70) + gsc_writel(gsc_dev->eim, gsc_dev->hpa + OFFSET_IAR); irq_data_update_effective_affinity(d, &tmask); diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index f94f5d384362e..47e466946bedb 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -207,12 +207,6 @@ config PCI_P2PDMA P2P DMA transactions must be between devices behind the same root port. - Enabling this option will reduce the entropy of x86 KASLR memory - regions. For example - on a 46 bit system, the entropy goes down - from 16 bits to 15 bits. The actual reduction in entropy depends - on the physical address bits, on processor features, kernel config - (5 level page table) and physical memory present on the system. - If unsure, say N. config PCI_LABEL diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c index f26aec6ff5889..67498c139b28d 100644 --- a/drivers/pci/bus.c +++ b/drivers/pci/bus.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -343,7 +344,6 @@ void __weak pcibios_bus_add_device(struct pci_dev *pdev) { } void pci_bus_add_device(struct pci_dev *dev) { struct device_node *dn = dev->dev.of_node; - struct platform_device *pdev; /* * Can not put in pci_device_add yet because resources @@ -357,23 +357,15 @@ void pci_bus_add_device(struct pci_dev *dev) pci_proc_attach_device(dev); pci_bridge_d3_update(dev); + /* Save config space for error recoverability */ + pci_save_state(dev); + /* - * If the PCI device is associated with a pwrctrl device with a - * power supply, create a device link between the PCI device and - * pwrctrl device. This ensures that pwrctrl drivers are probed - * before PCI client drivers. + * Enable runtime PM, which potentially allows the device to + * suspend immediately, only after the PCI state has been + * configured completely. */ - pdev = of_find_device_by_node(dn); - if (pdev) { - if (of_pci_supply_present(dn)) { - if (!device_link_add(&dev->dev, &pdev->dev, - DL_FLAG_AUTOREMOVE_CONSUMER)) { - pci_err(dev, "failed to add device link to power control device %s\n", - pdev->name); - } - } - put_device(&pdev->dev); - } + pm_runtime_enable(&dev->dev); if (!dn || of_device_is_available(dn)) pci_dev_allow_binding(dev); diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 41748d083b933..0452151a7bccc 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -259,12 +259,11 @@ config PCIE_RCAR_EP config PCI_RCAR_GEN2 bool "Renesas R-Car Gen2 Internal PCI controller" - depends on ARCH_RENESAS || COMPILE_TEST - depends on ARM + depends on (ARCH_RENESAS && ARM) || COMPILE_TEST help Say Y here if you want internal PCI support on R-Car Gen2 SoC. - There are 3 internal PCI controllers available with a single - built-in EHCI/OHCI host controller present on each one. + Each internal PCI controller contains a single built-in EHCI/OHCI + host controller. config PCIE_ROCKCHIP bool diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 5bc5ab20aa6d9..0413d163cfea0 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -479,7 +479,6 @@ static int j721e_pcie_probe(struct platform_device *pdev) struct cdns_pcie_ep *ep = NULL; struct gpio_desc *gpiod; void __iomem *base; - struct clk *clk; u32 num_lanes; u32 mode; int ret; @@ -603,19 +602,13 @@ static int j721e_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - clk = devm_clk_get_optional(dev, "pcie_refclk"); - if (IS_ERR(clk)) { - ret = dev_err_probe(dev, PTR_ERR(clk), "failed to get pcie_refclk\n"); + pcie->refclk = devm_clk_get_optional_enabled(dev, "pcie_refclk"); + if (IS_ERR(pcie->refclk)) { + ret = dev_err_probe(dev, PTR_ERR(pcie->refclk), + "failed to enable pcie_refclk\n"); goto err_pcie_setup; } - ret = clk_prepare_enable(clk); - if (ret) { - dev_err_probe(dev, ret, "failed to enable pcie_refclk\n"); - goto err_pcie_setup; - } - pcie->refclk = clk; - /* * Section 2.2 of the PCI Express Card Electromechanical * Specification (Revision 5.1) mandates that the deassertion @@ -628,10 +621,10 @@ static int j721e_pcie_probe(struct platform_device *pdev) gpiod_set_value_cansleep(gpiod, 1); } - ret = cdns_pcie_host_setup(rc); - if (ret < 0) { - clk_disable_unprepare(pcie->refclk); - goto err_pcie_setup; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_setup(rc); + if (ret < 0) + goto err_pcie_setup; } break; @@ -642,9 +635,11 @@ static int j721e_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - ret = cdns_pcie_ep_setup(ep); - if (ret < 0) - goto err_pcie_setup; + if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { + ret = cdns_pcie_ep_setup(ep); + if (ret < 0) + goto err_pcie_setup; + } break; } @@ -669,17 +664,17 @@ static void j721e_pcie_remove(struct platform_device *pdev) struct cdns_pcie_ep *ep; struct cdns_pcie_rc *rc; - if (pcie->mode == PCI_MODE_RC) { + if (IS_ENABLED(CONFIG_PCI_J721E_HOST) && + pcie->mode == PCI_MODE_RC) { rc = container_of(cdns_pcie, struct cdns_pcie_rc, pcie); cdns_pcie_host_disable(rc); - } else { + } else if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { ep = container_of(cdns_pcie, struct cdns_pcie_ep, pcie); cdns_pcie_ep_disable(ep); } gpiod_set_value_cansleep(pcie->reset_gpio, 0); - clk_disable_unprepare(pcie->refclk); cdns_pcie_disable_phy(cdns_pcie); j721e_pcie_disable_link_irq(pcie); pm_runtime_put(dev); @@ -739,10 +734,12 @@ static int j721e_pcie_resume_noirq(struct device *dev) gpiod_set_value_cansleep(pcie->reset_gpio, 1); } - ret = cdns_pcie_host_link_setup(rc); - if (ret < 0) { - clk_disable_unprepare(pcie->refclk); - return ret; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_link_setup(rc); + if (ret < 0) { + clk_disable_unprepare(pcie->refclk); + return ret; + } } /* @@ -752,10 +749,12 @@ static int j721e_pcie_resume_noirq(struct device *dev) for (enum cdns_pcie_rp_bar bar = RP_BAR0; bar <= RP_NO_BAR; bar++) rc->avail_ib_bar[bar] = true; - ret = cdns_pcie_host_init(rc); - if (ret) { - clk_disable_unprepare(pcie->refclk); - return ret; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_init(rc); + if (ret) { + clk_disable_unprepare(pcie->refclk); + return ret; + } } } diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c index bd683d0fecb22..d614452861f77 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.c +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -13,13 +13,13 @@ u8 cdns_pcie_find_capability(struct cdns_pcie *pcie, u8 cap) { return PCI_FIND_NEXT_CAP(cdns_pcie_read_cfg, PCI_CAPABILITY_LIST, - cap, pcie); + cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_capability); u16 cdns_pcie_find_ext_capability(struct cdns_pcie *pcie, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, pcie); + return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_ext_capability); diff --git a/drivers/pci/controller/cadence/pcie-sg2042.c b/drivers/pci/controller/cadence/pcie-sg2042.c index a077b28d48949..0c50c74d03eeb 100644 --- a/drivers/pci/controller/cadence/pcie-sg2042.c +++ b/drivers/pci/controller/cadence/pcie-sg2042.c @@ -74,15 +74,12 @@ static int sg2042_pcie_probe(struct platform_device *pdev) static void sg2042_pcie_remove(struct platform_device *pdev) { struct cdns_pcie *pcie = platform_get_drvdata(pdev); - struct device *dev = &pdev->dev; struct cdns_pcie_rc *rc; rc = container_of(pcie, struct cdns_pcie_rc, pcie); cdns_pcie_host_disable(rc); cdns_pcie_disable_phy(pcie); - - pm_runtime_disable(dev); } static int sg2042_pcie_suspend_noirq(struct device *dev) diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index 349d4657393c9..57be07198094c 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -299,7 +299,7 @@ config PCIE_QCOM select CRC8 select PCIE_QCOM_COMMON select PCI_HOST_COMMON - select PCI_PWRCTRL_SLOT + select PCI_PWRCTRL_GENERIC help Say Y here to enable PCIe controller support on Qualcomm SoCs. The PCIe controller uses the DesignWare core plus Qualcomm-specific diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 4668fc9648bff..b876b08b08aea 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -52,6 +52,8 @@ #define IMX95_PCIE_REF_CLKEN BIT(23) #define IMX95_PCIE_PHY_CR_PARA_SEL BIT(9) #define IMX95_PCIE_SS_RW_REG_1 0xf4 +#define IMX95_PCIE_CLKREQ_OVERRIDE_EN BIT(8) +#define IMX95_PCIE_CLKREQ_OVERRIDE_VAL BIT(9) #define IMX95_PCIE_SYS_AUX_PWR_DET BIT(31) #define IMX95_PE0_GEN_CTRL_1 0x1050 @@ -114,6 +116,7 @@ enum imx_pcie_variants { #define IMX_PCIE_FLAG_BROKEN_SUSPEND BIT(9) #define IMX_PCIE_FLAG_HAS_LUT BIT(10) #define IMX_PCIE_FLAG_8GT_ECN_ERR051586 BIT(11) +#define IMX_PCIE_FLAG_SKIP_L23_READY BIT(12) #define imx_check_flag(pci, val) (pci->drvdata->flags & val) @@ -706,6 +709,22 @@ static int imx7d_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) return 0; } +static void imx95_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_EN, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_EN : 0); + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_VAL, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_VAL : 0); +} + +static int imx95_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + imx95_pcie_clkreq_override(imx_pcie, enable); + return 0; +} + static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie) { struct dw_pcie *pci = imx_pcie->pci; @@ -1099,32 +1118,32 @@ static void imx_pcie_remove_lut(struct imx_pcie *imx_pcie, u16 rid) static int imx_pcie_add_lut_by_rid(struct imx_pcie *imx_pcie, u32 rid) { + struct of_phandle_args iommu_spec = {}; + struct of_phandle_args msi_spec = {}; struct device *dev = imx_pcie->pci->dev; - struct device_node *target; u32 sid_i, sid_m; int err_i, err_m; u32 sid = 0; - target = NULL; - err_i = of_map_id(dev->of_node, rid, "iommu-map", "iommu-map-mask", - &target, &sid_i); - if (target) { - of_node_put(target); - } else { + err_i = of_map_iommu_id(dev->of_node, rid, &iommu_spec); + if (!err_i) + sid_i = iommu_spec.args[0]; + of_node_put(iommu_spec.np); + if (!err_i && !iommu_spec.np) { /* - * "target == NULL && err_i == 0" means RID out of map range. - * Use 1:1 map RID to streamID. Hardware can't support this - * because the streamID is only 6 bits + * "iommu_spec.np == NULL && err_i == 0" means RID out of map + * range. Use 1:1 map RID to streamID. Hardware can't support + * this because the streamID is only 6 bits. */ err_i = -EINVAL; } - target = NULL; - err_m = of_map_id(dev->of_node, rid, "msi-map", "msi-map-mask", - &target, &sid_m); - + err_m = of_map_msi_id(dev->of_node, rid, NULL, &msi_spec); + if (!err_m) + sid_m = msi_spec.args[0]; + of_node_put(msi_spec.np); /* - * err_m target + * err_m msi_spec.np * 0 NULL RID out of range. Use 1:1 map RID to * streamID, Current hardware can't * support it, so return -EINVAL. @@ -1132,10 +1151,8 @@ static int imx_pcie_add_lut_by_rid(struct imx_pcie *imx_pcie, u32 rid) * 0 != NULL Get correct streamID from RID * != 0 != NULL Invalid combination */ - if (!err_m && !target) + if (!err_m && !msi_spec.np) return -EINVAL; - else if (target) - of_node_put(target); /* Find streamID map entry for RID in msi-map */ /* * msi-map iommu-map @@ -1777,6 +1794,8 @@ static int imx_pcie_probe(struct platform_device *pdev) */ imx_pcie_add_lut_by_rid(imx_pcie, 0); } else { + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY)) + pci->pp.skip_l23_ready = true; pci->pp.use_atu_msg = true; ret = dw_pcie_host_init(&pci->pp); if (ret < 0) @@ -1838,6 +1857,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .variant = IMX6QP, .flags = IMX_PCIE_FLAG_IMX_PHY | IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND | + IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .dbi_length = 0x200, .gpr = "fsl,imx6q-iomuxc-gpr", @@ -1854,6 +1874,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .variant = IMX7D, .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_HAS_PHY_RESET, .gpr = "fsl,imx7d-iomuxc-gpr", .mode_off[0] = IOMUXC_GPR12, @@ -1913,6 +1934,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .core_reset = imx95_pcie_core_reset, .init_phy = imx95_pcie_init_phy, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, + .enable_ref_clk = imx95_pcie_enable_ref_clk, }, [IMX8MQ_EP] = { .variant = IMX8MQ_EP, @@ -1969,6 +1991,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .core_reset = imx95_pcie_core_reset, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, .epc_features = &imx95_pcie_epc_features, + .enable_ref_clk = imx95_pcie_enable_ref_clk, .mode = DW_PCIE_EP_TYPE, }, }; diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index eb00aa3807220..25b8193ffbcf1 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -1337,6 +1337,8 @@ static int ks_pcie_probe(struct platform_device *pdev) break; default: dev_err(dev, "INVALID device type %d\n", mode); + ret = -EINVAL; + goto err_get_sync; } ks_pcie_enable_error_irq(ks_pcie); diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c index 787469d1b396d..0694084f612b7 100644 --- a/drivers/pci/controller/dwc/pci-meson.c +++ b/drivers/pci/controller/dwc/pci-meson.c @@ -37,7 +37,6 @@ #define PCIE_CFG_STATUS17 0x44 #define PM_CURRENT_STATE(x) (((x) >> 7) & 0x1) -#define WAIT_LINKUP_TIMEOUT 4000 #define PORT_CLK_RATE 100000000UL #define MAX_PAYLOAD_SIZE 256 #define MAX_READ_REQ_SIZE 256 @@ -108,10 +107,22 @@ static int meson_pcie_get_mems(struct platform_device *pdev, struct meson_pcie *mp) { struct dw_pcie *pci = &mp->pci; + struct resource *res; - pci->dbi_base = devm_platform_ioremap_resource_byname(pdev, "elbi"); - if (IS_ERR(pci->dbi_base)) - return PTR_ERR(pci->dbi_base); + /* + * For the broken DTs that supply 'dbi' as 'elbi', parse the 'elbi' + * region and assign it to both 'pci->elbi_base' and 'pci->dbi_space' so + * that the DWC core can skip parsing both regions. + */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "elbi"); + if (res) { + pci->elbi_base = devm_pci_remap_cfg_resource(pci->dev, res); + if (IS_ERR(pci->elbi_base)) + return PTR_ERR(pci->elbi_base); + + pci->dbi_base = pci->elbi_base; + pci->dbi_phys_addr = res->start; + } mp->cfg_base = devm_platform_ioremap_resource_byname(pdev, "cfg"); if (IS_ERR(mp->cfg_base)) @@ -338,40 +349,10 @@ static struct pci_ops meson_pci_ops = { static bool meson_pcie_link_up(struct dw_pcie *pci) { struct meson_pcie *mp = to_meson_pcie(pci); - struct device *dev = pci->dev; - u32 speed_okay = 0; - u32 cnt = 0; - u32 state12, state17, smlh_up, ltssm_up, rdlh_up; - - do { - state12 = meson_cfg_readl(mp, PCIE_CFG_STATUS12); - state17 = meson_cfg_readl(mp, PCIE_CFG_STATUS17); - smlh_up = IS_SMLH_LINK_UP(state12); - rdlh_up = IS_RDLH_LINK_UP(state12); - ltssm_up = IS_LTSSM_UP(state12); - - if (PM_CURRENT_STATE(state17) < PCIE_GEN3) - speed_okay = 1; - - if (smlh_up) - dev_dbg(dev, "smlh_link_up is on\n"); - if (rdlh_up) - dev_dbg(dev, "rdlh_link_up is on\n"); - if (ltssm_up) - dev_dbg(dev, "ltssm_up is on\n"); - if (speed_okay) - dev_dbg(dev, "speed_okay\n"); - - if (smlh_up && rdlh_up && ltssm_up && speed_okay) - return true; - - cnt++; - - udelay(10); - } while (cnt < WAIT_LINKUP_TIMEOUT); - - dev_err(dev, "error: wait linkup timeout\n"); - return false; + u32 state12; + + state12 = meson_cfg_readl(mp, PCIE_CFG_STATUS12); + return IS_SMLH_LINK_UP(state12) && IS_RDLH_LINK_UP(state12); } static int meson_pcie_host_init(struct dw_pcie_rp *pp) diff --git a/drivers/pci/controller/dwc/pcie-designware-debugfs.c b/drivers/pci/controller/dwc/pcie-designware-debugfs.c index 0fbf86c0b97e0..df98fee69892b 100644 --- a/drivers/pci/controller/dwc/pcie-designware-debugfs.c +++ b/drivers/pci/controller/dwc/pcie-designware-debugfs.c @@ -485,6 +485,8 @@ static const char *ltssm_status_string(enum dw_pcie_ltssm ltssm) DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ1); DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ2); DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ3); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_1); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_2); default: str = "DW_PCIE_LTSSM_UNKNOWN"; break; diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 7f2112c2fb215..e2e18beb2951d 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -72,47 +72,15 @@ EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_ep_read_cfg, PCI_CAPABILITY_LIST, - cap, ep, func_no); + cap, NULL, ep, func_no); } -/** - * dw_pcie_ep_hide_ext_capability - Hide a capability from the linked list - * @pci: DWC PCI device - * @prev_cap: Capability preceding the capability that should be hidden - * @cap: Capability that should be hidden - * - * Return: 0 if success, errno otherwise. - */ -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap) +static u16 dw_pcie_ep_find_ext_capability(struct dw_pcie_ep *ep, + u8 func_no, u8 cap) { - u16 prev_cap_offset, cap_offset; - u32 prev_cap_header, cap_header; - - prev_cap_offset = dw_pcie_find_ext_capability(pci, prev_cap); - if (!prev_cap_offset) - return -EINVAL; - - prev_cap_header = dw_pcie_readl_dbi(pci, prev_cap_offset); - cap_offset = PCI_EXT_CAP_NEXT(prev_cap_header); - cap_header = dw_pcie_readl_dbi(pci, cap_offset); - - /* cap must immediately follow prev_cap. */ - if (PCI_EXT_CAP_ID(cap_header) != cap) - return -EINVAL; - - /* Clear next ptr. */ - prev_cap_header &= ~GENMASK(31, 20); - - /* Set next ptr to next ptr of cap. */ - prev_cap_header |= cap_header & GENMASK(31, 20); - - dw_pcie_dbi_ro_wr_en(pci); - dw_pcie_writel_dbi(pci, prev_cap_offset, prev_cap_header); - dw_pcie_dbi_ro_wr_dis(pci); - - return 0; + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_ep_read_cfg, 0, + cap, NULL, ep, func_no); } -EXPORT_SYMBOL_GPL(dw_pcie_ep_hide_ext_capability); static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_header *hdr) @@ -217,22 +185,22 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, ep->bar_to_atu[bar] = 0; } -static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie *pci, +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) { u32 reg, bar_index; unsigned int offset, nbars; int i; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (!offset) return offset; - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar_index = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, reg); if (bar_index == bar) return offset; @@ -253,7 +221,7 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, u32 rebar_cap, rebar_ctrl; int ret; - rebar_offset = dw_pcie_ep_get_rebar_offset(pci, bar); + rebar_offset = dw_pcie_ep_get_rebar_offset(ep, func_no, bar); if (!rebar_offset) return -EINVAL; @@ -283,16 +251,16 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes" * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB. */ - rebar_ctrl = dw_pcie_readl_dbi(pci, rebar_offset + PCI_REBAR_CTRL); + rebar_ctrl = dw_pcie_ep_readl_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL); rebar_ctrl &= ~GENMASK(31, 16); - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); /* * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a. */ - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CAP, rebar_cap); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CAP, rebar_cap); dw_pcie_dbi_ro_wr_dis(pci); @@ -793,6 +761,9 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, writel(msg_data, ep->msi_mem + offset); + /* flush posted write before unmap */ + readl(ep->msi_mem + offset); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); return 0; @@ -835,20 +806,17 @@ void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) } EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); -static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +static void dw_pcie_ep_init_rebar_registers(struct dw_pcie_ep *ep, u8 func_no) { - struct dw_pcie_ep *ep = &pci->ep; unsigned int offset; unsigned int nbars; enum pci_barno bar; u32 reg, i, val; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); - - dw_pcie_dbi_ro_wr_en(pci); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (offset) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); /* @@ -869,16 +837,28 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) * the controller when RESBAR_CAP_REG is written, which * is why RESBAR_CAP_REG is written here. */ - val = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + val = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, val); if (ep->epf_bar[bar]) pci_epc_bar_size_to_rebar_cap(ep->epf_bar[bar]->size, &val); else val = BIT(4); - dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, val); + dw_pcie_ep_writel_dbi(ep, func_no, offset + PCI_REBAR_CAP, val); } } +} + +static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +{ + struct dw_pcie_ep *ep = &pci->ep; + u8 funcs = ep->epc->max_functions; + u8 func_no; + + dw_pcie_dbi_ro_wr_en(pci); + + for (func_no = 0; func_no < funcs; func_no++) + dw_pcie_ep_init_rebar_registers(ep, func_no); dw_pcie_setup(pci); dw_pcie_dbi_ro_wr_dis(pci); diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index e92513c5bda51..57293c180ad89 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -664,14 +664,8 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp) goto err_remove_edma; } - /* - * Note: Skip the link up delay only when a Link Up IRQ is present. - * If there is no Link Up IRQ, we should not bypass the delay - * because that would require users to manually rescan for devices. - */ - if (!pp->use_linkup_irq) - /* Ignore errors, the link may come up later */ - dw_pcie_wait_for_link(pci); + /* Ignore errors, the link may come up later */ + dw_pcie_wait_for_link(pci); ret = pci_host_probe(bridge); if (ret) @@ -942,7 +936,14 @@ static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp) dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n", pci->num_ob_windows); - pp->msg_atu_index = i; + if (pp->use_atu_msg) { + if (pci->num_ob_windows > ++i) { + pp->msg_atu_index = i; + } else { + dev_err(pci->dev, "Cannot add outbound window for MSG TLP\n"); + return -ENOMEM; + } + } i = 0; resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) { @@ -1080,6 +1081,8 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp) PCI_COMMAND_MASTER | PCI_COMMAND_SERR; dw_pcie_writel_dbi(pci, PCI_COMMAND, val); + dw_pcie_hide_unsupported_l1ss(pci); + dw_pcie_config_presets(pp); /* * If the platform provides its own child bus config accesses, it means @@ -1103,6 +1106,16 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp) dw_pcie_dbi_ro_wr_dis(pci); + /* + * If iMSI-RX module is used as the MSI controller, remove MSI and + * MSI-X capabilities from PCIe Root Ports to ensure fallback to INTx + * interrupt handling. + */ + if (pp->has_msi_ctrl) { + dw_pcie_remove_capability(pci, PCI_CAP_ID_MSI); + dw_pcie_remove_capability(pci, PCI_CAP_ID_MSIX); + } + return 0; } EXPORT_SYMBOL_GPL(dw_pcie_setup_rc); @@ -1145,17 +1158,17 @@ static int dw_pcie_pme_turn_off(struct dw_pcie *pci) int dw_pcie_suspend_noirq(struct dw_pcie *pci) { - u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + int ret = 0; u32 val; - int ret; - /* - * If L1SS is supported, then do not put the link into L2 as some - * devices such as NVMe expect low resume latency. - */ - if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1) + if (!dw_pcie_link_up(pci)) + goto stop_link; + + if (!pci_host_common_can_enter_d3cold(pci->pp.bridge)) return 0; + pci->pp.skip_pwrctrl_off = true; + if (pci->pp.ops->pme_turn_off) { pci->pp.ops->pme_turn_off(&pci->pp); } else { @@ -1164,6 +1177,16 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci) return ret; } + /* + * Some SoCs do not support reading the LTSSM register after + * PME_Turn_Off broadcast. For those SoCs, skip waiting for L2/L3 Ready + * state and wait 10ms as recommended in PCIe spec r6.0, sec 5.3.3.2.1. + */ + if (pci->pp.skip_l23_ready) { + mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000); + goto stop_link; + } + ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE || val <= DW_PCIE_LTSSM_DETECT_WAIT, @@ -1182,6 +1205,7 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci) */ udelay(1); +stop_link: dw_pcie_stop_link(pci); if (pci->pp.ops->deinit) pci->pp.ops->deinit(&pci->pp); @@ -1200,6 +1224,7 @@ int dw_pcie_resume_noirq(struct dw_pcie *pci) return 0; pci->suspended = false; + pci->pp.skip_pwrctrl_off = false; if (pci->pp.ops->init) { ret = pci->pp.ops->init(&pci->pp); diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index c644216995f69..345365ea97c74 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -168,11 +168,13 @@ int dw_pcie_get_resources(struct dw_pcie *pci) } /* ELBI is an optional resource */ - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "elbi"); - if (res) { - pci->elbi_base = devm_ioremap_resource(pci->dev, res); - if (IS_ERR(pci->elbi_base)) - return PTR_ERR(pci->elbi_base); + if (!pci->elbi_base) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "elbi"); + if (res) { + pci->elbi_base = devm_ioremap_resource(pci->dev, res); + if (IS_ERR(pci->elbi_base)) + return PTR_ERR(pci->elbi_base); + } } /* LLDD is supposed to manually switch the clocks and resets state */ @@ -224,16 +226,69 @@ void dw_pcie_version_detect(struct dw_pcie *pci) u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, - pci); + NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_capability); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, pci); + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_ext_capability); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap) +{ + u8 cap_pos, pre_pos, next_pos; + u16 reg; + + cap_pos = PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, + &pre_pos, pci); + if (!cap_pos) + return; + + reg = dw_pcie_readw_dbi(pci, cap_pos); + next_pos = (reg & 0xff00) >> 8; + + dw_pcie_dbi_ro_wr_en(pci); + if (pre_pos == PCI_CAPABILITY_LIST) + dw_pcie_writeb_dbi(pci, PCI_CAPABILITY_LIST, next_pos); + else + dw_pcie_writeb_dbi(pci, pre_pos + 1, next_pos); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_capability); + +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap) +{ + int cap_pos, next_pos, pre_pos; + u32 pre_header, header; + + cap_pos = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, &pre_pos, pci); + if (!cap_pos) + return; + + header = dw_pcie_readl_dbi(pci, cap_pos); + /* + * If the first cap at offset PCI_CFG_SPACE_SIZE is removed, + * only set it's capid to zero as it cannot be skipped. + */ + if (cap_pos == PCI_CFG_SPACE_SIZE) { + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, cap_pos, header & 0xffff0000); + dw_pcie_dbi_ro_wr_dis(pci); + return; + } + + pre_header = dw_pcie_readl_dbi(pci, pre_pos); + next_pos = PCI_EXT_CAP_NEXT(header); + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, pre_pos, + (pre_header & 0xfffff) | (next_pos << 20)); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_ext_capability); + static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, u16 vsec_id) { @@ -244,7 +299,7 @@ static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, return 0; while ((vsec = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, vsec, - PCI_EXT_CAP_ID_VNDR, pci))) { + PCI_EXT_CAP_ID_VNDR, NULL, pci))) { header = dw_pcie_readl_dbi(pci, vsec + PCI_VNDR_HEADER); if (PCI_VNDR_HEADER_ID(header) == vsec_id) return vsec; @@ -1081,6 +1136,30 @@ void dw_pcie_edma_remove(struct dw_pcie *pci) dw_edma_remove(&pci->edma); } +void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci) +{ + u16 l1ss; + u32 l1ss_cap; + + if (pci->l1ss_support) + return; + + l1ss = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); + if (!l1ss) + return; + + /* + * Unless the driver claims "l1ss_support", don't advertise L1 PM + * Substates because they require CLKREQ# and possibly other + * device-specific configuration. + */ + l1ss_cap = dw_pcie_readl_dbi(pci, l1ss + PCI_L1SS_CAP); + l1ss_cap &= ~(PCI_L1SS_CAP_PCIPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_1 | + PCI_L1SS_CAP_PCIPM_L1_2 | PCI_L1SS_CAP_ASPM_L1_2 | + PCI_L1SS_CAP_L1_PM_SS); + dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, l1ss_cap); +} + void dw_pcie_setup(struct dw_pcie *pci) { u32 val; diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index e995f692a1ecd..bc34858c0851f 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -26,6 +26,7 @@ #include #include +#include "../pci-host-common.h" #include "../../pci.h" /* DWC PCIe IP-core versions (native support since v4.70a) */ @@ -97,7 +98,7 @@ #define PORT_LANE_SKEW_INSERT_MASK GENMASK(23, 0) #define PCIE_PORT_DEBUG0 0x728 -#define PORT_LOGIC_LTSSM_STATE_MASK 0x1f +#define PORT_LOGIC_LTSSM_STATE_MASK 0x3f #define PORT_LOGIC_LTSSM_STATE_L0 0x11 #define PCIE_PORT_DEBUG1 0x72C #define PCIE_PORT_DEBUG1_LINK_UP BIT(4) @@ -380,6 +381,10 @@ enum dw_pcie_ltssm { DW_PCIE_LTSSM_RCVRY_EQ2 = 0x22, DW_PCIE_LTSSM_RCVRY_EQ3 = 0x23, + /* Vendor glue drivers provide pseudo L1 substates from get_ltssm() */ + DW_PCIE_LTSSM_L1_1 = 0x141, + DW_PCIE_LTSSM_L1_2 = 0x142, + DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF, }; @@ -426,11 +431,12 @@ struct dw_pcie_rp { bool use_atu_msg; int msg_atu_index; struct resource *msg_res; - bool use_linkup_irq; struct pci_eq_presets presets; struct pci_config_window *cfg; bool ecam_enabled; bool native_ecam; + bool skip_l23_ready; + bool skip_pwrctrl_off; }; struct dw_pcie_ep_ops { @@ -516,6 +522,7 @@ struct dw_pcie { int max_link_speed; u8 n_fts[2]; struct dw_edma_chip edma; + bool l1ss_support; /* L1 PM Substates support */ struct clk_bulk_data app_clks[DW_PCIE_NUM_APP_CLKS]; struct clk_bulk_data core_clks[DW_PCIE_NUM_CORE_CLKS]; struct reset_control_bulk_data app_rsts[DW_PCIE_NUM_APP_RSTS]; @@ -552,6 +559,8 @@ void dw_pcie_version_detect(struct dw_pcie *pci); u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_rasdes_capability(struct dw_pcie *pci); u16 dw_pcie_find_ptm_capability(struct dw_pcie *pci); @@ -573,6 +582,7 @@ int dw_pcie_prog_ep_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, int type, u64 parent_bus_addr, u8 bar, size_t size); void dw_pcie_disable_atu(struct dw_pcie *pci, u32 dir, int index); +void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci); void dw_pcie_setup(struct dw_pcie *pci); void dw_pcie_iatu_detect(struct dw_pcie *pci); int dw_pcie_edma_detect(struct dw_pcie *pci); @@ -880,7 +890,6 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num); void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap); struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no); #else @@ -938,12 +947,6 @@ static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { } -static inline int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, - u8 prev_cap, u8 cap) -{ - return 0; -} - static inline struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) { diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index 3e2752c7dd096..c2c36290be061 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -62,6 +62,17 @@ /* Interrupt Mask Register Related to Miscellaneous Operation */ #define PCIE_CLIENT_INTR_MASK_MISC 0x24 +/* Power Management Control Register */ +#define PCIE_CLIENT_POWER_CON 0x2c +#define PCIE_CLKREQ_READY FIELD_PREP_WM16(BIT(0), 1) +#define PCIE_CLKREQ_NOT_READY FIELD_PREP_WM16(BIT(0), 0) +#define PCIE_CLKREQ_PULL_DOWN FIELD_PREP_WM16(GENMASK(13, 12), 1) + +/* RASDES TBA information */ +#define PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN 0x154 +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_1 BIT(4) +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_2 BIT(5) + /* Hot Reset Control Register */ #define PCIE_CLIENT_HOT_RESET_CTRL 0x180 #define PCIE_LTSSM_APP_DLY2_EN BIT(1) @@ -74,6 +85,8 @@ #define PCIE_LINKUP_MASK GENMASK(17, 16) #define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0) +#define PCIE_TYPE0_HDR_DBI2_OFFSET 0x100000 + struct rockchip_pcie { struct dw_pcie pci; void __iomem *apb_base; @@ -85,6 +98,7 @@ struct rockchip_pcie { struct regulator *vpcie3v3; struct irq_domain *irq_domain; const struct rockchip_pcie_of_data *data; + bool supports_clkreq; }; struct rockchip_pcie_of_data { @@ -175,11 +189,26 @@ static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) return 0; } -static u32 rockchip_pcie_get_ltssm(struct rockchip_pcie *rockchip) +static u32 rockchip_pcie_get_ltssm_reg(struct rockchip_pcie *rockchip) { return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS); } +static enum dw_pcie_ltssm rockchip_pcie_get_ltssm(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + u32 val = rockchip_pcie_readl_apb(rockchip, + PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN); + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_1) + return DW_PCIE_LTSSM_L1_1; + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_2) + return DW_PCIE_LTSSM_L1_2; + + return rockchip_pcie_get_ltssm_reg(rockchip) & PCIE_LTSSM_STATUS_MASK; +} + static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip) { rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM, @@ -195,11 +224,40 @@ static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip) static bool rockchip_pcie_link_up(struct dw_pcie *pci) { struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); - u32 val = rockchip_pcie_get_ltssm(rockchip); + u32 val = rockchip_pcie_get_ltssm_reg(rockchip); return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP; } +/* + * See e.g. section '11.6.6.4 L1 Substate' in the RK3588 TRM V1.0 for the steps + * needed to support L1 substates. Currently, just enable L1 substates for RC + * mode if CLKREQ# is properly connected and supports-clkreq is present in DT. + * For EP mode, there are more things should be done to actually save power in + * L1 substates, so disable L1 substates until there is proper support. + */ +static void rockchip_pcie_configure_l1ss(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + + /* Enable L1 substates if CLKREQ# is properly connected */ + if (rockchip->supports_clkreq) { + rockchip_pcie_writel_apb(rockchip, PCIE_CLKREQ_READY, + PCIE_CLIENT_POWER_CON); + pci->l1ss_support = true; + return; + } + + /* + * Otherwise, assert CLKREQ# unconditionally. Since + * pci->l1ss_support is not set, the DWC core will prevent L1 + * Substates support from being advertised. + */ + rockchip_pcie_writel_apb(rockchip, + PCIE_CLKREQ_PULL_DOWN | PCIE_CLKREQ_NOT_READY, + PCIE_CLIENT_POWER_CON); +} + static void rockchip_pcie_enable_l0s(struct dw_pcie *pci) { u32 cap, lnkcap; @@ -257,6 +315,8 @@ static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) if (irq < 0) return irq; + pci->dbi_base2 = pci->dbi_base + PCIE_TYPE0_HDR_DBI2_OFFSET; + ret = rockchip_pcie_init_irq_domain(rockchip); if (ret < 0) dev_err(dev, "failed to init irq domain\n"); @@ -264,8 +324,13 @@ static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) irq_set_chained_handler_and_data(irq, rockchip_pcie_intx_handler, rockchip); + rockchip_pcie_configure_l1ss(pci); rockchip_pcie_enable_l0s(pci); + /* Disable Root Ports BAR0 and BAR1 as they report bogus size */ + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, 0x0); + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_1, 0x0); + return 0; } @@ -291,9 +356,7 @@ static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep) if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep")) return; - if (dw_pcie_ep_hide_ext_capability(pci, PCI_EXT_CAP_ID_SECPCI, - PCI_EXT_CAP_ID_ATS)) - dev_err(dev, "failed to hide ATS capability\n"); + dw_pcie_remove_ext_capability(pci, PCI_EXT_CAP_ID_ATS); } static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep) @@ -412,6 +475,9 @@ static int rockchip_pcie_resource_get(struct platform_device *pdev, return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst), "failed to get reset lines\n"); + rockchip->supports_clkreq = of_property_read_bool(pdev->dev.of_node, + "supports-clkreq"); + return 0; } @@ -446,36 +512,9 @@ static const struct dw_pcie_ops dw_pcie_ops = { .link_up = rockchip_pcie_link_up, .start_link = rockchip_pcie_start_link, .stop_link = rockchip_pcie_stop_link, + .get_ltssm = rockchip_pcie_get_ltssm, }; -static irqreturn_t rockchip_pcie_rc_sys_irq_thread(int irq, void *arg) -{ - struct rockchip_pcie *rockchip = arg; - struct dw_pcie *pci = &rockchip->pci; - struct dw_pcie_rp *pp = &pci->pp; - struct device *dev = pci->dev; - u32 reg; - - reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC); - rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); - - dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); - dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); - - if (reg & PCIE_RDLH_LINK_UP_CHGED) { - if (rockchip_pcie_link_up(pci)) { - msleep(PCIE_RESET_CONFIG_WAIT_MS); - dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); - /* Rescan the bus to enumerate endpoint devices */ - pci_lock_rescan_remove(); - pci_rescan_bus(pp->bridge->bus); - pci_unlock_rescan_remove(); - } - } - - return IRQ_HANDLED; -} - static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) { struct rockchip_pcie *rockchip = arg; @@ -487,7 +526,7 @@ static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); - dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); + dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip)); if (reg & PCIE_LINK_REQ_RST_NOT_INT) { dev_dbg(dev, "hot reset or link-down reset\n"); @@ -508,29 +547,14 @@ static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) return IRQ_HANDLED; } -static int rockchip_pcie_configure_rc(struct platform_device *pdev, - struct rockchip_pcie *rockchip) +static int rockchip_pcie_configure_rc(struct rockchip_pcie *rockchip) { - struct device *dev = &pdev->dev; struct dw_pcie_rp *pp; - int irq, ret; u32 val; if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST)) return -ENODEV; - irq = platform_get_irq_byname(pdev, "sys"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, - rockchip_pcie_rc_sys_irq_thread, - IRQF_ONESHOT, "pcie-sys-rc", rockchip); - if (ret) { - dev_err(dev, "failed to request PCIe sys IRQ\n"); - return ret; - } - /* LTSSM enable control mode */ val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1); rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); @@ -541,19 +565,8 @@ static int rockchip_pcie_configure_rc(struct platform_device *pdev, pp = &rockchip->pci.pp; pp->ops = &rockchip_pcie_host_ops; - pp->use_linkup_irq = true; - - ret = dw_pcie_host_init(pp); - if (ret) { - dev_err(dev, "failed to initialize host\n"); - return ret; - } - - /* unmask DLL up/down indicator */ - val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0); - rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); - return ret; + return dw_pcie_host_init(pp); } static int rockchip_pcie_configure_ep(struct platform_device *pdev, @@ -679,7 +692,7 @@ static int rockchip_pcie_probe(struct platform_device *pdev) switch (data->mode) { case DW_PCIE_RC_TYPE: - ret = rockchip_pcie_configure_rc(pdev, rockchip); + ret = rockchip_pcie_configure_rc(rockchip); if (ret) goto deinit_clk; break; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index c48a20602d7fa..4deed3ed5089a 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -24,7 +24,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -55,9 +57,6 @@ #define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 #define PARF_Q2A_FLUSH 0x1ac #define PARF_LTSSM 0x1b0 -#define PARF_INT_ALL_STATUS 0x224 -#define PARF_INT_ALL_CLEAR 0x228 -#define PARF_INT_ALL_MASK 0x22c #define PARF_SID_OFFSET 0x234 #define PARF_BDF_TRANSLATE_CFG 0x24c #define PARF_DBI_BASE_ADDR_V2 0x350 @@ -133,10 +132,7 @@ /* PARF_LTSSM register fields */ #define LTSSM_EN BIT(8) - -/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ -#define PARF_INT_ALL_LINK_UP BIT(13) -#define PARF_INT_MSI_DEV_0_7 GENMASK(30, 23) +#define PARF_LTSSM_STATE_MASK GENMASK(5, 0) /* PARF_NO_SNOOP_OVERRIDE register fields */ #define WR_NO_SNOOP_OVERRIDE_EN BIT(1) @@ -150,6 +146,7 @@ /* ELBI_SYS_CTRL register fields */ #define ELBI_SYS_CTRL_LT_ENABLE BIT(0) +#define ELBI_SYS_CTRL_PME_TURNOFF_MSG BIT(4) /* AXI_MSTR_RESP_COMP_CTRL0 register fields */ #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K 0x4 @@ -283,13 +280,12 @@ struct qcom_pcie { const struct qcom_pcie_cfg *cfg; struct dentry *debugfs; struct list_head ports; - bool suspended; bool use_pm_opp; }; #define to_qcom_pcie(x) dev_get_drvdata((x)->dev) -static void qcom_perst_assert(struct qcom_pcie *pcie, bool assert) +static void __qcom_pcie_perst_assert(struct qcom_pcie *pcie, bool assert) { struct qcom_pcie_port *port; int val = assert ? 1 : 0; @@ -300,16 +296,16 @@ static void qcom_perst_assert(struct qcom_pcie *pcie, bool assert) usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); } -static void qcom_ep_reset_assert(struct qcom_pcie *pcie) +static void qcom_pcie_perst_assert(struct qcom_pcie *pcie) { - qcom_perst_assert(pcie, true); + __qcom_pcie_perst_assert(pcie, true); } -static void qcom_ep_reset_deassert(struct qcom_pcie *pcie) +static void qcom_pcie_perst_deassert(struct qcom_pcie *pcie) { /* Ensure that PERST has been asserted for at least 100 ms */ msleep(PCIE_T_PVPERL_MS); - qcom_perst_assert(pcie, false); + __qcom_pcie_perst_assert(pcie, false); } static int qcom_pcie_start_link(struct dw_pcie *pci) @@ -1012,6 +1008,8 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) val &= ~REQ_NOT_ENTR_L1; writel(val, pcie->parf + PARF_PM_CTRL); + pci->l1ss_support = true; + val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); val |= EN; writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); @@ -1033,7 +1031,6 @@ static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) writel(WR_NO_SNOOP_OVERRIDE_EN | RD_NO_SNOOP_OVERRIDE_EN, pcie->parf + PARF_NO_SNOOP_OVERRIDE); - qcom_pcie_clear_aspm_l0s(pcie->pci); qcom_pcie_clear_hpc(pcie->pci); return 0; @@ -1061,6 +1058,12 @@ static void qcom_pcie_host_post_init_2_7_0(struct qcom_pcie *pcie) static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie) { struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + u32 val; + + /* Disable PCIe clocks and resets */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val |= PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); clk_bulk_disable_unprepare(res->num_clks, res->clks); @@ -1252,6 +1255,15 @@ static bool qcom_pcie_link_up(struct dw_pcie *pci) return val & PCI_EXP_LNKSTA_DLLLA; } +static enum dw_pcie_ltssm qcom_pcie_get_ltssm(struct dw_pcie *pci) +{ + struct qcom_pcie *pcie = to_qcom_pcie(pci); + u32 val; + + val = readl(pcie->parf + PARF_LTSSM); + return (enum dw_pcie_ltssm)FIELD_GET(PARF_LTSSM_STATE_MASK, val); +} + static void qcom_pcie_phy_power_off(struct qcom_pcie *pcie) { struct qcom_pcie_port *port; @@ -1286,7 +1298,7 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp) struct qcom_pcie *pcie = to_qcom_pcie(pci); int ret; - qcom_ep_reset_assert(pcie); + qcom_pcie_perst_assert(pcie); ret = pcie->cfg->ops->init(pcie); if (ret) @@ -1296,13 +1308,27 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp) if (ret) goto err_deinit; + if (!pci->suspended) { + ret = pci_pwrctrl_create_devices(pci->dev); + if (ret) + goto err_disable_phy; + } + + if (!pp->skip_pwrctrl_off) { + ret = pci_pwrctrl_power_on_devices(pci->dev); + if (ret) + goto err_pwrctrl_destroy; + } + if (pcie->cfg->ops->post_init) { ret = pcie->cfg->ops->post_init(pcie); if (ret) - goto err_disable_phy; + goto err_pwrctrl_power_off; } - qcom_ep_reset_deassert(pcie); + qcom_pcie_clear_aspm_l0s(pcie->pci); + + qcom_pcie_perst_deassert(pcie); if (pcie->cfg->ops->config_sid) { ret = pcie->cfg->ops->config_sid(pcie); @@ -1313,7 +1339,12 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp) return 0; err_assert_reset: - qcom_ep_reset_assert(pcie); + qcom_pcie_perst_assert(pcie); +err_pwrctrl_power_off: + pci_pwrctrl_power_off_devices(pci->dev); +err_pwrctrl_destroy: + if (ret != -EPROBE_DEFER) + pci_pwrctrl_destroy_devices(pci->dev); err_disable_phy: qcom_pcie_phy_power_off(pcie); err_deinit: @@ -1327,7 +1358,16 @@ static void qcom_pcie_host_deinit(struct dw_pcie_rp *pp) struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct qcom_pcie *pcie = to_qcom_pcie(pci); - qcom_ep_reset_assert(pcie); + qcom_pcie_perst_assert(pcie); + + if (!pci->pp.skip_pwrctrl_off) { + /* + * No need to destroy pwrctrl devices as this function only gets called + * during system suspend as of now. + */ + pci_pwrctrl_power_off_devices(pci->dev); + } + qcom_pcie_phy_power_off(pcie); pcie->cfg->ops->deinit(pcie); } @@ -1341,10 +1381,18 @@ static void qcom_pcie_host_post_init(struct dw_pcie_rp *pp) pcie->cfg->ops->host_post_init(pcie); } +static void qcom_pcie_host_pme_turn_off(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + + writel(ELBI_SYS_CTRL_PME_TURNOFF_MSG, pci->elbi_base + ELBI_SYS_CTRL); +} + static const struct dw_pcie_host_ops qcom_pcie_dw_ops = { .init = qcom_pcie_host_init, .deinit = qcom_pcie_host_deinit, .post_init = qcom_pcie_host_post_init, + .pme_turn_off = qcom_pcie_host_pme_turn_off, }; /* Qcom IP rev.: 2.1.0 Synopsys IP rev.: 4.01a */ @@ -1442,6 +1490,7 @@ static const struct qcom_pcie_cfg cfg_1_9_0 = { static const struct qcom_pcie_cfg cfg_1_34_0 = { .ops = &ops_1_9_0, .override_no_snoop = true, + .no_l0s = true, }; static const struct qcom_pcie_cfg cfg_2_1_0 = { @@ -1450,6 +1499,7 @@ static const struct qcom_pcie_cfg cfg_2_1_0 = { static const struct qcom_pcie_cfg cfg_2_3_2 = { .ops = &ops_2_3_2, + .no_l0s = true, }; static const struct qcom_pcie_cfg cfg_2_3_3 = { @@ -1480,6 +1530,7 @@ static const struct qcom_pcie_cfg cfg_fw_managed = { static const struct dw_pcie_ops dw_pcie_ops = { .link_up = qcom_pcie_link_up, .start_link = qcom_pcie_start_link, + .get_ltssm = qcom_pcie_get_ltssm, }; static int qcom_pcie_icc_init(struct qcom_pcie *pcie) @@ -1605,32 +1656,6 @@ static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie) qcom_pcie_link_transition_count); } -static irqreturn_t qcom_pcie_global_irq_thread(int irq, void *data) -{ - struct qcom_pcie *pcie = data; - struct dw_pcie_rp *pp = &pcie->pci->pp; - struct device *dev = pcie->pci->dev; - u32 status = readl_relaxed(pcie->parf + PARF_INT_ALL_STATUS); - - writel_relaxed(status, pcie->parf + PARF_INT_ALL_CLEAR); - - if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) { - msleep(PCIE_RESET_CONFIG_WAIT_MS); - dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); - /* Rescan the bus to enumerate endpoint devices */ - pci_lock_rescan_remove(); - pci_rescan_bus(pp->bridge->bus); - pci_unlock_rescan_remove(); - - qcom_pcie_icc_opp_update(pcie); - } else { - dev_WARN_ONCE(dev, 1, "Received unknown event. INT_STATUS: 0x%08x\n", - status); - } - - return IRQ_HANDLED; -} - static void qcom_pci_free_msi(void *ptr) { struct dw_pcie_rp *pp = (struct dw_pcie_rp *)ptr; @@ -1775,8 +1800,7 @@ static int qcom_pcie_probe(struct platform_device *pdev) struct dw_pcie_rp *pp; struct resource *res; struct dw_pcie *pci; - int ret, irq; - char *name; + int ret; pcie_cfg = of_device_get_match_data(dev); if (!pcie_cfg) { @@ -1927,37 +1951,12 @@ static int qcom_pcie_probe(struct platform_device *pdev) platform_set_drvdata(pdev, pcie); - irq = platform_get_irq_byname_optional(pdev, "global"); - if (irq > 0) - pp->use_linkup_irq = true; - ret = dw_pcie_host_init(pp); if (ret) { - dev_err(dev, "cannot initialize host\n"); + dev_err_probe(dev, ret, "cannot initialize host\n"); goto err_phy_exit; } - name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_global_irq%d", - pci_domain_nr(pp->bridge->bus)); - if (!name) { - ret = -ENOMEM; - goto err_host_deinit; - } - - if (irq > 0) { - ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - qcom_pcie_global_irq_thread, - IRQF_ONESHOT, name, pcie); - if (ret) { - dev_err_probe(&pdev->dev, ret, - "Failed to request Global IRQ\n"); - goto err_host_deinit; - } - - writel_relaxed(PARF_INT_ALL_LINK_UP | PARF_INT_MSI_DEV_0_7, - pcie->parf + PARF_INT_ALL_MASK); - } - qcom_pcie_icc_opp_update(pcie); if (pcie->mhi) @@ -1965,8 +1964,6 @@ static int qcom_pcie_probe(struct platform_device *pdev) return 0; -err_host_deinit: - dw_pcie_host_deinit(pp); err_phy_exit: list_for_each_entry_safe(port, tmp, &pcie->ports, list) { phy_exit(port->phy); @@ -1988,53 +1985,56 @@ static int qcom_pcie_suspend_noirq(struct device *dev) if (!pcie) return 0; - /* - * Set minimum bandwidth required to keep data path functional during - * suspend. - */ - if (pcie->icc_mem) { - ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1)); - if (ret) { - dev_err(dev, - "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", - ret); - return ret; - } - } + ret = dw_pcie_suspend_noirq(pcie->pci); + if (ret) + return ret; - /* - * Turn OFF the resources only for controllers without active PCIe - * devices. For controllers with active devices, the resources are kept - * ON and the link is expected to be in L0/L1 (sub)states. - * - * Turning OFF the resources for controllers with active PCIe devices - * will trigger access violation during the end of the suspend cycle, - * as kernel tries to access the PCIe devices config space for masking - * MSIs. - * - * Also, it is not desirable to put the link into L2/L3 state as that - * implies VDD supply will be removed and the devices may go into - * powerdown state. This will affect the lifetime of the storage devices - * like NVMe. - */ - if (!dw_pcie_link_up(pcie->pci)) { - qcom_pcie_host_deinit(&pcie->pci->pp); - pcie->suspended = true; - } + if (pcie->pci->suspended) + dev_pm_genpd_rpm_always_on(dev, false); + else + dev_pm_genpd_rpm_always_on(dev, true); + + if (pcie->pci->suspended) { + ret = icc_disable(pcie->icc_mem); + if (ret) + dev_err(dev, "Failed to disable PCIe-MEM interconnect path: %d\n", ret); - /* - * Only disable CPU-PCIe interconnect path if the suspend is non-S2RAM. - * Because on some platforms, DBI access can happen very late during the - * S2RAM and a non-active CPU-PCIe interconnect path may lead to NoC - * error. - */ - if (pm_suspend_target_state != PM_SUSPEND_MEM) { ret = icc_disable(pcie->icc_cpu); if (ret) dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n", ret); if (pcie->use_pm_opp) dev_pm_opp_set_opp(pcie->pci->dev, NULL); + } else { + /* + * Set minimum bandwidth required to keep data path functional during + * suspend. + */ + if (pcie->icc_mem) { + ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1)); + if (ret) { + dev_err(dev, + "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", + ret); + return ret; + } + } + + /* + * Only disable CPU-PCIe interconnect path if the suspend is non-S2RAM. + * Because on some platforms, DBI access can happen very late during the + * S2RAM and a non-active CPU-PCIe interconnect path may lead to NoC + * error. + */ + if (pm_suspend_target_state != PM_SUSPEND_MEM) { + ret = icc_disable(pcie->icc_cpu); + if (ret) + dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n", + ret); + + if (pcie->use_pm_opp) + dev_pm_opp_set_opp(pcie->pci->dev, NULL); + } } return ret; } @@ -2048,25 +2048,46 @@ static int qcom_pcie_resume_noirq(struct device *dev) if (!pcie) return 0; - if (pm_suspend_target_state != PM_SUSPEND_MEM) { + if (pcie->pci->suspended) { ret = icc_enable(pcie->icc_cpu); if (ret) { dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", ret); return ret; } - } - if (pcie->suspended) { - ret = qcom_pcie_host_init(&pcie->pci->pp); - if (ret) - return ret; + ret = icc_enable(pcie->icc_mem); + if (ret) { + dev_err(dev, "Failed to enable PCIe-MEM interconnect path: %d\n", ret); + goto disable_icc_cpu; + } - pcie->suspended = false; + /* + * Ignore -ENODEV & -EIO here since it is expected when no endpoint is + * connected to the PCIe link. + */ + ret = dw_pcie_resume_noirq(pcie->pci); + if (ret && ret != -ENODEV && ret != -EIO) + goto disable_icc_mem; + } else { + if (pm_suspend_target_state != PM_SUSPEND_MEM) { + ret = icc_enable(pcie->icc_cpu); + if (ret) { + dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", + ret); + return ret; + } + } } qcom_pcie_icc_opp_update(pcie); return 0; +disable_icc_mem: + icc_disable(pcie->icc_mem); +disable_icc_cpu: + icc_disable(pcie->icc_cpu); + + return ret; } static const struct of_device_id qcom_pcie_match[] = { diff --git a/drivers/pci/controller/dwc/pcie-sophgo.c b/drivers/pci/controller/dwc/pcie-sophgo.c index ad4baaa34ffa1..044088898819e 100644 --- a/drivers/pci/controller/dwc/pcie-sophgo.c +++ b/drivers/pci/controller/dwc/pcie-sophgo.c @@ -161,6 +161,22 @@ static void sophgo_pcie_msi_enable(struct dw_pcie_rp *pp) raw_spin_unlock_irqrestore(&pp->lock, flags); } +static void sophgo_pcie_disable_l0s_l1(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u32 offset, val; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, PCI_EXP_LNKCAP + offset); + val &= ~(PCI_EXP_LNKCAP_ASPM_L0S | PCI_EXP_LNKCAP_ASPM_L1); + dw_pcie_writel_dbi(pci, PCI_EXP_LNKCAP + offset, val); + + dw_pcie_dbi_ro_wr_dis(pci); +} + static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) { int irq; @@ -171,6 +187,8 @@ static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) irq_set_chained_handler_and_data(irq, sophgo_pcie_intx_handler, pp); + sophgo_pcie_disable_l0s_l1(pp); + sophgo_pcie_msi_enable(pp); return 0; diff --git a/drivers/pci/controller/dwc/pcie-stm32-ep.c b/drivers/pci/controller/dwc/pcie-stm32-ep.c index 3400c7cd2d88a..7d48038d576d9 100644 --- a/drivers/pci/controller/dwc/pcie-stm32-ep.c +++ b/drivers/pci/controller/dwc/pcie-stm32-ep.c @@ -37,36 +37,9 @@ static void stm32_pcie_ep_init(struct dw_pcie_ep *ep) dw_pcie_ep_reset_bar(pci, bar); } -static int stm32_pcie_enable_link(struct dw_pcie *pci) -{ - struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); - - regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, - STM32MP25_PCIECR_LTSSM_EN, - STM32MP25_PCIECR_LTSSM_EN); - - return dw_pcie_wait_for_link(pci); -} - -static void stm32_pcie_disable_link(struct dw_pcie *pci) -{ - struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); - - regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, STM32MP25_PCIECR_LTSSM_EN, 0); -} - static int stm32_pcie_start_link(struct dw_pcie *pci) { struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); - int ret; - - dev_dbg(pci->dev, "Enable link\n"); - - ret = stm32_pcie_enable_link(pci); - if (ret) { - dev_err(pci->dev, "PCIe cannot establish link: %d\n", ret); - return ret; - } enable_irq(stm32_pcie->perst_irq); @@ -77,11 +50,7 @@ static void stm32_pcie_stop_link(struct dw_pcie *pci) { struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); - dev_dbg(pci->dev, "Disable link\n"); - disable_irq(stm32_pcie->perst_irq); - - stm32_pcie_disable_link(pci); } static int stm32_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, @@ -152,6 +121,9 @@ static void stm32_pcie_perst_assert(struct dw_pcie *pci) dev_dbg(dev, "PERST asserted by host\n"); + regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_LTSSM_EN, 0); + pci_epc_deinit_notify(ep->epc); stm32_pcie_disable_resources(stm32_pcie); @@ -192,6 +164,11 @@ static void stm32_pcie_perst_deassert(struct dw_pcie *pci) pci_epc_init_notify(ep->epc); + /* Enable link training */ + regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_LTSSM_EN, + STM32MP25_PCIECR_LTSSM_EN); + return; err_disable_resources: @@ -237,6 +214,8 @@ static int stm32_add_pcie_ep(struct stm32_pcie *stm32_pcie, ep->ops = &stm32_pcie_ep_ops; + ep->page_size = stm32_pcie_epc_features.align; + ret = dw_pcie_ep_init(ep); if (ret) { dev_err(dev, "Failed to initialize ep: %d\n", ret); diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 10e74458e667b..3934757baa30c 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -703,6 +703,9 @@ static void init_host_aspm(struct tegra_pcie_dw *pcie) val |= (pcie->aspm_pwr_on_t << 19); dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); + if (pcie->supports_clkreq) + pci->l1ss_support = true; + /* Program L0s and L1 entrance latencies */ val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c index 810d1c8de24e9..aed69fc95241f 100644 --- a/drivers/pci/controller/pci-host-common.c +++ b/drivers/pci/controller/pci-host-common.c @@ -105,5 +105,34 @@ void pci_host_common_remove(struct platform_device *pdev) } EXPORT_SYMBOL_GPL(pci_host_common_remove); +static int pci_host_common_check_d3cold(struct pci_dev *pdev, void *userdata) +{ + bool *d3cold_allow = userdata; + + if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ENDPOINT) + return 0; + + if (pdev->current_state != PCI_D3hot) + goto exit; + + if (device_may_wakeup(&pdev->dev) && !pci_pme_capable(pdev, PCI_D3cold)) + goto exit; + + return 0; +exit: + *d3cold_allow = false; + return -EBUSY; +} + +bool pci_host_common_can_enter_d3cold(struct pci_host_bridge *bridge) +{ + bool d3cold_allow = true; + + pci_walk_bus(bridge->bus, pci_host_common_check_d3cold, &d3cold_allow); + + return d3cold_allow; +} +EXPORT_SYMBOL_GPL(pci_host_common_can_enter_d3cold); + MODULE_DESCRIPTION("Common library for PCI host controller drivers"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-host-common.h b/drivers/pci/controller/pci-host-common.h index 51c35ec0cf37d..5ec7e6929f4e0 100644 --- a/drivers/pci/controller/pci-host-common.h +++ b/drivers/pci/controller/pci-host-common.h @@ -19,4 +19,6 @@ void pci_host_common_remove(struct platform_device *pdev); struct pci_config_window *pci_host_common_ecam_create(struct device *dev, struct pci_host_bridge *bridge, const struct pci_ecam_ops *ops); + +bool pci_host_common_can_enter_d3cold(struct pci_host_bridge *bridge); #endif diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 146b43981b278..28b1572974879 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -2486,6 +2486,14 @@ static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus) if (!hv_dev) continue; + /* + * If the Hyper-V host doesn't provide a NUMA node for the + * device, default to node 0. With NUMA_NO_NODE the kernel + * may spread work across NUMA nodes, which degrades + * performance on Hyper-V. + */ + set_dev_node(&dev->dev, 0); + if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY && hv_dev->desc.virtual_numa_node < num_possible_nodes()) /* diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 0380d300adca6..8d4a554681969 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -782,6 +782,7 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d { u32 sid, rid = pci_dev_id(pdev); struct apple_pcie_port *port; + struct of_phandle_args iommu_spec = {}; int idx, err; port = apple_pcie_get_port(pdev); @@ -791,11 +792,12 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d dev_dbg(&pdev->dev, "added to bus %s, index %d\n", pci_name(pdev->bus->self), port->idx); - err = of_map_id(port->pcie->dev->of_node, rid, "iommu-map", - "iommu-map-mask", NULL, &sid); + err = of_map_iommu_id(port->pcie->dev->of_node, rid, &iommu_spec); if (err) return err; + of_node_put(iommu_spec.np); + sid = iommu_spec.args[0]; mutex_lock(&port->pcie->lock); idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0); diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 9afbd02ded35e..7e9b2f6a604a5 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -48,7 +48,6 @@ #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY 0x04dc #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK 0x1f0 -#define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK 0xc00 #define PCIE_RC_CFG_PRIV1_ROOT_CAP 0x4f8 #define PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK 0xf8 @@ -1075,7 +1074,7 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) void __iomem *base = pcie->base; struct pci_host_bridge *bridge; struct resource_entry *entry; - u32 tmp, burst, aspm_support, num_lanes, num_lanes_cap; + u32 tmp, burst, num_lanes, num_lanes_cap; u8 num_out_wins = 0; int num_inbound_wins = 0; int memc, ret; @@ -1175,12 +1174,9 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) /* Don't advertise L0s capability if 'aspm-no-l0s' */ - aspm_support = PCIE_LINK_STATE_L1; - if (!of_property_read_bool(pcie->np, "aspm-no-l0s")) - aspm_support |= PCIE_LINK_STATE_L0S; tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); - u32p_replace_bits(&tmp, aspm_support, - PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK); + if (of_property_read_bool(pcie->np, "aspm-no-l0s")) + tmp &= ~PCI_EXP_LNKCAP_ASPM_L0S; writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); /* 'tmp' still holds the contents of PRIV1_LINK_CAPABILITY */ diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 24cc30a2ab6c6..e0bf667c2b4c6 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -575,8 +575,10 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port, if (IS_ENABLED(CONFIG_PCI_MSI)) { ret = mtk_pcie_allocate_msi_domains(port); - if (ret) + if (ret) { + irq_domain_remove(port->irq_domain); return ret; + } } return 0; diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c index 937ea6ae1ac48..4aa139abac16e 100644 --- a/drivers/pci/controller/pcie-xilinx.c +++ b/drivers/pci/controller/pcie-xilinx.c @@ -302,9 +302,10 @@ static int xilinx_allocate_msi_domains(struct xilinx_pcie *pcie) return 0; } -static void xilinx_free_msi_domains(struct xilinx_pcie *pcie) +static void xilinx_free_irq_domains(struct xilinx_pcie *pcie) { irq_domain_remove(pcie->msi_domain); + irq_domain_remove(pcie->leg_domain); } /* INTx Functions */ @@ -480,8 +481,10 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie *pcie) phys_addr_t pa = ALIGN_DOWN(virt_to_phys(pcie), SZ_4K); ret = xilinx_allocate_msi_domains(pcie); - if (ret) + if (ret) { + irq_domain_remove(pcie->leg_domain); return ret; + } pcie_write(pcie, upper_32_bits(pa), XILINX_PCIE_REG_MSIBASE1); pcie_write(pcie, lower_32_bits(pa), XILINX_PCIE_REG_MSIBASE2); @@ -600,7 +603,7 @@ static int xilinx_pcie_probe(struct platform_device *pdev) err = pci_host_probe(bridge); if (err) - xilinx_free_msi_domains(pcie); + xilinx_free_irq_domains(pcie); return err; } diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 31617772ad516..b05e8db575c35 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -730,8 +730,9 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test, if (bar < BAR_0) goto err_doorbell_cleanup; - ret = request_irq(epf->db_msg[0].virq, pci_epf_test_doorbell_handler, 0, - "pci-ep-test-doorbell", epf_test); + ret = request_threaded_irq(epf->db_msg[0].virq, NULL, + pci_epf_test_doorbell_handler, IRQF_ONESHOT, + "pci-ep-test-doorbell", epf_test); if (ret) { dev_err(&epf->dev, "Failed to request doorbell IRQ: %d\n", diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 83e9ab10f9c4f..750a246f79c92 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -644,19 +644,6 @@ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws) } } -/** - * epf_ntb_epc_destroy() - Cleanup NTB EPC interface - * @ntb: NTB device that facilitates communication between HOST and VHOST - * - * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces - */ -static void epf_ntb_epc_destroy(struct epf_ntb *ntb) -{ - pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0); - pci_epc_put(ntb->epf->epc); -} - - /** * epf_ntb_is_bar_used() - Check if a bar is used in the ntb configuration * @ntb: NTB device that facilitates communication between HOST and VHOST @@ -836,6 +823,7 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb) */ static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) { + disable_delayed_work_sync(&ntb->cmd_handler); epf_ntb_mw_bar_clear(ntb, ntb->num_mws); epf_ntb_db_bar_clear(ntb); epf_ntb_config_sspad_bar_clear(ntb); @@ -1406,7 +1394,7 @@ static int epf_ntb_bind(struct pci_epf *epf) ret = epf_ntb_init_epc_bar(ntb); if (ret) { dev_err(dev, "Failed to create NTB EPC\n"); - goto err_bar_init; + return ret; } ret = epf_ntb_config_spad_bar_alloc(ntb); @@ -1446,9 +1434,6 @@ static int epf_ntb_bind(struct pci_epf *epf) err_bar_alloc: epf_ntb_config_spad_bar_free(ntb); -err_bar_init: - epf_ntb_epc_destroy(ntb); - return ret; } @@ -1464,7 +1449,6 @@ static void epf_ntb_unbind(struct pci_epf *epf) epf_ntb_epc_cleanup(ntb); epf_ntb_config_spad_bar_free(ntb); - epf_ntb_epc_destroy(ntb); pci_unregister_driver(&vntb_pci_driver); } diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index ef50c82e647f4..8b392a8363bb1 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -23,7 +23,6 @@ struct pci_epf_group { struct config_group group; struct config_group primary_epc_group; struct config_group secondary_epc_group; - struct delayed_work cfs_work; struct pci_epf *epf; int index; }; @@ -69,8 +68,8 @@ static int pci_secondary_epc_epf_link(struct config_item *epf_item, return 0; } -static void pci_secondary_epc_epf_unlink(struct config_item *epc_item, - struct config_item *epf_item) +static void pci_secondary_epc_epf_unlink(struct config_item *epf_item, + struct config_item *epc_item) { struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); @@ -103,7 +102,7 @@ static struct config_group secondary_epc_group = &epf_group->secondary_epc_group; config_group_init_type_name(secondary_epc_group, "secondary", &pci_secondary_epc_type); - configfs_register_group(&epf_group->group, secondary_epc_group); + configfs_add_default_group(secondary_epc_group, &epf_group->group); return secondary_epc_group; } @@ -133,8 +132,8 @@ static int pci_primary_epc_epf_link(struct config_item *epf_item, return 0; } -static void pci_primary_epc_epf_unlink(struct config_item *epc_item, - struct config_item *epf_item) +static void pci_primary_epc_epf_unlink(struct config_item *epf_item, + struct config_item *epc_item) { struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); @@ -166,7 +165,7 @@ static struct config_group config_group_init_type_name(primary_epc_group, "primary", &pci_primary_epc_type); - configfs_register_group(&epf_group->group, primary_epc_group); + configfs_add_default_group(primary_epc_group, &epf_group->group); return primary_epc_group; } @@ -570,15 +569,13 @@ static void pci_ep_cfs_add_type_group(struct pci_epf_group *epf_group) return; } - configfs_register_group(&epf_group->group, group); + configfs_add_default_group(group, &epf_group->group); } -static void pci_epf_cfs_work(struct work_struct *work) +static void pci_epf_cfs_add_sub_groups(struct pci_epf_group *epf_group) { - struct pci_epf_group *epf_group; struct config_group *group; - epf_group = container_of(work, struct pci_epf_group, cfs_work.work); group = pci_ep_cfs_add_primary_group(epf_group); if (IS_ERR(group)) { pr_err("failed to create 'primary' EPC interface\n"); @@ -637,9 +634,7 @@ static struct config_group *pci_epf_make(struct config_group *group, kfree(epf_name); - INIT_DELAYED_WORK(&epf_group->cfs_work, pci_epf_cfs_work); - queue_delayed_work(system_wq, &epf_group->cfs_work, - msecs_to_jiffies(1)); + pci_epf_cfs_add_sub_groups(epf_group); return &epf_group->group; diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 77dee43b78583..c6dc1b44bf602 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -495,7 +495,9 @@ static ssize_t sriov_numvfs_store(struct device *dev, if (num_vfs == 0) { /* disable VFs */ + pci_lock_rescan_remove(); ret = pdev->driver->sriov_configure(pdev, 0); + pci_unlock_rescan_remove(); goto exit; } @@ -507,7 +509,9 @@ static ssize_t sriov_numvfs_store(struct device *dev, goto exit; } + pci_lock_rescan_remove(); ret = pdev->driver->sriov_configure(pdev, num_vfs); + pci_unlock_rescan_remove(); if (ret < 0) goto exit; @@ -629,18 +633,15 @@ static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) if (dev->no_vf_scan) return 0; - pci_lock_rescan_remove(); for (i = 0; i < num_vfs; i++) { rc = pci_iov_add_virtfn(dev, i); if (rc) goto failed; } - pci_unlock_rescan_remove(); return 0; failed: while (i--) pci_iov_remove_virtfn(dev, i); - pci_unlock_rescan_remove(); return rc; } @@ -765,10 +766,8 @@ static void sriov_del_vfs(struct pci_dev *dev) struct pci_sriov *iov = dev->sriov; int i; - pci_lock_rescan_remove(); for (i = 0; i < iov->num_VFs; i++) pci_iov_remove_virtfn(dev, i); - pci_unlock_rescan_remove(); } static void sriov_disable(struct pci_dev *dev) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 34d664139f48f..e010ecd9f90dd 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -737,7 +737,7 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) - goto out_disable; + goto out_unmap; /* Disable INTX */ pci_intx_for_msi(dev, 0); @@ -758,6 +758,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, pcibios_free_irq(dev); return 0; +out_unmap: + iounmap(dev->msix_base); out_disable: dev->msix_enabled = 0; pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0); diff --git a/drivers/pci/of.c b/drivers/pci/of.c index 3579265f11984..9bb5f258759be 100644 --- a/drivers/pci/of.c +++ b/drivers/pci/of.c @@ -867,6 +867,7 @@ bool of_pci_supply_present(struct device_node *np) return false; } +EXPORT_SYMBOL_GPL(of_pci_supply_present); #endif /* CONFIG_PCI */ diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 78e108e47254a..49e395eb013d3 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -147,11 +147,19 @@ static int p2pmem_alloc_mmap(struct file *filp, struct kobject *kobj, * we have just allocated the page no one else should be * using it. */ - VM_WARN_ON_ONCE_PAGE(!page_ref_count(page), page); + VM_WARN_ON_ONCE_PAGE(page_ref_count(page), page); set_page_count(page, 1); ret = vm_insert_page(vma, vaddr, page); if (ret) { gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len); + + /* + * Reset the page count. We don't use put_page() + * because we don't want to trigger the + * p2pdma_folio_free() path. + */ + set_page_count(page, 0); + percpu_ref_put(ref); return ret; } percpu_ref_get(ref); diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index 9369377725fa0..0162acfb57896 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -271,21 +271,6 @@ static acpi_status decode_type1_hpx_record(union acpi_object *record, return AE_OK; } -static bool pcie_root_rcb_set(struct pci_dev *dev) -{ - struct pci_dev *rp = pcie_find_root_port(dev); - u16 lnkctl; - - if (!rp) - return false; - - pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &lnkctl); - if (lnkctl & PCI_EXP_LNKCTL_RCB) - return true; - - return false; -} - /* _HPX PCI Express Setting Record (Type 2) */ struct hpx_type2 { u32 revision; @@ -311,6 +296,7 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) { int pos; u32 reg32; + const struct pci_host_bridge *host; if (!hpx) return; @@ -318,6 +304,15 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) if (!pci_is_pcie(dev)) return; + host = pci_find_host_bridge(dev->bus); + + /* + * Only do the _HPX Type 2 programming if OS owns PCIe native + * hotplug but not AER. + */ + if (!host->native_pcie_hotplug || host->native_aer) + return; + if (hpx->revision > 1) { pci_warn(dev, "PCIe settings rev %d not supported\n", hpx->revision); @@ -325,33 +320,27 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) } /* - * Don't allow _HPX to change MPS or MRRS settings. We manage - * those to make sure they're consistent with the rest of the - * platform. + * We only allow _HPX to program DEVCTL bits related to AER, namely + * PCI_EXP_DEVCTL_CERE, PCI_EXP_DEVCTL_NFERE, PCI_EXP_DEVCTL_FERE, + * and PCI_EXP_DEVCTL_URRE. + * + * The rest of DEVCTL is managed by the OS to make sure it's + * consistent with the rest of the platform. */ - hpx->pci_exp_devctl_and |= PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ; - hpx->pci_exp_devctl_or &= ~(PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ); + hpx->pci_exp_devctl_and |= ~PCI_EXP_AER_FLAGS; + hpx->pci_exp_devctl_or &= PCI_EXP_AER_FLAGS; /* Initialize Device Control Register */ pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, ~hpx->pci_exp_devctl_and, hpx->pci_exp_devctl_or); - /* Initialize Link Control Register */ + /* Log if _HPX attempts to modify Link Control Register */ if (pcie_cap_has_lnkctl(dev)) { - - /* - * If the Root Port supports Read Completion Boundary of - * 128, set RCB to 128. Otherwise, clear it. - */ - hpx->pci_exp_lnkctl_and |= PCI_EXP_LNKCTL_RCB; - hpx->pci_exp_lnkctl_or &= ~PCI_EXP_LNKCTL_RCB; - if (pcie_root_rcb_set(dev)) - hpx->pci_exp_lnkctl_or |= PCI_EXP_LNKCTL_RCB; - - pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, - ~hpx->pci_exp_lnkctl_and, hpx->pci_exp_lnkctl_or); + if (hpx->pci_exp_lnkctl_and != 0xffff || + hpx->pci_exp_lnkctl_or != 0) + pci_info(dev, "_HPX attempts Link Control setting (AND %#06x OR %#06x)\n", + hpx->pci_exp_lnkctl_and, + hpx->pci_exp_lnkctl_or); } /* Find Advanced Error Reporting Enhanced Capability */ diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 302d61783f6c0..b4111c92c9572 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -629,6 +629,8 @@ static int pci_legacy_suspend(struct device *dev, pm_message_t state) struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *drv = pci_dev->driver; + pci_dev->state_saved = false; + if (drv && drv->suspend) { pci_power_t prev = pci_dev->current_state; int error; @@ -1036,6 +1038,8 @@ static int pci_pm_freeze(struct device *dev) if (!pm) { pci_pm_default_suspend(pci_dev); + if (!pm_runtime_suspended(dev)) + pci_dev->state_saved = false; return 0; } @@ -1648,6 +1652,14 @@ static int pci_dma_configure(struct device *dev) ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev)); } + /* + * Attempt to enable ACS regardless of capability because some Root + * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have + * the standard ACS capability but still support ACS via those + * quirks. + */ + pci_enable_acs(to_pci_dev(dev)); + pci_put_host_bridge_device(bridge); /* @drv may not be valid when we're called from the IOMMU layer */ diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index b14dd064006cc..e128696d5b761 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -426,7 +426,7 @@ static int pci_dev_str_match(struct pci_dev *dev, const char *p, static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, u8 pos, int cap) { - return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, bus, devfn); + return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, NULL, bus, devfn); } u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) @@ -531,7 +531,7 @@ u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) return 0; return PCI_FIND_NEXT_EXT_CAP(pci_bus_read_config, start, cap, - dev->bus, dev->devfn); + NULL, dev->bus, dev->devfn); } EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); @@ -600,7 +600,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) mask = HT_5BIT_CAP_MASK; pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, - PCI_CAP_ID_HT, dev->bus, dev->devfn); + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); while (pos) { rc = pci_read_config_byte(dev, pos + 3, &cap); if (rc != PCIBIOS_SUCCESSFUL) @@ -611,7 +611,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos + PCI_CAP_LIST_NEXT, - PCI_CAP_ID_HT, dev->bus, + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); } @@ -1015,7 +1015,7 @@ static void pci_std_enable_acs(struct pci_dev *dev, struct pci_acs *caps) * pci_enable_acs - enable ACS if hardware support it * @dev: the PCI device */ -static void pci_enable_acs(struct pci_dev *dev) +void pci_enable_acs(struct pci_dev *dev) { struct pci_acs caps; bool enable_acs = false; @@ -1488,6 +1488,9 @@ static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool || (state == PCI_D2 && !dev->d2_support)) return -EIO; + if (dev->current_state == state) + return 0; + pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); if (PCI_POSSIBLE_ERROR(pmcsr)) { pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n", @@ -1855,9 +1858,6 @@ static void pci_restore_rebar_state(struct pci_dev *pdev) */ void pci_restore_state(struct pci_dev *dev) { - if (!dev->state_saved) - return; - pci_restore_pcie_state(dev); pci_restore_pasid_state(dev); pci_restore_pri_state(dev); @@ -3225,8 +3225,14 @@ void pci_pm_init(struct pci_dev *dev) poweron: pci_pm_power_up_and_verify_state(dev); pm_runtime_forbid(&dev->dev); + + /* + * Runtime PM will be enabled for the device when it has been fully + * configured, but since its parent and suppliers may suspend in + * the meantime, prevent them from doing so by changing the + * device's runtime PM status to "active". + */ pm_runtime_set_active(&dev->dev); - pm_runtime_enable(&dev->dev); } static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) @@ -3677,14 +3683,6 @@ bool pci_acs_path_enabled(struct pci_dev *start, void pci_acs_init(struct pci_dev *dev) { dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); - - /* - * Attempt to enable ACS regardless of capability because some Root - * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have - * the standard ACS capability but still support ACS via those - * quirks. - */ - pci_enable_acs(dev); } void pci_rebar_init(struct pci_dev *pdev) @@ -5438,10 +5436,9 @@ static int pci_bus_trylock(struct pci_bus *bus) /* Do any devices on or below this slot prevent a bus reset? */ static bool pci_slot_resettable(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; - if (slot->bus->self && - (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) + if (bridge && (bridge->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) return false; list_for_each_entry(dev, &slot->bus->devices, bus_list) { @@ -5458,7 +5455,10 @@ static bool pci_slot_resettable(struct pci_slot *slot) /* Lock devices from the top of the tree down */ static void pci_slot_lock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; + + if (bridge) + pci_dev_lock(bridge); list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) @@ -5473,7 +5473,7 @@ static void pci_slot_lock(struct pci_slot *slot) /* Unlock devices from the bottom of the tree up */ static void pci_slot_unlock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) @@ -5483,21 +5483,25 @@ static void pci_slot_unlock(struct pci_slot *slot) else pci_dev_unlock(dev); } + + if (bridge) + pci_dev_unlock(bridge); } /* Return 1 on successful lock, 0 on contention */ static int pci_slot_trylock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; + + if (bridge && !pci_dev_trylock(bridge)) + return 0; list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) continue; if (dev->subordinate) { - if (!pci_bus_trylock(dev->subordinate)) { - pci_dev_unlock(dev); + if (!pci_bus_trylock(dev->subordinate)) goto unlock; - } } else if (!pci_dev_trylock(dev)) goto unlock; } @@ -5513,6 +5517,9 @@ static int pci_slot_trylock(struct pci_slot *slot) else pci_dev_unlock(dev); } + + if (bridge) + pci_dev_unlock(bridge); return 0; } @@ -6717,7 +6724,7 @@ static void of_pci_bus_release_domain_nr(struct device *parent, int domain_nr) return; /* Release domain from IDA where it was allocated. */ - if (of_get_pci_domain_nr(parent->of_node) == domain_nr) + if (parent && of_get_pci_domain_nr(parent->of_node) == domain_nr) ida_free(&pci_domain_nr_static_ida, domain_nr); else ida_free(&pci_domain_nr_dynamic_ida, domain_nr); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 36f8c0985430a..5510c103be2d3 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -88,6 +88,9 @@ struct pcie_tlp_log; #define PCI_BUS_BRIDGE_MEM_WINDOW 1 #define PCI_BUS_BRIDGE_PREF_MEM_WINDOW 2 +#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ + PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) + extern const unsigned char pcie_link_speed[]; extern bool pci_early_dump; @@ -103,17 +106,21 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search * @cap: Capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * - * Search the capability list in PCI config space to find @cap. + * Search the capability list in PCI config space to find @cap. If + * found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL (time-to-live) protection against infinite loops. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ int __ttl = PCI_FIND_CAP_TTL; \ - u8 __id, __found_pos = 0; \ + u8 __id, __found_pos = 0; \ + u8 __prev_pos = (start); \ u8 __pos = (start); \ u16 __ent; \ \ @@ -132,9 +139,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (__id == (cap)) { \ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u8 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = FIELD_GET(PCI_CAP_LIST_NEXT_MASK, __ent); \ } \ __found_pos; \ @@ -146,21 +156,26 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search (0 for initial search) * @cap: Extended capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * * Search the extended capability list in PCI config space to find @cap. + * If found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL protection against infinite loops using a calculated * maximum search count. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ u16 __pos = (start) ?: PCI_CFG_SPACE_SIZE; \ u16 __found_pos = 0; \ + u16 __prev_pos; \ int __ttl, __ret; \ u32 __header; \ \ + __prev_pos = __pos; \ __ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; \ while (__ttl-- > 0 && __pos >= PCI_CFG_SPACE_SIZE) { \ __ret = read_cfg##_dword(args, __pos, &__header); \ @@ -172,9 +187,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (PCI_EXT_CAP_ID(__header) == (cap) && __pos != start) {\ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u16 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = PCI_EXT_CAP_NEXT(__header); \ } \ __found_pos; \ @@ -917,6 +935,7 @@ static inline resource_size_t pci_resource_alignment(struct pci_dev *dev, } void pci_acs_init(struct pci_dev *dev); +void pci_enable_acs(struct pci_dev *dev); #ifdef CONFIG_PCI_QUIRKS int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags); int pci_dev_specific_enable_acs(struct pci_dev *dev); diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 0b5ed4722ac32..29504173425cd 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -238,9 +238,6 @@ void pcie_ecrc_get_policy(char *str) } #endif /* CONFIG_PCIE_ECRC */ -#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ - PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) - int pcie_aer_is_native(struct pci_dev *dev) { struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); @@ -1606,6 +1603,20 @@ static void aer_disable_irq(struct pci_dev *pdev) pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); } +static int clear_status_iter(struct pci_dev *dev, void *data) +{ + u16 devctl; + + /* Skip if pci_enable_pcie_error_reporting() hasn't been called yet */ + pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &devctl); + if (!(devctl & PCI_EXP_AER_FLAGS)) + return 0; + + pci_aer_clear_status(dev); + pcie_clear_device_status(dev); + return 0; +} + /** * aer_enable_rootport - enable Root Port's interrupts when receiving messages * @rpc: pointer to a Root Port data structure @@ -1627,9 +1638,19 @@ static void aer_enable_rootport(struct aer_rpc *rpc) pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, SYSTEM_ERROR_INTR_ON_MESG_MASK); - /* Clear error status */ + /* Clear error status of this Root Port or RCEC */ pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); + + /* Clear error status of agents reporting to this Root Port or RCEC */ + if (reg32 & AER_ERR_STATUS_MASK) { + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_EC) + pcie_walk_rcec(pdev, clear_status_iter, NULL); + else if (pdev->subordinate) + pci_walk_bus(pdev->subordinate, clear_status_iter, + NULL); + } + pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index cedea47a3547f..8b0874875da11 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -853,6 +853,18 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) parent_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); } + if (of_property_read_bool(child->dev.of_node, "aspm-no-l0s") || + of_property_read_bool(parent->dev.of_node, "aspm-no-l0s")) { + parent->aspm_l0s_support = 0; + child->aspm_l0s_support = 0; + } + + if (of_property_read_bool(child->dev.of_node, "aspm-no-l1") || + of_property_read_bool(parent->dev.of_node, "aspm-no-l1")) { + parent->aspm_l1_support = 0; + child->aspm_l1_support = 0; + } + /* * Setup L0s state * diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c index 36f939f23d34e..4ae92c9f912a8 100644 --- a/drivers/pci/pcie/bwctrl.c +++ b/drivers/pci/pcie/bwctrl.c @@ -250,6 +250,9 @@ static int pcie_bwnotif_probe(struct pcie_device *srv) struct pci_dev *port = srv->port; int ret; + if (port->no_bw_notif) + return -ENODEV; + /* Can happen if we run out of bus numbers during enumeration. */ if (!port->subordinate) return -ENODEV; diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index d1b68c18444f8..a0991da482136 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -557,10 +557,10 @@ static int pcie_port_remove_service(struct device *dev) pciedev = to_pcie_device(dev); driver = to_service_driver(dev->driver); - if (driver && driver->remove) { + if (driver && driver->remove) driver->remove(pciedev); - put_device(dev); - } + + put_device(dev); return 0; } @@ -760,7 +760,6 @@ static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev) device_for_each_child(&dev->dev, &off, pcie_port_device_iter); pci_restore_state(dev); - pci_save_state(dev); return PCI_ERS_RESULT_RECOVERED; } diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c index 65e4b008be00d..41d370f082ee4 100644 --- a/drivers/pci/pcie/ptm.c +++ b/drivers/pci/pcie/ptm.c @@ -519,8 +519,10 @@ struct pci_ptm_debugfs *pcie_ptm_create_debugfs(struct device *dev, void *pdata, return NULL; dirname = devm_kasprintf(dev, GFP_KERNEL, "pcie_ptm_%s", dev_name(dev)); - if (!dirname) + if (!dirname) { + kfree(ptm_debugfs); return NULL; + } ptm_debugfs->debugfs = debugfs_create_dir(dirname, NULL); ptm_debugfs->pdata = pdata; @@ -551,6 +553,7 @@ void pcie_ptm_destroy_debugfs(struct pci_ptm_debugfs *ptm_debugfs) mutex_destroy(&ptm_debugfs->lock); debugfs_remove_recursive(ptm_debugfs->debugfs); + kfree(ptm_debugfs); } EXPORT_SYMBOL_GPL(pcie_ptm_destroy_debugfs); #endif diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 9cd032dff31e5..fb0eb6b993ce1 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -287,8 +287,7 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8) && sz64 > 0x100000000ULL) { res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED; - res->start = 0; - res->end = 0; + resource_set_range(res, 0, 0); pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n", res_name, (unsigned long long)sz64); goto out; @@ -297,8 +296,7 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, if ((sizeof(pci_bus_addr_t) < 8) && l) { /* Above 32-bit boundary; try to reallocate */ res->flags |= IORESOURCE_UNSET; - res->start = 0; - res->end = sz64 - 1; + resource_set_range(res, 0, sz64); pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n", res_name, (unsigned long long)l64); goto out; @@ -2251,7 +2249,8 @@ int pci_configure_extended_tags(struct pci_dev *dev, void *ign) u16 ctl; int ret; - if (!pci_is_pcie(dev)) + /* PCI_EXP_DEVCTL_EXT_TAG is RsvdP in VFs */ + if (!pci_is_pcie(dev) || dev->is_virtfn) return 0; ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); @@ -2380,6 +2379,37 @@ static void pci_configure_serr(struct pci_dev *dev) } } +static void pci_configure_rcb(struct pci_dev *dev) +{ + struct pci_dev *rp; + u16 rp_lnkctl; + + /* + * Per PCIe r7.0, sec 7.5.3.7, RCB is only meaningful in Root Ports + * (where it is read-only), Endpoints, and Bridges. It may only be + * set for Endpoints and Bridges if it is set in the Root Port. For + * Endpoints, it is 'RsvdP' for Virtual Functions. + */ + if (!pci_is_pcie(dev) || + pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC || + dev->is_virtfn) + return; + + /* Root Port often not visible to virtualized guests */ + rp = pcie_find_root_port(dev); + if (!rp) + return; + + pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &rp_lnkctl); + pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_RCB, + (rp_lnkctl & PCI_EXP_LNKCTL_RCB) ? + PCI_EXP_LNKCTL_RCB : 0); +} + static void pci_configure_device(struct pci_dev *dev) { pci_configure_mps(dev); @@ -2389,6 +2419,7 @@ static void pci_configure_device(struct pci_dev *dev) pci_configure_aspm_l1ss(dev); pci_configure_eetlp_prefix(dev); pci_configure_serr(dev); + pci_configure_rcb(dev); pci_acpi_program_hp_params(dev); } @@ -2533,56 +2564,6 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l, } EXPORT_SYMBOL(pci_bus_read_dev_vendor_id); -#if IS_ENABLED(CONFIG_PCI_PWRCTRL) -static struct platform_device *pci_pwrctrl_create_device(struct pci_bus *bus, int devfn) -{ - struct pci_host_bridge *host = pci_find_host_bridge(bus); - struct platform_device *pdev; - struct device_node *np; - - np = of_pci_find_child_device(dev_of_node(&bus->dev), devfn); - if (!np) - return NULL; - - pdev = of_find_device_by_node(np); - if (pdev) { - put_device(&pdev->dev); - goto err_put_of_node; - } - - /* - * First check whether the pwrctrl device really needs to be created or - * not. This is decided based on at least one of the power supplies - * being defined in the devicetree node of the device. - */ - if (!of_pci_supply_present(np)) { - pr_debug("PCI/pwrctrl: Skipping OF node: %s\n", np->name); - goto err_put_of_node; - } - - /* Now create the pwrctrl device */ - pdev = of_platform_device_create(np, NULL, &host->dev); - if (!pdev) { - pr_err("PCI/pwrctrl: Failed to create pwrctrl device for node: %s\n", np->name); - goto err_put_of_node; - } - - of_node_put(np); - - return pdev; - -err_put_of_node: - of_node_put(np); - - return NULL; -} -#else -static struct platform_device *pci_pwrctrl_create_device(struct pci_bus *bus, int devfn) -{ - return NULL; -} -#endif - /* * Read the config data for a PCI device, sanity-check it, * and fill in the dev structure. @@ -2592,15 +2573,6 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) struct pci_dev *dev; u32 l; - /* - * Create pwrctrl device (if required) for the PCI device to handle the - * power state. If the pwrctrl device is created, then skip scanning - * further as the pwrctrl core will rescan the bus after powering on - * the device. - */ - if (pci_pwrctrl_create_device(bus, devfn)) - return NULL; - if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000)) return NULL; diff --git a/drivers/pci/pwrctrl/Kconfig b/drivers/pci/pwrctrl/Kconfig index 6956c18548114..0a93ac4cd11b5 100644 --- a/drivers/pci/pwrctrl/Kconfig +++ b/drivers/pci/pwrctrl/Kconfig @@ -11,17 +11,32 @@ config PCI_PWRCTRL_PWRSEQ select POWER_SEQUENCING select PCI_PWRCTRL -config PCI_PWRCTRL_SLOT - tristate "PCI Power Control driver for PCI slots" +config PCI_PWRCTRL_GENERIC + tristate "Generic PCI Power Control driver for PCI slots" select PCI_PWRCTRL help - Say Y here to enable the PCI Power Control driver to control the power - state of PCI slots. + Say Y here to enable the generic PCI Power Control driver to control + the power state of PCI slots. This is a generic driver that controls the power state of different PCI slots. The voltage regulators powering the rails of the PCI slots are expected to be defined in the devicetree node of the PCI bridge. +config PCI_PWRCTRL_TC9563 + tristate "PCI Power Control driver for TC9563 PCIe switch" + select PCI_PWRCTRL + default m if ARCH_QCOM + depends on I2C + help + Say Y here to enable the PCI Power Control driver of TC9563 PCIe + switch. + + This driver enables power and configures the TC9563 PCIe switch + through i2c. TC9563 is a PCIe switch which has one upstream and three + downstream ports. To one of the downstream ports integrated ethernet + MAC is connected as endpoint device. Other two downstream ports are + supposed to connect to external device. + # deprecated config HAVE_PWRCTL bool diff --git a/drivers/pci/pwrctrl/Makefile b/drivers/pci/pwrctrl/Makefile index a4e5808d7850c..f6bb4fb9a410b 100644 --- a/drivers/pci/pwrctrl/Makefile +++ b/drivers/pci/pwrctrl/Makefile @@ -5,5 +5,7 @@ pci-pwrctrl-core-y := core.o obj-$(CONFIG_PCI_PWRCTRL_PWRSEQ) += pci-pwrctrl-pwrseq.o -obj-$(CONFIG_PCI_PWRCTRL_SLOT) += pci-pwrctrl-slot.o -pci-pwrctrl-slot-y := slot.o +obj-$(CONFIG_PCI_PWRCTRL_GENERIC) += pci-pwrctrl-generic.o +pci-pwrctrl-generic-y := generic.o + +obj-$(CONFIG_PCI_PWRCTRL_TC9563) += pci-pwrctrl-tc9563.o diff --git a/drivers/pci/pwrctrl/core.c b/drivers/pci/pwrctrl/core.c index 6bdbfed584d6d..47a8771257615 100644 --- a/drivers/pci/pwrctrl/core.c +++ b/drivers/pci/pwrctrl/core.c @@ -3,14 +3,22 @@ * Copyright (C) 2024 Linaro Ltd. */ +#define dev_fmt(fmt) "pwrctrl: " fmt + #include #include #include +#include +#include +#include #include #include +#include #include #include +#include "../pci.h" + static int pci_pwrctrl_notify(struct notifier_block *nb, unsigned long action, void *data) { @@ -38,16 +46,6 @@ static int pci_pwrctrl_notify(struct notifier_block *nb, unsigned long action, return NOTIFY_DONE; } -static void rescan_work_func(struct work_struct *work) -{ - struct pci_pwrctrl *pwrctrl = container_of(work, - struct pci_pwrctrl, work); - - pci_lock_rescan_remove(); - pci_rescan_bus(to_pci_host_bridge(pwrctrl->dev->parent)->bus); - pci_unlock_rescan_remove(); -} - /** * pci_pwrctrl_init() - Initialize the PCI power control context struct * @@ -57,7 +55,7 @@ static void rescan_work_func(struct work_struct *work) void pci_pwrctrl_init(struct pci_pwrctrl *pwrctrl, struct device *dev) { pwrctrl->dev = dev; - INIT_WORK(&pwrctrl->work, rescan_work_func); + dev_set_drvdata(dev, pwrctrl); } EXPORT_SYMBOL_GPL(pci_pwrctrl_init); @@ -87,8 +85,6 @@ int pci_pwrctrl_device_set_ready(struct pci_pwrctrl *pwrctrl) if (ret) return ret; - schedule_work(&pwrctrl->work); - return 0; } EXPORT_SYMBOL_GPL(pci_pwrctrl_device_set_ready); @@ -101,8 +97,6 @@ EXPORT_SYMBOL_GPL(pci_pwrctrl_device_set_ready); */ void pci_pwrctrl_device_unset_ready(struct pci_pwrctrl *pwrctrl) { - cancel_work_sync(&pwrctrl->work); - /* * We don't have to delete the link here. Typically, this function * is only called when the power control device is being detached. If @@ -145,6 +139,290 @@ int devm_pci_pwrctrl_device_set_ready(struct device *dev, } EXPORT_SYMBOL_GPL(devm_pci_pwrctrl_device_set_ready); +/* + * Check whether the pwrctrl device really needs to be created or not. The + * pwrctrl device will only be created if the node satisfies below requirements: + * + * 1. Presence of compatible property with "pci" prefix to match against the + * pwrctrl driver (AND) + * 2. At least one of the power supplies defined in the devicetree node of the + * device (OR) in the remote endpoint parent node to indicate pwrctrl + * requirement. + */ +static bool pci_pwrctrl_is_required(struct device_node *np) +{ + struct device_node *endpoint; + const char *compat; + int ret; + + ret = of_property_read_string(np, "compatible", &compat); + if (ret < 0) + return false; + + if (!strstarts(compat, "pci")) + return false; + + if (of_pci_supply_present(np)) + return true; + + if (of_graph_is_present(np)) { + for_each_endpoint_of_node(np, endpoint) { + struct device_node *remote __free(device_node) = + of_graph_get_remote_port_parent(endpoint); + if (remote) { + if (of_pci_supply_present(remote)) { + of_node_put(endpoint); + return true; + } + } + } + } + + return false; +} + +static int __pci_pwrctrl_power_off_device(struct device *dev) +{ + struct pci_pwrctrl *pwrctrl = dev_get_drvdata(dev); + + if (!pwrctrl) + return 0; + + return pwrctrl->power_off(pwrctrl); +} + +static void pci_pwrctrl_power_off_device(struct device_node *np) +{ + struct platform_device *pdev; + int ret; + + for_each_available_child_of_node_scoped(np, child) + pci_pwrctrl_power_off_device(child); + + if (!pci_pwrctrl_is_required(np)) + return; + + pdev = of_find_device_by_node(np); + if (!pdev) + return; + + if (device_is_bound(&pdev->dev)) { + ret = __pci_pwrctrl_power_off_device(&pdev->dev); + if (ret) + dev_err(&pdev->dev, "Failed to power off device: %d", ret); + } + + platform_device_put(pdev); +} + +/** + * pci_pwrctrl_power_off_devices - Power off pwrctrl devices + * + * @parent: PCI host controller device + * + * Recursively traverse all pwrctrl devices for the devicetree hierarchy + * below the specified PCI host controller and power them off in a depth + * first manner. + */ +void pci_pwrctrl_power_off_devices(struct device *parent) +{ + struct device_node *np = parent->of_node; + + for_each_available_child_of_node_scoped(np, child) + pci_pwrctrl_power_off_device(child); +} +EXPORT_SYMBOL_GPL(pci_pwrctrl_power_off_devices); + +static int __pci_pwrctrl_power_on_device(struct device *dev) +{ + struct pci_pwrctrl *pwrctrl = dev_get_drvdata(dev); + + if (!pwrctrl) + return 0; + + return pwrctrl->power_on(pwrctrl); +} + +/* + * Power on the devices in a depth first manner. Before powering on the device, + * make sure its driver is bound. + */ +static int pci_pwrctrl_power_on_device(struct device_node *np) +{ + struct platform_device *pdev; + int ret; + + for_each_available_child_of_node_scoped(np, child) { + ret = pci_pwrctrl_power_on_device(child); + if (ret) + return ret; + } + + if (!pci_pwrctrl_is_required(np)) + return 0; + + pdev = of_find_device_by_node(np); + if (!pdev) + return 0; + + if (device_is_bound(&pdev->dev)) { + ret = __pci_pwrctrl_power_on_device(&pdev->dev); + } else { + /* FIXME: Use blocking wait instead of probe deferral */ + dev_dbg(&pdev->dev, "driver is not bound\n"); + ret = -EPROBE_DEFER; + } + + platform_device_put(pdev); + + return ret; +} + +/** + * pci_pwrctrl_power_on_devices - Power on pwrctrl devices + * + * @parent: PCI host controller device + * + * Recursively traverse all pwrctrl devices for the devicetree hierarchy + * below the specified PCI host controller and power them on in a depth + * first manner. On error, all powered on devices will be powered off. + * + * Return: 0 on success, -EPROBE_DEFER if any pwrctrl driver is not bound, an + * appropriate error code otherwise. + */ +int pci_pwrctrl_power_on_devices(struct device *parent) +{ + struct device_node *np = parent->of_node; + struct device_node *child = NULL; + int ret; + + for_each_available_child_of_node(np, child) { + ret = pci_pwrctrl_power_on_device(child); + if (ret) + goto err_power_off; + } + + return 0; + +err_power_off: + for_each_available_child_of_node_scoped(np, tmp) { + if (tmp == child) + break; + pci_pwrctrl_power_off_device(tmp); + } + of_node_put(child); + + return ret; +} +EXPORT_SYMBOL_GPL(pci_pwrctrl_power_on_devices); + +static int pci_pwrctrl_create_device(struct device_node *np, + struct device *parent) +{ + struct platform_device *pdev; + int ret; + + for_each_available_child_of_node_scoped(np, child) { + ret = pci_pwrctrl_create_device(child, parent); + if (ret) + return ret; + } + + /* Bail out if the platform device is already available for the node */ + pdev = of_find_device_by_node(np); + if (pdev) { + platform_device_put(pdev); + return 0; + } + + /* + * Sanity check to make sure that the node has the compatible property + * to allow driver binding. + */ + if (!of_property_present(np, "compatible")) + return 0; + + /* + * Check whether the pwrctrl device really needs to be created or not. + * This is decided based on at least one of the power supplies being + * defined in the devicetree node of the device. + */ + if (!of_pci_supply_present(np)) { + dev_dbg(parent, "Skipping OF node: %s\n", np->name); + return 0; + } + + /* Now create the pwrctrl device */ + pdev = of_platform_device_create(np, NULL, parent); + if (!pdev) { + dev_err(parent, "Failed to create pwrctrl device for node: %s\n", np->name); + return -EINVAL; + } + + return 0; +} + +/** + * pci_pwrctrl_create_devices - Create pwrctrl devices + * + * @parent: PCI host controller device + * + * Recursively create pwrctrl devices for the devicetree hierarchy below + * the specified PCI host controller in a depth first manner. On error, all + * created devices will be destroyed. + * + * Return: 0 on success, negative error number on error. + */ +int pci_pwrctrl_create_devices(struct device *parent) +{ + int ret; + + for_each_available_child_of_node_scoped(parent->of_node, child) { + ret = pci_pwrctrl_create_device(child, parent); + if (ret) { + pci_pwrctrl_destroy_devices(parent); + return ret; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(pci_pwrctrl_create_devices); + +static void pci_pwrctrl_destroy_device(struct device_node *np) +{ + struct platform_device *pdev; + + for_each_available_child_of_node_scoped(np, child) + pci_pwrctrl_destroy_device(child); + + pdev = of_find_device_by_node(np); + if (!pdev) + return; + + of_device_unregister(pdev); + platform_device_put(pdev); + + of_node_clear_flag(np, OF_POPULATED); +} + +/** + * pci_pwrctrl_destroy_devices - Destroy pwrctrl devices + * + * @parent: PCI host controller device + * + * Recursively destroy pwrctrl devices for the devicetree hierarchy below + * the specified PCI host controller in a depth first manner. + */ +void pci_pwrctrl_destroy_devices(struct device *parent) +{ + struct device_node *np = parent->of_node; + + for_each_available_child_of_node_scoped(np, child) + pci_pwrctrl_destroy_device(child); +} +EXPORT_SYMBOL_GPL(pci_pwrctrl_destroy_devices); + MODULE_AUTHOR("Bartosz Golaszewski "); MODULE_DESCRIPTION("PCI Device Power Control core driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/pci/pwrctrl/slot.c b/drivers/pci/pwrctrl/generic.c similarity index 57% rename from drivers/pci/pwrctrl/slot.c rename to drivers/pci/pwrctrl/generic.c index 3320494b62d89..e29b7bfdd6863 100644 --- a/drivers/pci/pwrctrl/slot.c +++ b/drivers/pci/pwrctrl/generic.c @@ -13,25 +13,51 @@ #include #include -struct pci_pwrctrl_slot_data { - struct pci_pwrctrl ctx; +struct pci_pwrctrl_slot { + struct pci_pwrctrl pwrctrl; struct regulator_bulk_data *supplies; int num_supplies; + struct clk *clk; }; -static void devm_pci_pwrctrl_slot_power_off(void *data) +static int pci_pwrctrl_slot_power_on(struct pci_pwrctrl *pwrctrl) { - struct pci_pwrctrl_slot_data *slot = data; + struct pci_pwrctrl_slot *slot = container_of(pwrctrl, + struct pci_pwrctrl_slot, pwrctrl); + int ret; + + ret = regulator_bulk_enable(slot->num_supplies, slot->supplies); + if (ret < 0) { + dev_err(slot->pwrctrl.dev, "Failed to enable slot regulators\n"); + return ret; + } + + return clk_prepare_enable(slot->clk); +} + +static int pci_pwrctrl_slot_power_off(struct pci_pwrctrl *pwrctrl) +{ + struct pci_pwrctrl_slot *slot = container_of(pwrctrl, + struct pci_pwrctrl_slot, pwrctrl); regulator_bulk_disable(slot->num_supplies, slot->supplies); + clk_disable_unprepare(slot->clk); + + return 0; +} + +static void devm_pci_pwrctrl_slot_release(void *data) +{ + struct pci_pwrctrl_slot *slot = data; + + pci_pwrctrl_slot_power_off(&slot->pwrctrl); regulator_bulk_free(slot->num_supplies, slot->supplies); } static int pci_pwrctrl_slot_probe(struct platform_device *pdev) { - struct pci_pwrctrl_slot_data *slot; + struct pci_pwrctrl_slot *slot; struct device *dev = &pdev->dev; - struct clk *clk; int ret; slot = devm_kzalloc(dev, sizeof(*slot), GFP_KERNEL); @@ -40,33 +66,27 @@ static int pci_pwrctrl_slot_probe(struct platform_device *pdev) ret = of_regulator_bulk_get_all(dev, dev_of_node(dev), &slot->supplies); - if (ret < 0) { - dev_err_probe(dev, ret, "Failed to get slot regulators\n"); - return ret; - } + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to get slot regulators\n"); slot->num_supplies = ret; - ret = regulator_bulk_enable(slot->num_supplies, slot->supplies); - if (ret < 0) { - dev_err_probe(dev, ret, "Failed to enable slot regulators\n"); - regulator_bulk_free(slot->num_supplies, slot->supplies); - return ret; - } - ret = devm_add_action_or_reset(dev, devm_pci_pwrctrl_slot_power_off, + ret = devm_add_action_or_reset(dev, devm_pci_pwrctrl_slot_release, slot); if (ret) return ret; - clk = devm_clk_get_optional_enabled(dev, NULL); - if (IS_ERR(clk)) { - return dev_err_probe(dev, PTR_ERR(clk), + slot->clk = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(slot->clk)) + return dev_err_probe(dev, PTR_ERR(slot->clk), "Failed to enable slot clock\n"); - } - pci_pwrctrl_init(&slot->ctx, dev); + slot->pwrctrl.power_on = pci_pwrctrl_slot_power_on; + slot->pwrctrl.power_off = pci_pwrctrl_slot_power_off; - ret = devm_pci_pwrctrl_device_set_ready(dev, &slot->ctx); + pci_pwrctrl_init(&slot->pwrctrl, dev); + + ret = devm_pci_pwrctrl_device_set_ready(dev, &slot->pwrctrl); if (ret) return dev_err_probe(dev, ret, "Failed to register pwrctrl driver\n"); @@ -77,6 +97,10 @@ static const struct of_device_id pci_pwrctrl_slot_of_match[] = { { .compatible = "pciclass,0604", }, + /* Renesas UPD720201/UPD720202 USB 3.0 xHCI Host Controller */ + { + .compatible = "pci1912,0014", + }, { } }; MODULE_DEVICE_TABLE(of, pci_pwrctrl_slot_of_match); diff --git a/drivers/pci/pwrctrl/pci-pwrctrl-pwrseq.c b/drivers/pci/pwrctrl/pci-pwrctrl-pwrseq.c index 4e664e7b8dd23..1df0cd73b416e 100644 --- a/drivers/pci/pwrctrl/pci-pwrctrl-pwrseq.c +++ b/drivers/pci/pwrctrl/pci-pwrctrl-pwrseq.c @@ -13,8 +13,8 @@ #include #include -struct pci_pwrctrl_pwrseq_data { - struct pci_pwrctrl ctx; +struct pci_pwrctrl_pwrseq { + struct pci_pwrctrl pwrctrl; struct pwrseq_desc *pwrseq; }; @@ -52,17 +52,33 @@ static const struct pci_pwrctrl_pwrseq_pdata pci_pwrctrl_pwrseq_qcom_wcn_pdata = .validate_device = pci_pwrctrl_pwrseq_qcm_wcn_validate_device, }; +static int pci_pwrctrl_pwrseq_power_on(struct pci_pwrctrl *pwrctrl) +{ + struct pci_pwrctrl_pwrseq *pwrseq = container_of(pwrctrl, + struct pci_pwrctrl_pwrseq, pwrctrl); + + return pwrseq_power_on(pwrseq->pwrseq); +} + +static int pci_pwrctrl_pwrseq_power_off(struct pci_pwrctrl *pwrctrl) +{ + struct pci_pwrctrl_pwrseq *pwrseq = container_of(pwrctrl, + struct pci_pwrctrl_pwrseq, pwrctrl); + + return pwrseq_power_off(pwrseq->pwrseq); +} + static void devm_pci_pwrctrl_pwrseq_power_off(void *data) { - struct pwrseq_desc *pwrseq = data; + struct pci_pwrctrl_pwrseq *pwrseq = data; - pwrseq_power_off(pwrseq); + pci_pwrctrl_pwrseq_power_off(&pwrseq->pwrctrl); } static int pci_pwrctrl_pwrseq_probe(struct platform_device *pdev) { const struct pci_pwrctrl_pwrseq_pdata *pdata; - struct pci_pwrctrl_pwrseq_data *data; + struct pci_pwrctrl_pwrseq *pwrseq; struct device *dev = &pdev->dev; int ret; @@ -76,28 +92,26 @@ static int pci_pwrctrl_pwrseq_probe(struct platform_device *pdev) return ret; } - data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); - if (!data) + pwrseq = devm_kzalloc(dev, sizeof(*pwrseq), GFP_KERNEL); + if (!pwrseq) return -ENOMEM; - data->pwrseq = devm_pwrseq_get(dev, pdata->target); - if (IS_ERR(data->pwrseq)) - return dev_err_probe(dev, PTR_ERR(data->pwrseq), + pwrseq->pwrseq = devm_pwrseq_get(dev, pdata->target); + if (IS_ERR(pwrseq->pwrseq)) + return dev_err_probe(dev, PTR_ERR(pwrseq->pwrseq), "Failed to get the power sequencer\n"); - ret = pwrseq_power_on(data->pwrseq); - if (ret) - return dev_err_probe(dev, ret, - "Failed to power-on the device\n"); - ret = devm_add_action_or_reset(dev, devm_pci_pwrctrl_pwrseq_power_off, - data->pwrseq); + pwrseq); if (ret) return ret; - pci_pwrctrl_init(&data->ctx, dev); + pwrseq->pwrctrl.power_on = pci_pwrctrl_pwrseq_power_on; + pwrseq->pwrctrl.power_off = pci_pwrctrl_pwrseq_power_off; + + pci_pwrctrl_init(&pwrseq->pwrctrl, dev); - ret = devm_pci_pwrctrl_device_set_ready(dev, &data->ctx); + ret = devm_pci_pwrctrl_device_set_ready(dev, &pwrseq->pwrctrl); if (ret) return dev_err_probe(dev, ret, "Failed to register the pwrctrl wrapper\n"); diff --git a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c new file mode 100644 index 0000000000000..38008d03903f2 --- /dev/null +++ b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../pci.h" + +#define TC9563_GPIO_CONFIG 0x801208 +#define TC9563_RESET_GPIO 0x801210 + +#define TC9563_PORT_L0S_DELAY 0x82496c +#define TC9563_PORT_L1_DELAY 0x824970 + +#define TC9563_EMBEDDED_ETH_DELAY 0x8200d8 +#define TC9563_ETH_L1_DELAY_MASK GENMASK(27, 18) +#define TC9563_ETH_L1_DELAY_VALUE(x) FIELD_PREP(TC9563_ETH_L1_DELAY_MASK, x) +#define TC9563_ETH_L0S_DELAY_MASK GENMASK(17, 13) +#define TC9563_ETH_L0S_DELAY_VALUE(x) FIELD_PREP(TC9563_ETH_L0S_DELAY_MASK, x) + +#define TC9563_NFTS_2_5_GT 0x824978 +#define TC9563_NFTS_5_GT 0x82497c + +#define TC9563_PORT_LANE_ACCESS_ENABLE 0x828000 + +#define TC9563_PHY_RATE_CHANGE_OVERRIDE 0x828040 +#define TC9563_PHY_RATE_CHANGE 0x828050 + +#define TC9563_TX_MARGIN 0x828234 + +#define TC9563_DFE_ENABLE 0x828a04 +#define TC9563_DFE_EQ0_MODE 0x828a08 +#define TC9563_DFE_EQ1_MODE 0x828a0c +#define TC9563_DFE_EQ2_MODE 0x828a14 +#define TC9563_DFE_PD_MASK 0x828254 + +#define TC9563_PORT_SELECT 0x82c02c +#define TC9563_PORT_ACCESS_ENABLE 0x82c030 + +#define TC9563_POWER_CONTROL 0x82b09c +#define TC9563_POWER_CONTROL_OVREN 0x82b2c8 + +#define TC9563_GPIO_MASK 0xfffffff3 +#define TC9563_GPIO_DEASSERT_BITS 0xc /* Clear to deassert GPIO */ + +#define TC9563_TX_MARGIN_MIN_UA 400000 + +/* + * From TC9563 PORSYS rev 0.2, figure 1.1 POR boot sequence + * wait for 10ms for the internal osc frequency to stabilize. + */ +#define TC9563_OSC_STAB_DELAY_US (10 * USEC_PER_MSEC) + +#define TC9563_L0S_L1_DELAY_UNIT_NS 256 /* Each unit represents 256 ns */ + +struct tc9563_pwrctrl_reg_setting { + unsigned int offset; + unsigned int val; +}; + +enum tc9563_pwrctrl_ports { + TC9563_USP, + TC9563_DSP1, + TC9563_DSP2, + TC9563_DSP3, + TC9563_ETHERNET, + TC9563_MAX +}; + +struct tc9563_pwrctrl_cfg { + u32 l0s_delay; + u32 l1_delay; + u32 tx_amp; + u8 nfts[2]; /* GEN1 & GEN2 */ + bool disable_dfe; + bool disable_port; +}; + +#define TC9563_PWRCTL_MAX_SUPPLY 6 + +static const char *const tc9563_supply_names[TC9563_PWRCTL_MAX_SUPPLY] = { + "vddc", + "vdd18", + "vdd09", + "vddio1", + "vddio2", + "vddio18", +}; + +struct pci_pwrctrl_tc9563 { + struct pci_pwrctrl pwrctrl; + struct regulator_bulk_data supplies[TC9563_PWRCTL_MAX_SUPPLY]; + struct tc9563_pwrctrl_cfg cfg[TC9563_MAX]; + struct gpio_desc *reset_gpio; + struct i2c_adapter *adapter; + struct i2c_client *client; +}; + +/* + * downstream port power off sequence, hardcoding the address + * as we don't know register names for these register offsets. + */ +static const struct tc9563_pwrctrl_reg_setting common_pwroff_seq[] = { + {0x82900c, 0x1}, + {0x829010, 0x1}, + {0x829018, 0x0}, + {0x829020, 0x1}, + {0x82902c, 0x1}, + {0x829030, 0x1}, + {0x82903c, 0x1}, + {0x829058, 0x0}, + {0x82905c, 0x1}, + {0x829060, 0x1}, + {0x8290cc, 0x1}, + {0x8290d0, 0x1}, + {0x8290d8, 0x1}, + {0x8290e0, 0x1}, + {0x8290e8, 0x1}, + {0x8290ec, 0x1}, + {0x8290f4, 0x1}, + {0x82910c, 0x1}, + {0x829110, 0x1}, + {0x829114, 0x1}, +}; + +static const struct tc9563_pwrctrl_reg_setting dsp1_pwroff_seq[] = { + {TC9563_PORT_ACCESS_ENABLE, 0x2}, + {TC9563_PORT_LANE_ACCESS_ENABLE, 0x3}, + {TC9563_POWER_CONTROL, 0x014f4804}, + {TC9563_POWER_CONTROL_OVREN, 0x1}, + {TC9563_PORT_ACCESS_ENABLE, 0x4}, +}; + +static const struct tc9563_pwrctrl_reg_setting dsp2_pwroff_seq[] = { + {TC9563_PORT_ACCESS_ENABLE, 0x8}, + {TC9563_PORT_LANE_ACCESS_ENABLE, 0x1}, + {TC9563_POWER_CONTROL, 0x014f4804}, + {TC9563_POWER_CONTROL_OVREN, 0x1}, + {TC9563_PORT_ACCESS_ENABLE, 0x8}, +}; + +/* + * Since all transfers are initiated by the probe, no locks are necessary, + * as there are no concurrent calls. + */ +static int tc9563_pwrctrl_i2c_write(struct i2c_client *client, + u32 reg_addr, u32 reg_val) +{ + struct i2c_msg msg; + u8 msg_buf[7]; + int ret; + + msg.addr = client->addr; + msg.len = 7; + msg.flags = 0; + + /* Big Endian for reg addr */ + put_unaligned_be24(reg_addr, &msg_buf[0]); + + /* Little Endian for reg val */ + put_unaligned_le32(reg_val, &msg_buf[3]); + + msg.buf = msg_buf; + ret = i2c_transfer(client->adapter, &msg, 1); + return ret == 1 ? 0 : ret; +} + +static int tc9563_pwrctrl_i2c_read(struct i2c_client *client, + u32 reg_addr, u32 *reg_val) +{ + struct i2c_msg msg[2]; + u8 wr_data[3]; + u32 rd_data; + int ret; + + msg[0].addr = client->addr; + msg[0].len = 3; + msg[0].flags = 0; + + /* Big Endian for reg addr */ + put_unaligned_be24(reg_addr, &wr_data[0]); + + msg[0].buf = wr_data; + + msg[1].addr = client->addr; + msg[1].len = 4; + msg[1].flags = I2C_M_RD; + + msg[1].buf = (u8 *)&rd_data; + + ret = i2c_transfer(client->adapter, &msg[0], 2); + if (ret == 2) { + *reg_val = get_unaligned_le32(&rd_data); + return 0; + } + + /* If only one message successfully completed, return -EIO */ + return ret == 1 ? -EIO : ret; +} + +static int tc9563_pwrctrl_i2c_bulk_write(struct i2c_client *client, + const struct tc9563_pwrctrl_reg_setting *seq, + int len) +{ + int ret, i; + + for (i = 0; i < len; i++) { + ret = tc9563_pwrctrl_i2c_write(client, seq[i].offset, seq[i].val); + if (ret) + return ret; + } + + return 0; +} + +static int tc9563_pwrctrl_disable_port(struct pci_pwrctrl_tc9563 *tc9563, + enum tc9563_pwrctrl_ports port) +{ + struct tc9563_pwrctrl_cfg *cfg = &tc9563->cfg[port]; + const struct tc9563_pwrctrl_reg_setting *seq; + int ret, len; + + if (!cfg->disable_port) + return 0; + + if (port == TC9563_DSP1) { + seq = dsp1_pwroff_seq; + len = ARRAY_SIZE(dsp1_pwroff_seq); + } else { + seq = dsp2_pwroff_seq; + len = ARRAY_SIZE(dsp2_pwroff_seq); + } + + ret = tc9563_pwrctrl_i2c_bulk_write(tc9563->client, seq, len); + if (ret) + return ret; + + return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, common_pwroff_seq, + ARRAY_SIZE(common_pwroff_seq)); +} + +static int tc9563_pwrctrl_set_l0s_l1_entry_delay(struct pci_pwrctrl_tc9563 *tc9563, + enum tc9563_pwrctrl_ports port, + bool is_l1, u32 ns) +{ + u32 rd_val, units; + int ret; + + if (ns < TC9563_L0S_L1_DELAY_UNIT_NS) + return 0; + + /* convert to units of 256ns */ + units = ns / TC9563_L0S_L1_DELAY_UNIT_NS; + + if (port == TC9563_ETHERNET) { + ret = tc9563_pwrctrl_i2c_read(tc9563->client, + TC9563_EMBEDDED_ETH_DELAY, + &rd_val); + if (ret) + return ret; + + if (is_l1) + rd_val = u32_replace_bits(rd_val, units, + TC9563_ETH_L1_DELAY_MASK); + else + rd_val = u32_replace_bits(rd_val, units, + TC9563_ETH_L0S_DELAY_MASK); + + return tc9563_pwrctrl_i2c_write(tc9563->client, + TC9563_EMBEDDED_ETH_DELAY, + rd_val); + } + + ret = tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_PORT_SELECT, + BIT(port)); + if (ret) + return ret; + + return tc9563_pwrctrl_i2c_write(tc9563->client, + is_l1 ? TC9563_PORT_L1_DELAY : TC9563_PORT_L0S_DELAY, + units); +} + +static int tc9563_pwrctrl_set_tx_amplitude(struct pci_pwrctrl_tc9563 *tc9563, + enum tc9563_pwrctrl_ports port) +{ + u32 amp = tc9563->cfg[port].tx_amp; + int port_access; + + if (amp < TC9563_TX_MARGIN_MIN_UA) + return 0; + + /* txmargin = (Amp(uV) - 400000) / 3125 */ + amp = (amp - TC9563_TX_MARGIN_MIN_UA) / 3125; + + switch (port) { + case TC9563_USP: + port_access = 0x1; + break; + case TC9563_DSP1: + port_access = 0x2; + break; + case TC9563_DSP2: + port_access = 0x8; + break; + default: + return -EINVAL; + } + + struct tc9563_pwrctrl_reg_setting tx_amp_seq[] = { + {TC9563_PORT_ACCESS_ENABLE, port_access}, + {TC9563_PORT_LANE_ACCESS_ENABLE, 0x3}, + {TC9563_TX_MARGIN, amp}, + }; + + return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, tx_amp_seq, + ARRAY_SIZE(tx_amp_seq)); +} + +static int tc9563_pwrctrl_disable_dfe(struct pci_pwrctrl_tc9563 *tc9563, + enum tc9563_pwrctrl_ports port) +{ + struct tc9563_pwrctrl_cfg *cfg = &tc9563->cfg[port]; + int port_access, lane_access = 0x3; + u32 phy_rate = 0x21; + + if (!cfg->disable_dfe) + return 0; + + switch (port) { + case TC9563_USP: + phy_rate = 0x1; + port_access = 0x1; + break; + case TC9563_DSP1: + port_access = 0x2; + break; + case TC9563_DSP2: + port_access = 0x8; + lane_access = 0x1; + break; + default: + return -EINVAL; + } + + struct tc9563_pwrctrl_reg_setting disable_dfe_seq[] = { + {TC9563_PORT_ACCESS_ENABLE, port_access}, + {TC9563_PORT_LANE_ACCESS_ENABLE, lane_access}, + {TC9563_DFE_ENABLE, 0x0}, + {TC9563_DFE_EQ0_MODE, 0x411}, + {TC9563_DFE_EQ1_MODE, 0x11}, + {TC9563_DFE_EQ2_MODE, 0x11}, + {TC9563_DFE_PD_MASK, 0x7}, + {TC9563_PHY_RATE_CHANGE_OVERRIDE, 0x10}, + {TC9563_PHY_RATE_CHANGE, phy_rate}, + {TC9563_PHY_RATE_CHANGE, 0x0}, + {TC9563_PHY_RATE_CHANGE_OVERRIDE, 0x0}, + }; + + return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, disable_dfe_seq, + ARRAY_SIZE(disable_dfe_seq)); +} + +static int tc9563_pwrctrl_set_nfts(struct pci_pwrctrl_tc9563 *tc9563, + enum tc9563_pwrctrl_ports port) +{ + u8 *nfts = tc9563->cfg[port].nfts; + struct tc9563_pwrctrl_reg_setting nfts_seq[] = { + {TC9563_NFTS_2_5_GT, nfts[0]}, + {TC9563_NFTS_5_GT, nfts[1]}, + }; + int ret; + + if (!nfts[0]) + return 0; + + ret = tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_PORT_SELECT, + BIT(port)); + if (ret) + return ret; + + return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, nfts_seq, + ARRAY_SIZE(nfts_seq)); +} + +static int tc9563_pwrctrl_assert_deassert_reset(struct pci_pwrctrl_tc9563 *tc9563, + bool deassert) +{ + int ret, val; + + ret = tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_GPIO_CONFIG, + TC9563_GPIO_MASK); + if (ret) + return ret; + + val = deassert ? TC9563_GPIO_DEASSERT_BITS : 0; + + return tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_RESET_GPIO, val); +} + +static int tc9563_pwrctrl_parse_device_dt(struct pci_pwrctrl_tc9563 *tc9563, + struct device_node *node, + enum tc9563_pwrctrl_ports port) +{ + struct tc9563_pwrctrl_cfg *cfg = &tc9563->cfg[port]; + int ret; + + /* Disable port if the status of the port is disabled. */ + if (!of_device_is_available(node)) { + cfg->disable_port = true; + return 0; + } + + ret = of_property_read_u32(node, "aspm-l0s-entry-delay-ns", &cfg->l0s_delay); + if (ret && ret != -EINVAL) + return ret; + + ret = of_property_read_u32(node, "aspm-l1-entry-delay-ns", &cfg->l1_delay); + if (ret && ret != -EINVAL) + return ret; + + ret = of_property_read_u32(node, "toshiba,tx-amplitude-microvolt", &cfg->tx_amp); + if (ret && ret != -EINVAL) + return ret; + + ret = of_property_read_u8_array(node, "n-fts", cfg->nfts, ARRAY_SIZE(cfg->nfts)); + if (ret && ret != -EINVAL) + return ret; + + cfg->disable_dfe = of_property_read_bool(node, "toshiba,no-dfe-support"); + + return 0; +} + +static int tc9563_pwrctrl_power_off(struct pci_pwrctrl *pwrctrl) +{ + struct pci_pwrctrl_tc9563 *tc9563 = container_of(pwrctrl, + struct pci_pwrctrl_tc9563, pwrctrl); + + gpiod_set_value(tc9563->reset_gpio, 1); + + regulator_bulk_disable(ARRAY_SIZE(tc9563->supplies), tc9563->supplies); + + return 0; +} + +static int tc9563_pwrctrl_power_on(struct pci_pwrctrl *pwrctrl) +{ + struct pci_pwrctrl_tc9563 *tc9563 = container_of(pwrctrl, + struct pci_pwrctrl_tc9563, pwrctrl); + struct device *dev = tc9563->pwrctrl.dev; + struct tc9563_pwrctrl_cfg *cfg; + int ret, i; + + ret = regulator_bulk_enable(ARRAY_SIZE(tc9563->supplies), + tc9563->supplies); + if (ret < 0) + return dev_err_probe(dev, ret, "cannot enable regulators\n"); + + gpiod_set_value(tc9563->reset_gpio, 0); + + fsleep(TC9563_OSC_STAB_DELAY_US); + + ret = tc9563_pwrctrl_assert_deassert_reset(tc9563, false); + if (ret) + goto power_off; + + for (i = 0; i < TC9563_MAX; i++) { + cfg = &tc9563->cfg[i]; + ret = tc9563_pwrctrl_disable_port(tc9563, i); + if (ret) { + dev_err(dev, "Disabling port failed\n"); + goto power_off; + } + + ret = tc9563_pwrctrl_set_l0s_l1_entry_delay(tc9563, i, false, cfg->l0s_delay); + if (ret) { + dev_err(dev, "Setting L0s entry delay failed\n"); + goto power_off; + } + + ret = tc9563_pwrctrl_set_l0s_l1_entry_delay(tc9563, i, true, cfg->l1_delay); + if (ret) { + dev_err(dev, "Setting L1 entry delay failed\n"); + goto power_off; + } + + ret = tc9563_pwrctrl_set_tx_amplitude(tc9563, i); + if (ret) { + dev_err(dev, "Setting Tx amplitude failed\n"); + goto power_off; + } + + ret = tc9563_pwrctrl_set_nfts(tc9563, i); + if (ret) { + dev_err(dev, "Setting N_FTS failed\n"); + goto power_off; + } + + ret = tc9563_pwrctrl_disable_dfe(tc9563, i); + if (ret) { + dev_err(dev, "Disabling DFE failed\n"); + goto power_off; + } + } + + ret = tc9563_pwrctrl_assert_deassert_reset(tc9563, true); + if (!ret) + return 0; + +power_off: + tc9563_pwrctrl_power_off(&tc9563->pwrctrl); + return ret; +} + +static int tc9563_pwrctrl_probe(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + struct device *dev = &pdev->dev; + enum tc9563_pwrctrl_ports port; + struct pci_pwrctrl_tc9563 *tc9563; + struct device_node *i2c_node; + int ret, addr; + + tc9563 = devm_kzalloc(dev, sizeof(*tc9563), GFP_KERNEL); + if (!tc9563) + return -ENOMEM; + + ret = of_property_read_u32_index(node, "i2c-parent", 1, &addr); + if (ret) + return dev_err_probe(dev, ret, "Failed to read i2c-parent property\n"); + + i2c_node = of_parse_phandle(dev->of_node, "i2c-parent", 0); + tc9563->adapter = of_find_i2c_adapter_by_node(i2c_node); + of_node_put(i2c_node); + if (!tc9563->adapter) + return dev_err_probe(dev, -EPROBE_DEFER, "Failed to find I2C adapter\n"); + + tc9563->client = i2c_new_dummy_device(tc9563->adapter, addr); + if (IS_ERR(tc9563->client)) { + dev_err(dev, "Failed to create I2C client\n"); + put_device(&tc9563->adapter->dev); + return PTR_ERR(tc9563->client); + } + + for (int i = 0; i < ARRAY_SIZE(tc9563_supply_names); i++) + tc9563->supplies[i].supply = tc9563_supply_names[i]; + + ret = devm_regulator_bulk_get(dev, TC9563_PWRCTL_MAX_SUPPLY, + tc9563->supplies); + if (ret) { + dev_err_probe(dev, ret, "failed to get supply regulator\n"); + goto remove_i2c; + } + + tc9563->reset_gpio = devm_gpiod_get(dev, "resx", GPIOD_OUT_HIGH); + if (IS_ERR(tc9563->reset_gpio)) { + ret = dev_err_probe(dev, PTR_ERR(tc9563->reset_gpio), "failed to get resx GPIO\n"); + goto remove_i2c; + } + + pci_pwrctrl_init(&tc9563->pwrctrl, dev); + + port = TC9563_USP; + ret = tc9563_pwrctrl_parse_device_dt(tc9563, node, port); + if (ret) { + dev_err(dev, "failed to parse device tree properties: %d\n", ret); + goto remove_i2c; + } + + /* + * Downstream ports are always children of the upstream port. + * The first node represents DSP1, the second node represents DSP2, + * and so on. + */ + for_each_child_of_node_scoped(node, child) { + port++; + ret = tc9563_pwrctrl_parse_device_dt(tc9563, child, port); + if (ret) + break; + /* Embedded ethernet device are under DSP3 */ + if (port == TC9563_DSP3) { + for_each_child_of_node_scoped(child, child1) { + port++; + ret = tc9563_pwrctrl_parse_device_dt(tc9563, + child1, port); + if (ret) + break; + } + } + } + if (ret) { + dev_err(dev, "failed to parse device tree properties: %d\n", ret); + goto remove_i2c; + } + + tc9563->pwrctrl.power_on = tc9563_pwrctrl_power_on; + tc9563->pwrctrl.power_off = tc9563_pwrctrl_power_off; + + ret = devm_pci_pwrctrl_device_set_ready(dev, &tc9563->pwrctrl); + if (ret) + goto power_off; + + return 0; + +power_off: + tc9563_pwrctrl_power_off(&tc9563->pwrctrl); +remove_i2c: + i2c_unregister_device(tc9563->client); + put_device(&tc9563->adapter->dev); + return ret; +} + +static void tc9563_pwrctrl_remove(struct platform_device *pdev) +{ + struct pci_pwrctrl *pwrctrl = dev_get_drvdata(&pdev->dev); + struct pci_pwrctrl_tc9563 *tc9563 = container_of(pwrctrl, + struct pci_pwrctrl_tc9563, pwrctrl); + + tc9563_pwrctrl_power_off(&tc9563->pwrctrl); + i2c_unregister_device(tc9563->client); + put_device(&tc9563->adapter->dev); +} + +static const struct of_device_id tc9563_pwrctrl_of_match[] = { + { .compatible = "pci1179,0623"}, + { } +}; +MODULE_DEVICE_TABLE(of, tc9563_pwrctrl_of_match); + +static struct platform_driver tc9563_pwrctrl_driver = { + .driver = { + .name = "pwrctrl-tc9563", + .of_match_table = tc9563_pwrctrl_of_match, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .probe = tc9563_pwrctrl_probe, + .remove = tc9563_pwrctrl_remove, +}; +module_platform_driver(tc9563_pwrctrl_driver); + +MODULE_AUTHOR("Krishna chaitanya chundru "); +MODULE_DESCRIPTION("TC956x power control driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index b9c252aa6fe08..4c4899deabad5 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -1359,6 +1359,16 @@ static void quirk_transparent_bridge(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82380FB, quirk_transparent_bridge); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TOSHIBA, 0x605, quirk_transparent_bridge); +/* + * Enabling Link Bandwidth Management Interrupts (BW notifications) can cause + * boot hangs on P45. + */ +static void quirk_p45_bw_notifications(struct pci_dev *dev) +{ + dev->no_bw_notif = 1; +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e21, quirk_p45_bw_notifications); + /* * Common misconfiguration of the MediaGX/Geode PCI master that will reduce * PCI bandwidth from 70MB/s to 25MB/s. See the GXM/GXLV/GX1 datasheets @@ -2527,6 +2537,57 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_FREESCALE, 0x0451, quirk_disable_aspm_l0s DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_PASEMI, 0xa002, quirk_disable_aspm_l0s_l1); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_HUAWEI, 0x1105, quirk_disable_aspm_l0s_l1); +static void quirk_disable_aspm_l1ss(struct pci_dev *dev) +{ + pci_disable_link_state(dev, PCIE_LINK_STATE_L1_1 | PCIE_LINK_STATE_L1_2); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_QCOM, 0x1103, quirk_disable_aspm_l1ss); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_QCOM, 0x1107, quirk_disable_aspm_l1ss); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_QCOM, 0x1112, quirk_disable_aspm_l1ss); + +static int disable_aspm_l1ss(struct pci_dev *dev, void *data) +{ + pci_disable_link_state(dev, PCIE_LINK_STATE_L1_1 | PCIE_LINK_STATE_L1_2); + return 0; +} + +static void quirk_disable_l1ss(struct pci_dev *dev) +{ + struct pci_dev *root_port; + struct pci_dev *upstream; + + pci_info(dev, "Disabling ASPM L1ss\n"); + + upstream = dev->bus->self; + while (upstream) { + if (pci_pcie_type(upstream) == PCI_EXP_TYPE_ROOT_PORT) { + root_port = upstream; + break; + } + + /* Move further up the hierarchy */ + upstream = upstream->bus->self; + } + + if (!root_port) { + pci_err(dev, "Could not find root port, cannot disable L1ss\n"); + return; + } + + pci_info(dev, "Found root-port with VID: %04x PID: %04x, disabling l1ss\n", + root_port->vendor, root_port->device); + + pci_walk_bus(root_port->subordinate, disable_aspm_l1ss, NULL); +} + +/* + * When ASPM L1.1/L1.2 is enabled on the upstream link of QPS615 switch, + * and Renesas uPD720201 USB 3.0 Controller is connected to one of the switch + * downstream ports, then uPD dies down and stops responding to host commands; + * disabling l1 sub-states in uPD attach case helps mitigate this issue. + */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_RENESAS, 0x0014, quirk_disable_l1ss); + /* * Some Pericom PCIe-to-PCI bridges in reverse mode need the PCIe Retrain * Link bit cleared after starting the link retrain process to allow this @@ -3748,6 +3809,14 @@ static void quirk_no_bus_reset(struct pci_dev *dev) dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; } +/* + * After asserting Secondary Bus Reset to downstream devices via a GB10 + * Root Port, the link may not retrain correctly. + * https://lore.kernel.org/r/20251113084441.2124737-1-Johnny-CC.Chang@mediatek.com + */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, 0x22CE, quirk_no_bus_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, 0x22D0, quirk_no_bus_reset); + /* * Some NVIDIA GPU devices do not work with bus reset, SBR needs to be * prevented for those affected devices. @@ -3791,6 +3860,16 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CAVIUM, 0xa100, quirk_no_bus_reset); */ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TI, 0xb005, quirk_no_bus_reset); +/* + * Reports from users making use of PCI device assignment with ASM1164 + * controllers indicate an issue with bus reset where the device fails to + * retrain. The issue appears more common in configurations with multiple + * controllers. The device does indicate PM reset support (NoSoftRst-), + * therefore this still leaves a viable reset method. + * https://forum.proxmox.com/threads/problems-with-pcie-passthrough-with-two-identical-devices.149003/ + */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ASMEDIA, 0x1164, quirk_no_bus_reset); + static void quirk_no_pm_reset(struct pci_dev *dev) { /* @@ -5107,6 +5186,10 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_QCOM, 0x0401, pci_quirk_qcom_rp_acs }, /* QCOM SA8775P root port */ { PCI_VENDOR_ID_QCOM, 0x0115, pci_quirk_qcom_rp_acs }, + /* QCOM Hamoa root port */ + { PCI_VENDOR_ID_QCOM, 0x0111, pci_quirk_qcom_rp_acs }, + /* QCOM Glymur root port */ + { PCI_VENDOR_ID_QCOM, 0x0120, pci_quirk_qcom_rp_acs }, /* HXT SD4800 root ports. The ACS design is same as QCOM QDF2xxx */ { PCI_VENDOR_ID_HXT, 0x0401, pci_quirk_qcom_rp_acs }, /* Intel PCH root ports */ @@ -5581,6 +5664,7 @@ static void quirk_no_ext_tags(struct pci_dev *pdev) pci_walk_bus(bridge->bus, pci_configure_extended_tags, NULL); } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1004, quirk_no_ext_tags); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1005, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0132, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0140, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0141, quirk_no_ext_tags); @@ -6188,6 +6272,10 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) { @@ -6341,3 +6429,13 @@ static void pci_mask_replay_timer_timeout(struct pci_dev *pdev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_GLI, 0x9750, pci_mask_replay_timer_timeout); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_GLI, 0x9755, pci_mask_replay_timer_timeout); #endif + +/* + * Renesas PCIe-to-USB bridge UPD720201 does not advertise D3cold + * capability by default until firmware is loaded post-enumeration. + */ +static void quirk_enable_d3cold(struct pci_dev *dev) +{ + dev->pme_support = dev->pme_support | (1 << PCI_D3cold); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_RENESAS, 0x0014, quirk_enable_d3cold); diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index ce5c25adef551..2a22595f38af6 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -17,25 +17,6 @@ static void pci_free_resources(struct pci_dev *dev) } } -static void pci_pwrctrl_unregister(struct device *dev) -{ - struct device_node *np; - struct platform_device *pdev; - - np = dev_of_node(dev); - if (!np) - return; - - pdev = of_find_device_by_node(np); - if (!pdev) - return; - - of_device_unregister(pdev); - put_device(&pdev->dev); - - of_node_clear_flag(np, OF_POPULATED); -} - static void pci_stop_dev(struct pci_dev *dev) { pci_pme_active(dev, false); @@ -66,7 +47,6 @@ static void pci_destroy_dev(struct pci_dev *dev) pci_doe_destroy(dev); pcie_aspm_exit_link_state(dev); pci_bridge_d3_update(dev); - pci_pwrctrl_unregister(&dev->dev); pci_free_resources(dev); put_device(&dev->dev); } diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 3645f392a9fd3..c2d640164f697 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -14,6 +14,7 @@ * tighter packing. Prefetchable range support. */ +#include #include #include #include @@ -221,14 +222,21 @@ static struct resource *pbus_select_window_for_type(struct pci_bus *bus, switch (iores_type) { case IORESOURCE_IO: - return pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + win = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + if (win && (win->flags & IORESOURCE_IO)) + return win; + return NULL; case IORESOURCE_MEM: mmio = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_MEM_WINDOW); mmio_pref = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_PREF_MEM_WINDOW); - if (!(type & IORESOURCE_PREFETCH) || - !(mmio_pref->flags & IORESOURCE_MEM)) + if (mmio && !(mmio->flags & IORESOURCE_MEM)) + mmio = NULL; + if (mmio_pref && !(mmio_pref->flags & IORESOURCE_MEM)) + mmio_pref = NULL; + + if (!(type & IORESOURCE_PREFETCH) || !mmio_pref) return mmio; if ((type & IORESOURCE_MEM_64) || @@ -452,7 +460,7 @@ static void reassign_resources_sorted(struct list_head *realloc_head, "%s %pR: ignoring failure in optional allocation\n", res_name, res); } - } else if (add_size > 0) { + } else if (add_size > 0 || !IS_ALIGNED(res->start, align)) { res->flags |= add_res->flags & (IORESOURCE_STARTALIGN|IORESOURCE_SIZEALIGN); if (pci_reassign_resource(dev, idx, add_size, align)) @@ -1066,16 +1074,13 @@ static resource_size_t calculate_memsize(resource_size_t size, resource_size_t min_size, resource_size_t add_size, resource_size_t children_add_size, - resource_size_t old_size, resource_size_t align) { if (size < min_size) size = min_size; - if (old_size == 1) - old_size = 0; size = max(size, add_size) + children_add_size; - return ALIGN(max(size, old_size), align); + return ALIGN(size, align); } resource_size_t __weak pcibios_window_alignment(struct pci_bus *bus, @@ -1223,66 +1228,43 @@ static inline resource_size_t calculate_mem_align(resource_size_t *aligns, return min_align; } -/** - * pbus_upstream_space_available - Check no upstream resource limits allocation - * @bus: The bus - * @res: The resource to help select the correct bridge window - * @size: The size required from the bridge window - * @align: Required alignment for the resource - * - * Check that @size can fit inside the upstream bridge resources that are - * already assigned. Select the upstream bridge window based on the type of - * @res. - * - * Return: %true if enough space is available on all assigned upstream +/* + * Calculate bridge window head alignment that leaves no gaps in between * resources. */ -static bool pbus_upstream_space_available(struct pci_bus *bus, - struct resource *res, - resource_size_t size, - resource_size_t align) -{ - struct resource_constraint constraint = { - .max = RESOURCE_SIZE_MAX, - .align = align, - }; - struct pci_bus *downstream = bus; - - while ((bus = bus->parent)) { - if (pci_is_root_bus(bus)) - break; +static resource_size_t calculate_head_align(resource_size_t *aligns, + int max_order) +{ + resource_size_t head_align = 1; + resource_size_t remainder = 0; + int order; - res = pbus_select_window(bus, res); - if (!res) - return false; - if (!res->parent) - continue; + /* Take the largest alignment as the starting point. */ + head_align <<= max_order + __ffs(SZ_1M); - if (resource_size(res) >= size) { - struct resource gap = {}; + for (order = max_order - 1; order >= 0; order--) { + resource_size_t align1 = 1; - if (find_resource_space(res, &gap, size, &constraint) == 0) { - gap.flags = res->flags; - pci_dbg(bus->self, - "Assigned bridge window %pR to %pR free space at %pR\n", - res, &bus->busn_res, &gap); - return true; - } - } + align1 <<= order + __ffs(SZ_1M); - if (bus->self) { - pci_info(bus->self, - "Assigned bridge window %pR to %pR cannot fit 0x%llx required for %s bridging to %pR\n", - res, &bus->busn_res, - (unsigned long long)size, - pci_name(downstream->self), - &downstream->busn_res); - } + /* + * Account smaller resources with alignment < max_order that + * could be used to fill head room if alignment less than + * max_order is used. + */ + remainder += aligns[order]; - return false; + /* + * Test if head fill is enough to satisfy the alignment of + * the larger resources after reducing the alignment. + */ + while ((head_align > align1) && (remainder >= head_align / 2)) { + head_align /= 2; + remainder -= head_align; + } } - return true; + return head_align; } /** @@ -1310,14 +1292,12 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, { struct pci_dev *dev; resource_size_t min_align, win_align, align, size, size0, size1 = 0; - resource_size_t aligns[28]; /* Alignments from 1MB to 128TB */ + resource_size_t aligns[28] = {}; /* Alignments from 1MB to 128TB */ int order, max_order; struct resource *b_res = pbus_select_window_for_type(bus, type); resource_size_t children_add_size = 0; resource_size_t children_add_align = 0; resource_size_t add_align = 0; - resource_size_t relaxed_align; - resource_size_t old_size; if (!b_res) return; @@ -1326,7 +1306,6 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, if (b_res->parent) return; - memset(aligns, 0, sizeof(aligns)); max_order = 0; size = 0; @@ -1371,12 +1350,8 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, continue; } size += max(r_size, align); - /* - * Exclude ranges with size > align from calculation of - * the alignment. - */ - if (r_size <= align) - aligns[order] += align; + + aligns[order] += align; if (order > max_order) max_order = order; @@ -1388,44 +1363,20 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, } } - old_size = resource_size(b_res); win_align = window_alignment(bus, b_res->flags); - min_align = calculate_mem_align(aligns, max_order); + min_align = calculate_head_align(aligns, max_order); min_align = max(min_align, win_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, min_align); + size0 = calculate_memsize(size, min_size, 0, 0, win_align); if (size0) { resource_set_range(b_res, min_align, size0); b_res->flags &= ~IORESOURCE_DISABLED; } - if (bus->self && size0 && - !pbus_upstream_space_available(bus, b_res, size0, min_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, win_align); - resource_set_range(b_res, min_align, size0); - pci_info(bus->self, "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } - if (realloc_head && (add_size > 0 || children_add_size > 0)) { add_align = max(min_align, add_align); size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, add_align); - - if (bus->self && size1 && - !pbus_upstream_space_available(bus, b_res, size1, add_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, win_align); - pci_info(bus->self, - "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } + win_align); } if (!size0 && !size1) { @@ -1438,12 +1389,13 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, resource_set_range(b_res, min_align, size0); b_res->flags |= IORESOURCE_STARTALIGN; - if (bus->self && size1 > size0 && realloc_head) { + if (bus->self && realloc_head && (size1 > size0 || add_align > min_align)) { b_res->flags &= ~IORESOURCE_DISABLED; - add_to_list(realloc_head, bus->self, b_res, size1-size0, add_align); + add_size = size1 > size0 ? size1 - size0 : 0; + add_to_list(realloc_head, bus->self, b_res, add_size, add_align); pci_info(bus->self, "bridge window %pR to %pR add_size %llx add_align %llx\n", b_res, &bus->busn_res, - (unsigned long long) (size1 - size0), + (unsigned long long) add_size, (unsigned long long) add_align); } } @@ -1729,6 +1681,8 @@ static void pci_claim_bridge_resources(struct pci_dev *dev) if (!r->flags || r->parent) continue; + if (r->flags & IORESOURCE_DISABLED) + continue; pci_claim_bridge_resource(dev, i); } @@ -2504,6 +2458,11 @@ int pbus_reassign_bridge_resources(struct pci_bus *bus, struct resource *res) bridge = dev_res->dev; i = pci_resource_num(bridge, res); + if (res->parent) { + release_child_resources(res); + pci_release_resource(bridge, i); + } + restore_dev_resource(dev_res); pci_claim_resource(bridge, i); diff --git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c index 436fa7f4c3873..baa242b140993 100644 --- a/drivers/pci/vgaarb.c +++ b/drivers/pci/vgaarb.c @@ -652,13 +652,6 @@ static bool vga_is_boot_device(struct vga_device *vgadev) return true; } - /* - * Vgadev has neither IO nor MEM enabled. If we haven't found any - * other VGA devices, it is the best candidate so far. - */ - if (!boot_vga) - return true; - return false; } diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 23245352a3fc0..4fbafc4b79843 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -210,6 +210,7 @@ enum cmn_model { enum cmn_part { PART_CMN600 = 0x434, PART_CMN650 = 0x436, + PART_CMN600AE = 0x438, PART_CMN700 = 0x43c, PART_CI700 = 0x43a, PART_CMN_S3 = 0x43e, @@ -2266,6 +2267,9 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) reg = readq_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_01); part = FIELD_GET(CMN_CFGM_PID0_PART_0, reg); part |= FIELD_GET(CMN_CFGM_PID1_PART_1, reg) << 8; + /* 600AE is close enough that it's not really worth more complexity */ + if (part == PART_CMN600AE) + part = PART_CMN600; if (cmn->part && cmn->part != part) dev_warn(cmn->dev, "Firmware binding mismatch: expected part number 0x%x, found 0x%x\n", @@ -2418,6 +2422,15 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn); dn->portid_bits = xp->portid_bits; dn->deviceid_bits = xp->deviceid_bits; + /* + * Logical IDs are assigned from 0 per node type, so as + * soon as we see one bigger than expected, we can assume + * there are more than we can cope with. + */ + if (dn->logid > CMN_MAX_NODES_PER_EVENT) { + dev_err(cmn->dev, "Node ID invalid for supported CMN versions: %d\n", dn->logid); + return -ENODEV; + } switch (dn->type) { case CMN_TYPE_DTC: @@ -2467,7 +2480,7 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) break; /* Something has gone horribly wrong */ default: - dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); + dev_err(cmn->dev, "Device node type invalid for supported CMN versions: 0x%x\n", dn->type); return -ENODEV; } } @@ -2495,6 +2508,10 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) cmn->mesh_x = cmn->num_xps; cmn->mesh_y = cmn->num_xps / cmn->mesh_x; + if (max(cmn->mesh_x, cmn->mesh_y) > CMN_MAX_DIMENSION) { + dev_err(cmn->dev, "Mesh size invalid for supported CMN versions: %dx%d\n", cmn->mesh_x, cmn->mesh_y); + return -ENODEV; + } /* 1x1 config plays havoc with XP event encodings */ if (cmn->num_xps == 1) dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index efa9b229e7012..e0d4293f06f93 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -1365,8 +1365,10 @@ void arm_cspmu_impl_unregister(const struct arm_cspmu_impl_match *impl_match) /* Unbind the driver from all matching backend devices. */ while ((dev = driver_find_device(&arm_cspmu_driver.driver, NULL, - match, arm_cspmu_match_device))) + match, arm_cspmu_match_device))) { device_release_driver(dev); + put_device(dev); + } mutex_lock(&arm_cspmu_lock); diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index fa50645feddad..e4e4e63c64c42 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -105,6 +105,8 @@ struct arm_spe_pmu { /* Keep track of our dynamic hotplug state */ static enum cpuhp_state arm_spe_pmu_online; +static void arm_spe_pmu_stop(struct perf_event *event, int flags); + enum arm_spe_pmu_buf_fault_action { SPE_PMU_BUF_FAULT_ACT_SPURIOUS, SPE_PMU_BUF_FAULT_ACT_FATAL, @@ -582,8 +584,8 @@ static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) return limit; } -static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, - struct perf_event *event) +static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, + struct perf_event *event) { u64 base, limit; struct arm_spe_pmu_buf *buf; @@ -597,7 +599,6 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, /* Start a new aux session */ buf = perf_aux_output_begin(handle, event); if (!buf) { - event->hw.state |= PERF_HES_STOPPED; /* * We still need to clear the limit pointer, since the * profiler might only be disabled by virtue of a fault. @@ -617,6 +618,7 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, out_write_limit: write_sysreg_s(limit, SYS_PMBLIMITR_EL1); + return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO; } static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) @@ -756,7 +758,10 @@ static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) * when we get to it. */ if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { - arm_spe_perf_aux_output_begin(handle, event); + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, PERF_EF_UPDATE); + break; + } isb(); } break; @@ -851,9 +856,10 @@ static void arm_spe_pmu_start(struct perf_event *event, int flags) struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); hwc->state = 0; - arm_spe_perf_aux_output_begin(handle, event); - if (hwc->state) + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, 0); return; + } reg = arm_spe_event_to_pmsfcr(event); write_sysreg_s(reg, SYS_PMSFCR_EL1); diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index d094030220bf2..68a54d97d2a8a 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -877,7 +877,7 @@ static int cxl_pmu_probe(struct device *dev) if (!irq_name) return -ENOMEM; - rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_ONESHOT, + rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_NO_THREAD, irq_name, info); if (rc) return rc; diff --git a/drivers/phy/broadcom/phy-bcm-ns-usb3.c b/drivers/phy/broadcom/phy-bcm-ns-usb3.c index 9f995e156f755..6e56498d0644b 100644 --- a/drivers/phy/broadcom/phy-bcm-ns-usb3.c +++ b/drivers/phy/broadcom/phy-bcm-ns-usb3.c @@ -203,7 +203,7 @@ static int bcm_ns_usb3_mdio_probe(struct mdio_device *mdiodev) usb3->dev = dev; usb3->mdiodev = mdiodev; - usb3->family = (enum bcm_ns_family)device_get_match_data(dev); + usb3->family = (unsigned long)device_get_match_data(dev); syscon_np = of_parse_phandle(dev->of_node, "usb3-dmp-syscon", 0); err = of_address_to_resource(syscon_np, 0, &res); diff --git a/drivers/phy/broadcom/phy-bcm63xx-usbh.c b/drivers/phy/broadcom/phy-bcm63xx-usbh.c index 647644de041bb..29fd6791bae64 100644 --- a/drivers/phy/broadcom/phy-bcm63xx-usbh.c +++ b/drivers/phy/broadcom/phy-bcm63xx-usbh.c @@ -375,7 +375,7 @@ static struct phy *bcm63xx_usbh_phy_xlate(struct device *dev, return of_phy_simple_xlate(dev, args); } -static int __init bcm63xx_usbh_phy_probe(struct platform_device *pdev) +static int bcm63xx_usbh_phy_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct bcm63xx_usbh_phy *usbh; @@ -432,7 +432,7 @@ static int __init bcm63xx_usbh_phy_probe(struct platform_device *pdev) return 0; } -static const struct of_device_id bcm63xx_usbh_phy_ids[] __initconst = { +static const struct of_device_id bcm63xx_usbh_phy_ids[] = { { .compatible = "brcm,bcm6318-usbh-phy", .data = &usbh_bcm6318 }, { .compatible = "brcm,bcm6328-usbh-phy", .data = &usbh_bcm6328 }, { .compatible = "brcm,bcm6358-usbh-phy", .data = &usbh_bcm6358 }, @@ -443,7 +443,7 @@ static const struct of_device_id bcm63xx_usbh_phy_ids[] __initconst = { }; MODULE_DEVICE_TABLE(of, bcm63xx_usbh_phy_ids); -static struct platform_driver bcm63xx_usbh_phy_driver __refdata = { +static struct platform_driver bcm63xx_usbh_phy_driver = { .driver = { .name = "bcm63xx-usbh-phy", .of_match_table = bcm63xx_usbh_phy_ids, diff --git a/drivers/phy/cadence/phy-cadence-torrent.c b/drivers/phy/cadence/phy-cadence-torrent.c index 37fa4bad6bd72..877f22177c699 100644 --- a/drivers/phy/cadence/phy-cadence-torrent.c +++ b/drivers/phy/cadence/phy-cadence-torrent.c @@ -397,6 +397,7 @@ struct cdns_torrent_refclk_driver { struct clk_hw hw; struct regmap_field *cmn_fields[REFCLK_OUT_NUM_CMN_CONFIG]; struct clk_init_data clk_data; + u8 parent_index; }; #define to_cdns_torrent_refclk_driver(_hw) \ @@ -3326,11 +3327,29 @@ static const struct cdns_torrent_vals sgmii_qsgmii_xcvr_diag_ln_vals = { .num_regs = ARRAY_SIZE(sgmii_qsgmii_xcvr_diag_ln_regs), }; +static void cdns_torrent_refclk_driver_suspend(struct cdns_torrent_phy *cdns_phy) +{ + struct clk_hw *hw = cdns_phy->clk_hw_data->hws[CDNS_TORRENT_REFCLK_DRIVER]; + struct cdns_torrent_refclk_driver *refclk_driver = to_cdns_torrent_refclk_driver(hw); + + refclk_driver->parent_index = cdns_torrent_refclk_driver_get_parent(hw); +} + +static int cdns_torrent_refclk_driver_resume(struct cdns_torrent_phy *cdns_phy) +{ + struct clk_hw *hw = cdns_phy->clk_hw_data->hws[CDNS_TORRENT_REFCLK_DRIVER]; + struct cdns_torrent_refclk_driver *refclk_driver = to_cdns_torrent_refclk_driver(hw); + + return cdns_torrent_refclk_driver_set_parent(hw, refclk_driver->parent_index); +} + static int cdns_torrent_phy_suspend_noirq(struct device *dev) { struct cdns_torrent_phy *cdns_phy = dev_get_drvdata(dev); int i; + cdns_torrent_refclk_driver_suspend(cdns_phy); + reset_control_assert(cdns_phy->phy_rst); reset_control_assert(cdns_phy->apb_rst); for (i = 0; i < cdns_phy->nsubnodes; i++) @@ -3352,6 +3371,10 @@ static int cdns_torrent_phy_resume_noirq(struct device *dev) int node = cdns_phy->nsubnodes; int ret, i; + ret = cdns_torrent_refclk_driver_resume(cdns_phy); + if (ret) + return ret; + ret = cdns_torrent_clk(cdns_phy); if (ret) return ret; diff --git a/drivers/phy/freescale/phy-fsl-imx8m-pcie.c b/drivers/phy/freescale/phy-fsl-imx8m-pcie.c index 68fcc8114d750..7f5600103a001 100644 --- a/drivers/phy/freescale/phy-fsl-imx8m-pcie.c +++ b/drivers/phy/freescale/phy-fsl-imx8m-pcie.c @@ -89,7 +89,8 @@ static int imx8_pcie_phy_power_on(struct phy *phy) writel(imx8_phy->tx_deemph_gen2, imx8_phy->base + PCIE_PHY_TRSV_REG6); break; - case IMX8MP: /* Do nothing. */ + case IMX8MP: + reset_control_assert(imx8_phy->reset); break; } diff --git a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c index b94f242420fc7..8e7b6e10e1f0b 100644 --- a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c +++ b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c @@ -124,8 +124,6 @@ struct imx8mq_usb_phy { static void tca_blk_orientation_set(struct tca_blk *tca, enum typec_orientation orientation); -#ifdef CONFIG_TYPEC - static int tca_blk_typec_switch_set(struct typec_switch_dev *sw, enum typec_orientation orientation) { @@ -173,18 +171,6 @@ static void tca_blk_put_typec_switch(struct typec_switch_dev *sw) typec_switch_unregister(sw); } -#else - -static struct typec_switch_dev *tca_blk_get_typec_switch(struct platform_device *pdev, - struct imx8mq_usb_phy *imx_phy) -{ - return NULL; -} - -static void tca_blk_put_typec_switch(struct typec_switch_dev *sw) {} - -#endif /* CONFIG_TYPEC */ - static void tca_blk_orientation_set(struct tca_blk *tca, enum typec_orientation orientation) { @@ -502,6 +488,7 @@ static void imx8m_phy_tune(struct imx8mq_usb_phy *imx_phy) if (imx_phy->pcs_tx_swing_full != PHY_TUNE_DEFAULT) { value = readl(imx_phy->base + PHY_CTRL5); + value &= ~PHY_CTRL5_PCS_TX_SWING_FULL_MASK; value |= FIELD_PREP(PHY_CTRL5_PCS_TX_SWING_FULL_MASK, imx_phy->pcs_tx_swing_full); writel(value, imx_phy->base + PHY_CTRL5); @@ -675,6 +662,8 @@ static int imx8mq_usb_phy_probe(struct platform_device *pdev) if (!imx_phy) return -ENOMEM; + platform_set_drvdata(pdev, imx_phy); + imx_phy->clk = devm_clk_get(dev, "phy"); if (IS_ERR(imx_phy->clk)) { dev_err(dev, "failed to get imx8mq usb phy clock\n"); @@ -724,6 +713,7 @@ static struct platform_driver imx8mq_usb_phy_driver = { .driver = { .name = "imx8mq-usb-phy", .of_match_table = imx8mq_usb_phy_of_match, + .suppress_bind_attrs = true, } }; module_platform_driver(imx8mq_usb_phy_driver); diff --git a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c index 5dca93cd325c8..279b8ac7822df 100644 --- a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c +++ b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c @@ -251,7 +251,7 @@ static void imx_hsio_configure_clk_pad(struct phy *phy) struct imx_hsio_lane *lane = phy_get_drvdata(phy); struct imx_hsio_priv *priv = lane->priv; - if (strncmp(priv->refclk_pad, "output", 6) == 0) { + if (priv->refclk_pad && strncmp(priv->refclk_pad, "output", 6) == 0) { pll = true; regmap_update_bits(priv->misc, HSIO_CTRL0, HSIO_IOB_A_0_TXOE | HSIO_IOB_A_0_M1M0_MASK, @@ -533,7 +533,7 @@ static struct phy *imx_hsio_xlate(struct device *dev, static int imx_hsio_probe(struct platform_device *pdev) { - int i; + int i, ret; void __iomem *off; struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; @@ -545,6 +545,9 @@ static int imx_hsio_probe(struct platform_device *pdev) return -ENOMEM; priv->dev = &pdev->dev; priv->drvdata = of_device_get_match_data(dev); + ret = devm_mutex_init(dev, &priv->lock); + if (ret) + return ret; /* Get HSIO configuration mode */ if (of_property_read_string(np, "fsl,hsio-cfg", &priv->hsio_cfg)) diff --git a/drivers/phy/marvell/phy-mvebu-cp110-utmi.c b/drivers/phy/marvell/phy-mvebu-cp110-utmi.c index 59903f86b13f5..dd3e515a8e865 100644 --- a/drivers/phy/marvell/phy-mvebu-cp110-utmi.c +++ b/drivers/phy/marvell/phy-mvebu-cp110-utmi.c @@ -338,7 +338,7 @@ static int mvebu_cp110_utmi_phy_probe(struct platform_device *pdev) return -ENOMEM; } - port->dr_mode = of_usb_get_dr_mode_by_phy(child, -1); + port->dr_mode = of_usb_get_dr_mode_by_phy(child, 0); if ((port->dr_mode != USB_DR_MODE_HOST) && (port->dr_mode != USB_DR_MODE_PERIPHERAL)) { dev_err(&pdev->dev, diff --git a/drivers/phy/qualcomm/Kconfig b/drivers/phy/qualcomm/Kconfig index 60a0ead127fa9..ea33025a40fd0 100644 --- a/drivers/phy/qualcomm/Kconfig +++ b/drivers/phy/qualcomm/Kconfig @@ -28,6 +28,19 @@ config PHY_QCOM_EDP Enable this driver to support the Qualcomm eDP PHY found in various Qualcomm chipsets. +config PHY_QCOM_MIPI_CSI2 + tristate "Qualcomm MIPI CSI2 PHY driver" + depends on ARCH_QCOM || COMPILE_TEST + depends on OF + depends on COMMON_CLK + select GENERIC_PHY + select GENERIC_PHY_MIPI_DPHY + help + Enable this to support the MIPI CSI2 PHY driver found in various + Qualcomm chipsets. This PHY is used to connect MIPI CSI2 + camera sensors to the CSI Decoder in the Qualcomm Camera Subsystem + CAMSS. + config PHY_QCOM_IPQ4019_USB tristate "Qualcomm IPQ4019 USB PHY driver" depends on OF && (ARCH_QCOM || COMPILE_TEST) diff --git a/drivers/phy/qualcomm/Makefile b/drivers/phy/qualcomm/Makefile index b71a6a0bed3f1..382cb594b06b6 100644 --- a/drivers/phy/qualcomm/Makefile +++ b/drivers/phy/qualcomm/Makefile @@ -6,6 +6,11 @@ obj-$(CONFIG_PHY_QCOM_IPQ4019_USB) += phy-qcom-ipq4019-usb.o obj-$(CONFIG_PHY_QCOM_IPQ806X_SATA) += phy-qcom-ipq806x-sata.o obj-$(CONFIG_PHY_QCOM_M31_USB) += phy-qcom-m31.o obj-$(CONFIG_PHY_QCOM_M31_EUSB) += phy-qcom-m31-eusb2.o + +phy-qcom-mipi-csi2-objs += phy-qcom-mipi-csi2-core.o \ + phy-qcom-mipi-csi2-3ph-dphy.o +obj-$(CONFIG_PHY_QCOM_MIPI_CSI2) += phy-qcom-mipi-csi2.o + obj-$(CONFIG_PHY_QCOM_PCIE2) += phy-qcom-pcie2.o obj-$(CONFIG_PHY_QCOM_QMP_COMBO) += phy-qcom-qmp-combo.o phy-qcom-qmp-usbc.o diff --git a/drivers/phy/qualcomm/phy-qcom-edp.c b/drivers/phy/qualcomm/phy-qcom-edp.c index f1b51018683d5..804363d58f3fa 100644 --- a/drivers/phy/qualcomm/phy-qcom-edp.c +++ b/drivers/phy/qualcomm/phy-qcom-edp.c @@ -103,7 +103,9 @@ struct qcom_edp { struct phy_configure_opts_dp dp_opts; - struct clk_bulk_data clks[2]; + struct clk_bulk_data *clks; + int num_clks; + struct regulator_bulk_data supplies[2]; bool is_edp; @@ -218,7 +220,7 @@ static int qcom_edp_phy_init(struct phy *phy) if (ret) return ret; - ret = clk_bulk_prepare_enable(ARRAY_SIZE(edp->clks), edp->clks); + ret = clk_bulk_prepare_enable(edp->num_clks, edp->clks); if (ret) goto out_disable_supplies; @@ -541,6 +543,7 @@ static const struct qcom_edp_phy_cfg sa8775p_dp_phy_cfg = { static const struct qcom_edp_phy_cfg sc7280_dp_phy_cfg = { .aux_cfg = edp_phy_aux_cfg_v4, + .swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg, .ver_ops = &qcom_edp_phy_ops_v4, }; @@ -885,7 +888,7 @@ static int qcom_edp_phy_exit(struct phy *phy) { struct qcom_edp *edp = phy_get_drvdata(phy); - clk_bulk_disable_unprepare(ARRAY_SIZE(edp->clks), edp->clks); + clk_bulk_disable_unprepare(edp->num_clks, edp->clks); regulator_bulk_disable(ARRAY_SIZE(edp->supplies), edp->supplies); return 0; @@ -1092,11 +1095,9 @@ static int qcom_edp_phy_probe(struct platform_device *pdev) if (IS_ERR(edp->pll)) return PTR_ERR(edp->pll); - edp->clks[0].id = "aux"; - edp->clks[1].id = "cfg_ahb"; - ret = devm_clk_bulk_get(dev, ARRAY_SIZE(edp->clks), edp->clks); - if (ret) - return ret; + edp->num_clks = devm_clk_bulk_get_all(dev, &edp->clks); + if (edp->num_clks < 0) + return dev_err_probe(dev, edp->num_clks, "failed to get clocks\n"); edp->supplies[0].supply = "vdda-phy"; edp->supplies[1].supply = "vdda-pll"; diff --git a/drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c new file mode 100644 index 0000000000000..b1eb2b28b2da2 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm MSM Camera Subsystem - CSIPHY Module 3phase v1.0 + * + * Copyright (c) 2011-2015, The Linux Foundation. All rights reserved. + * Copyright (C) 2016-2025 Linaro Ltd. + */ + +#include +#include +#include +#include + +#include "phy-qcom-mipi-csi2.h" + +#define CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(offset, n) ((offset) + 0x4 * (n)) +#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL0_PHY_SW_RESET BIT(0) +#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL5_CLK_ENABLE BIT(7) +#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_COMMON_PWRDN_B BIT(0) +#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_SHOW_REV_ID BIT(1) +#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL10_IRQ_CLEAR_CMD BIT(0) +#define CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(offset, n) ((offset) + 0xb0 + 0x4 * (n)) + +/* + * 3 phase CSI has 19 common status regs with only 0-10 being used + * and 11-18 being reserved. + */ +#define CSI_COMMON_STATUS_NUM 11 +/* + * There are a number of common control registers + * The offset to clear the CSIPHY IRQ status starts @ 22 + * So to clear CSI_COMMON_STATUS0 this is CSI_COMMON_CONTROL22, STATUS1 is + * CONTROL23 and so on + */ +#define CSI_CTRL_STATUS_INDEX 22 + +/* + * There are 43 COMMON_CTRL registers with regs after # 33 being reserved + */ +#define CSI_CTRL_MAX 33 + +#define CSIPHY_DEFAULT_PARAMS 0 +#define CSIPHY_SETTLE_CNT_LOWER_BYTE 2 +#define CSIPHY_SKEW_CAL 7 + +/* 4nm 2PH v 2.1.2 2p5Gbps 4 lane DPHY mode */ +static const struct +mipi_csi2phy_lane_regs lane_regs_x1e80100[] = { + /* Power up lanes 2ph mode */ + {.reg_addr = 0x1014, .reg_data = 0xd5, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x101c, .reg_data = 0x7a, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x1018, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + + {.reg_addr = 0x0094, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x00a0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0090, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0098, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0094, .reg_data = 0x07, .delay_us = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0030, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0000, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0038, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x002c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0034, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x001c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0014, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x003c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0004, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0020, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0008, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {.reg_addr = 0x0010, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0094, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x005c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0060, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0064, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL}, + + {.reg_addr = 0x0e94, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0ea0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e90, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e98, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e94, .reg_data = 0x07, .delay_us = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e30, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e28, .reg_data = 0x04, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e00, .reg_data = 0x80, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e0c, .reg_data = 0xff, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e38, .reg_data = 0x1f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e2c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e34, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e1c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e14, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e3c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e04, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e20, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0e08, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {.reg_addr = 0x0e10, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS}, + + {.reg_addr = 0x0494, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x04a0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0490, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0498, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0494, .reg_data = 0x07, .delay_us = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0430, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0400, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0438, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x042c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0434, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x041c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0414, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x043c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0404, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0420, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0408, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {.reg_addr = 0x0410, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0494, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x045c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0460, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0464, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL}, + + {.reg_addr = 0x0894, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x08a0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0890, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0898, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0894, .reg_data = 0x07, .delay_us = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0830, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0800, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0838, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x082c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0834, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x081c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0814, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x083c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0804, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0820, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0808, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {.reg_addr = 0x0810, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0894, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x085c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0860, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0864, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL}, + + {.reg_addr = 0x0c94, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0ca0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c90, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c98, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c94, .reg_data = 0x07, .delay_us = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c30, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c00, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c38, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c2c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c34, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c1c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c14, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c3c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c04, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c20, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c08, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE}, + {.reg_addr = 0x0c10, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS}, + {.reg_addr = 0x0c94, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0c5c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0c60, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL}, + {.reg_addr = 0x0c64, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL}, +}; + +static inline const struct mipi_csi2phy_device_regs * +csi2phy_dev_to_regs(struct mipi_csi2phy_device *csi2phy) +{ + return &csi2phy->soc_cfg->reg_info; +} + +static void phy_qcom_mipi_csi2_hw_version_read(struct mipi_csi2phy_device *csi2phy) +{ + const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy); + u32 tmp; + + writel(CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_SHOW_REV_ID, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 6)); + + tmp = readl_relaxed(csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 12)); + csi2phy->hw_version = tmp; + + tmp = readl_relaxed(csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 13)); + csi2phy->hw_version |= (tmp << 8) & 0xFF00; + + tmp = readl_relaxed(csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 14)); + csi2phy->hw_version |= (tmp << 16) & 0xFF0000; + + tmp = readl_relaxed(csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 15)); + csi2phy->hw_version |= (tmp << 24) & 0xFF000000; + + dev_dbg_once(csi2phy->dev, "CSIPHY 3PH HW Version = 0x%08x\n", csi2phy->hw_version); +} + +/* + * phy_qcom_mipi_csi2_reset - Perform software reset on CSIPHY module + * @phy_qcom_mipi_csi2: CSIPHY device + */ +static void phy_qcom_mipi_csi2_reset(struct mipi_csi2phy_device *csi2phy) +{ + const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy); + + writel(CSIPHY_3PH_CMN_CSI_COMMON_CTRL0_PHY_SW_RESET, + csi2phy->base + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 0)); + usleep_range(5000, 8000); + writel(0x0, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 0)); +} + +/* + * phy_qcom_mipi_csi2_settle_cnt_calc - Calculate settle count value + * + * Helper function to calculate settle count value. This is + * based on the CSI2 T_hs_settle parameter which in turn + * is calculated based on the CSI2 transmitter link frequency. + * + * Return settle count value or 0 if the CSI2 link frequency + * is not available + */ +static u8 phy_qcom_mipi_csi2_settle_cnt_calc(s64 link_freq, u32 timer_clk_rate) +{ + u32 t_hs_prepare_max_ps; + u32 timer_period_ps; + u32 t_hs_settle_ps; + u8 settle_cnt; + u32 ui_ps; + + if (link_freq <= 0) + return 0; + + ui_ps = div_u64(PSEC_PER_SEC, link_freq); + ui_ps /= 2; + t_hs_prepare_max_ps = 85000 + 6 * ui_ps; + t_hs_settle_ps = t_hs_prepare_max_ps; + + timer_period_ps = div_u64(PSEC_PER_SEC, timer_clk_rate); + settle_cnt = t_hs_settle_ps / timer_period_ps - 6; + + return settle_cnt; +} + +static void +phy_qcom_mipi_csi2_gen2_config_lanes(struct mipi_csi2phy_device *csi2phy, + u8 settle_cnt) +{ + const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy); + const struct mipi_csi2phy_lane_regs *r = regs->init_seq; + int i, array_size = regs->lane_array_size; + u32 val; + + for (i = 0; i < array_size; i++, r++) { + switch (r->param_type) { + case CSIPHY_SETTLE_CNT_LOWER_BYTE: + val = settle_cnt & 0xff; + break; + case CSIPHY_SKEW_CAL: + /* TODO: support application of skew from dt flag */ + continue; + default: + val = r->reg_data; + break; + } + writel(val, csi2phy->base + r->reg_addr); + if (r->delay_us) + udelay(r->delay_us); + } +} + +static int phy_qcom_mipi_csi2_lanes_enable(struct mipi_csi2phy_device *csi2phy, + struct mipi_csi2phy_stream_cfg *cfg) +{ + const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy); + struct mipi_csi2phy_lanes_cfg *lane_cfg = &cfg->lane_cfg; + u8 settle_cnt; + u8 val; + int i; + + settle_cnt = phy_qcom_mipi_csi2_settle_cnt_calc(cfg->link_freq, csi2phy->timer_clk_rate); + + val = CSIPHY_3PH_CMN_CSI_COMMON_CTRL5_CLK_ENABLE; + for (i = 0; i < cfg->num_data_lanes; i++) + val |= BIT(lane_cfg->data[i].pos * 2); + + writel(val, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 5)); + + val = CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_COMMON_PWRDN_B; + writel(val, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 6)); + + val = 0x02; + writel(val, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 7)); + + val = 0x00; + writel(val, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 0)); + + phy_qcom_mipi_csi2_gen2_config_lanes(csi2phy, settle_cnt); + + /* IRQ_MASK registers - disable all interrupts */ + for (i = CSI_COMMON_STATUS_NUM; i < CSI_CTRL_STATUS_INDEX; i++) { + writel(0, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, i)); + } + + return 0; +} + +static void +phy_qcom_mipi_csi2_lanes_disable(struct mipi_csi2phy_device *csi2phy, + struct mipi_csi2phy_stream_cfg *cfg) +{ + const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy); + + writel(0, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 5)); + + writel(0, csi2phy->base + + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 6)); +} + +static const struct mipi_csi2phy_hw_ops phy_qcom_mipi_csi2_ops_3ph_1_0 = { + .hw_version_read = phy_qcom_mipi_csi2_hw_version_read, + .reset = phy_qcom_mipi_csi2_reset, + .lanes_enable = phy_qcom_mipi_csi2_lanes_enable, + .lanes_disable = phy_qcom_mipi_csi2_lanes_disable, +}; + +static const char * const x1e_clks[] = { + "core", + "timer" +}; + +static const char * const x1e_supplies[] = { + "vdda-0p9", + "vdda-1p2" +}; + +static const char * const x1e_genpd_names[] = { + "mx", + "mmcx", +}; + +const struct mipi_csi2phy_soc_cfg mipi_csi2_dphy_4nm_x1e = { + .ops = &phy_qcom_mipi_csi2_ops_3ph_1_0, + .reg_info = { + .init_seq = lane_regs_x1e80100, + .lane_array_size = ARRAY_SIZE(lane_regs_x1e80100), + .common_regs_offset = 0x1000, + }, + .supply_names = (const char **)x1e_supplies, + .num_supplies = ARRAY_SIZE(x1e_supplies), + .clk_names = (const char **)x1e_clks, + .num_clk = ARRAY_SIZE(x1e_clks), + .opp_clk = x1e_clks[0], + .timer_clk = x1e_clks[1], + .genpd_names = (const char **)x1e_genpd_names, + .num_genpd_names = ARRAY_SIZE(x1e_genpd_names), +}; diff --git a/drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c new file mode 100644 index 0000000000000..47acf0d586a15 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2025, Linaro Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "phy-qcom-mipi-csi2.h" + +static int +phy_qcom_mipi_csi2_set_clock_rates(struct mipi_csi2phy_device *csi2phy, + s64 link_freq) +{ + struct device *dev = csi2phy->dev; + unsigned long opp_rate = link_freq / 4; + struct dev_pm_opp *opp; + long timer_rate; + int ret; + + opp = dev_pm_opp_find_freq_ceil(dev, &opp_rate); + if (IS_ERR(opp)) { + dev_err(csi2phy->dev, "Couldn't find ceiling for %lld Hz\n", + link_freq); + return PTR_ERR(opp); + } + + for (int i = 0; i < csi2phy->num_pds; i++) { + unsigned int perf = dev_pm_opp_get_required_pstate(opp, i); + + ret = dev_pm_genpd_set_performance_state(csi2phy->pds[i], perf); + if (ret) { + dev_err(csi2phy->dev, "Couldn't set perf state %u\n", + perf); + dev_pm_opp_put(opp); + return ret; + } + } + dev_pm_opp_put(opp); + + ret = dev_pm_opp_set_rate(dev, opp_rate); + if (ret) { + dev_err(csi2phy->dev, "dev_pm_opp_set_rate() fail\n"); + return ret; + } + + timer_rate = clk_round_rate(csi2phy->timer_clk, link_freq / 4); + if (timer_rate < 0) + return timer_rate; + + ret = clk_set_rate(csi2phy->timer_clk, timer_rate); + if (ret) + return ret; + + csi2phy->timer_clk_rate = timer_rate; + + return 0; +} + +static int phy_qcom_mipi_csi2_configure(struct phy *phy, + union phy_configure_opts *opts) +{ + struct mipi_csi2phy_device *csi2phy = phy_get_drvdata(phy); + struct phy_configure_opts_mipi_dphy *dphy_cfg = &opts->mipi_dphy; + struct mipi_csi2phy_stream_cfg *stream_cfg = &csi2phy->stream_cfg; + int ret; + int i; + + ret = phy_mipi_dphy_config_validate(dphy_cfg); + if (ret) + return ret; + + if (dphy_cfg->lanes < 1 || dphy_cfg->lanes > CSI2_MAX_DATA_LANES) + return -EINVAL; + + stream_cfg->link_freq = dphy_cfg->hs_clk_rate; + stream_cfg->num_data_lanes = dphy_cfg->lanes; + + for (i = 0; i < stream_cfg->num_data_lanes; i++) { + stream_cfg->lane_cfg.data[i].pol = dphy_cfg->lane_polarities[i]; + stream_cfg->lane_cfg.data[i].pos = dphy_cfg->lane_positions[i]; + } + + stream_cfg->lane_cfg.clk.pol = dphy_cfg->clock_lane_polarity; + stream_cfg->lane_cfg.clk.pos = dphy_cfg->clock_lane_position; + + return 0; +} + +static int phy_qcom_mipi_csi2_power_on(struct phy *phy) +{ + struct mipi_csi2phy_device *csi2phy = phy_get_drvdata(phy); + const struct mipi_csi2phy_hw_ops *ops = csi2phy->soc_cfg->ops; + struct device *dev = &phy->dev; + int ret; + + ret = regulator_bulk_enable(csi2phy->soc_cfg->num_supplies, + csi2phy->supplies); + if (ret) + return ret; + + ret = phy_qcom_mipi_csi2_set_clock_rates(csi2phy, csi2phy->stream_cfg.link_freq); + if (ret) + goto poweroff_phy; + + ret = clk_bulk_prepare_enable(csi2phy->soc_cfg->num_clk, + csi2phy->clks); + if (ret) { + dev_err(dev, "failed to enable clocks, %d\n", ret); + goto poweroff_phy; + } + + ops->reset(csi2phy); + + ops->hw_version_read(csi2phy); + + return ops->lanes_enable(csi2phy, &csi2phy->stream_cfg); + +poweroff_phy: + regulator_bulk_disable(csi2phy->soc_cfg->num_supplies, + csi2phy->supplies); + + return ret; +} + +static int phy_qcom_mipi_csi2_power_off(struct phy *phy) +{ + struct mipi_csi2phy_device *csi2phy = phy_get_drvdata(phy); + int i; + + for (i = 0; i < csi2phy->num_pds; i++) + dev_pm_genpd_set_performance_state(csi2phy->pds[i], 0); + + clk_bulk_disable_unprepare(csi2phy->soc_cfg->num_clk, + csi2phy->clks); + regulator_bulk_disable(csi2phy->soc_cfg->num_supplies, + csi2phy->supplies); + + return 0; +} + +static const struct phy_ops phy_qcom_mipi_csi2_ops = { + .configure = phy_qcom_mipi_csi2_configure, + .power_on = phy_qcom_mipi_csi2_power_on, + .power_off = phy_qcom_mipi_csi2_power_off, + .owner = THIS_MODULE, +}; + +static struct phy *qcom_csi2_phy_xlate(struct device *dev, + const struct of_phandle_args *args) +{ + struct mipi_csi2phy_device *csi2phy = dev_get_drvdata(dev); + + if (args->args[0] != PHY_QCOM_CSI2_MODE_DPHY) { + dev_err(csi2phy->dev, "mode %d -EOPNOTSUPP\n", args->args[0]); + return ERR_PTR(-EOPNOTSUPP); + } + + csi2phy->phy_mode = args->args[0]; + + return csi2phy->phy; +} + +static int phy_qcom_mipi_csi2_probe(struct platform_device *pdev) +{ + unsigned int i, num_clk, num_supplies, num_pds; + struct mipi_csi2phy_device *csi2phy; + struct phy_provider *phy_provider; + struct device *dev = &pdev->dev; + struct phy *generic_phy; + int ret; + + csi2phy = devm_kzalloc(dev, sizeof(*csi2phy), GFP_KERNEL); + if (!csi2phy) + return -ENOMEM; + + csi2phy->dev = dev; + dev_set_drvdata(dev, csi2phy); + + csi2phy->soc_cfg = device_get_match_data(&pdev->dev); + + if (!csi2phy->soc_cfg) + return -EINVAL; + + num_clk = csi2phy->soc_cfg->num_clk; + csi2phy->clks = devm_kzalloc(dev, sizeof(*csi2phy->clks) * num_clk, GFP_KERNEL); + if (!csi2phy->clks) + return -ENOMEM; + + num_pds = csi2phy->soc_cfg->num_genpd_names; + if (!num_pds) + return -EINVAL; + + csi2phy->pds = devm_kzalloc(dev, sizeof(*csi2phy->pds) * num_pds, GFP_KERNEL); + if (!csi2phy->pds) + return -ENOMEM; + + for (i = 0; i < num_pds; i++) { + csi2phy->pds[i] = dev_pm_domain_attach_by_name(dev, + csi2phy->soc_cfg->genpd_names[i]); + if (IS_ERR(csi2phy->pds[i])) { + return dev_err_probe(dev, PTR_ERR(csi2phy->pds[i]), + "Failed to attach %s\n", + csi2phy->soc_cfg->genpd_names[i]); + } + } + csi2phy->num_pds = num_pds; + + for (i = 0; i < num_clk; i++) + csi2phy->clks[i].id = csi2phy->soc_cfg->clk_names[i]; + + ret = devm_clk_bulk_get(dev, num_clk, csi2phy->clks); + if (ret) + return dev_err_probe(dev, ret, "Failed to get clocks\n"); + + csi2phy->timer_clk = devm_clk_get(dev, csi2phy->soc_cfg->timer_clk); + if (IS_ERR(csi2phy->timer_clk)) { + return dev_err_probe(dev, PTR_ERR(csi2phy->timer_clk), + "Failed to get timer clock\n"); + } + + ret = devm_pm_opp_set_clkname(dev, csi2phy->soc_cfg->opp_clk); + if (ret) + return dev_err_probe(dev, ret, "Failed to set opp clkname\n"); + + ret = devm_pm_opp_of_add_table(dev); + if (ret && ret != -ENODEV) + return dev_err_probe(dev, ret, "invalid OPP table in device tree\n"); + + num_supplies = csi2phy->soc_cfg->num_supplies; + csi2phy->supplies = devm_kzalloc(dev, sizeof(*csi2phy->supplies) * num_supplies, + GFP_KERNEL); + if (!csi2phy->supplies) + return -ENOMEM; + + for (i = 0; i < num_supplies; i++) + csi2phy->supplies[i].supply = csi2phy->soc_cfg->supply_names[i]; + + ret = devm_regulator_bulk_get(dev, num_supplies, csi2phy->supplies); + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); + + csi2phy->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(csi2phy->base)) + return PTR_ERR(csi2phy->base); + + generic_phy = devm_phy_create(dev, NULL, &phy_qcom_mipi_csi2_ops); + if (IS_ERR(generic_phy)) { + ret = PTR_ERR(generic_phy); + return dev_err_probe(dev, ret, "failed to create phy\n"); + } + csi2phy->phy = generic_phy; + + phy_set_drvdata(generic_phy, csi2phy); + + phy_provider = devm_of_phy_provider_register(dev, qcom_csi2_phy_xlate); + if (!IS_ERR(phy_provider)) + dev_dbg(dev, "Registered MIPI CSI2 PHY device\n"); + + return PTR_ERR_OR_ZERO(phy_provider); +} + +static const struct of_device_id phy_qcom_mipi_csi2_of_match_table[] = { + { .compatible = "qcom,x1e80100-csi2-phy", .data = &mipi_csi2_dphy_4nm_x1e }, + { } +}; +MODULE_DEVICE_TABLE(of, phy_qcom_mipi_csi2_of_match_table); + +static struct platform_driver phy_qcom_mipi_csi2_driver = { + .probe = phy_qcom_mipi_csi2_probe, + .driver = { + .name = "qcom-mipi-csi2-phy", + .of_match_table = phy_qcom_mipi_csi2_of_match_table, + }, +}; + +module_platform_driver(phy_qcom_mipi_csi2_driver); + +MODULE_DESCRIPTION("Qualcomm MIPI CSI2 PHY driver"); +MODULE_AUTHOR("Bryan O'Donoghue "); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/qualcomm/phy-qcom-mipi-csi2.h b/drivers/phy/qualcomm/phy-qcom-mipi-csi2.h new file mode 100644 index 0000000000000..27607dea412f1 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-mipi-csi2.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * Qualcomm MIPI CSI2 CPHY/DPHY driver + * + * Copyright (C) 2025 Linaro Ltd. + */ +#ifndef __PHY_QCOM_MIPI_CSI2_H__ +#define __PHY_QCOM_MIPI_CSI2_H__ + +#include + +#define CSI2_MAX_DATA_LANES 4 + +struct mipi_csi2phy_lane { + u8 pos; + u8 pol; +}; + +struct mipi_csi2phy_lanes_cfg { + struct mipi_csi2phy_lane data[CSI2_MAX_DATA_LANES]; + struct mipi_csi2phy_lane clk; +}; + +struct mipi_csi2phy_stream_cfg { + s64 link_freq; + u8 num_data_lanes; + struct mipi_csi2phy_lanes_cfg lane_cfg; +}; + +struct mipi_csi2phy_device; + +struct mipi_csi2phy_hw_ops { + void (*hw_version_read)(struct mipi_csi2phy_device *csi2phy_dev); + void (*reset)(struct mipi_csi2phy_device *csi2phy_dev); + int (*lanes_enable)(struct mipi_csi2phy_device *csi2phy_dev, + struct mipi_csi2phy_stream_cfg *cfg); + void (*lanes_disable)(struct mipi_csi2phy_device *csi2phy_dev, + struct mipi_csi2phy_stream_cfg *cfg); +}; + +struct mipi_csi2phy_lane_regs { + const s32 reg_addr; + const s32 reg_data; + const u32 delay_us; + const u32 param_type; +}; + +struct mipi_csi2phy_device_regs { + const struct mipi_csi2phy_lane_regs *init_seq; + const int lane_array_size; + const u32 common_regs_offset; +}; + +struct mipi_csi2phy_soc_cfg { + const struct mipi_csi2phy_hw_ops *ops; + const struct mipi_csi2phy_device_regs reg_info; + + const char ** const supply_names; + const unsigned int num_supplies; + + const char ** const clk_names; + const unsigned int num_clk; + + const char * const opp_clk; + const char * const timer_clk; + + const char ** const genpd_names; + const unsigned int num_genpd_names; +}; + +struct mipi_csi2phy_device { + struct device *dev; + u8 phy_mode; + + struct phy *phy; + void __iomem *base; + + struct clk_bulk_data *clks; + struct clk *timer_clk; + u32 timer_clk_rate; + + struct regulator_bulk_data *supplies; + struct device **pds; + unsigned int num_pds; + + const struct mipi_csi2phy_soc_cfg *soc_cfg; + struct mipi_csi2phy_stream_cfg stream_cfg; + + u32 hw_version; +}; + +extern const struct mipi_csi2phy_soc_cfg mipi_csi2_dphy_4nm_x1e; + +#endif /* __PHY_QCOM_MIPI_CSI2_H__ */ diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-dp-phy-v2.h b/drivers/phy/qualcomm/phy-qcom-qmp-dp-phy-v2.h new file mode 100644 index 0000000000000..8b9572d3cdebb --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-dp-phy-v2.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_DP_PHY_V2_H_ +#define QCOM_PHY_QMP_DP_PHY_V2_H_ + +// /* Only for QMP V2 PHY - DP PHY registers */ +#define QSERDES_V2_DP_PHY_AUX_INTERRUPT_MASK 0x048 +#define QSERDES_V2_DP_PHY_AUX_INTERRUPT_CLEAR 0x04c +#define QSERDES_V2_DP_PHY_AUX_BIST_CFG 0x050 + +#define QSERDES_V2_DP_PHY_VCO_DIV 0x068 +#define QSERDES_V2_DP_PHY_TX0_TX1_LANE_CTL 0x06c +#define QSERDES_V2_DP_PHY_TX2_TX3_LANE_CTL 0x088 + +#define QSERDES_V2_DP_PHY_SPARE0 0x0ac +#define QSERDES_V2_DP_PHY_STATUS 0x0c0 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c index 62b1c845b6275..d9571af517061 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c @@ -100,6 +100,18 @@ static const unsigned int pciephy_v7_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_POWER_DOWN_CONTROL] = QPHY_V7_PCS_POWER_DOWN_CONTROL, }; +static const unsigned int pciephy_v8_regs_layout[QPHY_LAYOUT_SIZE] = { + [QPHY_START_CTRL] = QPHY_V8_PCS_START_CONTROL, + [QPHY_PCS_STATUS] = QPHY_V8_PCS_PCS_STATUS1, + [QPHY_PCS_POWER_DOWN_CONTROL] = QPHY_V8_PCS_POWER_DOWN_CONTROL, +}; + +static const unsigned int pciephy_v8_50_regs_layout[QPHY_LAYOUT_SIZE] = { + [QPHY_START_CTRL] = QPHY_V8_50_PCS_START_CONTROL, + [QPHY_PCS_STATUS] = QPHY_V8_50_PCS_STATUS1, + [QPHY_PCS_POWER_DOWN_CONTROL] = QPHY_V8_50_PCS_POWER_DOWN_CONTROL, +}; + static const struct qmp_phy_init_tbl msm8998_pcie_serdes_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V3_COM_BIAS_EN_CLKBUFLR_EN, 0x14), QMP_PHY_INIT_CFG(QSERDES_V3_COM_CLK_SELECT, 0x30), @@ -3072,6 +3084,7 @@ struct qmp_pcie_offsets { u16 rx2; u16 txz; u16 rxz; + u16 txrxz; u16 ln_shrd; }; @@ -3356,6 +3369,21 @@ static const struct qmp_pcie_offsets qmp_pcie_offsets_v6_30 = { .ln_shrd = 0x8000, }; +static const struct qmp_pcie_offsets qmp_pcie_offsets_v8 = { + .serdes = 0x1000, + .pcs = 0x1400, + .tx = 0x0000, + .rx = 0x0200, + .tx2 = 0x0800, + .rx2 = 0x0a00, +}; + +static const struct qmp_pcie_offsets qmp_pcie_offsets_v8_50 = { + .serdes = 0x8000, + .pcs = 0x9000, + .txrxz = 0xd000, +}; + static const struct qmp_phy_cfg ipq8074_pciephy_cfg = { .lanes = 1, @@ -3544,8 +3572,8 @@ static const struct qmp_phy_cfg qcs8300_qmp_gen4x2_pciephy_cfg = { .reset_list = sdm845_pciephy_reset_l, .num_resets = ARRAY_SIZE(sdm845_pciephy_reset_l), - .vreg_list = qmp_phy_vreg_l, - .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .vreg_list = sm8550_qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(sm8550_qmp_phy_vreg_l), .regs = pciephy_v5_regs_layout, .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, @@ -4295,8 +4323,8 @@ static const struct qmp_phy_cfg sa8775p_qmp_gen4x4_pciephy_cfg = { .reset_list = sdm845_pciephy_reset_l, .num_resets = ARRAY_SIZE(sdm845_pciephy_reset_l), - .vreg_list = qmp_phy_vreg_l, - .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .vreg_list = sm8550_qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(sm8550_qmp_phy_vreg_l), .regs = pciephy_v5_regs_layout, .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, @@ -4412,6 +4440,38 @@ static const struct qmp_phy_cfg qmp_v6_gen4x4_pciephy_cfg = { .phy_status = PHYSTATUS_4_20, }; +static const struct qmp_phy_cfg glymur_qmp_gen5x4_pciephy_cfg = { + .lanes = 4, + + .offsets = &qmp_pcie_offsets_v8_50, + + .reset_list = sdm845_pciephy_reset_l, + .num_resets = ARRAY_SIZE(sdm845_pciephy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + + .regs = pciephy_v8_50_regs_layout, + + .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, + .phy_status = PHYSTATUS_4_20, +}; + +static const struct qmp_phy_cfg glymur_qmp_gen4x2_pciephy_cfg = { + .lanes = 2, + + .offsets = &qmp_pcie_offsets_v8, + + .reset_list = sdm845_pciephy_reset_l, + .num_resets = ARRAY_SIZE(sdm845_pciephy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + + .regs = pciephy_v8_regs_layout, + + .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, + .phy_status = PHYSTATUS_4_20, +}; + static void qmp_pcie_init_port_b(struct qmp_pcie *qmp, const struct qmp_phy_cfg_tbls *tbls) { const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -5163,6 +5223,12 @@ static int qmp_pcie_probe(struct platform_device *pdev) static const struct of_device_id qmp_pcie_of_match_table[] = { { + .compatible = "qcom,glymur-qmp-gen4x2-pcie-phy", + .data = &glymur_qmp_gen4x2_pciephy_cfg, + }, { + .compatible = "qcom,glymur-qmp-gen5x4-pcie-phy", + .data = &glymur_qmp_gen5x4_pciephy_cfg, + }, { .compatible = "qcom,ipq6018-qmp-pcie-phy", .data = &ipq6018_pciephy_cfg, }, { diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h new file mode 100644 index 0000000000000..325c127e8eb7a --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef QCOM_PHY_QMP_PCS_V8_50_H_ +#define QCOM_PHY_QMP_PCS_V8_50_H_ + +#define QPHY_V8_50_PCS_STATUS1 0x010 +#define QPHY_V8_50_PCS_START_CONTROL 0x05c +#define QPHY_V8_50_PCS_POWER_DOWN_CONTROL 0x64 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h new file mode 100644 index 0000000000000..3ea1884f35dd5 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_QSERDES_COM_V2_H_ +#define QCOM_PHY_QMP_QSERDES_COM_V2_H_ + +/* Only for QMP V2 PHY - QSERDES COM registers */ +#define QSERDES_V2_COM_ATB_SEL1 0x000 +#define QSERDES_V2_COM_ATB_SEL2 0x004 +#define QSERDES_V2_COM_FREQ_UPDATE 0x008 +#define QSERDES_V2_COM_BG_TIMER 0x00c +#define QSERDES_V2_COM_SSC_EN_CENTER 0x010 +#define QSERDES_V2_COM_SSC_ADJ_PER1 0x014 +#define QSERDES_V2_COM_SSC_ADJ_PER2 0x018 +#define QSERDES_V2_COM_SSC_PER1 0x01c +#define QSERDES_V2_COM_SSC_PER2 0x020 +#define QSERDES_V2_COM_SSC_STEP_SIZE1 0x024 +#define QSERDES_V2_COM_SSC_STEP_SIZE2 0x028 +#define QSERDES_V2_COM_POST_DIV 0x02c +#define QSERDES_V2_COM_POST_DIV_MUX 0x030 +#define QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN 0x034 +#define QSERDES_V2_COM_CLK_ENABLE1 0x038 +#define QSERDES_V2_COM_SYS_CLK_CTRL 0x03c +#define QSERDES_V2_COM_SYSCLK_BUF_ENABLE 0x040 +#define QSERDES_V2_COM_PLL_EN 0x044 +#define QSERDES_V2_COM_PLL_IVCO 0x048 +#define QSERDES_V2_COM_LOCK_CMP1_MODE0 0x04c +#define QSERDES_V2_COM_LOCK_CMP2_MODE0 0x050 +#define QSERDES_V2_COM_LOCK_CMP3_MODE0 0x054 +#define QSERDES_V2_COM_LOCK_CMP1_MODE1 0x058 +#define QSERDES_V2_COM_LOCK_CMP2_MODE1 0x05c +#define QSERDES_V2_COM_LOCK_CMP3_MODE1 0x060 +#define QSERDES_V2_COM_EP_CLOCK_DETECT_CTR 0x068 +#define QSERDES_V2_COM_SYSCLK_DET_COMP_STATUS 0x06c +#define QSERDES_V2_COM_CLK_EP_DIV 0x074 +#define QSERDES_V2_COM_CP_CTRL_MODE0 0x078 +#define QSERDES_V2_COM_CP_CTRL_MODE1 0x07c +#define QSERDES_V2_COM_PLL_RCTRL_MODE0 0x084 +#define QSERDES_V2_COM_PLL_RCTRL_MODE1 0x088 +#define QSERDES_V2_COM_PLL_CCTRL_MODE0 0x090 +#define QSERDES_V2_COM_PLL_CCTRL_MODE1 0x094 +#define QSERDES_V2_COM_PLL_CNTRL 0x09c +#define QSERDES_V2_COM_BIAS_EN_CTRL_BY_PSM 0x0a8 +#define QSERDES_V2_COM_SYSCLK_EN_SEL 0x0ac +#define QSERDES_V2_COM_CML_SYSCLK_SEL 0x0b0 +#define QSERDES_V2_COM_RESETSM_CNTRL 0x0b4 +#define QSERDES_V2_COM_RESETSM_CNTRL2 0x0b8 +#define QSERDES_V2_COM_LOCK_CMP_EN 0x0c8 +#define QSERDES_V2_COM_LOCK_CMP_CFG 0x0cc +#define QSERDES_V2_COM_DEC_START_MODE0 0x0d0 +#define QSERDES_V2_COM_DEC_START_MODE1 0x0d4 +#define QSERDES_V2_COM_VCOCAL_DEADMAN_CTRL 0x0d8 +#define QSERDES_V2_COM_DIV_FRAC_START1_MODE0 0x0dc +#define QSERDES_V2_COM_DIV_FRAC_START2_MODE0 0x0e0 +#define QSERDES_V2_COM_DIV_FRAC_START3_MODE0 0x0e4 +#define QSERDES_V2_COM_DIV_FRAC_START1_MODE1 0x0e8 +#define QSERDES_V2_COM_DIV_FRAC_START2_MODE1 0x0ec +#define QSERDES_V2_COM_DIV_FRAC_START3_MODE1 0x0f0 +#define QSERDES_V2_COM_VCO_TUNE_MINVAL1 0x0f4 +#define QSERDES_V2_COM_VCO_TUNE_MINVAL2 0x0f8 +#define QSERDES_V2_COM_INTEGLOOP_INITVAL 0x100 +#define QSERDES_V2_COM_INTEGLOOP_EN 0x104 +#define QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0 0x108 +#define QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0 0x10c +#define QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE1 0x110 +#define QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE1 0x114 +#define QSERDES_V2_COM_VCO_TUNE_MAXVAL1 0x118 +#define QSERDES_V2_COM_VCO_TUNE_MAXVAL2 0x11c +#define QSERDES_V2_COM_VCO_TUNE_CTRL 0x124 +#define QSERDES_V2_COM_VCO_TUNE_MAP 0x128 +#define QSERDES_V2_COM_VCO_TUNE1_MODE0 0x12c +#define QSERDES_V2_COM_VCO_TUNE2_MODE0 0x130 +#define QSERDES_V2_COM_VCO_TUNE1_MODE1 0x134 +#define QSERDES_V2_COM_VCO_TUNE2_MODE1 0x138 +#define QSERDES_V2_COM_VCO_TUNE_INITVAL1 0x13c +#define QSERDES_V2_COM_VCO_TUNE_INITVAL2 0x140 +#define QSERDES_V2_COM_VCO_TUNE_TIMER1 0x144 +#define QSERDES_V2_COM_VCO_TUNE_TIMER2 0x148 +#define QSERDES_V2_COM_CMN_STATUS 0x15c +#define QSERDES_V2_COM_RESET_SM_STATUS 0x160 +#define QSERDES_V2_COM_RESTRIM_CODE_STATUS 0x164 +#define QSERDES_V2_COM_PLLCAL_CODE1_STATUS 0x168 +#define QSERDES_V2_COM_PLLCAL_CODE2_STATUS 0x16c +#define QSERDES_V2_COM_CLK_SELECT 0x174 +#define QSERDES_V2_COM_HSCLK_SEL 0x178 +#define QSERDES_V2_COM_INTEGLOOP_BINCODE_STATUS 0x17c +#define QSERDES_V2_COM_PLL_ANALOG 0x180 +#define QSERDES_V2_COM_CORECLK_DIV 0x184 +#define QSERDES_V2_COM_SW_RESET 0x188 +#define QSERDES_V2_COM_CORE_CLK_EN 0x18c +#define QSERDES_V2_COM_C_READY_STATUS 0x190 +#define QSERDES_V2_COM_CMN_CONFIG 0x194 +#define QSERDES_V2_COM_CMN_RATE_OVERRIDE 0x198 +#define QSERDES_V2_COM_SVS_MODE_CLK_SEL 0x19c +#define QSERDES_V2_COM_DEBUG_BUS0 0x1a0 +#define QSERDES_V2_COM_DEBUG_BUS1 0x1a4 +#define QSERDES_V2_COM_DEBUG_BUS2 0x1a8 +#define QSERDES_V2_COM_DEBUG_BUS3 0x1ac +#define QSERDES_V2_COM_DEBUG_BUS_SEL 0x1b0 +#define QSERDES_V2_COM_CMN_MISC1 0x1b4 +#define QSERDES_V2_COM_CMN_MISC2 0x1b8 +#define QSERDES_V2_COM_CORECLK_DIV_MODE1 0x1bc + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h new file mode 100644 index 0000000000000..34919720b7bc4 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_QSERDES_TXRX_V2_H_ +#define QCOM_PHY_QMP_QSERDES_TXRX_V2_H_ + +/* Only for QMP V2 PHY - TX registers */ +#define QSERDES_V2_TX_BIST_MODE_LANENO 0x000 +#define QSERDES_V2_TX_CLKBUF_ENABLE 0x008 +#define QSERDES_V2_TX_TX_EMP_POST1_LVL 0x00c +#define QSERDES_V2_TX_TX_DRV_LVL 0x01c +#define QSERDES_V2_TX_RESET_TSYNC_EN 0x024 +#define QSERDES_V2_TX_PRE_STALL_LDO_BOOST_EN 0x028 +#define QSERDES_V2_TX_TX_BAND 0x02c +#define QSERDES_V2_TX_SLEW_CNTL 0x030 +#define QSERDES_V2_TX_INTERFACE_SELECT 0x034 +#define QSERDES_V2_TX_RES_CODE_LANE_TX 0x03c +#define QSERDES_V2_TX_RES_CODE_LANE_RX 0x040 +#define QSERDES_V2_TX_RES_CODE_LANE_OFFSET_TX 0x044 +#define QSERDES_V2_TX_RES_CODE_LANE_OFFSET_RX 0x048 +#define QSERDES_V2_TX_DEBUG_BUS_SEL 0x058 +#define QSERDES_V2_TX_TRANSCEIVER_BIAS_EN 0x05c +#define QSERDES_V2_TX_HIGHZ_DRVR_EN 0x060 +#define QSERDES_V2_TX_TX_POL_INV 0x064 +#define QSERDES_V2_TX_PARRATE_REC_DETECT_IDLE_EN 0x068 +#define QSERDES_V2_TX_LANE_MODE_1 0x08c +#define QSERDES_V2_TX_LANE_MODE_2 0x090 +#define QSERDES_V2_TX_LANE_MODE_3 0x094 +#define QSERDES_V2_TX_RCV_DETECT_LVL_2 0x0a4 +#define QSERDES_V2_TX_TRAN_DRVR_EMP_EN 0x0c0 +#define QSERDES_V2_TX_TX_INTERFACE_MODE 0x0c4 +#define QSERDES_V2_TX_VMODE_CTRL1 0x0f0 + +/* Only for QMP V2 PHY - RX registers */ +#define QSERDES_V2_RX_UCDR_FO_GAIN 0x008 +#define QSERDES_V2_RX_UCDR_SO_GAIN_HALF 0x00c +#define QSERDES_V2_RX_UCDR_SO_GAIN 0x014 +#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN_HALF 0x024 +#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN_QUARTER 0x028 +#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN 0x02c +#define QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN 0x030 +#define QSERDES_V2_RX_UCDR_SO_SATURATION_AND_ENABLE 0x034 +#define QSERDES_V2_RX_UCDR_FASTLOCK_COUNT_LOW 0x03c +#define QSERDES_V2_RX_UCDR_FASTLOCK_COUNT_HIGH 0x040 +#define QSERDES_V2_RX_UCDR_PI_CONTROLS 0x044 +#define QSERDES_V2_RX_RX_TERM_BW 0x07c +#define QSERDES_V2_RX_VGA_CAL_CNTRL1 0x0bc +#define QSERDES_V2_RX_VGA_CAL_CNTRL2 0x0c0 +#define QSERDES_V2_RX_RX_EQ_GAIN2_LSB 0x0c8 +#define QSERDES_V2_RX_RX_EQ_GAIN2_MSB 0x0cc +#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL1 0x0d0 +#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2 0x0d4 +#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3 0x0d8 +#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4 0x0dc +#define QSERDES_V2_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1 0x0f8 +#define QSERDES_V2_RX_RX_OFFSET_ADAPTOR_CNTRL2 0x0fc +#define QSERDES_V2_RX_SIGDET_ENABLES 0x100 +#define QSERDES_V2_RX_SIGDET_CNTRL 0x104 +#define QSERDES_V2_RX_SIGDET_LVL 0x108 +#define QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL 0x10c +#define QSERDES_V2_RX_RX_BAND 0x110 +#define QSERDES_V2_RX_RX_INTERFACE_MODE 0x11c +#define QSERDES_V2_RX_RX_MODE_00 0x164 +#define QSERDES_V2_RX_RX_MODE_01 0x168 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c index 8a280433a42b1..dda877561f8ca 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c @@ -928,6 +928,7 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = { QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0xc1), + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x68), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4, 0x0e), @@ -937,13 +938,11 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = { }; static const struct qmp_phy_init_tbl sm8650_ufsphy_g4_pcs[] = { - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x13), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x04), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x04), }; static const struct qmp_phy_init_tbl sm8650_ufsphy_g5_pcs[] = { - QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x05), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x05), QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HS_G5_SYNC_LENGTH_CAPABILITY, 0x4d), diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c index 5e7fcb26744a4..105cc0f4b8ddd 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c @@ -22,13 +22,19 @@ #include #include #include +#include #include "phy-qcom-qmp-common.h" #include "phy-qcom-qmp.h" #include "phy-qcom-qmp-pcs-misc-v3.h" +#include "phy-qcom-qmp-dp-phy.h" +#include "phy-qcom-qmp-dp-phy-v2.h" + #define PHY_INIT_COMPLETE_TIMEOUT 10000 +#define SW_PORTSELECT_VAL BIT(0) +#define SW_PORTSELECT_MUX BIT(1) /* set of registers with offsets different per-PHY */ enum qphy_reg_layout { @@ -284,6 +290,83 @@ static const struct qmp_phy_init_tbl qcm2290_usb3_pcs_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V3_PCS_RX_SIGDET_LVL, 0x88), }; +static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x01), + QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x37), + QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x06), + QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_COM_CLK_ENABLE1, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_COM_BG_CTRL, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_BUF_ENABLE, 0x06), + QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30), + QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28), + QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16), + QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b), + QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x40), + QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x08), + QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0x05), + QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x02), +}; + +static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl_rbr[] = { + QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x2c), + QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x69), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x80), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x07), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0xbf), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x21), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00), +}; + +static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl_hbr[] = { + QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x24), + QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x69), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x80), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x07), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x38), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00), +}; + +static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl_hbr2[] = { + QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x8c), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x70), + QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00), +}; + +static const struct qmp_phy_init_tbl qmp_v2_dp_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V2_TX_TRANSCEIVER_BIAS_EN, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_VMODE_CTRL1, 0x40), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_PRE_STALL_LDO_BOOST_EN, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_INTERFACE_SELECT, 0x3d), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_CLKBUF_ENABLE, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_RESET_TSYNC_EN, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_TRAN_DRVR_EMP_EN, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_PARRATE_REC_DETECT_IDLE_EN, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_INTERFACE_MODE, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_EMP_POST1_LVL, 0x2b), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_DRV_LVL, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_BAND, 0x4), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_RES_CODE_LANE_OFFSET_TX, 0x12), + QMP_PHY_INIT_CFG(QSERDES_V2_TX_RES_CODE_LANE_OFFSET_RX, 0x12), +}; + struct qmp_usbc_offsets { u16 serdes; u16 pcs; @@ -293,13 +376,18 @@ struct qmp_usbc_offsets { /* for PHYs with >= 2 lanes */ u16 tx2; u16 rx2; + + u16 dp_serdes; + u16 dp_txa; + u16 dp_txb; + u16 dp_dp_phy; }; -/* struct qmp_phy_cfg - per-PHY initialization config */ +struct qmp_usbc; struct qmp_phy_cfg { const struct qmp_usbc_offsets *offsets; - /* Init sequence for PHY blocks - serdes, tx, rx, pcs */ + /* Init sequence for USB PHY blocks - serdes, tx, rx, pcs */ const struct qmp_phy_init_tbl *serdes_tbl; int serdes_tbl_num; const struct qmp_phy_init_tbl *tx_tbl; @@ -309,8 +397,30 @@ struct qmp_phy_cfg { const struct qmp_phy_init_tbl *pcs_tbl; int pcs_tbl_num; - /* regulators to be requested */ - const char * const *vreg_list; + /* Init sequence for DP PHY blocks - serdes, tx, rbr, hbr, hbr2 */ + const struct qmp_phy_init_tbl *dp_serdes_tbl; + int dp_serdes_tbl_num; + const struct qmp_phy_init_tbl *dp_tx_tbl; + int dp_tx_tbl_num; + const struct qmp_phy_init_tbl *serdes_tbl_rbr; + int serdes_tbl_rbr_num; + const struct qmp_phy_init_tbl *serdes_tbl_hbr; + int serdes_tbl_hbr_num; + const struct qmp_phy_init_tbl *serdes_tbl_hbr2; + int serdes_tbl_hbr2_num; + + const u8 (*swing_tbl)[4][4]; + const u8 (*pre_emphasis_tbl)[4][4]; + + /* DP PHY callbacks */ + void (*dp_aux_init)(struct qmp_usbc *qmp); + void (*configure_dp_tx)(struct qmp_usbc *qmp); + int (*configure_dp_phy)(struct qmp_usbc *qmp); + int (*calibrate_dp_phy)(struct qmp_usbc *qmp); + + const char * const *reset_list; + int num_resets; + const struct regulator_bulk_data *vreg_list; int num_vregs; /* array of registers with different offsets */ @@ -329,25 +439,36 @@ struct qmp_usbc { void __iomem *rx; void __iomem *tx2; void __iomem *rx2; - - struct regmap *tcsr_map; - u32 vls_clamp_reg; + void __iomem *dp_dp_phy; + void __iomem *dp_tx; + void __iomem *dp_tx2; + void __iomem *dp_serdes; struct clk *pipe_clk; + struct clk_fixed_rate pipe_clk_fixed; + + struct clk_hw dp_link_hw; + struct clk_hw dp_pixel_hw; struct clk_bulk_data *clks; int num_clks; int num_resets; struct reset_control_bulk_data *resets; struct regulator_bulk_data *vregs; + struct regmap *tcsr_map; + u32 vls_clamp_reg; + u32 dp_phy_mode_reg; + struct mutex phy_mutex; + struct phy *usb_phy; enum phy_mode mode; unsigned int usb_init_count; - struct phy *phy; - - struct clk_fixed_rate pipe_clk_fixed; + struct phy *dp_phy; + unsigned int dp_aux_cfg; + struct phy_configure_opts_dp dp_opts; + unsigned int dp_init_count; struct typec_switch_dev *sw; enum typec_orientation orientation; @@ -391,9 +512,23 @@ static const char * const usb3phy_reset_l[] = { "phy_phy", "phy", }; -/* list of regulators */ -static const char * const qmp_phy_vreg_l[] = { - "vdda-phy", "vdda-pll", +static const char * const usb3dpphy_reset_l[] = { + "phy_phy", "dp_phy", +}; + +static const struct regulator_bulk_data qmp_phy_msm8998_vreg_l[] = { + { .supply = "vdda-phy", .init_load_uA = 68600 }, + { .supply = "vdda-pll", .init_load_uA = 14200 }, +}; + +static const struct regulator_bulk_data qmp_phy_sm2290_vreg_l[] = { + { .supply = "vdda-phy", .init_load_uA = 66100 }, + { .supply = "vdda-pll", .init_load_uA = 13300 }, +}; + +static const struct regulator_bulk_data qmp_phy_qcs615_vreg_l[] = { + { .supply = "vdda-phy", .init_load_uA = 50000 }, + { .supply = "vdda-pll", .init_load_uA = 20000 }, }; static const struct qmp_usbc_offsets qmp_usbc_offsets_v3_qcm2290 = { @@ -406,6 +541,34 @@ static const struct qmp_usbc_offsets qmp_usbc_offsets_v3_qcm2290 = { .rx2 = 0x800, }; +static const struct qmp_usbc_offsets qmp_usbc_usb3dp_offsets_qcs615 = { + .serdes = 0x0, + .pcs = 0xc00, + .pcs_misc = 0xa00, + .tx = 0x200, + .rx = 0x400, + .tx2 = 0x600, + .rx2 = 0x800, + .dp_serdes = 0x1c00, + .dp_txa = 0x1400, + .dp_txb = 0x1800, + .dp_dp_phy = 0x1000, +}; + +static const u8 qmp_v2_dp_pre_emphasis_hbr2_rbr[4][4] = { + {0x00, 0x0b, 0x12, 0xff}, + {0x00, 0x0a, 0x12, 0xff}, + {0x00, 0x0c, 0xff, 0xff}, + {0xff, 0xff, 0xff, 0xff} +}; + +static const u8 qmp_v2_dp_voltage_swing_hbr2_rbr[4][4] = { + {0x07, 0x0f, 0x14, 0xff}, + {0x11, 0x1d, 0x1f, 0xff}, + {0x18, 0x1f, 0xff, 0xff}, + {0xff, 0xff, 0xff, 0xff} +}; + static const struct qmp_phy_cfg msm8998_usb3phy_cfg = { .offsets = &qmp_usbc_offsets_v3_qcm2290, @@ -417,8 +580,10 @@ static const struct qmp_phy_cfg msm8998_usb3phy_cfg = { .rx_tbl_num = ARRAY_SIZE(msm8998_usb3_rx_tbl), .pcs_tbl = msm8998_usb3_pcs_tbl, .pcs_tbl_num = ARRAY_SIZE(msm8998_usb3_pcs_tbl), - .vreg_list = qmp_phy_vreg_l, - .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .reset_list = usb3phy_reset_l, + .num_resets = ARRAY_SIZE(usb3phy_reset_l), + .vreg_list = qmp_phy_msm8998_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_msm8998_vreg_l), .regs = qmp_v3_usb3phy_regs_layout, }; @@ -433,8 +598,10 @@ static const struct qmp_phy_cfg qcm2290_usb3phy_cfg = { .rx_tbl_num = ARRAY_SIZE(qcm2290_usb3_rx_tbl), .pcs_tbl = qcm2290_usb3_pcs_tbl, .pcs_tbl_num = ARRAY_SIZE(qcm2290_usb3_pcs_tbl), - .vreg_list = qmp_phy_vreg_l, - .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .reset_list = usb3phy_reset_l, + .num_resets = ARRAY_SIZE(usb3phy_reset_l), + .vreg_list = qmp_phy_sm2290_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_sm2290_vreg_l), .regs = qmp_v3_usb3phy_regs_layout_qcm2290, }; @@ -449,17 +616,86 @@ static const struct qmp_phy_cfg sdm660_usb3phy_cfg = { .rx_tbl_num = ARRAY_SIZE(sdm660_usb3_rx_tbl), .pcs_tbl = qcm2290_usb3_pcs_tbl, .pcs_tbl_num = ARRAY_SIZE(qcm2290_usb3_pcs_tbl), - .vreg_list = qmp_phy_vreg_l, - .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .reset_list = usb3phy_reset_l, + .num_resets = ARRAY_SIZE(usb3phy_reset_l), + .vreg_list = qmp_phy_msm8998_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_msm8998_vreg_l), .regs = qmp_v3_usb3phy_regs_layout_qcm2290, }; -static int qmp_usbc_init(struct phy *phy) +static const struct qmp_phy_cfg qcs615_usb3phy_cfg = { + .offsets = &qmp_usbc_offsets_v3_qcm2290, + + .serdes_tbl = qcm2290_usb3_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(qcm2290_usb3_serdes_tbl), + .tx_tbl = qcm2290_usb3_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(qcm2290_usb3_tx_tbl), + .rx_tbl = qcm2290_usb3_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(qcm2290_usb3_rx_tbl), + .pcs_tbl = qcm2290_usb3_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(qcm2290_usb3_pcs_tbl), + .reset_list = usb3phy_reset_l, + .num_resets = ARRAY_SIZE(usb3phy_reset_l), + .vreg_list = qmp_phy_qcs615_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_qcs615_vreg_l), + .regs = qmp_v3_usb3phy_regs_layout_qcm2290, +}; + +static void qmp_v2_dp_aux_init(struct qmp_usbc *qmp); +static void qmp_v2_configure_dp_tx(struct qmp_usbc *qmp); +static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp); +static int qmp_v2_calibrate_dp_phy(struct qmp_usbc *qmp); + +static const struct qmp_phy_cfg qcs615_usb3dp_phy_cfg = { + .offsets = &qmp_usbc_usb3dp_offsets_qcs615, + + .serdes_tbl = qcm2290_usb3_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(qcm2290_usb3_serdes_tbl), + .tx_tbl = qcm2290_usb3_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(qcm2290_usb3_tx_tbl), + .rx_tbl = qcm2290_usb3_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(qcm2290_usb3_rx_tbl), + .pcs_tbl = qcm2290_usb3_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(qcm2290_usb3_pcs_tbl), + + .regs = qmp_v3_usb3phy_regs_layout_qcm2290, + + .dp_serdes_tbl = qmp_v2_dp_serdes_tbl, + .dp_serdes_tbl_num = ARRAY_SIZE(qmp_v2_dp_serdes_tbl), + .dp_tx_tbl = qmp_v2_dp_tx_tbl, + .dp_tx_tbl_num = ARRAY_SIZE(qmp_v2_dp_tx_tbl), + + .serdes_tbl_rbr = qmp_v2_dp_serdes_tbl_rbr, + .serdes_tbl_rbr_num = ARRAY_SIZE(qmp_v2_dp_serdes_tbl_rbr), + .serdes_tbl_hbr = qmp_v2_dp_serdes_tbl_hbr, + .serdes_tbl_hbr_num = ARRAY_SIZE(qmp_v2_dp_serdes_tbl_hbr), + .serdes_tbl_hbr2 = qmp_v2_dp_serdes_tbl_hbr2, + .serdes_tbl_hbr2_num = ARRAY_SIZE(qmp_v2_dp_serdes_tbl_hbr2), + + .swing_tbl = &qmp_v2_dp_voltage_swing_hbr2_rbr, + .pre_emphasis_tbl = &qmp_v2_dp_pre_emphasis_hbr2_rbr, + + .dp_aux_init = qmp_v2_dp_aux_init, + .configure_dp_tx = qmp_v2_configure_dp_tx, + .configure_dp_phy = qmp_v2_configure_dp_phy, + .calibrate_dp_phy = qmp_v2_calibrate_dp_phy, + + .reset_list = usb3dpphy_reset_l, + .num_resets = ARRAY_SIZE(usb3dpphy_reset_l), + .vreg_list = qmp_phy_qcs615_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_qcs615_vreg_l), +}; + +static void qmp_usbc_set_phy_mode(struct qmp_usbc *qmp, bool is_dp) +{ + if (qmp->tcsr_map && qmp->dp_phy_mode_reg) + regmap_write(qmp->tcsr_map, qmp->dp_phy_mode_reg, is_dp); +} + +static int qmp_usbc_com_init(struct phy *phy) { struct qmp_usbc *qmp = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qmp->cfg; - void __iomem *pcs = qmp->pcs; - u32 val = 0; int ret; ret = regulator_bulk_enable(cfg->num_vregs, qmp->vregs); @@ -484,16 +720,6 @@ static int qmp_usbc_init(struct phy *phy) if (ret) goto err_assert_reset; - qphy_setbits(pcs, cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL], SW_PWRDN); - -#define SW_PORTSELECT_VAL BIT(0) -#define SW_PORTSELECT_MUX BIT(1) - /* Use software based port select and switch on typec orientation */ - val = SW_PORTSELECT_MUX; - if (qmp->orientation == TYPEC_ORIENTATION_REVERSE) - val |= SW_PORTSELECT_VAL; - writel(val, qmp->pcs_misc); - return 0; err_assert_reset: @@ -504,7 +730,7 @@ static int qmp_usbc_init(struct phy *phy) return ret; } -static int qmp_usbc_exit(struct phy *phy) +static int qmp_usbc_com_exit(struct phy *phy) { struct qmp_usbc *qmp = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -518,7 +744,254 @@ static int qmp_usbc_exit(struct phy *phy) return 0; } -static int qmp_usbc_power_on(struct phy *phy) +static void qmp_v2_dp_aux_init(struct qmp_usbc *qmp) +{ + writel(DP_PHY_PD_CTL_AUX_PWRDN | + DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN | + DP_PHY_PD_CTL_PLL_PWRDN, + qmp->dp_dp_phy + QSERDES_DP_PHY_PD_CTL); + + writel(DP_PHY_PD_CTL_PWRDN | DP_PHY_PD_CTL_AUX_PWRDN | + DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN | + DP_PHY_PD_CTL_PLL_PWRDN, + qmp->dp_dp_phy + QSERDES_DP_PHY_PD_CTL); + + writel(0x00, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG0); + writel(0x13, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG1); + writel(0x00, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG2); + writel(0x00, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG3); + writel(0x0a, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG4); + writel(0x26, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG5); + writel(0x0a, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG6); + writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG7); + writel(0xbb, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG8); + writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG9); + qmp->dp_aux_cfg = 0; + + writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | + PHY_AUX_SYNC_ERR_MASK | PHY_AUX_ALIGN_ERR_MASK | + PHY_AUX_REQ_ERR_MASK, + qmp->dp_dp_phy + QSERDES_V2_DP_PHY_AUX_INTERRUPT_MASK); +} + +static int qmp_v2_configure_dp_swing(struct qmp_usbc *qmp) +{ + const struct qmp_phy_cfg *cfg = qmp->cfg; + const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts; + void __iomem *tx = qmp->dp_tx; + void __iomem *tx2 = qmp->dp_tx2; + unsigned int v_level = 0, p_level = 0; + u8 voltage_swing_cfg, pre_emphasis_cfg; + int i; + + if (dp_opts->lanes > 4) { + dev_err(qmp->dev, "Invalid lane_num(%d)\n", dp_opts->lanes); + return -EINVAL; + } + + for (i = 0; i < dp_opts->lanes; i++) { + v_level = max(v_level, dp_opts->voltage[i]); + p_level = max(p_level, dp_opts->pre[i]); + } + + if (v_level > 4 || p_level > 4) { + dev_err(qmp->dev, "Invalid v(%d) | p(%d) level)\n", + v_level, p_level); + return -EINVAL; + } + + voltage_swing_cfg = (*cfg->swing_tbl)[v_level][p_level]; + pre_emphasis_cfg = (*cfg->pre_emphasis_tbl)[v_level][p_level]; + + voltage_swing_cfg |= DP_PHY_TXn_TX_DRV_LVL_MUX_EN; + pre_emphasis_cfg |= DP_PHY_TXn_TX_EMP_POST1_LVL_MUX_EN; + + if (voltage_swing_cfg == 0xff && pre_emphasis_cfg == 0xff) + return -EINVAL; + + writel(voltage_swing_cfg, tx + QSERDES_V2_TX_TX_DRV_LVL); + writel(pre_emphasis_cfg, tx + QSERDES_V2_TX_TX_EMP_POST1_LVL); + writel(voltage_swing_cfg, tx2 + QSERDES_V2_TX_TX_DRV_LVL); + writel(pre_emphasis_cfg, tx2 + QSERDES_V2_TX_TX_EMP_POST1_LVL); + + return 0; +} + +static void qmp_usbc_configure_dp_mode(struct qmp_usbc *qmp) +{ + bool reverse = (qmp->orientation == TYPEC_ORIENTATION_REVERSE); + u32 val; + + val = DP_PHY_PD_CTL_PWRDN | DP_PHY_PD_CTL_AUX_PWRDN | + DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN; + + writel(val, qmp->dp_dp_phy + QSERDES_DP_PHY_PD_CTL); + + if (reverse) + writel(0xc9, qmp->dp_dp_phy + QSERDES_DP_PHY_MODE); + else + writel(0xd9, qmp->dp_dp_phy + QSERDES_DP_PHY_MODE); +} + +static int qmp_usbc_configure_dp_clocks(struct qmp_usbc *qmp) +{ + const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts; + u32 phy_vco_div; + unsigned long pixel_freq; + + switch (dp_opts->link_rate) { + case 1620: + phy_vco_div = 0x1; + pixel_freq = 1620000000UL / 2; + break; + case 2700: + phy_vco_div = 0x1; + pixel_freq = 2700000000UL / 2; + break; + case 5400: + phy_vco_div = 0x2; + pixel_freq = 5400000000UL / 4; + break; + default: + dev_err(qmp->dev, "link rate:%d not supported\n", dp_opts->link_rate); + return -EINVAL; + } + writel(phy_vco_div, qmp->dp_dp_phy + QSERDES_V2_DP_PHY_VCO_DIV); + + clk_set_rate(qmp->dp_link_hw.clk, dp_opts->link_rate * 100000); + clk_set_rate(qmp->dp_pixel_hw.clk, pixel_freq); + + return 0; +} + +static void qmp_v2_configure_dp_tx(struct qmp_usbc *qmp) +{ + const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts; + void __iomem *tx = qmp->dp_tx; + void __iomem *tx2 = qmp->dp_tx2; + + /* program default setting first */ + writel(0x2a, tx + QSERDES_V2_TX_TX_DRV_LVL); + writel(0x20, tx + QSERDES_V2_TX_TX_EMP_POST1_LVL); + writel(0x2a, tx2 + QSERDES_V2_TX_TX_DRV_LVL); + writel(0x20, tx2 + QSERDES_V2_TX_TX_EMP_POST1_LVL); + + if (dp_opts->link_rate >= 2700) { + writel(0xc4, tx + QSERDES_V2_TX_LANE_MODE_1); + writel(0xc4, tx2 + QSERDES_V2_TX_LANE_MODE_1); + } else { + writel(0xc6, tx + QSERDES_V2_TX_LANE_MODE_1); + writel(0xc6, tx2 + QSERDES_V2_TX_LANE_MODE_1); + } + + qmp_v2_configure_dp_swing(qmp); +} + +static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp) +{ + u32 status; + int ret; + + qmp_usbc_configure_dp_mode(qmp); + + writel(0x05, qmp->dp_dp_phy + QSERDES_V2_DP_PHY_TX0_TX1_LANE_CTL); + writel(0x05, qmp->dp_dp_phy + QSERDES_V2_DP_PHY_TX2_TX3_LANE_CTL); + + ret = qmp_usbc_configure_dp_clocks(qmp); + if (ret) + return ret; + + writel(0x01, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + writel(0x05, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + writel(0x01, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + writel(0x09, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + + writel(0x20, qmp->dp_serdes + QSERDES_COM_RESETSM_CNTRL); + + if (readl_poll_timeout(qmp->dp_serdes + QSERDES_COM_C_READY_STATUS, + status, + ((status & BIT(0)) > 0), + 500, + 10000)) { + dev_err(qmp->dev, "C_READY not ready\n"); + return -ETIMEDOUT; + } + + if (readl_poll_timeout(qmp->dp_serdes + QSERDES_COM_CMN_STATUS, + status, + ((status & BIT(0)) > 0), + 500, + 10000)){ + dev_err(qmp->dev, "FREQ_DONE not ready\n"); + return -ETIMEDOUT; + } + + if (readl_poll_timeout(qmp->dp_serdes + QSERDES_COM_CMN_STATUS, + status, + ((status & BIT(1)) > 0), + 500, + 10000)){ + dev_err(qmp->dev, "PLL_LOCKED not ready\n"); + return -ETIMEDOUT; + } + + writel(0x19, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + + if (readl_poll_timeout(qmp->dp_dp_phy + QSERDES_V2_DP_PHY_STATUS, + status, + ((status & BIT(0)) > 0), + 500, + 10000)){ + dev_err(qmp->dev, "TSYNC_DONE not ready\n"); + return -ETIMEDOUT; + } + + if (readl_poll_timeout(qmp->dp_dp_phy + QSERDES_V2_DP_PHY_STATUS, + status, + ((status & BIT(1)) > 0), + 500, + 10000)){ + dev_err(qmp->dev, "PHY_READY not ready\n"); + return -ETIMEDOUT; + } + + writel(0x3f, qmp->dp_tx + QSERDES_V2_TX_TRANSCEIVER_BIAS_EN); + writel(0x10, qmp->dp_tx + QSERDES_V2_TX_HIGHZ_DRVR_EN); + writel(0x0a, qmp->dp_tx + QSERDES_V2_TX_TX_POL_INV); + writel(0x3f, qmp->dp_tx2 + QSERDES_V2_TX_TRANSCEIVER_BIAS_EN); + writel(0x10, qmp->dp_tx2 + QSERDES_V2_TX_HIGHZ_DRVR_EN); + writel(0x0a, qmp->dp_tx2 + QSERDES_V2_TX_TX_POL_INV); + + writel(0x18, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + writel(0x19, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + + if (readl_poll_timeout(qmp->dp_dp_phy + QSERDES_V2_DP_PHY_STATUS, + status, + ((status & BIT(1)) > 0), + 500, + 10000)){ + dev_err(qmp->dev, "PHY_READY not ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static int qmp_v2_calibrate_dp_phy(struct qmp_usbc *qmp) +{ + static const u8 cfg1_settings[] = {0x13, 0x23, 0x1d}; + u8 val; + + qmp->dp_aux_cfg++; + qmp->dp_aux_cfg %= ARRAY_SIZE(cfg1_settings); + val = cfg1_settings[qmp->dp_aux_cfg]; + + writel(val, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG1); + + return 0; +} + +static int qmp_usbc_usb_power_on(struct phy *phy) { struct qmp_usbc *qmp = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -526,6 +999,14 @@ static int qmp_usbc_power_on(struct phy *phy) unsigned int val; int ret; + qphy_setbits(qmp->pcs, cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL], SW_PWRDN); + + /* Use software based port select and switch on typec orientation */ + val = SW_PORTSELECT_MUX; + if (qmp->orientation == TYPEC_ORIENTATION_REVERSE) + val |= SW_PORTSELECT_VAL; + writel(val, qmp->pcs_misc); + qmp_configure(qmp->dev, qmp->serdes, cfg->serdes_tbl, cfg->serdes_tbl_num); @@ -566,7 +1047,7 @@ static int qmp_usbc_power_on(struct phy *phy) return ret; } -static int qmp_usbc_power_off(struct phy *phy) +static int qmp_usbc_usb_power_off(struct phy *phy) { struct qmp_usbc *qmp = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -587,20 +1068,39 @@ static int qmp_usbc_power_off(struct phy *phy) return 0; } -static int qmp_usbc_enable(struct phy *phy) +static int qmp_usbc_check_phy_status(struct qmp_usbc *qmp, bool is_dp) +{ + if ((is_dp && qmp->usb_init_count) || + (!is_dp && qmp->dp_init_count)) { + dev_err(qmp->dev, + "PHY is configured for %s, can not enable %s\n", + is_dp ? "USB" : "DP", is_dp ? "DP" : "USB"); + return -EBUSY; + } + + return 0; +} + +static int qmp_usbc_usb_enable(struct phy *phy) { struct qmp_usbc *qmp = phy_get_drvdata(phy); int ret; mutex_lock(&qmp->phy_mutex); - ret = qmp_usbc_init(phy); + ret = qmp_usbc_check_phy_status(qmp, false); + if (ret) + goto out_unlock; + + ret = qmp_usbc_com_init(phy); if (ret) goto out_unlock; - ret = qmp_usbc_power_on(phy); + qmp_usbc_set_phy_mode(qmp, false); + + ret = qmp_usbc_usb_power_on(phy); if (ret) { - qmp_usbc_exit(phy); + qmp_usbc_com_exit(phy); goto out_unlock; } @@ -611,19 +1111,19 @@ static int qmp_usbc_enable(struct phy *phy) return ret; } -static int qmp_usbc_disable(struct phy *phy) +static int qmp_usbc_usb_disable(struct phy *phy) { struct qmp_usbc *qmp = phy_get_drvdata(phy); int ret; qmp->usb_init_count--; - ret = qmp_usbc_power_off(phy); + ret = qmp_usbc_usb_power_off(phy); if (ret) return ret; - return qmp_usbc_exit(phy); + return qmp_usbc_com_exit(phy); } -static int qmp_usbc_set_mode(struct phy *phy, enum phy_mode mode, int submode) +static int qmp_usbc_usb_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct qmp_usbc *qmp = phy_get_drvdata(phy); @@ -632,10 +1132,185 @@ static int qmp_usbc_set_mode(struct phy *phy, enum phy_mode mode, int submode) return 0; } -static const struct phy_ops qmp_usbc_phy_ops = { - .init = qmp_usbc_enable, - .exit = qmp_usbc_disable, - .set_mode = qmp_usbc_set_mode, +static int qmp_usbc_dp_enable(struct phy *phy) +{ + struct qmp_usbc *qmp = phy_get_drvdata(phy); + const struct qmp_phy_cfg *cfg = qmp->cfg; + int ret; + + if (qmp->dp_init_count) { + dev_err(qmp->dev, "DP already inited\n"); + return 0; + } + + mutex_lock(&qmp->phy_mutex); + + ret = qmp_usbc_check_phy_status(qmp, true); + if (ret) + goto dp_init_unlock; + + ret = qmp_usbc_com_init(phy); + if (ret) + goto dp_init_unlock; + + qmp_usbc_set_phy_mode(qmp, true); + + cfg->dp_aux_init(qmp); + + qmp->dp_init_count++; + +dp_init_unlock: + mutex_unlock(&qmp->phy_mutex); + return ret; +} + +static int qmp_usbc_dp_disable(struct phy *phy) +{ + struct qmp_usbc *qmp = phy_get_drvdata(phy); + + mutex_lock(&qmp->phy_mutex); + + qmp_usbc_com_exit(phy); + + qmp->dp_init_count--; + + mutex_unlock(&qmp->phy_mutex); + + return 0; +} + +static int qmp_usbc_dp_configure(struct phy *phy, union phy_configure_opts *opts) +{ + const struct phy_configure_opts_dp *dp_opts = &opts->dp; + struct qmp_usbc *qmp = phy_get_drvdata(phy); + const struct qmp_phy_cfg *cfg = qmp->cfg; + + mutex_lock(&qmp->phy_mutex); + + memcpy(&qmp->dp_opts, dp_opts, sizeof(*dp_opts)); + if (qmp->dp_opts.set_voltages) { + cfg->configure_dp_tx(qmp); + qmp->dp_opts.set_voltages = 0; + } + + mutex_unlock(&qmp->phy_mutex); + + return 0; +} + +static int qmp_usbc_dp_calibrate(struct phy *phy) +{ + struct qmp_usbc *qmp = phy_get_drvdata(phy); + const struct qmp_phy_cfg *cfg = qmp->cfg; + int ret = 0; + + mutex_lock(&qmp->phy_mutex); + + if (cfg->calibrate_dp_phy) { + ret = cfg->calibrate_dp_phy(qmp); + if (ret) { + dev_err(qmp->dev, "dp calibrate err(%d)\n", ret); + mutex_unlock(&qmp->phy_mutex); + return ret; + } + } + + mutex_unlock(&qmp->phy_mutex); + return 0; +} + +static int qmp_usbc_dp_serdes_init(struct qmp_usbc *qmp) +{ + const struct qmp_phy_cfg *cfg = qmp->cfg; + void __iomem *serdes = qmp->dp_serdes; + const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts; + + qmp_configure(qmp->dev, serdes, cfg->dp_serdes_tbl, + cfg->dp_serdes_tbl_num); + + switch (dp_opts->link_rate) { + case 1620: + qmp_configure(qmp->dev, serdes, cfg->serdes_tbl_rbr, + cfg->serdes_tbl_rbr_num); + break; + case 2700: + qmp_configure(qmp->dev, serdes, cfg->serdes_tbl_hbr, + cfg->serdes_tbl_hbr_num); + break; + case 5400: + qmp_configure(qmp->dev, serdes, cfg->serdes_tbl_hbr2, + cfg->serdes_tbl_hbr2_num); + break; + default: + /* Other link rates aren't supported */ + return -EINVAL; + } + + return 0; +} + +static int qmp_usbc_dp_power_on(struct phy *phy) +{ + struct qmp_usbc *qmp = phy_get_drvdata(phy); + const struct qmp_phy_cfg *cfg = qmp->cfg; + + void __iomem *tx = qmp->dp_tx; + void __iomem *tx2 = qmp->dp_tx2; + + /* + * FIXME: SW_PORTSELECT handling for DP orientation flip is not implemented. + * Expected: + * - For standard lane mapping: configure SW_PORTSELECT in QSERDES_DP_PHY_CFG_1. + * - For non-standard mapping: pass orientation to dp_ctrl and handle flip + * via logical2physical lane remapping. + */ + + mutex_lock(&qmp->phy_mutex); + + qmp_usbc_dp_serdes_init(qmp); + + qmp_configure_lane(qmp->dev, tx, cfg->dp_tx_tbl, cfg->dp_tx_tbl_num, 1); + qmp_configure_lane(qmp->dev, tx2, cfg->dp_tx_tbl, cfg->dp_tx_tbl_num, 2); + + /* Configure special DP tx tunings */ + cfg->configure_dp_tx(qmp); + + /* Configure link rate, swing, etc. */ + cfg->configure_dp_phy(qmp); + + mutex_unlock(&qmp->phy_mutex); + + return 0; +} + +static int qmp_usbc_dp_power_off(struct phy *phy) +{ + struct qmp_usbc *qmp = phy_get_drvdata(phy); + + mutex_lock(&qmp->phy_mutex); + + /* Assert DP PHY power down */ + writel(DP_PHY_PD_CTL_PSR_PWRDN, qmp->dp_dp_phy + QSERDES_DP_PHY_PD_CTL); + + mutex_unlock(&qmp->phy_mutex); + + return 0; +} + +static const struct phy_ops qmp_usbc_usb_phy_ops = { + .init = qmp_usbc_usb_enable, + .exit = qmp_usbc_usb_disable, + .set_mode = qmp_usbc_usb_set_mode, + .owner = THIS_MODULE, +}; + +static const struct phy_ops qmp_usbc_dp_phy_ops = { + .init = qmp_usbc_dp_enable, + .exit = qmp_usbc_dp_disable, + .configure = qmp_usbc_dp_configure, + .calibrate = qmp_usbc_dp_calibrate, + .power_on = qmp_usbc_dp_power_on, + .power_off = qmp_usbc_dp_power_off, .owner = THIS_MODULE, }; @@ -690,7 +1365,7 @@ static int __maybe_unused qmp_usbc_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->usb_init_count && !qmp->dp_init_count) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -710,7 +1385,7 @@ static int __maybe_unused qmp_usbc_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->usb_init_count && !qmp->dp_init_count) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -736,23 +1411,6 @@ static const struct dev_pm_ops qmp_usbc_pm_ops = { qmp_usbc_runtime_resume, NULL) }; -static int qmp_usbc_vreg_init(struct qmp_usbc *qmp) -{ - const struct qmp_phy_cfg *cfg = qmp->cfg; - struct device *dev = qmp->dev; - int num = cfg->num_vregs; - int i; - - qmp->vregs = devm_kcalloc(dev, num, sizeof(*qmp->vregs), GFP_KERNEL); - if (!qmp->vregs) - return -ENOMEM; - - for (i = 0; i < num; i++) - qmp->vregs[i].supply = cfg->vreg_list[i]; - - return devm_regulator_bulk_get(dev, num, qmp->vregs); -} - static int qmp_usbc_reset_init(struct qmp_usbc *qmp, const char *const *reset_list, int num_resets) @@ -796,9 +1454,23 @@ static int qmp_usbc_clk_init(struct qmp_usbc *qmp) return devm_clk_bulk_get_optional(dev, num, qmp->clks); } -static void phy_clk_release_provider(void *res) +static struct clk_hw *qmp_usbc_clks_hw_get(struct of_phandle_args *clkspec, void *data) { - of_clk_del_provider(res); + struct qmp_usbc *qmp = data; + + if (clkspec->args_count == 0) + return &qmp->pipe_clk_fixed.hw; + + switch (clkspec->args[0]) { + case QMP_USB43DP_USB3_PIPE_CLK: + return &qmp->pipe_clk_fixed.hw; + case QMP_USB43DP_DP_LINK_CLK: + return &qmp->dp_link_hw; + case QMP_USB43DP_DP_VCO_DIV_CLK: + return &qmp->dp_pixel_hw; + } + + return ERR_PTR(-EINVAL); } /* @@ -823,12 +1495,14 @@ static int phy_pipe_clk_register(struct qmp_usbc *qmp, struct device_node *np) { struct clk_fixed_rate *fixed = &qmp->pipe_clk_fixed; struct clk_init_data init = { }; + char name[64]; int ret; ret = of_property_read_string(np, "clock-output-names", &init.name); if (ret) { - dev_err(qmp->dev, "%pOFn: No clock-output-names\n", np); - return ret; + /* Clock name is not mandatory. */ + snprintf(name, sizeof(name), "%s::pipe_clk", dev_name(qmp->dev)); + init.name = name; } init.ops = &clk_fixed_rate_ops; @@ -837,10 +1511,184 @@ static int phy_pipe_clk_register(struct qmp_usbc *qmp, struct device_node *np) fixed->fixed_rate = 125000000; fixed->hw.init = &init; - ret = devm_clk_hw_register(qmp->dev, &fixed->hw); + return devm_clk_hw_register(qmp->dev, &fixed->hw); +} + + +/* + * Display Port PLL driver block diagram for branch clocks + * + * +------------------------------+ + * | DP_VCO_CLK | + * | | + * | +-------------------+ | + * | | (DP PLL/VCO) | | + * | +---------+---------+ | + * | v | + * | +----------+-----------+ | + * | | hsclk_divsel_clk_src | | + * | +----------+-----------+ | + * +------------------------------+ + * | + * +---------<---------v------------>----------+ + * | | + * +--------v----------------+ | + * | dp_phy_pll_link_clk | | + * | link_clk | | + * +--------+----------------+ | + * | | + * | | + * v v + * Input to DISPCC block | + * for link clk, crypto clk | + * and interface clock | + * | + * | + * +--------<------------+-----------------+---<---+ + * | | | + * +----v---------+ +--------v-----+ +--------v------+ + * | vco_divided | | vco_divided | | vco_divided | + * | _clk_src | | _clk_src | | _clk_src | + * | | | | | | + * |divsel_six | | divsel_two | | divsel_four | + * +-------+------+ +-----+--------+ +--------+------+ + * | | | + * v---->----------v-------------<------v + * | + * +----------+-----------------+ + * | dp_phy_pll_vco_div_clk | + * +---------+------------------+ + * | + * v + * Input to DISPCC block + * for DP pixel clock + * + */ +static int qmp_dp_pixel_clk_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + switch (req->rate) { + case 1620000000UL / 2: + case 2700000000UL / 2: + /* 5.4 is same link rate as 2.7GHz, i.e. div 4 */ + return 0; + default: + return -EINVAL; + } +} + +static unsigned long qmp_dp_pixel_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + const struct qmp_usbc *qmp; + const struct phy_configure_opts_dp *dp_opts; + + qmp = container_of(hw, struct qmp_usbc, dp_pixel_hw); + + dp_opts = &qmp->dp_opts; + + switch (dp_opts->link_rate) { + case 1620: + return 1620000000UL / 2; + case 2700: + return 2700000000UL / 2; + case 5400: + return 5400000000UL / 4; + default: + return 0; + } +} + +static const struct clk_ops qmp_dp_pixel_clk_ops = { + .determine_rate = qmp_dp_pixel_clk_determine_rate, + .recalc_rate = qmp_dp_pixel_clk_recalc_rate, +}; + +static int qmp_dp_link_clk_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + switch (req->rate) { + case 162000000: + case 270000000: + case 540000000: + return 0; + default: + return -EINVAL; + } +} + +static unsigned long qmp_dp_link_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + const struct qmp_usbc *qmp; + const struct phy_configure_opts_dp *dp_opts; + + qmp = container_of(hw, struct qmp_usbc, dp_link_hw); + dp_opts = &qmp->dp_opts; + + switch (dp_opts->link_rate) { + case 1620: + case 2700: + case 5400: + return dp_opts->link_rate * 100000; + default: + return 0; + } +} + +static const struct clk_ops qmp_dp_link_clk_ops = { + .determine_rate = qmp_dp_link_clk_determine_rate, + .recalc_rate = qmp_dp_link_clk_recalc_rate, +}; + +static int phy_dp_clks_register(struct qmp_usbc *qmp, struct device_node *np) +{ + struct clk_init_data init = { }; + char name[64]; + int ret; + + snprintf(name, sizeof(name), "%s::link_clk", dev_name(qmp->dev)); + init.ops = &qmp_dp_link_clk_ops; + init.name = name; + qmp->dp_link_hw.init = &init; + ret = devm_clk_hw_register(qmp->dev, &qmp->dp_link_hw); + if (ret < 0) { + dev_err(qmp->dev, "link clk reg fail ret=%d\n", ret); + return ret; + } + + snprintf(name, sizeof(name), "%s::vco_div_clk", dev_name(qmp->dev)); + init.ops = &qmp_dp_pixel_clk_ops; + init.name = name; + qmp->dp_pixel_hw.init = &init; + ret = devm_clk_hw_register(qmp->dev, &qmp->dp_pixel_hw); + if (ret) { + dev_err(qmp->dev, "pxl clk reg fail ret=%d\n", ret); + return ret; + } + + return 0; +} + +static void phy_clk_release_provider(void *res) +{ + of_clk_del_provider(res); +} + +static int qmp_usbc_register_clocks(struct qmp_usbc *qmp, struct device_node *np) +{ + struct clk_fixed_rate *fixed = &qmp->pipe_clk_fixed; + int ret; + + ret = phy_pipe_clk_register(qmp, np); if (ret) return ret; + if (qmp->dp_serdes != 0) { + ret = phy_dp_clks_register(qmp, np); + if (ret) + return ret; + } + + if (np == qmp->dev->of_node) + return devm_of_clk_add_hw_provider(qmp->dev, qmp_usbc_clks_hw_get, qmp); + ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, &fixed->hw); if (ret) return ret; @@ -865,11 +1713,12 @@ static int qmp_usbc_typec_switch_set(struct typec_switch_dev *sw, qmp->orientation = orientation; if (qmp->usb_init_count) { - qmp_usbc_power_off(qmp->phy); - qmp_usbc_exit(qmp->phy); + qmp_usbc_usb_power_off(qmp->usb_phy); + qmp_usbc_com_exit(qmp->usb_phy); - qmp_usbc_init(qmp->phy); - qmp_usbc_power_on(qmp->phy); + qmp_usbc_com_init(qmp->usb_phy); + qmp_usbc_set_phy_mode(qmp, false); + qmp_usbc_usb_power_on(qmp->usb_phy); } mutex_unlock(&qmp->phy_mutex); @@ -985,6 +1834,13 @@ static int qmp_usbc_parse_dt(struct qmp_usbc *qmp) if (IS_ERR(base)) return PTR_ERR(base); + if (offs->dp_serdes != 0) { + qmp->dp_serdes = base + offs->dp_serdes; + qmp->dp_tx = base + offs->dp_txa; + qmp->dp_tx2 = base + offs->dp_txb; + qmp->dp_dp_phy = base + offs->dp_dp_phy; + } + qmp->serdes = base + offs->serdes; qmp->pcs = base + offs->pcs; if (offs->pcs_misc) @@ -1005,23 +1861,23 @@ static int qmp_usbc_parse_dt(struct qmp_usbc *qmp) "failed to get pipe clock\n"); } - ret = qmp_usbc_reset_init(qmp, usb3phy_reset_l, - ARRAY_SIZE(usb3phy_reset_l)); + ret = qmp_usbc_reset_init(qmp, cfg->reset_list, cfg->num_resets); if (ret) return ret; return 0; } -static int qmp_usbc_parse_vls_clamp(struct qmp_usbc *qmp) +static int qmp_usbc_parse_tcsr(struct qmp_usbc *qmp) { struct of_phandle_args tcsr_args; struct device *dev = qmp->dev; - int ret; + int ret, args_count; - /* for backwards compatibility ignore if there is no property */ - ret = of_parse_phandle_with_fixed_args(dev->of_node, "qcom,tcsr-reg", 1, 0, - &tcsr_args); + args_count = of_property_count_u32_elems(dev->of_node, "qcom,tcsr-reg"); + args_count = args_count - 1; + ret = of_parse_phandle_with_fixed_args(dev->of_node, "qcom,tcsr-reg", + args_count, 0, &tcsr_args); if (ret == -ENOENT) return 0; else if (ret < 0) @@ -1034,9 +1890,29 @@ static int qmp_usbc_parse_vls_clamp(struct qmp_usbc *qmp) qmp->vls_clamp_reg = tcsr_args.args[0]; + if (args_count > 1) + qmp->dp_phy_mode_reg = tcsr_args.args[1]; + return 0; } +static struct phy *qmp_usbc_phy_xlate(struct device *dev, const struct of_phandle_args *args) +{ + struct qmp_usbc *qmp = dev_get_drvdata(dev); + + if (args->args_count == 0) + return qmp->usb_phy; + + switch (args->args[0]) { + case QMP_USB43DP_USB3_PHY: + return qmp->usb_phy; + case QMP_USB43DP_DP_PHY: + return qmp->dp_phy ?: ERR_PTR(-ENODEV); + } + + return ERR_PTR(-EINVAL); +} + static int qmp_usbc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1060,7 +1936,8 @@ static int qmp_usbc_probe(struct platform_device *pdev) mutex_init(&qmp->phy_mutex); - ret = qmp_usbc_vreg_init(qmp); + ret = devm_regulator_bulk_get_const(qmp->dev, qmp->cfg->num_vregs, + qmp->cfg->vreg_list, &qmp->vregs); if (ret) return ret; @@ -1068,7 +1945,7 @@ static int qmp_usbc_probe(struct platform_device *pdev) if (ret) return ret; - ret = qmp_usbc_parse_vls_clamp(qmp); + ret = qmp_usbc_parse_tcsr(qmp); if (ret) return ret; @@ -1093,22 +1970,32 @@ static int qmp_usbc_probe(struct platform_device *pdev) */ pm_runtime_forbid(dev); - ret = phy_pipe_clk_register(qmp, np); + ret = qmp_usbc_register_clocks(qmp, np); if (ret) goto err_node_put; - qmp->phy = devm_phy_create(dev, np, &qmp_usbc_phy_ops); - if (IS_ERR(qmp->phy)) { - ret = PTR_ERR(qmp->phy); + qmp->usb_phy = devm_phy_create(dev, np, &qmp_usbc_usb_phy_ops); + if (IS_ERR(qmp->usb_phy)) { + ret = PTR_ERR(qmp->usb_phy); dev_err(dev, "failed to create PHY: %d\n", ret); goto err_node_put; } - phy_set_drvdata(qmp->phy, qmp); + phy_set_drvdata(qmp->usb_phy, qmp); + + if (qmp->dp_serdes != 0) { + qmp->dp_phy = devm_phy_create(dev, np, &qmp_usbc_dp_phy_ops); + if (IS_ERR(qmp->dp_phy)) { + ret = PTR_ERR(qmp->dp_phy); + dev_err(dev, "failed to create PHY: %d\n", ret); + goto err_node_put; + } + phy_set_drvdata(qmp->dp_phy, qmp); + } of_node_put(np); - phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); + phy_provider = devm_of_phy_provider_register(dev, qmp_usbc_phy_xlate); return PTR_ERR_OR_ZERO(phy_provider); @@ -1124,9 +2011,12 @@ static const struct of_device_id qmp_usbc_of_match_table[] = { }, { .compatible = "qcom,qcm2290-qmp-usb3-phy", .data = &qcm2290_usb3phy_cfg, + }, { + .compatible = "qcom,qcs615-qmp-usb3-dp-phy", + .data = &qcs615_usb3dp_phy_cfg, }, { .compatible = "qcom,qcs615-qmp-usb3-phy", - .data = &qcm2290_usb3phy_cfg, + .data = &qcs615_usb3phy_cfg, }, { .compatible = "qcom,sdm660-qmp-usb3-phy", .data = &sdm660_usb3phy_cfg, diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h index f58c82b2dd23e..836a222a2e786 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp.h @@ -9,6 +9,9 @@ #include "phy-qcom-qmp-qserdes-com.h" #include "phy-qcom-qmp-qserdes-txrx.h" +#include "phy-qcom-qmp-qserdes-com-v2.h" +#include "phy-qcom-qmp-qserdes-txrx-v2.h" + #include "phy-qcom-qmp-qserdes-com-v3.h" #include "phy-qcom-qmp-qserdes-txrx-v3.h" @@ -58,6 +61,8 @@ #include "phy-qcom-qmp-pcs-v8.h" +#include "phy-qcom-qmp-pcs-v8_50.h" + /* QPHY_SW_RESET bit */ #define SW_RESET BIT(0) /* QPHY_POWER_DOWN_CONTROL */ diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c index b5514a32ff8ff..eb93015be841f 100644 --- a/drivers/phy/qualcomm/phy-qcom-qusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c @@ -1093,29 +1093,29 @@ static int qusb2_phy_probe(struct platform_device *pdev) or->hsdisc_trim.override = true; } - pm_runtime_set_active(dev); - pm_runtime_enable(dev); + dev_set_drvdata(dev, qphy); + /* - * Prevent runtime pm from being ON by default. Users can enable - * it using power/control in sysfs. + * Enable runtime PM support, but forbid it by default. + * Users can allow it again via the power/control attribute in sysfs. */ + pm_runtime_set_active(dev); pm_runtime_forbid(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; generic_phy = devm_phy_create(dev, NULL, &qusb2_phy_gen_ops); if (IS_ERR(generic_phy)) { ret = PTR_ERR(generic_phy); dev_err(dev, "failed to create phy, %d\n", ret); - pm_runtime_disable(dev); return ret; } qphy->phy = generic_phy; - dev_set_drvdata(dev, qphy); phy_set_drvdata(generic_phy, qphy); phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (IS_ERR(phy_provider)) - pm_runtime_disable(dev); return PTR_ERR_OR_ZERO(phy_provider); } diff --git a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c index 5b1c82459c126..5044f244762f4 100644 --- a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c +++ b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "phy-qcom-qmp-pcs-sgmii.h" #include "phy-qcom-qmp-qserdes-com-v5.h" @@ -26,6 +27,7 @@ #define QSERDES_COM_C_PLL_LOCKED BIT(1) struct qcom_dwmac_sgmii_phy_data { + struct regulator *vdda_0p9; struct regmap *regmap; struct clk *refclk; int speed; @@ -266,9 +268,23 @@ static int qcom_dwmac_sgmii_phy_calibrate(struct phy *phy) static int qcom_dwmac_sgmii_phy_power_on(struct phy *phy) { + int ret; struct qcom_dwmac_sgmii_phy_data *data = phy_get_drvdata(phy); - return clk_prepare_enable(data->refclk); + ret = regulator_enable(data->vdda_0p9); + if (ret) + goto out_ret; + + ret = clk_prepare_enable(data->refclk); + if (ret) + goto out_reg_disable; + + return 0; + +out_reg_disable: + regulator_disable(data->vdda_0p9); +out_ret: + return ret; } static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy) @@ -283,6 +299,8 @@ static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy) clk_disable_unprepare(data->refclk); + regulator_disable(data->vdda_0p9); + return 0; } @@ -343,6 +361,10 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev) if (IS_ERR(data->refclk)) return PTR_ERR(data->refclk); + data->vdda_0p9 = devm_regulator_get(dev, "vdda-0p9"); + if (IS_ERR(data->vdda_0p9)) + return PTR_ERR(data->vdda_0p9); + provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); if (IS_ERR(provider)) return PTR_ERR(provider); diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c index 3f6b480e10922..a38ead7c8055d 100644 --- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c +++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c @@ -134,7 +134,6 @@ struct rcar_gen3_chan { struct extcon_dev *extcon; struct rcar_gen3_phy rphys[NUM_OF_PHYS]; struct regulator *vbus; - struct reset_control *rstc; struct work_struct work; spinlock_t lock; /* protects access to hardware and driver data structure. */ enum usb_dr_mode dr_mode; @@ -771,21 +770,31 @@ static enum usb_dr_mode rcar_gen3_get_dr_mode(struct device_node *np) return candidate; } +static void rcar_gen3_reset_assert(void *data) +{ + reset_control_assert(data); +} + static int rcar_gen3_phy_usb2_init_bus(struct rcar_gen3_chan *channel) { struct device *dev = channel->dev; + struct reset_control *rstc; int ret; u32 val; - channel->rstc = devm_reset_control_array_get_shared(dev); - if (IS_ERR(channel->rstc)) - return PTR_ERR(channel->rstc); + rstc = devm_reset_control_array_get_shared(dev); + if (IS_ERR(rstc)) + return PTR_ERR(rstc); ret = pm_runtime_resume_and_get(dev); if (ret) return ret; - ret = reset_control_deassert(channel->rstc); + ret = reset_control_deassert(rstc); + if (ret) + goto rpm_put; + + ret = devm_add_action_or_reset(dev, rcar_gen3_reset_assert, rstc); if (ret) goto rpm_put; @@ -924,7 +933,6 @@ static void rcar_gen3_phy_usb2_remove(struct platform_device *pdev) if (channel->is_otg_channel) device_remove_file(&pdev->dev, &dev_attr_role); - reset_control_assert(channel->rstc); pm_runtime_disable(&pdev->dev); }; diff --git a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c index b0f23690ec300..8f4c08e599aa2 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c @@ -821,17 +821,20 @@ static void rockchip_chg_detect_work(struct work_struct *work) container_of(work, struct rockchip_usb2phy_port, chg_work.work); struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent); struct regmap *base = get_reg_base(rphy); - bool is_dcd, tmout, vout; + bool is_dcd, tmout, vout, vbus_attach; unsigned long delay; + vbus_attach = property_enabled(rphy->grf, &rport->port_cfg->utmi_bvalid); + dev_dbg(&rport->phy->dev, "chg detection work state = %d\n", rphy->chg_state); switch (rphy->chg_state) { case USB_CHG_STATE_UNDEFINED: - if (!rport->suspended) + if (!rport->suspended && !vbus_attach) rockchip_usb2phy_power_off(rport->phy); /* put the controller in non-driving mode */ - property_enable(base, &rphy->phy_cfg->chg_det.opmode, false); + if (!vbus_attach) + property_enable(base, &rphy->phy_cfg->chg_det.opmode, false); /* Start DCD processing stage 1 */ rockchip_chg_enable_dcd(rphy, true); rphy->chg_state = USB_CHG_STATE_WAIT_FOR_DCD; @@ -894,7 +897,8 @@ static void rockchip_chg_detect_work(struct work_struct *work) fallthrough; case USB_CHG_STATE_DETECTED: /* put the controller in normal mode */ - property_enable(base, &rphy->phy_cfg->chg_det.opmode, true); + if (!vbus_attach) + property_enable(base, &rphy->phy_cfg->chg_det.opmode, true); rockchip_usb2phy_otg_sm_work(&rport->otg_sm_work.work); dev_dbg(&rport->phy->dev, "charger = %s\n", chg_to_string(rphy->chg_type)); @@ -1491,7 +1495,7 @@ static int rockchip_usb2phy_probe(struct platform_device *pdev) rphy); if (ret) { dev_err_probe(rphy->dev, ret, "failed to request usb2phy irq handle\n"); - goto put_child; + return ret; } } diff --git a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c index a3ef19807b9ef..7f8fc8e6d4890 100644 --- a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c +++ b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c @@ -21,6 +21,9 @@ #define REF_CLOCK_100MHz (100 * HZ_PER_MHZ) /* RK3528 COMBO PHY REG */ +#define RK3528_PHYREG5 0x14 +#define RK3528_PHYREG5_GATE_TX_PCK_SEL BIT(3) +#define RK3528_PHYREG5_GATE_TX_PCK_DLY_PLL_OFF BIT(3) #define RK3528_PHYREG6 0x18 #define RK3528_PHYREG6_PLL_KVCO GENMASK(12, 10) #define RK3528_PHYREG6_PLL_KVCO_VALUE 0x2 @@ -103,6 +106,10 @@ #define RK3568_PHYREG18 0x44 #define RK3568_PHYREG18_PLL_LOOP 0x32 +#define RK3568_PHYREG30 0x74 +#define RK3568_PHYREG30_GATE_TX_PCK_SEL BIT(7) +#define RK3568_PHYREG30_GATE_TX_PCK_DLY_PLL_OFF BIT(7) + #define RK3568_PHYREG32 0x7C #define RK3568_PHYREG32_SSC_MASK GENMASK(7, 4) #define RK3568_PHYREG32_SSC_DIR_MASK GENMASK(5, 4) @@ -504,6 +511,10 @@ static int rk3528_combphy_cfg(struct rockchip_combphy_priv *priv) case REF_CLOCK_100MHz: rockchip_combphy_param_write(priv->phy_grf, &cfg->pipe_clk_100m, true); if (priv->type == PHY_TYPE_PCIE) { + /* Gate_tx_pck_sel length select for L1ss support */ + rockchip_combphy_updatel(priv, RK3528_PHYREG5_GATE_TX_PCK_SEL, + RK3528_PHYREG5_GATE_TX_PCK_DLY_PLL_OFF, RK3528_PHYREG5); + /* PLL KVCO tuning fine */ val = FIELD_PREP(RK3528_PHYREG6_PLL_KVCO, RK3528_PHYREG6_PLL_KVCO_VALUE); rockchip_combphy_updatel(priv, RK3528_PHYREG6_PLL_KVCO, val, @@ -657,6 +668,10 @@ static int rk3562_combphy_cfg(struct rockchip_combphy_priv *priv) case REF_CLOCK_100MHz: rockchip_combphy_param_write(priv->phy_grf, &cfg->pipe_clk_100m, true); if (priv->type == PHY_TYPE_PCIE) { + /* Gate_tx_pck_sel length select for L1ss support */ + rockchip_combphy_updatel(priv, RK3568_PHYREG30_GATE_TX_PCK_SEL, + RK3568_PHYREG30_GATE_TX_PCK_DLY_PLL_OFF, + RK3568_PHYREG30); /* PLL KVCO tuning fine */ val = FIELD_PREP(RK3568_PHYREG33_PLL_KVCO_MASK, RK3568_PHYREG33_PLL_KVCO_VALUE); diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c index 01bbf668e05ef..cafa618d70fdc 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c @@ -414,6 +414,8 @@ struct rk_hdptx_phy { static const struct ropll_config ropll_tmds_cfg[] = { { 594000000ULL, 124, 124, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, + { 461101250ULL, 97, 97, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 71, 1, 53, 2, 6, + 35, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 371250000ULL, 155, 155, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 297000000ULL, 124, 124, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, @@ -500,9 +502,7 @@ static const struct reg_sequence rk_hdtpx_common_cmn_init_seq[] = { REG_SEQ0(CMN_REG(0043), 0x00), REG_SEQ0(CMN_REG(0044), 0x46), REG_SEQ0(CMN_REG(0045), 0x24), - REG_SEQ0(CMN_REG(0046), 0xff), REG_SEQ0(CMN_REG(0047), 0x00), - REG_SEQ0(CMN_REG(0048), 0x44), REG_SEQ0(CMN_REG(0049), 0xfa), REG_SEQ0(CMN_REG(004a), 0x08), REG_SEQ0(CMN_REG(004b), 0x00), @@ -575,6 +575,8 @@ static const struct reg_sequence rk_hdtpx_tmds_cmn_init_seq[] = { REG_SEQ0(CMN_REG(0034), 0x00), REG_SEQ0(CMN_REG(003d), 0x40), REG_SEQ0(CMN_REG(0042), 0x78), + REG_SEQ0(CMN_REG(0046), 0xdd), + REG_SEQ0(CMN_REG(0048), 0x11), REG_SEQ0(CMN_REG(004e), 0x34), REG_SEQ0(CMN_REG(005c), 0x25), REG_SEQ0(CMN_REG(005e), 0x4f), @@ -668,13 +670,9 @@ static const struct reg_sequence rk_hdtpx_common_lane_init_seq[] = { static const struct reg_sequence rk_hdtpx_tmds_lane_init_seq[] = { REG_SEQ0(LANE_REG(0312), 0x00), - REG_SEQ0(LANE_REG(031e), 0x00), REG_SEQ0(LANE_REG(0412), 0x00), - REG_SEQ0(LANE_REG(041e), 0x00), REG_SEQ0(LANE_REG(0512), 0x00), - REG_SEQ0(LANE_REG(051e), 0x00), REG_SEQ0(LANE_REG(0612), 0x00), - REG_SEQ0(LANE_REG(061e), 0x08), REG_SEQ0(LANE_REG(0303), 0x2f), REG_SEQ0(LANE_REG(0403), 0x2f), REG_SEQ0(LANE_REG(0503), 0x2f), @@ -687,6 +685,11 @@ static const struct reg_sequence rk_hdtpx_tmds_lane_init_seq[] = { REG_SEQ0(LANE_REG(0406), 0x1c), REG_SEQ0(LANE_REG(0506), 0x1c), REG_SEQ0(LANE_REG(0606), 0x1c), + /* Keep Inter-Pair Skew in the limits */ + REG_SEQ0(LANE_REG(031e), 0x02), + REG_SEQ0(LANE_REG(041e), 0x02), + REG_SEQ0(LANE_REG(051e), 0x02), + REG_SEQ0(LANE_REG(061e), 0x0a), }; static struct tx_drv_ctrl tx_drv_ctrl_rbr[4][4] = { @@ -1037,7 +1040,8 @@ static int rk_hdptx_ropll_tmds_cmn_config(struct rk_hdptx_phy *hdptx) ret = rk_hdptx_post_enable_pll(hdptx); if (!ret) - hdptx->hw_rate = hdptx->hdmi_cfg.tmds_char_rate; + hdptx->hw_rate = DIV_ROUND_CLOSEST_ULL(hdptx->hdmi_cfg.tmds_char_rate * 8, + hdptx->hdmi_cfg.bpc); return ret; } @@ -1895,19 +1899,20 @@ static long rk_hdptx_phy_clk_round_rate(struct clk_hw *hw, unsigned long rate, * hence ensure rk_hdptx_phy_clk_set_rate() won't be invoked with * a different rate argument. */ - return hdptx->hdmi_cfg.tmds_char_rate; + return DIV_ROUND_CLOSEST_ULL(hdptx->hdmi_cfg.tmds_char_rate * 8, hdptx->hdmi_cfg.bpc); } static int rk_hdptx_phy_clk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { struct rk_hdptx_phy *hdptx = to_rk_hdptx_phy(hw); + unsigned long long tmds_rate = DIV_ROUND_CLOSEST_ULL(rate * hdptx->hdmi_cfg.bpc, 8); /* Revert any unlikely TMDS char rate change since round_rate() */ - if (hdptx->hdmi_cfg.tmds_char_rate != rate) { - dev_warn(hdptx->dev, "Reverting unexpected rate change from %lu to %llu\n", - rate, hdptx->hdmi_cfg.tmds_char_rate); - hdptx->hdmi_cfg.tmds_char_rate = rate; + if (hdptx->hdmi_cfg.tmds_char_rate != tmds_rate) { + dev_warn(hdptx->dev, "Reverting unexpected rate change from %llu to %llu\n", + tmds_rate, hdptx->hdmi_cfg.tmds_char_rate); + hdptx->hdmi_cfg.tmds_char_rate = tmds_rate; } /* diff --git a/drivers/phy/samsung/phy-exynos5-usbdrd.c b/drivers/phy/samsung/phy-exynos5-usbdrd.c index a88ba95bdc8f5..1c8bf80119f11 100644 --- a/drivers/phy/samsung/phy-exynos5-usbdrd.c +++ b/drivers/phy/samsung/phy-exynos5-usbdrd.c @@ -1823,7 +1823,7 @@ static int exynos5_usbdrd_orien_sw_set(struct typec_switch_dev *sw, phy_drd->orientation = orientation; } - clk_bulk_disable(phy_drd->drv_data->n_clks, phy_drd->clks); + clk_bulk_disable_unprepare(phy_drd->drv_data->n_clks, phy_drd->clks); return 0; } diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index 27fe92f73f331..b44afbff8616b 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -712,7 +712,7 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) } ret = of_property_read_u32(child, "reg", &index); - if (ret || index > usbphyc->nphys) { + if (ret || index >= usbphyc->nphys) { dev_err(&phy->dev, "invalid reg property: %d\n", ret); if (!ret) ret = -EINVAL; diff --git a/drivers/phy/tegra/xusb-tegra186.c b/drivers/phy/tegra/xusb-tegra186.c index e818f6c3980e6..bec9616c4a2e0 100644 --- a/drivers/phy/tegra/xusb-tegra186.c +++ b/drivers/phy/tegra/xusb-tegra186.c @@ -84,6 +84,7 @@ #define XUSB_PADCTL_USB2_BIAS_PAD_CTL0 0x284 #define BIAS_PAD_PD BIT(11) #define HS_SQUELCH_LEVEL(x) (((x) & 0x7) << 0) +#define HS_DISCON_LEVEL(x) (((x) & 0x7) << 3) #define XUSB_PADCTL_USB2_BIAS_PAD_CTL1 0x288 #define USB2_TRK_START_TIMER(x) (((x) & 0x7f) << 12) @@ -623,6 +624,8 @@ static void tegra186_utmi_bias_pad_power_on(struct tegra_xusb_padctl *padctl) value &= ~BIAS_PAD_PD; value &= ~HS_SQUELCH_LEVEL(~0); value |= HS_SQUELCH_LEVEL(priv->calib.hs_squelch); + value &= ~HS_DISCON_LEVEL(~0); + value |= HS_DISCON_LEVEL(0x7); padctl_writel(padctl, value, XUSB_PADCTL_USB2_BIAS_PAD_CTL0); udelay(1); diff --git a/drivers/phy/ti/phy-da8xx-usb.c b/drivers/phy/ti/phy-da8xx-usb.c index 1d81a1e6ec6b6..62fa6f89c0e61 100644 --- a/drivers/phy/ti/phy-da8xx-usb.c +++ b/drivers/phy/ti/phy-da8xx-usb.c @@ -180,6 +180,7 @@ static int da8xx_usb_phy_probe(struct platform_device *pdev) struct da8xx_usb_phy_platform_data *pdata = dev->platform_data; struct device_node *node = dev->of_node; struct da8xx_usb_phy *d_phy; + int ret; d_phy = devm_kzalloc(dev, sizeof(*d_phy), GFP_KERNEL); if (!d_phy) @@ -233,8 +234,6 @@ static int da8xx_usb_phy_probe(struct platform_device *pdev) return PTR_ERR(d_phy->phy_provider); } } else { - int ret; - ret = phy_create_lookup(d_phy->usb11_phy, "usb-phy", "ohci-da8xx"); if (ret) @@ -249,7 +248,9 @@ static int da8xx_usb_phy_probe(struct platform_device *pdev) PHY_INIT_BITS, PHY_INIT_BITS); pm_runtime_set_active(dev); - devm_pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; /* * Prevent runtime pm from being ON by default. Users can enable * it using power/control in sysfs. diff --git a/drivers/phy/ti/phy-gmii-sel.c b/drivers/phy/ti/phy-gmii-sel.c index 50adabb867cb1..26209a89703a8 100644 --- a/drivers/phy/ti/phy-gmii-sel.c +++ b/drivers/phy/ti/phy-gmii-sel.c @@ -512,7 +512,7 @@ static int phy_gmii_sel_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(base), "failed to get base memory resource\n"); - priv->regmap = regmap_init_mmio(dev, base, &phy_gmii_sel_regmap_cfg); + priv->regmap = devm_regmap_init_mmio(dev, base, &phy_gmii_sel_regmap_cfg); if (IS_ERR(priv->regmap)) return dev_err_probe(dev, PTR_ERR(priv->regmap), "Failed to get syscon\n"); diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c index a8b440c6c46bb..77f18de6fdf62 100644 --- a/drivers/phy/ti/phy-j721e-wiz.c +++ b/drivers/phy/ti/phy-j721e-wiz.c @@ -393,6 +393,7 @@ struct wiz { struct clk *output_clks[WIZ_MAX_OUTPUT_CLOCKS]; struct clk_onecell_data clk_data; const struct wiz_data *data; + int mux_sel_status[WIZ_MUX_NUM_CLOCKS]; }; static int wiz_reset(struct wiz *wiz) @@ -1424,6 +1425,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz) dev_err(dev, "%s: Reading \"reg\" from \"%s\" failed: %d\n", __func__, subnode->name, ret); + of_node_put(serdes); return ret; } of_property_read_u32(subnode, "cdns,num-lanes", &num_lanes); @@ -1438,6 +1440,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz) } } + of_node_put(serdes); return 0; } @@ -1654,11 +1657,25 @@ static void wiz_remove(struct platform_device *pdev) pm_runtime_disable(dev); } +static int wiz_suspend_noirq(struct device *dev) +{ + struct wiz *wiz = dev_get_drvdata(dev); + int i; + + for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) + regmap_field_read(wiz->mux_sel_field[i], &wiz->mux_sel_status[i]); + + return 0; +} + static int wiz_resume_noirq(struct device *dev) { struct device_node *node = dev->of_node; struct wiz *wiz = dev_get_drvdata(dev); - int ret; + int ret, i; + + for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) + regmap_field_write(wiz->mux_sel_field[i], wiz->mux_sel_status[i]); /* Enable supplemental Control override if available */ if (wiz->sup_legacy_clk_override) @@ -1680,7 +1697,7 @@ static int wiz_resume_noirq(struct device *dev) return ret; } -static DEFINE_NOIRQ_DEV_PM_OPS(wiz_pm_ops, NULL, wiz_resume_noirq); +static DEFINE_NOIRQ_DEV_PM_OPS(wiz_pm_ops, wiz_suspend_noirq, wiz_resume_noirq); static struct platform_driver wiz_driver = { .probe = wiz_probe, diff --git a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c index a8f82104a3842..227c37c360e19 100644 --- a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c +++ b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c @@ -574,10 +574,9 @@ static int cs42l43_pin_probe(struct platform_device *pdev) if (child) { ret = devm_add_action_or_reset(&pdev->dev, cs42l43_fwnode_put, child); - if (ret) { - fwnode_handle_put(child); + if (ret) return ret; - } + if (!child->dev) child->dev = priv->dev; fwnode = child; diff --git a/drivers/pinctrl/intel/Kconfig b/drivers/pinctrl/intel/Kconfig index 248c2e558ff32..3ebf072371457 100644 --- a/drivers/pinctrl/intel/Kconfig +++ b/drivers/pinctrl/intel/Kconfig @@ -52,7 +52,10 @@ config PINCTRL_ALDERLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Alder Lake PCH pins and using them as GPIOs. + PCH pins of the following platforms and using them as GPIOs: + - Alder Lake HX, N, and S + - Raptor Lake HX, E, and S + - Twin Lake config PINCTRL_BROXTON tristate "Intel Broxton pinctrl and GPIO driver" @@ -136,15 +139,17 @@ config PINCTRL_METEORLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Meteor Lake pins and using them as GPIOs. + SoC pins of the following platforms and using them as GPIOs: + - Arrow Lake (all variants) + - Meteor Lake (all variants) config PINCTRL_METEORPOINT tristate "Intel Meteor Point pinctrl and GPIO driver" select PINCTRL_INTEL help - Meteor Point is the PCH of Intel Meteor Lake. This pinctrl driver - provides an interface that allows configuring of PCH pins and - using them as GPIOs. + This pinctrl driver provides an interface that allows configuring + PCH pins of the following platforms and using them as GPIOs: + - Arrow Lake HX and S config PINCTRL_SUNRISEPOINT tristate "Intel Sunrisepoint pinctrl and GPIO driver" @@ -159,7 +164,11 @@ config PINCTRL_TIGERLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Tiger Lake PCH pins and using them as GPIOs. + PCH pins of the following platforms and using them as GPIOs: + - Alder Lake H, P, PS, and U + - Raptor Lake H, P, PS, PX, and U + - Rocket Lake S + - Tiger Lake (all variants) source "drivers/pinctrl/intel/Kconfig.tng" endmenu diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c index d68cef4ec52ac..103eccc742a53 100644 --- a/drivers/pinctrl/intel/pinctrl-intel.c +++ b/drivers/pinctrl/intel/pinctrl-intel.c @@ -1606,7 +1606,7 @@ int intel_pinctrl_probe(struct platform_device *pdev, value = readl(regs + REVID); if (value == ~0u) return -ENODEV; - if (((value & REVID_MASK) >> REVID_SHIFT) >= 0x94) { + if (((value & REVID_MASK) >> REVID_SHIFT) >= 0x92) { community->features |= PINCTRL_FEATURE_DEBOUNCE; community->features |= PINCTRL_FEATURE_1K_PD; } diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 9f175c73613f8..2ea0902b4f660 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -539,24 +539,32 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) } } - eint->pin_list = devm_kmalloc(eint->dev, eint->nbase * sizeof(u16 *), GFP_KERNEL); + eint->pin_list = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->pin_list), GFP_KERNEL); if (!eint->pin_list) goto err_pin_list; - eint->wake_mask = devm_kmalloc(eint->dev, eint->nbase * sizeof(u32 *), GFP_KERNEL); + eint->wake_mask = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->wake_mask), GFP_KERNEL); if (!eint->wake_mask) goto err_wake_mask; - eint->cur_mask = devm_kmalloc(eint->dev, eint->nbase * sizeof(u32 *), GFP_KERNEL); + eint->cur_mask = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->cur_mask), GFP_KERNEL); if (!eint->cur_mask) goto err_cur_mask; for (i = 0; i < eint->nbase; i++) { - eint->pin_list[i] = devm_kzalloc(eint->dev, eint->base_pin_num[i] * sizeof(u16), + eint->pin_list[i] = devm_kzalloc(eint->dev, + eint->base_pin_num[i] * sizeof(**eint->pin_list), GFP_KERNEL); port = DIV_ROUND_UP(eint->base_pin_num[i], 32); - eint->wake_mask[i] = devm_kzalloc(eint->dev, port * sizeof(u32), GFP_KERNEL); - eint->cur_mask[i] = devm_kzalloc(eint->dev, port * sizeof(u32), GFP_KERNEL); + eint->wake_mask[i] = devm_kzalloc(eint->dev, + port * sizeof(**eint->wake_mask), + GFP_KERNEL); + eint->cur_mask[i] = devm_kzalloc(eint->dev, + port * sizeof(**eint->cur_mask), + GFP_KERNEL); if (!eint->pin_list[i] || !eint->wake_mask[i] || !eint->cur_mask[i]) goto err_eint; } @@ -592,12 +600,9 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) err_eint: for (i = 0; i < eint->nbase; i++) { - if (eint->cur_mask[i]) - devm_kfree(eint->dev, eint->cur_mask[i]); - if (eint->wake_mask[i]) - devm_kfree(eint->dev, eint->wake_mask[i]); - if (eint->pin_list[i]) - devm_kfree(eint->dev, eint->pin_list[i]); + devm_kfree(eint->dev, eint->cur_mask[i]); + devm_kfree(eint->dev, eint->wake_mask[i]); + devm_kfree(eint->dev, eint->pin_list[i]); } devm_kfree(eint->dev, eint->cur_mask); err_cur_mask: diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8189.c b/drivers/pinctrl/mediatek/pinctrl-mt8189.c index f6a3e584588b0..cd4cdff309a12 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt8189.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt8189.c @@ -1642,7 +1642,7 @@ static const struct mtk_pin_reg_calc mt8189_reg_cals[PINCTRL_PIN_REG_MAX] = { }; static const char * const mt8189_pinctrl_register_base_names[] = { - "base", "lm", "rb0", "rb1", "bm0", "bm1", "bm2", "lt0", "lt1", "rt", + "base", "bm0", "bm1", "bm2", "lm", "lt0", "lt1", "rb0", "rb1", "rt", }; static const struct mtk_eint_hw mt8189_eint_hw = { diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c index d6a46fe0cda89..3f518dce6d23f 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c @@ -1135,9 +1135,12 @@ int mtk_pctrl_init(struct platform_device *pdev, goto chip_error; } - ret = mtk_eint_init(pctl, pdev); - if (ret) - goto chip_error; + /* Only initialize EINT if we have EINT pins */ + if (data->eint_hw.ap_num > 0) { + ret = mtk_eint_init(pctl, pdev); + if (ret) + goto chip_error; + } return 0; diff --git a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c index d9e3a8d5932a8..e2293a872dcb7 100644 --- a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c +++ b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c @@ -24,6 +24,7 @@ #include #include "../core.h" +#include "../pinctrl-utils.h" #include "../pinconf.h" #define gpio_chip_to_bank(chip) \ @@ -672,11 +673,78 @@ static void aml_pin_dbg_show(struct pinctrl_dev *pcdev, struct seq_file *s, seq_printf(s, " %s", dev_name(pcdev->dev)); } +static int aml_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, + struct device_node *np, + struct pinctrl_map **map, + unsigned int *num_maps) +{ + struct device *dev = pctldev->dev; + unsigned long *configs = NULL; + unsigned int num_configs = 0; + struct property *prop; + unsigned int reserved_maps; + int reserve; + int ret; + + prop = of_find_property(np, "pinmux", NULL); + if (!prop) { + dev_info(dev, "Missing pinmux property\n"); + return -ENOENT; + } + + struct device_node *pnode __free(device_node) = of_get_parent(np); + if (!pnode) { + dev_info(dev, "Missing function node\n"); + return -EINVAL; + } + + reserved_maps = 0; + *map = NULL; + *num_maps = 0; + + ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, + &num_configs); + if (ret < 0) { + dev_err(dev, "%pOF: could not parse node property\n", np); + return ret; + } + + reserve = 1; + if (num_configs) + reserve++; + + ret = pinctrl_utils_reserve_map(pctldev, map, &reserved_maps, + num_maps, reserve); + if (ret < 0) + goto exit; + + ret = pinctrl_utils_add_map_mux(pctldev, map, + &reserved_maps, num_maps, np->name, + pnode->name); + if (ret < 0) + goto exit; + + if (num_configs) { + ret = pinctrl_utils_add_map_configs(pctldev, map, &reserved_maps, + num_maps, np->name, configs, + num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); + if (ret < 0) + goto exit; + } + +exit: + kfree(configs); + if (ret) + pinctrl_utils_free_map(pctldev, *map, *num_maps); + + return ret; +} + static const struct pinctrl_ops aml_pctrl_ops = { .get_groups_count = aml_get_groups_count, .get_group_name = aml_get_group_name, .get_group_pins = aml_get_group_pins, - .dt_node_to_map = pinconf_generic_dt_node_to_map_pinmux, + .dt_node_to_map = aml_dt_node_to_map_pinmux, .dt_free_map = pinconf_generic_dt_free_map, .pin_dbg_show = aml_pin_dbg_show, }; @@ -725,8 +793,9 @@ static u32 aml_bank_pins(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return 0; - else - return of_args.args[2]; + + of_node_put(of_args.np); + return of_args.args[2]; } static int aml_bank_number(struct device_node *np) @@ -736,8 +805,9 @@ static int aml_bank_number(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return -EINVAL; - else - return of_args.args[1] >> 8; + + of_node_put(of_args.np); + return of_args.args[1] >> 8; } static unsigned int aml_count_pins(struct device_node *np) @@ -893,7 +963,7 @@ static const struct gpio_chip aml_gpio_template = { .direction_input = aml_gpio_direction_input, .direction_output = aml_gpio_direction_output, .get_direction = aml_gpio_get_direction, - .can_sleep = false, + .can_sleep = true, }; static void init_bank_register_bit(struct aml_pinctrl *info, diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c index 18295b15ecd9d..4507dc8b5563c 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.c +++ b/drivers/pinctrl/meson/pinctrl-meson.c @@ -619,7 +619,7 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc) pc->chip.set = meson_gpio_set; pc->chip.base = -1; pc->chip.ngpio = pc->data->num_pins; - pc->chip.can_sleep = false; + pc->chip.can_sleep = true; ret = gpiochip_add_data(&pc->chip, pc); if (ret) { diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index 5de6ff62c69bd..cad29abe4050a 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -356,75 +356,6 @@ int pinconf_generic_parse_dt_config(struct device_node *np, } EXPORT_SYMBOL_GPL(pinconf_generic_parse_dt_config); -int pinconf_generic_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, - struct device_node *np, - struct pinctrl_map **map, - unsigned int *num_maps) -{ - struct device *dev = pctldev->dev; - struct device_node *pnode; - unsigned long *configs = NULL; - unsigned int num_configs = 0; - struct property *prop; - unsigned int reserved_maps; - int reserve; - int ret; - - prop = of_find_property(np, "pinmux", NULL); - if (!prop) { - dev_info(dev, "Missing pinmux property\n"); - return -ENOENT; - } - - pnode = of_get_parent(np); - if (!pnode) { - dev_info(dev, "Missing function node\n"); - return -EINVAL; - } - - reserved_maps = 0; - *map = NULL; - *num_maps = 0; - - ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, - &num_configs); - if (ret < 0) { - dev_err(dev, "%pOF: could not parse node property\n", np); - return ret; - } - - reserve = 1; - if (num_configs) - reserve++; - - ret = pinctrl_utils_reserve_map(pctldev, map, &reserved_maps, - num_maps, reserve); - if (ret < 0) - goto exit; - - ret = pinctrl_utils_add_map_mux(pctldev, map, - &reserved_maps, num_maps, np->name, - pnode->name); - if (ret < 0) - goto exit; - - if (num_configs) { - ret = pinctrl_utils_add_map_configs(pctldev, map, &reserved_maps, - num_maps, np->name, configs, - num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); - if (ret < 0) - goto exit; - } - -exit: - kfree(configs); - if (ret) - pinctrl_utils_free_map(pctldev, *map, *num_maps); - - return ret; -} -EXPORT_SYMBOL_GPL(pinconf_generic_dt_node_to_map_pinmux); - int pinconf_generic_dt_subnode_to_map(struct pinctrl_dev *pctldev, struct device_node *np, struct pinctrl_map **map, unsigned int *reserved_maps, unsigned int *num_maps, diff --git a/drivers/pinctrl/pinctrl-cy8c95x0.c b/drivers/pinctrl/pinctrl-cy8c95x0.c index a4b04bf6d081f..5c055d344ac9d 100644 --- a/drivers/pinctrl/pinctrl-cy8c95x0.c +++ b/drivers/pinctrl/pinctrl-cy8c95x0.c @@ -627,7 +627,7 @@ static int cy8c95x0_write_regs_mask(struct cy8c95x0_pinctrl *chip, int reg, bitmap_scatter(tmask, mask, chip->map, MAX_LINE); bitmap_scatter(tval, val, chip->map, MAX_LINE); - for_each_set_clump8(offset, bits, tmask, chip->tpin) { + for_each_set_clump8(offset, bits, tmask, chip->nport * BANK_SZ) { unsigned int i = offset / 8; write_val = bitmap_get_value8(tval, offset); @@ -655,7 +655,7 @@ static int cy8c95x0_read_regs_mask(struct cy8c95x0_pinctrl *chip, int reg, bitmap_scatter(tmask, mask, chip->map, MAX_LINE); bitmap_scatter(tval, val, chip->map, MAX_LINE); - for_each_set_clump8(offset, bits, tmask, chip->tpin) { + for_each_set_clump8(offset, bits, tmask, chip->nport * BANK_SZ) { unsigned int i = offset / 8; ret = cy8c95x0_regmap_read_bits(chip, reg, i, bits, &read_val); diff --git a/drivers/pinctrl/pinctrl-equilibrium.c b/drivers/pinctrl/pinctrl-equilibrium.c index 2d04829b29c99..ba1c867b7b891 100644 --- a/drivers/pinctrl/pinctrl-equilibrium.c +++ b/drivers/pinctrl/pinctrl-equilibrium.c @@ -23,7 +23,7 @@ #define PIN_NAME_LEN 10 #define PAD_REG_OFF 0x100 -static void eqbr_gpio_disable_irq(struct irq_data *d) +static void eqbr_irq_mask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -36,7 +36,7 @@ static void eqbr_gpio_disable_irq(struct irq_data *d) gpiochip_disable_irq(gc, offset); } -static void eqbr_gpio_enable_irq(struct irq_data *d) +static void eqbr_irq_unmask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -50,7 +50,7 @@ static void eqbr_gpio_enable_irq(struct irq_data *d) raw_spin_unlock_irqrestore(&gctrl->lock, flags); } -static void eqbr_gpio_ack_irq(struct irq_data *d) +static void eqbr_irq_ack(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -62,10 +62,17 @@ static void eqbr_gpio_ack_irq(struct irq_data *d) raw_spin_unlock_irqrestore(&gctrl->lock, flags); } -static void eqbr_gpio_mask_ack_irq(struct irq_data *d) +static void eqbr_irq_mask_ack(struct irq_data *d) { - eqbr_gpio_disable_irq(d); - eqbr_gpio_ack_irq(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); + unsigned int offset = irqd_to_hwirq(d); + unsigned long flags; + + raw_spin_lock_irqsave(&gctrl->lock, flags); + writel(BIT(offset), gctrl->membase + GPIO_IRNENCLR); + writel(BIT(offset), gctrl->membase + GPIO_IRNCR); + raw_spin_unlock_irqrestore(&gctrl->lock, flags); } static inline void eqbr_cfg_bit(void __iomem *addr, @@ -92,7 +99,7 @@ static int eqbr_irq_type_cfg(struct gpio_irq_type *type, return 0; } -static int eqbr_gpio_set_irq_type(struct irq_data *d, unsigned int type) +static int eqbr_irq_set_type(struct irq_data *d, unsigned int type) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -166,11 +173,11 @@ static void eqbr_irq_handler(struct irq_desc *desc) static const struct irq_chip eqbr_irq_chip = { .name = "gpio_irq", - .irq_mask = eqbr_gpio_disable_irq, - .irq_unmask = eqbr_gpio_enable_irq, - .irq_ack = eqbr_gpio_ack_irq, - .irq_mask_ack = eqbr_gpio_mask_ack_irq, - .irq_set_type = eqbr_gpio_set_irq_type, + .irq_ack = eqbr_irq_ack, + .irq_mask = eqbr_irq_mask, + .irq_mask_ack = eqbr_irq_mask_ack, + .irq_unmask = eqbr_irq_unmask, + .irq_set_type = eqbr_irq_set_type, .flags = IRQCHIP_IMMUTABLE, GPIOCHIP_IRQ_RESOURCE_HELPERS, }; @@ -846,6 +853,7 @@ static int pinbank_init(struct device_node *np, bank->pin_base = spec.args[1]; bank->nr_pins = spec.args[2]; + of_node_put(spec.np); bank->aval_pinmap = readl(bank->membase + REG_AVAIL); bank->id = id; diff --git a/drivers/pinctrl/pinctrl-k230.c b/drivers/pinctrl/pinctrl-k230.c index d716f23d837f7..20f7c0f70eb77 100644 --- a/drivers/pinctrl/pinctrl-k230.c +++ b/drivers/pinctrl/pinctrl-k230.c @@ -65,6 +65,7 @@ struct k230_pmx_func { }; struct k230_pinctrl { + struct device *dev; struct pinctrl_desc pctl; struct pinctrl_dev *pctl_dev; struct regmap *regmap_base; @@ -470,7 +471,7 @@ static int k230_pinctrl_parse_groups(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; const __be32 *list; int size, i, ret; @@ -511,7 +512,7 @@ static int k230_pinctrl_parse_functions(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; struct k230_pmx_func *func; struct k230_pin_group *grp; static unsigned int idx, i; @@ -596,6 +597,8 @@ static int k230_pinctrl_probe(struct platform_device *pdev) if (!info) return -ENOMEM; + info->dev = dev; + pctl = &info->pctl; pctl->name = "k230-pinctrl"; diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index 7a68a6237649c..f1cba3d2367a3 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -3184,10 +3184,9 @@ static int rockchip_pmx_set(struct pinctrl_dev *pctldev, unsigned selector, return 0; } -static int rockchip_pmx_gpio_set_direction(struct pinctrl_dev *pctldev, - struct pinctrl_gpio_range *range, - unsigned offset, - bool input) +static int rockchip_pmx_gpio_request_enable(struct pinctrl_dev *pctldev, + struct pinctrl_gpio_range *range, + unsigned int offset) { struct rockchip_pinctrl *info = pinctrl_dev_get_drvdata(pctldev); struct rockchip_pin_bank *bank; @@ -3201,7 +3200,7 @@ static const struct pinmux_ops rockchip_pmx_ops = { .get_function_name = rockchip_pmx_get_func_name, .get_function_groups = rockchip_pmx_get_groups, .set_mux = rockchip_pmx_set, - .gpio_set_direction = rockchip_pmx_gpio_set_direction, + .gpio_request_enable = rockchip_pmx_gpio_request_enable, }; /* diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 6d580aa282ec9..d85e6c1f63218 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -485,7 +485,8 @@ static int pcs_pinconf_get(struct pinctrl_dev *pctldev, struct pcs_device *pcs = pinctrl_dev_get_drvdata(pctldev); struct pcs_function *func; enum pin_config_param param; - unsigned offset = 0, data = 0, i, j, ret; + unsigned offset = 0, data = 0, i, j; + int ret; ret = pcs_get_function(pctldev, pin, &func); if (ret) @@ -549,9 +550,9 @@ static int pcs_pinconf_set(struct pinctrl_dev *pctldev, { struct pcs_device *pcs = pinctrl_dev_get_drvdata(pctldev); struct pcs_function *func; - unsigned offset = 0, shift = 0, i, data, ret; + unsigned offset = 0, shift = 0, i, data; u32 arg; - int j; + int j, ret; enum pin_config_param param; ret = pcs_get_function(pctldev, pin, &func); @@ -1358,6 +1359,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) } range = devm_kzalloc(pcs->dev, sizeof(*range), GFP_KERNEL); if (!range) { + of_node_put(gpiospec.np); ret = -ENOMEM; break; } @@ -1367,6 +1369,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) mutex_lock(&pcs->mutex); list_add_tail(&range->node, &pcs->gpiofuncs); mutex_unlock(&pcs->mutex); + of_node_put(gpiospec.np); } return ret; } diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig index c480e8b785032..f56592411cf6d 100644 --- a/drivers/pinctrl/qcom/Kconfig +++ b/drivers/pinctrl/qcom/Kconfig @@ -61,13 +61,14 @@ config PINCTRL_LPASS_LPI (Low Power Island) found on the Qualcomm Technologies Inc SoCs. config PINCTRL_SC7280_LPASS_LPI - tristate "Qualcomm Technologies Inc SC7280 LPASS LPI pin controller driver" + tristate "Qualcomm Technologies Inc SC7280 and SM8350 LPASS LPI pin controller driver" depends on ARM64 || COMPILE_TEST depends on PINCTRL_LPASS_LPI help This is the pinctrl, pinmux, pinconf and gpiolib driver for the Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI - (Low Power Island) found on the Qualcomm Technologies Inc SC7280 platform. + (Low Power Island) found on the Qualcomm Technologies Inc SC7280 + and SM8350 platforms. config PINCTRL_SDM660_LPASS_LPI tristate "Qualcomm Technologies Inc SDM660 LPASS LPI pin controller driver" @@ -106,16 +107,6 @@ config PINCTRL_SM8250_LPASS_LPI Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI (Low Power Island) found on the Qualcomm Technologies Inc SM8250 platform. -config PINCTRL_SM8350_LPASS_LPI - tristate "Qualcomm Technologies Inc SM8350 LPASS LPI pin controller driver" - depends on ARM64 || COMPILE_TEST - depends on PINCTRL_LPASS_LPI - help - This is the pinctrl, pinmux, pinconf and gpiolib driver for the - Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI - (Low Power Island) found on the Qualcomm Technologies Inc SM8350 - platform. - config PINCTRL_SM8450_LPASS_LPI tristate "Qualcomm Technologies Inc SM8450 LPASS LPI pin controller driver" depends on ARM64 || COMPILE_TEST diff --git a/drivers/pinctrl/qcom/Kconfig.msm b/drivers/pinctrl/qcom/Kconfig.msm index 69a5b47adedc2..3e9e02774001b 100644 --- a/drivers/pinctrl/qcom/Kconfig.msm +++ b/drivers/pinctrl/qcom/Kconfig.msm @@ -92,6 +92,14 @@ config PINCTRL_IPQ9574 Qualcomm Technologies Inc. IPQ9574 platform. Select this for IPQ9574. +config PINCTRL_KAANAPALI + tristate "Qualcomm Technologies Inc Kaanapali pin controller driver" + depends on ARM64 || COMPILE_TEST + help + This is the pinctrl, pinmux, pinconf and gpiolib driver for the + Qualcomm Technologies Inc TLMM block found on the Qualcomm + Technologies Inc Kaanapali platform. + config PINCTRL_MSM8226 tristate "Qualcomm 8226 pin controller driver" depends on ARM || COMPILE_TEST diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile index 567d3051e760d..4269d1781015e 100644 --- a/drivers/pinctrl/qcom/Makefile +++ b/drivers/pinctrl/qcom/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_PINCTRL_IPQ5424) += pinctrl-ipq5424.o obj-$(CONFIG_PINCTRL_IPQ8074) += pinctrl-ipq8074.o obj-$(CONFIG_PINCTRL_IPQ6018) += pinctrl-ipq6018.o obj-$(CONFIG_PINCTRL_IPQ9574) += pinctrl-ipq9574.o +obj-$(CONFIG_PINCTRL_KAANAPALI) += pinctrl-kaanapali.o obj-$(CONFIG_PINCTRL_MSM8226) += pinctrl-msm8226.o obj-$(CONFIG_PINCTRL_MSM8660) += pinctrl-msm8660.o obj-$(CONFIG_PINCTRL_MSM8960) += pinctrl-msm8960.o @@ -63,7 +64,6 @@ obj-$(CONFIG_PINCTRL_SM8150) += pinctrl-sm8150.o obj-$(CONFIG_PINCTRL_SM8250) += pinctrl-sm8250.o obj-$(CONFIG_PINCTRL_SM8250_LPASS_LPI) += pinctrl-sm8250-lpass-lpi.o obj-$(CONFIG_PINCTRL_SM8350) += pinctrl-sm8350.o -obj-$(CONFIG_PINCTRL_SM8350_LPASS_LPI) += pinctrl-sm8350-lpass-lpi.o obj-$(CONFIG_PINCTRL_SM8450) += pinctrl-sm8450.o obj-$(CONFIG_PINCTRL_SM8450_LPASS_LPI) += pinctrl-sm8450-lpass-lpi.o obj-$(CONFIG_PINCTRL_SM8550) += pinctrl-sm8550.o diff --git a/drivers/pinctrl/qcom/pinctrl-glymur.c b/drivers/pinctrl/qcom/pinctrl-glymur.c index 9913f98e95311..335005084b6bc 100644 --- a/drivers/pinctrl/qcom/pinctrl-glymur.c +++ b/drivers/pinctrl/qcom/pinctrl-glymur.c @@ -1316,7 +1316,7 @@ static const char *const wcn_sw_ctrl_groups[] = { }; static const struct pinfunction glymur_functions[] = { - MSM_PIN_FUNCTION(gpio), + MSM_GPIO_PIN_FUNCTION(gpio), MSM_PIN_FUNCTION(resout_gpio_n), MSM_PIN_FUNCTION(aoss_cti), MSM_PIN_FUNCTION(asc_cci), @@ -1342,7 +1342,7 @@ static const struct pinfunction glymur_functions[] = { MSM_PIN_FUNCTION(edp0_hot), MSM_PIN_FUNCTION(edp0_lcd), MSM_PIN_FUNCTION(edp1_lcd), - MSM_PIN_FUNCTION(egpio), + MSM_GPIO_PIN_FUNCTION(egpio), MSM_PIN_FUNCTION(eusb_ac_en), MSM_PIN_FUNCTION(gcc_gp1), MSM_PIN_FUNCTION(gcc_gp2), @@ -1743,7 +1743,7 @@ static const struct msm_pinctrl_soc_data glymur_tlmm = { }; static const struct of_device_id glymur_tlmm_of_match[] = { - { .compatible = "qcom,glymur-tlmm", .data = &glymur_tlmm }, + { .compatible = "qcom,glymur-tlmm", }, { } }; diff --git a/drivers/pinctrl/qcom/pinctrl-kaanapali.c b/drivers/pinctrl/qcom/pinctrl-kaanapali.c new file mode 100644 index 0000000000000..364e6d997337e --- /dev/null +++ b/drivers/pinctrl/qcom/pinctrl-kaanapali.c @@ -0,0 +1,1803 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include + +#include "pinctrl-msm.h" + +#define REG_SIZE 0x1000 +#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11) \ + { \ + .grp = PINCTRL_PINGROUP("gpio" #id, \ + gpio##id##_pins, \ + ARRAY_SIZE(gpio##id##_pins)), \ + .funcs = (int[]){ \ + msm_mux_gpio, /* gpio mode */ \ + msm_mux_##f1, \ + msm_mux_##f2, \ + msm_mux_##f3, \ + msm_mux_##f4, \ + msm_mux_##f5, \ + msm_mux_##f6, \ + msm_mux_##f7, \ + msm_mux_##f8, \ + msm_mux_##f9, \ + msm_mux_##f10, \ + msm_mux_##f11 /* egpio mode */ \ + }, \ + .nfuncs = 12, \ + .ctl_reg = REG_SIZE * id, \ + .io_reg = 0x4 + REG_SIZE * id, \ + .intr_cfg_reg = 0x8 + REG_SIZE * id, \ + .intr_status_reg = 0xc + REG_SIZE * id, \ + .intr_target_reg = 0x8 + REG_SIZE * id, \ + .mux_bit = 2, \ + .pull_bit = 0, \ + .drv_bit = 6, \ + .egpio_enable = 12, \ + .egpio_present = 11, \ + .oe_bit = 9, \ + .in_bit = 0, \ + .out_bit = 1, \ + .intr_enable_bit = 0, \ + .intr_status_bit = 0, \ + .intr_wakeup_present_bit = 6, \ + .intr_wakeup_enable_bit = 7, \ + .intr_target_bit = 8, \ + .intr_target_kpss_val = 3, \ + .intr_raw_status_bit = 4, \ + .intr_polarity_bit = 1, \ + .intr_detection_bit = 2, \ + .intr_detection_width = 2, \ + } + +#define SDC_QDSD_PINGROUP(pg_name, ctl, pull, drv) \ + { \ + .grp = PINCTRL_PINGROUP(#pg_name, \ + pg_name##_pins, \ + ARRAY_SIZE(pg_name##_pins)), \ + .ctl_reg = ctl, \ + .io_reg = 0, \ + .intr_cfg_reg = 0, \ + .intr_status_reg = 0, \ + .intr_target_reg = 0, \ + .mux_bit = -1, \ + .pull_bit = pull, \ + .drv_bit = drv, \ + .oe_bit = -1, \ + .in_bit = -1, \ + .out_bit = -1, \ + .intr_enable_bit = -1, \ + .intr_status_bit = -1, \ + .intr_target_bit = -1, \ + .intr_raw_status_bit = -1, \ + .intr_polarity_bit = -1, \ + .intr_detection_bit = -1, \ + .intr_detection_width = -1, \ + } + +#define UFS_RESET(pg_name, ctl, io) \ + { \ + .grp = PINCTRL_PINGROUP(#pg_name, \ + pg_name##_pins, \ + ARRAY_SIZE(pg_name##_pins)), \ + .ctl_reg = ctl, \ + .io_reg = io, \ + .intr_cfg_reg = 0, \ + .intr_status_reg = 0, \ + .intr_target_reg = 0, \ + .mux_bit = -1, \ + .pull_bit = 3, \ + .drv_bit = 0, \ + .oe_bit = -1, \ + .in_bit = -1, \ + .out_bit = 0, \ + .intr_enable_bit = -1, \ + .intr_status_bit = -1, \ + .intr_target_bit = -1, \ + .intr_raw_status_bit = -1, \ + .intr_polarity_bit = -1, \ + .intr_detection_bit = -1, \ + .intr_detection_width = -1, \ + } + +static const struct pinctrl_pin_desc kaanapali_pins[] = { + PINCTRL_PIN(0, "GPIO_0"), + PINCTRL_PIN(1, "GPIO_1"), + PINCTRL_PIN(2, "GPIO_2"), + PINCTRL_PIN(3, "GPIO_3"), + PINCTRL_PIN(4, "GPIO_4"), + PINCTRL_PIN(5, "GPIO_5"), + PINCTRL_PIN(6, "GPIO_6"), + PINCTRL_PIN(7, "GPIO_7"), + PINCTRL_PIN(8, "GPIO_8"), + PINCTRL_PIN(9, "GPIO_9"), + PINCTRL_PIN(10, "GPIO_10"), + PINCTRL_PIN(11, "GPIO_11"), + PINCTRL_PIN(12, "GPIO_12"), + PINCTRL_PIN(13, "GPIO_13"), + PINCTRL_PIN(14, "GPIO_14"), + PINCTRL_PIN(15, "GPIO_15"), + PINCTRL_PIN(16, "GPIO_16"), + PINCTRL_PIN(17, "GPIO_17"), + PINCTRL_PIN(18, "GPIO_18"), + PINCTRL_PIN(19, "GPIO_19"), + PINCTRL_PIN(20, "GPIO_20"), + PINCTRL_PIN(21, "GPIO_21"), + PINCTRL_PIN(22, "GPIO_22"), + PINCTRL_PIN(23, "GPIO_23"), + PINCTRL_PIN(24, "GPIO_24"), + PINCTRL_PIN(25, "GPIO_25"), + PINCTRL_PIN(26, "GPIO_26"), + PINCTRL_PIN(27, "GPIO_27"), + PINCTRL_PIN(28, "GPIO_28"), + PINCTRL_PIN(29, "GPIO_29"), + PINCTRL_PIN(30, "GPIO_30"), + PINCTRL_PIN(31, "GPIO_31"), + PINCTRL_PIN(32, "GPIO_32"), + PINCTRL_PIN(33, "GPIO_33"), + PINCTRL_PIN(34, "GPIO_34"), + PINCTRL_PIN(35, "GPIO_35"), + PINCTRL_PIN(36, "GPIO_36"), + PINCTRL_PIN(37, "GPIO_37"), + PINCTRL_PIN(38, "GPIO_38"), + PINCTRL_PIN(39, "GPIO_39"), + PINCTRL_PIN(40, "GPIO_40"), + PINCTRL_PIN(41, "GPIO_41"), + PINCTRL_PIN(42, "GPIO_42"), + PINCTRL_PIN(43, "GPIO_43"), + PINCTRL_PIN(44, "GPIO_44"), + PINCTRL_PIN(45, "GPIO_45"), + PINCTRL_PIN(46, "GPIO_46"), + PINCTRL_PIN(47, "GPIO_47"), + PINCTRL_PIN(48, "GPIO_48"), + PINCTRL_PIN(49, "GPIO_49"), + PINCTRL_PIN(50, "GPIO_50"), + PINCTRL_PIN(51, "GPIO_51"), + PINCTRL_PIN(52, "GPIO_52"), + PINCTRL_PIN(53, "GPIO_53"), + PINCTRL_PIN(54, "GPIO_54"), + PINCTRL_PIN(55, "GPIO_55"), + PINCTRL_PIN(56, "GPIO_56"), + PINCTRL_PIN(57, "GPIO_57"), + PINCTRL_PIN(58, "GPIO_58"), + PINCTRL_PIN(59, "GPIO_59"), + PINCTRL_PIN(60, "GPIO_60"), + PINCTRL_PIN(61, "GPIO_61"), + PINCTRL_PIN(62, "GPIO_62"), + PINCTRL_PIN(63, "GPIO_63"), + PINCTRL_PIN(64, "GPIO_64"), + PINCTRL_PIN(65, "GPIO_65"), + PINCTRL_PIN(66, "GPIO_66"), + PINCTRL_PIN(67, "GPIO_67"), + PINCTRL_PIN(68, "GPIO_68"), + PINCTRL_PIN(69, "GPIO_69"), + PINCTRL_PIN(70, "GPIO_70"), + PINCTRL_PIN(71, "GPIO_71"), + PINCTRL_PIN(72, "GPIO_72"), + PINCTRL_PIN(73, "GPIO_73"), + PINCTRL_PIN(74, "GPIO_74"), + PINCTRL_PIN(75, "GPIO_75"), + PINCTRL_PIN(76, "GPIO_76"), + PINCTRL_PIN(77, "GPIO_77"), + PINCTRL_PIN(78, "GPIO_78"), + PINCTRL_PIN(79, "GPIO_79"), + PINCTRL_PIN(80, "GPIO_80"), + PINCTRL_PIN(81, "GPIO_81"), + PINCTRL_PIN(82, "GPIO_82"), + PINCTRL_PIN(83, "GPIO_83"), + PINCTRL_PIN(84, "GPIO_84"), + PINCTRL_PIN(85, "GPIO_85"), + PINCTRL_PIN(86, "GPIO_86"), + PINCTRL_PIN(87, "GPIO_87"), + PINCTRL_PIN(88, "GPIO_88"), + PINCTRL_PIN(89, "GPIO_89"), + PINCTRL_PIN(90, "GPIO_90"), + PINCTRL_PIN(91, "GPIO_91"), + PINCTRL_PIN(92, "GPIO_92"), + PINCTRL_PIN(93, "GPIO_93"), + PINCTRL_PIN(94, "GPIO_94"), + PINCTRL_PIN(95, "GPIO_95"), + PINCTRL_PIN(96, "GPIO_96"), + PINCTRL_PIN(97, "GPIO_97"), + PINCTRL_PIN(98, "GPIO_98"), + PINCTRL_PIN(99, "GPIO_99"), + PINCTRL_PIN(100, "GPIO_100"), + PINCTRL_PIN(101, "GPIO_101"), + PINCTRL_PIN(102, "GPIO_102"), + PINCTRL_PIN(103, "GPIO_103"), + PINCTRL_PIN(104, "GPIO_104"), + PINCTRL_PIN(105, "GPIO_105"), + PINCTRL_PIN(106, "GPIO_106"), + PINCTRL_PIN(107, "GPIO_107"), + PINCTRL_PIN(108, "GPIO_108"), + PINCTRL_PIN(109, "GPIO_109"), + PINCTRL_PIN(110, "GPIO_110"), + PINCTRL_PIN(111, "GPIO_111"), + PINCTRL_PIN(112, "GPIO_112"), + PINCTRL_PIN(113, "GPIO_113"), + PINCTRL_PIN(114, "GPIO_114"), + PINCTRL_PIN(115, "GPIO_115"), + PINCTRL_PIN(116, "GPIO_116"), + PINCTRL_PIN(117, "GPIO_117"), + PINCTRL_PIN(118, "GPIO_118"), + PINCTRL_PIN(119, "GPIO_119"), + PINCTRL_PIN(120, "GPIO_120"), + PINCTRL_PIN(121, "GPIO_121"), + PINCTRL_PIN(122, "GPIO_122"), + PINCTRL_PIN(123, "GPIO_123"), + PINCTRL_PIN(124, "GPIO_124"), + PINCTRL_PIN(125, "GPIO_125"), + PINCTRL_PIN(126, "GPIO_126"), + PINCTRL_PIN(127, "GPIO_127"), + PINCTRL_PIN(128, "GPIO_128"), + PINCTRL_PIN(129, "GPIO_129"), + PINCTRL_PIN(130, "GPIO_130"), + PINCTRL_PIN(131, "GPIO_131"), + PINCTRL_PIN(132, "GPIO_132"), + PINCTRL_PIN(133, "GPIO_133"), + PINCTRL_PIN(134, "GPIO_134"), + PINCTRL_PIN(135, "GPIO_135"), + PINCTRL_PIN(136, "GPIO_136"), + PINCTRL_PIN(137, "GPIO_137"), + PINCTRL_PIN(138, "GPIO_138"), + PINCTRL_PIN(139, "GPIO_139"), + PINCTRL_PIN(140, "GPIO_140"), + PINCTRL_PIN(141, "GPIO_141"), + PINCTRL_PIN(142, "GPIO_142"), + PINCTRL_PIN(143, "GPIO_143"), + PINCTRL_PIN(144, "GPIO_144"), + PINCTRL_PIN(145, "GPIO_145"), + PINCTRL_PIN(146, "GPIO_146"), + PINCTRL_PIN(147, "GPIO_147"), + PINCTRL_PIN(148, "GPIO_148"), + PINCTRL_PIN(149, "GPIO_149"), + PINCTRL_PIN(150, "GPIO_150"), + PINCTRL_PIN(151, "GPIO_151"), + PINCTRL_PIN(152, "GPIO_152"), + PINCTRL_PIN(153, "GPIO_153"), + PINCTRL_PIN(154, "GPIO_154"), + PINCTRL_PIN(155, "GPIO_155"), + PINCTRL_PIN(156, "GPIO_156"), + PINCTRL_PIN(157, "GPIO_157"), + PINCTRL_PIN(158, "GPIO_158"), + PINCTRL_PIN(159, "GPIO_159"), + PINCTRL_PIN(160, "GPIO_160"), + PINCTRL_PIN(161, "GPIO_161"), + PINCTRL_PIN(162, "GPIO_162"), + PINCTRL_PIN(163, "GPIO_163"), + PINCTRL_PIN(164, "GPIO_164"), + PINCTRL_PIN(165, "GPIO_165"), + PINCTRL_PIN(166, "GPIO_166"), + PINCTRL_PIN(167, "GPIO_167"), + PINCTRL_PIN(168, "GPIO_168"), + PINCTRL_PIN(169, "GPIO_169"), + PINCTRL_PIN(170, "GPIO_170"), + PINCTRL_PIN(171, "GPIO_171"), + PINCTRL_PIN(172, "GPIO_172"), + PINCTRL_PIN(173, "GPIO_173"), + PINCTRL_PIN(174, "GPIO_174"), + PINCTRL_PIN(175, "GPIO_175"), + PINCTRL_PIN(176, "GPIO_176"), + PINCTRL_PIN(177, "GPIO_177"), + PINCTRL_PIN(178, "GPIO_178"), + PINCTRL_PIN(179, "GPIO_179"), + PINCTRL_PIN(180, "GPIO_180"), + PINCTRL_PIN(181, "GPIO_181"), + PINCTRL_PIN(182, "GPIO_182"), + PINCTRL_PIN(183, "GPIO_183"), + PINCTRL_PIN(184, "GPIO_184"), + PINCTRL_PIN(185, "GPIO_185"), + PINCTRL_PIN(186, "GPIO_186"), + PINCTRL_PIN(187, "GPIO_187"), + PINCTRL_PIN(188, "GPIO_188"), + PINCTRL_PIN(189, "GPIO_189"), + PINCTRL_PIN(190, "GPIO_190"), + PINCTRL_PIN(191, "GPIO_191"), + PINCTRL_PIN(192, "GPIO_192"), + PINCTRL_PIN(193, "GPIO_193"), + PINCTRL_PIN(194, "GPIO_194"), + PINCTRL_PIN(195, "GPIO_195"), + PINCTRL_PIN(196, "GPIO_196"), + PINCTRL_PIN(197, "GPIO_197"), + PINCTRL_PIN(198, "GPIO_198"), + PINCTRL_PIN(199, "GPIO_199"), + PINCTRL_PIN(200, "GPIO_200"), + PINCTRL_PIN(201, "GPIO_201"), + PINCTRL_PIN(202, "GPIO_202"), + PINCTRL_PIN(203, "GPIO_203"), + PINCTRL_PIN(204, "GPIO_204"), + PINCTRL_PIN(205, "GPIO_205"), + PINCTRL_PIN(206, "GPIO_206"), + PINCTRL_PIN(207, "GPIO_207"), + PINCTRL_PIN(208, "GPIO_208"), + PINCTRL_PIN(209, "GPIO_209"), + PINCTRL_PIN(210, "GPIO_210"), + PINCTRL_PIN(211, "GPIO_211"), + PINCTRL_PIN(212, "GPIO_212"), + PINCTRL_PIN(213, "GPIO_213"), + PINCTRL_PIN(214, "GPIO_214"), + PINCTRL_PIN(215, "GPIO_215"), + PINCTRL_PIN(216, "GPIO_216"), + PINCTRL_PIN(217, "UFS_RESET"), + PINCTRL_PIN(218, "SDC2_CLK"), + PINCTRL_PIN(219, "SDC2_CMD"), + PINCTRL_PIN(220, "SDC2_DATA"), +}; + +#define DECLARE_MSM_GPIO_PINS(pin) \ + static const unsigned int gpio##pin##_pins[] = { pin } +DECLARE_MSM_GPIO_PINS(0); +DECLARE_MSM_GPIO_PINS(1); +DECLARE_MSM_GPIO_PINS(2); +DECLARE_MSM_GPIO_PINS(3); +DECLARE_MSM_GPIO_PINS(4); +DECLARE_MSM_GPIO_PINS(5); +DECLARE_MSM_GPIO_PINS(6); +DECLARE_MSM_GPIO_PINS(7); +DECLARE_MSM_GPIO_PINS(8); +DECLARE_MSM_GPIO_PINS(9); +DECLARE_MSM_GPIO_PINS(10); +DECLARE_MSM_GPIO_PINS(11); +DECLARE_MSM_GPIO_PINS(12); +DECLARE_MSM_GPIO_PINS(13); +DECLARE_MSM_GPIO_PINS(14); +DECLARE_MSM_GPIO_PINS(15); +DECLARE_MSM_GPIO_PINS(16); +DECLARE_MSM_GPIO_PINS(17); +DECLARE_MSM_GPIO_PINS(18); +DECLARE_MSM_GPIO_PINS(19); +DECLARE_MSM_GPIO_PINS(20); +DECLARE_MSM_GPIO_PINS(21); +DECLARE_MSM_GPIO_PINS(22); +DECLARE_MSM_GPIO_PINS(23); +DECLARE_MSM_GPIO_PINS(24); +DECLARE_MSM_GPIO_PINS(25); +DECLARE_MSM_GPIO_PINS(26); +DECLARE_MSM_GPIO_PINS(27); +DECLARE_MSM_GPIO_PINS(28); +DECLARE_MSM_GPIO_PINS(29); +DECLARE_MSM_GPIO_PINS(30); +DECLARE_MSM_GPIO_PINS(31); +DECLARE_MSM_GPIO_PINS(32); +DECLARE_MSM_GPIO_PINS(33); +DECLARE_MSM_GPIO_PINS(34); +DECLARE_MSM_GPIO_PINS(35); +DECLARE_MSM_GPIO_PINS(36); +DECLARE_MSM_GPIO_PINS(37); +DECLARE_MSM_GPIO_PINS(38); +DECLARE_MSM_GPIO_PINS(39); +DECLARE_MSM_GPIO_PINS(40); +DECLARE_MSM_GPIO_PINS(41); +DECLARE_MSM_GPIO_PINS(42); +DECLARE_MSM_GPIO_PINS(43); +DECLARE_MSM_GPIO_PINS(44); +DECLARE_MSM_GPIO_PINS(45); +DECLARE_MSM_GPIO_PINS(46); +DECLARE_MSM_GPIO_PINS(47); +DECLARE_MSM_GPIO_PINS(48); +DECLARE_MSM_GPIO_PINS(49); +DECLARE_MSM_GPIO_PINS(50); +DECLARE_MSM_GPIO_PINS(51); +DECLARE_MSM_GPIO_PINS(52); +DECLARE_MSM_GPIO_PINS(53); +DECLARE_MSM_GPIO_PINS(54); +DECLARE_MSM_GPIO_PINS(55); +DECLARE_MSM_GPIO_PINS(56); +DECLARE_MSM_GPIO_PINS(57); +DECLARE_MSM_GPIO_PINS(58); +DECLARE_MSM_GPIO_PINS(59); +DECLARE_MSM_GPIO_PINS(60); +DECLARE_MSM_GPIO_PINS(61); +DECLARE_MSM_GPIO_PINS(62); +DECLARE_MSM_GPIO_PINS(63); +DECLARE_MSM_GPIO_PINS(64); +DECLARE_MSM_GPIO_PINS(65); +DECLARE_MSM_GPIO_PINS(66); +DECLARE_MSM_GPIO_PINS(67); +DECLARE_MSM_GPIO_PINS(68); +DECLARE_MSM_GPIO_PINS(69); +DECLARE_MSM_GPIO_PINS(70); +DECLARE_MSM_GPIO_PINS(71); +DECLARE_MSM_GPIO_PINS(72); +DECLARE_MSM_GPIO_PINS(73); +DECLARE_MSM_GPIO_PINS(74); +DECLARE_MSM_GPIO_PINS(75); +DECLARE_MSM_GPIO_PINS(76); +DECLARE_MSM_GPIO_PINS(77); +DECLARE_MSM_GPIO_PINS(78); +DECLARE_MSM_GPIO_PINS(79); +DECLARE_MSM_GPIO_PINS(80); +DECLARE_MSM_GPIO_PINS(81); +DECLARE_MSM_GPIO_PINS(82); +DECLARE_MSM_GPIO_PINS(83); +DECLARE_MSM_GPIO_PINS(84); +DECLARE_MSM_GPIO_PINS(85); +DECLARE_MSM_GPIO_PINS(86); +DECLARE_MSM_GPIO_PINS(87); +DECLARE_MSM_GPIO_PINS(88); +DECLARE_MSM_GPIO_PINS(89); +DECLARE_MSM_GPIO_PINS(90); +DECLARE_MSM_GPIO_PINS(91); +DECLARE_MSM_GPIO_PINS(92); +DECLARE_MSM_GPIO_PINS(93); +DECLARE_MSM_GPIO_PINS(94); +DECLARE_MSM_GPIO_PINS(95); +DECLARE_MSM_GPIO_PINS(96); +DECLARE_MSM_GPIO_PINS(97); +DECLARE_MSM_GPIO_PINS(98); +DECLARE_MSM_GPIO_PINS(99); +DECLARE_MSM_GPIO_PINS(100); +DECLARE_MSM_GPIO_PINS(101); +DECLARE_MSM_GPIO_PINS(102); +DECLARE_MSM_GPIO_PINS(103); +DECLARE_MSM_GPIO_PINS(104); +DECLARE_MSM_GPIO_PINS(105); +DECLARE_MSM_GPIO_PINS(106); +DECLARE_MSM_GPIO_PINS(107); +DECLARE_MSM_GPIO_PINS(108); +DECLARE_MSM_GPIO_PINS(109); +DECLARE_MSM_GPIO_PINS(110); +DECLARE_MSM_GPIO_PINS(111); +DECLARE_MSM_GPIO_PINS(112); +DECLARE_MSM_GPIO_PINS(113); +DECLARE_MSM_GPIO_PINS(114); +DECLARE_MSM_GPIO_PINS(115); +DECLARE_MSM_GPIO_PINS(116); +DECLARE_MSM_GPIO_PINS(117); +DECLARE_MSM_GPIO_PINS(118); +DECLARE_MSM_GPIO_PINS(119); +DECLARE_MSM_GPIO_PINS(120); +DECLARE_MSM_GPIO_PINS(121); +DECLARE_MSM_GPIO_PINS(122); +DECLARE_MSM_GPIO_PINS(123); +DECLARE_MSM_GPIO_PINS(124); +DECLARE_MSM_GPIO_PINS(125); +DECLARE_MSM_GPIO_PINS(126); +DECLARE_MSM_GPIO_PINS(127); +DECLARE_MSM_GPIO_PINS(128); +DECLARE_MSM_GPIO_PINS(129); +DECLARE_MSM_GPIO_PINS(130); +DECLARE_MSM_GPIO_PINS(131); +DECLARE_MSM_GPIO_PINS(132); +DECLARE_MSM_GPIO_PINS(133); +DECLARE_MSM_GPIO_PINS(134); +DECLARE_MSM_GPIO_PINS(135); +DECLARE_MSM_GPIO_PINS(136); +DECLARE_MSM_GPIO_PINS(137); +DECLARE_MSM_GPIO_PINS(138); +DECLARE_MSM_GPIO_PINS(139); +DECLARE_MSM_GPIO_PINS(140); +DECLARE_MSM_GPIO_PINS(141); +DECLARE_MSM_GPIO_PINS(142); +DECLARE_MSM_GPIO_PINS(143); +DECLARE_MSM_GPIO_PINS(144); +DECLARE_MSM_GPIO_PINS(145); +DECLARE_MSM_GPIO_PINS(146); +DECLARE_MSM_GPIO_PINS(147); +DECLARE_MSM_GPIO_PINS(148); +DECLARE_MSM_GPIO_PINS(149); +DECLARE_MSM_GPIO_PINS(150); +DECLARE_MSM_GPIO_PINS(151); +DECLARE_MSM_GPIO_PINS(152); +DECLARE_MSM_GPIO_PINS(153); +DECLARE_MSM_GPIO_PINS(154); +DECLARE_MSM_GPIO_PINS(155); +DECLARE_MSM_GPIO_PINS(156); +DECLARE_MSM_GPIO_PINS(157); +DECLARE_MSM_GPIO_PINS(158); +DECLARE_MSM_GPIO_PINS(159); +DECLARE_MSM_GPIO_PINS(160); +DECLARE_MSM_GPIO_PINS(161); +DECLARE_MSM_GPIO_PINS(162); +DECLARE_MSM_GPIO_PINS(163); +DECLARE_MSM_GPIO_PINS(164); +DECLARE_MSM_GPIO_PINS(165); +DECLARE_MSM_GPIO_PINS(166); +DECLARE_MSM_GPIO_PINS(167); +DECLARE_MSM_GPIO_PINS(168); +DECLARE_MSM_GPIO_PINS(169); +DECLARE_MSM_GPIO_PINS(170); +DECLARE_MSM_GPIO_PINS(171); +DECLARE_MSM_GPIO_PINS(172); +DECLARE_MSM_GPIO_PINS(173); +DECLARE_MSM_GPIO_PINS(174); +DECLARE_MSM_GPIO_PINS(175); +DECLARE_MSM_GPIO_PINS(176); +DECLARE_MSM_GPIO_PINS(177); +DECLARE_MSM_GPIO_PINS(178); +DECLARE_MSM_GPIO_PINS(179); +DECLARE_MSM_GPIO_PINS(180); +DECLARE_MSM_GPIO_PINS(181); +DECLARE_MSM_GPIO_PINS(182); +DECLARE_MSM_GPIO_PINS(183); +DECLARE_MSM_GPIO_PINS(184); +DECLARE_MSM_GPIO_PINS(185); +DECLARE_MSM_GPIO_PINS(186); +DECLARE_MSM_GPIO_PINS(187); +DECLARE_MSM_GPIO_PINS(188); +DECLARE_MSM_GPIO_PINS(189); +DECLARE_MSM_GPIO_PINS(190); +DECLARE_MSM_GPIO_PINS(191); +DECLARE_MSM_GPIO_PINS(192); +DECLARE_MSM_GPIO_PINS(193); +DECLARE_MSM_GPIO_PINS(194); +DECLARE_MSM_GPIO_PINS(195); +DECLARE_MSM_GPIO_PINS(196); +DECLARE_MSM_GPIO_PINS(197); +DECLARE_MSM_GPIO_PINS(198); +DECLARE_MSM_GPIO_PINS(199); +DECLARE_MSM_GPIO_PINS(200); +DECLARE_MSM_GPIO_PINS(201); +DECLARE_MSM_GPIO_PINS(202); +DECLARE_MSM_GPIO_PINS(203); +DECLARE_MSM_GPIO_PINS(204); +DECLARE_MSM_GPIO_PINS(205); +DECLARE_MSM_GPIO_PINS(206); +DECLARE_MSM_GPIO_PINS(207); +DECLARE_MSM_GPIO_PINS(208); +DECLARE_MSM_GPIO_PINS(209); +DECLARE_MSM_GPIO_PINS(210); +DECLARE_MSM_GPIO_PINS(211); +DECLARE_MSM_GPIO_PINS(212); +DECLARE_MSM_GPIO_PINS(213); +DECLARE_MSM_GPIO_PINS(214); +DECLARE_MSM_GPIO_PINS(215); +DECLARE_MSM_GPIO_PINS(216); + +static const unsigned int ufs_reset_pins[] = { 217 }; +static const unsigned int sdc2_clk_pins[] = { 218 }; +static const unsigned int sdc2_cmd_pins[] = { 219 }; +static const unsigned int sdc2_data_pins[] = { 220 }; + +enum kaanapali_functions { + msm_mux_gpio, + msm_mux_aoss_cti, + msm_mux_atest_char, + msm_mux_atest_usb, + msm_mux_audio_ext_mclk0, + msm_mux_audio_ext_mclk1, + msm_mux_audio_ref_clk, + msm_mux_cam_asc_mclk2, + msm_mux_cam_asc_mclk4, + msm_mux_cam_mclk, + msm_mux_cci_async_in, + msm_mux_cci_i2c_scl, + msm_mux_cci_i2c_sda, + msm_mux_cci_timer, + msm_mux_cmu_rng, + msm_mux_coex_uart1_rx, + msm_mux_coex_uart1_tx, + msm_mux_coex_uart2_rx, + msm_mux_coex_uart2_tx, + msm_mux_dbg_out_clk, + msm_mux_ddr_bist_complete, + msm_mux_ddr_bist_fail, + msm_mux_ddr_bist_start, + msm_mux_ddr_bist_stop, + msm_mux_ddr_pxi0, + msm_mux_ddr_pxi1, + msm_mux_ddr_pxi2, + msm_mux_ddr_pxi3, + msm_mux_dp_hot, + msm_mux_egpio, + msm_mux_gcc_gp1, + msm_mux_gcc_gp2, + msm_mux_gcc_gp3, + msm_mux_gnss_adc0, + msm_mux_gnss_adc1, + msm_mux_i2chub0_se0, + msm_mux_i2chub0_se1, + msm_mux_i2chub0_se2, + msm_mux_i2chub0_se3, + msm_mux_i2chub0_se4, + msm_mux_i2s0_data0, + msm_mux_i2s0_data1, + msm_mux_i2s0_sck, + msm_mux_i2s0_ws, + msm_mux_i2s1_data0, + msm_mux_i2s1_data1, + msm_mux_i2s1_sck, + msm_mux_i2s1_ws, + msm_mux_ibi_i3c, + msm_mux_jitter_bist, + msm_mux_mdp_esync0_out, + msm_mux_mdp_esync1_out, + msm_mux_mdp_vsync, + msm_mux_mdp_vsync0_out, + msm_mux_mdp_vsync1_out, + msm_mux_mdp_vsync2_out, + msm_mux_mdp_vsync3_out, + msm_mux_mdp_vsync5_out, + msm_mux_mdp_vsync_e, + msm_mux_nav_gpio0, + msm_mux_nav_gpio1, + msm_mux_nav_gpio2, + msm_mux_nav_gpio3, + msm_mux_pcie0_clk_req_n, + msm_mux_phase_flag, + msm_mux_pll_bist_sync, + msm_mux_pll_clk_aux, + msm_mux_prng_rosc0, + msm_mux_prng_rosc1, + msm_mux_prng_rosc2, + msm_mux_prng_rosc3, + msm_mux_qdss_cti, + msm_mux_qdss_gpio_traceclk, + msm_mux_qdss_gpio_tracectl, + msm_mux_qdss_gpio_tracedata, + msm_mux_qlink_big_enable, + msm_mux_qlink_big_request, + msm_mux_qlink_little_enable, + msm_mux_qlink_little_request, + msm_mux_qlink_wmss, + msm_mux_qspi0, + msm_mux_qspi1, + msm_mux_qspi2, + msm_mux_qspi3, + msm_mux_qspi_clk, + msm_mux_qspi_cs, + msm_mux_qup1_se0, + msm_mux_qup1_se1, + msm_mux_qup1_se2, + msm_mux_qup1_se3, + msm_mux_qup1_se4, + msm_mux_qup1_se5, + msm_mux_qup1_se6, + msm_mux_qup1_se7, + msm_mux_qup2_se0, + msm_mux_qup2_se1, + msm_mux_qup2_se2, + msm_mux_qup2_se3, + msm_mux_qup2_se4, + msm_mux_qup3_se0, + msm_mux_qup3_se1, + msm_mux_qup3_se2, + msm_mux_qup3_se3, + msm_mux_qup3_se4, + msm_mux_qup3_se5, + msm_mux_qup4_se0, + msm_mux_qup4_se1, + msm_mux_qup4_se2, + msm_mux_qup4_se3, + msm_mux_qup4_se4, + msm_mux_sd_write_protect, + msm_mux_sdc40, + msm_mux_sdc41, + msm_mux_sdc42, + msm_mux_sdc43, + msm_mux_sdc4_clk, + msm_mux_sdc4_cmd, + msm_mux_sys_throttle, + msm_mux_tb_trig_sdc2, + msm_mux_tb_trig_sdc4, + msm_mux_tmess_prng0, + msm_mux_tmess_prng1, + msm_mux_tmess_prng2, + msm_mux_tmess_prng3, + msm_mux_tsense_pwm1, + msm_mux_tsense_pwm2, + msm_mux_tsense_pwm3, + msm_mux_tsense_pwm4, + msm_mux_tsense_pwm5, + msm_mux_tsense_pwm6, + msm_mux_tsense_pwm7, + msm_mux_uim0_clk, + msm_mux_uim0_data, + msm_mux_uim0_present, + msm_mux_uim0_reset, + msm_mux_uim1_clk, + msm_mux_uim1_data, + msm_mux_uim1_present, + msm_mux_uim1_reset, + msm_mux_usb0_hs, + msm_mux_usb_phy, + msm_mux_vfr_0, + msm_mux_vfr_1, + msm_mux_vsense_trigger_mirnat, + msm_mux_wcn_sw, + msm_mux_wcn_sw_ctrl, + msm_mux__, +}; + +static const char *const gpio_groups[] = { + "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", + "gpio6", "gpio7", "gpio8", "gpio9", "gpio10", "gpio11", + "gpio12", "gpio13", "gpio14", "gpio15", "gpio16", "gpio17", + "gpio18", "gpio19", "gpio20", "gpio21", "gpio22", "gpio23", + "gpio24", "gpio25", "gpio26", "gpio27", "gpio28", "gpio29", + "gpio30", "gpio31", "gpio32", "gpio33", "gpio34", "gpio35", + "gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41", + "gpio42", "gpio43", "gpio44", "gpio45", "gpio46", "gpio47", + "gpio48", "gpio49", "gpio50", "gpio51", "gpio52", "gpio53", + "gpio54", "gpio55", "gpio56", "gpio57", "gpio58", "gpio59", + "gpio60", "gpio61", "gpio62", "gpio63", "gpio64", "gpio65", + "gpio66", "gpio67", "gpio68", "gpio69", "gpio70", "gpio71", + "gpio72", "gpio73", "gpio74", "gpio75", "gpio76", "gpio77", + "gpio78", "gpio79", "gpio80", "gpio81", "gpio82", "gpio83", + "gpio84", "gpio85", "gpio86", "gpio87", "gpio88", "gpio89", + "gpio90", "gpio91", "gpio92", "gpio93", "gpio94", "gpio95", + "gpio96", "gpio97", "gpio98", "gpio99", "gpio100", "gpio101", + "gpio102", "gpio103", "gpio104", "gpio105", "gpio106", "gpio107", + "gpio108", "gpio109", "gpio110", "gpio111", "gpio112", "gpio113", + "gpio114", "gpio115", "gpio116", "gpio117", "gpio118", "gpio119", + "gpio120", "gpio121", "gpio122", "gpio123", "gpio124", "gpio125", + "gpio126", "gpio127", "gpio128", "gpio129", "gpio130", "gpio131", + "gpio132", "gpio133", "gpio134", "gpio135", "gpio136", "gpio137", + "gpio138", "gpio139", "gpio140", "gpio141", "gpio142", "gpio143", + "gpio144", "gpio145", "gpio146", "gpio147", "gpio148", "gpio149", + "gpio150", "gpio151", "gpio152", "gpio153", "gpio154", "gpio155", + "gpio156", "gpio157", "gpio158", "gpio159", "gpio160", "gpio161", + "gpio162", "gpio163", "gpio164", "gpio165", "gpio166", "gpio167", + "gpio168", "gpio169", "gpio170", "gpio171", "gpio172", "gpio173", + "gpio174", "gpio175", "gpio176", "gpio177", "gpio178", "gpio179", + "gpio180", "gpio181", "gpio182", "gpio183", "gpio184", "gpio185", + "gpio186", "gpio187", "gpio188", "gpio189", "gpio190", "gpio191", + "gpio192", "gpio193", "gpio194", "gpio195", "gpio196", "gpio197", + "gpio198", "gpio199", "gpio200", "gpio201", "gpio202", "gpio203", + "gpio204", "gpio205", "gpio206", "gpio207", "gpio208", "gpio209", + "gpio210", "gpio211", "gpio212", "gpio213", "gpio214", "gpio215", + "gpio216", +}; + +static const char *const aoss_cti_groups[] = { + "gpio74", "gpio75", "gpio76", "gpio77", +}; + +static const char *const atest_char_groups[] = { + "gpio126", "gpio127", "gpio128", "gpio129", "gpio133", +}; + +static const char *const atest_usb_groups[] = { + "gpio70", "gpio71", "gpio72", "gpio73", "gpio129", +}; + +static const char *const audio_ext_mclk0_groups[] = { + "gpio121", +}; + +static const char *const audio_ext_mclk1_groups[] = { + "gpio120", +}; + +static const char *const audio_ref_clk_groups[] = { + "gpio120", +}; + +static const char *const cam_asc_mclk2_groups[] = { + "gpio91", +}; + +static const char *const cam_asc_mclk4_groups[] = { + "gpio93", +}; + +static const char *const cam_mclk_groups[] = { + "gpio89", "gpio90", "gpio92", "gpio94", "gpio95", "gpio96", +}; + +static const char *const cci_async_in_groups[] = { + "gpio10", "gpio11", "gpio15", +}; + +static const char *const cci_i2c_scl_groups[] = { + "gpio110", "gpio112", "gpio114", "gpio116", "gpio149", "gpio160", +}; + +static const char *const cci_i2c_sda_groups[] = { + "gpio107", "gpio108", "gpio109", "gpio111", "gpio113", "gpio115", +}; + +static const char *const cci_timer_groups[] = { + "gpio105", "gpio106", "gpio107", "gpio159", "gpio160", +}; + +static const char *const cmu_rng_groups[] = { + "gpio40", "gpio41", "gpio42", "gpio43", "gpio144", "gpio145", + "gpio146", "gpio147", +}; + +static const char *const coex_uart1_rx_groups[] = { + "gpio144", +}; + +static const char *const coex_uart1_tx_groups[] = { + "gpio145", +}; + +static const char *const coex_uart2_rx_groups[] = { + "gpio146", +}; + +static const char *const coex_uart2_tx_groups[] = { + "gpio147", +}; + +static const char *const dbg_out_clk_groups[] = { + "gpio42", +}; + +static const char *const ddr_bist_complete_groups[] = { + "gpio44", +}; + +static const char *const ddr_bist_fail_groups[] = { + "gpio40", +}; + +static const char *const ddr_bist_start_groups[] = { + "gpio41", +}; + +static const char *const ddr_bist_stop_groups[] = { + "gpio45", +}; + +static const char *const ddr_pxi0_groups[] = { + "gpio54", "gpio55", +}; + +static const char *const ddr_pxi1_groups[] = { + "gpio44", "gpio45", +}; + +static const char *const ddr_pxi2_groups[] = { + "gpio43", "gpio52", +}; + +static const char *const ddr_pxi3_groups[] = { + "gpio46", "gpio53", +}; + +static const char *const dp_hot_groups[] = { + "gpio47", +}; + +static const char *const egpio_groups[] = { + "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", + "gpio6", "gpio7", "gpio28", "gpio29", "gpio48", "gpio49", + "gpio50", "gpio51", "gpio163", "gpio164", "gpio165", "gpio166", + "gpio167", "gpio168", "gpio169", "gpio170", "gpio171", "gpio172", + "gpio173", "gpio174", "gpio175", "gpio176", "gpio177", "gpio178", + "gpio179", "gpio180", "gpio181", "gpio182", "gpio183", "gpio184", + "gpio185", "gpio186", "gpio187", "gpio188", "gpio189", "gpio190", + "gpio191", "gpio192", "gpio193", "gpio194", "gpio195", "gpio196", + "gpio197", "gpio198", "gpio199", "gpio200", "gpio201", "gpio202", + "gpio203", "gpio204", "gpio205", "gpio206", "gpio207", "gpio208", + "gpio209", "gpio210", "gpio211", "gpio212", "gpio213", "gpio214", + "gpio215", "gpio216", +}; + +static const char *const gcc_gp1_groups[] = { + "gpio130", "gpio158", +}; + +static const char *const gcc_gp2_groups[] = { + "gpio86", "gpio131", +}; + +static const char *const gcc_gp3_groups[] = { + "gpio87", "gpio132", +}; + +static const char *const gnss_adc0_groups[] = { + "gpio40", "gpio41", +}; + +static const char *const gnss_adc1_groups[] = { + "gpio42", "gpio77", +}; + +static const char *const i2chub0_se0_groups[] = { + "gpio66", "gpio67", +}; + +static const char *const i2chub0_se1_groups[] = { + "gpio78", "gpio79", +}; + +static const char *const i2chub0_se2_groups[] = { + "gpio68", "gpio69", +}; + +static const char *const i2chub0_se3_groups[] = { + "gpio70", "gpio71", +}; + +static const char *const i2chub0_se4_groups[] = { + "gpio72", "gpio73", +}; + +static const char *const i2s0_data0_groups[] = { + "gpio123", +}; + +static const char *const i2s0_data1_groups[] = { + "gpio124", +}; + +static const char *const i2s0_sck_groups[] = { + "gpio122", +}; + +static const char *const i2s0_ws_groups[] = { + "gpio125", +}; + +static const char *const i2s1_data0_groups[] = { + "gpio118", +}; + +static const char *const i2s1_data1_groups[] = { + "gpio120", +}; + +static const char *const i2s1_sck_groups[] = { + "gpio117", +}; + +static const char *const i2s1_ws_groups[] = { + "gpio119", +}; + +static const char *const ibi_i3c_groups[] = { + "gpio0", "gpio1", "gpio4", "gpio5", "gpio8", "gpio9", + "gpio12", "gpio13", "gpio28", "gpio29", "gpio32", "gpio33", + "gpio36", "gpio37", "gpio48", "gpio49", "gpio60", "gpio61", +}; + +static const char *const jitter_bist_groups[] = { + "gpio73", +}; + +static const char *const mdp_esync0_out_groups[] = { + "gpio88", +}; + +static const char *const mdp_esync1_out_groups[] = { + "gpio100", +}; + +static const char *const mdp_vsync_groups[] = { + "gpio86", "gpio87", "gpio97", "gpio98", +}; + +static const char *const mdp_vsync0_out_groups[] = { + "gpio86", +}; + +static const char *const mdp_vsync1_out_groups[] = { + "gpio86", +}; + +static const char *const mdp_vsync2_out_groups[] = { + "gpio87", +}; + +static const char *const mdp_vsync3_out_groups[] = { + "gpio87", +}; + +static const char *const mdp_vsync5_out_groups[] = { + "gpio87", +}; + +static const char *const mdp_vsync_e_groups[] = { + "gpio88", +}; + +static const char *const nav_gpio0_groups[] = { + "gpio150", +}; + +static const char *const nav_gpio1_groups[] = { + "gpio151", +}; + +static const char *const nav_gpio2_groups[] = { + "gpio148", +}; + +static const char *const nav_gpio3_groups[] = { + "gpio150", +}; + +static const char *const pcie0_clk_req_n_groups[] = { + "gpio103", +}; + +static const char *const phase_flag_groups[] = { + "gpio117", "gpio118", "gpio119", "gpio123", "gpio124", "gpio125", + "gpio169", "gpio170", "gpio171", "gpio172", "gpio173", "gpio175", + "gpio176", "gpio179", "gpio180", "gpio181", "gpio184", "gpio185", + "gpio192", "gpio196", "gpio197", "gpio198", "gpio199", "gpio204", + "gpio206", "gpio207", "gpio208", "gpio210", "gpio211", "gpio214", + "gpio215", "gpio216", +}; + +static const char *const pll_bist_sync_groups[] = { + "gpio104", +}; + +static const char *const pll_clk_aux_groups[] = { + "gpio94", +}; + +static const char *const prng_rosc0_groups[] = { + "gpio85", +}; + +static const char *const prng_rosc1_groups[] = { + "gpio64", +}; + +static const char *const prng_rosc2_groups[] = { + "gpio65", +}; + +static const char *const prng_rosc3_groups[] = { + "gpio66", +}; + +static const char *const qdss_cti_groups[] = { + "gpio27", "gpio31", "gpio72", "gpio73", "gpio82", "gpio83", + "gpio155", "gpio158", +}; + +static const char *const qdss_gpio_traceclk_groups[] = { + "gpio128", +}; + +static const char *const qdss_gpio_tracectl_groups[] = { + "gpio127", +}; + +static const char *const qdss_gpio_tracedata_groups[] = { + "gpio38", "gpio39", "gpio40", "gpio41", "gpio42", "gpio43", + "gpio62", "gpio63", "gpio68", "gpio69", "gpio126", "gpio129", + "gpio130", "gpio131", "gpio132", "gpio133", +}; + +static const char *const qlink_big_enable_groups[] = { + "gpio156", +}; + +static const char *const qlink_big_request_groups[] = { + "gpio155", +}; + +static const char *const qlink_little_enable_groups[] = { + "gpio153", +}; + +static const char *const qlink_little_request_groups[] = { + "gpio152", +}; + +static const char *const qlink_wmss_groups[] = { + "gpio154", +}; + +static const char *const qspi0_groups[] = { + "gpio80", +}; + +static const char *const qspi1_groups[] = { + "gpio147", +}; + +static const char *const qspi2_groups[] = { + "gpio81", +}; + +static const char *const qspi3_groups[] = { + "gpio82", +}; + +static const char *const qspi_clk_groups[] = { + "gpio83", +}; + +static const char *const qspi_cs_groups[] = { + "gpio146", "gpio148", +}; + +static const char *const qup1_se0_groups[] = { + "gpio80", "gpio81", "gpio82", "gpio83", +}; + +static const char *const qup1_se1_groups[] = { + "gpio74", "gpio75", "gpio76", "gpio77", +}; + +static const char *const qup1_se2_groups[] = { + "gpio40", "gpio41", "gpio42", "gpio43", "gpio130", "gpio131", "gpio132", +}; + +static const char *const qup1_se3_groups[] = { + "gpio44", "gpio45", "gpio46", "gpio47", +}; + +static const char *const qup1_se4_groups[] = { + "gpio36", "gpio37", "gpio38", "gpio39", +}; + +static const char *const qup1_se5_groups[] = { + "gpio52", "gpio53", "gpio54", "gpio55", +}; + +static const char *const qup1_se6_groups[] = { + "gpio56", "gpio57", "gpio58", "gpio59", +}; + +static const char *const qup1_se7_groups[] = { + "gpio60", "gpio61", "gpio62", "gpio63", +}; + +static const char *const qup2_se0_groups[] = { + "gpio0", "gpio1", "gpio2", "gpio3", +}; + +static const char *const qup2_se1_groups[] = { + "gpio4", "gpio5", "gpio6", "gpio7", +}; + +static const char *const qup2_se2_groups[] = { + "gpio117", "gpio118", "gpio119", "gpio120", +}; + +static const char *const qup2_se3_groups[] = { + "gpio122", "gpio123", "gpio124", "gpio125", +}; + +static const char *const qup2_se4_groups[] = { + "gpio208", "gpio209", +}; + +static const char *const qup3_se0_groups[] = { + "gpio64", "gpio65", +}; + +static const char *const qup3_se1_groups[] = { + "gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio15", +}; + +static const char *const qup3_se2_groups[] = { + "gpio12", "gpio13", "gpio14", "gpio15", +}; + +static const char *const qup3_se3_groups[] = { + "gpio16", "gpio17", "gpio18", "gpio19", +}; + +static const char *const qup3_se4_groups[] = { + "gpio20", "gpio21", "gpio22", "gpio23", +}; + +static const char *const qup3_se5_groups[] = { + "gpio24", "gpio25", "gpio26", "gpio27", +}; + +static const char *const qup4_se0_groups[] = { + "gpio48", "gpio49", "gpio50", "gpio51", +}; + +static const char *const qup4_se1_groups[] = { + "gpio28", "gpio29", "gpio30", "gpio31", +}; + +static const char *const qup4_se2_groups[] = { + "gpio32", "gpio33", "gpio34", "gpio35", +}; + +static const char *const qup4_se3_groups[] = { + "gpio84", "gpio121", +}; + +static const char *const qup4_se4_groups[] = { + "gpio161", "gpio162", +}; + +static const char *const sd_write_protect_groups[] = { + "gpio85", +}; + +static const char *const sdc40_groups[] = { + "gpio80", +}; + +static const char *const sdc41_groups[] = { + "gpio147", +}; + +static const char *const sdc42_groups[] = { + "gpio81", +}; + +static const char *const sdc43_groups[] = { + "gpio82", +}; + +static const char *const sdc4_clk_groups[] = { + "gpio83", +}; + +static const char *const sdc4_cmd_groups[] = { + "gpio148", +}; + +static const char *const sys_throttle_groups[] = { + "gpio99", +}; + +static const char *const tb_trig_sdc2_groups[] = { + "gpio88", +}; + +static const char *const tb_trig_sdc4_groups[] = { + "gpio146", +}; + +static const char *const tmess_prng0_groups[] = { + "gpio85", +}; + +static const char *const tmess_prng1_groups[] = { + "gpio64", +}; + +static const char *const tmess_prng2_groups[] = { + "gpio65", +}; + +static const char *const tmess_prng3_groups[] = { + "gpio66", +}; + +static const char *const tsense_pwm1_groups[] = { + "gpio87", +}; + +static const char *const tsense_pwm2_groups[] = { + "gpio10", +}; + +static const char *const tsense_pwm3_groups[] = { + "gpio97", +}; + +static const char *const tsense_pwm4_groups[] = { + "gpio99", +}; + +static const char *const tsense_pwm5_groups[] = { + "gpio105", +}; + +static const char *const tsense_pwm6_groups[] = { + "gpio106", +}; + +static const char *const tsense_pwm7_groups[] = { + "gpio159", +}; + +static const char *const uim0_clk_groups[] = { + "gpio127", +}; + +static const char *const uim0_data_groups[] = { + "gpio126", +}; + +static const char *const uim0_present_groups[] = { + "gpio129", +}; + +static const char *const uim0_reset_groups[] = { + "gpio128", +}; + +static const char *const uim1_clk_groups[] = { + "gpio37", "gpio55", "gpio71", "gpio131", +}; + +static const char *const uim1_data_groups[] = { + "gpio36", "gpio54", "gpio70", "gpio130", +}; + +static const char *const uim1_present_groups[] = { + "gpio133", +}; + +static const char *const uim1_reset_groups[] = { + "gpio39", "gpio56", "gpio72", "gpio132", +}; + +static const char *const usb0_hs_groups[] = { + "gpio79", +}; + +static const char *const usb_phy_groups[] = { + "gpio59", "gpio60", +}; + +static const char *const vfr_0_groups[] = { + "gpio146", +}; + +static const char *const vfr_1_groups[] = { + "gpio151", +}; + +static const char *const vsense_trigger_mirnat_groups[] = { + "gpio59", +}; + +static const char *const wcn_sw_groups[] = { + "gpio19", +}; + +static const char *const wcn_sw_ctrl_groups[] = { + "gpio18", +}; + +static const struct pinfunction kaanapali_functions[] = { + MSM_GPIO_PIN_FUNCTION(gpio), + MSM_PIN_FUNCTION(aoss_cti), + MSM_PIN_FUNCTION(atest_char), + MSM_PIN_FUNCTION(atest_usb), + MSM_PIN_FUNCTION(audio_ext_mclk0), + MSM_PIN_FUNCTION(audio_ext_mclk1), + MSM_PIN_FUNCTION(audio_ref_clk), + MSM_PIN_FUNCTION(cam_asc_mclk2), + MSM_PIN_FUNCTION(cam_asc_mclk4), + MSM_PIN_FUNCTION(cam_mclk), + MSM_PIN_FUNCTION(cci_async_in), + MSM_PIN_FUNCTION(cci_i2c_scl), + MSM_PIN_FUNCTION(cci_i2c_sda), + MSM_PIN_FUNCTION(cci_timer), + MSM_PIN_FUNCTION(cmu_rng), + MSM_PIN_FUNCTION(coex_uart1_rx), + MSM_PIN_FUNCTION(coex_uart1_tx), + MSM_PIN_FUNCTION(coex_uart2_rx), + MSM_PIN_FUNCTION(coex_uart2_tx), + MSM_PIN_FUNCTION(dbg_out_clk), + MSM_PIN_FUNCTION(ddr_bist_complete), + MSM_PIN_FUNCTION(ddr_bist_fail), + MSM_PIN_FUNCTION(ddr_bist_start), + MSM_PIN_FUNCTION(ddr_bist_stop), + MSM_PIN_FUNCTION(ddr_pxi0), + MSM_PIN_FUNCTION(ddr_pxi1), + MSM_PIN_FUNCTION(ddr_pxi2), + MSM_PIN_FUNCTION(ddr_pxi3), + MSM_PIN_FUNCTION(dp_hot), + MSM_PIN_FUNCTION(egpio), + MSM_PIN_FUNCTION(gcc_gp1), + MSM_PIN_FUNCTION(gcc_gp2), + MSM_PIN_FUNCTION(gcc_gp3), + MSM_PIN_FUNCTION(gnss_adc0), + MSM_PIN_FUNCTION(gnss_adc1), + MSM_PIN_FUNCTION(i2chub0_se0), + MSM_PIN_FUNCTION(i2chub0_se1), + MSM_PIN_FUNCTION(i2chub0_se2), + MSM_PIN_FUNCTION(i2chub0_se3), + MSM_PIN_FUNCTION(i2chub0_se4), + MSM_PIN_FUNCTION(i2s0_data0), + MSM_PIN_FUNCTION(i2s0_data1), + MSM_PIN_FUNCTION(i2s0_sck), + MSM_PIN_FUNCTION(i2s0_ws), + MSM_PIN_FUNCTION(i2s1_data0), + MSM_PIN_FUNCTION(i2s1_data1), + MSM_PIN_FUNCTION(i2s1_sck), + MSM_PIN_FUNCTION(i2s1_ws), + MSM_PIN_FUNCTION(ibi_i3c), + MSM_PIN_FUNCTION(jitter_bist), + MSM_PIN_FUNCTION(mdp_esync0_out), + MSM_PIN_FUNCTION(mdp_esync1_out), + MSM_PIN_FUNCTION(mdp_vsync), + MSM_PIN_FUNCTION(mdp_vsync0_out), + MSM_PIN_FUNCTION(mdp_vsync1_out), + MSM_PIN_FUNCTION(mdp_vsync2_out), + MSM_PIN_FUNCTION(mdp_vsync3_out), + MSM_PIN_FUNCTION(mdp_vsync5_out), + MSM_PIN_FUNCTION(mdp_vsync_e), + MSM_PIN_FUNCTION(nav_gpio0), + MSM_PIN_FUNCTION(nav_gpio1), + MSM_PIN_FUNCTION(nav_gpio2), + MSM_PIN_FUNCTION(nav_gpio3), + MSM_PIN_FUNCTION(pcie0_clk_req_n), + MSM_PIN_FUNCTION(phase_flag), + MSM_PIN_FUNCTION(pll_bist_sync), + MSM_PIN_FUNCTION(pll_clk_aux), + MSM_PIN_FUNCTION(prng_rosc0), + MSM_PIN_FUNCTION(prng_rosc1), + MSM_PIN_FUNCTION(prng_rosc2), + MSM_PIN_FUNCTION(prng_rosc3), + MSM_PIN_FUNCTION(qdss_cti), + MSM_PIN_FUNCTION(qdss_gpio_traceclk), + MSM_PIN_FUNCTION(qdss_gpio_tracectl), + MSM_PIN_FUNCTION(qdss_gpio_tracedata), + MSM_PIN_FUNCTION(qlink_big_enable), + MSM_PIN_FUNCTION(qlink_big_request), + MSM_PIN_FUNCTION(qlink_little_enable), + MSM_PIN_FUNCTION(qlink_little_request), + MSM_PIN_FUNCTION(qlink_wmss), + MSM_PIN_FUNCTION(qspi0), + MSM_PIN_FUNCTION(qspi1), + MSM_PIN_FUNCTION(qspi2), + MSM_PIN_FUNCTION(qspi3), + MSM_PIN_FUNCTION(qspi_clk), + MSM_PIN_FUNCTION(qspi_cs), + MSM_PIN_FUNCTION(qup1_se0), + MSM_PIN_FUNCTION(qup1_se1), + MSM_PIN_FUNCTION(qup1_se2), + MSM_PIN_FUNCTION(qup1_se3), + MSM_PIN_FUNCTION(qup1_se4), + MSM_PIN_FUNCTION(qup1_se5), + MSM_PIN_FUNCTION(qup1_se6), + MSM_PIN_FUNCTION(qup1_se7), + MSM_PIN_FUNCTION(qup2_se0), + MSM_PIN_FUNCTION(qup2_se1), + MSM_PIN_FUNCTION(qup2_se2), + MSM_PIN_FUNCTION(qup2_se3), + MSM_PIN_FUNCTION(qup2_se4), + MSM_PIN_FUNCTION(qup3_se0), + MSM_PIN_FUNCTION(qup3_se1), + MSM_PIN_FUNCTION(qup3_se2), + MSM_PIN_FUNCTION(qup3_se3), + MSM_PIN_FUNCTION(qup3_se4), + MSM_PIN_FUNCTION(qup3_se5), + MSM_PIN_FUNCTION(qup4_se0), + MSM_PIN_FUNCTION(qup4_se1), + MSM_PIN_FUNCTION(qup4_se2), + MSM_PIN_FUNCTION(qup4_se3), + MSM_PIN_FUNCTION(qup4_se4), + MSM_PIN_FUNCTION(sd_write_protect), + MSM_PIN_FUNCTION(sdc40), + MSM_PIN_FUNCTION(sdc41), + MSM_PIN_FUNCTION(sdc42), + MSM_PIN_FUNCTION(sdc43), + MSM_PIN_FUNCTION(sdc4_clk), + MSM_PIN_FUNCTION(sdc4_cmd), + MSM_PIN_FUNCTION(sys_throttle), + MSM_PIN_FUNCTION(tb_trig_sdc2), + MSM_PIN_FUNCTION(tb_trig_sdc4), + MSM_PIN_FUNCTION(tmess_prng0), + MSM_PIN_FUNCTION(tmess_prng1), + MSM_PIN_FUNCTION(tmess_prng2), + MSM_PIN_FUNCTION(tmess_prng3), + MSM_PIN_FUNCTION(tsense_pwm1), + MSM_PIN_FUNCTION(tsense_pwm2), + MSM_PIN_FUNCTION(tsense_pwm3), + MSM_PIN_FUNCTION(tsense_pwm4), + MSM_PIN_FUNCTION(tsense_pwm5), + MSM_PIN_FUNCTION(tsense_pwm6), + MSM_PIN_FUNCTION(tsense_pwm7), + MSM_PIN_FUNCTION(uim0_clk), + MSM_PIN_FUNCTION(uim0_data), + MSM_PIN_FUNCTION(uim0_present), + MSM_PIN_FUNCTION(uim0_reset), + MSM_PIN_FUNCTION(uim1_clk), + MSM_PIN_FUNCTION(uim1_data), + MSM_PIN_FUNCTION(uim1_present), + MSM_PIN_FUNCTION(uim1_reset), + MSM_PIN_FUNCTION(usb0_hs), + MSM_PIN_FUNCTION(usb_phy), + MSM_PIN_FUNCTION(vfr_0), + MSM_PIN_FUNCTION(vfr_1), + MSM_PIN_FUNCTION(vsense_trigger_mirnat), + MSM_PIN_FUNCTION(wcn_sw), + MSM_PIN_FUNCTION(wcn_sw_ctrl), +}; + +/* Every pin is maintained as a single group, and missing or non-existing pin + * would be maintained as dummy group to synchronize pin group index with + * pin descriptor registered with pinctrl core. + * Clients would not be able to request these dummy pin groups. + */ +static const struct msm_pingroup kaanapali_groups[] = { + [0] = PINGROUP(0, qup2_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [1] = PINGROUP(1, qup2_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [2] = PINGROUP(2, qup2_se0, _, _, _, _, _, _, _, _, _, egpio), + [3] = PINGROUP(3, qup2_se0, _, _, _, _, _, _, _, _, _, egpio), + [4] = PINGROUP(4, qup2_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [5] = PINGROUP(5, qup2_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [6] = PINGROUP(6, qup2_se1, _, _, _, _, _, _, _, _, _, egpio), + [7] = PINGROUP(7, qup2_se1, _, _, _, _, _, _, _, _, _, egpio), + [8] = PINGROUP(8, qup3_se1, ibi_i3c, _, _, _, _, _, _, _, _, _), + [9] = PINGROUP(9, qup3_se1, ibi_i3c, _, _, _, _, _, _, _, _, _), + [10] = PINGROUP(10, qup3_se1, cci_async_in, _, tsense_pwm2, _, _, _, _, _, _, _), + [11] = PINGROUP(11, qup3_se1, cci_async_in, _, _, _, _, _, _, _, _, _), + [12] = PINGROUP(12, qup3_se2, ibi_i3c, qup3_se1, _, _, _, _, _, _, _, _), + [13] = PINGROUP(13, qup3_se2, ibi_i3c, qup3_se1, _, _, _, _, _, _, _, _), + [14] = PINGROUP(14, qup3_se2, _, _, _, _, _, _, _, _, _, _), + [15] = PINGROUP(15, qup3_se2, cci_async_in, qup3_se1, _, _, _, _, _, _, _, _), + [16] = PINGROUP(16, qup3_se3, _, _, _, _, _, _, _, _, _, _), + [17] = PINGROUP(17, qup3_se3, _, _, _, _, _, _, _, _, _, _), + [18] = PINGROUP(18, wcn_sw_ctrl, qup3_se3, _, _, _, _, _, _, _, _, _), + [19] = PINGROUP(19, wcn_sw, qup3_se3, _, _, _, _, _, _, _, _, _), + [20] = PINGROUP(20, qup3_se4, _, _, _, _, _, _, _, _, _, _), + [21] = PINGROUP(21, qup3_se4, _, _, _, _, _, _, _, _, _, _), + [22] = PINGROUP(22, qup3_se4, _, _, _, _, _, _, _, _, _, _), + [23] = PINGROUP(23, qup3_se4, _, _, _, _, _, _, _, _, _, _), + [24] = PINGROUP(24, qup3_se5, _, _, _, _, _, _, _, _, _, _), + [25] = PINGROUP(25, qup3_se5, _, _, _, _, _, _, _, _, _, _), + [26] = PINGROUP(26, qup3_se5, _, _, _, _, _, _, _, _, _, _), + [27] = PINGROUP(27, qup3_se5, qdss_cti, _, _, _, _, _, _, _, _, _), + [28] = PINGROUP(28, qup4_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [29] = PINGROUP(29, qup4_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [30] = PINGROUP(30, qup4_se1, _, _, _, _, _, _, _, _, _, _), + [31] = PINGROUP(31, qup4_se1, qdss_cti, _, _, _, _, _, _, _, _, _), + [32] = PINGROUP(32, qup4_se2, ibi_i3c, _, _, _, _, _, _, _, _, _), + [33] = PINGROUP(33, qup4_se2, ibi_i3c, _, _, _, _, _, _, _, _, _), + [34] = PINGROUP(34, qup4_se2, _, _, _, _, _, _, _, _, _, _), + [35] = PINGROUP(35, qup4_se2, _, _, _, _, _, _, _, _, _, _), + [36] = PINGROUP(36, qup1_se4, uim1_data, ibi_i3c, _, _, _, _, _, _, _, _), + [37] = PINGROUP(37, qup1_se4, uim1_clk, ibi_i3c, _, _, _, _, _, _, _, _), + [38] = PINGROUP(38, qup1_se4, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _), + [39] = PINGROUP(39, qup1_se4, uim1_reset, qdss_gpio_tracedata, _, _, _, _, _, _, _, _), + [40] = PINGROUP(40, qup1_se2, cmu_rng, ddr_bist_fail, _, qdss_gpio_tracedata, gnss_adc0, + _, _, _, _, _), + [41] = PINGROUP(41, qup1_se2, cmu_rng, ddr_bist_start, _, qdss_gpio_tracedata, gnss_adc0, + _, _, _, _, _), + [42] = PINGROUP(42, qup1_se2, cmu_rng, dbg_out_clk, qdss_gpio_tracedata, gnss_adc1, _, _, + _, _, _, _), + [43] = PINGROUP(43, qup1_se2, cmu_rng, _, qdss_gpio_tracedata, ddr_pxi2, _, _, _, _, _, _), + [44] = PINGROUP(44, qup1_se3, ddr_bist_complete, ddr_pxi1, _, _, _, _, _, _, _, _), + [45] = PINGROUP(45, qup1_se3, ddr_bist_stop, ddr_pxi1, _, _, _, _, _, _, _, _), + [46] = PINGROUP(46, qup1_se3, ddr_pxi3, _, _, _, _, _, _, _, _, _), + [47] = PINGROUP(47, qup1_se3, dp_hot, _, _, _, _, _, _, _, _, _), + [48] = PINGROUP(48, qup4_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [49] = PINGROUP(49, qup4_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio), + [50] = PINGROUP(50, qup4_se0, _, _, _, _, _, _, _, _, _, egpio), + [51] = PINGROUP(51, qup4_se0, _, _, _, _, _, _, _, _, _, egpio), + [52] = PINGROUP(52, qup1_se5, ddr_pxi2, _, _, _, _, _, _, _, _, _), + [53] = PINGROUP(53, qup1_se5, _, ddr_pxi3, _, _, _, _, _, _, _, _), + [54] = PINGROUP(54, qup1_se5, uim1_data, ddr_pxi0, _, _, _, _, _, _, _, _), + [55] = PINGROUP(55, qup1_se5, uim1_clk, ddr_pxi0, _, _, _, _, _, _, _, _), + [56] = PINGROUP(56, qup1_se6, uim1_reset, _, _, _, _, _, _, _, _, _), + [57] = PINGROUP(57, qup1_se6, _, _, _, _, _, _, _, _, _, _), + [58] = PINGROUP(58, qup1_se6, _, _, _, _, _, _, _, _, _, _), + [59] = PINGROUP(59, qup1_se6, usb_phy, vsense_trigger_mirnat, _, _, _, _, _, _, _, _), + [60] = PINGROUP(60, qup1_se7, usb_phy, ibi_i3c, _, _, _, _, _, _, _, _), + [61] = PINGROUP(61, qup1_se7, ibi_i3c, _, _, _, _, _, _, _, _, _), + [62] = PINGROUP(62, qup1_se7, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _), + [63] = PINGROUP(63, qup1_se7, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _), + [64] = PINGROUP(64, qup3_se0, _, prng_rosc1, tmess_prng1, _, _, _, _, _, _, _), + [65] = PINGROUP(65, qup3_se0, _, prng_rosc2, tmess_prng2, _, _, _, _, _, _, _), + [66] = PINGROUP(66, i2chub0_se0, prng_rosc3, tmess_prng3, _, _, _, _, _, _, _, _), + [67] = PINGROUP(67, i2chub0_se0, _, _, _, _, _, _, _, _, _, _), + [68] = PINGROUP(68, i2chub0_se2, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _), + [69] = PINGROUP(69, i2chub0_se2, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _), + [70] = PINGROUP(70, i2chub0_se3, uim1_data, _, atest_usb, _, _, _, _, _, _, _), + [71] = PINGROUP(71, i2chub0_se3, uim1_clk, _, atest_usb, _, _, _, _, _, _, _), + [72] = PINGROUP(72, i2chub0_se4, uim1_reset, qdss_cti, _, atest_usb, _, _, _, _, _, _), + [73] = PINGROUP(73, i2chub0_se4, qdss_cti, jitter_bist, atest_usb, _, _, _, _, _, _, _), + [74] = PINGROUP(74, qup1_se1, aoss_cti, _, _, _, _, _, _, _, _, _), + [75] = PINGROUP(75, qup1_se1, aoss_cti, _, _, _, _, _, _, _, _, _), + [76] = PINGROUP(76, qup1_se1, aoss_cti, _, _, _, _, _, _, _, _, _), + [77] = PINGROUP(77, qup1_se1, aoss_cti, gnss_adc1, _, _, _, _, _, _, _, _), + [78] = PINGROUP(78, i2chub0_se1, _, _, _, _, _, _, _, _, _, _), + [79] = PINGROUP(79, i2chub0_se1, usb0_hs, _, _, _, _, _, _, _, _, _), + [80] = PINGROUP(80, qup1_se0, sdc40, qspi0, _, _, _, _, _, _, _, _), + [81] = PINGROUP(81, qup1_se0, sdc42, qspi2, _, _, _, _, _, _, _, _), + [82] = PINGROUP(82, qup1_se0, sdc43, qdss_cti, qspi3, _, _, _, _, _, _, _), + [83] = PINGROUP(83, qup1_se0, sdc4_clk, qdss_cti, qspi_clk, _, _, _, _, _, _, _), + [84] = PINGROUP(84, qup4_se3, _, _, _, _, _, _, _, _, _, _), + [85] = PINGROUP(85, sd_write_protect, prng_rosc0, tmess_prng0, _, _, _, _, _, _, _, _), + [86] = PINGROUP(86, mdp_vsync, mdp_vsync0_out, mdp_vsync1_out, gcc_gp2, _, _, _, _, _, _, + _), + [87] = PINGROUP(87, mdp_vsync, mdp_vsync2_out, mdp_vsync3_out, mdp_vsync5_out, gcc_gp3, _, + tsense_pwm1, _, _, _, _), + [88] = PINGROUP(88, mdp_vsync_e, mdp_esync0_out, tb_trig_sdc2, _, _, _, _, _, _, _, _), + [89] = PINGROUP(89, cam_mclk, _, _, _, _, _, _, _, _, _, _), + [90] = PINGROUP(90, cam_mclk, _, _, _, _, _, _, _, _, _, _), + [91] = PINGROUP(91, cam_asc_mclk2, _, _, _, _, _, _, _, _, _, _), + [92] = PINGROUP(92, cam_mclk, _, _, _, _, _, _, _, _, _, _), + [93] = PINGROUP(93, cam_asc_mclk4, _, _, _, _, _, _, _, _, _, _), + [94] = PINGROUP(94, cam_mclk, pll_clk_aux, _, _, _, _, _, _, _, _, _), + [95] = PINGROUP(95, cam_mclk, _, _, _, _, _, _, _, _, _, _), + [96] = PINGROUP(96, cam_mclk, _, _, _, _, _, _, _, _, _, _), + [97] = PINGROUP(97, mdp_vsync, tsense_pwm3, _, _, _, _, _, _, _, _, _), + [98] = PINGROUP(98, mdp_vsync, _, _, _, _, _, _, _, _, _, _), + [99] = PINGROUP(99, sys_throttle, tsense_pwm4, _, _, _, _, _, _, _, _, _), + [100] = PINGROUP(100, mdp_esync1_out, _, _, _, _, _, _, _, _, _, _), + [101] = PINGROUP(101, _, _, _, _, _, _, _, _, _, _, _), + [102] = PINGROUP(102, _, _, _, _, _, _, _, _, _, _, _), + [103] = PINGROUP(103, pcie0_clk_req_n, _, _, _, _, _, _, _, _, _, _), + [104] = PINGROUP(104, pll_bist_sync, _, _, _, _, _, _, _, _, _, _), + [105] = PINGROUP(105, cci_timer, tsense_pwm5, _, _, _, _, _, _, _, _, _), + [106] = PINGROUP(106, cci_timer, tsense_pwm6, _, _, _, _, _, _, _, _, _), + [107] = PINGROUP(107, cci_timer, cci_i2c_sda, _, _, _, _, _, _, _, _, _), + [108] = PINGROUP(108, cci_i2c_sda, _, _, _, _, _, _, _, _, _, _), + [109] = PINGROUP(109, cci_i2c_sda, _, _, _, _, _, _, _, _, _, _), + [110] = PINGROUP(110, cci_i2c_scl, _, _, _, _, _, _, _, _, _, _), + [111] = PINGROUP(111, cci_i2c_sda, _, _, _, _, _, _, _, _, _, _), + [112] = PINGROUP(112, cci_i2c_scl, _, _, _, _, _, _, _, _, _, _), + [113] = PINGROUP(113, cci_i2c_sda, _, _, _, _, _, _, _, _, _, _), + [114] = PINGROUP(114, cci_i2c_scl, _, _, _, _, _, _, _, _, _, _), + [115] = PINGROUP(115, cci_i2c_sda, _, _, _, _, _, _, _, _, _, _), + [116] = PINGROUP(116, cci_i2c_scl, _, _, _, _, _, _, _, _, _, _), + [117] = PINGROUP(117, i2s1_sck, qup2_se2, phase_flag, _, _, _, _, _, _, _, _), + [118] = PINGROUP(118, i2s1_data0, qup2_se2, phase_flag, _, _, _, _, _, _, _, _), + [119] = PINGROUP(119, i2s1_ws, qup2_se2, phase_flag, _, _, _, _, _, _, _, _), + [120] = PINGROUP(120, i2s1_data1, qup2_se2, audio_ext_mclk1, audio_ref_clk, _, _, _, _, _, + _, _), + [121] = PINGROUP(121, audio_ext_mclk0, qup4_se3, _, _, _, _, _, _, _, _, _), + [122] = PINGROUP(122, i2s0_sck, qup2_se3, _, _, _, _, _, _, _, _, _), + [123] = PINGROUP(123, i2s0_data0, qup2_se3, _, phase_flag, _, _, _, _, _, _, _), + [124] = PINGROUP(124, i2s0_data1, qup2_se3, _, phase_flag, _, _, _, _, _, _, _), + [125] = PINGROUP(125, i2s0_ws, qup2_se3, phase_flag, _, _, _, _, _, _, _, _), + [126] = PINGROUP(126, uim0_data, qdss_gpio_tracedata, atest_char, _, _, _, _, _, _, _, _), + [127] = PINGROUP(127, uim0_clk, qdss_gpio_tracectl, atest_char, _, _, _, _, _, _, _, _), + [128] = PINGROUP(128, uim0_reset, qdss_gpio_traceclk, atest_char, _, _, _, _, _, _, _, _), + [129] = PINGROUP(129, uim0_present, qdss_gpio_tracedata, atest_usb, atest_char, _, _, _, _, + _, _, _), + [130] = PINGROUP(130, uim1_data, qup1_se2, gcc_gp1, qdss_gpio_tracedata, _, _, _, _, _, _, + _), + [131] = PINGROUP(131, uim1_clk, qup1_se2, gcc_gp2, qdss_gpio_tracedata, _, _, _, _, _, _, + _), + [132] = PINGROUP(132, uim1_reset, qup1_se2, gcc_gp3, qdss_gpio_tracedata, _, _, _, _, _, _, + _), + [133] = PINGROUP(133, uim1_present, qdss_gpio_tracedata, atest_char, _, _, _, _, _, _, _, + _), + [134] = PINGROUP(134, _, _, _, _, _, _, _, _, _, _, _), + [135] = PINGROUP(135, _, _, _, _, _, _, _, _, _, _, _), + [136] = PINGROUP(136, _, _, _, _, _, _, _, _, _, _, _), + [137] = PINGROUP(137, _, _, _, _, _, _, _, _, _, _, _), + [138] = PINGROUP(138, _, _, _, _, _, _, _, _, _, _, _), + [139] = PINGROUP(139, _, _, _, _, _, _, _, _, _, _, _), + [140] = PINGROUP(140, _, _, _, _, _, _, _, _, _, _, _), + [141] = PINGROUP(141, _, _, _, _, _, _, _, _, _, _, _), + [142] = PINGROUP(142, _, _, _, _, _, _, _, _, _, _, _), + [143] = PINGROUP(143, _, _, _, _, _, _, _, _, _, _, _), + [144] = PINGROUP(144, coex_uart1_rx, cmu_rng, _, _, _, _, _, _, _, _, _), + [145] = PINGROUP(145, coex_uart1_tx, cmu_rng, _, _, _, _, _, _, _, _, _), + [146] = PINGROUP(146, _, vfr_0, coex_uart2_rx, cmu_rng, tb_trig_sdc4, qspi_cs, _, _, _, _, + _), + [147] = PINGROUP(147, _, coex_uart2_tx, cmu_rng, sdc41, qspi1, _, _, _, _, _, _), + [148] = PINGROUP(148, nav_gpio2, _, sdc4_cmd, qspi_cs, _, _, _, _, _, _, _), + [149] = PINGROUP(149, cci_i2c_scl, _, _, _, _, _, _, _, _, _, _), + [150] = PINGROUP(150, nav_gpio0, nav_gpio3, _, _, _, _, _, _, _, _, _), + [151] = PINGROUP(151, nav_gpio1, vfr_1, _, _, _, _, _, _, _, _, _), + [152] = PINGROUP(152, qlink_little_request, _, _, _, _, _, _, _, _, _, _), + [153] = PINGROUP(153, qlink_little_enable, _, _, _, _, _, _, _, _, _, _), + [154] = PINGROUP(154, qlink_wmss, _, _, _, _, _, _, _, _, _, _), + [155] = PINGROUP(155, qlink_big_request, qdss_cti, _, _, _, _, _, _, _, _, _), + [156] = PINGROUP(156, qlink_big_enable, _, _, _, _, _, _, _, _, _, _), + [157] = PINGROUP(157, _, _, _, _, _, _, _, _, _, _, _), + [158] = PINGROUP(158, qdss_cti, gcc_gp1, _, _, _, _, _, _, _, _, _), + [159] = PINGROUP(159, cci_timer, tsense_pwm7, _, _, _, _, _, _, _, _, _), + [160] = PINGROUP(160, cci_timer, cci_i2c_scl, _, _, _, _, _, _, _, _, _), + [161] = PINGROUP(161, qup4_se4, _, _, _, _, _, _, _, _, _, _), + [162] = PINGROUP(162, qup4_se4, _, _, _, _, _, _, _, _, _, _), + [163] = PINGROUP(163, _, _, _, _, _, _, _, _, _, _, egpio), + [164] = PINGROUP(164, _, _, _, _, _, _, _, _, _, _, egpio), + [165] = PINGROUP(165, _, _, _, _, _, _, _, _, _, _, egpio), + [166] = PINGROUP(166, _, _, _, _, _, _, _, _, _, _, egpio), + [167] = PINGROUP(167, _, _, _, _, _, _, _, _, _, _, egpio), + [168] = PINGROUP(168, _, _, _, _, _, _, _, _, _, _, egpio), + [169] = PINGROUP(169, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [170] = PINGROUP(170, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [171] = PINGROUP(171, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [172] = PINGROUP(172, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [173] = PINGROUP(173, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [174] = PINGROUP(174, _, _, _, _, _, _, _, _, _, _, egpio), + [175] = PINGROUP(175, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [176] = PINGROUP(176, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [177] = PINGROUP(177, _, _, _, _, _, _, _, _, _, _, egpio), + [178] = PINGROUP(178, _, _, _, _, _, _, _, _, _, _, egpio), + [179] = PINGROUP(179, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [180] = PINGROUP(180, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [181] = PINGROUP(181, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [182] = PINGROUP(182, _, _, _, _, _, _, _, _, _, _, egpio), + [183] = PINGROUP(183, _, _, _, _, _, _, _, _, _, _, egpio), + [184] = PINGROUP(184, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [185] = PINGROUP(185, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [186] = PINGROUP(186, _, _, _, _, _, _, _, _, _, _, egpio), + [187] = PINGROUP(187, _, _, _, _, _, _, _, _, _, _, egpio), + [188] = PINGROUP(188, _, _, _, _, _, _, _, _, _, _, egpio), + [189] = PINGROUP(189, _, _, _, _, _, _, _, _, _, _, egpio), + [190] = PINGROUP(190, _, _, _, _, _, _, _, _, _, _, egpio), + [191] = PINGROUP(191, _, _, _, _, _, _, _, _, _, _, egpio), + [192] = PINGROUP(192, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [193] = PINGROUP(193, _, _, _, _, _, _, _, _, _, _, egpio), + [194] = PINGROUP(194, _, _, _, _, _, _, _, _, _, _, egpio), + [195] = PINGROUP(195, _, _, _, _, _, _, _, _, _, _, egpio), + [196] = PINGROUP(196, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [197] = PINGROUP(197, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [198] = PINGROUP(198, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [199] = PINGROUP(199, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [200] = PINGROUP(200, _, _, _, _, _, _, _, _, _, _, egpio), + [201] = PINGROUP(201, _, _, _, _, _, _, _, _, _, _, egpio), + [202] = PINGROUP(202, _, _, _, _, _, _, _, _, _, _, egpio), + [203] = PINGROUP(203, _, _, _, _, _, _, _, _, _, _, egpio), + [204] = PINGROUP(204, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [205] = PINGROUP(205, _, _, _, _, _, _, _, _, _, _, egpio), + [206] = PINGROUP(206, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [207] = PINGROUP(207, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [208] = PINGROUP(208, qup2_se4, _, phase_flag, _, _, _, _, _, _, _, egpio), + [209] = PINGROUP(209, qup2_se4, _, _, _, _, _, _, _, _, _, egpio), + [210] = PINGROUP(210, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [211] = PINGROUP(211, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [212] = PINGROUP(212, _, _, _, _, _, _, _, _, _, _, egpio), + [213] = PINGROUP(213, _, _, _, _, _, _, _, _, _, _, egpio), + [214] = PINGROUP(214, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [215] = PINGROUP(215, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [216] = PINGROUP(216, phase_flag, _, _, _, _, _, _, _, _, _, egpio), + [217] = UFS_RESET(ufs_reset, 0xe8004, 0xe9000), + [218] = SDC_QDSD_PINGROUP(sdc2_clk, 0xdd000, 14, 6), + [219] = SDC_QDSD_PINGROUP(sdc2_cmd, 0xdd000, 11, 3), + [220] = SDC_QDSD_PINGROUP(sdc2_data, 0xdd000, 9, 0), +}; + +static const struct msm_gpio_wakeirq_map kaanapali_pdc_map[] = { + { 0, 89 }, { 3, 97 }, { 4, 90 }, { 7, 91 }, { 8, 92 }, { 11, 93 }, + { 12, 101 }, { 15, 115 }, { 17, 125 }, { 18, 127 }, { 19, 96 }, { 23, 99 }, + { 24, 100 }, { 27, 102 }, { 28, 103 }, { 31, 111 }, { 32, 109 }, { 35, 85 }, + { 36, 110 }, { 39, 112 }, { 43, 113 }, { 47, 138 }, { 48, 114 }, { 51, 98 }, + { 55, 88 }, { 57, 120 }, { 59, 121 }, { 60, 122 }, { 63, 108 }, { 64, 94 }, + { 65, 107 }, { 67, 116 }, { 68, 129 }, { 69, 130 }, { 75, 135 }, { 77, 123 }, + { 78, 119 }, { 79, 131 }, { 80, 139 }, { 81, 132 }, { 84, 118 }, { 85, 133 }, + { 86, 140 }, { 87, 141 }, { 88, 142 }, { 95, 143 }, { 96, 144 }, { 97, 117 }, + { 98, 134 }, { 99, 95 }, { 101, 145 }, { 102, 146 }, { 103, 147 }, { 104, 148 }, + { 120, 149 }, { 125, 150 }, { 129, 137 }, { 133, 84 }, { 144, 151 }, { 146, 152 }, + { 151, 153 }, { 152, 154 }, { 155, 106 }, { 158, 104 }, { 162, 126 }, { 164, 155 }, + { 167, 156 }, { 169, 157 }, { 170, 158 }, { 172, 159 }, { 174, 160 }, { 175, 161 }, + { 179, 162 }, { 180, 163 }, { 183, 164 }, { 186, 165 }, { 188, 128 }, { 189, 166 }, + { 190, 105 }, { 191, 167 }, { 194, 168 }, { 195, 169 }, { 196, 170 }, { 197, 171 }, + { 199, 136 }, { 200, 86 }, { 201, 172 }, { 202, 173 }, { 203, 174 }, { 205, 124 }, + { 209, 175 }, { 213, 87 }, { 216, 176 }, +}; + +static const struct msm_pinctrl_soc_data kaanapali_tlmm = { + .pins = kaanapali_pins, + .npins = ARRAY_SIZE(kaanapali_pins), + .functions = kaanapali_functions, + .nfunctions = ARRAY_SIZE(kaanapali_functions), + .groups = kaanapali_groups, + .ngroups = ARRAY_SIZE(kaanapali_groups), + .ngpios = 218, + .wakeirq_map = kaanapali_pdc_map, + .nwakeirq_map = ARRAY_SIZE(kaanapali_pdc_map), + .egpio_func = 11, +}; + +static int kaanapali_tlmm_probe(struct platform_device *pdev) +{ + return msm_pinctrl_probe(pdev, &kaanapali_tlmm); +} + +static const struct of_device_id kaanapali_tlmm_of_match[] = { + { .compatible = "qcom,kaanapali-tlmm",}, + {}, +}; + +static struct platform_driver kaanapali_tlmm_driver = { + .driver = { + .name = "kaanapali-tlmm", + .of_match_table = kaanapali_tlmm_of_match, + }, + .probe = kaanapali_tlmm_probe, +}; + +static int __init kaanapali_tlmm_init(void) +{ + return platform_driver_register(&kaanapali_tlmm_driver); +} +arch_initcall(kaanapali_tlmm_init); + +static void __exit kaanapali_tlmm_exit(void) +{ + platform_driver_unregister(&kaanapali_tlmm_driver); +} +module_exit(kaanapali_tlmm_exit); + +MODULE_DESCRIPTION("QTI Kaanapali TLMM driver"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, kaanapali_tlmm_of_match); diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c index 1c97ec44aa5ff..4275f27343c10 100644 --- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c @@ -14,15 +14,16 @@ #include #include +#include #include #include "../pinctrl-utils.h" #include "pinctrl-lpass-lpi.h" +#include #define MAX_NR_GPIO 32 #define GPIO_FUNC 0 -#define MAX_LPI_NUM_CLKS 2 struct lpi_pinctrl { struct device *dev; @@ -31,7 +32,6 @@ struct lpi_pinctrl { struct pinctrl_desc desc; char __iomem *tlmm_base; char __iomem *slew_base; - struct clk_bulk_data clks[MAX_LPI_NUM_CLKS]; /* Protects from concurrent register updates */ struct mutex lock; DECLARE_BITMAP(ever_gpio, MAX_NR_GPIO); @@ -39,30 +39,52 @@ struct lpi_pinctrl { }; static int lpi_gpio_read(struct lpi_pinctrl *state, unsigned int pin, - unsigned int addr) + unsigned int addr, u32 *val) { u32 pin_offset; + int ret; if (state->data->flags & LPI_FLAG_USE_PREDEFINED_PIN_OFFSET) pin_offset = state->data->groups[pin].pin_offset; else pin_offset = LPI_TLMM_REG_OFFSET * pin; - return ioread32(state->tlmm_base + pin_offset + addr); + ret = pm_runtime_get_sync(state->dev); + if (ret < 0) { + pm_runtime_put_noidle(state->dev); + return ret; + } + + *val = ioread32(state->tlmm_base + pin_offset + addr); + + pm_runtime_mark_last_busy(state->dev); + pm_runtime_put_autosuspend(state->dev); + + return 0; } static int lpi_gpio_write(struct lpi_pinctrl *state, unsigned int pin, unsigned int addr, unsigned int val) { u32 pin_offset; + int ret; if (state->data->flags & LPI_FLAG_USE_PREDEFINED_PIN_OFFSET) pin_offset = state->data->groups[pin].pin_offset; else pin_offset = LPI_TLMM_REG_OFFSET * pin; + ret = pm_runtime_get_sync(state->dev); + if (ret < 0) { + pm_runtime_put_noidle(state->dev); + return ret; + } + iowrite32(val, state->tlmm_base + pin_offset + addr); + pm_runtime_mark_last_busy(state->dev); + pm_runtime_put_autosuspend(state->dev); + return 0; } @@ -107,7 +129,8 @@ static int lpi_gpio_set_mux(struct pinctrl_dev *pctldev, unsigned int function, { struct lpi_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev); const struct lpi_pingroup *g = &pctrl->data->groups[group]; - u32 val; + u32 val, io_val; + int ret; int i, pin = g->pin; for (i = 0; i < g->nfuncs; i++) { @@ -119,7 +142,9 @@ static int lpi_gpio_set_mux(struct pinctrl_dev *pctldev, unsigned int function, return -EINVAL; mutex_lock(&pctrl->lock); - val = lpi_gpio_read(pctrl, pin, LPI_GPIO_CFG_REG); + ret = lpi_gpio_read(pctrl, pin, LPI_GPIO_CFG_REG, &val); + if (ret) + goto out_unlock; /* * If this is the first time muxing to GPIO and the direction is @@ -129,24 +154,28 @@ static int lpi_gpio_set_mux(struct pinctrl_dev *pctldev, unsigned int function, */ if (i == GPIO_FUNC && (val & LPI_GPIO_OE_MASK) && !test_and_set_bit(group, pctrl->ever_gpio)) { - u32 io_val = lpi_gpio_read(pctrl, group, LPI_GPIO_VALUE_REG); + ret = lpi_gpio_read(pctrl, group, LPI_GPIO_VALUE_REG, &io_val); + if (ret) + goto out_unlock; if (io_val & LPI_GPIO_VALUE_IN_MASK) { if (!(io_val & LPI_GPIO_VALUE_OUT_MASK)) - lpi_gpio_write(pctrl, group, LPI_GPIO_VALUE_REG, - io_val | LPI_GPIO_VALUE_OUT_MASK); + ret = lpi_gpio_write(pctrl, group, LPI_GPIO_VALUE_REG, + io_val | LPI_GPIO_VALUE_OUT_MASK); } else { if (io_val & LPI_GPIO_VALUE_OUT_MASK) - lpi_gpio_write(pctrl, group, LPI_GPIO_VALUE_REG, - io_val & ~LPI_GPIO_VALUE_OUT_MASK); + ret = lpi_gpio_write(pctrl, group, LPI_GPIO_VALUE_REG, + io_val & ~LPI_GPIO_VALUE_OUT_MASK); } } u32p_replace_bits(&val, i, LPI_GPIO_FUNCTION_MASK); - lpi_gpio_write(pctrl, pin, LPI_GPIO_CFG_REG, val); + ret = lpi_gpio_write(pctrl, pin, LPI_GPIO_CFG_REG, val); + +out_unlock: mutex_unlock(&pctrl->lock); - return 0; + return ret; } static const struct pinmux_ops lpi_gpio_pinmux_ops = { @@ -165,8 +194,11 @@ static int lpi_config_get(struct pinctrl_dev *pctldev, int is_out; int pull; u32 ctl_reg; + int ret; - ctl_reg = lpi_gpio_read(state, pin, LPI_GPIO_CFG_REG); + ret = lpi_gpio_read(state, pin, LPI_GPIO_CFG_REG, &ctl_reg); + if (ret) + return ret; is_out = ctl_reg & LPI_GPIO_OE_MASK; pull = FIELD_GET(LPI_GPIO_PULL_MASK, ctl_reg); @@ -293,17 +325,22 @@ static int lpi_config_set(struct pinctrl_dev *pctldev, unsigned int group, } mutex_lock(&pctrl->lock); - val = lpi_gpio_read(pctrl, group, LPI_GPIO_CFG_REG); + ret = lpi_gpio_read(pctrl, group, LPI_GPIO_CFG_REG, &val); + if (ret) { + mutex_unlock(&pctrl->lock); + goto out_unlock; + } u32p_replace_bits(&val, pullup, LPI_GPIO_PULL_MASK); u32p_replace_bits(&val, LPI_GPIO_DS_TO_VAL(strength), LPI_GPIO_OUT_STRENGTH_MASK); u32p_replace_bits(&val, output_enabled, LPI_GPIO_OE_MASK); - lpi_gpio_write(pctrl, group, LPI_GPIO_CFG_REG, val); - mutex_unlock(&pctrl->lock); + ret = lpi_gpio_write(pctrl, group, LPI_GPIO_CFG_REG, val); - return 0; +out_unlock: + mutex_unlock(&pctrl->lock); + return ret; } static const struct pinconf_ops lpi_gpio_pinconf_ops = { @@ -312,6 +349,22 @@ static const struct pinconf_ops lpi_gpio_pinconf_ops = { .pin_config_group_set = lpi_config_set, }; +static int lpi_gpio_get_direction(struct gpio_chip *chip, unsigned int pin) +{ + unsigned long config = pinconf_to_config_packed(PIN_CONFIG_LEVEL, 0); + struct lpi_pinctrl *state = gpiochip_get_data(chip); + unsigned long arg; + int ret; + + ret = lpi_config_get(state->ctrl, pin, &config); + if (ret) + return ret; + + arg = pinconf_to_config_argument(config); + + return arg ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + static int lpi_gpio_direction_input(struct gpio_chip *chip, unsigned int pin) { struct lpi_pinctrl *state = gpiochip_get_data(chip); @@ -336,9 +389,13 @@ static int lpi_gpio_direction_output(struct gpio_chip *chip, static int lpi_gpio_get(struct gpio_chip *chip, unsigned int pin) { struct lpi_pinctrl *state = gpiochip_get_data(chip); + u32 val; + int ret; - return lpi_gpio_read(state, pin, LPI_GPIO_VALUE_REG) & - LPI_GPIO_VALUE_IN_MASK; + ret = lpi_gpio_read(state, pin, LPI_GPIO_VALUE_REG, &val); + if (ret) + return ret; + return val & LPI_GPIO_VALUE_IN_MASK; } static int lpi_gpio_set(struct gpio_chip *chip, unsigned int pin, int value) @@ -371,6 +428,7 @@ static void lpi_gpio_dbg_show_one(struct seq_file *s, int drive; int pull; u32 ctl_reg; + int ret; static const char * const pulls[] = { "no pull", @@ -381,7 +439,11 @@ static void lpi_gpio_dbg_show_one(struct seq_file *s, pctldev = pctldev ? : state->ctrl; pindesc = pctldev->desc->pins[offset]; - ctl_reg = lpi_gpio_read(state, offset, LPI_GPIO_CFG_REG); + ret = lpi_gpio_read(state, offset, LPI_GPIO_CFG_REG, &ctl_reg); + if (ret) { + seq_printf(s, " %-8s: ", pindesc.name, ret); + return; + } is_out = ctl_reg & LPI_GPIO_OE_MASK; func = FIELD_GET(LPI_GPIO_FUNCTION_MASK, ctl_reg); @@ -409,6 +471,7 @@ static void lpi_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip) #endif static const struct gpio_chip lpi_gpio_template = { + .get_direction = lpi_gpio_get_direction, .direction_input = lpi_gpio_direction_input, .direction_output = lpi_gpio_direction_output, .get = lpi_gpio_get, @@ -463,9 +526,6 @@ int lpi_pinctrl_probe(struct platform_device *pdev) pctrl->data = data; pctrl->dev = &pdev->dev; - pctrl->clks[0].id = "core"; - pctrl->clks[1].id = "audio"; - pctrl->tlmm_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pctrl->tlmm_base)) return dev_err_probe(dev, PTR_ERR(pctrl->tlmm_base), @@ -478,13 +538,17 @@ int lpi_pinctrl_probe(struct platform_device *pdev) "Slew resource not provided\n"); } - ret = devm_clk_bulk_get_optional(dev, MAX_LPI_NUM_CLKS, pctrl->clks); + ret = devm_pm_clk_create(dev); if (ret) return ret; - ret = clk_bulk_prepare_enable(MAX_LPI_NUM_CLKS, pctrl->clks); - if (ret) - return dev_err_probe(dev, ret, "Can't enable clocks\n"); + ret = of_pm_clk_add_clks(dev); + if (ret < 0) + return ret; + + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); pctrl->desc.pctlops = &lpi_gpio_pinctrl_ops; pctrl->desc.pmxops = &lpi_gpio_pinmux_ops; @@ -498,7 +562,7 @@ int lpi_pinctrl_probe(struct platform_device *pdev) pctrl->chip.base = -1; pctrl->chip.ngpio = data->npins; pctrl->chip.label = dev_name(dev); - pctrl->chip.can_sleep = false; + pctrl->chip.can_sleep = true; mutex_init(&pctrl->lock); @@ -522,20 +586,33 @@ int lpi_pinctrl_probe(struct platform_device *pdev) return 0; err_pinctrl: + pm_runtime_disable(dev); mutex_destroy(&pctrl->lock); - clk_bulk_disable_unprepare(MAX_LPI_NUM_CLKS, pctrl->clks); return ret; } EXPORT_SYMBOL_GPL(lpi_pinctrl_probe); +int lpi_pinctrl_runtime_suspend(struct device *dev) +{ + return pm_clk_suspend(dev); +} +EXPORT_SYMBOL_GPL(lpi_pinctrl_runtime_suspend); + +int lpi_pinctrl_runtime_resume(struct device *dev) +{ + return pm_clk_resume(dev); +} +EXPORT_SYMBOL_GPL(lpi_pinctrl_runtime_resume); + void lpi_pinctrl_remove(struct platform_device *pdev) { struct lpi_pinctrl *pctrl = platform_get_drvdata(pdev); int i; + pm_runtime_disable(pctrl->dev); + mutex_destroy(&pctrl->lock); - clk_bulk_disable_unprepare(MAX_LPI_NUM_CLKS, pctrl->clks); for (i = 0; i < pctrl->data->npins; i++) pinctrl_generic_remove_group(pctrl->ctrl, i); diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h index f483684928613..ae94ef48da8ba 100644 --- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h +++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h @@ -107,5 +107,7 @@ struct lpi_pinctrl_variant_data { int lpi_pinctrl_probe(struct platform_device *pdev); void lpi_pinctrl_remove(struct platform_device *pdev); +int lpi_pinctrl_runtime_suspend(struct device *dev); +int lpi_pinctrl_runtime_resume(struct device *dev); #endif /*__PINCTRL_LPASS_LPI_H__*/ diff --git a/drivers/pinctrl/qcom/pinctrl-qcs615.c b/drivers/pinctrl/qcom/pinctrl-qcs615.c index 4dfa820d4e77c..f1c827ddbfbfa 100644 --- a/drivers/pinctrl/qcom/pinctrl-qcs615.c +++ b/drivers/pinctrl/qcom/pinctrl-qcs615.c @@ -1067,6 +1067,7 @@ static const struct msm_pinctrl_soc_data qcs615_tlmm = { .ntiles = ARRAY_SIZE(qcs615_tiles), .wakeirq_map = qcs615_pdc_map, .nwakeirq_map = ARRAY_SIZE(qcs615_pdc_map), + .wakeirq_dual_edge_errata = true, }; static const struct of_device_id qcs615_tlmm_of_match[] = { diff --git a/drivers/pinctrl/qcom/pinctrl-sc7280-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sc7280-lpass-lpi.c index 1161f0a91a002..2d955643d2f00 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc7280-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sc7280-lpass-lpi.c @@ -131,15 +131,23 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { { .compatible = "qcom,sc7280-lpass-lpi-pinctrl", .data = &sc7280_lpi_data, + }, { + .compatible = "qcom,sm8350-lpass-lpi-pinctrl", + .data = &sc7280_lpi_data, }, { } }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(lpi_pinctrl_runtime_suspend, lpi_pinctrl_runtime_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { .name = "qcom-sc7280-lpass-lpi-pinctrl", .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-sc8280xp-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sc8280xp-lpass-lpi.c index 0e839b6aaaf4b..1a61316c8c473 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc8280xp-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sc8280xp-lpass-lpi.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include "pinctrl-lpass-lpi.h" @@ -173,10 +175,15 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { - .name = "qcom-sc8280xp-lpass-lpi-pinctrl", - .of_match_table = lpi_pinctrl_of_match, + .name = "qcom-sc8280xp-lpass-lpi-pinctrl", + .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-sm4250-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm4250-lpass-lpi.c index c0e178be9cfc3..75bafa62426a2 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm4250-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm4250-lpass-lpi.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "pinctrl-lpass-lpi.h" @@ -221,10 +223,15 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { .name = "qcom-sm4250-lpass-lpi-pinctrl", .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-sm6115-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm6115-lpass-lpi.c index b7d9186861a2f..05435ea6e17a6 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm6115-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm6115-lpass-lpi.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "pinctrl-lpass-lpi.h" @@ -141,10 +143,15 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { .name = "qcom-sm6115-lpass-lpi-pinctrl", .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c index 64494a86490e2..656f22da7dde3 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "pinctrl-lpass-lpi.h" @@ -73,7 +75,7 @@ static const char * const i2s1_ws_groups[] = { "gpio7" }; static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" }; static const char * const wsa_swr_clk_groups[] = { "gpio10" }; static const char * const wsa_swr_data_groups[] = { "gpio11" }; -static const char * const i2s2_data_groups[] = { "gpio12", "gpio12" }; +static const char * const i2s2_data_groups[] = { "gpio12", "gpio13" }; static const struct lpi_pingroup sm8250_groups[] = { LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _), @@ -134,10 +136,15 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { - .name = "qcom-sm8250-lpass-lpi-pinctrl", - .of_match_table = lpi_pinctrl_of_match, + .name = "qcom-sm8250-lpass-lpi-pinctrl", + .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-sm8350-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8350-lpass-lpi.c deleted file mode 100644 index 7b146b4acfdf4..0000000000000 --- a/drivers/pinctrl/qcom/pinctrl-sm8350-lpass-lpi.c +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2020-2023 Linaro Ltd. - */ - -#include -#include -#include - -#include "pinctrl-lpass-lpi.h" - -enum lpass_lpi_functions { - LPI_MUX_dmic1_clk, - LPI_MUX_dmic1_data, - LPI_MUX_dmic2_clk, - LPI_MUX_dmic2_data, - LPI_MUX_dmic3_clk, - LPI_MUX_dmic3_data, - LPI_MUX_i2s1_clk, - LPI_MUX_i2s1_data, - LPI_MUX_i2s1_ws, - LPI_MUX_i2s2_clk, - LPI_MUX_i2s2_data, - LPI_MUX_i2s2_ws, - LPI_MUX_qua_mi2s_data, - LPI_MUX_qua_mi2s_sclk, - LPI_MUX_qua_mi2s_ws, - LPI_MUX_swr_rx_clk, - LPI_MUX_swr_rx_data, - LPI_MUX_swr_tx_clk, - LPI_MUX_swr_tx_data, - LPI_MUX_wsa_swr_clk, - LPI_MUX_wsa_swr_data, - LPI_MUX_gpio, - LPI_MUX__, -}; - -static const struct pinctrl_pin_desc sm8350_lpi_pins[] = { - PINCTRL_PIN(0, "gpio0"), - PINCTRL_PIN(1, "gpio1"), - PINCTRL_PIN(2, "gpio2"), - PINCTRL_PIN(3, "gpio3"), - PINCTRL_PIN(4, "gpio4"), - PINCTRL_PIN(5, "gpio5"), - PINCTRL_PIN(6, "gpio6"), - PINCTRL_PIN(7, "gpio7"), - PINCTRL_PIN(8, "gpio8"), - PINCTRL_PIN(9, "gpio9"), - PINCTRL_PIN(10, "gpio10"), - PINCTRL_PIN(11, "gpio11"), - PINCTRL_PIN(12, "gpio12"), - PINCTRL_PIN(13, "gpio13"), - PINCTRL_PIN(14, "gpio14"), -}; - -static const char * const swr_tx_clk_groups[] = { "gpio0" }; -static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio5", "gpio14" }; -static const char * const swr_rx_clk_groups[] = { "gpio3" }; -static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5" }; -static const char * const dmic1_clk_groups[] = { "gpio6" }; -static const char * const dmic1_data_groups[] = { "gpio7" }; -static const char * const dmic2_clk_groups[] = { "gpio8" }; -static const char * const dmic2_data_groups[] = { "gpio9" }; -static const char * const i2s2_clk_groups[] = { "gpio10" }; -static const char * const i2s2_ws_groups[] = { "gpio11" }; -static const char * const dmic3_clk_groups[] = { "gpio12" }; -static const char * const dmic3_data_groups[] = { "gpio13" }; -static const char * const qua_mi2s_sclk_groups[] = { "gpio0" }; -static const char * const qua_mi2s_ws_groups[] = { "gpio1" }; -static const char * const qua_mi2s_data_groups[] = { "gpio2", "gpio3", "gpio4" }; -static const char * const i2s1_clk_groups[] = { "gpio6" }; -static const char * const i2s1_ws_groups[] = { "gpio7" }; -static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" }; -static const char * const wsa_swr_clk_groups[] = { "gpio10" }; -static const char * const wsa_swr_data_groups[] = { "gpio11" }; -static const char * const i2s2_data_groups[] = { "gpio12", "gpio12" }; - -static const struct lpi_pingroup sm8350_groups[] = { - LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _), - LPI_PINGROUP(1, 2, swr_tx_data, qua_mi2s_ws, _, _), - LPI_PINGROUP(2, 4, swr_tx_data, qua_mi2s_data, _, _), - LPI_PINGROUP(3, 8, swr_rx_clk, qua_mi2s_data, _, _), - LPI_PINGROUP(4, 10, swr_rx_data, qua_mi2s_data, _, _), - LPI_PINGROUP(5, 12, swr_tx_data, swr_rx_data, _, _), - LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _, _), - LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _), - LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _), - LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, _, _), - LPI_PINGROUP(10, 16, i2s2_clk, wsa_swr_clk, _, _), - LPI_PINGROUP(11, 18, i2s2_ws, wsa_swr_data, _, _), - LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s2_data, _, _), - LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s2_data, _, _), - LPI_PINGROUP(14, 6, swr_tx_data, _, _, _), -}; - -static const struct lpi_function sm8350_functions[] = { - LPI_FUNCTION(dmic1_clk), - LPI_FUNCTION(dmic1_data), - LPI_FUNCTION(dmic2_clk), - LPI_FUNCTION(dmic2_data), - LPI_FUNCTION(dmic3_clk), - LPI_FUNCTION(dmic3_data), - LPI_FUNCTION(i2s1_clk), - LPI_FUNCTION(i2s1_data), - LPI_FUNCTION(i2s1_ws), - LPI_FUNCTION(i2s2_clk), - LPI_FUNCTION(i2s2_data), - LPI_FUNCTION(i2s2_ws), - LPI_FUNCTION(qua_mi2s_data), - LPI_FUNCTION(qua_mi2s_sclk), - LPI_FUNCTION(qua_mi2s_ws), - LPI_FUNCTION(swr_rx_clk), - LPI_FUNCTION(swr_rx_data), - LPI_FUNCTION(swr_tx_clk), - LPI_FUNCTION(swr_tx_data), - LPI_FUNCTION(wsa_swr_clk), - LPI_FUNCTION(wsa_swr_data), -}; - -static const struct lpi_pinctrl_variant_data sm8350_lpi_data = { - .pins = sm8350_lpi_pins, - .npins = ARRAY_SIZE(sm8350_lpi_pins), - .groups = sm8350_groups, - .ngroups = ARRAY_SIZE(sm8350_groups), - .functions = sm8350_functions, - .nfunctions = ARRAY_SIZE(sm8350_functions), -}; - -static const struct of_device_id lpi_pinctrl_of_match[] = { - { - .compatible = "qcom,sm8350-lpass-lpi-pinctrl", - .data = &sm8350_lpi_data, - }, - { } -}; -MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); - -static struct platform_driver lpi_pinctrl_driver = { - .driver = { - .name = "qcom-sm8350-lpass-lpi-pinctrl", - .of_match_table = lpi_pinctrl_of_match, - }, - .probe = lpi_pinctrl_probe, - .remove = lpi_pinctrl_remove, -}; -module_platform_driver(lpi_pinctrl_driver); - -MODULE_AUTHOR("Krzysztof Kozlowski "); -MODULE_DESCRIPTION("QTI SM8350 LPI GPIO pin control driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/pinctrl/qcom/pinctrl-sm8450-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8450-lpass-lpi.c index 439f6541622e9..a79f99ec6df9d 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8450-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8450-lpass-lpi.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include "pinctrl-lpass-lpi.h" @@ -202,10 +204,15 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { - .name = "qcom-sm8450-lpass-lpi-pinctrl", - .of_match_table = lpi_pinctrl_of_match, + .name = "qcom-sm8450-lpass-lpi-pinctrl", + .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-sm8550-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8550-lpass-lpi.c index 73065919c8c26..9037ef0020dac 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8550-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8550-lpass-lpi.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include "pinctrl-lpass-lpi.h" @@ -210,10 +212,15 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { - .name = "qcom-sm8550-lpass-lpi-pinctrl", - .of_match_table = lpi_pinctrl_of_match, + .name = "qcom-sm8550-lpass-lpi-pinctrl", + .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c index f9fcedf5a65d7..513ddc99dd378 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include "pinctrl-lpass-lpi.h" @@ -217,10 +219,15 @@ static const struct of_device_id lpi_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); +static const struct dev_pm_ops lpi_pinctrl_pm_ops = { + RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL) +}; + static struct platform_driver lpi_pinctrl_driver = { .driver = { - .name = "qcom-sm8650-lpass-lpi-pinctrl", - .of_match_table = lpi_pinctrl_of_match, + .name = "qcom-sm8650-lpass-lpi-pinctrl", + .of_match_table = lpi_pinctrl_of_match, + .pm = pm_ptr(&lpi_pinctrl_pm_ops), }, .probe = lpi_pinctrl_probe, .remove = lpi_pinctrl_remove, diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c index 485b68cc93f8e..83f940fe30b26 100644 --- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c +++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c @@ -42,6 +42,8 @@ #define PMIC_GPIO_SUBTYPE_GPIO_MV 0x11 #define PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2 0x12 #define PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3 0x13 +#define PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2_CLK 0x14 +#define PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3_CLK 0x15 #define PMIC_MPP_REG_RT_STS 0x10 #define PMIC_MPP_REG_RT_STS_VAL_MASK 0x1 @@ -852,11 +854,13 @@ static int pmic_gpio_populate(struct pmic_gpio_state *state, pad->lv_mv_type = true; break; case PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2: + case PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2_CLK: pad->num_sources = 2; pad->have_buffer = true; pad->lv_mv_type = true; break; case PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3: + case PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3_CLK: pad->num_sources = 3; pad->have_buffer = true; pad->lv_mv_type = true; @@ -1239,7 +1243,11 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pm8998-gpio", .data = (void *) 26 }, { .compatible = "qcom,pma8084-gpio", .data = (void *) 22 }, { .compatible = "qcom,pmc8380-gpio", .data = (void *) 10 }, + { .compatible = "qcom,pmcx0102-gpio", .data = (void *)14 }, { .compatible = "qcom,pmd8028-gpio", .data = (void *) 4 }, + { .compatible = "qcom,pmh0101-gpio", .data = (void *)18 }, + { .compatible = "qcom,pmh0104-gpio", .data = (void *)8 }, + { .compatible = "qcom,pmh0110-gpio", .data = (void *)14 }, { .compatible = "qcom,pmi632-gpio", .data = (void *) 8 }, { .compatible = "qcom,pmi8950-gpio", .data = (void *) 2 }, { .compatible = "qcom,pmi8994-gpio", .data = (void *) 10 }, @@ -1248,6 +1256,7 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pmiv0104-gpio", .data = (void *) 10 }, { .compatible = "qcom,pmk8350-gpio", .data = (void *) 4 }, { .compatible = "qcom,pmk8550-gpio", .data = (void *) 6 }, + { .compatible = "qcom,pmk8850-gpio", .data = (void *)8 }, { .compatible = "qcom,pmm8155au-gpio", .data = (void *) 10 }, { .compatible = "qcom,pmm8654au-gpio", .data = (void *) 12 }, /* pmp8074 has 12 GPIOs with holes on 1 and 12 */ diff --git a/drivers/pinctrl/qcom/pinctrl-x1e80100.c b/drivers/pinctrl/qcom/pinctrl-x1e80100.c index bb36f40b19fa5..04e08680f996b 100644 --- a/drivers/pinctrl/qcom/pinctrl-x1e80100.c +++ b/drivers/pinctrl/qcom/pinctrl-x1e80100.c @@ -1839,9 +1839,7 @@ static const struct msm_pinctrl_soc_data x1e80100_pinctrl = { .ngroups = ARRAY_SIZE(x1e80100_groups), .ngpios = 239, .wakeirq_map = x1e80100_pdc_map, - /* TODO: Enabling PDC currently breaks GPIO interrupts */ - .nwakeirq_map = 0, - /* .nwakeirq_map = ARRAY_SIZE(x1e80100_pdc_map), */ + .nwakeirq_map = ARRAY_SIZE(x1e80100_pdc_map), .egpio_func = 9, }; diff --git a/drivers/pinctrl/renesas/pinctrl-rza1.c b/drivers/pinctrl/renesas/pinctrl-rza1.c index f24e5915cbe4b..2677ee7035fce 100644 --- a/drivers/pinctrl/renesas/pinctrl-rza1.c +++ b/drivers/pinctrl/renesas/pinctrl-rza1.c @@ -590,7 +590,7 @@ static inline unsigned int rza1_get_bit(struct rza1_port *port, { void __iomem *mem = RZA1_ADDR(port->base, reg, port->id); - return ioread16(mem) & BIT(bit); + return !!(ioread16(mem) & BIT(bit)); } /** diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c index f524af6f586f4..c2711b2de374d 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c +++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c @@ -359,7 +359,7 @@ struct rzg2l_pinctrl { spinlock_t bitmap_lock; /* protect tint_slot bitmap */ unsigned int hwirq[RZG2L_TINT_MAX_INTERRUPT]; - spinlock_t lock; /* lock read/write registers */ + raw_spinlock_t lock; /* lock read/write registers */ struct mutex mutex; /* serialize adding groups and functions */ struct rzg2l_pinctrl_pin_settings *settings; @@ -543,7 +543,7 @@ static void rzg2l_pinctrl_set_pfc_mode(struct rzg2l_pinctrl *pctrl, unsigned long flags; u32 reg; - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); /* Set pin to 'Non-use (Hi-Z input protection)' */ reg = readw(pctrl->base + PM(off)); @@ -567,7 +567,7 @@ static void rzg2l_pinctrl_set_pfc_mode(struct rzg2l_pinctrl *pctrl, pctrl->data->pwpr_pfc_lock_unlock(pctrl, true); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); }; static int rzg2l_pinctrl_set_mux(struct pinctrl_dev *pctldev, @@ -882,10 +882,10 @@ static void rzg2l_rmw_pin_config(struct rzg2l_pinctrl *pctrl, u32 offset, addr += 4; } - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); reg = readl(addr) & ~(mask << (bit * 8)); writel(reg | (val << (bit * 8)), addr); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); } static int rzg2l_caps_to_pwr_reg(const struct rzg2l_register_offsets *regs, u32 caps) @@ -1121,7 +1121,7 @@ static int rzg2l_write_oen(struct rzg2l_pinctrl *pctrl, unsigned int _pin, u8 oe if (bit < 0) return -EINVAL; - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); val = readb(pctrl->base + oen_offset); if (oen) val &= ~BIT(bit); @@ -1134,7 +1134,7 @@ static int rzg2l_write_oen(struct rzg2l_pinctrl *pctrl, unsigned int _pin, u8 oe writeb(val, pctrl->base + oen_offset); if (pctrl->data->hwcfg->oen_pwpr_lock) writeb(pwpr & ~PWPR_REGWE_B, pctrl->base + regs->pwpr); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); return 0; } @@ -1687,14 +1687,14 @@ static int rzg2l_gpio_request(struct gpio_chip *chip, unsigned int offset) if (ret) return ret; - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); /* Select GPIO mode in PMC Register */ reg8 = readb(pctrl->base + PMC(off)); reg8 &= ~BIT(bit); pctrl->data->pmc_writeb(pctrl, reg8, PMC(off)); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); return 0; } @@ -1709,7 +1709,7 @@ static void rzg2l_gpio_set_direction(struct rzg2l_pinctrl *pctrl, u32 offset, unsigned long flags; u16 reg16; - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); reg16 = readw(pctrl->base + PM(off)); reg16 &= ~(PM_MASK << (bit * 2)); @@ -1717,7 +1717,7 @@ static void rzg2l_gpio_set_direction(struct rzg2l_pinctrl *pctrl, u32 offset, reg16 |= (output ? PM_OUTPUT : PM_INPUT) << (bit * 2); writew(reg16, pctrl->base + PM(off)); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); } static int rzg2l_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) @@ -1761,7 +1761,7 @@ static int rzg2l_gpio_set(struct gpio_chip *chip, unsigned int offset, unsigned long flags; u8 reg8; - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); reg8 = readb(pctrl->base + P(off)); @@ -1770,7 +1770,7 @@ static int rzg2l_gpio_set(struct gpio_chip *chip, unsigned int offset, else writeb(reg8 & ~BIT(bit), pctrl->base + P(off)); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); return 0; } @@ -2429,14 +2429,13 @@ static int rzg2l_gpio_get_gpioint(unsigned int virq, struct rzg2l_pinctrl *pctrl return gpioint; } -static void rzg2l_gpio_irq_endisable(struct rzg2l_pinctrl *pctrl, - unsigned int hwirq, bool enable) +static void __rzg2l_gpio_irq_endisable(struct rzg2l_pinctrl *pctrl, + unsigned int hwirq, bool enable) { const struct pinctrl_pin_desc *pin_desc = &pctrl->desc.pins[hwirq]; u64 *pin_data = pin_desc->drv_data; u32 off = RZG2L_PIN_CFG_TO_PORT_OFFSET(*pin_data); u8 bit = RZG2L_PIN_ID_TO_PIN(hwirq); - unsigned long flags; void __iomem *addr; addr = pctrl->base + ISEL(off); @@ -2445,12 +2444,20 @@ static void rzg2l_gpio_irq_endisable(struct rzg2l_pinctrl *pctrl, addr += 4; } - spin_lock_irqsave(&pctrl->lock, flags); if (enable) writel(readl(addr) | BIT(bit * 8), addr); else writel(readl(addr) & ~BIT(bit * 8), addr); - spin_unlock_irqrestore(&pctrl->lock, flags); +} + +static void rzg2l_gpio_irq_endisable(struct rzg2l_pinctrl *pctrl, + unsigned int hwirq, bool enable) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&pctrl->lock, flags); + __rzg2l_gpio_irq_endisable(pctrl, hwirq, enable); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); } static void rzg2l_gpio_irq_disable(struct irq_data *d) @@ -2462,15 +2469,25 @@ static void rzg2l_gpio_irq_disable(struct irq_data *d) gpiochip_disable_irq(gc, hwirq); } -static void rzg2l_gpio_irq_enable(struct irq_data *d) +static void __rzg2l_gpio_irq_enable(struct irq_data *d, bool lock) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct rzg2l_pinctrl *pctrl = container_of(gc, struct rzg2l_pinctrl, gpio_chip); unsigned int hwirq = irqd_to_hwirq(d); gpiochip_enable_irq(gc, hwirq); + if (lock) + rzg2l_gpio_irq_endisable(pctrl, hwirq, true); + else + __rzg2l_gpio_irq_endisable(pctrl, hwirq, true); irq_chip_enable_parent(d); } +static void rzg2l_gpio_irq_enable(struct irq_data *d) +{ + __rzg2l_gpio_irq_enable(d, true); +} + static int rzg2l_gpio_irq_set_type(struct irq_data *d, unsigned int type) { return irq_chip_set_type_parent(d, type); @@ -2616,11 +2633,11 @@ static void rzg2l_gpio_irq_restore(struct rzg2l_pinctrl *pctrl) * This has to be atomically executed to protect against a concurrent * interrupt. */ - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); ret = rzg2l_gpio_irq_set_type(data, irqd_get_trigger_type(data)); if (!ret && !irqd_irq_disabled(data)) - rzg2l_gpio_irq_enable(data); - spin_unlock_irqrestore(&pctrl->lock, flags); + __rzg2l_gpio_irq_enable(data, false); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); if (ret) dev_crit(pctrl->dev, "Failed to set IRQ type for virq=%u\n", virq); @@ -2950,7 +2967,7 @@ static int rzg2l_pinctrl_probe(struct platform_device *pdev) "failed to enable GPIO clk\n"); } - spin_lock_init(&pctrl->lock); + raw_spin_lock_init(&pctrl->lock); spin_lock_init(&pctrl->bitmap_lock); mutex_init(&pctrl->mutex); atomic_set(&pctrl->wakeup_path, 0); @@ -2993,7 +3010,11 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen * Now cache the registers or set them in the order suggested by * HW manual (section "Operation for GPIO Function"). */ - RZG2L_PCTRL_REG_ACCESS8(suspend, pctrl->base + PMC(off), cache->pmc[port]); + if (suspend) + RZG2L_PCTRL_REG_ACCESS8(suspend, pctrl->base + PMC(off), cache->pmc[port]); + else + pctrl->data->pmc_writeb(pctrl, cache->pmc[port], PMC(off)); + if (has_iolh) { RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + IOLH(off), cache->iolh[0][port]); @@ -3093,7 +3114,7 @@ static void rzg2l_pinctrl_pm_setup_pfc(struct rzg2l_pinctrl *pctrl) u32 nports = pctrl->data->n_port_pins / RZG2L_PINS_PER_PORT; unsigned long flags; - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); pctrl->data->pwpr_pfc_lock_unlock(pctrl, false); /* Restore port registers. */ @@ -3138,7 +3159,7 @@ static void rzg2l_pinctrl_pm_setup_pfc(struct rzg2l_pinctrl *pctrl) } pctrl->data->pwpr_pfc_lock_unlock(pctrl, true); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); } static int rzg2l_pinctrl_suspend_noirq(struct device *dev) @@ -3187,14 +3208,14 @@ static int rzg2l_pinctrl_resume_noirq(struct device *dev) writeb(cache->qspi, pctrl->base + QSPI); if (pctrl->data->hwcfg->oen_pwpr_lock) { - spin_lock_irqsave(&pctrl->lock, flags); + raw_spin_lock_irqsave(&pctrl->lock, flags); pwpr = readb(pctrl->base + regs->pwpr); writeb(pwpr | PWPR_REGWE_B, pctrl->base + regs->pwpr); } writeb(cache->oen, pctrl->base + pctrl->data->hwcfg->regs.oen); if (pctrl->data->hwcfg->oen_pwpr_lock) { writeb(pwpr & ~PWPR_REGWE_B, pctrl->base + regs->pwpr); - spin_unlock_irqrestore(&pctrl->lock, flags); + raw_spin_unlock_irqrestore(&pctrl->lock, flags); } for (u8 i = 0; i < 2; i++) { if (regs->sd_ch) diff --git a/drivers/pinctrl/renesas/pinctrl-rzt2h.c b/drivers/pinctrl/renesas/pinctrl-rzt2h.c index 3872638f5ebb3..595ad5c83aa00 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzt2h.c +++ b/drivers/pinctrl/renesas/pinctrl-rzt2h.c @@ -51,6 +51,7 @@ #define PFC_MASK GENMASK_ULL(5, 0) #define PFC_PIN_MASK(pin) (PFC_MASK << ((pin) * 8)) +#define PFC_FUNC_INTERRUPT 0 /* * Use 16 lower bits [15:0] for pin identifier @@ -486,6 +487,7 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) struct rzt2h_pinctrl *pctrl = gpiochip_get_data(chip); u8 port = RZT2H_PIN_ID_TO_PORT(offset); u8 bit = RZT2H_PIN_ID_TO_PIN(offset); + u64 reg64; u16 reg; int ret; @@ -493,8 +495,25 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) if (ret) return ret; - if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) + guard(spinlock_irqsave)(&pctrl->lock); + + if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) { + /* + * When a GPIO is being requested as an IRQ, the pinctrl + * framework expects to be able to read the GPIO's direction. + * IRQ function is separate from GPIO, and enabling it takes the + * pin out of GPIO mode. + * At this point, .child_to_parent_hwirq() has already been + * called to enable the IRQ function. + * Default to input direction for IRQ function. + */ + reg64 = rzt2h_pinctrl_readq(pctrl, port, PFC(port)); + reg64 = (reg64 >> (bit * 8)) & PFC_MASK; + if (reg64 == PFC_FUNC_INTERRUPT) + return GPIO_LINE_DIRECTION_IN; + return -EINVAL; + } reg = rzt2h_pinctrl_readw(pctrl, port, PM(port)); reg = (reg >> (bit * 2)) & PM_MASK; @@ -629,6 +648,7 @@ static int rzt2h_gpio_register(struct rzt2h_pinctrl *pctrl) if (ret) return dev_err_probe(dev, ret, "Unable to parse gpio-ranges\n"); + of_node_put(of_args.np); if (of_args.args[0] != 0 || of_args.args[1] != 0 || of_args.args[2] != pctrl->data->n_port_pins) return dev_err_probe(dev, -EINVAL, diff --git a/drivers/pinctrl/stm32/Kconfig b/drivers/pinctrl/stm32/Kconfig index 5f67e1ee66dd9..d6a1715230121 100644 --- a/drivers/pinctrl/stm32/Kconfig +++ b/drivers/pinctrl/stm32/Kconfig @@ -65,6 +65,7 @@ config PINCTRL_STM32_HDP select PINMUX select GENERIC_PINCONF select GPIOLIB + select GPIO_GENERIC help The Hardware Debug Port allows the observation of internal signals. It uses configurable multiplexer to route signals in a dedicated observation register. diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c index 3ebb468de830d..e73feb8ac90d1 100644 --- a/drivers/pinctrl/stm32/pinctrl-stm32.c +++ b/drivers/pinctrl/stm32/pinctrl-stm32.c @@ -1671,7 +1671,7 @@ int stm32_pctl_probe(struct platform_device *pdev) if (hwlock_id == -EPROBE_DEFER) return hwlock_id; } else { - pctl->hwlock = hwspin_lock_request_specific(hwlock_id); + pctl->hwlock = devm_hwspin_lock_request_specific(dev, hwlock_id); } spin_lock_init(&pctl->irqmux_lock); diff --git a/drivers/platform/arm64/Kconfig b/drivers/platform/arm64/Kconfig index 10f905d7d6bfa..e32e01b2a9bdd 100644 --- a/drivers/platform/arm64/Kconfig +++ b/drivers/platform/arm64/Kconfig @@ -90,4 +90,17 @@ config EC_LENOVO_THINKPAD_T14S Say M or Y here to include this support. +config EC_QCOM_HAMOA + tristate "Embedded Controller driver for Qualcomm Hamoa/Glymur reference devices" + depends on ARCH_QCOM || COMPILE_TEST + depends on I2C + depends on THERMAL || THERMAL=n + help + Say M or Y here to enable the Embedded Controller driver for Qualcomm + Snapdragon-based Hamoa/Glymur reference devices. The driver handles fan + control, temperature sensors, access to EC state changes and supports + reporting suspend entry/exit to the EC. + + This driver currently supports Hamoa/Purwa/Glymur reference devices. + endif # ARM64_PLATFORM_DEVICES diff --git a/drivers/platform/arm64/Makefile b/drivers/platform/arm64/Makefile index 60c131cff6a15..7681be4a46e94 100644 --- a/drivers/platform/arm64/Makefile +++ b/drivers/platform/arm64/Makefile @@ -9,3 +9,4 @@ obj-$(CONFIG_EC_ACER_ASPIRE1) += acer-aspire1-ec.o obj-$(CONFIG_EC_HUAWEI_GAOKUN) += huawei-gaokun-ec.o obj-$(CONFIG_EC_LENOVO_YOGA_C630) += lenovo-yoga-c630.o obj-$(CONFIG_EC_LENOVO_THINKPAD_T14S) += lenovo-thinkpad-t14s.o +obj-$(CONFIG_EC_QCOM_HAMOA) += qcom-hamoa-ec.o diff --git a/drivers/platform/arm64/qcom-hamoa-ec.c b/drivers/platform/arm64/qcom-hamoa-ec.c new file mode 100644 index 0000000000000..253f927c9aca7 --- /dev/null +++ b/drivers/platform/arm64/qcom-hamoa-ec.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024 Maya Matuszczyk + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EC_SCI_EVT_READ_CMD 0x05 +#define EC_FW_VERSION_CMD 0x0e +#define EC_MODERN_STANDBY_CMD 0x23 +#define EC_FAN_DBG_CONTROL_CMD 0x30 +#define EC_SCI_EVT_CONTROL_CMD 0x35 +#define EC_THERMAL_CAP_CMD 0x42 + +#define EC_FW_VERSION_RESP_LEN 4 +#define EC_THERMAL_CAP_RESP_LEN 3 +#define EC_FAN_DEBUG_CMD_LEN 6 +#define EC_FAN_SPEED_DATA_SIZE 4 + +#define EC_MODERN_STANDBY_ENTER 0x01 +#define EC_MODERN_STANDBY_EXIT 0x00 + +#define EC_FAN_DEBUG_MODE_OFF 0 +#define EC_FAN_DEBUG_MODE_ON BIT(0) +#define EC_FAN_ON BIT(1) +#define EC_FAN_DEBUG_TYPE_PWM BIT(2) +#define EC_MAX_FAN_CNT 2 +#define EC_FAN_NAME_SIZE 20 +#define EC_FAN_MAX_PWM 255 + +enum qcom_ec_sci_events { + EC_FAN1_STATUS_CHANGE_EVT = 0x30, + EC_FAN2_STATUS_CHANGE_EVT, + EC_FAN1_SPEED_CHANGE_EVT, + EC_FAN2_SPEED_CHANGE_EVT, + EC_NEW_LUT_SET_EVT, + EC_FAN_PROFILE_SWITCH_EVT, + EC_THERMISTOR_1_THRESHOLD_CROSS_EVT, + EC_THERMISTOR_2_THRESHOLD_CROSS_EVT, + EC_THERMISTOR_3_THRESHOLD_CROSS_EVT, + /* Reserved: 0x39 - 0x3c/0x3f */ + EC_RECOVERED_FROM_RESET_EVT = 0x3d, +}; + +struct qcom_ec_version { + u8 main_version; + u8 sub_version; + u8 test_version; +}; + +struct qcom_ec_thermal_cap { +#define EC_THERMAL_FAN_CNT(x) (FIELD_GET(GENMASK(1, 0), (x))) +#define EC_THERMAL_FAN_TYPE(x) (FIELD_GET(GENMASK(4, 2), (x))) +#define EC_THERMAL_THERMISTOR_MASK(x) (FIELD_GET(GENMASK(7, 0), (x))) + u8 fan_cnt; + u8 fan_type; + u8 thermistor_mask; +}; + +struct qcom_ec_cooling_dev { + struct thermal_cooling_device *cdev; + struct device *parent_dev; + u8 fan_id; + u8 state; +}; + +struct qcom_ec { + struct qcom_ec_cooling_dev *ec_cdev; + struct qcom_ec_thermal_cap thermal_cap; + struct qcom_ec_version version; + struct i2c_client *client; +}; + +static int qcom_ec_read(struct qcom_ec *ec, u8 cmd, u8 resp_len, u8 *resp) +{ + int ret; + + ret = i2c_smbus_read_i2c_block_data(ec->client, cmd, resp_len, resp); + + if (ret < 0) + return ret; + else if (ret == 0 || ret == 0xff) + return -EOPNOTSUPP; + + if (resp[0] >= resp_len) + return -EINVAL; + + return 0; +} + +/* + * EC Device Firmware Version: + * + * Read Response: + * ---------------------------------------------------------------------- + * | Offset | Name | Description | + * ---------------------------------------------------------------------- + * | 0x00 | Byte count | Number of bytes in response | + * | | | (excluding byte count) | + * ---------------------------------------------------------------------- + * | 0x01 | Test-version | Test-version of EC firmware | + * ---------------------------------------------------------------------- + * | 0x02 | Sub-version | Sub-version of EC firmware | + * ---------------------------------------------------------------------- + * | 0x03 | Main-version | Main-version of EC firmware | + * ---------------------------------------------------------------------- + * + */ +static int qcom_ec_read_fw_version(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct qcom_ec *ec = i2c_get_clientdata(client); + struct qcom_ec_version *version = &ec->version; + u8 resp[EC_FW_VERSION_RESP_LEN]; + int ret; + + ret = qcom_ec_read(ec, EC_FW_VERSION_CMD, EC_FW_VERSION_RESP_LEN, resp); + if (ret < 0) + return ret; + + version->main_version = resp[3]; + version->sub_version = resp[2]; + version->test_version = resp[1]; + + dev_dbg(dev, "EC Version %d.%d.%d\n", + version->main_version, version->sub_version, version->test_version); + + return 0; +} + +/* + * EC Device Thermal Capabilities: + * + * Read Response: + * ------------------------------------------------------------------------------ + * | Offset | Name | Description | + * ------------------------------------------------------------------------------ + * | 0x00 | Byte count | Number of bytes in response | + * | | | (excluding byte count) | + * ------------------------------------------------------------------------------ + * | 0x02 (LSB) | EC Thermal | Bit 0-1: Number of fans | + * | 0x03 | Capabilities | Bit 2-4: Type of fan | + * | | | Bit 5-6: Reserved | + * | | | Bit 7: Data Valid/Invalid | + * | | | (Valid - 1, Invalid - 0) | + * | | | Bit 8-15: Thermistor 0 - 7 presence | + * | | | (1 present, 0 absent) | + * ------------------------------------------------------------------------------ + * + */ +static int qcom_ec_thermal_capabilities(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct qcom_ec *ec = i2c_get_clientdata(client); + struct qcom_ec_thermal_cap *cap = &ec->thermal_cap; + u8 resp[EC_THERMAL_CAP_RESP_LEN]; + int ret; + + ret = qcom_ec_read(ec, EC_THERMAL_CAP_CMD, EC_THERMAL_CAP_RESP_LEN, resp); + if (ret < 0) + return ret; + + cap->fan_cnt = min(EC_MAX_FAN_CNT, EC_THERMAL_FAN_CNT(resp[1])); + cap->fan_type = EC_THERMAL_FAN_TYPE(resp[1]); + cap->thermistor_mask = EC_THERMAL_THERMISTOR_MASK(resp[2]); + + dev_dbg(dev, "Fan count: %d Fan Type: %d Thermistor Mask: %x\n", + cap->fan_cnt, cap->fan_type, cap->thermistor_mask); + + return 0; +} + +static irqreturn_t qcom_ec_irq(int irq, void *data) +{ + struct qcom_ec *ec = data; + struct device *dev = &ec->client->dev; + int val; + + val = i2c_smbus_read_byte_data(ec->client, EC_SCI_EVT_READ_CMD); + if (val < 0) { + dev_err_ratelimited(dev, "Failed to read EC SCI Event: %d\n", val); + return IRQ_HANDLED; + } + + switch (val) { + case EC_FAN1_STATUS_CHANGE_EVT: + dev_dbg_ratelimited(dev, "Fan1 status changed\n"); + break; + case EC_FAN2_STATUS_CHANGE_EVT: + dev_dbg_ratelimited(dev, "Fan2 status changed\n"); + break; + case EC_FAN1_SPEED_CHANGE_EVT: + dev_dbg_ratelimited(dev, "Fan1 speed crossed low/high trip point\n"); + break; + case EC_FAN2_SPEED_CHANGE_EVT: + dev_dbg_ratelimited(dev, "Fan2 speed crossed low/high trip point\n"); + break; + case EC_NEW_LUT_SET_EVT: + dev_dbg_ratelimited(dev, "New LUT set\n"); + break; + case EC_FAN_PROFILE_SWITCH_EVT: + dev_dbg_ratelimited(dev, "FAN Profile switched\n"); + break; + case EC_THERMISTOR_1_THRESHOLD_CROSS_EVT: + dev_dbg_ratelimited(dev, "Thermistor 1 threshold crossed\n"); + break; + case EC_THERMISTOR_2_THRESHOLD_CROSS_EVT: + dev_dbg_ratelimited(dev, "Thermistor 2 threshold crossed\n"); + break; + case EC_THERMISTOR_3_THRESHOLD_CROSS_EVT: + dev_dbg_ratelimited(dev, "Thermistor 3 threshold crossed\n"); + break; + case EC_RECOVERED_FROM_RESET_EVT: + dev_dbg_ratelimited(dev, "EC recovered from reset\n"); + break; + default: + dev_notice_ratelimited(dev, "Unknown EC event: %d\n", val); + break; + } + + return IRQ_HANDLED; +} + +static int qcom_ec_sci_evt_control(struct device *dev, bool enable) +{ + struct i2c_client *client = to_i2c_client(dev); + + return i2c_smbus_write_byte_data(client, EC_SCI_EVT_CONTROL_CMD, !!enable); +} + +static int qcom_ec_fan_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state) +{ + *state = EC_FAN_MAX_PWM; + + return 0; +} + +static int qcom_ec_fan_get_cur_state(struct thermal_cooling_device *cdev, unsigned long *state) +{ + struct qcom_ec_cooling_dev *ec_cdev = cdev->devdata; + + *state = ec_cdev->state; + + return 0; +} + +/* + * Fan Debug control command: + * + * Command Payload: + * -------------------------------------------------------------------------------------- + * | Offset | Name | Description | + * -------------------------------------------------------------------------------------- + * | 0x00 | Command | Fan control command | + * -------------------------------------------------------------------------------------- + * | 0x01 | Fan ID | 0x1 : Fan 1 | + * | | | 0x2 : Fan 2 | + * -------------------------------------------------------------------------------------- + * | 0x02 | Byte count = 4| Size of data to set fan speed | + * -------------------------------------------------------------------------------------- + * | 0x03 | Mode | Bit 0: Debug Mode On/Off (0 - OFF, 1 - ON ) | + * | | | Bit 1: Fan On/Off (0 - Off, 1 - ON) | + * | | | Bit 2: Debug Type (0 - RPM, 1 - PWM) | + * -------------------------------------------------------------------------------------- + * | 0x04 (LSB) | Speed in RPM | RPM value, if mode selected is RPM | + * | 0x05 | | | + * -------------------------------------------------------------------------------------- + * | 0x06 | Speed in PWM | PWM value, if mode selected is PWM (0 - 255) | + * ______________________________________________________________________________________ + * + */ +static int qcom_ec_fan_debug_mode_off(struct qcom_ec_cooling_dev *ec_cdev) +{ + struct device *dev = ec_cdev->parent_dev; + struct i2c_client *client = to_i2c_client(dev); + u8 request[6] = { ec_cdev->fan_id, EC_FAN_SPEED_DATA_SIZE, + EC_FAN_DEBUG_MODE_OFF, 0, 0, 0 }; + int ret; + + ret = i2c_smbus_write_i2c_block_data(client, EC_FAN_DBG_CONTROL_CMD, + sizeof(request), request); + if (ret) { + dev_err(dev, "Failed to turn off fan%d debug mode: %d\n", + ec_cdev->fan_id, ret); + } + + return ret; +} + +static int qcom_ec_fan_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) +{ + struct qcom_ec_cooling_dev *ec_cdev = cdev->devdata; + struct device *dev = ec_cdev->parent_dev; + struct i2c_client *client = to_i2c_client(dev); + u8 request[6] = { ec_cdev->fan_id, EC_FAN_SPEED_DATA_SIZE, + EC_FAN_DEBUG_MODE_ON | EC_FAN_ON | EC_FAN_DEBUG_TYPE_PWM, + 0, 0, state }; + int ret; + + ret = i2c_smbus_write_i2c_block_data(client, EC_FAN_DBG_CONTROL_CMD, + sizeof(request), request); + if (ret) { + dev_err(dev, "Failed to set fan pwm: %d\n", ret); + return ret; + } + + ec_cdev->state = state; + + return 0; +} + +static const struct thermal_cooling_device_ops qcom_ec_thermal_ops = { + .get_max_state = qcom_ec_fan_get_max_state, + .get_cur_state = qcom_ec_fan_get_cur_state, + .set_cur_state = qcom_ec_fan_set_cur_state, +}; + +static int qcom_ec_resume(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + + return i2c_smbus_write_byte_data(client, EC_MODERN_STANDBY_CMD, + EC_MODERN_STANDBY_EXIT); +} + +static int qcom_ec_suspend(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + + return i2c_smbus_write_byte_data(client, EC_MODERN_STANDBY_CMD, + EC_MODERN_STANDBY_ENTER); +} + +static int qcom_ec_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct qcom_ec *ec; + unsigned int i; + int ret; + + ec = devm_kzalloc(dev, sizeof(*ec), GFP_KERNEL); + if (!ec) + return -ENOMEM; + + ec->client = client; + + ret = devm_request_threaded_irq(dev, client->irq, NULL, qcom_ec_irq, + IRQF_ONESHOT, "qcom_ec", ec); + if (ret < 0) + return ret; + + i2c_set_clientdata(client, ec); + + ret = qcom_ec_read_fw_version(dev); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to read EC firmware version\n"); + + ret = qcom_ec_sci_evt_control(dev, true); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to enable SCI events\n"); + + ret = qcom_ec_thermal_capabilities(dev); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to read thermal capabilities\n"); + + if (ec->thermal_cap.fan_cnt == 0) { + dev_warn(dev, FW_BUG "Failed to get fan count, firmware update required\n"); + return 0; + } + + ec->ec_cdev = devm_kcalloc(dev, ec->thermal_cap.fan_cnt, sizeof(*ec->ec_cdev), GFP_KERNEL); + if (!ec->ec_cdev) + return -ENOMEM; + + for (i = 0; i < ec->thermal_cap.fan_cnt; i++) { + struct qcom_ec_cooling_dev *ec_cdev = &ec->ec_cdev[i]; + char name[EC_FAN_NAME_SIZE]; + + scnprintf(name, sizeof(name), "qcom_ec_fan_%u", i); + ec_cdev->fan_id = i + 1; + ec_cdev->parent_dev = dev; + + ec_cdev->cdev = devm_thermal_of_cooling_device_register(dev, NULL, name, ec_cdev, + &qcom_ec_thermal_ops); + if (IS_ERR(ec_cdev->cdev)) { + return dev_err_probe(dev, PTR_ERR(ec_cdev->cdev), + "Failed to register fan%d cooling device\n", i); + } + } + + return 0; +} + +static void qcom_ec_remove(struct i2c_client *client) +{ + struct qcom_ec *ec = i2c_get_clientdata(client); + struct device *dev = &client->dev; + int ret; + + ret = qcom_ec_sci_evt_control(dev, false); + if (ret < 0) + dev_err(dev, "Failed to disable SCI events: %d\n", ret); + + for (int i = 0; i < ec->thermal_cap.fan_cnt; i++) { + struct qcom_ec_cooling_dev *ec_cdev = &ec->ec_cdev[i]; + + qcom_ec_fan_debug_mode_off(ec_cdev); + } +} + +static const struct of_device_id qcom_ec_of_match[] = { + { .compatible = "qcom,hamoa-crd-ec" }, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_ec_of_match); + +static const struct i2c_device_id qcom_ec_i2c_id_table[] = { + { "qcom-hamoa-ec", }, + {} +}; +MODULE_DEVICE_TABLE(i2c, qcom_ec_i2c_id_table); + +static DEFINE_SIMPLE_DEV_PM_OPS(qcom_ec_pm_ops, + qcom_ec_suspend, + qcom_ec_resume); + +static struct i2c_driver qcom_ec_i2c_driver = { + .driver = { + .name = "qcom-hamoa-ec", + .of_match_table = qcom_ec_of_match, + .pm = &qcom_ec_pm_ops + }, + .probe = qcom_ec_probe, + .remove = qcom_ec_remove, + .id_table = qcom_ec_i2c_id_table, +}; +module_i2c_driver(qcom_ec_i2c_driver); + +MODULE_DESCRIPTION("QCOM Hamoa Embedded Controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/chrome/cros_ec_ishtp.c b/drivers/platform/chrome/cros_ec_ishtp.c index 4e74e702c5a24..3766cef81fe82 100644 --- a/drivers/platform/chrome/cros_ec_ishtp.c +++ b/drivers/platform/chrome/cros_ec_ishtp.c @@ -667,6 +667,7 @@ static void cros_ec_ishtp_remove(struct ishtp_cl_device *cl_device) cancel_work_sync(&client_data->work_ishtp_reset); cancel_work_sync(&client_data->work_ec_evt); + cros_ec_unregister(client_data->ec_dev); cros_ish_deinit(cros_ish_cl); ishtp_put_device(cl_device); } diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index 87634f6921b78..6b028615ee245 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -119,7 +119,7 @@ static int get_lightbar_version(struct cros_ec_dev *ec, param = (struct ec_params_lightbar *)msg->data; param->cmd = LIGHTBAR_CMD_VERSION; msg->outsize = sizeof(param->cmd); - msg->result = sizeof(resp->version); + msg->insize = sizeof(resp->version); ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0 && ret != -EINVAL) { ret = 0; diff --git a/drivers/platform/chrome/cros_typec_switch.c b/drivers/platform/chrome/cros_typec_switch.c index 8d7c34abb0a12..d8a28d4e51a85 100644 --- a/drivers/platform/chrome/cros_typec_switch.c +++ b/drivers/platform/chrome/cros_typec_switch.c @@ -230,20 +230,20 @@ static int cros_typec_register_switches(struct cros_typec_switch_data *sdata) adev = to_acpi_device_node(fwnode); if (!adev) { - dev_err(fwnode->dev, "Couldn't get ACPI device handle\n"); + dev_err(dev, "Couldn't get ACPI device handle for %pfwP\n", fwnode); ret = -ENODEV; goto err_switch; } ret = acpi_evaluate_integer(adev->handle, "_ADR", NULL, &index); if (ACPI_FAILURE(ret)) { - dev_err(fwnode->dev, "_ADR wasn't evaluated\n"); + dev_err(dev, "_ADR wasn't evaluated for %pfwP\n", fwnode); ret = -ENODATA; goto err_switch; } if (index >= EC_USB_PD_MAX_PORTS) { - dev_err(fwnode->dev, "Invalid port index number: %llu\n", index); + dev_err(dev, "%pfwP: Invalid port index number: %llu\n", fwnode, index); ret = -EINVAL; goto err_switch; } diff --git a/drivers/platform/mellanox/mlx-platform.c b/drivers/platform/mellanox/mlx-platform.c index d0df18be93c76..efd0c074ad937 100644 --- a/drivers/platform/mellanox/mlx-platform.c +++ b/drivers/platform/mellanox/mlx-platform.c @@ -7381,7 +7381,7 @@ static int __init mlxplat_dmi_ng400_hi171_matched(const struct dmi_system_id *dm mlxplat_hotplug = &mlxplat_mlxcpld_ng800_hi171_data; mlxplat_hotplug->deferred_nr = mlxplat_msn21xx_channels[MLXPLAT_CPLD_GRP_CHNL_NUM - 1]; - mlxplat_led = &mlxplat_default_ng_led_data; + mlxplat_led = &mlxplat_xdr_led_data; mlxplat_regs_io = &mlxplat_default_ng_regs_io_data; mlxplat_fan = &mlxplat_xdr_fan_data; diff --git a/drivers/platform/mellanox/mlxbf-pmc.c b/drivers/platform/mellanox/mlxbf-pmc.c index 16a2fd9fdd9b8..5ec1ad4716967 100644 --- a/drivers/platform/mellanox/mlxbf-pmc.c +++ b/drivers/platform/mellanox/mlxbf-pmc.c @@ -801,18 +801,18 @@ static const struct mlxbf_pmc_events mlxbf_pmc_llt_miss_events[] = { {11, "GDC_MISS_MACHINE_CHI_TXDAT"}, {12, "GDC_MISS_MACHINE_CHI_RXDAT"}, {13, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC0_0"}, - {14, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC0_1 "}, + {14, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC0_1"}, {15, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC0_2"}, - {16, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC0_3 "}, - {17, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_0 "}, - {18, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_1 "}, - {19, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_2 "}, - {20, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_3 "}, + {16, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC0_3"}, + {17, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_0"}, + {18, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_1"}, + {19, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_2"}, + {20, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC1_3"}, {21, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE0_0"}, {22, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE0_1"}, {23, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE0_2"}, {24, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE0_3"}, - {25, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE1_0 "}, + {25, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE1_0"}, {26, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE1_1"}, {27, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE1_2"}, {28, "GDC_MISS_MACHINE_G_FIFO_FF_EXEC_DONE1_3"}, diff --git a/drivers/platform/olpc/olpc-xo175-ec.c b/drivers/platform/olpc/olpc-xo175-ec.c index fa7b3bda688a6..bee271a4fda1a 100644 --- a/drivers/platform/olpc/olpc-xo175-ec.c +++ b/drivers/platform/olpc/olpc-xo175-ec.c @@ -482,7 +482,7 @@ static int olpc_xo175_ec_cmd(u8 cmd, u8 *inbuf, size_t inlen, u8 *resp, dev_dbg(dev, "CMD %x, %zd bytes expected\n", cmd, resp_len); if (inlen > 5) { - dev_err(dev, "command len %zd too big!\n", resp_len); + dev_err(dev, "command len %zd too big!\n", inlen); return -EOVERFLOW; } diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c index 404e62ad293a9..24506e3429430 100644 --- a/drivers/platform/x86/amd/pmc/pmc-quirks.c +++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c @@ -203,6 +203,15 @@ static const struct dmi_system_id fwbug_list[] = { DMI_MATCH(DMI_PRODUCT_NAME, "82XQ"), } }, + /* https://bugzilla.kernel.org/show_bug.cgi?id=221273 */ + { + .ident = "Thinkpad L14 Gen3", + .driver_data = &quirk_s2idle_bug, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21C6"), + } + }, /* https://gitlab.freedesktop.org/drm/amd/-/issues/4434 */ { .ident = "Lenovo Yoga 6 13ALC6", @@ -302,6 +311,13 @@ static const struct dmi_system_id fwbug_list[] = { DMI_MATCH(DMI_BOARD_NAME, "XxKK4NAx_XxSP4NAx"), } }, + { + .ident = "MECHREVO Wujie 15X Pro", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "WUJIE Series-X5SP4NAG"), + } + }, {} }; diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c index bc544a4a5266e..e787480f4df26 100644 --- a/drivers/platform/x86/amd/pmf/core.c +++ b/drivers/platform/x86/amd/pmf/core.c @@ -314,6 +314,61 @@ int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev) return 0; } +static int amd_pmf_reinit_ta(struct amd_pmf_dev *pdev) +{ + bool status; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(amd_pmf_ta_uuid); i++) { + ret = amd_pmf_tee_init(pdev, &amd_pmf_ta_uuid[i]); + if (ret) { + dev_err(pdev->dev, "TEE init failed for UUID[%d] ret: %d\n", i, ret); + return ret; + } + + ret = amd_pmf_start_policy_engine(pdev); + dev_dbg(pdev->dev, "start policy engine ret: %d (UUID idx: %d)\n", ret, i); + status = ret == TA_PMF_TYPE_SUCCESS; + if (status) + break; + amd_pmf_tee_deinit(pdev); + } + + return 0; +} + +static int amd_pmf_restore_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + int ret; + + if (pdev->buf) { + ret = amd_pmf_set_dram_addr(pdev, false); + if (ret) + return ret; + } + + if (pdev->smart_pc_enabled) + amd_pmf_reinit_ta(pdev); + + return 0; +} + +static int amd_pmf_freeze_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + + if (!pdev->smart_pc_enabled) + return 0; + + cancel_delayed_work_sync(&pdev->pb_work); + /* Clear all TEE resources */ + amd_pmf_tee_deinit(pdev); + pdev->session_id = 0; + + return 0; +} + static int amd_pmf_suspend_handler(struct device *dev) { struct amd_pmf_dev *pdev = dev_get_drvdata(dev); @@ -347,7 +402,12 @@ static int amd_pmf_resume_handler(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(amd_pmf_pm, amd_pmf_suspend_handler, amd_pmf_resume_handler); +static const struct dev_pm_ops amd_pmf_pm = { + .suspend = amd_pmf_suspend_handler, + .resume = amd_pmf_resume_handler, + .freeze = amd_pmf_freeze_handler, + .restore = amd_pmf_restore_handler, +}; static void amd_pmf_init_features(struct amd_pmf_dev *dev) { diff --git a/drivers/platform/x86/amd/pmf/pmf.h b/drivers/platform/x86/amd/pmf/pmf.h index bd19f2a6bc786..2da1885d87915 100644 --- a/drivers/platform/x86/amd/pmf/pmf.h +++ b/drivers/platform/x86/amd/pmf/pmf.h @@ -122,6 +122,12 @@ struct cookie_header { typedef void (*apmf_event_handler_t)(acpi_handle handle, u32 event, void *data); +static const uuid_t amd_pmf_ta_uuid[] __used = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, + 0x8a, 0xcc, 0x2b, 0x2b, 0x60, 0xd6), + UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, + 0xc5, 0x29, 0xb1, 0x3d, 0x85, 0x43), + }; + /* APTS PMF BIOS Interface */ struct amd_pmf_apts_output { u16 table_version; @@ -888,4 +894,8 @@ void amd_pmf_populate_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_tab void amd_pmf_dump_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *in); int amd_pmf_invoke_cmd_enact(struct amd_pmf_dev *dev); +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid); +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev); +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev); + #endif /* PMF_H */ diff --git a/drivers/platform/x86/amd/pmf/tee-if.c b/drivers/platform/x86/amd/pmf/tee-if.c index 6e8116bef4f6a..9030459352375 100644 --- a/drivers/platform/x86/amd/pmf/tee-if.c +++ b/drivers/platform/x86/amd/pmf/tee-if.c @@ -27,12 +27,6 @@ module_param(pb_side_load, bool, 0444); MODULE_PARM_DESC(pb_side_load, "Sideload policy binaries debug policy failures"); #endif -static const uuid_t amd_pmf_ta_uuid[] = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, 0x8a, - 0xcc, 0x2b, 0x2b, 0x60, 0xd6), - UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, 0xc5, - 0x29, 0xb1, 0x3d, 0x85, 0x43), - }; - static const char *amd_pmf_uevent_as_str(unsigned int state) { switch (state) { @@ -312,7 +306,7 @@ static void amd_pmf_invoke_cmd(struct work_struct *work) schedule_delayed_work(&dev->pb_work, msecs_to_jiffies(pb_actions_ms)); } -static int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) { struct cookie_header *header; int res; @@ -468,7 +462,7 @@ static int amd_pmf_register_input_device(struct amd_pmf_dev *dev) return 0; } -static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) { u32 size; int ret; @@ -516,7 +510,7 @@ static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) return ret; } -static void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) { if (!dev->tee_ctx) return; diff --git a/drivers/platform/x86/amd/wbrf.c b/drivers/platform/x86/amd/wbrf.c index dd197b3aebe06..0f58d252b620a 100644 --- a/drivers/platform/x86/amd/wbrf.c +++ b/drivers/platform/x86/amd/wbrf.c @@ -104,8 +104,10 @@ static int wbrf_record(struct acpi_device *adev, uint8_t action, struct wbrf_ran obj = acpi_evaluate_dsm(adev->handle, &wifi_acpi_dsm_guid, WBRF_REVISION, WBRF_RECORD, &argv4); - if (!obj) + if (!obj) { + kfree(tmp); return -EINVAL; + } if (obj->type != ACPI_TYPE_INTEGER) { ret = -EINVAL; diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c index 6a62bc5b02fda..8dad7bdb8f612 100644 --- a/drivers/platform/x86/asus-nb-wmi.c +++ b/drivers/platform/x86/asus-nb-wmi.c @@ -548,7 +548,7 @@ static const struct dmi_system_id asus_quirks[] = { .callback = dmi_matched, .ident = "ASUS ROG Z13", .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_SYS_VENDOR, "ASUS"), DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow Z13"), }, .driver_data = &quirk_asus_z13, diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index e72a2b5d158e9..8e3300f5c2943 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -1619,14 +1619,14 @@ static void do_kbd_led_set(struct led_classdev *led_cdev, int value) kbd_led_update(asus); } -static void kbd_led_set(struct led_classdev *led_cdev, - enum led_brightness value) +static int kbd_led_set(struct led_classdev *led_cdev, enum led_brightness value) { /* Prevent disabling keyboard backlight on module unregister */ if (led_cdev->flags & LED_UNREGISTERING) - return; + return 0; do_kbd_led_set(led_cdev, value); + return 0; } static void kbd_led_set_by_kbd(struct asus_wmi *asus, enum led_brightness value) @@ -1802,7 +1802,7 @@ static int asus_wmi_led_init(struct asus_wmi *asus) asus->kbd_led_wk = led_val; asus->kbd_led.name = "asus::kbd_backlight"; asus->kbd_led.flags = LED_BRIGHT_HW_CHANGED; - asus->kbd_led.brightness_set = kbd_led_set; + asus->kbd_led.brightness_set_blocking = kbd_led_set; asus->kbd_led.brightness_get = kbd_led_get; asus->kbd_led.max_brightness = 3; diff --git a/drivers/platform/x86/classmate-laptop.c b/drivers/platform/x86/classmate-laptop.c index 6b1b8e444e241..74d3eb83f56a6 100644 --- a/drivers/platform/x86/classmate-laptop.c +++ b/drivers/platform/x86/classmate-laptop.c @@ -207,7 +207,12 @@ static ssize_t cmpc_accel_sensitivity_show_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; return sysfs_emit(buf, "%d\n", accel->sensitivity); } @@ -224,7 +229,12 @@ static ssize_t cmpc_accel_sensitivity_store_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; r = kstrtoul(buf, 0, &sensitivity); if (r) @@ -256,7 +266,12 @@ static ssize_t cmpc_accel_g_select_show_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; return sysfs_emit(buf, "%d\n", accel->g_select); } @@ -273,7 +288,12 @@ static ssize_t cmpc_accel_g_select_store_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; r = kstrtoul(buf, 0, &g_select); if (r) @@ -302,6 +322,8 @@ static int cmpc_accel_open_v4(struct input_dev *input) acpi = to_acpi_device(input->dev.parent); accel = dev_get_drvdata(&input->dev); + if (!accel) + return -ENXIO; cmpc_accel_set_sensitivity_v4(acpi->handle, accel->sensitivity); cmpc_accel_set_g_select_v4(acpi->handle, accel->g_select); @@ -549,7 +571,12 @@ static ssize_t cmpc_accel_sensitivity_show(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; return sysfs_emit(buf, "%d\n", accel->sensitivity); } @@ -566,7 +593,12 @@ static ssize_t cmpc_accel_sensitivity_store(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; r = kstrtoul(buf, 0, &sensitivity); if (r) diff --git a/drivers/platform/x86/dell/alienware-wmi-wmax.c b/drivers/platform/x86/dell/alienware-wmi-wmax.c index fadf7aac6779f..fdae40689f178 100644 --- a/drivers/platform/x86/dell/alienware-wmi-wmax.c +++ b/drivers/platform/x86/dell/alienware-wmi-wmax.c @@ -89,6 +89,30 @@ static struct awcc_quirks generic_quirks = { static struct awcc_quirks empty_quirks; static const struct dmi_system_id awcc_dmi_table[] __initconst = { + { + .ident = "Alienware 16 Area-51", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alienware 16 Area-51"), + }, + .driver_data = &g_series_quirks, + }, + { + .ident = "Alienware 16X Aurora", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alienware 16X Aurora"), + }, + .driver_data = &g_series_quirks, + }, + { + .ident = "Alienware 18 Area-51", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alienware 18 Area-51"), + }, + .driver_data = &g_series_quirks, + }, { .ident = "Alienware 16 Aurora", .matches = { @@ -151,7 +175,7 @@ static const struct dmi_system_id awcc_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), DMI_MATCH(DMI_PRODUCT_NAME, "Alienware m18"), }, - .driver_data = &generic_quirks, + .driver_data = &g_series_quirks, }, { .ident = "Alienware x15", @@ -161,6 +185,14 @@ static const struct dmi_system_id awcc_dmi_table[] __initconst = { }, .driver_data = &generic_quirks, }, + { + .ident = "Alienware x16", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alienware x16"), + }, + .driver_data = &g_series_quirks, + }, { .ident = "Alienware x17", .matches = { diff --git a/drivers/platform/x86/dell/dell-lis3lv02d.c b/drivers/platform/x86/dell/dell-lis3lv02d.c index 77905a9ddde9d..fe52bcd896f78 100644 --- a/drivers/platform/x86/dell/dell-lis3lv02d.c +++ b/drivers/platform/x86/dell/dell-lis3lv02d.c @@ -44,6 +44,7 @@ static const struct dmi_system_id lis3lv02d_devices[] __initconst = { /* * Additional individual entries were added after verification. */ + DELL_LIS3LV02D_DMI_ENTRY("Latitude 5400", 0x29), DELL_LIS3LV02D_DMI_ENTRY("Latitude 5480", 0x29), DELL_LIS3LV02D_DMI_ENTRY("Latitude 5500", 0x29), DELL_LIS3LV02D_DMI_ENTRY("Latitude E6330", 0x29), diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c index 28076929d6af5..907f1da01c8db 100644 --- a/drivers/platform/x86/dell/dell-wmi-base.c +++ b/drivers/platform/x86/dell/dell-wmi-base.c @@ -80,6 +80,12 @@ static const struct dmi_system_id dell_wmi_smbios_list[] __initconst = { static const struct key_entry dell_wmi_keymap_type_0000[] = { { KE_IGNORE, 0x003a, { KEY_CAPSLOCK } }, + /* Audio mute toggle */ + { KE_KEY, 0x0109, { KEY_MUTE } }, + + /* Mic mute toggle */ + { KE_KEY, 0x0150, { KEY_MICMUTE } }, + /* Meta key lock */ { KE_IGNORE, 0xe000, { KEY_RIGHTMETA } }, diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c index 86ec962aace9b..e586f7957946b 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c @@ -93,7 +93,6 @@ int set_new_password(const char *password_type, const char *new) if (ret < 0) goto out; - print_hex_dump_bytes("set new password data: ", DUMP_PREFIX_NONE, buffer, buffer_size); ret = call_password_interface(wmi_priv.password_attr_wdev, buffer, buffer_size); /* on success copy the new password to current password */ if (!ret) diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c index 5bfa7159f5bcd..51e8977d3eb4a 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c +++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include "bioscfg.h" #include "../../firmware_attributes_class.h" @@ -694,6 +696,11 @@ static int hp_init_bios_package_attribute(enum hp_wmi_data_type attr_type, return ret; } + if (!str_value || !str_value[0]) { + pr_debug("Ignoring attribute with empty name\n"); + goto pack_attr_exit; + } + /* All duplicate attributes found are ignored */ duplicate = kset_find_obj(temp_kset, str_value); if (duplicate) { @@ -781,6 +788,12 @@ static int hp_init_bios_buffer_attribute(enum hp_wmi_data_type attr_type, if (ret < 0) goto buff_attr_exit; + if (strlen(str) == 0) { + pr_debug("Ignoring attribute with empty name\n"); + ret = 0; + goto buff_attr_exit; + } + if (attr_type == HPWMI_PASSWORD_TYPE || attr_type == HPWMI_SECURE_PLATFORM_TYPE) temp_kset = bioscfg_drv.authentication_dir_kset; diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h index 3166ef328ebae..f1eec0e4ba075 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h +++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -56,14 +57,14 @@ enum mechanism_values { #define PASSWD_MECHANISM_TYPES "password" -#define HP_WMI_BIOS_GUID "5FB7F034-2C63-45e9-BE91-3D44E2C707E4" +#define HP_WMI_BIOS_GUID "5FB7F034-2C63-45E9-BE91-3D44E2C707E4" -#define HP_WMI_BIOS_STRING_GUID "988D08E3-68F4-4c35-AF3E-6A1B8106F83C" +#define HP_WMI_BIOS_STRING_GUID "988D08E3-68F4-4C35-AF3E-6A1B8106F83C" #define HP_WMI_BIOS_INTEGER_GUID "8232DE3D-663D-4327-A8F4-E293ADB9BF05" #define HP_WMI_BIOS_ENUMERATION_GUID "2D114B49-2DFB-4130-B8FE-4A3C09E75133" #define HP_WMI_BIOS_ORDERED_LIST_GUID "14EA9746-CE1F-4098-A0E0-7045CB4DA745" #define HP_WMI_BIOS_PASSWORD_GUID "322F2028-0F84-4901-988E-015176049E2D" -#define HP_WMI_SET_BIOS_SETTING_GUID "1F4C91EB-DC5C-460b-951D-C7CB9B4B8D5E" +#define HP_WMI_SET_BIOS_SETTING_GUID "1F4C91EB-DC5C-460B-951D-C7CB9B4B8D5E" enum hp_wmi_spm_commandtype { HPWMI_SECUREPLATFORM_GET_STATE = 0x10, @@ -285,8 +286,9 @@ enum hp_wmi_data_elements { { \ int i; \ \ - for (i = 0; i <= bioscfg_drv.type##_instances_count; i++) { \ - if (!strcmp(kobj->name, bioscfg_drv.type##_data[i].attr_name_kobj->name)) \ + for (i = 0; i < bioscfg_drv.type##_instances_count; i++) { \ + if (bioscfg_drv.type##_data[i].attr_name_kobj && \ + !strcmp(kobj->name, bioscfg_drv.type##_data[i].attr_name_kobj->name)) \ return i; \ } \ return -EIO; \ diff --git a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c index c50ad58805038..af4d1920d4880 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c @@ -94,8 +94,11 @@ int hp_alloc_enumeration_data(void) bioscfg_drv.enumeration_instances_count = hp_get_instance_count(HP_WMI_BIOS_ENUMERATION_GUID); - bioscfg_drv.enumeration_data = kcalloc(bioscfg_drv.enumeration_instances_count, - sizeof(*bioscfg_drv.enumeration_data), GFP_KERNEL); + if (!bioscfg_drv.enumeration_instances_count) + return -EINVAL; + bioscfg_drv.enumeration_data = kvcalloc(bioscfg_drv.enumeration_instances_count, + sizeof(*bioscfg_drv.enumeration_data), GFP_KERNEL); + if (!bioscfg_drv.enumeration_data) { bioscfg_drv.enumeration_instances_count = 0; return -ENOMEM; @@ -207,7 +210,7 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum case PREREQUISITES: size = min_t(u32, enum_data->common.prerequisites_size, MAX_PREREQUISITES_SIZE); for (reqs = 0; reqs < size; reqs++) { - if (elem >= enum_obj_count) { + if (elem + reqs >= enum_obj_count) { pr_err("Error enum-objects package is too small\n"); return -EINVAL; } @@ -255,7 +258,7 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum for (pos_values = 0; pos_values < size && pos_values < MAX_VALUES_SIZE; pos_values++) { - if (elem >= enum_obj_count) { + if (elem + pos_values >= enum_obj_count) { pr_err("Error enum-objects package is too small\n"); return -EINVAL; } @@ -444,6 +447,6 @@ void hp_exit_enumeration_attributes(void) } bioscfg_drv.enumeration_instances_count = 0; - kfree(bioscfg_drv.enumeration_data); + kvfree(bioscfg_drv.enumeration_data); bioscfg_drv.enumeration_data = NULL; } diff --git a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c index 6c7f4d5fa9cb9..63b1fda2be4e2 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c @@ -227,7 +227,7 @@ static int hp_populate_integer_elements_from_package(union acpi_object *integer_ size = min_t(u32, integer_data->common.prerequisites_size, MAX_PREREQUISITES_SIZE); for (reqs = 0; reqs < size; reqs++) { - if (elem >= integer_obj_count) { + if (elem + reqs >= integer_obj_count) { pr_err("Error elem-objects package is too small\n"); return -EINVAL; } diff --git a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c index c6e57bb9d8b74..6a31f47ce3f5b 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c @@ -216,6 +216,11 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord size = min_t(u32, ordered_list_data->common.prerequisites_size, MAX_PREREQUISITES_SIZE); for (reqs = 0; reqs < size; reqs++) { + if (elem + reqs >= order_obj_count) { + pr_err("Error elem-objects package is too small\n"); + return -EINVAL; + } + ret = hp_convert_hexstr_to_str(order_obj[elem + reqs].string.pointer, order_obj[elem + reqs].string.length, &str_value, &value_len); diff --git a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c index 187b372123ed3..ec79d9d50377a 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c @@ -303,6 +303,11 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor MAX_PREREQUISITES_SIZE); for (reqs = 0; reqs < size; reqs++) { + if (elem + reqs >= password_obj_count) { + pr_err("Error elem-objects package is too small\n"); + return -EINVAL; + } + ret = hp_convert_hexstr_to_str(password_obj[elem + reqs].string.pointer, password_obj[elem + reqs].string.length, &str_value, &value_len); diff --git a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c index 27758b779b2d3..7b885d25650c5 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c @@ -217,7 +217,7 @@ static int hp_populate_string_elements_from_package(union acpi_object *string_ob MAX_PREREQUISITES_SIZE); for (reqs = 0; reqs < size; reqs++) { - if (elem >= string_obj_count) { + if (elem + reqs >= string_obj_count) { pr_err("Error elem-objects package is too small\n"); return -EINVAL; } diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index ad9d9f97960f2..31d099bd8db43 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -53,6 +53,66 @@ MODULE_ALIAS("wmi:5FB7F034-2C63-45E9-BE91-3D44E2C707E4"); #define zero_if_sup(tmp) (zero_insize_support?0:sizeof(tmp)) // use when zero insize is required +enum hp_thermal_profile_omen_v0 { + HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, + HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, +}; + +enum hp_thermal_profile_omen_v1 { + HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, + HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, + HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, +}; + +enum hp_thermal_profile_omen_flags { + HP_OMEN_EC_FLAGS_TURBO = 0x04, + HP_OMEN_EC_FLAGS_NOTIMER = 0x02, + HP_OMEN_EC_FLAGS_JUSTSET = 0x01, +}; + +enum hp_thermal_profile_victus { + HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, +}; + +enum hp_thermal_profile_victus_s { + HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, +}; + +enum hp_thermal_profile { + HP_THERMAL_PROFILE_PERFORMANCE = 0x00, + HP_THERMAL_PROFILE_DEFAULT = 0x01, + HP_THERMAL_PROFILE_COOL = 0x02, + HP_THERMAL_PROFILE_QUIET = 0x03, +}; + +struct thermal_profile_params { + u8 performance; + u8 balanced; + u8 low_power; +}; + +static const struct thermal_profile_params victus_s_thermal_params = { + .performance = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, + .low_power = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, +}; + +static const struct thermal_profile_params omen_v1_thermal_params = { + .performance = HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, + .low_power = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, +}; + +/* + * A generic pointer for the currently-active board's thermal profile + * parameters. + */ +static struct thermal_profile_params *active_thermal_profile_params; + /* DMI board names of devices that should use the omen specific path for * thermal profiles. * This was obtained by taking a look in the windows omen command center @@ -93,12 +153,52 @@ static const char * const victus_thermal_profile_boards[] = { }; /* DMI Board names of Victus 16-r and Victus 16-s laptops */ -static const char * const victus_s_thermal_profile_boards[] = { - "8BBE", "8BD4", "8BD5", - "8C78", "8C99", "8C9C", - "8D41", +static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = { + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BAB") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCD") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD4") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C76") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C99") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C9C") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + {}, }; +static bool is_victus_s_board; + enum hp_wmi_radio { HPWMI_WIFI = 0x0, HPWMI_BLUETOOTH = 0x1, @@ -219,42 +319,6 @@ enum hp_wireless2_bits { HPWMI_POWER_FW_OR_HW = HPWMI_POWER_BIOS | HPWMI_POWER_HARD, }; -enum hp_thermal_profile_omen_v0 { - HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, - HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, -}; - -enum hp_thermal_profile_omen_v1 { - HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, - HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, - HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, -}; - -enum hp_thermal_profile_omen_flags { - HP_OMEN_EC_FLAGS_TURBO = 0x04, - HP_OMEN_EC_FLAGS_NOTIMER = 0x02, - HP_OMEN_EC_FLAGS_JUSTSET = 0x01, -}; - -enum hp_thermal_profile_victus { - HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, -}; - -enum hp_thermal_profile_victus_s { - HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, -}; - -enum hp_thermal_profile { - HP_THERMAL_PROFILE_PERFORMANCE = 0x00, - HP_THERMAL_PROFILE_DEFAULT = 0x01, - HP_THERMAL_PROFILE_COOL = 0x02, - HP_THERMAL_PROFILE_QUIET = 0x03, -}; - #define IS_HWBLOCKED(x) ((x & HPWMI_POWER_FW_OR_HW) != HPWMI_POWER_FW_OR_HW) #define IS_SWBLOCKED(x) !(x & HPWMI_POWER_SOFT) @@ -1575,15 +1639,8 @@ static int platform_profile_victus_set_ec(enum platform_profile_option profile) static bool is_victus_s_thermal_profile(void) { - const char *board_name; - - board_name = dmi_get_system_info(DMI_BOARD_NAME); - if (!board_name) - return false; - - return match_string(victus_s_thermal_profile_boards, - ARRAY_SIZE(victus_s_thermal_profile_boards), - board_name) >= 0; + /* Initialised in driver init, hence safe to use here */ + return is_victus_s_board; } static int victus_s_gpu_thermal_profile_get(bool *ctgp_enable, @@ -1666,25 +1723,30 @@ static int victus_s_set_cpu_pl1_pl2(u8 pl1, u8 pl2) static int platform_profile_victus_s_set_ec(enum platform_profile_option profile) { + struct thermal_profile_params *params; bool gpu_ctgp_enable, gpu_ppab_enable; u8 gpu_dstate; /* Test shows 1 = 100%, 2 = 50%, 3 = 25%, 4 = 12.5% */ int err, tp; + params = active_thermal_profile_params; + if (!params) + return -ENODEV; + switch (profile) { case PLATFORM_PROFILE_PERFORMANCE: - tp = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE; + tp = params->performance; gpu_ctgp_enable = true; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_BALANCED: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->balanced; gpu_ctgp_enable = false; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_LOW_POWER: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->low_power; gpu_ctgp_enable = false; gpu_ppab_enable = false; gpu_dstate = 1; @@ -2221,6 +2283,26 @@ static int hp_wmi_hwmon_init(void) return 0; } +static void __init setup_active_thermal_profile_params(void) +{ + const struct dmi_system_id *id; + + /* + * Currently only victus_s devices use the + * active_thermal_profile_params + */ + id = dmi_first_match(victus_s_thermal_profile_boards); + if (id) { + /* + * Marking this boolean is required to ensure that + * is_victus_s_thermal_profile() behaves like a valid + * wrapper. + */ + is_victus_s_board = true; + active_thermal_profile_params = id->driver_data; + } +} + static int __init hp_wmi_init(void) { int event_capable = wmi_has_guid(HPWMI_EVENT_GUID); @@ -2248,6 +2330,11 @@ static int __init hp_wmi_init(void) goto err_destroy_input; } + /* + * Setup active board's thermal profile parameters before + * starting platform driver probe. + */ + setup_active_thermal_profile_params(); err = platform_driver_probe(&hp_wmi_driver, hp_wmi_bios_setup); if (err) goto err_unregister_device; diff --git a/drivers/platform/x86/ibm_rtl.c b/drivers/platform/x86/ibm_rtl.c index 231b379098017..139956168cf94 100644 --- a/drivers/platform/x86/ibm_rtl.c +++ b/drivers/platform/x86/ibm_rtl.c @@ -273,7 +273,7 @@ static int __init ibm_rtl_init(void) { /* search for the _RTL_ signature at the start of the table */ for (i = 0 ; i < ebda_size/sizeof(unsigned int); i++) { struct ibm_rtl_table __iomem * tmp; - tmp = (struct ibm_rtl_table __iomem *) (ebda_map+i); + tmp = (struct ibm_rtl_table __iomem *) (ebda_map + i*sizeof(unsigned int)); if ((readq(&tmp->signature) & RTL_MASK) == RTL_SIGNATURE) { phys_addr_t addr; unsigned int plen; diff --git a/drivers/platform/x86/intel/chtwc_int33fe.c b/drivers/platform/x86/intel/chtwc_int33fe.c index 29e8b5432f4c9..d183aa53c318b 100644 --- a/drivers/platform/x86/intel/chtwc_int33fe.c +++ b/drivers/platform/x86/intel/chtwc_int33fe.c @@ -77,7 +77,7 @@ static const struct software_node max17047_node = { * software node. */ static struct software_node_ref_args fusb302_mux_refs[] = { - { .node = NULL }, + SOFTWARE_NODE_REFERENCE(NULL), }; static const struct property_entry fusb302_properties[] = { @@ -190,11 +190,6 @@ static void cht_int33fe_remove_nodes(struct cht_int33fe_data *data) { software_node_unregister_node_group(node_group); - if (fusb302_mux_refs[0].node) { - fwnode_handle_put(software_node_fwnode(fusb302_mux_refs[0].node)); - fusb302_mux_refs[0].node = NULL; - } - if (data->dp) { data->dp->secondary = NULL; fwnode_handle_put(data->dp); @@ -202,7 +197,15 @@ static void cht_int33fe_remove_nodes(struct cht_int33fe_data *data) } } -static int cht_int33fe_add_nodes(struct cht_int33fe_data *data) +static void cht_int33fe_put_swnode(void *data) +{ + struct fwnode_handle *fwnode = data; + + fwnode_handle_put(fwnode); + fusb302_mux_refs[0] = SOFTWARE_NODE_REFERENCE(NULL); +} + +static int cht_int33fe_add_nodes(struct device *dev, struct cht_int33fe_data *data) { const struct software_node *mux_ref_node; int ret; @@ -212,17 +215,25 @@ static int cht_int33fe_add_nodes(struct cht_int33fe_data *data) * until the mux driver has created software node for the mux device. * It means we depend on the mux driver. This function will return * -EPROBE_DEFER until the mux device is registered. + * + * FIXME: the relevant software node exists in intel-xhci-usb-role-switch + * and - if exported - could be used to set up a static reference. */ mux_ref_node = software_node_find_by_name(NULL, "intel-xhci-usb-sw"); if (!mux_ref_node) return -EPROBE_DEFER; + ret = devm_add_action_or_reset(dev, cht_int33fe_put_swnode, + software_node_fwnode(mux_ref_node)); + if (ret) + return ret; + /* * Update node used in "usb-role-switch" property. Note that we * rely on software_node_register_node_group() to use the original * instance of properties instead of copying them. */ - fusb302_mux_refs[0].node = mux_ref_node; + fusb302_mux_refs[0] = SOFTWARE_NODE_REFERENCE(mux_ref_node); ret = software_node_register_node_group(node_group); if (ret) @@ -345,7 +356,7 @@ static int cht_int33fe_typec_probe(struct platform_device *pdev) return fusb302_irq; } - ret = cht_int33fe_add_nodes(data); + ret = cht_int33fe_add_nodes(dev, data); if (ret) return ret; diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c index 9c07a7faf18fe..c5e80887d0cb0 100644 --- a/drivers/platform/x86/intel/hid.c +++ b/drivers/platform/x86/intel/hid.c @@ -135,6 +135,13 @@ static const struct dmi_system_id button_array_table[] = { DMI_MATCH(DMI_PRODUCT_FAMILY, "ThinkPad X1 Tablet Gen 2"), }, }, + { + .ident = "Lenovo ThinkPad X1 Fold 16 Gen 1", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_FAMILY, "ThinkPad X1 Fold 16 Gen 1"), + }, + }, { .ident = "Microsoft Surface Go 3", .matches = { @@ -177,6 +184,24 @@ static const struct dmi_system_id dmi_vgbs_allow_list[] = { DMI_MATCH(DMI_PRODUCT_NAME, "HP Elite Dragonfly G2 Notebook PC"), }, }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell Pro Rugged 10 Tablet RA00260"), + }, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell Pro Rugged 12 Tablet RA02260"), + }, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell 14 Plus 2-in-1 DB04250"), + }, + }, { } }; @@ -407,6 +432,14 @@ static int intel_hid_pl_suspend_handler(struct device *device) return 0; } +static int intel_hid_pl_freeze_handler(struct device *device) +{ + struct intel_hid_priv *priv = dev_get_drvdata(device); + + priv->wakeup_mode = false; + return intel_hid_pl_suspend_handler(device); +} + static int intel_hid_pl_resume_handler(struct device *device) { intel_hid_pm_complete(device); @@ -421,7 +454,7 @@ static int intel_hid_pl_resume_handler(struct device *device) static const struct dev_pm_ops intel_hid_pl_pm_ops = { .prepare = intel_hid_pm_prepare, .complete = intel_hid_pm_complete, - .freeze = intel_hid_pl_suspend_handler, + .freeze = intel_hid_pl_freeze_handler, .thaw = intel_hid_pl_resume_handler, .restore = intel_hid_pl_resume_handler, .suspend = intel_hid_pl_suspend_handler, diff --git a/drivers/platform/x86/intel/int0002_vgpio.c b/drivers/platform/x86/intel/int0002_vgpio.c index 6f5629dc3f8db..562e880256436 100644 --- a/drivers/platform/x86/intel/int0002_vgpio.c +++ b/drivers/platform/x86/intel/int0002_vgpio.c @@ -206,8 +206,8 @@ static int int0002_probe(struct platform_device *pdev) * FIXME: augment this if we managed to pull handling of shared * IRQs into gpiolib. */ - ret = devm_request_irq(dev, irq, int0002_irq, - IRQF_ONESHOT | IRQF_SHARED, "INT0002", chip); + ret = devm_request_irq(dev, irq, int0002_irq, IRQF_SHARED, "INT0002", + chip); if (ret) { dev_err(dev, "Error requesting IRQ %d: %d\n", irq, ret); return ret; diff --git a/drivers/platform/x86/intel/plr_tpmi.c b/drivers/platform/x86/intel/plr_tpmi.c index 58132da477457..05727169f49c1 100644 --- a/drivers/platform/x86/intel/plr_tpmi.c +++ b/drivers/platform/x86/intel/plr_tpmi.c @@ -316,7 +316,7 @@ static int intel_plr_probe(struct auxiliary_device *auxdev, const struct auxilia snprintf(name, sizeof(name), "domain%d", i); dentry = debugfs_create_dir(name, plr->dbgfs_dir); - debugfs_create_file("status", 0444, dentry, &plr->die_info[i], + debugfs_create_file("status", 0644, dentry, &plr->die_info[i], &plr_status_fops); } diff --git a/drivers/platform/x86/intel/pmc/core.h b/drivers/platform/x86/intel/pmc/core.h index f4dadb696a314..d6818bd34768e 100644 --- a/drivers/platform/x86/intel/pmc/core.h +++ b/drivers/platform/x86/intel/pmc/core.h @@ -282,7 +282,7 @@ enum ppfear_regs { /* Die C6 from PUNIT telemetry */ #define MTL_PMT_DMU_DIE_C6_OFFSET 15 #define MTL_PMT_DMU_GUID 0x1A067102 -#define ARL_PMT_DMU_GUID 0x1A06A000 +#define ARL_PMT_DMU_GUID 0x1A06A102 #define LNL_PMC_MMIO_REG_LEN 0x2708 #define LNL_PMC_LTR_OSSE 0x1B88 diff --git a/drivers/platform/x86/intel/pmt/discovery.c b/drivers/platform/x86/intel/pmt/discovery.c index 32713a194a550..e500aa327d237 100644 --- a/drivers/platform/x86/intel/pmt/discovery.c +++ b/drivers/platform/x86/intel/pmt/discovery.c @@ -503,8 +503,10 @@ static int pmt_features_discovery(struct pmt_features_priv *priv, ret = kobject_init_and_add(&feature->kobj, ktype, &priv->dev->kobj, "%s", pmt_feature_names[feature->id]); - if (ret) + if (ret) { + kobject_put(&feature->kobj); return ret; + } kobject_uevent(&feature->kobj, KOBJ_ADD); pmt_features_add_feat(feature); @@ -546,9 +548,9 @@ static int pmt_features_probe(struct auxiliary_device *auxdev, const struct auxi priv->dev = device_create(&intel_pmt_class, &auxdev->dev, MKDEV(0, 0), priv, "%s-%s", "features", dev_name(priv->parent)); if (IS_ERR(priv->dev)) - return dev_err_probe(priv->dev, PTR_ERR(priv->dev), + return dev_err_probe(&auxdev->dev, PTR_ERR(priv->dev), "Could not create %s-%s device node\n", - "features", dev_name(priv->dev)); + "features", dev_name(priv->parent)); /* Initialize each feature */ for (i = 0; i < ivdev->num_resources; i++) { diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index 34bff2f65a835..c8de8688daedc 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -558,6 +558,9 @@ static bool disable_dynamic_sst_features(void) { u64 value; + if (!static_cpu_has(X86_FEATURE_HWP)) + return true; + rdmsrq(MSR_PM_ENABLE, value); return !(value & 0x1); } @@ -612,6 +615,9 @@ static long isst_if_core_power_state(void __user *argp) return -EINVAL; if (core_power.get_set) { + if (power_domain_info->write_blocked) + return -EPERM; + _write_cp_info("cp_enable", core_power.enable, SST_CP_CONTROL_OFFSET, SST_CP_ENABLE_START, SST_CP_ENABLE_WIDTH, SST_MUL_FACTOR_NONE) _write_cp_info("cp_prio_type", core_power.priority_type, SST_CP_CONTROL_OFFSET, @@ -865,7 +871,7 @@ static int isst_if_get_perf_level(void __user *argp) _read_pp_info("current_level", perf_level.current_level, SST_PP_STATUS_OFFSET, SST_PP_LEVEL_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE) _read_pp_info("locked", perf_level.locked, SST_PP_STATUS_OFFSET, - SST_PP_LOCK_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE) + SST_PP_LOCK_START, SST_PP_LOCK_WIDTH, SST_MUL_FACTOR_NONE) _read_pp_info("feature_state", perf_level.feature_state, SST_PP_STATUS_OFFSET, SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE) perf_level.enabled = !!(power_domain_info->sst_header.cap_mask & BIT(1)); @@ -1454,6 +1460,8 @@ static int isst_if_get_turbo_freq_info(void __user *argp) SST_MUL_FACTOR_FREQ) } + memset(turbo_freq.bucket_core_counts, 0, sizeof(turbo_freq.bucket_core_counts)); + if (feature_rev >= 2) { bool has_tf_info_8 = false; @@ -1720,55 +1728,67 @@ EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_remove, "INTEL_TPMI_SST"); void tpmi_sst_dev_suspend(struct auxiliary_device *auxdev) { struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev); - struct tpmi_per_power_domain_info *power_domain_info; + struct tpmi_per_power_domain_info *power_domain_info, *pd_info; struct oobmsm_plat_info *plat_info; void __iomem *cp_base; + int num_resources, i; plat_info = tpmi_get_platform_data(auxdev); if (!plat_info) return; power_domain_info = tpmi_sst->power_domain_info[plat_info->partition]; + num_resources = tpmi_sst->number_of_power_domains[plat_info->partition]; - cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset; - power_domain_info->saved_sst_cp_control = readq(cp_base + SST_CP_CONTROL_OFFSET); - - memcpy_fromio(power_domain_info->saved_clos_configs, cp_base + SST_CLOS_CONFIG_0_OFFSET, - sizeof(power_domain_info->saved_clos_configs)); + for (i = 0; i < num_resources; i++) { + pd_info = &power_domain_info[i]; + if (!pd_info || !pd_info->sst_base) + continue; - memcpy_fromio(power_domain_info->saved_clos_assocs, cp_base + SST_CLOS_ASSOC_0_OFFSET, - sizeof(power_domain_info->saved_clos_assocs)); + cp_base = pd_info->sst_base + pd_info->sst_header.cp_offset; + pd_info->saved_sst_cp_control = readq(cp_base + SST_CP_CONTROL_OFFSET); + memcpy_fromio(pd_info->saved_clos_configs, cp_base + SST_CLOS_CONFIG_0_OFFSET, + sizeof(pd_info->saved_clos_configs)); + memcpy_fromio(pd_info->saved_clos_assocs, cp_base + SST_CLOS_ASSOC_0_OFFSET, + sizeof(pd_info->saved_clos_assocs)); - power_domain_info->saved_pp_control = readq(power_domain_info->sst_base + - power_domain_info->sst_header.pp_offset + - SST_PP_CONTROL_OFFSET); + pd_info->saved_pp_control = readq(pd_info->sst_base + + pd_info->sst_header.pp_offset + + SST_PP_CONTROL_OFFSET); + } } EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_suspend, "INTEL_TPMI_SST"); void tpmi_sst_dev_resume(struct auxiliary_device *auxdev) { struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev); - struct tpmi_per_power_domain_info *power_domain_info; + struct tpmi_per_power_domain_info *power_domain_info, *pd_info; struct oobmsm_plat_info *plat_info; void __iomem *cp_base; + int num_resources, i; plat_info = tpmi_get_platform_data(auxdev); if (!plat_info) return; power_domain_info = tpmi_sst->power_domain_info[plat_info->partition]; + num_resources = tpmi_sst->number_of_power_domains[plat_info->partition]; - cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset; - writeq(power_domain_info->saved_sst_cp_control, cp_base + SST_CP_CONTROL_OFFSET); - - memcpy_toio(cp_base + SST_CLOS_CONFIG_0_OFFSET, power_domain_info->saved_clos_configs, - sizeof(power_domain_info->saved_clos_configs)); + for (i = 0; i < num_resources; i++) { + pd_info = &power_domain_info[i]; + if (!pd_info || !pd_info->sst_base) + continue; - memcpy_toio(cp_base + SST_CLOS_ASSOC_0_OFFSET, power_domain_info->saved_clos_assocs, - sizeof(power_domain_info->saved_clos_assocs)); + cp_base = pd_info->sst_base + pd_info->sst_header.cp_offset; + writeq(pd_info->saved_sst_cp_control, cp_base + SST_CP_CONTROL_OFFSET); + memcpy_toio(cp_base + SST_CLOS_CONFIG_0_OFFSET, pd_info->saved_clos_configs, + sizeof(pd_info->saved_clos_configs)); + memcpy_toio(cp_base + SST_CLOS_ASSOC_0_OFFSET, pd_info->saved_clos_assocs, + sizeof(pd_info->saved_clos_assocs)); - writeq(power_domain_info->saved_pp_control, power_domain_info->sst_base + - power_domain_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET); + writeq(pd_info->saved_pp_control, power_domain_info->sst_base + + pd_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET); + } } EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_resume, "INTEL_TPMI_SST"); diff --git a/drivers/platform/x86/intel/telemetry/debugfs.c b/drivers/platform/x86/intel/telemetry/debugfs.c index 70e5736c44c71..189c61ff7ff0c 100644 --- a/drivers/platform/x86/intel/telemetry/debugfs.c +++ b/drivers/platform/x86/intel/telemetry/debugfs.c @@ -449,7 +449,7 @@ static int telem_pss_states_show(struct seq_file *s, void *unused) for (index = 0; index < debugfs_conf->pss_ltr_evts; index++) { seq_printf(s, "%-32s\t%u\n", debugfs_conf->pss_ltr_data[index].name, - pss_s0ix_wakeup[index]); + pss_ltr_blkd[index]); } seq_puts(s, "\n--------------------------------------\n"); @@ -459,7 +459,7 @@ static int telem_pss_states_show(struct seq_file *s, void *unused) for (index = 0; index < debugfs_conf->pss_wakeup_evts; index++) { seq_printf(s, "%-32s\t%u\n", debugfs_conf->pss_wakeup[index].name, - pss_ltr_blkd[index]); + pss_s0ix_wakeup[index]); } return 0; diff --git a/drivers/platform/x86/intel/telemetry/pltdrv.c b/drivers/platform/x86/intel/telemetry/pltdrv.c index f23c170a55dc6..d9aa349f81e41 100644 --- a/drivers/platform/x86/intel/telemetry/pltdrv.c +++ b/drivers/platform/x86/intel/telemetry/pltdrv.c @@ -610,7 +610,7 @@ static int telemetry_setup(struct platform_device *pdev) /* Get telemetry Info */ events = (read_buf & TELEM_INFO_SRAMEVTS_MASK) >> TELEM_INFO_SRAMEVTS_SHIFT; - event_regs = read_buf & TELEM_INFO_SRAMEVTS_MASK; + event_regs = read_buf & TELEM_INFO_NENABLES_MASK; if ((events < TELEM_MAX_EVENTS_SRAM) || (event_regs < TELEM_MAX_EVENTS_SRAM)) { dev_err(&pdev->dev, "PSS:Insufficient Space for SRAM Trace\n"); diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c index 1237d95708865..4c7e64db478c0 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c @@ -537,6 +537,7 @@ static void set_cdie_id(int domain_id, struct tpmi_uncore_cluster_info *cluster_ #define UNCORE_VERSION_MASK GENMASK_ULL(7, 0) #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK GENMASK_ULL(15, 8) #define UNCORE_CLUSTER_OFF_MASK GENMASK_ULL(7, 0) +#define UNCORE_AUTONOMOUS_UFS_DISABLED BIT(32) #define UNCORE_MAX_CLUSTER_PER_DOMAIN 8 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id) @@ -598,6 +599,7 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_ for (i = 0; i < num_resources; ++i) { struct tpmi_uncore_power_domain_info *pd_info; + bool auto_ufs_enabled; struct resource *res; u64 cluster_offset; u8 cluster_mask; @@ -647,6 +649,8 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_ continue; } + auto_ufs_enabled = !(header & UNCORE_AUTONOMOUS_UFS_DISABLED); + /* Find out number of clusters in this resource */ pd_info->cluster_count = hweight8(cluster_mask); @@ -689,7 +693,9 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_ cluster_info->uncore_root = tpmi_uncore; - if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) >= UNCORE_ELC_SUPPORTED_VERSION) + if ((TPMI_MINOR_VERSION(pd_info->ufs_header_ver) >= + UNCORE_ELC_SUPPORTED_VERSION) && + auto_ufs_enabled) cluster_info->elc_supported = true; ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0); diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c index cc19fe520ea96..075543cd0e77e 100644 --- a/drivers/platform/x86/lenovo/thinkpad_acpi.c +++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c @@ -9525,14 +9525,16 @@ static int tpacpi_battery_get(int what, int battery, int *ret) { switch (what) { case THRESHOLD_START: - if ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_START, ret, battery)) + if (!battery_info.batteries[battery].start_support || + ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_START, ret, battery))) return -ENODEV; /* The value is in the low 8 bits of the response */ *ret = *ret & 0xFF; return 0; case THRESHOLD_STOP: - if ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_STOP, ret, battery)) + if (!battery_info.batteries[battery].stop_support || + ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_STOP, ret, battery))) return -ENODEV; /* Value is in lower 8 bits */ *ret = *ret & 0xFF; diff --git a/drivers/platform/x86/lenovo/wmi-gamezone.c b/drivers/platform/x86/lenovo/wmi-gamezone.c index 0eb7fe8222f4a..baf1238b4d1d9 100644 --- a/drivers/platform/x86/lenovo/wmi-gamezone.c +++ b/drivers/platform/x86/lenovo/wmi-gamezone.c @@ -31,8 +31,6 @@ #define LWMI_GZ_METHOD_ID_SMARTFAN_SET 44 #define LWMI_GZ_METHOD_ID_SMARTFAN_GET 45 -static BLOCKING_NOTIFIER_HEAD(gz_chain_head); - struct lwmi_gz_priv { enum thermal_mode current_mode; struct notifier_block event_nb; @@ -274,8 +272,23 @@ static const struct dmi_system_id fwbug_list[] = { }, .driver_data = &quirk_no_extreme_bug, }, + { + .ident = "Legion Go 8ASP2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_VERSION, "Legion Go 8ASP2"), + }, + .driver_data = &quirk_no_extreme_bug, + }, + { + .ident = "Legion Go 8AHP2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_VERSION, "Legion Go 8AHP2"), + }, + .driver_data = &quirk_no_extreme_bug, + }, {}, - }; /** diff --git a/drivers/platform/x86/msi-laptop.c b/drivers/platform/x86/msi-laptop.c index c4b150fa093fe..ddef6b78d2fa9 100644 --- a/drivers/platform/x86/msi-laptop.c +++ b/drivers/platform/x86/msi-laptop.c @@ -1130,6 +1130,9 @@ static void __exit msi_cleanup(void) sysfs_remove_group(&msipf_device->dev.kobj, &msipf_attribute_group); if (!quirks->old_ec_model && threeg_exists) device_remove_file(&msipf_device->dev, &dev_attr_threeg); + if (quirks->old_ec_model) + sysfs_remove_group(&msipf_device->dev.kobj, + &msipf_old_attribute_group); platform_device_unregister(msipf_device); platform_driver_unregister(&msipf_driver); backlight_device_unregister(msibl_device); diff --git a/drivers/platform/x86/oxpec.c b/drivers/platform/x86/oxpec.c index 54377b282ff88..a30845ba37969 100644 --- a/drivers/platform/x86/oxpec.c +++ b/drivers/platform/x86/oxpec.c @@ -13,7 +13,7 @@ * * Copyright (C) 2022 Joaquín I. Aramendía * Copyright (C) 2024 Derek J. Clark - * Copyright (C) 2025 Antheas Kapenekakis + * Copyright (C) 2025-2026 Antheas Kapenekakis */ #include @@ -124,6 +124,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)aok_zoe_a1, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "AOKZOE"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "AOKZOE A2 Pro"), + }, + .driver_data = (void *)aok_zoe_a1, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "AOKZOE"), @@ -208,6 +215,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)oxp_2, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "ONEXPLAYER APEX"), + }, + .driver_data = (void *)oxp_fly, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), @@ -278,6 +292,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)oxp_mini_amd_pro, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "ONEXPLAYER X1z"), + }, + .driver_data = (void *)oxp_x1, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), @@ -292,6 +313,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)oxp_x1, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "ONEXPLAYER X1Air"), + }, + .driver_data = (void *)oxp_x1, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ONE-NETBOOK"), diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c index 255317e6fec88..937f1a5b78edf 100644 --- a/drivers/platform/x86/panasonic-laptop.c +++ b/drivers/platform/x86/panasonic-laptop.c @@ -1089,7 +1089,7 @@ static int acpi_pcc_hotkey_add(struct acpi_device *device) PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pcc->platform)) { result = PTR_ERR(pcc->platform); - goto out_backlight; + goto out_sysfs; } result = device_create_file(&pcc->platform->dev, &dev_attr_cdpower); @@ -1105,6 +1105,8 @@ static int acpi_pcc_hotkey_add(struct acpi_device *device) out_platform: platform_device_unregister(pcc->platform); +out_sysfs: + sysfs_remove_group(&device->dev.kobj, &pcc_attr_group); out_backlight: backlight_device_unregister(pcc->backlight); out_input: diff --git a/drivers/platform/x86/samsung-galaxybook.c b/drivers/platform/x86/samsung-galaxybook.c index 3c13e13d48858..755cb82bdb606 100644 --- a/drivers/platform/x86/samsung-galaxybook.c +++ b/drivers/platform/x86/samsung-galaxybook.c @@ -442,12 +442,13 @@ static int galaxybook_battery_ext_property_get(struct power_supply *psy, union power_supply_propval *val) { struct samsung_galaxybook *galaxybook = ext_data; + u8 value; int err; if (psp != POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD) return -EINVAL; - err = charge_control_end_threshold_acpi_get(galaxybook, (u8 *)&val->intval); + err = charge_control_end_threshold_acpi_get(galaxybook, &value); if (err) return err; @@ -455,8 +456,10 @@ static int galaxybook_battery_ext_property_get(struct power_supply *psy, * device stores "no end threshold" as 0 instead of 100; * if device has 0, report 100 */ - if (val->intval == 0) - val->intval = 100; + if (value == 0) + value = 100; + + val->intval = value; return 0; } diff --git a/drivers/platform/x86/toshiba_haps.c b/drivers/platform/x86/toshiba_haps.c index 03dfddeee0c0a..e9324bf16aea4 100644 --- a/drivers/platform/x86/toshiba_haps.c +++ b/drivers/platform/x86/toshiba_haps.c @@ -183,7 +183,7 @@ static int toshiba_haps_add(struct acpi_device *acpi_dev) pr_info("Toshiba HDD Active Protection Sensor device\n"); - haps = kzalloc(sizeof(struct toshiba_haps_dev), GFP_KERNEL); + haps = devm_kzalloc(&acpi_dev->dev, sizeof(*haps), GFP_KERNEL); if (!haps) return -ENOMEM; diff --git a/drivers/platform/x86/touchscreen_dmi.c b/drivers/platform/x86/touchscreen_dmi.c index bdc19cd8d3edf..d83c387821ea1 100644 --- a/drivers/platform/x86/touchscreen_dmi.c +++ b/drivers/platform/x86/touchscreen_dmi.c @@ -410,6 +410,16 @@ static const struct ts_dmi_data gdix1002_upside_down_data = { .properties = gdix1001_upside_down_props, }; +static const struct property_entry gdix1001_y_inverted_props[] = { + PROPERTY_ENTRY_BOOL("touchscreen-inverted-y"), + { } +}; + +static const struct ts_dmi_data gdix1001_y_inverted_data = { + .acpi_name = "GDIX1001", + .properties = gdix1001_y_inverted_props, +}; + static const struct property_entry gp_electronic_t701_props[] = { PROPERTY_ENTRY_U32("touchscreen-size-x", 960), PROPERTY_ENTRY_U32("touchscreen-size-y", 640), @@ -1658,6 +1668,14 @@ const struct dmi_system_id touchscreen_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_SKU, "PN20170413488"), }, }, + { + /* SUPI S10 */ + .driver_data = (void *)&gdix1001_y_inverted_data, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "SUPI"), + DMI_MATCH(DMI_PRODUCT_NAME, "S10"), + }, + }, { /* Techbite Arc 11.6 */ .driver_data = (void *)&techbite_arc_11_6_data, diff --git a/drivers/pmdomain/bcm/bcm2835-power.c b/drivers/pmdomain/bcm/bcm2835-power.c index f5289fd184d0d..f2472f1e17521 100644 --- a/drivers/pmdomain/bcm/bcm2835-power.c +++ b/drivers/pmdomain/bcm/bcm2835-power.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -152,7 +153,6 @@ struct bcm2835_power { static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable) { void __iomem *base = power->asb; - u64 start; u32 val; switch (reg) { @@ -165,8 +165,6 @@ static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable break; } - start = ktime_get_ns(); - /* Enable the module's async AXI bridges. */ if (enable) { val = readl(base + reg) & ~ASB_REQ_STOP; @@ -175,11 +173,9 @@ static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable } writel(PM_PASSWORD | val, base + reg); - while (!!(readl(base + reg) & ASB_ACK) == enable) { - cpu_relax(); - if (ktime_get_ns() - start >= 1000) - return -ETIMEDOUT; - } + if (readl_poll_timeout_atomic(base + reg, val, + !!(val & ASB_ACK) != enable, 0, 5)) + return -ETIMEDOUT; return 0; } @@ -573,11 +569,11 @@ static int bcm2835_reset_status(struct reset_controller_dev *rcdev, switch (id) { case BCM2835_RESET_V3D: - return !PM_READ(PM_GRAFX & PM_V3DRSTN); + return !(PM_READ(PM_GRAFX) & PM_V3DRSTN); case BCM2835_RESET_H264: - return !PM_READ(PM_IMAGE & PM_H264RSTN); + return !(PM_READ(PM_IMAGE) & PM_H264RSTN); case BCM2835_RESET_ISP: - return !PM_READ(PM_IMAGE & PM_ISPRSTN); + return !(PM_READ(PM_IMAGE) & PM_ISPRSTN); default: return -EINVAL; } diff --git a/drivers/pmdomain/governor.c b/drivers/pmdomain/governor.c index 39359811a9304..a46470f2261a9 100644 --- a/drivers/pmdomain/governor.c +++ b/drivers/pmdomain/governor.c @@ -404,15 +404,21 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) if ((idle_duration_ns >= (genpd->states[i].residency_ns + genpd->states[i].power_off_latency_ns)) && (global_constraint >= (genpd->states[i].power_on_latency_ns + - genpd->states[i].power_off_latency_ns))) { - genpd->state_idx = i; - genpd->gd->last_enter = now; - genpd->gd->reflect_residency = true; - return true; - } + genpd->states[i].power_off_latency_ns))) + break; + } while (--i >= 0); - return false; + if (i < 0) + return false; + + if (cpus_peek_for_pending_ipi(genpd->cpus)) + return false; + + genpd->state_idx = i; + genpd->gd->last_enter = now; + genpd->gd->reflect_residency = true; + return true; } struct dev_power_governor pm_domain_cpu_gov = { diff --git a/drivers/pmdomain/imx/gpc.c b/drivers/pmdomain/imx/gpc.c index a34b260274f7b..de695f1944ab3 100644 --- a/drivers/pmdomain/imx/gpc.c +++ b/drivers/pmdomain/imx/gpc.c @@ -402,13 +402,12 @@ static int imx_gpc_old_dt_init(struct device *dev, struct regmap *regmap, static int imx_gpc_probe(struct platform_device *pdev) { const struct imx_gpc_dt_data *of_id_data = device_get_match_data(&pdev->dev); - struct device_node *pgc_node; + struct device_node *pgc_node __free(device_node) + = of_get_child_by_name(pdev->dev.of_node, "pgc"); struct regmap *regmap; void __iomem *base; int ret; - pgc_node = of_get_child_by_name(pdev->dev.of_node, "pgc"); - /* bail out if DT too old and doesn't provide the necessary info */ if (!of_property_present(pdev->dev.of_node, "#power-domain-cells") && !pgc_node) diff --git a/drivers/pmdomain/imx/gpcv2.c b/drivers/pmdomain/imx/gpcv2.c index 105fcaf13a34c..cff738e4d5462 100644 --- a/drivers/pmdomain/imx/gpcv2.c +++ b/drivers/pmdomain/imx/gpcv2.c @@ -165,13 +165,11 @@ #define IMX8M_VPU_HSK_PWRDNREQN BIT(5) #define IMX8M_DISP_HSK_PWRDNREQN BIT(4) -#define IMX8MM_GPUMIX_HSK_PWRDNACKN BIT(29) -#define IMX8MM_GPU_HSK_PWRDNACKN (BIT(27) | BIT(28)) +#define IMX8MM_GPU_HSK_PWRDNACKN GENMASK(29, 27) #define IMX8MM_VPUMIX_HSK_PWRDNACKN BIT(26) #define IMX8MM_DISPMIX_HSK_PWRDNACKN BIT(25) #define IMX8MM_HSIO_HSK_PWRDNACKN (BIT(23) | BIT(24)) -#define IMX8MM_GPUMIX_HSK_PWRDNREQN BIT(11) -#define IMX8MM_GPU_HSK_PWRDNREQN (BIT(9) | BIT(10)) +#define IMX8MM_GPU_HSK_PWRDNREQN GENMASK(11, 9) #define IMX8MM_VPUMIX_HSK_PWRDNREQN BIT(8) #define IMX8MM_DISPMIX_HSK_PWRDNREQN BIT(7) #define IMX8MM_HSIO_HSK_PWRDNREQN (BIT(5) | BIT(6)) @@ -794,8 +792,6 @@ static const struct imx_pgc_domain imx8mm_pgc_domains[] = { .bits = { .pxx = IMX8MM_GPUMIX_SW_Pxx_REQ, .map = IMX8MM_GPUMIX_A53_DOMAIN, - .hskreq = IMX8MM_GPUMIX_HSK_PWRDNREQN, - .hskack = IMX8MM_GPUMIX_HSK_PWRDNACKN, }, .pgc = BIT(IMX8MM_PGC_GPUMIX), .keep_clocks = true, diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c index 5c83e5599f1ea..19e992d2ee3b8 100644 --- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c @@ -340,7 +340,7 @@ static void imx8m_blk_ctrl_remove(struct platform_device *pdev) of_genpd_del_provider(pdev->dev.of_node); - for (i = 0; bc->onecell_data.num_domains; i++) { + for (i = 0; i < bc->onecell_data.num_domains; i++) { struct imx8m_blk_ctrl_domain *domain = &bc->domains[i]; pm_genpd_remove(&domain->genpd); @@ -846,22 +846,25 @@ static int imx8mq_vpu_power_notifier(struct notifier_block *nb, return NOTIFY_OK; } +/* + * For i.MX8MQ, the ADB in the VPUMIX domain has no separate reset and clock + * enable bits, but is ungated and reset together with the VPUs. + * Resetting G1 or G2 separately may led to system hang. + * Remove the rst_mask and clk_mask from the domain data of G1 and G2, + * Let imx8mq_vpu_power_notifier() do really vpu reset. + */ static const struct imx8m_blk_ctrl_domain_data imx8mq_vpu_blk_ctl_domain_data[] = { [IMX8MQ_VPUBLK_PD_G1] = { .name = "vpublk-g1", .clk_names = (const char *[]){ "g1", }, .num_clks = 1, .gpc_name = "g1", - .rst_mask = BIT(1), - .clk_mask = BIT(1), }, [IMX8MQ_VPUBLK_PD_G2] = { .name = "vpublk-g2", .clk_names = (const char *[]){ "g2", }, .num_clks = 1, .gpc_name = "g2", - .rst_mask = BIT(0), - .clk_mask = BIT(0), }, }; diff --git a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c index 34576be606e31..3f5b9499d30a0 100644 --- a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c @@ -53,6 +53,7 @@ struct imx8mp_blk_ctrl_domain_data { const char * const *path_names; int num_paths; const char *gpc_name; + const unsigned int flags; }; #define DOMAIN_MAX_CLKS 3 @@ -65,6 +66,7 @@ struct imx8mp_blk_ctrl_domain { struct icc_bulk_data paths[DOMAIN_MAX_PATHS]; struct device *power_dev; struct imx8mp_blk_ctrl *bc; + struct notifier_block power_nb; int num_paths; int id; }; @@ -264,10 +266,12 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hsio_domain_data[] = { [IMX8MP_HSIOBLK_PD_USB_PHY1] = { .name = "hsioblk-usb-phy1", .gpc_name = "usb-phy1", + .flags = GENPD_FLAG_ACTIVE_WAKEUP, }, [IMX8MP_HSIOBLK_PD_USB_PHY2] = { .name = "hsioblk-usb-phy2", .gpc_name = "usb-phy2", + .flags = GENPD_FLAG_ACTIVE_WAKEUP, }, [IMX8MP_HSIOBLK_PD_PCIE] = { .name = "hsioblk-pcie", @@ -348,9 +352,6 @@ static void imx8mp_hdmi_blk_ctrl_power_on(struct imx8mp_blk_ctrl *bc, regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(12)); regmap_clear_bits(bc->regmap, HDMI_TX_CONTROL0, BIT(3)); break; - case IMX8MP_HDMIBLK_PD_HDCP: - regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11)); - break; case IMX8MP_HDMIBLK_PD_HRV: regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5)); regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15)); @@ -404,9 +405,6 @@ static void imx8mp_hdmi_blk_ctrl_power_off(struct imx8mp_blk_ctrl *bc, regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(7)); regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(22) | BIT(24)); break; - case IMX8MP_HDMIBLK_PD_HDCP: - regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11)); - break; case IMX8MP_HDMIBLK_PD_HRV: regmap_clear_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15)); regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5)); @@ -435,7 +433,7 @@ static int imx8mp_hdmi_power_notifier(struct notifier_block *nb, regmap_write(bc->regmap, HDMI_RTX_CLK_CTL0, 0x0); regmap_write(bc->regmap, HDMI_RTX_CLK_CTL1, 0x0); regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL0, - BIT(0) | BIT(1) | BIT(10)); + BIT(0) | BIT(1) | BIT(10) | BIT(11)); regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(0)); /* @@ -594,6 +592,20 @@ static int imx8mp_blk_ctrl_power_off(struct generic_pm_domain *genpd) return 0; } +static int imx8mp_blk_ctrl_gpc_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct imx8mp_blk_ctrl_domain *domain = + container_of(nb, struct imx8mp_blk_ctrl_domain, power_nb); + + if (action == GENPD_NOTIFY_PRE_OFF) { + if (domain->genpd.status == GENPD_STATE_ON) + return NOTIFY_BAD; + } + + return NOTIFY_OK; +} + static struct lock_class_key blk_ctrl_genpd_lock_class; static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) @@ -698,15 +710,25 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) goto cleanup_pds; } + domain->power_nb.notifier_call = imx8mp_blk_ctrl_gpc_notifier; + ret = dev_pm_genpd_add_notifier(domain->power_dev, &domain->power_nb); + if (ret) { + dev_err_probe(dev, ret, "failed to add power notifier\n"); + dev_pm_domain_detach(domain->power_dev, true); + goto cleanup_pds; + } + domain->genpd.name = data->name; domain->genpd.power_on = imx8mp_blk_ctrl_power_on; domain->genpd.power_off = imx8mp_blk_ctrl_power_off; + domain->genpd.flags = data->flags; domain->bc = bc; domain->id = i; ret = pm_genpd_init(&domain->genpd, NULL, true); if (ret) { dev_err_probe(dev, ret, "failed to init power domain\n"); + dev_pm_genpd_remove_notifier(domain->power_dev); dev_pm_domain_detach(domain->power_dev, true); goto cleanup_pds; } @@ -755,6 +777,7 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) cleanup_pds: for (i--; i >= 0; i--) { pm_genpd_remove(&bc->domains[i].genpd); + dev_pm_genpd_remove_notifier(bc->domains[i].power_dev); dev_pm_domain_detach(bc->domains[i].power_dev, true); } @@ -774,6 +797,7 @@ static void imx8mp_blk_ctrl_remove(struct platform_device *pdev) struct imx8mp_blk_ctrl_domain *domain = &bc->domains[i]; pm_genpd_remove(&domain->genpd); + dev_pm_genpd_remove_notifier(domain->power_dev); dev_pm_domain_detach(domain->power_dev, true); } diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c index 9c9323c8c93a7..269634bcd9a40 100644 --- a/drivers/pmdomain/mediatek/mtk-pm-domains.c +++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c @@ -748,18 +748,6 @@ static void scpsys_domain_cleanup(struct scpsys *scpsys) } } -static struct device_node *scpsys_get_legacy_regmap(struct device_node *np, const char *pn) -{ - struct device_node *local_node; - - for_each_child_of_node(np, local_node) { - if (of_property_present(local_node, pn)) - return local_node; - } - - return NULL; -} - static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *scpsys) { const u8 bp_blocks[3] = { @@ -781,7 +769,8 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s * this makes it then possible to allocate the array of bus_prot * regmaps and convert all to the new style handling. */ - node = scpsys_get_legacy_regmap(np, "mediatek,infracfg"); + of_node_get(np); + node = of_find_node_with_property(np, "mediatek,infracfg"); if (node) { regmap[0] = syscon_regmap_lookup_by_phandle(node, "mediatek,infracfg"); of_node_put(node); @@ -794,7 +783,8 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s regmap[0] = NULL; } - node = scpsys_get_legacy_regmap(np, "mediatek,smi"); + of_node_get(np); + node = of_find_node_with_property(np, "mediatek,smi"); if (node) { smi_np = of_parse_phandle(node, "mediatek,smi", 0); of_node_put(node); @@ -812,7 +802,8 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s regmap[1] = NULL; } - node = scpsys_get_legacy_regmap(np, "mediatek,infracfg-nao"); + of_node_get(np); + node = of_find_node_with_property(np, "mediatek,infracfg-nao"); if (node) { regmap[2] = syscon_regmap_lookup_by_phandle(node, "mediatek,infracfg-nao"); num_regmaps++; diff --git a/drivers/pmdomain/qcom/rpmhpd.c b/drivers/pmdomain/qcom/rpmhpd.c index 4faa8a2561862..19849703be4ac 100644 --- a/drivers/pmdomain/qcom/rpmhpd.c +++ b/drivers/pmdomain/qcom/rpmhpd.c @@ -19,7 +19,7 @@ #define domain_to_rpmhpd(domain) container_of(domain, struct rpmhpd, pd) -#define RPMH_ARC_MAX_LEVELS 16 +#define RPMH_ARC_MAX_LEVELS 32 /** * struct rpmhpd - top level RPMh power domain resource data structure @@ -246,6 +246,8 @@ static struct rpmhpd *sa8540p_rpmhpds[] = { [SC8280XP_MMCX_AO] = &mmcx_ao, [SC8280XP_MX] = &mx, [SC8280XP_MX_AO] = &mx_ao, + [SC8280XP_MXC] = &mxc, + [SC8280XP_MXC_AO] = &mxc_ao, [SC8280XP_NSP] = &nsp, }; @@ -595,6 +597,31 @@ static const struct rpmhpd_desc sm8750_desc = { .num_pds = ARRAY_SIZE(sm8750_rpmhpds), }; +/* KAANAPALI RPMH powerdomains */ +static struct rpmhpd *kaanapali_rpmhpds[] = { + [RPMHPD_CX] = &cx, + [RPMHPD_CX_AO] = &cx_ao, + [RPMHPD_EBI] = &ebi, + [RPMHPD_GFX] = &gfx, + [RPMHPD_GMXC] = &gmxc, + [RPMHPD_LCX] = &lcx, + [RPMHPD_LMX] = &lmx, + [RPMHPD_MX] = &mx, + [RPMHPD_MX_AO] = &mx_ao, + [RPMHPD_MMCX] = &mmcx, + [RPMHPD_MMCX_AO] = &mmcx_ao, + [RPMHPD_MSS] = &mss, + [RPMHPD_MXC] = &mxc, + [RPMHPD_MXC_AO] = &mxc_ao, + [RPMHPD_NSP] = &nsp, + [RPMHPD_NSP2] = &nsp2, +}; + +static const struct rpmhpd_desc kaanapali_desc = { + .rpmhpds = kaanapali_rpmhpds, + .num_pds = ARRAY_SIZE(kaanapali_rpmhpds), +}; + /* QDU1000/QRU1000 RPMH powerdomains */ static struct rpmhpd *qdu1000_rpmhpds[] = { [QDU1000_CX] = &cx, @@ -675,6 +702,8 @@ static struct rpmhpd *sc8280xp_rpmhpds[] = { [SC8280XP_MMCX_AO] = &mmcx_ao, [SC8280XP_MX] = &mx, [SC8280XP_MX_AO] = &mx_ao, + [SC8280XP_MXC] = &mxc, + [SC8280XP_MXC_AO] = &mxc_ao, [SC8280XP_NSP] = &nsp, [SC8280XP_QPHY] = &qphy, }; @@ -767,6 +796,7 @@ static const struct rpmhpd_desc qcs615_desc = { static const struct of_device_id rpmhpd_match_table[] = { { .compatible = "qcom,glymur-rpmhpd", .data = &glymur_desc }, + { .compatible = "qcom,kaanapali-rpmhpd", .data = &kaanapali_desc }, { .compatible = "qcom,milos-rpmhpd", .data = &milos_desc }, { .compatible = "qcom,qcs615-rpmhpd", .data = &qcs615_desc }, { .compatible = "qcom,qcs8300-rpmhpd", .data = &qcs8300_desc }, diff --git a/drivers/pmdomain/qcom/rpmpd.c b/drivers/pmdomain/qcom/rpmpd.c index f8580ec0f7378..98ab4f9ea9bff 100644 --- a/drivers/pmdomain/qcom/rpmpd.c +++ b/drivers/pmdomain/qcom/rpmpd.c @@ -1001,7 +1001,7 @@ static int rpmpd_aggregate_corner(struct rpmpd *pd) /* Clamp to the highest corner/level if sync_state isn't done yet */ if (!pd->state_synced) - this_active_corner = this_sleep_corner = pd->max_state - 1; + this_active_corner = this_sleep_corner = pd->max_state; else to_active_sleep(pd, pd->corner, &this_active_corner, &this_sleep_corner); diff --git a/drivers/pmdomain/rockchip/pm-domains.c b/drivers/pmdomain/rockchip/pm-domains.c index 1955c6d453e4f..6e4a29ee439a4 100644 --- a/drivers/pmdomain/rockchip/pm-domains.c +++ b/drivers/pmdomain/rockchip/pm-domains.c @@ -861,6 +861,16 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu, pd->genpd.name = pd->info->name; else pd->genpd.name = kbasename(node->full_name); + + /* + * power domain's needing a regulator should default to off, since + * the regulator state is unknown at probe time. Also the regulator + * state cannot be checked, since that usually requires IP needing + * (a different) power domain. + */ + if (pd->info->need_regulator) + rockchip_pd_power(pd, false); + pd->genpd.power_off = rockchip_pd_power_off; pd->genpd.power_on = rockchip_pd_power_on; pd->genpd.attach_dev = rockchip_pd_attach_dev; @@ -1276,7 +1286,7 @@ static const struct rockchip_domain_info rk3576_pm_domains[] = { static const struct rockchip_domain_info rk3588_pm_domains[] = { [RK3588_PD_GPU] = DOMAIN_RK3588("gpu", 0x0, BIT(0), 0, 0x0, 0, BIT(1), 0x0, BIT(0), BIT(0), false, true), [RK3588_PD_NPU] = DOMAIN_RK3588("npu", 0x0, BIT(1), BIT(1), 0x0, 0, 0, 0x0, 0, 0, false, true), - [RK3588_PD_VCODEC] = DOMAIN_RK3588("vcodec", 0x0, BIT(2), BIT(2), 0x0, 0, 0, 0x0, 0, 0, false, false), + [RK3588_PD_VCODEC] = DOMAIN_RK3588("vcodec", 0x0, BIT(2), BIT(2), 0x0, 0, 0, 0x0, 0, 0, false, true), [RK3588_PD_NPUTOP] = DOMAIN_RK3588("nputop", 0x0, BIT(3), 0, 0x0, BIT(11), BIT(2), 0x0, BIT(1), BIT(1), false, false), [RK3588_PD_NPU1] = DOMAIN_RK3588("npu1", 0x0, BIT(4), 0, 0x0, BIT(12), BIT(3), 0x0, BIT(2), BIT(2), false, false), [RK3588_PD_NPU2] = DOMAIN_RK3588("npu2", 0x0, BIT(5), 0, 0x0, BIT(13), BIT(4), 0x0, BIT(3), BIT(3), false, false), diff --git a/drivers/power/reset/nvmem-reboot-mode.c b/drivers/power/reset/nvmem-reboot-mode.c index 41530b70cfc48..49f9777dfc115 100644 --- a/drivers/power/reset/nvmem-reboot-mode.c +++ b/drivers/power/reset/nvmem-reboot-mode.c @@ -10,21 +10,34 @@ #include #include #include +#include struct nvmem_reboot_mode { struct reboot_mode_driver reboot; struct nvmem_cell *cell; }; -static int nvmem_reboot_mode_write(struct reboot_mode_driver *reboot, - unsigned int magic) +static int nvmem_reboot_mode_write(struct reboot_mode_driver *reboot, u64 magic) { - int ret; struct nvmem_reboot_mode *nvmem_rbm; + u32 magic_32; + size_t buf_len; + void *buf; + int ret; nvmem_rbm = container_of(reboot, struct nvmem_reboot_mode, reboot); - ret = nvmem_cell_write(nvmem_rbm->cell, &magic, sizeof(magic)); + buf = nvmem_cell_read(nvmem_rbm->cell, &buf_len); + if (IS_ERR(buf)) + return PTR_ERR(buf); + kfree(buf); + + if (buf_len > sizeof(magic)) + return -EINVAL; + + magic_32 = magic; + + ret = nvmem_cell_write(nvmem_rbm->cell, &magic_32, buf_len); if (ret < 0) dev_err(reboot->dev, "update reboot mode bits failed\n"); diff --git a/drivers/power/reset/qcom-pon.c b/drivers/power/reset/qcom-pon.c index 7e108982a582e..d0ed9431a0231 100644 --- a/drivers/power/reset/qcom-pon.c +++ b/drivers/power/reset/qcom-pon.c @@ -27,17 +27,22 @@ struct qcom_pon { long reason_shift; }; -static int qcom_pon_reboot_mode_write(struct reboot_mode_driver *reboot, - unsigned int magic) +static int qcom_pon_reboot_mode_write(struct reboot_mode_driver *reboot, u64 magic) { struct qcom_pon *pon = container_of (reboot, struct qcom_pon, reboot_mode); + u32 magic_32; int ret; + if (magic > U32_MAX || (magic << pon->reason_shift) > U32_MAX) + return -EINVAL; + + magic_32 = magic << pon->reason_shift; + ret = regmap_update_bits(pon->regmap, pon->baseaddr + PON_SOFT_RB_SPARE, GENMASK(7, pon->reason_shift), - magic << pon->reason_shift); + magic_32); if (ret < 0) dev_err(pon->dev, "update reboot mode bits failed\n"); diff --git a/drivers/power/reset/reboot-mode.c b/drivers/power/reset/reboot-mode.c index fba53f638da04..1e78eb3d0fe51 100644 --- a/drivers/power/reset/reboot-mode.c +++ b/drivers/power/reset/reboot-mode.c @@ -3,24 +3,30 @@ * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd */ +#define pr_fmt(fmt) "reboot-mode: " fmt + #include +#include #include #include +#include #include #include #include #include +#include #define PREFIX "mode-" struct mode_info { const char *mode; - u32 magic; + u64 magic; struct list_head list; }; -static unsigned int get_reboot_mode_magic(struct reboot_mode_driver *reboot, - const char *cmd) +static struct class *rb_class; + +static u64 get_reboot_mode_magic(struct reboot_mode_driver *reboot, const char *cmd) { const char *normal = "normal"; struct mode_info *info; @@ -29,9 +35,11 @@ static unsigned int get_reboot_mode_magic(struct reboot_mode_driver *reboot, if (!cmd) cmd = normal; - list_for_each_entry(info, &reboot->head, list) - if (!strcmp(info->mode, cmd)) - return info->magic; + scoped_guard(mutex, &reboot->rb_lock) { + list_for_each_entry(info, &reboot->head, list) + if (!strcmp(info->mode, cmd)) + return info->magic; + } /* try to match again, replacing characters impossible in DT */ if (strscpy(cmd_, cmd, sizeof(cmd_)) == -E2BIG) @@ -41,9 +49,11 @@ static unsigned int get_reboot_mode_magic(struct reboot_mode_driver *reboot, strreplace(cmd_, ',', '-'); strreplace(cmd_, '/', '-'); - list_for_each_entry(info, &reboot->head, list) - if (!strcmp(info->mode, cmd_)) - return info->magic; + scoped_guard(mutex, &reboot->rb_lock) { + list_for_each_entry(info, &reboot->head, list) + if (!strcmp(info->mode, cmd_)) + return info->magic; + } return 0; } @@ -52,7 +62,7 @@ static int reboot_mode_notify(struct notifier_block *this, unsigned long mode, void *cmd) { struct reboot_mode_driver *reboot; - unsigned int magic; + u64 magic; reboot = container_of(this, struct reboot_mode_driver, reboot_notifier); magic = get_reboot_mode_magic(reboot, cmd); @@ -62,52 +72,159 @@ static int reboot_mode_notify(struct notifier_block *this, return NOTIFY_DONE; } +static void release_reboot_mode_device(struct device *dev, void *res); + +static ssize_t reboot_modes_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct reboot_mode_driver **devres_reboot; + struct reboot_mode_driver *reboot; + struct mode_info *info; + ssize_t size = 0; + + devres_reboot = devres_find(dev, release_reboot_mode_device, NULL, NULL); + if (!devres_reboot || !(*devres_reboot)) + return -ENODATA; + + reboot = *devres_reboot; + scoped_guard(mutex, &reboot->rb_lock) { + list_for_each_entry(info, &reboot->head, list) + size += sysfs_emit_at(buf, size, "%s ", info->mode); + } + + if (size) { + size += sysfs_emit_at(buf, size - 1, "\n"); + return size; + } + + return -ENODATA; +} +static DEVICE_ATTR_RO(reboot_modes); + +static void release_reboot_mode_device(struct device *dev, void *res) +{ + struct reboot_mode_driver *reboot = *(struct reboot_mode_driver **)res; + struct mode_info *info; + struct mode_info *next; + + unregister_reboot_notifier(&reboot->reboot_notifier); + + scoped_guard(mutex, &reboot->rb_lock) { + list_for_each_entry_safe(info, next, &reboot->head, list) { + list_del(&info->list); + kfree_const(info->mode); + kfree(info); + } + } + + device_remove_file(reboot->reboot_dev, &dev_attr_reboot_modes); +} + +static int create_reboot_mode_device(struct reboot_mode_driver *reboot) +{ + struct reboot_mode_driver **dr; + int ret = 0; + + if (!rb_class) { + rb_class = class_create("reboot-mode"); + if (IS_ERR(rb_class)) + return PTR_ERR(rb_class); + } + + reboot->reboot_dev = device_create(rb_class, NULL, 0, NULL, reboot->driver_name); + if (IS_ERR(reboot->reboot_dev)) + return PTR_ERR(reboot->reboot_dev); + + ret = device_create_file(reboot->reboot_dev, &dev_attr_reboot_modes); + if (ret) + goto create_file_err; + + dr = devres_alloc(release_reboot_mode_device, sizeof(*dr), GFP_KERNEL); + if (!dr) { + ret = -ENOMEM; + goto devres_alloc_error; + } + + *dr = reboot; + devres_add(reboot->reboot_dev, dr); + return ret; + +devres_alloc_error: + device_remove_file(reboot->reboot_dev, &dev_attr_reboot_modes); +create_file_err: + device_unregister(reboot->reboot_dev); + return ret; +} + /** * reboot_mode_register - register a reboot mode driver * @reboot: reboot mode driver + * @fwnode: Firmware node with reboot-mode configuration * * Returns: 0 on success or a negative error code on failure. */ -int reboot_mode_register(struct reboot_mode_driver *reboot) +int reboot_mode_register(struct reboot_mode_driver *reboot, struct fwnode_handle *fwnode) { struct mode_info *info; + struct device_node *np; struct property *prop; - struct device_node *np = reboot->dev->of_node; size_t len = strlen(PREFIX); + u32 magic_arg1; + u32 magic_arg2; int ret; - INIT_LIST_HEAD(&reboot->head); + if (!fwnode || !reboot->driver_name) + return -EINVAL; - for_each_property_of_node(np, prop) { - if (strncmp(prop->name, PREFIX, len)) - continue; + np = to_of_node(fwnode); + if (!np) + return -EINVAL; - info = devm_kzalloc(reboot->dev, sizeof(*info), GFP_KERNEL); - if (!info) { - ret = -ENOMEM; - goto error; - } + ret = create_reboot_mode_device(reboot); + if (ret) + return ret; - if (of_property_read_u32(np, prop->name, &info->magic)) { - dev_err(reboot->dev, "reboot mode %s without magic number\n", - info->mode); - devm_kfree(reboot->dev, info); - continue; - } + INIT_LIST_HEAD(&reboot->head); - info->mode = kstrdup_const(prop->name + len, GFP_KERNEL); - if (!info->mode) { - ret = -ENOMEM; - goto error; - } else if (info->mode[0] == '\0') { - kfree_const(info->mode); - ret = -EINVAL; - dev_err(reboot->dev, "invalid mode name(%s): too short!\n", - prop->name); - goto error; + mutex_init(&reboot->rb_lock); + + scoped_guard(mutex, &reboot->rb_lock) { + for_each_property_of_node(np, prop) { + if (strncmp(prop->name, PREFIX, len)) + continue; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + ret = -ENOMEM; + goto error; + } + + if (of_property_read_u32(np, prop->name, &magic_arg1)) { + pr_err("reboot mode without magic number\n"); + kfree(info); + continue; + } + + if (of_property_read_u32_index(np, prop->name, 1, &magic_arg2)) + magic_arg2 = 0; + + info->magic = magic_arg2; + info->magic = (info->magic << 32) | magic_arg1; + + info->mode = kstrdup_const(prop->name + len, GFP_KERNEL); + if (!info->mode) { + ret = -ENOMEM; + kfree(info); + goto error; + } else if (info->mode[0] == '\0') { + kfree_const(info->mode); + kfree(info); + ret = -EINVAL; + pr_err("invalid mode name(%s): too short!\n", prop->name); + goto error; + } + + list_add_tail(&info->list, &reboot->head); } - - list_add_tail(&info->list, &reboot->head); } reboot->reboot_notifier.notifier_call = reboot_mode_notify; @@ -116,9 +233,7 @@ int reboot_mode_register(struct reboot_mode_driver *reboot) return 0; error: - list_for_each_entry(info, &reboot->head, list) - kfree_const(info->mode); - + device_unregister(reboot->reboot_dev); return ret; } EXPORT_SYMBOL_GPL(reboot_mode_register); @@ -129,20 +244,16 @@ EXPORT_SYMBOL_GPL(reboot_mode_register); */ int reboot_mode_unregister(struct reboot_mode_driver *reboot) { - struct mode_info *info; - - unregister_reboot_notifier(&reboot->reboot_notifier); - - list_for_each_entry(info, &reboot->head, list) - kfree_const(info->mode); - + device_unregister(reboot->reboot_dev); return 0; } EXPORT_SYMBOL_GPL(reboot_mode_unregister); static void devm_reboot_mode_release(struct device *dev, void *res) { - reboot_mode_unregister(*(struct reboot_mode_driver **)res); + struct reboot_mode_driver *reboot = *(struct reboot_mode_driver **)res; + + device_unregister(reboot->reboot_dev); } /** @@ -158,11 +269,15 @@ int devm_reboot_mode_register(struct device *dev, struct reboot_mode_driver **dr; int rc; + if (!reboot->dev || !reboot->dev->of_node) + return -EINVAL; + dr = devres_alloc(devm_reboot_mode_release, sizeof(*dr), GFP_KERNEL); if (!dr) return -ENOMEM; - rc = reboot_mode_register(reboot); + reboot->driver_name = reboot->dev->driver->name; + rc = reboot_mode_register(reboot, of_fwnode_handle(reboot->dev->of_node)); if (rc) { devres_free(dr); return rc; diff --git a/drivers/power/reset/syscon-reboot-mode.c b/drivers/power/reset/syscon-reboot-mode.c index e0772c9f70f7a..3cbd000c51223 100644 --- a/drivers/power/reset/syscon-reboot-mode.c +++ b/drivers/power/reset/syscon-reboot-mode.c @@ -20,16 +20,21 @@ struct syscon_reboot_mode { u32 mask; }; -static int syscon_reboot_mode_write(struct reboot_mode_driver *reboot, - unsigned int magic) +static int syscon_reboot_mode_write(struct reboot_mode_driver *reboot, u64 magic) { struct syscon_reboot_mode *syscon_rbm; + u32 magic_32; int ret; + if (magic > U32_MAX) + return -EINVAL; + + magic_32 = magic; + syscon_rbm = container_of(reboot, struct syscon_reboot_mode, reboot); ret = regmap_update_bits(syscon_rbm->map, syscon_rbm->offset, - syscon_rbm->mask, magic); + syscon_rbm->mask, magic_32); if (ret < 0) dev_err(reboot->dev, "update reboot mode bits failed\n"); diff --git a/drivers/power/reset/tdx-ec-poweroff.c b/drivers/power/reset/tdx-ec-poweroff.c index 3302a127fce52..8040aa03d74d4 100644 --- a/drivers/power/reset/tdx-ec-poweroff.c +++ b/drivers/power/reset/tdx-ec-poweroff.c @@ -8,7 +8,10 @@ */ #include +#include +#include #include +#include #include #include #include @@ -31,6 +34,8 @@ #define EC_REG_MAX 0xD0 +#define EC_CMD_TIMEOUT_MS 1000 + static const struct regmap_range volatile_ranges[] = { regmap_reg_range(EC_CMD_REG, EC_CMD_REG), }; @@ -75,6 +80,13 @@ static int tdx_ec_power_off(struct sys_off_data *data) err = tdx_ec_cmd(regmap, EC_CMD_POWEROFF); + if (err) { + dev_err(data->dev, "Failed to send power off command\n"); + } else { + mdelay(EC_CMD_TIMEOUT_MS); + WARN_ONCE(1, "Unable to power off system\n"); + } + return err ? NOTIFY_BAD : NOTIFY_DONE; } @@ -85,6 +97,13 @@ static int tdx_ec_restart(struct sys_off_data *data) err = tdx_ec_cmd(regmap, EC_CMD_RESET); + if (err) { + dev_err(data->dev, "Failed to send restart command\n"); + } else { + mdelay(EC_CMD_TIMEOUT_MS); + WARN_ONCE(1, "Unable to restart system\n"); + } + return err ? NOTIFY_BAD : NOTIFY_DONE; } diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c index 190564e559885..1fcf0af7cc0bb 100644 --- a/drivers/power/sequencing/core.c +++ b/drivers/power/sequencing/core.c @@ -914,8 +914,10 @@ int pwrseq_power_on(struct pwrseq_desc *desc) if (target->post_enable) { ret = target->post_enable(pwrseq); if (ret) { - pwrseq_unit_disable(pwrseq, unit); - desc->powered_on = false; + scoped_guard(mutex, &pwrseq->state_lock) { + pwrseq_unit_disable(pwrseq, unit); + desc->powered_on = false; + } } } diff --git a/drivers/power/sequencing/pwrseq-qcom-wcn.c b/drivers/power/sequencing/pwrseq-qcom-wcn.c index 663d9a5370653..916fa056025f5 100644 --- a/drivers/power/sequencing/pwrseq-qcom-wcn.c +++ b/drivers/power/sequencing/pwrseq-qcom-wcn.c @@ -357,6 +357,20 @@ static int pwrseq_qcom_wcn_match(struct pwrseq_device *pwrseq, reg_node->parent->parent != ctx->of_node) return PWRSEQ_NO_MATCH; + /* + * If this is a Bluetooth consumer device but the bt-enable GPIO is not + * configured in the power sequencer (e.g. BT_EN is tied high via a + * hardware pull-up and therefore absent from the DT), don't match. + * The consumer driver will fall back to its legacy power control path + * and correctly set power_ctrl_enabled to false. + * + * BT device nodes are conventionally named "bluetooth" in the DT, + * so use of_node_name_eq() as a generic check rather than enumerating + * specific compatible strings. + */ + if (!ctx->bt_gpio && of_node_name_eq(dev_node, "bluetooth")) + return PWRSEQ_NO_MATCH; + return PWRSEQ_MATCH_OK; } diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c index 5f4537766e5b9..1813fbdfa1c1f 100644 --- a/drivers/power/supply/ab8500_charger.c +++ b/drivers/power/supply/ab8500_charger.c @@ -3466,26 +3466,6 @@ static int ab8500_charger_probe(struct platform_device *pdev) return ret; } - /* Request interrupts */ - for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { - irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, - irq, NULL, ab8500_charger_irq[i].isr, - IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, - ab8500_charger_irq[i].name, di); - - if (ret != 0) { - dev_err(dev, "failed to request %s IRQ %d: %d\n" - , ab8500_charger_irq[i].name, irq, ret); - return ret; - } - dev_dbg(dev, "Requested %s IRQ %d: %d\n", - ab8500_charger_irq[i].name, irq, ret); - } - /* initialize lock */ spin_lock_init(&di->usb_state.usb_lock); mutex_init(&di->usb_ipt_crnt_lock); @@ -3614,6 +3594,26 @@ static int ab8500_charger_probe(struct platform_device *pdev) return PTR_ERR(di->usb_chg.psy); } + /* Request interrupts */ + for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { + irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, + irq, NULL, ab8500_charger_irq[i].isr, + IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, + ab8500_charger_irq[i].name, di); + + if (ret != 0) { + dev_err(dev, "failed to request %s IRQ %d: %d\n" + , ab8500_charger_irq[i].name, irq, ret); + return ret; + } + dev_dbg(dev, "Requested %s IRQ %d: %d\n", + ab8500_charger_irq[i].name, irq, ret); + } + /* * Check what battery we have, since we always have the USB * psy, use that as a handle. diff --git a/drivers/power/supply/act8945a_charger.c b/drivers/power/supply/act8945a_charger.c index 3901a02f326a5..9dec4486b1439 100644 --- a/drivers/power/supply/act8945a_charger.c +++ b/drivers/power/supply/act8945a_charger.c @@ -597,14 +597,6 @@ static int act8945a_charger_probe(struct platform_device *pdev) return irq ?: -ENXIO; } - ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, - IRQF_TRIGGER_FALLING, "act8945a_interrupt", - charger); - if (ret) { - dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); - return ret; - } - charger->desc.name = "act8945a-charger"; charger->desc.get_property = act8945a_charger_get_property; charger->desc.properties = act8945a_charger_props; @@ -625,6 +617,14 @@ static int act8945a_charger_probe(struct platform_device *pdev) return PTR_ERR(charger->psy); } + ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, + IRQF_TRIGGER_FALLING, "act8945a_interrupt", + charger); + if (ret) { + dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); + return ret; + } + platform_set_drvdata(pdev, charger); INIT_WORK(&charger->work, act8945a_work); diff --git a/drivers/power/supply/apm_power.c b/drivers/power/supply/apm_power.c index 9236e00785786..9933cdc5c387a 100644 --- a/drivers/power/supply/apm_power.c +++ b/drivers/power/supply/apm_power.c @@ -364,7 +364,8 @@ static int __init apm_battery_init(void) static void __exit apm_battery_exit(void) { - apm_get_power_status = NULL; + if (apm_get_power_status == apm_battery_apm_get_power_status) + apm_get_power_status = NULL; } module_init(apm_battery_init); diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c index ae14162f017a9..d3de4f8b80db1 100644 --- a/drivers/power/supply/bq256xx_charger.c +++ b/drivers/power/supply/bq256xx_charger.c @@ -1741,6 +1741,12 @@ static int bq256xx_probe(struct i2c_client *client) usb_register_notifier(bq->usb3_phy, &bq->usb_nb); } + ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq256xx_irq_handler_thread, @@ -1753,12 +1759,6 @@ static int bq256xx_probe(struct i2c_client *client) } } - ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq256xx_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq25980_charger.c b/drivers/power/supply/bq25980_charger.c index 723858d62d141..73f06f09f134c 100644 --- a/drivers/power/supply/bq25980_charger.c +++ b/drivers/power/supply/bq25980_charger.c @@ -1241,6 +1241,12 @@ static int bq25980_probe(struct i2c_client *client) return ret; } + ret = bq25980_power_supply_init(bq, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq25980_irq_handler_thread, @@ -1251,12 +1257,6 @@ static int bq25980_probe(struct i2c_client *client) return ret; } - ret = bq25980_power_supply_init(bq, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq25980_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 19445e39651c7..45f0e39b8c2dd 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1172,7 +1172,7 @@ static inline int bq27xxx_write(struct bq27xxx_device_info *di, int reg_index, return -EINVAL; if (!di->bus.write) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write(di, di->regs[reg_index], value, single); if (ret < 0) @@ -1191,7 +1191,7 @@ static inline int bq27xxx_read_block(struct bq27xxx_device_info *di, int reg_ind return -EINVAL; if (!di->bus.read_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.read_bulk(di, di->regs[reg_index], data, len); if (ret < 0) @@ -1210,7 +1210,7 @@ static inline int bq27xxx_write_block(struct bq27xxx_device_info *di, int reg_in return -EINVAL; if (!di->bus.write_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write_bulk(di, di->regs[reg_index], data, len); if (ret < 0) diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index 8106d1edcbc26..507fdc1c866d5 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -1122,10 +1122,6 @@ static int cpcap_battery_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ddata); - error = cpcap_battery_init_interrupts(pdev, ddata); - if (error) - return error; - error = cpcap_battery_init_iio(ddata); if (error) return error; @@ -1142,6 +1138,10 @@ static int cpcap_battery_probe(struct platform_device *pdev) return error; } + error = cpcap_battery_init_interrupts(pdev, ddata); + if (error) + return error; + atomic_set(&ddata->active, 1); error = cpcap_battery_calibrate(ddata); diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c index 2263d5d3448fd..0806abea2372f 100644 --- a/drivers/power/supply/cw2015_battery.c +++ b/drivers/power/supply/cw2015_battery.c @@ -699,7 +699,13 @@ static int cw_bat_probe(struct i2c_client *client) if (!cw_bat->battery_workqueue) return -ENOMEM; - devm_delayed_work_autocancel(&client->dev, &cw_bat->battery_delay_work, cw_bat_work); + ret = devm_delayed_work_autocancel(&client->dev, &cw_bat->battery_delay_work, cw_bat_work); + if (ret) { + dev_err_probe(&client->dev, ret, + "Failed to register delayed work\n"); + return ret; + } + queue_delayed_work(cw_bat->battery_workqueue, &cw_bat->battery_delay_work, msecs_to_jiffies(10)); return 0; diff --git a/drivers/power/supply/goldfish_battery.c b/drivers/power/supply/goldfish_battery.c index 479195e35d734..5aa24e4dc4455 100644 --- a/drivers/power/supply/goldfish_battery.c +++ b/drivers/power/supply/goldfish_battery.c @@ -224,12 +224,6 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (data->irq < 0) return -ENODEV; - ret = devm_request_irq(&pdev->dev, data->irq, - goldfish_battery_interrupt, - IRQF_SHARED, pdev->name, data); - if (ret) - return ret; - psy_cfg.drv_data = data; data->ac = devm_power_supply_register(&pdev->dev, @@ -244,6 +238,12 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (IS_ERR(data->battery)) return PTR_ERR(data->battery); + ret = devm_request_irq(&pdev->dev, data->irq, + goldfish_battery_interrupt, + IRQF_SHARED, pdev->name, data); + if (ret) + return ret; + GOLDFISH_BATTERY_WRITE(data, BATTERY_INT_ENABLE, BATTERY_INT_MASK); return 0; } diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c index c1640bc6accd2..48453508688a4 100644 --- a/drivers/power/supply/max17040_battery.c +++ b/drivers/power/supply/max17040_battery.c @@ -388,6 +388,7 @@ static int max17040_get_property(struct power_supply *psy, union power_supply_propval *val) { struct max17040_chip *chip = power_supply_get_drvdata(psy); + int ret; switch (psp) { case POWER_SUPPLY_PROP_ONLINE: @@ -410,7 +411,10 @@ static int max17040_get_property(struct power_supply *psy, if (!chip->channel_temp) return -ENODATA; - iio_read_channel_processed(chip->channel_temp, &val->intval); + ret = iio_read_channel_processed(chip->channel_temp, &val->intval); + if (ret) + return ret; + val->intval /= 100; /* Convert from milli- to deci-degree */ break; diff --git a/drivers/power/supply/max77705_charger.c b/drivers/power/supply/max77705_charger.c index b1a227bf72e26..1044bf58aeaca 100644 --- a/drivers/power/supply/max77705_charger.c +++ b/drivers/power/supply/max77705_charger.c @@ -60,7 +60,7 @@ static const struct regmap_irq max77705_charger_irqs[] = { REGMAP_IRQ_REG_LINE(MAX77705_AICL_I, BITS_PER_BYTE), }; -static struct regmap_irq_chip max77705_charger_irq_chip = { +static const struct regmap_irq_chip max77705_charger_irq_chip = { .name = "max77705-charger", .status_base = MAX77705_CHG_REG_INT, .mask_base = MAX77705_CHG_REG_INT_MASK, @@ -567,6 +567,7 @@ static int max77705_charger_probe(struct i2c_client *i2c) { struct power_supply_config pscfg = {}; struct max77705_charger_data *chg; + struct regmap_irq_chip *chip_desc; struct device *dev; struct regmap_irq_chip_data *irq_data; int ret; @@ -580,6 +581,13 @@ static int max77705_charger_probe(struct i2c_client *i2c) chg->dev = dev; i2c_set_clientdata(i2c, chg); + chip_desc = devm_kmemdup(dev, &max77705_charger_irq_chip, + sizeof(max77705_charger_irq_chip), + GFP_KERNEL); + if (!chip_desc) + return -ENOMEM; + chip_desc->irq_drv_data = chg; + chg->regmap = devm_regmap_init_i2c(i2c, &max77705_chg_regmap_config); if (IS_ERR(chg->regmap)) return PTR_ERR(chg->regmap); @@ -599,11 +607,9 @@ static int max77705_charger_probe(struct i2c_client *i2c) if (IS_ERR(chg->psy_chg)) return PTR_ERR(chg->psy_chg); - max77705_charger_irq_chip.irq_drv_data = chg; ret = devm_regmap_add_irq_chip(chg->dev, chg->regmap, i2c->irq, IRQF_ONESHOT, 0, - &max77705_charger_irq_chip, - &irq_data); + chip_desc, &irq_data); if (ret) return dev_err_probe(dev, ret, "failed to add irq chip\n"); diff --git a/drivers/power/supply/pm8916_bms_vm.c b/drivers/power/supply/pm8916_bms_vm.c index 5120be086e6ff..de5d571c03e21 100644 --- a/drivers/power/supply/pm8916_bms_vm.c +++ b/drivers/power/supply/pm8916_bms_vm.c @@ -167,15 +167,6 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret < 0) return -EINVAL; - irq = platform_get_irq_byname(pdev, "fifo"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, - IRQF_ONESHOT, "pm8916_vm_bms", bat); - if (ret) - return ret; - ret = regmap_bulk_read(bat->regmap, bat->reg + PM8916_PERPH_TYPE, &tmp, 2); if (ret) goto comm_error; @@ -220,6 +211,15 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "fifo"); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, + IRQF_ONESHOT, "pm8916_vm_bms", bat); + if (ret) + return ret; + platform_set_drvdata(pdev, bat); return 0; diff --git a/drivers/power/supply/pm8916_lbc.c b/drivers/power/supply/pm8916_lbc.c index c74b75b1b2676..6b631012a7959 100644 --- a/drivers/power/supply/pm8916_lbc.c +++ b/drivers/power/supply/pm8916_lbc.c @@ -274,15 +274,6 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) return dev_err_probe(dev, -EINVAL, "Wrong amount of reg values: %d (4 expected)\n", len); - irq = platform_get_irq_byname(pdev, "usb_vbus"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, - IRQF_ONESHOT, "pm8916_lbc", chg); - if (ret) - return ret; - ret = device_property_read_u32_array(dev, "reg", chg->reg, len); if (ret) return ret; @@ -332,6 +323,10 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "usb_vbus"); + if (irq < 0) + return irq; + chg->edev = devm_extcon_dev_allocate(dev, pm8916_lbc_charger_cable); if (IS_ERR(chg->edev)) return PTR_ERR(chg->edev); @@ -340,6 +335,11 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret < 0) return dev_err_probe(dev, ret, "failed to register extcon device\n"); + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, + IRQF_ONESHOT, "pm8916_lbc", chg); + if (ret) + return ret; + ret = regmap_read(chg->regmap, chg->reg[LBC_USB] + PM8916_INT_RT_STS, &tmp); if (ret) goto comm_error; diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c index 3c2837ef34617..ff77dba29a3ef 100644 --- a/drivers/power/supply/qcom_battmgr.c +++ b/drivers/power/supply/qcom_battmgr.c @@ -257,6 +257,7 @@ struct qcom_battmgr_info { unsigned int capacity_warning; unsigned int cycle_count; unsigned int charge_count; + bool charge_ctrl_enable; unsigned int charge_ctrl_start; unsigned int charge_ctrl_end; char model_number[BATTMGR_STRING_LEN]; @@ -659,13 +660,13 @@ static int qcom_battmgr_bat_get_property(struct power_supply *psy, } static int qcom_battmgr_set_charge_control(struct qcom_battmgr *battmgr, - u32 target_soc, u32 delta_soc) + bool enable, u32 target_soc, u32 delta_soc) { struct qcom_battmgr_charge_ctrl_request request = { .hdr.owner = cpu_to_le32(PMIC_GLINK_OWNER_BATTMGR), .hdr.type = cpu_to_le32(PMIC_GLINK_REQ_RESP), .hdr.opcode = cpu_to_le32(BATTMGR_CHG_CTRL_LIMIT_EN), - .enable = cpu_to_le32(1), + .enable = cpu_to_le32(enable), .target_soc = cpu_to_le32(target_soc), .delta_soc = cpu_to_le32(delta_soc), }; @@ -677,13 +678,9 @@ static int qcom_battmgr_set_charge_start_threshold(struct qcom_battmgr *battmgr, { u32 target_soc, delta_soc; int ret; + bool enable = start_soc != 0; - if (start_soc < CHARGE_CTRL_START_THR_MIN || - start_soc > CHARGE_CTRL_START_THR_MAX) { - dev_err(battmgr->dev, "charge control start threshold exceed range: [%u - %u]\n", - CHARGE_CTRL_START_THR_MIN, CHARGE_CTRL_START_THR_MAX); - return -EINVAL; - } + start_soc = clamp(start_soc, CHARGE_CTRL_START_THR_MIN, CHARGE_CTRL_START_THR_MAX); /* * If the new start threshold is larger than the old end threshold, @@ -701,9 +698,10 @@ static int qcom_battmgr_set_charge_start_threshold(struct qcom_battmgr *battmgr, } mutex_lock(&battmgr->lock); - ret = qcom_battmgr_set_charge_control(battmgr, target_soc, delta_soc); + ret = qcom_battmgr_set_charge_control(battmgr, enable, target_soc, delta_soc); mutex_unlock(&battmgr->lock); if (!ret) { + battmgr->info.charge_ctrl_enable = enable; battmgr->info.charge_ctrl_start = start_soc; battmgr->info.charge_ctrl_end = target_soc; } @@ -715,19 +713,15 @@ static int qcom_battmgr_set_charge_end_threshold(struct qcom_battmgr *battmgr, i { u32 delta_soc = CHARGE_CTRL_DELTA_SOC; int ret; + bool enable = battmgr->info.charge_ctrl_enable; - if (end_soc < CHARGE_CTRL_END_THR_MIN || - end_soc > CHARGE_CTRL_END_THR_MAX) { - dev_err(battmgr->dev, "charge control end threshold exceed range: [%u - %u]\n", - CHARGE_CTRL_END_THR_MIN, CHARGE_CTRL_END_THR_MAX); - return -EINVAL; - } + end_soc = clamp(end_soc, CHARGE_CTRL_END_THR_MIN, CHARGE_CTRL_END_THR_MAX); if (battmgr->info.charge_ctrl_start && end_soc > battmgr->info.charge_ctrl_start) delta_soc = end_soc - battmgr->info.charge_ctrl_start; mutex_lock(&battmgr->lock); - ret = qcom_battmgr_set_charge_control(battmgr, end_soc, delta_soc); + ret = qcom_battmgr_set_charge_control(battmgr, enable, end_soc, delta_soc); mutex_unlock(&battmgr->lock); if (!ret) { battmgr->info.charge_ctrl_start = end_soc - delta_soc; @@ -1250,7 +1244,8 @@ static unsigned int qcom_battmgr_sc8280xp_parse_technology(const char *chemistry if ((!strncmp(chemistry, "LIO", BATTMGR_CHEMISTRY_LEN)) || (!strncmp(chemistry, "OOI", BATTMGR_CHEMISTRY_LEN))) return POWER_SUPPLY_TECHNOLOGY_LION; - if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN)) + if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN) || + !strncmp(chemistry, "LiP", BATTMGR_CHEMISTRY_LEN)) return POWER_SUPPLY_TECHNOLOGY_LIPO; pr_err("Unknown battery technology '%s'\n", chemistry); diff --git a/drivers/power/supply/rt5033_charger.c b/drivers/power/supply/rt5033_charger.c index 2fdc584397075..de724f23e453b 100644 --- a/drivers/power/supply/rt5033_charger.c +++ b/drivers/power/supply/rt5033_charger.c @@ -701,6 +701,8 @@ static int rt5033_charger_probe(struct platform_device *pdev) np_conn = of_parse_phandle(pdev->dev.of_node, "richtek,usb-connector", 0); np_edev = of_get_parent(np_conn); charger->edev = extcon_find_edev_by_node(np_edev); + of_node_put(np_edev); + of_node_put(np_conn); if (IS_ERR(charger->edev)) { dev_warn(charger->dev, "no extcon device found in device-tree\n"); goto out; diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 1ffe7f02932f6..5130d2395e88f 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1663,6 +1663,15 @@ static int rt9455_probe(struct i2c_client *client) rt9455_charger_config.supplied_to = rt9455_charger_supplied_to; rt9455_charger_config.num_supplicants = ARRAY_SIZE(rt9455_charger_supplied_to); + + info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, + &rt9455_charger_config); + if (IS_ERR(info->charger)) { + dev_err(dev, "Failed to register charger\n"); + ret = PTR_ERR(info->charger); + goto put_usb_notifier; + } + ret = devm_request_threaded_irq(dev, client->irq, NULL, rt9455_irq_handler_thread, IRQF_TRIGGER_LOW | IRQF_ONESHOT, @@ -1678,14 +1687,6 @@ static int rt9455_probe(struct i2c_client *client) goto put_usb_notifier; } - info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, - &rt9455_charger_config); - if (IS_ERR(info->charger)) { - dev_err(dev, "Failed to register charger\n"); - ret = PTR_ERR(info->charger); - goto put_usb_notifier; - } - return 0; put_usb_notifier: diff --git a/drivers/power/supply/rt9467-charger.c b/drivers/power/supply/rt9467-charger.c index fe773dd8b404f..44c26fb37a775 100644 --- a/drivers/power/supply/rt9467-charger.c +++ b/drivers/power/supply/rt9467-charger.c @@ -376,7 +376,7 @@ static int rt9467_set_value_from_ranges(struct rt9467_chg_data *data, if (rsel == RT9467_RANGE_VMIVR) { ret = linear_range_get_selector_high(range, value, &sel, &found); if (ret) - value = range->max_sel; + sel = range->max_sel; } else { linear_range_get_selector_within(range, value, &sel); } @@ -588,6 +588,10 @@ static int rt9467_run_aicl(struct rt9467_chg_data *data) aicl_vth = mivr_vth + RT9467_AICLVTH_GAP_uV; ret = rt9467_set_value_from_ranges(data, F_AICL_VTH, RT9467_RANGE_AICL_VTH, aicl_vth); + if (ret) { + dev_err(data->dev, "Failed to set AICL VTH\n"); + return ret; + } /* Trigger AICL function */ ret = regmap_field_write(data->rm_field[F_AICL_MEAS], 1); diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 943c82ee978f4..43c48196c1674 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -1174,24 +1174,6 @@ static int sbs_probe(struct i2c_client *client) i2c_set_clientdata(client, chip); - if (!chip->gpio_detect) - goto skip_gpio; - - irq = gpiod_to_irq(chip->gpio_detect); - if (irq <= 0) { - dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); - goto skip_gpio; - } - - rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, - IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, - dev_name(&client->dev), chip); - if (rc) { - dev_warn(&client->dev, "Failed to request irq: %d\n", rc); - goto skip_gpio; - } - -skip_gpio: /* * Before we register, we might need to make sure we can actually talk * to the battery. @@ -1217,6 +1199,24 @@ static int sbs_probe(struct i2c_client *client) return dev_err_probe(&client->dev, PTR_ERR(chip->power_supply), "Failed to register power supply\n"); + if (!chip->gpio_detect) + goto out; + + irq = gpiod_to_irq(chip->gpio_detect); + if (irq <= 0) { + dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); + goto out; + } + + rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, + IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + dev_name(&client->dev), chip); + if (rc) { + dev_warn(&client->dev, "Failed to request irq: %d\n", rc); + goto out; + } + +out: dev_info(&client->dev, "%s: battery gas gauge device registered\n", client->name); diff --git a/drivers/power/supply/wm831x_power.c b/drivers/power/supply/wm831x_power.c index 6acdba7885ca5..78fa0573ef25c 100644 --- a/drivers/power/supply/wm831x_power.c +++ b/drivers/power/supply/wm831x_power.c @@ -144,6 +144,7 @@ static int wm831x_usb_limit_change(struct notifier_block *nb, struct wm831x_power, usb_notify); unsigned int i, best; + int ret; /* Find the highest supported limit */ best = 0; @@ -156,8 +157,13 @@ static int wm831x_usb_limit_change(struct notifier_block *nb, dev_dbg(wm831x_power->wm831x->dev, "Limiting USB current to %umA", wm831x_usb_limits[best]); - wm831x_set_bits(wm831x_power->wm831x, WM831X_POWER_STATE, - WM831X_USB_ILIM_MASK, best); + ret = wm831x_set_bits(wm831x_power->wm831x, WM831X_POWER_STATE, + WM831X_USB_ILIM_MASK, best); + if (ret < 0) { + dev_err(wm831x_power->wm831x->dev, + "Failed to set USB current limit: %d\n", ret); + return ret; + } return 0; } diff --git a/drivers/power/supply/wm97xx_battery.c b/drivers/power/supply/wm97xx_battery.c index b3b0c37a9dd2d..f00722c88c6fe 100644 --- a/drivers/power/supply/wm97xx_battery.c +++ b/drivers/power/supply/wm97xx_battery.c @@ -178,12 +178,6 @@ static int wm97xx_bat_probe(struct platform_device *dev) "failed to get charge GPIO\n"); if (charge_gpiod) { gpiod_set_consumer_name(charge_gpiod, "BATT CHRG"); - ret = request_irq(gpiod_to_irq(charge_gpiod), - wm97xx_chrg_irq, 0, - "AC Detect", dev); - if (ret) - return dev_err_probe(&dev->dev, ret, - "failed to request GPIO irq\n"); props++; /* POWER_SUPPLY_PROP_STATUS */ } @@ -199,10 +193,8 @@ static int wm97xx_bat_probe(struct platform_device *dev) props++; /* POWER_SUPPLY_PROP_VOLTAGE_MIN */ prop = kcalloc(props, sizeof(*prop), GFP_KERNEL); - if (!prop) { - ret = -ENOMEM; - goto err3; - } + if (!prop) + return -ENOMEM; prop[i++] = POWER_SUPPLY_PROP_PRESENT; if (charge_gpiod) @@ -236,15 +228,27 @@ static int wm97xx_bat_probe(struct platform_device *dev) schedule_work(&bat_work); } else { ret = PTR_ERR(bat_psy); - goto err4; + goto free; + } + + if (charge_gpiod) { + ret = request_irq(gpiod_to_irq(charge_gpiod), wm97xx_chrg_irq, + 0, "AC Detect", dev); + if (ret) { + dev_err_probe(&dev->dev, ret, + "failed to request GPIO irq\n"); + goto unregister; + } } return 0; -err4: + +unregister: + power_supply_unregister(bat_psy); + +free: kfree(prop); -err3: - if (charge_gpiod) - free_irq(gpiod_to_irq(charge_gpiod), dev); + return ret; } diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index c7e7f9bf53137..57bebd07c7d0d 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1284,6 +1284,9 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_VFM(INTEL_LUNARLAKE_M, &rapl_defaults_core), X86_MATCH_VFM(INTEL_PANTHERLAKE_L, &rapl_defaults_core), + X86_MATCH_VFM(INTEL_WILDCATLAKE_L, &rapl_defaults_core), + X86_MATCH_VFM(INTEL_NOVALAKE, &rapl_defaults_core), + X86_MATCH_VFM(INTEL_NOVALAKE_L, &rapl_defaults_core), X86_MATCH_VFM(INTEL_ARROWLAKE_H, &rapl_defaults_core), X86_MATCH_VFM(INTEL_ARROWLAKE, &rapl_defaults_core), X86_MATCH_VFM(INTEL_ARROWLAKE_U, &rapl_defaults_core), diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 4ed06c71a3ac4..e827247683dbf 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -140,6 +140,7 @@ static int rapl_msr_write_raw(int cpu, struct reg_action *ra) /* List of verified CPUs. */ static const struct x86_cpu_id pl4_support_ids[] = { + X86_MATCH_VFM(INTEL_ICELAKE_L, NULL), X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL), X86_MATCH_VFM(INTEL_ALDERLAKE, NULL), X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL), @@ -151,6 +152,9 @@ static const struct x86_cpu_id pl4_support_ids[] = { X86_MATCH_VFM(INTEL_ARROWLAKE_U, NULL), X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL), X86_MATCH_VFM(INTEL_PANTHERLAKE_L, NULL), + X86_MATCH_VFM(INTEL_WILDCATLAKE_L, NULL), + X86_MATCH_VFM(INTEL_NOVALAKE, NULL), + X86_MATCH_VFM(INTEL_NOVALAKE_L, NULL), {} }; diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 82201bf4685d4..34c0bd1edd61a 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -157,7 +157,7 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) tpmi_domain_flags = tpmi_domain_header >> 32 & 0xffff; if (tpmi_domain_version == TPMI_VERSION_INVALID) { - pr_warn(FW_BUG "Invalid version\n"); + pr_debug("Invalid version, other instances may be valid\n"); return -ENODEV; } diff --git a/drivers/powercap/powercap_sys.c b/drivers/powercap/powercap_sys.c index 4112a00973382..1ff369880beb2 100644 --- a/drivers/powercap/powercap_sys.c +++ b/drivers/powercap/powercap_sys.c @@ -68,7 +68,7 @@ static ssize_t show_constraint_##_attr(struct device *dev, \ int id; \ struct powercap_zone_constraint *pconst;\ \ - if (!sscanf(dev_attr->attr.name, "constraint_%d_", &id)) \ + if (sscanf(dev_attr->attr.name, "constraint_%d_", &id) != 1) \ return -EINVAL; \ if (id >= power_zone->const_id_cnt) \ return -EINVAL; \ @@ -93,7 +93,7 @@ static ssize_t store_constraint_##_attr(struct device *dev,\ int id; \ struct powercap_zone_constraint *pconst;\ \ - if (!sscanf(dev_attr->attr.name, "constraint_%d_", &id)) \ + if (sscanf(dev_attr->attr.name, "constraint_%d_", &id) != 1) \ return -EINVAL; \ if (id >= power_zone->const_id_cnt) \ return -EINVAL; \ @@ -162,7 +162,7 @@ static ssize_t show_constraint_name(struct device *dev, ssize_t len = -ENODATA; struct powercap_zone_constraint *pconst; - if (!sscanf(dev_attr->attr.name, "constraint_%d_", &id)) + if (sscanf(dev_attr->attr.name, "constraint_%d_", &id) != 1) return -EINVAL; if (id >= power_zone->const_id_cnt) return -EINVAL; @@ -625,17 +625,23 @@ struct powercap_control_type *powercap_register_control_type( INIT_LIST_HEAD(&control_type->node); control_type->dev.class = &powercap_class; dev_set_name(&control_type->dev, "%s", name); - result = device_register(&control_type->dev); - if (result) { - put_device(&control_type->dev); - return ERR_PTR(result); - } idr_init(&control_type->idr); mutex_lock(&powercap_cntrl_list_lock); list_add_tail(&control_type->node, &powercap_cntrl_list); mutex_unlock(&powercap_cntrl_list_lock); + result = device_register(&control_type->dev); + if (result) { + mutex_lock(&powercap_cntrl_list_lock); + list_del(&control_type->node); + mutex_unlock(&powercap_cntrl_list_lock); + + idr_destroy(&control_type->idr); + put_device(&control_type->dev); + return ERR_PTR(result); + } + return control_type; } EXPORT_SYMBOL_GPL(powercap_register_control_type); diff --git a/drivers/ptp/ptp_vmclock.c b/drivers/ptp/ptp_vmclock.c index b3a83b03d9c14..cbbfc494680c7 100644 --- a/drivers/ptp/ptp_vmclock.c +++ b/drivers/ptp/ptp_vmclock.c @@ -591,6 +591,7 @@ static int vmclock_probe(struct platform_device *pdev) static const struct acpi_device_id vmclock_acpi_ids[] = { { "AMZNC10C", 0 }, + { "VMCLOCK", 0 }, {} }; MODULE_DEVICE_TABLE(acpi, vmclock_acpi_ids); diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index ea2ccf42e8144..462c91a034c8e 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -2294,8 +2294,9 @@ static long pwm_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long ar .duty_offset_ns = wf.duty_offset_ns, }; - return copy_to_user((struct pwmchip_waveform __user *)arg, - &cwf, sizeof(cwf)); + ret = copy_to_user((struct pwmchip_waveform __user *)arg, + &cwf, sizeof(cwf)); + return ret ? -EFAULT : 0; } case PWM_IOCTL_GETWF: @@ -2328,8 +2329,9 @@ static long pwm_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long ar .duty_offset_ns = wf.duty_offset_ns, }; - return copy_to_user((struct pwmchip_waveform __user *)arg, - &cwf, sizeof(cwf)); + ret = copy_to_user((struct pwmchip_waveform __user *)arg, + &cwf, sizeof(cwf)); + return ret ? -EFAULT : 0; } case PWM_IOCTL_SETROUNDEDWF: @@ -2696,11 +2698,10 @@ static int pwm_seq_show(struct seq_file *s, void *v) { struct pwm_chip *chip = v; - seq_printf(s, "%s%d: %s/%s, %d PWM device%s\n", + seq_printf(s, "%s%u: %s/%s, npwm: %u\n", (char *)s->private, chip->id, pwmchip_parent(chip)->bus ? pwmchip_parent(chip)->bus->name : "no-bus", - dev_name(pwmchip_parent(chip)), chip->npwm, - (chip->npwm != 1) ? "s" : ""); + dev_name(pwmchip_parent(chip)), chip->npwm); pwm_dbg_show(chip, s); diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c index 578e95e0296c6..532903da521fd 100644 --- a/drivers/pwm/pwm-bcm2835.c +++ b/drivers/pwm/pwm-bcm2835.c @@ -34,29 +34,6 @@ static inline struct bcm2835_pwm *to_bcm2835_pwm(struct pwm_chip *chip) return pwmchip_get_drvdata(chip); } -static int bcm2835_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) -{ - struct bcm2835_pwm *pc = to_bcm2835_pwm(chip); - u32 value; - - value = readl(pc->base + PWM_CONTROL); - value &= ~(PWM_CONTROL_MASK << PWM_CONTROL_SHIFT(pwm->hwpwm)); - value |= (PWM_MODE << PWM_CONTROL_SHIFT(pwm->hwpwm)); - writel(value, pc->base + PWM_CONTROL); - - return 0; -} - -static void bcm2835_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) -{ - struct bcm2835_pwm *pc = to_bcm2835_pwm(chip); - u32 value; - - value = readl(pc->base + PWM_CONTROL); - value &= ~(PWM_CONTROL_MASK << PWM_CONTROL_SHIFT(pwm->hwpwm)); - writel(value, pc->base + PWM_CONTROL); -} - static int bcm2835_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { @@ -102,6 +79,9 @@ static int bcm2835_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, /* set polarity */ val = readl(pc->base + PWM_CONTROL); + val &= ~(PWM_CONTROL_MASK << PWM_CONTROL_SHIFT(pwm->hwpwm)); + val |= PWM_MODE << PWM_CONTROL_SHIFT(pwm->hwpwm); + if (state->polarity == PWM_POLARITY_NORMAL) val &= ~(PWM_POLARITY << PWM_CONTROL_SHIFT(pwm->hwpwm)); else @@ -119,8 +99,6 @@ static int bcm2835_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, } static const struct pwm_ops bcm2835_pwm_ops = { - .request = bcm2835_pwm_request, - .free = bcm2835_pwm_free, .apply = bcm2835_pwm_apply, }; diff --git a/drivers/pwm/pwm-max7360.c b/drivers/pwm/pwm-max7360.c index ebf93a7aee5be..31972bd00ebe9 100644 --- a/drivers/pwm/pwm-max7360.c +++ b/drivers/pwm/pwm-max7360.c @@ -153,6 +153,7 @@ static int max7360_pwm_read_waveform(struct pwm_chip *chip, } static const struct pwm_ops max7360_pwm_ops = { + .sizeof_wfhw = sizeof(struct max7360_pwm_waveform), .request = max7360_pwm_request, .round_waveform_tohw = max7360_pwm_round_waveform_tohw, .round_waveform_fromhw = max7360_pwm_round_waveform_fromhw, diff --git a/drivers/pwm/pwm-rzg2l-gpt.c b/drivers/pwm/pwm-rzg2l-gpt.c index 360c8bf3b1903..4856af080e8e9 100644 --- a/drivers/pwm/pwm-rzg2l-gpt.c +++ b/drivers/pwm/pwm-rzg2l-gpt.c @@ -96,6 +96,11 @@ static inline unsigned int rzg2l_gpt_subchannel(unsigned int hwpwm) return hwpwm & 0x1; } +static inline unsigned int rzg2l_gpt_sibling(unsigned int hwpwm) +{ + return hwpwm ^ 0x1; +} + static void rzg2l_gpt_write(struct rzg2l_gpt_chip *rzg2l_gpt, u32 reg, u32 data) { writel(data, rzg2l_gpt->mmio + reg); @@ -271,10 +276,14 @@ static int rzg2l_gpt_config(struct pwm_chip *chip, struct pwm_device *pwm, * in use with different settings. */ if (rzg2l_gpt->channel_request_count[ch] > 1) { - if (period_ticks < rzg2l_gpt->period_ticks[ch]) - return -EBUSY; - else + u8 sibling_ch = rzg2l_gpt_sibling(pwm->hwpwm); + + if (rzg2l_gpt_is_ch_enabled(rzg2l_gpt, sibling_ch)) { + if (period_ticks < rzg2l_gpt->period_ticks[ch]) + return -EBUSY; + period_ticks = rzg2l_gpt->period_ticks[ch]; + } } prescale = rzg2l_gpt_calculate_prescale(rzg2l_gpt, period_ticks); diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index 7a86cb090f76f..2533c95b0ba9d 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -237,8 +237,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_cycles < 1) period_cycles = 1; - pm_runtime_get_sync(pwmchip_parent(chip)); - /* Update clock prescaler values */ ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_CLKDIV_MASK, tb_divval); @@ -290,8 +288,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (!(duty_cycles > period_cycles)) ehrpwm_write(pc->mmio_base, cmp_reg, duty_cycles); - pm_runtime_put_sync(pwmchip_parent(chip)); - return 0; } @@ -378,6 +374,8 @@ static int ehrpwm_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, int err; bool enabled = pwm->state.enabled; + guard(pm_runtime_active)(pwmchip_parent(chip)); + if (state->polarity != pwm->state.polarity) { if (enabled) { ehrpwm_pwm_disable(chip, pwm); diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c index c12941f71e2cb..dcd6619a4b027 100644 --- a/drivers/rapidio/rio-scan.c +++ b/drivers/rapidio/rio-scan.c @@ -854,7 +854,8 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport, if (idtab == NULL) { pr_err("RIO: failed to allocate destID table\n"); - rio_free_net(net); + kfree(net); + mport->net = NULL; net = NULL; } else { net->enum_data = idtab; diff --git a/drivers/ras/ras.c b/drivers/ras/ras.c index ac0e132ccc3eb..03df3db623346 100644 --- a/drivers/ras/ras.c +++ b/drivers/ras/ras.c @@ -53,9 +53,49 @@ void log_non_standard_event(const guid_t *sec_type, const guid_t *fru_id, } EXPORT_SYMBOL_GPL(log_non_standard_event); -void log_arm_hw_error(struct cper_sec_proc_arm *err) +void log_arm_hw_error(struct cper_sec_proc_arm *err, const u8 sev) { - trace_arm_event(err); + struct cper_arm_err_info *err_info; + struct cper_arm_ctx_info *ctx_info; + u8 *ven_err_data; + u32 ctx_len = 0; + int n, sz, cpu; + s32 vsei_len; + u32 pei_len; + u8 *pei_err, *ctx_err; + + pei_len = sizeof(struct cper_arm_err_info) * err->err_info_num; + pei_err = (u8 *)(err + 1); + + err_info = (struct cper_arm_err_info *)(err + 1); + ctx_info = (struct cper_arm_ctx_info *)(err_info + err->err_info_num); + ctx_err = (u8 *)ctx_info; + + for (n = 0; n < err->context_info_num; n++) { + sz = sizeof(struct cper_arm_ctx_info); + + if (sz + (long)ctx_info - (long)err >= err->section_length) + sz += ctx_info->size; + + ctx_info = (struct cper_arm_ctx_info *)((long)ctx_info + sz); + ctx_len += sz; + } + + vsei_len = err->section_length - (sizeof(struct cper_sec_proc_arm) + pei_len + ctx_len); + if (vsei_len < 0) { + pr_warn(FW_BUG "section length: %d\n", err->section_length); + pr_warn(FW_BUG "section length is too small\n"); + pr_warn(FW_BUG "firmware-generated error record is incorrect\n"); + vsei_len = 0; + } + ven_err_data = (u8 *)ctx_info; + + cpu = GET_LOGICAL_INDEX(err->mpidr); + if (cpu < 0) + cpu = -1; + + trace_arm_event(err, pei_err, pei_len, ctx_err, ctx_len, + ven_err_data, (u32)vsei_len, sev, cpu); } static int __init ras_init(void) diff --git a/drivers/regulator/bq257xx-regulator.c b/drivers/regulator/bq257xx-regulator.c index fc1ccede44688..dab8f1ab44503 100644 --- a/drivers/regulator/bq257xx-regulator.c +++ b/drivers/regulator/bq257xx-regulator.c @@ -115,11 +115,10 @@ static void bq257xx_reg_dt_parse_gpio(struct platform_device *pdev) return; subchild = of_get_child_by_name(child, pdata->desc.of_match); + of_node_put(child); if (!subchild) return; - of_node_put(child); - pdata->otg_en_gpio = devm_fwnode_gpiod_get_index(&pdev->dev, of_fwnode_handle(subchild), "enable", 0, diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index dd7b10e768c06..765bd1b5deb3a 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -1431,6 +1431,33 @@ static int set_machine_constraints(struct regulator_dev *rdev) int ret = 0; const struct regulator_ops *ops = rdev->desc->ops; + /* + * If there is no mechanism for controlling the regulator then + * flag it as always_on so we don't end up duplicating checks + * for this so much. Note that we could control the state of + * a supply to control the output on a regulator that has no + * direct control. + */ + if (!rdev->ena_pin && !ops->enable) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + + if (rdev->supply) + rdev->constraints->always_on = + rdev->supply->rdev->constraints->always_on; + else + rdev->constraints->always_on = true; + } + + /* + * If we want to enable this regulator, make sure that we know the + * supplying regulator. + */ + if (rdev->constraints->always_on || rdev->constraints->boot_on) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + } + ret = machine_constraints_voltage(rdev, rdev->constraints); if (ret != 0) return ret; @@ -1596,35 +1623,15 @@ static int set_machine_constraints(struct regulator_dev *rdev) } } - /* - * If there is no mechanism for controlling the regulator then - * flag it as always_on so we don't end up duplicating checks - * for this so much. Note that we could control the state of - * a supply to control the output on a regulator that has no - * direct control. - */ - if (!rdev->ena_pin && !ops->enable) { - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; - - if (rdev->supply) - rdev->constraints->always_on = - rdev->supply->rdev->constraints->always_on; - else - rdev->constraints->always_on = true; - } - /* If the constraints say the regulator should be on at this point * and we have control then make sure it is enabled. */ if (rdev->constraints->always_on || rdev->constraints->boot_on) { - /* If we want to enable this regulator, make sure that we know - * the supplying regulator. - */ - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; + bool supply_enabled = false; - /* If supplying regulator has already been enabled, + /* We have ensured a potential supply has been resolved above. + * + * If supplying regulator has already been enabled, * it's not intended to have use_count increment * when rdev is only boot-on. */ @@ -1637,11 +1644,14 @@ static int set_machine_constraints(struct regulator_dev *rdev) rdev->supply = NULL; return ret; } + supply_enabled = true; } ret = _regulator_do_enable(rdev); if (ret < 0 && ret != -EINVAL) { rdev_err(rdev, "failed to enable: %pe\n", ERR_PTR(ret)); + if (supply_enabled) + regulator_disable(rdev->supply); return ret; } @@ -1828,8 +1838,6 @@ static const struct file_operations constraint_flags_fops = { #endif }; -#define REG_STR_SIZE 64 - static void link_and_create_debugfs(struct regulator *regulator, struct regulator_dev *rdev, struct device *dev) { @@ -1877,15 +1885,7 @@ static struct regulator *create_regulator(struct regulator_dev *rdev, lockdep_assert_held_once(&rdev->mutex.base); if (dev) { - char buf[REG_STR_SIZE]; - int size; - - size = snprintf(buf, REG_STR_SIZE, "%s-%s", - dev->kobj.name, supply_name); - if (size >= REG_STR_SIZE) - return NULL; - - supply_name = kstrdup(buf, GFP_KERNEL); + supply_name = kasprintf(GFP_KERNEL, "%s-%s", dev->kobj.name, supply_name); if (supply_name == NULL) return NULL; } else { @@ -1942,6 +1942,7 @@ static void regulator_supply_alias(struct device **dev, const char **supply) { struct regulator_supply_alias *map; + mutex_lock(®ulator_list_mutex); map = regulator_find_supply_alias(*dev, *supply); if (map) { dev_dbg(*dev, "Mapping supply %s to %s,%s\n", @@ -1950,6 +1951,7 @@ static void regulator_supply_alias(struct device **dev, const char **supply) *dev = map->alias_dev; *supply = map->alias_supply; } + mutex_unlock(®ulator_list_mutex); } static int regulator_match(struct device *dev, const void *data) @@ -2492,22 +2494,26 @@ int regulator_register_supply_alias(struct device *dev, const char *id, const char *alias_id) { struct regulator_supply_alias *map; + struct regulator_supply_alias *new_map; - map = regulator_find_supply_alias(dev, id); - if (map) - return -EEXIST; - - map = kzalloc(sizeof(struct regulator_supply_alias), GFP_KERNEL); - if (!map) + new_map = kzalloc(sizeof(struct regulator_supply_alias), GFP_KERNEL); + if (!new_map) return -ENOMEM; - map->src_dev = dev; - map->src_supply = id; - map->alias_dev = alias_dev; - map->alias_supply = alias_id; - - list_add(&map->list, ®ulator_supply_alias_list); + mutex_lock(®ulator_list_mutex); + map = regulator_find_supply_alias(dev, id); + if (map) { + mutex_unlock(®ulator_list_mutex); + kfree(new_map); + return -EEXIST; + } + new_map->src_dev = dev; + new_map->src_supply = id; + new_map->alias_dev = alias_dev; + new_map->alias_supply = alias_id; + list_add(&new_map->list, ®ulator_supply_alias_list); + mutex_unlock(®ulator_list_mutex); pr_info("Adding alias for supply %s,%s -> %s,%s\n", id, dev_name(dev), alias_id, dev_name(alias_dev)); @@ -2527,11 +2533,13 @@ void regulator_unregister_supply_alias(struct device *dev, const char *id) { struct regulator_supply_alias *map; + mutex_lock(®ulator_list_mutex); map = regulator_find_supply_alias(dev, id); if (map) { list_del(&map->list); kfree(map); } + mutex_unlock(®ulator_list_mutex); } EXPORT_SYMBOL_GPL(regulator_unregister_supply_alias); diff --git a/drivers/regulator/fixed.c b/drivers/regulator/fixed.c index a2d16e9abfb58..254c0a8a45559 100644 --- a/drivers/regulator/fixed.c +++ b/drivers/regulator/fixed.c @@ -330,13 +330,10 @@ static int reg_fixed_voltage_probe(struct platform_device *pdev) drvdata->dev = devm_regulator_register(&pdev->dev, &drvdata->desc, &cfg); - if (IS_ERR(drvdata->dev)) { - ret = dev_err_probe(&pdev->dev, PTR_ERR(drvdata->dev), - "Failed to register regulator: %ld\n", - PTR_ERR(drvdata->dev)); - gpiod_put(cfg.ena_gpiod); - return ret; - } + if (IS_ERR(drvdata->dev)) + return dev_err_probe(&pdev->dev, PTR_ERR(drvdata->dev), + "Failed to register regulator: %ld\n", + PTR_ERR(drvdata->dev)); platform_set_drvdata(pdev, drvdata); diff --git a/drivers/regulator/pca9450-regulator.c b/drivers/regulator/pca9450-regulator.c index 4be270f4d6c35..f991dc9365f18 100644 --- a/drivers/regulator/pca9450-regulator.c +++ b/drivers/regulator/pca9450-regulator.c @@ -1117,6 +1117,143 @@ static int pca9450_i2c_restart_handler(struct sys_off_data *data) return 0; } +static int pca9450_of_init(struct pca9450 *pca9450) +{ + struct i2c_client *i2c = container_of(pca9450->dev, struct i2c_client, dev); + int ret; + unsigned int val; + unsigned int reset_ctrl; + unsigned int rstb_deb_ctrl; + unsigned int t_on_deb, t_off_deb; + unsigned int t_on_step, t_off_step; + unsigned int t_restart; + + if (of_property_read_bool(i2c->dev.of_node, "nxp,wdog_b-warm-reset")) + reset_ctrl = WDOG_B_CFG_WARM; + else + reset_ctrl = WDOG_B_CFG_COLD_LDO12; + + /* Set reset behavior on assertion of WDOG_B signal */ + ret = regmap_update_bits(pca9450->regmap, PCA9450_REG_RESET_CTRL, + WDOG_B_CFG_MASK, reset_ctrl); + if (ret) + return dev_err_probe(&i2c->dev, ret, "Failed to set WDOG_B reset behavior\n"); + + ret = of_property_read_u32(i2c->dev.of_node, "npx,pmic-rst-b-debounce-ms", &val); + if (ret == -EINVAL) + rstb_deb_ctrl = T_PMIC_RST_DEB_50MS; + else if (ret) + return ret; + else { + switch (val) { + case 10: rstb_deb_ctrl = T_PMIC_RST_DEB_10MS; break; + case 50: rstb_deb_ctrl = T_PMIC_RST_DEB_50MS; break; + case 100: rstb_deb_ctrl = T_PMIC_RST_DEB_100MS; break; + case 500: rstb_deb_ctrl = T_PMIC_RST_DEB_500MS; break; + case 1000: rstb_deb_ctrl = T_PMIC_RST_DEB_1S; break; + case 2000: rstb_deb_ctrl = T_PMIC_RST_DEB_2S; break; + case 4000: rstb_deb_ctrl = T_PMIC_RST_DEB_4S; break; + case 8000: rstb_deb_ctrl = T_PMIC_RST_DEB_8S; break; + default: return -EINVAL; + } + } + ret = regmap_update_bits(pca9450->regmap, PCA9450_REG_RESET_CTRL, + T_PMIC_RST_DEB_MASK, rstb_deb_ctrl); + if (ret) + return dev_err_probe(&i2c->dev, ret, "Failed to set PMIC_RST_B debounce time\n"); + + ret = of_property_read_u32(i2c->dev.of_node, "nxp,pmic-on-req-on-debounce-us", &val); + if (ret == -EINVAL) + t_on_deb = T_ON_DEB_20MS; + else if (ret) + return ret; + else { + switch (val) { + case 120: t_on_deb = T_ON_DEB_120US; break; + case 20000: t_on_deb = T_ON_DEB_20MS; break; + case 100000: t_on_deb = T_ON_DEB_100MS; break; + case 750000: t_on_deb = T_ON_DEB_750MS; break; + default: return -EINVAL; + } + } + + ret = of_property_read_u32(i2c->dev.of_node, "nxp,pmic-on-req-off-debounce-us", &val); + if (ret == -EINVAL) + t_off_deb = T_OFF_DEB_120US; + else if (ret) + return ret; + else { + switch (val) { + case 120: t_off_deb = T_OFF_DEB_120US; break; + case 2000: t_off_deb = T_OFF_DEB_2MS; break; + default: return -EINVAL; + } + } + + ret = of_property_read_u32(i2c->dev.of_node, "nxp,power-on-step-ms", &val); + if (ret == -EINVAL) + t_on_step = T_ON_STEP_2MS; + else if (ret) + return ret; + else { + switch (val) { + case 1: t_on_step = T_ON_STEP_1MS; break; + case 2: t_on_step = T_ON_STEP_2MS; break; + case 4: t_on_step = T_ON_STEP_4MS; break; + case 8: t_on_step = T_ON_STEP_8MS; break; + default: return -EINVAL; + } + } + + ret = of_property_read_u32(i2c->dev.of_node, "nxp,power-down-step-ms", &val); + if (ret == -EINVAL) + t_off_step = T_OFF_STEP_8MS; + else if (ret) + return ret; + else { + switch (val) { + case 2: t_off_step = T_OFF_STEP_2MS; break; + case 4: t_off_step = T_OFF_STEP_4MS; break; + case 8: t_off_step = T_OFF_STEP_8MS; break; + case 16: t_off_step = T_OFF_STEP_16MS; break; + default: return -EINVAL; + } + } + + ret = of_property_read_u32(i2c->dev.of_node, "nxp,restart-ms", &val); + if (ret == -EINVAL) + t_restart = T_RESTART_250MS; + else if (ret) + return ret; + else { + switch (val) { + case 250: t_restart = T_RESTART_250MS; break; + case 500: t_restart = T_RESTART_500MS; break; + default: return -EINVAL; + } + } + + ret = regmap_update_bits(pca9450->regmap, PCA9450_REG_PWRCTRL, + T_ON_DEB_MASK | T_OFF_DEB_MASK | T_ON_STEP_MASK | + T_OFF_STEP_MASK | T_RESTART_MASK, + t_on_deb | t_off_deb | t_on_step | + t_off_step | t_restart); + if (ret) + return dev_err_probe(&i2c->dev, ret, + "Failed to set PWR_CTRL debounce configuration\n"); + + if (of_property_read_bool(i2c->dev.of_node, "nxp,i2c-lt-enable")) { + /* Enable I2C Level Translator */ + ret = regmap_update_bits(pca9450->regmap, PCA9450_REG_CONFIG2, + I2C_LT_MASK, I2C_LT_ON_STANDBY_RUN); + if (ret) + return dev_err_probe(&i2c->dev, ret, + "Failed to enable I2C level translator\n"); + } + + return 0; +} + static int pca9450_i2c_probe(struct i2c_client *i2c) { enum pca9450_chip_type type = (unsigned int)(uintptr_t) @@ -1126,7 +1263,7 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) struct regulator_dev *ldo5; struct pca9450 *pca9450; unsigned int device_id, i; - unsigned int reset_ctrl; + const char *type_name; int ret; pca9450 = devm_kzalloc(&i2c->dev, sizeof(struct pca9450), GFP_KERNEL); @@ -1137,15 +1274,22 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) case PCA9450_TYPE_PCA9450A: regulator_desc = pca9450a_regulators; pca9450->rcnt = ARRAY_SIZE(pca9450a_regulators); + type_name = "pca9450a"; break; case PCA9450_TYPE_PCA9450BC: regulator_desc = pca9450bc_regulators; pca9450->rcnt = ARRAY_SIZE(pca9450bc_regulators); + type_name = "pca9450bc"; break; case PCA9450_TYPE_PCA9451A: + regulator_desc = pca9451a_regulators; + pca9450->rcnt = ARRAY_SIZE(pca9451a_regulators); + type_name = "pca9451a"; + break; case PCA9450_TYPE_PCA9452: regulator_desc = pca9451a_regulators; pca9450->rcnt = ARRAY_SIZE(pca9451a_regulators); + type_name = "pca9452"; break; default: dev_err(&i2c->dev, "Unknown device type"); @@ -1203,7 +1347,7 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) if (pca9450->irq) { ret = devm_request_threaded_irq(pca9450->dev, pca9450->irq, NULL, pca9450_irq_handler, - (IRQF_TRIGGER_FALLING | IRQF_ONESHOT), + (IRQF_TRIGGER_LOW | IRQF_ONESHOT), "pca9450-irq", pca9450); if (ret != 0) return dev_err_probe(pca9450->dev, ret, "Failed to request IRQ: %d\n", @@ -1224,25 +1368,9 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) if (ret) return dev_err_probe(&i2c->dev, ret, "Failed to clear PRESET_EN bit\n"); - if (of_property_read_bool(i2c->dev.of_node, "nxp,wdog_b-warm-reset")) - reset_ctrl = WDOG_B_CFG_WARM; - else - reset_ctrl = WDOG_B_CFG_COLD_LDO12; - - /* Set reset behavior on assertion of WDOG_B signal */ - ret = regmap_update_bits(pca9450->regmap, PCA9450_REG_RESET_CTRL, - WDOG_B_CFG_MASK, reset_ctrl); + ret = pca9450_of_init(pca9450); if (ret) - return dev_err_probe(&i2c->dev, ret, "Failed to set WDOG_B reset behavior\n"); - - if (of_property_read_bool(i2c->dev.of_node, "nxp,i2c-lt-enable")) { - /* Enable I2C Level Translator */ - ret = regmap_update_bits(pca9450->regmap, PCA9450_REG_CONFIG2, - I2C_LT_MASK, I2C_LT_ON_STANDBY_RUN); - if (ret) - return dev_err_probe(&i2c->dev, ret, - "Failed to enable I2C level translator\n"); - } + return dev_err_probe(&i2c->dev, ret, "Unable to parse OF data\n"); /* * For LDO5 we need to be able to check the status of the SD_VSEL input in @@ -1251,10 +1379,9 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) * to this signal (if SION bit is set in IOMUX). */ pca9450->sd_vsel_gpio = gpiod_get_optional(&ldo5->dev, "sd-vsel", GPIOD_IN); - if (IS_ERR(pca9450->sd_vsel_gpio)) { - dev_err(&i2c->dev, "Failed to get SD_VSEL GPIO\n"); - return ret; - } + if (IS_ERR(pca9450->sd_vsel_gpio)) + return dev_err_probe(&i2c->dev, PTR_ERR(pca9450->sd_vsel_gpio), + "Failed to get SD_VSEL GPIO\n"); pca9450->sd_vsel_fixed_low = of_property_read_bool(ldo5->dev.of_node, "nxp,sd-vsel-fixed-low"); @@ -1264,9 +1391,7 @@ static int pca9450_i2c_probe(struct i2c_client *i2c) pca9450_i2c_restart_handler, pca9450)) dev_warn(&i2c->dev, "Failed to register restart handler\n"); - dev_info(&i2c->dev, "%s probed.\n", - type == PCA9450_TYPE_PCA9450A ? "pca9450a" : - (type == PCA9450_TYPE_PCA9451A ? "pca9451a" : "pca9450bc")); + dev_info(&i2c->dev, "%s probed.\n", type_name); return 0; } diff --git a/drivers/regulator/pf9453-regulator.c b/drivers/regulator/pf9453-regulator.c index be627f49b6176..f9828af748170 100644 --- a/drivers/regulator/pf9453-regulator.c +++ b/drivers/regulator/pf9453-regulator.c @@ -819,7 +819,7 @@ static int pf9453_i2c_probe(struct i2c_client *i2c) } ret = devm_request_threaded_irq(pf9453->dev, pf9453->irq, NULL, pf9453_irq_handler, - (IRQF_TRIGGER_FALLING | IRQF_ONESHOT), + IRQF_ONESHOT, "pf9453-irq", pf9453); if (ret) return dev_err_probe(pf9453->dev, ret, "Failed to request IRQ: %d\n", pf9453->irq); diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c index 109f0aae09b1d..6e4cb2871fca8 100644 --- a/drivers/regulator/qcom-rpmh-regulator.c +++ b/drivers/regulator/qcom-rpmh-regulator.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. -// Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. #define pr_fmt(fmt) "%s: " fmt, __func__ @@ -32,6 +32,34 @@ enum rpmh_regulator_type { XOB, }; +/** + * enum regulator_hw_type - supported regulator types + * @SMPS: Switch mode power supply. + * @LDO: Linear Dropout regulator. + * @BOB: Buck/Boost type regulator. + * @VS: Simple voltage ON/OFF switch. + * @NUM_REGULATOR_TYPES: Number of regulator types. + */ +enum regulator_hw_type { + SMPS, + LDO, + BOB, + VS, + NUM_REGULATOR_TYPES, +}; + +struct resource_name_formats { + const char *rsc_name_fmt; + const char *rsc_name_fmt1; +}; + +static const struct resource_name_formats vreg_rsc_name_lookup[NUM_REGULATOR_TYPES] = { + [SMPS] = {"S%d%s", "smp%s%d"}, + [LDO] = {"L%d%s", "ldo%s%d"}, + [BOB] = {"B%d%s", "bob%s%d"}, + [VS] = {"VS%d%s", "vs%s%d"}, +}; + #define RPMH_REGULATOR_REG_VRM_VOLTAGE 0x0 #define RPMH_REGULATOR_REG_ENABLE 0x4 #define RPMH_REGULATOR_REG_VRM_MODE 0x8 @@ -64,6 +92,12 @@ enum rpmh_regulator_type { #define PMIC5_BOB_MODE_AUTO 6 #define PMIC5_BOB_MODE_PWM 7 +#define PMIC530_LDO_MODE_RETENTION 3 +#define PMIC530_LDO_MODE_LPM 4 +#define PMIC530_LDO_MODE_OPM 5 +#define PMIC530_LDO_MODE_HPM 7 + +#define PMIC_ID_LEN 4 /** * struct rpmh_vreg_hw_data - RPMh regulator hardware configurations * @regulator_type: RPMh accelerator type used to manage this @@ -136,17 +170,17 @@ struct rpmh_vreg { * struct rpmh_vreg_init_data - initialization data for an RPMh regulator * @name: Name for the regulator which also corresponds * to the device tree subnode name of the regulator - * @resource_name: RPMh regulator resource name format string. - * This must include exactly one field: '%s' which - * is filled at run-time with the PMIC ID provided - * by device tree property qcom,pmic-id. Example: - * "ldo%s1" for RPMh resource "ldoa1". + * @index: This is the index number of the regulator present + * on the PMIC. + * @vreg_hw_type: Regulator HW type enum, this must be BOB, SMPS, + * LDO, VS, based on the regulator HW type. * @supply_name: Parent supply regulator name * @hw_data: Configuration data for this PMIC regulator type */ struct rpmh_vreg_init_data { const char *name; - const char *resource_name; + enum regulator_hw_type vreg_hw_type; + int index; const char *supply_name; const struct rpmh_vreg_hw_data *hw_data; }; @@ -417,6 +451,7 @@ static int rpmh_regulator_init_vreg(struct rpmh_vreg *vreg, struct device *dev, { struct regulator_config reg_config = {}; char rpmh_resource_name[20] = ""; + const char *rsc_name; const struct rpmh_vreg_init_data *rpmh_data; struct regulator_init_data *init_data; struct regulator_dev *rdev; @@ -433,8 +468,16 @@ static int rpmh_regulator_init_vreg(struct rpmh_vreg *vreg, struct device *dev, return -EINVAL; } - scnprintf(rpmh_resource_name, sizeof(rpmh_resource_name), - rpmh_data->resource_name, pmic_id); + if (strnlen(pmic_id, PMIC_ID_LEN) > 1 && strnstr(pmic_id, "_E", PMIC_ID_LEN)) { + rsc_name = vreg_rsc_name_lookup[rpmh_data->vreg_hw_type].rsc_name_fmt; + scnprintf(rpmh_resource_name, sizeof(rpmh_resource_name), + rsc_name, rpmh_data->index, pmic_id); + + } else { + rsc_name = vreg_rsc_name_lookup[rpmh_data->vreg_hw_type].rsc_name_fmt1; + scnprintf(rpmh_resource_name, sizeof(rpmh_resource_name), + rsc_name, pmic_id, rpmh_data->index); + } vreg->addr = cmd_db_read_addr(rpmh_resource_name); if (!vreg->addr) { @@ -519,6 +562,14 @@ static const int pmic_mode_map_pmic5_ldo_hpm[REGULATOR_MODE_STANDBY + 1] = { [REGULATOR_MODE_FAST] = -EINVAL, }; +static const int pmic_mode_map_pmic530_ldo[REGULATOR_MODE_STANDBY + 1] = { + [REGULATOR_MODE_INVALID] = -EINVAL, + [REGULATOR_MODE_STANDBY] = PMIC530_LDO_MODE_RETENTION, + [REGULATOR_MODE_IDLE] = PMIC530_LDO_MODE_LPM, + [REGULATOR_MODE_NORMAL] = PMIC530_LDO_MODE_OPM, + [REGULATOR_MODE_FAST] = PMIC530_LDO_MODE_HPM, +}; + static unsigned int rpmh_regulator_pmic4_ldo_of_map_mode(unsigned int rpmh_mode) { unsigned int mode; @@ -541,6 +592,30 @@ static unsigned int rpmh_regulator_pmic4_ldo_of_map_mode(unsigned int rpmh_mode) return mode; } +static unsigned int rpmh_regulator_pmic530_ldo_of_map_mode(unsigned int rpmh_mode) +{ + unsigned int mode; + + switch (rpmh_mode) { + case RPMH_REGULATOR_MODE_HPM: + mode = REGULATOR_MODE_FAST; + break; + case RPMH_REGULATOR_MODE_AUTO: + mode = REGULATOR_MODE_NORMAL; + break; + case RPMH_REGULATOR_MODE_LPM: + mode = REGULATOR_MODE_IDLE; + break; + case RPMH_REGULATOR_MODE_RET: + mode = REGULATOR_MODE_STANDBY; + break; + default: + mode = REGULATOR_MODE_INVALID; + break; + } + return mode; +} + static const int pmic_mode_map_pmic4_smps[REGULATOR_MODE_STANDBY + 1] = { [REGULATOR_MODE_INVALID] = -EINVAL, [REGULATOR_MODE_STANDBY] = PMIC4_SMPS_MODE_RETENTION, @@ -904,671 +979,816 @@ static const struct rpmh_vreg_hw_data pmic5_bob = { .of_map_mode = rpmh_regulator_pmic4_bob_of_map_mode, }; -#define RPMH_VREG(_name, _resource_name, _hw_data, _supply_name) \ +static const struct rpmh_vreg_hw_data pmic5_nldo530 = { + .regulator_type = VRM, + .ops = &rpmh_regulator_vrm_drms_ops, + .voltage_ranges = (struct linear_range[]) { + REGULATOR_LINEAR_RANGE(320000, 0, 210, 8000), + }, + .n_linear_ranges = 1, + .n_voltages = 211, + .hpm_min_load_uA = 30000, + .pmic_mode_map = pmic_mode_map_pmic530_ldo, + .of_map_mode = rpmh_regulator_pmic530_ldo_of_map_mode, +}; + +static const struct rpmh_vreg_hw_data pmic5_pldo530_mvp150 = { + .regulator_type = VRM, + .ops = &rpmh_regulator_vrm_drms_ops, + .voltage_ranges = (struct linear_range[]) { + REGULATOR_LINEAR_RANGE(1504000, 0, 255, 8000), + }, + .n_linear_ranges = 1, + .n_voltages = 256, + .hpm_min_load_uA = 10000, + .pmic_mode_map = pmic_mode_map_pmic530_ldo, + .of_map_mode = rpmh_regulator_pmic530_ldo_of_map_mode, +}; + +static const struct rpmh_vreg_hw_data pmic5_pldo530_mvp300 = { + .regulator_type = VRM, + .ops = &rpmh_regulator_vrm_drms_ops, + .voltage_ranges = (struct linear_range[]) { + REGULATOR_LINEAR_RANGE(1504000, 0, 255, 8000), + }, + .n_linear_ranges = 1, + .n_voltages = 256, + .hpm_min_load_uA = 20000, + .pmic_mode_map = pmic_mode_map_pmic530_ldo, + .of_map_mode = rpmh_regulator_pmic530_ldo_of_map_mode, +}; + +static const struct rpmh_vreg_hw_data pmic5_pldo530_mvp600 = { + .regulator_type = VRM, + .ops = &rpmh_regulator_vrm_drms_ops, + .voltage_ranges = (struct linear_range[]) { + REGULATOR_LINEAR_RANGE(1504000, 0, 255, 8000), + }, + .n_linear_ranges = 1, + .n_voltages = 256, + .hpm_min_load_uA = 40000, + .pmic_mode_map = pmic_mode_map_pmic530_ldo, + .of_map_mode = rpmh_regulator_pmic530_ldo_of_map_mode, +}; + +static const struct rpmh_vreg_hw_data pmic5_ftsmps530 = { + .regulator_type = VRM, + .ops = &rpmh_regulator_vrm_ops, + .voltage_ranges = (struct linear_range[]) { + REGULATOR_LINEAR_RANGE(252000, 0, 305, 4000), + REGULATOR_LINEAR_RANGE(1480000, 306, 464, 8000), + }, + .n_linear_ranges = 2, + .n_voltages = 465, + .pmic_mode_map = pmic_mode_map_pmic5_smps, + .of_map_mode = rpmh_regulator_pmic4_smps_of_map_mode, +}; + +#define RPMH_VREG(_name, _vreg_hw_type, _index, _hw_data, _supply_name) \ { \ .name = _name, \ - .resource_name = _resource_name, \ + .vreg_hw_type = _vreg_hw_type, \ + .index = _index, \ .hw_data = _hw_data, \ .supply_name = _supply_name, \ } static const struct rpmh_vreg_init_data pm8998_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic4_ftsmps426, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic4_ftsmps426, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic4_hfsmps3, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic4_hfsmps3, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic4_hfsmps3, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic4_ftsmps426, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic4_ftsmps426, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic4_ftsmps426, "vdd-s8"), - RPMH_VREG("smps9", "smp%s9", &pmic4_ftsmps426, "vdd-s9"), - RPMH_VREG("smps10", "smp%s10", &pmic4_ftsmps426, "vdd-s10"), - RPMH_VREG("smps11", "smp%s11", &pmic4_ftsmps426, "vdd-s11"), - RPMH_VREG("smps12", "smp%s12", &pmic4_ftsmps426, "vdd-s12"), - RPMH_VREG("smps13", "smp%s13", &pmic4_ftsmps426, "vdd-s13"), - RPMH_VREG("ldo1", "ldo%s1", &pmic4_nldo, "vdd-l1-l27"), - RPMH_VREG("ldo2", "ldo%s2", &pmic4_nldo, "vdd-l2-l8-l17"), - RPMH_VREG("ldo3", "ldo%s3", &pmic4_nldo, "vdd-l3-l11"), - RPMH_VREG("ldo4", "ldo%s4", &pmic4_nldo, "vdd-l4-l5"), - RPMH_VREG("ldo5", "ldo%s5", &pmic4_nldo, "vdd-l4-l5"), - RPMH_VREG("ldo6", "ldo%s6", &pmic4_pldo, "vdd-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo8", "ldo%s8", &pmic4_nldo, "vdd-l2-l8-l17"), - RPMH_VREG("ldo9", "ldo%s9", &pmic4_pldo, "vdd-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic4_pldo, "vdd-l10-l23-l25"), - RPMH_VREG("ldo11", "ldo%s11", &pmic4_nldo, "vdd-l3-l11"), - RPMH_VREG("ldo12", "ldo%s12", &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo13", "ldo%s13", &pmic4_pldo, "vdd-l13-l19-l21"), - RPMH_VREG("ldo14", "ldo%s14", &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo15", "ldo%s15", &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic4_pldo, "vdd-l16-l28"), - RPMH_VREG("ldo17", "ldo%s17", &pmic4_nldo, "vdd-l2-l8-l17"), - RPMH_VREG("ldo18", "ldo%s18", &pmic4_pldo, "vdd-l18-l22"), - RPMH_VREG("ldo19", "ldo%s19", &pmic4_pldo, "vdd-l13-l19-l21"), - RPMH_VREG("ldo20", "ldo%s20", &pmic4_pldo, "vdd-l20-l24"), - RPMH_VREG("ldo21", "ldo%s21", &pmic4_pldo, "vdd-l13-l19-l21"), - RPMH_VREG("ldo22", "ldo%s22", &pmic4_pldo, "vdd-l18-l22"), - RPMH_VREG("ldo23", "ldo%s23", &pmic4_pldo, "vdd-l10-l23-l25"), - RPMH_VREG("ldo24", "ldo%s24", &pmic4_pldo, "vdd-l20-l24"), - RPMH_VREG("ldo25", "ldo%s25", &pmic4_pldo, "vdd-l10-l23-l25"), - RPMH_VREG("ldo26", "ldo%s26", &pmic4_nldo, "vdd-l26"), - RPMH_VREG("ldo27", "ldo%s27", &pmic4_nldo, "vdd-l1-l27"), - RPMH_VREG("ldo28", "ldo%s28", &pmic4_pldo, "vdd-l16-l28"), - RPMH_VREG("lvs1", "vs%s1", &pmic4_lvs, "vin-lvs-1-2"), - RPMH_VREG("lvs2", "vs%s2", &pmic4_lvs, "vin-lvs-1-2"), + RPMH_VREG("smps1", SMPS, 1, &pmic4_ftsmps426, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic4_ftsmps426, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic4_hfsmps3, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic4_hfsmps3, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic4_hfsmps3, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic4_ftsmps426, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic4_ftsmps426, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic4_ftsmps426, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic4_ftsmps426, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic4_ftsmps426, "vdd-s10"), + RPMH_VREG("smps11", SMPS, 11, &pmic4_ftsmps426, "vdd-s11"), + RPMH_VREG("smps12", SMPS, 12, &pmic4_ftsmps426, "vdd-s12"), + RPMH_VREG("smps13", SMPS, 13, &pmic4_ftsmps426, "vdd-s13"), + RPMH_VREG("ldo1", LDO, 1, &pmic4_nldo, "vdd-l1-l27"), + RPMH_VREG("ldo2", LDO, 2, &pmic4_nldo, "vdd-l2-l8-l17"), + RPMH_VREG("ldo3", LDO, 3, &pmic4_nldo, "vdd-l3-l11"), + RPMH_VREG("ldo4", LDO, 4, &pmic4_nldo, "vdd-l4-l5"), + RPMH_VREG("ldo5", LDO, 5, &pmic4_nldo, "vdd-l4-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic4_pldo, "vdd-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo8", LDO, 8, &pmic4_nldo, "vdd-l2-l8-l17"), + RPMH_VREG("ldo9", LDO, 9, &pmic4_pldo, "vdd-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic4_pldo, "vdd-l10-l23-l25"), + RPMH_VREG("ldo11", LDO, 11, &pmic4_nldo, "vdd-l3-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo13", LDO, 13, &pmic4_pldo, "vdd-l13-l19-l21"), + RPMH_VREG("ldo14", LDO, 14, &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo15", LDO, 15, &pmic4_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic4_pldo, "vdd-l16-l28"), + RPMH_VREG("ldo17", LDO, 17, &pmic4_nldo, "vdd-l2-l8-l17"), + RPMH_VREG("ldo18", LDO, 18, &pmic4_pldo, "vdd-l18-l22"), + RPMH_VREG("ldo19", LDO, 19, &pmic4_pldo, "vdd-l13-l19-l21"), + RPMH_VREG("ldo20", LDO, 20, &pmic4_pldo, "vdd-l20-l24"), + RPMH_VREG("ldo21", LDO, 21, &pmic4_pldo, "vdd-l13-l19-l21"), + RPMH_VREG("ldo22", LDO, 22, &pmic4_pldo, "vdd-l18-l22"), + RPMH_VREG("ldo23", LDO, 23, &pmic4_pldo, "vdd-l10-l23-l25"), + RPMH_VREG("ldo24", LDO, 24, &pmic4_pldo, "vdd-l20-l24"), + RPMH_VREG("ldo25", LDO, 25, &pmic4_pldo, "vdd-l10-l23-l25"), + RPMH_VREG("ldo26", LDO, 26, &pmic4_nldo, "vdd-l26"), + RPMH_VREG("ldo27", LDO, 27, &pmic4_nldo, "vdd-l1-l27"), + RPMH_VREG("ldo28", LDO, 28, &pmic4_pldo, "vdd-l16-l28"), + RPMH_VREG("lvs1", VS, 1, &pmic4_lvs, "vin-lvs-1-2"), + RPMH_VREG("lvs2", VS, 2, &pmic4_lvs, "vin-lvs-1-2"), {} }; static const struct rpmh_vreg_init_data pmg1110_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), {} }; static const struct rpmh_vreg_init_data pmi8998_vreg_data[] = { - RPMH_VREG("bob", "bob%s1", &pmic4_bob, "vdd-bob"), + RPMH_VREG("bob", BOB, 1, &pmic4_bob, "vdd-bob"), {} }; static const struct rpmh_vreg_init_data pm8005_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic4_ftsmps426, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic4_ftsmps426, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic4_ftsmps426, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic4_ftsmps426, "vdd-s4"), + RPMH_VREG("smps1", SMPS, 1, &pmic4_ftsmps426, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic4_ftsmps426, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic4_ftsmps426, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic4_ftsmps426, "vdd-s4"), {} }; static const struct rpmh_vreg_init_data pm8150_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_hfsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_hfsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps510, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps510, "vdd-s8"), - RPMH_VREG("smps9", "smp%s9", &pmic5_ftsmps510, "vdd-s9"), - RPMH_VREG("smps10", "smp%s10", &pmic5_ftsmps510, "vdd-s10"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1-l8-l11"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_pldo, "vdd-l2-l10"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3-l4-l5-l18"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l3-l4-l5-l18"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo, "vdd-l3-l4-l5-l18"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo, "vdd-l6-l9"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l1-l8-l11"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l6-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, "vdd-l2-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_nldo, "vdd-l1-l8-l11"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo, "vdd-l13-l16-l17"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_pldo, "vdd-l13-l16-l17"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_pldo, "vdd-l13-l16-l17"), - RPMH_VREG("ldo18", "ldo%s18", &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_hfsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_hfsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps510, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps510, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps510, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic5_ftsmps510, "vdd-s10"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1-l8-l11"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo, "vdd-l2-l10"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo, "vdd-l6-l9"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l1-l8-l11"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l6-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, "vdd-l2-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_nldo, "vdd-l1-l8-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo, "vdd-l13-l16-l17"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo, "vdd-l13-l16-l17"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo, "vdd-l13-l16-l17"), + RPMH_VREG("ldo18", LDO, 18, &pmic5_nldo, "vdd-l3-l4-l5-l18"), {} }; static const struct rpmh_vreg_init_data pm8150l_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps510, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_hfsmps510, "vdd-s8"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_pldo_lv, "vdd-l1-l8"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l2-l3"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l2-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_pldo, "vdd-l4-l5-l6"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l4-l5-l6"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l4-l5-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l7-l11"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_pldo_lv, "vdd-l1-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_pldo, "vdd-l9-l10"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, "vdd-l9-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo, "vdd-l7-l11"), - RPMH_VREG("bob", "bob%s1", &pmic5_bob, "vdd-bob"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps510, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_hfsmps510, "vdd-s8"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_pldo_lv, "vdd-l1-l8"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_pldo, "vdd-l4-l5-l6"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l4-l5-l6"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l4-l5-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l7-l11"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_pldo_lv, "vdd-l1-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_pldo, "vdd-l9-l10"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, "vdd-l9-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo, "vdd-l7-l11"), + RPMH_VREG("bob", BOB, 1, &pmic5_bob, "vdd-bob"), {} }; static const struct rpmh_vreg_init_data pmm8155au_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_hfsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_hfsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps510, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps510, "vdd-s8"), - RPMH_VREG("smps9", "smp%s9", &pmic5_ftsmps510, "vdd-s9"), - RPMH_VREG("smps10", "smp%s10", &pmic5_ftsmps510, "vdd-s10"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1-l8-l11"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_pldo, "vdd-l2-l10"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3-l4-l5-l18"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l3-l4-l5-l18"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo, "vdd-l3-l4-l5-l18"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo, "vdd-l6-l9"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l1-l8-l11"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l6-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, "vdd-l2-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_nldo, "vdd-l1-l8-l11"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo, "vdd-l13-l16-l17"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_pldo, "vdd-l13-l16-l17"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_pldo, "vdd-l13-l16-l17"), - RPMH_VREG("ldo18", "ldo%s18", &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_hfsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_hfsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps510, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps510, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps510, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic5_ftsmps510, "vdd-s10"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1-l8-l11"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo, "vdd-l2-l10"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo, "vdd-l3-l4-l5-l18"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo, "vdd-l6-l9"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l1-l8-l11"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l6-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, "vdd-l2-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_nldo, "vdd-l1-l8-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo, "vdd-l13-l16-l17"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_pldo_lv, "vdd-l7-l12-l14-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo, "vdd-l13-l16-l17"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo, "vdd-l13-l16-l17"), + RPMH_VREG("ldo18", LDO, 18, &pmic5_nldo, "vdd-l3-l4-l5-l18"), {} }; static const struct rpmh_vreg_init_data pmm8654au_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps527, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps527, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps527, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps527, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps527, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps527, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps527, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps527, "vdd-s8"), - RPMH_VREG("smps9", "smp%s9", &pmic5_ftsmps527, "vdd-s9"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo515, "vdd-s9"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo515, "vdd-l2-l3"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo515, "vdd-l2-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo515, "vdd-s9"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo515, "vdd-s9"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo515, "vdd-l6-l7"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_nldo515, "vdd-l6-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_pldo515_mv, "vdd-l8-l9"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_pldo, "vdd-l8-l9"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps527, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps527, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps527, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps527, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps527, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps527, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps527, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps527, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps527, "vdd-s9"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-s9"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo515, "vdd-l2-l3"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l2-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo515, "vdd-s9"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo515, "vdd-s9"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo515, "vdd-l6-l7"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo515, "vdd-l6-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_pldo515_mv, "vdd-l8-l9"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_pldo, "vdd-l8-l9"), {} }; static const struct rpmh_vreg_init_data pm8350_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps510, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps510, "vdd-s8"), - RPMH_VREG("smps9", "smp%s9", &pmic5_ftsmps510, "vdd-s9"), - RPMH_VREG("smps10", "smp%s10", &pmic5_hfsmps510, "vdd-s10"), - RPMH_VREG("smps11", "smp%s11", &pmic5_hfsmps510, "vdd-s11"), - RPMH_VREG("smps12", "smp%s12", &pmic5_hfsmps510, "vdd-s12"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1-l4"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_pldo, "vdd-l2-l7"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3-l5"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l1-l4"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo, "vdd-l3-l5"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo, "vdd-l6-l9-l10"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l2-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l6-l9-l10"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_nldo, "vdd-l6-l9-l10"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps510, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps510, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps510, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic5_hfsmps510, "vdd-s10"), + RPMH_VREG("smps11", SMPS, 11, &pmic5_hfsmps510, "vdd-s11"), + RPMH_VREG("smps12", SMPS, 12, &pmic5_hfsmps510, "vdd-s12"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1-l4"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo, "vdd-l2-l7"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3-l5"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l1-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo, "vdd-l3-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo, "vdd-l6-l9-l10"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l2-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l6-l9-l10"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_nldo, "vdd-l6-l9-l10"), {} }; static const struct rpmh_vreg_init_data pm8350c_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_hfsmps515, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps510, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps510, "vdd-s8"), - RPMH_VREG("smps9", "smp%s9", &pmic5_ftsmps510, "vdd-s9"), - RPMH_VREG("smps10", "smp%s10", &pmic5_ftsmps510, "vdd-s10"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_pldo_lv, "vdd-l1-l12"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_pldo_lv, "vdd-l2-l8"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l6-l9-l11"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_pldo_lv, "vdd-l2-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_pldo, "vdd-l6-l9-l11"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_nldo, "vdd-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo, "vdd-l6-l9-l11"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_pldo_lv, "vdd-l1-l12"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), - RPMH_VREG("bob", "bob%s1", &pmic5_bob, "vdd-bob"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_hfsmps515, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps510, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps510, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps510, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic5_ftsmps510, "vdd-s10"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_pldo_lv, "vdd-l1-l12"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo_lv, "vdd-l2-l8"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l6-l9-l11"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_pldo_lv, "vdd-l2-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_pldo, "vdd-l6-l9-l11"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_nldo, "vdd-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo, "vdd-l6-l9-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_pldo_lv, "vdd-l1-l12"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo, "vdd-l3-l4-l5-l7-l13"), + RPMH_VREG("bob", BOB, 1, &pmic5_bob, "vdd-bob"), {} }; static const struct rpmh_vreg_init_data pm8450_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps520, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps520, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps520, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps520, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps520, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps520, "vdd-s6"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_pldo_lv, "vdd-l4"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps520, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps520, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps520, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps520, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps520, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps520, "vdd-s6"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_pldo_lv, "vdd-l4"), {} }; static const struct rpmh_vreg_init_data pm8550_vreg_data[] = { - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo515, "vdd-l1-l4-l10"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_pldo, "vdd-l2-l13-l14"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo515, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo515, "vdd-l1-l4-l10"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l5-l16"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l6-l7"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l6-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_pldo, "vdd-l8-l9"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_pldo, "vdd-l8-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_nldo515, "vdd-l1-l4-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_nldo515, "vdd-l11"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_nldo515, "vdd-l12"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo, "vdd-l2-l13-l14"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_pldo, "vdd-l2-l13-l14"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_nldo515, "vdd-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_pldo, "vdd-l5-l16"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_pldo, "vdd-l17"), - RPMH_VREG("bob1", "bob%s1", &pmic5_bob, "vdd-bob1"), - RPMH_VREG("bob2", "bob%s2", &pmic5_bob, "vdd-bob2"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-l1-l4-l10"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo, "vdd-l2-l13-l14"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo515, "vdd-l1-l4-l10"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l5-l16"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l6-l7"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l6-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_pldo, "vdd-l8-l9"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_pldo, "vdd-l8-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_nldo515, "vdd-l1-l4-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_nldo515, "vdd-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_nldo515, "vdd-l12"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo, "vdd-l2-l13-l14"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo, "vdd-l2-l13-l14"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo515, "vdd-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo, "vdd-l5-l16"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo, "vdd-l17"), + RPMH_VREG("bob1", BOB, 1, &pmic5_bob, "vdd-bob1"), + RPMH_VREG("bob2", BOB, 2, &pmic5_bob, "vdd-bob2"), {} }; static const struct rpmh_vreg_init_data pm8550vs_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps525, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps525, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps525, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps525, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps525, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps525, "vdd-s6"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo515, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo515, "vdd-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo515, "vdd-l3"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps525, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps525, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps525, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps525, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps525, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps525, "vdd-s6"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo515, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l3"), {} }; static const struct rpmh_vreg_init_data pm8550ve_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps525, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps525, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps525, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps525, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps525, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps525, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps525, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps525, "vdd-s8"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo515, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo515, "vdd-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo515, "vdd-l3"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps525, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps525, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps525, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps525, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps525, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps525, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps525, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps525, "vdd-s8"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo515, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l3"), {} }; static const struct rpmh_vreg_init_data pmc8380_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps525, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps525, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps525, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps525, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps525, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps525, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps525, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps525, "vdd-s8"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo515, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo515, "vdd-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo515, "vdd-l3"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps525, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps525, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps525, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps525, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps525, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps525, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps525, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps525, "vdd-s8"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo515, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l3"), {} }; static const struct rpmh_vreg_init_data pm8009_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_hfsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_hfsmps515, "vdd-s2"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l4"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l5-l6"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l5-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo_lv, "vdd-l7"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_hfsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_hfsmps515, "vdd-s2"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l5-l6"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l5-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo_lv, "vdd-l7"), {} }; static const struct rpmh_vreg_init_data pm8009_1_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_hfsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_hfsmps515_1, "vdd-s2"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l4"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l5-l6"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l5-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo_lv, "vdd-l7"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_hfsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_hfsmps515_1, "vdd-s2"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l5-l6"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l5-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo_lv, "vdd-l7"), {} }; static const struct rpmh_vreg_init_data pm8010_vreg_data[] = { - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo502, "vdd-l1-l2"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo502, "vdd-l1-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_pldo502ln, "vdd-l3-l4"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_pldo502ln, "vdd-l3-l4"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo502, "vdd-l5"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo502ln, "vdd-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo502, "vdd-l7"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo502, "vdd-l1-l2"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo502, "vdd-l1-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_pldo502ln, "vdd-l3-l4"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_pldo502ln, "vdd-l3-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo502, "vdd-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo502ln, "vdd-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo502, "vdd-l7"), }; static const struct rpmh_vreg_init_data pm6150_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_hfsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_hfsmps510, "vdd-s5"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l2-l3"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l2-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l4-l7-l8"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo, "vdd-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_nldo, "vdd-l4-l7-l8"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l4-l7-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo_lv, "vdd-l10-l14-l15"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo_lv, "vdd-l11-l12-l13"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_pldo_lv, "vdd-l11-l12-l13"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo_lv, "vdd-l11-l12-l13"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_pldo_lv, "vdd-l10-l14-l15"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_pldo_lv, "vdd-l10-l14-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), - RPMH_VREG("ldo18", "ldo%s18", &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), - RPMH_VREG("ldo19", "ldo%s19", &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_hfsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_hfsmps510, "vdd-s5"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l4-l7-l8"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo, "vdd-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo, "vdd-l4-l7-l8"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l4-l7-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo_lv, "vdd-l10-l14-l15"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo_lv, "vdd-l11-l12-l13"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_pldo_lv, "vdd-l11-l12-l13"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo_lv, "vdd-l11-l12-l13"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo_lv, "vdd-l10-l14-l15"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_pldo_lv, "vdd-l10-l14-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), + RPMH_VREG("ldo18", LDO, 18, &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), + RPMH_VREG("ldo19", LDO, 19, &pmic5_pldo, "vdd-l5-l16-l17-l18-l19"), {} }; static const struct rpmh_vreg_init_data pm6150l_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps510, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_hfsmps510, "vdd-s8"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_pldo_lv, "vdd-l1-l8"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l2-l3"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l2-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_pldo, "vdd-l4-l5-l6"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l4-l5-l6"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l4-l5-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l7-l11"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_pldo, "vdd-l1-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_pldo, "vdd-l9-l10"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, "vdd-l9-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo, "vdd-l7-l11"), - RPMH_VREG("bob", "bob%s1", &pmic5_bob, "vdd-bob"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps510, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_hfsmps510, "vdd-s8"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_pldo_lv, "vdd-l1-l8"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_pldo, "vdd-l4-l5-l6"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l4-l5-l6"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l4-l5-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l7-l11"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_pldo, "vdd-l1-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_pldo, "vdd-l9-l10"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, "vdd-l9-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo, "vdd-l7-l11"), + RPMH_VREG("bob", BOB, 1, &pmic5_bob, "vdd-bob"), {} }; static const struct rpmh_vreg_init_data pm6350_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, NULL), - RPMH_VREG("smps2", "smp%s2", &pmic5_hfsmps510, NULL), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, NULL), + RPMH_VREG("smps2", SMPS, 2, &pmic5_hfsmps510, NULL), /* smps3 - smps5 not configured */ - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, NULL), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_pldo, NULL), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_pldo, NULL), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, NULL), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, NULL), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, NULL), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, NULL), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_pldo, NULL), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_pldo, NULL), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, NULL), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo, NULL), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_pldo, NULL), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_nldo, NULL), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_pldo, NULL), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_nldo, NULL), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_nldo, NULL), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, NULL), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo, NULL), + RPMH_VREG("ldo3", LDO, 3, &pmic5_pldo, NULL), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, NULL), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, NULL), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, NULL), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, NULL), + RPMH_VREG("ldo8", LDO, 8, &pmic5_pldo, NULL), + RPMH_VREG("ldo9", LDO, 9, &pmic5_pldo, NULL), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, NULL), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo, NULL), + RPMH_VREG("ldo12", LDO, 12, &pmic5_pldo, NULL), + RPMH_VREG("ldo13", LDO, 13, &pmic5_nldo, NULL), + RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo, NULL), + RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo, NULL), + RPMH_VREG("ldo16", LDO, 16, &pmic5_nldo, NULL), /* ldo17 not configured */ - RPMH_VREG("ldo18", "ldo%s18", &pmic5_nldo, NULL), - RPMH_VREG("ldo19", "ldo%s19", &pmic5_nldo, NULL), - RPMH_VREG("ldo20", "ldo%s20", &pmic5_nldo, NULL), - RPMH_VREG("ldo21", "ldo%s21", &pmic5_nldo, NULL), - RPMH_VREG("ldo22", "ldo%s22", &pmic5_nldo, NULL), + RPMH_VREG("ldo18", LDO, 18, &pmic5_nldo, NULL), + RPMH_VREG("ldo19", LDO, 19, &pmic5_nldo, NULL), + RPMH_VREG("ldo20", LDO, 20, &pmic5_nldo, NULL), + RPMH_VREG("ldo21", LDO, 21, &pmic5_nldo, NULL), + RPMH_VREG("ldo22", LDO, 22, &pmic5_nldo, NULL), +}; + +static const struct rpmh_vreg_init_data pmcx0102_vreg_data[] = { + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps530, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps530, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps530, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps530, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps530, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps530, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps530, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps530, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps530, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic5_ftsmps530, "vdd-s10"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo530, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo530, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo530, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo530, "vdd-l4"), + {} +}; + +static const struct rpmh_vreg_init_data pmh0101_vreg_data[] = { + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo530, "vdd-l1-l4-l10"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo530_mvp300, "vdd-l2-l13-l14"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo530, "vdd-l3-l11"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo530, "vdd-l1-l4-l10"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo530_mvp150, "vdd-l5-l16"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo530_mvp300, "vdd-l6-l7"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo530_mvp300, "vdd-l6-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_pldo530_mvp150, "vdd-l8-l9"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_pldo515_mv, "vdd-l8-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_nldo530, "vdd-l1-l4-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_nldo530, "vdd-l3-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_nldo530, "vdd-l12"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo530_mvp150, "vdd-l2-l13-l14"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo530_mvp150, "vdd-l2-l13-l14"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo530, "vdd-l15"), + RPMH_VREG("ldo16", LDO, 15, &pmic5_pldo530_mvp600, "vdd-l5-l16"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo515_mv, "vdd-l17"), + RPMH_VREG("ldo18", LDO, 18, &pmic5_nldo530, "vdd-l18"), + RPMH_VREG("bob1", BOB, 1, &pmic5_bob, "vdd-bob1"), + RPMH_VREG("bob2", BOB, 2, &pmic5_bob, "vdd-bob2"), + {} +}; + +static const struct rpmh_vreg_init_data pmh0104_vreg_data[] = { + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps530, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps530, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps530, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps530, "vdd-s4"), + {} +}; + +static const struct rpmh_vreg_init_data pmh0110_vreg_data[] = { + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps530, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps530, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps530, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps530, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps530, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps530, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps530, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps530, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps530, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic5_ftsmps530, "vdd-s10"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo530, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo530, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo530, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo530, "vdd-l4"), + {} }; static const struct rpmh_vreg_init_data pmx55_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_hfsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_hfsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_hfsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_hfsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_hfsmps510, "vdd-s7"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1-l2"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l1-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3-l9"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l4-l12"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l5-l6"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l5-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_nldo, "vdd-l7-l8"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l7-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l3-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, "vdd-l10-l11-l13"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo, "vdd-l10-l11-l13"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_nldo, "vdd-l4-l12"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo, "vdd-l10-l11-l13"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_nldo, "vdd-l14"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_nldo, "vdd-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_pldo, "vdd-l16"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_hfsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_hfsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_hfsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_hfsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_hfsmps510, "vdd-s7"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1-l2"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l1-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3-l9"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l4-l12"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l5-l6"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l5-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo, "vdd-l7-l8"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l7-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l3-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, "vdd-l10-l11-l13"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo, "vdd-l10-l11-l13"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_nldo, "vdd-l4-l12"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo, "vdd-l10-l11-l13"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_nldo, "vdd-l14"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo, "vdd-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo, "vdd-l16"), {} }; static const struct rpmh_vreg_init_data pmx65_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_hfsmps510, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_hfsmps510, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_hfsmps510, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_hfsmps510, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps510, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_hfsmps510, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_hfsmps510, "vdd-s8"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l2-l18"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l4"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo, "vdd-l5-l6-l16"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo, "vdd-l5-l6-l16"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_nldo, "vdd-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l8-l9"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l8-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, "vdd-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo, "vdd-l11-l13"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_nldo, "vdd-l12"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo, "vdd-l11-l13"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_nldo, "vdd-l14"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_nldo, "vdd-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_pldo, "vdd-l5-l6-l16"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_nldo, "vdd-l17"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_hfsmps510, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_hfsmps510, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_hfsmps510, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_hfsmps510, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps510, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_hfsmps510, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_hfsmps510, "vdd-s8"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l2-l18"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo, "vdd-l5-l6-l16"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo, "vdd-l5-l6-l16"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo, "vdd-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l8-l9"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l8-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, "vdd-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo, "vdd-l11-l13"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_nldo, "vdd-l12"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo, "vdd-l11-l13"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_nldo, "vdd-l14"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo, "vdd-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo, "vdd-l5-l6-l16"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_nldo, "vdd-l17"), /* ldo18 not configured */ - RPMH_VREG("ldo19", "ldo%s19", &pmic5_nldo, "vdd-l19"), - RPMH_VREG("ldo20", "ldo%s20", &pmic5_nldo, "vdd-l20"), - RPMH_VREG("ldo21", "ldo%s21", &pmic5_nldo, "vdd-l21"), + RPMH_VREG("ldo19", LDO, 19, &pmic5_nldo, "vdd-l19"), + RPMH_VREG("ldo20", LDO, 20, &pmic5_nldo, "vdd-l20"), + RPMH_VREG("ldo21", LDO, 21, &pmic5_nldo, "vdd-l21"), {} }; static const struct rpmh_vreg_init_data pmx75_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps525, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps525, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps525, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps525, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps525, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps525, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps525, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_ftsmps525, "vdd-s8"), - RPMH_VREG("smps9", "smp%s9", &pmic5_ftsmps525, "vdd-s9"), - RPMH_VREG("smps10", "smp%s10", &pmic5_ftsmps525, "vdd-s10"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo515, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo515, "vdd-l2-18"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo515, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo515, "vdd-l4-l16"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_pldo_lv, "vdd-l5-l6"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_pldo_lv, "vdd-l5-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_nldo515, "vdd-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo515, "vdd-l8-l9"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo515, "vdd-l8-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo, "vdd-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo, "vdd-l11-l13"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_nldo515, "vdd-l12"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo, "vdd-l11-l13"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_nldo515, "vdd-l14"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_nldo515, "vdd-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_nldo515, "vdd-l4-l16"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_nldo515, "vdd-l17"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps525, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps525, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps525, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps525, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps525, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps525, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps525, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_ftsmps525, "vdd-s8"), + RPMH_VREG("smps9", SMPS, 9, &pmic5_ftsmps525, "vdd-s9"), + RPMH_VREG("smps10", SMPS, 10, &pmic5_ftsmps525, "vdd-s10"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo515, "vdd-l2-18"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo515, "vdd-l4-l16"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_pldo_lv, "vdd-l5-l6"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_pldo_lv, "vdd-l5-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo515, "vdd-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo515, "vdd-l8-l9"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo515, "vdd-l8-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo, "vdd-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo, "vdd-l11-l13"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_nldo515, "vdd-l12"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo, "vdd-l11-l13"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_nldo515, "vdd-l14"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo515, "vdd-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_nldo515, "vdd-l4-l16"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_nldo515, "vdd-l17"), /* ldo18 not configured */ - RPMH_VREG("ldo19", "ldo%s19", &pmic5_nldo515, "vdd-l19"), - RPMH_VREG("ldo20", "ldo%s20", &pmic5_nldo515, "vdd-l20-l21"), - RPMH_VREG("ldo21", "ldo%s21", &pmic5_nldo515, "vdd-l20-l21"), + RPMH_VREG("ldo19", LDO, 19, &pmic5_nldo515, "vdd-l19"), + RPMH_VREG("ldo20", LDO, 20, &pmic5_nldo515, "vdd-l20-l21"), + RPMH_VREG("ldo21", LDO, 21, &pmic5_nldo515, "vdd-l20-l21"), }; static const struct rpmh_vreg_init_data pm7325_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_hfsmps510, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps520, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps520, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps520, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps520, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps520, "vdd-s6"), - RPMH_VREG("smps7", "smp%s7", &pmic5_ftsmps520, "vdd-s7"), - RPMH_VREG("smps8", "smp%s8", &pmic5_hfsmps510, "vdd-s8"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1-l4-l12-l15"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_pldo, "vdd-l2-l7"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo, "vdd-l1-l4-l12-l15"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo, "vdd-l5"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo, "vdd-l6-l9-l10"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l2-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l6-l9-l10"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_nldo, "vdd-l6-l9-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_nldo, "vdd-l1-l4-l12-l15"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_nldo, "vdd-l13"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_nldo, "vdd-l14-l16"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_nldo, "vdd-l1-l4-l12-l15"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_nldo, "vdd-l14-l16"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), - RPMH_VREG("ldo18", "ldo%s18", &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), - RPMH_VREG("ldo19", "ldo%s19", &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_hfsmps510, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps520, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps520, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps520, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps520, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps520, "vdd-s6"), + RPMH_VREG("smps7", SMPS, 7, &pmic5_ftsmps520, "vdd-s7"), + RPMH_VREG("smps8", SMPS, 8, &pmic5_hfsmps510, "vdd-s8"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1-l4-l12-l15"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_pldo, "vdd-l2-l7"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo, "vdd-l1-l4-l12-l15"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo, "vdd-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo, "vdd-l6-l9-l10"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l2-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l6-l9-l10"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_nldo, "vdd-l6-l9-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_nldo, "vdd-l1-l4-l12-l15"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_nldo, "vdd-l13"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_nldo, "vdd-l14-l16"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo, "vdd-l1-l4-l12-l15"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_nldo, "vdd-l14-l16"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), + RPMH_VREG("ldo18", LDO, 18, &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), + RPMH_VREG("ldo19", LDO, 19, &pmic5_pldo_lv, "vdd-l11-l17-l18-l19"), {} }; static const struct rpmh_vreg_init_data pm7550_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps525, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps525, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_ftsmps525, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic5_ftsmps525, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic5_ftsmps525, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic5_ftsmps525, "vdd-s6"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo515, "vdd-l1"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo515, "vdd-l2-l3"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo515, "vdd-l2-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_nldo515, "vdd-l4-l5"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo515, "vdd-l4-l5"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo515, "vdd-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_nldo515, "vdd-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo515, "vdd-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo515, "vdd-l9-l10"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_nldo515, "vdd-l9-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_nldo515, "vdd-l11"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_pldo515_mv, "vdd-l12-l14"), - RPMH_VREG("ldo13", "ldo%s13", &pmic5_pldo515_mv, "vdd-l13-l16"), - RPMH_VREG("ldo14", "ldo%s14", &pmic5_pldo, "vdd-l12-l14"), - RPMH_VREG("ldo15", "ldo%s15", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("ldo16", "ldo%s16", &pmic5_pldo, "vdd-l13-l16"), - RPMH_VREG("ldo17", "ldo%s17", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("ldo18", "ldo%s18", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("ldo19", "ldo%s19", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("ldo20", "ldo%s20", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("ldo21", "ldo%s21", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("ldo22", "ldo%s22", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("ldo23", "ldo%s23", &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), - RPMH_VREG("bob", "bob%s1", &pmic5_bob, "vdd-bob"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps525, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps525, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_ftsmps525, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic5_ftsmps525, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic5_ftsmps525, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic5_ftsmps525, "vdd-s6"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-l1"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo515, "vdd-l2-l3"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l2-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo515, "vdd-l4-l5"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo515, "vdd-l4-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo515, "vdd-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo515, "vdd-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo515, "vdd-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo515, "vdd-l9-l10"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_nldo515, "vdd-l9-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_nldo515, "vdd-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_pldo515_mv, "vdd-l12-l14"), + RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo515_mv, "vdd-l13-l16"), + RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo, "vdd-l12-l14"), + RPMH_VREG("ldo15", LDO, 15, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo, "vdd-l13-l16"), + RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("ldo18", LDO, 18, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("ldo19", LDO, 19, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("ldo20", LDO, 20, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("ldo21", LDO, 21, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("ldo22", LDO, 22, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("ldo23", LDO, 23, &pmic5_pldo, "vdd-l15-l17-l18-l19-l20-l21-l22-l23"), + RPMH_VREG("bob", BOB, 1, &pmic5_bob, "vdd-bob"), {} }; static const struct rpmh_vreg_init_data pmr735a_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic5_ftsmps520, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic5_ftsmps520, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic5_hfsmps515, "vdd-s3"), - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1-l2"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l1-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_pldo_lv, "vdd-l4"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo, "vdd-l5-l6"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo, "vdd-l5-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_pldo, "vdd-l7-bob"), + RPMH_VREG("smps1", SMPS, 1, &pmic5_ftsmps520, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic5_ftsmps520, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic5_hfsmps515, "vdd-s3"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1-l2"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l1-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_pldo_lv, "vdd-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo, "vdd-l5-l6"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo, "vdd-l5-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_pldo, "vdd-l7-bob"), {} }; static const struct rpmh_vreg_init_data pmr735b_vreg_data[] = { - RPMH_VREG("ldo1", "ldo%s1", &pmic5_nldo, "vdd-l1-l2"), - RPMH_VREG("ldo2", "ldo%s2", &pmic5_nldo, "vdd-l1-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic5_nldo, "vdd-l3"), - RPMH_VREG("ldo4", "ldo%s4", &pmic5_pldo_lv, "vdd-l4"), - RPMH_VREG("ldo5", "ldo%s5", &pmic5_nldo, "vdd-l5"), - RPMH_VREG("ldo6", "ldo%s6", &pmic5_nldo, "vdd-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic5_nldo, "vdd-l7-l8"), - RPMH_VREG("ldo8", "ldo%s8", &pmic5_nldo, "vdd-l7-l8"), - RPMH_VREG("ldo9", "ldo%s9", &pmic5_nldo, "vdd-l9"), - RPMH_VREG("ldo10", "ldo%s10", &pmic5_pldo_lv, "vdd-l10"), - RPMH_VREG("ldo11", "ldo%s11", &pmic5_nldo, "vdd-l11"), - RPMH_VREG("ldo12", "ldo%s12", &pmic5_nldo, "vdd-l12"), + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo, "vdd-l1-l2"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo, "vdd-l1-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo, "vdd-l3"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_pldo_lv, "vdd-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo, "vdd-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo, "vdd-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo, "vdd-l7-l8"), + RPMH_VREG("ldo8", LDO, 8, &pmic5_nldo, "vdd-l7-l8"), + RPMH_VREG("ldo9", LDO, 9, &pmic5_nldo, "vdd-l9"), + RPMH_VREG("ldo10", LDO, 10, &pmic5_pldo_lv, "vdd-l10"), + RPMH_VREG("ldo11", LDO, 11, &pmic5_nldo, "vdd-l11"), + RPMH_VREG("ldo12", LDO, 12, &pmic5_nldo, "vdd-l12"), + {} +}; + +static const struct rpmh_vreg_init_data pmr735d_vreg_data[] = { + RPMH_VREG("ldo1", LDO, 1, &pmic5_nldo515, "vdd-l1-l2-l5"), + RPMH_VREG("ldo2", LDO, 2, &pmic5_nldo515, "vdd-l1-l2-l5"), + RPMH_VREG("ldo3", LDO, 3, &pmic5_nldo515, "vdd-l3-l4"), + RPMH_VREG("ldo4", LDO, 4, &pmic5_nldo515, "vdd-l3-l4"), + RPMH_VREG("ldo5", LDO, 5, &pmic5_nldo515, "vdd-l1-l2-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic5_nldo515, "vdd-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic5_nldo515, "vdd-l7"), {} }; static const struct rpmh_vreg_init_data pm660_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic4_ftsmps426, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic4_ftsmps426, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic4_ftsmps426, "vdd-s3"), - RPMH_VREG("smps4", "smp%s4", &pmic4_hfsmps3, "vdd-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic4_hfsmps3, "vdd-s5"), - RPMH_VREG("smps6", "smp%s6", &pmic4_hfsmps3, "vdd-s6"), - RPMH_VREG("ldo1", "ldo%s1", &pmic4_nldo, "vdd-l1-l6-l7"), - RPMH_VREG("ldo2", "ldo%s2", &pmic4_nldo, "vdd-l2-l3"), - RPMH_VREG("ldo3", "ldo%s3", &pmic4_nldo, "vdd-l2-l3"), + RPMH_VREG("smps1", SMPS, 1, &pmic4_ftsmps426, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic4_ftsmps426, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic4_ftsmps426, "vdd-s3"), + RPMH_VREG("smps4", SMPS, 4, &pmic4_hfsmps3, "vdd-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic4_hfsmps3, "vdd-s5"), + RPMH_VREG("smps6", SMPS, 6, &pmic4_hfsmps3, "vdd-s6"), + RPMH_VREG("ldo1", LDO, 1, &pmic4_nldo, "vdd-l1-l6-l7"), + RPMH_VREG("ldo2", LDO, 2, &pmic4_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo3", LDO, 3, &pmic4_nldo, "vdd-l2-l3"), /* ldo4 is inaccessible on PM660 */ - RPMH_VREG("ldo5", "ldo%s5", &pmic4_nldo, "vdd-l5"), - RPMH_VREG("ldo6", "ldo%s6", &pmic4_nldo, "vdd-l1-l6-l7"), - RPMH_VREG("ldo7", "ldo%s7", &pmic4_nldo, "vdd-l1-l6-l7"), - RPMH_VREG("ldo8", "ldo%s8", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), - RPMH_VREG("ldo9", "ldo%s9", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), - RPMH_VREG("ldo10", "ldo%s10", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), - RPMH_VREG("ldo11", "ldo%s11", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), - RPMH_VREG("ldo12", "ldo%s12", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), - RPMH_VREG("ldo13", "ldo%s13", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), - RPMH_VREG("ldo14", "ldo%s14", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), - RPMH_VREG("ldo15", "ldo%s15", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), - RPMH_VREG("ldo16", "ldo%s16", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), - RPMH_VREG("ldo17", "ldo%s17", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), - RPMH_VREG("ldo18", "ldo%s18", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), - RPMH_VREG("ldo19", "ldo%s19", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo5", LDO, 5, &pmic4_nldo, "vdd-l5"), + RPMH_VREG("ldo6", LDO, 6, &pmic4_nldo, "vdd-l1-l6-l7"), + RPMH_VREG("ldo7", LDO, 7, &pmic4_nldo, "vdd-l1-l6-l7"), + RPMH_VREG("ldo8", LDO, 8, &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo9", LDO, 9, &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo10", LDO, 10, &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo11", LDO, 11, &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo12", LDO, 12, &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo13", LDO, 13, &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo14", LDO, 14, &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo15", LDO, 15, &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo16", LDO, 16, &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo17", LDO, 17, &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo18", LDO, 18, &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo19", LDO, 19, &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), {} }; static const struct rpmh_vreg_init_data pm660l_vreg_data[] = { - RPMH_VREG("smps1", "smp%s1", &pmic4_ftsmps426, "vdd-s1"), - RPMH_VREG("smps2", "smp%s2", &pmic4_ftsmps426, "vdd-s2"), - RPMH_VREG("smps3", "smp%s3", &pmic4_ftsmps426, "vdd-s3-s4"), - RPMH_VREG("smps5", "smp%s5", &pmic4_ftsmps426, "vdd-s5"), - RPMH_VREG("ldo1", "ldo%s1", &pmic4_nldo, "vdd-l1-l9-l10"), - RPMH_VREG("ldo2", "ldo%s2", &pmic4_pldo, "vdd-l2"), - RPMH_VREG("ldo3", "ldo%s3", &pmic4_pldo, "vdd-l3-l5-l7-l8"), - RPMH_VREG("ldo4", "ldo%s4", &pmic4_pldo, "vdd-l4-l6"), - RPMH_VREG("ldo5", "ldo%s5", &pmic4_pldo, "vdd-l3-l5-l7-l8"), - RPMH_VREG("ldo6", "ldo%s6", &pmic4_pldo, "vdd-l4-l6"), - RPMH_VREG("ldo7", "ldo%s7", &pmic4_pldo, "vdd-l3-l5-l7-l8"), - RPMH_VREG("ldo8", "ldo%s8", &pmic4_pldo, "vdd-l3-l5-l7-l8"), - RPMH_VREG("bob", "bob%s1", &pmic4_bob, "vdd-bob"), + RPMH_VREG("smps1", SMPS, 1, &pmic4_ftsmps426, "vdd-s1"), + RPMH_VREG("smps2", SMPS, 2, &pmic4_ftsmps426, "vdd-s2"), + RPMH_VREG("smps3", SMPS, 3, &pmic4_ftsmps426, "vdd-s3-s4"), + RPMH_VREG("smps5", SMPS, 5, &pmic4_ftsmps426, "vdd-s5"), + RPMH_VREG("ldo1", LDO, 1, &pmic4_nldo, "vdd-l1-l9-l10"), + RPMH_VREG("ldo2", LDO, 2, &pmic4_pldo, "vdd-l2"), + RPMH_VREG("ldo3", LDO, 3, &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("ldo4", LDO, 4, &pmic4_pldo, "vdd-l4-l6"), + RPMH_VREG("ldo5", LDO, 5, &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("ldo6", LDO, 6, &pmic4_pldo, "vdd-l4-l6"), + RPMH_VREG("ldo7", LDO, 7, &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("ldo8", LDO, 8, &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("bob", BOB, 1, &pmic4_bob, "vdd-bob"), {} }; @@ -1689,6 +1909,22 @@ static const struct of_device_id __maybe_unused rpmh_regulator_match_table[] = { .compatible = "qcom,pmc8380-rpmh-regulators", .data = pmc8380_vreg_data, }, + { + .compatible = "qcom,pmcx0102-rpmh-regulators", + .data = pmcx0102_vreg_data, + }, + { + .compatible = "qcom,pmh0101-rpmh-regulators", + .data = pmh0101_vreg_data, + }, + { + .compatible = "qcom,pmh0104-rpmh-regulators", + .data = pmh0104_vreg_data, + }, + { + .compatible = "qcom,pmh0110-rpmh-regulators", + .data = pmh0110_vreg_data, + }, { .compatible = "qcom,pmm8155au-rpmh-regulators", .data = pmm8155au_vreg_data, @@ -1725,6 +1961,10 @@ static const struct of_device_id __maybe_unused rpmh_regulator_match_table[] = { .compatible = "qcom,pmr735b-rpmh-regulators", .data = pmr735b_vreg_data, }, + { + .compatible = "qcom,pmr735d-rpmh-regulators", + .data = pmr735d_vreg_data, + }, { .compatible = "qcom,pm660-rpmh-regulators", .data = pm660_vreg_data, diff --git a/drivers/regulator/spacemit-p1.c b/drivers/regulator/spacemit-p1.c index d437e6738ea1e..2b585ba01a93d 100644 --- a/drivers/regulator/spacemit-p1.c +++ b/drivers/regulator/spacemit-p1.c @@ -87,13 +87,13 @@ static const struct linear_range p1_ldo_ranges[] = { } #define P1_BUCK_DESC(_n) \ - P1_REG_DESC(BUCK, buck, _n, "vcc", 0x47, BUCK_MASK, 254, p1_buck_ranges) + P1_REG_DESC(BUCK, buck, _n, "vin", 0x47, BUCK_MASK, 255, p1_buck_ranges) #define P1_ALDO_DESC(_n) \ - P1_REG_DESC(ALDO, aldo, _n, "vcc", 0x5b, LDO_MASK, 117, p1_ldo_ranges) + P1_REG_DESC(ALDO, aldo, _n, "vin", 0x5b, LDO_MASK, 128, p1_ldo_ranges) #define P1_DLDO_DESC(_n) \ - P1_REG_DESC(DLDO, dldo, _n, "buck5", 0x67, LDO_MASK, 117, p1_ldo_ranges) + P1_REG_DESC(DLDO, dldo, _n, "buck5", 0x67, LDO_MASK, 128, p1_ldo_ranges) static const struct regulator_desc p1_regulator_desc[] = { P1_BUCK_DESC(1), diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index 6e78a01755c7b..c0c2bda4964a7 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -1026,9 +1026,11 @@ static int imx_dsp_rproc_load(struct rproc *rproc, const struct firmware *fw) * Clear buffers after pm rumtime for internal ocram is not * accessible if power and clock are not enabled. */ - list_for_each_entry(carveout, &rproc->carveouts, node) { - if (carveout->va) - memset(carveout->va, 0, carveout->len); + if (rproc->state == RPROC_OFFLINE) { + list_for_each_entry(carveout, &rproc->carveouts, node) { + if (carveout->va) + memset(carveout->va, 0, carveout->len); + } } ret = imx_dsp_rproc_elf_load_segments(rproc, fw); @@ -1301,6 +1303,15 @@ static int imx_dsp_suspend(struct device *dev) if (rproc->state != RPROC_RUNNING) goto out; + /* + * No channel available for sending messages; + * indicates no mailboxes present, so trigger PM runtime suspend + */ + if (!priv->tx_ch) { + dev_dbg(dev, "No initialized mbox tx channel, suspend directly.\n"); + goto out; + } + reinit_completion(&priv->pm_comp); /* Tell DSP that suspend is happening */ diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index bb25221a4a898..7ef99eac37f1a 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -608,6 +608,10 @@ imx_rproc_elf_find_loaded_rsc_table(struct rproc *rproc, const struct firmware * { struct imx_rproc *priv = rproc->priv; + /* No resource table in the firmware */ + if (!rproc->table_ptr) + return NULL; + if (priv->rsc_table) return (struct resource_table *)priv->rsc_table; @@ -1136,11 +1140,16 @@ static int imx_rproc_probe(struct platform_device *pdev) ret = rproc_add(rproc); if (ret) { dev_err(dev, "rproc_add failed\n"); - goto err_put_clk; + goto err_put_pm; } return 0; +err_put_pm: + if (dcfg->method == IMX_RPROC_SCU_API) { + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); + } err_put_clk: clk_disable_unprepare(priv->clk); err_put_scu: @@ -1160,7 +1169,7 @@ static void imx_rproc_remove(struct platform_device *pdev) if (priv->dcfg->method == IMX_RPROC_SCU_API) { pm_runtime_disable(priv->dev); - pm_runtime_put(priv->dev); + pm_runtime_put_noidle(priv->dev); } clk_disable_unprepare(priv->clk); rproc_del(rproc); diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index 8206a17664818..eb8908ea3bab0 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -282,7 +282,7 @@ static irqreturn_t scp_irq_handler(int irq, void *priv) struct mtk_scp *scp = priv; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clocks\n"); return IRQ_NONE; @@ -290,7 +290,7 @@ static irqreturn_t scp_irq_handler(int irq, void *priv) scp->data->scp_irq_handler(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return IRQ_HANDLED; } @@ -664,7 +664,7 @@ static int scp_load(struct rproc *rproc, const struct firmware *fw) struct device *dev = scp->dev; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; @@ -679,7 +679,7 @@ static int scp_load(struct rproc *rproc, const struct firmware *fw) ret = scp_elf_load_segments(rproc, fw); leave: - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -690,14 +690,14 @@ static int scp_parse_fw(struct rproc *rproc, const struct firmware *fw) struct device *dev = scp->dev; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; } ret = scp_ipi_init(scp, fw); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -708,7 +708,7 @@ static int scp_start(struct rproc *rproc) struct scp_run *run = &scp->run; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; @@ -733,14 +733,14 @@ static int scp_start(struct rproc *rproc) goto stop; } - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); dev_info(dev, "SCP is ready. FW version %s\n", run->fw_ver); return 0; stop: scp->data->scp_reset_assert(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -908,7 +908,7 @@ static int scp_stop(struct rproc *rproc) struct mtk_scp *scp = rproc->priv; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clocks\n"); return ret; @@ -916,12 +916,29 @@ static int scp_stop(struct rproc *rproc) scp->data->scp_reset_assert(scp); scp->data->scp_stop(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return 0; } +static int scp_prepare(struct rproc *rproc) +{ + struct mtk_scp *scp = rproc->priv; + + return clk_prepare(scp->clk); +} + +static int scp_unprepare(struct rproc *rproc) +{ + struct mtk_scp *scp = rproc->priv; + + clk_unprepare(scp->clk); + return 0; +} + static const struct rproc_ops scp_ops = { + .prepare = scp_prepare, + .unprepare = scp_unprepare, .start = scp_start, .stop = scp_stop, .load = scp_load, @@ -1527,12 +1544,51 @@ static const struct of_device_id mtk_scp_of_match[] = { }; MODULE_DEVICE_TABLE(of, mtk_scp_of_match); +static int __maybe_unused scp_suspend(struct device *dev) +{ + struct mtk_scp *scp = dev_get_drvdata(dev); + struct rproc *rproc = scp->rproc; + + /* + * Only unprepare if the SCP is running and holding the clock. + * + * Note: `scp_ops` doesn't implement .attach() callback, hence + * `rproc->state` can never be RPROC_ATTACHED. Otherwise, it + * should also be checked here. + */ + if (rproc->state == RPROC_RUNNING) + clk_unprepare(scp->clk); + return 0; +} + +static int __maybe_unused scp_resume(struct device *dev) +{ + struct mtk_scp *scp = dev_get_drvdata(dev); + struct rproc *rproc = scp->rproc; + + /* + * Only prepare if the SCP was running and holding the clock. + * + * Note: `scp_ops` doesn't implement .attach() callback, hence + * `rproc->state` can never be RPROC_ATTACHED. Otherwise, it + * should also be checked here. + */ + if (rproc->state == RPROC_RUNNING) + return clk_prepare(scp->clk); + return 0; +} + +static const struct dev_pm_ops scp_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(scp_suspend, scp_resume) +}; + static struct platform_driver mtk_scp_driver = { .probe = scp_probe, .remove = scp_remove, .driver = { .name = "mtk-scp", .of_match_table = mtk_scp_of_match, + .pm = &scp_pm_ops, }, }; diff --git a/drivers/remoteproc/mtk_scp_ipi.c b/drivers/remoteproc/mtk_scp_ipi.c index c068227e251e7..7a37e273b3af8 100644 --- a/drivers/remoteproc/mtk_scp_ipi.c +++ b/drivers/remoteproc/mtk_scp_ipi.c @@ -171,7 +171,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, WARN_ON(len > scp_sizes->ipi_share_buffer_size) || WARN_ON(!buf)) return -EINVAL; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clock\n"); return ret; @@ -211,7 +211,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, unlock_mutex: mutex_unlock(&scp->send_lock); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } diff --git a/drivers/remoteproc/qcom_q6v5.c b/drivers/remoteproc/qcom_q6v5.c index 58d5b85e58cda..a02839c7ed8c2 100644 --- a/drivers/remoteproc/qcom_q6v5.c +++ b/drivers/remoteproc/qcom_q6v5.c @@ -6,6 +6,7 @@ * Copyright (C) 2014 Sony Mobile Communications AB * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. */ +#include #include #include #include @@ -351,6 +352,8 @@ int qcom_q6v5_init(struct qcom_q6v5 *q6v5, struct platform_device *pdev, return dev_err_probe(&pdev->dev, PTR_ERR(q6v5->path), "failed to acquire interconnect path\n"); + of_platform_populate(q6v5->dev->of_node, NULL, NULL, q6v5->dev); + return 0; } EXPORT_SYMBOL_GPL(qcom_q6v5_init); @@ -361,6 +364,7 @@ EXPORT_SYMBOL_GPL(qcom_q6v5_init); */ void qcom_q6v5_deinit(struct qcom_q6v5 *q6v5) { + of_platform_depopulate(q6v5->dev); qmp_put(q6v5->qmp); } EXPORT_SYMBOL_GPL(qcom_q6v5_deinit); diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 3087d895b87f4..e4f25be4f6438 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -238,7 +238,6 @@ struct q6v5 { struct qcom_rproc_pdm pdm_subdev; struct qcom_rproc_ssr ssr_subdev; struct qcom_sysmon *sysmon; - struct platform_device *bam_dmux; bool need_mem_protection; bool has_alt_reset; bool has_mba_logs; @@ -2051,7 +2050,6 @@ static int q6v5_alloc_memory_region(struct q6v5 *qproc) static int q6v5_probe(struct platform_device *pdev) { const struct rproc_hexagon_res *desc; - struct device_node *node; struct q6v5 *qproc; struct rproc *rproc; const char *mba_image; @@ -2185,10 +2183,6 @@ static int q6v5_probe(struct platform_device *pdev) if (ret) goto remove_sysmon_subdev; - node = of_get_compatible_child(pdev->dev.of_node, "qcom,bam-dmux"); - qproc->bam_dmux = of_platform_device_create(node, NULL, &pdev->dev); - of_node_put(node); - return 0; remove_sysmon_subdev: @@ -2208,8 +2202,6 @@ static void q6v5_remove(struct platform_device *pdev) struct q6v5 *qproc = platform_get_drvdata(pdev); struct rproc *rproc = qproc->rproc; - if (qproc->bam_dmux) - of_platform_device_destroy(&qproc->bam_dmux->dev, NULL); rproc_del(rproc); qcom_q6v5_deinit(&qproc->q6v5); diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index 158bcd6cc85c5..0cf8cbbbbb3f5 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -99,8 +100,9 @@ struct qcom_pas { phys_addr_t mem_reloc; phys_addr_t dtb_mem_reloc; phys_addr_t region_assign_phys[MAX_ASSIGN_COUNT]; + void *mem_region; - void *dtb_mem_region; + size_t mem_size; size_t dtb_mem_size; size_t region_assign_size[MAX_ASSIGN_COUNT]; @@ -117,8 +119,8 @@ struct qcom_pas { struct qcom_rproc_ssr ssr_subdev; struct qcom_sysmon *sysmon; - struct qcom_scm_pas_metadata pas_metadata; - struct qcom_scm_pas_metadata dtb_pas_metadata; + struct qcom_scm_pas_context *pas_ctx; + struct qcom_scm_pas_context *dtb_pas_ctx; }; static void qcom_pas_segment_dump(struct rproc *rproc, @@ -147,7 +149,16 @@ static void qcom_pas_minidump(struct rproc *rproc) if (rproc->dump_conf == RPROC_COREDUMP_DISABLED) return; + pas->mem_region = ioremap_wc(pas->mem_phys, pas->mem_size); + if (!pas->mem_region) { + dev_err(pas->dev, "unable to map memory region: %pa+%zx\n", + &pas->mem_phys, pas->mem_size); + return; + } + qcom_minidump(rproc, pas->minidump_id, qcom_pas_segment_dump); + iounmap(pas->mem_region); + pas->mem_region = NULL; } static int qcom_pas_pds_enable(struct qcom_pas *pas, struct device **pds, @@ -183,6 +194,18 @@ static void qcom_pas_pds_disable(struct qcom_pas *pas, struct device **pds, int i; for (i = 0; i < pd_count; i++) { + /* + * There is a race condition which occurs sometimes for RB8 platform when APPS + * removes it's vote on handover INT from fw - ADSP F/W side vote is not yet + * applied on the lcx and lmx rails because of which PMIC shutdowns shut and device + * goes into hung state. Carry this WA until a proper fix is finalized. + */ + if (of_device_is_compatible(dev_of_node(pas->dev), "qcom,sa8775p-adsp-pas")) { + /* Apply SVS_L1 vote to keep lcx and lmx rails ON */ + dev_pm_genpd_set_performance_state(pds[i], 192); + return; + } + dev_pm_genpd_set_performance_state(pds[i], 0); pm_runtime_put(pds[i]); } @@ -211,9 +234,9 @@ static int qcom_pas_unprepare(struct rproc *rproc) * auth_and_reset() was successful, but in other cases clean it up * here. */ - qcom_scm_pas_metadata_release(&pas->pas_metadata); + qcom_scm_pas_metadata_release(pas->pas_ctx); if (pas->dtb_pas_id) - qcom_scm_pas_metadata_release(&pas->dtb_pas_metadata); + qcom_scm_pas_metadata_release(pas->dtb_pas_ctx); return 0; } @@ -239,15 +262,9 @@ static int qcom_pas_load(struct rproc *rproc, const struct firmware *fw) return ret; } - ret = qcom_mdt_pas_init(pas->dev, pas->dtb_firmware, pas->dtb_firmware_name, - pas->dtb_pas_id, pas->dtb_mem_phys, - &pas->dtb_pas_metadata); - if (ret) - goto release_dtb_firmware; - - ret = qcom_mdt_load_no_init(pas->dev, pas->dtb_firmware, pas->dtb_firmware_name, - pas->dtb_mem_region, pas->dtb_mem_phys, - pas->dtb_mem_size, &pas->dtb_mem_reloc); + ret = qcom_mdt_pas_load(pas->dtb_pas_ctx, pas->dtb_firmware, + pas->dtb_firmware_name, + &pas->dtb_mem_reloc); if (ret) goto release_dtb_metadata; } @@ -255,14 +272,30 @@ static int qcom_pas_load(struct rproc *rproc, const struct firmware *fw) return 0; release_dtb_metadata: - qcom_scm_pas_metadata_release(&pas->dtb_pas_metadata); + if (pas->dtb_pas_id) + qcom_scm_pas_metadata_release(pas->dtb_pas_ctx); -release_dtb_firmware: release_firmware(pas->dtb_firmware); return ret; } +static void qcom_pas_unmap_carveout(struct rproc *rproc, phys_addr_t mem_phys, size_t size) +{ + if (rproc->has_iommu) + iommu_unmap(rproc->domain, mem_phys, size); +} + +static int qcom_pas_map_carveout(struct rproc *rproc, phys_addr_t mem_phys, size_t size) +{ + int ret = 0; + + if (rproc->has_iommu) + ret = iommu_map(rproc->domain, mem_phys, mem_phys, size, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + return ret; +} + static int qcom_pas_start(struct rproc *rproc) { struct qcom_pas *pas = rproc->priv; @@ -297,54 +330,62 @@ static int qcom_pas_start(struct rproc *rproc) } if (pas->dtb_pas_id) { - ret = qcom_scm_pas_auth_and_reset(pas->dtb_pas_id); + ret = qcom_pas_map_carveout(rproc, pas->dtb_mem_phys, pas->dtb_mem_size); + if (ret) + goto disable_px_supply; + + ret = qcom_scm_pas_prepare_and_auth_reset(pas->dtb_pas_ctx); if (ret) { dev_err(pas->dev, "failed to authenticate dtb image and release reset\n"); - goto disable_px_supply; + goto unmap_dtb_carveout; } } - ret = qcom_mdt_pas_init(pas->dev, pas->firmware, rproc->firmware, pas->pas_id, - pas->mem_phys, &pas->pas_metadata); - if (ret) - goto disable_px_supply; - - ret = qcom_mdt_load_no_init(pas->dev, pas->firmware, rproc->firmware, - pas->mem_region, pas->mem_phys, pas->mem_size, - &pas->mem_reloc); + ret = qcom_mdt_pas_load(pas->pas_ctx, pas->firmware, rproc->firmware, + &pas->mem_reloc); if (ret) goto release_pas_metadata; qcom_pil_info_store(pas->info_name, pas->mem_phys, pas->mem_size); - ret = qcom_scm_pas_auth_and_reset(pas->pas_id); + ret = qcom_pas_map_carveout(rproc, pas->mem_phys, pas->mem_size); + if (ret) + goto release_pas_metadata; + + ret = qcom_scm_pas_prepare_and_auth_reset(pas->pas_ctx); if (ret) { dev_err(pas->dev, "failed to authenticate image and release reset\n"); - goto release_pas_metadata; + goto unmap_carveout; } ret = qcom_q6v5_wait_for_start(&pas->q6v5, msecs_to_jiffies(5000)); if (ret == -ETIMEDOUT) { dev_err(pas->dev, "start timed out\n"); qcom_scm_pas_shutdown(pas->pas_id); - goto release_pas_metadata; + goto unmap_carveout; } - qcom_scm_pas_metadata_release(&pas->pas_metadata); + qcom_scm_pas_metadata_release(pas->pas_ctx); if (pas->dtb_pas_id) - qcom_scm_pas_metadata_release(&pas->dtb_pas_metadata); + qcom_scm_pas_metadata_release(pas->dtb_pas_ctx); /* firmware is used to pass reference from qcom_pas_start(), drop it now */ pas->firmware = NULL; return 0; +unmap_carveout: + qcom_pas_unmap_carveout(rproc, pas->mem_phys, pas->mem_size); release_pas_metadata: - qcom_scm_pas_metadata_release(&pas->pas_metadata); + qcom_scm_pas_metadata_release(pas->pas_ctx); + if (pas->dtb_pas_id) + qcom_scm_pas_metadata_release(pas->dtb_pas_ctx); + +unmap_dtb_carveout: if (pas->dtb_pas_id) - qcom_scm_pas_metadata_release(&pas->dtb_pas_metadata); + qcom_pas_unmap_carveout(rproc, pas->dtb_mem_phys, pas->dtb_mem_size); disable_px_supply: if (pas->px_supply) regulator_disable(pas->px_supply); @@ -400,8 +441,12 @@ static int qcom_pas_stop(struct rproc *rproc) ret = qcom_scm_pas_shutdown(pas->dtb_pas_id); if (ret) dev_err(pas->dev, "failed to shutdown dtb: %d\n", ret); + + qcom_pas_unmap_carveout(rproc, pas->dtb_mem_phys, pas->dtb_mem_size); } + qcom_pas_unmap_carveout(rproc, pas->mem_phys, pas->mem_size); + handover = qcom_q6v5_unprepare(&pas->q6v5); if (handover) qcom_pas_handover(&pas->q6v5); @@ -427,6 +472,61 @@ static void *qcom_pas_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *is return pas->mem_region + offset; } +static int qcom_pas_parse_firmware(struct rproc *rproc, const struct firmware *fw) +{ + struct qcom_pas *pas = rproc->priv; + struct resource_table *table = NULL; + size_t output_rt_size; + void *output_rt; + size_t table_sz; + int ret; + + ret = qcom_register_dump_segments(rproc, fw); + if (ret) { + dev_err(pas->dev, "Error in registering dump segments\n"); + return ret; + } + + if (!rproc->has_iommu) + return 0; + + ret = rproc_elf_load_rsc_table(rproc, fw); + if (ret) + dev_dbg(&rproc->dev, "Failed to load resource table from firmware\n"); + + table = rproc->table_ptr; + table_sz = rproc->table_sz; + + /* + * The resources consumed by Qualcomm remote processors fall into two categories: + * static (such as the memory carveouts for the rproc firmware) and dynamic (like + * shared memory pools). Both are managed by a Qualcomm hypervisor (such as QHEE + * or Gunyah), if one is present. Otherwise, a resource table must be retrieved + * via an SCM call. That table will list all dynamic resources (if any) and possibly + * the static ones. The static resources may also come from a resource table embedded + * in the rproc firmware instead. + * + * Here, we call rproc_elf_load_rsc_table() to check firmware binary has resources + * or not and if it is not having then we pass NULL and zero as input resource + * table pointer and size respectively to the argument of qcom_scm_pas_get_rsc_table() + * and this is even true for Qualcomm remote processor who does follow remoteproc + * framework. + */ + output_rt = qcom_scm_pas_get_rsc_table(pas->pas_ctx, table, table_sz, &output_rt_size); + ret = IS_ERR(output_rt) ? PTR_ERR(output_rt) : 0; + if (ret) { + dev_err(pas->dev, "Error in getting resource table: %d\n", ret); + return ret; + } + + kfree(rproc->cached_table); + rproc->cached_table = output_rt; + rproc->table_ptr = rproc->cached_table; + rproc->table_sz = output_rt_size; + + return ret; +} + static unsigned long qcom_pas_panic(struct rproc *rproc) { struct qcom_pas *pas = rproc->priv; @@ -434,14 +534,31 @@ static unsigned long qcom_pas_panic(struct rproc *rproc) return qcom_q6v5_panic(&pas->q6v5); } +static void qcom_pas_coredump(struct rproc *rproc) +{ + struct qcom_pas *pas = rproc->priv; + + pas->mem_region = ioremap_wc(pas->mem_phys, pas->mem_size); + if (!pas->mem_region) { + dev_err(pas->dev, "unable to map memory region: %pa+%zx\n", + &pas->mem_phys, pas->mem_size); + return; + } + + rproc_coredump(rproc); + iounmap(pas->mem_region); + pas->mem_region = NULL; +} + static const struct rproc_ops qcom_pas_ops = { .unprepare = qcom_pas_unprepare, .start = qcom_pas_start, .stop = qcom_pas_stop, .da_to_va = qcom_pas_da_to_va, - .parse_fw = qcom_register_dump_segments, + .parse_fw = qcom_pas_parse_firmware, .load = qcom_pas_load, .panic = qcom_pas_panic, + .coredump = qcom_pas_coredump, }; static const struct rproc_ops qcom_pas_minidump_ops = { @@ -449,7 +566,7 @@ static const struct rproc_ops qcom_pas_minidump_ops = { .start = qcom_pas_start, .stop = qcom_pas_stop, .da_to_va = qcom_pas_da_to_va, - .parse_fw = qcom_register_dump_segments, + .parse_fw = qcom_pas_parse_firmware, .load = qcom_pas_load, .panic = qcom_pas_panic, .coredump = qcom_pas_minidump, @@ -547,6 +664,7 @@ static void qcom_pas_pds_detach(struct qcom_pas *pas, struct device **pds, size_ static int qcom_pas_alloc_memory_region(struct qcom_pas *pas) { + struct rproc *rproc = pas->rproc; struct reserved_mem *rmem; struct device_node *node; @@ -565,13 +683,12 @@ static int qcom_pas_alloc_memory_region(struct qcom_pas *pas) pas->mem_phys = pas->mem_reloc = rmem->base; pas->mem_size = rmem->size; - pas->mem_region = devm_ioremap_wc(pas->dev, pas->mem_phys, pas->mem_size); - if (!pas->mem_region) { - dev_err(pas->dev, "unable to map memory region: %pa+%zx\n", - &rmem->base, pas->mem_size); - return -EBUSY; - } + pas->pas_ctx = devm_qcom_scm_pas_context_alloc(pas->dev, pas->pas_id, + pas->mem_phys, pas->mem_size); + if (IS_ERR(pas->pas_ctx)) + return PTR_ERR(pas->pas_ctx); + pas->pas_ctx->use_tzmem = rproc->has_iommu; if (!pas->dtb_pas_id) return 0; @@ -590,12 +707,13 @@ static int qcom_pas_alloc_memory_region(struct qcom_pas *pas) pas->dtb_mem_phys = pas->dtb_mem_reloc = rmem->base; pas->dtb_mem_size = rmem->size; - pas->dtb_mem_region = devm_ioremap_wc(pas->dev, pas->dtb_mem_phys, pas->dtb_mem_size); - if (!pas->dtb_mem_region) { - dev_err(pas->dev, "unable to map dtb memory region: %pa+%zx\n", - &rmem->base, pas->dtb_mem_size); - return -EBUSY; - } + pas->dtb_pas_ctx = devm_qcom_scm_pas_context_alloc(pas->dev, pas->dtb_pas_id, + pas->dtb_mem_phys, + pas->dtb_mem_size); + if (IS_ERR(pas->dtb_pas_ctx)) + return PTR_ERR(pas->dtb_pas_ctx); + + pas->dtb_pas_ctx->use_tzmem = rproc->has_iommu; return 0; } @@ -716,6 +834,7 @@ static int qcom_pas_probe(struct platform_device *pdev) return -ENOMEM; } + rproc->has_iommu = of_property_present(pdev->dev.of_node, "iommus"); rproc->auto_boot = desc->auto_boot; rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE); @@ -779,6 +898,7 @@ static int qcom_pas_probe(struct platform_device *pdev) } qcom_add_ssr_subdev(rproc, &pas->ssr_subdev, desc->ssr_name); + ret = rproc_add(rproc); if (ret) goto remove_ssr_sysmon; diff --git a/drivers/remoteproc/qcom_q6v5_wcss.c b/drivers/remoteproc/qcom_q6v5_wcss.c index 07c88623f5978..23ec87827d4f8 100644 --- a/drivers/remoteproc/qcom_q6v5_wcss.c +++ b/drivers/remoteproc/qcom_q6v5_wcss.c @@ -85,7 +85,7 @@ #define TCSR_WCSS_CLK_MASK 0x1F #define TCSR_WCSS_CLK_ENABLE 0x14 -#define MAX_HALT_REG 3 +#define MAX_HALT_REG 4 enum { WCSS_IPQ8074, WCSS_QCS404, @@ -864,9 +864,9 @@ static int q6v5_wcss_init_mmio(struct q6v5_wcss *wcss, return -EINVAL; } - wcss->halt_q6 = halt_reg[0]; - wcss->halt_wcss = halt_reg[1]; - wcss->halt_nc = halt_reg[2]; + wcss->halt_q6 = halt_reg[1]; + wcss->halt_wcss = halt_reg[2]; + wcss->halt_nc = halt_reg[3]; return 0; } diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c index 660ac6fc40821..c6cc6e519fe56 100644 --- a/drivers/remoteproc/qcom_sysmon.c +++ b/drivers/remoteproc/qcom_sysmon.c @@ -203,7 +203,7 @@ static const struct qmi_elem_info ssctl_shutdown_resp_ei[] = { }; struct ssctl_subsys_event_req { - u8 subsys_name_len; + u32 subsys_name_len; char subsys_name[SSCTL_SUBSYS_NAME_LENGTH]; u32 event; u8 evt_driven_valid; diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 78b7078478d46..b3b9e0f9d8c48 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -150,7 +150,7 @@ config RESET_K210 config RESET_K230 tristate "Reset controller driver for Canaan Kendryte K230 SoC" depends on ARCH_CANAAN || COMPILE_TEST - depends on OF + default ARCH_CANAAN help Support for the Canaan Kendryte K230 RISC-V SoC reset controller. Say Y if you want to control reset signals provided by this diff --git a/drivers/reset/reset-gpio.c b/drivers/reset/reset-gpio.c index 2290b25b67035..15353ba5758c3 100644 --- a/drivers/reset/reset-gpio.c +++ b/drivers/reset/reset-gpio.c @@ -110,6 +110,7 @@ static struct platform_driver reset_gpio_driver = { .id_table = reset_gpio_ids, .driver = { .name = "reset-gpio", + .suppress_bind_attrs = true, }, }; module_platform_driver(reset_gpio_driver); diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index 820a6ca5b1d7a..3a15d9d108089 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -1399,6 +1399,7 @@ static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) { struct glink_channel *channel = to_glink_channel(ept); struct qcom_glink *glink = channel->glink; + struct rpmsg_channel_info chinfo; unsigned long flags; spin_lock_irqsave(&channel->recv_lock, flags); @@ -1406,6 +1407,13 @@ static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) spin_unlock_irqrestore(&channel->recv_lock, flags); /* Decouple the potential rpdev from the channel */ + if (channel->rpdev) { + strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); + chinfo.src = RPMSG_ADDR_ANY; + chinfo.dst = RPMSG_ADDR_ANY; + + rpmsg_unregister_device(glink->dev, &chinfo); + } channel->rpdev = NULL; qcom_glink_send_close_req(glink, channel); diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index 5d661681a9b6c..96964745065b1 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -352,50 +352,38 @@ field##_show(struct device *dev, \ } \ static DEVICE_ATTR_RO(field); -#define rpmsg_string_attr(field, member) \ -static ssize_t \ -field##_store(struct device *dev, struct device_attribute *attr, \ - const char *buf, size_t sz) \ -{ \ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ - const char *old; \ - char *new; \ - \ - new = kstrndup(buf, sz, GFP_KERNEL); \ - if (!new) \ - return -ENOMEM; \ - new[strcspn(new, "\n")] = '\0'; \ - \ - device_lock(dev); \ - old = rpdev->member; \ - if (strlen(new)) { \ - rpdev->member = new; \ - } else { \ - kfree(new); \ - rpdev->member = NULL; \ - } \ - device_unlock(dev); \ - \ - kfree(old); \ - \ - return sz; \ -} \ -static ssize_t \ -field##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ - \ - return sprintf(buf, "%s\n", rpdev->member); \ -} \ -static DEVICE_ATTR_RW(field) - /* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */ rpmsg_show_attr(name, id.name, "%s\n"); rpmsg_show_attr(src, src, "0x%x\n"); rpmsg_show_attr(dst, dst, "0x%x\n"); rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n"); -rpmsg_string_attr(driver_override, driver_override); + +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + int ret; + + ret = driver_set_override(dev, &rpdev->driver_override, buf, count); + if (ret) + return ret; + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + ssize_t len; + + device_lock(dev); + len = sysfs_emit(buf, "%s\n", rpdev->driver_override); + device_unlock(dev); + return len; +} +static DEVICE_ATTR_RW(driver_override); static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c index b8b298efd9a9c..1906f4884a834 100644 --- a/drivers/rtc/interface.c +++ b/drivers/rtc/interface.c @@ -457,7 +457,7 @@ static int __rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) * are in, we can return -ETIME to signal that the timer has already * expired, which is true in both cases. */ - if ((scheduled - now) <= 1) { + if (!err && (scheduled - now) <= 1) { err = __rtc_read_time(rtc, &tm); if (err) return err; diff --git a/drivers/rtc/rtc-amlogic-a4.c b/drivers/rtc/rtc-amlogic-a4.c index 1928b29c10454..d766055d95848 100644 --- a/drivers/rtc/rtc-amlogic-a4.c +++ b/drivers/rtc/rtc-amlogic-a4.c @@ -371,7 +371,7 @@ static int aml_rtc_probe(struct platform_device *pdev) } ret = devm_request_irq(dev, rtc->irq, aml_rtc_handler, - IRQF_ONESHOT, "aml-rtc alarm", rtc); + 0, "aml-rtc alarm", rtc); if (ret) { dev_err_probe(dev, ret, "IRQ%d request failed, ret = %d\n", rtc->irq, ret); @@ -390,7 +390,6 @@ static int aml_rtc_probe(struct platform_device *pdev) return 0; err_clk: - clk_disable_unprepare(rtc->sys_clk); device_init_wakeup(dev, false); return ret; @@ -423,9 +422,6 @@ static SIMPLE_DEV_PM_OPS(aml_rtc_pm_ops, static void aml_rtc_remove(struct platform_device *pdev) { - struct aml_rtc_data *rtc = dev_get_drvdata(&pdev->dev); - - clk_disable_unprepare(rtc->sys_clk); device_init_wakeup(&pdev->dev, false); } diff --git a/drivers/rtc/rtc-gamecube.c b/drivers/rtc/rtc-gamecube.c index c828bc8e05b9c..045d5d45ab4b0 100644 --- a/drivers/rtc/rtc-gamecube.c +++ b/drivers/rtc/rtc-gamecube.c @@ -242,6 +242,10 @@ static int gamecube_rtc_read_offset_from_sram(struct priv *d) } hw_srnprot = ioremap(res.start, resource_size(&res)); + if (!hw_srnprot) { + pr_err("failed to ioremap hw_srnprot\n"); + return -ENOMEM; + } old = ioread32be(hw_srnprot); /* TODO: figure out why we use this magic constant. I obtained it by diff --git a/drivers/rtc/rtc-max31335.c b/drivers/rtc/rtc-max31335.c index dfb5bad3a3691..952b455071d68 100644 --- a/drivers/rtc/rtc-max31335.c +++ b/drivers/rtc/rtc-max31335.c @@ -391,10 +391,8 @@ static int max31335_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) if (ret) return ret; - ret = regmap_update_bits(max31335->regmap, max31335->chip->int_status_reg, - MAX31335_STATUS1_A1F, 0); - - return 0; + return regmap_update_bits(max31335->regmap, max31335->chip->int_status_reg, + MAX31335_STATUS1_A1F, 0); } static int max31335_alarm_irq_enable(struct device *dev, unsigned int enabled) @@ -593,7 +591,7 @@ static struct nvmem_config max31335_nvmem_cfg = { .size = MAX31335_RAM_SIZE, }; -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) static int max31335_read_temp(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { @@ -674,7 +672,7 @@ static int max31335_clkout_register(struct device *dev) static int max31335_probe(struct i2c_client *client) { struct max31335_data *max31335; -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) struct device *hwmon; #endif const struct chip_desc *match; @@ -729,7 +727,7 @@ static int max31335_probe(struct i2c_client *client) return dev_err_probe(&client->dev, ret, "cannot register rtc nvmem\n"); -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) if (max31335->chip->temp_reg) { hwmon = devm_hwmon_device_register_with_info(&client->dev, client->name, max31335, &max31335_chip_info, NULL); diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index 4e61011fb7a96..b281e9489df1d 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -424,7 +424,7 @@ static const struct clk_ops pcf8563_clkout_ops = { static struct clk *pcf8563_clkout_register_clk(struct pcf8563 *pcf8563) { - struct device_node *node = pcf8563->rtc->dev.of_node; + struct device_node *node = pcf8563->rtc->dev.parent->of_node; struct clk_init_data init; struct clk *clk; int ret; diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c index 3baa2b481d9f2..856bc1678e7d3 100644 --- a/drivers/rtc/rtc-zynqmp.c +++ b/drivers/rtc/rtc-zynqmp.c @@ -345,7 +345,10 @@ static int xlnx_rtc_probe(struct platform_device *pdev) &xrtcdev->freq); if (ret) xrtcdev->freq = RTC_CALIB_DEF; + } else { + xrtcdev->freq--; } + ret = readl(xrtcdev->reg_base + RTC_CALIB_RD); if (!ret) writel(xrtcdev->freq, (xrtcdev->reg_base + RTC_CALIB_WR)); diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 88fa17aea2ec8..40bbbada67f96 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -6146,10 +6146,12 @@ static void copy_pair_set_active(struct dasd_copy_relation *copy, char *new_busi static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid, char *sec_busid) { + struct dasd_eckd_private *prim_priv, *sec_priv; struct dasd_device *primary, *secondary; struct dasd_copy_relation *copy; struct dasd_block *block; struct gendisk *gdp; + int rc; copy = device->copy; if (!copy) @@ -6165,6 +6167,9 @@ static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid if (!secondary) return DASD_COPYPAIRSWAP_SECONDARY; + prim_priv = primary->private; + sec_priv = secondary->private; + /* * usually the device should be quiesced for swap * for paranoia stop device and requeue requests again @@ -6184,6 +6189,25 @@ static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid /* swap blocklayer device link */ gdp = block->gdp; dasd_add_link_to_gendisk(gdp, secondary); + rc = device_move(disk_to_dev(gdp), &secondary->cdev->dev, DPM_ORDER_NONE); + if (rc) { + dev_err(&primary->cdev->dev, + "copy_pair_swap: moving blockdevice parent %s->%s failed (%d)\n", + dev_name(&primary->cdev->dev), + dev_name(&secondary->cdev->dev), rc); + } + + if (primary->stopped & DASD_STOPPED_QUIESCE) { + dasd_device_set_stop_bits(secondary, DASD_STOPPED_QUIESCE); + dasd_device_remove_stop_bits(primary, DASD_STOPPED_QUIESCE); + } + + /* + * The secondary device never got through format detection, but since it + * is a copy of the primary device, the format is exactly the same; + * therefore, the detected layout can simply be copied. + */ + sec_priv->uses_cdl = prim_priv->uses_cdl; /* re-enable device */ dasd_device_remove_stop_bits(primary, DASD_STOPPED_PPRC); diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index be78a57f9bfde..8a70596a55447 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -236,7 +236,7 @@ struct subchannel *css_alloc_subchannel(struct subchannel_id schid, return sch; err: - kfree(sch); + put_device(&sch->dev); return ERR_PTR(ret); } diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 65f1a127cc3f6..dfd5d0f61a70d 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -2484,15 +2484,15 @@ static int __init ap_module_init(void) { int rc; - rc = ap_debug_init(); - if (rc) - return rc; - if (!ap_instructions_available()) { pr_warn("The hardware system does not support AP instructions\n"); return -ENODEV; } + rc = ap_debug_init(); + if (rc) + return rc; + /* init ap_queue hashtable */ hash_init(ap_queues); diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c index ce953cbbd5640..64a7f645eaf49 100644 --- a/drivers/s390/crypto/ap_card.c +++ b/drivers/s390/crypto/ap_card.c @@ -44,7 +44,7 @@ static ssize_t depth_show(struct device *dev, struct device_attribute *attr, { struct ap_card *ac = to_ap_card(dev); - return sysfs_emit(buf, "%d\n", ac->hwinfo.qd); + return sysfs_emit(buf, "%d\n", ac->hwinfo.qd + 1); } static DEVICE_ATTR_RO(depth); diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 8977866fab1bc..42ffb22aa1e2e 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -269,7 +269,7 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) list_move_tail(&ap_msg->list, &aq->pendingq); aq->requestq_count--; aq->pendingq_count++; - if (aq->queue_count < aq->card->hwinfo.qd) { + if (aq->queue_count < aq->card->hwinfo.qd + 1) { aq->sm_state = AP_SM_STATE_WORKING; return AP_SM_WAIT_AGAIN; } diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index a96e256143037..fe61df34f6146 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -1640,11 +1640,13 @@ int cca_get_info(u16 cardnr, u16 domain, struct cca_info *ci, u32 xflags) memset(ci, 0, sizeof(*ci)); - /* get first info from zcrypt device driver about this apqn */ - rc = zcrypt_device_status_ext(cardnr, domain, &devstat); - if (rc) - return rc; - ci->hwtype = devstat.hwtype; + /* if specific domain given, fetch status and hw info for this apqn */ + if (domain != AUTOSEL_DOM) { + rc = zcrypt_device_status_ext(cardnr, domain, &devstat); + if (rc) + return rc; + ci->hwtype = devstat.hwtype; + } /* * Prep memory for rule array and var array use. diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c index 6ba7fbddd3f7c..8aa78f4153363 100644 --- a/drivers/s390/crypto/zcrypt_cex4.c +++ b/drivers/s390/crypto/zcrypt_cex4.c @@ -84,8 +84,7 @@ static ssize_t cca_serialnr_show(struct device *dev, memset(&ci, 0, sizeof(ci)); - if (ap_domain_index >= 0) - cca_get_info(ac->id, ap_domain_index, &ci, 0); + cca_get_info(ac->id, AUTOSEL_DOM, &ci, 0); return sysfs_emit(buf, "%s\n", ci.serial); } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 9cefbb30960fd..5b9dce59db8b9 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -954,6 +954,10 @@ static atomic_t zcrypt_step = ATOMIC_INIT(0); /* * The request distributor calls this function if it picked the CEXxC * device to handle a modexpo request. + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. * @zq: pointer to zcrypt_queue structure that identifies the * CEXxC device to the request distributor * @mex: pointer to the modexpo request buffer @@ -965,21 +969,17 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, struct ap_response_type *resp_type = &ap_msg->response; int rc; - ap_msg->msg = (void *)get_zeroed_page(GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; - ap_msg->bufsize = PAGE_SIZE; ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); rc = icamex_msg_to_type6mex_msgx(zq, ap_msg, mex); if (rc) - goto out_free; + goto out; resp_type->type = CEXXC_RESPONSE_TYPE_ICA; init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) - goto out_free; + goto out; rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; @@ -992,15 +992,17 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq, ap_cancel_message(zq->queue, ap_msg); } -out_free: - free_page((unsigned long)ap_msg->msg); - ap_msg->msg = NULL; +out: return rc; } /* * The request distributor calls this function if it picked the CEXxC * device to handle a modexpo_crt request. + * This function assumes that ap_msg has been initialized with + * ap_init_apmsg() and thus a valid buffer with the size of + * ap_msg->bufsize is available within ap_msg. Also the caller has + * to make sure ap_release_apmsg() is always called even on failure. * @zq: pointer to zcrypt_queue structure that identifies the * CEXxC device to the request distributor * @crt: pointer to the modexpoc_crt request buffer @@ -1012,21 +1014,17 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, struct ap_response_type *resp_type = &ap_msg->response; int rc; - ap_msg->msg = (void *)get_zeroed_page(GFP_KERNEL); - if (!ap_msg->msg) - return -ENOMEM; - ap_msg->bufsize = PAGE_SIZE; ap_msg->receive = zcrypt_msgtype6_receive; ap_msg->psmid = (((unsigned long)current->pid) << 32) + atomic_inc_return(&zcrypt_step); rc = icacrt_msg_to_type6crt_msgx(zq, ap_msg, crt); if (rc) - goto out_free; + goto out; resp_type->type = CEXXC_RESPONSE_TYPE_ICA; init_completion(&resp_type->work); rc = ap_queue_message(zq->queue, ap_msg); if (rc) - goto out_free; + goto out; rc = wait_for_completion_interruptible(&resp_type->work); if (rc == 0) { rc = ap_msg->rc; @@ -1039,9 +1037,7 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq, ap_cancel_message(zq->queue, ap_msg); } -out_free: - free_page((unsigned long)ap_msg->msg); - ap_msg->msg = NULL; +out: return rc; } diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c index a86d780d1ba40..026c3e617cb1c 100644 --- a/drivers/scsi/BusLogic.c +++ b/drivers/scsi/BusLogic.c @@ -920,7 +920,8 @@ static int __init blogic_init_fp_probeinfo(struct blogic_adapter *adapter) a particular probe order. */ -static void __init blogic_init_probeinfo_list(struct blogic_adapter *adapter) +static noinline_for_stack void __init +blogic_init_probeinfo_list(struct blogic_adapter *adapter) { /* If a PCI BIOS is present, interrogate it for MultiMaster and @@ -1690,7 +1691,8 @@ static bool __init blogic_rdconfig(struct blogic_adapter *adapter) blogic_reportconfig reports the configuration of Host Adapter. */ -static bool __init blogic_reportconfig(struct blogic_adapter *adapter) +static noinline_for_stack bool __init +blogic_reportconfig(struct blogic_adapter *adapter) { unsigned short alltgt_mask = (1 << adapter->maxdev) - 1; unsigned short sync_ok, fast_ok; diff --git a/drivers/scsi/aic94xx/aic94xx_init.c b/drivers/scsi/aic94xx/aic94xx_init.c index adf3d91456064..95f3620059f7d 100644 --- a/drivers/scsi/aic94xx/aic94xx_init.c +++ b/drivers/scsi/aic94xx/aic94xx_init.c @@ -882,6 +882,9 @@ static void asd_pci_remove(struct pci_dev *dev) asd_disable_ints(asd_ha); + /* Ensure all scheduled tasklets complete before freeing resources */ + tasklet_kill(&asd_ha->seq.dl_tasklet); + asd_remove_dev_attrs(asd_ha); /* XXX more here as needed */ diff --git a/drivers/scsi/be2iscsi/be_mgmt.c b/drivers/scsi/be2iscsi/be_mgmt.c index 4e899ec1477d4..b1cba986f0fbd 100644 --- a/drivers/scsi/be2iscsi/be_mgmt.c +++ b/drivers/scsi/be2iscsi/be_mgmt.c @@ -1025,6 +1025,7 @@ unsigned int beiscsi_boot_get_sinfo(struct beiscsi_hba *phba) &nonemb_cmd->dma, GFP_KERNEL); if (!nonemb_cmd->va) { + free_mcc_wrb(ctrl, tag); mutex_unlock(&ctrl->mbox_lock); return 0; } diff --git a/drivers/scsi/bfa/bfad.c b/drivers/scsi/bfa/bfad.c index ff9adfc0b332e..bdfd065166718 100644 --- a/drivers/scsi/bfa/bfad.c +++ b/drivers/scsi/bfa/bfad.c @@ -1528,7 +1528,6 @@ bfad_pci_slot_reset(struct pci_dev *pdev) goto out_disable_device; } - pci_save_state(pdev); pci_set_master(pdev); rc = dma_set_mask_and_coherent(&bfad->pcidev->dev, DMA_BIT_MASK(64)); diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c index 79c8dafdd49ed..db0c2174430ac 100644 --- a/drivers/scsi/csiostor/csio_init.c +++ b/drivers/scsi/csiostor/csio_init.c @@ -1093,7 +1093,6 @@ csio_pci_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); /* Bring HW s/m to ready state. * but don't resume IOs. diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c index 34bde6650fae0..356a7c577ec3e 100644 --- a/drivers/scsi/csiostor/csio_scsi.c +++ b/drivers/scsi/csiostor/csio_scsi.c @@ -2074,7 +2074,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) struct csio_scsi_level_data sld; if (!rn) - goto fail; + goto fail_ret; csio_dbg(hw, "Request to reset LUN:%llu (ssni:0x%x tgtid:%d)\n", cmnd->device->lun, rn->flowid, rn->scsi_id); @@ -2220,6 +2220,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) csio_put_scsi_ioreq_lock(hw, scsim, ioreq); fail: CSIO_INC_STATS(rn, n_lun_rst_fail); +fail_ret: return FAILED; } diff --git a/drivers/scsi/elx/efct/efct_driver.c b/drivers/scsi/elx/efct/efct_driver.c index 1bd42f7db1773..528399f725d42 100644 --- a/drivers/scsi/elx/efct/efct_driver.c +++ b/drivers/scsi/elx/efct/efct_driver.c @@ -415,12 +415,6 @@ efct_intr_thread(int irq, void *handle) return IRQ_HANDLED; } -static irqreturn_t -efct_intr_msix(int irq, void *handle) -{ - return IRQ_WAKE_THREAD; -} - static int efct_setup_msix(struct efct *efct, u32 num_intrs) { @@ -450,7 +444,7 @@ efct_setup_msix(struct efct *efct, u32 num_intrs) intr_ctx->index = i; rc = request_threaded_irq(pci_irq_vector(efct->pci, i), - efct_intr_msix, efct_intr_thread, 0, + NULL, efct_intr_thread, IRQF_ONESHOT, EFCT_DRIVER_NAME, intr_ctx); if (rc) { dev_err(&efct->pci->dev, diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index 30a9c66126513..c2b082f1252c3 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -2578,7 +2578,7 @@ int hisi_sas_probe(struct platform_device *pdev, shost->transportt = hisi_sas_stt; shost->max_id = HISI_SAS_MAX_DEVICES; shost->max_lun = ~0; - shost->max_channel = 1; + shost->max_channel = 0; shost->max_cmd_len = HISI_SAS_MAX_CDB_LEN; if (hisi_hba->hw->slot_index_alloc) { shost->can_queue = HISI_SAS_MAX_COMMANDS; diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index 2f9e01717ef38..f69efc6494b8e 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -4993,7 +4993,7 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id) shost->transportt = hisi_sas_stt; shost->max_id = HISI_SAS_MAX_DEVICES; shost->max_lun = ~0; - shost->max_channel = 1; + shost->max_channel = 0; shost->max_cmd_len = HISI_SAS_MAX_CDB_LEN; shost->can_queue = HISI_SAS_UNRESERVED_IPTT; shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT; diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c index 228daffb286de..8d7c304636d99 100644 --- a/drivers/scsi/ibmvscsi/ibmvfc.c +++ b/drivers/scsi/ibmvscsi/ibmvfc.c @@ -4965,7 +4965,8 @@ static void ibmvfc_discover_targets_done(struct ibmvfc_event *evt) switch (mad_status) { case IBMVFC_MAD_SUCCESS: ibmvfc_dbg(vhost, "Discover Targets succeeded\n"); - vhost->num_targets = be32_to_cpu(rsp->num_written); + vhost->num_targets = min_t(u32, be32_to_cpu(rsp->num_written), + max_targets); ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_ALLOC_TGTS); break; case IBMVFC_MAD_FAILED: diff --git a/drivers/scsi/imm.c b/drivers/scsi/imm.c index 5c602c0577989..45b0e33293a59 100644 --- a/drivers/scsi/imm.c +++ b/drivers/scsi/imm.c @@ -1260,6 +1260,7 @@ static void imm_detach(struct parport *pb) imm_struct *dev; list_for_each_entry(dev, &imm_hosts, list) { if (dev->dev->port == pb) { + disable_delayed_work_sync(&dev->imm_tq); list_del_init(&dev->list); scsi_remove_host(dev->host); scsi_host_put(dev->host); diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c index 44214884deaf5..dbd58a7e7bc1f 100644 --- a/drivers/scsi/ipr.c +++ b/drivers/scsi/ipr.c @@ -61,8 +61,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -7843,6 +7843,30 @@ static int ipr_dump_mailbox_wait(struct ipr_cmnd *ipr_cmd) return IPR_RC_JOB_RETURN; } +/** + * ipr_set_affinity_nobalance + * @ioa_cfg: ipr_ioa_cfg struct for an ipr device + * @flag: bool + * true: ensable "IRQ_NO_BALANCING" bit for msix interrupt + * false: disable "IRQ_NO_BALANCING" bit for msix interrupt + * Description: This function will be called to disable/enable + * "IRQ_NO_BALANCING" to avoid irqbalance daemon + * kicking in during adapter reset. + **/ +static void ipr_set_affinity_nobalance(struct ipr_ioa_cfg *ioa_cfg, bool flag) +{ + int irq, i; + + for (i = 0; i < ioa_cfg->nvectors; i++) { + irq = pci_irq_vector(ioa_cfg->pdev, i); + + if (flag) + irq_set_status_flags(irq, IRQ_NO_BALANCING); + else + irq_clear_status_flags(irq, IRQ_NO_BALANCING); + } +} + /** * ipr_reset_restore_cfg_space - Restore PCI config space. * @ipr_cmd: ipr command struct @@ -7859,7 +7883,6 @@ static int ipr_reset_restore_cfg_space(struct ipr_cmnd *ipr_cmd) struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; ENTER; - ioa_cfg->pdev->state_saved = true; pci_restore_state(ioa_cfg->pdev); if (ipr_set_pcix_cmd_reg(ioa_cfg)) { @@ -7867,6 +7890,7 @@ static int ipr_reset_restore_cfg_space(struct ipr_cmnd *ipr_cmd) return IPR_RC_JOB_CONTINUE; } + ipr_set_affinity_nobalance(ioa_cfg, false); ipr_fail_all_ops(ioa_cfg); if (ioa_cfg->sis64) { @@ -7946,6 +7970,7 @@ static int ipr_reset_start_bist(struct ipr_cmnd *ipr_cmd) rc = pci_write_config_byte(ioa_cfg->pdev, PCI_BIST, PCI_BIST_START); if (rc == PCIBIOS_SUCCESSFUL) { + ipr_set_affinity_nobalance(ioa_cfg, true); ipr_cmd->job_step = ipr_reset_bist_done; ipr_reset_start_timer(ipr_cmd, IPR_WAIT_FOR_BIST_TIMEOUT); rc = IPR_RC_JOB_RETURN; diff --git a/drivers/scsi/libsas/sas_internal.h b/drivers/scsi/libsas/sas_internal.h index 6706f2be8d274..da5408c701cdd 100644 --- a/drivers/scsi/libsas/sas_internal.h +++ b/drivers/scsi/libsas/sas_internal.h @@ -145,20 +145,6 @@ static inline void sas_fail_probe(struct domain_device *dev, const char *func, i func, dev->parent ? "exp-attached" : "direct-attached", SAS_ADDR(dev->sas_addr), err); - - /* - * If the device probe failed, the expander phy attached address - * needs to be reset so that the phy will not be treated as flutter - * in the next revalidation - */ - if (dev->parent && !dev_is_expander(dev->dev_type)) { - struct sas_phy *phy = dev->phy; - struct domain_device *parent = dev->parent; - struct ex_phy *ex_phy = &parent->ex_dev.ex_phy[phy->number]; - - memset(ex_phy->attached_sas_addr, 0, SAS_ADDR_SIZE); - } - sas_unregister_dev(dev->port, dev); } diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index b71db7d7d747d..c08237f04bce2 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -934,10 +934,15 @@ lpfc_cmpl_els_flogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, /* Check to see if link went down during discovery */ if (lpfc_els_chk_latt(vport)) { /* One additional decrement on node reference count to - * trigger the release of the node + * trigger the release of the node. Make sure the ndlp + * is marked NLP_DROPPED. */ - if (!(ndlp->fc4_xpt_flags & SCSI_XPT_REGD)) + if (!test_bit(NLP_IN_DEV_LOSS, &ndlp->nlp_flag) && + !test_bit(NLP_DROPPED, &ndlp->nlp_flag) && + !(ndlp->fc4_xpt_flags & SCSI_XPT_REGD)) { + set_bit(NLP_DROPPED, &ndlp->nlp_flag); lpfc_nlp_put(ndlp); + } goto out; } @@ -995,9 +1000,10 @@ lpfc_cmpl_els_flogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, IOERR_LOOP_OPEN_FAILURE))) lpfc_vlog_msg(vport, KERN_WARNING, LOG_ELS, "2858 FLOGI Status:x%x/x%x TMO" - ":x%x Data x%lx x%x\n", + ":x%x Data x%lx x%x x%lx x%x\n", ulp_status, ulp_word4, tmo, - phba->hba_flag, phba->fcf.fcf_flag); + phba->hba_flag, phba->fcf.fcf_flag, + ndlp->nlp_flag, ndlp->fc4_xpt_flags); /* Check for retry */ if (lpfc_els_retry(phba, cmdiocb, rspiocb)) { @@ -1015,14 +1021,17 @@ lpfc_cmpl_els_flogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, * reference to trigger node release. */ if (!test_bit(NLP_IN_DEV_LOSS, &ndlp->nlp_flag) && - !(ndlp->fc4_xpt_flags & SCSI_XPT_REGD)) + !test_bit(NLP_DROPPED, &ndlp->nlp_flag) && + !(ndlp->fc4_xpt_flags & SCSI_XPT_REGD)) { + set_bit(NLP_DROPPED, &ndlp->nlp_flag); lpfc_nlp_put(ndlp); + } lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS, "0150 FLOGI Status:x%x/x%x " - "xri x%x TMO:x%x refcnt %d\n", + "xri x%x iotag x%x TMO:x%x refcnt %d\n", ulp_status, ulp_word4, cmdiocb->sli4_xritag, - tmo, kref_read(&ndlp->kref)); + cmdiocb->iotag, tmo, kref_read(&ndlp->kref)); /* If this is not a loop open failure, bail out */ if (!(ulp_status == IOSTAT_LOCAL_REJECT && @@ -1279,6 +1288,19 @@ lpfc_issue_els_flogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, uint32_t tmo, did; int rc; + /* It's possible for lpfc to reissue a FLOGI on an ndlp that is marked + * NLP_DROPPED. This happens when the FLOGI completed with the XB bit + * set causing lpfc to reference the ndlp until the XRI_ABORTED CQE is + * issued. The time window for the XRI_ABORTED CQE can be as much as + * 2*2*RA_TOV allowing for ndlp reuse of this type when the link is + * cycling quickly. When true, restore the initial reference and remove + * the NLP_DROPPED flag as lpfc is retrying. + */ + if (test_and_clear_bit(NLP_DROPPED, &ndlp->nlp_flag)) { + if (!lpfc_nlp_get(ndlp)) + return 1; + } + cmdsize = (sizeof(uint32_t) + sizeof(struct serv_parm)); elsiocb = lpfc_prep_els_iocb(vport, 1, cmdsize, retry, ndlp, ndlp->nlp_DID, ELS_CMD_FLOGI); diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 43d246c5c049c..717ae56c8e4bd 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -424,6 +424,7 @@ lpfc_check_nlp_post_devloss(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) { if (test_and_clear_bit(NLP_IN_RECOV_POST_DEV_LOSS, &ndlp->save_flags)) { + clear_bit(NLP_DROPPED, &ndlp->nlp_flag); lpfc_nlp_get(ndlp); lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY | LOG_NODE, "8438 Devloss timeout reversed on DID x%x " @@ -566,7 +567,8 @@ lpfc_dev_loss_tmo_handler(struct lpfc_nodelist *ndlp) return fcf_inuse; } - lpfc_nlp_put(ndlp); + if (!test_and_set_bit(NLP_DROPPED, &ndlp->nlp_flag)) + lpfc_nlp_put(ndlp); return fcf_inuse; } diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index f206267d9ecdc..adc0beaf5468f 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -12040,6 +12040,8 @@ lpfc_sli4_pci_mem_unset(struct lpfc_hba *phba) iounmap(phba->sli4_hba.conf_regs_memmap_p); if (phba->sli4_hba.dpp_regs_memmap_p) iounmap(phba->sli4_hba.dpp_regs_memmap_p); + if (phba->sli4_hba.dpp_regs_memmap_wc_p) + iounmap(phba->sli4_hba.dpp_regs_memmap_wc_p); break; case LPFC_SLI_INTF_IF_TYPE_1: break; @@ -14434,12 +14436,6 @@ lpfc_io_slot_reset_s3(struct pci_dev *pdev) pci_restore_state(pdev); - /* - * As the new kernel behavior of pci_restore_state() API call clears - * device saved_state flag, need to save the restored state again. - */ - pci_save_state(pdev); - if (pdev->is_busmaster) pci_set_master(pdev); diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 7ea7c4245c691..7b765719f4f6b 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -15911,6 +15911,32 @@ lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset) return NULL; } +static __maybe_unused void __iomem * +lpfc_dpp_wc_map(struct lpfc_hba *phba, uint8_t dpp_barset) +{ + + /* DPP region is supposed to cover 64-bit BAR2 */ + if (dpp_barset != WQ_PCI_BAR_4_AND_5) { + lpfc_log_msg(phba, KERN_WARNING, LOG_INIT, + "3273 dpp_barset x%x != WQ_PCI_BAR_4_AND_5\n", + dpp_barset); + return NULL; + } + + if (!phba->sli4_hba.dpp_regs_memmap_wc_p) { + void __iomem *dpp_map; + + dpp_map = ioremap_wc(phba->pci_bar2_map, + pci_resource_len(phba->pcidev, + PCI_64BIT_BAR4)); + + if (dpp_map) + phba->sli4_hba.dpp_regs_memmap_wc_p = dpp_map; + } + + return phba->sli4_hba.dpp_regs_memmap_wc_p; +} + /** * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs * @phba: HBA structure that EQs are on. @@ -16874,9 +16900,6 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, uint8_t dpp_barset; uint32_t dpp_offset; uint8_t wq_create_version; -#ifdef CONFIG_X86 - unsigned long pg_addr; -#endif /* sanity check on queue memory */ if (!wq || !cq) @@ -17062,14 +17085,15 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, #ifdef CONFIG_X86 /* Enable combined writes for DPP aperture */ - pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK; - rc = set_memory_wc(pg_addr, 1); - if (rc) { + bar_memmap_p = lpfc_dpp_wc_map(phba, dpp_barset); + if (!bar_memmap_p) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3272 Cannot setup Combined " "Write on WQ[%d] - disable DPP\n", wq->queue_id); phba->cfg_enable_dpp = 0; + } else { + wq->dpp_regaddr = bar_memmap_p + dpp_offset; } #else phba->cfg_enable_dpp = 0; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index fd6dab1578872..40f313e2769fc 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -785,6 +785,9 @@ struct lpfc_sli4_hba { void __iomem *dpp_regs_memmap_p; /* Kernel memory mapped address for * dpp registers */ + void __iomem *dpp_regs_memmap_wc_p;/* Kernel memory mapped address for + * dpp registers with write combining + */ union { struct { /* IF Type 0, BAR 0 PCI cfg space reg mem map */ diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h index b42933fcd4233..6561f98c3cb2d 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h @@ -166,6 +166,7 @@ struct mpi3_ioc_facts_data { #define MPI3_IOCFACTS_FLAGS_SIGNED_NVDATA_REQUIRED (0x00010000) #define MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK (0x0000ff00) #define MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT (8) +#define MPI3_IOCFACTS_FLAGS_MAX_REQ_PER_REPLY_QUEUE_LIMIT (0x00000040) #define MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK (0x00000030) #define MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_SHIFT (4) #define MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_NOT_STARTED (0x00000000) diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h index 6742684e2990a..31d68c151b207 100644 --- a/drivers/scsi/mpi3mr/mpi3mr.h +++ b/drivers/scsi/mpi3mr/mpi3mr.h @@ -56,8 +56,8 @@ extern struct list_head mrioc_list; extern int prot_mask; extern atomic64_t event_counter; -#define MPI3MR_DRIVER_VERSION "8.15.0.5.50" -#define MPI3MR_DRIVER_RELDATE "12-August-2025" +#define MPI3MR_DRIVER_VERSION "8.15.0.5.51" +#define MPI3MR_DRIVER_RELDATE "18-November-2025" #define MPI3MR_DRIVER_NAME "mpi3mr" #define MPI3MR_DRIVER_LICENSE "GPL" diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c index 8fe6e0bf342e2..4c8d78b840fc9 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_fw.c +++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c @@ -1530,6 +1530,7 @@ static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc) ioc_info(mrioc, "successfully transitioned to %s state\n", mpi3mr_iocstate_name(ioc_state)); + mpi3mr_clear_reset_history(mrioc); return 0; } ioc_status = readl(&mrioc->sysif_regs->ioc_status); @@ -1549,6 +1550,15 @@ static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc) elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000; } while (elapsed_time_sec < mrioc->ready_timeout); + ioc_state = mpi3mr_get_iocstate(mrioc); + if (ioc_state == MRIOC_STATE_READY) { + ioc_info(mrioc, + "successfully transitioned to %s state after %llu seconds\n", + mpi3mr_iocstate_name(ioc_state), elapsed_time_sec); + mpi3mr_clear_reset_history(mrioc); + return 0; + } + out_failed: elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000; if ((retry < 2) && (elapsed_time_sec < (mrioc->ready_timeout - 60))) { @@ -3158,6 +3168,8 @@ static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc, mrioc->facts.dma_mask = (facts_flags & MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >> MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT; + mrioc->facts.max_req_limit = (facts_flags & + MPI3_IOCFACTS_FLAGS_MAX_REQ_PER_REPLY_QUEUE_LIMIT); mrioc->facts.protocol_flags = facts_data->protocol_flags; mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word); mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests); @@ -4703,21 +4715,25 @@ void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc) } for (i = 0; i < mrioc->num_queues; i++) { - mrioc->op_reply_qinfo[i].qid = 0; - mrioc->op_reply_qinfo[i].ci = 0; - mrioc->op_reply_qinfo[i].num_replies = 0; - mrioc->op_reply_qinfo[i].ephase = 0; - atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0); - atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0); - mpi3mr_memset_op_reply_q_buffers(mrioc, i); - - mrioc->req_qinfo[i].ci = 0; - mrioc->req_qinfo[i].pi = 0; - mrioc->req_qinfo[i].num_requests = 0; - mrioc->req_qinfo[i].qid = 0; - mrioc->req_qinfo[i].reply_qid = 0; - spin_lock_init(&mrioc->req_qinfo[i].q_lock); - mpi3mr_memset_op_req_q_buffers(mrioc, i); + if (mrioc->op_reply_qinfo) { + mrioc->op_reply_qinfo[i].qid = 0; + mrioc->op_reply_qinfo[i].ci = 0; + mrioc->op_reply_qinfo[i].num_replies = 0; + mrioc->op_reply_qinfo[i].ephase = 0; + atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0); + atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0); + mpi3mr_memset_op_reply_q_buffers(mrioc, i); + } + + if (mrioc->req_qinfo) { + mrioc->req_qinfo[i].ci = 0; + mrioc->req_qinfo[i].pi = 0; + mrioc->req_qinfo[i].num_requests = 0; + mrioc->req_qinfo[i].qid = 0; + mrioc->req_qinfo[i].reply_qid = 0; + spin_lock_init(&mrioc->req_qinfo[i].q_lock); + mpi3mr_memset_op_req_q_buffers(mrioc, i); + } } atomic_set(&mrioc->pend_large_data_sz, 0); diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c index b88633e1efe27..d4ca878d08869 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_os.c +++ b/drivers/scsi/mpi3mr/mpi3mr_os.c @@ -1184,6 +1184,8 @@ static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc, if (is_added == true) tgtdev->io_throttle_enabled = (flags & MPI3_DEVICE0_FLAGS_IO_THROTTLING_REQUIRED) ? 1 : 0; + if (!mrioc->sas_transport_enabled) + tgtdev->non_stl = 1; switch (flags & MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_MASK) { case MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_256_LB: @@ -4844,7 +4846,7 @@ static int mpi3mr_target_alloc(struct scsi_target *starget) spin_lock_irqsave(&mrioc->tgtdev_lock, flags); if (starget->channel == mrioc->scsi_device_channel) { tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); - if (tgt_dev && !tgt_dev->is_hidden) { + if (tgt_dev && !tgt_dev->is_hidden && tgt_dev->non_stl) { scsi_tgt_priv_data->starget = starget; scsi_tgt_priv_data->dev_handle = tgt_dev->dev_handle; scsi_tgt_priv_data->perst_id = tgt_dev->perst_id; diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c index 6a8d35aea93a5..645524f3fe2d0 100644 --- a/drivers/scsi/pm8001/pm8001_sas.c +++ b/drivers/scsi/pm8001/pm8001_sas.c @@ -525,8 +525,9 @@ int pm8001_queue_command(struct sas_task *task, gfp_t gfp_flags) } else { task->task_done(task); } - rc = -ENODEV; - goto err_out; + spin_unlock_irqrestore(&pm8001_ha->lock, flags); + pm8001_dbg(pm8001_ha, IO, "pm8001_task_exec device gone\n"); + return 0; } ccb = pm8001_ccb_alloc(pm8001_ha, pm8001_dev, task); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index ccfc2d26dd372..0798bfd0372e4 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -1546,8 +1546,9 @@ qla2x00_update_optrom(struct bsg_job *bsg_job) ha->optrom_buffer = NULL; ha->optrom_state = QLA_SWAITING; mutex_unlock(&ha->optrom_mutex); - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!rval) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } @@ -2612,8 +2613,9 @@ qla2x00_manage_host_stats(struct bsg_job *bsg_job) sizeof(struct ql_vnd_mng_host_stats_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -2702,8 +2704,9 @@ qla2x00_get_host_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); kfree(data); host_stat_out: @@ -2802,8 +2805,9 @@ qla2x00_get_tgt_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); tgt_stat_out: kfree(data); @@ -2864,8 +2868,9 @@ qla2x00_manage_host_port(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, &rsp_data, sizeof(struct ql_vnd_mng_host_port_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -3240,7 +3245,8 @@ int qla2x00_mailbox_passthru(struct bsg_job *bsg_job) bsg_job->reply_len = sizeof(*bsg_job->reply); bsg_reply->result = DID_OK << 16; - bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); kfree(req_data); diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h index cb95b7b12051d..b3265952c4bed 100644 --- a/drivers/scsi/qla2xxx/qla_def.h +++ b/drivers/scsi/qla2xxx/qla_def.h @@ -3503,7 +3503,6 @@ struct isp_operations { #define QLA_MSIX_RSP_Q 0x01 #define QLA_ATIO_VECTOR 0x02 #define QLA_MSIX_QPAIR_MULTIQ_RSP_Q 0x03 -#define QLA_MSIX_QPAIR_MULTIQ_RSP_Q_HS 0x04 #define QLA_MIDX_DEFAULT 0 #define QLA_MIDX_RSP_Q 1 diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h index 145defc420f27..55d531c19e6b2 100644 --- a/drivers/scsi/qla2xxx/qla_gbl.h +++ b/drivers/scsi/qla2xxx/qla_gbl.h @@ -766,7 +766,7 @@ extern int qla2x00_dfs_remove(scsi_qla_host_t *); /* Globa function prototypes for multi-q */ extern int qla25xx_request_irq(struct qla_hw_data *, struct qla_qpair *, - struct qla_msix_entry *, int); + struct qla_msix_entry *); extern int qla25xx_init_req_que(struct scsi_qla_host *, struct req_que *); extern int qla25xx_init_rsp_que(struct scsi_qla_host *, struct rsp_que *); extern int qla25xx_create_req_que(struct qla_hw_data *, uint16_t, uint8_t, diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c index 51c7cea71f902..880cd73feaca4 100644 --- a/drivers/scsi/qla2xxx/qla_gs.c +++ b/drivers/scsi/qla2xxx/qla_gs.c @@ -3266,9 +3266,6 @@ void qla_fab_scan_finish(scsi_qla_host_t *vha, srb_t *sp) atomic_read(&fcport->state) == FCS_ONLINE) || do_delete) { if (fcport->loop_id != FC_NO_LOOP_ID) { - if (fcport->flags & FCF_FCP2_DEVICE) - continue; - ql_log(ql_log_warn, vha, 0x20f0, "%s %d %8phC post del sess\n", __func__, __LINE__, @@ -3535,8 +3532,8 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp) if (vha->scan.scan_flags & SF_SCANNING) { spin_unlock_irqrestore(&vha->work_lock, flags); ql_dbg(ql_dbg_disc + ql_dbg_verbose, vha, 0x2012, - "%s: scan active\n", __func__); - return rval; + "%s: scan active for sp:%p\n", __func__, sp); + goto done_free_sp; } vha->scan.scan_flags |= SF_SCANNING; if (!sp) @@ -3701,23 +3698,25 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp) return rval; done_free_sp: - if (sp->u.iocb_cmd.u.ctarg.req) { - dma_free_coherent(&vha->hw->pdev->dev, - sp->u.iocb_cmd.u.ctarg.req_allocated_size, - sp->u.iocb_cmd.u.ctarg.req, - sp->u.iocb_cmd.u.ctarg.req_dma); - sp->u.iocb_cmd.u.ctarg.req = NULL; - } - if (sp->u.iocb_cmd.u.ctarg.rsp) { - dma_free_coherent(&vha->hw->pdev->dev, - sp->u.iocb_cmd.u.ctarg.rsp_allocated_size, - sp->u.iocb_cmd.u.ctarg.rsp, - sp->u.iocb_cmd.u.ctarg.rsp_dma); - sp->u.iocb_cmd.u.ctarg.rsp = NULL; - } + if (sp) { + if (sp->u.iocb_cmd.u.ctarg.req) { + dma_free_coherent(&vha->hw->pdev->dev, + sp->u.iocb_cmd.u.ctarg.req_allocated_size, + sp->u.iocb_cmd.u.ctarg.req, + sp->u.iocb_cmd.u.ctarg.req_dma); + sp->u.iocb_cmd.u.ctarg.req = NULL; + } + if (sp->u.iocb_cmd.u.ctarg.rsp) { + dma_free_coherent(&vha->hw->pdev->dev, + sp->u.iocb_cmd.u.ctarg.rsp_allocated_size, + sp->u.iocb_cmd.u.ctarg.rsp, + sp->u.iocb_cmd.u.ctarg.rsp_dma); + sp->u.iocb_cmd.u.ctarg.rsp = NULL; + } - /* ref: INIT */ - kref_put(&sp->cmd_kref, qla2x00_sp_release); + /* ref: INIT */ + kref_put(&sp->cmd_kref, qla2x00_sp_release); + } spin_lock_irqsave(&vha->work_lock, flags); vha->scan.scan_flags &= ~SF_SCANNING; diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index 6a2e1c7fd1251..84f89445c7470 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -1859,15 +1859,6 @@ void qla2x00_handle_rscn(scsi_qla_host_t *vha, struct event_arg *ea) case RSCN_PORT_ADDR: fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1); if (fcport) { - if (ql2xfc2target && - fcport->flags & FCF_FCP2_DEVICE && - atomic_read(&fcport->state) == FCS_ONLINE) { - ql_dbg(ql_dbg_disc, vha, 0x2115, - "Delaying session delete for FCP2 portid=%06x %8phC ", - fcport->d_id.b24, fcport->port_name); - return; - } - if (vha->hw->flags.edif_enabled && DBELL_ACTIVE(vha)) { /* * On ipsec start by remote port, Target port @@ -2471,8 +2462,23 @@ qla24xx_handle_plogi_done_event(struct scsi_qla_host *vha, struct event_arg *ea) ea->sp->gen1, fcport->rscn_gen, ea->data[0], ea->data[1], ea->iop[0], ea->iop[1]); - if ((fcport->fw_login_state == DSC_LS_PLOGI_PEND) || - (fcport->fw_login_state == DSC_LS_PRLI_PEND)) { + if (fcport->fw_login_state == DSC_LS_PLOGI_PEND) { + ql_dbg(ql_dbg_disc, vha, 0x20ea, + "%s %d %8phC Remote is trying to login\n", + __func__, __LINE__, fcport->port_name); + /* + * If we get here, there is port thats already logged in, + * but it's state has not moved ahead. Recheck with FW on + * what state it is in and proceed ahead + */ + if (!N2N_TOPO(vha->hw)) { + fcport->fw_login_state = DSC_LS_PRLI_COMP; + qla24xx_post_gpdb_work(vha, fcport, 0); + } + return; + } + + if (fcport->fw_login_state == DSC_LS_PRLI_PEND) { ql_dbg(ql_dbg_disc, vha, 0x20ea, "%s %d %8phC Remote is trying to login\n", __func__, __LINE__, fcport->port_name); diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index c4c6b5c6658c0..608d2f36e7b4f 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -878,6 +878,9 @@ qla27xx_copy_multiple_pkt(struct scsi_qla_host *vha, void **pkt, payload_size = sizeof(purex->els_frame_payload); } + if (total_bytes > sizeof(item->iocb.iocb)) + total_bytes = sizeof(item->iocb.iocb); + pending_bytes = total_bytes; no_bytes = (pending_bytes > payload_size) ? payload_size : pending_bytes; @@ -1163,6 +1166,10 @@ qla27xx_copy_fpin_pkt(struct scsi_qla_host *vha, void **pkt, total_bytes = (le16_to_cpu(purex->frame_size) & 0x0FFF) - PURX_ELS_HEADER_SIZE; + + if (total_bytes > sizeof(item->iocb.iocb)) + total_bytes = sizeof(item->iocb.iocb); + pending_bytes = total_bytes; entry_count = entry_count_remaining = purex->entry_count; no_bytes = (pending_bytes > sizeof(purex->els_frame_payload)) ? @@ -1669,13 +1676,28 @@ qla2x00_async_event(scsi_qla_host_t *vha, struct rsp_que *rsp, uint16_t *mb) /* Port logout */ fcport = qla2x00_find_fcport_by_loopid(vha, mb[1]); - if (!fcport) + if (!fcport) { + ql_dbg(ql_dbg_async, vha, 0x5011, + "Could not find fcport:%04x %04x %04x\n", + mb[1], mb[2], mb[3]); break; - if (atomic_read(&fcport->state) != FCS_ONLINE) + } + + if (atomic_read(&fcport->state) != FCS_ONLINE) { + ql_dbg(ql_dbg_async, vha, 0x5012, + "Port state is not online State:0x%x \n", + atomic_read(&fcport->state)); + ql_dbg(ql_dbg_async, vha, 0x5012, + "Scheduling session for deletion \n"); + fcport->logout_on_delete = 0; + qlt_schedule_sess_for_deletion(fcport); break; + } + ql_dbg(ql_dbg_async, vha, 0x508a, "Marking port lost loopid=%04x portid=%06x.\n", fcport->loop_id, fcport->d_id.b24); + if (qla_ini_mode_enabled(vha)) { fcport->logout_on_delete = 0; qlt_schedule_sess_for_deletion(fcport); @@ -4467,32 +4489,6 @@ qla2xxx_msix_rsp_q(int irq, void *dev_id) return IRQ_HANDLED; } -irqreturn_t -qla2xxx_msix_rsp_q_hs(int irq, void *dev_id) -{ - struct qla_hw_data *ha; - struct qla_qpair *qpair; - struct device_reg_24xx __iomem *reg; - unsigned long flags; - - qpair = dev_id; - if (!qpair) { - ql_log(ql_log_info, NULL, 0x505b, - "%s: NULL response queue pointer.\n", __func__); - return IRQ_NONE; - } - ha = qpair->hw; - - reg = &ha->iobase->isp24; - spin_lock_irqsave(&ha->hardware_lock, flags); - wrt_reg_dword(®->hccr, HCCRX_CLR_RISC_INT); - spin_unlock_irqrestore(&ha->hardware_lock, flags); - - queue_work(ha->wq, &qpair->q_work); - - return IRQ_HANDLED; -} - /* Interrupt handling helpers. */ struct qla_init_msix_entry { @@ -4505,7 +4501,6 @@ static const struct qla_init_msix_entry msix_entries[] = { { "rsp_q", qla24xx_msix_rsp_q }, { "atio_q", qla83xx_msix_atio_q }, { "qpair_multiq", qla2xxx_msix_rsp_q }, - { "qpair_multiq_hs", qla2xxx_msix_rsp_q_hs }, }; static const struct qla_init_msix_entry qla82xx_msix_entries[] = { @@ -4792,9 +4787,10 @@ qla2x00_free_irqs(scsi_qla_host_t *vha) } int qla25xx_request_irq(struct qla_hw_data *ha, struct qla_qpair *qpair, - struct qla_msix_entry *msix, int vector_type) + struct qla_msix_entry *msix) { - const struct qla_init_msix_entry *intr = &msix_entries[vector_type]; + const struct qla_init_msix_entry *intr = + &msix_entries[QLA_MSIX_QPAIR_MULTIQ_RSP_Q]; scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev); int ret; diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c index 32eb0ce8b170d..1f01576f044b8 100644 --- a/drivers/scsi/qla2xxx/qla_mbx.c +++ b/drivers/scsi/qla2xxx/qla_mbx.c @@ -253,6 +253,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) /* Issue set host interrupt command to send cmd out. */ ha->flags.mbox_int = 0; clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags); + reinit_completion(&ha->mbx_intr_comp); /* Unlock mbx registers and wait for interrupt */ ql_dbg(ql_dbg_mbx, vha, 0x100f, @@ -279,6 +280,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) "cmd=%x Timeout.\n", command); spin_lock_irqsave(&ha->hardware_lock, flags); clear_bit(MBX_INTR_WAIT, &ha->mbx_cmd_flags); + reinit_completion(&ha->mbx_intr_comp); spin_unlock_irqrestore(&ha->hardware_lock, flags); if (chip_reset != ha->chip_reset) { diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c index 8b71ac0b1d999..0abc47e72e0bf 100644 --- a/drivers/scsi/qla2xxx/qla_mid.c +++ b/drivers/scsi/qla2xxx/qla_mid.c @@ -899,9 +899,7 @@ qla25xx_create_rsp_que(struct qla_hw_data *ha, uint16_t options, rsp->options, rsp->id, rsp->rsp_q_in, rsp->rsp_q_out); - ret = qla25xx_request_irq(ha, qpair, qpair->msix, - ha->flags.disable_msix_handshake ? - QLA_MSIX_QPAIR_MULTIQ_RSP_Q : QLA_MSIX_QPAIR_MULTIQ_RSP_Q_HS); + ret = qla25xx_request_irq(ha, qpair, qpair->msix); if (ret) goto que_failed; diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index 316594aa40cc5..42eb65a62f1f3 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -1292,7 +1292,7 @@ void qla2xxx_process_purls_iocb(void **pkt, struct rsp_que **rsp) a.reason = FCNVME_RJT_RC_LOGIC; a.explanation = FCNVME_RJT_EXP_NONE; xmt_reject = true; - kfree(item); + qla24xx_free_purex_item(item); goto out; } diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index 5ffd945866527..59ca4adcb4315 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -1182,7 +1182,8 @@ qla2x00_wait_for_hba_ready(scsi_qla_host_t *vha) while ((qla2x00_reset_active(vha) || ha->dpc_active || ha->flags.mbox_busy) || test_bit(FX00_RESET_RECOVERY, &vha->dpc_flags) || - test_bit(FX00_TARGET_SCAN, &vha->dpc_flags)) { + test_bit(FX00_TARGET_SCAN, &vha->dpc_flags) || + (vha->scan.scan_flags & SF_SCANNING)) { if (test_bit(UNLOADING, &base_vha->dpc_flags)) break; msleep(1000); @@ -1862,12 +1863,6 @@ __qla2x00_abort_all_cmds(struct qla_qpair *qp, int res) for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) { sp = req->outstanding_cmds[cnt]; if (sp) { - if (qla2x00_chip_is_down(vha)) { - req->outstanding_cmds[cnt] = NULL; - sp->done(sp, res); - continue; - } - switch (sp->cmd_type) { case TYPE_SRB: qla2x00_abort_srb(qp, sp, res, &flags); @@ -1881,10 +1876,26 @@ __qla2x00_abort_all_cmds(struct qla_qpair *qp, int res) continue; } cmd = (struct qla_tgt_cmd *)sp; - cmd->aborted = 1; + + if (cmd->sg_mapped) + qlt_unmap_sg(vha, cmd); + + if (cmd->state == QLA_TGT_STATE_NEED_DATA) { + cmd->aborted = 1; + cmd->write_data_transferred = 0; + cmd->state = QLA_TGT_STATE_DATA_IN; + ha->tgt.tgt_ops->handle_data(cmd); + } else { + ha->tgt.tgt_ops->free_cmd(cmd); + } break; case TYPE_TGT_TMCMD: - /* Skip task management functions. */ + /* + * Currently, only ABTS response gets on the + * outstanding_cmds[] + */ + ha->tgt.tgt_ops->free_mcmd( + (struct qla_tgt_mgmt_cmd *) sp); break; default: break; @@ -3444,13 +3455,7 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id) ha->mqenable = 0; if (ha->mqenable) { - bool startit = false; - - if (QLA_TGT_MODE_ENABLED()) - startit = false; - - if (ql2x_ini_mode == QLA2XXX_INI_MODE_ENABLED) - startit = true; + bool startit = !!(host->active_mode & MODE_INITIATOR); /* Create start of day qpairs for Block MQ */ for (i = 0; i < ha->max_qpairs; i++) @@ -4484,7 +4489,7 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, fail_elsrej: dma_pool_destroy(ha->purex_dma_pool); fail_flt: - dma_free_coherent(&ha->pdev->dev, SFP_DEV_SIZE, + dma_free_coherent(&ha->pdev->dev, sizeof(struct qla_flt_header) + FLT_REGIONS_SIZE, ha->flt, ha->flt_dma); fail_flt_buffer: @@ -7886,11 +7891,6 @@ qla2xxx_pci_slot_reset(struct pci_dev *pdev) pci_restore_state(pdev); - /* pci_restore_state() clears the saved_state flag of the device - * save restored state which resets saved_state flag - */ - pci_save_state(pdev); - if (ha->mem_only) rc = pci_enable_device_mem(pdev); else diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c index 1e81582085e38..4c6aff59fe3fb 100644 --- a/drivers/scsi/qla2xxx/qla_target.c +++ b/drivers/scsi/qla2xxx/qla_target.c @@ -2443,7 +2443,7 @@ static int qlt_pci_map_calc_cnt(struct qla_tgt_prm *prm) return -1; } -static void qlt_unmap_sg(struct scsi_qla_host *vha, struct qla_tgt_cmd *cmd) +void qlt_unmap_sg(struct scsi_qla_host *vha, struct qla_tgt_cmd *cmd) { struct qla_hw_data *ha; struct qla_qpair *qpair; @@ -3773,9 +3773,6 @@ int qlt_abort_cmd(struct qla_tgt_cmd *cmd) spin_lock_irqsave(&cmd->cmd_lock, flags); if (cmd->aborted) { - if (cmd->sg_mapped) - qlt_unmap_sg(vha, cmd); - spin_unlock_irqrestore(&cmd->cmd_lock, flags); /* * It's normal to see 2 calls in this path: diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h index 15a59c125c532..c483966d0a847 100644 --- a/drivers/scsi/qla2xxx/qla_target.h +++ b/drivers/scsi/qla2xxx/qla_target.h @@ -1058,6 +1058,7 @@ extern int qlt_abort_cmd(struct qla_tgt_cmd *); extern void qlt_xmit_tm_rsp(struct qla_tgt_mgmt_cmd *); extern void qlt_free_mcmd(struct qla_tgt_mgmt_cmd *); extern void qlt_free_cmd(struct qla_tgt_cmd *cmd); +extern void qlt_unmap_sg(struct scsi_qla_host *vha, struct qla_tgt_cmd *cmd); extern void qlt_async_event(uint16_t, struct scsi_qla_host *, uint16_t *); extern void qlt_enable_vha(struct scsi_qla_host *); extern void qlt_vport_create(struct scsi_qla_host *, struct qla_hw_data *); diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c index 83ff66f954e67..97329c97332f8 100644 --- a/drivers/scsi/qla4xxx/ql4_os.c +++ b/drivers/scsi/qla4xxx/ql4_os.c @@ -9796,11 +9796,6 @@ qla4xxx_pci_slot_reset(struct pci_dev *pdev) */ pci_restore_state(pdev); - /* pci_restore_state() clears the saved_state flag of the device - * save restored state which resets saved_state flag - */ - pci_save_state(pdev); - /* Initialize device or resume if in suspended state */ rc = pci_enable_device(pdev); if (rc) { diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index b2ab97be5db3d..047d56d23beab 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -7410,7 +7410,7 @@ MODULE_PARM_DESC(lbprz, MODULE_PARM_DESC(lbpu, "enable LBP, support UNMAP command (def=0)"); MODULE_PARM_DESC(lbpws, "enable LBP, support WRITE SAME(16) with UNMAP bit (def=0)"); MODULE_PARM_DESC(lbpws10, "enable LBP, support WRITE SAME(10) with UNMAP bit (def=0)"); -MODULE_PARM_DESC(atomic_write, "enable ATOMIC WRITE support, support WRITE ATOMIC(16) (def=0)"); +MODULE_PARM_DESC(atomic_wr, "enable ATOMIC WRITE support, support WRITE ATOMIC(16) (def=0)"); MODULE_PARM_DESC(lowest_aligned, "lowest aligned lba (def=0)"); MODULE_PARM_DESC(lun_format, "LUN format: 0->peripheral (def); 1 --> flat address method"); MODULE_PARM_DESC(max_luns, "number of LUNs per target to simulate(def=1)"); diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c index 78346b2b69c91..c51146882a1fa 100644 --- a/drivers/scsi/scsi_devinfo.c +++ b/drivers/scsi/scsi_devinfo.c @@ -190,7 +190,7 @@ static struct { {"IBM", "2076", NULL, BLIST_NO_VPD_SIZE}, {"IBM", "2105", NULL, BLIST_RETRY_HWERROR}, {"iomega", "jaz 1GB", "J.86", BLIST_NOTQ | BLIST_NOLUN}, - {"IOMEGA", "ZIP", NULL, BLIST_NOTQ | BLIST_NOLUN}, + {"IOMEGA", "ZIP", NULL, BLIST_NOTQ | BLIST_NOLUN | BLIST_SKIP_IO_HINTS}, {"IOMEGA", "Io20S *F", NULL, BLIST_KEY}, {"INSITE", "Floptical F*8I", NULL, BLIST_KEY}, {"INSITE", "I325VM", NULL, BLIST_KEY}, diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 1c13812a3f035..2fd4ca96b3089 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -282,11 +282,20 @@ static void scsi_eh_inc_host_failed(struct rcu_head *head) { struct scsi_cmnd *scmd = container_of(head, typeof(*scmd), rcu); struct Scsi_Host *shost = scmd->device->host; - unsigned int busy = scsi_host_busy(shost); + unsigned int busy; unsigned long flags; spin_lock_irqsave(shost->host_lock, flags); shost->host_failed++; + spin_unlock_irqrestore(shost->host_lock, flags); + /* + * The counting of busy requests needs to occur after adding to + * host_failed or after the lock acquire for adding to host_failed + * to prevent a race with host unbusy and missing an eh wakeup. + */ + busy = scsi_host_busy(shost); + + spin_lock_irqsave(shost->host_lock, flags); scsi_eh_wakeup(shost, busy); spin_unlock_irqrestore(shost->host_lock, flags); } @@ -1060,6 +1069,9 @@ void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, struct scsi_eh_save *ses, unsigned char *cmnd, int cmnd_size, unsigned sense_bytes) { struct scsi_device *sdev = scmd->device; +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct request *rq = scsi_cmd_to_rq(scmd); +#endif /* * We need saved copies of a number of fields - this is because @@ -1111,6 +1123,18 @@ void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, struct scsi_eh_save *ses, scmd->cmnd[1] = (scmd->cmnd[1] & 0x1f) | (sdev->lun << 5 & 0xe0); + /* + * Encryption must be disabled for the commands submitted by the error handler. + * Hence, clear the encryption context information. + */ +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + ses->rq_crypt_keyslot = rq->crypt_keyslot; + ses->rq_crypt_ctx = rq->crypt_ctx; + + rq->crypt_keyslot = NULL; + rq->crypt_ctx = NULL; +#endif + /* * Zero the sense buffer. The scsi spec mandates that any * untransferred sense data should be interpreted as being zero. @@ -1128,6 +1152,10 @@ EXPORT_SYMBOL(scsi_eh_prep_cmnd); */ void scsi_eh_restore_cmnd(struct scsi_cmnd* scmd, struct scsi_eh_save *ses) { +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct request *rq = scsi_cmd_to_rq(scmd); +#endif + /* * Restore original data */ @@ -1140,6 +1168,11 @@ void scsi_eh_restore_cmnd(struct scsi_cmnd* scmd, struct scsi_eh_save *ses) scmd->underflow = ses->underflow; scmd->prot_op = ses->prot_op; scmd->eh_eflags = ses->eh_eflags; + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + rq->crypt_keyslot = ses->rq_crypt_keyslot; + rq->crypt_ctx = ses->rq_crypt_ctx; +#endif } EXPORT_SYMBOL(scsi_eh_restore_cmnd); diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index d7e42293b8645..7ddb73cd6d9fe 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -376,6 +376,14 @@ static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd) rcu_read_lock(); __clear_bit(SCMD_STATE_INFLIGHT, &cmd->state); if (unlikely(scsi_host_in_recovery(shost))) { + /* + * Ensure the clear of SCMD_STATE_INFLIGHT is visible to + * other CPUs before counting busy requests. Otherwise, + * reordering can cause CPUs to race and miss an eh wakeup + * when no CPU sees all busy requests as done or timed out. + */ + smp_mb(); + unsigned int busy = scsi_host_busy(shost); spin_lock_irqsave(shost->host_lock, flags); diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 3c6e089e80c38..05ccd11e56a9b 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -355,11 +355,8 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, * default device queue depth to figure out sbitmap shift * since we use this queue depth most of times. */ - if (scsi_realloc_sdev_budget_map(sdev, depth)) { - put_device(&starget->dev); - kfree(sdev); - goto out; - } + if (scsi_realloc_sdev_budget_map(sdev, depth)) + goto out_device_destroy; scsi_change_queue_depth(sdev, depth); diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index d69c7c444a311..081c168094374 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c @@ -1734,7 +1734,7 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel, break; default: - if (channel < shost->max_channel) { + if (channel <= shost->max_channel) { res = scsi_scan_host_selected(shost, channel, id, lun, SCSI_SCAN_MANUAL); } else { diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c index 2c61624cb4b03..7e1f085ad350a 100644 --- a/drivers/scsi/ses.c +++ b/drivers/scsi/ses.c @@ -216,7 +216,7 @@ static unsigned char *ses_get_page2_descriptor(struct enclosure_device *edev, unsigned char *type_ptr = ses_dev->page1_types; unsigned char *desc_ptr = ses_dev->page2 + 8; - if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len) < 0) + if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len)) return NULL; for (i = 0; i < ses_dev->page1_num_types; i++, type_ptr += 4) { @@ -529,9 +529,8 @@ struct efd { }; static int ses_enclosure_find_by_addr(struct enclosure_device *edev, - void *data) + struct efd *efd) { - struct efd *efd = data; int i; struct ses_component *scomp; @@ -684,7 +683,7 @@ static void ses_match_to_enclosure(struct enclosure_device *edev, if (efd.addr) { efd.dev = &sdev->sdev_gendev; - enclosure_for_each_device(ses_enclosure_find_by_addr, &efd); + ses_enclosure_find_by_addr(edev, &efd); } } diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index b3af9b78fa123..57fba34832ad1 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -731,6 +731,8 @@ sg_new_write(Sg_fd *sfp, struct file *file, const char __user *buf, sg_remove_request(sfp, srp); return -EFAULT; } + hp->duration = jiffies_to_msecs(jiffies); + if (hp->interface_id != 'S') { sg_remove_request(sfp, srp); return -ENOSYS; @@ -815,7 +817,6 @@ sg_common_write(Sg_fd * sfp, Sg_request * srp, return -ENODEV; } - hp->duration = jiffies_to_msecs(jiffies); if (hp->interface_id != '\0' && /* v3 (or later) interface */ (SG_FLAG_Q_AT_TAIL & hp->flags)) at_head = 0; @@ -1338,9 +1339,6 @@ sg_rq_end_io(struct request *rq, blk_status_t status) "sg_cmd_done: pack_id=%d, res=0x%x\n", srp->header.pack_id, result)); srp->header.resid = resid; - ms = jiffies_to_msecs(jiffies); - srp->header.duration = (ms > srp->header.duration) ? - (ms - srp->header.duration) : 0; if (0 != result) { struct scsi_sense_hdr sshdr; @@ -1389,6 +1387,9 @@ sg_rq_end_io(struct request *rq, blk_status_t status) done = 0; } srp->done = done; + ms = jiffies_to_msecs(jiffies); + srp->header.duration = (ms > srp->header.duration) ? + (ms - srp->header.duration) : 0; write_unlock_irqrestore(&sfp->rq_list_lock, iflags); if (likely(done)) { @@ -2533,6 +2534,7 @@ static void sg_proc_debug_helper(struct seq_file *s, Sg_device * sdp) const sg_io_hdr_t *hp; const char * cp; unsigned int ms; + unsigned int duration; k = 0; list_for_each_entry(fp, &sdp->sfds, sfd_siblings) { @@ -2570,13 +2572,17 @@ static void sg_proc_debug_helper(struct seq_file *s, Sg_device * sdp) seq_printf(s, " id=%d blen=%d", srp->header.pack_id, blen); if (srp->done) - seq_printf(s, " dur=%d", hp->duration); + seq_printf(s, " dur=%u", hp->duration); else { ms = jiffies_to_msecs(jiffies); - seq_printf(s, " t_o/elap=%d/%d", + duration = READ_ONCE(hp->duration); + if (duration) + duration = (ms > duration ? + ms - duration : 0); + seq_printf(s, " t_o/elap=%u/%u", (new_interface ? hp->timeout : jiffies_to_msecs(fp->timeout)), - (ms > hp->duration ? ms - hp->duration : 0)); + duration); } seq_printf(s, "ms sgat=%d op=0x%02x\n", usg, (int) srp->data.cmd_opcode); diff --git a/drivers/scsi/sim710.c b/drivers/scsi/sim710.c index e519df68d603d..70c75ab1453a1 100644 --- a/drivers/scsi/sim710.c +++ b/drivers/scsi/sim710.c @@ -133,6 +133,7 @@ static int sim710_probe_common(struct device *dev, unsigned long base_addr, out_put_host: scsi_host_put(host); out_release: + ioport_unmap(hostdata->base); release_region(base_addr, 64); out_free: kfree(hostdata); @@ -148,6 +149,7 @@ static int sim710_device_remove(struct device *dev) scsi_remove_host(host); NCR_700_release(host); + ioport_unmap(hostdata->base); kfree(hostdata); free_irq(host->irq, host); release_region(host->base, 64); diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 03c97e60d36f6..5a6e1bb57e7c8 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -1241,7 +1241,8 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b dev_err(&ctrl_info->pci_dev->dev, "RPL returned unsupported data format %u\n", rpl_response_format); - return -EINVAL; + rc = -EINVAL; + goto out_free_rpl_list; } else { dev_warn(&ctrl_info->pci_dev->dev, "RPL returned extended format 2 instead of 4\n"); @@ -1253,8 +1254,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b rpl_16byte_wwid_list = kmalloc(struct_size(rpl_16byte_wwid_list, lun_entries, num_physicals), GFP_KERNEL); - if (!rpl_16byte_wwid_list) - return -ENOMEM; + if (!rpl_16byte_wwid_list) { + rc = -ENOMEM; + goto out_free_rpl_list; + } put_unaligned_be32(num_physicals * sizeof(struct report_phys_lun_16byte_wwid), &rpl_16byte_wwid_list->header.list_length); @@ -1275,6 +1278,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b *buffer = rpl_16byte_wwid_list; return 0; + +out_free_rpl_list: + kfree(rpl_list); + return rc; } static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer) @@ -6410,10 +6417,22 @@ static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev static int pqi_device_reset_handler(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun, struct scsi_cmnd *scmd, u8 scsi_opcode) { + unsigned long flags; int rc; mutex_lock(&ctrl_info->lun_reset_mutex); + spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); + if (pqi_find_scsi_dev(ctrl_info, device->bus, device->target, device->lun) == NULL) { + dev_warn(&ctrl_info->pci_dev->dev, + "skipping reset of scsi %d:%d:%d:%u, device has been removed\n", + ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun); + spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); + mutex_unlock(&ctrl_info->lun_reset_mutex); + return 0; + } + spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); + dev_err(&ctrl_info->pci_dev->dev, "resetting scsi %d:%d:%d:%u SCSI cmd at %p due to cmd opcode 0x%02x\n", ctrl_info->scsi_host->host_no, device->bus, device->target, lun, scmd, scsi_opcode); @@ -6594,7 +6613,9 @@ static void pqi_sdev_destroy(struct scsi_device *sdev) { struct pqi_ctrl_info *ctrl_info; struct pqi_scsi_dev *device; + struct pqi_tmf_work *tmf_work; int mutex_acquired; + unsigned int lun; unsigned long flags; ctrl_info = shost_to_hba(sdev->host); @@ -6621,8 +6642,13 @@ static void pqi_sdev_destroy(struct scsi_device *sdev) mutex_unlock(&ctrl_info->scan_mutex); + for (lun = 0, tmf_work = device->tmf_work; lun < PQI_MAX_LUNS_PER_DEVICE; lun++, tmf_work++) + cancel_work_sync(&tmf_work->work_struct); + + mutex_lock(&ctrl_info->lun_reset_mutex); pqi_dev_info(ctrl_info, "removed", device); pqi_free_device(device); + mutex_unlock(&ctrl_info->lun_reset_mutex); } static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg) @@ -10108,6 +10134,10 @@ static const struct pci_device_id pqi_pci_id_table[] = { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, 0x207d, 0x4240) }, + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, + 0x207d, 0x4840) + }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f, PCI_VENDOR_ID_ADVANTECH, 0x8312) diff --git a/drivers/scsi/stex.c b/drivers/scsi/stex.c index d8ad02c293205..e02f28e5a104e 100644 --- a/drivers/scsi/stex.c +++ b/drivers/scsi/stex.c @@ -1844,6 +1844,7 @@ static int stex_probe(struct pci_dev *pdev, const struct pci_device_id *id) out_scsi_host_put: scsi_host_put(host); out_disable: + unregister_reboot_notifier(&stex_notifier); pci_disable_device(pdev); return err; diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index 6e4112143c768..68c837146b9ea 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -1144,7 +1144,7 @@ static void storvsc_on_io_completion(struct storvsc_device *stor_device, * The current SCSI handling on the host side does * not correctly handle: * INQUIRY command with page code parameter set to 0x80 - * MODE_SENSE command with cmd[2] == 0x1c + * MODE_SENSE and MODE_SENSE_10 command with cmd[2] == 0x1c * MAINTENANCE_IN is not supported by HyperV FC passthrough * * Setup srb and scsi status so this won't be fatal. @@ -1154,6 +1154,7 @@ static void storvsc_on_io_completion(struct storvsc_device *stor_device, if ((stor_pkt->vm_srb.cdb[0] == INQUIRY) || (stor_pkt->vm_srb.cdb[0] == MODE_SENSE) || + (stor_pkt->vm_srb.cdb[0] == MODE_SENSE_10) || (stor_pkt->vm_srb.cdb[0] == MAINTENANCE_IN && hv_dev_is_fc(device))) { vstor_packet->vm_srb.scsi_status = 0; @@ -1854,8 +1855,9 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) cmd_request->payload_sz = payload_sz; /* Invokes the vsc to start an IO */ - ret = storvsc_do_io(dev, cmd_request, get_cpu()); - put_cpu(); + migrate_disable(); + ret = storvsc_do_io(dev, cmd_request, smp_processor_id()); + migrate_enable(); if (ret) scsi_dma_unmap(scmnd); diff --git a/drivers/slimbus/core.c b/drivers/slimbus/core.c index 005fa2ef100f5..0992ca35cef49 100644 --- a/drivers/slimbus/core.c +++ b/drivers/slimbus/core.c @@ -379,6 +379,8 @@ struct slim_device *slim_get_device(struct slim_controller *ctrl, sbdev = slim_alloc_device(ctrl, e_addr, NULL); if (!sbdev) return ERR_PTR(-ENOMEM); + + get_device(&sbdev->dev); } return sbdev; @@ -489,21 +491,24 @@ int slim_device_report_present(struct slim_controller *ctrl, if (ctrl->sched.clk_state != SLIM_CLK_ACTIVE) { dev_err(ctrl->dev, "slim ctrl not active,state:%d, ret:%d\n", ctrl->sched.clk_state, ret); - goto slimbus_not_active; + goto out_put_rpm; } sbdev = slim_get_device(ctrl, e_addr); - if (IS_ERR(sbdev)) - return -ENODEV; + if (IS_ERR(sbdev)) { + ret = -ENODEV; + goto out_put_rpm; + } if (sbdev->is_laddr_valid) { *laddr = sbdev->laddr; - return 0; + ret = 0; + } else { + ret = slim_device_alloc_laddr(sbdev, true); } - ret = slim_device_alloc_laddr(sbdev, true); - -slimbus_not_active: + put_device(&sbdev->dev); +out_put_rpm: pm_runtime_mark_last_busy(ctrl->dev); pm_runtime_put_autosuspend(ctrl->dev); return ret; diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c index cd40ab839c541..db45654f16955 100644 --- a/drivers/slimbus/qcom-ngd-ctrl.c +++ b/drivers/slimbus/qcom-ngd-ctrl.c @@ -1539,10 +1539,8 @@ static int of_qcom_slim_ngd_register(struct device *parent, ngd->id = id; ngd->pdev->dev.parent = parent; - ret = driver_set_override(&ngd->pdev->dev, - &ngd->pdev->driver_override, - QCOM_SLIM_NGD_DRV_NAME, - strlen(QCOM_SLIM_NGD_DRV_NAME)); + ret = device_set_driver_override(&ngd->pdev->dev, + QCOM_SLIM_NGD_DRV_NAME); if (ret) { platform_device_put(ngd->pdev); kfree(ngd); diff --git a/drivers/soc/amlogic/meson-canvas.c b/drivers/soc/amlogic/meson-canvas.c index b6e06c4d2117f..0711088da5dcd 100644 --- a/drivers/soc/amlogic/meson-canvas.c +++ b/drivers/soc/amlogic/meson-canvas.c @@ -73,10 +73,9 @@ struct meson_canvas *meson_canvas_get(struct device *dev) * current state, this driver probe cannot return -EPROBE_DEFER */ canvas = dev_get_drvdata(&canvas_pdev->dev); - if (!canvas) { - put_device(&canvas_pdev->dev); + put_device(&canvas_pdev->dev); + if (!canvas) return ERR_PTR(-EINVAL); - } return canvas; } diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 8f29108dc69ac..5c48455185c9b 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -302,11 +302,18 @@ struct apple_mbox *apple_mbox_get(struct device *dev, int index) return ERR_PTR(-EPROBE_DEFER); mbox = platform_get_drvdata(pdev); - if (!mbox) - return ERR_PTR(-EPROBE_DEFER); + if (!mbox) { + mbox = ERR_PTR(-EPROBE_DEFER); + goto out_put_pdev; + } + + if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) { + mbox = ERR_PTR(-ENODEV); + goto out_put_pdev; + } - if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) - return ERR_PTR(-ENODEV); +out_put_pdev: + put_device(&pdev->dev); return mbox; } diff --git a/drivers/soc/aspeed/aspeed-socinfo.c b/drivers/soc/aspeed/aspeed-socinfo.c index 67e9ac3d08ecc..a90b100f4d101 100644 --- a/drivers/soc/aspeed/aspeed-socinfo.c +++ b/drivers/soc/aspeed/aspeed-socinfo.c @@ -39,7 +39,7 @@ static const char *siliconid_to_name(u32 siliconid) unsigned int i; for (i = 0 ; i < ARRAY_SIZE(rev_table) ; ++i) { - if (rev_table[i].id == id) + if ((rev_table[i].id & 0xff00ffff) == id) return rev_table[i].name; } diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c index 9be240999f877..43a4e8d58b9bc 100644 --- a/drivers/soc/fsl/qbman/qman.c +++ b/drivers/soc/fsl/qbman/qman.c @@ -1827,6 +1827,8 @@ EXPORT_SYMBOL(qman_create_fq); void qman_destroy_fq(struct qman_fq *fq) { + int leaked; + /* * We don't need to lock the FQ as it is a pre-condition that the FQ be * quiesced. Instead, run some checks. @@ -1834,11 +1836,29 @@ void qman_destroy_fq(struct qman_fq *fq) switch (fq->state) { case qman_fq_state_parked: case qman_fq_state_oos: - if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) - qman_release_fqid(fq->fqid); + /* + * There's a race condition here on releasing the fqid, + * setting the fq_table to NULL, and freeing the fqid. + * To prevent it, this order should be respected: + */ + if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) { + leaked = qman_shutdown_fq(fq->fqid); + if (leaked) + pr_debug("FQID %d leaked\n", fq->fqid); + } DPAA_ASSERT(fq_table[fq->idx]); fq_table[fq->idx] = NULL; + + if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID) && !leaked) { + /* + * fq_table[fq->idx] should be set to null before + * freeing fq->fqid otherwise it could by allocated by + * qman_alloc_fqid() while still being !NULL + */ + smp_wmb(); + gen_pool_free(qm_fqalloc, fq->fqid | DPAA_GENALLOC_OFF, 1); + } return; default: break; diff --git a/drivers/soc/fsl/qe/qmc.c b/drivers/soc/fsl/qe/qmc.c index da5ea6d356184..6db5ab05c2c1c 100644 --- a/drivers/soc/fsl/qe/qmc.c +++ b/drivers/soc/fsl/qe/qmc.c @@ -1799,8 +1799,8 @@ static int qmc_qe_init_resources(struct qmc *qmc, struct platform_device *pdev) return -EINVAL; qmc->dpram_offset = res->start - qe_muram_dma(qe_muram_addr(0)); qmc->dpram = devm_ioremap_resource(qmc->dev, res); - if (IS_ERR(qmc->scc_pram)) - return PTR_ERR(qmc->scc_pram); + if (IS_ERR(qmc->dpram)) + return PTR_ERR(qmc->dpram); return 0; } diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c index 04a1b60f2f2b5..8e2322999f099 100644 --- a/drivers/soc/imx/soc-imx8m.c +++ b/drivers/soc/imx/soc-imx8m.c @@ -148,7 +148,11 @@ static int imx8m_soc_prepare(struct platform_device *pdev, const char *ocotp_com goto err_clk; } - return clk_prepare_enable(drvdata->clk); + ret = clk_prepare_enable(drvdata->clk); + if (ret) + goto err_clk; + + return 0; err_clk: iounmap(drvdata->ocotp_base); diff --git a/drivers/soc/mediatek/mtk-svs.c b/drivers/soc/mediatek/mtk-svs.c index f45537546553e..99edecb204f25 100644 --- a/drivers/soc/mediatek/mtk-svs.c +++ b/drivers/soc/mediatek/mtk-svs.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -789,7 +790,7 @@ static ssize_t svs_enable_debug_write(struct file *filp, struct svs_bank *svsb = file_inode(filp)->i_private; struct svs_platform *svsp = dev_get_drvdata(svsb->dev); int enabled, ret; - char *buf = NULL; + char *buf __free(kfree) = NULL; if (count >= PAGE_SIZE) return -EINVAL; @@ -807,8 +808,6 @@ static ssize_t svs_enable_debug_write(struct file *filp, svsb->mode_support = SVSB_MODE_ALL_DISABLE; } - kfree(buf); - return count; } diff --git a/drivers/soc/microchip/mpfs-sys-controller.c b/drivers/soc/microchip/mpfs-sys-controller.c index 30bc45d17d343..81636cfecd37e 100644 --- a/drivers/soc/microchip/mpfs-sys-controller.c +++ b/drivers/soc/microchip/mpfs-sys-controller.c @@ -142,8 +142,10 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev) sys_controller->flash = of_get_mtd_device_by_node(np); of_node_put(np); - if (IS_ERR(sys_controller->flash)) - return dev_err_probe(dev, PTR_ERR(sys_controller->flash), "Failed to get flash\n"); + if (IS_ERR(sys_controller->flash)) { + ret = dev_err_probe(dev, PTR_ERR(sys_controller->flash), "Failed to get flash\n"); + goto out_free; + } no_flash: sys_controller->client.dev = dev; @@ -155,8 +157,7 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev) if (IS_ERR(sys_controller->chan)) { ret = dev_err_probe(dev, PTR_ERR(sys_controller->chan), "Failed to get mbox channel\n"); - kfree(sys_controller); - return ret; + goto out_free; } init_completion(&sys_controller->c); @@ -174,6 +175,10 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev) dev_info(&pdev->dev, "Registered MPFS system controller\n"); return 0; + +out_free: + kfree(sys_controller); + return ret; } static void mpfs_sys_controller_remove(struct platform_device *pdev) diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index 2caadbbcf8307..905a24b42fe69 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -124,6 +124,19 @@ config QCOM_PMIC_GLINK Say yes here to support USB-C and battery status on modern Qualcomm platforms. +config QCOM_QMI_COOLING + tristate "Qualcomm QMI cooling drivers" + depends on QCOM_RPROC_COMMON + depends on ARCH_QCOM || COMPILE_TEST + select QCOM_QMI_HELPERS + help + This enables the remote subsystem cooling devices. These cooling + devices will be used by Qualcomm chipset to place various remote + subsystem mitigations like remote processor passive mitigation, + remote subsystem voltage restriction at low temperatures etc. + The QMI cooling device will interface with remote subsystem + using Qualcomm remoteproc interface. + config QCOM_QMI_HELPERS tristate depends on NET diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index b7f1d2a573674..b6728f54944b1 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -14,6 +14,7 @@ obj-$(CONFIG_QCOM_PMIC_GLINK) += pmic_glink.o obj-$(CONFIG_QCOM_PMIC_GLINK) += pmic_glink_altmode.o obj-$(CONFIG_QCOM_PMIC_PDCHARGER_ULOG) += pmic_pdcharger_ulog.o CFLAGS_pmic_pdcharger_ulog.o := -I$(src) +obj-$(CONFIG_QCOM_QMI_COOLING) += qmi-cooling.o obj-$(CONFIG_QCOM_QMI_HELPERS) += qmi_helpers.o qmi_helpers-y += qmi_encdec.o qmi_interface.o obj-$(CONFIG_QCOM_RAMP_CTRL) += ramp_controller.o diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c index ae66c2623d250..84a75d8c4b702 100644 --- a/drivers/soc/qcom/cmd-db.c +++ b/drivers/soc/qcom/cmd-db.c @@ -349,15 +349,16 @@ static int cmd_db_dev_probe(struct platform_device *pdev) return -EINVAL; } - cmd_db_header = memremap(rmem->base, rmem->size, MEMREMAP_WC); - if (!cmd_db_header) { - ret = -ENOMEM; + cmd_db_header = devm_memremap(&pdev->dev, rmem->base, rmem->size, MEMREMAP_WC); + if (IS_ERR(cmd_db_header)) { + ret = PTR_ERR(cmd_db_header); cmd_db_header = NULL; return ret; } if (!cmd_db_magic_matches(cmd_db_header)) { dev_err(&pdev->dev, "Invalid Command DB Magic\n"); + cmd_db_header = NULL; return -EINVAL; } diff --git a/drivers/soc/qcom/icc-bwmon.c b/drivers/soc/qcom/icc-bwmon.c index 597f9025e4228..e46975da7dba6 100644 --- a/drivers/soc/qcom/icc-bwmon.c +++ b/drivers/soc/qcom/icc-bwmon.c @@ -830,7 +830,7 @@ static const struct icc_bwmon_data msm8998_bwmon_data = { static const struct icc_bwmon_data sdm845_cpu_bwmon_data = { .sample_ms = 4, .count_unit_kb = 64, - .zone1_thres_count = 16, + .zone1_thres_count = 3, .zone3_thres_count = 1, .quirks = BWMON_HAS_GLOBAL_IRQ, .regmap_fields = sdm845_cpu_bwmon_reg_fields, @@ -849,7 +849,7 @@ static const struct icc_bwmon_data sdm845_llcc_bwmon_data = { static const struct icc_bwmon_data sc7280_llcc_bwmon_data = { .sample_ms = 4, .count_unit_kb = 64, - .zone1_thres_count = 16, + .zone1_thres_count = 3, .zone3_thres_count = 1, .quirks = BWMON_NEEDS_FORCE_CLEAR, .regmap_fields = sdm845_llcc_bwmon_reg_fields, diff --git a/drivers/soc/qcom/ice.c b/drivers/soc/qcom/ice.c index c467b55b41744..9e869e6abc630 100644 --- a/drivers/soc/qcom/ice.c +++ b/drivers/soc/qcom/ice.c @@ -16,13 +16,25 @@ #include #include #include +#include #include #include #define AES_256_XTS_KEY_SIZE 64 /* for raw keys only */ -#define QCOM_ICE_HWKM_WRAPPED_KEY_SIZE 100 /* assuming HWKM v2 */ + +#define QCOM_ICE_HWKM_V1 1 /* HWKM version 1 */ +#define QCOM_ICE_HWKM_V2 2 /* HWKM version 2 */ + +#define QCOM_ICE_HWKM_MAX_WRAPPED_KEY_SIZE 100 /* Maximum HWKM wrapped key size */ + +/* + * Wrapped key size depends upon HWKM version: + * HWKM version 1 supports 68 bytes + * HWKM version 2 supports 100 bytes + */ +#define QCOM_ICE_HWKM_WRAPPED_KEY_SIZE(v) ((v) == QCOM_ICE_HWKM_V1 ? 68 : 100) /* QCOM ICE registers */ @@ -62,13 +74,15 @@ union crypto_cfg { #define QCOM_ICE_REG_HWKM_TZ_KM_CTL (HWKM_OFFSET + 0x1000) #define QCOM_ICE_HWKM_DISABLE_CRC_CHECKS_VAL (BIT(1) | BIT(2)) +/* In HWKM v1 the ICE legacy mode is controlled from HWKM register space */ +#define QCOM_ICE_HWKM_ICE_LEGACY_MODE_ENABLED BIT(5) #define QCOM_ICE_REG_HWKM_TZ_KM_STATUS (HWKM_OFFSET + 0x1004) #define QCOM_ICE_HWKM_KT_CLEAR_DONE BIT(0) #define QCOM_ICE_HWKM_BOOT_CMD_LIST0_DONE BIT(1) #define QCOM_ICE_HWKM_BOOT_CMD_LIST1_DONE BIT(2) -#define QCOM_ICE_HWKM_CRYPTO_BIST_DONE_V2 BIT(7) -#define QCOM_ICE_HWKM_BIST_DONE_V2 BIT(9) +#define QCOM_ICE_HWKM_CRYPTO_BIST_DONE(v) (((v) == QCOM_ICE_HWKM_V1) ? BIT(14) : BIT(7)) +#define QCOM_ICE_HWKM_BIST_DONE(v) (((v) == QCOM_ICE_HWKM_V1) ? BIT(16) : BIT(9)) #define QCOM_ICE_REG_HWKM_BANK0_BANKN_IRQ_STATUS (HWKM_OFFSET + 0x2008) #define QCOM_ICE_HWKM_RSP_FIFO_CLEAR_VAL BIT(3) @@ -95,8 +109,12 @@ struct qcom_ice { void __iomem *base; struct clk *core_clk; + struct clk *iface_clk; bool use_hwkm; bool hwkm_init_complete; + u8 hwkm_version; + unsigned long core_clk_freq; + bool has_opp; }; static bool qcom_ice_check_supported(struct qcom_ice *ice) @@ -114,9 +132,24 @@ static bool qcom_ice_check_supported(struct qcom_ice *ice) return false; } + /* HWKM version v2 is present from ICE 3.2.1 onwards while version v1 + * is present only in ICE 3.2.0. Earlier ICE version don't have HWKM. + */ + if (major > 3 || + (major == 3 && (minor >= 3 || (minor == 2 && step >= 1)))) + ice->hwkm_version = QCOM_ICE_HWKM_V2; + else if ((major == 3) && (minor == 2)) + ice->hwkm_version = QCOM_ICE_HWKM_V1; + else + ice->hwkm_version = 0; + dev_info(dev, "Found QC Inline Crypto Engine (ICE) v%d.%d.%d\n", major, minor, step); + if (ice->hwkm_version) + dev_info(dev, "QC Hardware Key Manager (HWKM) version v%d\n", + ice->hwkm_version); + /* If fuses are blown, ICE might not work in the standard way. */ regval = qcom_ice_readl(ice, QCOM_ICE_REG_FUSE_SETTING); if (regval & (QCOM_ICE_FUSE_SETTING_MASK | @@ -131,19 +164,18 @@ static bool qcom_ice_check_supported(struct qcom_ice *ice) * v3.2.1 and later have HWKM v2. ICE v3.2.0 has HWKM v1. Earlier ICE * versions don't have HWKM at all. However, for HWKM to be fully * usable by Linux, the TrustZone software also needs to support certain - * SCM calls including the ones to generate and prepare keys. That - * effectively makes the earliest supported SoC be SM8650, which has - * HWKM v2. Therefore, this driver doesn't include support for HWKM v1, - * and it checks for the SCM call support before it decides to use HWKM. + * SCM calls including the ones to generate and prepare keys. Support + * for these SCM calls is present for SoCs with HWKM v2 and is being + * added for SoCs with HWKM v1 as well but not every SoC with HWKM v1 + * currently supports this. So, this driver checks for the SCM call + * support before it decides to use HWKM. * * Also, since HWKM and legacy mode are mutually exclusive, and * ICE-capable storage driver(s) need to know early on whether to * advertise support for raw keys or wrapped keys, HWKM cannot be used * unconditionally. A module parameter is used to opt into using it. */ - if ((major >= 4 || - (major == 3 && (minor >= 3 || (minor == 2 && step >= 1)))) && - qcom_scm_has_wrapped_key_support()) { + if (ice->hwkm_version && qcom_scm_has_wrapped_key_support()) { if (qcom_ice_use_wrapped_keys) { dev_info(dev, "Using HWKM. Supporting wrapped keys only.\n"); ice->use_hwkm = true; @@ -212,8 +244,8 @@ static int qcom_ice_wait_bist_status(struct qcom_ice *ice) (QCOM_ICE_HWKM_KT_CLEAR_DONE | QCOM_ICE_HWKM_BOOT_CMD_LIST0_DONE | QCOM_ICE_HWKM_BOOT_CMD_LIST1_DONE | - QCOM_ICE_HWKM_CRYPTO_BIST_DONE_V2 | - QCOM_ICE_HWKM_BIST_DONE_V2)) { + QCOM_ICE_HWKM_CRYPTO_BIST_DONE(ice->hwkm_version) | + QCOM_ICE_HWKM_BIST_DONE(ice->hwkm_version))) { dev_err(ice->dev, "HWKM self-test error!\n"); /* * Too late to revoke use_hwkm here, as it was already @@ -230,7 +262,7 @@ static void qcom_ice_hwkm_init(struct qcom_ice *ice) if (!ice->use_hwkm) return; - BUILD_BUG_ON(QCOM_ICE_HWKM_WRAPPED_KEY_SIZE > + BUILD_BUG_ON(QCOM_ICE_HWKM_MAX_WRAPPED_KEY_SIZE > BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE); /* * When ICE is in HWKM mode, it only supports wrapped keys. @@ -238,9 +270,15 @@ static void qcom_ice_hwkm_init(struct qcom_ice *ice) * * Put ICE in HWKM mode. ICE defaults to legacy mode. */ - regval = qcom_ice_readl(ice, QCOM_ICE_REG_CONTROL); - regval &= ~QCOM_ICE_LEGACY_MODE_ENABLED; - qcom_ice_writel(ice, regval, QCOM_ICE_REG_CONTROL); + if (ice->hwkm_version == QCOM_ICE_HWKM_V2) { + regval = qcom_ice_readl(ice, QCOM_ICE_REG_CONTROL); + regval &= ~QCOM_ICE_LEGACY_MODE_ENABLED; + qcom_ice_writel(ice, regval, QCOM_ICE_REG_CONTROL); + } else if (ice->hwkm_version == QCOM_ICE_HWKM_V1) { + regval = qcom_ice_readl(ice, QCOM_ICE_REG_HWKM_TZ_KM_CTL); + regval &= ~QCOM_ICE_HWKM_ICE_LEGACY_MODE_ENABLED; + qcom_ice_writel(ice, regval, QCOM_ICE_REG_HWKM_TZ_KM_CTL); + } /* Disable CRC checks. This HWKM feature is not used. */ qcom_ice_writel(ice, QCOM_ICE_HWKM_DISABLE_CRC_CHECKS_VAL, @@ -276,10 +314,19 @@ int qcom_ice_resume(struct qcom_ice *ice) struct device *dev = ice->dev; int err; + /* Restore the ICE core clk freq */ + if (ice->has_opp && ice->core_clk_freq) + dev_pm_opp_set_rate(ice->dev, ice->core_clk_freq); + err = clk_prepare_enable(ice->core_clk); if (err) { - dev_err(dev, "failed to enable core clock (%d)\n", - err); + dev_err(dev, "Failed to enable core clock: %d\n", err); + return err; + } + + err = clk_prepare_enable(ice->iface_clk); + if (err) { + dev_err(dev, "Failed to enable iface clock: %d\n", err); return err; } qcom_ice_hwkm_init(ice); @@ -289,7 +336,13 @@ EXPORT_SYMBOL_GPL(qcom_ice_resume); int qcom_ice_suspend(struct qcom_ice *ice) { + clk_disable_unprepare(ice->iface_clk); clk_disable_unprepare(ice->core_clk); + + /* Drop the clock votes while suspend */ + if (ice->has_opp) + dev_pm_opp_set_rate(ice->dev, 0); + ice->hwkm_init_complete = false; return 0; @@ -298,7 +351,7 @@ EXPORT_SYMBOL_GPL(qcom_ice_suspend); static unsigned int translate_hwkm_slot(struct qcom_ice *ice, unsigned int slot) { - return slot * 2; + return ice->hwkm_version == QCOM_ICE_HWKM_V1 ? slot : slot * 2; } static int qcom_ice_program_wrapped_key(struct qcom_ice *ice, unsigned int slot, @@ -451,11 +504,12 @@ int qcom_ice_generate_key(struct qcom_ice *ice, { int err; - err = qcom_scm_generate_ice_key(lt_key, QCOM_ICE_HWKM_WRAPPED_KEY_SIZE); + err = qcom_scm_generate_ice_key(lt_key, + QCOM_ICE_HWKM_WRAPPED_KEY_SIZE(ice->hwkm_version)); if (err) return err; - return QCOM_ICE_HWKM_WRAPPED_KEY_SIZE; + return QCOM_ICE_HWKM_WRAPPED_KEY_SIZE(ice->hwkm_version); } EXPORT_SYMBOL_GPL(qcom_ice_generate_key); @@ -478,13 +532,13 @@ int qcom_ice_prepare_key(struct qcom_ice *ice, int err; err = qcom_scm_prepare_ice_key(lt_key, lt_key_size, - eph_key, QCOM_ICE_HWKM_WRAPPED_KEY_SIZE); + eph_key, QCOM_ICE_HWKM_WRAPPED_KEY_SIZE(ice->hwkm_version)); if (err == -EIO || err == -EINVAL) err = -EBADMSG; /* probably invalid key */ if (err) return err; - return QCOM_ICE_HWKM_WRAPPED_KEY_SIZE; + return QCOM_ICE_HWKM_WRAPPED_KEY_SIZE(ice->hwkm_version); } EXPORT_SYMBOL_GPL(qcom_ice_prepare_key); @@ -506,14 +560,59 @@ int qcom_ice_import_key(struct qcom_ice *ice, int err; err = qcom_scm_import_ice_key(raw_key, raw_key_size, - lt_key, QCOM_ICE_HWKM_WRAPPED_KEY_SIZE); + lt_key, QCOM_ICE_HWKM_WRAPPED_KEY_SIZE(ice->hwkm_version)); if (err) return err; - return QCOM_ICE_HWKM_WRAPPED_KEY_SIZE; + return QCOM_ICE_HWKM_WRAPPED_KEY_SIZE(ice->hwkm_version); } EXPORT_SYMBOL_GPL(qcom_ice_import_key); +/** + * qcom_ice_scale_clk() - Scale ICE clock for DVFS-aware operations + * @ice: ICE driver data + * @target_freq: requested frequency in Hz + * @round_ceil: when true, selects nearest freq >= @target_freq; + * otherwise, selects nearest freq <= @target_freq + * + * Selects an OPP frequency based on @target_freq and the rounding direction + * specified by @round_ceil, then programs it using dev_pm_opp_set_rate(), + * including any voltage or power-domain transitions handled by the OPP + * framework. Updates ice->core_clk_freq on success. + * + * Return: 0 on success; -EOPNOTSUPP if no OPP table; or error from + * dev_pm_opp_set_rate()/OPP lookup. + */ +int qcom_ice_scale_clk(struct qcom_ice *ice, unsigned long target_freq, + bool round_ceil) +{ + unsigned long ice_freq = target_freq; + struct dev_pm_opp *opp; + int ret; + + if (!ice->has_opp) + return -EOPNOTSUPP; + + if (round_ceil) + opp = dev_pm_opp_find_freq_ceil(ice->dev, &ice_freq); + else + opp = dev_pm_opp_find_freq_floor(ice->dev, &ice_freq); + + if (IS_ERR(opp)) + return PTR_ERR(opp); + dev_pm_opp_put(opp); + + ret = dev_pm_opp_set_rate(ice->dev, ice_freq); + if (ret) { + dev_err(ice->dev, "Unable to scale ICE clock rate\n"); + return ret; + } + ice->core_clk_freq = ice_freq; + + return ret; +} +EXPORT_SYMBOL_GPL(qcom_ice_scale_clk); + static struct qcom_ice *qcom_ice_create(struct device *dev, void __iomem *base) { @@ -544,11 +643,17 @@ static struct qcom_ice *qcom_ice_create(struct device *dev, engine->core_clk = devm_clk_get_optional_enabled(dev, "ice_core_clk"); if (!engine->core_clk) engine->core_clk = devm_clk_get_optional_enabled(dev, "ice"); + if (!engine->core_clk) + engine->core_clk = devm_clk_get_optional_enabled(dev, "core"); if (!engine->core_clk) engine->core_clk = devm_clk_get_enabled(dev, NULL); if (IS_ERR(engine->core_clk)) return ERR_CAST(engine->core_clk); + engine->iface_clk = devm_clk_get_optional_enabled(dev, "iface"); + if (IS_ERR(engine->iface_clk)) + return ERR_CAST(engine->iface_clk); + if (!qcom_ice_check_supported(engine)) return ERR_PTR(-EOPNOTSUPP); @@ -683,6 +788,7 @@ static int qcom_ice_probe(struct platform_device *pdev) { struct qcom_ice *engine; void __iomem *base; + int err; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) { @@ -694,6 +800,40 @@ static int qcom_ice_probe(struct platform_device *pdev) if (IS_ERR(engine)) return PTR_ERR(engine); + /* qcom_ice_create() may return NULL if scm calls are not available */ + if (!engine) + return -EOPNOTSUPP; + + err = devm_pm_opp_set_clkname(&pdev->dev, "core"); + if (err && err != -ENOENT) { + dev_err(&pdev->dev, "Unable to set core clkname to OPP-table\n"); + return err; + } + + /* OPP table is optional */ + err = devm_pm_opp_of_add_table(&pdev->dev); + if (err && err != -ENODEV) { + dev_err(&pdev->dev, "Invalid OPP table in Device tree\n"); + return err; + } + + /* + * The OPP table is optional. devm_pm_opp_of_add_table() returns + * -ENODEV when no OPP table is present in DT, which is not treated + * as an error. Therefore, track successful OPP registration only + * when the return value is 0. + */ + engine->has_opp = (err == 0); + if (!engine->has_opp) + dev_info(&pdev->dev, "ICE OPP table is not registered, please update your DT\n"); + + /* + * Store the core clock rate for suspend resume cycles, + * against OPP aware DVFS operations. core_clk_freq will + * have a valid value only for non-legacy bindings. + */ + engine->core_clk_freq = clk_get_rate(engine->core_clk); + platform_set_drvdata(pdev, engine); return 0; diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c index 857ead56b37d0..13e1742672945 100644 --- a/drivers/soc/qcom/llcc-qcom.c +++ b/drivers/soc/qcom/llcc-qcom.c @@ -214,6 +214,364 @@ static const struct llcc_slice_config ipq5424_data[] = { }, }; +static const struct llcc_slice_config kaanapali_data[] = { + { + .usecase_id = LLCC_CPUSS, + .slice_id = 1, + .max_cap = 5120, + .priority = 1, + .bonus_ways = 0xffffffff, + .activate_on_init = true, + .write_scid_en = true, + .stale_en = true, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_VIDSC0, + .slice_id = 2, + .max_cap = 512, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_AUDIO, + .slice_id = 35, + .max_cap = 512, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_MDMHPGRW, + .slice_id = 25, + .max_cap = 1024, + .priority = 5, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_CMPT, + .slice_id = 34, + .max_cap = 4096, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_GPUHTW, + .slice_id = 11, + .max_cap = 512, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_GPU, + .slice_id = 9, + .max_cap = 5632, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .write_scid_cacheable_en = true, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_MMUHWT, + .slice_id = 18, + .max_cap = 768, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .activate_on_init = true, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_DISP, + .slice_id = 16, + .max_cap = 7168, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .cache_mode = 2, + .stale_en = true, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_MDMHPFX, + .slice_id = 24, + .max_cap = 1024, + .priority = 5, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_MDMPNG, + .slice_id = 27, + .max_cap = 256, + .priority = 5, + .bonus_ways = 0xfffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_CVP, + .slice_id = 8, + .max_cap = 800, + .priority = 5, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_MODPE, + .slice_id = 29, + .max_cap = 256, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xf0000000, + .mru_uncap_en = true, + .alloc_oneway_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_WRCACHE, + .slice_id = 31, + .max_cap = 512, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .activate_on_init = true, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_CVPFW, + .slice_id = 19, + .max_cap = 512, + .priority = 5, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_CPUMTE, + .slice_id = 7, + .max_cap = 256, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_CMPTHCP, + .slice_id = 15, + .max_cap = 256, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_LCPDARE, + .slice_id = 30, + .max_cap = 128, + .priority = 5, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .activate_on_init = true, + .mru_uncap_en = true, + .alloc_oneway_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_AENPU, + .slice_id = 3, + .max_cap = 3072, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .cache_mode = 2, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_ISLAND1, + .slice_id = 12, + .max_cap = 7936, + .priority = 7, + .fixed_size = true, + .bonus_ways = 0x7fffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_DISP_WB, + .slice_id = 23, + .max_cap = 512, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_VIDVSP, + .slice_id = 4, + .max_cap = 256, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_VIDDEC, + .slice_id = 5, + .max_cap = 512, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .cache_mode = 2, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_CAMOFE, + .slice_id = 33, + .max_cap = 6144, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .stale_en = true, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_CAMRTIP, + .slice_id = 13, + .max_cap = 6144, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .stale_en = true, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_CAMRTRF, + .slice_id = 10, + .max_cap = 3584, + .priority = 3, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .stale_en = true, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_CAMSRTRF, + .slice_id = 21, + .max_cap = 6144, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .stale_en = true, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_VIDEO_APV, + .slice_id = 6, + .max_cap = 768, + .priority = 4, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_COMPUTE1, + .slice_id = 22, + .max_cap = 4096, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_CPUSS_OPP, + .slice_id = 32, + .max_cap = 0, + .priority = 0, + .fixed_size = true, + .bonus_ways = 0, + .activate_on_init = true, + .write_scid_en = true, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_CPUSSMPAM, + .slice_id = 17, + .max_cap = 2048, + .priority = 1, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .activate_on_init = true, + .write_scid_en = true, + .stale_en = true, + .mru_uncap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_CAM_IPE_STROV, + .slice_id = 14, + .max_cap = 400, + .priority = 5, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_CAM_OFE_STROV, + .slice_id = 20, + .max_cap = 400, + .priority = 5, + .fixed_size = true, + .bonus_ways = 0xffffffff, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + .parent_slice_id = 33, + }, { + .usecase_id = LLCC_CPUSS_HEU, + .slice_id = 28, + .max_cap = 0, + .priority = 0, + .fixed_size = true, + .bonus_ways = 0, + .mru_uncap_en = true, + .ovcap_en = true, + .vict_prio = true, + }, { + .usecase_id = LLCC_MDM_PNG_FIXED, + .slice_id = 26, + .max_cap = 256, + .priority = 5, + .fixed_size = true, + .bonus_ways = 0xff000000, + .activate_on_init = true, + .write_scid_en = true, + .mru_uncap_en = true, + .vict_prio = true, + }, +}; + static const struct llcc_slice_config sa8775p_data[] = { { .usecase_id = LLCC_CPUSS, @@ -3505,6 +3863,15 @@ static const u32 llcc_v6_reg_offset[] = { [LLCC_TRP_WRS_CACHEABLE_EN] = 0x00042088, }; +static const struct qcom_llcc_config kaanapali_cfg[] = { + { + .sct_data = kaanapali_data, + .size = ARRAY_SIZE(kaanapali_data), + .reg_offset = llcc_v6_reg_offset, + .edac_reg_offset = &llcc_v6_edac_reg_offset, + }, +}; + static const struct qcom_llcc_config qcs615_cfg[] = { { .sct_data = qcs615_data, @@ -3731,6 +4098,11 @@ static const struct qcom_llcc_config x1e80100_cfg[] = { }, }; +static const struct qcom_sct_config kaanapali_cfgs = { + .llcc_config = kaanapali_cfg, + .num_config = ARRAY_SIZE(kaanapali_cfg), +}; + static const struct qcom_sct_config qcs615_cfgs = { .llcc_config = qcs615_cfg, .num_config = ARRAY_SIZE(qcs615_cfg), @@ -4570,6 +4942,7 @@ static int qcom_llcc_probe(struct platform_device *pdev) static const struct of_device_id qcom_llcc_of_match[] = { { .compatible = "qcom,ipq5424-llcc", .data = &ipq5424_cfgs}, + { .compatible = "qcom,kaanapali-llcc", .data = &kaanapali_cfgs}, { .compatible = "qcom,qcs615-llcc", .data = &qcs615_cfgs}, { .compatible = "qcom,qcs8300-llcc", .data = &qcs8300_cfgs}, { .compatible = "qcom,qdu1000-llcc", .data = &qdu1000_cfgs}, diff --git a/drivers/soc/qcom/mdt_loader.c b/drivers/soc/qcom/mdt_loader.c index a5c80d4fcc366..3936839da329e 100644 --- a/drivers/soc/qcom/mdt_loader.c +++ b/drivers/soc/qcom/mdt_loader.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -227,20 +228,9 @@ void *qcom_mdt_read_metadata(const struct firmware *fw, size_t *data_len, } EXPORT_SYMBOL_GPL(qcom_mdt_read_metadata); -/** - * qcom_mdt_pas_init() - initialize PAS region for firmware loading - * @dev: device handle to associate resources with - * @fw: firmware object for the mdt file - * @fw_name: name of the firmware, for construction of segment file names - * @pas_id: PAS identifier - * @mem_phys: physical address of allocated memory region - * @ctx: PAS metadata context, to be released by caller - * - * Returns 0 on success, negative errno otherwise. - */ -int qcom_mdt_pas_init(struct device *dev, const struct firmware *fw, - const char *fw_name, int pas_id, phys_addr_t mem_phys, - struct qcom_scm_pas_metadata *ctx) +static int __qcom_mdt_pas_init(struct device *dev, const struct firmware *fw, + const char *fw_name, int pas_id, phys_addr_t mem_phys, + struct qcom_scm_pas_context *ctx) { const struct elf32_phdr *phdrs; const struct elf32_phdr *phdr; @@ -302,7 +292,6 @@ int qcom_mdt_pas_init(struct device *dev, const struct firmware *fw, out: return ret; } -EXPORT_SYMBOL_GPL(qcom_mdt_pas_init); static bool qcom_mdt_bins_are_split(const struct firmware *fw) { @@ -456,7 +445,7 @@ int qcom_mdt_load(struct device *dev, const struct firmware *fw, { int ret; - ret = qcom_mdt_pas_init(dev, fw, firmware, pas_id, mem_phys, NULL); + ret = __qcom_mdt_pas_init(dev, fw, firmware, pas_id, mem_phys, NULL); if (ret) return ret; @@ -486,5 +475,45 @@ int qcom_mdt_load_no_init(struct device *dev, const struct firmware *fw, } EXPORT_SYMBOL_GPL(qcom_mdt_load_no_init); +/** + * qcom_mdt_pas_load - Loads and authenticates the metadata of the firmware + * (typically contained in the .mdt file), followed by loading the actual + * firmware segments (e.g., .bXX files). Authentication of the segments done + * by a separate call. + * + * The PAS context must be initialized using qcom_scm_pas_context_init() + * prior to invoking this function. + * + * @ctx: Pointer to the PAS (Peripheral Authentication Service) context + * @fw: Firmware object representing the .mdt file + * @firmware: Name of the firmware used to construct segment file names + * @reloc_base: Physical address adjusted after relocation + * + * Return: 0 on success or a negative error code on failure. + */ +int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx, const struct firmware *fw, + const char *firmware, phys_addr_t *reloc_base) +{ + void *mem_region; + int ret; + + ret = __qcom_mdt_pas_init(ctx->dev, fw, firmware, ctx->pas_id, ctx->mem_phys, ctx); + if (ret) + return ret; + + mem_region = ioremap_wc(ctx->mem_phys, ctx->mem_size); + if (!mem_region) { + dev_err(ctx->dev, "unable to map memory region: %pa+%zx\n", &ctx->mem_phys, + ctx->mem_size); + return -EINVAL; + } + + ret = qcom_mdt_load_no_init(ctx->dev, fw, firmware, mem_region, ctx->mem_phys, + ctx->mem_size, reloc_base); + iounmap(mem_region); + return ret; +} +EXPORT_SYMBOL_GPL(qcom_mdt_pas_load); + MODULE_DESCRIPTION("Firmware parser for Qualcomm MDT format"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/soc/qcom/ocmem.c b/drivers/soc/qcom/ocmem.c index 9c3bd37b6579d..71130a2f62e9e 100644 --- a/drivers/soc/qcom/ocmem.c +++ b/drivers/soc/qcom/ocmem.c @@ -202,9 +202,9 @@ struct ocmem *of_get_ocmem(struct device *dev) } ocmem = platform_get_drvdata(pdev); + put_device(&pdev->dev); if (!ocmem) { dev_err(dev, "Cannot get ocmem\n"); - put_device(&pdev->dev); return ERR_PTR(-ENODEV); } return ocmem; diff --git a/drivers/soc/qcom/pdr_internal.h b/drivers/soc/qcom/pdr_internal.h index 039508c1bbf7d..047c0160b6178 100644 --- a/drivers/soc/qcom/pdr_internal.h +++ b/drivers/soc/qcom/pdr_internal.h @@ -84,7 +84,7 @@ struct servreg_set_ack_resp { struct servreg_loc_pfr_req { char service[SERVREG_NAME_LENGTH + 1]; - char reason[257]; + char reason[SERVREG_PFR_LENGTH + 1]; }; struct servreg_loc_pfr_resp { diff --git a/drivers/soc/qcom/pmic_glink.c b/drivers/soc/qcom/pmic_glink.c index c0a4be5df9267..13bbc9a5181d2 100644 --- a/drivers/soc/qcom/pmic_glink.c +++ b/drivers/soc/qcom/pmic_glink.c @@ -23,13 +23,19 @@ enum { PMIC_GLINK_CLIENT_UCSI, }; +struct pmic_glink_data { + unsigned long client_mask; + char *charger_pdr_service_name; + char *charger_pdr_service_path; +}; + struct pmic_glink { struct device *dev; struct pdr_handle *pdr; struct rpmsg_endpoint *ept; - unsigned long client_mask; + const struct pmic_glink_data *data; struct auxiliary_device altmode_aux; struct auxiliary_device ps_aux; @@ -39,6 +45,7 @@ struct pmic_glink { struct mutex state_lock; unsigned int client_state; unsigned int pdr_state; + bool pdr_available; /* serializing clients list updates */ spinlock_t client_lock; @@ -246,9 +253,12 @@ static int pmic_glink_rpmsg_probe(struct rpmsg_device *rpdev) return dev_err_probe(&rpdev->dev, -ENODEV, "no pmic_glink device to attach to\n"); dev_set_drvdata(&rpdev->dev, pg); + pg->pdr_available = rpdev->id.driver_data; guard(mutex)(&pg->state_lock); pg->ept = rpdev->ept; + if (!pg->pdr_available) + pg->pdr_state = SERVREG_SERVICE_STATE_UP; pmic_glink_state_notify_clients(pg); return 0; @@ -265,11 +275,14 @@ static void pmic_glink_rpmsg_remove(struct rpmsg_device *rpdev) guard(mutex)(&pg->state_lock); pg->ept = NULL; + if (!pg->pdr_available) + pg->pdr_state = SERVREG_SERVICE_STATE_DOWN; pmic_glink_state_notify_clients(pg); } static const struct rpmsg_device_id pmic_glink_rpmsg_id_match[] = { - { "PMIC_RTR_ADSP_APPS" }, + {.name = "PMIC_RTR_ADSP_APPS", .driver_data = true }, + {.name = "PMIC_RTR_SOCCP_APPS", .driver_data = false }, {} }; @@ -285,7 +298,6 @@ static struct rpmsg_driver pmic_glink_rpmsg_driver = { static int pmic_glink_probe(struct platform_device *pdev) { - const unsigned long *match_data; struct pdr_service *service; struct pmic_glink *pg; int ret; @@ -302,12 +314,10 @@ static int pmic_glink_probe(struct platform_device *pdev) spin_lock_init(&pg->client_lock); mutex_init(&pg->state_lock); - match_data = (unsigned long *)of_device_get_match_data(&pdev->dev); - if (!match_data) + pg->data = of_device_get_match_data(&pdev->dev); + if (!pg->data) return -EINVAL; - pg->client_mask = *match_data; - pg->pdr = pdr_handle_alloc(pmic_glink_pdr_callback, pg); if (IS_ERR(pg->pdr)) { ret = dev_err_probe(&pdev->dev, PTR_ERR(pg->pdr), @@ -315,27 +325,30 @@ static int pmic_glink_probe(struct platform_device *pdev) return ret; } - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) { + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) { ret = pmic_glink_add_aux_device(pg, &pg->ucsi_aux, "ucsi"); if (ret) goto out_release_pdr_handle; } - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) { + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) { ret = pmic_glink_add_aux_device(pg, &pg->altmode_aux, "altmode"); if (ret) goto out_release_ucsi_aux; } - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) { + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) { ret = pmic_glink_add_aux_device(pg, &pg->ps_aux, "power-supply"); if (ret) goto out_release_altmode_aux; } - service = pdr_add_lookup(pg->pdr, "tms/servreg", "msm/adsp/charger_pd"); - if (IS_ERR(service)) { - ret = dev_err_probe(&pdev->dev, PTR_ERR(service), - "failed adding pdr lookup for charger_pd\n"); - goto out_release_aux_devices; + if (pg->data->charger_pdr_service_name && pg->data->charger_pdr_service_path) { + service = pdr_add_lookup(pg->pdr, pg->data->charger_pdr_service_name, + pg->data->charger_pdr_service_path); + if (IS_ERR(service)) { + ret = dev_err_probe(&pdev->dev, PTR_ERR(service), + "failed adding pdr lookup for charger_pd\n"); + goto out_release_aux_devices; + } } mutex_lock(&__pmic_glink_lock); @@ -345,13 +358,13 @@ static int pmic_glink_probe(struct platform_device *pdev) return 0; out_release_aux_devices: - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) pmic_glink_del_aux_device(pg, &pg->ps_aux); out_release_altmode_aux: - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) pmic_glink_del_aux_device(pg, &pg->altmode_aux); out_release_ucsi_aux: - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) pmic_glink_del_aux_device(pg, &pg->ucsi_aux); out_release_pdr_handle: pdr_handle_release(pg->pdr); @@ -365,23 +378,35 @@ static void pmic_glink_remove(struct platform_device *pdev) pdr_handle_release(pg->pdr); - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) pmic_glink_del_aux_device(pg, &pg->ps_aux); - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) pmic_glink_del_aux_device(pg, &pg->altmode_aux); - if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) + if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) pmic_glink_del_aux_device(pg, &pg->ucsi_aux); guard(mutex)(&__pmic_glink_lock); __pmic_glink = NULL; } -static const unsigned long pmic_glink_sm8450_client_mask = BIT(PMIC_GLINK_CLIENT_BATT) | - BIT(PMIC_GLINK_CLIENT_ALTMODE) | - BIT(PMIC_GLINK_CLIENT_UCSI); +static const struct pmic_glink_data pmic_glink_sm8450_data = { + .client_mask = BIT(PMIC_GLINK_CLIENT_BATT) | + BIT(PMIC_GLINK_CLIENT_ALTMODE) | + BIT(PMIC_GLINK_CLIENT_UCSI), + .charger_pdr_service_name = "tms/servreg", + .charger_pdr_service_path = "msm/adsp/charger_pd", +}; + +static const struct pmic_glink_data pmic_glink_kaanapali_data = { + .client_mask = BIT(PMIC_GLINK_CLIENT_BATT) | + BIT(PMIC_GLINK_CLIENT_ALTMODE) | + BIT(PMIC_GLINK_CLIENT_UCSI), +}; static const struct of_device_id pmic_glink_of_match[] = { - { .compatible = "qcom,pmic-glink", .data = &pmic_glink_sm8450_client_mask }, + { .compatible = "qcom,glymur-pmic-glink", .data = &pmic_glink_kaanapali_data }, + { .compatible = "qcom,kaanapali-pmic-glink", .data = &pmic_glink_kaanapali_data }, + { .compatible = "qcom,pmic-glink", .data = &pmic_glink_sm8450_data }, {} }; MODULE_DEVICE_TABLE(of, pmic_glink_of_match); diff --git a/drivers/soc/qcom/qcom-geni-se.c b/drivers/soc/qcom/qcom-geni-se.c index cd1779b6a91a7..1a60832ace168 100644 --- a/drivers/soc/qcom/qcom-geni-se.c +++ b/drivers/soc/qcom/qcom-geni-se.c @@ -597,10 +597,17 @@ int geni_se_resources_off(struct geni_se *se) if (has_acpi_companion(se->dev)) return 0; - - ret = pinctrl_pm_select_sleep_state(se->dev); - if (ret) - return ret; + /* + * Select the "sleep" pinctrl state only when the serial engine is + * exclusively owned by this system processor. For shared controller + * configurations, another system processor may still be using the pins, + * and switching them to "sleep" can disrupt ongoing transfers. + */ + if (!se->multi_owner) { + ret = pinctrl_pm_select_sleep_state(se->dev); + if (ret) + return ret; + } geni_se_clks_off(se); return 0; diff --git a/drivers/soc/qcom/qcom-pbs.c b/drivers/soc/qcom/qcom-pbs.c index 1cc5d045f9ddf..06b4a596e2757 100644 --- a/drivers/soc/qcom/qcom-pbs.c +++ b/drivers/soc/qcom/qcom-pbs.c @@ -173,6 +173,8 @@ struct pbs_dev *get_pbs_client_device(struct device *dev) return ERR_PTR(-EINVAL); } + platform_device_put(pdev); + return pbs; } EXPORT_SYMBOL_GPL(get_pbs_client_device); diff --git a/drivers/soc/qcom/qcom_gsbi.c b/drivers/soc/qcom/qcom_gsbi.c index 8f1158e0c6313..a25d1de592f06 100644 --- a/drivers/soc/qcom/qcom_gsbi.c +++ b/drivers/soc/qcom/qcom_gsbi.c @@ -212,13 +212,6 @@ static int gsbi_probe(struct platform_device *pdev) return of_platform_populate(node, NULL, NULL, &pdev->dev); } -static void gsbi_remove(struct platform_device *pdev) -{ - struct gsbi_info *gsbi = platform_get_drvdata(pdev); - - clk_disable_unprepare(gsbi->hclk); -} - static const struct of_device_id gsbi_dt_match[] = { { .compatible = "qcom,gsbi-v1.0.0", }, { }, @@ -232,7 +225,6 @@ static struct platform_driver gsbi_driver = { .of_match_table = gsbi_dt_match, }, .probe = gsbi_probe, - .remove = gsbi_remove, }; module_platform_driver(gsbi_driver); diff --git a/drivers/soc/qcom/qcom_pd_mapper.c b/drivers/soc/qcom/qcom_pd_mapper.c index 6384f271953d4..524e933c388c8 100644 --- a/drivers/soc/qcom/qcom_pd_mapper.c +++ b/drivers/soc/qcom/qcom_pd_mapper.c @@ -305,6 +305,18 @@ static const struct qcom_pdm_domain_data cdsp_root_pd = { .services = { NULL }, }; +static const struct qcom_pdm_domain_data gpdsp_root_pd = { + .domain = "msm/gpdsp/root_pd", + .instance_id = 192, + .services = { NULL }, +}; + +static const struct qcom_pdm_domain_data gpdsp1_root_pd = { + .domain = "msm/gpdsp1/root_pd", + .instance_id = 241, + .services = { NULL }, +}; + static const struct qcom_pdm_domain_data slpi_root_pd = { .domain = "msm/slpi/root_pd", .instance_id = 90, @@ -392,6 +404,32 @@ static const struct qcom_pdm_domain_data *qcs404_domains[] = { NULL, }; +static const struct qcom_pdm_domain_data *qcs615_domains[] = { + &adsp_audio_pd, + &adsp_root_pd, + &adsp_sensor_pd, + &cdsp_root_pd, + &mpss_root_pd, + &mpss_wlan_pd, + NULL, +}; + +static const struct qcom_pdm_domain_data *qcs8300_domains[] = { + &adsp_audio_pd, + &adsp_root_pd, + &cdsp_root_pd, + NULL, +}; + +static const struct qcom_pdm_domain_data *sa8775p_domains[] = { + &adsp_audio_pd, + &adsp_root_pd, + &cdsp_root_pd, + &gpdsp_root_pd, + &gpdsp1_root_pd, + NULL, +}; + static const struct qcom_pdm_domain_data *sc7180_domains[] = { &adsp_audio_pd, &adsp_root_pd_pdr, @@ -562,6 +600,9 @@ static const struct of_device_id qcom_pdm_domains[] __maybe_unused = { { .compatible = "qcom,qcm2290", .data = qcm2290_domains, }, { .compatible = "qcom,qcm6490", .data = sc7280_domains, }, { .compatible = "qcom,qcs404", .data = qcs404_domains, }, + { .compatible = "qcom,qcs615", .data = qcs615_domains, }, + { .compatible = "qcom,qcs8300", .data = qcs8300_domains, }, + { .compatible = "qcom,sa8775p", .data = sa8775p_domains, }, { .compatible = "qcom,sc7180", .data = sc7180_domains, }, { .compatible = "qcom,sc7280", .data = sc7280_domains, }, { .compatible = "qcom,sc8180x", .data = sc8180x_domains, }, diff --git a/drivers/soc/qcom/qcom_pdr_msg.c b/drivers/soc/qcom/qcom_pdr_msg.c index ca98932140d87..02022b11ecf05 100644 --- a/drivers/soc/qcom/qcom_pdr_msg.c +++ b/drivers/soc/qcom/qcom_pdr_msg.c @@ -325,7 +325,7 @@ const struct qmi_elem_info servreg_loc_pfr_req_ei[] = { }, { .data_type = QMI_STRING, - .elem_len = SERVREG_NAME_LENGTH + 1, + .elem_len = SERVREG_PFR_LENGTH + 1, .elem_size = sizeof(char), .array_type = VAR_LEN_ARRAY, .tlv_type = 0x02, diff --git a/drivers/soc/qcom/qmi-cooling.c b/drivers/soc/qcom/qmi-cooling.c new file mode 100644 index 0000000000000..5f3a72376033f --- /dev/null +++ b/drivers/soc/qcom/qmi-cooling.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2017, The Linux Foundation + * Copyright (c) 2025, Linaro Limited + * + * QMI Thermal Mitigation Device (TMD) client driver. + * This driver provides an in-kernel client to handle hot and cold thermal + * mitigations for remote subsystems (modem and DSPs) running the TMD service. + * It doesn't implement any handling of reports from remote subsystems. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qmi-cooling.h" + +#define MODEM0_INSTANCE_ID 0x0 +#define ADSP_INSTANCE_ID 0x1 +#define CDSP_INSTANCE_ID 0x43 +#define SLPI_INSTANCE_ID 0x53 + +#define QMI_TMD_RESP_TIMEOUT msecs_to_jiffies(100) + +/** + * struct qmi_tmd_client - TMD client state + * @dev: Device associated with this client + * @name: Friendly name for the remote TMD service + * @handle: QMI connection handle + * @mutex: Lock to synchronise QMI communication + * @id: The QMI TMD service instance ID + * @cdev_list: The list of cooling devices (controls) enabled for this instance + * @svc_arrive_work: Work item for initialising the client when the TMD service + * starts. + * @connection_active: Whether or not we're connected to the QMI TMD service + */ +struct qmi_tmd_client { + struct device *dev; + const char *name; + struct qmi_handle handle; + struct mutex mutex; + u32 id; + struct list_head cdev_list; + struct work_struct svc_arrive_work; + bool connection_active; +}; + +/** + * struct qmi_tmd - A TMD cooling device + * @np: OF node associated with this control + * @type: The control type (exposed via sysfs) + * @qmi_name: The common name of this control shared by the remote subsystem + * @rproc_cdev: Remote processor cooling device handle + * @cur_state: The current cooling/warming/mitigation state + * @max_state: The maximum state + * @client: The TMD client instance this control is associated with + */ +struct qmi_tmd { + struct device_node *np; + const char *type; + char qmi_name[QMI_TMD_MITIGATION_DEV_ID_LENGTH_MAX_V01 + 1]; + struct list_head node; + struct remoteproc_cdev *rproc_cdev; + unsigned int cur_state; + unsigned int max_state; + struct qmi_tmd_client *client; +}; + +/** + * struct qmi_instance_id - QMI instance match data + * @id: The QMI instance ID + * @name: Friendly name for this instance + */ +struct qmi_instance_data { + u32 id; + const char *name; +}; + +/* Notify the remote subsystem of the requested cooling state */ +static int qmi_tmd_send_state_request(struct qmi_tmd *tmd) +{ + struct tmd_set_mitigation_level_resp_msg_v01 tmd_resp = { 0 }; + struct tmd_set_mitigation_level_req_msg_v01 req = { 0 }; + struct qmi_tmd_client *client; + struct qmi_txn txn; + int ret = 0; + + client = tmd->client; + + guard(mutex)(&client->mutex); + + /* + * This function is called by qmi_set_cur_state() which does not know if + * the QMI service is actually online. If it isn't then we noop here. + * The state is cached in tmd->cur_state and will be broadcast via + * qmi_tmd_init_control() when the service comes up. + */ + if (!client->connection_active) + return 0; + + strscpy(req.mitigation_dev_id.mitigation_dev_id, tmd->qmi_name, + QMI_TMD_MITIGATION_DEV_ID_LENGTH_MAX_V01 + 1); + req.mitigation_level = tmd->cur_state; + + ret = qmi_txn_init(&client->handle, &txn, + tmd_set_mitigation_level_resp_msg_v01_ei, &tmd_resp); + if (ret < 0) { + dev_err(client->dev, "qmi set state %d txn init failed for %s ret %d\n", + tmd->cur_state, tmd->type, ret); + return ret; + } + + ret = qmi_send_request(&client->handle, NULL, &txn, + QMI_TMD_SET_MITIGATION_LEVEL_REQ_V01, + TMD_SET_MITIGATION_LEVEL_REQ_MSG_V01_MAX_MSG_LEN, + tmd_set_mitigation_level_req_msg_v01_ei, &req); + if (ret < 0) { + dev_err(client->dev, "qmi set state %d txn send failed for %s ret %d\n", + tmd->cur_state, tmd->type, ret); + qmi_txn_cancel(&txn); + return ret; + } + + ret = qmi_txn_wait(&txn, QMI_TMD_RESP_TIMEOUT); + if (ret < 0) { + dev_err(client->dev, "qmi set state %d txn wait failed for %s ret %d\n", + tmd->cur_state, tmd->type, ret); + return ret; + } + + if (tmd_resp.resp.result != QMI_RESULT_SUCCESS_V01) { + ret = -tmd_resp.resp.result; + dev_err(client->dev, "qmi set state %d NOT success for %s ret %d\n", + tmd->cur_state, tmd->type, ret); + return ret; + } + + dev_dbg(client->dev, "Requested state %d/%d for %s\n", tmd->cur_state, + tmd->max_state, tmd->type); + + return 0; +} + +static int qmi_get_max_level(void *devdata, unsigned long *level) +{ + struct qmi_tmd *tmd = devdata; + + if (!tmd) + return -EINVAL; + + *level = tmd->max_state; + + return 0; +} + +static int qmi_get_cur_level(void *devdata, unsigned long *level) +{ + struct qmi_tmd *tmd = devdata; + + if (!tmd) + return -EINVAL; + + *level = tmd->cur_state; + + return 0; +} + +static int qmi_set_cur_level(void *devdata, unsigned long level) +{ + struct qmi_tmd *tmd = devdata; + + if (!tmd) + return -EINVAL; + + if (level > tmd->max_state) + return -EINVAL; + + if (tmd->cur_state == level) + return 0; + + tmd->cur_state = level; + + return qmi_tmd_send_state_request(tmd); +} + +static const struct remoteproc_cooling_ops qmi_rproc_ops = { + .get_max_level = qmi_get_max_level, + .get_cur_level = qmi_get_cur_level, + .set_cur_level = qmi_set_cur_level, +}; + +static int qmi_register_cooling_device(struct qmi_tmd *tmd) +{ + struct remoteproc_cdev *rproc_cdev; + + rproc_cdev = remoteproc_cooling_register(tmd->np, + tmd->type, + &qmi_rproc_ops, + tmd); + + if (IS_ERR(rproc_cdev)) + return dev_err_probe(tmd->client->dev, PTR_ERR(rproc_cdev), + "Failed to register cooling device %s\n", + tmd->qmi_name); + + tmd->rproc_cdev = rproc_cdev; + return 0; +} + +/* + * Init a single TMD control by registering a cooling device for it, or + * synchronising state with the remote subsystem if recovering from a service + * restart. This is called when the TMD service starts up. + */ +static int qmi_tmd_init_control(struct qmi_tmd_client *client, const char *label, + u8 max_state) +{ + struct qmi_tmd *tmd = NULL; + + list_for_each_entry(tmd, &client->cdev_list, node) + if (!strncasecmp(tmd->qmi_name, label, + QMI_TMD_MITIGATION_DEV_ID_LENGTH_MAX_V01 + 1)) + goto found; + + dev_dbg(client->dev, + "TMD '%s' available in firmware but not specified in DT\n", + label); + return 0; + +found: + tmd->max_state = max_state; + /* + * If the cooling device already exists then the QMI service went away and + * came back. So just make sure the current cooling device state is + * reflected on the remote side and then return. + */ + if (tmd->rproc_cdev) + return qmi_tmd_send_state_request(tmd); + + return qmi_register_cooling_device(tmd); +} + +/* + * When the QMI service starts up on a remote subsystem this function will fetch + * the list of TMDs on the subsystem, match it to the TMDs specified in devicetree + * and call qmi_tmd_init_control() for each + */ +static void qmi_tmd_svc_arrive(struct work_struct *work) +{ + struct qmi_tmd_client *client = + container_of(work, struct qmi_tmd_client, svc_arrive_work); + + struct tmd_get_mitigation_device_list_req_msg_v01 req = { 0 }; + struct tmd_get_mitigation_device_list_resp_msg_v01 *resp __free(kfree); + int ret = 0, i; + struct qmi_txn txn; + + /* resp struct is 1.1kB, allocate it on the heap. */ + resp = kzalloc(sizeof(*resp), GFP_KERNEL); + if (!resp) + return; + + /* Get a list of TMDs supported by the remoteproc */ + scoped_guard(mutex, &client->mutex) { + ret = qmi_txn_init(&client->handle, &txn, + tmd_get_mitigation_device_list_resp_msg_v01_ei, resp); + if (ret < 0) { + dev_err(client->dev, + "Transaction init error for instance_id: %#x ret %d\n", + client->id, ret); + return; + } + + ret = qmi_send_request(&client->handle, NULL, &txn, + QMI_TMD_GET_MITIGATION_DEVICE_LIST_REQ_V01, + TMD_GET_MITIGATION_DEVICE_LIST_REQ_MSG_V01_MAX_MSG_LEN, + tmd_get_mitigation_device_list_req_msg_v01_ei, &req); + if (ret < 0) { + qmi_txn_cancel(&txn); + return; + } + + ret = qmi_txn_wait(&txn, QMI_TMD_RESP_TIMEOUT); + if (ret < 0) { + dev_err(client->dev, "Transaction wait error for client %#x ret:%d\n", + client->id, ret); + return; + } + if (resp->resp.result != QMI_RESULT_SUCCESS_V01) { + ret = resp->resp.result; + dev_err(client->dev, "Failed to get device list for client %#x ret:%d\n", + client->id, ret); + return; + } + + client->connection_active = true; + } + + for (i = 0; i < resp->mitigation_device_list_len; i++) { + struct tmd_mitigation_dev_list_type_v01 *device = + &resp->mitigation_device_list[i]; + + ret = qmi_tmd_init_control(client, + device->mitigation_dev_id.mitigation_dev_id, + device->max_mitigation_level); + if (ret) + break; + } +} + +static void thermal_qmi_net_reset(struct qmi_handle *qmi) +{ + struct qmi_tmd_client *client = container_of(qmi, struct qmi_tmd_client, handle); + struct qmi_tmd *tmd = NULL; + + list_for_each_entry(tmd, &client->cdev_list, node) { + qmi_tmd_send_state_request(tmd); + } +} + +static void thermal_qmi_del_server(struct qmi_handle *qmi, struct qmi_service *service) +{ + struct qmi_tmd_client *client = container_of(qmi, struct qmi_tmd_client, handle); + + scoped_guard(mutex, &client->mutex) + client->connection_active = false; +} + +static int thermal_qmi_new_server(struct qmi_handle *qmi, struct qmi_service *service) +{ + struct qmi_tmd_client *client = container_of(qmi, struct qmi_tmd_client, handle); + struct sockaddr_qrtr sq = { AF_QIPCRTR, service->node, service->port }; + + scoped_guard(mutex, &client->mutex) + kernel_connect(qmi->sock, (struct sockaddr *)&sq, sizeof(sq), 0); + + queue_work(system_highpri_wq, &client->svc_arrive_work); + + return 0; +} + +static struct qmi_ops thermal_qmi_event_ops = { + .new_server = thermal_qmi_new_server, + .del_server = thermal_qmi_del_server, + .net_reset = thermal_qmi_net_reset, +}; + +static void qmi_tmd_cleanup(struct qmi_tmd_client *client) +{ + struct qmi_tmd *tmd, *c_next; + + guard(mutex)(&client->mutex); + + client->connection_active = false; + + qmi_handle_release(&client->handle); + cancel_work(&client->svc_arrive_work); + list_for_each_entry_safe(tmd, c_next, &client->cdev_list, node) { + if (tmd->rproc_cdev) + remoteproc_cooling_unregister(tmd->rproc_cdev); + + list_del(&tmd->node); + } +} + +/* Parse the controls and allocate a qmi_tmd for each of them */ +static int qmi_tmd_alloc_cdevs(struct qmi_tmd_client *client) +{ + struct device *dev = client->dev; + struct qmi_tmd *tmd; + struct device_node *subnode, *node = dev->of_node; + int ret; + + for_each_available_child_of_node(node, subnode) { + const char *name; + + tmd = devm_kzalloc(dev, sizeof(*tmd), GFP_KERNEL); + if (!tmd) + return dev_err_probe(client->dev, -ENOMEM, + "Couldn't allocate tmd\n"); + + tmd->type = devm_kasprintf(client->dev, GFP_KERNEL, "%s:%s", + client->name, subnode->name); + if (!tmd->type) + return dev_err_probe(dev, -ENOMEM, + "Couldn't allocate cooling device name\n"); + + if (of_property_read_string(subnode, "label", &name)) { + return dev_err_probe(client->dev, -EINVAL, + "Failed to parse dev name for %s\n", + subnode->name); + } + + ret = strscpy(tmd->qmi_name, name, + QMI_TMD_MITIGATION_DEV_ID_LENGTH_MAX_V01 + 1); + if (ret == -E2BIG) { + return dev_err_probe(dev, -EINVAL, "TMD label %s is too long\n", + name); + } + + tmd->client = client; + tmd->np = subnode; + tmd->cur_state = 0; + list_add(&tmd->node, &client->cdev_list); + } + + if (list_empty(&client->cdev_list)) + return dev_err_probe(client->dev, -EINVAL, + "No cooling devices specified for client %s (%#x)\n", + client->name, client->id); + + return 0; +} + +static int qmi_tmd_client_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct qmi_tmd_client *client; + const struct qmi_instance_data *match; + int ret; + + client = devm_kzalloc(dev, sizeof(*client), GFP_KERNEL); + if (!client) + return -ENOMEM; + + client->dev = dev; + + match = of_device_get_match_data(dev); + if (!match) + return dev_err_probe(dev, -EINVAL, "No match data\n"); + + client->id = match->id; + client->name = match->name; + + mutex_init(&client->mutex); + INIT_LIST_HEAD(&client->cdev_list); + INIT_WORK(&client->svc_arrive_work, qmi_tmd_svc_arrive); + + ret = qmi_tmd_alloc_cdevs(client); + if (ret) + return ret; + + platform_set_drvdata(pdev, client); + + ret = qmi_handle_init(&client->handle, + TMD_GET_MITIGATION_DEVICE_LIST_RESP_MSG_V01_MAX_MSG_LEN, + &thermal_qmi_event_ops, NULL); + if (ret < 0) + return dev_err_probe(client->dev, ret, "QMI handle init failed for client %#x\n", + client->id); + + ret = qmi_add_lookup(&client->handle, TMD_SERVICE_ID_V01, TMD_SERVICE_VERS_V01, + client->id); + if (ret < 0) { + qmi_handle_release(&client->handle); + return dev_err_probe(client->dev, ret, "QMI register failed for client 0x%x\n", + client->id); + } + + return 0; +} + +static void qmi_tmd_client_remove(struct platform_device *pdev) +{ + struct qmi_tmd_client *client = platform_get_drvdata(pdev); + + qmi_tmd_cleanup(client); +} + +static const struct of_device_id qmi_tmd_device_table[] = { + { + .compatible = "qcom,qmi-cooling-cdsp", + .data = &((struct qmi_instance_data) { CDSP_INSTANCE_ID, "cdsp" }), + }, + {} +}; +MODULE_DEVICE_TABLE(of, qmi_tmd_device_table); + +static struct platform_driver qmi_tmd_device_driver = { + .probe = qmi_tmd_client_probe, + .remove = qmi_tmd_client_remove, + .driver = { + .name = "qcom-qmi-cooling", + .of_match_table = qmi_tmd_device_table, + }, +}; + +module_platform_driver(qmi_tmd_device_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/qmi-cooling.h b/drivers/soc/qcom/qmi-cooling.h new file mode 100644 index 0000000000000..f46b827b4ce64 --- /dev/null +++ b/drivers/soc/qcom/qmi-cooling.h @@ -0,0 +1,428 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2017, The Linux Foundation + * Copyright (c) 2023, Linaro Limited + */ + +#ifndef __QCOM_COOLING_H__ +#define __QCOM_COOLING_H__ + +#include + +#define TMD_SERVICE_ID_V01 0x18 +#define TMD_SERVICE_VERS_V01 0x01 + +#define QMI_TMD_GET_MITIGATION_DEVICE_LIST_RESP_V01 0x0020 +#define QMI_TMD_GET_MITIGATION_LEVEL_REQ_V01 0x0022 +#define QMI_TMD_GET_SUPPORTED_MSGS_REQ_V01 0x001E +#define QMI_TMD_SET_MITIGATION_LEVEL_REQ_V01 0x0021 +#define QMI_TMD_REGISTER_NOTIFICATION_MITIGATION_LEVEL_RESP_V01 0x0023 +#define QMI_TMD_GET_SUPPORTED_MSGS_RESP_V01 0x001E +#define QMI_TMD_SET_MITIGATION_LEVEL_RESP_V01 0x0021 +#define QMI_TMD_DEREGISTER_NOTIFICATION_MITIGATION_LEVEL_RESP_V01 0x0024 +#define QMI_TMD_MITIGATION_LEVEL_REPORT_IND_V01 0x0025 +#define QMI_TMD_GET_MITIGATION_LEVEL_RESP_V01 0x0022 +#define QMI_TMD_GET_SUPPORTED_FIELDS_REQ_V01 0x001F +#define QMI_TMD_GET_MITIGATION_DEVICE_LIST_REQ_V01 0x0020 +#define QMI_TMD_REGISTER_NOTIFICATION_MITIGATION_LEVEL_REQ_V01 0x0023 +#define QMI_TMD_DEREGISTER_NOTIFICATION_MITIGATION_LEVEL_REQ_V01 0x0024 +#define QMI_TMD_GET_SUPPORTED_FIELDS_RESP_V01 0x001F + +#define QMI_TMD_MITIGATION_DEV_ID_LENGTH_MAX_V01 32 +#define QMI_TMD_MITIGATION_DEV_LIST_MAX_V01 32 + +struct tmd_mitigation_dev_id_type_v01 { + char mitigation_dev_id[QMI_TMD_MITIGATION_DEV_ID_LENGTH_MAX_V01 + 1]; +}; + +static const struct qmi_elem_info tmd_mitigation_dev_id_type_v01_ei[] = { + { + .data_type = QMI_STRING, + .elem_len = QMI_TMD_MITIGATION_DEV_ID_LENGTH_MAX_V01 + 1, + .elem_size = sizeof(char), + .array_type = NO_ARRAY, + .tlv_type = 0, + .offset = offsetof(struct tmd_mitigation_dev_id_type_v01, + mitigation_dev_id), + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_mitigation_dev_list_type_v01 { + struct tmd_mitigation_dev_id_type_v01 mitigation_dev_id; + uint8_t max_mitigation_level; +}; + +static const struct qmi_elem_info tmd_mitigation_dev_list_type_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct tmd_mitigation_dev_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0, + .offset = offsetof(struct tmd_mitigation_dev_list_type_v01, + mitigation_dev_id), + .ei_array = tmd_mitigation_dev_id_type_v01_ei, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0, + .offset = offsetof(struct tmd_mitigation_dev_list_type_v01, + max_mitigation_level), + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_get_mitigation_device_list_req_msg_v01 { + char placeholder; +}; + +#define TMD_GET_MITIGATION_DEVICE_LIST_REQ_MSG_V01_MAX_MSG_LEN 0 +const struct qmi_elem_info tmd_get_mitigation_device_list_req_msg_v01_ei[] = { + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_get_mitigation_device_list_resp_msg_v01 { + struct qmi_response_type_v01 resp; + uint8_t mitigation_device_list_valid; + uint32_t mitigation_device_list_len; + struct tmd_mitigation_dev_list_type_v01 + mitigation_device_list[QMI_TMD_MITIGATION_DEV_LIST_MAX_V01]; +}; + +#define TMD_GET_MITIGATION_DEVICE_LIST_RESP_MSG_V01_MAX_MSG_LEN 1099 +static const struct qmi_elem_info tmd_get_mitigation_device_list_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct tmd_get_mitigation_device_list_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct tmd_get_mitigation_device_list_resp_msg_v01, + mitigation_device_list_valid), + }, + { + .data_type = QMI_DATA_LEN, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct tmd_get_mitigation_device_list_resp_msg_v01, + mitigation_device_list_len), + }, + { + .data_type = QMI_STRUCT, + .elem_len = QMI_TMD_MITIGATION_DEV_LIST_MAX_V01, + .elem_size = sizeof(struct tmd_mitigation_dev_list_type_v01), + .array_type = VAR_LEN_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct tmd_get_mitigation_device_list_resp_msg_v01, + mitigation_device_list), + .ei_array = tmd_mitigation_dev_list_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_set_mitigation_level_req_msg_v01 { + struct tmd_mitigation_dev_id_type_v01 mitigation_dev_id; + uint8_t mitigation_level; +}; + +#define TMD_SET_MITIGATION_LEVEL_REQ_MSG_V01_MAX_MSG_LEN 40 +static const struct qmi_elem_info tmd_set_mitigation_level_req_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct tmd_mitigation_dev_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof(struct tmd_set_mitigation_level_req_msg_v01, + mitigation_dev_id), + .ei_array = tmd_mitigation_dev_id_type_v01_ei, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct tmd_set_mitigation_level_req_msg_v01, + mitigation_level), + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_set_mitigation_level_resp_msg_v01 { + struct qmi_response_type_v01 resp; +}; + +#define TMD_SET_MITIGATION_LEVEL_RESP_MSG_V01_MAX_MSG_LEN 7 +static const struct qmi_elem_info tmd_set_mitigation_level_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct tmd_set_mitigation_level_resp_msg_v01, resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_get_mitigation_level_req_msg_v01 { + struct tmd_mitigation_dev_id_type_v01 mitigation_device; +}; +#define TMD_GET_MITIGATION_LEVEL_REQ_MSG_V01_MAX_MSG_LEN 36 + +static const struct qmi_elem_info tmd_get_mitigation_level_req_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct tmd_mitigation_dev_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof(struct tmd_get_mitigation_level_req_msg_v01, + mitigation_device), + .ei_array = tmd_mitigation_dev_id_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_get_mitigation_level_resp_msg_v01 { + struct qmi_response_type_v01 resp; + uint8_t current_mitigation_level_valid; + uint8_t current_mitigation_level; + uint8_t requested_mitigation_level_valid; + uint8_t requested_mitigation_level; +}; + +#define TMD_GET_MITIGATION_LEVEL_RESP_MSG_V01_MAX_MSG_LEN 15 +static const struct qmi_elem_info tmd_get_mitigation_level_resp_msg_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct tmd_get_mitigation_level_resp_msg_v01, resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct tmd_get_mitigation_level_resp_msg_v01, + current_mitigation_level_valid), + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct tmd_get_mitigation_level_resp_msg_v01, + current_mitigation_level), + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x11, + .offset = offsetof(struct tmd_get_mitigation_level_resp_msg_v01, + requested_mitigation_level_valid), + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x11, + .offset = offsetof(struct tmd_get_mitigation_level_resp_msg_v01, + requested_mitigation_level), + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct tmd_register_notification_mitigation_level_req_msg_v01 { + struct tmd_mitigation_dev_id_type_v01 mitigation_device; +}; + +#define TMD_REGISTER_NOTIFICATION_MITIGATION_LEVEL_REQ_MSG_V01_MAX_MSG_LEN 36 +static const struct qmi_elem_info + tmd_register_notification_mitigation_level_req_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct tmd_mitigation_dev_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof( + struct tmd_register_notification_mitigation_level_req_msg_v01, + mitigation_device), + .ei_array = tmd_mitigation_dev_id_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, + }; + +struct tmd_register_notification_mitigation_level_resp_msg_v01 { + struct qmi_response_type_v01 resp; +}; + +#define TMD_REGISTER_NOTIFICATION_MITIGATION_LEVEL_RESP_MSG_V01_MAX_MSG_LEN 7 +static const struct qmi_elem_info + tmd_register_notification_mitigation_level_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof( + struct tmd_register_notification_mitigation_level_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, + }; + +struct tmd_deregister_notification_mitigation_level_req_msg_v01 { + struct tmd_mitigation_dev_id_type_v01 mitigation_device; +}; + +#define TMD_DEREGISTER_NOTIFICATION_MITIGATION_LEVEL_REQ_MSG_V01_MAX_MSG_LEN 36 +static const struct qmi_elem_info + tmd_deregister_notification_mitigation_level_req_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct tmd_mitigation_dev_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof( + struct tmd_deregister_notification_mitigation_level_req_msg_v01, + mitigation_device), + .ei_array = tmd_mitigation_dev_id_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, + }; + +struct tmd_deregister_notification_mitigation_level_resp_msg_v01 { + struct qmi_response_type_v01 resp; +}; + +#define TMD_DEREGISTER_NOTIFICATION_MITIGATION_LEVEL_RESP_MSG_V01_MAX_MSG_LEN 7 +static const struct qmi_elem_info + tmd_deregister_notification_mitigation_level_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof( + struct tmd_deregister_notification_mitigation_level_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, + }; + +struct tmd_mitigation_level_report_ind_msg_v01 { + struct tmd_mitigation_dev_id_type_v01 mitigation_device; + uint8_t current_mitigation_level; +}; + +#define TMD_MITIGATION_LEVEL_REPORT_IND_MSG_V01_MAX_MSG_LEN 40 +static const struct qmi_elem_info tmd_mitigation_level_report_ind_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct tmd_mitigation_dev_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof(struct tmd_mitigation_level_report_ind_msg_v01, + mitigation_device), + .ei_array = tmd_mitigation_dev_id_type_v01_ei, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct tmd_mitigation_level_report_ind_msg_v01, + current_mitigation_level), + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +#endif /* __QMI_COOLING_INTERNAL_H__ */ diff --git a/drivers/soc/qcom/qmi_interface.c b/drivers/soc/qcom/qmi_interface.c index 6500f863aae5c..941612b1bd2e6 100644 --- a/drivers/soc/qcom/qmi_interface.c +++ b/drivers/soc/qcom/qmi_interface.c @@ -321,7 +321,7 @@ int qmi_txn_init(struct qmi_handle *qmi, struct qmi_txn *txn, mutex_lock(&qmi->txn_lock); ret = idr_alloc_cyclic(&qmi->txns, txn, 0, U16_MAX, GFP_KERNEL); if (ret < 0) - pr_err("failed to allocate transaction id\n"); + pr_err("failed to allocate transaction id: %d\n", ret); txn->id = ret; mutex_unlock(&qmi->txn_lock); @@ -413,7 +413,7 @@ static void qmi_invoke_handler(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, ret = qmi_decode_message(buf, len, handler->ei, dest); if (ret < 0) - pr_err("failed to decode incoming message\n"); + pr_err("failed to decode incoming message: %d\n", ret); else handler->fn(qmi, sq, txn, dest); @@ -502,7 +502,7 @@ static void qmi_handle_message(struct qmi_handle *qmi, if (txn->dest && txn->ei) { ret = qmi_decode_message(buf, len, txn->ei, txn->dest); if (ret < 0) - pr_err("failed to decode incoming message\n"); + pr_err("failed to decode incoming message: %d\n", ret); txn->result = ret; complete(&txn->completion); @@ -661,8 +661,8 @@ int qmi_handle_init(struct qmi_handle *qmi, size_t recv_buf_size, if (PTR_ERR(qmi->sock) == -EAFNOSUPPORT) { ret = -EPROBE_DEFER; } else { - pr_err("failed to create QMI socket\n"); ret = PTR_ERR(qmi->sock); + pr_err("failed to create QMI socket: %d\n", ret); } goto err_destroy_wq; } @@ -766,7 +766,7 @@ static ssize_t qmi_send_message(struct qmi_handle *qmi, if (qmi->sock) { ret = kernel_sendmsg(qmi->sock, &msghdr, &iv, 1, len); if (ret < 0) - pr_err("failed to send QMI message\n"); + pr_err("failed to send QMI message: %d\n", ret); } else { ret = -EPIPE; } diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c index c4c45f15dca4f..39177aa5793ad 100644 --- a/drivers/soc/qcom/smem.c +++ b/drivers/soc/qcom/smem.c @@ -1190,7 +1190,7 @@ static int qcom_smem_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, hwlock_id, "failed to retrieve hwlock\n"); - smem->hwlock = hwspin_lock_request_specific(hwlock_id); + smem->hwlock = devm_hwspin_lock_request_specific(&pdev->dev, hwlock_id); if (!smem->hwlock) return -ENXIO; @@ -1215,7 +1215,9 @@ static int qcom_smem_probe(struct platform_device *pdev) smem->item_count = qcom_smem_get_item_count(smem); break; case SMEM_GLOBAL_HEAP_VERSION: - qcom_smem_map_global(smem, size); + ret = qcom_smem_map_global(smem, size); + if (ret < 0) + return ret; smem->item_count = SMEM_ITEM_COUNT; break; default: @@ -1243,7 +1245,6 @@ static void qcom_smem_remove(struct platform_device *pdev) { platform_device_unregister(__smem->socinfo); - hwspin_lock_free(__smem->hwlock); __smem = NULL; } diff --git a/drivers/soc/qcom/smp2p.c b/drivers/soc/qcom/smp2p.c index cb515c2340c10..c35ca7535c144 100644 --- a/drivers/soc/qcom/smp2p.c +++ b/drivers/soc/qcom/smp2p.c @@ -36,6 +36,10 @@ * The driver uses the Linux GPIO and interrupt framework to expose a virtual * GPIO for each outbound entry and a virtual interrupt controller for each * inbound entry. + * + * Driver supports two versions: + * V1 - For processor that start after local host + * V2 - For processor that start in early boot sequence */ #define SMP2P_MAX_ENTRY 16 @@ -48,10 +52,14 @@ #define SMP2P_MAGIC 0x504d5324 #define SMP2P_ALL_FEATURES SMP2P_FEATURE_SSR_ACK +#define ONE 1 +#define TWO 2 +#define MAX_VERSION TWO + /** * struct smp2p_smem_item - in memory communication structure * @magic: magic number - * @version: version - must be 1 + * @version: version * @features: features flag - currently unused * @local_pid: processor id of sending end * @remote_pid: processor id of receiving end @@ -180,14 +188,23 @@ static void qcom_smp2p_kick(struct qcom_smp2p *smp2p) static bool qcom_smp2p_check_ssr(struct qcom_smp2p *smp2p) { struct smp2p_smem_item *in = smp2p->in; + struct smp2p_entry *entry; bool restart; if (!smp2p->ssr_ack_enabled) return false; restart = in->flags & BIT(SMP2P_FLAGS_RESTART_DONE_BIT); + restart = restart != smp2p->ssr_ack; + if (restart && in->version > ONE) { + list_for_each_entry(entry, &smp2p->inbound, node) { + if (!entry->value) + continue; + entry->last_value = 0; + } + } - return restart != smp2p->ssr_ack; + return restart; } static void qcom_smp2p_do_ssr_ack(struct qcom_smp2p *smp2p) @@ -222,6 +239,56 @@ static void qcom_smp2p_negotiate(struct qcom_smp2p *smp2p) } } +static int qcom_smp2p_in_version(struct qcom_smp2p *smp2p) +{ + unsigned int smem_id = smp2p->smem_items[SMP2P_INBOUND]; + unsigned int pid = smp2p->remote_pid; + struct smp2p_smem_item *in; + size_t size; + + in = qcom_smem_get(pid, smem_id, &size); + if (IS_ERR(in)) + return 0; + + return in->version; +} + +static void qcom_smp2p_start_in(struct qcom_smp2p *smp2p) +{ + unsigned int smem_id = smp2p->smem_items[SMP2P_INBOUND]; + unsigned int pid = smp2p->remote_pid; + char buf[SMP2P_MAX_ENTRY_NAME]; + struct smp2p_smem_item *in; + struct smp2p_entry *entry; + size_t size; + int i; + + in = qcom_smem_get(pid, smem_id, &size); + if (IS_ERR(in)) + return; + + smp2p->in = in; + + /* Check if version is initialized and set to v2. + * Early enumeration of inbound entries is required only + * for early boot processors which have smp2p version 2. + */ + if (in->version != TWO) + return; + + for (i = smp2p->valid_entries; i < in->valid_entries; i++) { + list_for_each_entry(entry, &smp2p->inbound, node) { + memcpy(buf, in->entries[i].name, sizeof(buf)); + if (!strcmp(buf, entry->name)) { + entry->value = &in->entries[i].value; + entry->last_value = readl(entry->value); + break; + } + } + } + smp2p->valid_entries = i; +} + static void qcom_smp2p_notify_in(struct qcom_smp2p *smp2p) { struct smp2p_smem_item *in; @@ -368,12 +435,31 @@ static void smp2p_irq_print_chip(struct irq_data *irqd, struct seq_file *p) seq_printf(p, "%8s", dev_name(entry->smp2p->dev)); } +static int smp2p_irq_get_irqchip_state(struct irq_data *irqd, enum irqchip_irq_state which, + bool *state) +{ + struct smp2p_entry *entry = irq_data_get_irq_chip_data(irqd); + u32 val; + + if (which != IRQCHIP_STATE_LINE_LEVEL) + return -EINVAL; + + if (!entry->value) + return -ENODEV; + + val = readl(entry->value); + *state = !!(val & BIT(irqd_to_hwirq(irqd))); + + return 0; +} + static struct irq_chip smp2p_irq_chip = { .name = "smp2p", .irq_mask = smp2p_mask_irq, .irq_unmask = smp2p_unmask_irq, .irq_set_type = smp2p_set_irq_type, .irq_print_chip = smp2p_irq_print_chip, + .irq_get_irqchip_state = smp2p_irq_get_irqchip_state, }; static int smp2p_irq_map(struct irq_domain *d, @@ -464,6 +550,7 @@ static int qcom_smp2p_alloc_outbound_item(struct qcom_smp2p *smp2p) struct smp2p_smem_item *out; unsigned smem_id = smp2p->smem_items[SMP2P_OUTBOUND]; unsigned pid = smp2p->remote_pid; + u8 in_version; int ret; ret = qcom_smem_alloc(pid, smem_id, sizeof(*out)); @@ -485,12 +572,18 @@ static int qcom_smp2p_alloc_outbound_item(struct qcom_smp2p *smp2p) out->valid_entries = 0; out->features = SMP2P_ALL_FEATURES; + in_version = qcom_smp2p_in_version(smp2p); + if (in_version > MAX_VERSION) { + dev_err(smp2p->dev, "Unsupported smp2p version\n"); + return -EINVAL; + } + /* * Make sure the rest of the header is written before we validate the * item by writing a valid version number. */ wmb(); - out->version = 1; + out->version = (in_version) ? in_version : 1; qcom_smp2p_kick(smp2p); @@ -618,6 +711,9 @@ static int qcom_smp2p_probe(struct platform_device *pdev) } } + /* Check inbound entries in the case of early boot processor */ + qcom_smp2p_start_in(smp2p); + /* Kick the outgoing edge after allocating entries */ qcom_smp2p_kick(smp2p); diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c index 963772f45489d..704235ad9ccf3 100644 --- a/drivers/soc/qcom/socinfo.c +++ b/drivers/soc/qcom/socinfo.c @@ -415,6 +415,7 @@ static const struct soc_id soc_id[] = { { qcom_board_id(SC7280) }, { qcom_board_id(SC7180P) }, { qcom_board_id(QCM6490) }, + { qcom_board_id(QCS6490) }, { qcom_board_id(SM7325P) }, { qcom_board_id(IPQ5000) }, { qcom_board_id(IPQ0509) }, @@ -461,6 +462,7 @@ static const struct soc_id soc_id[] = { { qcom_board_id(IPQ5424) }, { qcom_board_id(QCM6690) }, { qcom_board_id(QCS6690) }, + { qcom_board_id(SM8850) }, { qcom_board_id(IPQ5404) }, { qcom_board_id(QCS9100) }, { qcom_board_id(QCS8300) }, diff --git a/drivers/soc/qcom/ubwc_config.c b/drivers/soc/qcom/ubwc_config.c index 15d373bff231d..1c09796163b09 100644 --- a/drivers/soc/qcom/ubwc_config.c +++ b/drivers/soc/qcom/ubwc_config.c @@ -16,6 +16,16 @@ static const struct qcom_ubwc_cfg_data no_ubwc_data = { /* no UBWC, no HBB */ }; +static const struct qcom_ubwc_cfg_data kaanapali_data = { + .ubwc_enc_version = UBWC_6_0, + .ubwc_dec_version = UBWC_6_0, + .ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 | + UBWC_SWIZZLE_ENABLE_LVL3, + .ubwc_bank_spread = true, + .highest_bank_bit = 16, + .macrotile_mode = true, +}; + static const struct qcom_ubwc_cfg_data msm8937_data = { .ubwc_enc_version = UBWC_1_0, .ubwc_dec_version = UBWC_1_0, @@ -218,11 +228,24 @@ static const struct qcom_ubwc_cfg_data x1e80100_data = { .macrotile_mode = true, }; +static const struct qcom_ubwc_cfg_data glymur_data = { + .ubwc_enc_version = UBWC_5_0, + .ubwc_dec_version = UBWC_5_0, + .ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 | + UBWC_SWIZZLE_ENABLE_LVL3, + .ubwc_bank_spread = true, + /* TODO: highest_bank_bit = 15 for LP_DDR4 */ + .highest_bank_bit = 16, + .macrotile_mode = true, +}; + static const struct of_device_id qcom_ubwc_configs[] __maybe_unused = { { .compatible = "qcom,apq8016", .data = &no_ubwc_data }, { .compatible = "qcom,apq8026", .data = &no_ubwc_data }, { .compatible = "qcom,apq8074", .data = &no_ubwc_data }, { .compatible = "qcom,apq8096", .data = &msm8998_data }, + { .compatible = "qcom,kaanapali", .data = &kaanapali_data, }, + { .compatible = "qcom,glymur", .data = &glymur_data}, { .compatible = "qcom,msm8226", .data = &no_ubwc_data }, { .compatible = "qcom,msm8916", .data = &no_ubwc_data }, { .compatible = "qcom,msm8917", .data = &no_ubwc_data }, @@ -237,6 +260,7 @@ static const struct of_device_id qcom_ubwc_configs[] __maybe_unused = { { .compatible = "qcom,msm8998", .data = &msm8998_data }, { .compatible = "qcom,qcm2290", .data = &qcm2290_data, }, { .compatible = "qcom,qcm6490", .data = &sc7280_data, }, + { .compatible = "qcom,qcs8300", .data = &sc8280xp_data, }, { .compatible = "qcom,sa8155p", .data = &sm8150_data, }, { .compatible = "qcom,sa8540p", .data = &sc8280xp_data, }, { .compatible = "qcom,sa8775p", .data = &sa8775p_data, }, diff --git a/drivers/soc/renesas/r9a08g045-sysc.c b/drivers/soc/renesas/r9a08g045-sysc.c index 0504d4e687616..03d653d5cde55 100644 --- a/drivers/soc/renesas/r9a08g045-sysc.c +++ b/drivers/soc/renesas/r9a08g045-sysc.c @@ -6,10 +6,29 @@ */ #include +#include #include #include "rz-sysc.h" +#define SYS_XSPI_MAP_STAADD_CS0 0x348 +#define SYS_XSPI_MAP_ENDADD_CS0 0x34c +#define SYS_XSPI_MAP_STAADD_CS1 0x350 +#define SYS_XSPI_MAP_ENDADD_CS1 0x354 +#define SYS_GETH0_CFG 0x380 +#define SYS_GETH1_CFG 0x390 +#define SYS_PCIE_CFG 0x3a0 +#define SYS_PCIE_MON 0x3a4 +#define SYS_PCIE_ERR_MON 0x3ac +#define SYS_PCIE_PHY 0x3b4 +#define SYS_I2C0_CFG 0x400 +#define SYS_I2C1_CFG 0x410 +#define SYS_I2C2_CFG 0x420 +#define SYS_I2C3_CFG 0x430 +#define SYS_I3C_CFG 0x440 +#define SYS_USB_PWRRDY 0xd70 +#define SYS_PCIE_RST_RSM_B 0xd74 + static const struct rz_sysc_soc_id_init_data rzg3s_sysc_soc_id_init_data __initconst = { .family = "RZ/G3S", .id = 0x85e0447, @@ -18,7 +37,57 @@ static const struct rz_sysc_soc_id_init_data rzg3s_sysc_soc_id_init_data __initc .specific_id_mask = GENMASK(27, 0), }; +static bool rzg3s_regmap_readable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_XSPI_MAP_STAADD_CS0: + case SYS_XSPI_MAP_ENDADD_CS0: + case SYS_XSPI_MAP_STAADD_CS1: + case SYS_XSPI_MAP_ENDADD_CS1: + case SYS_GETH0_CFG: + case SYS_GETH1_CFG: + case SYS_PCIE_CFG: + case SYS_PCIE_MON: + case SYS_PCIE_ERR_MON: + case SYS_PCIE_PHY: + case SYS_I2C0_CFG: + case SYS_I2C1_CFG: + case SYS_I2C2_CFG: + case SYS_I2C3_CFG: + case SYS_I3C_CFG: + case SYS_USB_PWRRDY: + case SYS_PCIE_RST_RSM_B: + return true; + default: + return false; + } +} + +static bool rzg3s_regmap_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_XSPI_MAP_STAADD_CS0: + case SYS_XSPI_MAP_ENDADD_CS0: + case SYS_XSPI_MAP_STAADD_CS1: + case SYS_XSPI_MAP_ENDADD_CS1: + case SYS_PCIE_CFG: + case SYS_PCIE_PHY: + case SYS_I2C0_CFG: + case SYS_I2C1_CFG: + case SYS_I2C2_CFG: + case SYS_I2C3_CFG: + case SYS_I3C_CFG: + case SYS_USB_PWRRDY: + case SYS_PCIE_RST_RSM_B: + return true; + default: + return false; + } +} + const struct rz_sysc_init_data rzg3s_sysc_init_data __initconst = { .soc_id_init_data = &rzg3s_sysc_soc_id_init_data, + .readable_reg = rzg3s_regmap_readable_reg, + .writeable_reg = rzg3s_regmap_writeable_reg, .max_register = 0xe20, }; diff --git a/drivers/soc/renesas/r9a09g047-sys.c b/drivers/soc/renesas/r9a09g047-sys.c index 2e8426c030504..e413b0eff9bfd 100644 --- a/drivers/soc/renesas/r9a09g047-sys.c +++ b/drivers/soc/renesas/r9a09g047-sys.c @@ -29,6 +29,27 @@ #define SYS_LSI_PRR_CA55_DIS BIT(8) #define SYS_LSI_PRR_NPU_DIS BIT(1) +#define SYS_LSI_OTPTSU1TRMVAL0 0x330 +#define SYS_LSI_OTPTSU1TRMVAL1 0x334 +#define SYS_SPI_STAADDCS0 0x900 +#define SYS_SPI_ENDADDCS0 0x904 +#define SYS_SPI_STAADDCS1 0x908 +#define SYS_SPI_ENDADDCS1 0x90c +#define SYS_VSP_CLK 0xe00 +#define SYS_GBETH0_CFG 0xf00 +#define SYS_GBETH1_CFG 0xf04 +#define SYS_PCIE_INTX_CH0 0x1000 +#define SYS_PCIE_MSI1_CH0 0x1004 +#define SYS_PCIE_MSI2_CH0 0x1008 +#define SYS_PCIE_MSI3_CH0 0x100c +#define SYS_PCIE_MSI4_CH0 0x1010 +#define SYS_PCIE_MSI5_CH0 0x1014 +#define SYS_PCIE_PME_CH0 0x1018 +#define SYS_PCIE_ACK_CH0 0x101c +#define SYS_PCIE_MISC_CH0 0x1020 +#define SYS_PCIE_MODE_CH0 0x1024 +#define SYS_ADC_CFG 0x1600 + static void rzg3e_sys_print_id(struct device *dev, void __iomem *sysc_base, struct soc_device_attribute *soc_dev_attr) @@ -62,7 +83,65 @@ static const struct rz_sysc_soc_id_init_data rzg3e_sys_soc_id_init_data __initco .print_id = rzg3e_sys_print_id, }; +static bool rzg3e_regmap_readable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_LSI_OTPTSU1TRMVAL0: + case SYS_LSI_OTPTSU1TRMVAL1: + case SYS_SPI_STAADDCS0: + case SYS_SPI_ENDADDCS0: + case SYS_SPI_STAADDCS1: + case SYS_SPI_ENDADDCS1: + case SYS_VSP_CLK: + case SYS_GBETH0_CFG: + case SYS_GBETH1_CFG: + case SYS_PCIE_INTX_CH0: + case SYS_PCIE_MSI1_CH0: + case SYS_PCIE_MSI2_CH0: + case SYS_PCIE_MSI3_CH0: + case SYS_PCIE_MSI4_CH0: + case SYS_PCIE_MSI5_CH0: + case SYS_PCIE_PME_CH0: + case SYS_PCIE_ACK_CH0: + case SYS_PCIE_MISC_CH0: + case SYS_PCIE_MODE_CH0: + case SYS_ADC_CFG: + return true; + default: + return false; + } +} + +static bool rzg3e_regmap_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_SPI_STAADDCS0: + case SYS_SPI_ENDADDCS0: + case SYS_SPI_STAADDCS1: + case SYS_SPI_ENDADDCS1: + case SYS_VSP_CLK: + case SYS_GBETH0_CFG: + case SYS_GBETH1_CFG: + case SYS_PCIE_INTX_CH0: + case SYS_PCIE_MSI1_CH0: + case SYS_PCIE_MSI2_CH0: + case SYS_PCIE_MSI3_CH0: + case SYS_PCIE_MSI4_CH0: + case SYS_PCIE_MSI5_CH0: + case SYS_PCIE_PME_CH0: + case SYS_PCIE_ACK_CH0: + case SYS_PCIE_MISC_CH0: + case SYS_PCIE_MODE_CH0: + case SYS_ADC_CFG: + return true; + default: + return false; + } +} + const struct rz_sysc_init_data rzg3e_sys_init_data = { .soc_id_init_data = &rzg3e_sys_soc_id_init_data, + .readable_reg = rzg3e_regmap_readable_reg, + .writeable_reg = rzg3e_regmap_writeable_reg, .max_register = 0x170c, }; diff --git a/drivers/soc/renesas/r9a09g056-sys.c b/drivers/soc/renesas/r9a09g056-sys.c index 3ad1422eba36e..42f5eff291fd1 100644 --- a/drivers/soc/renesas/r9a09g056-sys.c +++ b/drivers/soc/renesas/r9a09g056-sys.c @@ -34,6 +34,24 @@ #define SYS_RZV2N_FEATURE_C55 BIT(1) #define SYS_RZV2N_FEATURE_SEC BIT(2) +#define SYS_LSI_OTPTSU0TRMVAL0 0x320 +#define SYS_LSI_OTPTSU0TRMVAL1 0x324 +#define SYS_LSI_OTPTSU1TRMVAL0 0x330 +#define SYS_LSI_OTPTSU1TRMVAL1 0x334 +#define SYS_GBETH0_CFG 0xf00 +#define SYS_GBETH1_CFG 0xf04 +#define SYS_PCIE_INTX_CH0 0x1000 +#define SYS_PCIE_MSI1_CH0 0x1004 +#define SYS_PCIE_MSI2_CH0 0x1008 +#define SYS_PCIE_MSI3_CH0 0x100c +#define SYS_PCIE_MSI4_CH0 0x1010 +#define SYS_PCIE_MSI5_CH0 0x1014 +#define SYS_PCIE_PME_CH0 0x1018 +#define SYS_PCIE_ACK_CH0 0x101c +#define SYS_PCIE_MISC_CH0 0x1020 +#define SYS_PCIE_MODE_CH0 0x1024 +#define SYS_ADC_CFG 0x1600 + static void rzv2n_sys_print_id(struct device *dev, void __iomem *sysc_base, struct soc_device_attribute *soc_dev_attr) @@ -70,6 +88,57 @@ static const struct rz_sysc_soc_id_init_data rzv2n_sys_soc_id_init_data __initco .print_id = rzv2n_sys_print_id, }; +static bool rzv2n_regmap_readable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_LSI_OTPTSU0TRMVAL0: + case SYS_LSI_OTPTSU0TRMVAL1: + case SYS_LSI_OTPTSU1TRMVAL0: + case SYS_LSI_OTPTSU1TRMVAL1: + case SYS_GBETH0_CFG: + case SYS_GBETH1_CFG: + case SYS_PCIE_INTX_CH0: + case SYS_PCIE_MSI1_CH0: + case SYS_PCIE_MSI2_CH0: + case SYS_PCIE_MSI3_CH0: + case SYS_PCIE_MSI4_CH0: + case SYS_PCIE_MSI5_CH0: + case SYS_PCIE_PME_CH0: + case SYS_PCIE_ACK_CH0: + case SYS_PCIE_MISC_CH0: + case SYS_PCIE_MODE_CH0: + case SYS_ADC_CFG: + return true; + default: + return false; + } +} + +static bool rzv2n_regmap_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_GBETH0_CFG: + case SYS_GBETH1_CFG: + case SYS_PCIE_INTX_CH0: + case SYS_PCIE_MSI1_CH0: + case SYS_PCIE_MSI2_CH0: + case SYS_PCIE_MSI3_CH0: + case SYS_PCIE_MSI4_CH0: + case SYS_PCIE_MSI5_CH0: + case SYS_PCIE_PME_CH0: + case SYS_PCIE_ACK_CH0: + case SYS_PCIE_MISC_CH0: + case SYS_PCIE_MODE_CH0: + case SYS_ADC_CFG: + return true; + default: + return false; + } +} + const struct rz_sysc_init_data rzv2n_sys_init_data = { .soc_id_init_data = &rzv2n_sys_soc_id_init_data, + .readable_reg = rzv2n_regmap_readable_reg, + .writeable_reg = rzv2n_regmap_writeable_reg, + .max_register = 0x170c, }; diff --git a/drivers/soc/renesas/r9a09g057-sys.c b/drivers/soc/renesas/r9a09g057-sys.c index e3390e7c7fe51..827c718ac7c54 100644 --- a/drivers/soc/renesas/r9a09g057-sys.c +++ b/drivers/soc/renesas/r9a09g057-sys.c @@ -29,6 +29,35 @@ #define SYS_LSI_PRR_GPU_DIS BIT(0) #define SYS_LSI_PRR_ISP_DIS BIT(4) +#define SYS_LSI_OTPTSU0TRMVAL0 0x320 +#define SYS_LSI_OTPTSU0TRMVAL1 0x324 +#define SYS_LSI_OTPTSU1TRMVAL0 0x330 +#define SYS_LSI_OTPTSU1TRMVAL1 0x334 +#define SYS_GBETH0_CFG 0xf00 +#define SYS_GBETH1_CFG 0xf04 +#define SYS_PCIE_INTX_CH0 0x1000 +#define SYS_PCIE_MSI1_CH0 0x1004 +#define SYS_PCIE_MSI2_CH0 0x1008 +#define SYS_PCIE_MSI3_CH0 0x100c +#define SYS_PCIE_MSI4_CH0 0x1010 +#define SYS_PCIE_MSI5_CH0 0x1014 +#define SYS_PCIE_PME_CH0 0x1018 +#define SYS_PCIE_ACK_CH0 0x101c +#define SYS_PCIE_MISC_CH0 0x1020 +#define SYS_PCIE_MODE_CH0 0x1024 +#define SYS_PCIE_INTX_CH1 0x1030 +#define SYS_PCIE_MSI1_CH1 0x1034 +#define SYS_PCIE_MSI2_CH1 0x1038 +#define SYS_PCIE_MSI3_CH1 0x103c +#define SYS_PCIE_MSI4_CH1 0x1040 +#define SYS_PCIE_MSI5_CH1 0x1044 +#define SYS_PCIE_PME_CH1 0x1048 +#define SYS_PCIE_ACK_CH1 0x104c +#define SYS_PCIE_MISC_CH1 0x1050 +#define SYS_PCIE_MODE_CH1 0x1054 +#define SYS_PCIE_MODE 0x1060 +#define SYS_ADC_CFG 0x1600 + static void rzv2h_sys_print_id(struct device *dev, void __iomem *sysc_base, struct soc_device_attribute *soc_dev_attr) @@ -62,7 +91,79 @@ static const struct rz_sysc_soc_id_init_data rzv2h_sys_soc_id_init_data __initco .print_id = rzv2h_sys_print_id, }; +static bool rzv2h_regmap_readable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_LSI_OTPTSU0TRMVAL0: + case SYS_LSI_OTPTSU0TRMVAL1: + case SYS_LSI_OTPTSU1TRMVAL0: + case SYS_LSI_OTPTSU1TRMVAL1: + case SYS_GBETH0_CFG: + case SYS_GBETH1_CFG: + case SYS_PCIE_INTX_CH0: + case SYS_PCIE_MSI1_CH0: + case SYS_PCIE_MSI2_CH0: + case SYS_PCIE_MSI3_CH0: + case SYS_PCIE_MSI4_CH0: + case SYS_PCIE_MSI5_CH0: + case SYS_PCIE_PME_CH0: + case SYS_PCIE_ACK_CH0: + case SYS_PCIE_MISC_CH0: + case SYS_PCIE_MODE_CH0: + case SYS_PCIE_INTX_CH1: + case SYS_PCIE_MSI1_CH1: + case SYS_PCIE_MSI2_CH1: + case SYS_PCIE_MSI3_CH1: + case SYS_PCIE_MSI4_CH1: + case SYS_PCIE_MSI5_CH1: + case SYS_PCIE_PME_CH1: + case SYS_PCIE_ACK_CH1: + case SYS_PCIE_MISC_CH1: + case SYS_PCIE_MODE_CH1: + case SYS_PCIE_MODE: + case SYS_ADC_CFG: + return true; + default: + return false; + } +} + +static bool rzv2h_regmap_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SYS_GBETH0_CFG: + case SYS_GBETH1_CFG: + case SYS_PCIE_INTX_CH0: + case SYS_PCIE_MSI1_CH0: + case SYS_PCIE_MSI2_CH0: + case SYS_PCIE_MSI3_CH0: + case SYS_PCIE_MSI4_CH0: + case SYS_PCIE_MSI5_CH0: + case SYS_PCIE_PME_CH0: + case SYS_PCIE_ACK_CH0: + case SYS_PCIE_MISC_CH0: + case SYS_PCIE_MODE_CH0: + case SYS_PCIE_INTX_CH1: + case SYS_PCIE_MSI1_CH1: + case SYS_PCIE_MSI2_CH1: + case SYS_PCIE_MSI3_CH1: + case SYS_PCIE_MSI4_CH1: + case SYS_PCIE_MSI5_CH1: + case SYS_PCIE_PME_CH1: + case SYS_PCIE_ACK_CH1: + case SYS_PCIE_MISC_CH1: + case SYS_PCIE_MODE_CH1: + case SYS_PCIE_MODE: + case SYS_ADC_CFG: + return true; + default: + return false; + } +} + const struct rz_sysc_init_data rzv2h_sys_init_data = { .soc_id_init_data = &rzv2h_sys_soc_id_init_data, + .readable_reg = rzv2h_regmap_readable_reg, + .writeable_reg = rzv2h_regmap_writeable_reg, .max_register = 0x170c, }; diff --git a/drivers/soc/renesas/rz-sysc.c b/drivers/soc/renesas/rz-sysc.c index 9f79e299e6f41..19c1e666279b7 100644 --- a/drivers/soc/renesas/rz-sysc.c +++ b/drivers/soc/renesas/rz-sysc.c @@ -140,6 +140,8 @@ static int rz_sysc_probe(struct platform_device *pdev) regmap_cfg->val_bits = 32; regmap_cfg->fast_io = true; regmap_cfg->max_register = data->max_register; + regmap_cfg->readable_reg = data->readable_reg; + regmap_cfg->writeable_reg = data->writeable_reg; regmap = devm_regmap_init_mmio(dev, sysc->base, regmap_cfg); if (IS_ERR(regmap)) diff --git a/drivers/soc/renesas/rz-sysc.h b/drivers/soc/renesas/rz-sysc.h index 8eec355d5d562..88929bf21cb11 100644 --- a/drivers/soc/renesas/rz-sysc.h +++ b/drivers/soc/renesas/rz-sysc.h @@ -34,10 +34,14 @@ struct rz_sysc_soc_id_init_data { /** * struct rz_sysc_init_data - RZ SYSC initialization data * @soc_id_init_data: RZ SYSC SoC ID initialization data + * @writeable_reg: Regmap writeable register check function + * @readable_reg: Regmap readable register check function * @max_register: Maximum SYSC register offset to be used by the regmap config */ struct rz_sysc_init_data { const struct rz_sysc_soc_id_init_data *soc_id_init_data; + bool (*writeable_reg)(struct device *dev, unsigned int reg); + bool (*readable_reg)(struct device *dev, unsigned int reg); u32 max_register; }; diff --git a/drivers/soc/rockchip/grf.c b/drivers/soc/rockchip/grf.c index 344870da7675f..1f070e0becb52 100644 --- a/drivers/soc/rockchip/grf.c +++ b/drivers/soc/rockchip/grf.c @@ -134,7 +134,7 @@ static const struct rockchip_grf_info rk3576_sysgrf __initconst = { .num_values = ARRAY_SIZE(rk3576_defaults_sys_grf), }; -#define RK3576_IOCGRF_MISC_CON 0x04F0 +#define RK3576_IOCGRF_MISC_CON 0x40F0 static const struct rockchip_grf_value rk3576_defaults_ioc_grf[] __initconst = { { "jtag switching", RK3576_IOCGRF_MISC_CON, FIELD_PREP_WM16_CONST(BIT(1), 0) }, @@ -202,34 +202,34 @@ static int __init rockchip_grf_init(void) struct regmap *grf; int ret, i; - np = of_find_matching_node_and_match(NULL, rockchip_grf_dt_match, - &match); - if (!np) - return -ENODEV; - if (!match || !match->data) { - pr_err("%s: missing grf data\n", __func__); - of_node_put(np); - return -EINVAL; - } - - grf_info = match->data; - - grf = syscon_node_to_regmap(np); - of_node_put(np); - if (IS_ERR(grf)) { - pr_err("%s: could not get grf syscon\n", __func__); - return PTR_ERR(grf); - } - - for (i = 0; i < grf_info->num_values; i++) { - const struct rockchip_grf_value *val = &grf_info->values[i]; - - pr_debug("%s: adjusting %s in %#6x to %#10x\n", __func__, - val->desc, val->reg, val->val); - ret = regmap_write(grf, val->reg, val->val); - if (ret < 0) - pr_err("%s: write to %#6x failed with %d\n", - __func__, val->reg, ret); + for_each_matching_node_and_match(np, rockchip_grf_dt_match, &match) { + if (!of_device_is_available(np)) + continue; + if (!match || !match->data) { + pr_err("%s: missing grf data\n", __func__); + of_node_put(np); + return -EINVAL; + } + + grf_info = match->data; + + grf = syscon_node_to_regmap(np); + if (IS_ERR(grf)) { + pr_err("%s: could not get grf syscon\n", __func__); + of_node_put(np); + return PTR_ERR(grf); + } + + for (i = 0; i < grf_info->num_values; i++) { + const struct rockchip_grf_value *val = &grf_info->values[i]; + + pr_debug("%s: adjusting %s in %#6x to %#10x\n", __func__, + val->desc, val->reg, val->val); + ret = regmap_write(grf, val->reg, val->val); + if (ret < 0) + pr_err("%s: write to %#6x failed with %d\n", + __func__, val->reg, ret); + } } return 0; diff --git a/drivers/soc/samsung/exynos-pmu.c b/drivers/soc/samsung/exynos-pmu.c index 22c50ca2aa79b..f8fe1a5965ab4 100644 --- a/drivers/soc/samsung/exynos-pmu.c +++ b/drivers/soc/samsung/exynos-pmu.c @@ -346,6 +346,8 @@ struct regmap *exynos_get_pmu_regmap_by_phandle(struct device_node *np, if (!dev) return ERR_PTR(-EPROBE_DEFER); + put_device(dev); + return syscon_node_to_regmap(pmu_np); } EXPORT_SYMBOL_GPL(exynos_get_pmu_regmap_by_phandle); @@ -585,10 +587,6 @@ static int setup_cpuhp_and_cpuidle(struct device *dev) if (!pmu_context->in_cpuhp) return -ENOMEM; - raw_spin_lock_init(&pmu_context->cpupm_lock); - pmu_context->sys_inreboot = false; - pmu_context->sys_insuspend = false; - /* set PMU to power on */ for_each_online_cpu(cpu) gs101_cpuhp_pmu_online(cpu); @@ -657,6 +655,9 @@ static int exynos_pmu_probe(struct platform_device *pdev) pmu_context->pmureg = regmap; pmu_context->dev = dev; + raw_spin_lock_init(&pmu_context->cpupm_lock); + pmu_context->sys_inreboot = false; + pmu_context->sys_insuspend = false; if (pmu_context->pmu_data && pmu_context->pmu_data->pmu_cpuhp) { ret = setup_cpuhp_and_cpuidle(dev); diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c index d276672838465..74d2fedea71ca 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra.c +++ b/drivers/soc/tegra/fuse/fuse-tegra.c @@ -182,8 +182,6 @@ static int tegra_fuse_probe(struct platform_device *pdev) } fuse->soc->init(fuse); - tegra_fuse_print_sku_info(&tegra_sku_info); - tegra_soc_device_register(); err = tegra_fuse_add_lookups(fuse); if (err) diff --git a/drivers/soc/tegra/fuse/speedo-tegra210.c b/drivers/soc/tegra/fuse/speedo-tegra210.c index 695d0b7f9a8ab..a8cc363297723 100644 --- a/drivers/soc/tegra/fuse/speedo-tegra210.c +++ b/drivers/soc/tegra/fuse/speedo-tegra210.c @@ -65,27 +65,51 @@ static void __init rev_sku_to_speedo_ids(struct tegra_sku_info *sku_info, sku_info->gpu_speedo_id = 0; *threshold = THRESHOLD_INDEX_0; - switch (sku) { - case 0x00: /* Engineering SKU */ - case 0x01: /* Engineering SKU */ - case 0x07: - case 0x17: - case 0x27: - if (speedo_rev >= 2) + if (sku_info->revision >= TEGRA_REVISION_A02) { + switch (sku) { + case 0x00: /* Engineering SKU */ + case 0x01: /* Engineering SKU */ + case 0x13: + sku_info->cpu_speedo_id = 5; + sku_info->gpu_speedo_id = 2; + break; + + case 0x07: + case 0x17: + case 0x1F: + sku_info->cpu_speedo_id = 7; + sku_info->gpu_speedo_id = 2; + break; + + case 0x27: + sku_info->cpu_speedo_id = 1; + sku_info->gpu_speedo_id = 2; + break; + + case 0x83: + sku_info->cpu_speedo_id = 3; + sku_info->gpu_speedo_id = 3; + break; + + case 0x87: + sku_info->cpu_speedo_id = 2; sku_info->gpu_speedo_id = 1; - break; - - case 0x13: - if (speedo_rev >= 2) - sku_info->gpu_speedo_id = 1; - - sku_info->cpu_speedo_id = 1; - break; - - default: + break; + + case 0x8F: + sku_info->cpu_speedo_id = 9; + sku_info->gpu_speedo_id = 2; + break; + + default: + pr_err("Tegra210: unknown revision 2 or newer SKU %#04x\n", sku); + /* Using the default for the error case */ + break; + } + } else if (sku == 0x00 || sku == 0x01 || sku == 0x07 || sku == 0x13 || sku == 0x17) { + sku_info->gpu_speedo_id = 1; + } else { pr_err("Tegra210: unknown SKU %#04x\n", sku); - /* Using the default for the error case */ - break; } } diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c index 50c170a995f90..42275cb5ba1c8 100644 --- a/drivers/soc/ti/k3-socinfo.c +++ b/drivers/soc/ti/k3-socinfo.c @@ -141,7 +141,7 @@ static int k3_chipinfo_probe(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - regmap = regmap_init_mmio(dev, base, &k3_chipinfo_regmap_cfg); + regmap = devm_regmap_init_mmio(dev, base, &k3_chipinfo_regmap_cfg); if (IS_ERR(regmap)) return PTR_ERR(regmap); diff --git a/drivers/soc/ti/pruss.c b/drivers/soc/ti/pruss.c index 038576805bfa0..0fd59c73f585d 100644 --- a/drivers/soc/ti/pruss.c +++ b/drivers/soc/ti/pruss.c @@ -366,12 +366,10 @@ static int pruss_clk_mux_setup(struct pruss *pruss, struct clk *clk_mux, ret = devm_add_action_or_reset(dev, pruss_of_free_clk_provider, clk_mux_np); - if (ret) { + if (ret) dev_err(dev, "failed to add clkmux free action %d", ret); - goto put_clk_mux_np; - } - return 0; + return ret; put_clk_mux_np: of_node_put(clk_mux_np); diff --git a/drivers/soundwire/Kconfig b/drivers/soundwire/Kconfig index ad56393e4c93b..196a7daaabdb8 100644 --- a/drivers/soundwire/Kconfig +++ b/drivers/soundwire/Kconfig @@ -40,6 +40,7 @@ config SOUNDWIRE_INTEL select AUXILIARY_BUS depends on ACPI && SND_SOC depends on SND_SOC_SOF_HDA_MLINK || !SND_SOC_SOF_HDA_MLINK + depends on SND_HDA_CORE || !SND_HDA_ALIGNED_MMIO help SoundWire Intel Master driver. If you have an Intel platform which has a SoundWire Master then diff --git a/drivers/soundwire/bus_type.c b/drivers/soundwire/bus_type.c index 91e70cb46fb57..5c67c13e57357 100644 --- a/drivers/soundwire/bus_type.c +++ b/drivers/soundwire/bus_type.c @@ -105,7 +105,7 @@ static int sdw_drv_probe(struct device *dev) if (ret) return ret; - ret = ida_alloc_max(&slave->bus->slave_ida, SDW_FW_MAX_DEVICES, GFP_KERNEL); + ret = ida_alloc_max(&slave->bus->slave_ida, SDW_FW_MAX_DEVICES - 1, GFP_KERNEL); if (ret < 0) { dev_err(dev, "Failed to allocated ID: %d\n", ret); return ret; diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c index 91ab97a456fa9..5854218e1a274 100644 --- a/drivers/soundwire/dmi-quirks.c +++ b/drivers/soundwire/dmi-quirks.c @@ -122,6 +122,17 @@ static const struct dmi_system_id adr_remap_quirk_table[] = { }, .driver_data = (void *)intel_tgl_bios, }, + { + /* + * quirk used for Avell B.ON (OEM rebrand of NUC15 'Bishop County' + * LAPBC510 and LAPBC710) + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Avell High Performance"), + DMI_MATCH(DMI_PRODUCT_NAME, "B.ON"), + }, + .driver_data = (void *)intel_tgl_bios, + }, { /* quirk used for NUC15 'Rooks County' LAPRC510 and LAPRC710 skews */ .matches = { diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c index 6df2601fff909..8752b0e3ce74c 100644 --- a/drivers/soundwire/intel_auxdevice.c +++ b/drivers/soundwire/intel_auxdevice.c @@ -52,6 +52,7 @@ struct wake_capable_part { static struct wake_capable_part wake_capable_list[] = { {0x01fa, 0x4243}, + {0x01fa, 0x4245}, {0x025d, 0x5682}, {0x025d, 0x700}, {0x025d, 0x711}, diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 55675750182e3..1872f9d54a5cc 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -706,15 +706,6 @@ config SPI_MESON_SPIFC This enables master mode support for the SPIFC (SPI flash controller) available in Amlogic Meson SoCs. -config SPI_MICROCHIP_CORE - tristate "Microchip FPGA SPI controllers" - depends on SPI_MASTER - help - This enables the SPI driver for Microchip FPGA SPI controllers. - Say Y or M here if you want to use the "hard" controllers on - PolarFire SoC. - If built as a module, it will be called spi-microchip-core. - config SPI_MICROCHIP_CORE_QSPI tristate "Microchip FPGA QSPI controllers" depends on SPI_MASTER @@ -871,6 +862,16 @@ config SPI_PL022 controller. If you have an embedded system with an AMBA(R) bus and a PL022 controller, say Y or M here. +config SPI_POLARFIRE_SOC + + tristate "Microchip FPGA SPI controllers" + depends on SPI_MASTER && ARCH_MICROCHIP + help + This enables the SPI driver for Microchip FPGA SPI controllers. + Say Y or M here if you want to use the "hard" controllers on + PolarFire SoC. + If built as a module, it will be called spi-mpfs. + config SPI_PPC4xx tristate "PPC4xx SPI Controller" depends on PPC32 && 4xx diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index 8ff74a13faaa8..1f7c06a3091d9 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -86,7 +86,6 @@ obj-$(CONFIG_SPI_LOONGSON_PLATFORM) += spi-loongson-plat.o obj-$(CONFIG_SPI_LP8841_RTC) += spi-lp8841-rtc.o obj-$(CONFIG_SPI_MESON_SPICC) += spi-meson-spicc.o obj-$(CONFIG_SPI_MESON_SPIFC) += spi-meson-spifc.o -obj-$(CONFIG_SPI_MICROCHIP_CORE) += spi-microchip-core.o obj-$(CONFIG_SPI_MICROCHIP_CORE_QSPI) += spi-microchip-core-qspi.o obj-$(CONFIG_SPI_MPC512x_PSC) += spi-mpc512x-psc.o obj-$(CONFIG_SPI_MPC52xx_PSC) += spi-mpc52xx-psc.o @@ -97,6 +96,7 @@ obj-$(CONFIG_SPI_MTK_NOR) += spi-mtk-nor.o obj-$(CONFIG_SPI_MTK_SNFI) += spi-mtk-snfi.o obj-$(CONFIG_SPI_MXIC) += spi-mxic.o obj-$(CONFIG_SPI_MXS) += spi-mxs.o +obj-$(CONFIG_SPI_POLARFIRE_SOC) += spi-mpfs.o obj-$(CONFIG_SPI_WPCM_FIU) += spi-wpcm-fiu.o obj-$(CONFIG_SPI_NPCM_FIU) += spi-npcm-fiu.o obj-$(CONFIG_SPI_NPCM_PSPI) += spi-npcm-pspi.o diff --git a/drivers/spi/spi-airoha-snfi.c b/drivers/spi/spi-airoha-snfi.c index b78163eaed61d..20b5d469d519a 100644 --- a/drivers/spi/spi-airoha-snfi.c +++ b/drivers/spi/spi-airoha-snfi.c @@ -1030,6 +1030,11 @@ static const struct spi_controller_mem_ops airoha_snand_mem_ops = { .dirmap_write = airoha_snand_dirmap_write, }; +static const struct spi_controller_mem_ops airoha_snand_nodma_mem_ops = { + .supports_op = airoha_snand_supports_op, + .exec_op = airoha_snand_exec_op, +}; + static int airoha_snand_setup(struct spi_device *spi) { struct airoha_snand_ctrl *as_ctrl; @@ -1104,7 +1109,9 @@ static int airoha_snand_probe(struct platform_device *pdev) struct airoha_snand_ctrl *as_ctrl; struct device *dev = &pdev->dev; struct spi_controller *ctrl; + bool dma_enable = true; void __iomem *base; + u32 sfc_strap; int err; ctrl = devm_spi_alloc_host(dev, sizeof(*as_ctrl)); @@ -1139,12 +1146,28 @@ static int airoha_snand_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(as_ctrl->spi_clk), "unable to get spi clk\n"); + if (device_is_compatible(dev, "airoha,en7523-snand")) { + err = regmap_read(as_ctrl->regmap_ctrl, + REG_SPI_CTRL_SFC_STRAP, &sfc_strap); + if (err) + return err; + + if (!(sfc_strap & 0x04)) { + dma_enable = false; + dev_warn(dev, "Detected booting in RESERVED mode (UART_TXD was short to GND).\n"); + dev_warn(dev, "This mode is known for incorrect DMA reading of some flashes.\n"); + dev_warn(dev, "Much slower PIO mode will be used to prevent flash data damage.\n"); + dev_warn(dev, "Unplug UART cable and power cycle board to get full performance.\n"); + } + } + err = dma_set_mask(as_ctrl->dev, DMA_BIT_MASK(32)); if (err) return err; ctrl->num_chipselect = 2; - ctrl->mem_ops = &airoha_snand_mem_ops; + ctrl->mem_ops = dma_enable ? &airoha_snand_mem_ops + : &airoha_snand_nodma_mem_ops; ctrl->bits_per_word_mask = SPI_BPW_MASK(8); ctrl->mode_bits = SPI_RX_DUAL; ctrl->setup = airoha_snand_setup; diff --git a/drivers/spi/spi-amlogic-spifc-a4.c b/drivers/spi/spi-amlogic-spifc-a4.c index 35a7c4965e113..3393e1f305709 100644 --- a/drivers/spi/spi-amlogic-spifc-a4.c +++ b/drivers/spi/spi-amlogic-spifc-a4.c @@ -411,7 +411,7 @@ static int aml_sfc_dma_buffer_setup(struct aml_sfc *sfc, void *databuf, ret = dma_mapping_error(sfc->dev, sfc->daddr); if (ret) { dev_err(sfc->dev, "DMA mapping error\n"); - goto out_map_data; + return ret; } cmd = CMD_DATA_ADDRL(sfc->daddr); @@ -429,7 +429,6 @@ static int aml_sfc_dma_buffer_setup(struct aml_sfc *sfc, void *databuf, ret = dma_mapping_error(sfc->dev, sfc->iaddr); if (ret) { dev_err(sfc->dev, "DMA mapping error\n"); - dma_unmap_single(sfc->dev, sfc->daddr, datalen, dir); goto out_map_data; } @@ -448,7 +447,7 @@ static int aml_sfc_dma_buffer_setup(struct aml_sfc *sfc, void *databuf, return 0; out_map_info: - dma_unmap_single(sfc->dev, sfc->iaddr, datalen, dir); + dma_unmap_single(sfc->dev, sfc->iaddr, infolen, dir); out_map_data: dma_unmap_single(sfc->dev, sfc->daddr, datalen, dir); @@ -1067,6 +1066,13 @@ static const struct nand_ecc_engine_ops aml_sfc_ecc_engine_ops = { .finish_io_req = aml_sfc_ecc_finish_io_req, }; +static void aml_sfc_unregister_ecc_engine(void *data) +{ + struct nand_ecc_engine *eng = data; + + nand_ecc_unregister_on_host_hw_engine(eng); +} + static int aml_sfc_clk_init(struct aml_sfc *sfc) { sfc->gate_clk = devm_clk_get_enabled(sfc->dev, "gate"); @@ -1084,14 +1090,6 @@ static int aml_sfc_clk_init(struct aml_sfc *sfc) return clk_set_rate(sfc->core_clk, SFC_BUS_DEFAULT_CLK); } -static int aml_sfc_disable_clk(struct aml_sfc *sfc) -{ - clk_disable_unprepare(sfc->core_clk); - clk_disable_unprepare(sfc->gate_clk); - - return 0; -} - static int aml_sfc_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -1142,16 +1140,12 @@ static int aml_sfc_probe(struct platform_device *pdev) /* Enable Amlogic flash controller spi mode */ ret = regmap_write(sfc->regmap_base, SFC_SPI_CFG, SPI_MODE_EN); - if (ret) { - dev_err(dev, "failed to enable SPI mode\n"); - goto err_out; - } + if (ret) + return dev_err_probe(dev, ret, "failed to enable SPI mode\n"); ret = dma_set_mask(sfc->dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(sfc->dev, "failed to set dma mask\n"); - goto err_out; - } + if (ret) + return dev_err_probe(sfc->dev, ret, "failed to set dma mask\n"); sfc->ecc_eng.dev = &pdev->dev; sfc->ecc_eng.integration = NAND_ECC_ENGINE_INTEGRATION_PIPELINED; @@ -1159,10 +1153,13 @@ static int aml_sfc_probe(struct platform_device *pdev) sfc->ecc_eng.priv = sfc; ret = nand_ecc_register_on_host_hw_engine(&sfc->ecc_eng); - if (ret) { - dev_err(&pdev->dev, "failed to register Aml host ecc engine.\n"); - goto err_out; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to register Aml host ecc engine.\n"); + + ret = devm_add_action_or_reset(dev, aml_sfc_unregister_ecc_engine, + &sfc->ecc_eng); + if (ret) + return dev_err_probe(dev, ret, "failed to add ECC unregister action\n"); ret = of_property_read_u32(np, "amlogic,rx-adj", &val); if (!ret) @@ -1178,24 +1175,7 @@ static int aml_sfc_probe(struct platform_device *pdev) ctrl->min_speed_hz = SFC_MIN_FREQUENCY; ctrl->num_chipselect = SFC_MAX_CS_NUM; - ret = devm_spi_register_controller(dev, ctrl); - if (ret) - goto err_out; - - return 0; - -err_out: - aml_sfc_disable_clk(sfc); - - return ret; -} - -static void aml_sfc_remove(struct platform_device *pdev) -{ - struct spi_controller *ctlr = platform_get_drvdata(pdev); - struct aml_sfc *sfc = spi_controller_get_devdata(ctlr); - - aml_sfc_disable_clk(sfc); + return devm_spi_register_controller(dev, ctrl); } static const struct of_device_id aml_sfc_of_match[] = { @@ -1213,7 +1193,6 @@ static struct platform_driver aml_sfc_driver = { .of_match_table = aml_sfc_of_match, }, .probe = aml_sfc_probe, - .remove = aml_sfc_remove, }; module_platform_driver(aml_sfc_driver); diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c index bcd7ec291ad07..6045c89c37c83 100644 --- a/drivers/spi/spi-amlogic-spisg.c +++ b/drivers/spi/spi-amlogic-spisg.c @@ -729,9 +729,9 @@ static int aml_spisg_probe(struct platform_device *pdev) }; if (of_property_read_bool(dev->of_node, "spi-slave")) - ctlr = spi_alloc_target(dev, sizeof(*spisg)); + ctlr = devm_spi_alloc_target(dev, sizeof(*spisg)); else - ctlr = spi_alloc_host(dev, sizeof(*spisg)); + ctlr = devm_spi_alloc_host(dev, sizeof(*spisg)); if (!ctlr) return -ENOMEM; @@ -750,10 +750,8 @@ static int aml_spisg_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(spisg->map), "regmap init failed\n"); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - ret = irq; - goto out_controller; - } + if (irq < 0) + return irq; ret = device_reset_optional(dev); if (ret) @@ -818,8 +816,6 @@ static int aml_spisg_probe(struct platform_device *pdev) if (spisg->core) clk_disable_unprepare(spisg->core); clk_disable_unprepare(spisg->pclk); -out_controller: - spi_controller_put(ctlr); return ret; } diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index af6d050da1c8a..d61bc678b6f83 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -769,6 +769,7 @@ static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata, readl(reg_base + CQSPI_REG_INDIRECTRD); /* Flush posted write. */ while (remaining > 0) { + ret = 0; if (use_irq && !wait_for_completion_timeout(&cqspi->transfer_complete, msecs_to_jiffies(CQSPI_READ_TIMEOUT_MS))) @@ -781,6 +782,14 @@ static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata, if (cqspi->slow_sram) writel(0x0, reg_base + CQSPI_REG_IRQMASK); + /* + * Prevent lost interrupt and race condition by reinitializing early. + * A spurious wakeup and another wait cycle can occur here, + * which is preferable to waiting until timeout if interrupt is lost. + */ + if (use_irq) + reinit_completion(&cqspi->transfer_complete); + bytes_to_read = cqspi_get_rd_sram_level(cqspi); if (ret && bytes_to_read == 0) { @@ -813,7 +822,6 @@ static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata, } if (use_irq && remaining > 0) { - reinit_completion(&cqspi->transfer_complete); if (cqspi->slow_sram) writel(CQSPI_REG_IRQ_WATERMARK, reg_base + CQSPI_REG_IRQMASK); } @@ -1479,14 +1487,6 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) if (refcount_read(&cqspi->inflight_ops) == 0) return -ENODEV; - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - ret = pm_runtime_resume_and_get(dev); - if (ret) { - dev_err(&mem->spi->dev, "resume failed with %d\n", ret); - return ret; - } - } - if (!refcount_read(&cqspi->refcount)) return -EBUSY; @@ -1498,6 +1498,14 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) return -EBUSY; } + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { + ret = pm_runtime_resume_and_get(dev); + if (ret) { + dev_err(&mem->spi->dev, "resume failed with %d\n", ret); + goto dec_inflight_refcount; + } + } + ret = cqspi_mem_process(mem, op); if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) @@ -1506,6 +1514,7 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) if (ret) dev_err(&mem->spi->dev, "operation failed with %d\n", ret); +dec_inflight_refcount: if (refcount_read(&cqspi->inflight_ops) > 1) refcount_dec(&cqspi->inflight_ops); @@ -1845,6 +1854,12 @@ static int cqspi_probe(struct platform_device *pdev) return -ENODEV; } + ret = cqspi_setup_flash(cqspi); + if (ret) { + dev_err(dev, "failed to setup flash parameters %d\n", ret); + return ret; + } + /* Obtain QSPI clock. */ cqspi->clk = devm_clk_get(dev, NULL); if (IS_ERR(cqspi->clk)) { @@ -1988,12 +2003,6 @@ static int cqspi_probe(struct platform_device *pdev) pm_runtime_get_noresume(dev); } - ret = cqspi_setup_flash(cqspi); - if (ret) { - dev_err(dev, "failed to setup flash parameters %d\n", ret); - goto probe_setup_failed; - } - host->num_chipselect = cqspi->num_chipselect; if (ddata && (ddata->quirks & CQSPI_SUPPORT_DEVICE_RESET)) @@ -2024,7 +2033,9 @@ static int cqspi_probe(struct platform_device *pdev) probe_reset_failed: if (cqspi->is_jh7110) cqspi_jh7110_disable_clk(pdev, cqspi); - clk_disable_unprepare(cqspi->clk); + + if (pm_runtime_get_sync(&pdev->dev) >= 0) + clk_disable_unprepare(cqspi->clk); probe_clk_failed: return ret; } @@ -2034,6 +2045,7 @@ static void cqspi_remove(struct platform_device *pdev) const struct cqspi_driver_platdata *ddata; struct cqspi_st *cqspi = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; + int ret = 0; ddata = of_device_get_match_data(dev); @@ -2049,8 +2061,10 @@ static void cqspi_remove(struct platform_device *pdev) dma_release_channel(cqspi->rx_chan); if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) - if (pm_runtime_get_sync(&pdev->dev) >= 0) - clk_disable(cqspi->clk); + ret = pm_runtime_get_sync(&pdev->dev); + + if (ret >= 0) + clk_disable(cqspi->clk); if (cqspi->is_jh7110) cqspi_jh7110_disable_clk(pdev, cqspi); diff --git a/drivers/spi/spi-ch341.c b/drivers/spi/spi-ch341.c index 46bc208f2d050..79d2f9ab4ef03 100644 --- a/drivers/spi/spi-ch341.c +++ b/drivers/spi/spi-ch341.c @@ -78,7 +78,7 @@ static int ch341_transfer_one(struct spi_controller *host, ch341->tx_buf[0] = CH341A_CMD_SPI_STREAM; - memcpy(ch341->tx_buf + 1, trans->tx_buf, len); + memcpy(ch341->tx_buf + 1, trans->tx_buf, len - 1); ret = usb_bulk_msg(ch341->udev, ch341->write_pipe, ch341->tx_buf, len, NULL, CH341_DEFAULT_TIMEOUT); diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index 065456aba2aea..47d372557e4f6 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -972,7 +972,7 @@ static int fsl_lpspi_probe(struct platform_device *pdev) enable_irq(irq); } - ret = devm_spi_register_controller(&pdev->dev, controller); + ret = spi_register_controller(controller); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "spi_register_controller error\n"); goto free_dma; @@ -998,6 +998,7 @@ static void fsl_lpspi_remove(struct platform_device *pdev) struct fsl_lpspi_data *fsl_lpspi = spi_controller_get_devdata(controller); + spi_unregister_controller(controller); fsl_lpspi_dma_exit(controller); pm_runtime_dont_use_autosuspend(fsl_lpspi->dev); diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c index 2f2082652a1a2..481a7b28aacd3 100644 --- a/drivers/spi/spi-fsl-spi.c +++ b/drivers/spi/spi-fsl-spi.c @@ -335,7 +335,7 @@ static int fsl_spi_prepare_message(struct spi_controller *ctlr, if (t->bits_per_word == 16 || t->bits_per_word == 32) t->bits_per_word = 8; /* pretend its 8 bits */ if (t->bits_per_word == 8 && t->len >= 256 && - (mpc8xxx_spi->flags & SPI_CPM1)) + !(t->len & 1) && (mpc8xxx_spi->flags & SPI_CPM1)) t->bits_per_word = 16; } } diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index a0d8d3425c6c6..736120107184f 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -160,24 +160,20 @@ static void handle_se_timeout(struct spi_controller *spi, xfer = mas->cur_xfer; mas->cur_xfer = NULL; - if (spi->target) { - /* - * skip CMD Cancel sequnece since spi target - * doesn`t support CMD Cancel sequnece - */ + /* The controller doesn't support the Cancel commnand in target mode */ + if (!spi->target) { + reinit_completion(&mas->cancel_done); + geni_se_cancel_m_cmd(se); + spin_unlock_irq(&mas->lock); - goto reset_if_dma; - } - reinit_completion(&mas->cancel_done); - geni_se_cancel_m_cmd(se); - spin_unlock_irq(&mas->lock); + time_left = wait_for_completion_timeout(&mas->cancel_done, HZ); + if (time_left) + goto reset_if_dma; - time_left = wait_for_completion_timeout(&mas->cancel_done, HZ); - if (time_left) - goto reset_if_dma; + spin_lock_irq(&mas->lock); + } - spin_lock_irq(&mas->lock); reinit_completion(&mas->abort_done); geni_se_abort_m_cmd(se); spin_unlock_irq(&mas->lock); @@ -548,10 +544,10 @@ static u32 get_xfer_len_in_words(struct spi_transfer *xfer, { u32 len; - if (!(mas->cur_bits_per_word % MIN_WORD_LEN)) - len = xfer->len * BITS_PER_BYTE / mas->cur_bits_per_word; + if (!(xfer->bits_per_word % MIN_WORD_LEN)) + len = xfer->len * BITS_PER_BYTE / xfer->bits_per_word; else - len = xfer->len / (mas->cur_bits_per_word / BITS_PER_BYTE + 1); + len = xfer->len / (xfer->bits_per_word / BITS_PER_BYTE + 1); len &= TRANS_LEN_MSK; return len; @@ -571,7 +567,7 @@ static bool geni_can_dma(struct spi_controller *ctlr, return true; len = get_xfer_len_in_words(xfer, mas); - fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / mas->cur_bits_per_word; + fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / xfer->bits_per_word; if (len > fifo_size) return true; @@ -724,6 +720,12 @@ static int spi_geni_init(struct spi_geni_master *mas) case 0: mas->cur_xfer_mode = GENI_SE_FIFO; geni_se_select_mode(se, GENI_SE_FIFO); + /* setup_fifo_params assumes that these registers start with a zero value */ + writel(0, se->base + SE_SPI_LOOPBACK); + writel(0, se->base + SE_SPI_DEMUX_SEL); + writel(0, se->base + SE_SPI_CPHA); + writel(0, se->base + SE_SPI_CPOL); + writel(0, se->base + SE_SPI_DEMUX_OUTPUT_INV); ret = 0; break; } @@ -956,10 +958,13 @@ static irqreturn_t geni_spi_isr(int irq, void *data) struct spi_controller *spi = data; struct spi_geni_master *mas = spi_controller_get_devdata(spi); struct geni_se *se = &mas->se; - u32 m_irq; + u32 m_irq, dma_tx_status, dma_rx_status; m_irq = readl(se->base + SE_GENI_M_IRQ_STATUS); - if (!m_irq) + dma_tx_status = readl_relaxed(se->base + SE_DMA_TX_IRQ_STAT); + dma_rx_status = readl_relaxed(se->base + SE_DMA_RX_IRQ_STAT); + + if (!m_irq && !dma_tx_status && !dma_rx_status) return IRQ_NONE; if (m_irq & (M_CMD_OVERRUN_EN | M_ILLEGAL_CMD_EN | M_CMD_FAILURE_EN | @@ -1007,8 +1012,6 @@ static irqreturn_t geni_spi_isr(int irq, void *data) } } else if (mas->cur_xfer_mode == GENI_SE_DMA) { const struct spi_transfer *xfer = mas->cur_xfer; - u32 dma_tx_status = readl_relaxed(se->base + SE_DMA_TX_IRQ_STAT); - u32 dma_rx_status = readl_relaxed(se->base + SE_DMA_RX_IRQ_STAT); if (dma_tx_status) writel(dma_tx_status, se->base + SE_DMA_TX_IRQ_CLR); diff --git a/drivers/spi/spi-hisi-kunpeng.c b/drivers/spi/spi-hisi-kunpeng.c index dadf558dd9c0c..80a1a15de0bc3 100644 --- a/drivers/spi/spi-hisi-kunpeng.c +++ b/drivers/spi/spi-hisi-kunpeng.c @@ -161,10 +161,8 @@ static const struct debugfs_reg32 hisi_spi_regs[] = { static int hisi_spi_debugfs_init(struct hisi_spi *hs) { char name[32]; + struct spi_controller *host = dev_get_drvdata(hs->dev); - struct spi_controller *host; - - host = container_of(hs->dev, struct spi_controller, dev); snprintf(name, 32, "hisi_spi%d", host->bus_num); hs->debugfs = debugfs_create_dir(name, NULL); if (IS_ERR(hs->debugfs)) diff --git a/drivers/spi/spi-intel-pci.c b/drivers/spi/spi-intel-pci.c index b8c572394aac2..d8ef8f89330ac 100644 --- a/drivers/spi/spi-intel-pci.c +++ b/drivers/spi/spi-intel-pci.c @@ -81,6 +81,7 @@ static const struct pci_device_id intel_spi_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x54a4), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0x5794), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0x5825), (unsigned long)&cnl_info }, + { PCI_VDEVICE(INTEL, 0x6e24), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0x7723), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0x7a24), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0x7aa4), (unsigned long)&cnl_info }, @@ -95,6 +96,7 @@ static const struct pci_device_id intel_spi_pci_ids[] = { { PCI_VDEVICE(INTEL, 0xa324), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa3a4), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa823), (unsigned long)&cnl_info }, + { PCI_VDEVICE(INTEL, 0xd323), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xe323), (unsigned long)&cnl_info }, { PCI_VDEVICE(INTEL, 0xe423), (unsigned long)&cnl_info }, { }, diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c index 064b99204d9ac..db9467535416d 100644 --- a/drivers/spi/spi-mem.c +++ b/drivers/spi/spi-mem.c @@ -175,8 +175,19 @@ bool spi_mem_default_supports_op(struct spi_mem *mem, if (op->data.swap16 && !spi_mem_controller_is_capable(ctlr, swap16)) return false; - if (op->cmd.nbytes != 2) - return false; + /* Extra 8D-8D-8D limitations */ + if (op->cmd.dtr && op->cmd.buswidth == 8) { + if (op->cmd.nbytes != 2) + return false; + + if ((op->addr.nbytes % 2) || + (op->dummy.nbytes % 2) || + (op->data.nbytes % 2)) { + dev_err(&ctlr->dev, + "Even byte numbers not allowed in octal DTR operations\n"); + return false; + } + } } else { if (op->cmd.nbytes != 1) return false; @@ -703,9 +714,18 @@ spi_mem_dirmap_create(struct spi_mem *mem, desc->mem = mem; desc->info = *info; - if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) + if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) { + ret = spi_mem_access_start(mem); + if (ret) { + kfree(desc); + return ERR_PTR(ret); + } + ret = ctlr->mem_ops->dirmap_create(desc); + spi_mem_access_end(mem); + } + if (ret) { desc->nodirmap = true; if (!spi_mem_supports_op(desc->mem, &desc->info.op_tmpl)) diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c index 6b91373075334..c99fab392add1 100644 --- a/drivers/spi/spi-meson-spicc.c +++ b/drivers/spi/spi-meson-spicc.c @@ -1102,8 +1102,6 @@ static void meson_spicc_remove(struct platform_device *pdev) /* Disable SPI */ writel(0, spicc->base + SPICC_CONREG); - - spi_controller_put(spicc->host); } static const struct meson_spicc_data meson_spicc_gx_data = { diff --git a/drivers/spi/spi-microchip-core.c b/drivers/spi/spi-mpfs.c similarity index 68% rename from drivers/spi/spi-microchip-core.c rename to drivers/spi/spi-mpfs.c index 9128b86c53660..7e9e64d8e6c81 100644 --- a/drivers/spi/spi-microchip-core.c +++ b/drivers/spi/spi-mpfs.c @@ -99,7 +99,7 @@ #define REG_CTRL2 (0x48) #define REG_FRAMESUP (0x50) -struct mchp_corespi { +struct mpfs_spi { void __iomem *regs; struct clk *clk; const u8 *tx_buf; @@ -113,34 +113,34 @@ struct mchp_corespi { int n_bytes; }; -static inline u32 mchp_corespi_read(struct mchp_corespi *spi, unsigned int reg) +static inline u32 mpfs_spi_read(struct mpfs_spi *spi, unsigned int reg) { return readl(spi->regs + reg); } -static inline void mchp_corespi_write(struct mchp_corespi *spi, unsigned int reg, u32 val) +static inline void mpfs_spi_write(struct mpfs_spi *spi, unsigned int reg, u32 val) { writel(val, spi->regs + reg); } -static inline void mchp_corespi_disable(struct mchp_corespi *spi) +static inline void mpfs_spi_disable(struct mpfs_spi *spi) { - u32 control = mchp_corespi_read(spi, REG_CONTROL); + u32 control = mpfs_spi_read(spi, REG_CONTROL); control &= ~CONTROL_ENABLE; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); } -static inline void mchp_corespi_read_fifo(struct mchp_corespi *spi, int fifo_max) +static inline void mpfs_spi_read_fifo(struct mpfs_spi *spi, int fifo_max) { for (int i = 0; i < fifo_max; i++) { u32 data; - while (mchp_corespi_read(spi, REG_STATUS) & STATUS_RXFIFO_EMPTY) + while (mpfs_spi_read(spi, REG_STATUS) & STATUS_RXFIFO_EMPTY) ; - data = mchp_corespi_read(spi, REG_RX_DATA); + data = mpfs_spi_read(spi, REG_RX_DATA); spi->rx_len -= spi->n_bytes; @@ -158,34 +158,34 @@ static inline void mchp_corespi_read_fifo(struct mchp_corespi *spi, int fifo_max } } -static void mchp_corespi_enable_ints(struct mchp_corespi *spi) +static void mpfs_spi_enable_ints(struct mpfs_spi *spi) { - u32 control = mchp_corespi_read(spi, REG_CONTROL); + u32 control = mpfs_spi_read(spi, REG_CONTROL); control |= INT_ENABLE_MASK; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); } -static void mchp_corespi_disable_ints(struct mchp_corespi *spi) +static void mpfs_spi_disable_ints(struct mpfs_spi *spi) { - u32 control = mchp_corespi_read(spi, REG_CONTROL); + u32 control = mpfs_spi_read(spi, REG_CONTROL); control &= ~INT_ENABLE_MASK; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); } -static inline void mchp_corespi_set_xfer_size(struct mchp_corespi *spi, int len) +static inline void mpfs_spi_set_xfer_size(struct mpfs_spi *spi, int len) { u32 control; u32 lenpart; - u32 frames = mchp_corespi_read(spi, REG_FRAMESUP); + u32 frames = mpfs_spi_read(spi, REG_FRAMESUP); /* * Writing to FRAMECNT in REG_CONTROL will reset the frame count, taking * a shortcut requires an explicit clear. */ if (frames == len) { - mchp_corespi_write(spi, REG_COMMAND, COMMAND_CLRFRAMECNT); + mpfs_spi_write(spi, REG_COMMAND, COMMAND_CLRFRAMECNT); return; } @@ -208,20 +208,20 @@ static inline void mchp_corespi_set_xfer_size(struct mchp_corespi *spi, int len) * that matches the documentation. */ lenpart = len & 0xffff; - control = mchp_corespi_read(spi, REG_CONTROL); + control = mpfs_spi_read(spi, REG_CONTROL); control &= ~CONTROL_FRAMECNT_MASK; control |= lenpart << CONTROL_FRAMECNT_SHIFT; - mchp_corespi_write(spi, REG_CONTROL, control); - mchp_corespi_write(spi, REG_FRAMESUP, len); + mpfs_spi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_FRAMESUP, len); } -static inline void mchp_corespi_write_fifo(struct mchp_corespi *spi, int fifo_max) +static inline void mpfs_spi_write_fifo(struct mpfs_spi *spi, int fifo_max) { int i = 0; - mchp_corespi_set_xfer_size(spi, fifo_max); + mpfs_spi_set_xfer_size(spi, fifo_max); - while ((i < fifo_max) && !(mchp_corespi_read(spi, REG_STATUS) & STATUS_TXFIFO_FULL)) { + while ((i < fifo_max) && !(mpfs_spi_read(spi, REG_STATUS) & STATUS_TXFIFO_FULL)) { u32 word; if (spi->n_bytes == 4) @@ -231,7 +231,7 @@ static inline void mchp_corespi_write_fifo(struct mchp_corespi *spi, int fifo_ma else word = spi->tx_buf ? *spi->tx_buf : 0xaa; - mchp_corespi_write(spi, REG_TX_DATA, word); + mpfs_spi_write(spi, REG_TX_DATA, word); if (spi->tx_buf) spi->tx_buf += spi->n_bytes; i++; @@ -240,9 +240,9 @@ static inline void mchp_corespi_write_fifo(struct mchp_corespi *spi, int fifo_ma spi->tx_len -= i * spi->n_bytes; } -static inline void mchp_corespi_set_framesize(struct mchp_corespi *spi, int bt) +static inline void mpfs_spi_set_framesize(struct mpfs_spi *spi, int bt) { - u32 frame_size = mchp_corespi_read(spi, REG_FRAME_SIZE); + u32 frame_size = mpfs_spi_read(spi, REG_FRAME_SIZE); u32 control; if ((frame_size & FRAME_SIZE_MASK) == bt) @@ -252,25 +252,25 @@ static inline void mchp_corespi_set_framesize(struct mchp_corespi *spi, int bt) * Disable the SPI controller. Writes to the frame size have * no effect when the controller is enabled. */ - control = mchp_corespi_read(spi, REG_CONTROL); + control = mpfs_spi_read(spi, REG_CONTROL); control &= ~CONTROL_ENABLE; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); - mchp_corespi_write(spi, REG_FRAME_SIZE, bt); + mpfs_spi_write(spi, REG_FRAME_SIZE, bt); control |= CONTROL_ENABLE; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); } -static void mchp_corespi_set_cs(struct spi_device *spi, bool disable) +static void mpfs_spi_set_cs(struct spi_device *spi, bool disable) { u32 reg; - struct mchp_corespi *corespi = spi_controller_get_devdata(spi->controller); + struct mpfs_spi *mspi = spi_controller_get_devdata(spi->controller); - reg = mchp_corespi_read(corespi, REG_SLAVE_SELECT); + reg = mpfs_spi_read(mspi, REG_SLAVE_SELECT); reg &= ~BIT(spi_get_chipselect(spi, 0)); reg |= !disable << spi_get_chipselect(spi, 0); - corespi->pending_slave_select = reg; + mspi->pending_slave_select = reg; /* * Only deassert chip select immediately. Writing to some registers @@ -281,12 +281,12 @@ static void mchp_corespi_set_cs(struct spi_device *spi, bool disable) * doesn't see any spurious clock transitions whilst CS is enabled. */ if (((spi->mode & SPI_CS_HIGH) == 0) == disable) - mchp_corespi_write(corespi, REG_SLAVE_SELECT, reg); + mpfs_spi_write(mspi, REG_SLAVE_SELECT, reg); } -static int mchp_corespi_setup(struct spi_device *spi) +static int mpfs_spi_setup(struct spi_device *spi) { - struct mchp_corespi *corespi = spi_controller_get_devdata(spi->controller); + struct mpfs_spi *mspi = spi_controller_get_devdata(spi->controller); u32 reg; if (spi_is_csgpiod(spi)) @@ -298,21 +298,21 @@ static int mchp_corespi_setup(struct spi_device *spi) * driving their select line low. */ if (spi->mode & SPI_CS_HIGH) { - reg = mchp_corespi_read(corespi, REG_SLAVE_SELECT); + reg = mpfs_spi_read(mspi, REG_SLAVE_SELECT); reg |= BIT(spi_get_chipselect(spi, 0)); - corespi->pending_slave_select = reg; - mchp_corespi_write(corespi, REG_SLAVE_SELECT, reg); + mspi->pending_slave_select = reg; + mpfs_spi_write(mspi, REG_SLAVE_SELECT, reg); } return 0; } -static void mchp_corespi_init(struct spi_controller *host, struct mchp_corespi *spi) +static void mpfs_spi_init(struct spi_controller *host, struct mpfs_spi *spi) { unsigned long clk_hz; - u32 control = mchp_corespi_read(spi, REG_CONTROL); + u32 control = mpfs_spi_read(spi, REG_CONTROL); control &= ~CONTROL_ENABLE; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); control |= CONTROL_MASTER; control &= ~CONTROL_MODE_MASK; @@ -328,15 +328,15 @@ static void mchp_corespi_init(struct spi_controller *host, struct mchp_corespi * */ control |= CONTROL_SPS | CONTROL_BIGFIFO; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); - mchp_corespi_set_framesize(spi, DEFAULT_FRAMESIZE); + mpfs_spi_set_framesize(spi, DEFAULT_FRAMESIZE); /* max. possible spi clock rate is the apb clock rate */ clk_hz = clk_get_rate(spi->clk); host->max_speed_hz = clk_hz; - mchp_corespi_enable_ints(spi); + mpfs_spi_enable_ints(spi); /* * It is required to enable direct mode, otherwise control over the chip @@ -344,34 +344,34 @@ static void mchp_corespi_init(struct spi_controller *host, struct mchp_corespi * * can deal with active high targets. */ spi->pending_slave_select = SSELOUT | SSEL_DIRECT; - mchp_corespi_write(spi, REG_SLAVE_SELECT, spi->pending_slave_select); + mpfs_spi_write(spi, REG_SLAVE_SELECT, spi->pending_slave_select); - control = mchp_corespi_read(spi, REG_CONTROL); + control = mpfs_spi_read(spi, REG_CONTROL); control &= ~CONTROL_RESET; control |= CONTROL_ENABLE; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); } -static inline void mchp_corespi_set_clk_gen(struct mchp_corespi *spi) +static inline void mpfs_spi_set_clk_gen(struct mpfs_spi *spi) { u32 control; - control = mchp_corespi_read(spi, REG_CONTROL); + control = mpfs_spi_read(spi, REG_CONTROL); if (spi->clk_mode) control |= CONTROL_CLKMODE; else control &= ~CONTROL_CLKMODE; - mchp_corespi_write(spi, REG_CLK_GEN, spi->clk_gen); - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CLK_GEN, spi->clk_gen); + mpfs_spi_write(spi, REG_CONTROL, control); } -static inline void mchp_corespi_set_mode(struct mchp_corespi *spi, unsigned int mode) +static inline void mpfs_spi_set_mode(struct mpfs_spi *spi, unsigned int mode) { u32 mode_val; - u32 control = mchp_corespi_read(spi, REG_CONTROL); + u32 control = mpfs_spi_read(spi, REG_CONTROL); switch (mode & SPI_MODE_X_MASK) { case SPI_MODE_0: @@ -394,22 +394,22 @@ static inline void mchp_corespi_set_mode(struct mchp_corespi *spi, unsigned int */ control &= ~CONTROL_ENABLE; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); control &= ~(SPI_MODE_X_MASK << MODE_X_MASK_SHIFT); control |= mode_val; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); control |= CONTROL_ENABLE; - mchp_corespi_write(spi, REG_CONTROL, control); + mpfs_spi_write(spi, REG_CONTROL, control); } -static irqreturn_t mchp_corespi_interrupt(int irq, void *dev_id) +static irqreturn_t mpfs_spi_interrupt(int irq, void *dev_id) { struct spi_controller *host = dev_id; - struct mchp_corespi *spi = spi_controller_get_devdata(host); - u32 intfield = mchp_corespi_read(spi, REG_MIS) & 0xf; + struct mpfs_spi *spi = spi_controller_get_devdata(host); + u32 intfield = mpfs_spi_read(spi, REG_MIS) & 0xf; bool finalise = false; /* Interrupt line may be shared and not for us at all */ @@ -417,7 +417,7 @@ static irqreturn_t mchp_corespi_interrupt(int irq, void *dev_id) return IRQ_NONE; if (intfield & INT_RX_CHANNEL_OVERFLOW) { - mchp_corespi_write(spi, REG_INT_CLEAR, INT_RX_CHANNEL_OVERFLOW); + mpfs_spi_write(spi, REG_INT_CLEAR, INT_RX_CHANNEL_OVERFLOW); finalise = true; dev_err(&host->dev, "%s: RX OVERFLOW: rxlen: %d, txlen: %d\n", __func__, @@ -425,7 +425,7 @@ static irqreturn_t mchp_corespi_interrupt(int irq, void *dev_id) } if (intfield & INT_TX_CHANNEL_UNDERRUN) { - mchp_corespi_write(spi, REG_INT_CLEAR, INT_TX_CHANNEL_UNDERRUN); + mpfs_spi_write(spi, REG_INT_CLEAR, INT_TX_CHANNEL_UNDERRUN); finalise = true; dev_err(&host->dev, "%s: TX UNDERFLOW: rxlen: %d, txlen: %d\n", __func__, @@ -438,8 +438,8 @@ static irqreturn_t mchp_corespi_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static int mchp_corespi_calculate_clkgen(struct mchp_corespi *spi, - unsigned long target_hz) +static int mpfs_spi_calculate_clkgen(struct mpfs_spi *spi, + unsigned long target_hz) { unsigned long clk_hz, spi_hz, clk_gen; @@ -475,20 +475,20 @@ static int mchp_corespi_calculate_clkgen(struct mchp_corespi *spi, return 0; } -static int mchp_corespi_transfer_one(struct spi_controller *host, - struct spi_device *spi_dev, - struct spi_transfer *xfer) +static int mpfs_spi_transfer_one(struct spi_controller *host, + struct spi_device *spi_dev, + struct spi_transfer *xfer) { - struct mchp_corespi *spi = spi_controller_get_devdata(host); + struct mpfs_spi *spi = spi_controller_get_devdata(host); int ret; - ret = mchp_corespi_calculate_clkgen(spi, (unsigned long)xfer->speed_hz); + ret = mpfs_spi_calculate_clkgen(spi, (unsigned long)xfer->speed_hz); if (ret) { dev_err(&host->dev, "failed to set clk_gen for target %u Hz\n", xfer->speed_hz); return ret; } - mchp_corespi_set_clk_gen(spi); + mpfs_spi_set_clk_gen(spi); spi->tx_buf = xfer->tx_buf; spi->rx_buf = xfer->rx_buf; @@ -496,45 +496,46 @@ static int mchp_corespi_transfer_one(struct spi_controller *host, spi->rx_len = xfer->len; spi->n_bytes = roundup_pow_of_two(DIV_ROUND_UP(xfer->bits_per_word, BITS_PER_BYTE)); - mchp_corespi_set_framesize(spi, xfer->bits_per_word); + mpfs_spi_set_framesize(spi, xfer->bits_per_word); - mchp_corespi_write(spi, REG_COMMAND, COMMAND_RXFIFORST | COMMAND_TXFIFORST); + mpfs_spi_write(spi, REG_COMMAND, COMMAND_RXFIFORST | COMMAND_TXFIFORST); - mchp_corespi_write(spi, REG_SLAVE_SELECT, spi->pending_slave_select); + mpfs_spi_write(spi, REG_SLAVE_SELECT, spi->pending_slave_select); while (spi->tx_len) { int fifo_max = DIV_ROUND_UP(min(spi->tx_len, FIFO_DEPTH), spi->n_bytes); - mchp_corespi_write_fifo(spi, fifo_max); - mchp_corespi_read_fifo(spi, fifo_max); + mpfs_spi_write_fifo(spi, fifo_max); + mpfs_spi_read_fifo(spi, fifo_max); } spi_finalize_current_transfer(host); return 1; } -static int mchp_corespi_prepare_message(struct spi_controller *host, - struct spi_message *msg) +static int mpfs_spi_prepare_message(struct spi_controller *host, + struct spi_message *msg) { struct spi_device *spi_dev = msg->spi; - struct mchp_corespi *spi = spi_controller_get_devdata(host); + struct mpfs_spi *spi = spi_controller_get_devdata(host); - mchp_corespi_set_mode(spi, spi_dev->mode); + mpfs_spi_set_mode(spi, spi_dev->mode); return 0; } -static int mchp_corespi_probe(struct platform_device *pdev) +static int mpfs_spi_probe(struct platform_device *pdev) { struct spi_controller *host; - struct mchp_corespi *spi; + struct mpfs_spi *spi; struct resource *res; u32 num_cs; int ret = 0; host = devm_spi_alloc_host(&pdev->dev, sizeof(*spi)); if (!host) - return -ENOMEM; + return dev_err_probe(&pdev->dev, -ENOMEM, + "unable to allocate host for SPI controller\n"); platform_set_drvdata(pdev, host); @@ -544,11 +545,11 @@ static int mchp_corespi_probe(struct platform_device *pdev) host->num_chipselect = num_cs; host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH; host->use_gpio_descriptors = true; - host->setup = mchp_corespi_setup; + host->setup = mpfs_spi_setup; host->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); - host->transfer_one = mchp_corespi_transfer_one; - host->prepare_message = mchp_corespi_prepare_message; - host->set_cs = mchp_corespi_set_cs; + host->transfer_one = mpfs_spi_transfer_one; + host->prepare_message = mpfs_spi_prepare_message; + host->set_cs = mpfs_spi_set_cs; host->dev.of_node = pdev->dev.of_node; spi = spi_controller_get_devdata(host); @@ -561,7 +562,7 @@ static int mchp_corespi_probe(struct platform_device *pdev) if (spi->irq < 0) return spi->irq; - ret = devm_request_irq(&pdev->dev, spi->irq, mchp_corespi_interrupt, + ret = devm_request_irq(&pdev->dev, spi->irq, mpfs_spi_interrupt, IRQF_SHARED, dev_name(&pdev->dev), host); if (ret) return dev_err_probe(&pdev->dev, ret, @@ -572,11 +573,12 @@ static int mchp_corespi_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(spi->clk), "could not get clk\n"); - mchp_corespi_init(host, spi); + mpfs_spi_init(host, spi); ret = devm_spi_register_controller(&pdev->dev, host); if (ret) { - mchp_corespi_disable(spi); + mpfs_spi_disable_ints(spi); + mpfs_spi_disable(spi); return dev_err_probe(&pdev->dev, ret, "unable to register host for SPI controller\n"); } @@ -586,13 +588,13 @@ static int mchp_corespi_probe(struct platform_device *pdev) return 0; } -static void mchp_corespi_remove(struct platform_device *pdev) +static void mpfs_spi_remove(struct platform_device *pdev) { struct spi_controller *host = platform_get_drvdata(pdev); - struct mchp_corespi *spi = spi_controller_get_devdata(host); + struct mpfs_spi *spi = spi_controller_get_devdata(host); - mchp_corespi_disable_ints(spi); - mchp_corespi_disable(spi); + mpfs_spi_disable_ints(spi); + mpfs_spi_disable(spi); } #define MICROCHIP_SPI_PM_OPS (NULL) @@ -602,23 +604,23 @@ static void mchp_corespi_remove(struct platform_device *pdev) */ #if defined(CONFIG_OF) -static const struct of_device_id mchp_corespi_dt_ids[] = { +static const struct of_device_id mpfs_spi_dt_ids[] = { { .compatible = "microchip,mpfs-spi" }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, mchp_corespi_dt_ids); +MODULE_DEVICE_TABLE(of, mpfs_spi_dt_ids); #endif -static struct platform_driver mchp_corespi_driver = { - .probe = mchp_corespi_probe, +static struct platform_driver mpfs_spi_driver = { + .probe = mpfs_spi_probe, .driver = { - .name = "microchip-corespi", + .name = "microchip-spi", .pm = MICROCHIP_SPI_PM_OPS, - .of_match_table = of_match_ptr(mchp_corespi_dt_ids), + .of_match_table = of_match_ptr(mpfs_spi_dt_ids), }, - .remove = mchp_corespi_remove, + .remove = mpfs_spi_remove, }; -module_platform_driver(mchp_corespi_driver); +module_platform_driver(mpfs_spi_driver); MODULE_DESCRIPTION("Microchip coreSPI SPI controller driver"); MODULE_AUTHOR("Daire McNamara "); MODULE_AUTHOR("Conor Dooley "); diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c index 4b40985af1eaf..90e5813cfdc33 100644 --- a/drivers/spi/spi-mt65xx.c +++ b/drivers/spi/spi-mt65xx.c @@ -1320,7 +1320,7 @@ static int mtk_spi_probe(struct platform_device *pdev) ret = devm_request_threaded_irq(dev, irq, mtk_spi_interrupt, mtk_spi_interrupt_thread, - IRQF_TRIGGER_NONE, dev_name(dev), host); + IRQF_ONESHOT, dev_name(dev), host); if (ret) return dev_err_probe(dev, ret, "failed to register irq\n"); diff --git a/drivers/spi/spi-qcom-qspi.c b/drivers/spi/spi-qcom-qspi.c index 3aeddada58e1c..e45dfd1458de9 100644 --- a/drivers/spi/spi-qcom-qspi.c +++ b/drivers/spi/spi-qcom-qspi.c @@ -174,6 +174,7 @@ struct qcom_qspi { void *virt_cmd_desc[QSPI_MAX_SG]; unsigned int n_cmd_desc; struct icc_path *icc_path_cpu_to_qspi; + struct icc_path *icc_path_mem; unsigned long last_speed; /* Lock to protect data accessed by IRQs */ spinlock_t lock; @@ -272,7 +273,7 @@ static void qcom_qspi_handle_err(struct spi_controller *host, static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz) { int ret; - unsigned int avg_bw_cpu; + unsigned int avg_bw_cpu, avg_bw_mem; if (speed_hz == ctrl->last_speed) return 0; @@ -285,7 +286,7 @@ static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz) } /* - * Set BW quota for CPU. + * Set BW quota for CPU and memory paths. * We don't have explicit peak requirement so keep it equal to avg_bw. */ avg_bw_cpu = Bps_to_icc(speed_hz); @@ -296,6 +297,13 @@ static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz) return ret; } + avg_bw_mem = Bps_to_icc(speed_hz); + ret = icc_set_bw(ctrl->icc_path_mem, avg_bw_mem, avg_bw_mem); + if (ret) { + dev_err(ctrl->dev, "ICC BW voting failed for memory: %d\n", ret); + return ret; + } + ctrl->last_speed = speed_hz; return 0; @@ -729,6 +737,11 @@ static int qcom_qspi_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi), "Failed to get cpu path\n"); + ctrl->icc_path_mem = devm_of_icc_get(dev, "qspi-memory"); + if (IS_ERR(ctrl->icc_path_mem)) + return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_mem), + "Failed to get memory path\n"); + /* Set BW vote for register access */ ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000), Bps_to_icc(1000)); @@ -830,6 +843,13 @@ static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev) return ret; } + ret = icc_disable(ctrl->icc_path_mem); + if (ret) { + dev_err_ratelimited(ctrl->dev, "ICC disable failed for memory: %d\n", ret); + icc_enable(ctrl->icc_path_cpu_to_qspi); + return ret; + } + pinctrl_pm_select_sleep_state(dev); return 0; @@ -850,9 +870,19 @@ static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev) return ret; } + ret = icc_enable(ctrl->icc_path_mem); + if (ret) { + dev_err_ratelimited(ctrl->dev, "ICC enable failed for memory: %d\n", ret); + icc_disable(ctrl->icc_path_cpu_to_qspi); + return ret; + } + ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks); - if (ret) + if (ret) { + icc_disable(ctrl->icc_path_cpu_to_qspi); + icc_disable(ctrl->icc_path_mem); return ret; + } return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4); } diff --git a/drivers/spi/spi-rockchip-sfc.c b/drivers/spi/spi-rockchip-sfc.c index b3c2b03b11535..8acf955636977 100644 --- a/drivers/spi/spi-rockchip-sfc.c +++ b/drivers/spi/spi-rockchip-sfc.c @@ -712,7 +712,7 @@ static int rockchip_sfc_probe(struct platform_device *pdev) } } - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) goto err_register; diff --git a/drivers/spi/spi-sg2044-nor.c b/drivers/spi/spi-sg2044-nor.c index af48b1fcda930..37f1cfe10be46 100644 --- a/drivers/spi/spi-sg2044-nor.c +++ b/drivers/spi/spi-sg2044-nor.c @@ -42,6 +42,7 @@ #define SPIFMC_TRAN_CSR_TRAN_MODE_RX BIT(0) #define SPIFMC_TRAN_CSR_TRAN_MODE_TX BIT(1) #define SPIFMC_TRAN_CSR_FAST_MODE BIT(3) +#define SPIFMC_TRAN_CSR_BUS_WIDTH_MASK GENMASK(5, 4) #define SPIFMC_TRAN_CSR_BUS_WIDTH_1_BIT (0x00 << 4) #define SPIFMC_TRAN_CSR_BUS_WIDTH_2_BIT (0x01 << 4) #define SPIFMC_TRAN_CSR_BUS_WIDTH_4_BIT (0x02 << 4) @@ -122,8 +123,7 @@ static u32 sg2044_spifmc_init_reg(struct sg2044_spifmc *spifmc) reg = readl(spifmc->io_base + SPIFMC_TRAN_CSR); reg &= ~(SPIFMC_TRAN_CSR_TRAN_MODE_MASK | SPIFMC_TRAN_CSR_FAST_MODE | - SPIFMC_TRAN_CSR_BUS_WIDTH_2_BIT | - SPIFMC_TRAN_CSR_BUS_WIDTH_4_BIT | + SPIFMC_TRAN_CSR_BUS_WIDTH_MASK | SPIFMC_TRAN_CSR_DMA_EN | SPIFMC_TRAN_CSR_ADDR_BYTES_MASK | SPIFMC_TRAN_CSR_WITH_CMD | diff --git a/drivers/spi/spi-sn-f-ospi.c b/drivers/spi/spi-sn-f-ospi.c index c4969f66a0ba9..84a5b327022e8 100644 --- a/drivers/spi/spi-sn-f-ospi.c +++ b/drivers/spi/spi-sn-f-ospi.c @@ -612,7 +612,7 @@ static int f_ospi_probe(struct platform_device *pdev) u32 num_cs = OSPI_NUM_CS; int ret; - ctlr = spi_alloc_host(dev, sizeof(*ospi)); + ctlr = devm_spi_alloc_host(dev, sizeof(*ospi)); if (!ctlr) return -ENOMEM; @@ -636,16 +636,12 @@ static int f_ospi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ospi); ospi->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(ospi->base)) { - ret = PTR_ERR(ospi->base); - goto err_put_ctlr; - } + if (IS_ERR(ospi->base)) + return PTR_ERR(ospi->base); ospi->clk = devm_clk_get_enabled(dev, NULL); - if (IS_ERR(ospi->clk)) { - ret = PTR_ERR(ospi->clk); - goto err_put_ctlr; - } + if (IS_ERR(ospi->clk)) + return PTR_ERR(ospi->clk); mutex_init(&ospi->mlock); @@ -662,9 +658,6 @@ static int f_ospi_probe(struct platform_device *pdev) err_destroy_mutex: mutex_destroy(&ospi->mlock); -err_put_ctlr: - spi_controller_put(ctlr); - return ret; } diff --git a/drivers/spi/spi-sprd-adi.c b/drivers/spi/spi-sprd-adi.c index 262c11d977ea3..f25b34a91756f 100644 --- a/drivers/spi/spi-sprd-adi.c +++ b/drivers/spi/spi-sprd-adi.c @@ -528,7 +528,7 @@ static int sprd_adi_probe(struct platform_device *pdev) pdev->id = of_alias_get_id(np, "spi"); num_chipselect = of_get_child_count(np); - ctlr = spi_alloc_host(&pdev->dev, sizeof(struct sprd_adi)); + ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(struct sprd_adi)); if (!ctlr) return -ENOMEM; @@ -536,10 +536,8 @@ static int sprd_adi_probe(struct platform_device *pdev) sadi = spi_controller_get_devdata(ctlr); sadi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); - if (IS_ERR(sadi->base)) { - ret = PTR_ERR(sadi->base); - goto put_ctlr; - } + if (IS_ERR(sadi->base)) + return PTR_ERR(sadi->base); sadi->slave_vbase = (unsigned long)sadi->base + data->slave_offset; @@ -551,18 +549,15 @@ static int sprd_adi_probe(struct platform_device *pdev) if (ret > 0 || (IS_ENABLED(CONFIG_HWSPINLOCK) && ret == 0)) { sadi->hwlock = devm_hwspin_lock_request_specific(&pdev->dev, ret); - if (!sadi->hwlock) { - ret = -ENXIO; - goto put_ctlr; - } + if (!sadi->hwlock) + return -ENXIO; } else { switch (ret) { case -ENOENT: dev_info(&pdev->dev, "no hardware spinlock supplied\n"); break; default: - dev_err_probe(&pdev->dev, ret, "failed to find hwlock id\n"); - goto put_ctlr; + return dev_err_probe(&pdev->dev, ret, "failed to find hwlock id\n"); } } @@ -579,26 +574,18 @@ static int sprd_adi_probe(struct platform_device *pdev) ctlr->transfer_one = sprd_adi_transfer_one; ret = devm_spi_register_controller(&pdev->dev, ctlr); - if (ret) { - dev_err(&pdev->dev, "failed to register SPI controller\n"); - goto put_ctlr; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to register SPI controller\n"); if (sadi->data->restart) { ret = devm_register_restart_handler(&pdev->dev, sadi->data->restart, sadi); - if (ret) { - dev_err(&pdev->dev, "can not register restart handler\n"); - goto put_ctlr; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "can not register restart handler\n"); } return 0; - -put_ctlr: - spi_controller_put(ctlr); - return ret; } static struct sprd_adi_data sc9860_data = { diff --git a/drivers/spi/spi-stm32-ospi.c b/drivers/spi/spi-stm32-ospi.c index f36fd36da2692..2988ff288ff02 100644 --- a/drivers/spi/spi-stm32-ospi.c +++ b/drivers/spi/spi-stm32-ospi.c @@ -939,13 +939,15 @@ static int stm32_ospi_probe(struct platform_device *pdev) if (ret) { /* Disable ospi */ writel_relaxed(0, ospi->regs_base + OSPI_CR); - goto err_pm_resume; + goto err_reset_control; } pm_runtime_put_autosuspend(ospi->dev); return 0; +err_reset_control: + reset_control_release(ospi->rstc); err_pm_resume: pm_runtime_put_sync_suspend(ospi->dev); @@ -963,11 +965,8 @@ static int stm32_ospi_probe(struct platform_device *pdev) static void stm32_ospi_remove(struct platform_device *pdev) { struct stm32_ospi *ospi = platform_get_drvdata(pdev); - int ret; - ret = pm_runtime_resume_and_get(ospi->dev); - if (ret < 0) - return; + pm_runtime_resume_and_get(ospi->dev); spi_unregister_controller(ospi->ctrl); /* Disable ospi */ diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index 2c804c1aef989..7a6ee93be9bd4 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1570,6 +1570,9 @@ static int stm32_spi_prepare_rx_dma_mdma_chaining(struct stm32_spi *spi, return -EINVAL; } + *rx_mdma_desc = _mdma_desc; + *rx_dma_desc = _dma_desc; + return 0; } @@ -1906,11 +1909,12 @@ static void stm32h7_spi_data_idleness(struct stm32_spi *spi, struct spi_transfer cfg2_clrb |= STM32H7_SPI_CFG2_MIDI; if ((len > 1) && (spi->cur_midi > 0)) { u32 sck_period_ns = DIV_ROUND_UP(NSEC_PER_SEC, spi->cur_speed); - u32 midi = min_t(u32, - DIV_ROUND_UP(spi->cur_midi, sck_period_ns), - FIELD_GET(STM32H7_SPI_CFG2_MIDI, - STM32H7_SPI_CFG2_MIDI)); + u32 midi = DIV_ROUND_UP(spi->cur_midi, sck_period_ns); + + if ((spi->cur_bpw + midi) < 8) + midi = 8 - spi->cur_bpw; + midi = min_t(u32, midi, FIELD_MAX(STM32H7_SPI_CFG2_MIDI)); dev_dbg(spi->dev, "period=%dns, midi=%d(=%dns)\n", sck_period_ns, midi, midi * sck_period_ns); diff --git a/drivers/spi/spi-tegra114.c b/drivers/spi/spi-tegra114.c index 795a8482c2c70..48fb11fea55f2 100644 --- a/drivers/spi/spi-tegra114.c +++ b/drivers/spi/spi-tegra114.c @@ -978,11 +978,14 @@ static int tegra_spi_setup(struct spi_device *spi) if (spi_get_csgpiod(spi, 0)) gpiod_set_value(spi_get_csgpiod(spi, 0), 0); + /* Update default register to include CS polarity and SPI mode */ val = tspi->def_command1_reg; if (spi->mode & SPI_CS_HIGH) val &= ~SPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0)); else val |= SPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0)); + val &= ~SPI_CONTROL_MODE_MASK; + val |= SPI_MODE_SEL(spi->mode & 0x3); tspi->def_command1_reg = val; tegra_spi_writel(tspi, tspi->def_command1_reg, SPI_COMMAND1); spin_unlock_irqrestore(&tspi->lock, flags); diff --git a/drivers/spi/spi-tegra20-slink.c b/drivers/spi/spi-tegra20-slink.c index fe452d03c1ee4..709669610840b 100644 --- a/drivers/spi/spi-tegra20-slink.c +++ b/drivers/spi/spi-tegra20-slink.c @@ -1086,8 +1086,10 @@ static int tegra_slink_probe(struct platform_device *pdev) reset_control_deassert(tspi->rst); spi_irq = platform_get_irq(pdev, 0); - if (spi_irq < 0) - return spi_irq; + if (spi_irq < 0) { + ret = spi_irq; + goto exit_pm_put; + } tspi->irq = spi_irq; ret = request_threaded_irq(tspi->irq, tegra_slink_isr, tegra_slink_isr_thread, IRQF_ONESHOT, diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c index 3be7499db21ec..83def82fe48c1 100644 --- a/drivers/spi/spi-tegra210-quad.c +++ b/drivers/spi/spi-tegra210-quad.c @@ -839,6 +839,7 @@ static u32 tegra_qspi_setup_transfer_one(struct spi_device *spi, struct spi_tran u32 command1, command2, speed = t->speed_hz; u8 bits_per_word = t->bits_per_word; u32 tx_tap = 0, rx_tap = 0; + unsigned long flags; int req_mode; if (!has_acpi_companion(tqspi->dev) && speed != tqspi->cur_speed) { @@ -846,10 +847,12 @@ static u32 tegra_qspi_setup_transfer_one(struct spi_device *spi, struct spi_tran tqspi->cur_speed = speed; } + spin_lock_irqsave(&tqspi->lock, flags); tqspi->cur_pos = 0; tqspi->cur_rx_pos = 0; tqspi->cur_tx_pos = 0; tqspi->curr_xfer = t; + spin_unlock_irqrestore(&tqspi->lock, flags); if (is_first_of_msg) { tegra_qspi_mask_clear_irq(tqspi); @@ -1024,8 +1027,10 @@ static void tegra_qspi_handle_error(struct tegra_qspi *tqspi) dev_err(tqspi->dev, "error in transfer, fifo status 0x%08x\n", tqspi->status_reg); tegra_qspi_dump_regs(tqspi); tegra_qspi_flush_fifos(tqspi, true); - if (device_reset(tqspi->dev) < 0) + if (device_reset(tqspi->dev) < 0) { dev_warn_once(tqspi->dev, "device reset failed\n"); + tegra_qspi_mask_clear_irq(tqspi); + } } static void tegra_qspi_transfer_end(struct spi_device *spi) @@ -1084,6 +1089,7 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi, u32 address_value = 0; u32 cmd_config = 0, addr_config = 0; u8 cmd_value = 0, val = 0; + unsigned long flags; /* Enable Combined sequence mode */ val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG); @@ -1176,9 +1182,11 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi, } /* Reset controller if timeout happens */ - if (device_reset(tqspi->dev) < 0) + if (device_reset(tqspi->dev) < 0) { dev_warn_once(tqspi->dev, "device reset failed\n"); + tegra_qspi_mask_clear_irq(tqspi); + } ret = -EIO; goto exit; } @@ -1200,11 +1208,17 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi, tegra_qspi_transfer_end(spi); spi_transfer_delay_exec(xfer); } + spin_lock_irqsave(&tqspi->lock, flags); + tqspi->curr_xfer = NULL; + spin_unlock_irqrestore(&tqspi->lock, flags); transfer_phase++; } ret = 0; exit: + spin_lock_irqsave(&tqspi->lock, flags); + tqspi->curr_xfer = NULL; + spin_unlock_irqrestore(&tqspi->lock, flags); msg->status = ret; return ret; @@ -1217,6 +1231,7 @@ static int tegra_qspi_non_combined_seq_xfer(struct tegra_qspi *tqspi, struct spi_transfer *transfer; bool is_first_msg = true; int ret = 0, val = 0; + unsigned long flags; msg->status = 0; msg->actual_length = 0; @@ -1290,6 +1305,10 @@ static int tegra_qspi_non_combined_seq_xfer(struct tegra_qspi *tqspi, msg->actual_length += xfer->len + dummy_bytes; complete_xfer: + spin_lock_irqsave(&tqspi->lock, flags); + tqspi->curr_xfer = NULL; + spin_unlock_irqrestore(&tqspi->lock, flags); + if (ret < 0) { tegra_qspi_transfer_end(spi); spi_transfer_delay_exec(xfer); @@ -1368,10 +1387,16 @@ static int tegra_qspi_transfer_one_message(struct spi_controller *host, static irqreturn_t handle_cpu_based_xfer(struct tegra_qspi *tqspi) { - struct spi_transfer *t = tqspi->curr_xfer; + struct spi_transfer *t; unsigned long flags; spin_lock_irqsave(&tqspi->lock, flags); + t = tqspi->curr_xfer; + + if (!t) { + spin_unlock_irqrestore(&tqspi->lock, flags); + return IRQ_HANDLED; + } if (tqspi->tx_status || tqspi->rx_status) { tegra_qspi_handle_error(tqspi); @@ -1395,13 +1420,14 @@ static irqreturn_t handle_cpu_based_xfer(struct tegra_qspi *tqspi) tegra_qspi_calculate_curr_xfer_param(tqspi, t); tegra_qspi_start_cpu_based_transfer(tqspi, t); exit: + tqspi->curr_xfer = NULL; spin_unlock_irqrestore(&tqspi->lock, flags); return IRQ_HANDLED; } static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi) { - struct spi_transfer *t = tqspi->curr_xfer; + struct spi_transfer *t; unsigned int total_fifo_words; unsigned long flags; long wait_status; @@ -1440,6 +1466,12 @@ static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi) } spin_lock_irqsave(&tqspi->lock, flags); + t = tqspi->curr_xfer; + + if (!t) { + spin_unlock_irqrestore(&tqspi->lock, flags); + return IRQ_HANDLED; + } if (num_errors) { tegra_qspi_dma_unmap_xfer(tqspi, t); @@ -1479,6 +1511,33 @@ static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi) static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data) { struct tegra_qspi *tqspi = context_data; + unsigned long flags; + u32 status; + + /* + * Read transfer status to check if interrupt was triggered by transfer + * completion + */ + status = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS); + + /* + * Occasionally the IRQ thread takes a long time to wake up (usually + * when the CPU that it's running on is excessively busy) and we have + * already reached the timeout before and cleaned up the timed out + * transfer. Avoid any processing in that case and bail out early. + * + * If no transfer is in progress, check if this was a real interrupt + * that the timeout handler already processed, or a spurious one. + */ + spin_lock_irqsave(&tqspi->lock, flags); + if (!tqspi->curr_xfer) { + spin_unlock_irqrestore(&tqspi->lock, flags); + /* Spurious interrupt - transfer not ready */ + if (!(status & QSPI_RDY)) + return IRQ_NONE; + /* Real interrupt, already handled by timeout path */ + return IRQ_HANDLED; + } tqspi->status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); @@ -1489,7 +1548,14 @@ static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data) tqspi->rx_status = tqspi->status_reg & (QSPI_RX_FIFO_OVF | QSPI_RX_FIFO_UNF); tegra_qspi_mask_clear_irq(tqspi); + spin_unlock_irqrestore(&tqspi->lock, flags); + /* + * Lock is released here but handlers safely re-check curr_xfer under + * lock before dereferencing. + * DMA handler also needs to sleep in wait_for_completion_*(), which + * cannot be done while holding spinlock. + */ if (!tqspi->is_curr_dma_xfer) return handle_cpu_based_xfer(tqspi); diff --git a/drivers/spi/spi-wpcm-fiu.c b/drivers/spi/spi-wpcm-fiu.c index a9aee2a6c7dcb..c47b56f0933f1 100644 --- a/drivers/spi/spi-wpcm-fiu.c +++ b/drivers/spi/spi-wpcm-fiu.c @@ -459,11 +459,11 @@ static int wpcm_fiu_probe(struct platform_device *pdev) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "memory"); fiu->memory = devm_ioremap_resource(dev, res); - fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); if (IS_ERR(fiu->memory)) return dev_err_probe(dev, PTR_ERR(fiu->memory), "Failed to map flash memory window\n"); + fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); fiu->shm_regmap = syscon_regmap_lookup_by_phandle_optional(dev->of_node, "nuvoton,shm"); wpcm_fiu_hw_init(fiu); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index e25df9990f82d..87d829d2a8427 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -50,7 +50,6 @@ static void spidev_release(struct device *dev) struct spi_device *spi = to_spi_device(dev); spi_controller_put(spi->controller); - kfree(spi->driver_override); free_percpu(spi->pcpu_statistics); kfree(spi); } @@ -73,10 +72,9 @@ static ssize_t driver_override_store(struct device *dev, struct device_attribute *a, const char *buf, size_t count) { - struct spi_device *spi = to_spi_device(dev); int ret; - ret = driver_set_override(dev, &spi->driver_override, buf, count); + ret = __device_set_driver_override(dev, buf, count); if (ret) return ret; @@ -86,13 +84,8 @@ static ssize_t driver_override_store(struct device *dev, static ssize_t driver_override_show(struct device *dev, struct device_attribute *a, char *buf) { - const struct spi_device *spi = to_spi_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", spi->driver_override ? : ""); - device_unlock(dev); - return len; + guard(spinlock)(&dev->driver_override.lock); + return sysfs_emit(buf, "%s\n", dev->driver_override.name ?: ""); } static DEVICE_ATTR_RW(driver_override); @@ -376,10 +369,12 @@ static int spi_match_device(struct device *dev, const struct device_driver *drv) { const struct spi_device *spi = to_spi_device(dev); const struct spi_driver *sdrv = to_spi_driver(drv); + int ret; /* Check override first, and if set, only use the named driver */ - if (spi->driver_override) - return strcmp(spi->driver_override, drv->name) == 0; + ret = device_match_driver_override(dev, drv); + if (ret >= 0) + return ret; /* Attempt an OF style match */ if (of_driver_match_device(dev, drv)) @@ -2914,6 +2909,8 @@ static void spi_controller_release(struct device *dev) struct spi_controller *ctlr; ctlr = container_of(dev, struct spi_controller, dev); + + free_percpu(ctlr->pcpu_statistics); kfree(ctlr); } @@ -3057,6 +3054,12 @@ struct spi_controller *__spi_alloc_controller(struct device *dev, if (!ctlr) return NULL; + ctlr->pcpu_statistics = spi_alloc_pcpu_stats(NULL); + if (!ctlr->pcpu_statistics) { + kfree(ctlr); + return NULL; + } + device_initialize(&ctlr->dev); INIT_LIST_HEAD(&ctlr->queue); spin_lock_init(&ctlr->queue_lock); @@ -3344,17 +3347,8 @@ int spi_register_controller(struct spi_controller *ctlr) dev_info(dev, "controller is unqueued, this is deprecated\n"); } else if (ctlr->transfer_one || ctlr->transfer_one_message) { status = spi_controller_initialize_queue(ctlr); - if (status) { - device_del(&ctlr->dev); - goto free_bus_id; - } - } - /* Add statistics */ - ctlr->pcpu_statistics = spi_alloc_pcpu_stats(dev); - if (!ctlr->pcpu_statistics) { - dev_err(dev, "Error allocating per-cpu statistics\n"); - status = -ENOMEM; - goto destroy_queue; + if (status) + goto del_ctrl; } mutex_lock(&board_lock); @@ -3368,8 +3362,8 @@ int spi_register_controller(struct spi_controller *ctlr) acpi_register_spi_devices(ctlr); return status; -destroy_queue: - spi_destroy_queue(ctlr); +del_ctrl: + device_del(&ctlr->dev); free_bus_id: mutex_lock(&board_lock); idr_remove(&spi_controller_idr, ctlr->bus_num); diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 5300c942a2a44..4d53c394101a4 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -74,7 +74,6 @@ struct spidev_data { struct list_head device_entry; /* TX/RX buffers are NULL unless this device is open (users > 0) */ - struct mutex buf_lock; unsigned users; u8 *tx_buffer; u8 *rx_buffer; @@ -102,24 +101,6 @@ spidev_sync_unlocked(struct spi_device *spi, struct spi_message *message) return status; } -static ssize_t -spidev_sync(struct spidev_data *spidev, struct spi_message *message) -{ - ssize_t status; - struct spi_device *spi; - - mutex_lock(&spidev->spi_lock); - spi = spidev->spi; - - if (spi == NULL) - status = -ESHUTDOWN; - else - status = spidev_sync_unlocked(spi, message); - - mutex_unlock(&spidev->spi_lock); - return status; -} - static inline ssize_t spidev_sync_write(struct spidev_data *spidev, size_t len) { @@ -132,7 +113,8 @@ spidev_sync_write(struct spidev_data *spidev, size_t len) spi_message_init(&m); spi_message_add_tail(&t, &m); - return spidev_sync(spidev, &m); + + return spidev_sync_unlocked(spidev->spi, &m); } static inline ssize_t @@ -147,7 +129,8 @@ spidev_sync_read(struct spidev_data *spidev, size_t len) spi_message_init(&m); spi_message_add_tail(&t, &m); - return spidev_sync(spidev, &m); + + return spidev_sync_unlocked(spidev->spi, &m); } /*-------------------------------------------------------------------------*/ @@ -157,7 +140,7 @@ static ssize_t spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct spidev_data *spidev; - ssize_t status; + ssize_t status = -ESHUTDOWN; /* chipselect only toggles at start or end of operation */ if (count > bufsiz) @@ -165,7 +148,11 @@ spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) spidev = filp->private_data; - mutex_lock(&spidev->buf_lock); + mutex_lock(&spidev->spi_lock); + + if (spidev->spi == NULL) + goto err_spi_removed; + status = spidev_sync_read(spidev, count); if (status > 0) { unsigned long missing; @@ -176,7 +163,9 @@ spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) else status = status - missing; } - mutex_unlock(&spidev->buf_lock); + +err_spi_removed: + mutex_unlock(&spidev->spi_lock); return status; } @@ -187,7 +176,7 @@ spidev_write(struct file *filp, const char __user *buf, size_t count, loff_t *f_pos) { struct spidev_data *spidev; - ssize_t status; + ssize_t status = -ESHUTDOWN; unsigned long missing; /* chipselect only toggles at start or end of operation */ @@ -196,13 +185,19 @@ spidev_write(struct file *filp, const char __user *buf, spidev = filp->private_data; - mutex_lock(&spidev->buf_lock); + mutex_lock(&spidev->spi_lock); + + if (spidev->spi == NULL) + goto err_spi_removed; + missing = copy_from_user(spidev->tx_buffer, buf, count); if (missing == 0) status = spidev_sync_write(spidev, count); else status = -EFAULT; - mutex_unlock(&spidev->buf_lock); + +err_spi_removed: + mutex_unlock(&spidev->spi_lock); return status; } @@ -379,14 +374,6 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ctlr = spi->controller; - /* use the buffer lock here for triple duty: - * - prevent I/O (from us) so calling spi_setup() is safe; - * - prevent concurrent SPI_IOC_WR_* from morphing - * data fields while SPI_IOC_RD_* reads them; - * - SPI_IOC_MESSAGE needs the buffer locked "normally". - */ - mutex_lock(&spidev->buf_lock); - switch (cmd) { /* read requests */ case SPI_IOC_RD_MODE: @@ -510,7 +497,6 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) break; } - mutex_unlock(&spidev->buf_lock); spi_dev_put(spi); mutex_unlock(&spidev->spi_lock); return retval; @@ -541,9 +527,6 @@ spidev_compat_ioc_message(struct file *filp, unsigned int cmd, return -ESHUTDOWN; } - /* SPI_IOC_MESSAGE needs the buffer locked "normally" */ - mutex_lock(&spidev->buf_lock); - /* Check message and copy into scratch area */ ioc = spidev_get_ioc_message(cmd, u_ioc, &n_ioc); if (IS_ERR(ioc)) { @@ -564,7 +547,6 @@ spidev_compat_ioc_message(struct file *filp, unsigned int cmd, kfree(ioc); done: - mutex_unlock(&spidev->buf_lock); spi_dev_put(spi); mutex_unlock(&spidev->spi_lock); return retval; @@ -800,7 +782,6 @@ static int spidev_probe(struct spi_device *spi) /* Initialize the driver data */ spidev->spi = spi; mutex_init(&spidev->spi_lock); - mutex_init(&spidev->buf_lock); INIT_LIST_HEAD(&spidev->device_entry); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 697b3e8bb0235..87e3ee9d4f2aa 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -149,6 +149,7 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { + { .compatible = "apple,t8103-spmi", }, { .compatible = "apple,spmi", }, {} }; diff --git a/drivers/spmi/spmi-pmic-arb.c b/drivers/spmi/spmi-pmic-arb.c index 91581974ef84e..612736973e4b0 100644 --- a/drivers/spmi/spmi-pmic-arb.c +++ b/drivers/spmi/spmi-pmic-arb.c @@ -3,6 +3,7 @@ * Copyright (c) 2012-2015, 2017, 2021, The Linux Foundation. All rights reserved. */ #include +#include #include #include #include @@ -25,10 +26,12 @@ #define PMIC_ARB_VERSION_V3_MIN 0x30000000 #define PMIC_ARB_VERSION_V5_MIN 0x50000000 #define PMIC_ARB_VERSION_V7_MIN 0x70000000 +#define PMIC_ARB_VERSION_V8_MIN 0x80000000 #define PMIC_ARB_INT_EN 0x0004 #define PMIC_ARB_FEATURES 0x0004 #define PMIC_ARB_FEATURES_PERIPH_MASK GENMASK(10, 0) +#define PMIC_ARB_FEATURES_V8_PERIPH_MASK GENMASK(12, 0) #define PMIC_ARB_FEATURES1 0x0008 @@ -50,9 +53,10 @@ #define SPMI_MAPPING_BIT_IS_1_RESULT(X) (((X) >> 0) & 0xFF) #define SPMI_MAPPING_TABLE_TREE_DEPTH 16 /* Maximum of 16-bits */ -#define PMIC_ARB_MAX_PPID BIT(12) /* PPID is 12bit */ +#define PMIC_ARB_MAX_PPID BIT(13) #define PMIC_ARB_APID_VALID BIT(15) #define PMIC_ARB_CHAN_IS_IRQ_OWNER(reg) ((reg) & BIT(24)) +#define PMIC_ARB_V8_CHAN_IS_IRQ_OWNER(reg) ((reg) & BIT(31)) #define INVALID_EE 0xFF /* Ownership Table */ @@ -96,30 +100,33 @@ enum pmic_arb_channel { PMIC_ARB_CHANNEL_OBS, }; -#define PMIC_ARB_MAX_BUSES 2 +#define PMIC_ARB_MAX_BUSES 4 +#define PMIC_ARB_MAX_BUSES_V8 4 /* Maximum number of support PMIC peripherals */ #define PMIC_ARB_MAX_PERIPHS 512 #define PMIC_ARB_MAX_PERIPHS_V7 1024 +#define PMIC_ARB_MAX_PERIPHS_V8 8192 #define PMIC_ARB_TIMEOUT_US 1000 #define PMIC_ARB_MAX_TRANS_BYTES (8) #define PMIC_ARB_APID_MASK 0xFF #define PMIC_ARB_PPID_MASK 0xFFF +#define PMIC_ARB_V8_PPID_MASK 0x1FFF /* interrupt enable bit */ #define SPMI_PIC_ACC_ENABLE_BIT BIT(0) #define spec_to_hwirq(slave_id, periph_id, irq_id, apid) \ - ((((slave_id) & 0xF) << 28) | \ - (((periph_id) & 0xFF) << 20) | \ - (((irq_id) & 0x7) << 16) | \ - (((apid) & 0x3FF) << 0)) + (FIELD_PREP(GENMASK(28, 24), (slave_id)) | \ + FIELD_PREP(GENMASK(23, 16), (periph_id)) | \ + FIELD_PREP(GENMASK(15, 13), (irq_id)) | \ + FIELD_PREP(GENMASK(12, 0), (apid))) -#define hwirq_to_sid(hwirq) (((hwirq) >> 28) & 0xF) -#define hwirq_to_per(hwirq) (((hwirq) >> 20) & 0xFF) -#define hwirq_to_irq(hwirq) (((hwirq) >> 16) & 0x7) -#define hwirq_to_apid(hwirq) (((hwirq) >> 0) & 0x3FF) +#define hwirq_to_sid(hwirq) FIELD_GET(GENMASK(28, 24), (hwirq)) +#define hwirq_to_per(hwirq) FIELD_GET(GENMASK(23, 16), (hwirq)) +#define hwirq_to_irq(hwirq) FIELD_GET(GENMASK(15, 13), (hwirq)) +#define hwirq_to_apid(hwirq) FIELD_GET(GENMASK(12, 0), (hwirq)) struct pmic_arb_ver_ops; @@ -138,11 +145,12 @@ struct spmi_pmic_arb; * @domain: irq domain object for PMIC IRQ domain * @intr: address of the SPMI interrupt control registers. * @cnfg: address of the PMIC Arbiter configuration registers. + * @apid_owner: on v8: address of APID owner mapping table registers * @spmic: spmi controller registered for this bus * @lock: lock to synchronize accesses. - * @base_apid: on v7: minimum APID associated with the particular SPMI - * bus instance - * @apid_count: on v5 and v7: number of APIDs associated with the + * @base_apid: on v7 and v8: minimum APID associated with the + * particular SPMI bus instance + * @apid_count: on v5, v7 and v8: number of APIDs associated with the * particular SPMI bus instance * @mapping_table: in-memory copy of PPID -> APID mapping table. * @mapping_table_valid:bitmap containing valid-only periphs @@ -159,6 +167,7 @@ struct spmi_pmic_arb_bus { struct irq_domain *domain; void __iomem *intr; void __iomem *cnfg; + void __iomem *apid_owner; struct spmi_controller *spmic; raw_spinlock_t lock; u16 base_apid; @@ -181,6 +190,7 @@ struct spmi_pmic_arb_bus { * @wr_base: on v1 "core", on v2 "chnls" register base off DT. * @core: core register base for v2 and above only (see above) * @core_size: core register base size + * @apid_map: on v8, APID mapping table register base * @channel: execution environment channel to use for accesses. * @ee: the current Execution Environment * @ver_ops: version dependent operations. @@ -193,6 +203,7 @@ struct spmi_pmic_arb { void __iomem *wr_base; void __iomem *core; resource_size_t core_size; + void __iomem *apid_map; u8 channel; u8 ee; const struct pmic_arb_ver_ops *ver_ops; @@ -206,6 +217,7 @@ struct spmi_pmic_arb { * * @ver_str: version string. * @get_core_resources: initializes the core, observer and channels + * @get_bus_resources: requests per-SPMI bus register resources * @init_apid: finds the apid base and count * @ppid_to_apid: finds the apid for a given ppid. * @non_data_cmd: on v1 issues an spmi non-data command. @@ -227,6 +239,9 @@ struct spmi_pmic_arb { struct pmic_arb_ver_ops { const char *ver_str; int (*get_core_resources)(struct platform_device *pdev, void __iomem *core); + int (*get_bus_resources)(struct platform_device *pdev, + struct device_node *node, + struct spmi_pmic_arb_bus *bus); int (*init_apid)(struct spmi_pmic_arb_bus *bus, int index); int (*ppid_to_apid)(struct spmi_pmic_arb_bus *bus, u16 ppid); /* spmi commands (read_cmd, write_cmd, cmd) functionality */ @@ -656,7 +671,7 @@ static int periph_interrupt(struct spmi_pmic_arb_bus *bus, u16 apid) unsigned int irq; u32 status, id; int handled = 0; - u8 sid = (bus->apid_data[apid].ppid >> 8) & 0xF; + u8 sid = (bus->apid_data[apid].ppid >> 8) & 0x1F; u8 per = bus->apid_data[apid].ppid & 0xFF; status = readl_relaxed(pmic_arb->ver_ops->irq_status(bus, apid)); @@ -686,7 +701,7 @@ static void pmic_arb_chained_irq(struct irq_desc *desc) int last = bus->max_apid; /* * acc_offset will be non-zero for the secondary SPMI bus instance on - * v7 controllers. + * v7 and v8 controllers. */ int acc_offset = bus->base_apid >> 5; u8 ee = pmic_arb->ee; @@ -913,7 +928,7 @@ static int qpnpint_irq_domain_translate(struct irq_domain *d, return -EINVAL; if (fwspec->param_count != 4) return -EINVAL; - if (intspec[0] > 0xF || intspec[1] > 0xFF || intspec[2] > 0x7) + if (intspec[0] > 0x1F || intspec[1] > 0xFF || intspec[2] > 0x7) return -EINVAL; ppid = intspec[0] << 8 | intspec[1]; @@ -1160,6 +1175,24 @@ static int pmic_arb_ppid_to_apid_v2(struct spmi_pmic_arb_bus *bus, u16 ppid) return apid_valid & ~PMIC_ARB_APID_VALID; } +static void pmic_arb_dump_apid_map(struct spmi_pmic_arb_bus *bus) +{ + struct apid_data *apidd; + u16 apid, ppid; + + /* Dump the mapping table for debug purposes. */ + dev_dbg(&bus->spmic->dev, "PPID APID Write-EE IRQ-EE\n"); + for (ppid = 0; ppid < PMIC_ARB_MAX_PPID; ppid++) { + apid = bus->ppid_to_apid[ppid]; + if (apid & PMIC_ARB_APID_VALID) { + apid &= ~PMIC_ARB_APID_VALID; + apidd = &bus->apid_data[apid]; + dev_dbg(&bus->spmic->dev, "%#03X %3u %2u %2u\n", + ppid, apid, apidd->write_ee, apidd->irq_ee); + } + } +} + static int pmic_arb_read_apid_map_v5(struct spmi_pmic_arb_bus *bus) { struct spmi_pmic_arb *pmic_arb = bus->pmic_arb; @@ -1222,17 +1255,7 @@ static int pmic_arb_read_apid_map_v5(struct spmi_pmic_arb_bus *bus) bus->last_apid = i; } - /* Dump the mapping table for debug purposes. */ - dev_dbg(&bus->spmic->dev, "PPID APID Write-EE IRQ-EE\n"); - for (ppid = 0; ppid < PMIC_ARB_MAX_PPID; ppid++) { - apid = bus->ppid_to_apid[ppid]; - if (apid & PMIC_ARB_APID_VALID) { - apid &= ~PMIC_ARB_APID_VALID; - apidd = &bus->apid_data[apid]; - dev_dbg(&bus->spmic->dev, "%#03X %3u %2u %2u\n", - ppid, apid, apidd->write_ee, apidd->irq_ee); - } - } + pmic_arb_dump_apid_map(bus); return 0; } @@ -1346,7 +1369,7 @@ static int pmic_arb_get_core_resources_v7(struct platform_device *pdev, } /* - * Only v7 supports 2 buses. Each bus will get a different apid count, read + * Arbiter v7 supports 2 buses. Each bus will get a different apid count, read * from different registers. */ static int pmic_arb_init_apid_v7(struct spmi_pmic_arb_bus *bus, int index) @@ -1424,6 +1447,185 @@ static int pmic_arb_offset_v7(struct spmi_pmic_arb_bus *bus, u8 sid, u16 addr, return offset; } +static int pmic_arb_get_core_resources_v8(struct platform_device *pdev, + void __iomem *core) +{ + struct spmi_pmic_arb *pmic_arb = platform_get_drvdata(pdev); + + pmic_arb->apid_map = devm_platform_ioremap_resource_byname(pdev, + "chnl_map"); + if (IS_ERR(pmic_arb->apid_map)) + return PTR_ERR(pmic_arb->apid_map); + + pmic_arb->core = core; + + pmic_arb->max_periphs = PMIC_ARB_MAX_PERIPHS_V8; + + return pmic_arb_get_obsrvr_chnls_v2(pdev); +} + +static int pmic_arb_get_bus_resources_v8(struct platform_device *pdev, + struct device_node *node, + struct spmi_pmic_arb_bus *bus) +{ + int index; + + index = of_property_match_string(node, "reg-names", "chnl_owner"); + if (index < 0) { + dev_err(&pdev->dev, "chnl_owner reg region missing\n"); + return -EINVAL; + } + + bus->apid_owner = devm_of_iomap(&pdev->dev, node, index, NULL); + + return PTR_ERR_OR_ZERO(bus->apid_owner); +} + +static int pmic_arb_read_apid_map_v8(struct spmi_pmic_arb_bus *bus) +{ + struct spmi_pmic_arb *pmic_arb = bus->pmic_arb; + struct apid_data *apidd; + struct apid_data *prev_apidd; + u16 i, apid, ppid, apid_max; + bool valid, is_irq_ee; + u32 regval, offset; + + /* + * In order to allow multiple EEs to write to a single PPID in arbiter + * version 8, there can be more than one APID mapped to each PPID. The + * owner field for each of these mappings specifies the EE which is + * allowed to write to the APID. The owner of the last (highest) APID + * which has the IRQ owner bit set for a given PPID will receive + * interrupts from the PPID. + * + * In arbiter version 8, the APID numbering space is divided between + * the SPMI buses according to this mapping: + * APID = 0 to N-1 --> bus 0 + * APID = N to N+M-1 --> bus 1 + * APID = N+M to N+M+P-1 --> bus 2 + * APID = N+M+P to N+M+P+Q-1 --> bus 3 + * where N = number of APIDs supported by bus 0 + * M = number of APIDs supported by bus 1 + * P = number of APIDs supported by bus 2 + * Q = number of APIDs supported by bus 3 + */ + apidd = &bus->apid_data[bus->base_apid]; + apid_max = bus->base_apid + bus->apid_count; + for (i = bus->base_apid; i < apid_max; i++, apidd++) { + offset = pmic_arb->ver_ops->apid_map_offset(i); + regval = readl_relaxed(pmic_arb->apid_map + offset); + if (!regval) + continue; + ppid = regval & PMIC_ARB_V8_PPID_MASK; + is_irq_ee = PMIC_ARB_V8_CHAN_IS_IRQ_OWNER(regval); + + regval = readl_relaxed(pmic_arb->ver_ops->apid_owner(bus, i)); + apidd->write_ee = SPMI_OWNERSHIP_PERIPH2OWNER(regval); + + apidd->irq_ee = is_irq_ee ? apidd->write_ee : INVALID_EE; + + valid = bus->ppid_to_apid[ppid] & PMIC_ARB_APID_VALID; + apid = bus->ppid_to_apid[ppid] & ~PMIC_ARB_APID_VALID; + prev_apidd = &bus->apid_data[apid]; + + if (!valid || apidd->write_ee == pmic_arb->ee) { + /* First PPID mapping or one for this EE */ + bus->ppid_to_apid[ppid] = i | PMIC_ARB_APID_VALID; + } else if (valid && is_irq_ee && + prev_apidd->write_ee == pmic_arb->ee) { + /* + * Duplicate PPID mapping after the one for this EE; + * override the irq owner + */ + prev_apidd->irq_ee = apidd->irq_ee; + } + + apidd->ppid = ppid; + bus->last_apid = i; + } + + pmic_arb_dump_apid_map(bus); + + return 0; +} + +static int pmic_arb_init_apid_v8(struct spmi_pmic_arb_bus *bus, int index) +{ + struct spmi_pmic_arb *pmic_arb = bus->pmic_arb; + int ret, i; + + if (index < 0 || index >= PMIC_ARB_MAX_BUSES_V8) { + dev_err(&bus->spmic->dev, "Unsupported bus index %d detected\n", + index); + return -EINVAL; + } + + bus->base_apid = 0; + bus->apid_count = 0; + for (i = 0; i <= index; i++) { + bus->base_apid += bus->apid_count; + bus->apid_count = readl_relaxed(pmic_arb->core + PMIC_ARB_FEATURES + i * 4) & + PMIC_ARB_FEATURES_V8_PERIPH_MASK; + } + + if (bus->apid_count == 0) { + dev_err(&bus->spmic->dev, "Bus %d not implemented\n", index); + return -EINVAL; + } else if (bus->base_apid + bus->apid_count > pmic_arb->max_periphs) { + dev_err(&bus->spmic->dev, "Unsupported max APID %d detected\n", + bus->base_apid + bus->apid_count); + return -EINVAL; + } + + ret = pmic_arb_init_apid_min_max(bus); + if (ret) + return ret; + + ret = pmic_arb_read_apid_map_v8(bus); + if (ret) { + dev_err(&bus->spmic->dev, "could not read APID->PPID mapping table, rc= %d\n", + ret); + return ret; + } + + return 0; +} + +/* + * v8 offset per ee and per apid for observer channels and per apid for + * read/write channels. + */ +static int pmic_arb_offset_v8(struct spmi_pmic_arb_bus *bus, u8 sid, u16 addr, + enum pmic_arb_channel ch_type) +{ + struct spmi_pmic_arb *pmic_arb = bus->pmic_arb; + u16 apid; + int rc; + u32 offset = 0; + u16 ppid = (sid << 8) | (addr >> 8); + + rc = pmic_arb->ver_ops->ppid_to_apid(bus, ppid); + if (rc < 0) + return rc; + + apid = rc; + switch (ch_type) { + case PMIC_ARB_CHANNEL_OBS: + offset = 0x40000 * pmic_arb->ee + 0x20 * apid; + break; + case PMIC_ARB_CHANNEL_RW: + if (bus->apid_data[apid].write_ee != pmic_arb->ee) { + dev_err(&bus->spmic->dev, "disallowed SPMI write to sid=%u, addr=0x%04X\n", + sid, addr); + return -EPERM; + } + offset = 0x200 * apid; + break; + } + + return offset; +} + static u32 pmic_arb_fmt_cmd_v1(u8 opc, u8 sid, u16 addr, u8 bc) { return (opc << 27) | ((sid & 0xf) << 20) | (addr << 4) | (bc & 0x7); @@ -1490,6 +1692,14 @@ pmic_arb_acc_enable_v7(struct spmi_pmic_arb_bus *bus, u16 n) return pmic_arb->wr_base + 0x100 + 0x1000 * n; } +static void __iomem * +pmic_arb_acc_enable_v8(struct spmi_pmic_arb_bus *bus, u16 n) +{ + struct spmi_pmic_arb *pmic_arb = bus->pmic_arb; + + return pmic_arb->wr_base + 0x100 + 0x200 * n; +} + static void __iomem * pmic_arb_irq_status_v1(struct spmi_pmic_arb_bus *bus, u16 n) { @@ -1516,6 +1726,14 @@ pmic_arb_irq_status_v7(struct spmi_pmic_arb_bus *bus, u16 n) return pmic_arb->wr_base + 0x104 + 0x1000 * n; } +static void __iomem * +pmic_arb_irq_status_v8(struct spmi_pmic_arb_bus *bus, u16 n) +{ + struct spmi_pmic_arb *pmic_arb = bus->pmic_arb; + + return pmic_arb->wr_base + 0x104 + 0x200 * n; +} + static void __iomem * pmic_arb_irq_clear_v1(struct spmi_pmic_arb_bus *bus, u16 n) { @@ -1542,6 +1760,14 @@ pmic_arb_irq_clear_v7(struct spmi_pmic_arb_bus *bus, u16 n) return pmic_arb->wr_base + 0x108 + 0x1000 * n; } +static void __iomem * +pmic_arb_irq_clear_v8(struct spmi_pmic_arb_bus *bus, u16 n) +{ + struct spmi_pmic_arb *pmic_arb = bus->pmic_arb; + + return pmic_arb->wr_base + 0x108 + 0x200 * n; +} + static u32 pmic_arb_apid_map_offset_v2(u16 n) { return 0x800 + 0x4 * n; @@ -1557,6 +1783,12 @@ static u32 pmic_arb_apid_map_offset_v7(u16 n) return 0x2000 + 0x4 * n; } +static u32 pmic_arb_apid_map_offset_v8(u16 n) +{ + /* For v8, offset is from "chnl_map" base register, not "core". */ + return 0x4 * n; +} + static void __iomem * pmic_arb_apid_owner_v2(struct spmi_pmic_arb_bus *bus, u16 n) { @@ -1564,7 +1796,7 @@ pmic_arb_apid_owner_v2(struct spmi_pmic_arb_bus *bus, u16 n) } /* - * For arbiter version 7, APID ownership table registers have independent + * For arbiter version 7 and 8, APID ownership table registers have independent * numbering space for each SPMI bus instance, so each is indexed starting from * 0. */ @@ -1574,6 +1806,12 @@ pmic_arb_apid_owner_v7(struct spmi_pmic_arb_bus *bus, u16 n) return bus->cnfg + 0x4 * (n - bus->base_apid); } +static void __iomem * +pmic_arb_apid_owner_v8(struct spmi_pmic_arb_bus *bus, u16 n) +{ + return bus->apid_owner + 0x4 * (n - bus->base_apid); +} + static const struct pmic_arb_ver_ops pmic_arb_v1 = { .ver_str = "v1", .get_core_resources = pmic_arb_get_core_resources_v1, @@ -1654,6 +1892,23 @@ static const struct pmic_arb_ver_ops pmic_arb_v7 = { .apid_owner = pmic_arb_apid_owner_v7, }; +static const struct pmic_arb_ver_ops pmic_arb_v8 = { + .ver_str = "v8", + .get_core_resources = pmic_arb_get_core_resources_v8, + .get_bus_resources = pmic_arb_get_bus_resources_v8, + .init_apid = pmic_arb_init_apid_v8, + .ppid_to_apid = pmic_arb_ppid_to_apid_v5, + .non_data_cmd = pmic_arb_non_data_cmd_v2, + .offset = pmic_arb_offset_v8, + .fmt_cmd = pmic_arb_fmt_cmd_v2, + .owner_acc_status = pmic_arb_owner_acc_status_v7, + .acc_enable = pmic_arb_acc_enable_v8, + .irq_status = pmic_arb_irq_status_v8, + .irq_clear = pmic_arb_irq_clear_v8, + .apid_map_offset = pmic_arb_apid_map_offset_v8, + .apid_owner = pmic_arb_apid_owner_v8, +}; + static const struct irq_domain_ops pmic_arb_irq_domain_ops = { .activate = qpnpint_irq_domain_activate, .alloc = qpnpint_irq_domain_alloc, @@ -1731,6 +1986,12 @@ static int spmi_pmic_arb_bus_init(struct platform_device *pdev, bus->spmic = ctrl; bus->id = bus_index; + if (pmic_arb->ver_ops->get_bus_resources) { + ret = pmic_arb->ver_ops->get_bus_resources(pdev, node, bus); + if (ret) + return ret; + } + ret = pmic_arb->ver_ops->init_apid(bus, bus_index); if (ret) return ret; @@ -1825,8 +2086,10 @@ static int spmi_pmic_arb_probe(struct platform_device *pdev) pmic_arb->ver_ops = &pmic_arb_v3; else if (hw_ver < PMIC_ARB_VERSION_V7_MIN) pmic_arb->ver_ops = &pmic_arb_v5; - else + else if (hw_ver < PMIC_ARB_VERSION_V8_MIN) pmic_arb->ver_ops = &pmic_arb_v7; + else + pmic_arb->ver_ops = &pmic_arb_v8; err = pmic_arb->ver_ops->get_core_resources(pdev, core); if (err) @@ -1875,6 +2138,7 @@ static void spmi_pmic_arb_remove(struct platform_device *pdev) static const struct of_device_id spmi_pmic_arb_match_table[] = { { .compatible = "qcom,spmi-pmic-arb", }, { .compatible = "qcom,x1e80100-spmi-pmic-arb", }, + { .compatible = "qcom,glymur-spmi-pmic-arb", }, {}, }; MODULE_DEVICE_TABLE(of, spmi_pmic_arb_match_table); diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c index 9e7b84071174c..8a5ccc8ae0a18 100644 --- a/drivers/staging/fbtft/fbtft-core.c +++ b/drivers/staging/fbtft/fbtft-core.c @@ -1171,8 +1171,8 @@ int fbtft_probe_common(struct fbtft_display *display, par->pdev = pdev; if (display->buswidth == 0) { - dev_err(dev, "buswidth is not set\n"); - return -EINVAL; + ret = dev_err_probe(dev, -EINVAL, "buswidth is not set\n"); + goto out_release; } /* write register functions */ diff --git a/drivers/staging/gpib/Kconfig b/drivers/staging/gpib/Kconfig index aa01538d5bebb..75418c58a1680 100644 --- a/drivers/staging/gpib/Kconfig +++ b/drivers/staging/gpib/Kconfig @@ -122,6 +122,7 @@ config GPIB_FLUKE depends on OF select GPIB_COMMON select GPIB_NEC7210 + depends on HAS_IOMEM help GPIB driver for Fluke based cda devices. diff --git a/drivers/staging/gpib/common/gpib_os.c b/drivers/staging/gpib/common/gpib_os.c index 9dbbac8b84360..baa2fea5ebf7e 100644 --- a/drivers/staging/gpib/common/gpib_os.c +++ b/drivers/staging/gpib/common/gpib_os.c @@ -888,10 +888,6 @@ static int read_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo if (read_cmd.completed_transfer_count > read_cmd.requested_transfer_count) return -EINVAL; - desc = handle_to_descriptor(file_priv, read_cmd.handle); - if (!desc) - return -EINVAL; - if (WARN_ON_ONCE(sizeof(userbuf) > sizeof(read_cmd.buffer_ptr))) return -EFAULT; @@ -904,6 +900,17 @@ static int read_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo if (!access_ok(userbuf, remain)) return -EFAULT; + /* Lock descriptors to prevent concurrent close from freeing descriptor */ + if (mutex_lock_interruptible(&file_priv->descriptors_mutex)) + return -ERESTARTSYS; + desc = handle_to_descriptor(file_priv, read_cmd.handle); + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); + atomic_set(&desc->io_in_progress, 1); /* Read buffer loads till we fill the user supplied buffer */ @@ -937,6 +944,7 @@ static int read_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo retval = copy_to_user((void __user *)arg, &read_cmd, sizeof(read_cmd)); atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); if (retval) @@ -964,10 +972,6 @@ static int command_ioctl(struct gpib_file_private *file_priv, if (cmd.completed_transfer_count > cmd.requested_transfer_count) return -EINVAL; - desc = handle_to_descriptor(file_priv, cmd.handle); - if (!desc) - return -EINVAL; - userbuf = (u8 __user *)(unsigned long)cmd.buffer_ptr; userbuf += cmd.completed_transfer_count; @@ -980,6 +984,17 @@ static int command_ioctl(struct gpib_file_private *file_priv, if (!access_ok(userbuf, remain)) return -EFAULT; + /* Lock descriptors to prevent concurrent close from freeing descriptor */ + if (mutex_lock_interruptible(&file_priv->descriptors_mutex)) + return -ERESTARTSYS; + desc = handle_to_descriptor(file_priv, cmd.handle); + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); + /* * Write buffer loads till we empty the user supplied buffer. * Call drivers at least once, even if remain is zero, in @@ -1003,6 +1018,7 @@ static int command_ioctl(struct gpib_file_private *file_priv, userbuf += bytes_written; if (retval < 0) { atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); break; @@ -1022,6 +1038,7 @@ static int command_ioctl(struct gpib_file_private *file_priv, */ if (!no_clear_io_in_prog || fault) atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); if (fault) @@ -1047,10 +1064,6 @@ static int write_ioctl(struct gpib_file_private *file_priv, struct gpib_board *b if (write_cmd.completed_transfer_count > write_cmd.requested_transfer_count) return -EINVAL; - desc = handle_to_descriptor(file_priv, write_cmd.handle); - if (!desc) - return -EINVAL; - userbuf = (u8 __user *)(unsigned long)write_cmd.buffer_ptr; userbuf += write_cmd.completed_transfer_count; @@ -1060,6 +1073,17 @@ static int write_ioctl(struct gpib_file_private *file_priv, struct gpib_board *b if (!access_ok(userbuf, remain)) return -EFAULT; + /* Lock descriptors to prevent concurrent close from freeing descriptor */ + if (mutex_lock_interruptible(&file_priv->descriptors_mutex)) + return -ERESTARTSYS; + desc = handle_to_descriptor(file_priv, write_cmd.handle); + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); + atomic_set(&desc->io_in_progress, 1); /* Write buffer loads till we empty the user supplied buffer */ @@ -1094,6 +1118,7 @@ static int write_ioctl(struct gpib_file_private *file_priv, struct gpib_board *b fault = copy_to_user((void __user *)arg, &write_cmd, sizeof(write_cmd)); atomic_set(&desc->io_in_progress, 0); + atomic_dec(&desc->descriptor_busy); wake_up_interruptible(&board->wait); if (fault) @@ -1276,6 +1301,9 @@ static int close_dev_ioctl(struct file *filep, struct gpib_board *board, unsigne { struct gpib_close_dev_ioctl cmd; struct gpib_file_private *file_priv = filep->private_data; + struct gpib_descriptor *desc; + unsigned int pad; + int sad; int retval; retval = copy_from_user(&cmd, (void __user *)arg, sizeof(cmd)); @@ -1284,19 +1312,27 @@ static int close_dev_ioctl(struct file *filep, struct gpib_board *board, unsigne if (cmd.handle >= GPIB_MAX_NUM_DESCRIPTORS) return -EINVAL; - if (!file_priv->descriptors[cmd.handle]) - return -EINVAL; - retval = decrement_open_device_count(board, &board->device_list, - file_priv->descriptors[cmd.handle]->pad, - file_priv->descriptors[cmd.handle]->sad); - if (retval < 0) - return retval; - - kfree(file_priv->descriptors[cmd.handle]); + mutex_lock(&file_priv->descriptors_mutex); + desc = file_priv->descriptors[cmd.handle]; + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EINVAL; + } + if (atomic_read(&desc->descriptor_busy)) { + mutex_unlock(&file_priv->descriptors_mutex); + return -EBUSY; + } + /* Remove from table while holding lock to prevent new IO from starting */ file_priv->descriptors[cmd.handle] = NULL; + pad = desc->pad; + sad = desc->sad; + mutex_unlock(&file_priv->descriptors_mutex); - return 0; + retval = decrement_open_device_count(board, &board->device_list, pad, sad); + + kfree(desc); + return retval; } static int serial_poll_ioctl(struct gpib_board *board, unsigned long arg) @@ -1331,12 +1367,25 @@ static int wait_ioctl(struct gpib_file_private *file_priv, struct gpib_board *bo if (retval) return -EFAULT; + /* + * Lock descriptors to prevent concurrent close from freeing + * descriptor. ibwait() releases big_gpib_mutex when wait_mask + * is non-zero, so desc must be pinned with descriptor_busy. + */ + mutex_lock(&file_priv->descriptors_mutex); desc = handle_to_descriptor(file_priv, wait_cmd.handle); - if (!desc) + if (!desc) { + mutex_unlock(&file_priv->descriptors_mutex); return -EINVAL; + } + atomic_inc(&desc->descriptor_busy); + mutex_unlock(&file_priv->descriptors_mutex); retval = ibwait(board, wait_cmd.wait_mask, wait_cmd.clear_mask, wait_cmd.set_mask, &wait_cmd.ibsta, wait_cmd.usec_timeout, desc); + + atomic_dec(&desc->descriptor_busy); + if (retval < 0) return retval; @@ -2035,6 +2084,7 @@ void init_gpib_descriptor(struct gpib_descriptor *desc) desc->is_board = 0; desc->autopoll_enabled = 0; atomic_set(&desc->io_in_progress, 0); + atomic_set(&desc->descriptor_busy, 0); } int gpib_register_driver(struct gpib_interface *interface, struct module *provider_module) diff --git a/drivers/staging/gpib/common/iblib.c b/drivers/staging/gpib/common/iblib.c index 7cbb6a467177d..b672dd6aad25f 100644 --- a/drivers/staging/gpib/common/iblib.c +++ b/drivers/staging/gpib/common/iblib.c @@ -227,11 +227,10 @@ int ibonline(struct gpib_board *board) #ifndef CONFIG_NIOS2 board->autospoll_task = kthread_run(&autospoll_thread, board, "gpib%d_autospoll_kthread", board->minor); - retval = IS_ERR(board->autospoll_task); - if (retval) { + if (IS_ERR(board->autospoll_task)) { dev_err(board->gpib_dev, "failed to create autospoll thread\n"); board->interface->detach(board); - return retval; + return PTR_ERR(board->autospoll_task); } #endif board->online = 1; diff --git a/drivers/staging/gpib/include/gpib_types.h b/drivers/staging/gpib/include/gpib_types.h index 998abb379749f..f9b5be1dc3663 100644 --- a/drivers/staging/gpib/include/gpib_types.h +++ b/drivers/staging/gpib/include/gpib_types.h @@ -364,6 +364,14 @@ struct gpib_descriptor { unsigned int pad; /* primary gpib address */ int sad; /* secondary gpib address (negative means disabled) */ atomic_t io_in_progress; + /* + * Kernel-only reference count to prevent descriptor from being + * freed while IO handlers hold a pointer to it. Incremented + * before each IO operation, decremented when done. Unlike + * io_in_progress, this cannot be modified from userspace via + * general_ibstatus(). + */ + atomic_t descriptor_busy; unsigned is_board : 1; unsigned autopoll_enabled : 1; }; diff --git a/drivers/staging/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c b/drivers/staging/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c index dd68c4843490a..e13d56c301e53 100644 --- a/drivers/staging/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c +++ b/drivers/staging/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c @@ -405,7 +405,7 @@ static int usb_gpib_attach(struct gpib_board *board, const struct gpib_board_con for (j = 0 ; j < MAX_DEV ; j++) { if ((assigned_usb_minors & 1 << j) == 0) continue; - udev = usb_get_dev(interface_to_usbdev(lpvo_usb_interfaces[j])); + udev = interface_to_usbdev(lpvo_usb_interfaces[j]); device_path = kobject_get_path(&udev->dev.kobj, GFP_KERNEL); match = gpib_match_device_path(&lpvo_usb_interfaces[j]->dev, config->device_path); @@ -420,7 +420,7 @@ static int usb_gpib_attach(struct gpib_board *board, const struct gpib_board_con for (j = 0 ; j < MAX_DEV ; j++) { if ((assigned_usb_minors & 1 << j) == 0) continue; - udev = usb_get_dev(interface_to_usbdev(lpvo_usb_interfaces[j])); + udev = interface_to_usbdev(lpvo_usb_interfaces[j]); DIA_LOG(1, "dev. %d: bus %d -> %d dev: %d -> %d\n", j, udev->bus->busnum, config->pci_bus, udev->devnum, config->pci_slot); if (config->pci_bus == udev->bus->busnum && diff --git a/drivers/staging/gpib/ni_usb/ni_usb_gpib.c b/drivers/staging/gpib/ni_usb/ni_usb_gpib.c index 1f8412de9fa32..b6fddb437f552 100644 --- a/drivers/staging/gpib/ni_usb/ni_usb_gpib.c +++ b/drivers/staging/gpib/ni_usb/ni_usb_gpib.c @@ -566,7 +566,7 @@ static int ni_usb_write_registers(struct ni_usb_priv *ni_priv, retval, bytes_read); ni_usb_dump_raw_block(in_data, bytes_read); kfree(in_data); - return retval; + return retval ?: -EINVAL; } mutex_unlock(&ni_priv->addressed_transfer_lock); @@ -1780,7 +1780,7 @@ static int ni_usb_setup_init(struct gpib_board *board, struct ni_usb_register *w i++; if (i > NUM_INIT_WRITES) { dev_err(&usb_dev->dev, "bug!, buffer overrun, i=%i\n", i); - return 0; + return -EINVAL; } return i; } @@ -1799,10 +1799,12 @@ static int ni_usb_init(struct gpib_board *board) return -ENOMEM; writes_len = ni_usb_setup_init(board, writes); - if (writes_len) - retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); - else - return -EFAULT; + if (writes_len < 0) { + kfree(writes); + return writes_len; + } + + retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); kfree(writes); if (retval) { dev_err(&usb_dev->dev, "register write failed, retval=%i\n", retval); diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c index e509fdc715dbb..38c233a706c48 100644 --- a/drivers/staging/greybus/light.c +++ b/drivers/staging/greybus/light.c @@ -1008,14 +1008,18 @@ static int gb_lights_light_config(struct gb_lights *glights, u8 id) if (!strlen(conf.name)) return -EINVAL; - light->channels_count = conf.channel_count; light->name = kstrndup(conf.name, NAMES_MAX, GFP_KERNEL); if (!light->name) return -ENOMEM; - light->channels = kcalloc(light->channels_count, + light->channels = kcalloc(conf.channel_count, sizeof(struct gb_channel), GFP_KERNEL); if (!light->channels) return -ENOMEM; + /* + * Publish channels_count only after channels allocation so cleanup + * doesn't walk a NULL channels pointer on allocation failure. + */ + light->channels_count = conf.channel_count; /* First we collect all the configurations for all channels */ for (i = 0; i < light->channels_count; i++) { diff --git a/drivers/staging/media/ipu7/ipu7-buttress.c b/drivers/staging/media/ipu7/ipu7-buttress.c index e5707f5e300ba..40c6c8473357c 100644 --- a/drivers/staging/media/ipu7/ipu7-buttress.c +++ b/drivers/staging/media/ipu7/ipu7-buttress.c @@ -342,14 +342,23 @@ irqreturn_t ipu_buttress_isr(int irq, void *isp_ptr) u32 disable_irqs = 0; u32 irq_status; unsigned int i; + int active; - pm_runtime_get_noresume(dev); + active = pm_runtime_get_if_active(dev); + if (active <= 0) + return IRQ_NONE; pb_irq = readl(isp->pb_base + INTERRUPT_STATUS); writel(pb_irq, isp->pb_base + INTERRUPT_STATUS); /* check btrs ATS, CFI and IMR errors, BIT(0) is unused for IPU */ pb_local_irq = readl(isp->pb_base + BTRS_LOCAL_INTERRUPT_MASK); + if (pb_local_irq == 0xffffffff) { + dev_warn_once(dev, "invalid PB irq status\n"); + pm_runtime_put_noidle(dev); + return IRQ_NONE; + } + if (pb_local_irq & ~BIT(0)) { dev_warn(dev, "PB interrupt status 0x%x local 0x%x\n", pb_irq, pb_local_irq); @@ -370,6 +379,12 @@ irqreturn_t ipu_buttress_isr(int irq, void *isp_ptr) return IRQ_NONE; } + if (irq_status == 0xffffffff) { + dev_warn_once(dev, "invalid irq status 0x%08x\n", irq_status); + pm_runtime_put_noidle(dev); + return IRQ_NONE; + } + do { writel(irq_status, isp->base + BUTTRESS_REG_IRQ_CLEAR); diff --git a/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c b/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c index b8c5db7ae3009..d8f3592ad6201 100644 --- a/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c +++ b/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c @@ -124,6 +124,7 @@ static const struct cdr_fbk_cap_prog_params table7[] = { { 1350, 1589, 4 }, { 1590, 1949, 5 }, { 1950, 2499, 6 }, + { 2500, 3500, 7 }, { } }; @@ -838,9 +839,10 @@ static void ipu7_isys_cphy_config(struct ipu7_isys *isys, u8 id, u8 lanes, dwc_phy_write_mask(isys, id, reg + 0x400 * i, reset_thresh, 9, 11); + /* Tuning ITMINRX to 2 for CPHY */ reg = CORE_DIG_CLANE_0_RW_LP_0; for (i = 0; i < trios; i++) - dwc_phy_write_mask(isys, id, reg + 0x400 * i, 1, 12, 15); + dwc_phy_write_mask(isys, id, reg + 0x400 * i, 2, 12, 15); reg = CORE_DIG_CLANE_0_RW_LP_2; for (i = 0; i < trios; i++) @@ -860,7 +862,11 @@ static void ipu7_isys_cphy_config(struct ipu7_isys *isys, u8 id, u8 lanes, for (i = 0; i < (lanes + 1); i++) { reg = CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9 + 0x400 * i; dwc_phy_write_mask(isys, id, reg, 4U, 0, 2); - dwc_phy_write_mask(isys, id, reg, 0U, 3, 4); + /* Set GMODE to 2 when CPHY >= 1.5Gsps */ + if (mbps >= 1500) + dwc_phy_write_mask(isys, id, reg, 2U, 3, 4); + else + dwc_phy_write_mask(isys, id, reg, 0U, 3, 4); reg = CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_7 + 0x400 * i; dwc_phy_write_mask(isys, id, reg, cap_prog, 10, 12); @@ -930,8 +936,9 @@ static int ipu7_isys_phy_config(struct ipu7_isys *isys, u8 id, u8 lanes, 7, 12, 14); dwc_phy_write_mask(isys, id, CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7, 0, 8, 10); + /* resistance tuning: 1 for 45ohm, 0 for 50ohm */ dwc_phy_write_mask(isys, id, CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_5, - 0, 8, 8); + 1, 8, 8); if (aggregation) phy_mode = isys->csi2[0].phy_mode; diff --git a/drivers/staging/media/ipu7/ipu7-mmu.c b/drivers/staging/media/ipu7/ipu7-mmu.c index ded1986eb8ba3..ea35cce4830ad 100644 --- a/drivers/staging/media/ipu7/ipu7-mmu.c +++ b/drivers/staging/media/ipu7/ipu7-mmu.c @@ -231,7 +231,7 @@ static u32 *alloc_l2_pt(struct ipu7_mmu_info *mmu_info) dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt); - for (i = 0; i < ISP_L1PT_PTES; i++) + for (i = 0; i < ISP_L2PT_PTES; i++) pt[i] = mmu_info->dummy_page_pteval; return pt; diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c index 5cddc09c72bf2..fa5a1867626f8 100644 --- a/drivers/staging/media/ipu7/ipu7.c +++ b/drivers/staging/media/ipu7/ipu7.c @@ -2620,7 +2620,7 @@ static int ipu7_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu)) ipu7_mmu_cleanup(isp->isys->mmu); if (!IS_ERR_OR_NULL(isp->psys)) - pm_runtime_put(&isp->psys->auxdev.dev); + pm_runtime_put_sync(&isp->psys->auxdev.dev); ipu7_bus_del_devices(pdev); release_firmware(isp->cpd_fw); buttress_exit: @@ -2684,6 +2684,10 @@ static void ipu7_pci_reset_done(struct pci_dev *pdev) */ static int ipu7_suspend(struct device *dev) { + struct pci_dev *pdev = to_pci_dev(dev); + + synchronize_irq(pdev->irq); + return 0; } diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c index c9276ff76157f..14b327afe045e 100644 --- a/drivers/staging/media/tegra-video/vi.c +++ b/drivers/staging/media/tegra-video/vi.c @@ -438,7 +438,7 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, .target = V4L2_SEL_TGT_CROP_BOUNDS, }; struct v4l2_rect *try_crop; - int ret; + int ret = 0; subdev = tegra_channel_get_remote_source_subdev(chan); if (!subdev) @@ -482,8 +482,10 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, } else { ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel); - if (ret) - return -EINVAL; + if (ret) { + ret = -EINVAL; + goto out_free; + } try_crop->width = sdsel.r.width; try_crop->height = sdsel.r.height; @@ -495,14 +497,15 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, ret = v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &fmt); if (ret < 0) - return ret; + goto out_free; v4l2_fill_pix_format(pix, &fmt.format); chan->vi->ops->vi_fmt_align(pix, fmtinfo->bpp); +out_free: __v4l2_subdev_state_free(sd_state); - return 0; + return ret; } static int tegra_channel_try_format(struct file *file, void *fh, diff --git a/drivers/staging/most/Kconfig b/drivers/staging/most/Kconfig index 6f420cbcdcfff..e89658df6f124 100644 --- a/drivers/staging/most/Kconfig +++ b/drivers/staging/most/Kconfig @@ -24,6 +24,4 @@ source "drivers/staging/most/video/Kconfig" source "drivers/staging/most/dim2/Kconfig" -source "drivers/staging/most/i2c/Kconfig" - endif diff --git a/drivers/staging/most/Makefile b/drivers/staging/most/Makefile index 8b3fc5a7af514..e45084df7803a 100644 --- a/drivers/staging/most/Makefile +++ b/drivers/staging/most/Makefile @@ -3,4 +3,3 @@ obj-$(CONFIG_MOST_NET) += net/ obj-$(CONFIG_MOST_VIDEO) += video/ obj-$(CONFIG_MOST_DIM2) += dim2/ -obj-$(CONFIG_MOST_I2C) += i2c/ diff --git a/drivers/staging/most/i2c/Kconfig b/drivers/staging/most/i2c/Kconfig deleted file mode 100644 index ff64283cbad18..0000000000000 --- a/drivers/staging/most/i2c/Kconfig +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# MOST I2C configuration -# - -config MOST_I2C - tristate "I2C" - depends on I2C - help - Say Y here if you want to connect via I2C to network transceiver. - - To compile this driver as a module, choose M here: the - module will be called most_i2c. diff --git a/drivers/staging/most/i2c/Makefile b/drivers/staging/most/i2c/Makefile deleted file mode 100644 index 71099dd0f85b9..0000000000000 --- a/drivers/staging/most/i2c/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_MOST_I2C) += most_i2c.o - -most_i2c-objs := i2c.o diff --git a/drivers/staging/most/i2c/i2c.c b/drivers/staging/most/i2c/i2c.c deleted file mode 100644 index 184b2dd11fc34..0000000000000 --- a/drivers/staging/most/i2c/i2c.c +++ /dev/null @@ -1,374 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * i2c.c - Hardware Dependent Module for I2C Interface - * - * Copyright (C) 2013-2015, Microchip Technology Germany II GmbH & Co. KG - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include - -enum { CH_RX, CH_TX, NUM_CHANNELS }; - -#define MAX_BUFFERS_CONTROL 32 -#define MAX_BUF_SIZE_CONTROL 256 - -/** - * list_first_mbo - get the first mbo from a list - * @ptr: the list head to take the mbo from. - */ -#define list_first_mbo(ptr) \ - list_first_entry(ptr, struct mbo, list) - -static unsigned int polling_rate; -module_param(polling_rate, uint, 0644); -MODULE_PARM_DESC(polling_rate, "Polling rate [Hz]. Default = 0 (use IRQ)"); - -struct hdm_i2c { - struct most_interface most_iface; - struct most_channel_capability capabilities[NUM_CHANNELS]; - struct i2c_client *client; - struct rx { - struct delayed_work dwork; - struct list_head list; - bool int_disabled; - unsigned int delay; - } rx; - char name[64]; -}; - -static inline struct hdm_i2c *to_hdm(struct most_interface *iface) -{ - return container_of(iface, struct hdm_i2c, most_iface); -} - -static irqreturn_t most_irq_handler(int, void *); -static void pending_rx_work(struct work_struct *); - -/** - * configure_channel - called from MOST core to configure a channel - * @most_iface: interface the channel belongs to - * @ch_idx: channel to be configured - * @channel_config: structure that holds the configuration information - * - * Return 0 on success, negative on failure. - * - * Receives configuration information from MOST core and initialize the - * corresponding channel. - */ -static int configure_channel(struct most_interface *most_iface, - int ch_idx, - struct most_channel_config *channel_config) -{ - int ret; - struct hdm_i2c *dev = to_hdm(most_iface); - unsigned int delay, pr; - - BUG_ON(ch_idx < 0 || ch_idx >= NUM_CHANNELS); - - if (channel_config->data_type != MOST_CH_CONTROL) { - pr_err("bad data type for channel %d\n", ch_idx); - return -EPERM; - } - - if (channel_config->direction != dev->capabilities[ch_idx].direction) { - pr_err("bad direction for channel %d\n", ch_idx); - return -EPERM; - } - - if (channel_config->direction == MOST_CH_RX) { - if (!polling_rate) { - if (dev->client->irq <= 0) { - pr_err("bad irq: %d\n", dev->client->irq); - return -ENOENT; - } - dev->rx.int_disabled = false; - ret = request_irq(dev->client->irq, most_irq_handler, 0, - dev->client->name, dev); - if (ret) { - pr_err("request_irq(%d) failed: %d\n", - dev->client->irq, ret); - return ret; - } - } else { - delay = msecs_to_jiffies(MSEC_PER_SEC / polling_rate); - dev->rx.delay = delay ? delay : 1; - pr = MSEC_PER_SEC / jiffies_to_msecs(dev->rx.delay); - pr_info("polling rate is %u Hz\n", pr); - } - } - - return 0; -} - -/** - * enqueue - called from MOST core to enqueue a buffer for data transfer - * @most_iface: intended interface - * @ch_idx: ID of the channel the buffer is intended for - * @mbo: pointer to the buffer object - * - * Return 0 on success, negative on failure. - * - * Transmit the data over I2C if it is a "write" request or push the buffer into - * list if it is an "read" request - */ -static int enqueue(struct most_interface *most_iface, - int ch_idx, struct mbo *mbo) -{ - struct hdm_i2c *dev = to_hdm(most_iface); - int ret; - - BUG_ON(ch_idx < 0 || ch_idx >= NUM_CHANNELS); - - if (ch_idx == CH_RX) { - /* RX */ - if (!polling_rate) - disable_irq(dev->client->irq); - cancel_delayed_work_sync(&dev->rx.dwork); - list_add_tail(&mbo->list, &dev->rx.list); - if (dev->rx.int_disabled || polling_rate) - pending_rx_work(&dev->rx.dwork.work); - if (!polling_rate) - enable_irq(dev->client->irq); - } else { - /* TX */ - ret = i2c_master_send(dev->client, mbo->virt_address, - mbo->buffer_length); - if (ret <= 0) { - mbo->processed_length = 0; - mbo->status = MBO_E_INVAL; - } else { - mbo->processed_length = mbo->buffer_length; - mbo->status = MBO_SUCCESS; - } - mbo->complete(mbo); - } - - return 0; -} - -/** - * poison_channel - called from MOST core to poison buffers of a channel - * @most_iface: pointer to the interface the channel to be poisoned belongs to - * @ch_idx: corresponding channel ID - * - * Return 0 on success, negative on failure. - * - * If channel direction is RX, complete the buffers in list with - * status MBO_E_CLOSE - */ -static int poison_channel(struct most_interface *most_iface, - int ch_idx) -{ - struct hdm_i2c *dev = to_hdm(most_iface); - struct mbo *mbo; - - BUG_ON(ch_idx < 0 || ch_idx >= NUM_CHANNELS); - - if (ch_idx == CH_RX) { - if (!polling_rate) - free_irq(dev->client->irq, dev); - cancel_delayed_work_sync(&dev->rx.dwork); - - while (!list_empty(&dev->rx.list)) { - mbo = list_first_mbo(&dev->rx.list); - list_del(&mbo->list); - - mbo->processed_length = 0; - mbo->status = MBO_E_CLOSE; - mbo->complete(mbo); - } - } - - return 0; -} - -static void do_rx_work(struct hdm_i2c *dev) -{ - struct mbo *mbo; - unsigned char msg[MAX_BUF_SIZE_CONTROL]; - int ret; - u16 pml, data_size; - - /* Read PML (2 bytes) */ - ret = i2c_master_recv(dev->client, msg, 2); - if (ret <= 0) { - pr_err("Failed to receive PML\n"); - return; - } - - pml = (msg[0] << 8) | msg[1]; - if (!pml) - return; - - data_size = pml + 2; - - /* Read the whole message, including PML */ - ret = i2c_master_recv(dev->client, msg, data_size); - if (ret <= 0) { - pr_err("Failed to receive a Port Message\n"); - return; - } - - mbo = list_first_mbo(&dev->rx.list); - list_del(&mbo->list); - - mbo->processed_length = min(data_size, mbo->buffer_length); - memcpy(mbo->virt_address, msg, mbo->processed_length); - mbo->status = MBO_SUCCESS; - mbo->complete(mbo); -} - -/** - * pending_rx_work - Read pending messages through I2C - * @work: definition of this work item - * - * Invoked by the Interrupt Service Routine, most_irq_handler() - */ -static void pending_rx_work(struct work_struct *work) -{ - struct hdm_i2c *dev = container_of(work, struct hdm_i2c, rx.dwork.work); - - if (list_empty(&dev->rx.list)) - return; - - do_rx_work(dev); - - if (polling_rate) { - schedule_delayed_work(&dev->rx.dwork, dev->rx.delay); - } else { - dev->rx.int_disabled = false; - enable_irq(dev->client->irq); - } -} - -/* - * most_irq_handler - Interrupt Service Routine - * @irq: irq number - * @_dev: private data - * - * Schedules a delayed work - * - * By default the interrupt line behavior is Active Low. Once an interrupt is - * generated by the device, until driver clears the interrupt (by reading - * the PMP message), device keeps the interrupt line in low state. Since i2c - * read is done in work queue, the interrupt line must be disabled temporarily - * to avoid ISR being called repeatedly. Re-enable the interrupt in workqueue, - * after reading the message. - * - * Note: If we use the interrupt line in Falling edge mode, there is a - * possibility to miss interrupts when ISR is getting executed. - * - */ -static irqreturn_t most_irq_handler(int irq, void *_dev) -{ - struct hdm_i2c *dev = _dev; - - disable_irq_nosync(irq); - dev->rx.int_disabled = true; - schedule_delayed_work(&dev->rx.dwork, 0); - - return IRQ_HANDLED; -} - -/* - * i2c_probe - i2c probe handler - * @client: i2c client device structure - * @id: i2c client device id - * - * Return 0 on success, negative on failure. - * - * Register the i2c client device as a MOST interface - */ -static int i2c_probe(struct i2c_client *client) -{ - struct hdm_i2c *dev; - int ret, i; - - dev = kzalloc(sizeof(*dev), GFP_KERNEL); - if (!dev) - return -ENOMEM; - - /* ID format: i2c--
*/ - snprintf(dev->name, sizeof(dev->name), "i2c-%d-%04x", - client->adapter->nr, client->addr); - - for (i = 0; i < NUM_CHANNELS; i++) { - dev->capabilities[i].data_type = MOST_CH_CONTROL; - dev->capabilities[i].num_buffers_packet = MAX_BUFFERS_CONTROL; - dev->capabilities[i].buffer_size_packet = MAX_BUF_SIZE_CONTROL; - } - dev->capabilities[CH_RX].direction = MOST_CH_RX; - dev->capabilities[CH_RX].name_suffix = "rx"; - dev->capabilities[CH_TX].direction = MOST_CH_TX; - dev->capabilities[CH_TX].name_suffix = "tx"; - - dev->most_iface.interface = ITYPE_I2C; - dev->most_iface.description = dev->name; - dev->most_iface.num_channels = NUM_CHANNELS; - dev->most_iface.channel_vector = dev->capabilities; - dev->most_iface.configure = configure_channel; - dev->most_iface.enqueue = enqueue; - dev->most_iface.poison_channel = poison_channel; - - INIT_LIST_HEAD(&dev->rx.list); - - INIT_DELAYED_WORK(&dev->rx.dwork, pending_rx_work); - - dev->client = client; - i2c_set_clientdata(client, dev); - - ret = most_register_interface(&dev->most_iface); - if (ret) { - pr_err("Failed to register i2c as a MOST interface\n"); - kfree(dev); - return ret; - } - - return 0; -} - -/* - * i2c_remove - i2c remove handler - * @client: i2c client device structure - * - * Return 0 on success. - * - * Unregister the i2c client device as a MOST interface - */ -static void i2c_remove(struct i2c_client *client) -{ - struct hdm_i2c *dev = i2c_get_clientdata(client); - - most_deregister_interface(&dev->most_iface); - kfree(dev); -} - -static const struct i2c_device_id i2c_id[] = { - { "most_i2c" }, - { } /* Terminating entry */ -}; - -MODULE_DEVICE_TABLE(i2c, i2c_id); - -static struct i2c_driver i2c_driver = { - .driver = { - .name = "hdm_i2c", - }, - .probe = i2c_probe, - .remove = i2c_remove, - .id_table = i2c_id, -}; - -module_i2c_driver(i2c_driver); - -MODULE_AUTHOR("Andrey Shvetsov "); -MODULE_DESCRIPTION("I2C Hardware Dependent Module"); -MODULE_LICENSE("GPL"); diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c index 53d4c113b19c8..9ecd00f309461 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c +++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c @@ -140,22 +140,24 @@ u8 *rtw_get_ie(u8 *pbuf, signed int index, signed int *len, signed int limit) signed int tmp, i; u8 *p; - if (limit < 1) + if (limit < 2) return NULL; p = pbuf; i = 0; *len = 0; - while (1) { + while (i + 2 <= limit) { + tmp = *(p + 1); + if (i + 2 + tmp > limit) + break; + if (*p == index) { - *len = *(p + 1); + *len = tmp; return p; } - tmp = *(p + 1); + p += (tmp + 2); i += (tmp + 2); - if (i >= limit) - break; } return NULL; } @@ -185,20 +187,25 @@ u8 *rtw_get_ie_ex(u8 *in_ie, uint in_len, u8 eid, u8 *oui, u8 oui_len, u8 *ie, u cnt = 0; - while (cnt < in_len) { + while (cnt + 2 <= in_len) { + u8 ie_len = in_ie[cnt + 1]; + + if (cnt + 2 + ie_len > in_len) + break; + if (eid == in_ie[cnt] - && (!oui || !memcmp(&in_ie[cnt+2], oui, oui_len))) { + && (!oui || (ie_len >= oui_len && !memcmp(&in_ie[cnt + 2], oui, oui_len)))) { target_ie = &in_ie[cnt]; if (ie) - memcpy(ie, &in_ie[cnt], in_ie[cnt+1]+2); + memcpy(ie, &in_ie[cnt], ie_len + 2); if (ielen) - *ielen = in_ie[cnt+1]+2; + *ielen = ie_len + 2; break; } - cnt += in_ie[cnt+1]+2; /* goto next */ + cnt += ie_len + 2; /* goto next */ } return target_ie; diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c index c06d990350e69..80f9cee1fb4a6 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c @@ -835,8 +835,10 @@ static void find_network(struct adapter *adapter) struct wlan_network *tgt_network = &pmlmepriv->cur_network; pwlan = rtw_find_network(&pmlmepriv->scanned_queue, tgt_network->network.mac_address); - if (pwlan) - pwlan->fixed = false; + if (!pwlan) + return; + + pwlan->fixed = false; if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) && (adapter->stapriv.asoc_sta_count == 1)) @@ -2006,7 +2008,10 @@ int rtw_restruct_wmm_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_ while (i < in_len) { ielength = initial_out_len; - if (in_ie[i] == 0xDD && in_ie[i+2] == 0x00 && in_ie[i+3] == 0x50 && in_ie[i+4] == 0xF2 && in_ie[i+5] == 0x02 && i+5 < in_len) { /* WMM element ID and OUI */ + if (i + 5 < in_len && + in_ie[i] == 0xDD && in_ie[i + 2] == 0x00 && + in_ie[i + 3] == 0x50 && in_ie[i + 4] == 0xF2 && + in_ie[i + 5] == 0x02) { for (j = i; j < i + 9; j++) { out_ie[ielength] = in_ie[j]; ielength++; diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c index a897c433d2b06..72eb48d554a35 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c @@ -588,9 +588,11 @@ unsigned int OnBeacon(struct adapter *padapter, union recv_frame *precv_frame) p = rtw_get_ie(pframe + sizeof(struct ieee80211_hdr_3addr) + _BEACON_IE_OFFSET_, WLAN_EID_EXT_SUPP_RATES, &ielen, precv_frame->u.hdr.len - sizeof(struct ieee80211_hdr_3addr) - _BEACON_IE_OFFSET_); if (p && ielen > 0) { - if ((*(p + 1 + ielen) == 0x2D) && (*(p + 2 + ielen) != 0x2D)) - /* Invalid value 0x2D is detected in Extended Supported Rates (ESR) IE. Try to fix the IE length to avoid failed Beacon parsing. */ - *(p + 1) = ielen - 1; + if (p + 2 + ielen < pframe + len) { + if ((*(p + 1 + ielen) == 0x2D) && (*(p + 2 + ielen) != 0x2D)) + /* Invalid value 0x2D is detected in Extended Supported Rates (ESR) IE. Try to fix the IE length to avoid failed Beacon parsing. */ + *(p + 1) = ielen - 1; + } } if (pmlmeext->sitesurvey_res.state == SCAN_PROCESS) { @@ -1042,6 +1044,9 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) status = WLAN_STATUS_CHALLENGE_FAIL; goto OnAssocReqFail; } else { + if (ie_len > sizeof(supportRate)) + ie_len = sizeof(supportRate); + memcpy(supportRate, p+2, ie_len); supportRateNum = ie_len; @@ -1049,7 +1054,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) pkt_len - WLAN_HDR_A3_LEN - ie_offset); if (p) { - if (supportRateNum <= sizeof(supportRate)) { + if (supportRateNum + ie_len <= sizeof(supportRate)) { memcpy(supportRate+supportRateNum, p+2, ie_len); supportRateNum += ie_len; } diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c index 3d99d045f4b6e..cef628f1352a3 100644 --- a/drivers/staging/rtl8723bs/core/rtw_security.c +++ b/drivers/staging/rtl8723bs/core/rtw_security.c @@ -1321,7 +1321,7 @@ u32 rtw_BIP_verify(struct adapter *padapter, u8 *precvframe) u8 mic[16]; struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; __le16 le_tmp; - __le64 le_tmp64; + __le64 le_tmp64 = 0; ori_len = pattrib->pkt_len-WLAN_HDR_A3_LEN+BIP_AAD_SIZE; BIP_AAD = rtw_zmalloc(ori_len); diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c index 315bab3737294..5d28dbf8b50ef 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c @@ -315,9 +315,10 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl len, notify_signal, GFP_ATOMIC); if (unlikely(!bss)) - goto exit; + goto free_buf; cfg80211_put_bss(wiphy, bss); +free_buf: kfree(buf); exit: diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c index f3caaa857c864..139ace51486d2 100644 --- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c +++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c @@ -377,7 +377,8 @@ static int rtw_drv_init( if (status != _SUCCESS) goto free_if1; - if (sdio_alloc_irq(dvobj) != _SUCCESS) + status = sdio_alloc_irq(dvobj); + if (status != _SUCCESS) goto free_if1; status = _SUCCESS; diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c index 3659af7e519db..24e927a1669d6 100644 --- a/drivers/staging/sm750fb/sm750.c +++ b/drivers/staging/sm750fb/sm750.c @@ -481,6 +481,9 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var, struct lynxfb_crtc *crtc; resource_size_t request; + if (!var->pixclock) + return -EINVAL; + ret = 0; par = info->par; crtc = &par->crtc; @@ -1118,6 +1121,7 @@ static void lynxfb_pci_remove(struct pci_dev *pdev) iounmap(sm750_dev->pvReg); iounmap(sm750_dev->pvMem); + pci_release_region(pdev, 1); kfree(g_settings); } diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c index ce46f240cbaf1..b3a16b22359c7 100644 --- a/drivers/staging/sm750fb/sm750_hw.c +++ b/drivers/staging/sm750fb/sm750_hw.c @@ -36,16 +36,11 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev) pr_info("mmio phyAddr = %lx\n", sm750_dev->vidreg_start); - /* - * reserve the vidreg space of smi adaptor - * if you do this, you need to add release region code - * in lynxfb_remove, or memory will not be mapped again - * successfully - */ + /* reserve the vidreg space of smi adaptor */ ret = pci_request_region(pdev, 1, "sm750fb"); if (ret) { pr_err("Can not request PCI regions.\n"); - goto exit; + return ret; } /* now map mmio and vidmem */ @@ -54,7 +49,7 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev) if (!sm750_dev->pvReg) { pr_err("mmio failed\n"); ret = -EFAULT; - goto exit; + goto err_release_region; } pr_info("mmio virtual addr = %p\n", sm750_dev->pvReg); @@ -79,13 +74,18 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev) sm750_dev->pvMem = ioremap_wc(sm750_dev->vidmem_start, sm750_dev->vidmem_size); if (!sm750_dev->pvMem) { - iounmap(sm750_dev->pvReg); pr_err("Map video memory failed\n"); ret = -EFAULT; - goto exit; + goto err_unmap_reg; } pr_info("video memory vaddr = %p\n", sm750_dev->pvMem); -exit: + + return 0; + +err_unmap_reg: + iounmap(sm750_dev->pvReg); +err_release_region: + pci_release_region(pdev, 1); return ret; } diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c index 5e6cf34929b55..c1888c42afdd5 100644 --- a/drivers/target/iscsi/iscsi_target_util.c +++ b/drivers/target/iscsi/iscsi_target_util.c @@ -741,8 +741,11 @@ void iscsit_dec_session_usage_count(struct iscsit_session *sess) spin_lock_bh(&sess->session_usage_lock); sess->session_usage_count--; - if (!sess->session_usage_count && sess->session_waiting_on_uc) + if (!sess->session_usage_count && sess->session_waiting_on_uc) { + spin_unlock_bh(&sess->session_usage_lock); complete(&sess->session_waiting_on_uc_comp); + return; + } spin_unlock_bh(&sess->session_usage_lock); } @@ -810,8 +813,11 @@ void iscsit_dec_conn_usage_count(struct iscsit_conn *conn) spin_lock_bh(&conn->conn_usage_lock); conn->conn_usage_count--; - if (!conn->conn_usage_count && conn->conn_waiting_on_uc) + if (!conn->conn_usage_count && conn->conn_waiting_on_uc) { + spin_unlock_bh(&conn->conn_usage_lock); complete(&conn->conn_waiting_on_uc_comp); + return; + } spin_unlock_bh(&conn->conn_usage_lock); } diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c index 01a8e349dc4d1..156f934049f19 100644 --- a/drivers/target/loopback/tcm_loop.c +++ b/drivers/target/loopback/tcm_loop.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -268,15 +269,27 @@ static int tcm_loop_device_reset(struct scsi_cmnd *sc) return (ret == TMR_FUNCTION_COMPLETE) ? SUCCESS : FAILED; } +static bool tcm_loop_flush_work_iter(struct request *rq, void *data) +{ + struct scsi_cmnd *sc = blk_mq_rq_to_pdu(rq); + struct tcm_loop_cmd *tl_cmd = scsi_cmd_priv(sc); + struct se_cmd *se_cmd = &tl_cmd->tl_se_cmd; + + flush_work(&se_cmd->work); + return true; +} + static int tcm_loop_target_reset(struct scsi_cmnd *sc) { struct tcm_loop_hba *tl_hba; struct tcm_loop_tpg *tl_tpg; + struct Scsi_Host *sh = sc->device->host; + int ret; /* * Locate the tcm_loop_hba_t pointer */ - tl_hba = *(struct tcm_loop_hba **)shost_priv(sc->device->host); + tl_hba = *(struct tcm_loop_hba **)shost_priv(sh); if (!tl_hba) { pr_err("Unable to perform device reset without active I_T Nexus\n"); return FAILED; @@ -285,11 +298,38 @@ static int tcm_loop_target_reset(struct scsi_cmnd *sc) * Locate the tl_tpg pointer from TargetID in sc->device->id */ tl_tpg = &tl_hba->tl_hba_tpgs[sc->device->id]; - if (tl_tpg) { - tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE; - return SUCCESS; - } - return FAILED; + if (!tl_tpg) + return FAILED; + + /* + * Issue a LUN_RESET to drain all commands that the target core + * knows about. This handles commands not yet marked CMD_T_COMPLETE. + */ + ret = tcm_loop_issue_tmr(tl_tpg, sc->device->lun, 0, TMR_LUN_RESET); + if (ret != TMR_FUNCTION_COMPLETE) + return FAILED; + + /* + * Flush any deferred target core completion work that may still be + * queued. Commands that already had CMD_T_COMPLETE set before the TMR + * are skipped by the TMR drain, but their async completion work + * (transport_lun_remove_cmd → percpu_ref_put, release_cmd → scsi_done) + * may still be pending in target_completion_wq. + * + * The SCSI EH will reuse in-flight scsi_cmnd structures for recovery + * commands (e.g. TUR) immediately after this handler returns SUCCESS — + * if deferred work is still pending, the memset in queuecommand would + * zero the se_cmd while the work accesses it, leaking the LUN + * percpu_ref and hanging configfs unlink forever. + * + * Use blk_mq_tagset_busy_iter() to find all started requests and + * flush_work() on each — the same pattern used by mpi3mr, scsi_debug, + * and other SCSI drivers to drain outstanding commands during reset. + */ + blk_mq_tagset_busy_iter(&sh->tag_set, tcm_loop_flush_work_iter, NULL); + + tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE; + return SUCCESS; } static const struct scsi_host_template tcm_loop_driver_template = { diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c index 3b89b5a70331f..ad03bf7929f8b 100644 --- a/drivers/target/sbp/sbp_target.c +++ b/drivers/target/sbp/sbp_target.c @@ -1961,12 +1961,12 @@ static struct se_portal_group *sbp_make_tpg(struct se_wwn *wwn, container_of(wwn, struct sbp_tport, tport_wwn); struct sbp_tpg *tpg; - unsigned long tpgt; + u16 tpgt; int ret; if (strstr(name, "tpgt_") != name) return ERR_PTR(-EINVAL); - if (kstrtoul(name + 5, 10, &tpgt) || tpgt > UINT_MAX) + if (kstrtou16(name + 5, 10, &tpgt)) return ERR_PTR(-EINVAL); if (tport->tpg) { diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c index b19acd662726d..a2bd2e81d2c68 100644 --- a/drivers/target/target_core_configfs.c +++ b/drivers/target/target_core_configfs.c @@ -108,8 +108,8 @@ static ssize_t target_core_item_dbroot_store(struct config_item *item, const char *page, size_t count) { ssize_t read_bytes; - struct file *fp; ssize_t r = -EINVAL; + struct path path = {}; mutex_lock(&target_devices_lock); if (target_devices) { @@ -131,17 +131,14 @@ static ssize_t target_core_item_dbroot_store(struct config_item *item, db_root_stage[read_bytes - 1] = '\0'; /* validate new db root before accepting it */ - fp = filp_open(db_root_stage, O_RDONLY, 0); - if (IS_ERR(fp)) { + r = kern_path(db_root_stage, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + if (r) { pr_err("db_root: cannot open: %s\n", db_root_stage); + if (r == -ENOTDIR) + pr_err("db_root: not a directory: %s\n", db_root_stage); goto unlock; } - if (!S_ISDIR(file_inode(fp)->i_mode)) { - filp_close(fp, NULL); - pr_err("db_root: not a directory: %s\n", db_root_stage); - goto unlock; - } - filp_close(fp, NULL); + path_put(&path); strscpy(db_root, db_root_stage); pr_debug("Target_Core_ConfigFS: db_root set to %s\n", db_root); @@ -2772,7 +2769,6 @@ static ssize_t target_lu_gp_members_show(struct config_item *item, char *page) cur_len = snprintf(buf, LU_GROUP_NAME_BUF, "%s/%s\n", config_item_name(&hba->hba_group.cg_item), config_item_name(&dev->dev_group.cg_item)); - cur_len++; /* Extra byte for NULL terminator */ if ((cur_len + len) > PAGE_SIZE || cur_len > LU_GROUP_NAME_BUF) { pr_warn("Ran out of lu_gp_show_attr" diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index 2d78ef74633c8..80d0fc2a98067 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -276,7 +276,7 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, ssize_t len = 0; int ret = 0, i; - aio_cmd = kmalloc(struct_size(aio_cmd, bvecs, sgl_nents), GFP_KERNEL); + aio_cmd = kzalloc(struct_size(aio_cmd, bvecs, sgl_nents), GFP_KERNEL); if (!aio_cmd) return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; diff --git a/drivers/target/target_core_stat.c b/drivers/target/target_core_stat.c index 6bdf2d8bd6942..4fdc307ea38bc 100644 --- a/drivers/target/target_core_stat.c +++ b/drivers/target/target_core_stat.c @@ -282,7 +282,7 @@ static ssize_t target_stat_lu_num_cmds_show(struct config_item *item, struct se_device *dev = to_stat_lu_dev(item); struct se_dev_io_stats *stats; unsigned int cpu; - u32 cmds = 0; + u64 cmds = 0; for_each_possible_cpu(cpu) { stats = per_cpu_ptr(dev->stats, cpu); @@ -290,7 +290,7 @@ static ssize_t target_stat_lu_num_cmds_show(struct config_item *item, } /* scsiLuNumCommands */ - return snprintf(page, PAGE_SIZE, "%u\n", cmds); + return snprintf(page, PAGE_SIZE, "%llu\n", cmds); } static ssize_t target_stat_lu_read_mbytes_show(struct config_item *item, @@ -299,7 +299,7 @@ static ssize_t target_stat_lu_read_mbytes_show(struct config_item *item, struct se_device *dev = to_stat_lu_dev(item); struct se_dev_io_stats *stats; unsigned int cpu; - u32 bytes = 0; + u64 bytes = 0; for_each_possible_cpu(cpu) { stats = per_cpu_ptr(dev->stats, cpu); @@ -307,7 +307,7 @@ static ssize_t target_stat_lu_read_mbytes_show(struct config_item *item, } /* scsiLuReadMegaBytes */ - return snprintf(page, PAGE_SIZE, "%u\n", bytes >> 20); + return snprintf(page, PAGE_SIZE, "%llu\n", bytes >> 20); } static ssize_t target_stat_lu_write_mbytes_show(struct config_item *item, @@ -316,7 +316,7 @@ static ssize_t target_stat_lu_write_mbytes_show(struct config_item *item, struct se_device *dev = to_stat_lu_dev(item); struct se_dev_io_stats *stats; unsigned int cpu; - u32 bytes = 0; + u64 bytes = 0; for_each_possible_cpu(cpu) { stats = per_cpu_ptr(dev->stats, cpu); @@ -324,7 +324,7 @@ static ssize_t target_stat_lu_write_mbytes_show(struct config_item *item, } /* scsiLuWrittenMegaBytes */ - return snprintf(page, PAGE_SIZE, "%u\n", bytes >> 20); + return snprintf(page, PAGE_SIZE, "%llu\n", bytes >> 20); } static ssize_t target_stat_lu_resets_show(struct config_item *item, char *page) @@ -1044,7 +1044,7 @@ static ssize_t target_stat_auth_num_cmds_show(struct config_item *item, struct se_dev_entry *deve; unsigned int cpu; ssize_t ret; - u32 cmds = 0; + u64 cmds = 0; rcu_read_lock(); deve = target_nacl_find_deve(nacl, lacl->mapped_lun); @@ -1059,7 +1059,7 @@ static ssize_t target_stat_auth_num_cmds_show(struct config_item *item, } /* scsiAuthIntrOutCommands */ - ret = snprintf(page, PAGE_SIZE, "%u\n", cmds); + ret = snprintf(page, PAGE_SIZE, "%llu\n", cmds); rcu_read_unlock(); return ret; } @@ -1073,7 +1073,7 @@ static ssize_t target_stat_auth_read_mbytes_show(struct config_item *item, struct se_dev_entry *deve; unsigned int cpu; ssize_t ret; - u32 bytes = 0; + u64 bytes = 0; rcu_read_lock(); deve = target_nacl_find_deve(nacl, lacl->mapped_lun); @@ -1088,7 +1088,7 @@ static ssize_t target_stat_auth_read_mbytes_show(struct config_item *item, } /* scsiAuthIntrReadMegaBytes */ - ret = snprintf(page, PAGE_SIZE, "%u\n", bytes >> 20); + ret = snprintf(page, PAGE_SIZE, "%llu\n", bytes >> 20); rcu_read_unlock(); return ret; } @@ -1102,7 +1102,7 @@ static ssize_t target_stat_auth_write_mbytes_show(struct config_item *item, struct se_dev_entry *deve; unsigned int cpu; ssize_t ret; - u32 bytes = 0; + u64 bytes = 0; rcu_read_lock(); deve = target_nacl_find_deve(nacl, lacl->mapped_lun); @@ -1117,7 +1117,7 @@ static ssize_t target_stat_auth_write_mbytes_show(struct config_item *item, } /* scsiAuthIntrWrittenMegaBytes */ - ret = snprintf(page, PAGE_SIZE, "%u\n", bytes >> 20); + ret = snprintf(page, PAGE_SIZE, "%llu\n", bytes >> 20); rcu_read_unlock(); return ret; } diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 0a76bdfe55282..88544c911949c 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1524,6 +1524,7 @@ target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb, gfp_t gfp) if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), gfp); if (!cmd->t_task_cdb) { + cmd->t_task_cdb = &cmd->__t_task_cdb[0]; pr_err("Unable to allocate cmd->t_task_cdb" " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", scsi_command_size(cdb), diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c index 4a47de4bb2e5c..898707ca21a8e 100644 --- a/drivers/tee/tee_shm.c +++ b/drivers/tee/tee_shm.c @@ -23,29 +23,11 @@ struct tee_shm_dma_mem { struct page *page; }; -static void shm_put_kernel_pages(struct page **pages, size_t page_count) -{ - size_t n; - - for (n = 0; n < page_count; n++) - put_page(pages[n]); -} - -static void shm_get_kernel_pages(struct page **pages, size_t page_count) -{ - size_t n; - - for (n = 0; n < page_count; n++) - get_page(pages[n]); -} - static void release_registered_pages(struct tee_shm *shm) { if (shm->pages) { if (shm->flags & TEE_SHM_USER_MAPPED) unpin_user_pages(shm->pages, shm->num_pages); - else - shm_put_kernel_pages(shm->pages, shm->num_pages); kfree(shm->pages); } @@ -477,13 +459,6 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags, goto err_put_shm_pages; } - /* - * iov_iter_extract_kvec_pages does not get reference on the pages, - * get a reference on them. - */ - if (iov_iter_is_kvec(iter)) - shm_get_kernel_pages(shm->pages, num_pages); - shm->offset = off; shm->size = len; shm->num_pages = num_pages; @@ -499,8 +474,6 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags, err_put_shm_pages: if (!iov_iter_is_kvec(iter)) unpin_user_pages(shm->pages, shm->num_pages); - else - shm_put_kernel_pages(shm->pages, shm->num_pages); err_free_shm_pages: kfree(shm->pages); err_free_shm: diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index a09c188b9ad11..c455a273a62d2 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -229,6 +229,17 @@ config PCIE_THERMAL If you want this support, you should say Y here. + +config REMOTEPROC_THERMAL + bool "Remote processor cooling support" + help + This implements a generic cooling mechanism for remote processors + (modem, DSP, etc.) that allows vendor-specific implementations to + register thermal cooling devices and provide callbacks for thermal + mitigation. + + If you want this support, you should say Y here. + config THERMAL_EMULATION bool "Thermal emulation mode support" help diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index d7718978db245..4f4c80afb1ec4 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -34,6 +34,8 @@ thermal_sys-$(CONFIG_DEVFREQ_THERMAL) += devfreq_cooling.o thermal_sys-$(CONFIG_PCIE_THERMAL) += pcie_cooling.o +thermal_sys-$(CONFIG_REMOTEPROC_THERMAL) += remoteproc_cooling.o + obj-$(CONFIG_K3_THERMAL) += k3_bandgap.o k3_j72xx_bandgap.o # platform thermal drivers obj-y += broadcom/ diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c index 1f3d22b659dbe..635bc8fded1e9 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c @@ -451,8 +451,11 @@ int proc_thermal_rfim_add(struct pci_dev *pdev, struct proc_thermal_device *proc break; } ret = sysfs_create_group(&pdev->dev.kobj, &dlvr_attribute_group); - if (ret) + if (ret) { + if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_FIVR) + sysfs_remove_group(&pdev->dev.kobj, &fivr_attribute_group); return ret; + } } if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DVFS) { diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c index 49ff3bae72710..91f291627132a 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c @@ -176,15 +176,21 @@ static inline void write_soc_slider(struct proc_thermal_device *proc_priv, u64 v static void set_soc_power_profile(struct proc_thermal_device *proc_priv, int slider) { + u8 offset; u64 val; val = read_soc_slider(proc_priv); val &= ~SLIDER_MASK; val |= FIELD_PREP(SLIDER_MASK, slider) | BIT(SLIDER_ENABLE_BIT); + if (slider == SOC_SLIDER_VALUE_MINIMUM || slider == SOC_SLIDER_VALUE_MAXIMUM) + offset = 0; + else + offset = slider_offset; + /* Set the slider offset from module params */ val &= ~SLIDER_OFFSET_MASK; - val |= FIELD_PREP(SLIDER_OFFSET_MASK, slider_offset); + val |= FIELD_PREP(SLIDER_OFFSET_MASK, offset); write_soc_slider(proc_priv, val); } diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 3fc679b6f11b1..aab5f9fca9c33 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -128,6 +128,9 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, u32 l, h, mask, shift, intr; int tj_max, val, ret; + if (temp == THERMAL_TEMP_INVALID) + temp = 0; + tj_max = intel_tcc_get_tjmax(zonedev->cpu); if (tj_max < 0) return tj_max; diff --git a/drivers/thermal/remoteproc_cooling.c b/drivers/thermal/remoteproc_cooling.c new file mode 100644 index 0000000000000..b5ea200f485b8 --- /dev/null +++ b/drivers/thermal/remoteproc_cooling.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Remote Processor Cooling Device + * + * Copyright (c) 2025, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define REMOTEPROC_PREFIX "rproc_" + +struct remoteproc_cooling_ops { + int (*get_max_level)(void *devdata, unsigned long *level); + int (*get_cur_level)(void *devdata, unsigned long *level); + int (*set_cur_level)(void *devdata, unsigned long level); +}; + +/** + * struct remoteproc_cdev - Remote processor cooling device + * @cdev: Thermal cooling device handle + * @ops: Vendor-specific operation callbacks + * @devdata: Private data for vendor implementation + * @np: Device tree node associated with this cooling device + * @lock: Mutex to protect cooling device operations + */ +struct remoteproc_cdev { + struct thermal_cooling_device *cdev; + const struct remoteproc_cooling_ops *ops; + void *devdata; + struct device_node *np; + struct mutex lock; +}; + +/* Thermal cooling device callbacks */ + +static int remoteproc_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct remoteproc_cdev *rproc_cdev = cdev->devdata; + int ret; + + if (!rproc_cdev || !rproc_cdev->ops) + return -EINVAL; + + mutex_lock(&rproc_cdev->lock); + ret = rproc_cdev->ops->get_max_level(rproc_cdev->devdata, state); + mutex_unlock(&rproc_cdev->lock); + + return ret; +} + +static int remoteproc_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct remoteproc_cdev *rproc_cdev = cdev->devdata; + int ret; + + if (!rproc_cdev || !rproc_cdev->ops) + return -EINVAL; + + mutex_lock(&rproc_cdev->lock); + ret = rproc_cdev->ops->get_cur_level(rproc_cdev->devdata, state); + mutex_unlock(&rproc_cdev->lock); + + return ret; +} + +static int remoteproc_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state) +{ + struct remoteproc_cdev *rproc_cdev = cdev->devdata; + int ret; + + if (!rproc_cdev || !rproc_cdev->ops) + return -EINVAL; + + mutex_lock(&rproc_cdev->lock); + ret = rproc_cdev->ops->set_cur_level(rproc_cdev->devdata, state); + mutex_unlock(&rproc_cdev->lock); + + return ret; +} + +static const struct thermal_cooling_device_ops remoteproc_cooling_ops = { + .get_max_state = remoteproc_get_max_state, + .get_cur_state = remoteproc_get_cur_state, + .set_cur_state = remoteproc_set_cur_state, +}; + +struct remoteproc_cdev * +remoteproc_cooling_register(struct device_node *np, + const char *name, const struct remoteproc_cooling_ops *ops, + void *devdata) +{ + struct remoteproc_cdev *rproc_cdev; + struct thermal_cooling_device *cdev; + int ret; + + if (!name || !ops) { + return ERR_PTR(-EINVAL); + } + + rproc_cdev = kzalloc(sizeof(*rproc_cdev), GFP_KERNEL); + if (!rproc_cdev) + return ERR_PTR(-ENOMEM); + + rproc_cdev->ops = ops; + rproc_cdev->devdata = devdata; + rproc_cdev->np = np; + mutex_init(&rproc_cdev->lock); + + char *rproc_name __free(kfree) = + kasprintf(GFP_KERNEL, REMOTEPROC_PREFIX "%s", name); + /* Register with thermal framework */ + if (np) { + cdev = thermal_of_cooling_device_register(np, rproc_name, rproc_cdev, + &remoteproc_cooling_ops); + } + + if (IS_ERR(cdev)) { + ret = PTR_ERR(cdev); + goto free_rproc_cdev; + } + + rproc_cdev->cdev = cdev; + + return rproc_cdev; + +free_rproc_cdev: + kfree(rproc_cdev); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(remoteproc_cooling_register); + +void remoteproc_cooling_unregister(struct remoteproc_cdev *rproc_cdev) +{ + if (!rproc_cdev) + return; + + thermal_cooling_device_unregister(rproc_cdev->cdev); + mutex_destroy(&rproc_cdev->lock); + kfree(rproc_cdev); +} +EXPORT_SYMBOL_GPL(remoteproc_cooling_unregister); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Remote Processor Cooling Device"); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 17ca5c0826435..f28c15dd0b926 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -41,6 +41,8 @@ static struct thermal_governor *def_governor; static bool thermal_pm_suspended; +static struct workqueue_struct *thermal_wq __ro_after_init; + /* * Governor section: set of functions to handle thermal governors * @@ -313,7 +315,7 @@ static void thermal_zone_device_set_polling(struct thermal_zone_device *tz, if (delay > HZ) delay = round_jiffies_relative(delay); - mod_delayed_work(system_freezable_power_efficient_wq, &tz->poll_queue, delay); + mod_delayed_work(thermal_wq, &tz->poll_queue, delay); } static void thermal_zone_recheck(struct thermal_zone_device *tz, int error) @@ -1636,6 +1638,7 @@ thermal_zone_device_register_with_trips(const char *type, device_del(&tz->device); release_device: put_device(&tz->device); + wait_for_completion(&tz->removal); remove_id: ida_free(&thermal_tz_ida, id); free_tzp: @@ -1781,6 +1784,10 @@ static void thermal_zone_device_resume(struct work_struct *work) guard(thermal_zone)(tz); + /* If the thermal zone is going away, there's nothing to do. */ + if (tz->state & TZ_STATE_FLAG_EXIT) + return; + tz->state &= ~(TZ_STATE_FLAG_SUSPENDED | TZ_STATE_FLAG_RESUMING); thermal_debug_tz_resume(tz); @@ -1807,6 +1814,9 @@ static void thermal_zone_pm_prepare(struct thermal_zone_device *tz) } tz->state |= TZ_STATE_FLAG_SUSPENDED; + + /* Prevent new work from getting to the workqueue subsequently. */ + cancel_delayed_work(&tz->poll_queue); } static void thermal_pm_notify_prepare(void) @@ -1825,8 +1835,6 @@ static void thermal_zone_pm_complete(struct thermal_zone_device *tz) { guard(thermal_zone)(tz); - cancel_delayed_work(&tz->poll_queue); - reinit_completion(&tz->resume); tz->state |= TZ_STATE_FLAG_RESUMING; @@ -1836,7 +1844,7 @@ static void thermal_zone_pm_complete(struct thermal_zone_device *tz) */ INIT_DELAYED_WORK(&tz->poll_queue, thermal_zone_device_resume); /* Queue up the work without a delay. */ - mod_delayed_work(system_freezable_power_efficient_wq, &tz->poll_queue, 0); + mod_delayed_work(thermal_wq, &tz->poll_queue, 0); } static void thermal_pm_notify_complete(void) @@ -1859,6 +1867,11 @@ static int thermal_pm_notify(struct notifier_block *nb, case PM_RESTORE_PREPARE: case PM_SUSPEND_PREPARE: thermal_pm_notify_prepare(); + /* + * Allow any leftover thermal work items already on the + * worqueue to complete so they don't get in the way later. + */ + flush_workqueue(thermal_wq); break; case PM_POST_HIBERNATION: case PM_POST_RESTORE: @@ -1891,9 +1904,16 @@ static int __init thermal_init(void) if (result) goto error; + thermal_wq = alloc_workqueue("thermal_events", + WQ_FREEZABLE | WQ_POWER_EFFICIENT | WQ_PERCPU, 0); + if (!thermal_wq) { + result = -ENOMEM; + goto unregister_netlink; + } + result = thermal_register_governors(); if (result) - goto unregister_netlink; + goto destroy_workqueue; thermal_class = kzalloc(sizeof(*thermal_class), GFP_KERNEL); if (!thermal_class) { @@ -1920,6 +1940,8 @@ static int __init thermal_init(void) unregister_governors: thermal_unregister_governors(); +destroy_workqueue: + destroy_workqueue(thermal_wq); unregister_netlink: thermal_netlink_exit(); error: diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 1a51a4d240ff6..b6d0c92f5522b 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -280,10 +280,10 @@ static bool thermal_of_cm_lookup(struct device_node *cm_np, struct cooling_spec *c) { for_each_child_of_node_scoped(cm_np, child) { - struct device_node *tr_np; int count, i; - tr_np = of_parse_phandle(child, "trip", 0); + struct device_node *tr_np __free(device_node) = + of_parse_phandle(child, "trip", 0); if (tr_np != trip->priv) continue; diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c index addb4a20d5ead..7e283dae8f808 100644 --- a/drivers/thunderbolt/nhi.c +++ b/drivers/thunderbolt/nhi.c @@ -1020,7 +1020,7 @@ static bool nhi_wake_supported(struct pci_dev *pdev) * If power rails are sustainable for wakeup from S4 this * property is set by the BIOS. */ - if (device_property_read_u8(&pdev->dev, "WAKE_SUPPORTED", &val)) + if (!device_property_read_u8(&pdev->dev, "WAKE_SUPPORTED", &val)) return !!val; return true; diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index e99f5193d8f1c..2d4f15df18f7f 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -184,7 +184,9 @@ static unsigned int __maybe_unused serial_icr_read(struct uart_8250_port *up, return value; } +void serial8250_clear_fifos(struct uart_8250_port *p); void serial8250_clear_and_reinit_fifos(struct uart_8250_port *p); +void serial8250_fifo_wait_for_lsr_thre(struct uart_8250_port *up, unsigned int count); void serial8250_rpm_get(struct uart_8250_port *p); void serial8250_rpm_put(struct uart_8250_port *p); @@ -409,6 +411,26 @@ static inline bool serial8250_tx_dma_running(struct uart_8250_port *p) return dma && dma->tx_running; } + +static inline void serial8250_tx_dma_pause(struct uart_8250_port *p) +{ + struct uart_8250_dma *dma = p->dma; + + if (!dma->tx_running) + return; + + dmaengine_pause(dma->txchan); +} + +static inline void serial8250_tx_dma_resume(struct uart_8250_port *p) +{ + struct uart_8250_dma *dma = p->dma; + + if (!dma->tx_running) + return; + + dmaengine_resume(dma->txchan); +} #else static inline int serial8250_tx_dma(struct uart_8250_port *p) { @@ -430,6 +452,9 @@ static inline bool serial8250_tx_dma_running(struct uart_8250_port *p) { return false; } + +static inline void serial8250_tx_dma_pause(struct uart_8250_port *p) { } +static inline void serial8250_tx_dma_resume(struct uart_8250_port *p) { } #endif static inline int ns16550a_goto_highspeed(struct uart_8250_port *up) diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c index bdd26c9f34bdf..3b6452e759d5b 100644 --- a/drivers/tty/serial/8250/8250_dma.c +++ b/drivers/tty/serial/8250/8250_dma.c @@ -162,7 +162,22 @@ void serial8250_tx_dma_flush(struct uart_8250_port *p) */ dma->tx_size = 0; + /* + * We can't use `dmaengine_terminate_sync` because `uart_flush_buffer` is + * holding the uart port spinlock. + */ dmaengine_terminate_async(dma->txchan); + + /* + * The callback might or might not run. If it doesn't run, we need to ensure + * that `tx_running` is cleared so that we can schedule new transactions. + * If it does run, then the zombie callback will clear `tx_running` again + * and perform a no-op since `tx_size` was cleared above. + * + * In either case, we ASSUME the DMA transaction will terminate before we + * issue a new `serial8250_tx_dma`. + */ + dma->tx_running = 0; } int serial8250_rx_dma(struct uart_8250_port *p) diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index 710ae4d40aec4..b1f0303aceab0 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -9,10 +9,14 @@ * LCR is written whilst busy. If it is, then a busy detect interrupt is * raised, the LCR needs to be rewritten and the uart status register read. */ +#include +#include +#include #include #include #include #include +#include #include #include #include @@ -40,8 +44,12 @@ #define RZN1_UART_RDMACR 0x110 /* DMA Control Register Receive Mode */ /* DesignWare specific register fields */ +#define DW_UART_IIR_IID GENMASK(3, 0) + #define DW_UART_MCR_SIRE BIT(6) +#define DW_UART_USR_BUSY BIT(0) + /* Renesas specific register fields */ #define RZN1_UART_xDMACR_DMA_EN BIT(0) #define RZN1_UART_xDMACR_1_WORD_BURST (0 << 1) @@ -56,6 +64,13 @@ #define DW_UART_QUIRK_IS_DMA_FC BIT(3) #define DW_UART_QUIRK_APMC0D08 BIT(4) #define DW_UART_QUIRK_CPR_VALUE BIT(5) +#define DW_UART_QUIRK_IER_KICK BIT(6) + +/* + * Number of consecutive IIR_NO_INT interrupts required to trigger interrupt + * storm prevention code. + */ +#define DW_UART_QUIRK_IER_KICK_THRES 4 struct dw8250_platform_data { u8 usr_reg; @@ -77,6 +92,9 @@ struct dw8250_data { unsigned int skip_autocfg:1; unsigned int uart_16550_compatible:1; + unsigned int in_idle:1; + + u8 no_int_count; }; static inline struct dw8250_data *to_dw8250_data(struct dw8250_port_data *data) @@ -107,78 +125,167 @@ static inline u32 dw8250_modify_msr(struct uart_port *p, unsigned int offset, u3 return value; } +static void dw8250_idle_exit(struct uart_port *p) +{ + struct dw8250_data *d = to_dw8250_data(p->private_data); + struct uart_8250_port *up = up_to_u8250p(p); + + if (d->uart_16550_compatible) + return; + + if (up->capabilities & UART_CAP_FIFO) + serial_port_out(p, UART_FCR, up->fcr); + serial_port_out(p, UART_MCR, up->mcr); + serial_port_out(p, UART_IER, up->ier); + + /* DMA Rx is restarted by IRQ handler as needed. */ + if (up->dma) + serial8250_tx_dma_resume(up); + + d->in_idle = 0; +} + /* - * This function is being called as part of the uart_port::serial_out() - * routine. Hence, it must not call serial_port_out() or serial_out() - * against the modified registers here, i.e. LCR. + * Ensure BUSY is not asserted. If DW UART is configured with + * !uart_16550_compatible, the writes to LCR, DLL, and DLH fail while + * BUSY is asserted. + * + * Context: port's lock must be held */ -static void dw8250_force_idle(struct uart_port *p) +static int dw8250_idle_enter(struct uart_port *p) { + struct dw8250_data *d = to_dw8250_data(p->private_data); + unsigned int usr_reg = d->pdata ? d->pdata->usr_reg : DW_UART_USR; struct uart_8250_port *up = up_to_u8250p(p); - unsigned int lsr; + int retries; + u32 lsr; - /* - * The following call currently performs serial_out() - * against the FCR register. Because it differs to LCR - * there will be no infinite loop, but if it ever gets - * modified, we might need a new custom version of it - * that avoids infinite recursion. - */ - serial8250_clear_and_reinit_fifos(up); + lockdep_assert_held_once(&p->lock); + + if (d->uart_16550_compatible) + return 0; + + d->in_idle = 1; + + /* Prevent triggering interrupt from RBR filling */ + serial_port_out(p, UART_IER, 0); + + if (up->dma) { + serial8250_rx_dma_flush(up); + if (serial8250_tx_dma_running(up)) + serial8250_tx_dma_pause(up); + } /* - * With PSLVERR_RESP_EN parameter set to 1, the device generates an - * error response when an attempt to read an empty RBR with FIFO - * enabled. + * Wait until Tx becomes empty + one extra frame time to ensure all bits + * have been sent on the wire. + * + * FIXME: frame_time delay is too long with very low baudrates. */ - if (up->fcr & UART_FCR_ENABLE_FIFO) { - lsr = serial_port_in(p, UART_LSR); - if (!(lsr & UART_LSR_DR)) - return; + serial8250_fifo_wait_for_lsr_thre(up, p->fifosize); + ndelay(p->frame_time); + + serial_port_out(p, UART_MCR, up->mcr | UART_MCR_LOOP); + + retries = 4; /* Arbitrary limit, 2 was always enough in tests */ + do { + serial8250_clear_fifos(up); + if (!(serial_port_in(p, usr_reg) & DW_UART_USR_BUSY)) + break; + /* FIXME: frame_time delay is too long with very low baudrates. */ + ndelay(p->frame_time); + } while (--retries); + + lsr = serial_lsr_in(up); + if (lsr & UART_LSR_DR) { + serial_port_in(p, UART_RX); + up->lsr_saved_flags = 0; + } + + /* Now guaranteed to have BUSY deasserted? Just sanity check */ + if (serial_port_in(p, usr_reg) & DW_UART_USR_BUSY) { + dw8250_idle_exit(p); + return -EBUSY; } - serial_port_in(p, UART_RX); + return 0; +} + +static void dw8250_set_divisor(struct uart_port *p, unsigned int baud, + unsigned int quot, unsigned int quot_frac) +{ + struct uart_8250_port *up = up_to_u8250p(p); + int ret; + + ret = dw8250_idle_enter(p); + if (ret < 0) + return; + + serial_port_out(p, UART_LCR, up->lcr | UART_LCR_DLAB); + if (!(serial_port_in(p, UART_LCR) & UART_LCR_DLAB)) + goto idle_failed; + + serial_dl_write(up, quot); + serial_port_out(p, UART_LCR, up->lcr); + +idle_failed: + dw8250_idle_exit(p); } /* * This function is being called as part of the uart_port::serial_out() - * routine. Hence, it must not call serial_port_out() or serial_out() - * against the modified registers here, i.e. LCR. + * routine. Hence, special care must be taken when serial_port_out() or + * serial_out() against the modified registers here, i.e. LCR (d->in_idle is + * used to break recursion loop). */ static void dw8250_check_lcr(struct uart_port *p, unsigned int offset, u32 value) { struct dw8250_data *d = to_dw8250_data(p->private_data); - void __iomem *addr = p->membase + (offset << p->regshift); - int tries = 1000; + u32 lcr; + int ret; if (offset != UART_LCR || d->uart_16550_compatible) return; + lcr = serial_port_in(p, UART_LCR); + /* Make sure LCR write wasn't ignored */ - while (tries--) { - u32 lcr = serial_port_in(p, offset); + if ((value & ~UART_LCR_SPAR) == (lcr & ~UART_LCR_SPAR)) + return; - if ((value & ~UART_LCR_SPAR) == (lcr & ~UART_LCR_SPAR)) - return; + if (d->in_idle) + goto write_err; - dw8250_force_idle(p); + ret = dw8250_idle_enter(p); + if (ret < 0) + goto write_err; -#ifdef CONFIG_64BIT - if (p->type == PORT_OCTEON) - __raw_writeq(value & 0xff, addr); - else -#endif - if (p->iotype == UPIO_MEM32) - writel(value, addr); - else if (p->iotype == UPIO_MEM32BE) - iowrite32be(value, addr); - else - writeb(value, addr); - } + serial_port_out(p, UART_LCR, value); + dw8250_idle_exit(p); + return; + +write_err: /* * FIXME: this deadlocks if port->lock is already held * dev_err(p->dev, "Couldn't set LCR to %d\n", value); */ + return; /* Silences "label at the end of compound statement" */ +} + +/* + * With BUSY, LCR writes can be very expensive (IRQ + complex retry logic). + * If the write does not change the value of the LCR register, skip it entirely. + */ +static bool dw8250_can_skip_reg_write(struct uart_port *p, unsigned int offset, u32 value) +{ + struct dw8250_data *d = to_dw8250_data(p->private_data); + u32 lcr; + + if (offset != UART_LCR || d->uart_16550_compatible) + return false; + + lcr = serial_port_in(p, offset); + return lcr == value; } /* Returns once the transmitter is empty or we run out of retries */ @@ -207,12 +314,18 @@ static void dw8250_tx_wait_empty(struct uart_port *p) static void dw8250_serial_out(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + writeb(value, p->membase + (offset << p->regshift)); dw8250_check_lcr(p, offset, value); } static void dw8250_serial_out38x(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + /* Allow the TX to drain before we reconfigure */ if (offset == UART_LCR) dw8250_tx_wait_empty(p); @@ -237,6 +350,9 @@ static u32 dw8250_serial_inq(struct uart_port *p, unsigned int offset) static void dw8250_serial_outq(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + value &= 0xff; __raw_writeq(value, p->membase + (offset << p->regshift)); /* Read back to ensure register write ordering. */ @@ -248,6 +364,9 @@ static void dw8250_serial_outq(struct uart_port *p, unsigned int offset, u32 val static void dw8250_serial_out32(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + writel(value, p->membase + (offset << p->regshift)); dw8250_check_lcr(p, offset, value); } @@ -261,6 +380,9 @@ static u32 dw8250_serial_in32(struct uart_port *p, unsigned int offset) static void dw8250_serial_out32be(struct uart_port *p, unsigned int offset, u32 value) { + if (dw8250_can_skip_reg_write(p, offset, value)) + return; + iowrite32be(value, p->membase + (offset << p->regshift)); dw8250_check_lcr(p, offset, value); } @@ -272,6 +394,29 @@ static u32 dw8250_serial_in32be(struct uart_port *p, unsigned int offset) return dw8250_modify_msr(p, offset, value); } +/* + * INTC10EE UART can IRQ storm while reporting IIR_NO_INT. Inducing IIR value + * change has been observed to break the storm. + * + * If Tx is empty (THRE asserted), we use here IER_THRI to cause IIR_NO_INT -> + * IIR_THRI transition. + */ +static void dw8250_quirk_ier_kick(struct uart_port *p) +{ + struct uart_8250_port *up = up_to_u8250p(p); + u32 lsr; + + if (up->ier & UART_IER_THRI) + return; + + lsr = serial_lsr_in(up); + if (!(lsr & UART_LSR_THRE)) + return; + + serial_port_out(p, UART_IER, up->ier | UART_IER_THRI); + serial_port_in(p, UART_LCR); /* safe, no side-effects */ + serial_port_out(p, UART_IER, up->ier); +} static int dw8250_handle_irq(struct uart_port *p) { @@ -281,7 +426,31 @@ static int dw8250_handle_irq(struct uart_port *p) bool rx_timeout = (iir & 0x3f) == UART_IIR_RX_TIMEOUT; unsigned int quirks = d->pdata->quirks; unsigned int status; - unsigned long flags; + + guard(uart_port_lock_irqsave)(p); + + switch (FIELD_GET(DW_UART_IIR_IID, iir)) { + case UART_IIR_NO_INT: + if (d->uart_16550_compatible || up->dma) + return 0; + + if (quirks & DW_UART_QUIRK_IER_KICK && + d->no_int_count == (DW_UART_QUIRK_IER_KICK_THRES - 1)) + dw8250_quirk_ier_kick(p); + d->no_int_count = (d->no_int_count + 1) % DW_UART_QUIRK_IER_KICK_THRES; + + return 0; + + case UART_IIR_BUSY: + /* Clear the USR */ + serial_port_in(p, d->pdata->usr_reg); + + d->no_int_count = 0; + + return 1; + } + + d->no_int_count = 0; /* * There are ways to get Designware-based UARTs into a state where @@ -294,20 +463,15 @@ static int dw8250_handle_irq(struct uart_port *p) * so we limit the workaround only to non-DMA mode. */ if (!up->dma && rx_timeout) { - uart_port_lock_irqsave(p, &flags); status = serial_lsr_in(up); if (!(status & (UART_LSR_DR | UART_LSR_BI))) serial_port_in(p, UART_RX); - - uart_port_unlock_irqrestore(p, flags); } /* Manually stop the Rx DMA transfer when acting as flow controller */ if (quirks & DW_UART_QUIRK_IS_DMA_FC && up->dma && up->dma->rx_running && rx_timeout) { - uart_port_lock_irqsave(p, &flags); status = serial_lsr_in(up); - uart_port_unlock_irqrestore(p, flags); if (status & (UART_LSR_DR | UART_LSR_BI)) { dw8250_writel_ext(p, RZN1_UART_RDMACR, 0); @@ -315,17 +479,9 @@ static int dw8250_handle_irq(struct uart_port *p) } } - if (serial8250_handle_irq(p, iir)) - return 1; + serial8250_handle_irq_locked(p, iir); - if ((iir & UART_IIR_BUSY) == UART_IIR_BUSY) { - /* Clear the USR */ - serial_port_in(p, d->pdata->usr_reg); - - return 1; - } - - return 0; + return 1; } static void dw8250_clk_work_cb(struct work_struct *work) @@ -527,6 +683,14 @@ static void dw8250_reset_control_assert(void *data) reset_control_assert(data); } +static void dw8250_shutdown(struct uart_port *port) +{ + struct dw8250_data *d = to_dw8250_data(port->private_data); + + serial8250_do_shutdown(port); + d->no_int_count = 0; +} + static int dw8250_probe(struct platform_device *pdev) { struct uart_8250_port uart = {}, *up = &uart; @@ -545,8 +709,10 @@ static int dw8250_probe(struct platform_device *pdev) p->type = PORT_8250; p->flags = UPF_FIXED_PORT; p->dev = dev; + p->set_ldisc = dw8250_set_ldisc; p->set_termios = dw8250_set_termios; + p->set_divisor = dw8250_set_divisor; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -650,10 +816,12 @@ static int dw8250_probe(struct platform_device *pdev) dw8250_quirks(p, data); /* If the Busy Functionality is not implemented, don't handle it */ - if (data->uart_16550_compatible) + if (data->uart_16550_compatible) { p->handle_irq = NULL; - else if (data->pdata) + } else if (data->pdata) { p->handle_irq = dw8250_handle_irq; + p->shutdown = dw8250_shutdown; + } dw8250_setup_dma_filter(p, data); @@ -741,11 +909,18 @@ static int dw8250_runtime_suspend(struct device *dev) static int dw8250_runtime_resume(struct device *dev) { + int ret; struct dw8250_data *data = dev_get_drvdata(dev); - clk_prepare_enable(data->pclk); + ret = clk_prepare_enable(data->pclk); + if (ret) + return ret; - clk_prepare_enable(data->clk); + ret = clk_prepare_enable(data->clk); + if (ret) { + clk_disable_unprepare(data->pclk); + return ret; + } return 0; } @@ -780,6 +955,11 @@ static const struct dw8250_platform_data dw8250_skip_set_rate_data = { .quirks = DW_UART_QUIRK_SKIP_SET_RATE, }; +static const struct dw8250_platform_data dw8250_intc10ee = { + .usr_reg = DW_UART_USR, + .quirks = DW_UART_QUIRK_IER_KICK, +}; + static const struct of_device_id dw8250_of_match[] = { { .compatible = "snps,dw-apb-uart", .data = &dw8250_dw_apb }, { .compatible = "cavium,octeon-3860-uart", .data = &dw8250_octeon_3860_data }, @@ -809,7 +989,7 @@ static const struct acpi_device_id dw8250_acpi_match[] = { { "INT33C5", (kernel_ulong_t)&dw8250_dw_apb }, { "INT3434", (kernel_ulong_t)&dw8250_dw_apb }, { "INT3435", (kernel_ulong_t)&dw8250_dw_apb }, - { "INTC10EE", (kernel_ulong_t)&dw8250_dw_apb }, + { "INTC10EE", (kernel_ulong_t)&dw8250_intc10ee }, { }, }; MODULE_DEVICE_TABLE(acpi, dw8250_acpi_match); @@ -827,6 +1007,7 @@ static struct platform_driver dw8250_platform_driver = { module_platform_driver(dw8250_platform_driver); +MODULE_IMPORT_NS("SERIAL_8250"); MODULE_AUTHOR("Jamie Iles"); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Synopsys DesignWare 8250 serial port driver"); diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 9e49ef48b851b..272bc07c9a6b5 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -100,6 +100,9 @@ #define OMAP_UART_REV_52 0x0502 #define OMAP_UART_REV_63 0x0603 +/* Resume register */ +#define UART_OMAP_RESUME 0x0B + /* Interrupt Enable Register 2 */ #define UART_OMAP_IER2 0x1B #define UART_OMAP_IER2_RHR_IT_DIS BIT(2) @@ -119,7 +122,6 @@ /* Timeout low and High */ #define UART_OMAP_TO_L 0x26 #define UART_OMAP_TO_H 0x27 - struct omap8250_priv { void __iomem *membase; int line; @@ -929,7 +931,6 @@ static void __dma_rx_do_complete(struct uart_8250_port *p) goto out; cookie = dma->rx_cookie; - dma->rx_running = 0; /* Re-enable RX FIFO interrupt now that transfer is complete */ if (priv->habit & UART_HAS_RHR_IT_DIS) { @@ -963,6 +964,7 @@ static void __dma_rx_do_complete(struct uart_8250_port *p) goto out; ret = tty_insert_flip_string(tty_port, dma->rx_buf, count); + dma->rx_running = 0; p->port.icount.rx += ret; p->port.icount.buf_overrun += count - ret; out: @@ -1256,6 +1258,20 @@ static u16 omap_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status return status; } +static void am654_8250_handle_uart_errors(struct uart_8250_port *up, u8 iir, u16 status) +{ + if (status & UART_LSR_OE) { + serial8250_clear_and_reinit_fifos(up); + serial_in(up, UART_LSR); + serial_in(up, UART_OMAP_RESUME); + } else { + if (status & (UART_LSR_FE | UART_LSR_PE | UART_LSR_BI)) + serial_in(up, UART_RX); + if (iir & UART_IIR_XOFF) + serial_in(up, UART_IIR); + } +} + static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status) { @@ -1266,7 +1282,8 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, * Queue a new transfer if FIFO has data. */ if ((status & (UART_LSR_DR | UART_LSR_BI)) && - (up->ier & UART_IER_RDI)) { + (up->ier & UART_IER_RDI) && !(status & UART_LSR_OE)) { + am654_8250_handle_uart_errors(up, iir, status); omap_8250_rx_dma(up); serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE); } else if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) { @@ -1282,6 +1299,8 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, serial_out(up, UART_OMAP_EFR2, 0x0); up->ier |= UART_IER_RLSI | UART_IER_RDI; serial_out(up, UART_IER, up->ier); + } else { + am654_8250_handle_uart_errors(up, iir, status); } } diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index 152f914c599dc..e4dc89b43ac6c 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -95,6 +95,11 @@ #define PCI_DEVICE_ID_MOXA_CP138E_A 0x1381 #define PCI_DEVICE_ID_MOXA_CP168EL_A 0x1683 +#define PCI_DEVICE_ID_ADDIDATA_CPCI7500 0x7003 +#define PCI_DEVICE_ID_ADDIDATA_CPCI7500_NG 0x7024 +#define PCI_DEVICE_ID_ADDIDATA_CPCI7420_NG 0x7025 +#define PCI_DEVICE_ID_ADDIDATA_CPCI7300_NG 0x7026 + /* Unknown vendors/cards - this should not be in linux/pci_ids.h */ #define PCI_SUBDEVICE_ID_UNKNOWN_0x1584 0x1584 #define PCI_SUBDEVICE_ID_UNKNOWN_0x1588 0x1588 @@ -132,6 +137,8 @@ struct serial_private { }; #define PCI_DEVICE_ID_HPE_PCI_SERIAL 0x37e +#define PCIE_VENDOR_ID_ASIX 0x125B +#define PCIE_DEVICE_ID_AX99100 0x9100 static const struct pci_device_id pci_use_msi[] = { { PCI_DEVICE_SUB(PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9900, @@ -144,6 +151,8 @@ static const struct pci_device_id pci_use_msi[] = { 0xA000, 0x1000) }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_HP_3PAR, PCI_DEVICE_ID_HPE_PCI_SERIAL, PCI_ANY_ID, PCI_ANY_ID) }, + { PCI_DEVICE_SUB(PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100, + 0xA000, 0x1000) }, { } }; @@ -907,6 +916,7 @@ static int pci_netmos_init(struct pci_dev *dev) case PCI_DEVICE_ID_NETMOS_9912: case PCI_DEVICE_ID_NETMOS_9922: case PCI_DEVICE_ID_NETMOS_9900: + case PCIE_DEVICE_ID_AX99100: num_serial = pci_netmos_9900_numports(dev); break; @@ -1645,7 +1655,7 @@ static int pci_fintek_rs485_config(struct uart_port *port, struct ktermios *term } static const struct serial_rs485 pci_fintek_rs485_supported = { - .flags = SER_RS485_ENABLED | SER_RS485_RTS_ON_SEND, + .flags = SER_RS485_ENABLED | SER_RS485_RTS_ON_SEND | SER_RS485_RTS_AFTER_SEND, /* F81504/508/512 does not support RTS delay before or after send */ }; @@ -2542,6 +2552,14 @@ static struct pci_serial_quirk pci_serial_quirks[] = { .init = pci_netmos_init, .setup = pci_netmos_9900_setup, }, + { + .vendor = PCIE_VENDOR_ID_ASIX, + .device = PCI_ANY_ID, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .init = pci_netmos_init, + .setup = pci_netmos_9900_setup, + }, /* * EndRun Technologies */ @@ -5996,6 +6014,38 @@ static const struct pci_device_id serial_pci_tbl[] = { 0, pbn_ADDIDATA_PCIe_8_3906250 }, + { PCI_VENDOR_ID_ADDIDATA, + PCI_DEVICE_ID_ADDIDATA_CPCI7500, + PCI_ANY_ID, + PCI_ANY_ID, + 0, + 0, + pbn_b0_4_115200 }, + + { PCI_VENDOR_ID_ADDIDATA, + PCI_DEVICE_ID_ADDIDATA_CPCI7500_NG, + PCI_ANY_ID, + PCI_ANY_ID, + 0, + 0, + pbn_b0_4_115200 }, + + { PCI_VENDOR_ID_ADDIDATA, + PCI_DEVICE_ID_ADDIDATA_CPCI7420_NG, + PCI_ANY_ID, + PCI_ANY_ID, + 0, + 0, + pbn_b0_2_115200 }, + + { PCI_VENDOR_ID_ADDIDATA, + PCI_DEVICE_ID_ADDIDATA_CPCI7300_NG, + PCI_ANY_ID, + PCI_ANY_ID, + 0, + 0, + pbn_b0_1_115200 }, + { PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9835, PCI_VENDOR_ID_IBM, 0x0299, 0, 0, pbn_b0_bt_2_115200 }, @@ -6031,6 +6081,10 @@ static const struct pci_device_id serial_pci_tbl[] = { 0xA000, 0x3002, 0, 0, pbn_NETMOS9900_2s_115200 }, + { PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100, + 0xA000, 0x1000, + 0, 0, pbn_b0_1_115200 }, + /* * Best Connectivity and Rosewill PCI Multi I/O cards */ @@ -6178,7 +6232,6 @@ static pci_ers_result_t serial8250_io_slot_reset(struct pci_dev *dev) return PCI_ERS_RESULT_DISCONNECT; pci_restore_state(dev); - pci_save_state(dev); return PCI_ERS_RESULT_RECOVERED; } diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 719faf92aa8ae..8785961a2a82e 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -488,7 +489,7 @@ serial_port_out_sync(struct uart_port *p, int offset, int value) /* * FIFO support. */ -static void serial8250_clear_fifos(struct uart_8250_port *p) +void serial8250_clear_fifos(struct uart_8250_port *p) { if (p->capabilities & UART_CAP_FIFO) { serial_out(p, UART_FCR, UART_FCR_ENABLE_FIFO); @@ -497,6 +498,7 @@ static void serial8250_clear_fifos(struct uart_8250_port *p) serial_out(p, UART_FCR, 0); } } +EXPORT_SYMBOL_NS_GPL(serial8250_clear_fifos, "SERIAL_8250"); static enum hrtimer_restart serial8250_em485_handle_start_tx(struct hrtimer *t); static enum hrtimer_restart serial8250_em485_handle_stop_tx(struct hrtimer *t); @@ -1782,20 +1784,16 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir) } /* - * This handles the interrupt from one port. + * Context: port's lock must be held by the caller. */ -int serial8250_handle_irq(struct uart_port *port, unsigned int iir) +void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir) { struct uart_8250_port *up = up_to_u8250p(port); struct tty_port *tport = &port->state->port; bool skip_rx = false; - unsigned long flags; u16 status; - if (iir & UART_IIR_NO_INT) - return 0; - - uart_port_lock_irqsave(port, &flags); + lockdep_assert_held_once(&port->lock); status = serial_lsr_in(up); @@ -1828,8 +1826,19 @@ int serial8250_handle_irq(struct uart_port *port, unsigned int iir) else if (!up->dma->tx_running) __stop_tx(up); } +} +EXPORT_SYMBOL_NS_GPL(serial8250_handle_irq_locked, "SERIAL_8250"); + +/* + * This handles the interrupt from one port. + */ +int serial8250_handle_irq(struct uart_port *port, unsigned int iir) +{ + if (iir & UART_IIR_NO_INT) + return 0; - uart_unlock_and_check_sysrq_irqrestore(port, flags); + guard(uart_port_lock_irqsave)(port); + serial8250_handle_irq_locked(port, iir); return 1; } @@ -2147,8 +2156,7 @@ static void serial8250_THRE_test(struct uart_port *port) if (up->port.flags & UPF_NO_THRE_TEST) return; - if (port->irqflags & IRQF_SHARED) - disable_irq_nosync(port->irq); + disable_irq(port->irq); /* * Test for UARTs that do not reassert THRE when the transmitter is idle and the interrupt @@ -2170,8 +2178,7 @@ static void serial8250_THRE_test(struct uart_port *port) serial_port_out(port, UART_IER, 0); } - if (port->irqflags & IRQF_SHARED) - enable_irq(port->irq); + enable_irq(port->irq); /* * If the interrupt is not reasserted, or we otherwise don't trust the iir, setup a timer to @@ -2350,6 +2357,7 @@ static int serial8250_startup(struct uart_port *port) void serial8250_do_shutdown(struct uart_port *port) { struct uart_8250_port *up = up_to_u8250p(port); + u32 lcr; serial8250_rpm_get(up); /* @@ -2376,13 +2384,13 @@ void serial8250_do_shutdown(struct uart_port *port) port->mctrl &= ~TIOCM_OUT2; serial8250_set_mctrl(port, port->mctrl); + + /* Disable break condition */ + lcr = serial_port_in(port, UART_LCR); + lcr &= ~UART_LCR_SBC; + serial_port_out(port, UART_LCR, lcr); } - /* - * Disable break condition and FIFOs - */ - serial_port_out(port, UART_LCR, - serial_port_in(port, UART_LCR) & ~UART_LCR_SBC); serial8250_clear_fifos(up); rsa_disable(up); @@ -2392,6 +2400,12 @@ void serial8250_do_shutdown(struct uart_port *port) * the IRQ chain. */ serial_port_in(port, UART_RX); + /* + * LCR writes on DW UART can trigger late (unmaskable) IRQs. + * Handle them before releasing the handler. + */ + synchronize_irq(port->irq); + serial8250_rpm_put(up); up->ops->release_irq(up); @@ -3185,6 +3199,17 @@ void serial8250_set_defaults(struct uart_8250_port *up) } EXPORT_SYMBOL_GPL(serial8250_set_defaults); +void serial8250_fifo_wait_for_lsr_thre(struct uart_8250_port *up, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) { + if (wait_for_lsr(up, UART_LSR_THRE)) + return; + } +} +EXPORT_SYMBOL_NS_GPL(serial8250_fifo_wait_for_lsr_thre, "SERIAL_8250"); + #ifdef CONFIG_SERIAL_8250_CONSOLE static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) @@ -3226,16 +3251,6 @@ static void serial8250_console_restore(struct uart_8250_port *up) serial8250_out_MCR(up, up->mcr | UART_MCR_DTR | UART_MCR_RTS); } -static void fifo_wait_for_lsr(struct uart_8250_port *up, unsigned int count) -{ - unsigned int i; - - for (i = 0; i < count; i++) { - if (wait_for_lsr(up, UART_LSR_THRE)) - return; - } -} - /* * Print a string to the serial port using the device FIFO * @@ -3254,7 +3269,7 @@ static void serial8250_console_fifo_write(struct uart_8250_port *up, while (s != end) { /* Allow timeout for each byte of a possibly full FIFO */ - fifo_wait_for_lsr(up, fifosize); + serial8250_fifo_wait_for_lsr_thre(up, fifosize); for (i = 0; i < fifosize && s != end; ++i) { if (*s == '\n' && !cr_sent) { @@ -3272,7 +3287,7 @@ static void serial8250_console_fifo_write(struct uart_8250_port *up, * Allow timeout for each byte written since the caller will only wait * for UART_LSR_BOTH_EMPTY using the timeout of a single character */ - fifo_wait_for_lsr(up, tx_count); + serial8250_fifo_wait_for_lsr_thre(up, tx_count); } /* diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 282116765e648..2b9c8b39d68f5 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -486,14 +486,14 @@ config SERIAL_IMX can enable its onboard serial port by enabling this option. config SERIAL_IMX_CONSOLE - tristate "Console on IMX serial port" + bool "Console on IMX serial port" depends on SERIAL_IMX select SERIAL_CORE_CONSOLE help If you have enabled the serial port on the Freescale IMX - CPU you can make it the console by answering Y/M to this option. + CPU you can make it the console by answering Y to this option. - Even if you say Y/M here, the currently visible virtual console + Even if you say Y here, the currently visible virtual console (/dev/tty0) will still be used as the system console by default, but you can alter that using a kernel command line option such as "console=ttymxc0". (Try "man bootparam" or see the documentation of @@ -671,7 +671,7 @@ config SERIAL_SH_SCI_EARLYCON default ARCH_RENESAS config SERIAL_SH_SCI_DMA - bool "DMA support" if EXPERT + bool "Support for DMA on SuperH SCI(F)" if EXPERT depends on SERIAL_SH_SCI && DMA_ENGINE default ARCH_RENESAS diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 500dfc009d03e..90e2ea1e8afe5 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -2697,8 +2697,22 @@ static void imx_uart_save_context(struct imx_port *sport) /* called with irq off */ static void imx_uart_enable_wakeup(struct imx_port *sport, bool on) { + struct tty_port *port = &sport->port.state->port; + struct device *tty_dev; + bool may_wake = false; u32 ucr3; + scoped_guard(tty_port_tty, port) { + struct tty_struct *tty = scoped_tty(); + + tty_dev = tty->dev; + may_wake = tty_dev && device_may_wakeup(tty_dev); + } + + /* only configure the wake register when device set as wakeup source */ + if (!may_wake) + return; + uart_port_lock_irq(&sport->port); ucr3 = imx_uart_readl(sport, UCR3); diff --git a/drivers/tty/serial/jsm/jsm_driver.c b/drivers/tty/serial/jsm/jsm_driver.c index 417a5b6bffc34..8d21373cae573 100644 --- a/drivers/tty/serial/jsm/jsm_driver.c +++ b/drivers/tty/serial/jsm/jsm_driver.c @@ -355,7 +355,6 @@ static void jsm_io_resume(struct pci_dev *pdev) struct jsm_board *brd = pci_get_drvdata(pdev); pci_restore_state(pdev); - pci_save_state(pdev); jsm_uart_port_init(brd); } diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index 8058b839b26ce..cbb4f3222116a 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -1022,8 +1022,20 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport) { struct qcom_geni_serial_port *port = to_dev_port(uport); struct tty_port *tport = &uport->state->port; + unsigned int fifo_len = kfifo_len(&tport->xmit_fifo); + + /* + * Only advance the kfifo if it still contains the bytes that were + * transferred. uart_flush_buffer() may have run before this IRQ + * fired: it calls kfifo_reset() under the port lock, making + * fifo_len = 0 while tx_remaining remains non-zero. Calling + * uart_xmit_advance() in that case would underflow kfifo->out past + * kfifo->in, making kfifo_len() wrap to UART_XMIT_SIZE - tx_remaining + * and triggering a spurious large DMA transfer of stale data. + */ + if (fifo_len >= port->tx_remaining) + uart_xmit_advance(uport, port->tx_remaining); - uart_xmit_advance(uport, port->tx_remaining); geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining); port->tx_dma_addr = 0; port->tx_remaining = 0; diff --git a/drivers/tty/serial/serial_base_bus.c b/drivers/tty/serial/serial_base_bus.c index 22749ab0428a7..1e1ad28d83fcf 100644 --- a/drivers/tty/serial/serial_base_bus.c +++ b/drivers/tty/serial/serial_base_bus.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -60,6 +60,7 @@ void serial_base_driver_unregister(struct device_driver *driver) driver_unregister(driver); } +/* On failure the caller must put device @dev with put_device() */ static int serial_base_device_init(struct uart_port *port, struct device *dev, struct device *parent_dev, @@ -73,7 +74,9 @@ static int serial_base_device_init(struct uart_port *port, dev->parent = parent_dev; dev->bus = &serial_base_bus_type; dev->release = release; - device_set_of_node_from_dev(dev, parent_dev); + dev->of_node_reused = true; + + device_set_node(dev, fwnode_handle_get(dev_fwnode(parent_dev))); if (!serial_base_initialized) { dev_dbg(port->dev, "uart_add_one_port() called before arch_initcall()?\n"); @@ -94,7 +97,7 @@ static void serial_base_ctrl_release(struct device *dev) { struct serial_ctrl_device *ctrl_dev = to_serial_base_ctrl_device(dev); - of_node_put(dev->of_node); + fwnode_handle_put(dev_fwnode(dev)); kfree(ctrl_dev); } @@ -142,7 +145,7 @@ static void serial_base_port_release(struct device *dev) { struct serial_port_device *port_dev = to_serial_base_port_device(dev); - of_node_put(dev->of_node); + fwnode_handle_put(dev_fwnode(dev)); kfree(port_dev); } diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 4757293ece8c6..ca72454b3eb09 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -643,7 +643,10 @@ static unsigned int uart_write_room(struct tty_struct *tty) unsigned int ret; port = uart_port_ref_lock(state, &flags); - ret = kfifo_avail(&state->port.xmit_fifo); + if (!state->port.xmit_buf) + ret = 0; + else + ret = kfifo_avail(&state->port.xmit_fifo); uart_port_unlock_deref(port, flags); return ret; } @@ -3102,6 +3105,12 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u if (uport->cons && uport->dev) of_console_check(uport->dev->of_node, uport->cons->name, uport->line); + /* + * TTY port has to be linked with the driver before register_console() + * in uart_configure_port(), because user-space could open the console + * immediately after. + */ + tty_port_link_device(port, drv->tty_driver, uport->line); uart_configure_port(drv, state, uport); port->console = uart_console(uport); diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c index 62bb62b82cbe5..d91b1fb69bebd 100644 --- a/drivers/tty/serial/sh-sci.c +++ b/drivers/tty/serial/sh-sci.c @@ -1024,8 +1024,16 @@ static int sci_handle_fifo_overrun(struct uart_port *port) status = s->ops->read_reg(port, s->params->overrun_reg); if (status & s->params->overrun_mask) { - status &= ~s->params->overrun_mask; - s->ops->write_reg(port, s->params->overrun_reg, status); + if (s->type == SCI_PORT_RSCI) { + /* + * All of the CFCLR_*C clearing bits match the corresponding + * CSR_*status bits. So, reuse the overrun mask for clearing. + */ + s->ops->clear_SCxSR(port, s->params->overrun_mask); + } else { + status &= ~s->params->overrun_mask; + s->ops->write_reg(port, s->params->overrun_reg, status); + } port->icount.overrun++; @@ -1732,7 +1740,7 @@ static void sci_dma_check_tx_occurred(struct sci_port *s) struct dma_tx_state state; enum dma_status status; - if (!s->chan_tx) + if (!s->chan_tx || s->cookie_tx <= 0) return; status = dmaengine_tx_status(s->chan_tx, s->cookie_tx, &state); diff --git a/drivers/tty/serial/sprd_serial.c b/drivers/tty/serial/sprd_serial.c index 8c9366321f8e7..092755f356836 100644 --- a/drivers/tty/serial/sprd_serial.c +++ b/drivers/tty/serial/sprd_serial.c @@ -1133,6 +1133,9 @@ static int sprd_clk_init(struct uart_port *uport) clk_uart = devm_clk_get(uport->dev, "uart"); if (IS_ERR(clk_uart)) { + if (PTR_ERR(clk_uart) == -EPROBE_DEFER) + return -EPROBE_DEFER; + dev_warn(uport->dev, "uart%d can't get uart clock\n", uport->line); clk_uart = NULL; @@ -1140,6 +1143,9 @@ static int sprd_clk_init(struct uart_port *uport) clk_parent = devm_clk_get(uport->dev, "source"); if (IS_ERR(clk_parent)) { + if (PTR_ERR(clk_parent) == -EPROBE_DEFER) + return -EPROBE_DEFER; + dev_warn(uport->dev, "uart%d can't get source clock\n", uport->line); clk_parent = NULL; diff --git a/drivers/tty/serial/uartlite.c b/drivers/tty/serial/uartlite.c index 39c1fd1ff9ced..6240c3d4dfd79 100644 --- a/drivers/tty/serial/uartlite.c +++ b/drivers/tty/serial/uartlite.c @@ -878,6 +878,7 @@ static int ulite_probe(struct platform_device *pdev) pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_set_autosuspend_delay(&pdev->dev, UART_AUTOSUSPEND_TIMEOUT); pm_runtime_set_active(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); pm_runtime_enable(&pdev->dev); ret = ulite_assign(&pdev->dev, id, res->start, irq, pdata); diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c index a66b44d21fba2..a354326a4242a 100644 --- a/drivers/tty/serial/xilinx_uartps.c +++ b/drivers/tty/serial/xilinx_uartps.c @@ -430,10 +430,17 @@ static void cdns_uart_handle_tx(void *dev_id) struct tty_port *tport = &port->state->port; unsigned int numbytes; unsigned char ch; + ktime_t rts_delay; if (kfifo_is_empty(&tport->xmit_fifo) || uart_tx_stopped(port)) { /* Disable the TX Empty interrupt */ writel(CDNS_UART_IXR_TXEMPTY, port->membase + CDNS_UART_IDR); + /* Set RTS line after delay */ + if (cdns_uart->port->rs485.flags & SER_RS485_ENABLED) { + cdns_uart->tx_timer.function = &cdns_rs485_rx_callback; + rts_delay = ns_to_ktime(cdns_calc_after_tx_delay(cdns_uart)); + hrtimer_start(&cdns_uart->tx_timer, rts_delay, HRTIMER_MODE_REL); + } return; } @@ -450,13 +457,6 @@ static void cdns_uart_handle_tx(void *dev_id) /* Enable the TX Empty interrupt */ writel(CDNS_UART_IXR_TXEMPTY, cdns_uart->port->membase + CDNS_UART_IER); - - if (cdns_uart->port->rs485.flags & SER_RS485_ENABLED && - (kfifo_is_empty(&tport->xmit_fifo) || uart_tx_stopped(port))) { - hrtimer_update_function(&cdns_uart->tx_timer, cdns_rs485_rx_callback); - hrtimer_start(&cdns_uart->tx_timer, - ns_to_ktime(cdns_calc_after_tx_delay(cdns_uart)), HRTIMER_MODE_REL); - } } /** diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 6e0089b85c270..a1117196cbb3c 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1345,6 +1345,8 @@ struct vc_data *vc_deallocate(unsigned int currcons) kfree(vc->vc_saved_screen); vc->vc_saved_screen = NULL; } + vc_uniscr_free(vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = NULL; } return vc; } @@ -1890,6 +1892,8 @@ static void enter_alt_screen(struct vc_data *vc) vc->vc_saved_screen = kmemdup((u16 *)vc->vc_origin, size, GFP_KERNEL); if (vc->vc_saved_screen == NULL) return; + vc->vc_saved_uni_lines = vc->vc_uni_lines; + vc->vc_uni_lines = NULL; vc->vc_saved_rows = vc->vc_rows; vc->vc_saved_cols = vc->vc_cols; save_cur(vc); @@ -1911,6 +1915,26 @@ static void leave_alt_screen(struct vc_data *vc) dest = ((u16 *)vc->vc_origin) + r * vc->vc_cols; memcpy(dest, src, 2 * cols); } + /* + * If the console was resized while in the alternate screen, + * resize the saved unicode buffer to the current dimensions. + * On allocation failure new_uniscr is NULL, causing the old + * buffer to be freed and vc_uni_lines to be lazily rebuilt + * via vc_uniscr_check() when next needed. + */ + if (vc->vc_saved_uni_lines && + (vc->vc_saved_rows != vc->vc_rows || + vc->vc_saved_cols != vc->vc_cols)) { + u32 **new_uniscr = vc_uniscr_alloc(vc->vc_cols, vc->vc_rows); + + if (new_uniscr) + vc_uniscr_copy_area(new_uniscr, vc->vc_cols, vc->vc_rows, + vc->vc_saved_uni_lines, cols, 0, rows); + vc_uniscr_free(vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = new_uniscr; + } + vc_uniscr_set(vc, vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = NULL; restore_cur(vc); /* Update the entire screen */ if (con_should_update(vc)) @@ -2233,6 +2257,8 @@ static void reset_terminal(struct vc_data *vc, int do_clear) if (vc->vc_saved_screen != NULL) { kfree(vc->vc_saved_screen); vc->vc_saved_screen = NULL; + vc_uniscr_free(vc->vc_saved_uni_lines); + vc->vc_saved_uni_lines = NULL; vc->vc_saved_rows = 0; vc->vc_saved_cols = 0; } diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index d0a2c963a27d3..1f0d38aa37f92 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -6,6 +6,8 @@ #include #include +void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs); + static inline bool ufshcd_is_user_access_allowed(struct ufs_hba *hba) { return !hba->shutting_down; diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index d6a060a724618..f2188e8b9a93e 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -117,7 +117,7 @@ static bool is_mcq_supported(struct ufs_hba *hba) module_param(use_mcq_mode, bool, 0644); MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); -static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT; +static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_MAX; static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp) { @@ -516,8 +516,8 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, if (hba->mcq_enabled) { struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); - - hwq_id = hwq->id; + if (hwq) + hwq_id = hwq->id; } else { doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); } @@ -3837,7 +3837,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, str[ret++] = '\0'; } else { - str = kmemdup(uc_str, uc_str->len, GFP_KERNEL); + str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); if (!str) { ret = -ENOMEM; goto out; @@ -4401,14 +4401,6 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) spin_unlock_irqrestore(hba->host->host_lock, flags); mutex_unlock(&hba->uic_cmd_mutex); - /* - * If the h8 exit fails during the runtime resume process, it becomes - * stuck and cannot be recovered through the error handler. To fix - * this, use link recovery instead of the error handler. - */ - if (ret && hba->pm_op_in_progress) - ret = ufshcd_link_recovery(hba); - return ret; } @@ -5954,6 +5946,7 @@ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) hba->auto_bkops_enabled = false; trace_ufshcd_auto_bkops_state(hba, "Disabled"); + hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; hba->is_urgent_bkops_lvl_checked = false; out: return err; @@ -6057,7 +6050,7 @@ static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba) * impacted or critical. Handle these device by determining their urgent * bkops status at runtime. */ - if (curr_status < BKOPS_STATUS_PERF_IMPACT) { + if ((curr_status > BKOPS_STATUS_NO_OP) && (curr_status < BKOPS_STATUS_PERF_IMPACT)) { dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n", __func__, curr_status); /* update the current status as the urgent bkops level */ @@ -6498,6 +6491,11 @@ static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend) static void ufshcd_err_handling_prepare(struct ufs_hba *hba) { + /* + * A WLUN resume failure could potentially lead to the HBA being + * runtime suspended, so take an extra reference on hba->dev. + */ + pm_runtime_get_sync(hba->dev); ufshcd_rpm_get_sync(hba); if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) || hba->is_sys_suspended) { @@ -6537,6 +6535,7 @@ static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) if (ufshcd_is_clkscaling_supported(hba)) ufshcd_clk_scaling_suspend(hba, false); ufshcd_rpm_put(hba); + pm_runtime_put(hba->dev); } static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) @@ -6551,28 +6550,42 @@ static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) #ifdef CONFIG_PM static void ufshcd_recover_pm_error(struct ufs_hba *hba) { + struct scsi_target *starget = hba->ufs_device_wlun->sdev_target; struct Scsi_Host *shost = hba->host; struct scsi_device *sdev; struct request_queue *q; - int ret; + bool resume_sdev_queues = false; hba->is_sys_suspended = false; + /* - * Set RPM status of wlun device to RPM_ACTIVE, - * this also clears its runtime error. + * Ensure the parent's error status is cleared before proceeding + * to the child, as the parent must be active to activate the child. */ - ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); + if (hba->dev->power.runtime_error) { + /* hba->dev has no functional parent thus simplily set RPM_ACTIVE */ + pm_runtime_set_active(hba->dev); + resume_sdev_queues = true; + } + + if (hba->ufs_device_wlun->sdev_gendev.power.runtime_error) { + /* + * starget, parent of wlun, might be suspended if wlun resume failed. + * Make sure parent is resumed before set child (wlun) active. + */ + pm_runtime_get_sync(&starget->dev); + pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); + pm_runtime_put_sync(&starget->dev); + resume_sdev_queues = true; + } - /* hba device might have a runtime error otherwise */ - if (ret) - ret = pm_runtime_set_active(hba->dev); /* * If wlun device had runtime error, we also need to resume those * consumer scsi devices in case any of them has failed to be * resumed due to supplier runtime resume failure. This is to unblock * blk_queue_enter in case there are bios waiting inside it. */ - if (!ret) { + if (resume_sdev_queues) { shost_for_each_device(sdev, shost) { q = sdev->request_queue; if (q->dev && (q->rpm_status == RPM_SUSPENDED || @@ -6670,19 +6683,22 @@ static void ufshcd_err_handler(struct work_struct *work) hba->saved_uic_err, hba->force_reset, ufshcd_is_link_broken(hba) ? "; link is broken" : ""); - /* - * Use ufshcd_rpm_get_noresume() here to safely perform link recovery - * even if an error occurs during runtime suspend or runtime resume. - * This avoids potential deadlocks that could happen if we tried to - * resume the device while a PM operation is already in progress. - */ - ufshcd_rpm_get_noresume(hba); - if (hba->pm_op_in_progress) { - ufshcd_link_recovery(hba); + if (hba->ufs_device_wlun) { + /* + * Use ufshcd_rpm_get_noresume() here to safely perform link + * recovery even if an error occurs during runtime suspend or + * runtime resume. This avoids potential deadlocks that could + * happen if we tried to resume the device while a PM operation + * is already in progress. + */ + ufshcd_rpm_get_noresume(hba); + if (hba->pm_op_in_progress) { + ufshcd_link_recovery(hba); + ufshcd_rpm_put(hba); + return; + } ufshcd_rpm_put(hba); - return; } - ufshcd_rpm_put(hba); down(&hba->host_sem); spin_lock_irqsave(hba->host->host_lock, flags); @@ -7062,7 +7078,7 @@ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); if (ret) - outstanding_cqs = (1U << hba->nr_hw_queues) - 1; + outstanding_cqs = (1ULL << hba->nr_hw_queues) - 1; /* Exclude the poll queues */ nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; @@ -9855,6 +9871,8 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && req_link_state == UIC_LINK_ACTIVE_STATE) { + ufshcd_disable_auto_bkops(hba); + flush_work(&hba->eeh_work); goto vops_suspend; } @@ -9907,6 +9925,7 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) } flush_work(&hba->eeh_work); + cancel_delayed_work_sync(&hba->ufs_rtc_update_work); ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); if (ret) @@ -9961,7 +9980,6 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (ret) goto set_link_active; - cancel_delayed_work_sync(&hba->ufs_rtc_update_work); goto out; set_link_active: @@ -10033,7 +10051,15 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) } else { dev_err(hba->dev, "%s: hibern8 exit failed %d\n", __func__, ret); - goto vendor_suspend; + /* + * If the h8 exit fails during the runtime resume + * process, it becomes stuck and cannot be recovered + * through the error handler. To fix this, use link + * recovery instead of the error handler. + */ + ret = ufshcd_link_recovery(hba); + if (ret) + goto vendor_suspend; } } else if (ufshcd_is_link_off(hba)) { /* @@ -10220,7 +10246,7 @@ static int ufshcd_suspend(struct ufs_hba *hba) ret = ufshcd_setup_clocks(hba, false); if (ret) { ufshcd_enable_irq(hba); - return ret; + goto out; } if (ufshcd_is_clkgating_allowed(hba)) { hba->clk_gating.state = CLKS_OFF; @@ -10232,6 +10258,9 @@ static int ufshcd_suspend(struct ufs_hba *hba) /* Put the host controller in low power mode if possible */ ufshcd_hba_vreg_set_lpm(hba); ufshcd_pm_qos_update(hba, false); +out: + if (ret) + ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret); return ret; } diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig index 191fbd799ec5b..48ee7e9b665ef 100644 --- a/drivers/ufs/host/Kconfig +++ b/drivers/ufs/host/Kconfig @@ -72,6 +72,7 @@ config SCSI_UFS_QCOM config SCSI_UFS_MEDIATEK tristate "Mediatek specific hooks to UFS controller platform driver" depends on SCSI_UFSHCD_PLATFORM && ARCH_MEDIATEK + depends on PM depends on RESET_CONTROLLER select PHY_MTK_UFS select RESET_TI_SYSCON diff --git a/drivers/ufs/host/ufs-mediatek-trace.h b/drivers/ufs/host/ufs-mediatek-trace.h index b5f2ec3140748..0df8ac843379a 100644 --- a/drivers/ufs/host/ufs-mediatek-trace.h +++ b/drivers/ufs/host/ufs-mediatek-trace.h @@ -33,19 +33,19 @@ TRACE_EVENT(ufs_mtk_clk_scale, TP_ARGS(name, scale_up, clk_rate), TP_STRUCT__entry( - __field(const char*, name) + __string(name, name) __field(bool, scale_up) __field(unsigned long, clk_rate) ), TP_fast_assign( - __entry->name = name; + __assign_str(name); __entry->scale_up = scale_up; __entry->clk_rate = clk_rate; ), TP_printk("ufs: clk (%s) scaled %s @ %lu", - __entry->name, + __get_str(name), __entry->scale_up ? "up" : "down", __entry->clk_rate) ); diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index 758a393a9de1a..3c63adca561d2 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -2366,13 +2366,17 @@ static void ufs_mtk_remove(struct platform_device *pdev) ufshcd_pltfrm_remove(pdev); } -#ifdef CONFIG_PM_SLEEP static int ufs_mtk_system_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); struct arm_smccc_res res; int ret; + if (hba->shutting_down) { + ret = -EBUSY; + goto out; + } + ret = ufshcd_system_suspend(dev); if (ret) goto out; @@ -2408,9 +2412,7 @@ static int ufs_mtk_system_resume(struct device *dev) return ret; } -#endif -#ifdef CONFIG_PM static int ufs_mtk_runtime_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); @@ -2449,13 +2451,10 @@ static int ufs_mtk_runtime_resume(struct device *dev) return ufshcd_runtime_resume(dev); } -#endif static const struct dev_pm_ops ufs_mtk_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, - ufs_mtk_system_resume) - SET_RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, - ufs_mtk_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, ufs_mtk_system_resume) + RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, ufs_mtk_runtime_resume, NULL) .prepare = ufshcd_suspend_prepare, .complete = ufshcd_resume_complete, }; @@ -2465,7 +2464,7 @@ static struct platform_driver ufs_mtk_pltform = { .remove = ufs_mtk_remove, .driver = { .name = "ufshcd-mtk", - .pm = &ufs_mtk_pm_ops, + .pm = pm_ptr(&ufs_mtk_pm_ops), .of_match_table = ufs_mtk_of_match, }, }; diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c index eba0e66174834..07e6f0ece6ef6 100644 --- a/drivers/ufs/host/ufs-qcom.c +++ b/drivers/ufs/host/ufs-qcom.c @@ -305,6 +305,15 @@ static int ufs_qcom_ice_prepare_key(struct blk_crypto_profile *profile, return qcom_ice_prepare_key(host->ice, lt_key, lt_key_size, eph_key); } +static int ufs_qcom_ice_scale_clk(struct ufs_qcom_host *host, unsigned long target_freq, + bool round_ceil) +{ + if (host->hba->caps & UFSHCD_CAP_CRYPTO) + return qcom_ice_scale_clk(host->ice, target_freq, round_ceil); + + return 0; +} + static const struct blk_crypto_ll_ops ufs_qcom_crypto_ops = { .keyslot_program = ufs_qcom_ice_keyslot_program, .keyslot_evict = ufs_qcom_ice_keyslot_evict, @@ -339,6 +348,12 @@ static void ufs_qcom_config_ice_allocator(struct ufs_qcom_host *host) { } +static int ufs_qcom_ice_scale_clk(struct ufs_qcom_host *host, unsigned long target_freq, + bool round_ceil) +{ + return 0; +} + #endif static void ufs_qcom_disable_lane_clks(struct ufs_qcom_host *host) @@ -1649,12 +1664,17 @@ static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba, bool scale_up, else err = ufs_qcom_clk_scale_down_post_change(hba, target_freq); - if (err) { ufshcd_uic_hibern8_exit(hba); return err; } + err = ufs_qcom_ice_scale_clk(host, target_freq, !scale_up); + if (err && err != -EOPNOTSUPP) { + ufshcd_uic_hibern8_exit(hba); + return err; + } + ufs_qcom_icc_update_bw(host); ufshcd_uic_hibern8_exit(hba); } diff --git a/drivers/ufs/host/ufs-rockchip.c b/drivers/ufs/host/ufs-rockchip.c index 8754085dd0ccf..8cecb28cdce41 100644 --- a/drivers/ufs/host/ufs-rockchip.c +++ b/drivers/ufs/host/ufs-rockchip.c @@ -20,9 +20,17 @@ #include "ufshcd-pltfrm.h" #include "ufs-rockchip.h" +static void ufs_rockchip_controller_reset(struct ufs_rockchip_host *host) +{ + reset_control_assert(host->rst); + udelay(1); + reset_control_deassert(host->rst); +} + static int ufs_rockchip_hce_enable_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) { + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); int err = 0; if (status == POST_CHANGE) { @@ -37,6 +45,9 @@ static int ufs_rockchip_hce_enable_notify(struct ufs_hba *hba, return ufshcd_vops_phy_initialization(hba); } + /* PRE_CHANGE */ + ufs_rockchip_controller_reset(host); + return 0; } @@ -156,9 +167,7 @@ static int ufs_rockchip_common_init(struct ufs_hba *hba) return dev_err_probe(dev, PTR_ERR(host->rst), "failed to get reset control\n"); - reset_control_assert(host->rst); - udelay(1); - reset_control_deassert(host->rst); + ufs_rockchip_controller_reset(host); host->ref_out_clk = devm_clk_get_enabled(dev, "ref_out"); if (IS_ERR(host->ref_out_clk)) @@ -282,9 +291,7 @@ static int ufs_rockchip_runtime_resume(struct device *dev) return err; } - reset_control_assert(host->rst); - udelay(1); - reset_control_deassert(host->rst); + ufs_rockchip_controller_reset(host); return ufshcd_runtime_resume(dev); } diff --git a/drivers/uio/uio_fsl_elbc_gpcm.c b/drivers/uio/uio_fsl_elbc_gpcm.c index 81454c3e2484c..338dd2aaabc87 100644 --- a/drivers/uio/uio_fsl_elbc_gpcm.c +++ b/drivers/uio/uio_fsl_elbc_gpcm.c @@ -384,6 +384,11 @@ static int uio_fsl_elbc_gpcm_probe(struct platform_device *pdev) /* set all UIO data */ info->mem[0].name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%pOFn", node); + if (!info->mem[0].name) { + ret = -ENODEV; + goto out_err3; + } + info->mem[0].addr = res.start; info->mem[0].size = resource_size(&res); info->mem[0].memtype = UIO_MEM_PHYS; @@ -423,6 +428,8 @@ static int uio_fsl_elbc_gpcm_probe(struct platform_device *pdev) out_err2: if (priv->shutdown) priv->shutdown(info, true); + +out_err3: iounmap(info->mem[0].internal_addr); return ret; } diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c index d9d8dc05b2359..6b8b0354b41a6 100644 --- a/drivers/usb/cdns3/cdns3-gadget.c +++ b/drivers/usb/cdns3/cdns3-gadget.c @@ -2589,6 +2589,9 @@ static int __cdns3_gadget_ep_queue(struct usb_ep *ep, struct cdns3_request *priv_req; int ret = 0; + if (!ep->desc) + return -ESHUTDOWN; + request->actual = 0; request->status = -EINPROGRESS; priv_req = to_cdns3_request(request); @@ -3429,6 +3432,7 @@ static int __cdns3_gadget_init(struct cdns *cdns) ret = cdns3_gadget_start(cdns); if (ret) { pm_runtime_put_sync(cdns->dev); + cdns_drd_gadget_off(cdns); return ret; } diff --git a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c index 1243a5cea91b5..f0e32227c0b79 100644 --- a/drivers/usb/cdns3/core.c +++ b/drivers/usb/cdns3/core.c @@ -551,7 +551,7 @@ int cdns_resume(struct cdns *cdns) } } - if (cdns->roles[cdns->role]->resume) + if (!role_changed && cdns->roles[cdns->role]->resume) cdns->roles[cdns->role]->resume(cdns, power_lost); return 0; diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c index 64a421ae0f05b..c8d931d9d4330 100644 --- a/drivers/usb/chipidea/udc.c +++ b/drivers/usb/chipidea/udc.c @@ -931,6 +931,13 @@ __acquires(hwep->lock) list_del_init(&hwreq->queue); hwreq->req.status = -ESHUTDOWN; + /* Unmap DMA and clean up bounce buffers before giving back */ + usb_gadget_unmap_request_by_dev(hwep->ci->dev->parent, + &hwreq->req, hwep->dir); + + if (hwreq->sgt.sgl) + sglist_do_debounce(hwreq, false); + if (hwreq->req.complete != NULL) { spin_unlock(hwep->lock); usb_gadget_giveback_request(&hwep->ep, &hwreq->req); diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 73f9476774ae6..d05b8806124a3 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -114,6 +114,8 @@ static int acm_ctrl_msg(struct acm *acm, int request, int value, int retval; retval = usb_autopm_get_interface(acm->control); +#define VENDOR_CLASS_DATA_IFACE BIT(9) /* data interface uses vendor-specific class */ +#define ALWAYS_POLL_CTRL BIT(10) /* keep ctrl URB active even without an open TTY */ if (retval) return retval; @@ -710,12 +712,14 @@ static int acm_port_activate(struct tty_port *port, struct tty_struct *tty) set_bit(TTY_NO_WRITE_SPLIT, &tty->flags); acm->control->needs_remote_wakeup = 1; - acm->ctrlurb->dev = acm->dev; - retval = usb_submit_urb(acm->ctrlurb, GFP_KERNEL); - if (retval) { - dev_err(&acm->control->dev, - "%s - usb_submit_urb(ctrl irq) failed\n", __func__); - goto error_submit_urb; + if (!(acm->quirks & ALWAYS_POLL_CTRL)) { + acm->ctrlurb->dev = acm->dev; + retval = usb_submit_urb(acm->ctrlurb, GFP_KERNEL); + if (retval) { + dev_err(&acm->control->dev, + "%s - usb_submit_urb(ctrl irq) failed\n", __func__); + goto error_submit_urb; + } } acm_tty_set_termios(tty, NULL); @@ -788,6 +792,14 @@ static void acm_port_shutdown(struct tty_port *port) acm_unpoison_urbs(acm); + if (acm->quirks & ALWAYS_POLL_CTRL) { + acm->ctrlurb->dev = acm->dev; + if (usb_submit_urb(acm->ctrlurb, GFP_KERNEL)) + dev_dbg(&acm->control->dev, + "ctrl polling restart failed after port close\n"); + /* port_shutdown() cleared DTR/RTS; restore them */ + acm_set_control(acm, USB_CDC_CTRL_DTR | USB_CDC_CTRL_RTS); + } } static void acm_tty_cleanup(struct tty_struct *tty) @@ -1225,6 +1237,12 @@ static int acm_probe(struct usb_interface *intf, if (!data_interface || !control_interface) return -ENODEV; goto skip_normal_probe; + } else if (quirks == NO_UNION_12) { + data_interface = usb_ifnum_to_if(usb_dev, 2); + control_interface = usb_ifnum_to_if(usb_dev, 1); + if (!data_interface || !control_interface) + return -ENODEV; + goto skip_normal_probe; } /* normal probing*/ @@ -1322,6 +1340,9 @@ static int acm_probe(struct usb_interface *intf, dev_dbg(&intf->dev, "Your device has switched interfaces.\n"); swap(control_interface, data_interface); + } else if (quirks & VENDOR_CLASS_DATA_IFACE) { + dev_dbg(&intf->dev, + "Vendor-specific data interface class, continuing.\n"); } else { return -EINVAL; } @@ -1379,6 +1400,8 @@ static int acm_probe(struct usb_interface *intf, acm->ctrl_caps = h.usb_cdc_acm_descriptor->bmCapabilities; if (quirks & NO_CAP_LINE) acm->ctrl_caps &= ~USB_CDC_CAP_LINE; + if (quirks & MISSING_CAP_BRK) + acm->ctrl_caps |= USB_CDC_CAP_BRK; acm->ctrlsize = ctrlsize; acm->readsize = readsize; acm->rx_buflimit = num_rx_buf; @@ -1514,6 +1537,9 @@ static int acm_probe(struct usb_interface *intf, acm->line.bDataBits = 8; acm_set_line(acm, &acm->line); + if (quirks & ALWAYS_POLL_CTRL) + acm_set_control(acm, USB_CDC_CTRL_DTR | USB_CDC_CTRL_RTS); + if (!acm->combined_interfaces) { rv = usb_driver_claim_interface(&acm_driver, data_interface, acm); if (rv) @@ -1535,6 +1561,13 @@ static int acm_probe(struct usb_interface *intf, dev_info(&intf->dev, "ttyACM%d: USB ACM device\n", minor); + if (acm->quirks & ALWAYS_POLL_CTRL) { + acm->ctrlurb->dev = acm->dev; + if (usb_submit_urb(acm->ctrlurb, GFP_KERNEL)) + dev_warn(&intf->dev, + "failed to start persistent ctrl polling\n"); + } + return 0; err_release_data_interface: @@ -1661,7 +1694,7 @@ static int acm_resume(struct usb_interface *intf) acm_unpoison_urbs(acm); - if (tty_port_initialized(&acm->port)) { + if (tty_port_initialized(&acm->port) || (acm->quirks & ALWAYS_POLL_CTRL)) { rv = usb_submit_urb(acm->ctrlurb, GFP_ATOMIC); for (;;) { @@ -1746,6 +1779,9 @@ static const struct usb_device_id acm_ids[] = { { USB_DEVICE(0x045b, 0x024D), /* Renesas R-Car E3 USB Download mode */ .driver_info = DISABLE_ECHO, /* Don't echo banner */ }, + { USB_DEVICE(0x04b8, 0x0d12), /* EPSON HMD Com&Sens */ + .driver_info = NO_UNION_12, /* union descriptor is garbage */ + }, { USB_DEVICE(0x0e8d, 0x0003), /* FIREFLY, MediaTek Inc; andrey.arapov@gmail.com */ .driver_info = NO_UNION_NORMAL, /* has no union descriptor */ }, @@ -2002,6 +2038,23 @@ static const struct usb_device_id acm_ids[] = { .driver_info = IGNORE_DEVICE, }, + /* CH343 supports CAP_BRK, but doesn't advertise it */ + { USB_DEVICE(0x1a86, 0x55d3), .driver_info = MISSING_CAP_BRK, }, + + /* + * Lenovo Yoga Book 9 14IAH10 (83KJ) — INGENIC 17EF:6161 touchscreen + * composite device. The CDC ACM control interface (0) uses a standard + * Union descriptor, but the data interface (1) is declared as vendor- + * specific class (0xff) with no CDC data descriptors, so cdc-acm would + * normally reject it. The firmware also requires continuous polling of + * the notification endpoint (EP 0x82) to suppress a 20-second watchdog + * reset; ALWAYS_POLL_CTRL keeps the ctrlurb active even when no TTY is + * open. Match only the control interface by class to avoid probing the + * vendor-specific data interface. + */ + { USB_DEVICE_INTERFACE_CLASS(0x17ef, 0x6161, USB_CLASS_COMM), + .driver_info = VENDOR_CLASS_DATA_IFACE | ALWAYS_POLL_CTRL }, + /* control interfaces without any protocol set */ { USB_INTERFACE_INFO(USB_CLASS_COMM, USB_CDC_SUBCLASS_ACM, USB_CDC_PROTO_NONE) }, diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h index 759ac15631d3e..25fd5329a8781 100644 --- a/drivers/usb/class/cdc-acm.h +++ b/drivers/usb/class/cdc-acm.h @@ -113,3 +113,5 @@ struct acm { #define CLEAR_HALT_CONDITIONS BIT(5) #define SEND_ZERO_PACKET BIT(6) #define DISABLE_ECHO BIT(7) +#define MISSING_CAP_BRK BIT(8) +#define NO_UNION_12 BIT(9) diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c index ecd6d1f39e498..92567324c5da7 100644 --- a/drivers/usb/class/cdc-wdm.c +++ b/drivers/usb/class/cdc-wdm.c @@ -225,7 +225,8 @@ static void wdm_in_callback(struct urb *urb) /* we may already be in overflow */ if (!test_bit(WDM_OVERFLOW, &desc->flags)) { memmove(desc->ubuf + desc->length, desc->inbuf, length); - desc->length += length; + smp_wmb(); /* against wdm_read() */ + WRITE_ONCE(desc->length, desc->length + length); } } skip_error: @@ -533,6 +534,7 @@ static ssize_t wdm_read return -ERESTARTSYS; cntr = READ_ONCE(desc->length); + smp_rmb(); /* against wdm_in_callback() */ if (cntr == 0) { desc->read = 0; retry: diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c index 75de29725a450..49459e3d34d6e 100644 --- a/drivers/usb/class/usbtmc.c +++ b/drivers/usb/class/usbtmc.c @@ -254,6 +254,9 @@ static int usbtmc_release(struct inode *inode, struct file *file) list_del(&file_data->file_elem); spin_unlock_irq(&file_data->data->dev_lock); + + /* flush anchored URBs */ + usbtmc_draw_down(file_data); mutex_unlock(&file_data->data->io_mutex); kref_put(&file_data->data->kref, usbtmc_delete); @@ -727,7 +730,7 @@ static int usbtmc488_ioctl_trigger(struct usbtmc_file_data *file_data) buffer[1] = data->bTag; buffer[2] = ~data->bTag; - retval = usb_bulk_msg(data->usb_dev, + retval = usb_bulk_msg_killable(data->usb_dev, usb_sndbulkpipe(data->usb_dev, data->bulk_out), buffer, USBTMC_HEADER_SIZE, @@ -1347,7 +1350,7 @@ static int send_request_dev_dep_msg_in(struct usbtmc_file_data *file_data, buffer[11] = 0; /* Reserved */ /* Send bulk URB */ - retval = usb_bulk_msg(data->usb_dev, + retval = usb_bulk_msg_killable(data->usb_dev, usb_sndbulkpipe(data->usb_dev, data->bulk_out), buffer, USBTMC_HEADER_SIZE, @@ -1419,7 +1422,7 @@ static ssize_t usbtmc_read(struct file *filp, char __user *buf, actual = 0; /* Send bulk URB */ - retval = usb_bulk_msg(data->usb_dev, + retval = usb_bulk_msg_killable(data->usb_dev, usb_rcvbulkpipe(data->usb_dev, data->bulk_in), buffer, bufsize, &actual, diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c index 4a2ee447b2137..d895cf6532a21 100644 --- a/drivers/usb/common/ulpi.c +++ b/drivers/usb/common/ulpi.c @@ -331,10 +331,9 @@ struct ulpi *ulpi_register_interface(struct device *dev, ulpi->ops = ops; ret = ulpi_register(dev, ulpi); - if (ret) { - kfree(ulpi); + if (ret) return ERR_PTR(ret); - } + return ulpi; } diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c index baf5bc844b6ff..3067e18ec4d8a 100644 --- a/drivers/usb/core/config.c +++ b/drivers/usb/core/config.c @@ -927,7 +927,11 @@ int usb_get_configuration(struct usb_device *dev) dev->descriptor.bNumConfigurations = ncfg = USB_MAXCONFIG; } - if (ncfg < 1) { + if (ncfg < 1 && dev->quirks & USB_QUIRK_FORCE_ONE_CONFIG) { + dev_info(ddev, "Device claims zero configurations, forcing to 1\n"); + dev->descriptor.bNumConfigurations = 1; + ncfg = 1; + } else if (ncfg < 1) { dev_err(ddev, "no configurations\n"); return -EINVAL; } @@ -1040,6 +1044,11 @@ int usb_get_bos_descriptor(struct usb_device *dev) __u8 cap_type; int ret; + if (dev->quirks & USB_QUIRK_NO_BOS) { + dev_dbg(ddev, "skipping BOS descriptor\n"); + return -ENOMSG; + } + bos = kzalloc(sizeof(*bos), GFP_KERNEL); if (!bos) return -ENOMEM; diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index d29edc7c616a2..74b8bdc27dbf5 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -1415,14 +1415,16 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) int status = 0; int i = 0, n = 0; struct usb_interface *intf; + bool offload_active = false; if (udev->state == USB_STATE_NOTATTACHED || udev->state == USB_STATE_SUSPENDED) goto done; + usb_offload_set_pm_locked(udev, true); if (msg.event == PM_EVENT_SUSPEND && usb_offload_check(udev)) { dev_dbg(&udev->dev, "device offloaded, skip suspend.\n"); - udev->offload_at_suspend = 1; + offload_active = true; } /* Suspend all the interfaces and then udev itself */ @@ -1436,8 +1438,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) * interrupt urbs, allowing interrupt events to be * handled during system suspend. */ - if (udev->offload_at_suspend && - intf->needs_remote_wakeup) { + if (offload_active && intf->needs_remote_wakeup) { dev_dbg(&intf->dev, "device offloaded, skip suspend.\n"); continue; @@ -1452,7 +1453,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } } if (status == 0) { - if (!udev->offload_at_suspend) + if (!offload_active) status = usb_suspend_device(udev, msg); /* @@ -1498,7 +1499,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) */ } else { udev->can_submit = 0; - if (!udev->offload_at_suspend) { + if (!offload_active) { for (i = 0; i < 16; ++i) { usb_hcd_flush_endpoint(udev, udev->ep_out[i]); usb_hcd_flush_endpoint(udev, udev->ep_in[i]); @@ -1507,6 +1508,8 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } done: + if (status != 0) + usb_offload_set_pm_locked(udev, false); dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status); return status; } @@ -1536,16 +1539,19 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) int status = 0; int i; struct usb_interface *intf; + bool offload_active = false; if (udev->state == USB_STATE_NOTATTACHED) { status = -ENODEV; goto done; } udev->can_submit = 1; + if (msg.event == PM_EVENT_RESUME) + offload_active = usb_offload_check(udev); /* Resume the device */ if (udev->state == USB_STATE_SUSPENDED || udev->reset_resume) { - if (!udev->offload_at_suspend) + if (!offload_active) status = usb_resume_device(udev, msg); else dev_dbg(&udev->dev, @@ -1562,8 +1568,7 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) * pending interrupt urbs, allowing interrupt events * to be handled during system suspend. */ - if (udev->offload_at_suspend && - intf->needs_remote_wakeup) { + if (offload_active && intf->needs_remote_wakeup) { dev_dbg(&intf->dev, "device offloaded, skip resume.\n"); continue; @@ -1572,11 +1577,11 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) udev->reset_resume); } } - udev->offload_at_suspend = 0; usb_mark_last_busy(udev); done: dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status); + usb_offload_set_pm_locked(udev, false); if (!status) udev->reset_resume = 0; return status; diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c index d2b2787be4092..43522f1d6b2ba 100644 --- a/drivers/usb/core/message.c +++ b/drivers/usb/core/message.c @@ -42,16 +42,19 @@ static void usb_api_blocking_completion(struct urb *urb) /* - * Starts urb and waits for completion or timeout. Note that this call - * is NOT interruptible. Many device driver i/o requests should be - * interruptible and therefore these drivers should implement their - * own interruptible routines. + * Starts urb and waits for completion or timeout. + * Whether or not the wait is killable depends on the flag passed in. + * For example, compare usb_bulk_msg() and usb_bulk_msg_killable(). + * + * For non-killable waits, we enforce a maximum limit on the timeout value. */ -static int usb_start_wait_urb(struct urb *urb, int timeout, int *actual_length) +static int usb_start_wait_urb(struct urb *urb, int timeout, int *actual_length, + bool killable) { struct api_context ctx; unsigned long expire; int retval; + long rc; init_completion(&ctx.done); urb->context = &ctx; @@ -60,13 +63,24 @@ static int usb_start_wait_urb(struct urb *urb, int timeout, int *actual_length) if (unlikely(retval)) goto out; - expire = timeout ? msecs_to_jiffies(timeout) : MAX_SCHEDULE_TIMEOUT; - if (!wait_for_completion_timeout(&ctx.done, expire)) { + if (!killable && (timeout <= 0 || timeout > USB_MAX_SYNCHRONOUS_TIMEOUT)) + timeout = USB_MAX_SYNCHRONOUS_TIMEOUT; + expire = (timeout > 0) ? msecs_to_jiffies(timeout) : MAX_SCHEDULE_TIMEOUT; + if (killable) + rc = wait_for_completion_killable_timeout(&ctx.done, expire); + else + rc = wait_for_completion_timeout(&ctx.done, expire); + if (rc <= 0) { usb_kill_urb(urb); - retval = (ctx.status == -ENOENT ? -ETIMEDOUT : ctx.status); + if (ctx.status != -ENOENT) + retval = ctx.status; + else if (rc == 0) + retval = -ETIMEDOUT; + else + retval = rc; dev_dbg(&urb->dev->dev, - "%s timed out on ep%d%s len=%u/%u\n", + "%s timed out or killed on ep%d%s len=%u/%u\n", current->comm, usb_endpoint_num(&urb->ep->desc), usb_urb_dir_in(urb) ? "in" : "out", @@ -100,7 +114,7 @@ static int usb_internal_control_msg(struct usb_device *usb_dev, usb_fill_control_urb(urb, usb_dev, pipe, (unsigned char *)cmd, data, len, usb_api_blocking_completion, NULL); - retv = usb_start_wait_urb(urb, timeout, &length); + retv = usb_start_wait_urb(urb, timeout, &length, false); if (retv < 0) return retv; else @@ -117,8 +131,7 @@ static int usb_internal_control_msg(struct usb_device *usb_dev, * @index: USB message index value * @data: pointer to the data to send * @size: length in bytes of the data to send - * @timeout: time in msecs to wait for the message to complete before timing - * out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * * Context: task context, might sleep. * @@ -173,8 +186,7 @@ EXPORT_SYMBOL_GPL(usb_control_msg); * @index: USB message index value * @driver_data: pointer to the data to send * @size: length in bytes of the data to send - * @timeout: time in msecs to wait for the message to complete before timing - * out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * @memflags: the flags for memory allocation for buffers * * Context: !in_interrupt () @@ -232,8 +244,7 @@ EXPORT_SYMBOL_GPL(usb_control_msg_send); * @index: USB message index value * @driver_data: pointer to the data to be filled in by the message * @size: length in bytes of the data to be received - * @timeout: time in msecs to wait for the message to complete before timing - * out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * @memflags: the flags for memory allocation for buffers * * Context: !in_interrupt () @@ -304,8 +315,7 @@ EXPORT_SYMBOL_GPL(usb_control_msg_recv); * @len: length in bytes of the data to send * @actual_length: pointer to a location to put the actual length transferred * in bytes - * @timeout: time in msecs to wait for the message to complete before - * timing out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * * Context: task context, might sleep. * @@ -337,8 +347,7 @@ EXPORT_SYMBOL_GPL(usb_interrupt_msg); * @len: length in bytes of the data to send * @actual_length: pointer to a location to put the actual length transferred * in bytes - * @timeout: time in msecs to wait for the message to complete before - * timing out (if 0 the wait is forever) + * @timeout: time in msecs to wait for the message to complete before timing out * * Context: task context, might sleep. * @@ -385,10 +394,59 @@ int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe, usb_fill_bulk_urb(urb, usb_dev, pipe, data, len, usb_api_blocking_completion, NULL); - return usb_start_wait_urb(urb, timeout, actual_length); + return usb_start_wait_urb(urb, timeout, actual_length, false); } EXPORT_SYMBOL_GPL(usb_bulk_msg); +/** + * usb_bulk_msg_killable - Builds a bulk urb, sends it off and waits for completion in a killable state + * @usb_dev: pointer to the usb device to send the message to + * @pipe: endpoint "pipe" to send the message to + * @data: pointer to the data to send + * @len: length in bytes of the data to send + * @actual_length: pointer to a location to put the actual length transferred + * in bytes + * @timeout: time in msecs to wait for the message to complete before + * timing out (if <= 0, the wait is as long as possible) + * + * Context: task context, might sleep. + * + * This function is just like usb_blk_msg(), except that it waits in a + * killable state and there is no limit on the timeout length. + * + * Return: + * If successful, 0. Otherwise a negative error number. The number of actual + * bytes transferred will be stored in the @actual_length parameter. + * + */ +int usb_bulk_msg_killable(struct usb_device *usb_dev, unsigned int pipe, + void *data, int len, int *actual_length, int timeout) +{ + struct urb *urb; + struct usb_host_endpoint *ep; + + ep = usb_pipe_endpoint(usb_dev, pipe); + if (!ep || len < 0) + return -EINVAL; + + urb = usb_alloc_urb(0, GFP_KERNEL); + if (!urb) + return -ENOMEM; + + if ((ep->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) == + USB_ENDPOINT_XFER_INT) { + pipe = (pipe & ~(3 << 30)) | (PIPE_INTERRUPT << 30); + usb_fill_int_urb(urb, usb_dev, pipe, data, len, + usb_api_blocking_completion, NULL, + ep->desc.bInterval); + } else + usb_fill_bulk_urb(urb, usb_dev, pipe, data, len, + usb_api_blocking_completion, NULL); + + return usb_start_wait_urb(urb, timeout, actual_length, true); +} +EXPORT_SYMBOL_GPL(usb_bulk_msg_killable); + /*-------------------------------------------------------------------*/ static void sg_clean(struct usb_sg_request *io) @@ -2431,7 +2489,7 @@ int cdc_parse_cdc_header(struct usb_cdc_parsed_header *hdr, break; case USB_CDC_MBIM_EXTENDED_TYPE: if (elength < sizeof(struct usb_cdc_mbim_extended_desc)) - break; + goto next_desc; hdr->usb_cdc_mbim_extended_desc = (struct usb_cdc_mbim_extended_desc *)buffer; break; diff --git a/drivers/usb/core/offload.c b/drivers/usb/core/offload.c index 7c699f1b8d2b7..9db3cfedd29c7 100644 --- a/drivers/usb/core/offload.c +++ b/drivers/usb/core/offload.c @@ -25,33 +25,30 @@ */ int usb_offload_get(struct usb_device *udev) { - int ret; + int ret = 0; - usb_lock_device(udev); - if (udev->state == USB_STATE_NOTATTACHED) { - usb_unlock_device(udev); + if (!usb_get_dev(udev)) return -ENODEV; - } - if (udev->state == USB_STATE_SUSPENDED || - udev->offload_at_suspend) { - usb_unlock_device(udev); - return -EBUSY; + if (pm_runtime_get_if_active(&udev->dev) != 1) { + ret = -EBUSY; + goto err_rpm; } - /* - * offload_usage could only be modified when the device is active, since - * it will alter the suspend flow of the device. - */ - ret = usb_autoresume_device(udev); - if (ret < 0) { - usb_unlock_device(udev); - return ret; + spin_lock(&udev->offload_lock); + + if (udev->offload_pm_locked) { + ret = -EAGAIN; + goto err; } udev->offload_usage++; - usb_autosuspend_device(udev); - usb_unlock_device(udev); + +err: + spin_unlock(&udev->offload_lock); + pm_runtime_put_autosuspend(&udev->dev); +err_rpm: + usb_put_dev(udev); return ret; } @@ -69,35 +66,32 @@ EXPORT_SYMBOL_GPL(usb_offload_get); */ int usb_offload_put(struct usb_device *udev) { - int ret; + int ret = 0; - usb_lock_device(udev); - if (udev->state == USB_STATE_NOTATTACHED) { - usb_unlock_device(udev); + if (!usb_get_dev(udev)) return -ENODEV; - } - if (udev->state == USB_STATE_SUSPENDED || - udev->offload_at_suspend) { - usb_unlock_device(udev); - return -EBUSY; + if (pm_runtime_get_if_active(&udev->dev) != 1) { + ret = -EBUSY; + goto err_rpm; } - /* - * offload_usage could only be modified when the device is active, since - * it will alter the suspend flow of the device. - */ - ret = usb_autoresume_device(udev); - if (ret < 0) { - usb_unlock_device(udev); - return ret; + spin_lock(&udev->offload_lock); + + if (udev->offload_pm_locked) { + ret = -EAGAIN; + goto err; } /* Drop the count when it wasn't 0, ignore the operation otherwise. */ if (udev->offload_usage) udev->offload_usage--; - usb_autosuspend_device(udev); - usb_unlock_device(udev); + +err: + spin_unlock(&udev->offload_lock); + pm_runtime_put_autosuspend(&udev->dev); +err_rpm: + usb_put_dev(udev); return ret; } @@ -112,25 +106,47 @@ EXPORT_SYMBOL_GPL(usb_offload_put); * management. * * The caller must hold @udev's device lock. In addition, the caller should - * ensure downstream usb devices are all either suspended or marked as - * "offload_at_suspend" to ensure the correctness of the return value. + * ensure the device itself and the downstream usb devices are all marked as + * "offload_pm_locked" to ensure the correctness of the return value. * * Returns true on any offload activity, false otherwise. */ bool usb_offload_check(struct usb_device *udev) __must_hold(&udev->dev->mutex) { struct usb_device *child; - bool active; + bool active = false; int port1; + if (udev->offload_usage) + return true; + usb_hub_for_each_child(udev, port1, child) { usb_lock_device(child); active = usb_offload_check(child); usb_unlock_device(child); + if (active) - return true; + break; } - return !!udev->offload_usage; + return active; } EXPORT_SYMBOL_GPL(usb_offload_check); + +/** + * usb_offload_set_pm_locked - set the PM lock state of a USB device + * @udev: the USB device to modify + * @locked: the new lock state + * + * Setting @locked to true prevents offload_usage from being modified. This + * ensures that offload activities cannot be started or stopped during critical + * power management transitions, maintaining a stable state for the duration + * of the transition. + */ +void usb_offload_set_pm_locked(struct usb_device *udev, bool locked) +{ + spin_lock(&udev->offload_lock); + udev->offload_pm_locked = locked; + spin_unlock(&udev->offload_lock); +} +EXPORT_SYMBOL_GPL(usb_offload_set_pm_locked); diff --git a/drivers/usb/core/phy.c b/drivers/usb/core/phy.c index faa20054ad5a1..4d966cc9cdc95 100644 --- a/drivers/usb/core/phy.c +++ b/drivers/usb/core/phy.c @@ -114,7 +114,7 @@ EXPORT_SYMBOL_GPL(usb_phy_roothub_alloc); struct usb_phy_roothub *usb_phy_roothub_alloc_usb3_phy(struct device *dev) { struct usb_phy_roothub *phy_roothub; - int num_phys; + int num_phys, usb2_phy_index; if (!IS_ENABLED(CONFIG_GENERIC_PHY)) return NULL; @@ -124,6 +124,16 @@ struct usb_phy_roothub *usb_phy_roothub_alloc_usb3_phy(struct device *dev) if (num_phys <= 0) return NULL; + /* + * If 'usb2-phy' is not present, usb_phy_roothub_alloc() added + * all PHYs to the primary HCD's phy_roothub already, so skip + * adding 'usb3-phy' here to avoid double use of that. + */ + usb2_phy_index = of_property_match_string(dev->of_node, "phy-names", + "usb2-phy"); + if (usb2_phy_index < 0) + return NULL; + phy_roothub = devm_kzalloc(dev, sizeof(*phy_roothub), GFP_KERNEL); if (!phy_roothub) return ERR_PTR(-ENOMEM); @@ -200,16 +210,10 @@ int usb_phy_roothub_set_mode(struct usb_phy_roothub *phy_roothub, list_for_each_entry(roothub_entry, head, list) { err = phy_set_mode(roothub_entry->phy, mode); if (err) - goto err_out; + return err; } return 0; - -err_out: - list_for_each_entry_continue_reverse(roothub_entry, head, list) - phy_power_off(roothub_entry->phy); - - return err; } EXPORT_SYMBOL_GPL(usb_phy_roothub_set_mode); diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index f54198171b6a3..a47df5d32f7c4 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -141,6 +141,7 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, usb_disconnect(&port_dev->child); rc = usb_hub_set_port_power(hdev, hub, port1, !disabled); + msleep(2 * hub_power_on_good_delay(hub)); if (disabled) { usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION); diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index 47f589c4104a3..34b1f7df3529a 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -141,6 +141,8 @@ static int quirks_param_set(const char *value, const struct kernel_param *kp) case 'p': flags |= USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT; break; + case 'q': + flags |= USB_QUIRK_FORCE_ONE_CONFIG; /* Ignore unrecognized flag characters */ } } @@ -208,6 +210,10 @@ static const struct usb_device_id usb_quirk_list[] = { /* HP v222w 16GB Mini USB Drive */ { USB_DEVICE(0x03f0, 0x3f40), .driver_info = USB_QUIRK_DELAY_INIT }, + /* Huawei 4G LTE module ME906S */ + { USB_DEVICE(0x03f0, 0xa31d), .driver_info = + USB_QUIRK_DISCONNECT_SUSPEND }, + /* Creative SB Audigy 2 NX */ { USB_DEVICE(0x041e, 0x3020), .driver_info = USB_QUIRK_RESET_RESUME }, @@ -377,6 +383,9 @@ static const struct usb_device_id usb_quirk_list[] = { /* SanDisk Extreme 55AE */ { USB_DEVICE(0x0781, 0x55ae), .driver_info = USB_QUIRK_NO_LPM }, + /* Avermedia Live Gamer Ultra 2.1 (GC553G2) - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x07ca, 0x2553), .driver_info = USB_QUIRK_NO_BOS }, + /* Realforce 87U Keyboard */ { USB_DEVICE(0x0853, 0x011b), .driver_info = USB_QUIRK_NO_LPM }, @@ -393,6 +402,7 @@ static const struct usb_device_id usb_quirk_list[] = { /* Silicon Motion Flash Drive */ { USB_DEVICE(0x090c, 0x1000), .driver_info = USB_QUIRK_DELAY_INIT }, + { USB_DEVICE(0x090c, 0x2000), .driver_info = USB_QUIRK_DELAY_INIT }, /* Sound Devices USBPre2 */ { USB_DEVICE(0x0926, 0x0202), .driver_info = @@ -437,6 +447,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x0b05, 0x17e0), .driver_info = USB_QUIRK_IGNORE_REMOTE_WAKEUP }, + /* ASUS TUF 4K PRO - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x0b05, 0x1ab9), .driver_info = USB_QUIRK_NO_BOS }, + /* Realtek Semiconductor Corp. Mass Storage Device (Multicard Reader)*/ { USB_DEVICE(0x0bda, 0x0151), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS }, @@ -450,6 +463,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x0c45, 0x7056), .driver_info = USB_QUIRK_IGNORE_REMOTE_WAKEUP }, + /* Elgato 4K X - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x0fd9, 0x009b), .driver_info = USB_QUIRK_NO_BOS }, + /* Sony Xperia XZ1 Compact (lilac) smartphone in fastboot mode */ { USB_DEVICE(0x0fce, 0x0dde), .driver_info = USB_QUIRK_NO_LPM }, @@ -478,6 +494,8 @@ static const struct usb_device_id usb_quirk_list[] = { /* Razer - Razer Blade Keyboard */ { USB_DEVICE(0x1532, 0x0116), .driver_info = USB_QUIRK_LINEAR_UFRAME_INTR_BINTERVAL }, + /* Razer - Razer Kiyo Pro Webcam */ + { USB_DEVICE(0x1532, 0x0e05), .driver_info = USB_QUIRK_NO_LPM }, /* Lenovo ThinkPad OneLink+ Dock twin hub controllers (VIA Labs VL812) */ { USB_DEVICE(0x17ef, 0x1018), .driver_info = USB_QUIRK_RESET_RESUME }, @@ -562,6 +580,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x2386, 0x350e), .driver_info = USB_QUIRK_NO_LPM }, + /* UGREEN 35871 - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x2b89, 0x5871), .driver_info = USB_QUIRK_NO_BOS }, + /* APTIV AUTOMOTIVE HUB */ { USB_DEVICE(0x2c48, 0x0132), .driver_info = USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT }, @@ -572,12 +593,18 @@ static const struct usb_device_id usb_quirk_list[] = { /* Alcor Link AK9563 SC Reader used in 2022 Lenovo ThinkPads */ { USB_DEVICE(0x2ce3, 0x9563), .driver_info = USB_QUIRK_NO_LPM }, + /* ezcap401 - BOS descriptor fetch hangs at SuperSpeed Plus */ + { USB_DEVICE(0x32ed, 0x0401), .driver_info = USB_QUIRK_NO_BOS }, + /* DELL USB GEN2 */ { USB_DEVICE(0x413c, 0xb062), .driver_info = USB_QUIRK_NO_LPM | USB_QUIRK_RESET_RESUME }, /* VCOM device */ { USB_DEVICE(0x4296, 0x7570), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS }, + /* Noji-MCS SmartCard Reader */ + { USB_DEVICE(0x5131, 0x2007), .driver_info = USB_QUIRK_FORCE_ONE_CONFIG }, + /* INTEL VALUE SSD */ { USB_DEVICE(0x8086, 0xf1a5), .driver_info = USB_QUIRK_RESET_RESUME }, diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c index b6b0b84895237..048cf28efd4ee 100644 --- a/drivers/usb/core/usb.c +++ b/drivers/usb/core/usb.c @@ -670,6 +670,7 @@ struct usb_device *usb_alloc_dev(struct usb_device *parent, set_dev_node(&dev->dev, dev_to_node(bus->sysdev)); dev->state = USB_STATE_ATTACHED; dev->lpm_disable_count = 1; + spin_lock_init(&dev->offload_lock); dev->offload_usage = 0; atomic_set(&dev->urbnum, 0); diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c index c3d24312db0fe..f375c5185bfe2 100644 --- a/drivers/usb/dwc2/core.c +++ b/drivers/usb/dwc2/core.c @@ -578,6 +578,7 @@ void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg) { switch (hsotg->dr_mode) { case USB_DR_MODE_HOST: + dwc2_force_mode(hsotg, true); /* * NOTE: This is required for some rockchip soc based * platforms on their host-only dwc2. diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 0637bfbc054e2..a6c2b8de39083 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -4607,7 +4607,9 @@ static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget) /* Exit clock gating when driver is stopped. */ if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_NONE && hsotg->bus_suspended && !hsotg->params.no_clock_gating) { + spin_lock_irqsave(&hsotg->lock, flags); dwc2_gadget_exit_clock_gating(hsotg, 0); + spin_unlock_irqrestore(&hsotg->lock, flags); } /* all endpoints should be shutdown */ diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c index 3f83ecc9fc236..ef0d730770347 100644 --- a/drivers/usb/dwc2/platform.c +++ b/drivers/usb/dwc2/platform.c @@ -369,11 +369,11 @@ static void dwc2_driver_shutdown(struct platform_device *dev) { struct dwc2_hsotg *hsotg = platform_get_drvdata(dev); - dwc2_disable_global_interrupts(hsotg); - synchronize_irq(hsotg->irq); - - if (hsotg->ll_hw_enabled) + if (hsotg->ll_hw_enabled) { + dwc2_disable_global_interrupts(hsotg); + synchronize_irq(hsotg->irq); dwc2_lowlevel_hw_disable(hsotg); + } } /** @@ -649,9 +649,13 @@ static int dwc2_driver_probe(struct platform_device *dev) static int __maybe_unused dwc2_suspend(struct device *dev) { struct dwc2_hsotg *dwc2 = dev_get_drvdata(dev); - bool is_device_mode = dwc2_is_device_mode(dwc2); + bool is_device_mode; int ret = 0; + if (!dwc2->ll_hw_enabled) + return 0; + + is_device_mode = dwc2_is_device_mode(dwc2); if (is_device_mode) dwc2_hsotg_suspend(dwc2); @@ -728,6 +732,9 @@ static int __maybe_unused dwc2_resume(struct device *dev) struct dwc2_hsotg *dwc2 = dev_get_drvdata(dev); int ret = 0; + if (!dwc2->ll_hw_enabled) + return 0; + if (dwc2->phy_off_for_suspend && dwc2->ll_hw_enabled) { ret = __dwc2_lowlevel_hw_enable(dwc2); if (ret) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index c2ce2f5e60a19..c6b7df5682b4e 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -991,6 +991,8 @@ static bool dwc3_core_is_valid(struct dwc3 *dwc) reg = dwc3_readl(dwc->regs, DWC3_GSNPSID); dwc->ip = DWC3_GSNPS_ID(reg); + if (dwc->ip == DWC4_IP) + dwc->ip = DWC32_IP; /* This should read as U3 followed by revision number */ if (DWC3_IP_IS(DWC3)) { @@ -2147,6 +2149,20 @@ static int dwc3_get_num_ports(struct dwc3 *dwc) return 0; } +static void dwc3_vbus_draw_work(struct work_struct *work) +{ + struct dwc3 *dwc = container_of(work, struct dwc3, vbus_draw_work); + union power_supply_propval val = {0}; + int ret; + + val.intval = 1000 * (dwc->current_limit); + ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); + + if (ret < 0) + dev_dbg(dwc->dev, "Error (%d) setting vbus draw (%d mA)\n", + ret, dwc->current_limit); +} + static struct power_supply *dwc3_get_usb_power_supply(struct dwc3 *dwc) { struct power_supply *usb_psy; @@ -2161,6 +2177,7 @@ static struct power_supply *dwc3_get_usb_power_supply(struct dwc3 *dwc) if (!usb_psy) return ERR_PTR(-EPROBE_DEFER); + INIT_WORK(&dwc->vbus_draw_work, dwc3_vbus_draw_work); return usb_psy; } @@ -2384,8 +2401,10 @@ void dwc3_core_remove(struct dwc3 *dwc) dwc3_free_event_buffers(dwc); - if (dwc->usb_psy) + if (dwc->usb_psy) { + cancel_work_sync(&dwc->vbus_draw_work); power_supply_put(dwc->usb_psy); + } } EXPORT_SYMBOL_GPL(dwc3_core_remove); diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index a5fc92c4ffa3b..9cfc36d4bc259 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1060,6 +1060,8 @@ struct dwc3_glue_ops { * @role_switch_default_mode: default operation mode of controller while * usb role is USB_ROLE_NONE. * @usb_psy: pointer to power supply interface. + * @vbus_draw_work: Work to set the vbus drawing limit + * @current_limit: How much current to draw from vbus, in milliAmperes. * @usb2_phy: pointer to USB2 PHY * @usb3_phy: pointer to USB3 PHY * @usb2_generic_phy: pointer to array of USB2 PHYs @@ -1246,6 +1248,8 @@ struct dwc3 { enum usb_dr_mode role_switch_default_mode; struct power_supply *usb_psy; + struct work_struct vbus_draw_work; + unsigned int current_limit; u32 fladj; u32 ref_clk_per; @@ -1265,6 +1269,7 @@ struct dwc3 { #define DWC3_IP 0x5533 #define DWC31_IP 0x3331 #define DWC32_IP 0x3332 +#define DWC4_IP 0x3430 u32 revision; diff --git a/drivers/usb/dwc3/dwc3-of-simple.c b/drivers/usb/dwc3/dwc3-of-simple.c index a4954a21be930..c116143335d9f 100644 --- a/drivers/usb/dwc3/dwc3-of-simple.c +++ b/drivers/usb/dwc3/dwc3-of-simple.c @@ -70,11 +70,11 @@ static int dwc3_of_simple_probe(struct platform_device *pdev) simple->num_clocks = ret; ret = clk_bulk_prepare_enable(simple->num_clocks, simple->clks); if (ret) - goto err_resetc_assert; + goto err_clk_put_all; ret = of_platform_populate(np, NULL, NULL, dev); if (ret) - goto err_clk_put; + goto err_clk_disable; pm_runtime_set_active(dev); pm_runtime_enable(dev); @@ -82,8 +82,9 @@ static int dwc3_of_simple_probe(struct platform_device *pdev) return 0; -err_clk_put: +err_clk_disable: clk_bulk_disable_unprepare(simple->num_clocks, simple->clks); +err_clk_put_all: clk_bulk_put_all(simple->num_clocks, simple->clks); err_resetc_assert: diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index 8f5faf632a8bf..6110bd96a60e3 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -56,6 +56,7 @@ #define PCI_DEVICE_ID_INTEL_CNPH 0xa36e #define PCI_DEVICE_ID_INTEL_CNPV 0xa3b0 #define PCI_DEVICE_ID_INTEL_RPL 0xa70e +#define PCI_DEVICE_ID_INTEL_NVLH 0xd37f #define PCI_DEVICE_ID_INTEL_PTLH 0xe332 #define PCI_DEVICE_ID_INTEL_PTLH_PCH 0xe37e #define PCI_DEVICE_ID_INTEL_PTLU 0xe432 @@ -448,6 +449,7 @@ static const struct pci_device_id dwc3_pci_id_table[] = { { PCI_DEVICE_DATA(INTEL, CNPH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, CNPV, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, RPL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, NVLH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLH_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLU, &dwc3_pci_intel_swnode) }, diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 5e4997f974ddd..db5e5b77b1eac 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -3123,8 +3123,6 @@ static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g, static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) { struct dwc3 *dwc = gadget_to_dwc(g); - union power_supply_propval val = {0}; - int ret; if (dwc->usb2_phy) return usb_phy_set_power(dwc->usb2_phy, mA); @@ -3132,10 +3130,10 @@ static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) if (!dwc->usb_psy) return -EOPNOTSUPP; - val.intval = 1000 * mA; - ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); + dwc->current_limit = mA; + schedule_work(&dwc->vbus_draw_work); - return ret; + return 0; } /** @@ -4825,7 +4823,7 @@ void dwc3_gadget_exit(struct dwc3 *dwc) if (!dwc->gadget) return; - dwc3_enable_susphy(dwc, false); + dwc3_enable_susphy(dwc, true); usb_del_gadget(dwc->gadget); dwc3_gadget_free_endpoints(dwc); usb_put_gadget(dwc->gadget); diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 1c513bf8002ec..3f075c57605cb 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -37,7 +37,10 @@ static void dwc3_power_off_all_roothub_ports(struct dwc3 *dwc) /* xhci regs are not mapped yet, do it temporarily here */ if (dwc->xhci_resources[0].start) { - xhci_regs = ioremap(dwc->xhci_resources[0].start, DWC3_XHCI_REGS_END); + if (dwc->xhci_resources[0].flags & IORESOURCE_MEM_NONPOSTED) + xhci_regs = ioremap_np(dwc->xhci_resources[0].start, DWC3_XHCI_REGS_END); + else + xhci_regs = ioremap(dwc->xhci_resources[0].start, DWC3_XHCI_REGS_END); if (!xhci_regs) { dev_err(dwc->dev, "Failed to ioremap xhci_regs\n"); return; @@ -223,7 +226,7 @@ void dwc3_host_exit(struct dwc3 *dwc) if (dwc->sys_wakeup) device_init_wakeup(&dwc->xhci->dev, false); - dwc3_enable_susphy(dwc, false); + dwc3_enable_susphy(dwc, true); platform_device_unregister(dwc->xhci); dwc->xhci = NULL; } diff --git a/drivers/usb/gadget/function/f_ecm.c b/drivers/usb/gadget/function/f_ecm.c index 675d2bc538a45..e315f18b7f9f0 100644 --- a/drivers/usb/gadget/function/f_ecm.c +++ b/drivers/usb/gadget/function/f_ecm.c @@ -681,6 +681,7 @@ ecm_bind(struct usb_configuration *c, struct usb_function *f) struct usb_ep *ep; struct f_ecm_opts *ecm_opts; + struct net_device *net __free(detach_gadget) = NULL; struct usb_request *request __free(free_usb_request) = NULL; if (!can_support_ecm(cdev->gadget)) @@ -688,18 +689,18 @@ ecm_bind(struct usb_configuration *c, struct usb_function *f) ecm_opts = container_of(f->fi, struct f_ecm_opts, func_inst); - mutex_lock(&ecm_opts->lock); - - gether_set_gadget(ecm_opts->net, cdev->gadget); - - if (!ecm_opts->bound) { - status = gether_register_netdev(ecm_opts->net); - ecm_opts->bound = true; - } - - mutex_unlock(&ecm_opts->lock); - if (status) - return status; + scoped_guard(mutex, &ecm_opts->lock) + if (ecm_opts->bind_count == 0 && !ecm_opts->bound) { + if (!device_is_registered(&ecm_opts->net->dev)) { + gether_set_gadget(ecm_opts->net, cdev->gadget); + status = gether_register_netdev(ecm_opts->net); + } else + status = gether_attach_gadget(ecm_opts->net, cdev->gadget); + + if (status) + return status; + net = ecm_opts->net; + } ecm_string_defs[1].s = ecm->ethaddr; @@ -790,6 +791,9 @@ ecm_bind(struct usb_configuration *c, struct usb_function *f) ecm->notify_req = no_free_ptr(request); + ecm_opts->bind_count++; + retain_and_null_ptr(net); + DBG(cdev, "CDC Ethernet: IN/%s OUT/%s NOTIFY/%s\n", ecm->port.in_ep->name, ecm->port.out_ep->name, ecm->notify->name); @@ -836,7 +840,7 @@ static void ecm_free_inst(struct usb_function_instance *f) struct f_ecm_opts *opts; opts = container_of(f, struct f_ecm_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -906,9 +910,12 @@ static void ecm_free(struct usb_function *f) static void ecm_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_ecm *ecm = func_to_ecm(f); + struct f_ecm_opts *ecm_opts; DBG(c->cdev, "ecm unbind\n"); + ecm_opts = container_of(f->fi, struct f_ecm_opts, func_inst); + usb_free_all_descriptors(f); if (atomic_read(&ecm->notify_count)) { @@ -918,6 +925,10 @@ static void ecm_unbind(struct usb_configuration *c, struct usb_function *f) kfree(ecm->notify_req->buf); usb_ep_free_request(ecm->notify, ecm->notify_req); + + ecm_opts->bind_count--; + if (ecm_opts->bind_count == 0 && !ecm_opts->bound) + gether_detach_gadget(ecm_opts->net); } static struct usb_function *ecm_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_eem.c b/drivers/usb/gadget/function/f_eem.c index edbbadad61381..36f4295e6d63d 100644 --- a/drivers/usb/gadget/function/f_eem.c +++ b/drivers/usb/gadget/function/f_eem.c @@ -7,6 +7,7 @@ * Copyright (C) 2009 EF Johnson Technologies */ +#include #include #include #include @@ -251,24 +252,22 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f) struct usb_ep *ep; struct f_eem_opts *eem_opts; + struct net_device *net __free(detach_gadget) = NULL; eem_opts = container_of(f->fi, struct f_eem_opts, func_inst); - /* - * in drivers/usb/gadget/configfs.c:configfs_composite_bind() - * configurations are bound in sequence with list_for_each_entry, - * in each configuration its functions are bound in sequence - * with list_for_each_entry, so we assume no race condition - * with regard to eem_opts->bound access - */ - if (!eem_opts->bound) { - mutex_lock(&eem_opts->lock); - gether_set_gadget(eem_opts->net, cdev->gadget); - status = gether_register_netdev(eem_opts->net); - mutex_unlock(&eem_opts->lock); - if (status) - return status; - eem_opts->bound = true; - } + + scoped_guard(mutex, &eem_opts->lock) + if (eem_opts->bind_count == 0 && !eem_opts->bound) { + if (!device_is_registered(&eem_opts->net->dev)) { + gether_set_gadget(eem_opts->net, cdev->gadget); + status = gether_register_netdev(eem_opts->net); + } else + status = gether_attach_gadget(eem_opts->net, cdev->gadget); + + if (status) + return status; + net = eem_opts->net; + } us = usb_gstrings_attach(cdev, eem_strings, ARRAY_SIZE(eem_string_defs)); @@ -279,21 +278,19 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f) /* allocate instance-specific interface IDs */ status = usb_interface_id(c, f); if (status < 0) - goto fail; + return status; eem->ctrl_id = status; eem_intf.bInterfaceNumber = status; - status = -ENODEV; - /* allocate instance-specific endpoints */ ep = usb_ep_autoconfig(cdev->gadget, &eem_fs_in_desc); if (!ep) - goto fail; + return -ENODEV; eem->port.in_ep = ep; ep = usb_ep_autoconfig(cdev->gadget, &eem_fs_out_desc); if (!ep) - goto fail; + return -ENODEV; eem->port.out_ep = ep; /* support all relevant hardware speeds... we expect that when @@ -309,16 +306,14 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f) status = usb_assign_descriptors(f, eem_fs_function, eem_hs_function, eem_ss_function, eem_ss_function); if (status) - goto fail; + return status; + + eem_opts->bind_count++; + retain_and_null_ptr(net); DBG(cdev, "CDC Ethernet (EEM): IN/%s OUT/%s\n", eem->port.in_ep->name, eem->port.out_ep->name); return 0; - -fail: - ERROR(cdev, "%s: can't bind, err %d\n", f->name, status); - - return status; } static void eem_cmd_complete(struct usb_ep *ep, struct usb_request *req) @@ -597,7 +592,7 @@ static void eem_free_inst(struct usb_function_instance *f) struct f_eem_opts *opts; opts = container_of(f, struct f_eem_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -640,9 +635,17 @@ static void eem_free(struct usb_function *f) static void eem_unbind(struct usb_configuration *c, struct usb_function *f) { + struct f_eem_opts *opts; + DBG(c->cdev, "eem unbind\n"); + opts = container_of(f->fi, struct f_eem_opts, func_inst); + usb_free_all_descriptors(f); + + opts->bind_count--; + if (opts->bind_count == 0 && !opts->bound) + gether_detach_gadget(opts->net); } static struct usb_function *eem_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 47cfbe41fdff8..497d20260b040 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -1489,7 +1489,7 @@ static int ffs_dmabuf_attach(struct file *file, int fd) goto err_dmabuf_detach; } - dir = epfile->in ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + dir = epfile->in ? DMA_TO_DEVICE : DMA_FROM_DEVICE; err = ffs_dma_resv_lock(dmabuf, nonblock); if (err) @@ -1619,7 +1619,7 @@ static int ffs_dmabuf_transfer(struct file *file, /* Make sure we don't have writers */ timeout = nonblock ? 0 : msecs_to_jiffies(DMABUF_ENQUEUE_TIMEOUT_MS); retl = dma_resv_wait_timeout(dmabuf->resv, - dma_resv_usage_rw(epfile->in), + dma_resv_usage_rw(!epfile->in), true, timeout); if (retl == 0) retl = -EBUSY; @@ -1664,7 +1664,7 @@ static int ffs_dmabuf_transfer(struct file *file, dma_fence_init(&fence->base, &ffs_dmabuf_fence_ops, &priv->lock, priv->context, seqno); - resv_dir = epfile->in ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ; + resv_dir = epfile->in ? DMA_RESV_USAGE_READ : DMA_RESV_USAGE_WRITE; dma_resv_add_fence(dmabuf->resv, &fence->base, resv_dir); dma_resv_unlock(dmabuf->resv); @@ -1724,10 +1724,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { int fd; - if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) + return -EFAULT; return ffs_dmabuf_attach(file, fd); } @@ -1735,10 +1733,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { int fd; - if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) + return -EFAULT; return ffs_dmabuf_detach(file, fd); } @@ -1746,10 +1742,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { struct usb_ffs_dmabuf_transfer_req req; - if (copy_from_user(&req, (void __user *)value, sizeof(req))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&req, (void __user *)value, sizeof(req))) + return -EFAULT; return ffs_dmabuf_transfer(file, &req); } diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c index 307ea563af95e..1004c577b50e2 100644 --- a/drivers/usb/gadget/function/f_hid.c +++ b/drivers/usb/gadget/function/f_hid.c @@ -106,7 +106,7 @@ struct f_hidg { struct list_head report_list; struct device dev; - struct cdev cdev; + struct cdev *cdev; struct usb_function func; struct usb_ep *in_ep; @@ -749,8 +749,9 @@ static int f_hidg_release(struct inode *inode, struct file *fd) static int f_hidg_open(struct inode *inode, struct file *fd) { + struct kobject *parent = inode->i_cdev->kobj.parent; struct f_hidg *hidg = - container_of(inode->i_cdev, struct f_hidg, cdev); + container_of(parent, struct f_hidg, dev.kobj); fd->private_data = hidg; @@ -1207,9 +1208,11 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) if (!hidg->interval_user_set) { hidg_fs_in_ep_desc.bInterval = 10; hidg_hs_in_ep_desc.bInterval = 4; + hidg_ss_in_ep_desc.bInterval = 4; } else { hidg_fs_in_ep_desc.bInterval = hidg->interval; hidg_hs_in_ep_desc.bInterval = hidg->interval; + hidg_ss_in_ep_desc.bInterval = hidg->interval; } hidg_ss_out_comp_desc.wBytesPerInterval = @@ -1239,9 +1242,11 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) if (!hidg->interval_user_set) { hidg_fs_out_ep_desc.bInterval = 10; hidg_hs_out_ep_desc.bInterval = 4; + hidg_ss_out_ep_desc.bInterval = 4; } else { hidg_fs_out_ep_desc.bInterval = hidg->interval; hidg_hs_out_ep_desc.bInterval = hidg->interval; + hidg_ss_out_ep_desc.bInterval = hidg->interval; } status = usb_assign_descriptors(f, hidg_fs_descriptors_intout, @@ -1258,17 +1263,8 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) if (status) goto fail; - spin_lock_init(&hidg->write_spinlock); hidg->write_pending = 1; hidg->req = NULL; - spin_lock_init(&hidg->read_spinlock); - spin_lock_init(&hidg->get_report_spinlock); - init_waitqueue_head(&hidg->write_queue); - init_waitqueue_head(&hidg->read_queue); - init_waitqueue_head(&hidg->get_queue); - init_waitqueue_head(&hidg->get_id_queue); - INIT_LIST_HEAD(&hidg->completed_out_req); - INIT_LIST_HEAD(&hidg->report_list); INIT_WORK(&hidg->work, get_report_workqueue_handler); hidg->workqueue = alloc_workqueue("report_work", @@ -1282,8 +1278,12 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) } /* create char device */ - cdev_init(&hidg->cdev, &f_hidg_fops); - status = cdev_device_add(&hidg->cdev, &hidg->dev); + hidg->cdev = cdev_alloc(); + if (!hidg->cdev) + goto fail_free_all; + hidg->cdev->ops = &f_hidg_fops; + + status = cdev_device_add(hidg->cdev, &hidg->dev); if (status) goto fail_free_all; @@ -1585,7 +1585,7 @@ static void hidg_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_hidg *hidg = func_to_hidg(f); - cdev_device_del(&hidg->cdev, &hidg->dev); + cdev_device_del(hidg->cdev, &hidg->dev); destroy_workqueue(hidg->workqueue); usb_free_all_descriptors(f); } @@ -1605,6 +1605,16 @@ static struct usb_function *hidg_alloc(struct usb_function_instance *fi) mutex_lock(&opts->lock); + spin_lock_init(&hidg->write_spinlock); + spin_lock_init(&hidg->read_spinlock); + spin_lock_init(&hidg->get_report_spinlock); + init_waitqueue_head(&hidg->write_queue); + init_waitqueue_head(&hidg->read_queue); + init_waitqueue_head(&hidg->get_queue); + init_waitqueue_head(&hidg->get_id_queue); + INIT_LIST_HEAD(&hidg->completed_out_req); + INIT_LIST_HEAD(&hidg->report_list); + device_initialize(&hidg->dev); hidg->dev.release = hidg_release; hidg->dev.class = &hidg_class; diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index 94d478b6bcd3d..6f275c3d11ac5 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -180,6 +180,7 @@ #include #include #include +#include #include #include #include @@ -1853,8 +1854,15 @@ static int check_command_size_in_blocks(struct fsg_common *common, int cmnd_size, enum data_direction data_dir, unsigned int mask, int needs_medium, const char *name) { - if (common->curlun) - common->data_size_from_cmnd <<= common->curlun->blkbits; + if (common->curlun) { + if (check_shl_overflow(common->data_size_from_cmnd, + common->curlun->blkbits, + &common->data_size_from_cmnd)) { + common->phase_error = 1; + return -EINVAL; + } + } + return check_command(common, cmnd_size, data_dir, mask, needs_medium, name); } diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index 0e38330271d5a..04be7047aa3d2 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -1210,8 +1210,8 @@ static int ncm_unwrap_ntb(struct gether *port, block_len = get_ncm(&tmp, opts->block_length); /* (d)wBlockLength */ - if (block_len > ntb_max) { - INFO(port->func.config->cdev, "OUT size exceeded\n"); + if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max)) { + INFO(port->func.config->cdev, "Bad block length: %#X\n", block_len); goto err; } @@ -1439,6 +1439,7 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) struct f_ncm_opts *ncm_opts; struct usb_os_desc_table *os_desc_table __free(kfree) = NULL; + struct net_device *net __free(detach_gadget) = NULL; struct usb_request *request __free(free_usb_request) = NULL; if (!can_support_ecm(cdev->gadget)) @@ -1452,18 +1453,19 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) return -ENOMEM; } - mutex_lock(&ncm_opts->lock); - gether_set_gadget(ncm_opts->net, cdev->gadget); - if (!ncm_opts->bound) { - ncm_opts->net->mtu = (ncm_opts->max_segment_size - ETH_HLEN); - status = gether_register_netdev(ncm_opts->net); - } - mutex_unlock(&ncm_opts->lock); - - if (status) - return status; - - ncm_opts->bound = true; + scoped_guard(mutex, &ncm_opts->lock) + if (ncm_opts->bind_count == 0) { + if (!device_is_registered(&ncm_opts->net->dev)) { + ncm_opts->net->mtu = (ncm_opts->max_segment_size - ETH_HLEN); + gether_set_gadget(ncm_opts->net, cdev->gadget); + status = gether_register_netdev(ncm_opts->net); + } else + status = gether_attach_gadget(ncm_opts->net, cdev->gadget); + + if (status) + return status; + net = ncm_opts->net; + } ncm_string_defs[1].s = ncm->ethaddr; @@ -1564,6 +1566,9 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) } ncm->notify_req = no_free_ptr(request); + ncm_opts->bind_count++; + retain_and_null_ptr(net); + DBG(cdev, "CDC Network: IN/%s OUT/%s NOTIFY/%s\n", ncm->port.in_ep->name, ncm->port.out_ep->name, ncm->notify->name); @@ -1655,7 +1660,7 @@ static void ncm_free_inst(struct usb_function_instance *f) struct f_ncm_opts *opts; opts = container_of(f, struct f_ncm_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -1718,9 +1723,12 @@ static void ncm_free(struct usb_function *f) static void ncm_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_ncm *ncm = func_to_ncm(f); + struct f_ncm_opts *ncm_opts; DBG(c->cdev, "ncm unbind\n"); + ncm_opts = container_of(f->fi, struct f_ncm_opts, func_inst); + hrtimer_cancel(&ncm->task_timer); kfree(f->os_desc_table); @@ -1736,6 +1744,10 @@ static void ncm_unbind(struct usb_configuration *c, struct usb_function *f) kfree(ncm->notify_req->buf); usb_ep_free_request(ncm->notify, ncm->notify_req); + + ncm_opts->bind_count--; + if (ncm_opts->bind_count == 0) + gether_detach_gadget(ncm_opts->net); } static struct usb_function *ncm_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_phonet.c b/drivers/usb/gadget/function/f_phonet.c index 0aa9e8224cae5..a3e11c2011a87 100644 --- a/drivers/usb/gadget/function/f_phonet.c +++ b/drivers/usb/gadget/function/f_phonet.c @@ -333,6 +333,15 @@ static void pn_rx_complete(struct usb_ep *ep, struct usb_request *req) if (unlikely(!skb)) break; + if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { + /* Frame count from host exceeds frags[] capacity */ + dev_kfree_skb_any(skb); + if (fp->rx.skb == skb) + fp->rx.skb = NULL; + dev->stats.rx_length_errors++; + break; + } + if (skb->len == 0) { /* First fragment */ skb->protocol = htons(ETH_P_PHONET); skb_reset_mac_header(skb); diff --git a/drivers/usb/gadget/function/f_rndis.c b/drivers/usb/gadget/function/f_rndis.c index 7451e7cb7a852..ec397291e40b1 100644 --- a/drivers/usb/gadget/function/f_rndis.c +++ b/drivers/usb/gadget/function/f_rndis.c @@ -11,6 +11,7 @@ /* #define VERBOSE_DEBUG */ +#include #include #include #include @@ -665,6 +666,7 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f) struct f_rndis_opts *rndis_opts; struct usb_os_desc_table *os_desc_table __free(kfree) = NULL; + struct net_device *net __free(detach_gadget) = NULL; struct usb_request *request __free(free_usb_request) = NULL; if (!can_support_rndis(c)) @@ -678,23 +680,22 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f) return -ENOMEM; } - rndis_iad_descriptor.bFunctionClass = rndis_opts->class; - rndis_iad_descriptor.bFunctionSubClass = rndis_opts->subclass; - rndis_iad_descriptor.bFunctionProtocol = rndis_opts->protocol; - - /* - * in drivers/usb/gadget/configfs.c:configfs_composite_bind() - * configurations are bound in sequence with list_for_each_entry, - * in each configuration its functions are bound in sequence - * with list_for_each_entry, so we assume no race condition - * with regard to rndis_opts->bound access - */ - if (!rndis_opts->bound) { - gether_set_gadget(rndis_opts->net, cdev->gadget); - status = gether_register_netdev(rndis_opts->net); - if (status) - return status; - rndis_opts->bound = true; + scoped_guard(mutex, &rndis_opts->lock) { + rndis_iad_descriptor.bFunctionClass = rndis_opts->class; + rndis_iad_descriptor.bFunctionSubClass = rndis_opts->subclass; + rndis_iad_descriptor.bFunctionProtocol = rndis_opts->protocol; + + if (rndis_opts->bind_count == 0 && !rndis_opts->borrowed_net) { + if (!device_is_registered(&rndis_opts->net->dev)) { + gether_set_gadget(rndis_opts->net, cdev->gadget); + status = gether_register_netdev(rndis_opts->net); + } else + status = gether_attach_gadget(rndis_opts->net, cdev->gadget); + + if (status) + return status; + net = rndis_opts->net; + } } us = usb_gstrings_attach(cdev, rndis_strings, @@ -793,6 +794,9 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f) } rndis->notify_req = no_free_ptr(request); + rndis_opts->bind_count++; + retain_and_null_ptr(net); + /* NOTE: all that is done without knowing or caring about * the network link ... which is unavailable to this code * until we're activated via set_alt(). @@ -809,11 +813,11 @@ void rndis_borrow_net(struct usb_function_instance *f, struct net_device *net) struct f_rndis_opts *opts; opts = container_of(f, struct f_rndis_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); - opts->borrowed_net = opts->bound = true; + opts->borrowed_net = true; opts->net = net; } EXPORT_SYMBOL_GPL(rndis_borrow_net); @@ -871,7 +875,7 @@ static void rndis_free_inst(struct usb_function_instance *f) opts = container_of(f, struct f_rndis_opts, func_inst); if (!opts->borrowed_net) { - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -940,6 +944,9 @@ static void rndis_free(struct usb_function *f) static void rndis_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_rndis *rndis = func_to_rndis(f); + struct f_rndis_opts *rndis_opts; + + rndis_opts = container_of(f->fi, struct f_rndis_opts, func_inst); kfree(f->os_desc_table); f->os_desc_n = 0; @@ -947,6 +954,10 @@ static void rndis_unbind(struct usb_configuration *c, struct usb_function *f) kfree(rndis->notify_req->buf); usb_ep_free_request(rndis->notify, rndis->notify_req); + + rndis_opts->bind_count--; + if (rndis_opts->bind_count == 0 && !rndis_opts->borrowed_net) + gether_detach_gadget(rndis_opts->net); } static struct usb_function *rndis_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_subset.c b/drivers/usb/gadget/function/f_subset.c index ea3fdd8424629..638e3138f8a0b 100644 --- a/drivers/usb/gadget/function/f_subset.c +++ b/drivers/usb/gadget/function/f_subset.c @@ -6,6 +6,7 @@ * Copyright (C) 2008 Nokia Corporation */ +#include #include #include #include @@ -298,25 +299,22 @@ geth_bind(struct usb_configuration *c, struct usb_function *f) struct usb_ep *ep; struct f_gether_opts *gether_opts; + struct net_device *net __free(detach_gadget) = NULL; gether_opts = container_of(f->fi, struct f_gether_opts, func_inst); - /* - * in drivers/usb/gadget/configfs.c:configfs_composite_bind() - * configurations are bound in sequence with list_for_each_entry, - * in each configuration its functions are bound in sequence - * with list_for_each_entry, so we assume no race condition - * with regard to gether_opts->bound access - */ - if (!gether_opts->bound) { - mutex_lock(&gether_opts->lock); - gether_set_gadget(gether_opts->net, cdev->gadget); - status = gether_register_netdev(gether_opts->net); - mutex_unlock(&gether_opts->lock); - if (status) - return status; - gether_opts->bound = true; - } + scoped_guard(mutex, &gether_opts->lock) + if (gether_opts->bind_count == 0 && !gether_opts->bound) { + if (!device_is_registered(&gether_opts->net->dev)) { + gether_set_gadget(gether_opts->net, cdev->gadget); + status = gether_register_netdev(gether_opts->net); + } else + status = gether_attach_gadget(gether_opts->net, cdev->gadget); + + if (status) + return status; + net = gether_opts->net; + } us = usb_gstrings_attach(cdev, geth_strings, ARRAY_SIZE(geth_string_defs)); @@ -329,20 +327,18 @@ geth_bind(struct usb_configuration *c, struct usb_function *f) /* allocate instance-specific interface IDs */ status = usb_interface_id(c, f); if (status < 0) - goto fail; + return status; subset_data_intf.bInterfaceNumber = status; - status = -ENODEV; - /* allocate instance-specific endpoints */ ep = usb_ep_autoconfig(cdev->gadget, &fs_subset_in_desc); if (!ep) - goto fail; + return -ENODEV; geth->port.in_ep = ep; ep = usb_ep_autoconfig(cdev->gadget, &fs_subset_out_desc); if (!ep) - goto fail; + return -ENODEV; geth->port.out_ep = ep; /* support all relevant hardware speeds... we expect that when @@ -360,21 +356,19 @@ geth_bind(struct usb_configuration *c, struct usb_function *f) status = usb_assign_descriptors(f, fs_eth_function, hs_eth_function, ss_eth_function, ss_eth_function); if (status) - goto fail; + return status; /* NOTE: all that is done without knowing or caring about * the network link ... which is unavailable to this code * until we're activated via set_alt(). */ + gether_opts->bind_count++; + retain_and_null_ptr(net); + DBG(cdev, "CDC Subset: IN/%s OUT/%s\n", geth->port.in_ep->name, geth->port.out_ep->name); return 0; - -fail: - ERROR(cdev, "%s: can't bind, err %d\n", f->name, status); - - return status; } static inline struct f_gether_opts *to_f_gether_opts(struct config_item *item) @@ -417,7 +411,7 @@ static void geth_free_inst(struct usb_function_instance *f) struct f_gether_opts *opts; opts = container_of(f, struct f_gether_opts, func_inst); - if (opts->bound) + if (device_is_registered(&opts->net->dev)) gether_cleanup(netdev_priv(opts->net)); else free_netdev(opts->net); @@ -449,15 +443,28 @@ static struct usb_function_instance *geth_alloc_inst(void) static void geth_free(struct usb_function *f) { struct f_gether *eth; + struct f_gether_opts *opts; + + opts = container_of(f->fi, struct f_gether_opts, func_inst); eth = func_to_geth(f); + scoped_guard(mutex, &opts->lock) + opts->refcnt--; kfree(eth); } static void geth_unbind(struct usb_configuration *c, struct usb_function *f) { + struct f_gether_opts *opts; + + opts = container_of(f->fi, struct f_gether_opts, func_inst); + geth_string_defs[0].id = 0; usb_free_all_descriptors(f); + + opts->bind_count--; + if (opts->bind_count == 0 && !opts->bound) + gether_detach_gadget(opts->net); } static struct usb_function *geth_alloc(struct usb_function_instance *fi) diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index 6e8804f04baa7..7b27f8082ace6 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -1222,6 +1222,13 @@ static void usbg_submit_cmd(struct usbg_cmd *cmd) se_cmd = &cmd->se_cmd; tpg = cmd->fu->tpg; tv_nexus = tpg->tpg_nexus; + if (!tv_nexus) { + struct usb_gadget *gadget = fuas_to_gadget(cmd->fu); + + dev_err(&gadget->dev, "Missing nexus, ignoring command\n"); + return; + } + dir = get_cmd_dir(cmd->cmd_buf); if (dir < 0) goto out; @@ -1482,6 +1489,13 @@ static void bot_cmd_work(struct work_struct *work) se_cmd = &cmd->se_cmd; tpg = cmd->fu->tpg; tv_nexus = tpg->tpg_nexus; + if (!tv_nexus) { + struct usb_gadget *gadget = fuas_to_gadget(cmd->fu); + + dev_err(&gadget->dev, "Missing nexus, ignoring command\n"); + return; + } + dir = get_cmd_dir(cmd->cmd_buf); if (dir < 0) goto out; diff --git a/drivers/usb/gadget/function/f_uac1_legacy.c b/drivers/usb/gadget/function/f_uac1_legacy.c index 49cf5aae90ca3..4981af8337ab8 100644 --- a/drivers/usb/gadget/function/f_uac1_legacy.c +++ b/drivers/usb/gadget/function/f_uac1_legacy.c @@ -360,19 +360,46 @@ static int f_audio_out_ep_complete(struct usb_ep *ep, struct usb_request *req) static void f_audio_complete(struct usb_ep *ep, struct usb_request *req) { struct f_audio *audio = req->context; - int status = req->status; - u32 data = 0; struct usb_ep *out_ep = audio->out_ep; - switch (status) { - - case 0: /* normal completion? */ - if (ep == out_ep) + switch (req->status) { + case 0: + if (ep == out_ep) { f_audio_out_ep_complete(ep, req); - else if (audio->set_con) { - memcpy(&data, req->buf, req->length); - audio->set_con->set(audio->set_con, audio->set_cmd, - le16_to_cpu(data)); + } else if (audio->set_con) { + struct usb_audio_control *con = audio->set_con; + u8 type = con->type; + u32 data; + bool valid_request = false; + + switch (type) { + case UAC_FU_MUTE: { + u8 value; + + if (req->actual == sizeof(value)) { + memcpy(&value, req->buf, sizeof(value)); + data = value; + valid_request = true; + } + break; + } + case UAC_FU_VOLUME: { + __le16 value; + + if (req->actual == sizeof(value)) { + memcpy(&value, req->buf, sizeof(value)); + data = le16_to_cpu(value); + valid_request = true; + } + break; + } + } + + if (valid_request) + con->set(con, audio->set_cmd, data); + else + usb_ep_set_halt(ep); + audio->set_con = NULL; } break; diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index aa6ab666741a9..842187a09cc03 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -362,6 +362,10 @@ uvc_function_set_alt(struct usb_function *f, unsigned interface, unsigned alt) return ret; usb_ep_enable(uvc->video.ep); + uvc->video.max_req_size = uvc->video.ep->maxpacket + * max_t(unsigned int, uvc->video.ep->maxburst, 1) + * (uvc->video.ep->mult); + memset(&v4l2_event, 0, sizeof(v4l2_event)); v4l2_event.type = UVC_EVENT_STREAMON; v4l2_event_queue(&uvc->vdev, &v4l2_event); @@ -409,6 +413,12 @@ uvc_function_disconnect(struct uvc_device *uvc) { int ret; + guard(mutex)(&uvc->lock); + if (uvc->func_unbound) { + dev_dbg(&uvc->vdev.dev, "skipping function deactivate (unbound)\n"); + return; + } + if ((ret = usb_function_deactivate(&uvc->func)) < 0) uvcg_info(&uvc->func, "UVC disconnect failed with %d\n", ret); } @@ -427,6 +437,15 @@ static ssize_t function_name_show(struct device *dev, static DEVICE_ATTR_RO(function_name); +static void uvc_vdev_release(struct video_device *vdev) +{ + struct uvc_device *uvc = video_get_drvdata(vdev); + + /* Signal uvc_function_unbind() that the video device has been released */ + if (uvc->vdev_release_done) + complete(uvc->vdev_release_done); +} + static int uvc_register_video(struct uvc_device *uvc) { @@ -439,7 +458,7 @@ uvc_register_video(struct uvc_device *uvc) uvc->vdev.v4l2_dev->dev = &cdev->gadget->dev; uvc->vdev.fops = &uvc_v4l2_fops; uvc->vdev.ioctl_ops = &uvc_v4l2_ioctl_ops; - uvc->vdev.release = video_device_release_empty; + uvc->vdev.release = uvc_vdev_release; uvc->vdev.vfl_dir = VFL_DIR_TX; uvc->vdev.lock = &uvc->video.mutex; uvc->vdev.device_caps = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING; @@ -655,6 +674,8 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f) int ret = -EINVAL; uvcg_info(f, "%s()\n", __func__); + scoped_guard(mutex, &uvc->lock) + uvc->func_unbound = false; opts = fi_to_f_uvc_opts(f->fi); /* Sanity check the streaming endpoint module parameters. */ @@ -984,12 +1005,19 @@ static void uvc_free(struct usb_function *f) static void uvc_function_unbind(struct usb_configuration *c, struct usb_function *f) { + DECLARE_COMPLETION_ONSTACK(vdev_release_done); struct usb_composite_dev *cdev = c->cdev; struct uvc_device *uvc = to_uvc(f); struct uvc_video *video = &uvc->video; long wait_ret = 1; + bool connected; uvcg_info(f, "%s()\n", __func__); + scoped_guard(mutex, &uvc->lock) { + uvc->func_unbound = true; + uvc->vdev_release_done = &vdev_release_done; + connected = uvc->func_connected; + } kthread_cancel_work_sync(&video->hw_submit); @@ -1002,7 +1030,7 @@ static void uvc_function_unbind(struct usb_configuration *c, * though the video device removal uevent. Allow some time for the * application to close out before things get deleted. */ - if (uvc->func_connected) { + if (connected) { uvcg_dbg(f, "waiting for clean disconnect\n"); wait_ret = wait_event_interruptible_timeout(uvc->func_connected_queue, uvc->func_connected == false, msecs_to_jiffies(500)); @@ -1013,7 +1041,10 @@ static void uvc_function_unbind(struct usb_configuration *c, video_unregister_device(&uvc->vdev); v4l2_device_unregister(&uvc->v4l2_dev); - if (uvc->func_connected) { + scoped_guard(mutex, &uvc->lock) + connected = uvc->func_connected; + + if (connected) { /* * Wait for the release to occur to ensure there are no longer any * pending operations that may cause panics when resources are cleaned @@ -1025,6 +1056,10 @@ static void uvc_function_unbind(struct usb_configuration *c, uvcg_dbg(f, "done waiting for release with ret: %ld\n", wait_ret); } + /* Wait for the video device to be released */ + wait_for_completion(&vdev_release_done); + uvc->vdev_release_done = NULL; + usb_ep_free_request(cdev->gadget->ep0, uvc->control_req); kfree(uvc->control_buf); @@ -1043,6 +1078,8 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi) return ERR_PTR(-ENOMEM); mutex_init(&uvc->video.mutex); + mutex_init(&uvc->lock); + uvc->func_unbound = true; uvc->state = UVC_STATE_DISCONNECTED; init_waitqueue_head(&uvc->func_connected_queue); opts = fi_to_f_uvc_opts(fi); diff --git a/drivers/usb/gadget/function/u_ecm.h b/drivers/usb/gadget/function/u_ecm.h index 77cfb89932bea..7f666b9dea026 100644 --- a/drivers/usb/gadget/function/u_ecm.h +++ b/drivers/usb/gadget/function/u_ecm.h @@ -15,17 +15,26 @@ #include +/** + * struct f_ecm_opts - ECM function options + * @func_inst: USB function instance. + * @net: The net_device associated with the ECM function. + * @bound: True if the net_device is shared and pre-registered during the + * legacy composite driver's bind phase (e.g., multi.c). If false, + * the ECM function will register the net_device during its own + * bind phase. + * @bind_count: Tracks the number of configurations the ECM function is + * bound to, preventing double-registration of the @net device. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_ecm_opts { struct usb_function_instance func_inst; struct net_device *net; bool bound; + int bind_count; - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/u_eem.h b/drivers/usb/gadget/function/u_eem.h index 3bd85dfcd71c8..78ef558152197 100644 --- a/drivers/usb/gadget/function/u_eem.h +++ b/drivers/usb/gadget/function/u_eem.h @@ -15,17 +15,26 @@ #include +/** + * struct f_eem_opts - EEM function options + * @func_inst: USB function instance. + * @net: The net_device associated with the EEM function. + * @bound: True if the net_device is shared and pre-registered during the + * legacy composite driver's bind phase (e.g., multi.c). If false, + * the EEM function will register the net_device during its own + * bind phase. + * @bind_count: Tracks the number of configurations the EEM function is + * bound to, preventing double-registration of the @net device. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_eem_opts { struct usb_function_instance func_inst; struct net_device *net; bool bound; + int bind_count; - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c index f58590bf5e02f..2b824db4d31b7 100644 --- a/drivers/usb/gadget/function/u_ether.c +++ b/drivers/usb/gadget/function/u_ether.c @@ -112,8 +112,10 @@ static void eth_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *p) strscpy(p->driver, "g_ether", sizeof(p->driver)); strscpy(p->version, UETH__VERSION, sizeof(p->version)); - strscpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version)); - strscpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info)); + if (dev->gadget) { + strscpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version)); + strscpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info)); + } } /* REVISIT can also support: @@ -896,6 +898,28 @@ void gether_set_gadget(struct net_device *net, struct usb_gadget *g) } EXPORT_SYMBOL_GPL(gether_set_gadget); +int gether_attach_gadget(struct net_device *net, struct usb_gadget *g) +{ + int ret; + + ret = device_move(&net->dev, &g->dev, DPM_ORDER_DEV_AFTER_PARENT); + if (ret) + return ret; + + gether_set_gadget(net, g); + return 0; +} +EXPORT_SYMBOL_GPL(gether_attach_gadget); + +void gether_detach_gadget(struct net_device *net) +{ + struct eth_dev *dev = netdev_priv(net); + + device_move(&net->dev, NULL, DPM_ORDER_NONE); + dev->gadget = NULL; +} +EXPORT_SYMBOL_GPL(gether_detach_gadget); + int gether_set_dev_addr(struct net_device *net, const char *dev_addr) { struct eth_dev *dev; @@ -1200,6 +1224,11 @@ void gether_disconnect(struct gether *link) DBG(dev, "%s\n", __func__); + spin_lock(&dev->lock); + dev->port_usb = NULL; + link->is_suspend = false; + spin_unlock(&dev->lock); + netif_stop_queue(dev->net); netif_carrier_off(dev->net); @@ -1237,11 +1266,6 @@ void gether_disconnect(struct gether *link) dev->header_len = 0; dev->unwrap = NULL; dev->wrap = NULL; - - spin_lock(&dev->lock); - dev->port_usb = NULL; - link->is_suspend = false; - spin_unlock(&dev->lock); } EXPORT_SYMBOL_GPL(gether_disconnect); diff --git a/drivers/usb/gadget/function/u_ether.h b/drivers/usb/gadget/function/u_ether.h index 34be220cef77c..c85a1cf3c115d 100644 --- a/drivers/usb/gadget/function/u_ether.h +++ b/drivers/usb/gadget/function/u_ether.h @@ -150,6 +150,32 @@ static inline struct net_device *gether_setup_default(void) */ void gether_set_gadget(struct net_device *net, struct usb_gadget *g); +/** + * gether_attach_gadget - Reparent net_device to the gadget device. + * @net: The network device to reparent. + * @g: The target USB gadget device to parent to. + * + * This function moves the network device to be a child of the USB gadget + * device in the device hierarchy. This is typically done when the function + * is bound to a configuration. + * + * Returns 0 on success, or a negative error code on failure. + */ +int gether_attach_gadget(struct net_device *net, struct usb_gadget *g); + +/** + * gether_detach_gadget - Detach net_device from its gadget parent. + * @net: The network device to detach. + * + * This function moves the network device to be a child of the virtual + * devices parent, effectively detaching it from the USB gadget device + * hierarchy. This is typically done when the function is unbound + * from a configuration but the instance is not yet freed. + */ +void gether_detach_gadget(struct net_device *net); + +DEFINE_FREE(detach_gadget, struct net_device *, if (_T) gether_detach_gadget(_T)) + /** * gether_set_dev_addr - initialize an ethernet-over-usb link with eth address * @net: device representing this link diff --git a/drivers/usb/gadget/function/u_gether.h b/drivers/usb/gadget/function/u_gether.h index 2f7a373ed4496..e7b6b51f69c12 100644 --- a/drivers/usb/gadget/function/u_gether.h +++ b/drivers/usb/gadget/function/u_gether.h @@ -15,17 +15,25 @@ #include +/** + * struct f_gether_opts - subset function options + * @func_inst: USB function instance. + * @net: The net_device associated with the subset function. + * @bound: True if the net_device is shared and pre-registered during the + * legacy composite driver's bind phase (e.g., multi.c). If false, + * the subset function will register the net_device during its own + * bind phase. + * @bind_count: Tracks the number of configurations the subset function is + * bound to, preventing double-registration of the @net device. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_gether_opts { struct usb_function_instance func_inst; struct net_device *net; bool bound; - - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ + int bind_count; struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/u_ncm.h b/drivers/usb/gadget/function/u_ncm.h index 49ec095cdb4b6..b1f3db8b68c15 100644 --- a/drivers/usb/gadget/function/u_ncm.h +++ b/drivers/usb/gadget/function/u_ncm.h @@ -18,7 +18,7 @@ struct f_ncm_opts { struct usb_function_instance func_inst; struct net_device *net; - bool bound; + int bind_count; struct config_group *ncm_interf_group; struct usb_os_desc ncm_os_desc; diff --git a/drivers/usb/gadget/function/u_rndis.h b/drivers/usb/gadget/function/u_rndis.h index a8c409b2f52f9..4e64619714dc2 100644 --- a/drivers/usb/gadget/function/u_rndis.h +++ b/drivers/usb/gadget/function/u_rndis.h @@ -15,12 +15,34 @@ #include +/** + * struct f_rndis_opts - RNDIS function options + * @func_inst: USB function instance. + * @vendor_id: Vendor ID. + * @manufacturer: Manufacturer string. + * @net: The net_device associated with the RNDIS function. + * @bind_count: Tracks the number of configurations the RNDIS function is + * bound to, preventing double-registration of the @net device. + * @borrowed_net: True if the net_device is shared and pre-registered during + * the legacy composite driver's bind phase (e.g., multi.c). + * If false, the RNDIS function will register the net_device + * during its own bind phase. + * @rndis_interf_group: ConfigFS group for RNDIS interface. + * @rndis_os_desc: USB OS descriptor for RNDIS. + * @rndis_ext_compat_id: Extended compatibility ID. + * @class: USB class. + * @subclass: USB subclass. + * @protocol: USB protocol. + * @lock: Protects the data from concurrent access by configfs read/write + * and create symlink/remove symlink operations. + * @refcnt: Reference counter for the function instance. + */ struct f_rndis_opts { struct usb_function_instance func_inst; u32 vendor_id; const char *manufacturer; struct net_device *net; - bool bound; + int bind_count; bool borrowed_net; struct config_group *rndis_interf_group; @@ -30,13 +52,6 @@ struct f_rndis_opts { u8 class; u8 subclass; u8 protocol; - - /* - * Read/write access to configfs attributes is handled by configfs. - * - * This is to protect the data from concurrent access by read/write - * and create symlink/remove symlink. - */ struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h index 9e79cbe507157..7abfdd5e1eefb 100644 --- a/drivers/usb/gadget/function/uvc.h +++ b/drivers/usb/gadget/function/uvc.h @@ -107,7 +107,7 @@ struct uvc_video { unsigned int width; unsigned int height; unsigned int imagesize; - unsigned int interval; + unsigned int interval; /* in 100ns units */ struct mutex mutex; /* protects frame parameters */ unsigned int uvc_num_requests; @@ -117,6 +117,7 @@ struct uvc_video { /* Requests */ bool is_enabled; /* tracks whether video stream is enabled */ unsigned int req_size; + unsigned int max_req_size; struct list_head ureqs; /* all uvc_requests allocated by uvc_video */ /* USB requests that the video pump thread can encode into */ @@ -154,6 +155,9 @@ struct uvc_device { enum uvc_state state; struct usb_function func; struct uvc_video video; + struct completion *vdev_release_done; + struct mutex lock; /* protects func_unbound and func_connected */ + bool func_unbound; bool func_connected; wait_queue_head_t func_connected_queue; diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index 9a1bbd79ff5af..21d80322cb614 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -86,10 +86,17 @@ static int uvc_buffer_prepare(struct vb2_buffer *vb) buf->bytesused = 0; } else { buf->bytesused = vb2_get_plane_payload(vb, 0); - buf->req_payload_size = - DIV_ROUND_UP(buf->bytesused + - (video->reqs_per_frame * UVCG_REQUEST_HEADER_LEN), - video->reqs_per_frame); + + if (video->reqs_per_frame != 0) { + buf->req_payload_size = + DIV_ROUND_UP(buf->bytesused + + (video->reqs_per_frame * UVCG_REQUEST_HEADER_LEN), + video->reqs_per_frame); + if (buf->req_payload_size > video->req_size) + buf->req_payload_size = video->req_size; + } else { + buf->req_payload_size = video->max_req_size; + } } return 0; diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c index fd4b998ccd160..23bafb07133f2 100644 --- a/drivers/usb/gadget/function/uvc_v4l2.c +++ b/drivers/usb/gadget/function/uvc_v4l2.c @@ -574,6 +574,8 @@ uvc_v4l2_subscribe_event(struct v4l2_fh *fh, if (sub->type < UVC_EVENT_FIRST || sub->type > UVC_EVENT_LAST) return -EINVAL; + guard(mutex)(&uvc->lock); + if (sub->type == UVC_EVENT_SETUP && uvc->func_connected) return -EBUSY; @@ -595,7 +597,8 @@ static void uvc_v4l2_disable(struct uvc_device *uvc) uvc_function_disconnect(uvc); uvcg_video_disable(&uvc->video); uvcg_free_buffers(&uvc->video.queue); - uvc->func_connected = false; + scoped_guard(mutex, &uvc->lock) + uvc->func_connected = false; wake_up_interruptible(&uvc->func_connected_queue); } diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index fb77b0b217901..f24746e728774 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -499,13 +499,11 @@ uvc_video_prep_requests(struct uvc_video *video) { struct uvc_device *uvc = container_of(video, struct uvc_device, video); struct usb_composite_dev *cdev = uvc->func.config->cdev; - unsigned int interval_duration = video->ep->desc->bInterval * 1250; + unsigned int interval_duration; unsigned int max_req_size, req_size, header_size; unsigned int nreq; - max_req_size = video->ep->maxpacket - * max_t(unsigned int, video->ep->maxburst, 1) - * (video->ep->mult); + max_req_size = video->max_req_size; if (!usb_endpoint_xfer_isoc(video->ep->desc)) { video->req_size = max_req_size; @@ -515,8 +513,11 @@ uvc_video_prep_requests(struct uvc_video *video) return; } + interval_duration = 1 << (video->ep->desc->bInterval - 1); if (cdev->gadget->speed < USB_SPEED_HIGH) - interval_duration = video->ep->desc->bInterval * 10000; + interval_duration *= 10000; + else + interval_duration *= 1250; nreq = DIV_ROUND_UP(video->interval, interval_duration); diff --git a/drivers/usb/gadget/legacy/raw_gadget.c b/drivers/usb/gadget/legacy/raw_gadget.c index b71680c58de6c..46f343ba48b3d 100644 --- a/drivers/usb/gadget/legacy/raw_gadget.c +++ b/drivers/usb/gadget/legacy/raw_gadget.c @@ -40,6 +40,7 @@ MODULE_LICENSE("GPL"); static DEFINE_IDA(driver_id_numbers); #define DRIVER_DRIVER_NAME_LENGTH_MAX 32 +#define USB_RAW_IO_LENGTH_MAX KMALLOC_MAX_SIZE #define RAW_EVENT_QUEUE_SIZE 16 @@ -667,6 +668,8 @@ static void *raw_alloc_io_data(struct usb_raw_ep_io *io, void __user *ptr, return ERR_PTR(-EINVAL); if (!usb_raw_io_flags_valid(io->flags)) return ERR_PTR(-EINVAL); + if (io->length > USB_RAW_IO_LENGTH_MAX) + return ERR_PTR(-EINVAL); if (get_from_user) data = memdup_user(ptr + sizeof(*io), io->length); else { diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c index 5c3d8b64c0e76..f47aac078f6be 100644 --- a/drivers/usb/gadget/udc/bdc/bdc_core.c +++ b/drivers/usb/gadget/udc/bdc/bdc_core.c @@ -35,8 +35,8 @@ static int poll_oip(struct bdc *bdc, u32 usec) u32 status; int ret; - ret = readl_poll_timeout(bdc->regs + BDC_BDCSC, status, - (BDC_CSTS(status) != BDC_OIP), 10, usec); + ret = readl_poll_timeout_atomic(bdc->regs + BDC_BDCSC, status, + (BDC_CSTS(status) != BDC_OIP), 10, usec); if (ret) dev_err(bdc->dev, "operation timedout BDCSC: 0x%08x\n", status); else diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c index 1cefca660773c..da271308d7534 100644 --- a/drivers/usb/gadget/udc/dummy_hcd.c +++ b/drivers/usb/gadget/udc/dummy_hcd.c @@ -462,8 +462,13 @@ static void set_link_state(struct dummy_hcd *dum_hcd) /* Report reset and disconnect events to the driver */ if (dum->ints_enabled && (disconnect || reset)) { - stop_activity(dum); ++dum->callback_usage; + /* + * stop_activity() can drop dum->lock, so it must + * not come between the dum->ints_enabled test + * and the ++dum->callback_usage. + */ + stop_activity(dum); spin_unlock(&dum->lock); if (reset) usb_gadget_udc_reset(&dum->gadget, dum->driver); @@ -908,21 +913,6 @@ static int dummy_pullup(struct usb_gadget *_gadget, int value) spin_lock_irqsave(&dum->lock, flags); dum->pullup = (value != 0); set_link_state(dum_hcd); - if (value == 0) { - /* - * Emulate synchronize_irq(): wait for callbacks to finish. - * This seems to be the best place to emulate the call to - * synchronize_irq() that's in usb_gadget_remove_driver(). - * Doing it in dummy_udc_stop() would be too late since it - * is called after the unbind callback and unbind shouldn't - * be invoked until all the other callbacks are finished. - */ - while (dum->callback_usage > 0) { - spin_unlock_irqrestore(&dum->lock, flags); - usleep_range(1000, 2000); - spin_lock_irqsave(&dum->lock, flags); - } - } spin_unlock_irqrestore(&dum->lock, flags); usb_hcd_poll_rh_status(dummy_hcd_to_hcd(dum_hcd)); @@ -945,6 +935,20 @@ static void dummy_udc_async_callbacks(struct usb_gadget *_gadget, bool enable) spin_lock_irq(&dum->lock); dum->ints_enabled = enable; + if (!enable) { + /* + * Emulate synchronize_irq(): wait for callbacks to finish. + * This has to happen after emulated interrupts are disabled + * (dum->ints_enabled is clear) and before the unbind callback, + * just like the call to synchronize_irq() in + * gadget/udc/core:gadget_unbind_driver(). + */ + while (dum->callback_usage > 0) { + spin_unlock_irq(&dum->lock); + usleep_range(1000, 2000); + spin_lock_irq(&dum->lock); + } + } spin_unlock_irq(&dum->lock); } @@ -1534,6 +1538,12 @@ static int transfer(struct dummy_hcd *dum_hcd, struct urb *urb, /* rescan to continue with any other queued i/o */ if (rescan) goto top; + + /* request not fully transferred; stop iterating to + * preserve data ordering across queued requests. + */ + if (req->req.actual < req->req.length) + break; } return sent; } diff --git a/drivers/usb/gadget/udc/lpc32xx_udc.c b/drivers/usb/gadget/udc/lpc32xx_udc.c index 1a7d3c4f652fe..a962d4294fbec 100644 --- a/drivers/usb/gadget/udc/lpc32xx_udc.c +++ b/drivers/usb/gadget/udc/lpc32xx_udc.c @@ -3020,7 +3020,7 @@ static int lpc32xx_udc_probe(struct platform_device *pdev) pdev->dev.dma_mask = &lpc32xx_usbd_dmamask; retval = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); if (retval) - return retval; + goto err_put_client; udc->board = &lpc32xx_usbddata; @@ -3038,28 +3038,32 @@ static int lpc32xx_udc_probe(struct platform_device *pdev) /* Get IRQs */ for (i = 0; i < 4; i++) { udc->udp_irq[i] = platform_get_irq(pdev, i); - if (udc->udp_irq[i] < 0) - return udc->udp_irq[i]; + if (udc->udp_irq[i] < 0) { + retval = udc->udp_irq[i]; + goto err_put_client; + } } udc->udp_baseaddr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(udc->udp_baseaddr)) { dev_err(udc->dev, "IO map failure\n"); - return PTR_ERR(udc->udp_baseaddr); + retval = PTR_ERR(udc->udp_baseaddr); + goto err_put_client; } /* Get USB device clock */ udc->usb_slv_clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(udc->usb_slv_clk)) { dev_err(udc->dev, "failed to acquire USB device clock\n"); - return PTR_ERR(udc->usb_slv_clk); + retval = PTR_ERR(udc->usb_slv_clk); + goto err_put_client; } /* Enable USB device clock */ retval = clk_prepare_enable(udc->usb_slv_clk); if (retval < 0) { dev_err(udc->dev, "failed to start USB device clock\n"); - return retval; + goto err_put_client; } /* Setup deferred workqueue data */ @@ -3162,6 +3166,9 @@ static int lpc32xx_udc_probe(struct platform_device *pdev) udc->udca_v_base, udc->udca_p_base); i2c_fail: clk_disable_unprepare(udc->usb_slv_clk); +err_put_client: + put_device(&udc->isp1301_i2c_client->dev); + dev_err(udc->dev, "%s probe failed, %d\n", driver_name, retval); return retval; @@ -3190,6 +3197,8 @@ static void lpc32xx_udc_remove(struct platform_device *pdev) udc->udca_v_base, udc->udca_p_base); clk_disable_unprepare(udc->usb_slv_clk); + + put_device(&udc->isp1301_i2c_client->dev); } #ifdef CONFIG_PM diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c index 7cdcc9d16b8b1..ea89bcb9ad35a 100644 --- a/drivers/usb/gadget/udc/renesas_usb3.c +++ b/drivers/usb/gadget/udc/renesas_usb3.c @@ -1669,6 +1669,10 @@ static bool usb3_std_req_get_status(struct renesas_usb3 *usb3, break; case USB_RECIP_ENDPOINT: num = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK; + if (num >= usb3->num_usb3_eps) { + stall = true; + break; + } usb3_ep = usb3_get_ep(usb3, num); if (usb3_ep->halt) status |= 1 << USB_ENDPOINT_HALT; @@ -1781,7 +1785,8 @@ static bool usb3_std_req_feature_endpoint(struct renesas_usb3 *usb3, struct renesas_usb3_ep *usb3_ep; struct renesas_usb3_request *usb3_req; - if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) + if ((le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) || + (num >= usb3->num_usb3_eps)) return true; /* stall */ usb3_ep = usb3_get_ep(usb3, num); diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c index 0c38fc37b6e66..7f7251c10e952 100644 --- a/drivers/usb/gadget/udc/tegra-xudc.c +++ b/drivers/usb/gadget/udc/tegra-xudc.c @@ -1558,12 +1558,6 @@ static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt) return -ENOTSUPP; } - if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) { - dev_dbg(xudc->dev, "EP %u already %s\n", ep->index, - halt ? "halted" : "not halted"); - return 0; - } - if (halt) { ep_halt(xudc, ep->index); } else { @@ -3398,17 +3392,18 @@ static void tegra_xudc_device_params_init(struct tegra_xudc *xudc) { u32 val, imod; + val = xudc_readl(xudc, BLCG); if (xudc->soc->has_ipfs) { - val = xudc_readl(xudc, BLCG); val |= BLCG_ALL; val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE | BLCG_COREPLL_PWRDN); val |= BLCG_IOPLL_0_PWRDN; val |= BLCG_IOPLL_1_PWRDN; val |= BLCG_IOPLL_2_PWRDN; - - xudc_writel(xudc, val, BLCG); + } else { + val &= ~BLCG_COREPLL_PWRDN; } + xudc_writel(xudc, val, BLCG); if (xudc->soc->port_speed_quirk) tegra_xudc_limit_port_speed(xudc); @@ -3959,6 +3954,7 @@ static void tegra_xudc_remove(struct platform_device *pdev) static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc) { unsigned long flags; + u32 val; dev_dbg(xudc->dev, "entering ELPG\n"); @@ -3971,6 +3967,10 @@ static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc) spin_unlock_irqrestore(&xudc->lock, flags); + val = xudc_readl(xudc, BLCG); + val |= BLCG_COREPLL_PWRDN; + xudc_writel(xudc, val, BLCG); + clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks); regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies); diff --git a/drivers/usb/host/ehci-brcm.c b/drivers/usb/host/ehci-brcm.c index 888e8f6670d2a..5e3156f94cc69 100644 --- a/drivers/usb/host/ehci-brcm.c +++ b/drivers/usb/host/ehci-brcm.c @@ -31,8 +31,8 @@ static inline void ehci_brcm_wait_for_sof(struct ehci_hcd *ehci, u32 delay) int res; /* Wait for next microframe (every 125 usecs) */ - res = readl_relaxed_poll_timeout(&ehci->regs->frame_index, val, - val != frame_idx, 1, 130); + res = readl_relaxed_poll_timeout_atomic(&ehci->regs->frame_index, + val, val != frame_idx, 1, 130); if (res) ehci_err(ehci, "Error waiting for SOF\n"); udelay(delay); diff --git a/drivers/usb/host/ohci-nxp.c b/drivers/usb/host/ohci-nxp.c index 24d5a1dc50560..509ca7d8d5138 100644 --- a/drivers/usb/host/ohci-nxp.c +++ b/drivers/usb/host/ohci-nxp.c @@ -223,6 +223,7 @@ static int ohci_hcd_nxp_probe(struct platform_device *pdev) fail_resource: usb_put_hcd(hcd); fail_disable: + put_device(&isp1301_i2c_client->dev); isp1301_i2c_client = NULL; return ret; } @@ -234,6 +235,7 @@ static void ohci_hcd_nxp_remove(struct platform_device *pdev) usb_remove_hcd(hcd); ohci_nxp_stop_hc(); usb_put_hcd(hcd); + put_device(&isp1301_i2c_client->dev); isp1301_i2c_client = NULL; } diff --git a/drivers/usb/host/ohci-platform.c b/drivers/usb/host/ohci-platform.c index f47ae12cde6a2..059621210d2c2 100644 --- a/drivers/usb/host/ohci-platform.c +++ b/drivers/usb/host/ohci-platform.c @@ -376,3 +376,4 @@ MODULE_DESCRIPTION(DRIVER_DESC); MODULE_AUTHOR("Hauke Mehrtens"); MODULE_AUTHOR("Alan Stern"); MODULE_LICENSE("GPL"); +MODULE_SOFTDEP("pre: ehci_platform"); diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index 0404489c2f6a9..bbf7606348d85 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1269,6 +1269,11 @@ static void quirk_usb_early_handoff(struct pci_dev *pdev) if (pdev->vendor == 0x184e) /* vendor Netlogic */ return; + /* Skip handoff for Renesas PCI USB controller on QCOM SOC */ + if ((pdev->vendor == PCI_VENDOR_ID_RENESAS) && + (pcie_find_root_port(pdev)->vendor == PCI_VENDOR_ID_QCOM)) + return; + /* * Bypass the Raspberry Pi 4 controller xHCI controller, things are * taken care of by the board's co-processor. diff --git a/drivers/usb/host/uhci-platform.c b/drivers/usb/host/uhci-platform.c index 62318291f5664..be8cba0242892 100644 --- a/drivers/usb/host/uhci-platform.c +++ b/drivers/usb/host/uhci-platform.c @@ -191,3 +191,4 @@ static struct platform_driver uhci_platform_driver = { .of_match_table = platform_uhci_ids, }, }; +MODULE_SOFTDEP("pre: ehci_platform"); diff --git a/drivers/usb/host/xhci-dbgtty.c b/drivers/usb/host/xhci-dbgtty.c index 57cdda4e09c8e..90282e51e23ec 100644 --- a/drivers/usb/host/xhci-dbgtty.c +++ b/drivers/usb/host/xhci-dbgtty.c @@ -554,7 +554,7 @@ static void xhci_dbc_tty_unregister_device(struct xhci_dbc *dbc) * Hang up the TTY. This wakes up any blocked * writers and causes subsequent writes to fail. */ - tty_vhangup(port->port.tty); + tty_port_tty_vhangup(&port->port); tty_unregister_device(dbc_tty_driver, port->minor); xhci_dbc_tty_exit_port(port); diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index b3a59ce1b3f41..5e1442e91743d 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -1671,7 +1671,7 @@ int xhci_hub_status_data(struct usb_hcd *hcd, char *buf) * SS devices are only visible to roothub after link training completes. * Keep polling roothubs for a grace period after xHC start */ - if (xhci->run_graceperiod) { + if (hcd->speed >= HCD_USB3 && xhci->run_graceperiod) { if (time_before(jiffies, xhci->run_graceperiod)) status = 1; else diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 5bdcf9ab2b99d..2bb7569c95054 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -128,11 +128,11 @@ static void inc_td_cnt(struct urb *urb) urb_priv->num_tds_done++; } -static void trb_to_noop(union xhci_trb *trb, u32 noop_type) +static void trb_to_noop(union xhci_trb *trb, u32 noop_type, bool unchain_links) { if (trb_is_link(trb)) { - /* unchain chained link TRBs */ - trb->link.control &= cpu_to_le32(~TRB_CHAIN); + if (unchain_links) + trb->link.control &= cpu_to_le32(~TRB_CHAIN); } else { trb->generic.field[0] = 0; trb->generic.field[1] = 0; @@ -465,7 +465,7 @@ static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, xhci_dbg(xhci, "Turn aborted command %p to no-op\n", i_cmd->command_trb); - trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP); + trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP, false); /* * caller waiting for completion is called when command @@ -797,13 +797,18 @@ static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci, * (The last TRB actually points to the ring enqueue pointer, which is not part * of this TD.) This is used to remove partially enqueued isoc TDs from a ring. */ -static void td_to_noop(struct xhci_td *td, bool flip_cycle) +static void td_to_noop(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, + struct xhci_td *td, bool flip_cycle) { + bool unchain_links; struct xhci_segment *seg = td->start_seg; union xhci_trb *trb = td->start_trb; + /* link TRBs should now be unchained, but some old HCs expect otherwise */ + unchain_links = !xhci_link_chain_quirk(xhci, ep->ring ? ep->ring->type : TYPE_STREAM); + while (1) { - trb_to_noop(trb, TRB_TR_NOOP); + trb_to_noop(trb, TRB_TR_NOOP, unchain_links); /* flip cycle if asked to */ if (flip_cycle && trb != td->start_trb && trb != td->end_trb) @@ -1091,16 +1096,16 @@ static int xhci_invalidate_cancelled_tds(struct xhci_virt_ep *ep) "Found multiple active URBs %p and %p in stream %u?\n", td->urb, cached_td->urb, td->urb->stream_id); - td_to_noop(cached_td, false); + td_to_noop(xhci, ep, cached_td, false); cached_td->cancel_status = TD_CLEARED; } - td_to_noop(td, false); + td_to_noop(xhci, ep, td, false); td->cancel_status = TD_CLEARING_CACHE; cached_td = td; break; } } else { - td_to_noop(td, false); + td_to_noop(xhci, ep, td, false); td->cancel_status = TD_CLEARED; } } @@ -1125,7 +1130,7 @@ static int xhci_invalidate_cancelled_tds(struct xhci_virt_ep *ep) continue; xhci_warn(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n", td->urb); - td_to_noop(td, false); + td_to_noop(xhci, ep, td, false); td->cancel_status = TD_CLEARED; } } @@ -3219,6 +3224,7 @@ irqreturn_t xhci_irq(struct usb_hcd *hcd) if (status & STS_HCE) { xhci_warn(xhci, "WARNING: Host Controller Error\n"); + xhci_halt(xhci); goto out; } @@ -4273,7 +4279,7 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, */ urb_priv->td[0].end_trb = ep_ring->enqueue; /* Every TRB except the first & last will have its cycle bit flipped. */ - td_to_noop(&urb_priv->td[0], true); + td_to_noop(xhci, xep, &urb_priv->td[0], true); /* Reset the ring enqueue back to the first TRB and its cycle bit. */ ep_ring->enqueue = urb_priv->td[0].start_trb; diff --git a/drivers/usb/host/xhci-sideband.c b/drivers/usb/host/xhci-sideband.c index a85f62a73313a..6519736061376 100644 --- a/drivers/usb/host/xhci-sideband.c +++ b/drivers/usb/host/xhci-sideband.c @@ -93,8 +93,6 @@ __xhci_sideband_remove_endpoint(struct xhci_sideband *sb, struct xhci_virt_ep *e static void __xhci_sideband_remove_interrupter(struct xhci_sideband *sb) { - struct usb_device *udev; - lockdep_assert_held(&sb->mutex); if (!sb->ir) @@ -102,10 +100,6 @@ __xhci_sideband_remove_interrupter(struct xhci_sideband *sb) xhci_remove_secondary_interrupter(xhci_to_hcd(sb->xhci), sb->ir); sb->ir = NULL; - udev = sb->vdev->udev; - - if (udev->state != USB_STATE_NOTATTACHED) - usb_offload_put(udev); } /* sideband api functions */ @@ -210,7 +204,6 @@ xhci_sideband_remove_endpoint(struct xhci_sideband *sb, return -ENODEV; __xhci_sideband_remove_endpoint(sb, ep); - xhci_initialize_ring_info(ep->ring); return 0; } @@ -292,8 +285,8 @@ EXPORT_SYMBOL_GPL(xhci_sideband_get_event_buffer); * Allow other drivers, such as usb controller driver, to check if there are * any sideband activity on the host controller. This information could be used * for power management or other forms of resource management. The caller should - * ensure downstream usb devices are all either suspended or marked as - * "offload_at_suspend" to ensure the correctness of the return value. + * ensure downstream usb devices are all marked as "offload_pm_locked" to ensure + * the correctness of the return value. * * Returns true on any active sideband existence, false otherwise. */ @@ -329,9 +322,6 @@ int xhci_sideband_create_interrupter(struct xhci_sideband *sb, int num_seg, bool ip_autoclear, u32 imod_interval, int intr_num) { - int ret = 0; - struct usb_device *udev; - if (!sb || !sb->xhci) return -ENODEV; @@ -349,12 +339,9 @@ xhci_sideband_create_interrupter(struct xhci_sideband *sb, int num_seg, if (!sb->ir) return -ENOMEM; - udev = sb->vdev->udev; - ret = usb_offload_get(udev); - sb->ir->ip_autoclear = ip_autoclear; - return ret; + return 0; } EXPORT_SYMBOL_GPL(xhci_sideband_create_interrupter); diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c index 5255b10028931..83b1766ff1521 100644 --- a/drivers/usb/host/xhci-tegra.c +++ b/drivers/usb/host/xhci-tegra.c @@ -1564,14 +1564,13 @@ static int tegra_xusb_setup_wakeup(struct platform_device *pdev, struct tegra_xu for (i = 0; i < tegra->soc->max_num_wakes; i++) { struct irq_data *data; - tegra->wake_irqs[i] = platform_get_irq(pdev, i + WAKE_IRQ_START_INDEX); + tegra->wake_irqs[i] = platform_get_irq_optional(pdev, i + WAKE_IRQ_START_INDEX); if (tegra->wake_irqs[i] < 0) break; data = irq_get_irq_data(tegra->wake_irqs[i]); if (!data) { dev_warn(tegra->dev, "get wake event %d irq data fail\n", i); - irq_dispose_mapping(tegra->wake_irqs[i]); break; } @@ -1584,16 +1583,6 @@ static int tegra_xusb_setup_wakeup(struct platform_device *pdev, struct tegra_xu return 0; } -static void tegra_xusb_dispose_wake(struct tegra_xusb *tegra) -{ - unsigned int i; - - for (i = 0; i < tegra->num_wakes; i++) - irq_dispose_mapping(tegra->wake_irqs[i]); - - tegra->num_wakes = 0; -} - static int tegra_xusb_probe(struct platform_device *pdev) { struct tegra_xusb *tegra; @@ -1649,10 +1638,8 @@ static int tegra_xusb_probe(struct platform_device *pdev) return err; tegra->padctl = tegra_xusb_padctl_get(&pdev->dev); - if (IS_ERR(tegra->padctl)) { - err = PTR_ERR(tegra->padctl); - goto dispose_wake; - } + if (IS_ERR(tegra->padctl)) + return PTR_ERR(tegra->padctl); np = of_parse_phandle(pdev->dev.of_node, "nvidia,xusb-padctl", 0); if (!np) { @@ -1976,8 +1963,6 @@ static int tegra_xusb_probe(struct platform_device *pdev) put_padctl: of_node_put(np); tegra_xusb_padctl_put(tegra->padctl); -dispose_wake: - tegra_xusb_dispose_wake(tegra); return err; } @@ -2010,8 +1995,6 @@ static void tegra_xusb_remove(struct platform_device *pdev) if (tegra->padctl_irq) pm_runtime_disable(&pdev->dev); - tegra_xusb_dispose_wake(tegra); - pm_runtime_put(&pdev->dev); tegra_xusb_disable(tegra); diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index a148a12801263..d84dded5333fb 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -2891,16 +2891,25 @@ int xhci_stop_endpoint_sync(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, int gfp_t gfp_flags) { struct xhci_command *command; + struct xhci_ep_ctx *ep_ctx; unsigned long flags; - int ret; + int ret = -ENODEV; command = xhci_alloc_command(xhci, true, gfp_flags); if (!command) return -ENOMEM; spin_lock_irqsave(&xhci->lock, flags); - ret = xhci_queue_stop_endpoint(xhci, command, ep->vdev->slot_id, - ep->ep_index, suspend); + + /* make sure endpoint exists and is running before stopping it */ + if (ep->ring) { + ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); + if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_RUNNING) + ret = xhci_queue_stop_endpoint(xhci, command, + ep->vdev->slot_id, + ep->ep_index, suspend); + } + if (ret < 0) { spin_unlock_irqrestore(&xhci->lock, flags); goto out; @@ -4130,7 +4139,7 @@ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id) if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) || (xhci->xhc_state & XHCI_STATE_HALTED)) { spin_unlock_irqrestore(&xhci->lock, flags); - kfree(command); + xhci_free_command(xhci, command); return -ENODEV; } @@ -4138,7 +4147,7 @@ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id) slot_id); if (ret) { spin_unlock_irqrestore(&xhci->lock, flags); - kfree(command); + xhci_free_command(xhci, command); return ret; } xhci_ring_cmd_db(xhci); diff --git a/drivers/usb/image/mdc800.c b/drivers/usb/image/mdc800.c index 7b7e1554ea20e..10d72562e4d29 100644 --- a/drivers/usb/image/mdc800.c +++ b/drivers/usb/image/mdc800.c @@ -707,7 +707,7 @@ static ssize_t mdc800_device_read (struct file *file, char __user *buf, size_t l if (signal_pending (current)) { mutex_unlock(&mdc800->io_lock); - return -EINTR; + return len == left ? -EINTR : len-left; } sts=left > (mdc800->out_count-mdc800->out_ptr)?mdc800->out_count-mdc800->out_ptr:left; @@ -730,9 +730,11 @@ static ssize_t mdc800_device_read (struct file *file, char __user *buf, size_t l mutex_unlock(&mdc800->io_lock); return len-left; } - wait_event_timeout(mdc800->download_wait, + retval = wait_event_timeout(mdc800->download_wait, mdc800->downloaded, msecs_to_jiffies(TO_DOWNLOAD_GET_READY)); + if (!retval) + usb_kill_urb(mdc800->download_urb); mdc800->downloaded = 0; if (mdc800->download_urb->status != 0) { diff --git a/drivers/usb/misc/chaoskey.c b/drivers/usb/misc/chaoskey.c index 225863321dc47..45cff32656c6e 100644 --- a/drivers/usb/misc/chaoskey.c +++ b/drivers/usb/misc/chaoskey.c @@ -444,9 +444,19 @@ static ssize_t chaoskey_read(struct file *file, goto bail; mutex_unlock(&dev->rng_lock); - result = mutex_lock_interruptible(&dev->lock); - if (result) - goto bail; + if (file->f_flags & O_NONBLOCK) { + result = mutex_trylock(&dev->lock); + if (result == 0) { + result = -EAGAIN; + goto bail; + } else { + result = 0; + } + } else { + result = mutex_lock_interruptible(&dev->lock); + if (result) + goto bail; + } if (dev->valid == dev->used) { result = _chaoskey_fill(dev); if (result < 0) { diff --git a/drivers/usb/misc/onboard_usb_dev.c b/drivers/usb/misc/onboard_usb_dev.c index 41360a7591e56..bde303b820d7e 100644 --- a/drivers/usb/misc/onboard_usb_dev.c +++ b/drivers/usb/misc/onboard_usb_dev.c @@ -661,6 +661,7 @@ static const struct usb_device_id onboard_dev_id_table[] = { { USB_DEVICE(VENDOR_ID_GENESYS, 0x0608) }, /* Genesys Logic GL850G USB 2.0 HUB */ { USB_DEVICE(VENDOR_ID_GENESYS, 0x0610) }, /* Genesys Logic GL852G USB 2.0 HUB */ { USB_DEVICE(VENDOR_ID_GENESYS, 0x0620) }, /* Genesys Logic GL3523 USB 3.1 HUB */ + { USB_DEVICE(VENDOR_ID_GENESYS, 0x0625) }, /* Genesys Logic GL3590 USB 3.2 HUB */ { USB_DEVICE(VENDOR_ID_MICROCHIP, 0x2412) }, /* USB2412 USB 2.0 HUB */ { USB_DEVICE(VENDOR_ID_MICROCHIP, 0x2514) }, /* USB2514B USB 2.0 HUB */ { USB_DEVICE(VENDOR_ID_MICROCHIP, 0x2517) }, /* USB2517 USB 2.0 HUB */ diff --git a/drivers/usb/misc/onboard_usb_dev.h b/drivers/usb/misc/onboard_usb_dev.h index c1462be5526d5..37054621f0d1c 100644 --- a/drivers/usb/misc/onboard_usb_dev.h +++ b/drivers/usb/misc/onboard_usb_dev.h @@ -108,6 +108,13 @@ static const struct onboard_dev_pdata genesys_gl852g_data = { .is_hub = true, }; +static const struct onboard_dev_pdata genesys_gl3590_data = { + .reset_us = 50, + .num_supplies = 2, + .supply_names = { "vdd", "vdd12" }, + .is_hub = true, +}; + static const struct onboard_dev_pdata vialab_vl817_data = { .reset_us = 10, .num_supplies = 1, @@ -140,6 +147,7 @@ static const struct of_device_id onboard_dev_match[] = { { .compatible = "usb5e3,608", .data = &genesys_gl850g_data, }, { .compatible = "usb5e3,610", .data = &genesys_gl852g_data, }, { .compatible = "usb5e3,620", .data = &genesys_gl852g_data, }, + { .compatible = "usb5e3,625", .data = &genesys_gl3590_data, }, { .compatible = "usb5e3,626", .data = &genesys_gl852g_data, }, { .compatible = "usbbda,179", .data = &realtek_rtl8188etv_data, }, { .compatible = "usbbda,411", .data = &realtek_rts5411_data, }, diff --git a/drivers/usb/misc/qcom_eud.c b/drivers/usb/misc/qcom_eud.c index 926419ca560fc..187cbffbab13d 100644 --- a/drivers/usb/misc/qcom_eud.c +++ b/drivers/usb/misc/qcom_eud.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -23,38 +25,111 @@ #define EUD_REG_VBUS_INT_CLR 0x0080 #define EUD_REG_CSR_EUD_EN 0x1014 #define EUD_REG_SW_ATTACH_DET 0x1018 +#define EUD_REG_PORT_SEL 0x1028 #define EUD_REG_EUD_EN2 0x0000 +#define EUD_MAX_PORTS 2 + #define EUD_ENABLE BIT(0) #define EUD_INT_PET_EUD BIT(0) #define EUD_INT_VBUS BIT(2) #define EUD_INT_SAFE_MODE BIT(4) #define EUD_INT_ALL (EUD_INT_VBUS | EUD_INT_SAFE_MODE) +static const char * const eud_port_names[] = { + "primary", + "secondary", +}; + +struct eud_path { + struct eud_chip *chip; + struct phy *phy; + struct usb_role_switch *controller_sw; + struct usb_role_switch *eud_sw; + enum usb_role curr_role; + char name[16]; + u8 num; +}; + struct eud_chip { struct device *dev; - struct usb_role_switch *role_sw; void __iomem *base; + struct eud_path *paths[EUD_MAX_PORTS]; + /* serializes EUD control operations */ + struct mutex state_lock; phys_addr_t mode_mgr; unsigned int int_status; int irq; bool enabled; bool usb_attached; + bool phy_enabled; + bool eud_disabled_for_host; + u8 port_idx; }; +static int eud_phy_enable(struct eud_chip *chip) +{ + struct phy *phy; + int ret; + + if (chip->phy_enabled) + return 0; + + phy = chip->paths[chip->port_idx]->phy; + + ret = phy_init(phy); + if (ret) { + dev_err(chip->dev, "Failed to initialize USB2 PHY for port %u: %d\n", + chip->port_idx, ret); + return ret; + } + + ret = phy_power_on(phy); + if (ret) { + dev_err(chip->dev, "Failed to power on USB2 PHY for port %u: %d\n", + chip->port_idx, ret); + phy_exit(phy); + return ret; + } + + chip->phy_enabled = true; + + return 0; +} + +static void eud_phy_disable(struct eud_chip *chip) +{ + struct phy *phy; + + if (!chip->phy_enabled) + return; + + phy = chip->paths[chip->port_idx]->phy; + + phy_power_off(phy); + phy_exit(phy); + chip->phy_enabled = false; +} + static int enable_eud(struct eud_chip *priv) { int ret; - ret = qcom_scm_io_writel(priv->mode_mgr + EUD_REG_EUD_EN2, 1); + ret = eud_phy_enable(priv); if (ret) return ret; + ret = qcom_scm_io_writel(priv->mode_mgr + EUD_REG_EUD_EN2, 1); + if (ret) { + eud_phy_disable(priv); + return ret; + } + writel(EUD_ENABLE, priv->base + EUD_REG_CSR_EUD_EN); writel(EUD_INT_VBUS | EUD_INT_SAFE_MODE, priv->base + EUD_REG_INT1_EN_MASK); - return usb_role_switch_set_role(priv->role_sw, USB_ROLE_DEVICE); + return 0; } static int disable_eud(struct eud_chip *priv) @@ -66,6 +141,8 @@ static int disable_eud(struct eud_chip *priv) return ret; writel(0, priv->base + EUD_REG_CSR_EUD_EN); + eud_phy_disable(priv); + return 0; } @@ -82,30 +159,103 @@ static ssize_t enable_store(struct device *dev, const char *buf, size_t count) { struct eud_chip *chip = dev_get_drvdata(dev); + struct eud_path *path; bool enable; int ret; if (kstrtobool(buf, &enable)) return -EINVAL; + guard(mutex)(&chip->state_lock); + + /* Skip operation if already in desired state */ + if (chip->enabled == enable) + return count; + + /* + * Handle double-disable scenario: User is disabling EUD that was already + * disabled due to host mode. Since the hardware is already disabled, we + * only need to clear the host-disabled flag to prevent unwanted re-enabling + * when exiting host mode. This respects the user's explicit disable request. + */ + if (!enable && chip->eud_disabled_for_host) { + chip->eud_disabled_for_host = false; + chip->enabled = false; + return count; + } + if (enable) { - ret = enable_eud(chip); - if (!ret) - chip->enabled = enable; - else - disable_eud(chip); + /* + * EUD functions by presenting itself as a USB device to the host PC for + * debugging, making it incompatible with USB host mode configuration. + * Prevent enabling EUD in this configuration to avoid hardware conflicts. + */ + path = chip->paths[chip->port_idx]; + if (path->curr_role == USB_ROLE_HOST) { + dev_err(chip->dev, "cannot enable EUD: USB port is in host mode\n"); + return -EBUSY; + } + ret = enable_eud(chip); + if (ret) { + dev_err(chip->dev, "failed to enable eud\n"); + return ret; + } } else { ret = disable_eud(chip); + if (ret) { + dev_err(chip->dev, "failed to disable eud\n"); + return ret; + } } - return ret < 0 ? ret : count; + chip->enabled = enable; + + return count; } static DEVICE_ATTR_RW(enable); +static ssize_t port_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct eud_chip *chip = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%s\n", eud_port_names[chip->port_idx]); +} + +static ssize_t port_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct eud_chip *chip = dev_get_drvdata(dev); + int port; + + port = sysfs_match_string(eud_port_names, buf); + if (port < 0) + return port; + + /* Check if the corresponding path is available */ + if (!chip->paths[port]) { + dev_err(chip->dev, "EUD not supported on selected port\n"); + return -EOPNOTSUPP; + } + + /* Port selection must be done before enabling EUD */ + if (chip->enabled) { + dev_err(chip->dev, "Cannot change port while EUD is enabled\n"); + return -EBUSY; + } + + writel(port, chip->base + EUD_REG_PORT_SEL); + chip->port_idx = port; + + return count; +} + +static DEVICE_ATTR_RW(port); + static struct attribute *eud_attrs[] = { &dev_attr_enable.attr, + &dev_attr_port.attr, NULL, }; ATTRIBUTE_GROUPS(eud); @@ -165,12 +315,31 @@ static irqreturn_t handle_eud_irq(int irq, void *data) static irqreturn_t handle_eud_irq_thread(int irq, void *data) { struct eud_chip *chip = data; + struct eud_path *path; int ret; + path = chip->paths[chip->port_idx]; + + /* + * EUD virtual attach/detach event handling for low power debugging: + * + * When EUD is enabled in debug mode, the device remains physically + * connected to the PC throughout the debug session, keeping the USB + * controller active. This prevents testing of low power scenarios that + * require USB disconnection. + * + * EUD solves this by providing virtual USB attach/detach events while + * maintaining the physical connection. These events are triggered from + * the Host PC via the enumerated EUD control interface and delivered + * to the EUD driver as interrupts. + * + * These notifications are forwarded to the USB controller through role + * switch framework. + */ if (chip->usb_attached) - ret = usb_role_switch_set_role(chip->role_sw, USB_ROLE_DEVICE); + ret = usb_role_switch_set_role(path->controller_sw, USB_ROLE_DEVICE); else - ret = usb_role_switch_set_role(chip->role_sw, USB_ROLE_HOST); + ret = usb_role_switch_set_role(path->controller_sw, USB_ROLE_NONE); if (ret) dev_err(chip->dev, "failed to set role switch\n"); @@ -181,15 +350,147 @@ static irqreturn_t handle_eud_irq_thread(int irq, void *data) return IRQ_HANDLED; } -static void eud_role_switch_release(void *data) +static int eud_role_switch_set(struct usb_role_switch *sw, enum usb_role role) { - struct eud_chip *chip = data; + struct eud_path *path = usb_role_switch_get_drvdata(sw); + struct eud_chip *chip = path->chip; + int ret; - usb_role_switch_put(chip->role_sw); + guard(mutex)(&chip->state_lock); + + /* + * EUD must be disabled when USB operates in host mode. EUD functions by + * presenting itself as a USB device to the host PC for debugging, making + * it incompatible with host mode configuration. + * + * chip->enabled preserves user's sysfs configuration and is not modified + * during host mode transitions to maintain user intent. + */ + + /* Only act if EUD is enabled and this is the active path */ + if (chip->enabled && path->num == chip->port_idx) { + if (role == USB_ROLE_HOST && !chip->eud_disabled_for_host) { + ret = disable_eud(chip); + if (ret) { + dev_err(chip->dev, "failed to disable EUD for host mode: %d\n", + ret); + return ret; + } + chip->eud_disabled_for_host = true; + } else if (role != USB_ROLE_HOST && chip->eud_disabled_for_host) { + ret = enable_eud(chip); + if (ret) { + dev_err(chip->dev, "failed to re-enable EUD after host mode: %d\n", + ret); + return ret; + } + chip->eud_disabled_for_host = false; + } + } + + ret = usb_role_switch_set_role(path->controller_sw, role); + if (ret) { + dev_err(path->chip->dev, "failed to set role %s for port %u: %d\n", + usb_role_string(role), path->num, ret); + return ret; + } + + path->curr_role = role; + + return 0; +} + +static void eud_path_role_switch_release(void *data) +{ + struct eud_path *path = data; + + usb_role_switch_unregister(path->eud_sw); + usb_role_switch_put(path->controller_sw); +} + +static int eud_init_path(struct eud_chip *chip, struct device_node *np) +{ + struct usb_role_switch_desc role_sw_desc; + struct usb_role_switch *sw; + struct device_node *controller_node; + struct eud_path *path; + u32 path_num; + int ret; + + ret = of_property_read_u32(np, "reg", &path_num); + if (ret) { + /* Legacy DT uses 'ports' node without 'reg' property; treat as path 0 */ + if (of_node_name_eq(np, "ports")) + path_num = 0; + else + return dev_err_probe(chip->dev, ret, "unexpected child node '%s'\n", + np->name); + } + + if (path_num >= EUD_MAX_PORTS) + return dev_err_probe(chip->dev, -EINVAL, "invalid path number: %u (max %d)\n", + path_num, EUD_MAX_PORTS - 1); + + path = devm_kzalloc(chip->dev, sizeof(*path), GFP_KERNEL); + if (!path) + return -ENOMEM; + + path->chip = chip; + path->num = path_num; + + controller_node = of_graph_get_remote_node(np, 0, -1); + if (!controller_node) + return dev_err_probe(chip->dev, -ENODEV, + "failed to get controller node for path %u\n", path_num); + + path->phy = devm_of_phy_get_by_index(chip->dev, controller_node, 0); + if (IS_ERR(path->phy)) { + of_node_put(controller_node); + return dev_err_probe(chip->dev, PTR_ERR(path->phy), + "failed to get PHY for path %d\n", path_num); + } + + path->curr_role = USB_ROLE_NONE; + + /* Fetch controller role switch if it is role switch capable */ + if (of_property_read_bool(controller_node, "usb-role-switch")) { + sw = usb_role_switch_find_by_fwnode(of_fwnode_handle(controller_node)); + if (!sw) { + of_node_put(controller_node); + return dev_err_probe(chip->dev, -EPROBE_DEFER, + "Failed to get controller role switch for path %d\n", + path_num); + } + path->controller_sw = sw; + } + + of_node_put(controller_node); + + role_sw_desc.fwnode = of_fwnode_handle(np); + role_sw_desc.set = eud_role_switch_set; + role_sw_desc.driver_data = path; + snprintf(path->name, sizeof(path->name), "eud-path%u", path_num); + role_sw_desc.name = path->name; + + path->eud_sw = usb_role_switch_register(chip->dev, &role_sw_desc); + if (IS_ERR(path->eud_sw)) { + usb_role_switch_put(path->controller_sw); + return dev_err_probe(chip->dev, PTR_ERR(path->eud_sw), + "Failed to register EUD role switch for path %d\n", path_num); + } + + ret = devm_add_action_or_reset(chip->dev, eud_path_role_switch_release, path); + if (ret) + return ret; + + chip->paths[path_num] = path; + + return 0; } static int eud_probe(struct platform_device *pdev) { + struct device_node *np = pdev->dev.of_node; struct eud_chip *chip; struct resource *res; int ret; @@ -200,14 +501,17 @@ static int eud_probe(struct platform_device *pdev) chip->dev = &pdev->dev; - chip->role_sw = usb_role_switch_get(&pdev->dev); - if (IS_ERR(chip->role_sw)) - return dev_err_probe(chip->dev, PTR_ERR(chip->role_sw), - "failed to get role switch\n"); + mutex_init(&chip->state_lock); - ret = devm_add_action_or_reset(chip->dev, eud_role_switch_release, chip); - if (ret) - return ret; + for_each_child_of_node_scoped(np, child) { + ret = eud_init_path(chip, child); + if (ret) + return ret; + } + + /* Primary path is mandatory. Secondary is optional */ + if (!chip->paths[0]) + return dev_err_probe(chip->dev, -ENODEV, "primary path not found\n"); chip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chip->base)) diff --git a/drivers/usb/misc/usbio.c b/drivers/usb/misc/usbio.c index 37644dddf157e..64815f8410acd 100644 --- a/drivers/usb/misc/usbio.c +++ b/drivers/usb/misc/usbio.c @@ -614,8 +614,10 @@ static int usbio_probe(struct usb_interface *intf, const struct usb_device_id *i usb_fill_bulk_urb(usbio->urb, udev, usbio->rx_pipe, usbio->rxbuf, usbio->rxbuf_len, usbio_bulk_recv, usbio); ret = usb_submit_urb(usbio->urb, GFP_KERNEL); - if (ret) - return dev_err_probe(dev, ret, "Submitting usb urb\n"); + if (ret) { + dev_err_probe(dev, ret, "Submitting usb urb\n"); + goto err_free_urb; + } mutex_lock(&usbio->ctrl_mutex); @@ -663,6 +665,7 @@ static int usbio_probe(struct usb_interface *intf, const struct usb_device_id *i err_unlock: mutex_unlock(&usbio->ctrl_mutex); usb_kill_urb(usbio->urb); +err_free_urb: usb_free_urb(usbio->urb); return ret; diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c index b26c1d382d599..3138f5dca6da4 100644 --- a/drivers/usb/misc/uss720.c +++ b/drivers/usb/misc/uss720.c @@ -736,7 +736,7 @@ static int uss720_probe(struct usb_interface *intf, ret = get_1284_register(pp, 0, ®, GFP_KERNEL); dev_dbg(&intf->dev, "reg: %7ph\n", priv->reg); if (ret < 0) - return ret; + goto probe_abort; ret = usb_find_last_int_in_endpoint(interface, &epd); if (!ret) { diff --git a/drivers/usb/misc/yurex.c b/drivers/usb/misc/yurex.c index 70dff0db5354f..6d03e689850a6 100644 --- a/drivers/usb/misc/yurex.c +++ b/drivers/usb/misc/yurex.c @@ -272,6 +272,7 @@ static int yurex_probe(struct usb_interface *interface, const struct usb_device_ dev->int_buffer, YUREX_BUF_SIZE, yurex_interrupt, dev, 1); dev->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; + dev->bbu = -1; if (usb_submit_urb(dev->urb, GFP_KERNEL)) { retval = -EIO; dev_err(&interface->dev, "Could not submitting URB\n"); @@ -280,7 +281,6 @@ static int yurex_probe(struct usb_interface *interface, const struct usb_device_ /* save our data pointer in this interface device */ usb_set_intfdata(interface, dev); - dev->bbu = -1; /* we can register the device now, as it is ready */ retval = usb_register_dev(interface, &yurex_class); diff --git a/drivers/usb/phy/phy-fsl-usb.c b/drivers/usb/phy/phy-fsl-usb.c index 40ac68e52cee7..e266a47c4d483 100644 --- a/drivers/usb/phy/phy-fsl-usb.c +++ b/drivers/usb/phy/phy-fsl-usb.c @@ -988,6 +988,7 @@ static void fsl_otg_remove(struct platform_device *pdev) { struct fsl_usb2_platform_data *pdata = dev_get_platdata(&pdev->dev); + disable_delayed_work_sync(&fsl_otg_dev->otg_event); usb_remove_phy(&fsl_otg_dev->phy); free_irq(fsl_otg_dev->irq, fsl_otg_dev); diff --git a/drivers/usb/phy/phy-isp1301.c b/drivers/usb/phy/phy-isp1301.c index f9b5c411aee4e..2940f0c84e1b7 100644 --- a/drivers/usb/phy/phy-isp1301.c +++ b/drivers/usb/phy/phy-isp1301.c @@ -149,7 +149,12 @@ struct i2c_client *isp1301_get_client(struct device_node *node) return client; /* non-DT: only one ISP1301 chip supported */ - return isp1301_i2c_client; + if (isp1301_i2c_client) { + get_device(&isp1301_i2c_client->dev); + return isp1301_i2c_client; + } + + return NULL; } EXPORT_SYMBOL_GPL(isp1301_get_client); diff --git a/drivers/usb/phy/phy.c b/drivers/usb/phy/phy.c index e1435bc596622..5a9b9353f343d 100644 --- a/drivers/usb/phy/phy.c +++ b/drivers/usb/phy/phy.c @@ -646,6 +646,8 @@ int usb_add_phy(struct usb_phy *x, enum usb_phy_type type) return -EINVAL; } + INIT_LIST_HEAD(&x->head); + usb_charger_init(x); ret = usb_add_extcon(x); if (ret) @@ -696,6 +698,8 @@ int usb_add_phy_dev(struct usb_phy *x) return -EINVAL; } + INIT_LIST_HEAD(&x->head); + usb_charger_init(x); ret = usb_add_extcon(x); if (ret) diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c index dc2fec9168b73..8a79548e1569b 100644 --- a/drivers/usb/renesas_usbhs/common.c +++ b/drivers/usb/renesas_usbhs/common.c @@ -815,6 +815,15 @@ static void usbhs_remove(struct platform_device *pdev) usbhs_platform_call(priv, hardware_exit, pdev); reset_control_assert(priv->rsts); + + /* + * Explicitly free the IRQ to ensure the interrupt handler is + * disabled and synchronized before freeing resources. + * devm_free_irq() calls free_irq() which waits for any running + * ISR to complete, preventing UAF. + */ + devm_free_irq(&pdev->dev, priv->irq, priv); + usbhs_mod_remove(priv); usbhs_fifo_remove(priv); usbhs_pipe_remove(priv); diff --git a/drivers/usb/renesas_usbhs/pipe.c b/drivers/usb/renesas_usbhs/pipe.c index 75fff2e4cbc65..56fc3ff5016fc 100644 --- a/drivers/usb/renesas_usbhs/pipe.c +++ b/drivers/usb/renesas_usbhs/pipe.c @@ -713,11 +713,13 @@ struct usbhs_pipe *usbhs_pipe_malloc(struct usbhs_priv *priv, /* make sure pipe is not busy */ ret = usbhsp_pipe_barrier(pipe); if (ret < 0) { + usbhsp_put_pipe(pipe); dev_err(dev, "pipe setup failed %d\n", usbhs_pipe_number(pipe)); return NULL; } if (usbhsp_setup_pipecfg(pipe, is_host, dir_in, &pipecfg)) { + usbhsp_put_pipe(pipe); dev_err(dev, "can't setup pipe\n"); return NULL; } diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c index 30482d4cf8267..b7cdc62d420fb 100644 --- a/drivers/usb/roles/class.c +++ b/drivers/usb/roles/class.c @@ -139,9 +139,14 @@ static void *usb_role_switch_match(const struct fwnode_handle *fwnode, const cha static struct usb_role_switch * usb_role_switch_is_parent(struct fwnode_handle *fwnode) { - struct fwnode_handle *parent = fwnode_get_parent(fwnode); + struct fwnode_handle *parent; struct device *dev; + if (!fwnode_device_is_compatible(fwnode, "usb-b-connector")) + return NULL; + + parent = fwnode_get_parent(fwnode); + if (!fwnode_property_present(parent, "usb-role-switch")) { fwnode_handle_put(parent); return NULL; diff --git a/drivers/usb/serial/belkin_sa.c b/drivers/usb/serial/belkin_sa.c index 44f5b58beec92..aa6b4c4ad5ecb 100644 --- a/drivers/usb/serial/belkin_sa.c +++ b/drivers/usb/serial/belkin_sa.c @@ -435,7 +435,7 @@ static int belkin_sa_tiocmset(struct tty_struct *tty, struct belkin_sa_private *priv = usb_get_serial_port_data(port); unsigned long control_state; unsigned long flags; - int retval; + int retval = 0; int rts = 0; int dtr = 0; @@ -452,26 +452,32 @@ static int belkin_sa_tiocmset(struct tty_struct *tty, } if (clear & TIOCM_RTS) { control_state &= ~TIOCM_RTS; - rts = 0; + rts = 1; } if (clear & TIOCM_DTR) { control_state &= ~TIOCM_DTR; - dtr = 0; + dtr = 1; } priv->control_state = control_state; spin_unlock_irqrestore(&priv->lock, flags); - retval = BSA_USB_CMD(BELKIN_SA_SET_RTS_REQUEST, rts); - if (retval < 0) { - dev_err(&port->dev, "Set RTS error %d\n", retval); - goto exit; + if (rts) { + retval = BSA_USB_CMD(BELKIN_SA_SET_RTS_REQUEST, + !!(control_state & TIOCM_RTS)); + if (retval < 0) { + dev_err(&port->dev, "Set RTS error %d\n", retval); + goto exit; + } } - retval = BSA_USB_CMD(BELKIN_SA_SET_DTR_REQUEST, dtr); - if (retval < 0) { - dev_err(&port->dev, "Set DTR error %d\n", retval); - goto exit; + if (dtr) { + retval = BSA_USB_CMD(BELKIN_SA_SET_DTR_REQUEST, + !!(control_state & TIOCM_DTR)); + if (retval < 0) { + dev_err(&port->dev, "Set DTR error %d\n", retval); + goto exit; + } } exit: return retval; diff --git a/drivers/usb/serial/f81232.c b/drivers/usb/serial/f81232.c index 530b77fc2f78d..9262a2ac97f58 100644 --- a/drivers/usb/serial/f81232.c +++ b/drivers/usb/serial/f81232.c @@ -70,7 +70,6 @@ MODULE_DEVICE_TABLE(usb, combined_id_table); #define F81232_REGISTER_REQUEST 0xa0 #define F81232_GET_REGISTER 0xc0 #define F81232_SET_REGISTER 0x40 -#define F81534A_ACCESS_REG_RETRY 2 #define SERIAL_BASE_ADDRESS 0x0120 #define RECEIVE_BUFFER_REGISTER (0x00 + SERIAL_BASE_ADDRESS) @@ -824,36 +823,31 @@ static void f81232_lsr_worker(struct work_struct *work) static int f81534a_ctrl_set_register(struct usb_interface *intf, u16 reg, u16 size, void *val) { - struct usb_device *dev = interface_to_usbdev(intf); - int retry = F81534A_ACCESS_REG_RETRY; - int status; - - while (retry--) { - status = usb_control_msg_send(dev, - 0, - F81232_REGISTER_REQUEST, - F81232_SET_REGISTER, - reg, - 0, - val, - size, - USB_CTRL_SET_TIMEOUT, - GFP_KERNEL); - if (status) { - status = usb_translate_errors(status); - if (status == -EIO) - continue; - } - - break; - } - - if (status) { - dev_err(&intf->dev, "failed to set register 0x%x: %d\n", - reg, status); - } + return usb_control_msg_send(interface_to_usbdev(intf), + 0, + F81232_REGISTER_REQUEST, + F81232_SET_REGISTER, + reg, + 0, + val, + size, + USB_CTRL_SET_TIMEOUT, + GFP_KERNEL); +} - return status; +static int f81534a_ctrl_get_register(struct usb_interface *intf, u16 reg, + u16 size, void *val) +{ + return usb_control_msg_recv(interface_to_usbdev(intf), + 0, + F81232_REGISTER_REQUEST, + F81232_GET_REGISTER, + reg, + 0, + val, + size, + USB_CTRL_GET_TIMEOUT, + GFP_KERNEL); } static int f81534a_ctrl_enable_all_ports(struct usb_interface *intf, bool en) @@ -869,6 +863,29 @@ static int f81534a_ctrl_enable_all_ports(struct usb_interface *intf, bool en) * bit 0~11 : Serial port enable bit. */ if (en) { + /* + * The Fintek F81532A/534A/535/536 family relies on the + * F81534A_CTRL_CMD_ENABLE_PORT (116h) register during + * initialization to both determine serial port status and + * control port creation. + * + * If the driver experiences fast load/unload cycles, the + * device state may becomes unstable, resulting in the + * incomplete generation of serial ports. + * + * Performing a dummy read operation on the register prior + * to the initial write command resolves the issue. + * + * This clears the device's stale internal state. Subsequent + * write operations will correctly generate all serial ports. + */ + status = f81534a_ctrl_get_register(intf, + F81534A_CTRL_CMD_ENABLE_PORT, + sizeof(enable), + enable); + if (status) + return status; + enable[0] = 0xff; enable[1] = 0x8f; } diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index b37fa31f56943..ceddfb1852e16 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -628,10 +628,8 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(FTDI_VID, FTDI_IBS_PEDO_PID) }, { USB_DEVICE(FTDI_VID, FTDI_IBS_PROD_PID) }, { USB_DEVICE(FTDI_VID, FTDI_TAVIR_STK500_PID) }, - { USB_DEVICE(FTDI_VID, FTDI_TIAO_UMPA_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, FTDI_NT_ORIONLXM_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, FTDI_TIAO_UMPA_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, FTDI_NT_ORIONLXM_PID, 1) }, { USB_DEVICE(FTDI_VID, FTDI_NT_ORIONLX_PLUS_PID) }, { USB_DEVICE(FTDI_VID, FTDI_NT_ORION_IO_PID) }, { USB_DEVICE(FTDI_VID, FTDI_NT_ORIONMX_PID) }, @@ -842,24 +840,18 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(FTDI_VID, FTDI_ELSTER_UNICOM_PID) }, { USB_DEVICE(FTDI_VID, FTDI_PROPOX_JTAGCABLEII_PID) }, { USB_DEVICE(FTDI_VID, FTDI_PROPOX_ISPCABLEIII_PID) }, - { USB_DEVICE(FTDI_VID, CYBER_CORTEX_AV_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, CYBER_CORTEX_AV_PID, 1) }, { USB_DEVICE_INTERFACE_NUMBER(OLIMEX_VID, OLIMEX_ARM_USB_OCD_PID, 1) }, { USB_DEVICE_INTERFACE_NUMBER(OLIMEX_VID, OLIMEX_ARM_USB_OCD_H_PID, 1) }, { USB_DEVICE_INTERFACE_NUMBER(OLIMEX_VID, OLIMEX_ARM_USB_TINY_PID, 1) }, { USB_DEVICE_INTERFACE_NUMBER(OLIMEX_VID, OLIMEX_ARM_USB_TINY_H_PID, 1) }, - { USB_DEVICE(FIC_VID, FIC_NEO1973_DEBUG_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, FTDI_OOCDLINK_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, LMI_LM3S_DEVEL_BOARD_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, LMI_LM3S_EVAL_BOARD_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, LMI_LM3S_ICDI_BOARD_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, FTDI_TURTELIZER_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(FIC_VID, FIC_NEO1973_DEBUG_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, FTDI_OOCDLINK_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, LMI_LM3S_DEVEL_BOARD_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, LMI_LM3S_EVAL_BOARD_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, LMI_LM3S_ICDI_BOARD_PID, 1) }, + { USB_DEVICE(FTDI_VID, FTDI_AXE027_PID) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, FTDI_TURTELIZER_PID, 1) }, { USB_DEVICE(RATOC_VENDOR_ID, RATOC_PRODUCT_ID_USB60F) }, { USB_DEVICE(RATOC_VENDOR_ID, RATOC_PRODUCT_ID_SCU18) }, { USB_DEVICE(FTDI_VID, FTDI_REU_TINY_PID) }, @@ -901,17 +893,14 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(ATMEL_VID, STK541_PID) }, { USB_DEVICE(DE_VID, STB_PID) }, { USB_DEVICE(DE_VID, WHT_PID) }, - { USB_DEVICE(ADI_VID, ADI_GNICE_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(ADI_VID, ADI_GNICEPLUS_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(ADI_VID, ADI_GNICE_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(ADI_VID, ADI_GNICEPLUS_PID, 1) }, { USB_DEVICE_AND_INTERFACE_INFO(MICROCHIP_VID, MICROCHIP_USB_BOARD_PID, USB_CLASS_VENDOR_SPEC, USB_SUBCLASS_VENDOR_SPEC, 0x00) }, { USB_DEVICE_INTERFACE_NUMBER(ACTEL_VID, MICROSEMI_ARROW_SF2PLUS_BOARD_PID, 2) }, { USB_DEVICE(JETI_VID, JETI_SPC1201_PID) }, - { USB_DEVICE(MARVELL_VID, MARVELL_SHEEVAPLUG_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(MARVELL_VID, MARVELL_SHEEVAPLUG_PID, 1) }, { USB_DEVICE(LARSENBRUSGAARD_VID, LB_ALTITRACK_PID) }, { USB_DEVICE(GN_OTOMETRICS_VID, AURICAL_USB_PID) }, { USB_DEVICE(FTDI_VID, PI_C865_PID) }, @@ -934,10 +923,8 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(PI_VID, PI_1016_PID) }, { USB_DEVICE(KONDO_VID, KONDO_USB_SERIAL_PID) }, { USB_DEVICE(BAYER_VID, BAYER_CONTOUR_CABLE_PID) }, - { USB_DEVICE(FTDI_VID, MARVELL_OPENRD_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, TI_XDS100V2_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, MARVELL_OPENRD_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, TI_XDS100V2_PID, 1) }, { USB_DEVICE(FTDI_VID, HAMEG_HO820_PID) }, { USB_DEVICE(FTDI_VID, HAMEG_HO720_PID) }, { USB_DEVICE(FTDI_VID, HAMEG_HO730_PID) }, @@ -946,18 +933,14 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(FTDI_VID, MJSG_SR_RADIO_PID) }, { USB_DEVICE(FTDI_VID, MJSG_HD_RADIO_PID) }, { USB_DEVICE(FTDI_VID, MJSG_XM_RADIO_PID) }, - { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_ST_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SLITE_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SH2_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, XVERVE_SIGNALYZER_ST_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, XVERVE_SIGNALYZER_SLITE_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, XVERVE_SIGNALYZER_SH2_PID, 1) }, { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SH4_PID), .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, { USB_DEVICE(FTDI_VID, SEGWAY_RMP200_PID) }, { USB_DEVICE(FTDI_VID, ACCESIO_COM4SM_PID) }, - { USB_DEVICE(IONICS_VID, IONICS_PLUGCOMPUTER_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(IONICS_VID, IONICS_PLUGCOMPUTER_PID, 1) }, { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_24_MASTER_WING_PID) }, { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_PC_WING_PID) }, { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_USB_DMX_PID) }, @@ -972,15 +955,12 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(FTDI_VID, FTDI_CINTERION_MC55I_PID) }, { USB_DEVICE(FTDI_VID, FTDI_FHE_PID) }, { USB_DEVICE(FTDI_VID, FTDI_DOTEC_PID) }, - { USB_DEVICE(QIHARDWARE_VID, MILKYMISTONE_JTAGSERIAL_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(ST_VID, ST_STMCLT_2232_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(QIHARDWARE_VID, MILKYMISTONE_JTAGSERIAL_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(ST_VID, ST_STMCLT_2232_PID, 1) }, { USB_DEVICE(ST_VID, ST_STMCLT_4232_PID), .driver_info = (kernel_ulong_t)&ftdi_stmclite_quirk }, { USB_DEVICE(FTDI_VID, FTDI_RF_R106) }, - { USB_DEVICE(FTDI_VID, FTDI_DISTORTEC_JTAG_LOCK_PICK_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, FTDI_DISTORTEC_JTAG_LOCK_PICK_PID, 1) }, { USB_DEVICE(FTDI_VID, FTDI_LUMEL_PD12_PID) }, /* Crucible Devices */ { USB_DEVICE(FTDI_VID, FTDI_CT_COMET_PID) }, @@ -1055,8 +1035,7 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(ICPDAS_VID, ICPDAS_I7561U_PID) }, { USB_DEVICE(ICPDAS_VID, ICPDAS_I7563U_PID) }, { USB_DEVICE(WICED_VID, WICED_USB20706V2_PID) }, - { USB_DEVICE(TI_VID, TI_CC3200_LAUNCHPAD_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(TI_VID, TI_CC3200_LAUNCHPAD_PID, 1) }, { USB_DEVICE(CYPRESS_VID, CYPRESS_WICED_BT_USB_PID) }, { USB_DEVICE(CYPRESS_VID, CYPRESS_WICED_WL_USB_PID) }, { USB_DEVICE(AIRBUS_DS_VID, AIRBUS_DS_P8GR) }, @@ -1076,10 +1055,8 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(UBLOX_VID, UBLOX_C099F9P_ODIN_PID) }, { USB_DEVICE_INTERFACE_NUMBER(UBLOX_VID, UBLOX_EVK_M101_PID, 2) }, /* FreeCalypso USB adapters */ - { USB_DEVICE(FTDI_VID, FTDI_FALCONIA_JTAG_BUF_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, - { USB_DEVICE(FTDI_VID, FTDI_FALCONIA_JTAG_UNBUF_PID), - .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, FTDI_FALCONIA_JTAG_BUF_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(FTDI_VID, FTDI_FALCONIA_JTAG_UNBUF_PID, 1) }, /* GMC devices */ { USB_DEVICE(GMC_VID, GMC_Z216C_PID) }, /* Altera USB Blaster 3 */ diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h index 2539b9e2f712c..6c76cfebfd0e4 100644 --- a/drivers/usb/serial/ftdi_sio_ids.h +++ b/drivers/usb/serial/ftdi_sio_ids.h @@ -96,6 +96,8 @@ #define LMI_LM3S_EVAL_BOARD_PID 0xbcd9 #define LMI_LM3S_ICDI_BOARD_PID 0xbcda +#define FTDI_AXE027_PID 0xBD90 /* PICAXE AXE027 USB download cable */ + #define FTDI_TURTELIZER_PID 0xBDC8 /* JTAG/RS-232 adapter by egnite GmbH */ /* OpenDCC (www.opendcc.de) product id */ diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c index 1fffda7647f98..ad73040b30c8d 100644 --- a/drivers/usb/serial/io_edgeport.c +++ b/drivers/usb/serial/io_edgeport.c @@ -73,6 +73,7 @@ static const struct usb_device_id edgeport_4port_id_table[] = { { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_22I) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_412_4) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_COMPATIBLE) }, + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_BLACKBOX_IC135A) }, { } }; @@ -121,6 +122,7 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_8R) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_8RR) }, { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_412_8) }, + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_BLACKBOX_IC135A) }, { USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0202) }, { USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0203) }, { USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0310) }, @@ -470,6 +472,7 @@ static void get_product_info(struct edgeport_serial *edge_serial) case ION_DEVICE_ID_EDGEPORT_2_DIN: case ION_DEVICE_ID_EDGEPORT_4_DIN: case ION_DEVICE_ID_EDGEPORT_16_DUAL_CPU: + case ION_DEVICE_ID_BLACKBOX_IC135A: product_info->IsRS232 = 1; break; diff --git a/drivers/usb/serial/io_usbvend.h b/drivers/usb/serial/io_usbvend.h index 9a6f742ad3abd..c82a275e8e76e 100644 --- a/drivers/usb/serial/io_usbvend.h +++ b/drivers/usb/serial/io_usbvend.h @@ -211,6 +211,7 @@ // // Definitions for other product IDs +#define ION_DEVICE_ID_BLACKBOX_IC135A 0x0801 // OEM device (rebranded Edgeport/4) #define ION_DEVICE_ID_MT4X56USB 0x1403 // OEM device #define ION_DEVICE_ID_E5805A 0x1A01 // OEM device (rebranded Edgeport/4) diff --git a/drivers/usb/serial/kobil_sct.c b/drivers/usb/serial/kobil_sct.c index 464433be20344..96ea571c436a7 100644 --- a/drivers/usb/serial/kobil_sct.c +++ b/drivers/usb/serial/kobil_sct.c @@ -418,7 +418,7 @@ static int kobil_tiocmset(struct tty_struct *tty, struct usb_serial_port *port = tty->driver_data; struct device *dev = &port->dev; struct kobil_private *priv; - int result; + int result = 0; int dtr = 0; int rts = 0; @@ -435,12 +435,12 @@ static int kobil_tiocmset(struct tty_struct *tty, if (set & TIOCM_DTR) dtr = 1; if (clear & TIOCM_RTS) - rts = 0; + rts = 1; if (clear & TIOCM_DTR) - dtr = 0; + dtr = 1; - if (priv->device_type == KOBIL_ADAPTER_B_PRODUCT_ID) { - if (dtr != 0) + if (dtr && priv->device_type == KOBIL_ADAPTER_B_PRODUCT_ID) { + if (set & TIOCM_DTR) dev_dbg(dev, "%s - Setting DTR\n", __func__); else dev_dbg(dev, "%s - Clearing DTR\n", __func__); @@ -448,13 +448,13 @@ static int kobil_tiocmset(struct tty_struct *tty, usb_sndctrlpipe(port->serial->dev, 0), SUSBCRequest_SetStatusLinesOrQueues, USB_TYPE_VENDOR | USB_RECIP_ENDPOINT | USB_DIR_OUT, - ((dtr != 0) ? SUSBCR_SSL_SETDTR : SUSBCR_SSL_CLRDTR), + ((set & TIOCM_DTR) ? SUSBCR_SSL_SETDTR : SUSBCR_SSL_CLRDTR), 0, NULL, 0, KOBIL_TIMEOUT); - } else { - if (rts != 0) + } else if (rts) { + if (set & TIOCM_RTS) dev_dbg(dev, "%s - Setting RTS\n", __func__); else dev_dbg(dev, "%s - Clearing RTS\n", __func__); @@ -462,7 +462,7 @@ static int kobil_tiocmset(struct tty_struct *tty, usb_sndctrlpipe(port->serial->dev, 0), SUSBCRequest_SetStatusLinesOrQueues, USB_TYPE_VENDOR | USB_RECIP_ENDPOINT | USB_DIR_OUT, - ((rts != 0) ? SUSBCR_SSL_SETRTS : SUSBCR_SSL_CLRRTS), + ((set & TIOCM_RTS) ? SUSBCR_SSL_SETRTS : SUSBCR_SSL_CLRRTS), 0, NULL, 0, diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index e9400727ad36e..5f16ea44084fe 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -1383,6 +1383,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(2) | RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1073, 0xff), /* Telit FN990A (ECM) */ .driver_info = NCTRL(0) | RSVD(1) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1074, 0xff), /* Telit FN990A (MBIM) */ + .driver_info = NCTRL(5) | RSVD(6) | RSVD(7) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1075, 0xff), /* Telit FN990A (PCIe) */ .driver_info = RSVD(0) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1077, 0xff), /* Telit FN990A (rmnet + audio) */ @@ -1401,12 +1403,16 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(0) | RSVD(1) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a0, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a1, 0xff), /* Telit FN20C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a2, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a3, 0xff), /* Telit FN920C04 (ECM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a4, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a6, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a7, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a8, 0xff), /* Telit FN920C04 (ECM) */ @@ -1415,6 +1421,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(0) | NCTRL(2) | RSVD(3) | RSVD(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10aa, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10ab, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x30), /* Telit FE990B (rmnet) */ .driver_info = NCTRL(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x40) }, @@ -1433,17 +1441,31 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b3, 0xff, 0xff, 0x60) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c0, 0xff), /* Telit FE910C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c1, 0xff), /* Telit FE910C04 (RNDIS) */ + .driver_info = NCTRL(4) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c2, 0xff), /* Telit FE910C04 (MBIM) */ + .driver_info = NCTRL(4) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c3, 0xff), /* Telit FE910C04 (ECM) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c4, 0xff), /* Telit FE910C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c5, 0xff), /* Telit FE910C04 (RNDIS) */ + .driver_info = NCTRL(4) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c6, 0xff), /* Telit FE910C04 (MBIM) */ + .driver_info = NCTRL(4) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10c7, 0xff, 0xff, 0x30), /* Telit FE910C04 (ECM) */ + .driver_info = NCTRL(4) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10c7, 0xff, 0xff, 0x40) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c8, 0xff), /* Telit FE910C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(2) | RSVD(3) | RSVD(4) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10c9, 0xff), /* Telit FE910C04 (MBIM) */ + .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10cb, 0xff), /* Telit FE910C04 (RNDIS) */ + .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d0, 0xff, 0xff, 0x30), /* Telit FN990B (rmnet) */ .driver_info = NCTRL(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d0, 0xff, 0xff, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d0, 0xff, 0xff, 0x60) }, - { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10c7, 0xff, 0xff, 0x30), /* Telit FE910C04 (ECM) */ - .driver_info = NCTRL(4) }, - { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10c7, 0xff, 0xff, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d1, 0xff, 0xff, 0x30), /* Telit FN990B (MBIM) */ .driver_info = NCTRL(6) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d1, 0xff, 0xff, 0x40) }, @@ -1491,6 +1513,7 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1231, 0xff), /* Telit LE910Cx (RNDIS) */ .driver_info = NCTRL(2) | RSVD(3) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x1250, 0xff, 0x00, 0x00) }, /* Telit LE910Cx (rmnet) */ + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1252, 0xff) }, /* Telit LE910Cx (MBIM) */ { USB_DEVICE(TELIT_VENDOR_ID, 0x1260), .driver_info = NCTRL(0) | RSVD(1) | RSVD(2) }, { USB_DEVICE(TELIT_VENDOR_ID, 0x1261), @@ -2376,6 +2399,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe0f0, 0xff), /* Foxconn T99W373 MBIM */ .driver_info = RSVD(3) }, + { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe123, 0xff), /* Foxconn T99W760 MBIM */ + .driver_info = RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe145, 0xff), /* Foxconn T99W651 RNDIS */ .driver_info = RSVD(5) | RSVD(6) }, { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe15f, 0xff), /* Foxconn T99W709 */ @@ -2418,6 +2443,9 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d22, 0xff, 0xff, 0x30) }, /* MeiG Smart SRM815 and SRM825L */ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d22, 0xff, 0xff, 0x40) }, /* MeiG Smart SRM825L */ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d22, 0xff, 0xff, 0x60) }, /* MeiG Smart SRM825L */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x30) }, /* MeiG Smart SRM825WN (Diag) */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x40) }, /* MeiG Smart SRM825WN (AT) */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x60) }, /* MeiG Smart SRM825WN (NMEA) */ { USB_DEVICE_INTERFACE_CLASS(0x2df3, 0x9d03, 0xff) }, /* LongSung M5710 */ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1404, 0xff) }, /* GosunCn GM500 RNDIS */ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) }, /* GosunCn GM500 MBIM */ @@ -2438,6 +2466,7 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0302, 0xff) }, /* Rolling RW101R-GL (laptop MBIM) */ { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0802, 0xff), /* Rolling RW350-GL (laptop MBIM) */ .driver_info = RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff) }, /* Rolling RW135R-GL (laptop MBIM) */ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) }, /* NetPrisma LCUK54-WWD for Global */ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0x00, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x40) }, diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index 47f50d7a385c8..255968f9ca42a 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -2350,10 +2350,11 @@ UNUSUAL_DEV( 0x2027, 0xa001, 0x0000, 0x9999, US_FL_SCM_MULT_TARG ), /* - * Reported by DocMAX - * and Thomas Weißschuh + * Reported by DocMAX , + * Thomas Weißschuh + * and Daniel Brát */ -UNUSUAL_DEV( 0x2109, 0x0715, 0x9999, 0x9999, +UNUSUAL_DEV( 0x2109, 0x0715, 0x0000, 0x9999, "VIA Labs, Inc.", "VL817 SATA Bridge", USB_SC_DEVICE, USB_PR_DEVICE, NULL, diff --git a/drivers/usb/storage/unusual_uas.h b/drivers/usb/storage/unusual_uas.h index 1477e31d77632..939a98c2d3f74 100644 --- a/drivers/usb/storage/unusual_uas.h +++ b/drivers/usb/storage/unusual_uas.h @@ -98,7 +98,7 @@ UNUSUAL_DEV(0x125f, 0xa94a, 0x0160, 0x0160, US_FL_NO_ATA_1X), /* Reported-by: Benjamin Tissoires */ -UNUSUAL_DEV(0x13fd, 0x3940, 0x0000, 0x9999, +UNUSUAL_DEV(0x13fd, 0x3940, 0x0000, 0x0309, "Initio Corporation", "INIC-3069", USB_SC_DEVICE, USB_PR_DEVICE, NULL, diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c index 1dcb77faf85d4..3edc9dc86b25b 100644 --- a/drivers/usb/typec/altmodes/displayport.c +++ b/drivers/usb/typec/altmodes/displayport.c @@ -100,9 +100,14 @@ static int dp_altmode_configure(struct dp_altmode *dp, u8 con) { u8 pin_assign = 0; u32 conf; + u32 signal; /* DP Signalling */ - conf = (dp->data.conf & DP_CONF_SIGNALLING_MASK) >> DP_CONF_SIGNALLING_SHIFT; + signal = DP_CAP_DP_SIGNALLING(dp->port->vdo) & DP_CAP_DP_SIGNALLING(dp->alt->vdo); + if (dp->plug_prime) + signal &= DP_CAP_DP_SIGNALLING(dp->plug_prime->vdo); + + conf = signal << DP_CONF_SIGNALLING_SHIFT; switch (con) { case DP_STATUS_CON_DISABLED: @@ -764,12 +769,16 @@ int dp_altmode_probe(struct typec_altmode *alt) if (!(DP_CAP_PIN_ASSIGN_DFP_D(port->vdo) & DP_CAP_PIN_ASSIGN_UFP_D(alt->vdo)) && !(DP_CAP_PIN_ASSIGN_UFP_D(port->vdo) & - DP_CAP_PIN_ASSIGN_DFP_D(alt->vdo))) + DP_CAP_PIN_ASSIGN_DFP_D(alt->vdo))) { + typec_altmode_put_plug(plug); return -ENODEV; + } dp = devm_kzalloc(&alt->dev, sizeof(*dp), GFP_KERNEL); - if (!dp) + if (!dp) { + typec_altmode_put_plug(plug); return -ENOMEM; + } INIT_WORK(&dp->work, dp_altmode_work); mutex_init(&dp->lock); diff --git a/drivers/usb/typec/altmodes/thunderbolt.c b/drivers/usb/typec/altmodes/thunderbolt.c index 6eadf7835f8f6..d09dd09cf1c31 100644 --- a/drivers/usb/typec/altmodes/thunderbolt.c +++ b/drivers/usb/typec/altmodes/thunderbolt.c @@ -39,28 +39,7 @@ static bool tbt_ready(struct typec_altmode *alt); static int tbt_enter_mode(struct tbt_altmode *tbt) { - struct typec_altmode *plug = tbt->plug[TYPEC_PLUG_SOP_P]; - u32 vdo; - - vdo = tbt->alt->vdo & (TBT_VENDOR_SPECIFIC_B0 | TBT_VENDOR_SPECIFIC_B1); - vdo |= tbt->alt->vdo & TBT_INTEL_SPECIFIC_B0; - vdo |= TBT_MODE; - - if (plug) { - if (typec_cable_is_active(tbt->cable)) - vdo |= TBT_ENTER_MODE_ACTIVE_CABLE; - - vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_SPEED(plug->vdo)); - vdo |= plug->vdo & TBT_CABLE_ROUNDED; - vdo |= plug->vdo & TBT_CABLE_OPTICAL; - vdo |= plug->vdo & TBT_CABLE_RETIMER; - vdo |= plug->vdo & TBT_CABLE_LINK_TRAINING; - } else { - vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_USB3_PASSIVE); - } - - tbt->enter_vdo = vdo; - return typec_altmode_enter(tbt->alt, &vdo); + return typec_altmode_enter(tbt->alt, &tbt->enter_vdo); } static void tbt_altmode_work(struct work_struct *work) @@ -337,6 +316,7 @@ static bool tbt_ready(struct typec_altmode *alt) { struct tbt_altmode *tbt = typec_altmode_get_drvdata(alt); struct typec_altmode *plug; + u32 vdo; if (tbt->cable) return true; @@ -364,6 +344,26 @@ static bool tbt_ready(struct typec_altmode *alt) tbt->plug[i] = plug; } + vdo = tbt->alt->vdo & (TBT_VENDOR_SPECIFIC_B0 | TBT_VENDOR_SPECIFIC_B1); + vdo |= tbt->alt->vdo & TBT_INTEL_SPECIFIC_B0; + vdo |= TBT_MODE; + plug = tbt->plug[TYPEC_PLUG_SOP_P]; + + if (plug) { + if (typec_cable_is_active(tbt->cable)) + vdo |= TBT_ENTER_MODE_ACTIVE_CABLE; + + vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_SPEED(plug->vdo)); + vdo |= plug->vdo & TBT_CABLE_ROUNDED; + vdo |= plug->vdo & TBT_CABLE_OPTICAL; + vdo |= plug->vdo & TBT_CABLE_RETIMER; + vdo |= plug->vdo & TBT_CABLE_LINK_TRAINING; + } else { + vdo |= TBT_ENTER_MODE_CABLE_SPEED(TBT_CABLE_USB3_PASSIVE); + } + + tbt->enter_vdo = vdo; + return true; } diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index 3ecc688dda82a..b56df9349f89b 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -15,6 +15,9 @@ #include #include #include +#include +#include +#include #define HD3SS3220_REG_CN_STAT 0x08 #define HD3SS3220_REG_CN_STAT_CTRL 0x09 @@ -54,6 +57,11 @@ struct hd3ss3220 { struct delayed_work output_poll_work; enum usb_role role_state; bool poll; + + struct gpio_desc *id_gpiod; + int id_irq; + + struct regulator *vbus; }; static int hd3ss3220_set_power_opmode(struct hd3ss3220 *hd3ss3220, int power_opmode) @@ -196,6 +204,23 @@ static const struct typec_operations hd3ss3220_ops = { .port_type_set = hd3ss3220_port_type_set, }; +static void hd3ss3220_regulator_control(struct hd3ss3220 *hd3ss3220, bool on) +{ + int ret; + + if (regulator_is_enabled(hd3ss3220->vbus) == on) + return; + + if (on) + ret = regulator_enable(hd3ss3220->vbus); + else + ret = regulator_disable(hd3ss3220->vbus); + + if (ret) + dev_err(hd3ss3220->dev, + "vbus regulator %s failed: %d\n", on ? "disable" : "enable", ret); +} + static void hd3ss3220_set_role(struct hd3ss3220 *hd3ss3220) { enum usb_role role_state = hd3ss3220_get_attached_state(hd3ss3220); @@ -213,6 +238,9 @@ static void hd3ss3220_set_role(struct hd3ss3220 *hd3ss3220) break; } + if (hd3ss3220->vbus && !hd3ss3220->id_gpiod) + hd3ss3220_regulator_control(hd3ss3220, role_state == USB_ROLE_HOST); + hd3ss3220->role_state = role_state; } @@ -319,13 +347,25 @@ static const struct regmap_config config = { .max_register = 0x0A, }; +static irqreturn_t hd3ss3220_id_isr(int irq, void *dev_id) +{ + struct hd3ss3220 *hd3ss3220 = dev_id; + int id; + + id = gpiod_get_value_cansleep(hd3ss3220->id_gpiod); + hd3ss3220_regulator_control(hd3ss3220, !id); + + return IRQ_HANDLED; +} + static int hd3ss3220_probe(struct i2c_client *client) { struct typec_capability typec_cap = { }; - struct hd3ss3220 *hd3ss3220; struct fwnode_handle *connector, *ep; - int ret; + struct hd3ss3220 *hd3ss3220; + struct regulator *vbus; unsigned int data; + int ret; hd3ss3220 = devm_kzalloc(&client->dev, sizeof(struct hd3ss3220), GFP_KERNEL); @@ -359,6 +399,49 @@ static int hd3ss3220_probe(struct i2c_client *client) goto err_put_fwnode; } + vbus = devm_of_regulator_get_optional(hd3ss3220->dev, + to_of_node(connector), + "vbus"); + if (IS_ERR(vbus) && vbus != ERR_PTR(-ENODEV)) { + ret = PTR_ERR(vbus); + dev_err(hd3ss3220->dev, "failed to get vbus: %d", ret); + goto err_put_fwnode; + } + + hd3ss3220->vbus = (vbus == ERR_PTR(-ENODEV) ? NULL : vbus); + + if (hd3ss3220->vbus) { + hd3ss3220->id_gpiod = devm_gpiod_get_optional(hd3ss3220->dev, + "id", + GPIOD_IN); + if (IS_ERR(hd3ss3220->id_gpiod)) { + ret = PTR_ERR(hd3ss3220->id_gpiod); + goto err_put_fwnode; + } + } + + if (hd3ss3220->id_gpiod) { + hd3ss3220->id_irq = gpiod_to_irq(hd3ss3220->id_gpiod); + if (hd3ss3220->id_irq < 0) { + ret = hd3ss3220->id_irq; + dev_err(hd3ss3220->dev, + "failed to get ID gpio: %d\n", + hd3ss3220->id_irq); + goto err_put_fwnode; + } + + ret = devm_request_threaded_irq(hd3ss3220->dev, + hd3ss3220->id_irq, NULL, + hd3ss3220_id_isr, + IRQF_TRIGGER_RISING | + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + dev_name(hd3ss3220->dev), hd3ss3220); + if (ret < 0) { + dev_err(hd3ss3220->dev, "failed to get ID irq: %d\n", ret); + goto err_put_fwnode; + } + } + typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; typec_cap.driver_data = hd3ss3220; typec_cap.type = TYPEC_PORT_DRP; @@ -418,6 +501,11 @@ static int hd3ss3220_probe(struct i2c_client *client) if (hd3ss3220->poll) schedule_delayed_work(&hd3ss3220->output_poll_work, HZ); + if (client->irq && device_property_read_bool(hd3ss3220->dev, "wakeup-source")) { + device_init_wakeup(&client->dev, true); + enable_irq_wake(client->irq); + } + dev_info(&client->dev, "probed revision=0x%x\n", ret); return 0; @@ -442,6 +530,35 @@ static void hd3ss3220_remove(struct i2c_client *client) usb_role_switch_put(hd3ss3220->role_sw); } +static int __maybe_unused hd3ss3220_suspend(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + + if (device_may_wakeup(dev)) + enable_irq_wake(client->irq); + else + disable_irq(client->irq); + + return 0; +} + +static int __maybe_unused hd3ss3220_resume(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + + if (device_may_wakeup(dev)) + disable_irq_wake(client->irq); + else + enable_irq(client->irq); + + return 0; +} + +static const struct dev_pm_ops hd3ss3220_pm_ops = { + .suspend = hd3ss3220_suspend, + .resume = hd3ss3220_resume, +}; + static const struct of_device_id dev_ids[] = { { .compatible = "ti,hd3ss3220"}, {} @@ -452,6 +569,7 @@ static struct i2c_driver hd3ss3220_driver = { .driver = { .name = "hd3ss3220", .of_match_table = dev_ids, + .pm = &hd3ss3220_pm_ops, }, .probe = hd3ss3220_probe, .remove = hd3ss3220_remove, diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index 870a71f953f6c..5b1f2750cfc3b 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -1755,9 +1755,9 @@ static int fusb302_probe(struct i2c_client *client) goto destroy_workqueue; } - ret = request_irq(chip->gpio_int_n_irq, fusb302_irq_intn, - IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "fsc_interrupt_int_n", chip); + ret = request_threaded_irq(chip->gpio_int_n_irq, NULL, fusb302_irq_intn, + IRQF_ONESHOT | IRQF_TRIGGER_LOW, + "fsc_interrupt_int_n", chip); if (ret < 0) { dev_err(dev, "cannot request IRQ for GPIO Int_N, ret=%d", ret); goto tcpm_unregister_port; diff --git a/drivers/usb/typec/ucsi/Kconfig b/drivers/usb/typec/ucsi/Kconfig index 7fcb1e1de5d6d..87dd992a4b9e9 100644 --- a/drivers/usb/typec/ucsi/Kconfig +++ b/drivers/usb/typec/ucsi/Kconfig @@ -73,7 +73,6 @@ config CROS_EC_UCSI tristate "UCSI Driver for ChromeOS EC" depends on MFD_CROS_EC_DEV depends on CROS_USBPD_NOTIFY - depends on !EXTCON_TCSS_CROS_EC default MFD_CROS_EC_DEV help This driver enables UCSI support for a ChromeOS EC. The EC is @@ -96,6 +95,7 @@ config UCSI_LENOVO_YOGA_C630 config UCSI_HUAWEI_GAOKUN tristate "UCSI Interface Driver for Huawei Matebook E Go" depends on EC_HUAWEI_GAOKUN + depends on DRM || !DRM select DRM_AUX_HPD_BRIDGE if DRM_BRIDGE && OF help This driver enables UCSI support on the Huawei Matebook E Go tablet, diff --git a/drivers/usb/typec/ucsi/psy.c b/drivers/usb/typec/ucsi/psy.c index 8ae900c8c1321..525c6fc2217da 100644 --- a/drivers/usb/typec/ucsi/psy.c +++ b/drivers/usb/typec/ucsi/psy.c @@ -88,15 +88,20 @@ static int ucsi_psy_get_voltage_max(struct ucsi_connector *con, union power_supply_propval *val) { u32 pdo; + int max_voltage = 0; switch (UCSI_CONSTAT(con, PWR_OPMODE)) { case UCSI_CONSTAT_PWR_OPMODE_PD: - if (con->num_pdos > 0) { - pdo = con->src_pdos[con->num_pdos - 1]; - val->intval = pdo_fixed_voltage(pdo) * 1000; - } else { - val->intval = 0; + for (int i = 0; i < con->num_pdos; i++) { + int pdo_voltage = 0; + + pdo = con->src_pdos[i]; + if (pdo_type(pdo) == PDO_TYPE_FIXED) + pdo_voltage = pdo_fixed_voltage(pdo) * 1000; + max_voltage = (pdo_voltage > max_voltage) ? pdo_voltage + : max_voltage; } + val->intval = max_voltage; break; case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0: case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5: @@ -144,6 +149,7 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, union power_supply_propval *val) { u32 pdo; + int max_current = 0; if (!UCSI_CONSTAT(con, CONNECTED)) { val->intval = 0; @@ -152,12 +158,16 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, switch (UCSI_CONSTAT(con, PWR_OPMODE)) { case UCSI_CONSTAT_PWR_OPMODE_PD: - if (con->num_pdos > 0) { - pdo = con->src_pdos[con->num_pdos - 1]; - val->intval = pdo_max_current(pdo) * 1000; - } else { - val->intval = 0; + for (int i = 0; i < con->num_pdos; i++) { + int pdo_current = 0; + + pdo = con->src_pdos[i]; + if (pdo_type(pdo) == PDO_TYPE_FIXED) + pdo_current = pdo_max_current(pdo) * 1000; + max_current = (pdo_current > max_current) ? pdo_current + : max_current; } + val->intval = max_current; break; case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5: val->intval = UCSI_TYPEC_1_5_CURRENT * 1000; diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 3f568f790f39b..47f9a80461099 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -42,8 +42,14 @@ void ucsi_notify_common(struct ucsi *ucsi, u32 cci) if (cci & UCSI_CCI_BUSY) return; - if (UCSI_CCI_CONNECTOR(cci)) - ucsi_connector_change(ucsi, UCSI_CCI_CONNECTOR(cci)); + if (UCSI_CCI_CONNECTOR(cci)) { + if (!ucsi->cap.num_connectors || + UCSI_CCI_CONNECTOR(cci) <= ucsi->cap.num_connectors) + ucsi_connector_change(ucsi, UCSI_CCI_CONNECTOR(cci)); + else + dev_err(ucsi->dev, "bogus connector number in CCI: %lu\n", + UCSI_CCI_CONNECTOR(cci)); + } if (cci & UCSI_CCI_ACK_COMPLETE && test_and_clear_bit(ACK_PENDING, &ucsi->flags)) @@ -1807,6 +1813,12 @@ static int ucsi_init(struct ucsi *ucsi) ret = -ENODEV; goto err_reset; } + /* Check if reserved bit set. This is out of spec but happens in buggy FW */ + if (ucsi->cap.num_connectors & 0x80) { + dev_warn(ucsi->dev, "UCSI: Invalid num_connectors %d. Likely buggy FW\n", + ucsi->cap.num_connectors); + ucsi->cap.num_connectors &= 0x7f; // clear bit and carry on + } /* Allocate the connectors. Released in ucsi_unregister() */ connector = kcalloc(ucsi->cap.num_connectors + 1, sizeof(*connector), GFP_KERNEL); diff --git a/drivers/usb/typec/ucsi/ucsi_glink.c b/drivers/usb/typec/ucsi/ucsi_glink.c index 8af79101a2fc7..11b3e24e34e2b 100644 --- a/drivers/usb/typec/ucsi/ucsi_glink.c +++ b/drivers/usb/typec/ucsi/ucsi_glink.c @@ -16,10 +16,10 @@ #define PMIC_GLINK_MAX_PORTS 3 -#define UCSI_BUF_SIZE 48 +#define UCSI_BUF_V1_SIZE (UCSI_MESSAGE_OUT + (UCSI_MESSAGE_OUT - UCSI_MESSAGE_IN)) +#define UCSI_BUF_V2_SIZE (UCSIv2_MESSAGE_OUT + (UCSIv2_MESSAGE_OUT - UCSI_MESSAGE_IN)) #define MSG_TYPE_REQ_RESP 1 -#define UCSI_BUF_SIZE 48 #define UC_NOTIFY_RECEIVER_UCSI 0x0 #define UC_UCSI_READ_BUF_REQ 0x11 @@ -30,24 +30,30 @@ struct ucsi_read_buf_req_msg { struct pmic_glink_hdr hdr; }; -struct ucsi_read_buf_resp_msg { +struct __packed ucsi_read_buf_resp_msg { struct pmic_glink_hdr hdr; - u8 buf[UCSI_BUF_SIZE]; + union { + u8 v2_buf[UCSI_BUF_V2_SIZE]; + u8 v1_buf[UCSI_BUF_V1_SIZE]; + } buf; u32 ret_code; }; -struct ucsi_write_buf_req_msg { +struct __packed ucsi_write_buf_req_msg { struct pmic_glink_hdr hdr; - u8 buf[UCSI_BUF_SIZE]; + union { + u8 v2_buf[UCSI_BUF_V2_SIZE]; + u8 v1_buf[UCSI_BUF_V1_SIZE]; + } buf; u32 reserved; }; -struct ucsi_write_buf_resp_msg { +struct __packed ucsi_write_buf_resp_msg { struct pmic_glink_hdr hdr; u32 ret_code; }; -struct ucsi_notify_ind_msg { +struct __packed ucsi_notify_ind_msg { struct pmic_glink_hdr hdr; u32 notification; u32 receiver; @@ -72,7 +78,7 @@ struct pmic_glink_ucsi { bool ucsi_registered; bool pd_running; - u8 read_buf[UCSI_BUF_SIZE]; + u8 read_buf[UCSI_BUF_V2_SIZE]; }; static int pmic_glink_ucsi_read(struct ucsi *__ucsi, unsigned int offset, @@ -132,17 +138,35 @@ static int pmic_glink_ucsi_locked_write(struct pmic_glink_ucsi *ucsi, unsigned i const void *val, size_t val_len) { struct ucsi_write_buf_req_msg req = {}; + size_t req_len, buf_len; unsigned long left; int ret; + u8 *buf; req.hdr.owner = PMIC_GLINK_OWNER_USBC; req.hdr.type = MSG_TYPE_REQ_RESP; req.hdr.opcode = UC_UCSI_WRITE_BUF_REQ; - memcpy(&req.buf[offset], val, val_len); + + if (ucsi->ucsi->version >= UCSI_VERSION_2_0) { + buf_len = UCSI_BUF_V2_SIZE; + buf = req.buf.v2_buf; + } else if (ucsi->ucsi->version) { + buf_len = UCSI_BUF_V1_SIZE; + buf = req.buf.v1_buf; + } else { + dev_err(ucsi->dev, "UCSI version unknown\n"); + return -EINVAL; + } + req_len = sizeof(struct pmic_glink_hdr) + buf_len + sizeof(u32); + + if (offset + val_len > buf_len) + return -EINVAL; + + memcpy(&buf[offset], val, val_len); reinit_completion(&ucsi->write_ack); - ret = pmic_glink_send(ucsi->client, &req, sizeof(req)); + ret = pmic_glink_send(ucsi->client, &req, req_len); if (ret < 0) { dev_err(ucsi->dev, "failed to send UCSI write request: %d\n", ret); return ret; @@ -216,12 +240,48 @@ static const struct ucsi_operations pmic_glink_ucsi_ops = { static void pmic_glink_ucsi_read_ack(struct pmic_glink_ucsi *ucsi, const void *data, int len) { - const struct ucsi_read_buf_resp_msg *resp = data; + u32 ret_code, resp_len, buf_len = 0; + u8 *buf; + + if (ucsi->ucsi->version) { + if (ucsi->ucsi->version >= UCSI_VERSION_2_0) { + buf = ((struct ucsi_read_buf_resp_msg *)data)->buf.v2_buf; + buf_len = UCSI_BUF_V2_SIZE; + } else { + buf = ((struct ucsi_read_buf_resp_msg *)data)->buf.v1_buf; + buf_len = UCSI_BUF_V1_SIZE; + } + } else if (!ucsi->ucsi_registered) { + /* + * If UCSI version is not known yet because device is not registered, choose buffer + * size which best fits incoming data + */ + if (len > sizeof(struct pmic_glink_hdr) + UCSI_BUF_V2_SIZE) { + buf = ((struct ucsi_read_buf_resp_msg *)data)->buf.v2_buf; + buf_len = UCSI_BUF_V2_SIZE; + } else { + buf = ((struct ucsi_read_buf_resp_msg *)data)->buf.v1_buf; + buf_len = UCSI_BUF_V1_SIZE; + } + } else { + dev_err(ucsi->dev, "Device has been registered but UCSI version is still unknown\n"); + return; + } - if (resp->ret_code) + resp_len = sizeof(struct pmic_glink_hdr) + buf_len + sizeof(u32); + + if (len > resp_len) + return; + + /* Ensure that buffer_len leaves space for ret_code to be read back from memory */ + if (buf_len > len - sizeof(struct pmic_glink_hdr) - sizeof(u32)) + buf_len = len - sizeof(struct pmic_glink_hdr) - sizeof(u32); + + memcpy(&ret_code, buf + buf_len, sizeof(u32)); + if (ret_code) return; - memcpy(ucsi->read_buf, resp->buf, UCSI_BUF_SIZE); + memcpy(ucsi->read_buf, buf, buf_len); complete(&ucsi->read_ack); } diff --git a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c index 7b5222081bbb5..c5965656babad 100644 --- a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c +++ b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c @@ -196,6 +196,7 @@ static void gaokun_ucsi_connector_status(struct ucsi_connector *con) const struct ucsi_operations gaokun_ucsi_ops = { .read_version = gaokun_ucsi_read_version, .read_cci = gaokun_ucsi_read_cci, + .poll_cci = gaokun_ucsi_read_cci, .read_message_in = gaokun_ucsi_read_message_in, .sync_control = ucsi_sync_control_common, .async_control = gaokun_ucsi_async_control, @@ -502,6 +503,7 @@ static void gaokun_ucsi_remove(struct auxiliary_device *adev) { struct gaokun_ucsi *uec = auxiliary_get_drvdata(adev); + disable_delayed_work_sync(&uec->work); gaokun_ec_unregister_notify(uec->ec, &uec->nb); ucsi_unregister(uec->ucsi); ucsi_destroy(uec->ucsi); diff --git a/drivers/usb/usbip/usbip_common.c b/drivers/usb/usbip/usbip_common.c index a2b2da1255dda..ba9e7c616e129 100644 --- a/drivers/usb/usbip/usbip_common.c +++ b/drivers/usb/usbip/usbip_common.c @@ -470,6 +470,18 @@ static void usbip_pack_ret_submit(struct usbip_header *pdu, struct urb *urb, urb->status = rpdu->status; urb->actual_length = rpdu->actual_length; urb->start_frame = rpdu->start_frame; + /* + * The number_of_packets field determines the length of + * iso_frame_desc[], which is a flexible array allocated + * at URB creation time. A response must never claim more + * packets than originally submitted; doing so would cause + * an out-of-bounds write in usbip_recv_iso() and + * usbip_pad_iso(). Clamp to zero on violation so both + * functions safely return early. + */ + if (rpdu->number_of_packets < 0 || + rpdu->number_of_packets > urb->number_of_packets) + rpdu->number_of_packets = 0; urb->number_of_packets = rpdu->number_of_packets; urb->error_count = rpdu->error_count; } diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c index 0d6c10a8490c0..f7e405abe6084 100644 --- a/drivers/usb/usbip/vhci_hcd.c +++ b/drivers/usb/usbip/vhci_hcd.c @@ -831,15 +831,15 @@ static int vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flag no_need_xmit: usb_hcd_unlink_urb_from_ep(hcd, urb); no_need_unlink: - spin_unlock_irqrestore(&vhci->lock, flags); if (!ret) { /* usb_hcd_giveback_urb() should be called with * irqs disabled */ - local_irq_disable(); + spin_unlock(&vhci->lock); usb_hcd_giveback_urb(hcd, urb, urb->status); - local_irq_enable(); + spin_lock(&vhci->lock); } + spin_unlock_irqrestore(&vhci->lock, flags); return ret; } diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index a7936bd1aabe1..44062e9d68f00 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1256,7 +1256,7 @@ static int query_virtqueues(struct mlx5_vdpa_net *ndev, int vq_idx = start_vq + i; if (cmd->err) { - mlx5_vdpa_err(mvdev, "query vq %d failed, err: %d\n", vq_idx, err); + mlx5_vdpa_err(mvdev, "query vq %d failed, err: %d\n", vq_idx, cmd->err); if (!err) err = cmd->err; continue; @@ -3640,9 +3640,6 @@ static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); int err = 0; - if (group >= MLX5_VDPA_NUMVQ_GROUPS) - return -EINVAL; - mvdev->mres.group2asid[group] = asid; mutex_lock(&mvdev->mres.lock); diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c index 9e8d07078606f..31a02e7fd7f27 100644 --- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c +++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c @@ -736,6 +736,7 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs) octep_vdpa_assign_barspace(vf_pdev, pdev, index); if (++index == num_vfs) { done = true; + pci_dev_put(vf_pdev); break; } } diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c index 36f61cc96e211..43426bd971acc 100644 --- a/drivers/vdpa/pds/vdpa_dev.c +++ b/drivers/vdpa/pds/vdpa_dev.c @@ -51,7 +51,7 @@ static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv) err = pdsc_register_notify(nb); if (err) { nb->notifier_call = NULL; - dev_err(dev, "failed to register pds event handler: %ps\n", + dev_err(dev, "failed to register pds event handler: %pe\n", ERR_PTR(err)); return -EINVAL; } diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index c1c6431950e1b..df9c7ddc5d782 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -606,12 +606,6 @@ static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group, struct vhost_iotlb *iommu; int i; - if (group > vdpasim->dev_attr.ngroups) - return -EINVAL; - - if (asid >= vdpasim->dev_attr.nas) - return -EINVAL; - iommu = &vdpasim->iommu[asid]; mutex_lock(&vdpasim->mutex); diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c index 480cac3a0c274..8ceca24ac136c 100644 --- a/drivers/vfio/device_cdev.c +++ b/drivers/vfio/device_cdev.c @@ -99,7 +99,7 @@ long vfio_df_ioctl_bind_iommufd(struct vfio_device_file *df, return ret; if (user_size < minsz) return -EINVAL; - ret = copy_struct_from_user(&bind, minsz, arg, user_size); + ret = copy_struct_from_user(&bind, sizeof(bind), arg, user_size); if (ret) return ret; diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index fde33f54e99ec..f141f21566c52 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -407,7 +407,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, ret = qm_get_vft(vf_qm, &vf_qm->qp_base); if (ret <= 0) { dev_err(dev, "failed to get vft qp nums\n"); - return ret; + return ret < 0 ? ret : -EINVAL; } if (ret != vf_data->qp_num) { @@ -1169,12 +1169,34 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, return 0; } +static void hisi_acc_vf_pci_reset_prepare(struct pci_dev *pdev) +{ + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + struct device *dev = &qm->pdev->dev; + u32 delay = 0; + + /* All reset requests need to be queued for processing */ + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + msleep(1); + if (++delay > QM_RESET_WAIT_TIMEOUT) { + dev_err(dev, "reset prepare failed\n"); + return; + } + } + + hisi_acc_vdev->set_reset_flag = true; +} + static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) { struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + + if (hisi_acc_vdev->set_reset_flag) + clear_bit(QM_RESETTING, &qm->misc_ctl); - if (hisi_acc_vdev->core_device.vdev.migration_flags != - VFIO_MIGRATION_STOP_COPY) + if (!hisi_acc_vdev->core_device.vdev.mig_ops) return; mutex_lock(&hisi_acc_vdev->state_mutex); @@ -1506,6 +1528,7 @@ static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) } hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; hisi_acc_vdev->dev_opened = true; + hisi_acc_vdev->match_done = 0; mutex_unlock(&hisi_acc_vdev->open_mutex); } @@ -1564,6 +1587,7 @@ static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { .mmap = hisi_acc_vfio_pci_mmap, .request = vfio_pci_core_request, .match = vfio_pci_core_match, + .match_token_uuid = vfio_pci_core_match_token_uuid, .bind_iommufd = vfio_iommufd_physical_bind, .unbind_iommufd = vfio_iommufd_physical_unbind, .attach_ioas = vfio_iommufd_physical_attach_ioas, @@ -1689,6 +1713,7 @@ static const struct pci_device_id hisi_acc_vfio_pci_table[] = { MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); static const struct pci_error_handlers hisi_acc_vf_err_handlers = { + .reset_prepare = hisi_acc_vf_pci_reset_prepare, .reset_done = hisi_acc_vf_pci_aer_reset_done, .error_detected = vfio_pci_core_aer_err_detected, }; diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h index 91002ceeebc18..6253fa074003e 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h @@ -27,6 +27,7 @@ #define ERROR_CHECK_TIMEOUT 100 #define CHECK_DELAY_TIME 100 +#define QM_RESET_WAIT_TIMEOUT 60000 #define QM_SQC_VFT_BASE_SHIFT_V2 28 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) @@ -110,6 +111,7 @@ struct hisi_acc_vf_migration_file { struct hisi_acc_vf_core_device { struct vfio_pci_core_device core_device; u8 match_done; + bool set_reset_flag; /* * io_base is only valid when dev_opened is true, * which is protected by open_mutex. diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index e346392b72f6a..3dc3c2432b5e0 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -491,7 +491,7 @@ nvgrace_gpu_map_and_read(struct nvgrace_gpu_pci_core_device *nvdev, ret = vfio_pci_core_do_io_rw(&nvdev->core_device, false, nvdev->resmem.ioaddr, buf, offset, mem_count, - 0, 0, false); + 0, 0, false, VFIO_PCI_IO_WIDTH_8); } return ret; @@ -609,7 +609,7 @@ nvgrace_gpu_map_and_write(struct nvgrace_gpu_pci_core_device *nvdev, ret = vfio_pci_core_do_io_rw(&nvdev->core_device, false, nvdev->resmem.ioaddr, (char __user *)buf, pos, mem_count, - 0, 0, true); + 0, 0, true, VFIO_PCI_IO_WIDTH_8); } return ret; diff --git a/drivers/vfio/pci/pds/dirty.c b/drivers/vfio/pci/pds/dirty.c index 481992142f790..4915a7c1c4916 100644 --- a/drivers/vfio/pci/pds/dirty.c +++ b/drivers/vfio/pci/pds/dirty.c @@ -292,8 +292,11 @@ static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio, len = num_ranges * sizeof(*region_info); node = interval_tree_iter_first(ranges, 0, ULONG_MAX); - if (!node) - return -EINVAL; + if (!node) { + err = -EINVAL; + goto out_free_region_info; + } + for (int i = 0; i < num_ranges; i++) { struct pds_lm_dirty_region_info *ri = ®ion_info[i]; u64 region_size = node->last - node->start + 1; diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 7dcf5439dedc9..085373d71e9c2 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -41,6 +41,40 @@ static bool nointxmask; static bool disable_vga; static bool disable_idle_d3; +static void vfio_pci_eventfd_rcu_free(struct rcu_head *rcu) +{ + struct vfio_pci_eventfd *eventfd = + container_of(rcu, struct vfio_pci_eventfd, rcu); + + eventfd_ctx_put(eventfd->ctx); + kfree(eventfd); +} + +int vfio_pci_eventfd_replace_locked(struct vfio_pci_core_device *vdev, + struct vfio_pci_eventfd __rcu **peventfd, + struct eventfd_ctx *ctx) +{ + struct vfio_pci_eventfd *new = NULL; + struct vfio_pci_eventfd *old; + + lockdep_assert_held(&vdev->igate); + + if (ctx) { + new = kzalloc(sizeof(*new), GFP_KERNEL_ACCOUNT); + if (!new) + return -ENOMEM; + + new->ctx = ctx; + } + + old = rcu_replace_pointer(*peventfd, new, + lockdep_is_held(&vdev->igate)); + if (old) + call_rcu(&old->rcu, vfio_pci_eventfd_rcu_free); + + return 0; +} + /* List of PF's that vfio_pci_core_sriov_configure() has been called on */ static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex); static LIST_HEAD(vfio_pci_sriov_pfs); @@ -555,6 +589,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_enable); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) { + struct pci_dev *bridge; struct pci_dev *pdev = vdev->pdev; struct vfio_pci_dummy_resource *dummy_res, *tmp; struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; @@ -661,12 +696,20 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) * We can not use the "try" reset interface here, which will * overwrite the previously restored configuration information. */ - if (vdev->reset_works && pci_dev_trylock(pdev)) { - if (!__pci_reset_function_locked(pdev)) - vdev->needs_reset = false; - pci_dev_unlock(pdev); + if (vdev->reset_works) { + bridge = pci_upstream_bridge(pdev); + if (bridge && !pci_dev_trylock(bridge)) + goto out_restore_state; + if (pci_dev_trylock(pdev)) { + if (!__pci_reset_function_locked(pdev)) + vdev->needs_reset = false; + pci_dev_unlock(pdev); + } + if (bridge) + pci_dev_unlock(bridge); } +out_restore_state: pci_restore_state(pdev); out: pci_disable_device(pdev); @@ -696,14 +739,8 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev) vfio_pci_core_disable(vdev); mutex_lock(&vdev->igate); - if (vdev->err_trigger) { - eventfd_ctx_put(vdev->err_trigger); - vdev->err_trigger = NULL; - } - if (vdev->req_trigger) { - eventfd_ctx_put(vdev->req_trigger); - vdev->req_trigger = NULL; - } + vfio_pci_eventfd_replace_locked(vdev, &vdev->err_trigger, NULL); + vfio_pci_eventfd_replace_locked(vdev, &vdev->req_trigger, NULL); mutex_unlock(&vdev->igate); } EXPORT_SYMBOL_GPL(vfio_pci_core_close_device); @@ -1800,21 +1837,21 @@ void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count) struct vfio_pci_core_device *vdev = container_of(core_vdev, struct vfio_pci_core_device, vdev); struct pci_dev *pdev = vdev->pdev; + struct vfio_pci_eventfd *eventfd; - mutex_lock(&vdev->igate); - - if (vdev->req_trigger) { + rcu_read_lock(); + eventfd = rcu_dereference(vdev->req_trigger); + if (eventfd) { if (!(count % 10)) pci_notice_ratelimited(pdev, "Relaying device request to user (#%u)\n", count); - eventfd_signal(vdev->req_trigger); + eventfd_signal(eventfd->ctx); } else if (count == 0) { pci_warn(pdev, "No device request channel registered, blocked until released by user\n"); } - - mutex_unlock(&vdev->igate); + rcu_read_unlock(); } EXPORT_SYMBOL_GPL(vfio_pci_core_request); @@ -2227,13 +2264,13 @@ pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev); + struct vfio_pci_eventfd *eventfd; - mutex_lock(&vdev->igate); - - if (vdev->err_trigger) - eventfd_signal(vdev->err_trigger); - - mutex_unlock(&vdev->igate); + rcu_read_lock(); + eventfd = rcu_dereference(vdev->err_trigger); + if (eventfd) + eventfd_signal(eventfd->ctx); + rcu_read_unlock(); return PCI_ERS_RESULT_CAN_RECOVER; } diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index 30d3e921cb0de..c76e753b3cecd 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -731,21 +731,27 @@ static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, return 0; } -static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, +static int vfio_pci_set_ctx_trigger_single(struct vfio_pci_core_device *vdev, + struct vfio_pci_eventfd __rcu **peventfd, unsigned int count, uint32_t flags, void *data) { /* DATA_NONE/DATA_BOOL enables loopback testing */ if (flags & VFIO_IRQ_SET_DATA_NONE) { - if (*ctx) { - if (count) { - eventfd_signal(*ctx); - } else { - eventfd_ctx_put(*ctx); - *ctx = NULL; - } + struct vfio_pci_eventfd *eventfd; + + eventfd = rcu_dereference_protected(*peventfd, + lockdep_is_held(&vdev->igate)); + + if (!eventfd) + return -EINVAL; + + if (count) { + eventfd_signal(eventfd->ctx); return 0; } + + return vfio_pci_eventfd_replace_locked(vdev, peventfd, NULL); } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { uint8_t trigger; @@ -753,8 +759,15 @@ static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, return -EINVAL; trigger = *(uint8_t *)data; - if (trigger && *ctx) - eventfd_signal(*ctx); + + if (trigger) { + struct vfio_pci_eventfd *eventfd = + rcu_dereference_protected(*peventfd, + lockdep_is_held(&vdev->igate)); + + if (eventfd) + eventfd_signal(eventfd->ctx); + } return 0; } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { @@ -765,22 +778,23 @@ static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, fd = *(int32_t *)data; if (fd == -1) { - if (*ctx) - eventfd_ctx_put(*ctx); - *ctx = NULL; + return vfio_pci_eventfd_replace_locked(vdev, + peventfd, NULL); } else if (fd >= 0) { struct eventfd_ctx *efdctx; + int ret; efdctx = eventfd_ctx_fdget(fd); if (IS_ERR(efdctx)) return PTR_ERR(efdctx); - if (*ctx) - eventfd_ctx_put(*ctx); + ret = vfio_pci_eventfd_replace_locked(vdev, + peventfd, efdctx); + if (ret) + eventfd_ctx_put(efdctx); - *ctx = efdctx; + return ret; } - return 0; } return -EINVAL; @@ -793,7 +807,7 @@ static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev, if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1) return -EINVAL; - return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger, + return vfio_pci_set_ctx_trigger_single(vdev, &vdev->err_trigger, count, flags, data); } @@ -804,7 +818,7 @@ static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1) return -EINVAL; - return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger, + return vfio_pci_set_ctx_trigger_single(vdev, &vdev->req_trigger, count, flags, data); } diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h index a9972eacb2936..97d992c063229 100644 --- a/drivers/vfio/pci/vfio_pci_priv.h +++ b/drivers/vfio/pci/vfio_pci_priv.h @@ -26,6 +26,10 @@ struct vfio_pci_ioeventfd { bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev); void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev); +int vfio_pci_eventfd_replace_locked(struct vfio_pci_core_device *vdev, + struct vfio_pci_eventfd __rcu **peventfd, + struct eventfd_ctx *ctx); + int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, unsigned index, unsigned start, unsigned count, void *data); diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 6192788c8ba39..25380b7dfe18a 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -135,7 +135,8 @@ VFIO_IORDWR(64) ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, void __iomem *io, char __user *buf, loff_t off, size_t count, size_t x_start, - size_t x_end, bool iswrite) + size_t x_end, bool iswrite, + enum vfio_pci_io_width max_width) { ssize_t done = 0; int ret; @@ -150,20 +151,19 @@ ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, else fillable = 0; - if (fillable >= 8 && !(off % 8)) { + if (fillable >= 8 && !(off % 8) && max_width >= 8) { ret = vfio_pci_iordwr64(vdev, iswrite, test_mem, io, buf, off, &filled); if (ret) return ret; - } else - if (fillable >= 4 && !(off % 4)) { + } else if (fillable >= 4 && !(off % 4) && max_width >= 4) { ret = vfio_pci_iordwr32(vdev, iswrite, test_mem, io, buf, off, &filled); if (ret) return ret; - } else if (fillable >= 2 && !(off % 2)) { + } else if (fillable >= 2 && !(off % 2) && max_width >= 2) { ret = vfio_pci_iordwr16(vdev, iswrite, test_mem, io, buf, off, &filled); if (ret) @@ -234,6 +234,7 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, void __iomem *io; struct resource *res = &vdev->pdev->resource[bar]; ssize_t done; + enum vfio_pci_io_width max_width = VFIO_PCI_IO_WIDTH_8; if (pci_resource_start(pdev, bar)) end = pci_resource_len(pdev, bar); @@ -262,6 +263,16 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, if (!io) return -ENOMEM; x_end = end; + + /* + * Certain devices (e.g. Intel X710) don't support qword + * access to the ROM bar. Otherwise PCI AER errors might be + * triggered. + * + * Disable qword access to the ROM bar universally, which + * worked reliably for years before qword access is enabled. + */ + max_width = VFIO_PCI_IO_WIDTH_4; } else { int ret = vfio_pci_core_setup_barmap(vdev, bar); if (ret) { @@ -278,7 +289,7 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, } done = vfio_pci_core_do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos, - count, x_start, x_end, iswrite); + count, x_start, x_end, iswrite, max_width); if (done >= 0) *ppos += done; @@ -352,7 +363,7 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, * to the memory enable bit in the command register. */ done = vfio_pci_core_do_io_rw(vdev, false, iomem, buf, off, count, - 0, 0, iswrite); + 0, 0, iswrite, VFIO_PCI_IO_WIDTH_8); vga_put(vdev->pdev, rsrc); diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 8f7f50acb6d63..1e77c0482b849 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -69,7 +69,7 @@ MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;" #define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN) -static const u64 vhost_net_features[VIRTIO_FEATURES_DWORDS] = { +static const u64 vhost_net_features[VIRTIO_FEATURES_U64S] = { VHOST_FEATURES | (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) | (1ULL << VIRTIO_NET_F_MRG_RXBUF) | @@ -1731,7 +1731,7 @@ static long vhost_net_set_owner(struct vhost_net *n) static long vhost_net_ioctl(struct file *f, unsigned int ioctl, unsigned long arg) { - u64 all_features[VIRTIO_FEATURES_DWORDS]; + u64 all_features[VIRTIO_FEATURES_U64S]; struct vhost_net *n = f->private_data; void __user *argp = (void __user *)arg; u64 __user *featurep = argp; @@ -1763,7 +1763,7 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, /* Copy the net features, up to the user-provided buffer size */ argp += sizeof(u64); - copied = min(count, VIRTIO_FEATURES_DWORDS); + copied = min(count, (u64)VIRTIO_FEATURES_U64S); if (copy_to_user(argp, vhost_net_features, copied * sizeof(u64))) return -EFAULT; @@ -1778,13 +1778,13 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, virtio_features_zero(all_features); argp += sizeof(u64); - copied = min(count, VIRTIO_FEATURES_DWORDS); + copied = min(count, (u64)VIRTIO_FEATURES_U64S); if (copy_from_user(all_features, argp, copied * sizeof(u64))) return -EFAULT; /* * Any feature specified by user-space above - * VIRTIO_FEATURES_MAX is not supported by definition. + * VIRTIO_FEATURES_BITS is not supported by definition. */ for (i = copied; i < count; ++i) { if (copy_from_user(&features, featurep + 1 + i, @@ -1794,7 +1794,7 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, return -EOPNOTSUPP; } - for (i = 0; i < VIRTIO_FEATURES_DWORDS; i++) + for (i = 0; i < VIRTIO_FEATURES_U64S; i++) if (all_features[i] & ~vhost_net_features[i]) return -EOPNOTSUPP; diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 05a481e4c385a..7e51eec842b8c 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -680,7 +680,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, case VHOST_VDPA_SET_GROUP_ASID: if (copy_from_user(&s, argp, sizeof(s))) return -EFAULT; - if (s.num >= vdpa->nas) + if (idx >= vdpa->ngroups || s.num >= vdpa->nas) return -EINVAL; if (!ops->set_group_asid) return -EOPNOTSUPP; @@ -1527,6 +1527,7 @@ static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) if (vma->vm_end - vma->vm_start != notify.size) return -ENOTSUPP; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); vma->vm_ops = &vhost_vdpa_vm_ops; return 0; diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index a78226b37739d..bccdc9eab267a 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -804,11 +804,13 @@ static int vhost_kthread_worker_create(struct vhost_worker *worker, ret = vhost_attach_task_to_cgroups(worker); if (ret) - goto stop_worker; + goto free_id; worker->id = id; return 0; +free_id: + xa_erase(&dev->worker_xa, id); stop_worker: vhost_kthread_do_stop(worker); return ret; diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index ae01457ea2cdb..78cc66fbb3dd6 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -64,14 +64,15 @@ static u32 vhost_transport_get_local_cid(void) return VHOST_VSOCK_DEFAULT_HOST_CID; } -/* Callers that dereference the return value must hold vhost_vsock_mutex or the - * RCU read lock. +/* Callers must be in an RCU read section or hold the vhost_vsock_mutex. + * The return value can only be dereferenced while within the section. */ static struct vhost_vsock *vhost_vsock_get(u32 guest_cid) { struct vhost_vsock *vsock; - hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) { + hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid, + lockdep_is_held(&vhost_vsock_mutex)) { u32 other_cid = vsock->guest_cid; /* Skip instances that have no CID yet */ @@ -707,9 +708,15 @@ static void vhost_vsock_reset_orphans(struct sock *sk) * executing. */ + rcu_read_lock(); + /* If the peer is still valid, no need to reset connection */ - if (vhost_vsock_get(vsk->remote_addr.svm_cid)) + if (vhost_vsock_get(vsk->remote_addr.svm_cid)) { + rcu_read_unlock(); return; + } + + rcu_read_unlock(); /* If the close timeout is pending, let it expire. This avoids races * with the timeout callback. diff --git a/drivers/video/backlight/led_bl.c b/drivers/video/backlight/led_bl.c index efc5e380669ae..f7ab9b3607313 100644 --- a/drivers/video/backlight/led_bl.c +++ b/drivers/video/backlight/led_bl.c @@ -210,6 +210,19 @@ static int led_bl_probe(struct platform_device *pdev) return PTR_ERR(priv->bl_dev); } + for (i = 0; i < priv->nb_leds; i++) { + struct device_link *link; + + link = device_link_add(&pdev->dev, priv->leds[i]->dev->parent, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + dev_err(&pdev->dev, "Failed to add devlink (consumer %s, supplier %s)\n", + dev_name(&pdev->dev), dev_name(priv->leds[i]->dev->parent)); + backlight_device_unregister(priv->bl_dev); + return -EINVAL; + } + } + for (i = 0; i < priv->nb_leds; i++) { mutex_lock(&priv->leds[i]->led_access); led_sysfs_disable(priv->leds[i]); diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c index a63bb42c8f8b0..8054e4787725e 100644 --- a/drivers/video/backlight/qcom-wled.c +++ b/drivers/video/backlight/qcom-wled.c @@ -1244,6 +1244,15 @@ static const struct wled_var_cfg wled4_ovp_cfg = { .size = ARRAY_SIZE(wled4_ovp_values), }; +static const u32 pmi8994_wled_ovp_values[] = { + 31000, 29500, 19400, 17800, +}; + +static const struct wled_var_cfg pmi8994_wled_ovp_cfg = { + .values = pmi8994_wled_ovp_values, + .size = ARRAY_SIZE(pmi8994_wled_ovp_values), +}; + static inline u32 wled5_ovp_values_fn(u32 idx) { /* @@ -1357,6 +1366,29 @@ static int wled_configure(struct wled *wled) }, }; + const struct wled_u32_opts pmi8994_wled_opts[] = { + { + .name = "qcom,current-boost-limit", + .val_ptr = &cfg->boost_i_limit, + .cfg = &wled4_boost_i_limit_cfg, + }, + { + .name = "qcom,current-limit-microamp", + .val_ptr = &cfg->string_i_limit, + .cfg = &wled4_string_i_limit_cfg, + }, + { + .name = "qcom,ovp-millivolt", + .val_ptr = &cfg->ovp, + .cfg = &pmi8994_wled_ovp_cfg, + }, + { + .name = "qcom,switching-freq", + .val_ptr = &cfg->switch_freq, + .cfg = &wled3_switch_freq_cfg, + }, + }; + const struct wled_u32_opts wled5_opts[] = { { .name = "qcom,current-boost-limit", @@ -1423,8 +1455,14 @@ static int wled_configure(struct wled *wled) break; case 4: - u32_opts = wled4_opts; - size = ARRAY_SIZE(wled4_opts); + if (of_device_is_compatible(dev->of_node, "qcom,pmi8950-wled") || + of_device_is_compatible(dev->of_node, "qcom,pmi8994-wled")) { + u32_opts = pmi8994_wled_opts; + size = ARRAY_SIZE(pmi8994_wled_opts); + } else { + u32_opts = wled4_opts; + size = ARRAY_SIZE(wled4_opts); + } *cfg = wled4_config_defaults; wled->wled_set_brightness = wled4_set_brightness; wled->wled_sync_toggle = wled3_sync_toggle; diff --git a/drivers/video/fbdev/au1200fb.c b/drivers/video/fbdev/au1200fb.c index ed770222660b5..685e629e7e164 100644 --- a/drivers/video/fbdev/au1200fb.c +++ b/drivers/video/fbdev/au1200fb.c @@ -1724,8 +1724,10 @@ static int au1200fb_drv_probe(struct platform_device *dev) /* Now hook interrupt too */ irq = platform_get_irq(dev, 0); - if (irq < 0) - return irq; + if (irq < 0) { + ret = irq; + goto failed; + } ret = request_irq(irq, au1200fb_handle_irq, IRQF_SHARED, "lcd", (void *)dev); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index e7e07eb2142eb..7453377f34336 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1047,7 +1047,8 @@ static void fbcon_init(struct vc_data *vc, bool init) return; if (!info->fbcon_par) - con2fb_acquire_newinfo(vc, info, vc->vc_num); + if (con2fb_acquire_newinfo(vc, info, vc->vc_num)) + return; /* If we are not the first console on this fb, copy the font from that console */ diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h index 4d97e6d8a16a2..7e21c8b336692 100644 --- a/drivers/video/fbdev/core/fbcon.h +++ b/drivers/video/fbdev/core/fbcon.h @@ -30,7 +30,6 @@ struct fbcon_display { #ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION u_short scrollmode; /* Scroll Method, use fb_scrollmode() */ #endif - u_short inverse; /* != 0 text black on white as default */ short yscroll; /* Hardware scrolling */ int vrows; /* number of virtual rows */ int cursor_shape; diff --git a/drivers/video/fbdev/core/fbsysfs.c b/drivers/video/fbdev/core/fbsysfs.c index b8344c40073b4..baa2bae0fb5b3 100644 --- a/drivers/video/fbdev/core/fbsysfs.c +++ b/drivers/video/fbdev/core/fbsysfs.c @@ -12,8 +12,6 @@ #include "fb_internal.h" -#define FB_SYSFS_FLAG_ATTR 1 - static int activate(struct fb_info *fb_info, struct fb_var_screeninfo *var) { int err; @@ -451,33 +449,7 @@ static struct attribute *fb_device_attrs[] = { NULL, }; -static const struct attribute_group fb_device_attr_group = { - .attrs = fb_device_attrs, -}; - -static int fb_init_device(struct fb_info *fb_info) -{ - int ret; - - dev_set_drvdata(fb_info->dev, fb_info); - - fb_info->class_flag |= FB_SYSFS_FLAG_ATTR; - - ret = device_add_group(fb_info->dev, &fb_device_attr_group); - if (ret) - fb_info->class_flag &= ~FB_SYSFS_FLAG_ATTR; - - return 0; -} - -static void fb_cleanup_device(struct fb_info *fb_info) -{ - if (fb_info->class_flag & FB_SYSFS_FLAG_ATTR) { - device_remove_group(fb_info->dev, &fb_device_attr_group); - - fb_info->class_flag &= ~FB_SYSFS_FLAG_ATTR; - } -} +ATTRIBUTE_GROUPS(fb_device); int fb_device_create(struct fb_info *fb_info) { @@ -485,14 +457,13 @@ int fb_device_create(struct fb_info *fb_info) dev_t devt = MKDEV(FB_MAJOR, node); int ret; - fb_info->dev = device_create(fb_class, fb_info->device, devt, NULL, "fb%d", node); + fb_info->dev = device_create_with_groups(fb_class, fb_info->device, devt, fb_info, + fb_device_groups, "fb%d", node); if (IS_ERR(fb_info->dev)) { /* Not fatal */ ret = PTR_ERR(fb_info->dev); pr_warn("Unable to create device for framebuffer %d; error %d\n", node, ret); fb_info->dev = NULL; - } else { - fb_init_device(fb_info); } return 0; @@ -505,7 +476,6 @@ void fb_device_destroy(struct fb_info *fb_info) if (!fb_info->dev) return; - fb_cleanup_device(fb_info); device_destroy(fb_class, devt); fb_info->dev = NULL; } diff --git a/drivers/video/fbdev/ffb.c b/drivers/video/fbdev/ffb.c index 34b6abff9493e..da531b4cb4513 100644 --- a/drivers/video/fbdev/ffb.c +++ b/drivers/video/fbdev/ffb.c @@ -335,6 +335,9 @@ struct ffb_dac { }; #define FFB_DAC_UCTRL 0x1001 /* User Control */ +#define FFB_DAC_UCTRL_OVENAB 0x00000008 /* Overlay Enable */ +#define FFB_DAC_UCTRL_WMODE 0x00000030 /* Window Mode */ +#define FFB_DAC_UCTRL_WM_COMB 0x00000000 /* Window Mode = Combined */ #define FFB_DAC_UCTRL_MANREV 0x00000f00 /* 4-bit Manufacturing Revision */ #define FFB_DAC_UCTRL_MANREV_SHIFT 8 #define FFB_DAC_TGEN 0x6000 /* Timing Generator */ @@ -425,7 +428,7 @@ static void ffb_switch_from_graph(struct ffb_par *par) { struct ffb_fbc __iomem *fbc = par->fbc; struct ffb_dac __iomem *dac = par->dac; - unsigned long flags; + unsigned long flags, uctrl; spin_lock_irqsave(&par->lock, flags); FFBWait(par); @@ -450,6 +453,15 @@ static void ffb_switch_from_graph(struct ffb_par *par) upa_writel((FFB_DAC_CUR_CTRL_P0 | FFB_DAC_CUR_CTRL_P1), &dac->value2); + /* Disable overlay and window modes. */ + upa_writel(FFB_DAC_UCTRL, &dac->type); + uctrl = upa_readl(&dac->value); + uctrl &= ~FFB_DAC_UCTRL_WMODE; + uctrl |= FFB_DAC_UCTRL_WM_COMB; + uctrl &= ~FFB_DAC_UCTRL_OVENAB; + upa_writel(FFB_DAC_UCTRL, &dac->type); + upa_writel(uctrl, &dac->value); + spin_unlock_irqrestore(&par->lock, flags); } diff --git a/drivers/video/fbdev/gbefb.c b/drivers/video/fbdev/gbefb.c index 4c36a3e409bea..cb6ff15a21dbf 100644 --- a/drivers/video/fbdev/gbefb.c +++ b/drivers/video/fbdev/gbefb.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -65,7 +66,7 @@ struct gbefb_par { static unsigned int gbe_mem_size = CONFIG_FB_GBE_MEM * 1024*1024; static void *gbe_mem; static dma_addr_t gbe_dma_addr; -static unsigned long gbe_mem_phys; +static phys_addr_t gbe_mem_phys; static struct { uint16_t *cpu; @@ -1183,7 +1184,7 @@ static int gbefb_probe(struct platform_device *p_dev) goto out_release_mem_region; } - gbe_mem_phys = (unsigned long) gbe_dma_addr; + gbe_mem_phys = dma_to_phys(&p_dev->dev, gbe_dma_addr); } par = info->par; diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c index b96a8a96bce8d..e418eee825fb2 100644 --- a/drivers/video/fbdev/pxafb.c +++ b/drivers/video/fbdev/pxafb.c @@ -419,12 +419,12 @@ static int pxafb_adjust_timing(struct pxafb_info *fbi, var->yres = max_t(int, var->yres, MIN_YRES); if (!(fbi->lccr0 & LCCR0_LCDT)) { - clamp_val(var->hsync_len, 1, 64); - clamp_val(var->vsync_len, 1, 64); - clamp_val(var->left_margin, 1, 255); - clamp_val(var->right_margin, 1, 255); - clamp_val(var->upper_margin, 1, 255); - clamp_val(var->lower_margin, 1, 255); + var->hsync_len = clamp(var->hsync_len, 1, 64); + var->vsync_len = clamp(var->vsync_len, 1, 64); + var->left_margin = clamp(var->left_margin, 1, 255); + var->right_margin = clamp(var->right_margin, 1, 255); + var->upper_margin = clamp(var->upper_margin, 1, 255); + var->lower_margin = clamp(var->lower_margin, 1, 255); } /* make sure each line is aligned on word boundary */ diff --git a/drivers/video/fbdev/riva/riva_hw.c b/drivers/video/fbdev/riva/riva_hw.c index 8b829b7200642..f292079566cfc 100644 --- a/drivers/video/fbdev/riva/riva_hw.c +++ b/drivers/video/fbdev/riva/riva_hw.c @@ -436,6 +436,9 @@ static char nv3_arb(nv3_fifo_info * res_info, nv3_sim_state * state, nv3_arb_in vmisses = 2; eburst_size = state->memory_width * 1; mburst_size = 32; + if (!state->mclk_khz) + return (0); + gns = 1000000 * (gmisses*state->mem_page_miss + state->mem_latency)/state->mclk_khz; ainfo->by_gfacc = gns*ainfo->gdrain_rate/1000000; ainfo->wcmocc = 0; diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c index 5f0dd01fd8349..891ce7b76d637 100644 --- a/drivers/video/fbdev/smscufx.c +++ b/drivers/video/fbdev/smscufx.c @@ -932,7 +932,6 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, unsigned long arg) { struct ufx_data *dev = info->par; - struct dloarea *area = NULL; if (!atomic_read(&dev->usb_active)) return 0; @@ -947,6 +946,10 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, /* TODO: Help propose a standard fb.h ioctl to report mmap damage */ if (cmd == UFX_IOCTL_REPORT_DAMAGE) { + struct dloarea *area __free(kfree) = kmalloc(sizeof(*area), GFP_KERNEL); + if (!area) + return -ENOMEM; + /* If we have a damage-aware client, turn fb_defio "off" * To avoid perf imact of unnecessary page fault handling. * Done by resetting the delay for this fb_info to a very @@ -956,7 +959,8 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, if (info->fbdefio) info->fbdefio->delay = UFX_DEFIO_WRITE_DISABLE; - area = (struct dloarea *)arg; + if (copy_from_user(area, (u8 __user *)arg, sizeof(*area))) + return -EFAULT; if (area->x < 0) area->x = 0; diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c index aa6cc0a8151ac..83dd31fa1fab5 100644 --- a/drivers/video/fbdev/ssd1307fb.c +++ b/drivers/video/fbdev/ssd1307fb.c @@ -680,7 +680,7 @@ static int ssd1307fb_probe(struct i2c_client *client) if (!ssd1307fb_defio) { dev_err(dev, "Couldn't allocate deferred io.\n"); ret = -ENOMEM; - goto fb_alloc_error; + goto fb_defio_error; } ssd1307fb_defio->delay = HZ / refreshrate; @@ -757,6 +757,8 @@ static int ssd1307fb_probe(struct i2c_client *client) regulator_disable(par->vbat_reg); reset_oled_error: fb_deferred_io_cleanup(info); +fb_defio_error: + __free_pages(vmem, get_order(vmem_size)); fb_alloc_error: framebuffer_release(info); return ret; diff --git a/drivers/video/fbdev/tcx.c b/drivers/video/fbdev/tcx.c index f9a0085ad72bf..ca9e84e8d8605 100644 --- a/drivers/video/fbdev/tcx.c +++ b/drivers/video/fbdev/tcx.c @@ -428,7 +428,7 @@ static int tcx_probe(struct platform_device *op) j = i; break; } - par->mmap_map[i].poff = op->resource[j].start; + par->mmap_map[i].poff = op->resource[j].start - info->fix.smem_start; } info->fbops = &tcx_ops; diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index 51ebe78359ec3..531fb8478e207 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -496,6 +496,9 @@ static int tdfxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info) } } + if (!var->pixclock) + return -EINVAL; + if (PICOS2KHZ(var->pixclock) > par->max_pixclock) { DPRINTK("pixclock too high (%ldKHz)\n", PICOS2KHZ(var->pixclock)); diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c index ccede85df1e1a..28e6d75e13ed3 100644 --- a/drivers/video/fbdev/udlfb.c +++ b/drivers/video/fbdev/udlfb.c @@ -1018,6 +1018,9 @@ static int dlfb_ops_check_var(struct fb_var_screeninfo *var, struct fb_videomode mode; struct dlfb_data *dlfb = info->par; + if (!var->pixclock) + return -EINVAL; + /* set device-specific elements of var unrelated to mode */ dlfb_var_color_format(var); diff --git a/drivers/video/fbdev/vt8500lcdfb.c b/drivers/video/fbdev/vt8500lcdfb.c index b08a6fdc53fd2..85c7a99a7d648 100644 --- a/drivers/video/fbdev/vt8500lcdfb.c +++ b/drivers/video/fbdev/vt8500lcdfb.c @@ -369,7 +369,7 @@ static int vt8500lcd_probe(struct platform_device *pdev) if (fbi->palette_cpu == NULL) { dev_err(&pdev->dev, "Failed to allocate palette buffer\n"); ret = -ENOMEM; - goto failed_free_io; + goto failed_free_mem_virt; } irq = platform_get_irq(pdev, 0); @@ -432,6 +432,9 @@ static int vt8500lcd_probe(struct platform_device *pdev) failed_free_palette: dma_free_coherent(&pdev->dev, fbi->palette_size, fbi->palette_cpu, fbi->palette_phys); +failed_free_mem_virt: + dma_free_coherent(&pdev->dev, fbi->fb.fix.smem_len, + fbi->fb.screen_buffer, fbi->fb.fix.smem_start); failed_free_io: iounmap(fbi->regbase); failed_free_res: diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c index bebd371c6b93e..a6ec392253c3e 100644 --- a/drivers/video/of_display_timing.c +++ b/drivers/video/of_display_timing.c @@ -181,7 +181,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) if (disp->num_timings == 0) { /* should never happen, as entry was already found above */ pr_err("%pOF: no timings specified\n", np); - goto entryfail; + goto timingfail; } disp->timings = kcalloc(disp->num_timings, @@ -189,13 +189,13 @@ struct display_timings *of_get_display_timings(const struct device_node *np) GFP_KERNEL); if (!disp->timings) { pr_err("%pOF: could not allocate timings array\n", np); - goto entryfail; + goto timingfail; } disp->num_timings = 0; disp->native_mode = 0; - for_each_child_of_node(timings_np, entry) { + for_each_child_of_node_scoped(timings_np, child) { struct display_timing *dt; int r; @@ -206,7 +206,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - r = of_parse_display_timing(entry, dt); + r = of_parse_display_timing(child, dt); if (r) { /* * to not encourage wrong devicetrees, fail in case of @@ -218,7 +218,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - if (native_mode == entry) + if (native_mode == child) disp->native_mode = disp->num_timings; disp->timings[disp->num_timings] = dt; diff --git a/drivers/virt/coco/tdx-guest/tdx-guest.c b/drivers/virt/coco/tdx-guest/tdx-guest.c index 4e239ec960c9b..40b53b4da7281 100644 --- a/drivers/virt/coco/tdx-guest/tdx-guest.c +++ b/drivers/virt/coco/tdx-guest/tdx-guest.c @@ -169,6 +169,8 @@ static void tdx_mr_deinit(const struct attribute_group *mr_grp) #define GET_QUOTE_SUCCESS 0 #define GET_QUOTE_IN_FLIGHT 0xffffffffffffffff +#define TDX_QUOTE_MAX_LEN (GET_QUOTE_BUF_SIZE - sizeof(struct tdx_quote_buf)) + /* struct tdx_quote_buf: Format of Quote request buffer. * @version: Quote format version, filled by TD. * @status: Status code of Quote request, filled by VMM. @@ -267,6 +269,7 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data) u8 *buf; struct tdx_quote_buf *quote_buf = quote_data; struct tsm_report_desc *desc = &report->desc; + u32 out_len; int ret; u64 err; @@ -304,12 +307,17 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data) return ret; } - buf = kvmemdup(quote_buf->data, quote_buf->out_len, GFP_KERNEL); + out_len = READ_ONCE(quote_buf->out_len); + + if (out_len > TDX_QUOTE_MAX_LEN) + return -EFBIG; + + buf = kvmemdup(quote_buf->data, out_len, GFP_KERNEL); if (!buf) return -ENOMEM; report->outblob = buf; - report->outblob_len = quote_buf->out_len; + report->outblob_len = out_len; /* * TODO: parse the PEM-formatted cert chain out of the quote buffer when diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index a09eb4d62f82d..5bdc6b82b30b4 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -53,7 +53,7 @@ static ssize_t features_show(struct device *_d, /* We actually represent this as a bitstring, as it could be * arbitrary length in future. */ - for (i = 0; i < VIRTIO_FEATURES_MAX; i++) + for (i = 0; i < VIRTIO_FEATURES_BITS; i++) len += sysfs_emit_at(buf, len, "%c", __virtio_test_bit(dev, i) ? '1' : '0'); len += sysfs_emit_at(buf, len, "\n"); @@ -272,8 +272,8 @@ static int virtio_dev_probe(struct device *_d) int err, i; struct virtio_device *dev = dev_to_virtio(_d); struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); - u64 device_features[VIRTIO_FEATURES_DWORDS]; - u64 driver_features[VIRTIO_FEATURES_DWORDS]; + u64 device_features[VIRTIO_FEATURES_U64S]; + u64 driver_features[VIRTIO_FEATURES_U64S]; u64 driver_features_legacy; /* We have a driver! */ @@ -286,7 +286,7 @@ static int virtio_dev_probe(struct device *_d) virtio_features_zero(driver_features); for (i = 0; i < drv->feature_table_size; i++) { unsigned int f = drv->feature_table[i]; - if (!WARN_ON_ONCE(f >= VIRTIO_FEATURES_MAX)) + if (!WARN_ON_ONCE(f >= VIRTIO_FEATURES_BITS)) virtio_features_set_bit(driver_features, f); } @@ -303,7 +303,7 @@ static int virtio_dev_probe(struct device *_d) } if (virtio_features_test_bit(device_features, VIRTIO_F_VERSION_1)) { - for (i = 0; i < VIRTIO_FEATURES_DWORDS; ++i) + for (i = 0; i < VIRTIO_FEATURES_U64S; ++i) dev->features_array[i] = driver_features[i] & device_features[i]; } else { @@ -325,7 +325,7 @@ static int virtio_dev_probe(struct device *_d) goto err; if (drv->validate) { - u64 features[VIRTIO_FEATURES_DWORDS]; + u64 features[VIRTIO_FEATURES_U64S]; virtio_features_copy(features, dev->features_array); err = drv->validate(dev); diff --git a/drivers/virtio/virtio_debug.c b/drivers/virtio/virtio_debug.c index d58713ddf2e58..ccf1955a1183a 100644 --- a/drivers/virtio/virtio_debug.c +++ b/drivers/virtio/virtio_debug.c @@ -8,12 +8,12 @@ static struct dentry *virtio_debugfs_dir; static int virtio_debug_device_features_show(struct seq_file *s, void *data) { - u64 device_features[VIRTIO_FEATURES_DWORDS]; + u64 device_features[VIRTIO_FEATURES_U64S]; struct virtio_device *dev = s->private; unsigned int i; virtio_get_features(dev, device_features); - for (i = 0; i < VIRTIO_FEATURES_MAX; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS; i++) { if (virtio_features_test_bit(device_features, i)) seq_printf(s, "%u\n", i); } @@ -26,7 +26,7 @@ static int virtio_debug_filter_features_show(struct seq_file *s, void *data) struct virtio_device *dev = s->private; unsigned int i; - for (i = 0; i < VIRTIO_FEATURES_MAX; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS; i++) { if (virtio_features_test_bit(dev->debugfs_filter_features, i)) seq_printf(s, "%u\n", i); } @@ -50,7 +50,7 @@ static int virtio_debug_filter_feature_add(void *data, u64 val) { struct virtio_device *dev = data; - if (val >= VIRTIO_FEATURES_MAX) + if (val >= VIRTIO_FEATURES_BITS) return -EINVAL; virtio_features_set_bit(dev->debugfs_filter_features, val); @@ -64,7 +64,7 @@ static int virtio_debug_filter_feature_del(void *data, u64 val) { struct virtio_device *dev = data; - if (val >= VIRTIO_FEATURES_MAX) + if (val >= VIRTIO_FEATURES_BITS) return -EINVAL; virtio_features_clear_bit(dev->debugfs_filter_features, val); diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index 9e503b7a58d81..413a8c3534638 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -401,7 +401,7 @@ void vp_modern_get_extended_features(struct virtio_pci_modern_device *mdev, int i; virtio_features_zero(features); - for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS / 32; i++) { u64 cur; vp_iowrite32(i, &cfg->device_feature_select); @@ -427,7 +427,7 @@ vp_modern_get_driver_extended_features(struct virtio_pci_modern_device *mdev, int i; virtio_features_zero(features); - for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS / 32; i++) { u64 cur; vp_iowrite32(i, &cfg->guest_feature_select); @@ -448,7 +448,7 @@ void vp_modern_set_extended_features(struct virtio_pci_modern_device *mdev, struct virtio_pci_common_cfg __iomem *cfg = mdev->common; int i; - for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS / 32; i++) { u32 cur = features[i >> 1] >> (32 * (i & 1)); vp_iowrite32(i, &cfg->guest_feature_select); diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 7b6205253b46b..ddab689596717 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -3166,6 +3166,7 @@ EXPORT_SYMBOL_GPL(virtqueue_map_alloc_coherent); * @vdev: the virtio device we are talking to * @map: metadata for performing mapping * @size: the size of the buffer + * @vaddr: the virtual address that needs to be freed * @map_handle: the mapped address that needs to be freed * */ @@ -3190,7 +3191,7 @@ EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent); * @dir: mapping direction * @attrs: mapping attributes * - * Returns mapped address. Caller should check that by virtqueue_mapping_error(). + * Returns mapped address. Caller should check that by virtqueue_map_mapping_error(). */ dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, struct page *page, @@ -3249,7 +3250,7 @@ EXPORT_SYMBOL_GPL(virtqueue_unmap_page_attrs); * The caller calls this to do dma mapping in advance. The DMA address can be * passed to this _vq when it is in pre-mapped mode. * - * return mapped address. Caller should check that by virtqueue_mapping_error(). + * return mapped address. Caller should check that by virtqueue_map_mapping_error(). */ dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr, size_t size, @@ -3299,7 +3300,7 @@ void virtqueue_unmap_single_attrs(const struct virtqueue *_vq, EXPORT_SYMBOL_GPL(virtqueue_unmap_single_attrs); /** - * virtqueue_mapping_error - check dma address + * virtqueue_map_mapping_error - check dma address * @_vq: the struct virtqueue we're talking about. * @addr: DMA address * diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index f9a29045eca0d..0a801f67b5996 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -80,7 +80,7 @@ static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) { struct vdpa_device *vdpa = vd_get_vdpa(vdev); - return vdpa_set_status(vdpa, status); + vdpa_set_status(vdpa, status); } static void virtio_vdpa_reset(struct virtio_device *vdev) diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c index 9ccedb3264fb2..832e3da94b203 100644 --- a/drivers/w1/slaves/w1_therm.c +++ b/drivers/w1/slaves/w1_therm.c @@ -1836,53 +1836,35 @@ static ssize_t alarms_store(struct device *device, struct w1_slave *sl = dev_to_w1_slave(device); struct therm_info info; u8 new_config_register[3]; /* array of data to be written */ - int temp, ret; - char *token = NULL; + long long temp; + int ret = 0; s8 tl, th; /* 1 byte per value + temp ring order */ - char *p_args, *orig; - - p_args = orig = kmalloc(size, GFP_KERNEL); - /* Safe string copys as buf is const */ - if (!p_args) { - dev_warn(device, - "%s: error unable to allocate memory %d\n", - __func__, -ENOMEM); - return size; - } - strcpy(p_args, buf); - - /* Split string using space char */ - token = strsep(&p_args, " "); - - if (!token) { - dev_info(device, - "%s: error parsing args %d\n", __func__, -EINVAL); - goto free_m; - } - - /* Convert 1st entry to int */ - ret = kstrtoint (token, 10, &temp); + const char *p = buf; + char *endp; + + temp = simple_strtoll(p, &endp, 10); + if (p == endp || *endp != ' ') + ret = -EINVAL; + else if (temp < INT_MIN || temp > INT_MAX) + ret = -ERANGE; if (ret) { dev_info(device, "%s: error parsing args %d\n", __func__, ret); - goto free_m; + return size; } tl = int_to_short(temp); - /* Split string using space char */ - token = strsep(&p_args, " "); - if (!token) { - dev_info(device, - "%s: error parsing args %d\n", __func__, -EINVAL); - goto free_m; - } - /* Convert 2nd entry to int */ - ret = kstrtoint (token, 10, &temp); + p = endp + 1; + temp = simple_strtoll(p, &endp, 10); + if (p == endp) + ret = -EINVAL; + else if (temp < INT_MIN || temp > INT_MAX) + ret = -ERANGE; if (ret) { dev_info(device, "%s: error parsing args %d\n", __func__, ret); - goto free_m; + return size; } /* Prepare to cast to short by eliminating out of range values */ @@ -1905,7 +1887,7 @@ static ssize_t alarms_store(struct device *device, dev_info(device, "%s: error reading from the slave device %d\n", __func__, ret); - goto free_m; + return size; } /* Write data in the device RAM */ @@ -1913,7 +1895,7 @@ static ssize_t alarms_store(struct device *device, dev_info(device, "%s: Device not supported by the driver %d\n", __func__, -ENODEV); - goto free_m; + return size; } ret = SLAVE_SPECIFIC_FUNC(sl)->write_data(sl, new_config_register); @@ -1922,10 +1904,6 @@ static ssize_t alarms_store(struct device *device, "%s: error writing to the slave device %d\n", __func__, ret); -free_m: - /* free allocated memory */ - kfree(orig); - return size; } diff --git a/drivers/w1/w1.c b/drivers/w1/w1.c index d0474a0532ec1..47e5a3f38ca33 100644 --- a/drivers/w1/w1.c +++ b/drivers/w1/w1.c @@ -758,8 +758,6 @@ int w1_attach_slave_device(struct w1_master *dev, struct w1_reg_num *rn) if (err < 0) { dev_err(&dev->dev, "%s: Attaching %s failed.\n", __func__, sl->name); - dev->slave_count--; - w1_family_put(sl->family); atomic_dec(&sl->master->refcnt); kfree(sl); return err; diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 05008d937e405..bc6db9a1c116b 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -2354,4 +2354,17 @@ config KEEMBAY_WATCHDOG To compile this driver as a module, choose M here: the module will be called keembay_wdt. +config GUNYAH_WATCHDOG + tristate "Qualcomm Gunyah Watchdog" + depends on ARCH_QCOM || COMPILE_TEST + depends on HAVE_ARM_SMCCC + select WATCHDOG_CORE + help + Say Y here to include support for watchdog timer provided by the + Gunyah hypervisor. The driver uses ARM SMC Calling Convention (SMCCC) + to interact with Gunyah Watchdog. + + To compile this driver as a module, choose M here: the + module will be called gunyah_wdt. + endif # WATCHDOG diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile index b680e4d3c1bc2..1215efb7816d2 100644 --- a/drivers/watchdog/Makefile +++ b/drivers/watchdog/Makefile @@ -102,6 +102,7 @@ obj-$(CONFIG_MSC313E_WATCHDOG) += msc313e_wdt.o obj-$(CONFIG_APPLE_WATCHDOG) += apple_wdt.o obj-$(CONFIG_SUNPLUS_WATCHDOG) += sunplus_wdt.o obj-$(CONFIG_MARVELL_GTI_WDT) += marvell_gti_wdt.o +obj-$(CONFIG_GUNYAH_WATCHDOG) += gunyah_wdt.o # X86 (i386 + ia64 + x86_64) Architecture obj-$(CONFIG_ACQUIRE_WDT) += acquirewdt.o diff --git a/drivers/watchdog/gunyah_wdt.c b/drivers/watchdog/gunyah_wdt.c new file mode 100644 index 0000000000000..49dfef459e847 --- /dev/null +++ b/drivers/watchdog/gunyah_wdt.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define GUNYAH_WDT_SMCCC_CALL_VAL(func_id) \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_32,\ + ARM_SMCCC_OWNER_VENDOR_HYP, func_id) + +/* SMCCC function IDs for watchdog operations */ +#define GUNYAH_WDT_CONTROL GUNYAH_WDT_SMCCC_CALL_VAL(0x0005) +#define GUNYAH_WDT_STATUS GUNYAH_WDT_SMCCC_CALL_VAL(0x0006) +#define GUNYAH_WDT_PING GUNYAH_WDT_SMCCC_CALL_VAL(0x0007) +#define GUNYAH_WDT_SET_TIME GUNYAH_WDT_SMCCC_CALL_VAL(0x0008) + +/* + * Control values for GUNYAH_WDT_CONTROL. + * Bit 0 is used to enable or disable the watchdog. If this bit is set, + * then the watchdog is enabled and vice versa. + * Bit 1 should always be set to 1 as this bit is reserved in Gunyah and + * it's expected to be 1. + */ +#define WDT_CTRL_ENABLE (BIT(1) | BIT(0)) +#define WDT_CTRL_DISABLE BIT(1) + +enum gunyah_error { + GUNYAH_ERROR_OK = 0, + GUNYAH_ERROR_UNIMPLEMENTED = -1, + GUNYAH_ERROR_ARG_INVAL = 1, +}; + +/** + * gunyah_error_remap() - Remap Gunyah hypervisor errors into a Linux error code + * @gunyah_error: Gunyah hypercall return value + */ +static inline int gunyah_error_remap(enum gunyah_error gunyah_error) +{ + switch (gunyah_error) { + case GUNYAH_ERROR_OK: + return 0; + case GUNYAH_ERROR_UNIMPLEMENTED: + return -EOPNOTSUPP; + default: + return -EINVAL; + } +} + +static int gunyah_wdt_call(unsigned long func_id, unsigned long arg1, + unsigned long arg2) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(func_id, arg1, arg2, &res); + return gunyah_error_remap(res.a0); +} + +static int gunyah_wdt_start(struct watchdog_device *wdd) +{ + unsigned int timeout_ms; + struct device *dev = wdd->parent; + int ret; + + ret = gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_DISABLE, 0); + if (ret && watchdog_active(wdd)) { + dev_err(dev, "%s: Failed to stop gunyah wdt %d\n", __func__, ret); + return ret; + } + + timeout_ms = wdd->timeout * 1000; + ret = gunyah_wdt_call(GUNYAH_WDT_SET_TIME, timeout_ms, timeout_ms); + if (ret) { + dev_err(dev, "%s: Failed to set timeout for gunyah wdt %d\n", + __func__, ret); + return ret; + } + + ret = gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_ENABLE, 0); + if (ret) + dev_err(dev, "%s: Failed to start gunyah wdt %d\n", __func__, ret); + + return ret; +} + +static int gunyah_wdt_stop(struct watchdog_device *wdd) +{ + return gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_DISABLE, 0); +} + +static int gunyah_wdt_ping(struct watchdog_device *wdd) +{ + return gunyah_wdt_call(GUNYAH_WDT_PING, 0, 0); +} + +static int gunyah_wdt_set_timeout(struct watchdog_device *wdd, + unsigned int timeout_sec) +{ + wdd->timeout = timeout_sec; + + if (watchdog_active(wdd)) + return gunyah_wdt_start(wdd); + + return 0; +} + +static int gunyah_wdt_get_time_since_last_ping(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(GUNYAH_WDT_STATUS, 0, 0, &res); + if (res.a0) + return gunyah_error_remap(res.a0); + + return res.a2 / 1000; +} + +static unsigned int gunyah_wdt_get_timeleft(struct watchdog_device *wdd) +{ + int seconds_since_last_ping; + + seconds_since_last_ping = gunyah_wdt_get_time_since_last_ping(); + if (seconds_since_last_ping < 0 || + seconds_since_last_ping > wdd->timeout) + return 0; + + return wdd->timeout - seconds_since_last_ping; +} + +static int gunyah_wdt_restart(struct watchdog_device *wdd, + unsigned long action, void *data) +{ + /* Set timeout to 1ms and send a ping */ + gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_DISABLE, 0); + gunyah_wdt_call(GUNYAH_WDT_SET_TIME, 1, 1); + gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_ENABLE, 0); + gunyah_wdt_call(GUNYAH_WDT_PING, 0, 0); + + /* Wait to make sure reset occurs */ + mdelay(100); + + return 0; +} + +static const struct watchdog_info gunyah_wdt_info = { + .identity = "Gunyah Watchdog", + .options = WDIOF_SETTIMEOUT + | WDIOF_KEEPALIVEPING + | WDIOF_MAGICCLOSE, +}; + +static const struct watchdog_ops gunyah_wdt_ops = { + .owner = THIS_MODULE, + .start = gunyah_wdt_start, + .stop = gunyah_wdt_stop, + .ping = gunyah_wdt_ping, + .set_timeout = gunyah_wdt_set_timeout, + .get_timeleft = gunyah_wdt_get_timeleft, + .restart = gunyah_wdt_restart +}; + +static int gunyah_wdt_probe(struct platform_device *pdev) +{ + struct watchdog_device *wdd; + struct device *dev = &pdev->dev; + int ret; + + ret = gunyah_wdt_call(GUNYAH_WDT_STATUS, 0, 0); + if (ret == -EOPNOTSUPP) + return -ENODEV; + + if (ret) + return dev_err_probe(dev, ret, "status check failed\n"); + + wdd = devm_kzalloc(dev, sizeof(*wdd), GFP_KERNEL); + if (!wdd) + return -ENOMEM; + + wdd->info = &gunyah_wdt_info; + wdd->ops = &gunyah_wdt_ops; + wdd->parent = dev; + + /* + * Although Gunyah expects 16-bit unsigned int values as timeout values + * in milliseconds, values above 0x8000 are reserved. This limits the + * max timeout value to 32 seconds. + */ + wdd->max_timeout = 32; /* seconds */ + wdd->min_timeout = 1; /* seconds */ + wdd->timeout = wdd->max_timeout; + + gunyah_wdt_stop(wdd); + platform_set_drvdata(pdev, wdd); + watchdog_set_restart_priority(wdd, 0); + + return devm_watchdog_register_device(dev, wdd); +} + +static void gunyah_wdt_remove(struct platform_device *pdev) +{ + struct watchdog_device *wdd = platform_get_drvdata(pdev); + + gunyah_wdt_stop(wdd); +} + +static int gunyah_wdt_suspend(struct device *dev) +{ + struct watchdog_device *wdd = dev_get_drvdata(dev); + + if (watchdog_active(wdd)) + gunyah_wdt_stop(wdd); + + return 0; +} + +static int gunyah_wdt_resume(struct device *dev) +{ + struct watchdog_device *wdd = dev_get_drvdata(dev); + + if (watchdog_active(wdd)) + gunyah_wdt_start(wdd); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(gunyah_wdt_pm_ops, gunyah_wdt_suspend, gunyah_wdt_resume); + +/* + * Gunyah watchdog is a vendor-specific hypervisor interface provided by the + * Gunyah hypervisor. Using QCOM SCM driver to detect Gunyah watchdog SMCCC + * hypervisor service and register platform device when the service is available + * allows this driver to operate independently of the devicetree and avoids + * adding the non-hardware nodes to the devicetree. + */ +static const struct platform_device_id gunyah_wdt_id[] = { + { .name = "gunyah-wdt" }, + {} +}; +MODULE_DEVICE_TABLE(platform, gunyah_wdt_id); + +static struct platform_driver gunyah_wdt_driver = { + .driver = { + .name = "gunyah-wdt", + .pm = pm_sleep_ptr(&gunyah_wdt_pm_ops), + }, + .id_table = gunyah_wdt_id, + .probe = gunyah_wdt_probe, + .remove = gunyah_wdt_remove, +}; + +module_platform_driver(gunyah_wdt_driver); + +MODULE_DESCRIPTION("Gunyah Watchdog Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c index 0f13a30533574..03479110453ce 100644 --- a/drivers/watchdog/imx7ulp_wdt.c +++ b/drivers/watchdog/imx7ulp_wdt.c @@ -346,6 +346,7 @@ static int imx7ulp_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(wdog); watchdog_stop_on_unregister(wdog); watchdog_set_drvdata(wdog, imx7ulp_wdt); + watchdog_set_nowayout(wdog, nowayout); imx7ulp_wdt->hw = of_device_get_match_data(dev); ret = imx7ulp_wdt_init(imx7ulp_wdt, wdog->timeout * imx7ulp_wdt->hw->wdog_clock_rate); diff --git a/drivers/watchdog/it87_wdt.c b/drivers/watchdog/it87_wdt.c index 3b8488c86a2f3..1d9f8591f38d8 100644 --- a/drivers/watchdog/it87_wdt.c +++ b/drivers/watchdog/it87_wdt.c @@ -188,6 +188,12 @@ static void _wdt_update_timeout(unsigned int t) superio_outb(t >> 8, WDTVALMSB); } +/* Internal function, should be called after superio_select(GPIO) */ +static bool _wdt_running(void) +{ + return superio_inb(WDTVALLSB) || (max_units > 255 && superio_inb(WDTVALMSB)); +} + static int wdt_update_timeout(unsigned int t) { int ret; @@ -374,6 +380,12 @@ static int __init it87_wdt_init(void) } } + /* wdt already left running by firmware? */ + if (_wdt_running()) { + pr_info("Left running by firmware.\n"); + set_bit(WDOG_HW_RUNNING, &wdt_dev.status); + } + superio_exit(); if (timeout < 1 || timeout > max_units * 60) { diff --git a/drivers/watchdog/rzv2h_wdt.c b/drivers/watchdog/rzv2h_wdt.c index a694786837e11..f9bb4ef3d327b 100644 --- a/drivers/watchdog/rzv2h_wdt.c +++ b/drivers/watchdog/rzv2h_wdt.c @@ -270,9 +270,7 @@ static int rzt2h_wdt_wdtdcr_init(struct platform_device *pdev, rzt2h_wdt_wdtdcr_count_stop(priv); - ret = pm_runtime_put(&pdev->dev); - if (ret < 0) - return ret; + pm_runtime_put(&pdev->dev); return 0; } diff --git a/drivers/watchdog/starfive-wdt.c b/drivers/watchdog/starfive-wdt.c index 355918d62f63d..af55adc4a3c69 100644 --- a/drivers/watchdog/starfive-wdt.c +++ b/drivers/watchdog/starfive-wdt.c @@ -446,7 +446,7 @@ static int starfive_wdt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, wdt); pm_runtime_enable(&pdev->dev); if (pm_runtime_enabled(&pdev->dev)) { - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; } else { @@ -500,12 +500,14 @@ static int starfive_wdt_probe(struct platform_device *pdev) if (pm_runtime_enabled(&pdev->dev)) { ret = pm_runtime_put_sync(&pdev->dev); if (ret) - goto err_exit; + goto err_unregister_wdt; } } return 0; +err_unregister_wdt: + watchdog_unregister_device(&wdt->wdd); err_exit: starfive_wdt_disable_clock(wdt); pm_runtime_disable(&pdev->dev); diff --git a/drivers/watchdog/via_wdt.c b/drivers/watchdog/via_wdt.c index d647923d68fed..f555763926518 100644 --- a/drivers/watchdog/via_wdt.c +++ b/drivers/watchdog/via_wdt.c @@ -165,6 +165,7 @@ static int wdt_probe(struct pci_dev *pdev, dev_err(&pdev->dev, "cannot enable PCI device\n"); return -ENODEV; } + wdt_res.name = "via_wdt"; /* * Allocate a MMIO region which contains watchdog control register diff --git a/drivers/watchdog/wdat_wdt.c b/drivers/watchdog/wdat_wdt.c index 650fdc7996e1c..dd3c2d69c9df1 100644 --- a/drivers/watchdog/wdat_wdt.c +++ b/drivers/watchdog/wdat_wdt.c @@ -326,19 +326,27 @@ static int wdat_wdt_probe(struct platform_device *pdev) return -ENODEV; wdat = devm_kzalloc(dev, sizeof(*wdat), GFP_KERNEL); - if (!wdat) - return -ENOMEM; + if (!wdat) { + ret = -ENOMEM; + goto out_put_table; + } regs = devm_kcalloc(dev, pdev->num_resources, sizeof(*regs), GFP_KERNEL); - if (!regs) - return -ENOMEM; + if (!regs) { + ret = -ENOMEM; + goto out_put_table; + } /* WDAT specification wants to have >= 1ms period */ - if (tbl->timer_period < 1) - return -EINVAL; - if (tbl->min_count > tbl->max_count) - return -EINVAL; + if (tbl->timer_period < 1) { + ret = -EINVAL; + goto out_put_table; + } + if (tbl->min_count > tbl->max_count) { + ret = -EINVAL; + goto out_put_table; + } wdat->period = tbl->timer_period; wdat->wdd.min_timeout = DIV_ROUND_UP(wdat->period * tbl->min_count, 1000); @@ -355,15 +363,20 @@ static int wdat_wdt_probe(struct platform_device *pdev) res = &pdev->resource[i]; if (resource_type(res) == IORESOURCE_MEM) { reg = devm_ioremap_resource(dev, res); - if (IS_ERR(reg)) - return PTR_ERR(reg); + if (IS_ERR(reg)) { + ret = PTR_ERR(reg); + goto out_put_table; + } } else if (resource_type(res) == IORESOURCE_IO) { reg = devm_ioport_map(dev, res->start, 1); - if (!reg) - return -ENOMEM; + if (!reg) { + ret = -ENOMEM; + goto out_put_table; + } } else { dev_err(dev, "Unsupported resource\n"); - return -EINVAL; + ret = -EINVAL; + goto out_put_table; } regs[i] = reg; @@ -385,8 +398,10 @@ static int wdat_wdt_probe(struct platform_device *pdev) } instr = devm_kzalloc(dev, sizeof(*instr), GFP_KERNEL); - if (!instr) - return -ENOMEM; + if (!instr) { + ret = -ENOMEM; + goto out_put_table; + } INIT_LIST_HEAD(&instr->node); instr->entry = entries[i]; @@ -417,7 +432,8 @@ static int wdat_wdt_probe(struct platform_device *pdev) if (!instr->reg) { dev_err(dev, "I/O resource not found\n"); - return -EINVAL; + ret = -EINVAL; + goto out_put_table; } instructions = wdat->instructions[action]; @@ -425,8 +441,10 @@ static int wdat_wdt_probe(struct platform_device *pdev) instructions = devm_kzalloc(dev, sizeof(*instructions), GFP_KERNEL); - if (!instructions) - return -ENOMEM; + if (!instructions) { + ret = -ENOMEM; + goto out_put_table; + } INIT_LIST_HEAD(instructions); wdat->instructions[action] = instructions; @@ -443,7 +461,7 @@ static int wdat_wdt_probe(struct platform_device *pdev) ret = wdat_wdt_enable_reboot(wdat); if (ret) - return ret; + goto out_put_table; platform_set_drvdata(pdev, wdat); @@ -460,12 +478,16 @@ static int wdat_wdt_probe(struct platform_device *pdev) ret = wdat_wdt_set_timeout(&wdat->wdd, timeout); if (ret) - return ret; + goto out_put_table; watchdog_set_nowayout(&wdat->wdd, nowayout); watchdog_stop_on_reboot(&wdat->wdd); watchdog_stop_on_unregister(&wdat->wdd); - return devm_watchdog_register_device(dev, &wdat->wdd); + ret = devm_watchdog_register_device(dev, &wdat->wdd); + +out_put_table: + acpi_put_table((struct acpi_table_header *)tbl); + return ret; } static int wdat_wdt_suspend_noirq(struct device *dev) diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 49c3f99263943..8c44a25a7d2b9 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -724,6 +724,7 @@ static int __init balloon_add_regions(void) static int __init balloon_init(void) { struct task_struct *task; + unsigned long current_pages; int rc; if (!xen_domain()) @@ -731,12 +732,18 @@ static int __init balloon_init(void) pr_info("Initialising balloon driver\n"); - if (xen_released_pages >= get_num_physpages()) { - WARN(1, "Released pages underflow current target"); - return -ERANGE; + if (xen_pv_domain()) { + if (xen_released_pages >= xen_start_info->nr_pages) + goto underflow; + current_pages = min(xen_start_info->nr_pages - + xen_released_pages, max_pfn); + } else { + if (xen_unpopulated_pages >= get_num_physpages()) + goto underflow; + current_pages = get_num_physpages() - xen_unpopulated_pages; } - balloon_stats.current_pages = get_num_physpages() - xen_released_pages; + balloon_stats.current_pages = current_pages; balloon_stats.target_pages = balloon_stats.current_pages; balloon_stats.balloon_low = 0; balloon_stats.balloon_high = 0; @@ -767,6 +774,10 @@ static int __init balloon_init(void) xen_balloon_init(); return 0; + + underflow: + WARN(1, "Released pages underflow current target"); + return -ERANGE; } subsys_initcall(balloon_init); diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 29257d2639dbf..f3b7529461172 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -315,14 +315,13 @@ static int xen_dt_grant_init_backend_domid(struct device *dev, struct device_node *np, domid_t *backend_domid) { - struct of_phandle_args iommu_spec = { .args_count = 1 }; + struct of_phandle_args iommu_spec = {}; if (dev_is_pci(dev)) { struct pci_dev *pdev = to_pci_dev(dev); u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn); - if (of_map_id(np, rid, "iommu-map", "iommu-map-mask", &iommu_spec.np, - iommu_spec.args)) { + if (of_map_iommu_id(np, rid, &iommu_spec)) { dev_dbg(dev, "Cannot translate ID\n"); return -ESRCH; } @@ -362,7 +361,8 @@ static int xen_grant_init_backend_domid(struct device *dev, if (np) { ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid); of_node_put(np); - } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) { + } else if (!xen_initial_domain() && + (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain())) { dev_info(dev, "Using dom0 as backend\n"); *backend_domid = 0; ret = 0; diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index f52a457b302d9..cbc62f0df11b7 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,10 @@ #include #include #include +#include +#include #include +#include #include #include @@ -46,6 +50,7 @@ #include #include #include +#include #ifdef CONFIG_XEN_ACPI #include #endif @@ -68,10 +73,20 @@ module_param_named(dm_op_buf_max_size, privcmd_dm_op_buf_max_size, uint, MODULE_PARM_DESC(dm_op_buf_max_size, "Maximum size of a dm_op hypercall buffer"); +static bool unrestricted; +module_param(unrestricted, bool, 0); +MODULE_PARM_DESC(unrestricted, + "Don't restrict hypercalls to target domain if running in a domU"); + struct privcmd_data { domid_t domid; }; +/* DOMID_INVALID implies no restriction */ +static domid_t target_domain = DOMID_INVALID; +static bool restrict_wait; +static DECLARE_WAIT_QUEUE_HEAD(restrict_wait_wq); + static int privcmd_vma_range_is_mapped( struct vm_area_struct *vma, unsigned long addr, @@ -1562,13 +1577,16 @@ static long privcmd_ioctl(struct file *file, static int privcmd_open(struct inode *ino, struct file *file) { - struct privcmd_data *data = kzalloc(sizeof(*data), GFP_KERNEL); + struct privcmd_data *data; + if (wait_event_interruptible(restrict_wait_wq, !restrict_wait) < 0) + return -EINTR; + + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; - /* DOMID_INVALID implies no restriction */ - data->domid = DOMID_INVALID; + data->domid = target_domain; file->private_data = data; return 0; @@ -1661,6 +1679,52 @@ static struct miscdevice privcmd_dev = { .fops = &xen_privcmd_fops, }; +static int init_restrict(struct notifier_block *notifier, + unsigned long event, + void *data) +{ + char *target; + unsigned int domid; + + /* Default to an guaranteed unused domain-id. */ + target_domain = DOMID_IDLE; + + target = xenbus_read(XBT_NIL, "target", "", NULL); + if (IS_ERR(target) || kstrtouint(target, 10, &domid)) { + pr_err("No target domain found, blocking all hypercalls\n"); + goto out; + } + + target_domain = domid; + + out: + if (!IS_ERR(target)) + kfree(target); + + restrict_wait = false; + wake_up_all(&restrict_wait_wq); + + return NOTIFY_DONE; +} + +static struct notifier_block xenstore_notifier = { + .notifier_call = init_restrict, +}; + +static void __init restrict_driver(void) +{ + if (unrestricted) { + if (security_locked_down(LOCKDOWN_XEN_USER_ACTIONS)) + pr_warn("Kernel is locked down, parameter \"unrestricted\" ignored\n"); + else + return; + } + + restrict_wait = true; + + register_xenstore_notifier(&xenstore_notifier); +} + static int __init privcmd_init(void) { int err; @@ -1668,6 +1732,9 @@ static int __init privcmd_init(void) if (!xen_domain()) return -ENODEV; + if (!xen_initial_domain()) + restrict_driver(); + err = misc_register(&privcmd_dev); if (err != 0) { pr_err("Could not register Xen privcmd device\n"); @@ -1697,6 +1764,9 @@ static int __init privcmd_init(void) static void __exit privcmd_exit(void) { + if (!xen_initial_domain()) + unregister_xenstore_notifier(&xenstore_notifier); + privcmd_ioeventfd_exit(); privcmd_irqfd_exit(); misc_deregister(&privcmd_dev); diff --git a/drivers/xen/unpopulated-alloc.c b/drivers/xen/unpopulated-alloc.c index d6fc2aefe2646..1dc0b495c8e59 100644 --- a/drivers/xen/unpopulated-alloc.c +++ b/drivers/xen/unpopulated-alloc.c @@ -18,6 +18,9 @@ static unsigned int list_count; static struct resource *target_resource; +/* Pages to subtract from the memory count when setting balloon target. */ +unsigned long xen_unpopulated_pages __initdata; + /* * If arch is not happy with system "iomem_resource" being used for * the region allocation it can provide it's own view by creating specific diff --git a/drivers/xen/xen-acpi-processor.c b/drivers/xen/xen-acpi-processor.c index 2967039398463..520756159d3d3 100644 --- a/drivers/xen/xen-acpi-processor.c +++ b/drivers/xen/xen-acpi-processor.c @@ -379,11 +379,8 @@ read_acpi_id(acpi_handle handle, u32 lvl, void *context, void **rv) acpi_psd[acpi_id].domain); } - status = acpi_evaluate_object(handle, "_CST", NULL, &buffer); - if (ACPI_FAILURE(status)) { - if (!pblk) - return AE_OK; - } + if (!pblk && !acpi_has_method(handle, "_CST")) + return AE_OK; /* .. and it has a C-state */ __set_bit(acpi_id, acpi_id_cst_present); diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c index 0c51edfd13dcd..7d5117e5efe0e 100644 --- a/drivers/xen/xen-scsiback.c +++ b/drivers/xen/xen-scsiback.c @@ -1262,6 +1262,7 @@ static void scsiback_remove(struct xenbus_device *dev) gnttab_page_cache_shrink(&info->free_pages, 0); dev_set_drvdata(&dev->dev, NULL); + kfree(info); } static int scsiback_probe(struct xenbus_device *dev, diff --git a/drivers/xen/xenbus/xenbus_probe_frontend.c b/drivers/xen/xenbus/xenbus_probe_frontend.c index 6d1819269cbe5..199917b6f77ca 100644 --- a/drivers/xen/xenbus/xenbus_probe_frontend.c +++ b/drivers/xen/xenbus/xenbus_probe_frontend.c @@ -148,11 +148,9 @@ static void xenbus_frontend_dev_shutdown(struct device *_dev) } static const struct dev_pm_ops xenbus_pm_ops = { - .suspend = xenbus_dev_suspend, - .resume = xenbus_frontend_dev_resume, .freeze = xenbus_dev_suspend, .thaw = xenbus_dev_cancel, - .restore = xenbus_dev_resume, + .restore = xenbus_frontend_dev_resume, }; static struct xen_bus_type xenbus_frontend = { diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index a020a8f00a1ac..bde3ffb0e319a 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -101,7 +101,7 @@ int v9fs_show_options(struct seq_file *m, struct dentry *root) struct v9fs_session_info *v9ses = root->d_sb->s_fs_info; if (v9ses->debug) - seq_printf(m, ",debug=%x", v9ses->debug); + seq_printf(m, ",debug=%#x", v9ses->debug); if (!uid_eq(v9ses->dfltuid, V9FS_DEFUID)) seq_printf(m, ",dfltuid=%u", from_kuid_munged(&init_user_ns, v9ses->dfltuid)); @@ -117,7 +117,7 @@ int v9fs_show_options(struct seq_file *m, struct dentry *root) if (v9ses->nodev) seq_puts(m, ",nodevmap"); if (v9ses->cache) - seq_printf(m, ",cache=%x", v9ses->cache); + seq_printf(m, ",cache=%#x", v9ses->cache); #ifdef CONFIG_9P_FSCACHE if (v9ses->cachetag && (v9ses->cache & CACHE_FSCACHE)) seq_printf(m, ",cachetag=%s", v9ses->cachetag); diff --git a/fs/9p/vfs_file.c b/fs/9p/vfs_file.c index eb0b083da269b..d1db03093d4c3 100644 --- a/fs/9p/vfs_file.c +++ b/fs/9p/vfs_file.c @@ -43,14 +43,18 @@ int v9fs_file_open(struct inode *inode, struct file *file) struct v9fs_session_info *v9ses; struct p9_fid *fid; int omode; + int o_append; p9_debug(P9_DEBUG_VFS, "inode: %p file: %p\n", inode, file); v9ses = v9fs_inode2v9ses(inode); - if (v9fs_proto_dotl(v9ses)) + if (v9fs_proto_dotl(v9ses)) { omode = v9fs_open_to_dotl_flags(file->f_flags); - else + o_append = P9_DOTL_APPEND; + } else { omode = v9fs_uflags2omode(file->f_flags, v9fs_proto_dotu(v9ses)); + o_append = P9_OAPPEND; + } fid = file->private_data; if (!fid) { fid = v9fs_fid_clone(file_dentry(file)); @@ -58,9 +62,10 @@ int v9fs_file_open(struct inode *inode, struct file *file) return PTR_ERR(fid); if ((v9ses->cache & CACHE_WRITEBACK) && (omode & P9_OWRITE)) { - int writeback_omode = (omode & ~P9_OWRITE) | P9_ORDWR; + int writeback_omode = (omode & ~(P9_OWRITE | o_append)) | P9_ORDWR; p9_debug(P9_DEBUG_CACHE, "write-only file with writeback enabled, try opening O_RDWR\n"); + err = p9_client_open(fid, writeback_omode); if (err < 0) { p9_debug(P9_DEBUG_CACHE, "could not open O_RDWR, disabling caches\n"); diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index d0c77ec31b1dd..0f3189a0a516a 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -786,7 +786,7 @@ v9fs_vfs_atomic_open(struct inode *dir, struct dentry *dentry, p9_omode = v9fs_uflags2omode(flags, v9fs_proto_dotu(v9ses)); if ((v9ses->cache & CACHE_WRITEBACK) && (p9_omode & P9_OWRITE)) { - p9_omode = (p9_omode & ~P9_OWRITE) | P9_ORDWR; + p9_omode = (p9_omode & ~(P9_OWRITE | P9_OAPPEND)) | P9_ORDWR; p9_debug(P9_DEBUG_CACHE, "write-only file with writeback enabled, creating w/ O_RDWR\n"); } @@ -1393,4 +1393,3 @@ static const struct inode_operations v9fs_symlink_inode_operations = { .getattr = v9fs_vfs_getattr, .setattr = v9fs_vfs_setattr, }; - diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index be297e3354688..6312b3590f743 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -282,7 +282,7 @@ v9fs_vfs_atomic_open_dotl(struct inode *dir, struct dentry *dentry, } if ((v9ses->cache & CACHE_WRITEBACK) && (p9_omode & P9_OWRITE)) { - p9_omode = (p9_omode & ~P9_OWRITE) | P9_ORDWR; + p9_omode = (p9_omode & ~(P9_OWRITE | P9_DOTL_APPEND)) | P9_ORDWR; p9_debug(P9_DEBUG_CACHE, "write-only file with writeback enabled, creating w/ O_RDWR\n"); } diff --git a/fs/afs/addr_list.c b/fs/afs/addr_list.c index e941da5b6dd92..b1704de3d95f5 100644 --- a/fs/afs/addr_list.c +++ b/fs/afs/addr_list.c @@ -298,8 +298,8 @@ int afs_merge_fs_addr4(struct afs_net *net, struct afs_addr_list *alist, srx.transport.sin.sin_addr.s_addr = xdr; peer = rxrpc_kernel_lookup_peer(net->socket, &srx, GFP_KERNEL); - if (!peer) - return -ENOMEM; + if (IS_ERR(peer)) + return PTR_ERR(peer); for (i = 0; i < alist->nr_ipv4; i++) { if (peer == alist->addrs[i].peer) { @@ -342,8 +342,8 @@ int afs_merge_fs_addr6(struct afs_net *net, struct afs_addr_list *alist, memcpy(&srx.transport.sin6.sin6_addr, xdr, 16); peer = rxrpc_kernel_lookup_peer(net->socket, &srx, GFP_KERNEL); - if (!peer) - return -ENOMEM; + if (IS_ERR(peer)) + return PTR_ERR(peer); for (i = alist->nr_ipv4; i < alist->nr_addrs; i++) { if (peer == alist->addrs[i].peer) { diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index 48fd2de3bca05..a3d4e6973b299 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -595,6 +595,12 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, #ifdef ELF_HWCAP2 nitems++; #endif +#ifdef ELF_HWCAP3 + nitems++; +#endif +#ifdef ELF_HWCAP4 + nitems++; +#endif csp = sp; sp -= nitems * 2 * sizeof(unsigned long); diff --git a/fs/btrfs/accessors.h b/fs/btrfs/accessors.h index 99b3ced12805b..78721412951c5 100644 --- a/fs/btrfs/accessors.h +++ b/fs/btrfs/accessors.h @@ -12,6 +12,7 @@ #include #include #include +#include "fs.h" #include "extent_io.h" struct extent_buffer; diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 21df48e6c4fa2..7d8aaa019c8ce 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -41,13 +41,17 @@ static bool bbio_has_ordered_extent(const struct btrfs_bio *bbio) * Initialize a btrfs_bio structure. This skips the embedded bio itself as it * is already initialized by the block layer. */ -void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info, +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private) { + /* @inode parameter is mandatory. */ + ASSERT(inode); + memset(bbio, 0, offsetof(struct btrfs_bio, bio)); - bbio->fs_info = fs_info; + bbio->inode = inode; bbio->end_io = end_io; bbio->private = private; + bbio->file_offset = file_offset; atomic_set(&bbio->pending_ios, 1); WRITE_ONCE(bbio->status, BLK_STS_OK); } @@ -60,7 +64,7 @@ void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info, * a mempool. */ struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, - struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private) { struct btrfs_bio *bbio; @@ -68,7 +72,7 @@ struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset); bbio = btrfs_bio(bio); - btrfs_bio_init(bbio, fs_info, end_io, private); + btrfs_bio_init(bbio, inode, file_offset, end_io, private); return bbio; } @@ -85,9 +89,7 @@ static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info, return ERR_CAST(bio); bbio = btrfs_bio(bio); - btrfs_bio_init(bbio, fs_info, NULL, orig_bbio); - bbio->inode = orig_bbio->inode; - bbio->file_offset = orig_bbio->file_offset; + btrfs_bio_init(bbio, orig_bbio->inode, orig_bbio->file_offset, NULL, orig_bbio); orig_bbio->file_offset += map_length; if (bbio_has_ordered_extent(bbio)) { refcount_inc(&orig_bbio->ordered->refs); @@ -100,6 +102,12 @@ static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info, void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) { + /* Make sure we're already in task context. */ + ASSERT(in_task()); + + if (bbio->async_csum) + wait_for_completion(&bbio->csum_done); + bbio->bio.bi_status = status; if (bbio->bio.bi_pool == &btrfs_clone_bioset) { struct btrfs_bio *orig_bbio = bbio->private; @@ -244,9 +252,8 @@ static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, bio_add_folio_nofail(repair_bio, folio, sectorsize, foff); repair_bbio = btrfs_bio(repair_bio); - btrfs_bio_init(repair_bbio, fs_info, NULL, fbio); - repair_bbio->inode = failed_bbio->inode; - repair_bbio->file_offset = failed_bbio->file_offset + bio_offset; + btrfs_bio_init(repair_bbio, failed_bbio->inode, failed_bbio->file_offset + bio_offset, + NULL, fbio); mirror = next_repair_mirror(fbio, failed_bbio->mirror_num); btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror); @@ -317,36 +324,35 @@ static struct workqueue_struct *btrfs_end_io_wq(const struct btrfs_fs_info *fs_i return fs_info->endio_workers; } -static void btrfs_end_bio_work(struct work_struct *work) +static void simple_end_io_work(struct work_struct *work) { struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; - /* Metadata reads are checked and repaired by the submitter. */ - if (is_data_bbio(bbio)) - btrfs_check_read_bio(bbio, bbio->bio.bi_private); - else - btrfs_bio_end_io(bbio, bbio->bio.bi_status); + if (bio_op(bio) == REQ_OP_READ) { + /* Metadata reads are checked and repaired by the submitter. */ + if (is_data_bbio(bbio)) + return btrfs_check_read_bio(bbio, bbio->bio.bi_private); + return btrfs_bio_end_io(bbio, bbio->bio.bi_status); + } + if (bio_is_zone_append(bio) && !bio->bi_status) + btrfs_record_physical_zoned(bbio); + btrfs_bio_end_io(bbio, bbio->bio.bi_status); } static void btrfs_simple_end_io(struct bio *bio) { struct btrfs_bio *bbio = btrfs_bio(bio); struct btrfs_device *dev = bio->bi_private; - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; btrfs_bio_counter_dec(fs_info); if (bio->bi_status) btrfs_log_dev_io_error(bio, dev); - if (bio_op(bio) == REQ_OP_READ) { - INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work); - queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); - } else { - if (bio_is_zone_append(bio) && !bio->bi_status) - btrfs_record_physical_zoned(bbio); - btrfs_bio_end_io(bbio, bbio->bio.bi_status); - } + INIT_WORK(&bbio->end_io_work, simple_end_io_work); + queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); } static void btrfs_raid56_end_io(struct bio *bio) @@ -354,6 +360,9 @@ static void btrfs_raid56_end_io(struct bio *bio) struct btrfs_io_context *bioc = bio->bi_private; struct btrfs_bio *bbio = btrfs_bio(bio); + /* RAID56 endio is always handled in workqueue. */ + ASSERT(in_task()); + btrfs_bio_counter_dec(bioc->fs_info); bbio->mirror_num = bioc->mirror_num; if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio)) @@ -364,11 +373,12 @@ static void btrfs_raid56_end_io(struct bio *bio) btrfs_put_bioc(bioc); } -static void btrfs_orig_write_end_io(struct bio *bio) +static void orig_write_end_io_work(struct work_struct *work) { + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; struct btrfs_io_stripe *stripe = bio->bi_private; struct btrfs_io_context *bioc = stripe->bioc; - struct btrfs_bio *bbio = btrfs_bio(bio); btrfs_bio_counter_dec(bioc->fs_info); @@ -393,8 +403,18 @@ static void btrfs_orig_write_end_io(struct bio *bio) btrfs_put_bioc(bioc); } -static void btrfs_clone_write_end_io(struct bio *bio) +static void btrfs_orig_write_end_io(struct bio *bio) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + + INIT_WORK(&bbio->end_io_work, orig_write_end_io_work); + queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); +} + +static void clone_write_end_io_work(struct work_struct *work) { + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; struct btrfs_io_stripe *stripe = bio->bi_private; if (bio->bi_status) { @@ -409,8 +429,18 @@ static void btrfs_clone_write_end_io(struct bio *bio) bio_put(bio); } +static void btrfs_clone_write_end_io(struct bio *bio) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + + INIT_WORK(&bbio->end_io_work, clone_write_end_io_work); + queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); +} + static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) { + u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || (btrfs_op(bio) == BTRFS_MAP_WRITE && @@ -425,12 +455,13 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) * For zone append writing, bi_sector must point the beginning of the * zone */ - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (btrfs_bio(bio)->can_use_append && btrfs_dev_is_sequential(dev, physical)) { u64 zone_start = round_down(physical, dev->fs_info->zone_size); ASSERT(btrfs_dev_is_sequential(dev, physical)); bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; + bio->bi_opf &= ~REQ_OP_WRITE; + bio->bi_opf |= REQ_OP_ZONE_APPEND; } btrfs_debug(dev->fs_info, "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", @@ -455,6 +486,7 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) { struct bio *orig_bio = bioc->orig_bio, *bio; + struct btrfs_bio *orig_bbio = btrfs_bio(orig_bio); ASSERT(bio_op(orig_bio) != REQ_OP_READ); @@ -463,8 +495,11 @@ static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) bio = orig_bio; bio->bi_end_io = btrfs_orig_write_end_io; } else { - bio = bio_alloc_clone(NULL, orig_bio, GFP_NOFS, &fs_bio_set); + /* We need to use endio_work to run end_io in task context. */ + bio = bio_alloc_clone(NULL, orig_bio, GFP_NOFS, &btrfs_bioset); bio_inc_remaining(orig_bio); + btrfs_bio_init(btrfs_bio(bio), orig_bbio->inode, + orig_bbio->file_offset, NULL, NULL); bio->bi_end_io = btrfs_clone_write_end_io; } @@ -509,7 +544,11 @@ static int btrfs_bio_csum(struct btrfs_bio *bbio) { if (bbio->bio.bi_opf & REQ_META) return btree_csum_one_bio(bbio); - return btrfs_csum_one_bio(bbio); +#ifdef CONFIG_BTRFS_EXPERIMENTAL + return btrfs_csum_one_bio(bbio, true); +#else + return btrfs_csum_one_bio(bbio, false); +#endif } /* @@ -581,20 +620,25 @@ static void run_one_async_done(struct btrfs_work *work, bool do_free) static bool should_async_write(struct btrfs_bio *bbio) { + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; bool auto_csum_mode = true; #ifdef CONFIG_BTRFS_EXPERIMENTAL - struct btrfs_fs_devices *fs_devices = bbio->fs_info->fs_devices; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; enum btrfs_offload_csum_mode csum_mode = READ_ONCE(fs_devices->offload_csum_mode); - if (csum_mode == BTRFS_OFFLOAD_CSUM_FORCE_OFF) - return false; - - auto_csum_mode = (csum_mode == BTRFS_OFFLOAD_CSUM_AUTO); + if (csum_mode == BTRFS_OFFLOAD_CSUM_FORCE_ON) + return true; + /* + * Write bios will calculate checksum and submit bio at the same time. + * Unless explicitly required don't offload serial csum calculate and bio + * submit into a workqueue. + */ + return false; #endif /* Submit synchronously if the checksum implementation is fast. */ - if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &bbio->fs_info->flags)) + if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) return false; /* @@ -605,7 +649,7 @@ static bool should_async_write(struct btrfs_bio *bbio) return false; /* Zoned devices require I/O to be submitted in order. */ - if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(bbio->fs_info)) + if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(fs_info)) return false; return true; @@ -620,7 +664,7 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, struct btrfs_io_context *bioc, struct btrfs_io_stripe *smap, int mirror_num) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct async_submit_bio *async; async = kmalloc(sizeof(*async), GFP_NOFS); @@ -639,11 +683,12 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) { + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; unsigned int nr_segs; int sector_offset; - map_length = min(map_length, bbio->fs_info->max_zone_append_size); - sector_offset = bio_split_rw_at(&bbio->bio, &bbio->fs_info->limits, + map_length = min(map_length, fs_info->max_zone_append_size); + sector_offset = bio_split_rw_at(&bbio->bio, &fs_info->limits, &nr_segs, map_length); if (sector_offset) { /* @@ -651,7 +696,7 @@ static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) * sectorsize and thus cause unaligned I/Os. Fix that by * always rounding down to the nearest boundary. */ - return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, bbio->fs_info->sectorsize); + return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, fs_info->sectorsize); } return map_length; } @@ -659,18 +704,17 @@ static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) { struct btrfs_inode *inode = bbio->inode; - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct bio *bio = &bbio->bio; u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; u64 length = bio->bi_iter.bi_size; u64 map_length = length; - bool use_append = btrfs_use_zone_append(bbio); struct btrfs_io_context *bioc = NULL; struct btrfs_io_stripe smap; blk_status_t status; int ret; - if (!bbio->inode || btrfs_is_data_reloc_root(inode->root)) + if (bbio->is_scrub || btrfs_is_data_reloc_root(inode->root)) smap.rst_search_commit_root = true; else smap.rst_search_commit_root = false; @@ -684,8 +728,10 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) goto end_bbio; } + bbio->can_use_append = btrfs_use_zone_append(bbio); + map_length = min(map_length, length); - if (use_append) + if (bbio->can_use_append) map_length = btrfs_append_map_length(bbio, map_length); if (map_length < length) { @@ -714,11 +760,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) } if (btrfs_op(bio) == BTRFS_MAP_WRITE) { - if (use_append) { - bio->bi_opf &= ~REQ_OP_WRITE; - bio->bi_opf |= REQ_OP_ZONE_APPEND; - } - if (is_data_bbio(bbio) && bioc && bioc->use_rst) { /* * No locking for the list update, as we only add to @@ -734,7 +775,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) * Csum items for reloc roots have already been cloned at this * point, so they are handled as part of the no-checksum case. */ - if (inode && !(inode->flags & BTRFS_INODE_NODATASUM) && + if (!(inode->flags & BTRFS_INODE_NODATASUM) && !test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state) && !btrfs_is_data_reloc_root(inode->root)) { if (should_async_write(bbio) && @@ -745,7 +786,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) status = errno_to_blk_status(ret); if (status) goto fail; - } else if (use_append || + } else if (bbio->can_use_append || (btrfs_is_zoned(fs_info) && inode && inode->flags & BTRFS_INODE_NODATASUM)) { ret = btrfs_alloc_dummy_sum(bbio); @@ -782,7 +823,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) static void assert_bbio_alignment(struct btrfs_bio *bbio) { #ifdef CONFIG_BTRFS_ASSERT - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct bio_vec bvec; struct bvec_iter iter; const u32 blocksize = fs_info->sectorsize; @@ -885,16 +926,16 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, */ void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; u64 length = bbio->bio.bi_iter.bi_size; struct btrfs_io_stripe smap = { 0 }; int ret; - ASSERT(fs_info); ASSERT(mirror_num > 0); ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE); - ASSERT(!bbio->inode); + ASSERT(!is_data_inode(bbio->inode)); + ASSERT(bbio->is_scrub); btrfs_bio_counter_inc_blocked(fs_info); ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num); diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 00883aea55d70..69fe54f564fcb 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -34,7 +34,10 @@ typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio); struct btrfs_bio { /* * Inode and offset into it that this I/O operates on. - * Only set for data I/O. + * + * If the inode is a data one, csum verification and read-repair + * will be done automatically. + * If the inode is a metadata one, everything is handled by the caller. */ struct btrfs_inode *inode; u64 file_offset; @@ -60,6 +63,9 @@ struct btrfs_bio { struct { struct btrfs_ordered_extent *ordered; struct btrfs_ordered_sum *sums; + struct work_struct csum_work; + struct completion csum_done; + struct bvec_iter csum_saved_iter; u64 orig_physical; }; @@ -76,14 +82,24 @@ struct btrfs_bio { atomic_t pending_ios; struct work_struct end_io_work; - /* File system that this I/O operates on. */ - struct btrfs_fs_info *fs_info; - /* Save the first error status of split bio. */ blk_status_t status; /* Use the commit root to look up csums (data read bio only). */ bool csum_search_commit_root; + + /* + * Since scrub will reuse btree inode, we need this flag to distinguish + * scrub bios. + */ + bool is_scrub; + + /* Whether the csum generation for data write is async. */ + bool async_csum; + + /* Whether the bio is written using zone append. */ + bool can_use_append; + /* * This member must come last, bio_alloc_bioset will allocate enough * bytes for entire btrfs_bio but relies on bio being last. @@ -99,10 +115,10 @@ static inline struct btrfs_bio *btrfs_bio(struct bio *bio) int __init btrfs_bioset_init(void); void __cold btrfs_bioset_exit(void); -void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info, +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private); struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, - struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private); void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status); diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 5322ef2ae015e..a277c8cc91661 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1877,6 +1877,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) while (!list_empty(&fs_info->reclaim_bgs)) { u64 used; u64 reserved; + u64 old_total; int ret = 0; bg = list_first_entry(&fs_info->reclaim_bgs, @@ -1942,6 +1943,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) } spin_unlock(&bg->lock); + old_total = space_info->total_bytes; spin_unlock(&space_info->lock); /* @@ -1994,14 +1996,14 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) reserved = 0; spin_lock(&space_info->lock); space_info->reclaim_errors++; - if (READ_ONCE(space_info->periodic_reclaim)) - space_info->periodic_reclaim_ready = false; spin_unlock(&space_info->lock); } spin_lock(&space_info->lock); space_info->reclaim_count++; space_info->reclaim_bytes += used; space_info->reclaim_bytes += reserved; + if (space_info->total_bytes < old_total) + btrfs_set_periodic_reclaim_ready(space_info, true); spin_unlock(&space_info->lock); next: @@ -3681,6 +3683,14 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) return ret; } +static void btrfs_maybe_reset_size_class(struct btrfs_block_group *bg) +{ + lockdep_assert_held(&bg->lock); + if (btrfs_block_group_should_use_size_class(bg) && + bg->used == 0 && bg->reserved == 0) + bg->size_class = BTRFS_BG_SZ_NONE; +} + int btrfs_update_block_group(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, bool alloc) { @@ -3745,6 +3755,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, old_val -= num_bytes; cache->used = old_val; cache->pinned += num_bytes; + btrfs_maybe_reset_size_class(cache); btrfs_space_info_update_bytes_pinned(space_info, num_bytes); space_info->bytes_used -= num_bytes; space_info->disk_used -= num_bytes * factor; @@ -3836,7 +3847,7 @@ int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, * that happens. */ if (num_bytes < ram_bytes) - btrfs_try_granting_tickets(cache->fs_info, space_info); + btrfs_try_granting_tickets(space_info); out: spin_unlock(&cache->lock); spin_unlock(&space_info->lock); @@ -3859,22 +3870,26 @@ void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, u64 num_bytes, bool is_delalloc) { struct btrfs_space_info *space_info = cache->space_info; + bool bg_ro; spin_lock(&space_info->lock); spin_lock(&cache->lock); - if (cache->ro) + bg_ro = cache->ro; + cache->reserved -= num_bytes; + btrfs_maybe_reset_size_class(cache); + if (is_delalloc) + cache->delalloc_bytes -= num_bytes; + spin_unlock(&cache->lock); + + if (bg_ro) space_info->bytes_readonly += num_bytes; else if (btrfs_is_zoned(cache->fs_info)) space_info->bytes_zone_unusable += num_bytes; - cache->reserved -= num_bytes; + space_info->bytes_reserved -= num_bytes; space_info->max_extent_size = 0; - if (is_delalloc) - cache->delalloc_bytes -= num_bytes; - spin_unlock(&cache->lock); - - btrfs_try_granting_tickets(cache->fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } @@ -4215,7 +4230,7 @@ int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, mutex_unlock(&fs_info->chunk_mutex); } else { /* Proceed with allocation */ - space_info->chunk_alloc = 1; + space_info->chunk_alloc = true; wait_for_alloc = false; spin_unlock(&space_info->lock); } @@ -4264,7 +4279,7 @@ int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, spin_lock(&space_info->lock); if (ret < 0) { if (ret == -ENOSPC) - space_info->full = 1; + space_info->full = true; else goto out; } else { @@ -4274,7 +4289,7 @@ int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; out: - space_info->chunk_alloc = 0; + space_info->chunk_alloc = false; spin_unlock(&space_info->lock); mutex_unlock(&fs_info->chunk_mutex); @@ -4449,7 +4464,7 @@ static void check_removing_space_info(struct btrfs_space_info *space_info) for (int i = 0; i < BTRFS_SPACE_INFO_SUB_GROUP_MAX; i++) { if (space_info->sub_group[i]) { check_removing_space_info(space_info->sub_group[i]); - kfree(space_info->sub_group[i]); + btrfs_sysfs_remove_space_info(space_info->sub_group[i]); space_info->sub_group[i] = NULL; } } diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index 5ad6de738aee0..fc378d2038a2d 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -278,10 +278,11 @@ u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, struct btrfs_block_rsv *target = NULL; /* - * If we are a delayed block reserve then push to the global rsv, - * otherwise dump into the global delayed reserve if it is not full. + * If we are a delayed refs block reserve then push to the global + * reserve, otherwise dump into the global delayed refs reserve if it is + * not full. */ - if (block_rsv->type == BTRFS_BLOCK_RSV_DELOPS) + if (block_rsv->type == BTRFS_BLOCK_RSV_DELREFS) target = global_rsv; else if (block_rsv != global_rsv && !btrfs_block_rsv_full(delayed_rsv)) target = delayed_rsv; @@ -387,7 +388,7 @@ void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info) num_bytes = block_rsv->reserved - block_rsv->size; btrfs_space_info_update_bytes_may_use(sinfo, -num_bytes); block_rsv->reserved = block_rsv->size; - btrfs_try_granting_tickets(fs_info, sinfo); + btrfs_try_granting_tickets(sinfo); } block_rsv->full = (block_rsv->reserved == block_rsv->size); diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index af373d50a901f..a66ca5531b5c5 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -18,20 +18,20 @@ #include #include #include +#include "ctree.h" #include "block-rsv.h" #include "extent_map.h" -#include "extent_io.h" #include "extent-io-tree.h" -#include "ordered-data.h" -#include "delayed-inode.h" -struct extent_state; struct posix_acl; struct iov_iter; struct writeback_control; struct btrfs_root; struct btrfs_fs_info; struct btrfs_trans_handle; +struct btrfs_bio; +struct btrfs_file_extent; +struct btrfs_delayed_node; /* * Since we search a directory based on f_pos (struct dir_context::pos) we have diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index bacad18357b33..8c3899832a1aa 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -67,9 +67,7 @@ static struct compressed_bio *alloc_compressed_bio(struct btrfs_inode *inode, bbio = btrfs_bio(bio_alloc_bioset(NULL, BTRFS_MAX_COMPRESSED_PAGES, op, GFP_NOFS, &btrfs_compressed_bioset)); - btrfs_bio_init(bbio, inode->root->fs_info, end_io, NULL); - bbio->inode = inode; - bbio->file_offset = start; + btrfs_bio_init(bbio, inode, start, end_io, NULL); return to_compressed_bio(bbio); } @@ -354,7 +352,7 @@ static void end_bbio_compressed_write(struct btrfs_bio *bbio) static void btrfs_add_compressed_bio_folios(struct compressed_bio *cb) { - struct btrfs_fs_info *fs_info = cb->bbio.fs_info; + struct btrfs_fs_info *fs_info = cb->bbio.inode->root->fs_info; struct bio *bio = &cb->bbio.bio; u32 offset = 0; diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index eba188a9e3bb5..062ebd9c2d32d 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -14,14 +14,12 @@ #include #include "bio.h" #include "fs.h" -#include "messages.h" +#include "btrfs_inode.h" struct address_space; -struct page; struct inode; struct btrfs_inode; struct btrfs_ordered_extent; -struct btrfs_bio; /* * We want to make sure that amount of RAM required to uncompress an extent is @@ -77,7 +75,7 @@ struct compressed_bio { static inline struct btrfs_fs_info *cb_to_fs_info(const struct compressed_bio *cb) { - return cb->bbio.fs_info; + return cb->bbio.inode->root->fs_info; } /* @range_end must be exclusive. */ diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 561658aca018b..6e053caa6e101 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -4566,9 +4566,9 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, if (btrfs_header_nritems(leaf) == 0) { path->slots[1] = slot; ret = btrfs_del_leaf(trans, root, path, leaf); + free_extent_buffer(leaf); if (ret < 0) return ret; - free_extent_buffer(leaf); ret = 0; } else { /* if we're still in the path, make sure diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index fe70b593c7cd9..16dd11c485313 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -17,9 +17,7 @@ #include #include #include "locking.h" -#include "fs.h" #include "accessors.h" -#include "extent-io-tree.h" struct extent_buffer; struct btrfs_block_rsv; diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index 7b277934f66f9..a4cc1bc635622 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -15,6 +15,7 @@ #include "defrag.h" #include "file-item.h" #include "super.h" +#include "compression.h" static struct kmem_cache *btrfs_inode_defrag_cachep; diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 3df7b9d7fbe8d..ea48706a3d810 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -152,37 +152,39 @@ static struct btrfs_delayed_node *btrfs_get_or_create_delayed_node( return ERR_PTR(-ENOMEM); btrfs_init_delayed_node(node, root, ino); + /* Cached in the inode and can be accessed. */ + refcount_set(&node->refs, 2); + btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_NOFS); + btrfs_delayed_node_ref_tracker_alloc(node, &node->inode_cache_tracker, GFP_NOFS); + /* Allocate and reserve the slot, from now it can return a NULL from xa_load(). */ ret = xa_reserve(&root->delayed_nodes, ino, GFP_NOFS); - if (ret == -ENOMEM) { - btrfs_delayed_node_ref_tracker_dir_exit(node); - kmem_cache_free(delayed_node_cache, node); - return ERR_PTR(-ENOMEM); - } + if (ret == -ENOMEM) + goto cleanup; + xa_lock(&root->delayed_nodes); ptr = xa_load(&root->delayed_nodes, ino); if (ptr) { /* Somebody inserted it, go back and read it. */ xa_unlock(&root->delayed_nodes); - btrfs_delayed_node_ref_tracker_dir_exit(node); - kmem_cache_free(delayed_node_cache, node); - node = NULL; - goto again; + goto cleanup; } ptr = __xa_store(&root->delayed_nodes, ino, node, GFP_ATOMIC); ASSERT(xa_err(ptr) != -EINVAL); ASSERT(xa_err(ptr) != -ENOMEM); ASSERT(ptr == NULL); - - /* Cached in the inode and can be accessed. */ - refcount_set(&node->refs, 2); - btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_ATOMIC); - btrfs_delayed_node_ref_tracker_alloc(node, &node->inode_cache_tracker, GFP_ATOMIC); - btrfs_inode->delayed_node = node; xa_unlock(&root->delayed_nodes); return node; +cleanup: + btrfs_delayed_node_ref_tracker_free(node, tracker); + btrfs_delayed_node_ref_tracker_free(node, &node->inode_cache_tracker); + btrfs_delayed_node_ref_tracker_dir_exit(node); + kmem_cache_free(delayed_node_cache, node); + if (ret) + return ERR_PTR(ret); + goto again; } /* @@ -1674,7 +1676,7 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, if (unlikely(ret)) { btrfs_err(trans->fs_info, "failed to add delayed dir index item, root: %llu, inode: %llu, index: %llu, error: %d", - index, btrfs_root_id(node->root), node->inode_id, ret); + btrfs_root_id(node->root), node->inode_id, index, ret); btrfs_delayed_item_release_metadata(dir->root, item); btrfs_release_delayed_item(item); } diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 481802efaa143..f8fc26272f76c 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -798,9 +798,13 @@ static void init_delayed_ref_head(struct btrfs_delayed_ref_head *head_ref, } /* - * helper function to actually insert a head node into the rbtree. - * this does all the dirty work in terms of maintaining the correct - * overall modification count. + * Helper function to actually insert a head node into the xarray. This does all + * the dirty work in terms of maintaining the correct overall modification + * count. + * + * The caller is responsible for calling kfree() on @qrecord. More specifically, + * if this function reports that it did not insert it as noted in + * @qrecord_inserted_ret, then it's safe to call kfree() on it. * * Returns an error pointer in case of an error. */ @@ -814,7 +818,14 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_head *existing; struct btrfs_delayed_ref_root *delayed_refs; const unsigned long index = (head_ref->bytenr >> fs_info->sectorsize_bits); - bool qrecord_inserted = false; + + /* + * If 'qrecord_inserted_ret' is provided, then the first thing we need + * to do is to initialize it to false just in case we have an exit + * before trying to insert the record. + */ + if (qrecord_inserted_ret) + *qrecord_inserted_ret = false; delayed_refs = &trans->transaction->delayed_refs; lockdep_assert_held(&delayed_refs->lock); @@ -833,6 +844,12 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, /* Record qgroup extent info if provided */ if (qrecord) { + /* + * Setting 'qrecord' but not 'qrecord_inserted_ret' will likely + * result in a memory leakage. + */ + ASSERT(qrecord_inserted_ret != NULL); + int ret; ret = btrfs_qgroup_trace_extent_nolock(fs_info, delayed_refs, qrecord, @@ -840,12 +857,10 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, if (ret) { /* Clean up if insertion fails or item exists. */ xa_release(&delayed_refs->dirty_extents, index); - /* Caller responsible for freeing qrecord on error. */ if (ret < 0) return ERR_PTR(ret); - kfree(qrecord); - } else { - qrecord_inserted = true; + } else if (qrecord_inserted_ret) { + *qrecord_inserted_ret = true; } } @@ -888,8 +903,6 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, delayed_refs->num_heads++; delayed_refs->num_heads_ready++; } - if (qrecord_inserted_ret) - *qrecord_inserted_ret = qrecord_inserted; return head_ref; } @@ -1049,6 +1062,14 @@ static int add_delayed_ref(struct btrfs_trans_handle *trans, xa_release(&delayed_refs->head_refs, index); spin_unlock(&delayed_refs->lock); ret = PTR_ERR(new_head_ref); + + /* + * It's only safe to call kfree() on 'qrecord' if + * add_delayed_ref_head() has _not_ inserted it for + * tracing. Otherwise we need to handle this here. + */ + if (!qrecord_reserved || qrecord_inserted) + goto free_head_ref; goto free_record; } head_ref = new_head_ref; @@ -1071,6 +1092,8 @@ static int add_delayed_ref(struct btrfs_trans_handle *trans, if (qrecord_inserted) return btrfs_qgroup_trace_extent_post(trans, record, generic_ref->bytenr); + + kfree(record); return 0; free_record: diff --git a/fs/btrfs/dir-item.c b/fs/btrfs/dir-item.c index 69863e398e223..77e1bcb2a74bf 100644 --- a/fs/btrfs/dir-item.c +++ b/fs/btrfs/dir-item.c @@ -9,6 +9,7 @@ #include "transaction.h" #include "accessors.h" #include "dir-item.h" +#include "delayed-inode.h" /* * insert a name into a directory, doing overflow properly if there is a hash diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 802d4dbe5b381..3836414cbe371 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -10,6 +10,8 @@ #include "fs.h" #include "transaction.h" #include "volumes.h" +#include "bio.h" +#include "ordered-data.h" struct btrfs_dio_data { ssize_t submitted; @@ -713,10 +715,8 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, container_of(bbio, struct btrfs_dio_private, bbio); struct btrfs_dio_data *dio_data = iter->private; - btrfs_bio_init(bbio, BTRFS_I(iter->inode)->root->fs_info, + btrfs_bio_init(bbio, BTRFS_I(iter->inode), file_offset, btrfs_dio_end_io, bio->bi_private); - bbio->inode = BTRFS_I(iter->inode); - bbio->file_offset = file_offset; dip->file_offset = file_offset; dip->bytes = bio->bi_iter.bi_size; @@ -814,6 +814,8 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) ssize_t ret; unsigned int ilock_flags = 0; struct iomap_dio *dio; + const u64 data_profile = btrfs_data_alloc_profile(fs_info) & + BTRFS_BLOCK_GROUP_PROFILE_MASK; if (iocb->ki_flags & IOCB_NOWAIT) ilock_flags |= BTRFS_ILOCK_TRY; @@ -827,6 +829,16 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode) && IS_NOSEC(inode)) ilock_flags |= BTRFS_ILOCK_SHARED; + /* + * If our data profile has duplication (either extra mirrors or RAID56), + * we can not trust the direct IO buffer, the content may change during + * writeback and cause different contents written to different mirrors. + * + * Thus only RAID0 and SINGLE can go true zero-copy direct IO. + */ + if (data_profile != BTRFS_BLOCK_GROUP_RAID0 && data_profile != 0) + goto buffered; + relock: ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags); if (ret < 0) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 0aa7e5d1b05f6..0f87f30c8dd27 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -50,6 +50,7 @@ #include "relocation.h" #include "scrub.h" #include "super.h" +#include "delayed-inode.h" #define BTRFS_SUPER_FLAG_SUPP (BTRFS_HEADER_FLAG_WRITTEN |\ BTRFS_HEADER_FLAG_RELOC |\ @@ -490,28 +491,6 @@ static int btree_migrate_folio(struct address_space *mapping, #define btree_migrate_folio NULL #endif -static int btree_writepages(struct address_space *mapping, - struct writeback_control *wbc) -{ - int ret; - - if (wbc->sync_mode == WB_SYNC_NONE) { - struct btrfs_fs_info *fs_info; - - if (wbc->for_kupdate) - return 0; - - fs_info = inode_to_fs_info(mapping->host); - /* this is a bit racy, but that's ok */ - ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes, - BTRFS_DIRTY_METADATA_THRESH, - fs_info->dirty_metadata_batch); - if (ret < 0) - return 0; - } - return btree_write_cache_pages(mapping, wbc); -} - static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags) { if (folio_test_writeback(folio) || folio_test_dirty(folio)) @@ -1666,7 +1645,7 @@ static void backup_super_roots(struct btrfs_fs_info *info) btrfs_set_backup_chunk_root_level(root_backup, btrfs_header_level(info->chunk_root->node)); - if (!btrfs_fs_compat_ro(info, BLOCK_GROUP_TREE)) { + if (!btrfs_fs_incompat(info, EXTENT_TREE_V2)) { struct btrfs_root *extent_root = btrfs_extent_root(info, 0); struct btrfs_root *csum_root = btrfs_csum_root(info, 0); @@ -2534,8 +2513,8 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info, if (mirror_num >= 0 && btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) { - btrfs_err(fs_info, "super offset mismatch %llu != %u", - btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET); + btrfs_err(fs_info, "super offset mismatch %llu != %llu", + btrfs_super_bytenr(sb), btrfs_sb_offset(mirror_num)); ret = -EINVAL; } @@ -3181,7 +3160,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) { btrfs_err(fs_info, "cannot mount because of unknown incompat features (0x%llx)", - incompat); + incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP); return -EINVAL; } @@ -3213,7 +3192,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) if (compat_ro_unsupp && is_rw_mount) { btrfs_err(fs_info, "cannot mount read-write because of unknown compat_ro features (0x%llx)", - compat_ro); + compat_ro_unsupp); return -EINVAL; } @@ -3226,7 +3205,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) !btrfs_test_opt(fs_info, NOLOGREPLAY)) { btrfs_err(fs_info, "cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay", - compat_ro); + compat_ro_unsupp); return -EINVAL; } @@ -3270,6 +3249,15 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) return 0; } +static bool fs_is_full_ro(const struct btrfs_fs_info *fs_info) +{ + if (!sb_rdonly(fs_info->sb)) + return false; + if (unlikely(fs_info->mount_opt & BTRFS_MOUNT_FULL_RO_MASK)) + return true; + return false; +} + int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices) { u32 sectorsize; @@ -3378,6 +3366,10 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR) WRITE_ONCE(fs_info->fs_error, -EUCLEAN); + /* If the fs has any rescue options, no transaction is allowed. */ + if (fs_is_full_ro(fs_info)) + WRITE_ONCE(fs_info->fs_error, -EROFS); + /* Set up fs_info before parsing mount options */ nodesize = btrfs_super_nodesize(disk_super); sectorsize = btrfs_super_sectorsize(disk_super); diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 57920f2c6fe4e..5320da83d0cf8 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -9,7 +9,8 @@ #include #include #include "ctree.h" -#include "fs.h" +#include "bio.h" +#include "ordered-data.h" struct block_device; struct super_block; diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index dc4ca98c37800..863b45092a190 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -40,6 +40,7 @@ #include "orphan.h" #include "tree-checker.h" #include "raid-stripe-tree.h" +#include "delayed-inode.h" #undef SCRAMBLE_DELAYED_REFS @@ -475,7 +476,7 @@ static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); if (key.objectid != bytenr || key.type != BTRFS_EXTENT_DATA_REF_KEY) - goto fail; + return -ENOENT; ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_data_ref); @@ -486,12 +487,11 @@ static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, btrfs_release_path(path); goto again; } - ret = 0; - break; + return 0; } path->slots[0]++; } -fail: + return ret; } @@ -2469,7 +2469,7 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, int i; int action; int level; - int ret = 0; + int ret; if (btrfs_is_testing(fs_info)) return 0; @@ -2521,7 +2521,7 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, else ret = btrfs_free_extent(trans, &ref); if (ret) - goto fail; + return ret; } else { /* We don't know the owning_root, leave as 0. */ ref.bytenr = btrfs_node_blockptr(buf, i); @@ -2534,12 +2534,10 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, else ret = btrfs_free_extent(trans, &ref); if (ret) - goto fail; + return ret; } } return 0; -fail: - return ret; } int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, @@ -3472,12 +3470,12 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, return 0; if (btrfs_header_generation(buf) != trans->transid) - goto out; + return 0; if (root_id != BTRFS_TREE_LOG_OBJECTID) { ret = check_ref_cleanup(trans, buf->start); if (!ret) - goto out; + return 0; } bg = btrfs_lookup_block_group(fs_info, buf->start); @@ -3485,7 +3483,7 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) { pin_down_extent(trans, bg, buf->start, buf->len, 1); btrfs_put_block_group(bg); - goto out; + return 0; } /* @@ -3509,7 +3507,7 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, || btrfs_is_zoned(fs_info)) { pin_down_extent(trans, bg, buf->start, buf->len, 1); btrfs_put_block_group(bg); - goto out; + return 0; } WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)); @@ -3519,7 +3517,6 @@ int btrfs_free_tree_block(struct btrfs_trans_handle *trans, btrfs_put_block_group(bg); trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len); -out: return 0; } @@ -6567,6 +6564,10 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) range->minlen); trimmed += group_trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) { + btrfs_put_block_group(cache); + break; + } if (ret) { bg_failed++; bg_ret = ret; @@ -6580,6 +6581,9 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) "failed to trim %llu block group(s), last error %d", bg_failed, bg_ret); + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; + mutex_lock(&fs_devices->device_list_mutex); list_for_each_entry(device, &fs_devices->devices, dev_list) { if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) @@ -6588,10 +6592,12 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) ret = btrfs_trim_free_extents(device, &group_trimmed); trimmed += group_trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) + break; if (ret) { dev_failed++; dev_ret = ret; - break; + continue; } } mutex_unlock(&fs_devices->device_list_mutex); @@ -6601,6 +6607,8 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) "failed to trim %llu device(s), last error %d", dev_failed, dev_ret); range->len = trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; if (bg_ret) return bg_ret; return dev_ret; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 23273d0e6f224..4b2bd3cc3ed33 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -518,7 +518,7 @@ static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 le */ static void end_bbio_data_write(struct btrfs_bio *bbio) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct bio *bio = &bbio->bio; int error = blk_status_to_errno(bio->bi_status); struct folio_iter fi; @@ -574,7 +574,7 @@ static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio) */ static void end_bbio_data_read(struct btrfs_bio *bbio) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct bio *bio = &bbio->bio; struct folio_iter fi; @@ -739,12 +739,10 @@ static void alloc_new_bio(struct btrfs_inode *inode, struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_bio *bbio; - bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info, - bio_ctrl->end_io_func, NULL); + bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, inode, + file_offset, bio_ctrl->end_io_func, NULL); bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; bbio->bio.bi_write_hint = inode->vfs_inode.i_write_hint; - bbio->inode = inode; - bbio->file_offset = file_offset; bio_ctrl->bbio = bbio; bio_ctrl->len_to_oe_boundary = U32_MAX; bio_ctrl->next_file_offset = file_offset; @@ -1691,14 +1689,15 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, unsigned long range_bitmap = 0; bool submitted_io = false; int found_error = 0; + const u64 end = start + len; const u64 folio_start = folio_pos(folio); + const u64 folio_end = folio_start + folio_size(folio); const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); u64 cur; int bit; int ret = 0; - ASSERT(start >= folio_start && - start + len <= folio_start + folio_size(folio)); + ASSERT(start >= folio_start && end <= folio_end); ret = btrfs_writepage_cow_fixup(folio); if (ret == -EAGAIN) { @@ -1714,7 +1713,7 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, return ret; } - for (cur = start; cur < start + len; cur += fs_info->sectorsize) + for (cur = start; cur < end; cur += fs_info->sectorsize) set_bit((cur - folio_start) >> fs_info->sectorsize_bits, &range_bitmap); bitmap_and(&bio_ctrl->submit_bitmap, &bio_ctrl->submit_bitmap, &range_bitmap, blocks_per_folio); @@ -1725,8 +1724,25 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits); if (cur >= i_size) { + struct btrfs_ordered_extent *ordered; + unsigned long flags; + + ordered = btrfs_lookup_first_ordered_range(inode, cur, + fs_info->sectorsize); + /* + * We have just run delalloc before getting here, so + * there must be an ordered extent. + */ + ASSERT(ordered != NULL); + spin_lock_irqsave(&inode->ordered_tree_lock, flags); + set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); + ordered->truncated_len = min(ordered->truncated_len, + cur - ordered->file_offset); + spin_unlock_irqrestore(&inode->ordered_tree_lock, flags); + btrfs_put_ordered_extent(ordered); + btrfs_mark_ordered_io_finished(inode, folio, cur, - start + len - cur, true); + fs_info->sectorsize, true); /* * This range is beyond i_size, thus we don't need to * bother writing back. @@ -1735,9 +1751,8 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, * writeback the sectors with subpage dirty bits, * causing writeback without ordered extent. */ - btrfs_folio_clear_dirty(fs_info, folio, cur, - start + len - cur); - break; + btrfs_folio_clear_dirty(fs_info, folio, cur, fs_info->sectorsize); + continue; } ret = submit_one_sector(inode, folio, cur, bio_ctrl, i_size); if (unlikely(ret < 0)) { @@ -2206,12 +2221,11 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc), - eb->fs_info, end_bbio_meta_write, eb); + BTRFS_I(fs_info->btree_inode), eb->start, + end_bbio_meta_write, eb); bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT; bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev); wbc_init_bio(wbc, &bbio->bio); - bbio->inode = BTRFS_I(eb->fs_info->btree_inode); - bbio->file_offset = eb->start; for (int i = 0; i < num_extent_folios(eb); i++) { struct folio *folio = eb->folios[i]; u64 range_start = max_t(u64, eb->start, folio_pos(folio)); @@ -2272,8 +2286,7 @@ void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, } } -int btree_write_cache_pages(struct address_space *mapping, - struct writeback_control *wbc) +int btree_writepages(struct address_space *mapping, struct writeback_control *wbc) { struct btrfs_eb_write_context ctx = { .wbc = wbc }; struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); @@ -3826,6 +3839,7 @@ static void end_bbio_meta_read(struct btrfs_bio *bbio) int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num, const struct btrfs_tree_parent_check *check) { + struct btrfs_fs_info *fs_info = eb->fs_info; struct btrfs_bio *bbio; if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) @@ -3859,11 +3873,9 @@ int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num, refcount_inc(&eb->refs); bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, - REQ_OP_READ | REQ_META, eb->fs_info, - end_bbio_meta_read, eb); + REQ_OP_READ | REQ_META, BTRFS_I(fs_info->btree_inode), + eb->start, end_bbio_meta_read, eb); bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT; - bbio->inode = BTRFS_I(eb->fs_info->btree_inode); - bbio->file_offset = eb->start; memcpy(&bbio->parent_check, check, sizeof(*check)); for (int i = 0; i < num_extent_folios(eb); i++) { struct folio *folio = eb->folios[i]; @@ -4466,6 +4478,7 @@ static int try_release_subpage_extent_buffer(struct folio *folio) */ if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { spin_unlock(&eb->refs_lock); + rcu_read_lock(); break; } diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 5fcbfe44218c4..73571d5d3d5ad 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -12,7 +12,6 @@ #include #include #include -#include "compression.h" #include "messages.h" #include "ulist.h" #include "misc.h" @@ -238,8 +237,7 @@ void extent_write_locked_range(struct inode *inode, const struct folio *locked_f u64 start, u64 end, struct writeback_control *wbc, bool pages_dirty); int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc); -int btree_write_cache_pages(struct address_space *mapping, - struct writeback_control *wbc); +int btree_writepages(struct address_space *mapping, struct writeback_control *wbc); void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, u64 end); void btrfs_readahead(struct readahead_control *rac); int set_folio_extent_mapped(struct folio *folio); diff --git a/fs/btrfs/extent_map.h b/fs/btrfs/extent_map.h index d4b81ee4d97bd..6f685f3c93272 100644 --- a/fs/btrfs/extent_map.h +++ b/fs/btrfs/extent_map.h @@ -8,8 +8,7 @@ #include #include #include -#include "misc.h" -#include "compression.h" +#include "fs.h" struct btrfs_inode; struct btrfs_fs_info; diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index a42e6d54e7cd7..4b7c40f05e8f9 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -18,6 +18,7 @@ #include "fs.h" #include "accessors.h" #include "file-item.h" +#include "volumes.h" #define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \ sizeof(struct btrfs_item) * 2) / \ @@ -764,21 +765,46 @@ int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path, return ret; } +static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); + struct bio *bio = &bbio->bio; + struct btrfs_ordered_sum *sums = bbio->sums; + struct bvec_iter iter = *src; + phys_addr_t paddr; + const u32 blocksize = fs_info->sectorsize; + int index = 0; + + shash->tfm = fs_info->csum_shash; + + btrfs_bio_for_each_block(paddr, bio, &iter, blocksize) { + btrfs_calculate_block_csum(fs_info, paddr, sums->sums + index); + index += fs_info->csum_size; + } +} + +static void csum_one_bio_work(struct work_struct *work) +{ + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, csum_work); + + ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE); + ASSERT(bbio->async_csum == true); + csum_one_bio(bbio, &bbio->csum_saved_iter); + complete(&bbio->csum_done); +} + /* * Calculate checksums of the data contained inside a bio. */ -int btrfs_csum_one_bio(struct btrfs_bio *bbio) +int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async) { struct btrfs_ordered_extent *ordered = bbio->ordered; struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); struct bio *bio = &bbio->bio; struct btrfs_ordered_sum *sums; - struct bvec_iter iter = bio->bi_iter; - phys_addr_t paddr; - const u32 blocksize = fs_info->sectorsize; - int index; unsigned nofs_flag; nofs_flag = memalloc_nofs_save(); @@ -789,21 +815,21 @@ int btrfs_csum_one_bio(struct btrfs_bio *bbio) if (!sums) return -ENOMEM; + sums->logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; sums->len = bio->bi_iter.bi_size; INIT_LIST_HEAD(&sums->list); - - sums->logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; - index = 0; - - shash->tfm = fs_info->csum_shash; - - btrfs_bio_for_each_block(paddr, bio, &iter, blocksize) { - btrfs_calculate_block_csum(fs_info, paddr, sums->sums + index); - index += fs_info->csum_size; - } - bbio->sums = sums; btrfs_add_ordered_sum(ordered, sums); + + if (!async) { + csum_one_bio(bbio, &bbio->bio.bi_iter); + return 0; + } + init_completion(&bbio->csum_done); + bbio->async_csum = true; + bbio->csum_saved_iter = bbio->bio.bi_iter; + INIT_WORK(&bbio->csum_work, csum_one_bio_work); + schedule_work(&bbio->csum_work); return 0; } diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h index 63216c43676de..5645c5e3abdb7 100644 --- a/fs/btrfs/file-item.h +++ b/fs/btrfs/file-item.h @@ -7,7 +7,7 @@ #include #include #include "ctree.h" -#include "accessors.h" +#include "ordered-data.h" struct extent_map; struct btrfs_file_extent_item; @@ -64,7 +64,7 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_ordered_sum *sums); -int btrfs_csum_one_bio(struct btrfs_bio *bbio); +int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async); int btrfs_alloc_dummy_sum(struct btrfs_bio *bbio); int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, struct list_head *list, int search_commit, diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index fa82def46e395..0fee45d35a5f8 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2018,13 +2018,14 @@ static vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) else btrfs_delalloc_release_space(inode, data_reserved, page_start, reserved_space, true); - extent_changeset_free(data_reserved); out_noreserve: if (only_release_metadata) btrfs_check_nocow_unlock(inode); sb_end_pagefault(inode->vfs_inode.i_sb); + extent_changeset_free(data_reserved); + if (ret < 0) return vmf_error(ret); diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index 814bbc9417d2a..37aa8d141a83d 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -250,6 +250,14 @@ enum { BTRFS_MOUNT_REF_TRACKER = (1ULL << 33), }; +/* These mount options require a full read-only fs, no new transaction is allowed. */ +#define BTRFS_MOUNT_FULL_RO_MASK \ + (BTRFS_MOUNT_NOLOGREPLAY | \ + BTRFS_MOUNT_IGNOREBADROOTS | \ + BTRFS_MOUNT_IGNOREDATACSUMS | \ + BTRFS_MOUNT_IGNOREMETACSUMS | \ + BTRFS_MOUNT_IGNORESUPERFLAGS) + /* * Compat flags that we support. If any incompat flags are set other than the * ones specified below then we will fail to mount diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 6282911e536f0..7fe868a6a51b4 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -71,6 +71,7 @@ #include "backref.h" #include "raid-stripe-tree.h" #include "fiemap.h" +#include "delayed-inode.h" #define COW_FILE_RANGE_KEEP_LOCKED (1UL << 0) #define COW_FILE_RANGE_NO_INLINE (1UL << 1) @@ -256,6 +257,7 @@ static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off if (ret < 0) { btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %d", logical, ret); + btrfs_release_path(&path); return; } eb = path.nodes[0]; @@ -613,19 +615,22 @@ static noinline int __cow_file_range_inline(struct btrfs_inode *inode, struct btrfs_drop_extents_args drop_args = { 0 }; struct btrfs_root *root = inode->root; struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_trans_handle *trans; + struct btrfs_trans_handle *trans = NULL; u64 data_len = (compressed_size ?: size); int ret; struct btrfs_path *path; path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } trans = btrfs_join_transaction(root); if (IS_ERR(trans)) { - btrfs_free_path(path); - return PTR_ERR(trans); + ret = PTR_ERR(trans); + trans = NULL; + goto out; } trans->block_rsv = &inode->block_rsv; @@ -669,10 +674,15 @@ static noinline int __cow_file_range_inline(struct btrfs_inode *inode, * it won't count as data extent, free them directly here. * And at reserve time, it's always aligned to page size, so * just free one page here. + * + * If we fallback to non-inline (ret == 1) due to -ENOSPC, then we need + * to keep the data reservation. */ - btrfs_qgroup_free_data(inode, NULL, 0, fs_info->sectorsize, NULL); + if (ret <= 0) + btrfs_qgroup_free_data(inode, NULL, 0, fs_info->sectorsize, NULL); btrfs_free_path(path); - btrfs_end_transaction(trans); + if (trans) + btrfs_end_transaction(trans); return ret; } @@ -3975,11 +3985,6 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path btrfs_set_inode_mapping_order(inode); cache_index: - ret = btrfs_init_file_extent_tree(inode); - if (ret) - goto out; - btrfs_inode_set_file_extent_range(inode, 0, - round_up(i_size_read(vfs_inode), fs_info->sectorsize)); /* * If we were modified in the current generation and evicted from memory * and then re-read we need to do a full sync since we don't have any @@ -4066,6 +4071,20 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path btrfs_ino(inode), btrfs_root_id(root), ret); } + /* + * We don't need the path anymore, so release it to avoid holding a read + * lock on a leaf while calling btrfs_init_file_extent_tree(), which can + * allocate memory that triggers reclaim (GFP_KERNEL) and cause a locking + * dependency. + */ + btrfs_release_path(path); + + ret = btrfs_init_file_extent_tree(inode); + if (ret) + goto out; + btrfs_inode_set_file_extent_range(inode, 0, + round_up(i_size_read(vfs_inode), fs_info->sectorsize)); + if (!maybe_acls) cache_no_acl(vfs_inode); @@ -4098,6 +4117,15 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path return 0; out: + /* + * We may have a read locked leaf and iget_failed() triggers inode + * eviction which needs to release the delayed inode and that needs + * to lock the delayed inode's mutex. This can cause a ABBA deadlock + * with a task running delayed items, as that require first locking + * the delayed inode's mutex and then modifying its subvolume btree. + * So release the path before iget_failed(). + */ + btrfs_release_path(path); iget_failed(vfs_inode); return ret; } @@ -4630,7 +4658,7 @@ int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry) spin_unlock(&dest->root_item_lock); btrfs_warn(fs_info, "attempt to delete subvolume %llu with active swapfile", - btrfs_root_id(root)); + btrfs_root_id(dest)); ret = -EPERM; goto out_up_write; } @@ -6450,6 +6478,25 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, int ret; bool xa_reserved = false; + if (!args->orphan && !args->subvol) { + /* + * Before anything else, check if we can add the name to the + * parent directory. We want to avoid a dir item overflow in + * case we have an existing dir item due to existing name + * hash collisions. We do this check here before we call + * btrfs_add_link() down below so that we can avoid a + * transaction abort (which could be exploited by malicious + * users). + * + * For subvolumes we already do this in btrfs_mksubvol(). + */ + ret = btrfs_check_dir_item_collision(BTRFS_I(dir)->root, + btrfs_ino(BTRFS_I(dir)), + name); + if (ret < 0) + return ret; + } + path = btrfs_alloc_path(); if (!path) return -ENOMEM; @@ -9395,7 +9442,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, u64 disk_bytenr, u64 disk_io_size, struct page **pages, void *uring_ctx) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_encoded_read_private *priv, sync_priv; struct completion sync_reads; unsigned long i = 0; @@ -9420,10 +9466,9 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, priv->status = 0; priv->uring_ctx = uring_ctx; - bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info, + bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, btrfs_encoded_read_endio, priv); bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; - bbio->inode = inode; do { size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE); @@ -9432,10 +9477,9 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, refcount_inc(&priv->pending_refs); btrfs_submit_bbio(bbio, 0); - bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info, + bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, btrfs_encoded_read_endio, priv); bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; - bbio->inode = inode; continue; } diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 8cb7d5a462ef7..bfe253c2849a5 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -674,6 +674,13 @@ static noinline int create_subvol(struct mnt_idmap *idmap, goto out; } + /* + * Subvolumes have orphans cleaned on first dentry lookup. A new + * subvolume cannot have any orphans, so we should set the bit before we + * add the subvolume dentry to the dentry cache, so that it is in the + * same state as a subvolume after first lookup. + */ + set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &new_root->state); d_instantiate_new(dentry, new_inode_args.inode); new_inode_args.inode = NULL; @@ -1913,10 +1920,8 @@ static int btrfs_search_path_in_tree_user(struct mnt_idmap *idmap, ret = inode_permission(idmap, &temp_inode->vfs_inode, MAY_READ | MAY_EXEC); iput(&temp_inode->vfs_inode); - if (ret) { - ret = -EACCES; + if (ret) goto out_put; - } if (key.offset == upper_limit) break; @@ -3744,7 +3749,8 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) } } - trans = btrfs_join_transaction(root); + /* 2 BTRFS_QGROUP_RELATION_KEY items. */ + trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; @@ -3816,7 +3822,11 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) goto out; } - trans = btrfs_join_transaction(root); + /* + * 1 BTRFS_QGROUP_INFO_KEY item. + * 1 BTRFS_QGROUP_LIMIT_KEY item. + */ + trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; @@ -3865,7 +3875,8 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) goto drop_write; } - trans = btrfs_join_transaction(root); + /* 1 BTRFS_QGROUP_LIMIT_KEY item. */ + trans = btrfs_start_transaction(root, 1); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; @@ -3986,6 +3997,25 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file, goto out; } + received_uuid_changed = memcmp(root_item->received_uuid, sa->uuid, + BTRFS_UUID_SIZE); + + /* + * Before we attempt to add the new received uuid, check if we have room + * for it in case there's already an item. If the size of the existing + * item plus this root's ID (u64) exceeds the maximum item size, we can + * return here without the need to abort a transaction. If we don't do + * this check, the btrfs_uuid_tree_add() call below would fail with + * -EOVERFLOW and result in a transaction abort. Malicious users could + * exploit this to turn the fs into RO mode. + */ + if (received_uuid_changed && !btrfs_is_empty_uuid(sa->uuid)) { + ret = btrfs_uuid_tree_check_overflow(fs_info, sa->uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL); + if (ret < 0) + goto out; + } + /* * 1 - root item * 2 - uuid items (received uuid + subvol uuid) @@ -4001,8 +4031,6 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file, sa->rtime.sec = ct.tv_sec; sa->rtime.nsec = ct.tv_nsec; - received_uuid_changed = memcmp(root_item->received_uuid, sa->uuid, - BTRFS_UUID_SIZE); if (received_uuid_changed && !btrfs_is_empty_uuid(root_item->received_uuid)) { ret = btrfs_uuid_tree_remove(trans, root_item->received_uuid, @@ -4024,7 +4052,8 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file, ret = btrfs_update_root(trans, fs_info->tree_root, &root->root_key, &root->root_item); - if (ret < 0) { + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); btrfs_end_transaction(trans); goto out; } @@ -4727,7 +4756,7 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, { struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); struct extent_io_tree *io_tree = &inode->io_tree; - struct page **pages; + struct page **pages = NULL; struct btrfs_uring_priv *priv = NULL; unsigned long nr_pages; int ret; @@ -4785,6 +4814,11 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, btrfs_unlock_extent(io_tree, start, lockend, &cached_state); btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); kfree(priv); + for (int i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); return ret; } diff --git a/fs/btrfs/misc.h b/fs/btrfs/misc.h index 60f9b000d644b..a82032c66ccd3 100644 --- a/fs/btrfs/misc.h +++ b/fs/btrfs/misc.h @@ -13,6 +13,13 @@ #include #include +/* + * Convenience macros to define a pointer with the __free(kfree) and + * __free(kvfree) cleanup attributes and initialized to NULL. + */ +#define AUTO_KFREE(name) *name __free(kfree) = NULL +#define AUTO_KVFREE(name) *name __free(kvfree) = NULL + /* * Enumerate bits using enum autoincrement. Define the @name as the n-th bit. */ diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 2829f20d7bb59..8a8aa6ed405bd 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -1098,8 +1098,9 @@ struct btrfs_ordered_extent *btrfs_lookup_first_ordered_range( struct rb_node *prev; struct rb_node *next; struct btrfs_ordered_extent *entry = NULL; + unsigned long flags; - spin_lock_irq(&inode->ordered_tree_lock); + spin_lock_irqsave(&inode->ordered_tree_lock, flags); node = inode->ordered_tree.rb_node; /* * Here we don't want to use tree_search() which will use tree->last @@ -1154,7 +1155,7 @@ struct btrfs_ordered_extent *btrfs_lookup_first_ordered_range( trace_btrfs_ordered_extent_lookup_first_range(inode, entry); } - spin_unlock_irq(&inode->ordered_tree_lock); + spin_unlock_irqrestore(&inode->ordered_tree_lock, flags); return entry; } diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 31ad8580322a6..302bb3ecf39a3 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -1157,11 +1157,14 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info, } if (ret > 0) { /* - * Shouldn't happen, but in case it does we - * don't need to do the btrfs_next_item, just - * continue. + * Shouldn't happen because the key should still + * be there (return 0), but in case it does it + * means we have reached the end of the tree - + * there are no more leaves with items that have + * a key greater than or equals to @found_key, + * so just stop the search loop. */ - continue; + break; } } ret = btrfs_next_item(tree_root, path); @@ -1627,8 +1630,10 @@ static int __del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, if (ret < 0 && ret != -ENOENT) goto out; ret2 = del_qgroup_relation_item(trans, dst, src); - if (ret2 < 0 && ret2 != -ENOENT) + if (ret2 < 0 && ret2 != -ENOENT) { + ret = ret2; goto out; + } /* At least one deletion succeeded, return 0 */ if (!ret || !ret2) @@ -3244,9 +3249,15 @@ static int qgroup_snapshot_quick_inherit(struct btrfs_fs_info *fs_info, { struct btrfs_qgroup *src; struct btrfs_qgroup *parent; + struct btrfs_qgroup *qgroup; struct btrfs_qgroup_list *list; + LIST_HEAD(qgroup_list); + const u32 nodesize = fs_info->nodesize; int nr_parents = 0; + if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_FULL) + return 0; + src = find_qgroup_rb(fs_info, srcid); if (!src) return -ENOENT; @@ -3281,8 +3292,19 @@ static int qgroup_snapshot_quick_inherit(struct btrfs_fs_info *fs_info, if (parent->excl != parent->rfer) return 1; - parent->excl += fs_info->nodesize; - parent->rfer += fs_info->nodesize; + qgroup_iterator_add(&qgroup_list, parent); + list_for_each_entry(qgroup, &qgroup_list, iterator) { + qgroup->rfer += nodesize; + qgroup->rfer_cmpr += nodesize; + qgroup->excl += nodesize; + qgroup->excl_cmpr += nodesize; + qgroup_dirty(fs_info, qgroup); + + /* Append parent qgroups to @qgroup_list. */ + list_for_each_entry(list, &qgroup->groups, next_group) + qgroup_iterator_add(&qgroup_list, list->group); + } + qgroup_iterator_clean(&qgroup_list); return 0; } diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index ba20d9286a340..16936d17166ee 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -747,7 +747,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad fsid, has %pU want %pU", logical, stripe->mirror_num, - header->fsid, fs_info->fs_devices->fsid); + header->fsid, fs_info->fs_devices->metadata_uuid); return; } if (memcmp(header->chunk_tree_uuid, fs_info->chunk_tree_uuid, @@ -929,10 +929,11 @@ static int calc_next_mirror(int mirror, int num_copies) static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *stripe, int sector_nr) { + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; void *kaddr = scrub_stripe_get_kaddr(stripe, sector_nr); int ret; - ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), bbio->fs_info->sectorsize, + ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), fs_info->sectorsize, offset_in_page(kaddr)); /* * Caller should ensure the bbio has enough size. @@ -942,7 +943,21 @@ static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *st * to create the minimal amount of bio vectors, for fs block size < page * size cases. */ - ASSERT(ret == bbio->fs_info->sectorsize); + ASSERT(ret == fs_info->sectorsize); +} + +static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info, + unsigned int nr_vecs, blk_opf_t opf, + u64 logical, + btrfs_bio_end_io_t end_io, void *private) +{ + struct btrfs_bio *bbio; + + bbio = btrfs_bio_alloc(nr_vecs, opf, BTRFS_I(fs_info->btree_inode), + logical, end_io, private); + bbio->is_scrub = true; + bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT; + return bbio; } static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, @@ -968,12 +983,10 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, bbio = NULL; } - if (!bbio) { - bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ, - fs_info, scrub_repair_read_endio, stripe); - bbio->bio.bi_iter.bi_sector = (stripe->logical + - (i << fs_info->sectorsize_bits)) >> SECTOR_SHIFT; - } + if (!bbio) + bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, + stripe->logical + (i << fs_info->sectorsize_bits), + scrub_repair_read_endio, stripe); scrub_bio_add_sector(bbio, stripe, i); } @@ -1352,13 +1365,10 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *str scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); bbio = NULL; } - if (!bbio) { - bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_WRITE, - fs_info, scrub_write_endio, stripe); - bbio->bio.bi_iter.bi_sector = (stripe->logical + - (sector_nr << fs_info->sectorsize_bits)) >> - SECTOR_SHIFT; - } + if (!bbio) + bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_WRITE, + stripe->logical + (sector_nr << fs_info->sectorsize_bits), + scrub_write_endio, stripe); scrub_bio_add_sector(bbio, stripe, sector_nr); } if (bbio) @@ -1849,9 +1859,8 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) continue; } - bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ, - fs_info, scrub_read_endio, stripe); - bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT; + bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, + logical, scrub_read_endio, stripe); } scrub_bio_add_sector(bbio, stripe, i); @@ -1888,10 +1897,8 @@ static void scrub_submit_initial_read(struct scrub_ctx *sctx, return; } - bbio = btrfs_bio_alloc(BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ, fs_info, - scrub_read_endio, stripe); - - bbio->bio.bi_iter.bi_sector = stripe->logical >> SECTOR_SHIFT; + bbio = alloc_scrub_bbio(fs_info, BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ, + stripe->logical, scrub_read_endio, stripe); /* Read the whole range inside the chunk boundary. */ for (unsigned int cur = 0; cur < nr_sectors; cur++) scrub_bio_add_sector(bbio, stripe, cur); @@ -2230,9 +2237,9 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx, bio_put(bio); btrfs_bio_counter_dec(fs_info); +out: btrfs_release_path(&extent_path); btrfs_release_path(&csum_path); -out: return ret; } @@ -3039,6 +3046,10 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, unsigned int nofs_flag; bool need_commit = false; + /* Set the basic fallback @last_physical before we got a sctx. */ + if (progress) + progress->last_physical = start; + if (btrfs_fs_closing(fs_info)) return -EAGAIN; @@ -3057,6 +3068,7 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, sctx = scrub_setup_ctx(fs_info, is_dev_replace); if (IS_ERR(sctx)) return PTR_ERR(sctx); + sctx->stat.last_physical = start; ret = scrub_workers_get(fs_info); if (ret) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 96a030d28e091..9012ce7a742f4 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -6399,6 +6399,8 @@ static int range_is_hole_in_parent(struct send_ctx *sctx, extent_end = btrfs_file_extent_end(path); if (extent_end <= start) goto next; + if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) + return 0; if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) { search_start = extent_end; goto next; diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 97452fb5d29b0..194f590201658 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -15,6 +15,7 @@ #include "accessors.h" #include "extent-tree.h" #include "zoned.h" +#include "delayed-inode.h" /* * HOW DOES SPACE RESERVATION WORK @@ -192,7 +193,7 @@ void btrfs_clear_space_info_full(struct btrfs_fs_info *info) struct btrfs_space_info *found; list_for_each_entry(found, head, list) - found->full = 0; + found->full = false; } /* @@ -305,18 +306,22 @@ static int create_space_info(struct btrfs_fs_info *info, u64 flags) 0); if (ret) - return ret; + goto out_free; } ret = btrfs_sysfs_add_space_info_type(info, space_info); if (ret) - return ret; + goto out_free; list_add(&space_info->list, &info->space_info); if (flags & BTRFS_BLOCK_GROUP_DATA) info->data_sinfo = space_info; return ret; + +out_free: + kfree(space_info); + return ret; } int btrfs_init_space_info(struct btrfs_fs_info *fs_info) @@ -372,8 +377,8 @@ void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info, space_info->bytes_readonly += block_group->bytes_super; btrfs_space_info_update_bytes_zone_unusable(space_info, block_group->zone_unusable); if (block_group->length > 0) - space_info->full = 0; - btrfs_try_granting_tickets(info, space_info); + space_info->full = false; + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); block_group->space_info = space_info; @@ -523,9 +528,9 @@ static void remove_ticket(struct btrfs_space_info *space_info, * This is for space we already have accounted in space_info->bytes_may_use, so * basically when we're returning space from block_rsv's. */ -void btrfs_try_granting_tickets(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info) +void btrfs_try_granting_tickets(struct btrfs_space_info *space_info) { + struct btrfs_fs_info *fs_info = space_info->fs_info; struct list_head *head; enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH; @@ -1124,7 +1129,7 @@ static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info, * the list. */ if (!aborted) - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); } return (tickets_id != space_info->tickets_id); } @@ -1146,7 +1151,7 @@ static void do_async_reclaim_metadata_space(struct btrfs_space_info *space_info) spin_lock(&space_info->lock); to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info); if (!to_reclaim) { - space_info->flush = 0; + space_info->flush = false; spin_unlock(&space_info->lock); return; } @@ -1158,7 +1163,7 @@ static void do_async_reclaim_metadata_space(struct btrfs_space_info *space_info) flush_space(fs_info, space_info, to_reclaim, flush_state, false); spin_lock(&space_info->lock); if (list_empty(&space_info->tickets)) { - space_info->flush = 0; + space_info->flush = false; spin_unlock(&space_info->lock); return; } @@ -1201,7 +1206,7 @@ static void do_async_reclaim_metadata_space(struct btrfs_space_info *space_info) flush_state = FLUSH_DELAYED_ITEMS_NR; commit_cycles--; } else { - space_info->flush = 0; + space_info->flush = false; } } else { flush_state = FLUSH_DELAYED_ITEMS_NR; @@ -1383,7 +1388,7 @@ static void do_async_reclaim_data_space(struct btrfs_space_info *space_info) spin_lock(&space_info->lock); if (list_empty(&space_info->tickets)) { - space_info->flush = 0; + space_info->flush = false; spin_unlock(&space_info->lock); return; } @@ -1394,7 +1399,7 @@ static void do_async_reclaim_data_space(struct btrfs_space_info *space_info) flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE, false); spin_lock(&space_info->lock); if (list_empty(&space_info->tickets)) { - space_info->flush = 0; + space_info->flush = false; spin_unlock(&space_info->lock); return; } @@ -1411,7 +1416,7 @@ static void do_async_reclaim_data_space(struct btrfs_space_info *space_info) data_flush_states[flush_state], false); spin_lock(&space_info->lock); if (list_empty(&space_info->tickets)) { - space_info->flush = 0; + space_info->flush = false; spin_unlock(&space_info->lock); return; } @@ -1428,7 +1433,7 @@ static void do_async_reclaim_data_space(struct btrfs_space_info *space_info) if (maybe_fail_all_tickets(fs_info, space_info)) flush_state = 0; else - space_info->flush = 0; + space_info->flush = false; } else { flush_state = 0; } @@ -1444,7 +1449,7 @@ static void do_async_reclaim_data_space(struct btrfs_space_info *space_info) aborted_fs: maybe_fail_all_tickets(fs_info, space_info); - space_info->flush = 0; + space_info->flush = false; spin_unlock(&space_info->lock); } @@ -1544,7 +1549,7 @@ static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info, * ticket in front of a smaller ticket that can now be satisfied with * the available space. */ - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } @@ -1572,7 +1577,7 @@ static void priority_reclaim_data_space(struct btrfs_fs_info *fs_info, ticket->error = -ENOSPC; remove_ticket(space_info, ticket); - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } @@ -1825,7 +1830,7 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, */ maybe_clamp_preempt(fs_info, space_info); - space_info->flush = 1; + space_info->flush = true; trace_btrfs_trigger_flush(fs_info, space_info->flags, orig_bytes, flush, @@ -2069,11 +2074,11 @@ static bool is_reclaim_urgent(struct btrfs_space_info *space_info) return unalloc < data_chunk_size; } -static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) +static bool do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) { struct btrfs_block_group *bg; int thresh_pct; - bool try_again = true; + bool will_reclaim = false; bool urgent; spin_lock(&space_info->lock); @@ -2091,7 +2096,7 @@ static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) spin_lock(&bg->lock); thresh = mult_perc(bg->length, thresh_pct); if (bg->used < thresh && bg->reclaim_mark) { - try_again = false; + will_reclaim = true; reclaim = true; } bg->reclaim_mark++; @@ -2108,12 +2113,13 @@ static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) * If we have any staler groups, we don't touch the fresher ones, but if we * really need a block group, do take a fresh one. */ - if (try_again && urgent) { - try_again = false; + if (!will_reclaim && urgent) { + urgent = false; goto again; } up_read(&space_info->groups_sem); + return will_reclaim; } void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s64 bytes) @@ -2123,7 +2129,8 @@ void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s6 lockdep_assert_held(&space_info->lock); space_info->reclaimable_bytes += bytes; - if (space_info->reclaimable_bytes >= chunk_sz) + if (space_info->reclaimable_bytes > 0 && + space_info->reclaimable_bytes >= chunk_sz) btrfs_set_periodic_reclaim_ready(space_info, true); } @@ -2150,7 +2157,6 @@ static bool btrfs_should_periodic_reclaim(struct btrfs_space_info *space_info) spin_lock(&space_info->lock); ret = space_info->periodic_reclaim_ready; - btrfs_set_periodic_reclaim_ready(space_info, false); spin_unlock(&space_info->lock); return ret; @@ -2164,8 +2170,13 @@ void btrfs_reclaim_sweep(const struct btrfs_fs_info *fs_info) list_for_each_entry(space_info, &fs_info->space_info, list) { if (!btrfs_should_periodic_reclaim(space_info)) continue; - for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) - do_reclaim_sweep(space_info, raid); + for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) { + if (do_reclaim_sweep(space_info, raid)) { + spin_lock(&space_info->lock); + btrfs_set_periodic_reclaim_ready(space_info, false); + spin_unlock(&space_info->lock); + } + } } } @@ -2195,5 +2206,5 @@ void btrfs_return_free_space(struct btrfs_space_info *space_info, u64 len) grant: /* Add to any tickets we may have. */ if (len) - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); } diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 679f22efb4073..596a1e923ddfe 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -142,11 +142,11 @@ struct btrfs_space_info { flushing. The value is >> clamp, so turns out to be a 2^clamp divisor. */ - unsigned int full:1; /* indicates that we cannot allocate any more + bool full; /* indicates that we cannot allocate any more chunks for this space */ - unsigned int chunk_alloc:1; /* set if we are allocating a chunk */ + bool chunk_alloc; /* set if we are allocating a chunk */ - unsigned int flush:1; /* set if we are trying to make space */ + bool flush; /* set if we are trying to make space */ unsigned int force_alloc; /* set if we need to force a chunk alloc for this space */ @@ -283,8 +283,7 @@ int btrfs_reserve_metadata_bytes(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info, u64 orig_bytes, enum btrfs_reserve_flush_enum flush); -void btrfs_try_granting_tickets(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info); +void btrfs_try_granting_tickets(struct btrfs_space_info *space_info); int btrfs_can_overcommit(struct btrfs_fs_info *fs_info, const struct btrfs_space_info *space_info, u64 bytes, enum btrfs_reserve_flush_enum flush); @@ -295,7 +294,7 @@ static inline void btrfs_space_info_free_bytes_may_use( { spin_lock(&space_info->lock); btrfs_space_info_update_bytes_may_use(space_info, -num_bytes); - btrfs_try_granting_tickets(space_info->fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes, diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index ad0552db7c7dc..d81a0ade559fd 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -7,7 +7,6 @@ #include #include #include "btrfs_inode.h" -#include "fs.h" struct address_space; struct folio; diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 430e7419349c9..c40944ca7b948 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -736,14 +736,12 @@ bool btrfs_check_options(const struct btrfs_fs_info *info, */ void btrfs_set_free_space_cache_settings(struct btrfs_fs_info *fs_info) { - if (fs_info->sectorsize < PAGE_SIZE) { + if (fs_info->sectorsize < PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) { + btrfs_info(fs_info, + "forcing free space tree for sector size %u with page size %lu", + fs_info->sectorsize, PAGE_SIZE); btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE); - if (!btrfs_test_opt(fs_info, FREE_SPACE_TREE)) { - btrfs_info(fs_info, - "forcing free space tree for sector size %u with page size %lu", - fs_info->sectorsize, PAGE_SIZE); - btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE); - } + btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE); } /* diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 89ae0c7a610aa..fc3953136d23c 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -32,6 +32,8 @@ #include "ioctl.h" #include "relocation.h" #include "scrub.h" +#include "ordered-data.h" +#include "delayed-inode.h" static struct kmem_cache *btrfs_trans_handle_cachep; @@ -518,13 +520,14 @@ static inline int is_transaction_blocked(struct btrfs_transaction *trans) * when this is done, it is safe to start a new transaction, but the current * transaction might not be fully on disk. */ -static void wait_current_trans(struct btrfs_fs_info *fs_info) +static void wait_current_trans(struct btrfs_fs_info *fs_info, unsigned int type) { struct btrfs_transaction *cur_trans; spin_lock(&fs_info->trans_lock); cur_trans = fs_info->running_transaction; - if (cur_trans && is_transaction_blocked(cur_trans)) { + if (cur_trans && is_transaction_blocked(cur_trans) && + (btrfs_blocked_trans_types[cur_trans->state] & type)) { refcount_inc(&cur_trans->use_count); spin_unlock(&fs_info->trans_lock); @@ -699,12 +702,12 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, sb_start_intwrite(fs_info->sb); if (may_wait_transaction(fs_info, type)) - wait_current_trans(fs_info); + wait_current_trans(fs_info, type); do { ret = join_transaction(fs_info, type); if (ret == -EBUSY) { - wait_current_trans(fs_info); + wait_current_trans(fs_info, type); if (unlikely(type == TRANS_ATTACH || type == TRANS_JOIN_NOSTART)) ret = -ENOENT; @@ -723,7 +726,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, h->type = type; INIT_LIST_HEAD(&h->new_bgs); - btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELOPS); + btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELREFS); smp_mb(); if (cur_trans->state >= TRANS_STATE_COMMIT_START && @@ -1001,7 +1004,7 @@ int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid) void btrfs_throttle(struct btrfs_fs_info *fs_info) { - wait_current_trans(fs_info); + wait_current_trans(fs_info, TRANS_START); } bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans) @@ -1887,6 +1890,22 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, ret = btrfs_uuid_tree_add(trans, new_root_item->received_uuid, BTRFS_UUID_KEY_RECEIVED_SUBVOL, objectid); + /* + * We are creating of lot of snapshots of the same root that was + * received (has a received UUID) and reached a leaf's limit for + * an item. We can safely ignore this and avoid a transaction + * abort. A deletion of this snapshot will still work since we + * ignore if an item with a BTRFS_UUID_KEY_RECEIVED_SUBVOL key + * is missing (see btrfs_delete_subvolume()). Send/receive will + * work too since it peeks the first root id from the existing + * item (it could peek any), and in case it's missing it + * falls back to search by BTRFS_UUID_KEY_SUBVOL keys. + * Creation of a snapshot does not require CAP_SYS_ADMIN, so + * we don't want users triggering transaction aborts, either + * intentionally or not. + */ + if (ret == -EOVERFLOW) + ret = 0; if (unlikely(ret && ret != -EEXIST)) { btrfs_abort_transaction(trans, ret); goto fail; @@ -2481,13 +2500,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) list_add_tail(&fs_info->chunk_root->dirty_list, &cur_trans->switch_commits); - if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { - btrfs_set_root_node(&fs_info->block_group_root->root_item, - fs_info->block_group_root->node); - list_add_tail(&fs_info->block_group_root->dirty_list, - &cur_trans->switch_commits); - } - switch_commit_roots(trans); ASSERT(list_empty(&cur_trans->dirty_bgs)); diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 9f7c777af6356..18ef069197e5b 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -14,10 +14,6 @@ #include #include "btrfs_inode.h" #include "delayed-ref.h" -#include "extent-io-tree.h" -#include "block-rsv.h" -#include "messages.h" -#include "misc.h" struct dentry; struct inode; diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index c10b4c242acfc..33a45737c4cf4 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1256,10 +1256,27 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, } if (unlikely(btrfs_root_drop_level(&ri) >= BTRFS_MAX_LEVEL)) { generic_err(leaf, slot, - "invalid root level, have %u expect [0, %u]", + "invalid root drop_level, have %u expect [0, %u]", btrfs_root_drop_level(&ri), BTRFS_MAX_LEVEL - 1); return -EUCLEAN; } + /* + * If drop_progress.objectid is non-zero, a btrfs_drop_snapshot() was + * interrupted and the resume point was recorded in drop_progress and + * drop_level. In that case drop_level must be >= 1: level 0 is the + * leaf level and drop_snapshot never saves a checkpoint there (it + * only records checkpoints at internal node levels in DROP_REFERENCE + * stage). A zero drop_level combined with a non-zero drop_progress + * objectid indicates on-disk corruption and would cause a BUG_ON in + * merge_reloc_root() and btrfs_drop_snapshot() at mount time. + */ + if (unlikely(btrfs_disk_key_objectid(&ri.drop_progress) != 0 && + btrfs_root_drop_level(&ri) == 0)) { + generic_err(leaf, slot, + "invalid root drop_level 0 with non-zero drop_progress objectid %llu", + btrfs_disk_key_objectid(&ri.drop_progress)); + return -EUCLEAN; + } /* Flags check */ if (unlikely(btrfs_root_flags(&ri) & ~valid_root_flags)) { @@ -1713,7 +1730,7 @@ static int check_extent_data_ref(struct extent_buffer *leaf, objectid > BTRFS_LAST_FREE_OBJECTID)) { extent_err(leaf, slot, "invalid extent data backref objectid value %llu", - root); + objectid); return -EUCLEAN; } if (unlikely(!IS_ALIGNED(offset, leaf->fs_info->sectorsize))) { @@ -1894,7 +1911,7 @@ static int check_dev_extent_item(const struct extent_buffer *leaf, if (unlikely(prev_key->offset + prev_len > key->offset)) { generic_err(leaf, slot, "dev extent overlap, prev offset %llu len %llu current offset %llu", - prev_key->objectid, prev_len, key->offset); + prev_key->offset, prev_len, key->offset); return -EUCLEAN; } } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 30f3c3b849c14..c45c5112c0350 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -29,6 +29,7 @@ #include "orphan.h" #include "print-tree.h" #include "tree-checker.h" +#include "delayed-inode.h" #define MAX_CONFLICT_INODES 10 @@ -189,7 +190,7 @@ static void do_abort_log_replay(struct walk_control *wc, const char *function, btrfs_abort_transaction(wc->trans, error); - if (wc->subvol_path->nodes[0]) { + if (wc->subvol_path && wc->subvol_path->nodes[0]) { btrfs_crit(fs_info, "subvolume (root %llu) leaf currently being processed:", btrfs_root_id(wc->root)); @@ -2800,7 +2801,7 @@ static int replay_one_buffer(struct extent_buffer *eb, nritems = btrfs_header_nritems(eb); for (wc->log_slot = 0; wc->log_slot < nritems; wc->log_slot++) { - struct btrfs_inode_item *inode_item; + struct btrfs_inode_item *inode_item = NULL; btrfs_item_key_to_cpu(eb, &wc->log_key, wc->log_slot); @@ -4607,21 +4608,32 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, struct inode *inode, bool log_inode_only, u64 logged_isize) { + u64 gen = BTRFS_I(inode)->generation; u64 flags; if (log_inode_only) { - /* set the generation to zero so the recover code - * can tell the difference between an logging - * just to say 'this inode exists' and a logging - * to say 'update this inode with these values' + /* + * Set the generation to zero so the recover code can tell the + * difference between a logging just to say 'this inode exists' + * and a logging to say 'update this inode with these values'. + * But only if the inode was not already logged before. + * We access ->logged_trans directly since it was already set + * up in the call chain by btrfs_log_inode(), and data_race() + * to avoid false alerts from KCSAN and since it was set already + * and one can set it to 0 since that only happens on eviction + * and we are holding a ref on the inode. */ - btrfs_set_inode_generation(leaf, item, 0); + ASSERT(data_race(BTRFS_I(inode)->logged_trans) > 0); + if (data_race(BTRFS_I(inode)->logged_trans) < trans->transid) + gen = 0; + btrfs_set_inode_size(leaf, item, logged_isize); } else { - btrfs_set_inode_generation(leaf, item, BTRFS_I(inode)->generation); btrfs_set_inode_size(leaf, item, inode->i_size); } + btrfs_set_inode_generation(leaf, item, gen); + btrfs_set_inode_uid(leaf, item, i_uid_read(inode)); btrfs_set_inode_gid(leaf, item, i_gid_read(inode)); btrfs_set_inode_mode(leaf, item, inode->i_mode); @@ -5427,42 +5439,63 @@ static int btrfs_log_changed_extents(struct btrfs_trans_handle *trans, return 0; } -static int logged_inode_size(struct btrfs_root *log, struct btrfs_inode *inode, - struct btrfs_path *path, u64 *size_ret) +static int get_inode_size_to_log(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, u64 *size_ret) { struct btrfs_key key; + struct btrfs_inode_item *item; int ret; key.objectid = btrfs_ino(inode); key.type = BTRFS_INODE_ITEM_KEY; key.offset = 0; - ret = btrfs_search_slot(NULL, log, &key, path, 0, 0); - if (ret < 0) { - return ret; - } else if (ret > 0) { - *size_ret = 0; - } else { - struct btrfs_inode_item *item; + /* + * Our caller called inode_logged(), so logged_trans is up to date. + * Use data_race() to silence any warning from KCSAN. Once logged_trans + * is set, it can only be reset to 0 after inode eviction. + */ + if (data_race(inode->logged_trans) == trans->transid) { + ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); + } else if (inode->generation < trans->transid) { + path->search_commit_root = true; + path->skip_locking = true; + ret = btrfs_search_slot(NULL, inode->root, &key, path, 0, 0); + path->search_commit_root = false; + path->skip_locking = false; - item = btrfs_item_ptr(path->nodes[0], path->slots[0], - struct btrfs_inode_item); - *size_ret = btrfs_inode_size(path->nodes[0], item); - /* - * If the in-memory inode's i_size is smaller then the inode - * size stored in the btree, return the inode's i_size, so - * that we get a correct inode size after replaying the log - * when before a power failure we had a shrinking truncate - * followed by addition of a new name (rename / new hard link). - * Otherwise return the inode size from the btree, to avoid - * data loss when replaying a log due to previously doing a - * write that expands the inode's size and logging a new name - * immediately after. - */ - if (*size_ret > inode->vfs_inode.i_size) - *size_ret = inode->vfs_inode.i_size; + } else { + *size_ret = 0; + return 0; } + /* + * If the inode was logged before or is from a past transaction, then + * its inode item must exist in the log root or in the commit root. + */ + ASSERT(ret <= 0); + if (WARN_ON_ONCE(ret > 0)) + ret = -ENOENT; + + if (ret < 0) + return ret; + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + *size_ret = btrfs_inode_size(path->nodes[0], item); + /* + * If the in-memory inode's i_size is smaller then the inode size stored + * in the btree, return the inode's i_size, so that we get a correct + * inode size after replaying the log when before a power failure we had + * a shrinking truncate followed by addition of a new name (rename / new + * hard link). Otherwise return the inode size from the btree, to avoid + * data loss when replaying a log due to previously doing a write that + * expands the inode's size and logging a new name immediately after. + */ + if (*size_ret > inode->vfs_inode.i_size) + *size_ret = inode->vfs_inode.i_size; + btrfs_release_path(path); return 0; } @@ -5872,14 +5905,6 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, struct btrfs_inode *curr_inode = start_inode; int ret = 0; - /* - * If we are logging a new name, as part of a link or rename operation, - * don't bother logging new dentries, as we just want to log the names - * of an inode and that any new parents exist. - */ - if (ctx->logging_new_name) - return 0; - path = btrfs_alloc_path(); if (!path) return -ENOMEM; @@ -6058,6 +6083,33 @@ static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino, return ret; } +static bool can_log_conflicting_inode(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode) +{ + if (!S_ISDIR(inode->vfs_inode.i_mode)) + return true; + + if (inode->last_unlink_trans < trans->transid) + return true; + + /* + * If this is a directory and its unlink_trans is not from a past + * transaction then we must fallback to a transaction commit in order + * to avoid getting a directory with 2 hard links after log replay. + * + * This happens if a directory A is renamed, moved from one parent + * directory to another one, a new file is created in the old parent + * directory with the old name of our directory A, the new file is + * fsynced, then we moved the new file to some other parent directory + * and fsync again the new file. This results in a log tree where we + * logged that directory A existed, with the INODE_REF item for the + * new location but without having logged its old parent inode, so + * that on log replay we add a new link for the new location but the + * old link remains, resulting in a link count of 2. + */ + return false; +} + static int add_conflicting_inode(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_path *path, @@ -6161,6 +6213,11 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, return 0; } + if (!can_log_conflicting_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); + return BTRFS_LOG_FORCE_COMMIT; + } + btrfs_add_delayed_iput(inode); ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); @@ -6178,6 +6235,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_log_ctx *ctx) { + const bool orig_log_new_dentries = ctx->log_new_dentries; int ret = 0; /* @@ -6225,6 +6283,12 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, break; } + if (!can_log_conflicting_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); + ret = BTRFS_LOG_FORCE_COMMIT; + break; + } + /* * Always log the directory, we cannot make this * conditional on need_log_inode() because the directory @@ -6233,7 +6297,11 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, * dir index key range logged for the directory. So we * must make sure the deletion is recorded. */ + ctx->log_new_dentries = false; ret = btrfs_log_inode(trans, inode, LOG_INODE_ALL, ctx); + if (!ret && ctx->log_new_dentries) + ret = log_new_dir_dentries(trans, inode, ctx); + btrfs_add_delayed_iput(inode); if (ret) break; @@ -6268,6 +6336,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, break; } + ctx->log_new_dentries = orig_log_new_dentries; ctx->logging_conflict_inodes = false; if (ret) free_conflicting_inodes(ctx); @@ -6318,10 +6387,8 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, * and no keys greater than that, so bail out. */ break; - } else if ((min_key->type == BTRFS_INODE_REF_KEY || - min_key->type == BTRFS_INODE_EXTREF_KEY) && - (inode->generation == trans->transid || - ctx->logging_conflict_inodes)) { + } else if (min_key->type == BTRFS_INODE_REF_KEY || + min_key->type == BTRFS_INODE_EXTREF_KEY) { u64 other_ino = 0; u64 other_parent = 0; @@ -6943,7 +7010,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, ret = drop_inode_items(trans, log, path, inode, BTRFS_XATTR_ITEM_KEY); } else { - if (inode_only == LOG_INODE_EXISTS && ctx->logged_before) { + if (inode_only == LOG_INODE_EXISTS) { /* * Make sure the new inode item we write to the log has * the same isize as the current one (if it exists). @@ -6957,7 +7024,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * (zeroes), as if an expanding truncate happened, * instead of getting a file of 4Kb only. */ - ret = logged_inode_size(log, inode, path, &logged_isize); + ret = get_inode_size_to_log(trans, inode, path, &logged_isize); if (ret) goto out_unlock; } diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h index dc313e6bb2faa..4f149d7d4fdee 100644 --- a/fs/btrfs/tree-log.h +++ b/fs/btrfs/tree-log.h @@ -8,8 +8,7 @@ #include #include -#include "messages.h" -#include "ctree.h" +#include #include "transaction.h" struct inode; diff --git a/fs/btrfs/uuid-tree.c b/fs/btrfs/uuid-tree.c index 17b5e81123a1f..146d78fac8f83 100644 --- a/fs/btrfs/uuid-tree.c +++ b/fs/btrfs/uuid-tree.c @@ -227,6 +227,44 @@ int btrfs_uuid_tree_remove(struct btrfs_trans_handle *trans, const u8 *uuid, u8 return ret; } +/* + * Check if we can add one root ID to a UUID key. + * If the key does not yet exists, we can, otherwise only if extended item does + * not exceeds the maximum item size permitted by the leaf size. + * + * Returns 0 on success, negative value on error. + */ +int btrfs_uuid_tree_check_overflow(struct btrfs_fs_info *fs_info, + const u8 *uuid, u8 type) +{ + BTRFS_PATH_AUTO_FREE(path); + int ret; + u32 item_size; + struct btrfs_key key; + + if (WARN_ON_ONCE(!fs_info->uuid_root)) + return -EINVAL; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + btrfs_uuid_to_key(uuid, type, &key); + ret = btrfs_search_slot(NULL, fs_info->uuid_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0) + return 0; + + item_size = btrfs_item_size(path->nodes[0], path->slots[0]); + + if (sizeof(struct btrfs_item) + item_size + sizeof(u64) > + BTRFS_LEAF_DATA_SIZE(fs_info)) + return -EOVERFLOW; + + return 0; +} + static int btrfs_uuid_iter_rem(struct btrfs_root *uuid_root, u8 *uuid, u8 type, u64 subid) { diff --git a/fs/btrfs/uuid-tree.h b/fs/btrfs/uuid-tree.h index c60ad20325cce..02b235a3653f0 100644 --- a/fs/btrfs/uuid-tree.h +++ b/fs/btrfs/uuid-tree.h @@ -12,6 +12,8 @@ int btrfs_uuid_tree_add(struct btrfs_trans_handle *trans, const u8 *uuid, u8 typ u64 subid); int btrfs_uuid_tree_remove(struct btrfs_trans_handle *trans, const u8 *uuid, u8 type, u64 subid); +int btrfs_uuid_tree_check_overflow(struct btrfs_fs_info *fs_info, + const u8 *uuid, u8 type); int btrfs_uuid_tree_iterate(struct btrfs_fs_info *fs_info); int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info); int btrfs_uuid_scan_kthread(void *data); diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 2bec544d8ba30..ef9f24076ccae 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1365,7 +1365,9 @@ struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev, (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT); } + filemap_invalidate_lock(mapping); page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); + filemap_invalidate_unlock(mapping); if (IS_ERR(page)) return ERR_CAST(page); @@ -1504,30 +1506,158 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, } /* - * Try to find a chunk that intersects [start, start + len] range and when one - * such is found, record the end of it in *start + * Find the first pending extent intersecting a range. + * + * @device: the device to search + * @start: start of the range to check + * @len: length of the range to check + * @pending_start: output pointer for the start of the found pending extent + * @pending_end: output pointer for the end of the found pending extent (inclusive) + * + * Search for a pending chunk allocation that intersects the half-open range + * [start, start + len). + * + * Return: true if a pending extent was found, false otherwise. + * If the return value is true, store the first pending extent in + * [*pending_start, *pending_end]. Otherwise, the two output variables + * may still be modified, to something outside the range and should not + * be used. */ -static bool contains_pending_extent(struct btrfs_device *device, u64 *start, - u64 len) +static bool first_pending_extent(struct btrfs_device *device, u64 start, u64 len, + u64 *pending_start, u64 *pending_end) { - u64 physical_start, physical_end; - lockdep_assert_held(&device->fs_info->chunk_mutex); - if (btrfs_find_first_extent_bit(&device->alloc_state, *start, - &physical_start, &physical_end, + if (btrfs_find_first_extent_bit(&device->alloc_state, start, + pending_start, pending_end, CHUNK_ALLOCATED, NULL)) { - if (in_range(physical_start, *start, len) || - in_range(*start, physical_start, - physical_end + 1 - physical_start)) { - *start = physical_end + 1; + if (in_range(*pending_start, start, len) || + in_range(start, *pending_start, *pending_end + 1 - *pending_start)) { return true; } } return false; } +/* + * Find the first real hole accounting for pending extents. + * + * @device: the device containing the candidate hole + * @start: input/output pointer for the hole start position + * @len: input/output pointer for the hole length + * @min_hole_size: the size of hole we are looking for + * + * Given a potential hole specified by [*start, *start + *len), check for pending + * chunk allocations within that range. If pending extents are found, the hole is + * adjusted to represent the first true free space that is large enough when + * accounting for pending chunks. + * + * Note that this function must handle various cases involving non consecutive + * pending extents. + * + * Returns: true if a suitable hole was found and false otherwise. + * If the return value is true, then *start and *len are set to represent the hole. + * If the return value is false, then *start is set to the largest hole we + * found and *len is set to its length. + * If there are no holes at all, then *start is set to the end of the range and + * *len is set to 0. + */ +static bool find_hole_in_pending_extents(struct btrfs_device *device, u64 *start, + u64 *len, u64 min_hole_size) +{ + u64 pending_start, pending_end; + u64 end; + u64 max_hole_start = 0; + u64 max_hole_len = 0; + + lockdep_assert_held(&device->fs_info->chunk_mutex); + + if (*len == 0) + return false; + + end = *start + *len - 1; + + /* + * Loop until we either see a large enough hole or check every pending + * extent overlapping the candidate hole. + * At every hole that we observe, record it if it is the new max. + * At the end of the iteration, set the output variables to the max hole. + */ + while (true) { + if (first_pending_extent(device, *start, *len, &pending_start, &pending_end)) { + /* + * Case 1: the pending extent overlaps the start of + * candidate hole. That means the true hole is after the + * pending extent, but we need to find the next pending + * extent to properly size the hole. In the next loop, + * we will reduce to case 2 or 3. + * e.g., + * + * |----pending A----| real hole |----pending B----| + * | candidate hole | + * *start end + */ + if (pending_start <= *start) { + *start = pending_end + 1; + goto next; + } + /* + * Case 2: The pending extent starts after *start (and overlaps + * [*start, end), so the first hole just goes up to the start + * of the pending extent. + * e.g., + * + * | real hole |----pending A----| + * | candidate hole | + * *start end + */ + *len = pending_start - *start; + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + if (*len >= min_hole_size) + break; + /* + * If the hole wasn't big enough, then we advance past + * the pending extent and keep looking. + */ + *start = pending_end + 1; + goto next; + } else { + /* + * Case 3: There is no pending extent overlapping the + * range [*start, *start + *len - 1], so the only remaining + * hole is the remaining range. + * e.g., + * + * | candidate hole | + * | real hole | + * *start end + */ + + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + break; + } +next: + if (*start > end) + break; + *len = end - *start + 1; + } + if (max_hole_len) { + *start = max_hole_start; + *len = max_hole_len; + } else { + *start = end + 1; + *len = 0; + } + return max_hole_len >= min_hole_size; +} + static u64 dev_extent_search_start(struct btrfs_device *device) { switch (device->fs_devices->chunk_alloc_policy) { @@ -1592,59 +1722,57 @@ static bool dev_extent_hole_check_zoned(struct btrfs_device *device, } /* - * Check if specified hole is suitable for allocation. + * Validate and adjust a hole for chunk allocation + * + * @device: the device containing the candidate hole + * @hole_start: input/output pointer for the hole start position + * @hole_size: input/output pointer for the hole size + * @num_bytes: minimum allocation size required * - * @device: the device which we have the hole - * @hole_start: starting position of the hole - * @hole_size: the size of the hole - * @num_bytes: the size of the free space that we need + * Check if the specified hole is suitable for allocation and adjust it if + * necessary. The hole may be modified to skip over pending chunk allocations + * and to satisfy stricter zoned requirements on zoned filesystems. * - * This function may modify @hole_start and @hole_size to reflect the suitable - * position for allocation. Returns 1 if hole position is updated, 0 otherwise. + * For regular (non-zoned) allocation, if the hole after adjustment is smaller + * than @num_bytes, the search continues past additional pending extents until + * either a sufficiently large hole is found or no more pending extents exist. + * + * Return: true if a suitable hole was found and false otherwise. + * If the return value is true, then *hole_start and *hole_size are set to + * represent the hole we found. + * If the return value is false, then *hole_start is set to the largest + * hole we found and *hole_size is set to its length. + * If there are no holes at all, then *hole_start is set to the end of the range + * and *hole_size is set to 0. */ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, u64 *hole_size, u64 num_bytes) { - bool changed = false; - u64 hole_end = *hole_start + *hole_size; + bool found = false; + const u64 hole_end = *hole_start + *hole_size - 1; - for (;;) { - /* - * Check before we set max_hole_start, otherwise we could end up - * sending back this offset anyway. - */ - if (contains_pending_extent(device, hole_start, *hole_size)) { - if (hole_end >= *hole_start) - *hole_size = hole_end - *hole_start; - else - *hole_size = 0; - changed = true; - } + ASSERT(*hole_size > 0); - switch (device->fs_devices->chunk_alloc_policy) { - default: - btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); - fallthrough; - case BTRFS_CHUNK_ALLOC_REGULAR: - /* No extra check */ - break; - case BTRFS_CHUNK_ALLOC_ZONED: - if (dev_extent_hole_check_zoned(device, hole_start, - hole_size, num_bytes)) { - changed = true; - /* - * The changed hole can contain pending extent. - * Loop again to check that. - */ - continue; - } - break; - } +again: + *hole_size = hole_end - *hole_start + 1; + found = find_hole_in_pending_extents(device, hole_start, hole_size, num_bytes); + if (!found) + return found; + ASSERT(*hole_size >= num_bytes); + switch (device->fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + return found; + case BTRFS_CHUNK_ALLOC_ZONED: + if (dev_extent_hole_check_zoned(device, hole_start, hole_size, num_bytes)) + goto again; break; } - return changed; + return found; } /* @@ -1703,7 +1831,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, ret = -ENOMEM; goto out; } -again: + if (search_start >= search_end || test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { ret = -ENOSPC; @@ -1790,11 +1918,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, */ if (search_end > search_start) { hole_size = search_end - search_start; - if (dev_extent_hole_check(device, &search_start, &hole_size, - num_bytes)) { - btrfs_release_path(path); - goto again; - } + dev_extent_hole_check(device, &search_start, &hole_size, num_bytes); if (hole_size > max_hole_size) { max_hole_start = search_start; @@ -4140,8 +4264,14 @@ static int __btrfs_balance(struct btrfs_fs_info *fs_info) * this shouldn't happen, it means the last relocate * failed */ - if (ret == 0) - BUG(); /* FIXME break ? */ + if (unlikely(ret == 0)) { + btrfs_err(fs_info, + "unexpected exact match of CHUNK_ITEM in chunk tree, offset 0x%llx", + key.offset); + mutex_unlock(&fs_info->reclaim_bgs_lock); + ret = -EUCLEAN; + goto error; + } ret = btrfs_previous_item(chunk_root, path, 0, BTRFS_CHUNK_ITEM_KEY); @@ -4880,6 +5010,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) u64 diff; u64 start; u64 free_diff = 0; + u64 pending_start, pending_end; new_size = round_down(new_size, fs_info->sectorsize); start = new_size; @@ -4925,7 +5056,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) * in-memory chunks are synced to disk so that the loop below sees them * and relocates them accordingly. */ - if (contains_pending_extent(device, &start, diff)) { + if (first_pending_extent(device, start, diff, &pending_start, &pending_end)) { mutex_unlock(&fs_info->chunk_mutex); ret = btrfs_commit_transaction(trans); if (ret) @@ -6628,8 +6759,10 @@ int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, return PTR_ERR(map); num_copies = btrfs_chunk_map_num_copies(map); - if (io_geom.mirror_num > num_copies) - return -EINVAL; + if (io_geom.mirror_num > num_copies) { + ret = -EINVAL; + goto out; + } map_offset = logical - map->start; io_geom.raid56_full_stripe_start = (u64)-1; @@ -7178,6 +7311,7 @@ static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info, fs_devices->seeding = true; fs_devices->opened = 1; + list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list); return fs_devices; } @@ -7795,8 +7929,9 @@ int btrfs_run_dev_stats(struct btrfs_trans_handle *trans) smp_rmb(); ret = update_dev_stat_item(trans, device); - if (!ret) - atomic_sub(stats_cnt, &device->dev_stats_ccnt); + if (ret) + break; + atomic_sub(stats_cnt, &device->dev_stats_ccnt); } mutex_unlock(&fs_devices->device_list_mutex); diff --git a/fs/btrfs/zlib.c b/fs/btrfs/zlib.c index 6caba8be7c845..10ed48d4a8466 100644 --- a/fs/btrfs/zlib.c +++ b/fs/btrfs/zlib.c @@ -139,6 +139,7 @@ static int copy_data_into_buffer(struct address_space *mapping, data_in = kmap_local_folio(folio, offset); memcpy(workspace->buf + cur - filepos, data_in, copy_length); kunmap_local(data_in); + folio_put(folio); cur += copy_length; } return 0; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index d1db7fa1fe583..16818dbf48a46 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -335,7 +335,10 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) if (!btrfs_fs_incompat(fs_info, ZONED)) return 0; - mutex_lock(&fs_devices->device_list_mutex); + /* + * No need to take the device_list mutex here, we're still in the mount + * path and devices cannot be added to or removed from the list yet. + */ list_for_each_entry(device, &fs_devices->devices, dev_list) { /* We can skip reading of zone info for missing devices */ if (!device->bdev) @@ -345,7 +348,6 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) if (ret) break; } - mutex_unlock(&fs_devices->device_list_mutex); return ret; } @@ -1442,6 +1444,20 @@ static int btrfs_load_block_group_dup(struct btrfs_block_group *bg, return -EIO; } + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + if (zone_info[0].alloc_offset == WP_CONVENTIONAL) zone_info[0].alloc_offset = last_alloc; @@ -1483,6 +1499,21 @@ static int btrfs_load_block_group_raid1(struct btrfs_block_group *bg, /* In case a device is missing we have a cap of 0, so don't use it. */ bg->zone_capacity = min_not_zero(zone_info[0].capacity, zone_info[1].capacity); + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + for (i = 0; i < map->num_stripes; i++) { if (zone_info[i].alloc_offset == WP_MISSING_DEV) continue; @@ -1524,7 +1555,9 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, { struct btrfs_fs_info *fs_info = bg->fs_info; u64 stripe_nr = 0, stripe_offset = 0; + u64 prev_offset = 0; u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", @@ -1532,6 +1565,35 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, return -EINVAL; } + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + u64 alloc; + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + + stripe_nr = zone_info[i].alloc_offset >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + alloc = ((stripe_nr * map->num_stripes + i) << BTRFS_STRIPE_LEN_SHIFT) + + stripe_offset; + last_alloc = max(last_alloc, alloc); + + /* Partially written stripe found. It should be last. */ + if (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) + break; + } + stripe_nr = 0; + stripe_offset = 0; + if (last_alloc) { u32 factor = map->num_stripes; @@ -1545,7 +1607,7 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, continue; if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { - + has_conventional = true; zone_info[i].alloc_offset = btrfs_stripe_nr_to_offset(stripe_nr); if (stripe_index > i) @@ -1554,6 +1616,28 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, zone_info[i].alloc_offset += stripe_offset; } + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < zone_info[i].alloc_offset)) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_partial && + (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = zone_info[i].alloc_offset; + + if ((zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + if (test_bit(0, active) != test_bit(i, active)) { if (unlikely(!btrfs_zone_activate(bg))) return -EIO; @@ -1565,6 +1649,19 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, bg->alloc_offset += zone_info[i].alloc_offset; } + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + return 0; } @@ -1575,8 +1672,11 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, u64 last_alloc) { struct btrfs_fs_info *fs_info = bg->fs_info; + u64 AUTO_KFREE(raid0_allocs); u64 stripe_nr = 0, stripe_offset = 0; u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; + u64 prev_offset = 0; if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", @@ -1584,6 +1684,60 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, return -EINVAL; } + raid0_allocs = kcalloc(map->num_stripes / map->sub_stripes, sizeof(*raid0_allocs), + GFP_NOFS); + if (!raid0_allocs) + return -ENOMEM; + + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i += map->sub_stripes) { + u64 alloc = zone_info[i].alloc_offset; + + for (int j = 1; j < map->sub_stripes; j++) { + int idx = i + j; + + if (zone_info[idx].alloc_offset == WP_MISSING_DEV || + zone_info[idx].alloc_offset == WP_CONVENTIONAL) + continue; + if (alloc == WP_MISSING_DEV || alloc == WP_CONVENTIONAL) { + alloc = zone_info[idx].alloc_offset; + } else if (unlikely(zone_info[idx].alloc_offset != alloc)) { + btrfs_err(fs_info, + "zoned: write pointer mismatch found in block group %llu", + bg->start); + return -EIO; + } + } + + raid0_allocs[i / map->sub_stripes] = alloc; + if (alloc == WP_CONVENTIONAL) + continue; + if (unlikely(alloc == WP_MISSING_DEV)) { + btrfs_err(fs_info, + "zoned: cannot recover write pointer of block group %llu due to missing device", + bg->start); + return -EIO; + } + + stripe_nr = alloc >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = alloc & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + + alloc = ((stripe_nr * (map->num_stripes / map->sub_stripes) + + (i / map->sub_stripes)) << + BTRFS_STRIPE_LEN_SHIFT) + stripe_offset; + last_alloc = max(last_alloc, alloc); + } + stripe_nr = 0; + stripe_offset = 0; + if (last_alloc) { u32 factor = map->num_stripes / map->sub_stripes; @@ -1593,24 +1747,51 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, } for (int i = 0; i < map->num_stripes; i++) { - if (zone_info[i].alloc_offset == WP_MISSING_DEV) - continue; + int idx = i / map->sub_stripes; - if (test_bit(0, active) != test_bit(i, active)) { - if (unlikely(!btrfs_zone_activate(bg))) - return -EIO; - } else { - if (test_bit(0, active)) - set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + if (raid0_allocs[idx] == WP_CONVENTIONAL) { + has_conventional = true; + raid0_allocs[idx] = btrfs_stripe_nr_to_offset(stripe_nr); + + if (stripe_index > idx) + raid0_allocs[idx] += BTRFS_STRIPE_LEN; + else if (stripe_index == idx) + raid0_allocs[idx] += stripe_offset; } - if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { - zone_info[i].alloc_offset = btrfs_stripe_nr_to_offset(stripe_nr); + if ((i % map->sub_stripes) == 0) { + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < raid0_allocs[idx])) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } - if (stripe_index > (i / map->sub_stripes)) - zone_info[i].alloc_offset += BTRFS_STRIPE_LEN; - else if (stripe_index == (i / map->sub_stripes)) - zone_info[i].alloc_offset += stripe_offset; + if (unlikely(has_partial && + (raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = raid0_allocs[idx]; + + if ((raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + } + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + zone_info[i].alloc_offset = raid0_allocs[idx]; + + if (test_bit(0, active) != test_bit(i, active)) { + if (unlikely(!btrfs_zone_activate(bg))) + return -EIO; + } else if (test_bit(0, active)) { + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); } if ((i % map->sub_stripes) == 0) { @@ -1619,6 +1800,20 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, } } + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + return 0; } @@ -1809,14 +2004,14 @@ bool btrfs_use_zone_append(struct btrfs_bio *bbio) { u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT); struct btrfs_inode *inode = bbio->inode; - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_block_group *cache; bool ret = false; if (!btrfs_is_zoned(fs_info)) return false; - if (!inode || !is_data_inode(inode)) + if (!is_data_inode(inode)) return false; if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE) diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 17c5656580dd9..2b807a02d1a8a 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -15,7 +15,6 @@ #include "disk-io.h" #include "block-group.h" #include "btrfs_inode.h" -#include "fs.h" struct block_device; struct extent_buffer; diff --git a/fs/buffer.c b/fs/buffer.c index 6a8752f7bbedb..b6b477ff7b75d 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2948,6 +2948,10 @@ bool try_to_free_buffers(struct folio *folio) if (folio_test_writeback(folio)) return false; + /* Misconfigured folio check */ + if (WARN_ON_ONCE(!folio_buffers(folio))) + return true; + if (mapping == NULL) { /* can this still happen? */ ret = drop_buffers(folio, &buffers_to_free); goto out; diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 322ed268f14aa..390f122feeaaf 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1000,7 +1000,8 @@ unsigned int ceph_define_write_size(struct address_space *mapping) { struct inode *inode = mapping->host; struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); - unsigned int wsize = i_blocksize(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + unsigned int wsize = ci->i_layout.stripe_unit; if (fsc->mount_options->wsize < wsize) wsize = fsc->mount_options->wsize; @@ -1333,7 +1334,6 @@ int ceph_process_folio_batch(struct address_space *mapping, } else if (rc == -E2BIG) { rc = 0; folio_unlock(folio); - ceph_wbc->fbatch.folios[i] = NULL; break; } @@ -1373,6 +1373,7 @@ int ceph_process_folio_batch(struct address_space *mapping, rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc, folio); if (rc) { + rc = 0; folio_redirty_for_writepage(wbc, folio); folio_unlock(folio); break; @@ -2203,6 +2204,7 @@ int ceph_uninline_data(struct file *file) struct ceph_osd_request *req = NULL; struct ceph_cap_flush *prealloc_cf = NULL; struct folio *folio = NULL; + struct ceph_snap_context *snapc = NULL; u64 inline_version = CEPH_INLINE_NONE; struct page *pages[1]; int err = 0; @@ -2230,6 +2232,24 @@ int ceph_uninline_data(struct file *file) if (inline_version == 1) /* initial version, no data */ goto out_uninline; + down_read(&fsc->mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + if (!ci->i_head_snapc) { + ci->i_head_snapc = ceph_get_snap_context( + ci->i_snap_realm->cached_context); + } + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + up_read(&fsc->mdsc->snap_rwsem); + folio = read_mapping_folio(inode->i_mapping, 0, file); if (IS_ERR(folio)) { err = PTR_ERR(folio); @@ -2245,7 +2265,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 0, 1, CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { err = PTR_ERR(req); goto out_unlock; @@ -2261,7 +2281,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 1, 3, CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, - NULL, ci->i_truncate_seq, + snapc, ci->i_truncate_seq, ci->i_truncate_size, false); if (IS_ERR(req)) { err = PTR_ERR(req); @@ -2324,6 +2344,7 @@ int ceph_uninline_data(struct file *file) folio_put(folio); } out: + ceph_put_snap_context(snapc); ceph_free_cap_flush(prealloc_cf); doutc(cl, "%llx.%llx inline_version %llu = %d\n", ceph_vinop(inode), inline_version, err); diff --git a/fs/ceph/crypto.c b/fs/ceph/crypto.c index 7026e794813ca..de823a50af9c5 100644 --- a/fs/ceph/crypto.c +++ b/fs/ceph/crypto.c @@ -219,12 +219,13 @@ static struct inode *parse_longname(const struct inode *parent, struct ceph_vino vino = { .snap = CEPH_NOSNAP }; char *name_end, *inode_number; int ret = -EIO; - /* NUL-terminate */ - char *str __free(kfree) = kmemdup_nul(name, *name_len, GFP_KERNEL); + /* Snapshot name must start with an underscore */ + if (*name_len <= 0 || name[0] != '_') + return ERR_PTR(-EIO); + /* Skip initial '_' and NUL-terminate */ + char *str __free(kfree) = kmemdup_nul(name + 1, *name_len - 1, GFP_KERNEL); if (!str) return ERR_PTR(-ENOMEM); - /* Skip initial '_' */ - str++; name_end = strrchr(str, '_'); if (!name_end) { doutc(cl, "failed to parse long snapshot name: %s\n", str); diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c index f3fe786b4143d..7dc3077902401 100644 --- a/fs/ceph/debugfs.c +++ b/fs/ceph/debugfs.c @@ -79,7 +79,7 @@ static int mdsc_show(struct seq_file *s, void *p) if (req->r_inode) { seq_printf(s, " #%llx", ceph_ino(req->r_inode)); } else if (req->r_dentry) { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, req->r_dentry, &path_info, 0); if (IS_ERR(path)) path = NULL; @@ -98,7 +98,7 @@ static int mdsc_show(struct seq_file *s, void *p) } if (req->r_old_dentry) { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, req->r_old_dentry, &path_info, 0); if (IS_ERR(path)) path = NULL; diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index d18c0eaef9b7e..45e5edecc0cbc 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1338,6 +1338,7 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) struct ceph_client *cl = fsc->client; struct ceph_mds_client *mdsc = fsc->mdsc; struct inode *inode = d_inode(dentry); + struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_mds_request *req; bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS); struct dentry *dn; @@ -1362,7 +1363,7 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) if (!dn) { try_async = false; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dn, &path_info, 0); if (IS_ERR(path)) { try_async = false; @@ -1423,7 +1424,19 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) * We have enough caps, so we assume that the unlink * will succeed. Fix up the target inode and dcache. */ - drop_nlink(inode); + + /* + * Protect the i_nlink update with i_ceph_lock + * to precent racing against ceph_fill_inode() + * handling our completion on a worker thread + * and don't decrement if i_nlink has already + * been updated to zero by this completion. + */ + spin_lock(&ci->i_ceph_lock); + if (inode->i_nlink > 0) + drop_nlink(inode); + spin_unlock(&ci->i_ceph_lock); + d_delete(dentry); } else { spin_lock(&fsc->async_unlink_conflict_lock); diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 99b30f784ee24..ceb5706fe3665 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -397,7 +397,7 @@ int ceph_open(struct inode *inode, struct file *file) if (!dentry) { do_sync = true; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0); if (IS_ERR(path)) { do_sync = true; @@ -807,7 +807,7 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, if (!dn) { try_async = false; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dn, &path_info, 0); if (IS_ERR(path)) { try_async = false; @@ -2568,6 +2568,7 @@ static int ceph_zero_partial_object(struct inode *inode, struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); struct ceph_osd_request *req; + struct ceph_snap_context *snapc; int ret = 0; loff_t zero = 0; int op; @@ -2582,12 +2583,25 @@ static int ceph_zero_partial_object(struct inode *inode, op = CEPH_OSD_OP_ZERO; } + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), offset, length, 0, 1, op, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { ret = PTR_ERR(req); goto out; @@ -2601,6 +2615,7 @@ static int ceph_zero_partial_object(struct inode *inode, ceph_osdc_put_request(req); out: + ceph_put_snap_context(snapc); return ret; } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index a6e260d9e4205..b6c60d787692e 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -2562,7 +2562,7 @@ int __ceph_setattr(struct mnt_idmap *idmap, struct inode *inode, if (!dentry) { do_sync = true; } else { - struct ceph_path_info path_info; + struct ceph_path_info path_info = {0}; path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0); if (IS_ERR(path)) { do_sync = true; diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 1740047aef0f9..ba9f96efc8ee7 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2767,6 +2767,7 @@ char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry, if (ret < 0) { dput(parent); dput(cur); + __putname(path); return ERR_PTR(ret); } @@ -2776,6 +2777,7 @@ char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry, if (len < 0) { dput(parent); dput(cur); + __putname(path); return ERR_PTR(len); } } @@ -2812,6 +2814,7 @@ char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry, * cannot ever succeed. Creating paths that long is * possible with Ceph, but Linux cannot use them. */ + __putname(path); return ERR_PTR(-ENAMETOOLONG); } @@ -5655,7 +5658,7 @@ static int ceph_mds_auth_match(struct ceph_mds_client *mdsc, u32 caller_uid = from_kuid(&init_user_ns, cred->fsuid); u32 caller_gid = from_kgid(&init_user_ns, cred->fsgid); struct ceph_client *cl = mdsc->fsc->client; - const char *fs_name = mdsc->fsc->mount_options->mds_namespace; + const char *fs_name = mdsc->mdsmap->m_fs_name; const char *spath = mdsc->fsc->mount_options->server_path; bool gid_matched = false; u32 gid, tlen, len; @@ -5663,7 +5666,8 @@ static int ceph_mds_auth_match(struct ceph_mds_client *mdsc, doutc(cl, "fsname check fs_name=%s match.fs_name=%s\n", fs_name, auth->match.fs_name ? auth->match.fs_name : ""); - if (auth->match.fs_name && strcmp(auth->match.fs_name, fs_name)) { + + if (!ceph_namespace_match(auth->match.fs_name, fs_name)) { /* fsname mismatch, try next one */ return 0; } diff --git a/fs/ceph/mdsmap.c b/fs/ceph/mdsmap.c index 2c7b151a7c95c..b228e5ecfb926 100644 --- a/fs/ceph/mdsmap.c +++ b/fs/ceph/mdsmap.c @@ -353,22 +353,33 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, __decode_and_drop_type(p, end, u8, bad_ext); } if (mdsmap_ev >= 8) { - u32 fsname_len; + size_t fsname_len; + /* enabled */ ceph_decode_8_safe(p, end, m->m_enabled, bad_ext); + /* fs_name */ - ceph_decode_32_safe(p, end, fsname_len, bad_ext); + m->m_fs_name = ceph_extract_encoded_string(p, end, + &fsname_len, + GFP_NOFS); + if (IS_ERR(m->m_fs_name)) { + m->m_fs_name = NULL; + goto nomem; + } /* validate fsname against mds_namespace */ - if (!namespace_equals(mdsc->fsc->mount_options, *p, + if (!namespace_equals(mdsc->fsc->mount_options, m->m_fs_name, fsname_len)) { - pr_warn_client(cl, "fsname %*pE doesn't match mds_namespace %s\n", - (int)fsname_len, (char *)*p, + pr_warn_client(cl, "fsname %s doesn't match mds_namespace %s\n", + m->m_fs_name, mdsc->fsc->mount_options->mds_namespace); goto bad; } - /* skip fsname after validation */ - ceph_decode_skip_n(p, end, fsname_len, bad); + } else { + m->m_enabled = false; + m->m_fs_name = kstrdup(CEPH_OLD_FS_NAME, GFP_NOFS); + if (!m->m_fs_name) + goto nomem; } /* damaged */ if (mdsmap_ev >= 9) { @@ -430,6 +441,7 @@ void ceph_mdsmap_destroy(struct ceph_mdsmap *m) kfree(m->m_info); } kfree(m->m_data_pg_pools); + kfree(m->m_fs_name); kfree(m); } diff --git a/fs/ceph/mdsmap.h b/fs/ceph/mdsmap.h index 1f2171dd01bfa..d48d07c3516d4 100644 --- a/fs/ceph/mdsmap.h +++ b/fs/ceph/mdsmap.h @@ -45,6 +45,7 @@ struct ceph_mdsmap { bool m_enabled; bool m_damaged; int m_num_laggy; + char *m_fs_name; }; static inline struct ceph_entity_addr * diff --git a/fs/ceph/super.h b/fs/ceph/super.h index a1f781c46b41d..29a980e22dc26 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -104,14 +104,26 @@ struct ceph_mount_options { struct fscrypt_dummy_policy dummy_enc_policy; }; +#define CEPH_NAMESPACE_WILDCARD "*" + +static inline bool ceph_namespace_match(const char *pattern, + const char *target) +{ + if (!pattern || !pattern[0] || + !strcmp(pattern, CEPH_NAMESPACE_WILDCARD)) + return true; + + return !strcmp(pattern, target); +} + /* * Check if the mds namespace in ceph_mount_options matches * the passed in namespace string. First time match (when * ->mds_namespace is NULL) is treated specially, since * ->mds_namespace needs to be initialized by the caller. */ -static inline int namespace_equals(struct ceph_mount_options *fsopt, - const char *namespace, size_t len) +static inline bool namespace_equals(struct ceph_mount_options *fsopt, + const char *namespace, size_t len) { return !(fsopt->mds_namespace && (strlen(fsopt->mds_namespace) != len || diff --git a/fs/dcache.c b/fs/dcache.c index 035cccbc92765..8bf82b002b4d4 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -3207,7 +3207,7 @@ static void __init dcache_init_early(void) HASH_EARLY | HASH_ZERO, &d_hash_shift, NULL, - 0, + 2, 0); d_hash_shift = 32 - d_hash_shift; @@ -3238,7 +3238,7 @@ static void __init dcache_init(void) HASH_ZERO, &d_hash_shift, NULL, - 0, + 2, 0); d_hash_shift = 32 - d_hash_shift; diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index be938fdf17d96..a393ecaf3442a 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -626,7 +626,8 @@ int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len, struct dlm_rsb **r_ret) { char key[DLM_RESNAME_MAXLEN] = {}; - + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; memcpy(key, name, len); *r_ret = rhashtable_lookup_fast(rhash, &key, dlm_rhash_rsb_params); if (*r_ret) @@ -5014,25 +5015,8 @@ void dlm_receive_buffer(const union dlm_packet *p, int nodeid) static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb, struct dlm_message *ms_local) { - if (middle_conversion(lkb)) { - log_rinfo(ls, "%s %x middle convert in progress", __func__, - lkb->lkb_id); - - /* We sent this lock to the new master. The new master will - * tell us when it's granted. We no longer need a reply, so - * use a fake reply to put the lkb into the right state. - */ - hold_lkb(lkb); - memset(ms_local, 0, sizeof(struct dlm_message)); - ms_local->m_type = cpu_to_le32(DLM_MSG_CONVERT_REPLY); - ms_local->m_result = cpu_to_le32(to_dlm_errno(-EINPROGRESS)); - ms_local->m_header.h_nodeid = cpu_to_le32(lkb->lkb_nodeid); - _receive_convert_reply(lkb, ms_local, true); - unhold_lkb(lkb); - - } else if (lkb->lkb_rqmode >= lkb->lkb_grmode) { + if (middle_conversion(lkb) || lkb->lkb_rqmode >= lkb->lkb_grmode) set_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); - } /* lkb->lkb_rqmode < lkb->lkb_grmode shouldn't happen since down conversions are async; there's no reply from the remote master */ diff --git a/fs/efivarfs/vars.c b/fs/efivarfs/vars.c index 6edc10958ecf5..70e13db260dba 100644 --- a/fs/efivarfs/vars.c +++ b/fs/efivarfs/vars.c @@ -552,7 +552,7 @@ int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, err = __efivar_entry_get(entry, attributes, size, data); efivar_unlock(); - return 0; + return err; } /** diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c index b7b3432a9882e..740efd2097bf1 100644 --- a/fs/erofs/fileio.c +++ b/fs/erofs/fileio.c @@ -10,6 +10,7 @@ struct erofs_fileio_rq { struct bio bio; struct kiocb iocb; struct super_block *sb; + refcount_t ref; }; struct erofs_fileio { @@ -24,32 +25,27 @@ static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret) container_of(iocb, struct erofs_fileio_rq, iocb); struct folio_iter fi; - if (ret > 0) { - if (ret != rq->bio.bi_iter.bi_size) { - bio_advance(&rq->bio, ret); - zero_fill_bio(&rq->bio); - } - ret = 0; - } - if (rq->bio.bi_end_io) { - if (ret < 0 && !rq->bio.bi_status) - rq->bio.bi_status = errno_to_blk_status(ret); - rq->bio.bi_end_io(&rq->bio); - } else { + if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) + ret = -EIO; + if (!rq->bio.bi_end_io) { bio_for_each_folio_all(fi, &rq->bio) { DBG_BUGON(folio_test_uptodate(fi.folio)); - erofs_onlinefolio_end(fi.folio, ret, false); + erofs_onlinefolio_end(fi.folio, ret < 0, false); } + } else if (ret < 0 && !rq->bio.bi_status) { + rq->bio.bi_status = errno_to_blk_status(ret); } + bio_endio(&rq->bio); bio_uninit(&rq->bio); - kfree(rq); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); } static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) { const struct cred *old_cred; struct iov_iter iter; - int ret; + ssize_t ret; if (!rq) return; @@ -66,6 +62,8 @@ static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) revert_creds(old_cred); if (ret != -EIOCBQUEUED) erofs_fileio_ki_complete(&rq->iocb, ret); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); } static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev) @@ -76,6 +74,7 @@ static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev) bio_init(&rq->bio, NULL, rq->bvecs, ARRAY_SIZE(rq->bvecs), REQ_OP_READ); rq->iocb.ki_filp = mdev->m_dif->file; rq->sb = mdev->m_sb; + refcount_set(&rq->ref, 2); return rq; } diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c index 362acf828279f..7a346e20f7b7a 100644 --- a/fs/erofs/fscache.c +++ b/fs/erofs/fscache.c @@ -185,7 +185,7 @@ static void erofs_fscache_bio_endio(void *priv, ssize_t transferred_or_error) if (IS_ERR_VALUE(transferred_or_error)) io->bio.bi_status = errno_to_blk_status(transferred_or_error); - io->bio.bi_end_io(&io->bio); + bio_endio(&io->bio); BUILD_BUG_ON(offsetof(struct erofs_fscache_bio, io) != 0); erofs_fscache_io_put(&io->io); } @@ -216,7 +216,7 @@ void erofs_fscache_submit_bio(struct bio *bio) if (!ret) return; bio->bi_status = errno_to_blk_status(ret); - bio->bi_end_io(bio); + bio_endio(bio); } static int erofs_fscache_meta_read_folio(struct file *data, struct folio *folio) diff --git a/fs/erofs/super.c b/fs/erofs/super.c index f3f8d8c066e4e..ee37628ec99fb 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -174,15 +174,15 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, if (!erofs_is_fileio_mode(sbi)) { dif->dax_dev = fs_dax_get_by_bdev(file_bdev(file), &dif->dax_part_off, NULL, NULL); - if (!dif->dax_dev && test_opt(&sbi->opt, DAX_ALWAYS)) { - erofs_info(sb, "DAX unsupported by %s. Turning off DAX.", - dif->path); - clear_opt(&sbi->opt, DAX_ALWAYS); - } } else if (!S_ISREG(file_inode(file)->i_mode)) { fput(file); return -EINVAL; } + if (!dif->dax_dev && test_opt(&sbi->opt, DAX_ALWAYS)) { + erofs_info(sb, "DAX unsupported by %s. Turning off DAX.", + dif->path); + clear_opt(&sbi->opt, DAX_ALWAYS); + } dif->file = file; } @@ -215,13 +215,13 @@ static int erofs_scan_devices(struct super_block *sb, ondisk_extradevs, sbi->devs->extra_devices); return -EINVAL; } - if (!ondisk_extradevs) { - if (test_opt(&sbi->opt, DAX_ALWAYS) && !sbi->dif0.dax_dev) { - erofs_info(sb, "DAX unsupported by block device. Turning off DAX."); - clear_opt(&sbi->opt, DAX_ALWAYS); - } - return 0; + + if (test_opt(&sbi->opt, DAX_ALWAYS) && !sbi->dif0.dax_dev) { + erofs_info(sb, "DAX unsupported by block device. Turning off DAX."); + clear_opt(&sbi->opt, DAX_ALWAYS); } + if (!ondisk_extradevs) + return 0; if (!sbi->devs->extra_devices && !erofs_is_fscache_mode(sb)) sbi->devs->flatdev = true; @@ -330,12 +330,13 @@ static int erofs_read_superblock(struct super_block *sb) } sbi->packed_nid = le64_to_cpu(dsb->packed_nid); if (erofs_sb_has_metabox(sbi)) { + ret = -EFSCORRUPTED; if (sbi->sb_size <= offsetof(struct erofs_super_block, metabox_nid)) - return -EFSCORRUPTED; + goto out; sbi->metabox_nid = le64_to_cpu(dsb->metabox_nid); if (sbi->metabox_nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT)) - return -EFSCORRUPTED; /* self-loop detection */ + goto out; /* self-loop detection */ } sbi->inos = le64_to_cpu(dsb->inos); @@ -346,8 +347,10 @@ static int erofs_read_superblock(struct super_block *sb) if (dsb->volume_name[0]) { sbi->volume_name = kstrndup(dsb->volume_name, sizeof(dsb->volume_name), GFP_KERNEL); - if (!sbi->volume_name) - return -ENOMEM; + if (!sbi->volume_name) { + ret = -ENOMEM; + goto out; + } } /* parse on-disk compression configurations */ @@ -639,6 +642,29 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sbi->blkszbits = PAGE_SHIFT; if (!sb->s_bdev) { + /* + * (File-backed mounts) EROFS claims it's safe to nest other + * fs contexts (including its own) due to self-controlled RO + * accesses/contexts and no side-effect changes that need to + * context save & restore so it can reuse the current thread + * context. + * However, we still need to prevent kernel stack overflow due + * to filesystem nesting: just ensure that s_stack_depth is 0 + * to disallow mounting EROFS on stacked filesystems. + * Note: s_stack_depth is not incremented here for now, since + * EROFS is the only fs supporting file-backed mounts for now. + * It MUST change if another fs plans to support them, which + * may also require adjusting FILESYSTEM_MAX_STACK_DEPTH. + */ + if (erofs_is_fileio_mode(sbi)) { + inode = file_inode(sbi->dif0.file); + if ((inode->i_sb->s_op == &erofs_sops && + !inode->i_sb->s_bdev) || + inode->i_sb->s_stack_depth) { + erofs_err(sb, "file-backed mounts cannot be applied to stacked fses"); + return -ENOTBLK; + } + } sb->s_blocksize = PAGE_SIZE; sb->s_blocksize_bits = PAGE_SHIFT; diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index bc80cfe482f73..71f01f0a07435 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -805,14 +805,26 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) struct erofs_map_blocks *map = &fe->map; struct super_block *sb = fe->inode->i_sb; struct z_erofs_pcluster *pcl = NULL; - void *ptr; + void *ptr = NULL; int ret; DBG_BUGON(fe->pcl); /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ DBG_BUGON(!fe->head); - if (!(map->m_flags & EROFS_MAP_META)) { + if (map->m_flags & EROFS_MAP_META) { + ret = erofs_init_metabuf(&map->buf, sb, + erofs_inode_in_metabox(fe->inode)); + if (ret) + return ret; + ptr = erofs_bread(&map->buf, map->m_pa, false); + if (IS_ERR(ptr)) { + erofs_err(sb, "failed to read inline data %pe @ pa %llu of nid %llu", + ptr, map->m_pa, EROFS_I(fe->inode)->nid); + return PTR_ERR(ptr); + } + ptr = map->buf.page; + } else { while (1) { rcu_read_lock(); pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa); @@ -852,18 +864,8 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) /* bind cache first when cached decompression is preferred */ z_erofs_bind_cache(fe); } else { - ret = erofs_init_metabuf(&map->buf, sb, - erofs_inode_in_metabox(fe->inode)); - if (ret) - return ret; - ptr = erofs_bread(&map->buf, map->m_pa, false); - if (IS_ERR(ptr)) { - ret = PTR_ERR(ptr); - erofs_err(sb, "failed to get inline folio %d", ret); - return ret; - } - folio_get(page_folio(map->buf.page)); - WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page); + folio_get(page_folio((struct page *)ptr)); + WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, ptr); fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK; fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; } @@ -1262,17 +1264,17 @@ static int z_erofs_parse_in_bvecs(struct z_erofs_backend *be, bool *overlapped) return err; } -static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, int err) +static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) { struct erofs_sb_info *const sbi = EROFS_SB(be->sb); struct z_erofs_pcluster *pcl = be->pcl; unsigned int pclusterpages = z_erofs_pclusterpages(pcl); const struct z_erofs_decompressor *decomp = z_erofs_decomp[pcl->algorithmformat]; - int i, j, jtop, err2; + bool try_free = true; + int i, j, jtop, err2, err = eio ? -EIO : 0; struct page *page; bool overlapped; - bool try_free = true; mutex_lock(&pcl->lock); be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; @@ -1400,12 +1402,12 @@ static int z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, .pcl = io->head, }; struct z_erofs_pcluster *next; - int err = io->eio ? -EIO : 0; + int err = 0; for (; be.pcl != Z_EROFS_PCLUSTER_TAIL; be.pcl = next) { DBG_BUGON(!be.pcl); next = READ_ONCE(be.pcl->next); - err = z_erofs_decompress_pcluster(&be, err) ?: err; + err = z_erofs_decompress_pcluster(&be, io->eio) ?: err; } return err; } @@ -1443,6 +1445,7 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, int bios) { struct erofs_sb_info *const sbi = EROFS_SB(io->sb); + int gfp_flag; /* wake up the caller thread for sync decompression */ if (io->sync) { @@ -1475,7 +1478,9 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; return; } + gfp_flag = memalloc_noio_save(); z_erofs_decompressqueue_work(&io->u.work); + memalloc_noio_restore(gfp_flag); } static void z_erofs_fill_bio_vec(struct bio_vec *bvec, diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index c8d8e129eb4ba..30775502b56da 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -513,6 +513,7 @@ static int z_erofs_map_blocks_ext(struct inode *inode, unsigned int recsz = z_erofs_extent_recsize(vi->z_advise); erofs_off_t pos = round_up(Z_EROFS_MAP_HEADER_END(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize), recsz); + unsigned int bmask = sb->s_blocksize - 1; bool in_mbox = erofs_inode_in_metabox(inode); erofs_off_t lend = inode->i_size; erofs_off_t l, r, mid, pa, la, lstart; @@ -596,17 +597,17 @@ static int z_erofs_map_blocks_ext(struct inode *inode, map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | EROFS_MAP_ENCODED; fmt = map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT; + if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; if (fmt) map->m_algorithmformat = fmt - 1; - else if (interlaced && !erofs_blkoff(sb, map->m_pa)) + else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) - map->m_flags |= EROFS_MAP_PARTIAL_REF; - map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; } } map->m_llen = lend - map->m_la; diff --git a/fs/eventpoll.c b/fs/eventpoll.c index ee7c4b683ec3d..a8e30414d996c 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -226,6 +226,9 @@ struct eventpoll { */ refcount_t refcount; + /* used to defer freeing past ep_get_upwards_depth_proc() RCU walk */ + struct rcu_head rcu; + #ifdef CONFIG_NET_RX_BUSY_POLL /* used to track busy poll napi_id */ unsigned int napi_id; @@ -819,7 +822,8 @@ static void ep_free(struct eventpoll *ep) mutex_destroy(&ep->mtx); free_uid(ep->user); wakeup_source_unregister(ep->ws); - kfree(ep); + /* ep_get_upwards_depth_proc() may still hold epi->ep under RCU */ + kfree_rcu(ep, rcu); } /* @@ -2061,7 +2065,8 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, * @ep: the &struct eventpoll to be currently checked. * @depth: Current depth of the path being checked. * - * Return: depth of the subtree, or INT_MAX if we found a loop or went too deep. + * Return: depth of the subtree, or a value bigger than EP_MAX_NESTS if we found + * a loop or went too deep. */ static int ep_loop_check_proc(struct eventpoll *ep, int depth) { @@ -2080,7 +2085,7 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth) struct eventpoll *ep_tovisit; ep_tovisit = epi->ffd.file->private_data; if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS) - result = INT_MAX; + result = EP_MAX_NESTS+1; else result = max(result, ep_loop_check_proc(ep_tovisit, depth + 1) + 1); if (result > EP_MAX_NESTS) diff --git a/fs/exfat/balloc.c b/fs/exfat/balloc.c index 2d2d510f2372c..0b6466b3490a0 100644 --- a/fs/exfat/balloc.c +++ b/fs/exfat/balloc.c @@ -106,7 +106,7 @@ static int exfat_allocate_bitmap(struct super_block *sb, (PAGE_SHIFT - sb->s_blocksize_bits); for (i = 0; i < sbi->map_sectors; i++) { /* Trigger the next readahead in advance. */ - if (0 == (i % max_ra_count)) { + if (max_ra_count && 0 == (i % max_ra_count)) { blk_start_plug(&plug); for (j = i; j < min(max_ra_count, sbi->map_sectors - i) + i; j++) sb_breadahead(sb, sector + j); diff --git a/fs/exfat/file.c b/fs/exfat/file.c index adc37b4d7fc2d..536c8078f0c19 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -25,6 +25,8 @@ static int exfat_cont_expand(struct inode *inode, loff_t size) struct exfat_sb_info *sbi = EXFAT_SB(sb); struct exfat_chain clu; + truncate_pagecache(inode, i_size_read(inode)); + ret = inode_newsize_ok(inode, size); if (ret) return ret; @@ -639,6 +641,9 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) inode_lock(inode); + if (pos > i_size_read(inode)) + truncate_pagecache(inode, i_size_read(inode)); + valid_size = ei->valid_size; ret = generic_write_checks(iocb, iter); diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c index 745dce29ddb53..dfe957493d49e 100644 --- a/fs/exfat/namei.c +++ b/fs/exfat/namei.c @@ -645,16 +645,6 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, info->valid_size = le64_to_cpu(ep2->dentry.stream.valid_size); info->size = le64_to_cpu(ep2->dentry.stream.size); - if (info->valid_size < 0) { - exfat_fs_error(sb, "data valid size is invalid(%lld)", info->valid_size); - return -EIO; - } - - if (unlikely(EXFAT_B_TO_CLU_ROUND_UP(info->size, sbi) > sbi->used_clusters)) { - exfat_fs_error(sb, "data size is invalid(%lld)", info->size); - return -EIO; - } - info->start_clu = le32_to_cpu(ep2->dentry.stream.start_clu); if (!is_valid_cluster(sbi, info->start_clu) && info->size) { exfat_warn(sb, "start_clu is invalid cluster(0x%x)", @@ -692,6 +682,16 @@ static int exfat_find(struct inode *dir, const struct qstr *qname, 0); exfat_put_dentry_set(&es, false); + if (info->valid_size < 0) { + exfat_fs_error(sb, "data valid size is invalid(%lld)", info->valid_size); + return -EIO; + } + + if (unlikely(EXFAT_B_TO_CLU_ROUND_UP(info->size, sbi) > sbi->used_clusters)) { + exfat_fs_error(sb, "data size is invalid(%lld)", info->size); + return -EIO; + } + if (ei->start_clu == EXFAT_FREE_CLUSTER) { exfat_fs_error(sb, "non-zero size file starts with zero cluster (size : %llu, p_dir : %u, entry : 0x%08x)", diff --git a/fs/exfat/super.c b/fs/exfat/super.c index 74d451f732c73..581754001128b 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -813,10 +813,21 @@ static int exfat_init_fs_context(struct fs_context *fc) ratelimit_state_init(&sbi->ratelimit, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); - sbi->options.fs_uid = current_uid(); - sbi->options.fs_gid = current_gid(); - sbi->options.fs_fmask = current->fs->umask; - sbi->options.fs_dmask = current->fs->umask; + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE && fc->root) { + struct super_block *sb = fc->root->d_sb; + struct exfat_mount_options *cur_opts = &EXFAT_SB(sb)->options; + + sbi->options.fs_uid = cur_opts->fs_uid; + sbi->options.fs_gid = cur_opts->fs_gid; + sbi->options.fs_fmask = cur_opts->fs_fmask; + sbi->options.fs_dmask = cur_opts->fs_dmask; + } else { + sbi->options.fs_uid = current_uid(); + sbi->options.fs_gid = current_gid(); + sbi->options.fs_fmask = current->fs->umask; + sbi->options.fs_dmask = current->fs->umask; + } + sbi->options.allow_utime = -1; sbi->options.errors = EXFAT_ERRORS_RO; exfat_set_iocharset(&sbi->options, exfat_default_iocharset); diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c index cf0a0970c0956..f41f320f4437b 100644 --- a/fs/ext4/crypto.c +++ b/fs/ext4/crypto.c @@ -163,10 +163,17 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, */ if (handle) { + /* + * Since the inode is new it is ok to pass the + * XATTR_CREATE flag. This is necessary to match the + * remaining journal credits check in the set_handle + * function with the credits allocated for the new + * inode. + */ res = ext4_xattr_set_handle(handle, inode, EXT4_XATTR_INDEX_ENCRYPTION, EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, - ctx, len, 0); + ctx, len, XATTR_CREATE); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); ext4_clear_inode_state(inode, diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 57087da6c7bee..9bf82b568cfe2 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1564,6 +1564,7 @@ struct ext4_sb_info { struct proc_dir_entry *s_proc; struct kobject s_kobj; struct completion s_kobj_unregister; + struct mutex s_error_notify_mutex; /* protects sysfs_notify vs kobject_del */ struct super_block *s_sb; struct buffer_head *s_mmp_bh; @@ -1771,6 +1772,10 @@ struct ext4_sb_info { * Main fast commit lock. This lock protects accesses to the * following fields: * ei->i_fc_list, s_fc_dentry_q, s_fc_q, s_fc_bytes, s_fc_bh. + * + * s_fc_lock can be taken from reclaim context (inode eviction) and is + * thus reclaim unsafe. Use ext4_fc_lock()/ext4_fc_unlock() helpers + * when acquiring / releasing the lock. */ struct mutex s_fc_lock; struct buffer_head *s_fc_bh; @@ -1815,6 +1820,18 @@ static inline void ext4_writepages_up_write(struct super_block *sb, int ctx) percpu_up_write(&EXT4_SB(sb)->s_writepages_rwsem); } +static inline int ext4_fc_lock(struct super_block *sb) +{ + mutex_lock(&EXT4_SB(sb)->s_fc_lock); + return memalloc_nofs_save(); +} + +static inline void ext4_fc_unlock(struct super_block *sb, int ctx) +{ + memalloc_nofs_restore(ctx); + mutex_unlock(&EXT4_SB(sb)->s_fc_lock); +} + static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) { return ino == EXT4_ROOT_INO || diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index ca5499e9412b0..21fba12520a8e 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -43,8 +43,13 @@ #define EXT4_EXT_MARK_UNWRIT1 0x2 /* mark first half unwritten */ #define EXT4_EXT_MARK_UNWRIT2 0x4 /* mark second half unwritten */ -#define EXT4_EXT_DATA_VALID1 0x8 /* first half contains valid data */ -#define EXT4_EXT_DATA_VALID2 0x10 /* second half contains valid data */ +/* first half contains valid data */ +#define EXT4_EXT_DATA_ENTIRE_VALID1 0x8 /* has entirely valid data */ +#define EXT4_EXT_DATA_PARTIAL_VALID1 0x10 /* has partially valid data */ +#define EXT4_EXT_DATA_VALID1 (EXT4_EXT_DATA_ENTIRE_VALID1 | \ + EXT4_EXT_DATA_PARTIAL_VALID1) + +#define EXT4_EXT_DATA_VALID2 0x20 /* second half contains valid data */ static __le32 ext4_extent_block_csum(struct inode *inode, struct ext4_extent_header *eh) @@ -1736,6 +1741,13 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, err = ext4_ext_get_access(handle, inode, path + k); if (err) return err; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + return -EFSCORRUPTED; + } path[k].p_idx->ei_block = border; err = ext4_ext_dirty(handle, inode, path + k); if (err) @@ -1748,6 +1760,14 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, err = ext4_ext_get_access(handle, inode, path + k); if (err) goto clean; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + err = -EFSCORRUPTED; + goto clean; + } path[k].p_idx->ei_block = border; err = ext4_ext_dirty(handle, inode, path + k); if (err) @@ -3190,8 +3210,12 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, unsigned int ee_len, depth; int err = 0; - BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) == - (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) == EXT4_EXT_DATA_VALID1); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) && + (split_flag & EXT4_EXT_DATA_VALID2)); + + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; ext_debug(inode, "logical block %llu\n", (unsigned long long)split); @@ -3258,7 +3282,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, err = PTR_ERR(path); if (err != -ENOSPC && err != -EDQUOT && err != -ENOMEM) - return path; + goto out_path; /* * Get a new path to try to zeroout or fix the extent length. @@ -3272,7 +3296,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, if (IS_ERR(path)) { EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld", split, PTR_ERR(path)); - return path; + goto out_path; } depth = ext_depth(inode); ex = path[depth].p_ext; @@ -3304,6 +3328,23 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, } if (!err) { + /* + * The first half contains partially valid data, the + * splitting of this extent has not been completed, fix + * extent length and ext4_split_extent() split will the + * first half again. + */ + if (split_flag & EXT4_EXT_DATA_PARTIAL_VALID1) { + /* + * Drop extent cache to prevent stale unwritten + * extents remaining after zeroing out. + */ + ext4_es_remove_extent(inode, + le32_to_cpu(zero_ex.ee_block), + ext4_ext_get_actual_len(&zero_ex)); + goto fix_extent_len; + } + /* update the extent length and mark as initialized */ ex->ee_len = cpu_to_le16(ee_len); ext4_ext_try_to_merge(handle, inode, path, ex); @@ -3332,6 +3373,10 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, ext4_free_ext_path(path); path = ERR_PTR(err); } +out_path: + if (IS_ERR(path)) + /* Remove all remaining potentially stale extents. */ + ext4_es_remove_extent(inode, ee_block, ee_len); ext4_ext_show_leaf(inode, path); return path; } @@ -3366,6 +3411,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, ee_len = ext4_ext_get_actual_len(ex); unwritten = ext4_ext_is_unwritten(ex); + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; + if (map->m_lblk + map->m_len < ee_block + ee_len) { split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT; flags1 = flags | EXT4_GET_BLOCKS_PRE_IO; @@ -3373,7 +3421,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, split_flag1 |= EXT4_EXT_MARK_UNWRIT1 | EXT4_EXT_MARK_UNWRIT2; if (split_flag & EXT4_EXT_DATA_VALID2) - split_flag1 |= EXT4_EXT_DATA_VALID1; + split_flag1 |= map->m_lblk > ee_block ? + EXT4_EXT_DATA_PARTIAL_VALID1 : + EXT4_EXT_DATA_ENTIRE_VALID1; path = ext4_split_extent_at(handle, inode, path, map->m_lblk + map->m_len, split_flag1, flags1); if (IS_ERR(path)) @@ -3721,10 +3771,6 @@ static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle, >> inode->i_sb->s_blocksize_bits; if (eof_block < map->m_lblk + map->m_len) eof_block = map->m_lblk + map->m_len; - /* - * It is safe to convert extent to initialized via explicit - * zeroout only if extent is fully inside i_size or new_size. - */ depth = ext_depth(inode); ex = path[depth].p_ext; ee_block = le32_to_cpu(ex->ee_block); @@ -3732,12 +3778,20 @@ static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle, /* Convert to unwritten */ if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) { - split_flag |= EXT4_EXT_DATA_VALID1; - /* Convert to initialized */ - } else if (flags & EXT4_GET_BLOCKS_CONVERT) { + split_flag |= EXT4_EXT_DATA_ENTIRE_VALID1; + /* Split the existing unwritten extent */ + } else if (flags & (EXT4_GET_BLOCKS_UNWRIT_EXT | + EXT4_GET_BLOCKS_CONVERT)) { + /* + * It is safe to convert extent to initialized via explicit + * zeroout only if extent is fully inside i_size or new_size. + */ split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0; - split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2); + split_flag |= EXT4_EXT_MARK_UNWRIT2; + /* Convert to initialized */ + if (flags & EXT4_GET_BLOCKS_CONVERT) + split_flag |= EXT4_EXT_DATA_VALID2; } flags |= EXT4_GET_BLOCKS_PRE_IO; return ext4_split_extent(handle, inode, path, map, split_flag, flags, @@ -3770,6 +3824,8 @@ ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode, * illegal. */ if (ee_block != map->m_lblk || ee_len > map->m_len) { + int flags = EXT4_GET_BLOCKS_CONVERT | + EXT4_GET_BLOCKS_METADATA_NOFAIL; #ifdef CONFIG_EXT4_DEBUG ext4_warning(inode->i_sb, "Inode (%ld) finished: extent logical block %llu," " len %u; IO logical block %llu, len %u", @@ -3777,7 +3833,7 @@ ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode, (unsigned long long)map->m_lblk, map->m_len); #endif path = ext4_split_convert_extents(handle, inode, map, path, - EXT4_GET_BLOCKS_CONVERT, NULL); + flags, NULL); if (IS_ERR(path)) return path; @@ -3816,6 +3872,7 @@ static struct ext4_ext_path * convert_initialized_extent(handle_t *handle, struct inode *inode, struct ext4_map_blocks *map, struct ext4_ext_path *path, + int flags, unsigned int *allocated) { struct ext4_extent *ex; @@ -3841,11 +3898,11 @@ convert_initialized_extent(handle_t *handle, struct inode *inode, if (ee_block != map->m_lblk || ee_len > map->m_len) { path = ext4_split_convert_extents(handle, inode, map, path, - EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, NULL); + flags, NULL); if (IS_ERR(path)) return path; - path = ext4_find_extent(inode, map->m_lblk, path, 0); + path = ext4_find_extent(inode, map->m_lblk, path, flags); if (IS_ERR(path)) return path; depth = ext_depth(inode); @@ -3913,7 +3970,7 @@ ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode, /* get_block() before submitting IO, split the extent */ if (flags & EXT4_GET_BLOCKS_PRE_IO) { path = ext4_split_convert_extents(handle, inode, map, path, - flags | EXT4_GET_BLOCKS_CONVERT, allocated); + flags, allocated); if (IS_ERR(path)) return path; /* @@ -4257,7 +4314,7 @@ int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, if ((!ext4_ext_is_unwritten(ex)) && (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) { path = convert_initialized_extent(handle, - inode, map, path, &allocated); + inode, map, path, flags, &allocated); if (IS_ERR(path)) err = PTR_ERR(path); goto out; @@ -4404,9 +4461,13 @@ int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, path = ext4_ext_insert_extent(handle, inode, path, &newex, flags); if (IS_ERR(path)) { err = PTR_ERR(path); - if (allocated_clusters) { + /* + * Gracefully handle out of space conditions. If the filesystem + * is inconsistent, we'll just leak allocated blocks to avoid + * causing even more damage. + */ + if (allocated_clusters && (err == -EDQUOT || err == -ENOSPC)) { int fb_flags = 0; - /* * free data blocks we just allocated. * not a good idea to call discard here directly, @@ -5375,7 +5436,8 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle, if (!extent) { EXT4_ERROR_INODE(inode, "unexpected hole at %lu", (unsigned long) *iterator); - return -EFSCORRUPTED; + ret = -EFSCORRUPTED; + goto out; } if (SHIFT == SHIFT_LEFT && *iterator > le32_to_cpu(extent->ee_block)) { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index fa66b08de9994..bab522ca1573d 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -231,16 +231,16 @@ static bool ext4_fc_disabled(struct super_block *sb) void ext4_fc_del(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); struct ext4_fc_dentry_update *fc_dentry; wait_queue_head_t *wq; + int alloc_ctx; if (ext4_fc_disabled(inode->i_sb)) return; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -275,9 +275,9 @@ void ext4_fc_del(struct inode *inode) #endif prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); if (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); schedule(); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); } finish_wait(wq, &wait.wq_entry); } @@ -288,7 +288,7 @@ void ext4_fc_del(struct inode *inode) * dentry create references, since it is not needed to log it anyways. */ if (list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -298,7 +298,7 @@ void ext4_fc_del(struct inode *inode) list_del_init(&fc_dentry->fcd_dilist); WARN_ON(!list_empty(&ei->i_fc_dilist)); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); release_dentry_name_snapshot(&fc_dentry->fcd_name); kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); @@ -315,6 +315,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl tid_t tid; bool has_transaction = true; bool is_ineligible; + int alloc_ctx; if (ext4_fc_disabled(sb)) return; @@ -329,12 +330,12 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl has_transaction = false; read_unlock(&sbi->s_journal->j_state_lock); } - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid))) sbi->s_fc_ineligible_tid = tid; ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); WARN_ON(reason >= EXT4_FC_REASON_MAX); sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; } @@ -358,6 +359,7 @@ static int ext4_fc_track_template( struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); tid_t tid = 0; + int alloc_ctx; int ret; tid = handle->h_transaction->t_tid; @@ -373,14 +375,14 @@ static int ext4_fc_track_template( if (!enqueue) return ret; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&EXT4_I(inode)->i_fc_list)) list_add_tail(&EXT4_I(inode)->i_fc_list, (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? &sbi->s_fc_q[FC_Q_STAGING] : &sbi->s_fc_q[FC_Q_MAIN]); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return ret; } @@ -402,6 +404,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, struct inode *dir = dentry->d_parent->d_inode; struct super_block *sb = inode->i_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); + int alloc_ctx; spin_unlock(&ei->i_fc_lock); @@ -425,7 +428,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, take_dentry_name_snapshot(&node->fcd_name, dentry); INIT_LIST_HEAD(&node->fcd_dilist); INIT_LIST_HEAD(&node->fcd_list); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) list_add_tail(&node->fcd_list, @@ -446,7 +449,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, WARN_ON(!list_empty(&ei->i_fc_dilist)); list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); spin_lock(&ei->i_fc_lock); return 0; @@ -972,13 +975,13 @@ static int ext4_fc_flush_data(journal_t *journal) int ret = 0; list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { - ret = jbd2_submit_inode_data(journal, ei->jinode); + ret = jbd2_submit_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; } list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { - ret = jbd2_wait_inode_data(journal, ei->jinode); + ret = jbd2_wait_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; } @@ -1046,18 +1049,19 @@ static int ext4_fc_perform_commit(journal_t *journal) struct blk_plug plug; int ret = 0; u32 crc = 0; + int alloc_ctx; /* * Step 1: Mark all inodes on s_fc_q[MAIN] with * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being * freed until the data flush is over. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* Step 2: Flush data for all the eligible inodes. */ ret = ext4_fc_flush_data(journal); @@ -1067,7 +1071,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * any error from step 2. This ensures that waiters waiting on * EXT4_STATE_FC_FLUSHING_DATA can resume. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); @@ -1084,7 +1088,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * prepare_to_wait() in ext4_fc_del(). */ smp_mb(); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* * If we encountered error in Step 2, return it now after clearing @@ -1101,12 +1105,12 @@ static int ext4_fc_perform_commit(journal_t *journal) * previous handles are now drained. We now mark the inodes on the * commit queue as being committed. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_COMMITTING); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); jbd2_journal_unlock_updates(journal); /* @@ -1117,6 +1121,7 @@ static int ext4_fc_perform_commit(journal_t *journal) blkdev_issue_flush(journal->j_fs_dev); blk_start_plug(&plug); + alloc_ctx = ext4_fc_lock(sb); /* Step 6: Write fast commit blocks to disk. */ if (sbi->s_fc_bytes == 0) { /* @@ -1134,7 +1139,6 @@ static int ext4_fc_perform_commit(journal_t *journal) } /* Step 6.2: Now write all the dentry updates. */ - mutex_lock(&sbi->s_fc_lock); ret = ext4_fc_commit_dentry_updates(journal, &crc); if (ret) goto out; @@ -1156,7 +1160,7 @@ static int ext4_fc_perform_commit(journal_t *journal) ret = ext4_fc_write_tail(sb, crc); out: - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); blk_finish_plug(&plug); return ret; } @@ -1290,6 +1294,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *ei; struct ext4_fc_dentry_update *fc_dentry; + int alloc_ctx; if (full && sbi->s_fc_bh) sbi->s_fc_bh = NULL; @@ -1297,7 +1302,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) trace_ext4_fc_cleanup(journal, full, tid); jbd2_fc_release_bufs(journal); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], struct ext4_inode_info, @@ -1356,7 +1361,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) if (full) sbi->s_fc_bytes = 0; - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); trace_ext4_fc_stats(sb); } @@ -1608,19 +1613,21 @@ static int ext4_fc_replay_inode(struct super_block *sb, /* Immediately update the inode on disk. */ ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); if (ret) - goto out; + goto out_brelse; ret = sync_dirty_buffer(iloc.bh); if (ret) - goto out; + goto out_brelse; ret = ext4_mark_inode_used(sb, ino); if (ret) - goto out; + goto out_brelse; /* Given that we just wrote the inode on disk, this SHOULD succeed. */ inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode not found."); - return -EFSCORRUPTED; + inode = NULL; + ret = -EFSCORRUPTED; + goto out_brelse; } /* @@ -1637,13 +1644,14 @@ static int ext4_fc_replay_inode(struct super_block *sb, ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode)); ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); sync_dirty_buffer(iloc.bh); +out_brelse: brelse(iloc.bh); out: iput(inode); if (!ret) blkdev_issue_flush(sb->s_bdev); - return 0; + return ret; } /* diff --git a/fs/ext4/fsync.c b/fs/ext4/fsync.c index e476c6de30740..bd8f230fa507e 100644 --- a/fs/ext4/fsync.c +++ b/fs/ext4/fsync.c @@ -83,11 +83,23 @@ static int ext4_fsync_nojournal(struct file *file, loff_t start, loff_t end, int datasync, bool *needs_barrier) { struct inode *inode = file->f_inode; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0, + }; int ret; ret = generic_buffers_fsync_noflush(file, start, end, datasync); - if (!ret) - ret = ext4_sync_parent(inode); + if (ret) + return ret; + + /* Force writeout of inode table buffer to disk */ + ret = ext4_write_inode(inode, &wbc); + if (ret) + return ret; + + ret = ext4_sync_parent(inode); + if (test_opt(inode->i_sb, BARRIER)) *needs_barrier = true; diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index ba4fd9aba1c14..b1bc1950c9f03 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -686,6 +686,12 @@ static int recently_deleted(struct super_block *sb, ext4_group_t group, int ino) if (unlikely(!gdp)) return 0; + /* Inode was never used in this filesystem? */ + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT) || + ino >= EXT4_INODES_PER_GROUP(sb) - ext4_itable_unused_count(sb, gdp))) + return 0; + bh = sb_find_get_block(sb, ext4_inode_table(sb, gdp) + (ino / inodes_per_block)); if (!bh || !buffer_uptodate(bh)) @@ -1293,7 +1299,6 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen)); } - ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */ ext4_set_inode_state(inode, EXT4_STATE_NEW); ei->i_extra_isize = sbi->s_want_extra_isize; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 1b094a4f38663..408677fa81967 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -418,7 +418,12 @@ static int ext4_prepare_inline_data(handle_t *handle, struct inode *inode, return -ENOSPC; ext4_write_lock_xattr(inode, &no_expand); - + /* + * ei->i_inline_size may have changed since the initial check + * if other xattrs were added. Recalculate to ensure + * ext4_update_inline_data() validates against current capacity. + */ + (void) ext4_find_inline_data_nolock(inode); if (ei->i_inline_off) ret = ext4_update_inline_data(handle, inode, len); else @@ -446,9 +451,13 @@ static int ext4_destroy_inline_data_nolock(handle_t *handle, if (!ei->i_inline_off) return 0; + down_write(&ei->i_data_sem); + error = ext4_get_inode_loc(inode, &is.iloc); - if (error) + if (error) { + up_write(&ei->i_data_sem); return error; + } error = ext4_xattr_ibody_find(inode, &i, &is); if (error) @@ -487,6 +496,7 @@ static int ext4_destroy_inline_data_nolock(handle_t *handle, brelse(is.iloc.bh); if (error == -ENODATA) error = 0; + up_write(&ei->i_data_sem); return error; } @@ -512,7 +522,15 @@ static int ext4_read_inline_folio(struct inode *inode, struct folio *folio) goto out; len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode)); - BUG_ON(len > PAGE_SIZE); + + if (len > PAGE_SIZE) { + ext4_error_inode(inode, __func__, __LINE__, 0, + "inline size %zu exceeds PAGE_SIZE", len); + ret = -EFSCORRUPTED; + brelse(iloc.bh); + goto out; + } + kaddr = kmap_local_folio(folio, 0); ret = ext4_read_inline_data(inode, kaddr, len, &iloc); kaddr = folio_zero_tail(folio, len, kaddr + len); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index e99306a8f47ce..3ed0c2656e2e1 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -126,6 +126,8 @@ void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, static inline int ext4_begin_ordered_truncate(struct inode *inode, loff_t new_size) { + struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode); + trace_ext4_begin_ordered_truncate(inode, new_size); /* * If jinode is zero, then we never opened the file for @@ -133,10 +135,10 @@ static inline int ext4_begin_ordered_truncate(struct inode *inode, * jbd2_journal_begin_ordered_truncate() since there's no * outstanding writes we need to flush. */ - if (!EXT4_I(inode)->jinode) + if (!jinode) return 0; return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), - EXT4_I(inode)->jinode, + jinode, new_size); } @@ -182,6 +184,14 @@ void ext4_evict_inode(struct inode *inode) if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL) ext4_evict_ea_inode(inode); if (inode->i_nlink) { + /* + * If there's dirty page will lead to data loss, user + * could see stale data. + */ + if (unlikely(!ext4_emergency_state(inode->i_sb) && + mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY))) + ext4_warning_inode(inode, "data will be lost"); + truncate_inode_pages_final(&inode->i_data); goto no_delete; @@ -4501,8 +4511,13 @@ int ext4_inode_attach_jinode(struct inode *inode) spin_unlock(&inode->i_lock); return -ENOMEM; } - ei->jinode = jinode; - jbd2_journal_init_jbd_inode(ei->jinode, inode); + jbd2_journal_init_jbd_inode(jinode, inode); + /* + * Publish ->jinode only after it is fully initialized so that + * readers never observe a partially initialized jbd2_inode. + */ + smp_wmb(); + WRITE_ONCE(ei->jinode, jinode); jinode = NULL; } spin_unlock(&inode->i_lock); @@ -5288,7 +5303,6 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ei->i_projid = make_kprojid(&init_user_ns, i_projid); set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); - ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */ ei->i_inline_off = 0; ei->i_dir_start_lookup = 0; ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); @@ -5466,18 +5480,36 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, inode->i_op = &ext4_encrypted_symlink_inode_operations; } else if (ext4_inode_is_fast_symlink(inode)) { inode->i_op = &ext4_fast_symlink_inode_operations; - if (inode->i_size == 0 || - inode->i_size >= sizeof(ei->i_data) || - strnlen((char *)ei->i_data, inode->i_size + 1) != - inode->i_size) { - ext4_error_inode(inode, function, line, 0, - "invalid fast symlink length %llu", - (unsigned long long)inode->i_size); - ret = -EFSCORRUPTED; - goto bad_inode; + + /* + * Orphan cleanup can see inodes with i_size == 0 + * and i_data uninitialized. Skip size checks in + * that case. This is safe because the first thing + * ext4_evict_inode() does for fast symlinks is + * clearing of i_data and i_size. + */ + if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) { + if (inode->i_nlink != 0) { + ext4_error_inode(inode, function, line, 0, + "invalid orphan symlink nlink %d", + inode->i_nlink); + ret = -EFSCORRUPTED; + goto bad_inode; + } + } else { + if (inode->i_size == 0 || + inode->i_size >= sizeof(ei->i_data) || + strnlen((char *)ei->i_data, inode->i_size + 1) != + inode->i_size) { + ext4_error_inode(inode, function, line, 0, + "invalid fast symlink length %llu", + (unsigned long long)inode->i_size); + ret = -EFSCORRUPTED; + goto bad_inode; + } + inode_set_cached_link(inode, (char *)ei->i_data, + inode->i_size); } - inode_set_cached_link(inode, (char *)ei->i_data, - inode->i_size); } else { inode->i_op = &ext4_symlink_inode_operations; } @@ -5916,6 +5948,18 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (attr->ia_size == inode->i_size) inc_ivers = false; + /* + * If file has inline data but new size exceeds inline capacity, + * convert to extent-based storage first to prevent inconsistent + * state (inline flag set but size exceeds inline capacity). + */ + if (ext4_has_inline_data(inode) && + attr->ia_size > EXT4_I(inode)->i_inline_size) { + error = ext4_convert_inline_data(inode); + if (error) + goto err_out; + } + if (shrink) { if (ext4_should_order_data(inode)) { error = ext4_begin_ordered_truncate(inode, diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index a93a7baae990c..acc28aa5744b1 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -966,6 +966,7 @@ static long ext4_ioctl_group_add(struct file *file, err = ext4_group_add(sb, input); if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, NULL); jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); @@ -1394,6 +1395,10 @@ static int ext4_ioctl_set_tune_sb(struct file *filp, if (copy_from_user(¶ms, in, sizeof(params))) return -EFAULT; + if (strnlen(params.mount_opts, sizeof(params.mount_opts)) == + sizeof(params.mount_opts)) + return -E2BIG; + if ((params.set_flags & ~TUNE_OPS_SUPPORTED) != 0) return -EOPNOTSUPP; @@ -1607,6 +1612,8 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) err = ext4_group_extend(sb, EXT4_SB(sb)->s_es, n_blocks_count); if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, + NULL); jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index a9416b20ff64c..4abb40d4561ce 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -567,7 +567,7 @@ test_mark_diskspace_used_range(struct kunit *test, bitmap = mbt_ctx_bitmap(sb, TEST_GOAL_GROUP); memset(bitmap, 0, sb->s_blocksize); - ret = ext4_mb_mark_diskspace_used(ac, NULL, 0); + ret = ext4_mb_mark_diskspace_used(ac, NULL); KUNIT_ASSERT_EQ(test, ret, 0); max = EXT4_CLUSTERS_PER_GROUP(sb); diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 9087183602e44..71b8b47e213ab 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -682,6 +682,24 @@ do { \ } \ } while (0) +/* + * Perform buddy integrity check with the following steps: + * + * 1. Top-down validation (from highest order down to order 1, excluding order-0 bitmap): + * For each pair of adjacent orders, if a higher-order bit is set (indicating a free block), + * at most one of the two corresponding lower-order bits may be clear (free). + * + * 2. Order-0 (bitmap) validation, performed on bit pairs: + * - If either bit in a pair is set (1, allocated), then all corresponding higher-order bits + * must not be free (0). + * - If both bits in a pair are clear (0, free), then exactly one of the corresponding + * higher-order bits must be free (0). + * + * 3. Preallocation (pa) list validation: + * For each preallocated block (pa) in the group: + * - Verify that pa_pstart falls within the bounds of this block group. + * - Ensure the corresponding bit(s) in the order-0 bitmap are marked as allocated (1). + */ static void __mb_check_buddy(struct ext4_buddy *e4b, char *file, const char *function, int line) { @@ -723,15 +741,6 @@ static void __mb_check_buddy(struct ext4_buddy *e4b, char *file, continue; } - /* both bits in buddy2 must be 1 */ - MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); - MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); - - for (j = 0; j < (1 << order); j++) { - k = (i * (1 << order)) + j; - MB_CHECK_ASSERT( - !mb_test_bit(k, e4b->bd_bitmap)); - } count++; } MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); @@ -747,15 +756,21 @@ static void __mb_check_buddy(struct ext4_buddy *e4b, char *file, fragments++; fstart = i; } - continue; + } else { + fstart = -1; } - fstart = -1; - /* check used bits only */ - for (j = 0; j < e4b->bd_blkbits + 1; j++) { - buddy2 = mb_find_buddy(e4b, j, &max2); - k = i >> j; - MB_CHECK_ASSERT(k < max2); - MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); + if (!(i & 1)) { + int in_use, zero_bit_count = 0; + + in_use = mb_test_bit(i, buddy) || mb_test_bit(i + 1, buddy); + for (j = 1; j < e4b->bd_blkbits + 2; j++) { + buddy2 = mb_find_buddy(e4b, j, &max2); + k = i >> j; + MB_CHECK_ASSERT(k < max2); + if (!mb_test_bit(k, buddy2)) + zero_bit_count++; + } + MB_CHECK_ASSERT(zero_bit_count == !in_use); } } MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); @@ -768,6 +783,8 @@ static void __mb_check_buddy(struct ext4_buddy *e4b, char *file, ext4_group_t groupnr; struct ext4_prealloc_space *pa; pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); + if (!pa->pa_len) + continue; ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); MB_CHECK_ASSERT(groupnr == e4b->bd_group); for (i = 0; i < pa->pa_len; i++) @@ -875,6 +892,21 @@ mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) } } +static ext4_group_t ext4_get_allocation_groups_count( + struct ext4_allocation_context *ac) +{ + ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); + + /* non-extent files are limited to low blocks/groups */ + if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) + ngroups = EXT4_SB(ac->ac_sb)->s_blockfile_groups; + + /* Pairs with smp_wmb() in ext4_update_super() */ + smp_rmb(); + + return ngroups; +} + static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct xarray *xa, ext4_group_t start, ext4_group_t end) @@ -882,7 +914,7 @@ static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct super_block *sb = ac->ac_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); enum criteria cr = ac->ac_criteria; - ext4_group_t ngroups = ext4_get_groups_count(sb); + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); unsigned long group = start; struct ext4_group_info *grp; @@ -934,7 +966,7 @@ static int ext4_mb_scan_groups_p2_aligned(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { ret = ext4_mb_scan_groups_largest_free_order_range(ac, i, @@ -984,7 +1016,7 @@ static int ext4_mb_scan_groups_goal_fast(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); for (; i < MB_NUM_ORDERS(ac->ac_sb); i++) { @@ -1066,7 +1098,7 @@ static int ext4_mb_scan_groups_best_avail(struct ext4_allocation_context *ac, min_order = fls(ac->ac_o_ex.fe_len); start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = order; i >= min_order; i--) { int frag_order; @@ -1116,8 +1148,6 @@ static inline int should_optimize_scan(struct ext4_allocation_context *ac) return 0; if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) return 0; - if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) - return 0; return 1; } @@ -1165,14 +1195,12 @@ static int ext4_mb_scan_groups(struct ext4_allocation_context *ac) int ret = 0; ext4_group_t start; struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); - ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); - - /* non-extent files are limited to low blocks/groups */ - if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) - ngroups = sbi->s_blockfile_groups; + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); /* searching for the right group start from the goal value specified */ start = ac->ac_g_ex.fe_group; + if (start >= ngroups) + start = 0; ac->ac_prefetch_grp = start; ac->ac_prefetch_nr = 0; @@ -1695,16 +1723,17 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, /* Avoid locking the folio in the fast path ... */ folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { + /* + * folio_test_locked is employed to detect ongoing folio + * migrations, since concurrent migrations can lead to + * bitmap inconsistency. And if we are not uptodate that + * implies somebody just created the folio but is yet to + * initialize it. We can drop the folio reference and + * try to get the folio with lock in both cases to avoid + * concurrency. + */ if (!IS_ERR(folio)) - /* - * drop the folio reference and try - * to get the folio with lock. If we - * are not uptodate that implies - * somebody just created the folio but - * is yet to initialize it. So - * wait for it to initialize. - */ folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); @@ -1746,7 +1775,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, poff = block % blocks_per_page; folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { if (!IS_ERR(folio)) folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, @@ -2413,8 +2442,12 @@ int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, return 0; err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); - if (err) + if (err) { + if (EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info) && + !(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) + return 0; return err; + } ext4_lock_group(ac->ac_sb, group); if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) @@ -3548,9 +3581,7 @@ static int ext4_mb_init_backend(struct super_block *sb) rcu_read_unlock(); iput(sbi->s_buddy_cache); err_freesgi: - rcu_read_lock(); - kvfree(rcu_dereference(sbi->s_group_info)); - rcu_read_unlock(); + kvfree(rcu_access_pointer(sbi->s_group_info)); return -ENOMEM; } @@ -3859,15 +3890,14 @@ void ext4_mb_release(struct super_block *sb) struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); int count; - if (test_opt(sb, DISCARD)) { - /* - * wait the discard work to drain all of ext4_free_data - */ - flush_work(&sbi->s_discard_work); - WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); - } + /* + * wait the discard work to drain all of ext4_free_data + */ + flush_work(&sbi->s_discard_work); + WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); - if (sbi->s_group_info) { + group_info = rcu_access_pointer(sbi->s_group_info); + if (group_info) { for (i = 0; i < ngroups; i++) { cond_resched(); grinfo = ext4_get_group_info(sb, i); @@ -3885,12 +3915,9 @@ void ext4_mb_release(struct super_block *sb) num_meta_group_infos = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> EXT4_DESC_PER_BLOCK_BITS(sb); - rcu_read_lock(); - group_info = rcu_dereference(sbi->s_group_info); for (i = 0; i < num_meta_group_infos; i++) kfree(group_info[i]); kvfree(group_info); - rcu_read_unlock(); } ext4_mb_avg_fragment_size_destroy(sbi); ext4_mb_largest_free_orders_destroy(sbi); @@ -4163,8 +4190,7 @@ ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, * Returns 0 if success or error code */ static noinline_for_stack int -ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, - handle_t *handle, unsigned int reserv_clstrs) +ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, handle_t *handle) { struct ext4_group_desc *gdp; struct ext4_sb_info *sbi; @@ -4219,13 +4245,6 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, BUG_ON(changed != ac->ac_b_ex.fe_len); #endif percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); - /* - * Now reduce the dirty block count also. Should not go negative - */ - if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); return err; } @@ -6310,7 +6329,7 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, ext4_mb_pa_put_free(ac); } if (likely(ac->ac_status == AC_STATUS_FOUND)) { - *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); + *errp = ext4_mb_mark_diskspace_used(ac, handle); if (*errp) { ext4_discard_allocated_blocks(ac); goto errout; @@ -6341,12 +6360,9 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, out: if (inquota && ar->len < inquota) dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); - if (!ar->len) { - if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); - } + /* release any reserved blocks */ + if (reserv_clstrs) + percpu_counter_sub(&sbi->s_dirtyclusters_counter, reserv_clstrs); trace_ext4_allocate_blocks(ar, (unsigned long long)block); diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c index 4b091c21908fd..0f4b7c89edd39 100644 --- a/fs/ext4/move_extent.c +++ b/fs/ext4/move_extent.c @@ -485,7 +485,7 @@ mext_check_arguments(struct inode *orig_inode, return -ETXTBSY; } - if (ext4_is_quota_file(orig_inode) && ext4_is_quota_file(donor_inode)) { + if (ext4_is_quota_file(orig_inode) || ext4_is_quota_file(donor_inode)) { ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %lu, donor %lu]\n", orig_inode->i_ino, donor_inode->i_ino); return -EOPNOTSUPP; diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index 82d5e75014555..fb57bba0d19d1 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -8,6 +8,8 @@ #include "ext4.h" #include "ext4_jbd2.h" +#define EXT4_MAX_ORPHAN_FILE_BLOCKS 512 + static int ext4_orphan_file_add(handle_t *handle, struct inode *inode) { int i, j, start; @@ -588,7 +590,7 @@ int ext4_init_orphan_info(struct super_block *sb) * consuming absurd amounts of memory when pinning blocks of orphan * file in memory. */ - if (inode->i_size > 8 << 20) { + if (inode->i_size > (EXT4_MAX_ORPHAN_FILE_BLOCKS << inode->i_blkbits)) { ext4_msg(sb, KERN_ERR, "orphan file too big: %llu", (unsigned long long)inode->i_size); ret = -EFSCORRUPTED; diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 39abfeec5f36c..0a3ef9bd68030 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -523,9 +523,15 @@ int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, nr_to_submit++; } while ((bh = bh->b_this_page) != head); - /* Nothing to submit? Just unlock the folio... */ - if (!nr_to_submit) + if (!nr_to_submit) { + /* + * We have nothing to submit. Just cycle the folio through + * writeback state to properly update xarray tags. + */ + __folio_start_writeback(folio, keep_towrite); + folio_end_writeback(folio); return 0; + } bh = head = folio_buffers(folio); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 33e7c08c9529c..cf4285db7fb8b 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1249,12 +1249,10 @@ static void ext4_group_desc_free(struct ext4_sb_info *sbi) struct buffer_head **group_desc; int i; - rcu_read_lock(); - group_desc = rcu_dereference(sbi->s_group_desc); + group_desc = rcu_access_pointer(sbi->s_group_desc); for (i = 0; i < sbi->s_gdb_count; i++) brelse(group_desc[i]); kvfree(group_desc); - rcu_read_unlock(); } static void ext4_flex_groups_free(struct ext4_sb_info *sbi) @@ -1262,14 +1260,12 @@ static void ext4_flex_groups_free(struct ext4_sb_info *sbi) struct flex_groups **flex_groups; int i; - rcu_read_lock(); - flex_groups = rcu_dereference(sbi->s_flex_groups); + flex_groups = rcu_access_pointer(sbi->s_flex_groups); if (flex_groups) { for (i = 0; i < sbi->s_flex_groups_allocated; i++) kvfree(flex_groups[i]); kvfree(flex_groups); } - rcu_read_unlock(); } static void ext4_put_super(struct super_block *sb) @@ -1396,6 +1392,7 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) inode_set_iversion(&ei->vfs_inode, 1); ei->i_flags = 0; + ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */ spin_lock_init(&ei->i_raw_lock); ei->i_prealloc_node = RB_ROOT; atomic_set(&ei->i_prealloc_active, 0); @@ -2475,7 +2472,7 @@ static int parse_apply_sb_mount_options(struct super_block *sb, struct ext4_fs_context *m_ctx) { struct ext4_sb_info *sbi = EXT4_SB(sb); - char s_mount_opts[65]; + char s_mount_opts[64]; struct ext4_fs_context *s_ctx = NULL; struct fs_context *fc = NULL; int ret = -ENOMEM; @@ -2483,7 +2480,8 @@ static int parse_apply_sb_mount_options(struct super_block *sb, if (!sbi->s_es->s_mount_opts[0]) return 0; - strscpy_pad(s_mount_opts, sbi->s_es->s_mount_opts); + if (strscpy_pad(s_mount_opts, sbi->s_es->s_mount_opts) < 0) + return -E2BIG; fc = kzalloc(sizeof(struct fs_context), GFP_KERNEL); if (!fc) @@ -3622,6 +3620,13 @@ int ext4_feature_set_ok(struct super_block *sb, int readonly) "extents feature\n"); return 0; } + if (ext4_has_feature_bigalloc(sb) && + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) { + ext4_msg(sb, KERN_WARNING, + "bad geometry: bigalloc file system with non-zero " + "first_data_block\n"); + return 0; + } #if !IS_ENABLED(CONFIG_QUOTA) || !IS_ENABLED(CONFIG_QFMT_V2) if (!readonly && (ext4_has_feature_quota(sb) || @@ -5352,6 +5357,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) timer_setup(&sbi->s_err_report, print_daily_error_info, 0); spin_lock_init(&sbi->s_error_lock); + mutex_init(&sbi->s_error_notify_mutex); INIT_WORK(&sbi->s_sb_upd_work, update_super_work); err = ext4_group_desc_init(sb, es, logical_sb_block, &first_not_zeroed); @@ -5563,6 +5569,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) clear_opt2(sb, MB_OPTIMIZE_SCAN); } + err = ext4_percpu_param_init(sbi); + if (err) + goto failed_mount5; + err = ext4_mb_init(sb); if (err) { ext4_msg(sb, KERN_ERR, "failed to initialize mballoc (%d)", @@ -5578,10 +5588,6 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) sbi->s_journal->j_commit_callback = ext4_journal_commit_callback; - err = ext4_percpu_param_init(sbi); - if (err) - goto failed_mount6; - if (ext4_has_feature_flex_bg(sb)) if (!ext4_fill_flex_info(sb)) { ext4_msg(sb, KERN_ERR, @@ -5663,8 +5669,8 @@ failed_mount8: __maybe_unused failed_mount6: ext4_mb_release(sb); ext4_flex_groups_free(sbi); - ext4_percpu_param_destroy(sbi); failed_mount5: + ext4_percpu_param_destroy(sbi); ext4_ext_release(sb); ext4_release_system_zone(sb); failed_mount4a: diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 987bd00f916af..3ec79a33d246e 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -555,7 +555,10 @@ static const struct kobj_type ext4_feat_ktype = { void ext4_notify_error_sysfs(struct ext4_sb_info *sbi) { - sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); + mutex_lock(&sbi->s_error_notify_mutex); + if (sbi->s_kobj.state_in_sysfs) + sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); + mutex_unlock(&sbi->s_error_notify_mutex); } static struct kobject *ext4_root; @@ -568,8 +571,10 @@ int ext4_register_sysfs(struct super_block *sb) int err; init_completion(&sbi->s_kobj_unregister); + mutex_lock(&sbi->s_error_notify_mutex); err = kobject_init_and_add(&sbi->s_kobj, &ext4_sb_ktype, ext4_root, "%s", sb->s_id); + mutex_unlock(&sbi->s_error_notify_mutex); if (err) { kobject_put(&sbi->s_kobj); wait_for_completion(&sbi->s_kobj_unregister); @@ -602,7 +607,10 @@ void ext4_unregister_sysfs(struct super_block *sb) if (sbi->s_proc) remove_proc_subtree(sb->s_id, ext4_proc_root); + + mutex_lock(&sbi->s_error_notify_mutex); kobject_del(&sbi->s_kobj); + mutex_unlock(&sbi->s_error_notify_mutex); } int __init ext4_init_sysfs(void) diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index ce7253b3f5499..4ed8ddf2a60b3 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -1037,6 +1037,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, ext4_error_inode(ea_inode, __func__, __LINE__, 0, "EA inode %lu ref wraparound: ref_count=%lld ref_change=%d", ea_inode->i_ino, ref_count, ref_change); + brelse(iloc.bh); ret = -EFSCORRUPTED; goto out; } @@ -1174,7 +1175,11 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, if (block_csum) end = (void *)bh->b_data + bh->b_size; else { - ext4_get_inode_loc(parent, &iloc); + err = ext4_get_inode_loc(parent, &iloc); + if (err) { + EXT4_ERROR_INODE(parent, "parent inode loc (error %d)", err); + return; + } end = (void *)ext4_raw_inode(&iloc) + EXT4_SB(parent->i_sb)->s_inode_size; } diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 6ad8d3bc6df7a..ae36bb1f93519 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -759,10 +759,7 @@ void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) ret = -EFSCORRUPTED; /* Avoid f2fs_commit_super in irq context */ - if (!in_task) - f2fs_handle_error_async(sbi, ERROR_FAIL_DECOMPRESSION); - else - f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); + f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); goto out_release; } @@ -1492,10 +1489,10 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) f2fs_compress_free_page(page); - dec_page_count(sbi, type); - - if (atomic_dec_return(&cic->pending_pages)) + if (atomic_dec_return(&cic->pending_pages)) { + dec_page_count(sbi, type); return; + } for (i = 0; i < cic->nr_rpages; i++) { WARN_ON(!cic->rpages[i]); @@ -1505,6 +1502,14 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) page_array_free(sbi, cic->rpages, cic->nr_rpages); kmem_cache_free(cic_entry_slab, cic); + + /* + * Make sure dec_page_count() is the last access to sbi. + * Once it drops the F2FS_WB_CP_DATA counter to zero, the + * unmount thread can proceed to destroy sbi and + * sbi->page_array_slab. + */ + dec_page_count(sbi, type); } static int f2fs_write_raw_pages(struct compress_ctx *cc, diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 775aa4f63aa30..dbbe134257b23 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -151,6 +151,13 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) } dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); + + if (bio->bi_status == BLK_STS_OK && + F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && + f2fs_sanity_check_node_footer(F2FS_F_SB(folio), + folio, folio->index, NODE_TYPE_REGULAR, true)) + bio->bi_status = BLK_STS_IOERR; + folio_end_read(folio, bio->bi_status == BLK_STS_OK); } @@ -352,18 +359,27 @@ static void f2fs_write_end_io(struct bio *bio) STOP_CP_REASON_WRITE_FAIL); } - f2fs_bug_on(sbi, is_node_folio(folio) && - folio->index != nid_of_node(folio)); - - dec_page_count(sbi, type); + if (is_node_folio(folio)) { + f2fs_sanity_check_node_footer(sbi, folio, + folio->index, NODE_TYPE_REGULAR, true); + f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); + } if (f2fs_in_warm_node_list(sbi, folio)) f2fs_del_fsync_node_entry(sbi, folio); + + dec_page_count(sbi, type); + + /* + * we should access sbi before folio_end_writeback() to + * avoid racing w/ kill_f2fs_super() + */ + if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && + wq_has_sleeper(&sbi->cp_wait)) + wake_up(&sbi->cp_wait); + folio_clear_f2fs_gcing(folio); folio_end_writeback(folio); } - if (!get_pages(sbi, F2FS_WB_CP_DATA) && - wq_has_sleeper(&sbi->cp_wait)) - wake_up(&sbi->cp_wait); bio_put(bio); } @@ -1793,7 +1809,8 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) return err; } -bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len, + bool check_first) { struct f2fs_map_blocks map; block_t last_lblk; @@ -1815,10 +1832,17 @@ bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) if (err || map.m_len == 0) return false; map.m_lblk += map.m_len; + if (check_first) + break; } return true; } +bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +{ + return __f2fs_overwrite_io(inode, pos, len, false); +} + static int f2fs_xattr_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo) { @@ -3224,6 +3248,19 @@ static inline bool __should_serialize_io(struct inode *inode, return false; } +static inline void account_writeback(struct inode *inode, bool inc) +{ + if (!f2fs_sb_has_compression(F2FS_I_SB(inode))) + return; + + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (inc) + atomic_inc(&F2FS_I(inode)->writeback); + else + atomic_dec(&F2FS_I(inode)->writeback); + f2fs_up_read(&F2FS_I(inode)->i_sem); +} + static int __f2fs_write_data_pages(struct address_space *mapping, struct writeback_control *wbc, enum iostat_type io_type) @@ -3269,10 +3306,14 @@ static int __f2fs_write_data_pages(struct address_space *mapping, locked = true; } + account_writeback(inode, true); + blk_start_plug(&plug); ret = f2fs_write_cache_pages(mapping, wbc, io_type); blk_finish_plug(&plug); + account_writeback(inode, false); + if (locked) mutex_unlock(&sbi->writepages); @@ -3916,6 +3957,7 @@ static int check_swap_activate(struct swap_info_struct *sis, while (cur_lblock < last_lblock && cur_lblock < sis->max) { struct f2fs_map_blocks map; + bool last_extent = false; retry: cond_resched(); @@ -3941,11 +3983,10 @@ static int check_swap_activate(struct swap_info_struct *sis, pblock = map.m_pblk; nr_pblocks = map.m_len; - if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || - nr_pblocks % blks_per_sec || - f2fs_is_sequential_zone_area(sbi, pblock)) { - bool last_extent = false; - + if (!last_extent && + ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || + nr_pblocks % blks_per_sec || + f2fs_is_sequential_zone_area(sbi, pblock))) { not_aligned++; nr_pblocks = roundup(nr_pblocks, blks_per_sec); @@ -3966,8 +4007,8 @@ static int check_swap_activate(struct swap_info_struct *sis, goto out; } - if (!last_extent) - goto retry; + /* lookup block mapping info after block migration */ + goto retry; } if (cur_lblock + nr_pblocks >= sis->max) @@ -4168,7 +4209,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, * f2fs_map_lock and f2fs_balance_fs are not necessary. */ if ((flags & IOMAP_WRITE) && - !f2fs_overwrite_io(inode, offset, length)) + !__f2fs_overwrite_io(inode, offset, length, true)) map.m_may_create = true; err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 33e09c453c703..0ed84cc065a7e 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -808,7 +808,7 @@ static void __update_extent_tree_range(struct inode *inode, } goto out_read_extent_cache; update_age_extent_cache: - if (!tei->last_blocks) + if (tei->last_blocks == F2FS_EXTENT_AGE_INVALID) goto out_read_extent_cache; __set_extent_info(&ei, fofs, len, 0, false, @@ -912,7 +912,7 @@ static int __get_new_block_age(struct inode *inode, struct extent_info *ei, cur_age = cur_blocks - tei.last_blocks; else /* allocated_data_blocks overflow */ - cur_age = ULLONG_MAX - tei.last_blocks + cur_blocks; + cur_age = (ULLONG_MAX - 1) - tei.last_blocks + cur_blocks; if (tei.age) ei->age = __calculate_block_age(sbi, cur_age, tei.age); @@ -1114,6 +1114,7 @@ void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn, struct extent_info ei = { .fofs = fofs, .len = len, + .last_blocks = F2FS_EXTENT_AGE_INVALID, }; if (!__may_extent_tree(dn->inode, EX_BLOCK_AGE)) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 5b4e9548a231f..77ff143624f9c 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -245,6 +245,7 @@ struct f2fs_mount_info { #define F2FS_FEATURE_COMPRESSION 0x00002000 #define F2FS_FEATURE_RO 0x00004000 #define F2FS_FEATURE_DEVICE_ALIAS 0x00008000 +#define F2FS_FEATURE_PACKED_SSA 0x00010000 #define __F2FS_HAS_FEATURE(raw_super, mask) \ ((raw_super->feature & cpu_to_le32(mask)) != 0) @@ -507,13 +508,25 @@ struct fsync_inode_entry { #define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats)) #define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits)) -#define nat_in_journal(jnl, i) ((jnl)->nat_j.entries[i].ne) -#define nid_in_journal(jnl, i) ((jnl)->nat_j.entries[i].nid) -#define sit_in_journal(jnl, i) ((jnl)->sit_j.entries[i].se) -#define segno_in_journal(jnl, i) ((jnl)->sit_j.entries[i].segno) - -#define MAX_NAT_JENTRIES(jnl) (NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl)) -#define MAX_SIT_JENTRIES(jnl) (SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl)) +#define nat_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].ne) +#define nid_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].nid) +#define sit_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].se) +#define segno_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].segno) + +#define sum_entries(sum) ((struct f2fs_summary *)(sum)) +#define sum_journal(sbi, sum) \ + ((struct f2fs_journal *)((char *)(sum) + \ + ((sbi)->entries_in_sum * sizeof(struct f2fs_summary)))) +#define sum_footer(sbi, sum) \ + ((struct summary_footer *)((char *)(sum) + (sbi)->sum_blocksize - \ + sizeof(struct summary_footer))) + +#define MAX_NAT_JENTRIES(sbi, jnl) ((sbi)->nat_journal_entries - nats_in_cursum(jnl)) +#define MAX_SIT_JENTRIES(sbi, jnl) ((sbi)->sit_journal_entries - sits_in_cursum(jnl)) static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i) { @@ -531,14 +544,6 @@ static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i) return before; } -static inline bool __has_cursum_space(struct f2fs_journal *journal, - int size, int type) -{ - if (type == NAT_JOURNAL) - return size <= MAX_NAT_JENTRIES(journal); - return size <= MAX_SIT_JENTRIES(journal); -} - /* for inline stuff */ #define DEF_INLINE_RESERVED_SIZE 1 static inline int get_extra_isize(struct inode *inode); @@ -707,6 +712,12 @@ enum extent_type { NR_EXTENT_CACHES, }; +/* + * Reserved value to mark invalid age extents, hence valid block range + * from 0 to ULLONG_MAX-1 + */ +#define F2FS_EXTENT_AGE_INVALID ULLONG_MAX + struct extent_info { unsigned int fofs; /* start offset in a file */ unsigned int len; /* length of the extent */ @@ -947,6 +958,7 @@ struct f2fs_inode_info { unsigned char i_compress_level; /* compress level (lz4hc,zstd) */ unsigned char i_compress_flag; /* compress flag */ unsigned int i_cluster_size; /* cluster size */ + atomic_t writeback; /* count # of writeback thread */ unsigned int atomic_write_cnt; loff_t original_i_size; /* original i_size before atomic write */ @@ -1504,6 +1516,15 @@ enum f2fs_lookup_mode { LOOKUP_AUTO, }; +/* For node type in __get_node_folio() */ +enum node_type { + NODE_TYPE_REGULAR, + NODE_TYPE_INODE, + NODE_TYPE_XATTR, + NODE_TYPE_NON_INODE, +}; + + static inline int f2fs_test_bit(unsigned int nr, char *addr); static inline void f2fs_set_bit(unsigned int nr, char *addr); static inline void f2fs_clear_bit(unsigned int nr, char *addr); @@ -1742,6 +1763,15 @@ struct f2fs_sb_info { bool readdir_ra; /* readahead inode in readdir */ u64 max_io_bytes; /* max io bytes to merge IOs */ + /* variable summary block units */ + unsigned int sum_blocksize; /* sum block size */ + unsigned int sums_per_block; /* sum block count per block */ + unsigned int entries_in_sum; /* entry count in sum block */ + unsigned int sum_entry_size; /* total entry size in sum block */ + unsigned int sum_journal_size; /* journal size in sum block */ + unsigned int nat_journal_entries; /* nat journal entry count in the journal */ + unsigned int sit_journal_entries; /* sit journal entry count in the journal */ + block_t user_block_count; /* # of user blocks */ block_t total_valid_block_count; /* # of valid blocks */ block_t discard_blks; /* discard command candidats */ @@ -1884,9 +1914,6 @@ struct f2fs_sb_info { spinlock_t error_lock; /* protect errors/stop_reason array */ bool error_dirty; /* errors of sb is dirty */ - struct kmem_cache *inline_xattr_slab; /* inline xattr entry */ - unsigned int inline_xattr_slab_size; /* default inline xattr slab size */ - /* For reclaimed segs statistics per each GC mode */ unsigned int gc_segment_mode; /* GC state for reclaimed segments */ unsigned int gc_reclaimed_segs[MAX_GC_MODE]; /* Reclaimed segs for each mode */ @@ -2794,6 +2821,14 @@ static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); } +static inline bool __has_cursum_space(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int size, int type) +{ + if (type == NAT_JOURNAL) + return size <= MAX_NAT_JENTRIES(sbi, journal); + return size <= MAX_SIT_JENTRIES(sbi, journal); +} + extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync); static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, struct inode *inode, bool is_inode) @@ -3810,7 +3845,6 @@ void f2fs_quota_off_umount(struct super_block *sb); void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag); void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason); void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error); -void f2fs_handle_error_async(struct f2fs_sb_info *sbi, unsigned char error); int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover); int f2fs_sync_fs(struct super_block *sb, int sync); int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi); @@ -3849,6 +3883,9 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs); void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid); struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, enum node_type node_type); +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t nid, + enum node_type ntype, bool in_irq); struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino); struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid); int f2fs_move_node_folio(struct folio *node_folio, int gc_type); @@ -3893,7 +3930,7 @@ bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr); int f2fs_start_discard_thread(struct f2fs_sb_info *sbi); void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi); -bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi); +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check); void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc); void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi); @@ -3948,7 +3985,8 @@ void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, block_t len); void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk); void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk); -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, unsigned int val, int alloc); void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi); @@ -4674,7 +4712,7 @@ static inline bool f2fs_disable_compressed_file(struct inode *inode) f2fs_up_write(&fi->i_sem); return true; } - if (f2fs_is_mmap_file(inode) || + if (f2fs_is_mmap_file(inode) || atomic_read(&fi->writeback) || (S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))) { f2fs_up_write(&fi->i_sem); return false; @@ -4710,6 +4748,7 @@ F2FS_FEATURE_FUNCS(casefold, CASEFOLD); F2FS_FEATURE_FUNCS(compression, COMPRESSION); F2FS_FEATURE_FUNCS(readonly, RO); F2FS_FEATURE_FUNCS(device_alias, DEVICE_ALIAS); +F2FS_FEATURE_FUNCS(packed_ssa, PACKED_SSA); #ifdef CONFIG_BLK_DEV_ZONED static inline bool f2fs_zone_is_seq(struct f2fs_sb_info *sbi, int devi, diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index ffa045b39c01d..6d42e2d28861c 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1654,8 +1654,11 @@ static int f2fs_do_zero_range(struct dnode_of_data *dn, pgoff_t start, f2fs_set_data_blkaddr(dn, NEW_ADDR); } - f2fs_update_read_extent_cache_range(dn, start, 0, index - start); - f2fs_update_age_extent_cache_range(dn, start, index - start); + if (index > start) { + f2fs_update_read_extent_cache_range(dn, start, 0, + index - start); + f2fs_update_age_extent_cache_range(dn, start, index - start); + } return ret; } @@ -2125,8 +2128,9 @@ static int f2fs_setflags_common(struct inode *inode, u32 iflags, u32 mask) f2fs_down_write(&fi->i_sem); if (!f2fs_may_compress(inode) || - (S_ISREG(inode->i_mode) && - F2FS_HAS_BLOCKS(inode))) { + atomic_read(&fi->writeback) || + (S_ISREG(inode->i_mode) && + F2FS_HAS_BLOCKS(inode))) { f2fs_up_write(&fi->i_sem); return -EINVAL; } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index a7708cf80c04e..9d2f4f22fd396 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -38,13 +38,14 @@ static int gc_thread_func(void *data) struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, .should_migrate_blocks = false, - .err_gc_skipped = false }; + .err_gc_skipped = false, + .one_time = false }; wait_ms = gc_th->min_sleep_time; set_freezable(); do { - bool sync_mode, foreground = false; + bool sync_mode, foreground = false, gc_boost = false; wait_event_freezable_timeout(*wq, kthread_should_stop() || @@ -52,8 +53,12 @@ static int gc_thread_func(void *data) gc_th->gc_wake, msecs_to_jiffies(wait_ms)); - if (test_opt(sbi, GC_MERGE) && waitqueue_active(fggc_wq)) + if (test_opt(sbi, GC_MERGE) && waitqueue_active(fggc_wq)) { foreground = true; + gc_control.one_time = false; + } else if (f2fs_sb_has_blkzoned(sbi)) { + gc_control.one_time = true; + } /* give it a try one time */ if (gc_th->gc_wake) @@ -81,8 +86,6 @@ static int gc_thread_func(void *data) continue; } - gc_control.one_time = false; - /* * [GC triggering condition] * 0. GC is not conducted currently. @@ -132,7 +135,7 @@ static int gc_thread_func(void *data) if (need_to_boost_gc(sbi)) { decrease_sleep_time(gc_th, &wait_ms); if (f2fs_sb_has_blkzoned(sbi)) - gc_control.one_time = true; + gc_boost = true; } else { increase_sleep_time(gc_th, &wait_ms); } @@ -141,7 +144,7 @@ static int gc_thread_func(void *data) FOREGROUND : BACKGROUND); sync_mode = (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC) || - (gc_control.one_time && gc_th->boost_gc_greedy); + (gc_boost && gc_th->boost_gc_greedy); /* foreground GC was been triggered via f2fs_balance_fs() */ if (foreground && !f2fs_sb_has_blkzoned(sbi)) @@ -771,7 +774,7 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, { struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); struct sit_info *sm = SIT_I(sbi); - struct victim_sel_policy p; + struct victim_sel_policy p = {0}; unsigned int secno, last_victim; unsigned int last_segment; unsigned int nsearched; @@ -1729,7 +1732,7 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, unsigned char type = IS_DATASEG(get_seg_entry(sbi, segno)->type) ? SUM_TYPE_DATA : SUM_TYPE_NODE; unsigned char data_type = (type == SUM_TYPE_DATA) ? DATA : NODE; - int submitted = 0; + int submitted = 0, sum_blk_cnt; if (__is_large_section(sbi)) { sec_end_segno = rounddown(end_segno, SEGS_PER_SEC(sbi)); @@ -1763,22 +1766,28 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, sanity_check_seg_type(sbi, get_seg_entry(sbi, segno)->type); + segno = rounddown(segno, sbi->sums_per_block); + sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, sbi->sums_per_block); /* readahead multi ssa blocks those have contiguous address */ if (__is_large_section(sbi)) f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno), - end_segno - segno, META_SSA, true); + sum_blk_cnt, META_SSA, true); /* reference all summary page */ while (segno < end_segno) { - struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno++); + struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno); + + segno += sbi->sums_per_block; if (IS_ERR(sum_folio)) { int err = PTR_ERR(sum_folio); - end_segno = segno - 1; - for (segno = start_segno; segno < end_segno; segno++) { + end_segno = segno - sbi->sums_per_block; + segno = rounddown(start_segno, sbi->sums_per_block); + while (segno < end_segno) { sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); folio_put_refs(sum_folio, 2); + segno += sbi->sums_per_block; } return err; } @@ -1787,68 +1796,84 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, blk_start_plug(&plug); - for (segno = start_segno; segno < end_segno; segno++) { - struct f2fs_summary_block *sum; + segno = start_segno; + while (segno < end_segno) { + unsigned int cur_segno; /* find segment summary of victim */ struct folio *sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); + unsigned int block_end_segno = rounddown(segno, sbi->sums_per_block) + + sbi->sums_per_block; + + if (block_end_segno > end_segno) + block_end_segno = end_segno; if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, segno))) { f2fs_err(sbi, "%s: segment %u is used by log", __func__, segno); f2fs_bug_on(sbi, 1); - goto skip; + goto next_block; } - if (get_valid_blocks(sbi, segno, false) == 0) - goto freed; - if (gc_type == BG_GC && __is_large_section(sbi) && - migrated >= sbi->migration_granularity) - goto skip; if (!folio_test_uptodate(sum_folio) || unlikely(f2fs_cp_error(sbi))) - goto skip; + goto next_block; - sum = folio_address(sum_folio); - if (type != GET_SUM_TYPE((&sum->footer))) { - f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SIT and SSA", - segno, type, GET_SUM_TYPE((&sum->footer))); - f2fs_stop_checkpoint(sbi, false, - STOP_CP_REASON_CORRUPTED_SUMMARY); - goto skip; - } + for (cur_segno = segno; cur_segno < block_end_segno; + cur_segno++) { + struct f2fs_summary_block *sum; - /* - * this is to avoid deadlock: - * - lock_page(sum_page) - f2fs_replace_block - * - check_valid_map() - down_write(sentry_lock) - * - down_read(sentry_lock) - change_curseg() - * - lock_page(sum_page) - */ - if (type == SUM_TYPE_NODE) - submitted += gc_node_segment(sbi, sum->entries, segno, - gc_type); - else - submitted += gc_data_segment(sbi, sum->entries, gc_list, - segno, gc_type, - force_migrate); + if (get_valid_blocks(sbi, cur_segno, false) == 0) + goto freed; + if (gc_type == BG_GC && __is_large_section(sbi) && + migrated >= sbi->migration_granularity) + continue; - stat_inc_gc_seg_count(sbi, data_type, gc_type); - sbi->gc_reclaimed_segs[sbi->gc_mode]++; - migrated++; + sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno); + if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) { + f2fs_err(sbi, "Inconsistent segment (%u) type " + "[%d, %d] in SSA and SIT", + cur_segno, type, + GET_SUM_TYPE( + sum_footer(sbi, sum))); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_CORRUPTED_SUMMARY); + continue; + } -freed: - if (gc_type == FG_GC && - get_valid_blocks(sbi, segno, false) == 0) - seg_freed++; + /* + * this is to avoid deadlock: + * - lock_page(sum_page) - f2fs_replace_block + * - check_valid_map() - down_write(sentry_lock) + * - down_read(sentry_lock) - change_curseg() + * - lock_page(sum_page) + */ + if (type == SUM_TYPE_NODE) + submitted += gc_node_segment(sbi, sum->entries, + cur_segno, gc_type); + else + submitted += gc_data_segment(sbi, sum->entries, + gc_list, cur_segno, + gc_type, force_migrate); - if (__is_large_section(sbi)) - sbi->next_victim_seg[gc_type] = - (segno + 1 < sec_end_segno) ? - segno + 1 : NULL_SEGNO; -skip: + stat_inc_gc_seg_count(sbi, data_type, gc_type); + sbi->gc_reclaimed_segs[sbi->gc_mode]++; + migrated++; + +freed: + if (gc_type == FG_GC && + get_valid_blocks(sbi, cur_segno, false) == 0) + seg_freed++; + + if (__is_large_section(sbi)) + sbi->next_victim_seg[gc_type] = + (cur_segno + 1 < sec_end_segno) ? + cur_segno + 1 : NULL_SEGNO; + } +next_block: folio_put_refs(sum_folio, 2); + segno = block_end_segno; } if (submitted) @@ -2069,6 +2094,7 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, if (unlikely(f2fs_cp_error(sbi))) return -EIO; + stat_inc_gc_call_count(sbi, FOREGROUND); for (segno = start_seg; segno <= end_seg; segno += SEGS_PER_SEC(sbi)) { struct gc_inode_list gc_list = { .ilist = LIST_HEAD_INIT(gc_list.ilist), diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index b882771e46997..fb0b0b35ad26d 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -949,6 +949,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, return err; err = f2fs_create_whiteout(idmap, old_dir, &whiteout, &fname); + f2fs_free_filename(&fname); if (err) return err; } @@ -1053,9 +1054,11 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, if (whiteout) { set_inode_flag(whiteout, FI_INC_LINK); err = f2fs_add_link(old_dentry, whiteout); - if (err) + if (err) { + d_invalidate(old_dentry); + d_invalidate(new_dentry); goto put_out_dir; - + } spin_lock(&whiteout->i_lock); whiteout->i_state &= ~I_LINKABLE; spin_unlock(&whiteout->i_lock); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 482a362f26254..591fcdf3ba77b 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -606,7 +606,7 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, goto retry; } - i = f2fs_lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0); + i = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 0); if (i >= 0) { ne = nat_in_journal(journal, i); node_info_from_raw_nat(ni, &ne); @@ -1500,9 +1500,9 @@ void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) f2fs_folio_put(afolio, err ? true : false); } -static int sanity_check_node_footer(struct f2fs_sb_info *sbi, +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, struct folio *folio, pgoff_t nid, - enum node_type ntype) + enum node_type ntype, bool in_irq) { if (unlikely(nid != nid_of_node(folio))) goto out_err; @@ -1527,12 +1527,13 @@ static int sanity_check_node_footer(struct f2fs_sb_info *sbi, goto out_err; return 0; out_err: - f2fs_warn(sbi, "inconsistent node block, node_type:%d, nid:%lu, " - "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", - ntype, nid, nid_of_node(folio), ino_of_node(folio), - ofs_of_node(folio), cpver_of_node(folio), - next_blkaddr_of_node(folio)); set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, " + "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", + ntype, nid, nid_of_node(folio), ino_of_node(folio), + ofs_of_node(folio), cpver_of_node(folio), + next_blkaddr_of_node(folio)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); return -EFSCORRUPTED; } @@ -1578,7 +1579,7 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, goto out_err; } page_hit: - err = sanity_check_node_footer(sbi, folio, nid, ntype); + err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false); if (!err) return folio; out_err: @@ -1751,7 +1752,12 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted /* get old block addr of this node page */ nid = nid_of_node(folio); - f2fs_bug_on(sbi, folio->index != nid); + + if (f2fs_sanity_check_node_footer(sbi, folio, nid, + NODE_TYPE_REGULAR, false)) { + f2fs_handle_critical_error(sbi, STOP_CP_REASON_CORRUPTED_NID); + goto redirty_out; + } if (f2fs_get_node_info(sbi, nid, &ni, !do_balance)) goto redirty_out; @@ -1774,8 +1780,13 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted goto redirty_out; } - if (atomic && !test_opt(sbi, NOBARRIER)) - fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (atomic) { + if (!test_opt(sbi, NOBARRIER)) + fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (IS_INODE(folio)) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino_of_node(folio))); + } /* should add to global list before clearing PAGECACHE status */ if (f2fs_in_warm_node_list(sbi, folio)) { @@ -1916,8 +1927,9 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, if (is_inode_flag_set(inode, FI_DIRTY_INODE)) f2fs_update_inode(inode, folio); - set_dentry_mark(folio, - f2fs_need_dentry_mark(sbi, ino)); + if (!atomic) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino)); } /* may be written by other thread */ if (!folio_test_dirty(folio)) @@ -2937,7 +2949,7 @@ int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, /* scan the node segment */ last_offset = BLKS_PER_SEG(sbi); addr = START_BLOCK(sbi, segno); - sum_entry = &sum->entries[0]; + sum_entry = sum_entries(sum); for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { nrpages = bio_max_segs(last_offset - i); @@ -3078,7 +3090,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, * #2, flush nat entries to nat page. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL)) + !__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL)) to_journal = false; if (to_journal) { @@ -3101,7 +3113,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, nat_get_blkaddr(ne) == NEW_ADDR); if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 1); f2fs_bug_on(sbi, offset < 0); raw_ne = &nat_in_journal(journal, offset); @@ -3172,7 +3184,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * into nat entry set. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, + !__has_cursum_space(sbi, journal, nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL)) remove_nats_in_journal(sbi); @@ -3183,7 +3195,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) set_idx = setvec[found - 1]->set + 1; for (idx = 0; idx < found; idx++) __adjust_nat_entry_set(setvec[idx], &sets, - MAX_NAT_JENTRIES(journal)); + MAX_NAT_JENTRIES(sbi, journal)); } /* flush dirty nats in nat entry set */ diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index 9cb8dcf8d4176..824ac9f0e6e42 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -52,14 +52,6 @@ enum { IS_PREALLOC, /* nat entry is preallocated */ }; -/* For node type in __get_node_folio() */ -enum node_type { - NODE_TYPE_REGULAR, - NODE_TYPE_INODE, - NODE_TYPE_XATTR, - NODE_TYPE_NON_INODE, -}; - /* * For node information */ diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 215e442db72c8..06674e694b271 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -399,7 +399,7 @@ static int sanity_check_node_chain(struct f2fs_sb_info *sbi, block_t blkaddr, } static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head, - bool check_only) + bool check_only, bool *new_inode) { struct curseg_info *curseg; block_t blkaddr, blkaddr_fast; @@ -447,16 +447,19 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head, quota_inode = true; } - /* - * CP | dnode(F) | inode(DF) - * For this case, we should not give up now. - */ entry = add_fsync_inode(sbi, head, ino_of_node(folio), quota_inode); if (IS_ERR(entry)) { err = PTR_ERR(entry); - if (err == -ENOENT) + /* + * CP | dnode(F) | inode(DF) + * For this case, we should not give up now. + */ + if (err == -ENOENT) { + if (check_only) + *new_inode = true; goto next; + } f2fs_folio_put(folio, true); break; } @@ -511,7 +514,7 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, struct curseg_info *curseg = CURSEG_I(sbi, i); if (curseg->segno == segno) { - sum = curseg->sum_blk->entries[blkoff]; + sum = sum_entries(curseg->sum_blk)[blkoff]; goto got_it; } } @@ -519,8 +522,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, sum_folio = f2fs_get_sum_folio(sbi, segno); if (IS_ERR(sum_folio)) return PTR_ERR(sum_folio); - sum_node = folio_address(sum_folio); - sum = sum_node->entries[blkoff]; + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, segno); + sum = sum_entries(sum_node)[blkoff]; f2fs_folio_put(sum_folio, true); got_it: /* Use the locked dnode page and inode */ @@ -875,6 +878,7 @@ int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only) int ret = 0; unsigned long s_flags = sbi->sb->s_flags; bool need_writecp = false; + bool new_inode = false; f2fs_notice(sbi, "f2fs_recover_fsync_data: recovery fsync data, " "check_only: %d", check_only); @@ -890,8 +894,8 @@ int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only) f2fs_down_write(&sbi->cp_global_sem); /* step #1: find fsynced inode numbers */ - err = find_fsync_dnodes(sbi, &inode_list, check_only); - if (err || list_empty(&inode_list)) + err = find_fsync_dnodes(sbi, &inode_list, check_only, &new_inode); + if (err < 0 || (list_empty(&inode_list) && (!check_only || !new_inode))) goto skip; if (check_only) { diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index b45eace879d74..a7cf8627d8881 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -1895,7 +1895,7 @@ void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) * * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false. */ -bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check) { struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct discard_policy dpolicy; @@ -1912,7 +1912,7 @@ bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) /* just to make sure there is no pending discard commands */ __wait_all_discard_cmd(sbi, NULL); - f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); + f2fs_bug_on(sbi, need_check && atomic_read(&dcc->discard_cmd_cnt)); return !dropped; } @@ -2382,7 +2382,7 @@ static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) * Recovery can cache discard commands, so in error path of * fill_super(), it needs to give a chance to handle them. */ - f2fs_issue_discard_timeout(sbi); + f2fs_issue_discard_timeout(sbi, true); kfree(dcc); SM_I(sbi)->dcc_info = NULL; @@ -2689,12 +2689,12 @@ int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); } - sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - + sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size - SUM_FOOTER_SIZE) / SUMMARY_SIZE; if (valid_sum_count <= sum_in_page) return 1; else if ((valid_sum_count - sum_in_page) <= - (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) + (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) return 2; return 3; } @@ -2712,7 +2712,15 @@ struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno) void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr) { - struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr); + struct folio *folio; + + if (!f2fs_sb_has_packed_ssa(sbi)) + folio = f2fs_grab_meta_folio(sbi, blk_addr); + else + folio = f2fs_get_meta_folio_retry(sbi, blk_addr); + + if (IS_ERR(folio)) + return; memcpy(folio_address(folio), src, PAGE_SIZE); folio_mark_dirty(folio); @@ -2720,9 +2728,22 @@ void f2fs_update_meta_page(struct f2fs_sb_info *sbi, } static void write_sum_page(struct f2fs_sb_info *sbi, - struct f2fs_summary_block *sum_blk, block_t blk_addr) + struct f2fs_summary_block *sum_blk, unsigned int segno) { - f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr); + struct folio *folio; + + if (!f2fs_sb_has_packed_ssa(sbi)) + return f2fs_update_meta_page(sbi, (void *)sum_blk, + GET_SUM_BLOCK(sbi, segno)); + + folio = f2fs_get_sum_folio(sbi, segno); + if (IS_ERR(folio)) + return; + + memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk, + sbi->sum_blocksize); + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); } static void write_current_sum_page(struct f2fs_sb_info *sbi, @@ -2739,11 +2760,11 @@ static void write_current_sum_page(struct f2fs_sb_info *sbi, mutex_lock(&curseg->curseg_mutex); down_read(&curseg->journal_rwsem); - memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); + memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size); up_read(&curseg->journal_rwsem); - memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); - memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size); + memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE); mutex_unlock(&curseg->curseg_mutex); @@ -2916,7 +2937,7 @@ static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) curseg->next_blkoff = 0; curseg->next_segno = NULL_SEGNO; - sum_footer = &(curseg->sum_blk->footer); + sum_footer = sum_footer(sbi, curseg->sum_blk); memset(sum_footer, 0, sizeof(struct summary_footer)); sanity_check_seg_type(sbi, seg_type); @@ -2987,7 +3008,7 @@ static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) int ret; if (curseg->inited) - write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, segno)); + write_sum_page(sbi, curseg->sum_blk, segno); segno = __get_next_segno(sbi, type); ret = get_new_segment(sbi, &segno, new_sec, pinning); @@ -3046,7 +3067,7 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) struct folio *sum_folio; if (curseg->inited) - write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno)); + write_sum_page(sbi, curseg->sum_blk, curseg->segno); __set_test_and_inuse(sbi, new_segno); @@ -3062,11 +3083,11 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) sum_folio = f2fs_get_sum_folio(sbi, new_segno); if (IS_ERR(sum_folio)) { /* GC won't be able to use stale summary pages by cp_error */ - memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE); + memset(curseg->sum_blk, 0, sbi->sum_entry_size); return PTR_ERR(sum_folio); } - sum_node = folio_address(sum_folio); - memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno); + memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); f2fs_folio_put(sum_folio, true); return 0; } @@ -3154,8 +3175,7 @@ static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) goto out; if (get_valid_blocks(sbi, curseg->segno, false)) { - write_sum_page(sbi, curseg->sum_blk, - GET_SUM_BLOCK(sbi, curseg->segno)); + write_sum_page(sbi, curseg->sum_blk, curseg->segno); } else { mutex_lock(&DIRTY_I(sbi)->seglist_lock); __set_test_and_free(sbi, curseg->segno, true); @@ -3799,7 +3819,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, f2fs_wait_discard_bio(sbi, *new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (curseg->alloc_type == SSR) { curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { @@ -3833,8 +3853,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, if (segment_full) { if (type == CURSEG_COLD_DATA_PINNED && !((curseg->segno + 1) % sbi->segs_per_sec)) { - write_sum_page(sbi, curseg->sum_blk, - GET_SUM_BLOCK(sbi, curseg->segno)); + write_sum_page(sbi, curseg->sum_blk, curseg->segno); reset_curseg_fields(curseg); goto skip_new_segment; } @@ -3863,8 +3882,13 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); - if (IS_DATASEG(curseg->seg_type)) - atomic64_inc(&sbi->allocated_data_blocks); + if (IS_DATASEG(curseg->seg_type)) { + unsigned long long new_val; + + new_val = atomic64_inc_return(&sbi->allocated_data_blocks); + if (unlikely(new_val == ULLONG_MAX)) + atomic64_set(&sbi->allocated_data_blocks, 0); + } up_write(&sit_i->sentry_lock); @@ -4164,7 +4188,7 @@ void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, } curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (!recover_curseg || recover_newaddr) { if (!from_gc) @@ -4284,12 +4308,12 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) /* Step 1: restore nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); + memcpy(seg_i->journal, kaddr, sbi->sum_journal_size); /* Step 2: restore sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); - offset = 2 * SUM_JOURNAL_SIZE; + memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size); + offset = 2 * sbi->sum_journal_size; /* Step 3: restore summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4311,9 +4335,9 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) struct f2fs_summary *s; s = (struct f2fs_summary *)(kaddr + offset); - seg_i->sum_blk->entries[j] = *s; + sum_entries(seg_i->sum_blk)[j] = *s; offset += SUMMARY_SIZE; - if (offset + SUMMARY_SIZE <= PAGE_SIZE - + if (offset + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4369,7 +4393,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) if (IS_NODESEG(type)) { if (__exist_node_summaries(sbi)) { - struct f2fs_summary *ns = &sum->entries[0]; + struct f2fs_summary *ns = sum_entries(sum); int i; for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) { @@ -4389,11 +4413,13 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) /* update journal info */ down_write(&curseg->journal_rwsem); - memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); + memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size); up_write(&curseg->journal_rwsem); - memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); - memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(curseg->sum_blk), sum_entries(sum), + sbi->sum_entry_size); + memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum), + SUM_FOOTER_SIZE); curseg->next_segno = segno; reset_curseg(sbi, type, 0); curseg->alloc_type = ckpt->alloc_type[type]; @@ -4437,8 +4463,8 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi) } /* sanity check for summary blocks */ - if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || - sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) { + if (nats_in_cursum(nat_j) > sbi->nat_journal_entries || + sits_in_cursum(sit_j) > sbi->sit_journal_entries) { f2fs_err(sbi, "invalid journal entries nats %u sits %u", nats_in_cursum(nat_j), sits_in_cursum(sit_j)); return -EINVAL; @@ -4462,13 +4488,13 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) /* Step 1: write nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 2: write sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 3: write summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4481,10 +4507,10 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) written_size = 0; } summary = (struct f2fs_summary *)(kaddr + written_size); - *summary = seg_i->sum_blk->entries[j]; + *summary = sum_entries(seg_i->sum_blk)[j]; written_size += SUMMARY_SIZE; - if (written_size + SUMMARY_SIZE <= PAGE_SIZE - + if (written_size + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4526,8 +4552,9 @@ void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); } -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, - unsigned int val, int alloc) +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, + unsigned int val, int alloc) { int i; @@ -4536,13 +4563,13 @@ int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, if (le32_to_cpu(nid_in_journal(journal, i)) == val) return i; } - if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL)) return update_nats_in_cursum(journal, 1); } else if (type == SIT_JOURNAL) { for (i = 0; i < sits_in_cursum(journal); i++) if (le32_to_cpu(segno_in_journal(journal, i)) == val) return i; - if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL)) return update_sits_in_cursum(journal, 1); } return -1; @@ -4690,8 +4717,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * entries, remove all entries from journal and add and account * them in sit entry set. */ - if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) || - !to_journal) + if (!__has_cursum_space(sbi, journal, + sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal) remove_sits_in_journal(sbi); /* @@ -4708,7 +4735,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) unsigned int segno = start_segno; if (to_journal && - !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) + !__has_cursum_space(sbi, journal, ses->entry_cnt, + SIT_JOURNAL)) to_journal = false; if (to_journal) { @@ -4736,7 +4764,7 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) } if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, SIT_JOURNAL, segno, 1); f2fs_bug_on(sbi, offset < 0); segno_in_journal(journal, offset) = @@ -4943,12 +4971,13 @@ static int build_curseg(struct f2fs_sb_info *sbi) for (i = 0; i < NO_CHECK_TYPE; i++) { mutex_init(&array[i].curseg_mutex); - array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); + array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize, + GFP_KERNEL); if (!array[i].sum_blk) return -ENOMEM; init_rwsem(&array[i].journal_rwsem); array[i].journal = f2fs_kzalloc(sbi, - sizeof(struct f2fs_journal), GFP_KERNEL); + sbi->sum_journal_size, GFP_KERNEL); if (!array[i].journal) return -ENOMEM; array[i].seg_type = log_type_to_seg_type(i); diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 1ce2c8abaf488..d05d133c89aff 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -85,8 +85,11 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, #define GET_ZONE_FROM_SEG(sbi, segno) \ GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno)) -#define GET_SUM_BLOCK(sbi, segno) \ - ((sbi)->sm_info->ssa_blkaddr + (segno)) +#define GET_SUM_BLOCK(sbi, segno) \ + (SM_I(sbi)->ssa_blkaddr + (segno / (sbi)->sums_per_block)) +#define GET_SUM_BLKOFF(sbi, segno) (segno % (sbi)->sums_per_block) +#define SUM_BLK_PAGE_ADDR(sbi, folio, segno) \ + (folio_address(folio) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize) #define GET_SUM_TYPE(footer) ((footer)->entry_type) #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type)) @@ -603,10 +606,12 @@ static inline int reserved_sections(struct f2fs_sb_info *sbi) static inline unsigned int get_left_section_blocks(struct f2fs_sb_info *sbi, enum log_type type, unsigned int segno) { - if (f2fs_lfs_mode(sbi) && __is_large_section(sbi)) - return CAP_BLKS_PER_SEC(sbi) - SEGS_TO_BLKS(sbi, - (segno - GET_START_SEG_FROM_SEC(sbi, segno))) - + if (f2fs_lfs_mode(sbi)) { + unsigned int used_blocks = __is_large_section(sbi) ? SEGS_TO_BLKS(sbi, + (segno - GET_START_SEG_FROM_SEC(sbi, segno))) : 0; + return CAP_BLKS_PER_SEC(sbi) - used_blocks - CURSEG_I(sbi, type)->next_blkoff; + } return CAP_BLKS_PER_SEC(sbi) - get_ckpt_valid_blocks(sbi, segno, true); } diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index db7afb8064115..96325bbc70386 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -1759,6 +1759,7 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb) atomic_set(&fi->dirty_pages, 0); atomic_set(&fi->i_compr_blocks, 0); atomic_set(&fi->open_count, 0); + atomic_set(&fi->writeback, 0); init_f2fs_rwsem(&fi->i_sem); spin_lock_init(&fi->i_size_lock); INIT_LIST_HEAD(&fi->dirty_list); @@ -1960,7 +1961,7 @@ static void f2fs_put_super(struct super_block *sb) } /* be sure to wait for any on-going discard commands */ - done = f2fs_issue_discard_timeout(sbi); + done = f2fs_issue_discard_timeout(sbi, true); if (f2fs_realtime_discard_enable(sbi) && !sbi->discard_blks && done) { struct cp_control cpc = { .reason = CP_UMOUNT | CP_TRIMMED, @@ -1988,14 +1989,6 @@ static void f2fs_put_super(struct super_block *sb) truncate_inode_pages_final(META_MAPPING(sbi)); } - for (i = 0; i < NR_COUNT_TYPE; i++) { - if (!get_pages(sbi, i)) - continue; - f2fs_err(sbi, "detect filesystem reference count leak during " - "umount, type: %d, count: %lld", i, get_pages(sbi, i)); - f2fs_bug_on(sbi, 1); - } - f2fs_bug_on(sbi, sbi->fsync_node_num); f2fs_destroy_compress_inode(sbi); @@ -2006,6 +1999,15 @@ static void f2fs_put_super(struct super_block *sb) iput(sbi->meta_inode); sbi->meta_inode = NULL; + /* Should check the page counts after dropping all node/meta pages */ + for (i = 0; i < NR_COUNT_TYPE; i++) { + if (!get_pages(sbi, i)) + continue; + f2fs_err(sbi, "detect filesystem reference count leak during " + "umount, type: %d, count: %lld", i, get_pages(sbi, i)); + f2fs_bug_on(sbi, 1); + } + /* * iput() can update stat information, if f2fs_write_checkpoint() * above failed with error. @@ -2026,7 +2028,6 @@ static void f2fs_put_super(struct super_block *sb) kfree(sbi->raw_super); f2fs_destroy_page_array_cache(sbi); - f2fs_destroy_xattr_caches(sbi); #ifdef CONFIG_QUOTA for (i = 0; i < MAXQUOTAS; i++) kfree(F2FS_OPTION(sbi).s_qf_names[i]); @@ -2103,7 +2104,7 @@ static int f2fs_unfreeze(struct super_block *sb) * will recover after removal of snapshot. */ if (test_opt(sbi, DISCARD) && !f2fs_hw_support_discard(sbi)) - f2fs_issue_discard_timeout(sbi); + f2fs_issue_discard_timeout(sbi, true); clear_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING); return 0; @@ -2632,10 +2633,11 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) return err; } -static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) +static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) { unsigned int nr_pages = get_pages(sbi, F2FS_DIRTY_DATA) / 16; long long start, writeback, end; + int ret; f2fs_info(sbi, "f2fs_enable_checkpoint() starts, meta: %lld, node: %lld, data: %lld", get_pages(sbi, F2FS_DIRTY_META), @@ -2669,7 +2671,9 @@ static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) set_sbi_flag(sbi, SBI_IS_DIRTY); f2fs_up_write(&sbi->gc_lock); - f2fs_sync_fs(sbi->sb, 1); + ret = f2fs_sync_fs(sbi->sb, 1); + if (ret) + f2fs_err(sbi, "%s sync_fs failed, ret: %d", __func__, ret); /* Let's ensure there's no pending checkpoint anymore */ f2fs_flush_ckpt_thread(sbi); @@ -2679,6 +2683,7 @@ static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) f2fs_info(sbi, "f2fs_enable_checkpoint() finishes, writeback:%llu, sync:%llu", ktime_ms_delta(writeback, start), ktime_ms_delta(end, writeback)); + return ret; } static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) @@ -2879,7 +2884,12 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) need_stop_discard = true; } else { f2fs_stop_discard_thread(sbi); - f2fs_issue_discard_timeout(sbi); + /* + * f2fs_ioc_fitrim() won't race w/ "remount ro" + * so it's safe to check discard_cmd_cnt in + * f2fs_issue_discard_timeout(). + */ + f2fs_issue_discard_timeout(sbi, flags & SB_RDONLY); need_restart_discard = true; } } @@ -2892,7 +2902,9 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) goto restore_discard; need_enable_checkpoint = true; } else { - f2fs_enable_checkpoint(sbi); + err = f2fs_enable_checkpoint(sbi); + if (err) + goto restore_discard; need_disable_checkpoint = true; } } @@ -2935,7 +2947,8 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) return 0; restore_checkpoint: if (need_enable_checkpoint) { - f2fs_enable_checkpoint(sbi); + if (f2fs_enable_checkpoint(sbi)) + f2fs_warn(sbi, "checkpoint has not been enabled"); } else if (need_disable_checkpoint) { if (f2fs_disable_checkpoint(sbi)) f2fs_warn(sbi, "checkpoint has not been disabled"); @@ -4261,6 +4274,18 @@ static void init_sb_info(struct f2fs_sb_info *sbi) spin_lock_init(&sbi->gc_remaining_trials_lock); atomic64_set(&sbi->current_atomic_write, 0); + sbi->sum_blocksize = f2fs_sb_has_packed_ssa(sbi) ? + 4096 : sbi->blocksize; + sbi->sums_per_block = sbi->blocksize / sbi->sum_blocksize; + sbi->entries_in_sum = sbi->sum_blocksize / 8; + sbi->sum_entry_size = SUMMARY_SIZE * sbi->entries_in_sum; + sbi->sum_journal_size = sbi->sum_blocksize - SUM_FOOTER_SIZE - + sbi->sum_entry_size; + sbi->nat_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct nat_journal_entry); + sbi->sit_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct sit_journal_entry); + sbi->dir_level = DEF_DIR_LEVEL; sbi->interval_time[CP_TIME] = DEF_CP_INTERVAL; sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL; @@ -4544,48 +4569,7 @@ void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag) spin_unlock_irqrestore(&sbi->error_lock, flags); } -static bool f2fs_update_errors(struct f2fs_sb_info *sbi) -{ - unsigned long flags; - bool need_update = false; - - spin_lock_irqsave(&sbi->error_lock, flags); - if (sbi->error_dirty) { - memcpy(F2FS_RAW_SUPER(sbi)->s_errors, sbi->errors, - MAX_F2FS_ERRORS); - sbi->error_dirty = false; - need_update = true; - } - spin_unlock_irqrestore(&sbi->error_lock, flags); - - return need_update; -} - -static void f2fs_record_errors(struct f2fs_sb_info *sbi, unsigned char error) -{ - int err; - - f2fs_down_write(&sbi->sb_lock); - - if (!f2fs_update_errors(sbi)) - goto out_unlock; - - err = f2fs_commit_super(sbi, false); - if (err) - f2fs_err_ratelimited(sbi, - "f2fs_commit_super fails to record errors:%u, err:%d", - error, err); -out_unlock: - f2fs_up_write(&sbi->sb_lock); -} - void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error) -{ - f2fs_save_errors(sbi, error); - f2fs_record_errors(sbi, error); -} - -void f2fs_handle_error_async(struct f2fs_sb_info *sbi, unsigned char error) { f2fs_save_errors(sbi, error); @@ -5015,13 +4999,9 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto free_iostat; - /* init per sbi slab cache */ - err = f2fs_init_xattr_caches(sbi); - if (err) - goto free_percpu; err = f2fs_init_page_array_cache(sbi); if (err) - goto free_xattr_cache; + goto free_percpu; /* get an inode for meta space */ sbi->meta_inode = f2fs_iget(sb, F2FS_META_INO(sbi)); @@ -5226,11 +5206,15 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) } } else { err = f2fs_recover_fsync_data(sbi, true); - - if (!f2fs_readonly(sb) && err > 0) { - err = -EINVAL; - f2fs_err(sbi, "Need to recover fsync data"); - goto free_meta; + if (err > 0) { + if (!f2fs_readonly(sb)) { + f2fs_err(sbi, "Need to recover fsync data"); + err = -EINVAL; + goto free_meta; + } else { + f2fs_info(sbi, "drop all fsynced data"); + err = 0; + } } } @@ -5257,13 +5241,12 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto sync_free_meta; - if (test_opt(sbi, DISABLE_CHECKPOINT)) { + if (test_opt(sbi, DISABLE_CHECKPOINT)) err = f2fs_disable_checkpoint(sbi); - if (err) - goto sync_free_meta; - } else if (is_set_ckpt_flags(sbi, CP_DISABLED_FLAG)) { - f2fs_enable_checkpoint(sbi); - } + else if (is_set_ckpt_flags(sbi, CP_DISABLED_FLAG)) + err = f2fs_enable_checkpoint(sbi); + if (err) + goto sync_free_meta; /* * If filesystem is not mounted as read-only then @@ -5350,8 +5333,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) sbi->meta_inode = NULL; free_page_array_cache: f2fs_destroy_page_array_cache(sbi); -free_xattr_cache: - f2fs_destroy_xattr_caches(sbi); free_percpu: destroy_percpu_info(sbi); free_iostat: @@ -5554,10 +5535,15 @@ static int __init init_f2fs_fs(void) err = f2fs_create_casefold_cache(); if (err) goto free_compress_cache; - err = register_filesystem(&f2fs_fs_type); + err = f2fs_init_xattr_cache(); if (err) goto free_casefold_cache; + err = register_filesystem(&f2fs_fs_type); + if (err) + goto free_xattr_cache; return 0; +free_xattr_cache: + f2fs_destroy_xattr_cache(); free_casefold_cache: f2fs_destroy_casefold_cache(); free_compress_cache: @@ -5598,6 +5584,7 @@ static int __init init_f2fs_fs(void) static void __exit exit_f2fs_fs(void) { unregister_filesystem(&f2fs_fs_type); + f2fs_destroy_xattr_cache(); f2fs_destroy_casefold_cache(); f2fs_destroy_compress_cache(); f2fs_destroy_compress_mempool(); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 6d2a4fba68a29..c6fa8f74b91c2 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -58,6 +58,7 @@ struct f2fs_attr { const char *buf, size_t len); int struct_type; int offset; + int size; int id; }; @@ -235,6 +236,9 @@ static ssize_t features_show(struct f2fs_attr *a, if (f2fs_sb_has_compression(sbi)) len += sysfs_emit_at(buf, len, "%s%s", len ? ", " : "", "compression"); + if (f2fs_sb_has_packed_ssa(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "packed_ssa"); len += sysfs_emit_at(buf, len, "%s%s", len ? ", " : "", "pin_file"); len += sysfs_emit_at(buf, len, "\n"); @@ -341,11 +345,30 @@ static ssize_t main_blkaddr_show(struct f2fs_attr *a, (unsigned long long)MAIN_BLKADDR(sbi)); } +static ssize_t __sbi_show_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf, + unsigned char *value) +{ + switch (a->size) { + case 1: + return sysfs_emit(buf, "%u\n", *(u8 *)value); + case 2: + return sysfs_emit(buf, "%u\n", *(u16 *)value); + case 4: + return sysfs_emit(buf, "%u\n", *(u32 *)value); + case 8: + return sysfs_emit(buf, "%llu\n", *(u64 *)value); + default: + f2fs_bug_on(sbi, 1); + return sysfs_emit(buf, + "show sysfs node value with wrong type\n"); + } +} + static ssize_t f2fs_sbi_show(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf) { unsigned char *ptr = NULL; - unsigned int *ui; ptr = __struct_ptr(sbi, a->struct_type); if (!ptr) @@ -425,9 +448,30 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, atomic_read(&sbi->cp_call_count[BACKGROUND])); #endif - ui = (unsigned int *)(ptr + a->offset); + return __sbi_show_value(a, sbi, buf, ptr + a->offset); +} - return sysfs_emit(buf, "%u\n", *ui); +static void __sbi_store_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + unsigned char *ui, unsigned long value) +{ + switch (a->size) { + case 1: + *(u8 *)ui = value; + break; + case 2: + *(u16 *)ui = value; + break; + case 4: + *(u32 *)ui = value; + break; + case 8: + *(u64 *)ui = value; + break; + default: + f2fs_bug_on(sbi, 1); + f2fs_err(sbi, "store sysfs node value with wrong type"); + } } static ssize_t __sbi_store(struct f2fs_attr *a, @@ -746,7 +790,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - if (!strcmp(a->attr.name, "gc_pin_file_threshold")) { + if (!strcmp(a->attr.name, "gc_pin_file_thresh")) { if (t > MAX_GC_FAILED_PINNED_FILES) return -EINVAL; sbi->gc_pin_file_threshold = t; @@ -903,7 +947,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - *ui = (unsigned int)t; + __sbi_store_value(a, sbi, ptr + a->offset, t); return count; } @@ -1050,24 +1094,27 @@ static struct f2fs_attr f2fs_attr_sb_##_name = { \ .id = F2FS_FEATURE_##_feat, \ } -#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset) \ +#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset, _size) \ static struct f2fs_attr f2fs_attr_##_name = { \ .attr = {.name = __stringify(_name), .mode = _mode }, \ .show = _show, \ .store = _store, \ .struct_type = _struct_type, \ - .offset = _offset \ + .offset = _offset, \ + .size = _size \ } #define F2FS_RO_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0444, \ f2fs_sbi_show, NULL, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_RW_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0644, \ f2fs_sbi_show, f2fs_sbi_store, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_GENERAL_RO_ATTR(name) \ static struct f2fs_attr f2fs_attr_##name = __ATTR(name, 0444, name##_show, NULL) @@ -1296,6 +1343,7 @@ F2FS_FEATURE_RO_ATTR(pin_file); #ifdef CONFIG_UNICODE F2FS_FEATURE_RO_ATTR(linear_lookup); #endif +F2FS_FEATURE_RO_ATTR(packed_ssa); #define ATTR_LIST(name) (&f2fs_attr_##name.attr) static struct attribute *f2fs_attrs[] = { @@ -1455,6 +1503,7 @@ static struct attribute *f2fs_feat_attrs[] = { #ifdef CONFIG_UNICODE BASE_ATTR_LIST(linear_lookup), #endif + BASE_ATTR_LIST(packed_ssa), NULL, }; ATTRIBUTE_GROUPS(f2fs_feat); @@ -1490,6 +1539,7 @@ F2FS_SB_FEATURE_RO_ATTR(casefold, CASEFOLD); F2FS_SB_FEATURE_RO_ATTR(compression, COMPRESSION); F2FS_SB_FEATURE_RO_ATTR(readonly, RO); F2FS_SB_FEATURE_RO_ATTR(device_alias, DEVICE_ALIAS); +F2FS_SB_FEATURE_RO_ATTR(packed_ssa, PACKED_SSA); static struct attribute *f2fs_sb_feat_attrs[] = { ATTR_LIST(sb_encryption), @@ -1507,6 +1557,7 @@ static struct attribute *f2fs_sb_feat_attrs[] = { ATTR_LIST(sb_compression), ATTR_LIST(sb_readonly), ATTR_LIST(sb_device_alias), + ATTR_LIST(sb_packed_ssa), NULL, }; ATTRIBUTE_GROUPS(f2fs_sb_feat); diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 58632a2b66136..b4e5c406632f8 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -23,11 +23,12 @@ #include "xattr.h" #include "segment.h" +static struct kmem_cache *inline_xattr_slab; static void *xattr_alloc(struct f2fs_sb_info *sbi, int size, bool *is_inline) { - if (likely(size == sbi->inline_xattr_slab_size)) { + if (likely(size == DEFAULT_XATTR_SLAB_SIZE)) { *is_inline = true; - return f2fs_kmem_cache_alloc(sbi->inline_xattr_slab, + return f2fs_kmem_cache_alloc(inline_xattr_slab, GFP_F2FS_ZERO, false, sbi); } *is_inline = false; @@ -38,7 +39,7 @@ static void xattr_free(struct f2fs_sb_info *sbi, void *xattr_addr, bool is_inline) { if (is_inline) - kmem_cache_free(sbi->inline_xattr_slab, xattr_addr); + kmem_cache_free(inline_xattr_slab, xattr_addr); else kfree(xattr_addr); } @@ -830,25 +831,14 @@ int f2fs_setxattr(struct inode *inode, int index, const char *name, return err; } -int f2fs_init_xattr_caches(struct f2fs_sb_info *sbi) +int __init f2fs_init_xattr_cache(void) { - dev_t dev = sbi->sb->s_bdev->bd_dev; - char slab_name[32]; - - sprintf(slab_name, "f2fs_xattr_entry-%u:%u", MAJOR(dev), MINOR(dev)); - - sbi->inline_xattr_slab_size = F2FS_OPTION(sbi).inline_xattr_size * - sizeof(__le32) + XATTR_PADDING_SIZE; - - sbi->inline_xattr_slab = f2fs_kmem_cache_create(slab_name, - sbi->inline_xattr_slab_size); - if (!sbi->inline_xattr_slab) - return -ENOMEM; - - return 0; + inline_xattr_slab = f2fs_kmem_cache_create("f2fs_xattr_entry", + DEFAULT_XATTR_SLAB_SIZE); + return inline_xattr_slab ? 0 : -ENOMEM; } -void f2fs_destroy_xattr_caches(struct f2fs_sb_info *sbi) +void f2fs_destroy_xattr_cache(void) { - kmem_cache_destroy(sbi->inline_xattr_slab); -} + kmem_cache_destroy(inline_xattr_slab); +} \ No newline at end of file diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h index 4fc0b2305fbd8..bce3d93e4755b 100644 --- a/fs/f2fs/xattr.h +++ b/fs/f2fs/xattr.h @@ -89,6 +89,8 @@ struct f2fs_xattr_entry { F2FS_TOTAL_EXTRA_ATTR_SIZE / sizeof(__le32) - \ DEF_INLINE_RESERVED_SIZE - \ MIN_INLINE_DENTRY_SIZE / sizeof(__le32)) +#define DEFAULT_XATTR_SLAB_SIZE (DEFAULT_INLINE_XATTR_ADDRS * \ + sizeof(__le32) + XATTR_PADDING_SIZE) /* * On-disk structure of f2fs_xattr @@ -132,8 +134,8 @@ int f2fs_setxattr(struct inode *, int, const char *, const void *, int f2fs_getxattr(struct inode *, int, const char *, void *, size_t, struct folio *); ssize_t f2fs_listxattr(struct dentry *, char *, size_t); -int f2fs_init_xattr_caches(struct f2fs_sb_info *); -void f2fs_destroy_xattr_caches(struct f2fs_sb_info *); +int __init f2fs_init_xattr_cache(void); +void f2fs_destroy_xattr_cache(void); #else #define f2fs_xattr_handlers NULL @@ -150,8 +152,8 @@ static inline int f2fs_getxattr(struct inode *inode, int index, { return -EOPNOTSUPP; } -static inline int f2fs_init_xattr_caches(struct f2fs_sb_info *sbi) { return 0; } -static inline void f2fs_destroy_xattr_caches(struct f2fs_sb_info *sbi) { } +static inline int __init f2fs_init_xattr_cache(void) { return 0; } +static inline void f2fs_destroy_xattr_cache(void) { } #endif #ifdef CONFIG_F2FS_FS_SECURITY diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c index 0b920ee40a7f9..262ec1b790b56 100644 --- a/fs/fat/namei_msdos.c +++ b/fs/fat/namei_msdos.c @@ -325,7 +325,12 @@ static int msdos_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_CTIME); diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c index 5dbc4cbb8fce3..47ff083cfc7e6 100644 --- a/fs/fat/namei_vfat.c +++ b/fs/fat/namei_vfat.c @@ -803,7 +803,12 @@ static int vfat_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_ATIME|S_MTIME); diff --git a/fs/file_attr.c b/fs/file_attr.c index 1dcec88c06805..9d3e177ad7d1d 100644 --- a/fs/file_attr.c +++ b/fs/file_attr.c @@ -379,7 +379,7 @@ SYSCALL_DEFINE5(file_getattr, int, dfd, const char __user *, filename, struct filename *name __free(putname) = NULL; unsigned int lookup_flags = 0; struct file_attr fattr; - struct file_kattr fa; + struct file_kattr fa = { .flags_valid = true }; /* hint only */ int error; BUILD_BUG_ON(sizeof(struct file_attr) < FILE_ATTR_SIZE_VER0); diff --git a/fs/file_table.c b/fs/file_table.c index cd4a3db4659ac..34244fccf2edf 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -176,6 +176,11 @@ static int init_file(struct file *f, int flags, const struct cred *cred) f->f_flags = flags; f->f_mode = OPEN_FMODE(flags); + /* + * Disable permission and pre-content events for all files by default. + * They may be enabled later by fsnotify_open_perm_and_set_mode(). + */ + file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); f->f_op = NULL; f->f_mapping = NULL; @@ -197,11 +202,6 @@ static int init_file(struct file *f, int flags, const struct cred *cred) * refcount bumps we should reinitialize the reused file first. */ file_ref_init(&f->f_ref, 1); - /* - * Disable permission and pre-content events for all files by default. - * They may be enabled later by fsnotify_open_perm_and_set_mode(). - */ - file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); return 0; } diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 2b35e80037fee..5d43bbaf37b16 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -558,28 +558,30 @@ void inode_switch_wbs_work_fn(struct work_struct *work) struct inode_switch_wbs_context *isw, *next_isw; struct llist_node *list; + list = llist_del_all(&new_wb->switch_wbs_ctxs); /* - * Grab out reference to wb so that it cannot get freed under us + * Nothing to do? That would be a problem as references held by isw + * items protect wb from freeing... + */ + if (WARN_ON_ONCE(!list)) + return; + + /* + * Grab our reference to wb so that it cannot get freed under us * after we process all the isw items. */ wb_get(new_wb); - while (1) { - list = llist_del_all(&new_wb->switch_wbs_ctxs); - /* Nothing to do? */ - if (!list) - break; - /* - * In addition to synchronizing among switchers, I_WB_SWITCH - * tells the RCU protected stat update paths to grab the i_page - * lock so that stat transfer can synchronize against them. - * Let's continue after I_WB_SWITCH is guaranteed to be - * visible. - */ - synchronize_rcu(); + /* + * In addition to synchronizing among switchers, I_WB_SWITCH + * tells the RCU protected stat update paths to grab the i_page + * lock so that stat transfer can synchronize against them. + * Let's continue after I_WB_SWITCH is guaranteed to be + * visible. + */ + synchronize_rcu(); - llist_for_each_entry_safe(isw, next_isw, list, list) - process_inode_switch_wbs(new_wb, isw); - } + llist_for_each_entry_safe(isw, next_isw, list, list) + process_inode_switch_wbs(new_wb, isw); wb_put(new_wb); } @@ -2471,7 +2473,8 @@ static void wakeup_dirtytime_writeback(struct work_struct *w) wb_wakeup(wb); } rcu_read_unlock(); - schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ); + if (dirtytime_expire_interval) + schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ); } static int dirtytime_interval_handler(const struct ctl_table *table, int write, @@ -2480,8 +2483,12 @@ static int dirtytime_interval_handler(const struct ctl_table *table, int write, int ret; ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); - if (ret == 0 && write) - mod_delayed_work(system_percpu_wq, &dirtytime_work, 0); + if (ret == 0 && write) { + if (dirtytime_expire_interval) + mod_delayed_work(system_percpu_wq, &dirtytime_work, 0); + else + cancel_delayed_work_sync(&dirtytime_work); + } return ret; } @@ -2498,7 +2505,8 @@ static const struct ctl_table vm_fs_writeback_table[] = { static int __init start_dirtytime_writeback(void) { - schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ); + if (dirtytime_expire_interval) + schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ); register_sysctl_init("vm", vm_fs_writeback_table); return 0; } @@ -2865,6 +2873,17 @@ void sync_inodes_sb(struct super_block *sb) */ if (bdi == &noop_backing_dev_info) return; + + /* + * If the superblock has SB_I_NO_DATA_INTEGRITY set, there's no need to + * wait for the writeout to complete, as the filesystem cannot guarantee + * data persistence on sync. Just kick off writeback and return. + */ + if (sb->s_iflags & SB_I_NO_DATA_INTEGRITY) { + wakeup_flusher_threads_bdi(bdi, WB_REASON_SYNC); + return; + } + WARN_ON(!rwsem_is_locked(&sb->s_umount)); /* protect against inode wb switch, see inode_switch_wbs_work_fn() */ diff --git a/fs/fs_struct.c b/fs/fs_struct.c index 28be762ac1c63..a0b40ad5e7423 100644 --- a/fs/fs_struct.c +++ b/fs/fs_struct.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "internal.h" /* diff --git a/fs/fuse/control.c b/fs/fuse/control.c index bb407705603c2..ce11f4b427c28 100644 --- a/fs/fuse/control.c +++ b/fs/fuse/control.c @@ -121,7 +121,7 @@ static ssize_t fuse_conn_max_background_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - unsigned val; + unsigned int val = 0; ssize_t ret; ret = fuse_conn_limit_write(file, buf, count, ppos, &val, @@ -163,7 +163,7 @@ static ssize_t fuse_conn_congestion_threshold_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - unsigned val; + unsigned int val = 0; struct fuse_conn *fc; ssize_t ret; @@ -205,8 +205,7 @@ static const struct file_operations fuse_conn_congestion_threshold_ops = { static struct dentry *fuse_ctl_add_dentry(struct dentry *parent, struct fuse_conn *fc, - const char *name, - int mode, int nlink, + const char *name, int mode, const struct inode_operations *iop, const struct file_operations *fop) { @@ -232,7 +231,10 @@ static struct dentry *fuse_ctl_add_dentry(struct dentry *parent, if (iop) inode->i_op = iop; inode->i_fop = fop; - set_nlink(inode, nlink); + if (S_ISDIR(mode)) { + inc_nlink(d_inode(parent)); + inc_nlink(inode); + } inode->i_private = fc; d_add(dentry, inode); @@ -252,22 +254,21 @@ int fuse_ctl_add_conn(struct fuse_conn *fc) return 0; parent = fuse_control_sb->s_root; - inc_nlink(d_inode(parent)); sprintf(name, "%u", fc->dev); - parent = fuse_ctl_add_dentry(parent, fc, name, S_IFDIR | 0500, 2, + parent = fuse_ctl_add_dentry(parent, fc, name, S_IFDIR | 0500, &simple_dir_inode_operations, &simple_dir_operations); if (!parent) goto err; - if (!fuse_ctl_add_dentry(parent, fc, "waiting", S_IFREG | 0400, 1, + if (!fuse_ctl_add_dentry(parent, fc, "waiting", S_IFREG | 0400, NULL, &fuse_ctl_waiting_ops) || - !fuse_ctl_add_dentry(parent, fc, "abort", S_IFREG | 0200, 1, + !fuse_ctl_add_dentry(parent, fc, "abort", S_IFREG | 0200, NULL, &fuse_ctl_abort_ops) || !fuse_ctl_add_dentry(parent, fc, "max_background", S_IFREG | 0600, - 1, NULL, &fuse_conn_max_background_ops) || + NULL, &fuse_conn_max_background_ops) || !fuse_ctl_add_dentry(parent, fc, "congestion_threshold", - S_IFREG | 0600, 1, NULL, + S_IFREG | 0600, NULL, &fuse_conn_congestion_threshold_ops)) goto err; diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 132f38619d707..039fe9c0c3cba 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -570,6 +570,11 @@ static void request_wait_answer(struct fuse_req *req) if (!err) return; + if (req->args->abort_on_kill) { + fuse_abort_conn(fc); + return; + } + if (test_bit(FR_URING, &req->flags)) removed = fuse_uring_remove_pending_req(req); else @@ -676,7 +681,8 @@ ssize_t __fuse_simple_request(struct mnt_idmap *idmap, fuse_force_creds(req); __set_bit(FR_WAITING, &req->flags); - __set_bit(FR_FORCE, &req->flags); + if (!args->abort_on_kill) + __set_bit(FR_FORCE, &req->flags); } else { WARN_ON(args->nocreds); req = fuse_get_req(idmap, fm, false); @@ -846,7 +852,7 @@ void fuse_copy_init(struct fuse_copy_state *cs, bool write, } /* Unmap and put previous page of userspace buffer */ -static void fuse_copy_finish(struct fuse_copy_state *cs) +void fuse_copy_finish(struct fuse_copy_state *cs) { if (cs->currbuf) { struct pipe_buffer *buf = cs->currbuf; @@ -1011,6 +1017,9 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop folio_clear_uptodate(newfolio); folio_clear_mappedtodisk(newfolio); + if (folio_test_large(newfolio)) + goto out_fallback_unlock; + if (fuse_check_folio(newfolio) != 0) goto out_fallback_unlock; @@ -2590,9 +2599,8 @@ static int fuse_device_clone(struct fuse_conn *fc, struct file *new) static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) { - int res; int oldfd; - struct fuse_dev *fud = NULL; + struct fuse_dev *fud; if (get_user(oldfd, argp)) return -EFAULT; @@ -2605,17 +2613,15 @@ static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) * Check against file->f_op because CUSE * uses the same ioctl handler. */ - if (fd_file(f)->f_op == file->f_op) - fud = __fuse_get_dev(fd_file(f)); + if (fd_file(f)->f_op != file->f_op) + return -EINVAL; - res = -EINVAL; - if (fud) { - mutex_lock(&fuse_mutex); - res = fuse_device_clone(fud->fc, file); - mutex_unlock(&fuse_mutex); - } + fud = fuse_get_dev(fd_file(f)); + if (IS_ERR(fud)) + return PTR_ERR(fud); - return res; + guard(mutex)(&fuse_mutex); + return fuse_device_clone(fud->fc, file); } static long fuse_dev_ioctl_backing_open(struct file *file, diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index f6b12aebb8bbe..f0846694822dd 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -86,6 +86,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, lockdep_assert_not_held(&queue->lock); spin_lock(&queue->lock); ent->fuse_req = NULL; + list_del_init(&req->list); if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; spin_lock(&fc->bg_lock); @@ -598,7 +599,9 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, cs.is_uring = true; cs.req = req; - return fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); + err = fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); + fuse_copy_finish(&cs); + return err; } /* @@ -649,6 +652,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, /* copy the payload */ err = fuse_copy_args(&cs, num_args, args->in_pages, (struct fuse_arg *)in_args, 0); + fuse_copy_finish(&cs); if (err) { pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); return err; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f1ef77a0be05b..6014d588845cd 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -110,7 +110,9 @@ static void fuse_file_put(struct fuse_file *ff, bool sync) fuse_file_io_release(ff, ra->inode); if (!args) { - /* Do nothing when server does not implement 'open' */ + /* Do nothing when server does not implement 'opendir' */ + } else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) { + fuse_release_end(ff->fm, args, 0); } else if (sync) { fuse_simple_request(ff->fm, args); fuse_release_end(ff->fm, args, 0); @@ -131,8 +133,17 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, struct fuse_file *ff; int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN; bool open = isdir ? !fc->no_opendir : !fc->no_open; + bool release = !isdir || open; - ff = fuse_file_alloc(fm, open); + /* + * ff->args->release_args still needs to be allocated (so we can hold an + * inode reference while there are pending inflight file operations when + * ->release() is called, see fuse_prepare_release()) even if + * fc->no_open is set else it becomes possible for reclaim to deadlock + * if while servicing the readahead request the server triggers reclaim + * and reclaim evicts the inode of the file being read ahead. + */ + ff = fuse_file_alloc(fm, release); if (!ff) return ERR_PTR(-ENOMEM); @@ -152,13 +163,14 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, fuse_file_free(ff); return ERR_PTR(err); } else { - /* No release needed */ - kfree(ff->args); - ff->args = NULL; - if (isdir) + if (isdir) { + /* No release needed */ + kfree(ff->args); + ff->args = NULL; fc->no_opendir = 1; - else + } else { fc->no_open = 1; + } } } @@ -1607,7 +1619,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (!ia) return -ENOMEM; - if (fopen_direct_io && fc->direct_io_allow_mmap) { + if (fopen_direct_io) { res = filemap_write_and_wait_range(mapping, pos, pos + count - 1); if (res) { fuse_io_free(ia); @@ -1681,6 +1693,15 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (res > 0) *ppos = pos; + if (res > 0 && write && fopen_direct_io) { + /* + * As in generic_file_direct_write(), invalidate after the + * write, to invalidate read-ahead cache that may have competed + * with the write. + */ + invalidate_inode_pages2_range(mapping, idx_from, idx_to); + } + return res > 0 ? res : err; } EXPORT_SYMBOL_GPL(fuse_direct_io); diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 6e8373f970409..134bf44aff0d3 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -62,6 +62,7 @@ void fuse_dev_end_requests(struct list_head *head); void fuse_copy_init(struct fuse_copy_state *cs, bool write, struct iov_iter *iter); +void fuse_copy_finish(struct fuse_copy_state *cs); int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs, unsigned int argpages, struct fuse_arg *args, int zeroing); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c2f2a48156d6c..c288f28f6c6ea 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -333,6 +333,7 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; + bool abort_on_kill:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d1babf56f2547..e9ed693fc7b37 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1548,6 +1548,7 @@ int fuse_send_init(struct fuse_mount *fm) int err; if (fm->fc->sync_init) { + ia->args.abort_on_kill = true; err = fuse_simple_request(fm, &ia->args); /* Ignore size of init reply */ if (err > 0) @@ -1706,6 +1707,7 @@ static void fuse_sb_defaults(struct super_block *sb) sb->s_export_op = &fuse_export_operations; sb->s_iflags |= SB_I_IMA_UNVERIFIABLE_SIGNATURE; sb->s_iflags |= SB_I_NOIDMAP; + sb->s_iflags |= SB_I_NO_DATA_INTEGRITY; if (sb->s_user_ns != &init_user_ns) sb->s_iflags |= SB_I_UNTRUSTED_MOUNTER; sb->s_flags &= ~(SB_NOSEC | SB_I_VERSION); diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index c2aae2eef0868..aae657fd56c0e 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -41,6 +41,10 @@ static void fuse_add_dirent_to_cache(struct file *file, unsigned int offset; void *addr; + /* Dirent doesn't fit in readdir cache page? Skip caching. */ + if (reclen > PAGE_SIZE) + return; + spin_lock(&fi->rdc.lock); /* * Is cache already completed? Or this entry does not go at the end of diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 131091520de6b..fdcac8e3f2ba2 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1127,10 +1127,18 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length, goto out_unlock; break; default: - goto out_unlock; + goto out; } ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp); + if (ret) + goto out_unlock; + +out: + if (iomap->type == IOMAP_INLINE) { + iomap->private = metapath_dibh(&mp); + get_bh(iomap->private); + } out_unlock: release_metapath(&mp); @@ -1144,6 +1152,9 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_sbd *sdp = GFS2_SB(inode); + if (iomap->private) + brelse(iomap->private); + switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) { case IOMAP_WRITE: if (flags & IOMAP_DIRECT) diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index b677c0e6b9ab3..7aec6cfdfd91d 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -1211,10 +1211,13 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number, mapping = gfs2_glock2aspace(gl); if (mapping) { + gfp_t gfp_mask; + mapping->a_ops = &gfs2_meta_aops; mapping->host = sdp->sd_inode; mapping->flags = 0; - mapping_set_gfp_mask(mapping, GFP_NOFS); + gfp_mask = mapping_gfp_mask(sdp->sd_inode->i_mapping); + mapping_set_gfp_mask(mapping, gfp_mask); mapping->i_private_data = NULL; mapping->writeback_index = 0; } @@ -1351,31 +1354,45 @@ static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs) * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions * @num_gh: the number of holders in the array * @ghs: the glock holder array + * @retries: number of retries attempted so far * * Returns: 0 on success, meaning all glocks have been granted and are held. * -ESTALE if the request timed out, meaning all glocks were released, * and the caller should retry the operation. */ -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries) { struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd; - int i, ret = 0, timeout = 0; unsigned long start_time = jiffies; + int i, ret = 0; + long timeout; might_sleep(); - /* - * Total up the (minimum hold time * 2) of all glocks and use that to - * determine the max amount of time we should wait. - */ - for (i = 0; i < num_gh; i++) - timeout += ghs[i].gh_gl->gl_hold_time << 1; - if (!wait_event_timeout(sdp->sd_async_glock_wait, + timeout = GL_GLOCK_MIN_HOLD; + if (retries) { + unsigned int max_shift; + long incr; + + /* Add a random delay and increase the timeout exponentially. */ + max_shift = BITS_PER_LONG - 2 - __fls(GL_GLOCK_HOLD_INCR); + incr = min(GL_GLOCK_HOLD_INCR << min(retries - 1, max_shift), + 10 * HZ - GL_GLOCK_MIN_HOLD); + schedule_timeout_interruptible(get_random_long() % (incr / 3)); + if (signal_pending(current)) + goto interrupted; + timeout += (incr / 3) + get_random_long() % (incr / 3); + } + + if (!wait_event_interruptible_timeout(sdp->sd_async_glock_wait, !glocks_pending(num_gh, ghs), timeout)) { ret = -ESTALE; /* request timed out. */ goto out; } + if (signal_pending(current)) + goto interrupted; for (i = 0; i < num_gh; i++) { struct gfs2_holder *gh = &ghs[i]; @@ -1399,6 +1416,10 @@ int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) } } return ret; + +interrupted: + ret = -EINTR; + goto out; } /** diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h index d041b922b45e3..2d4fd1a2bbbb8 100644 --- a/fs/gfs2/glock.h +++ b/fs/gfs2/glock.h @@ -204,7 +204,8 @@ int gfs2_glock_poll(struct gfs2_holder *gh); int gfs2_instantiate(struct gfs2_holder *gh); int gfs2_glock_holder_ready(struct gfs2_holder *gh); int gfs2_glock_wait(struct gfs2_holder *gh); -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs); +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries); void gfs2_glock_dq(struct gfs2_holder *gh); void gfs2_glock_dq_wait(struct gfs2_holder *gh); void gfs2_glock_dq_uninit(struct gfs2_holder *gh); diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c index 0c0a80b3bacab..0c68ab4432b08 100644 --- a/fs/gfs2/glops.c +++ b/fs/gfs2/glops.c @@ -630,8 +630,7 @@ static void iopen_go_callback(struct gfs2_glock *gl, bool remote) struct gfs2_inode *ip = gl->gl_object; struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; - if (!remote || sb_rdonly(sdp->sd_vfs) || - test_bit(SDF_KILL, &sdp->sd_flags)) + if (!remote || test_bit(SDF_KILL, &sdp->sd_flags)) return; if (gl->gl_demote_state == LM_ST_UNLOCKED && diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 8a7ed80d9f2d6..03dd54fb7e8c8 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -89,6 +89,19 @@ static int iget_set(struct inode *inode, void *opaque) return 0; } +void gfs2_setup_inode(struct inode *inode) +{ + gfp_t gfp_mask; + + /* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into the filesystem and blowing + * stacks or deadlocking. + */ + gfp_mask = mapping_gfp_mask(inode->i_mapping); + mapping_set_gfp_mask(inode->i_mapping, gfp_mask & ~__GFP_FS); +} + /** * gfs2_inode_lookup - Lookup an inode * @sb: The super block @@ -132,6 +145,7 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type, struct gfs2_glock *io_gl; int extra_flags = 0; + gfs2_setup_inode(inode); error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE, &ip->i_gl); if (unlikely(error)) @@ -752,6 +766,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, error = -ENOMEM; if (!inode) goto fail_gunlock; + gfs2_setup_inode(inode); ip = GFS2_I(inode); error = posix_acl_create(dir, &mode, &default_acl, &acl); @@ -1480,7 +1495,7 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, unsigned int num_gh; int dir_rename = 0; struct gfs2_diradd da = { .nr_blocks = 0, .save_loc = 0, }; - unsigned int x; + unsigned int retries = 0, x; int error; gfs2_holder_mark_uninitialized(&r_gh); @@ -1530,12 +1545,17 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, num_gh++; } +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; @@ -1724,7 +1744,7 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, struct gfs2_sbd *sdp = GFS2_SB(odir); struct gfs2_holder ghs[4], r_gh; unsigned int num_gh; - unsigned int x; + unsigned int retries = 0, x; umode_t old_mode = oip->i_inode.i_mode; umode_t new_mode = nip->i_inode.i_mode; int error; @@ -1768,13 +1788,18 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh); num_gh++; +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; @@ -2167,6 +2192,14 @@ static int gfs2_getattr(struct mnt_idmap *idmap, return 0; } +static bool fault_in_fiemap(struct fiemap_extent_info *fi) +{ + struct fiemap_extent __user *dest = fi->fi_extents_start; + size_t size = sizeof(*dest) * fi->fi_extents_max; + + return fault_in_safe_writeable((char __user *)dest, size) == 0; +} + static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { @@ -2176,14 +2209,22 @@ static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, inode_lock_shared(inode); +retry: ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh); if (ret) goto out; + pagefault_disable(); ret = iomap_fiemap(inode, fieinfo, start, len, &gfs2_iomap_ops); + pagefault_enable(); gfs2_glock_dq_uninit(&gh); + if (ret == -EFAULT && fault_in_fiemap(fieinfo)) { + fieinfo->fi_extents_mapped = 0; + goto retry; + } + out: inode_unlock_shared(inode); return ret; diff --git a/fs/gfs2/inode.h b/fs/gfs2/inode.h index e43f08eb26e72..2fcd96dd13613 100644 --- a/fs/gfs2/inode.h +++ b/fs/gfs2/inode.h @@ -86,6 +86,7 @@ static inline int gfs2_check_internal_file_size(struct inode *inode, return -EIO; } +void gfs2_setup_inode(struct inode *inode); struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned type, u64 no_addr, u64 no_formal_ino, unsigned int blktype); diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index aa15183f9a168..1a2db8053da04 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -1183,7 +1183,7 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) mapping = gfs2_aspace(sdp); mapping->a_ops = &gfs2_rgrp_aops; - mapping_set_gfp_mask(mapping, GFP_NOFS); + gfs2_setup_inode(sdp->sd_inode); error = init_names(sdp, silent); if (error) diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index 2298e06797ac3..898fc3937b449 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -334,6 +334,7 @@ static void qd_put(struct gfs2_quota_data *qd) lockref_mark_dead(&qd->qd_lockref); spin_unlock(&qd->qd_lockref.lock); + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); gfs2_qd_dispose(qd); return; } @@ -1616,7 +1617,7 @@ int gfs2_quotad(void *data) t = min(quotad_timeo, statfs_timeo); - t = wait_event_freezable_timeout(sdp->sd_quota_wait, + t -= wait_event_freezable_timeout(sdp->sd_quota_wait, sdp->sd_statfs_force_sync || gfs2_withdrawing_or_withdrawn(sdp) || kthread_should_stop(), diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 644b2d1e72769..54c6f2098f01e 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -749,9 +749,7 @@ static int gfs2_freeze_super(struct super_block *sb, enum freeze_holder who, break; } - error = gfs2_do_thaw(sdp, who, freeze_owner); - if (error) - goto out; + (void)gfs2_do_thaw(sdp, who, freeze_owner); if (error == -EBUSY) fs_err(sdp, "waiting for recovery before freeze\n"); diff --git a/fs/hfs/catalog.c b/fs/hfs/catalog.c index caebabb6642f1..b80ba40e38776 100644 --- a/fs/hfs/catalog.c +++ b/fs/hfs/catalog.c @@ -322,9 +322,9 @@ int hfs_correct_next_unused_CNID(struct super_block *sb, u32 cnid) } } + node_id = node->prev; hfs_bnode_put(node); - node_id = node->prev; } while (node_id >= leaf_head); return -ENOENT; diff --git a/fs/hfs/mdb.c b/fs/hfs/mdb.c index 53f3fae602179..f28cd24dee842 100644 --- a/fs/hfs/mdb.c +++ b/fs/hfs/mdb.c @@ -92,7 +92,7 @@ int hfs_mdb_get(struct super_block *sb) /* See if this is an HFS filesystem */ bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); if (!bh) - goto out; + return -EIO; if (mdb->drSigWord == cpu_to_be16(HFS_SUPER_MAGIC)) break; @@ -102,13 +102,14 @@ int hfs_mdb_get(struct super_block *sb) * (should do this only for cdrom/loop though) */ if (hfs_part_find(sb, &part_start, &part_size)) - goto out; + return -EIO; } HFS_SB(sb)->alloc_blksz = size = be32_to_cpu(mdb->drAlBlkSiz); if (!size || (size & (HFS_SECTOR_SIZE - 1))) { pr_err("bad allocation block size %d\n", size); - goto out_bh; + brelse(bh); + return -EIO; } size = min(HFS_SB(sb)->alloc_blksz, (u32)PAGE_SIZE); @@ -125,14 +126,16 @@ int hfs_mdb_get(struct super_block *sb) brelse(bh); if (!sb_set_blocksize(sb, size)) { pr_err("unable to set blocksize to %u\n", size); - goto out; + return -EIO; } bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); if (!bh) - goto out; - if (mdb->drSigWord != cpu_to_be16(HFS_SUPER_MAGIC)) - goto out_bh; + return -EIO; + if (mdb->drSigWord != cpu_to_be16(HFS_SUPER_MAGIC)) { + brelse(bh); + return -EIO; + } HFS_SB(sb)->mdb_bh = bh; HFS_SB(sb)->mdb = mdb; @@ -174,7 +177,7 @@ int hfs_mdb_get(struct super_block *sb) HFS_SB(sb)->bitmap = kzalloc(8192, GFP_KERNEL); if (!HFS_SB(sb)->bitmap) - goto out; + return -EIO; /* read in the bitmap */ block = be16_to_cpu(mdb->drVBMSt) + part_start; @@ -185,7 +188,7 @@ int hfs_mdb_get(struct super_block *sb) bh = sb_bread(sb, off >> sb->s_blocksize_bits); if (!bh) { pr_err("unable to read volume bitmap\n"); - goto out; + return -EIO; } off2 = off & (sb->s_blocksize - 1); len = min((int)sb->s_blocksize - off2, size); @@ -199,12 +202,12 @@ int hfs_mdb_get(struct super_block *sb) HFS_SB(sb)->ext_tree = hfs_btree_open(sb, HFS_EXT_CNID, hfs_ext_keycmp); if (!HFS_SB(sb)->ext_tree) { pr_err("unable to open extent tree\n"); - goto out; + return -EIO; } HFS_SB(sb)->cat_tree = hfs_btree_open(sb, HFS_CAT_CNID, hfs_cat_keycmp); if (!HFS_SB(sb)->cat_tree) { pr_err("unable to open catalog tree\n"); - goto out; + return -EIO; } attrib = mdb->drAtrb; @@ -229,12 +232,6 @@ int hfs_mdb_get(struct super_block *sb) } return 0; - -out_bh: - brelse(bh); -out: - hfs_mdb_put(sb); - return -EIO; } /* @@ -359,8 +356,6 @@ void hfs_mdb_close(struct super_block *sb) * Release the resources associated with the in-core MDB. */ void hfs_mdb_put(struct super_block *sb) { - if (!HFS_SB(sb)) - return; /* free the B-trees */ hfs_btree_close(HFS_SB(sb)->ext_tree); hfs_btree_close(HFS_SB(sb)->cat_tree); @@ -373,6 +368,4 @@ void hfs_mdb_put(struct super_block *sb) unload_nls(HFS_SB(sb)->nls_disk); kfree(HFS_SB(sb)->bitmap); - kfree(HFS_SB(sb)); - sb->s_fs_info = NULL; } diff --git a/fs/hfs/super.c b/fs/hfs/super.c index 47f50fa555a45..df289cbdd4e85 100644 --- a/fs/hfs/super.c +++ b/fs/hfs/super.c @@ -431,10 +431,18 @@ static int hfs_init_fs_context(struct fs_context *fc) return 0; } +static void hfs_kill_super(struct super_block *sb) +{ + struct hfs_sb_info *hsb = HFS_SB(sb); + + kill_block_super(sb); + kfree(hsb); +} + static struct file_system_type hfs_fs_type = { .owner = THIS_MODULE, .name = "hfs", - .kill_sb = kill_block_super, + .kill_sb = hfs_kill_super, .fs_flags = FS_REQUIRES_DEV, .init_fs_context = hfs_init_fs_context, }; diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index 63e652ad1e0de..8e60e04c427bd 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -481,6 +481,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) tree->node_hash[hash] = node; tree->node_hash_cnt++; } else { + hfs_bnode_get(node2); spin_unlock(&tree->hash_lock); kfree(node); wait_event(node2->lock_wq, @@ -628,7 +629,7 @@ struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) if (node) { pr_crit("new node %u already hashed?\n", num); WARN_ON(1); - return node; + return ERR_PTR(-EEXIST); } node = __hfs_bnode_create(tree, num); if (!node) @@ -704,6 +705,5 @@ bool hfs_bnode_need_zeroout(struct hfs_btree *tree) struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); const u32 volume_attr = be32_to_cpu(sbi->s_vhdr->attributes); - return tree->cnid == HFSPLUS_CAT_CNID && - volume_attr & HFSPLUS_VOL_UNUSED_NODE_FIX; + return volume_attr & HFSPLUS_VOL_UNUSED_NODE_FIX; } diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index 1b3e27a0d5e03..ca5f74a140ec1 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -313,6 +313,9 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, if (!S_ISREG(inode->i_mode)) return -EPERM; + hfs_dbg("src_dir->i_ino %lu, dst_dir->i_ino %lu, inode->i_ino %lu\n", + src_dir->i_ino, dst_dir->i_ino, inode->i_ino); + mutex_lock(&sbi->vh_mutex); if (inode->i_ino == (u32)(unsigned long)src_dentry->d_fsdata) { for (;;) { @@ -332,7 +335,7 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, cnid = sbi->next_cnid++; src_dentry->d_fsdata = (void *)(unsigned long)cnid; res = hfsplus_create_cat(cnid, src_dir, - &src_dentry->d_name, inode); + &src_dentry->d_name, inode); if (res) /* panic? */ goto out; @@ -350,6 +353,21 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, mark_inode_dirty(inode); sbi->file_count++; hfsplus_mark_mdb_dirty(dst_dir->i_sb); + + res = hfsplus_cat_write_inode(src_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(dst_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); + out: mutex_unlock(&sbi->vh_mutex); return res; @@ -367,6 +385,9 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry) if (HFSPLUS_IS_RSRC(inode)) return -EPERM; + hfs_dbg("dir->i_ino %lu, inode->i_ino %lu\n", + dir->i_ino, inode->i_ino); + mutex_lock(&sbi->vh_mutex); cnid = (u32)(unsigned long)dentry->d_fsdata; if (inode->i_ino == cnid && @@ -408,6 +429,15 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry) inode_set_ctime_current(inode); mark_inode_dirty(inode); out: + if (!res) { + res = hfsplus_cat_write_inode(dir); + if (!res) { + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (!res) + res = hfsplus_cat_write_inode(inode); + } + } + mutex_unlock(&sbi->vh_mutex); return res; } @@ -429,6 +459,8 @@ static int hfsplus_rmdir(struct inode *dir, struct dentry *dentry) inode_set_ctime_current(inode); hfsplus_delete_inode(inode); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); out: mutex_unlock(&sbi->vh_mutex); return res; @@ -465,6 +497,12 @@ static int hfsplus_symlink(struct mnt_idmap *idmap, struct inode *dir, hfsplus_instantiate(dentry, inode, inode->i_ino); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); goto out; out_err: @@ -506,6 +544,12 @@ static int hfsplus_mknod(struct mnt_idmap *idmap, struct inode *dir, hfsplus_instantiate(dentry, inode, inode->i_ino); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); goto out; failed_mknod: @@ -552,8 +596,13 @@ static int hfsplus_rename(struct mnt_idmap *idmap, res = hfsplus_rename_cat((u32)(unsigned long)old_dentry->d_fsdata, old_dir, &old_dentry->d_name, new_dir, &new_dentry->d_name); - if (!res) + if (!res) { new_dentry->d_fsdata = old_dentry->d_fsdata; + + res = hfsplus_cat_write_inode(old_dir); + if (!res) + res = hfsplus_cat_write_inode(new_dir); + } return res; } diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index 89e8b19c127b0..de801942ae471 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -477,6 +477,8 @@ int hfs_part_find(struct super_block *sb, sector_t *part_start, /* super.c */ struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino); void hfsplus_mark_mdb_dirty(struct super_block *sb); +void hfsplus_prepare_volume_header_for_commit(struct hfsplus_vh *vhdr); +int hfsplus_commit_superblock(struct super_block *sb); /* tables.c */ extern u16 hfsplus_case_fold_table[]; diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index b51a411ecd237..6153e5cc6eb65 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -180,13 +180,29 @@ const struct dentry_operations hfsplus_dentry_operations = { .d_compare = hfsplus_compare_dentry, }; -static void hfsplus_get_perms(struct inode *inode, - struct hfsplus_perm *perms, int dir) +static int hfsplus_get_perms(struct inode *inode, + struct hfsplus_perm *perms, int dir) { struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); u16 mode; mode = be16_to_cpu(perms->mode); + if (dir) { + if (mode && !S_ISDIR(mode)) + goto bad_type; + } else if (mode) { + switch (mode & S_IFMT) { + case S_IFREG: + case S_IFLNK: + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + break; + default: + goto bad_type; + } + } i_uid_write(inode, be32_to_cpu(perms->owner)); if ((test_bit(HFSPLUS_SB_UID, &sbi->flags)) || (!i_uid_read(inode) && !mode)) @@ -212,6 +228,10 @@ static void hfsplus_get_perms(struct inode *inode, inode->i_flags |= S_APPEND; else inode->i_flags &= ~S_APPEND; + return 0; +bad_type: + pr_err("invalid file type 0%04o for inode %lu\n", mode, inode->i_ino); + return -EIO; } static int hfsplus_file_open(struct inode *inode, struct file *file) @@ -305,8 +325,12 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, struct inode *inode = file->f_mapping->host; struct hfsplus_inode_info *hip = HFSPLUS_I(inode); struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + struct hfsplus_vh *vhdr = sbi->s_vhdr; int error = 0, error2; + hfs_dbg("inode->i_ino %lu, start %llu, end %llu\n", + inode->i_ino, start, end); + error = file_write_and_wait_range(file, start, end); if (error) return error; @@ -348,6 +372,14 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, error = error2; } + mutex_lock(&sbi->vh_mutex); + hfsplus_prepare_volume_header_for_commit(vhdr); + mutex_unlock(&sbi->vh_mutex); + + error2 = hfsplus_commit_superblock(inode->i_sb); + if (!error) + error = error2; + if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) blkdev_issue_flush(inode->i_sb->s_bdev); @@ -516,7 +548,9 @@ int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd) } hfs_bnode_read(fd->bnode, &entry, fd->entryoffset, sizeof(struct hfsplus_cat_folder)); - hfsplus_get_perms(inode, &folder->permissions, 1); + res = hfsplus_get_perms(inode, &folder->permissions, 1); + if (res) + goto out; set_nlink(inode, 1); inode->i_size = 2 + be32_to_cpu(folder->valence); inode_set_atime_to_ts(inode, hfsp_mt2ut(folder->access_date)); @@ -545,7 +579,9 @@ int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd) hfsplus_inode_read_fork(inode, HFSPLUS_IS_RSRC(inode) ? &file->rsrc_fork : &file->data_fork); - hfsplus_get_perms(inode, &file->permissions, 0); + res = hfsplus_get_perms(inode, &file->permissions, 0); + if (res) + goto out; set_nlink(inode, 1); if (S_ISREG(inode->i_mode)) { if (file->permissions.dev) @@ -579,17 +615,20 @@ int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd) int hfsplus_cat_write_inode(struct inode *inode) { struct inode *main_inode = inode; + struct hfs_btree *tree = HFSPLUS_SB(inode->i_sb)->cat_tree; struct hfs_find_data fd; hfsplus_cat_entry entry; int res = 0; + hfs_dbg("inode->i_ino %lu\n", inode->i_ino); + if (HFSPLUS_IS_RSRC(inode)) main_inode = HFSPLUS_I(inode)->rsrc_inode; if (!main_inode->i_nlink) return 0; - if (hfs_find_init(HFSPLUS_SB(main_inode->i_sb)->cat_tree, &fd)) + if (hfs_find_init(tree, &fd)) /* panic? */ return -EIO; @@ -654,6 +693,15 @@ int hfsplus_cat_write_inode(struct inode *inode) set_bit(HFSPLUS_I_CAT_DIRTY, &HFSPLUS_I(inode)->flags); out: hfs_find_exit(&fd); + + if (!res) { + res = hfs_btree_write(tree); + if (res) { + pr_err("b-tree write err: %d, ino %lu\n", + res, inode->i_ino); + } + } + return res; } diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 16bc4abc67e08..5230d368bd4f2 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -53,6 +53,12 @@ static int hfsplus_system_read_inode(struct inode *inode) return -EIO; } + /* + * Assign a dummy file type, for may_open() requires that + * an inode has a valid file type. + */ + inode->i_mode = S_IFREG; + return 0; } @@ -187,40 +193,15 @@ static void hfsplus_evict_inode(struct inode *inode) } } -static int hfsplus_sync_fs(struct super_block *sb, int wait) +int hfsplus_commit_superblock(struct super_block *sb) { struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); struct hfsplus_vh *vhdr = sbi->s_vhdr; int write_backup = 0; - int error, error2; - - if (!wait) - return 0; + int error = 0, error2; hfs_dbg("starting...\n"); - /* - * Explicitly write out the special metadata inodes. - * - * While these special inodes are marked as hashed and written - * out peridocically by the flusher threads we redirty them - * during writeout of normal inodes, and thus the life lock - * prevents us from getting the latest state to disk. - */ - error = filemap_write_and_wait(sbi->cat_tree->inode->i_mapping); - error2 = filemap_write_and_wait(sbi->ext_tree->inode->i_mapping); - if (!error) - error = error2; - if (sbi->attr_tree) { - error2 = - filemap_write_and_wait(sbi->attr_tree->inode->i_mapping); - if (!error) - error = error2; - } - error2 = filemap_write_and_wait(sbi->alloc_file->i_mapping); - if (!error) - error = error2; - mutex_lock(&sbi->vh_mutex); mutex_lock(&sbi->alloc_mutex); vhdr->free_blocks = cpu_to_be32(sbi->free_blocks); @@ -249,11 +230,52 @@ static int hfsplus_sync_fs(struct super_block *sb, int wait) sbi->part_start + sbi->sect_count - 2, sbi->s_backup_vhdr_buf, NULL, REQ_OP_WRITE); if (!error) - error2 = error; + error = error2; out: mutex_unlock(&sbi->alloc_mutex); mutex_unlock(&sbi->vh_mutex); + hfs_dbg("finished: err %d\n", error); + + return error; +} + +static int hfsplus_sync_fs(struct super_block *sb, int wait) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + int error, error2; + + if (!wait) + return 0; + + hfs_dbg("starting...\n"); + + /* + * Explicitly write out the special metadata inodes. + * + * While these special inodes are marked as hashed and written + * out peridocically by the flusher threads we redirty them + * during writeout of normal inodes, and thus the life lock + * prevents us from getting the latest state to disk. + */ + error = filemap_write_and_wait(sbi->cat_tree->inode->i_mapping); + error2 = filemap_write_and_wait(sbi->ext_tree->inode->i_mapping); + if (!error) + error = error2; + if (sbi->attr_tree) { + error2 = + filemap_write_and_wait(sbi->attr_tree->inode->i_mapping); + if (!error) + error = error2; + } + error2 = filemap_write_and_wait(sbi->alloc_file->i_mapping); + if (!error) + error = error2; + + error2 = hfsplus_commit_superblock(sb); + if (!error) + error = error2; + if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) blkdev_issue_flush(sb->s_bdev); @@ -328,8 +350,6 @@ static void hfsplus_put_super(struct super_block *sb) hfs_btree_close(sbi->ext_tree); kfree(sbi->s_vhdr_buf); kfree(sbi->s_backup_vhdr_buf); - call_rcu(&sbi->rcu, delayed_free); - hfs_dbg("finished\n"); } @@ -395,6 +415,15 @@ static const struct super_operations hfsplus_sops = { .show_options = hfsplus_show_options, }; +void hfsplus_prepare_volume_header_for_commit(struct hfsplus_vh *vhdr) +{ + vhdr->last_mount_vers = cpu_to_be32(HFSP_MOUNT_VERSION); + vhdr->modify_date = hfsp_now2mt(); + be32_add_cpu(&vhdr->write_count, 1); + vhdr->attributes &= cpu_to_be32(~HFSPLUS_VOL_UNMNT); + vhdr->attributes |= cpu_to_be32(HFSPLUS_VOL_INCNSTNT); +} + static int hfsplus_fill_super(struct super_block *sb, struct fs_context *fc) { struct hfsplus_vh *vhdr; @@ -562,11 +591,7 @@ static int hfsplus_fill_super(struct super_block *sb, struct fs_context *fc) * H+LX == hfsplusutils, H+Lx == this driver, H+lx is unused * all three are registered with Apple for our use */ - vhdr->last_mount_vers = cpu_to_be32(HFSP_MOUNT_VERSION); - vhdr->modify_date = hfsp_now2mt(); - be32_add_cpu(&vhdr->write_count, 1); - vhdr->attributes &= cpu_to_be32(~HFSPLUS_VOL_UNMNT); - vhdr->attributes |= cpu_to_be32(HFSPLUS_VOL_INCNSTNT); + hfsplus_prepare_volume_header_for_commit(vhdr); hfsplus_sync_fs(sb, 1); if (!sbi->hidden_dir) { @@ -629,7 +654,6 @@ static int hfsplus_fill_super(struct super_block *sb, struct fs_context *fc) out_unload_nls: unload_nls(sbi->nls); unload_nls(nls); - kfree(sbi); return err; } @@ -688,10 +712,18 @@ static int hfsplus_init_fs_context(struct fs_context *fc) return 0; } +static void hfsplus_kill_super(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + + kill_block_super(sb); + call_rcu(&sbi->rcu, delayed_free); +} + static struct file_system_type hfsplus_fs_type = { .owner = THIS_MODULE, .name = "hfsplus", - .kill_sb = kill_block_super, + .kill_sb = hfsplus_kill_super, .fs_flags = FS_REQUIRES_DEV, .init_fs_context = hfsplus_init_fs_context, }; diff --git a/fs/inode.c b/fs/inode.c index cff1d3af0d577..2c55ec49b0239 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2322,42 +2322,40 @@ struct timespec64 current_time(struct inode *inode) } EXPORT_SYMBOL(current_time); -static int inode_needs_update_time(struct inode *inode) +static int file_update_time_flags(struct file *file, unsigned int flags) { + struct inode *inode = file_inode(file); struct timespec64 now, ts; - int sync_it = 0; + int sync_mode = 0; + int ret = 0; /* First try to exhaust all avenues to not sync */ if (IS_NOCMTIME(inode)) return 0; + if (unlikely(file->f_mode & FMODE_NOCMTIME)) + return 0; now = current_time(inode); ts = inode_get_mtime(inode); if (!timespec64_equal(&ts, &now)) - sync_it |= S_MTIME; - + sync_mode |= S_MTIME; ts = inode_get_ctime(inode); if (!timespec64_equal(&ts, &now)) - sync_it |= S_CTIME; - + sync_mode |= S_CTIME; if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode)) - sync_it |= S_VERSION; - - return sync_it; -} + sync_mode |= S_VERSION; -static int __file_update_time(struct file *file, int sync_mode) -{ - int ret = 0; - struct inode *inode = file_inode(file); + if (!sync_mode) + return 0; - /* try to update time settings */ - if (!mnt_get_write_access_file(file)) { - ret = inode_update_time(inode, sync_mode); - mnt_put_write_access_file(file); - } + if (flags & IOCB_NOWAIT) + return -EAGAIN; + if (mnt_get_write_access_file(file)) + return 0; + ret = inode_update_time(inode, sync_mode); + mnt_put_write_access_file(file); return ret; } @@ -2377,14 +2375,7 @@ static int __file_update_time(struct file *file, int sync_mode) */ int file_update_time(struct file *file) { - int ret; - struct inode *inode = file_inode(file); - - ret = inode_needs_update_time(inode); - if (ret <= 0) - return ret; - - return __file_update_time(file, ret); + return file_update_time_flags(file, 0); } EXPORT_SYMBOL(file_update_time); @@ -2406,7 +2397,6 @@ EXPORT_SYMBOL(file_update_time); static int file_modified_flags(struct file *file, int flags) { int ret; - struct inode *inode = file_inode(file); /* * Clear the security bits if the process is not being run by root. @@ -2415,17 +2405,7 @@ static int file_modified_flags(struct file *file, int flags) ret = file_remove_privs_flags(file, flags); if (ret) return ret; - - if (unlikely(file->f_mode & FMODE_NOCMTIME)) - return 0; - - ret = inode_needs_update_time(inode); - if (ret <= 0) - return ret; - if (flags & IOCB_NOWAIT) - return -EAGAIN; - - return __file_update_time(file, ret); + return file_update_time_flags(file, flags); } /** diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 8b847a1e27f13..c0fa6acf19375 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -217,6 +217,22 @@ static void ifs_free(struct folio *folio) kfree(ifs); } +/* + * Calculate how many bytes to truncate based off the number of blocks to + * truncate and the end position to start truncating from. + */ +static size_t iomap_bytes_to_truncate(loff_t end_pos, unsigned block_bits, + unsigned blocks_truncated) +{ + unsigned block_size = 1 << block_bits; + unsigned block_offset = end_pos & (block_size - 1); + + if (!block_offset) + return blocks_truncated << block_bits; + + return ((blocks_truncated - 1) << block_bits) + block_offset; +} + /* * Calculate the range inside the folio that we actually need to read. */ @@ -240,22 +256,30 @@ static void iomap_adjust_read_range(struct inode *inode, struct folio *folio, * to avoid reading in already uptodate ranges. */ if (ifs) { - unsigned int i; + unsigned int i, blocks_skipped; /* move forward for each leading block marked uptodate */ - for (i = first; i <= last; i++) { + for (i = first; i <= last; i++) if (!ifs_block_is_uptodate(ifs, i)) break; - *pos += block_size; - poff += block_size; - plen -= block_size; - first++; + + blocks_skipped = i - first; + if (blocks_skipped) { + unsigned long block_offset = *pos & (block_size - 1); + unsigned bytes_skipped = + (blocks_skipped << block_bits) - block_offset; + + *pos += bytes_skipped; + poff += bytes_skipped; + plen -= bytes_skipped; } + first = i; /* truncate len if we find any trailing uptodate block(s) */ while (++i <= last) { if (ifs_block_is_uptodate(ifs, i)) { - plen -= (last - i + 1) * block_size; + plen -= iomap_bytes_to_truncate(*pos + plen, + block_bits, last - i + 1); last = i - 1; break; } @@ -271,7 +295,8 @@ static void iomap_adjust_read_range(struct inode *inode, struct folio *folio, unsigned end = offset_in_folio(folio, isize - 1) >> block_bits; if (first <= end && last > end) - plen -= (last - end) * block_size; + plen -= iomap_bytes_to_truncate(*pos + plen, block_bits, + last - end); } *offp = poff; diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 5d5d63efbd576..e73c71f39bd45 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -179,7 +179,18 @@ static void iomap_dio_done(struct iomap_dio *dio) WRITE_ONCE(dio->submit.waiter, NULL); blk_wake_io_task(waiter); - } else if (dio->flags & IOMAP_DIO_INLINE_COMP) { + return; + } + + /* + * Always run error completions in user context. These are not + * performance critical and some code relies on taking sleeping locks + * for error handling. + */ + if (dio->error) + dio->flags &= ~IOMAP_DIO_INLINE_COMP; + + if (dio->flags & IOMAP_DIO_INLINE_COMP) { WRITE_ONCE(iocb->private, NULL); iomap_dio_complete_work(&dio->aio.work); } else if (dio->flags & IOMAP_DIO_CALLER_COMP) { @@ -418,9 +429,13 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); do { size_t n; - if (dio->error) { - iov_iter_revert(dio->submit.iter, copied); - copied = ret = 0; + + /* + * If completions already occurred and reported errors, give up now and + * don't bother submitting more bios. + */ + if (unlikely(data_race(dio->error))) { + ret = 0; goto out; } @@ -717,12 +732,12 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, } goto out_free_dio; } + } - if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) { - ret = sb_init_dio_done_wq(inode->i_sb); - if (ret < 0) - goto out_free_dio; - } + if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) { + ret = sb_init_dio_done_wq(inode->i_sb); + if (ret < 0) + goto out_free_dio; } inode_dio_begin(inode); diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index b49fa75eab260..fb26cfd9c4cc3 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -163,17 +163,18 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, WARN_ON_ONCE(!folio->private && map_len < dirty_len); switch (wpc->iomap.type) { - case IOMAP_INLINE: - WARN_ON_ONCE(1); - return -EIO; + case IOMAP_UNWRITTEN: + ioend_flags |= IOMAP_IOEND_UNWRITTEN; + break; + case IOMAP_MAPPED: + break; case IOMAP_HOLE: return map_len; default: - break; + WARN_ON_ONCE(1); + return -EIO; } - if (wpc->iomap.type == IOMAP_UNWRITTEN) - ioend_flags |= IOMAP_IOEND_UNWRITTEN; if (wpc->iomap.flags & IOMAP_F_SHARED) ioend_flags |= IOMAP_IOEND_SHARED; if (folio_test_dropbehind(folio)) diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 2d0719bf6d87c..6dbfb9cb07d7a 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -267,7 +267,15 @@ int jbd2_log_do_checkpoint(journal_t *journal) */ BUFFER_TRACE(bh, "queue"); get_bh(bh); - J_ASSERT_BH(bh, !buffer_jwrite(bh)); + if (WARN_ON_ONCE(buffer_jwrite(bh))) { + put_bh(bh); /* drop the ref we just took */ + spin_unlock(&journal->j_list_lock); + /* Clean up any previously batched buffers */ + if (batch_count) + __flush_batch(journal, &batch_count); + jbd2_journal_abort(journal, -EFSCORRUPTED); + return -EFSCORRUPTED; + } journal->j_chkpt_bhs[batch_count++] = bh; transaction->t_chp_stats.cs_written++; transaction->t_checkpoint_list = jh->b_cpnext; @@ -325,7 +333,10 @@ int jbd2_cleanup_journal_tail(journal_t *journal) if (!jbd2_journal_get_log_tail(journal, &first_tid, &blocknr)) return 1; - J_ASSERT(blocknr != 0); + if (WARN_ON_ONCE(blocknr == 0)) { + jbd2_journal_abort(journal, -EFSCORRUPTED); + return -EFSCORRUPTED; + } /* * We need to make sure that any blocks that were recently written out diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index d480b94117cd7..d02fc04e60d08 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -1521,7 +1521,6 @@ static journal_t *journal_init_common(struct block_device *bdev, struct block_device *fs_dev, unsigned long long start, int len, int blocksize) { - static struct lock_class_key jbd2_trans_commit_key; journal_t *journal; int err; int n; @@ -1530,6 +1529,7 @@ static journal_t *journal_init_common(struct block_device *bdev, if (!journal) return ERR_PTR(-ENOMEM); + lockdep_register_key(&journal->jbd2_trans_commit_key); journal->j_blocksize = blocksize; journal->j_dev = bdev; journal->j_fs_dev = fs_dev; @@ -1560,7 +1560,7 @@ static journal_t *journal_init_common(struct block_device *bdev, journal->j_max_batch_time = 15000; /* 15ms */ atomic_set(&journal->j_reserved_credits, 0); lockdep_init_map(&journal->j_trans_commit_map, "jbd2_handle", - &jbd2_trans_commit_key, 0); + &journal->jbd2_trans_commit_key, 0); /* The journal is marked for error until we succeed with recovery! */ journal->j_flags = JBD2_ABORT; @@ -1611,6 +1611,7 @@ static journal_t *journal_init_common(struct block_device *bdev, kfree(journal->j_wbuf); jbd2_journal_destroy_revoke(journal); journal_fail_superblock(journal); + lockdep_unregister_key(&journal->jbd2_trans_commit_key); kfree(journal); return ERR_PTR(err); } @@ -2187,6 +2188,7 @@ int jbd2_journal_destroy(journal_t *journal) jbd2_journal_destroy_revoke(journal); kfree(journal->j_fc_wbuf); kfree(journal->j_wbuf); + lockdep_unregister_key(&journal->jbd2_trans_commit_key); kfree(journal); return err; @@ -2349,6 +2351,12 @@ int jbd2_journal_set_features(journal_t *journal, unsigned long compat, sb->s_feature_compat |= cpu_to_be32(compat); sb->s_feature_ro_compat |= cpu_to_be32(ro); sb->s_feature_incompat |= cpu_to_be32(incompat); + /* + * Update the checksum now so that it is valid even for read-only + * filesystems where jbd2_write_superblock() doesn't get called. + */ + if (jbd2_journal_has_csum_v2or3(journal)) + sb->s_checksum = jbd2_superblock_csum(sb); unlock_buffer(journal->j_sb_buffer); jbd2_journal_init_transaction_limits(journal); @@ -2378,9 +2386,17 @@ void jbd2_journal_clear_features(journal_t *journal, unsigned long compat, sb = journal->j_superblock; + lock_buffer(journal->j_sb_buffer); sb->s_feature_compat &= ~cpu_to_be32(compat); sb->s_feature_ro_compat &= ~cpu_to_be32(ro); sb->s_feature_incompat &= ~cpu_to_be32(incompat); + /* + * Update the checksum now so that it is valid even for read-only + * filesystems where jbd2_write_superblock() doesn't get called. + */ + if (jbd2_journal_has_csum_v2or3(journal)) + sb->s_checksum = jbd2_superblock_csum(sb); + unlock_buffer(journal->j_sb_buffer); jbd2_journal_init_transaction_limits(journal); } EXPORT_SYMBOL(jbd2_journal_clear_features); diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c index 3e510564de6ee..653ee540c4a1b 100644 --- a/fs/jbd2/transaction.c +++ b/fs/jbd2/transaction.c @@ -441,7 +441,7 @@ static int start_this_handle(journal_t *journal, handle_t *handle, read_unlock(&journal->j_state_lock); current->journal_info = handle; - rwsem_acquire_read(&journal->j_trans_commit_map, 0, 0, _THIS_IP_); + rwsem_acquire_read(&journal->j_trans_commit_map, 0, 1, _THIS_IP_); jbd2_journal_free_transaction(new_transaction); /* * Ensure that no allocations done while the transaction is open are @@ -1284,14 +1284,23 @@ int jbd2_journal_get_create_access(handle_t *handle, struct buffer_head *bh) * committing transaction's lists, but it HAS to be in Forget state in * that case: the transaction must have deleted the buffer for it to be * reused here. + * In the case of file system data inconsistency, for example, if the + * block bitmap of a referenced block is not set, it can lead to the + * situation where a block being committed is allocated and used again. + * As a result, the following condition will not be satisfied, so here + * we directly trigger a JBD abort instead of immediately invoking + * bugon. */ spin_lock(&jh->b_state_lock); - J_ASSERT_JH(jh, (jh->b_transaction == transaction || - jh->b_transaction == NULL || - (jh->b_transaction == journal->j_committing_transaction && - jh->b_jlist == BJ_Forget))); + if (!(jh->b_transaction == transaction || jh->b_transaction == NULL || + (jh->b_transaction == journal->j_committing_transaction && + jh->b_jlist == BJ_Forget)) || jh->b_next_transaction != NULL) { + err = -EROFS; + spin_unlock(&jh->b_state_lock); + jbd2_journal_abort(journal, err); + goto out; + } - J_ASSERT_JH(jh, jh->b_next_transaction == NULL); J_ASSERT_JH(jh, buffer_locked(jh2bh(jh))); if (jh->b_transaction == NULL) { diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c index 0ab83bb7bbdf9..9ab3f2fc61d17 100644 --- a/fs/jfs/jfs_dtree.c +++ b/fs/jfs/jfs_dtree.c @@ -2903,7 +2903,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) stbl = DT_GETSTBL(p); for (i = index; i < p->header.nextindex; i++) { - if (stbl[i] < 0 || stbl[i] >= DTPAGEMAXSLOT) { + if (stbl[i] < 0) { jfs_err("JFS: Invalid stbl[%d] = %d for inode %ld, block = %lld", i, stbl[i], (long)ip->i_ino, (long long)bn); free_page(dirent_buf); @@ -3108,7 +3108,7 @@ static int dtReadFirst(struct inode *ip, struct btstack * btstack) /* get the leftmost entry */ stbl = DT_GETSTBL(p); - if (stbl[0] < 0 || stbl[0] >= DTPAGEMAXSLOT) { + if (stbl[0] < 0) { DT_PUTPAGE(mp); jfs_error(ip->i_sb, "stbl[0] out of bound\n"); return -EIO; diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c index b343c5ea11592..5b1c5da041630 100644 --- a/fs/jfs/jfs_logmgr.c +++ b/fs/jfs/jfs_logmgr.c @@ -2311,6 +2311,7 @@ int jfsIOWait(void *arg) { struct lbuf *bp; + set_freezable(); do { spin_lock_irq(&log_redrive_lock); while ((bp = log_redrive_list)) { diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c index 65a218eba8faf..7879c049632b3 100644 --- a/fs/jfs/namei.c +++ b/fs/jfs/namei.c @@ -1228,7 +1228,7 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, jfs_err("jfs_rename: dtInsert returned -EIO"); goto out_tx; } - if (S_ISDIR(old_ip->i_mode)) + if (S_ISDIR(old_ip->i_mode) && old_dir != new_dir) inc_nlink(new_dir); } /* @@ -1244,7 +1244,9 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, goto out_tx; } if (S_ISDIR(old_ip->i_mode)) { - drop_nlink(old_dir); + if (new_ip || old_dir != new_dir) + drop_nlink(old_dir); + if (old_dir != new_dir) { /* * Change inode number of parent for moved directory diff --git a/fs/libfs.c b/fs/libfs.c index ce8c496a6940a..6be233c787fd8 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -346,22 +346,22 @@ void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry) * User space expects the directory offset value of the replaced * (new) directory entry to be unchanged after a rename. * - * Returns zero on success, a negative errno value on failure. + * Caller must have grabbed a slot for new_dentry in the maple_tree + * associated with new_dir, even if dentry is negative. */ -int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry) +void simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) { struct offset_ctx *old_ctx = old_dir->i_op->get_offset_ctx(old_dir); struct offset_ctx *new_ctx = new_dir->i_op->get_offset_ctx(new_dir); long new_offset = dentry2offset(new_dentry); - simple_offset_remove(old_ctx, old_dentry); + if (WARN_ON(!new_offset)) + return; - if (new_offset) { - offset_set(new_dentry, 0); - return simple_offset_replace(new_ctx, old_dentry, new_offset); - } - return simple_offset_add(new_ctx, old_dentry); + simple_offset_remove(old_ctx, old_dentry); + offset_set(new_dentry, 0); + WARN_ON(simple_offset_replace(new_ctx, old_dentry, new_offset)); } /** @@ -388,31 +388,23 @@ int simple_offset_rename_exchange(struct inode *old_dir, long new_index = dentry2offset(new_dentry); int ret; - simple_offset_remove(old_ctx, old_dentry); - simple_offset_remove(new_ctx, new_dentry); + if (WARN_ON(!old_index || !new_index)) + return -EINVAL; - ret = simple_offset_replace(new_ctx, old_dentry, new_index); - if (ret) - goto out_restore; + ret = mtree_store(&new_ctx->mt, new_index, old_dentry, GFP_KERNEL); + if (WARN_ON(ret)) + return ret; - ret = simple_offset_replace(old_ctx, new_dentry, old_index); - if (ret) { - simple_offset_remove(new_ctx, old_dentry); - goto out_restore; + ret = mtree_store(&old_ctx->mt, old_index, new_dentry, GFP_KERNEL); + if (WARN_ON(ret)) { + mtree_store(&new_ctx->mt, new_index, new_dentry, GFP_KERNEL); + return ret; } - ret = simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry); - if (ret) { - simple_offset_remove(new_ctx, old_dentry); - simple_offset_remove(old_ctx, new_dentry); - goto out_restore; - } + offset_set(old_dentry, new_index); + offset_set(new_dentry, old_index); + simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry); return 0; - -out_restore: - (void)simple_offset_replace(old_ctx, old_dentry, old_index); - (void)simple_offset_replace(new_ctx, new_dentry, new_index); - return ret; } /** diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index 109e5caae8c70..4b6f18d977343 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -97,7 +97,6 @@ __nlm4svc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) struct nlm_args *argp = rqstp->rq_argp; struct nlm_host *host; struct nlm_file *file; - struct nlm_lockowner *test_owner; __be32 rc = rpc_success; dprintk("lockd: TEST4 called\n"); @@ -107,7 +106,6 @@ __nlm4svc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file))) return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; - test_owner = argp->lock.fl.c.flc_owner; /* Now check for conflicting locks */ resp->status = nlmsvc_testlock(rqstp, file, host, &argp->lock, &resp->lock); @@ -116,7 +114,7 @@ __nlm4svc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) else dprintk("lockd: TEST4 status %d\n", ntohl(resp->status)); - nlmsvc_put_lockowner(test_owner); + nlmsvc_release_lockowner(&argp->lock); nlmsvc_release_host(host); nlm_release_file(file); return rc; diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index a31dc9588eb80..d66e828515999 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -627,7 +627,13 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, } mode = lock_to_openmode(&lock->fl); - error = vfs_test_lock(file->f_file[mode], &lock->fl); + locks_init_lock(&conflock->fl); + /* vfs_test_lock only uses start, end, and owner, but tests flc_file */ + conflock->fl.c.flc_file = lock->fl.c.flc_file; + conflock->fl.fl_start = lock->fl.fl_start; + conflock->fl.fl_end = lock->fl.fl_end; + conflock->fl.c.flc_owner = lock->fl.c.flc_owner; + error = vfs_test_lock(file->f_file[mode], &conflock->fl); if (error) { /* We can't currently deal with deferred test requests */ if (error == FILE_LOCK_DEFERRED) @@ -637,22 +643,19 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, goto out; } - if (lock->fl.c.flc_type == F_UNLCK) { + if (conflock->fl.c.flc_type == F_UNLCK) { ret = nlm_granted; goto out; } dprintk("lockd: conflicting lock(ty=%d, %Ld-%Ld)\n", - lock->fl.c.flc_type, (long long)lock->fl.fl_start, - (long long)lock->fl.fl_end); + conflock->fl.c.flc_type, (long long)conflock->fl.fl_start, + (long long)conflock->fl.fl_end); conflock->caller = "somehost"; /* FIXME */ conflock->len = strlen(conflock->caller); conflock->oh.len = 0; /* don't return OH info */ - conflock->svid = lock->fl.c.flc_pid; - conflock->fl.c.flc_type = lock->fl.c.flc_type; - conflock->fl.fl_start = lock->fl.fl_start; - conflock->fl.fl_end = lock->fl.fl_end; - locks_release_private(&lock->fl); + conflock->svid = conflock->fl.c.flc_pid; + locks_release_private(&conflock->fl); ret = nlm_lck_denied; out: diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index f53d5177f2673..5817ef272332d 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -117,7 +117,6 @@ __nlmsvc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) struct nlm_args *argp = rqstp->rq_argp; struct nlm_host *host; struct nlm_file *file; - struct nlm_lockowner *test_owner; __be32 rc = rpc_success; dprintk("lockd: TEST called\n"); @@ -127,8 +126,6 @@ __nlmsvc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; - test_owner = argp->lock.fl.c.flc_owner; - /* Now check for conflicting locks */ resp->status = cast_status(nlmsvc_testlock(rqstp, file, host, &argp->lock, &resp->lock)); @@ -138,7 +135,7 @@ __nlmsvc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) dprintk("lockd: TEST status %d vers %d\n", ntohl(resp->status), rqstp->rq_vers); - nlmsvc_put_lockowner(test_owner); + nlmsvc_release_lockowner(&argp->lock); nlmsvc_release_host(host); nlm_release_file(file); return rc; diff --git a/fs/locks.c b/fs/locks.c index 04a3f0e207246..bf5e0d05a0269 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -2185,13 +2185,21 @@ SYSCALL_DEFINE2(flock, unsigned int, fd, unsigned int, cmd) /** * vfs_test_lock - test file byte range lock * @filp: The file to test lock for - * @fl: The lock to test; also used to hold result + * @fl: The byte-range in the file to test; also used to hold result * + * On entry, @fl does not contain a lock, but identifies a range (fl_start, fl_end) + * in the file (c.flc_file), and an owner (c.flc_owner) for whom existing locks + * should be ignored. c.flc_type and c.flc_flags are ignored. + * Both fl_lmops and fl_ops in @fl must be NULL. * Returns -ERRNO on failure. Indicates presence of conflicting lock by - * setting conf->fl_type to something other than F_UNLCK. + * setting fl->fl_type to something other than F_UNLCK. + * + * If vfs_test_lock() does find a lock and return it, the caller must + * use locks_free_lock() or locks_release_private() on the returned lock. */ int vfs_test_lock(struct file *filp, struct file_lock *fl) { + WARN_ON_ONCE(fl->fl_ops || fl->fl_lmops); WARN_ON_ONCE(filp != fl->c.flc_file); if (filp->f_op->lock) return filp->f_op->lock(filp, F_GETLK, fl); diff --git a/fs/minix/inode.c b/fs/minix/inode.c index 32db676127a9e..7bdd240ea1584 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -154,10 +154,38 @@ static int minix_reconfigure(struct fs_context *fc) static bool minix_check_superblock(struct super_block *sb) { struct minix_sb_info *sbi = minix_sb(sb); + unsigned long block; - if (sbi->s_imap_blocks == 0 || sbi->s_zmap_blocks == 0) + if (sbi->s_log_zone_size != 0) { + printk("minix-fs error: zone size must equal block size. " + "s_log_zone_size > 0 is not supported.\n"); + return false; + } + + if (sbi->s_ninodes < 1 || sbi->s_firstdatazone <= 4 || + sbi->s_firstdatazone >= sbi->s_nzones) return false; + /* Apparently minix can create filesystems that allocate more blocks for + * the bitmaps than needed. We simply ignore that, but verify it didn't + * create one with not enough blocks and bail out if so. + */ + block = minix_blocks_needed(sbi->s_ninodes, sb->s_blocksize); + if (sbi->s_imap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "imap blocks allocated. Refusing to mount.\n"); + return false; + } + + block = minix_blocks_needed( + (sbi->s_nzones - sbi->s_firstdatazone + 1), + sb->s_blocksize); + if (sbi->s_zmap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "zmap blocks allocated. Refusing to mount.\n"); + return false; + } + /* * s_max_size must not exceed the block mapping limitation. This check * is only needed for V1 filesystems, since V2/V3 support an extra level @@ -277,26 +305,6 @@ static int minix_fill_super(struct super_block *s, struct fs_context *fc) minix_set_bit(0,sbi->s_imap[0]->b_data); minix_set_bit(0,sbi->s_zmap[0]->b_data); - /* Apparently minix can create filesystems that allocate more blocks for - * the bitmaps than needed. We simply ignore that, but verify it didn't - * create one with not enough blocks and bail out if so. - */ - block = minix_blocks_needed(sbi->s_ninodes, s->s_blocksize); - if (sbi->s_imap_blocks < block) { - printk("MINIX-fs: file system does not have enough " - "imap blocks allocated. Refusing to mount.\n"); - goto out_no_bitmap; - } - - block = minix_blocks_needed( - (sbi->s_nzones - sbi->s_firstdatazone + 1), - s->s_blocksize); - if (sbi->s_zmap_blocks < block) { - printk("MINIX-fs: file system does not have enough " - "zmap blocks allocated. Refusing to mount.\n"); - goto out_no_bitmap; - } - /* set up enough so that it can read an inode */ s->s_op = &minix_sops; s->s_time_min = 0; diff --git a/fs/namespace.c b/fs/namespace.c index 4272349650b14..8531b8deee416 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -221,7 +221,7 @@ static int mnt_alloc_id(struct mount *mnt) int res; xa_lock(&mnt_id_xa); - res = __xa_alloc(&mnt_id_xa, &mnt->mnt_id, mnt, XA_LIMIT(1, INT_MAX), GFP_KERNEL); + res = __xa_alloc(&mnt_id_xa, &mnt->mnt_id, mnt, xa_limit_31b, GFP_KERNEL); if (!res) mnt->mnt_id_unique = ++mnt_id_ctr; xa_unlock(&mnt_id_xa); @@ -1531,23 +1531,33 @@ static struct mount *mnt_find_id_at_reverse(struct mnt_namespace *ns, u64 mnt_id static void *m_start(struct seq_file *m, loff_t *pos) { struct proc_mounts *p = m->private; + struct mount *mnt; down_read(&namespace_sem); - return mnt_find_id_at(p->ns, *pos); + mnt = mnt_find_id_at(p->ns, *pos); + if (mnt) + *pos = mnt->mnt_id_unique; + return mnt; } static void *m_next(struct seq_file *m, void *v, loff_t *pos) { - struct mount *next = NULL, *mnt = v; + struct mount *mnt = v; struct rb_node *node = rb_next(&mnt->mnt_node); - ++*pos; if (node) { - next = node_to_mount(node); + struct mount *next = node_to_mount(node); *pos = next->mnt_id_unique; + return next; } - return next; + + /* + * No more mounts. Set pos past current mount's ID so that if + * iteration restarts, mnt_find_id_at() returns NULL. + */ + *pos = mnt->mnt_id_unique + 1; + return NULL; } static void m_stop(struct seq_file *m, void *v) @@ -5796,7 +5806,7 @@ SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req, if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) && !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN)) - return -ENOENT; + return -EPERM; ks = kmalloc(sizeof(*ks), GFP_KERNEL_ACCOUNT); if (!ks) diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 37ab6f28b5ad0..88361e8c70961 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -171,9 +171,8 @@ static void netfs_queue_read(struct netfs_io_request *rreq, spin_lock(&rreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { - stream->front = subreq; if (!stream->active) { - stream->collected_to = stream->front->start; + stream->collected_to = subreq->start; /* Store list pointers before active flag */ smp_store_release(&stream->active, true); } diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c index a498ee8d66745..f72e6da88cca7 100644 --- a/fs/netfs/direct_read.c +++ b/fs/netfs/direct_read.c @@ -71,9 +71,8 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) spin_lock(&rreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { - stream->front = subreq; if (!stream->active) { - stream->collected_to = stream->front->start; + stream->collected_to = subreq->start; /* Store list pointers before active flag */ smp_store_release(&stream->active, true); } diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index a9d1c3b2c0842..f9ab69de3e298 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -9,6 +9,209 @@ #include #include "internal.h" +/* + * Perform the cleanup rituals after an unbuffered write is complete. + */ +static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) +{ + struct netfs_inode *ictx = netfs_inode(wreq->inode); + + _enter("R=%x", wreq->debug_id); + + /* Okay, declare that all I/O is complete. */ + trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); + + if (!wreq->error) + netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); + + if (wreq->origin == NETFS_DIO_WRITE && + wreq->mapping->nrpages) { + /* mmap may have got underfoot and we may now have folios + * locally covering the region we just wrote. Attempt to + * discard the folios, but leave in place any modified locally. + * ->write_iter() is prevented from interfering by the DIO + * counter. + */ + pgoff_t first = wreq->start >> PAGE_SHIFT; + pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT; + + invalidate_inode_pages2_range(wreq->mapping, first, last); + } + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_end(wreq->inode); + + _debug("finished"); + netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); + /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ + + if (wreq->iocb) { + size_t written = umin(wreq->transferred, wreq->len); + + wreq->iocb->ki_pos += written; + if (wreq->iocb->ki_complete) { + trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete); + wreq->iocb->ki_complete(wreq->iocb, wreq->error ?: written); + } + wreq->iocb = VFS_PTR_POISON; + } + + netfs_clear_subrequests(wreq); +} + +/* + * Collect the subrequest results of unbuffered write subrequests. + */ +static void netfs_unbuffered_write_collect(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + struct netfs_io_subrequest *subreq) +{ + trace_netfs_collect_sreq(wreq, subreq); + + spin_lock(&wreq->lock); + list_del_init(&subreq->rreq_link); + spin_unlock(&wreq->lock); + + wreq->transferred += subreq->transferred; + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + + stream->collected_to = subreq->start + subreq->transferred; + wreq->collected_to = stream->collected_to; + netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); + + trace_netfs_collect_stream(wreq, stream); + trace_netfs_collect_state(wreq, wreq->collected_to, 0); +} + +/* + * Write data to the server without going through the pagecache and without + * writing it to the local cache. We dispatch the subrequests serially and + * wait for each to complete before dispatching the next, lest we leave a gap + * in the data written due to a failure such as ENOSPC. We could, however + * attempt to do preparation such as content encryption for the next subreq + * whilst the current is in progress. + */ +static int netfs_unbuffered_write(struct netfs_io_request *wreq) +{ + struct netfs_io_subrequest *subreq = NULL; + struct netfs_io_stream *stream = &wreq->io_streams[0]; + int ret; + + _enter("%llx", wreq->len); + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_begin(wreq->inode); + + stream->collected_to = wreq->start; + + for (;;) { + bool retry = false; + + if (!subreq) { + netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred); + subreq = stream->construct; + stream->construct = NULL; + } + + /* Check if (re-)preparation failed. */ + if (unlikely(test_bit(NETFS_SREQ_FAILED, &subreq->flags))) { + netfs_write_subrequest_terminated(subreq, subreq->error); + wreq->error = subreq->error; + break; + } + + iov_iter_truncate(&subreq->io_iter, wreq->len - wreq->transferred); + if (!iov_iter_count(&subreq->io_iter)) + break; + + subreq->len = netfs_limit_iter(&subreq->io_iter, 0, + stream->sreq_max_len, + stream->sreq_max_segs); + iov_iter_truncate(&subreq->io_iter, subreq->len); + stream->submit_extendable_to = subreq->len; + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + stream->issue_write(subreq); + + /* Async, need to wait. */ + netfs_wait_for_in_progress_stream(wreq, stream); + + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + retry = true; + } else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { + ret = subreq->error; + wreq->error = ret; + netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed); + subreq = NULL; + break; + } + ret = 0; + + if (!retry) { + netfs_unbuffered_write_collect(wreq, stream, subreq); + subreq = NULL; + if (wreq->transferred >= wreq->len) + break; + if (!wreq->iocb && signal_pending(current)) { + ret = wreq->transferred ? -EINTR : -ERESTARTSYS; + trace_netfs_rreq(wreq, netfs_rreq_trace_intr); + break; + } + continue; + } + + /* We need to retry the last subrequest, so first reset the + * iterator, taking into account what, if anything, we managed + * to transfer. + */ + subreq->error = -EAGAIN; + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + if (subreq->transferred > 0) + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + + if (stream->source == NETFS_UPLOAD_TO_SERVER && + wreq->netfs_ops->retry_request) + wreq->netfs_ops->retry_request(wreq, stream); + + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); + __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); + subreq->io_iter = wreq->buffer.iter; + subreq->start = wreq->start + wreq->transferred; + subreq->len = wreq->len - wreq->transferred; + subreq->transferred = 0; + subreq->retry_count += 1; + stream->sreq_max_len = UINT_MAX; + stream->sreq_max_segs = INT_MAX; + + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + + if (stream->prepare_write) { + stream->prepare_write(subreq); + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + netfs_stat(&netfs_n_wh_retry_write_subreq); + } else { + struct iov_iter source; + + netfs_reset_iter(subreq); + source = subreq->io_iter; + netfs_reissue_write(stream, subreq, &source); + } + } + + netfs_unbuffered_write_done(wreq); + _leave(" = %d", ret); + return ret; +} + +static void netfs_unbuffered_write_async(struct work_struct *work) +{ + struct netfs_io_request *wreq = container_of(work, struct netfs_io_request, work); + + netfs_unbuffered_write(wreq); + netfs_put_request(wreq, netfs_rreq_trace_put_complete); +} + /* * Perform an unbuffered write where we may have to do an RMW operation on an * encrypted file. This can also be used for direct I/O writes. @@ -70,35 +273,35 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * */ wreq->buffer.iter = *iter; } + + wreq->len = iov_iter_count(&wreq->buffer.iter); } __set_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags); - if (async) - __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); /* Copy the data into the bounce buffer and encrypt it. */ // TODO /* Dispatch the write. */ __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); - if (async) - wreq->iocb = iocb; - wreq->len = iov_iter_count(&wreq->buffer.iter); - ret = netfs_unbuffered_write(wreq, is_sync_kiocb(iocb), wreq->len); - if (ret < 0) { - _debug("begin = %zd", ret); - goto out; - } - if (!async) { - ret = netfs_wait_for_write(wreq); - if (ret > 0) - iocb->ki_pos += ret; - } else { + if (async) { + INIT_WORK(&wreq->work, netfs_unbuffered_write_async); + wreq->iocb = iocb; + queue_work(system_dfl_wq, &wreq->work); ret = -EIOCBQUEUED; + } else { + ret = netfs_unbuffered_write(wreq); + if (ret < 0) { + _debug("begin = %zd", ret); + } else { + iocb->ki_pos += wreq->transferred; + ret = wreq->transferred ?: wreq->error; + } + + netfs_put_request(wreq, netfs_rreq_trace_put_complete); } -out: netfs_put_request(wreq, netfs_rreq_trace_put_return); return ret; diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 4319611f53544..d436e20d34185 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -198,6 +198,9 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, struct file *file, loff_t start, enum netfs_io_origin origin); +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start); void netfs_reissue_write(struct netfs_io_stream *stream, struct netfs_io_subrequest *subreq, struct iov_iter *source); @@ -212,7 +215,6 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c struct folio **writethrough_cache); ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, struct folio *writethrough_cache); -int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t len); /* * write_retry.c diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c index 72a435e5fc6da..154a14bb2d7f7 100644 --- a/fs/netfs/iterator.c +++ b/fs/netfs/iterator.c @@ -142,6 +142,47 @@ static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset, return min(span, max_size); } +/* + * Select the span of a kvec iterator we're going to use. Limit it by both + * maximum size and maximum number of segments. Returns the size of the span + * in bytes. + */ +static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset, + size_t max_size, size_t max_segs) +{ + const struct kvec *kvecs = iter->kvec; + unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0; + size_t len, span = 0, n = iter->count; + size_t skip = iter->iov_offset + start_offset; + + if (WARN_ON(!iov_iter_is_kvec(iter)) || + WARN_ON(start_offset > n) || + n == 0) + return 0; + + while (n && ix < nkv && skip) { + len = kvecs[ix].iov_len; + if (skip < len) + break; + skip -= len; + n -= len; + ix++; + } + + while (n && ix < nkv) { + len = min3(n, kvecs[ix].iov_len - skip, max_size); + span += len; + nsegs++; + ix++; + if (span >= max_size || nsegs >= max_segs) + break; + skip = 0; + n -= len; + } + + return min(span, max_size); +} + /* * Select the span of an xarray iterator we're going to use. Limit it by both * maximum size and maximum number of segments. It is assumed that segments @@ -245,6 +286,8 @@ size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, return netfs_limit_bvec(iter, start_offset, max_size, max_segs); if (iov_iter_is_xarray(iter)) return netfs_limit_xarray(iter, start_offset, max_size, max_segs); + if (iov_iter_is_kvec(iter)) + return netfs_limit_kvec(iter, start_offset, max_size, max_segs); BUG(); } EXPORT_SYMBOL(netfs_limit_iter); diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index a95e7aadafd07..e5f6665b3341e 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -137,7 +137,7 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq, rreq->front_folio_order = order; fsize = PAGE_SIZE << order; fpos = folio_pos(folio); - fend = umin(fpos + fsize, rreq->i_size); + fend = fpos + fsize; trace_netfs_collect_folio(rreq, folio, fend, collected_to); @@ -205,7 +205,8 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq) * in progress. The issuer thread may be adding stuff to the tail * whilst we're doing this. */ - front = READ_ONCE(stream->front); + front = list_first_entry_or_null(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); while (front) { size_t transferred; @@ -301,7 +302,6 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq) list_del_init(&front->rreq_link); front = list_first_entry_or_null(&stream->subrequests, struct netfs_io_subrequest, rreq_link); - stream->front = front; spin_unlock(&rreq->lock); netfs_put_subrequest(remove, notes & ABANDON_SREQ ? @@ -546,6 +546,15 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq) } } + /* If need retry is set, error should not matter unless we hit too many + * retries. Pause the generation of new subreqs + */ + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause); + set_bit(NETFS_RREQ_PAUSE, &rreq->flags); + goto skip_error_checks; + } + if (unlikely(subreq->error < 0)) { trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read); if (subreq->source == NETFS_READ_FROM_CACHE) { @@ -559,6 +568,7 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq) set_bit(NETFS_RREQ_PAUSE, &rreq->flags); } +skip_error_checks: trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); netfs_subreq_clear_in_progress(subreq); netfs_put_subrequest(subreq, netfs_sreq_trace_put_terminated); diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index b99e84a8170af..cca9ac43c0773 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -12,6 +12,7 @@ static void netfs_reissue_read(struct netfs_io_request *rreq, struct netfs_io_subrequest *subreq) { + subreq->error = 0; __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); netfs_stat(&netfs_n_rh_retry_read_subreq); @@ -92,8 +93,10 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) from->start, from->transferred, from->len); if (test_bit(NETFS_SREQ_FAILED, &from->flags) || - !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) + !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) { + subreq = from; goto abandon; + } list_for_each_continue(next, &stream->subrequests) { subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); @@ -177,6 +180,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) if (subreq == to) break; } + subreq = NULL; continue; } @@ -242,8 +246,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) subreq = list_next_entry(subreq, rreq_link); abandon: list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { - if (!subreq->error && - !test_bit(NETFS_SREQ_FAILED, &subreq->flags) && + if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) continue; subreq->error = -ENOMEM; diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c index 5c0dc4efc7922..9d48ced80d1fa 100644 --- a/fs/netfs/read_single.c +++ b/fs/netfs/read_single.c @@ -107,7 +107,6 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) spin_lock(&rreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); trace_netfs_sreq(subreq, netfs_sreq_trace_added); - stream->front = subreq; /* Store list pointers before active flag */ smp_store_release(&stream->active, true); spin_unlock(&rreq->lock); diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c index cbf3d9194c7bf..b194447f4b111 100644 --- a/fs/netfs/write_collect.c +++ b/fs/netfs/write_collect.c @@ -228,7 +228,8 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) if (!smp_load_acquire(&stream->active)) continue; - front = stream->front; + front = list_first_entry_or_null(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); while (front) { trace_netfs_collect_sreq(wreq, front); //_debug("sreq [%x] %llx %zx/%zx", @@ -279,7 +280,6 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) list_del_init(&front->rreq_link); front = list_first_entry_or_null(&stream->subrequests, struct netfs_io_subrequest, rreq_link); - stream->front = front; spin_unlock(&wreq->lock); netfs_put_subrequest(remove, notes & SAW_FAILURE ? @@ -399,27 +399,6 @@ bool netfs_write_collection(struct netfs_io_request *wreq) ictx->ops->invalidate_cache(wreq); } - if ((wreq->origin == NETFS_UNBUFFERED_WRITE || - wreq->origin == NETFS_DIO_WRITE) && - !wreq->error) - netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); - - if (wreq->origin == NETFS_DIO_WRITE && - wreq->mapping->nrpages) { - /* mmap may have got underfoot and we may now have folios - * locally covering the region we just wrote. Attempt to - * discard the folios, but leave in place any modified locally. - * ->write_iter() is prevented from interfering by the DIO - * counter. - */ - pgoff_t first = wreq->start >> PAGE_SHIFT; - pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT; - invalidate_inode_pages2_range(wreq->mapping, first, last); - } - - if (wreq->origin == NETFS_DIO_WRITE) - inode_dio_end(wreq->inode); - _debug("finished"); netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ @@ -492,11 +471,11 @@ void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error) if (IS_ERR_VALUE(transferred_or_error)) { subreq->error = transferred_or_error; - if (subreq->error == -EAGAIN) - set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); - else + /* if need retry is set, error should not matter */ + if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { set_bit(NETFS_SREQ_FAILED, &subreq->flags); - trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + } switch (subreq->source) { case NETFS_WRITE_TO_CACHE: diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index dd8743bc8d7fe..2db688f941251 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -154,9 +154,9 @@ EXPORT_SYMBOL(netfs_prepare_write_failed); * Prepare a write subrequest. We need to allocate a new subrequest * if we don't have one. */ -static void netfs_prepare_write(struct netfs_io_request *wreq, - struct netfs_io_stream *stream, - loff_t start) +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start) { struct netfs_io_subrequest *subreq; struct iov_iter *wreq_iter = &wreq->buffer.iter; @@ -206,9 +206,8 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, spin_lock(&wreq->lock); list_add_tail(&subreq->rreq_link, &stream->subrequests); if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { - stream->front = subreq; if (!stream->active) { - stream->collected_to = stream->front->start; + stream->collected_to = subreq->start; /* Write list pointers before active flag */ smp_store_release(&stream->active, true); } @@ -250,6 +249,7 @@ void netfs_reissue_write(struct netfs_io_stream *stream, iov_iter_truncate(&subreq->io_iter, size); subreq->retry_count++; + subreq->error = 0; __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); netfs_stat(&netfs_n_wh_retry_write_subreq); @@ -697,41 +697,6 @@ ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_c return ret; } -/* - * Write data to the server without going through the pagecache and without - * writing it to the local cache. - */ -int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t len) -{ - struct netfs_io_stream *upload = &wreq->io_streams[0]; - ssize_t part; - loff_t start = wreq->start; - int error = 0; - - _enter("%zx", len); - - if (wreq->origin == NETFS_DIO_WRITE) - inode_dio_begin(wreq->inode); - - while (len) { - // TODO: Prepare content encryption - - _debug("unbuffered %zx", len); - part = netfs_advance_write(wreq, upload, start, len, false); - start += part; - len -= part; - rolling_buffer_advance(&wreq->buffer, part); - if (test_bit(NETFS_RREQ_PAUSE, &wreq->flags)) - netfs_wait_for_paused_write(wreq); - if (test_bit(NETFS_RREQ_FAILED, &wreq->flags)) - break; - } - - netfs_end_issue_write(wreq); - _leave(" = %d", error); - return error; -} - /* * Write some of a pending folio data back to the server and/or the cache. */ diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index fc9c3e0d34d81..29489a23a2209 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -98,7 +98,6 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, subreq->start = start; subreq->len = len; __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); - subreq->retry_count++; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); /* Renegotiate max_len (wsize) */ diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index ab76120705e20..134d7f760a33a 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -417,8 +417,10 @@ bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d, d->map = bl_map_simple; d->pr_key = v->scsi.pr_key; - if (d->len == 0) - return -ENODEV; + if (d->len == 0) { + error = -ENODEV; + goto out_blkdev_put; + } ops = bdev->bd_disk->fops->pr_ops; if (!ops) { diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 54699299d5b16..2aaea9c98c2cd 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -784,10 +784,18 @@ static int nfs_init_server(struct nfs_server *server, server->fattr_valid = NFS_ATTR_FATTR_V4; } - if (ctx->rsize) + if (ctx->bsize) { + server->bsize = ctx->bsize; + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_BSIZE; + } + if (ctx->rsize) { server->rsize = nfs_io_size(ctx->rsize, clp->cl_proto); - if (ctx->wsize) + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_RSIZE; + } + if (ctx->wsize) { server->wsize = nfs_io_size(ctx->wsize, clp->cl_proto); + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_WSIZE; + } server->acregmin = ctx->acregmin * HZ; server->acregmax = ctx->acregmax * HZ; @@ -977,8 +985,13 @@ EXPORT_SYMBOL_GPL(nfs_probe_server); void nfs_server_copy_userdata(struct nfs_server *target, struct nfs_server *source) { target->flags = source->flags; - target->rsize = source->rsize; - target->wsize = source->wsize; + target->automount_inherit = source->automount_inherit; + if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_BSIZE) + target->bsize = source->bsize; + if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_RSIZE) + target->rsize = source->rsize; + if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_WSIZE) + target->wsize = source->wsize; target->acregmin = source->acregmin; target->acregmax = source->acregmax; target->acdirmin = source->acdirmin; diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index ea9f6ca8f30fa..a653a401b7970 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -72,7 +72,7 @@ const struct address_space_operations nfs_dir_aops = { .free_folio = nfs_readdir_clear_array, }; -#define NFS_INIT_DTSIZE PAGE_SIZE +#define NFS_INIT_DTSIZE SZ_64K static struct nfs_open_dir_context * alloc_nfs_open_dir_context(struct inode *dir) @@ -83,7 +83,7 @@ alloc_nfs_open_dir_context(struct inode *dir) ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); if (ctx != NULL) { ctx->attr_gencount = nfsi->attr_gencount; - ctx->dtsize = NFS_INIT_DTSIZE; + ctx->dtsize = min(NFS_SERVER(dir)->dtsize, NFS_INIT_DTSIZE); spin_lock(&dir->i_lock); if (list_empty(&nfsi->open_files) && (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) @@ -789,16 +789,17 @@ void nfs_prime_dcache(struct dentry *parent, struct nfs_entry *entry, goto out; } + nfs_set_verifier(dentry, dir_verifier); inode = nfs_fhget(dentry->d_sb, entry->fh, entry->fattr); alias = d_splice_alias(inode, dentry); d_lookup_done(dentry); if (alias) { if (IS_ERR(alias)) goto out; + nfs_set_verifier(alias, dir_verifier); dput(dentry); dentry = alias; } - nfs_set_verifier(dentry, dir_verifier); trace_nfs_readdir_lookup(d_inode(parent), dentry, 0); out: dput(dentry); @@ -1894,13 +1895,15 @@ static int nfs_dentry_delete(const struct dentry *dentry) } /* Ensure that we revalidate inode->i_nlink */ -static void nfs_drop_nlink(struct inode *inode) +static void nfs_drop_nlink(struct inode *inode, unsigned long gencount) { + struct nfs_inode *nfsi = NFS_I(inode); + spin_lock(&inode->i_lock); /* drop the inode if we're reasonably sure this is the last link */ - if (inode->i_nlink > 0) + if (inode->i_nlink > 0 && gencount == nfsi->attr_gencount) drop_nlink(inode); - NFS_I(inode)->attr_gencount = nfs_inc_attr_generation_counter(); + nfsi->attr_gencount = nfs_inc_attr_generation_counter(); nfs_set_cache_invalid( inode, NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_CTIME | NFS_INO_INVALID_NLINK); @@ -1914,8 +1917,9 @@ static void nfs_drop_nlink(struct inode *inode) static void nfs_dentry_iput(struct dentry *dentry, struct inode *inode) { if (dentry->d_flags & DCACHE_NFSFS_RENAMED) { + unsigned long gencount = READ_ONCE(NFS_I(inode)->attr_gencount); nfs_complete_unlink(dentry, inode); - nfs_drop_nlink(inode); + nfs_drop_nlink(inode, gencount); } iput(inode); } @@ -1991,13 +1995,14 @@ struct dentry *nfs_lookup(struct inode *dir, struct dentry * dentry, unsigned in nfs_lookup_advise_force_readdirplus(dir, flags); no_entry: + nfs_set_verifier(dentry, dir_verifier); res = d_splice_alias(inode, dentry); if (res != NULL) { if (IS_ERR(res)) goto out; + nfs_set_verifier(res, dir_verifier); dentry = res; } - nfs_set_verifier(dentry, dir_verifier); out: trace_nfs_lookup_exit(dir, dentry, flags, PTR_ERR_OR_ZERO(res)); nfs_free_fattr(fattr); @@ -2139,12 +2144,12 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry, d_drop(dentry); switch (err) { case -ENOENT: - d_splice_alias(NULL, dentry); if (nfs_server_capable(dir, NFS_CAP_CASE_INSENSITIVE)) dir_verifier = inode_peek_iversion_raw(dir); else dir_verifier = nfs_save_change_attribute(dir); nfs_set_verifier(dentry, dir_verifier); + d_splice_alias(NULL, dentry); break; case -EISDIR: case -ENOTDIR: @@ -2507,9 +2512,11 @@ static int nfs_safe_remove(struct dentry *dentry) trace_nfs_remove_enter(dir, dentry); if (inode != NULL) { + unsigned long gencount = READ_ONCE(NFS_I(inode)->attr_gencount); + error = NFS_PROTO(dir)->remove(dir, dentry); if (error == 0) - nfs_drop_nlink(inode); + nfs_drop_nlink(inode, gencount); } else error = NFS_PROTO(dir)->remove(dir, dentry); if (error == -ENOENT) @@ -2709,6 +2716,7 @@ int nfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, { struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); + unsigned long new_gencount = 0; struct dentry *dentry = NULL; struct rpc_task *task; bool must_unblock = false; @@ -2761,6 +2769,7 @@ int nfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, } else { block_revalidate(new_dentry); must_unblock = true; + new_gencount = NFS_I(new_inode)->attr_gencount; spin_unlock(&new_dentry->d_lock); } @@ -2800,7 +2809,7 @@ int nfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, new_dir, new_dentry, error); if (!error) { if (new_inode != NULL) - nfs_drop_nlink(new_inode); + nfs_drop_nlink(new_inode, new_gencount); /* * The d_move() should be here instead of in an async RPC completion * handler because we need the proper locks to move the dentry. If diff --git a/fs/nfs/file.c b/fs/nfs/file.c index d020aab40c64e..d1c138a416cfb 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -511,7 +511,8 @@ static bool nfs_release_folio(struct folio *folio, gfp_t gfp) if ((current_gfp_context(gfp) & GFP_KERNEL) != GFP_KERNEL || current_is_kswapd() || current_is_kcompactd()) return false; - if (nfs_wb_folio(folio->mapping->host, folio) < 0) + if (nfs_wb_folio_reclaim(folio->mapping->host, folio) < 0 || + folio_test_private(folio)) return false; } return nfs_fscache_release_folio(folio, gfp); diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index c55ea8fa3bfa5..c2d8a13a9dbdd 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -103,7 +103,7 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, sizeof(struct nfs4_ff_ds_version), gfp_flags); if (!ds_versions) - goto out_scratch; + goto out_err_drain_dsaddrs; for (i = 0; i < version_count; i++) { /* 20 = version(4) + minor_version(4) + rsize(4) + wsize(4) + diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 13ad70fc00d84..8c2bfcc323e02 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -716,7 +716,7 @@ nfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, { struct inode *inode = d_inode(dentry); struct nfs_fattr *fattr; - loff_t oldsize = i_size_read(inode); + loff_t oldsize; int error = 0; kuid_t task_uid = current_fsuid(); kuid_t owner_uid = inode->i_uid; @@ -727,6 +727,10 @@ nfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (attr->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) attr->ia_valid &= ~ATTR_MODE; + if (S_ISREG(inode->i_mode)) + nfs_file_block_o_direct(NFS_I(inode)); + + oldsize = i_size_read(inode); if (attr->ia_valid & ATTR_SIZE) { BUG_ON(!S_ISREG(inode->i_mode)); @@ -774,10 +778,8 @@ nfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, trace_nfs_setattr_enter(inode); /* Write all dirty data */ - if (S_ISREG(inode->i_mode)) { - nfs_file_block_o_direct(NFS_I(inode)); + if (S_ISREG(inode->i_mode)) nfs_sync_inode(inode); - } fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode)); if (fattr == NULL) { diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 2ecd38e1d17a8..2e596244799f3 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -13,7 +13,7 @@ #include #include -#define NFS_SB_MASK (SB_NOSUID|SB_NODEV|SB_NOEXEC|SB_SYNCHRONOUS) +#define NFS_SB_MASK (SB_RDONLY|SB_NOSUID|SB_NODEV|SB_NOEXEC|SB_SYNCHRONOUS) extern const struct export_operations nfs_export_ops; @@ -152,7 +152,6 @@ struct nfs_fs_context { struct super_block *sb; struct dentry *dentry; struct nfs_fattr *fattr; - unsigned int inherited_bsize; } clone_data; }; diff --git a/fs/nfs/io.c b/fs/nfs/io.c index d275b0a250bf3..8337f0ae852d4 100644 --- a/fs/nfs/io.c +++ b/fs/nfs/io.c @@ -84,6 +84,7 @@ nfs_start_io_write(struct inode *inode) nfs_file_block_o_direct(NFS_I(inode)); return err; } +EXPORT_SYMBOL_GPL(nfs_start_io_write); /** * nfs_end_io_write - declare that the buffered write operation is done @@ -97,6 +98,7 @@ nfs_end_io_write(struct inode *inode) { up_write(&inode->i_rwsem); } +EXPORT_SYMBOL_GPL(nfs_end_io_write); /* Call with exclusively locked inode->i_rwsem */ static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode) diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index 656976b4f42ce..358d686d2ae33 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -43,8 +43,8 @@ struct nfs_local_kiocb { size_t end_len; short int end_iter_index; atomic_t n_iters; + struct iov_iter iters[NFSLOCAL_MAX_IOS]; bool iter_is_dio_aligned[NFSLOCAL_MAX_IOS]; - struct iov_iter iters[NFSLOCAL_MAX_IOS] ____cacheline_aligned; /* End mostly DIO-specific members */ }; @@ -58,6 +58,11 @@ struct nfs_local_fsync_ctx { static bool localio_enabled __read_mostly = true; module_param(localio_enabled, bool, 0644); +static int nfs_local_do_read(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); +static int nfs_local_do_write(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); + static inline bool nfs_client_is_local(const struct nfs_client *clp) { return !!rcu_access_pointer(clp->cl_uuid.net); @@ -286,6 +291,18 @@ nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, } EXPORT_SYMBOL_GPL(nfs_local_open_fh); +/* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into NFS via nfs_writepages. + */ +static void +nfs_local_mapping_set_gfp_nofs_context(struct address_space *m) +{ + gfp_t gfp_mask = mapping_gfp_mask(m); + + mapping_set_gfp_mask(m, (gfp_mask & ~(__GFP_FS))); +} + static void nfs_local_iocb_free(struct nfs_local_kiocb *iocb) { @@ -310,6 +327,7 @@ nfs_local_iocb_alloc(struct nfs_pgio_header *hdr, return NULL; } + nfs_local_mapping_set_gfp_nofs_context(file->f_mapping); init_sync_kiocb(&iocb->kiocb, file); iocb->hdr = hdr; @@ -339,8 +357,6 @@ nfs_is_local_dio_possible(struct nfs_local_kiocb *iocb, int rw, if (unlikely(!nf_dio_mem_align || !nf_dio_offset_align)) return false; - if (unlikely(nf_dio_offset_align > PAGE_SIZE)) - return false; if (unlikely(len < nf_dio_offset_align)) return false; @@ -463,6 +479,8 @@ nfs_local_iters_init(struct nfs_local_kiocb *iocb, int rw) v = 0; total = hdr->args.count; base = hdr->args.pgbase; + pagevec += base >> PAGE_SHIFT; + base &= ~PAGE_MASK; while (total && v < hdr->page_array.npages) { len = min_t(size_t, total, PAGE_SIZE - base); bvec_set_page(&iocb->bvec[v], *pagevec, len, base); @@ -542,13 +560,50 @@ nfs_local_iocb_release(struct nfs_local_kiocb *iocb) nfs_local_iocb_free(iocb); } -static void -nfs_local_pgio_release(struct nfs_local_kiocb *iocb) +static void nfs_local_pgio_restart(struct nfs_local_kiocb *iocb, + struct nfs_pgio_header *hdr) +{ + int status = 0; + + iocb->kiocb.ki_pos = hdr->args.offset; + iocb->kiocb.ki_flags &= ~(IOCB_DSYNC | IOCB_SYNC | IOCB_DIRECT); + iocb->kiocb.ki_complete = NULL; + iocb->aio_complete_work = NULL; + iocb->end_iter_index = -1; + + switch (hdr->rw_mode) { + case FMODE_READ: + nfs_local_iters_init(iocb, ITER_DEST); + status = nfs_local_do_read(iocb, hdr->task.tk_ops); + break; + case FMODE_WRITE: + nfs_local_iters_init(iocb, ITER_SOURCE); + status = nfs_local_do_write(iocb, hdr->task.tk_ops); + break; + default: + status = -EOPNOTSUPP; + } + + if (status != 0) { + nfs_local_iocb_release(iocb); + hdr->task.tk_status = status; + nfs_local_hdr_release(hdr, hdr->task.tk_ops); + } +} + +static void nfs_local_pgio_release(struct nfs_local_kiocb *iocb) { struct nfs_pgio_header *hdr = iocb->hdr; + struct rpc_task *task = &hdr->task; - nfs_local_iocb_release(iocb); - nfs_local_hdr_release(hdr, hdr->task.tk_ops); + task->tk_action = NULL; + task->tk_ops->rpc_call_done(task, hdr); + + if (task->tk_action == NULL) { + nfs_local_iocb_release(iocb); + task->tk_ops->rpc_release(hdr); + } else + nfs_local_pgio_restart(iocb, hdr); } /* @@ -625,8 +680,6 @@ static void nfs_local_call_read(struct work_struct *work) ssize_t status; int n_iters; - save_cred = override_creds(filp->f_cred); - n_iters = atomic_read(&iocb->n_iters); for (int i = 0; i < n_iters ; i++) { if (iocb->iter_is_dio_aligned[i]) { @@ -639,7 +692,10 @@ static void nfs_local_call_read(struct work_struct *work) } else iocb->kiocb.ki_flags &= ~IOCB_DIRECT; + save_cred = override_creds(filp->f_cred); status = filp->f_op->read_iter(&iocb->kiocb, &iocb->iters[i]); + revert_creds(save_cred); + if (status != -EIOCBQUEUED) { if (unlikely(status >= 0 && status < iocb->iters[i].count)) force_done = true; /* Partial read */ @@ -649,8 +705,6 @@ static void nfs_local_call_read(struct work_struct *work) } } } - - revert_creds(save_cred); } static int @@ -777,19 +831,7 @@ static void nfs_local_write_done(struct nfs_local_kiocb *iocb) pr_info_ratelimited("nfs: Unexpected direct I/O write alignment failure\n"); } - /* Handle short writes as if they are ENOSPC */ - status = hdr->res.count; - if (status > 0 && status < hdr->args.count) { - hdr->mds_offset += status; - hdr->args.offset += status; - hdr->args.pgbase += status; - hdr->args.count -= status; - nfs_set_pgio_error(hdr, -ENOSPC, hdr->args.offset); - status = -ENOSPC; - /* record -ENOSPC in terms of nfs_local_pgio_done */ - (void) nfs_local_pgio_done(iocb, status, true); - } - if (hdr->task.tk_status < 0) + if (status < 0) nfs_reset_boot_verifier(hdr->inode); } @@ -832,7 +874,6 @@ static void nfs_local_call_write(struct work_struct *work) int n_iters; current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; - save_cred = override_creds(filp->f_cred); file_start_write(filp); n_iters = atomic_read(&iocb->n_iters); @@ -847,7 +888,10 @@ static void nfs_local_call_write(struct work_struct *work) } else iocb->kiocb.ki_flags &= ~IOCB_DIRECT; + save_cred = override_creds(filp->f_cred); status = filp->f_op->write_iter(&iocb->kiocb, &iocb->iters[i]); + revert_creds(save_cred); + if (status != -EIOCBQUEUED) { if (unlikely(status >= 0 && status < iocb->iters[i].count)) force_done = true; /* Partial write */ @@ -859,7 +903,6 @@ static void nfs_local_call_write(struct work_struct *work) } file_end_write(filp); - revert_creds(save_cred); current->flags = old_flags; } @@ -952,8 +995,6 @@ int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio, } if (status != 0) { - if (status == -EAGAIN) - nfs_localio_disable_client(clp); nfs_local_iocb_release(iocb); hdr->task.tk_status = status; nfs_local_hdr_release(hdr, call_ops); @@ -980,6 +1021,8 @@ nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data) end = LLONG_MAX; } + nfs_local_mapping_set_gfp_nofs_context(filp->f_mapping); + dprintk("%s: commit %llu - %llu\n", __func__, start, end); return vfs_fsync_range(filp, start, end, 0); } @@ -1021,17 +1064,22 @@ nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx) static void nfs_local_fsync_work(struct work_struct *work) { + unsigned long old_flags = current->flags; struct nfs_local_fsync_ctx *ctx; int status; ctx = container_of(work, struct nfs_local_fsync_ctx, work); + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio), ctx->data); nfs_local_commit_done(ctx->data, status); if (ctx->done != NULL) complete(ctx->done); nfs_local_fsync_ctx_free(ctx); + + current->flags = old_flags; } static struct nfs_local_fsync_ctx * @@ -1055,7 +1103,7 @@ int nfs_local_commit(struct nfsd_file *localio, { struct nfs_local_fsync_ctx *ctx; - ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_KERNEL); + ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_NOIO); if (!ctx) { nfs_local_commit_done(data, -ENOMEM); nfs_local_release_commit_data(localio, data, call_ops); @@ -1067,10 +1115,10 @@ int nfs_local_commit(struct nfsd_file *localio, if (how & FLUSH_SYNC) { DECLARE_COMPLETION_ONSTACK(done); ctx->done = &done; - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); wait_for_completion(&done); } else - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); return 0; } diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index 5a4d193da1a98..af9be0c5f5163 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -149,6 +149,7 @@ struct vfsmount *nfs_d_automount(struct path *path) struct vfsmount *mnt = ERR_PTR(-ENOMEM); struct nfs_server *server = NFS_SB(path->dentry->d_sb); struct nfs_client *client = server->nfs_client; + unsigned long s_flags = path->dentry->d_sb->s_flags; int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout); int ret; @@ -169,11 +170,21 @@ struct vfsmount *nfs_d_automount(struct path *path) if (!ctx->clone_data.fattr) goto out_fc; + if (fc->cred != server->cred) { + put_cred(fc->cred); + fc->cred = get_cred(server->cred); + } + if (fc->net_ns != client->cl_net) { put_net(fc->net_ns); fc->net_ns = get_net(client->cl_net); } + /* Inherit the flags covered by NFS_SB_MASK */ + fc->sb_flags_mask |= NFS_SB_MASK; + fc->sb_flags &= ~NFS_SB_MASK; + fc->sb_flags |= s_flags & NFS_SB_MASK; + /* for submounts we want the same server; referrals will reassign */ memcpy(&ctx->nfs_server._address, &client->cl_addr, client->cl_addrlen); ctx->nfs_server.addrlen = client->cl_addrlen; @@ -184,6 +195,10 @@ struct vfsmount *nfs_d_automount(struct path *path) ctx->nfs_mod = client->cl_nfs_mod; get_nfs_version(ctx->nfs_mod); + /* Inherit block sizes if they were specified as mount parameters */ + if (server->automount_inherit & NFS_AUTOMOUNT_INHERIT_BSIZE) + ctx->bsize = server->bsize; + ret = client->rpc_ops->submount(fc, server); if (ret < 0) { mnt = ERR_PTR(ret); @@ -283,7 +298,6 @@ int nfs_do_submount(struct fs_context *fc) return -ENOMEM; ctx->internal = true; - ctx->clone_data.inherited_bsize = ctx->clone_data.sb->s_blocksize_bits; p = nfs_devname(dentry, buffer, 4096); if (IS_ERR(p)) { diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index a4cb67573aa7c..993f62636a772 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -392,8 +392,13 @@ nfs3_proc_create(struct inode *dir, struct dentry *dentry, struct iattr *sattr, if (status != 0) goto out_release_acls; - if (d_alias) + if (d_alias) { + if (d_is_dir(d_alias)) { + status = -EISDIR; + goto out_dput; + } dentry = d_alias; + } /* When we created the file with exclusive semantics, make * sure we set the attributes afterwards. */ diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index d537fb0c230e8..c08520828708b 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -114,7 +114,6 @@ static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, exception.inode = inode; exception.state = lock->open_context->state; - nfs_file_block_o_direct(NFS_I(inode)); err = nfs_sync_inode(inode); if (err) goto out; @@ -138,13 +137,17 @@ int nfs42_proc_allocate(struct file *filep, loff_t offset, loff_t len) .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ALLOCATE], }; struct inode *inode = file_inode(filep); - loff_t oldsize = i_size_read(inode); + loff_t oldsize; int err; if (!nfs_server_capable(inode, NFS_CAP_ALLOCATE)) return -EOPNOTSUPP; - inode_lock(inode); + err = nfs_start_io_write(inode); + if (err) + return err; + + oldsize = i_size_read(inode); err = nfs42_proc_fallocate(&msg, filep, offset, len); @@ -155,7 +158,7 @@ int nfs42_proc_allocate(struct file *filep, loff_t offset, loff_t len) NFS_SERVER(inode)->caps &= ~(NFS_CAP_ALLOCATE | NFS_CAP_ZERO_RANGE); - inode_unlock(inode); + nfs_end_io_write(inode); return err; } @@ -170,7 +173,9 @@ int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len) if (!nfs_server_capable(inode, NFS_CAP_DEALLOCATE)) return -EOPNOTSUPP; - inode_lock(inode); + err = nfs_start_io_write(inode); + if (err) + return err; err = nfs42_proc_fallocate(&msg, filep, offset, len); if (err == 0) @@ -179,7 +184,7 @@ int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len) NFS_SERVER(inode)->caps &= ~(NFS_CAP_DEALLOCATE | NFS_CAP_ZERO_RANGE); - inode_unlock(inode); + nfs_end_io_write(inode); return err; } @@ -189,14 +194,17 @@ int nfs42_proc_zero_range(struct file *filep, loff_t offset, loff_t len) .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ZERO_RANGE], }; struct inode *inode = file_inode(filep); - loff_t oldsize = i_size_read(inode); + loff_t oldsize; int err; if (!nfs_server_capable(inode, NFS_CAP_ZERO_RANGE)) return -EOPNOTSUPP; - inode_lock(inode); + err = nfs_start_io_write(inode); + if (err) + return err; + oldsize = i_size_read(inode); err = nfs42_proc_fallocate(&msg, filep, offset, len); if (err == 0) { nfs_truncate_last_folio(inode->i_mapping, oldsize, @@ -205,7 +213,7 @@ int nfs42_proc_zero_range(struct file *filep, loff_t offset, loff_t len) } else if (err == -EOPNOTSUPP) NFS_SERVER(inode)->caps &= ~NFS_CAP_ZERO_RANGE; - inode_unlock(inode); + nfs_end_io_write(inode); return err; } @@ -416,7 +424,7 @@ static ssize_t _nfs42_proc_copy(struct file *src, struct nfs_server *src_server = NFS_SERVER(src_inode); loff_t pos_src = args->src_pos; loff_t pos_dst = args->dst_pos; - loff_t oldsize_dst = i_size_read(dst_inode); + loff_t oldsize_dst; size_t count = args->count; ssize_t status; @@ -461,6 +469,7 @@ static ssize_t _nfs42_proc_copy(struct file *src, &src_lock->open_context->state->flags); set_bit(NFS_CLNT_DST_SSC_COPY_STATE, &dst_lock->open_context->state->flags); + oldsize_dst = i_size_read(dst_inode); status = nfs4_call_sync(dst_server->client, dst_server, &msg, &args->seq_args, &res->seq_res, 0); diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 3a4baed993c96..4ff0e9dd1145e 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -1174,10 +1174,20 @@ static int nfs4_init_server(struct nfs_server *server, struct fs_context *fc) if (error < 0) return error; - if (ctx->rsize) - server->rsize = nfs_io_size(ctx->rsize, server->nfs_client->cl_proto); - if (ctx->wsize) - server->wsize = nfs_io_size(ctx->wsize, server->nfs_client->cl_proto); + if (ctx->bsize) { + server->bsize = ctx->bsize; + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_BSIZE; + } + if (ctx->rsize) { + server->rsize = + nfs_io_size(ctx->rsize, server->nfs_client->cl_proto); + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_RSIZE; + } + if (ctx->wsize) { + server->wsize = + nfs_io_size(ctx->wsize, server->nfs_client->cl_proto); + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_WSIZE; + } server->acregmin = ctx->acregmin * HZ; server->acregmax = ctx->acregmax * HZ; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 93c6ce04332b8..3745c59f0af25 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -1780,8 +1780,17 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state, if (nfs_stateid_is_sequential(state, stateid)) break; - if (status) - break; + if (status) { + if (nfs4_stateid_match_other(stateid, &state->open_stateid) && + !nfs4_stateid_is_newer(stateid, &state->open_stateid)) { + trace_nfs4_open_stateid_update_skip(state->inode, + stateid, status); + return; + } else { + break; + } + } + /* Rely on seqids for serialisation with NFSv4.0 */ if (!nfs4_has_session(NFS_SERVER(state->inode)->nfs_client)) break; @@ -3174,18 +3183,6 @@ static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata, if (opendata->o_res.rflags & NFS4_OPEN_RESULT_PRESERVE_UNLINKED) set_bit(NFS_INO_PRESERVE_UNLINKED, &NFS_I(state->inode)->flags); - dentry = opendata->dentry; - if (d_really_is_negative(dentry)) { - struct dentry *alias; - d_drop(dentry); - alias = d_splice_alias(igrab(state->inode), dentry); - /* d_splice_alias() can't fail here - it's a non-directory */ - if (alias) { - dput(ctx->dentry); - ctx->dentry = dentry = alias; - } - } - switch(opendata->o_arg.claim) { default: break; @@ -3196,7 +3193,20 @@ static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata, break; if (opendata->o_res.delegation.type != 0) dir_verifier = nfs_save_change_attribute(dir); - nfs_set_verifier(dentry, dir_verifier); + } + + dentry = opendata->dentry; + nfs_set_verifier(dentry, dir_verifier); + if (d_really_is_negative(dentry)) { + struct dentry *alias; + d_drop(dentry); + alias = d_splice_alias(igrab(state->inode), dentry); + /* d_splice_alias() can't fail here - it's a non-directory */ + if (alias) { + dput(ctx->dentry); + nfs_set_verifier(alias, dir_verifier); + ctx->dentry = dentry = alias; + } } /* Parse layoutget results before we check for access */ @@ -3884,8 +3894,8 @@ int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait) calldata->res.seqid = calldata->arg.seqid; calldata->res.server = server; calldata->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT; - calldata->lr.roc = pnfs_roc(state->inode, - &calldata->lr.arg, &calldata->lr.res, msg.rpc_cred); + calldata->lr.roc = pnfs_roc(state->inode, &calldata->lr.arg, + &calldata->lr.res, msg.rpc_cred, wait); if (calldata->lr.roc) { calldata->arg.lr_args = &calldata->lr.arg; calldata->res.lr_res = &calldata->lr.res; @@ -6936,7 +6946,7 @@ static int _nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, data->inode = nfs_igrab_and_active(inode); if (data->inode || issync) { data->lr.roc = pnfs_roc(inode, &data->lr.arg, &data->lr.res, - cred); + cred, issync); if (data->lr.roc) { data->args.lr_args = &data->lr.arg; data->res.lr_res = &data->lr.res; diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h index 9776d220cec33..6285128e631a5 100644 --- a/fs/nfs/nfs4trace.h +++ b/fs/nfs/nfs4trace.h @@ -1353,6 +1353,7 @@ DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_setattr); DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_delegreturn); DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_open_stateid_update); DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_open_stateid_update_wait); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_open_stateid_update_skip); DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_close_stateid_update_wait); DECLARE_EVENT_CLASS(nfs4_getattr_event, diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index 6ce55e8e6b67c..9f9ce4a565ea6 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -1062,6 +1062,9 @@ DECLARE_EVENT_CLASS(nfs_folio_event_done, DEFINE_NFS_FOLIO_EVENT(nfs_aop_readpage); DEFINE_NFS_FOLIO_EVENT_DONE(nfs_aop_readpage_done); +DEFINE_NFS_FOLIO_EVENT(nfs_writeback_folio_reclaim); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_writeback_folio_reclaim_done); + DEFINE_NFS_FOLIO_EVENT(nfs_writeback_folio); DEFINE_NFS_FOLIO_EVENT_DONE(nfs_writeback_folio_done); diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index a3135b5af7eec..b3cb5ee9d821e 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -463,7 +463,9 @@ pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, }; struct pnfs_layout_segment *lseg, *next; - set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); + if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) + return !list_empty(&lo->plh_segs); + clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) pnfs_clear_lseg_state(lseg, lseg_list); pnfs_clear_layoutreturn_info(lo); @@ -1532,10 +1534,9 @@ static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) PNFS_FL_LAYOUTRETURN_PRIVILEGED); } -bool pnfs_roc(struct inode *ino, - struct nfs4_layoutreturn_args *args, - struct nfs4_layoutreturn_res *res, - const struct cred *cred) +bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, const struct cred *cred, + bool sync) { struct nfs_inode *nfsi = NFS_I(ino); struct nfs_open_context *ctx; @@ -1546,7 +1547,7 @@ bool pnfs_roc(struct inode *ino, nfs4_stateid stateid; enum pnfs_iomode iomode = 0; bool layoutreturn = false, roc = false; - bool skip_read = false; + bool skip_read; if (!nfs_have_layout(ino)) return false; @@ -1559,20 +1560,14 @@ bool pnfs_roc(struct inode *ino, lo = NULL; goto out_noroc; } - pnfs_get_layout_hdr(lo); - if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { - spin_unlock(&ino->i_lock); - rcu_read_unlock(); - wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, - TASK_UNINTERRUPTIBLE); - pnfs_put_layout_hdr(lo); - goto retry; - } /* no roc if we hold a delegation */ + skip_read = false; if (nfs4_check_delegation(ino, FMODE_READ)) { - if (nfs4_check_delegation(ino, FMODE_WRITE)) + if (nfs4_check_delegation(ino, FMODE_WRITE)) { + lo = NULL; goto out_noroc; + } skip_read = true; } @@ -1581,12 +1576,43 @@ bool pnfs_roc(struct inode *ino, if (state == NULL) continue; /* Don't return layout if there is open file state */ - if (state->state & FMODE_WRITE) + if (state->state & FMODE_WRITE) { + lo = NULL; goto out_noroc; + } if (state->state & FMODE_READ) skip_read = true; } + if (skip_read) { + bool writes = false; + + list_for_each_entry(lseg, &lo->plh_segs, pls_list) { + if (lseg->pls_range.iomode != IOMODE_READ) { + writes = true; + break; + } + } + if (!writes) { + lo = NULL; + goto out_noroc; + } + } + + pnfs_get_layout_hdr(lo); + if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { + if (!sync) { + pnfs_set_plh_return_info( + lo, skip_read ? IOMODE_RW : IOMODE_ANY, 0); + goto out_noroc; + } + spin_unlock(&ino->i_lock); + rcu_read_unlock(); + wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, + TASK_UNINTERRUPTIBLE); + pnfs_put_layout_hdr(lo); + goto retry; + } list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { if (skip_read && lseg->pls_range.iomode == IOMODE_READ) @@ -1626,7 +1652,7 @@ bool pnfs_roc(struct inode *ino, out_noroc: spin_unlock(&ino->i_lock); rcu_read_unlock(); - pnfs_layoutcommit_inode(ino, true); + pnfs_layoutcommit_inode(ino, sync); if (roc) { struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; if (ld->prepare_layoutreturn) diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 91ff877185c8a..3db8f13d8fe4e 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -303,10 +303,9 @@ int pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, u32 seq); int pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, struct list_head *lseg_list); -bool pnfs_roc(struct inode *ino, - struct nfs4_layoutreturn_args *args, - struct nfs4_layoutreturn_res *res, - const struct cred *cred); +bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, const struct cred *cred, + bool sync); int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, struct nfs4_layoutreturn_res **respp, int *ret); void pnfs_roc_release(struct nfs4_layoutreturn_args *args, @@ -773,12 +772,10 @@ pnfs_layoutcommit_outstanding(struct inode *inode) return false; } - -static inline bool -pnfs_roc(struct inode *ino, - struct nfs4_layoutreturn_args *args, - struct nfs4_layoutreturn_res *res, - const struct cred *cred) +static inline bool pnfs_roc(struct inode *ino, + struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, + const struct cred *cred, bool sync) { return false; } diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 72dee6f3050e6..57d372db03b93 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -1051,16 +1051,6 @@ int nfs_reconfigure(struct fs_context *fc) sync_filesystem(sb); - /* - * The SB_RDONLY flag has been removed from the superblock during - * mounts to prevent interference between different filesystems. - * Similarly, it is also necessary to ignore the SB_RDONLY flag - * during reconfiguration; otherwise, it may also result in the - * creation of redundant superblocks when mounting a directory with - * different rw and ro flags multiple times. - */ - fc->sb_flags_mask &= ~SB_RDONLY; - /* * Userspace mount programs that send binary options generally send * them populated with default values. We have no way to know which @@ -1101,8 +1091,9 @@ static void nfs_fill_super(struct super_block *sb, struct nfs_fs_context *ctx) sb->s_blocksize = 0; sb->s_xattr = server->nfs_client->cl_nfs_mod->xattr; sb->s_op = server->nfs_client->cl_nfs_mod->sops; - if (ctx->bsize) - sb->s_blocksize = nfs_block_size(ctx->bsize, &sb->s_blocksize_bits); + if (server->bsize) + sb->s_blocksize = + nfs_block_size(server->bsize, &sb->s_blocksize_bits); switch (server->nfs_client->rpc_ops->version) { case 2: @@ -1318,26 +1309,13 @@ int nfs_get_tree_common(struct fs_context *fc) if (IS_ERR(server)) return PTR_ERR(server); - /* - * When NFS_MOUNT_UNSHARED is not set, NFS forces the sharing of a - * superblock among each filesystem that mounts sub-directories - * belonging to a single exported root path. - * To prevent interference between different filesystems, the - * SB_RDONLY flag should be removed from the superblock. - */ if (server->flags & NFS_MOUNT_UNSHARED) compare_super = NULL; - else - fc->sb_flags &= ~SB_RDONLY; /* -o noac implies -o sync */ if (server->flags & NFS_MOUNT_NOAC) fc->sb_flags |= SB_SYNCHRONOUS; - if (ctx->clone_data.sb) - if (ctx->clone_data.sb->s_flags & SB_SYNCHRONOUS) - fc->sb_flags |= SB_SYNCHRONOUS; - /* Get a superblock - note that we may end up sharing one that already exists */ fc->s_fs_info = server; s = sget_fc(fc, compare_super, nfs_set_super); @@ -1361,13 +1339,8 @@ int nfs_get_tree_common(struct fs_context *fc) } if (!s->s_root) { - unsigned bsize = ctx->clone_data.inherited_bsize; /* initial superblock/root creation */ nfs_fill_super(s, ctx); - if (bsize) { - s->s_blocksize_bits = bsize; - s->s_blocksize = 1U << bsize; - } error = nfs_get_cache_cookie(s, ctx); if (error < 0) goto error_splat_super; diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 336c510f37502..bf412455e8edf 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -2024,6 +2024,39 @@ int nfs_wb_folio_cancel(struct inode *inode, struct folio *folio) return ret; } +/** + * nfs_wb_folio_reclaim - Write back all requests on one page + * @inode: pointer to page + * @folio: pointer to folio + * + * Assumes that the folio has been locked by the caller + */ +int nfs_wb_folio_reclaim(struct inode *inode, struct folio *folio) +{ + loff_t range_start = folio_pos(folio); + size_t len = folio_size(folio); + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0, + .range_start = range_start, + .range_end = range_start + len - 1, + .for_sync = 1, + }; + int ret; + + if (folio_test_writeback(folio)) + return -EBUSY; + if (folio_clear_dirty_for_io(folio)) { + trace_nfs_writeback_folio_reclaim(inode, range_start, len); + ret = nfs_writepage_locked(folio, &wbc); + trace_nfs_writeback_folio_reclaim_done(inode, range_start, len, + ret); + return ret; + } + nfs_commit_inode(inode, 0); + return 0; +} + /** * nfs_wb_folio - Write back all requests on one page * @inode: pointer to page diff --git a/fs/nfs_common/common.c b/fs/nfs_common/common.c index af09aed09fd27..0778743ae2c2c 100644 --- a/fs/nfs_common/common.c +++ b/fs/nfs_common/common.c @@ -17,7 +17,6 @@ static const struct { { NFSERR_NOENT, -ENOENT }, { NFSERR_IO, -EIO }, { NFSERR_NXIO, -ENXIO }, -/* { NFSERR_EAGAIN, -EAGAIN }, */ { NFSERR_ACCES, -EACCES }, { NFSERR_EXIST, -EEXIST }, { NFSERR_XDEV, -EXDEV }, diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c index fde5539cf6a69..e9512c34c9dcf 100644 --- a/fs/nfsd/blocklayout.c +++ b/fs/nfsd/blocklayout.c @@ -23,6 +23,7 @@ nfsd4_block_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, { struct nfsd4_layout_seg *seg = &args->lg_seg; struct super_block *sb = inode->i_sb; + u64 length; u32 block_size = i_blocksize(inode); struct pnfs_block_extent *bex; struct iomap iomap; @@ -56,7 +57,8 @@ nfsd4_block_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, goto out_error; } - if (iomap.length < args->lg_minlength) { + length = iomap.offset + iomap.length - seg->offset; + if (length < args->lg_minlength) { dprintk("pnfsd: extent smaller than minlength\n"); goto out_layoutunavailable; } @@ -342,7 +344,8 @@ nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file) struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev; bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY, - nfsd4_scsi_pr_key(clp), 0, true); + nfsd4_scsi_pr_key(clp), + PR_EXCLUSIVE_ACCESS_REG_ONLY, true); } const struct nfsd4_layout_ops scsi_layout_ops = { diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 9d55512d0cc97..18933ca407bef 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -36,19 +36,30 @@ * second map contains a reference to the entry in the first map. */ +static struct workqueue_struct *nfsd_export_wq; + #define EXPKEY_HASHBITS 8 #define EXPKEY_HASHMAX (1 << EXPKEY_HASHBITS) #define EXPKEY_HASHMASK (EXPKEY_HASHMAX -1) -static void expkey_put(struct kref *ref) +static void expkey_release(struct work_struct *work) { - struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); + struct svc_expkey *key = container_of(to_rcu_work(work), + struct svc_expkey, ek_rwork); if (test_bit(CACHE_VALID, &key->h.flags) && !test_bit(CACHE_NEGATIVE, &key->h.flags)) path_put(&key->ek_path); auth_domain_put(key->ek_client); - kfree_rcu(key, ek_rcu); + kfree(key); +} + +static void expkey_put(struct kref *ref) +{ + struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); + + INIT_RCU_WORK(&key->ek_rwork, expkey_release); + queue_rcu_work(nfsd_export_wq, &key->ek_rwork); } static int expkey_upcall(struct cache_detail *cd, struct cache_head *h) @@ -353,11 +364,13 @@ static void export_stats_destroy(struct export_stats *stats) EXP_STATS_COUNTERS_NUM); } -static void svc_export_release(struct rcu_head *rcu_head) +static void svc_export_release(struct work_struct *work) { - struct svc_export *exp = container_of(rcu_head, struct svc_export, - ex_rcu); + struct svc_export *exp = container_of(to_rcu_work(work), + struct svc_export, ex_rwork); + path_put(&exp->ex_path); + auth_domain_put(exp->ex_client); nfsd4_fslocs_free(&exp->ex_fslocs); export_stats_destroy(exp->ex_stats); kfree(exp->ex_stats); @@ -369,9 +382,8 @@ static void svc_export_put(struct kref *ref) { struct svc_export *exp = container_of(ref, struct svc_export, h.ref); - path_put(&exp->ex_path); - auth_domain_put(exp->ex_client); - call_rcu(&exp->ex_rcu, svc_export_release); + INIT_RCU_WORK(&exp->ex_rwork, svc_export_release); + queue_rcu_work(nfsd_export_wq, &exp->ex_rwork); } static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h) @@ -1024,7 +1036,7 @@ exp_rootfh(struct net *net, struct auth_domain *clp, char *name, { struct svc_export *exp; struct path path; - struct inode *inode; + struct inode *inode __maybe_unused; struct svc_fh fh; int err; struct nfsd_net *nn = net_generic(net, nfsd_net_id); @@ -1478,6 +1490,36 @@ const struct seq_operations nfs_exports_op = { .show = e_show, }; +/** + * nfsd_export_wq_init - allocate the export release workqueue + * + * Called once at module load. The workqueue runs deferred svc_export and + * svc_expkey release work scheduled by queue_rcu_work() in the cache put + * callbacks. + * + * Return values: + * %0: workqueue allocated + * %-ENOMEM: allocation failed + */ +int nfsd_export_wq_init(void) +{ + nfsd_export_wq = alloc_workqueue("nfsd_export", WQ_UNBOUND, 0); + if (!nfsd_export_wq) + return -ENOMEM; + return 0; +} + +/** + * nfsd_export_wq_shutdown - drain and free the export release workqueue + * + * Called once at module unload. Per-namespace teardown in + * nfsd_export_shutdown() has already drained all deferred work. + */ +void nfsd_export_wq_shutdown(void) +{ + destroy_workqueue(nfsd_export_wq); +} + /* * Initialize the exports module. */ @@ -1539,6 +1581,9 @@ nfsd_export_shutdown(struct net *net) cache_unregister_net(nn->svc_expkey_cache, net); cache_unregister_net(nn->svc_export_cache, net); + /* Drain deferred export and expkey release work. */ + rcu_barrier(); + flush_workqueue(nfsd_export_wq); cache_destroy_net(nn->svc_expkey_cache, net); cache_destroy_net(nn->svc_export_cache, net); svcauth_unix_purge(net); diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index d2b09cd761453..b053993745749 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -75,7 +76,7 @@ struct svc_export { u32 ex_layout_types; struct nfsd4_deviceid_map *ex_devid_map; struct cache_detail *cd; - struct rcu_head ex_rcu; + struct rcu_work ex_rwork; unsigned long ex_xprtsec_modes; struct export_stats *ex_stats; }; @@ -92,7 +93,7 @@ struct svc_expkey { u32 ek_fsid[6]; struct path ek_path; - struct rcu_head ek_rcu; + struct rcu_work ek_rwork; }; #define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC)) @@ -110,6 +111,8 @@ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, /* * Function declarations */ +int nfsd_export_wq_init(void); +void nfsd_export_wq_shutdown(void); int nfsd_export_init(struct net *); void nfsd_export_shutdown(struct net *); void nfsd_export_flush(struct net *); diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 3e2d0fde80a7c..fe8338735e7cc 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -66,6 +66,8 @@ struct nfsd_net { struct lock_manager nfsd4_manager; bool grace_ended; + bool grace_end_forced; + bool client_tracking_active; time64_t boot_time; struct dentry *nfsd_client_dir; diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 5fb202acb0fd0..0ac538c761800 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -45,7 +45,7 @@ static __be32 nfsacld_proc_getacl(struct svc_rqst *rqstp) inode = d_inode(fh->fh_dentry); if (argp->mask & ~NFS_ACL_MASK) { - resp->status = nfserr_inval; + resp->status = nfserr_io; goto out; } resp->mask = argp->mask; diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index 8cca1329f3485..c319c31b0f647 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -643,34 +643,74 @@ static __be32 encode_name_from_id(struct xdr_stream *xdr, return idmap_id_to_name(xdr, rqstp, type, id); } -__be32 -nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kuid_t *uid) +/** + * nfsd_map_name_to_uid - Map user@domain to local UID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @uid: OUT: mapped local UID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kuid_t *uid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; status = do_name_to_id(rqstp, IDMAP_TYPE_USER, name, namelen, &id); + if (status) + return status; *uid = make_kuid(nfsd_user_namespace(rqstp), id); if (!uid_valid(*uid)) status = nfserr_badowner; return status; } -__be32 -nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kgid_t *gid) +/** + * nfsd_map_name_to_gid - Map user@domain to local GID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @gid: OUT: mapped local GID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kgid_t *gid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; status = do_name_to_id(rqstp, IDMAP_TYPE_GROUP, name, namelen, &id); + if (status) + return status; *gid = make_kgid(nfsd_user_namespace(rqstp), id); if (!gid_valid(*gid)) status = nfserr_badowner; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 7f7e6bb23a90d..8dada7ef97cb1 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1506,7 +1506,7 @@ static __be32 nfsd4_ssc_setup_dul(struct nfsd_net *nn, char *ipaddr, (schedule_timeout(20*HZ) == 0)) { finish_wait(&nn->nfsd_ssc_waitq, &wait); kfree(work); - return nfserr_eagain; + return nfserr_jukebox; } finish_wait(&nn->nfsd_ssc_waitq, &wait); goto try_again; @@ -2995,8 +2995,6 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) BUG_ON(cstate->replay_owner); out: cstate->status = status; - /* Reset deferral mechanism for RPC deferrals */ - set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); return rpc_success; } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 8a69605002174..c5dba49c90356 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -84,7 +84,7 @@ static u64 current_sessionid = 1; /* forward declarations */ static bool check_for_locks(struct nfs4_file *fp, struct nfs4_lockowner *lowner); static void nfs4_free_ol_stateid(struct nfs4_stid *stid); -void nfsd4_end_grace(struct nfsd_net *nn); +static void nfsd4_end_grace(struct nfsd_net *nn); static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps); static void nfsd4_file_hash_remove(struct nfs4_file *fi); static void deleg_reaper(struct nfsd_net *nn); @@ -1218,13 +1218,15 @@ static void put_deleg_file(struct nfs4_file *fp) if (nf) nfsd_file_put(nf); - if (rnf) + if (rnf) { + nfsd_file_put(rnf); nfs4_file_put_access(fp, NFS4_SHARE_ACCESS_READ); + } } static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct file *f) { - struct iattr ia = { .ia_valid = ATTR_ATIME | ATTR_CTIME | ATTR_MTIME }; + struct iattr ia = { .ia_valid = ATTR_ATIME | ATTR_CTIME | ATTR_MTIME | ATTR_DELEG }; struct inode *inode = file_inode(f); int ret; @@ -1757,7 +1759,7 @@ static struct nfs4_stid *find_one_sb_stid(struct nfs4_client *clp, /** * nfsd4_revoke_states - revoke all nfsv4 states associated with given filesystem - * @net: used to identify instance of nfsd (there is one per net namespace) + * @nn: used to identify instance of nfsd (there is one per net namespace) * @sb: super_block used to identify target filesystem * * All nfs4 states (open, lock, delegation, layout) held by the server instance @@ -1769,16 +1771,15 @@ static struct nfs4_stid *find_one_sb_stid(struct nfs4_client *clp, * The clients which own the states will subsequently being notified that the * states have been "admin-revoked". */ -void nfsd4_revoke_states(struct net *net, struct super_block *sb) +void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) { - struct nfsd_net *nn = net_generic(net, nfsd_net_id); unsigned int idhashval; unsigned int sc_types; sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; spin_lock(&nn->client_lock); - for (idhashval = 0; idhashval < CLIENT_HASH_MASK; idhashval++) { + for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) { struct list_head *head = &nn->conf_id_hashtbl[idhashval]; struct nfs4_client *clp; retry: @@ -3097,8 +3098,10 @@ static int client_states_open(struct inode *inode, struct file *file) return -ENXIO; ret = seq_open(file, &states_seq_ops); - if (ret) + if (ret) { + drop_client(clp); return ret; + } s = file->private_data; s->private = clp; return 0; @@ -6253,10 +6256,14 @@ nfsd4_add_rdaccess_to_wrdeleg(struct svc_rqst *rqstp, struct nfsd4_open *open, fp = stp->st_stid.sc_file; spin_lock(&fp->fi_lock); __nfs4_file_get_access(fp, NFS4_SHARE_ACCESS_READ); - fp = stp->st_stid.sc_file; - fp->fi_fds[O_RDONLY] = nf; - fp->fi_rdeleg_file = nf; + if (!fp->fi_fds[O_RDONLY]) { + fp->fi_fds[O_RDONLY] = nf; + nf = NULL; + } + fp->fi_rdeleg_file = nfsd_file_get(fp->fi_fds[O_RDONLY]); spin_unlock(&fp->fi_lock); + if (nf) + nfsd_file_put(nf); } return true; } @@ -6349,7 +6356,8 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, dp->dl_ctime = stat.ctime; dp->dl_mtime = stat.mtime; spin_lock(&f->f_lock); - f->f_mode |= FMODE_NOCMTIME; + if (deleg_ts) + f->f_mode |= FMODE_NOCMTIME; spin_unlock(&f->f_lock); trace_nfsd_deleg_write(&dp->dl_stid.sc_stateid); } else { @@ -6589,7 +6597,7 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, return nfs_ok; } -void +static void nfsd4_end_grace(struct nfsd_net *nn) { /* do nothing if grace period already ended */ @@ -6622,6 +6630,33 @@ nfsd4_end_grace(struct nfsd_net *nn) */ } +/** + * nfsd4_force_end_grace - forcibly end the NFSv4 grace period + * @nn: network namespace for the server instance to be updated + * + * Forces bypass of normal grace period completion, then schedules + * the laundromat to end the grace period immediately. Does not wait + * for the grace period to fully terminate before returning. + * + * Return values: + * %true: Grace termination schedule + * %false: No action was taken + */ +bool nfsd4_force_end_grace(struct nfsd_net *nn) +{ + if (!nn->client_tracking_ops) + return false; + spin_lock(&nn->client_lock); + if (nn->grace_ended || !nn->client_tracking_active) { + spin_unlock(&nn->client_lock); + return false; + } + WRITE_ONCE(nn->grace_end_forced, true); + mod_delayed_work(laundry_wq, &nn->laundromat_work, 0); + spin_unlock(&nn->client_lock); + return true; +} + /* * If we've waited a lease period but there are still clients trying to * reclaim, wait a little longer to give them a chance to finish. @@ -6631,6 +6666,8 @@ static bool clients_still_reclaiming(struct nfsd_net *nn) time64_t double_grace_period_end = nn->boot_time + 2 * nn->nfsd4_lease; + if (READ_ONCE(nn->grace_end_forced)) + return false; if (nn->track_reclaim_completes && atomic_read(&nn->nr_reclaim_complete) == nn->reclaim_str_hashtbl_size) @@ -8934,6 +8971,8 @@ static int nfs4_state_create_net(struct net *net) nn->unconf_name_tree = RB_ROOT; nn->boot_time = ktime_get_real_seconds(); nn->grace_ended = false; + nn->grace_end_forced = false; + nn->client_tracking_active = false; nn->nfsd4_manager.block_opens = true; INIT_LIST_HEAD(&nn->nfsd4_manager.list); INIT_LIST_HEAD(&nn->client_lru); @@ -9014,6 +9053,10 @@ nfs4_state_start_net(struct net *net) return ret; locks_start_grace(net, &nn->nfsd4_manager); nfsd4_client_tracking_init(net); + /* safe for laundromat to run now */ + spin_lock(&nn->client_lock); + nn->client_tracking_active = true; + spin_unlock(&nn->client_lock); if (nn->track_reclaim_completes && nn->reclaim_str_hashtbl_size == 0) goto skip_grace; printk(KERN_INFO "NFSD: starting %lld-second grace period (net %x)\n", @@ -9062,6 +9105,9 @@ nfs4_state_shutdown_net(struct net *net) shrinker_free(nn->nfsd_client_shrinker); cancel_work_sync(&nn->nfsd_shrinker_work); + spin_lock(&nn->client_lock); + nn->client_tracking_active = false; + spin_unlock(&nn->client_lock); cancel_delayed_work_sync(&nn->laundromat_work); locks_end_grace(&nn->nfsd4_manager); diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 67bb9c0b9fcb1..b1b52c816ebdf 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -3375,6 +3375,11 @@ static __be32 nfsd4_encode_fattr4_suppattr_exclcreat(struct xdr_stream *xdr, u32 supp[3]; memcpy(supp, nfsd_suppattrs[resp->cstate.minorversion], sizeof(supp)); + if (!IS_POSIXACL(d_inode(args->dentry))) + supp[0] &= ~FATTR4_WORD0_ACL; + if (!args->contextsupport) + supp[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + supp[0] &= NFSD_SUPPATTR_EXCLCREAT_WORD0; supp[1] &= NFSD_SUPPATTR_EXCLCREAT_WORD1; supp[2] &= NFSD_SUPPATTR_EXCLCREAT_WORD2; @@ -5929,9 +5934,14 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op) int len = xdr->buf->len - (op_status_offset + XDR_UNIT); so->so_replay.rp_status = op->status; - so->so_replay.rp_buflen = len; - read_bytes_from_xdr_buf(xdr->buf, op_status_offset + XDR_UNIT, + if (len <= NFSD4_REPLAY_ISIZE) { + so->so_replay.rp_buflen = len; + read_bytes_from_xdr_buf(xdr->buf, + op_status_offset + XDR_UNIT, so->so_replay.rp_buf, len); + } else { + so->so_replay.rp_buflen = 0; + } } status: op->status = nfsd4_map_status(op->status, @@ -5996,6 +6006,22 @@ nfs4svc_decode_compoundargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) args->ops = args->iops; args->rqstp = rqstp; + /* + * NFSv4 operation decoders can invoke svc cache lookups + * that trigger svc_defer() when RQ_USEDEFERRAL is set, + * setting RQ_DROPME. This creates two problems: + * + * 1. Non-idempotency: Compounds make it too hard to avoid + * problems if a request is deferred and replayed. + * + * 2. Session slot leakage (NFSv4.1+): If RQ_DROPME is set + * during decode but SEQUENCE executes successfully, the + * session slot will be marked INUSE. The request is then + * dropped before encoding, so the slot is never released, + * rendering it permanently unusable by the client. + */ + clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); + return nfsd4_decode_compound(args); } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 2b79129703d54..4ed19d81577b6 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -149,9 +149,19 @@ static int exports_net_open(struct net *net, struct file *file) seq = file->private_data; seq->private = nn->svc_export_cache; + get_net(net); return 0; } +static int exports_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct cache_detail *cd = seq->private; + + put_net(cd->net); + return seq_release(inode, file); +} + static int exports_nfsd_open(struct inode *inode, struct file *file) { return exports_net_open(inode->i_sb->s_fs_info, file); @@ -161,7 +171,7 @@ static const struct file_operations exports_nfsd_operations = { .open = exports_nfsd_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = exports_release, }; static int export_features_show(struct seq_file *m, void *v) @@ -259,6 +269,7 @@ static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size) struct path path; char *fo_path; int error; + struct nfsd_net *nn; /* sanity check */ if (size == 0) @@ -285,7 +296,13 @@ static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size) * 3. Is that directory the root of an exported file system? */ error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); - nfsd4_revoke_states(netns(file), path.dentry->d_sb); + mutex_lock(&nfsd_mutex); + nn = net_generic(netns(file), nfsd_net_id); + if (nn->nfsd_serv) + nfsd4_revoke_states(nn, path.dentry->d_sb); + else + error = -EINVAL; + mutex_unlock(&nfsd_mutex); path_put(&path); return error; @@ -1082,10 +1099,9 @@ static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size) case 'Y': case 'y': case '1': - if (!nn->nfsd_serv) + if (!nfsd4_force_end_grace(nn)) return -EBUSY; trace_nfsd_end_grace(netns(file)); - nfsd4_end_grace(nn); break; default: return -EINVAL; @@ -1369,7 +1385,7 @@ static const struct proc_ops exports_proc_ops = { .proc_open = exports_proc_open, .proc_read = seq_read, .proc_lseek = seq_lseek, - .proc_release = seq_release, + .proc_release = exports_release, }; static int create_proc_exports_entry(void) @@ -1636,7 +1652,7 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) scope = nla_data(attr); } - ret = nfsd_svc(nrpools, nthreads, net, get_current_cred(), scope); + ret = nfsd_svc(nrpools, nthreads, net, current_cred(), scope); if (ret > 0) ret = 0; out_unlock: @@ -1987,7 +2003,7 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) } ret = svc_xprt_create_from_sa(serv, xcl_name, net, sa, 0, - get_current_cred()); + current_cred()); /* always save the latest error */ if (ret < 0) err = ret; @@ -2246,9 +2262,12 @@ static int __init init_nfsd(void) if (retval) goto out_free_pnfs; nfsd_lockd_init(); /* lockd->nfsd callbacks */ + retval = nfsd_export_wq_init(); + if (retval) + goto out_free_lockd; retval = register_pernet_subsys(&nfsd_net_ops); if (retval < 0) - goto out_free_lockd; + goto out_free_export_wq; retval = register_cld_notifier(); if (retval) goto out_free_subsys; @@ -2277,6 +2296,8 @@ static int __init init_nfsd(void) unregister_cld_notifier(); out_free_subsys: unregister_pernet_subsys(&nfsd_net_ops); +out_free_export_wq: + nfsd_export_wq_shutdown(); out_free_lockd: nfsd_lockd_shutdown(); nfsd_drc_slab_free(); @@ -2297,6 +2318,7 @@ static void __exit exit_nfsd(void) nfsd4_destroy_laundry_wq(); unregister_cld_notifier(); unregister_pernet_subsys(&nfsd_net_ops); + nfsd_export_wq_shutdown(); nfsd_drc_slab_free(); nfsd_lockd_shutdown(); nfsd4_free_slabs(); diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index b752433c3c2c4..8cecccdaf9ee5 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -232,7 +232,6 @@ void nfsd_lockd_shutdown(void); #define nfserr_noent cpu_to_be32(NFSERR_NOENT) #define nfserr_io cpu_to_be32(NFSERR_IO) #define nfserr_nxio cpu_to_be32(NFSERR_NXIO) -#define nfserr_eagain cpu_to_be32(NFSERR_EAGAIN) #define nfserr_acces cpu_to_be32(NFSERR_ACCES) #define nfserr_exist cpu_to_be32(NFSERR_EXIST) #define nfserr_xdev cpu_to_be32(NFSERR_XDEV) @@ -547,8 +546,14 @@ static inline bool nfsd_attrs_supported(u32 minorversion, const u32 *bmval) #define NFSD_SUPPATTR_EXCLCREAT_WORD1 \ (NFSD_WRITEABLE_ATTRS_WORD1 & \ ~(FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET)) +/* + * The FATTR4_WORD2_TIME_DELEG attributes are not to be allowed for + * OPEN(create) with EXCLUSIVE4_1. It doesn't make sense to set a + * delegated timestamp on a new file. + */ #define NFSD_SUPPATTR_EXCLCREAT_WORD2 \ - NFSD_WRITEABLE_ATTRS_WORD2 + (NFSD_WRITEABLE_ATTRS_WORD2 & \ + ~(FATTR4_WORD2_TIME_DELEG_ACCESS | FATTR4_WORD2_TIME_DELEG_MODIFY)) extern int nfsd4_is_junction(struct dentry *dentry); extern int register_cld_notifier(void); diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 8f71f5748c75b..906a672578900 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -33,7 +33,7 @@ static __be32 nfsd_map_status(__be32 status) break; case nfserr_symlink: case nfserr_wrong_type: - status = nfserr_inval; + status = nfserr_io; break; } return status; diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 7057ddd7a0a87..fcb47f344e32f 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -424,26 +424,26 @@ static void nfsd_shutdown_net(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - if (!nn->nfsd_net_up) - return; - - percpu_ref_kill_and_confirm(&nn->nfsd_net_ref, nfsd_net_done); - wait_for_completion(&nn->nfsd_net_confirm_done); - - nfsd_export_flush(net); - nfs4_state_shutdown_net(net); - nfsd_reply_cache_shutdown(nn); - nfsd_file_cache_shutdown_net(net); - if (nn->lockd_up) { - lockd_down(net); - nn->lockd_up = false; + if (nn->nfsd_net_up) { + percpu_ref_kill_and_confirm(&nn->nfsd_net_ref, nfsd_net_done); + wait_for_completion(&nn->nfsd_net_confirm_done); + + nfsd_export_flush(net); + nfs4_state_shutdown_net(net); + nfsd_reply_cache_shutdown(nn); + nfsd_file_cache_shutdown_net(net); + if (nn->lockd_up) { + lockd_down(net); + nn->lockd_up = false; + } + wait_for_completion(&nn->nfsd_net_free_done); } - wait_for_completion(&nn->nfsd_net_free_done); percpu_ref_exit(&nn->nfsd_net_ref); + if (nn->nfsd_net_up) + nfsd_shutdown_generic(); nn->nfsd_net_up = false; - nfsd_shutdown_generic(); } static DEFINE_SPINLOCK(nfsd_notifier_lock); @@ -633,12 +633,15 @@ int nfsd_create_serv(struct net *net) serv = svc_create_pooled(nfsd_programs, ARRAY_SIZE(nfsd_programs), &nn->nfsd_svcstats, nfsd_max_blksize, nfsd); - if (serv == NULL) + if (serv == NULL) { + percpu_ref_exit(&nn->nfsd_net_ref); return -ENOMEM; + } error = svc_bind(serv, net); if (error < 0) { svc_destroy(&serv); + percpu_ref_exit(&nn->nfsd_net_ref); return error; } spin_lock(&nfsd_notifier_lock); diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 1e736f4024263..aaf513ed91045 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -541,11 +541,18 @@ struct nfs4_client_reclaim { struct xdr_netobj cr_princhash; }; -/* A reasonable value for REPLAY_ISIZE was estimated as follows: - * The OPEN response, typically the largest, requires - * 4(status) + 8(stateid) + 20(changeinfo) + 4(rflags) + 8(verifier) + - * 4(deleg. type) + 8(deleg. stateid) + 4(deleg. recall flag) + - * 20(deleg. space limit) + ~32(deleg. ace) = 112 bytes +/* + * REPLAY_ISIZE is sized for an OPEN response with delegation: + * 4(status) + 8(stateid) + 20(changeinfo) + 4(rflags) + + * 8(verifier) + 4(deleg. type) + 8(deleg. stateid) + + * 4(deleg. recall flag) + 20(deleg. space limit) + + * ~32(deleg. ace) = 112 bytes + * + * Some responses can exceed this. A LOCK denial includes the conflicting + * lock owner, which can be up to 1024 bytes (NFS4_OPAQUE_LIMIT). Responses + * larger than REPLAY_ISIZE are not cached in rp_ibuf; only rp_status is + * saved. Enlarging this constant increases the size of every + * nfs4_stateowner. */ #define NFSD4_REPLAY_ISIZE 112 @@ -841,15 +848,15 @@ static inline void get_nfs4_file(struct nfs4_file *fi) struct nfsd_file *find_any_file(struct nfs4_file *f); #ifdef CONFIG_NFSD_V4 -void nfsd4_revoke_states(struct net *net, struct super_block *sb); +void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb); #else -static inline void nfsd4_revoke_states(struct net *net, struct super_block *sb) +static inline void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) { } #endif /* grace period management */ -void nfsd4_end_grace(struct nfsd_net *nn); +bool nfsd4_force_end_grace(struct nfsd_net *nn); /* nfs4recover operations */ extern int nfsd4_client_tracking_init(struct net *net); diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 9cb20d4aeab15..e32a5fcd6ac85 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1228,8 +1228,18 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, stable = NFS_UNSTABLE; init_sync_kiocb(&kiocb, file); kiocb.ki_pos = offset; - if (stable && !fhp->fh_use_wgather) - kiocb.ki_flags |= IOCB_DSYNC; + if (likely(!fhp->fh_use_wgather)) { + switch (stable) { + case NFS_FILE_SYNC: + /* persist data and timestamps */ + kiocb.ki_flags |= IOCB_DSYNC | IOCB_SYNC; + break; + case NFS_DATA_SYNC: + /* persist data only */ + kiocb.ki_flags |= IOCB_DSYNC; + break; + } + } nvecs = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, payload); iov_iter_bvec(&iter, ITER_SOURCE, rqstp->rq_bvec, nvecs, *cnt); @@ -2673,8 +2683,8 @@ nfsd_permission(struct svc_cred *cred, struct svc_export *exp, /* Allow read access to binaries even when mode 111 */ if (err == -EACCES && S_ISREG(inode->i_mode) && - (acc == (NFSD_MAY_READ | NFSD_MAY_OWNER_OVERRIDE) || - acc == (NFSD_MAY_READ | NFSD_MAY_READ_IF_EXEC))) + (((acc & NFSD_MAY_MASK) == NFSD_MAY_READ) && + (acc & (NFSD_MAY_OWNER_OVERRIDE | NFSD_MAY_READ_IF_EXEC)))) err = inode_permission(&nop_mnt_idmap, inode, MAY_EXEC); return err? nfserrno(err) : 0; diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index 0c0292611c6de..9652c1de53ec1 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -67,7 +67,8 @@ static inline bool nfsd_attrs_valid(struct nfsd_attrs *attrs) struct iattr *iap = attrs->na_iattr; return (iap->ia_valid || (attrs->na_seclabel && - attrs->na_seclabel->len)); + attrs->na_seclabel->len) || + attrs->na_pacl || attrs->na_dpacl); } __be32 nfserrno (int errno); diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c index c664daba56ae6..5e2e6107c2a81 100644 --- a/fs/nilfs2/dat.c +++ b/fs/nilfs2/dat.c @@ -524,6 +524,9 @@ int nilfs_dat_read(struct super_block *sb, size_t entry_size, if (err) goto failed; + err = nilfs_attach_btree_node_cache(dat); + if (err) + goto failed; err = nilfs_read_inode_common(dat, raw_inode); if (err) goto failed; diff --git a/fs/nilfs2/sufile.c b/fs/nilfs2/sufile.c index 330f269abedf5..71ee217eb072c 100644 --- a/fs/nilfs2/sufile.c +++ b/fs/nilfs2/sufile.c @@ -1093,6 +1093,9 @@ int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range) else end_block = start_block + len - 1; + if (end_block < nilfs->ns_first_data_block) + goto out; + segnum = nilfs_get_segnum_of_block(nilfs, start_block); segnum_end = nilfs_get_segnum_of_block(nilfs, end_block); @@ -1191,6 +1194,7 @@ int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range) out_sem: up_read(&NILFS_MDT(sufile)->mi_sem); +out: range->len = ndiscarded << nilfs->ns_blocksize_bits; return ret; } diff --git a/fs/nls/nls_base.c b/fs/nls/nls_base.c index 18d597e49a194..a5c3a9f1b8dc5 100644 --- a/fs/nls/nls_base.c +++ b/fs/nls/nls_base.c @@ -67,19 +67,22 @@ int utf8_to_utf32(const u8 *s, int inlen, unicode_t *pu) l &= t->lmask; if (l < t->lval || l > UNICODE_MAX || (l & SURROGATE_MASK) == SURROGATE_PAIR) - return -1; + return -EILSEQ; + *pu = (unicode_t) l; return nc; } if (inlen <= nc) - return -1; + return -EOVERFLOW; + s++; c = (*s ^ 0x80) & 0xFF; if (c & 0xC0) - return -1; + return -EILSEQ; + l = (l << 6) | c; } - return -1; + return -EILSEQ; } EXPORT_SYMBOL(utf8_to_utf32); @@ -94,7 +97,7 @@ int utf32_to_utf8(unicode_t u, u8 *s, int maxout) l = u; if (l > UNICODE_MAX || (l & SURROGATE_MASK) == SURROGATE_PAIR) - return -1; + return -EILSEQ; nc = 0; for (t = utf8_table; t->cmask && maxout; t++, maxout--) { @@ -110,7 +113,7 @@ int utf32_to_utf8(unicode_t u, u8 *s, int maxout) return nc; } } - return -1; + return -EOVERFLOW; } EXPORT_SYMBOL(utf32_to_utf8); @@ -217,8 +220,16 @@ int utf16s_to_utf8s(const wchar_t *pwcs, int inlen, enum utf16_endian endian, inlen--; } size = utf32_to_utf8(u, op, maxout); - if (size == -1) { - /* Ignore character and move on */ + if (size < 0) { + if (size == -EILSEQ) { + /* Ignore character and move on */ + continue; + } + /* + * Stop filling the buffer with data once a character + * does not fit anymore. + */ + break; } else { op += size; maxout -= size; diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 46bfc543f9467..63dd44931989d 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -270,8 +270,15 @@ int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data, /* * Include parent/name in notification either if some notification * groups require parent info or the parent is interested in this event. + * The parent interest in ACCESS/MODIFY events does not apply to special + * files, where read/write are not on the filesystem of the parent and + * events can provide an undesirable side-channel for information + * exfiltration. */ - parent_interested = mask & p_mask & ALL_FSNOTIFY_EVENTS; + parent_interested = mask & p_mask & ALL_FSNOTIFY_EVENTS && + !(data_type == FSNOTIFY_EVENT_PATH && + d_is_special(dentry) && + (mask & (FS_ACCESS | FS_MODIFY))); if (parent_needed || parent_interested) { /* When notifying parent, child should be passed as data */ WARN_ON_ONCE(inode != fsnotify_data_inode(data, data_type)); diff --git a/fs/nsfs.c b/fs/nsfs.c index 79b026a36fb62..f22c2a636e8f3 100644 --- a/fs/nsfs.c +++ b/fs/nsfs.c @@ -194,6 +194,17 @@ static bool nsfs_ioctl_valid(unsigned int cmd) return false; } +static bool may_use_nsfs_ioctl(unsigned int cmd) +{ + switch (_IOC_NR(cmd)) { + case _IOC_NR(NS_MNT_GET_NEXT): + fallthrough; + case _IOC_NR(NS_MNT_GET_PREV): + return may_see_all_namespaces(); + } + return true; +} + static long ns_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -209,6 +220,8 @@ static long ns_ioctl(struct file *filp, unsigned int ioctl, if (!nsfs_ioctl_valid(ioctl)) return -ENOIOCTLCMD; + if (!may_use_nsfs_ioctl(ioctl)) + return -EPERM; ns = get_proc_ns(file_inode(filp)); switch (ioctl) { diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index eced9013a8818..5a7675112e7b8 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -448,8 +448,10 @@ int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, is_ext = is_attr_ext(attr_b); align = sbi->cluster_size; - if (is_ext) + if (is_ext) { align <<= attr_b->nres.c_unit; + keep_prealloc = false; + } old_valid = le64_to_cpu(attr_b->nres.valid_size); old_size = le64_to_cpu(attr_b->nres.data_size); @@ -1354,19 +1356,28 @@ int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, CLST vcn; CLST vcn_last = (to - 1) >> cluster_bits; CLST lcn, clen; - int err; + int err = 0; + int retry = 0; for (vcn = from >> cluster_bits; vcn <= vcn_last; vcn += clen) { if (!run_lookup_entry(run, vcn, &lcn, &clen, NULL)) { + if (retry != 0) { /* Next run_lookup_entry(vcn) also failed. */ + err = -EINVAL; + break; + } err = attr_load_runs_vcn(ni, type, name, name_len, run, vcn); if (err) - return err; + break; + clen = 0; /* Next run_lookup_entry(vcn) must be success. */ + retry++; } + else + retry = 0; } - return 0; + return err; } #ifdef CONFIG_NTFS3_LZX_XPRESS diff --git a/fs/ntfs3/attrlist.c b/fs/ntfs3/attrlist.c index a4d74bed74fab..098bd7e8c3d64 100644 --- a/fs/ntfs3/attrlist.c +++ b/fs/ntfs3/attrlist.c @@ -52,6 +52,11 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) if (!attr->non_res) { lsize = le32_to_cpu(attr->res.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } + /* attr is resident: lsize < record_size (1K or 4K) */ le = kvmalloc(al_aligned(lsize), GFP_KERNEL); if (!le) { @@ -66,6 +71,10 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) u16 run_off = le16_to_cpu(attr->nres.run_off); lsize = le64_to_cpu(attr->nres.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } run_init(&ni->attr_list.run); diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index 1b5c865a0339a..460df046482c7 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -424,8 +424,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) if (!dir_emit_dots(file, ctx)) return 0; - /* Allocate PATH_MAX bytes. */ - name = __getname(); + name = kmalloc(PATH_MAX, GFP_KERNEL); if (!name) return -ENOMEM; @@ -503,7 +502,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) out: - __putname(name); + kfree(name); put_indx_node(node); if (err == 1) { diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 4c90ec2fa2eae..cd7aaeef45fe9 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -930,7 +930,7 @@ static int ntfs_get_frame_pages(struct address_space *mapping, pgoff_t index, folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, - gfp_mask); + gfp_mask | __GFP_ZERO); if (IS_ERR(folio)) { while (npages--) { folio = page_folio(pages[npages]); @@ -1012,8 +1012,12 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) goto out; if (lcn == SPARSE_LCN) { - ni->i_valid = valid = - frame_vbo + ((u64)clen << sbi->cluster_bits); + valid = frame_vbo + ((u64)clen << sbi->cluster_bits); + if (ni->i_valid == valid) { + err = -EINVAL; + goto out; + } + ni->i_valid = valid; continue; } @@ -1375,6 +1379,18 @@ static ssize_t ntfs_file_splice_write(struct pipe_inode_info *pipe, return iter_file_splice_write(pipe, file, ppos, len, flags); } +/* + * ntfs_file_fsync - file_operations::fsync + */ +static int ntfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file_inode(file); + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + return generic_file_fsync(file, start, end, datasync); +} + // clang-format off const struct inode_operations ntfs_file_inode_operations = { .getattr = ntfs_getattr, @@ -1397,7 +1413,7 @@ const struct file_operations ntfs_file_operations = { .splice_write = ntfs_file_splice_write, .mmap_prepare = ntfs_file_mmap_prepare, .open = ntfs_file_open, - .fsync = generic_file_fsync, + .fsync = ntfs_file_fsync, .fallocate = ntfs_fallocate, .release = ntfs_file_release, }; diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 8f9fe1d7a6908..295f6936a3e58 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -325,8 +325,10 @@ bool ni_add_subrecord(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi) mi_get_ref(&ni->mi, &m->mrec->parent_ref); - ni_add_mi(ni, m); - *mi = m; + *mi = ni_ins_mi(ni, &ni->mi_tree, m->rno, &m->node); + if (*mi != m) + mi_put(m); + return true; } @@ -1015,9 +1017,9 @@ static int ni_ins_attr_ext(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le, out2: ni_remove_mi(ni, mi); - mi_put(mi); out1: + mi_put(mi); ntfs_mark_rec_free(sbi, rno, is_mft); out: @@ -2020,6 +2022,29 @@ int ni_fiemap(struct ntfs_inode *ni, struct fiemap_extent_info *fieinfo, return err; } +static struct page *ntfs_lock_new_page(struct address_space *mapping, + pgoff_t index, gfp_t gfp) +{ + struct folio *folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + struct page *page; + + if (IS_ERR(folio)) + return ERR_CAST(folio); + + if (!folio_test_uptodate(folio)) + return folio_file_page(folio, index); + + /* Use a temporary page to avoid data corruption */ + folio_unlock(folio); + folio_put(folio); + page = alloc_page(gfp); + if (!page) + return ERR_PTR(-ENOMEM); + __SetPageLocked(page); + return page; +} + /* * ni_readpage_cmpr * @@ -2074,9 +2099,9 @@ int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio) if (i == idx) continue; - pg = find_or_create_page(mapping, index, gfp_mask); - if (!pg) { - err = -ENOMEM; + pg = ntfs_lock_new_page(mapping, index, gfp_mask); + if (IS_ERR(pg)) { + err = PTR_ERR(pg); goto out1; } pages[i] = pg; @@ -2175,13 +2200,13 @@ int ni_decompress_file(struct ntfs_inode *ni) for (i = 0; i < pages_per_frame; i++, index++) { struct page *pg; - pg = find_or_create_page(mapping, index, gfp_mask); - if (!pg) { + pg = ntfs_lock_new_page(mapping, index, gfp_mask); + if (IS_ERR(pg)) { while (i--) { unlock_page(pages[i]); put_page(pages[i]); } - err = -ENOMEM; + err = PTR_ERR(pg); goto out; } pages[i] = pg; diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 38934e6978ece..90d1de22fee68 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -2789,13 +2789,14 @@ static inline bool check_file_record(const struct MFT_REC *rec, u16 fn = le16_to_cpu(rec->rhdr.fix_num); u16 ao = le16_to_cpu(rec->attr_off); u32 rs = sbi->record_size; + u32 used = le32_to_cpu(rec->used); /* Check the file record header for consistency. */ if (rec->rhdr.sign != NTFS_FILE_SIGNATURE || fo > (SECTOR_SIZE - ((rs >> SECTOR_SHIFT) + 1) * sizeof(short)) || (fn - 1) * SECTOR_SIZE != rs || ao < MFTRECORD_FIXUP_OFFSET_1 || ao > sbi->record_size - SIZEOF_RESIDENT || !is_rec_inuse(rec) || - le32_to_cpu(rec->total) != rs) { + le32_to_cpu(rec->total) != rs || used > rs || used < ao) { return false; } @@ -2807,6 +2808,15 @@ static inline bool check_file_record(const struct MFT_REC *rec, return false; } + /* + * The do_action() handlers compute memmove lengths as + * "rec->used - ", which underflows when + * rec->used is smaller than the attribute walk reached. At this + * point attr is the ATTR_END marker; rec->used must cover it. + */ + if (used < PtrOffset(rec, attr) + sizeof(attr->type)) + return false; + return true; } @@ -3429,6 +3439,9 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); esize = le16_to_cpu(e1->size); + if (PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + e2 = Add2Ptr(e1, esize); memmove(e1, e2, PtrOffset(e2, Add2Ptr(hdr, used))); diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index c7a2f191254da..83df92df1ee0c 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -1252,6 +1252,12 @@ int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, } while (len32); + if (!run) { + err = -EINVAL; + goto out; + } + + /* Get next fragment to read. */ vcn_next = vcn + clen; if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || vcn != vcn_next) { @@ -1349,7 +1355,14 @@ int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, } if (buffer_locked(bh)) __wait_on_buffer(bh); - set_buffer_uptodate(bh); + + lock_buffer(bh); + if (!buffer_uptodate(bh)) + { + memset(bh->b_data, 0, blocksize); + set_buffer_uptodate(bh); + } + unlock_buffer(bh); } else { bh = ntfs_bread(sb, block); if (!bh) { @@ -2633,7 +2646,7 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) u32 uni_bytes; struct ntfs_inode *ni = sbi->volume.ni; /* Allocate PATH_MAX bytes. */ - struct cpu_str *uni = __getname(); + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); if (!uni) return -ENOMEM; @@ -2677,6 +2690,6 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) err = _ni_write_inode(&ni->vfs_inode, 0); out: - __putname(uni); + kfree(uni); return err; } diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 6d1bf890929d9..050b3709e0204 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -1190,7 +1190,12 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni, return -EINVAL; } - fnd_push(fnd, node, e); + err = fnd_push(fnd, node, e); + + if (err) { + put_indx_node(node); + return err; + } } *entry = e; diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 3959f23c487a2..205baa791f998 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -472,6 +472,7 @@ static struct inode *ntfs_read_mft(struct inode *inode, /* Records in $Extend are not a files or general directories. */ inode->i_op = &ntfs_file_inode_operations; mode = S_IFREG; + init_rwsem(&ni->file.run_lock); } else { err = -EINVAL; goto out; @@ -1278,7 +1279,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, fa |= FILE_ATTRIBUTE_READONLY; /* Allocate PATH_MAX bytes. */ - new_de = __getname(); + new_de = kzalloc(PATH_MAX, GFP_KERNEL); if (!new_de) { err = -ENOMEM; goto out1; @@ -1698,7 +1699,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, ntfs_mark_rec_free(sbi, ino, false); out2: - __putname(new_de); + kfree(new_de); kfree(rp); out1: @@ -1719,7 +1720,7 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) struct NTFS_DE *de; /* Allocate PATH_MAX bytes. */ - de = __getname(); + de = kzalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1733,7 +1734,7 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) err = ni_add_name(ntfs_i(d_inode(dentry->d_parent)), ni, de); out: - __putname(de); + kfree(de); return err; } @@ -1756,8 +1757,7 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) if (ntfs_is_meta_file(sbi, ni->mi.rno)) return -EINVAL; - /* Allocate PATH_MAX bytes. */ - de = __getname(); + de = kzalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1793,7 +1793,7 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) out: ni_unlock(ni); - __putname(de); + kfree(de); return err; } diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c index 82c8ae56beee6..612923d93f4dc 100644 --- a/fs/ntfs3/namei.c +++ b/fs/ntfs3/namei.c @@ -68,7 +68,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, u32 flags) { struct ntfs_inode *ni = ntfs_i(dir); - struct cpu_str *uni = __getname(); + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); struct inode *inode; int err; @@ -85,7 +85,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, inode = dir_search_u(dir, uni, NULL); ni_unlock(ni); } - __putname(uni); + kfree(uni); } /* @@ -303,8 +303,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, return err; } - /* Allocate PATH_MAX bytes. */ - de = __getname(); + de = kmalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -349,7 +348,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, ni_unlock(ni); ni_unlock(dir_ni); out: - __putname(de); + kfree(de); return err; } @@ -407,7 +406,7 @@ static int ntfs_d_hash(const struct dentry *dentry, struct qstr *name) /* * Try slow way with current upcase table */ - uni = kmem_cache_alloc(names_cachep, GFP_NOWAIT); + uni = kmalloc(PATH_MAX, GFP_NOWAIT); if (!uni) return -ENOMEM; @@ -429,7 +428,7 @@ static int ntfs_d_hash(const struct dentry *dentry, struct qstr *name) err = 0; out: - kmem_cache_free(names_cachep, uni); + kfree(uni); return err; } @@ -468,7 +467,7 @@ static int ntfs_d_compare(const struct dentry *dentry, unsigned int len1, * Try slow way with current upcase table */ sbi = dentry->d_sb->s_fs_info; - uni1 = __getname(); + uni1 = kmalloc(PATH_MAX, GFP_NOWAIT); if (!uni1) return -ENOMEM; @@ -498,7 +497,7 @@ static int ntfs_d_compare(const struct dentry *dentry, unsigned int len1, ret = !ntfs_cmp_names_cpu(uni1, uni2, sbi->upcase, false) ? 0 : 1; out: - __putname(uni1); + kfree(uni1); return ret; } diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index 630128716ea73..2649fbe16669d 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -979,11 +979,12 @@ static inline __le64 kernel2nt(const struct timespec64 *ts) */ static inline void nt2kernel(const __le64 tm, struct timespec64 *ts) { - u64 t = le64_to_cpu(tm) - _100ns2seconds * SecondsToStartOf1970; + s32 t32; + /* use signed 64 bit to support timestamps prior to epoch. xfstest 258. */ + s64 t = le64_to_cpu(tm) - _100ns2seconds * SecondsToStartOf1970; - // WARNING: do_div changes its first argument(!) - ts->tv_nsec = do_div(t, _100ns2seconds) * 100; - ts->tv_sec = t; + ts->tv_sec = div_s64_rem(t, _100ns2seconds, &t32); + ts->tv_nsec = t32 * 100; } static inline struct ntfs_sb_info *ntfs_sb(struct super_block *sb) diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 88550085f7457..21decc2922823 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -984,8 +984,12 @@ int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, if (!dlcn) return -EINVAL; - if (check_add_overflow(prev_lcn, dlcn, &lcn)) + /* Check special combination: 0 + SPARSE_LCN64. */ + if (!prev_lcn && dlcn == SPARSE_LCN64) { + lcn = SPARSE_LCN64; + } else if (check_add_overflow(prev_lcn, dlcn, &lcn)) { return -EINVAL; + } prev_lcn = lcn; } else { /* The size of 'dlcn' can't be > 8. */ @@ -1120,11 +1124,14 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, struct rw_semaphore *lock = is_mounted(sbi) ? &sbi->mft.ni->file.run_lock : NULL; - if (lock) - down_read(lock); - ntfs_refresh_zone(sbi); - if (lock) - up_read(lock); + if (lock) { + if (down_read_trylock(lock)) { + ntfs_refresh_zone(sbi); + up_read(lock); + } + } else { + ntfs_refresh_zone(sbi); + } } up_write(&wnd->rw_lock); if (err) diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index ddff94c091b8c..e6c0908e27c29 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -933,6 +933,11 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, sbi->volume.blocks = dev_size >> PAGE_SHIFT; + /* Set dummy blocksize to read boot_block. */ + if (!sb_min_blocksize(sb, PAGE_SIZE)) { + return -EINVAL; + } + read_boot: bh = ntfs_bread(sb, boot_block); if (!bh) diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index e519e21596a70..142ecb3847e59 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -556,8 +556,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, if (unlikely(is_bad_ni(ni))) return ERR_PTR(-EINVAL); - /* Allocate PATH_MAX bytes. */ - buf = __getname(); + buf = kmalloc(PATH_MAX, GFP_KERNEL); if (!buf) return ERR_PTR(-ENOMEM); @@ -588,7 +587,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, if (!IS_ERR(acl)) set_cached_acl(inode, type, acl); - __putname(buf); + kfree(buf); return acl; } diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c index 162711cc5b201..a0ced11e0c24a 100644 --- a/fs/ocfs2/alloc.c +++ b/fs/ocfs2/alloc.c @@ -3654,7 +3654,6 @@ static int ocfs2_merge_rec_left(struct ocfs2_path *right_path, * So we use the new rightmost path. */ ocfs2_mv_path(right_path, left_path); - left_path = NULL; } else ocfs2_complete_edge_insert(handle, left_path, right_path, subtree_index); diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 76c86f1c2b1ca..7a65d5a36a3e1 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -2295,8 +2295,6 @@ static int ocfs2_dio_end_io_write(struct inode *inode, goto out; } - down_write(&oi->ip_alloc_sem); - /* Delete orphan before acquire i_rwsem. */ if (dwc->dw_orphaned) { BUG_ON(dwc->dw_writer_pid != task_pid_nr(current)); @@ -2309,6 +2307,7 @@ static int ocfs2_dio_end_io_write(struct inode *inode, mlog_errno(ret); } + down_write(&oi->ip_alloc_sem); di = (struct ocfs2_dinode *)di_bh->b_data; ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh); diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index fcc89856ab95b..a840dde611c01 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -201,13 +201,15 @@ struct inode *ocfs2_iget(struct ocfs2_super *osb, u64 blkno, unsigned flags, static int ocfs2_dinode_has_extents(struct ocfs2_dinode *di) { /* inodes flagged with other stuff in id2 */ - if (di->i_flags & (OCFS2_SUPER_BLOCK_FL | OCFS2_LOCAL_ALLOC_FL | - OCFS2_CHAIN_FL | OCFS2_DEALLOC_FL)) + if (le32_to_cpu(di->i_flags) & + (OCFS2_SUPER_BLOCK_FL | OCFS2_LOCAL_ALLOC_FL | OCFS2_CHAIN_FL | + OCFS2_DEALLOC_FL)) return 0; /* i_flags doesn't indicate when id2 is a fast symlink */ - if (S_ISLNK(di->i_mode) && di->i_size && di->i_clusters == 0) + if (S_ISLNK(le16_to_cpu(di->i_mode)) && le64_to_cpu(di->i_size) && + !le32_to_cpu(di->i_clusters)) return 0; - if (di->i_dyn_features & OCFS2_INLINE_DATA_FL) + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) return 0; return 1; @@ -1503,6 +1505,37 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { + struct ocfs2_inline_data *data = &di->id2.i_data; + + if (le32_to_cpu(di->i_clusters)) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: %u clusters\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(di->i_clusters)); + goto bail; + } + + if (le16_to_cpu(data->id_count) > + ocfs2_max_inline_data_with_xattr(sb, di)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: inline data id_count %u exceeds max %d\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(data->id_count), + ocfs2_max_inline_data_with_xattr(sb, di)); + goto bail; + } + + if (le64_to_cpu(di->i_size) > le16_to_cpu(data->id_count)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: inline data i_size %llu exceeds id_count %u\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size), + le16_to_cpu(data->id_count)); + goto bail; + } + } + rc = 0; bail: diff --git a/fs/ocfs2/mmap.c b/fs/ocfs2/mmap.c index 50e2faf64c197..6c570157caf16 100644 --- a/fs/ocfs2/mmap.c +++ b/fs/ocfs2/mmap.c @@ -30,7 +30,8 @@ static vm_fault_t ocfs2_fault(struct vm_fault *vmf) { - struct vm_area_struct *vma = vmf->vma; + unsigned long long ip_blkno = + OCFS2_I(file_inode(vmf->vma->vm_file))->ip_blkno; sigset_t oldset; vm_fault_t ret; @@ -38,11 +39,9 @@ static vm_fault_t ocfs2_fault(struct vm_fault *vmf) ret = filemap_fault(vmf); ocfs2_unblock_signals(&oldset); - trace_ocfs2_fault(OCFS2_I(vma->vm_file->f_mapping->host)->ip_blkno, - vma, vmf->page, vmf->pgoff); + trace_ocfs2_fault(ip_blkno, vmf->page, vmf->pgoff); return ret; } - static vm_fault_t __ocfs2_page_mkwrite(struct file *file, struct buffer_head *di_bh, struct folio *folio) { diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c index 10923bf7c8b84..26e150c5f25eb 100644 --- a/fs/ocfs2/move_extents.c +++ b/fs/ocfs2/move_extents.c @@ -98,7 +98,13 @@ static int __ocfs2_move_extent(handle_t *handle, rec = &el->l_recs[index]; - BUG_ON(ext_flags != rec->e_flags); + if (ext_flags != rec->e_flags) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has corrupted extent %d with flags 0x%x at cpos %u\n", + (unsigned long long)ino, index, rec->e_flags, cpos); + goto out; + } + /* * after moving/defraging to new location, the extent is not going * to be refcounted anymore. diff --git a/fs/ocfs2/ocfs2_trace.h b/fs/ocfs2/ocfs2_trace.h index 54ed1495de9a7..90a69f44f041c 100644 --- a/fs/ocfs2/ocfs2_trace.h +++ b/fs/ocfs2/ocfs2_trace.h @@ -1246,22 +1246,20 @@ TRACE_EVENT(ocfs2_write_end_inline, TRACE_EVENT(ocfs2_fault, TP_PROTO(unsigned long long ino, - void *area, void *page, unsigned long pgoff), - TP_ARGS(ino, area, page, pgoff), + void *page, unsigned long pgoff), + TP_ARGS(ino, page, pgoff), TP_STRUCT__entry( __field(unsigned long long, ino) - __field(void *, area) __field(void *, page) __field(unsigned long, pgoff) ), TP_fast_assign( __entry->ino = ino; - __entry->area = area; __entry->page = page; __entry->pgoff = pgoff; ), - TP_printk("%llu %p %p %lu", - __entry->ino, __entry->area, __entry->page, __entry->pgoff) + TP_printk("%llu %p %lu", + __entry->ino, __entry->page, __entry->pgoff) ); /* End of trace events for fs/ocfs2/mmap.c. */ diff --git a/fs/ocfs2/resize.c b/fs/ocfs2/resize.c index b0733c08ed13b..ed7ed15ad9a73 100644 --- a/fs/ocfs2/resize.c +++ b/fs/ocfs2/resize.c @@ -303,9 +303,13 @@ int ocfs2_group_extend(struct inode * inode, int new_clusters) fe = (struct ocfs2_dinode *)main_bm_bh->b_data; - /* main_bm_bh is validated by inode read inside ocfs2_inode_lock(), - * so any corruption is a code bug. */ - BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + /* JBD-managed buffers can bypass validation, so treat this as corruption. */ + if (!OCFS2_IS_VALID_DINODE(fe)) { + ret = ocfs2_error(main_bm_inode->i_sb, + "Invalid dinode #%llu\n", + (unsigned long long)OCFS2_I(main_bm_inode)->ip_blkno); + goto out_unlock; + } if (le16_to_cpu(fe->id2.i_chain.cl_cpg) != ocfs2_group_bitmap_size(osb->sb, 0, diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index 6ac4dcd54588c..e93fc842bb203 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -1992,6 +1992,16 @@ static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac, } cl = (struct ocfs2_chain_list *) &fe->id2.i_chain; + if (!le16_to_cpu(cl->cl_next_free_rec) || + le16_to_cpu(cl->cl_next_free_rec) > le16_to_cpu(cl->cl_count)) { + status = ocfs2_error(ac->ac_inode->i_sb, + "Chain allocator dinode %llu has invalid next " + "free chain record %u, but only %u total\n", + (unsigned long long)le64_to_cpu(fe->i_blkno), + le16_to_cpu(cl->cl_next_free_rec), + le16_to_cpu(cl->cl_count)); + goto bail; + } victim = ocfs2_find_victim_chain(cl); ac->ac_chain = victim; diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index d70a20d29e3e9..64ba3946f8408 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -6364,6 +6364,10 @@ static int ocfs2_reflink_xattr_header(handle_t *handle, (void *)last - (void *)xe); memset(last, 0, sizeof(struct ocfs2_xattr_entry)); + last = &new_xh->xh_entries[le16_to_cpu(new_xh->xh_count)] - 1; + } else { + memset(xe, 0, sizeof(struct ocfs2_xattr_entry)); + last = NULL; } /* diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 604a82acd1645..97f373fd98797 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -1159,15 +1159,15 @@ static int ovl_copy_up_one(struct dentry *parent, struct dentry *dentry, return -EOVERFLOW; /* - * With metacopy disabled, we fsync after final metadata copyup, for + * With "fsync=strict", we fsync after final metadata copyup, for * both regular files and directories to get atomic copyup semantics * on filesystems that do not use strict metadata ordering (e.g. ubifs). * - * With metacopy enabled we want to avoid fsync on all meta copyup + * By default, we want to avoid fsync on all meta copyup, because * that will hurt performance of workloads such as chown -R, so we * only fsync on data copyup as legacy behavior. */ - ctx.metadata_fsync = !OVL_FS(dentry->d_sb)->config.metacopy && + ctx.metadata_fsync = ovl_should_sync_metadata(OVL_FS(dentry->d_sb)) && (S_ISREG(ctx.stat.mode) || S_ISDIR(ctx.stat.mode)); ctx.metacopy = ovl_need_meta_copy_up(dentry, ctx.stat.mode, flags); diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index c8fd5951fc5ec..d1eb1cbe7a456 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -99,6 +99,12 @@ enum { OVL_VERITY_REQUIRE, }; +enum { + OVL_FSYNC_VOLATILE, + OVL_FSYNC_AUTO, + OVL_FSYNC_STRICT, +}; + /* * The tuple (fh,uuid) is a universal unique identifier for a copy up origin, * where: @@ -634,6 +640,21 @@ static inline bool ovl_xino_warn(struct ovl_fs *ofs) return ofs->config.xino == OVL_XINO_ON; } +static inline bool ovl_should_sync(struct ovl_fs *ofs) +{ + return ofs->config.fsync_mode != OVL_FSYNC_VOLATILE; +} + +static inline bool ovl_should_sync_metadata(struct ovl_fs *ofs) +{ + return ofs->config.fsync_mode == OVL_FSYNC_STRICT; +} + +static inline bool ovl_is_volatile(struct ovl_config *config) +{ + return config->fsync_mode == OVL_FSYNC_VOLATILE; +} + /* * To avoid regressions in existing setups with overlay lower offline changes, * we allow lower changes only if none of the new features are used. diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h index 1d4828dbcf7ac..80cad4ea96a3e 100644 --- a/fs/overlayfs/ovl_entry.h +++ b/fs/overlayfs/ovl_entry.h @@ -18,7 +18,7 @@ struct ovl_config { int xino; bool metacopy; bool userxattr; - bool ovl_volatile; + int fsync_mode; }; struct ovl_sb { @@ -120,11 +120,6 @@ static inline struct ovl_fs *OVL_FS(struct super_block *sb) return (struct ovl_fs *)sb->s_fs_info; } -static inline bool ovl_should_sync(struct ovl_fs *ofs) -{ - return !ofs->config.ovl_volatile; -} - static inline unsigned int ovl_numlower(struct ovl_entry *oe) { return oe ? oe->__numlower : 0; diff --git a/fs/overlayfs/params.c b/fs/overlayfs/params.c index 63b7346c5ee1c..7d8bc6d3863c1 100644 --- a/fs/overlayfs/params.c +++ b/fs/overlayfs/params.c @@ -58,6 +58,7 @@ enum ovl_opt { Opt_xino, Opt_metacopy, Opt_verity, + Opt_fsync, Opt_volatile, Opt_override_creds, }; @@ -140,6 +141,23 @@ static int ovl_verity_mode_def(void) return OVL_VERITY_OFF; } +static const struct constant_table ovl_parameter_fsync[] = { + { "volatile", OVL_FSYNC_VOLATILE }, + { "auto", OVL_FSYNC_AUTO }, + { "strict", OVL_FSYNC_STRICT }, + {} +}; + +static const char *ovl_fsync_mode(struct ovl_config *config) +{ + return ovl_parameter_fsync[config->fsync_mode].name; +} + +static int ovl_fsync_mode_def(void) +{ + return OVL_FSYNC_AUTO; +} + const struct fs_parameter_spec ovl_parameter_spec[] = { fsparam_string_empty("lowerdir", Opt_lowerdir), fsparam_file_or_string("lowerdir+", Opt_lowerdir_add), @@ -155,6 +173,7 @@ const struct fs_parameter_spec ovl_parameter_spec[] = { fsparam_enum("xino", Opt_xino, ovl_parameter_xino), fsparam_enum("metacopy", Opt_metacopy, ovl_parameter_bool), fsparam_enum("verity", Opt_verity, ovl_parameter_verity), + fsparam_enum("fsync", Opt_fsync, ovl_parameter_fsync), fsparam_flag("volatile", Opt_volatile), fsparam_flag_no("override_creds", Opt_override_creds), {} @@ -665,8 +684,11 @@ static int ovl_parse_param(struct fs_context *fc, struct fs_parameter *param) case Opt_verity: config->verity_mode = result.uint_32; break; + case Opt_fsync: + config->fsync_mode = result.uint_32; + break; case Opt_volatile: - config->ovl_volatile = true; + config->fsync_mode = OVL_FSYNC_VOLATILE; break; case Opt_userxattr: config->userxattr = true; @@ -800,6 +822,7 @@ int ovl_init_fs_context(struct fs_context *fc) ofs->config.nfs_export = ovl_nfs_export_def; ofs->config.xino = ovl_xino_def(); ofs->config.metacopy = ovl_metacopy_def; + ofs->config.fsync_mode = ovl_fsync_mode_def(); fc->s_fs_info = ofs; fc->fs_private = ctx; @@ -870,9 +893,9 @@ int ovl_fs_params_verify(const struct ovl_fs_context *ctx, config->index = false; } - if (!config->upperdir && config->ovl_volatile) { + if (!config->upperdir && ovl_is_volatile(config)) { pr_info("option \"volatile\" is meaningless in a non-upper mount, ignoring it.\n"); - config->ovl_volatile = false; + config->fsync_mode = ovl_fsync_mode_def(); } if (!config->upperdir && config->uuid == OVL_UUID_ON) { @@ -1070,8 +1093,8 @@ int ovl_show_options(struct seq_file *m, struct dentry *dentry) seq_printf(m, ",xino=%s", ovl_xino_mode(&ofs->config)); if (ofs->config.metacopy != ovl_metacopy_def) seq_printf(m, ",metacopy=%s", str_on_off(ofs->config.metacopy)); - if (ofs->config.ovl_volatile) - seq_puts(m, ",volatile"); + if (ofs->config.fsync_mode != ovl_fsync_mode_def()) + seq_printf(m, ",fsync=%s", ovl_fsync_mode(&ofs->config)); if (ofs->config.userxattr) seq_puts(m, ",userxattr"); if (ofs->config.verity_mode != ovl_verity_mode_def()) diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 1e9792cc557b8..3c27e7a16f94b 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -761,7 +761,7 @@ static bool ovl_fill_real(struct dir_context *ctx, const char *name, struct dir_context *orig_ctx = rdt->orig_ctx; bool res; - if (rdt->parent_ino && strcmp(name, "..") == 0) { + if (rdt->parent_ino && namelen == 2 && !strncmp(name, "..", 2)) { ino = rdt->parent_ino; } else if (rdt->cache) { struct ovl_cache_entry *p; diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 43ee4c7296a77..0e971ea97580f 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -773,7 +773,7 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, * For volatile mount, create a incompat/volatile/dirty file to keep * track of it. */ - if (ofs->config.ovl_volatile) { + if (ovl_is_volatile(&ofs->config)) { err = ovl_create_volatile_dirty(ofs); if (err < 0) { pr_err("Failed to create volatile/dirty file.\n"); diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 82373dd1ce6e8..19f4fe7b98f29 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -90,7 +90,10 @@ int ovl_can_decode_fh(struct super_block *sb) if (!exportfs_can_decode_fh(sb->s_export_op)) return 0; - return sb->s_export_op->encode_fh ? -1 : FILEID_INO32_GEN; + if (sb->s_export_op->encode_fh == generic_encode_ino32_fh) + return FILEID_INO32_GEN; + + return -1; } struct dentry *ovl_indexdir(struct super_block *sb) diff --git a/fs/pidfs.c b/fs/pidfs.c index 0ef5b47d796a2..34987fcdd9a87 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -306,6 +306,8 @@ static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg) const struct cred *c; __u64 mask; + BUILD_BUG_ON(sizeof(struct pidfd_info) != PIDFD_INFO_SIZE_VER1); + if (!uinfo) return -EINVAL; if (usize < PIDFD_INFO_SIZE_VER0) @@ -319,7 +321,7 @@ static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg) * namespace hierarchy. */ if (!pid_in_current_pidns(pid)) - return -ESRCH; + return -EREMOTE; attr = READ_ONCE(pid->attr); if (mask & PIDFD_INFO_EXIT) { diff --git a/fs/proc/array.c b/fs/proc/array.c index 2ae63189091e0..038d4b57127fe 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -529,7 +529,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, } sid = task_session_nr_ns(task, ns); - ppid = task_tgid_nr_ns(task->real_parent, ns); + ppid = task_ppid_nr_ns(task, ns); pgid = task_pgrp_nr_ns(task, ns); unlock_task_sighand(task, &flags); diff --git a/fs/proc/base.c b/fs/proc/base.c index 6299878e3d97e..a314fd9af17c0 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2128,6 +2128,9 @@ bool proc_fill_cache(struct file *file, struct dir_context *ctx, ino_t ino = 1; child = try_lookup_noperm(&qname, dir); + if (IS_ERR(child)) + goto end_instantiate; + if (!child) { DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); child = d_alloc_parallel(dir, &qname, &wq); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index fc35a0543f019..b490245ff9be7 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -656,6 +656,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) struct proc_maps_locking_ctx lock_ctx = { .mm = mm }; struct procmap_query karg; struct vm_area_struct *vma; + struct file *vm_file = NULL; const char *name = NULL; char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL; __u64 usize; @@ -727,21 +728,6 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) karg.inode = 0; } - if (karg.build_id_size) { - __u32 build_id_sz; - - err = build_id_parse(vma, build_id_buf, &build_id_sz); - if (err) { - karg.build_id_size = 0; - } else { - if (karg.build_id_size < build_id_sz) { - err = -ENAMETOOLONG; - goto out; - } - karg.build_id_size = build_id_sz; - } - } - if (karg.vma_name_size) { size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size); const struct path *path; @@ -775,10 +761,34 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) karg.vma_name_size = name_sz; } + if (karg.build_id_size && vma->vm_file) + vm_file = get_file(vma->vm_file); + /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ query_vma_teardown(&lock_ctx); mmput(mm); + if (karg.build_id_size) { + __u32 build_id_sz; + + if (vm_file) + err = build_id_parse_file(vm_file, build_id_buf, &build_id_sz); + else + err = -ENOENT; + if (err) { + karg.build_id_size = 0; + } else { + if (karg.build_id_size < build_id_sz) { + err = -ENAMETOOLONG; + goto out_file; + } + karg.build_id_size = build_id_sz; + } + } + + if (vm_file) + fput(vm_file); + if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr), name, karg.vma_name_size)) { kfree(name_buf); @@ -798,6 +808,9 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) out: query_vma_teardown(&lock_ctx); mmput(mm); +out_file: + if (vm_file) + fput(vm_file); kfree(name_buf); return err; } diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index f1848cdd6d348..7b6d6378a3b87 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -298,6 +298,17 @@ void persistent_ram_save_old(struct persistent_ram_zone *prz) if (!size) return; + /* + * If the existing buffer is differently sized, free it so a new + * one is allocated. This can happen when persistent_ram_save_old() + * is called early in boot and later for a timer-triggered + * survivable crash when the crash dumps don't match in size + * (which would be extremely unlikely given kmsg buffers usually + * exceed prz buffer sizes). + */ + if (prz->old_log && prz->old_log_size != size) + persistent_ram_free_old(prz); + if (!prz->old_log) { persistent_ram_ecc_old(prz); prz->old_log = kvzalloc(size, GFP_KERNEL); @@ -446,6 +457,13 @@ static void *persistent_ram_vmap(phys_addr_t start, size_t size, vaddr = vmap(pages, page_count, VM_MAP | VM_IOREMAP, prot); kfree(pages); + /* + * vmap() may fail and return NULL. Do not add the offset in this + * case, otherwise a NULL mapping would appear successful. + */ + if (!vaddr) + return NULL; + /* * Since vmap() uses page granularity, we must add the offset * into the page here, to get the byte granularity address diff --git a/fs/quota/quota.c b/fs/quota/quota.c index 7c2b75a444852..de4379a9c7920 100644 --- a/fs/quota/quota.c +++ b/fs/quota/quota.c @@ -899,6 +899,7 @@ static struct super_block *quotactl_block(const char __user *special, int cmd) sb_start_write(sb); sb_end_write(sb); put_super(sb); + cond_resched(); goto retry; } return sb; diff --git a/fs/readdir.c b/fs/readdir.c index 7764b86389788..73707b6816e9a 100644 --- a/fs/readdir.c +++ b/fs/readdir.c @@ -316,6 +316,7 @@ SYSCALL_DEFINE3(getdents, unsigned int, fd, struct getdents_callback buf = { .ctx.actor = filldir, .ctx.count = count, + .ctx.dt_flags_mask = FILLDIR_FLAG_NOINTR, .current_dir = dirent }; int error; @@ -400,6 +401,7 @@ SYSCALL_DEFINE3(getdents64, unsigned int, fd, struct getdents_callback64 buf = { .ctx.actor = filldir64, .ctx.count = count, + .ctx.dt_flags_mask = FILLDIR_FLAG_NOINTR, .current_dir = dirent }; int error; @@ -569,6 +571,7 @@ COMPAT_SYSCALL_DEFINE3(getdents, unsigned int, fd, struct compat_getdents_callback buf = { .ctx.actor = compat_filldir, .ctx.count = count, + .ctx.dt_flags_mask = FILLDIR_FLAG_NOINTR, .current_dir = dirent, }; int error; diff --git a/fs/romfs/super.c b/fs/romfs/super.c index 0addcc849ff2c..e83f9b78d7a16 100644 --- a/fs/romfs/super.c +++ b/fs/romfs/super.c @@ -458,7 +458,10 @@ static int romfs_fill_super(struct super_block *sb, struct fs_context *fc) #ifdef CONFIG_BLOCK if (!sb->s_mtd) { - sb_set_blocksize(sb, ROMBSIZE); + if (!sb_set_blocksize(sb, ROMBSIZE)) { + errorf(fc, "romfs: unable to set blocksize\n"); + return -EINVAL; + } } else { sb->s_blocksize = ROMBSIZE; sb->s_blocksize_bits = blksize_bits(ROMBSIZE); diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index e3ea6fe7edb47..09939fe9666ea 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -788,11 +788,11 @@ static void cfids_laundromat_worker(struct work_struct *work) cfid->dentry = NULL; if (cfid->is_open) { - spin_lock(&cifs_tcp_ses_lock); + spin_lock(&cfid->tcon->tc_lock); ++cfid->tcon->tc_count; trace_smb3_tcon_ref(cfid->tcon->debug_id, cfid->tcon->tc_count, netfs_trace_tcon_ref_get_cached_laundromat); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&cfid->tcon->tc_lock); queue_work(serverclose_wq, &cfid->close_work); } else /* diff --git a/fs/smb/client/cached_dir.h b/fs/smb/client/cached_dir.h index 1e383db7c3374..5091bf45345e8 100644 --- a/fs/smb/client/cached_dir.h +++ b/fs/smb/client/cached_dir.h @@ -36,10 +36,10 @@ struct cached_fid { struct list_head entry; struct cached_fids *cfids; const char *path; - bool has_lease:1; - bool is_open:1; - bool on_list:1; - bool file_all_info_is_valid:1; + bool has_lease; + bool is_open; + bool on_list; + bool file_all_info_is_valid; unsigned long time; /* jiffies of when lease was taken */ unsigned long last_access_time; /* jiffies of when last accessed */ struct kref refcount; diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index ce2ebc213a1d9..1d9e2e742ed71 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -831,6 +831,7 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, dump_ace(ppace[i], end_of_acl); #endif if (mode_from_special_sid && + ppace[i]->sid.num_subauth >= 3 && (compare_sids(&(ppace[i]->sid), &sid_unix_NFS_mode) == 0)) { /* diff --git a/fs/smb/client/cifsencrypt.c b/fs/smb/client/cifsencrypt.c index 801824825ecf2..b9e20bcaa8b5c 100644 --- a/fs/smb/client/cifsencrypt.c +++ b/fs/smb/client/cifsencrypt.c @@ -26,6 +26,7 @@ #include #include #include +#include static int cifs_sig_update(struct cifs_calc_sig_ctx *ctx, const u8 *data, size_t len) @@ -277,7 +278,7 @@ int cifs_verify_signature(struct smb_rqst *rqst, /* cifs_dump_mem("what we think it should be: ", what_we_think_sig_should_be, 16); */ - if (memcmp(server_response_sig, what_we_think_sig_should_be, 8)) + if (crypto_memneq(server_response_sig, what_we_think_sig_should_be, 8)) return -EACCES; else return 0; diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index 185ac41bd7e93..4b34a4304edb7 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -330,10 +330,14 @@ static void cifs_kill_sb(struct super_block *sb) /* * We need to release all dentries for the cached directories - * before we kill the sb. + * and close all deferred file handles before we kill the sb. */ if (cifs_sb->root) { close_all_cached_dirs(cifs_sb); + cifs_close_all_deferred_files_sb(cifs_sb); + + /* Wait for all pending oplock breaks to complete */ + flush_workqueue(cifsoplockd_wq); /* finally release root dentry */ dput(cifs_sb->root); @@ -864,7 +868,6 @@ static void cifs_umount_begin(struct super_block *sb) spin_unlock(&tcon->tc_lock); spin_unlock(&cifs_tcp_ses_lock); - cifs_close_all_deferred_files(tcon); /* cancel_brl_requests(tcon); */ /* BB mark all brl mids as exiting */ /* cancel_notify_requests(tcon); */ if (tcon->ses && tcon->ses->server) { diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h index 203e2aaa3c250..3059fcf12ed13 100644 --- a/fs/smb/client/cifsglob.h +++ b/fs/smb/client/cifsglob.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "cifs_fs_sb.h" #include "cifsacl.h" #include @@ -2396,4 +2397,20 @@ static inline void mid_execute_callback(struct mid_q_entry *mid) (le32_to_cpu((tcon)->fsAttrInfo.Attributes) & \ FILE_SUPPORTS_REPARSE_POINTS)) +static inline int cifs_open_create_options(unsigned int oflags, int opts) +{ + /* O_SYNC also has bit for O_DSYNC so following check picks up either */ + if (oflags & O_SYNC) + opts |= CREATE_WRITE_THROUGH; + if (oflags & O_DIRECT) + opts |= CREATE_NO_BUFFER; + return opts; +} + +/* + * The number of blocks is not related to (i_size / i_blksize), but instead + * 512 byte (2**9) size is required for calculating num blocks. + */ +#define CIFS_INO_BLOCKS(size) DIV_ROUND_UP_ULL((u64)(size), 512) + #endif /* _CIFS_GLOB_H */ diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h index 3528c365a4529..4be719c9fd520 100644 --- a/fs/smb/client/cifsproto.h +++ b/fs/smb/client/cifsproto.h @@ -313,6 +313,7 @@ extern void cifs_close_deferred_file(struct cifsInodeInfo *cifs_inode); extern void cifs_close_all_deferred_files(struct cifs_tcon *cifs_tcon); +void cifs_close_all_deferred_files_sb(struct cifs_sb_info *cifs_sb); void cifs_close_deferred_file_under_dentry(struct cifs_tcon *cifs_tcon, struct dentry *dentry); diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c index dcc50a2bfa4b2..bfc9b1ea76fac 100644 --- a/fs/smb/client/cifssmb.c +++ b/fs/smb/client/cifssmb.c @@ -1374,7 +1374,7 @@ cifs_readv_callback(struct mid_q_entry *mid) } else { size_t trans = rdata->subreq.transferred + rdata->got_bytes; if (trans < rdata->subreq.len && - rdata->subreq.start + trans == ictx->remote_i_size) { + rdata->subreq.start + trans >= ictx->remote_i_size) { rdata->result = 0; __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); } else if (rdata->got_bytes > 0) { diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index 2f94d93b95e91..7bc912d00030c 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -1954,6 +1954,10 @@ static int match_session(struct cifs_ses *ses, case Kerberos: if (!uid_eq(ctx->cred_uid, ses->cred_uid)) return 0; + if (strncmp(ses->user_name ?: "", + ctx->username ?: "", + CIFS_MAX_USERNAME_LEN)) + return 0; break; case NTLMv2: case RawNTLMSSP: @@ -2235,7 +2239,6 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) /* find first : in payload */ payload = upayload->data; delim = strnchr(payload, upayload->datalen, ':'); - cifs_dbg(FYI, "payload=%s\n", payload); if (!delim) { cifs_dbg(FYI, "Unable to find ':' in payload (datalen=%d)\n", upayload->datalen); @@ -4267,7 +4270,9 @@ cifs_setup_session(const unsigned int xid, struct cifs_ses *ses, ses->ses_status = SES_IN_SETUP; /* force iface_list refresh */ + spin_lock(&ses->iface_lock); ses->iface_last_update = 0; + spin_unlock(&ses->iface_lock); } spin_unlock(&ses->ses_lock); diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c index da5597dbf5b9f..50c7c6ec068f9 100644 --- a/fs/smb/client/dir.c +++ b/fs/smb/client/dir.c @@ -307,6 +307,7 @@ static int cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned goto out; } + create_options |= cifs_open_create_options(oflags, create_options); /* * if we're not using unix extensions, see if we need to set * ATTR_READONLY on the create call diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index 474dadeb15933..f0e7a8f69e8b8 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -584,15 +584,8 @@ static int cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_ *********************************************************************/ disposition = cifs_get_disposition(f_flags); - /* BB pass O_SYNC flag through on file attributes .. BB */ - - /* O_SYNC also has bit for O_DSYNC so following check picks up either */ - if (f_flags & O_SYNC) - create_options |= CREATE_WRITE_THROUGH; - - if (f_flags & O_DIRECT) - create_options |= CREATE_NO_BUFFER; + create_options |= cifs_open_create_options(f_flags, create_options); retry_open: oparms = (struct cifs_open_parms) { @@ -711,8 +704,6 @@ struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, mutex_init(&cfile->fh_mutex); spin_lock_init(&cfile->file_info_lock); - cifs_sb_active(inode->i_sb); - /* * If the server returned a read oplock and we have mandatory brlocks, * set oplock level to None. @@ -767,7 +758,6 @@ static void cifsFileInfo_put_final(struct cifsFileInfo *cifs_file) struct inode *inode = d_inode(cifs_file->dentry); struct cifsInodeInfo *cifsi = CIFS_I(inode); struct cifsLockInfo *li, *tmp; - struct super_block *sb = inode->i_sb; /* * Delete any outstanding lock records. We'll lose them when the file @@ -785,7 +775,6 @@ static void cifsFileInfo_put_final(struct cifsFileInfo *cifs_file) cifs_put_tlink(cifs_file->tlink); dput(cifs_file->dentry); - cifs_sb_deactive(sb); kfree(cifs_file->symlink_target); kfree(cifs_file); } @@ -1004,7 +993,6 @@ static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry) if (!rc) { netfs_resize_file(&cinode->netfs, 0, true); cifs_setsize(inode, 0); - inode->i_blocks = 0; } } if (cfile) @@ -1318,13 +1306,8 @@ cifs_reopen_file(struct cifsFileInfo *cfile, bool can_flush) rdwr_for_fscache = 1; desired_access = cifs_convert_flags(cfile->f_flags, rdwr_for_fscache); - - /* O_SYNC also has bit for O_DSYNC so following check picks up either */ - if (cfile->f_flags & O_SYNC) - create_options |= CREATE_WRITE_THROUGH; - - if (cfile->f_flags & O_DIRECT) - create_options |= CREATE_NO_BUFFER; + create_options |= cifs_open_create_options(cfile->f_flags, + create_options); if (server->ops->get_lease_key) server->ops->get_lease_key(inode, &cfile->fid); @@ -3162,12 +3145,6 @@ void cifs_oplock_break(struct work_struct *work) __u64 persistent_fid, volatile_fid; __u16 net_fid; - /* - * Hold a reference to the superblock to prevent it and its inodes from - * being freed while we are accessing cinode. Otherwise, _cifsFileInfo_put() - * may release the last reference to the sb and trigger inode eviction. - */ - cifs_sb_active(sb); wait_on_bit(&cinode->flags, CIFS_INODE_PENDING_WRITERS, TASK_UNINTERRUPTIBLE); @@ -3240,7 +3217,6 @@ void cifs_oplock_break(struct work_struct *work) cifs_put_tlink(tlink); out: cifs_done_oplock_break(cinode); - cifs_sb_deactive(sb); } static int cifs_swap_activate(struct swap_info_struct *sis, diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index 2a0d8b87bd8ea..f207c7cef0467 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -589,6 +589,10 @@ char *cifs_sanitize_prepath(char *prepath, gfp_t gfp) while (IS_DELIM(*cursor1)) cursor1++; + /* exit in case of only delimiters */ + if (!*cursor1) + return NULL; + /* copy the first letter */ *cursor2 = *cursor1; @@ -1080,6 +1084,8 @@ static int smb3_reconfigure(struct fs_context *fc) rc = smb3_sync_session_ctx_passwords(cifs_sb, ses); if (rc) { mutex_unlock(&ses->session_mutex); + kfree_sensitive(new_password); + kfree_sensitive(new_password2); return rc; } @@ -1918,7 +1924,7 @@ int smb3_init_fs_context(struct fs_context *fc) ctx->backupuid_specified = false; /* no backup intent for a user */ ctx->backupgid_specified = false; /* no backup intent for a group */ - ctx->retrans = 1; + ctx->retrans = 0; ctx->reparse_type = CIFS_REPARSE_TYPE_DEFAULT; ctx->symlink_type = CIFS_SYMLINK_TYPE_DEFAULT; ctx->nonativesocket = 0; diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index cac355364e43c..7cd718dd7cb70 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -218,13 +218,7 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr, */ if (is_size_safe_to_change(cifs_i, fattr->cf_eof, from_readdir)) { i_size_write(inode, fattr->cf_eof); - - /* - * i_blocks is not related to (i_size / i_blksize), - * but instead 512 byte (2**9) size is required for - * calculating num blocks. - */ - inode->i_blocks = (512 - 1 + fattr->cf_bytes) >> 9; + inode->i_blocks = CIFS_INO_BLOCKS(fattr->cf_bytes); } if (S_ISLNK(fattr->cf_mode) && fattr->cf_symlink_target) { @@ -3008,6 +3002,11 @@ void cifs_setsize(struct inode *inode, loff_t offset) { spin_lock(&inode->i_lock); i_size_write(inode, offset); + /* + * Until we can query the server for actual allocation size, + * this is best estimate we have for blocks allocated for a file. + */ + inode->i_blocks = CIFS_INO_BLOCKS(offset); spin_unlock(&inode->i_lock); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); truncate_pagecache(inode, offset); @@ -3080,14 +3079,6 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, if (rc == 0) { netfs_resize_file(&cifsInode->netfs, size, true); cifs_setsize(inode, size); - /* - * i_blocks is not related to (i_size / i_blksize), but instead - * 512 byte (2**9) size is required for calculating num blocks. - * Until we can query the server for actual allocation size, - * this is best estimate we have for blocks allocated for a file - * Number of blocks must be rounded up so size 1 is not 0 blocks - */ - inode->i_blocks = (512 - 1 + size) >> 9; } return rc; diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c index e10123d8cd7d9..3770f8348541d 100644 --- a/fs/smb/client/misc.c +++ b/fs/smb/client/misc.c @@ -27,6 +27,11 @@ #include "fs_context.h" #include "cached_dir.h" +struct tcon_list { + struct list_head entry; + struct cifs_tcon *tcon; +}; + /* The xid serves as a useful identifier for each incoming vfs request, in a similar way to the mid which is useful to track each sent smb, and CurrentXid can also provide a running counter (although it @@ -833,6 +838,43 @@ cifs_close_all_deferred_files(struct cifs_tcon *tcon) } } +void cifs_close_all_deferred_files_sb(struct cifs_sb_info *cifs_sb) +{ + struct rb_root *root = &cifs_sb->tlink_tree; + struct rb_node *node; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + struct tcon_list *tmp_list, *q; + LIST_HEAD(tcon_head); + + spin_lock(&cifs_sb->tlink_tree_lock); + for (node = rb_first(root); node; node = rb_next(node)) { + tlink = rb_entry(node, struct tcon_link, tl_rbnode); + tcon = tlink_tcon(tlink); + if (IS_ERR(tcon)) + continue; + tmp_list = kmalloc(sizeof(struct tcon_list), GFP_ATOMIC); + if (tmp_list == NULL) + break; + tmp_list->tcon = tcon; + /* Take a reference on tcon to prevent it from being freed */ + spin_lock(&tcon->tc_lock); + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_close_defer_files); + spin_unlock(&tcon->tc_lock); + list_add_tail(&tmp_list->entry, &tcon_head); + } + spin_unlock(&cifs_sb->tlink_tree_lock); + + list_for_each_entry_safe(tmp_list, q, &tcon_head, entry) { + cifs_close_all_deferred_files(tmp_list->tcon); + list_del(&tmp_list->entry); + cifs_put_tcon(tmp_list->tcon, netfs_trace_tcon_ref_put_close_defer_files); + kfree(tmp_list); + } +} + void cifs_close_deferred_file_under_dentry(struct cifs_tcon *tcon, struct dentry *dentry) { diff --git a/fs/smb/client/nterr.h b/fs/smb/client/nterr.h index 180602c22355e..09263c91d07a4 100644 --- a/fs/smb/client/nterr.h +++ b/fs/smb/client/nterr.h @@ -41,10 +41,10 @@ extern const struct nt_err_code_struct nt_errs[]; #define NT_STATUS_MEDIA_CHANGED 0x8000001c #define NT_STATUS_END_OF_MEDIA 0x8000001e #define NT_STATUS_MEDIA_CHECK 0x80000020 -#define NT_STATUS_NO_DATA_DETECTED 0x8000001c +#define NT_STATUS_NO_DATA_DETECTED 0x80000022 #define NT_STATUS_STOPPED_ON_SYMLINK 0x8000002d #define NT_STATUS_DEVICE_REQUIRES_CLEANING 0x80000288 -#define NT_STATUS_DEVICE_DOOR_OPEN 0x80000288 +#define NT_STATUS_DEVICE_DOOR_OPEN 0x80000289 #define NT_STATUS_UNSUCCESSFUL 0xC0000000 | 0x0001 #define NT_STATUS_NOT_IMPLEMENTED 0xC0000000 | 0x0002 #define NT_STATUS_INVALID_INFO_CLASS 0xC0000000 | 0x0003 @@ -70,7 +70,7 @@ extern const struct nt_err_code_struct nt_errs[]; #define NT_STATUS_NO_MEMORY 0xC0000000 | 0x0017 #define NT_STATUS_CONFLICTING_ADDRESSES 0xC0000000 | 0x0018 #define NT_STATUS_NOT_MAPPED_VIEW 0xC0000000 | 0x0019 -#define NT_STATUS_UNABLE_TO_FREE_VM 0x80000000 | 0x001a +#define NT_STATUS_UNABLE_TO_FREE_VM 0xC0000000 | 0x001a #define NT_STATUS_UNABLE_TO_DELETE_SECTION 0xC0000000 | 0x001b #define NT_STATUS_INVALID_SYSTEM_SERVICE 0xC0000000 | 0x001c #define NT_STATUS_ILLEGAL_INSTRUCTION 0xC0000000 | 0x001d diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index a7f6292388306..30fff678c7456 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -27,10 +27,11 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) { struct smb2_err_rsp *err = iov->iov_base; struct smb2_symlink_err_rsp *sym = ERR_PTR(-EINVAL); + u8 *end = (u8 *)err + iov->iov_len; u32 len; if (err->ErrorContextCount) { - struct smb2_error_context_rsp *p, *end; + struct smb2_error_context_rsp *p; len = (u32)err->ErrorContextCount * (offsetof(struct smb2_error_context_rsp, ErrorContextData) + @@ -39,8 +40,7 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) return ERR_PTR(-EINVAL); p = (struct smb2_error_context_rsp *)err->ErrorData; - end = (struct smb2_error_context_rsp *)((u8 *)err + iov->iov_len); - do { + while ((u8 *)p + sizeof(*p) <= end) { if (le32_to_cpu(p->ErrorId) == SMB2_ERROR_ID_DEFAULT) { sym = (struct smb2_symlink_err_rsp *)p->ErrorContextData; break; @@ -50,14 +50,16 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); p = (struct smb2_error_context_rsp *)(p->ErrorContextData + len); - } while (p < end); + } } else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) && iov->iov_len >= SMB2_SYMLINK_STRUCT_SIZE) { sym = (struct smb2_symlink_err_rsp *)err->ErrorData; } - if (!IS_ERR(sym) && (le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || - le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) + if (!IS_ERR(sym) && + ((u8 *)sym + sizeof(*sym) > end || + le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || + le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) sym = ERR_PTR(-EINVAL); return sym; @@ -128,8 +130,10 @@ int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec print_len = le16_to_cpu(sym->PrintNameLength); print_offs = le16_to_cpu(sym->PrintNameOffset); - if (iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offs + sub_len || - iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + print_offs + print_len) + if ((char *)sym->PathBuffer + sub_offs + sub_len > + (char *)iov->iov_base + iov->iov_len || + (char *)sym->PathBuffer + print_offs + print_len > + (char *)iov->iov_base + iov->iov_len) return -EINVAL; return smb2_parse_native_symlink(path, @@ -177,6 +181,9 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, &err_buftype); if (rc == -EACCES && retry_without_read_attributes) { + free_rsp_buf(err_buftype, err_iov.iov_base); + memset(&err_iov, 0, sizeof(err_iov)); + err_buftype = CIFS_NO_BUFFER; oparms->desired_access &= ~FILE_READ_ATTRIBUTES; rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, &err_buftype); diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 69cb81fa0d3ab..17052b988951f 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -125,7 +125,7 @@ static int check_wsl_eas(struct kvec *rsp_iov) nlen = ea->ea_name_length; vlen = le16_to_cpu(ea->ea_value_length); if (nlen != SMB2_WSL_XATTR_NAME_LEN || - (u8 *)ea + nlen + 1 + vlen > end) + (u8 *)ea->ea_data + nlen + 1 + vlen > end) return -EINVAL; switch (vlen) { @@ -322,7 +322,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, cfile->fid.volatile_fid, SMB_FIND_FILE_POSIX_INFO, SMB2_O_INFO_FILE, 0, - sizeof(struct smb311_posix_qinfo *) + + sizeof(struct smb311_posix_qinfo) + (PATH_MAX * 2) + (sizeof(struct smb_sid) * 2), 0, NULL); } else { @@ -332,7 +332,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, SMB_FIND_FILE_POSIX_INFO, SMB2_O_INFO_FILE, 0, - sizeof(struct smb311_posix_qinfo *) + + sizeof(struct smb311_posix_qinfo) + (PATH_MAX * 2) + (sizeof(struct smb_sid) * 2), 0, NULL); } @@ -1205,6 +1205,7 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, memset(resp_buftype, 0, sizeof(resp_buftype)); memset(rsp_iov, 0, sizeof(rsp_iov)); + memset(open_iov, 0, sizeof(open_iov)); rqst[0].rq_iov = open_iov; rqst[0].rq_nvec = ARRAY_SIZE(open_iov); @@ -1229,14 +1230,15 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, creq = rqst[0].rq_iov[0].iov_base; creq->ShareAccess = FILE_SHARE_DELETE_LE; + memset(&close_iov, 0, sizeof(close_iov)); rqst[1].rq_iov = &close_iov; rqst[1].rq_nvec = 1; rc = SMB2_close_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID, false); - smb2_set_related(&rqst[1]); if (rc) goto err_free; + smb2_set_related(&rqst[1]); if (retries) { for (int i = 0; i < ARRAY_SIZE(rqst); i++) diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 96bfe4c63ccf9..a6fc8ff398ebf 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -819,14 +819,14 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, int rc; cifs_dbg(FYI, "%s: tc_count=%d\n", __func__, tcon->tc_count); - spin_lock(&cifs_tcp_ses_lock); + spin_lock(&tcon->tc_lock); if (tcon->tc_count <= 0) { struct TCP_Server_Info *server = NULL; trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_see_cancelled_close); WARN_ONCE(tcon->tc_count < 0, "tcon refcount is negative"); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&tcon->tc_lock); if (tcon->ses) { server = tcon->ses->server; @@ -840,7 +840,7 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, tcon->tc_count++; trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_cancelled_close); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&tcon->tc_lock); rc = __smb2_handle_cancelled_cmd(tcon, SMB2_CLOSE_HE, 0, persistent_fid, volatile_fid); diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 1e39f2165e427..0497c97195644 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -628,6 +628,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, struct iface_info_ipv6 *p6; struct cifs_server_iface *info = NULL, *iface = NULL, *niface = NULL; struct cifs_server_iface tmp_iface; + __be16 port; ssize_t bytes_left; size_t next = 0; int nb_iface = 0; @@ -637,13 +638,6 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, p = buf; spin_lock(&ses->iface_lock); - /* do not query too frequently, this time with lock held */ - if (ses->iface_last_update && - time_before(jiffies, ses->iface_last_update + - (SMB_INTERFACE_POLL_INTERVAL * HZ))) { - spin_unlock(&ses->iface_lock); - return 0; - } /* * Go through iface_list and mark them as inactive @@ -666,10 +660,18 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, "Empty network interface list returned by server %s\n", ses->server->hostname); rc = -EOPNOTSUPP; - ses->iface_last_update = jiffies; goto out; } + spin_lock(&ses->server->srv_lock); + if (ses->server->dstaddr.ss_family == AF_INET) + port = ((struct sockaddr_in *)&ses->server->dstaddr)->sin_port; + else if (ses->server->dstaddr.ss_family == AF_INET6) + port = ((struct sockaddr_in6 *)&ses->server->dstaddr)->sin6_port; + else + port = cpu_to_be16(CIFS_PORT); + spin_unlock(&ses->server->srv_lock); + while (bytes_left >= (ssize_t)sizeof(*p)) { memset(&tmp_iface, 0, sizeof(tmp_iface)); /* default to 1Gbps when link speed is unset */ @@ -690,7 +692,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, memcpy(&addr4->sin_addr, &p4->IPv4Address, 4); /* [MS-SMB2] 2.2.32.5.1.1 Clients MUST ignore these */ - addr4->sin_port = cpu_to_be16(CIFS_PORT); + addr4->sin_port = port; cifs_dbg(FYI, "%s: ipv4 %pI4\n", __func__, &addr4->sin_addr); @@ -704,7 +706,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, /* [MS-SMB2] 2.2.32.5.1.2 Clients MUST ignore these */ addr6->sin6_flowinfo = 0; addr6->sin6_scope_id = 0; - addr6->sin6_port = cpu_to_be16(CIFS_PORT); + addr6->sin6_port = port; cifs_dbg(FYI, "%s: ipv6 %pI6\n", __func__, &addr6->sin6_addr); @@ -795,8 +797,6 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, + sizeof(p->Next) && p->Next)) cifs_dbg(VFS, "%s: incomplete interface info\n", __func__); - ses->iface_last_update = jiffies; - out: /* * Go through the list again and put the inactive entries @@ -825,10 +825,17 @@ SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_ struct TCP_Server_Info *pserver; /* do not query too frequently */ + spin_lock(&ses->iface_lock); if (ses->iface_last_update && time_before(jiffies, ses->iface_last_update + - (SMB_INTERFACE_POLL_INTERVAL * HZ))) + (SMB_INTERFACE_POLL_INTERVAL * HZ))) { + spin_unlock(&ses->iface_lock); return 0; + } + + ses->iface_last_update = jiffies; + + spin_unlock(&ses->iface_lock); rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID, FSCTL_QUERY_NETWORK_INTERFACE_INFO, @@ -1187,6 +1194,7 @@ smb2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + used_len = 0; flags = CIFS_CP_CREATE_CLOSE_OP; oplock = SMB2_OPLOCK_LEVEL_NONE; server = cifs_pick_channel(ses); @@ -1484,6 +1492,7 @@ smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon, { struct smb2_file_network_open_info file_inf; struct inode *inode; + u64 asize; int rc; rc = __SMB2_close(xid, tcon, cfile->fid.persistent_fid, @@ -1507,14 +1516,9 @@ smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon, inode_set_atime_to_ts(inode, cifs_NTtimeToUnix(file_inf.LastAccessTime)); - /* - * i_blocks is not related to (i_size / i_blksize), - * but instead 512 byte (2**9) size is required for - * calculating num blocks. - */ - if (le64_to_cpu(file_inf.AllocationSize) > 4096) - inode->i_blocks = - (512 - 1 + le64_to_cpu(file_inf.AllocationSize)) >> 9; + asize = le64_to_cpu(file_inf.AllocationSize); + if (asize > 4096) + inode->i_blocks = CIFS_INO_BLOCKS(asize); /* End of file and Attributes should not have to be updated on close */ spin_unlock(&inode->i_lock); @@ -1585,6 +1589,7 @@ smb2_ioctl_query_info(const unsigned int xid, replay_again: /* reinitialize for possible replay */ + buffer = NULL; flags = CIFS_CP_CREATE_CLOSE_OP; oplock = SMB2_OPLOCK_LEVEL_NONE; server = cifs_pick_channel(ses); @@ -1770,6 +1775,12 @@ smb2_ioctl_query_info(const unsigned int xid, qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; if (le32_to_cpu(qi_rsp->OutputBufferLength) < qi.input_buffer_length) qi.input_buffer_length = le32_to_cpu(qi_rsp->OutputBufferLength); + if (qi.input_buffer_length > 0 && + struct_size(qi_rsp, Buffer, qi.input_buffer_length) > + rsp_iov[1].iov_len) { + rc = -EFAULT; + goto out; + } if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length, sizeof(qi.input_buffer_length))) { @@ -2174,14 +2185,6 @@ smb2_duplicate_extents(const unsigned int xid, rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); if (rc) goto duplicate_extents_out; - - /* - * Although also could set plausible allocation size (i_blocks) - * here in addition to setting the file size, in reflink - * it is likely that the target file is sparse. Its allocation - * size will be queried on next revalidate, but it is important - * to make sure that file's cached size is updated immediately - */ netfs_resize_file(netfs_inode(inode), dest_off + len, true); cifs_setsize(inode, dest_off + len); } @@ -3077,7 +3080,9 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, struct cifs_tcon, tcon_list); if (tcon) { + spin_lock(&tcon->tc_lock); tcon->tc_count++; + spin_unlock(&tcon->tc_lock); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_dfs_refer); } @@ -3146,13 +3151,9 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, out: if (tcon && !tcon->ipc) { /* ipc tcons are not refcounted */ - spin_lock(&cifs_tcp_ses_lock); - tcon->tc_count--; + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_dfs_refer); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_dec_dfs_refer); - /* tc_count can never go negative */ - WARN_ON(tcon->tc_count < 0); - spin_unlock(&cifs_tcp_ses_lock); } kfree(utf16_path); kfree(dfs_req); diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 8b4a4573e9c37..3e49c5b396b5a 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -1650,19 +1650,17 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) is_binding = (ses->ses_status == SES_GOOD); spin_unlock(&ses->ses_lock); - /* keep session key if binding */ - if (!is_binding) { - kfree_sensitive(ses->auth_key.response); - ses->auth_key.response = kmemdup(msg->data, msg->sesskey_len, - GFP_KERNEL); - if (!ses->auth_key.response) { - cifs_dbg(VFS, "Kerberos can't allocate (%u bytes) memory\n", - msg->sesskey_len); - rc = -ENOMEM; - goto out_put_spnego_key; - } - ses->auth_key.len = msg->sesskey_len; + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = kmemdup(msg->data, + msg->sesskey_len, + GFP_KERNEL); + if (!ses->auth_key.response) { + cifs_dbg(VFS, "%s: can't allocate (%u bytes) memory\n", + __func__, msg->sesskey_len); + rc = -ENOMEM; + goto out_put_spnego_key; } + ses->auth_key.len = msg->sesskey_len; sess_data->iov[1].iov_base = msg->data + msg->sesskey_len; sess_data->iov[1].iov_len = msg->secblob_len; @@ -2845,6 +2843,7 @@ int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, replay_again: /* reinitialize for possible replay */ + pc_buf = NULL; flags = 0; n_iov = 2; server = cifs_pick_channel(ses); @@ -3916,7 +3915,7 @@ int SMB311_posix_query_info(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, struct smb311_posix_qinfo *data, u32 *plen) { - size_t output_len = sizeof(struct smb311_posix_qinfo *) + + size_t output_len = sizeof(struct smb311_posix_qinfo) + (sizeof(struct smb_sid) * 2) + (PATH_MAX * 2); *plen = 0; @@ -4174,7 +4173,9 @@ void smb2_reconnect_server(struct work_struct *work) list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { if (tcon->need_reconnect || tcon->need_reopen_files) { + spin_lock(&tcon->tc_lock); tcon->tc_count++; + spin_unlock(&tcon->tc_lock); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_reconnect_server); list_add_tail(&tcon->rlist, &tmp_list); @@ -4629,7 +4630,7 @@ smb2_readv_callback(struct mid_q_entry *mid) } else { size_t trans = rdata->subreq.transferred + rdata->got_bytes; if (trans < rdata->subreq.len && - rdata->subreq.start + trans == ictx->remote_i_size) { + rdata->subreq.start + trans >= ictx->remote_i_size) { __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); rdata->result = 0; } @@ -5171,7 +5172,10 @@ SMB2_write(const unsigned int xid, struct cifs_io_parms *io_parms, memset(&rqst, 0, sizeof(struct smb_rqst)); rqst.rq_iov = iov; - rqst.rq_nvec = n_vec + 1; + /* iov[0] is the SMB header; move payload to rq_iter for encryption safety */ + rqst.rq_nvec = 1; + iov_iter_kvec(&rqst.rq_iter, ITER_SOURCE, &iov[1], n_vec, + io_parms->length); if (retries) smb2_set_replay(server, &rqst); diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c index 6a9b80385b86b..211305d43f8d4 100644 --- a/fs/smb/client/smb2transport.c +++ b/fs/smb/client/smb2transport.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "cifsglob.h" #include "cifsproto.h" #include "smb2proto.h" @@ -617,7 +618,8 @@ smb2_verify_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server) if (rc) return rc; - if (memcmp(server_response_sig, shdr->Signature, SMB2_SIGNATURE_SIZE)) { + if (crypto_memneq(server_response_sig, shdr->Signature, + SMB2_SIGNATURE_SIZE)) { cifs_dbg(VFS, "sign fail cmd 0x%x message id 0x%llx\n", shdr->Command, shdr->MessageId); return -EACCES; diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index c6c428c2e08dd..ff44a2dc49938 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -7,6 +7,7 @@ #include #include #include +#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) smbd_disconnect_rdma_connection(__sc) #include "../common/smbdirect/smbdirect_pdu.h" #include "smbdirect.h" #include "cifs_debug.h" @@ -34,6 +35,10 @@ static void enqueue_reassembly( static struct smbdirect_recv_io *_get_first_reassembly( struct smbdirect_socket *sc); +static int smbd_post_send(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *request); + static int smbd_post_recv( struct smbdirect_socket *sc, struct smbdirect_recv_io *response); @@ -96,8 +101,23 @@ int smbd_send_credit_target = 255; /* The maximum single message size can be sent to remote peer */ int smbd_max_send_size = 1364; -/* The maximum fragmented upper-layer payload receive size supported */ -int smbd_max_fragmented_recv_size = 1024 * 1024; +/* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smbd_max_receive_size - 24 = 1340 + * + * The maximum number would be + * smbd_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + */ +int smbd_max_fragmented_recv_size = (1364 * 255) / 2; /* The maximum single-message size which can be received */ int smbd_max_receive_size = 1364; @@ -186,6 +206,9 @@ static void smbd_disconnect_rdma_work(struct work_struct *work) struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, disconnect_work); + if (sc->first_error == 0) + sc->first_error = -ECONNABORTED; + /* * make sure this and other work is not queued again * but here we don't block and avoid @@ -197,9 +220,6 @@ static void smbd_disconnect_rdma_work(struct work_struct *work) disable_work(&sc->idle.immediate_work); disable_delayed_work(&sc->idle.timer_work); - if (sc->first_error == 0) - sc->first_error = -ECONNABORTED; - switch (sc->status) { case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: @@ -242,6 +262,9 @@ static void smbd_disconnect_rdma_work(struct work_struct *work) static void smbd_disconnect_rdma_connection(struct smbdirect_socket *sc) { + if (sc->first_error == 0) + sc->first_error = -ECONNABORTED; + /* * make sure other work (than disconnect_work) is * not queued again but here we don't block and avoid @@ -252,9 +275,6 @@ static void smbd_disconnect_rdma_connection(struct smbdirect_socket *sc) disable_work(&sc->idle.immediate_work); disable_delayed_work(&sc->idle.timer_work); - if (sc->first_error == 0) - sc->first_error = -ECONNABORTED; - switch (sc->status) { case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: @@ -322,27 +342,27 @@ static int smbd_conn_upcall( switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING); + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING)) + break; sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED; wake_up(&sc->status_wait); break; case RDMA_CM_EVENT_ROUTE_RESOLVED: - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING); + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING)) + break; sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; wake_up(&sc->status_wait); break; case RDMA_CM_EVENT_ADDR_ERROR: log_rdma_event(ERR, "connecting failed event=%s\n", event_name); - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING); sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; smbd_disconnect_rdma_work(&sc->disconnect_work); break; case RDMA_CM_EVENT_ROUTE_ERROR: log_rdma_event(ERR, "connecting failed event=%s\n", event_name); - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING); sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; smbd_disconnect_rdma_work(&sc->disconnect_work); break; @@ -428,7 +448,8 @@ static int smbd_conn_upcall( min_t(u8, sp->responder_resources, peer_responder_resources); - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) + break; sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; wake_up(&sc->status_wait); break; @@ -437,7 +458,6 @@ static int smbd_conn_upcall( case RDMA_CM_EVENT_UNREACHABLE: case RDMA_CM_EVENT_REJECTED: log_rdma_event(ERR, "connecting failed event=%s\n", event_name); - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; smbd_disconnect_rdma_work(&sc->disconnect_work); break; @@ -492,27 +512,103 @@ static inline void *smbdirect_recv_io_payload(struct smbdirect_recv_io *response return (void *)response->packet; } +static struct smbdirect_send_io *smbd_alloc_send_io(struct smbdirect_socket *sc) +{ + struct smbdirect_send_io *msg; + + msg = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); + if (!msg) + return ERR_PTR(-ENOMEM); + msg->socket = sc; + INIT_LIST_HEAD(&msg->sibling_list); + msg->num_sge = 0; + + return msg; +} + +static void smbd_free_send_io(struct smbdirect_send_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + size_t i; + + /* + * The list needs to be empty! + * The caller should take care of it. + */ + WARN_ON_ONCE(!list_empty(&msg->sibling_list)); + + /* + * Note we call ib_dma_unmap_page(), even if some sges are mapped using + * ib_dma_map_single(). + * + * The difference between _single() and _page() only matters for the + * ib_dma_map_*() case. + * + * For the ib_dma_unmap_*() case it does not matter as both take the + * dma_addr_t and dma_unmap_single_attrs() is just an alias to + * dma_unmap_page_attrs(). + */ + for (i = 0; i < msg->num_sge; i++) + ib_dma_unmap_page(sc->ib.dev, + msg->sge[i].addr, + msg->sge[i].length, + DMA_TO_DEVICE); + + mempool_free(msg, sc->send_io.mem.pool); +} + /* Called when a RDMA send is done */ static void send_done(struct ib_cq *cq, struct ib_wc *wc) { - int i; struct smbdirect_send_io *request = container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); struct smbdirect_socket *sc = request->socket; + struct smbdirect_send_io *sibling, *next; int lcredits = 0; log_rdma_send(INFO, "smbdirect_send_io 0x%p completed wc->status=%s\n", request, ib_wc_status_msg(wc->status)); - for (i = 0; i < request->num_sge; i++) - ib_dma_unmap_single(sc->ib.dev, - request->sge[i].addr, - request->sge[i].length, - DMA_TO_DEVICE); - mempool_free(request, sc->send_io.mem.pool); + if (unlikely(!(request->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + log_rdma_send(ERR, + "unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbd_disconnect_rdma_connection(sc); + return; + } + + /* + * Free possible siblings and then the main send_io + */ + list_for_each_entry_safe(sibling, next, &request->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbd_free_send_io(sibling); + lcredits += 1; + } + /* Note this frees wc->wr_cqe, but not wc */ + smbd_free_send_io(request); lcredits += 1; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { +skip_free: if (wc->status != IB_WC_WR_FLUSH_ERR) log_rdma_send(ERR, "wc->status=%s wc->opcode=%d\n", ib_wc_status_msg(wc->status), wc->opcode); @@ -607,6 +703,7 @@ static bool process_negotiation_response( sp->max_frmr_depth * PAGE_SIZE); sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; + atomic_set(&sc->send_io.bcredits.count, 1); sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; return true; } @@ -617,6 +714,7 @@ static void smbd_post_send_credits(struct work_struct *work) struct smbdirect_recv_io *response; struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); + int posted = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { return; @@ -639,9 +737,21 @@ static void smbd_post_send_credits(struct work_struct *work) } atomic_inc(&sc->recv_io.posted.count); + posted += 1; } } + atomic_add(posted, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (posted && + atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + /* Promptly send an immediate packet as defined in [MS-SMBD] 3.1.1.1 */ if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target - 1) { @@ -658,6 +768,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); struct smbdirect_socket *sc = response->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; + int current_recv_credits; u16 old_recv_credit_target; u32 data_offset = 0; u32 data_length = 0; @@ -699,7 +810,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) negotiate_done = process_negotiation_response(response, wc->byte_len); put_receive_buffer(sc, response); - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING); + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)) + negotiate_done = false; if (!negotiate_done) { sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; smbd_disconnect_rdma_connection(sc); @@ -741,7 +853,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } atomic_dec(&sc->recv_io.posted.count); - atomic_dec(&sc->recv_io.credits.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); + old_recv_credit_target = sc->recv_io.credits.target; sc->recv_io.credits.target = le16_to_cpu(data_transfer->credits_requested); @@ -777,7 +890,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) * reassembly queue and wake up the reading thread */ if (data_length) { - if (sc->recv_io.credits.target > old_recv_credit_target) + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); enqueue_reassembly(sc, response, data_length); @@ -953,16 +1067,13 @@ static int smbd_ia_open( static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) { struct smbdirect_socket_parameters *sp = &sc->parameters; - struct ib_send_wr send_wr; - int rc = -ENOMEM; + int rc; struct smbdirect_send_io *request; struct smbdirect_negotiate_req *packet; - request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); - if (!request) - return rc; - - request->socket = sc; + request = smbd_alloc_send_io(sc); + if (IS_ERR(request)) + return PTR_ERR(request); packet = smbdirect_send_io_payload(request); packet->min_version = cpu_to_le16(SMBDIRECT_V1); @@ -974,7 +1085,6 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) packet->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); - request->num_sge = 1; request->sge[0].addr = ib_dma_map_single( sc->ib.dev, (void *)packet, sizeof(*packet), DMA_TO_DEVICE); @@ -982,42 +1092,20 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) rc = -EIO; goto dma_mapping_failed; } + request->num_sge = 1; request->sge[0].length = sizeof(*packet); request->sge[0].lkey = sc->ib.pd->local_dma_lkey; - ib_dma_sync_single_for_device( - sc->ib.dev, request->sge[0].addr, - request->sge[0].length, DMA_TO_DEVICE); - - request->cqe.done = send_done; - - send_wr.next = NULL; - send_wr.wr_cqe = &request->cqe; - send_wr.sg_list = request->sge; - send_wr.num_sge = request->num_sge; - send_wr.opcode = IB_WR_SEND; - send_wr.send_flags = IB_SEND_SIGNALED; - - log_rdma_send(INFO, "sge addr=0x%llx length=%u lkey=0x%x\n", - request->sge[0].addr, - request->sge[0].length, request->sge[0].lkey); - - atomic_inc(&sc->send_io.pending.count); - rc = ib_post_send(sc->ib.qp, &send_wr, NULL); + rc = smbd_post_send(sc, NULL, request); if (!rc) return 0; - /* if we reach here, post send failed */ - log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); - atomic_dec(&sc->send_io.pending.count); - ib_dma_unmap_single(sc->ib.dev, request->sge[0].addr, - request->sge[0].length, DMA_TO_DEVICE); - - smbd_disconnect_rdma_connection(sc); + if (rc == -EAGAIN) + rc = -EIO; dma_mapping_failed: - mempool_free(request, sc->send_io.mem.pool); + smbd_free_send_io(request); return rc; } @@ -1031,19 +1119,38 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) */ static int manage_credits_prior_sending(struct smbdirect_socket *sc) { + int missing; + int available; int new_credits; if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) return 0; - new_credits = atomic_read(&sc->recv_io.posted.count); - if (new_credits == 0) + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = (u16)min3(U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); return 0; + } - new_credits -= atomic_read(&sc->recv_io.credits.count); - if (new_credits <= 0) - return 0; + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + /* + * Remember we granted the credits + */ + atomic_add(new_credits, &sc->recv_io.credits.count); return new_credits; } @@ -1073,12 +1180,27 @@ static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) return 0; } +static int smbd_ib_post_send(struct smbdirect_socket *sc, + struct ib_send_wr *wr) +{ + int ret; + + atomic_inc(&sc->send_io.pending.count); + ret = ib_post_send(sc->ib.qp, wr, NULL); + if (ret) { + pr_err("failed to post send: %d\n", ret); + smbd_disconnect_rdma_connection(sc); + ret = -EAGAIN; + } + return ret; +} + /* Post the send request */ static int smbd_post_send(struct smbdirect_socket *sc, - struct smbdirect_send_io *request) + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *request) { - struct ib_send_wr send_wr; - int rc, i; + int i; for (i = 0; i < request->num_sge; i++) { log_rdma_send(INFO, @@ -1092,79 +1214,245 @@ static int smbd_post_send(struct smbdirect_socket *sc, } request->cqe.done = send_done; + request->wr.next = NULL; + request->wr.sg_list = request->sge; + request->wr.num_sge = request->num_sge; + request->wr.opcode = IB_WR_SEND; + + if (batch) { + request->wr.wr_cqe = NULL; + request->wr.send_flags = 0; + if (!list_empty(&batch->msg_list)) { + struct smbdirect_send_io *last; + + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last->wr.next = &request->wr; + } + list_add_tail(&request->sibling_list, &batch->msg_list); + batch->wr_cnt++; + return 0; + } - send_wr.next = NULL; - send_wr.wr_cqe = &request->cqe; - send_wr.sg_list = request->sge; - send_wr.num_sge = request->num_sge; - send_wr.opcode = IB_WR_SEND; - send_wr.send_flags = IB_SEND_SIGNALED; + request->wr.wr_cqe = &request->cqe; + request->wr.send_flags = IB_SEND_SIGNALED; + return smbd_ib_post_send(sc, &request->wr); +} - rc = ib_post_send(sc->ib.qp, &send_wr, NULL); - if (rc) { - log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); - smbd_disconnect_rdma_connection(sc); - rc = -EAGAIN; +static void smbd_send_batch_init(struct smbdirect_send_batch *batch, + bool need_invalidate_rkey, + unsigned int remote_key) +{ + INIT_LIST_HEAD(&batch->msg_list); + batch->wr_cnt = 0; + batch->need_invalidate_rkey = need_invalidate_rkey; + batch->remote_key = remote_key; + batch->credit = 0; +} + +static int smbd_send_batch_flush(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + bool is_last) +{ + struct smbdirect_send_io *first, *last; + int ret = 0; + + if (list_empty(&batch->msg_list)) + goto release_credit; + + first = list_first_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + + if (batch->need_invalidate_rkey) { + first->wr.opcode = IB_WR_SEND_WITH_INV; + first->wr.ex.invalidate_rkey = batch->remote_key; + batch->need_invalidate_rkey = false; + batch->remote_key = 0; } - return rc; + last->wr.send_flags = IB_SEND_SIGNALED; + last->wr.wr_cqe = &last->cqe; + + /* + * Remove last from batch->msg_list + * and splice the rest of batch->msg_list + * to last->sibling_list. + * + * batch->msg_list is a valid empty list + * at the end. + */ + list_del_init(&last->sibling_list); + list_splice_tail_init(&batch->msg_list, &last->sibling_list); + batch->wr_cnt = 0; + + ret = smbd_ib_post_send(sc, &first->wr); + if (ret) { + struct smbdirect_send_io *sibling, *next; + + list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbd_free_send_io(sibling); + } + smbd_free_send_io(last); + } + +release_credit: + if (is_last && !ret && batch->credit) { + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + + return ret; +} + +static int wait_for_credits(struct smbdirect_socket *sc, + wait_queue_head_t *waitq, atomic_t *total_credits, + int needed) +{ + int ret; + + do { + if (atomic_sub_return(needed, total_credits) >= 0) + return 0; + + atomic_add(needed, total_credits); + ret = wait_event_interruptible(*waitq, + atomic_read(total_credits) >= needed || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return -ENOTCONN; + else if (ret < 0) + return ret; + } while (true); +} + +static int wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + int ret; + + if (batch->credit) + return 0; + + ret = wait_for_credits(sc, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + batch->credit = 1; + return 0; +} + +static int wait_for_send_lcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && (atomic_read(&sc->send_io.lcredits.count) <= 1)) { + int ret; + + ret = smbd_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return wait_for_credits(sc, + &sc->send_io.lcredits.wait_queue, + &sc->send_io.lcredits.count, + 1); +} + +static int wait_for_send_credits(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && + (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { + int ret; + + ret = smbd_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return wait_for_credits(sc, + &sc->send_io.credits.wait_queue, + &sc->send_io.credits.count, + 1); } static int smbd_post_send_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, struct iov_iter *iter, int *_remaining_data_length) { struct smbdirect_socket_parameters *sp = &sc->parameters; - int i, rc; + int rc; int header_length; int data_length; struct smbdirect_send_io *request; struct smbdirect_data_transfer *packet; int new_credits = 0; + struct smbdirect_send_batch _batch; -wait_lcredit: - /* Wait for local send credits */ - rc = wait_event_interruptible(sc->send_io.lcredits.wait_queue, - atomic_read(&sc->send_io.lcredits.count) > 0 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (rc) - goto err_wait_lcredit; + if (!batch) { + smbd_send_batch_init(&_batch, false, 0); + batch = &_batch; + } - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - log_outgoing(ERR, "disconnected not sending on wait_credit\n"); + rc = wait_for_send_bcredit(sc, batch); + if (rc) { + log_outgoing(ERR, "disconnected not sending on wait_bcredit\n"); rc = -EAGAIN; - goto err_wait_lcredit; - } - if (unlikely(atomic_dec_return(&sc->send_io.lcredits.count) < 0)) { - atomic_inc(&sc->send_io.lcredits.count); - goto wait_lcredit; + goto err_wait_bcredit; } -wait_credit: - /* Wait for send credits. A SMBD packet needs one credit */ - rc = wait_event_interruptible(sc->send_io.credits.wait_queue, - atomic_read(&sc->send_io.credits.count) > 0 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (rc) - goto err_wait_credit; + rc = wait_for_send_lcredit(sc, batch); + if (rc) { + log_outgoing(ERR, "disconnected not sending on wait_lcredit\n"); + rc = -EAGAIN; + goto err_wait_lcredit; + } - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + rc = wait_for_send_credits(sc, batch); + if (rc) { log_outgoing(ERR, "disconnected not sending on wait_credit\n"); rc = -EAGAIN; goto err_wait_credit; } - if (unlikely(atomic_dec_return(&sc->send_io.credits.count) < 0)) { - atomic_inc(&sc->send_io.credits.count); - goto wait_credit; + + new_credits = manage_credits_prior_sending(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); + rc = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + rc = -ENOTCONN; + if (rc < 0) { + log_outgoing(ERR, "disconnected not sending on last credit\n"); + rc = -EAGAIN; + goto err_wait_credit; + } + + new_credits = manage_credits_prior_sending(sc); } - request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); - if (!request) { - rc = -ENOMEM; + request = smbd_alloc_send_io(sc); + if (IS_ERR(request)) { + rc = PTR_ERR(request); goto err_alloc; } - request->socket = sc; memset(request->sge, 0, sizeof(request->sge)); /* Map the packet to DMA */ @@ -1213,9 +1501,6 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, /* Fill in the packet header */ packet->credits_requested = cpu_to_le16(sp->send_credit_target); - - new_credits = manage_credits_prior_sending(sc); - atomic_add(new_credits, &sc->recv_io.credits.count); packet->credits_granted = cpu_to_le16(new_credits); packet->flags = 0; @@ -1238,33 +1523,27 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, le32_to_cpu(packet->data_length), le32_to_cpu(packet->remaining_data_length)); - /* - * Now that we got a local and a remote credit - * we add us as pending - */ - atomic_inc(&sc->send_io.pending.count); + rc = smbd_post_send(sc, batch, request); + if (!rc) { + /* + * From here request is moved to batch + * and we should not free it explicitly. + */ - rc = smbd_post_send(sc, request); - if (!rc) - return 0; + if (batch != &_batch) + return 0; - if (atomic_dec_and_test(&sc->send_io.pending.count)) - wake_up(&sc->send_io.pending.zero_wait_queue); + rc = smbd_send_batch_flush(sc, batch, true); + if (!rc) + return 0; - wake_up(&sc->send_io.pending.dec_wait_queue); + goto err_flush; + } err_dma: - for (i = 0; i < request->num_sge; i++) - if (request->sge[i].addr) - ib_dma_unmap_single(sc->ib.dev, - request->sge[i].addr, - request->sge[i].length, - DMA_TO_DEVICE); - mempool_free(request, sc->send_io.mem.pool); - - /* roll back the granted receive credits */ - atomic_sub(new_credits, &sc->recv_io.credits.count); + smbd_free_send_io(request); +err_flush: err_alloc: atomic_inc(&sc->send_io.credits.count); wake_up(&sc->send_io.credits.wait_queue); @@ -1274,6 +1553,11 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, wake_up(&sc->send_io.lcredits.wait_queue); err_wait_lcredit: + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + +err_wait_bcredit: return rc; } @@ -1287,10 +1571,11 @@ static int smbd_post_send_empty(struct smbdirect_socket *sc) int remaining_data_length = 0; sc->statistics.send_empty++; - return smbd_post_send_iter(sc, NULL, &remaining_data_length); + return smbd_post_send_iter(sc, NULL, NULL, &remaining_data_length); } static int smbd_post_send_full_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, struct iov_iter *iter, int *_remaining_data_length) { @@ -1303,7 +1588,7 @@ static int smbd_post_send_full_iter(struct smbdirect_socket *sc, */ while (iov_iter_count(iter) > 0) { - rc = smbd_post_send_iter(sc, iter, _remaining_data_length); + rc = smbd_post_send_iter(sc, batch, iter, _remaining_data_length); if (rc < 0) break; } @@ -2225,8 +2510,10 @@ int smbd_send(struct TCP_Server_Info *server, struct smbdirect_socket_parameters *sp = &sc->parameters; struct smb_rqst *rqst; struct iov_iter iter; + struct smbdirect_send_batch batch; unsigned int remaining_data_length, klen; int rc, i, rqst_idx; + int error = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -EAGAIN; @@ -2251,6 +2538,7 @@ int smbd_send(struct TCP_Server_Info *server, num_rqst, remaining_data_length); rqst_idx = 0; + smbd_send_batch_init(&batch, false, 0); do { rqst = &rqst_array[rqst_idx]; @@ -2269,20 +2557,28 @@ int smbd_send(struct TCP_Server_Info *server, klen += rqst->rq_iov[i].iov_len; iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen); - rc = smbd_post_send_full_iter(sc, &iter, &remaining_data_length); - if (rc < 0) + rc = smbd_post_send_full_iter(sc, &batch, &iter, &remaining_data_length); + if (rc < 0) { + error = rc; break; + } if (iov_iter_count(&rqst->rq_iter) > 0) { /* And then the data pages if there are any */ - rc = smbd_post_send_full_iter(sc, &rqst->rq_iter, + rc = smbd_post_send_full_iter(sc, &batch, &rqst->rq_iter, &remaining_data_length); - if (rc < 0) + if (rc < 0) { + error = rc; break; + } } } while (++rqst_idx < num_rqst); + rc = smbd_send_batch_flush(sc, &batch, true); + if (unlikely(!rc && error)) + rc = error; + /* * As an optimization, we don't wait for individual I/O to finish * before sending the next one. diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index 28e00c34df1cb..79a246af664d2 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -47,6 +47,7 @@ EM(netfs_trace_tcon_ref_get_cached_laundromat, "GET Ch-Lau") \ EM(netfs_trace_tcon_ref_get_cached_lease_break, "GET Ch-Lea") \ EM(netfs_trace_tcon_ref_get_cancelled_close, "GET Cn-Cls") \ + EM(netfs_trace_tcon_ref_get_close_defer_files, "GET Cl-Def") \ EM(netfs_trace_tcon_ref_get_dfs_refer, "GET DfsRef") \ EM(netfs_trace_tcon_ref_get_find, "GET Find ") \ EM(netfs_trace_tcon_ref_get_find_sess_tcon, "GET FndSes") \ @@ -58,7 +59,9 @@ EM(netfs_trace_tcon_ref_put_cancelled_close, "PUT Cn-Cls") \ EM(netfs_trace_tcon_ref_put_cancelled_close_fid, "PUT Cn-Fid") \ EM(netfs_trace_tcon_ref_put_cancelled_mid, "PUT Cn-Mid") \ + EM(netfs_trace_tcon_ref_put_close_defer_files, "PUT Cl-Def") \ EM(netfs_trace_tcon_ref_put_mnt_ctx, "PUT MntCtx") \ + EM(netfs_trace_tcon_ref_put_dfs_refer, "PUT DfsRfr") \ EM(netfs_trace_tcon_ref_put_reconnect_server, "PUT Reconn") \ EM(netfs_trace_tcon_ref_put_tlink, "PUT Tlink ") \ EM(netfs_trace_tcon_ref_see_cancelled_close, "SEE Cn-Cls") \ diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index 915cedde5d664..2d7140767dc43 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -806,16 +806,21 @@ cifs_cancelled_callback(struct mid_q_entry *mid) } /* - * Return a channel (master if none) of @ses that can be used to send - * regular requests. + * cifs_pick_channel - pick an eligible channel for network operations * - * If we are currently binding a new channel (negprot/sess.setup), - * return the new incomplete channel. + * @ses: session reference + * + * Select an eligible channel (not terminating and not marked as needing + * reconnect), preferring the least loaded one. If no eligible channel is + * found, fall back to the primary channel (index 0). + * + * Return: TCP_Server_Info pointer for the chosen channel, or NULL if @ses is + * NULL. */ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) { uint index = 0; - unsigned int min_in_flight = UINT_MAX, max_in_flight = 0; + unsigned int min_in_flight = UINT_MAX; struct TCP_Server_Info *server = NULL; int i, start, cur; @@ -845,14 +850,8 @@ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) min_in_flight = server->in_flight; index = cur; } - if (server->in_flight > max_in_flight) - max_in_flight = server->in_flight; } - /* if all channels are equally loaded, fall back to round-robin */ - if (min_in_flight == max_in_flight) - index = (uint)start % ses->chan_count; - server = ses->chans[index].server; spin_unlock(&ses->chan_lock); diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h index ee5a90d691c89..ca22b7c2d3a7f 100644 --- a/fs/smb/common/smbdirect/smbdirect_socket.h +++ b/fs/smb/common/smbdirect/smbdirect_socket.h @@ -74,6 +74,19 @@ const char *smbdirect_socket_status_string(enum smbdirect_socket_status status) return ""; } +/* + * This can be used with %1pe to print errors as strings or '0' + * And it avoids warnings like: warn: passing zero to 'ERR_PTR' + * from smatch -p=kernel --pedantic + */ +static __always_inline +const void * __must_check SMBDIRECT_DEBUG_ERR_PTR(long error) +{ + if (error == 0) + return NULL; + return ERR_PTR(error); +} + enum smbdirect_keepalive_status { SMBDIRECT_KEEPALIVE_NONE, SMBDIRECT_KEEPALIVE_PENDING, @@ -141,6 +154,17 @@ struct smbdirect_socket { mempool_t *pool; } mem; + /* + * This is a coordination for smbdirect_send_batch. + * + * There's only one possible credit, which means + * only one instance is running at a time. + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } bcredits; + /* * The local credit state for ib_post_send() */ @@ -218,6 +242,7 @@ struct smbdirect_socket { */ struct { u16 target; + atomic_t available; atomic_t count; } credits; @@ -345,6 +370,9 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); disable_delayed_work_sync(&sc->idle.timer_work); + atomic_set(&sc->send_io.bcredits.count, 0); + init_waitqueue_head(&sc->send_io.bcredits.wait_queue); + atomic_set(&sc->send_io.lcredits.count, 0); init_waitqueue_head(&sc->send_io.lcredits.wait_queue); @@ -362,6 +390,7 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); disable_work_sync(&sc->recv_io.posted.refill_work); + atomic_set(&sc->recv_io.credits.available, 0); atomic_set(&sc->recv_io.credits.count, 0); INIT_LIST_HEAD(&sc->recv_io.reassembly.list); @@ -381,6 +410,44 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) init_waitqueue_head(&sc->mr_io.cleanup.wait_queue); } +#define __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, __error_cmd, __unexpected_cmd) ({ \ + bool __failed = false; \ + if (unlikely((__sc)->first_error)) { \ + __failed = true; \ + __error_cmd \ + } else if (unlikely((__sc)->status != (__expected_status))) { \ + __failed = true; \ + __unexpected_cmd \ + } \ + __failed; \ +}) + +#define __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, __unexpected_cmd) \ + __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, \ + , \ + { \ + const struct sockaddr_storage *__src = NULL; \ + const struct sockaddr_storage *__dst = NULL; \ + if ((__sc)->rdma.cm_id) { \ + __src = &(__sc)->rdma.cm_id->route.addr.src_addr; \ + __dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \ + } \ + WARN_ONCE(1, \ + "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ + smbdirect_socket_status_string(__expected_status), \ + smbdirect_socket_status_string((__sc)->status), \ + SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ + __src, __dst); \ + __unexpected_cmd \ + }) + +#define SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status) \ + __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, /* nothing */) + +#define SMBDIRECT_CHECK_STATUS_DISCONNECT(__sc, __expected_status) \ + __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, \ + __SMBDIRECT_SOCKET_DISCONNECT(__sc);) + struct smbdirect_send_io { struct smbdirect_socket *socket; struct ib_cqe cqe; @@ -420,6 +487,8 @@ struct smbdirect_send_batch { */ bool need_invalidate_rkey; u32 remote_key; + + int credit; }; struct smbdirect_recv_io { diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig index 098cac98d31e6..6200c71298f66 100644 --- a/fs/smb/server/Kconfig +++ b/fs/smb/server/Kconfig @@ -13,6 +13,7 @@ config SMB_SERVER select CRYPTO_LIB_ARC4 select CRYPTO_LIB_DES select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_UTILS select CRYPTO_SHA256 select CRYPTO_CMAC select CRYPTO_SHA512 diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index b4020bb55a268..c37b86a83cac8 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -283,7 +284,8 @@ int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, goto out; } - if (memcmp(ntlmv2->ntlmv2_hash, ntlmv2_rsp, CIFS_HMAC_MD5_HASH_SIZE) != 0) + if (crypto_memneq(ntlmv2->ntlmv2_hash, ntlmv2_rsp, + CIFS_HMAC_MD5_HASH_SIZE)) rc = -EINVAL; out: if (ctx) @@ -800,12 +802,8 @@ static int generate_smb3signingkey(struct ksmbd_session *sess, if (!(conn->dialect >= SMB30_PROT_ID && signing->binding)) memcpy(chann->smb3signingkey, key, SMB3_SIGN_KEY_SIZE); - ksmbd_debug(AUTH, "dumping generated AES signing keys\n"); + ksmbd_debug(AUTH, "generated SMB3 signing key\n"); ksmbd_debug(AUTH, "Session Id %llu\n", sess->id); - ksmbd_debug(AUTH, "Session Key %*ph\n", - SMB2_NTLMV2_SESSKEY_SIZE, sess->sess_key); - ksmbd_debug(AUTH, "Signing Key %*ph\n", - SMB3_SIGN_KEY_SIZE, key); return 0; } @@ -869,23 +867,9 @@ static int generate_smb3encryptionkey(struct ksmbd_conn *conn, if (rc) return rc; - ksmbd_debug(AUTH, "dumping generated AES encryption keys\n"); + ksmbd_debug(AUTH, "generated SMB3 encryption/decryption keys\n"); ksmbd_debug(AUTH, "Cipher type %d\n", conn->cipher_type); ksmbd_debug(AUTH, "Session Id %llu\n", sess->id); - ksmbd_debug(AUTH, "Session Key %*ph\n", - SMB2_NTLMV2_SESSKEY_SIZE, sess->sess_key); - if (conn->cipher_type == SMB2_ENCRYPTION_AES256_CCM || - conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) { - ksmbd_debug(AUTH, "ServerIn Key %*ph\n", - SMB3_GCM256_CRYPTKEY_SIZE, sess->smb3encryptionkey); - ksmbd_debug(AUTH, "ServerOut Key %*ph\n", - SMB3_GCM256_CRYPTKEY_SIZE, sess->smb3decryptionkey); - } else { - ksmbd_debug(AUTH, "ServerIn Key %*ph\n", - SMB3_GCM128_CRYPTKEY_SIZE, sess->smb3encryptionkey); - ksmbd_debug(AUTH, "ServerOut Key %*ph\n", - SMB3_GCM128_CRYPTKEY_SIZE, sess->smb3decryptionkey); - } return 0; } diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index b6b4f1286b9cf..e810b339dd4e5 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -39,6 +39,7 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) xa_destroy(&conn->sessions); kvfree(conn->request_buf); kfree(conn->preauth_info); + kfree(conn->mechToken); if (atomic_dec_and_test(&conn->refcnt)) { conn->transport->ops->free_transport(conn->transport); kfree(conn); @@ -179,7 +180,7 @@ int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) { struct ksmbd_conn *conn; int rc, retry_count = 0, max_timeout = 120; - int rcount = 1, bkt; + int rcount, bkt; retry_idle: if (retry_count >= max_timeout) @@ -188,8 +189,7 @@ int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { if (conn->binding || xa_load(&conn->sessions, sess_id)) { - if (conn == curr_conn) - rcount = 2; + rcount = (conn == curr_conn) ? 2 : 1; if (atomic_read(&conn->req_running) >= rcount) { rc = wait_event_timeout(conn->req_running_q, atomic_read(&conn->req_running) < rcount, diff --git a/fs/smb/server/mgmt/tree_connect.c b/fs/smb/server/mgmt/tree_connect.c index ecfc575086712..d3483d9c757c7 100644 --- a/fs/smb/server/mgmt/tree_connect.c +++ b/fs/smb/server/mgmt/tree_connect.c @@ -78,7 +78,6 @@ ksmbd_tree_conn_connect(struct ksmbd_work *work, const char *share_name) tree_conn->t_state = TREE_NEW; status.tree_conn = tree_conn; atomic_set(&tree_conn->refcount, 1); - init_waitqueue_head(&tree_conn->refcount_q); ret = xa_err(xa_store(&sess->tree_conns, tree_conn->id, tree_conn, KSMBD_DEFAULT_GFP)); @@ -100,14 +99,8 @@ ksmbd_tree_conn_connect(struct ksmbd_work *work, const char *share_name) void ksmbd_tree_connect_put(struct ksmbd_tree_connect *tcon) { - /* - * Checking waitqueue to releasing tree connect on - * tree disconnect. waitqueue_active is safe because it - * uses atomic operation for condition. - */ - if (!atomic_dec_return(&tcon->refcount) && - waitqueue_active(&tcon->refcount_q)) - wake_up(&tcon->refcount_q); + if (atomic_dec_and_test(&tcon->refcount)) + kfree(tcon); } int ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, @@ -119,14 +112,11 @@ int ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, xa_erase(&sess->tree_conns, tree_conn->id); write_unlock(&sess->tree_conns_lock); - if (!atomic_dec_and_test(&tree_conn->refcount)) - wait_event(tree_conn->refcount_q, - atomic_read(&tree_conn->refcount) == 0); - ret = ksmbd_ipc_tree_disconnect_request(sess->id, tree_conn->id); ksmbd_release_tree_conn_id(sess, tree_conn->id); ksmbd_share_config_put(tree_conn->share_conf); - kfree(tree_conn); + if (atomic_dec_and_test(&tree_conn->refcount)) + kfree(tree_conn); return ret; } diff --git a/fs/smb/server/mgmt/tree_connect.h b/fs/smb/server/mgmt/tree_connect.h index a42cdd0510411..f0023d86716f2 100644 --- a/fs/smb/server/mgmt/tree_connect.h +++ b/fs/smb/server/mgmt/tree_connect.h @@ -33,7 +33,6 @@ struct ksmbd_tree_connect { int maximal_access; bool posix_extensions; atomic_t refcount; - wait_queue_head_t refcount_q; unsigned int t_state; }; diff --git a/fs/smb/server/mgmt/user_config.c b/fs/smb/server/mgmt/user_config.c index 56c9a38ca8789..d051ac3bc831c 100644 --- a/fs/smb/server/mgmt/user_config.c +++ b/fs/smb/server/mgmt/user_config.c @@ -56,12 +56,6 @@ struct ksmbd_user *ksmbd_alloc_user(struct ksmbd_login_response *resp, goto err_free; if (resp_ext) { - if (resp_ext->ngroups > NGROUPS_MAX) { - pr_err("ngroups(%u) from login response exceeds max groups(%d)\n", - resp_ext->ngroups, NGROUPS_MAX); - goto err_free; - } - user->sgid = kmemdup(resp_ext->____payload, resp_ext->ngroups * sizeof(gid_t), KSMBD_DEFAULT_GFP); diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index 1c181ef999295..ed343807660fa 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -32,12 +32,14 @@ static void free_channel_list(struct ksmbd_session *sess) struct channel *chann; unsigned long index; + down_write(&sess->chann_lock); xa_for_each(&sess->ksmbd_chann_list, index, chann) { xa_erase(&sess->ksmbd_chann_list, index); kfree(chann); } xa_destroy(&sess->ksmbd_chann_list); + up_write(&sess->chann_lock); } static void __session_rpc_close(struct ksmbd_session *sess, @@ -159,11 +161,10 @@ void ksmbd_session_destroy(struct ksmbd_session *sess) if (!sess) return; + ksmbd_tree_conn_session_logoff(sess); + ksmbd_destroy_file_table(sess); if (sess->user) ksmbd_free_user(sess->user); - - ksmbd_tree_conn_session_logoff(sess); - ksmbd_destroy_file_table(&sess->file_table); ksmbd_launch_ksmbd_durable_scavenger(); ksmbd_session_rpc_clear_list(sess); free_channel_list(sess); @@ -220,7 +221,9 @@ static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) { struct channel *chann; + down_write(&sess->chann_lock); chann = xa_erase(&sess->ksmbd_chann_list, (long)conn); + up_write(&sess->chann_lock); if (!chann) return -ENOENT; @@ -325,8 +328,10 @@ struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, sess = ksmbd_session_lookup(conn, id); if (!sess && conn->binding) sess = ksmbd_session_lookup_slowpath(id); - if (sess && sess->state != SMB2_SESSION_VALID) + if (sess && sess->state != SMB2_SESSION_VALID) { + ksmbd_user_session_put(sess); sess = NULL; + } return sess; } @@ -390,7 +395,7 @@ void destroy_previous_session(struct ksmbd_conn *conn, goto out; } - ksmbd_destroy_file_table(&prev_sess->file_table); + ksmbd_destroy_file_table(prev_sess); prev_sess->state = SMB2_SESSION_EXPIRED; ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); ksmbd_launch_ksmbd_durable_scavenger(); @@ -452,6 +457,7 @@ static struct ksmbd_session *__session_create(int protocol) rwlock_init(&sess->tree_conns_lock); atomic_set(&sess->refcnt, 2); init_rwsem(&sess->rpc_lock); + init_rwsem(&sess->chann_lock); ret = __init_smb2_session(sess); if (ret) diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index c5749d6ec7151..cba7f688f6b57 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -49,6 +49,7 @@ struct ksmbd_session { char sess_key[CIFS_KEY_SIZE]; struct hlist_node hlist; + struct rw_semaphore chann_lock; struct xarray ksmbd_chann_list; struct xarray tree_conns; struct ida tree_conn_ida; diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index a04d5702820d0..bbb2cb3782d0c 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -82,11 +82,19 @@ static void lease_del_list(struct oplock_info *opinfo) spin_unlock(&lb->lb_lock); } -static void lb_add(struct lease_table *lb) +static struct lease_table *alloc_lease_table(struct oplock_info *opinfo) { - write_lock(&lease_list_lock); - list_add(&lb->l_entry, &lease_table_list); - write_unlock(&lease_list_lock); + struct lease_table *lb; + + lb = kmalloc(sizeof(struct lease_table), KSMBD_DEFAULT_GFP); + if (!lb) + return NULL; + + memcpy(lb->client_guid, opinfo->conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE); + INIT_LIST_HEAD(&lb->lease_list); + spin_lock_init(&lb->lb_lock); + return lb; } static int alloc_lease(struct oplock_info *opinfo, struct lease_ctx_info *lctx) @@ -120,7 +128,7 @@ static void free_lease(struct oplock_info *opinfo) kfree(lease); } -static void free_opinfo(struct oplock_info *opinfo) +static void __free_opinfo(struct oplock_info *opinfo) { if (opinfo->is_lease) free_lease(opinfo); @@ -129,6 +137,18 @@ static void free_opinfo(struct oplock_info *opinfo) kfree(opinfo); } +static void free_opinfo_rcu(struct rcu_head *rcu) +{ + struct oplock_info *opinfo = container_of(rcu, struct oplock_info, rcu); + + __free_opinfo(opinfo); +} + +static void free_opinfo(struct oplock_info *opinfo) +{ + call_rcu(&opinfo->rcu, free_opinfo_rcu); +} + struct oplock_info *opinfo_get(struct ksmbd_file *fp) { struct oplock_info *opinfo; @@ -176,9 +196,9 @@ void opinfo_put(struct oplock_info *opinfo) free_opinfo(opinfo); } -static void opinfo_add(struct oplock_info *opinfo) +static void opinfo_add(struct oplock_info *opinfo, struct ksmbd_file *fp) { - struct ksmbd_inode *ci = opinfo->o_fp->f_ci; + struct ksmbd_inode *ci = fp->f_ci; down_write(&ci->m_lock); list_add(&opinfo->op_entry, &ci->m_op_list); @@ -1030,34 +1050,27 @@ static void copy_lease(struct oplock_info *op1, struct oplock_info *op2) lease2->version = lease1->version; } -static int add_lease_global_list(struct oplock_info *opinfo) +static void add_lease_global_list(struct oplock_info *opinfo, + struct lease_table *new_lb) { struct lease_table *lb; - read_lock(&lease_list_lock); + write_lock(&lease_list_lock); list_for_each_entry(lb, &lease_table_list, l_entry) { if (!memcmp(lb->client_guid, opinfo->conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) { opinfo->o_lease->l_lb = lb; lease_add_list(opinfo); - read_unlock(&lease_list_lock); - return 0; + write_unlock(&lease_list_lock); + kfree(new_lb); + return; } } - read_unlock(&lease_list_lock); - - lb = kmalloc(sizeof(struct lease_table), KSMBD_DEFAULT_GFP); - if (!lb) - return -ENOMEM; - memcpy(lb->client_guid, opinfo->conn->ClientGUID, - SMB2_CLIENT_GUID_SIZE); - INIT_LIST_HEAD(&lb->lease_list); - spin_lock_init(&lb->lb_lock); - opinfo->o_lease->l_lb = lb; + opinfo->o_lease->l_lb = new_lb; lease_add_list(opinfo); - lb_add(lb); - return 0; + list_add(&new_lb->l_entry, &lease_table_list); + write_unlock(&lease_list_lock); } static void set_oplock_level(struct oplock_info *opinfo, int level, @@ -1123,10 +1136,12 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) rcu_read_lock(); opinfo = rcu_dereference(fp->f_opinfo); - rcu_read_unlock(); - if (!opinfo || !opinfo->is_lease || opinfo->o_lease->version != 2) + if (!opinfo || !opinfo->is_lease || opinfo->o_lease->version != 2) { + rcu_read_unlock(); return; + } + rcu_read_unlock(); p_ci = ksmbd_inode_lookup_lock(fp->filp->f_path.dentry->d_parent); if (!p_ci) @@ -1175,6 +1190,7 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, int err = 0; struct oplock_info *opinfo = NULL, *prev_opinfo = NULL; struct ksmbd_inode *ci = fp->f_ci; + struct lease_table *new_lb = NULL; bool prev_op_has_lease; __le32 prev_op_state = 0; @@ -1277,20 +1293,37 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, set_oplock_level(opinfo, req_op_level, lctx); out: - rcu_assign_pointer(fp->f_opinfo, opinfo); + /* + * Set o_fp before any publication so that concurrent readers + * (e.g. find_same_lease_key() on the lease list) that + * dereference opinfo->o_fp don't hit a NULL pointer. + * + * Keep the original publication order so concurrent opens can + * still observe the in-flight grant via ci->m_op_list, but make + * everything after opinfo_add() no-fail by preallocating any new + * lease_table first. + */ opinfo->o_fp = fp; - - opinfo_count_inc(fp); - opinfo_add(opinfo); if (opinfo->is_lease) { - err = add_lease_global_list(opinfo); - if (err) + new_lb = alloc_lease_table(opinfo); + if (!new_lb) { + err = -ENOMEM; goto err_out; + } } + opinfo_count_inc(fp); + opinfo_add(opinfo, fp); + + if (opinfo->is_lease) + add_lease_global_list(opinfo, new_lb); + + rcu_assign_pointer(fp->f_opinfo, opinfo); + return 0; err_out: - free_opinfo(opinfo); + kfree(new_lb); + opinfo_put(opinfo); return err; } @@ -1808,6 +1841,7 @@ int smb2_check_durable_oplock(struct ksmbd_conn *conn, struct ksmbd_share_config *share, struct ksmbd_file *fp, struct lease_ctx_info *lctx, + struct ksmbd_user *user, char *name) { struct oplock_info *opinfo = opinfo_get(fp); @@ -1816,6 +1850,12 @@ int smb2_check_durable_oplock(struct ksmbd_conn *conn, if (!opinfo) return 0; + if (ksmbd_vfs_compare_durable_owner(fp, user) == false) { + ksmbd_debug(SMB, "Durable handle reconnect failed: owner mismatch\n"); + ret = -EBADF; + goto out; + } + if (opinfo->is_lease == false) { if (lctx) { pr_err("create context include lease\n"); diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h index 9a56eaadd0dd8..d91a8266e065e 100644 --- a/fs/smb/server/oplock.h +++ b/fs/smb/server/oplock.h @@ -69,8 +69,9 @@ struct oplock_info { struct lease *o_lease; struct list_head op_entry; struct list_head lease_entry; - wait_queue_head_t oplock_q; /* Other server threads */ - wait_queue_head_t oplock_brk; /* oplock breaking wait */ + wait_queue_head_t oplock_q; /* Other server threads */ + wait_queue_head_t oplock_brk; /* oplock breaking wait */ + struct rcu_head rcu; }; struct lease_break_info { @@ -125,5 +126,6 @@ int smb2_check_durable_oplock(struct ksmbd_conn *conn, struct ksmbd_share_config *share, struct ksmbd_file *fp, struct lease_ctx_info *lctx, + struct ksmbd_user *user, char *name); #endif /* __KSMBD_OPLOCK_H */ diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index 40420544cc25a..ef7a24e31f506 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -126,21 +126,21 @@ static int __process_request(struct ksmbd_work *work, struct ksmbd_conn *conn, andx_again: if (command >= conn->max_cmds) { conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } cmds = &conn->cmds[command]; if (!cmds->proc) { ksmbd_debug(SMB, "*** not implemented yet cmd = %x\n", command); conn->ops->set_rsp_status(work, STATUS_NOT_IMPLEMENTED); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } if (work->sess && conn->ops->is_sign_req(work, command)) { ret = conn->ops->check_sign_req(work); if (!ret) { conn->ops->set_rsp_status(work, STATUS_ACCESS_DENIED); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } } diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index f901ae18e68ad..006b386cf9122 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -4,6 +4,7 @@ * Copyright (C) 2018 Samsung Electronics Co., Ltd. */ +#include #include #include #include @@ -79,7 +80,13 @@ static inline bool check_session_id(struct ksmbd_conn *conn, u64 id) struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn *conn) { - return xa_load(&sess->ksmbd_chann_list, (long)conn); + struct channel *chann; + + down_read(&sess->chann_lock); + chann = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + + return chann; } /** @@ -118,6 +125,8 @@ int smb2_get_ksmbd_tcon(struct ksmbd_work *work) pr_err("The first operation in the compound does not have tcon\n"); return -EINVAL; } + if (work->tcon->t_state != TREE_CONNECTED) + return -ENOENT; if (tree_id != UINT_MAX && work->tcon->id != tree_id) { pr_err("tree id(%u) is different with id(%u) in first operation\n", tree_id, work->tcon->id); @@ -1563,8 +1572,10 @@ static int ntlm_authenticate(struct ksmbd_work *work, return -ENOMEM; chann->conn = conn; + down_write(&sess->chann_lock); old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, KSMBD_DEFAULT_GFP); + up_write(&sess->chann_lock); if (xa_is_err(old)) { kfree(chann); return xa_err(old); @@ -1661,8 +1672,10 @@ static int krb5_authenticate(struct ksmbd_work *work, return -ENOMEM; chann->conn = conn; + down_write(&sess->chann_lock); old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, KSMBD_DEFAULT_GFP); + up_write(&sess->chann_lock); if (xa_is_err(old)) { kfree(chann); return xa_err(old); @@ -1911,7 +1924,7 @@ int smb2_sess_setup(struct ksmbd_work *work) else if (rc) rsp->hdr.Status = STATUS_LOGON_FAILURE; - if (conn->use_spnego && conn->mechToken) { + if (conn->mechToken) { kfree(conn->mechToken); conn->mechToken = NULL; } @@ -1935,8 +1948,14 @@ int smb2_sess_setup(struct ksmbd_work *work) if (sess->user && sess->user->flags & KSMBD_USER_FLAG_DELAY_SESSION) try_delay = true; - sess->last_active = jiffies; - sess->state = SMB2_SESSION_EXPIRED; + /* + * For binding requests, session belongs to another + * connection. Do not expire it. + */ + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + sess->last_active = jiffies; + sess->state = SMB2_SESSION_EXPIRED; + } ksmbd_user_session_put(sess); work->sess = NULL; if (try_delay) { @@ -1946,6 +1965,7 @@ int smb2_sess_setup(struct ksmbd_work *work) } } smb2_set_err_rsp(work); + conn->binding = false; } else { unsigned int iov_len; @@ -2200,7 +2220,6 @@ int smb2_tree_disconnect(struct ksmbd_work *work) goto err_out; } - WARN_ON_ONCE(atomic_dec_and_test(&tcon->refcount)); tcon->t_state = TREE_DISCONNECTED; write_unlock(&sess->tree_conns_lock); @@ -2210,8 +2229,6 @@ int smb2_tree_disconnect(struct ksmbd_work *work) goto err_out; } - work->tcon = NULL; - rsp->StructureSize = cpu_to_le16(4); err = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_tree_disconnect_rsp)); @@ -2294,7 +2311,7 @@ static noinline int create_smb2_pipe(struct ksmbd_work *work) { struct smb2_create_rsp *rsp; struct smb2_create_req *req; - int id; + int id = -1; int err; char *name; @@ -2351,6 +2368,9 @@ static noinline int create_smb2_pipe(struct ksmbd_work *work) break; } + if (id >= 0) + ksmbd_session_rpc_close(work->sess, id); + if (!IS_ERR(name)) kfree(name); @@ -2376,7 +2396,7 @@ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len, int rc = 0; unsigned int next = 0; - if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + + if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 + le16_to_cpu(eabuf->EaValueLength)) return -EINVAL; @@ -2453,7 +2473,7 @@ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len, break; } - if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + + if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 + le16_to_cpu(eabuf->EaValueLength)) { rc = -EINVAL; break; @@ -2822,10 +2842,15 @@ static int parse_durable_handle_context(struct ksmbd_work *work, SMB2_CLIENT_GUID_SIZE)) { if (!(req->hdr.Flags & SMB2_FLAGS_REPLAY_OPERATION)) { err = -ENOEXEC; + ksmbd_put_durable_fd(dh_info->fp); goto out; } - dh_info->fp->conn = conn; + if (dh_info->fp->conn) { + ksmbd_put_durable_fd(dh_info->fp); + err = -EBADF; + goto out; + } dh_info->reconnected = true; goto out; } @@ -2997,7 +3022,8 @@ int smb2_open(struct ksmbd_work *work) } if (dh_info.reconnected == true) { - rc = smb2_check_durable_oplock(conn, share, dh_info.fp, lc, name); + rc = smb2_check_durable_oplock(conn, share, dh_info.fp, + lc, sess->user, name); if (rc) { ksmbd_put_durable_fd(dh_info.fp); goto err_out2; @@ -3009,20 +3035,21 @@ int smb2_open(struct ksmbd_work *work) goto err_out2; } + fp = dh_info.fp; + if (ksmbd_override_fsids(work)) { rc = -ENOMEM; ksmbd_put_durable_fd(dh_info.fp); goto err_out2; } - fp = dh_info.fp; file_info = FILE_OPENED; rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); + ksmbd_put_durable_fd(fp); if (rc) goto err_out2; - ksmbd_put_durable_fd(fp); goto reconnected_fp; } } else if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) @@ -3385,20 +3412,24 @@ int smb2_open(struct ksmbd_work *work) KSMBD_SHARE_FLAG_ACL_XATTR)) { struct smb_fattr fattr; struct smb_ntsd *pntsd; - int pntsd_size, ace_num = 0; + int pntsd_size; + size_t scratch_len; ksmbd_acls_fattr(&fattr, idmap, inode); - if (fattr.cf_acls) - ace_num = fattr.cf_acls->a_count; - if (fattr.cf_dacls) - ace_num += fattr.cf_dacls->a_count; - - pntsd = kmalloc(sizeof(struct smb_ntsd) + - sizeof(struct smb_sid) * 3 + - sizeof(struct smb_acl) + - sizeof(struct smb_ace) * ace_num * 2, - KSMBD_DEFAULT_GFP); + scratch_len = smb_acl_sec_desc_scratch_len(&fattr, + NULL, 0, + OWNER_SECINFO | GROUP_SECINFO | + DACL_SECINFO); + if (!scratch_len || scratch_len == SIZE_MAX) { + rc = -EFBIG; + posix_acl_release(fattr.cf_acls); + posix_acl_release(fattr.cf_dacls); + goto err_out; + } + + pntsd = kvzalloc(scratch_len, KSMBD_DEFAULT_GFP); if (!pntsd) { + rc = -ENOMEM; posix_acl_release(fattr.cf_acls); posix_acl_release(fattr.cf_dacls); goto err_out; @@ -3413,7 +3444,7 @@ int smb2_open(struct ksmbd_work *work) posix_acl_release(fattr.cf_acls); posix_acl_release(fattr.cf_dacls); if (rc) { - kfree(pntsd); + kvfree(pntsd); goto err_out; } @@ -3423,7 +3454,7 @@ int smb2_open(struct ksmbd_work *work) pntsd, pntsd_size, false); - kfree(pntsd); + kvfree(pntsd); if (rc) pr_err("failed to store ntacl in xattr : %d\n", rc); @@ -3613,10 +3644,8 @@ int smb2_open(struct ksmbd_work *work) reconnected_fp: rsp->StructureSize = cpu_to_le16(89); - rcu_read_lock(); - opinfo = rcu_dereference(fp->f_opinfo); + opinfo = opinfo_get(fp); rsp->OplockLevel = opinfo != NULL ? opinfo->level : 0; - rcu_read_unlock(); rsp->Flags = 0; rsp->CreateAction = cpu_to_le32(file_info); rsp->CreationTime = cpu_to_le64(fp->create_time); @@ -3657,6 +3686,7 @@ int smb2_open(struct ksmbd_work *work) next_ptr = &lease_ccontext->Next; next_off = conn->vals->create_lease_size; } + opinfo_put(opinfo); if (maximal_access_ctxt) { struct create_context *mxac_ccontext; @@ -4436,8 +4466,9 @@ int smb2_query_dir(struct ksmbd_work *work) d_info.wptr = (char *)rsp->Buffer; d_info.rptr = (char *)rsp->Buffer; d_info.out_buf_len = - smb2_calc_max_out_buf_len(work, 8, - le32_to_cpu(req->OutputBufferLength)); + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_directory_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); if (d_info.out_buf_len < 0) { rc = -EINVAL; goto err_out; @@ -4695,6 +4726,11 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, ea_req = (struct smb2_ea_info_req *)((char *)req + le16_to_cpu(req->InputBufferOffset)); + + if (le32_to_cpu(req->InputBufferLength) < + offsetof(struct smb2_ea_info_req, name) + + ea_req->EaNameLength) + return -EINVAL; } else { /* need to send all EAs, if no specific EA is requested*/ if (le32_to_cpu(req->Flags) & SL_RETURN_SINGLE_ENTRY) @@ -4704,8 +4740,9 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, } buf_free_len = - smb2_calc_max_out_buf_len(work, 8, - le32_to_cpu(req->OutputBufferLength)); + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); if (buf_free_len < 0) return -EINVAL; @@ -4794,6 +4831,8 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, /* align next xattr entry at 4 byte bundary */ alignment_bytes = ((next_offset + 3) & ~3) - next_offset; if (alignment_bytes) { + if (buf_free_len < alignment_bytes) + break; memset(ptr, '\0', alignment_bytes); ptr += alignment_bytes; next_offset += alignment_bytes; @@ -4922,7 +4961,8 @@ static int get_file_all_info(struct ksmbd_work *work, int conv_len; char *filename; u64 time; - int ret; + int ret, buf_free_len, filename_len; + struct smb2_query_info_req *req = ksmbd_req_buf_next(work); if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { ksmbd_debug(SMB, "no right to read the attributes : 0x%x\n", @@ -4934,10 +4974,22 @@ static int get_file_all_info(struct ksmbd_work *work, if (IS_ERR(filename)) return PTR_ERR(filename); + filename_len = strlen(filename); + buf_free_len = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer) + + offsetof(struct smb2_file_all_info, FileName), + le32_to_cpu(req->OutputBufferLength)); + if (buf_free_len < (filename_len + 1) * 2) { + kfree(filename); + return -EINVAL; + } + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); - if (ret) + if (ret) { + kfree(filename); return ret; + } ksmbd_debug(SMB, "filename = %s\n", filename); delete_pending = ksmbd_inode_pending_delete(fp); @@ -4975,7 +5027,8 @@ static int get_file_all_info(struct ksmbd_work *work, file_info->Mode = fp->coption; file_info->AlignmentRequirement = 0; conv_len = smbConvertToUTF16((__le16 *)file_info->FileName, filename, - PATH_MAX, conn->local_nls, 0); + min(filename_len, PATH_MAX), + conn->local_nls, 0); conv_len *= 2; file_info->FileNameLength = cpu_to_le32(conv_len); rsp->OutputBufferLength = @@ -5029,8 +5082,9 @@ static int get_file_stream_info(struct ksmbd_work *work, file_info = (struct smb2_file_stream_info *)rsp->Buffer; buf_free_len = - smb2_calc_max_out_buf_len(work, 8, - le32_to_cpu(req->OutputBufferLength)); + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); if (buf_free_len < 0) goto out; @@ -5339,8 +5393,9 @@ static int smb2_get_info_file(struct ksmbd_work *work, if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { /* smb2 info file called for pipe */ - return smb2_get_info_file_pipe(work->sess, req, rsp, + rc = smb2_get_info_file_pipe(work->sess, req, rsp, work->response_buf); + goto iov_pin_out; } if (work->next_smb2_rcv_hdr_off) { @@ -5440,6 +5495,12 @@ static int smb2_get_info_file(struct ksmbd_work *work, rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), rsp, work->response_buf); ksmbd_fd_put(work, fp); + +iov_pin_out: + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer) + + le32_to_cpu(rsp->OutputBufferLength)); return rc; } @@ -5447,7 +5508,6 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, struct smb2_query_info_req *req, struct smb2_query_info_rsp *rsp) { - struct ksmbd_session *sess = work->sess; struct ksmbd_conn *conn = work->conn; struct ksmbd_share_config *share = work->tcon->share_conf; int fsinfoclass = 0; @@ -5577,10 +5637,11 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info = (struct object_id_info *)(rsp->Buffer); - if (!user_guest(sess->user)) - memcpy(info->objid, user_passkey(sess->user), 16); + if (path.mnt->mnt_sb->s_uuid_len == 16) + memcpy(info->objid, path.mnt->mnt_sb->s_uuid.b, + path.mnt->mnt_sb->s_uuid_len); else - memset(info->objid, 0, 16); + memcpy(info->objid, &stfs.f_fsid, sizeof(stfs.f_fsid)); info->extended_info.magic = cpu_to_le32(EXTENDED_INFO_MAGIC); info->extended_info.version = cpu_to_le32(1); @@ -5659,6 +5720,11 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), rsp, work->response_buf); path_put(&path); + + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer) + + le32_to_cpu(rsp->OutputBufferLength)); return rc; } @@ -5668,13 +5734,14 @@ static int smb2_get_info_sec(struct ksmbd_work *work, { struct ksmbd_file *fp; struct mnt_idmap *idmap; - struct smb_ntsd *pntsd = (struct smb_ntsd *)rsp->Buffer, *ppntsd = NULL; + struct smb_ntsd *pntsd = NULL, *ppntsd = NULL; struct smb_fattr fattr = {{0}}; struct inode *inode; __u32 secdesclen = 0; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; int addition_info = le32_to_cpu(req->AdditionalInformation); - int rc = 0, ppntsd_size = 0; + int rc = 0, ppntsd_size = 0, max_len; + size_t scratch_len = 0; if (addition_info & ~(OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO | PROTECTED_DACL_SECINFO | @@ -5682,6 +5749,11 @@ static int smb2_get_info_sec(struct ksmbd_work *work, ksmbd_debug(SMB, "Unsupported addition info: 0x%x)\n", addition_info); + pntsd = kzalloc(ALIGN(sizeof(struct smb_ntsd), 8), + KSMBD_DEFAULT_GFP); + if (!pntsd) + return -ENOMEM; + pntsd->revision = cpu_to_le16(1); pntsd->type = cpu_to_le16(SELF_RELATIVE | DACL_PROTECTED); pntsd->osidoffset = 0; @@ -5690,9 +5762,7 @@ static int smb2_get_info_sec(struct ksmbd_work *work, pntsd->dacloffset = 0; secdesclen = sizeof(struct smb_ntsd); - rsp->OutputBufferLength = cpu_to_le32(secdesclen); - - return 0; + goto iov_pin; } if (work->next_smb2_rcv_hdr_off) { @@ -5724,18 +5794,58 @@ static int smb2_get_info_sec(struct ksmbd_work *work, &ppntsd); /* Check if sd buffer size exceeds response buffer size */ - if (smb2_resp_buf_len(work, 8) > ppntsd_size) - rc = build_sec_desc(idmap, pntsd, ppntsd, ppntsd_size, - addition_info, &secdesclen, &fattr); + max_len = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); + if (max_len < 0) { + rc = -EINVAL; + goto release_acl; + } + + scratch_len = smb_acl_sec_desc_scratch_len(&fattr, ppntsd, + ppntsd_size, addition_info); + if (!scratch_len || scratch_len == SIZE_MAX) { + rc = -EFBIG; + goto release_acl; + } + + pntsd = kvzalloc(scratch_len, KSMBD_DEFAULT_GFP); + if (!pntsd) { + rc = -ENOMEM; + goto release_acl; + } + + rc = build_sec_desc(idmap, pntsd, ppntsd, ppntsd_size, + addition_info, &secdesclen, &fattr); + +release_acl: posix_acl_release(fattr.cf_acls); posix_acl_release(fattr.cf_dacls); kfree(ppntsd); ksmbd_fd_put(work, fp); + + if (!rc && ALIGN(secdesclen, 8) > scratch_len) + rc = -EFBIG; if (rc) - return rc; + goto err_out; +iov_pin: rsp->OutputBufferLength = cpu_to_le32(secdesclen); - return 0; + rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), + rsp, work->response_buf); + if (rc) + goto err_out; + + rc = ksmbd_iov_pin_rsp_read(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer), + pntsd, secdesclen); +err_out: + if (rc) { + rsp->OutputBufferLength = 0; + kvfree(pntsd); + } + + return rc; } /** @@ -5759,6 +5869,9 @@ int smb2_query_info(struct ksmbd_work *work) goto err_out; } + rsp->StructureSize = cpu_to_le16(9); + rsp->OutputBufferOffset = cpu_to_le16(72); + switch (req->InfoType) { case SMB2_O_INFO_FILE: ksmbd_debug(SMB, "GOT SMB2_O_INFO_FILE\n"); @@ -5779,14 +5892,6 @@ int smb2_query_info(struct ksmbd_work *work) } ksmbd_revert_fsids(work); - if (!rc) { - rsp->StructureSize = cpu_to_le16(9); - rsp->OutputBufferOffset = cpu_to_le16(72); - rc = ksmbd_iov_pin_rsp(work, (void *)rsp, - offsetof(struct smb2_query_info_rsp, Buffer) + - le32_to_cpu(rsp->OutputBufferLength)); - } - err_out: if (rc < 0) { if (rc == -EACCES) @@ -5797,6 +5902,8 @@ int smb2_query_info(struct ksmbd_work *work) rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; else if (rc == -ENOMEM) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (rc == -EINVAL && rsp->hdr.Status == 0) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; else if (rc == -EOPNOTSUPP || rsp->hdr.Status == 0) rsp->hdr.Status = STATUS_INVALID_INFO_CLASS; smb2_set_err_rsp(work); @@ -6104,14 +6211,14 @@ static int smb2_create_link(struct ksmbd_work *work, rc = -EINVAL; ksmbd_debug(SMB, "cannot delete %s\n", link_name); - goto out; } } else { rc = -EEXIST; ksmbd_debug(SMB, "link already exists\n"); - goto out; } ksmbd_vfs_kern_path_unlock(&path); + if (rc) + goto out; } rc = ksmbd_vfs_link(work, target_name, link_name); if (rc) @@ -7566,14 +7673,15 @@ int smb2_lock(struct ksmbd_work *work) rc = vfs_lock_file(filp, smb_lock->cmd, flock, NULL); skip: if (smb_lock->flags & SMB2_LOCKFLAG_UNLOCK) { + locks_free_lock(flock); + kfree(smb_lock); if (!rc) { ksmbd_debug(SMB, "File unlocked\n"); } else if (rc == -ENOENT) { rsp->hdr.Status = STATUS_NOT_LOCKED; + err = rc; goto out; } - locks_free_lock(flock); - kfree(smb_lock); } else { if (rc == FILE_LOCK_DEFERRED) { void **argv; @@ -7642,6 +7750,9 @@ int smb2_lock(struct ksmbd_work *work) spin_unlock(&work->conn->llist_lock); ksmbd_debug(SMB, "successful in taking lock\n"); } else { + locks_free_lock(flock); + kfree(smb_lock); + err = rc; goto out; } } @@ -7672,13 +7783,17 @@ int smb2_lock(struct ksmbd_work *work) struct file_lock *rlock = NULL; rlock = smb_flock_init(filp); - rlock->c.flc_type = F_UNLCK; - rlock->fl_start = smb_lock->start; - rlock->fl_end = smb_lock->end; + if (rlock) { + rlock->c.flc_type = F_UNLCK; + rlock->fl_start = smb_lock->start; + rlock->fl_end = smb_lock->end; - rc = vfs_lock_file(filp, F_SETLK, rlock, NULL); - if (rc) - pr_err("rollback unlock fail : %d\n", rc); + rc = vfs_lock_file(filp, F_SETLK, rlock, NULL); + if (rc) + pr_err("rollback unlock fail : %d\n", rc); + } else { + pr_err("rollback unlock alloc failed\n"); + } list_del(&smb_lock->llist); spin_lock(&work->conn->llist_lock); @@ -7688,7 +7803,8 @@ int smb2_lock(struct ksmbd_work *work) spin_unlock(&work->conn->llist_lock); locks_free_lock(smb_lock->fl); - locks_free_lock(rlock); + if (rlock) + locks_free_lock(rlock); kfree(smb_lock); } out2: @@ -8164,15 +8280,16 @@ int smb2_ioctl(struct ksmbd_work *work) id = req->VolatileFileId; if (req->Flags != cpu_to_le32(SMB2_0_IOCTL_IS_FSCTL)) { - rsp->hdr.Status = STATUS_NOT_SUPPORTED; + ret = -EOPNOTSUPP; goto out; } buffer = (char *)req + le32_to_cpu(req->InputOffset); cnt_code = le32_to_cpu(req->CtlCode); - ret = smb2_calc_max_out_buf_len(work, 48, - le32_to_cpu(req->MaxOutputResponse)); + ret = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_ioctl_rsp, Buffer), + le32_to_cpu(req->MaxOutputResponse)); if (ret < 0) { rsp->hdr.Status = STATUS_INVALID_PARAMETER; goto out; @@ -8184,8 +8301,9 @@ int smb2_ioctl(struct ksmbd_work *work) case FSCTL_DFS_GET_REFERRALS: case FSCTL_DFS_GET_REFERRALS_EX: /* Not support DFS yet */ + ret = -EOPNOTSUPP; rsp->hdr.Status = STATUS_FS_DRIVER_REQUIRED; - goto out; + goto out2; case FSCTL_CREATE_OR_GET_OBJECT_ID: { struct file_object_buf_type1_ioctl_rsp *obj_buf; @@ -8475,8 +8593,10 @@ int smb2_ioctl(struct ksmbd_work *work) rsp->hdr.Status = STATUS_BUFFER_TOO_SMALL; else if (ret < 0 || rsp->hdr.Status == 0) rsp->hdr.Status = STATUS_INVALID_PARAMETER; + +out2: smb2_set_err_rsp(work); - return 0; + return ret; } /** @@ -8865,7 +8985,7 @@ int smb2_check_sign_req(struct ksmbd_work *work) signature)) return 0; - if (memcmp(signature, signature_req, SMB2_SIGNATURE_SIZE)) { + if (crypto_memneq(signature, signature_req, SMB2_SIGNATURE_SIZE)) { pr_err("bad smb2 signature\n"); return 0; } @@ -8953,7 +9073,7 @@ int smb3_check_sign_req(struct ksmbd_work *work) if (ksmbd_sign_smb3_pdu(conn, signing_key, iov, 1, signature)) return 0; - if (memcmp(signature, signature_req, SMB2_SIGNATURE_SIZE)) { + if (crypto_memneq(signature, signature_req, SMB2_SIGNATURE_SIZE)) { pr_err("bad smb2 signature\n"); return 0; } diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index 5aa7a66334d93..eae9daeb0a414 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -451,7 +451,8 @@ static void parse_dacl(struct mnt_idmap *idmap, ppace[i]->access_req = smb_map_generic_desired_access(ppace[i]->access_req); - if (!(compare_sids(&ppace[i]->sid, &sid_unix_NFS_mode))) { + if (ppace[i]->sid.num_subauth >= 3 && + !(compare_sids(&ppace[i]->sid, &sid_unix_NFS_mode))) { fattr->cf_mode = le32_to_cpu(ppace[i]->sid.sub_auth[2]); break; @@ -595,6 +596,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, struct smb_sid *sid; struct smb_ace *ntace; int i, j; + u16 ace_sz; if (!fattr->cf_acls) goto posix_default_acl; @@ -639,8 +641,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, flags = 0x03; ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -649,8 +653,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, if (S_ISDIR(fattr->cf_mode) && (pace->e_tag == ACL_USER || pace->e_tag == ACL_GROUP)) { ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x03, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -690,8 +696,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, } ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -727,7 +735,8 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap, break; memcpy((char *)pndace + size, ntace, nt_ace_size); - size += nt_ace_size; + if (check_add_overflow(size, nt_ace_size, &size)) + break; aces_size -= nt_ace_size; ntace = (struct smb_ace *)((char *)ntace + nt_ace_size); num_aces++; @@ -915,6 +924,49 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, return 0; } +size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr, + struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info) +{ + size_t len = sizeof(struct smb_ntsd); + size_t tmp; + + if (addition_info & OWNER_SECINFO) + len += sizeof(struct smb_sid); + if (addition_info & GROUP_SECINFO) + len += sizeof(struct smb_sid); + if (!(addition_info & DACL_SECINFO)) + return len; + + len += sizeof(struct smb_acl); + if (ppntsd && ppntsd_size > 0) { + unsigned int dacl_offset = le32_to_cpu(ppntsd->dacloffset); + + if (dacl_offset < ppntsd_size && + check_add_overflow(len, ppntsd_size - dacl_offset, &len)) + return 0; + } + + if (fattr->cf_acls) { + if (check_mul_overflow((size_t)fattr->cf_acls->a_count, + 2 * sizeof(struct smb_ace), &tmp) || + check_add_overflow(len, tmp, &len)) + return 0; + } else { + /* default/minimum DACL */ + if (check_add_overflow(len, 5 * sizeof(struct smb_ace), &len)) + return 0; + } + + if (fattr->cf_dacls) { + if (check_mul_overflow((size_t)fattr->cf_dacls->a_count, + sizeof(struct smb_ace), &tmp) || + check_add_overflow(len, tmp, &len)) + return 0; + } + + return len; +} + /* Convert permission bits from mode to equivalent CIFS ACL */ int build_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, struct smb_ntsd *ppntsd, @@ -1062,8 +1114,24 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, goto free_parent_pntsd; } - aces_base = kmalloc(sizeof(struct smb_ace) * num_aces * 2, - KSMBD_DEFAULT_GFP); + aces_size = pdacl_size - sizeof(struct smb_acl); + + /* + * Validate num_aces against the DACL payload before allocating. + * Each ACE must be at least as large as its fixed-size header + * (up to the SID base), so num_aces cannot exceed the payload + * divided by the minimum ACE size. This mirrors the existing + * check in parse_dacl(). + */ + if (num_aces > aces_size / (offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth) + + sizeof(__le16))) { + rc = -EINVAL; + goto free_parent_pntsd; + } + + aces_base = kmalloc_array(num_aces * 2, sizeof(struct smb_ace), + KSMBD_DEFAULT_GFP); if (!aces_base) { rc = -ENOMEM; goto free_parent_pntsd; @@ -1072,7 +1140,6 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, aces = (struct smb_ace *)aces_base; parent_aces = (struct smb_ace *)((char *)parent_pdacl + sizeof(struct smb_acl)); - aces_size = acl_len - sizeof(struct smb_acl); if (pntsd_type & DACL_AUTO_INHERITED) inherited_flags = INHERITED_ACE; @@ -1080,11 +1147,14 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, for (i = 0; i < num_aces; i++) { int pace_size; - if (offsetof(struct smb_ace, access_req) > aces_size) + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; pace_size = le16_to_cpu(parent_aces->size); - if (pace_size > aces_size) + if (pace_size > aces_size || + pace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= pace_size; @@ -1298,10 +1368,13 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, access_req) > aces_size) + if (offsetof(struct smb_ace, sid) + + aces_size < CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); - if (ace_size > aces_size) + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; granted |= le32_to_cpu(ace->access_req); @@ -1319,13 +1392,19 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, access_req) > aces_size) + if (offsetof(struct smb_ace, sid) + + aces_size < CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); - if (ace_size > aces_size) + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) + break; + if (!compare_sids(&sid, &ace->sid) || !compare_sids(&sid_unix_NFS_mode, &ace->sid)) { found = 1; diff --git a/fs/smb/server/smbacl.h b/fs/smb/server/smbacl.h index 355adaee39b87..ab21ba2cd4df3 100644 --- a/fs/smb/server/smbacl.h +++ b/fs/smb/server/smbacl.h @@ -101,6 +101,8 @@ int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, bool type_check, bool get_write); void id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid); void ksmbd_init_domain(u32 *sub_auth); +size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr, + struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info); static inline uid_t posix_acl_uid_translate(struct mnt_idmap *idmap, struct posix_acl_entry *pace) diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c index 2c08cccfa6809..1c5645238bd3f 100644 --- a/fs/smb/server/transport_ipc.c +++ b/fs/smb/server/transport_ipc.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "vfs_cache.h" #include "transport_ipc.h" @@ -497,7 +498,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_rpc_command *resp = entry->response; - msg_sz = sizeof(struct ksmbd_rpc_command) + resp->payload_sz; + if (check_add_overflow(sizeof(struct ksmbd_rpc_command), + resp->payload_sz, &msg_sz)) + return -EINVAL; break; } case KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST: @@ -516,8 +519,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) if (resp->payload_sz < resp->veto_list_sz) return -EINVAL; - msg_sz = sizeof(struct ksmbd_share_config_response) + - resp->payload_sz; + if (check_add_overflow(sizeof(struct ksmbd_share_config_response), + resp->payload_sz, &msg_sz)) + return -EINVAL; } break; } @@ -526,6 +530,12 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) struct ksmbd_login_response_ext *resp = entry->response; if (resp->ngroups) { + if (resp->ngroups < 0 || + resp->ngroups > NGROUPS_MAX) { + pr_err("ngroups(%d) from login response exceeds max groups(%d)\n", + resp->ngroups, NGROUPS_MAX); + return -EINVAL; + } msg_sz = sizeof(struct ksmbd_login_response_ext) + resp->ngroups * sizeof(gid_t); } @@ -553,12 +563,16 @@ static void *ipc_msg_send_request(struct ksmbd_ipc_msg *msg, unsigned int handle up_write(&ipc_msg_table_lock); ret = ipc_msg_send(msg); - if (ret) + if (ret) { + down_write(&ipc_msg_table_lock); goto out; + } ret = wait_event_interruptible_timeout(entry.wait, entry.response != NULL, IPC_WAIT_TIMEOUT); + + down_write(&ipc_msg_table_lock); if (entry.response) { ret = ipc_validate_msg(&entry); if (ret) { @@ -567,7 +581,6 @@ static void *ipc_msg_send_request(struct ksmbd_ipc_msg *msg, unsigned int handle } } out: - down_write(&ipc_msg_table_lock); hash_del(&entry.ipc_table_hlist); up_write(&ipc_msg_table_lock); return entry.response; diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index e2be9a4961546..3f9bcd10a0b3f 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -19,6 +19,8 @@ #include #include +#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) smb_direct_disconnect_rdma_connection(__sc) + #include "glob.h" #include "connection.h" #include "smb_common.h" @@ -219,6 +221,7 @@ static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc) * in order to notice the broken connection. */ wake_up_all(&sc->status_wait); + wake_up_all(&sc->send_io.bcredits.wait_queue); wake_up_all(&sc->send_io.lcredits.wait_queue); wake_up_all(&sc->send_io.credits.wait_queue); wake_up_all(&sc->send_io.pending.zero_wait_queue); @@ -231,6 +234,9 @@ static void smb_direct_disconnect_rdma_work(struct work_struct *work) struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, disconnect_work); + if (sc->first_error == 0) + sc->first_error = -ECONNABORTED; + /* * make sure this and other work is not queued again * but here we don't block and avoid @@ -241,9 +247,6 @@ static void smb_direct_disconnect_rdma_work(struct work_struct *work) disable_delayed_work(&sc->idle.timer_work); disable_work(&sc->idle.immediate_work); - if (sc->first_error == 0) - sc->first_error = -ECONNABORTED; - switch (sc->status) { case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: @@ -287,6 +290,9 @@ static void smb_direct_disconnect_rdma_work(struct work_struct *work) static void smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc) { + if (sc->first_error == 0) + sc->first_error = -ECONNABORTED; + /* * make sure other work (than disconnect_work) is * not queued again but here we don't block and avoid @@ -296,9 +302,6 @@ smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc) disable_work(&sc->idle.immediate_work); disable_delayed_work(&sc->idle.timer_work); - if (sc->first_error == 0) - sc->first_error = -ECONNABORTED; - switch (sc->status) { case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: @@ -639,7 +642,18 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) return; } sc->recv_io.reassembly.full_packet_received = true; - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED); + /* + * Some drivers (at least mlx5_ib) might post a + * recv completion before RDMA_CM_EVENT_ESTABLISHED, + * we need to adjust our expectation in that case. + */ + if (!sc->first_error && sc->status == SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING) + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) { + put_recvmsg(sc, recvmsg); + smb_direct_disconnect_rdma_connection(sc); + return; + } sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; enqueue_reassembly(sc, recvmsg, 0); wake_up(&sc->status_wait); @@ -648,6 +662,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) struct smbdirect_data_transfer *data_transfer = (struct smbdirect_data_transfer *)recvmsg->packet; u32 remaining_data_length, data_offset, data_length; + int current_recv_credits; u16 old_recv_credit_target; if (wc->byte_len < @@ -686,7 +701,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } atomic_dec(&sc->recv_io.posted.count); - atomic_dec(&sc->recv_io.credits.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); old_recv_credit_target = sc->recv_io.credits.target; sc->recv_io.credits.target = @@ -706,7 +721,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) wake_up(&sc->send_io.credits.wait_queue); if (data_length) { - if (sc->recv_io.credits.target > old_recv_credit_target) + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); enqueue_reassembly(sc, recvmsg, (int)data_length); @@ -913,6 +929,17 @@ static void smb_direct_post_recv_credits(struct work_struct *work) } } + atomic_add(credits, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (credits && + atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + if (credits) queue_work(sc->workqueue, &sc->idle.immediate_work); } @@ -930,6 +957,31 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) ib_wc_status_msg(wc->status), wc->status, wc->opcode); + if (unlikely(!(sendmsg->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + pr_err("unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smb_direct_disconnect_rdma_connection(sc); + return; + } + /* * Free possible siblings and then the main send_io */ @@ -943,6 +995,7 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) lcredits += 1; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { +skip_free: pr_err("Send error. status='%s (%d)', opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); @@ -959,19 +1012,37 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) static int manage_credits_prior_sending(struct smbdirect_socket *sc) { + int missing; + int available; int new_credits; if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) return 0; - new_credits = atomic_read(&sc->recv_io.posted.count); - if (new_credits == 0) + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = (u16)min3(U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); return 0; + } - new_credits -= atomic_read(&sc->recv_io.credits.count); - if (new_credits <= 0) - return 0; + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + /* + * Remember we granted the credits + */ atomic_add(new_credits, &sc->recv_io.credits.count); return new_credits; } @@ -1015,6 +1086,7 @@ static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx, send_ctx->wr_cnt = 0; send_ctx->need_invalidate_rkey = need_invalidate_rkey; send_ctx->remote_key = remote_key; + send_ctx->credit = 0; } static int smb_direct_flush_send_list(struct smbdirect_socket *sc, @@ -1022,10 +1094,10 @@ static int smb_direct_flush_send_list(struct smbdirect_socket *sc, bool is_last) { struct smbdirect_send_io *first, *last; - int ret; + int ret = 0; if (list_empty(&send_ctx->msg_list)) - return 0; + goto release_credit; first = list_first_entry(&send_ctx->msg_list, struct smbdirect_send_io, @@ -1067,6 +1139,13 @@ static int smb_direct_flush_send_list(struct smbdirect_socket *sc, smb_direct_free_sendmsg(sc, last); } +release_credit: + if (is_last && !ret && send_ctx->credit) { + atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); + send_ctx->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + return ret; } @@ -1092,6 +1171,25 @@ static int wait_for_credits(struct smbdirect_socket *sc, } while (true); } +static int wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *send_ctx) +{ + int ret; + + if (send_ctx->credit) + return 0; + + ret = wait_for_credits(sc, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + send_ctx->credit = 1; + return 0; +} + static int wait_for_send_lcredit(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx) { @@ -1141,6 +1239,7 @@ static int calc_rw_credits(struct smbdirect_socket *sc, static int smb_direct_create_header(struct smbdirect_socket *sc, int size, int remaining_data_length, + int new_credits, struct smbdirect_send_io **sendmsg_out) { struct smbdirect_socket_parameters *sp = &sc->parameters; @@ -1156,7 +1255,7 @@ static int smb_direct_create_header(struct smbdirect_socket *sc, /* Fill in the packet header */ packet = (struct smbdirect_data_transfer *)sendmsg->packet; packet->credits_requested = cpu_to_le16(sp->send_credit_target); - packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); + packet->credits_granted = cpu_to_le16(new_credits); packet->flags = 0; if (manage_keep_alive_before_sending(sc)) @@ -1238,14 +1337,12 @@ static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nen static int get_mapped_sg_list(struct ib_device *device, void *buf, int size, struct scatterlist *sg_list, int nentries, - enum dma_data_direction dir) + enum dma_data_direction dir, int *npages) { - int npages; - - npages = get_sg_list(buf, size, sg_list, nentries); - if (npages < 0) + *npages = get_sg_list(buf, size, sg_list, nentries); + if (*npages < 0) return -EINVAL; - return ib_dma_map_sg(device, sg_list, npages, dir); + return ib_dma_map_sg(device, sg_list, *npages, dir); } static int post_sendmsg(struct smbdirect_socket *sc, @@ -1295,6 +1392,17 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, struct smbdirect_send_io *msg; int data_length; struct scatterlist sg[SMBDIRECT_SEND_IO_MAX_SGE - 1]; + struct smbdirect_send_batch _send_ctx; + int new_credits; + + if (!send_ctx) { + smb_direct_send_ctx_init(&_send_ctx, false, 0); + send_ctx = &_send_ctx; + } + + ret = wait_for_send_bcredit(sc, send_ctx); + if (ret) + goto bcredit_failed; ret = wait_for_send_lcredit(sc, send_ctx); if (ret) @@ -1304,24 +1412,42 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, if (ret) goto credit_failed; + new_credits = manage_credits_prior_sending(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); + ret = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + ret = -ENOTCONN; + if (ret < 0) + goto credit_failed; + + new_credits = manage_credits_prior_sending(sc); + } + data_length = 0; for (i = 0; i < niov; i++) data_length += iov[i].iov_len; ret = smb_direct_create_header(sc, data_length, remaining_data_length, - &msg); + new_credits, &msg); if (ret) goto header_failed; for (i = 0; i < niov; i++) { struct ib_sge *sge; int sg_cnt; + int npages; sg_init_table(sg, SMBDIRECT_SEND_IO_MAX_SGE - 1); sg_cnt = get_mapped_sg_list(sc->ib.dev, iov[i].iov_base, iov[i].iov_len, sg, SMBDIRECT_SEND_IO_MAX_SGE - 1, - DMA_TO_DEVICE); + DMA_TO_DEVICE, &npages); if (sg_cnt <= 0) { pr_err("failed to map buffer\n"); ret = -ENOMEM; @@ -1329,7 +1455,7 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, } else if (sg_cnt + msg->num_sge > SMBDIRECT_SEND_IO_MAX_SGE) { pr_err("buffer not fitted into sges\n"); ret = -E2BIG; - ib_dma_unmap_sg(sc->ib.dev, sg, sg_cnt, + ib_dma_unmap_sg(sc->ib.dev, sg, npages, DMA_TO_DEVICE); goto err; } @@ -1346,14 +1472,30 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, ret = post_sendmsg(sc, send_ctx, msg); if (ret) goto err; + + /* + * From here msg is moved to send_ctx + * and we should not free it explicitly. + */ + + if (send_ctx == &_send_ctx) { + ret = smb_direct_flush_send_list(sc, send_ctx, true); + if (ret) + goto flush_failed; + } + return 0; err: smb_direct_free_sendmsg(sc, msg); +flush_failed: header_failed: atomic_inc(&sc->send_io.credits.count); credit_failed: atomic_inc(&sc->send_io.lcredits.count); lcredit_failed: + atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); + send_ctx->credit = 0; +bcredit_failed: return ret; } @@ -1725,7 +1867,18 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, switch (event->event) { case RDMA_CM_EVENT_ESTABLISHED: { - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); + /* + * Some drivers (at least mlx5_ib) might post a + * recv completion before RDMA_CM_EVENT_ESTABLISHED, + * we need to adjust our expectation in that case. + * + * As we already started the negotiation, we just + * ignore RDMA_CM_EVENT_ESTABLISHED here. + */ + if (!sc->first_error && sc->status > SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING) + break; + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) + break; sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; wake_up(&sc->status_wait); break; @@ -1804,6 +1957,7 @@ static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc, resp->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); + atomic_set(&sc->send_io.bcredits.count, 1); sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; sc->status = SMBDIRECT_SOCKET_CONNECTED; } @@ -2266,9 +2420,9 @@ static int smb_direct_prepare(struct ksmbd_transport *t) goto put; req = (struct smbdirect_negotiate_req *)recvmsg->packet; - sp->max_recv_size = min_t(int, sp->max_recv_size, + sp->max_recv_size = min_t(u32, sp->max_recv_size, le32_to_cpu(req->preferred_send_size)); - sp->max_send_size = min_t(int, sp->max_send_size, + sp->max_send_size = min_t(u32, sp->max_send_size, le32_to_cpu(req->max_receive_size)); sp->max_fragmented_send_size = le32_to_cpu(req->max_fragmented_size); diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index d2e391c29464e..a09b07b3b108d 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -41,6 +41,7 @@ static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops; static void tcp_stop_kthread(struct task_struct *kthread); static struct interface *alloc_iface(char *ifname); +static void ksmbd_tcp_disconnect(struct ksmbd_transport *t); #define KSMBD_TRANS(t) (&(t)->transport) #define TCP_TRANS(t) ((struct tcp_transport *)container_of(t, \ @@ -196,6 +197,8 @@ static int ksmbd_tcp_new_connection(struct socket *client_sk) t = alloc_transport(client_sk); if (!t) { sock_release(client_sk); + if (server_conf.max_connections) + atomic_dec(&active_num_conn); return -ENOMEM; } @@ -216,7 +219,7 @@ static int ksmbd_tcp_new_connection(struct socket *client_sk) if (IS_ERR(handler)) { pr_err("cannot start conn thread\n"); rc = PTR_ERR(handler); - free_transport(t); + ksmbd_tcp_disconnect(KSMBD_TRANS(t)); } return rc; } @@ -298,7 +301,7 @@ static int ksmbd_kthread_fn(void *p) skip_max_ip_conns_limit: if (server_conf.max_connections && - atomic_inc_return(&active_num_conn) >= server_conf.max_connections) { + atomic_inc_return(&active_num_conn) > server_conf.max_connections) { pr_info_ratelimited("Limit the maximum number of connections(%u)\n", atomic_read(&active_num_conn)); atomic_dec(&active_num_conn); diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index 891ed2dc2b735..90e693e4e4149 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -324,6 +324,9 @@ static int check_lock_range(struct file *filp, loff_t start, loff_t end, struct file_lock_context *ctx = locks_inode_context(file_inode(filp)); int error = 0; + if (start == end) + return 0; + if (!ctx || list_empty_careful(&ctx->flc_posix)) return 0; @@ -828,7 +831,7 @@ int ksmbd_vfs_truncate(struct ksmbd_work *work, if (size < inode->i_size) { err = check_lock_range(filp, size, inode->i_size - 1, WRITE); - } else { + } else if (size > inode->i_size) { err = check_lock_range(filp, inode->i_size, size - 1, WRITE); } diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index dfed6fce89049..d29cc1d01bd2c 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -18,6 +18,7 @@ #include "connection.h" #include "mgmt/tree_connect.h" #include "mgmt/user_session.h" +#include "mgmt/user_config.h" #include "smb_common.h" #include "server.h" @@ -112,40 +113,62 @@ int ksmbd_query_inode_status(struct dentry *dentry) read_lock(&inode_hash_lock); ci = __ksmbd_inode_lookup(dentry); - if (ci) { - ret = KSMBD_INODE_STATUS_OK; - if (ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS)) - ret = KSMBD_INODE_STATUS_PENDING_DELETE; - atomic_dec(&ci->m_count); - } read_unlock(&inode_hash_lock); + if (!ci) + return ret; + + down_read(&ci->m_lock); + if (ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS)) + ret = KSMBD_INODE_STATUS_PENDING_DELETE; + else + ret = KSMBD_INODE_STATUS_OK; + up_read(&ci->m_lock); + + atomic_dec(&ci->m_count); return ret; } bool ksmbd_inode_pending_delete(struct ksmbd_file *fp) { - return (fp->f_ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS)); + struct ksmbd_inode *ci = fp->f_ci; + int ret; + + down_read(&ci->m_lock); + ret = (ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS)); + up_read(&ci->m_lock); + + return ret; } void ksmbd_set_inode_pending_delete(struct ksmbd_file *fp) { - fp->f_ci->m_flags |= S_DEL_PENDING; + struct ksmbd_inode *ci = fp->f_ci; + + down_write(&ci->m_lock); + ci->m_flags |= S_DEL_PENDING; + up_write(&ci->m_lock); } void ksmbd_clear_inode_pending_delete(struct ksmbd_file *fp) { - fp->f_ci->m_flags &= ~S_DEL_PENDING; + struct ksmbd_inode *ci = fp->f_ci; + + down_write(&ci->m_lock); + ci->m_flags &= ~S_DEL_PENDING; + up_write(&ci->m_lock); } void ksmbd_fd_set_delete_on_close(struct ksmbd_file *fp, int file_info) { - if (ksmbd_stream_fd(fp)) { - fp->f_ci->m_flags |= S_DEL_ON_CLS_STREAM; - return; - } + struct ksmbd_inode *ci = fp->f_ci; - fp->f_ci->m_flags |= S_DEL_ON_CLS; + down_write(&ci->m_lock); + if (ksmbd_stream_fd(fp)) + ci->m_flags |= S_DEL_ON_CLS_STREAM; + else + ci->m_flags |= S_DEL_ON_CLS; + up_write(&ci->m_lock); } static void ksmbd_inode_hash(struct ksmbd_inode *ci) @@ -257,27 +280,41 @@ static void __ksmbd_inode_close(struct ksmbd_file *fp) struct file *filp; filp = fp->filp; - if (ksmbd_stream_fd(fp) && (ci->m_flags & S_DEL_ON_CLS_STREAM)) { - ci->m_flags &= ~S_DEL_ON_CLS_STREAM; - err = ksmbd_vfs_remove_xattr(file_mnt_idmap(filp), - &filp->f_path, - fp->stream.name, - true); - if (err) - pr_err("remove xattr failed : %s\n", - fp->stream.name); + + if (ksmbd_stream_fd(fp)) { + bool remove_stream_xattr = false; + + down_write(&ci->m_lock); + if (ci->m_flags & S_DEL_ON_CLS_STREAM) { + ci->m_flags &= ~S_DEL_ON_CLS_STREAM; + remove_stream_xattr = true; + } + up_write(&ci->m_lock); + + if (remove_stream_xattr) { + err = ksmbd_vfs_remove_xattr(file_mnt_idmap(filp), + &filp->f_path, + fp->stream.name, + true); + if (err) + pr_err("remove xattr failed : %s\n", + fp->stream.name); + } } if (atomic_dec_and_test(&ci->m_count)) { + bool do_unlink = false; + down_write(&ci->m_lock); if (ci->m_flags & (S_DEL_ON_CLS | S_DEL_PENDING)) { ci->m_flags &= ~(S_DEL_ON_CLS | S_DEL_PENDING); - up_write(&ci->m_lock); - ksmbd_vfs_unlink(filp); - down_write(&ci->m_lock); + do_unlink = true; } up_write(&ci->m_lock); + if (do_unlink) + ksmbd_vfs_unlink(filp); + ksmbd_inode_free(ci); } } @@ -334,9 +371,11 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) * there are not accesses to fp->lock_list. */ list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - spin_lock(&fp->conn->llist_lock); - list_del(&smb_lock->clist); - spin_unlock(&fp->conn->llist_lock); + if (!list_empty(&smb_lock->clist) && fp->conn) { + spin_lock(&fp->conn->llist_lock); + list_del(&smb_lock->clist); + spin_unlock(&fp->conn->llist_lock); + } list_del(&smb_lock->flist); locks_free_lock(smb_lock->fl); @@ -345,6 +384,8 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) if (ksmbd_stream_fd(fp)) kfree(fp->stream.name); + kfree(fp->owner.name); + kmem_cache_free(filp_cache, fp); } @@ -656,11 +697,13 @@ void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, } static int -__close_file_table_ids(struct ksmbd_file_table *ft, +__close_file_table_ids(struct ksmbd_session *sess, struct ksmbd_tree_connect *tcon, bool (*skip)(struct ksmbd_tree_connect *tcon, - struct ksmbd_file *fp)) + struct ksmbd_file *fp, + struct ksmbd_user *user)) { + struct ksmbd_file_table *ft = &sess->file_table; struct ksmbd_file *fp; unsigned int id = 0; int num = 0; @@ -673,7 +716,7 @@ __close_file_table_ids(struct ksmbd_file_table *ft, break; } - if (skip(tcon, fp) || + if (skip(tcon, fp, sess->user) || !atomic_dec_and_test(&fp->refcount)) { id++; write_unlock(&ft->lock); @@ -725,7 +768,8 @@ static inline bool is_reconnectable(struct ksmbd_file *fp) } static bool tree_conn_fd_check(struct ksmbd_tree_connect *tcon, - struct ksmbd_file *fp) + struct ksmbd_file *fp, + struct ksmbd_user *user) { return fp->tcon != tcon; } @@ -860,16 +904,74 @@ void ksmbd_stop_durable_scavenger(void) kthread_stop(server_conf.dh_task); } +/* + * ksmbd_vfs_copy_durable_owner - Copy owner info for durable reconnect + * @fp: ksmbd file pointer to store owner info + * @user: user pointer to copy from + * + * This function binds the current user's identity to the file handle + * to satisfy MS-SMB2 Step 8 (SecurityContext matching) during reconnect. + * + * Return: 0 on success, or negative error code on failure + */ +static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) +{ + if (!user) + return -EINVAL; + + /* Duplicate the user name to ensure identity persistence */ + fp->owner.name = kstrdup(user->name, GFP_KERNEL); + if (!fp->owner.name) + return -ENOMEM; + + fp->owner.uid = user->uid; + fp->owner.gid = user->gid; + + return 0; +} + +/** + * ksmbd_vfs_compare_durable_owner - Verify if the requester is original owner + * @fp: existing ksmbd file pointer + * @user: user pointer of the reconnect requester + * + * Compares the UID, GID, and name of the current requester against the + * original owner stored in the file handle. + * + * Return: true if the user matches, false otherwise + */ +bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) +{ + if (!user || !fp->owner.name) + return false; + + /* Check if the UID and GID match first (fast path) */ + if (fp->owner.uid != user->uid || fp->owner.gid != user->gid) + return false; + + /* Validate the account name to ensure the same SecurityContext */ + if (strcmp(fp->owner.name, user->name)) + return false; + + return true; +} + static bool session_fd_check(struct ksmbd_tree_connect *tcon, - struct ksmbd_file *fp) + struct ksmbd_file *fp, struct ksmbd_user *user) { struct ksmbd_inode *ci; struct oplock_info *op; struct ksmbd_conn *conn; + struct ksmbd_lock *smb_lock, *tmp_lock; if (!is_reconnectable(fp)) return false; + if (ksmbd_vfs_copy_durable_owner(fp, user)) + return false; + conn = fp->conn; ci = fp->f_ci; down_write(&ci->m_lock); @@ -882,6 +984,12 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, } up_write(&ci->m_lock); + list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { + spin_lock(&fp->conn->llist_lock); + list_del_init(&smb_lock->clist); + spin_unlock(&fp->conn->llist_lock); + } + fp->conn = NULL; fp->tcon = NULL; fp->volatile_id = KSMBD_NO_FID; @@ -895,7 +1003,7 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, void ksmbd_close_tree_conn_fds(struct ksmbd_work *work) { - int num = __close_file_table_ids(&work->sess->file_table, + int num = __close_file_table_ids(work->sess, work->tcon, tree_conn_fd_check); @@ -904,7 +1012,7 @@ void ksmbd_close_tree_conn_fds(struct ksmbd_work *work) void ksmbd_close_session_fds(struct ksmbd_work *work) { - int num = __close_file_table_ids(&work->sess->file_table, + int num = __close_file_table_ids(work->sess, work->tcon, session_fd_check); @@ -960,6 +1068,9 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) { struct ksmbd_inode *ci; struct oplock_info *op; + struct ksmbd_conn *conn = work->conn; + struct ksmbd_lock *smb_lock; + unsigned int old_f_state; if (!fp->is_durable || fp->conn || fp->tcon) { pr_err("Invalid durable fd [%p:%p]\n", fp->conn, fp->tcon); @@ -971,9 +1082,23 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) return -EBADF; } - fp->conn = work->conn; + old_f_state = fp->f_state; + fp->f_state = FP_NEW; + __open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID); + if (!has_file_id(fp->volatile_id)) { + fp->f_state = old_f_state; + return -EBADF; + } + + fp->conn = conn; fp->tcon = work->tcon; + list_for_each_entry(smb_lock, &fp->lock_list, flist) { + spin_lock(&conn->llist_lock); + list_add_tail(&smb_lock->clist, &conn->lock_list); + spin_unlock(&conn->llist_lock); + } + ci = fp->f_ci; down_write(&ci->m_lock); list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) { @@ -984,13 +1109,10 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) } up_write(&ci->m_lock); - fp->f_state = FP_NEW; - __open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID); - if (!has_file_id(fp->volatile_id)) { - fp->conn = NULL; - fp->tcon = NULL; - return -EBADF; - } + fp->owner.uid = fp->owner.gid = 0; + kfree(fp->owner.name); + fp->owner.name = NULL; + return 0; } @@ -1005,12 +1127,14 @@ int ksmbd_init_file_table(struct ksmbd_file_table *ft) return 0; } -void ksmbd_destroy_file_table(struct ksmbd_file_table *ft) +void ksmbd_destroy_file_table(struct ksmbd_session *sess) { + struct ksmbd_file_table *ft = &sess->file_table; + if (!ft->idr) return; - __close_file_table_ids(ft, NULL, session_fd_check); + __close_file_table_ids(sess, NULL, session_fd_check); idr_destroy(ft->idr); kfree(ft->idr); ft->idr = NULL; diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index 78b506c5ef03b..866f32c10d4dd 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -68,6 +68,13 @@ enum { FP_CLOSED }; +/* Owner information for durable handle reconnect */ +struct durable_owner { + unsigned int uid; + unsigned int gid; + char *name; +}; + struct ksmbd_file { struct file *filp; u64 persistent_id; @@ -114,6 +121,7 @@ struct ksmbd_file { bool is_resilient; bool is_posix_ctxt; + struct durable_owner owner; }; static inline void set_ctx_actor(struct dir_context *ctx, @@ -140,7 +148,7 @@ static inline bool ksmbd_stream_fd(struct ksmbd_file *fp) } int ksmbd_init_file_table(struct ksmbd_file_table *ft); -void ksmbd_destroy_file_table(struct ksmbd_file_table *ft); +void ksmbd_destroy_file_table(struct ksmbd_session *sess); int ksmbd_close_fd(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_fd_fast(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_foreign_fd(struct ksmbd_work *work, u64 id); @@ -166,6 +174,8 @@ void ksmbd_free_global_file_table(void); void ksmbd_set_fd_limit(unsigned long limit); void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, unsigned int state); +bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user); /* * INODE hash diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c index 181260e72680c..92fb857d2c761 100644 --- a/fs/squashfs/cache.c +++ b/fs/squashfs/cache.c @@ -344,6 +344,9 @@ int squashfs_read_metadata(struct super_block *sb, void *buffer, if (unlikely(length < 0)) return -EIO; + if (unlikely(*offset < 0 || *offset >= SQUASHFS_METADATA_SIZE)) + return -EIO; + while (length) { entry = squashfs_cache_get(sb, msblk->block_cache, *block, 0); if (entry->error) { diff --git a/fs/super.c b/fs/super.c index 277b84e5c2791..4c79f170ac0d2 100644 --- a/fs/super.c +++ b/fs/super.c @@ -1188,7 +1188,7 @@ static void filesystems_freeze_callback(struct super_block *sb, void *freeze_all if (!sb->s_op->freeze_fs && !sb->s_op->freeze_super) return; - if (freeze_all_ptr && !(sb->s_type->fs_flags & FS_POWER_FREEZE)) + if (!freeze_all_ptr && !(sb->s_type->fs_flags & FS_POWER_FREEZE)) return; if (!get_active_super(sb)) diff --git a/fs/tests/exec_kunit.c b/fs/tests/exec_kunit.c index 7c77d039680bb..1c32cac098cf5 100644 --- a/fs/tests/exec_kunit.c +++ b/fs/tests/exec_kunit.c @@ -87,9 +87,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_argmin = ULONG_MAX - sizeof(void *) }, /* Raising rlim_stack / 4 to _STK_LIM / 4 * 3 will see more space. */ - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, @@ -97,12 +94,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3 + sizeof(void *)), .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * + sizeof(void *)), - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 8705c77a9e75a..93c231601c8e2 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -757,7 +757,7 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry const struct eventfs_entry *entries, int size, void *data) { - struct dentry *dentry = tracefs_start_creating(name, parent); + struct dentry *dentry; struct eventfs_root_inode *rei; struct eventfs_inode *ei; struct tracefs_inode *ti; @@ -768,6 +768,7 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry if (security_locked_down(LOCKDOWN_TRACEFS)) return NULL; + dentry = tracefs_start_creating(name, parent); if (IS_ERR(dentry)) return ERR_CAST(dentry); diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 8c04acd30d489..b88e65c7e45de 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -50,7 +50,6 @@ STATIC int xfs_attr_shortform_addname(xfs_da_args_t *args); */ STATIC int xfs_attr_leaf_get(xfs_da_args_t *args); STATIC int xfs_attr_leaf_removename(xfs_da_args_t *args); -STATIC int xfs_attr_leaf_hasname(struct xfs_da_args *args, struct xfs_buf **bp); /* * Internal routines when attribute list is more than one block. @@ -979,11 +978,12 @@ xfs_attr_lookup( return error; if (xfs_attr_is_leaf(dp)) { - error = xfs_attr_leaf_hasname(args, &bp); - - if (bp) - xfs_trans_brelse(args->trans, bp); - + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + xfs_trans_brelse(args->trans, bp); return error; } @@ -1222,27 +1222,6 @@ xfs_attr_shortform_addname( * External routines when attribute list is one block *========================================================================*/ -/* - * Return EEXIST if attr is found, or ENOATTR if not - */ -STATIC int -xfs_attr_leaf_hasname( - struct xfs_da_args *args, - struct xfs_buf **bp) -{ - int error = 0; - - error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, bp); - if (error) - return error; - - error = xfs_attr3_leaf_lookup_int(*bp, args); - if (error != -ENOATTR && error != -EEXIST) - xfs_trans_brelse(args->trans, *bp); - - return error; -} - /* * Remove a name from the leaf attribute list structure * @@ -1253,25 +1232,22 @@ STATIC int xfs_attr_leaf_removename( struct xfs_da_args *args) { - struct xfs_inode *dp; - struct xfs_buf *bp; + struct xfs_inode *dp = args->dp; int error, forkoff; + struct xfs_buf *bp; trace_xfs_attr_leaf_removename(args); - /* - * Remove the attribute. - */ - dp = args->dp; - - error = xfs_attr_leaf_hasname(args, &bp); - if (error == -ENOATTR) { + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error != -EEXIST) { xfs_trans_brelse(args->trans, bp); - if (args->op_flags & XFS_DA_OP_RECOVERY) + if (error == -ENOATTR && (args->op_flags & XFS_DA_OP_RECOVERY)) return 0; return error; - } else if (error != -EEXIST) - return error; + } xfs_attr3_leaf_remove(bp, args); @@ -1295,23 +1271,20 @@ xfs_attr_leaf_removename( * Returns 0 on successful retrieval, otherwise an error. */ STATIC int -xfs_attr_leaf_get(xfs_da_args_t *args) +xfs_attr_leaf_get( + struct xfs_da_args *args) { - struct xfs_buf *bp; - int error; + struct xfs_buf *bp; + int error; trace_xfs_attr_leaf_get(args); - error = xfs_attr_leaf_hasname(args, &bp); - - if (error == -ENOATTR) { - xfs_trans_brelse(args->trans, bp); - return error; - } else if (error != -EEXIST) + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) return error; - - - error = xfs_attr3_leaf_getvalue(bp, args); + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error == -EEXIST) + error = xfs_attr3_leaf_getvalue(bp, args); xfs_trans_brelse(args->trans, bp); return error; } diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 91c1b30ebaab3..b858e3c2ad50a 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -1476,6 +1476,7 @@ xfs_attr3_leaf_add_work( struct xfs_attr_leaf_name_local *name_loc; struct xfs_attr_leaf_name_remote *name_rmt; struct xfs_mount *mp; + int old_end, new_end; int tmp; int i; @@ -1568,17 +1569,49 @@ xfs_attr3_leaf_add_work( if (be16_to_cpu(entry->nameidx) < ichdr->firstused) ichdr->firstused = be16_to_cpu(entry->nameidx); - ASSERT(ichdr->firstused >= ichdr->count * sizeof(xfs_attr_leaf_entry_t) - + xfs_attr3_leaf_hdr_size(leaf)); - tmp = (ichdr->count - 1) * sizeof(xfs_attr_leaf_entry_t) - + xfs_attr3_leaf_hdr_size(leaf); + new_end = ichdr->count * sizeof(struct xfs_attr_leaf_entry) + + xfs_attr3_leaf_hdr_size(leaf); + old_end = new_end - sizeof(struct xfs_attr_leaf_entry); + + ASSERT(ichdr->firstused >= new_end); for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { - if (ichdr->freemap[i].base == tmp) { - ichdr->freemap[i].base += sizeof(xfs_attr_leaf_entry_t); + int diff = 0; + + if (ichdr->freemap[i].base == old_end) { + /* + * This freemap entry starts at the old end of the + * leaf entry array, so we need to adjust its base + * upward to accomodate the larger array. + */ + diff = sizeof(struct xfs_attr_leaf_entry); + } else if (ichdr->freemap[i].size > 0 && + ichdr->freemap[i].base < new_end) { + /* + * This freemap entry starts in the space claimed by + * the new leaf entry. Adjust its base upward to + * reflect that. + */ + diff = new_end - ichdr->freemap[i].base; + } + + if (diff) { + ichdr->freemap[i].base += diff; ichdr->freemap[i].size -= - min_t(uint16_t, ichdr->freemap[i].size, - sizeof(xfs_attr_leaf_entry_t)); + min_t(uint16_t, ichdr->freemap[i].size, diff); + } + + /* + * Don't leave zero-length freemaps with nonzero base lying + * around, because we don't want the code in _remove that + * matches on base address to get confused and create + * overlapping freemaps. If we end up with no freemap entries + * then the next _add will compact the leaf block and + * regenerate the freemaps. + */ + if (ichdr->freemap[i].size == 0 && ichdr->freemap[i].base > 0) { + ichdr->freemap[i].base = 0; + ichdr->holes = 1; } } ichdr->usedbytes += xfs_attr_leaf_entsize(leaf, args->index); diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 5b377cbbb1f7e..e8db2f6149e07 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -809,7 +809,7 @@ xfs_defer_can_append( /* Paused items cannot absorb more work */ if (dfp->dfp_flags & XFS_DEFER_PAUSED) - return NULL; + return false; /* Already full? */ if (ops->max_items && dfp->dfp_count >= ops->max_items) diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index d97295eaebe63..c19d6d713780c 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -848,15 +848,16 @@ xfs_ialloc_ag_alloc( * invalid inode records, such as records that start at agbno 0 * or extend beyond the AG. * - * Set min agbno to the first aligned, non-zero agbno and max to - * the last aligned agbno that is at least one full chunk from - * the end of the AG. + * Set min agbno to the first chunk aligned, non-zero agbno and + * max to one less than the last chunk aligned agbno from the + * end of the AG. We subtract 1 from max so that the cluster + * allocation alignment takes over and allows allocation within + * the last full inode chunk in the AG. */ args.min_agbno = args.mp->m_sb.sb_inoalignmt; args.max_agbno = round_down(xfs_ag_block_count(args.mp, pag_agno(pag)), - args.mp->m_sb.sb_inoalignmt) - - igeo->ialloc_blks; + args.mp->m_sb.sb_inoalignmt) - 1; error = xfs_alloc_vextent_near_bno(&args, xfs_agbno_to_fsb(pag, diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index cdd16dd805d77..94c272a2ae262 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -301,6 +301,21 @@ xfs_validate_rt_geometry( sbp->sb_rbmblocks != xfs_expected_rbmblocks(sbp)) return false; + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED)) { + uint32_t mod; + + /* + * Zoned RT devices must be aligned to the RT group size, + * because garbage collection assumes that all zones have the + * same size to avoid insane complexity if that weren't the + * case. + */ + div_u64_rem(sbp->sb_rextents, sbp->sb_rgextents, &mod); + if (mod) + return false; + } + return true; } diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index cd6f0223879f4..6e3fef36d6614 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -837,8 +837,12 @@ xrep_agi_buf_cleanup( { struct xrep_agi *ragi = buf; - xfarray_destroy(ragi->iunlink_prev); - xfarray_destroy(ragi->iunlink_next); + if (ragi->iunlink_prev) + xfarray_destroy(ragi->iunlink_prev); + ragi->iunlink_prev = NULL; + if (ragi->iunlink_next) + xfarray_destroy(ragi->iunlink_next); + ragi->iunlink_next = NULL; xagino_bitmap_destroy(&ragi->iunlink_bmp); } @@ -1708,7 +1712,6 @@ xrep_agi( { struct xrep_agi *ragi; struct xfs_mount *mp = sc->mp; - char *descr; unsigned int i; int error; @@ -1742,17 +1745,13 @@ xrep_agi( xagino_bitmap_init(&ragi->iunlink_bmp); sc->buf_cleanup = xrep_agi_buf_cleanup; - descr = xchk_xfile_ag_descr(sc, "iunlinked next pointers"); - error = xfarray_create(descr, 0, sizeof(xfs_agino_t), - &ragi->iunlink_next); - kfree(descr); + error = xfarray_create("iunlinked next pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_next); if (error) return error; - descr = xchk_xfile_ag_descr(sc, "iunlinked prev pointers"); - error = xfarray_create(descr, 0, sizeof(xfs_agino_t), - &ragi->iunlink_prev); - kfree(descr); + error = xfarray_create("iunlinked prev pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_prev); if (error) return error; diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c index bed6a09aa7911..35035d02a2316 100644 --- a/fs/xfs/scrub/alloc_repair.c +++ b/fs/xfs/scrub/alloc_repair.c @@ -850,7 +850,6 @@ xrep_allocbt( struct xrep_abt *ra; struct xfs_mount *mp = sc->mp; unsigned int busy_gen; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -876,11 +875,9 @@ xrep_allocbt( } /* Set up enough storage to handle maximally fragmented free space. */ - descr = xchk_xfile_ag_descr(sc, "free space records"); - error = xfarray_create(descr, mp->m_sb.sb_agblocks / 2, + error = xfarray_create("free space records", mp->m_sb.sb_agblocks / 2, sizeof(struct xfs_alloc_rec_incore), &ra->free_records); - kfree(descr); if (error) goto out_ra; @@ -926,7 +923,22 @@ xrep_revalidate_allocbt( if (error) goto out; + /* + * If the bnobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the bnobt fails cross-examination with the cntbt, the scan will + * free the cntbt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL cntbt cursor. + */ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + sc->sm->sm_type = XFS_SCRUB_TYPE_CNTBT; + if (!sc->sa.cnt_cur) { + xchk_set_incomplete(sc); + goto out; + } error = xchk_allocbt(sc); out: sc->sm->sm_type = old_type; diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 708334f9b2bd1..a0878fdbcf386 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -287,32 +287,6 @@ xchk_xattr_set_map( return ret; } -/* - * Check the leaf freemap from the usage bitmap. Returns false if the - * attr freemap has problems or points to used space. - */ -STATIC bool -xchk_xattr_check_freemap( - struct xfs_scrub *sc, - struct xfs_attr3_icleaf_hdr *leafhdr) -{ - struct xchk_xattr_buf *ab = sc->buf; - unsigned int mapsize = sc->mp->m_attr_geo->blksize; - int i; - - /* Construct bitmap of freemap contents. */ - bitmap_zero(ab->freemap, mapsize); - for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { - if (!xchk_xattr_set_map(sc, ab->freemap, - leafhdr->freemap[i].base, - leafhdr->freemap[i].size)) - return false; - } - - /* Look for bits that are set in freemap and are marked in use. */ - return !bitmap_intersects(ab->freemap, ab->usedmap, mapsize); -} - /* * Check this leaf entry's relations to everything else. * Returns the number of bytes used for the name/value data. @@ -364,7 +338,10 @@ xchk_xattr_entry( rentry = xfs_attr3_leaf_name_remote(leaf, idx); namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); name_end = (char *)rentry + namesize; - if (rentry->namelen == 0 || rentry->valueblk == 0) + if (rentry->namelen == 0) + xchk_da_set_corrupt(ds, level); + if (rentry->valueblk == 0 && + !(ent->flags & XFS_ATTR_INCOMPLETE)) xchk_da_set_corrupt(ds, level); } if (name_end > buf_end) @@ -403,6 +380,7 @@ xchk_xattr_block( *last_checked = blk->blkno; bitmap_zero(ab->usedmap, mp->m_attr_geo->blksize); + bitmap_zero(ab->freemap, mp->m_attr_geo->blksize); /* Check all the padding. */ if (xfs_has_crc(ds->sc->mp)) { @@ -449,6 +427,9 @@ xchk_xattr_block( if ((char *)&entries[leafhdr.count] > (char *)leaf + leafhdr.firstused) xchk_da_set_corrupt(ds, level); + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; for (i = 0, ent = entries; i < leafhdr.count; ent++, i++) { /* Mark the leaf entry itself. */ @@ -467,7 +448,29 @@ xchk_xattr_block( goto out; } - if (!xchk_xattr_check_freemap(ds->sc, &leafhdr)) + /* Construct bitmap of freemap contents. */ + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + if (!xchk_xattr_set_map(ds->sc, ab->freemap, + leafhdr.freemap[i].base, + leafhdr.freemap[i].size)) + xchk_da_set_corrupt(ds, level); + + /* + * freemap entries with zero length and nonzero base can cause + * problems with older kernels, so we mark these for preening + * even though there's no inconsistency. + */ + if (leafhdr.freemap[i].size == 0 && + leafhdr.freemap[i].base > 0) + xchk_da_set_preen(ds, level); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + } + + /* Look for bits that are set in freemap and are marked in use. */ + if (bitmap_intersects(ab->freemap, ab->usedmap, + mp->m_attr_geo->blksize)) xchk_da_set_corrupt(ds, level); if (leafhdr.usedbytes != usedbytes) diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index c7eb94069cafc..dd24044c44efd 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -333,7 +333,6 @@ xrep_xattr_salvage_remote_attr( .attr_filter = ent->flags & XFS_ATTR_NSP_ONDISK_MASK, .namelen = rentry->namelen, .name = rentry->name, - .value = ab->value, .valuelen = be32_to_cpu(rentry->valuelen), }; unsigned int namesize; @@ -363,6 +362,7 @@ xrep_xattr_salvage_remote_attr( error = -EDEADLOCK; if (error) return error; + args.value = ab->value; /* Look up the remote value and stash it for reconstruction. */ error = xfs_attr3_leaf_getvalue(leaf_bp, &args); @@ -1516,8 +1516,10 @@ xrep_xattr_teardown( xfblob_destroy(rx->pptr_names); if (rx->pptr_recs) xfarray_destroy(rx->pptr_recs); - xfblob_destroy(rx->xattr_blobs); - xfarray_destroy(rx->xattr_records); + if (rx->xattr_blobs) + xfblob_destroy(rx->xattr_blobs); + if (rx->xattr_records) + xfarray_destroy(rx->xattr_records); mutex_destroy(&rx->lock); kfree(rx); } @@ -1529,7 +1531,6 @@ xrep_xattr_setup_scan( struct xrep_xattr **rxp) { struct xrep_xattr *rx; - char *descr; int max_len; int error; @@ -1555,35 +1556,26 @@ xrep_xattr_setup_scan( goto out_rx; /* Set up some staging for salvaged attribute keys and values */ - descr = xchk_xfile_ino_descr(sc, "xattr keys"); - error = xfarray_create(descr, 0, sizeof(struct xrep_xattr_key), + error = xfarray_create("xattr keys", 0, sizeof(struct xrep_xattr_key), &rx->xattr_records); - kfree(descr); if (error) goto out_rx; - descr = xchk_xfile_ino_descr(sc, "xattr names"); - error = xfblob_create(descr, &rx->xattr_blobs); - kfree(descr); + error = xfblob_create("xattr names", &rx->xattr_blobs); if (error) goto out_keys; if (xfs_has_parent(sc->mp)) { ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); - descr = xchk_xfile_ino_descr(sc, - "xattr retained parent pointer entries"); - error = xfarray_create(descr, 0, + error = xfarray_create("xattr parent pointer entries", 0, sizeof(struct xrep_xattr_pptr), &rx->pptr_recs); - kfree(descr); if (error) goto out_values; - descr = xchk_xfile_ino_descr(sc, - "xattr retained parent pointer names"); - error = xfblob_create(descr, &rx->pptr_names); - kfree(descr); + error = xfblob_create("xattr parent pointer names", + &rx->pptr_names); if (error) goto out_pprecs; diff --git a/fs/xfs/scrub/bmap_repair.c b/fs/xfs/scrub/bmap_repair.c index 1084213b8e9b8..747cd9389b491 100644 --- a/fs/xfs/scrub/bmap_repair.c +++ b/fs/xfs/scrub/bmap_repair.c @@ -923,7 +923,6 @@ xrep_bmap( bool allow_unwritten) { struct xrep_bmap *rb; - char *descr; xfs_extnum_t max_bmbt_recs; bool large_extcount; int error = 0; @@ -945,11 +944,8 @@ xrep_bmap( /* Set up enough storage to handle the max records for this fork. */ large_extcount = xfs_has_large_extent_counts(sc->mp); max_bmbt_recs = xfs_iext_max_nextents(large_extcount, whichfork); - descr = xchk_xfile_ino_descr(sc, "%s fork mapping records", - whichfork == XFS_DATA_FORK ? "data" : "attr"); - error = xfarray_create(descr, max_bmbt_recs, + error = xfarray_create("fork mapping records", max_bmbt_recs, sizeof(struct xfs_bmbt_rec), &rb->bmap_records); - kfree(descr); if (error) goto out_rb; diff --git a/fs/xfs/scrub/btree.c b/fs/xfs/scrub/btree.c index cd6f0ff382a7c..b497f6a474c77 100644 --- a/fs/xfs/scrub/btree.c +++ b/fs/xfs/scrub/btree.c @@ -42,6 +42,8 @@ __xchk_btree_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -370,12 +372,15 @@ xchk_btree_check_block_owner( { xfs_agnumber_t agno; xfs_agblock_t agbno; + bool is_bnobt, is_rmapbt; bool init_sa; int error = 0; if (!bs->cur) return 0; + is_bnobt = xfs_btree_is_bno(bs->cur->bc_ops); + is_rmapbt = xfs_btree_is_rmap(bs->cur->bc_ops); agno = xfs_daddr_to_agno(bs->cur->bc_mp, daddr); agbno = xfs_daddr_to_agbno(bs->cur->bc_mp, daddr); @@ -398,11 +403,11 @@ xchk_btree_check_block_owner( * have to nullify it (to shut down further block owner checks) if * self-xref encounters problems. */ - if (!bs->sc->sa.bno_cur && xfs_btree_is_bno(bs->cur->bc_ops)) + if (!bs->sc->sa.bno_cur && is_bnobt) bs->cur = NULL; xchk_xref_is_only_owned_by(bs->sc, agbno, 1, bs->oinfo); - if (!bs->sc->sa.rmap_cur && xfs_btree_is_rmap(bs->cur->bc_ops)) + if (!bs->sc->sa.rmap_cur && is_rmapbt) bs->cur = NULL; out_free: diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 2ef7742be7d3d..975c879c8d7f5 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -103,6 +103,8 @@ __xchk_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -177,6 +179,8 @@ __xchk_fblock_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -1395,6 +1399,9 @@ xchk_metadata_inode_subtype( int error; sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + error = sub->sc.ops->scrub(&sub->sc); xchk_scrub_free_subord(sub); return error; diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index ddbc065c798cd..f2ecc68538f0c 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -246,31 +246,6 @@ static inline bool xchk_could_repair(const struct xfs_scrub *sc) int xchk_metadata_inode_forks(struct xfs_scrub *sc); -/* - * Helper macros to allocate and format xfile description strings. - * Callers must kfree the pointer returned. - */ -#define xchk_xfile_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): " fmt, \ - (sc)->mp->m_super->s_id, ##__VA_ARGS__) -#define xchk_xfile_ag_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): AG 0x%x " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->sa.pag ? \ - pag_agno((sc)->sa.pag) : (sc)->sm->sm_agno, \ - ##__VA_ARGS__) -#define xchk_xfile_ino_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): inode 0x%llx " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->ip ? (sc)->ip->i_ino : (sc)->sm->sm_ino, \ - ##__VA_ARGS__) -#define xchk_xfile_rtgroup_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): rtgroup 0x%x " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->sa.pag ? \ - rtg_rgno((sc)->sr.rtg) : (sc)->sm->sm_agno, \ - ##__VA_ARGS__) - /* * Setting up a hook to wait for intents to drain is costly -- we have to take * the CPU hotplug lock and force an i-cache flush on all CPUs once to set it diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 056de4819f866..a6a5d3a75d994 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -45,6 +45,8 @@ xchk_da_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; *error = 0; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index c877bde71e628..4f849d98cbdd2 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -1102,22 +1102,17 @@ xchk_directory( sd->xname.name = sd->namebuf; if (xfs_has_parent(sc->mp)) { - char *descr; - /* * Set up some staging memory for dirents that we can't check * due to locking contention. */ - descr = xchk_xfile_ino_descr(sc, "slow directory entries"); - error = xfarray_create(descr, 0, sizeof(struct xchk_dirent), - &sd->dir_entries); - kfree(descr); + error = xfarray_create("slow directory entries", 0, + sizeof(struct xchk_dirent), &sd->dir_entries); if (error) goto out_sd; - descr = xchk_xfile_ino_descr(sc, "slow directory entry names"); - error = xfblob_create(descr, &sd->dir_names); - kfree(descr); + error = xfblob_create("slow directory entry names", + &sd->dir_names); if (error) goto out_entries; } diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 8d3b550990b58..e0fa5e6ca1fe8 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -172,8 +172,12 @@ xrep_dir_teardown( struct xrep_dir *rd = sc->buf; xrep_findparent_scan_teardown(&rd->pscan); - xfblob_destroy(rd->dir_names); - xfarray_destroy(rd->dir_entries); + if (rd->dir_names) + xfblob_destroy(rd->dir_names); + rd->dir_names = NULL; + if (rd->dir_entries) + xfarray_destroy(rd->dir_entries); + rd->dir_entries = NULL; } /* Set up for a directory repair. */ @@ -1784,20 +1788,15 @@ xrep_dir_setup_scan( struct xrep_dir *rd) { struct xfs_scrub *sc = rd->sc; - char *descr; int error; /* Set up some staging memory for salvaging dirents. */ - descr = xchk_xfile_ino_descr(sc, "directory entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_dirent), - &rd->dir_entries); - kfree(descr); + error = xfarray_create("directory entries", 0, + sizeof(struct xrep_dirent), &rd->dir_entries); if (error) return error; - descr = xchk_xfile_ino_descr(sc, "directory entry names"); - error = xfblob_create(descr, &rd->dir_names); - kfree(descr); + error = xfblob_create("directory entry names", &rd->dir_names); if (error) goto out_xfarray; diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 3a9cdf8738b6d..3e0bbe75c44cf 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -81,8 +81,12 @@ xchk_dirtree_buf_cleanup( kfree(path); } - xfblob_destroy(dl->path_names); - xfarray_destroy(dl->path_steps); + if (dl->path_names) + xfblob_destroy(dl->path_names); + dl->path_names = NULL; + if (dl->path_steps) + xfarray_destroy(dl->path_steps); + dl->path_steps = NULL; mutex_destroy(&dl->lock); } @@ -92,7 +96,6 @@ xchk_setup_dirtree( struct xfs_scrub *sc) { struct xchk_dirtree *dl; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); @@ -116,16 +119,12 @@ xchk_setup_dirtree( mutex_init(&dl->lock); - descr = xchk_xfile_ino_descr(sc, "dirtree path steps"); - error = xfarray_create(descr, 0, sizeof(struct xchk_dirpath_step), - &dl->path_steps); - kfree(descr); + error = xfarray_create("dirtree path steps", 0, + sizeof(struct xchk_dirpath_step), &dl->path_steps); if (error) goto out_dl; - descr = xchk_xfile_ino_descr(sc, "dirtree path names"); - error = xfblob_create(descr, &dl->path_names); - kfree(descr); + error = xfblob_create("dirtree path names", &dl->path_names); if (error) goto out_steps; diff --git a/fs/xfs/scrub/ialloc_repair.c b/fs/xfs/scrub/ialloc_repair.c index 14e48d3f1912b..f28459f58832f 100644 --- a/fs/xfs/scrub/ialloc_repair.c +++ b/fs/xfs/scrub/ialloc_repair.c @@ -797,7 +797,6 @@ xrep_iallocbt( { struct xrep_ibt *ri; struct xfs_mount *mp = sc->mp; - char *descr; xfs_agino_t first_agino, last_agino; int error = 0; @@ -816,11 +815,9 @@ xrep_iallocbt( /* Set up enough storage to handle an AG with nothing but inodes. */ xfs_agino_range(mp, pag_agno(sc->sa.pag), &first_agino, &last_agino); last_agino /= XFS_INODES_PER_CHUNK; - descr = xchk_xfile_ag_descr(sc, "inode index records"); - error = xfarray_create(descr, last_agino, + error = xfarray_create("inode index records", last_agino, sizeof(struct xfs_inobt_rec_incore), &ri->inode_records); - kfree(descr); if (error) goto out_ri; @@ -866,10 +863,24 @@ xrep_revalidate_iallocbt( if (error) goto out; - if (xfs_has_finobt(sc->mp)) { - sc->sm->sm_type = XFS_SCRUB_TYPE_FINOBT; - error = xchk_iallocbt(sc); + /* + * If the inobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the inobt fails cross-examination with the finobt, the scan will + * free the finobt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL finobt cursor. + */ + if (!xfs_has_finobt(sc->mp) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + goto out; + + sc->sm->sm_type = XFS_SCRUB_TYPE_FINOBT; + if (!sc->sa.fino_cur) { + xchk_set_incomplete(sc); + goto out; } + error = xchk_iallocbt(sc); out: sc->sm->sm_type = old_type; diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index 091c79e432e59..dec3b9b47453e 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -971,7 +971,8 @@ xchk_nlinks_teardown_scan( xfs_dir_hook_del(xnc->sc->mp, &xnc->dhook); - xfarray_destroy(xnc->nlinks); + if (xnc->nlinks) + xfarray_destroy(xnc->nlinks); xnc->nlinks = NULL; xchk_iscan_teardown(&xnc->collect_iscan); @@ -990,7 +991,6 @@ xchk_nlinks_setup_scan( struct xchk_nlink_ctrs *xnc) { struct xfs_mount *mp = sc->mp; - char *descr; unsigned long long max_inos; xfs_agnumber_t last_agno = mp->m_sb.sb_agcount - 1; xfs_agino_t first_agino, last_agino; @@ -1007,10 +1007,9 @@ xchk_nlinks_setup_scan( */ xfs_agino_range(mp, last_agno, &first_agino, &last_agino); max_inos = XFS_AGINO_TO_INO(mp, last_agno, last_agino) + 1; - descr = xchk_xfile_descr(sc, "file link counts"); - error = xfarray_create(descr, min(XFS_MAXINUMBER + 1, max_inos), + error = xfarray_create("file link counts", + min(XFS_MAXINUMBER + 1, max_inos), sizeof(struct xchk_nlink), &xnc->nlinks); - kfree(descr); if (error) goto out_teardown; diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 9c12cb8442311..3c4f3b542c20f 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -445,6 +445,11 @@ xrep_adoption_check_dcache( return 0; d_child = try_lookup_noperm(&qname, d_orphanage); + if (IS_ERR(d_child)) { + dput(d_orphanage); + return PTR_ERR(d_child); + } + if (d_child) { trace_xrep_adoption_check_child(sc->mp, d_child); @@ -482,7 +487,7 @@ xrep_adoption_zap_dcache( return; d_child = try_lookup_noperm(&qname, d_orphanage); - while (d_child != NULL) { + while (!IS_ERR_OR_NULL(d_child)) { trace_xrep_adoption_invalidate_child(sc->mp, d_child); ASSERT(d_is_negative(d_child)); diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 3b692c4acc1e6..f2ee520cc9429 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -755,7 +755,6 @@ xchk_parent_pptr( struct xfs_scrub *sc) { struct xchk_pptrs *pp; - char *descr; int error; pp = kvzalloc(sizeof(struct xchk_pptrs), XCHK_GFP_FLAGS); @@ -768,16 +767,12 @@ xchk_parent_pptr( * Set up some staging memory for parent pointers that we can't check * due to locking contention. */ - descr = xchk_xfile_ino_descr(sc, "slow parent pointer entries"); - error = xfarray_create(descr, 0, sizeof(struct xchk_pptr), - &pp->pptr_entries); - kfree(descr); + error = xfarray_create("slow parent pointer entries", 0, + sizeof(struct xchk_pptr), &pp->pptr_entries); if (error) goto out_pp; - descr = xchk_xfile_ino_descr(sc, "slow parent pointer names"); - error = xfblob_create(descr, &pp->pptr_names); - kfree(descr); + error = xfblob_create("slow parent pointer names", &pp->pptr_names); if (error) goto out_entries; diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 2949feda62717..897902c54178d 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -1497,7 +1497,6 @@ xrep_parent_setup_scan( struct xrep_parent *rp) { struct xfs_scrub *sc = rp->sc; - char *descr; struct xfs_da_geometry *geo = sc->mp->m_attr_geo; int max_len; int error; @@ -1525,32 +1524,22 @@ xrep_parent_setup_scan( goto out_xattr_name; /* Set up some staging memory for logging parent pointer updates. */ - descr = xchk_xfile_ino_descr(sc, "parent pointer entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_pptr), - &rp->pptr_recs); - kfree(descr); + error = xfarray_create("parent pointer entries", 0, + sizeof(struct xrep_pptr), &rp->pptr_recs); if (error) goto out_xattr_value; - descr = xchk_xfile_ino_descr(sc, "parent pointer names"); - error = xfblob_create(descr, &rp->pptr_names); - kfree(descr); + error = xfblob_create("parent pointer names", &rp->pptr_names); if (error) goto out_recs; /* Set up some storage for copying attrs before the mapping exchange */ - descr = xchk_xfile_ino_descr(sc, - "parent pointer retained xattr entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_parent_xattr), - &rp->xattr_records); - kfree(descr); + error = xfarray_create("parent pointer xattr entries", 0, + sizeof(struct xrep_parent_xattr), &rp->xattr_records); if (error) goto out_names; - descr = xchk_xfile_ino_descr(sc, - "parent pointer retained xattr values"); - error = xfblob_create(descr, &rp->xattr_blobs); - kfree(descr); + error = xfblob_create("parent pointer xattr values", &rp->xattr_blobs); if (error) goto out_attr_keys; diff --git a/fs/xfs/scrub/quota.c b/fs/xfs/scrub/quota.c index 58d6d4ed2853b..5d43a8924f090 100644 --- a/fs/xfs/scrub/quota.c +++ b/fs/xfs/scrub/quota.c @@ -174,8 +174,10 @@ xchk_quota_item( error = xchk_quota_item_bmap(sc, dq, offset); xchk_iunlock(sc, XFS_ILOCK_SHARED); - if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) { + mutex_unlock(&dq->q_qlock); return error; + } /* * Warn if the hard limits are larger than the fs. diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c index e4105aaafe845..38e855fe0feeb 100644 --- a/fs/xfs/scrub/quotacheck.c +++ b/fs/xfs/scrub/quotacheck.c @@ -742,7 +742,6 @@ xqcheck_setup_scan( struct xfs_scrub *sc, struct xqcheck *xqc) { - char *descr; struct xfs_quotainfo *qi = sc->mp->m_quotainfo; unsigned long long max_dquots = XFS_DQ_ID_MAX + 1ULL; int error; @@ -757,28 +756,22 @@ xqcheck_setup_scan( error = -ENOMEM; if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_USER)) { - descr = xchk_xfile_descr(sc, "user dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("user dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->ucounts); - kfree(descr); if (error) goto out_teardown; } if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_GROUP)) { - descr = xchk_xfile_descr(sc, "group dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("group dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->gcounts); - kfree(descr); if (error) goto out_teardown; } if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_PROJ)) { - descr = xchk_xfile_descr(sc, "project dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("project dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->pcounts); - kfree(descr); if (error) goto out_teardown; } diff --git a/fs/xfs/scrub/refcount_repair.c b/fs/xfs/scrub/refcount_repair.c index 9c8cb5332da04..360fd7354880a 100644 --- a/fs/xfs/scrub/refcount_repair.c +++ b/fs/xfs/scrub/refcount_repair.c @@ -123,13 +123,7 @@ int xrep_setup_ag_refcountbt( struct xfs_scrub *sc) { - char *descr; - int error; - - descr = xchk_xfile_ag_descr(sc, "rmap record bag"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); - return error; + return xrep_setup_xfbtree(sc, "rmap record bag"); } /* Check for any obvious conflicts with this shared/CoW staging extent. */ @@ -704,7 +698,6 @@ xrep_refcountbt( { struct xrep_refc *rr; struct xfs_mount *mp = sc->mp; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -717,11 +710,9 @@ xrep_refcountbt( rr->sc = sc; /* Set up enough storage to handle one refcount record per block. */ - descr = xchk_xfile_ag_descr(sc, "reference count records"); - error = xfarray_create(descr, mp->m_sb.sb_agblocks, + error = xfarray_create("reference count records", mp->m_sb.sb_agblocks, sizeof(struct xfs_refcount_irec), &rr->refcount_records); - kfree(descr); if (error) goto out_rr; diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index efd5a7ccdf624..4d45d39e67f11 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -1136,6 +1136,9 @@ xrep_metadata_inode_subtype( * setup/teardown routines. */ sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + error = sub->sc.ops->scrub(&sub->sc); if (error) goto out; diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index 17d4a38d735cb..cfd1cf403b37e 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -164,14 +164,11 @@ xrep_setup_ag_rmapbt( struct xfs_scrub *sc) { struct xrep_rmap *rr; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); - descr = xchk_xfile_ag_descr(sc, "reverse mapping records"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); + error = xrep_setup_xfbtree(sc, "reverse mapping records"); if (error) return error; diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c index 203a1a97c5026..41d6736a529d0 100644 --- a/fs/xfs/scrub/rtbitmap_repair.c +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -43,7 +43,6 @@ xrep_setup_rtbitmap( struct xchk_rtbitmap *rtb) { struct xfs_mount *mp = sc->mp; - char *descr; unsigned long long blocks = mp->m_sb.sb_rbmblocks; int error; @@ -52,9 +51,8 @@ xrep_setup_rtbitmap( return error; /* Create an xfile to hold our reconstructed bitmap. */ - descr = xchk_xfile_rtgroup_descr(sc, "bitmap file"); - error = xfile_create(descr, blocks * mp->m_sb.sb_blocksize, &sc->xfile); - kfree(descr); + error = xfile_create("realtime bitmap file", + blocks * mp->m_sb.sb_blocksize, &sc->xfile); if (error) return error; diff --git a/fs/xfs/scrub/rtrefcount_repair.c b/fs/xfs/scrub/rtrefcount_repair.c index 983362447826d..b35e39cce7ad5 100644 --- a/fs/xfs/scrub/rtrefcount_repair.c +++ b/fs/xfs/scrub/rtrefcount_repair.c @@ -128,13 +128,7 @@ int xrep_setup_rtrefcountbt( struct xfs_scrub *sc) { - char *descr; - int error; - - descr = xchk_xfile_ag_descr(sc, "rmap record bag"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); - return error; + return xrep_setup_xfbtree(sc, "realtime rmap record bag"); } /* Check for any obvious conflicts with this shared/CoW staging extent. */ @@ -704,7 +698,6 @@ xrep_rtrefcountbt( { struct xrep_rtrefc *rr; struct xfs_mount *mp = sc->mp; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -722,11 +715,9 @@ xrep_rtrefcountbt( rr->sc = sc; /* Set up enough storage to handle one refcount record per rt extent. */ - descr = xchk_xfile_ag_descr(sc, "reference count records"); - error = xfarray_create(descr, mp->m_sb.sb_rextents, - sizeof(struct xfs_refcount_irec), + error = xfarray_create("realtime reference count records", + mp->m_sb.sb_rextents, sizeof(struct xfs_refcount_irec), &rr->refcount_records); - kfree(descr); if (error) goto out_rr; diff --git a/fs/xfs/scrub/rtrmap_repair.c b/fs/xfs/scrub/rtrmap_repair.c index 7561941a337a1..749977a66e40f 100644 --- a/fs/xfs/scrub/rtrmap_repair.c +++ b/fs/xfs/scrub/rtrmap_repair.c @@ -103,14 +103,11 @@ xrep_setup_rtrmapbt( struct xfs_scrub *sc) { struct xrep_rtrmap *rr; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); - descr = xchk_xfile_rtgroup_descr(sc, "reverse mapping records"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); + error = xrep_setup_xfbtree(sc, "realtime reverse mapping records"); if (error) return error; diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c index 4ac679c1bd29c..fb78cff2ac3a1 100644 --- a/fs/xfs/scrub/rtsummary.c +++ b/fs/xfs/scrub/rtsummary.c @@ -43,7 +43,6 @@ xchk_setup_rtsummary( struct xfs_scrub *sc) { struct xfs_mount *mp = sc->mp; - char *descr; struct xchk_rtsummary *rts; int error; @@ -70,10 +69,8 @@ xchk_setup_rtsummary( * Create an xfile to construct a new rtsummary file. The xfile allows * us to avoid pinning kernel memory for this purpose. */ - descr = xchk_xfile_descr(sc, "realtime summary file"); - error = xfile_create(descr, XFS_FSB_TO_B(mp, mp->m_rsumblocks), - &sc->xfile); - kfree(descr); + error = xfile_create("realtime summary file", + XFS_FSB_TO_B(mp, mp->m_rsumblocks), &sc->xfile); if (error) return error; diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 3c3b0d25006ff..c312f0a672e65 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -634,7 +634,7 @@ xchk_scrub_create_subord( sub = kzalloc(sizeof(*sub), XCHK_GFP_FLAGS); if (!sub) - return ERR_PTR(-ENOMEM); + return NULL; sub->old_smtype = sc->sm->sm_type; sub->old_smflags = sc->sm->sm_flags; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 39ea651cbb751..286c5f5e05444 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -972,20 +972,12 @@ TRACE_EVENT(xfile_create, TP_STRUCT__entry( __field(dev_t, dev) __field(unsigned long, ino) - __array(char, pathname, MAXNAMELEN) ), TP_fast_assign( - char *path; - __entry->ino = file_inode(xf->file)->i_ino; - path = file_path(xf->file, __entry->pathname, MAXNAMELEN); - if (IS_ERR(path)) - strncpy(__entry->pathname, "(unknown)", - sizeof(__entry->pathname)); ), - TP_printk("xfino 0x%lx path '%s'", - __entry->ino, - __entry->pathname) + TP_printk("xfino 0x%lx", + __entry->ino) ); TRACE_EVENT(xfile_destroy, diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index c3a593319bee7..8beaa474fd33c 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -656,7 +656,6 @@ xfs_attri_recover_work( break; } if (error) { - xfs_irele(ip); XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attrp, sizeof(*attrp)); return ERR_PTR(-EFSCORRUPTED); @@ -737,7 +736,7 @@ xfs_attr_recover_work( struct xfs_attri_log_item *attrip = ATTRI_ITEM(lip); struct xfs_attr_intent *attr; struct xfs_mount *mp = lip->li_log->l_mp; - struct xfs_inode *ip; + struct xfs_inode *ip = NULL; struct xfs_da_args *args; struct xfs_trans *tp; struct xfs_trans_res resv; @@ -1050,8 +1049,8 @@ xlog_recover_attri_commit_pass2( break; case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: - /* Log item, attr name, attr value */ - if (item->ri_total != 3) { + /* Log item, attr name, optional attr value */ + if (item->ri_total != 2 + !!attri_formatp->alfi_value_len) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, len); return -EFSCORRUPTED; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 80f0c4bcc483f..bcc89d65de7fb 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -247,7 +247,7 @@ xfs_bmap_update_diff_items( struct xfs_bmap_intent *ba = bi_entry(a); struct xfs_bmap_intent *bb = bi_entry(b); - return ba->bi_owner->i_ino - bb->bi_owner->i_ino; + return cmp_int(ba->bi_owner->i_ino, bb->bi_owner->i_ino); } /* Log bmap updates in the intent item. */ diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 8d85b5eee4444..f4c5be67826e2 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -896,6 +896,7 @@ xfs_buf_item_init( map_size = DIV_ROUND_UP(chunks, NBWORD); if (map_size > XFS_BLF_DATAMAP_SIZE) { + xfs_buf_item_free_format(bip); kmem_cache_free(xfs_buf_item_cache, bip); xfs_err(mp, "buffer item dirty bitmap (%u uints) too small to reflect %u bytes!", diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 0bd8022e47b4f..92a8863bee36c 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1464,9 +1464,15 @@ xfs_qm_dqflush( return 0; out_abort: + /* + * Shut down the log before removing the dquot item from the AIL. + * Otherwise, the log tail may advance past this item's LSN while + * log writes are still in progress, making these unflushed changes + * unrecoverable on the next mount. + */ + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); dqp->q_flags &= ~XFS_DQFLAG_DIRTY; xfs_trans_ail_delete(lip, 0); - xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); xfs_dqfunlock(dqp); return error; } diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c index 271b195ebb932..9a0abbfe295e8 100644 --- a/fs/xfs/xfs_dquot_item.c +++ b/fs/xfs/xfs_dquot_item.c @@ -126,6 +126,7 @@ xfs_qm_dquot_logitem_push( struct xfs_dq_logitem *qlip = DQUOT_ITEM(lip); struct xfs_dquot *dqp = qlip->qli_dquot; struct xfs_buf *bp; + struct xfs_ail *ailp = lip->li_ailp; uint rval = XFS_ITEM_SUCCESS; int error; @@ -154,7 +155,7 @@ xfs_qm_dquot_logitem_push( goto out_unlock; } - spin_unlock(&lip->li_ailp->ail_lock); + spin_unlock(&ailp->ail_lock); error = xfs_dquot_use_attached_buf(dqp, &bp); if (error == -EAGAIN) { @@ -173,9 +174,13 @@ xfs_qm_dquot_logitem_push( rval = XFS_ITEM_FLUSHING; } xfs_buf_relse(bp); + /* + * The buffer no longer protects the log item from reclaim, so + * do not reference lip after this point. + */ out_relock_ail: - spin_lock(&lip->li_ailp->ail_lock); + spin_lock(&ailp->ail_lock); out_unlock: xfs_dqunlock(dqp); return rval; diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 1bd411a1114c7..dbc604a9cea61 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -749,6 +749,7 @@ xfs_inode_item_push( struct xfs_inode_log_item *iip = INODE_ITEM(lip); struct xfs_inode *ip = iip->ili_inode; struct xfs_buf *bp = lip->li_buf; + struct xfs_ail *ailp = lip->li_ailp; uint rval = XFS_ITEM_SUCCESS; int error; @@ -774,7 +775,7 @@ xfs_inode_item_push( if (!xfs_buf_trylock(bp)) return XFS_ITEM_LOCKED; - spin_unlock(&lip->li_ailp->ail_lock); + spin_unlock(&ailp->ail_lock); /* * We need to hold a reference for flushing the cluster buffer as it may @@ -798,7 +799,11 @@ xfs_inode_item_push( rval = XFS_ITEM_LOCKED; } - spin_lock(&lip->li_ailp->ail_lock); + /* + * The buffer no longer protects the log item from reclaim, so + * do not reference lip after this point. + */ + spin_lock(&ailp->ail_lock); return rval; } diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 603e85c1ab4cd..614b3c385178b 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1397,6 +1397,8 @@ xlog_alloc_log( if (xfs_has_logv2(mp) && mp->m_sb.sb_logsunit > 1) log->l_iclog_roundoff = mp->m_sb.sb_logsunit; + else if (mp->m_sb.sb_logsectsize > 0) + log->l_iclog_roundoff = mp->m_sb.sb_logsectsize; else log->l_iclog_roundoff = BBSIZE; diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 0953f6ae94abc..8848aea99c87f 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -607,8 +607,9 @@ xfs_unmount_check( * have been retrying in the background. This will prevent never-ending * retries in AIL pushing from hanging the unmount. * - * Finally, we can push the AIL to clean all the remaining dirty objects, then - * reclaim the remaining inodes that are still in memory at this point in time. + * Stop inodegc and background reclaim before pushing the AIL so that they + * are not running while the AIL is being flushed. Then push the AIL to + * clean all the remaining dirty objects and reclaim the remaining inodes. */ static void xfs_unmount_flush_inodes( @@ -620,9 +621,9 @@ xfs_unmount_flush_inodes( xfs_set_unmounting(mp); - xfs_ail_push_all_sync(mp->m_ail); xfs_inodegc_stop(mp); cancel_delayed_work_sync(&mp->m_reclaim_work); + xfs_ail_push_all_sync(mp->m_ail); xfs_reclaim_inodes(mp); xfs_health_unmount(mp); } diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c index b176728899420..0700a723f38e7 100644 --- a/fs/xfs/xfs_notify_failure.c +++ b/fs/xfs/xfs_notify_failure.c @@ -293,7 +293,7 @@ xfs_dax_notify_dev_failure( error = xfs_alloc_read_agf(pag, tp, 0, &agf_bp); if (error) { - xfs_perag_put(pag); + xfs_perag_rele(pag); break; } @@ -329,7 +329,7 @@ xfs_dax_notify_dev_failure( if (rtg) xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); if (error) { - xfs_group_put(xg); + xfs_group_rele(xg); break; } } diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 23ba84ec919a4..18a19947bbdba 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1318,7 +1318,7 @@ xfs_qm_quotacheck_dqadjust( error = xfs_dquot_attach_buf(NULL, dqp); if (error) - return error; + goto out_unlock; trace_xfs_dqadjust(dqp); @@ -1348,8 +1348,9 @@ xfs_qm_quotacheck_dqadjust( } dqp->q_flags |= XFS_DQFLAG_DIRTY; +out_unlock: xfs_qm_dqput(dqp); - return 0; + return error; } /* diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index 6907e871fa151..59fad43063a78 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -126,7 +126,7 @@ xfs_rtcopy_summary( error = 0; out: xfs_rtbuf_cache_relse(oargs); - return 0; + return error; } /* * Mark an extent specified by start and len allocated. @@ -1255,12 +1255,10 @@ xfs_growfs_check_rtgeom( min_logfsbs = min_t(xfs_extlen_t, xfs_log_calc_minimum_size(nmp), nmp->m_rsumblocks * 2); - kfree(nmp); - trace_xfs_growfs_check_rtgeom(mp, min_logfsbs); if (min_logfsbs > mp->m_sb.sb_logblocks) - return -EINVAL; + goto out_inval; if (xfs_has_zoned(mp)) { uint32_t gblocks = mp->m_groups[XG_TYPE_RTG].blocks; @@ -1268,16 +1266,20 @@ xfs_growfs_check_rtgeom( if (rextsize != 1) return -EINVAL; - div_u64_rem(mp->m_sb.sb_rblocks, gblocks, &rem); + div_u64_rem(nmp->m_sb.sb_rblocks, gblocks, &rem); if (rem) { xfs_warn(mp, "new RT volume size (%lld) not aligned to RT group size (%d)", - mp->m_sb.sb_rblocks, gblocks); - return -EINVAL; + nmp->m_sb.sb_rblocks, gblocks); + goto out_inval; } } + kfree(nmp); return 0; +out_inval: + kfree(nmp); + return -EINVAL; } /* diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 79b8641880ab9..434829a5ce7e0 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -56,6 +56,7 @@ #include struct xfs_agf; +struct xfs_ail; struct xfs_alloc_arg; struct xfs_attr_list_context; struct xfs_buf_log_item; @@ -1649,16 +1650,43 @@ TRACE_EVENT(xfs_log_force, DEFINE_EVENT(xfs_log_item_class, name, \ TP_PROTO(struct xfs_log_item *lip), \ TP_ARGS(lip)) -DEFINE_LOG_ITEM_EVENT(xfs_ail_push); -DEFINE_LOG_ITEM_EVENT(xfs_ail_pinned); -DEFINE_LOG_ITEM_EVENT(xfs_ail_locked); -DEFINE_LOG_ITEM_EVENT(xfs_ail_flushing); DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_mark); DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_skip); DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_unpin); DEFINE_LOG_ITEM_EVENT(xlog_ail_insert_abort); DEFINE_LOG_ITEM_EVENT(xfs_trans_free_abort); +DECLARE_EVENT_CLASS(xfs_ail_push_class, + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), + TP_ARGS(ailp, type, flags, lsn), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint, type) + __field(unsigned long, flags) + __field(xfs_lsn_t, lsn) + ), + TP_fast_assign( + __entry->dev = ailp->ail_log->l_mp->m_super->s_dev; + __entry->type = type; + __entry->flags = flags; + __entry->lsn = lsn; + ), + TP_printk("dev %d:%d lsn %d/%d type %s flags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + CYCLE_LSN(__entry->lsn), BLOCK_LSN(__entry->lsn), + __print_symbolic(__entry->type, XFS_LI_TYPE_DESC), + __print_flags(__entry->flags, "|", XFS_LI_FLAGS)) +) + +#define DEFINE_AIL_PUSH_EVENT(name) \ +DEFINE_EVENT(xfs_ail_push_class, name, \ + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), \ + TP_ARGS(ailp, type, flags, lsn)) +DEFINE_AIL_PUSH_EVENT(xfs_ail_push); +DEFINE_AIL_PUSH_EVENT(xfs_ail_pinned); +DEFINE_AIL_PUSH_EVENT(xfs_ail_locked); +DEFINE_AIL_PUSH_EVENT(xfs_ail_flushing); + DECLARE_EVENT_CLASS(xfs_ail_class, TP_PROTO(struct xfs_log_item *lip, xfs_lsn_t old_lsn, xfs_lsn_t new_lsn), TP_ARGS(lip, old_lsn, new_lsn), @@ -5089,23 +5117,16 @@ TRACE_EVENT(xmbuf_create, TP_STRUCT__entry( __field(dev_t, dev) __field(unsigned long, ino) - __array(char, pathname, MAXNAMELEN) ), TP_fast_assign( - char *path; struct file *file = btp->bt_file; __entry->dev = btp->bt_mount->m_super->s_dev; __entry->ino = file_inode(file)->i_ino; - path = file_path(file, __entry->pathname, MAXNAMELEN); - if (IS_ERR(path)) - strncpy(__entry->pathname, "(unknown)", - sizeof(__entry->pathname)); ), - TP_printk("dev %d:%d xmino 0x%lx path '%s'", + TP_printk("dev %d:%d xmino 0x%lx", MAJOR(__entry->dev), MINOR(__entry->dev), - __entry->ino, - __entry->pathname) + __entry->ino) ); TRACE_EVENT(xmbuf_free, diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 38983c6777df3..68386db824646 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -365,6 +365,12 @@ xfsaild_resubmit_item( return XFS_ITEM_SUCCESS; } +/* + * Push a single log item from the AIL. + * + * @lip may have been released and freed by the time this function returns, + * so callers must not dereference the log item afterwards. + */ static inline uint xfsaild_push_item( struct xfs_ail *ailp, @@ -505,7 +511,10 @@ xfsaild_push( lsn = lip->li_lsn; while ((XFS_LSN_CMP(lip->li_lsn, ailp->ail_target) <= 0)) { - int lock_result; + int lock_result; + uint type = lip->li_type; + unsigned long flags = lip->li_flags; + xfs_lsn_t item_lsn = lip->li_lsn; if (test_bit(XFS_LI_FLUSHING, &lip->li_flags)) goto next_item; @@ -514,14 +523,17 @@ xfsaild_push( * Note that iop_push may unlock and reacquire the AIL lock. We * rely on the AIL cursor implementation to be able to deal with * the dropped lock. + * + * The log item may have been freed by the push, so it must not + * be accessed or dereferenced below this line. */ lock_result = xfsaild_push_item(ailp, lip); switch (lock_result) { case XFS_ITEM_SUCCESS: XFS_STATS_INC(mp, xs_push_ail_success); - trace_xfs_ail_push(lip); + trace_xfs_ail_push(ailp, type, flags, item_lsn); - ailp->ail_last_pushed_lsn = lsn; + ailp->ail_last_pushed_lsn = item_lsn; break; case XFS_ITEM_FLUSHING: @@ -537,22 +549,22 @@ xfsaild_push( * AIL is being flushed. */ XFS_STATS_INC(mp, xs_push_ail_flushing); - trace_xfs_ail_flushing(lip); + trace_xfs_ail_flushing(ailp, type, flags, item_lsn); flushing++; - ailp->ail_last_pushed_lsn = lsn; + ailp->ail_last_pushed_lsn = item_lsn; break; case XFS_ITEM_PINNED: XFS_STATS_INC(mp, xs_push_ail_pinned); - trace_xfs_ail_pinned(lip); + trace_xfs_ail_pinned(ailp, type, flags, item_lsn); stuck++; ailp->ail_log_flush++; break; case XFS_ITEM_LOCKED: XFS_STATS_INC(mp, xs_push_ail_locked); - trace_xfs_ail_locked(lip); + trace_xfs_ail_locked(ailp, type, flags, item_lsn); stuck++; break; diff --git a/include/acpi/ghes.h b/include/acpi/ghes.h index ebd21b05fe6ed..93db60da5934e 100644 --- a/include/acpi/ghes.h +++ b/include/acpi/ghes.h @@ -21,6 +21,7 @@ struct ghes { struct acpi_hest_generic_v2 *generic_v2; }; struct acpi_hest_generic_status *estatus; + unsigned int estatus_length; unsigned long flags; union { struct list_head list; diff --git a/include/acpi/pcc.h b/include/acpi/pcc.h index 9af3b502f8395..840bfc95bae33 100644 --- a/include/acpi/pcc.h +++ b/include/acpi/pcc.h @@ -17,35 +17,6 @@ struct pcc_mbox_chan { u32 latency; u32 max_access_rate; u16 min_turnaround_time; - - /* Set to true to indicate that the mailbox should manage - * writing the dat to the shared buffer. This differs from - * the case where the drivesr are writing to the buffer and - * using send_data only to ring the doorbell. If this flag - * is set, then the void * data parameter of send_data must - * point to a kernel-memory buffer formatted in accordance with - * the PCC specification. - * - * The active buffer management will include reading the - * notify_on_completion flag, and will then - * call mbox_chan_txdone when the acknowledgment interrupt is - * received. - */ - bool manage_writes; - - /* Optional callback that allows the driver - * to allocate the memory used for receiving - * messages. The return value is the location - * inside the buffer where the mailbox should write the data. - */ - void *(*rx_alloc)(struct mbox_client *cl, int size); -}; - -struct pcc_header { - u32 signature; - u32 flags; - u32 length; - u32 command; }; /* Generic Communications Channel Shared Memory Region */ diff --git a/include/asm-generic/audit_change_attr.h b/include/asm-generic/audit_change_attr.h index cc840537885fb..ddd90bbe40dfc 100644 --- a/include/asm-generic/audit_change_attr.h +++ b/include/asm-generic/audit_change_attr.h @@ -26,6 +26,9 @@ __NR_fremovexattr, __NR_fchownat, __NR_fchmodat, #endif +#ifdef __NR_fchmodat2 +__NR_fchmodat2, +#endif #ifdef __NR_chown32 __NR_chown32, __NR_fchown32, diff --git a/include/asm-generic/audit_read.h b/include/asm-generic/audit_read.h index 7bb7b5a83ae2e..fb9991f53fb6f 100644 --- a/include/asm-generic/audit_read.h +++ b/include/asm-generic/audit_read.h @@ -4,9 +4,15 @@ __NR_readlink, #endif __NR_quotactl, __NR_listxattr, +#ifdef __NR_listxattrat +__NR_listxattrat, +#endif __NR_llistxattr, __NR_flistxattr, __NR_getxattr, +#ifdef __NR_getxattrat +__NR_getxattrat, +#endif __NR_lgetxattr, __NR_fgetxattr, #ifdef __NR_readlinkat diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index 64ba6bc807d98..b89c7e3a20474 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -124,10 +124,12 @@ static inline unsigned int hv_repcomp(u64 status) /* * Rep hypercalls. Callers of this functions are supposed to ensure that - * rep_count and varhead_size comply with Hyper-V hypercall definition. + * rep_count, varhead_size, and rep_start comply with Hyper-V hypercall + * definition. */ -static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, - void *input, void *output) +static inline u64 hv_do_rep_hypercall_ex(u16 code, u16 rep_count, + u16 varhead_size, u16 rep_start, + void *input, void *output) { u64 control = code; u64 status; @@ -135,6 +137,7 @@ static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET; control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET; + control |= (u64)rep_start << HV_HYPERCALL_REP_START_OFFSET; do { status = hv_do_hypercall(control, input, output); @@ -152,6 +155,14 @@ static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, return status; } +/* For the typical case where rep_start is 0 */ +static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, + void *input, void *output) +{ + return hv_do_rep_hypercall_ex(code, rep_count, varhead_size, 0, + input, output); +} + /* Generate the guest OS identifier as described in the Hyper-V TLFS */ static inline u64 hv_generate_guest_id(u64 kernel_version) { diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 3c8ec3bfea447..b9d2a7c79b93a 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -28,6 +28,8 @@ static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm) return NULL; } + ptdesc_set_kernel(ptdesc); + return ptdesc_address(ptdesc); } #define __pte_alloc_one_kernel(...) alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__)) @@ -146,6 +148,10 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad pagetable_free(ptdesc); return NULL; } + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + return ptdesc_address(ptdesc); } #define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) @@ -179,6 +185,10 @@ static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long return NULL; pagetable_pud_ctor(ptdesc); + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + return ptdesc_address(ptdesc); } #define __pud_alloc_one(...) alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__)) @@ -233,6 +243,10 @@ static inline p4d_t *__p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long return NULL; pagetable_p4d_ctor(ptdesc); + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + return ptdesc_address(ptdesc); } #define __p4d_alloc_one(...) alloc_hooks(__p4d_alloc_one_noprof(__VA_ARGS__)) @@ -277,6 +291,10 @@ static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order return NULL; pagetable_pgd_ctor(ptdesc); + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + return ptdesc_address(ptdesc); } #define __pgd_alloc(...) alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__)) diff --git a/include/asm-generic/rqspinlock.h b/include/asm-generic/rqspinlock.h index 6d4244d643df3..5c5cf2f7fc395 100644 --- a/include/asm-generic/rqspinlock.h +++ b/include/asm-generic/rqspinlock.h @@ -129,8 +129,8 @@ static __always_inline void release_held_lock_entry(void) * for lock B * release_held_lock_entry * - * try_cmpxchg_acquire for lock A * grab_held_lock_entry + * try_cmpxchg_acquire for lock A * * Lack of any ordering means reordering may occur such that dec, inc * are done before entry is overwritten. This permits a remote lock @@ -139,13 +139,8 @@ static __always_inline void release_held_lock_entry(void) * CPU holds a lock it is attempting to acquire, leading to false ABBA * diagnosis). * - * In case of unlock, we will always do a release on the lock word after - * releasing the entry, ensuring that other CPUs cannot hold the lock - * (and make conclusions about deadlocks) until the entry has been - * cleared on the local CPU, preventing any anomalies. Reordering is - * still possible there, but a remote CPU cannot observe a lock in our - * table which it is already holding, since visibility entails our - * release store for the said lock has not retired. + * The case of unlock is treated differently due to NMI reentrancy, see + * comments in res_spin_unlock. * * In theory we don't have a problem if the dec and WRITE_ONCE above get * reordered with each other, we either notice an empty NULL entry on @@ -175,16 +170,28 @@ static __always_inline int res_spin_lock(rqspinlock_t *lock) { int val = 0; - if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) { - grab_held_lock_entry(lock); + /* + * Grab the deadlock detection entry before doing the cmpxchg, so that + * reentrancy due to NMIs between the succeeding cmpxchg and creation of + * held lock entry can correctly detect an acquisition attempt in the + * interrupted context. + * + * cmpxchg lock A + * + * res_spin_lock(A) --> missed AA, leads to timeout + * + * grab_held_lock_entry(A) + */ + grab_held_lock_entry(lock); + + if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) return 0; - } return resilient_queued_spin_lock_slowpath(lock, val); } #else -#define res_spin_lock(lock) resilient_tas_spin_lock(lock) +#define res_spin_lock(lock) ({ grab_held_lock_entry(lock); resilient_tas_spin_lock(lock); }) #endif /* CONFIG_QUEUED_SPINLOCKS */ @@ -192,28 +199,25 @@ static __always_inline void res_spin_unlock(rqspinlock_t *lock) { struct rqspinlock_held *rqh = this_cpu_ptr(&rqspinlock_held_locks); - if (unlikely(rqh->cnt > RES_NR_HELD)) - goto unlock; - WRITE_ONCE(rqh->locks[rqh->cnt - 1], NULL); -unlock: /* - * Release barrier, ensures correct ordering. See release_held_lock_entry - * for details. Perform release store instead of queued_spin_unlock, - * since we use this function for test-and-set fallback as well. When we - * have CONFIG_QUEUED_SPINLOCKS=n, we clear the full 4-byte lockword. + * Release barrier, ensures correct ordering. Perform release store + * instead of queued_spin_unlock, since we use this function for the TAS + * fallback as well. When we have CONFIG_QUEUED_SPINLOCKS=n, we clear + * the full 4-byte lockword. * - * Like release_held_lock_entry, we can do the release before the dec. - * We simply care about not seeing the 'lock' in our table from a remote - * CPU once the lock has been released, which doesn't rely on the dec. + * Perform the smp_store_release before clearing the lock entry so that + * NMIs landing in the unlock path can correctly detect AA issues. The + * opposite order shown below may lead to missed AA checks: * - * Unlike smp_wmb(), release is not a two way fence, hence it is - * possible for a inc to move up and reorder with our clearing of the - * entry. This isn't a problem however, as for a misdiagnosis of ABBA, - * the remote CPU needs to hold this lock, which won't be released until - * the store below is done, which would ensure the entry is overwritten - * to NULL, etc. + * WRITE_ONCE(rqh->locks[rqh->cnt - 1], NULL) + * + * res_spin_lock(A) --> missed AA, leads to timeout + * + * smp_store_release(A->locked, 0) */ smp_store_release(&lock->locked, 0); + if (likely(rqh->cnt <= RES_NR_HELD)) + WRITE_ONCE(rqh->locks[rqh->cnt - 1], NULL); this_cpu_dec(rqspinlock_held_locks.cnt); } diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 1fff717cae510..4d679d2a206b4 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -46,7 +46,8 @@ * * The mmu_gather API consists of: * - * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu() + * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_gather_mmu_vma() / + * tlb_finish_mmu() * * start and finish a mmu_gather * @@ -364,6 +365,20 @@ struct mmu_gather { unsigned int vma_huge : 1; unsigned int vma_pfn : 1; + /* + * Did we unshare (unmap) any shared page tables? For now only + * used for hugetlb PMD table sharing. + */ + unsigned int unshared_tables : 1; + + /* + * Did we unshare any page tables such that they are now exclusive + * and could get reused+modified by the new owner? When setting this + * flag, "unshared_tables" will be set as well. For now only used + * for hugetlb PMD table sharing. + */ + unsigned int fully_unshared_tables : 1; + unsigned int batch_count; #ifndef CONFIG_MMU_GATHER_NO_GATHER @@ -400,6 +415,7 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) tlb->cleared_pmds = 0; tlb->cleared_puds = 0; tlb->cleared_p4ds = 0; + tlb->unshared_tables = 0; /* * Do not reset mmu_gather::vma_* fields here, we do not * call into tlb_start_vma() again to set them if there is an @@ -484,7 +500,7 @@ static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) * these bits. */ if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds || - tlb->cleared_puds || tlb->cleared_p4ds)) + tlb->cleared_puds || tlb->cleared_p4ds || tlb->unshared_tables)) return; tlb_flush(tlb); @@ -773,6 +789,63 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd) } #endif +#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING +static inline void tlb_unshare_pmd_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt, + unsigned long addr) +{ + /* + * The caller must make sure that concurrent unsharing + exclusive + * reuse is impossible until tlb_flush_unshared_tables() was called. + */ + VM_WARN_ON_ONCE(!ptdesc_pmd_is_shared(pt)); + ptdesc_pmd_pts_dec(pt); + + /* Clearing a PUD pointing at a PMD table with PMD leaves. */ + tlb_flush_pmd_range(tlb, addr & PUD_MASK, PUD_SIZE); + + /* + * If the page table is now exclusively owned, we fully unshared + * a page table. + */ + if (!ptdesc_pmd_is_shared(pt)) + tlb->fully_unshared_tables = true; + tlb->unshared_tables = true; +} + +static inline void tlb_flush_unshared_tables(struct mmu_gather *tlb) +{ + /* + * As soon as the caller drops locks to allow for reuse of + * previously-shared tables, these tables could get modified and + * even reused outside of hugetlb context, so we have to make sure that + * any page table walkers (incl. TLB, GUP-fast) are aware of that + * change. + * + * Even if we are not fully unsharing a PMD table, we must + * flush the TLB for the unsharer now. + */ + if (tlb->unshared_tables) + tlb_flush_mmu_tlbonly(tlb); + + /* + * Similarly, we must make sure that concurrent GUP-fast will not + * walk previously-shared page tables that are getting modified+reused + * elsewhere. So broadcast an IPI to wait for any concurrent GUP-fast. + * + * We only perform this when we are the last sharer of a page table, + * as the IPI will reach all CPUs: any GUP-fast. + * + * Note that on configs where tlb_remove_table_sync_one() is a NOP, + * the expectation is that the tlb_flush_mmu_tlbonly() would have issued + * required IPIs already for us. + */ + if (tlb->fully_unshared_tables) { + tlb_remove_table_sync_one(); + tlb->fully_unshared_tables = false; + } +} +#endif /* CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */ + #endif /* CONFIG_MMU */ #endif /* _ASM_GENERIC__TLB_H */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index e04d56a5332e6..6640e06ef2f13 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -832,12 +832,14 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) /* Required sections not related to debugging. */ #define ELF_DETAILS \ - .modinfo : { *(.modinfo) . = ALIGN(8); } \ .comment 0 : { *(.comment) } \ .symtab 0 : { *(.symtab) } \ .strtab 0 : { *(.strtab) } \ .shstrtab 0 : { *(.shstrtab) } +#define MODINFO \ + .modinfo : { *(.modinfo) . = ALIGN(8); } + #ifdef CONFIG_GENERIC_BUG #define BUG_TABLE \ . = ALIGN(8); \ diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 107b797c33ecf..0cc8fa749f68d 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -230,9 +230,8 @@ static inline bool af_alg_readable(struct sock *sk) return PAGE_SIZE <= af_alg_rcvbuf(sk); } -unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset); -void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, - size_t dst_offset); +unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes); +void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst); void af_alg_wmem_wakeup(struct sock *sk); int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min); int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 83d14376ff2bc..f485454e3955c 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -227,6 +227,34 @@ static inline void scatterwalk_done_src(struct scatter_walk *walk, scatterwalk_advance(walk, nbytes); } +/* + * Flush the dcache of any pages that overlap the region + * [offset, offset + nbytes) relative to base_page. + * + * This should be called only when ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE, to ensure + * that all relevant code (including the call to sg_page() in the caller, if + * applicable) gets fully optimized out when !ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE. + */ +static inline void __scatterwalk_flush_dcache_pages(struct page *base_page, + unsigned int offset, + unsigned int nbytes) +{ + unsigned int num_pages; + + base_page += offset / PAGE_SIZE; + offset %= PAGE_SIZE; + + /* + * This is an overflow-safe version of + * num_pages = DIV_ROUND_UP(offset + nbytes, PAGE_SIZE). + */ + num_pages = nbytes / PAGE_SIZE; + num_pages += DIV_ROUND_UP(offset + (nbytes % PAGE_SIZE), PAGE_SIZE); + + for (unsigned int i = 0; i < num_pages; i++) + flush_dcache_page(base_page + i); +} + /** * scatterwalk_done_dst() - Finish one step of a walk of destination scatterlist * @walk: the scatter_walk @@ -240,27 +268,9 @@ static inline void scatterwalk_done_dst(struct scatter_walk *walk, unsigned int nbytes) { scatterwalk_unmap(walk); - /* - * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just - * relying on flush_dcache_page() being a no-op when not implemented, - * since otherwise the BUG_ON in sg_page() does not get optimized out. - * This also avoids having to consider whether the loop would get - * reliably optimized out or not. - */ - if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) { - struct page *base_page; - unsigned int offset; - int start, end, i; - - base_page = sg_page(walk->sg); - offset = walk->offset; - start = offset >> PAGE_SHIFT; - end = start + (nbytes >> PAGE_SHIFT); - end += (offset_in_page(offset) + offset_in_page(nbytes) + - PAGE_SIZE - 1) >> PAGE_SHIFT; - for (i = start; i < end; i++) - flush_dcache_page(base_page + i); - } + if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) + __scatterwalk_flush_dcache_pages(sg_page(walk->sg), + walk->offset, nbytes); scatterwalk_advance(walk, nbytes); } diff --git a/include/cxl/event.h b/include/cxl/event.h index 6fd90f9cc2034..4d7d1036ea9cb 100644 --- a/include/cxl/event.h +++ b/include/cxl/event.h @@ -320,4 +320,14 @@ static inline int cxl_cper_prot_err_kfifo_get(struct cxl_cper_prot_err_work_data } #endif +#ifdef CONFIG_ACPI_APEI_PCIEAER +int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err); +#else +static inline int +cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err) +{ + return -EOPNOTSUPP; +} +#endif + #endif /* _LINUX_CXL_EVENT_H */ diff --git a/include/drm/bridge/dw_hdmi_qp.h b/include/drm/bridge/dw_hdmi_qp.h index e9be6d507ad9c..8955450663e53 100644 --- a/include/drm/bridge/dw_hdmi_qp.h +++ b/include/drm/bridge/dw_hdmi_qp.h @@ -28,5 +28,6 @@ struct dw_hdmi_qp_plat_data { struct dw_hdmi_qp *dw_hdmi_qp_bind(struct platform_device *pdev, struct drm_encoder *encoder, const struct dw_hdmi_qp_plat_data *plat_data); +void dw_hdmi_qp_suspend(struct device *dev, struct dw_hdmi_qp *hdmi); void dw_hdmi_qp_resume(struct device *dev, struct dw_hdmi_qp *hdmi); #endif /* __DW_HDMI_QP__ */ diff --git a/include/drm/drm_atomic_helper.h b/include/drm/drm_atomic_helper.h index 53382fe93537b..e154ee4f0696c 100644 --- a/include/drm/drm_atomic_helper.h +++ b/include/drm/drm_atomic_helper.h @@ -60,6 +60,12 @@ int drm_atomic_helper_check_plane_state(struct drm_plane_state *plane_state, int drm_atomic_helper_check_planes(struct drm_device *dev, struct drm_atomic_state *state); int drm_atomic_helper_check_crtc_primary_plane(struct drm_crtc_state *crtc_state); +void drm_atomic_helper_commit_encoder_bridge_disable(struct drm_device *dev, + struct drm_atomic_state *state); +void drm_atomic_helper_commit_crtc_disable(struct drm_device *dev, + struct drm_atomic_state *state); +void drm_atomic_helper_commit_encoder_bridge_post_disable(struct drm_device *dev, + struct drm_atomic_state *state); int drm_atomic_helper_check(struct drm_device *dev, struct drm_atomic_state *state); void drm_atomic_helper_commit_tail(struct drm_atomic_state *state); @@ -89,8 +95,24 @@ drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev, void drm_atomic_helper_calc_timestamping_constants(struct drm_atomic_state *state); +void drm_atomic_helper_commit_crtc_set_mode(struct drm_device *dev, + struct drm_atomic_state *state); + void drm_atomic_helper_commit_modeset_disables(struct drm_device *dev, struct drm_atomic_state *state); + +void drm_atomic_helper_commit_writebacks(struct drm_device *dev, + struct drm_atomic_state *state); + +void drm_atomic_helper_commit_encoder_bridge_pre_enable(struct drm_device *dev, + struct drm_atomic_state *state); + +void drm_atomic_helper_commit_crtc_enable(struct drm_device *dev, + struct drm_atomic_state *state); + +void drm_atomic_helper_commit_encoder_bridge_enable(struct drm_device *dev, + struct drm_atomic_state *state); + void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev, struct drm_atomic_state *old_state); diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h index 76e05930f50e0..57d9a3c062124 100644 --- a/include/drm/drm_bridge.h +++ b/include/drm/drm_bridge.h @@ -176,33 +176,17 @@ struct drm_bridge_funcs { /** * @disable: * - * The @disable callback should disable the bridge. + * This callback should disable the bridge. It is called right before + * the preceding element in the display pipe is disabled. If the + * preceding element is a bridge this means it's called before that + * bridge's @disable vfunc. If the preceding element is a &drm_encoder + * it's called right before the &drm_encoder_helper_funcs.disable, + * &drm_encoder_helper_funcs.prepare or &drm_encoder_helper_funcs.dpms + * hook. * * The bridge can assume that the display pipe (i.e. clocks and timing * signals) feeding it is still running when this callback is called. * - * - * If the preceding element is a &drm_bridge, then this is called before - * that bridge is disabled via one of: - * - * - &drm_bridge_funcs.disable - * - &drm_bridge_funcs.atomic_disable - * - * If the preceding element of the bridge is a display controller, then - * this callback is called before the encoder is disabled via one of: - * - * - &drm_encoder_helper_funcs.atomic_disable - * - &drm_encoder_helper_funcs.prepare - * - &drm_encoder_helper_funcs.disable - * - &drm_encoder_helper_funcs.dpms - * - * and the CRTC is disabled via one of: - * - * - &drm_crtc_helper_funcs.prepare - * - &drm_crtc_helper_funcs.atomic_disable - * - &drm_crtc_helper_funcs.disable - * - &drm_crtc_helper_funcs.dpms. - * * The @disable callback is optional. * * NOTE: @@ -215,34 +199,17 @@ struct drm_bridge_funcs { /** * @post_disable: * - * The bridge must assume that the display pipe (i.e. clocks and timing - * signals) feeding this bridge is no longer running when the - * @post_disable is called. + * This callback should disable the bridge. It is called right after the + * preceding element in the display pipe is disabled. If the preceding + * element is a bridge this means it's called after that bridge's + * @post_disable function. If the preceding element is a &drm_encoder + * it's called right after the encoder's + * &drm_encoder_helper_funcs.disable, &drm_encoder_helper_funcs.prepare + * or &drm_encoder_helper_funcs.dpms hook. * - * This callback should perform all the actions required by the hardware - * after it has stopped receiving signals from the preceding element. - * - * If the preceding element is a &drm_bridge, then this is called after - * that bridge is post-disabled (unless marked otherwise by the - * @pre_enable_prev_first flag) via one of: - * - * - &drm_bridge_funcs.post_disable - * - &drm_bridge_funcs.atomic_post_disable - * - * If the preceding element of the bridge is a display controller, then - * this callback is called after the encoder is disabled via one of: - * - * - &drm_encoder_helper_funcs.atomic_disable - * - &drm_encoder_helper_funcs.prepare - * - &drm_encoder_helper_funcs.disable - * - &drm_encoder_helper_funcs.dpms - * - * and the CRTC is disabled via one of: - * - * - &drm_crtc_helper_funcs.prepare - * - &drm_crtc_helper_funcs.atomic_disable - * - &drm_crtc_helper_funcs.disable - * - &drm_crtc_helper_funcs.dpms + * The bridge must assume that the display pipe (i.e. clocks and timing + * signals) feeding it is no longer running when this callback is + * called. * * The @post_disable callback is optional. * @@ -285,30 +252,18 @@ struct drm_bridge_funcs { /** * @pre_enable: * - * The display pipe (i.e. clocks and timing signals) feeding this bridge - * will not yet be running when the @pre_enable is called. - * - * This callback should perform all the necessary actions to prepare the - * bridge to accept signals from the preceding element. - * - * If the preceding element is a &drm_bridge, then this is called before - * that bridge is pre-enabled (unless marked otherwise by - * @pre_enable_prev_first flag) via one of: - * - * - &drm_bridge_funcs.pre_enable - * - &drm_bridge_funcs.atomic_pre_enable - * - * If the preceding element of the bridge is a display controller, then - * this callback is called before the CRTC is enabled via one of: - * - * - &drm_crtc_helper_funcs.atomic_enable - * - &drm_crtc_helper_funcs.commit - * - * and the encoder is enabled via one of: + * This callback should enable the bridge. It is called right before + * the preceding element in the display pipe is enabled. If the + * preceding element is a bridge this means it's called before that + * bridge's @pre_enable function. If the preceding element is a + * &drm_encoder it's called right before the encoder's + * &drm_encoder_helper_funcs.enable, &drm_encoder_helper_funcs.commit or + * &drm_encoder_helper_funcs.dpms hook. * - * - &drm_encoder_helper_funcs.atomic_enable - * - &drm_encoder_helper_funcs.enable - * - &drm_encoder_helper_funcs.commit + * The display pipe (i.e. clocks and timing signals) feeding this bridge + * will not yet be running when this callback is called. The bridge must + * not enable the display link feeding the next bridge in the chain (if + * there is one) when this callback is called. * * The @pre_enable callback is optional. * @@ -322,31 +277,19 @@ struct drm_bridge_funcs { /** * @enable: * - * The @enable callback should enable the bridge. + * This callback should enable the bridge. It is called right after + * the preceding element in the display pipe is enabled. If the + * preceding element is a bridge this means it's called after that + * bridge's @enable function. If the preceding element is a + * &drm_encoder it's called right after the encoder's + * &drm_encoder_helper_funcs.enable, &drm_encoder_helper_funcs.commit or + * &drm_encoder_helper_funcs.dpms hook. * * The bridge can assume that the display pipe (i.e. clocks and timing * signals) feeding it is running when this callback is called. This * callback must enable the display link feeding the next bridge in the * chain if there is one. * - * If the preceding element is a &drm_bridge, then this is called after - * that bridge is enabled via one of: - * - * - &drm_bridge_funcs.enable - * - &drm_bridge_funcs.atomic_enable - * - * If the preceding element of the bridge is a display controller, then - * this callback is called after the CRTC is enabled via one of: - * - * - &drm_crtc_helper_funcs.atomic_enable - * - &drm_crtc_helper_funcs.commit - * - * and the encoder is enabled via one of: - * - * - &drm_encoder_helper_funcs.atomic_enable - * - &drm_encoder_helper_funcs.enable - * - drm_encoder_helper_funcs.commit - * * The @enable callback is optional. * * NOTE: @@ -359,30 +302,17 @@ struct drm_bridge_funcs { /** * @atomic_pre_enable: * - * The display pipe (i.e. clocks and timing signals) feeding this bridge - * will not yet be running when the @atomic_pre_enable is called. - * - * This callback should perform all the necessary actions to prepare the - * bridge to accept signals from the preceding element. - * - * If the preceding element is a &drm_bridge, then this is called before - * that bridge is pre-enabled (unless marked otherwise by - * @pre_enable_prev_first flag) via one of: - * - * - &drm_bridge_funcs.pre_enable - * - &drm_bridge_funcs.atomic_pre_enable + * This callback should enable the bridge. It is called right before + * the preceding element in the display pipe is enabled. If the + * preceding element is a bridge this means it's called before that + * bridge's @atomic_pre_enable or @pre_enable function. If the preceding + * element is a &drm_encoder it's called right before the encoder's + * &drm_encoder_helper_funcs.atomic_enable hook. * - * If the preceding element of the bridge is a display controller, then - * this callback is called before the CRTC is enabled via one of: - * - * - &drm_crtc_helper_funcs.atomic_enable - * - &drm_crtc_helper_funcs.commit - * - * and the encoder is enabled via one of: - * - * - &drm_encoder_helper_funcs.atomic_enable - * - &drm_encoder_helper_funcs.enable - * - &drm_encoder_helper_funcs.commit + * The display pipe (i.e. clocks and timing signals) feeding this bridge + * will not yet be running when this callback is called. The bridge must + * not enable the display link feeding the next bridge in the chain (if + * there is one) when this callback is called. * * The @atomic_pre_enable callback is optional. */ @@ -392,31 +322,18 @@ struct drm_bridge_funcs { /** * @atomic_enable: * - * The @atomic_enable callback should enable the bridge. + * This callback should enable the bridge. It is called right after + * the preceding element in the display pipe is enabled. If the + * preceding element is a bridge this means it's called after that + * bridge's @atomic_enable or @enable function. If the preceding element + * is a &drm_encoder it's called right after the encoder's + * &drm_encoder_helper_funcs.atomic_enable hook. * * The bridge can assume that the display pipe (i.e. clocks and timing * signals) feeding it is running when this callback is called. This * callback must enable the display link feeding the next bridge in the * chain if there is one. * - * If the preceding element is a &drm_bridge, then this is called after - * that bridge is enabled via one of: - * - * - &drm_bridge_funcs.enable - * - &drm_bridge_funcs.atomic_enable - * - * If the preceding element of the bridge is a display controller, then - * this callback is called after the CRTC is enabled via one of: - * - * - &drm_crtc_helper_funcs.atomic_enable - * - &drm_crtc_helper_funcs.commit - * - * and the encoder is enabled via one of: - * - * - &drm_encoder_helper_funcs.atomic_enable - * - &drm_encoder_helper_funcs.enable - * - drm_encoder_helper_funcs.commit - * * The @atomic_enable callback is optional. */ void (*atomic_enable)(struct drm_bridge *bridge, @@ -424,32 +341,16 @@ struct drm_bridge_funcs { /** * @atomic_disable: * - * The @atomic_disable callback should disable the bridge. + * This callback should disable the bridge. It is called right before + * the preceding element in the display pipe is disabled. If the + * preceding element is a bridge this means it's called before that + * bridge's @atomic_disable or @disable vfunc. If the preceding element + * is a &drm_encoder it's called right before the + * &drm_encoder_helper_funcs.atomic_disable hook. * * The bridge can assume that the display pipe (i.e. clocks and timing * signals) feeding it is still running when this callback is called. * - * If the preceding element is a &drm_bridge, then this is called before - * that bridge is disabled via one of: - * - * - &drm_bridge_funcs.disable - * - &drm_bridge_funcs.atomic_disable - * - * If the preceding element of the bridge is a display controller, then - * this callback is called before the encoder is disabled via one of: - * - * - &drm_encoder_helper_funcs.atomic_disable - * - &drm_encoder_helper_funcs.prepare - * - &drm_encoder_helper_funcs.disable - * - &drm_encoder_helper_funcs.dpms - * - * and the CRTC is disabled via one of: - * - * - &drm_crtc_helper_funcs.prepare - * - &drm_crtc_helper_funcs.atomic_disable - * - &drm_crtc_helper_funcs.disable - * - &drm_crtc_helper_funcs.dpms. - * * The @atomic_disable callback is optional. */ void (*atomic_disable)(struct drm_bridge *bridge, @@ -458,34 +359,16 @@ struct drm_bridge_funcs { /** * @atomic_post_disable: * - * The bridge must assume that the display pipe (i.e. clocks and timing - * signals) feeding this bridge is no longer running when the - * @atomic_post_disable is called. - * - * This callback should perform all the actions required by the hardware - * after it has stopped receiving signals from the preceding element. + * This callback should disable the bridge. It is called right after the + * preceding element in the display pipe is disabled. If the preceding + * element is a bridge this means it's called after that bridge's + * @atomic_post_disable or @post_disable function. If the preceding + * element is a &drm_encoder it's called right after the encoder's + * &drm_encoder_helper_funcs.atomic_disable hook. * - * If the preceding element is a &drm_bridge, then this is called after - * that bridge is post-disabled (unless marked otherwise by the - * @pre_enable_prev_first flag) via one of: - * - * - &drm_bridge_funcs.post_disable - * - &drm_bridge_funcs.atomic_post_disable - * - * If the preceding element of the bridge is a display controller, then - * this callback is called after the encoder is disabled via one of: - * - * - &drm_encoder_helper_funcs.atomic_disable - * - &drm_encoder_helper_funcs.prepare - * - &drm_encoder_helper_funcs.disable - * - &drm_encoder_helper_funcs.dpms - * - * and the CRTC is disabled via one of: - * - * - &drm_crtc_helper_funcs.prepare - * - &drm_crtc_helper_funcs.atomic_disable - * - &drm_crtc_helper_funcs.disable - * - &drm_crtc_helper_funcs.dpms + * The bridge must assume that the display pipe (i.e. clocks and timing + * signals) feeding it is no longer running when this callback is + * called. * * The @atomic_post_disable callback is optional. */ diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index 04afd7c21a82e..c2e05a2812522 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -44,7 +45,11 @@ struct drm_buddy_block { * a list, if so desired. As soon as the block is freed with * drm_buddy_free* ownership is given back to the mm. */ - struct list_head link; + union { + struct rb_node rb; + struct list_head link; + }; + struct list_head tmp_link; }; @@ -59,7 +64,7 @@ struct drm_buddy_block { */ struct drm_buddy { /* Maintain a free list for each order. */ - struct list_head *free_list; + struct rb_root **free_trees; /* * Maintain explicit binary tree(s) to track the allocation of the @@ -85,7 +90,7 @@ struct drm_buddy { }; static inline u64 -drm_buddy_block_offset(struct drm_buddy_block *block) +drm_buddy_block_offset(const struct drm_buddy_block *block) { return block->header & DRM_BUDDY_HEADER_OFFSET; } diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h index 3d1aecfec9b2a..04f7a7f1f1082 100644 --- a/include/drm/drm_edid.h +++ b/include/drm/drm_edid.h @@ -340,6 +340,12 @@ struct drm_edid_ident { const char *name; }; +#define DRM_EDID_IDENT_INIT(_vend_chr_0, _vend_chr_1, _vend_chr_2, _product_id, _name) \ +{ \ + .panel_id = drm_edid_encode_panel_id(_vend_chr_0, _vend_chr_1, _vend_chr_2, _product_id), \ + .name = _name, \ +} + #define EDID_PRODUCT_ID(e) ((e)->prod_code[0] | ((e)->prod_code[1] << 8)) /* Short Audio Descriptor */ diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 92f5db84b9c22..6802896e30c7c 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -301,4 +301,15 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev, .gem_prime_import = drm_gem_shmem_prime_import_no_map, \ .dumb_create = drm_gem_shmem_dumb_create +/* + * Kunit helpers + */ + +#if IS_ENABLED(CONFIG_KUNIT) +int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map); +void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map); +int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv); +int drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); +#endif + #endif /* __DRM_GEM_SHMEM_HELPER_H__ */ diff --git a/include/drm/drm_of.h b/include/drm/drm_of.h index 7f0256dae3f13..f3e55ea2174c0 100644 --- a/include/drm/drm_of.h +++ b/include/drm/drm_of.h @@ -5,6 +5,7 @@ #include #include #if IS_ENABLED(CONFIG_OF) && IS_ENABLED(CONFIG_DRM_PANEL_BRIDGE) +#include #include #endif @@ -173,6 +174,8 @@ static inline int drm_of_panel_bridge_remove(const struct device_node *np, bridge = of_drm_find_bridge(remote); drm_panel_bridge_remove(bridge); + of_node_put(remote); + return 0; #else return -EINVAL; diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h index f6e7e234c0892..eb29e5309f0ab 100644 --- a/include/drm/drm_pagemap.h +++ b/include/drm/drm_pagemap.h @@ -8,6 +8,7 @@ #define NR_PAGES(order) (1U << (order)) +struct dma_fence; struct drm_pagemap; struct drm_pagemap_zdd; struct device; @@ -174,6 +175,8 @@ struct drm_pagemap_devmem_ops { * @pages: Pointer to array of device memory pages (destination) * @pagemap_addr: Pointer to array of DMA information (source) * @npages: Number of pages to copy + * @pre_migrate_fence: dma-fence to wait for before migration start. + * May be NULL. * * Copy pages to device memory. If the order of a @pagemap_addr entry * is greater than 0, the entry is populated but subsequent entries @@ -183,13 +186,16 @@ struct drm_pagemap_devmem_ops { */ int (*copy_to_devmem)(struct page **pages, struct drm_pagemap_addr *pagemap_addr, - unsigned long npages); + unsigned long npages, + struct dma_fence *pre_migrate_fence); /** * @copy_to_ram: Copy to system RAM (required for migration) * @pages: Pointer to array of device memory pages (source) * @pagemap_addr: Pointer to array of DMA information (destination) * @npages: Number of pages to copy + * @pre_migrate_fence: dma-fence to wait for before migration start. + * May be NULL. * * Copy pages to system RAM. If the order of a @pagemap_addr entry * is greater than 0, the entry is populated but subsequent entries @@ -199,9 +205,23 @@ struct drm_pagemap_devmem_ops { */ int (*copy_to_ram)(struct page **pages, struct drm_pagemap_addr *pagemap_addr, - unsigned long npages); + unsigned long npages, + struct dma_fence *pre_migrate_fence); }; +#if IS_ENABLED(CONFIG_ZONE_DEVICE) + +struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page); + +#else + +static inline struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page) +{ + return NULL; +} + +#endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */ + /** * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation * @@ -212,6 +232,8 @@ struct drm_pagemap_devmem_ops { * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to. * @size: Size of device memory allocation * @timeslice_expiration: Timeslice expiration in jiffies + * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. + * (May be NULL). */ struct drm_pagemap_devmem { struct device *dev; @@ -221,8 +243,11 @@ struct drm_pagemap_devmem { struct drm_pagemap *dpagemap; size_t size; u64 timeslice_expiration; + struct dma_fence *pre_migrate_fence; }; +#if IS_ENABLED(CONFIG_ZONE_DEVICE) + int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, struct mm_struct *mm, unsigned long start, unsigned long end, @@ -233,16 +258,17 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation); const struct dev_pagemap_ops *drm_pagemap_pagemap_ops_get(void); -struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page); - void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, struct device *dev, struct mm_struct *mm, const struct drm_pagemap_devmem_ops *ops, - struct drm_pagemap *dpagemap, size_t size); + struct drm_pagemap *dpagemap, size_t size, + struct dma_fence *pre_migrate_fence); int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, unsigned long start, unsigned long end, struct mm_struct *mm, unsigned long timeslice_ms); +#endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */ + #endif diff --git a/include/drm/intel/intel_lb_mei_interface.h b/include/drm/intel/intel_lb_mei_interface.h index d65be2cba2ab9..0850738a30fc7 100644 --- a/include/drm/intel/intel_lb_mei_interface.h +++ b/include/drm/intel/intel_lb_mei_interface.h @@ -53,7 +53,8 @@ enum intel_lb_status { */ struct intel_lb_component_ops { /** - * push_payload - Sends a payload to the authentication firmware + * @push_payload: Sends a payload to the authentication firmware + * * @dev: Device struct corresponding to the mei device * @type: Payload type (see &enum intel_lb_type) * @flags: Payload flags bitmap (e.g. %INTEL_LB_FLAGS_IS_PERSISTENT) diff --git a/include/dt-bindings/arm/qcom,ids.h b/include/dt-bindings/arm/qcom,ids.h index cb8ce53146f0c..8776844e0eeb3 100644 --- a/include/dt-bindings/arm/qcom,ids.h +++ b/include/dt-bindings/arm/qcom,ids.h @@ -240,6 +240,7 @@ #define QCOM_ID_SC7280 487 #define QCOM_ID_SC7180P 495 #define QCOM_ID_QCM6490 497 +#define QCOM_ID_QCS6490 498 #define QCOM_ID_SM7325P 499 #define QCOM_ID_IPQ5000 503 #define QCOM_ID_IPQ0509 504 @@ -286,6 +287,7 @@ #define QCOM_ID_IPQ5424 651 #define QCOM_ID_QCM6690 657 #define QCOM_ID_QCS6690 658 +#define QCOM_ID_SM8850 660 #define QCOM_ID_IPQ5404 671 #define QCOM_ID_QCS9100 667 #define QCOM_ID_QCS8300 674 diff --git a/include/dt-bindings/camera/msm-camera.h b/include/dt-bindings/camera/msm-camera.h new file mode 100644 index 0000000000000..97228c26e7040 --- /dev/null +++ b/include/dt-bindings/camera/msm-camera.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __MSM_CAMERA_H +#define __MSM_CAMERA_H + +/* CPAS path data types */ +#define CAM_CPAS_PATH_DATA_IFE_START_OFFSET 0 +#define CAM_CPAS_PATH_DATA_IFE_LINEAR (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 0) +#define CAM_CPAS_PATH_DATA_IFE_VID (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_IFE_DISP (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 2) +#define CAM_CPAS_PATH_DATA_IFE_STATS (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 3) +#define CAM_CPAS_PATH_DATA_IFE_RDI0 (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 4) +#define CAM_CPAS_PATH_DATA_IFE_RDI1 (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 5) +#define CAM_CPAS_PATH_DATA_IFE_RDI2 (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 6) +#define CAM_CPAS_PATH_DATA_IFE_RDI3 (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 7) +#define CAM_CPAS_PATH_DATA_IFE_PDAF (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 8) +#define CAM_CPAS_PATH_DATA_IFE_PIXEL_RAW (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 9) +#define CAM_CPAS_PATH_DATA_IFE_MAX_OFFSET (CAM_CPAS_PATH_DATA_IFE_START_OFFSET + 31) + +#define CAM_CPAS_PATH_DATA_IPE_START_OFFSET 32 +#define CAM_CPAS_PATH_DATA_IPE_RD_IN (CAM_CPAS_PATH_DATA_IPE_START_OFFSET + 0) +#define CAM_CPAS_PATH_DATA_IPE_RD_REF (CAM_CPAS_PATH_DATA_IPE_START_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_IPE_WR_VID (CAM_CPAS_PATH_DATA_IPE_START_OFFSET + 2) +#define CAM_CPAS_PATH_DATA_IPE_WR_DISP (CAM_CPAS_PATH_DATA_IPE_START_OFFSET + 3) +#define CAM_CPAS_PATH_DATA_IPE_WR_REF (CAM_CPAS_PATH_DATA_IPE_START_OFFSET + 4) +#define CAM_CPAS_PATH_DATA_IPE_WR_APP (CAM_CPAS_PATH_DATA_IPE_START_OFFSET + 5) +#define CAM_CPAS_PATH_DATA_IPE_MAX_OFFSET (CAM_CPAS_PATH_DATA_IPE_START_OFFSET + 31) + +#define CAM_CPAS_PATH_DATA_OPE_START_OFFSET 64 +#define CAM_CPAS_PATH_DATA_OPE_RD_IN (CAM_CPAS_PATH_DATA_OPE_START_OFFSET + 0) +#define CAM_CPAS_PATH_DATA_OPE_RD_REF (CAM_CPAS_PATH_DATA_OPE_START_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_OPE_WR_VID (CAM_CPAS_PATH_DATA_OPE_START_OFFSET + 2) +#define CAM_CPAS_PATH_DATA_OPE_WR_DISP (CAM_CPAS_PATH_DATA_OPE_START_OFFSET + 3) +#define CAM_CPAS_PATH_DATA_OPE_WR_REF (CAM_CPAS_PATH_DATA_OPE_START_OFFSET + 4) +#define CAM_CPAS_PATH_DATA_OPE_MAX_OFFSET (CAM_CPAS_PATH_DATA_OPE_START_OFFSET + 31) + +#define CAM_CPAS_PATH_DATA_SFE_START_OFFSET 96 +#define CAM_CPAS_PATH_DATA_SFE_NRDI (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 0) +#define CAM_CPAS_PATH_DATA_SFE_RDI0 (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_SFE_RDI1 (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 2) +#define CAM_CPAS_PATH_DATA_SFE_RDI2 (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 3) +#define CAM_CPAS_PATH_DATA_SFE_RDI3 (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 4) +#define CAM_CPAS_PATH_DATA_SFE_RDI4 (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 5) +#define CAM_CPAS_PATH_DATA_SFE_STATS (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 6) +#define CAM_CPAS_PATH_DATA_SFE_MAX_OFFSET (CAM_CPAS_PATH_DATA_SFE_START_OFFSET + 31) + +#define CAM_CPAS_PATH_DATA_CRE_START_OFFSET (CAM_CPAS_PATH_DATA_SFE_MAX_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_CRE_RD_IN (CAM_CPAS_PATH_DATA_CRE_START_OFFSET + 0) +#define CAM_CPAS_PATH_DATA_CRE_WR_OUT (CAM_CPAS_PATH_DATA_CRE_START_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_CRE_MAX_OFFSET (CAM_CPAS_PATH_DATA_CRE_START_OFFSET + 31) + +#define CAM_CPAS_PATH_DATA_OFE_START_OFFSET (CAM_CPAS_PATH_DATA_CRE_MAX_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_OFE_RD_EXT (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 0) +#define CAM_CPAS_PATH_DATA_OFE_RD_INT_PDI (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_OFE_RD_INT_HDR (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 2) +#define CAM_CPAS_PATH_DATA_OFE_WR_VID (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 3) +#define CAM_CPAS_PATH_DATA_OFE_WR_DISP (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 4) +#define CAM_CPAS_PATH_DATA_OFE_WR_IR (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 5) +#define CAM_CPAS_PATH_DATA_OFE_WR_HDR_LTM (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 6) +#define CAM_CPAS_PATH_DATA_OFE_WR_DC4 (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 7) +#define CAM_CPAS_PATH_DATA_OFE_WR_AI (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 8) +#define CAM_CPAS_PATH_DATA_OFE_WR_PDI (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 9) +#define CAM_CPAS_PATH_DATA_OFE_WR_IDEALRAW (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 10) +#define CAM_CPAS_PATH_DATA_OFE_WR_STATS (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 11) +#define CAM_CPAS_PATH_DATA_OFE_MAX_OFFSET (CAM_CPAS_PATH_DATA_OFE_START_OFFSET + 31) + +#define CAM_CPAS_PATH_DATA_CONSO_OFFSET 256 +#define CAM_CPAS_PATH_DATA_ALL (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 0) + +/* IFE consolidated paths */ +#define CAM_CPAS_PATH_DATA_IFE_LINEAR_PDAF (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 1) +#define CAM_CPAS_PATH_DATA_IFE_UBWC_STATS (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 2) +#define CAM_CPAS_PATH_DATA_IFE_PIXEL_ALL (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 3) +#define CAM_CPAS_PATH_DATA_IFE_RDI_PIXEL_RAW (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 4) +#define CAM_CPAS_PATH_DATA_IFE_RDI_ALL (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 5) +#define CAM_CPAS_PATH_DATA_IFE_UBWC (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 6) +#define CAM_CPAS_PATH_DATA_IFE_LINEAR_STATS (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 7) +#define CAM_CPAS_PATH_DATA_IFE_UBWC_LINEAR (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 8) +#define CAM_CPAS_PATH_DATA_IFE_PDAF_LINEAR (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 9) + +/* IPE Consolidated paths */ +#define CAM_CPAS_PATH_DATA_IPE_WR_VID_DISP (CAM_CPAS_PATH_DATA_CONSO_OFFSET + 1) + +/* CPAS transaction types */ +#define CAM_CPAS_TRANSACTION_READ 0 +#define CAM_CPAS_TRANSACTION_WRITE 1 + +/* CPAS traffic merge types */ +#define CAM_CPAS_TRAFFIC_MERGE_SUM 0 +#define CAM_CPAS_TRAFFIC_MERGE_SUM_INTERLEAVE 1 + +/* Feature bit type */ +#define CAM_CPAS_FEATURE_TYPE_DISABLE 0 +#define CAM_CPAS_FEATURE_TYPE_ENABLE 1 +#define CAM_CPAS_FEATURE_TYPE_VALUE 2 + +/* Feature support bit positions in feature fuse register*/ +#define CAM_CPAS_QCFA_BINNING_ENABLE 0 +#define CAM_CPAS_SECURE_CAMERA_ENABLE 1 +#define CAM_CPAS_MF_HDR_ENABLE 2 +#define CAM_CPAS_MP_LIMIT_FUSE 3 +#define CAM_CPAS_ISP_FUSE 4 +#define CAM_CPAS_ISP_PIX_FUSE 5 +#define CAM_CPAS_ISP_LITE_FUSE 6 +#define CAM_CPAS_CSIPHY_FUSE 7 +#define CAM_CPAS_IPE_VID_OUT_8BPP_LIMIT_ENABLE 8 +#define CAM_CPAS_SFE_FUSE 9 +#define CAM_CPAS_CUSTOM_FUSE 10 +#define CAM_CPAS_CAM_FUSE 11 +#define CAM_CPAS_SHDR_FUSE 12 +#define CAM_CPAS_RT_OT_FUSE 13 +#define CAM_CPAS_FUSE_FEATURE_MAX 14 + +/* Flash type*/ +#define CAM_FLASH_TYPE_PMIC 0 +#define CAM_FLASH_TYPE_I2C 1 +#define CAM_FLASH_TYPE_GPIO 2 + +/* CCI master */ +#define CCI_MASTER_0 0 +#define CCI_MASTER_1 1 +#define CCI_MASTER_MAX 2 + +/* AON Camera IDs*/ +#define AON_CAM1 0 +#define AON_CAM2 1 +#define MAX_AON_CAM 2 +#define NOT_AON_CAM 255 + +/* Camera DRV enable masks */ +#define CAM_DDR_DRV 0x1 +#define CAM_CLK_DRV 0x2 + +/* Port index for BW voting */ +#define CAM_CPAS_PORT_HLOS_DRV 0 +#define CAM_CPAS_PORT_DRV_0 1 +#define CAM_CPAS_PORT_DRV_1 2 +#define CAM_CPAS_PORT_DRV_2 3 +#define CAM_CPAS_PORT_DRV_DYN 32 + +/* Domain ID types */ +#define CAM_CPAS_NON_SECURE_DOMAIN 0 +#define CAM_CPAS_SECURE_DOMAIN 1 + +/* Debug bypass driver */ +#define CAM_BYPASS_RGLTR 0x1 +#define CAM_BYPASS_RGLTR_MODE 0x2 +#define CAM_BYPASS_CLKS 0x4 +#define CAM_BYPASS_CESTA 0x8 +#define CAM_BYPASS_ICC 0x10 + +#endif diff --git a/include/dt-bindings/clock/qcom,kaanapali-gcc.h b/include/dt-bindings/clock/qcom,kaanapali-gcc.h new file mode 100644 index 0000000000000..890e48709f09f --- /dev/null +++ b/include/dt-bindings/clock/qcom,kaanapali-gcc.h @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_GCC_KAANAPALI_H +#define _DT_BINDINGS_CLK_QCOM_GCC_KAANAPALI_H + +/* GCC clocks */ +#define GCC_AGGRE_NOC_PCIE_AXI_CLK 0 +#define GCC_AGGRE_UFS_PHY_AXI_CLK 1 +#define GCC_AGGRE_USB3_PRIM_AXI_CLK 2 +#define GCC_BOOT_ROM_AHB_CLK 3 +#define GCC_CAM_BIST_MCLK_AHB_CLK 4 +#define GCC_CAMERA_AHB_CLK 5 +#define GCC_CAMERA_HF_AXI_CLK 6 +#define GCC_CAMERA_SF_AXI_CLK 7 +#define GCC_CAMERA_XO_CLK 8 +#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK 9 +#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK 10 +#define GCC_CNOC_PCIE_SF_AXI_CLK 11 +#define GCC_DDRSS_PCIE_SF_QTB_CLK 12 +#define GCC_QMIP_CAMERA_CMD_AHB_CLK 13 +#define GCC_DISP_HF_AXI_CLK 14 +#define GCC_DISP_SF_AXI_CLK 15 +#define GCC_EVA_AHB_CLK 16 +#define GCC_EVA_AXI0_CLK 17 +#define GCC_EVA_AXI0C_CLK 18 +#define GCC_EVA_XO_CLK 19 +#define GCC_GP1_CLK 20 +#define GCC_GP1_CLK_SRC 21 +#define GCC_GP2_CLK 22 +#define GCC_GP2_CLK_SRC 23 +#define GCC_GP3_CLK 24 +#define GCC_GP3_CLK_SRC 25 +#define GCC_GPLL0 26 +#define GCC_GPLL0_OUT_EVEN 27 +#define GCC_GPLL1 28 +#define GCC_GPLL4 29 +#define GCC_GPLL7 30 +#define GCC_GPLL9 31 +#define GCC_GPU_CFG_AHB_CLK 32 +#define GCC_GPU_GEMNOC_GFX_CLK 33 +#define GCC_GPU_GPLL0_CLK_SRC 34 +#define GCC_GPU_GPLL0_DIV_CLK_SRC 35 +#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK 36 +#define GCC_QMIP_GPU_AHB_CLK 37 +#define GCC_PCIE_0_AUX_CLK 38 +#define GCC_PCIE_0_AUX_CLK_SRC 39 +#define GCC_PCIE_0_CFG_AHB_CLK 40 +#define GCC_PCIE_0_MSTR_AXI_CLK 41 +#define GCC_PCIE_0_PHY_AUX_CLK 42 +#define GCC_PCIE_0_PHY_AUX_CLK_SRC 43 +#define GCC_PCIE_0_PHY_RCHNG_CLK 44 +#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC 45 +#define GCC_PCIE_0_PIPE_CLK 46 +#define GCC_PCIE_0_PIPE_CLK_SRC 47 +#define GCC_PCIE_0_SLV_AXI_CLK 48 +#define GCC_PCIE_0_SLV_Q2A_AXI_CLK 49 +#define GCC_PCIE_RSCC_CFG_AHB_CLK 50 +#define GCC_PCIE_RSCC_XO_CLK 51 +#define GCC_PDM2_CLK 52 +#define GCC_PDM2_CLK_SRC 53 +#define GCC_PDM_AHB_CLK 54 +#define GCC_PDM_XO4_CLK 55 +#define GCC_QUPV3_I2C_CORE_CLK 56 +#define GCC_QUPV3_I2C_S0_CLK 57 +#define GCC_QUPV3_I2C_S0_CLK_SRC 58 +#define GCC_QUPV3_I2C_S1_CLK 59 +#define GCC_QUPV3_I2C_S1_CLK_SRC 60 +#define GCC_QUPV3_I2C_S2_CLK 61 +#define GCC_QUPV3_I2C_S2_CLK_SRC 62 +#define GCC_QUPV3_I2C_S3_CLK 63 +#define GCC_QUPV3_I2C_S3_CLK_SRC 64 +#define GCC_QUPV3_I2C_S4_CLK 65 +#define GCC_QUPV3_I2C_S4_CLK_SRC 66 +#define GCC_QUPV3_I2C_S_AHB_CLK 67 +#define GCC_QUPV3_WRAP1_CORE_2X_CLK 68 +#define GCC_QUPV3_WRAP1_CORE_CLK 69 +#define GCC_QUPV3_WRAP1_QSPI_REF_CLK 70 +#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC 71 +#define GCC_QUPV3_WRAP1_S0_CLK 72 +#define GCC_QUPV3_WRAP1_S0_CLK_SRC 73 +#define GCC_QUPV3_WRAP1_S1_CLK 74 +#define GCC_QUPV3_WRAP1_S1_CLK_SRC 75 +#define GCC_QUPV3_WRAP1_S2_CLK 76 +#define GCC_QUPV3_WRAP1_S2_CLK_SRC 77 +#define GCC_QUPV3_WRAP1_S3_CLK 78 +#define GCC_QUPV3_WRAP1_S3_CLK_SRC 79 +#define GCC_QUPV3_WRAP1_S4_CLK 80 +#define GCC_QUPV3_WRAP1_S4_CLK_SRC 81 +#define GCC_QUPV3_WRAP1_S5_CLK 82 +#define GCC_QUPV3_WRAP1_S5_CLK_SRC 83 +#define GCC_QUPV3_WRAP1_S6_CLK 84 +#define GCC_QUPV3_WRAP1_S6_CLK_SRC 85 +#define GCC_QUPV3_WRAP1_S7_CLK 86 +#define GCC_QUPV3_WRAP1_S7_CLK_SRC 87 +#define GCC_QUPV3_WRAP2_CORE_2X_CLK 88 +#define GCC_QUPV3_WRAP2_CORE_CLK 89 +#define GCC_QUPV3_WRAP2_S0_CLK 90 +#define GCC_QUPV3_WRAP2_S0_CLK_SRC 91 +#define GCC_QUPV3_WRAP2_S1_CLK 92 +#define GCC_QUPV3_WRAP2_S1_CLK_SRC 93 +#define GCC_QUPV3_WRAP2_S2_CLK 94 +#define GCC_QUPV3_WRAP2_S2_CLK_SRC 95 +#define GCC_QUPV3_WRAP2_S3_CLK 96 +#define GCC_QUPV3_WRAP2_S3_CLK_SRC 97 +#define GCC_QUPV3_WRAP2_S4_CLK 98 +#define GCC_QUPV3_WRAP2_S4_CLK_SRC 99 +#define GCC_QUPV3_WRAP3_CORE_2X_CLK 100 +#define GCC_QUPV3_WRAP3_CORE_CLK 101 +#define GCC_QUPV3_WRAP3_IBI_CTRL_0_CLK_SRC 102 +#define GCC_QUPV3_WRAP3_IBI_CTRL_1_CLK 103 +#define GCC_QUPV3_WRAP3_IBI_CTRL_2_CLK 104 +#define GCC_QUPV3_WRAP3_S0_CLK 105 +#define GCC_QUPV3_WRAP3_S0_CLK_SRC 106 +#define GCC_QUPV3_WRAP3_S1_CLK 107 +#define GCC_QUPV3_WRAP3_S1_CLK_SRC 108 +#define GCC_QUPV3_WRAP3_S2_CLK 109 +#define GCC_QUPV3_WRAP3_S2_CLK_SRC 110 +#define GCC_QUPV3_WRAP3_S3_CLK 111 +#define GCC_QUPV3_WRAP3_S3_CLK_SRC 112 +#define GCC_QUPV3_WRAP3_S4_CLK 113 +#define GCC_QUPV3_WRAP3_S4_CLK_SRC 114 +#define GCC_QUPV3_WRAP3_S5_CLK 115 +#define GCC_QUPV3_WRAP3_S5_CLK_SRC 116 +#define GCC_QUPV3_WRAP4_CORE_2X_CLK 117 +#define GCC_QUPV3_WRAP4_CORE_CLK 118 +#define GCC_QUPV3_WRAP4_S0_CLK 119 +#define GCC_QUPV3_WRAP4_S0_CLK_SRC 120 +#define GCC_QUPV3_WRAP4_S1_CLK 121 +#define GCC_QUPV3_WRAP4_S1_CLK_SRC 122 +#define GCC_QUPV3_WRAP4_S2_CLK 123 +#define GCC_QUPV3_WRAP4_S2_CLK_SRC 124 +#define GCC_QUPV3_WRAP4_S3_CLK 125 +#define GCC_QUPV3_WRAP4_S3_CLK_SRC 126 +#define GCC_QUPV3_WRAP4_S4_CLK 127 +#define GCC_QUPV3_WRAP4_S4_CLK_SRC 128 +#define GCC_QUPV3_WRAP_1_M_AXI_CLK 129 +#define GCC_QUPV3_WRAP_1_S_AHB_CLK 130 +#define GCC_QUPV3_WRAP_2_M_AHB_CLK 131 +#define GCC_QUPV3_WRAP_2_S_AHB_CLK 132 +#define GCC_QUPV3_WRAP_3_IBI_1_AHB_CLK 133 +#define GCC_QUPV3_WRAP_3_IBI_2_AHB_CLK 134 +#define GCC_QUPV3_WRAP_3_M_AHB_CLK 135 +#define GCC_QUPV3_WRAP_3_S_AHB_CLK 136 +#define GCC_QUPV3_WRAP_4_M_AHB_CLK 137 +#define GCC_QUPV3_WRAP_4_S_AHB_CLK 138 +#define GCC_SDCC2_AHB_CLK 139 +#define GCC_SDCC2_APPS_CLK 140 +#define GCC_SDCC2_APPS_CLK_SRC 141 +#define GCC_SDCC4_AHB_CLK 142 +#define GCC_SDCC4_APPS_CLK 143 +#define GCC_SDCC4_APPS_CLK_SRC 144 +#define GCC_UFS_PHY_AHB_CLK 145 +#define GCC_UFS_PHY_AXI_CLK 146 +#define GCC_UFS_PHY_AXI_CLK_SRC 147 +#define GCC_UFS_PHY_ICE_CORE_CLK 148 +#define GCC_UFS_PHY_ICE_CORE_CLK_SRC 149 +#define GCC_UFS_PHY_PHY_AUX_CLK 150 +#define GCC_UFS_PHY_PHY_AUX_CLK_SRC 151 +#define GCC_UFS_PHY_RX_SYMBOL_0_CLK 152 +#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC 153 +#define GCC_UFS_PHY_RX_SYMBOL_1_CLK 154 +#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC 155 +#define GCC_UFS_PHY_TX_SYMBOL_0_CLK 156 +#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC 157 +#define GCC_UFS_PHY_UNIPRO_CORE_CLK 158 +#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC 159 +#define GCC_USB30_PRIM_MASTER_CLK 160 +#define GCC_USB30_PRIM_MASTER_CLK_SRC 161 +#define GCC_USB30_PRIM_MOCK_UTMI_CLK 162 +#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC 163 +#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC 164 +#define GCC_USB30_PRIM_SLEEP_CLK 165 +#define GCC_USB3_PRIM_PHY_AUX_CLK 166 +#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC 167 +#define GCC_USB3_PRIM_PHY_COM_AUX_CLK 168 +#define GCC_USB3_PRIM_PHY_PIPE_CLK 169 +#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC 170 +#define GCC_VIDEO_AHB_CLK 171 +#define GCC_VIDEO_AXI0_CLK 172 +#define GCC_VIDEO_AXI1_CLK 173 +#define GCC_VIDEO_XO_CLK 174 +#define GCC_QMIP_CAMERA_NRT_AHB_CLK 175 +#define GCC_QMIP_CAMERA_RT_AHB_CLK 176 +#define GCC_QMIP_DISP_DCP_SF_AHB_CLK 177 +#define GCC_QMIP_PCIE_AHB_CLK 178 +#define GCC_QMIP_VIDEO_CV_CPU_AHB_CLK 179 +#define GCC_QMIP_VIDEO_CVP_AHB_CLK 180 +#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK 181 +#define GCC_DISP_AHB_CLK 182 + +/* GCC power domains */ +#define GCC_PCIE_0_GDSC 0 +#define GCC_PCIE_0_PHY_GDSC 1 +#define GCC_UFS_MEM_PHY_GDSC 2 +#define GCC_UFS_PHY_GDSC 3 +#define GCC_USB30_PRIM_GDSC 4 +#define GCC_USB3_PHY_GDSC 5 + +/* GCC resets */ +#define GCC_CAMERA_BCR 0 +#define GCC_DISPLAY_BCR 1 +#define GCC_EVA_AXI0_CLK_ARES 2 +#define GCC_EVA_AXI0C_CLK_ARES 3 +#define GCC_EVA_BCR 4 +#define GCC_GPU_BCR 5 +#define GCC_PCIE_0_BCR 6 +#define GCC_PCIE_0_LINK_DOWN_BCR 7 +#define GCC_PCIE_0_NOCSR_COM_PHY_BCR 8 +#define GCC_PCIE_0_PHY_BCR 9 +#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR 10 +#define GCC_PCIE_PHY_BCR 11 +#define GCC_PCIE_PHY_CFG_AHB_BCR 12 +#define GCC_PCIE_PHY_COM_BCR 13 +#define GCC_PCIE_RSCC_BCR 14 +#define GCC_PDM_BCR 15 +#define GCC_QUPV3_WRAPPER_1_BCR 16 +#define GCC_QUPV3_WRAPPER_2_BCR 17 +#define GCC_QUPV3_WRAPPER_3_BCR 18 +#define GCC_QUPV3_WRAPPER_4_BCR 19 +#define GCC_QUPV3_WRAPPER_I2C_BCR 20 +#define GCC_QUSB2PHY_PRIM_BCR 21 +#define GCC_QUSB2PHY_SEC_BCR 22 +#define GCC_SDCC2_BCR 23 +#define GCC_SDCC4_BCR 24 +#define GCC_UFS_PHY_BCR 25 +#define GCC_USB30_PRIM_BCR 26 +#define GCC_USB3_DP_PHY_PRIM_BCR 27 +#define GCC_USB3_DP_PHY_SEC_BCR 28 +#define GCC_USB3_PHY_PRIM_BCR 29 +#define GCC_USB3_PHY_SEC_BCR 30 +#define GCC_USB3PHY_PHY_PRIM_BCR 31 +#define GCC_USB3PHY_PHY_SEC_BCR 32 +#define GCC_VIDEO_AXI0_CLK_ARES 33 +#define GCC_VIDEO_AXI1_CLK_ARES 34 +#define GCC_VIDEO_BCR 35 +#define GCC_VIDEO_XO_CLK_ARES 36 + +#endif diff --git a/include/dt-bindings/clock/qcom,mmcc-sdm660.h b/include/dt-bindings/clock/qcom,mmcc-sdm660.h index f9dbc21cb5c7c..ee2a89dae72dc 100644 --- a/include/dt-bindings/clock/qcom,mmcc-sdm660.h +++ b/include/dt-bindings/clock/qcom,mmcc-sdm660.h @@ -157,6 +157,7 @@ #define BIMC_SMMU_GDSC 7 #define CAMSS_MICRO_BCR 0 +#define MDSS_BCR 1 #endif diff --git a/include/dt-bindings/clock/qcom,x1e80100-camcc.h b/include/dt-bindings/clock/qcom,x1e80100-camcc.h index d72fdfb06a7c7..06c316022fb0d 100644 --- a/include/dt-bindings/clock/qcom,x1e80100-camcc.h +++ b/include/dt-bindings/clock/qcom,x1e80100-camcc.h @@ -115,6 +115,9 @@ #define CAM_CC_SLEEP_CLK_SRC 105 #define CAM_CC_SLOW_AHB_CLK_SRC 106 #define CAM_CC_XO_CLK_SRC 107 +#define CAM_CC_QDSS_DEBUG_CLK 108 +#define CAM_CC_QDSS_DEBUG_CLK_SRC 109 +#define CAM_CC_QDSS_DEBUG_XO_CLK 110 /* CAM_CC power domains */ #define CAM_CC_BPS_GDSC 0 diff --git a/include/dt-bindings/clock/qcom,x1e80100-gcc.h b/include/dt-bindings/clock/qcom,x1e80100-gcc.h index 710c340f24a57..62aa124255927 100644 --- a/include/dt-bindings/clock/qcom,x1e80100-gcc.h +++ b/include/dt-bindings/clock/qcom,x1e80100-gcc.h @@ -363,6 +363,30 @@ #define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC 353 #define GCC_USB3_SEC_PHY_PIPE_CLK_SRC 354 #define GCC_USB3_TERT_PHY_PIPE_CLK_SRC 355 +#define GCC_USB34_PRIM_PHY_PIPE_CLK_SRC 356 +#define GCC_USB34_SEC_PHY_PIPE_CLK_SRC 357 +#define GCC_USB34_TERT_PHY_PIPE_CLK_SRC 358 +#define GCC_USB4_0_PHY_DP0_CLK_SRC 359 +#define GCC_USB4_0_PHY_DP1_CLK_SRC 360 +#define GCC_USB4_0_PHY_P2RR2P_PIPE_CLK_SRC 361 +#define GCC_USB4_0_PHY_PCIE_PIPE_MUX_CLK_SRC 362 +#define GCC_USB4_0_PHY_RX0_CLK_SRC 363 +#define GCC_USB4_0_PHY_RX1_CLK_SRC 364 +#define GCC_USB4_0_PHY_SYS_CLK_SRC 365 +#define GCC_USB4_1_PHY_DP0_CLK_SRC 366 +#define GCC_USB4_1_PHY_DP1_CLK_SRC 367 +#define GCC_USB4_1_PHY_P2RR2P_PIPE_CLK_SRC 368 +#define GCC_USB4_1_PHY_PCIE_PIPE_MUX_CLK_SRC 369 +#define GCC_USB4_1_PHY_RX0_CLK_SRC 370 +#define GCC_USB4_1_PHY_RX1_CLK_SRC 371 +#define GCC_USB4_1_PHY_SYS_CLK_SRC 372 +#define GCC_USB4_2_PHY_DP0_CLK_SRC 373 +#define GCC_USB4_2_PHY_DP1_CLK_SRC 374 +#define GCC_USB4_2_PHY_P2RR2P_PIPE_CLK_SRC 375 +#define GCC_USB4_2_PHY_PCIE_PIPE_MUX_CLK_SRC 376 +#define GCC_USB4_2_PHY_RX0_CLK_SRC 377 +#define GCC_USB4_2_PHY_RX1_CLK_SRC 378 +#define GCC_USB4_2_PHY_SYS_CLK_SRC 379 /* GCC power domains */ #define GCC_PCIE_0_TUNNEL_GDSC 0 @@ -484,4 +508,41 @@ #define GCC_VIDEO_BCR 87 #define GCC_VIDEO_AXI0_CLK_ARES 88 #define GCC_VIDEO_AXI1_CLK_ARES 89 +#define GCC_USB4_0_MISC_USB4_SYS_BCR 90 +#define GCC_USB4_0_MISC_RX_CLK_0_BCR 91 +#define GCC_USB4_0_MISC_RX_CLK_1_BCR 92 +#define GCC_USB4_0_MISC_USB_PIPE_BCR 93 +#define GCC_USB4_0_MISC_PCIE_PIPE_BCR 94 +#define GCC_USB4_0_MISC_TMU_BCR 95 +#define GCC_USB4_0_MISC_SB_IF_BCR 96 +#define GCC_USB4_0_MISC_HIA_MSTR_BCR 97 +#define GCC_USB4_0_MISC_AHB_BCR 98 +#define GCC_USB4_0_MISC_DP0_MAX_PCLK_BCR 99 +#define GCC_USB4_0_MISC_DP1_MAX_PCLK_BCR 100 +#define GCC_USB4_1_MISC_USB4_SYS_BCR 101 +#define GCC_USB4_1_MISC_RX_CLK_0_BCR 102 +#define GCC_USB4_1_MISC_RX_CLK_1_BCR 103 +#define GCC_USB4_1_MISC_USB_PIPE_BCR 104 +#define GCC_USB4_1_MISC_PCIE_PIPE_BCR 105 +#define GCC_USB4_1_MISC_TMU_BCR 106 +#define GCC_USB4_1_MISC_SB_IF_BCR 107 +#define GCC_USB4_1_MISC_HIA_MSTR_BCR 108 +#define GCC_USB4_1_MISC_AHB_BCR 109 +#define GCC_USB4_1_MISC_DP0_MAX_PCLK_BCR 110 +#define GCC_USB4_1_MISC_DP1_MAX_PCLK_BCR 111 +#define GCC_USB4_2_MISC_USB4_SYS_BCR 112 +#define GCC_USB4_2_MISC_RX_CLK_0_BCR 113 +#define GCC_USB4_2_MISC_RX_CLK_1_BCR 114 +#define GCC_USB4_2_MISC_USB_PIPE_BCR 115 +#define GCC_USB4_2_MISC_PCIE_PIPE_BCR 116 +#define GCC_USB4_2_MISC_TMU_BCR 117 +#define GCC_USB4_2_MISC_SB_IF_BCR 118 +#define GCC_USB4_2_MISC_HIA_MSTR_BCR 119 +#define GCC_USB4_2_MISC_AHB_BCR 120 +#define GCC_USB4_2_MISC_DP0_MAX_PCLK_BCR 121 +#define GCC_USB4_2_MISC_DP1_MAX_PCLK_BCR 122 +#define GCC_USB4PHY_PHY_PRIM_BCR 123 +#define GCC_USB4PHY_PHY_SEC_BCR 124 +#define GCC_USB4PHY_PHY_TERT_BCR 125 + #endif diff --git a/include/dt-bindings/clock/qcom,x1p42100-videocc.h b/include/dt-bindings/clock/qcom,x1p42100-videocc.h new file mode 100644 index 0000000000000..996408d1a0c32 --- /dev/null +++ b/include/dt-bindings/clock/qcom,x1p42100-videocc.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_X1P42100_H +#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_X1P42100_H + +/* VIDEO_CC clocks */ +#define VIDEO_CC_MVS0_CLK 0 +#define VIDEO_CC_MVS0_CLK_SRC 1 +#define VIDEO_CC_MVS0_DIV_CLK_SRC 2 +#define VIDEO_CC_MVS0C_CLK 3 +#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC 4 +#define VIDEO_CC_MVS1_CLK 5 +#define VIDEO_CC_MVS1_CLK_SRC 6 +#define VIDEO_CC_MVS1_DIV_CLK_SRC 7 +#define VIDEO_CC_MVS1C_CLK 8 +#define VIDEO_CC_MVS1C_DIV2_DIV_CLK_SRC 9 +#define VIDEO_CC_PLL0 10 +#define VIDEO_CC_PLL1 11 +#define VIDEO_CC_MVS0_SHIFT_CLK 12 +#define VIDEO_CC_MVS0C_SHIFT_CLK 13 +#define VIDEO_CC_MVS1_SHIFT_CLK 14 +#define VIDEO_CC_MVS1C_SHIFT_CLK 15 +#define VIDEO_CC_XO_CLK_SRC 16 +#define VIDEO_CC_MVS0_BSE_CLK 17 +#define VIDEO_CC_MVS0_BSE_CLK_SRC 18 +#define VIDEO_CC_MVS0_BSE_DIV4_DIV_CLK_SRC 19 + +/* VIDEO_CC power domains */ +#define VIDEO_CC_MVS0C_GDSC 0 +#define VIDEO_CC_MVS0_GDSC 1 +#define VIDEO_CC_MVS1C_GDSC 2 +#define VIDEO_CC_MVS1_GDSC 3 + +/* VIDEO_CC resets */ +#define CVP_VIDEO_CC_INTERFACE_BCR 0 +#define CVP_VIDEO_CC_MVS0_BCR 1 +#define CVP_VIDEO_CC_MVS0C_BCR 2 +#define CVP_VIDEO_CC_MVS1_BCR 3 +#define CVP_VIDEO_CC_MVS1C_BCR 4 +#define VIDEO_CC_MVS0C_CLK_ARES 5 +#define VIDEO_CC_MVS1C_CLK_ARES 6 +#define VIDEO_CC_XO_CLK_ARES 7 +#define VIDEO_CC_MVS0_BSE_BCR 8 + +#endif diff --git a/include/dt-bindings/interconnect/qcom,kaanapali-rpmh.h b/include/dt-bindings/interconnect/qcom,kaanapali-rpmh.h new file mode 100644 index 0000000000000..dde3f9abd6776 --- /dev/null +++ b/include/dt-bindings/interconnect/qcom,kaanapali-rpmh.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_KAANAPALI_H +#define __DT_BINDINGS_INTERCONNECT_QCOM_KAANAPALI_H + +#define MASTER_QSPI_0 0 +#define MASTER_CRYPTO 1 +#define MASTER_QUP_1 2 +#define MASTER_SDCC_4 3 +#define MASTER_UFS_MEM 4 +#define MASTER_USB3 5 +#define MASTER_QUP_2 6 +#define MASTER_QUP_3 7 +#define MASTER_QUP_4 8 +#define MASTER_IPA 9 +#define MASTER_SOCCP_PROC 10 +#define MASTER_SP 11 +#define MASTER_QDSS_ETR 12 +#define MASTER_QDSS_ETR_1 13 +#define MASTER_SDCC_2 14 +#define SLAVE_A1NOC_SNOC 15 +#define SLAVE_A2NOC_SNOC 16 + +#define MASTER_QUP_CORE_0 0 +#define MASTER_QUP_CORE_1 1 +#define MASTER_QUP_CORE_2 2 +#define MASTER_QUP_CORE_3 3 +#define MASTER_QUP_CORE_4 4 +#define SLAVE_QUP_CORE_0 5 +#define SLAVE_QUP_CORE_1 6 +#define SLAVE_QUP_CORE_2 7 +#define SLAVE_QUP_CORE_3 8 +#define SLAVE_QUP_CORE_4 9 + +#define MASTER_CNOC_CFG 0 +#define SLAVE_AHB2PHY_SOUTH 1 +#define SLAVE_AHB2PHY_NORTH 2 +#define SLAVE_CAMERA_CFG 3 +#define SLAVE_CLK_CTL 4 +#define SLAVE_CRYPTO_0_CFG 5 +#define SLAVE_DISPLAY_CFG 6 +#define SLAVE_EVA_CFG 7 +#define SLAVE_GFX3D_CFG 8 +#define SLAVE_I2C 9 +#define SLAVE_I3C_IBI0_CFG 10 +#define SLAVE_I3C_IBI1_CFG 11 +#define SLAVE_IMEM_CFG 12 +#define SLAVE_IPC_ROUTER_CFG 13 +#define SLAVE_CNOC_MSS 14 +#define SLAVE_PCIE_CFG 15 +#define SLAVE_PRNG 16 +#define SLAVE_QDSS_CFG 17 +#define SLAVE_QSPI_0 18 +#define SLAVE_QUP_1 19 +#define SLAVE_QUP_2 20 +#define SLAVE_QUP_3 21 +#define SLAVE_QUP_4 22 +#define SLAVE_SDCC_2 23 +#define SLAVE_SDCC_4 24 +#define SLAVE_SPSS_CFG 25 +#define SLAVE_TCSR 26 +#define SLAVE_TLMM 27 +#define SLAVE_UFS_MEM_CFG 28 +#define SLAVE_USB3 29 +#define SLAVE_VENUS_CFG 30 +#define SLAVE_VSENSE_CTRL_CFG 31 +#define SLAVE_CNOC_MNOC_CFG 32 +#define SLAVE_PCIE_ANOC_CFG 33 +#define SLAVE_QDSS_STM 34 +#define SLAVE_TCU 35 + +#define MASTER_GEM_NOC_CNOC 0 +#define MASTER_GEM_NOC_PCIE_SNOC 1 +#define SLAVE_AOSS 2 +#define SLAVE_IPA_CFG 3 +#define SLAVE_IPC_ROUTER_FENCE 4 +#define SLAVE_SOCCP 5 +#define SLAVE_TME_CFG 6 +#define SLAVE_APPSS 7 +#define SLAVE_CNOC_CFG 8 +#define SLAVE_DDRSS_CFG 9 +#define SLAVE_BOOT_IMEM 10 +#define SLAVE_IMEM 11 +#define SLAVE_PCIE_0 12 + +#define MASTER_GPU_TCU 0 +#define MASTER_SYS_TCU 1 +#define MASTER_APPSS_PROC 2 +#define MASTER_GFX3D 3 +#define MASTER_LPASS_GEM_NOC 4 +#define MASTER_MSS_PROC 5 +#define MASTER_MNOC_HF_MEM_NOC 6 +#define MASTER_MNOC_SF_MEM_NOC 7 +#define MASTER_COMPUTE_NOC 8 +#define MASTER_ANOC_PCIE_GEM_NOC 9 +#define MASTER_QPACE 10 +#define MASTER_SNOC_SF_MEM_NOC 11 +#define MASTER_WLAN_Q6 12 +#define MASTER_GIC 13 +#define SLAVE_GEM_NOC_CNOC 14 +#define SLAVE_LLCC 15 +#define SLAVE_MEM_NOC_PCIE_SNOC 16 + +#define MASTER_LPIAON_NOC 0 +#define SLAVE_LPASS_GEM_NOC 1 + +#define MASTER_LPASS_LPINOC 0 +#define SLAVE_LPIAON_NOC_LPASS_AG_NOC 1 + +#define MASTER_LPASS_PROC 0 +#define SLAVE_LPICX_NOC_LPIAON_NOC 1 + +#define MASTER_LLCC 0 +#define SLAVE_EBI1 1 + +#define MASTER_CAMNOC_HF 0 +#define MASTER_CAMNOC_NRT_ICP_SF 1 +#define MASTER_CAMNOC_RT_CDM_SF 2 +#define MASTER_CAMNOC_SF 3 +#define MASTER_MDP 4 +#define MASTER_MDSS_DCP 5 +#define MASTER_CDSP_HCP 6 +#define MASTER_VIDEO_CV_PROC 7 +#define MASTER_VIDEO_EVA 8 +#define MASTER_VIDEO_MVP 9 +#define MASTER_VIDEO_V_PROC 10 +#define MASTER_CNOC_MNOC_CFG 11 +#define SLAVE_MNOC_HF_MEM_NOC 12 +#define SLAVE_MNOC_SF_MEM_NOC 13 +#define SLAVE_SERVICE_MNOC 14 + +#define MASTER_CDSP_PROC 0 +#define SLAVE_CDSP_MEM_NOC 1 + +#define MASTER_PCIE_ANOC_CFG 0 +#define MASTER_PCIE_0 1 +#define SLAVE_ANOC_PCIE_GEM_NOC 2 +#define SLAVE_SERVICE_PCIE_ANOC 3 + +#define MASTER_A1NOC_SNOC 0 +#define MASTER_A2NOC_SNOC 1 +#define MASTER_APSS_NOC 2 +#define MASTER_CNOC_SNOC 3 +#define SLAVE_SNOC_GEM_NOC_SF 4 + +#endif diff --git a/include/dt-bindings/mailbox/qcom-ipcc.h b/include/dt-bindings/mailbox/qcom-ipcc.h index fd85a79381b31..dc07ca27db09d 100644 --- a/include/dt-bindings/mailbox/qcom-ipcc.h +++ b/include/dt-bindings/mailbox/qcom-ipcc.h @@ -36,4 +36,52 @@ #define IPCC_CLIENT_GPDSP0 31 #define IPCC_CLIENT_GPDSP1 32 +/* Physical client IDs */ +#define IPCC_MPROC_AOP 0 +#define IPCC_MPROC_TZ 1 +#define IPCC_MPROC_MPSS 2 +#define IPCC_MPROC_LPASS 3 +#define IPCC_MPROC_SDC 4 +#define IPCC_MPROC_CDSP 5 +#define IPCC_MPROC_APSS 6 +#define IPCC_MPROC_SOCCP 13 +#define IPCC_MPROC_DCP 14 +#define IPCC_MPROC_SPSS 15 +#define IPCC_MPROC_TME 16 + +#define IPCC_COMPUTE_L0_CDSP 2 +#define IPCC_COMPUTE_L0_APSS 3 +#define IPCC_COMPUTE_L0_GPU 4 +#define IPCC_COMPUTE_L0_CVP 8 +#define IPCC_COMPUTE_L0_CAM 9 +#define IPCC_COMPUTE_L0_CAM1 10 +#define IPCC_COMPUTE_L0_DCP 11 +#define IPCC_COMPUTE_L0_VPU 12 +#define IPCC_COMPUTE_L0_SOCCP 16 + +#define IPCC_COMPUTE_L1_CDSP 2 +#define IPCC_COMPUTE_L1_APSS 3 +#define IPCC_COMPUTE_L1_GPU 4 +#define IPCC_COMPUTE_L1_CVP 8 +#define IPCC_COMPUTE_L1_CAM 9 +#define IPCC_COMPUTE_L1_CAM1 10 +#define IPCC_COMPUTE_L1_DCP 11 +#define IPCC_COMPUTE_L1_VPU 12 +#define IPCC_COMPUTE_L1_SOCCP 16 + +#define IPCC_PERIPH_CDSP 2 +#define IPCC_PERIPH_APSS 3 +#define IPCC_PERIPH_PCIE0 4 +#define IPCC_PERIPH_PCIE1 5 + +#define IPCC_FENCE_CDSP 2 +#define IPCC_FENCE_APSS 3 +#define IPCC_FENCE_GPU 4 +#define IPCC_FENCE_CVP 8 +#define IPCC_FENCE_CAM 8 +#define IPCC_FENCE_VPU 20 +#define IPCC_FENCE_SOCCP 24 +#define IPCC_FENCE_CAM1 10 +#define IPCC_FENCE_DCP 11 + #endif diff --git a/include/dt-bindings/media/qcom,qcs615-venus.h b/include/dt-bindings/media/qcom,qcs615-venus.h new file mode 100644 index 0000000000000..ca755f2f6aa40 --- /dev/null +++ b/include/dt-bindings/media/qcom,qcs615-venus.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_MEDIA_QCOM_QCS615_VENUS_H_ +#define _DT_BINDINGS_MEDIA_QCOM_QCS615_VENUS_H_ + +#define VENUS_FIRMWARE 0 + +#endif diff --git a/include/dt-bindings/media/qcom,sm8550-iris.h b/include/dt-bindings/media/qcom,sm8550-iris.h new file mode 100644 index 0000000000000..5165ef817f8bb --- /dev/null +++ b/include/dt-bindings/media/qcom,sm8550-iris.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_MEDIA_QCOM_SM8550_IRIS_H_ +#define _DT_BINDINGS_MEDIA_QCOM_SM8550_IRIS_H_ + +/* Function identifiers for iommu-map to attach for the context bank devices */ +#define IRIS_NON_PIXEL_VCODEC 0 +#define IRIS_PIXEL 1 + +#endif diff --git a/include/dt-bindings/phy/phy-qcom-mipi-csi2.h b/include/dt-bindings/phy/phy-qcom-mipi-csi2.h new file mode 100644 index 0000000000000..fa48fd75c58d8 --- /dev/null +++ b/include/dt-bindings/phy/phy-qcom-mipi-csi2.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ +/* + * Qualcomm MIPI CSI2 PHY constants + * + * Copyright (C) 2026 Linaro Limited + */ + +#ifndef __DT_BINDINGS_PHY_MIPI_CSI2__ +#define __DT_BINDINGS_PHY_MIPI_CSI2__ + +#define PHY_QCOM_CSI2_MODE_DPHY 0 +#define PHY_QCOM_CSI2_MODE_CPHY 1 +#define PHY_QCOM_CSI2_MODE_SPLIT_DPHY 2 + +#endif /* __DT_BINDINGS_PHY_MIPI_CSI2__ */ diff --git a/include/dt-bindings/power/qcom,rpmhpd.h b/include/dt-bindings/power/qcom,rpmhpd.h index 73cceb88953f7..06851363ae0ea 100644 --- a/include/dt-bindings/power/qcom,rpmhpd.h +++ b/include/dt-bindings/power/qcom,rpmhpd.h @@ -33,11 +33,14 @@ #define RPMH_REGULATOR_LEVEL_RETENTION 16 #define RPMH_REGULATOR_LEVEL_MIN_SVS 48 #define RPMH_REGULATOR_LEVEL_LOW_SVS_D3 50 +#define RPMH_REGULATOR_LEVEL_LOW_SVS_D2_1 51 #define RPMH_REGULATOR_LEVEL_LOW_SVS_D2 52 +#define RPMH_REGULATOR_LEVEL_LOW_SVS_D1_1 54 #define RPMH_REGULATOR_LEVEL_LOW_SVS_D1 56 #define RPMH_REGULATOR_LEVEL_LOW_SVS_D0 60 #define RPMH_REGULATOR_LEVEL_LOW_SVS 64 #define RPMH_REGULATOR_LEVEL_LOW_SVS_P1 72 +#define RPMH_REGULATOR_LEVEL_LOW_SVS_L0 76 #define RPMH_REGULATOR_LEVEL_LOW_SVS_L1 80 #define RPMH_REGULATOR_LEVEL_LOW_SVS_L2 96 #define RPMH_REGULATOR_LEVEL_SVS 128 @@ -261,5 +264,6 @@ #define SC8280XP_NSP 13 #define SC8280XP_QPHY 14 #define SC8280XP_XO 15 +#define SC8280XP_MXC_AO 16 #endif diff --git a/include/kunit/run-in-irq-context.h b/include/kunit/run-in-irq-context.h index 108e96433ea45..bfe60d6cf28d8 100644 --- a/include/kunit/run-in-irq-context.h +++ b/include/kunit/run-in-irq-context.h @@ -12,16 +12,16 @@ #include #include -#define KUNIT_IRQ_TEST_HRTIMER_INTERVAL us_to_ktime(5) - struct kunit_irq_test_state { bool (*func)(void *test_specific_state); void *test_specific_state; bool task_func_reported_failure; bool hardirq_func_reported_failure; bool softirq_func_reported_failure; - unsigned long hardirq_func_calls; - unsigned long softirq_func_calls; + atomic_t task_func_calls; + atomic_t hardirq_func_calls; + atomic_t softirq_func_calls; + ktime_t interval; struct hrtimer timer; struct work_struct bh_work; }; @@ -30,14 +30,25 @@ static enum hrtimer_restart kunit_irq_test_timer_func(struct hrtimer *timer) { struct kunit_irq_test_state *state = container_of(timer, typeof(*state), timer); + int task_calls, hardirq_calls, softirq_calls; WARN_ON_ONCE(!in_hardirq()); - state->hardirq_func_calls++; + task_calls = atomic_read(&state->task_func_calls); + hardirq_calls = atomic_inc_return(&state->hardirq_func_calls); + softirq_calls = atomic_read(&state->softirq_func_calls); + + /* + * If the timer is firing too often for the softirq or task to ever have + * a chance to run, increase the timer interval. This is needed on very + * slow systems. + */ + if (hardirq_calls >= 20 && (softirq_calls == 0 || task_calls == 0)) + state->interval = ktime_add_ns(state->interval, 250); if (!state->func(state->test_specific_state)) state->hardirq_func_reported_failure = true; - hrtimer_forward_now(&state->timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL); + hrtimer_forward_now(&state->timer, state->interval); queue_work(system_bh_wq, &state->bh_work); return HRTIMER_RESTART; } @@ -48,7 +59,7 @@ static void kunit_irq_test_bh_work_func(struct work_struct *work) container_of(work, typeof(*state), bh_work); WARN_ON_ONCE(!in_serving_softirq()); - state->softirq_func_calls++; + atomic_inc(&state->softirq_func_calls); if (!state->func(state->test_specific_state)) state->softirq_func_reported_failure = true; @@ -59,7 +70,10 @@ static void kunit_irq_test_bh_work_func(struct work_struct *work) * hardirq context concurrently, and reports a failure to KUnit if any * invocation of @func in any context returns false. @func is passed * @test_specific_state as its argument. At most 3 invocations of @func will - * run concurrently: one in each of task, softirq, and hardirq context. + * run concurrently: one in each of task, softirq, and hardirq context. @func + * will continue running until either @max_iterations calls have been made (so + * long as at least one each runs in task, softirq, and hardirq contexts), or + * one second has passed. * * The main purpose of this interrupt context testing is to validate fallback * code paths that run in contexts where the normal code path cannot be used, @@ -83,8 +97,14 @@ static inline void kunit_run_irq_test(struct kunit *test, bool (*func)(void *), struct kunit_irq_test_state state = { .func = func, .test_specific_state = test_specific_state, + /* + * Start with a 5us timer interval. If the system can't keep + * up, kunit_irq_test_timer_func() will increase it. + */ + .interval = us_to_ktime(5), }; unsigned long end_jiffies; + int task_calls, hardirq_calls, softirq_calls; /* * Set up a hrtimer (the way we access hardirq context) and a work @@ -94,36 +114,41 @@ static inline void kunit_run_irq_test(struct kunit *test, bool (*func)(void *), CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); INIT_WORK_ONSTACK(&state.bh_work, kunit_irq_test_bh_work_func); - /* Run for up to max_iterations or 1 second, whichever comes first. */ + /* + * Run for up to max_iterations (including at least one task, softirq, + * and hardirq), or 1 second, whichever comes first. + */ end_jiffies = jiffies + HZ; - hrtimer_start(&state.timer, KUNIT_IRQ_TEST_HRTIMER_INTERVAL, - HRTIMER_MODE_REL_HARD); - for (int i = 0; i < max_iterations && !time_after(jiffies, end_jiffies); - i++) { + hrtimer_start(&state.timer, state.interval, HRTIMER_MODE_REL_HARD); + do { if (!func(test_specific_state)) state.task_func_reported_failure = true; - } + + task_calls = atomic_inc_return(&state.task_func_calls); + hardirq_calls = atomic_read(&state.hardirq_func_calls); + softirq_calls = atomic_read(&state.softirq_func_calls); + } while ((task_calls + hardirq_calls + softirq_calls < max_iterations || + (task_calls == 0 || hardirq_calls == 0 || + softirq_calls == 0)) && + !time_after(jiffies, end_jiffies)); /* Cancel the timer and work. */ hrtimer_cancel(&state.timer); flush_work(&state.bh_work); /* Sanity check: the timer and BH functions should have been run. */ - KUNIT_EXPECT_GT_MSG(test, state.hardirq_func_calls, 0, + KUNIT_EXPECT_GT_MSG(test, atomic_read(&state.hardirq_func_calls), 0, "Timer function was not called"); - KUNIT_EXPECT_GT_MSG(test, state.softirq_func_calls, 0, + KUNIT_EXPECT_GT_MSG(test, atomic_read(&state.softirq_func_calls), 0, "BH work function was not called"); - /* Check for incorrect hash values reported from any context. */ - KUNIT_EXPECT_FALSE_MSG( - test, state.task_func_reported_failure, - "Incorrect hash values reported from task context"); - KUNIT_EXPECT_FALSE_MSG( - test, state.hardirq_func_reported_failure, - "Incorrect hash values reported from hardirq context"); - KUNIT_EXPECT_FALSE_MSG( - test, state.softirq_func_reported_failure, - "Incorrect hash values reported from softirq context"); + /* Check for failure reported from any context. */ + KUNIT_EXPECT_FALSE_MSG(test, state.task_func_reported_failure, + "Failure reported from task context"); + KUNIT_EXPECT_FALSE_MSG(test, state.hardirq_func_reported_failure, + "Failure reported from hardirq context"); + KUNIT_EXPECT_FALSE_MSG(test, state.softirq_func_reported_failure, + "Failure reported from softirq context"); } #endif /* _KUNIT_RUN_IN_IRQ_CONTEXT_H */ diff --git a/include/linux/audit.h b/include/linux/audit.h index 536f8ee8da818..b8d8029c6c480 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -128,12 +128,6 @@ enum audit_nfcfgop { extern int __init audit_register_class(int class, unsigned *list); extern int audit_classify_syscall(int abi, unsigned syscall); extern int audit_classify_arch(int arch); -/* only for compat system calls */ -extern unsigned compat_write_class[]; -extern unsigned compat_read_class[]; -extern unsigned compat_dir_class[]; -extern unsigned compat_chattr_class[]; -extern unsigned compat_signal_class[]; /* audit_names->type values */ #define AUDIT_TYPE_UNKNOWN 0 /* we don't know yet */ diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index 0e34d673ef171..2b8153791e6a5 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -23,4 +23,11 @@ enum auditsc_class_t { extern int audit_classify_compat_syscall(int abi, unsigned syscall); +/* only for compat system calls */ +extern unsigned compat_write_class[]; +extern unsigned compat_read_class[]; +extern unsigned compat_dir_class[]; +extern unsigned compat_chattr_class[]; +extern unsigned compat_signal_class[]; + #endif diff --git a/include/linux/auxvec.h b/include/linux/auxvec.h index 407f7005e6d60..8bcb9b7262628 100644 --- a/include/linux/auxvec.h +++ b/include/linux/auxvec.h @@ -4,6 +4,6 @@ #include -#define AT_VECTOR_SIZE_BASE 22 /* NEW_AUX_ENT entries in auxiliary table */ +#define AT_VECTOR_SIZE_BASE 24 /* NEW_AUX_ENT entries in auxiliary table */ /* number of "#define AT_.*" above, minus {AT_NULL, AT_IGNORE, AT_NOTELF} */ #endif /* _LINUX_AUXVEC_H */ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 44c30183ecc34..4e2e3aed32f5f 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -469,10 +469,7 @@ static inline bool op_is_discard(blk_opf_t op) } /* - * Check if a bio or request operation is a zone management operation, with - * the exception of REQ_OP_ZONE_RESET_ALL which is treated as a special case - * due to its different handling in the block layer and device response in - * case of command failure. + * Check if a bio or request operation is a zone management operation. */ static inline bool op_is_zone_mgmt(enum req_op op) { diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 70b671a9a7f77..59e54550a053f 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -485,7 +485,7 @@ struct request_queue { */ unsigned long queue_flags; - unsigned int rq_timeout; + unsigned int __data_racy rq_timeout; unsigned int queue_depth; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d808253f2e945..e2dd3a6d495af 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3200,6 +3200,11 @@ static inline void bpf_prog_report_arena_violation(bool write, unsigned long add } #endif /* CONFIG_BPF_SYSCALL */ +static inline bool bpf_net_capable(void) +{ + return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); +} + static __always_inline int bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) { diff --git a/include/linux/bpf_mprog.h b/include/linux/bpf_mprog.h index 929225f7b0959..0b9f4caeeb0a3 100644 --- a/include/linux/bpf_mprog.h +++ b/include/linux/bpf_mprog.h @@ -340,4 +340,14 @@ static inline bool bpf_mprog_supported(enum bpf_prog_type type) return false; } } + +static inline bool bpf_mprog_detach_empty(enum bpf_prog_type type) +{ + switch (type) { + case BPF_PROG_TYPE_SCHED_CLS: + return bpf_net_capable(); + default: + return false; + } +} #endif /* __BPF_MPROG_H */ diff --git a/include/linux/buildid.h b/include/linux/buildid.h index 014a88c410739..5e0a14866cc18 100644 --- a/include/linux/buildid.h +++ b/include/linux/buildid.h @@ -7,7 +7,10 @@ #define BUILD_ID_SIZE_MAX 20 struct vm_area_struct; +struct file; + int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size); +int build_id_parse_file(struct file *file, unsigned char *build_id, __u32 *size); int build_id_parse_nofault(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size); int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size); diff --git a/include/linux/capability.h b/include/linux/capability.h index 1fb08922552c7..37db92b3d6f89 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -203,6 +203,12 @@ static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns) ns_capable(ns, CAP_SYS_ADMIN); } +static inline bool checkpoint_restore_ns_capable_noaudit(struct user_namespace *ns) +{ + return ns_capable_noaudit(ns, CAP_CHECKPOINT_RESTORE) || + ns_capable_noaudit(ns, CAP_SYS_ADMIN); +} + /* audit system wants to get cap info from files as well */ int get_vfs_caps_from_disk(struct mnt_idmap *idmap, const struct dentry *dentry, diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index c7f2c63b3bc3f..08e5dbe15ca44 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -31,6 +31,12 @@ #define CEPH_INO_CEPH 2 /* hidden .ceph dir */ #define CEPH_INO_GLOBAL_SNAPREALM 3 /* global dummy snaprealm */ +/* + * name for "old" CephFS file systems, + * see ceph.git e2b151d009640114b2565c901d6f41f6cd5ec652 + */ +#define CEPH_OLD_FS_NAME "cephfs" + /* arbitrary limit on max # of monitors (cluster of 3 is typical) */ #define CEPH_MAX_MON 31 diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index 2573585b7f068..19c7e475d3a4d 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -290,15 +290,16 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \ class_##_name##_t var __cleanup(class_##_name##_destructor) = \ class_##_name##_constructor -#define scoped_class(_name, var, args) \ - for (CLASS(_name, var)(args); \ - __guard_ptr(_name)(&var) || !__is_cond_ptr(_name); \ - ({ goto _label; })) \ - if (0) { \ -_label: \ - break; \ +#define __scoped_class(_name, var, _label, args...) \ + for (CLASS(_name, var)(args); ; ({ goto _label; })) \ + if (0) { \ +_label: \ + break; \ } else +#define scoped_class(_name, var, args...) \ + __scoped_class(_name, var, __UNIQUE_ID(label), args) + /* * DEFINE_GUARD(name, type, lock, unlock): * trivial wrapper around DEFINE_CLASS() above specifically diff --git a/include/linux/clk.h b/include/linux/clk.h index b607482ca77e9..64ff118ffb1a1 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -228,6 +228,23 @@ int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk); */ void clk_rate_exclusive_put(struct clk *clk); +/** + * clk_save_context - save clock context for poweroff + * + * Saves the context of the clock register for powerstates in which the + * contents of the registers will be lost. Occurs deep within the suspend + * code so locking is not necessary. + */ +int clk_save_context(void); + +/** + * clk_restore_context - restore clock context after poweroff + * + * This occurs with all clocks enabled. Occurs deep within the resume code + * so locking is not necessary. + */ +void clk_restore_context(void); + #else static inline int clk_notifier_register(struct clk *clk, @@ -293,6 +310,13 @@ static inline int devm_clk_rate_exclusive_get(struct device *dev, struct clk *cl static inline void clk_rate_exclusive_put(struct clk *clk) {} +static inline int clk_save_context(void) +{ + return 0; +} + +static inline void clk_restore_context(void) {} + #endif #ifdef CONFIG_HAVE_CLK_PREPARE @@ -933,23 +957,6 @@ struct clk *clk_get_parent(struct clk *clk); */ struct clk *clk_get_sys(const char *dev_id, const char *con_id); -/** - * clk_save_context - save clock context for poweroff - * - * Saves the context of the clock register for powerstates in which the - * contents of the registers will be lost. Occurs deep within the suspend - * code so locking is not necessary. - */ -int clk_save_context(void); - -/** - * clk_restore_context - restore clock context after poweroff - * - * This occurs with all clocks enabled. Occurs deep within the resume code - * so locking is not necessary. - */ -void clk_restore_context(void); - #else /* !CONFIG_HAVE_CLK */ static inline struct clk *clk_get(struct device *dev, const char *id) @@ -1129,13 +1136,6 @@ static inline struct clk *clk_get_sys(const char *dev_id, const char *con_id) return NULL; } -static inline int clk_save_context(void) -{ - return 0; -} - -static inline void clk_restore_context(void) {} - #endif /* clk_prepare_enable helps cases using clk_enable in non-atomic context. */ diff --git a/include/linux/cma.h b/include/linux/cma.h index 62d9c1cf63265..2e6931735880b 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -57,6 +57,15 @@ extern bool cma_intersects(struct cma *cma, unsigned long start, unsigned long e extern void cma_reserve_pages_on_error(struct cma *cma); +#ifdef CONFIG_DMA_CMA +extern bool cma_skip_dt_default_reserved_mem(void); +#else +static inline bool cma_skip_dt_default_reserved_mem(void) +{ + return false; +} +#endif + #ifdef CONFIG_CMA struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp); bool cma_free_folio(struct cma *cma, const struct folio *folio); diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 107ce05bd16eb..cccc72fd336b8 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -152,4 +152,4 @@ * Bindgen uses LLVM even if our C compiler is GCC, so we cannot * rely on the auto-detected CONFIG_CC_HAS_TYPEOF_UNQUAL. */ -#define CC_HAS_TYPEOF_UNQUAL (__clang_major__ >= 19) +#define CC_HAS_TYPEOF_UNQUAL (__clang_major__ > 19 || (__clang_major__ == 19 && __clang_minor__ > 0)) diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 0a1b9598940d0..2998af80cbef3 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -13,6 +13,19 @@ #ifndef __ASSEMBLY__ +/* + * C23 introduces "auto" as a standard way to define type-inferred + * variables, but "auto" has been a (useless) keyword even since K&R C, + * so it has always been "namespace reserved." + * + * Until at some future time we require C23 support, we need the gcc + * extension __auto_type, but there is no reason to put that elsewhere + * in the source code. + */ +#if __STDC_VERSION__ < 202311L +# define auto __auto_type +#endif + /* * Skipped when running bindgen due to a libclang issue; * see https://github.com/rust-lang/rust-bindgen/issues/2244. @@ -290,6 +303,22 @@ struct ftrace_likely_data { # define __no_kasan_or_inline __always_inline #endif +#ifdef CONFIG_KCSAN +/* + * Type qualifier to mark variables where all data-racy accesses should be + * ignored by KCSAN. Note, the implementation simply marks these variables as + * volatile, since KCSAN will treat such accesses as "marked". + * + * Defined here because defining __data_racy as volatile for KCSAN objects only + * causes problems in BPF Type Format (BTF) generation since struct members + * of core kernel data structs will be volatile in some objects and not in + * others. Instead define it globally for KCSAN kernels. + */ +# define __data_racy volatile +#else +# define __data_racy +#endif + #ifdef __SANITIZE_THREAD__ /* * Clang still emits instrumentation for __tsan_func_{entry,exit}() and builtin @@ -301,16 +330,9 @@ struct ftrace_likely_data { * disable all instrumentation. See Kconfig.kcsan where this is mandatory. */ # define __no_kcsan __no_sanitize_thread __disable_sanitizer_instrumentation -/* - * Type qualifier to mark variables where all data-racy accesses should be - * ignored by KCSAN. Note, the implementation simply marks these variables as - * volatile, since KCSAN will treat such accesses as "marked". - */ -# define __data_racy volatile # define __no_sanitize_or_inline __no_kcsan notrace __maybe_unused #else # define __no_kcsan -# define __data_racy #endif #ifdef __SANITIZE_MEMORY__ diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h index 13b35637bd5ae..d5ca855116df4 100644 --- a/include/linux/console_struct.h +++ b/include/linux/console_struct.h @@ -160,6 +160,7 @@ struct vc_data { struct uni_pagedict **uni_pagedict_loc; /* [!] Location of uni_pagedict variable for this console */ u32 **vc_uni_lines; /* unicode screen content */ u16 *vc_saved_screen; + u32 **vc_saved_uni_lines; unsigned int vc_saved_cols; unsigned int vc_saved_rows; /* additional information is in vt_kern.h */ diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 6de59ce8ef8ca..bf8f1dc7b429c 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -251,15 +251,11 @@ struct coresight_trace_id_map { * by @coresight_ops. * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. - * @mode: This tracer's mode, i.e sysFS, Perf or disabled. This is - * actually an 'enum cs_mode', but is stored in an atomic type. - * This is always accessed through local_read() and local_set(), - * but wherever it's done from within the Coresight device's lock, - * a non-atomic read would also work. This is the main point of - * synchronisation between code happening inside the sysfs mode's - * coresight_mutex and outside when running in Perf mode. A compare - * and exchange swap is done to atomically claim one mode or the - * other. + * @mode: The device mode, i.e sysFS, Perf or disabled. This is actually + * an 'enum cs_mode' but stored in an atomic type. Access is always + * through atomic APIs, ensuring SMP-safe synchronisation between + * racing from sysFS and Perf mode. A compare-and-exchange + * operation is done to atomically claim one mode or the other. * @refcnt: keep track of what is in use. Only access this outside of the * device's spinlock when the coresight_mutex held and mode == * CS_MODE_SYSFS. Otherwise it must be accessed from inside the @@ -288,7 +284,7 @@ struct coresight_device { const struct coresight_ops *ops; struct csdev_access access; struct device dev; - local_t mode; + atomic_t mode; int refcnt; bool orphan; /* sink specific fields */ @@ -332,12 +328,14 @@ static struct coresight_dev_list (var) = { \ /** * struct coresight_path - data needed by enable/disable path - * @path_list: path from source to sink. - * @trace_id: trace_id of the whole path. + * @path_list: path from source to sink. + * @trace_id: trace_id of the whole path. + * @handle: handle of the aux_event. */ struct coresight_path { - struct list_head path_list; - u8 trace_id; + struct list_head path_list; + u8 trace_id; + struct perf_output_handle *handle; }; enum cs_mode { @@ -385,7 +383,8 @@ struct coresight_ops_sink { struct coresight_ops_link { int (*enable)(struct coresight_device *csdev, struct coresight_connection *in, - struct coresight_connection *out); + struct coresight_connection *out, + enum cs_mode mode); void (*disable)(struct coresight_device *csdev, struct coresight_connection *in, struct coresight_connection *out); @@ -621,13 +620,14 @@ static inline bool coresight_is_percpu_sink(struct coresight_device *csdev) static inline bool coresight_take_mode(struct coresight_device *csdev, enum cs_mode new_mode) { - return local_cmpxchg(&csdev->mode, CS_MODE_DISABLED, new_mode) == - CS_MODE_DISABLED; + int curr = CS_MODE_DISABLED; + + return atomic_try_cmpxchg_acquire(&csdev->mode, &curr, new_mode); } static inline enum cs_mode coresight_get_mode(struct coresight_device *csdev) { - return local_read(&csdev->mode); + return atomic_read_acquire(&csdev->mode); } static inline void coresight_set_mode(struct coresight_device *csdev, @@ -643,7 +643,7 @@ static inline void coresight_set_mode(struct coresight_device *csdev, WARN(new_mode != CS_MODE_DISABLED && current_mode != CS_MODE_DISABLED && current_mode != new_mode, "Device already in use\n"); - local_set(&csdev->mode, new_mode); + atomic_set_release(&csdev->mode, new_mode); } struct coresight_device *coresight_register(struct coresight_desc *desc); diff --git a/include/linux/cper.h b/include/linux/cper.h index 0ed60a91eca9d..440b35e459e53 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -297,11 +297,11 @@ enum { #define CPER_ARM_INFO_FLAGS_PROPAGATED BIT(2) #define CPER_ARM_INFO_FLAGS_OVERFLOW BIT(3) -#define CPER_ARM_CACHE_ERROR 0 -#define CPER_ARM_TLB_ERROR 1 -#define CPER_ARM_BUS_ERROR 2 -#define CPER_ARM_VENDOR_ERROR 3 -#define CPER_ARM_MAX_TYPE CPER_ARM_VENDOR_ERROR +#define CPER_ARM_ERR_TYPE_MASK GENMASK(4,1) +#define CPER_ARM_CACHE_ERROR BIT(1) +#define CPER_ARM_TLB_ERROR BIT(2) +#define CPER_ARM_BUS_ERROR BIT(3) +#define CPER_ARM_VENDOR_ERROR BIT(4) #define CPER_ARM_ERR_VALID_TRANSACTION_TYPE BIT(0) #define CPER_ARM_ERR_VALID_OPERATION_TYPE BIT(1) @@ -588,12 +588,15 @@ const char *cper_mem_err_type_str(unsigned int); const char *cper_mem_err_status_str(u64 status); void cper_print_bits(const char *prefix, unsigned int bits, const char * const strs[], unsigned int strs_size); +int cper_bits_to_str(char *buf, int buf_size, unsigned long bits, + const char * const strs[], unsigned int strs_size); void cper_mem_err_pack(const struct cper_sec_mem_err *, struct cper_mem_err_compact *); const char *cper_mem_err_unpack(struct trace_seq *, struct cper_mem_err_compact *); void cper_print_proc_arm(const char *pfx, - const struct cper_sec_proc_arm *proc); + const struct cper_sec_proc_arm *proc, + u32 length); void cper_print_proc_ia(const char *pfx, const struct cper_sec_proc_ia *proc); int cper_mem_err_location(struct cper_mem_err_compact *mem, char *msg); diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 2ddb256187b51..75283991e77ab 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -173,7 +173,7 @@ static inline void set_mems_allowed(nodemask_t nodemask) task_unlock(current); } -extern bool cpuset_node_allowed(struct cgroup *cgroup, int nid); +extern void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask); #else /* !CONFIG_CPUSETS */ static inline bool cpusets_enabled(void) { return false; } @@ -294,9 +294,9 @@ static inline bool read_mems_allowed_retry(unsigned int seq) return false; } -static inline bool cpuset_node_allowed(struct cgroup *cgroup, int nid) +static inline void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) { - return true; + nodes_copy(*mask, node_states[N_MEMORY]); } #endif /* !CONFIG_CPUSETS */ diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h index 7b44b41d0a20d..f0dc03d94ca2c 100644 --- a/include/linux/crash_reserve.h +++ b/include/linux/crash_reserve.h @@ -32,6 +32,12 @@ int __init parse_crashkernel(char *cmdline, unsigned long long system_ram, void __init reserve_crashkernel_cma(unsigned long long cma_size); #ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +#ifndef arch_add_crash_res_to_iomem +static inline bool arch_add_crash_res_to_iomem(void) +{ + return true; +} +#endif #ifndef DEFAULT_CRASH_KERNEL_LOW_SIZE #define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20) #endif diff --git a/include/linux/damon.h b/include/linux/damon.h index cae8c613c5fca..1a8a79d7e4e8d 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -786,6 +786,12 @@ struct damon_ctx { struct damos_walk_control *walk_control; struct mutex walk_control_lock; + /* + * indicate if this may be corrupted. Currentonly this is set only for + * damon_commit_ctx() failure. + */ + bool maybe_corrupted; + /* public: */ struct task_struct *kdamond; struct mutex kdamond_lock; diff --git a/include/linux/device.h b/include/linux/device.h index b031ff71a5bdf..8733a4edf3ccd 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -502,6 +502,8 @@ struct device_physical_location { * on. This shrinks the "Board Support Packages" (BSPs) and * minimizes board-specific #ifdefs in drivers. * @driver_data: Private pointer for driver specific info. + * @driver_override: Driver name to force a match. Do not touch directly; use + * device_set_driver_override() instead. * @links: Links to suppliers and consumers of this device. * @power: For device power management. * See Documentation/driver-api/pm/devices.rst for details. @@ -595,6 +597,10 @@ struct device { core doesn't touch it */ void *driver_data; /* Driver data, set and get with dev_set_drvdata/dev_get_drvdata */ + struct { + const char *name; + spinlock_t lock; + } driver_override; struct mutex mutex; /* mutex to synchronize calls to * its driver. */ @@ -720,6 +726,54 @@ struct device_link { #define kobj_to_dev(__kobj) container_of_const(__kobj, struct device, kobj) +int __device_set_driver_override(struct device *dev, const char *s, size_t len); + +/** + * device_set_driver_override() - Helper to set or clear driver override. + * @dev: Device to change + * @s: NUL-terminated string, new driver name to force a match, pass empty + * string to clear it ("" or "\n", where the latter is only for sysfs + * interface). + * + * Helper to set or clear driver override of a device. + * + * Returns: 0 on success or a negative error code on failure. + */ +static inline int device_set_driver_override(struct device *dev, const char *s) +{ + return __device_set_driver_override(dev, s, s ? strlen(s) : 0); +} + +/** + * device_has_driver_override() - Check if a driver override has been set. + * @dev: device to check + * + * Returns true if a driver override has been set for this device. + */ +static inline bool device_has_driver_override(struct device *dev) +{ + guard(spinlock)(&dev->driver_override.lock); + return !!dev->driver_override.name; +} + +/** + * device_match_driver_override() - Match a driver against the device's driver_override. + * @dev: device to check + * @drv: driver to match against + * + * Returns > 0 if a driver override is set and matches the given driver, 0 if a + * driver override is set but does not match, or < 0 if a driver override is not + * set at all. + */ +static inline int device_match_driver_override(struct device *dev, + const struct device_driver *drv) +{ + guard(spinlock)(&dev->driver_override.lock); + if (dev->driver_override.name) + return !strcmp(dev->driver_override.name, drv->name); + return -1; +} + /** * device_iommu_mapped - Returns true when the device DMA is translated * by an IOMMU diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index f5a56efd2bd6a..15de0d7881f9e 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -63,6 +63,9 @@ struct fwnode_handle; * this bus. * @pm: Power management operations of this bus, callback the specific * device driver's pm-ops. + * @driver_override: Set to true if this bus supports the driver_override + * mechanism, which allows userspace to force a specific + * driver to bind to a device via a sysfs attribute. * @need_parent_lock: When probing or removing a device on this bus, the * device core should lock the device's parent. * @@ -104,6 +107,7 @@ struct bus_type { const struct dev_pm_ops *pm; + bool driver_override; bool need_parent_lock; }; diff --git a/include/linux/device_cgroup.h b/include/linux/device_cgroup.h index 0864773a57e8f..822085bc2d202 100644 --- a/include/linux/device_cgroup.h +++ b/include/linux/device_cgroup.h @@ -21,7 +21,7 @@ static inline int devcgroup_inode_permission(struct inode *inode, int mask) if (likely(!S_ISBLK(inode->i_mode) && !S_ISCHR(inode->i_mode))) return 0; - if (likely(!inode->i_rdev)) + if (!inode->i_rdev) return 0; if (S_ISBLK(inode->i_mode)) diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 2ceda49c609fb..3e63046b899bc 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -78,6 +78,13 @@ */ #define DMA_ATTR_MMIO (1UL << 10) +/* + * DMA_ATTR_CPU_CACHE_CLEAN: Indicates the CPU will not dirty any cacheline + * overlapping this buffer while it is mapped for DMA. All mappings sharing + * a cacheline must have this attribute for this to be considered safe. + */ +#define DMA_ATTR_CPU_CACHE_CLEAN (1UL << 11) + /* * A dma_addr_t can hold any valid DMA or bus address for the platform. It can * be given to a device to use as a DMA source or target. It is specific to a @@ -90,7 +97,7 @@ */ #define DMA_MAPPING_ERROR (~(dma_addr_t)0) -#define DMA_BIT_MASK(n) GENMASK_ULL(n - 1, 0) +#define DMA_BIT_MASK(n) GENMASK_ULL((n) - 1, 0) struct dma_iova_state { dma_addr_t addr; @@ -240,8 +247,8 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, { return NULL; } -static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, unsigned long attrs) +static inline void dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) { } static inline void *dmam_alloc_attrs(struct device *dev, size_t size, diff --git a/include/linux/dma/qcom-gpi-dma.h b/include/linux/dma/qcom-gpi-dma.h index 6680dd1a43c68..36cbb85499b4c 100644 --- a/include/linux/dma/qcom-gpi-dma.h +++ b/include/linux/dma/qcom-gpi-dma.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2020, Linaro Limited + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #ifndef QCOM_GPI_DMA_H @@ -51,6 +52,21 @@ enum i2c_op { I2C_READ, }; +/** + * enum gpi_lock_action - request lock/unlock TRE sequencing + * @GPI_LOCK_NONE: No lock/unlock TRE requested for this transfer + * @GPI_LOCK_ACQUIRE: Emit a lock TRE before the transfer + * @GPI_LOCK_RELEASE: Emit an unlock TRE after the transfer + * + * Used by protocol drivers for multi-owner controller setups (e.g. when + * DeviceTree indicates the controller is shared via qcom,qup-multi-owner). + */ +enum gpi_lock_action { + GPI_LOCK_NONE = 0, + GPI_LOCK_ACQUIRE, + GPI_LOCK_RELEASE, +}; + /** * struct gpi_i2c_config - i2c config for peripheral * @@ -65,6 +81,7 @@ enum i2c_op { * @rx_len: receive length for buffer * @op: i2c cmd * @muli-msg: is part of multi i2c r-w msgs + * @lock_action: request lock/unlock TRE sequencing for this transfer */ struct gpi_i2c_config { u8 set_config; @@ -78,6 +95,7 @@ struct gpi_i2c_config { u32 rx_len; enum i2c_op op; bool multi_msg; + enum gpi_lock_action lock_action; }; #endif /* QCOM_GPI_DMA_H */ diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 99efe2b9b4ea9..c02be4bc8ac4c 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -200,6 +200,10 @@ struct dma_vec { * transaction is marked with DMA_PREP_REPEAT will cause the new transaction * to never be processed and stay in the issued queue forever. The flag is * ignored if the previous transaction is not a repeated transaction. + * @DMA_PREP_LOCK: tell the driver that there is a lock bit set on command + * descriptor. + * @DMA_PREP_UNLOCK: tell the driver that there is a un-lock bit set on command + * descriptor. */ enum dma_ctrl_flags { DMA_PREP_INTERRUPT = (1 << 0), @@ -212,6 +216,8 @@ enum dma_ctrl_flags { DMA_PREP_CMD = (1 << 7), DMA_PREP_REPEAT = (1 << 8), DMA_PREP_LOAD_EOT = (1 << 9), + DMA_PREP_LOCK = (1 << 10), + DMA_PREP_UNLOCK = (1 << 11), }; /** diff --git a/include/linux/dpll.h b/include/linux/dpll.h index 25be745bf41f1..562f520b23c27 100644 --- a/include/linux/dpll.h +++ b/include/linux/dpll.h @@ -163,6 +163,7 @@ struct dpll_pin_properties { u32 freq_supported_num; struct dpll_pin_frequency *freq_supported; struct dpll_pin_phase_adjust_range phase_range; + u32 phase_gran; }; #if IS_ENABLED(CONFIG_DPLL) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 61d50571ad88a..ce2db5447d221 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -18,7 +18,7 @@ * @power: The power consumed at this level (by 1 CPU or by a registered * device). It can be a total power: static and dynamic. * @cost: The cost coefficient associated with this level, used during - * energy calculation. Equal to: power * max_frequency / frequency + * energy calculation. Equal to: 10 * power * max_frequency / frequency * @flags: see "em_perf_state flags" description below. */ struct em_perf_state { diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h index 9a1eacf35d370..df8f88f63a706 100644 --- a/include/linux/etherdevice.h +++ b/include/linux/etherdevice.h @@ -42,7 +42,8 @@ extern const struct header_ops eth_header_ops; int eth_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, const void *daddr, const void *saddr, unsigned len); -int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr); +int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr); int eth_header_cache(const struct neighbour *neigh, struct hh_cache *hh, __be16 type); void eth_header_cache_update(struct hh_cache *hh, const struct net_device *dev, diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 6afb4a13b81d6..dc41722fcc9dd 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -441,10 +441,8 @@ struct f2fs_sit_block { * from node's page's beginning to get a data block address. * ex) data_blkaddr = (block_t)(nodepage_start_address + ofs_in_node) */ -#define ENTRIES_IN_SUM (F2FS_BLKSIZE / 8) #define SUMMARY_SIZE (7) /* sizeof(struct f2fs_summary) */ #define SUM_FOOTER_SIZE (5) /* sizeof(struct summary_footer) */ -#define SUM_ENTRY_SIZE (SUMMARY_SIZE * ENTRIES_IN_SUM) /* a summary entry for a block in a segment */ struct f2fs_summary { @@ -467,22 +465,6 @@ struct summary_footer { __le32 check_sum; /* summary checksum */ } __packed; -#define SUM_JOURNAL_SIZE (F2FS_BLKSIZE - SUM_FOOTER_SIZE -\ - SUM_ENTRY_SIZE) -#define NAT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct nat_journal_entry)) -#define NAT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct nat_journal_entry)) -#define SIT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct sit_journal_entry)) -#define SIT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct sit_journal_entry)) - -/* Reserved area should make size of f2fs_extra_info equals to - * that of nat_journal and sit_journal. - */ -#define EXTRA_INFO_RESERVED (SUM_JOURNAL_SIZE - 2 - 8) - /* * frequently updated NAT/SIT entries can be stored in the spare area in * summary blocks @@ -497,9 +479,16 @@ struct nat_journal_entry { struct f2fs_nat_entry ne; } __packed; +/* + * The nat_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->nat_journal_entries, nat_in_journal(), + * nid_in_journal(), MAX_NAT_JENTRIES(). + */ struct nat_journal { - struct nat_journal_entry entries[NAT_JOURNAL_ENTRIES]; - __u8 reserved[NAT_JOURNAL_RESERVED]; + struct nat_journal_entry entries[0]; } __packed; struct sit_journal_entry { @@ -507,14 +496,21 @@ struct sit_journal_entry { struct f2fs_sit_entry se; } __packed; +/* + * The sit_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->sit_journal_entries, sit_in_journal(), + * segno_in_journal(), MAX_SIT_JENTRIES(). + */ struct sit_journal { - struct sit_journal_entry entries[SIT_JOURNAL_ENTRIES]; - __u8 reserved[SIT_JOURNAL_RESERVED]; + struct sit_journal_entry entries[0]; } __packed; struct f2fs_extra_info { __le64 kbytes_written; - __u8 reserved[EXTRA_INFO_RESERVED]; + __u8 reserved[]; } __packed; struct f2fs_journal { @@ -530,11 +526,33 @@ struct f2fs_journal { }; } __packed; -/* Block-sized summary block structure */ +/* + * Block-sized summary block structure + * + * The f2fs_summary_block structure is a placeholder whose actual size varies + * depending on the use of packed_ssa. Therefore, it must always be accessed + * only through specific sets of macros and fields, and size calculations should + * use size-related macros instead of sizeof(). + * Relevant macros: sbi->sum_blocksize, sbi->entries_in_sum, + * sbi->sum_entry_size, sum_entries(), sum_journal(), sum_footer(). + * + * Summary Block Layout + * + * +-----------------------+ <--- Block Start + * | struct f2fs_summary | + * | entries[0] | + * | ... | + * | entries[N-1] | + * +-----------------------+ + * | struct f2fs_journal | + * +-----------------------+ + * | struct summary_footer | + * +-----------------------+ <--- Block End + */ struct f2fs_summary_block { - struct f2fs_summary entries[ENTRIES_IN_SUM]; - struct f2fs_journal journal; - struct summary_footer footer; + struct f2fs_summary entries[0]; + // struct f2fs_journal journal; + // struct summary_footer footer; } __packed; /* diff --git a/include/linux/fb.h b/include/linux/fb.h index 05cc251035da9..c3302d5135466 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -497,7 +497,6 @@ struct fb_info { #if defined(CONFIG_FB_DEVICE) struct device *dev; /* This is this fb device */ #endif - int class_flag; /* private sysfs flags */ #ifdef CONFIG_FB_TILEBLITTING struct fb_tile_ops *tileops; /* Tile Blitting */ #endif diff --git a/include/linux/filter.h b/include/linux/filter.h index 973233b82dc1f..cf7a0bce1bb6d 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -712,11 +712,13 @@ static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog, ret = dfunc(ctx, prog->insnsi, prog->bpf_func); duration = sched_clock() - start; - stats = this_cpu_ptr(prog->stats); - flags = u64_stats_update_begin_irqsave(&stats->syncp); - u64_stats_inc(&stats->cnt); - u64_stats_add(&stats->nsecs, duration); - u64_stats_update_end_irqrestore(&stats->syncp, flags); + if (likely(prog->stats)) { + stats = this_cpu_ptr(prog->stats); + flags = u64_stats_update_begin_irqsave(&stats->syncp); + u64_stats_inc(&stats->cnt); + u64_stats_add(&stats->nsecs, duration); + u64_stats_update_end_irqrestore(&stats->syncp, flags); + } } else { ret = dfunc(ctx, prog->insnsi, prog->bpf_func); } @@ -1373,24 +1375,13 @@ static inline bool bpf_jit_kallsyms_enabled(void) return false; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym); +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym); bool is_bpf_text_address(unsigned long addr); int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *sym); struct bpf_prog *bpf_prog_ksym_find(unsigned long addr); -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - int ret = __bpf_address_lookup(addr, size, off, sym); - - if (ret && modname) - *modname = NULL; - return ret; -} - void bpf_prog_kallsyms_add(struct bpf_prog *fp); void bpf_prog_kallsyms_del(struct bpf_prog *fp); @@ -1429,8 +1420,8 @@ static inline bool bpf_jit_kallsyms_enabled(void) } static inline int -__bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { return 0; } @@ -1451,13 +1442,6 @@ static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) return NULL; } -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - return 0; -} - static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp) { } diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index a66eb7624730c..69959032f8f51 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -188,8 +188,8 @@ struct cs_dsp { #ifdef CONFIG_DEBUG_FS struct dentry *debugfs_root; - char *wmfw_file_name; - char *bin_file_name; + const char *wmfw_file_name; + const char *bin_file_name; #endif }; diff --git a/include/linux/firmware/qcom/memory_dump.h b/include/linux/firmware/qcom/memory_dump.h new file mode 100644 index 0000000000000..ead7add4265c5 --- /dev/null +++ b/include/linux/firmware/qcom/memory_dump.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2012, 2014-2017, 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __MSM_MEMORY_DUMP_H +#define __MSM_MEMORY_DUMP_H + +#include +#include + +enum dump_client_type { + MSM_CPU_CTXT = 0, + MSM_L1_CACHE, + MSM_L2_CACHE, + MSM_OCMEM, + MSM_TMC_ETFETB, + MSM_ETM0_REG, + MSM_ETM1_REG, + MSM_ETM2_REG, + MSM_ETM3_REG, + MSM_TMC0_REG, /* TMC_ETR */ + MSM_TMC1_REG, /* TMC_ETF */ + MSM_LOG_BUF, + MSM_LOG_BUF_FIRST_IDX, + MAX_NUM_CLIENTS, +}; + +struct msm_client_dump { + enum dump_client_type id; + unsigned long start_addr; + unsigned long end_addr; +}; + +void __init reserve_memdump_cma(void); +extern struct cma *memdump_cma; + +#ifdef CONFIG_QCOM_MEMORY_DUMP +extern int msm_dump_tbl_register(struct msm_client_dump *client_entry); +#else +static inline int msm_dump_tbl_register(struct msm_client_dump *entry) +{ + return -EIO; +} +#endif + +#if IS_ENABLED(CONFIG_QCOM_MEMORY_DUMP_V2) +extern uint32_t msm_dump_table_version(void); +#else +static inline uint32_t msm_dump_table_version(void) +{ + return 0; +} +#endif + +#define MSM_DUMP_MAKE_VERSION(ma, mi) ((ma << 20) | mi) +#define MSM_DUMP_MAJOR(val) (val >> 20) +#define MSM_DUMP_MINOR(val) (val & 0xFFFFF) + + +#define MAX_NUM_ENTRIES 0x150 + +enum msm_dump_data_ids { + MSM_DUMP_DATA_CPU_CTX = 0x00, + MSM_DUMP_DATA_L1_INST_CACHE = 0x60, + MSM_DUMP_DATA_L1_DATA_CACHE = 0x80, + MSM_DUMP_DATA_ETM_REG = 0xA0, + MSM_DUMP_DATA_L2_CACHE = 0xC0, + MSM_DUMP_DATA_L3_CACHE = 0xD0, + MSM_DUMP_DATA_OCMEM = 0xE0, + MSM_DUMP_DATA_CNSS_WLAN = 0xE1, + MSM_DUMP_DATA_WIGIG = 0xE2, + MSM_DUMP_DATA_PMIC = 0xE4, + MSM_DUMP_DATA_DBGUI_REG = 0xE5, + MSM_DUMP_DATA_DCC_REG = 0xE6, + MSM_DUMP_DATA_DCC_SRAM = 0xE7, + MSM_DUMP_DATA_MISC = 0xE8, + MSM_DUMP_DATA_VSENSE = 0xE9, + MSM_DUMP_DATA_RPM = 0xEA, + MSM_DUMP_DATA_SCANDUMP = 0xEB, + MSM_DUMP_DATA_RPMH = 0xEC, + MSM_DUMP_DATA_TMC_ETF = 0xF0, + MSM_DUMP_DATA_TMC_ETF_SWAO = 0xF1, + MSM_DUMP_DATA_TMC_REG = 0x100, + MSM_DUMP_DATA_TMC_ETF_SWAO_REG = 0x102, + MSM_DUMP_DATA_LOG_BUF = 0x110, + MSM_DUMP_DATA_LOG_BUF_FIRST_IDX = 0x111, + MSM_DUMP_DATA_SCANDUMP_PER_CPU = 0x130, + MSM_DUMP_DATA_LLCC_PER_INSTANCE = 0x140, + MSM_DUMP_DATA_MAX = MAX_NUM_ENTRIES, +}; + +enum msm_dump_table_ids { + MSM_DUMP_TABLE_APPS, + MSM_DUMP_TABLE_MAX = MAX_NUM_ENTRIES, +}; + +enum msm_dump_type { + MSM_DUMP_TYPE_DATA, + MSM_DUMP_TYPE_TABLE, +}; + +struct msm_dump_data { + uint32_t version; + uint32_t magic; + char name[32]; + uint64_t addr; + uint64_t len; + uint32_t reserved; +}; + +struct msm_dump_entry { + uint32_t id; + char name[32]; + uint32_t type; + uint64_t addr; +}; + +struct dump_item { + u32 dump_id; + size_t size; + const char *name; +}; + +struct dump_table { + const struct dump_item *items; + u32 num_of_items; + phys_addr_t imem_base; + resource_size_t imem_size; +}; + +#if IS_ENABLED(CONFIG_QCOM_MEMORY_DUMP_V2) +extern int msm_dump_data_register(enum msm_dump_table_ids id, + struct msm_dump_entry *entry); +extern int msm_dump_data_register_nominidump(enum msm_dump_table_ids id, + struct msm_dump_entry *entry); +#else +static inline int msm_dump_data_register(enum msm_dump_table_ids id, + struct msm_dump_entry *entry) +{ + return -EINVAL; +} +static inline int msm_dump_data_register_nominidump(enum msm_dump_table_ids id, + struct msm_dump_entry *entry) +{ + return -EINVAL; +} +#endif + +#endif diff --git a/include/linux/firmware/qcom/qcom_scm.h b/include/linux/firmware/qcom/qcom_scm.h index a55ca771286bf..e505e9b6f49e9 100644 --- a/include/linux/firmware/qcom/qcom_scm.h +++ b/include/linux/firmware/qcom/qcom_scm.h @@ -10,10 +10,12 @@ #include #include +#include #define QCOM_SCM_VERSION(major, minor) (((major) << 16) | ((minor) & 0xFF)) #define QCOM_SCM_CPU_PWR_DOWN_L2_ON 0x0 #define QCOM_SCM_CPU_PWR_DOWN_L2_OFF 0x1 +#define QCOM_SCM_CAMERA_MAX_QOS_CNT 2 #define QCOM_SCM_HDCP_MAX_REQ_CNT 5 struct qcom_scm_hdcp_req { @@ -66,19 +68,41 @@ int qcom_scm_set_warm_boot_addr(void *entry); void qcom_scm_cpu_power_down(u32 flags); int qcom_scm_set_remote_state(u32 state, u32 id); -struct qcom_scm_pas_metadata { +struct qcom_scm_camera_qos { + u32 offset; + u32 val; +}; + +int qcom_scm_camera_update_camnoc_qos(uint32_t use_case_id, + uint32_t qos_cnt, struct qcom_scm_camera_qos *scm_buf); + +struct qcom_scm_pas_context { + struct device *dev; + u32 pas_id; + phys_addr_t mem_phys; + size_t mem_size; void *ptr; dma_addr_t phys; ssize_t size; + bool use_tzmem; }; -int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, - struct qcom_scm_pas_metadata *ctx); -void qcom_scm_pas_metadata_release(struct qcom_scm_pas_metadata *ctx); -int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size); -int qcom_scm_pas_auth_and_reset(u32 peripheral); -int qcom_scm_pas_shutdown(u32 peripheral); -bool qcom_scm_pas_supported(u32 peripheral); +struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev, + u32 pas_id, + phys_addr_t mem_phys, + size_t mem_size); +int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size, + struct qcom_scm_pas_context *ctx); +void qcom_scm_pas_metadata_release(struct qcom_scm_pas_context *ctx); +int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size); +int qcom_scm_pas_auth_and_reset(u32 pas_id); +int qcom_scm_pas_shutdown(u32 pas_id); +bool qcom_scm_pas_supported(u32 pas_id); +struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *ctx, + void *input_rt, size_t input_rt_size, + size_t *output_rt_size); + +int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx); int qcom_scm_io_readl(phys_addr_t addr, unsigned int *val); int qcom_scm_io_writel(phys_addr_t addr, unsigned int val); diff --git a/include/linux/firmware/qcom/qcom_tzmem.h b/include/linux/firmware/qcom/qcom_tzmem.h index 48ac0e5454c7f..23173e0c3dddd 100644 --- a/include/linux/firmware/qcom/qcom_tzmem.h +++ b/include/linux/firmware/qcom/qcom_tzmem.h @@ -17,11 +17,20 @@ struct qcom_tzmem_pool; * enum qcom_tzmem_policy - Policy for pool growth. */ enum qcom_tzmem_policy { - /**< Static pool, never grow above initial size. */ + /** + * @QCOM_TZMEM_POLICY_STATIC: Static pool, + * never grow above initial size. + */ QCOM_TZMEM_POLICY_STATIC = 1, - /**< When out of memory, add increment * current size of memory. */ + /** + * @QCOM_TZMEM_POLICY_MULTIPLIER: When out of memory, + * add increment * current size of memory. + */ QCOM_TZMEM_POLICY_MULTIPLIER, - /**< When out of memory add as much as is needed until max_size. */ + /** + * @QCOM_TZMEM_POLICY_ON_DEMAND: When out of memory + * add as much as is needed until max_size. + */ QCOM_TZMEM_POLICY_ON_DEMAND, }; diff --git a/include/linux/firmware/thead/thead,th1520-aon.h b/include/linux/firmware/thead/thead,th1520-aon.h index dae132b66873a..d81f5f6f5b905 100644 --- a/include/linux/firmware/thead/thead,th1520-aon.h +++ b/include/linux/firmware/thead/thead,th1520-aon.h @@ -97,80 +97,6 @@ struct th1520_aon_rpc_ack_common { #define RPC_GET_SVC_FLAG_ACK_TYPE(MESG) (((MESG)->svc & 0x40) >> 6) #define RPC_SET_SVC_FLAG_ACK_TYPE(MESG, ACK) ((MESG)->svc |= (ACK) << 6) -#define RPC_SET_BE64(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 7] = _set_data & 0xFF; \ - data[_offset + 6] = (_set_data & 0xFF00) >> 8; \ - data[_offset + 5] = (_set_data & 0xFF0000) >> 16; \ - data[_offset + 4] = (_set_data & 0xFF000000) >> 24; \ - data[_offset + 3] = (_set_data & 0xFF00000000) >> 32; \ - data[_offset + 2] = (_set_data & 0xFF0000000000) >> 40; \ - data[_offset + 1] = (_set_data & 0xFF000000000000) >> 48; \ - data[_offset + 0] = (_set_data & 0xFF00000000000000) >> 56; \ - } while (0) - -#define RPC_SET_BE32(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 3] = (_set_data) & 0xFF; \ - data[_offset + 2] = (_set_data & 0xFF00) >> 8; \ - data[_offset + 1] = (_set_data & 0xFF0000) >> 16; \ - data[_offset + 0] = (_set_data & 0xFF000000) >> 24; \ - } while (0) - -#define RPC_SET_BE16(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 1] = (_set_data) & 0xFF; \ - data[_offset + 0] = (_set_data & 0xFF00) >> 8; \ - } while (0) - -#define RPC_SET_U8(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - data[OFFSET] = (SET_DATA) & 0xFF; \ - } while (0) - -#define RPC_GET_BE64(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u32 *)(PTR) = \ - (data[_offset + 7] | data[_offset + 6] << 8 | \ - data[_offset + 5] << 16 | data[_offset + 4] << 24 | \ - data[_offset + 3] << 32 | data[_offset + 2] << 40 | \ - data[_offset + 1] << 48 | data[_offset + 0] << 56); \ - } while (0) - -#define RPC_GET_BE32(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u32 *)(PTR) = \ - (data[_offset + 3] | data[_offset + 2] << 8 | \ - data[_offset + 1] << 16 | data[_offset + 0] << 24); \ - } while (0) - -#define RPC_GET_BE16(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u16 *)(PTR) = (data[_offset + 1] | data[_offset + 0] << 8); \ - } while (0) - -#define RPC_GET_U8(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - *(u8 *)(PTR) = (data[OFFSET]); \ - } while (0) - /* * Defines for SC PM Power Mode */ diff --git a/include/linux/fs.h b/include/linux/fs.h index dd3b57cfadeeb..014cb04eefbe6 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1419,6 +1419,7 @@ extern int send_sigurg(struct file *file); #define SB_I_NOUMASK 0x00001000 /* VFS does not apply umask */ #define SB_I_NOIDMAP 0x00002000 /* No idmapped mounts on this superblock */ #define SB_I_ALLOW_HSM 0x00004000 /* Allow HSM events on this superblock */ +#define SB_I_NO_DATA_INTEGRITY 0x00008000 /* fs cannot guarantee data persistence on sync */ /* Possible states of 'frozen' field */ enum { @@ -2207,6 +2208,8 @@ struct dir_context { * INT_MAX unlimited */ int count; + /* @actor supports these flags in d_type high bits */ + unsigned int dt_flags_mask; }; /* If OR-ed with d_type, pending signals are not checked */ @@ -3676,7 +3679,7 @@ struct offset_ctx { void simple_offset_init(struct offset_ctx *octx); int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry); void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry); -int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry, +void simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry); int simple_offset_rename_exchange(struct inode *old_dir, struct dentry *old_dentry, @@ -3985,7 +3988,9 @@ static inline bool dir_emit(struct dir_context *ctx, const char *name, int namelen, u64 ino, unsigned type) { - return ctx->actor(ctx, name, namelen, ctx->pos, ino, type); + unsigned int dt_mask = S_DT_MASK | ctx->dt_flags_mask; + + return ctx->actor(ctx, name, namelen, ctx->pos, ino, type & dt_mask); } static inline bool dir_emit_dot(struct file *file, struct dir_context *ctx) { diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 07f8c309e4327..c3b1c74bdd7a4 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -87,11 +87,13 @@ struct ftrace_hash; defined(CONFIG_DYNAMIC_FTRACE) int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym); + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym); #else static inline int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { return 0; } @@ -1030,10 +1032,17 @@ static inline bool is_ftrace_trampoline(unsigned long addr) #ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifndef ftrace_graph_func -#define ftrace_graph_func ftrace_stub -#define FTRACE_OPS_GRAPH_STUB FTRACE_OPS_FL_STUB +# define ftrace_graph_func ftrace_stub +# define FTRACE_OPS_GRAPH_STUB FTRACE_OPS_FL_STUB +/* + * The function graph is called every time the function tracer is called. + * It must always test the ops hash and cannot just directly call + * the handler. + */ +# define FGRAPH_NO_DIRECT 1 #else -#define FTRACE_OPS_GRAPH_STUB 0 +# define FTRACE_OPS_GRAPH_STUB 0 +# define FGRAPH_NO_DIRECT 0 #endif #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/include/linux/ftrace_regs.h b/include/linux/ftrace_regs.h index 15627ceea9bcc..386fa48c4a957 100644 --- a/include/linux/ftrace_regs.h +++ b/include/linux/ftrace_regs.h @@ -33,6 +33,31 @@ struct ftrace_regs; #define ftrace_regs_get_frame_pointer(fregs) \ frame_pointer(&arch_ftrace_regs(fregs)->regs) +static __always_inline void +ftrace_partial_regs_update(struct ftrace_regs *fregs, struct pt_regs *regs) { } + +#else + +/* + * ftrace_partial_regs_update - update the original ftrace_regs from regs + * @fregs: The ftrace_regs to update from @regs + * @regs: The partial regs from ftrace_partial_regs() that was updated + * + * Some architectures have the partial regs living in the ftrace_regs + * structure, whereas other architectures need to make a different copy + * of the @regs. If a partial @regs is retrieved by ftrace_partial_regs() and + * if the code using @regs updates a field (like the instruction pointer or + * stack pointer) it may need to propagate that change to the original @fregs + * it retrieved the partial @regs from. Use this function to guarantee that + * update happens. + */ +static __always_inline void +ftrace_partial_regs_update(struct ftrace_regs *fregs, struct pt_regs *regs) +{ + ftrace_regs_set_instruction_pointer(fregs, instruction_pointer(regs)); + ftrace_regs_set_return_value(fregs, regs_return_value(regs)); +} + #endif /* HAVE_ARCH_FTRACE_REGS */ /* This can be overridden by the architectures */ diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index 0bd581003cd5d..60de63e46b33d 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -44,6 +44,7 @@ struct gen_pool; * @nr: The number of zeroed bits we're looking for * @data: optional additional data used by the callback * @pool: the pool being allocated from + * @start_addr: start address of memory chunk */ typedef unsigned long (*genpool_algo_t)(unsigned long *map, unsigned long size, diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 623bee335383e..b155929af5b11 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -387,7 +387,7 @@ extern void free_pages(unsigned long addr, unsigned int order); #define free_page(addr) free_pages((addr), 0) void page_alloc_init_cpuhp(void); -int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); +bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); void drain_all_pages(struct zone *zone); void drain_local_pages(struct zone *zone); diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index c4690e365ade9..4cf418a41fe43 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -444,12 +444,16 @@ struct hisi_qp_ops { int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm); }; +struct instance_backlog { + struct list_head list; + spinlock_t lock; +}; + struct hisi_qp { u32 qp_id; u16 sq_depth; u16 cq_depth; u8 alg_type; - u8 req_type; struct qm_dma qdma; void *sqe; @@ -459,7 +463,6 @@ struct hisi_qp { struct hisi_qp_status qp_status; struct hisi_qp_ops *hw_ops; - void *qp_ctx; void (*req_cb)(struct hisi_qp *qp, void *data); void (*event_cb)(struct hisi_qp *qp); @@ -468,6 +471,10 @@ struct hisi_qp { bool is_in_kernel; u16 pasid; struct uacce_queue *uacce_q; + + spinlock_t qp_lock; + struct instance_backlog backlog; + const void **msg; }; static inline int vfs_num_set(const char *val, const struct kernel_param *kp) @@ -572,7 +579,7 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool); int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps); + u8 *alg_type, int node, struct hisi_qp **qps); void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); void hisi_qm_dev_shutdown(struct pci_dev *pdev); void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 71ac78b9f834f..240cbc6764809 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -369,10 +369,8 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order); int min_order_for_split(struct folio *folio); int split_folio_to_list(struct folio *folio, struct list_head *list); -bool uniform_split_supported(struct folio *folio, unsigned int new_order, - bool warns); -bool non_uniform_split_supported(struct folio *folio, unsigned int new_order, - bool warns); +bool folio_split_supported(struct folio *folio, unsigned int new_order, + bool uniform_split, bool warns); int folio_split(struct folio *folio, unsigned int new_order, struct page *page, struct list_head *list); /* @@ -392,7 +390,7 @@ int folio_split(struct folio *folio, unsigned int new_order, struct page *page, static inline int try_folio_split_to_order(struct folio *folio, struct page *page, unsigned int new_order) { - if (!non_uniform_split_supported(folio, new_order, /* warns= */ false)) + if (!folio_split_supported(folio, new_order, false, /* warns= */ false)) return split_huge_page_to_list_to_order(&folio->page, NULL, new_order); return folio_split(folio, new_order, page, NULL); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 8e63e46b8e1f0..082b39ac34ffd 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -241,8 +241,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz); unsigned long hugetlb_mask_last_page(struct hstate *h); -int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep); +int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma); void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end); @@ -302,13 +303,17 @@ static inline struct address_space *hugetlb_folio_mapping_lock_write( return NULL; } -static inline int huge_pmd_unshare(struct mm_struct *mm, - struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +static inline int huge_pmd_unshare(struct mmu_gather *tlb, + struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { return 0; } +static inline void huge_pmd_unshare_flush(struct mmu_gather *tlb, + struct vm_area_struct *vma) +{ +} + static inline void adjust_range_if_pmd_sharing_possible( struct vm_area_struct *vma, unsigned long *start, unsigned long *end) @@ -788,6 +793,23 @@ static inline unsigned huge_page_shift(struct hstate *h) return h->order + PAGE_SHIFT; } +/** + * hugetlb_linear_page_index() - linear_page_index() but in hugetlb + * page size granularity. + * @vma: the hugetlb VMA + * @address: the virtual address within the VMA + * + * Return: the page offset within the mapping in huge page units. + */ +static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma, + unsigned long address) +{ + struct hstate *h = hstate_vma(vma); + + return ((address - vma->vm_start) >> huge_page_shift(h)) + + (vma->vm_pgoff >> huge_page_order(h)); +} + static inline bool order_is_gigantic(unsigned int order) { return order > MAX_PAGE_ORDER; @@ -1323,7 +1345,7 @@ static inline __init void hugetlb_cma_reserve(int order) #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING static inline bool hugetlb_pmd_shared(pte_t *pte) { - return page_count(virt_to_page(pte)) > 1; + return ptdesc_pmd_is_shared(virt_to_ptdesc(pte)); } #else static inline bool hugetlb_pmd_shared(pte_t *pte) diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index b424555753b11..b77bc55a4cf35 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -15,6 +15,7 @@ #include #include #include +#include /** * struct hwrng - Hardware Random Number Generator driver @@ -48,6 +49,7 @@ struct hwrng { /* internal. */ struct list_head list; struct kref ref; + struct work_struct cleanup_work; struct completion cleanup_done; struct completion dying; }; diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index ddff9102f6332..1f4679092e69d 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -3594,8 +3594,8 @@ enum ieee80211_statuscode { WLAN_STATUS_DENIED_WITH_SUGGESTED_BAND_AND_CHANNEL = 99, WLAN_STATUS_DENIED_DUE_TO_SPECTRUM_MANAGEMENT = 103, /* 802.11ai */ - WLAN_STATUS_FILS_AUTHENTICATION_FAILURE = 108, - WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 109, + WLAN_STATUS_FILS_AUTHENTICATION_FAILURE = 112, + WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 113, WLAN_STATUS_SAE_HASH_TO_ELEMENT = 126, WLAN_STATUS_SAE_PK = 127, WLAN_STATUS_DENIED_TID_TO_LINK_MAPPING = 133, diff --git a/include/linux/if_ether.h b/include/linux/if_ether.h index 61b7335aa037c..ca9afa824aa4f 100644 --- a/include/linux/if_ether.h +++ b/include/linux/if_ether.h @@ -40,7 +40,8 @@ static inline struct ethhdr *inner_eth_hdr(const struct sk_buff *skb) return (struct ethhdr *)skb_inner_mac_header(skb); } -int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr); +int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr); extern ssize_t sysfs_format_mac(char *buf, const unsigned char *addr, int len); diff --git a/include/linux/if_hsr.h b/include/linux/if_hsr.h index d7941fd880329..f4cf2dd36d193 100644 --- a/include/linux/if_hsr.h +++ b/include/linux/if_hsr.h @@ -43,6 +43,8 @@ extern bool is_hsr_master(struct net_device *dev); extern int hsr_get_version(struct net_device *dev, enum hsr_version *ver); struct net_device *hsr_get_port_ndev(struct net_device *ndev, enum hsr_port_type pt); +int hsr_get_port_type(struct net_device *hsr_dev, struct net_device *dev, + enum hsr_port_type *type); #else static inline bool is_hsr_master(struct net_device *dev) { @@ -59,6 +61,13 @@ static inline struct net_device *hsr_get_port_ndev(struct net_device *ndev, { return ERR_PTR(-EINVAL); } + +static inline int hsr_get_port_type(struct net_device *hsr_dev, + struct net_device *dev, + enum hsr_port_type *type) +{ + return -EINVAL; +} #endif /* CONFIG_HSR */ #endif /*_LINUX_IF_HSR_H_*/ diff --git a/include/linux/iio/iio-opaque.h b/include/linux/iio/iio-opaque.h index 4247497f3f8bd..b87841a355f80 100644 --- a/include/linux/iio/iio-opaque.h +++ b/include/linux/iio/iio-opaque.h @@ -14,6 +14,7 @@ * @mlock: lock used to prevent simultaneous device state changes * @mlock_key: lockdep class for iio_dev lock * @info_exist_lock: lock to prevent use during removal + * @info_exist_key: lockdep class for info_exist lock * @trig_readonly: mark the current trigger immutable * @event_interface: event chrdevs associated with interrupt lines * @attached_buffers: array of buffers statically attached by the driver @@ -47,6 +48,7 @@ struct iio_dev_opaque { struct mutex mlock; struct lock_class_key mlock_key; struct mutex info_exist_lock; + struct lock_class_key info_exist_key; bool trig_readonly; struct iio_event_interface *event_interface; struct iio_buffer **attached_buffers; diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 872ebdf0dd77a..82ab8d9a8bf57 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -816,6 +816,18 @@ static inline void *iio_device_get_drvdata(const struct iio_dev *indio_dev) #define IIO_DECLARE_DMA_BUFFER_WITH_TS(type, name, count) \ __IIO_DECLARE_BUFFER_WITH_TS(type, name, count) __aligned(IIO_DMA_MINALIGN) +/** + * IIO_DECLARE_QUATERNION() - Declare a quaternion element + * @type: element type of the individual vectors + * @name: identifier name + * + * Quaternions are a vector composed of 4 elements (W, X, Y, Z). Use this macro + * to declare a quaternion element in a struct to ensure proper alignment in + * an IIO buffer. + */ +#define IIO_DECLARE_QUATERNION(type, name) \ + type name[4] __aligned(sizeof(type) * 4) + struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv); /* The information at the returned address is guaranteed to be cacheline aligned */ diff --git a/include/linux/ima.h b/include/linux/ima.h index 8e29cb4e6a01d..abf8923f8fc51 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -69,6 +69,7 @@ static inline int ima_measure_critical_data(const char *event_label, #ifdef CONFIG_HAVE_IMA_KEXEC int __init ima_free_kexec_buffer(void); int __init ima_get_kexec_buffer(void **addr, size_t *size); +int ima_validate_range(phys_addr_t phys, size_t size); #endif #ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h index 35227d47cfc98..dc272b514a01b 100644 --- a/include/linux/indirect_call_wrapper.h +++ b/include/linux/indirect_call_wrapper.h @@ -16,22 +16,26 @@ */ #define INDIRECT_CALL_1(f, f1, ...) \ ({ \ - likely(f == f1) ? f1(__VA_ARGS__) : f(__VA_ARGS__); \ + typeof(f) __f1 = (f); \ + likely(__f1 == f1) ? f1(__VA_ARGS__) : __f1(__VA_ARGS__); \ }) #define INDIRECT_CALL_2(f, f2, f1, ...) \ ({ \ - likely(f == f2) ? f2(__VA_ARGS__) : \ - INDIRECT_CALL_1(f, f1, __VA_ARGS__); \ + typeof(f) __f2 = (f); \ + likely(__f2 == f2) ? f2(__VA_ARGS__) : \ + INDIRECT_CALL_1(__f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALL_3(f, f3, f2, f1, ...) \ ({ \ - likely(f == f3) ? f3(__VA_ARGS__) : \ - INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__); \ + typeof(f) __f3 = (f); \ + likely(__f3 == f3) ? f3(__VA_ARGS__) : \ + INDIRECT_CALL_2(__f3, f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALL_4(f, f4, f3, f2, f1, ...) \ ({ \ - likely(f == f4) ? f4(__VA_ARGS__) : \ - INDIRECT_CALL_3(f, f3, f2, f1, __VA_ARGS__); \ + typeof(f) __f4 = (f); \ + likely(__f4 == f4) ? f4(__VA_ARGS__) : \ + INDIRECT_CALL_3(__f4, f3, f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALLABLE_DECLARE(f) f diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index 5730ba6b1cfaf..dccbeb25f7014 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -38,11 +38,11 @@ struct in_device { struct ip_mc_list *mc_tomb; unsigned long mr_v1_seen; unsigned long mr_v2_seen; - unsigned long mr_maxdelay; unsigned long mr_qi; /* Query Interval */ unsigned long mr_qri; /* Query Response Interval */ unsigned char mr_qrv; /* Query Robustness Variable */ unsigned char mr_gq_running; + u32 mr_maxdelay; u32 mr_ifc_count; struct timer_list mr_gq_timer; /* general query timer */ struct timer_list mr_ifc_timer; /* interface change timer */ diff --git a/include/linux/input/adp5589.h b/include/linux/input/adp5589.h deleted file mode 100644 index 0e4742c8c81e3..0000000000000 --- a/include/linux/input/adp5589.h +++ /dev/null @@ -1,180 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Analog Devices ADP5589/ADP5585 I/O Expander and QWERTY Keypad Controller - * - * Copyright 2010-2011 Analog Devices Inc. - */ - -#ifndef _ADP5589_H -#define _ADP5589_H - -/* - * ADP5589 specific GPI and Keymap defines - */ - -#define ADP5589_KEYMAPSIZE 88 - -#define ADP5589_GPI_PIN_ROW0 97 -#define ADP5589_GPI_PIN_ROW1 98 -#define ADP5589_GPI_PIN_ROW2 99 -#define ADP5589_GPI_PIN_ROW3 100 -#define ADP5589_GPI_PIN_ROW4 101 -#define ADP5589_GPI_PIN_ROW5 102 -#define ADP5589_GPI_PIN_ROW6 103 -#define ADP5589_GPI_PIN_ROW7 104 -#define ADP5589_GPI_PIN_COL0 105 -#define ADP5589_GPI_PIN_COL1 106 -#define ADP5589_GPI_PIN_COL2 107 -#define ADP5589_GPI_PIN_COL3 108 -#define ADP5589_GPI_PIN_COL4 109 -#define ADP5589_GPI_PIN_COL5 110 -#define ADP5589_GPI_PIN_COL6 111 -#define ADP5589_GPI_PIN_COL7 112 -#define ADP5589_GPI_PIN_COL8 113 -#define ADP5589_GPI_PIN_COL9 114 -#define ADP5589_GPI_PIN_COL10 115 -#define GPI_LOGIC1 116 -#define GPI_LOGIC2 117 - -#define ADP5589_GPI_PIN_ROW_BASE ADP5589_GPI_PIN_ROW0 -#define ADP5589_GPI_PIN_ROW_END ADP5589_GPI_PIN_ROW7 -#define ADP5589_GPI_PIN_COL_BASE ADP5589_GPI_PIN_COL0 -#define ADP5589_GPI_PIN_COL_END ADP5589_GPI_PIN_COL10 - -#define ADP5589_GPI_PIN_BASE ADP5589_GPI_PIN_ROW_BASE -#define ADP5589_GPI_PIN_END ADP5589_GPI_PIN_COL_END - -#define ADP5589_GPIMAPSIZE_MAX (ADP5589_GPI_PIN_END - ADP5589_GPI_PIN_BASE + 1) - -/* - * ADP5585 specific GPI and Keymap defines - */ - -#define ADP5585_KEYMAPSIZE 30 - -#define ADP5585_GPI_PIN_ROW0 37 -#define ADP5585_GPI_PIN_ROW1 38 -#define ADP5585_GPI_PIN_ROW2 39 -#define ADP5585_GPI_PIN_ROW3 40 -#define ADP5585_GPI_PIN_ROW4 41 -#define ADP5585_GPI_PIN_ROW5 42 -#define ADP5585_GPI_PIN_COL0 43 -#define ADP5585_GPI_PIN_COL1 44 -#define ADP5585_GPI_PIN_COL2 45 -#define ADP5585_GPI_PIN_COL3 46 -#define ADP5585_GPI_PIN_COL4 47 -#define GPI_LOGIC 48 - -#define ADP5585_GPI_PIN_ROW_BASE ADP5585_GPI_PIN_ROW0 -#define ADP5585_GPI_PIN_ROW_END ADP5585_GPI_PIN_ROW5 -#define ADP5585_GPI_PIN_COL_BASE ADP5585_GPI_PIN_COL0 -#define ADP5585_GPI_PIN_COL_END ADP5585_GPI_PIN_COL4 - -#define ADP5585_GPI_PIN_BASE ADP5585_GPI_PIN_ROW_BASE -#define ADP5585_GPI_PIN_END ADP5585_GPI_PIN_COL_END - -#define ADP5585_GPIMAPSIZE_MAX (ADP5585_GPI_PIN_END - ADP5585_GPI_PIN_BASE + 1) - -struct adp5589_gpi_map { - unsigned short pin; - unsigned short sw_evt; -}; - -/* scan_cycle_time */ -#define ADP5589_SCAN_CYCLE_10ms 0 -#define ADP5589_SCAN_CYCLE_20ms 1 -#define ADP5589_SCAN_CYCLE_30ms 2 -#define ADP5589_SCAN_CYCLE_40ms 3 - -/* RESET_CFG */ -#define RESET_PULSE_WIDTH_500us 0 -#define RESET_PULSE_WIDTH_1ms 1 -#define RESET_PULSE_WIDTH_2ms 2 -#define RESET_PULSE_WIDTH_10ms 3 - -#define RESET_TRIG_TIME_0ms (0 << 2) -#define RESET_TRIG_TIME_1000ms (1 << 2) -#define RESET_TRIG_TIME_1500ms (2 << 2) -#define RESET_TRIG_TIME_2000ms (3 << 2) -#define RESET_TRIG_TIME_2500ms (4 << 2) -#define RESET_TRIG_TIME_3000ms (5 << 2) -#define RESET_TRIG_TIME_3500ms (6 << 2) -#define RESET_TRIG_TIME_4000ms (7 << 2) - -#define RESET_PASSTHRU_EN (1 << 5) -#define RESET1_POL_HIGH (1 << 6) -#define RESET1_POL_LOW (0 << 6) -#define RESET2_POL_HIGH (1 << 7) -#define RESET2_POL_LOW (0 << 7) - -/* ADP5589 Mask Bits: - * C C C C C C C C C C C | R R R R R R R R - * 1 9 8 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 - * 0 - * ---------------- BIT ------------------ - * 1 1 1 1 1 1 1 1 1 0 0 | 0 0 0 0 0 0 0 0 - * 8 7 6 5 4 3 2 1 0 9 8 | 7 6 5 4 3 2 1 0 - */ - -#define ADP_ROW(x) (1 << (x)) -#define ADP_COL(x) (1 << (x + 8)) -#define ADP5589_ROW_MASK 0xFF -#define ADP5589_COL_MASK 0xFF -#define ADP5589_COL_SHIFT 8 -#define ADP5589_MAX_ROW_NUM 7 -#define ADP5589_MAX_COL_NUM 10 - -/* ADP5585 Mask Bits: - * C C C C C | R R R R R R - * 4 3 2 1 0 | 5 4 3 2 1 0 - * - * ---- BIT -- ----------- - * 1 0 0 0 0 | 0 0 0 0 0 0 - * 0 9 8 7 6 | 5 4 3 2 1 0 - */ - -#define ADP5585_ROW_MASK 0x3F -#define ADP5585_COL_MASK 0x1F -#define ADP5585_ROW_SHIFT 0 -#define ADP5585_COL_SHIFT 6 -#define ADP5585_MAX_ROW_NUM 5 -#define ADP5585_MAX_COL_NUM 4 - -#define ADP5585_ROW(x) (1 << ((x) & ADP5585_ROW_MASK)) -#define ADP5585_COL(x) (1 << (((x) & ADP5585_COL_MASK) + ADP5585_COL_SHIFT)) - -/* Put one of these structures in i2c_board_info platform_data */ - -struct adp5589_kpad_platform_data { - unsigned keypad_en_mask; /* Keypad (Rows/Columns) enable mask */ - const unsigned short *keymap; /* Pointer to keymap */ - unsigned short keymapsize; /* Keymap size */ - bool repeat; /* Enable key repeat */ - bool en_keylock; /* Enable key lock feature (ADP5589 only)*/ - unsigned char unlock_key1; /* Unlock Key 1 (ADP5589 only) */ - unsigned char unlock_key2; /* Unlock Key 2 (ADP5589 only) */ - unsigned char unlock_timer; /* Time in seconds [0..7] between the two unlock keys 0=disable (ADP5589 only) */ - unsigned char scan_cycle_time; /* Time between consecutive scan cycles */ - unsigned char reset_cfg; /* Reset config */ - unsigned short reset1_key_1; /* Reset Key 1 */ - unsigned short reset1_key_2; /* Reset Key 2 */ - unsigned short reset1_key_3; /* Reset Key 3 */ - unsigned short reset2_key_1; /* Reset Key 1 */ - unsigned short reset2_key_2; /* Reset Key 2 */ - unsigned debounce_dis_mask; /* Disable debounce mask */ - unsigned pull_dis_mask; /* Disable all pull resistors mask */ - unsigned pullup_en_100k; /* Pull-Up 100k Enable Mask */ - unsigned pullup_en_300k; /* Pull-Up 300k Enable Mask */ - unsigned pulldown_en_300k; /* Pull-Down 300k Enable Mask */ - const struct adp5589_gpi_map *gpimap; - unsigned short gpimapsize; - const struct adp5589_gpio_platform_data *gpio_data; -}; - -struct i2c_client; /* forward declaration */ - -struct adp5589_gpio_platform_data { - int gpio_start; /* GPIO Chip base # */ -}; - -#endif diff --git a/include/linux/intel-ish-client-if.h b/include/linux/intel-ish-client-if.h index dfbf7d9d7bb5a..b235fd84f4780 100644 --- a/include/linux/intel-ish-client-if.h +++ b/include/linux/intel-ish-client-if.h @@ -87,6 +87,8 @@ bool ishtp_wait_resume(struct ishtp_device *dev); ishtp_print_log ishtp_trace_callback(struct ishtp_cl_device *cl_device); /* Get device pointer of PCI device for DMA acces */ struct device *ishtp_get_pci_device(struct ishtp_cl_device *cl_device); +/* Get the ISHTP workqueue */ +struct workqueue_struct *ishtp_get_workqueue(struct ishtp_cl_device *cl_device); struct ishtp_cl *ishtp_cl_allocate(struct ishtp_cl_device *cl_device); void ishtp_cl_free(struct ishtp_cl *cl); diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 51b6484c04934..8f1166bc3b1ce 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -215,7 +215,7 @@ static inline int __must_check devm_request_irq(struct device *dev, unsigned int irq, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id) { - return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags, + return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags | IRQF_COND_ONESHOT, devname, dev_id); } diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index c2ea6280901dc..2ee408d076b3f 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -372,6 +372,7 @@ struct io_ring_ctx { * regularly bounce b/w CPUs. */ struct { + struct io_rings __rcu *rings_rcu; struct llist_head work_llist; struct llist_head retry_llist; unsigned long check_cq; @@ -439,6 +440,9 @@ struct io_ring_ctx { struct list_head defer_list; unsigned nr_drained; + /* protected by ->completion_lock */ + unsigned nr_req_allocated; + #ifdef CONFIG_NET_RX_BUSY_POLL struct list_head napi_list; /* track busy poll napi_id */ spinlock_t napi_lock; /* napi_list lock */ @@ -451,10 +455,6 @@ struct io_ring_ctx { DECLARE_HASHTABLE(napi_ht, 4); #endif - /* protected by ->completion_lock */ - unsigned evfd_last_cq_tail; - unsigned nr_req_allocated; - /* * Protection for resize vs mmap races - both the mmap and resize * side will need to grab this lock, to prevent either side from @@ -518,6 +518,7 @@ enum { REQ_F_BL_NO_RECYCLE_BIT, REQ_F_BUFFERS_COMMIT_BIT, REQ_F_BUF_NODE_BIT, + REQ_F_BUF_MORE_BIT, REQ_F_HAS_METADATA_BIT, REQ_F_IMPORT_BUFFER_BIT, REQ_F_SQE_COPIED_BIT, @@ -603,6 +604,8 @@ enum { REQ_F_BUFFERS_COMMIT = IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT), /* buf node is valid */ REQ_F_BUF_NODE = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT), + /* incremental buffer consumption, more space available */ + REQ_F_BUF_MORE = IO_REQ_FLAG(REQ_F_BUF_MORE_BIT), /* request has read/write metadata assigned */ REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT), /* diff --git a/include/linux/iommu.h b/include/linux/iommu.h index c30d12e16473d..66e4abb2df0dc 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1134,7 +1134,9 @@ struct iommu_sva { struct iommu_mm_data { u32 pasid; + struct mm_struct *mm; struct list_head sva_domains; + struct list_head mm_list_elm; }; int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode); @@ -1615,6 +1617,7 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm); void iommu_sva_unbind_device(struct iommu_sva *handle); u32 iommu_sva_get_pasid(struct iommu_sva *handle); +void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end); #else static inline struct iommu_sva * iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) @@ -1639,6 +1642,7 @@ static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm) } static inline void mm_pasid_drop(struct mm_struct *mm) {} +static inline void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end) {} #endif /* CONFIG_IOMMU_SVA */ #ifdef CONFIG_IOMMU_IOPF diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index 43b7bb8287388..7294e4e89b797 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -271,7 +271,6 @@ struct ipv6_pinfo { struct ipv6_mc_socklist __rcu *ipv6_mc_list; struct ipv6_ac_socklist *ipv6_ac_list; - struct ipv6_fl_socklist __rcu *ipv6_fl_list; }; /* We currently use available bits from inet_sk(sk)->inet_flags, diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index d643c7c87822e..ba1ed42f8a1c2 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -253,11 +253,11 @@ static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs) static __always_inline void exit_to_user_mode(void) { instrumentation_begin(); + unwind_reset_info(); trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); instrumentation_end(); - unwind_reset_info(); user_enter_irqoff(); arch_exit_to_user_mode(); lockdep_hardirqs_on(CALLER_ADDR0); diff --git a/include/linux/irqchip.h b/include/linux/irqchip.h index d5e6024cb2a8c..bc4ddacd6ddc1 100644 --- a/include/linux/irqchip.h +++ b/include/linux/irqchip.h @@ -17,12 +17,18 @@ #include #include +typedef int (*platform_irq_probe_t)(struct platform_device *, struct device_node *); + /* Undefined on purpose */ extern of_irq_init_cb_t typecheck_irq_init_cb; +extern platform_irq_probe_t typecheck_irq_probe; #define typecheck_irq_init_cb(fn) \ (__typecheck(typecheck_irq_init_cb, &fn) ? fn : fn) +#define typecheck_irq_probe(fn) \ + (__typecheck(typecheck_irq_probe, &fn) ? fn : fn) + /* * This macro must be used by the different irqchip drivers to declare * the association between their DT compatible string and their @@ -42,7 +48,7 @@ extern int platform_irqchip_probe(struct platform_device *pdev); static const struct of_device_id drv_name##_irqchip_match_table[] = { #define IRQCHIP_MATCH(compat, fn) { .compatible = compat, \ - .data = typecheck_irq_init_cb(fn), }, + .data = typecheck_irq_probe(fn), }, #define IRQCHIP_PLATFORM_DRIVER_END(drv_name, ...) \ diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 70c0948f978eb..0225121f30138 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -394,6 +394,7 @@ #define GITS_TYPER_VLPIS (1UL << 1) #define GITS_TYPER_ITT_ENTRY_SIZE_SHIFT 4 #define GITS_TYPER_ITT_ENTRY_SIZE GENMASK_ULL(7, 4) +#define GITS_TYPER_IDBITS GENMASK_ULL(12, 8) #define GITS_TYPER_IDBITS_SHIFT 8 #define GITS_TYPER_DEVBITS_SHIFT 13 #define GITS_TYPER_DEVBITS GENMASK_ULL(17, 13) diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 43b9297fe8a75..f5eaf76198f37 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -1253,6 +1253,12 @@ struct journal_s */ struct lockdep_map j_trans_commit_map; #endif + /** + * @jbd2_trans_commit_key: + * + * "struct lock_class_key" for @j_trans_commit_map + */ + struct lock_class_key jbd2_trans_commit_key; /** * @j_fc_cleanup_callback: diff --git a/include/linux/kasan.h b/include/linux/kasan.h index d12e1a5f5a9af..864472c9bfc97 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -28,6 +28,7 @@ typedef unsigned int __bitwise kasan_vmalloc_flags_t; #define KASAN_VMALLOC_INIT ((__force kasan_vmalloc_flags_t)0x01u) #define KASAN_VMALLOC_VM_ALLOC ((__force kasan_vmalloc_flags_t)0x02u) #define KASAN_VMALLOC_PROT_NORMAL ((__force kasan_vmalloc_flags_t)0x04u) +#define KASAN_VMALLOC_KEEP_TAG ((__force kasan_vmalloc_flags_t)0x08u) #define KASAN_VMALLOC_PAGE_RANGE 0x1 /* Apply exsiting page range */ #define KASAN_VMALLOC_TLB_FLUSH 0x2 /* TLB flush */ @@ -614,6 +615,27 @@ static __always_inline void kasan_poison_vmalloc(const void *start, __kasan_poison_vmalloc(start, size); } +void __kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, + kasan_vmalloc_flags_t flags); +static __always_inline void +kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, + kasan_vmalloc_flags_t flags) +{ + if (kasan_enabled()) + __kasan_unpoison_vmap_areas(vms, nr_vms, flags); +} + +void __kasan_vrealloc(const void *start, unsigned long old_size, + unsigned long new_size); + +static __always_inline void kasan_vrealloc(const void *start, + unsigned long old_size, + unsigned long new_size) +{ + if (kasan_enabled()) + __kasan_vrealloc(start, old_size, new_size); +} + #else /* CONFIG_KASAN_VMALLOC */ static inline void kasan_populate_early_vm_area_shadow(void *start, @@ -638,6 +660,14 @@ static inline void *kasan_unpoison_vmalloc(const void *start, static inline void kasan_poison_vmalloc(const void *start, unsigned long size) { } +static __always_inline void +kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, + kasan_vmalloc_flags_t flags) +{ } + +static inline void kasan_vrealloc(const void *start, unsigned long old_size, + unsigned long new_size) { } + #endif /* CONFIG_KASAN_VMALLOC */ #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ diff --git a/include/linux/kexec.h b/include/linux/kexec.h index ff7e231b0485a..8a22bc9b8c6c8 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -530,7 +530,7 @@ extern bool kexec_file_dbg_print; #define kexec_dprintk(fmt, arg...) \ do { if (kexec_file_dbg_print) pr_info(fmt, ##arg); } while (0) -extern void *kimage_map_segment(struct kimage *image, unsigned long addr, unsigned long size); +extern void *kimage_map_segment(struct kimage *image, int idx); extern void kimage_unmap_segment(void *buffer); #else /* !CONFIG_KEXEC_CORE */ struct pt_regs; @@ -540,7 +540,7 @@ static inline void __crash_kexec(struct pt_regs *regs) { } static inline void crash_kexec(struct pt_regs *regs) { } static inline int kexec_should_crash(struct task_struct *p) { return 0; } static inline int kexec_crash_loaded(void) { return 0; } -static inline void *kimage_map_segment(struct kimage *image, unsigned long addr, unsigned long size) +static inline void *kimage_map_segment(struct kimage *image, int idx) { return NULL; } static inline void kimage_unmap_segment(void *buffer) { } #define kexec_in_progress false diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 0ad1ddbb8b996..e5822f6e7f279 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -211,6 +211,7 @@ struct kmem_obj_info; * __kfence_obj_info() - fill kmem_obj_info struct * @kpp: kmem_obj_info to be filled * @object: the object + * @slab: the slab * * Return: * * false - not a KFENCE object diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 067538fc4d585..c982694c987b9 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -17,7 +17,7 @@ #ifdef CONFIG_KSM int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, vm_flags_t *vm_flags); -vm_flags_t ksm_vma_flags(const struct mm_struct *mm, const struct file *file, +vm_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file, vm_flags_t vm_flags); int ksm_enable_merge_any(struct mm_struct *mm); int ksm_disable_merge_any(struct mm_struct *mm); @@ -103,7 +103,7 @@ bool ksm_process_mergeable(struct mm_struct *mm); #else /* !CONFIG_KSM */ -static inline vm_flags_t ksm_vma_flags(const struct mm_struct *mm, +static inline vm_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file, vm_flags_t vm_flags) { return vm_flags; diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 8d27403888ce9..68d4b31d8989e 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -7,6 +7,24 @@ struct mm_struct; +/* opaque kthread data */ +struct kthread; + +/* + * When "(p->flags & PF_KTHREAD)" is set the task is a kthread and will + * always remain a kthread. For kthreads p->worker_private always + * points to a struct kthread. For tasks that are not kthreads + * p->worker_private is used to point to other things. + * + * Return NULL for any task that is not a kthread. + */ +static inline struct kthread *tsk_is_kthread(struct task_struct *p) +{ + if (p->flags & PF_KTHREAD) + return p->worker_private; + return NULL; +} + __printf(4, 5) struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), void *data, @@ -98,8 +116,9 @@ void *kthread_probe_data(struct task_struct *k); int kthread_park(struct task_struct *k); void kthread_unpark(struct task_struct *k); void kthread_parkme(void); -void kthread_exit(long result) __noreturn; +#define kthread_exit(result) do_exit(result) void kthread_complete_and_exit(struct completion *, long) __noreturn; +void kthread_do_exit(struct kthread *, long); int kthreadd(void *unused); extern struct task_struct *kthreadd_task; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 5bd76cf394fa4..398e5695dc076 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -320,7 +320,8 @@ static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop) struct kvm_mmio_fragment { gpa_t gpa; void *data; - unsigned len; + u64 val; + unsigned int len; }; struct kvm_vcpu { @@ -1030,6 +1031,13 @@ static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) return NULL; } +static inline bool kvm_is_vcpu_creation_in_progress(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->lock); + + return kvm->created_vcpus != atomic_read(&kvm->online_vcpus); +} + void kvm_destroy_vcpus(struct kvm *kvm); int kvm_trylock_all_vcpus(struct kvm *kvm); @@ -1925,56 +1933,43 @@ enum kvm_stat_kind { struct kvm_stat_data { struct kvm *kvm; - const struct _kvm_stats_desc *desc; + const struct kvm_stats_desc *desc; enum kvm_stat_kind kind; }; -struct _kvm_stats_desc { - struct kvm_stats_desc desc; - char name[KVM_STATS_NAME_SIZE]; -}; - -#define STATS_DESC_COMMON(type, unit, base, exp, sz, bsz) \ - .flags = type | unit | base | \ - BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ - BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ - BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ - .exponent = exp, \ - .size = sz, \ +#define STATS_DESC_COMMON(type, unit, base, exp, sz, bsz) \ + .flags = type | unit | base | \ + BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ + BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ + BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ + .exponent = exp, \ + .size = sz, \ .bucket_size = bsz -#define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vm_stat, generic.stat) \ - }, \ - .name = #stat, \ - } -#define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vcpu_stat, generic.stat) \ - }, \ - .name = #stat, \ - } -#define VM_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vm_stat, stat) \ - }, \ - .name = #stat, \ - } -#define VCPU_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ - { \ - { \ - STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ - .offset = offsetof(struct kvm_vcpu_stat, stat) \ - }, \ - .name = #stat, \ - } +#define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vm_stat, generic.stat), \ + .name = #stat, \ +} +#define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vcpu_stat, generic.stat), \ + .name = #stat, \ +} +#define VM_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vm_stat, stat), \ + .name = #stat, \ +} +#define VCPU_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ +{ \ + STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ + .offset = offsetof(struct kvm_vcpu_stat, stat), \ + .name = #stat, \ +} /* SCOPE: VM, VM_GENERIC, VCPU, VCPU_GENERIC */ #define STATS_DESC(SCOPE, stat, type, unit, base, exp, sz, bsz) \ SCOPE##_STATS_DESC(stat, type, unit, base, exp, sz, bsz) @@ -2051,7 +2046,7 @@ struct _kvm_stats_desc { STATS_DESC_IBOOLEAN(VCPU_GENERIC, blocking) ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, - const struct _kvm_stats_desc *desc, + const struct kvm_stats_desc *desc, void *stats, size_t size_stats, char __user *user_buffer, size_t size, loff_t *offset); @@ -2096,9 +2091,9 @@ static inline void kvm_stats_log_hist_update(u64 *data, size_t size, u64 value) extern const struct kvm_stats_header kvm_vm_stats_header; -extern const struct _kvm_stats_desc kvm_vm_stats_desc[]; +extern const struct kvm_stats_desc kvm_vm_stats_desc[]; extern const struct kvm_stats_header kvm_vcpu_stats_header; -extern const struct _kvm_stats_desc kvm_vcpu_stats_desc[]; +extern const struct kvm_stats_desc kvm_vcpu_stats_desc[]; #ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq) diff --git a/include/linux/leds-expresswire.h b/include/linux/leds-expresswire.h index a422921f4159f..7f8c4795f69fa 100644 --- a/include/linux/leds-expresswire.h +++ b/include/linux/leds-expresswire.h @@ -30,9 +30,6 @@ struct expresswire_common_props { void expresswire_power_off(struct expresswire_common_props *props); void expresswire_enable(struct expresswire_common_props *props); -void expresswire_start(struct expresswire_common_props *props); -void expresswire_end(struct expresswire_common_props *props); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit); void expresswire_write_u8(struct expresswire_common_props *props, u8 val); #endif /* _LEDS_EXPRESSWIRE_H */ diff --git a/include/linux/libata.h b/include/linux/libata.h index 7a98de1cc995c..3b8bdea8516d0 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -899,6 +899,9 @@ struct ata_port { u64 qc_active; int nr_active_links; /* #links with active qcs */ + struct work_struct deferred_qc_work; + struct ata_queued_cmd *deferred_qc; + struct ata_link link; /* host default link */ struct ata_link *slave_link; /* see ata_slave_link_init() */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 873e510d6f8d9..133591199933a 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1789,7 +1789,7 @@ static inline void count_objcg_events(struct obj_cgroup *objcg, rcu_read_unlock(); } -bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid); +void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask); #else static inline bool mem_cgroup_kmem_disabled(void) @@ -1853,9 +1853,9 @@ static inline ino_t page_cgroup_ino(struct page *page) return 0; } -static inline bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid) +static inline void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, + nodemask_t *mask) { - return true; } #endif /* CONFIG_MEMCG */ diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 0fe96f3ab3ef0..65c732d440d2f 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -55,6 +55,7 @@ struct mempolicy { nodemask_t cpuset_mems_allowed; /* relative to these nodes */ nodemask_t user_nodemask; /* nodemask passed by user */ } w; + struct rcu_head rcu; }; /* diff --git a/include/linux/mfd/tps65219.h b/include/linux/mfd/tps65219.h index 55234e771ba73..3abf937191d0c 100644 --- a/include/linux/mfd/tps65219.h +++ b/include/linux/mfd/tps65219.h @@ -149,6 +149,8 @@ enum pmic_id { #define TPS65215_ENABLE_LDO2_EN_MASK BIT(5) #define TPS65214_ENABLE_LDO1_EN_MASK BIT(5) #define TPS65219_ENABLE_LDO4_EN_MASK BIT(6) +/* Register Unlock */ +#define TPS65214_LOCK_ACCESS_CMD 0x5a /* power ON-OFF sequence slot */ #define TPS65219_BUCKS_LDOS_SEQUENCE_OFF_SLOT_MASK GENMASK(3, 0) #define TPS65219_BUCKS_LDOS_SEQUENCE_ON_SLOT_MASK GENMASK(7, 4) diff --git a/include/linux/mfd/wm8350/core.h b/include/linux/mfd/wm8350/core.h index 5f70d3b5d1b1a..097ef4dfcdac8 100644 --- a/include/linux/mfd/wm8350/core.h +++ b/include/linux/mfd/wm8350/core.h @@ -667,7 +667,7 @@ static inline int wm8350_register_irq(struct wm8350 *wm8350, int irq, return -ENODEV; return request_threaded_irq(irq + wm8350->irq_base, NULL, - handler, flags, name, data); + handler, flags | IRQF_ONESHOT, name, data); } static inline void wm8350_free_irq(struct wm8350 *wm8350, int irq, void *data) diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 1f0ac122c3bfc..8061efd89041e 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -97,6 +97,14 @@ static inline int set_movable_ops(const struct movable_operations *ops, enum pag return -ENOSYS; } +static inline void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl) + __releases(ptl) +{ + WARN_ON_ONCE(1); + + spin_unlock(ptl); +} + #endif /* CONFIG_MIGRATION */ #ifdef CONFIG_NUMA_BALANCING diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 5405ca1038f9e..85c2b3d358ec1 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1274,12 +1274,12 @@ static inline bool mlx5_rl_is_supported(struct mlx5_core_dev *dev) static inline int mlx5_core_is_mp_slave(struct mlx5_core_dev *dev) { return MLX5_CAP_GEN(dev, affiliate_nic_vport_criteria) && - MLX5_CAP_GEN(dev, num_vhca_ports) <= 1; + MLX5_CAP_GEN_MAX(dev, num_vhca_ports) <= 1; } static inline int mlx5_core_is_mp_master(struct mlx5_core_dev *dev) { - return MLX5_CAP_GEN(dev, num_vhca_ports) > 1; + return MLX5_CAP_GEN_MAX(dev, num_vhca_ports) > 1; } static inline int mlx5_core_mp_enabled(struct mlx5_core_dev *dev) diff --git a/include/linux/mm.h b/include/linux/mm.h index 7c79b3369b82c..1e74eb7267acb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2272,10 +2272,10 @@ static inline int folio_expected_ref_count(const struct folio *folio) if (WARN_ON_ONCE(page_has_type(&folio->page) && !folio_test_hugetlb(folio))) return 0; - if (folio_test_anon(folio)) { - /* One reference per page from the swapcache. */ - ref_count += folio_test_swapcache(folio) << order; - } else { + /* One reference per page from the swapcache. */ + ref_count += folio_test_swapcache(folio) << order; + + if (!folio_test_anon(folio)) { /* One reference per page from the pagecache. */ ref_count += !!folio->mapping << order; /* One reference from PG_private. */ @@ -2947,6 +2947,7 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a #endif /* CONFIG_MMU */ enum pt_flags { + PT_kernel = PG_referenced, PT_reserved = PG_reserved, /* High bits are used for zone/node/section */ }; @@ -2972,6 +2973,46 @@ static inline bool pagetable_is_reserved(struct ptdesc *pt) return test_bit(PT_reserved, &pt->pt_flags.f); } +/** + * ptdesc_set_kernel - Mark a ptdesc used to map the kernel + * @ptdesc: The ptdesc to be marked + * + * Kernel page tables often need special handling. Set a flag so that + * the handling code knows this ptdesc will not be used for userspace. + */ +static inline void ptdesc_set_kernel(struct ptdesc *ptdesc) +{ + set_bit(PT_kernel, &ptdesc->pt_flags.f); +} + +/** + * ptdesc_clear_kernel - Mark a ptdesc as no longer used to map the kernel + * @ptdesc: The ptdesc to be unmarked + * + * Use when the ptdesc is no longer used to map the kernel and no longer + * needs special handling. + */ +static inline void ptdesc_clear_kernel(struct ptdesc *ptdesc) +{ + /* + * Note: the 'PG_referenced' bit does not strictly need to be + * cleared before freeing the page. But this is nice for + * symmetry. + */ + clear_bit(PT_kernel, &ptdesc->pt_flags.f); +} + +/** + * ptdesc_test_kernel - Check if a ptdesc is used to map the kernel + * @ptdesc: The ptdesc being tested + * + * Call to tell if the ptdesc used to map the kernel. + */ +static inline bool ptdesc_test_kernel(const struct ptdesc *ptdesc) +{ + return test_bit(PT_kernel, &ptdesc->pt_flags.f); +} + /** * pagetable_alloc - Allocate pagetables * @gfp: GFP flags @@ -2990,6 +3031,21 @@ static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int orde } #define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__)) +static inline void __pagetable_free(struct ptdesc *pt) +{ + struct page *page = ptdesc_page(pt); + + __free_pages(page, compound_order(page)); +} + +#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE +void pagetable_free_kernel(struct ptdesc *pt); +#else +static inline void pagetable_free_kernel(struct ptdesc *pt) +{ + __pagetable_free(pt); +} +#endif /** * pagetable_free - Free pagetables * @pt: The page table descriptor @@ -2999,9 +3055,12 @@ static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int orde */ static inline void pagetable_free(struct ptdesc *pt) { - struct page *page = ptdesc_page(pt); - - __free_pages(page, compound_order(page)); + if (ptdesc_test_kernel(pt)) { + ptdesc_clear_kernel(pt); + pagetable_free_kernel(pt); + } else { + __pagetable_free(pt); + } } #if defined(CONFIG_SPLIT_PTE_PTLOCKS) @@ -3079,26 +3138,21 @@ static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; } static inline void ptlock_free(struct ptdesc *ptdesc) {} #endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */ -static inline unsigned long ptdesc_nr_pages(const struct ptdesc *ptdesc) -{ - return compound_nr(ptdesc_page(ptdesc)); -} - static inline void __pagetable_ctor(struct ptdesc *ptdesc) { - pg_data_t *pgdat = NODE_DATA(memdesc_nid(ptdesc->pt_flags)); + struct folio *folio = ptdesc_folio(ptdesc); - __SetPageTable(ptdesc_page(ptdesc)); - mod_node_page_state(pgdat, NR_PAGETABLE, ptdesc_nr_pages(ptdesc)); + __folio_set_pgtable(folio); + lruvec_stat_add_folio(folio, NR_PAGETABLE); } static inline void pagetable_dtor(struct ptdesc *ptdesc) { - pg_data_t *pgdat = NODE_DATA(memdesc_nid(ptdesc->pt_flags)); + struct folio *folio = ptdesc_folio(ptdesc); ptlock_free(ptdesc); - __ClearPageTable(ptdesc_page(ptdesc)); - mod_node_page_state(pgdat, NR_PAGETABLE, -ptdesc_nr_pages(ptdesc)); + __folio_clear_pgtable(folio); + lruvec_stat_sub_folio(folio, NR_PAGETABLE); } static inline void pagetable_dtor_free(struct ptdesc *ptdesc) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 90e5790c318f0..8b1045c51e0a6 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1490,6 +1490,7 @@ static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumas struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); +void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma); extern void tlb_finish_mmu(struct mmu_gather *tlb); struct vm_fault; diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 2c9fffa58714f..95ee1f224c492 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -322,7 +322,7 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) __mmap_lock_trace_acquire_returned(mm, true, true); } -static inline int mmap_write_lock_killable(struct mm_struct *mm) +static inline int __must_check mmap_write_lock_killable(struct mm_struct *mm) { int ret; @@ -369,7 +369,7 @@ static inline void mmap_read_lock(struct mm_struct *mm) __mmap_lock_trace_acquire_returned(mm, false, true); } -static inline int mmap_read_lock_killable(struct mm_struct *mm) +static inline int __must_check mmap_read_lock_killable(struct mm_struct *mm) { int ret; @@ -379,7 +379,7 @@ static inline int mmap_read_lock_killable(struct mm_struct *mm) return ret; } -static inline bool mmap_read_trylock(struct mm_struct *mm) +static inline bool __must_check mmap_read_trylock(struct mm_struct *mm) { bool ret; diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index e0e2c265e5d10..335950f3a6c8a 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -463,6 +463,11 @@ struct mmc_host { #define MMC_CAP2_CRYPTO 0 #endif #define MMC_CAP2_ALT_GPT_TEGRA (1 << 28) /* Host with eMMC that has GPT entry at a non-standard location */ +#ifdef CONFIG_MMC_CRYPTO +#define MMC_CAP2_CRYPTO_NO_REPROG (1 << 29) /* Host handles inline crypto key reprogramming */ +#else +#define MMC_CAP2_CRYPTO_NO_REPROG 0 +#endif bool uhs2_sd_tran; /* UHS-II flag for SD_TRAN state */ bool uhs2_app_cmd; /* UHS-II flag for APP command */ @@ -486,14 +491,12 @@ struct mmc_host { struct mmc_ios ios; /* current io bus settings */ + bool claimed; /* host exclusively claimed */ + /* group bitfields together to minimize padding */ unsigned int use_spi_crc:1; - unsigned int claimed:1; /* host exclusively claimed */ unsigned int doing_init_tune:1; /* initial tuning in progress */ - unsigned int can_retune:1; /* re-tuning can be used */ unsigned int doing_retune:1; /* re-tuning in progress */ - unsigned int retune_now:1; /* do re-tuning at next req */ - unsigned int retune_paused:1; /* re-tuning is temporarily disabled */ unsigned int retune_crc_disable:1; /* don't trigger retune upon crc */ unsigned int can_dma_map_merge:1; /* merging can be used */ unsigned int vqmmc_enabled:1; /* vqmmc regulator is enabled */ @@ -508,6 +511,9 @@ struct mmc_host { int rescan_disable; /* disable card detection */ int rescan_entered; /* used with nonremovable devices */ + bool can_retune; /* re-tuning can be used */ + bool retune_now; /* do re-tuning at next req */ + bool retune_paused; /* re-tuning is temporarily disabled */ int need_retune; /* re-tuning is needed */ int hold_retune; /* hold off re-tuning */ unsigned int retune_period; /* re-tuning period in secs */ diff --git a/include/linux/module.h b/include/linux/module.h index e135cc79aceea..4decae2b16755 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -747,6 +747,15 @@ static inline void __module_get(struct module *module) __mod ? __mod->name : "kernel"; \ }) +static inline const unsigned char *module_buildid(struct module *mod) +{ +#ifdef CONFIG_STACKTRACE_BUILD_ID + return mod->build_id; +#else + return NULL; +#endif +} + /* Dereference module function descriptor */ void *dereference_module_function_descriptor(struct module *mod, void *ptr); diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 927c10d787695..1c741145e4971 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -287,7 +287,7 @@ struct spinand_device; /** * struct spinand_id - SPI NAND id structure - * @data: buffer containing the id bytes. Currently 4 bytes large, but can + * @data: buffer containing the id bytes. Currently 5 bytes large, but can * be extended if required * @len: ID length */ diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 7a01c518e5730..93e4da7046a10 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -255,6 +255,24 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start) NETIF_F_GSO_UDP_TUNNEL | \ NETIF_F_GSO_UDP_TUNNEL_CSUM) +/* virtual device features */ +#define MASTER_UPPER_DEV_VLAN_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ + NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE | \ + NETIF_F_GSO_ENCAP_ALL | \ + NETIF_F_HIGHDMA | NETIF_F_LRO) + +#define MASTER_UPPER_DEV_ENC_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ + NETIF_F_RXCSUM | NETIF_F_GSO_SOFTWARE | \ + NETIF_F_GSO_PARTIAL) + +#define MASTER_UPPER_DEV_MPLS_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ + NETIF_F_GSO_SOFTWARE) + +#define MASTER_UPPER_DEV_XFRM_FEATURES (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM | \ + NETIF_F_GSO_ESP) + +#define MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES (NETIF_F_GSO_ESP) + static inline netdev_features_t netdev_base_features(netdev_features_t features) { features &= ~NETIF_F_ONE_FOR_ALL; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index d1a687444b275..fc55157a44860 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -311,7 +311,9 @@ struct header_ops { int (*create) (struct sk_buff *skb, struct net_device *dev, unsigned short type, const void *daddr, const void *saddr, unsigned int len); - int (*parse)(const struct sk_buff *skb, unsigned char *haddr); + int (*parse)(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr); int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type); void (*cache_update)(struct hh_cache *hh, const struct net_device *dev, @@ -1815,6 +1817,8 @@ enum netdev_reg_state { * * @mpls_features: Mask of features inheritable by MPLS * @gso_partial_features: value(s) from NETIF_F_GSO\* + * @mangleid_features: Mask of features requiring MANGLEID, will be + * disabled together with the latter. * * @ifindex: interface index * @group: The group the device belongs to @@ -2203,6 +2207,7 @@ struct net_device { netdev_features_t vlan_features; netdev_features_t hw_enc_features; netdev_features_t mpls_features; + netdev_features_t mangleid_features; unsigned int min_mtu; unsigned int max_mtu; @@ -3401,6 +3406,7 @@ struct net_device *dev_get_by_index(struct net *net, int ifindex); struct net_device *__dev_get_by_index(struct net *net, int ifindex); struct net_device *netdev_get_by_index(struct net *net, int ifindex, netdevice_tracker *tracker, gfp_t gfp); +struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex); struct net_device *netdev_get_by_name(struct net *net, const char *name, netdevice_tracker *tracker, gfp_t gfp); struct net_device *netdev_get_by_flags_rcu(struct net *net, netdevice_tracker *tracker, @@ -3426,7 +3432,7 @@ static inline int dev_parse_header(const struct sk_buff *skb, if (!dev->header_ops || !dev->header_ops->parse) return 0; - return dev->header_ops->parse(skb, haddr); + return dev->header_ops->parse(skb, dev, haddr); } static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb) @@ -3548,17 +3554,49 @@ struct page_pool_bh { }; DECLARE_PER_CPU(struct page_pool_bh, system_page_pool); +#define XMIT_RECURSION_LIMIT 8 + #ifndef CONFIG_PREEMPT_RT static inline int dev_recursion_level(void) { return this_cpu_read(softnet_data.xmit.recursion); } + +static inline bool dev_xmit_recursion(void) +{ + return unlikely(__this_cpu_read(softnet_data.xmit.recursion) > + XMIT_RECURSION_LIMIT); +} + +static inline void dev_xmit_recursion_inc(void) +{ + __this_cpu_inc(softnet_data.xmit.recursion); +} + +static inline void dev_xmit_recursion_dec(void) +{ + __this_cpu_dec(softnet_data.xmit.recursion); +} #else static inline int dev_recursion_level(void) { return current->net_xmit.recursion; } +static inline bool dev_xmit_recursion(void) +{ + return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT); +} + +static inline void dev_xmit_recursion_inc(void) +{ + current->net_xmit.recursion++; +} + +static inline void dev_xmit_recursion_dec(void) +{ + current->net_xmit.recursion--; +} #endif void __netif_schedule(struct Qdisc *q); @@ -4683,7 +4721,7 @@ static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits) static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu) { spin_lock(&txq->_xmit_lock); - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, cpu); } @@ -4701,7 +4739,7 @@ static inline void __netif_tx_release(struct netdev_queue *txq) static inline void __netif_tx_lock_bh(struct netdev_queue *txq) { spin_lock_bh(&txq->_xmit_lock); - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id()); } @@ -4710,7 +4748,7 @@ static inline bool __netif_tx_trylock(struct netdev_queue *txq) bool ok = spin_trylock(&txq->_xmit_lock); if (likely(ok)) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id()); } return ok; @@ -4718,14 +4756,14 @@ static inline bool __netif_tx_trylock(struct netdev_queue *txq) static inline void __netif_tx_unlock(struct netdev_queue *txq) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, -1); spin_unlock(&txq->_xmit_lock); } static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, -1); spin_unlock_bh(&txq->_xmit_lock); } @@ -4818,6 +4856,23 @@ static inline void netif_tx_disable(struct net_device *dev) local_bh_enable(); } +#ifndef CONFIG_PREEMPT_RT +static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu) +{ + /* Other cpus might concurrently change txq->xmit_lock_owner + * to -1 or to their cpu id, but not to our id. + */ + return READ_ONCE(txq->xmit_lock_owner) == cpu; +} + +#else +static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu) +{ + return rt_mutex_owner(&txq->_xmit_lock.lock) == current; +} + +#endif + static inline void netif_addr_lock(struct net_device *dev) { unsigned char nest_level = 0; @@ -5298,12 +5353,14 @@ netdev_features_t netdev_increment_features(netdev_features_t all, static inline netdev_features_t netdev_add_tso_features(netdev_features_t features, netdev_features_t mask) { - return netdev_increment_features(features, NETIF_F_ALL_TSO, mask); + return netdev_increment_features(features, NETIF_F_ALL_TSO | + NETIF_F_ALL_FOR_ALL, mask); } int __netdev_update_features(struct net_device *dev); void netdev_update_features(struct net_device *dev); void netdev_change_features(struct net_device *dev); +void netdev_compute_master_upper_features(struct net_device *dev, bool update_header); void netif_stacked_transfer_operstate(const struct net_device *rootdev, struct net_device *dev); diff --git a/include/linux/netfilter/ipset/ip_set.h b/include/linux/netfilter/ipset/ip_set.h index e9f4f845d760a..b98331572ad29 100644 --- a/include/linux/netfilter/ipset/ip_set.h +++ b/include/linux/netfilter/ipset/ip_set.h @@ -309,7 +309,7 @@ enum { /* register and unregister set references */ extern ip_set_id_t ip_set_get_byname(struct net *net, - const char *name, struct ip_set **set); + const struct nlattr *name, struct ip_set **set); extern void ip_set_put_byindex(struct net *net, ip_set_id_t index); extern void ip_set_name_byindex(struct net *net, ip_set_id_t index, char *name); extern ip_set_id_t ip_set_nfnl_get_byindex(struct net *net, ip_set_id_t index); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 72ee7d210a744..ba17ac5bf356a 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -140,7 +140,6 @@ struct netfs_io_stream { void (*issue_write)(struct netfs_io_subrequest *subreq); /* Collection tracking */ struct list_head subrequests; /* Contributory I/O operations */ - struct netfs_io_subrequest *front; /* Op being collected */ unsigned long long collected_to; /* Position we've collected results to */ size_t transferred; /* The amount transferred from this stream */ unsigned short error; /* Aggregate error for the stream */ diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index c585939b6cd60..2cf490a3a239b 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -636,6 +636,7 @@ extern int nfs_update_folio(struct file *file, struct folio *folio, extern int nfs_sync_inode(struct inode *inode); extern int nfs_wb_all(struct inode *inode); extern int nfs_wb_folio(struct inode *inode, struct folio *folio); +extern int nfs_wb_folio_reclaim(struct inode *inode, struct folio *folio); int nfs_wb_folio_cancel(struct inode *inode, struct folio *folio); extern int nfs_commit_inode(struct inode *, int); extern struct nfs_commit_data *nfs_commitdata_alloc(void); diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index d30c0245031c0..30ac384e011a4 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -172,6 +172,11 @@ struct nfs_server { #define NFS_MOUNT_FORCE_RDIRPLUS 0x20000000 #define NFS_MOUNT_NETUNREACH_FATAL 0x40000000 + unsigned int automount_inherit; /* Properties inherited by automount */ +#define NFS_AUTOMOUNT_INHERIT_BSIZE 0x0001 +#define NFS_AUTOMOUNT_INHERIT_RSIZE 0x0002 +#define NFS_AUTOMOUNT_INHERIT_WSIZE 0x0004 + unsigned int caps; /* server capabilities */ __u64 fattr_valid; /* Valid attributes */ unsigned int rsize; /* read size */ diff --git a/include/linux/ns_common.h b/include/linux/ns_common.h index f5b68b8abb543..f3c52904343e7 100644 --- a/include/linux/ns_common.h +++ b/include/linux/ns_common.h @@ -119,6 +119,17 @@ void __ns_common_free(struct ns_common *ns); struct user_namespace *: CLONE_NEWUSER, \ struct uts_namespace *: CLONE_NEWUTS) +#define NS_COMMON_INIT(nsname, refs) \ +{ \ + .ns_type = ns_common_type(&nsname), \ + .ns_id = 0, \ + .inum = ns_init_inum(&nsname), \ + .ops = to_ns_operations(&nsname), \ + .stashed = NULL, \ + .__ns_ref = REFCOUNT_INIT(refs), \ + .ns_list_node = LIST_HEAD_INIT(nsname.ns.ns_list_node), \ +} + #define ns_common_init(__ns) \ __ns_common_init(to_ns_common(__ns), \ ns_common_type(__ns), \ @@ -133,6 +144,8 @@ void __ns_common_free(struct ns_common *ns); #define ns_common_free(__ns) __ns_common_free(to_ns_common((__ns))) +bool may_see_all_namespaces(void); + static __always_inline __must_check bool __ns_ref_put(struct ns_common *ns) { return refcount_dec_and_test(&ns->__ns_ref); diff --git a/include/linux/of.h b/include/linux/of.h index 121a288ca92df..aab51acfbbfae 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -457,8 +457,15 @@ const char *of_prop_next_string(const struct property *prop, const char *cur); bool of_console_check(const struct device_node *dn, char *name, int index); int of_map_id(const struct device_node *np, u32 id, - const char *map_name, const char *map_mask_name, - struct device_node **target, u32 *id_out); + const char *map_name, const char *cells_name, + const char *map_mask_name, + const struct device_node *filter_np, struct of_phandle_args *arg); + +int of_map_iommu_id(const struct device_node *np, u32 id, + struct of_phandle_args *arg); + +int of_map_msi_id(const struct device_node *np, u32 id, + const struct device_node *filter_np, struct of_phandle_args *arg); phys_addr_t of_dma_get_max_cpu_address(struct device_node *np); @@ -906,8 +913,23 @@ static inline void of_property_clear_flag(struct property *p, unsigned long flag } static inline int of_map_id(const struct device_node *np, u32 id, - const char *map_name, const char *map_mask_name, - struct device_node **target, u32 *id_out) + const char *map_name, const char *cells_name, + const char *map_mask_name, + const struct device_node *filter_np, + struct of_phandle_args *arg) +{ + return -EINVAL; +} + +static inline int of_map_iommu_id(const struct device_node *np, u32 id, + struct of_phandle_args *arg) +{ + return -EINVAL; +} + +static inline int of_map_msi_id(const struct device_node *np, u32 id, + const struct device_node *filter_np, + struct of_phandle_args *arg) { return -EINVAL; } diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h index b8d6c0c208760..51dadbaa3d63a 100644 --- a/include/linux/of_fdt.h +++ b/include/linux/of_fdt.h @@ -55,6 +55,15 @@ extern int of_get_flat_dt_subnode_by_name(unsigned long node, const char *uname); extern const void *of_get_flat_dt_prop(unsigned long node, const char *name, int *size); + +extern const __be32 *of_flat_dt_get_addr_size_prop(unsigned long node, + const char *name, + int *entries); +extern bool of_flat_dt_get_addr_size(unsigned long node, const char *name, + u64 *addr, u64 *size); +extern void of_flat_dt_read_addr_size(const __be32 *prop, int entry_index, + u64 *addr, u64 *size); + extern int of_flat_dt_is_compatible(unsigned long node, const char *name); extern unsigned long of_get_flat_dt_root(void); extern uint32_t of_get_flat_dt_phandle(unsigned long node); diff --git a/include/linux/pci-pwrctrl.h b/include/linux/pci-pwrctrl.h index 4aefc7901cd18..d2b83f5b4961b 100644 --- a/include/linux/pci-pwrctrl.h +++ b/include/linux/pci-pwrctrl.h @@ -31,6 +31,8 @@ struct device_link; /** * struct pci_pwrctrl - PCI device power control context. * @dev: Address of the power controlling device. + * @power_on: Callback to power on the power controlling device. + * @power_off: Callback to power off the power controlling device. * * An object of this type must be allocated by the PCI power control device and * passed to the pwrctrl subsystem to trigger a bus rescan and setup a device @@ -38,6 +40,8 @@ struct device_link; */ struct pci_pwrctrl { struct device *dev; + int (*power_on)(struct pci_pwrctrl *pwrctrl); + int (*power_off)(struct pci_pwrctrl *pwrctrl); /* private: internal use only */ struct notifier_block nb; @@ -50,5 +54,15 @@ int pci_pwrctrl_device_set_ready(struct pci_pwrctrl *pwrctrl); void pci_pwrctrl_device_unset_ready(struct pci_pwrctrl *pwrctrl); int devm_pci_pwrctrl_device_set_ready(struct device *dev, struct pci_pwrctrl *pwrctrl); - +#if IS_REACHABLE(CONFIG_PCI_PWRCTRL) +int pci_pwrctrl_create_devices(struct device *parent); +void pci_pwrctrl_destroy_devices(struct device *parent); +int pci_pwrctrl_power_on_devices(struct device *parent); +void pci_pwrctrl_power_off_devices(struct device *parent); +#else +static inline int pci_pwrctrl_create_devices(struct device *parent) { return 0; } +static void pci_pwrctrl_destroy_devices(struct device *parent) { } +static inline int pci_pwrctrl_power_on_devices(struct device *parent) { return 0; } +static void pci_pwrctrl_power_off_devices(struct device *parent) { } +#endif #endif /* __PCI_PWRCTRL_H__ */ diff --git a/include/linux/pci.h b/include/linux/pci.h index bf97d49c23cf5..05aeee8c8844a 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -406,6 +406,7 @@ struct pci_dev { user sysfs */ unsigned int clear_retrain_link:1; /* Need to clear Retrain Link bit manually */ + unsigned int no_bw_notif:1; /* BW notifications may cause issues */ unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 92ffc4373f6de..03b7c0380f71c 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -3142,6 +3142,7 @@ #define PCI_DEVICE_ID_INTEL_HDA_CML_S 0xa3f0 #define PCI_DEVICE_ID_INTEL_HDA_LNL_P 0xa828 #define PCI_DEVICE_ID_INTEL_S21152BB 0xb152 +#define PCI_DEVICE_ID_INTEL_HDA_NVL 0xd328 #define PCI_DEVICE_ID_INTEL_HDA_BMG 0xe2f7 #define PCI_DEVICE_ID_INTEL_HDA_PTL_H 0xe328 #define PCI_DEVICE_ID_INTEL_HDA_PTL 0xe428 diff --git a/include/linux/phy.h b/include/linux/phy.h index 3c7634482356e..0bc00a4cceb24 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1813,6 +1813,9 @@ static inline bool phy_is_pseudo_fixed_link(struct phy_device *phydev) return phydev->is_pseudo_fixed_link; } +phy_interface_t phy_fix_phy_mode_for_mac_delays(phy_interface_t interface, + bool mac_txid, bool mac_rxid); + int phy_save_page(struct phy_device *phydev); int phy_select_page(struct phy_device *phydev, int page); int phy_restore_page(struct phy_device *phydev, int oldpage, int ret); diff --git a/include/linux/phy/phy-mipi-dphy.h b/include/linux/phy/phy-mipi-dphy.h index 1ac128d78dfeb..f7b4ad29e6f83 100644 --- a/include/linux/phy/phy-mipi-dphy.h +++ b/include/linux/phy/phy-mipi-dphy.h @@ -6,6 +6,8 @@ #ifndef __PHY_MIPI_DPHY_H_ #define __PHY_MIPI_DPHY_H_ +#define PHY_MIPI_DPHY_MAX_DATA_LANES 4 + /** * struct phy_configure_opts_mipi_dphy - MIPI D-PHY configuration set * @@ -269,10 +271,41 @@ struct phy_configure_opts_mipi_dphy { /** * @lanes: * - * Number of active, consecutive, data lanes, starting from - * lane 0, used for the transmissions. + * Number of active data lanes used for the transmission. + * When @lane_positions is not populated, lanes are consecutive + * starting from lane 0. */ unsigned char lanes; + + /** + * @lane_positions: + * + * Array representing the physical positions of the data-lanes. + * Indexed by logical lane number. + */ + unsigned char lane_positions[PHY_MIPI_DPHY_MAX_DATA_LANES]; + + /** + * @lane_polarities: + * + * Array representing data-lane polarities. True means inverted. + * Indexed by logical lane number. + */ + bool lane_polarities[PHY_MIPI_DPHY_MAX_DATA_LANES]; + + /** + * @clock_lane_position: + * + * Physical lane number used as the clock lane. + */ + unsigned char clock_lane_position; + + /** + * @clock_lane_polarity: + * + * Clock lane polarity. True means inverted. + */ + bool clock_lane_polarity; }; int phy_mipi_dphy_get_default_config(unsigned long pixel_clock, diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h index d9245ecec71dc..4e22aa44bcd4c 100644 --- a/include/linux/pinctrl/pinconf-generic.h +++ b/include/linux/pinctrl/pinconf-generic.h @@ -235,9 +235,4 @@ static inline int pinconf_generic_dt_node_to_map_all(struct pinctrl_dev *pctldev return pinconf_generic_dt_node_to_map(pctldev, np_config, map, num_maps, PIN_MAP_TYPE_INVALID); } - -int pinconf_generic_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, - struct device_node *np, - struct pinctrl_map **map, - unsigned int *num_maps); #endif /* __LINUX_PINCTRL_PINCONF_GENERIC_H */ diff --git a/include/linux/platform_data/lp855x.h b/include/linux/platform_data/lp855x.h index ab222dd05bbc2..3b4a891acefe9 100644 --- a/include/linux/platform_data/lp855x.h +++ b/include/linux/platform_data/lp855x.h @@ -124,12 +124,12 @@ struct lp855x_rom_data { }; /** - * struct lp855x_platform_data + * struct lp855x_platform_data - lp855 platform-specific data * @name : Backlight driver name. If it is not defined, default name is set. * @device_control : value of DEVICE CONTROL register * @initial_brightness : initial value of backlight brightness * @period_ns : platform specific pwm period value. unit is nano. - Only valid when mode is PWM_BASED. + * Only valid when mode is PWM_BASED. * @size_program : total size of lp855x_rom_data * @rom_data : list of new eeprom/eprom registers */ diff --git a/include/linux/platform_data/x86/intel_pmc_ipc.h b/include/linux/platform_data/x86/intel_pmc_ipc.h index 1d34435b70016..85ea381e4a275 100644 --- a/include/linux/platform_data/x86/intel_pmc_ipc.h +++ b/include/linux/platform_data/x86/intel_pmc_ipc.h @@ -9,6 +9,7 @@ #ifndef INTEL_PMC_IPC_H #define INTEL_PMC_IPC_H #include +#include #define IPC_SOC_REGISTER_ACCESS 0xAA #define IPC_SOC_SUB_CMD_READ 0x00 @@ -48,7 +49,6 @@ static inline int intel_pmc_ipc(struct pmc_ipc_cmd *ipc_cmd, struct pmc_ipc_rbuf {.type = ACPI_TYPE_INTEGER,}, }; struct acpi_object_list arg_list = { PMC_IPCS_PARAM_COUNT, params }; - union acpi_object *obj; int status; if (!ipc_cmd || !rbuf) @@ -72,7 +72,7 @@ static inline int intel_pmc_ipc(struct pmc_ipc_cmd *ipc_cmd, struct pmc_ipc_rbuf if (ACPI_FAILURE(status)) return -ENODEV; - obj = buffer.pointer; + union acpi_object *obj __free(kfree) = buffer.pointer; if (obj && obj->type == ACPI_TYPE_PACKAGE && obj->package.count == VALID_IPC_RESPONSE) { diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 074754c23d330..bc843d76066a5 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -31,11 +31,6 @@ struct platform_device { struct resource *resource; const struct platform_device_id *id_entry; - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; /* MFD cell pointer */ struct mfd_cell *mfd_cell; diff --git a/include/linux/pm.h b/include/linux/pm.h index cc7b2dc28574c..12782f775a17e 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -677,10 +677,10 @@ struct dev_pm_info { struct list_head entry; struct completion completion; struct wakeup_source *wakeup; + bool work_in_progress; /* Owned by the PM core */ bool wakeup_path:1; bool syscore:1; bool no_pm_callbacks:1; /* Owned by the PM core */ - bool work_in_progress:1; /* Owned by the PM core */ bool smart_suspend:1; /* Owned by the PM core */ bool must_resume:1; /* Owned by the PM core */ bool may_skip_resume:1; /* Set by subsystems */ diff --git a/include/linux/psp.h b/include/linux/psp.h index 92e60aeef21e1..b337dcce1e991 100644 --- a/include/linux/psp.h +++ b/include/linux/psp.h @@ -18,6 +18,7 @@ * and should include an appropriate local definition in their source file. */ #define PSP_CMDRESP_STS GENMASK(15, 0) +#define PSP_TEE_STS_RING_BUSY 0x0000000d /* Ring already initialized */ #define PSP_CMDRESP_CMD GENMASK(23, 16) #define PSP_CMDRESP_RESERVED GENMASK(29, 24) #define PSP_CMDRESP_RECOVERY BIT(30) diff --git a/include/linux/ras.h b/include/linux/ras.h index a64182bc72ad3..468941bfe855f 100644 --- a/include/linux/ras.h +++ b/include/linux/ras.h @@ -24,8 +24,7 @@ int __init parse_cec_param(char *str); void log_non_standard_event(const guid_t *sec_type, const guid_t *fru_id, const char *fru_text, const u8 sev, const u8 *err, const u32 len); -void log_arm_hw_error(struct cper_sec_proc_arm *err); - +void log_arm_hw_error(struct cper_sec_proc_arm *err, const u8 sev); #else static inline void log_non_standard_event(const guid_t *sec_type, @@ -33,7 +32,7 @@ log_non_standard_event(const guid_t *sec_type, const u8 sev, const u8 *err, const u32 len) { return; } static inline void -log_arm_hw_error(struct cper_sec_proc_arm *err) { return; } +log_arm_hw_error(struct cper_sec_proc_arm *err, const u8 sev) { return; } #endif struct atl_err { @@ -53,4 +52,15 @@ static inline unsigned long amd_convert_umc_mca_addr_to_sys_addr(struct atl_err *err) { return -EINVAL; } #endif /* CONFIG_AMD_ATL */ +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) +#include +/* + * Include ARM-specific SMP header which provides a function mapping mpidr to + * CPU logical index. + */ +#define GET_LOGICAL_INDEX(mpidr) get_logical_index(mpidr & MPIDR_HWID_BITMASK) +#else +#define GET_LOGICAL_INDEX(mpidr) -EINVAL +#endif /* CONFIG_ARM || CONFIG_ARM64 */ + #endif /* __RAS_H__ */ diff --git a/include/linux/reboot-mode.h b/include/linux/reboot-mode.h index 4a2abb38d1d61..c68a671f6947f 100644 --- a/include/linux/reboot-mode.h +++ b/include/linux/reboot-mode.h @@ -2,14 +2,22 @@ #ifndef __REBOOT_MODE_H__ #define __REBOOT_MODE_H__ +#include +#include +#include + struct reboot_mode_driver { struct device *dev; + struct device *reboot_dev; + const char *driver_name; struct list_head head; - int (*write)(struct reboot_mode_driver *reboot, unsigned int magic); + int (*write)(struct reboot_mode_driver *reboot, u64 magic); struct notifier_block reboot_notifier; + /*Protects access to reboot mode list*/ + struct mutex rb_lock; }; -int reboot_mode_register(struct reboot_mode_driver *reboot); +int reboot_mode_register(struct reboot_mode_driver *reboot, struct fwnode_handle *fwnode); int reboot_mode_unregister(struct reboot_mode_driver *reboot); int devm_reboot_mode_register(struct device *dev, struct reboot_mode_driver *reboot); diff --git a/include/linux/regulator/pca9450.h b/include/linux/regulator/pca9450.h index 85b4fecc10d82..0df8b3c48082f 100644 --- a/include/linux/regulator/pca9450.h +++ b/include/linux/regulator/pca9450.h @@ -223,12 +223,44 @@ enum { #define IRQ_THERM_105 0x02 #define IRQ_THERM_125 0x01 +/* PCA9450_REG_PWRCTRL bits */ +#define T_ON_DEB_MASK 0xC0 +#define T_ON_DEB_120US (0 << 6) +#define T_ON_DEB_20MS (1 << 6) +#define T_ON_DEB_100MS (2 << 6) +#define T_ON_DEB_750MS (3 << 6) +#define T_OFF_DEB_MASK 0x20 +#define T_OFF_DEB_120US (0 << 5) +#define T_OFF_DEB_2MS (1 << 5) +#define T_ON_STEP_MASK 0x18 +#define T_ON_STEP_1MS (0 << 3) +#define T_ON_STEP_2MS (1 << 3) +#define T_ON_STEP_4MS (2 << 3) +#define T_ON_STEP_8MS (3 << 3) +#define T_OFF_STEP_MASK 0x06 +#define T_OFF_STEP_2MS (0 << 1) +#define T_OFF_STEP_4MS (1 << 1) +#define T_OFF_STEP_8MS (2 << 1) +#define T_OFF_STEP_16MS (3 << 1) +#define T_RESTART_MASK 0x01 +#define T_RESTART_250MS 0 +#define T_RESTART_500MS 1 + /* PCA9450_REG_RESET_CTRL bits */ #define WDOG_B_CFG_MASK 0xC0 #define WDOG_B_CFG_NONE 0x00 #define WDOG_B_CFG_WARM 0x40 #define WDOG_B_CFG_COLD_LDO12 0x80 #define WDOG_B_CFG_COLD 0xC0 +#define T_PMIC_RST_DEB_MASK 0x07 +#define T_PMIC_RST_DEB_10MS 0x00 +#define T_PMIC_RST_DEB_50MS 0x01 +#define T_PMIC_RST_DEB_100MS 0x02 +#define T_PMIC_RST_DEB_500MS 0x03 +#define T_PMIC_RST_DEB_1S 0x04 +#define T_PMIC_RST_DEB_2S 0x05 +#define T_PMIC_RST_DEB_4S 0x06 +#define T_PMIC_RST_DEB_8S 0x07 /* PCA9450_REG_CONFIG2 bits */ #define I2C_LT_MASK 0x03 diff --git a/include/linux/remoteproc_cooling.h b/include/linux/remoteproc_cooling.h new file mode 100644 index 0000000000000..721912d1a5eca --- /dev/null +++ b/include/linux/remoteproc_cooling.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Remote Processor Cooling Device + * + * Copyright (c) 2025, Qualcomm Innovation Center + */ + +#ifndef __REMOTEPROC_COOLING_H__ +#define __REMOTEPROC_COOLING_H__ + +#include + +struct device; +struct device_node; + +struct remoteproc_cooling_ops { + int (*get_max_level)(void *devdata, unsigned long *level); + int (*get_cur_level)(void *devdata, unsigned long *level); + int (*set_cur_level)(void *devdata, unsigned long level); +}; + +struct remoteproc_cdev; + +#ifdef CONFIG_REMOTEPROC_THERMAL + +struct remoteproc_cdev * +remoteproc_cooling_register(struct device_node *np, + const char *name, + const struct remoteproc_cooling_ops *ops, + void *devdata); + +void remoteproc_cooling_unregister(struct remoteproc_cdev *rproc_cdev); + +#else /* !CONFIG_REMOTEPROC_THERMAL */ + +static inline struct remoteproc_cdev * +remoteproc_cooling_register(struct device_node *np, + const char *name, + const struct remoteproc_cooling_ops *ops, + void *devdata) +{ + return ERR_PTR(-EINVAL); +} + +static inline void +remoteproc_cooling_unregister(struct remoteproc_cdev *rproc_cdev) +{ +} + +#endif /* CONFIG_REMOTEPROC_THERMAL */ + +#endif /* __REMOTEPROC_COOLING_H__ */ diff --git a/include/linux/reset.h b/include/linux/reset.h index 840d75d172f62..44f9e3415f92c 100644 --- a/include/linux/reset.h +++ b/include/linux/reset.h @@ -2,6 +2,7 @@ #ifndef _LINUX_RESET_H_ #define _LINUX_RESET_H_ +#include #include #include #include diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 876358cfe1b12..d862fa610270b 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -248,6 +248,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node); int ring_buffer_map(struct trace_buffer *buffer, int cpu, struct vm_area_struct *vma); +void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu); int ring_buffer_unmap(struct trace_buffer *buffer, int cpu); int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu); #endif /* _LINUX_RING_BUFFER_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index b469878de25c8..3e2005e9e2f0b 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -574,6 +574,7 @@ struct sched_entity { u64 deadline; u64 min_vruntime; u64 min_slice; + u64 max_slice; struct list_head group_node; unsigned char on_rq; @@ -1798,6 +1799,11 @@ static __always_inline bool is_percpu_thread(void) (current->nr_cpus_allowed == 1); } +static __always_inline bool is_user_task(struct task_struct *task) +{ + return task->mm && !(task->flags & (PF_KTHREAD | PF_USER_WORKER)); +} + /* Per-process atomic flags. */ #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 0232d983b7153..a3094379b5790 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -323,6 +323,7 @@ static inline void might_alloc(gfp_t gfp_mask) /** * memalloc_flags_save - Add a PF_* flag to current->flags, save old value + * @flags: Flags to add. * * This allows PF_* flags to be conveniently added, irrespective of current * value, and then the old version restored with memalloc_flags_restore(). diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index bbcfdf12aa6e5..45c0022b91ced 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -92,6 +92,9 @@ struct sched_domain { unsigned int nr_balance_failed; /* initialise to 0 */ /* idle_balance() stats */ + unsigned int newidle_call; + unsigned int newidle_success; + unsigned int newidle_ratio; u64 max_newidle_lb_cost; unsigned long last_decay_max_lb_cost; diff --git a/include/linux/security.h b/include/linux/security.h index 92ac3f27b9733..b64598e5d65d7 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -144,6 +144,7 @@ enum lockdown_reason { LOCKDOWN_BPF_WRITE_USER, LOCKDOWN_DBG_WRITE_KERNEL, LOCKDOWN_RTAS_ERROR_INJECTION, + LOCKDOWN_XEN_USER_ACTIONS, LOCKDOWN_INTEGRITY_MAX, LOCKDOWN_KCORE, LOCKDOWN_KPROBES, diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index 01efdce0fda07..a95b2d143d248 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -195,6 +195,7 @@ void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl); void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud, unsigned int quot); int fsl8250_handle_irq(struct uart_port *port); +void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir); int serial8250_handle_irq(struct uart_port *port, unsigned int iir); u16 serial8250_rx_chars(struct uart_8250_port *up, u16 lsr); void serial8250_read_char(struct uart_8250_port *up, u16 lsr); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index a7cc3d1f4fd11..def9cca948178 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -4301,6 +4301,18 @@ skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer) skb_headlen(skb), buffer); } +/* Variant of skb_header_pointer() where @offset is user-controlled + * and potentially negative. + */ +static inline void * __must_check +skb_header_pointer_careful(const struct sk_buff *skb, int offset, + int len, void *buffer) +{ + if (unlikely(offset < 0 && -offset > skb_headroom(skb))) + return NULL; + return skb_header_pointer(skb, offset, len, buffer); +} + static inline void * __must_check skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len) { @@ -5006,6 +5018,7 @@ static inline bool skb_has_extensions(struct sk_buff *skb) return unlikely(skb->active_extensions); } #else +static inline void __skb_ext_put(struct skb_ext *ext) {} static inline void skb_ext_put(struct sk_buff *skb) {} static inline void skb_ext_reset(struct sk_buff *skb) {} static inline void skb_ext_del(struct sk_buff *skb, int unused) {} diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 49847888c287a..829b281d6c9c2 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -97,6 +97,8 @@ struct sk_psock { struct sk_buff_head ingress_skb; struct list_head ingress_msg; spinlock_t ingress_lock; + /** @msg_tot_len: Total bytes queued in ingress_msg list. */ + u32 msg_tot_len; unsigned long state; struct list_head link; spinlock_t link_lock; @@ -141,6 +143,8 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, struct sk_msg *msg, u32 bytes); int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, int len, int flags); +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self); bool sk_msg_is_readable(struct sock *sk); static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes) @@ -319,6 +323,27 @@ static inline void sock_drop(struct sock *sk, struct sk_buff *skb) kfree_skb(skb); } +static inline u32 sk_psock_get_msg_len_nolock(struct sk_psock *psock) +{ + /* Used by ioctl to read msg_tot_len only; lock-free for performance */ + return READ_ONCE(psock->msg_tot_len); +} + +static inline void sk_psock_msg_len_add_locked(struct sk_psock *psock, int diff) +{ + /* Use WRITE_ONCE to ensure correct read in sk_psock_get_msg_len_nolock(). + * ingress_lock should be held to prevent concurrent updates to msg_tot_len + */ + WRITE_ONCE(psock->msg_tot_len, psock->msg_tot_len + diff); +} + +static inline void sk_psock_msg_len_add(struct sk_psock *psock, int diff) +{ + spin_lock_bh(&psock->ingress_lock); + sk_psock_msg_len_add_locked(psock, diff); + spin_unlock_bh(&psock->ingress_lock); +} + static inline bool sk_psock_queue_msg(struct sk_psock *psock, struct sk_msg *msg) { @@ -327,6 +352,7 @@ static inline bool sk_psock_queue_msg(struct sk_psock *psock, spin_lock_bh(&psock->ingress_lock); if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { list_add_tail(&msg->list, &psock->ingress_msg); + sk_psock_msg_len_add_locked(psock, msg->sg.size); ret = true; } else { sk_msg_free(psock->sk, msg); @@ -343,18 +369,25 @@ static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock) spin_lock_bh(&psock->ingress_lock); msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); - if (msg) + if (msg) { list_del(&msg->list); + sk_psock_msg_len_add_locked(psock, -msg->sg.size); + } spin_unlock_bh(&psock->ingress_lock); return msg; } +static inline struct sk_msg *sk_psock_peek_msg_locked(struct sk_psock *psock) +{ + return list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); +} + static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock) { struct sk_msg *msg; spin_lock_bh(&psock->ingress_lock); - msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); + msg = sk_psock_peek_msg_locked(psock); spin_unlock_bh(&psock->ingress_lock); return msg; } @@ -521,6 +554,39 @@ static inline bool sk_psock_strp_enabled(struct sk_psock *psock) return !!psock->saved_data_ready; } +/* for tcp only, sk is locked */ +static inline ssize_t sk_psock_msg_inq(struct sock *sk) +{ + struct sk_psock *psock; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + inq = sk_psock_get_msg_len_nolock(psock); + sk_psock_put(sk, psock); + } + return inq; +} + +/* for udp only, sk is not locked */ +static inline ssize_t sk_msg_first_len(struct sock *sk) +{ + struct sk_psock *psock; + struct sk_msg *msg; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + spin_lock_bh(&psock->ingress_lock); + msg = sk_psock_peek_msg_locked(psock); + if (msg) + inq = msg->sg.size; + spin_unlock_bh(&psock->ingress_lock); + sk_psock_put(sk, psock); + } + return inq; +} + #if IS_ENABLED(CONFIG_NET_SOCK_MSG) #define BPF_F_STRPARSER (1UL << 1) diff --git a/include/linux/slab.h b/include/linux/slab.h index cf443f064a667..2482992248dc9 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -1150,10 +1150,17 @@ static inline void kvfree_rcu_barrier(void) rcu_barrier(); } +static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) +{ + rcu_barrier(); +} + static inline void kfree_rcu_scheduler_running(void) { } #else void kvfree_rcu_barrier(void); +void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); + void kfree_rcu_scheduler_running(void); #endif diff --git a/include/linux/smp.h b/include/linux/smp.h index 18e9c918325e5..91d0ecf3b8d3b 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -168,6 +168,7 @@ int smp_call_function_any(const struct cpumask *mask, void kick_all_cpus_sync(void); void wake_up_all_idle_cpus(void); +bool cpus_peek_for_pending_ipi(const struct cpumask *mask); /* * Generic and arch helpers @@ -216,6 +217,10 @@ smp_call_function_any(const struct cpumask *mask, smp_call_func_t func, static inline void kick_all_cpus_sync(void) { } static inline void wake_up_all_idle_cpus(void) { } +static inline bool cpus_peek_for_pending_ipi(const struct cpumask *mask) +{ + return false; +} #define setup_max_cpus 0 diff --git a/include/linux/soc/airoha/airoha_offload.h b/include/linux/soc/airoha/airoha_offload.h index 6f66eb339b3fc..0e82f1f4d36c4 100644 --- a/include/linux/soc/airoha/airoha_offload.h +++ b/include/linux/soc/airoha/airoha_offload.h @@ -51,8 +51,8 @@ static inline void airoha_ppe_put_dev(struct airoha_ppe_dev *dev) { } -static inline int airoha_ppe_setup_tc_block_cb(struct airoha_ppe_dev *dev, - void *type_data) +static inline int airoha_ppe_dev_setup_tc_block_cb(struct airoha_ppe_dev *dev, + void *type_data) { return -EOPNOTSUPP; } @@ -70,12 +70,12 @@ static inline void airoha_ppe_dev_check_skb(struct airoha_ppe_dev *dev, #define NPU_RX1_DESC_NUM 512 /* CTRL */ -#define NPU_RX_DMA_DESC_LAST_MASK BIT(29) -#define NPU_RX_DMA_DESC_LEN_MASK GENMASK(28, 15) -#define NPU_RX_DMA_DESC_CUR_LEN_MASK GENMASK(14, 1) +#define NPU_RX_DMA_DESC_LAST_MASK BIT(27) +#define NPU_RX_DMA_DESC_LEN_MASK GENMASK(26, 14) +#define NPU_RX_DMA_DESC_CUR_LEN_MASK GENMASK(13, 1) #define NPU_RX_DMA_DESC_DONE_MASK BIT(0) /* INFO */ -#define NPU_RX_DMA_PKT_COUNT_MASK GENMASK(31, 28) +#define NPU_RX_DMA_PKT_COUNT_MASK GENMASK(31, 29) #define NPU_RX_DMA_PKT_ID_MASK GENMASK(28, 26) #define NPU_RX_DMA_SRC_PORT_MASK GENMASK(25, 21) #define NPU_RX_DMA_CRSN_MASK GENMASK(20, 16) diff --git a/include/linux/soc/mediatek/mtk_wed.h b/include/linux/soc/mediatek/mtk_wed.h index c4ff6bab176db..3fa93bd650044 100644 --- a/include/linux/soc/mediatek/mtk_wed.h +++ b/include/linux/soc/mediatek/mtk_wed.h @@ -154,6 +154,7 @@ struct mtk_wed_device { bool wcid_512; bool hw_rro; bool msi; + bool hif2; u16 token_start; unsigned int nbuf; diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index 0a984e2579fe2..326744e311cea 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -63,6 +63,7 @@ struct geni_icc_path { * @num_clk_levels: Number of valid clock levels in clk_perf_tbl * @clk_perf_tbl: Table of clock frequency input to serial engine clock * @icc_paths: Array of ICC paths for SE + * @multi_owner: True if SE is shared between multiprocessors. */ struct geni_se { void __iomem *base; @@ -72,6 +73,7 @@ struct geni_se { unsigned int num_clk_levels; unsigned long *clk_perf_tbl; struct geni_icc_path icc_paths[3]; + bool multi_owner; }; /* Common SE registers */ diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h index 7a69210a250c4..0287f9182c4d7 100644 --- a/include/linux/soc/qcom/llcc-qcom.h +++ b/include/linux/soc/qcom/llcc-qcom.h @@ -74,7 +74,14 @@ #define LLCC_CAMSRTIP 73 #define LLCC_CAMRTRF 74 #define LLCC_CAMSRTRF 75 +#define LLCC_VIDEO_APV 83 +#define LLCC_COMPUTE1 87 +#define LLCC_CPUSS_OPP 88 #define LLCC_CPUSSMPAM 89 +#define LLCC_CAM_IPE_STROV 92 +#define LLCC_CAM_OFE_STROV 93 +#define LLCC_CPUSS_HEU 94 +#define LLCC_MDM_PNG_FIXED 100 /** * struct llcc_slice_desc - Cache slice descriptor diff --git a/include/linux/soc/qcom/mdt_loader.h b/include/linux/soc/qcom/mdt_loader.h index 8ea8230579a20..7c551b98e1822 100644 --- a/include/linux/soc/qcom/mdt_loader.h +++ b/include/linux/soc/qcom/mdt_loader.h @@ -10,19 +10,19 @@ struct device; struct firmware; -struct qcom_scm_pas_metadata; +struct qcom_scm_pas_context; #if IS_ENABLED(CONFIG_QCOM_MDT_LOADER) ssize_t qcom_mdt_get_size(const struct firmware *fw); -int qcom_mdt_pas_init(struct device *dev, const struct firmware *fw, - const char *fw_name, int pas_id, phys_addr_t mem_phys, - struct qcom_scm_pas_metadata *pas_metadata_ctx); int qcom_mdt_load(struct device *dev, const struct firmware *fw, const char *fw_name, int pas_id, void *mem_region, phys_addr_t mem_phys, size_t mem_size, phys_addr_t *reloc_base); +int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx, const struct firmware *fw, + const char *firmware, phys_addr_t *reloc_base); + int qcom_mdt_load_no_init(struct device *dev, const struct firmware *fw, const char *fw_name, void *mem_region, phys_addr_t mem_phys, size_t mem_size, @@ -37,13 +37,6 @@ static inline ssize_t qcom_mdt_get_size(const struct firmware *fw) return -ENODEV; } -static inline int qcom_mdt_pas_init(struct device *dev, const struct firmware *fw, - const char *fw_name, int pas_id, phys_addr_t mem_phys, - struct qcom_scm_pas_metadata *pas_metadata_ctx) -{ - return -ENODEV; -} - static inline int qcom_mdt_load(struct device *dev, const struct firmware *fw, const char *fw_name, int pas_id, void *mem_region, phys_addr_t mem_phys, @@ -52,6 +45,13 @@ static inline int qcom_mdt_load(struct device *dev, const struct firmware *fw, return -ENODEV; } +static inline int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx, + const struct firmware *fw, const char *firmware, + phys_addr_t *reloc_base) +{ + return -ENODEV; +} + static inline int qcom_mdt_load_no_init(struct device *dev, const struct firmware *fw, const char *fw_name, void *mem_region, diff --git a/include/linux/soc/qcom/pdr.h b/include/linux/soc/qcom/pdr.h index 83a8ea612e69a..2b7691e47c2a9 100644 --- a/include/linux/soc/qcom/pdr.h +++ b/include/linux/soc/qcom/pdr.h @@ -5,6 +5,7 @@ #include #define SERVREG_NAME_LENGTH 64 +#define SERVREG_PFR_LENGTH 256 struct pdr_service; struct pdr_handle; diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index 1ed8b1b16bc90..f052e241736c4 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,6 +8,7 @@ #define __QCOM_UBWC_H__ #include +#include #include struct qcom_ubwc_cfg_data { @@ -52,6 +53,7 @@ struct qcom_ubwc_cfg_data { #define UBWC_4_0 0x40000000 #define UBWC_4_3 0x40030000 #define UBWC_5_0 0x50000000 +#define UBWC_6_0 0x60000000 #if IS_ENABLED(CONFIG_QCOM_UBWC_CONFIG) const struct qcom_ubwc_cfg_data *qcom_ubwc_config_get_data(void); diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index cb2c2df310899..fe9dd430cc03a 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -156,10 +156,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, * @modalias: Name of the driver to use with this device, or an alias * for that name. This appears in the sysfs "modalias" attribute * for driver coldplugging, and in uevents used for hotplugging - * @driver_override: If the name of a driver is written to this attribute, then - * the device will bind to the named driver and only the named driver. - * Do not set directly, because core frees it; use driver_set_override() to - * set or clear it. * @pcpu_statistics: statistics for the spi_device * @word_delay: delay to be inserted between consecutive * words of a transfer @@ -217,7 +213,6 @@ struct spi_device { void *controller_state; void *controller_data; char modalias[SPI_NAME_SIZE]; - const char *driver_override; /* The statistics */ struct spi_statistics __percpu *pcpu_statistics; diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index fa1318bac06c4..7e989d0edead2 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -171,7 +171,13 @@ struct dwmac4_addrs { u32 mtl_low_cred_offset; }; -#define STMMAC_FLAG_HAS_INTEGRATED_PCS BIT(0) +enum dwmac_core_type { + DWMAC_CORE_MAC100, + DWMAC_CORE_GMAC, + DWMAC_CORE_GMAC4, + DWMAC_CORE_XGMAC, +}; + #define STMMAC_FLAG_SPH_DISABLE BIT(1) #define STMMAC_FLAG_USE_PHY_WOL BIT(2) #define STMMAC_FLAG_HAS_SUN8I BIT(3) @@ -187,6 +193,7 @@ struct dwmac4_addrs { #define STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY BIT(13) struct plat_stmmacenet_data { + enum dwmac_core_type core_type; int bus_id; int phy_addr; /* MAC ----- optional PCS ----- SerDes ----- optional PHY ----- Media @@ -220,7 +227,6 @@ struct plat_stmmacenet_data { struct stmmac_dma_cfg *dma_cfg; struct stmmac_safety_feature_cfg *safety_feat_cfg; int clk_csr; - int has_gmac; int enh_desc; int tx_coe; int rx_coe; @@ -283,16 +289,13 @@ struct plat_stmmacenet_data { struct reset_control *stmmac_rst; struct reset_control *stmmac_ahb_rst; struct stmmac_axi *axi; - int has_gmac4; int rss_en; int mac_port_sel_speed; - int has_xgmac; u8 vlan_fail_q; struct pci_dev *pdev; int int_snapshot_num; int msi_mac_vec; int msi_wol_vec; - int msi_lpi_vec; int msi_sfty_ce_vec; int msi_sfty_ue_vec; int msi_rx_base_vec; diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h index 66ca3ece951ab..a5ab75d2db044 100644 --- a/include/linux/sunrpc/xdrgen/_builtins.h +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -188,12 +188,10 @@ xdrgen_decode_string(struct xdr_stream *xdr, string *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (unsigned char *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (unsigned char *)p; ptr->len = len; return true; } @@ -219,12 +217,10 @@ xdrgen_decode_opaque(struct xdr_stream *xdr, opaque *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (u8 *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (u8 *)p; ptr->len = len; return true; } diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 64ea151a7ae39..a73c5f14b5912 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -487,15 +487,29 @@ static inline int pte_none_mostly(pte_t pte) return pte_none(pte) || is_pte_marker(pte); } -static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) +static inline void swap_entry_migration_sync(swp_entry_t entry, + struct folio *folio) { - struct page *p = pfn_to_page(swp_offset_pfn(entry)); + /* + * Ensure we do not race with split, which might alter tail pages into new + * folios and thus result in observing an unlocked folio. + * This matches the write barrier in __split_folio_to_order(). + */ + smp_rmb(); /* * Any use of migration entries may only occur while the * corresponding page is locked */ - BUG_ON(is_migration_entry(entry) && !PageLocked(p)); + BUG_ON(!folio_test_locked(folio)); +} + +static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) +{ + struct page *p = pfn_to_page(swp_offset_pfn(entry)); + + if (is_migration_entry(entry)) + swap_entry_migration_sync(entry, page_folio(p)); return p; } @@ -504,11 +518,8 @@ static inline struct folio *pfn_swap_entry_folio(swp_entry_t entry) { struct folio *folio = pfn_folio(swp_offset_pfn(entry)); - /* - * Any use of migration entries may only occur while the - * corresponding folio is locked - */ - BUG_ON(is_migration_entry(entry) && !folio_test_locked(folio)); + if (is_migration_entry(entry)) + swap_entry_migration_sync(entry, folio); return folio; } diff --git a/include/linux/textsearch.h b/include/linux/textsearch.h index 6673e4d4ac2e1..4933777404d61 100644 --- a/include/linux/textsearch.h +++ b/include/linux/textsearch.h @@ -35,6 +35,7 @@ struct ts_state * @get_pattern: return head of pattern * @get_pattern_len: return length of pattern * @owner: module reference to algorithm + * @list: list to search */ struct ts_ops { diff --git a/include/linux/tnum.h b/include/linux/tnum.h index c52b862dad45b..ca2cfec8de08a 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -63,6 +63,11 @@ struct tnum tnum_union(struct tnum t1, struct tnum t2); /* Return @a with all but the lowest @size bytes cleared */ struct tnum tnum_cast(struct tnum a, u8 size); +/* Swap the bytes of a tnum */ +struct tnum tnum_bswap16(struct tnum a); +struct tnum tnum_bswap32(struct tnum a); +struct tnum tnum_bswap64(struct tnum a); + /* Returns true if @a is a known constant */ static inline bool tnum_is_const(struct tnum a) { @@ -126,4 +131,7 @@ static inline bool tnum_subreg_is_const(struct tnum a) return !(tnum_subreg(a)).mask; } +/* Returns the smallest member of t larger than z */ +u64 tnum_step(struct tnum t, u64 z); + #endif /* _LINUX_TNUM_H */ diff --git a/include/linux/tpm.h b/include/linux/tpm.h index dc0338a783f37..5d5c926a71119 100644 --- a/include/linux/tpm.h +++ b/include/linux/tpm.h @@ -26,7 +26,9 @@ #include #define TPM_DIGEST_SIZE 20 /* Max TPM v1.2 PCR size */ -#define TPM_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE + +#define TPM2_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE +#define TPM2_MAX_PCR_BANKS 8 struct tpm_chip; struct trusted_key_payload; @@ -68,7 +70,7 @@ enum tpm2_curves { struct tpm_digest { u16 alg_id; - u8 digest[TPM_MAX_DIGEST_SIZE]; + u8 digest[TPM2_MAX_DIGEST_SIZE]; } __packed; struct tpm_bank_info { @@ -189,7 +191,7 @@ struct tpm_chip { unsigned int groups_cnt; u32 nr_allocated_banks; - struct tpm_bank_info *allocated_banks; + struct tpm_bank_info allocated_banks[TPM2_MAX_PCR_BANKS]; #ifdef CONFIG_ACPI acpi_handle acpi_dev_handle; char ppi_version[TPM_PPI_VERSION_LEN + 1]; @@ -524,8 +526,8 @@ static inline struct tpm2_auth *tpm2_chip_auth(struct tpm_chip *chip) #endif } -void tpm_buf_append_name(struct tpm_chip *chip, struct tpm_buf *buf, - u32 handle, u8 *name); +int tpm_buf_append_name(struct tpm_chip *chip, struct tpm_buf *buf, + u32 handle, u8 *name); void tpm_buf_append_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf, u8 attributes, u8 *passphrase, int passphraselen); @@ -558,7 +560,7 @@ static inline void tpm_buf_append_hmac_session_opt(struct tpm_chip *chip, #ifdef CONFIG_TCG_TPM2_HMAC int tpm2_start_auth_session(struct tpm_chip *chip); -void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf); +int tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf); int tpm_buf_check_hmac_response(struct tpm_chip *chip, struct tpm_buf *buf, int rc); void tpm2_end_auth_session(struct tpm_chip *chip); @@ -572,10 +574,13 @@ static inline int tpm2_start_auth_session(struct tpm_chip *chip) static inline void tpm2_end_auth_session(struct tpm_chip *chip) { } -static inline void tpm_buf_fill_hmac_session(struct tpm_chip *chip, - struct tpm_buf *buf) + +static inline int tpm_buf_fill_hmac_session(struct tpm_chip *chip, + struct tpm_buf *buf) { + return 0; } + static inline int tpm_buf_check_hmac_response(struct tpm_chip *chip, struct tpm_buf *buf, int rc) diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 04307a19cde30..64ada9cc38864 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -682,6 +682,11 @@ static inline void hist_poll_wakeup(void) #define hist_poll_wait(file, wait) \ poll_wait(file, &hist_poll_wq, wait) + +#else +static inline void hist_poll_wakeup(void) +{ +} #endif #define __TRACE_EVENT_FLAGS(name, value) \ diff --git a/include/linux/trace_recursion.h b/include/linux/trace_recursion.h index ae04054a1be36..e6ca052b2a85a 100644 --- a/include/linux/trace_recursion.h +++ b/include/linux/trace_recursion.h @@ -34,6 +34,13 @@ enum { TRACE_INTERNAL_SIRQ_BIT, TRACE_INTERNAL_TRANSITION_BIT, + /* Internal event use recursion bits */ + TRACE_INTERNAL_EVENT_BIT, + TRACE_INTERNAL_EVENT_NMI_BIT, + TRACE_INTERNAL_EVENT_IRQ_BIT, + TRACE_INTERNAL_EVENT_SIRQ_BIT, + TRACE_INTERNAL_EVENT_TRANSITION_BIT, + TRACE_BRANCH_BIT, /* * Abuse of the trace_recursion. @@ -58,6 +65,8 @@ enum { #define TRACE_LIST_START TRACE_INTERNAL_BIT +#define TRACE_EVENT_START TRACE_INTERNAL_EVENT_BIT + #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1) /* diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h index 457879938fc19..3366090a86bd2 100644 --- a/include/linux/u64_stats_sync.h +++ b/include/linux/u64_stats_sync.h @@ -89,6 +89,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) local64_add(val, &p->v); } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + local64_sub(val, &p->v); +} + static inline void u64_stats_inc(u64_stats_t *p) { local64_inc(&p->v); @@ -130,6 +135,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) p->v += val; } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + p->v -= val; +} + static inline void u64_stats_inc(u64_stats_t *p) { p->v++; diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 1beb5b395d81d..7657904c8db9c 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -320,16 +320,21 @@ static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size) #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */ -#ifndef ARCH_HAS_NOCACHE_UACCESS +#ifndef ARCH_HAS_NONTEMPORAL_UACCESS static inline __must_check unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, +copy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n) { + if (can_do_masked_user_access()) + from = mask_user_address(from); + else + if (!access_ok(from, n)) + return n; return __copy_from_user_inatomic(to, from, n); } -#endif /* ARCH_HAS_NOCACHE_UACCESS */ +#endif /* ARCH_HAS_NONTEMPORAL_UACCESS */ extern __must_check int check_zeroed_user(const void __user *from, size_t size); diff --git a/include/linux/unwind_user_types.h b/include/linux/unwind_user_types.h index a449f15be8902..412729a269bcc 100644 --- a/include/linux/unwind_user_types.h +++ b/include/linux/unwind_user_types.h @@ -36,8 +36,10 @@ struct unwind_user_state { unsigned long ip; unsigned long sp; unsigned long fp; + unsigned int ws; enum unwind_user_type current_type; unsigned int available_types; + bool topmost; bool done; }; diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index ee3d36eda45dd..f548fea2adec8 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -242,6 +242,7 @@ extern void arch_uprobe_clear_state(struct mm_struct *mm); extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); +extern unsigned long arch_uprobe_get_xol_area(void); #else /* !CONFIG_UPROBES */ struct uprobes_state { }; diff --git a/include/linux/usb.h b/include/linux/usb.h index e85105939af8e..2511f1e5b114d 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -21,6 +21,7 @@ #include /* for struct completion */ #include /* for current && schedule_timeout */ #include /* for struct mutex */ +#include /* for spinlock_t */ #include /* for runtime PM */ struct usb_device; @@ -636,8 +637,9 @@ struct usb3_lpm_parameters { * @do_remote_wakeup: remote wakeup should be enabled * @reset_resume: needs reset instead of resume * @port_is_suspended: the upstream port is suspended (L2 or U3) - * @offload_at_suspend: offload activities during suspend is enabled. + * @offload_pm_locked: prevents offload_usage changes during PM transitions. * @offload_usage: number of offload activities happening on this usb device. + * @offload_lock: protects offload_usage and offload_pm_locked * @slot_id: Slot ID assigned by xHCI * @l1_params: best effor service latency for USB2 L1 LPM state, and L1 timeout. * @u1_params: exit latencies for USB3 U1 LPM state, and hub-initiated timeout. @@ -726,8 +728,9 @@ struct usb_device { unsigned do_remote_wakeup:1; unsigned reset_resume:1; unsigned port_is_suspended:1; - unsigned offload_at_suspend:1; + unsigned offload_pm_locked:1; int offload_usage; + spinlock_t offload_lock; enum usb_link_tunnel_mode tunnel_mode; struct device_link *usb4_link; @@ -849,6 +852,7 @@ static inline void usb_mark_last_busy(struct usb_device *udev) int usb_offload_get(struct usb_device *udev); int usb_offload_put(struct usb_device *udev); bool usb_offload_check(struct usb_device *udev); +void usb_offload_set_pm_locked(struct usb_device *udev, bool locked); #else static inline int usb_offload_get(struct usb_device *udev) @@ -857,6 +861,8 @@ static inline int usb_offload_put(struct usb_device *udev) { return 0; } static inline bool usb_offload_check(struct usb_device *udev) { return false; } +static inline void usb_offload_set_pm_locked(struct usb_device *udev, bool locked) +{ } #endif extern int usb_disable_lpm(struct usb_device *udev); @@ -1863,14 +1869,18 @@ void usb_free_noncoherent(struct usb_device *dev, size_t size, * SYNCHRONOUS CALL SUPPORT * *-------------------------------------------------------------------*/ +/* Maximum value allowed for timeout in synchronous routines below */ +#define USB_MAX_SYNCHRONOUS_TIMEOUT 60000 /* ms */ + extern int usb_control_msg(struct usb_device *dev, unsigned int pipe, __u8 request, __u8 requesttype, __u16 value, __u16 index, void *data, __u16 size, int timeout); extern int usb_interrupt_msg(struct usb_device *usb_dev, unsigned int pipe, void *data, int len, int *actual_length, int timeout); extern int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe, - void *data, int len, int *actual_length, - int timeout); + void *data, int len, int *actual_length, int timeout); +extern int usb_bulk_msg_killable(struct usb_device *usb_dev, unsigned int pipe, + void *data, int len, int *actual_length, int timeout); /* wrappers around usb_control_msg() for the most common standard requests */ int usb_control_msg_send(struct usb_device *dev, __u8 endpoint, __u8 request, diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index 59409c1fc3dee..b3cc7beab4a3c 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -75,4 +75,10 @@ /* short SET_ADDRESS request timeout */ #define USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT BIT(16) +/* skip BOS descriptor request */ +#define USB_QUIRK_NO_BOS BIT(17) + +/* Device claims zero configurations, forcing to 1 */ +#define USB_QUIRK_FORCE_ONE_CONFIG BIT(18) + #endif /* __LINUX_USB_QUIRKS_H */ diff --git a/include/linux/usb/r8152.h b/include/linux/usb/r8152.h index 2ca60828f28bb..1502b2a355f98 100644 --- a/include/linux/usb/r8152.h +++ b/include/linux/usb/r8152.h @@ -32,6 +32,7 @@ #define VENDOR_ID_DLINK 0x2001 #define VENDOR_ID_DELL 0x413c #define VENDOR_ID_ASUS 0x0b05 +#define VENDOR_ID_TRENDNET 0x20f4 #if IS_REACHABLE(CONFIG_USB_RTL8152) extern u8 rtl8152_get_version(struct usb_interface *intf); diff --git a/include/linux/usb/usbnet.h b/include/linux/usb/usbnet.h index a2d54122823da..bef88ba522164 100644 --- a/include/linux/usb/usbnet.h +++ b/include/linux/usb/usbnet.h @@ -130,6 +130,7 @@ struct driver_info { #define FLAG_MULTI_PACKET 0x2000 #define FLAG_RX_ASSEMBLE 0x4000 /* rx packets may span >1 frames */ #define FLAG_NOARP 0x8000 /* device can't do ARP */ +#define FLAG_NOMAXMTU 0x10000 /* allow max_mtu above hard_mtu */ /* init device ... can sleep, or cause probe() failure */ int (*bind)(struct usbnet *, struct usb_interface *); diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index f541044e42a2a..7aa29428982aa 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,11 @@ struct vfio_pci_core_device; struct vfio_pci_region; +struct vfio_pci_eventfd { + struct eventfd_ctx *ctx; + struct rcu_head rcu; +}; + struct vfio_pci_regops { ssize_t (*rw)(struct vfio_pci_core_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite); @@ -83,8 +89,8 @@ struct vfio_pci_core_device { struct pci_saved_state *pci_saved_state; struct pci_saved_state *pm_save; int ioeventfds_nr; - struct eventfd_ctx *err_trigger; - struct eventfd_ctx *req_trigger; + struct vfio_pci_eventfd __rcu *err_trigger; + struct vfio_pci_eventfd __rcu *req_trigger; struct eventfd_ctx *pm_wake_eventfd_ctx; struct list_head dummy_resources_list; struct mutex ioeventfds_lock; @@ -96,6 +102,13 @@ struct vfio_pci_core_device { struct rw_semaphore memory_lock; }; +enum vfio_pci_io_width { + VFIO_PCI_IO_WIDTH_1 = 1, + VFIO_PCI_IO_WIDTH_2 = 2, + VFIO_PCI_IO_WIDTH_4 = 4, + VFIO_PCI_IO_WIDTH_8 = 8, +}; + /* Will be exported for vfio pci drivers usage */ int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, unsigned int type, unsigned int subtype, @@ -133,7 +146,8 @@ pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, void __iomem *io, char __user *buf, loff_t off, size_t count, size_t x_start, - size_t x_end, bool iswrite); + size_t x_end, bool iswrite, + enum vfio_pci_io_width max_width); bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, loff_t reg_start, size_t reg_cnt, loff_t *buf_offset, diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 96c66126c0741..132a474e59140 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -177,7 +177,7 @@ struct virtio_device { union virtio_map vmap; #ifdef CONFIG_VIRTIO_DEBUG struct dentry *debugfs_dir; - u64 debugfs_filter_features[VIRTIO_FEATURES_DWORDS]; + u64 debugfs_filter_features[VIRTIO_FEATURES_U64S]; #endif }; diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 16001e9f9b391..69f84ea85d71a 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -24,7 +24,7 @@ typedef void vq_callback_t(struct virtqueue *); * a virtqueue unused by the driver. * @callback: A callback to invoke on a used buffer notification. * NULL for a virtqueue that does not need a callback. - * @ctx: A flag to indicate to maintain an extra context per virtqueue. + * @ctx: whether to maintain an extra context per virtqueue. */ struct virtqueue_info { const char *name; @@ -80,13 +80,13 @@ struct virtqueue_info { * Returns the first 64 feature bits. * @get_extended_features: * vdev: the virtio_device - * Returns the first VIRTIO_FEATURES_MAX feature bits (all we currently + * Returns the first VIRTIO_FEATURES_BITS feature bits (all we currently * need). * @finalize_features: confirm what device features we'll be using. * vdev: the virtio_device * This sends the driver feature bits to the device: it can change * the dev->feature bits if it wants. - * Note that despite the name this can be called any number of + * Note that despite the name this can be called any number of * times. * Returns 0 on success or error status * @bus_name: return the bus name associated with the device (optional) @@ -141,8 +141,8 @@ struct virtio_config_ops { /** * struct virtio_map_ops - operations for mapping buffer for a virtio device - * Note: For transport that has its own mapping logic it must - * implements all of the operations + * Note: For a transport that has its own mapping logic it must + * implement all of the operations * @map_page: map a buffer to the device * map: metadata for performing mapping * page: the page that will be mapped by the device @@ -150,7 +150,7 @@ struct virtio_config_ops { * size: the buffer size * dir: mapping direction * attrs: mapping attributes - * Returns: the mapped address + * Returns the mapped address * @unmap_page: unmap a buffer from the device * map: device specific mapping map * map_handle: the mapped address @@ -172,23 +172,23 @@ struct virtio_config_ops { * size: the size of the buffer * map_handle: the mapping address to sync * gfp: allocation flag (GFP_XXX) - * Returns: virtual address of the allocated buffer + * Returns virtual address of the allocated buffer * @free: free a coherent buffer mapping * map: metadata for performing mapping * size: the size of the buffer * vaddr: virtual address of the buffer - * map_handle: the mapping address to sync + * map_handle: the mapping address that needs to be freed * attrs: unmapping attributes * @need_sync: if the buffer needs synchronization * map: metadata for performing mapping * map_handle: the mapped address - * Returns: whether the buffer needs synchronization + * Returns whether the buffer needs synchronization * @mapping_error: if the mapping address is error * map: metadata for performing mapping * map_handle: the mapped address * @max_mapping_size: get the maximum buffer size that can be mapped * map: metadata for performing mapping - * Returns: the maximum buffer size that can be mapped + * Returns the maximum buffer size that can be mapped */ struct virtio_map_ops { dma_addr_t (*map_page)(union virtio_map map, struct page *page, @@ -362,7 +362,7 @@ void virtio_device_ready(struct virtio_device *dev) * specific set_status() method. * * A well behaved device will only notify a virtqueue after - * DRIVER_OK, this means the device should "see" the coherenct + * DRIVER_OK, this means the device should "see" the coherent * memory write that set vq->broken as false which is done by * the driver when it sees DRIVER_OK, then the following * driver's vring_interrupt() will see vq->broken as false so @@ -384,7 +384,7 @@ const char *virtio_bus_name(struct virtio_device *vdev) * @vq: the virtqueue * @cpu_mask: the cpu mask * - * Pay attention the function are best-effort: the affinity hint may not be set + * Note that this function is best-effort: the affinity hint may not be set * due to config support, irq type and sharing. * */ diff --git a/include/linux/virtio_features.h b/include/linux/virtio_features.h index f748f2f87de8d..ea2ad8717882e 100644 --- a/include/linux/virtio_features.h +++ b/include/linux/virtio_features.h @@ -4,15 +4,16 @@ #include -#define VIRTIO_FEATURES_DWORDS 2 -#define VIRTIO_FEATURES_MAX (VIRTIO_FEATURES_DWORDS * 64) -#define VIRTIO_FEATURES_WORDS (VIRTIO_FEATURES_DWORDS * 2) +#define VIRTIO_FEATURES_U64S 2 +#define VIRTIO_FEATURES_BITS (VIRTIO_FEATURES_U64S * 64) + #define VIRTIO_BIT(b) BIT_ULL((b) & 0x3f) -#define VIRTIO_DWORD(b) ((b) >> 6) +#define VIRTIO_U64(b) ((b) >> 6) + #define VIRTIO_DECLARE_FEATURES(name) \ union { \ u64 name; \ - u64 name##_array[VIRTIO_FEATURES_DWORDS];\ + u64 name##_array[VIRTIO_FEATURES_U64S];\ } static inline bool virtio_features_chk_bit(unsigned int bit) @@ -22,9 +23,9 @@ static inline bool virtio_features_chk_bit(unsigned int bit) * Don't care returning the correct value: the build * will fail before any bad features access */ - BUILD_BUG_ON(bit >= VIRTIO_FEATURES_MAX); + BUILD_BUG_ON(bit >= VIRTIO_FEATURES_BITS); } else { - if (WARN_ON_ONCE(bit >= VIRTIO_FEATURES_MAX)) + if (WARN_ON_ONCE(bit >= VIRTIO_FEATURES_BITS)) return false; } return true; @@ -34,26 +35,26 @@ static inline bool virtio_features_test_bit(const u64 *features, unsigned int bit) { return virtio_features_chk_bit(bit) && - !!(features[VIRTIO_DWORD(bit)] & VIRTIO_BIT(bit)); + !!(features[VIRTIO_U64(bit)] & VIRTIO_BIT(bit)); } static inline void virtio_features_set_bit(u64 *features, unsigned int bit) { if (virtio_features_chk_bit(bit)) - features[VIRTIO_DWORD(bit)] |= VIRTIO_BIT(bit); + features[VIRTIO_U64(bit)] |= VIRTIO_BIT(bit); } static inline void virtio_features_clear_bit(u64 *features, unsigned int bit) { if (virtio_features_chk_bit(bit)) - features[VIRTIO_DWORD(bit)] &= ~VIRTIO_BIT(bit); + features[VIRTIO_U64(bit)] &= ~VIRTIO_BIT(bit); } static inline void virtio_features_zero(u64 *features) { - memset(features, 0, sizeof(features[0]) * VIRTIO_FEATURES_DWORDS); + memset(features, 0, sizeof(features[0]) * VIRTIO_FEATURES_U64S); } static inline void virtio_features_from_u64(u64 *features, u64 from) @@ -66,7 +67,7 @@ static inline bool virtio_features_equal(const u64 *f1, const u64 *f2) { int i; - for (i = 0; i < VIRTIO_FEATURES_DWORDS; ++i) + for (i = 0; i < VIRTIO_FEATURES_U64S; ++i) if (f1[i] != f2[i]) return false; return true; @@ -74,14 +75,14 @@ static inline bool virtio_features_equal(const u64 *f1, const u64 *f2) static inline void virtio_features_copy(u64 *to, const u64 *from) { - memcpy(to, from, sizeof(to[0]) * VIRTIO_FEATURES_DWORDS); + memcpy(to, from, sizeof(to[0]) * VIRTIO_FEATURES_U64S); } static inline void virtio_features_andnot(u64 *to, const u64 *f1, const u64 *f2) { int i; - for (i = 0; i < VIRTIO_FEATURES_DWORDS; i++) + for (i = 0; i < VIRTIO_FEATURES_U64S; i++) to[i] = f1[i] & ~f2[i]; } diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index 75dabb763c650..f36d21b5bc19e 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -207,6 +207,39 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, return __virtio_net_hdr_to_skb(skb, hdr, little_endian, hdr->gso_type); } +/* This function must be called after virtio_net_hdr_from_skb(). */ +static inline void __virtio_net_set_hdrlen(const struct sk_buff *skb, + struct virtio_net_hdr *hdr, + bool little_endian) +{ + u16 hdr_len; + + hdr_len = skb_transport_offset(skb); + + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) + hdr_len += sizeof(struct udphdr); + else + hdr_len += tcp_hdrlen(skb); + + hdr->hdr_len = __cpu_to_virtio16(little_endian, hdr_len); +} + +/* This function must be called after virtio_net_hdr_from_skb(). */ +static inline void __virtio_net_set_tnl_hdrlen(const struct sk_buff *skb, + struct virtio_net_hdr *hdr) +{ + u16 hdr_len; + + hdr_len = skb_inner_transport_offset(skb); + + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) + hdr_len += sizeof(struct udphdr); + else + hdr_len += inner_tcp_hdrlen(skb); + + hdr->hdr_len = __cpu_to_virtio16(true, hdr_len); +} + static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, struct virtio_net_hdr *hdr, bool little_endian, @@ -385,7 +418,8 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, bool tnl_hdr_negotiated, bool little_endian, int vlan_hlen, - bool has_data_valid) + bool has_data_valid, + bool feature_hdrlen) { struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)vhdr; unsigned int inner_nh, outer_th; @@ -394,9 +428,17 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, tnl_gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM); - if (!tnl_gso_type) - return virtio_net_hdr_from_skb(skb, hdr, little_endian, - has_data_valid, vlan_hlen); + if (!tnl_gso_type) { + ret = virtio_net_hdr_from_skb(skb, hdr, little_endian, + has_data_valid, vlan_hlen); + if (ret) + return ret; + + if (feature_hdrlen && hdr->hdr_len) + __virtio_net_set_hdrlen(skb, hdr, little_endian); + + return ret; + } /* Tunnel support not negotiated but skb ask for it. */ if (!tnl_hdr_negotiated) @@ -414,6 +456,9 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, if (ret) return ret; + if (feature_hdrlen && hdr->hdr_len) + __virtio_net_set_tnl_hdrlen(skb, hdr); + if (skb->protocol == htons(ETH_P_IPV6)) hdr->gso_type |= VIRTIO_NET_HDR_GSO_UDP_TUNNEL_IPV6; else diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_modern.h index 48bc12d1045bd..9a3f2fc53bd65 100644 --- a/include/linux/virtio_pci_modern.h +++ b/include/linux/virtio_pci_modern.h @@ -107,7 +107,7 @@ void vp_modern_set_extended_features(struct virtio_pci_modern_device *mdev, static inline u64 vp_modern_get_features(struct virtio_pci_modern_device *mdev) { - u64 features_array[VIRTIO_FEATURES_DWORDS]; + u64 features_array[VIRTIO_FEATURES_U64S]; vp_modern_get_extended_features(mdev, features_array); return features_array[0]; @@ -116,11 +116,11 @@ vp_modern_get_features(struct virtio_pci_modern_device *mdev) static inline u64 vp_modern_get_driver_features(struct virtio_pci_modern_device *mdev) { - u64 features_array[VIRTIO_FEATURES_DWORDS]; + u64 features_array[VIRTIO_FEATURES_U64S]; int i; vp_modern_get_driver_extended_features(mdev, features_array); - for (i = 1; i < VIRTIO_FEATURES_DWORDS; ++i) + for (i = 1; i < VIRTIO_FEATURES_U64S; ++i) WARN_ON_ONCE(features_array[i]); return features_array[0]; } @@ -128,7 +128,7 @@ vp_modern_get_driver_features(struct virtio_pci_modern_device *mdev) static inline void vp_modern_set_features(struct virtio_pci_modern_device *mdev, u64 features) { - u64 features_array[VIRTIO_FEATURES_DWORDS]; + u64 features_array[VIRTIO_FEATURES_U64S]; virtio_features_from_u64(features_array, features); vp_modern_set_extended_features(mdev, features_array); diff --git a/include/media/dvb_vb2.h b/include/media/dvb_vb2.h index 8cb88452cd6c2..0fbbfc65157e6 100644 --- a/include/media/dvb_vb2.h +++ b/include/media/dvb_vb2.h @@ -124,7 +124,7 @@ static inline int dvb_vb2_release(struct dvb_vb2_ctx *ctx) return 0; }; #define dvb_vb2_is_streaming(ctx) (0) -#define dvb_vb2_fill_buffer(ctx, file, wait, flags) (0) +#define dvb_vb2_fill_buffer(ctx, file, wait, flags, flush) (0) static inline __poll_t dvb_vb2_poll(struct dvb_vb2_ctx *ctx, struct file *file, @@ -166,10 +166,12 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx); * @buffer_flags: * pointer to buffer flags as defined by &enum dmx_buffer_flags. * can be NULL. + * @flush: flush the buffer, even if it isn't full. */ int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, const unsigned char *src, int len, - enum dmx_buffer_flags *buffer_flags); + enum dmx_buffer_flags *buffer_flags, + bool flush); /** * dvb_vb2_poll - Wrapper to vb2_core_streamon() for Digital TV diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h index 09c6164577ccf..500f81f399dfa 100644 --- a/include/media/v4l2-mem2mem.h +++ b/include/media/v4l2-mem2mem.h @@ -192,8 +192,7 @@ void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *m2m_ctx); * other instances to take control of the device. * * This function has to be called only after &v4l2_m2m_ops->device_run - * callback has been called on the driver. To prevent recursion, it should - * not be called directly from the &v4l2_m2m_ops->device_run callback though. + * callback has been called on the driver. */ void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev, struct v4l2_m2m_ctx *m2m_ctx); diff --git a/include/net/act_api.h b/include/net/act_api.h index 91a24b5e0b93e..2ba40eb45aad2 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -70,6 +70,7 @@ struct tc_action { #define TCA_ACT_FLAGS_REPLACE (1U << (TCA_ACT_FLAGS_USER_BITS + 2)) #define TCA_ACT_FLAGS_NO_RTNL (1U << (TCA_ACT_FLAGS_USER_BITS + 3)) #define TCA_ACT_FLAGS_AT_INGRESS (1U << (TCA_ACT_FLAGS_USER_BITS + 4)) +#define TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT (1U << (TCA_ACT_FLAGS_USER_BITS + 5)) /* Update lastuse only if needed, to avoid dirtying a cache line. * We use a temp variable to avoid fetching jiffies twice. diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index cb4c02d007590..0ce5eea563e29 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -651,6 +651,8 @@ enum { #define HCI_LE_CIS_PERIPHERAL 0x20 #define HCI_LE_ISO_BROADCASTER 0x40 #define HCI_LE_ISO_SYNC_RECEIVER 0x80 +#define HCI_LE_CS 0x40 +#define HCI_LE_CS_HOST 0x80 /* Connection modes */ #define HCI_CM_ACTIVE 0x0000 @@ -2215,6 +2217,204 @@ struct hci_cp_le_set_host_feature { __u8 bit_value; } __packed; +/* Channel Sounding Commands */ +#define HCI_OP_LE_CS_RD_LOCAL_SUPP_CAP 0x2089 +struct hci_rp_le_cs_rd_local_supp_cap { + __u8 status; + __u8 num_config_supported; + __le16 max_consecutive_procedures_supported; + __u8 num_antennas_supported; + __u8 max_antenna_paths_supported; + __u8 roles_supported; + __u8 modes_supported; + __u8 rtt_capability; + __u8 rtt_aa_only_n; + __u8 rtt_sounding_n; + __u8 rtt_random_payload_n; + __le16 nadm_sounding_capability; + __le16 nadm_random_capability; + __u8 cs_sync_phys_supported; + __le16 subfeatures_supported; + __le16 t_ip1_times_supported; + __le16 t_ip2_times_supported; + __le16 t_fcs_times_supported; + __le16 t_pm_times_supported; + __u8 t_sw_time_supported; + __u8 tx_snr_capability; +} __packed; + +#define HCI_OP_LE_CS_RD_RMT_SUPP_CAP 0x208A +struct hci_cp_le_cs_rd_local_supp_cap { + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_WR_CACHED_RMT_SUPP_CAP 0x208B +struct hci_cp_le_cs_wr_cached_rmt_supp_cap { + __le16 handle; + __u8 num_config_supported; + __le16 max_consecutive_procedures_supported; + __u8 num_antennas_supported; + __u8 max_antenna_paths_supported; + __u8 roles_supported; + __u8 modes_supported; + __u8 rtt_capability; + __u8 rtt_aa_only_n; + __u8 rtt_sounding_n; + __u8 rtt_random_payload_n; + __le16 nadm_sounding_capability; + __le16 nadm_random_capability; + __u8 cs_sync_phys_supported; + __le16 subfeatures_supported; + __le16 t_ip1_times_supported; + __le16 t_ip2_times_supported; + __le16 t_fcs_times_supported; + __le16 t_pm_times_supported; + __u8 t_sw_time_supported; + __u8 tx_snr_capability; +} __packed; + +struct hci_rp_le_cs_wr_cached_rmt_supp_cap { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_SEC_ENABLE 0x208C +struct hci_cp_le_cs_sec_enable { + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_SET_DEFAULT_SETTINGS 0x208D +struct hci_cp_le_cs_set_default_settings { + __le16 handle; + __u8 role_enable; + __u8 cs_sync_ant_sel; + __s8 max_tx_power; +} __packed; + +struct hci_rp_le_cs_set_default_settings { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_RD_RMT_FAE_TABLE 0x208E +struct hci_cp_le_cs_rd_rmt_fae_table { + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_WR_CACHED_RMT_FAE_TABLE 0x208F +struct hci_cp_le_cs_wr_rmt_cached_fae_table { + __le16 handle; + __u8 remote_fae_table[72]; +} __packed; + +struct hci_rp_le_cs_wr_rmt_cached_fae_table { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_CREATE_CONFIG 0x2090 +struct hci_cp_le_cs_create_config { + __le16 handle; + __u8 config_id; + __u8 create_context; + __u8 main_mode_type; + __u8 sub_mode_type; + __u8 min_main_mode_steps; + __u8 max_main_mode_steps; + __u8 main_mode_repetition; + __u8 mode_0_steps; + __u8 role; + __u8 rtt_type; + __u8 cs_sync_phy; + __u8 channel_map[10]; + __u8 channel_map_repetition; + __u8 channel_selection_type; + __u8 ch3c_shape; + __u8 ch3c_jump; + __u8 reserved; +} __packed; + +#define HCI_OP_LE_CS_REMOVE_CONFIG 0x2091 +struct hci_cp_le_cs_remove_config { + __le16 handle; + __u8 config_id; +} __packed; + +#define HCI_OP_LE_CS_SET_CH_CLASSIFICATION 0x2092 +struct hci_cp_le_cs_set_ch_classification { + __u8 ch_classification[10]; +} __packed; + +struct hci_rp_le_cs_set_ch_classification { + __u8 status; +} __packed; + +#define HCI_OP_LE_CS_SET_PROC_PARAM 0x2093 +struct hci_cp_le_cs_set_proc_param { + __le16 handle; + __u8 config_id; + __le16 max_procedure_len; + __le16 min_procedure_interval; + __le16 max_procedure_interval; + __le16 max_procedure_count; + __u8 min_subevent_len[3]; + __u8 max_subevent_len[3]; + __u8 tone_antenna_config_selection; + __u8 phy; + __u8 tx_power_delta; + __u8 preferred_peer_antenna; + __u8 snr_control_initiator; + __u8 snr_control_reflector; +} __packed; + +struct hci_rp_le_cs_set_proc_param { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_SET_PROC_ENABLE 0x2094 +struct hci_cp_le_cs_set_proc_enable { + __le16 handle; + __u8 config_id; + __u8 enable; +} __packed; + +#define HCI_OP_LE_CS_TEST 0x2095 +struct hci_cp_le_cs_test { + __u8 main_mode_type; + __u8 sub_mode_type; + __u8 main_mode_repetition; + __u8 mode_0_steps; + __u8 role; + __u8 rtt_type; + __u8 cs_sync_phy; + __u8 cs_sync_antenna_selection; + __u8 subevent_len[3]; + __le16 subevent_interval; + __u8 max_num_subevents; + __u8 transmit_power_level; + __u8 t_ip1_time; + __u8 t_ip2_time; + __u8 t_fcs_time; + __u8 t_pm_time; + __u8 t_sw_time; + __u8 tone_antenna_config_selection; + __u8 reserved; + __u8 snr_control_initiator; + __u8 snr_control_reflector; + __le16 drbg_nonce; + __u8 channel_map_repetition; + __le16 override_config; + __u8 override_parameters_length; + __u8 override_parameters_data[]; +} __packed; + +struct hci_rp_le_cs_test { + __u8 status; +} __packed; + +#define HCI_OP_LE_CS_TEST_END 0x2096 + /* ---- HCI Events ---- */ struct hci_ev_status { __u8 status; @@ -2883,6 +3083,129 @@ struct hci_evt_le_big_info_adv_report { __u8 encryption; } __packed; +/* Channel Sounding Events */ +#define HCI_EVT_LE_CS_READ_RMT_SUPP_CAP_COMPLETE 0x2C +struct hci_evt_le_cs_read_rmt_supp_cap_complete { + __u8 status; + __le16 handle; + __u8 num_configs_supp; + __le16 max_consec_proc_supp; + __u8 num_ant_supp; + __u8 max_ant_path_supp; + __u8 roles_supp; + __u8 modes_supp; + __u8 rtt_cap; + __u8 rtt_aa_only_n; + __u8 rtt_sounding_n; + __u8 rtt_rand_payload_n; + __le16 nadm_sounding_cap; + __le16 nadm_rand_cap; + __u8 cs_sync_phys_supp; + __le16 sub_feat_supp; + __le16 t_ip1_times_supp; + __le16 t_ip2_times_supp; + __le16 t_fcs_times_supp; + __le16 t_pm_times_supp; + __u8 t_sw_times_supp; + __u8 tx_snr_cap; +} __packed; + +#define HCI_EVT_LE_CS_READ_RMT_FAE_TABLE_COMPLETE 0x2D +struct hci_evt_le_cs_read_rmt_fae_table_complete { + __u8 status; + __le16 handle; + __u8 remote_fae_table[72]; +} __packed; + +#define HCI_EVT_LE_CS_SECURITY_ENABLE_COMPLETE 0x2E +struct hci_evt_le_cs_security_enable_complete { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_EVT_LE_CS_CONFIG_COMPLETE 0x2F +struct hci_evt_le_cs_config_complete { + __u8 status; + __le16 handle; + __u8 config_id; + __u8 action; + __u8 main_mode_type; + __u8 sub_mode_type; + __u8 min_main_mode_steps; + __u8 max_main_mode_steps; + __u8 main_mode_rep; + __u8 mode_0_steps; + __u8 role; + __u8 rtt_type; + __u8 cs_sync_phy; + __u8 channel_map[10]; + __u8 channel_map_rep; + __u8 channel_sel_type; + __u8 ch3c_shape; + __u8 ch3c_jump; + __u8 reserved; + __u8 t_ip1_time; + __u8 t_ip2_time; + __u8 t_fcs_time; + __u8 t_pm_time; +} __packed; + +#define HCI_EVT_LE_CS_PROCEDURE_ENABLE_COMPLETE 0x30 +struct hci_evt_le_cs_procedure_enable_complete { + __u8 status; + __le16 handle; + __u8 config_id; + __u8 state; + __u8 tone_ant_config_sel; + __s8 sel_tx_pwr; + __u8 sub_evt_len[3]; + __u8 sub_evts_per_evt; + __le16 sub_evt_intrvl; + __le16 evt_intrvl; + __le16 proc_intrvl; + __le16 proc_counter; + __le16 max_proc_len; +} __packed; + +#define HCI_EVT_LE_CS_SUBEVENT_RESULT 0x31 +struct hci_evt_le_cs_subevent_result { + __le16 handle; + __u8 config_id; + __le16 start_acl_conn_evt_counter; + __le16 proc_counter; + __le16 freq_comp; + __u8 ref_pwr_lvl; + __u8 proc_done_status; + __u8 subevt_done_status; + __u8 abort_reason; + __u8 num_ant_paths; + __u8 num_steps_reported; + __u8 step_mode[0]; /* depends on num_steps_reported */ + __u8 step_channel[0]; /* depends on num_steps_reported */ + __u8 step_data_length[0]; /* depends on num_steps_reported */ + __u8 step_data[0]; /* depends on num_steps_reported */ +} __packed; + +#define HCI_EVT_LE_CS_SUBEVENT_RESULT_CONTINUE 0x32 +struct hci_evt_le_cs_subevent_result_continue { + __le16 handle; + __u8 config_id; + __u8 proc_done_status; + __u8 subevt_done_status; + __u8 abort_reason; + __u8 num_ant_paths; + __u8 num_steps_reported; + __u8 step_mode[0]; /* depends on num_steps_reported */ + __u8 step_channel[0]; /* depends on num_steps_reported */ + __u8 step_data_length[0]; /* depends on num_steps_reported */ + __u8 step_data[0]; /* depends on num_steps_reported */ +} __packed; + +#define HCI_EVT_LE_CS_TEST_END_COMPLETE 0x33 +struct hci_evt_le_cs_test_end_complete { + __u8 status; +} __packed; + #define HCI_EV_VENDOR 0xff /* Internal events generated by Bluetooth stack */ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 0cb87687837f7..313ed47e84d14 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -2054,6 +2054,12 @@ void hci_conn_del_sysfs(struct hci_conn *conn); ((dev)->le_features[3] & HCI_LE_ISO_SYNC_RECEIVER) #define sync_recv_enabled(dev) (le_enabled(dev) && sync_recv_capable(dev)) +/* Channel sounding support */ +#define le_cs_capable(dev) \ + ((dev)->le_features[5] & HCI_LE_CS) +#define le_cs_host_capable(dev) \ + ((dev)->le_features[5] & HCI_LE_CS_HOST) + #define mws_transport_config_capable(dev) (((dev)->commands[30] & 0x08) && \ (!hci_test_quirk((dev), HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG))) diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 00e182a22720a..5172afee54943 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -284,9 +284,9 @@ struct l2cap_conn_rsp { #define L2CAP_CR_LE_BAD_KEY_SIZE 0x0007 #define L2CAP_CR_LE_ENCRYPTION 0x0008 #define L2CAP_CR_LE_INVALID_SCID 0x0009 -#define L2CAP_CR_LE_SCID_IN_USE 0X000A -#define L2CAP_CR_LE_UNACCEPT_PARAMS 0X000B -#define L2CAP_CR_LE_INVALID_PARAMS 0X000C +#define L2CAP_CR_LE_SCID_IN_USE 0x000A +#define L2CAP_CR_LE_UNACCEPT_PARAMS 0x000B +#define L2CAP_CR_LE_INVALID_PARAMS 0x000C /* connect/create channel status */ #define L2CAP_CS_NO_INFO 0x0000 @@ -493,6 +493,8 @@ struct l2cap_ecred_reconf_req { #define L2CAP_RECONF_SUCCESS 0x0000 #define L2CAP_RECONF_INVALID_MTU 0x0001 #define L2CAP_RECONF_INVALID_MPS 0x0002 +#define L2CAP_RECONF_INVALID_CID 0x0003 +#define L2CAP_RECONF_INVALID_PARAMS 0x0004 struct l2cap_ecred_reconf_rsp { __le16 result; @@ -655,8 +657,8 @@ struct l2cap_conn { struct sk_buff *rx_skb; __u32 rx_len; + struct ida tx_ida; __u8 tx_ident; - struct mutex ident_lock; struct sk_buff_head pending_rx; struct work_struct pending_rx_work; diff --git a/include/net/bonding.h b/include/net/bonding.h index 49edc7da05867..99c1bdadcd11a 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -521,13 +521,14 @@ static inline int bond_is_ip6_target_ok(struct in6_addr *addr) static inline unsigned long slave_oldest_target_arp_rx(struct bonding *bond, struct slave *slave) { + unsigned long tmp, ret = READ_ONCE(slave->target_last_arp_rx[0]); int i = 1; - unsigned long ret = slave->target_last_arp_rx[0]; - - for (; (i < BOND_MAX_ARP_TARGETS) && bond->params.arp_targets[i]; i++) - if (time_before(slave->target_last_arp_rx[i], ret)) - ret = slave->target_last_arp_rx[i]; + for (; (i < BOND_MAX_ARP_TARGETS) && bond->params.arp_targets[i]; i++) { + tmp = READ_ONCE(slave->target_last_arp_rx[i]); + if (time_before(tmp, ret)) + ret = tmp; + } return ret; } @@ -537,7 +538,7 @@ static inline unsigned long slave_last_rx(struct bonding *bond, if (bond->params.arp_all_targets == BOND_ARP_TARGETS_ALL) return slave_oldest_target_arp_rx(bond, slave); - return slave->last_rx; + return READ_ONCE(slave->last_rx); } static inline void slave_update_last_tx(struct slave *slave) @@ -697,6 +698,7 @@ void bond_debug_register(struct bonding *bond); void bond_debug_unregister(struct bonding *bond); void bond_debug_reregister(struct bonding *bond); const char *bond_mode_name(int mode); +bool __bond_xdp_check(int mode, int xmit_policy); bool bond_xdp_check(struct bonding *bond, int mode); void bond_setup(struct net_device *bond_dev); unsigned int bond_get_num_tx_queues(void); diff --git a/include/net/dsa.h b/include/net/dsa.h index d73ea08800660..5cb456bf46394 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -298,6 +298,7 @@ struct dsa_port { struct devlink_port devlink_port; struct phylink *pl; struct phylink_config pl_config; + netdevice_tracker conduit_tracker; struct dsa_lag *lag; struct net_device *hsr_dev; diff --git a/include/net/inet6_hashtables.h b/include/net/inet6_hashtables.h index 282e29237d936..c16de5b7963fd 100644 --- a/include/net/inet6_hashtables.h +++ b/include/net/inet6_hashtables.h @@ -175,7 +175,7 @@ static inline bool inet6_match(const struct net *net, const struct sock *sk, { if (!net_eq(sock_net(sk), net) || sk->sk_family != AF_INET6 || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || !ipv6_addr_equal(&sk->sk_v6_daddr, saddr) || !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return false; diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index b4b8866476075..afec146b8bafd 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -42,7 +42,9 @@ struct inet_connection_sock_af_ops { struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); u16 net_header_len; int (*setsockopt)(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); diff --git a/include/net/inet_ecn.h b/include/net/inet_ecn.h index ea32393464a29..827b87a95dab3 100644 --- a/include/net/inet_ecn.h +++ b/include/net/inet_ecn.h @@ -51,11 +51,25 @@ static inline __u8 INET_ECN_encapsulate(__u8 outer, __u8 inner) return outer; } +/* Apply either ECT(0) or ECT(1) */ +static inline void __INET_ECN_xmit(struct sock *sk, bool use_ect_1) +{ + __u8 ect = use_ect_1 ? INET_ECN_ECT_1 : INET_ECN_ECT_0; + + /* Mask the complete byte in case the connection alternates between + * ECT(0) and ECT(1). + */ + inet_sk(sk)->tos &= ~INET_ECN_MASK; + inet_sk(sk)->tos |= ect; + if (inet6_sk(sk)) { + inet6_sk(sk)->tclass &= ~INET_ECN_MASK; + inet6_sk(sk)->tclass |= ect; + } +} + static inline void INET_ECN_xmit(struct sock *sk) { - inet_sk(sk)->tos |= INET_ECN_ECT_0; - if (inet6_sk(sk) != NULL) - inet6_sk(sk)->tclass |= INET_ECN_ECT_0; + __INET_ECN_xmit(sk, false); } static inline void INET_ECN_dontxmit(struct sock *sk) diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h index 0eccd9c3a883f..365925c9d2628 100644 --- a/include/net/inet_frag.h +++ b/include/net/inet_frag.h @@ -123,27 +123,15 @@ void inet_frags_fini(struct inet_frags *); int fqdir_init(struct fqdir **fqdirp, struct inet_frags *f, struct net *net); -static inline void fqdir_pre_exit(struct fqdir *fqdir) -{ - /* Prevent creation of new frags. - * Pairs with READ_ONCE() in inet_frag_find(). - */ - WRITE_ONCE(fqdir->high_thresh, 0); - - /* Pairs with READ_ONCE() in inet_frag_kill(), ip_expire() - * and ip6frag_expire_frag_queue(). - */ - WRITE_ONCE(fqdir->dead, true); -} +void fqdir_pre_exit(struct fqdir *fqdir); void fqdir_exit(struct fqdir *fqdir); void inet_frag_kill(struct inet_frag_queue *q, int *refs); void inet_frag_destroy(struct inet_frag_queue *q); struct inet_frag_queue *inet_frag_find(struct fqdir *fqdir, void *key); -/* Free all skbs in the queue; return the sum of their truesizes. */ -unsigned int inet_frag_rbtree_purge(struct rb_root *root, - enum skb_drop_reason reason); +void inet_frag_queue_flush(struct inet_frag_queue *q, + enum skb_drop_reason reason); static inline void inet_frag_putn(struct inet_frag_queue *q, int refs) { diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index ac05a52d9e138..6d936e9f2fd32 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -264,6 +264,20 @@ inet_bhashfn_portaddr(const struct inet_hashinfo *hinfo, const struct sock *sk, return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)]; } +static inline bool inet_use_hash2_on_bind(const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) { + if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) + return false; + + if (!ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) + return true; + } +#endif + return sk->sk_rcv_saddr != htonl(INADDR_ANY); +} + struct inet_bind_hashbucket * inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port); @@ -345,7 +359,7 @@ static inline bool inet_match(const struct net *net, const struct sock *sk, int dif, int sdif) { if (!net_eq(sock_net(sk), net) || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || sk->sk_addrpair != cookie) return false; diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 1086256549faa..b6ec08072533a 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -214,6 +214,7 @@ struct inet_sock { struct sock sk; #if IS_ENABLED(CONFIG_IPV6) struct ipv6_pinfo *pinet6; + struct ipv6_fl_socklist __rcu *ipv6_fl_list; #endif /* Socket demultiplex comparisons on incoming packets. */ #define inet_daddr sk.__sk_common.skc_daddr diff --git a/include/net/ioam6.h b/include/net/ioam6.h index 2cbbee6e806aa..a75912fe247e6 100644 --- a/include/net/ioam6.h +++ b/include/net/ioam6.h @@ -60,6 +60,8 @@ void ioam6_fill_trace_data(struct sk_buff *skb, struct ioam6_trace_hdr *trace, bool is_input); +u8 ioam6_trace_compute_nodelen(u32 trace_type); + int ioam6_init(void); void ioam6_exit(void); diff --git a/include/net/ip.h b/include/net/ip.h index 380afb691c419..1ce79e62a76fb 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -101,7 +101,7 @@ static inline void ipcm_init_sk(struct ipcm_cookie *ipcm, ipcm->oif = READ_ONCE(inet->sk.sk_bound_dev_if); ipcm->addr = inet->inet_saddr; - ipcm->protocol = inet->inet_num; + ipcm->protocol = READ_ONCE(inet->inet_num); } #define IPCB(skb) ((struct inet_skb_parm*)((skb)->cb)) diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h index 88b0dd4d8e094..9f8b6814a96a0 100644 --- a/include/net/ip6_fib.h +++ b/include/net/ip6_fib.h @@ -507,12 +507,14 @@ void fib6_rt_update(struct net *net, struct fib6_info *rt, void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, unsigned int flags); +void fib6_age_exceptions(struct fib6_info *rt, struct fib6_gc_args *gc_args, + unsigned long now); void fib6_run_gc(unsigned long expires, struct net *net, bool force); - void fib6_gc_cleanup(void); int fib6_init(void); +#if IS_ENABLED(CONFIG_IPV6) /* Add the route to the gc list if it is not already there * * The callers should hold f6i->fib6_table->tb6_lock. @@ -545,6 +547,23 @@ static inline void fib6_remove_gc_list(struct fib6_info *f6i) hlist_del_init(&f6i->gc_link); } +static inline void fib6_may_remove_gc_list(struct net *net, + struct fib6_info *f6i) +{ + struct fib6_gc_args gc_args; + + if (hlist_unhashed(&f6i->gc_link)) + return; + + gc_args.timeout = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_interval); + gc_args.more = 0; + + rcu_read_lock(); + fib6_age_exceptions(f6i, &gc_args, jiffies); + rcu_read_unlock(); +} +#endif + struct ipv6_route_iter { struct seq_net_private p; struct fib6_walker w; diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index 120db28658112..359b595f1df93 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -156,6 +156,18 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, { int pkt_len, err; + if (unlikely(dev_recursion_level() > IP_TUNNEL_RECURSION_LIMIT)) { + if (dev) { + net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", + dev->name); + DEV_STATS_INC(dev, tx_errors); + } + kfree_skb(skb); + return; + } + + dev_xmit_recursion_inc(); + memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); IP6CB(skb)->flags = ip6cb_flags; pkt_len = skb->len - skb_inner_network_offset(skb); @@ -166,6 +178,8 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, pkt_len = -1; iptunnel_xmit_stats(dev, pkt_len); } + + dev_xmit_recursion_dec(); } #endif #endif diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index b4495c38e0a01..318593743b6e1 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -559,7 +559,7 @@ static inline u32 fib_multipath_hash_from_keys(const struct net *net, siphash_aligned_key_t hash_key; u32 mp_seed; - mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed).mp_seed; + mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.mp_seed); fib_multipath_hash_construct_key(&hash_key, mp_seed); return flow_hash_from_keys_seed(keys, &hash_key); diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index ecae35512b9b4..253ed3930f6ef 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -19,6 +19,7 @@ #include #include #include +#include #if IS_ENABLED(CONFIG_IPV6) #include @@ -26,6 +27,13 @@ #include #endif +/* Recursion limit for tunnel xmit to detect routing loops. + * Unlike XMIT_RECURSION_LIMIT (8) used in the no-qdisc path, tunnel + * recursion involves route lookups and full IP output, consuming much + * more stack per level, so a lower limit is needed. + */ +#define IP_TUNNEL_RECURSION_LIMIT 5 + /* Keep error state on tunnel for 30 sec */ #define IPTUNNEL_ERR_TIMEO (30*HZ) @@ -372,7 +380,17 @@ static inline void ip_tunnel_init_flow(struct flowi4 *fl4, fl4->flowi4_flags = flow_flags; } -int ip_tunnel_init(struct net_device *dev); +int __ip_tunnel_init(struct net_device *dev); +#define ip_tunnel_init(DEV) \ +({ \ + struct net_device *__dev = (DEV); \ + int __res = __ip_tunnel_init(__dev); \ + \ + if (!__res) \ + netdev_lockdep_set_classes(__dev);\ + __res; \ +}) + void ip_tunnel_uninit(struct net_device *dev); void ip_tunnel_dellink(struct net_device *dev, struct list_head *head); struct net *ip_tunnel_get_link_net(const struct net_device *dev); diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 2ccdf85f34f16..7e984e75f3345 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -1010,11 +1010,11 @@ static inline int ip6_default_np_autolabel(struct net *net) #if IS_ENABLED(CONFIG_IPV6) static inline int ip6_multipath_hash_policy(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_policy; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_policy); } static inline u32 ip6_multipath_hash_fields(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_fields; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_fields); } #else static inline int ip6_multipath_hash_policy(const struct net *net) @@ -1280,12 +1280,15 @@ int ipv6_sock_mc_drop(struct sock *sk, int ifindex, static inline int ip6_sock_set_v6only(struct sock *sk) { - if (inet_sk(sk)->inet_num) - return -EINVAL; + int ret = 0; + lock_sock(sk); - sk->sk_ipv6only = true; + if (inet_sk(sk)->inet_num) + ret = -EINVAL; + else + sk->sk_ipv6only = true; release_sock(sk); - return 0; + return ret; } static inline void ip6_sock_set_recverr(struct sock *sk) diff --git a/include/net/ipv6_frag.h b/include/net/ipv6_frag.h index 38ef66826939e..41d9fc6965f9a 100644 --- a/include/net/ipv6_frag.h +++ b/include/net/ipv6_frag.h @@ -69,9 +69,6 @@ ip6frag_expire_frag_queue(struct net *net, struct frag_queue *fq) int refs = 1; rcu_read_lock(); - /* Paired with the WRITE_ONCE() in fqdir_pre_exit(). */ - if (READ_ONCE(fq->q.fqdir->dead)) - goto out_rcu_unlock; spin_lock(&fq->q.lock); if (fq->q.flags & INET_FRAG_COMPLETE) @@ -80,6 +77,12 @@ ip6frag_expire_frag_queue(struct net *net, struct frag_queue *fq) fq->q.flags |= INET_FRAG_DROP; inet_frag_kill(&fq->q, &refs); + /* Paired with the WRITE_ONCE() in fqdir_pre_exit(). */ + if (READ_ONCE(fq->q.fqdir->dead)) { + inet_frag_queue_flush(&fq->q, 0); + goto out; + } + dev = dev_get_by_index_rcu(net, fq->iif); if (!dev) goto out; diff --git a/include/net/mac80211.h b/include/net/mac80211.h index a55085cf4ec49..ac2546b121385 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -7289,7 +7289,9 @@ void ieee80211_report_wowlan_wakeup(struct ieee80211_vif *vif, * @band: the band to transmit on * @sta: optional pointer to get the station to send the frame to * - * Return: %true if the skb was prepared, %false otherwise + * Return: %true if the skb was prepared, %false otherwise. + * On failure, the skb is freed by this function; callers must not + * free it again. * * Note: must be called under RCU lock */ diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index 3384859a89210..8883575adcc1e 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -83,6 +83,11 @@ void nf_conntrack_lock(spinlock_t *lock); extern spinlock_t nf_conntrack_expect_lock; +static inline void lockdep_nfct_expect_lock_held(void) +{ + lockdep_assert_held(&nf_conntrack_expect_lock); +} + /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ static inline void __nf_ct_set_timeout(struct nf_conn *ct, u64 timeout) diff --git a/include/net/netfilter/nf_conntrack_count.h b/include/net/netfilter/nf_conntrack_count.h index 1b58b5b91ff6a..cf0166520cf33 100644 --- a/include/net/netfilter/nf_conntrack_count.h +++ b/include/net/netfilter/nf_conntrack_count.h @@ -13,20 +13,20 @@ struct nf_conncount_list { u32 last_gc; /* jiffies at most recent gc */ struct list_head head; /* connections with the same filtering key */ unsigned int count; /* length of list */ + unsigned int last_gc_count; /* length of list at most recent gc */ }; struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen); void nf_conncount_destroy(struct net *net, struct nf_conncount_data *data); -unsigned int nf_conncount_count(struct net *net, - struct nf_conncount_data *data, - const u32 *key, - const struct nf_conntrack_tuple *tuple, - const struct nf_conntrack_zone *zone); +unsigned int nf_conncount_count_skb(struct net *net, + const struct sk_buff *skb, + u16 l3num, + struct nf_conncount_data *data, + const u32 *key); -int nf_conncount_add(struct net *net, struct nf_conncount_list *list, - const struct nf_conntrack_tuple *tuple, - const struct nf_conntrack_zone *zone); +int nf_conncount_add_skb(struct net *net, const struct sk_buff *skb, + u16 l3num, struct nf_conncount_list *list); void nf_conncount_list_init(struct nf_conncount_list *list); diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index 165e7a03b8e9d..e9a8350e7ccfb 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -22,10 +22,16 @@ struct nf_conntrack_expect { /* Hash member */ struct hlist_node hnode; + /* Network namespace */ + possible_net_t net; + /* We expect this tuple, with the following mask */ struct nf_conntrack_tuple tuple; struct nf_conntrack_tuple_mask mask; +#ifdef CONFIG_NF_CONNTRACK_ZONES + struct nf_conntrack_zone zone; +#endif /* Usage count. */ refcount_t use; @@ -40,7 +46,7 @@ struct nf_conntrack_expect { struct nf_conntrack_expect *this); /* Helper to assign to new connection */ - struct nf_conntrack_helper *helper; + struct nf_conntrack_helper __rcu *helper; /* The conntrack of the master connection */ struct nf_conn *master; @@ -62,7 +68,17 @@ struct nf_conntrack_expect { static inline struct net *nf_ct_exp_net(struct nf_conntrack_expect *exp) { - return nf_ct_net(exp->master); + return read_pnet(&exp->net); +} + +static inline bool nf_ct_exp_zone_equal_any(const struct nf_conntrack_expect *a, + const struct nf_conntrack_zone *b) +{ +#ifdef CONFIG_NF_CONNTRACK_ZONES + return a->zone.id == b->id; +#else + return true; +#endif } #define NF_CT_EXP_POLICY_NAME_LEN 16 diff --git a/include/net/netfilter/nf_conntrack_timeout.h b/include/net/netfilter/nf_conntrack_timeout.h index 9fdaba911de64..3a66d4abb6d68 100644 --- a/include/net/netfilter/nf_conntrack_timeout.h +++ b/include/net/netfilter/nf_conntrack_timeout.h @@ -14,6 +14,7 @@ struct nf_ct_timeout { __u16 l3num; const struct nf_conntrack_l4proto *l4proto; + struct rcu_head rcu; char data[]; }; diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 4aeffddb75861..d17035d14d96c 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -6,11 +6,13 @@ #include #include #include +#include #include /* Each queued (to userspace) skbuff has one of these. */ struct nf_queue_entry { struct list_head list; + struct rhash_head hash_node; struct sk_buff *skb; unsigned int id; unsigned int hook_index; /* index in hook_entries->hook[] */ @@ -19,6 +21,7 @@ struct nf_queue_entry { struct net_device *physout; #endif struct nf_hook_state state; + bool nf_ct_is_unconfirmed; u16 size; /* sizeof(entry) + saved route keys */ /* extra space to store route keys */ diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index fab7dc73f738c..4dc080f7f27c6 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -317,11 +317,13 @@ static inline void *nft_elem_priv_cast(const struct nft_elem_priv *priv) * @NFT_ITER_UNSPEC: unspecified, to catch errors * @NFT_ITER_READ: read-only iteration over set elements * @NFT_ITER_UPDATE: iteration under mutex to update set element state + * @NFT_ITER_UPDATE_CLONE: clone set before iteration under mutex to update element */ enum nft_iter_type { NFT_ITER_UNSPEC, NFT_ITER_READ, NFT_ITER_UPDATE, + NFT_ITER_UPDATE_CLONE, }; struct nft_set; @@ -452,6 +454,7 @@ struct nft_set_ext; * @init: initialize private data of new set instance * @destroy: destroy private data of set instance * @gc_init: initialize garbage collection + * @abort_skip_removal: skip removal of elements from abort path * @elemsize: element private size * * Operations lookup, update and delete have simpler interfaces, are faster @@ -509,6 +512,7 @@ struct nft_set_ops { const struct nft_set *set); void (*gc_init)(const struct nft_set *set); + bool abort_skip_removal; unsigned int elemsize; }; @@ -871,6 +875,8 @@ struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set, u64 timeout, u64 expiration, gfp_t gfp); int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, struct nft_expr *expr_array[]); +void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, + struct nft_set_elem_expr *elem_expr); void nft_set_elem_destroy(const struct nft_set *set, const struct nft_elem_priv *elem_priv, bool destroy_expr); @@ -1091,6 +1097,29 @@ struct nft_rule_blob { __attribute__((aligned(__alignof__(struct nft_rule_dp)))); }; +enum nft_chain_types { + NFT_CHAIN_T_DEFAULT = 0, + NFT_CHAIN_T_ROUTE, + NFT_CHAIN_T_NAT, + NFT_CHAIN_T_MAX +}; + +/** + * struct nft_chain_validate_state - validation state + * + * If a chain is encountered again during table validation it is + * possible to avoid revalidation provided the calling context is + * compatible. This structure stores relevant calling context of + * previous validations. + * + * @hook_mask: the hook numbers and locations the chain is linked to + * @depth: the deepest call chain level the chain is linked to + */ +struct nft_chain_validate_state { + u8 hook_mask[NFT_CHAIN_T_MAX]; + u8 depth; +}; + /** * struct nft_chain - nf_tables chain * @@ -1109,6 +1138,7 @@ struct nft_rule_blob { * @udlen: user data length * @udata: user data in the chain * @blob_next: rule blob pointer to the next in the chain + * @vstate: validation state */ struct nft_chain { struct nft_rule_blob __rcu *blob_gen_0; @@ -1128,9 +1158,10 @@ struct nft_chain { /* Only used during control plane commit phase: */ struct nft_rule_blob *blob_next; + struct nft_chain_validate_state vstate; }; -int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain); +int nft_chain_validate(const struct nft_ctx *ctx, struct nft_chain *chain); int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set, const struct nft_set_iter *iter, struct nft_elem_priv *elem_priv); @@ -1138,13 +1169,6 @@ int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set); int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain); void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain); -enum nft_chain_types { - NFT_CHAIN_T_DEFAULT = 0, - NFT_CHAIN_T_ROUTE, - NFT_CHAIN_T_NAT, - NFT_CHAIN_T_MAX -}; - /** * struct nft_chain_type - nf_tables chain type info * @@ -1838,6 +1862,11 @@ struct nft_trans_gc { struct rcu_head rcu; }; +static inline int nft_trans_gc_space(const struct nft_trans_gc *trans) +{ + return NFT_TRANS_GC_BATCHCOUNT - trans->count; +} + static inline void nft_ctx_update(struct nft_ctx *ctx, const struct nft_trans *trans) { diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 34eb3aecb3f21..62166da045548 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -87,6 +87,12 @@ struct netns_ipv4 { int sysctl_tcp_rmem[3]; __cacheline_group_end(netns_ipv4_read_rx); + /* ICMP rate limiter hot cache line. */ + __cacheline_group_begin_aligned(icmp); + atomic_t icmp_global_credit; + u32 icmp_global_stamp; + __cacheline_group_end_aligned(icmp); + struct inet_timewait_death_row tcp_death_row; struct udp_table *udp_table; @@ -139,8 +145,7 @@ struct netns_ipv4 { int sysctl_icmp_ratemask; int sysctl_icmp_msgs_per_sec; int sysctl_icmp_msgs_burst; - atomic_t icmp_global_credit; - u32 icmp_global_stamp; + u32 ip_rt_min_pmtu; int ip_rt_mtu_expires; int ip_rt_min_advmss; diff --git a/include/net/nfc/nfc.h b/include/net/nfc/nfc.h index 127e6c7d910dc..c54df042db6be 100644 --- a/include/net/nfc/nfc.h +++ b/include/net/nfc/nfc.h @@ -219,6 +219,8 @@ static inline void nfc_free_device(struct nfc_dev *dev) int nfc_register_device(struct nfc_dev *dev); +void nfc_unregister_rfkill(struct nfc_dev *dev); +void nfc_remove_device(struct nfc_dev *dev); void nfc_unregister_device(struct nfc_dev *dev); /** diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index 1509a536cb855..fb4f03ccd6156 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -246,7 +246,7 @@ struct page_pool { /* User-facing fields, protected by page_pools_lock */ struct { struct hlist_node list; - u64 detach_time; + ktime_t detach_time; u32 id; } user; }; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 738cd5b13c62f..059eb6cb54f13 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -696,6 +696,34 @@ void qdisc_destroy(struct Qdisc *qdisc); void qdisc_put(struct Qdisc *qdisc); void qdisc_put_unlocked(struct Qdisc *qdisc); void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len); + +static inline void dev_reset_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_unused) +{ + struct Qdisc *qdisc; + bool nolock; + + qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); + if (!qdisc) + return; + + nolock = qdisc->flags & TCQ_F_NOLOCK; + + if (nolock) + spin_lock_bh(&qdisc->seqlock); + spin_lock_bh(qdisc_lock(qdisc)); + + qdisc_reset(qdisc); + + spin_unlock_bh(qdisc_lock(qdisc)); + if (nolock) { + clear_bit(__QDISC_STATE_MISSED, &qdisc->state); + clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); + spin_unlock_bh(&qdisc->seqlock); + } +} + #ifdef CONFIG_NET_SCHED int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, void *type_data); @@ -758,13 +786,23 @@ static inline bool skb_skip_tc_classify(struct sk_buff *skb) static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) { struct Qdisc *qdisc; + bool nolock; for (; i < dev->num_tx_queues; i++) { qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); if (qdisc) { + nolock = qdisc->flags & TCQ_F_NOLOCK; + + if (nolock) + spin_lock_bh(&qdisc->seqlock); spin_lock_bh(qdisc_lock(qdisc)); qdisc_reset(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); + if (nolock) { + clear_bit(__QDISC_STATE_MISSED, &qdisc->state); + clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); + spin_unlock_bh(&qdisc->seqlock); + } } } } @@ -1373,6 +1411,11 @@ void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp, struct tcf_block *block); +static inline bool mini_qdisc_pair_inited(struct mini_Qdisc_pair *miniqp) +{ + return !!miniqp->p_miniq; +} + void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx); int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)); diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h index cddebafb9f779..6f996229167b3 100644 --- a/include/net/secure_seq.h +++ b/include/net/secure_seq.h @@ -5,16 +5,47 @@ #include struct net; +extern struct net init_net; + +union tcp_seq_and_ts_off { + struct { + u32 seq; + u32 ts_off; + }; + u64 hash64; +}; u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport); -u32 secure_tcp_seq(__be32 saddr, __be32 daddr, - __be16 sport, __be16 dport); -u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr); -u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, - __be16 sport, __be16 dport); -u32 secure_tcpv6_ts_off(const struct net *net, - const __be32 *saddr, const __be32 *daddr); +union tcp_seq_and_ts_off +secure_tcp_seq_and_ts_off(const struct net *net, __be32 saddr, __be32 daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcp_seq(__be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcp_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + + return ts.seq; +} + +union tcp_seq_and_ts_off +secure_tcpv6_seq_and_ts_off(const struct net *net, const __be32 *saddr, + const __be32 *daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcpv6_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + return ts.seq; +} #endif /* _NET_SECURE_SEQ */ diff --git a/include/net/sock.h b/include/net/sock.h index 60bcb13f045c3..9540dcc5a0c01 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2067,7 +2067,7 @@ static inline int sk_rx_queue_get(const struct sock *sk) static inline void sk_set_socket(struct sock *sk, struct socket *sock) { - sk->sk_socket = sock; + WRITE_ONCE(sk->sk_socket, sock); if (sock) { WRITE_ONCE(sk->sk_uid, SOCK_INODE(sock)->i_uid); WRITE_ONCE(sk->sk_ino, SOCK_INODE(sock)->i_ino); diff --git a/include/net/tc_act/tc_gate.h b/include/net/tc_act/tc_gate.h index c1a67149c6b62..5223c00279d5a 100644 --- a/include/net/tc_act/tc_gate.h +++ b/include/net/tc_act/tc_gate.h @@ -32,6 +32,7 @@ struct tcf_gate_params { s32 tcfg_clockid; size_t num_entries; struct list_head entries; + struct rcu_head rcu; }; #define GATE_ACT_GATE_OPEN BIT(0) @@ -39,7 +40,7 @@ struct tcf_gate_params { struct tcf_gate { struct tc_action common; - struct tcf_gate_params param; + struct tcf_gate_params __rcu *param; u8 current_gate_status; ktime_t current_close_time; u32 current_entry_octets; @@ -51,47 +52,65 @@ struct tcf_gate { #define to_gate(a) ((struct tcf_gate *)a) +static inline struct tcf_gate_params *tcf_gate_params_locked(const struct tc_action *a) +{ + struct tcf_gate *gact = to_gate(a); + + return rcu_dereference_protected(gact->param, + lockdep_is_held(&gact->tcf_lock)); +} + static inline s32 tcf_gate_prio(const struct tc_action *a) { + struct tcf_gate_params *p; s32 tcfg_prio; - tcfg_prio = to_gate(a)->param.tcfg_priority; + p = tcf_gate_params_locked(a); + tcfg_prio = p->tcfg_priority; return tcfg_prio; } static inline u64 tcf_gate_basetime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_basetime; - tcfg_basetime = to_gate(a)->param.tcfg_basetime; + p = tcf_gate_params_locked(a); + tcfg_basetime = p->tcfg_basetime; return tcfg_basetime; } static inline u64 tcf_gate_cycletime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletime; - tcfg_cycletime = to_gate(a)->param.tcfg_cycletime; + p = tcf_gate_params_locked(a); + tcfg_cycletime = p->tcfg_cycletime; return tcfg_cycletime; } static inline u64 tcf_gate_cycletimeext(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletimeext; - tcfg_cycletimeext = to_gate(a)->param.tcfg_cycletime_ext; + p = tcf_gate_params_locked(a); + tcfg_cycletimeext = p->tcfg_cycletime_ext; return tcfg_cycletimeext; } static inline u32 tcf_gate_num_entries(const struct tc_action *a) { + struct tcf_gate_params *p; u32 num_entries; - num_entries = to_gate(a)->param.num_entries; + p = tcf_gate_params_locked(a); + num_entries = p->num_entries; return num_entries; } @@ -105,7 +124,7 @@ static inline struct action_gate_entry u32 num_entries; int i = 0; - p = &to_gate(a)->param; + p = tcf_gate_params_locked(a); num_entries = p->num_entries; list_for_each_entry(entry, &p->entries, list) diff --git a/include/net/tc_act/tc_ife.h b/include/net/tc_act/tc_ife.h index c7f24a2da1cad..24d4d5a62b3c2 100644 --- a/include/net/tc_act/tc_ife.h +++ b/include/net/tc_act/tc_ife.h @@ -13,15 +13,13 @@ struct tcf_ife_params { u8 eth_src[ETH_ALEN]; u16 eth_type; u16 flags; - + struct list_head metalist; struct rcu_head rcu; }; struct tcf_ife_info { struct tc_action common; struct tcf_ife_params __rcu *params; - /* list of metaids allowed */ - struct list_head metalist; }; #define to_ife(a) ((struct tcf_ife_info *)a) diff --git a/include/net/tcp.h b/include/net/tcp.h index ab20f549b8f91..7647ed5c732c1 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -528,7 +529,9 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); int tcp_connect(struct sock *sk); @@ -1190,7 +1193,15 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED BIT(0) /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN BIT(1) -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Require successfully negotiated AccECN capability */ +#define TCP_CONG_NEEDS_ACCECN BIT(2) +/* Use ECT(1) instead of ECT(0) while the CA is uninitialized */ +#define TCP_CONG_ECT_1_NEGOTIATION BIT(3) +/* Cannot fallback to RFC3168 during AccECN negotiation */ +#define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ + TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ + TCP_CONG_NO_FALLBACK_RFC3168) union tcp_cc_info; @@ -1322,6 +1333,27 @@ static inline bool tcp_ca_needs_ecn(const struct sock *sk) return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; } +static inline bool tcp_ca_needs_accecn(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN; +} + +static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION; +} + +static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168; +} + static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -2404,8 +2436,9 @@ struct tcp_request_sock_ops { struct flowi *fl, struct request_sock *req, u32 tw_isn); - u32 (*init_seq)(const struct sk_buff *skb); - u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); + union tcp_seq_and_ts_off (*init_seq_and_ts_off)( + const struct net *net, + const struct sk_buff *skb); int (*send_synack)(const struct sock *sk, struct dst_entry *dst, struct flowi *fl, struct request_sock *req, struct tcp_fastopen_cookie *foc, diff --git a/include/net/tcp_ecn.h b/include/net/tcp_ecn.h index f13e5cd2b1ac3..a709fb1756eb7 100644 --- a/include/net/tcp_ecn.h +++ b/include/net/tcp_ecn.h @@ -31,6 +31,12 @@ enum tcp_accecn_option { TCP_ACCECN_OPTION_FULL = 2, }; +/* Apply either ECT(0) or ECT(1) based on TCP_CONG_ECT_1_NEGOTIATION flag */ +static inline void INET_ECN_xmit_ect_1_negotiation(struct sock *sk) +{ + __INET_ECN_xmit(sk, tcp_ca_ect_1_negotiation(sk)); +} + static inline void tcp_ecn_queue_cwr(struct tcp_sock *tp) { /* Do not set CWR if in AccECN mode! */ @@ -467,6 +473,26 @@ static inline u8 tcp_accecn_option_init(const struct sk_buff *skb, return TCP_ACCECN_OPT_COUNTER_SEEN; } +static inline void tcp_ecn_rcv_synack_accecn(struct sock *sk, + const struct sk_buff *skb, u8 dsf) +{ + struct tcp_sock *tp = tcp_sk(sk); + + tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); + tp->syn_ect_rcv = dsf & INET_ECN_MASK; + /* Demand Accurate ECN option in response to the SYN on the SYN/ACK + * and the TCP server will try to send one more packet with an AccECN + * Option at a later point during the connection. + */ + if (tp->rx_opt.accecn && + tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { + u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); + + tcp_accecn_saw_opt_fail_recv(tp, saw_opt); + tp->accecn_opt_demand = 2; + } +} + /* See Table 2 of the AccECN draft */ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb, const struct tcphdr *th, u8 ip_dsfield) @@ -489,32 +515,32 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); break; case 0x1: - case 0x5: /* +========+========+============+=============+ * | A | B | SYN/ACK | Feedback | * | | | B->A | Mode of A | * | | | AE CWR ECE | | * +========+========+============+=============+ - * | AccECN | Nonce | 1 0 1 | (Reserved) | * | AccECN | ECN | 0 0 1 | Classic ECN | * | Nonce | AccECN | 0 0 1 | Classic ECN | * | ECN | AccECN | 0 0 1 | Classic ECN | * +========+========+============+=============+ */ - if (tcp_ecn_mode_pending(tp)) - /* Downgrade from AccECN, or requested initially */ + if (tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); + else tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); break; - default: - tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); - tp->syn_ect_rcv = ip_dsfield & INET_ECN_MASK; - if (tp->rx_opt.accecn && - tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { - u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); - - tcp_accecn_saw_opt_fail_recv(tp, saw_opt); - tp->accecn_opt_demand = 2; + case 0x5: + if (tcp_ecn_mode_pending(tp)) { + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); + if (INET_ECN_is_ce(ip_dsfield)) { + tp->received_ce++; + tp->received_ce_pending++; + } } + break; + default: + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); if (INET_ECN_is_ce(ip_dsfield) && tcp_accecn_validate_syn_feedback(sk, ace, tp->syn_ect_snt)) { @@ -525,9 +551,11 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb } } -static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, +static inline void tcp_ecn_rcv_syn(struct sock *sk, const struct tcphdr *th, const struct sk_buff *skb) { + struct tcp_sock *tp = tcp_sk(sk); + if (tcp_ecn_mode_pending(tp)) { if (!tcp_accecn_syn_requested(th)) { /* Downgrade to classic ECN feedback */ @@ -539,7 +567,8 @@ static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); } } - if (tcp_ecn_mode_rfc3168(tp) && (!th->ece || !th->cwr)) + if (tcp_ecn_mode_rfc3168(tp) && + (!th->ece || !th->cwr || tcp_ca_no_fallback_rfc3168(sk))) tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } @@ -561,7 +590,7 @@ static inline void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; else if (tcp_ca_needs_ecn(sk) || tcp_bpf_ca_needs_ecn(sk)) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); if (tp->ecn_flags & TCP_ECN_MODE_ACCECN) { TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ACE; @@ -579,7 +608,8 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool use_ecn, use_accecn; u8 tcp_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn); - use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN; + use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN || + tcp_ca_needs_accecn(sk); use_ecn = tcp_ecn == TCP_ECN_IN_ECN_OUT_ECN || tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ECN || tcp_ca_needs_ecn(sk) || bpf_needs_ecn || use_accecn; @@ -595,7 +625,7 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) if (use_ecn) { if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; if (use_accecn) { diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index 9acef2fbd2fdc..d97ee26ba4f66 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -47,7 +47,7 @@ int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, struct socket **sockp) { - return 0; + return -EPFNOSUPPORT; } #endif diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index ce587a2256618..7c2bc46c67050 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -14,7 +14,7 @@ #include #include -#define XDP_UMEM_SG_FLAG (1 << 1) +#define XDP_UMEM_SG_FLAG BIT(3) struct net_device; struct xsk_queue; diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 4f2d3268a6769..dd1d3a6e1b780 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -37,16 +37,42 @@ static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool) return XDP_PACKET_HEADROOM + pool->headroom; } +static inline u32 xsk_pool_get_tailroom(bool mbuf) +{ + return mbuf ? SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : 0; +} + static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool) { return pool->chunk_size; } -static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) +static inline u32 __xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) { return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool); } +static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) +{ + u32 frame_size = __xsk_pool_get_rx_frame_size(pool); + struct xdp_umem *umem = pool->umem; + bool mbuf; + + /* Reserve tailroom only for zero-copy pools that opted into + * multi-buffer. The reserved area is used for skb_shared_info, + * matching the XDP core's xdp_data_hard_end() layout. + */ + mbuf = pool->dev && (umem->flags & XDP_UMEM_SG_FLAG); + frame_size -= xsk_pool_get_tailroom(mbuf); + + return ALIGN_DOWN(frame_size, 128); +} + +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return pool->unaligned ? 0 : xsk_pool_get_chunk_size(pool); +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { @@ -118,7 +144,7 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) goto out; list_for_each_entry_safe(pos, tmp, xskb_list, list_node) { - list_del(&pos->list_node); + list_del_init(&pos->list_node); xp_free(pos); } @@ -153,7 +179,7 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) frag = list_first_entry_or_null(&xskb->pool->xskb_list, struct xdp_buff_xsk, list_node); if (frag) { - list_del(&frag->list_node); + list_del_init(&frag->list_node); ret = &frag->xdp; } @@ -164,7 +190,7 @@ static inline void xsk_buff_del_frag(struct xdp_buff *xdp) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); - list_del(&xskb->list_node); + list_del_init(&xskb->list_node); } static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) @@ -333,6 +359,11 @@ static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) return 0; } +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return 0; +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { diff --git a/include/ras/ras_event.h b/include/ras/ras_event.h index c8cd0f00c8454..c9f0b1018bcce 100644 --- a/include/ras/ras_event.h +++ b/include/ras/ras_event.h @@ -168,11 +168,25 @@ TRACE_EVENT(mc_event, * This event is generated when hardware detects an ARM processor error * has occurred. UEFI 2.6 spec section N.2.4.4. */ +#define APEIL "ARM Processor Err Info data len" +#define APEID "ARM Processor Err Info raw data" +#define APECIL "ARM Processor Err Context Info data len" +#define APECID "ARM Processor Err Context Info raw data" +#define VSEIL "Vendor Specific Err Info data len" +#define VSEID "Vendor Specific Err Info raw data" TRACE_EVENT(arm_event, - TP_PROTO(const struct cper_sec_proc_arm *proc), + TP_PROTO(const struct cper_sec_proc_arm *proc, + const u8 *pei_err, + const u32 pei_len, + const u8 *ctx_err, + const u32 ctx_len, + const u8 *oem, + const u32 oem_len, + u8 sev, + int cpu), - TP_ARGS(proc), + TP_ARGS(proc, pei_err, pei_len, ctx_err, ctx_len, oem, oem_len, sev, cpu), TP_STRUCT__entry( __field(u64, mpidr) @@ -180,6 +194,14 @@ TRACE_EVENT(arm_event, __field(u32, running_state) __field(u32, psci_state) __field(u8, affinity) + __field(u32, pei_len) + __dynamic_array(u8, pei_buf, pei_len) + __field(u32, ctx_len) + __dynamic_array(u8, ctx_buf, ctx_len) + __field(u32, oem_len) + __dynamic_array(u8, oem_buf, oem_len) + __field(u8, sev) + __field(int, cpu) ), TP_fast_assign( @@ -199,12 +221,29 @@ TRACE_EVENT(arm_event, __entry->running_state = ~0; __entry->psci_state = ~0; } + __entry->pei_len = pei_len; + memcpy(__get_dynamic_array(pei_buf), pei_err, pei_len); + __entry->ctx_len = ctx_len; + memcpy(__get_dynamic_array(ctx_buf), ctx_err, ctx_len); + __entry->oem_len = oem_len; + memcpy(__get_dynamic_array(oem_buf), oem, oem_len); + __entry->sev = sev; + __entry->cpu = cpu; ), - TP_printk("affinity level: %d; MPIDR: %016llx; MIDR: %016llx; " - "running state: %d; PSCI state: %d", + TP_printk("cpu: %d; error: %d; affinity level: %d; MPIDR: %016llx; MIDR: %016llx; " + "running state: %d; PSCI state: %d; " + "%s: %d; %s: %s; %s: %d; %s: %s; %s: %d; %s: %s", + __entry->cpu, + __entry->sev, __entry->affinity, __entry->mpidr, __entry->midr, - __entry->running_state, __entry->psci_state) + __entry->running_state, __entry->psci_state, + APEIL, __entry->pei_len, APEID, + __print_hex(__get_dynamic_array(pei_buf), __entry->pei_len), + APECIL, __entry->ctx_len, APECID, + __print_hex(__get_dynamic_array(ctx_buf), __entry->ctx_len), + VSEIL, __entry->oem_len, VSEID, + __print_hex(__get_dynamic_array(oem_buf), __entry->oem_len)) ); /* diff --git a/include/rdma/rw.h b/include/rdma/rw.h index d606cac482338..9a8f4b76ce588 100644 --- a/include/rdma/rw.h +++ b/include/rdma/rw.h @@ -66,6 +66,8 @@ int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, unsigned int maxpages); +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags); void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr); int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr); void rdma_rw_cleanup_mrs(struct ib_qp *qp); diff --git a/include/scsi/scsi_eh.h b/include/scsi/scsi_eh.h index 1ae08e81339fa..15679be90c5c3 100644 --- a/include/scsi/scsi_eh.h +++ b/include/scsi/scsi_eh.h @@ -41,6 +41,12 @@ struct scsi_eh_save { unsigned char cmnd[32]; struct scsi_data_buffer sdb; struct scatterlist sense_sgl; + + /* struct request fields */ +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct bio_crypt_ctx *rq_crypt_ctx; + struct blk_crypto_keyslot *rq_crypt_keyslot; +#endif }; extern void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, diff --git a/include/soc/qcom/ice.h b/include/soc/qcom/ice.h index 4bee553f0a59d..4eb58a264d416 100644 --- a/include/soc/qcom/ice.h +++ b/include/soc/qcom/ice.h @@ -30,5 +30,7 @@ int qcom_ice_import_key(struct qcom_ice *ice, const u8 *raw_key, size_t raw_key_size, u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]); struct qcom_ice *devm_of_qcom_ice_get(struct device *dev); +int qcom_ice_scale_clk(struct qcom_ice *ice, unsigned long target_freq, + bool round_ceil); #endif /* __QCOM_ICE_H__ */ diff --git a/include/sound/cs35l56.h b/include/sound/cs35l56.h index ab044ce2aa8b3..0374c251901f5 100644 --- a/include/sound/cs35l56.h +++ b/include/sound/cs35l56.h @@ -344,6 +344,7 @@ extern const struct cirrus_amp_cal_controls cs35l56_calibration_controls; extern const char * const cs35l56_tx_input_texts[CS35L56_NUM_INPUT_SRC]; extern const unsigned int cs35l56_tx_input_values[CS35L56_NUM_INPUT_SRC]; +int cs35l56_set_asp_patch(struct cs35l56_base *cs35l56_base); int cs35l56_set_patch(struct cs35l56_base *cs35l56_base); int cs35l56_mbox_send(struct cs35l56_base *cs35l56_base, unsigned int command); int cs35l56_firmware_shutdown(struct cs35l56_base *cs35l56_base); diff --git a/include/sound/pcm.h b/include/sound/pcm.h index 58fd6e84f9613..a7860c047503a 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -1402,7 +1402,7 @@ int snd_pcm_lib_mmap_iomem(struct snd_pcm_substream *substream, struct vm_area_s #define snd_pcm_lib_mmap_iomem NULL #endif -void snd_pcm_runtime_buffer_set_silence(struct snd_pcm_runtime *runtime); +int snd_pcm_runtime_buffer_set_silence(struct snd_pcm_runtime *runtime); /** * snd_pcm_limit_isa_dma_size - Get the max size fitting with ISA DMA transfer diff --git a/include/sound/sdca_function.h b/include/sound/sdca_function.h index ea68856e4c8c4..86fd74146c33d 100644 --- a/include/sound/sdca_function.h +++ b/include/sound/sdca_function.h @@ -771,6 +771,7 @@ struct sdca_control { u8 layers; bool deferrable; + bool is_volatile; bool has_default; bool has_fixed; }; diff --git a/include/sound/soc.h b/include/sound/soc.h index ddc508ff7b9be..a9c058b06ab42 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -1225,6 +1225,7 @@ struct soc_mixer_control { unsigned int sign_bit; unsigned int invert:1; unsigned int autodisable:1; + unsigned int sdca_q78:1; #ifdef CONFIG_SND_SOC_TOPOLOGY struct snd_soc_dobj dobj; #endif diff --git a/include/sound/tas2781.h b/include/sound/tas2781.h index 0fbcdb15c74b5..29d15ba65f04d 100644 --- a/include/sound/tas2781.h +++ b/include/sound/tas2781.h @@ -197,7 +197,6 @@ struct tasdevice_priv { struct acoustic_data acou_data; #endif struct tasdevice_fw *fmw; - struct gpio_desc *speaker_id; struct gpio_desc *reset; struct mutex codec_lock; struct regmap *regmap; @@ -215,6 +214,7 @@ struct tasdevice_priv { unsigned int magic_num; unsigned int chip_id; unsigned int sysclk; + int speaker_id; int irq; int cur_prog; diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h index c4d9116904aa0..27e1f9d5f0c6c 100644 --- a/include/target/target_core_base.h +++ b/include/target/target_core_base.h @@ -671,9 +671,9 @@ struct se_lun_acl { }; struct se_dev_entry_io_stats { - u32 total_cmds; - u32 read_bytes; - u32 write_bytes; + u64 total_cmds; + u64 read_bytes; + u64 write_bytes; }; struct se_dev_entry { @@ -806,9 +806,9 @@ struct se_device_queue { }; struct se_dev_io_stats { - u32 total_cmds; - u32 read_bytes; - u32 write_bytes; + u64 total_cmds; + u64 read_bytes; + u64 write_bytes; }; struct se_device { diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 7e418f065b945..0864700f76e0a 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -224,7 +224,8 @@ DECLARE_EVENT_CLASS(btrfs__inode, __entry->generation = BTRFS_I(inode)->generation; __entry->last_trans = BTRFS_I(inode)->last_trans; __entry->logged_trans = BTRFS_I(inode)->logged_trans; - __entry->root_objectid = btrfs_root_id(BTRFS_I(inode)->root); + __entry->root_objectid = BTRFS_I(inode)->root ? + btrfs_root_id(BTRFS_I(inode)->root) : 0; ), TP_printk_btrfs("root=%llu(%s) gen=%llu ino=%llu blocks=%llu " @@ -768,12 +769,15 @@ TRACE_EVENT(btrfs_sync_file, ), TP_fast_assign( - const struct dentry *dentry = file->f_path.dentry; - const struct inode *inode = d_inode(dentry); + struct dentry *dentry = file_dentry(file); + struct inode *inode = file_inode(file); + struct dentry *parent = dget_parent(dentry); + struct inode *parent_inode = d_inode(parent); - TP_fast_assign_fsid(btrfs_sb(file->f_path.dentry->d_sb)); + dput(parent); + TP_fast_assign_fsid(btrfs_sb(inode->i_sb)); __entry->ino = btrfs_ino(BTRFS_I(inode)); - __entry->parent = btrfs_ino(BTRFS_I(d_inode(dentry->d_parent))); + __entry->parent = btrfs_ino(BTRFS_I(parent_inode)); __entry->datasync = datasync; __entry->root_objectid = btrfs_root_id(BTRFS_I(inode)->root); ), diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h index b3fef140ae155..69cb3805ee81c 100644 --- a/include/trace/events/dma.h +++ b/include/trace/events/dma.h @@ -32,7 +32,8 @@ TRACE_DEFINE_ENUM(DMA_NONE); { DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \ { DMA_ATTR_NO_WARN, "NO_WARN" }, \ { DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \ - { DMA_ATTR_MMIO, "MMIO" }) + { DMA_ATTR_MMIO, "MMIO" }, \ + { DMA_ATTR_CPU_CACHE_CLEAN, "CACHE_CLEAN" }) DECLARE_EVENT_CLASS(dma_map, TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, @@ -275,6 +276,8 @@ TRACE_EVENT(dma_free_sgt, sizeof(u64), sizeof(u64))) ); +#define DMA_TRACE_MAX_ENTRIES 128 + TRACE_EVENT(dma_map_sg, TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents, int ents, enum dma_data_direction dir, unsigned long attrs), @@ -282,9 +285,12 @@ TRACE_EVENT(dma_map_sg, TP_STRUCT__entry( __string(device, dev_name(dev)) - __dynamic_array(u64, phys_addrs, nents) - __dynamic_array(u64, dma_addrs, ents) - __dynamic_array(unsigned int, lengths, ents) + __field(int, full_nents) + __field(int, full_ents) + __field(bool, truncated) + __dynamic_array(u64, phys_addrs, min(nents, DMA_TRACE_MAX_ENTRIES)) + __dynamic_array(u64, dma_addrs, min(ents, DMA_TRACE_MAX_ENTRIES)) + __dynamic_array(unsigned int, lengths, min(ents, DMA_TRACE_MAX_ENTRIES)) __field(enum dma_data_direction, dir) __field(unsigned long, attrs) ), @@ -292,11 +298,16 @@ TRACE_EVENT(dma_map_sg, TP_fast_assign( struct scatterlist *sg; int i; + int traced_nents = min_t(int, nents, DMA_TRACE_MAX_ENTRIES); + int traced_ents = min_t(int, ents, DMA_TRACE_MAX_ENTRIES); __assign_str(device); - for_each_sg(sgl, sg, nents, i) + __entry->full_nents = nents; + __entry->full_ents = ents; + __entry->truncated = (nents > DMA_TRACE_MAX_ENTRIES) || (ents > DMA_TRACE_MAX_ENTRIES); + for_each_sg(sgl, sg, traced_nents, i) ((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg); - for_each_sg(sgl, sg, ents, i) { + for_each_sg(sgl, sg, traced_ents, i) { ((u64 *)__get_dynamic_array(dma_addrs))[i] = sg_dma_address(sg); ((unsigned int *)__get_dynamic_array(lengths))[i] = @@ -306,9 +317,12 @@ TRACE_EVENT(dma_map_sg, __entry->attrs = attrs; ), - TP_printk("%s dir=%s dma_addrs=%s sizes=%s phys_addrs=%s attrs=%s", + TP_printk("%s dir=%s nents=%d/%d ents=%d/%d%s dma_addrs=%s sizes=%s phys_addrs=%s attrs=%s", __get_str(device), decode_dma_data_direction(__entry->dir), + min_t(int, __entry->full_nents, DMA_TRACE_MAX_ENTRIES), __entry->full_nents, + min_t(int, __entry->full_ents, DMA_TRACE_MAX_ENTRIES), __entry->full_ents, + __entry->truncated ? " [TRUNCATED]" : "", __print_array(__get_dynamic_array(dma_addrs), __get_dynamic_array_len(dma_addrs) / sizeof(u64), sizeof(u64)), diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index 7f93e754da5c2..cd7920c81f85b 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -440,7 +440,13 @@ TRACE_EVENT(rss_stat, TP_fast_assign( __entry->mm_id = mm_ptr_to_hash(mm); - __entry->curr = !!(current->mm == mm); + /* + * curr is true if the mm matches the current task's mm_struct. + * Since kthreads (PF_KTHREAD) have no mm_struct of their own + * but can borrow one via kthread_use_mm(), we must filter them + * out to avoid incorrectly attributing the RSS update to them. + */ + __entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD); __entry->member = member; __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member]) << PAGE_SHIFT); diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index 64a382fbc31a8..cbe28211106c5 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -57,6 +57,7 @@ EM(netfs_rreq_trace_done, "DONE ") \ EM(netfs_rreq_trace_end_copy_to_cache, "END-C2C") \ EM(netfs_rreq_trace_free, "FREE ") \ + EM(netfs_rreq_trace_intr, "INTR ") \ EM(netfs_rreq_trace_ki_complete, "KI-CMPL") \ EM(netfs_rreq_trace_recollect, "RECLLCT") \ EM(netfs_rreq_trace_redirty, "REDIRTY") \ @@ -169,7 +170,8 @@ EM(netfs_sreq_trace_put_oom, "PUT OOM ") \ EM(netfs_sreq_trace_put_wip, "PUT WIP ") \ EM(netfs_sreq_trace_put_work, "PUT WORK ") \ - E_(netfs_sreq_trace_put_terminated, "PUT TERM ") + EM(netfs_sreq_trace_put_terminated, "PUT TERM ") \ + E_(netfs_sreq_trace_see_failed, "SEE FAILED ") #define netfs_folio_traces \ EM(netfs_folio_is_uptodate, "mod-uptodate") \ @@ -738,19 +740,19 @@ TRACE_EVENT(netfs_collect_stream, __field(unsigned int, wreq) __field(unsigned char, stream) __field(unsigned long long, collected_to) - __field(unsigned long long, front) + __field(unsigned long long, issued_to) ), TP_fast_assign( __entry->wreq = wreq->debug_id; __entry->stream = stream->stream_nr; __entry->collected_to = stream->collected_to; - __entry->front = stream->front ? stream->front->start : UINT_MAX; + __entry->issued_to = atomic64_read(&wreq->issued_to); ), - TP_printk("R=%08x[%x:] cto=%llx frn=%llx", + TP_printk("R=%08x[%x:] cto=%llx ito=%llx", __entry->wreq, __entry->stream, - __entry->collected_to, __entry->front) + __entry->collected_to, __entry->issued_to) ); TRACE_EVENT(netfs_folioq, diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h index de6f6d25767c6..578b8038b2117 100644 --- a/include/trace/events/rxrpc.h +++ b/include/trace/events/rxrpc.h @@ -185,6 +185,7 @@ EM(rxrpc_skb_put_input, "PUT input ") \ EM(rxrpc_skb_put_jumbo_subpacket, "PUT jumbo-sub") \ EM(rxrpc_skb_put_oob, "PUT oob ") \ + EM(rxrpc_skb_put_old_response, "PUT old-resp ") \ EM(rxrpc_skb_put_purge, "PUT purge ") \ EM(rxrpc_skb_put_purge_oob, "PUT purge-oob") \ EM(rxrpc_skb_put_response, "PUT response ") \ @@ -322,6 +323,7 @@ EM(rxrpc_call_put_kernel, "PUT kernel ") \ EM(rxrpc_call_put_poke, "PUT poke ") \ EM(rxrpc_call_put_recvmsg, "PUT recvmsg ") \ + EM(rxrpc_call_put_recvmsg_peek_nowait, "PUT peek-nwt") \ EM(rxrpc_call_put_release_recvmsg_q, "PUT rls-rcmq") \ EM(rxrpc_call_put_release_sock, "PUT rls-sock") \ EM(rxrpc_call_put_release_sock_tba, "PUT rls-sk-a") \ @@ -340,10 +342,13 @@ EM(rxrpc_call_see_input, "SEE input ") \ EM(rxrpc_call_see_notify_released, "SEE nfy-rlsd") \ EM(rxrpc_call_see_recvmsg, "SEE recvmsg ") \ + EM(rxrpc_call_see_recvmsg_requeue, "SEE recv-rqu") \ + EM(rxrpc_call_see_recvmsg_requeue_first, "SEE recv-rqF") \ + EM(rxrpc_call_see_recvmsg_requeue_move, "SEE recv-rqM") \ EM(rxrpc_call_see_release, "SEE release ") \ EM(rxrpc_call_see_userid_exists, "SEE u-exists") \ EM(rxrpc_call_see_waiting_call, "SEE q-conn ") \ - E_(rxrpc_call_see_zap, "SEE zap ") + E_(rxrpc_call_see_still_live, "SEE !still-l") #define rxrpc_txqueue_traces \ EM(rxrpc_txqueue_await_reply, "AWR") \ @@ -516,6 +521,7 @@ #define rxrpc_req_ack_traces \ EM(rxrpc_reqack_ack_lost, "ACK-LOST ") \ EM(rxrpc_reqack_app_stall, "APP-STALL ") \ + EM(rxrpc_reqack_jumbo_win, "JUMBO-WIN ") \ EM(rxrpc_reqack_more_rtt, "MORE-RTT ") \ EM(rxrpc_reqack_no_srv_last, "NO-SRVLAST") \ EM(rxrpc_reqack_old_rtt, "OLD-RTT ") \ diff --git a/include/trace/events/task.h b/include/trace/events/task.h index 4f0759634306c..b9a129eb54d9e 100644 --- a/include/trace/events/task.h +++ b/include/trace/events/task.h @@ -38,19 +38,22 @@ TRACE_EVENT(task_rename, TP_ARGS(task, comm), TP_STRUCT__entry( + __field( pid_t, pid) __array( char, oldcomm, TASK_COMM_LEN) __array( char, newcomm, TASK_COMM_LEN) __field( short, oom_score_adj) ), TP_fast_assign( + __entry->pid = task->pid; memcpy(entry->oldcomm, task->comm, TASK_COMM_LEN); strscpy(entry->newcomm, comm, TASK_COMM_LEN); __entry->oom_score_adj = task->signal->oom_score_adj; ), - TP_printk("oldcomm=%s newcomm=%s oom_score_adj=%hd", - __entry->oldcomm, __entry->newcomm, __entry->oom_score_adj) + TP_printk("pid=%d oldcomm=%s newcomm=%s oom_score_adj=%hd", + __entry->pid, __entry->oldcomm, + __entry->newcomm, __entry->oom_score_adj) ); /** diff --git a/include/trace/events/tlb.h b/include/trace/events/tlb.h index b4d8e7dc38f88..fb83695116856 100644 --- a/include/trace/events/tlb.h +++ b/include/trace/events/tlb.h @@ -12,8 +12,9 @@ EM( TLB_FLUSH_ON_TASK_SWITCH, "flush on task switch" ) \ EM( TLB_REMOTE_SHOOTDOWN, "remote shootdown" ) \ EM( TLB_LOCAL_SHOOTDOWN, "local shootdown" ) \ - EM( TLB_LOCAL_MM_SHOOTDOWN, "local mm shootdown" ) \ - EMe( TLB_REMOTE_SEND_IPI, "remote ipi send" ) + EM( TLB_LOCAL_MM_SHOOTDOWN, "local MM shootdown" ) \ + EM( TLB_REMOTE_SEND_IPI, "remote IPI send" ) \ + EMe( TLB_REMOTE_WRONG_CPU, "remote wrong CPU" ) /* * First define the enums in TLB_FLUSH_REASON to be exported to userspace diff --git a/include/trace/misc/nfs.h b/include/trace/misc/nfs.h index c82233e950ac0..a394b4d38e18f 100644 --- a/include/trace/misc/nfs.h +++ b/include/trace/misc/nfs.h @@ -16,7 +16,6 @@ TRACE_DEFINE_ENUM(NFSERR_PERM); TRACE_DEFINE_ENUM(NFSERR_NOENT); TRACE_DEFINE_ENUM(NFSERR_IO); TRACE_DEFINE_ENUM(NFSERR_NXIO); -TRACE_DEFINE_ENUM(NFSERR_EAGAIN); TRACE_DEFINE_ENUM(NFSERR_ACCES); TRACE_DEFINE_ENUM(NFSERR_EXIST); TRACE_DEFINE_ENUM(NFSERR_XDEV); @@ -52,7 +51,6 @@ TRACE_DEFINE_ENUM(NFSERR_JUKEBOX); { NFSERR_NXIO, "NXIO" }, \ { ECHILD, "CHILD" }, \ { ETIMEDOUT, "TIMEDOUT" }, \ - { NFSERR_EAGAIN, "AGAIN" }, \ { NFSERR_ACCES, "ACCES" }, \ { NFSERR_EXIST, "EXIST" }, \ { NFSERR_XDEV, "XDEV" }, \ diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h index e527b24bd824b..c89aede3cb120 100644 --- a/include/uapi/drm/drm_fourcc.h +++ b/include/uapi/drm/drm_fourcc.h @@ -401,8 +401,8 @@ extern "C" { * implementation can multiply the values by 2^6=64. For that reason the padding * must only contain zeros. * index 0 = Y plane, [15:0] z:Y [6:10] little endian - * index 1 = Cr plane, [15:0] z:Cr [6:10] little endian - * index 2 = Cb plane, [15:0] z:Cb [6:10] little endian + * index 1 = Cb plane, [15:0] z:Cb [6:10] little endian + * index 2 = Cr plane, [15:0] z:Cr [6:10] little endian */ #define DRM_FORMAT_S010 fourcc_code('S', '0', '1', '0') /* 2x2 subsampled Cb (1) and Cr (2) planes 10 bits per channel */ #define DRM_FORMAT_S210 fourcc_code('S', '2', '1', '0') /* 2x1 subsampled Cb (1) and Cr (2) planes 10 bits per channel */ @@ -414,8 +414,8 @@ extern "C" { * implementation can multiply the values by 2^4=16. For that reason the padding * must only contain zeros. * index 0 = Y plane, [15:0] z:Y [4:12] little endian - * index 1 = Cr plane, [15:0] z:Cr [4:12] little endian - * index 2 = Cb plane, [15:0] z:Cb [4:12] little endian + * index 1 = Cb plane, [15:0] z:Cb [4:12] little endian + * index 2 = Cr plane, [15:0] z:Cr [4:12] little endian */ #define DRM_FORMAT_S012 fourcc_code('S', '0', '1', '2') /* 2x2 subsampled Cb (1) and Cr (2) planes 12 bits per channel */ #define DRM_FORMAT_S212 fourcc_code('S', '2', '1', '2') /* 2x1 subsampled Cb (1) and Cr (2) planes 12 bits per channel */ @@ -424,8 +424,8 @@ extern "C" { /* * 3 plane YCbCr * index 0 = Y plane, [15:0] Y little endian - * index 1 = Cr plane, [15:0] Cr little endian - * index 2 = Cb plane, [15:0] Cb little endian + * index 1 = Cb plane, [15:0] Cb little endian + * index 2 = Cr plane, [15:0] Cr little endian */ #define DRM_FORMAT_S016 fourcc_code('S', '0', '1', '6') /* 2x2 subsampled Cb (1) and Cr (2) planes 16 bits per channel */ #define DRM_FORMAT_S216 fourcc_code('S', '2', '1', '6') /* 2x1 subsampled Cb (1) and Cr (2) planes 16 bits per channel */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 517489a7ec60a..400555a8af187 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1459,6 +1459,7 @@ struct drm_xe_exec { /** @exec_queue_id: Exec queue ID for the batch buffer */ __u32 exec_queue_id; +#define DRM_XE_MAX_SYNCS 1024 /** @num_syncs: Amount of struct drm_xe_sync in array. */ __u32 num_syncs; diff --git a/include/uapi/linux/comedi.h b/include/uapi/linux/comedi.h index 7314e5ee0a1e8..798ec9a39e129 100644 --- a/include/uapi/linux/comedi.h +++ b/include/uapi/linux/comedi.h @@ -640,7 +640,7 @@ struct comedi_chaninfo { /** * struct comedi_rangeinfo - used to retrieve the range table for a channel - * @range_type: Encodes subdevice index (bits 27:24), channel index + * @range_type: Encodes subdevice index (bits 31:24), channel index * (bits 23:16) and range table length (bits 15:0). * @range_ptr: Pointer to array of @struct comedi_krange to be filled * in with the range table for the channel or subdevice. diff --git a/include/uapi/linux/dma-buf.h b/include/uapi/linux/dma-buf.h index 5a6fda66d9adf..e827c9d20c5d3 100644 --- a/include/uapi/linux/dma-buf.h +++ b/include/uapi/linux/dma-buf.h @@ -20,6 +20,7 @@ #ifndef _DMA_BUF_UAPI_H_ #define _DMA_BUF_UAPI_H_ +#include #include /** diff --git a/include/uapi/linux/dpll.h b/include/uapi/linux/dpll.h index ab1725a954d74..69d35570ac4f1 100644 --- a/include/uapi/linux/dpll.h +++ b/include/uapi/linux/dpll.h @@ -251,6 +251,7 @@ enum dpll_a_pin { DPLL_A_PIN_ESYNC_FREQUENCY_SUPPORTED, DPLL_A_PIN_ESYNC_PULSE, DPLL_A_PIN_REFERENCE_SYNC, + DPLL_A_PIN_PHASE_ADJUST_GRAN, __DPLL_A_PIN_MAX, DPLL_A_PIN_MAX = (__DPLL_A_PIN_MAX - 1) diff --git a/include/uapi/linux/ext4.h b/include/uapi/linux/ext4.h index 411dcc1e4a35c..9c683991c32f9 100644 --- a/include/uapi/linux/ext4.h +++ b/include/uapi/linux/ext4.h @@ -139,7 +139,7 @@ struct ext4_tune_sb_params { __u32 clear_feature_incompat_mask; __u32 clear_feature_ro_compat_mask; __u8 mount_opts[64]; - __u8 pad[64]; + __u8 pad[68]; }; #define EXT4_TUNE_FL_ERRORS_BEHAVIOR 0x00000001 diff --git a/include/uapi/linux/hyperv.h b/include/uapi/linux/hyperv.h index aaa502a7bff46..1749b35ab2c21 100644 --- a/include/uapi/linux/hyperv.h +++ b/include/uapi/linux/hyperv.h @@ -362,7 +362,7 @@ struct hv_kvp_exchg_msg_value { __u8 value[HV_KVP_EXCHANGE_MAX_VALUE_SIZE]; __u32 value_u32; __u64 value_u64; - }; + } __attribute__((packed)); } __attribute__((packed)); struct hv_kvp_msg_enumerate { diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 52f6000ab0208..954e0511ce91c 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -11,9 +11,14 @@ #include #include #include +#include #include #include +#ifdef __KERNEL__ +#include +#endif + #define KVM_API_VERSION 12 /* @@ -519,7 +524,7 @@ struct kvm_coalesced_mmio { struct kvm_coalesced_mmio_ring { __u32 first, last; - struct kvm_coalesced_mmio coalesced_mmio[]; + __DECLARE_FLEX_ARRAY(struct kvm_coalesced_mmio, coalesced_mmio); }; #define KVM_COALESCED_MMIO_MAX \ @@ -569,7 +574,7 @@ struct kvm_clear_dirty_log { /* for KVM_SET_SIGNAL_MASK */ struct kvm_signal_mask { __u32 len; - __u8 sigset[]; + __DECLARE_FLEX_ARRAY(__u8, sigset); }; /* for KVM_TPR_ACCESS_REPORTING */ @@ -1025,7 +1030,7 @@ struct kvm_irq_routing_entry { struct kvm_irq_routing { __u32 nr; __u32 flags; - struct kvm_irq_routing_entry entries[]; + __DECLARE_FLEX_ARRAY(struct kvm_irq_routing_entry, entries); }; #define KVM_IRQFD_FLAG_DEASSIGN (1 << 0) @@ -1116,7 +1121,7 @@ struct kvm_dirty_tlb { struct kvm_reg_list { __u64 n; /* number of regs */ - __u64 reg[]; + __DECLARE_FLEX_ARRAY(__u64, reg); }; struct kvm_one_reg { @@ -1568,7 +1573,11 @@ struct kvm_stats_desc { __u16 size; __u32 offset; __u32 bucket_size; - char name[]; +#ifdef __KERNEL__ + char name[KVM_STATS_NAME_SIZE]; +#else + __DECLARE_FLEX_ARRAY(char, name); +#endif }; #define KVM_GET_STATS_FD _IO(KVMIO, 0xce) diff --git a/include/uapi/linux/mptcp.h b/include/uapi/linux/mptcp.h index 87cfab874e241..49e9b4e98c8b4 100644 --- a/include/uapi/linux/mptcp.h +++ b/include/uapi/linux/mptcp.h @@ -40,6 +40,7 @@ #define MPTCP_PM_ADDR_FLAG_FULLMESH _BITUL(3) #define MPTCP_PM_ADDR_FLAG_IMPLICIT _BITUL(4) #define MPTCP_PM_ADDR_FLAG_LAMINAR _BITUL(5) +#define MPTCP_PM_ADDR_FLAGS_MASK GENMASK(5, 0) struct mptcp_info { __u8 mptcpi_subflows; diff --git a/include/uapi/linux/netfilter/nf_conntrack_common.h b/include/uapi/linux/netfilter/nf_conntrack_common.h index 26071021e986f..56b6b60a814f5 100644 --- a/include/uapi/linux/netfilter/nf_conntrack_common.h +++ b/include/uapi/linux/netfilter/nf_conntrack_common.h @@ -159,5 +159,9 @@ enum ip_conntrack_expect_events { #define NF_CT_EXPECT_INACTIVE 0x2 #define NF_CT_EXPECT_USERSPACE 0x4 +#ifdef __KERNEL__ +#define NF_CT_EXPECT_MASK (NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE | \ + NF_CT_EXPECT_USERSPACE) +#endif #endif /* _UAPI_NF_CONNTRACK_COMMON_H */ diff --git a/include/uapi/linux/netfilter_bridge.h b/include/uapi/linux/netfilter_bridge.h index 1610fdbab98df..ad520d3e9df8f 100644 --- a/include/uapi/linux/netfilter_bridge.h +++ b/include/uapi/linux/netfilter_bridge.h @@ -5,6 +5,10 @@ /* bridge-specific defines for netfilter. */ +#ifndef __KERNEL__ +#include /* for __UAPI_DEF_ETHHDR if defined */ +#endif + #include #include #include diff --git a/include/uapi/linux/nfs.h b/include/uapi/linux/nfs.h index f356f2ba38142..e629c49535345 100644 --- a/include/uapi/linux/nfs.h +++ b/include/uapi/linux/nfs.h @@ -49,14 +49,13 @@ NFSERR_NOENT = 2, /* v2 v3 v4 */ NFSERR_IO = 5, /* v2 v3 v4 */ NFSERR_NXIO = 6, /* v2 v3 v4 */ - NFSERR_EAGAIN = 11, /* v2 v3 */ NFSERR_ACCES = 13, /* v2 v3 v4 */ NFSERR_EXIST = 17, /* v2 v3 v4 */ NFSERR_XDEV = 18, /* v3 v4 */ NFSERR_NODEV = 19, /* v2 v3 v4 */ NFSERR_NOTDIR = 20, /* v2 v3 v4 */ NFSERR_ISDIR = 21, /* v2 v3 v4 */ - NFSERR_INVAL = 22, /* v2 v3 v4 */ + NFSERR_INVAL = 22, /* v3 v4 */ NFSERR_FBIG = 27, /* v2 v3 v4 */ NFSERR_NOSPC = 28, /* v2 v3 v4 */ NFSERR_ROFS = 30, /* v2 v3 v4 */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 07e06aafec502..1172bda87abff 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -706,7 +706,7 @@ #define PCI_EXP_LNKCTL2_HASD 0x0020 /* HW Autonomous Speed Disable */ #define PCI_EXP_LNKSTA2 0x32 /* Link Status 2 */ #define PCI_EXP_LNKSTA2_FLIT 0x0400 /* Flit Mode Status */ -#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 0x32 /* end of v2 EPs w/ link */ +#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 0x34 /* end of v2 EPs w/ link */ #define PCI_EXP_SLTCAP2 0x34 /* Slot Capabilities 2 */ #define PCI_EXP_SLTCAP2_IBPD 0x00000001 /* In-band PD Disable Supported */ #define PCI_EXP_SLTCTL2 0x38 /* Slot Control 2 */ diff --git a/include/uapi/linux/pidfd.h b/include/uapi/linux/pidfd.h index 957db425d459a..6ccbabd9a68d8 100644 --- a/include/uapi/linux/pidfd.h +++ b/include/uapi/linux/pidfd.h @@ -28,6 +28,7 @@ #define PIDFD_INFO_COREDUMP (1UL << 4) /* Only returned if requested. */ #define PIDFD_INFO_SIZE_VER0 64 /* sizeof first published struct */ +#define PIDFD_INFO_SIZE_VER1 72 /* sizeof second published struct */ /* * Values for @coredump_mask in pidfd_info. diff --git a/include/uapi/linux/vbox_vmmdev_types.h b/include/uapi/linux/vbox_vmmdev_types.h index 6073858d52a2e..11f3627c3729b 100644 --- a/include/uapi/linux/vbox_vmmdev_types.h +++ b/include/uapi/linux/vbox_vmmdev_types.h @@ -236,7 +236,7 @@ struct vmmdev_hgcm_function_parameter32 { /** Relative to the request header. */ __u32 offset; } page_list; - } u; + } __packed u; } __packed; VMMDEV_ASSERT_SIZE(vmmdev_hgcm_function_parameter32, 4 + 8); @@ -251,7 +251,7 @@ struct vmmdev_hgcm_function_parameter64 { union { __u64 phys_addr; __u64 linear_addr; - } u; + } __packed u; } __packed pointer; struct { /** Size of the buffer described by the page list. */ diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index becd08fdbddb8..cf0b71bbe0f9d 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -775,6 +775,7 @@ struct v4l2_pix_format { #define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4') /* H264 parsed slices */ #define V4L2_PIX_FMT_HEVC_SLICE v4l2_fourcc('S', '2', '6', '5') /* HEVC parsed slices */ #define V4L2_PIX_FMT_AV1_FRAME v4l2_fourcc('A', 'V', '1', 'F') /* AV1 parsed frame */ +#define V4L2_PIX_FMT_AV1 v4l2_fourcc('A', 'V', '0', '1') /* AV1 */ #define V4L2_PIX_FMT_SPK v4l2_fourcc('S', 'P', 'K', '0') /* Sorenson Spark */ #define V4L2_PIX_FMT_RV30 v4l2_fourcc('R', 'V', '3', '0') /* RealVideo 8 */ #define V4L2_PIX_FMT_RV40 v4l2_fourcc('R', 'V', '4', '0') /* RealVideo 9 & 10 */ diff --git a/include/uapi/rdma/irdma-abi.h b/include/uapi/rdma/irdma-abi.h index f7788d33376b8..36f20802bcc84 100644 --- a/include/uapi/rdma/irdma-abi.h +++ b/include/uapi/rdma/irdma-abi.h @@ -57,8 +57,8 @@ struct irdma_alloc_ucontext_resp { __u8 rsvd2; __aligned_u64 comp_mask; __u16 min_hw_wq_size; + __u8 revd3[2]; __u32 max_hw_srq_quanta; - __u8 rsvd3[2]; }; struct irdma_alloc_pd_resp { diff --git a/include/uapi/rdma/rdma_user_cm.h b/include/uapi/rdma/rdma_user_cm.h index 5ded174687ee0..838f8d4602560 100644 --- a/include/uapi/rdma/rdma_user_cm.h +++ b/include/uapi/rdma/rdma_user_cm.h @@ -192,6 +192,7 @@ struct rdma_ucm_query_path_resp { struct rdma_ucm_query_ib_service_resp { __u32 num_service_recs; + __u32 reserved; struct ib_user_service_rec recs[]; }; @@ -354,7 +355,7 @@ enum { #define RDMA_USER_CM_IB_SERVICE_NAME_SIZE 64 struct rdma_ucm_ib_service { - __u64 service_id; + __aligned_u64 service_id; __u8 service_name[RDMA_USER_CM_IB_SERVICE_NAME_SIZE]; __u32 flags; __u32 reserved; @@ -362,6 +363,7 @@ struct rdma_ucm_ib_service { struct rdma_ucm_resolve_ib_service { __u32 id; + __u32 reserved; struct rdma_ucm_ib_service ibs; }; diff --git a/include/uapi/sound/asound.h b/include/uapi/sound/asound.h index 5a049eeaeccea..d3ce75ba938a8 100644 --- a/include/uapi/sound/asound.h +++ b/include/uapi/sound/asound.h @@ -60,7 +60,7 @@ struct snd_cea_861_aud_if { unsigned char db2_sf_ss; /* sample frequency and size */ unsigned char db3; /* not used, all zeros */ unsigned char db4_ca; /* channel allocation code */ - unsigned char db5_dminh_lsv; /* downmix inhibit & level-shit values */ + unsigned char db5_dminh_lsv; /* downmix inhibit & level-shift values */ }; /**************************************************************************** diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 0f95576bf1f6c..17fe07dac6a7e 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1302,7 +1302,6 @@ static inline void ufshcd_rmwl(struct ufs_hba *hba, u32 mask, u32 val, u32 reg) void ufshcd_enable_irq(struct ufs_hba *hba); void ufshcd_disable_irq(struct ufs_hba *hba); -void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs); int ufshcd_alloc_host(struct device *, struct ufs_hba **); int ufshcd_hba_enable(struct ufs_hba *hba); int ufshcd_init(struct ufs_hba *, void __iomem *, unsigned int); @@ -1351,17 +1350,13 @@ static inline void *ufshcd_get_variant(struct ufs_hba *hba) return hba->priv; } -#ifdef CONFIG_PM extern int ufshcd_runtime_suspend(struct device *dev); extern int ufshcd_runtime_resume(struct device *dev); -#endif -#ifdef CONFIG_PM_SLEEP extern int ufshcd_system_suspend(struct device *dev); extern int ufshcd_system_resume(struct device *dev); extern int ufshcd_system_freeze(struct device *dev); extern int ufshcd_system_thaw(struct device *dev); extern int ufshcd_system_restore(struct device *dev); -#endif extern int ufshcd_dme_reset(struct ufs_hba *hba); extern int ufshcd_dme_enable(struct ufs_hba *hba); diff --git a/include/xen/xen.h b/include/xen/xen.h index 61854e3f28377..f280c5dcf9236 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -69,11 +69,13 @@ extern u64 xen_saved_max_mem_size; #endif #ifdef CONFIG_XEN_UNPOPULATED_ALLOC +extern unsigned long xen_unpopulated_pages; int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages); void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages); #include int arch_xen_unpopulated_init(struct resource **res); #else +#define xen_unpopulated_pages 0UL #include static inline int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 43e9bb74e9d19..eaa4069e258c9 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -6,10 +6,8 @@ struct io_cancel_data { struct io_ring_ctx *ctx; - union { - u64 data; - struct file *file; - }; + u64 data; + struct file *file; u8 opcode; u32 flags; int seq; diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c index 3b75931bd5695..54d05a205e623 100644 --- a/io_uring/cmd_net.c +++ b/io_uring/cmd_net.c @@ -139,16 +139,19 @@ int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags) struct proto *prot = READ_ONCE(sk->sk_prot); int ret, arg = 0; - if (!prot || !prot->ioctl) - return -EOPNOTSUPP; - switch (cmd->cmd_op) { case SOCKET_URING_OP_SIOCINQ: + if (!prot || !prot->ioctl) + return -EOPNOTSUPP; + ret = prot->ioctl(sk, SIOCINQ, &arg); if (ret) return ret; return arg; case SOCKET_URING_OP_SIOCOUTQ: + if (!prot || !prot->ioctl) + return -EOPNOTSUPP; + ret = prot->ioctl(sk, SIOCOUTQ, &arg); if (ret) return ret; diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index 78f8ab7db104f..ab789e1ebe91e 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -76,11 +76,15 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) { bool skip = false; struct io_ev_fd *ev_fd; - - if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) - return; + struct io_rings *rings; guard(rcu)(); + + rings = rcu_dereference(ctx->rings_rcu); + if (!rings) + return; + if (READ_ONCE(rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) + return; ev_fd = rcu_dereference(ctx->io_ev_fd); /* * Check again if ev_fd exists in case an io_eventfd_unregister call diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 294c75a8a3bdb..3585ad8308504 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -65,7 +65,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) unsigned int cq_head = READ_ONCE(r->cq.head); unsigned int cq_tail = READ_ONCE(r->cq.tail); unsigned int sq_shift = 0; - unsigned int sq_entries; + unsigned int cq_entries, sq_entries; int sq_pid = -1, sq_cpu = -1; u64 sq_total_time = 0, sq_work_time = 0; unsigned int i; @@ -119,9 +119,11 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) } } seq_printf(m, "\n"); + cond_resched(); } seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head); - while (cq_head < cq_tail) { + cq_entries = min(cq_tail - cq_head, ctx->cq_entries); + for (i = 0; i < cq_entries; i++) { struct io_uring_cqe *cqe; bool cqe32 = false; @@ -136,8 +138,11 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) cqe->big_cqe[0], cqe->big_cqe[1]); seq_printf(m, "\n"); cq_head++; - if (cqe32) + if (cqe32) { cq_head++; + i++; + } + cond_resched(); } if (ctx->flags & IORING_SETUP_SQPOLL) { diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 794ef95df293c..cb1838c9fc377 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -22,6 +22,10 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) if (!table->bitmap) return -ENFILE; + if (table->alloc_hint < ctx->file_alloc_start || + table->alloc_hint >= ctx->file_alloc_end) + table->alloc_hint = ctx->file_alloc_start; + do { ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint); if (ret != nr) diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 1d03b2fc4b259..49a9c914b4e97 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -34,6 +34,7 @@ enum { enum { IO_WQ_BIT_EXIT = 0, /* wq exiting */ + IO_WQ_BIT_EXIT_ON_IDLE = 1, /* allow all workers to exit on idle */ }; enum { @@ -598,9 +599,9 @@ static void io_worker_handle_work(struct io_wq_acct *acct, __releases(&acct->lock) { struct io_wq *wq = worker->wq; - bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); do { + bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); struct io_wq_work *work; /* @@ -706,9 +707,13 @@ static int io_wq_worker(void *data) raw_spin_lock(&acct->workers_lock); /* * Last sleep timed out. Exit if we're not the last worker, - * or if someone modified our affinity. + * or if someone modified our affinity. If wq is marked + * idle-exit, drop the worker as well. This is used to avoid + * keeping io-wq workers around for tasks that no longer have + * any active io_uring instances. */ - if (last_timeout && (exit_mask || acct->nr_workers > 1)) { + if ((last_timeout && (exit_mask || acct->nr_workers > 1)) || + test_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) { acct->nr_workers--; raw_spin_unlock(&acct->workers_lock); __set_current_state(TASK_RUNNING); @@ -951,11 +956,11 @@ static bool io_wq_for_each_worker(struct io_wq *wq, void *data) { for (int i = 0; i < IO_WQ_ACCT_NR; i++) { - if (!io_acct_for_each_worker(&wq->acct[i], func, data)) - return false; + if (io_acct_for_each_worker(&wq->acct[i], func, data)) + return true; } - return true; + return false; } static bool io_wq_worker_wake(struct io_worker *worker, void *data) @@ -965,6 +970,24 @@ static bool io_wq_worker_wake(struct io_worker *worker, void *data) return false; } +void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable) +{ + if (!wq->task) + return; + + if (!enable) { + clear_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state); + return; + } + + if (test_and_set_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) + return; + + rcu_read_lock(); + io_wq_for_each_worker(wq, io_wq_worker_wake, NULL); + rcu_read_unlock(); +} + static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq) { do { diff --git a/io_uring/io-wq.h b/io_uring/io-wq.h index 774abab54732e..94b14742b7032 100644 --- a/io_uring/io-wq.h +++ b/io_uring/io-wq.h @@ -41,6 +41,7 @@ struct io_wq_data { struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); void io_wq_exit_start(struct io_wq *wq); void io_wq_put_and_exit(struct io_wq *wq); +void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 02339b74ba8d4..99cd5bcac201d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -148,7 +148,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags); static void __io_req_caches_free(struct io_ring_ctx *ctx); -static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray); +static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ); struct kmem_cache *req_cachep; static struct workqueue_struct *iou_wq __ro_after_init; @@ -199,12 +199,15 @@ static void io_poison_req(struct io_kiocb *req) static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) { - return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); + struct io_rings *rings = io_get_rings(ctx); + return ctx->cached_cq_tail - READ_ONCE(rings->cq.head); } static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) { - return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head); + struct io_rings *rings = io_get_rings(ctx); + + return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head); } static bool io_match_linked(struct io_kiocb *head) @@ -897,7 +900,7 @@ static __cold bool io_cqe_overflow_locked(struct io_ring_ctx *ctx, { struct io_overflow_cqe *ocqe; - ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_ATOMIC); + ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_NOWAIT); return io_cqring_add_overflow(ctx, ocqe); } @@ -1238,6 +1241,21 @@ void tctx_task_work(struct callback_head *cb) WARN_ON_ONCE(ret); } +/* + * Sets IORING_SQ_TASKRUN in the sq_flags shared with userspace, using the + * RCU protected rings pointer to be safe against concurrent ring resizing. + */ +static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx) +{ + lockdep_assert_in_rcu_read_lock(); + + if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) { + struct io_rings *rings = rcu_dereference(ctx->rings_rcu); + + atomic_or(IORING_SQ_TASKRUN, &rings->sq_flags); + } +} + static void io_req_local_work_add(struct io_kiocb *req, unsigned flags) { struct io_ring_ctx *ctx = req->ctx; @@ -1292,8 +1310,7 @@ static void io_req_local_work_add(struct io_kiocb *req, unsigned flags) */ if (!head) { - if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) - atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); + io_ctx_mark_taskrun(ctx); if (ctx->has_evfd) io_eventfd_signal(ctx, false); } @@ -1317,6 +1334,10 @@ static void io_req_normal_work_add(struct io_kiocb *req) if (!llist_add(&req->io_task_work.node, &tctx->task_list)) return; + /* + * Doesn't need to use ->rings_rcu, as resizing isn't supported for + * !DEFER_TASKRUN. + */ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); @@ -2390,7 +2411,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) unsigned mask = ctx->sq_entries - 1; unsigned head = ctx->cached_sq_head++ & mask; - if (static_branch_unlikely(&io_key_has_sqarray) && + if (static_branch_unlikely(&io_key_has_sqarray.key) && (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) { head = READ_ONCE(ctx->sq_array[head]); if (unlikely(head >= ctx->sq_entries)) { @@ -2532,12 +2553,15 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) if (io_has_work(ctx)) goto out_wake; /* got events since we started waiting, min timeout is done */ - if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail)) - goto out_wake; - /* if we have any events and min timeout expired, we're done */ - if (io_cqring_events(ctx)) - goto out_wake; + scoped_guard(rcu) { + struct io_rings *rings = io_get_rings(ctx); + if (iowq->cq_min_tail != READ_ONCE(rings->cq.tail)) + goto out_wake; + /* if we have any events and min timeout expired, we're done */ + if (io_cqring_events(ctx)) + goto out_wake; + } /* * If using deferred task_work running and application is waiting on * more than one request, ensure we reset it now where we are switching @@ -2551,6 +2575,9 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) goto out_wake; } + /* any generated CQE posted past this time should wake us up */ + iowq->cq_tail = iowq->cq_min_tail; + hrtimer_update_function(&iowq->t, io_cqring_timer_wakeup); hrtimer_set_expires(timer, iowq->timeout); return HRTIMER_RESTART; @@ -2645,9 +2672,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, struct ext_arg *ext_arg) { struct io_wait_queue iowq; - struct io_rings *rings = ctx->rings; + struct io_rings *rings; ktime_t start_time; - int ret; + int ret, nr_wait; min_events = min_t(int, min_events, ctx->cq_entries); @@ -2660,15 +2687,23 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))) io_cqring_do_overflow_flush(ctx); - if (__io_cqring_events_user(ctx) >= min_events) + + rcu_read_lock(); + rings = io_get_rings(ctx); + if (__io_cqring_events_user(ctx) >= min_events) { + rcu_read_unlock(); return 0; + } init_waitqueue_func_entry(&iowq.wq, io_wake_function); iowq.wq.private = current; INIT_LIST_HEAD(&iowq.wq.entry); iowq.ctx = ctx; - iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; - iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail); + iowq.cq_tail = READ_ONCE(rings->cq.head) + min_events; + iowq.cq_min_tail = READ_ONCE(rings->cq.tail); + nr_wait = (int) iowq.cq_tail - READ_ONCE(rings->cq.tail); + rcu_read_unlock(); + rings = NULL; iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); iowq.hit_timeout = 0; iowq.min_timeout = ext_arg->min_time; @@ -2699,14 +2734,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, trace_io_uring_cqring_wait(ctx, min_events); do { unsigned long check_cq; - int nr_wait; - - /* if min timeout has been hit, don't reset wait count */ - if (!iowq.hit_timeout) - nr_wait = (int) iowq.cq_tail - - READ_ONCE(ctx->rings->cq.tail); - else - nr_wait = 1; if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { atomic_set(&ctx->cq_wait_nr, nr_wait); @@ -2757,13 +2784,22 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, break; } cond_resched(); + + /* if min timeout has been hit, don't reset wait count */ + if (!iowq.hit_timeout) + scoped_guard(rcu) + nr_wait = (int) iowq.cq_tail - + READ_ONCE(io_get_rings(ctx)->cq.tail); + else + nr_wait = 1; } while (1); if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) finish_wait(&ctx->cq_wait, &iowq.wq); restore_saved_sigmask_unless(ret == -EINTR); - return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; + guard(rcu)(); + return READ_ONCE(io_get_rings(ctx)->cq.head) == READ_ONCE(io_get_rings(ctx)->cq.tail) ? ret : 0; } static void io_rings_free(struct io_ring_ctx *ctx) @@ -2771,6 +2807,7 @@ static void io_rings_free(struct io_ring_ctx *ctx) io_free_region(ctx, &ctx->sq_region); io_free_region(ctx, &ctx->ring_region); ctx->rings = NULL; + RCU_INIT_POINTER(ctx->rings_rcu, NULL); ctx->sq_sqes = NULL; } @@ -2866,7 +2903,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_rings_free(ctx); if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_dec(&io_key_has_sqarray); + static_branch_slow_dec_deferred(&io_key_has_sqarray); percpu_ref_exit(&ctx->refs); free_uid(ctx->user); @@ -2934,7 +2971,9 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) */ poll_wait(file, &ctx->poll_wq, wait); - if (!io_sqring_full(ctx)) + rcu_read_lock(); + + if (!__io_sqring_full(ctx)) mask |= EPOLLOUT | EPOLLWRNORM; /* @@ -2954,6 +2993,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) if (__io_cqring_events_user(ctx) || io_has_work(ctx)) mask |= EPOLLIN | EPOLLRDNORM; + rcu_read_unlock(); return mask; } @@ -3014,12 +3054,12 @@ static __cold void io_ring_exit_work(struct work_struct *work) mutex_unlock(&ctx->uring_lock); } - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) - io_move_task_work_from_local(ctx); - /* The SQPOLL thread never reaches this path */ - while (io_uring_try_cancel_requests(ctx, NULL, true, false)) + do { + if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + io_move_task_work_from_local(ctx); cond_resched(); + } while (io_uring_try_cancel_requests(ctx, NULL, true, false)); if (ctx->sq_data) { struct io_sq_data *sqd = ctx->sq_data; @@ -3487,7 +3527,11 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, ctx = file->private_data; ret = -EBADFD; - if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED)) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_uring_add_tctx_node() -> __io_uring_add_tctx_node_from_submit() + */ + if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED)) goto out; /* @@ -3620,13 +3664,9 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, if (ret) return ret; ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); - + rcu_assign_pointer(ctx->rings_rcu, rings); if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); - rings->sq_ring_mask = p->sq_entries - 1; - rings->cq_ring_mask = p->cq_entries - 1; - rings->sq_ring_entries = p->sq_entries; - rings->cq_ring_entries = p->cq_entries; if (p->flags & IORING_SETUP_SQE128) size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries); @@ -3649,6 +3689,12 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, return ret; } ctx->sq_sqes = io_region_get_ptr(&ctx->sq_region); + + memset(rings, 0, sizeof(*rings)); + WRITE_ONCE(rings->sq_ring_mask, ctx->sq_entries - 1); + WRITE_ONCE(rings->cq_ring_mask, ctx->cq_entries - 1); + WRITE_ONCE(rings->sq_ring_entries, ctx->sq_entries); + WRITE_ONCE(rings->cq_ring_entries, ctx->cq_entries); return 0; } @@ -3808,7 +3854,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->clock_offset = 0; if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_inc(&io_key_has_sqarray); + static_branch_deferred_inc(&io_key_has_sqarray); if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && !(ctx->flags & IORING_SETUP_IOPOLL) && diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 46d9141d772a7..05702288465b7 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -112,16 +112,28 @@ struct io_wait_queue { #endif }; +static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) +{ + return rcu_dereference_check(ctx->rings_rcu, + lockdep_is_held(&ctx->uring_lock) || + lockdep_is_held(&ctx->completion_lock)); +} + static inline bool io_should_wake(struct io_wait_queue *iowq) { struct io_ring_ctx *ctx = iowq->ctx; - int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail; + struct io_rings *rings; + int dist; + + guard(rcu)(); + rings = io_get_rings(ctx); /* * Wake up if we have enough events, or if a timeout occurred since we * started waiting. For timeouts, we always want to return to userspace, * regardless of event count. */ + dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail; return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; } @@ -413,9 +425,9 @@ static inline void io_cqring_wake(struct io_ring_ctx *ctx) __io_wq_wake(&ctx->cq_wait); } -static inline bool io_sqring_full(struct io_ring_ctx *ctx) +static inline bool __io_sqring_full(struct io_ring_ctx *ctx) { - struct io_rings *r = ctx->rings; + struct io_rings *r = io_get_rings(ctx); /* * SQPOLL must use the actual sqring head, as using the cached_sq_head @@ -427,9 +439,15 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; } -static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) +static inline bool io_sqring_full(struct io_ring_ctx *ctx) { - struct io_rings *rings = ctx->rings; + guard(rcu)(); + return __io_sqring_full(ctx); +} + +static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx) +{ + struct io_rings *rings = io_get_rings(ctx); unsigned int entries; /* make sure SQ entry isn't read before tail */ @@ -490,6 +508,12 @@ static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw) lockdep_assert_held(&ctx->uring_lock); } +static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) +{ + guard(rcu)(); + return __io_sqring_entries(ctx); +} + /* * Don't complete immediately but use deferred completion infrastructure. * Protected by ->uring_lock and can only be used either with diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index a727e020fe036..9e8166e24dc81 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -34,6 +34,10 @@ struct io_provide_buf { static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len) { + /* No data consumed, return false early to avoid consuming the buffer */ + if (!len) + return false; + while (len) { struct io_uring_buf *buf; u32 buf_len, this_len; @@ -44,7 +48,7 @@ static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len) buf_len -= this_len; /* Stop looping for invalid buffer length of 0 */ if (buf_len || !this_len) { - buf->addr += this_len; + buf->addr = READ_ONCE(buf->addr) + this_len; buf->len = buf_len; return false; } @@ -111,9 +115,18 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags) buf = req->kbuf; bl = io_buffer_get_list(ctx, buf->bgid); - list_add(&buf->list, &bl->buf_list); - bl->nbufs++; + /* + * If the buffer list was upgraded to a ring-based one, or removed, + * while the request was in-flight in io-wq, drop it. + */ + if (bl && !(bl->flags & IOBL_BUF_RING)) { + list_add(&buf->list, &bl->buf_list); + bl->nbufs++; + } else { + kfree(buf); + } req->flags &= ~REQ_F_BUFFER_SELECTED; + req->kbuf = NULL; io_ring_submit_unlock(ctx, issue_flags); return true; @@ -198,12 +211,13 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, if (*len == 0 || *len > buf_len) *len = buf_len; req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT; - req->buf_index = buf->bid; + req->buf_index = READ_ONCE(buf->bid); sel.buf_list = bl; - sel.addr = u64_to_user_ptr(buf->addr); + sel.addr = u64_to_user_ptr(READ_ONCE(buf->addr)); if (io_should_commit(req, issue_flags)) { - io_kbuf_commit(req, sel.buf_list, *len, 1); + if (!io_kbuf_commit(req, sel.buf_list, *len, 1)) + req->flags |= REQ_F_BUF_MORE; sel.buf_list = NULL; } return sel; @@ -280,7 +294,7 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, if (!arg->max_len) arg->max_len = INT_MAX; - req->buf_index = buf->bid; + req->buf_index = READ_ONCE(buf->bid); do { u32 len = READ_ONCE(buf->len); @@ -295,7 +309,7 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, } } - iov->iov_base = u64_to_user_ptr(buf->addr); + iov->iov_base = u64_to_user_ptr(READ_ONCE(buf->addr)); iov->iov_len = len; iov++; @@ -336,7 +350,8 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, */ if (ret > 0) { req->flags |= REQ_F_BUFFERS_COMMIT | REQ_F_BL_NO_RECYCLE; - io_kbuf_commit(req, sel->buf_list, arg->out_len, ret); + if (!io_kbuf_commit(req, sel->buf_list, arg->out_len, ret)) + req->flags |= REQ_F_BUF_MORE; } } else { ret = io_provided_buffers_select(req, &arg->out_len, sel->buf_list, arg->iovs); @@ -382,8 +397,10 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, if (bl) ret = io_kbuf_commit(req, bl, len, nr); + if (ret && (req->flags & REQ_F_BUF_MORE)) + ret = false; - req->flags &= ~REQ_F_BUFFER_RING; + req->flags &= ~(REQ_F_BUFFER_RING | REQ_F_BUF_MORE); return ret; } @@ -669,8 +686,9 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) bl->buf_ring = br; if (reg.flags & IOU_PBUF_RING_INC) bl->flags |= IOBL_INC; - io_buffer_add_list(ctx, bl, reg.bgid); - return 0; + ret = io_buffer_add_list(ctx, bl, reg.bgid); + if (!ret) + return 0; fail: io_free_region(ctx, &bl->region); kfree(bl); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 5e5b94236d720..bce74a8b64c6e 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -124,7 +124,11 @@ static int __io_msg_ring_data(struct io_ring_ctx *target_ctx, return -EINVAL; if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS) && msg->dst_fd) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_data_remote() -> io_msg_remote_post() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (io_msg_need_remote(target_ctx)) @@ -244,7 +248,11 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) return -EINVAL; if (target_ctx == ctx) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_fd_remote() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (!msg->src_file) { int ret = io_msg_grab_file(req, issue_flags); diff --git a/io_uring/net.c b/io_uring/net.c index 43d77f95db51d..ad08f693bccb9 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -376,6 +376,8 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) kmsg->msg.msg_namelen = addr_len; } if (sr->flags & IORING_RECVSEND_FIXED_BUF) { + if (sr->flags & IORING_SEND_VECTORIZED) + return -EINVAL; req->flags |= REQ_F_IMPORT_BUFFER; return 0; } @@ -420,6 +422,8 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->done_io = 0; sr->len = READ_ONCE(sqe->len); + if (unlikely(sr->len < 0)) + return -EINVAL; sr->flags = READ_ONCE(sqe->ioprio); if (sr->flags & ~SENDMSG_FLAGS) return -EINVAL; @@ -516,7 +520,11 @@ static inline bool io_send_finish(struct io_kiocb *req, cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); - if (bundle_finished || req->flags & REQ_F_BL_EMPTY) + /* + * Don't start new bundles if the buffer list is empty, or if the + * current operation needed to go through polling to complete. + */ + if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED)) goto finish; /* @@ -786,6 +794,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); sr->len = READ_ONCE(sqe->len); + if (unlikely(sr->len < 0)) + return -EINVAL; sr->flags = READ_ONCE(sqe->ioprio); if (sr->flags & ~RECVMSG_FLAGS) return -EINVAL; diff --git a/io_uring/openclose.c b/io_uring/openclose.c index bfeb91b31bba5..606ce0664e6a4 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -73,13 +73,13 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe open->filename = NULL; return ret; } + req->flags |= REQ_F_NEED_CLEANUP; open->file_slot = READ_ONCE(sqe->file_index); if (open->file_slot && (open->how.flags & O_CLOEXEC)) return -EINVAL; open->nofile = rlimit(RLIMIT_NOFILE); - req->flags |= REQ_F_NEED_CLEANUP; if (io_openat_force_async(open)) req->flags |= REQ_F_FORCE_ASYNC; return 0; @@ -336,31 +336,34 @@ static int io_pipe_fixed(struct io_kiocb *req, struct file **files, { struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); struct io_ring_ctx *ctx = req->ctx; + bool alloc_slot; int ret, fds[2] = { -1, -1 }; int slot = p->file_slot; if (p->flags & O_CLOEXEC) return -EINVAL; + alloc_slot = slot == IORING_FILE_INDEX_ALLOC; + io_ring_submit_lock(ctx, issue_flags); ret = __io_fixed_fd_install(ctx, files[0], slot); if (ret < 0) goto err; - fds[0] = ret; + fds[0] = alloc_slot ? ret : slot - 1; files[0] = NULL; /* * If a specific slot is given, next one will be used for * the write side. */ - if (slot != IORING_FILE_INDEX_ALLOC) + if (!alloc_slot) slot++; ret = __io_fixed_fd_install(ctx, files[1], slot); if (ret < 0) goto err; - fds[1] = ret; + fds[1] = alloc_slot ? ret : slot - 1; files[1] = NULL; io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/poll.c b/io_uring/poll.c index b9681d0f9f131..869c7b2a08b64 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -254,6 +254,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); v &= ~IO_POLL_RETRY_FLAG; } + v &= IO_POLL_REF_MASK; } /* the mask was stashed in __io_poll_execute */ @@ -286,8 +287,13 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) return IOU_POLL_REMOVE_POLL_USE_RES; } } else { - int ret = io_poll_issue(req, tw); + int ret; + /* multiple refs and HUP, ensure we loop once more */ + if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1) + v--; + + ret = io_poll_issue(req, tw); if (ret == IOU_COMPLETE) return IOU_POLL_REMOVE_POLL_USE_RES; else if (ret == IOU_REQUEUE) @@ -303,7 +309,6 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) * Release all references, retry if someone tried to restart * task_work while we were executing it. */ - v &= IO_POLL_REF_MASK; } while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK); io_napi_add(req); @@ -936,12 +941,17 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) ret2 = io_poll_add(preq, issue_flags & ~IO_URING_F_UNLOCKED); /* successfully updated, don't complete poll request */ - if (!ret2 || ret2 == -EIOCBQUEUED) + if (ret2 == IOU_ISSUE_SKIP_COMPLETE) goto out; + /* request completed as part of the update, complete it */ + else if (ret2 == IOU_COMPLETE) + goto complete; } - req_set_fail(preq); io_req_set_res(preq, -ECANCELED, 0); +complete: + if (preq->cqe.res < 0) + req_set_fail(preq); preq->io_task_work.func = io_req_task_complete; io_req_task_work_add(preq); out: diff --git a/io_uring/register.c b/io_uring/register.c index d189b266b8cce..faa44dd32cd5a 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -193,7 +193,8 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (ctx->restrictions.registered) ctx->restricted = 1; - ctx->flags &= ~IORING_SETUP_R_DISABLED; + /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ + smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); return 0; @@ -555,7 +556,15 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ctx->sq_entries = p.sq_entries; ctx->cq_entries = p.cq_entries; + /* + * Just mark any flag we may have missed and that the application + * should act on unconditionally. Worst case it'll be an extra + * syscall. + */ + atomic_or(IORING_SQ_TASKRUN | IORING_SQ_NEED_WAKEUP, &n.rings->sq_flags); ctx->rings = n.rings; + rcu_assign_pointer(ctx->rings_rcu, n.rings); + ctx->sq_sqes = n.sq_sqes; swap_old(ctx, o, n, ring_region); swap_old(ctx, o, n, sq_region); @@ -564,6 +573,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) out: spin_unlock(&ctx->completion_lock); mutex_unlock(&ctx->mmap_lock); + /* Wait for concurrent io_ctx_mark_taskrun() */ + if (to_free == &o) + synchronize_rcu_expedited(); io_register_free_rings(ctx, &p, to_free); if (ctx->sq_data) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 0010c49924906..dc87c6a86e346 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1057,6 +1057,7 @@ static int io_import_kbuf(int ddir, struct iov_iter *iter, if (count < imu->len) { const struct bio_vec *bvec = iter->bvec; + len += iter->iov_offset; while (len > bvec->bv_len) { len -= bvec->bv_len; bvec++; @@ -1081,6 +1082,10 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, return ret; if (!(imu->dir & (1 << ddir))) return -EFAULT; + if (unlikely(!len)) { + iov_iter_bvec(iter, ddir, NULL, 0, 0); + return 0; + } offset = buf_addr - imu->ubuf; @@ -1184,12 +1189,16 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx return -EBUSY; nbufs = src_ctx->buf_table.nr; + if (!nbufs) + return -ENXIO; if (!arg->nr) arg->nr = nbufs; else if (arg->nr > nbufs) return -EINVAL; else if (arg->nr > IORING_MAX_REG_BUFFERS) return -EINVAL; + if (check_add_overflow(arg->nr, arg->src_off, &off) || off > nbufs) + return -EOVERFLOW; if (check_add_overflow(arg->nr, arg->dst_off, &nbufs)) return -EOVERFLOW; if (nbufs > IORING_MAX_REG_BUFFERS) @@ -1199,7 +1208,7 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx if (ret) return ret; - /* Fill entries in data from dst that won't overlap with src */ + /* Copy original dst nodes from before the cloned range */ for (i = 0; i < min(arg->dst_off, ctx->buf_table.nr); i++) { struct io_rsrc_node *src_node = ctx->buf_table.nodes[i]; @@ -1209,21 +1218,6 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx } } - ret = -ENXIO; - nbufs = src_ctx->buf_table.nr; - if (!nbufs) - goto out_free; - ret = -EINVAL; - if (!arg->nr) - arg->nr = nbufs; - else if (arg->nr > nbufs) - goto out_free; - ret = -EOVERFLOW; - if (check_add_overflow(arg->nr, arg->src_off, &off)) - goto out_free; - if (off > nbufs) - goto out_free; - off = arg->dst_off; i = arg->src_off; nr = arg->nr; @@ -1236,8 +1230,8 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx } else { dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!dst_node) { - ret = -ENOMEM; - goto out_free; + io_rsrc_data_free(ctx, &data); + return -ENOMEM; } refcount_inc(&src_node->buf->refs); @@ -1247,6 +1241,16 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx i++; } + /* Copy original dst nodes from after the cloned range */ + for (i = nbufs; i < ctx->buf_table.nr; i++) { + struct io_rsrc_node *node = ctx->buf_table.nodes[i]; + + if (node) { + data.nodes[i] = node; + node->refs++; + } + } + /* * If asked for replace, put the old table. data->nodes[] holds both * old and new nodes at this point. @@ -1263,10 +1267,6 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx WARN_ON_ONCE(ctx->buf_table.nr); ctx->buf_table = data; return 0; - -out_free: - io_rsrc_data_free(ctx, &data); - return ret; } /* diff --git a/io_uring/rw.c b/io_uring/rw.c index abe68ba9c9dc8..d7388a4a3ea5e 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -144,19 +144,22 @@ static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, return 0; } -static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) +static bool io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_rw *rw = req->async_data; if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) - return; + return false; io_alloc_cache_vec_kasan(&rw->vec); if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP) io_vec_free(&rw->vec); - if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) + if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) { io_req_async_data_clear(req, 0); + return true; + } + return false; } static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) @@ -190,7 +193,11 @@ static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) */ if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) { req->flags &= ~REQ_F_NEED_CLEANUP; - io_rw_recycle(req, issue_flags); + if (!io_rw_recycle(req, issue_flags)) { + struct io_async_rw *rw = req->async_data; + + io_vec_free(&rw->vec); + } } } diff --git a/io_uring/sync.c b/io_uring/sync.c index cea2d381ffd2a..ab7fa1cd7dd63 100644 --- a/io_uring/sync.c +++ b/io_uring/sync.c @@ -62,6 +62,8 @@ int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EINVAL; sync->off = READ_ONCE(sqe->off); + if (sync->off < 0) + return -EINVAL; sync->len = READ_ONCE(sqe->len); req->flags |= REQ_F_FORCE_ASYNC; return 0; diff --git a/io_uring/tctx.c b/io_uring/tctx.c index 5b66755579c08..03c278aa58120 100644 --- a/io_uring/tctx.c +++ b/io_uring/tctx.c @@ -122,6 +122,14 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) return ret; } } + + /* + * Re-activate io-wq keepalive on any new io_uring usage. The wq may have + * been marked for idle-exit when the task temporarily had no active + * io_uring instances. + */ + if (tctx->io_wq) + io_wq_set_exit_on_idle(tctx->io_wq, false); if (!xa_load(&tctx->xa, (unsigned long)ctx)) { node = kmalloc(sizeof(*node), GFP_KERNEL); if (!node) @@ -183,6 +191,9 @@ __cold void io_uring_del_tctx_node(unsigned long index) if (tctx->last == node->ctx) tctx->last = NULL; kfree(node); + + if (xa_empty(&tctx->xa) && tctx->io_wq) + io_wq_set_exit_on_idle(tctx->io_wq, true); } __cold void io_uring_clean_tctx(struct io_uring_task *tctx) diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 17e3aab0af367..cd42316d0f3c0 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -129,7 +129,7 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx) u32 seq; raw_spin_lock_irq(&ctx->timeout_lock); - seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); + seq = READ_ONCE(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts); list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { struct io_kiocb *req = cmd_to_io_kiocb(timeout); diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index b1b723222cdb8..d17ff07779dee 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -196,6 +196,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq, GFP_KERNEL_ACCOUNT); if (ret) { unpin_user_pages(pages, nr_pages); + kvfree(pages); return ret; } @@ -286,6 +287,9 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) } ret = io_populate_area_dma(ifq, area); + if (ret && !area->mem.is_dmabuf) + dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, + DMA_FROM_DEVICE, IO_DMA_ATTR); if (ret == 0) area->is_mapped = true; return ret; @@ -332,10 +336,14 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov) static bool io_zcrx_put_niov_uref(struct net_iov *niov) { atomic_t *uref = io_get_user_counter(niov); + int old; + + old = atomic_read(uref); + do { + if (unlikely(old == 0)) + return false; + } while (!atomic_try_cmpxchg(uref, &old, old - 1)); - if (unlikely(!atomic_read(uref))) - return false; - atomic_dec(uref); return true; } @@ -599,29 +607,30 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, if (ret) goto err; - ifq->netdev = netdev_get_by_index(current->nsproxy->net_ns, reg.if_idx, - &ifq->netdev_tracker, GFP_KERNEL); + ifq->netdev = netdev_get_by_index_lock(current->nsproxy->net_ns, reg.if_idx); if (!ifq->netdev) { ret = -ENODEV; goto err; } + netdev_hold(ifq->netdev, &ifq->netdev_tracker, GFP_KERNEL); ifq->dev = netdev_queue_get_dma_dev(ifq->netdev, reg.if_rxq); if (!ifq->dev) { ret = -EOPNOTSUPP; - goto err; + goto netdev_put_unlock; } get_device(ifq->dev); ret = io_zcrx_create_area(ifq, &area); if (ret) - goto err; + goto netdev_put_unlock; mp_param.mp_ops = &io_uring_pp_zc_ops; mp_param.mp_priv = ifq; - ret = net_mp_open_rxq(ifq->netdev, reg.if_rxq, &mp_param); + ret = __net_mp_open_rxq(ifq->netdev, reg.if_rxq, &mp_param, NULL); if (ret) - goto err; + goto netdev_put_unlock; + netdev_unlock(ifq->netdev); ifq->if_rxq = reg.if_rxq; reg.zcrx_id = id; @@ -640,6 +649,9 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, goto err; } return 0; +netdev_put_unlock: + netdev_put(ifq->netdev, &ifq->netdev_tracker); + netdev_unlock(ifq->netdev); err: scoped_guard(mutex, &ctx->mmap_lock) xa_erase(&ctx->zcrx_ctxs, id); @@ -755,11 +767,12 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, struct io_zcrx_ifq *ifq, struct net_iov **ret_niov) { + __u64 off = READ_ONCE(rqe->off); unsigned niov_idx, area_idx; struct io_zcrx_area *area; - area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; - niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; + area_idx = off >> IORING_ZCRX_AREA_SHIFT; + niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; if (unlikely(rqe->__pad || area_idx)) return false; diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 15b17e86e198c..9b087ebeb643b 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -214,7 +214,7 @@ static int ipc_permissions(struct ctl_table_header *head, const struct ctl_table if (((table->data == &ns->ids[IPC_SEM_IDS].next_id) || (table->data == &ns->ids[IPC_MSG_IDS].next_id) || (table->data == &ns->ids[IPC_SHM_IDS].next_id)) && - checkpoint_restore_ns_capable(ns->user_ns)) + checkpoint_restore_ns_capable_noaudit(ns->user_ns)) mode = 0666; else #endif diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 0de8fc8a0e0b3..75a5df36f9170 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1676,7 +1676,16 @@ static void btf_free_id(struct btf *btf) * of the _bh() version. */ spin_lock_irqsave(&btf_idr_lock, flags); - idr_remove(&btf_idr, btf->id); + if (btf->id) { + idr_remove(&btf_idr, btf->id); + /* + * Clear the id here to make this function idempotent, since it will get + * called a couple of times for module BTFs: on module unload, and then + * the final btf_put(). btf_alloc_id() starts IDs with 1, so we can use + * 0 as sentinel value. + */ + WRITE_ONCE(btf->id, 0); + } spin_unlock_irqrestore(&btf_idr_lock, flags); } @@ -7995,7 +8004,7 @@ static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) { const struct btf *btf = filp->private_data; - seq_printf(m, "btf_id:\t%u\n", btf->id); + seq_printf(m, "btf_id:\t%u\n", READ_ONCE(btf->id)); } #endif @@ -8077,7 +8086,7 @@ int btf_get_info_by_fd(const struct btf *btf, if (copy_from_user(&info, uinfo, info_copy)) return -EFAULT; - info.id = btf->id; + info.id = READ_ONCE(btf->id); ubtf = u64_to_user_ptr(info.btf); btf_copy = min_t(u32, btf->data_size, info.btf_size); if (copy_to_user(ubtf, btf->data, btf_copy)) @@ -8140,7 +8149,7 @@ int btf_get_fd_by_id(u32 id) u32 btf_obj_id(const struct btf *btf) { - return btf->id; + return READ_ONCE(btf->id); } bool btf_is_kernel(const struct btf *btf) @@ -8262,6 +8271,13 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, if (btf_mod->module != module) continue; + /* + * For modules, we do the freeing of BTF IDR as soon as + * module goes away to disable BTF discovery, since the + * btf_try_get_module() on such BTFs will fail. This may + * be called again on btf_put(), but it's ok to do so. + */ + btf_free_id(btf_mod->btf); list_del(&btf_mod->list); if (btf_mod->sysfs_attr) sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d595fe512498c..8de006d388f67 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -713,8 +713,8 @@ static struct bpf_ksym *bpf_ksym_find(unsigned long addr) return n ? container_of(n, struct bpf_ksym, tnode) : NULL; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { struct bpf_ksym *ksym; int ret = 0; @@ -1403,6 +1403,27 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); break; + + case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: + case BPF_ST | BPF_PROBE_MEM32 | BPF_W: + case BPF_ST | BPF_PROBE_MEM32 | BPF_H: + case BPF_ST | BPF_PROBE_MEM32 | BPF_B: + *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ + from->imm); + *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); + /* + * Cannot use BPF_STX_MEM() macro here as it + * hardcodes BPF_MEM mode, losing PROBE_MEM32 + * and breaking arena addressing in the JIT. + */ + *to++ = (struct bpf_insn) { + .code = BPF_STX | BPF_PROBE_MEM32 | + BPF_SIZE(from->code), + .dst_reg = from->dst_reg, + .src_reg = BPF_REG_AX, + .off = from->off, + }; + break; } out: return to - to_buff; @@ -1696,6 +1717,12 @@ bool bpf_opcode_in_insntable(u8 code) } #ifndef CONFIG_BPF_JIT_ALWAYS_ON +/* Absolute value of s32 without undefined behavior for S32_MIN */ +static u32 abs_s32(s32 x) +{ + return x >= 0 ? (u32)x : -(u32)x; +} + /** * ___bpf_prog_run - run eBPF program on a given context * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers @@ -1860,8 +1887,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = do_div(AX, (u32) SRC); break; case 1: - AX = abs((s32)DST); - AX = do_div(AX, abs((s32)SRC)); + AX = abs_s32((s32)DST); + AX = do_div(AX, abs_s32((s32)SRC)); if ((s32)DST < 0) DST = (u32)-AX; else @@ -1888,8 +1915,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = do_div(AX, (u32) IMM); break; case 1: - AX = abs((s32)DST); - AX = do_div(AX, abs((s32)IMM)); + AX = abs_s32((s32)DST); + AX = do_div(AX, abs_s32((s32)IMM)); if ((s32)DST < 0) DST = (u32)-AX; else @@ -1915,8 +1942,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = (u32) AX; break; case 1: - AX = abs((s32)DST); - do_div(AX, abs((s32)SRC)); + AX = abs_s32((s32)DST); + do_div(AX, abs_s32((s32)SRC)); if (((s32)DST < 0) == ((s32)SRC < 0)) DST = (u32)AX; else @@ -1942,8 +1969,8 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) DST = (u32) AX; break; case 1: - AX = abs((s32)DST); - do_div(AX, abs((s32)IMM)); + AX = abs_s32((s32)DST); + do_div(AX, abs_s32((s32)IMM)); if (((s32)DST < 0) == ((s32)IMM < 0)) DST = (u32)AX; else diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 703e5df1f4ef9..306bf98378041 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ struct xdp_bulk_queue { struct list_head flush_node; struct bpf_cpu_map_entry *obj; unsigned int count; + local_lock_t bq_lock; }; /* Struct for every remote "destination" CPU in map */ @@ -451,6 +453,7 @@ __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value, for_each_possible_cpu(i) { bq = per_cpu_ptr(rcpu->bulkq, i); bq->obj = rcpu; + local_lock_init(&bq->bq_lock); } /* Alloc queue */ @@ -717,6 +720,8 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) struct ptr_ring *q; int i; + lockdep_assert_held(&bq->bq_lock); + if (unlikely(!bq->count)) return; @@ -744,11 +749,15 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) } /* Runs under RCU-read-side, plus in softirq under NAPI protection. - * Thus, safe percpu variable access. + * Thus, safe percpu variable access. PREEMPT_RT relies on + * local_lock_nested_bh() to serialise access to the per-CPU bq. */ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) { - struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq); + struct xdp_bulk_queue *bq; + + local_lock_nested_bh(&rcpu->bulkq->bq_lock); + bq = this_cpu_ptr(rcpu->bulkq); if (unlikely(bq->count == CPU_MAP_BULK_SIZE)) bq_flush_to_queue(bq); @@ -769,6 +778,8 @@ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) list_add(&bq->flush_node, flush_list); } + + local_unlock_nested_bh(&rcpu->bulkq->bq_lock); } int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, @@ -805,7 +816,9 @@ void __cpu_map_flush(struct list_head *flush_list) struct xdp_bulk_queue *bq, *tmp; list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { + local_lock_nested_bh(&bq->obj->bulkq->bq_lock); bq_flush_to_queue(bq); + local_unlock_nested_bh(&bq->obj->bulkq->bq_lock); /* If already running, costs spin_lock_irqsave + smb_mb */ wake_up_process(bq->obj->kthread); diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 83c4d9943084b..1d024fe7248ac 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -261,6 +261,12 @@ __bpf_kfunc void bpf_crypto_ctx_release(struct bpf_crypto_ctx *ctx) call_rcu(&ctx->rcu, crypto_free_cb); } +__bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx) +{ + bpf_crypto_ctx_release(ctx); +} +CFI_NOSEAL(bpf_crypto_ctx_release_dtor); + static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, const struct bpf_dynptr_kern *src, const struct bpf_dynptr_kern *dst, @@ -368,7 +374,7 @@ static const struct btf_kfunc_id_set crypt_kfunc_set = { BTF_ID_LIST(bpf_crypto_dtor_ids) BTF_ID(struct, bpf_crypto_ctx) -BTF_ID(func, bpf_crypto_ctx_release) +BTF_ID(func, bpf_crypto_ctx_release_dtor) static int __init crypto_kfunc_init(void) { diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 2625601de76e9..3d619d01088e3 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -45,6 +45,7 @@ * types of devmap; only the lookup and insertion is different. */ #include +#include #include #include #include @@ -60,6 +61,7 @@ struct xdp_dev_bulk_queue { struct net_device *dev_rx; struct bpf_prog *xdp_prog; unsigned int count; + local_lock_t bq_lock; }; struct bpf_dtab_netdev { @@ -381,6 +383,8 @@ static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags) int to_send = cnt; int i; + lockdep_assert_held(&bq->bq_lock); + if (unlikely(!cnt)) return; @@ -425,10 +429,12 @@ void __dev_flush(struct list_head *flush_list) struct xdp_dev_bulk_queue *bq, *tmp; list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { + local_lock_nested_bh(&bq->dev->xdp_bulkq->bq_lock); bq_xmit_all(bq, XDP_XMIT_FLUSH); bq->dev_rx = NULL; bq->xdp_prog = NULL; __list_del_clearprev(&bq->flush_node); + local_unlock_nested_bh(&bq->dev->xdp_bulkq->bq_lock); } } @@ -451,12 +457,16 @@ static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key) /* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu * variable access, and map elements stick around. See comment above - * xdp_do_flush() in filter.c. + * xdp_do_flush() in filter.c. PREEMPT_RT relies on local_lock_nested_bh() + * to serialise access to the per-CPU bq. */ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_prog *xdp_prog) { - struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq); + struct xdp_dev_bulk_queue *bq; + + local_lock_nested_bh(&dev->xdp_bulkq->bq_lock); + bq = this_cpu_ptr(dev->xdp_bulkq); if (unlikely(bq->count == DEV_MAP_BULK_SIZE)) bq_xmit_all(bq, 0); @@ -477,6 +487,8 @@ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, } bq->q[bq->count++] = xdpf; + + local_unlock_nested_bh(&dev->xdp_bulkq->bq_lock); } static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, @@ -588,18 +600,22 @@ static inline bool is_ifindex_excluded(int *excluded, int num_excluded, int ifin } /* Get ifindex of each upper device. 'indexes' must be able to hold at - * least MAX_NEST_DEV elements. - * Returns the number of ifindexes added. + * least 'max' elements. + * Returns the number of ifindexes added, or -EOVERFLOW if there are too + * many upper devices. */ -static int get_upper_ifindexes(struct net_device *dev, int *indexes) +static int get_upper_ifindexes(struct net_device *dev, int *indexes, int max) { struct net_device *upper; struct list_head *iter; int n = 0; netdev_for_each_upper_dev_rcu(dev, upper, iter) { + if (n >= max) + return -EOVERFLOW; indexes[n++] = upper->ifindex; } + return n; } @@ -615,7 +631,11 @@ int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, int err; if (exclude_ingress) { - num_excluded = get_upper_ifindexes(dev_rx, excluded_devices); + num_excluded = get_upper_ifindexes(dev_rx, excluded_devices, + ARRAY_SIZE(excluded_devices) - 1); + if (num_excluded < 0) + return num_excluded; + excluded_devices[num_excluded++] = dev_rx->ifindex; } @@ -733,7 +753,11 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, int err; if (exclude_ingress) { - num_excluded = get_upper_ifindexes(dev, excluded_devices); + num_excluded = get_upper_ifindexes(dev, excluded_devices, + ARRAY_SIZE(excluded_devices) - 1); + if (num_excluded < 0) + return num_excluded; + excluded_devices[num_excluded++] = dev->ifindex; } @@ -1115,8 +1139,13 @@ static int dev_map_notification(struct notifier_block *notifier, if (!netdev->xdp_bulkq) return NOTIFY_BAD; - for_each_possible_cpu(cpu) - per_cpu_ptr(netdev->xdp_bulkq, cpu)->dev = netdev; + for_each_possible_cpu(cpu) { + struct xdp_dev_bulk_queue *bq; + + bq = per_cpu_ptr(netdev->xdp_bulkq, cpu); + bq->dev = netdev; + local_lock_init(&bq->bq_lock); + } break; case NETDEV_UNREGISTER: /* This rcu_read_lock/unlock pair is needed because diff --git a/kernel/bpf/dmabuf_iter.c b/kernel/bpf/dmabuf_iter.c index 4dd7ef7c145ca..cd500248abd95 100644 --- a/kernel/bpf/dmabuf_iter.c +++ b/kernel/bpf/dmabuf_iter.c @@ -6,10 +6,33 @@ #include #include +struct dmabuf_iter_priv { + /* + * If this pointer is non-NULL, the buffer's refcount is elevated to + * prevent destruction between stop/start. If reading is not resumed and + * start is never called again, then dmabuf_iter_seq_fini drops the + * reference when the iterator is released. + */ + struct dma_buf *dmabuf; +}; + static void *dmabuf_iter_seq_start(struct seq_file *seq, loff_t *pos) { - if (*pos) - return NULL; + struct dmabuf_iter_priv *p = seq->private; + + if (*pos) { + struct dma_buf *dmabuf = p->dmabuf; + + if (!dmabuf) + return NULL; + + /* + * Always resume from where we stopped, regardless of the value + * of pos. + */ + p->dmabuf = NULL; + return dmabuf; + } return dma_buf_iter_begin(); } @@ -54,8 +77,11 @@ static void dmabuf_iter_seq_stop(struct seq_file *seq, void *v) { struct dma_buf *dmabuf = v; - if (dmabuf) - dma_buf_put(dmabuf); + if (dmabuf) { + struct dmabuf_iter_priv *p = seq->private; + + p->dmabuf = dmabuf; + } } static const struct seq_operations dmabuf_iter_seq_ops = { @@ -71,11 +97,27 @@ static void bpf_iter_dmabuf_show_fdinfo(const struct bpf_iter_aux_info *aux, seq_puts(seq, "dmabuf iter\n"); } +static int dmabuf_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux) +{ + struct dmabuf_iter_priv *p = (struct dmabuf_iter_priv *)priv; + + p->dmabuf = NULL; + return 0; +} + +static void dmabuf_iter_seq_fini(void *priv) +{ + struct dmabuf_iter_priv *p = (struct dmabuf_iter_priv *)priv; + + if (p->dmabuf) + dma_buf_put(p->dmabuf); +} + static const struct bpf_iter_seq_info dmabuf_iter_seq_info = { .seq_ops = &dmabuf_iter_seq_ops, - .init_seq_private = NULL, - .fini_seq_private = NULL, - .seq_priv_size = 0, + .init_seq_private = dmabuf_iter_seq_init, + .fini_seq_private = dmabuf_iter_seq_fini, + .seq_priv_size = sizeof(struct dmabuf_iter_priv), }; static struct bpf_iter_reg bpf_dmabuf_reg_info = { diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index c2fcd0cd51e51..e7721f0776c72 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -947,15 +947,21 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, void *value, bool onallcpus) { + void *ptr; + if (!onallcpus) { /* copy true value_size bytes */ - copy_map_value(&htab->map, this_cpu_ptr(pptr), value); + ptr = this_cpu_ptr(pptr); + copy_map_value(&htab->map, ptr, value); + bpf_obj_free_fields(htab->map.record, ptr); } else { u32 size = round_up(htab->map.value_size, 8); int off = 0, cpu; for_each_possible_cpu(cpu) { - copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value + off); + ptr = per_cpu_ptr(pptr, cpu); + copy_map_value_long(&htab->map, ptr, value + off); + bpf_obj_free_fields(htab->map.record, ptr); off += size; } } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index e4007fea49091..68da6dcfb4bb7 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -777,9 +777,11 @@ int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs) { int nest_level; + preempt_disable(); nest_level = this_cpu_inc_return(bpf_bprintf_nest_level); if (WARN_ON_ONCE(nest_level > MAX_BPRINTF_NEST_LEVEL)) { this_cpu_dec(bpf_bprintf_nest_level); + preempt_enable(); return -EBUSY; } *bufs = this_cpu_ptr(&bpf_bprintf_bufs[nest_level - 1]); @@ -792,6 +794,7 @@ void bpf_put_buffers(void) if (WARN_ON_ONCE(this_cpu_read(bpf_bprintf_nest_level) == 0)) return; this_cpu_dec(bpf_bprintf_nest_level); + preempt_enable(); } void bpf_bprintf_cleanup(struct bpf_bprintf_data *data) @@ -1077,7 +1080,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index a00561b1d3e51..c0c93a0f5af63 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -89,15 +89,14 @@ struct rqspinlock_timeout { DEFINE_PER_CPU_ALIGNED(struct rqspinlock_held, rqspinlock_held_locks); EXPORT_SYMBOL_GPL(rqspinlock_held_locks); -static bool is_lock_released(rqspinlock_t *lock, u32 mask, struct rqspinlock_timeout *ts) +static bool is_lock_released(rqspinlock_t *lock, u32 mask) { if (!(atomic_read_acquire(&lock->val) & (mask))) return true; return false; } -static noinline int check_deadlock_AA(rqspinlock_t *lock, u32 mask, - struct rqspinlock_timeout *ts) +static noinline int check_deadlock_AA(rqspinlock_t *lock) { struct rqspinlock_held *rqh = this_cpu_ptr(&rqspinlock_held_locks); int cnt = min(RES_NR_HELD, rqh->cnt); @@ -118,8 +117,7 @@ static noinline int check_deadlock_AA(rqspinlock_t *lock, u32 mask, * more locks, which reduce to ABBA). This is not exhaustive, and we rely on * timeouts as the final line of defense. */ -static noinline int check_deadlock_ABBA(rqspinlock_t *lock, u32 mask, - struct rqspinlock_timeout *ts) +static noinline int check_deadlock_ABBA(rqspinlock_t *lock, u32 mask) { struct rqspinlock_held *rqh = this_cpu_ptr(&rqspinlock_held_locks); int rqh_cnt = min(RES_NR_HELD, rqh->cnt); @@ -142,7 +140,7 @@ static noinline int check_deadlock_ABBA(rqspinlock_t *lock, u32 mask, * Let's ensure to break out of this loop if the lock is available for * us to potentially acquire. */ - if (is_lock_released(lock, mask, ts)) + if (is_lock_released(lock, mask)) return 0; /* @@ -198,15 +196,14 @@ static noinline int check_deadlock_ABBA(rqspinlock_t *lock, u32 mask, return 0; } -static noinline int check_deadlock(rqspinlock_t *lock, u32 mask, - struct rqspinlock_timeout *ts) +static noinline int check_deadlock(rqspinlock_t *lock, u32 mask) { int ret; - ret = check_deadlock_AA(lock, mask, ts); + ret = check_deadlock_AA(lock); if (ret) return ret; - ret = check_deadlock_ABBA(lock, mask, ts); + ret = check_deadlock_ABBA(lock, mask); if (ret) return ret; @@ -234,7 +231,7 @@ static noinline int check_timeout(rqspinlock_t *lock, u32 mask, */ if (prev + NSEC_PER_MSEC < time) { ts->cur = time; - return check_deadlock(lock, mask, ts); + return check_deadlock(lock, mask); } return 0; @@ -278,7 +275,12 @@ int __lockfunc resilient_tas_spin_lock(rqspinlock_t *lock) int val, ret = 0; RES_INIT_TIMEOUT(ts); - grab_held_lock_entry(lock); + /* + * We are either called directly from res_spin_lock after grabbing the + * deadlock detection entry when queued spinlocks are disabled, or from + * resilient_queued_spin_lock_slowpath after grabbing the deadlock + * detection entry. No need to obtain it here. + */ /* * Since the waiting loop's time is dependent on the amount of @@ -400,10 +402,7 @@ int __lockfunc resilient_queued_spin_lock_slowpath(rqspinlock_t *lock, u32 val) goto queue; } - /* - * Grab an entry in the held locks array, to enable deadlock detection. - */ - grab_held_lock_entry(lock); + /* Deadlock detection entry already held after failing fast path. */ /* * We're pending, wait for the owner to go away. @@ -451,11 +450,7 @@ int __lockfunc resilient_queued_spin_lock_slowpath(rqspinlock_t *lock, u32 val) */ queue: lockevent_inc(lock_slowpath); - /* - * Grab deadlock detection entry for the queue path. - */ - grab_held_lock_entry(lock); - + /* Deadlock detection entry already held after failing fast path. */ node = this_cpu_ptr(&rqnodes[0].mcs); idx = node->count++; tail = encode_tail(smp_processor_id(), idx); @@ -471,7 +466,7 @@ int __lockfunc resilient_queued_spin_lock_slowpath(rqspinlock_t *lock, u32 val) * any MCS node. This is not the most elegant solution, but is * simple enough. */ - if (unlikely(idx >= _Q_MAX_NODES || in_nmi())) { + if (unlikely(idx >= _Q_MAX_NODES || (in_nmi() && idx > 0))) { lockevent_inc(lock_no_node); RES_RESET_TIMEOUT(ts, RES_DEF_TIMEOUT); while (!queued_spin_trylock(lock)) { diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 4d53cdd1374cf..2365541c81dd1 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -42,6 +42,28 @@ static inline int stack_map_data_size(struct bpf_map *map) sizeof(struct bpf_stack_build_id) : sizeof(u64); } +/** + * stack_map_calculate_max_depth - Calculate maximum allowed stack trace depth + * @size: Size of the buffer/map value in bytes + * @elem_size: Size of each stack trace element + * @flags: BPF stack trace flags (BPF_F_USER_STACK, BPF_F_USER_BUILD_ID, ...) + * + * Return: Maximum number of stack trace entries that can be safely stored + */ +static u32 stack_map_calculate_max_depth(u32 size, u32 elem_size, u64 flags) +{ + u32 skip = flags & BPF_F_SKIP_FIELD_MASK; + u32 max_depth; + u32 curr_sysctl_max_stack = READ_ONCE(sysctl_perf_event_max_stack); + + max_depth = size / elem_size; + max_depth += skip; + if (max_depth > curr_sysctl_max_stack) + return curr_sysctl_max_stack; + + return max_depth; +} + static int prealloc_elems_and_freelist(struct bpf_stack_map *smap) { u64 elem_size = sizeof(struct stack_map_bucket) + @@ -229,8 +251,8 @@ static long __bpf_get_stackid(struct bpf_map *map, { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); struct stack_map_bucket *bucket, *new_bucket, *old_bucket; + u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - u32 hash, id, trace_nr, trace_len, i; bool user = flags & BPF_F_USER_STACK; u64 *ips; bool hash_matches; @@ -239,7 +261,8 @@ static long __bpf_get_stackid(struct bpf_map *map, /* skipping more than usable stack trace */ return -EFAULT; - trace_nr = trace->nr - skip; + max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); + trace_nr = min_t(u32, trace->nr - skip, max_depth - skip); trace_len = trace_nr * sizeof(u64); ips = trace->ip + skip; hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); @@ -300,20 +323,17 @@ static long __bpf_get_stackid(struct bpf_map *map, BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, u64, flags) { - u32 max_depth = map->value_size / stack_map_data_size(map); - u32 skip = flags & BPF_F_SKIP_FIELD_MASK; + u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; bool kernel = !user; + u32 max_depth; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) return -EINVAL; - max_depth += skip; - if (max_depth > sysctl_perf_event_max_stack) - max_depth = sysctl_perf_event_max_stack; - + max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags); trace = get_perf_callchain(regs, kernel, user, max_depth, false, false); @@ -371,15 +391,11 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; nr_kernel = count_kernel_ip(trace); + __u64 nr = trace->nr; /* save original */ if (kernel) { - __u64 nr = trace->nr; - trace->nr = nr_kernel; ret = __bpf_get_stackid(map, trace, flags); - - /* restore nr */ - trace->nr = nr; } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -390,6 +406,10 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; ret = __bpf_get_stackid(map, trace, flags); } + + /* restore nr */ + trace->nr = nr; + return ret; } @@ -406,7 +426,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) { - u32 trace_nr, copy_len, elem_size, num_elem, max_depth; + u32 trace_nr, copy_len, elem_size, max_depth; bool user_build_id = flags & BPF_F_USER_BUILD_ID; bool crosstask = task && task != current; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -438,21 +458,20 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, goto clear; } - num_elem = size / elem_size; - max_depth = num_elem + skip; - if (sysctl_perf_event_max_stack < max_depth) - max_depth = sysctl_perf_event_max_stack; + max_depth = stack_map_calculate_max_depth(size, elem_size, flags); if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ - if (trace_in) + if (trace_in) { trace = trace_in; - else if (kernel && task) + trace->nr = min_t(u32, trace->nr, max_depth); + } else if (kernel && task) { trace = get_callchain_entry_for_task(task, max_depth); - else + } else { trace = get_perf_callchain(regs, kernel, user, max_depth, crosstask, false); + } if (unlikely(!trace) || trace->nr < skip) { if (may_fault) @@ -461,7 +480,6 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, } trace_nr = trace->nr - skip; - trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem; copy_len = trace_nr * elem_size; ips = trace->ip + skip; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8a129746bd6cc..586ece78f783a 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1365,11 +1365,6 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, return ret; } -static bool bpf_net_capable(void) -{ - return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); -} - #define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size /* called via syscall */ static int map_create(union bpf_attr *attr, bpfptr_t uattr) @@ -1585,7 +1580,8 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) goto free_map; } } else if (attr->excl_prog_hash_size) { - return -EINVAL; + err = -EINVAL; + goto free_map; } err = security_bpf_map_create(map, attr, token, uattr.is_kernel); @@ -2462,6 +2458,9 @@ void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog) struct bpf_prog_stats *stats; unsigned int flags; + if (unlikely(!prog->stats)) + return; + stats = this_cpu_ptr(prog->stats); flags = u64_stats_update_begin_irqsave(&stats->syncp); u64_stats_inc(&stats->misses); @@ -2826,6 +2825,13 @@ static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr void *sig; int err = 0; + /* + * Don't attempt to use kmalloc_large or vmalloc for signatures. + * Practical signature for BPF program should be below this limit. + */ + if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); else @@ -4550,6 +4556,8 @@ static int bpf_prog_detach(const union bpf_attr *attr) prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); if (IS_ERR(prog)) return PTR_ERR(prog); + } else if (!bpf_mprog_detach_empty(ptype)) { + return -EPERM; } } else if (is_cgroup_prog_type(ptype, 0, false)) { if (attr->attach_flags || attr->relative_fd) @@ -5295,6 +5303,9 @@ static int bpf_map_get_info_by_fd(struct file *file, if (info.hash_size != SHA256_DIGEST_SIZE) return -EINVAL; + if (!READ_ONCE(map->frozen)) + return -EPERM; + err = map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, map->sha); if (err != 0) return err; @@ -6392,7 +6403,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .func = bpf_kallsyms_lookup_name, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index f8e70e9c3998d..4abc359b3db01 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -8,6 +8,7 @@ */ #include #include +#include #define TNUM(_v, _m) (struct tnum){.value = _v, .mask = _m} /* A completely unknown value */ @@ -253,3 +254,74 @@ struct tnum tnum_const_subreg(struct tnum a, u32 value) { return tnum_with_subreg(a, tnum_const(value)); } + +struct tnum tnum_bswap16(struct tnum a) +{ + return TNUM(swab16(a.value & 0xFFFF), swab16(a.mask & 0xFFFF)); +} + +struct tnum tnum_bswap32(struct tnum a) +{ + return TNUM(swab32(a.value & 0xFFFFFFFF), swab32(a.mask & 0xFFFFFFFF)); +} + +struct tnum tnum_bswap64(struct tnum a) +{ + return TNUM(swab64(a.value), swab64(a.mask)); +} + +/* Given tnum t, and a number z such that tmin <= z < tmax, where tmin + * is the smallest member of the t (= t.value) and tmax is the largest + * member of t (= t.value | t.mask), returns the smallest member of t + * larger than z. + * + * For example, + * t = x11100x0 + * z = 11110001 (241) + * result = 11110010 (242) + * + * Note: if this function is called with z >= tmax, it just returns + * early with tmax; if this function is called with z < tmin, the + * algorithm already returns tmin. + */ +u64 tnum_step(struct tnum t, u64 z) +{ + u64 tmax, j, p, q, r, s, v, u, w, res; + u8 k; + + tmax = t.value | t.mask; + + /* if z >= largest member of t, return largest member of t */ + if (z >= tmax) + return tmax; + + /* if z < smallest member of t, return smallest member of t */ + if (z < t.value) + return t.value; + + /* keep t's known bits, and match all unknown bits to z */ + j = t.value | (z & t.mask); + + if (j > z) { + p = ~z & t.value & ~t.mask; + k = fls64(p); /* k is the most-significant 0-to-1 flip */ + q = U64_MAX << k; + r = q & z; /* positions > k matched to z */ + s = ~q & t.value; /* positions <= k matched to t.value */ + v = r | s; + res = v; + } else { + p = z & ~t.value & ~t.mask; + k = fls64(p); /* k is the most-significant 1-to-0 flip */ + q = U64_MAX << k; + r = q & t.mask & z; /* unknown positions > k, matched to z */ + s = q & ~t.mask; /* known positions > k, set to 1 */ + v = r | s; + /* add 1 to unknown positions > k to make value greater than z */ + u = v + (1ULL << k); + /* extract bits in unknown positions > k from u, rest from t.value */ + w = (u & t.mask) | t.value; + res = w; + } + return res; +} diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index f2cb0b0970933..40f19147f227e 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -220,7 +220,9 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) } if (tr->func.ftrace_managed) { - ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); + ret = ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); + if (ret) + return ret; ret = register_ftrace_direct(tr->fops, (long)new_addr); } else { ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); @@ -749,10 +751,8 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, mutex_lock(&tr->mutex); shim_link = cgroup_shim_find(tr, bpf_func); - if (shim_link) { + if (shim_link && !IS_ERR(bpf_link_inc_not_zero(&shim_link->link.link))) { /* Reusing existing shim attached by the other program. */ - bpf_link_inc(&shim_link->link.link); - mutex_unlock(&tr->mutex); bpf_trampoline_put(tr); /* bpf_trampoline_get above */ return 0; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fbe4bb91c564a..db1c591a1da3b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -515,6 +515,7 @@ static bool is_callback_calling_kfunc(u32 btf_id); static bool is_bpf_throw_kfunc(struct bpf_insn *insn); static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id); +static bool is_task_work_add_kfunc(u32 func_id); static bool is_sync_callback_calling_function(enum bpf_func_id func_id) { @@ -547,6 +548,21 @@ static bool is_async_callback_calling_insn(struct bpf_insn *insn) (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm)); } +static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn *insn) +{ + /* bpf_timer callbacks are never sleepable. */ + if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) + return false; + + /* bpf_wq and bpf_task_work callbacks are always sleepable. */ + if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && + (is_bpf_wq_set_callback_impl_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm))) + return true; + + verifier_bug(env, "unhandled async callback in is_async_cb_sleepable"); + return false; +} + static bool is_may_goto_insn(struct bpf_insn *insn) { return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; @@ -2331,6 +2347,9 @@ static void __update_reg32_bounds(struct bpf_reg_state *reg) static void __update_reg64_bounds(struct bpf_reg_state *reg) { + u64 tnum_next, tmax; + bool umin_in_tnum; + /* min signed is max(sign bit) | min(other bits) */ reg->smin_value = max_t(s64, reg->smin_value, reg->var_off.value | (reg->var_off.mask & S64_MIN)); @@ -2340,6 +2359,33 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg) reg->umin_value = max(reg->umin_value, reg->var_off.value); reg->umax_value = min(reg->umax_value, reg->var_off.value | reg->var_off.mask); + + /* Check if u64 and tnum overlap in a single value */ + tnum_next = tnum_step(reg->var_off, reg->umin_value); + umin_in_tnum = (reg->umin_value & ~reg->var_off.mask) == reg->var_off.value; + tmax = reg->var_off.value | reg->var_off.mask; + if (umin_in_tnum && tnum_next > reg->umax_value) { + /* The u64 range and the tnum only overlap in umin. + * u64: ---[xxxxxx]----- + * tnum: --xx----------x- + */ + ___mark_reg_known(reg, reg->umin_value); + } else if (!umin_in_tnum && tnum_next == tmax) { + /* The u64 range and the tnum only overlap in the maximum value + * represented by the tnum, called tmax. + * u64: ---[xxxxxx]----- + * tnum: xx-----x-------- + */ + ___mark_reg_known(reg, tmax); + } else if (!umin_in_tnum && tnum_next <= reg->umax_value && + tnum_step(reg->var_off, tnum_next) > reg->umax_value) { + /* The u64 range and the tnum only overlap in between umin + * (excluded) and umax. + * u64: ---[xxxxxx]----- + * tnum: xx----x-------x- + */ + ___mark_reg_known(reg, tnum_next); + } } static void __update_reg_bounds(struct bpf_reg_state *reg) @@ -2433,6 +2479,30 @@ static void __reg32_deduce_bounds(struct bpf_reg_state *reg) if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) { reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value); reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value); + } else { + if (reg->u32_max_value < (u32)reg->s32_min_value) { + /* See __reg64_deduce_bounds() for detailed explanation. + * Refine ranges in the following situation: + * + * 0 U32_MAX + * | [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx] | + * |----------------------------|----------------------------| + * |xxxxx s32 range xxxxxxxxx] [xxxxxxx| + * 0 S32_MAX S32_MIN -1 + */ + reg->s32_min_value = (s32)reg->u32_min_value; + reg->u32_max_value = min_t(u32, reg->u32_max_value, reg->s32_max_value); + } else if ((u32)reg->s32_max_value < reg->u32_min_value) { + /* + * 0 U32_MAX + * | [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx] | + * |----------------------------|----------------------------| + * |xxxxxxxxx] [xxxxxxxxxxxx s32 range | + * 0 S32_MAX S32_MIN -1 + */ + reg->s32_max_value = (s32)reg->u32_max_value; + reg->u32_min_value = max_t(u32, reg->u32_min_value, reg->s32_min_value); + } } } @@ -4392,6 +4462,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } + if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call + && subseq_idx - idx != 1) { + if (bt_subprog_enter(bt)) + return -EFAULT; + } } else if (opcode == BPF_EXIT) { bool r0_precise; @@ -5826,8 +5901,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, static bool in_sleepable(struct bpf_verifier_env *env) { - return env->prog->sleepable || - (env->cur_state && env->cur_state->in_sleepable); + return env->cur_state->in_sleepable; } /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock() @@ -7663,7 +7737,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } else if (reg->type == CONST_PTR_TO_MAP) { err = check_ptr_to_map_access(env, regs, regno, off, size, t, value_regno); - } else if (base_type(reg->type) == PTR_TO_BUF) { + } else if (base_type(reg->type) == PTR_TO_BUF && + !type_may_be_null(reg->type)) { bool rdonly_mem = type_is_rdonly_mem(reg->type); u32 *max_access; @@ -8464,6 +8539,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno, case BPF_TASK_WORK: field_off = map->record->task_work_off; break; + case BPF_WORKQUEUE: + field_off = map->record->wq_off; + break; default: verifier_bug(env, "unsupported BTF field type: %s\n", struct_name); return -EINVAL; @@ -8505,13 +8583,17 @@ static int process_wq_func(struct bpf_verifier_env *env, int regno, { struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; struct bpf_map *map = reg->map_ptr; - u64 val = reg->var_off.value; + int err; - if (map->record->wq_off != val + reg->off) { - verbose(env, "off %lld doesn't point to 'struct bpf_wq' that is at %d\n", - val + reg->off, map->record->wq_off); - return -EINVAL; + err = check_map_field_pointer(env, regno, BPF_WORKQUEUE); + if (err) + return err; + + if (meta->map.ptr) { + verifier_bug(env, "Two map pointers in a bpf_wq helper"); + return -EFAULT; } + meta->map.uid = reg->map_uid; meta->map.ptr = map; return 0; @@ -10368,8 +10450,6 @@ typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, struct bpf_func_state *callee, int insn_idx); -static bool is_task_work_add_kfunc(u32 func_id); - static int set_callee_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx); @@ -10588,8 +10668,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins env->subprog_info[subprog].is_async_cb = true; async_cb = push_async_cb(env, env->subprog_info[subprog].start, insn_idx, subprog, - is_bpf_wq_set_callback_impl_kfunc(insn->imm) || - is_task_work_add_kfunc(insn->imm)); + is_async_cb_sleepable(env, insn)); if (!async_cb) return -EFAULT; callee = async_cb->frame[0]; @@ -10976,6 +11055,10 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) bool in_callback_fn; int err; + err = bpf_update_live_stack(env); + if (err) + return err; + callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; if (r0->type == PTR_TO_STACK) { @@ -11428,7 +11511,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (in_sleepable(env) && is_storage_get_function(func_id)) + if (is_storage_get_function(func_id)) env->insn_aux_data[insn_idx].storage_get_func_atomic = true; } @@ -11439,7 +11522,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (in_sleepable(env) && is_storage_get_function(func_id)) + if (is_storage_get_function(func_id)) env->insn_aux_data[insn_idx].storage_get_func_atomic = true; } @@ -11450,10 +11533,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (in_sleepable(env) && is_storage_get_function(func_id)) + if (is_storage_get_function(func_id)) env->insn_aux_data[insn_idx].storage_get_func_atomic = true; } + /* + * Non-sleepable contexts in sleepable programs (e.g., timer callbacks) + * are atomic and must use GFP_ATOMIC for storage_get helpers. + */ + if (!in_sleepable(env) && is_storage_get_function(func_id)) + env->insn_aux_data[insn_idx].storage_get_func_atomic = true; + meta.func_id = func_id; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { @@ -11886,6 +11976,25 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->prog->call_get_func_ip = true; } + if (func_id == BPF_FUNC_tail_call) { + if (env->cur_state->curframe) { + struct bpf_verifier_state *branch; + + mark_reg_scratched(env, BPF_REG_0); + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + clear_all_pkt_pointers(env); + mark_reg_unknown(env, regs, BPF_REG_0); + err = prepare_func_exit(env, &env->insn_idx); + if (err) + return err; + env->insn_idx--; + } else { + changes_data = false; + } + } + if (changes_data) clear_all_pkt_pointers(env); return 0; @@ -15170,21 +15279,17 @@ static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, u64 umin_val, u64 umax_val) { /* Special case <<32 because it is a common compiler pattern to sign - * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are - * positive we know this shift will also be positive so we can track - * bounds correctly. Otherwise we lose all sign bit information except - * what we can pick up from var_off. Perhaps we can generalize this - * later to shifts of any length. + * extend subreg by doing <<32 s>>32. smin/smax assignments are correct + * because s32 bounds don't flip sign when shifting to the left by + * 32bits. */ - if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) + if (umin_val == 32 && umax_val == 32) { dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; - else - dst_reg->smax_value = S64_MAX; - - if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; - else + } else { + dst_reg->smax_value = S64_MAX; dst_reg->smin_value = S64_MIN; + } /* If we might shift our top bit out, then we know nothing */ if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { @@ -15327,6 +15432,55 @@ static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, __update_reg_bounds(dst_reg); } +static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *insn) +{ + /* + * Byte swap operation - update var_off using tnum_bswap. + * Three cases: + * 1. bswap(16|32|64): opcode=0xd7 (BPF_END | BPF_ALU64 | BPF_TO_LE) + * unconditional swap + * 2. to_le(16|32|64): opcode=0xd4 (BPF_END | BPF_ALU | BPF_TO_LE) + * swap on big-endian, truncation or no-op on little-endian + * 3. to_be(16|32|64): opcode=0xdc (BPF_END | BPF_ALU | BPF_TO_BE) + * swap on little-endian, truncation or no-op on big-endian + */ + + bool alu64 = BPF_CLASS(insn->code) == BPF_ALU64; + bool to_le = BPF_SRC(insn->code) == BPF_TO_LE; + bool is_big_endian; +#ifdef CONFIG_CPU_BIG_ENDIAN + is_big_endian = true; +#else + is_big_endian = false; +#endif + /* Apply bswap if alu64 or switch between big-endian and little-endian machines */ + bool need_bswap = alu64 || (to_le == is_big_endian); + + /* + * If the register is mutated, manually reset its scalar ID to break + * any existing ties and avoid incorrect bounds propagation. + */ + if (need_bswap || insn->imm == 16 || insn->imm == 32) + dst_reg->id = 0; + + if (need_bswap) { + if (insn->imm == 16) + dst_reg->var_off = tnum_bswap16(dst_reg->var_off); + else if (insn->imm == 32) + dst_reg->var_off = tnum_bswap32(dst_reg->var_off); + else if (insn->imm == 64) + dst_reg->var_off = tnum_bswap64(dst_reg->var_off); + /* + * Byteswap scrambles the range, so we must reset bounds. + * Bounds will be re-derived from the new tnum later. + */ + __mark_reg_unbounded(dst_reg); + } + /* For bswap16/32, truncate dst register to match the swapped size */ + if (insn->imm == 16 || insn->imm == 32) + coerce_reg_to_size(dst_reg, insn->imm / 8); +} + static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, const struct bpf_reg_state *src_reg) { @@ -15353,6 +15507,7 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, case BPF_XOR: case BPF_OR: case BPF_MUL: + case BPF_END: return true; /* Shift operators range is only computable if shift dimension operand @@ -15368,6 +15523,35 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, } } +static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct bpf_reg_state *dst_reg) +{ + struct bpf_verifier_state *branch; + struct bpf_reg_state *regs; + bool alu32; + + if (dst_reg->smin_value == -1 && dst_reg->smax_value == 0) + alu32 = false; + else if (dst_reg->s32_min_value == -1 && dst_reg->s32_max_value == 0) + alu32 = true; + else + return 0; + + branch = push_stack(env, env->insn_idx, env->insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + + regs = branch->frame[branch->curframe]->regs; + if (alu32) { + __mark_reg32_known(®s[insn->dst_reg], 0); + __mark_reg32_known(dst_reg, -1ull); + } else { + __mark_reg_known(®s[insn->dst_reg], 0); + __mark_reg_known(dst_reg, -1ull); + } + return 0; +} + /* WARNING: This function does calculations on 64-bit values, but the actual * execution may occur on 32-bit values. Therefore, things like bitshifts * need extra checks in the 32-bit case. @@ -15430,11 +15614,21 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_mul(dst_reg, &src_reg); break; case BPF_AND: + if (tnum_is_const(src_reg.var_off)) { + ret = maybe_fork_scalars(env, insn, dst_reg); + if (ret) + return ret; + } dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); break; case BPF_OR: + if (tnum_is_const(src_reg.var_off)) { + ret = maybe_fork_scalars(env, insn, dst_reg); + if (ret) + return ret; + } dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); @@ -15462,12 +15656,23 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, else scalar_min_max_arsh(dst_reg, &src_reg); break; + case BPF_END: + scalar_byte_swap(dst_reg, insn); + break; default: break; } - /* ALU32 ops are zero extended into 64bit register */ - if (alu32) + /* + * ALU32 ops are zero extended into 64bit register. + * + * BPF_END is already handled inside the helper (truncation), + * so skip zext here to avoid unexpected zero extension. + * e.g., le64: opcode=(BPF_END|BPF_ALU|BPF_TO_LE), imm=0x40 + * This is a 64bit byte swap operation with alu32==true, + * but we should not zero extend the result. + */ + if (alu32 && opcode != BPF_END) zext_32_to_64(dst_reg); reg_bounds_sync(dst_reg); return 0; @@ -15647,7 +15852,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) } /* check dest operand */ - if (opcode == BPF_NEG && + if ((opcode == BPF_NEG || opcode == BPF_END) && regs[insn->dst_reg].type == SCALAR_VALUE) { err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); err = err ?: adjust_scalar_min_max_vals(env, insn, @@ -16610,17 +16815,24 @@ static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_st * in verifier state, save R in linked_regs if R->id == id. * If there are too many Rs sharing same id, reset id for leftover Rs. */ -static void collect_linked_regs(struct bpf_verifier_state *vstate, u32 id, +static void collect_linked_regs(struct bpf_verifier_env *env, + struct bpf_verifier_state *vstate, + u32 id, struct linked_regs *linked_regs) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_func_state *func; struct bpf_reg_state *reg; + u16 live_regs; int i, j; id = id & ~BPF_ADD_CONST; for (i = vstate->curframe; i >= 0; i--) { + live_regs = aux[frame_insn_idx(vstate, i)].live_regs_before; func = vstate->frame[i]; for (j = 0; j < BPF_REG_FP; j++) { + if (!(live_regs & BIT(j))) + continue; reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } @@ -16661,6 +16873,7 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s } else { s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->off; + u32 saved_id = reg->id; fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off); @@ -16668,10 +16881,11 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s /* reg = known_reg; reg += delta */ copy_register_state(reg, known_reg); /* - * Must preserve off, id and add_const flag, + * Must preserve off, id and subreg_def flag, * otherwise another sync_linked_regs() will be incorrect. */ reg->off = saved_off; + reg->id = saved_id; reg->subreg_def = saved_subreg_def; scalar32_min_max_add(reg, &fake_reg); @@ -16824,9 +17038,9 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * if parent state is created. */ if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) - collect_linked_regs(this_branch, src_reg->id, &linked_regs); + collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); if (dst_reg->type == SCALAR_VALUE && dst_reg->id) - collect_linked_regs(this_branch, dst_reg->id, &linked_regs); + collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs)); if (err) @@ -18772,8 +18986,13 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, * since someone could have accessed through (ptr - k), or * even done ptr -= k in a register, to get a safe access. */ - if (rold->range > rcur->range) + if (rold->range < 0 || rcur->range < 0) { + /* special case for [BEYOND|AT]_PKT_END */ + if (rold->range != rcur->range) + return false; + } else if (rold->range > rcur->range) { return false; + } /* If the offsets don't match, we can't trust our alignment; * nor can we be sure that we won't fall out of range. */ @@ -19766,7 +19985,8 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env, * state when it exits. */ int err = check_resource_leak(env, exception_exit, - !env->cur_state->curframe, + exception_exit || !env->cur_state->curframe, + exception_exit ? "bpf_throw" : "BPF_EXIT instruction in main prog"); if (err) return err; @@ -19784,9 +20004,6 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env, return PROCESS_BPF_EXIT; if (env->cur_state->curframe) { - err = bpf_update_live_stack(env); - if (err) - return err; /* exit from nested function */ err = prepare_func_exit(env, &env->insn_idx); if (err) @@ -20096,17 +20313,19 @@ static int do_check(struct bpf_verifier_env *env) * may skip a nospec patched-in after the jump. This can * currently never happen because nospec_result is only * used for the write-ops - * `*(size*)(dst_reg+off)=src_reg|imm32` which must - * never skip the following insn. Still, add a warning - * to document this in case nospec_result is used - * elsewhere in the future. + * `*(size*)(dst_reg+off)=src_reg|imm32` and helper + * calls. These must never skip the following insn + * (i.e., bpf_insn_successors()'s opcode_info.can_jump + * is false). Still, add a warning to document this in + * case nospec_result is used elsewhere in the future. * * All non-branch instructions have a single * fall-through edge. For these, nospec_result should * already work. */ - if (verifier_bug_if(BPF_CLASS(insn->code) == BPF_JMP || - BPF_CLASS(insn->code) == BPF_JMP32, env, + if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP || + BPF_CLASS(insn->code) == BPF_JMP32) && + BPF_OP(insn->code) != BPF_CALL, env, "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: @@ -20164,29 +20383,29 @@ static int find_btf_percpu_datasec(struct btf *btf) } /* - * Add btf to the used_btfs array and return the index. (If the btf was - * already added, then just return the index.) Upon successful insertion - * increase btf refcnt, and, if present, also refcount the corresponding - * kernel module. + * Add btf to the env->used_btfs array. If needed, refcount the + * corresponding kernel module. To simplify caller's logic + * in case of error or if btf was added before the function + * decreases the btf refcount. */ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) { struct btf_mod_pair *btf_mod; + int ret = 0; int i; /* check whether we recorded this BTF (and maybe module) already */ for (i = 0; i < env->used_btf_cnt; i++) if (env->used_btfs[i].btf == btf) - return i; + goto ret_put; if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); - return -E2BIG; + ret = -E2BIG; + goto ret_put; } - btf_get(btf); - btf_mod = &env->used_btfs[env->used_btf_cnt]; btf_mod->btf = btf; btf_mod->module = NULL; @@ -20195,12 +20414,18 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) if (btf_is_module(btf)) { btf_mod->module = btf_try_get_module(btf); if (!btf_mod->module) { - btf_put(btf); - return -ENXIO; + ret = -ENXIO; + goto ret_put; } } - return env->used_btf_cnt++; + env->used_btf_cnt++; + return 0; + +ret_put: + /* Either error or this BTF was already added */ + btf_put(btf); + return ret; } /* replace pseudo btf_id with kernel symbol address */ @@ -20297,9 +20522,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, btf_fd = insn[1].imm; if (btf_fd) { - CLASS(fd, f)(btf_fd); - - btf = __btf_get_by_fd(f); + btf = btf_get_by_fd(btf_fd); if (IS_ERR(btf)) { verbose(env, "invalid module BTF object FD specified.\n"); return -EINVAL; @@ -20309,17 +20532,17 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); return -EINVAL; } + btf_get(btf_vmlinux); btf = btf_vmlinux; } err = __check_pseudo_btf_id(env, insn, aux, btf); - if (err) + if (err) { + btf_put(btf); return err; + } - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + return __add_used_btf(env, btf); } static bool is_tracing_prog_type(enum bpf_prog_type type) @@ -22485,8 +22708,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) } if (is_storage_get_function(insn->imm)) { - if (!in_sleepable(env) || - env->insn_aux_data[i + delta].storage_get_func_atomic) + if (env->insn_aux_data[i + delta].storage_get_func_atomic) insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); else insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); @@ -23156,6 +23378,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) state->curframe = 0; state->speculative = false; state->branches = 1; + state->in_sleepable = env->prog->sleepable; state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -23896,7 +24119,6 @@ BTF_ID(func, __x64_sys_exit_group) BTF_ID(func, do_exit) BTF_ID(func, do_group_exit) BTF_ID(func, kthread_complete_and_exit) -BTF_ID(func, kthread_exit) BTF_ID(func, make_task_dead) BTF_SET_END(noreturn_deny) @@ -24041,10 +24263,8 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + btf_get(btf); + return __add_used_btf(env, btf); } verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); @@ -24200,6 +24420,12 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, case BPF_JMP32: switch (code) { case BPF_JA: + def = 0; + if (BPF_SRC(insn->code) == BPF_X) + use = dst; + else + use = 0; + break; case BPF_JCOND: def = 0; use = 0; diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index fdee387f0d6be..da5f6f5400afa 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -2611,6 +2611,7 @@ static void cgroup_migrate_add_task(struct task_struct *task, mgctx->tset.nr_tasks++; + css_set_skip_task_iters(cset, task); list_move_tail(&task->cg_list, &cset->mg_tasks); if (list_empty(&cset->mg_node)) list_add_tail(&cset->mg_node, diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 52468d2c178a3..08b0c264bd268 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -352,6 +352,15 @@ static inline bool is_in_v2_mode(void) (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); } +static inline bool cpuset_is_populated(struct cpuset *cs) +{ + lockdep_assert_held(&cpuset_mutex); + + /* Cpusets in the process of attaching should be considered as populated */ + return cgroup_is_populated(cs->css.cgroup) || + cs->attach_in_progress; +} + /** * partition_is_populated - check if partition has tasks * @cs: partition root to be checked @@ -364,21 +373,31 @@ static inline bool is_in_v2_mode(void) static inline bool partition_is_populated(struct cpuset *cs, struct cpuset *excluded_child) { - struct cgroup_subsys_state *css; - struct cpuset *child; + struct cpuset *cp; + struct cgroup_subsys_state *pos_css; - if (cs->css.cgroup->nr_populated_csets) + /* + * We cannot call cs_is_populated(cs) directly, as + * nr_populated_domain_children may include populated + * csets from descendants that are partitions. + */ + if (cs->css.cgroup->nr_populated_csets || + cs->attach_in_progress) return true; if (!excluded_child && !cs->nr_subparts) - return cgroup_is_populated(cs->css.cgroup); + return cpuset_is_populated(cs); rcu_read_lock(); - cpuset_for_each_child(child, css, cs) { - if (child == excluded_child) + cpuset_for_each_descendant_pre(cp, pos_css, cs) { + if (cp == cs || cp == excluded_child) continue; - if (is_partition_valid(child)) + + if (is_partition_valid(cp)) { + pos_css = css_rightmost_descendant(pos_css); continue; - if (cgroup_is_populated(child->css.cgroup)) { + } + + if (cpuset_is_populated(cp)) { rcu_read_unlock(); return true; } @@ -580,33 +599,31 @@ static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2) /** * cpus_excl_conflict - Check if two cpusets have exclusive CPU conflicts - * @cs1: first cpuset to check - * @cs2: second cpuset to check + * @trial: the trial cpuset to be checked + * @sibling: a sibling cpuset to be checked against + * @xcpus_changed: set if exclusive_cpus has been set * * Returns: true if CPU exclusivity conflict exists, false otherwise * * Conflict detection rules: * 1. If either cpuset is CPU exclusive, they must be mutually exclusive * 2. exclusive_cpus masks cannot intersect between cpusets - * 3. The allowed CPUs of one cpuset cannot be a subset of another's exclusive CPUs + * 3. The allowed CPUs of a sibling cpuset cannot be a subset of the new exclusive CPUs */ -static inline bool cpus_excl_conflict(struct cpuset *cs1, struct cpuset *cs2) +static inline bool cpus_excl_conflict(struct cpuset *trial, struct cpuset *sibling, + bool xcpus_changed) { /* If either cpuset is exclusive, check if they are mutually exclusive */ - if (is_cpu_exclusive(cs1) || is_cpu_exclusive(cs2)) - return !cpusets_are_exclusive(cs1, cs2); + if (is_cpu_exclusive(trial) || is_cpu_exclusive(sibling)) + return !cpusets_are_exclusive(trial, sibling); /* Exclusive_cpus cannot intersect */ - if (cpumask_intersects(cs1->exclusive_cpus, cs2->exclusive_cpus)) - return true; - - /* The cpus_allowed of one cpuset cannot be a subset of another cpuset's exclusive_cpus */ - if (!cpumask_empty(cs1->cpus_allowed) && - cpumask_subset(cs1->cpus_allowed, cs2->exclusive_cpus)) + if (cpumask_intersects(trial->exclusive_cpus, sibling->exclusive_cpus)) return true; - if (!cpumask_empty(cs2->cpus_allowed) && - cpumask_subset(cs2->cpus_allowed, cs1->exclusive_cpus)) + /* The cpus_allowed of a sibling cpuset cannot be a subset of the new exclusive_cpus */ + if (xcpus_changed && !cpumask_empty(sibling->cpus_allowed) && + cpumask_subset(sibling->cpus_allowed, trial->exclusive_cpus)) return true; return false; @@ -643,6 +660,7 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) { struct cgroup_subsys_state *css; struct cpuset *c, *par; + bool xcpus_changed; int ret = 0; rcu_read_lock(); @@ -663,7 +681,7 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * be changed to have empty cpus_allowed or mems_allowed. */ ret = -ENOSPC; - if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) { + if (cpuset_is_populated(cur)) { if (!cpumask_empty(cur->cpus_allowed) && cpumask_empty(trial->cpus_allowed)) goto out; @@ -699,10 +717,11 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * overlap. exclusive_cpus cannot overlap with each other if set. */ ret = -EINVAL; + xcpus_changed = !cpumask_equal(cur->exclusive_cpus, trial->exclusive_cpus); cpuset_for_each_child(c, css, par) { if (c == cur) continue; - if (cpus_excl_conflict(trial, c)) + if (cpus_excl_conflict(trial, c, xcpus_changed)) goto out; if (mems_excl_conflict(trial, c)) goto out; @@ -1581,7 +1600,14 @@ static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp) bool isolcpus_updated; WARN_ON_ONCE(!is_remote_partition(cs)); - WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); + /* + * When a CPU is offlined, top_cpuset may end up with no available CPUs, + * which should clear subpartitions_cpus. We should not emit a warning for this + * scenario: the hierarchy is updated from top to bottom, so subpartitions_cpus + * may already be cleared when disabling the partition. + */ + WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus) && + !cpumask_empty(subpartitions_cpus)); spin_lock_irq(&callback_lock); list_del_init(&cs->remote_sibling); @@ -2275,7 +2301,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, WARN_ON(!is_in_v2_mode() && !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); - cpuset_update_tasks_cpumask(cp, cp->effective_cpus); + cpuset_update_tasks_cpumask(cp, tmp->new_cpus); /* * On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE @@ -3908,8 +3934,9 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) if (remote || (is_partition_valid(cs) && is_partition_valid(parent))) compute_partition_effective_cpumask(cs, &new_cpus); - if (remote && cpumask_empty(&new_cpus) && - partition_is_populated(cs, NULL)) { + if (remote && (cpumask_empty(subpartitions_cpus) || + (cpumask_empty(&new_cpus) && + partition_is_populated(cs, NULL)))) { cs->prs_err = PERR_HOTPLUG; remote_partition_disable(cs, tmp); compute_effective_cpumask(&new_cpus, cs, parent); @@ -3922,9 +3949,12 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) * 1) empty effective cpus but not valid empty partition. * 2) parent is invalid or doesn't grant any cpus to child * partitions. + * 3) subpartitions_cpus is empty. */ - if (is_local_partition(cs) && (!is_partition_valid(parent) || - tasks_nocpu_error(parent, cs, &new_cpus))) + if (is_local_partition(cs) && + (!is_partition_valid(parent) || + tasks_nocpu_error(parent, cs, &new_cpus) || + cpumask_empty(subpartitions_cpus))) partcmd = partcmd_invalidate; /* * On the other hand, an invalid partition root may be transitioned @@ -4328,40 +4358,58 @@ bool cpuset_current_node_allowed(int node, gfp_t gfp_mask) return allowed; } -bool cpuset_node_allowed(struct cgroup *cgroup, int nid) +/** + * cpuset_nodes_allowed - return effective_mems mask from a cgroup cpuset. + * @cgroup: pointer to struct cgroup. + * @mask: pointer to struct nodemask_t to be returned. + * + * Returns effective_mems mask from a cgroup cpuset if it is cgroup v2 and + * has cpuset subsys. Otherwise, returns node_states[N_MEMORY]. + * + * This function intentionally avoids taking the cpuset_mutex or callback_lock + * when accessing effective_mems. This is because the obtained effective_mems + * is stale immediately after the query anyway (e.g., effective_mems is updated + * immediately after releasing the lock but before returning). + * + * As a result, returned @mask may be empty because cs->effective_mems can be + * rebound during this call. Besides, nodes in @mask are not guaranteed to be + * online due to hot plugins. Callers should check the mask for validity on + * return based on its subsequent use. + **/ +void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) { struct cgroup_subsys_state *css; struct cpuset *cs; - bool allowed; /* * In v1, mem_cgroup and cpuset are unlikely in the same hierarchy * and mems_allowed is likely to be empty even if we could get to it, - * so return true to avoid taking a global lock on the empty check. + * so return directly to avoid taking a global lock on the empty check. */ - if (!cpuset_v2()) - return true; + if (!cgroup || !cpuset_v2()) { + nodes_copy(*mask, node_states[N_MEMORY]); + return; + } css = cgroup_get_e_css(cgroup, &cpuset_cgrp_subsys); - if (!css) - return true; + if (!css) { + nodes_copy(*mask, node_states[N_MEMORY]); + return; + } /* - * Normally, accessing effective_mems would require the cpuset_mutex - * or callback_lock - but node_isset is atomic and the reference - * taken via cgroup_get_e_css is sufficient to protect css. - * - * Since this interface is intended for use by migration paths, we - * relax locking here to avoid taking global locks - while accepting - * there may be rare scenarios where the result may be innaccurate. + * The reference taken via cgroup_get_e_css is sufficient to + * protect css, but it does not imply safe accesses to effective_mems. * - * Reclaim and migration are subject to these same race conditions, and - * cannot make strong isolation guarantees, so this is acceptable. + * Normally, accessing effective_mems would require the cpuset_mutex + * or callback_lock - but the correctness of this information is stale + * immediately after the query anyway. We do not acquire the lock + * during this process to save lock contention in exchange for racing + * against mems_allowed rebinds. */ cs = container_of(css, struct cpuset, css); - allowed = node_isset(nid, cs->effective_mems); + nodes_copy(*mask, cs->effective_mems); css_put(css); - return allowed; } /** diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index e12b946278b6c..1ea6afffa985c 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -71,7 +72,9 @@ struct dmem_cgroup_pool_state { struct rcu_head rcu; struct page_counter cnt; + struct dmem_cgroup_pool_state *parent; + refcount_t ref; bool inited; }; @@ -88,6 +91,9 @@ struct dmem_cgroup_pool_state { static DEFINE_SPINLOCK(dmemcg_lock); static LIST_HEAD(dmem_cgroup_regions); +static void dmemcg_free_region(struct kref *ref); +static void dmemcg_pool_free_rcu(struct rcu_head *rcu); + static inline struct dmemcg_state * css_to_dmemcs(struct cgroup_subsys_state *css) { @@ -104,10 +110,38 @@ static struct dmemcg_state *parent_dmemcs(struct dmemcg_state *cg) return cg->css.parent ? css_to_dmemcs(cg->css.parent) : NULL; } +static void dmemcg_pool_get(struct dmem_cgroup_pool_state *pool) +{ + refcount_inc(&pool->ref); +} + +static bool dmemcg_pool_tryget(struct dmem_cgroup_pool_state *pool) +{ + return refcount_inc_not_zero(&pool->ref); +} + +static void dmemcg_pool_put(struct dmem_cgroup_pool_state *pool) +{ + if (!refcount_dec_and_test(&pool->ref)) + return; + + call_rcu(&pool->rcu, dmemcg_pool_free_rcu); +} + +static void dmemcg_pool_free_rcu(struct rcu_head *rcu) +{ + struct dmem_cgroup_pool_state *pool = container_of(rcu, typeof(*pool), rcu); + + if (pool->parent) + dmemcg_pool_put(pool->parent); + kref_put(&pool->region->ref, dmemcg_free_region); + kfree(pool); +} + static void free_cg_pool(struct dmem_cgroup_pool_state *pool) { list_del(&pool->region_node); - kfree(pool); + dmemcg_pool_put(pool); } static void @@ -342,6 +376,12 @@ alloc_pool_single(struct dmemcg_state *dmemcs, struct dmem_cgroup_region *region page_counter_init(&pool->cnt, ppool ? &ppool->cnt : NULL, true); reset_all_resource_limits(pool); + refcount_set(&pool->ref, 1); + kref_get(®ion->ref); + if (ppool && !pool->parent) { + pool->parent = ppool; + dmemcg_pool_get(ppool); + } list_add_tail_rcu(&pool->css_node, &dmemcs->pools); list_add_tail(&pool->region_node, ®ion->pools); @@ -389,6 +429,10 @@ get_cg_pool_locked(struct dmemcg_state *dmemcs, struct dmem_cgroup_region *regio /* Fix up parent links, mark as inited. */ pool->cnt.parent = &ppool->cnt; + if (ppool && !pool->parent) { + pool->parent = ppool; + dmemcg_pool_get(ppool); + } pool->inited = true; pool = ppool; @@ -423,7 +467,7 @@ static void dmemcg_free_region(struct kref *ref) */ void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region) { - struct list_head *entry; + struct dmem_cgroup_pool_state *pool, *next; if (!region) return; @@ -433,11 +477,10 @@ void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region) /* Remove from global region list */ list_del_rcu(®ion->region_node); - list_for_each_rcu(entry, ®ion->pools) { - struct dmem_cgroup_pool_state *pool = - container_of(entry, typeof(*pool), region_node); - + list_for_each_entry_safe(pool, next, ®ion->pools, region_node) { list_del_rcu(&pool->css_node); + list_del(&pool->region_node); + dmemcg_pool_put(pool); } /* @@ -518,8 +561,10 @@ static struct dmem_cgroup_region *dmemcg_get_region_by_name(const char *name) */ void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool) { - if (pool) + if (pool) { css_put(&pool->cs->css); + dmemcg_pool_put(pool); + } } EXPORT_SYMBOL_GPL(dmem_cgroup_pool_state_put); @@ -533,6 +578,8 @@ get_cg_pool_unlocked(struct dmemcg_state *cg, struct dmem_cgroup_region *region) pool = find_cg_pool_locked(cg, region); if (pool && !READ_ONCE(pool->inited)) pool = NULL; + if (pool && !dmemcg_pool_tryget(pool)) + pool = NULL; rcu_read_unlock(); while (!pool) { @@ -541,6 +588,8 @@ get_cg_pool_unlocked(struct dmemcg_state *cg, struct dmem_cgroup_region *region) pool = get_cg_pool_locked(cg, region, &allocpool); else pool = ERR_PTR(-ENODEV); + if (!IS_ERR(pool)) + dmemcg_pool_get(pool); spin_unlock(&dmemcg_lock); if (pool == ERR_PTR(-ENOMEM)) { @@ -576,6 +625,7 @@ void dmem_cgroup_uncharge(struct dmem_cgroup_pool_state *pool, u64 size) page_counter_uncharge(&pool->cnt, size); css_put(&pool->cs->css); + dmemcg_pool_put(pool); } EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge); @@ -627,7 +677,9 @@ int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size, if (ret_limit_pool) { *ret_limit_pool = container_of(fail, struct dmem_cgroup_pool_state, cnt); css_get(&(*ret_limit_pool)->cs->css); + dmemcg_pool_get(*ret_limit_pool); } + dmemcg_pool_put(pool); ret = -EAGAIN; goto err; } @@ -700,6 +752,9 @@ static ssize_t dmemcg_limit_write(struct kernfs_open_file *of, if (!region_name[0]) continue; + if (!options || !*options) + return -EINVAL; + rcu_read_lock(); region = dmemcg_get_region_by_name(region_name); rcu_read_unlock(); @@ -719,6 +774,7 @@ static ssize_t dmemcg_limit_write(struct kernfs_open_file *of, /* And commit */ apply(pool, new_limit); + dmemcg_pool_put(pool); out_put: kref_put(®ion->ref, dmemcg_free_region); diff --git a/kernel/cgroup/namespace.c b/kernel/cgroup/namespace.c index fdbe57578e688..db9617556dd70 100644 --- a/kernel/cgroup/namespace.c +++ b/kernel/cgroup/namespace.c @@ -30,7 +30,6 @@ static struct cgroup_namespace *alloc_cgroup_ns(void) ret = ns_common_init(new_ns); if (ret) return ERR_PTR(ret); - ns_tree_add(new_ns); return no_free_ptr(new_ns); } @@ -86,6 +85,7 @@ struct cgroup_namespace *copy_cgroup_ns(u64 flags, new_ns->ucounts = ucounts; new_ns->root_cset = cset; + ns_tree_add(new_ns); return new_ns; } diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c index a198e40c799b4..150e5871e66f2 100644 --- a/kernel/cgroup/rstat.c +++ b/kernel/cgroup/rstat.c @@ -71,7 +71,6 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu) { struct llist_head *lhead; struct css_rstat_cpu *rstatc; - struct css_rstat_cpu __percpu *rstatc_pcpu; struct llist_node *self; /* @@ -104,18 +103,22 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu) /* * This function can be renentered by irqs and nmis for the same cgroup * and may try to insert the same per-cpu lnode into the llist. Note - * that llist_add() does not protect against such scenarios. + * that llist_add() does not protect against such scenarios. In addition + * this same per-cpu lnode can be modified through init_llist_node() + * from css_rstat_flush() running on a different CPU. * * To protect against such stacked contexts of irqs/nmis, we use the * fact that lnode points to itself when not on a list and then use - * this_cpu_cmpxchg() to atomically set to NULL to select the winner + * try_cmpxchg() to atomically set to NULL to select the winner * which will call llist_add(). The losers can assume the insertion is * successful and the winner will eventually add the per-cpu lnode to * the llist. + * + * Please note that we can not use this_cpu_cmpxchg() here as on some + * archs it is not safe against modifications from multiple CPUs. */ self = &rstatc->lnode; - rstatc_pcpu = css->rstat_cpu; - if (this_cpu_cmpxchg(rstatc_pcpu->lnode.next, self, NULL) != self) + if (!try_cmpxchg(&rstatc->lnode.next, &self, NULL)) return; lhead = ss_lhead_cpu(css->ss, cpu); diff --git a/kernel/configs/debug.config b/kernel/configs/debug.config index e81327d2cd639..dc5f7dd084f91 100644 --- a/kernel/configs/debug.config +++ b/kernel/configs/debug.config @@ -29,16 +29,15 @@ CONFIG_SECTION_MISMATCH_WARN_ONLY=y # CONFIG_UBSAN_ALIGNMENT is not set # CONFIG_UBSAN_DIV_ZERO is not set # CONFIG_UBSAN_TRAP is not set -# CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set CONFIG_DEBUG_FS=y CONFIG_DEBUG_FS_ALLOW_ALL=y CONFIG_DEBUG_IRQFLAGS=y -CONFIG_UBSAN=y -CONFIG_UBSAN_BOOL=y -CONFIG_UBSAN_BOUNDS=y -CONFIG_UBSAN_ENUM=y -CONFIG_UBSAN_SHIFT=y -CONFIG_UBSAN_UNREACHABLE=y +# CONFIG_UBSAN is not set +# CONFIG_UBSAN_BOOL is not set +# CONFIG_UBSAN_BOUNDS is not set +# CONFIG_UBSAN_ENUM is not set +# CONFIG_UBSAN_SHIFT is not set +# CONFIG_UBSAN_UNREACHABLE is not set # # Networking Debugging # @@ -57,8 +56,8 @@ CONFIG_DEBUG_NET=y # CONFIG_SLUB_STATS is not set CONFIG_PAGE_EXTENSION=y CONFIG_PAGE_OWNER=y -CONFIG_DEBUG_KMEMLEAK=y -CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN=y +# CONFIG_DEBUG_KMEMLEAK is not set +# CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN is not set CONFIG_DEBUG_OBJECTS=y CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT=1 CONFIG_DEBUG_OBJECTS_FREE=y @@ -73,10 +72,10 @@ CONFIG_DEBUG_VM=y CONFIG_DEBUG_VM_PGFLAGS=y CONFIG_DEBUG_VM_RB=y CONFIG_DEBUG_VM_VMACACHE=y -CONFIG_KASAN=y -CONFIG_KASAN_GENERIC=y -CONFIG_KASAN_INLINE=y -CONFIG_KASAN_VMALLOC=y +# CONFIG_KASAN is not set +# CONFIG_KASAN_GENERIC is not set +# CONFIG_KASAN_INLINE is not set +# CONFIG_KASAN_VMALLOC is not set CONFIG_PTDUMP_DEBUGFS=y CONFIG_SCHED_STACK_END_CHECK=y CONFIG_SLUB_DEBUG_ON=y @@ -94,7 +93,7 @@ CONFIG_SOFTLOCKUP_DETECTOR=y # Lock Debugging (spinlocks, mutexes, etc...) # # CONFIG_PROVE_RAW_LOCK_NESTING is not set -CONFIG_PROVE_LOCKING=y +# CONFIG_PROVE_LOCKING is not set # # Debug kernel data structures # @@ -102,18 +101,55 @@ CONFIG_BUG_ON_DATA_CORRUPTION=y # # RCU Debugging # -CONFIG_RCU_EXPERT=y -CONFIG_PROVE_RCU=y -CONFIG_PROVE_RCU_LIST=y +# CONFIG_RCU_EXPERT is not set +# CONFIG_PROVE_RCU is not set +# CONFIG_PROVE_RCU_LIST is not set # # Tracers # CONFIG_BRANCH_PROFILE_NONE=y CONFIG_DYNAMIC_FTRACE=y CONFIG_FTRACE=y -CONFIG_FUNCTION_TRACER=y +# CONFIG_FUNCTION_TRACER is not set # # Preemption # CONFIG_DEBUG_PREEMPT=y CONFIG_PREEMPT=y + +# +# Qualcomm Debug Configs +# +CONFIG_RELAY=y +CONFIG_ATH11K_CFR=y +CONFIG_ATH11K_COREDUMP=y +CONFIG_ATH11K_DEBUG=y +CONFIG_ATH11K_DEBUGFS=y +CONFIG_ATH11K_TRACING=y +CONFIG_ATH12K_DEBUG=y +CONFIG_ATH12K_DEBUGFS=y +CONFIG_ATH12K_TRACING=y +CONFIG_ATH12K_COREDUMP=y +CONFIG_CFG80211_DEBUGFS=y +CONFIG_CMA_DEBUG=y +CONFIG_DEBUG_LIST=y +CONFIG_DEBUG_PAGEALLOC=y +CONFIG_DYNAMIC_FTRACE_WITH_REGS=y +CONFIG_FTRACE_SYSCALLS=y +CONFIG_FUNCTION_GRAPH_TRACER=y +# CONFIG_FW_LOADER_DEBUG is not set +CONFIG_HARDLOCKUP_DETECTOR=y +CONFIG_IRQSOFF_TRACER=y +CONFIG_KPROBE_EVENTS=y +CONFIG_MAC80211_DEBUGFS=y +CONFIG_NL80211_TESTMODE=y +CONFIG_PREEMPT_TRACER=y +CONFIG_PREEMPTIRQ_TRACEPOINTS=y +CONFIG_PM_ADVANCED_DEBUG=y +CONFIG_PM_DEBUG=y +CONFIG_PM_SLEEP_DEBUG=y +CONFIG_SCHED_TRACER=y +CONFIG_STACK_TRACER=y +CONFIG_TRACEPOINTS=y +CONFIG_TRACING=y +CONFIG_UPROBE_EVENTS=y diff --git a/kernel/cpu.c b/kernel/cpu.c index db9f6c539b28c..15000c7abc659 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -249,6 +249,14 @@ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, return ret; } +/* + * The former STARTING/DYING states, ran with IRQs disabled and must not fail. + */ +static bool cpuhp_is_atomic_state(enum cpuhp_state state) +{ + return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE; +} + #ifdef CONFIG_SMP static bool cpuhp_is_ap_state(enum cpuhp_state state) { @@ -271,14 +279,6 @@ static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup) complete(done); } -/* - * The former STARTING/DYING states, ran with IRQs disabled and must not fail. - */ -static bool cpuhp_is_atomic_state(enum cpuhp_state state) -{ - return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE; -} - /* Synchronization state management */ enum cpuhp_sync_state { SYNC_STATE_DEAD, @@ -2364,7 +2364,14 @@ static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup, else ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); #else - ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); + if (cpuhp_is_atomic_state(state)) { + guard(irqsave)(); + ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); + /* STARTING/DYING must not fail! */ + WARN_ON_ONCE(ret); + } else { + ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); + } #endif BUG_ON(ret && !bringup); return ret; diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c index 401423ba477da..9501b0704f19f 100644 --- a/kernel/crash_dump_dm_crypt.c +++ b/kernel/crash_dump_dm_crypt.c @@ -143,6 +143,7 @@ static int read_key_from_user_keying(struct dm_crypt_key *dm_key) { const struct user_key_payload *ukp; struct key *key; + int ret = 0; kexec_dprintk("Requesting logon key %s", dm_key->key_desc); key = request_key(&key_type_logon, dm_key->key_desc, NULL); @@ -152,20 +153,28 @@ static int read_key_from_user_keying(struct dm_crypt_key *dm_key) return PTR_ERR(key); } + down_read(&key->sem); ukp = user_key_payload_locked(key); - if (!ukp) - return -EKEYREVOKED; + if (!ukp) { + ret = -EKEYREVOKED; + goto out; + } if (ukp->datalen > KEY_SIZE_MAX) { pr_err("Key size %u exceeds maximum (%u)\n", ukp->datalen, KEY_SIZE_MAX); - return -EINVAL; + ret = -EINVAL; + goto out; } memcpy(dm_key->data, ukp->data, ukp->datalen); dm_key->key_size = ukp->datalen; - kexec_dprintk("Get dm crypt key (size=%u) %s: %8ph\n", dm_key->key_size, - dm_key->key_desc, dm_key->data); - return 0; + kexec_dprintk("Get dm crypt key (size=%u) %s\n", dm_key->key_size, + dm_key->key_desc); + +out: + up_read(&key->sem); + key_put(key); + return ret; } struct config_key { diff --git a/kernel/crash_reserve.c b/kernel/crash_reserve.c index 87bf4d41eabba..62e60e0223cff 100644 --- a/kernel/crash_reserve.c +++ b/kernel/crash_reserve.c @@ -524,6 +524,9 @@ void __init reserve_crashkernel_cma(unsigned long long cma_size) #ifndef HAVE_ARCH_ADD_CRASH_RES_TO_IOMEM_EARLY static __init int insert_crashkernel_resources(void) { + if (!arch_add_crash_res_to_iomem()) + return 0; + if (crashk_res.start < crashk_res.end) insert_resource(&iomem_resource, &crashk_res); diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index d9b9dcba6ff7c..9071c08650e3a 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -90,6 +90,16 @@ static int __init early_cma(char *p) } early_param("cma", early_cma); +/* + * cma_skip_dt_default_reserved_mem - This is called from the + * reserved_mem framework to detect if the default cma region is being + * set by the "cma=" kernel parameter. + */ +bool __init cma_skip_dt_default_reserved_mem(void) +{ + return size_cmdline != -1; +} + #ifdef CONFIG_DMA_NUMA_CMA static struct cma *dma_contiguous_numa_area[MAX_NUMNODES]; @@ -463,12 +473,6 @@ static int __init rmem_cma_setup(struct reserved_mem *rmem) struct cma *cma; int err; - if (size_cmdline != -1 && default_cma) { - pr_info("Reserved memory: bypass %s node, using cmdline CMA params instead\n", - rmem->name); - return -EBUSY; - } - if (!of_get_flat_dt_prop(node, "reusable", NULL) || of_get_flat_dt_prop(node, "no-map", NULL)) return -EINVAL; diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 138ede653de40..21db331185911 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -63,6 +63,7 @@ enum map_err_types { * @sg_mapped_ents: 'mapped_ents' from dma_map_sg * @paddr: physical start address of the mapping * @map_err_type: track whether dma_mapping_error() was checked + * @is_cache_clean: driver promises not to write to buffer while mapped * @stack_len: number of backtrace entries in @stack_entries * @stack_entries: stack of backtrace history */ @@ -76,7 +77,8 @@ struct dma_debug_entry { int sg_call_ents; int sg_mapped_ents; phys_addr_t paddr; - enum map_err_types map_err_type; + enum map_err_types map_err_type; + bool is_cache_clean; #ifdef CONFIG_STACKTRACE unsigned int stack_len; unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; @@ -451,7 +453,7 @@ static int active_cacheline_set_overlap(phys_addr_t cln, int overlap) return overlap; } -static void active_cacheline_inc_overlap(phys_addr_t cln) +static void active_cacheline_inc_overlap(phys_addr_t cln, bool is_cache_clean) { int overlap = active_cacheline_read_overlap(cln); @@ -460,7 +462,7 @@ static void active_cacheline_inc_overlap(phys_addr_t cln) /* If we overflowed the overlap counter then we're potentially * leaking dma-mappings. */ - WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP, + WARN_ONCE(!is_cache_clean && overlap > ACTIVE_CACHELINE_MAX_OVERLAP, pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"), ACTIVE_CACHELINE_MAX_OVERLAP, &cln); } @@ -472,12 +474,15 @@ static int active_cacheline_dec_overlap(phys_addr_t cln) return active_cacheline_set_overlap(cln, --overlap); } -static int active_cacheline_insert(struct dma_debug_entry *entry) +static int active_cacheline_insert(struct dma_debug_entry *entry, + bool *overlap_cache_clean) { phys_addr_t cln = to_cacheline_number(entry); unsigned long flags; int rc; + *overlap_cache_clean = false; + /* If the device is not writing memory then we don't have any * concerns about the cpu consuming stale data. This mitigates * legitimate usages of overlapping mappings. @@ -487,8 +492,16 @@ static int active_cacheline_insert(struct dma_debug_entry *entry) spin_lock_irqsave(&radix_lock, flags); rc = radix_tree_insert(&dma_active_cacheline, cln, entry); - if (rc == -EEXIST) - active_cacheline_inc_overlap(cln); + if (rc == -EEXIST) { + struct dma_debug_entry *existing; + + active_cacheline_inc_overlap(cln, entry->is_cache_clean); + existing = radix_tree_lookup(&dma_active_cacheline, cln); + /* A lookup failure here after we got -EEXIST is unexpected. */ + WARN_ON(!existing); + if (existing) + *overlap_cache_clean = existing->is_cache_clean; + } spin_unlock_irqrestore(&radix_lock, flags); return rc; @@ -583,19 +596,25 @@ DEFINE_SHOW_ATTRIBUTE(dump); */ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) { + bool overlap_cache_clean; struct hash_bucket *bucket; unsigned long flags; int rc; + entry->is_cache_clean = !!(attrs & DMA_ATTR_CPU_CACHE_CLEAN); + bucket = get_hash_bucket(entry, &flags); hash_bucket_add(bucket, entry); put_hash_bucket(bucket, flags); - rc = active_cacheline_insert(entry); + rc = active_cacheline_insert(entry, &overlap_cache_clean); if (rc == -ENOMEM) { pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n"); global_disable = true; - } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + } else if (rc == -EEXIST && + !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + !(entry->is_cache_clean && overlap_cache_clean) && + dma_get_cache_alignment() >= L1_CACHE_BYTES && !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && is_swiotlb_active(entry->dev))) { err_printk(entry->dev, entry, diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index da2fadf45bcd6..62f0d9d0ba02e 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -88,7 +88,7 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev, if (is_swiotlb_force_bounce(dev)) { if (attrs & DMA_ATTR_MMIO) - goto err_overflow; + return DMA_MAPPING_ERROR; return swiotlb_map(dev, phys, size, dir, attrs); } diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index ee45dee33d491..985d6aa102b67 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -93,7 +93,7 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, page = dma_alloc_from_contiguous(NULL, 1 << order, order, false); if (!page) - page = alloc_pages(gfp, order); + page = alloc_pages(gfp | __GFP_NOWARN, order); } while (!page && order-- > 0); if (!page) goto out; @@ -268,15 +268,20 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size, { struct gen_pool *pool = NULL; struct page *page; + bool pool_found = false; while ((pool = dma_guess_pool(pool, gfp))) { + pool_found = true; page = __dma_alloc_from_pool(dev, size, pool, cpu_addr, phys_addr_ok); if (page) return page; } - WARN(1, "Failed to get suitable pool for %s\n", dev_name(dev)); + if (pool_found) + WARN(!(gfp & __GFP_NOWARN), "DMA pool exhausted for %s\n", dev_name(dev)); + else + WARN(1, "Failed to get suitable pool for %s\n", dev_name(dev)); return NULL; } diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 0d37da3d95b65..e27225f8aeb27 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -903,10 +904,19 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size local_irq_save(flags); page = pfn_to_page(pfn); - if (dir == DMA_TO_DEVICE) + if (dir == DMA_TO_DEVICE) { + /* + * Ideally, kmsan_check_highmem_page() + * could be used here to detect infoleaks, + * but callers may map uninitialized buffers + * that will be written by the device, + * causing false positives. + */ memcpy_from_page(vaddr, page, offset, sz); - else + } else { + kmsan_unpoison_memory(vaddr, sz); memcpy_to_page(page, offset, vaddr, sz); + } local_irq_restore(flags); size -= sz; @@ -915,8 +925,15 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size offset = 0; } } else if (dir == DMA_TO_DEVICE) { + /* + * Ideally, kmsan_check_memory() could be used here to detect + * infoleaks (uninitialized data being sent to device), but + * callers may map uninitialized buffers that will be written + * by the device, causing false positives. + */ memcpy(vaddr, phys_to_virt(orig_addr), size); } else { + kmsan_unpoison_memory(vaddr, size); memcpy(phys_to_virt(orig_addr), vaddr, size); } } diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c index 808c0d7a31faf..575154dbd2319 100644 --- a/kernel/events/callchain.c +++ b/kernel/events/callchain.c @@ -246,7 +246,7 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, if (user && !crosstask) { if (!user_mode(regs)) { - if (current->flags & (PF_KTHREAD | PF_USER_WORKER)) + if (!is_user_task(current)) goto exit_put; regs = task_pt_regs(current); } diff --git a/kernel/events/core.c b/kernel/events/core.c index 2c35acc2722b0..6b6fea8d33e04 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -2316,8 +2316,6 @@ static void perf_group_detach(struct perf_event *event) perf_event__header_size(leader); } -static void sync_child_event(struct perf_event *child_event); - static void perf_child_detach(struct perf_event *event) { struct perf_event *parent_event = event->parent; @@ -2336,7 +2334,6 @@ static void perf_child_detach(struct perf_event *event) lockdep_assert_held(&parent_event->child_mutex); */ - sync_child_event(event); list_del_init(&event->child_list); } @@ -4019,7 +4016,8 @@ static int merge_sched_in(struct perf_event *event, void *data) if (*perf_event_fasync(event)) event->pending_kill = POLL_ERR; - perf_event_wakeup(event); + event->pending_wakeup = 1; + irq_work_queue(&event->pending_irq); } else { struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu); @@ -4587,6 +4585,7 @@ static void perf_event_enable_on_exec(struct perf_event_context *ctx) static void perf_remove_from_owner(struct perf_event *event); static void perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx, + struct task_struct *task, bool revoke); /* @@ -4614,7 +4613,7 @@ static void perf_event_remove_on_exec(struct perf_event_context *ctx) modified = true; - perf_event_exit_event(event, ctx, false); + perf_event_exit_event(event, ctx, ctx->task, false); } raw_spin_lock_irqsave(&ctx->lock, flags); @@ -4672,7 +4671,7 @@ static void __perf_event_read(void *info) struct perf_event *sub, *event = data->event; struct perf_event_context *ctx = event->ctx; struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); - struct pmu *pmu = event->pmu; + struct pmu *pmu; /* * If this is a task context, we need to check whether it is @@ -4684,7 +4683,7 @@ static void __perf_event_read(void *info) if (ctx->task && cpuctx->task_ctx != ctx) return; - raw_spin_lock(&ctx->lock); + guard(raw_spinlock)(&ctx->lock); ctx_time_update_event(ctx, event); perf_event_update_time(event); @@ -4692,25 +4691,22 @@ static void __perf_event_read(void *info) perf_event_update_sibling_time(event); if (event->state != PERF_EVENT_STATE_ACTIVE) - goto unlock; + return; if (!data->group) { - pmu->read(event); + perf_pmu_read(event); data->ret = 0; - goto unlock; + return; } + pmu = event->pmu_ctx->pmu; pmu->start_txn(pmu, PERF_PMU_TXN_READ); - pmu->read(event); - + perf_pmu_read(event); for_each_sibling_event(sub, event) perf_pmu_read(sub); data->ret = pmu->commit_txn(pmu); - -unlock: - raw_spin_unlock(&ctx->lock); } static inline u64 perf_event_count(struct perf_event *event, bool self) @@ -5281,9 +5277,20 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, return -ENOMEM; for (;;) { - if (try_cmpxchg((struct perf_ctx_data **)&task->perf_ctx_data, &old, cd)) { + if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) { if (old) perf_free_ctx_data_rcu(old); + /* + * Above try_cmpxchg() pairs with try_cmpxchg() from + * detach_task_ctx_data() such that + * if we race with perf_event_exit_task(), we must + * observe PF_EXITING. + */ + if (task->flags & PF_EXITING) { + /* detach_task_ctx_data() may free it already */ + if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL)) + perf_free_ctx_data_rcu(cd); + } return 0; } @@ -5329,6 +5336,8 @@ attach_global_ctx_data(struct kmem_cache *ctx_cache) /* Allocate everything */ scoped_guard (rcu) { for_each_process_thread(g, p) { + if (p->flags & PF_EXITING) + continue; cd = rcu_dereference(p->perf_ctx_data); if (cd && !cd->global) { cd->global = 1; @@ -6998,6 +7007,15 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (data_page_nr(event->rb) != nr_pages) return -EINVAL; + /* + * If this event doesn't have mmap_count, we're attempting to + * create an alias of another event's mmap(); this would mean + * both events will end up scribbling the same user_page; + * which makes no sense. + */ + if (!refcount_read(&event->mmap_count)) + return -EBUSY; + if (refcount_inc_not_zero(&event->rb->mmap_count)) { /* * Success -- managed to mmap() the same buffer @@ -7167,28 +7185,28 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) ret = perf_mmap_aux(vma, event, nr_pages); if (ret) return ret; - } - /* - * Since pinned accounting is per vm we cannot allow fork() to copy our - * vma. - */ - vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_ops = &perf_mmap_vmops; + /* + * Since pinned accounting is per vm we cannot allow fork() to copy our + * vma. + */ + vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP); + vma->vm_ops = &perf_mmap_vmops; - mapped = get_mapped(event, event_mapped); - if (mapped) - mapped(event, vma->vm_mm); + mapped = get_mapped(event, event_mapped); + if (mapped) + mapped(event, vma->vm_mm); - /* - * Try to map it into the page table. On fail, invoke - * perf_mmap_close() to undo the above, as the callsite expects - * full cleanup in this case and therefore does not invoke - * vmops::close(). - */ - ret = map_range(event->rb, vma); - if (ret) - perf_mmap_close(vma); + /* + * Try to map it into the page table. On fail, invoke + * perf_mmap_close() to undo the above, as the callsite expects + * full cleanup in this case and therefore does not invoke + * vmops::close(). + */ + ret = map_range(event->rb, vma); + if (ret) + perf_mmap_close(vma); + } return ret; } @@ -7452,7 +7470,7 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, if (user_mode(regs)) { regs_user->abi = perf_reg_abi(current); regs_user->regs = regs; - } else if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) { + } else if (is_user_task(current)) { perf_get_regs_user(regs_user, regs); } else { regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; @@ -8092,7 +8110,7 @@ static u64 perf_virt_to_phys(u64 virt) * Try IRQ-safe get_user_page_fast_only first. * If failed, leave phys_addr as 0. */ - if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) { + if (is_user_task(current)) { struct page *p; pagefault_disable(); @@ -8205,7 +8223,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs) { bool kernel = !event->attr.exclude_callchain_kernel; bool user = !event->attr.exclude_callchain_user && - !(current->flags & (PF_KTHREAD | PF_USER_WORKER)); + is_user_task(current); /* Disallow cross-task user callchains. */ bool crosstask = event->ctx->task && event->ctx->task != current; const u32 max_stack = event->attr.sample_max_stack; @@ -10406,6 +10424,13 @@ int perf_event_overflow(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { + /* + * Entry point from hardware PMI, interrupts should be disabled here. + * This serializes us against perf_event_remove_from_context() in + * things like perf_event_release_kernel(). + */ + lockdep_assert_irqs_disabled(); + return __perf_event_overflow(event, 1, data, regs); } @@ -10482,6 +10507,19 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, { struct hw_perf_event *hwc = &event->hw; + /* + * This is: + * - software preempt + * - tracepoint preempt + * - tp_target_task irq (ctx->lock) + * - uprobes preempt/irq + * - kprobes preempt/irq + * - hw_breakpoint irq + * + * Any of these are sufficient to hold off RCU and thus ensure @event + * exists. + */ + lockdep_assert_preemption_disabled(); local64_add(nr, &event->count); if (!regs) @@ -10490,6 +10528,16 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, if (!is_sampling_event(event)) return; + /* + * Serialize against event_function_call() IPIs like normal overflow + * event handling. Specifically, must not allow + * perf_event_release_kernel() -> perf_remove_from_context() to make + * progress and 'release' the event from under us. + */ + guard(irqsave)(); + if (event->state != PERF_EVENT_STATE_ACTIVE) + return; + if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { data->period = nr; return perf_swevent_overflow(event, 1, data, regs); @@ -10988,6 +11036,11 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, struct perf_sample_data data; struct perf_event *event; + /* + * Per being a tracepoint, this runs with preemption disabled. + */ + lockdep_assert_preemption_disabled(); + struct perf_raw_record raw = { .frag = { .size = entry_size, @@ -11320,6 +11373,11 @@ void perf_bp_event(struct perf_event *bp, void *data) struct perf_sample_data sample; struct pt_regs *regs = data; + /* + * Exception context, will have interrupts disabled. + */ + lockdep_assert_irqs_disabled(); + perf_sample_data_init(&sample, bp->attr.bp_addr, 0); if (!bp->hw.state && !perf_exclude_event(bp, regs)) @@ -11784,7 +11842,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) if (regs && !perf_exclude_event(event, regs)) { if (!(event->attr.exclude_idle && is_idle_task(current))) - if (__perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) ret = HRTIMER_NORESTART; } @@ -11837,6 +11895,11 @@ static void perf_swevent_cancel_hrtimer(struct perf_event *event) } } +static void perf_swevent_destroy_hrtimer(struct perf_event *event) +{ + hrtimer_cancel(&event->hw.hrtimer); +} + static void perf_swevent_init_hrtimer(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -11845,6 +11908,7 @@ static void perf_swevent_init_hrtimer(struct perf_event *event) return; hrtimer_setup(&hwc->hrtimer, perf_swevent_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); + event->destroy = perf_swevent_destroy_hrtimer; /* * Since hrtimers have a fixed rate, we can do a static freq->period @@ -12447,7 +12511,7 @@ static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event, /* * De-schedule the event and mark it REVOKED. */ - perf_event_exit_event(event, ctx, true); + perf_event_exit_event(event, ctx, ctx->task, true); /* * All _free_event() bits that rely on event->pmu: @@ -14004,14 +14068,13 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) } EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); -static void sync_child_event(struct perf_event *child_event) +static void sync_child_event(struct perf_event *child_event, + struct task_struct *task) { struct perf_event *parent_event = child_event->parent; u64 child_val; if (child_event->attr.inherit_stat) { - struct task_struct *task = child_event->ctx->task; - if (task && task != TASK_TOMBSTONE) perf_event_read_event(child_event, task); } @@ -14030,7 +14093,9 @@ static void sync_child_event(struct perf_event *child_event) static void perf_event_exit_event(struct perf_event *event, - struct perf_event_context *ctx, bool revoke) + struct perf_event_context *ctx, + struct task_struct *task, + bool revoke) { struct perf_event *parent_event = event->parent; unsigned long detach_flags = DETACH_EXIT; @@ -14053,6 +14118,9 @@ perf_event_exit_event(struct perf_event *event, mutex_lock(&parent_event->child_mutex); /* PERF_ATTACH_ITRACE might be set concurrently */ attach_state = READ_ONCE(event->attach_state); + + if (attach_state & PERF_ATTACH_CHILD) + sync_child_event(event, task); } if (revoke) @@ -14144,7 +14212,7 @@ static void perf_event_exit_task_context(struct task_struct *task, bool exit) perf_event_task(task, ctx, 0); list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry) - perf_event_exit_event(child_event, ctx, false); + perf_event_exit_event(child_event, ctx, exit ? task : NULL, false); mutex_unlock(&ctx->mutex); @@ -14206,8 +14274,11 @@ void perf_event_exit_task(struct task_struct *task) /* * Detach the perf_ctx_data for the system-wide event. + * + * Done without holding global_ctx_data_rwsem; typically + * attach_global_ctx_data() will skip over this task, but otherwise + * attach_task_ctx_data() will observe PF_EXITING. */ - guard(percpu_read)(&global_ctx_data_rwsem); detach_task_ctx_data(task); } @@ -14316,7 +14387,7 @@ inherit_event(struct perf_event *parent_event, get_ctx(child_ctx); child_event->ctx = child_ctx; - pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event); + pmu_ctx = find_get_pmu_context(parent_event->pmu_ctx->pmu, child_ctx, child_event); if (IS_ERR(pmu_ctx)) { free_event(child_event); return ERR_CAST(pmu_ctx); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index f11ceb8be8c41..725c5772429dc 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1138,7 +1138,7 @@ static bool filter_chain(struct uprobe *uprobe, struct mm_struct *mm) bool ret = false; down_read(&uprobe->consumer_rwsem); - list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) { + list_for_each_entry(uc, &uprobe->consumers, cons_node) { ret = consumer_filter(uc, mm); if (ret) break; @@ -1694,6 +1694,12 @@ static const struct vm_special_mapping xol_mapping = { .mremap = xol_mremap, }; +unsigned long __weak arch_uprobe_get_xol_area(void) +{ + /* Try to map as high as possible, this is only a hint. */ + return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); +} + /* Slot allocation for XOL */ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) { @@ -1709,9 +1715,7 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) } if (!area->vaddr) { - /* Try to map as high as possible, this is only a hint. */ - area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, - PAGE_SIZE, 0, 0); + area->vaddr = arch_uprobe_get_xol_area(); if (IS_ERR_VALUE(area->vaddr)) { ret = area->vaddr; goto fail; diff --git a/kernel/exit.c b/kernel/exit.c index 9f74e8f1c431b..c8c3ff935a84b 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -897,11 +897,16 @@ static void synchronize_group_exit(struct task_struct *tsk, long code) void __noreturn do_exit(long code) { struct task_struct *tsk = current; + struct kthread *kthread; int group_dead; WARN_ON(irqs_disabled()); WARN_ON(tsk->plug); + kthread = tsk_is_kthread(tsk); + if (unlikely(kthread)) + kthread_do_exit(kthread, code); + kcov_task_exit(tsk); kmsan_task_exit(tsk); @@ -1008,6 +1013,7 @@ void __noreturn do_exit(long code) lockdep_free_task(tsk); do_task_dead(); } +EXPORT_SYMBOL(do_exit); void __noreturn make_task_dead(int signr) { diff --git a/kernel/fork.c b/kernel/fork.c index 3da0f08615a95..924a9e10106b3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -3040,7 +3040,7 @@ static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) return 0; /* don't need lock here; in the worst case we'll do useless copy */ - if (fs->users == 1) + if (!(unshare_flags & CLONE_NEWNS) && fs->users == 1) return 0; *new_fsp = copy_fs_struct(fs); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 2e77a6e5c8657..9e7dea6fc0ccd 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -342,7 +342,7 @@ static int __futex_key_to_node(struct mm_struct *mm, unsigned long addr) if (!vma) return FUTEX_NO_NODE; - mpol = vma_policy(vma); + mpol = READ_ONCE(vma->vm_policy); if (!mpol) return FUTEX_NO_NODE; diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index dacb2330f1fbc..64cb87d3a73e8 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -918,7 +918,7 @@ int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked) int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock) { struct hrtimer_sleeper timeout, *to; - struct task_struct *exiting = NULL; + struct task_struct *exiting; struct rt_mutex_waiter rt_waiter; struct futex_q q = futex_q_init; DEFINE_WAKE_Q(wake_q); @@ -933,6 +933,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl to = futex_setup_timer(time, &timeout, flags, 0); retry: + exiting = NULL; ret = get_futex_key(uaddr, flags, &q.key, FUTEX_WRITE); if (unlikely(ret != 0)) goto out; diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 880c9bf2f3150..99723189c8cf7 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -459,6 +459,14 @@ SYSCALL_DEFINE4(futex_requeue, if (ret) return ret; + /* + * For now mandate both flags are identical, like the sys_futex() + * interface has. If/when we merge the variable sized futex support, + * that patch can modify this test to allow a difference in size. + */ + if (futexes[0].w.flags != futexes[1].w.flags) + return -EINVAL; + cmpval = futexes[0].w.val; return futex_requeue(u64_to_user_ptr(futexes[0].w.uaddr), futexes[0].w.flags, diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index 755346ea98196..cd5689e383b00 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -177,9 +177,11 @@ void irq_migrate_all_off_this_cpu(void) bool affinity_broken; desc = irq_to_desc(irq); - scoped_guard(raw_spinlock, &desc->lock) + scoped_guard(raw_spinlock, &desc->lock) { affinity_broken = migrate_one_irq(desc); - + if (affinity_broken && desc->affinity_notify) + irq_affinity_schedule_notify_work(desc); + } if (affinity_broken) { pr_debug_ratelimited("IRQ %u: no longer affine to CPU%u\n", irq, smp_processor_id()); diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 0164ca48da59e..5568ed3a8b852 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -135,6 +135,7 @@ extern bool irq_can_set_affinity_usr(unsigned int irq); extern int irq_do_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force); +extern void irq_affinity_schedule_notify_work(struct irq_desc *desc); #ifdef CONFIG_SMP extern int irq_setup_affinity(struct irq_desc *desc); @@ -142,7 +143,6 @@ extern int irq_setup_affinity(struct irq_desc *desc); static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; } #endif - #define for_each_action_of_desc(desc, act) \ for (act = desc->action; act; act = act->next) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 400856abf6721..c09751b7a0c49 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -347,6 +347,21 @@ static bool irq_set_affinity_deactivated(struct irq_data *data, return true; } +/** + * irq_affinity_schedule_notify_work - Schedule work to notify about affinity change + * @desc: Interrupt descriptor whose affinity changed + */ +void irq_affinity_schedule_notify_work(struct irq_desc *desc) +{ + lockdep_assert_held(&desc->lock); + + kref_get(&desc->affinity_notify->kref); + if (!schedule_work(&desc->affinity_notify->work)) { + /* Work was already scheduled, drop our extra ref */ + kref_put(&desc->affinity_notify->kref, desc->affinity_notify->release); + } +} + int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask, bool force) { @@ -367,14 +382,9 @@ int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask, irq_copy_pending(desc, mask); } - if (desc->affinity_notify) { - kref_get(&desc->affinity_notify->kref); - if (!schedule_work(&desc->affinity_notify->work)) { - /* Work was already scheduled, drop our extra ref */ - kref_put(&desc->affinity_notify->kref, - desc->affinity_notify->release); - } - } + if (desc->affinity_notify) + irq_affinity_schedule_notify_work(desc); + irqd_set(data, IRQD_AFFINITY_SET); return ret; diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 1e76358641247..e0813ca9469a3 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -103,8 +103,11 @@ static char kallsyms_get_symbol_type(unsigned int off) { /* * Get just the first code, look it up in the token table, - * and return the first char from this token. + * and return the first char from this token. If MSB of length + * is 1, it is a "big" symbol, so needs an additional byte. */ + if (kallsyms_names[off] & 0x80) + off++; return kallsyms_token_table[kallsyms_token_index[kallsyms_names[off + 1]]]; } @@ -342,7 +345,7 @@ int kallsyms_lookup_size_offset(unsigned long addr, unsigned long *symbolsize, return 1; } return !!module_address_lookup(addr, symbolsize, offset, NULL, NULL, namebuf) || - !!__bpf_address_lookup(addr, symbolsize, offset, namebuf); + !!bpf_address_lookup(addr, symbolsize, offset, namebuf); } static int kallsyms_lookup_buildid(unsigned long addr, @@ -352,8 +355,21 @@ static int kallsyms_lookup_buildid(unsigned long addr, { int ret; - namebuf[KSYM_NAME_LEN - 1] = 0; + /* + * kallsyms_lookus() returns pointer to namebuf on success and + * NULL on error. But some callers ignore the return value. + * Instead they expect @namebuf filled either with valid + * or empty string. + */ namebuf[0] = 0; + /* + * Initialize the module-related return values. They are not set + * when the symbol is in vmlinux or it is a bpf address. + */ + if (modname) + *modname = NULL; + if (modbuildid) + *modbuildid = NULL; if (is_ksym_addr(addr)) { unsigned long pos; @@ -362,10 +378,6 @@ static int kallsyms_lookup_buildid(unsigned long addr, /* Grab name */ kallsyms_expand_symbol(get_symbol_offset(pos), namebuf, KSYM_NAME_LEN); - if (modname) - *modname = NULL; - if (modbuildid) - *modbuildid = NULL; return strlen(namebuf); } @@ -374,12 +386,11 @@ static int kallsyms_lookup_buildid(unsigned long addr, ret = module_address_lookup(addr, symbolsize, offset, modname, modbuildid, namebuf); if (!ret) - ret = bpf_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = bpf_address_lookup(addr, symbolsize, offset, namebuf); if (!ret) - ret = ftrace_mod_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = ftrace_mod_address_lookup(addr, symbolsize, offset, + modname, modbuildid, namebuf); return ret; } @@ -423,6 +434,37 @@ int lookup_symbol_name(unsigned long addr, char *symname) return lookup_module_symbol_name(addr, symname); } +#ifdef CONFIG_STACKTRACE_BUILD_ID + +static int append_buildid(char *buffer, const char *modname, + const unsigned char *buildid) +{ + if (!modname) + return 0; + + if (!buildid) { + pr_warn_once("Undefined buildid for the module %s\n", modname); + return 0; + } + + /* build ID should match length of sprintf */ +#ifdef CONFIG_MODULES + static_assert(sizeof(typeof_member(struct module, build_id)) == 20); +#endif + + return sprintf(buffer, " %20phN", buildid); +} + +#else /* CONFIG_STACKTRACE_BUILD_ID */ + +static int append_buildid(char *buffer, const char *modname, + const unsigned char *buildid) +{ + return 0; +} + +#endif /* CONFIG_STACKTRACE_BUILD_ID */ + /* Look up a kernel symbol and return it in a text buffer. */ static int __sprint_symbol(char *buffer, unsigned long address, int symbol_offset, int add_offset, int add_buildid) @@ -432,6 +474,9 @@ static int __sprint_symbol(char *buffer, unsigned long address, unsigned long offset, size; int len; + /* Prevent module removal until modname and modbuildid are printed */ + guard(rcu)(); + address += symbol_offset; len = kallsyms_lookup_buildid(address, &size, &offset, &modname, &buildid, buffer); @@ -445,15 +490,8 @@ static int __sprint_symbol(char *buffer, unsigned long address, if (modname) { len += sprintf(buffer + len, " [%s", modname); -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) - if (add_buildid && buildid) { - /* build ID should match length of sprintf */ -#if IS_ENABLED(CONFIG_MODULES) - static_assert(sizeof(typeof_member(struct module, build_id)) == 20); -#endif - len += sprintf(buffer + len, " %20phN", buildid); - } -#endif + if (add_buildid) + len += append_buildid(buffer + len, modname, buildid); len += sprintf(buffer + len, "]"); } diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index fa00b239c5d9a..08ae3c64caca5 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -960,17 +960,24 @@ int kimage_load_segment(struct kimage *image, int idx) return result; } -void *kimage_map_segment(struct kimage *image, - unsigned long addr, unsigned long size) +void *kimage_map_segment(struct kimage *image, int idx) { + unsigned long addr, size, eaddr; unsigned long src_page_addr, dest_page_addr = 0; - unsigned long eaddr = addr + size; kimage_entry_t *ptr, entry; struct page **src_pages; unsigned int npages; + struct page *cma; void *vaddr = NULL; int i; + cma = image->segment_cma[idx]; + if (cma) + return page_address(cma); + + addr = image->segment[idx].mem; + size = image->segment[idx].memsz; + eaddr = addr + size; /* * Collect the source pages and map them in a contiguous VA range. */ @@ -1011,7 +1018,8 @@ void *kimage_map_segment(struct kimage *image, void kimage_unmap_segment(void *segment_buffer) { - vunmap(segment_buffer); + if (is_vmalloc_addr(segment_buffer)) + vunmap(segment_buffer); } struct kexec_load_limit { diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index eb62a97942428..2bfbb2d144e69 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -882,6 +882,60 @@ static int kexec_calculate_store_digests(struct kimage *image) } #ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY +/* + * kexec_purgatory_find_symbol - find a symbol in the purgatory + * @pi: Purgatory to search in. + * @name: Name of the symbol. + * + * Return: pointer to symbol in read-only symtab on success, NULL on error. + */ +static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi, + const char *name) +{ + const Elf_Shdr *sechdrs; + const Elf_Ehdr *ehdr; + const Elf_Sym *syms; + const char *strtab; + int i, k; + + if (!pi->ehdr) + return NULL; + + ehdr = pi->ehdr; + sechdrs = (void *)ehdr + ehdr->e_shoff; + + for (i = 0; i < ehdr->e_shnum; i++) { + if (sechdrs[i].sh_type != SHT_SYMTAB) + continue; + + if (sechdrs[i].sh_link >= ehdr->e_shnum) + /* Invalid strtab section number */ + continue; + strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset; + syms = (void *)ehdr + sechdrs[i].sh_offset; + + /* Go through symbols for a match */ + for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) { + if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL) + continue; + + if (strcmp(strtab + syms[k].st_name, name) != 0) + continue; + + if (syms[k].st_shndx == SHN_UNDEF || + syms[k].st_shndx >= ehdr->e_shnum) { + pr_debug("Symbol: %s has bad section index %d.\n", + name, syms[k].st_shndx); + return NULL; + } + + /* Found the symbol we are looking for */ + return &syms[k]; + } + } + + return NULL; +} /* * kexec_purgatory_setup_kbuf - prepare buffer to load purgatory. * @pi: Purgatory to be loaded. @@ -960,6 +1014,10 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, unsigned long offset; size_t sechdrs_size; Elf_Shdr *sechdrs; + const Elf_Sym *entry_sym; + u16 entry_shndx = 0; + unsigned long entry_off = 0; + bool start_fixed = false; int i; /* @@ -977,6 +1035,12 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, bss_addr = kbuf->mem + kbuf->bufsz; kbuf->image->start = pi->ehdr->e_entry; + entry_sym = kexec_purgatory_find_symbol(pi, "purgatory_start"); + if (entry_sym) { + entry_shndx = entry_sym->st_shndx; + entry_off = entry_sym->st_value; + } + for (i = 0; i < pi->ehdr->e_shnum; i++) { unsigned long align; void *src, *dst; @@ -994,6 +1058,13 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, offset = ALIGN(offset, align); + if (!start_fixed && entry_sym && i == entry_shndx && + (sechdrs[i].sh_flags & SHF_EXECINSTR) && + entry_off < sechdrs[i].sh_size) { + kbuf->image->start = kbuf->mem + offset + entry_off; + start_fixed = true; + } + /* * Check if the segment contains the entry point, if so, * calculate the value of image->start based on it. @@ -1004,13 +1075,14 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, * is not set to the initial value, and warn the user so they * have a chance to fix their purgatory's linker script. */ - if (sechdrs[i].sh_flags & SHF_EXECINSTR && + if (!start_fixed && sechdrs[i].sh_flags & SHF_EXECINSTR && pi->ehdr->e_entry >= sechdrs[i].sh_addr && pi->ehdr->e_entry < (sechdrs[i].sh_addr + sechdrs[i].sh_size) && - !WARN_ON(kbuf->image->start != pi->ehdr->e_entry)) { + kbuf->image->start == pi->ehdr->e_entry) { kbuf->image->start -= sechdrs[i].sh_addr; kbuf->image->start += kbuf->mem + offset; + start_fixed = true; } src = (void *)pi->ehdr + sechdrs[i].sh_offset; @@ -1128,61 +1200,6 @@ int kexec_load_purgatory(struct kimage *image, struct kexec_buf *kbuf) return ret; } -/* - * kexec_purgatory_find_symbol - find a symbol in the purgatory - * @pi: Purgatory to search in. - * @name: Name of the symbol. - * - * Return: pointer to symbol in read-only symtab on success, NULL on error. - */ -static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi, - const char *name) -{ - const Elf_Shdr *sechdrs; - const Elf_Ehdr *ehdr; - const Elf_Sym *syms; - const char *strtab; - int i, k; - - if (!pi->ehdr) - return NULL; - - ehdr = pi->ehdr; - sechdrs = (void *)ehdr + ehdr->e_shoff; - - for (i = 0; i < ehdr->e_shnum; i++) { - if (sechdrs[i].sh_type != SHT_SYMTAB) - continue; - - if (sechdrs[i].sh_link >= ehdr->e_shnum) - /* Invalid strtab section number */ - continue; - strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset; - syms = (void *)ehdr + sechdrs[i].sh_offset; - - /* Go through symbols for a match */ - for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) { - if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL) - continue; - - if (strcmp(strtab + syms[k].st_name, name) != 0) - continue; - - if (syms[k].st_shndx == SHN_UNDEF || - syms[k].st_shndx >= ehdr->e_shnum) { - pr_debug("Symbol: %s has bad section index %d.\n", - name, syms[k].st_shndx); - return NULL; - } - - /* Found the symbol we are looking for */ - return &syms[k]; - } - } - - return NULL; -} - void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name) { struct purgatory_info *pi = &image->purgatory_info; diff --git a/kernel/kexec_handover.c b/kernel/kexec_handover.c index 03d12e27189fc..2da4bd994322f 100644 --- a/kernel/kexec_handover.c +++ b/kernel/kexec_handover.c @@ -619,7 +619,7 @@ static void __init kho_reserve_scratch(void) scratch_size_update(); /* FIXME: deal with node hot-plug/remove */ - kho_scratch_cnt = num_online_nodes() + 2; + kho_scratch_cnt = nodes_weight(node_states[N_MEMORY]) + 2; size = kho_scratch_cnt * sizeof(*kho_scratch); kho_scratch = memblock_alloc(size, PAGE_SIZE); if (!kho_scratch) @@ -649,7 +649,11 @@ static void __init kho_reserve_scratch(void) kho_scratch[i].size = size; i++; - for_each_online_node(nid) { + /* + * Loop over nodes that have both memory and are online. Skip + * memoryless nodes, as we can not allocate scratch areas there. + */ + for_each_node_state(nid, N_MEMORY) { size = scratch_size_node(nid); addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES, 0, MEMBLOCK_ALLOC_ACCESSIBLE, diff --git a/kernel/kprobes.c b/kernel/kprobes.c index ab8f9fc1f0d17..87e6f4d61b95b 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1070,12 +1070,12 @@ static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, lockdep_assert_held(&kprobe_mutex); ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0); - if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret)) + if (ret < 0) return ret; if (*cnt == 0) { ret = register_ftrace_function(ops); - if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret)) { + if (ret < 0) { /* * At this point, sinec ops is not registered, we should be sefe from * registering empty filter. @@ -1104,6 +1104,10 @@ static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, int ret; lockdep_assert_held(&kprobe_mutex); + if (unlikely(kprobe_ftrace_disabled)) { + /* Now ftrace is disabled forever, disarm is already done. */ + return 0; + } if (*cnt == 1) { ret = unregister_ftrace_function(ops); diff --git a/kernel/kthread.c b/kernel/kthread.c index 31b072e8d4279..d2ee1d982a4d4 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -85,24 +85,6 @@ static inline struct kthread *to_kthread(struct task_struct *k) return k->worker_private; } -/* - * Variant of to_kthread() that doesn't assume @p is a kthread. - * - * When "(p->flags & PF_KTHREAD)" is set the task is a kthread and will - * always remain a kthread. For kthreads p->worker_private always - * points to a struct kthread. For tasks that are not kthreads - * p->worker_private is used to point to other things. - * - * Return NULL for any task that is not a kthread. - */ -static inline struct kthread *__to_kthread(struct task_struct *p) -{ - void *kthread = p->worker_private; - if (kthread && !(p->flags & PF_KTHREAD)) - kthread = NULL; - return kthread; -} - void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk) { struct kthread *kthread = to_kthread(tsk); @@ -193,7 +175,7 @@ EXPORT_SYMBOL_GPL(kthread_should_park); bool kthread_should_stop_or_park(void) { - struct kthread *kthread = __to_kthread(current); + struct kthread *kthread = tsk_is_kthread(current); if (!kthread) return false; @@ -234,7 +216,7 @@ EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); */ void *kthread_func(struct task_struct *task) { - struct kthread *kthread = __to_kthread(task); + struct kthread *kthread = tsk_is_kthread(task); if (kthread) return kthread->threadfn; return NULL; @@ -266,7 +248,7 @@ EXPORT_SYMBOL_GPL(kthread_data); */ void *kthread_probe_data(struct task_struct *task) { - struct kthread *kthread = __to_kthread(task); + struct kthread *kthread = tsk_is_kthread(task); void *data = NULL; if (kthread) @@ -309,19 +291,8 @@ void kthread_parkme(void) } EXPORT_SYMBOL_GPL(kthread_parkme); -/** - * kthread_exit - Cause the current kthread return @result to kthread_stop(). - * @result: The integer value to return to kthread_stop(). - * - * While kthread_exit can be called directly, it exists so that - * functions which do some additional work in non-modular code such as - * module_put_and_kthread_exit can be implemented. - * - * Does not return. - */ -void __noreturn kthread_exit(long result) +void kthread_do_exit(struct kthread *kthread, long result) { - struct kthread *kthread = to_kthread(current); kthread->result = result; if (!list_empty(&kthread->hotplug_node)) { mutex_lock(&kthreads_hotplug_lock); @@ -333,9 +304,7 @@ void __noreturn kthread_exit(long result) kthread->preferred_affinity = NULL; } } - do_exit(0); } -EXPORT_SYMBOL(kthread_exit); /** * kthread_complete_and_exit - Exit the current kthread. @@ -682,7 +651,7 @@ void kthread_set_per_cpu(struct task_struct *k, int cpu) bool kthread_is_per_cpu(struct task_struct *p) { - struct kthread *kthread = __to_kthread(p); + struct kthread *kthread = tsk_is_kthread(p); if (!kthread) return false; diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 0e73fac55f8eb..4e7a5cbc40a91 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -88,8 +88,14 @@ static struct klp_func *klp_find_func(struct klp_object *obj, struct klp_func *func; klp_for_each_func(obj, func) { + /* + * Besides identical old_sympos, also consider old_sympos + * of 0 and 1 are identical. + */ if ((strcmp(old_func->old_name, func->old_name) == 0) && - (old_func->old_sympos == func->old_sympos)) { + ((old_func->old_sympos == func->old_sympos) || + (old_func->old_sympos == 0 && func->old_sympos == 1) || + (old_func->old_sympos == 1 && func->old_sympos == 0))) { return func; } } diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index ce0362f0a8719..6567e5eeacc0e 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -103,8 +103,8 @@ static const struct kernel_param_ops lt_bind_ops = { .get = param_get_cpumask, }; -module_param_cb(bind_readers, <_bind_ops, &bind_readers, 0644); -module_param_cb(bind_writers, <_bind_ops, &bind_writers, 0644); +module_param_cb(bind_readers, <_bind_ops, &bind_readers, 0444); +module_param_cb(bind_writers, <_bind_ops, &bind_writers, 0444); long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn); @@ -1211,6 +1211,10 @@ static void lock_torture_cleanup(void) cxt.cur_ops->exit(); cxt.init_called = false; } + + free_cpumask_var(bind_readers); + free_cpumask_var(bind_writers); + torture_cleanup_end(); } diff --git a/kernel/locking/spinlock_debug.c b/kernel/locking/spinlock_debug.c index 87b03d2e41dbb..2338b3adfb55f 100644 --- a/kernel/locking/spinlock_debug.c +++ b/kernel/locking/spinlock_debug.c @@ -184,8 +184,8 @@ void do_raw_read_unlock(rwlock_t *lock) static inline void debug_write_lock_before(rwlock_t *lock) { RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic"); - RWLOCK_BUG_ON(lock->owner == current, lock, "recursion"); - RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(), + RWLOCK_BUG_ON(READ_ONCE(lock->owner) == current, lock, "recursion"); + RWLOCK_BUG_ON(READ_ONCE(lock->owner_cpu) == raw_smp_processor_id(), lock, "cpu recursion"); } diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 00a60796327c0..0fc11e45df9b9 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -334,13 +334,8 @@ int module_address_lookup(unsigned long addr, if (mod) { if (modname) *modname = mod->name; - if (modbuildid) { -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) - *modbuildid = mod->build_id; -#else - *modbuildid = NULL; -#endif - } + if (modbuildid) + *modbuildid = module_buildid(mod); sym = find_kallsyms_symbol(mod, addr, size, offset); diff --git a/kernel/module/main.c b/kernel/module/main.c index c66b261849362..66d4efbddfffe 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1568,6 +1568,13 @@ static int simplify_symbols(struct module *mod, const struct load_info *info) break; default: + if (sym[i].st_shndx >= info->hdr->e_shnum) { + pr_err("%s: Symbol %s has an invalid section index %u (max %u)\n", + mod->name, name, sym[i].st_shndx, info->hdr->e_shnum - 1); + ret = -ENOEXEC; + break; + } + /* Divert to percpu allocation if a percpu var. */ if (sym[i].st_shndx == info->index.pcpu) secbase = (unsigned long)mod_percpu(mod); @@ -3544,12 +3551,6 @@ static int load_module(struct load_info *info, const char __user *uargs, mutex_unlock(&module_mutex); free_module: mod_stat_bump_invalid(info, flags); - /* Free lock-classes; relies on the preceding sync_rcu() */ - for_class_mod_mem_type(type, core_data) { - lockdep_free_key_range(mod->mem[type].base, - mod->mem[type].size); - } - module_memory_restore_rox(mod); module_deallocate(mod, info); free_copy: diff --git a/kernel/nscommon.c b/kernel/nscommon.c index c1fb2bad6d729..22b5c5d0385f5 100644 --- a/kernel/nscommon.c +++ b/kernel/nscommon.c @@ -75,3 +75,9 @@ void __ns_common_free(struct ns_common *ns) { proc_free_inum(ns->inum); } + +bool may_see_all_namespaces(void) +{ + return (task_active_pid_ns(current) == &init_pid_ns) && + ns_capable_noaudit(init_pid_ns.user_ns, CAP_SYS_ADMIN); +} diff --git a/kernel/panic.c b/kernel/panic.c index 24cc3eec1805f..b13f040680ac1 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -131,7 +131,8 @@ static int proc_taint(const struct ctl_table *table, int write, static int sysctl_panic_print_handler(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { - panic_print_deprecated(); + if (write) + panic_print_deprecated(); return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); } @@ -1010,7 +1011,6 @@ static int panic_print_set(const char *val, const struct kernel_param *kp) static int panic_print_get(char *val, const struct kernel_param *kp) { - panic_print_deprecated(); return param_get_ulong(val, kp); } diff --git a/kernel/power/main.c b/kernel/power/main.c index 549f51ca3a1e6..ed7b5953d1ce0 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -38,7 +38,7 @@ void pm_restore_gfp_mask(void) { WARN_ON(!mutex_is_locked(&system_transition_mutex)); - if (WARN_ON(!saved_gfp_count) || --saved_gfp_count) + if (!saved_gfp_count || --saved_gfp_count) return; gfp_allowed_mask = saved_gfp_mask; diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 645f42e404789..e249e5786fbcd 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -2856,6 +2856,17 @@ int snapshot_write_finalize(struct snapshot_handle *handle) { int error; + /* + * Call snapshot_write_next() to drain any trailing zero pages, + * but make sure we're in the data page region first. + * This function can return PAGE_SIZE if the kernel was expecting + * another copy page. Return -ENODATA in that situation. + */ + if (handle->cur > nr_meta_pages + 1) { + error = snapshot_write_next(handle); + if (error) + return error > 0 ? -ENODATA : error; + } copy_last_highmem_page(); error = hibernate_restore_protect_page(handle->buffer); /* Do that only if we have loaded the image entirely */ diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 70ae21f7370d4..f910a250ccddb 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -897,8 +897,11 @@ static int save_compressed_image(struct swap_map_handle *handle, for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); - acomp_request_free(data[thr].cr); - crypto_free_acomp(data[thr].cc); + if (data[thr].cr) + acomp_request_free(data[thr].cr); + + if (!IS_ERR_OR_NULL(data[thr].cc)) + crypto_free_acomp(data[thr].cc); } vfree(data); } @@ -1519,8 +1522,11 @@ static int load_compressed_image(struct swap_map_handle *handle, for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); - acomp_request_free(data[thr].cr); - crypto_free_acomp(data[thr].cc); + if (data[thr].cr) + acomp_request_free(data[thr].cr); + + if (!IS_ERR_OR_NULL(data[thr].cc)) + crypto_free_acomp(data[thr].cc); } vfree(data); } diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index f72bbfa266d6c..b20929b7d71f5 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -230,6 +230,8 @@ struct console_flush_type { bool legacy_offload; }; +extern bool console_irqwork_blocked; + /* * Identify which console flushing methods should be used in the context of * the caller. @@ -241,7 +243,7 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft) switch (nbcon_get_default_prio()) { case NBCON_PRIO_NORMAL: if (have_nbcon_console && !have_boot_console) { - if (printk_kthreads_running) + if (printk_kthreads_running && !console_irqwork_blocked) ft->nbcon_offload = true; else ft->nbcon_atomic = true; @@ -251,7 +253,7 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft) if (have_legacy_console || have_boot_console) { if (!is_printk_legacy_deferred()) ft->legacy_direct = true; - else + else if (!console_irqwork_blocked) ft->legacy_offload = true; } break; @@ -264,7 +266,7 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft) if (have_legacy_console || have_boot_console) { if (!is_printk_legacy_deferred()) ft->legacy_direct = true; - else + else if (!console_irqwork_blocked) ft->legacy_offload = true; } break; diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 558ef31779760..730d14f6cbc58 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1276,6 +1276,13 @@ void nbcon_kthreads_wake(void) if (!printk_kthreads_running) return; + /* + * It is not allowed to call this function when console irq_work + * is blocked. + */ + if (WARN_ON_ONCE(console_irqwork_blocked)) + return; + cookie = console_srcu_read_lock(); for_each_console_srcu(con) { if (!(console_srcu_read_flags(con) & CON_NBCON)) @@ -1849,7 +1856,7 @@ void nbcon_device_release(struct console *con) if (console_trylock()) console_unlock(); } else if (ft.legacy_offload) { - printk_trigger_flush(); + defer_console_output(); } } console_srcu_read_unlock(cookie); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 5aee9ffb16b9a..c27fc7fc64eb5 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -462,6 +462,9 @@ bool have_boot_console; /* See printk_legacy_allow_panic_sync() for details. */ bool legacy_allow_panic_sync; +/* Avoid using irq_work when suspending. */ +bool console_irqwork_blocked; + #ifdef CONFIG_PRINTK DECLARE_WAIT_QUEUE_HEAD(log_wait); static DECLARE_WAIT_QUEUE_HEAD(legacy_wait); @@ -2390,7 +2393,7 @@ asmlinkage int vprintk_emit(int facility, int level, /* If called from the scheduler, we can not call up(). */ if (level == LOGLEVEL_SCHED) { level = LOGLEVEL_DEFAULT; - ft.legacy_offload |= ft.legacy_direct; + ft.legacy_offload |= ft.legacy_direct && !console_irqwork_blocked; ft.legacy_direct = false; } @@ -2426,7 +2429,7 @@ asmlinkage int vprintk_emit(int facility, int level, if (ft.legacy_offload) defer_console_output(); - else + else if (!console_irqwork_blocked) wake_up_klogd(); return printed_len; @@ -2730,10 +2733,20 @@ void console_suspend_all(void) { struct console *con; + if (console_suspend_enabled) + pr_info("Suspending console(s) (use no_console_suspend to debug)\n"); + + /* + * Flush any console backlog and then avoid queueing irq_work until + * console_resume_all(). Until then deferred printing is no longer + * triggered, NBCON consoles transition to atomic flushing, and + * any klogd waiters are not triggered. + */ + pr_flush(1000, true); + console_irqwork_blocked = true; + if (!console_suspend_enabled) return; - pr_info("Suspending console(s) (use no_console_suspend to debug)\n"); - pr_flush(1000, true); console_list_lock(); for_each_console(con) @@ -2754,26 +2767,34 @@ void console_resume_all(void) struct console_flush_type ft; struct console *con; - if (!console_suspend_enabled) - return; - - console_list_lock(); - for_each_console(con) - console_srcu_write_flags(con, con->flags & ~CON_SUSPENDED); - console_list_unlock(); - /* - * Ensure that all SRCU list walks have completed. All printing - * contexts must be able to see they are no longer suspended so - * that they are guaranteed to wake up and resume printing. + * Allow queueing irq_work. After restoring console state, deferred + * printing and any klogd waiters need to be triggered in case there + * is now a console backlog. */ - synchronize_srcu(&console_srcu); + console_irqwork_blocked = false; + + if (console_suspend_enabled) { + console_list_lock(); + for_each_console(con) + console_srcu_write_flags(con, con->flags & ~CON_SUSPENDED); + console_list_unlock(); + + /* + * Ensure that all SRCU list walks have completed. All printing + * contexts must be able to see they are no longer suspended so + * that they are guaranteed to wake up and resume printing. + */ + synchronize_srcu(&console_srcu); + } printk_get_console_flush_type(&ft); if (ft.nbcon_offload) nbcon_kthreads_wake(); if (ft.legacy_offload) defer_console_output(); + else + wake_up_klogd(); pr_flush(1000, true); } @@ -4511,6 +4532,13 @@ static void __wake_up_klogd(int val) if (!printk_percpu_data_ready()) return; + /* + * It is not allowed to call this function when console irq_work + * is blocked. + */ + if (WARN_ON_ONCE(console_irqwork_blocked)) + return; + preempt_disable(); /* * Guarantee any new records can be seen by tasks preparing to wait @@ -4567,9 +4595,30 @@ void defer_console_output(void) __wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT); } +/** + * printk_trigger_flush - Attempt to flush printk buffer to consoles. + * + * If possible, flush the printk buffer to all consoles in the caller's + * context. If offloading is available, trigger deferred printing. + * + * This is best effort. Depending on the system state, console states, + * and caller context, no actual flushing may result from this call. + */ void printk_trigger_flush(void) { - defer_console_output(); + struct console_flush_type ft; + + printk_get_console_flush_type(&ft); + if (ft.nbcon_atomic) + nbcon_atomic_flush_pending(); + if (ft.nbcon_offload) + nbcon_kthreads_wake(); + if (ft.legacy_direct) { + if (console_trylock()) + console_unlock(); + } + if (ft.legacy_offload) + defer_console_output(); } int vprintk_deferred(const char *fmt, va_list args) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index b8bbe7960cda7..2265b9c2906e1 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -203,7 +203,7 @@ struct rcu_data { /* during and after the last grace */ /* period it is aware of. */ struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ - int defer_qs_iw_pending; /* Scheduler attention pending? */ + int defer_qs_pending; /* irqwork or softirq pending? */ struct work_struct strict_work; /* Schedule readers for strict GPs. */ /* 2) batch handling */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index d85763336b3c0..cafb1cc8eff84 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -487,8 +487,8 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) union rcu_special special; rdp = this_cpu_ptr(&rcu_data); - if (rdp->defer_qs_iw_pending == DEFER_QS_PENDING) - rdp->defer_qs_iw_pending = DEFER_QS_IDLE; + if (rdp->defer_qs_pending == DEFER_QS_PENDING) + rdp->defer_qs_pending = DEFER_QS_IDLE; /* * If RCU core is waiting for this CPU to exit its critical section, @@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_iw_pending, DEFER_QS_IDLE); + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); } /* @@ -747,7 +747,10 @@ static void rcu_read_unlock_special(struct task_struct *t) // Using softirq, safe to awaken, and either the // wakeup is free or there is either an expedited // GP in flight or a potential need to deboost. - raise_softirq_irqoff(RCU_SOFTIRQ); + if (rdp->defer_qs_pending != DEFER_QS_PENDING) { + rdp->defer_qs_pending = DEFER_QS_PENDING; + raise_softirq_irqoff(RCU_SOFTIRQ); + } } else { // Enabling BH or preempt does reschedule, so... // Also if no expediting and no possible deboosting, @@ -756,11 +759,11 @@ static void rcu_read_unlock_special(struct task_struct *t) set_tsk_need_resched(current); set_preempt_need_resched(); if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled && - needs_exp && rdp->defer_qs_iw_pending != DEFER_QS_PENDING && + needs_exp && rdp->defer_qs_pending != DEFER_QS_PENDING && cpu_online(rdp->cpu)) { // Get scheduler to re-evaluate and call hooks. // If !IRQ_WORK, FQS scan will eventually IPI. - rdp->defer_qs_iw_pending = DEFER_QS_PENDING; + rdp->defer_qs_pending = DEFER_QS_PENDING; irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); } } diff --git a/kernel/resource.c b/kernel/resource.c index b9fa2a4ce089c..e4e9bac12e6e1 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -341,6 +341,8 @@ static int find_next_iomem_res(resource_size_t start, resource_size_t end, unsigned long flags, unsigned long desc, struct resource *res) { + /* Skip children until we find a top level range that matches */ + bool skip_children = true; struct resource *p; if (!res) @@ -351,7 +353,7 @@ static int find_next_iomem_res(resource_size_t start, resource_size_t end, read_lock(&resource_lock); - for_each_resource(&iomem_resource, p, false) { + for_each_resource(&iomem_resource, p, skip_children) { /* If we passed the resource we are looking for, stop */ if (p->start > end) { p = NULL; @@ -362,6 +364,12 @@ static int find_next_iomem_res(resource_size_t start, resource_size_t end, if (p->end < start) continue; + /* + * We found a top level range that matches what we are looking + * for. Time to start checking children too. + */ + skip_children = false; + /* Found a match, break */ if (is_type_match(p, flags, desc)) break; diff --git a/kernel/rseq.c b/kernel/rseq.c index 2452b7366b00e..07b0b46ec640f 100644 --- a/kernel/rseq.c +++ b/kernel/rseq.c @@ -519,8 +519,9 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, * auxiliary vector AT_RSEQ_ALIGN. If rseq_len is the original rseq * size, the required alignment is the original struct rseq alignment. * - * In order to be valid, rseq_len is either the original rseq size, or - * large enough to contain all supported fields, as communicated to + * The rseq_len is required to be greater or equal to the original rseq + * size. In order to be valid, rseq_len is either the original rseq size, + * or large enough to contain all supported fields, as communicated to * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE. */ if (rseq_len < ORIG_RSEQ_SIZE || diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f754a60de8484..582c3847f483a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -119,8 +119,12 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_entry_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_exit_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_set_need_resched_tp); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); +DEFINE_PER_CPU(struct rnd_state, sched_rnd_state); #ifdef CONFIG_SCHED_PROXY_EXEC DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec); @@ -1135,6 +1139,7 @@ void __trace_set_need_resched(struct task_struct *curr, int tif) { trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif); } +EXPORT_SYMBOL_GPL(__trace_set_need_resched); void resched_curr(struct rq *rq) { @@ -7382,7 +7387,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) trace_sched_pi_setprio(p, pi_task); oldprio = p->prio; - if (oldprio == prio) + if (oldprio == prio && !dl_prio(prio)) queue_flag &= ~DEQUEUE_MOVE; prev_class = p->sched_class; @@ -8591,6 +8596,8 @@ void __init sched_init_smp(void) { sched_init_numa(NUMA_NO_NODE); + prandom_init_once(&sched_rnd_state); + /* * There's no userspace yet to cause hotplug operations; hence all the * CPU masks are stable and all blatant races in the below code cannot diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c index cdd740b3f7743..37b572cc8aca2 100644 --- a/kernel/sched/cpudeadline.c +++ b/kernel/sched/cpudeadline.c @@ -166,12 +166,13 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p, * cpudl_clear - remove a CPU from the cpudl max-heap * @cp: the cpudl max-heap context * @cpu: the target CPU + * @online: the online state of the deadline runqueue * * Notes: assumes cpu_rq(cpu)->lock is locked * * Returns: (void) */ -void cpudl_clear(struct cpudl *cp, int cpu) +void cpudl_clear(struct cpudl *cp, int cpu, bool online) { int old_idx, new_cpu; unsigned long flags; @@ -184,7 +185,7 @@ void cpudl_clear(struct cpudl *cp, int cpu) if (old_idx == IDX_INVALID) { /* * Nothing to remove if old_idx was invalid. - * This could happen if a rq_offline_dl is + * This could happen if rq_online_dl or rq_offline_dl is * called for a CPU without -dl tasks running. */ } else { @@ -195,9 +196,12 @@ void cpudl_clear(struct cpudl *cp, int cpu) cp->elements[new_cpu].idx = old_idx; cp->elements[cpu].idx = IDX_INVALID; cpudl_heapify(cp, old_idx); - - cpumask_set_cpu(cpu, cp->free_cpus); } + if (likely(online)) + __cpumask_set_cpu(cpu, cp->free_cpus); + else + __cpumask_clear_cpu(cpu, cp->free_cpus); + raw_spin_unlock_irqrestore(&cp->lock, flags); } @@ -228,7 +232,7 @@ void cpudl_set(struct cpudl *cp, int cpu, u64 dl) cp->elements[new_idx].cpu = cpu; cp->elements[cpu].idx = new_idx; cpudl_heapify_up(cp, new_idx); - cpumask_clear_cpu(cpu, cp->free_cpus); + __cpumask_clear_cpu(cpu, cp->free_cpus); } else { cp->elements[old_idx].dl = dl; cpudl_heapify(cp, old_idx); @@ -237,26 +241,6 @@ void cpudl_set(struct cpudl *cp, int cpu, u64 dl) raw_spin_unlock_irqrestore(&cp->lock, flags); } -/* - * cpudl_set_freecpu - Set the cpudl.free_cpus - * @cp: the cpudl max-heap context - * @cpu: rd attached CPU - */ -void cpudl_set_freecpu(struct cpudl *cp, int cpu) -{ - cpumask_set_cpu(cpu, cp->free_cpus); -} - -/* - * cpudl_clear_freecpu - Clear the cpudl.free_cpus - * @cp: the cpudl max-heap context - * @cpu: rd attached CPU - */ -void cpudl_clear_freecpu(struct cpudl *cp, int cpu) -{ - cpumask_clear_cpu(cpu, cp->free_cpus); -} - /* * cpudl_init - initialize the cpudl structure * @cp: the cpudl max-heap context diff --git a/kernel/sched/cpudeadline.h b/kernel/sched/cpudeadline.h index 11c0f1faa7e11..d7699468eedd5 100644 --- a/kernel/sched/cpudeadline.h +++ b/kernel/sched/cpudeadline.h @@ -19,8 +19,6 @@ struct cpudl { int cpudl_find(struct cpudl *cp, struct task_struct *p, struct cpumask *later_mask); void cpudl_set(struct cpudl *cp, int cpu, u64 dl); -void cpudl_clear(struct cpudl *cp, int cpu); +void cpudl_clear(struct cpudl *cp, int cpu, bool online); int cpudl_init(struct cpudl *cp); -void cpudl_set_freecpu(struct cpudl *cp, int cpu); -void cpudl_clear_freecpu(struct cpudl *cp, int cpu); void cpudl_cleanup(struct cpudl *cp); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 7b7671060bf9e..d5052f238adf7 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -761,8 +761,6 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) struct dl_rq *dl_rq = dl_rq_of_se(dl_se); struct rq *rq = rq_of_dl_rq(dl_rq); - update_rq_clock(rq); - WARN_ON(is_dl_boosted(dl_se)); WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); @@ -1038,13 +1036,19 @@ static void update_dl_entity(struct sched_dl_entity *dl_se) if (dl_time_before(dl_se->deadline, rq_clock(rq)) || dl_entity_overflow(dl_se, rq_clock(rq))) { - if (unlikely(!dl_is_implicit(dl_se) && + if (unlikely((!dl_is_implicit(dl_se) || dl_se->dl_defer) && !dl_time_before(dl_se->deadline, rq_clock(rq)) && !is_dl_boosted(dl_se))) { update_dl_revised_wakeup(dl_se, rq); return; } + /* + * When [4] D->A is followed by [1] A->B, dl_defer_running + * needs to be cleared, otherwise it will fail to properly + * start the zero-laxity timer. + */ + dl_se->dl_defer_running = 0; replenish_dl_new_period(dl_se, rq); } else if (dl_server(dl_se) && dl_se->dl_defer) { /* @@ -1575,6 +1579,206 @@ void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec) update_curr_dl_se(dl_se->rq, dl_se, delta_exec); } +/* + * dl_server && dl_defer: + * + * 6 + * +--------------------+ + * v | + * +-------------+ 4 +-----------+ 5 +------------------+ + * +-> | A:init | <--- | D:running | -----> | E:replenish-wait | + * | +-------------+ +-----------+ +------------------+ + * | | | 1 ^ ^ | + * | | 1 +----------+ | 3 | + * | v | | + * | +--------------------------------+ 2 | + * | | | ----+ | + * | 8 | B:zero_laxity-wait | | | + * | | | <---+ | + * | +--------------------------------+ | + * | | ^ ^ 2 | + * | | 7 | 2 +--------------------+ + * | v | + * | +-------------+ | + * +-- | C:idle-wait | -+ + * +-------------+ + * ^ 7 | + * +---------+ + * + * + * [A] - init + * dl_server_active = 0 + * dl_throttled = 0 + * dl_defer_armed = 0 + * dl_defer_running = 0/1 + * dl_defer_idle = 0 + * + * [B] - zero_laxity-wait + * dl_server_active = 1 + * dl_throttled = 1 + * dl_defer_armed = 1 + * dl_defer_running = 0 + * dl_defer_idle = 0 + * + * [C] - idle-wait + * dl_server_active = 1 + * dl_throttled = 1 + * dl_defer_armed = 1 + * dl_defer_running = 0 + * dl_defer_idle = 1 + * + * [D] - running + * dl_server_active = 1 + * dl_throttled = 0 + * dl_defer_armed = 0 + * dl_defer_running = 1 + * dl_defer_idle = 0 + * + * [E] - replenish-wait + * dl_server_active = 1 + * dl_throttled = 1 + * dl_defer_armed = 0 + * dl_defer_running = 1 + * dl_defer_idle = 0 + * + * + * [1] A->B, A->D + * dl_server_start() + * dl_server_active = 1; + * enqueue_dl_entity() + * update_dl_entity(WAKEUP) + * if (dl_time_before() || dl_entity_overflow()) + * dl_defer_running = 0; + * replenish_dl_new_period(); + * // fwd period + * dl_throttled = 1; + * dl_defer_armed = 1; + * if (!dl_defer_running) + * dl_defer_armed = 1; + * dl_throttled = 1; + * if (dl_throttled && start_dl_timer()) + * return; // [B] + * __enqueue_dl_entity(); + * // [D] + * + * // deplete server runtime from client-class + * [2] B->B, C->B, E->B + * dl_server_update() + * update_curr_dl_se() // idle = false + * if (dl_defer_idle) + * dl_defer_idle = 0; + * if (dl_defer && dl_throttled && dl_runtime_exceeded()) + * dl_defer_running = 0; + * hrtimer_try_to_cancel(); // stop timer + * replenish_dl_new_period() + * // fwd period + * dl_throttled = 1; + * dl_defer_armed = 1; + * start_dl_timer(); // restart timer + * // [B] + * + * // timer actually fires means we have runtime + * [3] B->D + * dl_server_timer() + * if (dl_defer_armed) + * dl_defer_running = 1; + * enqueue_dl_entity(REPLENISH) + * replenish_dl_entity() + * // fwd period + * if (dl_throttled) + * dl_throttled = 0; + * if (dl_defer_armed) + * dl_defer_armed = 0; + * __enqueue_dl_entity(); + * // [D] + * + * // schedule server + * [4] D->A + * pick_task_dl() + * p = server_pick_task(); + * if (!p) + * dl_server_stop() + * dequeue_dl_entity(); + * hrtimer_try_to_cancel(); + * dl_defer_armed = 0; + * dl_throttled = 0; + * dl_server_active = 0; + * // [A] + * return p; + * + * // server running + * [5] D->E + * update_curr_dl_se() + * if (dl_runtime_exceeded()) + * dl_throttled = 1; + * dequeue_dl_entity(); + * start_dl_timer(); + * // [E] + * + * // server replenished + * [6] E->D + * dl_server_timer() + * enqueue_dl_entity(REPLENISH) + * replenish_dl_entity() + * fwd-period + * if (dl_throttled) + * dl_throttled = 0; + * __enqueue_dl_entity(); + * // [D] + * + * // deplete server runtime from idle + * [7] B->C, C->C + * dl_server_update_idle() + * update_curr_dl_se() // idle = true + * if (dl_defer && dl_throttled && dl_runtime_exceeded()) + * if (dl_defer_idle) + * return; + * dl_defer_running = 0; + * hrtimer_try_to_cancel(); + * replenish_dl_new_period() + * // fwd period + * dl_throttled = 1; + * dl_defer_armed = 1; + * dl_defer_idle = 1; + * start_dl_timer(); // restart timer + * // [C] + * + * // stop idle server + * [8] C->A + * dl_server_timer() + * if (dl_defer_idle) + * dl_server_stop(); + * // [A] + * + * + * digraph dl_server { + * "A:init" -> "B:zero_laxity-wait" [label="1:dl_server_start"] + * "A:init" -> "D:running" [label="1:dl_server_start"] + * "B:zero_laxity-wait" -> "B:zero_laxity-wait" [label="2:dl_server_update"] + * "B:zero_laxity-wait" -> "C:idle-wait" [label="7:dl_server_update_idle"] + * "B:zero_laxity-wait" -> "D:running" [label="3:dl_server_timer"] + * "C:idle-wait" -> "A:init" [label="8:dl_server_timer"] + * "C:idle-wait" -> "B:zero_laxity-wait" [label="2:dl_server_update"] + * "C:idle-wait" -> "C:idle-wait" [label="7:dl_server_update_idle"] + * "D:running" -> "A:init" [label="4:pick_task_dl"] + * "D:running" -> "E:replenish-wait" [label="5:update_curr_dl_se"] + * "E:replenish-wait" -> "B:zero_laxity-wait" [label="2:dl_server_update"] + * "E:replenish-wait" -> "D:running" [label="6:dl_server_timer"] + * } + * + * + * Notes: + * + * - When there are fair tasks running the most likely loop is [2]->[2]. + * the dl_server never actually runs, the timer never fires. + * + * - When there is actual fair starvation; the timer fires and starts the + * dl_server. This will then throttle and replenish like a normal DL + * task. Notably it will not 'defer' again. + * + * - When idle it will push the actication forward once, and then wait + * for the timer to hit or a non-idle update to restart things. + */ void dl_server_start(struct sched_dl_entity *dl_se) { struct rq *rq = dl_se->rq; @@ -1623,6 +1827,7 @@ void sched_init_dl_servers(void) rq = cpu_rq(cpu); guard(rq_lock_irq)(rq); + update_rq_clock(rq); dl_se = &rq->fair_server; @@ -1811,7 +2016,7 @@ static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) if (!dl_rq->dl_nr_running) { dl_rq->earliest_dl.curr = 0; dl_rq->earliest_dl.next = 0; - cpudl_clear(&rq->rd->cpudl, rq->cpu); + cpudl_clear(&rq->rd->cpudl, rq->cpu, rq->online); cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr); } else { struct rb_node *leftmost = rb_first_cached(&dl_rq->root); @@ -2143,7 +2348,7 @@ static void yield_task_dl(struct rq *rq) * it and the bandwidth timer will wake it up and will give it * new scheduling parameters (thanks to dl_yielded=1). */ - rq->curr->dl.dl_yielded = 1; + rq->donor->dl.dl_yielded = 1; update_rq_clock(rq); update_curr_dl(rq); @@ -2883,9 +3088,10 @@ static void rq_online_dl(struct rq *rq) if (rq->dl.overloaded) dl_set_overload(rq); - cpudl_set_freecpu(&rq->rd->cpudl, rq->cpu); if (rq->dl.dl_nr_running > 0) cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr); + else + cpudl_clear(&rq->rd->cpudl, rq->cpu, true); } /* Assumes rq->lock is held */ @@ -2894,8 +3100,7 @@ static void rq_offline_dl(struct rq *rq) if (rq->dl.overloaded) dl_clear_overload(rq); - cpudl_clear(&rq->rd->cpudl, rq->cpu); - cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu); + cpudl_clear(&rq->rd->cpudl, rq->cpu, false); } void __init init_sched_dl_class(void) @@ -3386,6 +3591,9 @@ static void __dl_clear_params(struct sched_dl_entity *dl_se) dl_se->dl_non_contending = 0; dl_se->dl_overrun = 0; dl_se->dl_server = 0; + dl_se->dl_defer = 0; + dl_se->dl_defer_running = 0; + dl_se->dl_defer_armed = 0; #ifdef CONFIG_RT_MUTEXES dl_se->pi_se = dl_se; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 02e16b70a7901..3504ec9bd7307 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -345,8 +345,8 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu long cpu = (long) ((struct seq_file *) filp->private_data)->private; struct rq *rq = cpu_rq(cpu); u64 runtime, period; + int retval = 0; size_t err; - int retval; u64 value; err = kstrtoull_from_user(ubuf, cnt, 10, &value); @@ -380,8 +380,6 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu dl_server_stop(&rq->fair_server); retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0); - if (retval) - cnt = retval; if (!runtime) printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n", @@ -389,6 +387,9 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu if (rq->cfs.h_nr_queued) dl_server_start(&rq->fair_server); + + if (retval < 0) + return retval; } *ppos += cnt; @@ -796,7 +797,8 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) { - s64 left_vruntime = -1, min_vruntime, right_vruntime = -1, left_deadline = -1, spread; + s64 left_vruntime = -1, zero_vruntime, right_vruntime = -1, left_deadline = -1, spread; + u64 avruntime; struct sched_entity *last, *first, *root; struct rq *rq = cpu_rq(cpu); unsigned long flags; @@ -819,17 +821,18 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) last = __pick_last_entity(cfs_rq); if (last) right_vruntime = last->vruntime; - min_vruntime = cfs_rq->min_vruntime; + zero_vruntime = cfs_rq->zero_vruntime; + avruntime = avg_vruntime(cfs_rq); raw_spin_rq_unlock_irqrestore(rq, flags); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "left_deadline", SPLIT_NS(left_deadline)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "left_vruntime", SPLIT_NS(left_vruntime)); - SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime", - SPLIT_NS(min_vruntime)); + SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "zero_vruntime", + SPLIT_NS(zero_vruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "avg_vruntime", - SPLIT_NS(avg_vruntime(cfs_rq))); + SPLIT_NS(avruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "right_vruntime", SPLIT_NS(right_vruntime)); spread = right_vruntime - left_vruntime; diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 979484dab2d3d..ee031ba877d9c 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -33,13 +33,19 @@ static DEFINE_MUTEX(scx_enable_mutex); DEFINE_STATIC_KEY_FALSE(__scx_enabled); DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); -static unsigned long scx_in_softlockup; -static atomic_t scx_breather_depth = ATOMIC_INIT(0); static int scx_bypass_depth; +static bool scx_aborting; static bool scx_init_task_enabled; static bool scx_switching_all; DEFINE_STATIC_KEY_FALSE(__scx_switched_all); +/* + * Tracks whether scx_enable() called scx_bypass(true). Used to balance bypass + * depth on enable failure. Will be removed when bypass depth is moved into the + * sched instance. + */ +static bool scx_bypassed_for_enable; + static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); @@ -203,6 +209,14 @@ static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) return rhashtable_lookup_fast(&sch->dsq_hash, &dsq_id, dsq_hash_params); } +static const struct sched_class *scx_setscheduler_class(struct task_struct *p) +{ + if (p->sched_class == &stop_sched_class) + return &stop_sched_class; + + return __setscheduler_class(p->policy, p->prio); +} + /* * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate @@ -906,6 +920,30 @@ static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); } +static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) +{ + struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); + bool preempt = false; + + /* + * If @rq is in balance, the CPU is already vacant and looking for the + * next task to run. No need to preempt or trigger resched after moving + * @p into its local DSQ. + */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) + return; + + if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && + rq->curr->sched_class == &ext_sched_class) { + rq->curr->scx.slice = 0; + preempt = true; + } + + if (preempt || sched_class_above(&ext_sched_class, rq->curr->sched_class)) + resched_curr(rq); +} + static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { @@ -981,21 +1019,12 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq, } /* seq records the order tasks are queued, used by BPF DSQ iterator */ - dsq->seq++; + WRITE_ONCE(dsq->seq, dsq->seq + 1); p->scx.dsq_seq = dsq->seq; dsq_mod_nr(dsq, 1); p->scx.dsq = dsq; - /* - * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the - * direct dispatch path, but we clear them here because the direct - * dispatch verdict may be overridden on the enqueue path during e.g. - * bypass. - */ - p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; - p->scx.ddsp_enq_flags = 0; - /* * We're transitioning out of QUEUEING or DISPATCHING. store_release to * match waiters' load_acquire. @@ -1003,22 +1032,10 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq, if (enq_flags & SCX_ENQ_CLEAR_OPSS) atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); - if (is_local) { - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - bool preempt = false; - - if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && - rq->curr->sched_class == &ext_sched_class) { - rq->curr->scx.slice = 0; - preempt = true; - } - - if (preempt || sched_class_above(&ext_sched_class, - rq->curr->sched_class)) - resched_curr(rq); - } else { + if (is_local) + local_dsq_post_enq(dsq, p, enq_flags); + else raw_spin_unlock(&dsq->lock); - } } static void task_unlink_from_dsq(struct task_struct *p, @@ -1150,12 +1167,34 @@ static void mark_direct_dispatch(struct scx_sched *sch, p->scx.ddsp_enq_flags = enq_flags; } +/* + * Clear @p direct dispatch state when leaving the scheduler. + * + * Direct dispatch state must be cleared in the following cases: + * - direct_dispatch(): cleared on the synchronous enqueue path, deferred + * dispatch keeps the state until consumed + * - process_ddsp_deferred_locals(): cleared after consuming deferred state, + * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch + * verdict is ignored (local/global/bypass) + * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred + * cancellation and holding_cpu races + * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by + * the scx_bypass() loop, so that stale state is not reused by a subsequent + * scheduler instance + */ +static inline void clear_direct_dispatch(struct task_struct *p) +{ + p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; + p->scx.ddsp_enq_flags = 0; +} + static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, u64 enq_flags) { struct rq *rq = task_rq(p); struct scx_dispatch_q *dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p); + u64 ddsp_enq_flags; touch_core_sched_dispatch(rq, p); @@ -1196,8 +1235,10 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, return; } - dispatch_enqueue(sch, dsq, p, - p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); + ddsp_enq_flags = p->scx.ddsp_enq_flags; + clear_direct_dispatch(p); + + dispatch_enqueue(sch, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); } static bool scx_rq_online(struct rq *rq) @@ -1217,6 +1258,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, { struct scx_sched *sch = scx_root; struct task_struct **ddsp_taskp; + struct scx_dispatch_q *dsq; unsigned long qseq; WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); @@ -1284,8 +1326,17 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, direct: direct_dispatch(sch, p, enq_flags); return; - +local_norefill: + dispatch_enqueue(sch, &rq->scx.local_dsq, p, enq_flags); + return; local: + dsq = &rq->scx.local_dsq; + goto enqueue; +global: + dsq = find_global_dsq(sch, p); + goto enqueue; + +enqueue: /* * For task-ordering, slice refill must be treated as implying the end * of the current slice. Otherwise, the longer @p stays on the CPU, the @@ -1293,14 +1344,8 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, */ touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); -local_norefill: - dispatch_enqueue(sch, &rq->scx.local_dsq, p, enq_flags); - return; - -global: - touch_core_sched(rq, p); /* see the comment in local: */ - refill_task_slice_dfl(sch, p); - dispatch_enqueue(sch, find_global_dsq(sch, p), p, enq_flags); + clear_direct_dispatch(p); + dispatch_enqueue(sch, dsq, p, enq_flags); } static bool task_runnable(const struct task_struct *p) @@ -1331,16 +1376,15 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } -static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) +static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) { struct scx_sched *sch = scx_root; int sticky_cpu = p->scx.sticky_cpu; + u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; if (enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; - enq_flags |= rq->scx.extra_enq_flags; - if (sticky_cpu >= 0) p->scx.sticky_cpu = -1; @@ -1468,13 +1512,14 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags sub_nr_running(rq, 1); dispatch_dequeue(rq, p); + clear_direct_dispatch(p); return true; } static void yield_task_scx(struct rq *rq) { struct scx_sched *sch = scx_root; - struct task_struct *p = rq->curr; + struct task_struct *p = rq->donor; if (SCX_HAS_OP(sch, yield)) SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq, p, NULL); @@ -1485,7 +1530,7 @@ static void yield_task_scx(struct rq *rq) static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) { struct scx_sched *sch = scx_root; - struct task_struct *from = rq->curr; + struct task_struct *from = rq->donor; if (SCX_HAS_OP(sch, yield)) return SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq, @@ -1513,6 +1558,8 @@ static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags, dsq_mod_nr(dst_dsq, 1); p->scx.dsq = dst_dsq; + + local_dsq_post_enq(dst_dsq, p, enq_flags); } /** @@ -1763,7 +1810,7 @@ static void scx_breather(struct rq *rq) lockdep_assert_rq_held(rq); - if (likely(!atomic_read(&scx_breather_depth))) + if (likely(!READ_ONCE(scx_aborting))) return; raw_spin_rq_unlock(rq); @@ -1772,9 +1819,9 @@ static void scx_breather(struct rq *rq) do { int cnt = 1024; - while (atomic_read(&scx_breather_depth) && --cnt) + while (READ_ONCE(scx_aborting) && --cnt) cpu_relax(); - } while (atomic_read(&scx_breather_depth) && + } while (READ_ONCE(scx_aborting) && time_before64(ktime_get_ns(), until)); raw_spin_rq_lock(rq); @@ -2206,13 +2253,15 @@ static void process_ddsp_deferred_locals(struct rq *rq) struct task_struct, scx.dsq_list.node))) { struct scx_sched *sch = scx_root; struct scx_dispatch_q *dsq; + u64 dsq_id = p->scx.ddsp_dsq_id; + u64 enq_flags = p->scx.ddsp_enq_flags; list_del_init(&p->scx.dsq_list.node); + clear_direct_dispatch(p); - dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p); + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, p); if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - dispatch_to_local_dsq(sch, rq, dsq, p, - p->scx.ddsp_enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); } } @@ -2277,12 +2326,6 @@ static void switch_class(struct rq *rq, struct task_struct *next) struct scx_sched *sch = scx_root; const struct sched_class *next_class = next->sched_class; - /* - * Pairs with the smp_load_acquire() issued by a CPU in - * kick_cpus_irq_workfn() who is waiting for this CPU to perform a - * resched. - */ - smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) return; @@ -2322,6 +2365,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_root; + + /* see kick_cpus_irq_workfn() */ + smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); + update_curr_scx(rq); /* see dequeue_task_scx() on why we skip when !QUEUED */ @@ -2375,6 +2422,9 @@ static struct task_struct *pick_task_scx(struct rq *rq) bool keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; bool kick_idle = false; + /* see kick_cpus_irq_workfn() */ + smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); + /* * WORKAROUND: * @@ -2850,6 +2900,8 @@ static void scx_disable_task(struct task_struct *p) lockdep_assert_rq_held(rq); WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); + clear_direct_dispatch(p); + if (SCX_HAS_OP(sch, disable)) SCX_CALL_OP_TASK(sch, SCX_KF_REST, disable, rq, p); scx_set_task_state(p, SCX_TASK_READY); @@ -3420,7 +3472,6 @@ static int scx_cgroup_init(struct scx_sched *sch) ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, cgroup_init, NULL, css->cgroup, &args); if (ret) { - css_put(css); scx_error(sch, "ops.cgroup_init() failed (%d)", ret); return ret; } @@ -3508,7 +3559,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) int node; irq_work_sync(&sch->error_irq_work); - kthread_stop(sch->helper->task); + kthread_destroy_worker(sch->helper); free_percpu(sch->pcpu); @@ -3690,30 +3741,14 @@ void scx_softlockup(u32 dur_s) goto out_unlock; } - /* allow only one instance, cleared at the end of scx_bypass() */ - if (test_and_set_bit(0, &scx_in_softlockup)) - goto out_unlock; - printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU%d stuck for %us, disabling \"%s\"\n", smp_processor_id(), dur_s, scx_root->ops.name); - /* - * Some CPUs may be trapped in the dispatch paths. Enable breather - * immediately; otherwise, we might even be able to get to scx_bypass(). - */ - atomic_inc(&scx_breather_depth); - scx_error(sch, "soft lockup - CPU#%d stuck for %us", smp_processor_id(), dur_s); out_unlock: rcu_read_unlock(); } -static void scx_clear_softlockup(void) -{ - if (test_and_clear_bit(0, &scx_in_softlockup)) - atomic_dec(&scx_breather_depth); -} - /** * scx_bypass - [Un]bypass scx_ops and guarantee forward progress * @bypass: true for bypass, false for unbypass @@ -3774,8 +3809,6 @@ static void scx_bypass(bool bypass) ktime_get_ns() - bypass_timestamp); } - atomic_inc(&scx_breather_depth); - /* * No task property is changing. We just need to make sure all currently * queued tasks are re-queued according to the new scx_rq_bypassing() @@ -3832,10 +3865,8 @@ static void scx_bypass(bool bypass) raw_spin_rq_unlock(rq); } - atomic_dec(&scx_breather_depth); unlock: raw_spin_unlock_irqrestore(&bypass_lock, flags); - scx_clear_softlockup(); } static void free_exit_info(struct scx_exit_info *ei) @@ -3930,6 +3961,7 @@ static void scx_disable_workfn(struct kthread_work *work) /* guarantee forward progress by bypassing scx_ops */ scx_bypass(true); + WRITE_ONCE(scx_aborting, false); switch (scx_set_enable_state(SCX_DISABLING)) { case SCX_DISABLING: @@ -3973,8 +4005,7 @@ static void scx_disable_workfn(struct kthread_work *work) scx_task_iter_start(&sti); while ((p = scx_task_iter_next_locked(&sti))) { const struct sched_class *old_class = p->sched_class; - const struct sched_class *new_class = - __setscheduler_class(p->policy, p->prio); + const struct sched_class *new_class = scx_setscheduler_class(p); struct sched_enq_and_set_ctx ctx; if (old_class != new_class && p->se.sched_delayed) @@ -4047,6 +4078,11 @@ static void scx_disable_workfn(struct kthread_work *work) scx_dsp_max_batch = 0; free_kick_pseqs(); + if (scx_bypassed_for_enable) { + scx_bypassed_for_enable = false; + scx_bypass(false); + } + mutex_unlock(&scx_enable_mutex); WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); @@ -4054,9 +4090,33 @@ static void scx_disable_workfn(struct kthread_work *work) scx_bypass(false); } -static void scx_disable(enum scx_exit_kind kind) +/* + * Claim the exit on @sch. The caller must ensure that the helper kthread work + * is kicked before the current task can be preempted. Once exit_kind is + * claimed, scx_error() can no longer trigger, so if the current task gets + * preempted and the BPF scheduler fails to schedule it back, the helper work + * will never be kicked and the whole system can wedge. + */ +static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) { int none = SCX_EXIT_NONE; + + lockdep_assert_preemption_disabled(); + + if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) + return false; + + /* + * Some CPUs may be trapped in the dispatch paths. Enable breather + * immediately; otherwise, we might not even be able to get to + * scx_bypass(). + */ + WRITE_ONCE(scx_aborting, true); + return true; +} + +static void scx_disable(enum scx_exit_kind kind) +{ struct scx_sched *sch; if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) @@ -4065,7 +4125,8 @@ static void scx_disable(enum scx_exit_kind kind) rcu_read_lock(); sch = rcu_dereference(scx_root); if (sch) { - atomic_try_cmpxchg(&sch->exit_kind, &none, kind); + guard(preempt)(); + scx_claim_exit(sch, kind); kthread_queue_work(sch->helper, &sch->disable_work); } rcu_read_unlock(); @@ -4386,9 +4447,10 @@ static void scx_vexit(struct scx_sched *sch, const char *fmt, va_list args) { struct scx_exit_info *ei = sch->exit_info; - int none = SCX_EXIT_NONE; - if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) + guard(preempt)(); + + if (!scx_claim_exit(sch, kind)) return; ei->exit_code = exit_code; @@ -4475,8 +4537,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops) } sch->pcpu = alloc_percpu(struct scx_sched_pcpu); - if (!sch->pcpu) + if (!sch->pcpu) { + ret = -ENOMEM; goto err_free_gdsqs; + } sch->helper = kthread_run_worker(0, "sched_ext_helper"); if (IS_ERR(sch->helper)) { @@ -4500,7 +4564,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops) return sch; err_stop_helper: - kthread_stop(sch->helper->task); + kthread_destroy_worker(sch->helper); err_free_pcpu: free_percpu(sch->pcpu); err_free_gdsqs: @@ -4564,20 +4628,30 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return 0; } -static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + struct sched_ext_ops *ops; + int ret; +}; + +static void scx_enable_workfn(struct kthread_work *work) { + struct scx_enable_cmd *cmd = + container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; struct scx_task_iter sti; struct task_struct *p; unsigned long timeout; int i, cpu, ret; - if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), - cpu_possible_mask)) { - pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); - return -EINVAL; - } - mutex_lock(&scx_enable_mutex); if (scx_enable_state() != SCX_DISABLED) { @@ -4601,6 +4675,8 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) */ WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); WARN_ON_ONCE(scx_root); + if (WARN_ON_ONCE(READ_ONCE(scx_aborting))) + WRITE_ONCE(scx_aborting, false); atomic_long_set(&scx_nr_rejected, 0); @@ -4672,6 +4748,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) * Init in bypass mode to guarantee forward progress. */ scx_bypass(true); + scx_bypassed_for_enable = true; for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) if (((void (**)(void))ops)[i]) @@ -4752,8 +4829,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) scx_task_iter_start(&sti); while ((p = scx_task_iter_next_locked(&sti))) { const struct sched_class *old_class = p->sched_class; - const struct sched_class *new_class = - __setscheduler_class(p->policy, p->prio); + const struct sched_class *new_class = scx_setscheduler_class(p); struct sched_enq_and_set_ctx ctx; if (!tryget_task_struct(p)) @@ -4776,6 +4852,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) scx_task_iter_stop(&sti); percpu_up_write(&scx_fork_rwsem); + scx_bypassed_for_enable = false; scx_bypass(false); if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { @@ -4793,13 +4870,15 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) atomic_long_inc(&scx_enable_seq); - return 0; + cmd->ret = 0; + return; err_free_pseqs: free_kick_pseqs(); err_unlock: mutex_unlock(&scx_enable_mutex); - return ret; + cmd->ret = ret; + return; err_disable_unlock_all: scx_cgroup_unlock(); @@ -4818,7 +4897,41 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) */ scx_error(sch, "scx_enable() failed (%d)", ret); kthread_flush_work(&sch->disable_work); - return 0; + cmd->ret = 0; +} + +static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) +{ + static struct kthread_worker *helper; + static DEFINE_MUTEX(helper_mutex); + struct scx_enable_cmd cmd; + + if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), + cpu_possible_mask)) { + pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); + return -EINVAL; + } + + if (!READ_ONCE(helper)) { + mutex_lock(&helper_mutex); + if (!helper) { + helper = kthread_run_worker(0, "scx_enable_helper"); + if (IS_ERR_OR_NULL(helper)) { + helper = NULL; + mutex_unlock(&helper_mutex); + return -ENOMEM; + } + sched_set_fifo(helper->task); + } + mutex_unlock(&helper_mutex); + } + + kthread_init_work(&cmd.work, scx_enable_workfn); + cmd.ops = ops; + + kthread_queue_work(READ_ONCE(helper), &cmd.work); + kthread_flush_work(&cmd.work); + return cmd.ret; } @@ -5128,25 +5241,34 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *pseqs) { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; + const struct sched_class *cur_class; bool should_wait = false; unsigned long flags; raw_spin_rq_lock_irqsave(rq, flags); + cur_class = rq->curr->sched_class; /* * During CPU hotplug, a CPU may depend on kicking itself to make - * forward progress. Allow kicking self regardless of online state. + * forward progress. Allow kicking self regardless of online state. If + * @cpu is running a higher class task, we have no control over @cpu. + * Skip kicking. */ - if (cpu_online(cpu) || cpu == cpu_of(this_rq)) { + if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && + !sched_class_above(cur_class, &ext_sched_class)) { if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { - if (rq->curr->sched_class == &ext_sched_class) + if (cur_class == &ext_sched_class) rq->curr->scx.slice = 0; cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); } if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { - pseqs[cpu] = rq->scx.pnt_seq; - should_wait = true; + if (cur_class == &ext_sched_class) { + pseqs[cpu] = rq->scx.pnt_seq; + should_wait = true; + } else { + cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + } } resched_curr(rq); @@ -5207,18 +5329,19 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) for_each_cpu(cpu, this_scx->cpus_to_wait) { unsigned long *wait_pnt_seq = &cpu_rq(cpu)->scx.pnt_seq; - if (cpu != cpu_of(this_rq)) { - /* - * Pairs with smp_store_release() issued by this CPU in - * switch_class() on the resched path. - * - * We busy-wait here to guarantee that no other task can - * be scheduled on our core before the target CPU has - * entered the resched path. - */ - while (smp_load_acquire(wait_pnt_seq) == pseqs[cpu]) - cpu_relax(); - } + /* + * Busy-wait until the task running at the time of kicking is no + * longer running. This can be used to implement e.g. core + * scheduling. + * + * smp_cond_load_acquire() pairs with store_releases in + * pick_task_scx() and put_prev_task_scx(). The former breaks + * the wait if SCX's scheduling path is entered even if the same + * task is picked subsequently. The latter is necessary to break + * the wait when $cpu is taken by a higher sched class. + */ + if (cpu != cpu_of(this_rq)) + smp_cond_load_acquire(wait_pnt_seq, VAL != pseqs[cpu]); cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); } diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index d2434c954848e..f880e3dcadfa6 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -543,7 +543,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, * piled up on it even if there is an idle core elsewhere on * the system. */ - waker_node = cpu_to_node(cpu); + waker_node = scx_cpu_node_if_enabled(cpu); if (!(current->flags & PF_EXITING) && cpu_rq(cpu)->scx.local_dsq.nr == 0 && (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && @@ -861,25 +861,32 @@ static bool check_builtin_idle_enabled(struct scx_sched *sch) * code. * * We can't simply check whether @p->migration_disabled is set in a - * sched_ext callback, because migration is always disabled for the current - * task while running BPF code. + * sched_ext callback, because the BPF prolog (__bpf_prog_enter) may disable + * migration for the current task while running BPF code. * - * The prolog (__bpf_prog_enter) and epilog (__bpf_prog_exit) respectively - * disable and re-enable migration. For this reason, the current task - * inside a sched_ext callback is always a migration-disabled task. + * Since the BPF prolog calls migrate_disable() only when CONFIG_PREEMPT_RCU + * is enabled (via rcu_read_lock_dont_migrate()), migration_disabled == 1 for + * the current task is ambiguous only in that case: it could be from the BPF + * prolog rather than a real migrate_disable() call. * - * Therefore, when @p->migration_disabled == 1, check whether @p is the - * current task or not: if it is, then migration was not disabled before - * entering the callback, otherwise migration was disabled. + * Without CONFIG_PREEMPT_RCU, the BPF prolog never calls migrate_disable(), + * so migration_disabled == 1 always means the task is truly + * migration-disabled. + * + * Therefore, when migration_disabled == 1 and CONFIG_PREEMPT_RCU is enabled, + * check whether @p is the current task or not: if it is, then migration was + * not disabled before entering the callback, otherwise migration was disabled. * * Returns true if @p is migration-disabled, false otherwise. */ static bool is_bpf_migration_disabled(const struct task_struct *p) { - if (p->migration_disabled == 1) - return p != current; - else - return p->migration_disabled; + if (p->migration_disabled == 1) { + if (IS_ENABLED(CONFIG_PREEMPT_RCU)) + return p != current; + return true; + } + return p->migration_disabled; } static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index b3617abed5108..8039a750490f8 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -69,7 +69,7 @@ enum scx_exit_flags { * info communication. The following flag indicates whether ops.init() * finished successfully. */ - SCX_EFLAG_INITIALIZED, + SCX_EFLAG_INITIALIZED = 1LLU << 0, }; /* @@ -986,8 +986,10 @@ enum scx_kick_flags { SCX_KICK_PREEMPT = 1LLU << 1, /* - * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will - * return after the target CPU finishes picking the next task. + * The scx_bpf_kick_cpu() call will return after the current SCX task of + * the target CPU switches out. This can be used to implement e.g. core + * scheduling. This has no effect if the current task on the target CPU + * is not on SCX. */ SCX_KICK_WAIT = 1LLU << 2, }; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5b752324270b0..d9777c81db0da 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -524,10 +524,48 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); * Scheduling class tree data structure manipulation methods: */ +extern void __BUILD_BUG_vruntime_cmp(void); + +/* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ + +#define vruntime_cmp(A, CMP_STR, B) ({ \ + int __res = 0; \ + \ + if (!__builtin_strcmp(CMP_STR, "<")) { \ + __res = ((s64)((A)-(B)) < 0); \ + } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ + __res = ((s64)((A)-(B)) <= 0); \ + } else if (!__builtin_strcmp(CMP_STR, ">")) { \ + __res = ((s64)((A)-(B)) > 0); \ + } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ + __res = ((s64)((A)-(B)) >= 0); \ + } else { \ + /* Unknown operator throws linker error: */ \ + __BUILD_BUG_vruntime_cmp(); \ + } \ + \ + __res; \ +}) + +extern void __BUILD_BUG_vruntime_op(void); + +#define vruntime_op(A, OP_STR, B) ({ \ + s64 __res = 0; \ + \ + if (!__builtin_strcmp(OP_STR, "-")) { \ + __res = (s64)((A)-(B)); \ + } else { \ + /* Unknown operator throws linker error: */ \ + __BUILD_BUG_vruntime_op(); \ + } \ + \ + __res; \ +}) + + static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - max_vruntime); - if (delta > 0) + if (vruntime_cmp(vruntime, ">", max_vruntime)) max_vruntime = vruntime; return max_vruntime; @@ -535,8 +573,7 @@ static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - min_vruntime); - if (delta < 0) + if (vruntime_cmp(vruntime, "<", min_vruntime)) min_vruntime = vruntime; return min_vruntime; @@ -549,12 +586,27 @@ static inline bool entity_before(const struct sched_entity *a, * Tiebreak on vruntime seems unnecessary since it can * hardly happen. */ - return (s64)(a->deadline - b->deadline) < 0; + return vruntime_cmp(a->deadline, "<", b->deadline); } +/* + * Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale + * and this value should be no more than two lag bounds. Which puts it in the + * general order of: + * + * (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT + * + * which is around 44 bits in size (on 64bit); that is 20 for + * NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for + * however many msec the actual slice+tick ends up begin. + * + * (disregarding the actual divide-by-weight part makes for the worst case + * weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually + * being the zero-lag point). + */ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) { - return (s64)(se->vruntime - cfs_rq->min_vruntime); + return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); } #define __node_2_se(node) \ @@ -606,74 +658,98 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) * * Which we track using: * - * v0 := cfs_rq->min_vruntime - * \Sum (v_i - v0) * w_i := cfs_rq->avg_vruntime - * \Sum w_i := cfs_rq->avg_load + * v0 := cfs_rq->zero_vruntime + * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime + * \Sum w_i := cfs_rq->sum_weight * - * Since min_vruntime is a monotonic increasing variable that closely tracks - * the per-task service, these deltas: (v_i - v), will be in the order of the - * maximal (virtual) lag induced in the system due to quantisation. + * Since zero_vruntime closely tracks the per-task service, these + * deltas: (v_i - v), will be in the order of the maximal (virtual) lag + * induced in the system due to quantisation. * * Also, we use scale_load_down() to reduce the size. * * As measured, the max (key * weight) value was ~44 bits for a kernel build. */ static void -avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) +sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { unsigned long weight = scale_load_down(se->load.weight); s64 key = entity_key(cfs_rq, se); - cfs_rq->avg_vruntime += key * weight; - cfs_rq->avg_load += weight; + cfs_rq->sum_w_vruntime += key * weight; + cfs_rq->sum_weight += weight; } static void -avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) +sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { unsigned long weight = scale_load_down(se->load.weight); s64 key = entity_key(cfs_rq, se); - cfs_rq->avg_vruntime -= key * weight; - cfs_rq->avg_load -= weight; + cfs_rq->sum_w_vruntime -= key * weight; + cfs_rq->sum_weight -= weight; } static inline -void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) +void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta) { /* - * v' = v + d ==> avg_vruntime' = avg_runtime - d*avg_load + * v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight */ - cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta; + cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; + cfs_rq->zero_vruntime += delta; } /* - * Specifically: avg_runtime() + 0 must result in entity_eligible() := true + * Specifically: avg_vruntime() + 0 must result in entity_eligible() := true * For this to be so, the result of this function must have a left bias. + * + * Called in: + * - place_entity() -- before enqueue + * - update_entity_lag() -- before dequeue + * - update_deadline() -- slice expiration + * + * This means it is one entry 'behind' but that puts it close enough to where + * the bound on entity_key() is at most two lag bounds. */ u64 avg_vruntime(struct cfs_rq *cfs_rq) { struct sched_entity *curr = cfs_rq->curr; - s64 avg = cfs_rq->avg_vruntime; - long load = cfs_rq->avg_load; + long weight = cfs_rq->sum_weight; + s64 delta = 0; - if (curr && curr->on_rq) { - unsigned long weight = scale_load_down(curr->load.weight); + if (curr && !curr->on_rq) + curr = NULL; - avg += entity_key(cfs_rq, curr) * weight; - load += weight; - } + if (weight) { + s64 runtime = cfs_rq->sum_w_vruntime; + + if (curr) { + unsigned long w = scale_load_down(curr->load.weight); + + runtime += entity_key(cfs_rq, curr) * w; + weight += w; + } - if (load) { /* sign flips effective floor / ceiling */ - if (avg < 0) - avg -= (load - 1); - avg = div_s64(avg, load); + if (runtime < 0) + runtime -= (weight - 1); + + delta = div_s64(runtime, weight); + } else if (curr) { + /* + * When there is but one element, it is the average. + */ + delta = curr->vruntime - cfs_rq->zero_vruntime; } - return cfs_rq->min_vruntime + avg; + update_zero_vruntime(cfs_rq, delta); + + return cfs_rq->zero_vruntime; } +static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); + /* * lag_i = S - s_i = w_i * (V - v_i) * @@ -687,17 +763,16 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) * EEVDF gives the following limit for a steady state system: * * -r_max < lag < max(r_max, q) - * - * XXX could add max_slice to the augmented data to track this. */ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) { + u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; s64 vlag, limit; WARN_ON_ONCE(!se->on_rq); vlag = avg_vruntime(cfs_rq) - se->vruntime; - limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); + limit = calc_delta_fair(max_slice, se); se->vlag = clamp(vlag, -limit, limit); } @@ -722,8 +797,8 @@ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) { struct sched_entity *curr = cfs_rq->curr; - s64 avg = cfs_rq->avg_vruntime; - long load = cfs_rq->avg_load; + s64 avg = cfs_rq->sum_w_vruntime; + long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { unsigned long weight = scale_load_down(curr->load.weight); @@ -732,7 +807,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) load += weight; } - return avg >= (s64)(vruntime - cfs_rq->min_vruntime) * load; + return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load; } int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) @@ -740,57 +815,34 @@ int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) return vruntime_eligible(cfs_rq, se->vruntime); } -static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime) -{ - u64 min_vruntime = cfs_rq->min_vruntime; - /* - * open coded max_vruntime() to allow updating avg_vruntime - */ - s64 delta = (s64)(vruntime - min_vruntime); - if (delta > 0) { - avg_vruntime_update(cfs_rq, delta); - min_vruntime = vruntime; - } - return min_vruntime; -} - -static void update_min_vruntime(struct cfs_rq *cfs_rq) +static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) { - struct sched_entity *se = __pick_root_entity(cfs_rq); + struct sched_entity *root = __pick_root_entity(cfs_rq); struct sched_entity *curr = cfs_rq->curr; - u64 vruntime = cfs_rq->min_vruntime; + u64 min_slice = ~0ULL; - if (curr) { - if (curr->on_rq) - vruntime = curr->vruntime; - else - curr = NULL; - } + if (curr && curr->on_rq) + min_slice = curr->slice; - if (se) { - if (!curr) - vruntime = se->min_vruntime; - else - vruntime = min_vruntime(vruntime, se->min_vruntime); - } + if (root) + min_slice = min(min_slice, root->min_slice); - /* ensure we never gain time by being placed backwards. */ - cfs_rq->min_vruntime = __update_min_vruntime(cfs_rq, vruntime); + return min_slice; } -static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) +static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq) { struct sched_entity *root = __pick_root_entity(cfs_rq); struct sched_entity *curr = cfs_rq->curr; - u64 min_slice = ~0ULL; + u64 max_slice = 0ULL; if (curr && curr->on_rq) - min_slice = curr->slice; + max_slice = curr->slice; if (root) - min_slice = min(min_slice, root->min_slice); + max_slice = max(max_slice, root->max_slice); - return min_slice; + return max_slice; } static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) @@ -798,13 +850,12 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) return entity_before(__node_2_se(a), __node_2_se(b)); } -#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; }) - static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) { if (node) { struct sched_entity *rse = __node_2_se(node); - if (vruntime_gt(min_vruntime, se, rse)) + + if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) se->min_vruntime = rse->min_vruntime; } } @@ -818,6 +869,15 @@ static inline void __min_slice_update(struct sched_entity *se, struct rb_node *n } } +static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) +{ + if (node) { + struct sched_entity *rse = __node_2_se(node); + if (rse->max_slice > se->max_slice) + se->max_slice = rse->max_slice; + } +} + /* * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) */ @@ -825,6 +885,7 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) { u64 old_min_vruntime = se->min_vruntime; u64 old_min_slice = se->min_slice; + u64 old_max_slice = se->max_slice; struct rb_node *node = &se->run_node; se->min_vruntime = se->vruntime; @@ -835,8 +896,13 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) __min_slice_update(se, node->rb_right); __min_slice_update(se, node->rb_left); + se->max_slice = se->slice; + __max_slice_update(se, node->rb_right); + __max_slice_update(se, node->rb_left); + return se->min_vruntime == old_min_vruntime && - se->min_slice == old_min_slice; + se->min_slice == old_min_slice && + se->max_slice == old_max_slice; } RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, @@ -847,7 +913,7 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - avg_vruntime_add(cfs_rq, se); + sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, @@ -858,7 +924,7 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); - avg_vruntime_sub(cfs_rq, se); + sum_w_vruntime_sub(cfs_rq, se); } struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) @@ -913,7 +979,7 @@ static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_enti static inline bool protect_slice(struct sched_entity *se) { - return ((s64)(se->vprot - se->vruntime) > 0); + return vruntime_cmp(se->vruntime, "<", se->vprot); } static inline void cancel_protect_slice(struct sched_entity *se) @@ -1040,7 +1106,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); */ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) { - if ((s64)(se->vruntime - se->deadline) < 0) + if (vruntime_cmp(se->vruntime, "<", se->deadline)) return false; /* @@ -1055,6 +1121,7 @@ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) * EEVDF: vd_i = ve_i + r_i / w_i */ se->deadline = se->vruntime + calc_delta_fair(se->slice, se); + avg_vruntime(cfs_rq); /* * The task has consumed its request, reschedule. @@ -1226,7 +1293,6 @@ static void update_curr(struct cfs_rq *cfs_rq) curr->vruntime += calc_delta_fair(delta_exec, curr); resched = update_deadline(cfs_rq, curr); - update_min_vruntime(cfs_rq); if (entity_is_task(curr)) { /* @@ -3771,6 +3837,8 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight) { bool curr = cfs_rq->curr == se; + bool rel_vprot = false; + u64 vprot; if (se->on_rq) { /* commit outstanding execution time */ @@ -3778,6 +3846,11 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, update_entity_lag(cfs_rq, se); se->deadline -= se->vruntime; se->rel_deadline = 1; + if (curr && protect_slice(se)) { + vprot = se->vprot - se->vruntime; + rel_vprot = true; + } + cfs_rq->nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); @@ -3793,6 +3866,9 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, if (se->rel_deadline) se->deadline = div_s64(se->deadline * se->load.weight, weight); + if (rel_vprot) + vprot = div_s64(vprot * se->load.weight, weight); + update_load_set(&se->load, weight); do { @@ -3804,19 +3880,12 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, enqueue_load_avg(cfs_rq, se); if (se->on_rq) { place_entity(cfs_rq, se, 0); + if (rel_vprot) + se->vprot = se->vruntime + vprot; update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); cfs_rq->nr_queued++; - - /* - * The entity's vruntime has been adjusted, so let's check - * whether the rq-wide min_vruntime needs updated too. Since - * the calculations above require stable min_vruntime rather - * than up-to-date one, we do the update at the end of the - * reweight process. - */ - update_min_vruntime(cfs_rq); } } @@ -4059,6 +4128,9 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) if (child_cfs_rq_on_list(cfs_rq)) return false; + if (cfs_rq->tg_load_avg_contrib) + return false; + return true; } @@ -5197,7 +5269,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * vl_i = (W + w_i)*vl'_i / W */ - load = cfs_rq->avg_load; + load = cfs_rq->sum_weight; if (curr && curr->on_rq) load += scale_load_down(curr->load.weight); @@ -5429,15 +5501,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_cfs_group(se); - /* - * Now advance min_vruntime if @se was the entity holding it back, - * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be - * put back on, and if we advance min_vruntime, we'll be placed back - * further than we started -- i.e. we'll be penalized. - */ - if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) - update_min_vruntime(cfs_rq); - if (flags & DEQUEUE_DELAYED) finish_delayed_dequeue_entity(se); @@ -5458,7 +5521,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) { clear_buddies(cfs_rq, se); @@ -5473,7 +5536,8 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); - set_protect_slice(cfs_rq, se); + if (first) + set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); @@ -8908,13 +8972,13 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf pse = parent_entity(pse); } if (se_depth >= pse_depth) { - set_next_entity(cfs_rq_of(se), se); + set_next_entity(cfs_rq_of(se), se, true); se = parent_entity(se); } } put_prev_entity(cfs_rq, pse); - set_next_entity(cfs_rq, se); + set_next_entity(cfs_rq, se, true); __set_next_task_fair(rq, p, true); } @@ -8943,12 +9007,6 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf goto again; } - /* - * rq is about to be idle, check if we need to update the - * lost_idle_time of clock_pelt - */ - update_idle_rq_clock_pelt(rq); - return NULL; } @@ -8990,7 +9048,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t */ static void yield_task_fair(struct rq *rq) { - struct task_struct *curr = rq->curr; + struct task_struct *curr = rq->donor; struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se; @@ -9014,7 +9072,18 @@ static void yield_task_fair(struct rq *rq) */ rq_clock_skip_update(rq); - se->deadline += calc_delta_fair(se->slice, se); + /* + * Forfeit the remaining vruntime, only if the entity is eligible. This + * condition is necessary because in core scheduling we prefer to run + * ineligible tasks rather than force idling. If this happens we may + * end up in a loop where the core scheduler picks the yielding task, + * which yields immediately again; without the condition the vruntime + * ends up quickly running away. + */ + if (entity_eligible(cfs_rq, se)) { + se->vruntime = se->deadline; + update_deadline(cfs_rq, se); + } } static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) @@ -11729,6 +11798,21 @@ static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd } } +/* + * This flag serializes load-balancing passes over large domains + * (above the NODE topology level) - only one load-balancing instance + * may run at a time, to reduce overhead on very large systems with + * lots of CPUs and large NUMA distances. + * + * - Note that load-balancing passes triggered while another one + * is executing are skipped and not re-tried. + * + * - Also note that this does not serialize rebalance_domains() + * execution, as non-SD_SERIALIZE domains will still be + * load-balanced in parallel. + */ +static atomic_t sched_balance_running = ATOMIC_INIT(0); + /* * Check this_cpu to ensure it is balanced within domain. Attempt to move * tasks if there is an imbalance. @@ -11754,6 +11838,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, .fbq_type = all, .tasks = LIST_HEAD_INIT(env.tasks), }; + bool need_unlock = false; cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); @@ -11765,6 +11850,14 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, goto out_balanced; } + if (!need_unlock && (sd->flags & SD_SERIALIZE)) { + int zero = 0; + if (!atomic_try_cmpxchg_acquire(&sched_balance_running, &zero, 1)) + goto out_balanced; + + need_unlock = true; + } + group = sched_balance_find_src_group(&env); if (!group) { schedstat_inc(sd->lb_nobusyg[idle]); @@ -12005,6 +12098,9 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, sd->balance_interval < sd->max_interval) sd->balance_interval *= 2; out: + if (need_unlock) + atomic_set_release(&sched_balance_running, 0); + return ld_moved; } @@ -12129,21 +12225,6 @@ static int active_load_balance_cpu_stop(void *data) return 0; } -/* - * This flag serializes load-balancing passes over large domains - * (above the NODE topology level) - only one load-balancing instance - * may run at a time, to reduce overhead on very large systems with - * lots of CPUs and large NUMA distances. - * - * - Note that load-balancing passes triggered while another one - * is executing are skipped and not re-tried. - * - * - Also note that this does not serialize rebalance_domains() - * execution, as non-SD_SERIALIZE domains will still be - * load-balanced in parallel. - */ -static atomic_t sched_balance_running = ATOMIC_INIT(0); - /* * Scale the max sched_balance_rq interval with the number of CPUs in the system. * This trades load-balance latency on larger machines for less cross talk. @@ -12153,30 +12234,43 @@ void update_max_interval(void) max_load_balance_interval = HZ*num_online_cpus()/10; } -static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost) +static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success) { + sd->newidle_call++; + sd->newidle_success += success; + + if (sd->newidle_call >= 1024) { + sd->newidle_ratio = sd->newidle_success; + sd->newidle_call /= 2; + sd->newidle_success /= 2; + } +} + +static inline bool +update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success) +{ + unsigned long next_decay = sd->last_decay_max_lb_cost + HZ; + unsigned long now = jiffies; + + if (cost) + update_newidle_stats(sd, success); + if (cost > sd->max_newidle_lb_cost) { /* * Track max cost of a domain to make sure to not delay the * next wakeup on the CPU. - * - * sched_balance_newidle() bumps the cost whenever newidle - * balance fails, and we don't want things to grow out of - * control. Use the sysctl_sched_migration_cost as the upper - * limit, plus a litle extra to avoid off by ones. */ - sd->max_newidle_lb_cost = - min(cost, sysctl_sched_migration_cost + 200); - sd->last_decay_max_lb_cost = jiffies; - } else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) { + sd->max_newidle_lb_cost = cost; + sd->last_decay_max_lb_cost = now; + + } else if (time_after(now, next_decay)) { /* * Decay the newidle max times by ~1% per second to ensure that * it is not outdated and the current max cost is actually * shorter. */ sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; - sd->last_decay_max_lb_cost = jiffies; - + sd->last_decay_max_lb_cost = now; return true; } @@ -12199,7 +12293,7 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) /* Earliest time when we have to do rebalance again */ unsigned long next_balance = jiffies + 60*HZ; int update_next_balance = 0; - int need_serialize, need_decay = 0; + int need_decay = 0; u64 max_cost = 0; rcu_read_lock(); @@ -12208,7 +12302,7 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) * Decay the newidle max times here because this is a regular * visit to all the domains. */ - need_decay = update_newidle_cost(sd, 0); + need_decay = update_newidle_cost(sd, 0, 0); max_cost += sd->max_newidle_lb_cost; /* @@ -12223,13 +12317,6 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) } interval = get_sd_balance_interval(sd, busy); - - need_serialize = sd->flags & SD_SERIALIZE; - if (need_serialize) { - if (atomic_cmpxchg_acquire(&sched_balance_running, 0, 1)) - goto out; - } - if (time_after_eq(jiffies, sd->last_balance + interval)) { if (sched_balance_rq(cpu, rq, sd, idle, &continue_balancing)) { /* @@ -12243,9 +12330,6 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) sd->last_balance = jiffies; interval = get_sd_balance_interval(sd, busy); } - if (need_serialize) - atomic_set_release(&sched_balance_running, 0); -out: if (time_after(next_balance, sd->last_balance + interval)) { next_balance = sd->last_balance + interval; update_next_balance = 1; @@ -12824,14 +12908,16 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) rcu_read_lock(); sd = rcu_dereference_check_sched_domain(this_rq->sd); + if (!sd) { + rcu_read_unlock(); + goto out; + } if (!get_rd_overloaded(this_rq->rd) || - (sd && this_rq->avg_idle < sd->max_newidle_lb_cost)) { + this_rq->avg_idle < sd->max_newidle_lb_cost) { - if (sd) - update_next_balance(sd, &next_balance); + update_next_balance(sd, &next_balance); rcu_read_unlock(); - goto out; } rcu_read_unlock(); @@ -12851,6 +12937,22 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) break; if (sd->flags & SD_BALANCE_NEWIDLE) { + unsigned int weight = 1; + + if (sched_feat(NI_RANDOM)) { + /* + * Throw a 1k sided dice; and only run + * newidle_balance according to the success + * rate. + */ + u32 d1k = sched_rng() % 1024; + weight = 1 + sd->newidle_ratio; + if (d1k > weight) { + update_newidle_stats(sd, 0); + continue; + } + weight = (1024 + weight/2) / weight; + } pulled_task = sched_balance_rq(this_cpu, this_rq, sd, CPU_NEWLY_IDLE, @@ -12862,13 +12964,10 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) t0 = t1; /* - * Failing newidle means it is not effective; - * bump the cost so we end up doing less of it. + * Track max cost of a domain to make sure to not delay the + * next wakeup on the CPU. */ - if (!pulled_task) - domain_cost = (3 * sd->max_newidle_lb_cost) / 2; - - update_newidle_cost(sd, domain_cost); + update_newidle_cost(sd, domain_cost, weight * !!pulled_task); } /* @@ -13012,7 +13111,170 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) } /* - * se_fi_update - Update the cfs_rq->min_vruntime_fi in a CFS hierarchy if needed. + * Consider any infeasible weight scenario. Take for instance two tasks, + * each bound to their respective sibling, one with weight 1 and one with + * weight 2. Then the lower weight task will run ahead of the higher weight + * task without bound. + * + * This utterly destroys the concept of a shared time base. + * + * Remember; all this is about a proportionally fair scheduling, where each + * tasks receives: + * + * w_i + * dt_i = ---------- dt (1) + * \Sum_j w_j + * + * which we do by tracking a virtual time, s_i: + * + * 1 + * s_i = --- d[t]_i (2) + * w_i + * + * Where d[t] is a delta of discrete time, while dt is an infinitesimal. + * The immediate corollary is that the ideal schedule S, where (2) to use + * an infinitesimal delta, is: + * + * 1 + * S = ---------- dt (3) + * \Sum_i w_i + * + * From which we can define the lag, or deviation from the ideal, as: + * + * lag(i) = S - s_i (4) + * + * And since the one and only purpose is to approximate S, we get that: + * + * \Sum_i w_i lag(i) := 0 (5) + * + * If this were not so, we no longer converge to S, and we can no longer + * claim our scheduler has any of the properties we derive from S. This is + * exactly what you did above, you broke it! + * + * + * Let's continue for a while though; to see if there is anything useful to + * be learned. We can combine (1)-(3) or (4)-(5) and express S in s_i: + * + * \Sum_i w_i s_i + * S = -------------- (6) + * \Sum_i w_i + * + * Which gives us a way to compute S, given our s_i. Now, if you've read + * our code, you know that we do not in fact do this, the reason for this + * is two-fold. Firstly, computing S in that way requires a 64bit division + * for every time we'd use it (see 12), and secondly, this only describes + * the steady-state, it doesn't handle dynamics. + * + * Anyway, in (6): s_i -> x + (s_i - x), to get: + * + * \Sum_i w_i (s_i - x) + * S - x = -------------------- (7) + * \Sum_i w_i + * + * Which shows that S and s_i transform alike (which makes perfect sense + * given that S is basically the (weighted) average of s_i). + * + * So the thing to remember is that the above is strictly UP. It is + * possible to generalize to multiple runqueues -- however it gets really + * yuck when you have to add affinity support, as illustrated by our very + * first counter-example. + * + * Luckily I think we can avoid needing a full multi-queue variant for + * core-scheduling (or load-balancing). The crucial observation is that we + * only actually need this comparison in the presence of forced-idle; only + * then do we need to tell if the stalled rq has higher priority over the + * other. + * + * [XXX assumes SMT2; better consider the more general case, I suspect + * it'll work out because our comparison is always between 2 rqs and the + * answer is only interesting if one of them is forced-idle] + * + * And (under assumption of SMT2) when there is forced-idle, there is only + * a single queue, so everything works like normal. + * + * Let, for our runqueue 'k': + * + * T_k = \Sum_i w_i s_i + * W_k = \Sum_i w_i ; for all i of k (8) + * + * Then we can write (6) like: + * + * T_k + * S_k = --- (9) + * W_k + * + * From which immediately follows that: + * + * T_k + T_l + * S_k+l = --------- (10) + * W_k + W_l + * + * On which we can define a combined lag: + * + * lag_k+l(i) := S_k+l - s_i (11) + * + * And that gives us the tools to compare tasks across a combined runqueue. + * + * + * Combined this gives the following: + * + * a) when a runqueue enters force-idle, sync it against it's sibling rq(s) + * using (7); this only requires storing single 'time'-stamps. + * + * b) when comparing tasks between 2 runqueues of which one is forced-idle, + * compare the combined lag, per (11). + * + * Now, of course cgroups (I so hate them) make this more interesting in + * that a) seems to suggest we need to iterate all cgroup on a CPU at such + * boundaries, but I think we can avoid that. The force-idle is for the + * whole CPU, all it's rqs. So we can mark it in the root and lazily + * propagate downward on demand. + */ + +/* + * So this sync is basically a relative reset of S to 0. + * + * So with 2 queues, when one goes idle, we drop them both to 0 and one + * then increases due to not being idle, and the idle one builds up lag to + * get re-elected. So far so simple, right? + * + * When there's 3, we can have the situation where 2 run and one is idle, + * we sync to 0 and let the idle one build up lag to get re-election. Now + * suppose another one also drops idle. At this point dropping all to 0 + * again would destroy the built-up lag from the queue that was already + * idle, not good. + * + * So instead of syncing everything, we can: + * + * less := !((s64)(s_a - s_b) <= 0) + * + * (v_a - S_a) - (v_b - S_b) == v_a - v_b - S_a + S_b + * == v_a - (v_b - S_a + S_b) + * + * IOW, we can recast the (lag) comparison to a one-sided difference. + * So if then, instead of syncing the whole queue, sync the idle queue + * against the active queue with S_a + S_b at the point where we sync. + * + * (XXX consider the implication of living in a cyclic group: N / 2^n N) + * + * This gives us means of syncing single queues against the active queue, + * and for already idle queues to preserve their build-up lag. + * + * Of course, then we get the situation where there's 2 active and one + * going idle, who do we pick to sync against? Theory would have us sync + * against the combined S, but as we've already demonstrated, there is no + * such thing in infeasible weight scenarios. + * + * One thing I've considered; and this is where that core_active rudiment + * came from, is having active queues sync up between themselves after + * every tick. This limits the observed divergence due to the work + * conservancy. + * + * On top of that, we can improve upon things by employing (10) here. + */ + +/* + * se_fi_update - Update the cfs_rq->zero_vruntime_fi in a CFS hierarchy if needed. */ static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, bool forceidle) @@ -13026,7 +13288,7 @@ static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, cfs_rq->forceidle_seq = fi_seq; } - cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime; + cfs_rq->zero_vruntime_fi = cfs_rq->zero_vruntime; } } @@ -13079,11 +13341,11 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, /* * Find delta after normalizing se's vruntime with its cfs_rq's - * min_vruntime_fi, which would have been updated in prior calls + * zero_vruntime_fi, which would have been updated in prior calls * to se_fi_update(). */ - delta = (s64)(sea->vruntime - seb->vruntime) + - (s64)(cfs_rqb->min_vruntime_fi - cfs_rqa->min_vruntime_fi); + delta = vruntime_op(sea->vruntime, "-", seb->vruntime) + + vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi); return delta > 0; } @@ -13308,7 +13570,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - set_next_entity(cfs_rq, se); + set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ account_cfs_rq_runtime(cfs_rq, 0); } @@ -13319,7 +13581,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) void init_cfs_rq(struct cfs_rq *cfs_rq) { cfs_rq->tasks_timeline = RB_ROOT_CACHED; - cfs_rq->min_vruntime = (u64)(-(1LL << 20)); + cfs_rq->zero_vruntime = (u64)(-(1LL << 20)); raw_spin_lock_init(&cfs_rq->removed.lock); } diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 3c12d9f93331d..136a6584be797 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -121,3 +121,8 @@ SCHED_FEAT(WA_BIAS, true) SCHED_FEAT(UTIL_EST, true) SCHED_FEAT(LATENCY_WARN, false) + +/* + * Do newidle balancing proportional to its success rate using randomization. + */ +SCHED_FEAT(NI_RANDOM, true) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index c39b089d4f09b..bf92ae29361ed 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -160,6 +160,14 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, return cpuidle_enter(drv, dev, next_state); } +static void idle_call_stop_or_retain_tick(bool stop_tick) +{ + if (stop_tick || tick_nohz_tick_stopped()) + tick_nohz_idle_stop_tick(); + else + tick_nohz_idle_retain_tick(); +} + /** * cpuidle_idle_call - the main idle function * @@ -169,7 +177,7 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, * set, and it returns with polling set. If it ever stops polling, it * must clear the polling bit. */ -static void cpuidle_idle_call(void) +static void cpuidle_idle_call(bool stop_tick) { struct cpuidle_device *dev = cpuidle_get_device(); struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); @@ -185,7 +193,7 @@ static void cpuidle_idle_call(void) } if (cpuidle_not_available(drv, dev)) { - tick_nohz_idle_stop_tick(); + idle_call_stop_or_retain_tick(stop_tick); default_idle_call(); goto exit_idle; @@ -219,24 +227,35 @@ static void cpuidle_idle_call(void) next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns); call_cpuidle(drv, dev, next_state); - } else { - bool stop_tick = true; + } else if (drv->state_count > 1) { + /* + * stop_tick is expected to be true by default by cpuidle + * governors, which allows them to select idle states with + * target residency above the tick period length. + */ + stop_tick = true; /* * Ask the cpuidle framework to choose a convenient idle state. */ next_state = cpuidle_select(drv, dev, &stop_tick); - if (stop_tick || tick_nohz_tick_stopped()) - tick_nohz_idle_stop_tick(); - else - tick_nohz_idle_retain_tick(); + idle_call_stop_or_retain_tick(stop_tick); entered_state = call_cpuidle(drv, dev, next_state); /* * Give the governor an opportunity to reflect on the outcome */ cpuidle_reflect(dev, entered_state); + } else { + idle_call_stop_or_retain_tick(stop_tick); + + /* + * If there is only a single idle state (or none), there is + * nothing meaningful for the governor to choose. Skip the + * governor and always use state 0. + */ + call_cpuidle(drv, dev, 0); } exit_idle: @@ -257,6 +276,7 @@ static void cpuidle_idle_call(void) static void do_idle(void) { int cpu = smp_processor_id(); + bool got_tick = false; /* * Check if we need to update blocked load @@ -327,8 +347,9 @@ static void do_idle(void) tick_nohz_idle_restart_tick(); cpu_idle_poll(); } else { - cpuidle_idle_call(); + cpuidle_idle_call(got_tick); } + got_tick = tick_nohz_idle_got_tick(); arch_cpu_idle_exit(); } @@ -464,6 +485,12 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir scx_update_idle(rq, true, true); schedstat_inc(rq->sched_goidle); next->se.exec_start = rq_clock_task(rq); + + /* + * rq is about to be idle, check if we need to update the + * lost_idle_time of clock_pelt + */ + update_idle_rq_clock_pelt(rq); } struct task_struct *pick_task_idle(struct rq *rq) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 7936d43337313..d4d994fb8999a 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1490,7 +1490,7 @@ static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head) static void yield_task_rt(struct rq *rq) { - requeue_task_rt(rq, rq->curr, 0); + requeue_task_rt(rq, rq->donor, 0); } static int find_lowest_rq(struct task_struct *task); @@ -2100,6 +2100,7 @@ static void push_rt_tasks(struct rq *rq) */ static int rto_next_cpu(struct root_domain *rd) { + int this_cpu = smp_processor_id(); int next; int cpu; @@ -2123,6 +2124,10 @@ static int rto_next_cpu(struct root_domain *rd) rd->rto_cpu = cpu; + /* Do not send IPI to self */ + if (cpu == this_cpu) + continue; + if (cpu < nr_cpu_ids) return cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index adfb6e3409d72..ed37ab9209e59 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -5,6 +5,7 @@ #ifndef _KERNEL_SCHED_SCHED_H #define _KERNEL_SCHED_SCHED_H +#include #include #include #include @@ -679,13 +680,13 @@ struct cfs_rq { unsigned int h_nr_runnable; /* SCHED_{NORMAL,BATCH,IDLE} */ unsigned int h_nr_idle; /* SCHED_IDLE */ - s64 avg_vruntime; - u64 avg_load; + s64 sum_w_vruntime; + u64 sum_weight; - u64 min_vruntime; + u64 zero_vruntime; #ifdef CONFIG_SCHED_CORE unsigned int forceidle_seq; - u64 min_vruntime_fi; + u64 zero_vruntime_fi; #endif struct rb_root_cached tasks_timeline; @@ -1349,6 +1350,12 @@ static inline bool is_migration_disabled(struct task_struct *p) } DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); +DECLARE_PER_CPU(struct rnd_state, sched_rnd_state); + +static inline u32 sched_rng(void) +{ + return prandom_u32_state(this_cpu_ptr(&sched_rnd_state)); +} #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) #define this_rq() this_cpu_ptr(&runqueues) diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 26f3fd4d34cea..73bd6bca4d310 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -180,8 +180,13 @@ static inline void psi_dequeue(struct task_struct *p, int flags) * avoid walking all ancestors twice, psi_task_switch() handles * TSK_RUNNING and TSK_IOWAIT for us when it moves TSK_ONCPU. * Do nothing here. + * + * In the SCHED_PROXY_EXECUTION case we may do sleeping + * dequeues that are not followed by a task switch, so check + * TSK_ONCPU is set to ensure the task switch is imminent. + * Otherwise clear the flags as usual. */ - if (flags & DEQUEUE_SLEEP) + if ((flags & DEQUEUE_SLEEP) && (p->psi_flags & TSK_ONCPU)) return; /* diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index 77ae87f36e841..6805a63d47af7 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -688,7 +688,7 @@ int __sched_setscheduler(struct task_struct *p, * itself. */ newprio = rt_effective_prio(p, newprio); - if (newprio == oldprio) + if (newprio == oldprio && !dl_prio(newprio)) queue_flags &= ~DEQUEUE_MOVE; } @@ -1351,7 +1351,7 @@ static void do_sched_yield(void) rq = this_rq_lock_irq(&rf); schedstat_inc(rq->yld_count); - current->sched_class->yield_task(rq); + rq->donor->sched_class->yield_task(rq); preempt_disable(); rq_unlock_irq(rq, &rf); @@ -1420,12 +1420,13 @@ EXPORT_SYMBOL(yield); */ int __sched yield_to(struct task_struct *p, bool preempt) { - struct task_struct *curr = current; + struct task_struct *curr; struct rq *rq, *p_rq; int yielded = 0; scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { rq = this_rq(); + curr = rq->donor; again: p_rq = task_rq(p); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 444bdfdab7318..c7a4d2fff5718 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1662,6 +1662,12 @@ sd_init(struct sched_domain_topology_level *tl, .last_balance = jiffies, .balance_interval = sd_weight, + + /* 50% success rate */ + .newidle_call = 512, + .newidle_success = 256, + .newidle_ratio = 512, + .max_newidle_lb_cost = 0, .last_decay_max_lb_cost = jiffies, .child = child, diff --git a/kernel/scs.c b/kernel/scs.c index d7809affe7404..772488afd5b97 100644 --- a/kernel/scs.c +++ b/kernel/scs.c @@ -135,7 +135,7 @@ static void scs_check_usage(struct task_struct *tsk) if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) return; - for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { + for (p = task_scs(tsk); p < __scs_magic(task_scs(tsk)); ++p) { if (!READ_ONCE_NOCHECK(*p)) break; used += sizeof(*p); diff --git a/kernel/smp.c b/kernel/smp.c index 02f52291fae42..f349960f79cad 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -1087,6 +1087,28 @@ void wake_up_all_idle_cpus(void) } EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus); +/** + * cpus_peek_for_pending_ipi - Check for pending IPI for CPUs + * @mask: The CPU mask for the CPUs to check. + * + * This function walks through the @mask to check if there are any pending IPIs + * scheduled, for any of the CPUs in the @mask. It does not guarantee + * correctness as it only provides a racy snapshot. + * + * Returns true if there is a pending IPI scheduled and false otherwise. + */ +bool cpus_peek_for_pending_ipi(const struct cpumask *mask) +{ + unsigned int cpu; + + for_each_cpu(cpu, mask) { + if (!llist_empty(per_cpu_ptr(&call_single_queue, cpu))) + return true; + } + + return false; +} + /** * struct smp_call_on_cpu_struct - Call a function on a specific CPU * @work: &work_struct diff --git a/kernel/sysctl.c b/kernel/sysctl.c index cb6196e3fa993..970525e6c76c7 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1213,7 +1213,7 @@ int proc_do_large_bitmap(const struct ctl_table *table, int write, unsigned long bitmap_len = table->maxlen; unsigned long *bitmap = *(unsigned long **) table->data; unsigned long *tmp_bitmap = NULL; - char tr_a[] = { '-', ',', '\n' }, tr_b[] = { ',', '\n', 0 }, c; + char tr_a[] = { '-', ',', '\n' }, tr_b[] = { ',', '\n', 0 }, c = 0; if (!bitmap || !bitmap_len || !left || (*ppos && !write)) { *lenp = 0; diff --git a/kernel/task_work.c b/kernel/task_work.c index d1efec571a4a4..0f7519f8e7c93 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -9,7 +9,12 @@ static struct callback_head work_exited; /* all we need is ->next == NULL */ #ifdef CONFIG_IRQ_WORK static void task_work_set_notify_irq(struct irq_work *entry) { - test_and_set_tsk_thread_flag(current, TIF_NOTIFY_RESUME); + /* + * no-op IPI + * + * TWA_NMI_CURRENT will already have set the TIF flag, all + * this interrupt does it tickle the return-to-user path. + */ } static DEFINE_PER_CPU(struct irq_work, irq_work_NMI_resume) = IRQ_WORK_INIT_HARD(task_work_set_notify_irq); @@ -86,6 +91,7 @@ int task_work_add(struct task_struct *task, struct callback_head *work, break; #ifdef CONFIG_IRQ_WORK case TWA_NMI_CURRENT: + set_tsk_thread_flag(current, TIF_NOTIFY_RESUME); irq_work_queue(this_cpu_ptr(&irq_work_NMI_resume)); break; #endif diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 069d93bfb0c75..b64db405ba5c7 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -540,7 +540,7 @@ static s64 alarm_timer_forward(struct k_itimer *timr, ktime_t now) { struct alarm *alarm = &timr->it.alarm.alarmtimer; - return alarm_forward(alarm, timr->it_interval, now); + return alarm_forward(alarm, now, timr->it_interval); } /** diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index a1890a073196b..df71949616584 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -252,7 +252,7 @@ enum wd_read_status { static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow) { - int64_t md = 2 * watchdog->uncertainty_margin; + int64_t md = watchdog->uncertainty_margin; unsigned int nretries, max_retries; int64_t wd_delay, wd_seq_delay; u64 wd_end, wd_end2; diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 88aa062b8a556..21b6d93401480 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -913,7 +913,7 @@ static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, return true; /* Extra check for softirq clock bases */ - if (base->clockid < HRTIMER_BASE_MONOTONIC_SOFT) + if (base->index < HRTIMER_BASE_MONOTONIC_SOFT) continue; if (cpu_base->softirq_activated) continue; @@ -1742,7 +1742,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, lockdep_assert_held(&cpu_base->lock); - debug_deactivate(timer); + debug_hrtimer_deactivate(timer); base->running = timer; /* diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index cc1afec306b3f..425d429906d0d 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -215,7 +215,7 @@ void sched_clock_register(u64 (*read)(void), int bits, unsigned long rate) update_clock_read_data(&rd); - if (sched_clock_timer.function != NULL) { + if (ACCESS_PRIVATE(&sched_clock_timer, function) != NULL) { /* update timeout for clock wrap */ hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD); diff --git a/kernel/time/time.c b/kernel/time/time.c index 0ba8e3c50d625..155cf7def9146 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -702,7 +702,7 @@ EXPORT_SYMBOL(clock_t_to_jiffies); * * Return: jiffies_64 value converted to 64-bit "clock_t" (CLOCKS_PER_SEC) */ -u64 jiffies_64_to_clock_t(u64 x) +notrace u64 jiffies_64_to_clock_t(u64 x) { #if (TICK_NSEC % (NSEC_PER_SEC / USER_HZ)) == 0 # if HZ < USER_HZ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 4790da895203c..c7dcccc5f3d6b 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2639,7 +2639,8 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux if (aux_clock) { /* Auxiliary clocks are similar to TAI and do not have leap seconds */ - if (txc->status & (STA_INS | STA_DEL)) + if (txc->modes & ADJ_STATUS && + txc->status & (STA_INS | STA_DEL)) return -EINVAL; /* No TAI offset setting */ @@ -2647,7 +2648,8 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux return -EINVAL; /* No PPS support either */ - if (txc->status & (STA_PPSFREQ | STA_PPSTIME)) + if (txc->modes & ADJ_STATUS && + txc->status & (STA_PPSFREQ | STA_PPSTIME)) return -EINVAL; } @@ -2721,7 +2723,7 @@ static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc, timekeeping_update_from_shadow(tkd, TK_CLOCK_WAS_SET); result->clock_set = true; } else { - tk_update_leap_state_all(&tk_core); + tk_update_leap_state_all(tkd); } /* Update the multiplier immediately if frequency was set directly */ diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index c0c54dc5314c3..49635a2b7ee28 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -420,6 +420,8 @@ static struct list_head *tmigr_level_list __read_mostly; static unsigned int tmigr_hierarchy_levels __read_mostly; static unsigned int tmigr_crossnode_level __read_mostly; +static struct tmigr_group *tmigr_root; + static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu); #define TMIGR_NONE 0xFF @@ -522,11 +524,9 @@ struct tmigr_walk { typedef bool (*up_f)(struct tmigr_group *, struct tmigr_group *, struct tmigr_walk *); -static void __walk_groups(up_f up, struct tmigr_walk *data, - struct tmigr_cpu *tmc) +static void __walk_groups_from(up_f up, struct tmigr_walk *data, + struct tmigr_group *child, struct tmigr_group *group) { - struct tmigr_group *child = NULL, *group = tmc->tmgroup; - do { WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels); @@ -544,6 +544,12 @@ static void __walk_groups(up_f up, struct tmigr_walk *data, } while (group); } +static void __walk_groups(up_f up, struct tmigr_walk *data, + struct tmigr_cpu *tmc) +{ + __walk_groups_from(up, data, NULL, tmc->tmgroup); +} + static void walk_groups(up_f up, struct tmigr_walk *data, struct tmigr_cpu *tmc) { lockdep_assert_held(&tmc->lock); @@ -1498,21 +1504,6 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, s.seq = 0; atomic_set(&group->migr_state, s.state); - /* - * If this is a new top-level, prepare its groupmask in advance. - * This avoids accidents where yet another new top-level is - * created in the future and made visible before the current groupmask. - */ - if (list_empty(&tmigr_level_list[lvl])) { - group->groupmask = BIT(0); - /* - * The previous top level has prepared its groupmask already, - * simply account it as the first child. - */ - if (lvl > 0) - group->num_children = 1; - } - timerqueue_init_head(&group->events); timerqueue_init(&group->groupevt.nextevt); group->groupevt.nextevt.expires = KTIME_MAX; @@ -1567,25 +1558,51 @@ static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node, return group; } +static bool tmigr_init_root(struct tmigr_group *group, bool activate) +{ + if (!group->parent && group != tmigr_root) { + /* + * This is the new top-level, prepare its groupmask in advance + * to avoid accidents where yet another new top-level is + * created in the future and made visible before this groupmask. + */ + group->groupmask = BIT(0); + WARN_ON_ONCE(activate); + + return true; + } + + return false; + +} + static void tmigr_connect_child_parent(struct tmigr_group *child, struct tmigr_group *parent, bool activate) { - struct tmigr_walk data; - - raw_spin_lock_irq(&child->lock); - raw_spin_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING); + if (tmigr_init_root(parent, activate)) { + /* + * The previous top level had prepared its groupmask already, + * simply account it in advance as the first child. If some groups + * have been created between the old and new root due to node + * mismatch, the new root's child will be intialized accordingly. + */ + parent->num_children = 1; + } - if (activate) { + /* Connecting old root to new root ? */ + if (!parent->parent && activate) { /* - * @child is the old top and @parent the new one. In this - * case groupmask is pre-initialized and @child already - * accounted, along with its new sibling corresponding to the - * CPU going up. + * @child is the old top, or in case of node mismatch, some + * intermediate group between the old top and the new one in + * @parent. In this case the @child must be pre-accounted above + * as the first child. Its new inactive sibling corresponding + * to the CPU going up has been accounted as the second child. */ - WARN_ON_ONCE(child->groupmask != BIT(0) || parent->num_children != 2); + WARN_ON_ONCE(parent->num_children != 2); + child->groupmask = BIT(0); } else { - /* Adding @child for the CPU going up to @parent. */ + /* Common case adding @child for the CPU going up to @parent. */ child->groupmask = BIT(parent->num_children++); } @@ -1596,87 +1613,61 @@ static void tmigr_connect_child_parent(struct tmigr_group *child, */ smp_store_release(&child->parent, parent); - raw_spin_unlock(&parent->lock); - raw_spin_unlock_irq(&child->lock); - trace_tmigr_connect_child_parent(child); - - if (!activate) - return; - - /* - * To prevent inconsistent states, active children need to be active in - * the new parent as well. Inactive children are already marked inactive - * in the parent group: - * - * * When new groups were created by tmigr_setup_groups() starting from - * the lowest level (and not higher then one level below the current - * top level), then they are not active. They will be set active when - * the new online CPU comes active. - * - * * But if a new group above the current top level is required, it is - * mandatory to propagate the active state of the already existing - * child to the new parent. So tmigr_connect_child_parent() is - * executed with the formerly top level group (child) and the newly - * created group (parent). - * - * * It is ensured that the child is active, as this setup path is - * executed in hotplug prepare callback. This is exectued by an - * already connected and !idle CPU. Even if all other CPUs go idle, - * the CPU executing the setup will be responsible up to current top - * level group. And the next time it goes inactive, it will release - * the new childmask and parent to subsequent walkers through this - * @child. Therefore propagate active state unconditionally. - */ - data.childmask = child->groupmask; - - /* - * There is only one new level per time (which is protected by - * tmigr_mutex). When connecting the child and the parent and set the - * child active when the parent is inactive, the parent needs to be the - * uppermost level. Otherwise there went something wrong! - */ - WARN_ON(!tmigr_active_up(parent, child, &data) && parent->parent); } -static int tmigr_setup_groups(unsigned int cpu, unsigned int node) +static int tmigr_setup_groups(unsigned int cpu, unsigned int node, + struct tmigr_group *start, bool activate) { struct tmigr_group *group, *child, **stack; - int top = 0, err = 0, i = 0; - struct list_head *lvllist; + int i, top = 0, err = 0, start_lvl = 0; + bool root_mismatch = false; stack = kcalloc(tmigr_hierarchy_levels, sizeof(*stack), GFP_KERNEL); if (!stack) return -ENOMEM; - do { + if (start) { + stack[start->level] = start; + start_lvl = start->level + 1; + } + + if (tmigr_root) + root_mismatch = tmigr_root->numa_node != node; + + for (i = start_lvl; i < tmigr_hierarchy_levels; i++) { group = tmigr_get_group(cpu, node, i); if (IS_ERR(group)) { err = PTR_ERR(group); + i--; break; } top = i; - stack[i++] = group; + stack[i] = group; /* * When booting only less CPUs of a system than CPUs are - * available, not all calculated hierarchy levels are required. + * available, not all calculated hierarchy levels are required, + * unless a node mismatch is detected. * * The loop is aborted as soon as the highest level, which might * be different from tmigr_hierarchy_levels, contains only a - * single group. + * single group, unless the nodes mismatch below tmigr_crossnode_level */ - if (group->parent || list_is_singular(&tmigr_level_list[i - 1])) + if (group->parent) break; + if ((!root_mismatch || i >= tmigr_crossnode_level) && + list_is_singular(&tmigr_level_list[i])) + break; + } - } while (i < tmigr_hierarchy_levels); - - /* Assert single root */ - WARN_ON_ONCE(!err && !group->parent && !list_is_singular(&tmigr_level_list[top])); + /* Assert single root without parent */ + if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) + return -EINVAL; - while (i > 0) { - group = stack[--i]; + for (; i >= start_lvl; i--) { + group = stack[i]; if (err < 0) { list_del(&group->list); @@ -1692,12 +1683,10 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) if (i == 0) { struct tmigr_cpu *tmc = per_cpu_ptr(&tmigr_cpu, cpu); - raw_spin_lock_irq(&group->lock); - tmc->tmgroup = group; tmc->groupmask = BIT(group->num_children++); - raw_spin_unlock_irq(&group->lock); + tmigr_init_root(group, activate); trace_tmigr_connect_cpu_parent(tmc); @@ -1705,42 +1694,55 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) continue; } else { child = stack[i - 1]; - /* Will be activated at online time */ - tmigr_connect_child_parent(child, group, false); + tmigr_connect_child_parent(child, group, activate); } + } - /* check if uppermost level was newly created */ - if (top != i) - continue; - - WARN_ON_ONCE(top == 0); + if (err < 0) + goto out; - lvllist = &tmigr_level_list[top]; + if (activate) { + struct tmigr_walk data; /* - * Newly created root level should have accounted the upcoming - * CPU's child group and pre-accounted the old root. + * To prevent inconsistent states, active children need to be active in + * the new parent as well. Inactive children are already marked inactive + * in the parent group: + * + * * When new groups were created by tmigr_setup_groups() starting from + * the lowest level, then they are not active. They will be set active + * when the new online CPU comes active. + * + * * But if new groups above the current top level are required, it is + * mandatory to propagate the active state of the already existing + * child to the new parents. So tmigr_active_up() activates the + * new parents while walking up from the old root to the new. + * + * * It is ensured that @start is active, as this setup path is + * executed in hotplug prepare callback. This is executed by an + * already connected and !idle CPU. Even if all other CPUs go idle, + * the CPU executing the setup will be responsible up to current top + * level group. And the next time it goes inactive, it will release + * the new childmask and parent to subsequent walkers through this + * @child. Therefore propagate active state unconditionally. */ - if (group->num_children == 2 && list_is_singular(lvllist)) { - /* - * The target CPU must never do the prepare work, except - * on early boot when the boot CPU is the target. Otherwise - * it may spuriously activate the old top level group inside - * the new one (nevertheless whether old top level group is - * active or not) and/or release an uninitialized childmask. - */ - WARN_ON_ONCE(cpu == raw_smp_processor_id()); - - lvllist = &tmigr_level_list[top - 1]; - list_for_each_entry(child, lvllist, list) { - if (child->parent) - continue; + WARN_ON_ONCE(!start->parent); + data.childmask = start->groupmask; + __walk_groups_from(tmigr_active_up, &data, start, start->parent); + } - tmigr_connect_child_parent(child, group, true); - } + /* Root update */ + if (list_is_singular(&tmigr_level_list[top])) { + group = list_first_entry(&tmigr_level_list[top], + typeof(*group), list); + WARN_ON_ONCE(group->parent); + if (tmigr_root) { + /* Old root should be the same or below */ + WARN_ON_ONCE(tmigr_root->level > top); } + tmigr_root = group; } - +out: kfree(stack); return err; @@ -1748,12 +1750,26 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) static int tmigr_add_cpu(unsigned int cpu) { + struct tmigr_group *old_root = tmigr_root; int node = cpu_to_node(cpu); int ret; - mutex_lock(&tmigr_mutex); - ret = tmigr_setup_groups(cpu, node); - mutex_unlock(&tmigr_mutex); + guard(mutex)(&tmigr_mutex); + + ret = tmigr_setup_groups(cpu, node, NULL, false); + + /* Root has changed? Connect the old one to the new */ + if (ret >= 0 && old_root && old_root != tmigr_root) { + /* + * The target CPU must never do the prepare work, except + * on early boot when the boot CPU is the target. Otherwise + * it may spuriously activate the old top level group inside + * the new one (nevertheless whether old top level group is + * active or not) and/or release an uninitialized childmask. + */ + WARN_ON_ONCE(cpu == raw_smp_processor_id()); + ret = tmigr_setup_groups(-1, old_root->numa_node, old_root, true); + } return ret; } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 4f87c16d915a0..70f1292b7ddbd 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -965,7 +965,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1022,7 +1022,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, @@ -1526,7 +1526,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1661,7 +1661,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -2441,8 +2441,10 @@ static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) { struct bpf_kprobe_multi_link *kmulti_link; + bool has_cookies; kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); + has_cookies = !!kmulti_link->cookies; seq_printf(seq, "kprobe_cnt:\t%u\n" @@ -2454,7 +2456,7 @@ static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, for (int i = 0; i < kmulti_link->cnt; i++) { seq_printf(seq, "%llu\t %pS\n", - kmulti_link->cookies[i], + has_cookies ? kmulti_link->cookies[i] : 0, (void *)kmulti_link->addrs[i]); } } @@ -2564,6 +2566,7 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); err = bpf_prog_run(link->link.prog, regs); bpf_reset_run_ctx(old_run_ctx); + ftrace_partial_regs_update(fregs, bpf_kprobe_multi_pt_regs_ptr()); rcu_read_unlock(); out: @@ -2736,6 +2739,10 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr if (!is_kprobe_multi(prog)) return -EINVAL; + /* kprobe_multi is not allowed to be sleepable. */ + if (prog->sleepable) + return -EINVAL; + /* Writing to context is not allowed for kprobes. */ if (prog->aux->kprobe_write_ctx) return -EINVAL; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 484ad7a18463a..234fd9d45b56e 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -542,7 +542,11 @@ static struct fgraph_ops fgraph_stub = { static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub; DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub); DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub); +#if FGRAPH_NO_DIRECT +static DEFINE_STATIC_KEY_FALSE(fgraph_do_direct); +#else static DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); +#endif /** * ftrace_graph_stop - set to permanently disable function graph tracing @@ -846,7 +850,7 @@ __ftrace_return_to_handler(struct ftrace_regs *fregs, unsigned long frame_pointe bitmap = get_bitmap_bits(current, offset); #ifdef CONFIG_HAVE_STATIC_CALL - if (static_branch_likely(&fgraph_do_direct)) { + if (!FGRAPH_NO_DIRECT && static_branch_likely(&fgraph_do_direct)) { if (test_bit(fgraph_direct_gops->idx, &bitmap)) static_call(fgraph_retfunc)(&trace, fgraph_direct_gops, fregs); } else @@ -1019,6 +1023,7 @@ void fgraph_init_ops(struct ftrace_ops *dst_ops, mutex_init(&dst_ops->local_hash.regex_lock); INIT_LIST_HEAD(&dst_ops->subop_list); dst_ops->flags |= FTRACE_OPS_FL_INITIALIZED; + dst_ops->private = src_ops->private; } #endif } @@ -1292,6 +1297,9 @@ static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *go trace_func_graph_ret_t retfunc = NULL; int i; + if (FGRAPH_NO_DIRECT) + return; + if (gops) { func = gops->entryfunc; retfunc = gops->retfunc; @@ -1310,11 +1318,14 @@ static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *go static_call_update(fgraph_func, func); static_call_update(fgraph_retfunc, retfunc); if (enable_branch) - static_branch_disable(&fgraph_do_direct); + static_branch_enable(&fgraph_do_direct); } static void ftrace_graph_disable_direct(bool disable_branch) { + if (FGRAPH_NO_DIRECT) + return; + if (disable_branch) static_branch_disable(&fgraph_do_direct); static_call_update(fgraph_func, ftrace_graph_entry_stub); @@ -1376,6 +1387,13 @@ int register_ftrace_graph(struct fgraph_ops *gops) ftrace_graph_active++; + /* Always save the function, and reset at unregistering */ + gops->saved_func = gops->entryfunc; +#ifdef CONFIG_DYNAMIC_FTRACE + if (ftrace_pids_enabled(&gops->ops)) + gops->entryfunc = fgraph_pid_func; +#endif + if (ftrace_graph_active == 2) ftrace_graph_disable_direct(true); @@ -1395,8 +1413,6 @@ int register_ftrace_graph(struct fgraph_ops *gops) } else { init_task_vars(gops->idx); } - /* Always save the function, and reset at unregistering */ - gops->saved_func = gops->entryfunc; gops->ops.flags |= FTRACE_OPS_FL_GRAPH; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 59cfacb8a5bbd..905f4d1679559 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1122,7 +1122,6 @@ struct ftrace_page { }; #define ENTRY_SIZE sizeof(struct dyn_ftrace) -#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE) static struct ftrace_page *ftrace_pages_start; static struct ftrace_page *ftrace_pages; @@ -3808,7 +3807,8 @@ static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) return 0; } -static int ftrace_allocate_records(struct ftrace_page *pg, int count) +static int ftrace_allocate_records(struct ftrace_page *pg, int count, + unsigned long *num_pages) { int order; int pages; @@ -3818,7 +3818,7 @@ static int ftrace_allocate_records(struct ftrace_page *pg, int count) return -EINVAL; /* We want to fill as much as possible, with no empty pages */ - pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE); + pages = DIV_ROUND_UP(count * ENTRY_SIZE, PAGE_SIZE); order = fls(pages) - 1; again: @@ -3833,6 +3833,7 @@ static int ftrace_allocate_records(struct ftrace_page *pg, int count) } ftrace_number_of_pages += 1 << order; + *num_pages += 1 << order; ftrace_number_of_groups++; cnt = (PAGE_SIZE << order) / ENTRY_SIZE; @@ -3861,12 +3862,14 @@ static void ftrace_free_pages(struct ftrace_page *pages) } static struct ftrace_page * -ftrace_allocate_pages(unsigned long num_to_init) +ftrace_allocate_pages(unsigned long num_to_init, unsigned long *num_pages) { struct ftrace_page *start_pg; struct ftrace_page *pg; int cnt; + *num_pages = 0; + if (!num_to_init) return NULL; @@ -3880,7 +3883,7 @@ ftrace_allocate_pages(unsigned long num_to_init) * waste as little space as possible. */ for (;;) { - cnt = ftrace_allocate_records(pg, num_to_init); + cnt = ftrace_allocate_records(pg, num_to_init, num_pages); if (cnt < 0) goto free_pages; @@ -7148,8 +7151,6 @@ static int ftrace_process_locs(struct module *mod, if (!count) return 0; - pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE); - /* * Sorting mcount in vmlinux at build time depend on * CONFIG_BUILDTIME_MCOUNT_SORT, while mcount loc in @@ -7162,7 +7163,7 @@ static int ftrace_process_locs(struct module *mod, test_is_sorted(start, count); } - start_pg = ftrace_allocate_pages(count); + start_pg = ftrace_allocate_pages(count, &pages); if (!start_pg) return -ENOMEM; @@ -7261,27 +7262,27 @@ static int ftrace_process_locs(struct module *mod, /* We should have used all pages unless we skipped some */ if (pg_unuse) { unsigned long pg_remaining, remaining = 0; - unsigned long skip; + long skip; /* Count the number of entries unused and compare it to skipped. */ - pg_remaining = (ENTRIES_PER_PAGE << pg->order) - pg->index; + pg_remaining = (PAGE_SIZE << pg->order) / ENTRY_SIZE - pg->index; if (!WARN(skipped < pg_remaining, "Extra allocated pages for ftrace")) { skip = skipped - pg_remaining; - for (pg = pg_unuse; pg; pg = pg->next) + for (pg = pg_unuse; pg && skip > 0; pg = pg->next) { remaining += 1 << pg->order; + skip -= (PAGE_SIZE << pg->order) / ENTRY_SIZE; + } pages -= remaining; - skip = DIV_ROUND_UP(skip, ENTRIES_PER_PAGE); - /* * Check to see if the number of pages remaining would * just fit the number of entries skipped. */ - WARN(skip != remaining, "Extra allocated pages for ftrace: %lu with %lu skipped", + WARN(pg || skip > 0, "Extra allocated pages for ftrace: %lu with %lu skipped", remaining, skipped); } /* Need to synchronize with ftrace_location_range() */ @@ -7708,7 +7709,8 @@ ftrace_func_address_lookup(struct ftrace_mod_map *mod_map, int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { struct ftrace_mod_map *mod_map; int ret = 0; @@ -7720,6 +7722,8 @@ ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, if (ret) { if (modname) *modname = mod_map->mod->name; + if (modbuildid) + *modbuildid = module_buildid(mod_map->mod); break; } } diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index afcd3747264d2..656cd8df2d55d 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1813,6 +1813,7 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu struct ring_buffer_event *event; u64 ts, delta; int events = 0; + int len; int e; *delta_ptr = 0; @@ -1820,9 +1821,12 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu ts = dpage->time_stamp; - for (e = 0; e < tail; e += rb_event_length(event)) { + for (e = 0; e < tail; e += len) { event = (struct ring_buffer_event *)(dpage->data + e); + len = rb_event_length(event); + if (len <= 0 || len > tail - e) + return -1; switch (event->type_len) { @@ -1883,6 +1887,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (!meta || !meta->head_buffer) return; + orig_head = head_page = cpu_buffer->head_page; + /* Do the reader page first */ ret = rb_validate_buffer(cpu_buffer->reader_page->page, cpu_buffer->cpu); if (ret < 0) { @@ -1893,7 +1899,6 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entry_bytes += local_read(&cpu_buffer->reader_page->page->commit); local_set(&cpu_buffer->reader_page->entries, ret); - orig_head = head_page = cpu_buffer->head_page; ts = head_page->page->time_stamp; /* @@ -2012,7 +2017,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entries += ret; entry_bytes += local_read(&head_page->page->commit); - local_set(&cpu_buffer->head_page->entries, ret); + local_set(&head_page->entries, ret); if (head_page == cpu_buffer->commit_page) break; @@ -3121,6 +3126,8 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, list) { list_del_init(&bpage->list); free_buffer_page(bpage); + + cond_resched(); } } out_err_unlock: @@ -7276,6 +7283,27 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, return err; } +/* + * This is called when a VMA is duplicated (e.g., on fork()) to increment + * the user_mapped counter without remapping pages. + */ +void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu) +{ + struct ring_buffer_per_cpu *cpu_buffer; + + if (WARN_ON(!cpumask_test_cpu(cpu, buffer->cpumask))) + return; + + cpu_buffer = buffer->buffers[cpu]; + + guard(mutex)(&cpu_buffer->mapping_lock); + + if (cpu_buffer->user_mapped) + __rb_inc_dec_mapped(cpu_buffer, true); + else + WARN(1, "Unexpected buffer stat, it should be mapped"); +} + int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) { struct ring_buffer_per_cpu *cpu_buffer; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 304e93597126e..2be9e47d64b08 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -141,7 +141,7 @@ cpumask_var_t __read_mostly tracing_buffer_mask; * by commas. */ /* Set to string format zero to disable by default */ -char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0"; +static char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0"; /* When set, tracing will stop when a WARN*() is hit */ static int __disable_trace_on_warning; @@ -569,7 +569,7 @@ static bool update_marker_trace(struct trace_array *tr, int enabled) lockdep_assert_held(&event_mutex); if (enabled) { - if (!list_empty(&tr->marker_list)) + if (tr->trace_flags & TRACE_ITER_COPY_MARKER) return false; list_add_rcu(&tr->marker_list, &marker_copies); @@ -577,10 +577,10 @@ static bool update_marker_trace(struct trace_array *tr, int enabled) return true; } - if (list_empty(&tr->marker_list)) + if (!(tr->trace_flags & TRACE_ITER_COPY_MARKER)) return false; - list_del_init(&tr->marker_list); + list_del_rcu(&tr->marker_list); tr->trace_flags &= ~TRACE_ITER_COPY_MARKER; return true; } @@ -3003,6 +3003,11 @@ static void __ftrace_trace_stack(struct trace_array *tr, struct ftrace_stack *fstack; struct stack_entry *entry; int stackidx; + int bit; + + bit = trace_test_and_set_recursion(_THIS_IP_, _RET_IP_, TRACE_EVENT_START); + if (bit < 0) + return; /* * Add one, for this function and the call to save_stack_trace() @@ -3071,6 +3076,7 @@ static void __ftrace_trace_stack(struct trace_array *tr, /* Again, don't let gcc optimize things here */ barrier(); __this_cpu_dec(ftrace_stack_reserve); + trace_clear_recursion(bit); } static inline void ftrace_trace_stack(struct trace_array *tr, @@ -4846,6 +4852,8 @@ static int tracing_single_release_tr(struct inode *inode, struct file *file) return single_release(inode, file); } +static bool update_last_data_if_empty(struct trace_array *tr); + static int tracing_open(struct inode *inode, struct file *file) { struct trace_array *tr = inode->i_private; @@ -4870,6 +4878,8 @@ static int tracing_open(struct inode *inode, struct file *file) tracing_reset_online_cpus(trace_buf); else tracing_reset_cpu(trace_buf, cpu); + + update_last_data_if_empty(tr); } if (file->f_mode & FMODE_READ) { @@ -5911,6 +5921,7 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf, int tracer_init(struct tracer *t, struct trace_array *tr) { tracing_reset_online_cpus(&tr->array_buffer); + update_last_data_if_empty(tr); return t->init(tr); } @@ -6055,10 +6066,10 @@ static int cmp_mod_entry(const void *key, const void *pivot) unsigned long addr = (unsigned long)key; const struct trace_mod_entry *ent = pivot; - if (addr >= ent[0].mod_addr && addr < ent[1].mod_addr) - return 0; - else - return addr - ent->mod_addr; + if (addr < ent[0].mod_addr) + return -1; + + return addr >= ent[1].mod_addr; } /** @@ -7334,6 +7345,23 @@ static char *trace_user_fault_read(struct trace_user_buf_info *tinfo, */ do { + /* + * It is possible that something is trying to migrate this + * task. What happens then, is when preemption is enabled, + * the migration thread will preempt this task, try to + * migrate it, fail, then let it run again. That will + * cause this to loop again and never succeed. + * On failures, enabled and disable preemption with + * migration enabled, to allow the migration thread to + * migrate this task. + */ + if (trys) { + preempt_enable_notrace(); + preempt_disable_notrace(); + cpu = smp_processor_id(); + buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf; + } + /* * If for some reason, copy_from_user() always causes a context * switch, this would then cause an infinite loop. @@ -7576,6 +7604,7 @@ int tracing_set_clock(struct trace_array *tr, const char *clockstr) ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); tracing_reset_online_cpus(&tr->max_buffer); #endif + update_last_data_if_empty(tr); if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) { struct trace_scratch *tscratch = tr->scratch; @@ -8772,6 +8801,18 @@ static inline int get_snapshot_map(struct trace_array *tr) { return 0; } static inline void put_snapshot_map(struct trace_array *tr) { } #endif +/* + * This is called when a VMA is duplicated (e.g., on fork()) to increment + * the user_mapped counter without remapping pages. + */ +static void tracing_buffers_mmap_open(struct vm_area_struct *vma) +{ + struct ftrace_buffer_info *info = vma->vm_file->private_data; + struct trace_iterator *iter = &info->iter; + + ring_buffer_map_dup(iter->array_buffer->buffer, iter->cpu_file); +} + static void tracing_buffers_mmap_close(struct vm_area_struct *vma) { struct ftrace_buffer_info *info = vma->vm_file->private_data; @@ -8791,6 +8832,7 @@ static int tracing_buffers_may_split(struct vm_area_struct *vma, unsigned long a } static const struct vm_operations_struct tracing_buffers_vmops = { + .open = tracing_buffers_mmap_open, .close = tracing_buffers_mmap_close, .may_split = tracing_buffers_may_split, }; @@ -9177,7 +9219,7 @@ tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu, tr, cpu, &tracing_stats_fops); - trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu, + trace_create_cpu_file("buffer_size_kb", TRACE_MODE_WRITE, d_cpu, tr, cpu, &tracing_entries_fops); if (tr->range_addr_start) @@ -9824,7 +9866,7 @@ static void setup_trace_scratch(struct trace_array *tr, } static int -allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size) +allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, unsigned long size) { enum ring_buffer_flags rb_flags; struct trace_scratch *tscratch; @@ -9879,7 +9921,7 @@ static void free_trace_buffer(struct array_buffer *buf) } } -static int allocate_trace_buffers(struct trace_array *tr, int size) +static int allocate_trace_buffers(struct trace_array *tr, unsigned long size) { int ret; @@ -10190,18 +10232,19 @@ static int __remove_instance(struct trace_array *tr) list_del(&tr->list); - /* Disable all the flags that were enabled coming in */ - for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { - if ((1 << i) & ZEROED_TRACE_FLAGS) - set_tracer_flag(tr, 1 << i, 0); - } - if (printk_trace == tr) update_printk_trace(&global_trace); + /* Must be done before disabling all the flags */ if (update_marker_trace(tr, 0)) synchronize_rcu(); + /* Disable all the flags that were enabled coming in */ + for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { + if ((1 << i) & ZEROED_TRACE_FLAGS) + set_tracer_flag(tr, 1 << i, 0); + } + tracing_set_nop(tr); clear_ftrace_function_probes(tr); event_trace_del_tracer(tr); @@ -11161,7 +11204,7 @@ __init static void enable_instances(void) __init static int tracer_alloc_buffers(void) { - int ring_buf_size; + unsigned long ring_buf_size; int ret = -ENOMEM; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 85eabb454bee9..ec372e0f2e716 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -67,14 +67,17 @@ enum trace_type { #undef __field_fn #define __field_fn(type, item) type item; +#undef __field_packed +#define __field_packed(type, item) type item; + #undef __field_struct #define __field_struct(type, item) __field(type, item) #undef __field_desc #define __field_desc(type, container, item) -#undef __field_packed -#define __field_packed(type, container, item) +#undef __field_desc_packed +#define __field_desc_packed(type, container, item) #undef __array #define __array(type, item, size) type item[size]; diff --git a/kernel/trace/trace_entries.h b/kernel/trace/trace_entries.h index de294ae2c5c5a..a649dfcf9b7cd 100644 --- a/kernel/trace/trace_entries.h +++ b/kernel/trace/trace_entries.h @@ -79,8 +79,8 @@ FTRACE_ENTRY(funcgraph_entry, ftrace_graph_ent_entry, F_STRUCT( __field_struct( struct ftrace_graph_ent, graph_ent ) - __field_packed( unsigned long, graph_ent, func ) - __field_packed( unsigned int, graph_ent, depth ) + __field_desc_packed(unsigned long, graph_ent, func ) + __field_desc_packed(unsigned int, graph_ent, depth ) __dynamic_array(unsigned long, args ) ), @@ -96,9 +96,9 @@ FTRACE_ENTRY_PACKED(fgraph_retaddr_entry, fgraph_retaddr_ent_entry, F_STRUCT( __field_struct( struct fgraph_retaddr_ent, graph_ent ) - __field_packed( unsigned long, graph_ent, func ) - __field_packed( unsigned int, graph_ent, depth ) - __field_packed( unsigned long, graph_ent, retaddr ) + __field_desc_packed( unsigned long, graph_ent, func ) + __field_desc_packed( unsigned int, graph_ent, depth ) + __field_desc_packed( unsigned long, graph_ent, retaddr ) ), F_printk("--> %ps (%u) <- %ps", (void *)__entry->func, __entry->depth, @@ -122,12 +122,12 @@ FTRACE_ENTRY_PACKED(funcgraph_exit, ftrace_graph_ret_entry, F_STRUCT( __field_struct( struct ftrace_graph_ret, ret ) - __field_packed( unsigned long, ret, func ) - __field_packed( unsigned long, ret, retval ) - __field_packed( unsigned int, ret, depth ) - __field_packed( unsigned int, ret, overrun ) - __field(unsigned long long, calltime ) - __field(unsigned long long, rettime ) + __field_desc_packed( unsigned long, ret, func ) + __field_desc_packed( unsigned long, ret, retval ) + __field_desc_packed( unsigned int, ret, depth ) + __field_desc_packed( unsigned int, ret, overrun ) + __field_packed(unsigned long long, calltime) + __field_packed(unsigned long long, rettime ) ), F_printk("<-- %ps (%u) (start: %llx end: %llx) over: %u retval: %lx", @@ -145,11 +145,11 @@ FTRACE_ENTRY_PACKED(funcgraph_exit, ftrace_graph_ret_entry, F_STRUCT( __field_struct( struct ftrace_graph_ret, ret ) - __field_packed( unsigned long, ret, func ) - __field_packed( unsigned int, ret, depth ) - __field_packed( unsigned int, ret, overrun ) - __field(unsigned long long, calltime ) - __field(unsigned long long, rettime ) + __field_desc_packed( unsigned long, ret, func ) + __field_desc_packed( unsigned int, ret, depth ) + __field_desc_packed( unsigned int, ret, overrun ) + __field_packed(unsigned long long, calltime ) + __field_packed(unsigned long long, rettime ) ), F_printk("<-- %ps (%u) (start: %llx end: %llx) over: %u", diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index e00da4182deb7..b86f128ff99fb 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -700,6 +700,8 @@ int trace_event_reg(struct trace_event_call *call, #ifdef CONFIG_PERF_EVENTS case TRACE_REG_PERF_REGISTER: + if (!call->class->perf_probe) + return -ENODEV; return tracepoint_probe_register(call->tp, call->class->perf_probe, call); @@ -1294,6 +1296,9 @@ static void remove_event_file_dir(struct trace_event_file *file) free_event_filter(file->filter); file->flags |= EVENT_FILE_FL_FREED; event_file_put(file); + + /* Wake up hist poll waiters to notice the EVENT_FILE_FL_FREED flag. */ + hist_poll_wakeup(); } /* @@ -3962,11 +3967,6 @@ void trace_put_event_file(struct trace_event_file *file) EXPORT_SYMBOL_GPL(trace_put_event_file); #ifdef CONFIG_DYNAMIC_FTRACE - -/* Avoid typos */ -#define ENABLE_EVENT_STR "enable_event" -#define DISABLE_EVENT_STR "disable_event" - struct event_probe_data { struct trace_event_file *file; unsigned long count; @@ -4342,7 +4342,11 @@ static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata; static __init int setup_trace_event(char *str) { - strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE); + if (bootup_event_buf[0] != '\0') + strlcat(bootup_event_buf, ",", COMMAND_LINE_SIZE); + + strlcat(bootup_event_buf, str, COMMAND_LINE_SIZE); + trace_set_ring_buffer_expanded(NULL); disable_tracing_selftest("running event tracing"); @@ -4513,26 +4517,22 @@ static __init int event_trace_memsetup(void) return 0; } -__init void -early_enable_events(struct trace_array *tr, char *buf, bool disable_first) +/* + * Helper function to enable or disable a comma-separated list of events + * from the bootup buffer. + */ +static __init void __early_set_events(struct trace_array *tr, char *buf, bool enable) { char *token; - int ret; - - while (true) { - token = strsep(&buf, ","); - - if (!token) - break; + while ((token = strsep(&buf, ","))) { if (*token) { - /* Restarting syscalls requires that we stop them first */ - if (disable_first) + if (enable) { + if (ftrace_set_clr_event(tr, token, 1)) + pr_warn("Failed to enable trace event: %s\n", token); + } else { ftrace_set_clr_event(tr, token, 0); - - ret = ftrace_set_clr_event(tr, token, 1); - if (ret) - pr_warn("Failed to enable trace event: %s\n", token); + } } /* Put back the comma to allow this to be called again */ @@ -4541,6 +4541,32 @@ early_enable_events(struct trace_array *tr, char *buf, bool disable_first) } } +/** + * early_enable_events - enable events from the bootup buffer + * @tr: The trace array to enable the events in + * @buf: The buffer containing the comma separated list of events + * @disable_first: If true, disable all events in @buf before enabling them + * + * This function enables events from the bootup buffer. If @disable_first + * is true, it will first disable all events in the buffer before enabling + * them. + * + * For syscall events, which rely on a global refcount to register the + * SYSCALL_WORK_SYSCALL_TRACEPOINT flag (especially for pid 1), we must + * ensure the refcount hits zero before re-enabling them. A simple + * "disable then enable" per-event is not enough if multiple syscalls are + * used, as the refcount will stay above zero. Thus, we need a two-phase + * approach: disable all, then enable all. + */ +__init void +early_enable_events(struct trace_array *tr, char *buf, bool disable_first) +{ + if (disable_first) + __early_set_events(tr, buf, false); + + __early_set_events(tr, buf, true); +} + static __init int event_trace_enable(void) { struct trace_array *tr = top_trace_array(); diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 6bfaf1210dd24..2a0726e1bc97f 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -2057,6 +2057,15 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING; else hist_field->fn_num = HIST_FIELD_FN_PSTRING; + } else if (field->filter_type == FILTER_STACKTRACE) { + flags |= HIST_FIELD_FL_STACKTRACE; + + hist_field->size = MAX_FILTER_STR_VAL; + hist_field->type = kstrdup_const(field->type, GFP_KERNEL); + if (!hist_field->type) + goto free; + + hist_field->fn_num = HIST_FIELD_FN_STACK; } else { hist_field->size = field->size; hist_field->is_signed = field->is_signed; @@ -5769,7 +5778,7 @@ static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wai guard(mutex)(&event_mutex); - event_file = event_file_data(file); + event_file = event_file_file(file); if (!event_file) return EPOLLERR; @@ -5807,7 +5816,7 @@ static int event_hist_open(struct inode *inode, struct file *file) guard(mutex)(&event_mutex); - event_file = event_file_data(file); + event_file = event_file_file(file); if (!event_file) { ret = -ENODEV; goto err; @@ -6900,7 +6909,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, remove_hist_vars(hist_data); - kfree(trigger_data); + trigger_data_free(trigger_data); destroy_hist_data(hist_data); goto out; diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index f24ee61f8884f..934dafbb009e5 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -130,7 +130,9 @@ static int synth_event_define_fields(struct trace_event_call *call) struct synth_event *event = call->data; unsigned int i, size, n_u64; char *name, *type; + int filter_type; bool is_signed; + bool is_stack; int ret = 0; for (i = 0, n_u64 = 0; i < event->n_fields; i++) { @@ -138,8 +140,12 @@ static int synth_event_define_fields(struct trace_event_call *call) is_signed = event->fields[i]->is_signed; type = event->fields[i]->type; name = event->fields[i]->name; + is_stack = event->fields[i]->is_stack; + + filter_type = is_stack ? FILTER_STACKTRACE : FILTER_OTHER; + ret = trace_define_field(call, type, name, offset, size, - is_signed, FILTER_OTHER); + is_signed, filter_type); if (ret) break; @@ -375,7 +381,6 @@ static enum print_line_t print_synth_event(struct trace_iterator *iter, n_u64++; } else { trace_seq_printf(s, print_fmt, se->fields[i]->name, - STR_VAR_LEN_MAX, (char *)&entry->fields[n_u64].as_u64, i == se->n_fields - 1 ? "" : " "); n_u64 += STR_VAR_LEN_MAX / sizeof(u64); diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index cbfc306c0159a..98b8d5df15c79 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -19,6 +19,9 @@ static DEFINE_MUTEX(trigger_cmd_mutex); void trigger_data_free(struct event_trigger_data *data) { + if (!data) + return; + if (data->cmd_ops->set_filter) data->cmd_ops->set_filter(NULL, data, NULL); diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c index 1698fc22afa0a..32a42ef31855d 100644 --- a/kernel/trace/trace_export.c +++ b/kernel/trace/trace_export.c @@ -42,11 +42,14 @@ static int ftrace_event_register(struct trace_event_call *call, #undef __field_fn #define __field_fn(type, item) type item; +#undef __field_packed +#define __field_packed(type, item) type item; + #undef __field_desc #define __field_desc(type, container, item) type item; -#undef __field_packed -#define __field_packed(type, container, item) type item; +#undef __field_desc_packed +#define __field_desc_packed(type, container, item) type item; #undef __array #define __array(type, item, size) type item[size]; @@ -104,11 +107,14 @@ static void __always_unused ____ftrace_check_##name(void) \ #undef __field_fn #define __field_fn(_type, _item) __field_ext(_type, _item, FILTER_TRACE_FN) +#undef __field_packed +#define __field_packed(_type, _item) __field_ext_packed(_type, _item, FILTER_OTHER) + #undef __field_desc #define __field_desc(_type, _container, _item) __field_ext(_type, _item, FILTER_OTHER) -#undef __field_packed -#define __field_packed(_type, _container, _item) __field_ext_packed(_type, _item, FILTER_OTHER) +#undef __field_desc_packed +#define __field_desc_packed(_type, _container, _item) __field_ext_packed(_type, _item, FILTER_OTHER) #undef __array #define __array(_type, _item, _len) { \ @@ -146,11 +152,14 @@ static struct trace_event_fields ftrace_event_fields_##name[] = { \ #undef __field_fn #define __field_fn(type, item) +#undef __field_packed +#define __field_packed(type, item) + #undef __field_desc #define __field_desc(type, container, item) -#undef __field_packed -#define __field_packed(type, container, item) +#undef __field_desc_packed +#define __field_desc_packed(type, container, item) #undef __array #define __array(type, item, len) diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index a7f4b9a47a71a..0e65d1f452657 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -378,9 +378,14 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs) { + struct trace_array *tr = gops->private; struct fgraph_times *ftimes; + unsigned int trace_ctx; + u64 calltime, rettime; int size; + rettime = trace_clock_local(); + ftrace_graph_addr_finish(gops, trace); if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { @@ -394,11 +399,13 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, handle_nosleeptime(trace, ftimes, size); - if (tracing_thresh && - (trace_clock_local() - ftimes->calltime < tracing_thresh)) + calltime = ftimes->calltime; + + if (tracing_thresh && (rettime - calltime < tracing_thresh)) return; - else - trace_graph_return(trace, gops, fregs); + + trace_ctx = tracing_gen_ctx(); + __trace_graph_return(tr, trace, trace_ctx, calltime, rettime); } static struct fgraph_ops funcgraph_ops = { diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index 2f7b94e98317c..3fe274b84f1c2 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -102,9 +102,9 @@ struct hwlat_sample { /* keep the global state somewhere. */ static struct hwlat_data { - struct mutex lock; /* protect changes */ + struct mutex lock; /* protect changes */ - u64 count; /* total since reset */ + atomic64_t count; /* total since reset */ u64 sample_window; /* total sampling window (on+off) */ u64 sample_width; /* active sampling portion of window */ @@ -193,8 +193,7 @@ void trace_hwlat_callback(bool enter) * get_sample - sample the CPU TSC and look for likely hardware latencies * * Used to repeatedly capture the CPU TSC (or similar), looking for potential - * hardware-induced latency. Called with interrupts disabled and with - * hwlat_data.lock held. + * hardware-induced latency. Called with interrupts disabled. */ static int get_sample(void) { @@ -204,6 +203,7 @@ static int get_sample(void) time_type start, t1, t2, last_t2; s64 diff, outer_diff, total, last_total = 0; u64 sample = 0; + u64 sample_width = READ_ONCE(hwlat_data.sample_width); u64 thresh = tracing_thresh; u64 outer_sample = 0; int ret = -1; @@ -267,7 +267,7 @@ static int get_sample(void) if (diff > sample) sample = diff; /* only want highest value */ - } while (total <= hwlat_data.sample_width); + } while (total <= sample_width); barrier(); /* finish the above in the view for NMIs */ trace_hwlat_callback_enabled = false; @@ -285,8 +285,7 @@ static int get_sample(void) if (kdata->nmi_total_ts) do_div(kdata->nmi_total_ts, NSEC_PER_USEC); - hwlat_data.count++; - s.seqnum = hwlat_data.count; + s.seqnum = atomic64_inc_return(&hwlat_data.count); s.duration = sample; s.outer_duration = outer_sample; s.nmi_total_ts = kdata->nmi_total_ts; @@ -832,7 +831,7 @@ static int hwlat_tracer_init(struct trace_array *tr) hwlat_trace = tr; - hwlat_data.count = 0; + atomic64_set(&hwlat_data.count, 0); tr->max_latency = 0; save_tracing_thresh = tracing_thresh; diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index a9962d4497e82..b93ab382d833c 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -2073,8 +2073,8 @@ static void osnoise_hotplug_workfn(struct work_struct *dummy) if (!osnoise_has_registered_instances()) return; - guard(mutex)(&interface_lock); guard(cpus_read_lock)(); + guard(mutex)(&interface_lock); if (!cpu_online(cpu)) return; @@ -2237,11 +2237,11 @@ static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf, if (running) stop_per_cpu_kthreads(); - mutex_lock(&interface_lock); /* * avoid CPU hotplug operations that might read options. */ cpus_read_lock(); + mutex_lock(&interface_lock); retval = cnt; @@ -2257,8 +2257,8 @@ static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf, clear_bit(option, &osnoise_options); } - cpus_read_unlock(); mutex_unlock(&interface_lock); + cpus_read_unlock(); if (running) start_per_cpu_kthreads(); @@ -2345,16 +2345,16 @@ osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, if (running) stop_per_cpu_kthreads(); - mutex_lock(&interface_lock); /* * osnoise_cpumask is read by CPU hotplug operations. */ cpus_read_lock(); + mutex_lock(&interface_lock); cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new); - cpus_read_unlock(); mutex_unlock(&interface_lock); + cpus_read_unlock(); if (running) start_per_cpu_kthreads(); diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 5cbdc423afebc..d7adbf1536c8b 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -1068,7 +1068,7 @@ static int __parse_imm_string(char *str, char **pbuf, int offs) { size_t len = strlen(str); - if (str[len - 1] != '"') { + if (!len || str[len - 1] != '"') { trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE); return -EINVAL; } diff --git a/kernel/ucount.c b/kernel/ucount.c index 586af49fc03e4..fc4a8f2d30965 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -47,7 +47,7 @@ static int set_permissions(struct ctl_table_header *head, int mode; /* Allow users with CAP_SYS_RESOURCE unrestrained access */ - if (ns_capable(user_ns, CAP_SYS_RESOURCE)) + if (ns_capable_noaudit(user_ns, CAP_SYS_RESOURCE)) mode = (table->mode & S_IRWXU) >> 6; else /* Allow all others at most read-only access */ diff --git a/kernel/unwind/user.c b/kernel/unwind/user.c index 97a8415e3216f..39e2707894447 100644 --- a/kernel/unwind/user.c +++ b/kernel/unwind/user.c @@ -8,18 +8,28 @@ #include #include -static const struct unwind_user_frame fp_frame = { - ARCH_INIT_USER_FP_FRAME -}; - #define for_each_user_frame(state) \ for (unwind_user_start(state); !(state)->done; unwind_user_next(state)) -static int unwind_user_next_fp(struct unwind_user_state *state) +static inline int +get_user_word(unsigned long *word, unsigned long base, int off, unsigned int ws) +{ + unsigned long __user *addr = (void __user *)base + off; +#ifdef CONFIG_COMPAT + if (ws == sizeof(int)) { + unsigned int data; + int ret = get_user(data, (unsigned int __user *)addr); + *word = data; + return ret; + } +#endif + return get_user(*word, addr); +} + +static int unwind_user_next_common(struct unwind_user_state *state, + const struct unwind_user_frame *frame) { - const struct unwind_user_frame *frame = &fp_frame; unsigned long cfa, fp, ra; - unsigned int shift; if (frame->use_fp) { if (state->fp < state->sp) @@ -37,24 +47,45 @@ static int unwind_user_next_fp(struct unwind_user_state *state) return -EINVAL; /* Make sure that the address is word aligned */ - shift = sizeof(long) == 4 ? 2 : 3; - if (cfa & ((1 << shift) - 1)) + if (cfa & (state->ws - 1)) return -EINVAL; /* Find the Return Address (RA) */ - if (get_user(ra, (unsigned long *)(cfa + frame->ra_off))) + if (get_user_word(&ra, cfa, frame->ra_off, state->ws)) return -EINVAL; - if (frame->fp_off && get_user(fp, (unsigned long __user *)(cfa + frame->fp_off))) + if (frame->fp_off && get_user_word(&fp, cfa, frame->fp_off, state->ws)) return -EINVAL; state->ip = ra; state->sp = cfa; if (frame->fp_off) state->fp = fp; + state->topmost = false; return 0; } +static int unwind_user_next_fp(struct unwind_user_state *state) +{ +#ifdef CONFIG_HAVE_UNWIND_USER_FP + struct pt_regs *regs = task_pt_regs(current); + + if (state->topmost && unwind_user_at_function_start(regs)) { + const struct unwind_user_frame fp_entry_frame = { + ARCH_INIT_USER_FP_ENTRY_FRAME(state->ws) + }; + return unwind_user_next_common(state, &fp_entry_frame); + } + + const struct unwind_user_frame fp_frame = { + ARCH_INIT_USER_FP_FRAME(state->ws) + }; + return unwind_user_next_common(state, &fp_frame); +#else + return -EINVAL; +#endif +} + static int unwind_user_next(struct unwind_user_state *state) { unsigned long iter_mask = state->available_types; @@ -102,6 +133,12 @@ static int unwind_user_start(struct unwind_user_state *state) state->ip = instruction_pointer(regs); state->sp = user_stack_pointer(regs); state->fp = frame_pointer(regs); + state->ws = unwind_user_word_size(regs); + if (!state->ws) { + state->done = true; + return -EINVAL; + } + state->topmost = true; return 0; } diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 5b62d10027836..25d134306e146 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -521,7 +521,7 @@ static bool need_counting_irqs(void) u8 util; int tail = __this_cpu_read(cpustat_tail); - tail = (tail + NUM_HARDIRQ_REPORT - 1) % NUM_HARDIRQ_REPORT; + tail = (tail + NUM_SAMPLE_PERIODS - 1) % NUM_SAMPLE_PERIODS; util = __this_cpu_read(cpustat_util[tail][STATS_HARDIRQ]); return util > HARDIRQ_PERCENT_THRESH; } diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 45320e27a16c4..751b8cba52fc6 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -117,6 +117,8 @@ enum wq_internal_consts { MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ CREATE_COOLDOWN = HZ, /* time to breath after fail */ + RESCUER_BATCH = 16, /* process items per turn */ + /* * Rescue workers are used only on emergencies and shared by * all cpus. Give MIN_NICE. @@ -286,6 +288,7 @@ struct pool_workqueue { struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */ struct list_head pwqs_node; /* WR: node on wq->pwqs */ struct list_head mayday_node; /* MD: node on wq->maydays */ + struct work_struct mayday_cursor; /* L: cursor on pool->worklist */ u64 stats[PWQ_NR_STATS]; @@ -1126,6 +1129,12 @@ static struct worker *find_worker_executing_work(struct worker_pool *pool, return NULL; } +static void mayday_cursor_func(struct work_struct *work) +{ + /* should not be processed, only for marking position */ + BUG(); +} + /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled @@ -1188,6 +1197,16 @@ static bool assign_work(struct work_struct *work, struct worker *worker, lockdep_assert_held(&pool->lock); + /* The cursor work should not be processed */ + if (unlikely(work->func == mayday_cursor_func)) { + /* only worker_thread() can possibly take this branch */ + WARN_ON_ONCE(worker->rescue_wq); + if (nextp) + *nextp = list_next_entry(work, entry); + list_del_init(&work->entry); + return false; + } + /* * A single work shouldn't be executed concurrently by multiple workers. * __queue_work() ensures that @work doesn't jump to a different pool @@ -1836,8 +1855,20 @@ static void unplug_oldest_pwq(struct workqueue_struct *wq) raw_spin_lock_irq(&pwq->pool->lock); if (pwq->plugged) { pwq->plugged = false; - if (pwq_activate_first_inactive(pwq, true)) + if (pwq_activate_first_inactive(pwq, true)) { + /* + * While plugged, queueing skips activation which + * includes bumping the nr_active count and adding the + * pwq to nna->pending_pwqs if the count can't be + * obtained. We need to restore both for the pwq being + * unplugged. The first call activates the first + * inactive work item and the second, if there are more + * inactive, puts the pwq on pending_pwqs. + */ + pwq_activate_first_inactive(pwq, false); + kick_pool(pwq->pool); + } } raw_spin_unlock_irq(&pwq->pool->lock); } @@ -3443,6 +3474,35 @@ static int worker_thread(void *__worker) goto woke_up; } +static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer) +{ + struct worker_pool *pool = pwq->pool; + struct work_struct *cursor = &pwq->mayday_cursor; + struct work_struct *work, *n; + + /* need rescue? */ + if (!pwq->nr_active || !need_to_create_worker(pool)) + return false; + + /* search from the start or cursor if available */ + if (list_empty(&cursor->entry)) + work = list_first_entry(&pool->worklist, struct work_struct, entry); + else + work = list_next_entry(cursor, entry); + + /* find the next work item to rescue */ + list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { + if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { + pwq->stats[PWQ_STAT_RESCUED]++; + /* put the cursor for next search */ + list_move_tail(&cursor->entry, &n->entry); + return true; + } + } + + return false; +} + /** * rescuer_thread - the rescuer thread function * @__rescuer: self @@ -3497,7 +3557,7 @@ static int rescuer_thread(void *__rescuer) struct pool_workqueue *pwq = list_first_entry(&wq->maydays, struct pool_workqueue, mayday_node); struct worker_pool *pool = pwq->pool; - struct work_struct *work, *n; + unsigned int count = 0; __set_current_state(TASK_RUNNING); list_del_init(&pwq->mayday_node); @@ -3508,30 +3568,18 @@ static int rescuer_thread(void *__rescuer) raw_spin_lock_irq(&pool->lock); - /* - * Slurp in all works issued via this workqueue and - * process'em. - */ WARN_ON_ONCE(!list_empty(&rescuer->scheduled)); - list_for_each_entry_safe(work, n, &pool->worklist, entry) { - if (get_work_pwq(work) == pwq && - assign_work(work, rescuer, &n)) - pwq->stats[PWQ_STAT_RESCUED]++; - } - if (!list_empty(&rescuer->scheduled)) { + while (assign_rescuer_work(pwq, rescuer)) { process_scheduled_works(rescuer); /* - * The above execution of rescued work items could - * have created more to rescue through - * pwq_activate_first_inactive() or chained - * queueing. Let's put @pwq back on mayday list so - * that such back-to-back work items, which may be - * being used to relieve memory pressure, don't - * incur MAYDAY_INTERVAL delay inbetween. + * If the per-turn work item limit is reached and other + * PWQs are in mayday, requeue mayday for this PWQ and + * let the rescuer handle the other PWQs first. */ - if (pwq->nr_active && need_to_create_worker(pool)) { + if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) && + pwq->nr_active && need_to_create_worker(pool)) { raw_spin_lock(&wq_mayday_lock); /* * Queue iff we aren't racing destruction @@ -3542,9 +3590,14 @@ static int rescuer_thread(void *__rescuer) list_add_tail(&pwq->mayday_node, &wq->maydays); } raw_spin_unlock(&wq_mayday_lock); + break; } } + /* The cursor can not be left behind without the rescuer watching it. */ + if (!list_empty(&pwq->mayday_cursor.entry) && list_empty(&pwq->mayday_node)) + list_del_init(&pwq->mayday_cursor.entry); + /* * Leave this pool. Notify regular workers; otherwise, we end up * with 0 concurrency and stalling the execution. @@ -5163,6 +5216,19 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, INIT_LIST_HEAD(&pwq->pwqs_node); INIT_LIST_HEAD(&pwq->mayday_node); kthread_init_work(&pwq->release_work, pwq_release_workfn); + + /* + * Set the dummy cursor work with valid function and get_work_pwq(). + * + * The cursor work should only be in the pwq->pool->worklist, and + * should not be treated as a processable work item. + * + * WORK_STRUCT_PENDING and WORK_STRUCT_INACTIVE just make it less + * surprise for kernel debugging tools and reviewers. + */ + INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func); + atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq | + WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | WORK_STRUCT_INACTIVE); } /* sync @pwq with the current state of its associated wq and link it */ @@ -6215,7 +6281,7 @@ static void pr_cont_worker_id(struct worker *worker) { struct worker_pool *pool = worker->pool; - if (pool->flags & WQ_BH) + if (pool->flags & POOL_BH) pr_cont("bh%s", pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : ""); else diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 713cc94caa02e..21cd68084e468 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -721,6 +721,7 @@ source "mm/Kconfig.debug" config DEBUG_OBJECTS bool "Debug object operations" + depends on PREEMPT_COUNT || !DEFERRED_STRUCT_PAGE_INIT depends on DEBUG_KERNEL help If you say Y here, additional code will be inserted into the @@ -1684,33 +1685,6 @@ config STACKTRACE It is also used by various kernel debugging features that require stack trace generation. -config WARN_ALL_UNSEEDED_RANDOM - bool "Warn for all uses of unseeded randomness" - default n - help - Some parts of the kernel contain bugs relating to their use of - cryptographically secure random numbers before it's actually possible - to generate those numbers securely. This setting ensures that these - flaws don't go unnoticed, by enabling a message, should this ever - occur. This will allow people with obscure setups to know when things - are going wrong, so that they might contact developers about fixing - it. - - Unfortunately, on some models of some architectures getting - a fully seeded CRNG is extremely difficult, and so this can - result in dmesg getting spammed for a surprisingly long - time. This is really bad from a security perspective, and - so architecture maintainers really need to do what they can - to get the CRNG seeded sooner after the system is booted. - However, since users cannot do anything actionable to - address this, by default this option is disabled. - - Say Y here if you want to receive warnings for all uses of - unseeded randomness. This will be of use primarily for - those developers interested in improving the security of - Linux kernels running on their architecture (or - subarchitecture). - config DEBUG_KOBJECT bool "kobject debugging" depends on DEBUG_KERNEL diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 81f29c29f47b6..5d3802eba52a3 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -316,7 +316,7 @@ int __init xbc_node_compose_key_after(struct xbc_node *root, depth ? "." : ""); if (ret < 0) return ret; - if (ret > size) { + if (ret >= size) { size = 0; } else { size -= ret; @@ -532,9 +532,9 @@ static char *skip_spaces_until_newline(char *p) static int __init __xbc_open_brace(char *p) { /* Push the last key as open brace */ - open_brace[brace_index++] = xbc_node_index(last_parent); if (brace_index >= XBC_DEPTH_MAX) return xbc_parse_error("Exceed max depth of braces", p); + open_brace[brace_index++] = xbc_node_index(last_parent); return 0; } @@ -712,7 +712,8 @@ static int __init xbc_parse_kv(char **k, char *v, int op) if (op == ':') { unsigned short nidx = child->next; - xbc_init_node(child, v, XBC_VALUE); + if (xbc_init_node(child, v, XBC_VALUE) < 0) + return xbc_parse_error("Failed to override value", v); child->next = nidx; /* keep subkeys */ goto array; } @@ -791,7 +792,7 @@ static int __init xbc_verify_tree(void) /* Brace closing */ if (brace_index) { - n = &xbc_nodes[open_brace[brace_index]]; + n = &xbc_nodes[open_brace[brace_index - 1]]; return xbc_parse_error("Brace is not closed", xbc_node_get_data(n)); } diff --git a/lib/buildid.c b/lib/buildid.c index c4b0f376fb341..ef112a7084ef1 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #define BUILD_ID 3 @@ -65,20 +66,9 @@ static int freader_get_folio(struct freader *r, loff_t file_off) freader_put_folio(r); - /* reject secretmem folios created with memfd_secret() */ - if (secretmem_mapping(r->file->f_mapping)) - return -EFAULT; - + /* only use page cache lookup - fail if not already cached */ r->folio = filemap_get_folio(r->file->f_mapping, file_off >> PAGE_SHIFT); - /* if sleeping is allowed, wait for the page, if necessary */ - if (r->may_fault && (IS_ERR(r->folio) || !folio_test_uptodate(r->folio))) { - filemap_invalidate_lock_shared(r->file->f_mapping); - r->folio = read_cache_folio(r->file->f_mapping, file_off >> PAGE_SHIFT, - NULL, r->file); - filemap_invalidate_unlock_shared(r->file->f_mapping); - } - if (IS_ERR(r->folio) || !folio_test_uptodate(r->folio)) { if (!IS_ERR(r->folio)) folio_put(r->folio); @@ -116,6 +106,24 @@ static const void *freader_fetch(struct freader *r, loff_t file_off, size_t sz) return r->data + file_off; } + /* reject secretmem folios created with memfd_secret() */ + if (secretmem_mapping(r->file->f_mapping)) { + r->err = -EFAULT; + return NULL; + } + + /* use __kernel_read() for sleepable context */ + if (r->may_fault) { + ssize_t ret; + + ret = __kernel_read(r->file, r->buf, sz, &file_off); + if (ret != sz) { + r->err = (ret < 0) ? ret : -EIO; + return NULL; + } + return r->buf; + } + /* fetch or reuse folio for given file offset */ r->err = freader_get_folio(r, file_off); if (r->err) @@ -287,7 +295,7 @@ static int get_build_id_64(struct freader *r, unsigned char *build_id, __u32 *si /* enough for Elf64_Ehdr, Elf64_Phdr, and all the smaller requests */ #define MAX_FREADER_BUF_SZ 64 -static int __build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, +static int __build_id_parse(struct file *file, unsigned char *build_id, __u32 *size, bool may_fault) { const Elf32_Ehdr *ehdr; @@ -295,11 +303,7 @@ static int __build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, char buf[MAX_FREADER_BUF_SZ]; int ret; - /* only works for page backed storage */ - if (!vma->vm_file) - return -EINVAL; - - freader_init_from_file(&r, buf, sizeof(buf), vma->vm_file, may_fault); + freader_init_from_file(&r, buf, sizeof(buf), file, may_fault); /* fetch first 18 bytes of ELF header for checks */ ehdr = freader_fetch(&r, 0, offsetofend(Elf32_Ehdr, e_type)); @@ -327,8 +331,8 @@ static int __build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, return ret; } -/* - * Parse build ID of ELF file mapped to vma +/** + * build_id_parse_nofault() - Parse build ID of ELF file mapped to vma * @vma: vma object * @build_id: buffer to store build id, at least BUILD_ID_SIZE long * @size: returns actual build id size in case of success @@ -340,11 +344,14 @@ static int __build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, */ int build_id_parse_nofault(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size) { - return __build_id_parse(vma, build_id, size, false /* !may_fault */); + if (!vma->vm_file) + return -EINVAL; + + return __build_id_parse(vma->vm_file, build_id, size, false /* !may_fault */); } -/* - * Parse build ID of ELF file mapped to VMA +/** + * build_id_parse() - Parse build ID of ELF file mapped to VMA * @vma: vma object * @build_id: buffer to store build id, at least BUILD_ID_SIZE long * @size: returns actual build id size in case of success @@ -356,7 +363,26 @@ int build_id_parse_nofault(struct vm_area_struct *vma, unsigned char *build_id, */ int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size) { - return __build_id_parse(vma, build_id, size, true /* may_fault */); + if (!vma->vm_file) + return -EINVAL; + + return __build_id_parse(vma->vm_file, build_id, size, true /* may_fault */); +} + +/** + * build_id_parse_file() - Parse build ID of ELF file + * @file: file object + * @build_id: buffer to store build id, at least BUILD_ID_SIZE long + * @size: returns actual build id size in case of success + * + * Assumes faultable context and can cause page faults to bring in file data + * into page cache. + * + * Return: 0 on success; negative error, otherwise + */ +int build_id_parse_file(struct file *file, unsigned char *build_id, __u32 *size) +{ + return __build_id_parse(file, build_id, size, true /* may_fault */); } /** diff --git a/lib/crc/.kunitconfig b/lib/crc/.kunitconfig new file mode 100644 index 0000000000000..0a3671ba573f0 --- /dev/null +++ b/lib/crc/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=y +CONFIG_CRC_KUNIT_TEST=y diff --git a/lib/crc/Kconfig b/lib/crc/Kconfig index 70e7a6016de32..cca228879bb5a 100644 --- a/lib/crc/Kconfig +++ b/lib/crc/Kconfig @@ -99,18 +99,27 @@ config CRC_OPTIMIZATIONS config CRC_KUNIT_TEST tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS - depends on KUNIT + depends on KUNIT && (CRC7 || CRC16 || CRC_T10DIF || CRC32 || CRC64) default KUNIT_ALL_TESTS + help + Unit tests for the CRC library functions. + + This is intended to help people writing architecture-specific + optimized versions. If unsure, say N. + +config CRC_ENABLE_ALL_FOR_KUNIT + tristate "Enable all CRC functions for KUnit test" + depends on KUNIT select CRC7 select CRC16 select CRC_T10DIF select CRC32 select CRC64 help - Unit tests for the CRC library functions. + Enable all CRC functions that have test code in CRC_KUNIT_TEST. - This is intended to help people writing architecture-specific - optimized versions. If unsure, say N. + Enable this only if you'd like the CRC KUnit test suite to test all + the CRC variants, even ones that wouldn't otherwise need to be built. config CRC_BENCHMARK bool "Benchmark for the CRC functions" diff --git a/lib/crc/tests/crc_kunit.c b/lib/crc/tests/crc_kunit.c index 9a450e25ac811..9428cd9136253 100644 --- a/lib/crc/tests/crc_kunit.c +++ b/lib/crc/tests/crc_kunit.c @@ -268,8 +268,7 @@ crc_benchmark(struct kunit *test, } } -/* crc7_be */ - +#if IS_REACHABLE(CONFIG_CRC7) static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len) { /* @@ -294,9 +293,9 @@ static void crc7_be_benchmark(struct kunit *test) { crc_benchmark(test, crc7_be_wrapper); } +#endif /* CONFIG_CRC7 */ -/* crc16 */ - +#if IS_REACHABLE(CONFIG_CRC16) static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len) { return crc16(crc, p, len); @@ -318,9 +317,9 @@ static void crc16_benchmark(struct kunit *test) { crc_benchmark(test, crc16_wrapper); } +#endif /* CONFIG_CRC16 */ -/* crc_t10dif */ - +#if IS_REACHABLE(CONFIG_CRC_T10DIF) static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len) { return crc_t10dif_update(crc, p, len); @@ -342,6 +341,9 @@ static void crc_t10dif_benchmark(struct kunit *test) { crc_benchmark(test, crc_t10dif_wrapper); } +#endif /* CONFIG_CRC_T10DIF */ + +#if IS_REACHABLE(CONFIG_CRC32) /* crc32_le */ @@ -414,6 +416,9 @@ static void crc32c_benchmark(struct kunit *test) { crc_benchmark(test, crc32c_wrapper); } +#endif /* CONFIG_CRC32 */ + +#if IS_REACHABLE(CONFIG_CRC64) /* crc64_be */ @@ -463,24 +468,35 @@ static void crc64_nvme_benchmark(struct kunit *test) { crc_benchmark(test, crc64_nvme_wrapper); } +#endif /* CONFIG_CRC64 */ static struct kunit_case crc_test_cases[] = { +#if IS_REACHABLE(CONFIG_CRC7) KUNIT_CASE(crc7_be_test), KUNIT_CASE(crc7_be_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC16) KUNIT_CASE(crc16_test), KUNIT_CASE(crc16_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC_T10DIF) KUNIT_CASE(crc_t10dif_test), KUNIT_CASE(crc_t10dif_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC32) KUNIT_CASE(crc32_le_test), KUNIT_CASE(crc32_le_benchmark), KUNIT_CASE(crc32_be_test), KUNIT_CASE(crc32_be_benchmark), KUNIT_CASE(crc32c_test), KUNIT_CASE(crc32c_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC64) KUNIT_CASE(crc64_be_test), KUNIT_CASE(crc64_be_benchmark), KUNIT_CASE(crc64_nvme_test), KUNIT_CASE(crc64_nvme_benchmark), +#endif {}, }; diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig new file mode 100644 index 0000000000000..2edc7fc23aab7 --- /dev/null +++ b/lib/crypto/.kunitconfig @@ -0,0 +1,11 @@ +CONFIG_KUNIT=y + +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y + +CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_POLY1305_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST=y diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 16859c6226dd4..fa882108ba62c 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -50,7 +50,8 @@ config CRYPTO_LIB_CHACHA_ARCH default y if ARM64 && KERNEL_MODE_NEON default y if MIPS && CPU_MIPS32_R2 default y if PPC64 && CPU_LITTLE_ENDIAN && VSX - default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if S390 default y if X86_64 @@ -161,7 +162,8 @@ config CRYPTO_LIB_SHA256_ARCH default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC && SPE - default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if S390 default y if SPARC64 default y if X86_64 @@ -179,7 +181,8 @@ config CRYPTO_LIB_SHA512_ARCH default y if ARM && !CPU_V7M default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON - default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if S390 default y if SPARC64 default y if X86_64 diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index b57fda3460f1b..102aaa76bc8d7 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -13,7 +13,7 @@ * Emit the sbox as volatile const to prevent the compiler from doing * constant folding on sbox references involving fixed indexes. */ -static volatile const u8 __cacheline_aligned aes_sbox[] = { +static volatile const u8 ____cacheline_aligned aes_sbox[] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, @@ -48,7 +48,7 @@ static volatile const u8 __cacheline_aligned aes_sbox[] = { 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, }; -static volatile const u8 __cacheline_aligned aes_inv_sbox[] = { +static volatile const u8 ____cacheline_aligned aes_inv_sbox[] = { 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, diff --git a/lib/crypto/chacha-block-generic.c b/lib/crypto/chacha-block-generic.c index 77f68de71066a..4a6d627580cb6 100644 --- a/lib/crypto/chacha-block-generic.c +++ b/lib/crypto/chacha-block-generic.c @@ -87,6 +87,8 @@ void chacha_block_generic(struct chacha_state *state, &out[i * sizeof(u32)]); state->x[12]++; + + chacha_zeroize_state(&permuted_state); } EXPORT_SYMBOL(chacha_block_generic); @@ -110,5 +112,7 @@ void hchacha_block_generic(const struct chacha_state *state, memcpy(&out[0], &permuted_state.x[0], 16); memcpy(&out[4], &permuted_state.x[12], 16); + + chacha_zeroize_state(&permuted_state); } EXPORT_SYMBOL(hchacha_block_generic); diff --git a/lib/crypto/riscv/.gitignore b/lib/crypto/riscv/.gitignore new file mode 100644 index 0000000000000..0d47d4f21c6de --- /dev/null +++ b/lib/crypto/riscv/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +poly1305-core.S diff --git a/lib/crypto/riscv/chacha-riscv64-zvkb.S b/lib/crypto/riscv/chacha-riscv64-zvkb.S index b777d0b4e3796..3d183ec818f52 100644 --- a/lib/crypto/riscv/chacha-riscv64-zvkb.S +++ b/lib/crypto/riscv/chacha-riscv64-zvkb.S @@ -60,7 +60,8 @@ #define VL t2 #define STRIDE t3 #define ROUND_CTR t4 -#define KEY0 s0 +#define KEY0 t5 +// Avoid s0/fp to allow for unwinding #define KEY1 s1 #define KEY2 s2 #define KEY3 s3 @@ -143,7 +144,6 @@ // The updated 32-bit counter is written back to state->x[12] before returning. SYM_FUNC_START(chacha_zvkb) addi sp, sp, -96 - sd s0, 0(sp) sd s1, 8(sp) sd s2, 16(sp) sd s3, 24(sp) @@ -280,7 +280,6 @@ SYM_FUNC_START(chacha_zvkb) bnez NBLOCKS, .Lblock_loop sw COUNTER, 48(STATEP) - ld s0, 0(sp) ld s1, 8(sp) ld s2, 16(sp) ld s3, 24(sp) diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 578af717e13a7..a91815ec9b3ad 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -3,47 +3,43 @@ config CRYPTO_LIB_BLAKE2S_KUNIT_TEST tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE - # No need to select CRYPTO_LIB_BLAKE2S here, as that option doesn't + # No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't # exist; the BLAKE2s code is always built-in for the /dev/random driver. help KUnit tests for the BLAKE2s cryptographic hash function. config CRYPTO_LIB_CURVE25519_KUNIT_TEST tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_CURVE25519 + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE - select CRYPTO_LIB_CURVE25519 help KUnit tests for the Curve25519 Diffie-Hellman function. config CRYPTO_LIB_MD5_KUNIT_TEST tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_MD5 + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE - select CRYPTO_LIB_MD5 help KUnit tests for the MD5 cryptographic hash function and its corresponding HMAC. config CRYPTO_LIB_POLY1305_KUNIT_TEST tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_POLY1305 + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE - select CRYPTO_LIB_POLY1305 help KUnit tests for the Poly1305 library functions. config CRYPTO_LIB_SHA1_KUNIT_TEST tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_SHA1 + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE - select CRYPTO_LIB_SHA1 help KUnit tests for the SHA-1 cryptographic hash function and its corresponding HMAC. @@ -52,10 +48,9 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST # included, for consistency with the naming used elsewhere (e.g. CRYPTO_SHA256). config CRYPTO_LIB_SHA256_KUNIT_TEST tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_SHA256 + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE - select CRYPTO_LIB_SHA256 help KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions and their corresponding HMACs. @@ -64,14 +59,31 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST # included, for consistency with the naming used elsewhere (e.g. CRYPTO_SHA512). config CRYPTO_LIB_SHA512_KUNIT_TEST tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_SHA512 + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE - select CRYPTO_LIB_SHA512 help KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions and their corresponding HMACs. +config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT + tristate "Enable all crypto library code for KUnit tests" + depends on KUNIT + select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_MD5 + select CRYPTO_LIB_POLY1305 + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + help + Enable all the crypto library code that has KUnit tests. + + Enable this only if you'd like to test all the crypto library code, + even code that wouldn't otherwise need to be built. + + You'll still need to enable the tests themselves, either individually + or using KUNIT_ALL_TESTS. + config CRYPTO_LIB_BENCHMARK_VISIBLE bool diff --git a/lib/crypto/x86/blake2s-core.S b/lib/crypto/x86/blake2s-core.S index ef8e9f427aab3..093e7814f3879 100644 --- a/lib/crypto/x86/blake2s-core.S +++ b/lib/crypto/x86/blake2s-core.S @@ -52,7 +52,7 @@ SYM_FUNC_START(blake2s_compress_ssse3) movdqa ROT16(%rip),%xmm12 movdqa ROR328(%rip),%xmm13 movdqu 0x20(%rdi),%xmm14 - movq %rcx,%xmm15 + movd %ecx,%xmm15 leaq SIGMA+0xa0(%rip),%r8 jmp .Lbeginofloop .align 32 @@ -176,7 +176,7 @@ SYM_FUNC_START(blake2s_compress_avx512) vmovdqu (%rdi),%xmm0 vmovdqu 0x10(%rdi),%xmm1 vmovdqu 0x20(%rdi),%xmm4 - vmovq %rcx,%xmm5 + vmovd %ecx,%xmm5 vmovdqa IV(%rip),%xmm14 vmovdqa IV+16(%rip),%xmm15 jmp .Lblake2s_compress_avx512_mainloop diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 7f50c4480a4e3..b3151679d0d30 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -398,9 +398,26 @@ static void fill_pool(void) atomic_inc(&cpus_allocating); while (pool_should_refill(&pool_global)) { + gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; HLIST_HEAD(head); - if (!kmem_alloc_batch(&head, obj_cache, __GFP_HIGH | __GFP_NOWARN)) + /* + * Allow reclaim only in preemptible context and during + * early boot. If not preemptible, the caller might hold + * locks causing a deadlock in the allocator. + * + * If the reclaim flag is not set during early boot then + * allocations, which happen before deferred page + * initialization has completed, will fail. + * + * In preemptible context the flag is harmless and not a + * performance issue as that's usually invoked from slow + * path initialization context. + */ + if (preemptible() || system_state < SYSTEM_SCHEDULING) + gfp |= __GFP_KSWAPD_RECLAIM; + + if (!kmem_alloc_batch(&head, obj_cache, gfp)) break; guard(raw_spinlock_irqsave)(&pool_lock); diff --git a/lib/flex_proportions.c b/lib/flex_proportions.c index 84ecccddc7718..012d5614efb9a 100644 --- a/lib/flex_proportions.c +++ b/lib/flex_proportions.c @@ -64,13 +64,14 @@ void fprop_global_destroy(struct fprop_global *p) bool fprop_new_period(struct fprop_global *p, int periods) { s64 events = percpu_counter_sum(&p->events); + unsigned long flags; /* * Don't do anything if there are no events. */ if (events <= 1) return false; - preempt_disable_nested(); + local_irq_save(flags); write_seqcount_begin(&p->sequence); if (periods < 64) events -= events >> periods; @@ -78,7 +79,7 @@ bool fprop_new_period(struct fprop_global *p, int periods) percpu_counter_add(&p->events, -events); p->period += periods; write_seqcount_end(&p->sequence); - preempt_enable_nested(); + local_irq_restore(flags); return true; } diff --git a/lib/idr.c b/lib/idr.c index e2adc457abb4b..457430cff8c5e 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -40,6 +40,8 @@ int idr_alloc_u32(struct idr *idr, void *ptr, u32 *nextid, if (WARN_ON_ONCE(!(idr->idr_rt.xa_flags & ROOT_IS_IDR))) idr->idr_rt.xa_flags |= IDR_RT_MARKER; + if (max < base) + return -ENOSPC; id = (id < base) ? 0 : id - base; radix_tree_iter_init(&iter, id); diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 2fe66a6b8789e..4bc7c19337558 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -265,7 +265,7 @@ static __always_inline size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); + return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len); } size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) @@ -284,7 +284,7 @@ static __always_inline size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_flushcache(to + progress, iter_from, len); + return copy_from_user_flushcache(to + progress, iter_from, len); } static __always_inline diff --git a/lib/kstrtox.c b/lib/kstrtox.c index bdde40cd69d78..97be2a39f5371 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -340,8 +340,8 @@ EXPORT_SYMBOL(kstrtos8); * @s: input string * @res: result * - * This routine returns 0 iff the first character is one of 'YyTt1NnFf0', or - * [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value + * This routine returns 0 iff the first character is one of 'EeYyTt1DdNnFf0', + * or [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value * pointed to by res is updated upon finding a match. */ noinline diff --git a/lib/objpool.c b/lib/objpool.c index b998b720c7329..d98fadf1de169 100644 --- a/lib/objpool.c +++ b/lib/objpool.c @@ -142,7 +142,7 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, pool->gfp = gfp & ~__GFP_ZERO; pool->context = context; pool->release = release; - slot_size = nr_cpu_ids * sizeof(struct objpool_slot); + slot_size = nr_cpu_ids * sizeof(struct objpool_slot *); pool->cpu_slots = kzalloc(slot_size, pool->gfp); if (!pool->cpu_slots) return -ENOMEM; diff --git a/lib/vsprintf.c b/lib/vsprintf.c index eb0cb11d0d126..a356965c1d734 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -1928,9 +1928,6 @@ char *rtc_str(char *buf, char *end, const struct rtc_time *tm, bool found = true; int count = 2; - if (check_pointer(&buf, end, tm, spec)) - return buf; - switch (fmt[count]) { case 'd': have_t = false; @@ -1996,6 +1993,9 @@ static noinline_for_stack char *time_and_date(char *buf, char *end, void *ptr, struct printf_spec spec, const char *fmt) { + if (check_pointer(&buf, end, ptr, spec)) + return buf; + switch (fmt[1]) { case 'R': return rtc_str(buf, end, (const struct rtc_time *)ptr, spec, fmt); diff --git a/mm/Kconfig b/mm/Kconfig index ca3f146bc7053..76001e9ba04f1 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -915,6 +915,9 @@ config HAVE_GIGANTIC_FOLIOS def_bool (HUGETLB_PAGE && ARCH_HAS_GIGANTIC_PAGE) || \ (ZONE_DEVICE && HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) +config ASYNC_KERNEL_PGTABLE_FREE + def_bool n + # TODO: Allow to be enabled without THP config ARCH_SUPPORTS_HUGE_PFNMAP def_bool n @@ -1135,10 +1138,14 @@ config ZONE_DEVICE Device memory hotplug support allows for establishing pmem, or other device driver discovered memory regions, in the memmap. This allows pfn_to_page() lookups of otherwise - "device-physical" addresses which is needed for using a DAX - mapping in an O_DIRECT operation, among other things. - - If FS_DAX is enabled, then say Y. + "device-physical" addresses which is needed for DAX, PCI_P2PDMA, and + DEVICE_PRIVATE features among others. + + Enabling this option will reduce the entropy of x86 KASLR memory + regions. For example - on a 46 bit system, the entropy goes down + from 16 bits to 15 bits. The actual reduction in entropy depends + on the physical address bits, on processor features, kernel config + (5 level page table) and physical memory present on the system. # # Helpers to mirror range of the CPU page tables of a process into device page diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 41b6c9386b692..014252f948a76 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -618,12 +618,13 @@ static void cgwb_release_workfn(struct work_struct *work) wb_shutdown(wb); css_put(wb->memcg_css); - css_put(wb->blkcg_css); - mutex_unlock(&wb->bdi->cgwb_release_mutex); /* triggers blkg destruction if no online users left */ blkcg_unpin_online(wb->blkcg_css); + css_put(wb->blkcg_css); + mutex_unlock(&wb->bdi->cgwb_release_mutex); + fprop_local_destroy_percpu(&wb->memcg_completions); spin_lock_irq(&cgwb_lock); diff --git a/mm/damon/core.c b/mm/damon/core.c index 109b050c795ad..87b6c9c2d6471 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1236,6 +1236,10 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) { int err; + dst->maybe_corrupted = true; + if (!is_power_of_2(src->min_sz_region)) + return -EINVAL; + err = damon_commit_schemes(dst, src); if (err) return err; @@ -1258,6 +1262,7 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) dst->addr_unit = src->addr_unit; dst->min_sz_region = src->min_sz_region; + dst->maybe_corrupted = false; return 0; } @@ -1426,6 +1431,35 @@ bool damon_is_running(struct damon_ctx *ctx) return running; } +/* + * damon_call_handle_inactive_ctx() - handle DAMON call request that added to + * an inactive context. + * @ctx: The inactive DAMON context. + * @control: Control variable of the call request. + * + * This function is called in a case that @control is added to @ctx but @ctx is + * not running (inactive). See if @ctx handled @control or not, and cleanup + * @control if it was not handled. + * + * Returns 0 if @control was handled by @ctx, negative error code otherwise. + */ +static int damon_call_handle_inactive_ctx( + struct damon_ctx *ctx, struct damon_call_control *control) +{ + struct damon_call_control *c; + + mutex_lock(&ctx->call_controls_lock); + list_for_each_entry(c, &ctx->call_controls, list) { + if (c == control) { + list_del(&control->list); + mutex_unlock(&ctx->call_controls_lock); + return -EINVAL; + } + } + mutex_unlock(&ctx->call_controls_lock); + return 0; +} + /** * damon_call() - Invoke a given function on DAMON worker thread (kdamond). * @ctx: DAMON context to call the function for. @@ -1456,7 +1490,7 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) list_add_tail(&control->list, &ctx->call_controls); mutex_unlock(&ctx->call_controls_lock); if (!damon_is_running(ctx)) - return -EINVAL; + return damon_call_handle_inactive_ctx(ctx, control); if (control->repeat) return 0; wait_for_completion(&control->completion); @@ -1497,8 +1531,13 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) } ctx->walk_control = control; mutex_unlock(&ctx->walk_control_lock); - if (!damon_is_running(ctx)) + if (!damon_is_running(ctx)) { + mutex_lock(&ctx->walk_control_lock); + if (ctx->walk_control == control) + ctx->walk_control = NULL; + mutex_unlock(&ctx->walk_control_lock); return -EINVAL; + } wait_for_completion(&control->completion); if (control->canceled) return -ECANCELED; @@ -2525,6 +2564,8 @@ static void kdamond_call(struct damon_ctx *ctx, bool cancel) } else { list_add(&control->list, &repeat_controls); } + if (!cancel && ctx->maybe_corrupted) + break; } control = list_first_entry_or_null(&repeat_controls, struct damon_call_control, list); @@ -2557,6 +2598,8 @@ static int kdamond_wait_activation(struct damon_ctx *ctx) kdamond_usleep(min_wait_time); kdamond_call(ctx, false); + if (ctx->maybe_corrupted) + return -EINVAL; damos_walk_cancel(ctx); } return -EBUSY; @@ -2642,6 +2685,8 @@ static int kdamond_fn(void *data) * kdamond_merge_regions() if possible, to reduce overhead */ kdamond_call(ctx, false); + if (ctx->maybe_corrupted) + break; if (!list_empty(&ctx->schemes)) kdamond_apply_schemes(ctx); else @@ -2704,13 +2749,13 @@ static int kdamond_fn(void *data) if (ctx->ops.cleanup) ctx->ops.cleanup(ctx); kfree(ctx->regions_score_histogram); + kdamond_call(ctx, true); pr_debug("kdamond (%d) finishes\n", current->pid); mutex_lock(&ctx->kdamond_lock); ctx->kdamond = NULL; mutex_unlock(&ctx->kdamond_lock); - kdamond_call(ctx, true); damos_walk_cancel(ctx); mutex_lock(&damon_lock); diff --git a/mm/damon/stat.c b/mm/damon/stat.c index bf8626859902f..3a55f6e41dc4c 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -145,12 +145,44 @@ static int damon_stat_damon_call_fn(void *data) return 0; } +struct damon_stat_system_ram_range_walk_arg { + bool walked; + struct resource res; +}; + +static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg) +{ + struct damon_stat_system_ram_range_walk_arg *a = arg; + + if (!a->walked) { + a->walked = true; + a->res.start = res->start; + } + a->res.end = res->end; + return 0; +} + +static int damon_stat_set_monitoring_region(struct damon_target *t, + unsigned long addr_unit) +{ + struct damon_addr_range addr_range; + struct damon_stat_system_ram_range_walk_arg arg = {}; + + walk_system_ram_res(0, ULONG_MAX, &arg, damon_stat_system_ram_walk_fn); + if (!arg.walked) + return -EINVAL; + addr_range.start = arg.res.start; + addr_range.end = arg.res.end + 1; + if (addr_range.end <= addr_range.start) + return -EINVAL; + return damon_set_regions(t, &addr_range, 1, DAMON_MIN_REGION); +} + static struct damon_ctx *damon_stat_build_ctx(void) { struct damon_ctx *ctx; struct damon_attrs attrs; struct damon_target *target; - unsigned long start = 0, end = 0; ctx = damon_new_ctx(); if (!ctx) @@ -188,7 +220,7 @@ static struct damon_ctx *damon_stat_build_ctx(void) if (!target) goto free_out; damon_add_target(ctx, target); - if (damon_set_region_biggest_system_ram_default(target, &start, &end)) + if (damon_stat_set_monitoring_region(target, ctx->addr_unit)) goto free_out; return ctx; free_out: @@ -205,6 +237,12 @@ static int damon_stat_start(void) { int err; + if (damon_stat_context) { + if (damon_is_running(damon_stat_context)) + return -EAGAIN; + damon_destroy_ctx(damon_stat_context); + } + damon_stat_context = damon_stat_build_ctx(); if (!damon_stat_context) return -ENOMEM; @@ -221,6 +259,7 @@ static void damon_stat_stop(void) { damon_stop(&damon_stat_context, 1); damon_destroy_ctx(damon_stat_context); + damon_stat_context = NULL; } static int damon_stat_enabled_store( diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 6536f16006c90..50d000d61c907 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2111,13 +2111,13 @@ static int damon_sysfs_scheme_add_dirs(struct damon_sysfs_scheme *scheme) return err; err = damos_sysfs_set_dests(scheme); if (err) - goto put_access_pattern_out; + goto rmdir_put_access_pattern_out; err = damon_sysfs_scheme_set_quotas(scheme); if (err) goto put_dests_out; err = damon_sysfs_scheme_set_watermarks(scheme); if (err) - goto put_quotas_access_pattern_out; + goto rmdir_put_quotas_access_pattern_out; err = damos_sysfs_set_filter_dirs(scheme); if (err) goto put_watermarks_quotas_access_pattern_out; @@ -2142,13 +2142,15 @@ static int damon_sysfs_scheme_add_dirs(struct damon_sysfs_scheme *scheme) put_watermarks_quotas_access_pattern_out: kobject_put(&scheme->watermarks->kobj); scheme->watermarks = NULL; -put_quotas_access_pattern_out: +rmdir_put_quotas_access_pattern_out: + damon_sysfs_quotas_rm_dirs(scheme->quotas); kobject_put(&scheme->quotas->kobj); scheme->quotas = NULL; put_dests_out: kobject_put(&scheme->dests->kobj); scheme->dests = NULL; -put_access_pattern_out: +rmdir_put_access_pattern_out: + damon_sysfs_access_pattern_rm_dirs(scheme->access_pattern); kobject_put(&scheme->access_pattern->kobj); scheme->access_pattern = NULL; return err; diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 3c0d727788c87..a6c3e83a3dad5 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -764,7 +764,7 @@ static int damon_sysfs_attrs_add_dirs(struct damon_sysfs_attrs *attrs) nr_regions_range = damon_sysfs_ul_range_alloc(10, 1000); if (!nr_regions_range) { err = -ENOMEM; - goto put_intervals_out; + goto rmdir_put_intervals_out; } err = kobject_init_and_add(&nr_regions_range->kobj, @@ -778,6 +778,8 @@ static int damon_sysfs_attrs_add_dirs(struct damon_sysfs_attrs *attrs) put_nr_regions_intervals_out: kobject_put(&nr_regions_range->kobj); attrs->nr_regions_range = NULL; +rmdir_put_intervals_out: + damon_sysfs_intervals_rm_dirs(intervals); put_intervals_out: kobject_put(&intervals->kobj); attrs->intervals = NULL; @@ -920,7 +922,7 @@ static int damon_sysfs_context_add_dirs(struct damon_sysfs_context *context) err = damon_sysfs_context_set_targets(context); if (err) - goto put_attrs_out; + goto rmdir_put_attrs_out; err = damon_sysfs_context_set_schemes(context); if (err) @@ -930,7 +932,8 @@ static int damon_sysfs_context_add_dirs(struct damon_sysfs_context *context) put_targets_attrs_out: kobject_put(&context->targets->kobj); context->targets = NULL; -put_attrs_out: +rmdir_put_attrs_out: + damon_sysfs_attrs_rm_dirs(context->attrs); kobject_put(&context->attrs->kobj); context->attrs = NULL; return err; @@ -1264,7 +1267,7 @@ enum damon_sysfs_cmd { DAMON_SYSFS_CMD_UPDATE_SCHEMES_EFFECTIVE_QUOTAS, /* * @DAMON_SYSFS_CMD_UPDATE_TUNED_INTERVALS: Update the tuned monitoring - * intevals. + * intervals. */ DAMON_SYSFS_CMD_UPDATE_TUNED_INTERVALS, /* @@ -1473,8 +1476,10 @@ static int damon_sysfs_commit_input(void *data) if (IS_ERR(param_ctx)) return PTR_ERR(param_ctx); test_ctx = damon_new_ctx(); - if (!test_ctx) + if (!test_ctx) { + damon_destroy_ctx(param_ctx); return -ENOMEM; + } err = damon_commit_ctx(test_ctx, param_ctx); if (err) goto out; @@ -1567,9 +1572,12 @@ static int damon_sysfs_repeat_call_fn(void *data) if (!mutex_trylock(&damon_sysfs_lock)) return 0; + if (sysfs_kdamond->contexts->nr != 1) + goto out; damon_sysfs_upd_tuned_intervals(sysfs_kdamond); damon_sysfs_upd_schemes_stats(sysfs_kdamond); damon_sysfs_upd_schemes_effective_quotas(sysfs_kdamond); +out: mutex_unlock(&damon_sysfs_lock); return 0; } @@ -1615,7 +1623,8 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) repeat_call_control->data = kdamond; repeat_call_control->repeat = true; repeat_call_control->dealloc_on_cancel = true; - damon_call(ctx, repeat_call_control); + if (damon_call(ctx, repeat_call_control)) + kfree(repeat_call_control); return err; } @@ -1697,6 +1706,9 @@ static int damon_sysfs_update_schemes_tried_regions( static int damon_sysfs_handle_cmd(enum damon_sysfs_cmd cmd, struct damon_sysfs_kdamond *kdamond) { + if (cmd != DAMON_SYSFS_CMD_OFF && kdamond->contexts->nr != 1) + return -EINVAL; + switch (cmd) { case DAMON_SYSFS_CMD_ON: return damon_sysfs_turn_damon_on(kdamond); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 51369e35298b1..6687778752728 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -20,11 +20,17 @@ static void damon_test_regions(struct kunit *test) struct damon_target *t; r = damon_new_region(1, 2); + if (!r) + kunit_skip(test, "region alloc fail"); KUNIT_EXPECT_EQ(test, 1ul, r->ar.start); KUNIT_EXPECT_EQ(test, 2ul, r->ar.end); KUNIT_EXPECT_EQ(test, 0u, r->nr_accesses); t = damon_new_target(); + if (!t) { + damon_free_region(r); + kunit_skip(test, "target alloc fail"); + } KUNIT_EXPECT_EQ(test, 0u, damon_nr_regions(t)); damon_add_region(r, t); @@ -52,7 +58,14 @@ static void damon_test_target(struct kunit *test) struct damon_ctx *c = damon_new_ctx(); struct damon_target *t; + if (!c) + kunit_skip(test, "ctx alloc fail"); + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } KUNIT_EXPECT_EQ(test, 0u, nr_damon_targets(c)); damon_add_target(c, t); @@ -84,8 +97,15 @@ static void damon_test_aggregate(struct kunit *test) struct damon_region *r; int it, ir; + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + for (it = 0; it < 3; it++) { t = damon_new_target(); + if (!t) { + damon_destroy_ctx(ctx); + kunit_skip(test, "target alloc fail"); + } damon_add_target(ctx, t); } @@ -93,6 +113,10 @@ static void damon_test_aggregate(struct kunit *test) damon_for_each_target(t, ctx) { for (ir = 0; ir < 3; ir++) { r = damon_new_region(saddr[it][ir], eaddr[it][ir]); + if (!r) { + damon_destroy_ctx(ctx); + kunit_skip(test, "region alloc fail"); + } r->nr_accesses = accesses[it][ir]; r->nr_accesses_bp = accesses[it][ir] * 10000; damon_add_region(r, t); @@ -124,8 +148,19 @@ static void damon_test_split_at(struct kunit *test) struct damon_target *t; struct damon_region *r, *r_new; + if (!c) + kunit_skip(test, "ctx alloc fail"); t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } r = damon_new_region(0, 100); + if (!r) { + damon_destroy_ctx(c); + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } r->nr_accesses_bp = 420000; r->nr_accesses = 42; r->last_nr_accesses = 15; @@ -153,11 +188,21 @@ static void damon_test_merge_two(struct kunit *test) int i; t = damon_new_target(); + if (!t) + kunit_skip(test, "target alloc fail"); r = damon_new_region(0, 100); + if (!r) { + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } r->nr_accesses = 10; r->nr_accesses_bp = 100000; damon_add_region(r, t); r2 = damon_new_region(100, 300); + if (!r2) { + damon_free_target(t); + kunit_skip(test, "second region alloc fail"); + } r2->nr_accesses = 20; r2->nr_accesses_bp = 200000; damon_add_region(r2, t); @@ -203,8 +248,14 @@ static void damon_test_merge_regions_of(struct kunit *test) int i; t = damon_new_target(); + if (!t) + kunit_skip(test, "target alloc fail"); for (i = 0; i < ARRAY_SIZE(sa); i++) { r = damon_new_region(sa[i], ea[i]); + if (!r) { + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } r->nr_accesses = nrs[i]; r->nr_accesses_bp = nrs[i] * 10000; damon_add_region(r, t); @@ -227,15 +278,35 @@ static void damon_test_split_regions_of(struct kunit *test) struct damon_target *t; struct damon_region *r; + if (!c) + kunit_skip(test, "ctx alloc fail"); t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } r = damon_new_region(0, 22); + if (!r) { + damon_destroy_ctx(c); + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } damon_add_region(r, t); damon_split_regions_of(t, 2, DAMON_MIN_REGION); KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u); damon_free_target(t); t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "second target alloc fail"); + } r = damon_new_region(0, 220); + if (!r) { + damon_destroy_ctx(c); + damon_free_target(t); + kunit_skip(test, "second region alloc fail"); + } damon_add_region(r, t); damon_split_regions_of(t, 4, DAMON_MIN_REGION); KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u); @@ -249,6 +320,9 @@ static void damon_test_ops_registration(struct kunit *test) struct damon_operations ops = {.id = DAMON_OPS_VADDR}, bak; bool need_cleanup = false; + if (!c) + kunit_skip(test, "ctx alloc fail"); + /* DAMON_OPS_VADDR is registered only if CONFIG_DAMON_VADDR is set */ if (!damon_is_registered_ops(DAMON_OPS_VADDR)) { bak.id = DAMON_OPS_VADDR; @@ -294,13 +368,26 @@ static void damon_test_ops_registration(struct kunit *test) static void damon_test_set_regions(struct kunit *test) { struct damon_target *t = damon_new_target(); - struct damon_region *r1 = damon_new_region(4, 16); - struct damon_region *r2 = damon_new_region(24, 32); + struct damon_region *r1, *r2; struct damon_addr_range range = {.start = 8, .end = 28}; unsigned long expects[] = {8, 16, 16, 24, 24, 28}; int expect_idx = 0; struct damon_region *r; + if (!t) + kunit_skip(test, "target alloc fail"); + r1 = damon_new_region(4, 16); + if (!r1) { + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } + r2 = damon_new_region(24, 32); + if (!r2) { + damon_free_target(t); + damon_free_region(r1); + kunit_skip(test, "second region alloc fail"); + } + damon_add_region(r1, t); damon_add_region(r2, t); damon_set_regions(t, &range, 1, DAMON_MIN_REGION); @@ -342,6 +429,9 @@ static void damon_test_update_monitoring_result(struct kunit *test) struct damon_attrs new_attrs; struct damon_region *r = damon_new_region(3, 7); + if (!r) + kunit_skip(test, "region alloc fail"); + r->nr_accesses = 15; r->nr_accesses_bp = 150000; r->age = 20; @@ -375,6 +465,9 @@ static void damon_test_set_attrs(struct kunit *test) .sample_interval = 5000, .aggr_interval = 100000,}; struct damon_attrs invalid_attrs; + if (!c) + kunit_skip(test, "ctx alloc fail"); + KUNIT_EXPECT_EQ(test, damon_set_attrs(c, &valid_attrs), 0); invalid_attrs = valid_attrs; @@ -412,6 +505,8 @@ static void damos_test_new_filter(struct kunit *test) struct damos_filter *filter; filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true, false); + if (!filter) + kunit_skip(test, "filter alloc fail"); KUNIT_EXPECT_EQ(test, filter->type, DAMOS_FILTER_TYPE_ANON); KUNIT_EXPECT_EQ(test, filter->matching, true); KUNIT_EXPECT_PTR_EQ(test, filter->list.prev, &filter->list); @@ -421,11 +516,16 @@ static void damos_test_new_filter(struct kunit *test) static void damos_test_commit_filter(struct kunit *test) { - struct damos_filter *src_filter = damos_new_filter( - DAMOS_FILTER_TYPE_ANON, true, true); - struct damos_filter *dst_filter = damos_new_filter( - DAMOS_FILTER_TYPE_ACTIVE, false, false); - + struct damos_filter *src_filter, *dst_filter; + + src_filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true, true); + if (!src_filter) + kunit_skip(test, "src filter alloc fail"); + dst_filter = damos_new_filter(DAMOS_FILTER_TYPE_ACTIVE, false, false); + if (!dst_filter) { + damos_destroy_filter(src_filter); + kunit_skip(test, "dst filter alloc fail"); + } damos_commit_filter(dst_filter, src_filter); KUNIT_EXPECT_EQ(test, dst_filter->type, src_filter->type); KUNIT_EXPECT_EQ(test, dst_filter->matching, src_filter->matching); @@ -442,11 +542,22 @@ static void damos_test_filter_out(struct kunit *test) struct damos_filter *f; f = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false); + if (!f) + kunit_skip(test, "filter alloc fail"); f->addr_range = (struct damon_addr_range){ .start = DAMON_MIN_REGION * 2, .end = DAMON_MIN_REGION * 6}; t = damon_new_target(); + if (!t) { + damos_destroy_filter(f); + kunit_skip(test, "target alloc fail"); + } r = damon_new_region(DAMON_MIN_REGION * 3, DAMON_MIN_REGION * 5); + if (!r) { + damos_destroy_filter(f); + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } damon_add_region(r, t); /* region in the range */ @@ -548,6 +659,8 @@ static void damon_test_set_filters_default_reject(struct kunit *test) KUNIT_EXPECT_EQ(test, scheme.ops_filters_default_reject, false); target_filter = damos_new_filter(DAMOS_FILTER_TYPE_TARGET, true, true); + if (!target_filter) + kunit_skip(test, "filter alloc fail"); damos_add_filter(&scheme, target_filter); damos_set_filters_default_reject(&scheme); /* @@ -573,6 +686,10 @@ static void damon_test_set_filters_default_reject(struct kunit *test) KUNIT_EXPECT_EQ(test, scheme.ops_filters_default_reject, false); anon_filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true, true); + if (!anon_filter) { + damos_free_filter(target_filter); + kunit_skip(test, "anon_filter alloc fail"); + } damos_add_filter(&scheme, anon_filter); damos_set_filters_default_reject(&scheme); @@ -598,6 +715,9 @@ static void damon_test_set_filters_default_reject(struct kunit *test) */ KUNIT_EXPECT_EQ(test, scheme.core_filters_default_reject, false); KUNIT_EXPECT_EQ(test, scheme.ops_filters_default_reject, true); + + damos_free_filter(anon_filter); + damos_free_filter(target_filter); } static struct kunit_case damon_test_cases[] = { diff --git a/mm/damon/tests/sysfs-kunit.h b/mm/damon/tests/sysfs-kunit.h index 7b5c7b307da99..ce7218469f20b 100644 --- a/mm/damon/tests/sysfs-kunit.h +++ b/mm/damon/tests/sysfs-kunit.h @@ -45,16 +45,41 @@ static void damon_sysfs_test_add_targets(struct kunit *test) struct damon_ctx *ctx; sysfs_targets = damon_sysfs_targets_alloc(); + if (!sysfs_targets) + kunit_skip(test, "sysfs_targets alloc fail"); sysfs_targets->nr = 1; sysfs_targets->targets_arr = kmalloc_array(1, sizeof(*sysfs_targets->targets_arr), GFP_KERNEL); + if (!sysfs_targets->targets_arr) { + kfree(sysfs_targets); + kunit_skip(test, "targets_arr alloc fail"); + } sysfs_target = damon_sysfs_target_alloc(); + if (!sysfs_target) { + kfree(sysfs_targets->targets_arr); + kfree(sysfs_targets); + kunit_skip(test, "sysfs_target alloc fail"); + } sysfs_target->pid = __damon_sysfs_test_get_any_pid(12, 100); sysfs_target->regions = damon_sysfs_regions_alloc(); + if (!sysfs_target->regions) { + kfree(sysfs_targets->targets_arr); + kfree(sysfs_targets); + kfree(sysfs_target); + kunit_skip(test, "sysfs_regions alloc fail"); + } + sysfs_targets->targets_arr[0] = sysfs_target; ctx = damon_new_ctx(); + if (!ctx) { + kfree(sysfs_targets->targets_arr); + kfree(sysfs_targets); + kfree(sysfs_target); + kfree(sysfs_target->regions); + kunit_skip(test, "ctx alloc fail"); + } damon_sysfs_add_targets(ctx, sysfs_targets); KUNIT_EXPECT_EQ(test, 1u, nr_damon_targets(ctx)); diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index fce38dd53cf8f..30dc5459f1d2c 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -136,8 +136,14 @@ static void damon_do_test_apply_three_regions(struct kunit *test, int i; t = damon_new_target(); + if (!t) + kunit_skip(test, "target alloc fail"); for (i = 0; i < nr_regions / 2; i++) { r = damon_new_region(regions[i * 2], regions[i * 2 + 1]); + if (!r) { + damon_destroy_target(t, NULL); + kunit_skip(test, "region alloc fail"); + } damon_add_region(r, t); } @@ -250,7 +256,16 @@ static void damon_test_split_evenly_fail(struct kunit *test, unsigned long start, unsigned long end, unsigned int nr_pieces) { struct damon_target *t = damon_new_target(); - struct damon_region *r = damon_new_region(start, end); + struct damon_region *r; + + if (!t) + kunit_skip(test, "target alloc fail"); + + r = damon_new_region(start, end); + if (!r) { + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } damon_add_region(r, t); KUNIT_EXPECT_EQ(test, @@ -269,10 +284,17 @@ static void damon_test_split_evenly_succ(struct kunit *test, unsigned long start, unsigned long end, unsigned int nr_pieces) { struct damon_target *t = damon_new_target(); - struct damon_region *r = damon_new_region(start, end); + struct damon_region *r; unsigned long expected_width = (end - start) / nr_pieces; unsigned long i = 0; + if (!t) + kunit_skip(test, "target alloc fail"); + r = damon_new_region(start, end); + if (!r) { + damon_free_target(t); + kunit_skip(test, "region alloc fail"); + } damon_add_region(r, t); KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, nr_pieces), 0); diff --git a/mm/filemap.c b/mm/filemap.c index 024b71da5224d..91dcfe14a67b7 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1386,14 +1386,16 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, #ifdef CONFIG_MIGRATION /** - * migration_entry_wait_on_locked - Wait for a migration entry to be removed - * @entry: migration swap entry. + * migration_entry_wait_on_locked - Wait for a migration entry or + * device_private entry to be removed. + * @entry: migration or device_private swap entry. * @ptl: already locked ptl. This function will drop the lock. * - * Wait for a migration entry referencing the given page to be removed. This is + * Wait for a migration entry referencing the given page, or device_private + * entry referencing a dvice_private page to be unlocked. This is * equivalent to folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE) except * this can be called without taking a reference on the page. Instead this - * should be called while holding the ptl for the migration entry referencing + * should be called while holding the ptl for @entry referencing * the page. * * Returns after unlocking the ptl. @@ -1435,6 +1437,9 @@ void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl) * If a migration entry exists for the page the migration path must hold * a valid reference to the page, and it must take the ptl to remove the * migration entry. So the page is valid until the ptl is dropped. + * Similarly any path attempting to drop the last reference to a + * device-private page needs to grab the ptl to remove the device-private + * entry. */ spin_unlock(ptl); @@ -3818,14 +3823,19 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned int nr_pages = 0, folio_type; unsigned short mmap_miss = 0, mmap_miss_saved; + /* + * Recalculate end_pgoff based on file_end before calling + * next_uptodate_folio() to avoid races with concurrent + * truncation. + */ + file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE) - 1; + end_pgoff = min(end_pgoff, file_end); + rcu_read_lock(); folio = next_uptodate_folio(&xas, mapping, end_pgoff); if (!folio) goto out; - file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE) - 1; - end_pgoff = min(end_pgoff, file_end); - /* * Do not allow to map with PMD across i_size to preserve * SIGBUS semantics. diff --git a/mm/gup.c b/mm/gup.c index a8ba5112e4d09..95d948c8e86c9 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -950,7 +950,7 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, struct mm_struct *mm = vma->vm_mm; pudp = pud_offset(p4dp, address); - pud = READ_ONCE(*pudp); + pud = pudp_get(pudp); if (!pud_present(pud)) return no_page_table(vma, flags, address); if (pud_leaf(pud)) { @@ -975,7 +975,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, p4d_t *p4dp, p4d; p4dp = p4d_offset(pgdp, address); - p4d = READ_ONCE(*p4dp); + p4d = p4dp_get(p4dp); BUILD_BUG_ON(p4d_leaf(p4d)); if (!p4d_present(p4d) || p4d_bad(p4d)) @@ -2710,7 +2710,7 @@ EXPORT_SYMBOL(get_user_pages_unlocked); * * *) ptes can be read atomically by the architecture. * - * *) valid user addesses are below TASK_MAX_SIZE + * *) valid user addresses are below TASK_MAX_SIZE * * The last two assumptions can be relaxed by the addition of helper functions. * @@ -3060,7 +3060,7 @@ static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, pudp = pud_offset_lockless(p4dp, p4d, addr); do { - pud_t pud = READ_ONCE(*pudp); + pud_t pud = pudp_get(pudp); next = pud_addr_end(addr, end); if (unlikely(!pud_present(pud))) @@ -3086,7 +3086,7 @@ static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, p4dp = p4d_offset_lockless(pgdp, pgd, addr); do { - p4d_t p4d = READ_ONCE(*p4dp); + p4d_t p4d = p4dp_get(p4dp); next = p4d_addr_end(addr, end); if (!p4d_present(p4d)) @@ -3108,7 +3108,7 @@ static void gup_fast_pgd_range(unsigned long addr, unsigned long end, pgdp = pgd_offset(current->mm, addr); do { - pgd_t pgd = READ_ONCE(*pgdp); + pgd_t pgd = pgdp_get(pgdp); next = pgd_addr_end(addr, end); if (pgd_none(pgd)) diff --git a/mm/highmem.c b/mm/highmem.c index b5c8e4c2d5d49..a33e411839517 100644 --- a/mm/highmem.c +++ b/mm/highmem.c @@ -180,12 +180,13 @@ struct page *__kmap_to_page(void *vaddr) for (i = 0; i < kctrl->idx; i++) { unsigned long base_addr; int idx; + pte_t pteval = kctrl->pteval[i]; idx = arch_kmap_local_map_idx(i, pte_pfn(pteval)); base_addr = __fix_to_virt(FIX_KMAP_BEGIN + idx); if (base_addr == base) - return pte_page(kctrl->pteval[i]); + return pte_page(pteval); } } diff --git a/mm/hmm.c b/mm/hmm.c index 87562914670a1..a56081d67ad69 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -491,7 +491,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, /* Normally we don't want to split the huge page */ walk->action = ACTION_CONTINUE; - pud = READ_ONCE(*pudp); + pud = pudp_get(pudp); if (!pud_present(pud)) { spin_unlock(ptl); return hmm_vma_walk_hole(start, end, -1, walk); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 6cba1cb14b23a..8218e9d1887b7 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -94,6 +94,9 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) inode = file_inode(vma->vm_file); + if (IS_ANON_FILE(inode)) + return false; + return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); } @@ -1233,6 +1236,7 @@ static void map_anon_folio_pmd(struct folio *folio, pmd_t *pmd, count_vm_event(THP_FAULT_ALLOC); count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); + deferred_split_folio(folio, false); } static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) @@ -1273,7 +1277,6 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); mm_inc_nr_ptes(vma->vm_mm); - deferred_split_folio(folio, false); spin_unlock(vmf->ptl); } @@ -2636,7 +2639,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); } else { src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); - _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); + _dst_pmd = move_soft_dirty_pmd(src_pmdval); + _dst_pmd = clear_uffd_wp_pmd(_dst_pmd); } set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); @@ -3434,6 +3438,7 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, { int order = folio_order(folio); int start_order = uniform_split ? new_order : order - 1; + struct folio *old_folio = folio; bool stop_split = false; struct folio *next; int split_order; @@ -3464,12 +3469,16 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, * uniform split has xas_split_alloc() called before * irq is disabled to allocate enough memory, whereas * non-uniform split can handle ENOMEM. + * Use the to-be-split folio, so that a parallel + * folio_try_get() waits on it until xarray is updated + * with after-split folios and the original one is + * unfrozen. */ if (uniform_split) - xas_split(xas, folio, old_order); + xas_split(xas, old_folio, old_order); else { xas_set_order(xas, folio->index, split_order); - xas_try_split(xas, folio, old_order); + xas_try_split(xas, old_folio, old_order); if (xas_error(xas)) { ret = xas_error(xas); stop_split = true; @@ -3515,8 +3524,8 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, return ret; } -bool non_uniform_split_supported(struct folio *folio, unsigned int new_order, - bool warns) +bool folio_split_supported(struct folio *folio, unsigned int new_order, + bool uniform_split, bool warns) { if (folio_test_anon(folio)) { /* order-1 is not supported for anonymous THP. */ @@ -3524,48 +3533,41 @@ bool non_uniform_split_supported(struct folio *folio, unsigned int new_order, "Cannot split to order-1 folio"); if (new_order == 1) return false; - } else if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && - !mapping_large_folio_support(folio->mapping)) { - /* - * No split if the file system does not support large folio. - * Note that we might still have THPs in such mappings due to - * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping - * does not actually support large folios properly. - */ - VM_WARN_ONCE(warns, - "Cannot split file folio to non-0 order"); - return false; - } - - /* Only swapping a whole PMD-mapped folio is supported */ - if (folio_test_swapcache(folio)) { - VM_WARN_ONCE(warns, - "Cannot split swapcache folio to non-0 order"); - return false; - } - - return true; -} - -/* See comments in non_uniform_split_supported() */ -bool uniform_split_supported(struct folio *folio, unsigned int new_order, - bool warns) -{ - if (folio_test_anon(folio)) { - VM_WARN_ONCE(warns && new_order == 1, - "Cannot split to order-1 folio"); - if (new_order == 1) - return false; - } else if (new_order) { + } else if (!uniform_split || new_order) { if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !mapping_large_folio_support(folio->mapping)) { + /* + * We can always split a folio down to a single page + * (new_order == 0) uniformly. + * + * For any other scenario + * a) uniform split targeting a large folio + * (new_order > 0) + * b) any non-uniform split + * we must confirm that the file system supports large + * folios. + * + * Note that we might still have THPs in such + * mappings, which is created from khugepaged when + * CONFIG_READ_ONLY_THP_FOR_FS is enabled. But in that + * case, the mapping does not actually support large + * folios properly. + */ VM_WARN_ONCE(warns, "Cannot split file folio to non-0 order"); return false; } } - if (new_order && folio_test_swapcache(folio)) { + /* + * swapcache folio could only be split to order 0 + * + * non-uniform split creates after-split folios with orders from + * folio_order(folio) - 1 to new_order, making it not suitable for any + * swapcache folio split. Only uniform split to order-0 can be used + * here. + */ + if ((!uniform_split || new_order) && folio_test_swapcache(folio)) { VM_WARN_ONCE(warns, "Cannot split swapcache folio to non-0 order"); return false; @@ -3632,11 +3634,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, if (new_order >= folio_order(folio)) return -EINVAL; - if (uniform_split && !uniform_split_supported(folio, new_order, true)) - return -EINVAL; - - if (!uniform_split && - !non_uniform_split_supported(folio, new_order, true)) + if (!folio_split_supported(folio, new_order, uniform_split, /* warn = */ true)) return -EINVAL; is_hzp = is_huge_zero_folio(folio); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 0455119716ec0..13293976e0568 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2934,7 +2934,7 @@ typedef enum { * NOTE: This is mostly identical to MAP_CHG_NEEDED, except * that currently vma_needs_reservation() has an unwanted side * effect to either use end() or commit() to complete the - * transaction. Hence it needs to differenciate from NEEDED. + * transaction. Hence it needs to differentiate from NEEDED. */ MAP_CHG_ENFORCED = 2, } map_chg_state; @@ -5797,7 +5797,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, unsigned long last_addr_mask; pte_t *src_pte, *dst_pte; struct mmu_notifier_range range; - bool shared_pmd = false; + struct mmu_gather tlb; mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr, old_end); @@ -5807,6 +5807,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, * range. */ flush_cache_range(vma, range.start, range.end); + tlb_gather_mmu_vma(&tlb, vma); mmu_notifier_invalidate_range_start(&range); last_addr_mask = hugetlb_mask_last_page(h); @@ -5823,8 +5824,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte))) continue; - if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) { - shared_pmd = true; + if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) { old_addr |= last_addr_mask; new_addr |= last_addr_mask; continue; @@ -5835,15 +5835,16 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, break; move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz); + tlb_remove_huge_tlb_entry(h, &tlb, src_pte, old_addr); } - if (shared_pmd) - flush_hugetlb_tlb_range(vma, range.start, range.end); - else - flush_hugetlb_tlb_range(vma, old_end - len, old_end); + tlb_flush_mmu_tlbonly(&tlb); + huge_pmd_unshare_flush(&tlb, vma); + mmu_notifier_invalidate_range_end(&range); i_mmap_unlock_write(mapping); hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); return len + old_addr - old_end; } @@ -5862,7 +5863,6 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long sz = huge_page_size(h); bool adjust_reservation; unsigned long last_addr_mask; - bool force_flush = false; WARN_ON(!is_vm_hugetlb_page(vma)); BUG_ON(start & ~huge_page_mask(h)); @@ -5885,10 +5885,8 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, } ptl = huge_pte_lock(h, mm, ptep); - if (huge_pmd_unshare(mm, vma, address, ptep)) { + if (huge_pmd_unshare(tlb, vma, address, ptep)) { spin_unlock(ptl); - tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE); - force_flush = true; address |= last_addr_mask; continue; } @@ -6004,21 +6002,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, } tlb_end_vma(tlb, vma); - /* - * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We - * could defer the flush until now, since by holding i_mmap_rwsem we - * guaranteed that the last refernece would not be dropped. But we must - * do the flushing before we return, as otherwise i_mmap_rwsem will be - * dropped and the last reference to the shared PMDs page might be - * dropped as well. - * - * In theory we could defer the freeing of the PMD pages as well, but - * huge_pmd_unshare() relies on the exact page_count for the PMD page to - * detect sharing, so we cannot defer the release of the page either. - * Instead, do flush now. - */ - if (force_flush) - tlb_flush_mmu_tlbonly(tlb); + huge_pmd_unshare_flush(tlb, vma); } void __hugetlb_zap_begin(struct vm_area_struct *vma, @@ -7111,11 +7095,11 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pte_t pte; struct hstate *h = hstate_vma(vma); long pages = 0, psize = huge_page_size(h); - bool shared_pmd = false; struct mmu_notifier_range range; unsigned long last_addr_mask; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + struct mmu_gather tlb; /* * In the case of shared PMDs, the area to flush could be beyond @@ -7128,6 +7112,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, BUG_ON(address >= end); flush_cache_range(vma, range.start, range.end); + tlb_gather_mmu_vma(&tlb, vma); mmu_notifier_invalidate_range_start(&range); hugetlb_vma_lock_write(vma); @@ -7152,7 +7137,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, } } ptl = huge_pte_lock(h, mm, ptep); - if (huge_pmd_unshare(mm, vma, address, ptep)) { + if (huge_pmd_unshare(&tlb, vma, address, ptep)) { /* * When uffd-wp is enabled on the vma, unshare * shouldn't happen at all. Warn about it if it @@ -7161,7 +7146,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma, WARN_ON_ONCE(uffd_wp || uffd_wp_resolve); pages++; spin_unlock(ptl); - shared_pmd = true; address |= last_addr_mask; continue; } @@ -7193,7 +7177,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, } else if (unlikely(is_pte_marker(pte))) { /* * Do nothing on a poison marker; page is - * corrupted, permissons do not apply. Here + * corrupted, permissions do not apply. Here * pte_marker_uffd_wp()==true implies !poison * because they're mutual exclusive. */ @@ -7213,6 +7197,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pte = huge_pte_clear_uffd_wp(pte); huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; + tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); } else { /* None pte */ if (unlikely(uffd_wp)) @@ -7225,17 +7210,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma, cond_resched(); } - /* - * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare - * may have cleared our pud entry and done put_page on the page table: - * once we release i_mmap_rwsem, another task can do the final put_page - * and that page table be reused and filled with junk. If we actually - * did unshare a page of pmds, flush the range corresponding to the pud. - */ - if (shared_pmd) - flush_hugetlb_tlb_range(vma, range.start, range.end); - else - flush_hugetlb_tlb_range(vma, start, end); + + tlb_flush_mmu_tlbonly(&tlb); + huge_pmd_unshare_flush(&tlb, vma); /* * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are * downgrading page table protection not changing it to point to a new @@ -7246,6 +7223,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, i_mmap_unlock_write(vma->vm_file->f_mapping); hugetlb_vma_unlock_write(vma); mmu_notifier_invalidate_range_end(&range); + tlb_finish_mmu(&tlb); return pages > 0 ? (pages << h->order) : pages; } @@ -7408,6 +7386,15 @@ long hugetlb_reserve_pages(struct inode *inode, */ hugetlb_acct_memory(h, -gbl_resv); } + /* Restore used_hpages for pages that failed global reservation */ + if (gbl_reserve && spool) { + unsigned long flags; + + spin_lock_irqsave(&spool->lock, flags); + if (spool->max_hpages != -1) + spool->used_hpages -= gbl_reserve; + unlock_or_release_subpool(spool, flags); + } out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), chg * pages_per_huge_page(h), h_cg); @@ -7598,18 +7585,27 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, return pte; } -/* - * unmap huge page backed by shared pte. +/** + * huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users + * @tlb: the current mmu_gather. + * @vma: the vma covering the pmd table. + * @addr: the address we are trying to unshare. + * @ptep: pointer into the (pmd) page table. + * + * Called with the page table lock held, the i_mmap_rwsem held in write mode + * and the hugetlb vma lock held in write mode. * - * Called with page table lock held. + * Note: The caller must call huge_pmd_unshare_flush() before dropping the + * i_mmap_rwsem. * - * returns: 1 successfully unmapped a shared pte page - * 0 the underlying pte page is not shared, or it is the last user + * Returns: 1 if it was a shared PMD table and it got unmapped, or 0 if it + * was not a shared PMD table. */ -int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { unsigned long sz = huge_page_size(hstate_vma(vma)); + struct mm_struct *mm = vma->vm_mm; pgd_t *pgd = pgd_offset(mm, addr); p4d_t *p4d = p4d_offset(pgd, addr); pud_t *pud = pud_offset(p4d, addr); @@ -7621,18 +7617,36 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, i_mmap_assert_write_locked(vma->vm_file->f_mapping); hugetlb_vma_assert_locked(vma); pud_clear(pud); - /* - * Once our caller drops the rmap lock, some other process might be - * using this page table as a normal, non-hugetlb page table. - * Wait for pending gup_fast() in other threads to finish before letting - * that happen. - */ - tlb_remove_table_sync_one(); - ptdesc_pmd_pts_dec(virt_to_ptdesc(ptep)); + + tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr); + mm_dec_nr_pmds(mm); return 1; } +/* + * huge_pmd_unshare_flush - Complete a sequence of huge_pmd_unshare() calls + * @tlb: the current mmu_gather. + * @vma: the vma covering the pmd table. + * + * Perform necessary TLB flushes or IPI broadcasts to synchronize PMD table + * unsharing with concurrent page table walkers. + * + * This function must be called after a sequence of huge_pmd_unshare() + * calls while still holding the i_mmap_rwsem. + */ +void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ + /* + * We must synchronize page table unsharing such that nobody will + * try reusing a previously-shared page table while it might still + * be in use by previous sharers (TLB, GUP_fast). + */ + i_mmap_assert_write_locked(vma->vm_file->f_mapping); + + tlb_flush_unshared_tables(tlb); +} + #else /* !CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, @@ -7641,12 +7655,16 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, return NULL; } -int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { return 0; } +void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ +} + void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end) { @@ -7913,6 +7931,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, unsigned long sz = huge_page_size(h); struct mm_struct *mm = vma->vm_mm; struct mmu_notifier_range range; + struct mmu_gather tlb; unsigned long address; spinlock_t *ptl; pte_t *ptep; @@ -7924,6 +7943,8 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, return; flush_cache_range(vma, start, end); + tlb_gather_mmu_vma(&tlb, vma); + /* * No need to call adjust_range_if_pmd_sharing_possible(), because * we have already done the PUD_SIZE alignment. @@ -7942,10 +7963,10 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, if (!ptep) continue; ptl = huge_pte_lock(h, mm, ptep); - huge_pmd_unshare(mm, vma, address, ptep); + huge_pmd_unshare(&tlb, vma, address, ptep); spin_unlock(ptl); } - flush_hugetlb_tlb_range(vma, start, end); + huge_pmd_unshare_flush(&tlb, vma); if (take_locks) { i_mmap_unlock_write(vma->vm_file->f_mapping); hugetlb_vma_unlock_write(vma); @@ -7955,6 +7976,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, * Documentation/mm/mmu_notifier.rst. */ mmu_notifier_invalidate_range_end(&range); + tlb_finish_mmu(&tlb); } /* diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index ba0fb1b6a5a8e..96ee2bd16ee15 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -75,7 +75,7 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start, if (likely(pmd_leaf(*pmd))) { /* * Higher order allocations from buddy allocator must be able to - * be treated as indepdenent small pages (as they can be freed + * be treated as independent small pages (as they can be freed * individually). */ if (!PageReserved(head)) @@ -684,7 +684,7 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, ret = hugetlb_vmemmap_split_folio(h, folio); /* - * Spliting the PMD requires allocating a page, thus lets fail + * Splitting the PMD requires allocating a page, thus let's fail * early once we encounter the first OOM. No point in retrying * as it can be dynamically done on remap with the memory * we get back from the vmemmap deduplication. @@ -715,7 +715,7 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, /* * Pages to be freed may have been accumulated. If we * encounter an ENOMEM, free what we have and try again. - * This can occur in the case that both spliting fails + * This can occur in the case that both splitting fails * halfway and head page allocation also failed. In this * case __hugetlb_vmemmap_optimize_folio() would free memory * allowing more vmemmap remaps to occur. diff --git a/mm/internal.h b/mm/internal.h index 1561fc2ff5b83..c80c6f566c2d9 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -538,16 +538,8 @@ extern unsigned long highest_memmap_pfn; bool folio_isolate_lru(struct folio *folio); void folio_putback_lru(struct folio *folio); extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason); -#ifdef CONFIG_NUMA int user_proactive_reclaim(char *buf, struct mem_cgroup *memcg, pg_data_t *pgdat); -#else -static inline int user_proactive_reclaim(char *buf, - struct mem_cgroup *memcg, pg_data_t *pgdat) -{ - return 0; -} -#endif /* * in mm/rmap.c: diff --git a/mm/kasan/common.c b/mm/kasan/common.c index d4c14359feaf9..3341474507f95 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "kasan.h" #include "../slab.h" @@ -582,3 +583,55 @@ bool __kasan_check_byte(const void *address, unsigned long ip) } return true; } + +#ifdef CONFIG_KASAN_VMALLOC +void __kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, + kasan_vmalloc_flags_t flags) +{ + unsigned long size; + void *addr; + int area; + u8 tag; + + /* + * If KASAN_VMALLOC_KEEP_TAG was set at this point, all vms[] pointers + * would be unpoisoned with the KASAN_TAG_KERNEL which would disable + * KASAN checks down the line. + */ + if (WARN_ON_ONCE(flags & KASAN_VMALLOC_KEEP_TAG)) + return; + + size = vms[0]->size; + addr = vms[0]->addr; + vms[0]->addr = __kasan_unpoison_vmalloc(addr, size, flags); + tag = get_tag(vms[0]->addr); + + for (area = 1 ; area < nr_vms ; area++) { + size = vms[area]->size; + addr = set_tag(vms[area]->addr, tag); + vms[area]->addr = + __kasan_unpoison_vmalloc(addr, size, flags | KASAN_VMALLOC_KEEP_TAG); + } +} + +void __kasan_vrealloc(const void *addr, unsigned long old_size, + unsigned long new_size) +{ + if (new_size < old_size) { + kasan_poison_last_granule(addr, new_size); + + new_size = round_up(new_size, KASAN_GRANULE_SIZE); + old_size = round_up(old_size, KASAN_GRANULE_SIZE); + if (new_size < old_size) + __kasan_poison_vmalloc(addr + new_size, + old_size - new_size); + } else if (new_size > old_size) { + old_size = round_down(old_size, KASAN_GRANULE_SIZE); + __kasan_unpoison_vmalloc(addr + old_size, + new_size - old_size, + KASAN_VMALLOC_PROT_NORMAL | + KASAN_VMALLOC_VM_ALLOC | + KASAN_VMALLOC_KEEP_TAG); + } +} +#endif diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index 1c373cc4b3fa5..cbef5e450954e 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -361,7 +361,7 @@ void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, return (void *)start; } - tag = kasan_random_tag(); + tag = (flags & KASAN_VMALLOC_KEEP_TAG) ? get_tag(start) : kasan_random_tag(); start = set_tag(start, tag); /* Unpoison and initialize memory up to size. */ diff --git a/mm/kasan/init.c b/mm/kasan/init.c index f084e7a5df1e5..9c880f607c6a2 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -292,7 +292,7 @@ static void kasan_free_pte(pte_t *pte_start, pmd_t *pmd) return; } - pte_free_kernel(&init_mm, (pte_t *)page_to_virt(pmd_page(*pmd))); + pte_free_kernel(&init_mm, pte_start); pmd_clear(pmd); } @@ -307,7 +307,7 @@ static void kasan_free_pmd(pmd_t *pmd_start, pud_t *pud) return; } - pmd_free(&init_mm, (pmd_t *)page_to_virt(pud_page(*pud))); + pmd_free(&init_mm, pmd_start); pud_clear(pud); } @@ -322,7 +322,7 @@ static void kasan_free_pud(pud_t *pud_start, p4d_t *p4d) return; } - pud_free(&init_mm, (pud_t *)page_to_virt(p4d_page(*p4d))); + pud_free(&init_mm, pud_start); p4d_clear(p4d); } @@ -337,7 +337,7 @@ static void kasan_free_p4d(p4d_t *p4d_start, pgd_t *pgd) return; } - p4d_free(&init_mm, (p4d_t *)page_to_virt(pgd_page(*pgd))); + p4d_free(&init_mm, p4d_start); pgd_clear(pgd); } diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 5d2a876035d65..5e47ae7fdd590 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -648,7 +648,9 @@ void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, !(flags & KASAN_VMALLOC_PROT_NORMAL)) return (void *)start; - start = set_tag(start, kasan_random_tag()); + if (unlikely(!(flags & KASAN_VMALLOC_KEEP_TAG))) + start = set_tag(start, kasan_random_tag()); + kasan_unpoison(start, size, false); return (void *)start; } diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 727c20c94ac59..c5d525fcfccaf 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -595,7 +596,7 @@ static void rcu_guarded_free(struct rcu_head *h) static unsigned long kfence_init_pool(void) { unsigned long addr, start_pfn; - int i; + int i, rand; if (!arch_kfence_init_pool()) return (unsigned long)__kfence_pool; @@ -645,13 +646,27 @@ static unsigned long kfence_init_pool(void) INIT_LIST_HEAD(&meta->list); raw_spin_lock_init(&meta->lock); meta->state = KFENCE_OBJECT_UNUSED; - meta->addr = addr; /* Initialize for validation in metadata_to_pageaddr(). */ - list_add_tail(&meta->list, &kfence_freelist); + /* Use addr to randomize the freelist. */ + meta->addr = i; /* Protect the right redzone. */ - if (unlikely(!kfence_protect(addr + PAGE_SIZE))) + if (unlikely(!kfence_protect(addr + 2 * i * PAGE_SIZE + PAGE_SIZE))) goto reset_slab; + } + + for (i = CONFIG_KFENCE_NUM_OBJECTS; i > 0; i--) { + rand = get_random_u32_below(i); + swap(kfence_metadata_init[i - 1].addr, kfence_metadata_init[rand].addr); + } + + for (i = 0; i < CONFIG_KFENCE_NUM_OBJECTS; i++) { + struct kfence_metadata *meta_1 = &kfence_metadata_init[i]; + struct kfence_metadata *meta_2 = &kfence_metadata_init[meta_1->addr]; + list_add_tail(&meta_2->list, &kfence_freelist); + } + for (i = 0; i < CONFIG_KFENCE_NUM_OBJECTS; i++) { + kfence_metadata_init[i].addr = addr; addr += 2 * PAGE_SIZE; } @@ -664,6 +679,7 @@ static unsigned long kfence_init_pool(void) return 0; reset_slab: + addr += 2 * i * PAGE_SIZE; for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { struct slab *slab; @@ -867,6 +883,20 @@ void __init kfence_alloc_pool_and_metadata(void) if (!kfence_sample_interval) return; + /* + * If KASAN hardware tags are enabled, disable KFENCE, because it + * does not support MTE yet. + */ + if (kasan_hw_tags_enabled()) { + pr_info("disabled as KASAN HW tags are enabled\n"); + if (__kfence_pool) { + memblock_free(__kfence_pool, KFENCE_POOL_SIZE); + __kfence_pool = NULL; + } + kfence_sample_interval = 0; + return; + } + /* * If the pool has already been initialized by arch, there is no need to * re-allocate the memory pool. @@ -936,14 +966,14 @@ static int kfence_init_late(void) #ifdef CONFIG_CONTIG_ALLOC struct page *pages; - pages = alloc_contig_pages(nr_pages_pool, GFP_KERNEL, first_online_node, - NULL); + pages = alloc_contig_pages(nr_pages_pool, GFP_KERNEL | __GFP_SKIP_KASAN, + first_online_node, NULL); if (!pages) return -ENOMEM; __kfence_pool = page_to_virt(pages); - pages = alloc_contig_pages(nr_pages_meta, GFP_KERNEL, first_online_node, - NULL); + pages = alloc_contig_pages(nr_pages_meta, GFP_KERNEL | __GFP_SKIP_KASAN, + first_online_node, NULL); if (pages) kfence_metadata_init = page_to_virt(pages); #else @@ -953,11 +983,13 @@ static int kfence_init_late(void) return -EINVAL; } - __kfence_pool = alloc_pages_exact(KFENCE_POOL_SIZE, GFP_KERNEL); + __kfence_pool = alloc_pages_exact(KFENCE_POOL_SIZE, + GFP_KERNEL | __GFP_SKIP_KASAN); if (!__kfence_pool) return -ENOMEM; - kfence_metadata_init = alloc_pages_exact(KFENCE_METADATA_SIZE, GFP_KERNEL); + kfence_metadata_init = alloc_pages_exact(KFENCE_METADATA_SIZE, + GFP_KERNEL | __GFP_SKIP_KASAN); #endif if (!kfence_metadata_init) diff --git a/mm/kmsan/core.c b/mm/kmsan/core.c index 35ceaa8adb41e..90f427b95a213 100644 --- a/mm/kmsan/core.c +++ b/mm/kmsan/core.c @@ -33,7 +33,7 @@ bool kmsan_enabled __read_mostly; /* * Per-CPU KMSAN context to be used in interrupts, where current->kmsan is - * unavaliable. + * unavailable. */ DEFINE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx); diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c index 55fdea199aaf0..8bd50f5b1092e 100644 --- a/mm/kmsan/shadow.c +++ b/mm/kmsan/shadow.c @@ -207,7 +207,7 @@ void kmsan_free_page(struct page *page, unsigned int order) if (!kmsan_enabled || kmsan_in_runtime()) return; kmsan_enter_runtime(); - kmsan_internal_poison_memory(page_address(page), page_size(page), + kmsan_internal_poison_memory(page_address(page), PAGE_SIZE << order, GFP_KERNEL & ~(__GFP_RECLAIM), KMSAN_POISON_CHECK | KMSAN_POISON_FREE); kmsan_leave_runtime(); diff --git a/mm/ksm.c b/mm/ksm.c index c4e7304099499..4f672f4f21407 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -389,7 +389,7 @@ static unsigned long ewma(unsigned long prev, unsigned long curr) * exponentially weighted moving average. The new pages_to_scan value is * multiplied with that change factor: * - * new_pages_to_scan *= change facor + * new_pages_to_scan *= change factor * * The new_pages_to_scan value is limited by the cpu min and max values. It * calculates the cpu percent for the last scan and calculates the new @@ -2712,8 +2712,14 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) spin_unlock(&ksm_mmlist_lock); mm_slot_free(mm_slot_cache, mm_slot); + /* + * Only clear MMF_VM_MERGEABLE. We must not clear + * MMF_VM_MERGE_ANY, because for those MMF_VM_MERGE_ANY process, + * perhaps their mm_struct has just been added to ksm_mm_slot + * list, and its process has not yet officially started running + * or has not yet performed mmap/brk to allocate anonymous VMAS. + */ mm_flags_clear(MMF_VM_MERGEABLE, mm); - mm_flags_clear(MMF_VM_MERGE_ANY, mm); mmap_read_unlock(mm); mmdrop(mm); } else { @@ -2831,12 +2837,20 @@ static int __ksm_del_vma(struct vm_area_struct *vma) * * Returns: @vm_flags possibly updated to mark mergeable. */ -vm_flags_t ksm_vma_flags(const struct mm_struct *mm, const struct file *file, +vm_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file, vm_flags_t vm_flags) { if (mm_flags_test(MMF_VM_MERGE_ANY, mm) && - __ksm_should_add_vma(file, vm_flags)) + __ksm_should_add_vma(file, vm_flags)) { vm_flags |= VM_MERGEABLE; + /* + * Generally, the flags here always include MMF_VM_MERGEABLE. + * However, in rare cases, this flag may be cleared by ksmd who + * scans a cycle without finding any mergeable vma. + */ + if (unlikely(!mm_flags_test(MMF_VM_MERGEABLE, mm))) + __ksm_enter(mm); + } return vm_flags; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 4deda33625f41..61cf6af26f3c9 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3028,7 +3028,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, if (!local_trylock(&obj_stock.lock)) { if (pgdat) - mod_objcg_mlstate(objcg, pgdat, idx, nr_bytes); + mod_objcg_mlstate(objcg, pgdat, idx, nr_acct); nr_pages = nr_bytes >> PAGE_SHIFT; nr_bytes = nr_bytes & (PAGE_SIZE - 1); atomic_add(nr_bytes, &objcg->nr_charged_bytes); @@ -5597,7 +5597,19 @@ subsys_initcall(mem_cgroup_swap_init); #endif /* CONFIG_SWAP */ -bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid) +void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask) { - return memcg ? cpuset_node_allowed(memcg->css.cgroup, nid) : true; + nodemask_t allowed; + + if (!memcg) + return; + + /* + * Since this interface is intended for use by migration paths, and + * reclaim and migration are subject to race conditions such as changes + * in effective_mems and hot-unpluging of nodes, inaccurate allowed + * mask is acceptable. + */ + cpuset_nodes_allowed(memcg->css.cgroup, &allowed); + nodes_and(*mask, *mask, allowed); } diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 3edebb0cda30b..6e770bad79ce3 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -684,6 +684,8 @@ static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, unsigned long poisoned_pfn, struct to_kill *tk) { unsigned long pfn = 0; + unsigned long hwpoison_vaddr; + unsigned long mask; if (pte_present(pte)) { pfn = pte_pfn(pte); @@ -694,10 +696,12 @@ static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, pfn = swp_offset_pfn(swp); } - if (!pfn || pfn != poisoned_pfn) + mask = ~((1UL << (shift - PAGE_SHIFT)) - 1); + if (!pfn || pfn != (poisoned_pfn & mask)) return 0; - set_to_kill(tk, addr, shift); + hwpoison_vaddr = addr + ((poisoned_pfn - pfn) << PAGE_SHIFT); + set_to_kill(tk, hwpoison_vaddr, shift); return 1; } @@ -1873,12 +1877,22 @@ static unsigned long __folio_free_raw_hwp(struct folio *folio, bool move_flag) return count; } -static int folio_set_hugetlb_hwpoison(struct folio *folio, struct page *page) +#define MF_HUGETLB_FREED 0 /* freed hugepage */ +#define MF_HUGETLB_IN_USED 1 /* in-use hugepage */ +#define MF_HUGETLB_NON_HUGEPAGE 2 /* not a hugepage */ +#define MF_HUGETLB_FOLIO_PRE_POISONED 3 /* folio already poisoned */ +#define MF_HUGETLB_PAGE_PRE_POISONED 4 /* exact page already poisoned */ +#define MF_HUGETLB_RETRY 5 /* hugepage is busy, retry */ +/* + * Set hugetlb folio as hwpoisoned, update folio private raw hwpoison list + * to keep track of the poisoned pages. + */ +static int hugetlb_update_hwpoison(struct folio *folio, struct page *page) { struct llist_head *head; struct raw_hwp_page *raw_hwp; struct raw_hwp_page *p; - int ret = folio_test_set_hwpoison(folio) ? -EHWPOISON : 0; + int ret = folio_test_set_hwpoison(folio) ? MF_HUGETLB_FOLIO_PRE_POISONED : 0; /* * Once the hwpoison hugepage has lost reliable raw error info, @@ -1886,20 +1900,17 @@ static int folio_set_hugetlb_hwpoison(struct folio *folio, struct page *page) * so skip to add additional raw error info. */ if (folio_test_hugetlb_raw_hwp_unreliable(folio)) - return -EHWPOISON; + return MF_HUGETLB_FOLIO_PRE_POISONED; head = raw_hwp_list_head(folio); llist_for_each_entry(p, head->first, node) { if (p->page == page) - return -EHWPOISON; + return MF_HUGETLB_PAGE_PRE_POISONED; } raw_hwp = kmalloc(sizeof(struct raw_hwp_page), GFP_ATOMIC); if (raw_hwp) { raw_hwp->page = page; llist_add(&raw_hwp->node, head); - /* the first error event will be counted in action_result(). */ - if (ret) - num_poisoned_pages_inc(page_to_pfn(page)); } else { /* * Failed to save raw error info. We no longer trace all @@ -1947,42 +1958,39 @@ void folio_clear_hugetlb_hwpoison(struct folio *folio) /* * Called from hugetlb code with hugetlb_lock held. - * - * Return values: - * 0 - free hugepage - * 1 - in-use hugepage - * 2 - not a hugepage - * -EBUSY - the hugepage is busy (try to retry) - * -EHWPOISON - the hugepage is already hwpoisoned */ int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, bool *migratable_cleared) { struct page *page = pfn_to_page(pfn); struct folio *folio = page_folio(page); - int ret = 2; /* fallback to normal page handling */ bool count_increased = false; + int ret, rc; - if (!folio_test_hugetlb(folio)) + if (!folio_test_hugetlb(folio)) { + ret = MF_HUGETLB_NON_HUGEPAGE; goto out; - - if (flags & MF_COUNT_INCREASED) { - ret = 1; + } else if (flags & MF_COUNT_INCREASED) { + ret = MF_HUGETLB_IN_USED; count_increased = true; } else if (folio_test_hugetlb_freed(folio)) { - ret = 0; + ret = MF_HUGETLB_FREED; } else if (folio_test_hugetlb_migratable(folio)) { - ret = folio_try_get(folio); - if (ret) + if (folio_try_get(folio)) { + ret = MF_HUGETLB_IN_USED; count_increased = true; + } else { + ret = MF_HUGETLB_FREED; + } } else { - ret = -EBUSY; + ret = MF_HUGETLB_RETRY; if (!(flags & MF_NO_RETRY)) goto out; } - if (folio_set_hugetlb_hwpoison(folio, page)) { - ret = -EHWPOISON; + rc = hugetlb_update_hwpoison(folio, page); + if (rc >= MF_HUGETLB_FOLIO_PRE_POISONED) { + ret = rc; goto out; } @@ -2007,10 +2015,16 @@ int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, * with basic operations like hugepage allocation/free/demotion. * So some of prechecks for hwpoison (pinning, and testing/setting * PageHWPoison) should be done in single hugetlb_lock range. + * Returns: + * 0 - not hugetlb, or recovered + * -EBUSY - not recovered + * -EOPNOTSUPP - hwpoison_filter'ed + * -EHWPOISON - folio or exact page already poisoned + * -EFAULT - kill_accessing_process finds current->mm null */ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb) { - int res; + int res, rv; struct page *p = pfn_to_page(pfn); struct folio *folio; unsigned long page_flags; @@ -2019,22 +2033,29 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb *hugetlb = 1; retry: res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared); - if (res == 2) { /* fallback to normal page handling */ + switch (res) { + case MF_HUGETLB_NON_HUGEPAGE: /* fallback to normal page handling */ *hugetlb = 0; return 0; - } else if (res == -EHWPOISON) { - if (flags & MF_ACTION_REQUIRED) { - folio = page_folio(p); - res = kill_accessing_process(current, folio_pfn(folio), flags); - } - action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED); - return res; - } else if (res == -EBUSY) { + case MF_HUGETLB_RETRY: if (!(flags & MF_NO_RETRY)) { flags |= MF_NO_RETRY; goto retry; } return action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED); + case MF_HUGETLB_FOLIO_PRE_POISONED: + case MF_HUGETLB_PAGE_PRE_POISONED: + rv = -EHWPOISON; + if (flags & MF_ACTION_REQUIRED) + rv = kill_accessing_process(current, pfn, flags); + if (res == MF_HUGETLB_PAGE_PRE_POISONED) + action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED); + else + action_result(pfn, MF_MSG_HUGE, MF_FAILED); + return rv; + default: + WARN_ON((res != MF_HUGETLB_FREED) && (res != MF_HUGETLB_IN_USED)); + break; } folio = page_folio(p); @@ -2045,7 +2066,7 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb if (migratable_cleared) folio_set_hugetlb_migratable(folio); folio_unlock(folio); - if (res == 1) + if (res == MF_HUGETLB_IN_USED) folio_put(folio); return -EOPNOTSUPP; } @@ -2054,7 +2075,7 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb * Handling free hugepage. The possible race with hugepage allocation * or demotion can be prevented by PageHWPoison flag. */ - if (res == 0) { + if (res == MF_HUGETLB_FREED) { folio_unlock(folio); if (__page_handle_poison(p) > 0) { page_ref_inc(p); diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 0ea5c13f10a23..864811fff4093 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -519,7 +519,7 @@ static inline void __init_node_memory_type(int node, struct memory_dev_type *mem * for each device getting added in the same NUMA node * with this specific memtype, bump the map count. We * Only take memtype device reference once, so that - * changing a node memtype can be done by droping the + * changing a node memtype can be done by dropping the * only reference count taken here. */ diff --git a/mm/memory.c b/mm/memory.c index b59ae7ce42ebc..94bf107a47caf 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4328,7 +4328,7 @@ static inline bool should_try_to_free_swap(struct folio *folio, * If we want to map a page that's in the swapcache writable, we * have to detect via the refcount if we're really the exclusive * user. Try freeing the swapcache to get rid of the swapcache - * reference only in case it's likely that we'll be the exlusive user. + * reference only in case it's likely that we'll be the exclusive user. */ return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) && folio_ref_count(folio) == (1 + folio_nr_pages(folio)); @@ -4642,7 +4642,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) unlock_page(vmf->page); put_page(vmf->page); } else { - pte_unmap_unlock(vmf->pte, vmf->ptl); + pte_unmap(vmf->pte); + migration_entry_wait_on_locked(entry, vmf->ptl); } } else if (is_hwpoison_entry(entry)) { ret = VM_FAULT_HWPOISON; @@ -5405,7 +5406,7 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa /** * set_pte_range - Set a range of PTEs to point to pages in a folio. - * @vmf: Fault decription. + * @vmf: Fault description. * @folio: The folio that contains @page. * @page: The first page to create a PTE for. * @nr: The number of PTEs to create. @@ -6690,17 +6691,22 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto out; p4dp = p4d_offset(pgdp, address); - p4d = READ_ONCE(*p4dp); + p4d = p4dp_get(p4dp); if (p4d_none(p4d) || unlikely(p4d_bad(p4d))) goto out; pudp = pud_offset(p4dp, address); - pud = READ_ONCE(*pudp); - if (pud_none(pud)) + pud = pudp_get(pudp); + if (!pud_present(pud)) goto out; if (pud_leaf(pud)) { lock = pud_lock(mm, pudp); - if (!unlikely(pud_leaf(pud))) { + pud = pudp_get(pudp); + + if (unlikely(!pud_present(pud))) { + spin_unlock(lock); + goto out; + } else if (unlikely(!pud_leaf(pud))) { spin_unlock(lock); goto retry; } @@ -6712,9 +6718,16 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pmdp = pmd_offset(pudp, address); pmd = pmdp_get_lockless(pmdp); + if (!pmd_present(pmd)) + goto out; if (pmd_leaf(pmd)) { lock = pmd_lock(mm, pmdp); - if (!unlikely(pmd_leaf(pmd))) { + pmd = pmdp_get(pmdp); + + if (unlikely(!pmd_present(pmd))) { + spin_unlock(lock); + goto out; + } else if (unlikely(!pmd_leaf(pmd))) { spin_unlock(lock); goto retry; } diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 0be83039c3b5e..aa1f744143077 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1218,6 +1218,13 @@ int online_pages(unsigned long pfn, unsigned long nr_pages, if (node_arg.nid >= 0) node_set_state(nid, N_MEMORY); + /* + * Check whether we are adding normal memory to the node for the first + * time. + */ + if (!node_state(nid, N_NORMAL_MEMORY) && zone_idx(zone) <= ZONE_NORMAL) + node_set_state(nid, N_NORMAL_MEMORY); + if (need_zonelists_rebuild) build_all_zonelists(NULL); @@ -1919,6 +1926,8 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages, unsigned long flags; char *reason; int ret; + unsigned long normal_pages = 0; + enum zone_type zt; /* * {on,off}lining is constrained to full memory sections (or more @@ -2066,6 +2075,17 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages, /* reinitialise watermarks and update pcp limits */ init_per_zone_wmark_min(); + /* + * Check whether this operation removes the last normal memory from + * the node. We do this before clearing N_MEMORY to avoid the possible + * transient "!N_MEMORY && N_NORMAL_MEMORY" state. + */ + if (zone_idx(zone) <= ZONE_NORMAL) { + for (zt = 0; zt <= ZONE_NORMAL; zt++) + normal_pages += pgdat->node_zones[zt].present_pages; + if (!normal_pages) + node_clear_state(node, N_NORMAL_MEMORY); + } /* * Make sure to mark the node as memory-less before rebuilding the zone * list. Otherwise this node would still appear in the fallback lists. diff --git a/mm/mempolicy.c b/mm/mempolicy.c index eb83cff7db8c3..94327574fbbbb 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -485,7 +485,13 @@ void __mpol_put(struct mempolicy *pol) { if (!atomic_dec_and_test(&pol->refcnt)) return; - kmem_cache_free(policy_cache, pol); + /* + * Required to allow mmap_lock_speculative*() access, see for example + * futex_key_to_node_opt(). All accesses are serialized by mmap_lock, + * however the speculative lock section unbound by the normal lock + * boundaries, requiring RCU freeing. + */ + kfree_rcu(pol, rcu); } static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes) @@ -951,7 +957,7 @@ static int vma_replace_policy(struct vm_area_struct *vma, } old = vma->vm_policy; - vma->vm_policy = new; /* protected by mmap_lock */ + WRITE_ONCE(vma->vm_policy, new); /* protected by mmap_lock */ mpol_put(old); return 0; diff --git a/mm/migrate.c b/mm/migrate.c index c0e9f15be2a22..a936623d0b237 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1450,6 +1450,7 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, int page_was_mapped = 0; struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; + enum ttu_flags ttu = 0; if (folio_ref_count(src) == 1) { /* page was freed from under us. So we are done. */ @@ -1490,8 +1491,6 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, goto put_anon; if (folio_mapped(src)) { - enum ttu_flags ttu = 0; - if (!folio_test_anon(src)) { /* * In shared mappings, try_to_unmap could potentially @@ -1508,16 +1507,17 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, try_to_migrate(src, ttu); page_was_mapped = 1; - - if (ttu & TTU_RMAP_LOCKED) - i_mmap_unlock_write(mapping); } if (!folio_mapped(src)) rc = move_to_new_folio(dst, src, mode); if (page_was_mapped) - remove_migration_ptes(src, !rc ? dst : src, 0); + remove_migration_ptes(src, !rc ? dst : src, + ttu ? RMP_LOCKED : 0); + + if (ttu & TTU_RMAP_LOCKED) + i_mmap_unlock_write(mapping); unlock_put_anon: folio_unlock(dst); diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 374aa6f021c6b..5619ee967f3c7 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -426,6 +427,7 @@ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, #endif tlb->vma_pfn = 0; + tlb->fully_unshared_tables = 0; __tlb_reset_range(tlb); inc_tlb_flush_pending(tlb->mm); } @@ -459,6 +461,31 @@ void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) __tlb_gather_mmu(tlb, mm, true); } +/** + * tlb_gather_mmu_vma - initialize an mmu_gather structure for operating on a + * single VMA + * @tlb: the mmu_gather structure to initialize + * @vma: the vm_area_struct + * + * Called to initialize an (on-stack) mmu_gather structure for operating on + * a single VMA. In contrast to tlb_gather_mmu(), calling this function will + * not require another call to tlb_start_vma(). In contrast to tlb_start_vma(), + * this function will *not* call flush_cache_range(). + * + * For hugetlb VMAs, this function will also initialize the mmu_gather + * page_size accordingly, not requiring a separate call to + * tlb_change_page_size(). + * + */ +void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ + tlb_gather_mmu(tlb, vma->vm_mm); + tlb_update_vma_flags(tlb, vma); + if (is_vm_hugetlb_page(vma)) + /* All entries have the same size. */ + tlb_change_page_size(tlb, huge_page_size(hstate_vma(vma))); +} + /** * tlb_finish_mmu - finish an mmu_gather structure * @tlb: the mmu_gather structure to finish @@ -468,6 +495,12 @@ void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) */ void tlb_finish_mmu(struct mmu_gather *tlb) { + /* + * We expect an earlier huge_pmd_unshare_flush() call to sort this out, + * due to complicated locking requirements with page table unsharing. + */ + VM_WARN_ON_ONCE(tlb->fully_unshared_tables); + /* * If there are parallel threads are doing PTE changes on same range * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB diff --git a/mm/mprotect.c b/mm/mprotect.c index 113b489858341..988c366137d50 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -599,7 +599,7 @@ static inline long change_pud_range(struct mmu_gather *tlb, break; } - pud = READ_ONCE(*pudp); + pud = pudp_get(pudp); if (pud_none(pud)) continue; diff --git a/mm/mseal.c b/mm/mseal.c index e5b205562d2ec..c561f0ea93e84 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -56,7 +56,6 @@ static int mseal_apply(struct mm_struct *mm, unsigned long start, unsigned long end) { struct vm_area_struct *vma, *prev; - unsigned long curr_start = start; VMA_ITERATOR(vmi, mm, start); /* We know there are no gaps so this will be non-NULL. */ @@ -66,7 +65,8 @@ static int mseal_apply(struct mm_struct *mm, prev = vma; for_each_vma_range(vmi, vma, end) { - unsigned long curr_end = MIN(vma->vm_end, end); + const unsigned long curr_start = MAX(vma->vm_start, start); + const unsigned long curr_end = MIN(vma->vm_end, end); if (!(vma->vm_flags & VM_SEALED)) { vma = vma_modify_flags(&vmi, prev, vma, @@ -78,7 +78,6 @@ static int mseal_apply(struct mm_struct *mm, } prev = vma; - curr_start = curr_end; } return 0; diff --git a/mm/numa_memblks.c b/mm/numa_memblks.c index 5b009a9cd8b4c..3f53464240e8d 100644 --- a/mm/numa_memblks.c +++ b/mm/numa_memblks.c @@ -7,6 +7,8 @@ #include #include +#include + int numa_distance_cnt; static u8 *numa_distance; @@ -568,15 +570,16 @@ static int meminfo_to_nid(struct numa_meminfo *mi, u64 start) int phys_to_target_node(u64 start) { int nid = meminfo_to_nid(&numa_meminfo, start); + int reserved_nid = meminfo_to_nid(&numa_reserved_meminfo, start); /* - * Prefer online nodes, but if reserved memory might be - * hot-added continue the search with reserved ranges. + * Prefer online nodes unless the address is also described + * by reserved ranges, in which case use the reserved nid. */ - if (nid != NUMA_NO_NODE) + if (nid != NUMA_NO_NODE && reserved_nid == NUMA_NO_NODE) return nid; - return meminfo_to_nid(&numa_reserved_meminfo, start); + return reserved_nid; } EXPORT_SYMBOL_GPL(phys_to_target_node); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ed82ee55e66af..6288c7e4b971b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -166,6 +166,33 @@ static DEFINE_MUTEX(pcp_batch_high_lock); #define pcp_spin_unlock(ptr) \ pcpu_spin_unlock(lock, ptr) +/* + * With the UP spinlock implementation, when we spin_lock(&pcp->lock) (for i.e. + * a potentially remote cpu drain) and get interrupted by an operation that + * attempts pcp_spin_trylock(), we can't rely on the trylock failure due to UP + * spinlock assumptions making the trylock a no-op. So we have to turn that + * spin_lock() to a spin_lock_irqsave(). This works because on UP there are no + * remote cpu's so we can only be locking the only existing local one. + */ +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) +static inline void __flags_noop(unsigned long *flags) { } +#define pcp_spin_lock_maybe_irqsave(ptr, flags) \ +({ \ + __flags_noop(&(flags)); \ + spin_lock(&(ptr)->lock); \ +}) +#define pcp_spin_unlock_maybe_irqrestore(ptr, flags) \ +({ \ + spin_unlock(&(ptr)->lock); \ + __flags_noop(&(flags)); \ +}) +#else +#define pcp_spin_lock_maybe_irqsave(ptr, flags) \ + spin_lock_irqsave(&(ptr)->lock, flags) +#define pcp_spin_unlock_maybe_irqrestore(ptr, flags) \ + spin_unlock_irqrestore(&(ptr)->lock, flags) +#endif + #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); @@ -913,6 +940,17 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, NULL) != NULL; } +static void change_pageblock_range(struct page *pageblock_page, + int start_order, int migratetype) +{ + int nr_pageblocks = 1 << (start_order - pageblock_order); + + while (nr_pageblocks--) { + set_pageblock_migratetype(pageblock_page, migratetype); + pageblock_page += pageblock_nr_pages; + } +} + /* * Freeing function for a buddy system allocator. * @@ -999,7 +1037,7 @@ static inline void __free_one_page(struct page *page, * expand() down the line puts the sub-blocks * on the right freelists. */ - set_pageblock_migratetype(buddy, migratetype); + change_pageblock_range(buddy, order, migratetype); } combined_pfn = buddy_pfn & pfn; @@ -1301,8 +1339,8 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #endif /* CONFIG_MEM_ALLOC_PROFILING */ -__always_inline bool free_pages_prepare(struct page *page, - unsigned int order) +__always_inline bool __free_pages_prepare(struct page *page, + unsigned int order, fpi_t fpi_flags) { int bad = 0; bool skip_kasan_poison = should_skip_kasan_poison(page); @@ -1391,11 +1429,12 @@ __always_inline bool free_pages_prepare(struct page *page, page_cpupid_reset_last(page); page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; + page->private = 0; reset_page_owner(page, order); page_table_check_free(page, order); pgalloc_tag_sub(page, 1 << order); - if (!PageHighMem(page)) { + if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) { debug_check_no_locks_freed(page_address(page), PAGE_SIZE << order); debug_check_no_obj_freed(page_address(page), @@ -1434,6 +1473,11 @@ __always_inline bool free_pages_prepare(struct page *page, return true; } +bool free_pages_prepare(struct page *page, unsigned int order) +{ + return __free_pages_prepare(page, order, FPI_NONE); +} + /* * Frees a number of pages from the PCP lists * Assumes all pages on list are in same zone. @@ -1567,7 +1611,7 @@ static void __free_pages_ok(struct page *page, unsigned int order, unsigned long pfn = page_to_pfn(page); struct zone *zone = page_zone(page); - if (free_pages_prepare(page, order)) + if (__free_pages_prepare(page, order, fpi_flags)) free_one_page(zone, page, pfn, order, fpi_flags); } @@ -2146,17 +2190,6 @@ bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *pag #endif /* CONFIG_MEMORY_ISOLATION */ -static void change_pageblock_range(struct page *pageblock_page, - int start_order, int migratetype) -{ - int nr_pageblocks = 1 << (start_order - pageblock_order); - - while (nr_pageblocks--) { - set_pageblock_migratetype(pageblock_page, migratetype); - pageblock_page += pageblock_nr_pages; - } -} - static inline bool boost_watermark(struct zone *zone) { unsigned long max_boost; @@ -2552,10 +2585,11 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, * Called from the vmstat counter updater to decay the PCP high. * Return whether there are addition works to do. */ -int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp) +bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp) { - int high_min, to_drain, batch; - int todo = 0; + int high_min, to_drain, to_drain_batched, batch; + unsigned long UP_flags; + bool todo = false; high_min = READ_ONCE(pcp->high_min); batch = READ_ONCE(pcp->batch); @@ -2568,15 +2602,18 @@ int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp) pcp->high = max3(pcp->count - (batch << CONFIG_PCP_BATCH_SCALE_MAX), pcp->high - (pcp->high >> 3), high_min); if (pcp->high > high_min) - todo++; + todo = true; } to_drain = pcp->count - pcp->high; - if (to_drain > 0) { - spin_lock(&pcp->lock); - free_pcppages_bulk(zone, to_drain, pcp, 0); - spin_unlock(&pcp->lock); - todo++; + while (to_drain > 0) { + to_drain_batched = min(to_drain, batch); + pcp_spin_lock_maybe_irqsave(pcp, UP_flags); + free_pcppages_bulk(zone, to_drain_batched, pcp, 0); + pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); + todo = true; + + to_drain -= to_drain_batched; } return todo; @@ -2590,14 +2627,15 @@ int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp) */ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) { + unsigned long UP_flags; int to_drain, batch; batch = READ_ONCE(pcp->batch); to_drain = min(pcp->count, batch); if (to_drain > 0) { - spin_lock(&pcp->lock); + pcp_spin_lock_maybe_irqsave(pcp, UP_flags); free_pcppages_bulk(zone, to_drain, pcp, 0); - spin_unlock(&pcp->lock); + pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); } } #endif @@ -2608,10 +2646,11 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) static void drain_pages_zone(unsigned int cpu, struct zone *zone) { struct per_cpu_pages *pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); + unsigned long UP_flags; int count; do { - spin_lock(&pcp->lock); + pcp_spin_lock_maybe_irqsave(pcp, UP_flags); count = pcp->count; if (count) { int to_drain = min(count, @@ -2620,7 +2659,7 @@ static void drain_pages_zone(unsigned int cpu, struct zone *zone) free_pcppages_bulk(zone, to_drain, pcp, 0); count -= to_drain; } - spin_unlock(&pcp->lock); + pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); } while (count); } @@ -2898,7 +2937,7 @@ static void __free_frozen_pages(struct page *page, unsigned int order, return; } - if (!free_pages_prepare(page, order)) + if (!__free_pages_prepare(page, order, fpi_flags)) return; /* @@ -2955,7 +2994,7 @@ void free_unref_folios(struct folio_batch *folios) unsigned long pfn = folio_pfn(folio); unsigned int order = folio_order(folio); - if (!free_pages_prepare(&folio->page, order)) + if (!__free_pages_prepare(&folio->page, order, FPI_NONE)) continue; /* * Free orders not handled on the PCP directly to the @@ -4756,6 +4795,20 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, compact_result == COMPACT_DEFERRED) goto nopage; + /* + * THP page faults may attempt local node only first, + * but are then allowed to only compact, not reclaim, + * see alloc_pages_mpol(). + * + * Compaction can fail for other reasons than those + * checked above and we don't want such THP allocations + * to put reclaim pressure on a single node in a + * situation where other nodes might have plenty of + * available memory. + */ + if (gfp_mask & __GFP_THISNODE) + goto nopage; + /* * Looks like reclaim/compaction is worth trying, but * sync compaction could be very expensive, so keep @@ -6078,6 +6131,7 @@ static void zone_pcp_update_cacheinfo(struct zone *zone, unsigned int cpu) { struct per_cpu_pages *pcp; struct cpu_cacheinfo *cci; + unsigned long UP_flags; pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); cci = get_cpu_cacheinfo(cpu); @@ -6088,12 +6142,12 @@ static void zone_pcp_update_cacheinfo(struct zone *zone, unsigned int cpu) * This can reduce zone lock contention without hurting * cache-hot pages sharing. */ - spin_lock(&pcp->lock); + pcp_spin_lock_maybe_irqsave(pcp, UP_flags); if ((cci->per_cpu_data_slice_size >> PAGE_SHIFT) > 3 * pcp->batch) pcp->flags |= PCPF_FREE_HIGH_BATCH; else pcp->flags &= ~PCPF_FREE_HIGH_BATCH; - spin_unlock(&pcp->lock); + pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); } void setup_pcp_cacheinfo(unsigned int cpu) @@ -6611,11 +6665,19 @@ static int percpu_pagelist_high_fraction_sysctl_handler(const struct ctl_table * int old_percpu_pagelist_high_fraction; int ret; + /* + * Avoid using pcp_batch_high_lock for reads as the value is read + * atomically and a race with offlining is harmless. + */ + + if (!write) + return proc_dointvec_minmax(table, write, buffer, length, ppos); + mutex_lock(&pcp_batch_high_lock); old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction; ret = proc_dointvec_minmax(table, write, buffer, length, ppos); - if (!write || ret < 0) + if (ret < 0) goto out; /* Sanity checking to avoid pcp imbalance */ @@ -6826,7 +6888,8 @@ static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask) { const gfp_t reclaim_mask = __GFP_IO | __GFP_FS | __GFP_RECLAIM; const gfp_t action_mask = __GFP_COMP | __GFP_RETRY_MAYFAIL | __GFP_NOWARN | - __GFP_ZERO | __GFP_ZEROTAGS | __GFP_SKIP_ZERO; + __GFP_ZERO | __GFP_ZEROTAGS | __GFP_SKIP_ZERO | + __GFP_SKIP_KASAN; const gfp_t cc_action_mask = __GFP_RETRY_MAYFAIL | __GFP_NOWARN; /* diff --git a/mm/page_owner.c b/mm/page_owner.c index 589ec37c94aab..bc26764142ba5 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -936,7 +936,7 @@ static const struct file_operations page_owner_stack_operations = { .open = page_owner_stack_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = seq_release_private, }; static int page_owner_threshold_get(void *data, u64 *val) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 9f91cf85a5be7..2d19d6a9b034d 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -97,6 +97,7 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, struct mm_walk *walk) { + pud_t pudval = pudp_get(pud); pmd_t *pmd; unsigned long next; const struct mm_walk_ops *ops = walk->ops; @@ -105,6 +106,24 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(3); + /* + * For PTE handling, pte_offset_map_lock() takes care of checking + * whether there actually is a page table. But it also has to be + * very careful about concurrent page table reclaim. + * + * Similarly, we have to be careful here - a PUD entry that points + * to a PMD table cannot go away, so we can just walk it. But if + * it's something else, we need to ensure we didn't race something, + * so need to retry. + * + * A pertinent example of this is a PUD refault after PUD split - + * we will need to split again or risk accessing invalid memory. + */ + if (!pud_present(pudval) || pud_leaf(pudval)) { + walk->action = ACTION_AGAIN; + return 0; + } + pmd = pmd_offset(pud, addr); do { again: @@ -218,12 +237,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, else if (pud_leaf(*pud) || !pud_present(*pud)) continue; /* Nothing to do. */ - if (pud_none(*pud)) - goto again; - err = walk_pmd_range(pud, addr, next, walk); if (err) break; + + if (walk->action == ACTION_AGAIN) + goto again; } while (pud++, addr = next, addr != end); return err; diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index 567e2d084071e..8c22be79b7343 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -406,3 +407,41 @@ pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, pte_unmap_unlock(pte, ptl); goto again; } + +#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE +static void kernel_pgtable_work_func(struct work_struct *work); + +static struct { + struct list_head list; + /* protect above ptdesc lists */ + spinlock_t lock; + struct work_struct work; +} kernel_pgtable_work = { + .list = LIST_HEAD_INIT(kernel_pgtable_work.list), + .lock = __SPIN_LOCK_UNLOCKED(kernel_pgtable_work.lock), + .work = __WORK_INITIALIZER(kernel_pgtable_work.work, kernel_pgtable_work_func), +}; + +static void kernel_pgtable_work_func(struct work_struct *work) +{ + struct ptdesc *pt, *next; + LIST_HEAD(page_list); + + spin_lock(&kernel_pgtable_work.lock); + list_splice_tail_init(&kernel_pgtable_work.list, &page_list); + spin_unlock(&kernel_pgtable_work.lock); + + iommu_sva_invalidate_kva_range(PAGE_OFFSET, TLB_FLUSH_ALL); + list_for_each_entry_safe(pt, next, &page_list, pt_list) + __pagetable_free(pt); +} + +void pagetable_free_kernel(struct ptdesc *pt) +{ + spin_lock(&kernel_pgtable_work.lock); + list_add(&pt->pt_list, &kernel_pgtable_work.list); + spin_unlock(&kernel_pgtable_work.lock); + + schedule_work(&kernel_pgtable_work.work); +} +#endif diff --git a/mm/rmap.c b/mm/rmap.c index ac4f783d6ec2f..8f3c50bd1125f 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -76,7 +76,7 @@ #include #include -#include +#include #define CREATE_TRACE_POINTS #include @@ -1856,7 +1856,14 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, if (pte_unused(pte)) return 1; - return folio_pte_batch(folio, pvmw->pte, pte, max_nr); + /* + * If unmap fails, we need to restore the ptes. To avoid accidentally + * upgrading write permissions for ptes that were not originally + * writable, and to avoid losing the soft-dirty bit, use the + * appropriate FPB flags. + */ + return folio_pte_batch_flags(folio, vma, pvmw->pte, &pte, max_nr, + FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } /* @@ -2019,26 +2026,25 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * if unsuccessful. */ if (!anon) { + struct mmu_gather tlb; + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); if (!hugetlb_vma_trylock_write(vma)) goto walk_abort; - if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { + + tlb_gather_mmu_vma(&tlb, vma); + if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { hugetlb_vma_unlock_write(vma); - flush_tlb_range(vma, - range.start, range.end); + huge_pmd_unshare_flush(&tlb, vma); + tlb_finish_mmu(&tlb); /* - * The ref count of the PMD page was - * dropped which is part of the way map - * counting is done for shared PMDs. - * Return 'true' here. When there is - * no other sharing, huge_pmd_unshare - * returns false and we will unmap the - * actual page and drop map count - * to zero. + * The PMD table was unmapped, + * consequently unmapping the folio. */ goto walk_done; } hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); } pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); if (pte_dirty(pteval)) @@ -2340,11 +2346,17 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* PMD-mapped THP migration entry */ if (!pvmw.pte) { if (flags & TTU_SPLIT_HUGE_PMD) { + /* + * split_huge_pmd_locked() might leave the + * folio mapped through PTEs. Retry the walk + * so we can detect this scenario and properly + * abort the walk. + */ split_huge_pmd_locked(vma, pvmw.address, pvmw.pmd, true); - ret = false; - page_vma_mapped_walk_done(&pvmw); - break; + flags &= ~TTU_SPLIT_HUGE_PMD; + page_vma_mapped_walk_restart(&pvmw); + continue; } #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION subpage = folio_page(folio, @@ -2404,31 +2416,29 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * fail if unsuccessful. */ if (!anon) { + struct mmu_gather tlb; + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); if (!hugetlb_vma_trylock_write(vma)) { page_vma_mapped_walk_done(&pvmw); ret = false; break; } - if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { - hugetlb_vma_unlock_write(vma); - flush_tlb_range(vma, - range.start, range.end); + tlb_gather_mmu_vma(&tlb, vma); + if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); + huge_pmd_unshare_flush(&tlb, vma); + tlb_finish_mmu(&tlb); /* - * The ref count of the PMD page was - * dropped which is part of the way map - * counting is done for shared PMDs. - * Return 'true' here. When there is - * no other sharing, huge_pmd_unshare - * returns false and we will unmap the - * actual page and drop map count - * to zero. + * The PMD table was unmapped, + * consequently unmapping the folio. */ page_vma_mapped_walk_done(&pvmw); break; } hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); } /* Nuke the hugetlb page table entry */ pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); diff --git a/mm/secretmem.c b/mm/secretmem.c index b59350daffe31..9b0f5d9ec6f4b 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -227,7 +227,7 @@ SYSCALL_DEFINE1(memfd_secret, unsigned int, flags) struct file *file; int fd, err; - /* make sure local flags do not confict with global fcntl.h */ + /* make sure local flags do not conflict with global fcntl.h */ BUILD_BUG_ON(SECRETMEM_FLAGS_MASK & O_CLOEXEC); if (!secretmem_enable || !can_set_direct_map()) diff --git a/mm/shmem.c b/mm/shmem.c index 5a3f0f754dc0c..94c5b0d78ac31 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -944,17 +944,29 @@ static void shmem_delete_from_page_cache(struct folio *folio, void *radswap) * being freed). */ static long shmem_free_swap(struct address_space *mapping, - pgoff_t index, void *radswap) + pgoff_t index, pgoff_t end, void *radswap) { - int order = xa_get_order(&mapping->i_pages, index); - void *old; + XA_STATE(xas, &mapping->i_pages, index); + unsigned int nr_pages = 0; + pgoff_t base; + void *entry; - old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0); - if (old != radswap) - return 0; - free_swap_and_cache_nr(radix_to_swp_entry(radswap), 1 << order); + xas_lock_irq(&xas); + entry = xas_load(&xas); + if (entry == radswap) { + nr_pages = 1 << xas_get_order(&xas); + base = round_down(xas.xa_index, nr_pages); + if (base < index || base + nr_pages - 1 > end) + nr_pages = 0; + else + xas_store(&xas, NULL); + } + xas_unlock_irq(&xas); + + if (nr_pages) + free_swap_and_cache_nr(radix_to_swp_entry(radswap), nr_pages); - return 1 << order; + return nr_pages; } /* @@ -1106,8 +1118,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, if (xa_is_value(folio)) { if (unfalloc) continue; - nr_swaps_freed += shmem_free_swap(mapping, - indices[i], folio); + nr_swaps_freed += shmem_free_swap(mapping, indices[i], + end - 1, folio); continue; } @@ -1173,14 +1185,30 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, folio = fbatch.folios[i]; if (xa_is_value(folio)) { + int order; long swaps_freed; if (unfalloc) continue; - swaps_freed = shmem_free_swap(mapping, indices[i], folio); + swaps_freed = shmem_free_swap(mapping, indices[i], + end - 1, folio); if (!swaps_freed) { - /* Swap was replaced by page: retry */ - index = indices[i]; + pgoff_t base = indices[i]; + + order = shmem_confirm_swap(mapping, indices[i], + radix_to_swp_entry(folio)); + /* + * If found a large swap entry cross the end or start + * border, skip it as the truncate_inode_partial_folio + * above should have at least zerod its content once. + */ + if (order > 0) { + base = round_down(base, 1 << order); + if (base < start || base + (1 << order) > end) + continue; + } + /* Swap was replaced by page or extended, retry */ + index = base; break; } nr_swaps_freed += swaps_freed; @@ -4039,6 +4067,7 @@ static int shmem_rename2(struct mnt_idmap *idmap, { struct inode *inode = d_inode(old_dentry); int they_are_dirs = S_ISDIR(inode->i_mode); + bool had_offset = false; int error; if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) @@ -4051,16 +4080,23 @@ static int shmem_rename2(struct mnt_idmap *idmap, if (!simple_empty(new_dentry)) return -ENOTEMPTY; + error = simple_offset_add(shmem_get_offset_ctx(new_dir), new_dentry); + if (error == -EBUSY) + had_offset = true; + else if (unlikely(error)) + return error; + if (flags & RENAME_WHITEOUT) { error = shmem_whiteout(idmap, old_dir, old_dentry); - if (error) + if (error) { + if (!had_offset) + simple_offset_remove(shmem_get_offset_ctx(new_dir), + new_dentry); return error; + } } - error = simple_offset_rename(old_dir, old_dentry, new_dir, new_dentry); - if (error) - return error; - + simple_offset_rename(old_dir, old_dentry, new_dir, new_dentry); if (d_really_is_positive(new_dentry)) { (void) shmem_unlink(new_dir, new_dentry); if (they_are_dirs) { diff --git a/mm/slab.h b/mm/slab.h index 078daecc7cf50..36893299fa67c 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -236,10 +236,8 @@ struct kmem_cache_order_objects { * Slab cache management. */ struct kmem_cache { -#ifndef CONFIG_SLUB_TINY struct kmem_cache_cpu __percpu *cpu_slab; struct lock_class_key lock_key; -#endif struct slub_percpu_sheaves __percpu *cpu_sheaves; /* Used for retrieving partial slabs, etc. */ slab_flags_t flags; @@ -442,6 +440,7 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s) bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj); void flush_all_rcu_sheaves(void); +void flush_rcu_sheaves_on_cache(struct kmem_cache *s); #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \ SLAB_CACHE_DMA32 | SLAB_PANIC | \ diff --git a/mm/slab_common.c b/mm/slab_common.c index 932d13ada36c0..0f58265ca200e 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -259,7 +259,7 @@ static struct kmem_cache *create_cache(const char *name, * @object_size: The size of objects to be created in this cache. * @args: Additional arguments for the cache creation (see * &struct kmem_cache_args). - * @flags: See the desriptions of individual flags. The common ones are listed + * @flags: See the descriptions of individual flags. The common ones are listed * in the description below. * * Not to be called directly, use the kmem_cache_create() wrapper with the same @@ -492,7 +492,7 @@ void kmem_cache_destroy(struct kmem_cache *s) return; /* in-flight kfree_rcu()'s may include objects from our cache */ - kvfree_rcu_barrier(); + kvfree_rcu_barrier_on_cache(s); if (IS_ENABLED(CONFIG_SLUB_RCU_DEBUG) && (s->flags & SLAB_TYPESAFE_BY_RCU)) { @@ -2039,25 +2039,13 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) } EXPORT_SYMBOL_GPL(kvfree_call_rcu); -/** - * kvfree_rcu_barrier - Wait until all in-flight kvfree_rcu() complete. - * - * Note that a single argument of kvfree_rcu() call has a slow path that - * triggers synchronize_rcu() following by freeing a pointer. It is done - * before the return from the function. Therefore for any single-argument - * call that will result in a kfree() to a cache that is to be destroyed - * during module exit, it is developer's responsibility to ensure that all - * such calls have returned before the call to kmem_cache_destroy(). - */ -void kvfree_rcu_barrier(void) +static inline void __kvfree_rcu_barrier(void) { struct kfree_rcu_cpu_work *krwp; struct kfree_rcu_cpu *krcp; bool queued; int i, cpu; - flush_all_rcu_sheaves(); - /* * Firstly we detach objects and queue them over an RCU-batch * for all CPUs. Finally queued works are flushed for each CPU. @@ -2119,8 +2107,46 @@ void kvfree_rcu_barrier(void) } } } + +/** + * kvfree_rcu_barrier - Wait until all in-flight kvfree_rcu() complete. + * + * Note that a single argument of kvfree_rcu() call has a slow path that + * triggers synchronize_rcu() following by freeing a pointer. It is done + * before the return from the function. Therefore for any single-argument + * call that will result in a kfree() to a cache that is to be destroyed + * during module exit, it is developer's responsibility to ensure that all + * such calls have returned before the call to kmem_cache_destroy(). + */ +void kvfree_rcu_barrier(void) +{ + flush_all_rcu_sheaves(); + __kvfree_rcu_barrier(); +} EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); +/** + * kvfree_rcu_barrier_on_cache - Wait for in-flight kvfree_rcu() calls on a + * specific slab cache. + * @s: slab cache to wait for + * + * See the description of kvfree_rcu_barrier() for details. + */ +void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) +{ + if (s->cpu_sheaves) { + flush_rcu_sheaves_on_cache(s); + rcu_barrier(); + } + + /* + * TODO: Introduce a version of __kvfree_rcu_barrier() that works + * on a specific slab cache. + */ + __kvfree_rcu_barrier(); +} +EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); + static unsigned long kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) { @@ -2216,4 +2242,3 @@ void __init kvfree_rcu_init(void) } #endif /* CONFIG_KVFREE_RCU_BATCHED */ - diff --git a/mm/slub.c b/mm/slub.c index a0b905c2a557e..5b038d1c8250e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -410,7 +411,6 @@ enum stat_item { NR_SLUB_STAT_ITEMS }; -#ifndef CONFIG_SLUB_TINY /* * When changing the layout, make sure freelist and tid are still compatible * with this_cpu_cmpxchg_double() alignment requirements. @@ -432,7 +432,6 @@ struct kmem_cache_cpu { unsigned int stat[NR_SLUB_STAT_ITEMS]; #endif }; -#endif /* CONFIG_SLUB_TINY */ static inline void stat(const struct kmem_cache *s, enum stat_item si) { @@ -594,12 +593,10 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object) return freelist_ptr_decode(s, p, ptr_addr); } -#ifndef CONFIG_SLUB_TINY static void prefetch_freepointer(const struct kmem_cache *s, void *object) { prefetchw(object + s->offset); } -#endif /* * When running under KMSAN, get_freepointer_safe() may return an uninitialized @@ -711,10 +708,12 @@ static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) return s->cpu_partial_slabs; } #else +#ifdef SLAB_SUPPORTS_SYSFS static inline void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) { } +#endif static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) { @@ -863,7 +862,7 @@ static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab, * request size in the meta data area, for better debug and sanity check. */ static inline void set_orig_size(struct kmem_cache *s, - void *object, unsigned int orig_size) + void *object, unsigned long orig_size) { void *p = kasan_reset_tag(object); @@ -873,10 +872,10 @@ static inline void set_orig_size(struct kmem_cache *s, p += get_info_end(s); p += sizeof(struct track) * 2; - *(unsigned int *)p = orig_size; + *(unsigned long *)p = orig_size; } -static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) +static inline unsigned long get_orig_size(struct kmem_cache *s, void *object) { void *p = kasan_reset_tag(object); @@ -889,7 +888,7 @@ static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) p += get_info_end(s); p += sizeof(struct track) * 2; - return *(unsigned int *)p; + return *(unsigned long *)p; } #ifdef CONFIG_SLUB_DEBUG @@ -1204,7 +1203,7 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) off += 2 * sizeof(struct track); if (slub_debug_orig_size(s)) - off += sizeof(unsigned int); + off += sizeof(unsigned long); off += kasan_metadata_size(s, false); @@ -1400,7 +1399,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) off += 2 * sizeof(struct track); if (s->flags & SLAB_KMALLOC) - off += sizeof(unsigned int); + off += sizeof(unsigned long); } off += kasan_metadata_size(s, false); @@ -2023,13 +2022,11 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) {} static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) {} -#ifndef CONFIG_SLUB_TINY static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, void **freelist, void *nextfree) { return false; } -#endif #endif /* CONFIG_SLUB_DEBUG */ #ifdef CONFIG_SLAB_OBJ_EXT @@ -2099,6 +2096,42 @@ static inline void init_slab_obj_exts(struct slab *slab) slab->obj_exts = 0; } +/* + * Calculate the allocation size for slabobj_ext array. + * + * When memory allocation profiling is enabled, the obj_exts array + * could be allocated from the same slab cache it's being allocated for. + * This would prevent the slab from ever being freed because it would + * always contain at least one allocated object (its own obj_exts array). + * + * To avoid this, increase the allocation size when we detect the array + * may come from the same cache, forcing it to use a different cache. + */ +static inline size_t obj_exts_alloc_size(struct kmem_cache *s, + struct slab *slab, gfp_t gfp) +{ + size_t sz = sizeof(struct slabobj_ext) * slab->objects; + struct kmem_cache *obj_exts_cache; + + if (sz > KMALLOC_MAX_CACHE_SIZE) + return sz; + + if (!is_kmalloc_normal(s)) + return sz; + + obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0); + /* + * We can't simply compare s with obj_exts_cache, because random kmalloc + * caches have multiple caches per size, selected by caller address. + * Since caller address may differ between kmalloc_slab() and actual + * allocation, bump size when sizes are equal. + */ + if (s->object_size == obj_exts_cache->object_size) + return obj_exts_cache->object_size + 1; + + return sz; +} + int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, bool new_slab) { @@ -2107,26 +2140,26 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, unsigned long new_exts; unsigned long old_exts; struct slabobj_ext *vec; + size_t sz; gfp &= ~OBJCGS_CLEAR_MASK; /* Prevent recursive extension vector allocation */ gfp |= __GFP_NO_OBJ_EXT; + sz = obj_exts_alloc_size(s, slab, gfp); + /* * Note that allow_spin may be false during early boot and its * restricted GFP_BOOT_MASK. Due to kmalloc_nolock() only supporting * architectures with cmpxchg16b, early obj_exts will be missing for * very early allocations on those. */ - if (unlikely(!allow_spin)) { - size_t sz = objects * sizeof(struct slabobj_ext); - + if (unlikely(!allow_spin)) vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT, slab_nid(slab)); - } else { - vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp, - slab_nid(slab)); - } + else + vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab)); + if (!vec) { /* * Try to mark vectors which failed to allocate. @@ -2140,6 +2173,9 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, return -ENOMEM; } + VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL && + virt_to_slab(vec)->slab_cache == s); + new_exts = (unsigned long)vec; if (unlikely(!allow_spin)) new_exts |= OBJEXTS_NOSPIN_ALLOC; @@ -2533,7 +2569,7 @@ bool slab_free_hook(struct kmem_cache *s, void *x, bool init, memset((char *)kasan_reset_tag(x) + inuse, 0, s->size - inuse - rsize); /* - * Restore orig_size, otherwize kmalloc redzone overwritten + * Restore orig_size, otherwise kmalloc redzone overwritten * would be reported */ set_orig_size(s, x, orig_size); @@ -2677,19 +2713,19 @@ static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); * object pointers are moved to a on-stack array under the lock. To bound the * stack usage, limit each batch to PCS_BATCH_MAX. * - * returns true if at least partially flushed + * Must be called with s->cpu_sheaves->lock locked, returns with the lock + * unlocked. + * + * Returns how many objects are remaining to be flushed */ -static bool sheaf_flush_main(struct kmem_cache *s) +static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s) { struct slub_percpu_sheaves *pcs; unsigned int batch, remaining; void *objects[PCS_BATCH_MAX]; struct slab_sheaf *sheaf; - bool ret = false; -next_batch: - if (!local_trylock(&s->cpu_sheaves->lock)) - return ret; + lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); pcs = this_cpu_ptr(s->cpu_sheaves); sheaf = pcs->main; @@ -2707,10 +2743,37 @@ static bool sheaf_flush_main(struct kmem_cache *s) stat_add(s, SHEAF_FLUSH, batch); - ret = true; + return remaining; +} - if (remaining) - goto next_batch; +static void sheaf_flush_main(struct kmem_cache *s) +{ + unsigned int remaining; + + do { + local_lock(&s->cpu_sheaves->lock); + + remaining = __sheaf_flush_main_batch(s); + + } while (remaining); +} + +/* + * Returns true if the main sheaf was at least partially flushed. + */ +static bool sheaf_try_flush_main(struct kmem_cache *s) +{ + unsigned int remaining; + bool ret = false; + + do { + if (!local_trylock(&s->cpu_sheaves->lock)) + return ret; + + ret = true; + remaining = __sheaf_flush_main_batch(s); + + } while (remaining); return ret; } @@ -3134,8 +3197,11 @@ static void *next_freelist_entry(struct kmem_cache *s, return (char *)start + idx; } +static DEFINE_PER_CPU(struct rnd_state, slab_rnd_state); + /* Shuffle the single linked freelist based on a random pre-computed sequence */ -static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) +static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, + bool allow_spin) { void *start; void *cur; @@ -3146,7 +3212,19 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) return false; freelist_count = oo_objects(s->oo); - pos = get_random_u32_below(freelist_count); + if (allow_spin) { + pos = get_random_u32_below(freelist_count); + } else { + struct rnd_state *state; + + /* + * An interrupt or NMI handler might interrupt and change + * the state in the middle, but that's safe. + */ + state = &get_cpu_var(slab_rnd_state); + pos = prandom_u32_state(state) % freelist_count; + put_cpu_var(slab_rnd_state); + } page_limit = slab->objects * s->size; start = fixup_red_left(s, slab_address(slab)); @@ -3173,7 +3251,8 @@ static inline int init_cache_random_seq(struct kmem_cache *s) return 0; } static inline void init_freelist_randomization(void) { } -static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) +static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, + bool allow_spin) { return false; } @@ -3258,7 +3337,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) setup_slab_debug(s, slab, start); - shuffle = shuffle_freelist(s, slab); + shuffle = shuffle_freelist(s, slab, allow_spin); if (!shuffle) { start = fixup_red_left(s, start); @@ -3548,6 +3627,7 @@ static struct slab *get_any_partial(struct kmem_cache *s, enum zone_type highest_zoneidx = gfp_zone(pc->flags); struct slab *slab; unsigned int cpuset_mems_cookie; + bool allow_spin = gfpflags_allow_spinning(pc->flags); /* * The defrag ratio allows a configuration of the tradeoffs between @@ -3572,7 +3652,15 @@ static struct slab *get_any_partial(struct kmem_cache *s, return NULL; do { - cpuset_mems_cookie = read_mems_allowed_begin(); + /* + * read_mems_allowed_begin() accesses current->mems_allowed_seq, + * a seqcount_spinlock_t that is not NMI-safe. Do not access + * current->mems_allowed_seq and avoid retry when GFP flags + * indicate spinning is not allowed. + */ + if (allow_spin) + cpuset_mems_cookie = read_mems_allowed_begin(); + zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { struct kmem_cache_node *n; @@ -3594,7 +3682,7 @@ static struct slab *get_any_partial(struct kmem_cache *s, } } } - } while (read_mems_allowed_retry(cpuset_mems_cookie)); + } while (allow_spin && read_mems_allowed_retry(cpuset_mems_cookie)); #endif /* CONFIG_NUMA */ return NULL; } @@ -3618,8 +3706,6 @@ static struct slab *get_partial(struct kmem_cache *s, int node, return get_any_partial(s, pc); } -#ifndef CONFIG_SLUB_TINY - #ifdef CONFIG_PREEMPTION /* * Calculate the next globally unique transaction for disambiguation @@ -4019,12 +4105,6 @@ static bool has_cpu_slab(int cpu, struct kmem_cache *s) return c->slab || slub_percpu_partial(c); } -#else /* CONFIG_SLUB_TINY */ -static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { } -static inline bool has_cpu_slab(int cpu, struct kmem_cache *s) { return false; } -static inline void flush_this_cpu_slab(struct kmem_cache *s) { } -#endif /* CONFIG_SLUB_TINY */ - static bool has_pcs_used(int cpu, struct kmem_cache *s) { struct slub_percpu_sheaves *pcs; @@ -4118,42 +4198,47 @@ static void flush_rcu_sheaf(struct work_struct *w) /* needed for kvfree_rcu_barrier() */ -void flush_all_rcu_sheaves(void) +void flush_rcu_sheaves_on_cache(struct kmem_cache *s) { struct slub_flush_work *sfw; - struct kmem_cache *s; unsigned int cpu; - cpus_read_lock(); - mutex_lock(&slab_mutex); + mutex_lock(&flush_lock); - list_for_each_entry(s, &slab_caches, list) { - if (!s->cpu_sheaves) - continue; + for_each_online_cpu(cpu) { + sfw = &per_cpu(slub_flush, cpu); - mutex_lock(&flush_lock); + /* + * we don't check if rcu_free sheaf exists - racing + * __kfree_rcu_sheaf() might have just removed it. + * by executing flush_rcu_sheaf() on the cpu we make + * sure the __kfree_rcu_sheaf() finished its call_rcu() + */ - for_each_online_cpu(cpu) { - sfw = &per_cpu(slub_flush, cpu); + INIT_WORK(&sfw->work, flush_rcu_sheaf); + sfw->s = s; + queue_work_on(cpu, flushwq, &sfw->work); + } - /* - * we don't check if rcu_free sheaf exists - racing - * __kfree_rcu_sheaf() might have just removed it. - * by executing flush_rcu_sheaf() on the cpu we make - * sure the __kfree_rcu_sheaf() finished its call_rcu() - */ + for_each_online_cpu(cpu) { + sfw = &per_cpu(slub_flush, cpu); + flush_work(&sfw->work); + } - INIT_WORK(&sfw->work, flush_rcu_sheaf); - sfw->s = s; - queue_work_on(cpu, flushwq, &sfw->work); - } + mutex_unlock(&flush_lock); +} - for_each_online_cpu(cpu) { - sfw = &per_cpu(slub_flush, cpu); - flush_work(&sfw->work); - } +void flush_all_rcu_sheaves(void) +{ + struct kmem_cache *s; + + cpus_read_lock(); + mutex_lock(&slab_mutex); - mutex_unlock(&flush_lock); + list_for_each_entry(s, &slab_caches, list) { + if (!s->cpu_sheaves) + continue; + flush_rcu_sheaves_on_cache(s); } mutex_unlock(&slab_mutex); @@ -4365,7 +4450,6 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) return true; } -#ifndef CONFIG_SLUB_TINY static inline bool __update_cpu_freelist_fast(struct kmem_cache *s, void *freelist_old, void *freelist_new, @@ -4629,7 +4713,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, pc.orig_size = orig_size; slab = get_partial(s, node, &pc); if (slab) { - if (kmem_cache_debug(s)) { + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { freelist = pc.object; /* * For debug caches here we had to go through @@ -4667,7 +4751,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, stat(s, ALLOC_SLAB); - if (kmem_cache_debug(s)) { + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); if (unlikely(!freelist)) { @@ -4879,32 +4963,6 @@ static __always_inline void *__slab_alloc_node(struct kmem_cache *s, return object; } -#else /* CONFIG_SLUB_TINY */ -static void *__slab_alloc_node(struct kmem_cache *s, - gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) -{ - struct partial_context pc; - struct slab *slab; - void *object; - - pc.flags = gfpflags; - pc.orig_size = orig_size; - slab = get_partial(s, node, &pc); - - if (slab) - return pc.object; - - slab = new_slab(s, gfpflags, node); - if (unlikely(!slab)) { - slab_out_of_memory(s, gfpflags, node); - return NULL; - } - - object = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); - - return object; -} -#endif /* CONFIG_SLUB_TINY */ /* * If the object has been wiped upon free, make sure it's fully initialized by @@ -5687,8 +5745,12 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) if (unlikely(!size)) return ZERO_SIZE_PTR; - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - /* kmalloc_nolock() in PREEMPT_RT is not supported from irq */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && !preemptible()) + /* + * kmalloc_nolock() in PREEMPT_RT is not supported from + * non-preemptible context because local_lock becomes a + * sleeping lock on RT. + */ return NULL; retry: if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) @@ -5723,9 +5785,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) * it did local_lock_irqsave(&s->cpu_slab->lock, flags). * In this case fast path with __update_cpu_freelist_fast() is not safe. */ -#ifndef CONFIG_SLUB_TINY if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) -#endif ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); if (PTR_ERR(ret) == -EBUSY) { @@ -6107,7 +6167,7 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) if (put_fail) stat(s, BARN_PUT_FAIL); - if (!sheaf_flush_main(s)) + if (!sheaf_try_flush_main(s)) return NULL; if (!local_trylock(&s->cpu_sheaves->lock)) @@ -6223,11 +6283,29 @@ static void rcu_free_sheaf(struct rcu_head *head) free_empty_sheaf(s, sheaf); } +/* + * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since + * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), + * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids + * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. + * + * However, lockdep still complains that it is invalid to acquire spinlock_t + * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a + * spinning lock. Tell lockdep that acquiring spinlock_t is valid here + * by temporarily raising the wait-type to LD_WAIT_CONFIG. + */ +static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); + bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; + if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) + return false; + + lock_map_acquire_try(&kfree_rcu_sheaf_map); + if (!local_trylock(&s->cpu_sheaves->lock)) goto fail; @@ -6304,10 +6382,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); stat(s, FREE_RCU_SHEAF); + lock_map_release(&kfree_rcu_sheaf_map); return true; fail: stat(s, FREE_RCU_SHEAF_FAIL); + lock_map_release(&kfree_rcu_sheaf_map); return false; } @@ -6487,14 +6567,10 @@ static void free_deferred_objects(struct irq_work *work) llist_for_each_safe(pos, t, llnode) { struct slab *slab = container_of(pos, struct slab, llnode); -#ifdef CONFIG_SLUB_TINY - free_slab(slab->slab_cache, slab); -#else if (slab->frozen) deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); else free_slab(slab->slab_cache, slab); -#endif } } @@ -6504,6 +6580,8 @@ static void defer_free(struct kmem_cache *s, void *head) guard(preempt)(); + head = kasan_reset_tag(head); + df = this_cpu_ptr(&defer_free_objects); if (llist_add(head + s->offset, &df->objects)) irq_work_queue(&df->work); @@ -6530,7 +6608,6 @@ void defer_free_barrier(void) irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); } -#ifndef CONFIG_SLUB_TINY /* * Fastpath with forced inlining to produce a kfree and kmem_cache_free that * can perform fastpath freeing without additional function calls. @@ -6623,14 +6700,6 @@ static __always_inline void do_slab_free(struct kmem_cache *s, } stat_add(s, FREE_FASTPATH, cnt); } -#else /* CONFIG_SLUB_TINY */ -static void do_slab_free(struct kmem_cache *s, - struct slab *slab, void *head, void *tail, - int cnt, unsigned long addr) -{ - __slab_free(s, slab, head, tail, cnt, addr); -} -#endif /* CONFIG_SLUB_TINY */ static __fastpath_inline void slab_free(struct kmem_cache *s, struct slab *slab, void *object, @@ -6656,8 +6725,12 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object, static noinline void memcg_alloc_abort_single(struct kmem_cache *s, void *object) { + struct slab *slab = virt_to_slab(object); + + alloc_tagging_slab_free_hook(s, slab, &object, 1); + if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false))) - do_slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_); + do_slab_free(s, slab, object, object, 1, _RET_IP_); } #endif @@ -6907,11 +6980,7 @@ void kfree_nolock(const void *object) * since kasan quarantine takes locks and not supported from NMI. */ kasan_slab_free(s, x, false, false, /* skip quarantine */true); -#ifndef CONFIG_SLUB_TINY do_slab_free(s, slab, x, x, 0, _RET_IP_); -#else - defer_free(s, x); -#endif } EXPORT_SYMBOL_GPL(kfree_nolock); @@ -7361,7 +7430,6 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) } EXPORT_SYMBOL(kmem_cache_free_bulk); -#ifndef CONFIG_SLUB_TINY static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p) @@ -7432,35 +7500,6 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, return 0; } -#else /* CONFIG_SLUB_TINY */ -static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, - size_t size, void **p) -{ - int i; - - for (i = 0; i < size; i++) { - void *object = kfence_alloc(s, s->object_size, flags); - - if (unlikely(object)) { - p[i] = object; - continue; - } - - p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE, - _RET_IP_, s->object_size); - if (unlikely(!p[i])) - goto error; - - maybe_wipe_obj_freeptr(s, p[i]); - } - - return i; - -error: - __kmem_cache_free_bulk(s, i, p); - return 0; -} -#endif /* CONFIG_SLUB_TINY */ /* Note that interrupts must be enabled when calling this function. */ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, @@ -7651,7 +7690,6 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn) barn_init(barn); } -#ifndef CONFIG_SLUB_TINY static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) { BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < @@ -7672,12 +7710,6 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) return 1; } -#else -static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) -{ - return 1; -} -#endif /* CONFIG_SLUB_TINY */ static int init_percpu_sheaves(struct kmem_cache *s) { @@ -7767,13 +7799,11 @@ void __kmem_cache_release(struct kmem_cache *s) cache_random_seq_destroy(s); if (s->cpu_sheaves) pcs_destroy(s); -#ifndef CONFIG_SLUB_TINY #ifdef CONFIG_PREEMPT_RT if (s->cpu_slab) lockdep_unregister_key(&s->lock_key); #endif free_percpu(s->cpu_slab); -#endif free_kmem_cache_nodes(s); } @@ -7932,7 +7962,7 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) /* Save the original kmalloc request size */ if (flags & SLAB_KMALLOC) - size += sizeof(unsigned int); + size += sizeof(unsigned long); } #endif @@ -8516,9 +8546,10 @@ void __init kmem_cache_init(void) void __init kmem_cache_init_late(void) { -#ifndef CONFIG_SLUB_TINY flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); WARN_ON(!flushwq); +#ifdef CONFIG_SLAB_FREELIST_RANDOM + prandom_init_once(&slab_rnd_state); #endif } diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index dbd8daccade28..37522d6cb3988 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -439,7 +439,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, return -ENOMEM; pmd = pmd_offset(pud, addr); - if (pmd_none(READ_ONCE(*pmd))) { + if (pmd_none(pmdp_get(pmd))) { void *p; p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap); diff --git a/mm/swap.h b/mm/swap.h index 8d8efdf1297a7..c6bdc5d215934 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -198,7 +198,7 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug); void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug); /* linux/mm/swap_state.c */ -extern struct address_space swap_space __ro_after_init; +extern struct address_space swap_space __read_mostly; static inline struct address_space *swap_address_space(swp_entry_t entry) { return &swap_space; diff --git a/mm/swap_state.c b/mm/swap_state.c index f4980dde5394b..b8d7744b4436a 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -37,8 +37,7 @@ static const struct address_space_operations swap_aops = { #endif }; -/* Set swap_space as read only as swap cache is handled by swap table */ -struct address_space swap_space __ro_after_init = { +struct address_space swap_space __read_mostly = { .a_ops = &swap_aops, }; diff --git a/mm/swapfile.c b/mm/swapfile.c index a1b4b9d80e3b5..89746abc47373 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1101,13 +1101,6 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int o goto done; } - /* - * We don't have free cluster but have some clusters in discarding, - * do discard now and reclaim them. - */ - if ((si->flags & SWP_PAGE_DISCARD) && swap_do_scheduled_discard(si)) - goto new_cluster; - if (order) goto done; @@ -1394,6 +1387,33 @@ static bool swap_alloc_slow(swp_entry_t *entry, return false; } +/* + * Discard pending clusters in a synchronized way when under high pressure. + * Return: true if any cluster is discarded. + */ +static bool swap_sync_discard(void) +{ + bool ret = false; + int nid = numa_node_id(); + struct swap_info_struct *si, *next; + + spin_lock(&swap_avail_lock); + plist_for_each_entry_safe(si, next, &swap_avail_heads[nid], avail_lists[nid]) { + spin_unlock(&swap_avail_lock); + if (get_swap_device_info(si)) { + if (si->flags & SWP_PAGE_DISCARD) + ret = swap_do_scheduled_discard(si); + put_swap_device(si); + } + if (ret) + return true; + spin_lock(&swap_avail_lock); + } + spin_unlock(&swap_avail_lock); + + return false; +} + /** * folio_alloc_swap - allocate swap space for a folio * @folio: folio we want to move to swap @@ -1432,11 +1452,17 @@ int folio_alloc_swap(struct folio *folio, gfp_t gfp) } } +again: local_lock(&percpu_swap_cluster.lock); if (!swap_alloc_fast(&entry, order)) swap_alloc_slow(&entry, order); local_unlock(&percpu_swap_cluster.lock); + if (unlikely(!order && !entry.val)) { + if (swap_sync_discard()) + goto again; + } + /* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */ if (mem_cgroup_try_charge_swap(folio, entry)) goto out_free; @@ -1677,7 +1703,7 @@ static bool swap_entries_put_map_nr(struct swap_info_struct *si, /* * Check if it's the last ref of swap entry in the freeing path. - * Qualified vlaue includes 1, SWAP_HAS_CACHE or SWAP_MAP_SHMEM. + * Qualified value includes 1, SWAP_HAS_CACHE or SWAP_MAP_SHMEM. */ static inline bool __maybe_unused swap_is_last_ref(unsigned char count) { diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index af61b95c89e4e..1b0fa239aa752 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -569,7 +569,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( * in the case of shared pmds. fault mutex prevents * races with other faulting threads. */ - idx = linear_page_index(dst_vma, dst_addr); + idx = hugetlb_linear_page_index(dst_vma, dst_addr); mapping = dst_vma->vm_file->f_mapping; hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_lock(&hugetlb_fault_mutex_table[hash]); @@ -1578,7 +1578,7 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, /* * For now, we keep it simple and only move between writable VMAs. - * Access flags are equal, therefore cheching only the source is enough. + * Access flags are equal, therefore checking only the source is enough. */ if (!(src_vma->vm_flags & VM_WRITE)) return -EINVAL; diff --git a/mm/vma.c b/mm/vma.c index abe0da33c8446..5815ae9e57703 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -65,18 +65,13 @@ struct mmap_state { .state = VMA_MERGE_START, \ } -/* - * If, at any point, the VMA had unCoW'd mappings from parents, it will maintain - * more than one anon_vma_chain connecting it to more than one anon_vma. A merge - * would mean a wider range of folios sharing the root anon_vma lock, and thus - * potential lock contention, we do not wish to encourage merging such that this - * scales to a problem. - */ -static bool vma_had_uncowed_parents(struct vm_area_struct *vma) +/* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ +static bool vma_is_fork_child(struct vm_area_struct *vma) { /* * The list_is_singular() test is to avoid merging VMA cloned from - * parents. This can improve scalability caused by anon_vma lock. + * parents. This can improve scalability caused by the anon_vma root + * lock. */ return vma && vma->anon_vma && !list_is_singular(&vma->anon_vma_chain); } @@ -109,7 +104,7 @@ static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_nex static bool is_mergeable_anon_vma(struct vma_merge_struct *vmg, bool merge_next) { struct vm_area_struct *tgt = merge_next ? vmg->next : vmg->prev; - struct vm_area_struct *src = vmg->middle; /* exisitng merge case. */ + struct vm_area_struct *src = vmg->middle; /* existing merge case. */ struct anon_vma *tgt_anon = tgt->anon_vma; struct anon_vma *src_anon = vmg->anon_vma; @@ -121,11 +116,19 @@ static bool is_mergeable_anon_vma(struct vma_merge_struct *vmg, bool merge_next) VM_WARN_ON(src && src_anon != src->anon_vma); /* Case 1 - we will dup_anon_vma() from src into tgt. */ - if (!tgt_anon && src_anon) - return !vma_had_uncowed_parents(src); + if (!tgt_anon && src_anon) { + struct vm_area_struct *copied_from = vmg->copied_from; + + if (vma_is_fork_child(src)) + return false; + if (vma_is_fork_child(copied_from)) + return false; + + return true; + } /* Case 2 - we will simply use tgt's anon_vma. */ if (tgt_anon && !src_anon) - return !vma_had_uncowed_parents(tgt); + return !vma_is_fork_child(tgt); /* Case 3 - the anon_vma's are already shared. */ return src_anon == tgt_anon; } @@ -798,7 +801,7 @@ static bool can_merge_remove_vma(struct vm_area_struct *vma) * Returns: The merged VMA if merge succeeds, or NULL otherwise. * * ASSUMPTIONS: - * - The caller must assign the VMA to be modifed to @vmg->middle. + * - The caller must assign the VMA to be modified to @vmg->middle. * - The caller must have set @vmg->prev to the previous VMA, if there is one. * - The caller must not set @vmg->next, as we determine this. * - The caller must hold a WRITE lock on the mm_struct->mmap_lock. @@ -835,6 +838,8 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( VM_WARN_ON_VMG(middle && !(vma_iter_addr(vmg->vmi) >= middle->vm_start && vma_iter_addr(vmg->vmi) < middle->vm_end), vmg); + /* An existing merge can never be used by the mremap() logic. */ + VM_WARN_ON_VMG(vmg->copied_from, vmg); vmg->state = VMA_MERGE_NOMERGE; @@ -1101,6 +1106,33 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) return NULL; } +/* + * vma_merge_copied_range - Attempt to merge a VMA that is being copied by + * mremap() + * + * @vmg: Describes the VMA we are adding, in the copied-to range @vmg->start to + * @vmg->end (exclusive), which we try to merge with any adjacent VMAs if + * possible. + * + * vmg->prev, next, start, end, pgoff should all be relative to the COPIED TO + * range, i.e. the target range for the VMA. + * + * Returns: In instances where no merge was possible, NULL. Otherwise, a pointer + * to the VMA we expanded. + * + * ASSUMPTIONS: Same as vma_merge_new_range(), except vmg->middle must contain + * the copied-from VMA. + */ +static struct vm_area_struct *vma_merge_copied_range(struct vma_merge_struct *vmg) +{ + /* We must have a copied-from VMA. */ + VM_WARN_ON_VMG(!vmg->middle, vmg); + + vmg->copied_from = vmg->middle; + vmg->middle = NULL; + return vma_merge_new_range(vmg); +} + /* * vma_expand - Expand an existing VMA * @@ -1123,38 +1155,45 @@ int vma_expand(struct vma_merge_struct *vmg) bool remove_next = false; struct vm_area_struct *target = vmg->target; struct vm_area_struct *next = vmg->next; + int ret = 0; VM_WARN_ON_VMG(!target, vmg); mmap_assert_write_locked(vmg->mm); - vma_start_write(target); - if (next && (target != next) && (vmg->end == next->vm_end)) { - int ret; + if (next && target != next && vmg->end == next->vm_end) remove_next = true; - /* This should already have been checked by this point. */ - VM_WARN_ON_VMG(!can_merge_remove_vma(next), vmg); - vma_start_write(next); - /* - * In this case we don't report OOM, so vmg->give_up_on_mm is - * safe. - */ - ret = dup_anon_vma(target, next, &anon_dup); - if (ret) - return ret; - } + /* We must have a target. */ + VM_WARN_ON_VMG(!target, vmg); + /* This should have already been checked by this point. */ + VM_WARN_ON_VMG(remove_next && !can_merge_remove_vma(next), vmg); /* Not merging but overwriting any part of next is not handled. */ VM_WARN_ON_VMG(next && !remove_next && next != target && vmg->end > next->vm_start, vmg); - /* Only handles expanding */ + /* Only handles expanding. */ VM_WARN_ON_VMG(target->vm_start < vmg->start || target->vm_end > vmg->end, vmg); + /* + * If we are removing the next VMA or copying from a VMA + * (e.g. mremap()'ing), we must propagate anon_vma state. + * + * Note that, by convention, callers ignore OOM for this case, so + * we don't need to account for vmg->give_up_on_mm here. + */ if (remove_next) - vmg->__remove_next = true; + ret = dup_anon_vma(target, next, &anon_dup); + if (!ret && vmg->copied_from) + ret = dup_anon_vma(target, vmg->copied_from, &anon_dup); + if (ret) + return ret; + if (remove_next) { + vma_start_write(next); + vmg->__remove_next = true; + } if (commit_merge(vmg)) goto nomem; @@ -1837,10 +1876,9 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (new_vma && new_vma->vm_start < addr + len) return NULL; /* should never get here */ - vmg.middle = NULL; /* New VMA range. */ vmg.pgoff = pgoff; vmg.next = vma_iter_next_rewind(&vmi, NULL); - new_vma = vma_merge_new_range(&vmg); + new_vma = vma_merge_copied_range(&vmg); if (new_vma) { /* diff --git a/mm/vma.h b/mm/vma.h index 9183fe5490090..d73e1b324bfd1 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -106,6 +106,9 @@ struct vma_merge_struct { struct anon_vma_name *anon_name; enum vma_merge_state state; + /* If copied from (i.e. mremap()'d) the VMA from which we are copying. */ + struct vm_area_struct *copied_from; + /* Flags which callers can use to modify merge behaviour: */ /* diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 798b2ed21e460..e2f526ad7abba 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2241,11 +2241,14 @@ decay_va_pool_node(struct vmap_node *vn, bool full_decay) reclaim_list_global(&decay_list); } +#define KASAN_RELEASE_BATCH_SIZE 32 + static void kasan_release_vmalloc_node(struct vmap_node *vn) { struct vmap_area *va; unsigned long start, end; + unsigned int batch_count = 0; start = list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start; end = list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end; @@ -2255,6 +2258,11 @@ kasan_release_vmalloc_node(struct vmap_node *vn) kasan_release_vmalloc(va->va_start, va->va_end, va->va_start, va->va_end, KASAN_VMALLOC_PAGE_RANGE); + + if (need_resched() || (++batch_count >= KASAN_RELEASE_BATCH_SIZE)) { + cond_resched(); + batch_count = 0; + } } kasan_release_vmalloc(start, end, start, end, KASAN_VMALLOC_TLB_FLUSH); @@ -4167,7 +4175,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align if (want_init_on_free() || want_init_on_alloc(flags)) memset((void *)p + size, 0, old_size - size); vm->requested_size = size; - kasan_poison_vmalloc(p + size, old_size - size); + kasan_vrealloc(p, old_size, size); return (void *)p; } @@ -4175,14 +4183,13 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align * We already have the bytes available in the allocation; use them. */ if (size <= alloced_size) { - kasan_unpoison_vmalloc(p + old_size, size - old_size, - KASAN_VMALLOC_PROT_NORMAL); /* * No need to zero memory here, as unused memory will have * already been zeroed at initial allocation time or during * realloc shrink time. */ vm->requested_size = size; + kasan_vrealloc(p, old_size, size); return (void *)p; } @@ -4870,9 +4877,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, * With hardware tag-based KASAN, marking is skipped for * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). */ - for (area = 0; area < nr_vms; area++) - vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr, - vms[area]->size, KASAN_VMALLOC_PROT_NORMAL); + kasan_unpoison_vmap_areas(vms, nr_vms, KASAN_VMALLOC_PROT_NORMAL); kfree(vas); return vms; diff --git a/mm/vmscan.c b/mm/vmscan.c index b2fc8b626d3df..95b1179a14e78 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -344,19 +344,21 @@ static void flush_reclaim_state(struct scan_control *sc) static bool can_demote(int nid, struct scan_control *sc, struct mem_cgroup *memcg) { - int demotion_nid; + struct pglist_data *pgdat = NODE_DATA(nid); + nodemask_t allowed_mask; - if (!numa_demotion_enabled) + if (!pgdat || !numa_demotion_enabled) return false; if (sc && sc->no_demotion) return false; - demotion_nid = next_demotion_node(nid); - if (demotion_nid == NUMA_NO_NODE) + node_get_allowed_targets(pgdat, &allowed_mask); + if (nodes_empty(allowed_mask)) return false; - /* If demotion node isn't in the cgroup's mems_allowed, fall back */ - return mem_cgroup_node_allowed(memcg, demotion_nid); + /* Filter out nodes that are not in cgroup's mems_allowed. */ + mem_cgroup_node_filter_allowed(memcg, &allowed_mask); + return !nodes_empty(allowed_mask); } static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, @@ -1042,9 +1044,10 @@ static struct folio *alloc_demote_folio(struct folio *src, * Folios which are not demoted are left on @demote_folios. */ static unsigned int demote_folio_list(struct list_head *demote_folios, - struct pglist_data *pgdat) + struct pglist_data *pgdat, + struct mem_cgroup *memcg) { - int target_nid = next_demotion_node(pgdat->node_id); + int target_nid; unsigned int nr_succeeded; nodemask_t allowed_mask; @@ -1056,7 +1059,6 @@ static unsigned int demote_folio_list(struct list_head *demote_folios, */ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | __GFP_NOMEMALLOC | GFP_NOWAIT, - .nid = target_nid, .nmask = &allowed_mask, .reason = MR_DEMOTION, }; @@ -1064,10 +1066,18 @@ static unsigned int demote_folio_list(struct list_head *demote_folios, if (list_empty(demote_folios)) return 0; - if (target_nid == NUMA_NO_NODE) + node_get_allowed_targets(pgdat, &allowed_mask); + mem_cgroup_node_filter_allowed(memcg, &allowed_mask); + if (nodes_empty(allowed_mask)) return 0; - node_get_allowed_targets(pgdat, &allowed_mask); + target_nid = next_demotion_node(pgdat->node_id); + if (target_nid == NUMA_NO_NODE) + /* No lower-tier nodes or nodes were hot-unplugged. */ + return 0; + if (!node_isset(target_nid, allowed_mask)) + target_nid = node_random(&allowed_mask); + mtc.nid = target_nid; /* Demotion ignores all cpuset and mempolicy settings */ migrate_pages(demote_folios, alloc_demote_folio, NULL, @@ -1601,7 +1611,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, /* 'folio_list' is always empty here */ /* Migrate folios selected for demotion */ - nr_demoted = demote_folio_list(&demote_folios, pgdat); + nr_demoted = demote_folio_list(&demote_folios, pgdat, memcg); nr_reclaimed += nr_demoted; stat->nr_demoted += nr_demoted; /* Folios that could not be demoted are still in @demote_folios */ @@ -3773,7 +3783,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, pud = pud_offset(p4d, start & P4D_MASK); restart: for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { - pud_t val = READ_ONCE(pud[i]); + pud_t val = pudp_get(pud + i); next = pud_addr_end(addr, end); @@ -7740,6 +7750,17 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) return ret; } +#else + +static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, + unsigned long nr_pages, + struct scan_control *sc) +{ + return 0; +} + +#endif + enum { MEMORY_RECLAIM_SWAPPINESS = 0, MEMORY_RECLAIM_SWAPPINESS_MAX, @@ -7847,8 +7868,6 @@ int user_proactive_reclaim(char *buf, return 0; } -#endif - /** * check_move_unevictable_folios - Move evictable folios to appropriate zone * lru list diff --git a/mm/vmstat.c b/mm/vmstat.c index bb09c032eecfa..98855f31294dd 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -771,25 +771,25 @@ EXPORT_SYMBOL(dec_node_page_state); /* * Fold a differential into the global counters. - * Returns the number of counters updated. + * Returns whether counters were updated. */ static int fold_diff(int *zone_diff, int *node_diff) { int i; - int changes = 0; + bool changed = false; for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) if (zone_diff[i]) { atomic_long_add(zone_diff[i], &vm_zone_stat[i]); - changes++; + changed = true; } for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) if (node_diff[i]) { atomic_long_add(node_diff[i], &vm_node_stat[i]); - changes++; + changed = true; } - return changes; + return changed; } /* @@ -806,16 +806,16 @@ static int fold_diff(int *zone_diff, int *node_diff) * with the global counters. These could cause remote node cache line * bouncing and will have to be only done when necessary. * - * The function returns the number of global counters updated. + * The function returns whether global counters were updated. */ -static int refresh_cpu_vm_stats(bool do_pagesets) +static bool refresh_cpu_vm_stats(bool do_pagesets) { struct pglist_data *pgdat; struct zone *zone; int i; int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; - int changes = 0; + bool changed = false; for_each_populated_zone(zone) { struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; @@ -839,7 +839,8 @@ static int refresh_cpu_vm_stats(bool do_pagesets) if (do_pagesets) { cond_resched(); - changes += decay_pcp_high(zone, this_cpu_ptr(pcp)); + if (decay_pcp_high(zone, this_cpu_ptr(pcp))) + changed = true; #ifdef CONFIG_NUMA /* * Deal with draining the remote pageset of this @@ -861,13 +862,13 @@ static int refresh_cpu_vm_stats(bool do_pagesets) } if (__this_cpu_dec_return(pcp->expire)) { - changes++; + changed = true; continue; } if (__this_cpu_read(pcp->count)) { drain_zone_pages(zone, this_cpu_ptr(pcp)); - changes++; + changed = true; } #endif } @@ -887,8 +888,9 @@ static int refresh_cpu_vm_stats(bool do_pagesets) } } - changes += fold_diff(global_zone_diff, global_node_diff); - return changes; + if (fold_diff(global_zone_diff, global_node_diff)) + changed = true; + return changed; } /* diff --git a/mm/zswap.c b/mm/zswap.c index c1af782e54ec6..12fb0e3954129 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -787,7 +787,7 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) return 0; fail: - if (acomp) + if (!IS_ERR_OR_NULL(acomp)) crypto_free_acomp(acomp); kfree(buffer); return ret; diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c index b9ff69c7522a1..068d57515dd58 100644 --- a/net/9p/trans_xen.c +++ b/net/9p/trans_xen.c @@ -274,45 +274,52 @@ static void xen_9pfs_front_free(struct xen_9pfs_front_priv *priv) { int i, j; - write_lock(&xen_9pfs_lock); - list_del(&priv->list); - write_unlock(&xen_9pfs_lock); - - for (i = 0; i < XEN_9PFS_NUM_RINGS; i++) { - struct xen_9pfs_dataring *ring = &priv->rings[i]; - - cancel_work_sync(&ring->work); - - if (!priv->rings[i].intf) - break; - if (priv->rings[i].irq > 0) - unbind_from_irqhandler(priv->rings[i].irq, ring); - if (priv->rings[i].data.in) { - for (j = 0; - j < (1 << priv->rings[i].intf->ring_order); - j++) { - grant_ref_t ref; - - ref = priv->rings[i].intf->ref[j]; - gnttab_end_foreign_access(ref, NULL); - } - free_pages_exact(priv->rings[i].data.in, + if (priv->rings) { + for (i = 0; i < XEN_9PFS_NUM_RINGS; i++) { + struct xen_9pfs_dataring *ring = &priv->rings[i]; + + cancel_work_sync(&ring->work); + + if (!priv->rings[i].intf) + break; + if (priv->rings[i].irq > 0) + unbind_from_irqhandler(priv->rings[i].irq, ring); + if (priv->rings[i].data.in) { + for (j = 0; + j < (1 << priv->rings[i].intf->ring_order); + j++) { + grant_ref_t ref; + + ref = priv->rings[i].intf->ref[j]; + gnttab_end_foreign_access(ref, NULL); + } + free_pages_exact(priv->rings[i].data.in, 1UL << (priv->rings[i].intf->ring_order + XEN_PAGE_SHIFT)); + } + gnttab_end_foreign_access(priv->rings[i].ref, NULL); + free_page((unsigned long)priv->rings[i].intf); } - gnttab_end_foreign_access(priv->rings[i].ref, NULL); - free_page((unsigned long)priv->rings[i].intf); + kfree(priv->rings); } - kfree(priv->rings); kfree(priv->tag); kfree(priv); } static void xen_9pfs_front_remove(struct xenbus_device *dev) { - struct xen_9pfs_front_priv *priv = dev_get_drvdata(&dev->dev); + struct xen_9pfs_front_priv *priv; + write_lock(&xen_9pfs_lock); + priv = dev_get_drvdata(&dev->dev); + if (priv == NULL) { + write_unlock(&xen_9pfs_lock); + return; + } dev_set_drvdata(&dev->dev, NULL); + list_del(&priv->list); + write_unlock(&xen_9pfs_lock); + xen_9pfs_front_free(priv); } @@ -379,7 +386,7 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) { int ret, i; struct xenbus_transaction xbt; - struct xen_9pfs_front_priv *priv = dev_get_drvdata(&dev->dev); + struct xen_9pfs_front_priv *priv; char *versions, *v; unsigned int max_rings, max_ring_order, len = 0; @@ -407,6 +414,10 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) if (p9_xen_trans.maxsize > XEN_FLEX_RING_SIZE(max_ring_order)) p9_xen_trans.maxsize = XEN_FLEX_RING_SIZE(max_ring_order) / 2; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; priv->rings = kcalloc(XEN_9PFS_NUM_RINGS, sizeof(*priv->rings), GFP_KERNEL); if (!priv->rings) { @@ -465,6 +476,11 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) goto error; } + write_lock(&xen_9pfs_lock); + dev_set_drvdata(&dev->dev, priv); + list_add_tail(&priv->list, &xen_9pfs_devs); + write_unlock(&xen_9pfs_lock); + xenbus_switch_state(dev, XenbusStateInitialised); return 0; @@ -479,19 +495,6 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) static int xen_9pfs_front_probe(struct xenbus_device *dev, const struct xenbus_device_id *id) { - struct xen_9pfs_front_priv *priv = NULL; - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->dev = dev; - dev_set_drvdata(&dev->dev, priv); - - write_lock(&xen_9pfs_lock); - list_add_tail(&priv->list, &xen_9pfs_devs); - write_unlock(&xen_9pfs_lock); - return 0; } diff --git a/net/atm/lec.c b/net/atm/lec.c index afb8d3eb21850..b6f764e524f7c 100644 --- a/net/atm/lec.c +++ b/net/atm/lec.c @@ -154,10 +154,19 @@ static void lec_handle_bridge(struct sk_buff *skb, struct net_device *dev) /* 0x01 is topology change */ priv = netdev_priv(dev); - atm_force_charge(priv->lecd, skb2->truesize); - sk = sk_atm(priv->lecd); - skb_queue_tail(&sk->sk_receive_queue, skb2); - sk->sk_data_ready(sk); + struct atm_vcc *vcc; + + rcu_read_lock(); + vcc = rcu_dereference(priv->lecd); + if (vcc) { + atm_force_charge(vcc, skb2->truesize); + sk = sk_atm(vcc); + skb_queue_tail(&sk->sk_receive_queue, skb2); + sk->sk_data_ready(sk); + } else { + dev_kfree_skb(skb2); + } + rcu_read_unlock(); } } #endif /* IS_ENABLED(CONFIG_BRIDGE) */ @@ -216,7 +225,7 @@ static netdev_tx_t lec_start_xmit(struct sk_buff *skb, int is_rdesc; pr_debug("called\n"); - if (!priv->lecd) { + if (!rcu_access_pointer(priv->lecd)) { pr_info("%s:No lecd attached\n", dev->name); dev->stats.tx_errors++; netif_stop_queue(dev); @@ -449,10 +458,19 @@ static int lec_atm_send(struct atm_vcc *vcc, struct sk_buff *skb) break; skb2->len = sizeof(struct atmlec_msg); skb_copy_to_linear_data(skb2, mesg, sizeof(*mesg)); - atm_force_charge(priv->lecd, skb2->truesize); - sk = sk_atm(priv->lecd); - skb_queue_tail(&sk->sk_receive_queue, skb2); - sk->sk_data_ready(sk); + struct atm_vcc *vcc; + + rcu_read_lock(); + vcc = rcu_dereference(priv->lecd); + if (vcc) { + atm_force_charge(vcc, skb2->truesize); + sk = sk_atm(vcc); + skb_queue_tail(&sk->sk_receive_queue, skb2); + sk->sk_data_ready(sk); + } else { + dev_kfree_skb(skb2); + } + rcu_read_unlock(); } } #endif /* IS_ENABLED(CONFIG_BRIDGE) */ @@ -468,23 +486,16 @@ static int lec_atm_send(struct atm_vcc *vcc, struct sk_buff *skb) static void lec_atm_close(struct atm_vcc *vcc) { - struct sk_buff *skb; struct net_device *dev = (struct net_device *)vcc->proto_data; struct lec_priv *priv = netdev_priv(dev); - priv->lecd = NULL; + rcu_assign_pointer(priv->lecd, NULL); + synchronize_rcu(); /* Do something needful? */ netif_stop_queue(dev); lec_arp_destroy(priv); - if (skb_peek(&sk_atm(vcc)->sk_receive_queue)) - pr_info("%s closing with messages pending\n", dev->name); - while ((skb = skb_dequeue(&sk_atm(vcc)->sk_receive_queue))) { - atm_return(vcc, skb->truesize); - dev_kfree_skb(skb); - } - pr_info("%s: Shut down!\n", dev->name); module_put(THIS_MODULE); } @@ -510,12 +521,14 @@ send_to_lecd(struct lec_priv *priv, atmlec_msg_type type, const unsigned char *mac_addr, const unsigned char *atm_addr, struct sk_buff *data) { + struct atm_vcc *vcc; struct sock *sk; struct sk_buff *skb; struct atmlec_msg *mesg; - if (!priv || !priv->lecd) + if (!priv || !rcu_access_pointer(priv->lecd)) return -1; + skb = alloc_skb(sizeof(struct atmlec_msg), GFP_ATOMIC); if (!skb) return -1; @@ -532,18 +545,27 @@ send_to_lecd(struct lec_priv *priv, atmlec_msg_type type, if (atm_addr) memcpy(&mesg->content.normal.atm_addr, atm_addr, ATM_ESA_LEN); - atm_force_charge(priv->lecd, skb->truesize); - sk = sk_atm(priv->lecd); + rcu_read_lock(); + vcc = rcu_dereference(priv->lecd); + if (!vcc) { + rcu_read_unlock(); + kfree_skb(skb); + return -1; + } + + atm_force_charge(vcc, skb->truesize); + sk = sk_atm(vcc); skb_queue_tail(&sk->sk_receive_queue, skb); sk->sk_data_ready(sk); if (data != NULL) { pr_debug("about to send %d bytes of data\n", data->len); - atm_force_charge(priv->lecd, data->truesize); + atm_force_charge(vcc, data->truesize); skb_queue_tail(&sk->sk_receive_queue, data); sk->sk_data_ready(sk); } + rcu_read_unlock(); return 0; } @@ -618,7 +640,7 @@ static void lec_push(struct atm_vcc *vcc, struct sk_buff *skb) atm_return(vcc, skb->truesize); if (*(__be16 *) skb->data == htons(priv->lecid) || - !priv->lecd || !(dev->flags & IFF_UP)) { + !rcu_access_pointer(priv->lecd) || !(dev->flags & IFF_UP)) { /* * Probably looping back, or if lecd is missing, * lecd has gone down @@ -753,12 +775,12 @@ static int lecd_attach(struct atm_vcc *vcc, int arg) priv = netdev_priv(dev_lec[i]); } else { priv = netdev_priv(dev_lec[i]); - if (priv->lecd) + if (rcu_access_pointer(priv->lecd)) return -EADDRINUSE; } lec_arp_init(priv); priv->itfnum = i; /* LANE2 addition */ - priv->lecd = vcc; + rcu_assign_pointer(priv->lecd, vcc); vcc->dev = &lecatm_dev; vcc_insert_socket(sk_atm(vcc)); @@ -1260,24 +1282,28 @@ static void lec_arp_clear_vccs(struct lec_arp_table *entry) struct lec_vcc_priv *vpriv = LEC_VCC_PRIV(vcc); struct net_device *dev = (struct net_device *)vcc->proto_data; - vcc->pop = vpriv->old_pop; - if (vpriv->xoff) - netif_wake_queue(dev); - kfree(vpriv); - vcc->user_back = NULL; - vcc->push = entry->old_push; - vcc_release_async(vcc, -EPIPE); + if (vpriv) { + vcc->pop = vpriv->old_pop; + if (vpriv->xoff) + netif_wake_queue(dev); + kfree(vpriv); + vcc->user_back = NULL; + vcc->push = entry->old_push; + vcc_release_async(vcc, -EPIPE); + } entry->vcc = NULL; } if (entry->recv_vcc) { struct atm_vcc *vcc = entry->recv_vcc; struct lec_vcc_priv *vpriv = LEC_VCC_PRIV(vcc); - kfree(vpriv); - vcc->user_back = NULL; + if (vpriv) { + kfree(vpriv); + vcc->user_back = NULL; - entry->recv_vcc->push = entry->old_recv_push; - vcc_release_async(entry->recv_vcc, -EPIPE); + entry->recv_vcc->push = entry->old_recv_push; + vcc_release_async(entry->recv_vcc, -EPIPE); + } entry->recv_vcc = NULL; } } diff --git a/net/atm/lec.h b/net/atm/lec.h index be0e2667bd8c3..ec85709bf8185 100644 --- a/net/atm/lec.h +++ b/net/atm/lec.h @@ -91,7 +91,7 @@ struct lec_priv { */ spinlock_t lec_arp_lock; struct atm_vcc *mcast_vcc; /* Default Multicast Send VCC */ - struct atm_vcc *lecd; + struct atm_vcc __rcu *lecd; struct delayed_work lec_arp_work; /* C10 */ unsigned int maximum_unknown_frame_count; /* diff --git a/net/atm/signaling.c b/net/atm/signaling.c index e70ae2c113f95..358fbe5e4d1d0 100644 --- a/net/atm/signaling.c +++ b/net/atm/signaling.c @@ -22,6 +22,36 @@ struct atm_vcc *sigd = NULL; +/* + * find_get_vcc - validate and get a reference to a vcc pointer + * @vcc: the vcc pointer to validate + * + * This function validates that @vcc points to a registered VCC in vcc_hash. + * If found, it increments the socket reference count and returns the vcc. + * The caller must call sock_put(sk_atm(vcc)) when done. + * + * Returns the vcc pointer if valid, NULL otherwise. + */ +static struct atm_vcc *find_get_vcc(struct atm_vcc *vcc) +{ + int i; + + read_lock(&vcc_sklist_lock); + for (i = 0; i < VCC_HTABLE_SIZE; i++) { + struct sock *s; + + sk_for_each(s, &vcc_hash[i]) { + if (atm_sk(s) == vcc) { + sock_hold(s); + read_unlock(&vcc_sklist_lock); + return vcc; + } + } + } + read_unlock(&vcc_sklist_lock); + return NULL; +} + static void sigd_put_skb(struct sk_buff *skb) { if (!sigd) { @@ -69,7 +99,14 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) msg = (struct atmsvc_msg *) skb->data; WARN_ON(refcount_sub_and_test(skb->truesize, &sk_atm(vcc)->sk_wmem_alloc)); - vcc = *(struct atm_vcc **) &msg->vcc; + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->vcc); + if (!vcc) { + pr_debug("invalid vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + pr_debug("%d (0x%lx)\n", (int)msg->type, (unsigned long)vcc); sk = sk_atm(vcc); @@ -100,7 +137,16 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) clear_bit(ATM_VF_WAITING, &vcc->flags); break; case as_indicate: - vcc = *(struct atm_vcc **)&msg->listen_vcc; + /* Release the reference from msg->vcc, we'll use msg->listen_vcc instead */ + sock_put(sk); + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->listen_vcc); + if (!vcc) { + pr_debug("invalid listen_vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + sk = sk_atm(vcc); pr_debug("as_indicate!!!\n"); lock_sock(sk); @@ -115,6 +161,8 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) sk->sk_state_change(sk); as_indicate_complete: release_sock(sk); + /* Paired with find_get_vcc(msg->listen_vcc) above */ + sock_put(sk); return 0; case as_close: set_bit(ATM_VF_RELEASED, &vcc->flags); @@ -131,11 +179,15 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) break; default: pr_alert("bad message type %d\n", (int)msg->type); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return -EINVAL; } sk->sk_state_change(sk); out: dev_kfree_skb(skb); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return 0; } diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c index b75c2228e69a6..f28e9cbf8ad5f 100644 --- a/net/batman-adv/bat_iv_ogm.c +++ b/net/batman-adv/bat_iv_ogm.c @@ -473,6 +473,9 @@ batadv_iv_ogm_can_aggregate(const struct batadv_ogm_packet *new_bat_ogm_packet, if (aggregated_bytes > max_bytes) return false; + if (skb_tailroom(forw_packet->skb) < packet_len) + return false; + if (packet_num >= BATADV_MAX_AGGREGATION_PACKETS) return false; diff --git a/net/batman-adv/bat_v_elp.c b/net/batman-adv/bat_v_elp.c index cb16c1ed2a58f..fe832093d421f 100644 --- a/net/batman-adv/bat_v_elp.c +++ b/net/batman-adv/bat_v_elp.c @@ -111,7 +111,15 @@ static bool batadv_v_elp_get_throughput(struct batadv_hardif_neigh_node *neigh, /* unsupported WiFi driver version */ goto default_throughput; - real_netdev = batadv_get_real_netdev(hard_iface->net_dev); + /* only use rtnl_trylock because the elp worker will be cancelled while + * the rntl_lock is held. the cancel_delayed_work_sync() would otherwise + * wait forever when the elp work_item was started and it is then also + * trying to rtnl_lock + */ + if (!rtnl_trylock()) + return false; + real_netdev = __batadv_get_real_netdev(hard_iface->net_dev); + rtnl_unlock(); if (!real_netdev) goto default_throughput; diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index b992ba12aa247..d4f4e97a27f10 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -2165,6 +2165,7 @@ batadv_bla_claim_dump_entry(struct sk_buff *msg, u32 portid, struct batadv_bla_claim *claim) { const u8 *primary_addr = primary_if->net_dev->dev_addr; + struct batadv_bla_backbone_gw *backbone_gw; u16 backbone_crc; bool is_own; void *hdr; @@ -2180,32 +2181,35 @@ batadv_bla_claim_dump_entry(struct sk_buff *msg, u32 portid, genl_dump_check_consistent(cb, hdr); - is_own = batadv_compare_eth(claim->backbone_gw->orig, - primary_addr); + backbone_gw = batadv_bla_claim_get_backbone_gw(claim); + + is_own = batadv_compare_eth(backbone_gw->orig, primary_addr); - spin_lock_bh(&claim->backbone_gw->crc_lock); - backbone_crc = claim->backbone_gw->crc; - spin_unlock_bh(&claim->backbone_gw->crc_lock); + spin_lock_bh(&backbone_gw->crc_lock); + backbone_crc = backbone_gw->crc; + spin_unlock_bh(&backbone_gw->crc_lock); if (is_own) if (nla_put_flag(msg, BATADV_ATTR_BLA_OWN)) { genlmsg_cancel(msg, hdr); - goto out; + goto put_backbone_gw; } if (nla_put(msg, BATADV_ATTR_BLA_ADDRESS, ETH_ALEN, claim->addr) || nla_put_u16(msg, BATADV_ATTR_BLA_VID, claim->vid) || nla_put(msg, BATADV_ATTR_BLA_BACKBONE, ETH_ALEN, - claim->backbone_gw->orig) || + backbone_gw->orig) || nla_put_u16(msg, BATADV_ATTR_BLA_CRC, backbone_crc)) { genlmsg_cancel(msg, hdr); - goto out; + goto put_backbone_gw; } genlmsg_end(msg, hdr); ret = 0; +put_backbone_gw: + batadv_backbone_gw_put(backbone_gw); out: return ret; } @@ -2483,6 +2487,7 @@ int batadv_bla_backbone_dump(struct sk_buff *msg, struct netlink_callback *cb) bool batadv_bla_check_claim(struct batadv_priv *bat_priv, u8 *addr, unsigned short vid) { + struct batadv_bla_backbone_gw *backbone_gw; struct batadv_bla_claim search_claim; struct batadv_bla_claim *claim = NULL; struct batadv_hard_iface *primary_if = NULL; @@ -2505,9 +2510,13 @@ bool batadv_bla_check_claim(struct batadv_priv *bat_priv, * return false. */ if (claim) { - if (!batadv_compare_eth(claim->backbone_gw->orig, + backbone_gw = batadv_bla_claim_get_backbone_gw(claim); + + if (!batadv_compare_eth(backbone_gw->orig, primary_if->net_dev->dev_addr)) ret = false; + + batadv_backbone_gw_put(backbone_gw); batadv_claim_put(claim); } diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c index 5113f879736b5..1c488049d5546 100644 --- a/net/batman-adv/hard-interface.c +++ b/net/batman-adv/hard-interface.c @@ -204,7 +204,7 @@ static bool batadv_is_valid_iface(const struct net_device *net_dev) } /** - * batadv_get_real_netdevice() - check if the given netdev struct is a virtual + * __batadv_get_real_netdev() - check if the given netdev struct is a virtual * interface on top of another 'real' interface * @netdev: the device to check * @@ -214,7 +214,7 @@ static bool batadv_is_valid_iface(const struct net_device *net_dev) * Return: the 'real' net device or the original net device and NULL in case * of an error. */ -static struct net_device *batadv_get_real_netdevice(struct net_device *netdev) +struct net_device *__batadv_get_real_netdev(struct net_device *netdev) { struct batadv_hard_iface *hard_iface = NULL; struct net_device *real_netdev = NULL; @@ -267,7 +267,7 @@ struct net_device *batadv_get_real_netdev(struct net_device *net_device) struct net_device *real_netdev; rtnl_lock(); - real_netdev = batadv_get_real_netdevice(net_device); + real_netdev = __batadv_get_real_netdev(net_device); rtnl_unlock(); return real_netdev; @@ -336,7 +336,7 @@ static u32 batadv_wifi_flags_evaluate(struct net_device *net_device) if (batadv_is_cfg80211_netdev(net_device)) wifi_flags |= BATADV_HARDIF_WIFI_CFG80211_DIRECT; - real_netdev = batadv_get_real_netdevice(net_device); + real_netdev = __batadv_get_real_netdev(net_device); if (!real_netdev) return wifi_flags; diff --git a/net/batman-adv/hard-interface.h b/net/batman-adv/hard-interface.h index 9db8a310961ea..9ba8fb2bdceb4 100644 --- a/net/batman-adv/hard-interface.h +++ b/net/batman-adv/hard-interface.h @@ -67,6 +67,7 @@ enum batadv_hard_if_bcast { extern struct notifier_block batadv_hard_if_notifier; +struct net_device *__batadv_get_real_netdev(struct net_device *net_device); struct net_device *batadv_get_real_netdev(struct net_device *net_device); bool batadv_is_cfg80211_hardif(struct batadv_hard_iface *hard_iface); bool batadv_is_wifi_hardif(struct batadv_hard_iface *hard_iface); diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c index 6e95e883c2bf0..05cddcf994f65 100644 --- a/net/batman-adv/translation-table.c +++ b/net/batman-adv/translation-table.c @@ -798,8 +798,8 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node, { u16 num_vlan = 0; u16 num_entries = 0; - u16 change_offset; - u16 tvlv_len; + u16 tvlv_len = 0; + unsigned int change_offset; struct batadv_tvlv_tt_vlan_data *tt_vlan; struct batadv_orig_node_vlan *vlan; u8 *tt_change_ptr; @@ -816,6 +816,11 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node, if (*tt_len < 0) *tt_len = batadv_tt_len(num_entries); + if (change_offset > U16_MAX || *tt_len > U16_MAX - change_offset) { + *tt_len = 0; + goto out; + } + tvlv_len = *tt_len; tvlv_len += change_offset; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 6fc0692abf057..71a24be2a6d67 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -990,6 +990,7 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t switch (type) { case ACL_LINK: conn->pkt_type = hdev->pkt_type & ACL_PTYPE_MASK; + conn->link_policy = hdev->link_policy; conn->mtu = hdev->acl_mtu; break; case LE_LINK: @@ -1825,9 +1826,13 @@ static int set_cig_params_sync(struct hci_dev *hdev, void *data) u8 aux_num_cis = 0; u8 cis_id; + hci_dev_lock(hdev); + conn = hci_conn_hash_lookup_cig(hdev, cig_id); - if (!conn) + if (!conn) { + hci_dev_unlock(hdev); return 0; + } qos = &conn->iso_qos; pdu->cig_id = cig_id; @@ -1866,6 +1871,8 @@ static int set_cig_params_sync(struct hci_dev *hdev, void *data) } pdu->num_cis = aux_num_cis; + hci_dev_unlock(hdev); + if (!pdu->num_cis) return 0; @@ -1926,6 +1933,8 @@ static bool hci_le_set_cig_params(struct hci_conn *conn, struct bt_iso_qos *qos) return false; done: + conn->iso_qos = *qos; + if (hci_cmd_sync_queue(hdev, set_cig_params_sync, UINT_PTR(qos->ucast.cig), NULL) < 0) return false; @@ -1995,8 +2004,6 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, } hci_conn_hold(cis); - - cis->iso_qos = *qos; cis->state = BT_BOUND; return cis; @@ -2591,8 +2598,8 @@ void hci_conn_enter_active_mode(struct hci_conn *conn, __u8 force_active) timer: if (hdev->idle_timeout > 0) - queue_delayed_work(hdev->workqueue, &conn->idle_work, - msecs_to_jiffies(hdev->idle_timeout)); + mod_delayed_work(hdev->workqueue, &conn->idle_work, + msecs_to_jiffies(hdev->idle_timeout)); } /* Drop all connection on the device */ @@ -2960,7 +2967,7 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) * hci_connect_le serializes the connection attempts so only one * connection can be in BT_CONNECT at time. */ - if (conn->state == BT_CONNECT && hdev->req_status == HCI_REQ_PEND) { + if (conn->state == BT_CONNECT && READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { switch (hci_skb_event(hdev->sent_cmd)) { case HCI_EV_CONN_COMPLETE: case HCI_EV_LE_CONN_COMPLETE: diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 8ccec73dce45c..0f86b81b39730 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -4125,7 +4125,7 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) kfree_skb(skb); } - if (hdev->req_status == HCI_REQ_PEND && + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND && !hci_dev_test_and_set_flag(hdev, HCI_CMD_PENDING)) { kfree_skb(hdev->req_skb); hdev->req_skb = skb_clone(hdev->sent_cmd, GFP_KERNEL); diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 3838b90343d91..7794f4f981596 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -6653,25 +6653,31 @@ static void hci_le_remote_conn_param_req_evt(struct hci_dev *hdev, void *data, latency = le16_to_cpu(ev->latency); timeout = le16_to_cpu(ev->timeout); + hci_dev_lock(hdev); + hcon = hci_conn_hash_lookup_handle(hdev, handle); - if (!hcon || hcon->state != BT_CONNECTED) - return send_conn_param_neg_reply(hdev, handle, - HCI_ERROR_UNKNOWN_CONN_ID); + if (!hcon || hcon->state != BT_CONNECTED) { + send_conn_param_neg_reply(hdev, handle, + HCI_ERROR_UNKNOWN_CONN_ID); + goto unlock; + } - if (max > hcon->le_conn_max_interval) - return send_conn_param_neg_reply(hdev, handle, - HCI_ERROR_INVALID_LL_PARAMS); + if (max > hcon->le_conn_max_interval) { + send_conn_param_neg_reply(hdev, handle, + HCI_ERROR_INVALID_LL_PARAMS); + goto unlock; + } - if (hci_check_conn_params(min, max, latency, timeout)) - return send_conn_param_neg_reply(hdev, handle, - HCI_ERROR_INVALID_LL_PARAMS); + if (hci_check_conn_params(min, max, latency, timeout)) { + send_conn_param_neg_reply(hdev, handle, + HCI_ERROR_INVALID_LL_PARAMS); + goto unlock; + } if (hcon->role == HCI_ROLE_MASTER) { struct hci_conn_params *params; u8 store_hint; - hci_dev_lock(hdev); - params = hci_conn_params_lookup(hdev, &hcon->dst, hcon->dst_type); if (params) { @@ -6684,8 +6690,6 @@ static void hci_le_remote_conn_param_req_evt(struct hci_dev *hdev, void *data, store_hint = 0x00; } - hci_dev_unlock(hdev); - mgmt_new_conn_param(hdev, &hcon->dst, hcon->dst_type, store_hint, min, max, latency, timeout); } @@ -6699,6 +6703,9 @@ static void hci_le_remote_conn_param_req_evt(struct hci_dev *hdev, void *data, cp.max_ce_len = 0; hci_send_cmd(hdev, HCI_OP_LE_CONN_PARAM_REQ_REPLY, sizeof(cp), &cp); + +unlock: + hci_dev_unlock(hdev); } static void hci_le_direct_adv_report_evt(struct hci_dev *hdev, void *data, diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index ad19022ae127a..5295ff001e38c 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -2166,6 +2166,7 @@ static void hci_sock_destruct(struct sock *sk) mgmt_cleanup(sk); skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static const struct proto_ops hci_sock_ops = { diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 6e76798ec786b..ce9f4f8011c13 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -25,11 +25,11 @@ static void hci_cmd_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, { bt_dev_dbg(hdev, "result 0x%2.2x", result); - if (hdev->req_status != HCI_REQ_PEND) + if (READ_ONCE(hdev->req_status) != HCI_REQ_PEND) return; hdev->req_result = result; - hdev->req_status = HCI_REQ_DONE; + WRITE_ONCE(hdev->req_status, HCI_REQ_DONE); /* Free the request command so it is not used as response */ kfree_skb(hdev->req_skb); @@ -167,20 +167,20 @@ struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, hci_cmd_sync_add(&req, opcode, plen, param, event, sk); - hdev->req_status = HCI_REQ_PEND; + WRITE_ONCE(hdev->req_status, HCI_REQ_PEND); err = hci_req_sync_run(&req); if (err < 0) return ERR_PTR(err); err = wait_event_interruptible_timeout(hdev->req_wait_q, - hdev->req_status != HCI_REQ_PEND, + READ_ONCE(hdev->req_status) != HCI_REQ_PEND, timeout); if (err == -ERESTARTSYS) return ERR_PTR(-EINTR); - switch (hdev->req_status) { + switch (READ_ONCE(hdev->req_status)) { case HCI_REQ_DONE: err = -bt_to_errno(hdev->req_result); break; @@ -194,7 +194,7 @@ struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, break; } - hdev->req_status = 0; + WRITE_ONCE(hdev->req_status, 0); hdev->req_result = 0; skb = hdev->req_rsp; hdev->req_rsp = NULL; @@ -665,9 +665,9 @@ void hci_cmd_sync_cancel(struct hci_dev *hdev, int err) { bt_dev_dbg(hdev, "err 0x%2.2x", err); - if (hdev->req_status == HCI_REQ_PEND) { + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { hdev->req_result = err; - hdev->req_status = HCI_REQ_CANCELED; + WRITE_ONCE(hdev->req_status, HCI_REQ_CANCELED); queue_work(hdev->workqueue, &hdev->cmd_sync_cancel_work); } @@ -683,12 +683,12 @@ void hci_cmd_sync_cancel_sync(struct hci_dev *hdev, int err) { bt_dev_dbg(hdev, "err 0x%2.2x", err); - if (hdev->req_status == HCI_REQ_PEND) { + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { /* req_result is __u32 so error must be positive to be properly * propagated. */ hdev->req_result = err < 0 ? -err : err; - hdev->req_status = HCI_REQ_CANCELED; + WRITE_ONCE(hdev->req_status, HCI_REQ_CANCELED); wake_up_interruptible(&hdev->req_wait_q); } @@ -801,8 +801,15 @@ int hci_cmd_sync_run(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, return -ENETDOWN; /* If on cmd_sync_work then run immediately otherwise queue */ - if (current_work() == &hdev->cmd_sync_work) - return func(hdev, data); + if (current_work() == &hdev->cmd_sync_work) { + int err; + + err = func(hdev, data); + if (destroy) + destroy(hdev, data, err); + + return 0; + } return hci_cmd_sync_submit(hdev, func, data, destroy); } @@ -4402,6 +4409,7 @@ static int hci_le_set_event_mask_sync(struct hci_dev *hdev) if (bis_capable(hdev)) { events[1] |= 0x20; /* LE PA Report */ events[1] |= 0x40; /* LE PA Sync Established */ + events[1] |= 0x80; /* LE PA Sync Lost */ events[3] |= 0x04; /* LE Create BIG Complete */ events[3] |= 0x08; /* LE Terminate BIG Complete */ events[3] |= 0x10; /* LE BIG Sync Established */ @@ -4409,6 +4417,18 @@ static int hci_le_set_event_mask_sync(struct hci_dev *hdev) events[4] |= 0x02; /* LE BIG Info Advertising Report */ } + if (le_cs_capable(hdev)) { + /* Channel Sounding events */ + events[5] |= 0x08; /* LE CS Read Remote Supported Cap Complete event */ + events[5] |= 0x10; /* LE CS Read Remote FAE Table Complete event */ + events[5] |= 0x20; /* LE CS Security Enable Complete event */ + events[5] |= 0x40; /* LE CS Config Complete event */ + events[5] |= 0x80; /* LE CS Procedure Enable Complete event */ + events[6] |= 0x01; /* LE CS Subevent Result event */ + events[6] |= 0x02; /* LE CS Subevent Result Continue event */ + events[6] |= 0x04; /* LE CS Test End Complete event */ + } + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EVENT_MASK, sizeof(events), events, HCI_CMD_TIMEOUT); } @@ -4541,23 +4561,43 @@ static int hci_set_le_support_sync(struct hci_dev *hdev) } /* LE Set Host Feature */ -static int hci_le_set_host_feature_sync(struct hci_dev *hdev) +static int hci_le_set_host_feature_sync(struct hci_dev *hdev, + u8 bit, u8 value) { struct hci_cp_le_set_host_feature cp; - if (!iso_capable(hdev)) - return 0; - memset(&cp, 0, sizeof(cp)); - /* Connected Isochronous Channels (Host Support) */ - cp.bit_number = 32; - cp.bit_value = iso_enabled(hdev) ? 0x01 : 0x00; + cp.bit_number = bit; + cp.bit_value = value; return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_HOST_FEATURE, sizeof(cp), &cp, HCI_CMD_TIMEOUT); } +/* Set Host Features, each feature needs to be sent separately since + * HCI_OP_LE_SET_HOST_FEATURE doesn't support setting all of them at once. + */ +static int hci_le_set_host_features_sync(struct hci_dev *hdev) +{ + int err; + + if (iso_capable(hdev)) { + /* Connected Isochronous Channels (Host Support) */ + err = hci_le_set_host_feature_sync(hdev, 32, + (iso_enabled(hdev) ? 0x01 : + 0x00)); + if (err) + return err; + } + + if (le_cs_capable(hdev)) + /* Channel Sounding (Host Support) */ + err = hci_le_set_host_feature_sync(hdev, 47, 0x01); + + return err; +} + /* LE Controller init stage 3 command sequence */ static const struct hci_init_stage le_init3[] = { /* HCI_OP_LE_SET_EVENT_MASK */ @@ -4585,7 +4625,7 @@ static const struct hci_init_stage le_init3[] = { /* HCI_OP_WRITE_LE_HOST_SUPPORTED */ HCI_INIT(hci_set_le_support_sync), /* HCI_OP_LE_SET_HOST_FEATURE */ - HCI_INIT(hci_le_set_host_feature_sync), + HCI_INIT(hci_le_set_host_features_sync), {} }; @@ -6577,8 +6617,8 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) * state. */ if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) { - hci_scan_disable_sync(hdev); hci_dev_set_flag(hdev, HCI_LE_SCAN_INTERRUPTED); + hci_scan_disable_sync(hdev); } /* Update random address, but set require_privacy to false so @@ -6878,8 +6918,6 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) conn->attempt++; - conn->link_policy = hdev->link_policy; - memset(&cp, 0, sizeof(cp)); bacpy(&cp.bdaddr, &conn->dst); cp.pscan_rep_mode = 0x02; @@ -7125,7 +7163,8 @@ static void create_big_complete(struct hci_dev *hdev, void *data, int err) static int hci_le_big_create_sync(struct hci_dev *hdev, void *data) { - DEFINE_FLEX(struct hci_cp_le_big_create_sync, cp, bis, num_bis, 0x11); + DEFINE_FLEX(struct hci_cp_le_big_create_sync, cp, bis, num_bis, + HCI_MAX_ISO_BIS); struct hci_conn *conn = data; struct bt_iso_qos *qos = &conn->iso_qos; int err; diff --git a/net/bluetooth/hidp/core.c b/net/bluetooth/hidp/core.c index 6724adce615b6..e0e4003815500 100644 --- a/net/bluetooth/hidp/core.c +++ b/net/bluetooth/hidp/core.c @@ -986,7 +986,8 @@ static void session_free(struct kref *ref) skb_queue_purge(&session->intr_transmit); fput(session->intr_sock->file); fput(session->ctrl_sock->file); - l2cap_conn_put(session->conn); + if (session->conn) + l2cap_conn_put(session->conn); kfree(session); } @@ -1164,6 +1165,15 @@ static void hidp_session_remove(struct l2cap_conn *conn, down_write(&hidp_session_sem); + /* Drop L2CAP reference immediately to indicate that + * l2cap_unregister_user() shall not be called as it is already + * considered removed. + */ + if (session->conn) { + l2cap_conn_put(session->conn); + session->conn = NULL; + } + hidp_session_terminate(session); cancel_work_sync(&session->dev_init); @@ -1301,7 +1311,9 @@ static int hidp_session_thread(void *arg) * Instead, this call has the same semantics as if user-space tried to * delete the session. */ - l2cap_unregister_user(session->conn, &session->user); + if (session->conn) + l2cap_unregister_user(session->conn, &session->user); + hidp_session_put(session); module_put_and_kthread_exit(0); diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index 616c2fef91d24..46ebd69026fe2 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -728,6 +728,7 @@ static void iso_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void iso_sock_cleanup_listen(struct sock *parent) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 07b493331fd78..29e23f20dc438 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -924,26 +924,41 @@ int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator) initiator); } -static u8 l2cap_get_ident(struct l2cap_conn *conn) +static int l2cap_get_ident(struct l2cap_conn *conn) { - u8 id; + u8 max; + int ident; + /* LE link does not support tools like l2ping so use the full range */ + if (conn->hcon->type == LE_LINK) + max = 255; /* Get next available identificator. * 1 - 128 are used by kernel. * 129 - 199 are reserved. * 200 - 254 are used by utilities like l2ping, etc. */ + else + max = 128; + + /* Allocate ident using min as last used + 1 (cyclic) */ + ident = ida_alloc_range(&conn->tx_ida, READ_ONCE(conn->tx_ident) + 1, + max, GFP_ATOMIC); + /* Force min 1 to start over */ + if (ident <= 0) { + ident = ida_alloc_range(&conn->tx_ida, 1, max, GFP_ATOMIC); + if (ident <= 0) { + /* If all idents are in use, log an error, this is + * extremely unlikely to happen and would indicate a bug + * in the code that idents are not being freed properly. + */ + BT_ERR("Unable to allocate ident: %d", ident); + return 0; + } + } - mutex_lock(&conn->ident_lock); - - if (++conn->tx_ident > 128) - conn->tx_ident = 1; - - id = conn->tx_ident; - - mutex_unlock(&conn->ident_lock); + WRITE_ONCE(conn->tx_ident, ident); - return id; + return ident; } static void l2cap_send_acl(struct l2cap_conn *conn, struct sk_buff *skb, @@ -1686,17 +1701,15 @@ static void l2cap_info_timeout(struct work_struct *work) int l2cap_register_user(struct l2cap_conn *conn, struct l2cap_user *user) { - struct hci_dev *hdev = conn->hcon->hdev; int ret; /* We need to check whether l2cap_conn is registered. If it is not, we - * must not register the l2cap_user. l2cap_conn_del() is unregisters - * l2cap_conn objects, but doesn't provide its own locking. Instead, it - * relies on the parent hci_conn object to be locked. This itself relies - * on the hci_dev object to be locked. So we must lock the hci device - * here, too. */ + * must not register the l2cap_user. l2cap_conn_del() unregisters + * l2cap_conn objects under conn->lock, and we use the same lock here + * to protect access to conn->users and conn->hchan. + */ - hci_dev_lock(hdev); + mutex_lock(&conn->lock); if (!list_empty(&user->list)) { ret = -EINVAL; @@ -1717,16 +1730,14 @@ int l2cap_register_user(struct l2cap_conn *conn, struct l2cap_user *user) ret = 0; out_unlock: - hci_dev_unlock(hdev); + mutex_unlock(&conn->lock); return ret; } EXPORT_SYMBOL(l2cap_register_user); void l2cap_unregister_user(struct l2cap_conn *conn, struct l2cap_user *user) { - struct hci_dev *hdev = conn->hcon->hdev; - - hci_dev_lock(hdev); + mutex_lock(&conn->lock); if (list_empty(&user->list)) goto out_unlock; @@ -1735,7 +1746,7 @@ void l2cap_unregister_user(struct l2cap_conn *conn, struct l2cap_user *user) user->remove(conn, user); out_unlock: - hci_dev_unlock(hdev); + mutex_unlock(&conn->lock); } EXPORT_SYMBOL(l2cap_unregister_user); @@ -1760,6 +1771,9 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) BT_DBG("hcon %p conn %p, err %d", hcon, conn, err); + disable_delayed_work_sync(&conn->info_timer); + disable_delayed_work_sync(&conn->id_addr_timer); + mutex_lock(&conn->lock); kfree_skb(conn->rx_skb); @@ -1773,7 +1787,7 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) if (work_pending(&conn->pending_rx_work)) cancel_work_sync(&conn->pending_rx_work); - cancel_delayed_work_sync(&conn->id_addr_timer); + ida_destroy(&conn->tx_ida); l2cap_unregister_all_users(conn); @@ -1793,9 +1807,6 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) l2cap_chan_put(chan); } - if (conn->info_state & L2CAP_INFO_FEAT_MASK_REQ_SENT) - cancel_delayed_work_sync(&conn->info_timer); - hci_chan_del(conn->hchan); conn->hchan = NULL; @@ -2387,6 +2398,9 @@ static int l2cap_segment_sdu(struct l2cap_chan *chan, /* Remote device may have requested smaller PDUs */ pdu_len = min_t(size_t, pdu_len, chan->remote_mps); + if (!pdu_len) + return -EINVAL; + if (len <= pdu_len) { sar = L2CAP_SAR_UNSEGMENTED; sdu_len = 0; @@ -4322,14 +4336,16 @@ static inline int l2cap_config_req(struct l2cap_conn *conn, if (test_bit(CONF_INPUT_DONE, &chan->conf_state)) { set_default_fcs(chan); - if (chan->mode == L2CAP_MODE_ERTM || - chan->mode == L2CAP_MODE_STREAMING) - err = l2cap_ertm_init(chan); + if (chan->state != BT_CONNECTED) { + if (chan->mode == L2CAP_MODE_ERTM || + chan->mode == L2CAP_MODE_STREAMING) + err = l2cap_ertm_init(chan); - if (err < 0) - l2cap_send_disconn_req(chan, -err); - else - l2cap_chan_ready(chan); + if (err < 0) + l2cap_send_disconn_req(chan, -err); + else + l2cap_chan_ready(chan); + } goto unlock; } @@ -4622,7 +4638,8 @@ static inline int l2cap_information_rsp(struct l2cap_conn *conn, switch (type) { case L2CAP_IT_FEAT_MASK: - conn->feat_mask = get_unaligned_le32(rsp->data); + if (cmd_len >= sizeof(*rsp) + sizeof(u32)) + conn->feat_mask = get_unaligned_le32(rsp->data); if (conn->feat_mask & L2CAP_FEAT_FIXED_CHAN) { struct l2cap_info_req req; @@ -4641,7 +4658,8 @@ static inline int l2cap_information_rsp(struct l2cap_conn *conn, break; case L2CAP_IT_FIXED_CHAN: - conn->remote_fixed_chan = rsp->data[0]; + if (cmd_len >= sizeof(*rsp) + sizeof(rsp->data[0])) + conn->remote_fixed_chan = rsp->data[0]; conn->info_state |= L2CAP_INFO_FEAT_MASK_REQ_DONE; conn->info_ident = 0; @@ -4782,12 +4800,34 @@ static int l2cap_le_connect_rsp(struct l2cap_conn *conn, return err; } +static void l2cap_put_ident(struct l2cap_conn *conn, u8 code, u8 id) +{ + switch (code) { + case L2CAP_COMMAND_REJ: + case L2CAP_CONN_RSP: + case L2CAP_CONF_RSP: + case L2CAP_DISCONN_RSP: + case L2CAP_ECHO_RSP: + case L2CAP_INFO_RSP: + case L2CAP_CONN_PARAM_UPDATE_RSP: + case L2CAP_ECRED_CONN_RSP: + case L2CAP_ECRED_RECONF_RSP: + /* First do a lookup since the remote may send bogus ids that + * would make ida_free to generate warnings. + */ + if (ida_find_first_range(&conn->tx_ida, id, id) >= 0) + ida_free(&conn->tx_ida, id); + } +} + static inline int l2cap_bredr_sig_cmd(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, u16 cmd_len, u8 *data) { int err = 0; + l2cap_put_ident(conn, cmd->code, cmd->ident); + switch (cmd->code) { case L2CAP_COMMAND_REJ: l2cap_command_rej(conn, cmd, cmd_len, data); @@ -4900,6 +4940,13 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } + /* Check if Key Size is sufficient for the security level */ + if (!l2cap_check_enc_key_size(conn->hcon, pchan)) { + result = L2CAP_CR_LE_BAD_KEY_SIZE; + chan = NULL; + goto response_unlock; + } + /* Check for valid dynamic CID range */ if (scid < L2CAP_CID_DYN_START || scid > L2CAP_CID_LE_DYN_END) { result = L2CAP_CR_LE_INVALID_SCID; @@ -5035,18 +5082,28 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, struct l2cap_chan *chan, *pchan; u16 mtu, mps; __le16 psm; - u8 result, len = 0; - int i, num_scid; + u8 result, rsp_len = 0; + int i, num_scid = 0; bool defer = false; if (!enable_ecred) return -EINVAL; + memset(pdu, 0, sizeof(*pdu)); + if (cmd_len < sizeof(*req) || (cmd_len - sizeof(*req)) % sizeof(u16)) { result = L2CAP_CR_LE_INVALID_PARAMS; goto response; } + /* Check if there are no pending channels with the same ident */ + __l2cap_chan_list_id(conn, cmd->ident, l2cap_ecred_list_defer, + &num_scid); + if (num_scid) { + result = L2CAP_CR_LE_INVALID_PARAMS; + goto response; + } + cmd_len -= sizeof(*req); num_scid = cmd_len / sizeof(u16); @@ -5055,11 +5112,14 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, goto response; } + /* Always respond with the same number of scids as in the request */ + rsp_len = cmd_len; + mtu = __le16_to_cpu(req->mtu); mps = __le16_to_cpu(req->mps); if (mtu < L2CAP_ECRED_MIN_MTU || mps < L2CAP_ECRED_MIN_MPS) { - result = L2CAP_CR_LE_UNACCEPT_PARAMS; + result = L2CAP_CR_LE_INVALID_PARAMS; goto response; } @@ -5079,8 +5139,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, BT_DBG("psm 0x%2.2x mtu %u mps %u", __le16_to_cpu(psm), mtu, mps); - memset(pdu, 0, sizeof(*pdu)); - /* Check if we have socket listening on psm */ pchan = l2cap_global_chan_by_psm(BT_LISTEN, psm, &conn->hcon->src, &conn->hcon->dst, LE_LINK); @@ -5093,7 +5151,16 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, if (!smp_sufficient_security(conn->hcon, pchan->sec_level, SMP_ALLOW_STK)) { - result = L2CAP_CR_LE_AUTHENTICATION; + result = pchan->sec_level == BT_SECURITY_MEDIUM ? + L2CAP_CR_LE_ENCRYPTION : L2CAP_CR_LE_AUTHENTICATION; + goto unlock; + } + + /* Check if the listening channel has set an output MTU then the + * requested MTU shall be less than or equal to that value. + */ + if (pchan->omtu && mtu < pchan->omtu) { + result = L2CAP_CR_LE_UNACCEPT_PARAMS; goto unlock; } @@ -5105,7 +5172,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, BT_DBG("scid[%d] 0x%4.4x", i, scid); pdu->dcid[i] = 0x0000; - len += sizeof(*pdu->dcid); /* Check for valid dynamic CID range */ if (scid < L2CAP_CID_DYN_START || scid > L2CAP_CID_LE_DYN_END) { @@ -5172,7 +5238,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, return 0; l2cap_send_cmd(conn, cmd->ident, L2CAP_ECRED_CONN_RSP, - sizeof(*pdu) + len, pdu); + sizeof(*pdu) + rsp_len, pdu); return 0; } @@ -5294,14 +5360,14 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn, struct l2cap_ecred_reconf_req *req = (void *) data; struct l2cap_ecred_reconf_rsp rsp; u16 mtu, mps, result; - struct l2cap_chan *chan; + struct l2cap_chan *chan[L2CAP_ECRED_MAX_CID] = {}; int i, num_scid; if (!enable_ecred) return -EINVAL; - if (cmd_len < sizeof(*req) || cmd_len - sizeof(*req) % sizeof(u16)) { - result = L2CAP_CR_LE_INVALID_PARAMS; + if (cmd_len < sizeof(*req) || (cmd_len - sizeof(*req)) % sizeof(u16)) { + result = L2CAP_RECONF_INVALID_CID; goto respond; } @@ -5311,42 +5377,69 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn, BT_DBG("mtu %u mps %u", mtu, mps); if (mtu < L2CAP_ECRED_MIN_MTU) { - result = L2CAP_RECONF_INVALID_MTU; + result = L2CAP_RECONF_INVALID_PARAMS; goto respond; } if (mps < L2CAP_ECRED_MIN_MPS) { - result = L2CAP_RECONF_INVALID_MPS; + result = L2CAP_RECONF_INVALID_PARAMS; goto respond; } cmd_len -= sizeof(*req); num_scid = cmd_len / sizeof(u16); + + if (num_scid > L2CAP_ECRED_MAX_CID) { + result = L2CAP_RECONF_INVALID_PARAMS; + goto respond; + } + result = L2CAP_RECONF_SUCCESS; + /* Check if each SCID, MTU and MPS are valid */ for (i = 0; i < num_scid; i++) { u16 scid; scid = __le16_to_cpu(req->scid[i]); - if (!scid) - return -EPROTO; + if (!scid) { + result = L2CAP_RECONF_INVALID_CID; + goto respond; + } - chan = __l2cap_get_chan_by_dcid(conn, scid); - if (!chan) - continue; + chan[i] = __l2cap_get_chan_by_dcid(conn, scid); + if (!chan[i]) { + result = L2CAP_RECONF_INVALID_CID; + goto respond; + } - /* If the MTU value is decreased for any of the included - * channels, then the receiver shall disconnect all - * included channels. + /* The MTU field shall be greater than or equal to the greatest + * current MTU size of these channels. */ - if (chan->omtu > mtu) { - BT_ERR("chan %p decreased MTU %u -> %u", chan, - chan->omtu, mtu); + if (chan[i]->omtu > mtu) { + BT_ERR("chan %p decreased MTU %u -> %u", chan[i], + chan[i]->omtu, mtu); result = L2CAP_RECONF_INVALID_MTU; + goto respond; } - chan->omtu = mtu; - chan->remote_mps = mps; + /* If more than one channel is being configured, the MPS field + * shall be greater than or equal to the current MPS size of + * each of these channels. If only one channel is being + * configured, the MPS field may be less than the current MPS + * of that channel. + */ + if (chan[i]->remote_mps >= mps && i) { + BT_ERR("chan %p decreased MPS %u -> %u", chan[i], + chan[i]->remote_mps, mps); + result = L2CAP_RECONF_INVALID_MPS; + goto respond; + } + } + + /* Commit the new MTU and MPS values after checking they are valid */ + for (i = 0; i < num_scid; i++) { + chan[i]->omtu = mtu; + chan[i]->remote_mps = mps; } respond: @@ -5363,7 +5456,7 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn, u8 *data) { struct l2cap_chan *chan, *tmp; - struct l2cap_ecred_conn_rsp *rsp = (void *) data; + struct l2cap_ecred_reconf_rsp *rsp = (void *)data; u16 result; if (cmd_len < sizeof(*rsp)) @@ -5371,7 +5464,7 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn, result = __le16_to_cpu(rsp->result); - BT_DBG("result 0x%4.4x", rsp->result); + BT_DBG("result 0x%4.4x", result); if (!result) return 0; @@ -5419,6 +5512,8 @@ static inline int l2cap_le_sig_cmd(struct l2cap_conn *conn, { int err = 0; + l2cap_put_ident(conn, cmd->code, cmd->ident); + switch (cmd->code) { case L2CAP_COMMAND_REJ: l2cap_le_command_rej(conn, cmd, cmd_len, data); @@ -6538,6 +6633,10 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) struct l2cap_le_credits pkt; u16 return_credits = l2cap_le_rx_credits(chan); + if (chan->mode != L2CAP_MODE_LE_FLOWCTL && + chan->mode != L2CAP_MODE_EXT_FLOWCTL) + return; + if (chan->rx_credits >= return_credits) return; @@ -6599,8 +6698,10 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) return -ENOBUFS; } - if (chan->imtu < skb->len) { - BT_ERR("Too big LE L2CAP PDU"); + if (skb->len > chan->imtu) { + BT_ERR("Too big LE L2CAP PDU: len %u > %u", skb->len, + chan->imtu); + l2cap_send_disconn_req(chan, ECONNRESET); return -ENOBUFS; } @@ -6619,6 +6720,11 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) if (!chan->sdu) { u16 sdu_len; + if (!pskb_may_pull(skb, L2CAP_SDULEN_SIZE)) { + err = -EINVAL; + goto failed; + } + sdu_len = get_unaligned_le16(skb->data); skb_pull(skb, L2CAP_SDULEN_SIZE); @@ -6626,7 +6732,9 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) sdu_len, skb->len, chan->imtu); if (sdu_len > chan->imtu) { - BT_ERR("Too big LE L2CAP SDU length received"); + BT_ERR("Too big LE L2CAP SDU length: len %u > %u", + skb->len, sdu_len); + l2cap_send_disconn_req(chan, ECONNRESET); err = -EMSGSIZE; goto failed; } @@ -6662,6 +6770,7 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) if (chan->sdu->len + skb->len > chan->sdu_len) { BT_ERR("Too much LE L2CAP data received"); + l2cap_send_disconn_req(chan, ECONNRESET); err = -EINVAL; goto failed; } @@ -6907,13 +7016,13 @@ static struct l2cap_conn *l2cap_conn_add(struct hci_conn *hcon) hci_dev_test_flag(hcon->hdev, HCI_FORCE_BREDR_SMP))) conn->local_fixed_chan |= L2CAP_FC_SMP_BREDR; - mutex_init(&conn->ident_lock); mutex_init(&conn->lock); INIT_LIST_HEAD(&conn->chan_l); INIT_LIST_HEAD(&conn->users); INIT_DELAYED_WORK(&conn->info_timer, l2cap_info_timeout); + ida_init(&conn->tx_ida); skb_queue_head_init(&conn->pending_rx); INIT_WORK(&conn->pending_rx_work, process_pending_rx); diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 814fb8610ac43..80a37d56b040f 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -1029,10 +1029,17 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, break; } - /* Setting is not supported as it's the remote side that - * decides this. - */ - err = -EPERM; + /* Only allow setting output MTU when not connected */ + if (sk->sk_state == BT_CONNECTED) { + err = -EISCONN; + break; + } + + err = copy_safe_from_sockptr(&mtu, sizeof(mtu), optval, optlen); + if (err) + break; + + chan->omtu = mtu; break; case BT_RCVMTU: @@ -1674,6 +1681,9 @@ static void l2cap_sock_ready_cb(struct l2cap_chan *chan) struct sock *sk = chan->data; struct sock *parent; + if (!sk) + return; + lock_sock(sk); parent = bt_sk(sk)->parent; @@ -1799,6 +1809,7 @@ static void l2cap_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void l2cap_skb_msg_name(struct sk_buff *skb, void *msg_name, diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 262bf984d2aaf..9065a864bc65d 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -849,6 +849,12 @@ static u32 get_supported_settings(struct hci_dev *hdev) if (cis_peripheral_capable(hdev)) settings |= MGMT_SETTING_CIS_PERIPHERAL; + if (bis_capable(hdev)) + settings |= MGMT_SETTING_ISO_BROADCASTER; + + if (sync_recv_capable(hdev)) + settings |= MGMT_SETTING_ISO_SYNC_RECEIVER; + if (ll_privacy_capable(hdev)) settings |= MGMT_SETTING_LL_PRIVACY; @@ -1948,6 +1954,7 @@ static void set_ssp_complete(struct hci_dev *hdev, void *data, int err) } mgmt_cmd_status(cmd->sk, cmd->hdev->id, cmd->opcode, mgmt_err); + mgmt_pending_free(cmd); return; } @@ -1966,6 +1973,7 @@ static void set_ssp_complete(struct hci_dev *hdev, void *data, int err) sock_put(match.sk); hci_update_eir_sync(hdev); + mgmt_pending_free(cmd); } static int set_ssp_sync(struct hci_dev *hdev, void *data) @@ -2175,10 +2183,7 @@ static void set_mesh_complete(struct hci_dev *hdev, void *data, int err) sk = cmd->sk; if (status) { - mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, - status); - mgmt_pending_foreach(MGMT_OP_SET_MESH_RECEIVER, hdev, true, - cmd_status_rsp, &status); + mgmt_cmd_status(cmd->sk, hdev->id, cmd->opcode, status); goto done; } @@ -2461,6 +2466,7 @@ static int mesh_send(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) struct mgmt_mesh_tx *mesh_tx; struct mgmt_cp_mesh_send *send = data; struct mgmt_rp_mesh_read_features rp; + u16 expected_len; bool sending; int err = 0; @@ -2468,12 +2474,19 @@ static int mesh_send(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL)) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, MGMT_STATUS_NOT_SUPPORTED); - if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED) || - len <= MGMT_MESH_SEND_SIZE || - len > (MGMT_MESH_SEND_SIZE + 31)) + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, MGMT_STATUS_REJECTED); + if (!send->adv_data_len || send->adv_data_len > 31) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_REJECTED); + + expected_len = struct_size(send, adv_data, send->adv_data_len); + if (expected_len != len) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_INVALID_PARAMS); + hci_dev_lock(hdev); memset(&rp, 0, sizeof(rp)); @@ -5268,7 +5281,7 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, * hci_adv_monitors_clear is about to be called which will take care of * freeing the adv_monitor instances. */ - if (status == -ECANCELED && !mgmt_pending_valid(hdev, cmd)) + if (status == -ECANCELED || !mgmt_pending_valid(hdev, cmd)) return; monitor = cmd->user_data; @@ -5287,7 +5300,7 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, mgmt_status(status), &rp, sizeof(rp)); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); hci_dev_unlock(hdev); bt_dev_dbg(hdev, "add monitor %d complete, status %d", @@ -6350,6 +6363,7 @@ static void set_advertising_complete(struct hci_dev *hdev, void *data, int err) hci_dev_clear_flag(hdev, HCI_ADVERTISING); settings_rsp(cmd, &match); + mgmt_pending_free(cmd); new_settings(hdev, match.sk); @@ -7160,6 +7174,9 @@ static bool ltk_is_valid(struct mgmt_ltk_info *key) if (key->initiator != 0x00 && key->initiator != 0x01) return false; + if (key->enc_size > sizeof(key->val)) + return false; + switch (key->addr.type) { case BDADDR_LE_PUBLIC: return true; diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 298c2a9ab4df8..1a38577135d44 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -298,7 +298,7 @@ static int sco_chan_add(struct sco_conn *conn, struct sock *sk, int err = 0; sco_conn_lock(conn); - if (conn->sk) + if (conn->sk || sco_pi(sk)->conn) err = -EBUSY; else __sco_chan_add(conn, sk, parent); @@ -353,9 +353,20 @@ static int sco_connect(struct sock *sk) lock_sock(sk); + /* Recheck state after reacquiring the socket lock, as another + * thread may have changed it (e.g., closed the socket). + */ + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { + release_sock(sk); + hci_conn_drop(hcon); + err = -EBADFD; + goto unlock; + } + err = sco_chan_add(conn, sk, NULL); if (err) { release_sock(sk); + hci_conn_drop(hcon); goto unlock; } @@ -401,7 +412,7 @@ static void sco_recv_frame(struct sco_conn *conn, struct sk_buff *skb) struct sock *sk; sco_conn_lock(conn); - sk = conn->sk; + sk = sco_sock_hold(conn); sco_conn_unlock(conn); if (!sk) @@ -410,11 +421,15 @@ static void sco_recv_frame(struct sco_conn *conn, struct sk_buff *skb) BT_DBG("sk %p len %u", sk, skb->len); if (sk->sk_state != BT_CONNECTED) - goto drop; + goto drop_put; - if (!sock_queue_rcv_skb(sk, skb)) + if (!sock_queue_rcv_skb(sk, skb)) { + sock_put(sk); return; + } +drop_put: + sock_put(sk); drop: kfree_skb(skb); } @@ -470,6 +485,7 @@ static void sco_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void sco_sock_cleanup_listen(struct sock *parent) @@ -651,13 +667,18 @@ static int sco_sock_connect(struct socket *sock, struct sockaddr *addr, int alen addr->sa_family != AF_BLUETOOTH) return -EINVAL; - if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) + lock_sock(sk); + + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { + release_sock(sk); return -EBADFD; + } - if (sk->sk_type != SOCK_SEQPACKET) - err = -EINVAL; + if (sk->sk_type != SOCK_SEQPACKET) { + release_sock(sk); + return -EINVAL; + } - lock_sock(sk); /* Set destination address and psm */ bacpy(&sco_pi(sk)->dst, &sa->sco_bdaddr); release_sock(sk); diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 3a1ce04a7a536..abf3ab7479ff2 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -1018,10 +1018,7 @@ static u8 smp_random(struct smp_chan *smp) smp_s1(smp->tk, smp->prnd, smp->rrnd, stk); - if (hcon->pending_sec_level == BT_SECURITY_HIGH) - auth = 1; - else - auth = 0; + auth = test_bit(SMP_FLAG_MITM_AUTH, &smp->flags) ? 1 : 0; /* Even though there's no _RESPONDER suffix this is the * responder STK we're adding for later lookup (the initiator @@ -1826,7 +1823,7 @@ static u8 smp_cmd_pairing_req(struct l2cap_conn *conn, struct sk_buff *skb) if (sec_level > conn->hcon->pending_sec_level) conn->hcon->pending_sec_level = sec_level; - /* If we need MITM check that it can be achieved */ + /* If we need MITM check that it can be achieved. */ if (conn->hcon->pending_sec_level >= BT_SECURITY_HIGH) { u8 method; @@ -1834,6 +1831,10 @@ static u8 smp_cmd_pairing_req(struct l2cap_conn *conn, struct sk_buff *skb) req->io_capability); if (method == JUST_WORKS || method == JUST_CFM) return SMP_AUTH_REQUIREMENTS; + + /* Force MITM bit if it isn't set by the initiator. */ + auth |= SMP_AUTH_MITM; + rsp.auth_req |= SMP_AUTH_MITM; } key_size = min(req->max_key_size, rsp.max_key_size); @@ -2743,7 +2744,7 @@ static int smp_cmd_public_key(struct l2cap_conn *conn, struct sk_buff *skb) if (!test_bit(SMP_FLAG_DEBUG_KEY, &smp->flags) && !crypto_memneq(key, smp->local_pk, 64)) { bt_dev_err(hdev, "Remote and local public keys are identical"); - return SMP_UNSPECIFIED; + return SMP_DHKEY_CHECK_FAILED; } memcpy(smp->remote_pk, key, 64); diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 8b7d0b90fea76..6b04f47301c1e 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -939,6 +939,11 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) if (__skb->gso_segs > GSO_MAX_SEGS) return -EINVAL; + + /* Currently GSO type is zero/unset. If this gets extended with + * a small list of accepted GSO types in future, the filter for + * an unset GSO type in bpf_clone_redirect() can be lifted. + */ skb_shinfo(skb)->gso_segs = __skb->gso_segs; skb_shinfo(skb)->gso_size = __skb->gso_size; skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; @@ -1226,8 +1231,6 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, batch_size = NAPI_POLL_WEIGHT; else if (batch_size > TEST_XDP_MAX_BATCH) return -E2BIG; - - headroom += sizeof(struct xdp_page_head); } else if (batch_size) { return -EINVAL; } @@ -1240,16 +1243,26 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, /* There can't be user provided data before the meta data */ if (ctx->data_meta || ctx->data_end > kattr->test.data_size_in || ctx->data > ctx->data_end || - unlikely(xdp_metalen_invalid(ctx->data)) || (do_live && (kattr->test.data_out || kattr->test.ctx_out))) goto free_ctx; - /* Meta data is allocated from the headroom */ - headroom -= ctx->data; meta_sz = ctx->data; + if (xdp_metalen_invalid(meta_sz) || meta_sz > headroom - sizeof(struct xdp_frame)) + goto free_ctx; + + /* Meta data is allocated from the headroom */ + headroom -= meta_sz; linear_sz = ctx->data_end; } + /* The xdp_page_head structure takes up space in each page, limiting the + * size of the packet data; add the extra size to headroom here to make + * sure it's accounted in the length checks below, but not in the + * metadata size check above. + */ + if (do_live) + headroom += sizeof(struct xdp_page_head); + max_linear_sz = PAGE_SIZE - headroom - tailroom; linear_sz = min_t(u32, linear_sz, max_linear_sz); @@ -1287,13 +1300,13 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, if (sinfo->nr_frags == MAX_SKB_FRAGS) { ret = -ENOMEM; - goto out; + goto out_put_dev; } page = alloc_page(GFP_KERNEL); if (!page) { ret = -ENOMEM; - goto out; + goto out_put_dev; } frag = &sinfo->frags[sinfo->nr_frags++]; @@ -1305,7 +1318,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, if (copy_from_user(page_address(page), data_in + size, data_len)) { ret = -EFAULT; - goto out; + goto out_put_dev; } sinfo->xdp_frags_size += data_len; size += data_len; @@ -1320,6 +1333,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration); else ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); +out_put_dev: /* We convert the xdp_buff back to an xdp_md before checking the return * code so the reference count of any held netdevice will be decremented * even if the test run failed. diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index 1e2b51769eec8..6b5595868a39c 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -251,12 +251,12 @@ struct nd_msg *br_is_nd_neigh_msg(const struct sk_buff *skb, struct nd_msg *msg) static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, struct sk_buff *request, struct neighbour *n, - __be16 vlan_proto, u16 vlan_tci, struct nd_msg *ns) + __be16 vlan_proto, u16 vlan_tci) { struct net_device *dev = request->dev; struct net_bridge_vlan_group *vg; + struct nd_msg *na, *ns; struct sk_buff *reply; - struct nd_msg *na; struct ipv6hdr *pip6; int na_olen = 8; /* opt hdr + ETH_ALEN for target */ int ns_olen; @@ -264,7 +264,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, u8 *daddr; u16 pvid; - if (!dev) + if (!dev || skb_linearize(request)) return; len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) + @@ -281,17 +281,21 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, skb_set_mac_header(reply, 0); daddr = eth_hdr(request)->h_source; + ns = (struct nd_msg *)(skb_network_header(request) + + sizeof(struct ipv6hdr)); /* Do we need option processing ? */ ns_olen = request->len - (skb_network_offset(request) + sizeof(struct ipv6hdr)) - sizeof(*ns); for (i = 0; i < ns_olen - 1; i += (ns->opt[i + 1] << 3)) { - if (!ns->opt[i + 1]) { + if (!ns->opt[i + 1] || i + (ns->opt[i + 1] << 3) > ns_olen) { kfree_skb(reply); return; } if (ns->opt[i] == ND_OPT_SOURCE_LL_ADDR) { - daddr = ns->opt + i + sizeof(struct nd_opt_hdr); + if ((ns->opt[i + 1] << 3) >= + sizeof(struct nd_opt_hdr) + ETH_ALEN) + daddr = ns->opt + i + sizeof(struct nd_opt_hdr); break; } } @@ -472,9 +476,9 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, if (vid != 0) br_nd_send(br, p, skb, n, skb->vlan_proto, - skb_vlan_tag_get(skb), msg); + skb_vlan_tag_get(skb)); else - br_nd_send(br, p, skb, n, 0, 0, msg); + br_nd_send(br, p, skb, n, 0, 0); replied = true; } diff --git a/net/bridge/br_cfm.c b/net/bridge/br_cfm.c index c2c1c7d44c615..f4ca77d9b0e96 100644 --- a/net/bridge/br_cfm.c +++ b/net/bridge/br_cfm.c @@ -576,7 +576,7 @@ static void mep_delete_implementation(struct net_bridge *br, /* Empty and free peer MEP list */ hlist_for_each_entry_safe(peer_mep, n_store, &mep->peer_mep_list, head) { - cancel_delayed_work_sync(&peer_mep->ccm_rx_dwork); + disable_delayed_work_sync(&peer_mep->ccm_rx_dwork); hlist_del_rcu(&peer_mep->head); kfree_rcu(peer_mep, rcu); } @@ -732,7 +732,7 @@ int br_cfm_cc_peer_mep_remove(struct net_bridge *br, const u32 instance, return -ENOENT; } - cc_peer_disable(peer_mep); + disable_delayed_work_sync(&peer_mep->ccm_rx_dwork); hlist_del_rcu(&peer_mep->head); kfree_rcu(peer_mep, rcu); diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index a818fdc22da9a..525d4eccd194a 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -74,7 +74,7 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) eth_hdr(skb)->h_proto == htons(ETH_P_RARP)) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED)) { br_do_proxy_suppress_arp(skb, br, vid, NULL); - } else if (IS_ENABLED(CONFIG_IPV6) && + } else if (ipv6_mod_enabled() && skb->protocol == htons(ETH_P_IPV6) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED) && pskb_may_pull(skb, sizeof(struct ipv6hdr) + diff --git a/net/bridge/br_fdb.c b/net/bridge/br_fdb.c index 58d22e2b85fc3..e2c17f620f009 100644 --- a/net/bridge/br_fdb.c +++ b/net/bridge/br_fdb.c @@ -70,7 +70,7 @@ static inline int has_expired(const struct net_bridge *br, { return !test_bit(BR_FDB_STATIC, &fdb->flags) && !test_bit(BR_FDB_ADDED_BY_EXT_LEARN, &fdb->flags) && - time_before_eq(fdb->updated + hold_time(br), jiffies); + time_before_eq(READ_ONCE(fdb->updated) + hold_time(br), jiffies); } static int fdb_to_nud(const struct net_bridge *br, @@ -126,9 +126,9 @@ static int fdb_fill_info(struct sk_buff *skb, const struct net_bridge *br, if (nla_put_u32(skb, NDA_FLAGS_EXT, ext_flags)) goto nla_put_failure; - ci.ndm_used = jiffies_to_clock_t(now - fdb->used); + ci.ndm_used = jiffies_to_clock_t(now - READ_ONCE(fdb->used)); ci.ndm_confirmed = 0; - ci.ndm_updated = jiffies_to_clock_t(now - fdb->updated); + ci.ndm_updated = jiffies_to_clock_t(now - READ_ONCE(fdb->updated)); ci.ndm_refcnt = 0; if (nla_put(skb, NDA_CACHEINFO, sizeof(ci), &ci)) goto nla_put_failure; @@ -551,7 +551,7 @@ void br_fdb_cleanup(struct work_struct *work) */ rcu_read_lock(); hlist_for_each_entry_rcu(f, &br->fdb_list, fdb_node) { - unsigned long this_timer = f->updated + delay; + unsigned long this_timer = READ_ONCE(f->updated) + delay; if (test_bit(BR_FDB_STATIC, &f->flags) || test_bit(BR_FDB_ADDED_BY_EXT_LEARN, &f->flags)) { @@ -597,6 +597,9 @@ static void br_fdb_delete_locals_per_vlan_port(struct net_bridge *br, dev = br->dev; } + if (!vg) + return; + list_for_each_entry(v, &vg->vlan_list, vlist) br_fdb_find_delete_local(br, p, dev->dev_addr, v->vid); } @@ -630,6 +633,9 @@ static int br_fdb_insert_locals_per_vlan_port(struct net_bridge *br, dev = br->dev; } + if (!vg) + return 0; + list_for_each_entry(v, &vg->vlan_list, vlist) { if (!br_vlan_should_use(v)) continue; @@ -924,6 +930,7 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf, { struct net_bridge_fdb_entry *f; struct __fdb_entry *fe = buf; + unsigned long delta; int num = 0; memset(buf, 0, maxnum*sizeof(struct __fdb_entry)); @@ -953,8 +960,11 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf, fe->port_hi = f->dst->port_no >> 8; fe->is_local = test_bit(BR_FDB_LOCAL, &f->flags); - if (!test_bit(BR_FDB_STATIC, &f->flags)) - fe->ageing_timer_value = jiffies_delta_to_clock_t(jiffies - f->updated); + if (!test_bit(BR_FDB_STATIC, &f->flags)) { + delta = jiffies - READ_ONCE(f->updated); + fe->ageing_timer_value = + jiffies_delta_to_clock_t(delta); + } ++fe; ++num; } @@ -1002,8 +1012,8 @@ void br_fdb_update(struct net_bridge *br, struct net_bridge_port *source, unsigned long now = jiffies; bool fdb_modified = false; - if (now != fdb->updated) { - fdb->updated = now; + if (now != READ_ONCE(fdb->updated)) { + WRITE_ONCE(fdb->updated, now); fdb_modified = __fdb_mark_active(fdb); } @@ -1242,10 +1252,10 @@ static int fdb_add_entry(struct net_bridge *br, struct net_bridge_port *source, if (fdb_handle_notify(fdb, notify)) modified = true; - fdb->used = jiffies; + WRITE_ONCE(fdb->used, jiffies); if (modified) { if (refresh) - fdb->updated = jiffies; + WRITE_ONCE(fdb->updated, jiffies); fdb_notify(br, fdb, RTM_NEWNEIGH, true); } @@ -1556,7 +1566,7 @@ int br_fdb_external_learn_add(struct net_bridge *br, struct net_bridge_port *p, goto err_unlock; } - fdb->updated = jiffies; + WRITE_ONCE(fdb->updated, jiffies); if (READ_ONCE(fdb->dst) != p) { WRITE_ONCE(fdb->dst, p); @@ -1565,7 +1575,7 @@ int br_fdb_external_learn_add(struct net_bridge *br, struct net_bridge_port *p, if (test_and_set_bit(BR_FDB_ADDED_BY_EXT_LEARN, &fdb->flags)) { /* Refresh entry */ - fdb->used = jiffies; + WRITE_ONCE(fdb->used, jiffies); } else { modified = true; } diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index 777fa869c1a14..2cbae0f9ae1f0 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -170,7 +170,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb (skb->protocol == htons(ETH_P_ARP) || skb->protocol == htons(ETH_P_RARP))) { br_do_proxy_suppress_arp(skb, br, vid, p); - } else if (IS_ENABLED(CONFIG_IPV6) && + } else if (ipv6_mod_enabled() && skb->protocol == htons(ETH_P_IPV6) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED) && pskb_may_pull(skb, sizeof(struct ipv6hdr) + @@ -221,8 +221,8 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb if (test_bit(BR_FDB_LOCAL, &dst->flags)) return br_pass_frame_up(skb, false); - if (now != dst->used) - dst->used = now; + if (now != READ_ONCE(dst->used)) + WRITE_ONCE(dst->used, now); br_forward(dst->dst, skb, local_rcv, false); } else { if (!mcast_hit) @@ -274,7 +274,7 @@ static int nf_hook_bridge_pre(struct sk_buff *skb, struct sk_buff **pskb) int ret; net = dev_net(skb->dev); -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL if (!static_key_false(&nf_hooks_needed[NFPROTO_BRIDGE][NF_BR_PRE_ROUTING])) goto frame_finish; #endif diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 22d12e5459668..5855eb0502085 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -244,14 +244,11 @@ br_multicast_port_vid_to_port_ctx(struct net_bridge_port *port, u16 vid) lockdep_assert_held_once(&port->br->multicast_lock); - if (!br_opt_get(port->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) - return NULL; - /* Take RCU to access the vlan. */ rcu_read_lock(); vlan = br_vlan_find(nbp_vlan_group_rcu(port), vid); - if (vlan && !br_multicast_port_ctx_vlan_disabled(&vlan->port_mcast_ctx)) + if (vlan) pmctx = &vlan->port_mcast_ctx; rcu_read_unlock(); @@ -701,7 +698,10 @@ br_multicast_port_ngroups_inc_one(struct net_bridge_mcast_port *pmctx, u32 max = READ_ONCE(pmctx->mdb_max_entries); u32 n = READ_ONCE(pmctx->mdb_n_entries); - if (max && n >= max) { + /* enforce the max limit when it's a port pmctx or a port-vlan pmctx + * with snooping enabled + */ + if (!br_multicast_port_ctx_vlan_disabled(pmctx) && max && n >= max) { NL_SET_ERR_MSG_FMT_MOD(extack, "%s is already in %u groups, and mcast_max_groups=%u", what, n, max); return -E2BIG; @@ -736,9 +736,7 @@ static int br_multicast_port_ngroups_inc(struct net_bridge_port *port, return err; } - /* Only count on the VLAN context if VID is given, and if snooping on - * that VLAN is enabled. - */ + /* Only count on the VLAN context if VID is given */ if (!group->vid) return 0; @@ -2011,6 +2009,18 @@ void br_multicast_port_ctx_init(struct net_bridge_port *port, timer_setup(&pmctx->ip6_own_query.timer, br_ip6_multicast_port_query_expired, 0); #endif + /* initialize mdb_n_entries if a new port vlan is being created */ + if (vlan) { + struct net_bridge_port_group *pg; + u32 n = 0; + + spin_lock_bh(&port->br->multicast_lock); + hlist_for_each_entry(pg, &port->mglist, mglist) + if (pg->key.addr.vid == vlan->vid) + n++; + WRITE_ONCE(pmctx->mdb_n_entries, n); + spin_unlock_bh(&port->br->multicast_lock); + } } void br_multicast_port_ctx_deinit(struct net_bridge_mcast_port *pmctx) @@ -2094,25 +2104,6 @@ static void __br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) br_ip4_multicast_add_router(brmctx, pmctx); br_ip6_multicast_add_router(brmctx, pmctx); } - - if (br_multicast_port_ctx_is_vlan(pmctx)) { - struct net_bridge_port_group *pg; - u32 n = 0; - - /* The mcast_n_groups counter might be wrong. First, - * BR_VLFLAG_MCAST_ENABLED is toggled before temporary entries - * are flushed, thus mcast_n_groups after the toggle does not - * reflect the true values. And second, permanent entries added - * while BR_VLFLAG_MCAST_ENABLED was disabled, are not reflected - * either. Thus we have to refresh the counter. - */ - - hlist_for_each_entry(pg, &pmctx->port->mglist, mglist) { - if (pg->key.addr.vid == pmctx->vlan->vid) - n++; - } - WRITE_ONCE(pmctx->mdb_n_entries, n); - } } static void br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index 7280c4e9305f3..b9b2981c48414 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -247,6 +247,7 @@ struct net_bridge_vlan { * struct net_bridge_vlan_group * * @vlan_hash: VLAN entry rhashtable + * @tunnel_hash: Hash table to map from tunnel key ID (e.g. VXLAN VNI) to VLAN * @vlan_list: sorted VLAN entry list * @num_vlans: number of total VLAN entries * @pvid: PVID VLAN id diff --git a/net/bridge/br_vlan_tunnel.c b/net/bridge/br_vlan_tunnel.c index a966a6ec82634..257cae9f15698 100644 --- a/net/bridge/br_vlan_tunnel.c +++ b/net/bridge/br_vlan_tunnel.c @@ -189,7 +189,6 @@ int br_handle_egress_vlan_tunnel(struct sk_buff *skb, IP_TUNNEL_DECLARE_FLAGS(flags) = { }; struct metadata_dst *tunnel_dst; __be64 tunnel_id; - int err; if (!vlan) return 0; @@ -199,9 +198,13 @@ int br_handle_egress_vlan_tunnel(struct sk_buff *skb, return 0; skb_dst_drop(skb); - err = skb_vlan_pop(skb); - if (err) - return err; + /* For 802.1ad (QinQ), skb_vlan_pop() incorrectly moves the C-VLAN + * from payload to hwaccel after clearing S-VLAN. We only need to + * clear the hwaccel S-VLAN; the C-VLAN must stay in payload for + * correct VXLAN encapsulation. This is also correct for 802.1Q + * where no C-VLAN exists in payload. + */ + __vlan_hwaccel_clear_tag(skb); if (BR_INPUT_SKB_CB(skb)->backup_nhid) { __set_bit(IP_TUNNEL_KEY_BIT, flags); diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index 5697e3949a365..a04fc17575289 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -1299,7 +1299,7 @@ int ebt_register_template(const struct ebt_table *t, int (*table_init)(struct ne list_for_each_entry(tmpl, &template_tables, list) { if (WARN_ON_ONCE(strcmp(t->name, tmpl->name) == 0)) { mutex_unlock(&ebt_mutex); - return -EEXIST; + return -EBUSY; } } diff --git a/net/caif/cffrml.c b/net/caif/cffrml.c index 6651a8dc62e04..d4d63586053ad 100644 --- a/net/caif/cffrml.c +++ b/net/caif/cffrml.c @@ -92,8 +92,15 @@ static int cffrml_receive(struct cflayer *layr, struct cfpkt *pkt) len = le16_to_cpu(tmp); /* Subtract for FCS on length if FCS is not used. */ - if (!this->dofcs) + if (!this->dofcs) { + if (len < 2) { + ++cffrml_rcv_error; + pr_err("Invalid frame length (%d)\n", len); + cfpkt_destroy(pkt); + return -EPROTO; + } len -= 2; + } if (cfpkt_setlen(pkt, len) < 0) { ++cffrml_rcv_error; diff --git a/net/can/af_can.c b/net/can/af_can.c index 770173d8db428..a624c04ed5c63 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -469,7 +469,7 @@ int can_rx_register(struct net *net, struct net_device *dev, canid_t can_id, rcv->can_id = can_id; rcv->mask = mask; - rcv->matches = 0; + atomic_long_set(&rcv->matches, 0); rcv->func = func; rcv->data = data; rcv->ident = ident; @@ -573,7 +573,7 @@ EXPORT_SYMBOL(can_rx_unregister); static inline void deliver(struct sk_buff *skb, struct receiver *rcv) { rcv->func(skb, rcv->data); - rcv->matches++; + atomic_long_inc(&rcv->matches); } static int can_rcv_filter(struct can_dev_rcv_lists *dev_rcv_lists, struct sk_buff *skb) diff --git a/net/can/af_can.h b/net/can/af_can.h index 22f3352c77fec..87887014f5628 100644 --- a/net/can/af_can.h +++ b/net/can/af_can.h @@ -52,7 +52,7 @@ struct receiver { struct hlist_node list; canid_t can_id; canid_t mask; - unsigned long matches; + atomic_long_t matches; void (*func)(struct sk_buff *skb, void *data); void *data; char *ident; diff --git a/net/can/bcm.c b/net/can/bcm.c index 5e690a2377e48..756acfd20c6c2 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -1170,6 +1170,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (!op) return -ENOMEM; + spin_lock_init(&op->bcm_tx_lock); op->can_id = msg_head->can_id; op->nframes = msg_head->nframes; op->cfsiz = CFSIZ(msg_head->flags); diff --git a/net/can/gw.c b/net/can/gw.c index 55eccb1c7620c..79fa58cb232ed 100644 --- a/net/can/gw.c +++ b/net/can/gw.c @@ -374,10 +374,10 @@ static void cgw_csum_crc8_rel(struct canfd_frame *cf, return; if (from <= to) { - for (i = crc8->from_idx; i <= crc8->to_idx; i++) + for (i = from; i <= to; i++) crc = crc8->crctab[crc ^ cf->data[i]]; } else { - for (i = crc8->from_idx; i >= crc8->to_idx; i--) + for (i = from; i >= to; i--) crc = crc8->crctab[crc ^ cf->data[i]]; } @@ -396,7 +396,7 @@ static void cgw_csum_crc8_rel(struct canfd_frame *cf, break; } - cf->data[crc8->result_idx] = crc ^ crc8->final_xor_val; + cf->data[res] = crc ^ crc8->final_xor_val; } static void cgw_csum_crc8_pos(struct canfd_frame *cf, diff --git a/net/can/isotp.c b/net/can/isotp.c index 74ee1e52249b2..12eabfd47d824 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -1230,12 +1230,6 @@ static int isotp_release(struct socket *sock) so->ifindex = 0; so->bound = 0; - if (so->rx.buf != so->rx.sbuf) - kfree(so->rx.buf); - - if (so->tx.buf != so->tx.sbuf) - kfree(so->tx.buf); - sock_orphan(sk); sock->sk = NULL; @@ -1604,6 +1598,21 @@ static int isotp_notifier(struct notifier_block *nb, unsigned long msg, return NOTIFY_DONE; } +static void isotp_sock_destruct(struct sock *sk) +{ + struct isotp_sock *so = isotp_sk(sk); + + /* do the standard CAN sock destruct work */ + can_sock_destruct(sk); + + /* free potential extended PDU buffers */ + if (so->rx.buf != so->rx.sbuf) + kfree(so->rx.buf); + + if (so->tx.buf != so->tx.sbuf) + kfree(so->tx.buf); +} + static int isotp_init(struct sock *sk) { struct isotp_sock *so = isotp_sk(sk); @@ -1648,6 +1657,9 @@ static int isotp_init(struct sock *sk) list_add_tail(&so->notifier, &isotp_notifier_list); spin_unlock(&isotp_notifier_lock); + /* re-assign default can_sock_destruct() reference */ + sk->sk_destruct = isotp_sock_destruct; + return 0; } diff --git a/net/can/j1939/socket.c b/net/can/j1939/socket.c index 88e7160d42489..e3ba2e9fc0e9b 100644 --- a/net/can/j1939/socket.c +++ b/net/can/j1939/socket.c @@ -482,6 +482,12 @@ static int j1939_sk_bind(struct socket *sock, struct sockaddr *uaddr, int len) goto out_release_sock; } + if (ndev->reg_state != NETREG_REGISTERED) { + dev_put(ndev); + ret = -ENODEV; + goto out_release_sock; + } + can_ml = can_get_ml_priv(ndev); if (!can_ml) { dev_put(ndev); diff --git a/net/can/j1939/transport.c b/net/can/j1939/transport.c index fbf5c8001c9df..8656ab388c83e 100644 --- a/net/can/j1939/transport.c +++ b/net/can/j1939/transport.c @@ -1567,6 +1567,8 @@ int j1939_session_activate(struct j1939_session *session) if (active) { j1939_session_put(active); ret = -EAGAIN; + } else if (priv->ndev->reg_state != NETREG_REGISTERED) { + ret = -ENODEV; } else { WARN_ON_ONCE(session->state != J1939_SESSION_NEW); list_add_tail(&session->active_session_list_entry, @@ -1693,8 +1695,16 @@ static int j1939_xtp_rx_rts_session_active(struct j1939_session *session, j1939_session_timers_cancel(session); j1939_session_cancel(session, J1939_XTP_ABORT_BUSY); - if (session->transmission) + if (session->transmission) { j1939_session_deactivate_activate_next(session); + } else if (session->state == J1939_SESSION_WAITING_ABORT) { + /* Force deactivation for the receiver. + * If we rely on the timer starting in j1939_session_cancel, + * a second RTS call here will cancel that timer and fail + * to restart it because the state is already WAITING_ABORT. + */ + j1939_session_deactivate_activate_next(session); + } return -EBUSY; } diff --git a/net/can/proc.c b/net/can/proc.c index 0938bf7dd646a..de4d05ae34597 100644 --- a/net/can/proc.c +++ b/net/can/proc.c @@ -196,7 +196,8 @@ static void can_print_rcvlist(struct seq_file *m, struct hlist_head *rx_list, " %-5s %03x %08x %pK %pK %8ld %s\n"; seq_printf(m, fmt, DNAME(dev), r->can_id, r->mask, - r->func, r->data, r->matches, r->ident); + r->func, r->data, atomic_long_read(&r->matches), + r->ident); } } diff --git a/net/can/raw.c b/net/can/raw.c index a53853f5e9afc..263e7167d2f52 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -360,6 +360,14 @@ static int raw_notifier(struct notifier_block *nb, unsigned long msg, return NOTIFY_DONE; } +static void raw_sock_destruct(struct sock *sk) +{ + struct raw_sock *ro = raw_sk(sk); + + free_percpu(ro->uniq); + can_sock_destruct(sk); +} + static int raw_init(struct sock *sk) { struct raw_sock *ro = raw_sk(sk); @@ -386,6 +394,8 @@ static int raw_init(struct sock *sk) if (unlikely(!ro->uniq)) return -ENOMEM; + sk->sk_destruct = raw_sock_destruct; + /* set notifier */ spin_lock(&raw_notifier_lock); list_add_tail(&ro->notifier, &raw_notifier_list); @@ -435,7 +445,6 @@ static int raw_release(struct socket *sock) ro->bound = 0; ro->dev = NULL; ro->count = 0; - free_percpu(ro->uniq); sock_orphan(sk); sock->sk = NULL; diff --git a/net/ceph/auth.c b/net/ceph/auth.c index d38c9eadbe2f1..0d75679c6a7ed 100644 --- a/net/ceph/auth.c +++ b/net/ceph/auth.c @@ -205,9 +205,9 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, s32 result; u64 global_id; void *payload, *payload_end; - int payload_len; + u32 payload_len; char *result_msg; - int result_msg_len; + u32 result_msg_len; int ret = -EINVAL; mutex_lock(&ac->mutex); @@ -217,10 +217,12 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, result = ceph_decode_32(&p); global_id = ceph_decode_64(&p); payload_len = ceph_decode_32(&p); + ceph_decode_need(&p, end, payload_len, bad); payload = p; p += payload_len; ceph_decode_need(&p, end, sizeof(u32), bad); result_msg_len = ceph_decode_32(&p); + ceph_decode_need(&p, end, result_msg_len, bad); result_msg = p; p += result_msg_len; if (p != end) diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 01b2ce1e8fc06..5601732cf4faa 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -37,9 +37,6 @@ static int set_secret(struct ceph_crypto_key *key, void *buf) return -ENOTSUPP; } - if (!key->len) - return -EINVAL; - key->key = kmemdup(buf, key->len, GFP_NOIO); if (!key->key) { ret = -ENOMEM; @@ -83,6 +80,11 @@ int ceph_crypto_key_decode(struct ceph_crypto_key *key, void **p, void *end) ceph_decode_copy(p, &key->created, sizeof(key->created)); key->len = ceph_decode_16(p); ceph_decode_need(p, end, key->len, bad); + if (key->len > CEPH_MAX_KEY_LEN) { + pr_err("secret too big %d\n", key->len); + return -EINVAL; + } + ret = set_secret(key, *p); memzero_explicit(*p, key->len); *p += key->len; diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h index 23de29fc613cf..a20bad6d1e964 100644 --- a/net/ceph/crypto.h +++ b/net/ceph/crypto.h @@ -5,7 +5,7 @@ #include #include -#define CEPH_KEY_LEN 16 +#define CEPH_MAX_KEY_LEN 16 #define CEPH_MAX_CON_SECRET_LEN 64 /* diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index 9e48623018a36..5c7435fc6483f 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -392,7 +392,7 @@ static int head_onwire_len(int ctrl_len, bool secure) int head_len; int rem_len; - BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN); + BUG_ON(ctrl_len < 1 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN); if (secure) { head_len = CEPH_PREAMBLE_SECURE_LEN; @@ -401,9 +401,7 @@ static int head_onwire_len(int ctrl_len, bool secure) head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN; } } else { - head_len = CEPH_PREAMBLE_PLAIN_LEN; - if (ctrl_len) - head_len += ctrl_len + CEPH_CRC_LEN; + head_len = CEPH_PREAMBLE_PLAIN_LEN + ctrl_len + CEPH_CRC_LEN; } return head_len; } @@ -528,11 +526,16 @@ static int decode_preamble(void *p, struct ceph_frame_desc *desc) desc->fd_aligns[i] = ceph_decode_16(&p); } - if (desc->fd_lens[0] < 0 || + /* + * This would fire for FRAME_TAG_WAIT (it has one empty + * segment), but we should never get it as client. + */ + if (desc->fd_lens[0] < 1 || desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) { pr_err("bad control segment length %d\n", desc->fd_lens[0]); return -EINVAL; } + if (desc->fd_lens[1] < 0 || desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) { pr_err("bad front segment length %d\n", desc->fd_lens[1]); @@ -549,10 +552,6 @@ static int decode_preamble(void *p, struct ceph_frame_desc *desc) return -EINVAL; } - /* - * This would fire for FRAME_TAG_WAIT (it has one empty - * segment), but we should never get it as client. - */ if (!desc->fd_lens[desc->fd_seg_cnt - 1]) { pr_err("last segment empty, segment count %d\n", desc->fd_seg_cnt); @@ -2361,7 +2360,7 @@ static int process_auth_reply_more(struct ceph_connection *con, */ static int process_auth_done(struct ceph_connection *con, void *p, void *end) { - u8 session_key_buf[CEPH_KEY_LEN + 16]; + u8 session_key_buf[CEPH_MAX_KEY_LEN + 16]; u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16]; u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16); u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16); @@ -2377,7 +2376,9 @@ static int process_auth_done(struct ceph_connection *con, void *p, void *end) ceph_decode_64_safe(&p, end, global_id, bad); ceph_decode_32_safe(&p, end, con->v2.con_mode, bad); + ceph_decode_32_safe(&p, end, payload_len, bad); + ceph_decode_need(&p, end, payload_len, bad); dout("%s con %p global_id %llu con_mode %d payload_len %d\n", __func__, con, global_id, con->v2.con_mode, payload_len); @@ -2833,12 +2834,15 @@ static int process_message_header(struct ceph_connection *con, void *p, void *end) { struct ceph_frame_desc *desc = &con->v2.in_desc; - struct ceph_msg_header2 *hdr2 = p; + struct ceph_msg_header2 *hdr2; struct ceph_msg_header hdr; int skip; int ret; u64 seq; + ceph_decode_need(&p, end, sizeof(*hdr2), bad); + hdr2 = p; + /* verify seq# */ seq = le64_to_cpu(hdr2->seq); if ((s64)seq - (s64)con->in_seq < 1) { @@ -2869,6 +2873,10 @@ static int process_message_header(struct ceph_connection *con, WARN_ON(!con->in_msg); WARN_ON(con->in_msg->con != con); return 1; + +bad: + pr_err("failed to decode message header\n"); + return -EINVAL; } static int process_message(struct ceph_connection *con) @@ -2898,6 +2906,11 @@ static int __handle_control(struct ceph_connection *con, void *p) if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE) return process_control(con, p, end); + if (con->state != CEPH_CON_S_OPEN) { + con->error_msg = "protocol error, unexpected message"; + return -EINVAL; + } + ret = process_message_header(con, p, end); if (ret < 0) return ret; diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index c227ececa9254..94a7a82ca4756 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -72,8 +72,8 @@ static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2) struct ceph_monmap *monmap = NULL; struct ceph_fsid fsid; u32 struct_len; - int blob_len; - int num_mon; + u32 blob_len; + u32 num_mon; u8 struct_v; u32 epoch; int ret; @@ -112,7 +112,7 @@ static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2) } ceph_decode_32_safe(p, end, num_mon, e_inval); - dout("%s fsid %pU epoch %u num_mon %d\n", __func__, &fsid, epoch, + dout("%s fsid %pU epoch %u num_mon %u\n", __func__, &fsid, epoch, num_mon); if (num_mon > CEPH_MAX_MON) goto e_inval; @@ -1417,7 +1417,7 @@ static int mon_handle_auth_done(struct ceph_connection *con, if (!ret) finish_hunting(monc); mutex_unlock(&monc->mutex); - return 0; + return ret; } static int mon_handle_auth_bad_method(struct ceph_connection *con, diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 6664ea73ccf81..6d7d8c7d7d3fe 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -1588,6 +1588,7 @@ static enum calc_target_result calc_target(struct ceph_osd_client *osdc, struct ceph_pg_pool_info *pi; struct ceph_pg pgid, last_pgid; struct ceph_osds up, acting; + bool should_be_paused; bool is_read = t->flags & CEPH_OSD_FLAG_READ; bool is_write = t->flags & CEPH_OSD_FLAG_WRITE; bool force_resend = false; @@ -1656,10 +1657,16 @@ static enum calc_target_result calc_target(struct ceph_osd_client *osdc, &last_pgid)) force_resend = true; - if (t->paused && !target_should_be_paused(osdc, t, pi)) { - t->paused = false; + should_be_paused = target_should_be_paused(osdc, t, pi); + if (t->paused && !should_be_paused) { unpaused = true; } + if (t->paused != should_be_paused) { + dout("%s t %p paused %d -> %d\n", __func__, t, t->paused, + should_be_paused); + t->paused = should_be_paused; + } + legacy_change = ceph_pg_compare(&t->pgid, &pgid) || ceph_osds_changed(&t->acting, &acting, t->used_replica || any_change); @@ -4283,6 +4290,9 @@ static void osd_fault(struct ceph_connection *con) goto out_unlock; } + osd->o_sparse_op_idx = -1; + ceph_init_sparse_read(&osd->o_sparse_read); + if (!reopen_osd(osd)) kick_osd_requests(osd); maybe_request_map(osdc); diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index d245fa508e1cc..7c76eb9d6ceec 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -241,22 +241,26 @@ static struct crush_choose_arg_map *alloc_choose_arg_map(void) static void free_choose_arg_map(struct crush_choose_arg_map *arg_map) { - if (arg_map) { - int i, j; + int i, j; - WARN_ON(!RB_EMPTY_NODE(&arg_map->node)); + if (!arg_map) + return; + WARN_ON(!RB_EMPTY_NODE(&arg_map->node)); + + if (arg_map->args) { for (i = 0; i < arg_map->size; i++) { struct crush_choose_arg *arg = &arg_map->args[i]; - - for (j = 0; j < arg->weight_set_size; j++) - kfree(arg->weight_set[j].weights); - kfree(arg->weight_set); + if (arg->weight_set) { + for (j = 0; j < arg->weight_set_size; j++) + kfree(arg->weight_set[j].weights); + kfree(arg->weight_set); + } kfree(arg->ids); } kfree(arg_map->args); - kfree(arg_map); } + kfree(arg_map); } DEFINE_RB_FUNCS(choose_arg_map, struct crush_choose_arg_map, choose_args_index, @@ -806,51 +810,49 @@ static int decode_pool(void **p, void *end, struct ceph_pg_pool_info *pi) ceph_decode_need(p, end, len, bad); pool_end = *p + len; + ceph_decode_need(p, end, 4 + 4 + 4, bad); pi->type = ceph_decode_8(p); pi->size = ceph_decode_8(p); pi->crush_ruleset = ceph_decode_8(p); pi->object_hash = ceph_decode_8(p); - pi->pg_num = ceph_decode_32(p); pi->pgp_num = ceph_decode_32(p); - *p += 4 + 4; /* skip lpg* */ - *p += 4; /* skip last_change */ - *p += 8 + 4; /* skip snap_seq, snap_epoch */ + /* lpg*, last_change, snap_seq, snap_epoch */ + ceph_decode_skip_n(p, end, 8 + 4 + 8 + 4, bad); /* skip snaps */ - num = ceph_decode_32(p); + ceph_decode_32_safe(p, end, num, bad); while (num--) { - *p += 8; /* snapid key */ - *p += 1 + 1; /* versions */ - len = ceph_decode_32(p); - *p += len; + /* snapid key, pool snap (with versions) */ + ceph_decode_skip_n(p, end, 8 + 2, bad); + ceph_decode_skip_string(p, end, bad); } - /* skip removed_snaps */ - num = ceph_decode_32(p); - *p += num * (8 + 8); + /* removed_snaps */ + ceph_decode_skip_map(p, end, 64, 64, bad); + ceph_decode_need(p, end, 8 + 8 + 4, bad); *p += 8; /* skip auid */ pi->flags = ceph_decode_64(p); *p += 4; /* skip crash_replay_interval */ if (ev >= 7) - pi->min_size = ceph_decode_8(p); + ceph_decode_8_safe(p, end, pi->min_size, bad); else pi->min_size = pi->size - pi->size / 2; if (ev >= 8) - *p += 8 + 8; /* skip quota_max_* */ + /* quota_max_* */ + ceph_decode_skip_n(p, end, 8 + 8, bad); if (ev >= 9) { - /* skip tiers */ - num = ceph_decode_32(p); - *p += num * 8; + /* tiers */ + ceph_decode_skip_set(p, end, 64, bad); + ceph_decode_need(p, end, 8 + 1 + 8 + 8, bad); *p += 8; /* skip tier_of */ *p += 1; /* skip cache_mode */ - pi->read_tier = ceph_decode_64(p); pi->write_tier = ceph_decode_64(p); } else { @@ -858,86 +860,76 @@ static int decode_pool(void **p, void *end, struct ceph_pg_pool_info *pi) pi->write_tier = -1; } - if (ev >= 10) { - /* skip properties */ - num = ceph_decode_32(p); - while (num--) { - len = ceph_decode_32(p); - *p += len; /* key */ - len = ceph_decode_32(p); - *p += len; /* val */ - } - } + if (ev >= 10) + /* properties */ + ceph_decode_skip_map(p, end, string, string, bad); if (ev >= 11) { - /* skip hit_set_params */ - *p += 1 + 1; /* versions */ - len = ceph_decode_32(p); - *p += len; + /* hit_set_params (with versions) */ + ceph_decode_skip_n(p, end, 2, bad); + ceph_decode_skip_string(p, end, bad); - *p += 4; /* skip hit_set_period */ - *p += 4; /* skip hit_set_count */ + /* hit_set_period, hit_set_count */ + ceph_decode_skip_n(p, end, 4 + 4, bad); } if (ev >= 12) - *p += 4; /* skip stripe_width */ + /* stripe_width */ + ceph_decode_skip_32(p, end, bad); - if (ev >= 13) { - *p += 8; /* skip target_max_bytes */ - *p += 8; /* skip target_max_objects */ - *p += 4; /* skip cache_target_dirty_ratio_micro */ - *p += 4; /* skip cache_target_full_ratio_micro */ - *p += 4; /* skip cache_min_flush_age */ - *p += 4; /* skip cache_min_evict_age */ - } + if (ev >= 13) + /* target_max_*, cache_target_*, cache_min_* */ + ceph_decode_skip_n(p, end, 16 + 8 + 8, bad); - if (ev >= 14) { - /* skip erasure_code_profile */ - len = ceph_decode_32(p); - *p += len; - } + if (ev >= 14) + /* erasure_code_profile */ + ceph_decode_skip_string(p, end, bad); /* * last_force_op_resend_preluminous, will be overridden if the * map was encoded with RESEND_ON_SPLIT */ if (ev >= 15) - pi->last_force_request_resend = ceph_decode_32(p); + ceph_decode_32_safe(p, end, pi->last_force_request_resend, bad); else pi->last_force_request_resend = 0; if (ev >= 16) - *p += 4; /* skip min_read_recency_for_promote */ + /* min_read_recency_for_promote */ + ceph_decode_skip_32(p, end, bad); if (ev >= 17) - *p += 8; /* skip expected_num_objects */ + /* expected_num_objects */ + ceph_decode_skip_64(p, end, bad); if (ev >= 19) - *p += 4; /* skip cache_target_dirty_high_ratio_micro */ + /* cache_target_dirty_high_ratio_micro */ + ceph_decode_skip_32(p, end, bad); if (ev >= 20) - *p += 4; /* skip min_write_recency_for_promote */ + /* min_write_recency_for_promote */ + ceph_decode_skip_32(p, end, bad); if (ev >= 21) - *p += 1; /* skip use_gmt_hitset */ + /* use_gmt_hitset */ + ceph_decode_skip_8(p, end, bad); if (ev >= 22) - *p += 1; /* skip fast_read */ + /* fast_read */ + ceph_decode_skip_8(p, end, bad); - if (ev >= 23) { - *p += 4; /* skip hit_set_grade_decay_rate */ - *p += 4; /* skip hit_set_search_last_n */ - } + if (ev >= 23) + /* hit_set_grade_decay_rate, hit_set_search_last_n */ + ceph_decode_skip_n(p, end, 4 + 4, bad); if (ev >= 24) { - /* skip opts */ - *p += 1 + 1; /* versions */ - len = ceph_decode_32(p); - *p += len; + /* opts (with versions) */ + ceph_decode_skip_n(p, end, 2, bad); + ceph_decode_skip_string(p, end, bad); } if (ev >= 25) - pi->last_force_request_resend = ceph_decode_32(p); + ceph_decode_32_safe(p, end, pi->last_force_request_resend, bad); /* ignore the rest */ @@ -1991,11 +1983,13 @@ struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, bool msgr2, sizeof(u64) + sizeof(u32), e_inval); ceph_decode_copy(p, &fsid, sizeof(fsid)); epoch = ceph_decode_32(p); - BUG_ON(epoch != map->epoch+1); ceph_decode_copy(p, &modified, sizeof(modified)); new_pool_max = ceph_decode_64(p); new_flags = ceph_decode_32(p); + if (epoch != map->epoch + 1) + goto e_inval; + /* full map? */ ceph_decode_32_safe(p, end, len, e_inval); if (len > 0) { diff --git a/net/core/dev.c b/net/core/dev.c index 2acfa44927daa..29f2f35ae5ebe 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -478,15 +478,21 @@ static const unsigned short netdev_lock_type[] = { ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, + ARPHRD_CAN, ARPHRD_MCTP, ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, - ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, + ARPHRD_RAWHDLC, ARPHRD_RAWIP, + ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, - ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, - ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; + ARPHRD_IEEE80211_RADIOTAP, + ARPHRD_IEEE802154, ARPHRD_IEEE802154_MONITOR, + ARPHRD_PHONET, ARPHRD_PHONET_PIPE, + ARPHRD_CAIF, ARPHRD_IP6GRE, ARPHRD_NETLINK, ARPHRD_6LOWPAN, + ARPHRD_VSOCKMON, + ARPHRD_VOID, ARPHRD_NONE}; static const char *const netdev_lock_name[] = { "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", @@ -495,15 +501,21 @@ static const char *const netdev_lock_name[] = { "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", + "_xmit_CAN", "_xmit_MCTP", "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", - "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", + "_xmit_RAWHDLC", "_xmit_RAWIP", + "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", - "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", - "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; + "_xmit_IEEE80211_RADIOTAP", + "_xmit_IEEE802154", "_xmit_IEEE802154_MONITOR", + "_xmit_PHONET", "_xmit_PHONET_PIPE", + "_xmit_CAIF", "_xmit_IP6GRE", "_xmit_NETLINK", "_xmit_6LOWPAN", + "_xmit_VSOCKMON", + "_xmit_VOID", "_xmit_NONE"}; static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; @@ -516,6 +528,7 @@ static inline unsigned short netdev_lock_pos(unsigned short dev_type) if (netdev_lock_type[i] == dev_type) return i; /* the last key is used by default */ + WARN_ONCE(1, "netdev_lock_pos() could not find dev_type=%u\n", dev_type); return ARRAY_SIZE(netdev_lock_type) - 1; } @@ -725,7 +738,7 @@ static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack) { int k = stack->num_paths++; - if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX)) + if (k >= NET_DEVICE_PATH_STACK_MAX) return NULL; return &stack->path[k]; @@ -3742,6 +3755,22 @@ static netdev_features_t dflt_features_check(struct sk_buff *skb, return vlan_features_check(skb, features); } +static bool skb_gso_has_extension_hdr(const struct sk_buff *skb) +{ + if (!skb->encapsulation) + return ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && + vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && + skb_transport_header_was_set(skb) && + skb_network_header_len(skb) != sizeof(struct ipv6hdr)); + else + return (!skb_inner_network_header_was_set(skb) || + ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && + inner_ip_hdr(skb)->version == 6)) && + skb_inner_network_header_len(skb) != sizeof(struct ipv6hdr))); +} + static netdev_features_t gso_features_check(const struct sk_buff *skb, struct net_device *dev, netdev_features_t features) @@ -3778,22 +3807,23 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, * segmentation-offloads.rst). */ if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { - struct iphdr *iph = skb->encapsulation ? - inner_ip_hdr(skb) : ip_hdr(skb); + const struct iphdr *iph; + struct iphdr _iph; + int nhoff = skb->encapsulation ? + skb_inner_network_offset(skb) : + skb_network_offset(skb); + + iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); - if (!(iph->frag_off & htons(IP_DF))) - features &= ~NETIF_F_TSO_MANGLEID; + if (!iph || !(iph->frag_off & htons(IP_DF))) + features &= ~dev->mangleid_features; } /* NETIF_F_IPV6_CSUM does not support IPv6 extension headers, * so neither does TSO that depends on it. */ if (features & NETIF_F_IPV6_CSUM && - (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || - (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && - vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && - skb_transport_header_was_set(skb) && - skb_network_header_len(skb) != sizeof(struct ipv6hdr) && + skb_gso_has_extension_hdr(skb) && !ipv6_has_hopopt_jumbo(skb)) features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4); @@ -3962,7 +3992,7 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, if (shinfo->nr_frags > 0) { niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0])); if (net_is_devmem_iov(niov) && - net_devmem_iov_binding(niov)->dev != dev) + READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev) goto out_free; } @@ -4745,10 +4775,9 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) if (dev->flags & IFF_UP) { int cpu = smp_processor_id(); /* ok because BHs are off */ - /* Other cpus might concurrently change txq->xmit_lock_owner - * to -1 or to their cpu id, but not to our id. - */ - if (READ_ONCE(txq->xmit_lock_owner) != cpu) { + if (!netif_tx_owned(txq, cpu)) { + bool is_list = false; + if (dev_xmit_recursion()) goto recursion_alert; @@ -4759,17 +4788,28 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) HARD_TX_LOCK(dev, txq, cpu); if (!netif_xmit_stopped(txq)) { + is_list = !!skb->next; + dev_xmit_recursion_inc(); skb = dev_hard_start_xmit(skb, dev, txq, &rc); dev_xmit_recursion_dec(); - if (dev_xmit_complete(rc)) { - HARD_TX_UNLOCK(dev, txq); - goto out; - } + + /* GSO segments a single SKB into + * a list of frames. TCP expects error + * to mean none of the data was sent. + */ + if (is_list) + rc = NETDEV_TX_OK; } HARD_TX_UNLOCK(dev, txq); + if (!skb) /* xmit completed */ + goto out; + net_crit_ratelimited("Virtual device %s asks to queue packet!\n", dev->name); + /* NETDEV_TX_BUSY or queue was stopped */ + if (!is_list) + rc = -ENETDOWN; } else { /* Recursion is detected! It is possible, * unfortunately @@ -4777,10 +4817,10 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) recursion_alert: net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", dev->name); + rc = -ENETDOWN; } } - rc = -ENETDOWN; rcu_read_unlock_bh(); dev_core_stats_tx_dropped_inc(dev); @@ -4919,8 +4959,7 @@ static bool rps_flow_is_active(struct rps_dev_flow *rflow, static struct rps_dev_flow * set_rps_cpu(struct net_device *dev, struct sk_buff *skb, - struct rps_dev_flow *rflow, u16 next_cpu, u32 hash, - u32 flow_id) + struct rps_dev_flow *rflow, u16 next_cpu, u32 hash) { if (next_cpu < nr_cpu_ids) { u32 head; @@ -4931,6 +4970,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, struct rps_dev_flow *tmp_rflow; unsigned int tmp_cpu; u16 rxq_index; + u32 flow_id; int rc; /* Should we steer this flow to a different hardware queue? */ @@ -4946,6 +4986,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, if (!flow_table) goto out; + flow_id = rfs_slot(hash, flow_table); tmp_rflow = &flow_table->flows[flow_id]; tmp_cpu = READ_ONCE(tmp_rflow->cpu); @@ -4993,7 +5034,6 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, struct rps_dev_flow_table *flow_table; struct rps_map *map; int cpu = -1; - u32 flow_id; u32 tcpu; u32 hash; @@ -5040,8 +5080,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, /* OK, now we know there is a match, * we can look at the local (per receive queue) flow table */ - flow_id = rfs_slot(hash, flow_table); - rflow = &flow_table->flows[flow_id]; + rflow = &flow_table->flows[rfs_slot(hash, flow_table)]; tcpu = rflow->cpu; /* @@ -5060,8 +5099,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - rflow->last_qtail)) >= 0)) { tcpu = next_cpu; - rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash, - flow_id); + rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash); } if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { @@ -11274,6 +11312,9 @@ int register_netdevice(struct net_device *dev) if (dev->hw_enc_features & NETIF_F_TSO) dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; + /* TSO_MANGLEID belongs in mangleid_features by definition */ + dev->mangleid_features |= NETIF_F_TSO_MANGLEID; + /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. */ dev->vlan_features |= NETIF_F_HIGHDMA; @@ -12646,6 +12687,94 @@ netdev_features_t netdev_increment_features(netdev_features_t all, } EXPORT_SYMBOL(netdev_increment_features); +/** + * netdev_compute_master_upper_features - compute feature from lowers + * @dev: the upper device + * @update_header: whether to update upper device's header_len/headroom/tailroom + * + * Recompute the upper device's feature based on all lower devices. + */ +void netdev_compute_master_upper_features(struct net_device *dev, bool update_header) +{ + unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; + netdev_features_t gso_partial_features = MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES; + netdev_features_t xfrm_features = MASTER_UPPER_DEV_XFRM_FEATURES; + netdev_features_t mpls_features = MASTER_UPPER_DEV_MPLS_FEATURES; + netdev_features_t vlan_features = MASTER_UPPER_DEV_VLAN_FEATURES; + netdev_features_t enc_features = MASTER_UPPER_DEV_ENC_FEATURES; + unsigned short max_header_len = ETH_HLEN; + unsigned int tso_max_size = TSO_MAX_SIZE; + unsigned short max_headroom = 0; + unsigned short max_tailroom = 0; + u16 tso_max_segs = TSO_MAX_SEGS; + struct net_device *lower_dev; + struct list_head *iter; + + mpls_features = netdev_base_features(mpls_features); + vlan_features = netdev_base_features(vlan_features); + enc_features = netdev_base_features(enc_features); + + netdev_for_each_lower_dev(dev, lower_dev, iter) { + gso_partial_features = netdev_increment_features(gso_partial_features, + lower_dev->gso_partial_features, + MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES); + + vlan_features = netdev_increment_features(vlan_features, + lower_dev->vlan_features, + MASTER_UPPER_DEV_VLAN_FEATURES); + + enc_features = netdev_increment_features(enc_features, + lower_dev->hw_enc_features, + MASTER_UPPER_DEV_ENC_FEATURES); + + if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) + xfrm_features = netdev_increment_features(xfrm_features, + lower_dev->hw_enc_features, + MASTER_UPPER_DEV_XFRM_FEATURES); + + mpls_features = netdev_increment_features(mpls_features, + lower_dev->mpls_features, + MASTER_UPPER_DEV_MPLS_FEATURES); + + dst_release_flag &= lower_dev->priv_flags; + + if (update_header) { + max_header_len = max(max_header_len, lower_dev->hard_header_len); + max_headroom = max(max_headroom, lower_dev->needed_headroom); + max_tailroom = max(max_tailroom, lower_dev->needed_tailroom); + } + + tso_max_size = min(tso_max_size, lower_dev->tso_max_size); + tso_max_segs = min(tso_max_segs, lower_dev->tso_max_segs); + } + + dev->gso_partial_features = gso_partial_features; + dev->vlan_features = vlan_features; + dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_STAG_TX; + if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) + dev->hw_enc_features |= xfrm_features; + dev->mpls_features = mpls_features; + + dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; + if ((dev->priv_flags & IFF_XMIT_DST_RELEASE_PERM) && + dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM)) + dev->priv_flags |= IFF_XMIT_DST_RELEASE; + + if (update_header) { + dev->hard_header_len = max_header_len; + dev->needed_headroom = max_headroom; + dev->needed_tailroom = max_tailroom; + } + + netif_set_tso_max_segs(dev, tso_max_segs); + netif_set_tso_max_size(dev, tso_max_size); + + netdev_change_features(dev); +} +EXPORT_SYMBOL(netdev_compute_master_upper_features); + static struct hlist_head * __net_init netdev_create_hash(void) { int i; diff --git a/net/core/dev.h b/net/core/dev.h index 900880e8b5b4b..b458e2777725e 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -29,7 +29,6 @@ struct napi_struct * netdev_napi_by_id_lock(struct net *net, unsigned int napi_id); struct net_device *dev_get_by_napi_id(unsigned int napi_id); -struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex); struct net_device *__netdev_put_lock(struct net_device *dev, struct net *net); struct net_device * netdev_xa_find_lock(struct net *net, struct net_device *dev, @@ -359,41 +358,6 @@ static inline void napi_assert_will_not_race(const struct napi_struct *napi) void kick_defer_list_purge(unsigned int cpu); -#define XMIT_RECURSION_LIMIT 8 - -#ifndef CONFIG_PREEMPT_RT -static inline bool dev_xmit_recursion(void) -{ - return unlikely(__this_cpu_read(softnet_data.xmit.recursion) > - XMIT_RECURSION_LIMIT); -} - -static inline void dev_xmit_recursion_inc(void) -{ - __this_cpu_inc(softnet_data.xmit.recursion); -} - -static inline void dev_xmit_recursion_dec(void) -{ - __this_cpu_dec(softnet_data.xmit.recursion); -} -#else -static inline bool dev_xmit_recursion(void) -{ - return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT); -} - -static inline void dev_xmit_recursion_inc(void) -{ - current->net_xmit.recursion++; -} - -static inline void dev_xmit_recursion_dec(void) -{ - current->net_xmit.recursion--; -} -#endif - int dev_set_hwtstamp_phylib(struct net_device *dev, struct kernel_hwtstamp_config *cfg, struct netlink_ext_ack *extack); diff --git a/net/core/devmem.c b/net/core/devmem.c index 1d04754bc756d..448f6582ac1ae 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -387,7 +387,8 @@ struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, * net_device. */ dst_dev = dst_dev_rcu(dst); - if (unlikely(!dst_dev) || unlikely(dst_dev != binding->dev)) { + if (unlikely(!dst_dev) || + unlikely(dst_dev != READ_ONCE(binding->dev))) { err = -ENODEV; goto out_unlock; } @@ -504,7 +505,8 @@ static void mp_dmabuf_devmem_uninstall(void *mp_priv, xa_erase(&binding->bound_rxqs, xa_idx); if (xa_empty(&binding->bound_rxqs)) { mutex_lock(&binding->lock); - binding->dev = NULL; + ASSERT_EXCLUSIVE_WRITER(binding->dev); + WRITE_ONCE(binding->dev, NULL); mutex_unlock(&binding->lock); } break; diff --git a/net/core/dst.c b/net/core/dst.c index e9d35f49c9e78..1dae26c51ebec 100644 --- a/net/core/dst.c +++ b/net/core/dst.c @@ -68,6 +68,7 @@ void dst_init(struct dst_entry *dst, struct dst_ops *ops, dst->lwtstate = NULL; rcuref_init(&dst->__rcuref, 1); INIT_LIST_HEAD(&dst->rt_uncached); + dst->rt_uncached_list = NULL; dst->__use = 0; dst->lastuse = jiffies; dst->flags = flags; diff --git a/net/core/filter.c b/net/core/filter.c index fa06c5a08e22f..3d4bf4d2a1a4b 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -2289,12 +2289,12 @@ static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev, err = bpf_out_neigh_v6(net, skb, dev, nh); if (unlikely(net_xmit_eval(err))) - DEV_STATS_INC(dev, tx_errors); + dev_core_stats_tx_dropped_inc(dev); else ret = NET_XMIT_SUCCESS; goto out_xmit; out_drop: - DEV_STATS_INC(dev, tx_errors); + dev_core_stats_tx_dropped_inc(dev); kfree_skb(skb); out_xmit: return ret; @@ -2396,12 +2396,12 @@ static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev, err = bpf_out_neigh_v4(net, skb, dev, nh); if (unlikely(net_xmit_eval(err))) - DEV_STATS_INC(dev, tx_errors); + dev_core_stats_tx_dropped_inc(dev); else ret = NET_XMIT_SUCCESS; goto out_xmit; out_drop: - DEV_STATS_INC(dev, tx_errors); + dev_core_stats_tx_dropped_inc(dev); kfree_skb(skb); out_xmit: return ret; @@ -2458,6 +2458,13 @@ BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags) if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))) return -EINVAL; + /* BPF test infra's convert___skb_to_skb() can create type-less + * GSO packets. gso_features_check() will detect this as a bad + * offload. However, lets not leak them out in the first place. + */ + if (unlikely(skb_is_gso(skb) && !skb_shinfo(skb)->gso_type)) + return -EBADMSG; + dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex); if (unlikely(!dev)) return -EINVAL; @@ -3345,6 +3352,7 @@ static int bpf_skb_proto_4_to_6(struct sk_buff *skb) shinfo->gso_type &= ~SKB_GSO_TCPV4; shinfo->gso_type |= SKB_GSO_TCPV6; } + shinfo->gso_type |= SKB_GSO_DODGY; } bpf_skb_change_protocol(skb, ETH_P_IPV6); @@ -3375,6 +3383,7 @@ static int bpf_skb_proto_6_to_4(struct sk_buff *skb) shinfo->gso_type &= ~SKB_GSO_TCPV6; shinfo->gso_type |= SKB_GSO_TCPV4; } + shinfo->gso_type |= SKB_GSO_DODGY; } bpf_skb_change_protocol(skb, ETH_P_IP); @@ -4124,7 +4133,7 @@ static const struct bpf_func_proto bpf_xdp_store_bytes_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, }; @@ -4138,12 +4147,14 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset) struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags - 1]; struct xdp_rxq_info *rxq = xdp->rxq; - unsigned int tailroom; + int tailroom; if (!rxq->frag_size || rxq->frag_size > xdp->frame_sz) return -EOPNOTSUPP; - tailroom = rxq->frag_size - skb_frag_size(frag) - skb_frag_off(frag); + tailroom = rxq->frag_size - skb_frag_size(frag) - + skb_frag_off(frag) % rxq->frag_size; + WARN_ON_ONCE(tailroom < 0); if (unlikely(offset > tailroom)) return -EINVAL; @@ -6316,7 +6327,7 @@ static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6371,7 +6382,7 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6422,9 +6433,12 @@ BPF_CALL_5(bpf_skb_check_mtu, struct sk_buff *, skb, */ if (skb_is_gso(skb)) { ret = BPF_MTU_CHK_RET_SUCCESS; - if (flags & BPF_MTU_CHK_SEGS && - !skb_gso_validate_network_len(skb, mtu)) - ret = BPF_MTU_CHK_RET_SEGS_TOOBIG; + if (flags & BPF_MTU_CHK_SEGS) { + if (!skb_transport_header_was_set(skb)) + return -EINVAL; + if (!skb_gso_validate_network_len(skb, mtu)) + ret = BPF_MTU_CHK_RET_SEGS_TOOBIG; + } } out: *mtu_len = mtu; @@ -7922,9 +7936,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -7954,9 +7968,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -7974,9 +7988,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; @@ -7998,9 +8012,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; #endif /* CONFIG_SYN_COOKIES */ diff --git a/net/core/gro.c b/net/core/gro.c index 76f9c37124221..ef61695fbdbb6 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -265,6 +265,8 @@ static void gro_complete(struct gro_node *gro, struct sk_buff *skb) goto out; } + /* NICs can feed encapsulated packets into GRO */ + skb->encapsulation = 0; rcu_read_lock(); list_for_each_entry_rcu(ptype, head, list) { if (ptype->type != type || !ptype->callbacks.gro_complete) @@ -415,7 +417,7 @@ static void gro_pull_from_frag0(struct sk_buff *skb, int grow) { struct skb_shared_info *pinfo = skb_shinfo(skb); - BUG_ON(skb->end - skb->tail < grow); + DEBUG_NET_WARN_ON_ONCE(skb->end - skb->tail < grow); memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); diff --git a/net/core/link_watch.c b/net/core/link_watch.c index 212cde35affa7..25c455c10a01c 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -185,10 +185,6 @@ static void linkwatch_do_dev(struct net_device *dev) netif_state_change(dev); } - /* Note: our callers are responsible for calling netdev_tracker_free(). - * This is the reason we use __dev_put() instead of dev_put(). - */ - __dev_put(dev); } static void __linkwatch_run_queue(int urgent_only) @@ -243,6 +239,11 @@ static void __linkwatch_run_queue(int urgent_only) netdev_lock_ops(dev); linkwatch_do_dev(dev); netdev_unlock_ops(dev); + /* Use __dev_put() because netdev_tracker_free() was already + * called above. Must be after netdev_unlock_ops() to prevent + * netdev_run_todo() from freeing the device while still in use. + */ + __dev_put(dev); do_dev--; spin_lock_irq(&lweventlist_lock); } @@ -278,8 +279,13 @@ void __linkwatch_sync_dev(struct net_device *dev) { netdev_ops_assert_locked(dev); - if (linkwatch_clean_dev(dev)) + if (linkwatch_clean_dev(dev)) { linkwatch_do_dev(dev); + /* Use __dev_put() because netdev_tracker_free() was already + * called inside linkwatch_clean_dev(). + */ + __dev_put(dev); + } } void linkwatch_sync_dev(struct net_device *dev) @@ -288,6 +294,10 @@ void linkwatch_sync_dev(struct net_device *dev) netdev_lock_ops(dev); linkwatch_do_dev(dev); netdev_unlock_ops(dev); + /* Use __dev_put() because netdev_tracker_free() was already + * called inside linkwatch_clean_dev(). + */ + __dev_put(dev); } } diff --git a/net/core/neighbour.c b/net/core/neighbour.c index bddfa389effa7..6dab4d1c2263d 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -821,7 +821,8 @@ int pneigh_create(struct neigh_table *tbl, struct net *net, update: WRITE_ONCE(n->flags, flags); n->permanent = permanent; - WRITE_ONCE(n->protocol, protocol); + if (protocol) + WRITE_ONCE(n->protocol, protocol); out: mutex_unlock(&tbl->phash_lock); return err; diff --git a/net/core/net-procfs.c b/net/core/net-procfs.c index 70e0e9a3b650c..7dbfa6109f0b8 100644 --- a/net/core/net-procfs.c +++ b/net/core/net-procfs.c @@ -170,8 +170,14 @@ static const struct seq_operations softnet_seq_ops = { .show = softnet_seq_show, }; +struct ptype_iter_state { + struct seq_net_private p; + struct net_device *dev; +}; + static void *ptype_get_idx(struct seq_file *seq, loff_t pos) { + struct ptype_iter_state *iter = seq->private; struct list_head *ptype_list = NULL; struct packet_type *pt = NULL; struct net_device *dev; @@ -181,12 +187,16 @@ static void *ptype_get_idx(struct seq_file *seq, loff_t pos) for_each_netdev_rcu(seq_file_net(seq), dev) { ptype_list = &dev->ptype_all; list_for_each_entry_rcu(pt, ptype_list, list) { - if (i == pos) + if (i == pos) { + iter->dev = dev; return pt; + } ++i; } } + iter->dev = NULL; + list_for_each_entry_rcu(pt, &seq_file_net(seq)->ptype_all, list) { if (i == pos) return pt; @@ -218,6 +228,7 @@ static void *ptype_seq_start(struct seq_file *seq, loff_t *pos) static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) { + struct ptype_iter_state *iter = seq->private; struct net *net = seq_file_net(seq); struct net_device *dev; struct packet_type *pt; @@ -229,19 +240,21 @@ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) return ptype_get_idx(seq, 0); pt = v; - nxt = pt->list.next; - if (pt->dev) { - if (nxt != &pt->dev->ptype_all) + nxt = READ_ONCE(pt->list.next); + dev = iter->dev; + if (dev) { + if (nxt != &dev->ptype_all) goto found; - dev = pt->dev; for_each_netdev_continue_rcu(seq_file_net(seq), dev) { - if (!list_empty(&dev->ptype_all)) { - nxt = dev->ptype_all.next; + nxt = READ_ONCE(dev->ptype_all.next); + if (nxt != &dev->ptype_all) { + iter->dev = dev; goto found; } } - nxt = net->ptype_all.next; + iter->dev = NULL; + nxt = READ_ONCE(net->ptype_all.next); goto net_ptype_all; } @@ -252,20 +265,20 @@ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) if (nxt == &net->ptype_all) { /* continue with ->ptype_specific if it's not empty */ - nxt = net->ptype_specific.next; + nxt = READ_ONCE(net->ptype_specific.next); if (nxt != &net->ptype_specific) goto found; } hash = 0; - nxt = ptype_base[0].next; + nxt = READ_ONCE(ptype_base[0].next); } else hash = ntohs(pt->type) & PTYPE_HASH_MASK; while (nxt == &ptype_base[hash]) { if (++hash >= PTYPE_HASH_SIZE) return NULL; - nxt = ptype_base[hash].next; + nxt = READ_ONCE(ptype_base[hash].next); } found: return list_entry(nxt, struct packet_type, list); @@ -279,19 +292,24 @@ static void ptype_seq_stop(struct seq_file *seq, void *v) static int ptype_seq_show(struct seq_file *seq, void *v) { + struct ptype_iter_state *iter = seq->private; struct packet_type *pt = v; + struct net_device *dev; - if (v == SEQ_START_TOKEN) + if (v == SEQ_START_TOKEN) { seq_puts(seq, "Type Device Function\n"); - else if ((!pt->af_packet_net || net_eq(pt->af_packet_net, seq_file_net(seq))) && - (!pt->dev || net_eq(dev_net(pt->dev), seq_file_net(seq)))) { + return 0; + } + dev = iter->dev; + if ((!pt->af_packet_net || net_eq(pt->af_packet_net, seq_file_net(seq))) && + (!dev || net_eq(dev_net(dev), seq_file_net(seq)))) { if (pt->type == htons(ETH_P_ALL)) seq_puts(seq, "ALL "); else seq_printf(seq, "%04x", ntohs(pt->type)); seq_printf(seq, " %-8s %ps\n", - pt->dev ? pt->dev->name : "", pt->func); + dev ? dev->name : "", pt->func); } return 0; @@ -315,7 +333,7 @@ static int __net_init dev_proc_net_init(struct net *net) &softnet_seq_ops)) goto out_dev; if (!proc_create_net("ptype", 0444, net->proc_net, &ptype_seq_ops, - sizeof(struct seq_net_private))) + sizeof(struct ptype_iter_state))) goto out_softnet; if (wext_proc_init(net)) diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 331764845e8fa..5af14f14a3623 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -132,7 +132,7 @@ static int netif_local_xmit_active(struct net_device *dev) for (i = 0; i < dev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - if (READ_ONCE(txq->xmit_lock_owner) == smp_processor_id()) + if (netif_tx_owned(txq, smp_processor_id())) return 1; } @@ -554,6 +554,7 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev) int err; skb_queue_head_init(&np->skb_pool); + INIT_WORK(&np->refill_wq, refill_skbs_work_handler); if (ndev->priv_flags & IFF_DISABLE_NETPOLL) { np_err(np, "%s doesn't support polling, aborting\n", @@ -591,7 +592,6 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev) /* fill up the skb queue */ refill_skbs(np); - INIT_WORK(&np->refill_wq, refill_skbs_work_handler); /* last thing to do is link it to the net device structure */ rcu_assign_pointer(ndev->npinfo, npinfo); diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c index c82a95beceff8..ee5060d8eec0e 100644 --- a/net/core/page_pool_user.c +++ b/net/core/page_pool_user.c @@ -245,7 +245,7 @@ page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, goto err_cancel; if (pool->user.detach_time && nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, - pool->user.detach_time)) + ktime_divns(pool->user.detach_time, NSEC_PER_SEC))) goto err_cancel; if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL)) @@ -337,7 +337,7 @@ int page_pool_list(struct page_pool *pool) void page_pool_detached(struct page_pool *pool) { mutex_lock(&page_pools_lock); - pool->user.detach_time = ktime_get_boottime_seconds(); + pool->user.detach_time = ktime_get_boottime(); netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); mutex_unlock(&page_pools_lock); } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 576d5ec3bb364..f4ed60bd9a256 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -629,6 +629,9 @@ int rtnl_link_register(struct rtnl_link_ops *ops) unlock: mutex_unlock(&link_ops_mutex); + if (err) + cleanup_srcu_struct(&ops->srcu); + return err; } EXPORT_SYMBOL_GPL(rtnl_link_register); @@ -707,11 +710,14 @@ static size_t rtnl_link_get_slave_info_data_size(const struct net_device *dev) goto out; ops = master_dev->rtnl_link_ops; - if (!ops || !ops->get_slave_size) + if (!ops) + goto out; + size += nla_total_size(strlen(ops->kind) + 1); /* IFLA_INFO_SLAVE_KIND */ + if (!ops->get_slave_size) goto out; /* IFLA_INFO_SLAVE_DATA + nested data */ - size = nla_total_size(sizeof(struct nlattr)) + - ops->get_slave_size(master_dev, dev); + size += nla_total_size(sizeof(struct nlattr)) + + ops->get_slave_size(master_dev, dev); out: rcu_read_unlock(); @@ -1267,6 +1273,21 @@ static size_t rtnl_dpll_pin_size(const struct net_device *dev) return size; } +static size_t rtnl_dev_parent_size(const struct net_device *dev) +{ + size_t size = 0; + + /* IFLA_PARENT_DEV_NAME */ + if (dev->dev.parent) + size += nla_total_size(strlen(dev_name(dev->dev.parent)) + 1); + + /* IFLA_PARENT_DEV_BUS_NAME */ + if (dev->dev.parent && dev->dev.parent->bus) + size += nla_total_size(strlen(dev->dev.parent->bus->name) + 1); + + return size; +} + static noinline size_t if_nlmsg_size(const struct net_device *dev, u32 ext_filter_mask) { @@ -1328,6 +1349,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, + nla_total_size(8) /* IFLA_MAX_PACING_OFFLOAD_HORIZON */ + nla_total_size(2) /* IFLA_HEADROOM */ + nla_total_size(2) /* IFLA_TAILROOM */ + + rtnl_dev_parent_size(dev) + 0; } @@ -3865,28 +3887,42 @@ static int rtnl_newlink_create(struct sk_buff *skb, struct ifinfomsg *ifm, goto out; } -static struct net *rtnl_get_peer_net(const struct rtnl_link_ops *ops, +static struct net *rtnl_get_peer_net(struct sk_buff *skb, + const struct rtnl_link_ops *ops, struct nlattr *tbp[], struct nlattr *data[], struct netlink_ext_ack *extack) { - struct nlattr *tb[IFLA_MAX + 1]; + struct nlattr *tb[IFLA_MAX + 1], **attrs; + struct net *net; int err; - if (!data || !data[ops->peer_type]) - return rtnl_link_get_net_ifla(tbp); - - err = rtnl_nla_parse_ifinfomsg(tb, data[ops->peer_type], extack); - if (err < 0) - return ERR_PTR(err); - - if (ops->validate) { - err = ops->validate(tb, NULL, extack); + if (!data || !data[ops->peer_type]) { + attrs = tbp; + } else { + err = rtnl_nla_parse_ifinfomsg(tb, data[ops->peer_type], extack); if (err < 0) return ERR_PTR(err); + + if (ops->validate) { + err = ops->validate(tb, NULL, extack); + if (err < 0) + return ERR_PTR(err); + } + + attrs = tb; } - return rtnl_link_get_net_ifla(tb); + net = rtnl_link_get_net_ifla(attrs); + if (IS_ERR_OR_NULL(net)) + return net; + + if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) { + put_net(net); + return ERR_PTR(-EPERM); + } + + return net; } static int __rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh, @@ -4025,7 +4061,7 @@ static int rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh, } if (ops->peer_type) { - peer_net = rtnl_get_peer_net(ops, tb, data, extack); + peer_net = rtnl_get_peer_net(skb, ops, tb, data, extack); if (IS_ERR(peer_net)) { ret = PTR_ERR(peer_net); goto put_ops; diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index 9a39656804513..6a6f2cda5aaef 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -20,7 +20,6 @@ #include static siphash_aligned_key_t net_secret; -static siphash_aligned_key_t ts_secret; #define EPHEMERAL_PORT_SHUFFLE_PERIOD (10 * HZ) @@ -28,11 +27,6 @@ static __always_inline void net_secret_init(void) { net_get_random_once(&net_secret, sizeof(net_secret)); } - -static __always_inline void ts_secret_init(void) -{ - net_get_random_once(&ts_secret, sizeof(ts_secret)); -} #endif #ifdef CONFIG_INET @@ -53,28 +47,9 @@ static u32 seq_scale(u32 seq) #endif #if IS_ENABLED(CONFIG_IPV6) -u32 secure_tcpv6_ts_off(const struct net *net, - const __be32 *saddr, const __be32 *daddr) -{ - const struct { - struct in6_addr saddr; - struct in6_addr daddr; - } __aligned(SIPHASH_ALIGNMENT) combined = { - .saddr = *(struct in6_addr *)saddr, - .daddr = *(struct in6_addr *)daddr, - }; - - if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) - return 0; - - ts_secret_init(); - return siphash(&combined, offsetofend(typeof(combined), daddr), - &ts_secret); -} -EXPORT_IPV6_MOD(secure_tcpv6_ts_off); - -u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, - __be16 sport, __be16 dport) +union tcp_seq_and_ts_off +secure_tcpv6_seq_and_ts_off(const struct net *net, const __be32 *saddr, + const __be32 *daddr, __be16 sport, __be16 dport) { const struct { struct in6_addr saddr; @@ -87,14 +62,20 @@ u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, .sport = sport, .dport = dport }; - u32 hash; + union tcp_seq_and_ts_off st; net_secret_init(); - hash = siphash(&combined, offsetofend(typeof(combined), dport), - &net_secret); - return seq_scale(hash); + + st.hash64 = siphash(&combined, offsetofend(typeof(combined), dport), + &net_secret); + + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) + st.ts_off = 0; + + st.seq = seq_scale(st.seq); + return st; } -EXPORT_SYMBOL(secure_tcpv6_seq); +EXPORT_SYMBOL(secure_tcpv6_seq_and_ts_off); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport) @@ -118,33 +99,30 @@ EXPORT_SYMBOL(secure_ipv6_port_ephemeral); #endif #ifdef CONFIG_INET -u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr) -{ - if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) - return 0; - - ts_secret_init(); - return siphash_2u32((__force u32)saddr, (__force u32)daddr, - &ts_secret); -} - /* secure_tcp_seq_and_tsoff(a, b, 0, d) == secure_ipv4_port_ephemeral(a, b, d), * but fortunately, `sport' cannot be 0 in any circumstances. If this changes, * it would be easy enough to have the former function use siphash_4u32, passing * the arguments as separate u32. */ -u32 secure_tcp_seq(__be32 saddr, __be32 daddr, - __be16 sport, __be16 dport) +union tcp_seq_and_ts_off +secure_tcp_seq_and_ts_off(const struct net *net, __be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) { - u32 hash; + u32 ports = (__force u32)sport << 16 | (__force u32)dport; + union tcp_seq_and_ts_off st; net_secret_init(); - hash = siphash_3u32((__force u32)saddr, (__force u32)daddr, - (__force u32)sport << 16 | (__force u32)dport, - &net_secret); - return seq_scale(hash); + + st.hash64 = siphash_3u32((__force u32)saddr, (__force u32)daddr, + ports, &net_secret); + + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) + st.ts_off = 0; + + st.seq = seq_scale(st.seq); + return st; } -EXPORT_SYMBOL_GPL(secure_tcp_seq); +EXPORT_SYMBOL_GPL(secure_tcp_seq_and_ts_off); u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 6be01454f262a..a4695882d1c4c 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1042,10 +1042,7 @@ static int skb_pp_frag_ref(struct sk_buff *skb) static void skb_kfree_head(void *head, unsigned int end_offset) { - if (end_offset == SKB_SMALL_HEAD_HEADROOM) - kmem_cache_free(net_hotdata.skb_small_head_cache, head); - else - kfree(head); + kfree(head); } static void skb_free_head(struct sk_buff *skb) @@ -4597,12 +4594,14 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb, { struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; unsigned int tnl_hlen = skb_tnl_header_len(skb); - unsigned int delta_truesize = 0; unsigned int delta_len = 0; struct sk_buff *tail = NULL; struct sk_buff *nskb, *tmp; int len_diff, err; + /* Only skb_gro_receive_list generated skbs arrive here */ + DEBUG_NET_WARN_ON_ONCE(!(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)); + skb_push(skb, -skb_network_offset(skb) + offset); /* Ensure the head is writeable before touching the shared info */ @@ -4616,8 +4615,9 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb, nskb = list_skb; list_skb = list_skb->next; + DEBUG_NET_WARN_ON_ONCE(nskb->sk); + err = 0; - delta_truesize += nskb->truesize; if (skb_shared(nskb)) { tmp = skb_clone(nskb, GFP_ATOMIC); if (tmp) { @@ -4660,7 +4660,6 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb, goto err_linearize; } - skb->truesize = skb->truesize - delta_truesize; skb->data_len = skb->data_len - delta_len; skb->len = skb->len - delta_len; @@ -5511,15 +5510,28 @@ static void __skb_complete_tx_timestamp(struct sk_buff *skb, static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) { - bool ret; + struct socket *sock; + struct file *file; + bool ret = false; if (likely(tsonly || READ_ONCE(sock_net(sk)->core.sysctl_tstamp_allow_data))) return true; - read_lock_bh(&sk->sk_callback_lock); - ret = sk->sk_socket && sk->sk_socket->file && - file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); - read_unlock_bh(&sk->sk_callback_lock); + /* The sk pointer remains valid as long as the skb is. The sk_socket and + * file pointer may become NULL if the socket is closed. Both structures + * (including file->cred) are RCU freed which means they can be accessed + * within a RCU read section. + */ + rcu_read_lock(); + sock = READ_ONCE(sk->sk_socket); + if (!sock) + goto out; + file = READ_ONCE(sock->file); + if (!file) + goto out; + ret = file_ns_capable(file, &init_user_ns, CAP_NET_RAW); +out: + rcu_read_unlock(); return ret; } diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 2ac7731e1e0a7..35a6acbf9a579 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -409,22 +409,26 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, } EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter); -/* Receive sk_msg from psock->ingress_msg to @msg. */ -int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, - int len, int flags) +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self) { struct iov_iter *iter = &msg->msg_iter; int peek = flags & MSG_PEEK; struct sk_msg *msg_rx; int i, copied = 0; + bool from_self; msg_rx = sk_psock_peek_msg(psock); + if (copied_from_self) + *copied_from_self = 0; + while (copied != len) { struct scatterlist *sge; if (unlikely(!msg_rx)) break; + from_self = msg_rx->sk == sk; i = msg_rx->sg.start; do { struct page *page; @@ -443,6 +447,9 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, } copied += copy; + if (from_self && copied_from_self) + *copied_from_self += copy; + if (likely(!peek)) { sge->offset += copy; sge->length -= copy; @@ -451,6 +458,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, atomic_sub(copy, &sk->sk_rmem_alloc); } msg_rx->sg.size -= copy; + sk_psock_msg_len_add(psock, -copy); if (!sge->length) { sk_msg_iter_var_next(i); @@ -487,6 +495,13 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, out: return copied; } + +/* Receive sk_msg from psock->ingress_msg to @msg. */ +int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags) +{ + return __sk_msg_recvmsg(sk, psock, msg, len, flags, NULL); +} EXPORT_SYMBOL_GPL(sk_msg_recvmsg); bool sk_msg_is_readable(struct sock *sk) @@ -616,6 +631,12 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb if (unlikely(!msg)) return -EAGAIN; skb_set_owner_r(skb, sk); + + /* This is used in tcp_bpf_recvmsg_parser() to determine whether the + * data originates from the socket's own protocol stack. No need to + * refcount sk because msg's lifetime is bound to sk via the ingress_msg. + */ + msg->sk = sk; err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref); if (err < 0) kfree(msg); @@ -801,9 +822,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) list_del(&msg->list); if (!msg->skb) atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc); + sk_psock_msg_len_add(psock, -msg->sg.size); sk_msg_free(psock->sk, msg); kfree(msg); } + WARN_ON_ONCE(psock->msg_tot_len); } static void __sk_psock_zap_ingress(struct sk_psock *psock) @@ -909,6 +932,7 @@ int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, sk_msg_compute_data_pointers(msg); msg->sk = sk; ret = bpf_prog_run_pin_on_cpu(prog, msg); + msg->sk = NULL; ret = sk_psock_map_verd(ret, msg->sk_redir); psock->apply_bytes = msg->apply_bytes; if (ret == __SK_REDIRECT) { @@ -1181,8 +1205,8 @@ void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) return; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_strp_data_ready; - sk->sk_write_space = sk_psock_write_space; + WRITE_ONCE(sk->sk_data_ready, sk_psock_strp_data_ready); + WRITE_ONCE(sk->sk_write_space, sk_psock_write_space); } void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) @@ -1192,8 +1216,8 @@ void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) if (!psock->saved_data_ready) return; - sk->sk_data_ready = psock->saved_data_ready; - psock->saved_data_ready = NULL; + WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready); + WRITE_ONCE(psock->saved_data_ready, NULL); strp_stop(&psock->strp); } @@ -1243,17 +1267,20 @@ static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb) static void sk_psock_verdict_data_ready(struct sock *sk) { - struct socket *sock = sk->sk_socket; - const struct proto_ops *ops; + const struct proto_ops *ops = NULL; + struct socket *sock; int copied; trace_sk_data_ready(sk); - if (unlikely(!sock)) - return; - ops = READ_ONCE(sock->ops); + rcu_read_lock(); + sock = READ_ONCE(sk->sk_socket); + if (likely(sock)) + ops = READ_ONCE(sock->ops); + rcu_read_unlock(); if (!ops || !ops->read_skb) return; + copied = ops->read_skb(sk, sk_psock_verdict_recv); if (copied >= 0) { struct sk_psock *psock; @@ -1272,8 +1299,8 @@ void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) return; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_verdict_data_ready; - sk->sk_write_space = sk_psock_write_space; + WRITE_ONCE(sk->sk_data_ready, sk_psock_verdict_data_ready); + WRITE_ONCE(sk->sk_write_space, sk_psock_write_space); } void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) @@ -1284,6 +1311,6 @@ void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) if (!psock->saved_data_ready) return; - sk->sk_data_ready = psock->saved_data_ready; + WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready); psock->saved_data_ready = NULL; } diff --git a/net/core/sock.c b/net/core/sock.c index dc03d4b5909a2..5a38837a58384 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -3836,7 +3836,7 @@ void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level, int type) { - struct sock_exterr_skb *serr; + struct sock_extended_err ee; struct sk_buff *skb; int copied, err; @@ -3856,8 +3856,9 @@ int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, sock_recv_timestamp(msg, sk, skb); - serr = SKB_EXT_ERR(skb); - put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); + /* We must use a bounce buffer for CONFIG_HARDENED_USERCOPY=y */ + ee = SKB_EXT_ERR(skb)->ee; + put_cmsg(msg, level, type, sizeof(ee), &ee); msg->msg_flags |= MSG_ERRQUEUE; err = copied; diff --git a/net/devlink/health.c b/net/devlink/health.c index 136a67c36a20d..0798c82096bdc 100644 --- a/net/devlink/health.c +++ b/net/devlink/health.c @@ -1327,7 +1327,7 @@ void devlink_fmsg_dump_skb(struct devlink_fmsg *fmsg, const struct sk_buff *skb) if (sk) { devlink_fmsg_pair_nest_start(fmsg, "sk"); devlink_fmsg_obj_nest_start(fmsg); - devlink_fmsg_put(fmsg, "family", sk->sk_type); + devlink_fmsg_put(fmsg, "family", sk->sk_family); devlink_fmsg_put(fmsg, "type", sk->sk_type); devlink_fmsg_put(fmsg, "proto", sk->sk_protocol); devlink_fmsg_obj_nest_end(fmsg); diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c index 5b01a0e43ebe8..0505e90033f23 100644 --- a/net/dsa/dsa.c +++ b/net/dsa/dsa.c @@ -157,7 +157,7 @@ unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) bridge_num = find_next_zero_bit(&dsa_fwd_offloading_bridges, DSA_MAX_NUM_OFFLOADING_BRIDGES, 1); - if (bridge_num >= max) + if (bridge_num > max) return 0; set_bit(bridge_num, &dsa_fwd_offloading_bridges); @@ -366,16 +366,10 @@ static struct dsa_port *dsa_tree_find_first_cpu(struct dsa_switch_tree *dst) struct net_device *dsa_tree_find_first_conduit(struct dsa_switch_tree *dst) { - struct device_node *ethernet; - struct net_device *conduit; struct dsa_port *cpu_dp; cpu_dp = dsa_tree_find_first_cpu(dst); - ethernet = of_parse_phandle(cpu_dp->dn, "ethernet", 0); - conduit = of_find_net_device_by_node(ethernet); - of_node_put(ethernet); - - return conduit; + return cpu_dp->conduit; } /* Assign the default CPU port (the first one in the tree) to all ports of the @@ -1252,14 +1246,25 @@ static int dsa_port_parse_of(struct dsa_port *dp, struct device_node *dn) if (ethernet) { struct net_device *conduit; const char *user_protocol; + int err; + rtnl_lock(); conduit = of_find_net_device_by_node(ethernet); of_node_put(ethernet); - if (!conduit) + if (!conduit) { + rtnl_unlock(); return -EPROBE_DEFER; + } + + netdev_hold(conduit, &dp->conduit_tracker, GFP_KERNEL); + put_device(&conduit->dev); + rtnl_unlock(); user_protocol = of_get_property(dn, "dsa-tag-protocol", NULL); - return dsa_port_parse_cpu(dp, conduit, user_protocol); + err = dsa_port_parse_cpu(dp, conduit, user_protocol); + if (err) + netdev_put(conduit, &dp->conduit_tracker); + return err; } if (link) @@ -1392,37 +1397,30 @@ static struct device *dev_find_class(struct device *parent, char *class) return device_find_child(parent, class, dev_is_class); } -static struct net_device *dsa_dev_to_net_device(struct device *dev) -{ - struct device *d; - - d = dev_find_class(dev, "net"); - if (d != NULL) { - struct net_device *nd; - - nd = to_net_dev(d); - dev_hold(nd); - put_device(d); - - return nd; - } - - return NULL; -} - static int dsa_port_parse(struct dsa_port *dp, const char *name, struct device *dev) { if (!strcmp(name, "cpu")) { struct net_device *conduit; + struct device *d; + int err; - conduit = dsa_dev_to_net_device(dev); - if (!conduit) + rtnl_lock(); + d = dev_find_class(dev, "net"); + if (!d) { + rtnl_unlock(); return -EPROBE_DEFER; + } - dev_put(conduit); + conduit = to_net_dev(d); + netdev_hold(conduit, &dp->conduit_tracker, GFP_KERNEL); + put_device(d); + rtnl_unlock(); - return dsa_port_parse_cpu(dp, conduit, NULL); + err = dsa_port_parse_cpu(dp, conduit, NULL); + if (err) + netdev_put(conduit, &dp->conduit_tracker); + return err; } if (!strcmp(name, "dsa")) @@ -1490,6 +1488,9 @@ static void dsa_switch_release_ports(struct dsa_switch *ds) struct dsa_vlan *v, *n; dsa_switch_for_each_port_safe(dp, next, ds) { + if (dsa_port_is_cpu(dp) && dp->conduit) + netdev_put(dp->conduit, &dp->conduit_tracker); + /* These are either entries that upper layers lost track of * (probably due to bugs), or installed through interfaces * where one does not necessarily have to remove them, like @@ -1634,8 +1635,10 @@ void dsa_switch_shutdown(struct dsa_switch *ds) /* Disconnect from further netdevice notifiers on the conduit, * since netdev_uses_dsa() will now return false. */ - dsa_switch_for_each_cpu_port(dp, ds) + dsa_switch_for_each_cpu_port(dp, ds) { dp->conduit->dsa_ptr = NULL; + netdev_put(dp->conduit, &dp->conduit_tracker); + } rtnl_unlock(); out: diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index 43e211e611b16..ca4e3a01237d0 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -193,14 +193,11 @@ __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev) } EXPORT_SYMBOL(eth_type_trans); -/** - * eth_header_parse - extract hardware address from packet - * @skb: packet to extract header from - * @haddr: destination buffer - */ -int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr) +int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr) { const struct ethhdr *eth = eth_hdr(skb); + memcpy(haddr, eth->h_source, ETH_ALEN); return ETH_ALEN; } diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 55223ebc2a7e6..146c7eaedc5ac 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -854,9 +854,6 @@ ethtool_rxfh_ctx_alloc(const struct ethtool_ops *ops, ctx->key_off = key_off; ctx->priv_size = ops->rxfh_priv_size; - ctx->hfunc = ETH_RSS_HASH_NO_CHANGE; - ctx->input_xfrm = RXH_XFRM_NO_CHANGE; - return ctx; } diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index fa83ddade4f81..9431e305b2333 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -2383,7 +2383,10 @@ static int ethtool_get_strings(struct net_device *dev, void __user *useraddr) return -ENOMEM; WARN_ON_ONCE(!ret); - gstrings.len = ret; + if (gstrings.len && gstrings.len != ret) + gstrings.len = 0; + else + gstrings.len = ret; if (gstrings.len) { data = vzalloc(array_size(gstrings.len, ETH_GSTRING_LEN)); @@ -2509,10 +2512,13 @@ static int ethtool_get_stats(struct net_device *dev, void __user *useraddr) if (copy_from_user(&stats, useraddr, sizeof(stats))) return -EFAULT; - stats.n_stats = n_stats; + if (stats.n_stats && stats.n_stats != n_stats) + stats.n_stats = 0; + else + stats.n_stats = n_stats; - if (n_stats) { - data = vzalloc(array_size(n_stats, sizeof(u64))); + if (stats.n_stats) { + data = vzalloc(array_size(stats.n_stats, sizeof(u64))); if (!data) return -ENOMEM; ops->get_ethtool_stats(dev, &stats, data); @@ -2524,7 +2530,9 @@ static int ethtool_get_stats(struct net_device *dev, void __user *useraddr) if (copy_to_user(useraddr, &stats, sizeof(stats))) goto out; useraddr += sizeof(stats); - if (n_stats && copy_to_user(useraddr, data, array_size(n_stats, sizeof(u64)))) + if (stats.n_stats && + copy_to_user(useraddr, data, + array_size(stats.n_stats, sizeof(u64)))) goto out; ret = 0; @@ -2560,6 +2568,10 @@ static int ethtool_get_phy_stats_phydev(struct phy_device *phydev, return -EOPNOTSUPP; n_stats = phy_ops->get_sset_count(phydev); + if (stats->n_stats && stats->n_stats != n_stats) { + stats->n_stats = 0; + return 0; + } ret = ethtool_vzalloc_stats_array(n_stats, data); if (ret) @@ -2580,6 +2592,10 @@ static int ethtool_get_phy_stats_ethtool(struct net_device *dev, return -EOPNOTSUPP; n_stats = ops->get_sset_count(dev, ETH_SS_PHY_STATS); + if (stats->n_stats && stats->n_stats != n_stats) { + stats->n_stats = 0; + return 0; + } ret = ethtool_vzalloc_stats_array(n_stats, data); if (ret) @@ -2616,7 +2632,9 @@ static int ethtool_get_phy_stats(struct net_device *dev, void __user *useraddr) } useraddr += sizeof(stats); - if (copy_to_user(useraddr, data, array_size(stats.n_stats, sizeof(u64)))) + if (stats.n_stats && + copy_to_user(useraddr, data, + array_size(stats.n_stats, sizeof(u64)))) ret = -EFAULT; out: diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c index 4dced53be4b3b..da5934cceb075 100644 --- a/net/ethtool/rss.c +++ b/net/ethtool/rss.c @@ -824,8 +824,8 @@ rss_set_ctx_update(struct ethtool_rxfh_context *ctx, struct nlattr **tb, static int ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info) { - bool indir_reset = false, indir_mod, xfrm_sym = false; struct rss_req_info *request = RSS_REQINFO(req_info); + bool indir_reset = false, indir_mod, xfrm_sym; struct ethtool_rxfh_context *ctx = NULL; struct net_device *dev = req_info->dev; bool mod = false, fields_mod = false; @@ -860,12 +860,7 @@ ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info) rxfh.input_xfrm = data.input_xfrm; ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod); - /* For drivers which don't support input_xfrm it will be set to 0xff - * in the RSS context info. In all other case input_xfrm != 0 means - * symmetric hashing is requested. - */ - if (!request->rss_context || ops->rxfh_per_ctx_key) - xfrm_sym = rxfh.input_xfrm || data.input_xfrm; + xfrm_sym = rxfh.input_xfrm || data.input_xfrm; if (rxfh.input_xfrm == data.input_xfrm) rxfh.input_xfrm = RXH_XFRM_NO_CHANGE; diff --git a/net/handshake/request.c b/net/handshake/request.c index 274d2c89b6b20..6b7e3e0bf3996 100644 --- a/net/handshake/request.c +++ b/net/handshake/request.c @@ -276,6 +276,8 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req, out_unlock: spin_unlock(&hn->hn_lock); out_err: + /* Restore original destructor so socket teardown still runs on failure */ + req->hr_sk->sk_destruct = req->hr_odestruct; trace_handshake_submit_err(net, req, req->hr_sk, ret); handshake_req_destroy(req); return ret; @@ -324,7 +326,11 @@ bool handshake_req_cancel(struct sock *sk) hn = handshake_pernet(net); if (hn && remove_pending(hn, req)) { - /* Request hadn't been accepted */ + /* Request hadn't been accepted - mark cancelled */ + if (test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) { + trace_handshake_cancel_busy(net, req, sk); + return false; + } goto out_true; } if (test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) { diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index 492cbc78ab75a..fd2fea25eff0d 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -532,8 +532,8 @@ static void hsr_change_rx_flags(struct net_device *dev, int change) static int hsr_ndo_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) { - bool is_slave_a_added = false; - bool is_slave_b_added = false; + struct net_device *slave_a_dev = NULL; + struct net_device *slave_b_dev = NULL; struct hsr_port *port; struct hsr_priv *hsr; int ret = 0; @@ -549,33 +549,35 @@ static int hsr_ndo_vlan_rx_add_vid(struct net_device *dev, switch (port->type) { case HSR_PT_SLAVE_A: if (ret) { - /* clean up Slave-B */ netdev_err(dev, "add vid failed for Slave-A\n"); - if (is_slave_b_added) - vlan_vid_del(port->dev, proto, vid); - return ret; + goto unwind; } - - is_slave_a_added = true; + slave_a_dev = port->dev; break; - case HSR_PT_SLAVE_B: if (ret) { - /* clean up Slave-A */ netdev_err(dev, "add vid failed for Slave-B\n"); - if (is_slave_a_added) - vlan_vid_del(port->dev, proto, vid); - return ret; + goto unwind; } - - is_slave_b_added = true; + slave_b_dev = port->dev; break; default: + if (ret) + goto unwind; break; } } return 0; + +unwind: + if (slave_a_dev) + vlan_vid_del(slave_a_dev, proto, vid); + + if (slave_b_dev) + vlan_vid_del(slave_b_dev, proto, vid); + + return ret; } static int hsr_ndo_vlan_rx_kill_vid(struct net_device *dev, @@ -690,6 +692,26 @@ struct net_device *hsr_get_port_ndev(struct net_device *ndev, } EXPORT_SYMBOL(hsr_get_port_ndev); +int hsr_get_port_type(struct net_device *hsr_dev, struct net_device *dev, + enum hsr_port_type *type) +{ + struct hsr_priv *hsr = netdev_priv(hsr_dev); + struct hsr_port *port; + + rcu_read_lock(); + hsr_for_each_port(hsr, port) { + if (port->dev == dev) { + *type = port->type; + rcu_read_unlock(); + return 0; + } + } + rcu_read_unlock(); + + return -EINVAL; +} +EXPORT_SYMBOL(hsr_get_port_type); + /* Default multicast address for HSR Supervision frames */ static const unsigned char def_multicast_addr[ETH_ALEN] __aligned(2) = { 0x01, 0x15, 0x4e, 0x00, 0x01, 0x00 diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c index 339f0d2202129..aefc9b6936ba0 100644 --- a/net/hsr/hsr_forward.c +++ b/net/hsr/hsr_forward.c @@ -205,6 +205,8 @@ struct sk_buff *prp_get_untagged_frame(struct hsr_frame_info *frame, __pskb_copy(frame->skb_prp, skb_headroom(frame->skb_prp), GFP_ATOMIC); + if (!frame->skb_std) + return NULL; } else { /* Unexpected */ WARN_ONCE(1, "%s:%d: Unexpected frame received (port_src %s)\n", diff --git a/net/hsr/hsr_slave.c b/net/hsr/hsr_slave.c index 8177ac6c2d26d..afe06ba00ea44 100644 --- a/net/hsr/hsr_slave.c +++ b/net/hsr/hsr_slave.c @@ -207,14 +207,14 @@ int hsr_add_port(struct hsr_priv *hsr, struct net_device *dev, port->type = type; ether_addr_copy(port->original_macaddress, dev->dev_addr); + list_add_tail_rcu(&port->port_list, &hsr->ports); + if (type != HSR_PT_MASTER) { res = hsr_portdev_setup(hsr, dev, port, extack); if (res) goto fail_dev_setup; } - list_add_tail_rcu(&port->port_list, &hsr->ports); - master = hsr_port_get_hsr(hsr, HSR_PT_MASTER); netdev_update_features(master->dev); dev_set_mtu(master->dev, hsr_get_max_mtu(hsr)); @@ -222,7 +222,8 @@ int hsr_add_port(struct hsr_priv *hsr, struct net_device *dev, return 0; fail_dev_setup: - kfree(port); + list_del_rcu(&port->port_list); + kfree_rcu(port, rcu); return res; } diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index 12850a277251d..06cb9c9c5b9e7 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -748,6 +748,7 @@ config TCP_SIGPOOL config TCP_AO bool "TCP: Authentication Option (RFC5925)" select CRYPTO + select CRYPTO_LIB_UTILS select TCP_SIGPOOL depends on 64BIT && IPV6 != m # seq-number extension needs WRITE_ONCE(u64) help diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index 833f2cf97178e..3ce1664e8632a 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -564,7 +564,7 @@ struct sk_buff *arp_create(int type, int ptype, __be32 dest_ip, skb_reserve(skb, hlen); skb_reset_network_header(skb); - arp = skb_put(skb, arp_hdr_len(dev)); + skb_put(skb, arp_hdr_len(dev)); skb->dev = dev; skb->protocol = htons(ETH_P_ARP); if (!src_hw) @@ -572,12 +572,13 @@ struct sk_buff *arp_create(int type, int ptype, __be32 dest_ip, if (!dest_hw) dest_hw = dev->broadcast; - /* - * Fill the device header for the ARP frame + /* Fill the device header for the ARP frame. + * Note: skb->head can be changed. */ if (dev_hard_header(skb, dev, ptype, dest_hw, src_hw, skb->len) < 0) goto out; + arp = arp_hdr(skb); /* * Fill out the arp protocol part. * diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 2c922afadb8f6..6dfc0bcdef654 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -235,10 +235,13 @@ static void esp_output_done(void *data, int err) xfrm_dev_resume(skb); } else { if (!err && - x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) - esp_output_tail_tcp(x, skb); - else + x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) { + err = esp_output_tail_tcp(x, skb); + if (err != -EINPROGRESS) + kfree_skb(skb); + } else { xfrm_output_resume(skb_to_full_sk(skb), skb, err); + } } } diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c index 05828d4cb6cdb..abd77162f5e75 100644 --- a/net/ipv4/esp4_offload.c +++ b/net/ipv4/esp4_offload.c @@ -122,8 +122,8 @@ static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, - XFRM_MODE_SKB_CB(skb)->protocol); + struct xfrm_offload *xo = xfrm_offload(skb); + const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, xo->proto); __be16 type = inner_mode->family == AF_INET6 ? htons(ETH_P_IPV6) : htons(ETH_P_IP); diff --git a/net/ipv4/fib_lookup.h b/net/ipv4/fib_lookup.h index f9b9e26c32c19..0b72796dd1ad3 100644 --- a/net/ipv4/fib_lookup.h +++ b/net/ipv4/fib_lookup.h @@ -28,8 +28,10 @@ struct fib_alias { /* Don't write on fa_state unless needed, to keep it shared on all cpus */ static inline void fib_alias_accessed(struct fib_alias *fa) { - if (!(fa->fa_state & FA_S_ACCESSED)) - fa->fa_state |= FA_S_ACCESSED; + u8 fa_state = READ_ONCE(fa->fa_state); + + if (!(fa_state & FA_S_ACCESSED)) + WRITE_ONCE(fa->fa_state, fa_state | FA_S_ACCESSED); } /* Exported by fib_semantics.c */ diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index a5f3c8459758f..0caf38e44c738 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -2167,8 +2167,8 @@ void fib_select_multipath(struct fib_result *res, int hash, { struct fib_info *fi = res->fi; struct net *net = fi->fib_net; - bool found = false; bool use_neigh; + int score = -1; __be32 saddr; if (unlikely(res->fi->nh)) { @@ -2180,7 +2180,7 @@ void fib_select_multipath(struct fib_result *res, int hash, saddr = fl4 ? fl4->saddr : 0; change_nexthops(fi) { - int nh_upper_bound; + int nh_upper_bound, nh_score = 0; /* Nexthops without a carrier are assigned an upper bound of * minus one when "ignore_routes_with_linkdown" is set. @@ -2190,24 +2190,18 @@ void fib_select_multipath(struct fib_result *res, int hash, (use_neigh && !fib_good_nh(nexthop_nh))) continue; - if (!found) { + if (saddr && nexthop_nh->nh_saddr == saddr) + nh_score += 2; + if (hash <= nh_upper_bound) + nh_score++; + if (score < nh_score) { res->nh_sel = nhsel; res->nhc = &nexthop_nh->nh_common; - found = !saddr || nexthop_nh->nh_saddr == saddr; + if (nh_score == 3 || (!saddr && nh_score == 1)) + return; + score = nh_score; } - if (hash > nh_upper_bound) - continue; - - if (!saddr || nexthop_nh->nh_saddr == saddr) { - res->nh_sel = nhsel; - res->nhc = &nexthop_nh->nh_common; - return; - } - - if (found) - return; - } endfor_nexthops(fi); } #endif diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 59a6f0a9638f9..1308213791f19 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -1280,7 +1280,7 @@ int fib_table_insert(struct net *net, struct fib_table *tb, new_fa->fa_dscp = fa->fa_dscp; new_fa->fa_info = fi; new_fa->fa_type = cfg->fc_type; - state = fa->fa_state; + state = READ_ONCE(fa->fa_state); new_fa->fa_state = state & ~FA_S_ACCESSED; new_fa->fa_slen = fa->fa_slen; new_fa->tb_id = tb->tb_id; @@ -1745,7 +1745,7 @@ int fib_table_delete(struct net *net, struct fib_table *tb, fib_remove_alias(t, tp, l, fa_to_delete); - if (fa_to_delete->fa_state & FA_S_ACCESSED) + if (READ_ONCE(fa_to_delete->fa_state) & FA_S_ACCESSED) rt_cache_flush(cfg->fc_nlinfo.nl_net); fib_release_info(fa_to_delete->fa_info); @@ -2053,10 +2053,11 @@ int fib_table_flush(struct net *net, struct fib_table *tb, bool flush_all) continue; } - /* Do not flush error routes if network namespace is - * not being dismantled + /* When not flushing the entire table, skip error + * routes that are not marked for deletion. */ - if (!flush_all && fib_props[fa->fa_type].error) { + if (!flush_all && fib_props[fa->fa_type].error && + !(fi->fib_flags & RTNH_F_DEAD)) { slen = fa->fa_slen; continue; } diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c index 3970b6b7ace53..ab8f309f8925d 100644 --- a/net/ipv4/fou_core.c +++ b/net/ipv4/fou_core.c @@ -215,6 +215,9 @@ static int gue_udp_recv(struct sock *sk, struct sk_buff *skb) return gue_control_message(skb, guehdr); proto_ctype = guehdr->proto_ctype; + if (unlikely(!proto_ctype)) + goto drop; + __skb_pull(skb, sizeof(struct udphdr) + hdrlen); skb_reset_transport_header(skb); diff --git a/net/ipv4/fou_nl.c b/net/ipv4/fou_nl.c index 506260b4a4dc2..9ff7797ef7c4f 100644 --- a/net/ipv4/fou_nl.c +++ b/net/ipv4/fou_nl.c @@ -14,7 +14,7 @@ const struct nla_policy fou_nl_policy[FOU_ATTR_IFINDEX + 1] = { [FOU_ATTR_PORT] = { .type = NLA_BE16, }, [FOU_ATTR_AF] = { .type = NLA_U8, }, - [FOU_ATTR_IPPROTO] = { .type = NLA_U8, }, + [FOU_ATTR_IPPROTO] = NLA_POLICY_MIN(NLA_U8, 1), [FOU_ATTR_TYPE] = { .type = NLA_U8, }, [FOU_ATTR_REMCSUM_NOPARTIAL] = { .type = NLA_FLAG, }, [FOU_ATTR_LOCAL_V4] = { .type = NLA_U32, }, diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 1b7fb5d935edf..980aa17f3534d 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -248,7 +248,8 @@ bool icmp_global_allow(struct net *net) if (delta < HZ / 50) return false; - incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec) * delta / HZ; + incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec); + incr = div_u64((u64)incr * delta, HZ); if (!incr) return false; @@ -554,6 +555,21 @@ static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4, /* steal dst entry from skb_in, don't drop refcnt */ skb_dstref_steal(skb_in); skb_dstref_restore(skb_in, orefdst); + + /* + * At this point, fl4_dec.daddr should NOT be local (we + * checked fl4_dec.saddr above). However, a race condition + * may occur if the address is added to the interface + * concurrently. In that case, ip_route_input() returns a + * LOCAL route with dst.output=ip_rt_bug, which must not + * be used for output. + */ + if (!err && rt2 && rt2->rt_type == RTN_LOCAL) { + net_warn_ratelimited("detected local route for %pI4 during ICMP sending, src %pI4\n", + &fl4_dec.daddr, &fl4_dec.saddr); + dst_release(&rt2->dst); + err = -EINVAL; + } } if (err) @@ -843,24 +859,32 @@ static void icmp_socket_deliver(struct sk_buff *skb, u32 info) /* Checkin full IP header plus 8 bytes of protocol to * avoid additional coding at protocol handlers. */ - if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) { - __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); - return; - } + if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) + goto out; + + /* IPPROTO_RAW sockets are not supposed to receive anything. */ + if (protocol == IPPROTO_RAW) + goto out; raw_icmp_error(skb, protocol, info); ipprot = rcu_dereference(inet_protos[protocol]); if (ipprot && ipprot->err_handler) ipprot->err_handler(skb, info); + return; + +out: + __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); } static bool icmp_tag_validation(int proto) { + const struct net_protocol *ipprot; bool ok; rcu_read_lock(); - ok = rcu_dereference(inet_protos[proto])->icmp_strict_tag_validation; + ipprot = rcu_dereference(inet_protos[proto]); + ok = ipprot ? ipprot->icmp_strict_tag_validation : false; rcu_read_unlock(); return ok; } @@ -1121,6 +1145,13 @@ bool icmp_build_probe(struct sk_buff *skb, struct icmphdr *icmphdr) if (iio->ident.addr.ctype3_hdr.addrlen != sizeof(struct in6_addr)) goto send_mal_query; dev = ipv6_stub->ipv6_dev_find(net, &iio->ident.addr.ip_addr.ipv6_addr, dev); + /* + * If IPv6 identifier lookup is unavailable, silently + * discard the request instead of misreporting NO_IF. + */ + if (IS_ERR(dev)) + return false; + dev_hold(dev); break; #endif diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index 7182f1419c2a4..0adc993c211d7 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -227,7 +227,7 @@ static void igmp_start_timer(struct ip_mc_list *im, int max_delay) static void igmp_gq_start_timer(struct in_device *in_dev) { - int tv = get_random_u32_below(in_dev->mr_maxdelay); + int tv = get_random_u32_below(READ_ONCE(in_dev->mr_maxdelay)); unsigned long exp = jiffies + tv + 2; if (in_dev->mr_gq_running && @@ -1009,7 +1009,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb, max_delay = IGMPV3_MRC(ih3->code)*(HZ/IGMP_TIMER_SCALE); if (!max_delay) max_delay = 1; /* can't mod w/ 0 */ - in_dev->mr_maxdelay = max_delay; + WRITE_ONCE(in_dev->mr_maxdelay, max_delay); /* RFC3376, 4.1.6. QRV and 4.1.7. QQIC, when the most recently * received value was zero, use the default or statically diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index cdd1e12aac8c0..7a2f116106e96 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -153,20 +153,6 @@ bool inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high) } EXPORT_SYMBOL(inet_sk_get_local_port_range); -static bool inet_use_bhash2_on_bind(const struct sock *sk) -{ -#if IS_ENABLED(CONFIG_IPV6) - if (sk->sk_family == AF_INET6) { - if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) - return false; - - if (!ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) - return true; - } -#endif - return sk->sk_rcv_saddr != htonl(INADDR_ANY); -} - static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2, kuid_t uid, bool relax, bool reuseport_cb_ok, bool reuseport_ok) @@ -258,7 +244,7 @@ static int inet_csk_bind_conflict(const struct sock *sk, * checks separately because their spinlocks have to be acquired/released * independently of each other, to prevent possible deadlocks */ - if (inet_use_bhash2_on_bind(sk)) + if (inet_use_hash2_on_bind(sk)) return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, reuseport_ok); @@ -375,7 +361,7 @@ inet_csk_find_open_port(const struct sock *sk, struct inet_bind_bucket **tb_ret, head = &hinfo->bhash[inet_bhashfn(net, port, hinfo->bhash_size)]; spin_lock_bh(&head->lock); - if (inet_use_bhash2_on_bind(sk)) { + if (inet_use_hash2_on_bind(sk)) { if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false)) goto next_port; } @@ -561,7 +547,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) check_bind_conflict = false; } - if (check_bind_conflict && inet_use_bhash2_on_bind(sk)) { + if (check_bind_conflict && inet_use_hash2_on_bind(sk)) { if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true)) goto fail_unlock; } diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c index 025895eb6ec59..4e6d7467ed444 100644 --- a/net/ipv4/inet_fragment.c +++ b/net/ipv4/inet_fragment.c @@ -218,6 +218,41 @@ static int __init inet_frag_wq_init(void) pure_initcall(inet_frag_wq_init); +void fqdir_pre_exit(struct fqdir *fqdir) +{ + struct inet_frag_queue *fq; + struct rhashtable_iter hti; + + /* Prevent creation of new frags. + * Pairs with READ_ONCE() in inet_frag_find(). + */ + WRITE_ONCE(fqdir->high_thresh, 0); + + /* Pairs with READ_ONCE() in inet_frag_kill(), ip_expire() + * and ip6frag_expire_frag_queue(). + */ + WRITE_ONCE(fqdir->dead, true); + + rhashtable_walk_enter(&fqdir->rhashtable, &hti); + rhashtable_walk_start(&hti); + + while ((fq = rhashtable_walk_next(&hti))) { + if (IS_ERR(fq)) { + if (PTR_ERR(fq) != -EAGAIN) + break; + continue; + } + spin_lock_bh(&fq->lock); + if (!(fq->flags & INET_FRAG_COMPLETE)) + inet_frag_queue_flush(fq, 0); + spin_unlock_bh(&fq->lock); + } + + rhashtable_walk_stop(&hti); + rhashtable_walk_exit(&hti); +} +EXPORT_SYMBOL(fqdir_pre_exit); + void fqdir_exit(struct fqdir *fqdir) { INIT_WORK(&fqdir->destroy_work, fqdir_work_fn); @@ -263,8 +298,8 @@ static void inet_frag_destroy_rcu(struct rcu_head *head) kmem_cache_free(f->frags_cachep, q); } -unsigned int inet_frag_rbtree_purge(struct rb_root *root, - enum skb_drop_reason reason) +static unsigned int +inet_frag_rbtree_purge(struct rb_root *root, enum skb_drop_reason reason) { struct rb_node *p = rb_first(root); unsigned int sum = 0; @@ -284,7 +319,17 @@ unsigned int inet_frag_rbtree_purge(struct rb_root *root, } return sum; } -EXPORT_SYMBOL(inet_frag_rbtree_purge); + +void inet_frag_queue_flush(struct inet_frag_queue *q, + enum skb_drop_reason reason) +{ + unsigned int sum; + + reason = reason ?: SKB_DROP_REASON_FRAG_REASM_TIMEOUT; + sum = inet_frag_rbtree_purge(&q->rb_fragments, reason); + sub_frag_mem_limit(q->fqdir, sum); +} +EXPORT_SYMBOL(inet_frag_queue_flush); void inet_frag_destroy(struct inet_frag_queue *q) { @@ -327,7 +372,9 @@ static struct inet_frag_queue *inet_frag_alloc(struct fqdir *fqdir, timer_setup(&q->timer, f->frag_expire, 0); spin_lock_init(&q->lock); - /* One reference for the timer, one for the hash table. */ + /* One reference for the timer, one for the hash table. + * We never take any extra references, only decrement this field. + */ refcount_set(&q->refcnt, 2); return q; @@ -441,6 +488,8 @@ int inet_frag_queue_insert(struct inet_frag_queue *q, struct sk_buff *skb, } FRAG_CB(skb)->ip_defrag_offset = offset; + if (offset) + nf_reset_ct(skb); return IPFRAG_OK; } diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index b7024e3d9ac3d..a57e33ff92d77 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -200,7 +200,7 @@ static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2, void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, struct inet_bind2_bucket *tb2, unsigned short port) { - inet_sk(sk)->inet_num = port; + WRITE_ONCE(inet_sk(sk)->inet_num, port); inet_csk(sk)->icsk_bind_hash = tb; inet_csk(sk)->icsk_bind2_hash = tb2; sk_add_bind_node(sk, &tb2->owners); @@ -224,7 +224,7 @@ static void __inet_put_port(struct sock *sk) spin_lock(&head->lock); tb = inet_csk(sk)->icsk_bind_hash; inet_csk(sk)->icsk_bind_hash = NULL; - inet_sk(sk)->inet_num = 0; + WRITE_ONCE(inet_sk(sk)->inet_num, 0); sk->sk_userlocks &= ~SOCK_CONNECT_BIND; spin_lock(&head2->lock); @@ -352,7 +352,7 @@ static inline int compute_score(struct sock *sk, const struct net *net, { int score = -1; - if (net_eq(sock_net(sk), net) && sk->sk_num == hnum && + if (net_eq(sock_net(sk), net) && READ_ONCE(sk->sk_num) == hnum && !ipv6_only_sock(sk)) { if (sk->sk_rcv_saddr != daddr) return -1; @@ -1202,7 +1202,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, sk->sk_hash = 0; inet_sk(sk)->inet_sport = 0; - inet_sk(sk)->inet_num = 0; + WRITE_ONCE(inet_sk(sk)->inet_num, 0); if (tw) inet_twsk_bind_unhash(tw, hinfo); diff --git a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c index f7012479713ba..56b0f738d2f27 100644 --- a/net/ipv4/ip_fragment.c +++ b/net/ipv4/ip_fragment.c @@ -134,11 +134,6 @@ static void ip_expire(struct timer_list *t) net = qp->q.fqdir->net; rcu_read_lock(); - - /* Paired with WRITE_ONCE() in fqdir_pre_exit(). */ - if (READ_ONCE(qp->q.fqdir->dead)) - goto out_rcu_unlock; - spin_lock(&qp->q.lock); if (qp->q.flags & INET_FRAG_COMPLETE) @@ -146,6 +141,13 @@ static void ip_expire(struct timer_list *t) qp->q.flags |= INET_FRAG_DROP; inet_frag_kill(&qp->q, &refs); + + /* Paired with WRITE_ONCE() in fqdir_pre_exit(). */ + if (READ_ONCE(qp->q.fqdir->dead)) { + inet_frag_queue_flush(&qp->q, 0); + goto out; + } + __IP_INC_STATS(net, IPSTATS_MIB_REASMFAILS); __IP_INC_STATS(net, IPSTATS_MIB_REASMTIMEOUT); @@ -240,16 +242,10 @@ static int ip_frag_too_far(struct ipq *qp) static int ip_frag_reinit(struct ipq *qp) { - unsigned int sum_truesize = 0; - - if (!mod_timer(&qp->q.timer, jiffies + qp->q.fqdir->timeout)) { - refcount_inc(&qp->q.refcnt); + if (!mod_timer_pending(&qp->q.timer, jiffies + qp->q.fqdir->timeout)) return -ETIMEDOUT; - } - sum_truesize = inet_frag_rbtree_purge(&qp->q.rb_fragments, - SKB_DROP_REASON_FRAG_TOO_FAR); - sub_frag_mem_limit(qp->q.fqdir, sum_truesize); + inet_frag_queue_flush(&qp->q, SKB_DROP_REASON_FRAG_TOO_FAR); qp->q.flags = 0; qp->q.len = 0; diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 761a53c6a89a6..35f0baa99d409 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -330,6 +330,10 @@ static int erspan_rcv(struct sk_buff *skb, struct tnl_ptk_info *tpi, if (!tun_dst) return PACKET_REJECT; + /* MUST set options_len before referencing options */ + info = &tun_dst->u.tun_info; + info->options_len = sizeof(*md); + /* skb can be uncloned in __iptunnel_pull_header, so * old pkt_md is no longer valid and we need to reset * it @@ -344,10 +348,8 @@ static int erspan_rcv(struct sk_buff *skb, struct tnl_ptk_info *tpi, memcpy(md2, pkt_md, ver == 1 ? ERSPAN_V1_MDSIZE : ERSPAN_V2_MDSIZE); - info = &tun_dst->u.tun_info; __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, info->key.tun_flags); - info->options_len = sizeof(*md); } skb_reset_mac_header(skb); @@ -889,10 +891,17 @@ static int ipgre_header(struct sk_buff *skb, struct net_device *dev, const void *daddr, const void *saddr, unsigned int len) { struct ip_tunnel *t = netdev_priv(dev); - struct iphdr *iph; struct gre_base_hdr *greh; + struct iphdr *iph; + int needed; + + needed = t->hlen + sizeof(*iph); + if (skb_headroom(skb) < needed && + pskb_expand_head(skb, HH_DATA_ALIGN(needed - skb_headroom(skb)), + 0, GFP_ATOMIC)) + return -needed; - iph = skb_push(skb, t->hlen + sizeof(*iph)); + iph = skb_push(skb, needed); greh = (struct gre_base_hdr *)(iph+1); greh->flags = gre_tnl_flags_to_gre_flags(t->parms.o_flags); greh->protocol = htons(type); @@ -910,7 +919,8 @@ static int ipgre_header(struct sk_buff *skb, struct net_device *dev, return -(t->hlen + sizeof(*iph)); } -static int ipgre_header_parse(const struct sk_buff *skb, unsigned char *haddr) +static int ipgre_header_parse(const struct sk_buff *skb, const struct net_device *dev, + unsigned char *haddr) { const struct iphdr *iph = (const struct iphdr *) skb_mac_header(skb); memcpy(haddr, &iph->saddr, 4); diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index 158a30ae7c5f2..50d0f5fe4e4c6 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -1281,7 +1281,7 @@ int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[], } EXPORT_SYMBOL_GPL(ip_tunnel_changelink); -int ip_tunnel_init(struct net_device *dev) +int __ip_tunnel_init(struct net_device *dev) { struct ip_tunnel *tunnel = netdev_priv(dev); struct iphdr *iph = &tunnel->parms.iph; @@ -1308,10 +1308,9 @@ int ip_tunnel_init(struct net_device *dev) if (tunnel->collect_md) netif_keep_dst(dev); - netdev_lockdep_set_classes(dev); return 0; } -EXPORT_SYMBOL_GPL(ip_tunnel_init); +EXPORT_SYMBOL_GPL(__ip_tunnel_init); void ip_tunnel_uninit(struct net_device *dev) { diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index 2e61ac1371289..5683c328990f4 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -58,6 +58,19 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, struct iphdr *iph; int err; + if (unlikely(dev_recursion_level() > IP_TUNNEL_RECURSION_LIMIT)) { + if (dev) { + net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", + dev->name); + DEV_STATS_INC(dev, tx_errors); + } + ip_rt_put(rt); + kfree_skb(skb); + return; + } + + dev_xmit_recursion_inc(); + skb_scrub_packet(skb, xnet); skb_clear_hash_if_not_l4(skb); @@ -88,6 +101,8 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, pkt_len = 0; iptunnel_xmit_stats(dev, pkt_len); } + + dev_xmit_recursion_dec(); } EXPORT_SYMBOL_GPL(iptunnel_xmit); diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 7b9d70f9b31c7..c958b8edfe540 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -905,8 +905,7 @@ static int nla_put_nh_group(struct sk_buff *skb, struct nexthop *nh, goto nla_put_failure; if (op_flags & NHA_OP_FLAG_DUMP_STATS && - (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats) || - nla_put_nh_group_stats(skb, nh, op_flags))) + nla_put_nh_group_stats(skb, nh, op_flags)) goto nla_put_failure; return 0; @@ -1007,16 +1006,32 @@ static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg) nla_total_size_64bit(8);/* NHA_RES_GROUP_UNBALANCED_TIME */ } -static size_t nh_nlmsg_size_grp(struct nexthop *nh) +static size_t nh_nlmsg_size_grp(struct nexthop *nh, u32 op_flags) { struct nh_group *nhg = rtnl_dereference(nh->nh_grp); size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh; size_t tot = nla_total_size(sz) + - nla_total_size(2); /* NHA_GROUP_TYPE */ + nla_total_size(2) + /* NHA_GROUP_TYPE */ + nla_total_size(0); /* NHA_FDB */ if (nhg->resilient) tot += nh_nlmsg_size_grp_res(nhg); + if (op_flags & NHA_OP_FLAG_DUMP_STATS) { + tot += nla_total_size(0) + /* NHA_GROUP_STATS */ + nla_total_size(4); /* NHA_HW_STATS_ENABLE */ + tot += nhg->num_nh * + (nla_total_size(0) + /* NHA_GROUP_STATS_ENTRY */ + nla_total_size(4) + /* NHA_GROUP_STATS_ENTRY_ID */ + nla_total_size_64bit(8)); /* NHA_GROUP_STATS_ENTRY_PACKETS */ + + if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS) { + tot += nhg->num_nh * + nla_total_size_64bit(8); /* NHA_GROUP_STATS_ENTRY_PACKETS_HW */ + tot += nla_total_size(4); /* NHA_HW_STATS_USED */ + } + } + return tot; } @@ -1051,14 +1066,14 @@ static size_t nh_nlmsg_size_single(struct nexthop *nh) return sz; } -static size_t nh_nlmsg_size(struct nexthop *nh) +static size_t nh_nlmsg_size(struct nexthop *nh, u32 op_flags) { size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg)); sz += nla_total_size(4); /* NHA_ID */ if (nh->is_group) - sz += nh_nlmsg_size_grp(nh) + + sz += nh_nlmsg_size_grp(nh, op_flags) + nla_total_size(4) + /* NHA_OP_FLAGS */ 0; else @@ -1074,7 +1089,7 @@ static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info) struct sk_buff *skb; int err = -ENOBUFS; - skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any()); + skb = nlmsg_new(nh_nlmsg_size(nh, 0), gfp_any()); if (!skb) goto errout; @@ -2005,7 +2020,8 @@ static void nh_hthr_group_rebalance(struct nh_group *nhg) } static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, - struct nl_info *nlinfo) + struct nl_info *nlinfo, + struct list_head *deferred_free) { struct nh_grp_entry *nhges, *new_nhges; struct nexthop *nhp = nhge->nh_parent; @@ -2065,8 +2081,8 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, rcu_assign_pointer(nhp->nh_grp, newg); list_del(&nhge->nh_list); - free_percpu(nhge->stats); nexthop_put(nhge->nh); + list_add(&nhge->nh_list, deferred_free); /* Removal of a NH from a resilient group is notified through * bucket notifications. @@ -2086,6 +2102,7 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, struct nl_info *nlinfo) { struct nh_grp_entry *nhge, *tmp; + LIST_HEAD(deferred_free); /* If there is nothing to do, let's avoid the costly call to * synchronize_net() @@ -2094,10 +2111,16 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, return; list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list) - remove_nh_grp_entry(net, nhge, nlinfo); + remove_nh_grp_entry(net, nhge, nlinfo, &deferred_free); /* make sure all see the newly published array before releasing rtnl */ synchronize_net(); + + /* Now safe to free percpu stats — all RCU readers have finished */ + list_for_each_entry_safe(nhge, tmp, &deferred_free, nh_list) { + list_del(&nhge->nh_list); + free_percpu(nhge->stats); + } } static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo) @@ -3372,15 +3395,15 @@ static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh, if (err) return err; - err = -ENOBUFS; - skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); - if (!skb) - goto out; - err = -ENOENT; nh = nexthop_find_by_id(net, id); if (!nh) - goto errout_free; + goto out; + + err = -ENOBUFS; + skb = nlmsg_new(nh_nlmsg_size(nh, op_flags), GFP_KERNEL); + if (!skb) + goto out; err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, 0, op_flags); diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 5321c5801c64d..690f486173e01 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -148,7 +148,7 @@ void ping_unhash(struct sock *sk) pr_debug("ping_unhash(isk=%p,isk->num=%u)\n", isk, isk->inet_num); spin_lock(&ping_table.lock); if (sk_del_node_init_rcu(sk)) { - isk->inet_num = 0; + WRITE_ONCE(isk->inet_num, 0); isk->inet_sport = 0; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); } @@ -181,31 +181,35 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) } sk_for_each_rcu(sk, hslot) { + int bound_dev_if; + if (!net_eq(sock_net(sk), net)) continue; isk = inet_sk(sk); pr_debug("iterate\n"); - if (isk->inet_num != ident) + if (READ_ONCE(isk->inet_num) != ident) continue; + bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); if (skb->protocol == htons(ETH_P_IP) && sk->sk_family == AF_INET) { + __be32 rcv_saddr = READ_ONCE(isk->inet_rcv_saddr); + pr_debug("found: %p: num=%d, daddr=%pI4, dif=%d\n", sk, - (int) isk->inet_num, &isk->inet_rcv_saddr, - sk->sk_bound_dev_if); + ident, &rcv_saddr, + bound_dev_if); - if (isk->inet_rcv_saddr && - isk->inet_rcv_saddr != ip_hdr(skb)->daddr) + if (rcv_saddr && rcv_saddr != ip_hdr(skb)->daddr) continue; #if IS_ENABLED(CONFIG_IPV6) } else if (skb->protocol == htons(ETH_P_IPV6) && sk->sk_family == AF_INET6) { pr_debug("found: %p: num=%d, daddr=%pI6c, dif=%d\n", sk, - (int) isk->inet_num, + ident, &sk->sk_v6_rcv_saddr, - sk->sk_bound_dev_if); + bound_dev_if); if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) && !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, @@ -216,8 +220,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) continue; } - if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif && - sk->sk_bound_dev_if != sdif) + if (bound_dev_if && bound_dev_if != dif && + bound_dev_if != sdif) continue; goto exit; @@ -392,7 +396,9 @@ static void ping_set_saddr(struct sock *sk, struct sockaddr *saddr) if (saddr->sa_family == AF_INET) { struct inet_sock *isk = inet_sk(sk); struct sockaddr_in *addr = (struct sockaddr_in *) saddr; - isk->inet_rcv_saddr = isk->inet_saddr = addr->sin_addr.s_addr; + + isk->inet_saddr = addr->sin_addr.s_addr; + WRITE_ONCE(isk->inet_rcv_saddr, addr->sin_addr.s_addr); #if IS_ENABLED(CONFIG_IPV6) } else if (saddr->sa_family == AF_INET6) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *) saddr; @@ -828,10 +834,8 @@ static int ping_v4_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) out_free: if (free) kfree(ipc.opt); - if (!err) { - icmp_out_count(sock_net(sk), user_icmph.type); + if (!err) return len; - } return err; do_confirm: @@ -851,7 +855,8 @@ int ping_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, struct sk_buff *skb; int copied, err; - pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, isk->inet_num); + pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, + READ_ONCE(isk->inet_num)); err = -EOPNOTSUPP; if (flags & MSG_OOB) diff --git a/net/ipv4/route.c b/net/ipv4/route.c index b549d6a573073..11d990703d31a 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -1537,9 +1537,9 @@ void rt_add_uncached_list(struct rtable *rt) void rt_del_uncached_list(struct rtable *rt) { - if (!list_empty(&rt->dst.rt_uncached)) { - struct uncached_list *ul = rt->dst.rt_uncached_list; + struct uncached_list *ul = rt->dst.rt_uncached_list; + if (ul) { spin_lock_bh(&ul->lock); list_del_init(&rt->dst.rt_uncached); spin_unlock_bh(&ul->lock); diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index 569befcf021ba..fc3affd9c8014 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -203,7 +203,7 @@ struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, bool own_req; child = icsk->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, - NULL, &own_req); + NULL, &own_req, NULL); if (child) { refcount_set(&req->rsk_refcnt, 1); sock_rps_save_rxhash(child, skb); @@ -378,9 +378,14 @@ static struct request_sock *cookie_tcp_check(struct net *net, struct sock *sk, tcp_parse_options(net, skb, &tcp_opt, 0, NULL); if (tcp_opt.saw_tstamp && tcp_opt.rcv_tsecr) { - tsoff = secure_tcp_ts_off(net, - ip_hdr(skb)->daddr, - ip_hdr(skb)->saddr); + union tcp_seq_and_ts_off st; + + st = secure_tcp_seq_and_ts_off(net, + ip_hdr(skb)->daddr, + ip_hdr(skb)->saddr, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); + tsoff = st.ts_off; tcp_opt.rcv_tsecr -= tsoff; } diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index 24dbc603cc44d..0f1dd75dbf37b 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -484,7 +484,8 @@ static void proc_fib_multipath_hash_set_seed(struct net *net, u32 user_seed) proc_fib_multipath_hash_rand_seed), }; - WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed, new); + WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.user_seed, new.user_seed); + WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.mp_seed, new.mp_seed); } static int proc_fib_multipath_hash_seed(const struct ctl_table *table, int write, @@ -498,7 +499,7 @@ static int proc_fib_multipath_hash_seed(const struct ctl_table *table, int write int ret; mphs = &net->ipv4.sysctl_fib_multipath_hash_seed; - user_seed = mphs->user_seed; + user_seed = READ_ONCE(mphs->user_seed); tmp = *table; tmp.data = &user_seed; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 8a18aeca7ab07..94e029c70247d 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -244,6 +244,7 @@ #define pr_fmt(fmt) "TCP: " fmt #include +#include #include #include #include @@ -503,6 +504,9 @@ static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc) struct sk_buff *skb = tcp_write_queue_tail(sk); u32 tsflags = sockc->tsflags; + if (unlikely(!skb)) + skb = skb_rb_last(&sk->tcp_rtx_queue); + if (tsflags && skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); @@ -1395,7 +1399,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) err = sk_stream_error(sk, flags, err); /* make sure we wake any epoll edge trigger waiter */ if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { - sk->sk_write_space(sk); + READ_ONCE(sk->sk_write_space)(sk); tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); } if (binding) @@ -2651,10 +2655,8 @@ static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, if (sk->sk_state == TCP_LISTEN) goto out; - if (tp->recvmsg_inq) { + if (tp->recvmsg_inq) *cmsg_flags = TCP_CMSG_INQ; - msg->msg_get_inq = 1; - } timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); /* Urgent data needs to be handled specially. */ @@ -2928,10 +2930,10 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags); release_sock(sk); - if ((cmsg_flags || msg->msg_get_inq) && ret >= 0) { + if ((cmsg_flags | msg->msg_get_inq) && ret >= 0) { if (cmsg_flags & TCP_CMSG_TS) tcp_recv_timestamp(msg, sk, &tss); - if (msg->msg_get_inq) { + if ((cmsg_flags & TCP_CMSG_INQ) | msg->msg_get_inq) { msg->msg_inq = tcp_inq_hint(sk); if (cmsg_flags & TCP_CMSG_INQ) put_cmsg(msg, SOL_TCP, TCP_CM_INQ, @@ -4110,7 +4112,7 @@ int do_tcp_setsockopt(struct sock *sk, int level, int optname, break; case TCP_NOTSENT_LOWAT: WRITE_ONCE(tp->notsent_lowat, val); - sk->sk_write_space(sk); + READ_ONCE(sk->sk_write_space)(sk); break; case TCP_INQ: if (val > 1 || val < 0) @@ -4898,7 +4900,7 @@ tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, else genhash = tp->af_specific->calc_md5_hash(newhash, key, NULL, skb); - if (genhash || memcmp(hash_location, newhash, 16) != 0) { + if (genhash || crypto_memneq(hash_location, newhash, 16)) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE); trace_tcp_hash_md5_mismatch(sk, skb); return SKB_DROP_REASON_TCP_MD5FAILURE; diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c index 34b8450829d0d..849a69c1f497f 100644 --- a/net/ipv4/tcp_ao.c +++ b/net/ipv4/tcp_ao.c @@ -10,6 +10,7 @@ #define pr_fmt(fmt) "TCP: " fmt #include +#include #include #include @@ -922,7 +923,7 @@ tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb, /* XXX: make it per-AF callback? */ tcp_ao_hash_skb(family, hash_buf, key, sk, skb, traffic_key, (phash - (u8 *)th), sne); - if (memcmp(phash, hash_buf, maclen)) { + if (crypto_memneq(phash, hash_buf, maclen)) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD); atomic64_inc(&info->counters.pkt_bad); atomic64_inc(&key->pkt_bad); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index a268e1595b22a..d3d6a47af5270 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -10,6 +10,7 @@ #include #include +#include void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) { @@ -226,6 +227,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, int peek = flags & MSG_PEEK; struct sk_psock *psock; struct tcp_sock *tcp; + int copied_from_self = 0; int copied = 0; u32 seq; @@ -262,7 +264,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, } msg_bytes_ready: - copied = sk_msg_recvmsg(sk, psock, msg, len, flags); + copied = __sk_msg_recvmsg(sk, psock, msg, len, flags, &copied_from_self); /* The typical case for EFAULT is the socket was gracefully * shutdown with a FIN pkt. So check here the other case is * some error on copy_page_to_iter which would be unexpected. @@ -277,7 +279,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, goto out; } } - seq += copied; + seq += copied_from_self; if (!copied) { long timeo; int data; @@ -331,6 +333,24 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, return copied; } +static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + bool slow; + + if (cmd != SIOCINQ) + return tcp_ioctl(sk, cmd, karg); + + /* works similar as tcp_ioctl */ + if (sk->sk_state == TCP_LISTEN) + return -EINVAL; + + slow = lock_sock_fast(sk); + *karg = sk_psock_msg_inq(sk); + unlock_sock_fast(sk, slow); + + return 0; +} + static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, int *addr_len) { @@ -609,6 +629,7 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], prot[TCP_BPF_BASE].close = sock_map_close; prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; + prot[TCP_BPF_BASE].ioctl = tcp_bpf_ioctl; prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; @@ -704,7 +725,7 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash); tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space); } else { - sk->sk_write_space = psock->saved_write_space; + WRITE_ONCE(sk->sk_write_space, psock->saved_write_space); /* Pairs with lockless read in sk_clone_lock() */ sock_replace_proto(sk, psock->sk_proto); } diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index df758adbb445f..e9f6c77e06316 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static DEFINE_SPINLOCK(tcp_cong_list_lock); @@ -227,7 +228,7 @@ void tcp_assign_congestion_control(struct sock *sk) memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); } @@ -257,7 +258,7 @@ static void tcp_reinit_congestion_control(struct sock *sk, memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); diff --git a/net/ipv4/tcp_diag.c b/net/ipv4/tcp_diag.c index d83efd91f461c..7935702e394b2 100644 --- a/net/ipv4/tcp_diag.c +++ b/net/ipv4/tcp_diag.c @@ -509,7 +509,7 @@ static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb, if (r->sdiag_family != AF_UNSPEC && sk->sk_family != r->sdiag_family) goto next_normal; - if (r->id.idiag_sport != htons(sk->sk_num) && + if (r->id.idiag_sport != htons(READ_ONCE(sk->sk_num)) && r->id.idiag_sport) goto next_normal; if (r->id.idiag_dport != sk->sk_dport && diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 7d945a527daf0..444306af444ae 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -247,7 +247,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, bool own_req; child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, - NULL, &own_req); + NULL, &own_req, NULL); if (!child) return NULL; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index e4a979b75cc66..96486eea26724 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -4595,15 +4595,24 @@ static enum skb_drop_reason tcp_disordered_ack_check(const struct sock *sk, */ static enum skb_drop_reason tcp_sequence(const struct sock *sk, - u32 seq, u32 end_seq) + u32 seq, u32 end_seq, + const struct tcphdr *th) { const struct tcp_sock *tp = tcp_sk(sk); + u32 seq_limit; if (before(end_seq, tp->rcv_wup)) return SKB_DROP_REASON_TCP_OLD_SEQUENCE; - if (after(end_seq, tp->rcv_nxt + tcp_receive_window(tp))) { - if (after(seq, tp->rcv_nxt + tcp_receive_window(tp))) + seq_limit = tp->rcv_nxt + tcp_receive_window(tp); + if (unlikely(after(end_seq, seq_limit))) { + /* Some stacks are known to handle FIN incorrectly; allow the + * FIN to extend beyond the window and check it in detail later. + */ + if (!after(end_seq - th->fin, seq_limit)) + return SKB_NOT_DROPPED_YET; + + if (after(seq, seq_limit)) return SKB_DROP_REASON_TCP_INVALID_SEQUENCE; /* Only accept this packet if receive queue is empty. */ @@ -5091,25 +5100,11 @@ static void tcp_ofo_queue(struct sock *sk) static bool tcp_prune_ofo_queue(struct sock *sk, const struct sk_buff *in_skb); static int tcp_prune_queue(struct sock *sk, const struct sk_buff *in_skb); -/* Check if this incoming skb can be added to socket receive queues - * while satisfying sk->sk_rcvbuf limit. - * - * In theory we should use skb->truesize, but this can cause problems - * when applications use too small SO_RCVBUF values. - * When LRO / hw gro is used, the socket might have a high tp->scaling_ratio, - * allowing RWIN to be close to available space. - * Whenever the receive queue gets full, we can receive a small packet - * filling RWIN, but with a high skb->truesize, because most NIC use 4K page - * plus sk_buff metadata even when receiving less than 1500 bytes of payload. - * - * Note that we use skb->len to decide to accept or drop this packet, - * but sk->sk_rmem_alloc is the sum of all skb->truesize. - */ static bool tcp_can_ingest(const struct sock *sk, const struct sk_buff *skb) { unsigned int rmem = atomic_read(&sk->sk_rmem_alloc); - return rmem + skb->len <= sk->sk_rcvbuf; + return rmem <= sk->sk_rcvbuf; } static int tcp_try_rmem_schedule(struct sock *sk, const struct sk_buff *skb, @@ -5142,7 +5137,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP); - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); tcp_drop_reason(sk, skb, SKB_DROP_REASON_PROTO_MEM); return; } @@ -5352,7 +5347,7 @@ int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size) void tcp_data_ready(struct sock *sk) { if (tcp_epollin_ready(sk, sk->sk_rcvlowat) || sock_flag(sk, SOCK_DONE)) - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); } static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) @@ -5408,7 +5403,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) inet_csk(sk)->icsk_ack.pending |= (ICSK_ACK_NOMEM | ICSK_ACK_NOW); inet_csk_schedule_ack(sk); - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); if (skb_queue_len(&sk->sk_receive_queue) && skb->len) { reason = SKB_DROP_REASON_PROTO_MEM; @@ -5850,7 +5845,9 @@ static void tcp_new_space(struct sock *sk) tp->snd_cwnd_stamp = tcp_jiffies32; } - INDIRECT_CALL_1(sk->sk_write_space, sk_stream_write_space, sk); + INDIRECT_CALL_1(READ_ONCE(sk->sk_write_space), + sk_stream_write_space, + sk); } /* Caller made space either from: @@ -6056,7 +6053,7 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *t BUG(); WRITE_ONCE(tp->urg_data, TCP_URG_VALID | tmp); if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); } } } @@ -6119,7 +6116,8 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, step1: /* Step 1: check sequence number */ - reason = tcp_sequence(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); + reason = tcp_sequence(sk, TCP_SKB_CB(skb)->seq, + TCP_SKB_CB(skb)->end_seq, th); if (reason) { /* RFC793, page 37: "In all states except SYN-SENT, all reset * (RST) segments are validated by checking their SEQ-fields." @@ -6817,7 +6815,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, tp->snd_wl1 = TCP_SKB_CB(skb)->seq; tp->max_window = tp->snd_wnd; - tcp_ecn_rcv_syn(tp, th, skb); + tcp_ecn_rcv_syn(sk, th, skb); tcp_mtup_init(sk); tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); @@ -7222,7 +7220,8 @@ static void tcp_ecn_create_request(struct request_sock *req, u32 ecn_ok_dst; if (tcp_accecn_syn_requested(th) && - READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3) { + (READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3 || + tcp_ca_needs_accecn(listen_sk))) { inet_rsk(req)->ecn_ok = 1; tcp_rsk(req)->accecn_ok = 1; tcp_rsk(req)->syn_ect_rcv = TCP_SKB_CB(skb)->ip_dsfield & @@ -7386,6 +7385,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, const struct tcp_sock *tp = tcp_sk(sk); struct net *net = sock_net(sk); struct sock *fastopen_sk = NULL; + union tcp_seq_and_ts_off st; struct request_sock *req; bool want_cookie = false; struct dst_entry *dst; @@ -7455,9 +7455,12 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, if (!dst) goto drop_and_free; + if (tmp_opt.tstamp_ok || (!want_cookie && !isn)) + st = af_ops->init_seq_and_ts_off(net, skb); + if (tmp_opt.tstamp_ok) { tcp_rsk(req)->req_usec_ts = dst_tcp_usec_ts(dst); - tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb); + tcp_rsk(req)->ts_off = st.ts_off; } if (!want_cookie && !isn) { int max_syn_backlog = READ_ONCE(net->ipv4.sysctl_max_syn_backlog); @@ -7479,7 +7482,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, goto drop_and_release; } - isn = af_ops->init_seq(skb); + isn = st.seq; } tcp_ecn_create_request(req, skb, sk, dst); @@ -7520,7 +7523,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, sock_put(fastopen_sk); goto drop_and_free; } - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); bh_unlock_sock(fastopen_sk); sock_put(fastopen_sk); } else { diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index b1fcf3e4e1ce0..702fdff58f7a3 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -87,6 +87,7 @@ #include #include +#include #include #include @@ -104,17 +105,14 @@ static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = { static DEFINE_MUTEX(tcp_exit_batch_mutex); -static u32 tcp_v4_init_seq(const struct sk_buff *skb) +static union tcp_seq_and_ts_off +tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb) { - return secure_tcp_seq(ip_hdr(skb)->daddr, - ip_hdr(skb)->saddr, - tcp_hdr(skb)->dest, - tcp_hdr(skb)->source); -} - -static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb) -{ - return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr); + return secure_tcp_seq_and_ts_off(net, + ip_hdr(skb)->daddr, + ip_hdr(skb)->saddr, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); } int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp) @@ -326,15 +324,16 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) rt = NULL; if (likely(!tp->repair)) { + union tcp_seq_and_ts_off st; + + st = secure_tcp_seq_and_ts_off(net, + inet->inet_saddr, + inet->inet_daddr, + inet->inet_sport, + usin->sin_port); if (!tp->write_seq) - WRITE_ONCE(tp->write_seq, - secure_tcp_seq(inet->inet_saddr, - inet->inet_daddr, - inet->inet_sport, - usin->sin_port)); - WRITE_ONCE(tp->tsoffset, - secure_tcp_ts_off(net, inet->inet_saddr, - inet->inet_daddr)); + WRITE_ONCE(tp->write_seq, st.seq); + WRITE_ONCE(tp->tsoffset, st.ts_off); } atomic_set(&inet->inet_id, get_random_u16()); @@ -842,7 +841,7 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb, genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb); - if (genhash || memcmp(md5_hash_location, newhash, 16) != 0) + if (genhash || crypto_memneq(md5_hash_location, newhash, 16)) goto out; } @@ -1727,8 +1726,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = { .cookie_init_seq = cookie_v4_init_sequence, #endif .route_req = tcp_v4_route_req, - .init_seq = tcp_v4_init_seq, - .init_ts_off = tcp_v4_init_ts_off, + .init_seq_and_ts_off = tcp_v4_init_seq_and_ts_off, .send_synack = tcp_v4_send_synack, }; @@ -1756,7 +1754,9 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct inet_request_sock *ireq; bool found_dup_sk = false; @@ -1808,6 +1808,10 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, } sk_setup_caps(newsk, dst); +#if IS_ENABLED(CONFIG_IPV6) + if (opt_child_init) + opt_child_init(newsk, sk); +#endif tcp_ca_openreq_child(newsk, dst); tcp_sync_mss(newsk, dst_mtu(dst)); diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 2ec8c6f1cdccc..c70c29a3a0905 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -488,9 +488,10 @@ static void tcp_ecn_openreq_child(struct sock *sk, tp->accecn_opt_demand = 1; tcp_ecn_received_counters_payload(sk, skb); } else { - tcp_ecn_mode_set(tp, inet_rsk(req)->ecn_ok ? - TCP_ECN_MODE_RFC3168 : - TCP_ECN_DISABLED); + if (inet_rsk(req)->ecn_ok && !tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); + else + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } } @@ -910,7 +911,7 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, * socket is created, wait for troubles. */ child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, - req, &own_req); + req, &own_req, NULL); if (!child) goto listen_overflow; @@ -989,7 +990,7 @@ enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child, reason = tcp_rcv_state_process(child, skb); /* Wakeup parent, send SIGIO */ if (state == TCP_SYN_RECV && child->sk_state != state) - parent->sk_data_ready(parent); + READ_ONCE(parent->sk_data_ready)(parent); } else { /* Alas, it is possible again, because we do lookup * in main socket hash table and lock on listening diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c index 2cb93da93abc2..ed4bc3209c973 100644 --- a/net/ipv4/tcp_offload.c +++ b/net/ipv4/tcp_offload.c @@ -107,7 +107,8 @@ static struct sk_buff *tcp4_gso_segment(struct sk_buff *skb, if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) { struct tcphdr *th = tcp_hdr(skb); - if (skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size) + if ((skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size) && + !(skb_shinfo(skb)->gso_type & SKB_GSO_DODGY)) return __tcp4_gso_segment_list(skb, features); skb->ip_summed = CHECKSUM_NONE; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 30dfbf73729da..de0deded74f0a 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -287,7 +287,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, } else { hslot = udp_hashslot(udptable, net, snum); spin_lock_bh(&hslot->lock); - if (hslot->count > 10) { + if (inet_use_hash2_on_bind(sk) && hslot->count > 10) { int exist; unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; @@ -1786,7 +1786,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) * using prepare_to_wait_exclusive(). */ while (nb) { - INDIRECT_CALL_1(sk->sk_data_ready, + INDIRECT_CALL_1(READ_ONCE(sk->sk_data_ready), sock_def_readable, sk); nb--; } @@ -1851,6 +1851,7 @@ void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) sk_peek_offset_bwd(sk, len); if (!skb_shared(skb)) { + skb_orphan(skb); skb_attempt_defer_free(skb); return; } @@ -2265,7 +2266,6 @@ void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4) udp_sk(sk)->udp_port_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); nhslot2 = udp_hashslot2(udptable, newhash); - udp_sk(sk)->udp_portaddr_hash = newhash; if (hslot2 != nhslot2 || rcu_access_pointer(sk->sk_reuseport_cb)) { @@ -2299,19 +2299,25 @@ void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4) if (udp_hashed4(sk)) { spin_lock_bh(&hslot->lock); - udp_rehash4(udptable, sk, newhash4); - if (hslot2 != nhslot2) { - spin_lock(&hslot2->lock); - udp_hash4_dec(hslot2); - spin_unlock(&hslot2->lock); - - spin_lock(&nhslot2->lock); - udp_hash4_inc(nhslot2); - spin_unlock(&nhslot2->lock); + if (inet_rcv_saddr_any(sk)) { + udp_unhash4(udptable, sk); + } else { + udp_rehash4(udptable, sk, newhash4); + if (hslot2 != nhslot2) { + spin_lock(&hslot2->lock); + udp_hash4_dec(hslot2); + spin_unlock(&hslot2->lock); + + spin_lock(&nhslot2->lock); + udp_hash4_inc(nhslot2); + spin_unlock(&nhslot2->lock); + } } spin_unlock_bh(&hslot->lock); } + + udp_sk(sk)->udp_portaddr_hash = newhash; } } EXPORT_IPV6_MOD(udp_lib_rehash); diff --git a/net/ipv4/udp_bpf.c b/net/ipv4/udp_bpf.c index 0735d820e413f..779a3a03762f1 100644 --- a/net/ipv4/udp_bpf.c +++ b/net/ipv4/udp_bpf.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "udp_impl.h" @@ -111,12 +112,26 @@ enum { static DEFINE_SPINLOCK(udpv6_prot_lock); static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS]; +static int udp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + if (cmd != SIOCINQ) + return udp_ioctl(sk, cmd, karg); + + /* Since we don't hold a lock, sk_receive_queue may contain data. + * BPF might only be processing this data at the moment. We only + * care about the data in the ingress_msg here. + */ + *karg = sk_msg_first_len(sk); + return 0; +} + static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base) { - *prot = *base; - prot->close = sock_map_close; - prot->recvmsg = udp_bpf_recvmsg; - prot->sock_is_readable = sk_msg_is_readable; + *prot = *base; + prot->close = sock_map_close; + prot->recvmsg = udp_bpf_recvmsg; + prot->sock_is_readable = sk_msg_is_readable; + prot->ioctl = udp_bpf_ioctl; } static void udp_bpf_check_v6_needs_rebuild(struct proto *ops) @@ -143,7 +158,7 @@ int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6; if (restore) { - sk->sk_write_space = psock->saved_write_space; + WRITE_ONCE(sk->sk_write_space, psock->saved_write_space); sock_replace_proto(sk, psock->sk_proto); return 0; } diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 19d0b5b09ffae..589456bd8b5f1 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -514,7 +514,8 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST) { /* Detect modified geometry and pass those to skb_segment. */ - if (skb_pagelen(gso_skb) - sizeof(*uh) == skb_shinfo(gso_skb)->gso_size) + if ((skb_pagelen(gso_skb) - sizeof(*uh) == skb_shinfo(gso_skb)->gso_size) && + !(skb_shinfo(gso_skb)->gso_type & SKB_GSO_DODGY)) return __udp_gso_segment_list(gso_skb, features, is_ipv6); ret = __skb_linearize(gso_skb); diff --git a/net/ipv4/udplite.c b/net/ipv4/udplite.c index d3e621a11a1aa..826e9e79eb19c 100644 --- a/net/ipv4/udplite.c +++ b/net/ipv4/udplite.c @@ -20,10 +20,9 @@ EXPORT_SYMBOL(udplite_table); /* Designate sk as UDP-Lite socket */ static int udplite_sk_init(struct sock *sk) { - udp_init_sock(sk); pr_warn_once("UDP-Lite is deprecated and scheduled to be removed in 2025, " "please contact the netdev mailing list\n"); - return 0; + return udp_init_sock(sk); } static int udplite_rcv(struct sk_buff *skb) diff --git a/net/ipv4/xfrm4_input.c b/net/ipv4/xfrm4_input.c index f28cfd88eaf59..c2eac844bcdb4 100644 --- a/net/ipv4/xfrm4_input.c +++ b/net/ipv4/xfrm4_input.c @@ -50,6 +50,7 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async) { struct xfrm_offload *xo = xfrm_offload(skb); struct iphdr *iph = ip_hdr(skb); + struct net_device *dev = skb->dev; iph->protocol = XFRM_MODE_SKB_CB(skb)->protocol; @@ -73,8 +74,10 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async) } NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING, - dev_net(skb->dev), NULL, skb, skb->dev, NULL, + dev_net(dev), NULL, skb, dev, NULL, xfrm4_rcv_encap_finish); + if (async) + dev_put(dev); return 0; } diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 40e9c336f6c55..2d4c3d9c1a2a5 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -2863,7 +2863,7 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao) fib6_add_gc_list(rt); } else { fib6_clean_expires(rt); - fib6_remove_gc_list(rt); + fib6_may_remove_gc_list(net, rt); } spin_unlock_bh(&table->tb6_lock); @@ -3112,12 +3112,12 @@ static int inet6_addr_del(struct net *net, int ifindex, u32 ifa_flags, in6_ifa_hold(ifp); read_unlock_bh(&idev->lock); - ipv6_del_addr(ifp); - if (!(ifp->flags & IFA_F_TEMPORARY) && (ifp->flags & IFA_F_MANAGETEMPADDR)) delete_tempaddrs(idev, ifp); + ipv6_del_addr(ifp); + addrconf_verify_rtnl(net); if (ipv6_addr_is_multicast(pfx)) { ipv6_mc_config(net->ipv6.mc_autojoin_sk, @@ -3621,12 +3621,12 @@ static void addrconf_permanent_addr(struct net *net, struct net_device *dev) if ((ifp->flags & IFA_F_PERMANENT) && fixup_permanent_addr(net, idev, ifp) < 0) { write_unlock_bh(&idev->lock); - in6_ifa_hold(ifp); - ipv6_del_addr(ifp); - write_lock_bh(&idev->lock); net_info_ratelimited("%s: Failed to add prefix route for address %pI6c; dropping\n", idev->dev->name, &ifp->addr); + in6_ifa_hold(ifp); + ipv6_del_addr(ifp); + write_lock_bh(&idev->lock); } } @@ -4836,7 +4836,7 @@ static int modify_prefix_route(struct net *net, struct inet6_ifaddr *ifp, if (!(flags & RTF_EXPIRES)) { fib6_clean_expires(f6i); - fib6_remove_gc_list(f6i); + fib6_may_remove_gc_list(net, f6i); } else { fib6_set_expires(f6i, expires); fib6_add_gc_list(f6i); diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 1b0314644e0cc..3709f213d33de 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -224,8 +224,8 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, inet6_set_bit(MC6_LOOP, sk); inet6_set_bit(MC6_ALL, sk); np->pmtudisc = IPV6_PMTUDISC_WANT; - inet6_assign_bit(REPFLOW, sk, net->ipv6.sysctl.flowlabel_reflect & - FLOWLABEL_REFLECT_ESTABLISHED); + inet6_assign_bit(REPFLOW, sk, READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_ESTABLISHED); sk->sk_ipv6only = net->ipv6.sysctl.bindv6only; sk->sk_txrehash = READ_ONCE(net->core.sysctl_txrehash); @@ -955,7 +955,7 @@ static int __net_init inet6_net_init(struct net *net) int err = 0; net->ipv6.sysctl.bindv6only = 0; - net->ipv6.sysctl.icmpv6_time = 1*HZ; + net->ipv6.sysctl.icmpv6_time = HZ / 10; net->ipv6.sysctl.icmpv6_echo_ignore_all = 0; net->ipv6.sysctl.icmpv6_echo_ignore_multicast = 0; net->ipv6.sysctl.icmpv6_echo_ignore_anycast = 0; diff --git a/net/ipv6/calipso.c b/net/ipv6/calipso.c index df1986973430c..21f6ed126253a 100644 --- a/net/ipv6/calipso.c +++ b/net/ipv6/calipso.c @@ -1342,7 +1342,8 @@ static int calipso_skbuff_setattr(struct sk_buff *skb, /* At this point new_end aligns to 4n, so (new_end & 4) pads to 8n */ pad = ((new_end & 4) + (end & 7)) & 7; len_delta = new_end - (int)end + pad; - ret_val = skb_cow(skb, skb_headroom(skb) + len_delta); + ret_val = skb_cow(skb, + skb_headroom(skb) + (len_delta > 0 ? len_delta : 0)); if (ret_val < 0) return ret_val; diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c index 33ebe93d80e3c..8933fa4c3dd50 100644 --- a/net/ipv6/datagram.c +++ b/net/ipv6/datagram.c @@ -762,6 +762,7 @@ int ip6_datagram_send_ctl(struct net *net, struct sock *sk, { struct in6_pktinfo *src_info; struct cmsghdr *cmsg; + struct ipv6_rt_hdr *orthdr; struct ipv6_rt_hdr *rthdr; struct ipv6_opt_hdr *hdr; struct ipv6_txoptions *opt = ipc6->opt; @@ -923,9 +924,13 @@ int ip6_datagram_send_ctl(struct net *net, struct sock *sk, goto exit_f; } if (cmsg->cmsg_type == IPV6_DSTOPTS) { + if (opt->dst1opt) + opt->opt_flen -= ipv6_optlen(opt->dst1opt); opt->opt_flen += len; opt->dst1opt = hdr; } else { + if (opt->dst0opt) + opt->opt_nflen -= ipv6_optlen(opt->dst0opt); opt->opt_nflen += len; opt->dst0opt = hdr; } @@ -968,12 +973,17 @@ int ip6_datagram_send_ctl(struct net *net, struct sock *sk, goto exit_f; } + orthdr = opt->srcrt; + if (orthdr) + opt->opt_nflen -= ((orthdr->hdrlen + 1) << 3); opt->opt_nflen += len; opt->srcrt = rthdr; if (cmsg->cmsg_type == IPV6_2292RTHDR && opt->dst1opt) { int dsthdrlen = ((opt->dst1opt->hdrlen+1)<<3); + if (opt->dst0opt) + opt->opt_nflen -= ipv6_optlen(opt->dst0opt); opt->opt_nflen += dsthdrlen; opt->dst0opt = opt->dst1opt; opt->dst1opt = NULL; diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index e75da98f52838..9f75313734f8c 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -271,10 +271,13 @@ static void esp_output_done(void *data, int err) xfrm_dev_resume(skb); } else { if (!err && - x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) - esp_output_tail_tcp(x, skb); - else + x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) { + err = esp_output_tail_tcp(x, skb); + if (err != -EINPROGRESS) + kfree_skb(skb); + } else { xfrm_output_resume(skb_to_full_sk(skb), skb, err); + } } } diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c index 22410243ebe88..22895521a57d0 100644 --- a/net/ipv6/esp6_offload.c +++ b/net/ipv6/esp6_offload.c @@ -158,8 +158,8 @@ static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, - XFRM_MODE_SKB_CB(skb)->protocol); + struct xfrm_offload *xo = xfrm_offload(skb); + const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, xo->proto); __be16 type = inner_mode->family == AF_INET ? htons(ETH_P_IP) : htons(ETH_P_IPV6); diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index a23eb8734e151..1d509b6d16bbd 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -314,7 +314,7 @@ static int ipv6_destopt_rcv(struct sk_buff *skb) } extlen = (skb_transport_header(skb)[1] + 1) << 3; - if (extlen > net->ipv6.sysctl.max_dst_opts_len) + if (extlen > READ_ONCE(net->ipv6.sysctl.max_dst_opts_len)) goto fail_and_free; opt->lastopt = opt->dst1 = skb_network_header_len(skb); @@ -322,7 +322,8 @@ static int ipv6_destopt_rcv(struct sk_buff *skb) dstbuf = opt->dst1; #endif - if (ip6_parse_tlv(false, skb, net->ipv6.sysctl.max_dst_opts_cnt)) { + if (ip6_parse_tlv(false, skb, + READ_ONCE(net->ipv6.sysctl.max_dst_opts_cnt))) { skb->transport_header += extlen; opt = IP6CB(skb); #if IS_ENABLED(CONFIG_IPV6_MIP6) @@ -378,6 +379,10 @@ static int ipv6_srh_rcv(struct sk_buff *skb) hdr = (struct ipv6_sr_hdr *)skb_transport_header(skb); idev = __in6_dev_get(skb->dev); + if (!idev) { + kfree_skb(skb); + return -1; + } accept_seg6 = min(READ_ONCE(net->ipv6.devconf_all->seg6_enabled), READ_ONCE(idev->cnf.seg6_enabled)); @@ -930,6 +935,11 @@ static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff) if (hdr->opt_len < 2 + sizeof(*trace) + trace->remlen * 4) goto drop; + /* Inconsistent Pre-allocated Trace header */ + if (trace->nodelen != + ioam6_trace_compute_nodelen(be32_to_cpu(trace->type_be32))) + goto drop; + /* Ignore if the IOAM namespace is unknown */ ns = ioam6_namespace(dev_net(skb->dev), trace->namespace_id); if (!ns) @@ -1049,11 +1059,12 @@ int ipv6_parse_hopopts(struct sk_buff *skb) } extlen = (skb_transport_header(skb)[1] + 1) << 3; - if (extlen > net->ipv6.sysctl.max_hbh_opts_len) + if (extlen > READ_ONCE(net->ipv6.sysctl.max_hbh_opts_len)) goto fail_and_free; opt->flags |= IP6SKB_HOPBYHOP; - if (ip6_parse_tlv(true, skb, net->ipv6.sysctl.max_hbh_opts_cnt)) { + if (ip6_parse_tlv(true, skb, + READ_ONCE(net->ipv6.sysctl.max_hbh_opts_cnt))) { skb->transport_header += extlen; opt = IP6CB(skb); opt->nhoff = sizeof(struct ipv6hdr); diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index 56c974cf75d15..aa39aabe4417e 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -217,14 +217,9 @@ static bool icmpv6_xrlim_allow(struct sock *sk, u8 type, } else if (dev && (dev->flags & IFF_LOOPBACK)) { res = true; } else { - struct rt6_info *rt = dst_rt6_info(dst); - int tmo = net->ipv6.sysctl.icmpv6_time; + int tmo = READ_ONCE(net->ipv6.sysctl.icmpv6_time); struct inet_peer *peer; - /* Give more bandwidth to wider prefixes. */ - if (rt->rt6i_dst.plen < 128) - tmo >>= ((128 - rt->rt6i_dst.plen)>>5); - peer = inet_getpeer_v6(net->ipv6.peers, &fl6->daddr); res = inet_peer_xrlim_allow(peer, tmo); } @@ -680,6 +675,9 @@ int ip6_err_gen_icmpv6_unreach(struct sk_buff *skb, int nhs, int type, if (!skb2) return 1; + /* Remove debris left by IPv4 stack. */ + memset(IP6CB(skb2), 0, sizeof(*IP6CB(skb2))); + skb_dst_drop(skb2); skb_pull(skb2, nhs); skb_reset_network_header(skb2); @@ -762,14 +760,17 @@ static enum skb_drop_reason icmpv6_echo_reply(struct sk_buff *skb) tmp_hdr.icmp6_type = type; memset(&fl6, 0, sizeof(fl6)); - if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_ICMPV6_ECHO_REPLIES) + if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_ICMPV6_ECHO_REPLIES) fl6.flowlabel = ip6_flowlabel(ipv6_hdr(skb)); fl6.flowi6_proto = IPPROTO_ICMPV6; fl6.daddr = ipv6_hdr(skb)->saddr; if (saddr) fl6.saddr = *saddr; - fl6.flowi6_oif = icmp6_iif(skb); + fl6.flowi6_oif = ipv6_addr_loopback(&fl6.daddr) ? + skb->dev->ifindex : + icmp6_iif(skb); fl6.fl6_icmp_type = type; fl6.flowi6_mark = mark; fl6.flowi6_uid = sock_net_uid(net, NULL); @@ -868,6 +869,12 @@ enum skb_drop_reason icmpv6_notify(struct sk_buff *skb, u8 type, if (reason != SKB_NOT_DROPPED_YET) goto out; + if (nexthdr == IPPROTO_RAW) { + /* Add a more specific reason later ? */ + reason = SKB_DROP_REASON_NOT_SPECIFIED; + goto out; + } + /* BUGGG_FUTURE: we should try to parse exthdrs in this packet. Without this we will not able f.e. to make source routed pmtu discovery. diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 5e1da088d8e11..182d38e6d6d8d 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -95,7 +95,8 @@ static inline int compute_score(struct sock *sk, const struct net *net, { int score = -1; - if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum && + if (net_eq(sock_net(sk), net) && + READ_ONCE(inet_sk(sk)->inet_num) == hnum && sk->sk_family == PF_INET6) { if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return -1; diff --git a/net/ipv6/ioam6.c b/net/ipv6/ioam6.c index 9553a32000813..b91de51ffa9ea 100644 --- a/net/ipv6/ioam6.c +++ b/net/ipv6/ioam6.c @@ -690,13 +690,29 @@ struct ioam6_namespace *ioam6_namespace(struct net *net, __be16 id) return rhashtable_lookup_fast(&nsdata->namespaces, &id, rht_ns_params); } +#define IOAM6_MASK_SHORT_FIELDS 0xff1ffc00 +#define IOAM6_MASK_WIDE_FIELDS 0x00e00000 + +u8 ioam6_trace_compute_nodelen(u32 trace_type) +{ + u8 nodelen = hweight32(trace_type & IOAM6_MASK_SHORT_FIELDS) + * (sizeof(__be32) / 4); + + nodelen += hweight32(trace_type & IOAM6_MASK_WIDE_FIELDS) + * (sizeof(__be64) / 4); + + return nodelen; +} + static void __ioam6_fill_trace_data(struct sk_buff *skb, struct ioam6_namespace *ns, struct ioam6_trace_hdr *trace, struct ioam6_schema *sc, - u8 sclen, bool is_input) + unsigned int sclen, bool is_input) { - struct net_device *dev = skb_dst_dev(skb); + /* Note: skb_dst_dev_rcu() can't be NULL at this point. */ + struct net_device *dev = skb_dst_dev_rcu(skb); + struct inet6_dev *i_skb_dev, *idev; struct timespec64 ts; ktime_t tstamp; u64 raw64; @@ -707,13 +723,16 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, data = trace->data + trace->remlen * 4 - trace->nodelen * 4 - sclen * 4; + i_skb_dev = skb->dev ? __in6_dev_get(skb->dev) : NULL; + idev = __in6_dev_get(dev); + /* hop_lim and node_id */ if (trace->type.bit0) { byte = ipv6_hdr(skb)->hop_limit; if (is_input) byte--; - raw32 = dev_net(dev)->ipv6.sysctl.ioam6_id; + raw32 = READ_ONCE(dev_net(dev)->ipv6.sysctl.ioam6_id); *(__be32 *)data = cpu_to_be32((byte << 24) | raw32); data += sizeof(__be32); @@ -721,18 +740,18 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, /* ingress_if_id and egress_if_id */ if (trace->type.bit1) { - if (!skb->dev) + if (!i_skb_dev) raw16 = IOAM6_U16_UNAVAILABLE; else - raw16 = (__force u16)READ_ONCE(__in6_dev_get(skb->dev)->cnf.ioam6_id); + raw16 = (__force u16)READ_ONCE(i_skb_dev->cnf.ioam6_id); *(__be16 *)data = cpu_to_be16(raw16); data += sizeof(__be16); - if (dev->flags & IFF_LOOPBACK) + if ((dev->flags & IFF_LOOPBACK) || !idev) raw16 = IOAM6_U16_UNAVAILABLE; else - raw16 = (__force u16)READ_ONCE(__in6_dev_get(dev)->cnf.ioam6_id); + raw16 = (__force u16)READ_ONCE(idev->cnf.ioam6_id); *(__be16 *)data = cpu_to_be16(raw16); data += sizeof(__be16); @@ -784,12 +803,16 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, struct Qdisc *qdisc; __u32 qlen, backlog; - if (dev->flags & IFF_LOOPBACK) { + if (dev->flags & IFF_LOOPBACK || + skb_get_queue_mapping(skb) >= dev->num_tx_queues) { *(__be32 *)data = cpu_to_be32(IOAM6_U32_UNAVAILABLE); } else { queue = skb_get_tx_queue(dev, skb); qdisc = rcu_dereference(queue->qdisc); + + spin_lock_bh(qdisc_lock(qdisc)); qdisc_qstats_qlen_backlog(qdisc, &qlen, &backlog); + spin_unlock_bh(qdisc_lock(qdisc)); *(__be32 *)data = cpu_to_be32(backlog); } @@ -808,7 +831,7 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, if (is_input) byte--; - raw64 = dev_net(dev)->ipv6.sysctl.ioam6_id_wide; + raw64 = READ_ONCE(dev_net(dev)->ipv6.sysctl.ioam6_id_wide); *(__be64 *)data = cpu_to_be64(((u64)byte << 56) | raw64); data += sizeof(__be64); @@ -816,18 +839,18 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb, /* ingress_if_id and egress_if_id (wide) */ if (trace->type.bit9) { - if (!skb->dev) + if (!i_skb_dev) raw32 = IOAM6_U32_UNAVAILABLE; else - raw32 = READ_ONCE(__in6_dev_get(skb->dev)->cnf.ioam6_id_wide); + raw32 = READ_ONCE(i_skb_dev->cnf.ioam6_id_wide); *(__be32 *)data = cpu_to_be32(raw32); data += sizeof(__be32); - if (dev->flags & IFF_LOOPBACK) + if ((dev->flags & IFF_LOOPBACK) || !idev) raw32 = IOAM6_U32_UNAVAILABLE; else - raw32 = READ_ONCE(__in6_dev_get(dev)->cnf.ioam6_id_wide); + raw32 = READ_ONCE(idev->cnf.ioam6_id_wide); *(__be32 *)data = cpu_to_be32(raw32); data += sizeof(__be32); @@ -925,7 +948,7 @@ void ioam6_fill_trace_data(struct sk_buff *skb, bool is_input) { struct ioam6_schema *sc; - u8 sclen = 0; + unsigned int sclen = 0; /* Skip if Overflow flag is set */ diff --git a/net/ipv6/ioam6_iptunnel.c b/net/ipv6/ioam6_iptunnel.c index 1fe7894f14dd9..b9f6d892a566c 100644 --- a/net/ipv6/ioam6_iptunnel.c +++ b/net/ipv6/ioam6_iptunnel.c @@ -22,9 +22,6 @@ #include #include -#define IOAM6_MASK_SHORT_FIELDS 0xff100000 -#define IOAM6_MASK_WIDE_FIELDS 0xe00000 - struct ioam6_lwt_encap { struct ipv6_hopopt_hdr eh; u8 pad[2]; /* 2-octet padding for 4n-alignment */ @@ -93,13 +90,8 @@ static bool ioam6_validate_trace_hdr(struct ioam6_trace_hdr *trace) trace->type.bit21 | trace->type.bit23) return false; - trace->nodelen = 0; fields = be32_to_cpu(trace->type_be32); - - trace->nodelen += hweight32(fields & IOAM6_MASK_SHORT_FIELDS) - * (sizeof(__be32) / 4); - trace->nodelen += hweight32(fields & IOAM6_MASK_WIDE_FIELDS) - * (sizeof(__be64) / 4); + trace->nodelen = ioam6_trace_compute_nodelen(fields); return true; } diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index 02c16909f6182..ffa7733598333 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -1133,11 +1133,16 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, return -EEXIST; if (!(rt->fib6_flags & RTF_EXPIRES)) { fib6_clean_expires(iter); - fib6_remove_gc_list(iter); + fib6_may_remove_gc_list(info->nl_net, iter); } else { fib6_set_expires(iter, rt->expires); fib6_add_gc_list(iter); } + if (!(rt->fib6_flags & (RTF_ADDRCONF | RTF_PREFIX_RT)) && + (iter->nh || !iter->fib6_nh->fib_nh_gw_family)) { + iter->fib6_flags &= ~RTF_ADDRCONF; + iter->fib6_flags &= ~RTF_PREFIX_RT; + } if (rt->fib6_pmtu) fib6_metric_set(iter, RTAX_MTU, @@ -2343,6 +2348,17 @@ static void fib6_flush_trees(struct net *net) /* * Garbage collection */ +void fib6_age_exceptions(struct fib6_info *rt, struct fib6_gc_args *gc_args, + unsigned long now) +{ + bool may_expire = rt->fib6_flags & RTF_EXPIRES && rt->expires; + int old_more = gc_args->more; + + rt6_age_exceptions(rt, gc_args, now); + + if (!may_expire && old_more == gc_args->more) + fib6_remove_gc_list(rt); +} static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args) { @@ -2365,7 +2381,7 @@ static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args) * Note, that clones are aged out * only if they are not in use now. */ - rt6_age_exceptions(rt, gc_args, now); + fib6_age_exceptions(rt, gc_args, now); return 0; } diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c index a3ff575798dda..8aa29b3d3daca 100644 --- a/net/ipv6/ip6_flowlabel.c +++ b/net/ipv6/ip6_flowlabel.c @@ -66,8 +66,8 @@ EXPORT_SYMBOL(ipv6_flowlabel_exclusive); fl != NULL; \ fl = rcu_dereference(fl->next)) -#define for_each_sk_fl_rcu(np, sfl) \ - for (sfl = rcu_dereference(np->ipv6_fl_list); \ +#define for_each_sk_fl_rcu(sk, sfl) \ + for (sfl = rcu_dereference(inet_sk(sk)->ipv6_fl_list); \ sfl != NULL; \ sfl = rcu_dereference(sfl->next)) @@ -133,11 +133,6 @@ static void fl_release(struct ip6_flowlabel *fl) if (time_after(ttd, fl->expires)) fl->expires = ttd; ttd = fl->expires; - if (fl->opt && fl->share == IPV6_FL_S_EXCL) { - struct ipv6_txoptions *opt = fl->opt; - fl->opt = NULL; - kfree(opt); - } if (!timer_pending(&ip6_fl_gc_timer) || time_after(ip6_fl_gc_timer.expires, ttd)) mod_timer(&ip6_fl_gc_timer, ttd); @@ -262,12 +257,11 @@ static struct ip6_flowlabel *fl_intern(struct net *net, struct ip6_flowlabel *__fl6_sock_lookup(struct sock *sk, __be32 label) { struct ipv6_fl_socklist *sfl; - struct ipv6_pinfo *np = inet6_sk(sk); label &= IPV6_FLOWLABEL_MASK; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { struct ip6_flowlabel *fl = sfl->fl; if (fl->label == label && atomic_inc_not_zero(&fl->users)) { @@ -283,16 +277,16 @@ EXPORT_SYMBOL_GPL(__fl6_sock_lookup); void fl6_free_socklist(struct sock *sk) { - struct ipv6_pinfo *np = inet6_sk(sk); + struct inet_sock *inet = inet_sk(sk); struct ipv6_fl_socklist *sfl; - if (!rcu_access_pointer(np->ipv6_fl_list)) + if (!rcu_access_pointer(inet->ipv6_fl_list)) return; spin_lock_bh(&ip6_sk_fl_lock); - while ((sfl = rcu_dereference_protected(np->ipv6_fl_list, + while ((sfl = rcu_dereference_protected(inet->ipv6_fl_list, lockdep_is_held(&ip6_sk_fl_lock))) != NULL) { - np->ipv6_fl_list = sfl->next; + inet->ipv6_fl_list = sfl->next; spin_unlock_bh(&ip6_sk_fl_lock); fl_release(sfl->fl); @@ -470,16 +464,15 @@ fl_create(struct net *net, struct sock *sk, struct in6_flowlabel_req *freq, static int mem_check(struct sock *sk) { - struct ipv6_pinfo *np = inet6_sk(sk); - struct ipv6_fl_socklist *sfl; int room = FL_MAX_SIZE - atomic_read(&fl_size); + struct ipv6_fl_socklist *sfl; int count = 0; if (room > FL_MAX_SIZE - FL_MAX_PER_SOCK) return 0; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) + for_each_sk_fl_rcu(sk, sfl) count++; rcu_read_unlock(); @@ -492,13 +485,15 @@ static int mem_check(struct sock *sk) return 0; } -static inline void fl_link(struct ipv6_pinfo *np, struct ipv6_fl_socklist *sfl, - struct ip6_flowlabel *fl) +static inline void fl_link(struct sock *sk, struct ipv6_fl_socklist *sfl, + struct ip6_flowlabel *fl) { + struct inet_sock *inet = inet_sk(sk); + spin_lock_bh(&ip6_sk_fl_lock); sfl->fl = fl; - sfl->next = np->ipv6_fl_list; - rcu_assign_pointer(np->ipv6_fl_list, sfl); + sfl->next = inet->ipv6_fl_list; + rcu_assign_pointer(inet->ipv6_fl_list, sfl); spin_unlock_bh(&ip6_sk_fl_lock); } @@ -520,7 +515,7 @@ int ipv6_flowlabel_opt_get(struct sock *sk, struct in6_flowlabel_req *freq, rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { if (sfl->fl->label == (np->flow_label & IPV6_FLOWLABEL_MASK)) { spin_lock_bh(&ip6_fl_lock); freq->flr_label = sfl->fl->label; @@ -559,7 +554,7 @@ static int ipv6_flowlabel_put(struct sock *sk, struct in6_flowlabel_req *freq) } spin_lock_bh(&ip6_sk_fl_lock); - for (sflp = &np->ipv6_fl_list; + for (sflp = &inet_sk(sk)->ipv6_fl_list; (sfl = socklist_dereference(*sflp)) != NULL; sflp = &sfl->next) { if (sfl->fl->label == freq->flr_label) @@ -579,13 +574,12 @@ static int ipv6_flowlabel_put(struct sock *sk, struct in6_flowlabel_req *freq) static int ipv6_flowlabel_renew(struct sock *sk, struct in6_flowlabel_req *freq) { - struct ipv6_pinfo *np = inet6_sk(sk); struct net *net = sock_net(sk); struct ipv6_fl_socklist *sfl; int err; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { if (sfl->fl->label == freq->flr_label) { err = fl6_renew(sfl->fl, freq->flr_linger, freq->flr_expires); @@ -614,7 +608,6 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, { struct ipv6_fl_socklist *sfl, *sfl1 = NULL; struct ip6_flowlabel *fl, *fl1 = NULL; - struct ipv6_pinfo *np = inet6_sk(sk); struct net *net = sock_net(sk); int err; @@ -645,7 +638,7 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, if (freq->flr_label) { err = -EEXIST; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { if (sfl->fl->label == freq->flr_label) { if (freq->flr_flags & IPV6_FL_F_EXCL) { rcu_read_unlock(); @@ -682,7 +675,7 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, fl1->linger = fl->linger; if ((long)(fl->expires - fl1->expires) > 0) fl1->expires = fl->expires; - fl_link(np, sfl1, fl1); + fl_link(sk, sfl1, fl1); fl_free(fl); return 0; @@ -716,7 +709,7 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, } } - fl_link(np, sfl1, fl); + fl_link(sk, sfl1, fl); return 0; done: fl_free(fl); diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index c82a75510c0e2..d19d86ed43766 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -535,6 +535,10 @@ static int ip6erspan_rcv(struct sk_buff *skb, if (!tun_dst) return PACKET_REJECT; + /* MUST set options_len before referencing options */ + info = &tun_dst->u.tun_info; + info->options_len = sizeof(*md); + /* skb can be uncloned in __iptunnel_pull_header, so * old pkt_md is no longer valid and we need to reset * it @@ -543,7 +547,6 @@ static int ip6erspan_rcv(struct sk_buff *skb, skb_network_header_len(skb); pkt_md = (struct erspan_metadata *)(gh + gre_hdr_len + sizeof(*ershdr)); - info = &tun_dst->u.tun_info; md = ip_tunnel_info_opts(info); md->version = ver; md2 = &md->u.md2; @@ -551,7 +554,6 @@ static int ip6erspan_rcv(struct sk_buff *skb, ERSPAN_V2_MDSIZE); __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, info->key.tun_flags); - info->options_len = sizeof(*md); ip6_tnl_rcv(tunnel, skb, tpi, tun_dst, log_ecn_error); @@ -1366,9 +1368,16 @@ static int ip6gre_header(struct sk_buff *skb, struct net_device *dev, { struct ip6_tnl *t = netdev_priv(dev); struct ipv6hdr *ipv6h; + int needed; __be16 *p; - ipv6h = skb_push(skb, t->hlen + sizeof(*ipv6h)); + needed = t->hlen + sizeof(*ipv6h); + if (skb_headroom(skb) < needed && + pskb_expand_head(skb, HH_DATA_ALIGN(needed - skb_headroom(skb)), + 0, GFP_ATOMIC)) + return -needed; + + ipv6h = skb_push(skb, needed); ip6_flow_hdr(ipv6h, 0, ip6_make_flowlabel(dev_net(dev), skb, t->fl.u.ip6.flowlabel, true, &t->fl.u.ip6)); diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 6405072050e0e..9e2449db0bdf2 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -601,11 +601,16 @@ ip4ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, if (!skb2) return 0; + /* Remove debris left by IPv6 stack. */ + memset(IPCB(skb2), 0, sizeof(*IPCB(skb2))); + skb_dst_drop(skb2); skb_pull(skb2, offset); skb_reset_network_header(skb2); eiph = ip_hdr(skb2); + if (eiph->version != 4 || eiph->ihl < 5) + goto out; /* Try to guess incoming interface */ rt = ip_route_output_ports(dev_net(skb->dev), &fl4, NULL, eiph->saddr, @@ -844,7 +849,7 @@ static int __ip6_tnl_rcv(struct ip6_tnl *tunnel, struct sk_buff *skb, skb_reset_network_header(skb); - if (!pskb_inet_may_pull(skb)) { + if (skb_vlan_inet_prepare(skb, true)) { DEV_STATS_INC(tunnel->dev, rx_length_errors); DEV_STATS_INC(tunnel->dev, rx_errors); goto drop; diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index f427e41e9c49b..ded2d3a0660c3 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1209,6 +1209,9 @@ static void ndisc_ra_useropt(struct sk_buff *ra, struct nd_opt_hdr *opt) ndmsg->nduseropt_icmp_type = icmp6h->icmp6_type; ndmsg->nduseropt_icmp_code = icmp6h->icmp6_code; ndmsg->nduseropt_opts_len = opt->nd_opt_len << 3; + ndmsg->nduseropt_pad1 = 0; + ndmsg->nduseropt_pad2 = 0; + ndmsg->nduseropt_pad3 = 0; memcpy(ndmsg + 1, opt, opt->nd_opt_len << 3); @@ -1555,8 +1558,8 @@ static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb) memcpy(&n, ((u8 *)(ndopts.nd_opts_mtu+1))+2, sizeof(mtu)); mtu = ntohl(n); - if (in6_dev->ra_mtu != mtu) { - in6_dev->ra_mtu = mtu; + if (READ_ONCE(in6_dev->ra_mtu) != mtu) { + WRITE_ONCE(in6_dev->ra_mtu, mtu); send_ifinfo_notify = true; } diff --git a/net/ipv6/netfilter/ip6t_eui64.c b/net/ipv6/netfilter/ip6t_eui64.c index d704f7ed300c2..da69a27e8332c 100644 --- a/net/ipv6/netfilter/ip6t_eui64.c +++ b/net/ipv6/netfilter/ip6t_eui64.c @@ -22,8 +22,7 @@ eui64_mt6(const struct sk_buff *skb, struct xt_action_param *par) unsigned char eui64[8]; if (!(skb_mac_header(skb) >= skb->head && - skb_mac_header(skb) + ETH_HLEN <= skb->data) && - par->fragoff != 0) { + skb_mac_header(skb) + ETH_HLEN <= skb->data)) { par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 4ad8b2032f1f9..5561bd9cea818 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -157,6 +157,10 @@ static int rt_mt6_check(const struct xt_mtchk_param *par) pr_debug("unknown flags %X\n", rtinfo->invflags); return -EINVAL; } + if (rtinfo->addrnr > IP6T_RT_HOPS) { + pr_debug("too many addresses specified\n"); + return -EINVAL; + } if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) && (!(rtinfo->flags & IP6T_RT_TYP) || (rtinfo->rt_type != 0) || diff --git a/net/ipv6/route.c b/net/ipv6/route.c index aee6a10b112aa..446f4de7d6a22 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -148,9 +148,9 @@ void rt6_uncached_list_add(struct rt6_info *rt) void rt6_uncached_list_del(struct rt6_info *rt) { - if (!list_empty(&rt->dst.rt_uncached)) { - struct uncached_list *ul = rt->dst.rt_uncached_list; + struct uncached_list *ul = rt->dst.rt_uncached_list; + if (ul) { spin_lock_bh(&ul->lock); list_del_init(&rt->dst.rt_uncached); spin_unlock_bh(&ul->lock); @@ -1033,7 +1033,7 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, if (!addrconf_finite_timeout(lifetime)) { fib6_clean_expires(rt); - fib6_remove_gc_list(rt); + fib6_may_remove_gc_list(net, rt); } else { fib6_set_expires(rt, jiffies + HZ * lifetime); fib6_add_gc_list(rt); @@ -1063,7 +1063,8 @@ static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) */ if (netif_is_l3_slave(dev) && !rt6_need_strict(&res->f6i->fib6_dst.addr)) - dev = l3mdev_master_dev_rcu(dev); + dev = l3mdev_master_dev_rcu(dev) ? : + dev_net(dev)->loopback_dev; else if (!netif_is_l3_master(dev)) dev = dev_net(dev)->loopback_dev; /* last case is netif_is_l3_master(dev) is true in which @@ -1470,7 +1471,18 @@ static struct rt6_info *rt6_make_pcpu_route(struct net *net, p = this_cpu_ptr(res->nh->rt6i_pcpu); prev = cmpxchg(p, NULL, pcpu_rt); - BUG_ON(prev); + if (unlikely(prev)) { + /* + * Another task on this CPU already installed a pcpu_rt. + * This can happen on PREEMPT_RT where preemption is possible. + * Free our allocation and return the existing one. + */ + WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RT)); + + dst_dev_put(&pcpu_rt->dst); + dst_release(&pcpu_rt->dst); + return prev; + } if (res->f6i->fib6_destroying) { struct fib6_info *from; @@ -2884,7 +2896,7 @@ static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) dst_metric_set(&rt->dst, RTAX_MTU, mtu); rt->rt6i_flags |= RTF_MODIFIED; - rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); + rt6_update_expires(rt, READ_ONCE(net->ipv6.sysctl.ip6_rt_mtu_expires)); } static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) @@ -3245,8 +3257,8 @@ static unsigned int ip6_default_advmss(const struct dst_entry *dst) rcu_read_lock(); net = dst_dev_net_rcu(dst); - if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) - mtu = net->ipv6.sysctl.ip6_rt_min_advmss; + mtu = max_t(unsigned int, mtu, + READ_ONCE(net->ipv6.sysctl.ip6_rt_min_advmss)); rcu_read_unlock(); @@ -3348,10 +3360,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev, static void ip6_dst_gc(struct dst_ops *ops) { struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); - int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; - int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; - int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; - unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; + int rt_min_interval = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_min_interval); + int rt_elasticity = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_elasticity); + int rt_gc_timeout = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_timeout); + unsigned long rt_last_gc = READ_ONCE(net->ipv6.ip6_rt_last_gc); unsigned int val; int entries; @@ -3572,7 +3584,6 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker; struct net_device *dev = NULL; struct inet6_dev *idev = NULL; - int addr_type; int err; fib6_nh->fib_nh_family = AF_INET6; @@ -3614,11 +3625,10 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, fib6_nh->fib_nh_weight = 1; - /* We cannot add true routes via loopback here, - * they would result in kernel looping; promote them to reject routes + /* Reset the nexthop device to the loopback device in case of reject + * routes. */ - addr_type = ipv6_addr_type(&cfg->fc_dst); - if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { + if (cfg->fc_flags & RTF_REJECT) { /* hold loopback dev/idev if we haven't done so. */ if (dev != net->loopback_dev) { if (dev) { @@ -4997,7 +5007,7 @@ void rt6_sync_down_dev(struct net_device *dev, unsigned long event) }; struct net *net = dev_net(dev); - if (net->ipv6.sysctl.skip_notify_on_dev_down) + if (READ_ONCE(net->ipv6.sysctl.skip_notify_on_dev_down)) fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); else fib6_clean_all(net, fib6_ifdown, &arg); @@ -6397,6 +6407,7 @@ void fib6_rt_update(struct net *net, struct fib6_info *rt, void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, bool offload, bool trap, bool offload_failed) { + u8 fib_notify_on_flag_change; struct sk_buff *skb; int err; @@ -6408,8 +6419,9 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, WRITE_ONCE(f6i->offload, offload); WRITE_ONCE(f6i->trap, trap); + fib_notify_on_flag_change = READ_ONCE(net->ipv6.sysctl.fib_notify_on_flag_change); /* 2 means send notifications only if offload_failed was changed. */ - if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && + if (fib_notify_on_flag_change == 2 && READ_ONCE(f6i->offload_failed) == offload_failed) return; @@ -6421,7 +6433,7 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, */ return; - if (!net->ipv6.sysctl.fib_notify_on_flag_change) + if (!fib_notify_on_flag_change) return; skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); @@ -6518,7 +6530,7 @@ static int ipv6_sysctl_rtcache_flush(const struct ctl_table *ctl, int write, return ret; net = (struct net *)ctl->extra1; - delay = net->ipv6.sysctl.flush_delay; + delay = READ_ONCE(net->ipv6.sysctl.flush_delay); fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); return 0; } diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index ee6bac0160ace..e6964c6b0d381 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -184,6 +184,8 @@ bool seg6_hmac_validate_skb(struct sk_buff *skb) int require_hmac; idev = __in6_dev_get(skb->dev); + if (!idev) + return false; srh = (struct ipv6_sr_hdr *)skb_transport_header(skb); diff --git a/net/ipv6/seg6_iptunnel.c b/net/ipv6/seg6_iptunnel.c index 3e1b9991131a2..d6a0f7df90807 100644 --- a/net/ipv6/seg6_iptunnel.c +++ b/net/ipv6/seg6_iptunnel.c @@ -48,7 +48,8 @@ static size_t seg6_lwt_headroom(struct seg6_iptunnel_encap *tuninfo) } struct seg6_lwt { - struct dst_cache cache; + struct dst_cache cache_input; + struct dst_cache cache_output; struct seg6_iptunnel_encap tuninfo[]; }; @@ -488,7 +489,7 @@ static int seg6_input_core(struct net *net, struct sock *sk, slwt = seg6_lwt_lwtunnel(lwtst); local_bh_disable(); - dst = dst_cache_get(&slwt->cache); + dst = dst_cache_get(&slwt->cache_input); local_bh_enable(); err = seg6_do_srh(skb, dst); @@ -504,7 +505,7 @@ static int seg6_input_core(struct net *net, struct sock *sk, /* cache only if we don't create a dst reference loop */ if (!dst->error && lwtst != dst->lwtstate) { local_bh_disable(); - dst_cache_set_ip6(&slwt->cache, dst, + dst_cache_set_ip6(&slwt->cache_input, dst, &ipv6_hdr(skb)->saddr); local_bh_enable(); } @@ -564,7 +565,7 @@ static int seg6_output_core(struct net *net, struct sock *sk, slwt = seg6_lwt_lwtunnel(orig_dst->lwtstate); local_bh_disable(); - dst = dst_cache_get(&slwt->cache); + dst = dst_cache_get(&slwt->cache_output); local_bh_enable(); err = seg6_do_srh(skb, dst); @@ -591,7 +592,7 @@ static int seg6_output_core(struct net *net, struct sock *sk, /* cache only if we don't create a dst reference loop */ if (orig_dst->lwtstate != dst->lwtstate) { local_bh_disable(); - dst_cache_set_ip6(&slwt->cache, dst, &fl6.saddr); + dst_cache_set_ip6(&slwt->cache_output, dst, &fl6.saddr); local_bh_enable(); } @@ -701,11 +702,13 @@ static int seg6_build_state(struct net *net, struct nlattr *nla, slwt = seg6_lwt_lwtunnel(newts); - err = dst_cache_init(&slwt->cache, GFP_ATOMIC); - if (err) { - kfree(newts); - return err; - } + err = dst_cache_init(&slwt->cache_input, GFP_ATOMIC); + if (err) + goto err_free_newts; + + err = dst_cache_init(&slwt->cache_output, GFP_ATOMIC); + if (err) + goto err_destroy_input; memcpy(&slwt->tuninfo, tuninfo, tuninfo_len); @@ -720,11 +723,20 @@ static int seg6_build_state(struct net *net, struct nlattr *nla, *ts = newts; return 0; + +err_destroy_input: + dst_cache_destroy(&slwt->cache_input); +err_free_newts: + kfree(newts); + return err; } static void seg6_destroy_state(struct lwtunnel_state *lwt) { - dst_cache_destroy(&seg6_lwt_lwtunnel(lwt)->cache); + struct seg6_lwt *slwt = seg6_lwt_lwtunnel(lwt); + + dst_cache_destroy(&slwt->cache_input); + dst_cache_destroy(&slwt->cache_output); } static int seg6_fill_encap_info(struct sk_buff *skb, diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index 7e007f013ec82..4f6f0d751d6c5 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -151,9 +151,14 @@ static struct request_sock *cookie_tcp_check(struct net *net, struct sock *sk, tcp_parse_options(net, skb, &tcp_opt, 0, NULL); if (tcp_opt.saw_tstamp && tcp_opt.rcv_tsecr) { - tsoff = secure_tcpv6_ts_off(net, - ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32); + union tcp_seq_and_ts_off st; + + st = secure_tcpv6_seq_and_ts_off(net, + ipv6_hdr(skb)->daddr.s6_addr32, + ipv6_hdr(skb)->saddr.s6_addr32, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); + tsoff = st.ts_off; tcp_opt.rcv_tsecr -= tsoff; } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 59c4977a811a0..7f20db11e8ce9 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -68,6 +68,7 @@ #include #include +#include #include #include @@ -105,18 +106,14 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) } } -static u32 tcp_v6_init_seq(const struct sk_buff *skb) +static union tcp_seq_and_ts_off +tcp_v6_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb) { - return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32, - tcp_hdr(skb)->dest, - tcp_hdr(skb)->source); -} - -static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb) -{ - return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32); + return secure_tcpv6_seq_and_ts_off(net, + ipv6_hdr(skb)->daddr.s6_addr32, + ipv6_hdr(skb)->saddr.s6_addr32, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); } static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr, @@ -319,14 +316,16 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, sk_set_txhash(sk); if (likely(!tp->repair)) { + union tcp_seq_and_ts_off st; + + st = secure_tcpv6_seq_and_ts_off(net, + np->saddr.s6_addr32, + sk->sk_v6_daddr.s6_addr32, + inet->inet_sport, + inet->inet_dport); if (!tp->write_seq) - WRITE_ONCE(tp->write_seq, - secure_tcpv6_seq(np->saddr.s6_addr32, - sk->sk_v6_daddr.s6_addr32, - inet->inet_sport, - inet->inet_dport)); - tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, - sk->sk_v6_daddr.s6_addr32); + WRITE_ONCE(tp->write_seq, st.seq); + tp->tsoffset = st.ts_off; } if (tcp_fastopen_defer_connect(sk, &err)) @@ -859,8 +858,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { .cookie_init_seq = cookie_v6_init_sequence, #endif .route_req = tcp_v6_route_req, - .init_seq = tcp_v6_init_seq, - .init_ts_off = tcp_v6_init_ts_off, + .init_seq_and_ts_off = tcp_v6_init_seq_and_ts_off, .send_synack = tcp_v6_send_synack, }; @@ -1092,7 +1090,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, key.type = TCP_KEY_MD5; genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb); - if (genhash || memcmp(md5_hash_location, newhash, 16) != 0) + if (genhash || crypto_memneq(md5_hash_location, newhash, 16)) goto out; } #endif @@ -1131,7 +1129,8 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, txhash = inet_twsk(sk)->tw_txhash; } } else { - if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) + if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_TCP_RESET) label = ip6_flowlabel(ipv6h); } @@ -1355,11 +1354,48 @@ static void tcp_v6_restore_cb(struct sk_buff *skb) sizeof(struct inet6_skb_parm)); } +/* Called from tcp_v4_syn_recv_sock() for v6_mapped children. */ +static void tcp_v6_mapped_child_init(struct sock *newsk, const struct sock *sk) +{ + struct inet_sock *newinet = inet_sk(newsk); + struct ipv6_pinfo *newnp; + + newinet->pinet6 = newnp = tcp_inet6_sk(newsk); + newinet->ipv6_fl_list = NULL; + + memcpy(newnp, tcp_inet6_sk(sk), sizeof(struct ipv6_pinfo)); + + newnp->saddr = newsk->sk_v6_rcv_saddr; + + inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; + if (sk_is_mptcp(newsk)) + mptcpv6_handle_mapped(newsk, true); + newsk->sk_backlog_rcv = tcp_v4_do_rcv; +#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) + tcp_sk(newsk)->af_specific = &tcp_sock_ipv6_mapped_specific; +#endif + + newnp->ipv6_mc_list = NULL; + newnp->ipv6_ac_list = NULL; + newnp->pktoptions = NULL; + newnp->opt = NULL; + + /* tcp_v4_syn_recv_sock() has initialized newinet->mc_{index,ttl} */ + newnp->mcast_oif = newinet->mc_index; + newnp->mcast_hops = newinet->mc_ttl; + + newnp->rcv_flowinfo = 0; + if (inet6_test_bit(REPFLOW, sk)) + newnp->flow_label = 0; +} + static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct inet_request_sock *ireq; struct ipv6_pinfo *newnp; @@ -1375,60 +1411,10 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * #endif struct flowi6 fl6; - if (skb->protocol == htons(ETH_P_IP)) { - /* - * v6 mapped - */ - - newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst, - req_unhash, own_req); - - if (!newsk) - return NULL; - - inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); - - newnp = tcp_inet6_sk(newsk); - newtp = tcp_sk(newsk); - - memcpy(newnp, np, sizeof(struct ipv6_pinfo)); - - newnp->saddr = newsk->sk_v6_rcv_saddr; - - inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; - if (sk_is_mptcp(newsk)) - mptcpv6_handle_mapped(newsk, true); - newsk->sk_backlog_rcv = tcp_v4_do_rcv; -#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) - newtp->af_specific = &tcp_sock_ipv6_mapped_specific; -#endif - - newnp->ipv6_mc_list = NULL; - newnp->ipv6_ac_list = NULL; - newnp->ipv6_fl_list = NULL; - newnp->pktoptions = NULL; - newnp->opt = NULL; - newnp->mcast_oif = inet_iif(skb); - newnp->mcast_hops = ip_hdr(skb)->ttl; - newnp->rcv_flowinfo = 0; - if (inet6_test_bit(REPFLOW, sk)) - newnp->flow_label = 0; - - /* - * No need to charge this sock to the relevant IPv6 refcnt debug socks count - * here, tcp_create_openreq_child now does this for us, see the comment in - * that function for the gory details. -acme - */ - - /* It is tricky place. Until this moment IPv4 tcp - worked with IPv6 icsk.icsk_af_ops. - Sync it now. - */ - tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie); - - return newsk; - } - + if (skb->protocol == htons(ETH_P_IP)) + return tcp_v4_syn_recv_sock(sk, skb, req, dst, + req_unhash, own_req, + tcp_v6_mapped_child_init); ireq = inet_rsk(req); if (sk_acceptq_is_full(sk)) @@ -1453,10 +1439,12 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * newsk->sk_gso_type = SKB_GSO_TCPV6; inet6_sk_rx_dst_set(newsk, skb); - inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); + newinet = inet_sk(newsk); + newinet->pinet6 = tcp_inet6_sk(newsk); + newinet->ipv6_fl_list = NULL; + newinet->inet_opt = NULL; newtp = tcp_sk(newsk); - newinet = inet_sk(newsk); newnp = tcp_inet6_sk(newsk); memcpy(newnp, np, sizeof(struct ipv6_pinfo)); @@ -1469,10 +1457,8 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * First: no IPv4 options. */ - newinet->inet_opt = NULL; newnp->ipv6_mc_list = NULL; newnp->ipv6_ac_list = NULL; - newnp->ipv6_fl_list = NULL; /* Clone RX bits */ newnp->rxopt.all = np->rxopt.all; diff --git a/net/ipv6/tcpv6_offload.c b/net/ipv6/tcpv6_offload.c index effeba58630b5..5670d32c27f80 100644 --- a/net/ipv6/tcpv6_offload.c +++ b/net/ipv6/tcpv6_offload.c @@ -170,7 +170,8 @@ static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb, if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) { struct tcphdr *th = tcp_hdr(skb); - if (skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size) + if ((skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size) && + !(skb_shinfo(skb)->gso_type & SKB_GSO_DODGY)) return __tcp6_gso_segment_list(skb, features); skb->ip_summed = CHECKSUM_NONE; diff --git a/net/ipv6/udplite.c b/net/ipv6/udplite.c index 2cec542437f74..e867721cda4d3 100644 --- a/net/ipv6/udplite.c +++ b/net/ipv6/udplite.c @@ -16,10 +16,9 @@ static int udplitev6_sk_init(struct sock *sk) { - udpv6_init_sock(sk); pr_warn_once("UDP-Lite is deprecated and scheduled to be removed in 2025, " "please contact the netdev mailing list\n"); - return 0; + return udpv6_init_sock(sk); } static int udplitev6_rcv(struct sk_buff *skb) diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 9005fc156a20e..699a001ac1662 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -43,6 +43,7 @@ static int xfrm6_transport_finish2(struct net *net, struct sock *sk, int xfrm6_transport_finish(struct sk_buff *skb, int async) { struct xfrm_offload *xo = xfrm_offload(skb); + struct net_device *dev = skb->dev; int nhlen = -skb_network_offset(skb); skb_network_header(skb)[IP6CB(skb)->nhoff] = @@ -68,8 +69,10 @@ int xfrm6_transport_finish(struct sk_buff *skb, int async) } NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING, - dev_net(skb->dev), NULL, skb, skb->dev, NULL, + dev_net(dev), NULL, skb, dev, NULL, xfrm6_transport_finish2); + if (async) + dev_put(dev); return 0; } diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index 1f19b6f14484c..125ea9a5b8a08 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -57,6 +57,7 @@ static int xfrm6_get_saddr(xfrm_address_t *saddr, struct dst_entry *dst; struct net_device *dev; struct inet6_dev *idev; + int err; dst = xfrm6_dst_lookup(params); if (IS_ERR(dst)) @@ -68,9 +69,11 @@ static int xfrm6_get_saddr(xfrm_address_t *saddr, return -EHOSTUNREACH; } dev = idev->dev; - ipv6_dev_get_saddr(dev_net(dev), dev, ¶ms->daddr->in6, 0, - &saddr->in6); + err = ipv6_dev_get_saddr(dev_net(dev), dev, ¶ms->daddr->in6, 0, + &saddr->in6); dst_release(dst); + if (err) + return -EHOSTUNREACH; return 0; } diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index b4f01cb075615..f6d44481954cd 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -628,7 +628,7 @@ static int kcm_write_msgs(struct kcm_sock *kcm) skb = txm->frag_skb; } - if (WARN_ON(!skb_shinfo(skb)->nr_frags) || + if (WARN_ON_ONCE(!skb_shinfo(skb)->nr_frags) || WARN_ON_ONCE(!skb_frag_page(&skb_shinfo(skb)->frags[0]))) { ret = -EINVAL; goto out; @@ -749,7 +749,7 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) { struct sock *sk = sock->sk; struct kcm_sock *kcm = kcm_sk(sk); - struct sk_buff *skb = NULL, *head = NULL; + struct sk_buff *skb = NULL, *head = NULL, *frag_prev = NULL; size_t copy, copied = 0; long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); int eor = (sock->type == SOCK_DGRAM) ? @@ -824,6 +824,7 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) else skb->next = tskb; + frag_prev = skb; skb = tskb; skb->ip_summed = CHECKSUM_UNNECESSARY; continue; @@ -933,6 +934,22 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) out_error: kcm_push(kcm); + /* When MAX_SKB_FRAGS was reached, a new skb was allocated and + * linked into the frag_list before data copy. If the copy + * subsequently failed, this skb has zero frags. Remove it from + * the frag_list to prevent kcm_write_msgs from later hitting + * WARN_ON(!skb_shinfo(skb)->nr_frags). + */ + if (frag_prev && !skb_shinfo(skb)->nr_frags) { + if (head == frag_prev) + skb_shinfo(head)->frag_list = NULL; + else + frag_prev->next = NULL; + kfree_skb(skb); + /* Update skb as it may be saved in partial_message via goto */ + skb = frag_prev; + } + if (sock->type == SOCK_SEQPACKET) { /* Wrote some bytes before encountering an * error, return partial success. diff --git a/net/key/af_key.c b/net/key/af_key.c index 2ebde03522459..ceaa82bc78acc 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -3518,7 +3518,7 @@ static int set_sadb_kmaddress(struct sk_buff *skb, const struct xfrm_kmaddress * static int set_ipsecrequest(struct sk_buff *skb, uint8_t proto, uint8_t mode, int level, - uint32_t reqid, uint8_t family, + uint32_t reqid, sa_family_t family, const xfrm_address_t *src, const xfrm_address_t *dst) { struct sadb_x_ipsecrequest *rq; @@ -3583,12 +3583,17 @@ static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, /* ipsecrequests */ for (i = 0, mp = m; i < num_bundles; i++, mp++) { - /* old locator pair */ - size_pol += sizeof(struct sadb_x_ipsecrequest) + - pfkey_sockaddr_pair_size(mp->old_family); - /* new locator pair */ - size_pol += sizeof(struct sadb_x_ipsecrequest) + - pfkey_sockaddr_pair_size(mp->new_family); + int pair_size; + + pair_size = pfkey_sockaddr_pair_size(mp->old_family); + if (!pair_size) + return -EINVAL; + size_pol += sizeof(struct sadb_x_ipsecrequest) + pair_size; + + pair_size = pfkey_sockaddr_pair_size(mp->new_family); + if (!pair_size) + return -EINVAL; + size_pol += sizeof(struct sadb_x_ipsecrequest) + pair_size; } size += sizeof(struct sadb_msg) + size_pol; diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c index 0710281dd95aa..9156a937334ae 100644 --- a/net/l2tp/l2tp_core.c +++ b/net/l2tp/l2tp_core.c @@ -1086,8 +1086,10 @@ int l2tp_udp_encap_recv(struct sock *sk, struct sk_buff *skb) tunnel = session->tunnel; /* Check protocol version */ - if (version != tunnel->version) + if (version != tunnel->version) { + l2tp_session_put(session); goto invalid; + } if (version == L2TP_HDR_VER_3 && l2tp_v3_ensure_opt_in_linear(session, skb, &ptr, &optr)) { @@ -1288,6 +1290,11 @@ static int l2tp_xmit_core(struct l2tp_session *session, struct sk_buff *skb, uns uh->source = inet->inet_sport; uh->dest = inet->inet_dport; udp_len = uhlen + session->hdr_len + data_len; + if (udp_len > U16_MAX) { + kfree_skb(skb); + ret = NET_XMIT_DROP; + goto out_unlock; + } uh->len = htons(udp_len); /* Calculate UDP checksum if configured to do so */ @@ -1414,8 +1421,6 @@ static void l2tp_tunnel_del_work(struct work_struct *work) { struct l2tp_tunnel *tunnel = container_of(work, struct l2tp_tunnel, del_work); - struct sock *sk = tunnel->sock; - struct socket *sock = sk->sk_socket; l2tp_tunnel_closeall(tunnel); @@ -1423,6 +1428,8 @@ static void l2tp_tunnel_del_work(struct work_struct *work) * the sk API to release it here. */ if (tunnel->fd < 0) { + struct socket *sock = tunnel->sock->sk_socket; + if (sock) { kernel_sock_shutdown(sock, SHUT_RDWR); sock_release(sock); diff --git a/net/mac80211/aes_cmac.c b/net/mac80211/aes_cmac.c index 48c04f89de20a..65989c7dfc680 100644 --- a/net/mac80211/aes_cmac.c +++ b/net/mac80211/aes_cmac.c @@ -22,50 +22,77 @@ static const u8 zero[CMAC_TLEN_256]; -void ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic) +int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic) { + int err; SHASH_DESC_ON_STACK(desc, tfm); u8 out[AES_BLOCK_SIZE]; const __le16 *fc; desc->tfm = tfm; - crypto_shash_init(desc); - crypto_shash_update(desc, aad, AAD_LEN); + err = crypto_shash_init(desc); + if (err) + return err; + err = crypto_shash_update(desc, aad, AAD_LEN); + if (err) + return err; fc = (const __le16 *)aad; if (ieee80211_is_beacon(*fc)) { /* mask Timestamp field to zero */ - crypto_shash_update(desc, zero, 8); - crypto_shash_update(desc, data + 8, data_len - 8 - CMAC_TLEN); + err = crypto_shash_update(desc, zero, 8); + if (err) + return err; + err = crypto_shash_update(desc, data + 8, + data_len - 8 - CMAC_TLEN); + if (err) + return err; } else { - crypto_shash_update(desc, data, data_len - CMAC_TLEN); + err = crypto_shash_update(desc, data, + data_len - CMAC_TLEN); + if (err) + return err; } - crypto_shash_finup(desc, zero, CMAC_TLEN, out); - + err = crypto_shash_finup(desc, zero, CMAC_TLEN, out); + if (err) + return err; memcpy(mic, out, CMAC_TLEN); + + return 0; } -void ieee80211_aes_cmac_256(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic) +int ieee80211_aes_cmac_256(struct crypto_shash *tfm, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic) { + int err; SHASH_DESC_ON_STACK(desc, tfm); const __le16 *fc; desc->tfm = tfm; - crypto_shash_init(desc); - crypto_shash_update(desc, aad, AAD_LEN); + err = crypto_shash_init(desc); + if (err) + return err; + err = crypto_shash_update(desc, aad, AAD_LEN); + if (err) + return err; fc = (const __le16 *)aad; if (ieee80211_is_beacon(*fc)) { /* mask Timestamp field to zero */ - crypto_shash_update(desc, zero, 8); - crypto_shash_update(desc, data + 8, - data_len - 8 - CMAC_TLEN_256); + err = crypto_shash_update(desc, zero, 8); + if (err) + return err; + err = crypto_shash_update(desc, data + 8, + data_len - 8 - CMAC_TLEN_256); + if (err) + return err; } else { - crypto_shash_update(desc, data, data_len - CMAC_TLEN_256); + err = crypto_shash_update(desc, data, data_len - CMAC_TLEN_256); + if (err) + return err; } - crypto_shash_finup(desc, zero, CMAC_TLEN_256, mic); + return crypto_shash_finup(desc, zero, CMAC_TLEN_256, mic); } struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[], diff --git a/net/mac80211/aes_cmac.h b/net/mac80211/aes_cmac.h index 76817446fb838..f74150542142a 100644 --- a/net/mac80211/aes_cmac.h +++ b/net/mac80211/aes_cmac.h @@ -11,10 +11,10 @@ struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[], size_t key_len); -void ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic); -void ieee80211_aes_cmac_256(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic); +int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic); +int ieee80211_aes_cmac_256(struct crypto_shash *tfm, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic); void ieee80211_aes_cmac_key_free(struct crypto_shash *tfm); #endif /* AES_CMAC_H */ diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index c52b0456039dd..d32eacbb7517d 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1328,7 +1328,6 @@ ieee80211_assign_beacon(struct ieee80211_sub_if_data *sdata, size = sizeof(*new) + new_head_len + new_tail_len; - /* new or old multiple BSSID elements? */ if (params->mbssid_ies) { mbssid = params->mbssid_ies; size += struct_size(new->mbssid_ies, elem, mbssid->cnt); @@ -1338,15 +1337,6 @@ ieee80211_assign_beacon(struct ieee80211_sub_if_data *sdata, } size += ieee80211_get_mbssid_beacon_len(mbssid, rnr, mbssid->cnt); - } else if (old && old->mbssid_ies) { - mbssid = old->mbssid_ies; - size += struct_size(new->mbssid_ies, elem, mbssid->cnt); - if (old && old->rnr_ies) { - rnr = old->rnr_ies; - size += struct_size(new->rnr_ies, elem, rnr->cnt); - } - size += ieee80211_get_mbssid_beacon_len(mbssid, rnr, - mbssid->cnt); } new = kzalloc(size, GFP_KERNEL); @@ -1882,12 +1872,6 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, __sta_info_flush(sdata, true, link_id, NULL); - ieee80211_remove_link_keys(link, &keys); - if (!list_empty(&keys)) { - synchronize_net(); - ieee80211_free_key_list(local, &keys); - } - ieee80211_stop_mbssid(sdata); RCU_INIT_POINTER(link_conf->tx_bss_conf, NULL); @@ -1899,6 +1883,12 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, ieee80211_link_info_change_notify(sdata, link, BSS_CHANGED_BEACON_ENABLED); + ieee80211_remove_link_keys(link, &keys); + if (!list_empty(&keys)) { + synchronize_net(); + ieee80211_free_key_list(local, &keys); + } + if (sdata->wdev.links[link_id].cac_started) { chandef = link_conf->chanreq.oper; wiphy_delayed_work_cancel(wiphy, &link->dfs_cac_timer_work); diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index 7f8799fd673eb..a6895b40d4322 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -441,14 +441,16 @@ static void ieee80211_chan_bw_change(struct ieee80211_local *local, rcu_read_lock(); list_for_each_entry_rcu(sta, &local->sta_list, list) { - struct ieee80211_sub_if_data *sdata = sta->sdata; + struct ieee80211_sub_if_data *sdata; enum ieee80211_sta_rx_bandwidth new_sta_bw; unsigned int link_id; if (!ieee80211_sdata_running(sta->sdata)) continue; - for (link_id = 0; link_id < ARRAY_SIZE(sta->sdata->link); link_id++) { + sdata = get_bss_sdata(sta->sdata); + + for (link_id = 0; link_id < ARRAY_SIZE(sdata->link); link_id++) { struct ieee80211_link_data *link = rcu_dereference(sdata->link[link_id]); struct ieee80211_bss_conf *link_conf; diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c index d02f07368c511..687a66cd49433 100644 --- a/net/mac80211/debugfs.c +++ b/net/mac80211/debugfs.c @@ -320,7 +320,6 @@ static ssize_t aql_enable_read(struct file *file, char __user *user_buf, static ssize_t aql_enable_write(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { - bool aql_disabled = static_key_false(&aql_disable.key); char buf[3]; size_t len; @@ -335,15 +334,12 @@ static ssize_t aql_enable_write(struct file *file, const char __user *user_buf, if (len > 0 && buf[len - 1] == '\n') buf[len - 1] = 0; - if (buf[0] == '0' && buf[1] == '\0') { - if (!aql_disabled) - static_branch_inc(&aql_disable); - } else if (buf[0] == '1' && buf[1] == '\0') { - if (aql_disabled) - static_branch_dec(&aql_disable); - } else { + if (buf[0] == '0' && buf[1] == '\0') + static_branch_enable(&aql_disable); + else if (buf[0] == '1' && buf[1] == '\0') + static_branch_disable(&aql_disable); + else return -EINVAL; - } return count; } diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 878c3b14aeb80..5c0c833fcf7a9 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -451,8 +451,6 @@ struct ieee80211_mgd_assoc_data { struct ieee80211_conn_settings conn; u16 status; - - bool disabled; } link[IEEE80211_MLD_MAX_NUM_LINKS]; u8 ap_addr[ETH_ALEN] __aligned(2); diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 0ca55b9655a7f..72c129478da08 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -350,6 +350,8 @@ static int ieee80211_check_concurrent_iface(struct ieee80211_sub_if_data *sdata, /* we hold the RTNL here so can safely walk the list */ list_for_each_entry(nsdata, &local->interfaces, list) { if (nsdata != sdata && ieee80211_sdata_running(nsdata)) { + struct ieee80211_link_data *link; + /* * Only OCB and monitor mode may coexist */ @@ -376,8 +378,10 @@ static int ieee80211_check_concurrent_iface(struct ieee80211_sub_if_data *sdata, * will not add another interface while any channel * switch is active. */ - if (nsdata->vif.bss_conf.csa_active) - return -EBUSY; + for_each_link_data(nsdata, link) { + if (link->conf->csa_active) + return -EBUSY; + } /* * The remaining checks are only performed for interfaces diff --git a/net/mac80211/key.c b/net/mac80211/key.c index d5da7ccea66e0..04c8809173d7f 100644 --- a/net/mac80211/key.c +++ b/net/mac80211/key.c @@ -987,7 +987,8 @@ void ieee80211_reenable_keys(struct ieee80211_sub_if_data *sdata) if (ieee80211_sdata_running(sdata)) { list_for_each_entry(key, &sdata->key_list, list) { - increment_tailroom_need_count(sdata); + if (!(key->flags & KEY_FLAG_TAINTED)) + increment_tailroom_need_count(sdata); ieee80211_key_enable_hw_accel(key); } } diff --git a/net/mac80211/link.c b/net/mac80211/link.c index 4a19b765ccb69..05b0472bda40e 100644 --- a/net/mac80211/link.c +++ b/net/mac80211/link.c @@ -286,6 +286,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]; struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS]; bool use_deflink = old_links == 0; /* set for error case */ + bool non_sta = sdata->vif.type != NL80211_IFTYPE_STATION; lockdep_assert_wiphy(sdata->local->hw.wiphy); @@ -342,6 +343,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, link = links[link_id]; ieee80211_link_init(sdata, link_id, &link->data, &link->conf); ieee80211_link_setup(&link->data); + ieee80211_set_wmm_default(&link->data, true, non_sta); } if (new_links == 0) diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c index f37068a533f4e..4b0eebd5c7cf8 100644 --- a/net/mac80211/mesh.c +++ b/net/mac80211/mesh.c @@ -79,6 +79,9 @@ bool mesh_matches_local(struct ieee80211_sub_if_data *sdata, * - MDA enabled * - Power management control on fc */ + if (!ie->mesh_config) + return false; + if (!(ifmsh->mesh_id_len == ie->mesh_id_len && memcmp(ifmsh->mesh_id, ie->mesh_id, ie->mesh_id_len) == 0 && (ifmsh->mesh_pp_id == ie->mesh_config->meshconf_psel) && @@ -1629,6 +1632,9 @@ static void mesh_rx_csa_frame(struct ieee80211_sub_if_data *sdata, if (!mesh_matches_local(sdata, elems)) goto free; + if (!elems->mesh_chansw_params_ie) + goto free; + ifmsh->chsw_ttl = elems->mesh_chansw_params_ie->mesh_ttl; if (!--ifmsh->chsw_ttl) fwd_csa = false; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index f3138d1585353..f119149bcc1c1 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -8,7 +8,7 @@ * Copyright 2007, Michael Wu * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2015 - 2017 Intel Deutschland GmbH - * Copyright (C) 2018 - 2025 Intel Corporation + * Copyright (C) 2018 - 2026 Intel Corporation */ #include @@ -1126,7 +1126,10 @@ ieee80211_determine_chan_mode(struct ieee80211_sub_if_data *sdata, while (!ieee80211_chandef_usable(sdata, &chanreq->oper, IEEE80211_CHAN_DISABLED)) { - if (WARN_ON(chanreq->oper.width == NL80211_CHAN_WIDTH_20_NOHT)) { + if (chanreq->oper.width == NL80211_CHAN_WIDTH_20_NOHT) { + link_id_info(sdata, link_id, + "unusable channel (%d MHz) for connection\n", + chanreq->oper.chan->center_freq); ret = -EINVAL; goto free; } @@ -6161,6 +6164,100 @@ static bool ieee80211_get_dtim(const struct cfg80211_bss_ies *ies, return true; } +static u16 ieee80211_get_ttlm(u8 bm_size, u8 *data) +{ + if (bm_size == 1) + return *data; + + return get_unaligned_le16(data); +} + +static int +ieee80211_parse_adv_t2l(struct ieee80211_sub_if_data *sdata, + const struct ieee80211_ttlm_elem *ttlm, + struct ieee80211_adv_ttlm_info *ttlm_info) +{ + /* The element size was already validated in + * ieee80211_tid_to_link_map_size_ok() + */ + u8 control, link_map_presence, map_size, tid; + u8 *pos; + + memset(ttlm_info, 0, sizeof(*ttlm_info)); + pos = (void *)ttlm->optional; + control = ttlm->control; + + if ((control & IEEE80211_TTLM_CONTROL_DIRECTION) != + IEEE80211_TTLM_DIRECTION_BOTH) { + sdata_info(sdata, "Invalid advertised T2L map direction\n"); + return -EINVAL; + } + + if (!(control & IEEE80211_TTLM_CONTROL_DEF_LINK_MAP)) { + link_map_presence = *pos; + pos++; + } + + if (control & IEEE80211_TTLM_CONTROL_SWITCH_TIME_PRESENT) { + ttlm_info->switch_time = get_unaligned_le16(pos); + + /* Since ttlm_info->switch_time == 0 means no switch time, bump + * it by 1. + */ + if (!ttlm_info->switch_time) + ttlm_info->switch_time = 1; + + pos += 2; + } + + if (control & IEEE80211_TTLM_CONTROL_EXPECTED_DUR_PRESENT) { + ttlm_info->duration = pos[0] | pos[1] << 8 | pos[2] << 16; + pos += 3; + } + + if (control & IEEE80211_TTLM_CONTROL_DEF_LINK_MAP) { + ttlm_info->map = 0xffff; + return 0; + } + + if (control & IEEE80211_TTLM_CONTROL_LINK_MAP_SIZE) + map_size = 1; + else + map_size = 2; + + /* According to Draft P802.11be_D3.0 clause 35.3.7.1.7, an AP MLD shall + * not advertise a TID-to-link mapping that does not map all TIDs to the + * same link set, reject frame if not all links have mapping + */ + if (link_map_presence != 0xff) { + sdata_info(sdata, + "Invalid advertised T2L mapping presence indicator\n"); + return -EINVAL; + } + + ttlm_info->map = ieee80211_get_ttlm(map_size, pos); + if (!ttlm_info->map) { + sdata_info(sdata, + "Invalid advertised T2L map for TID 0\n"); + return -EINVAL; + } + + pos += map_size; + + for (tid = 1; tid < 8; tid++) { + u16 map = ieee80211_get_ttlm(map_size, pos); + + if (map != ttlm_info->map) { + sdata_info(sdata, "Invalid advertised T2L map for tid %d\n", + tid); + return -EINVAL; + } + + pos += map_size; + } + return 0; +} + static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, struct ieee80211_mgmt *mgmt, struct ieee802_11_elems *elems, @@ -6192,8 +6289,6 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, continue; valid_links |= BIT(link_id); - if (assoc_data->link[link_id].disabled) - dormant_links |= BIT(link_id); if (link_id != assoc_data->assoc_link_id) { err = ieee80211_sta_allocate_link(sta, link_id); @@ -6202,6 +6297,33 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, } } + /* + * We do not support setting a negotiated TTLM during + * association. As such, we can assume that if there is a TTLM, + * then it is the currently active advertised TTLM. + * In that case, there must be exactly one TTLM that does not + * have a switch time set. This mapping should also leave us + * with at least one usable link. + */ + if (elems->ttlm_num > 1) { + sdata_info(sdata, + "More than one advertised TTLM in association response\n"); + goto out_err; + } else if (elems->ttlm_num == 1) { + if (ieee80211_parse_adv_t2l(sdata, elems->ttlm[0], + &sdata->u.mgd.ttlm_info) || + sdata->u.mgd.ttlm_info.switch_time != 0 || + !(valid_links & sdata->u.mgd.ttlm_info.map)) { + sdata_info(sdata, + "Invalid advertised TTLM in association response\n"); + goto out_err; + } + + sdata->u.mgd.ttlm_info.active = true; + dormant_links = + valid_links & ~sdata->u.mgd.ttlm_info.map; + } + ieee80211_vif_set_links(sdata, valid_links, dormant_links); } @@ -6855,6 +6977,9 @@ static void ieee80211_ml_reconfiguration(struct ieee80211_sub_if_data *sdata, control = le16_to_cpu(prof->control); link_id = control & IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID; + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + continue; + removed_links |= BIT(link_id); /* the MAC address should not be included, but handle it */ @@ -6991,95 +7116,6 @@ static void ieee80211_tid_to_link_map_work(struct wiphy *wiphy, sdata->u.mgd.ttlm_info.switch_time = 0; } -static u16 ieee80211_get_ttlm(u8 bm_size, u8 *data) -{ - if (bm_size == 1) - return *data; - else - return get_unaligned_le16(data); -} - -static int -ieee80211_parse_adv_t2l(struct ieee80211_sub_if_data *sdata, - const struct ieee80211_ttlm_elem *ttlm, - struct ieee80211_adv_ttlm_info *ttlm_info) -{ - /* The element size was already validated in - * ieee80211_tid_to_link_map_size_ok() - */ - u8 control, link_map_presence, map_size, tid; - u8 *pos; - - memset(ttlm_info, 0, sizeof(*ttlm_info)); - pos = (void *)ttlm->optional; - control = ttlm->control; - - if ((control & IEEE80211_TTLM_CONTROL_DEF_LINK_MAP) || - !(control & IEEE80211_TTLM_CONTROL_SWITCH_TIME_PRESENT)) - return 0; - - if ((control & IEEE80211_TTLM_CONTROL_DIRECTION) != - IEEE80211_TTLM_DIRECTION_BOTH) { - sdata_info(sdata, "Invalid advertised T2L map direction\n"); - return -EINVAL; - } - - link_map_presence = *pos; - pos++; - - ttlm_info->switch_time = get_unaligned_le16(pos); - - /* Since ttlm_info->switch_time == 0 means no switch time, bump it - * by 1. - */ - if (!ttlm_info->switch_time) - ttlm_info->switch_time = 1; - - pos += 2; - - if (control & IEEE80211_TTLM_CONTROL_EXPECTED_DUR_PRESENT) { - ttlm_info->duration = pos[0] | pos[1] << 8 | pos[2] << 16; - pos += 3; - } - - if (control & IEEE80211_TTLM_CONTROL_LINK_MAP_SIZE) - map_size = 1; - else - map_size = 2; - - /* According to Draft P802.11be_D3.0 clause 35.3.7.1.7, an AP MLD shall - * not advertise a TID-to-link mapping that does not map all TIDs to the - * same link set, reject frame if not all links have mapping - */ - if (link_map_presence != 0xff) { - sdata_info(sdata, - "Invalid advertised T2L mapping presence indicator\n"); - return -EINVAL; - } - - ttlm_info->map = ieee80211_get_ttlm(map_size, pos); - if (!ttlm_info->map) { - sdata_info(sdata, - "Invalid advertised T2L map for TID 0\n"); - return -EINVAL; - } - - pos += map_size; - - for (tid = 1; tid < 8; tid++) { - u16 map = ieee80211_get_ttlm(map_size, pos); - - if (map != ttlm_info->map) { - sdata_info(sdata, "Invalid advertised T2L map for tid %d\n", - tid); - return -EINVAL; - } - - pos += map_size; - } - return 0; -} - static void ieee80211_process_adv_ttlm(struct ieee80211_sub_if_data *sdata, struct ieee802_11_elems *elems, u64 beacon_ts) @@ -9726,7 +9762,6 @@ int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata, req, true, i, &assoc_data->link[i].conn); assoc_data->link[i].bss = link_cbss; - assoc_data->link[i].disabled = req->links[i].disabled; if (!bss->uapsd_supported) uapsd_supported = false; @@ -10708,8 +10743,6 @@ int ieee80211_mgd_assoc_ml_reconf(struct ieee80211_sub_if_data *sdata, &data->link[link_id].conn); data->link[link_id].bss = link_cbss; - data->link[link_id].disabled = - req->add_links[link_id].disabled; data->link[link_id].elems = (u8 *)req->add_links[link_id].elems; data->link[link_id].elems_len = diff --git a/net/mac80211/ocb.c b/net/mac80211/ocb.c index a5d4358f122ae..ebb4f4d88c237 100644 --- a/net/mac80211/ocb.c +++ b/net/mac80211/ocb.c @@ -47,6 +47,9 @@ void ieee80211_ocb_rx_no_sta(struct ieee80211_sub_if_data *sdata, struct sta_info *sta; int band; + if (!ifocb->joined) + return; + /* XXX: Consider removing the least recently used entry and * allow new one to be added. */ diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 5b4c3fe9970ad..5091a3c15b0b4 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -3428,6 +3428,11 @@ ieee80211_rx_h_mgmt_check(struct ieee80211_rx_data *rx) rx->skb->len < IEEE80211_MIN_ACTION_SIZE) return RX_DROP_U_RUNT_ACTION; + /* Drop non-broadcast Beacon frames */ + if (ieee80211_is_beacon(mgmt->frame_control) && + !is_broadcast_ether_addr(mgmt->da)) + return RX_DROP; + if (rx->sdata->vif.type == NL80211_IFTYPE_AP && ieee80211_is_beacon(mgmt->frame_control) && !(rx->flags & IEEE80211_RX_BEACON_REPORTED)) { diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c index bb9563f50e7b4..1e06a465b49e3 100644 --- a/net/mac80211/scan.c +++ b/net/mac80211/scan.c @@ -343,8 +343,13 @@ void ieee80211_scan_rx(struct ieee80211_local *local, struct sk_buff *skb) mgmt->da)) return; } else { - /* Beacons are expected only with broadcast address */ - if (!is_broadcast_ether_addr(mgmt->da)) + /* + * Non-S1G beacons are expected only with broadcast address. + * S1G beacons only carry the SA so no DA check is required + * nor possible. + */ + if (!ieee80211_is_s1g_beacon(mgmt->frame_control) && + !is_broadcast_ether_addr(mgmt->da)) return; } diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index f4d3b67fda062..b0d9bb830f293 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -1533,6 +1533,10 @@ static void __sta_info_destroy_part2(struct sta_info *sta, bool recalc) } } + sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL); + if (sinfo) + sta_set_sinfo(sta, sinfo, true); + if (sta->uploaded) { ret = drv_sta_state(local, sdata, sta, IEEE80211_STA_NONE, IEEE80211_STA_NOTEXIST); @@ -1541,9 +1545,6 @@ static void __sta_info_destroy_part2(struct sta_info *sta, bool recalc) sta_dbg(sdata, "Removed STA %pM\n", sta->sta.addr); - sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL); - if (sinfo) - sta_set_sinfo(sta, sinfo, true); cfg80211_del_sta_sinfo(sdata->dev, sta->sta.addr, sinfo, GFP_KERNEL); kfree(sinfo); @@ -2758,7 +2759,9 @@ static void sta_set_link_sinfo(struct sta_info *sta, } link_sinfo->inactive_time = - jiffies_to_msecs(jiffies - ieee80211_sta_last_active(sta, link_id)); + jiffies_delta_to_msecs(jiffies - + ieee80211_sta_last_active(sta, + link_id)); if (!(link_sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | BIT_ULL(NL80211_STA_INFO_TX_BYTES)))) { @@ -2991,7 +2994,8 @@ void sta_set_sinfo(struct sta_info *sta, struct station_info *sinfo, sinfo->connected_time = ktime_get_seconds() - sta->last_connected; sinfo->assoc_at = sta->assoc_at; sinfo->inactive_time = - jiffies_to_msecs(jiffies - ieee80211_sta_last_active(sta, -1)); + jiffies_delta_to_msecs(jiffies - + ieee80211_sta_last_active(sta, -1)); if (!(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | BIT_ULL(NL80211_STA_INFO_TX_BYTES)))) { diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c index ba5fbacbeeda6..1536cd71878c7 100644 --- a/net/mac80211/tdls.c +++ b/net/mac80211/tdls.c @@ -1449,7 +1449,7 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, } sta = sta_info_get(sdata, peer); - if (!sta) + if (!sta || !sta->sta.tdls) return -ENOLINK; iee80211_tdls_recalc_chanctx(sdata, sta); diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index e7b141c55f7a9..2f830001b0cd6 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -1896,8 +1896,10 @@ bool ieee80211_tx_prepare_skb(struct ieee80211_hw *hw, struct ieee80211_tx_data tx; struct sk_buff *skb2; - if (ieee80211_tx_prepare(sdata, &tx, NULL, skb) == TX_DROP) + if (ieee80211_tx_prepare(sdata, &tx, NULL, skb) == TX_DROP) { + kfree_skb(skb); return false; + } info->band = band; info->control.vif = vif; @@ -2395,6 +2397,8 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, if (chanctx_conf) chandef = &chanctx_conf->def; + else if (local->emulate_chanctx) + chandef = &local->hw.conf.chandef; else goto fail_rcu; diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c index 40d5d9e484791..bb0fa505cdcae 100644 --- a/net/mac80211/wpa.c +++ b/net/mac80211/wpa.c @@ -869,8 +869,9 @@ ieee80211_crypto_aes_cmac_encrypt(struct ieee80211_tx_data *tx) /* * MIC = AES-128-CMAC(IGTK, AAD || Management Frame Body || MMIE, 64) */ - ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, mmie->mic); + if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, + skb->data + 24, skb->len - 24, mmie->mic)) + return TX_DROP; return TX_CONTINUE; } @@ -916,8 +917,9 @@ ieee80211_crypto_aes_cmac_256_encrypt(struct ieee80211_tx_data *tx) /* MIC = AES-256-CMAC(IGTK, AAD || Management Frame Body || MMIE, 128) */ - ieee80211_aes_cmac_256(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, mmie->mic); + if (ieee80211_aes_cmac_256(key->u.aes_cmac.tfm, aad, + skb->data + 24, skb->len - 24, mmie->mic)) + return TX_DROP; return TX_CONTINUE; } @@ -956,8 +958,9 @@ ieee80211_crypto_aes_cmac_decrypt(struct ieee80211_rx_data *rx) if (!(status->flag & RX_FLAG_DECRYPTED)) { /* hardware didn't decrypt/verify MIC */ bip_aad(skb, aad); - ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, mic); + if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, + skb->data + 24, skb->len - 24, mic)) + return RX_DROP_U_DECRYPT_FAIL; if (crypto_memneq(mic, mmie->mic, sizeof(mmie->mic))) { key->u.aes_cmac.icverrors++; return RX_DROP_U_MIC_FAIL; @@ -1006,8 +1009,9 @@ ieee80211_crypto_aes_cmac_256_decrypt(struct ieee80211_rx_data *rx) if (!(status->flag & RX_FLAG_DECRYPTED)) { /* hardware didn't decrypt/verify MIC */ bip_aad(skb, aad); - ieee80211_aes_cmac_256(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, mic); + if (ieee80211_aes_cmac_256(key->u.aes_cmac.tfm, aad, + skb->data + 24, skb->len - 24, mic)) + return RX_DROP_U_DECRYPT_FAIL; if (crypto_memneq(mic, mmie->mic, sizeof(mmie->mic))) { key->u.aes_cmac.icverrors++; return RX_DROP_U_MIC_FAIL; diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index 9e4631fade90c..000be60d95803 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -469,7 +469,9 @@ static int mac802154_header_create(struct sk_buff *skb, } static int -mac802154_header_parse(const struct sk_buff *skb, unsigned char *haddr) +mac802154_header_parse(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr) { struct ieee802154_hdr hdr; diff --git a/net/mctp/route.c b/net/mctp/route.c index 2ac4011a953ff..bee225c821ed8 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -359,6 +359,7 @@ static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev) { struct mctp_sk_key *key; struct mctp_flow *flow; + unsigned long flags; flow = skb_ext_find(skb, SKB_EXT_MCTP); if (!flow) @@ -366,12 +367,14 @@ static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev) key = flow->key; - if (key->dev) { + spin_lock_irqsave(&key->lock, flags); + + if (!key->dev) + mctp_dev_set_key(dev, key); + else WARN_ON(key->dev != dev); - return; - } - mctp_dev_set_key(dev, key); + spin_unlock_irqrestore(&key->lock, flags); } #else static void mctp_skb_set_flow(struct sk_buff *skb, struct mctp_sk_key *key) {} diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 25c88cba5c48b..1c70cb26e7ba1 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -2777,6 +2777,7 @@ static int __init mpls_init(void) rtnl_af_unregister(&mpls_af_ops); out_unregister_dev_type: dev_remove_pack(&mpls_packet_type); + unregister_netdevice_notifier(&mpls_dev_notifier); out_unregister_pernet: unregister_pernet_subsys(&mpls_net_ops); goto out; diff --git a/net/mptcp/options.c b/net/mptcp/options.c index f24ae7d40e883..43df4293f58bf 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -408,6 +408,16 @@ bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb, */ subflow->snd_isn = TCP_SKB_CB(skb)->end_seq; if (subflow->request_mptcp) { + if (unlikely(subflow_simultaneous_connect(sk))) { + WARN_ON_ONCE(!mptcp_try_fallback(sk, MPTCP_MIB_SIMULTCONNFALLBACK)); + + /* Ensure mptcp_finish_connect() will not process the + * MPC handshake. + */ + subflow->request_mptcp = 0; + return false; + } + opts->suboptions = OPTION_MPTCP_MPC_SYN; opts->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk)); opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk)); diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 9604b91902b8b..e22d53af0ffa0 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -212,9 +212,24 @@ void mptcp_pm_send_ack(struct mptcp_sock *msk, spin_lock_bh(&msk->pm.lock); } -void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) +static bool subflow_in_rm_list(const struct mptcp_subflow_context *subflow, + const struct mptcp_rm_list *rm_list) +{ + u8 i, id = subflow_get_local_id(subflow); + + for (i = 0; i < rm_list->nr; i++) { + if (rm_list->ids[i] == id) + return true; + } + + return false; +} + +static void +mptcp_pm_addr_send_ack_avoid_list(struct mptcp_sock *msk, + const struct mptcp_rm_list *rm_list) { - struct mptcp_subflow_context *subflow, *alt = NULL; + struct mptcp_subflow_context *subflow, *stale = NULL, *same_id = NULL; msk_owned_by_me(msk); lockdep_assert_held(&msk->pm.lock); @@ -224,19 +239,35 @@ void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) return; mptcp_for_each_subflow(msk, subflow) { - if (__mptcp_subflow_active(subflow)) { - if (!subflow->stale) { - mptcp_pm_send_ack(msk, subflow, false, false); - return; - } + if (!__mptcp_subflow_active(subflow)) + continue; - if (!alt) - alt = subflow; + if (unlikely(subflow->stale)) { + if (!stale) + stale = subflow; + } else if (unlikely(rm_list && + subflow_in_rm_list(subflow, rm_list))) { + if (!same_id) + same_id = subflow; + } else { + goto send_ack; } } - if (alt) - mptcp_pm_send_ack(msk, alt, false, false); + if (same_id) + subflow = same_id; + else if (stale) + subflow = stale; + else + return; + +send_ack: + mptcp_pm_send_ack(msk, subflow, false, false); +} + +void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) +{ + mptcp_pm_addr_send_ack_avoid_list(msk, NULL); } int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, @@ -470,7 +501,7 @@ int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_ msk->pm.rm_list_tx = *rm_list; rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL); WRITE_ONCE(msk->pm.addr_signal, rm_addr); - mptcp_pm_addr_send_ack(msk); + mptcp_pm_addr_send_ack_avoid_list(msk, rm_list); return 0; } diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c index 0a50fd5edc06d..4494cf4c38282 100644 --- a/net/mptcp/pm_kernel.c +++ b/net/mptcp/pm_kernel.c @@ -407,6 +407,15 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) } exit: + /* If an endpoint has both the signal and subflow flags, but it is not + * possible to create subflows -- the 'while' loop body above never + * executed -- then still mark the endp as used, which is somehow the + * case. This avoids issues later when removing the endpoint and calling + * __mark_subflow_endp_available(), which expects the increment here. + */ + if (signal_and_subflow && local.addr.id != msk->mpc_endpoint_id) + msk->pm.local_addr_used++; + mptcp_pm_nl_check_work_pending(msk); } @@ -701,7 +710,7 @@ static void __mptcp_pm_release_addr_entry(struct mptcp_pm_addr_entry *entry) static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, struct mptcp_pm_addr_entry *entry, - bool needs_id, bool replace) + bool replace) { struct mptcp_pm_addr_entry *cur, *del_entry = NULL; int ret = -EINVAL; @@ -760,7 +769,7 @@ static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, } } - if (!entry->addr.id && needs_id) { + if (!entry->addr.id) { find_next: entry->addr.id = find_next_zero_bit(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1, @@ -771,7 +780,7 @@ static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, } } - if (!entry->addr.id && needs_id) + if (!entry->addr.id) goto out; __set_bit(entry->addr.id, pernet->id_bitmap); @@ -815,7 +824,7 @@ static struct lock_class_key mptcp_keys[2]; static int mptcp_pm_nl_create_listen_socket(struct sock *sk, struct mptcp_pm_addr_entry *entry) { - bool is_ipv6 = sk->sk_family == AF_INET6; + bool is_ipv6 = entry->addr.family == AF_INET6; int addrlen = sizeof(struct sockaddr_in); struct sockaddr_storage addr; struct sock *newsk, *ssk; @@ -900,7 +909,7 @@ int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, return -ENOMEM; entry->addr.port = 0; - ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true, false); + ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, false); if (ret < 0) kfree(entry); @@ -954,18 +963,6 @@ static int mptcp_nl_add_subflow_or_signal_addr(struct net *net, return 0; } -static bool mptcp_pm_has_addr_attr_id(const struct nlattr *attr, - struct genl_info *info) -{ - struct nlattr *tb[MPTCP_PM_ADDR_ATTR_MAX + 1]; - - if (!nla_parse_nested_deprecated(tb, MPTCP_PM_ADDR_ATTR_MAX, attr, - mptcp_pm_address_nl_policy, info->extack) && - tb[MPTCP_PM_ADDR_ATTR_ID]) - return true; - return false; -} - /* Add an MPTCP endpoint */ int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info) { @@ -1014,9 +1011,7 @@ int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info) goto out_free; } } - ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, - !mptcp_pm_has_addr_attr_id(attr, info), - true); + ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true); if (ret < 0) { GENL_SET_ERR_MSG_FMT(info, "too many addresses or duplicate one: %d", ret); goto out_free; @@ -1042,10 +1037,8 @@ static bool mptcp_pm_remove_anno_addr(struct mptcp_sock *msk, ret = mptcp_remove_anno_list_by_saddr(msk, addr); if (ret || force) { spin_lock_bh(&msk->pm.lock); - if (ret) { - __set_bit(addr->id, msk->pm.id_avail_bitmap); + if (ret) msk->pm.add_addr_signaled--; - } mptcp_pm_remove_addr(msk, &list); spin_unlock_bh(&msk->pm.lock); } @@ -1083,17 +1076,15 @@ static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net, !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)); list.ids[0] = mptcp_endp_get_local_id(msk, addr); - if (remove_subflow) { - spin_lock_bh(&msk->pm.lock); - mptcp_pm_rm_subflow(msk, &list); - spin_unlock_bh(&msk->pm.lock); - } - if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { - spin_lock_bh(&msk->pm.lock); + spin_lock_bh(&msk->pm.lock); + if (remove_subflow) + mptcp_pm_rm_subflow(msk, &list); + if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) __mark_subflow_endp_available(msk, list.ids[0]); - spin_unlock_bh(&msk->pm.lock); - } + else /* mark endp ID as available, e.g. Signal or MPC endp */ + __set_bit(addr->id, msk->pm.id_avail_bitmap); + spin_unlock_bh(&msk->pm.lock); if (msk->mpc_endpoint_id == entry->addr.id) msk->mpc_endpoint_id = 0; @@ -1276,16 +1267,26 @@ static void __reset_counters(struct pm_nl_pernet *pernet) int mptcp_pm_nl_flush_addrs_doit(struct sk_buff *skb, struct genl_info *info) { struct pm_nl_pernet *pernet = genl_info_pm_nl(info); - LIST_HEAD(free_list); + struct list_head free_list; spin_lock_bh(&pernet->lock); - list_splice_init(&pernet->endp_list, &free_list); + free_list = pernet->endp_list; + INIT_LIST_HEAD_RCU(&pernet->endp_list); __reset_counters(pernet); pernet->next_id = 1; bitmap_zero(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); spin_unlock_bh(&pernet->lock); - mptcp_nl_flush_addrs_list(sock_net(skb->sk), &free_list); + + if (free_list.next == &pernet->endp_list) + return 0; + synchronize_rcu(); + + /* Adjust the pointers to free_list instead of pernet->endp_list */ + free_list.prev->next = &free_list; + free_list.next->prev = &free_list; + + mptcp_nl_flush_addrs_list(sock_net(skb->sk), &free_list); __flush_addrs(&free_list); return 0; } diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c index d5b383870f799..7aa42de9c47b5 100644 --- a/net/mptcp/pm_netlink.c +++ b/net/mptcp/pm_netlink.c @@ -119,7 +119,8 @@ int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info, } if (tb[MPTCP_PM_ADDR_ATTR_FLAGS]) - entry->flags = nla_get_u32(tb[MPTCP_PM_ADDR_ATTR_FLAGS]); + entry->flags = nla_get_u32(tb[MPTCP_PM_ADDR_ATTR_FLAGS]) & + MPTCP_PM_ADDR_FLAGS_MASK; if (tb[MPTCP_PM_ADDR_ATTR_PORT]) entry->addr.port = htons(nla_get_u16(tb[MPTCP_PM_ADDR_ATTR_PORT])); diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 1e413426deeec..09e1a93b7daab 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -224,9 +224,6 @@ static bool mptcp_rcvbuf_grow(struct sock *sk, u32 newval) do_div(grow, oldval); rcvwin += grow << 1; - if (!RB_EMPTY_ROOT(&msk->out_of_order_queue)) - rcvwin += MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq - msk->ack_seq; - cap = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]); rcvbuf = min_t(u32, mptcp_space_from_win(sk, rcvwin), cap); @@ -350,9 +347,6 @@ static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) end: skb_condense(skb); skb_set_owner_r(skb, sk); - /* do not grow rcvbuf for not-yet-accepted or orphaned sockets. */ - if (sk->sk_socket) - mptcp_rcvbuf_grow(sk, msk->rcvq_space.space); } static void mptcp_init_skb(struct sock *ssk, struct sk_buff *skb, int offset, @@ -784,11 +778,8 @@ static bool __mptcp_ofo_queue(struct mptcp_sock *msk) static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk) { - int err = sock_error(ssk); int ssk_state; - - if (!err) - return false; + int err; /* only propagate errors on fallen-back sockets or * on MPC connect @@ -796,6 +787,10 @@ static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk) if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk))) return false; + err = sock_error(ssk); + if (!err) + return false; + /* We need to propagate only transition to CLOSE state. * Orphaned socket will see such state change via * subflow_sched_work_if_closed() and that path will properly @@ -1597,7 +1592,7 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags) struct mptcp_sendmsg_info info = { .flags = flags, }; - bool do_check_data_fin = false; + bool copied = false; int push_count = 1; while (mptcp_send_head(sk) && (push_count > 0)) { @@ -1639,7 +1634,7 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags) push_count--; continue; } - do_check_data_fin = true; + copied = true; } } } @@ -1648,11 +1643,14 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags) if (ssk) mptcp_push_release(ssk, &info); - /* ensure the rtx timer is running */ - if (!mptcp_rtx_timer_pending(sk)) - mptcp_reset_rtx_timer(sk); - if (do_check_data_fin) + /* Avoid scheduling the rtx timer if no data has been pushed; the timer + * will be updated on positive acks by __mptcp_cleanup_una(). + */ + if (copied) { + if (!mptcp_rtx_timer_pending(sk)) + mptcp_reset_rtx_timer(sk); mptcp_check_send_data_fin(sk); + } } static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first) @@ -1965,10 +1963,21 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied); +static void mptcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) +{ + /* avoid the indirect call, we know the destructor is sock_rfree */ + skb->destructor = NULL; + skb->sk = NULL; + atomic_sub(skb->truesize, &sk->sk_rmem_alloc); + sk_mem_uncharge(sk, skb->truesize); + __skb_unlink(skb, &sk->sk_receive_queue); + skb_attempt_defer_free(skb); +} + static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, size_t len, int flags, int copied_total, struct scm_timestamping_internal *tss, - int *cmsg_flags) + int *cmsg_flags, struct sk_buff **last) { struct mptcp_sock *msk = mptcp_sk(sk); struct sk_buff *skb, *tmp; @@ -1985,6 +1994,7 @@ static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, /* skip already peeked skbs */ if (total_data_len + data_len <= copied_total) { total_data_len += data_len; + *last = skb; continue; } @@ -2019,13 +2029,9 @@ static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, break; } - /* avoid the indirect call, we know the destructor is sock_rfree */ - skb->destructor = NULL; - skb->sk = NULL; - atomic_sub(skb->truesize, &sk->sk_rmem_alloc); - sk_mem_uncharge(sk, skb->truesize); - __skb_unlink(skb, &sk->sk_receive_queue); - skb_attempt_defer_free(skb); + mptcp_eat_recv_skb(sk, skb); + } else { + *last = skb; } if (copied >= len) @@ -2058,8 +2064,8 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) msk->rcvq_space.copied += copied; - mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); - time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); + mstamp = mptcp_stamp(); + time = tcp_stamp_us_delta(mstamp, READ_ONCE(msk->rcvq_space.time)); rtt_us = msk->rcvq_space.rtt_us; if (rtt_us && time < (rtt_us >> 3)) @@ -2220,10 +2226,12 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, cmsg_flags = MPTCP_CMSG_INQ; while (copied < len) { + struct sk_buff *last = NULL; int err, bytes_read; bytes_read = __mptcp_recvmsg_mskq(sk, msg, len - copied, flags, - copied, &tss, &cmsg_flags); + copied, &tss, &cmsg_flags, + &last); if (unlikely(bytes_read < 0)) { if (!copied) copied = bytes_read; @@ -2275,7 +2283,7 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, pr_debug("block timeout %ld\n", timeo); mptcp_cleanup_rbuf(msk, copied); - err = sk_wait_data(sk, &timeo, NULL); + err = sk_wait_data(sk, &timeo, last); if (err < 0) { err = copied ? : err; goto out_err; @@ -2418,10 +2426,10 @@ bool __mptcp_retransmit_pending_data(struct sock *sk) */ static void __mptcp_subflow_disconnect(struct sock *ssk, struct mptcp_subflow_context *subflow, - unsigned int flags) + bool fastclosing) { if (((1 << ssk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) || - subflow->send_fastclose) { + fastclosing) { /* The MPTCP code never wait on the subflow sockets, TCP-level * disconnect should never fail */ @@ -2473,7 +2481,7 @@ static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); if (!dispose_it) { - __mptcp_subflow_disconnect(ssk, subflow, flags); + __mptcp_subflow_disconnect(ssk, subflow, msk->fastclosing); release_sock(ssk); goto out; @@ -2530,8 +2538,8 @@ static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, void mptcp_close_ssk(struct sock *sk, struct sock *ssk, struct mptcp_subflow_context *subflow) { - /* The first subflow can already be closed and still in the list */ - if (subflow->close_event_done) + /* The first subflow can already be closed or disconnected */ + if (subflow->close_event_done || READ_ONCE(subflow->local_id) < 0) return; subflow->close_event_done = true; @@ -2690,10 +2698,13 @@ static void __mptcp_retrans(struct sock *sk) /* * make the whole retrans decision, xmit, disallow - * fallback atomic + * fallback atomic, note that we can't retrans even + * when an infinite fallback is in progress, i.e. new + * subflows are disallowed. */ spin_lock_bh(&msk->fallback_lock); - if (__mptcp_check_fallback(msk)) { + if (__mptcp_check_fallback(msk) || + !msk->allow_subflows) { spin_unlock_bh(&msk->fallback_lock); release_sock(ssk); goto clear_scheduled; @@ -2783,6 +2794,7 @@ static void mptcp_do_fastclose(struct sock *sk) struct mptcp_sock *msk = mptcp_sk(sk); mptcp_set_state(sk, TCP_CLOSE); + msk->fastclosing = 1; /* Explicitly send the fastclose reset as need */ if (__mptcp_check_fallback(msk)) @@ -3293,6 +3305,7 @@ static int mptcp_disconnect(struct sock *sk, int flags) msk->bytes_sent = 0; msk->bytes_retrans = 0; msk->rcvspace_init = 0; + msk->fastclosing = 0; WRITE_ONCE(sk->sk_shutdown, 0); sk_error_report(sk); @@ -3418,6 +3431,7 @@ struct sock *mptcp_sk_clone_init(const struct sock *sk, __mptcp_propagate_sndbuf(nsk, ssk); mptcp_rcv_space_init(msk, ssk); + msk->rcvq_space.time = mptcp_stamp(); if (mp_opt->suboptions & OPTION_MPTCP_MPC_ACK) __mptcp_subflow_fully_established(msk, subflow, mp_opt); @@ -3435,8 +3449,6 @@ void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) msk->rcvq_space.copied = 0; msk->rcvq_space.rtt_us = 0; - msk->rcvq_space.time = tp->tcp_mstamp; - /* initial rcv_space offering made to peer */ msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, TCP_INIT_CWND * tp->advmss); @@ -3649,6 +3661,7 @@ void mptcp_finish_connect(struct sock *ssk) * accessing the field below */ WRITE_ONCE(msk->local_key, subflow->local_key); + WRITE_ONCE(msk->rcvq_space.time, mptcp_stamp()); mptcp_pm_new_connection(msk, ssk, 0); } @@ -4258,6 +4271,8 @@ int __init mptcp_proto_v6_init(void) { int err; + mptcp_subflow_v6_init(); + mptcp_v6_prot = mptcp_prot; strscpy(mptcp_v6_prot.name, "MPTCPv6", sizeof(mptcp_v6_prot.name)); mptcp_v6_prot.slab = NULL; diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 6ca97096607c3..5a03c8824ab68 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -321,7 +321,8 @@ struct mptcp_sock { fastopening:1, in_accept_queue:1, free_first:1, - rcvspace_init:1; + rcvspace_init:1, + fastclosing:1; u32 notsent_lowat; int keepalive_cnt; int keepalive_idle; @@ -829,6 +830,7 @@ static inline void mptcp_subflow_tcp_fallback(struct sock *sk, void __init mptcp_proto_init(void); #if IS_ENABLED(CONFIG_MPTCP_IPV6) int __init mptcp_proto_v6_init(void); +void __init mptcp_subflow_v6_init(void); #endif struct sock *mptcp_sk_clone_init(const struct sock *sk, @@ -869,6 +871,11 @@ static inline bool mptcp_is_fully_established(struct sock *sk) READ_ONCE(mptcp_sk(sk)->fully_established); } +static inline u64 mptcp_stamp(void) +{ + return div_u64(tcp_clock_ns(), NSEC_PER_USEC); +} + void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk); void mptcp_data_ready(struct sock *sk, struct sock *ssk); bool mptcp_finish_join(struct sock *sk); @@ -1293,10 +1300,8 @@ static inline bool subflow_simultaneous_connect(struct sock *sk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); - return (1 << sk->sk_state) & - (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING) && - is_active_ssk(subflow) && - !subflow->conn_finished; + /* Note that the sk state implies !subflow->conn_finished. */ + return sk->sk_state == TCP_SYN_RECV && is_active_ssk(subflow); } #ifdef CONFIG_SYN_COOKIES diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index af707ce0f6244..90927f6cad26a 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -807,7 +807,9 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); struct mptcp_subflow_request_sock *subflow_req; @@ -854,7 +856,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, create_child: child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, - req_unhash, own_req); + req_unhash, own_req, opt_child_init); if (child && *own_req) { struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); @@ -1856,12 +1858,6 @@ static void subflow_state_change(struct sock *sk) __subflow_state_change(sk); - if (subflow_simultaneous_connect(sk)) { - WARN_ON_ONCE(!mptcp_try_fallback(sk, MPTCP_MIB_SIMULTCONNFALLBACK)); - subflow->conn_finished = 1; - mptcp_propagate_state(parent, sk, subflow, NULL); - } - /* as recvmsg() does not acquire the subflow socket for ssk selection * a fin packet carrying a DSS can be unnoticed if we don't trigger * the data available machinery here. @@ -2149,7 +2145,15 @@ void __init mptcp_subflow_init(void) tcp_prot_override.psock_update_sk_prot = NULL; #endif + mptcp_diag_subflow_init(&subflow_ulp_ops); + + if (tcp_register_ulp(&subflow_ulp_ops) != 0) + panic("MPTCP: failed to register subflows to ULP\n"); +} + #if IS_ENABLED(CONFIG_MPTCP_IPV6) +void __init mptcp_subflow_v6_init(void) +{ /* In struct mptcp_subflow_request_sock, we assume the TCP request sock * structures for v4 and v6 have the same size. It should not changed in * the future but better to make sure to be warned if it is no longer @@ -2188,10 +2192,5 @@ void __init mptcp_subflow_init(void) /* Disable sockmap processing for subflows */ tcpv6_prot_override.psock_update_sk_prot = NULL; #endif -#endif - - mptcp_diag_subflow_init(&subflow_ulp_ops); - - if (tcp_register_ulp(&subflow_ulp_ops) != 0) - panic("MPTCP: failed to register subflows to ULP\n"); } +#endif diff --git a/net/ncsi/ncsi-aen.c b/net/ncsi/ncsi-aen.c index 62fb1031763d1..040a31557201b 100644 --- a/net/ncsi/ncsi-aen.c +++ b/net/ncsi/ncsi-aen.c @@ -224,7 +224,8 @@ int ncsi_aen_handler(struct ncsi_dev_priv *ndp, struct sk_buff *skb) if (!nah) { netdev_warn(ndp->ndev.dev, "Invalid AEN (0x%x) received\n", h->type); - return -ENOENT; + ret = -ENOENT; + goto out; } ret = ncsi_validate_aen_pkt(h, nah->payload); diff --git a/net/ncsi/ncsi-rsp.c b/net/ncsi/ncsi-rsp.c index 271ec6c3929e8..fbd84bc8026a3 100644 --- a/net/ncsi/ncsi-rsp.c +++ b/net/ncsi/ncsi-rsp.c @@ -1176,8 +1176,10 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, /* Find the NCSI device */ nd = ncsi_find_dev(orig_dev); ndp = nd ? TO_NCSI_DEV_PRIV(nd) : NULL; - if (!ndp) - return -ENODEV; + if (!ndp) { + ret = -ENODEV; + goto err_free_skb; + } /* Check if it is AEN packet */ hdr = (struct ncsi_pkt_hdr *)skb_network_header(skb); @@ -1199,7 +1201,8 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, if (!nrh) { netdev_err(nd->dev, "Received unrecognized packet (0x%x)\n", hdr->type); - return -ENOENT; + ret = -ENOENT; + goto err_free_skb; } /* Associate with the request */ @@ -1207,7 +1210,8 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, nr = &ndp->requests[hdr->id]; if (!nr->used) { spin_unlock_irqrestore(&ndp->lock, flags); - return -ENODEV; + ret = -ENODEV; + goto err_free_skb; } nr->rsp = skb; @@ -1261,4 +1265,8 @@ int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev, out: ncsi_free_request(nr); return ret; + +err_free_skb: + kfree_skb(skb); + return ret; } diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index cc20e6d56807c..a4e1d7951b2c6 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -821,7 +821,7 @@ EXPORT_SYMBOL_GPL(ip_set_del); * */ ip_set_id_t -ip_set_get_byname(struct net *net, const char *name, struct ip_set **set) +ip_set_get_byname(struct net *net, const struct nlattr *name, struct ip_set **set) { ip_set_id_t i, index = IPSET_INVALID_ID; struct ip_set *s; @@ -830,7 +830,7 @@ ip_set_get_byname(struct net *net, const char *name, struct ip_set **set) rcu_read_lock(); for (i = 0; i < inst->ip_set_max; i++) { s = rcu_dereference(inst->ip_set_list)[i]; - if (s && STRNCMP(s->name, name)) { + if (s && nla_strcmp(name, s->name) == 0) { __ip_set_get(s); index = i; *set = s; diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index 5e4453e9ef8e7..4e56269efef28 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -1099,7 +1099,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext, if (!test_bit(i, n->used)) k++; } - if (n->pos == 0 && k == 0) { + if (k == n->pos) { t->hregion[r].ext_size -= ext_size(n->size, dsize); rcu_assign_pointer(hbucket(t, key), NULL); kfree_rcu(n, rcu); diff --git a/net/netfilter/ipset/ip_set_list_set.c b/net/netfilter/ipset/ip_set_list_set.c index 13c7a08aa868c..34bb84d7b174c 100644 --- a/net/netfilter/ipset/ip_set_list_set.c +++ b/net/netfilter/ipset/ip_set_list_set.c @@ -367,7 +367,7 @@ list_set_uadt(struct ip_set *set, struct nlattr *tb[], ret = ip_set_get_extensions(set, tb, &ext); if (ret) return ret; - e.id = ip_set_get_byname(map->net, nla_data(tb[IPSET_ATTR_NAME]), &s); + e.id = ip_set_get_byname(map->net, tb[IPSET_ATTR_NAME], &s); if (e.id == IPSET_INVALID_ID) return -IPSET_ERR_NAME; /* "Loop detection" */ @@ -389,7 +389,7 @@ list_set_uadt(struct ip_set *set, struct nlattr *tb[], if (tb[IPSET_ATTR_NAMEREF]) { e.refid = ip_set_get_byname(map->net, - nla_data(tb[IPSET_ATTR_NAMEREF]), + tb[IPSET_ATTR_NAMEREF], &s); if (e.refid == IPSET_INVALID_ID) { ret = -IPSET_ERR_NAMEREF; diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index 4c8fa22be88ad..e442ba6033d5f 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -1453,7 +1453,6 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u, ret = ip_vs_bind_scheduler(svc, sched); if (ret) goto out_err; - sched = NULL; } ret = ip_vs_start_estimator(ipvs, &svc->stats); diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index 95af252b29397..ecbcdc43263d6 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -295,6 +295,12 @@ static inline bool decrement_ttl(struct netns_ipvs *ipvs, return true; } +/* rt has device that is down */ +static bool rt_dev_is_down(const struct net_device *dev) +{ + return dev && !netif_running(dev); +} + /* Get route to destination or remote server */ static int __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, @@ -310,9 +316,11 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rtable(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + } else { dest_dst = ip_vs_dest_dst_alloc(); spin_lock_bh(&dest->dst_lock); if (!dest_dst) { @@ -328,14 +336,22 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, ip_vs_dest_dst_free(dest_dst); goto err_unreach; } - __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n", &dest->addr.ip, &dest_dst->dst_saddr.ip, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.ip; } else { noref = 0; @@ -409,6 +425,9 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, return -1; err_unreach: + if (!skb->dev) + skb->dev = skb_dst(skb)->dev; + dst_link_failure(skb); return -1; } @@ -469,9 +488,11 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rt6_info(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + } else { u32 cookie; dest_dst = ip_vs_dest_dst_alloc(); @@ -492,14 +513,22 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, } rt = dst_rt6_info(dst); cookie = rt6_get_cookie(rt); - __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n", &dest->addr.in6, &dest_dst->dst_saddr.in6, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.in6; } else { noref = 0; dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm, diff --git a/net/netfilter/nf_bpf_link.c b/net/netfilter/nf_bpf_link.c index 46e667a50d988..248840dbca1b2 100644 --- a/net/netfilter/nf_bpf_link.c +++ b/net/netfilter/nf_bpf_link.c @@ -170,7 +170,7 @@ static int bpf_nf_link_update(struct bpf_link *link, struct bpf_prog *new_prog, static const struct bpf_link_ops bpf_nf_link_lops = { .release = bpf_nf_link_release, - .dealloc = bpf_nf_link_dealloc, + .dealloc_deferred = bpf_nf_link_dealloc, .detach = bpf_nf_link_detach, .show_fdinfo = bpf_nf_link_show_info, .fill_link_info = bpf_nf_link_fill_link_info, diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index 913ede2f57f9a..14e62b3263cd9 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -34,8 +34,9 @@ #define CONNCOUNT_SLOTS 256U -#define CONNCOUNT_GC_MAX_NODES 8 -#define MAX_KEYLEN 5 +#define CONNCOUNT_GC_MAX_NODES 8 +#define CONNCOUNT_GC_MAX_COLLECT 64 +#define MAX_KEYLEN 5 /* we will save the tuples of all connections we care about */ struct nf_conncount_tuple { @@ -122,32 +123,94 @@ find_or_evict(struct net *net, struct nf_conncount_list *list, return ERR_PTR(-EAGAIN); } +static bool get_ct_or_tuple_from_skb(struct net *net, + const struct sk_buff *skb, + u16 l3num, + struct nf_conn **ct, + struct nf_conntrack_tuple *tuple, + const struct nf_conntrack_zone **zone, + bool *refcounted) +{ + const struct nf_conntrack_tuple_hash *h; + enum ip_conntrack_info ctinfo; + struct nf_conn *found_ct; + + found_ct = nf_ct_get(skb, &ctinfo); + if (found_ct && !nf_ct_is_template(found_ct)) { + *tuple = found_ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; + *zone = nf_ct_zone(found_ct); + *ct = found_ct; + return true; + } + + if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), l3num, net, tuple)) + return false; + + if (found_ct) + *zone = nf_ct_zone(found_ct); + + h = nf_conntrack_find_get(net, *zone, tuple); + if (!h) + return true; + + found_ct = nf_ct_tuplehash_to_ctrack(h); + *refcounted = true; + *ct = found_ct; + + return true; +} + static int __nf_conncount_add(struct net *net, - struct nf_conncount_list *list, - const struct nf_conntrack_tuple *tuple, - const struct nf_conntrack_zone *zone) + const struct sk_buff *skb, + u16 l3num, + struct nf_conncount_list *list) { + const struct nf_conntrack_zone *zone = &nf_ct_zone_dflt; const struct nf_conntrack_tuple_hash *found; struct nf_conncount_tuple *conn, *conn_n; + struct nf_conntrack_tuple tuple; + struct nf_conn *ct = NULL; struct nf_conn *found_ct; unsigned int collect = 0; + bool refcounted = false; + int err = 0; + + if (!get_ct_or_tuple_from_skb(net, skb, l3num, &ct, &tuple, &zone, &refcounted)) + return -ENOENT; + + if (ct && nf_ct_is_confirmed(ct)) { + /* local connections are confirmed in postrouting so confirmation + * might have happened before hitting connlimit + */ + if (skb->skb_iif != LOOPBACK_IFINDEX) { + err = -EEXIST; + goto out_put; + } + + /* this is likely a local connection, skip optimization to avoid + * adding duplicates from a 'packet train' + */ + goto check_connections; + } - if ((u32)jiffies == list->last_gc) + if ((u32)jiffies == list->last_gc && + (list->count - list->last_gc_count) < CONNCOUNT_GC_MAX_COLLECT) goto add_new_node; +check_connections: /* check the saved connections */ list_for_each_entry_safe(conn, conn_n, &list->head, node) { - if (collect > CONNCOUNT_GC_MAX_NODES) + if (collect > CONNCOUNT_GC_MAX_COLLECT) break; found = find_or_evict(net, list, conn); if (IS_ERR(found)) { /* Not found, but might be about to be confirmed */ if (PTR_ERR(found) == -EAGAIN) { - if (nf_ct_tuple_equal(&conn->tuple, tuple) && + if (nf_ct_tuple_equal(&conn->tuple, &tuple) && nf_ct_zone_id(&conn->zone, conn->zone.dir) == nf_ct_zone_id(zone, zone->dir)) - return 0; /* already exists */ + goto out_put; /* already exists */ } else { collect++; } @@ -156,7 +219,7 @@ static int __nf_conncount_add(struct net *net, found_ct = nf_ct_tuplehash_to_ctrack(found); - if (nf_ct_tuple_equal(&conn->tuple, tuple) && + if (nf_ct_tuple_equal(&conn->tuple, &tuple) && nf_ct_zone_equal(found_ct, zone, zone->dir)) { /* * We should not see tuples twice unless someone hooks @@ -165,7 +228,7 @@ static int __nf_conncount_add(struct net *net, * Attempt to avoid a re-add in this case. */ nf_ct_put(found_ct); - return 0; + goto out_put; } else if (already_closed(found_ct)) { /* * we do not care about connections which are @@ -179,53 +242,63 @@ static int __nf_conncount_add(struct net *net, nf_ct_put(found_ct); } + list->last_gc = (u32)jiffies; + list->last_gc_count = list->count; add_new_node: - if (WARN_ON_ONCE(list->count > INT_MAX)) - return -EOVERFLOW; + if (WARN_ON_ONCE(list->count > INT_MAX)) { + err = -EOVERFLOW; + goto out_put; + } conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC); - if (conn == NULL) - return -ENOMEM; + if (conn == NULL) { + err = -ENOMEM; + goto out_put; + } - conn->tuple = *tuple; + conn->tuple = tuple; conn->zone = *zone; conn->cpu = raw_smp_processor_id(); conn->jiffies32 = (u32)jiffies; list_add_tail(&conn->node, &list->head); list->count++; - list->last_gc = (u32)jiffies; - return 0; + +out_put: + if (refcounted) + nf_ct_put(ct); + return err; } -int nf_conncount_add(struct net *net, - struct nf_conncount_list *list, - const struct nf_conntrack_tuple *tuple, - const struct nf_conntrack_zone *zone) +int nf_conncount_add_skb(struct net *net, + const struct sk_buff *skb, + u16 l3num, + struct nf_conncount_list *list) { int ret; /* check the saved connections */ spin_lock_bh(&list->list_lock); - ret = __nf_conncount_add(net, list, tuple, zone); + ret = __nf_conncount_add(net, skb, l3num, list); spin_unlock_bh(&list->list_lock); return ret; } -EXPORT_SYMBOL_GPL(nf_conncount_add); +EXPORT_SYMBOL_GPL(nf_conncount_add_skb); void nf_conncount_list_init(struct nf_conncount_list *list) { spin_lock_init(&list->list_lock); INIT_LIST_HEAD(&list->head); list->count = 0; + list->last_gc_count = 0; list->last_gc = (u32)jiffies; } EXPORT_SYMBOL_GPL(nf_conncount_list_init); /* Return true if the list is empty. Must be called with BH disabled. */ -bool nf_conncount_gc_list(struct net *net, - struct nf_conncount_list *list) +static bool __nf_conncount_gc_list(struct net *net, + struct nf_conncount_list *list) { const struct nf_conntrack_tuple_hash *found; struct nf_conncount_tuple *conn, *conn_n; @@ -237,10 +310,6 @@ bool nf_conncount_gc_list(struct net *net, if ((u32)jiffies == READ_ONCE(list->last_gc)) return false; - /* don't bother if other cpu is already doing GC */ - if (!spin_trylock(&list->list_lock)) - return false; - list_for_each_entry_safe(conn, conn_n, &list->head, node) { found = find_or_evict(net, list, conn); if (IS_ERR(found)) { @@ -262,14 +331,29 @@ bool nf_conncount_gc_list(struct net *net, } nf_ct_put(found_ct); - if (collected > CONNCOUNT_GC_MAX_NODES) + if (collected > CONNCOUNT_GC_MAX_COLLECT) break; } if (!list->count) ret = true; list->last_gc = (u32)jiffies; - spin_unlock(&list->list_lock); + list->last_gc_count = list->count; + + return ret; +} + +bool nf_conncount_gc_list(struct net *net, + struct nf_conncount_list *list) +{ + bool ret; + + /* don't bother if other cpu is already doing GC */ + if (!spin_trylock_bh(&list->list_lock)) + return false; + + ret = __nf_conncount_gc_list(net, list); + spin_unlock_bh(&list->list_lock); return ret; } @@ -309,19 +393,22 @@ static void schedule_gc_worker(struct nf_conncount_data *data, int tree) static unsigned int insert_tree(struct net *net, + const struct sk_buff *skb, + u16 l3num, struct nf_conncount_data *data, struct rb_root *root, unsigned int hash, - const u32 *key, - const struct nf_conntrack_tuple *tuple, - const struct nf_conntrack_zone *zone) + const u32 *key) { struct nf_conncount_rb *gc_nodes[CONNCOUNT_GC_MAX_NODES]; + const struct nf_conntrack_zone *zone = &nf_ct_zone_dflt; + bool do_gc = true, refcounted = false; + unsigned int count = 0, gc_count = 0; struct rb_node **rbnode, *parent; - struct nf_conncount_rb *rbconn; + struct nf_conntrack_tuple tuple; struct nf_conncount_tuple *conn; - unsigned int count = 0, gc_count = 0; - bool do_gc = true; + struct nf_conncount_rb *rbconn; + struct nf_conn *ct = NULL; spin_lock_bh(&nf_conncount_locks[hash]); restart: @@ -340,8 +427,8 @@ insert_tree(struct net *net, } else { int ret; - ret = nf_conncount_add(net, &rbconn->list, tuple, zone); - if (ret) + ret = nf_conncount_add_skb(net, skb, l3num, &rbconn->list); + if (ret && ret != -EEXIST) count = 0; /* hotdrop */ else count = rbconn->list.count; @@ -364,41 +451,45 @@ insert_tree(struct net *net, goto restart; } - /* expected case: match, insert new node */ - rbconn = kmem_cache_alloc(conncount_rb_cachep, GFP_ATOMIC); - if (rbconn == NULL) - goto out_unlock; + if (get_ct_or_tuple_from_skb(net, skb, l3num, &ct, &tuple, &zone, &refcounted)) { + /* expected case: match, insert new node */ + rbconn = kmem_cache_alloc(conncount_rb_cachep, GFP_ATOMIC); + if (rbconn == NULL) + goto out_unlock; - conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC); - if (conn == NULL) { - kmem_cache_free(conncount_rb_cachep, rbconn); - goto out_unlock; - } + conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC); + if (conn == NULL) { + kmem_cache_free(conncount_rb_cachep, rbconn); + goto out_unlock; + } - conn->tuple = *tuple; - conn->zone = *zone; - conn->cpu = raw_smp_processor_id(); - conn->jiffies32 = (u32)jiffies; - memcpy(rbconn->key, key, sizeof(u32) * data->keylen); + conn->tuple = tuple; + conn->zone = *zone; + conn->cpu = raw_smp_processor_id(); + conn->jiffies32 = (u32)jiffies; + memcpy(rbconn->key, key, sizeof(u32) * data->keylen); - nf_conncount_list_init(&rbconn->list); - list_add(&conn->node, &rbconn->list.head); - count = 1; - rbconn->list.count = count; + nf_conncount_list_init(&rbconn->list); + list_add(&conn->node, &rbconn->list.head); + count = 1; + rbconn->list.count = count; - rb_link_node_rcu(&rbconn->node, parent, rbnode); - rb_insert_color(&rbconn->node, root); + rb_link_node_rcu(&rbconn->node, parent, rbnode); + rb_insert_color(&rbconn->node, root); + } out_unlock: + if (refcounted) + nf_ct_put(ct); spin_unlock_bh(&nf_conncount_locks[hash]); return count; } static unsigned int count_tree(struct net *net, + const struct sk_buff *skb, + u16 l3num, struct nf_conncount_data *data, - const u32 *key, - const struct nf_conntrack_tuple *tuple, - const struct nf_conntrack_zone *zone) + const u32 *key) { struct rb_root *root; struct rb_node *parent; @@ -422,7 +513,7 @@ count_tree(struct net *net, } else { int ret; - if (!tuple) { + if (!skb) { nf_conncount_gc_list(net, &rbconn->list); return rbconn->list.count; } @@ -437,19 +528,23 @@ count_tree(struct net *net, } /* same source network -> be counted! */ - ret = __nf_conncount_add(net, &rbconn->list, tuple, zone); + ret = __nf_conncount_add(net, skb, l3num, &rbconn->list); spin_unlock_bh(&rbconn->list.list_lock); - if (ret) + if (ret && ret != -EEXIST) { return 0; /* hotdrop */ - else + } else { + /* -EEXIST means add was skipped, update the list */ + if (ret == -EEXIST) + nf_conncount_gc_list(net, &rbconn->list); return rbconn->list.count; + } } } - if (!tuple) + if (!skb) return 0; - return insert_tree(net, data, root, hash, key, tuple, zone); + return insert_tree(net, skb, l3num, data, root, hash, key); } static void tree_gc_worker(struct work_struct *work) @@ -511,18 +606,19 @@ static void tree_gc_worker(struct work_struct *work) } /* Count and return number of conntrack entries in 'net' with particular 'key'. - * If 'tuple' is not null, insert it into the accounting data structure. - * Call with RCU read lock. + * If 'skb' is not null, insert the corresponding tuple into the accounting + * data structure. Call with RCU read lock. */ -unsigned int nf_conncount_count(struct net *net, - struct nf_conncount_data *data, - const u32 *key, - const struct nf_conntrack_tuple *tuple, - const struct nf_conntrack_zone *zone) +unsigned int nf_conncount_count_skb(struct net *net, + const struct sk_buff *skb, + u16 l3num, + struct nf_conncount_data *data, + const u32 *key) { - return count_tree(net, data, key, tuple, zone); + return count_tree(net, skb, l3num, data, key); + } -EXPORT_SYMBOL_GPL(nf_conncount_count); +EXPORT_SYMBOL_GPL(nf_conncount_count_skb); struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen) { diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c index a7552a46d6acf..4f39bf7c843f2 100644 --- a/net/netfilter/nf_conntrack_broadcast.c +++ b/net/netfilter/nf_conntrack_broadcast.c @@ -21,6 +21,7 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, unsigned int timeout) { const struct nf_conntrack_helper *helper; + struct net *net = read_pnet(&ct->ct_net); struct nf_conntrack_expect *exp; struct iphdr *iph = ip_hdr(skb); struct rtable *rt = skb_rtable(skb); @@ -70,8 +71,11 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, exp->expectfn = NULL; exp->flags = NF_CT_EXPECT_PERMANENT; exp->class = NF_CT_EXPECT_CLASS_DEFAULT; - exp->helper = NULL; - + rcu_assign_pointer(exp->helper, helper); + write_pnet(&exp->net, net); +#ifdef CONFIG_NF_CONNTRACK_ZONES + exp->zone = ct->zone; +#endif nf_ct_expect_related(exp, 0); nf_ct_expect_put(exp); diff --git a/net/netfilter/nf_conntrack_ecache.c b/net/netfilter/nf_conntrack_ecache.c index 81baf20826046..9df159448b897 100644 --- a/net/netfilter/nf_conntrack_ecache.c +++ b/net/netfilter/nf_conntrack_ecache.c @@ -247,6 +247,8 @@ void nf_ct_expect_event_report(enum ip_conntrack_expect_events event, struct nf_ct_event_notifier *notify; struct nf_conntrack_ecache *e; + lockdep_nfct_expect_lock_held(); + rcu_read_lock(); notify = rcu_dereference(net->ct.nf_conntrack_event_cb); if (!notify) diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index cfc2daa3fc7f3..24d0576d84b7f 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -51,6 +51,7 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, struct net *net = nf_ct_exp_net(exp); struct nf_conntrack_net *cnet; + lockdep_nfct_expect_lock_held(); WARN_ON(!master_help); WARN_ON(timer_pending(&exp->timeout)); @@ -112,12 +113,14 @@ nf_ct_exp_equal(const struct nf_conntrack_tuple *tuple, const struct net *net) { return nf_ct_tuple_mask_cmp(tuple, &i->tuple, &i->mask) && - net_eq(net, nf_ct_net(i->master)) && - nf_ct_zone_equal_any(i->master, zone); + net_eq(net, read_pnet(&i->net)) && + nf_ct_exp_zone_equal_any(i, zone); } bool nf_ct_remove_expect(struct nf_conntrack_expect *exp) { + lockdep_nfct_expect_lock_held(); + if (timer_delete(&exp->timeout)) { nf_ct_unlink_expect(exp); nf_ct_expect_put(exp); @@ -177,6 +180,8 @@ nf_ct_find_expectation(struct net *net, struct nf_conntrack_expect *i, *exp = NULL; unsigned int h; + lockdep_nfct_expect_lock_held(); + if (!cnet->expect_count) return NULL; @@ -309,12 +314,20 @@ struct nf_conntrack_expect *nf_ct_expect_alloc(struct nf_conn *me) } EXPORT_SYMBOL_GPL(nf_ct_expect_alloc); +/* This function can only be used from packet path, where accessing + * master's helper is safe, because the packet holds a reference on + * the conntrack object. Never use it from control plane. + */ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, u_int8_t family, const union nf_inet_addr *saddr, const union nf_inet_addr *daddr, u_int8_t proto, const __be16 *src, const __be16 *dst) { + struct nf_conntrack_helper *helper = NULL; + struct nf_conn *ct = exp->master; + struct net *net = read_pnet(&ct->ct_net); + struct nf_conn_help *help; int len; if (family == AF_INET) @@ -325,7 +338,16 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, exp->flags = 0; exp->class = class; exp->expectfn = NULL; - exp->helper = NULL; + + help = nfct_help(ct); + if (help) + helper = rcu_dereference(help->helper); + + rcu_assign_pointer(exp->helper, helper); + write_pnet(&exp->net, net); +#ifdef CONFIG_NF_CONNTRACK_ZONES + exp->zone = ct->zone; +#endif exp->tuple.src.l3num = family; exp->tuple.dst.protonum = proto; @@ -442,6 +464,8 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, unsigned int h; int ret = 0; + lockdep_nfct_expect_lock_held(); + if (!master_help) { ret = -ESHUTDOWN; goto out; @@ -498,8 +522,9 @@ int nf_ct_expect_related_report(struct nf_conntrack_expect *expect, nf_ct_expect_insert(expect); - spin_unlock_bh(&nf_conntrack_expect_lock); nf_ct_expect_event_report(IPEXP_NEW, expect, portid, report); + spin_unlock_bh(&nf_conntrack_expect_lock); + return 0; out: spin_unlock_bh(&nf_conntrack_expect_lock); @@ -627,11 +652,15 @@ static int exp_seq_show(struct seq_file *s, void *v) { struct nf_conntrack_expect *expect; struct nf_conntrack_helper *helper; + struct net *net = seq_file_net(s); struct hlist_node *n = v; char *delim = ""; expect = hlist_entry(n, struct nf_conntrack_expect, hnode); + if (!net_eq(nf_ct_exp_net(expect), net)) + return 0; + if (expect->timeout.function) seq_printf(s, "%ld ", timer_pending(&expect->timeout) ? (long)(expect->timeout.expires - jiffies)/HZ : 0); @@ -654,7 +683,7 @@ static int exp_seq_show(struct seq_file *s, void *v) if (expect->flags & NF_CT_EXPECT_USERSPACE) seq_printf(s, "%sUSERSPACE", delim); - helper = rcu_dereference(nfct_help(expect->master)->helper); + helper = rcu_dereference(expect->helper); if (helper) { seq_printf(s, "%s%s", expect->flags ? " " : "", helper->name); if (helper->expect_policy[expect->class].name[0]) diff --git a/net/netfilter/nf_conntrack_h323_asn1.c b/net/netfilter/nf_conntrack_h323_asn1.c index 540d97715bd23..7b1497ed97d26 100644 --- a/net/netfilter/nf_conntrack_h323_asn1.c +++ b/net/netfilter/nf_conntrack_h323_asn1.c @@ -331,6 +331,8 @@ static int decode_int(struct bitstr *bs, const struct field_t *f, if (nf_h323_error_boundary(bs, 0, 2)) return H323_ERROR_BOUND; len = get_bits(bs, 2) + 1; + if (nf_h323_error_boundary(bs, len, 0)) + return H323_ERROR_BOUND; BYTE_ALIGN(bs); if (base && (f->attr & DECODE)) { /* timeToLive */ unsigned int v = get_uint(bs, len) + f->lb; @@ -796,7 +798,7 @@ static int decode_choice(struct bitstr *bs, const struct field_t *f, if (ext || (son->attr & OPEN)) { BYTE_ALIGN(bs); - if (nf_h323_error_boundary(bs, len, 0)) + if (nf_h323_error_boundary(bs, 2, 0)) return H323_ERROR_BOUND; len = get_len(bs); if (nf_h323_error_boundary(bs, len, 0)) @@ -922,6 +924,8 @@ int DecodeQ931(unsigned char *buf, size_t sz, Q931 *q931) break; p++; len--; + if (len <= 0) + break; return DecodeH323_UserInformation(buf, p, len, &q931->UUIE); } diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index 14f73872f6477..bd7e9e13e4f68 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -642,7 +642,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - exp->helper = &nf_conntrack_helper_h245; + rcu_assign_pointer(exp->helper, &nf_conntrack_helper_h245); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -766,7 +766,7 @@ static int expect_callforwarding(struct sk_buff *skb, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -1186,13 +1186,13 @@ static struct nf_conntrack_expect *find_expect(struct nf_conn *ct, { struct net *net = nf_ct_net(ct); struct nf_conntrack_expect *exp; - struct nf_conntrack_tuple tuple; + struct nf_conntrack_tuple tuple = { + .src.l3num = nf_ct_l3num(ct), + .dst.protonum = IPPROTO_TCP, + .dst.u.tcp.port = port, + }; - memset(&tuple.src.u3, 0, sizeof(tuple.src.u3)); - tuple.src.u.tcp.port = 0; memcpy(&tuple.dst.u3, addr, sizeof(tuple.dst.u3)); - tuple.dst.u.tcp.port = port; - tuple.dst.protonum = IPPROTO_TCP; exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); if (exp && exp->master == ct) @@ -1233,7 +1233,7 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3 : NULL, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */ nathook = rcu_dereference(nfct_h323_nat_hook); @@ -1305,7 +1305,7 @@ static int process_gcf(struct sk_buff *skb, struct nf_conn *ct, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_UDP, NULL, &port); - exp->helper = nf_conntrack_helper_ras; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_ras); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect RAS "); @@ -1522,7 +1522,7 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); @@ -1576,7 +1576,7 @@ static int process_lcf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - exp->helper = nf_conntrack_helper_q931; + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index ceb48c3ca0a43..a715304a53d8c 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -395,14 +395,10 @@ EXPORT_SYMBOL_GPL(nf_conntrack_helper_register); static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data) { - struct nf_conn_help *help = nfct_help(exp->master); const struct nf_conntrack_helper *me = data; const struct nf_conntrack_helper *this; - if (exp->helper == me) - return true; - - this = rcu_dereference_protected(help->helper, + this = rcu_dereference_protected(exp->helper, lockdep_is_held(&nf_conntrack_expect_lock)); return this == me; } @@ -419,8 +415,13 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) */ synchronize_rcu(); - nf_ct_expect_iterate_destroy(expect_iter_me, NULL); + nf_ct_expect_iterate_destroy(expect_iter_me, me); nf_ct_iterate_destroy(unhelp, me); + + /* nf_ct_iterate_destroy() does an unconditional synchronize_rcu() as + * last step, this ensures rcu readers of exp->helper are done. + * No need for another synchronize_rcu() here. + */ } EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 3a04665adf992..fbe9e3f1036f8 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -909,8 +909,8 @@ struct ctnetlink_filter { }; static const struct nla_policy cta_filter_nla_policy[CTA_FILTER_MAX + 1] = { - [CTA_FILTER_ORIG_FLAGS] = { .type = NLA_U32 }, - [CTA_FILTER_REPLY_FLAGS] = { .type = NLA_U32 }, + [CTA_FILTER_ORIG_FLAGS] = NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL), + [CTA_FILTER_REPLY_FLAGS] = NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL), }; static int ctnetlink_parse_filter(const struct nlattr *attr, @@ -924,17 +924,11 @@ static int ctnetlink_parse_filter(const struct nlattr *attr, if (ret) return ret; - if (tb[CTA_FILTER_ORIG_FLAGS]) { + if (tb[CTA_FILTER_ORIG_FLAGS]) filter->orig_flags = nla_get_u32(tb[CTA_FILTER_ORIG_FLAGS]); - if (filter->orig_flags & ~CTA_FILTER_F_ALL) - return -EOPNOTSUPP; - } - if (tb[CTA_FILTER_REPLY_FLAGS]) { + if (tb[CTA_FILTER_REPLY_FLAGS]) filter->reply_flags = nla_get_u32(tb[CTA_FILTER_REPLY_FLAGS]); - if (filter->reply_flags & ~CTA_FILTER_F_ALL) - return -EOPNOTSUPP; - } return 0; } @@ -2633,7 +2627,7 @@ static const struct nla_policy exp_nla_policy[CTA_EXPECT_MAX+1] = { [CTA_EXPECT_HELP_NAME] = { .type = NLA_NUL_STRING, .len = NF_CT_HELPER_NAME_LEN - 1 }, [CTA_EXPECT_ZONE] = { .type = NLA_U16 }, - [CTA_EXPECT_FLAGS] = { .type = NLA_U32 }, + [CTA_EXPECT_FLAGS] = NLA_POLICY_MASK(NLA_BE32, NF_CT_EXPECT_MASK), [CTA_EXPECT_CLASS] = { .type = NLA_U32 }, [CTA_EXPECT_NAT] = { .type = NLA_NESTED }, [CTA_EXPECT_FN] = { .type = NLA_NUL_STRING }, @@ -2641,7 +2635,6 @@ static const struct nla_policy exp_nla_policy[CTA_EXPECT_MAX+1] = { static struct nf_conntrack_expect * ctnetlink_alloc_expect(const struct nlattr *const cda[], struct nf_conn *ct, - struct nf_conntrack_helper *helper, struct nf_conntrack_tuple *tuple, struct nf_conntrack_tuple *mask); @@ -2870,7 +2863,6 @@ ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct, { struct nlattr *cda[CTA_EXPECT_MAX+1]; struct nf_conntrack_tuple tuple, mask; - struct nf_conntrack_helper *helper = NULL; struct nf_conntrack_expect *exp; int err; @@ -2884,17 +2876,8 @@ ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct, if (err < 0) return err; - if (cda[CTA_EXPECT_HELP_NAME]) { - const char *helpname = nla_data(cda[CTA_EXPECT_HELP_NAME]); - - helper = __nf_conntrack_helper_find(helpname, nf_ct_l3num(ct), - nf_ct_protonum(ct)); - if (helper == NULL) - return -EOPNOTSUPP; - } - exp = ctnetlink_alloc_expect((const struct nlattr * const *)cda, ct, - helper, &tuple, &mask); + &tuple, &mask); if (IS_ERR(exp)) return PTR_ERR(exp); @@ -3011,7 +2994,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, { struct nf_conn *master = exp->master; long timeout = ((long)exp->timeout.expires - (long)jiffies) / HZ; - struct nf_conn_help *help; + struct nf_conntrack_helper *helper; #if IS_ENABLED(CONFIG_NF_NAT) struct nlattr *nest_parms; struct nf_conntrack_tuple nat_tuple = {}; @@ -3056,15 +3039,12 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags)) || nla_put_be32(skb, CTA_EXPECT_CLASS, htonl(exp->class))) goto nla_put_failure; - help = nfct_help(master); - if (help) { - struct nf_conntrack_helper *helper; - helper = rcu_dereference(help->helper); - if (helper && - nla_put_string(skb, CTA_EXPECT_HELP_NAME, helper->name)) - goto nla_put_failure; - } + helper = rcu_dereference(exp->helper); + if (helper && + nla_put_string(skb, CTA_EXPECT_HELP_NAME, helper->name)) + goto nla_put_failure; + expfn = nf_ct_helper_expectfn_find_by_symbol(exp->expectfn); if (expfn != NULL && nla_put_string(skb, CTA_EXPECT_FN, expfn->name)) @@ -3211,7 +3191,7 @@ ctnetlink_exp_ct_dump_table(struct sk_buff *skb, struct netlink_callback *cb) { struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh); struct nf_conn *ct = cb->data; - struct nf_conn_help *help = nfct_help(ct); + struct nf_conn_help *help; u_int8_t l3proto = nfmsg->nfgen_family; unsigned long last_id = cb->args[1]; struct nf_conntrack_expect *exp; @@ -3219,6 +3199,10 @@ ctnetlink_exp_ct_dump_table(struct sk_buff *skb, struct netlink_callback *cb) if (cb->args[0]) return 0; + help = nfct_help(ct); + if (!help) + return 0; + rcu_read_lock(); restart: @@ -3248,6 +3232,24 @@ ctnetlink_exp_ct_dump_table(struct sk_buff *skb, struct netlink_callback *cb) return skb->len; } +static int ctnetlink_dump_exp_ct_start(struct netlink_callback *cb) +{ + struct nf_conn *ct = cb->data; + + if (!refcount_inc_not_zero(&ct->ct_general.use)) + return -ENOENT; + return 0; +} + +static int ctnetlink_dump_exp_ct_done(struct netlink_callback *cb) +{ + struct nf_conn *ct = cb->data; + + if (ct) + nf_ct_put(ct); + return 0; +} + static int ctnetlink_dump_exp_ct(struct net *net, struct sock *ctnl, struct sk_buff *skb, const struct nlmsghdr *nlh, @@ -3263,6 +3265,8 @@ static int ctnetlink_dump_exp_ct(struct net *net, struct sock *ctnl, struct nf_conntrack_zone zone; struct netlink_dump_control c = { .dump = ctnetlink_exp_ct_dump_table, + .start = ctnetlink_dump_exp_ct_start, + .done = ctnetlink_dump_exp_ct_done, }; err = ctnetlink_parse_tuple(cda, &tuple, CTA_EXPECT_MASTER, @@ -3333,31 +3337,37 @@ static int ctnetlink_get_expect(struct sk_buff *skb, if (err < 0) return err; + skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb2) + return -ENOMEM; + + spin_lock_bh(&nf_conntrack_expect_lock); exp = nf_ct_expect_find_get(info->net, &zone, &tuple); - if (!exp) + if (!exp) { + spin_unlock_bh(&nf_conntrack_expect_lock); + kfree_skb(skb2); return -ENOENT; + } if (cda[CTA_EXPECT_ID]) { __be32 id = nla_get_be32(cda[CTA_EXPECT_ID]); if (id != nf_expect_get_id(exp)) { nf_ct_expect_put(exp); + spin_unlock_bh(&nf_conntrack_expect_lock); + kfree_skb(skb2); return -ENOENT; } } - skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); - if (!skb2) { - nf_ct_expect_put(exp); - return -ENOMEM; - } - rcu_read_lock(); err = ctnetlink_exp_fill_info(skb2, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, IPCTNL_MSG_EXP_NEW, exp); rcu_read_unlock(); nf_ct_expect_put(exp); + spin_unlock_bh(&nf_conntrack_expect_lock); + if (err <= 0) { kfree_skb(skb2); return -ENOMEM; @@ -3369,12 +3379,9 @@ static int ctnetlink_get_expect(struct sk_buff *skb, static bool expect_iter_name(struct nf_conntrack_expect *exp, void *data) { struct nf_conntrack_helper *helper; - const struct nf_conn_help *m_help; const char *name = data; - m_help = nfct_help(exp->master); - - helper = rcu_dereference(m_help->helper); + helper = rcu_dereference(exp->helper); if (!helper) return false; @@ -3407,22 +3414,26 @@ static int ctnetlink_del_expect(struct sk_buff *skb, if (err < 0) return err; + spin_lock_bh(&nf_conntrack_expect_lock); + /* bump usage count to 2 */ exp = nf_ct_expect_find_get(info->net, &zone, &tuple); - if (!exp) + if (!exp) { + spin_unlock_bh(&nf_conntrack_expect_lock); return -ENOENT; + } if (cda[CTA_EXPECT_ID]) { __be32 id = nla_get_be32(cda[CTA_EXPECT_ID]); if (id != nf_expect_get_id(exp)) { nf_ct_expect_put(exp); + spin_unlock_bh(&nf_conntrack_expect_lock); return -ENOENT; } } /* after list removal, usage count == 1 */ - spin_lock_bh(&nf_conntrack_expect_lock); if (timer_delete(&exp->timeout)) { nf_ct_unlink_expect_report(exp, NETLINK_CB(skb).portid, nlmsg_report(info->nlh)); @@ -3464,7 +3475,7 @@ ctnetlink_change_expect(struct nf_conntrack_expect *x, #if IS_ENABLED(CONFIG_NF_NAT) static const struct nla_policy exp_nat_nla_policy[CTA_EXPECT_NAT_MAX+1] = { - [CTA_EXPECT_NAT_DIR] = { .type = NLA_U32 }, + [CTA_EXPECT_NAT_DIR] = NLA_POLICY_MAX(NLA_BE32, IP_CT_DIR_REPLY), [CTA_EXPECT_NAT_TUPLE] = { .type = NLA_NESTED }, }; #endif @@ -3505,20 +3516,25 @@ ctnetlink_parse_expect_nat(const struct nlattr *attr, static struct nf_conntrack_expect * ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, - struct nf_conntrack_helper *helper, struct nf_conntrack_tuple *tuple, struct nf_conntrack_tuple *mask) { - u_int32_t class = 0; + struct net *net = read_pnet(&ct->ct_net); + struct nf_conntrack_helper *helper; struct nf_conntrack_expect *exp; struct nf_conn_help *help; + u32 class = 0; int err; help = nfct_help(ct); if (!help) return ERR_PTR(-EOPNOTSUPP); - if (cda[CTA_EXPECT_CLASS] && helper) { + helper = rcu_dereference(help->helper); + if (!helper) + return ERR_PTR(-EOPNOTSUPP); + + if (cda[CTA_EXPECT_CLASS]) { class = ntohl(nla_get_be32(cda[CTA_EXPECT_CLASS])); if (class > helper->expect_class_max) return ERR_PTR(-EINVAL); @@ -3548,7 +3564,11 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, exp->class = class; exp->master = ct; - exp->helper = helper; + write_pnet(&exp->net, net); +#ifdef CONFIG_NF_CONNTRACK_ZONES + exp->zone = ct->zone; +#endif + rcu_assign_pointer(exp->helper, helper); exp->tuple = *tuple; exp->mask.src.u3 = mask->src.u3; exp->mask.src.u.all = mask->src.u.all; @@ -3558,6 +3578,12 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, exp, nf_ct_l3num(ct)); if (err < 0) goto err_out; +#if IS_ENABLED(CONFIG_NF_NAT) + } else { + memset(&exp->saved_addr, 0, sizeof(exp->saved_addr)); + memset(&exp->saved_proto, 0, sizeof(exp->saved_proto)); + exp->dir = 0; +#endif } return exp; err_out: @@ -3573,7 +3599,6 @@ ctnetlink_create_expect(struct net *net, { struct nf_conntrack_tuple tuple, mask, master_tuple; struct nf_conntrack_tuple_hash *h = NULL; - struct nf_conntrack_helper *helper = NULL; struct nf_conntrack_expect *exp; struct nf_conn *ct; int err; @@ -3599,33 +3624,7 @@ ctnetlink_create_expect(struct net *net, ct = nf_ct_tuplehash_to_ctrack(h); rcu_read_lock(); - if (cda[CTA_EXPECT_HELP_NAME]) { - const char *helpname = nla_data(cda[CTA_EXPECT_HELP_NAME]); - - helper = __nf_conntrack_helper_find(helpname, u3, - nf_ct_protonum(ct)); - if (helper == NULL) { - rcu_read_unlock(); -#ifdef CONFIG_MODULES - if (request_module("nfct-helper-%s", helpname) < 0) { - err = -EOPNOTSUPP; - goto err_ct; - } - rcu_read_lock(); - helper = __nf_conntrack_helper_find(helpname, u3, - nf_ct_protonum(ct)); - if (helper) { - err = -EAGAIN; - goto err_rcu; - } - rcu_read_unlock(); -#endif - err = -EOPNOTSUPP; - goto err_ct; - } - } - - exp = ctnetlink_alloc_expect(cda, ct, helper, &tuple, &mask); + exp = ctnetlink_alloc_expect(cda, ct, &tuple, &mask); if (IS_ERR(exp)) { err = PTR_ERR(exp); goto err_rcu; @@ -3635,8 +3634,8 @@ ctnetlink_create_expect(struct net *net, nf_ct_expect_put(exp); err_rcu: rcu_read_unlock(); -err_ct: nf_ct_put(ct); + return err; } diff --git a/net/netfilter/nf_conntrack_proto_generic.c b/net/netfilter/nf_conntrack_proto_generic.c index e831637bc8ca8..cb260eb3d012c 100644 --- a/net/netfilter/nf_conntrack_proto_generic.c +++ b/net/netfilter/nf_conntrack_proto_generic.c @@ -67,6 +67,7 @@ void nf_conntrack_generic_init_net(struct net *net) const struct nf_conntrack_l4proto nf_conntrack_l4proto_generic = { .l4proto = 255, + .allow_clash = true, #ifdef CONFIG_NF_CONNTRACK_TIMEOUT .ctnl_timeout = { .nlattr_to_obj = generic_timeout_nlattr_to_obj, diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c index 7c6f7c9f73320..645d2c43ebf7a 100644 --- a/net/netfilter/nf_conntrack_proto_sctp.c +++ b/net/netfilter/nf_conntrack_proto_sctp.c @@ -582,7 +582,8 @@ static int sctp_to_nlattr(struct sk_buff *skb, struct nlattr *nla, } static const struct nla_policy sctp_nla_policy[CTA_PROTOINFO_SCTP_MAX+1] = { - [CTA_PROTOINFO_SCTP_STATE] = { .type = NLA_U8 }, + [CTA_PROTOINFO_SCTP_STATE] = NLA_POLICY_MAX(NLA_U8, + SCTP_CONNTRACK_HEARTBEAT_SENT), [CTA_PROTOINFO_SCTP_VTAG_ORIGINAL] = { .type = NLA_U32 }, [CTA_PROTOINFO_SCTP_VTAG_REPLY] = { .type = NLA_U32 }, }; diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c index 0c1d086e96cb3..b67426c2189b2 100644 --- a/net/netfilter/nf_conntrack_proto_tcp.c +++ b/net/netfilter/nf_conntrack_proto_tcp.c @@ -1385,9 +1385,9 @@ static int tcp_to_nlattr(struct sk_buff *skb, struct nlattr *nla, } static const struct nla_policy tcp_nla_policy[CTA_PROTOINFO_TCP_MAX+1] = { - [CTA_PROTOINFO_TCP_STATE] = { .type = NLA_U8 }, - [CTA_PROTOINFO_TCP_WSCALE_ORIGINAL] = { .type = NLA_U8 }, - [CTA_PROTOINFO_TCP_WSCALE_REPLY] = { .type = NLA_U8 }, + [CTA_PROTOINFO_TCP_STATE] = NLA_POLICY_MAX(NLA_U8, TCP_CONNTRACK_SYN_SENT2), + [CTA_PROTOINFO_TCP_WSCALE_ORIGINAL] = NLA_POLICY_MAX(NLA_U8, TCP_MAX_WSCALE), + [CTA_PROTOINFO_TCP_WSCALE_REPLY] = NLA_POLICY_MAX(NLA_U8, TCP_MAX_WSCALE), [CTA_PROTOINFO_TCP_FLAGS_ORIGINAL] = { .len = sizeof(struct nf_ct_tcp_flags) }, [CTA_PROTOINFO_TCP_FLAGS_REPLY] = { .len = sizeof(struct nf_ct_tcp_flags) }, }; @@ -1414,10 +1414,6 @@ static int nlattr_to_tcp(struct nlattr *cda[], struct nf_conn *ct) if (err < 0) return err; - if (tb[CTA_PROTOINFO_TCP_STATE] && - nla_get_u8(tb[CTA_PROTOINFO_TCP_STATE]) >= TCP_CONNTRACK_MAX) - return -EINVAL; - spin_lock_bh(&ct->lock); if (tb[CTA_PROTOINFO_TCP_STATE]) ct->proto.tcp.state = nla_get_u8(tb[CTA_PROTOINFO_TCP_STATE]); diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index ca748f8dbff13..939502ff7c871 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -924,7 +924,7 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); if (!exp || exp->master == ct || - nfct_help(exp->master)->helper != nfct_help(ct)->helper || + exp->helper != nfct_help(ct)->helper || exp->class != class) break; #if IS_ENABLED(CONFIG_NF_NAT) @@ -1040,6 +1040,7 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, unsigned int port; const struct sdp_media_type *t; int ret = NF_ACCEPT; + bool have_rtp_addr = false; hooks = rcu_dereference(nf_nat_sip_hooks); @@ -1056,8 +1057,11 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, caddr_len = 0; if (ct_sip_parse_sdp_addr(ct, *dptr, sdpoff, *datalen, SDP_HDR_CONNECTION, SDP_HDR_MEDIA, - &matchoff, &matchlen, &caddr) > 0) + &matchoff, &matchlen, &caddr) > 0) { caddr_len = matchlen; + memcpy(&rtp_addr, &caddr, sizeof(rtp_addr)); + have_rtp_addr = true; + } mediaoff = sdpoff; for (i = 0; i < ARRAY_SIZE(sdp_media_types); ) { @@ -1091,9 +1095,11 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, &matchoff, &matchlen, &maddr) > 0) { maddr_len = matchlen; memcpy(&rtp_addr, &maddr, sizeof(rtp_addr)); - } else if (caddr_len) + have_rtp_addr = true; + } else if (caddr_len) { memcpy(&rtp_addr, &caddr, sizeof(rtp_addr)); - else { + have_rtp_addr = true; + } else { nf_ct_helper_log(skb, ct, "cannot parse SDP message"); return NF_DROP; } @@ -1125,7 +1131,7 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, /* Update session connection and owner addresses */ hooks = rcu_dereference(nf_nat_sip_hooks); - if (hooks && ct->status & IPS_NAT_MASK) + if (hooks && ct->status & IPS_NAT_MASK && have_rtp_addr) ret = hooks->sdp_session(skb, protoff, dataoff, dptr, datalen, sdpoff, &rtp_addr); @@ -1297,7 +1303,7 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), saddr, &daddr, proto, NULL, &port); exp->timeout.expires = sip_timeout * HZ; - exp->helper = helper; + rcu_assign_pointer(exp->helper, helper); exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE; hooks = rcu_dereference(nf_nat_sip_hooks); @@ -1534,11 +1540,12 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff, { struct tcphdr *th, _tcph; unsigned int dataoff, datalen; - unsigned int matchoff, matchlen, clen; + unsigned int matchoff, matchlen; unsigned int msglen, origlen; const char *dptr, *end; s16 diff, tdiff = 0; int ret = NF_ACCEPT; + unsigned long clen; bool term; if (ctinfo != IP_CT_ESTABLISHED && @@ -1573,6 +1580,9 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff, if (dptr + matchoff == end) break; + if (clen > datalen) + break; + term = false; for (; end + strlen("\r\n\r\n") <= dptr + datalen; end++) { if (end[0] == '\r' && end[1] == '\n' && diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index e06bc36f49fe7..4f346f51d7d74 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -13,6 +13,8 @@ #include #include +#define NF_FLOW_RULE_ACTION_MAX 24 + static struct workqueue_struct *nf_flow_offload_add_wq; static struct workqueue_struct *nf_flow_offload_del_wq; static struct workqueue_struct *nf_flow_offload_stats_wq; @@ -215,7 +217,12 @@ static void flow_offload_mangle(struct flow_action_entry *entry, static inline struct flow_action_entry * flow_action_entry_next(struct nf_flow_rule *flow_rule) { - int i = flow_rule->rule->action.num_entries++; + int i; + + if (unlikely(flow_rule->rule->action.num_entries >= NF_FLOW_RULE_ACTION_MAX)) + return NULL; + + i = flow_rule->rule->action.num_entries++; return &flow_rule->rule->action.entries[i]; } @@ -233,6 +240,9 @@ static int flow_offload_eth_src(struct net *net, u32 mask, val; u16 val16; + if (!entry0 || !entry1) + return -E2BIG; + this_tuple = &flow->tuplehash[dir].tuple; switch (this_tuple->xmit_type) { @@ -283,6 +293,9 @@ static int flow_offload_eth_dst(struct net *net, u8 nud_state; u16 val16; + if (!entry0 || !entry1) + return -E2BIG; + this_tuple = &flow->tuplehash[dir].tuple; switch (this_tuple->xmit_type) { @@ -324,16 +337,19 @@ static int flow_offload_eth_dst(struct net *net, return 0; } -static void flow_offload_ipv4_snat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv4_snat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask = ~htonl(0xffffffff); __be32 addr; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; @@ -344,23 +360,27 @@ static void flow_offload_ipv4_snat(struct net *net, offset = offsetof(struct iphdr, daddr); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, &addr, &mask); + return 0; } -static void flow_offload_ipv4_dnat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv4_dnat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask = ~htonl(0xffffffff); __be32 addr; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; @@ -371,14 +391,15 @@ static void flow_offload_ipv4_dnat(struct net *net, offset = offsetof(struct iphdr, saddr); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, &addr, &mask); + return 0; } -static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, +static int flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, unsigned int offset, const __be32 *addr, const __be32 *mask) { @@ -387,15 +408,20 @@ static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; + flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6, offset + i * sizeof(u32), &addr[i], mask); } + + return 0; } -static void flow_offload_ipv6_snat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv6_snat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { u32 mask = ~htonl(0xffffffff); const __be32 *addr; @@ -411,16 +437,16 @@ static void flow_offload_ipv6_snat(struct net *net, offset = offsetof(struct ipv6hdr, daddr); break; default: - return; + return -EOPNOTSUPP; } - flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); + return flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); } -static void flow_offload_ipv6_dnat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv6_dnat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { u32 mask = ~htonl(0xffffffff); const __be32 *addr; @@ -436,10 +462,10 @@ static void flow_offload_ipv6_dnat(struct net *net, offset = offsetof(struct ipv6hdr, saddr); break; default: - return; + return -EOPNOTSUPP; } - flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); + return flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); } static int flow_offload_l4proto(const struct flow_offload *flow) @@ -461,15 +487,18 @@ static int flow_offload_l4proto(const struct flow_offload *flow) return type; } -static void flow_offload_port_snat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_port_snat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask, port; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port); @@ -484,22 +513,26 @@ static void flow_offload_port_snat(struct net *net, mask = ~htonl(0xffff); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, &port, &mask); + return 0; } -static void flow_offload_port_dnat(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_port_dnat(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { struct flow_action_entry *entry = flow_action_entry_next(flow_rule); u32 mask, port; u32 offset; + if (!entry) + return -E2BIG; + switch (dir) { case FLOW_OFFLOAD_DIR_ORIGINAL: port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port); @@ -514,20 +547,24 @@ static void flow_offload_port_dnat(struct net *net, mask = ~htonl(0xffff0000); break; default: - return; + return -EOPNOTSUPP; } flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, &port, &mask); + return 0; } -static void flow_offload_ipv4_checksum(struct net *net, - const struct flow_offload *flow, - struct nf_flow_rule *flow_rule) +static int flow_offload_ipv4_checksum(struct net *net, + const struct flow_offload *flow, + struct nf_flow_rule *flow_rule) { u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; struct flow_action_entry *entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; + entry->id = FLOW_ACTION_CSUM; entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR; @@ -539,12 +576,14 @@ static void flow_offload_ipv4_checksum(struct net *net, entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP; break; } + + return 0; } -static void flow_offload_redirect(struct net *net, - const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_redirect(struct net *net, + const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { const struct flow_offload_tuple *this_tuple, *other_tuple; struct flow_action_entry *entry; @@ -562,21 +601,28 @@ static void flow_offload_redirect(struct net *net, ifindex = other_tuple->iifidx; break; default: - return; + return -EOPNOTSUPP; } dev = dev_get_by_index(net, ifindex); if (!dev) - return; + return -ENODEV; entry = flow_action_entry_next(flow_rule); + if (!entry) { + dev_put(dev); + return -E2BIG; + } + entry->id = FLOW_ACTION_REDIRECT; entry->dev = dev; + + return 0; } -static void flow_offload_encap_tunnel(const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_encap_tunnel(const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { const struct flow_offload_tuple *this_tuple; struct flow_action_entry *entry; @@ -584,7 +630,7 @@ static void flow_offload_encap_tunnel(const struct flow_offload *flow, this_tuple = &flow->tuplehash[dir].tuple; if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) - return; + return 0; dst = this_tuple->dst_cache; if (dst && dst->lwtstate) { @@ -593,15 +639,19 @@ static void flow_offload_encap_tunnel(const struct flow_offload *flow, tun_info = lwt_tun_info(dst->lwtstate); if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; entry->id = FLOW_ACTION_TUNNEL_ENCAP; entry->tunnel = tun_info; } } + + return 0; } -static void flow_offload_decap_tunnel(const struct flow_offload *flow, - enum flow_offload_tuple_dir dir, - struct nf_flow_rule *flow_rule) +static int flow_offload_decap_tunnel(const struct flow_offload *flow, + enum flow_offload_tuple_dir dir, + struct nf_flow_rule *flow_rule) { const struct flow_offload_tuple *other_tuple; struct flow_action_entry *entry; @@ -609,7 +659,7 @@ static void flow_offload_decap_tunnel(const struct flow_offload *flow, other_tuple = &flow->tuplehash[!dir].tuple; if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) - return; + return 0; dst = other_tuple->dst_cache; if (dst && dst->lwtstate) { @@ -618,9 +668,13 @@ static void flow_offload_decap_tunnel(const struct flow_offload *flow, tun_info = lwt_tun_info(dst->lwtstate); if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -E2BIG; entry->id = FLOW_ACTION_TUNNEL_DECAP; } } + + return 0; } static int @@ -632,8 +686,9 @@ nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, const struct flow_offload_tuple *tuple; int i; - flow_offload_decap_tunnel(flow, dir, flow_rule); - flow_offload_encap_tunnel(flow, dir, flow_rule); + if (flow_offload_decap_tunnel(flow, dir, flow_rule) < 0 || + flow_offload_encap_tunnel(flow, dir, flow_rule) < 0) + return -1; if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 || flow_offload_eth_dst(net, flow, dir, flow_rule) < 0) @@ -649,6 +704,8 @@ nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, if (tuple->encap[i].proto == htons(ETH_P_8021Q)) { entry = flow_action_entry_next(flow_rule); + if (!entry) + return -1; entry->id = FLOW_ACTION_VLAN_POP; } } @@ -662,6 +719,8 @@ nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, continue; entry = flow_action_entry_next(flow_rule); + if (!entry) + return -1; switch (other_tuple->encap[i].proto) { case htons(ETH_P_PPP_SES): @@ -687,18 +746,22 @@ int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow, return -1; if (test_bit(NF_FLOW_SNAT, &flow->flags)) { - flow_offload_ipv4_snat(net, flow, dir, flow_rule); - flow_offload_port_snat(net, flow, dir, flow_rule); + if (flow_offload_ipv4_snat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_snat(net, flow, dir, flow_rule) < 0) + return -1; } if (test_bit(NF_FLOW_DNAT, &flow->flags)) { - flow_offload_ipv4_dnat(net, flow, dir, flow_rule); - flow_offload_port_dnat(net, flow, dir, flow_rule); + if (flow_offload_ipv4_dnat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_dnat(net, flow, dir, flow_rule) < 0) + return -1; } if (test_bit(NF_FLOW_SNAT, &flow->flags) || test_bit(NF_FLOW_DNAT, &flow->flags)) - flow_offload_ipv4_checksum(net, flow, flow_rule); + if (flow_offload_ipv4_checksum(net, flow, flow_rule) < 0) + return -1; - flow_offload_redirect(net, flow, dir, flow_rule); + if (flow_offload_redirect(net, flow, dir, flow_rule) < 0) + return -1; return 0; } @@ -712,22 +775,23 @@ int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow, return -1; if (test_bit(NF_FLOW_SNAT, &flow->flags)) { - flow_offload_ipv6_snat(net, flow, dir, flow_rule); - flow_offload_port_snat(net, flow, dir, flow_rule); + if (flow_offload_ipv6_snat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_snat(net, flow, dir, flow_rule) < 0) + return -1; } if (test_bit(NF_FLOW_DNAT, &flow->flags)) { - flow_offload_ipv6_dnat(net, flow, dir, flow_rule); - flow_offload_port_dnat(net, flow, dir, flow_rule); + if (flow_offload_ipv6_dnat(net, flow, dir, flow_rule) < 0 || + flow_offload_port_dnat(net, flow, dir, flow_rule) < 0) + return -1; } - flow_offload_redirect(net, flow, dir, flow_rule); + if (flow_offload_redirect(net, flow, dir, flow_rule) < 0) + return -1; return 0; } EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6); -#define NF_FLOW_RULE_ACTION_MAX 16 - static struct nf_flow_rule * nf_flow_offload_rule_alloc(struct net *net, const struct flow_offload_work *offload, diff --git a/net/netfilter/nf_log.c b/net/netfilter/nf_log.c index 74cef8bf554c5..62cf6a30875e3 100644 --- a/net/netfilter/nf_log.c +++ b/net/netfilter/nf_log.c @@ -89,7 +89,7 @@ int nf_log_register(u_int8_t pf, struct nf_logger *logger) if (pf == NFPROTO_UNSPEC) { for (i = NFPROTO_UNSPEC; i < NFPROTO_NUMPROTO; i++) { if (rcu_access_pointer(loggers[i][logger->type])) { - ret = -EEXIST; + ret = -EBUSY; goto unlock; } } @@ -97,7 +97,7 @@ int nf_log_register(u_int8_t pf, struct nf_logger *logger) rcu_assign_pointer(loggers[i][logger->type], logger); } else { if (rcu_access_pointer(loggers[pf][logger->type])) { - ret = -EEXIST; + ret = -EBUSY; goto unlock; } rcu_assign_pointer(loggers[pf][logger->type], logger); diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 78a61dac4ade8..e6b24586d2fed 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -294,25 +294,13 @@ nf_nat_used_tuple_new(const struct nf_conntrack_tuple *tuple, ct = nf_ct_tuplehash_to_ctrack(thash); - /* NB: IP_CT_DIR_ORIGINAL should be impossible because - * nf_nat_used_tuple() handles origin collisions. - * - * Handle remote chance other CPU confirmed its ct right after. - */ - if (thash->tuple.dst.dir != IP_CT_DIR_REPLY) - goto out; - /* clashing connection subject to NAT? Retry with new tuple. */ if (READ_ONCE(ct->status) & uses_nat) goto out; if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, - &ignored_ct->tuplehash[IP_CT_DIR_REPLY].tuple) && - nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple, - &ignored_ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple)) { + &ignored_ct->tuplehash[IP_CT_DIR_REPLY].tuple)) taken = false; - goto out; - } out: nf_ct_put(ct); return taken; diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index eed434e0a9702..5b25b032e285d 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -123,6 +123,29 @@ static void nft_validate_state_update(struct nft_table *table, u8 new_validate_s table->validate_state = new_validate_state; } + +static bool nft_chain_vstate_valid(const struct nft_ctx *ctx, + const struct nft_chain *chain) +{ + const struct nft_base_chain *base_chain; + enum nft_chain_types type; + u8 hooknum; + + if (WARN_ON_ONCE(!nft_is_base_chain(ctx->chain))) + return false; + + base_chain = nft_base_chain(ctx->chain); + hooknum = base_chain->ops.hooknum; + type = base_chain->type->type; + + /* chain is already validated for this call depth */ + if (chain->vstate.depth >= ctx->level && + chain->vstate.hook_mask[type] & BIT(hooknum)) + return true; + + return false; +} + static void nf_tables_trans_destroy_work(struct work_struct *w); static void nft_trans_gc_work(struct work_struct *work); @@ -805,10 +828,14 @@ static void nft_map_catchall_deactivate(const struct nft_ctx *ctx, nft_set_elem_change_active(ctx->net, set, ext); nft_setelem_data_deactivate(ctx->net, set, catchall->elem); - break; } } +/* Use NFT_ITER_UPDATE iterator even if this may be called from the preparation + * phase, the set clone might already exist from a previous command, or it might + * be a set that is going away and does not require a clone. The netns and + * netlink release paths also need to work on the live set. + */ static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set) { struct nft_set_iter iter = { @@ -2799,6 +2826,7 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 policy, err_register_hook: nft_chain_del(chain); + synchronize_rcu(); err_chain_add: nft_trans_destroy(trans); err_trans: @@ -4079,6 +4107,29 @@ static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *r nf_tables_rule_destroy(ctx, rule); } +static void nft_chain_vstate_update(const struct nft_ctx *ctx, struct nft_chain *chain) +{ + const struct nft_base_chain *base_chain; + enum nft_chain_types type; + u8 hooknum; + + /* ctx->chain must hold the calling base chain. */ + if (WARN_ON_ONCE(!nft_is_base_chain(ctx->chain))) { + memset(&chain->vstate, 0, sizeof(chain->vstate)); + return; + } + + base_chain = nft_base_chain(ctx->chain); + hooknum = base_chain->ops.hooknum; + type = base_chain->type->type; + + BUILD_BUG_ON(BIT(NF_INET_NUMHOOKS) > U8_MAX); + + chain->vstate.hook_mask[type] |= BIT(hooknum); + if (chain->vstate.depth < ctx->level) + chain->vstate.depth = ctx->level; +} + /** nft_chain_validate - loop detection and hook validation * * @ctx: context containing call depth and base chain @@ -4088,15 +4139,25 @@ static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *r * and set lookups until either the jump limit is hit or all reachable * chains have been validated. */ -int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain) +int nft_chain_validate(const struct nft_ctx *ctx, struct nft_chain *chain) { struct nft_expr *expr, *last; struct nft_rule *rule; int err; + BUILD_BUG_ON(NFT_JUMP_STACK_SIZE > 255); if (ctx->level == NFT_JUMP_STACK_SIZE) return -EMLINK; + if (ctx->level > 0) { + /* jumps to base chains are not allowed. */ + if (nft_is_base_chain(chain)) + return -ELOOP; + + if (nft_chain_vstate_valid(ctx, chain)) + return 0; + } + list_for_each_entry(rule, &chain->rules, list) { if (fatal_signal_pending(current)) return -EINTR; @@ -4117,6 +4178,7 @@ int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain) } } + nft_chain_vstate_update(ctx, chain); return 0; } EXPORT_SYMBOL_GPL(nft_chain_validate); @@ -4128,7 +4190,7 @@ static int nft_table_validate(struct net *net, const struct nft_table *table) .net = net, .family = table->family, }; - int err; + int err = 0; list_for_each_entry(chain, &table->chains, list) { if (!nft_is_base_chain(chain)) @@ -4137,12 +4199,16 @@ static int nft_table_validate(struct net *net, const struct nft_table *table) ctx.chain = chain; err = nft_chain_validate(&ctx, chain); if (err < 0) - return err; + goto err; cond_resched(); } - return 0; +err: + list_for_each_entry(chain, &table->chains, list) + memset(&chain->vstate, 0, sizeof(chain->vstate)); + + return err; } int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set, @@ -4378,7 +4444,7 @@ static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info, if (!nft_use_inc(&chain->use)) { err = -EMFILE; - goto err_release_rule; + goto err_destroy_flow; } if (info->nlh->nlmsg_flags & NLM_F_REPLACE) { @@ -4428,6 +4494,7 @@ static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info, err_destroy_flow_rule: nft_use_dec_restore(&chain->use); +err_destroy_flow: if (flow) nft_flow_rule_destroy(flow); err_release_rule: @@ -5855,12 +5922,11 @@ static void nft_map_catchall_activate(const struct nft_ctx *ctx, list_for_each_entry(catchall, &set->catchall_list, list) { ext = nft_set_elem_ext(set, catchall->elem); - if (!nft_set_elem_active(ext, genmask)) + if (nft_set_elem_active(ext, genmask)) continue; nft_clear(ctx->net, ext); nft_setelem_data_activate(ctx->net, set, catchall->elem); - break; } } @@ -6797,8 +6863,8 @@ static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx, } } -static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, - struct nft_set_elem_expr *elem_expr) +void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, + struct nft_set_elem_expr *elem_expr) { struct nft_expr *expr; u32 size; @@ -7225,6 +7291,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, struct nft_data_desc desc; enum nft_registers dreg; struct nft_trans *trans; + bool set_full = false; u64 expiration; u64 timeout; int err, i; @@ -7511,10 +7578,18 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, if (err < 0) goto err_elem_free; + if (!(flags & NFT_SET_ELEM_CATCHALL)) { + unsigned int max = nft_set_maxsize(set), nelems; + + nelems = atomic_inc_return(&set->nelems); + if (nelems > max) + set_full = true; + } + trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set); if (trans == NULL) { err = -ENOMEM; - goto err_elem_free; + goto err_set_size; } ext->genmask = nft_genmask_cur(ctx->net); @@ -7566,7 +7641,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, ue->priv = elem_priv; nft_trans_commit_list_add_elem(ctx->net, trans); - goto err_elem_free; + goto err_set_size; } } } @@ -7575,27 +7650,25 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, * and an existing one. */ err = -EEXIST; + } else if (err == -ECANCELED) { + /* ECANCELED reports an existing nul-element in + * interval sets. + */ + err = 0; } goto err_element_clash; } - if (!(flags & NFT_SET_ELEM_CATCHALL)) { - unsigned int max = nft_set_maxsize(set); - - if (!atomic_add_unless(&set->nelems, 1, max)) { - err = -ENFILE; - goto err_set_full; - } - } - nft_trans_container_elem(trans)->elems[0].priv = elem.priv; nft_trans_commit_list_add_elem(ctx->net, trans); - return 0; -err_set_full: - nft_setelem_remove(ctx->net, set, elem.priv); + return set_full ? -ENFILE : 0; + err_element_clash: kfree(trans); +err_set_size: + if (!(flags & NFT_SET_ELEM_CATCHALL)) + atomic_dec(&set->nelems); err_elem_free: nf_tables_set_elem_destroy(ctx, set, elem.priv); err_parse_data: @@ -7746,7 +7819,8 @@ static bool nft_trans_elems_new_abort(const struct nft_ctx *ctx, continue; } - if (!te->set->ops->abort || nft_setelem_is_catchall(te->set, te->elems[i].priv)) + if (!te->set->ops->abort_skip_removal || + nft_setelem_is_catchall(te->set, te->elems[i].priv)) nft_setelem_remove(ctx->net, te->set, te->elems[i].priv); if (!nft_setelem_is_catchall(te->set, te->elems[i].priv)) @@ -7939,9 +8013,12 @@ static int nft_set_catchall_flush(const struct nft_ctx *ctx, static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask) { + /* The set backend might need to clone the set, do it now from the + * preparation phase, use NFT_ITER_UPDATE_CLONE iterator type. + */ struct nft_set_iter iter = { .genmask = genmask, - .type = NFT_ITER_UPDATE, + .type = NFT_ITER_UPDATE_CLONE, .fn = nft_setelem_flush, }; @@ -9292,6 +9369,7 @@ static int nf_tables_newflowtable(struct sk_buff *skb, return 0; err_flowtable_hooks: + synchronize_rcu(); nft_trans_destroy(trans); err_flowtable_trans: nft_hooks_destroy(&flowtable->hook_list); @@ -9762,7 +9840,7 @@ static int nft_flowtable_event(unsigned long event, struct net_device *dev, break; case NETDEV_REGISTER: /* NOP if not matching or already registered */ - if (!match || (changename && ops)) + if (!match || ops) continue; ops = kzalloc(sizeof(struct nf_hook_ops), @@ -10567,11 +10645,6 @@ static void nft_trans_gc_queue_work(struct nft_trans_gc *trans) schedule_work(&trans_gc_work); } -static int nft_trans_gc_space(struct nft_trans_gc *trans) -{ - return NFT_TRANS_GC_BATCHCOUNT - trans->count; -} - struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc, unsigned int gc_seq, gfp_t gfp) { @@ -11476,6 +11549,13 @@ static int nf_tables_abort(struct net *net, struct sk_buff *skb, ret = __nf_tables_abort(net, action); nft_gc_seq_end(nft_net, gc_seq); + if (action == NFNL_ABORT_NONE) { + struct nft_table *table; + + list_for_each_entry(table, &nft_net->tables, list) + table->validate_state = NFT_VALIDATE_SKIP; + } + WARN_ON_ONCE(!list_empty(&nft_net->commit_list)); /* module autoload needs to happen after GC sequence update because it @@ -11678,21 +11758,10 @@ static int nft_validate_register_store(const struct nft_ctx *ctx, enum nft_data_types type, unsigned int len) { - int err; - switch (reg) { case NFT_REG_VERDICT: if (type != NFT_DATA_VERDICT) return -EINVAL; - - if (data != NULL && - (data->verdict.code == NFT_GOTO || - data->verdict.code == NFT_JUMP)) { - err = nft_chain_validate(ctx, data->verdict.chain); - if (err < 0) - return err; - } - break; default: if (type != NFT_DATA_VALUE) @@ -11764,8 +11833,6 @@ static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data, switch (data->verdict.code) { case NF_ACCEPT: case NF_DROP: - case NF_QUEUE: - break; case NFT_CONTINUE: case NFT_BREAK: case NFT_RETURN: @@ -11800,6 +11867,11 @@ static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data, data->verdict.chain = chain; break; + case NF_QUEUE: + /* The nft_queue expression is used for this purpose, an + * immediate NF_QUEUE verdict should not ever be seen here. + */ + fallthrough; default: return -EINVAL; } diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c index 97248963a7d3b..71a248cca746a 100644 --- a/net/netfilter/nfnetlink_cthelper.c +++ b/net/netfilter/nfnetlink_cthelper.c @@ -603,10 +603,10 @@ nfnl_cthelper_dump_table(struct sk_buff *skb, struct netlink_callback *cb) goto out; } } - } - if (cb->args[1]) { - cb->args[1] = 0; - goto restart; + if (cb->args[1]) { + cb->args[1] = 0; + goto restart; + } } out: rcu_read_unlock(); diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index bfcb9cd335bff..b1f3eda85989c 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -361,10 +361,10 @@ static void __nfulnl_send(struct nfulnl_instance *inst) { if (inst->qlen > 1) { - struct nlmsghdr *nlh = nlmsg_put(inst->skb, 0, 0, - NLMSG_DONE, - sizeof(struct nfgenmsg), - 0); + struct nlmsghdr *nlh = nfnl_msg_put(inst->skb, 0, 0, + NLMSG_DONE, 0, + AF_UNSPEC, NFNETLINK_V0, + htons(inst->group_num)); if (WARN_ONCE(!nlh, "bad nlskb size: %u, tailroom %d\n", inst->skb->len, skb_tailroom(inst->skb))) { kfree_skb(inst->skb); @@ -647,15 +647,11 @@ __build_packet_message(struct nfnl_log_net *log, if (data_len) { struct nlattr *nla; - int size = nla_attr_size(data_len); - if (skb_tailroom(inst->skb) < nla_total_size(data_len)) + nla = nla_reserve(inst->skb, NFULA_PAYLOAD, data_len); + if (!nla) goto nla_put_failure; - nla = skb_put(inst->skb, nla_total_size(data_len)); - nla->nla_type = NFULA_PAYLOAD; - nla->nla_len = size; - if (skb_copy_bits(skb, 0, nla_data(nla), data_len)) BUG(); } @@ -730,7 +726,7 @@ nfulnl_log_packet(struct net *net, + nla_total_size(plen) /* prefix */ + nla_total_size(sizeof(struct nfulnl_msg_packet_hw)) + nla_total_size(sizeof(struct nfulnl_msg_packet_timestamp)) - + nla_total_size(sizeof(struct nfgenmsg)); /* NLMSG_DONE */ + + nlmsg_total_size(sizeof(struct nfgenmsg)); /* NLMSG_DONE */ if (in && skb_mac_header_was_set(skb)) { size += nla_total_size(skb->dev->hard_header_len) diff --git a/net/netfilter/nfnetlink_osf.c b/net/netfilter/nfnetlink_osf.c index c0fc431991e88..9fc9544d4bc53 100644 --- a/net/netfilter/nfnetlink_osf.c +++ b/net/netfilter/nfnetlink_osf.c @@ -302,7 +302,9 @@ static int nfnl_osf_add_callback(struct sk_buff *skb, { struct nf_osf_user_finger *f; struct nf_osf_finger *kf = NULL, *sf; + unsigned int tot_opt_len = 0; int err = 0; + int i; if (!capable(CAP_NET_ADMIN)) return -EPERM; @@ -318,6 +320,17 @@ static int nfnl_osf_add_callback(struct sk_buff *skb, if (f->opt_num > ARRAY_SIZE(f->opt)) return -EINVAL; + for (i = 0; i < f->opt_num; i++) { + if (!f->opt[i].length || f->opt[i].length > MAX_IPOPTLEN) + return -EINVAL; + if (f->opt[i].kind == OSFOPT_MSS && f->opt[i].length < 4) + return -EINVAL; + + tot_opt_len += f->opt[i].length; + if (tot_opt_len > MAX_IPOPTLEN) + return -EINVAL; + } + if (!memchr(f->genre, 0, MAXGENRELEN) || !memchr(f->subtype, 0, MAXGENRELEN) || !memchr(f->version, 0, MAXGENRELEN)) diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 8b7b39d8a1091..fe5942535245d 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include #include @@ -47,6 +49,8 @@ #endif #define NFQNL_QMAX_DEFAULT 1024 +#define NFQNL_HASH_MIN 8 +#define NFQNL_HASH_MAX 32768 /* We're using struct nlattr which has 16bit nla_len. Note that nla_len * includes the header length. Thus, the maximum packet length that we @@ -58,7 +62,8 @@ struct nfqnl_instance { struct hlist_node hlist; /* global list of queues */ - struct rcu_head rcu; + struct rhashtable nfqnl_packet_map; + struct rcu_work rwork; u32 peer_portid; unsigned int queue_maxlen; @@ -82,6 +87,7 @@ struct nfqnl_instance { typedef int (*nfqnl_cmpfn)(struct nf_queue_entry *, unsigned long); +static struct workqueue_struct *nfq_cleanup_wq __read_mostly; static unsigned int nfnl_queue_net_id __read_mostly; #define INSTANCE_BUCKETS 16 @@ -100,6 +106,15 @@ static inline u_int8_t instance_hashfn(u_int16_t queue_num) return ((queue_num >> 8) ^ queue_num) % INSTANCE_BUCKETS; } +static const struct rhashtable_params nfqnl_rhashtable_params = { + .head_offset = offsetof(struct nf_queue_entry, hash_node), + .key_offset = offsetof(struct nf_queue_entry, id), + .key_len = sizeof(u32), + .automatic_shrinking = true, + .min_size = NFQNL_HASH_MIN, + .max_size = NFQNL_HASH_MAX, +}; + static struct nfqnl_instance * instance_lookup(struct nfnl_queue_net *q, u_int16_t queue_num) { @@ -121,17 +136,9 @@ instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid) unsigned int h; int err; - spin_lock(&q->instances_lock); - if (instance_lookup(q, queue_num)) { - err = -EEXIST; - goto out_unlock; - } - - inst = kzalloc(sizeof(*inst), GFP_ATOMIC); - if (!inst) { - err = -ENOMEM; - goto out_unlock; - } + inst = kzalloc(sizeof(*inst), GFP_KERNEL_ACCOUNT); + if (!inst) + return ERR_PTR(-ENOMEM); inst->queue_num = queue_num; inst->peer_portid = portid; @@ -141,9 +148,19 @@ instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid) spin_lock_init(&inst->lock); INIT_LIST_HEAD(&inst->queue_list); + err = rhashtable_init(&inst->nfqnl_packet_map, &nfqnl_rhashtable_params); + if (err < 0) + goto out_free; + + spin_lock(&q->instances_lock); + if (instance_lookup(q, queue_num)) { + err = -EEXIST; + goto out_unlock; + } + if (!try_module_get(THIS_MODULE)) { err = -EAGAIN; - goto out_free; + goto out_unlock; } h = instance_hashfn(queue_num); @@ -153,25 +170,29 @@ instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid) return inst; -out_free: - kfree(inst); out_unlock: spin_unlock(&q->instances_lock); + rhashtable_destroy(&inst->nfqnl_packet_map); +out_free: + kfree(inst); return ERR_PTR(err); } static void nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data); -static void -instance_destroy_rcu(struct rcu_head *head) +static void instance_destroy_work(struct work_struct *work) { - struct nfqnl_instance *inst = container_of(head, struct nfqnl_instance, - rcu); + struct nfqnl_instance *inst; + inst = container_of(to_rcu_work(work), struct nfqnl_instance, + rwork); rcu_read_lock(); nfqnl_flush(inst, NULL, 0); rcu_read_unlock(); + + rhashtable_destroy(&inst->nfqnl_packet_map); + kfree(inst); module_put(THIS_MODULE); } @@ -180,7 +201,9 @@ static void __instance_destroy(struct nfqnl_instance *inst) { hlist_del_rcu(&inst->hlist); - call_rcu(&inst->rcu, instance_destroy_rcu); + + INIT_RCU_WORK(&inst->rwork, instance_destroy_work); + queue_rcu_work(nfq_cleanup_wq, &inst->rwork); } static void @@ -191,16 +214,27 @@ instance_destroy(struct nfnl_queue_net *q, struct nfqnl_instance *inst) spin_unlock(&q->instances_lock); } -static inline void +static int __enqueue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { - list_add_tail(&entry->list, &queue->queue_list); - queue->queue_total++; + int err; + + err = rhashtable_insert_fast(&queue->nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); + if (unlikely(err)) + return err; + + list_add_tail(&entry->list, &queue->queue_list); + queue->queue_total++; + + return 0; } static void __dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { + rhashtable_remove_fast(&queue->nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); list_del(&entry->list); queue->queue_total--; } @@ -208,16 +242,11 @@ __dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) static struct nf_queue_entry * find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id) { - struct nf_queue_entry *entry = NULL, *i; + struct nf_queue_entry *entry; spin_lock_bh(&queue->lock); - - list_for_each_entry(i, &queue->queue_list, list) { - if (i->id == id) { - entry = i; - break; - } - } + entry = rhashtable_lookup_fast(&queue->nfqnl_packet_map, &id, + nfqnl_rhashtable_params); if (entry) __dequeue_entry(queue, entry); @@ -369,6 +398,34 @@ static void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) nf_queue_entry_free(entry); } +/* return true if the entry has an unconfirmed conntrack attached that isn't owned by us + * exclusively. + */ +static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry, bool *is_unconfirmed) +{ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) + struct nf_conn *ct = (void *)skb_nfct(entry->skb); + + if (!ct || nf_ct_is_confirmed(ct)) + return false; + + if (is_unconfirmed) + *is_unconfirmed = true; + + /* in some cases skb_clone() can occur after initial conntrack + * pickup, but conntrack assumes exclusive skb->_nfct ownership for + * unconfirmed entries. + * + * This happens for br_netfilter and with ip multicast routing. + * This can't be solved with serialization here because one clone + * could have been queued for local delivery or could be transmitted + * in parallel on another CPU. + */ + return refcount_read(&ct->ct_general.use) > 1; +#endif + return false; +} + static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) { const struct nf_ct_hook *ct_hook; @@ -396,6 +453,24 @@ static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) break; } } + + if (verdict != NF_DROP && entry->nf_ct_is_unconfirmed) { + /* If first queued segment was already reinjected then + * there is a good chance the ct entry is now confirmed. + * + * Handle the rare cases: + * - out-of-order verdict + * - threaded userspace reinjecting in parallel + * - first segment was dropped + * + * In all of those cases we can't handle this packet + * because we can't be sure that another CPU won't modify + * nf_conn->ext in parallel which isn't allowed. + */ + if (nf_ct_drop_unconfirmed(entry, NULL)) + verdict = NF_DROP; + } + nf_reinject(entry, verdict); } @@ -407,8 +482,7 @@ nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data) spin_lock_bh(&queue->lock); list_for_each_entry_safe(entry, next, &queue->queue_list, list) { if (!cmpfn || cmpfn(entry, data)) { - list_del(&entry->list); - queue->queue_total--; + __dequeue_entry(queue, entry); nfqnl_reinject(entry, NF_DROP); } } @@ -826,49 +900,6 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, return NULL; } -static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry) -{ -#if IS_ENABLED(CONFIG_NF_CONNTRACK) - static const unsigned long flags = IPS_CONFIRMED | IPS_DYING; - struct nf_conn *ct = (void *)skb_nfct(entry->skb); - unsigned long status; - unsigned int use; - - if (!ct) - return false; - - status = READ_ONCE(ct->status); - if ((status & flags) == IPS_DYING) - return true; - - if (status & IPS_CONFIRMED) - return false; - - /* in some cases skb_clone() can occur after initial conntrack - * pickup, but conntrack assumes exclusive skb->_nfct ownership for - * unconfirmed entries. - * - * This happens for br_netfilter and with ip multicast routing. - * We can't be solved with serialization here because one clone could - * have been queued for local delivery. - */ - use = refcount_read(&ct->ct_general.use); - if (likely(use == 1)) - return false; - - /* Can't decrement further? Exclusive ownership. */ - if (!refcount_dec_not_one(&ct->ct_general.use)) - return false; - - skb_set_nfct(entry->skb, 0); - /* No nf_ct_put(): we already decremented .use and it cannot - * drop down to 0. - */ - return true; -#endif - return false; -} - static int __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, struct nf_queue_entry *entry) @@ -885,26 +916,23 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, } spin_lock_bh(&queue->lock); - if (nf_ct_drop_unconfirmed(entry)) - goto err_out_free_nskb; + if (queue->queue_total >= queue->queue_maxlen) + goto err_out_queue_drop; - if (queue->queue_total >= queue->queue_maxlen) { - if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { - failopen = 1; - err = 0; - } else { - queue->queue_dropped++; - net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", - queue->queue_total); - } - goto err_out_free_nskb; - } entry->id = ++queue->id_sequence; *packet_id_ptr = htonl(entry->id); + /* Insert into hash BEFORE unicast. If failure don't send to userspace. */ + err = __enqueue_entry(queue, entry); + if (unlikely(err)) + goto err_out_queue_drop; + /* nfnetlink_unicast will either free the nskb or add it to a socket */ err = nfnetlink_unicast(nskb, net, queue->peer_portid); if (err < 0) { + /* Unicast failed - remove entry we just inserted */ + __dequeue_entry(queue, entry); + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { failopen = 1; err = 0; @@ -914,12 +942,22 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, goto err_out_unlock; } - __enqueue_entry(queue, entry); - spin_unlock_bh(&queue->lock); return 0; -err_out_free_nskb: +err_out_queue_drop: + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { + failopen = 1; + err = 0; + } else { + queue->queue_dropped++; + + if (queue->queue_total >= queue->queue_maxlen) + net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", + queue->queue_total); + else + net_warn_ratelimited("nf_queue: hash insert failed: %d\n", err); + } kfree_skb(nskb); err_out_unlock: spin_unlock_bh(&queue->lock); @@ -998,9 +1036,10 @@ __nfqnl_enqueue_packet_gso(struct net *net, struct nfqnl_instance *queue, static int nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) { - unsigned int queued; - struct nfqnl_instance *queue; struct sk_buff *skb, *segs, *nskb; + bool ct_is_unconfirmed = false; + struct nfqnl_instance *queue; + unsigned int queued; int err = -ENOBUFS; struct net *net = entry->state.net; struct nfnl_queue_net *q = nfnl_queue_pernet(net); @@ -1024,6 +1063,15 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) break; } + /* Check if someone already holds another reference to + * unconfirmed ct. If so, we cannot queue the skb: + * concurrent modifications of nf_conn->ext are not + * allowed and we can't know if another CPU isn't + * processing the same nf_conn entry in parallel. + */ + if (nf_ct_drop_unconfirmed(entry, &ct_is_unconfirmed)) + return -EINVAL; + if (!skb_is_gso(skb) || ((queue->flags & NFQA_CFG_F_GSO) && !skb_is_gso_sctp(skb))) return __nfqnl_enqueue_packet(net, queue, entry); @@ -1037,7 +1085,23 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) goto out_err; queued = 0; err = 0; + skb_list_walk_safe(segs, segs, nskb) { + if (ct_is_unconfirmed && queued > 0) { + /* skb_gso_segment() increments the ct refcount. + * This is a problem for unconfirmed (not in hash) + * entries, those can race when reinjections happen + * in parallel. + * + * Annotate this for all queued entries except the + * first one. + * + * As long as the first one is reinjected first it + * will do the confirmation for us. + */ + entry->nf_ct_is_unconfirmed = ct_is_unconfirmed; + } + if (err == 0) err = __nfqnl_enqueue_packet_gso(net, queue, segs, entry); @@ -1445,8 +1509,10 @@ static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info, if (entry->state.pf == PF_BRIDGE) { err = nfqa_parse_bridge(entry, nfqa); - if (err < 0) + if (err < 0) { + nfqnl_reinject(entry, NF_DROP); return err; + } } if (nfqa[NFQA_PAYLOAD]) { @@ -1498,7 +1564,8 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, struct nfqnl_msg_config_cmd *cmd = NULL; struct nfqnl_instance *queue; __u32 flags = 0, mask = 0; - int ret = 0; + + WARN_ON_ONCE(!lockdep_nfnl_is_held(NFNL_SUBSYS_QUEUE)); if (nfqa[NFQA_CFG_CMD]) { cmd = nla_data(nfqa[NFQA_CFG_CMD]); @@ -1544,47 +1611,44 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, } } + /* Lookup queue under RCU. After peer_portid check (or for new queue + * in BIND case), the queue is owned by the socket sending this message. + * A socket cannot simultaneously send a message and close, so while + * processing this CONFIG message, nfqnl_rcv_nl_event() (triggered by + * socket close) cannot destroy this queue. Safe to use without RCU. + */ rcu_read_lock(); queue = instance_lookup(q, queue_num); if (queue && queue->peer_portid != NETLINK_CB(skb).portid) { - ret = -EPERM; - goto err_out_unlock; + rcu_read_unlock(); + return -EPERM; } + rcu_read_unlock(); if (cmd != NULL) { switch (cmd->command) { case NFQNL_CFG_CMD_BIND: - if (queue) { - ret = -EBUSY; - goto err_out_unlock; - } - queue = instance_create(q, queue_num, - NETLINK_CB(skb).portid); - if (IS_ERR(queue)) { - ret = PTR_ERR(queue); - goto err_out_unlock; - } + if (queue) + return -EBUSY; + queue = instance_create(q, queue_num, NETLINK_CB(skb).portid); + if (IS_ERR(queue)) + return PTR_ERR(queue); break; case NFQNL_CFG_CMD_UNBIND: - if (!queue) { - ret = -ENODEV; - goto err_out_unlock; - } + if (!queue) + return -ENODEV; instance_destroy(q, queue); - goto err_out_unlock; + return 0; case NFQNL_CFG_CMD_PF_BIND: case NFQNL_CFG_CMD_PF_UNBIND: break; default: - ret = -ENOTSUPP; - goto err_out_unlock; + return -EOPNOTSUPP; } } - if (!queue) { - ret = -ENODEV; - goto err_out_unlock; - } + if (!queue) + return -ENODEV; if (nfqa[NFQA_CFG_PARAMS]) { struct nfqnl_msg_config_params *params = @@ -1609,9 +1673,7 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, spin_unlock_bh(&queue->lock); } -err_out_unlock: - rcu_read_unlock(); - return ret; + return 0; } static const struct nfnl_callback nfqnl_cb[NFQNL_MSG_MAX] = { @@ -1781,35 +1843,38 @@ static int __init nfnetlink_queue_init(void) { int status; + nfq_cleanup_wq = alloc_ordered_workqueue("nfq_workqueue", 0); + if (!nfq_cleanup_wq) + return -ENOMEM; + status = register_pernet_subsys(&nfnl_queue_net_ops); - if (status < 0) { - pr_err("failed to register pernet ops\n"); - goto out; - } + if (status < 0) + goto cleanup_pernet_subsys; - netlink_register_notifier(&nfqnl_rtnl_notifier); - status = nfnetlink_subsys_register(&nfqnl_subsys); - if (status < 0) { - pr_err("failed to create netlink socket\n"); - goto cleanup_netlink_notifier; - } + status = netlink_register_notifier(&nfqnl_rtnl_notifier); + if (status < 0) + goto cleanup_rtnl_notifier; status = register_netdevice_notifier(&nfqnl_dev_notifier); - if (status < 0) { - pr_err("failed to register netdevice notifier\n"); - goto cleanup_netlink_subsys; - } + if (status < 0) + goto cleanup_dev_notifier; + + status = nfnetlink_subsys_register(&nfqnl_subsys); + if (status < 0) + goto cleanup_nfqnl_subsys; nf_register_queue_handler(&nfqh); return status; -cleanup_netlink_subsys: - nfnetlink_subsys_unregister(&nfqnl_subsys); -cleanup_netlink_notifier: +cleanup_nfqnl_subsys: + unregister_netdevice_notifier(&nfqnl_dev_notifier); +cleanup_dev_notifier: netlink_unregister_notifier(&nfqnl_rtnl_notifier); +cleanup_rtnl_notifier: unregister_pernet_subsys(&nfnl_queue_net_ops); -out: +cleanup_pernet_subsys: + destroy_workqueue(nfq_cleanup_wq); return status; } @@ -1820,7 +1885,7 @@ static void __exit nfnetlink_queue_fini(void) nfnetlink_subsys_unregister(&nfqnl_subsys); netlink_unregister_notifier(&nfqnl_rtnl_notifier); unregister_pernet_subsys(&nfnl_queue_net_ops); - + destroy_workqueue(nfq_cleanup_wq); rcu_barrier(); /* Wait for completion of call_rcu()'s */ } diff --git a/net/netfilter/nft_chain_filter.c b/net/netfilter/nft_chain_filter.c index b16185e9a6dd7..041426e3bdbf1 100644 --- a/net/netfilter/nft_chain_filter.c +++ b/net/netfilter/nft_chain_filter.c @@ -344,7 +344,7 @@ static int nft_netdev_event(unsigned long event, struct net_device *dev, break; case NETDEV_REGISTER: /* NOP if not matching or already registered */ - if (!match || (changename && ops)) + if (!match || ops) continue; ops = kmemdup(&basechain->ops, diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index 72711d62fddfa..08f620311b03f 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -134,7 +134,8 @@ static void nft_target_eval_bridge(const struct nft_expr *expr, } static const struct nla_policy nft_target_policy[NFTA_TARGET_MAX + 1] = { - [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN, }, [NFTA_TARGET_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_TARGET_INFO] = { .type = NLA_BINARY }, }; @@ -434,7 +435,8 @@ static void nft_match_eval(const struct nft_expr *expr, } static const struct nla_policy nft_match_policy[NFTA_MATCH_MAX + 1] = { - [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN }, [NFTA_MATCH_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_MATCH_INFO] = { .type = NLA_BINARY }, }; @@ -693,7 +695,12 @@ static int nfnl_compat_get_rcu(struct sk_buff *skb, name = nla_data(tb[NFTA_COMPAT_NAME]); rev = ntohl(nla_get_be32(tb[NFTA_COMPAT_REV])); - target = ntohl(nla_get_be32(tb[NFTA_COMPAT_TYPE])); + /* x_tables api checks for 'target == 1' to mean target, + * everything else means 'match'. + * In x_tables world, the number is set by kernel, not + * userspace. + */ + target = nla_get_be32(tb[NFTA_COMPAT_TYPE]) == htonl(1); switch(family) { case AF_INET: diff --git a/net/netfilter/nft_connlimit.c b/net/netfilter/nft_connlimit.c index fc35a11cdca20..714a594859357 100644 --- a/net/netfilter/nft_connlimit.c +++ b/net/netfilter/nft_connlimit.c @@ -24,28 +24,22 @@ static inline void nft_connlimit_do_eval(struct nft_connlimit *priv, const struct nft_pktinfo *pkt, const struct nft_set_ext *ext) { - const struct nf_conntrack_zone *zone = &nf_ct_zone_dflt; - const struct nf_conntrack_tuple *tuple_ptr; - struct nf_conntrack_tuple tuple; - enum ip_conntrack_info ctinfo; - const struct nf_conn *ct; unsigned int count; + int err; - tuple_ptr = &tuple; - - ct = nf_ct_get(pkt->skb, &ctinfo); - if (ct != NULL) { - tuple_ptr = &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; - zone = nf_ct_zone(ct); - } else if (!nf_ct_get_tuplepr(pkt->skb, skb_network_offset(pkt->skb), - nft_pf(pkt), nft_net(pkt), &tuple)) { - regs->verdict.code = NF_DROP; - return; - } - - if (nf_conncount_add(nft_net(pkt), priv->list, tuple_ptr, zone)) { - regs->verdict.code = NF_DROP; - return; + err = nf_conncount_add_skb(nft_net(pkt), pkt->skb, nft_pf(pkt), priv->list); + if (err) { + if (err == -EEXIST) { + /* Call gc to update the list count if any connection has + * been closed already. This is useful for softlimit + * connections like limiting bandwidth based on a number + * of open connections. + */ + nf_conncount_gc_list(nft_net(pkt), priv->list); + } else { + regs->verdict.code = NF_DROP; + return; + } } count = READ_ONCE(priv->list->count); @@ -238,13 +232,8 @@ static void nft_connlimit_destroy_clone(const struct nft_ctx *ctx, static bool nft_connlimit_gc(struct net *net, const struct nft_expr *expr) { struct nft_connlimit *priv = nft_expr_priv(expr); - bool ret; - - local_bh_disable(); - ret = nf_conncount_gc_list(net, priv->list); - local_bh_enable(); - return ret; + return nf_conncount_gc_list(net, priv->list); } static struct nft_expr_type nft_connlimit_type; diff --git a/net/netfilter/nft_counter.c b/net/netfilter/nft_counter.c index cc73253294963..0d70325280cc5 100644 --- a/net/netfilter/nft_counter.c +++ b/net/netfilter/nft_counter.c @@ -117,8 +117,8 @@ static void nft_counter_reset(struct nft_counter_percpu_priv *priv, nft_sync = this_cpu_ptr(&nft_counter_sync); u64_stats_update_begin(nft_sync); - u64_stats_add(&this_cpu->packets, -total->packets); - u64_stats_add(&this_cpu->bytes, -total->bytes); + u64_stats_sub(&this_cpu->packets, total->packets); + u64_stats_sub(&this_cpu->bytes, total->bytes); u64_stats_update_end(nft_sync); local_bh_enable(); diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 6f2ae7cad7310..4f52ca4c48d51 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -23,6 +23,7 @@ #include #include #include +#include "nf_internals.h" struct nft_ct_helper_obj { struct nf_conntrack_helper *helper4; @@ -543,6 +544,7 @@ static void __nft_ct_set_destroy(const struct nft_ctx *ctx, struct nft_ct *priv) #endif #ifdef CONFIG_NF_CONNTRACK_ZONES case NFT_CT_ZONE: + nf_queue_nf_hook_drop(ctx->net); mutex_lock(&nft_ct_pcpu_mutex); if (--nft_ct_pcpu_template_refcnt == 0) nft_ct_tmpl_put_pcpu(); @@ -1016,9 +1018,10 @@ static void nft_ct_timeout_obj_destroy(const struct nft_ctx *ctx, struct nft_ct_timeout_obj *priv = nft_obj_data(obj); struct nf_ct_timeout *timeout = priv->timeout; + nf_queue_nf_hook_drop(ctx->net); nf_ct_untimeout(ctx->net, timeout); nf_ct_netns_put(ctx->net, ctx->family); - kfree(priv->timeout); + kfree_rcu(priv->timeout, rcu); } static int nft_ct_timeout_obj_dump(struct sk_buff *skb, @@ -1148,6 +1151,7 @@ static void nft_ct_helper_obj_destroy(const struct nft_ctx *ctx, { struct nft_ct_helper_obj *priv = nft_obj_data(obj); + nf_queue_nf_hook_drop(ctx->net); if (priv->helper4) nf_conntrack_helper_put(priv->helper4); if (priv->helper6) diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index 7807d81296646..9123277be03ce 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -30,18 +30,26 @@ static int nft_dynset_expr_setup(const struct nft_dynset *priv, const struct nft_set_ext *ext) { struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext); + struct nft_ctx ctx = { + .net = read_pnet(&priv->set->net), + .family = priv->set->table->family, + }; struct nft_expr *expr; int i; for (i = 0; i < priv->num_exprs; i++) { expr = nft_setelem_expr_at(elem_expr, elem_expr->size); if (nft_expr_clone(expr, priv->expr_array[i], GFP_ATOMIC) < 0) - return -1; + goto err_out; elem_expr->size += priv->expr_array[i]->ops->size; } return 0; +err_out: + nft_set_elem_expr_destroy(&ctx, elem_expr); + + return -1; } struct nft_elem_priv *nft_dynset_new(struct nft_set *set, diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c index 14dd1c0698c3c..e95e5f59a3d6a 100644 --- a/net/netfilter/nft_flow_offload.c +++ b/net/netfilter/nft_flow_offload.c @@ -141,12 +141,19 @@ static void nft_dev_path_info(const struct net_device_path_stack *stack, info->ingress_vlans |= BIT(info->num_encaps - 1); break; case DEV_PATH_BR_VLAN_TAG: + if (info->num_encaps >= NF_FLOW_TABLE_ENCAP_MAX) { + info->indev = NULL; + break; + } info->encap[info->num_encaps].id = path->bridge.vlan_id; info->encap[info->num_encaps].proto = path->bridge.vlan_proto; info->num_encaps++; break; case DEV_PATH_BR_VLAN_UNTAG: - info->num_encaps--; + if (WARN_ON_ONCE(info->num_encaps-- == 0)) { + info->indev = NULL; + break; + } break; case DEV_PATH_BR_VLAN_KEEP: break; diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c index ba01ce75d6dea..b0e571c8e3f38 100644 --- a/net/netfilter/nft_set_hash.c +++ b/net/netfilter/nft_set_hash.c @@ -374,6 +374,7 @@ static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set, { switch (iter->type) { case NFT_ITER_UPDATE: + case NFT_ITER_UPDATE_CLONE: /* only relevant for netlink dumps which use READ type */ WARN_ON_ONCE(iter->skip != 0); @@ -619,15 +620,20 @@ static struct nft_elem_priv * nft_hash_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { + const u32 *key = (const u32 *)&elem->key.val; struct nft_hash *priv = nft_set_priv(set); u8 genmask = nft_genmask_cur(net); struct nft_hash_elem *he; u32 hash; - hash = jhash(elem->key.val.data, set->klen, priv->seed); + if (set->klen == 4) + hash = jhash_1word(*key, priv->seed); + else + hash = jhash(key, set->klen, priv->seed); + hash = reciprocal_scale(hash, priv->buckets); hlist_for_each_entry_rcu(he, &priv->table[hash], node) { - if (!memcmp(nft_set_ext_key(&he->ext), elem->key.val.data, set->klen) && + if (!memcmp(nft_set_ext_key(&he->ext), key, set->klen) && nft_set_elem_active(&he->ext, genmask)) return &he->priv; } diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 112fe46788b6f..394b78a00a6a5 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -1317,8 +1317,8 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, else dup_end = dup_key; - if (!memcmp(start, dup_key->data, sizeof(*dup_key->data)) && - !memcmp(end, dup_end->data, sizeof(*dup_end->data))) { + if (!memcmp(start, dup_key->data, set->klen) && + !memcmp(end, dup_end->data, set->klen)) { *elem_priv = &dup->priv; return -EEXIST; } @@ -1641,6 +1641,7 @@ static void pipapo_drop(struct nft_pipapo_match *m, int i; nft_pipapo_for_each_field(f, i, m) { + bool last = i == m->field_count - 1; int g; for (g = 0; g < f->groups; g++) { @@ -1660,7 +1661,7 @@ static void pipapo_drop(struct nft_pipapo_match *m, } pipapo_unmap(f->mt, f->rules, rulemap[i].to, rulemap[i].n, - rulemap[i + 1].n, i == m->field_count - 1); + last ? 0 : rulemap[i + 1].n, last); if (pipapo_resize(f, f->rules, f->rules - rulemap[i].n)) { /* We can ignore this, a failure to shrink tables down * doesn't make tables invalid. @@ -1681,11 +1682,11 @@ static void nft_pipapo_gc_deactivate(struct net *net, struct nft_set *set, } /** - * pipapo_gc() - Drop expired entries from set, destroy start and end elements + * pipapo_gc_scan() - Drop expired entries from set and link them to gc list * @set: nftables API set representation * @m: Matching data */ -static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) +static void pipapo_gc_scan(struct nft_set *set, struct nft_pipapo_match *m) { struct nft_pipapo *priv = nft_set_priv(set); struct net *net = read_pnet(&set->net); @@ -1698,6 +1699,8 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) if (!gc) return; + list_add(&gc->list, &priv->gc_head); + while ((rules_f0 = pipapo_rules_same_key(m->f, first_rule))) { union nft_pipapo_map_bucket rulemap[NFT_PIPAPO_MAX_FIELDS]; const struct nft_pipapo_field *f; @@ -1725,9 +1728,13 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) * NFT_SET_ELEM_DEAD_BIT. */ if (__nft_set_elem_expired(&e->ext, tstamp)) { - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - return; + if (!nft_trans_gc_space(gc)) { + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); + if (!gc) + return; + + list_add(&gc->list, &priv->gc_head); + } nft_pipapo_gc_deactivate(net, set, e); pipapo_drop(m, rulemap); @@ -1741,10 +1748,30 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) } } - gc = nft_trans_gc_catchall_sync(gc); + priv->last_gc = jiffies; +} + +/** + * pipapo_gc_queue() - Free expired elements + * @set: nftables API set representation + */ +static void pipapo_gc_queue(struct nft_set *set) +{ + struct nft_pipapo *priv = nft_set_priv(set); + struct nft_trans_gc *gc, *next; + + /* always do a catchall cycle: */ + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); if (gc) { + gc = nft_trans_gc_catchall_sync(gc); + if (gc) + nft_trans_gc_queue_sync_done(gc); + } + + /* always purge queued gc elements. */ + list_for_each_entry_safe(gc, next, &priv->gc_head, list) { + list_del(&gc->list); nft_trans_gc_queue_sync_done(gc); - priv->last_gc = jiffies; } } @@ -1798,6 +1825,10 @@ static void pipapo_reclaim_match(struct rcu_head *rcu) * * We also need to create a new working copy for subsequent insertions and * deletions. + * + * After the live copy has been replaced by the clone, we can safely queue + * expired elements that have been collected by pipapo_gc_scan() for + * memory reclaim. */ static void nft_pipapo_commit(struct nft_set *set) { @@ -1808,7 +1839,7 @@ static void nft_pipapo_commit(struct nft_set *set) return; if (time_after_eq(jiffies, priv->last_gc + nft_set_gc_interval(set))) - pipapo_gc(set, priv->clone); + pipapo_gc_scan(set, priv->clone); old = rcu_replace_pointer(priv->match, priv->clone, nft_pipapo_transaction_mutex_held(set)); @@ -1816,6 +1847,8 @@ static void nft_pipapo_commit(struct nft_set *set) if (old) call_rcu(&old->rcu, pipapo_reclaim_match); + + pipapo_gc_queue(set); } static void nft_pipapo_abort(const struct nft_set *set) @@ -2145,13 +2178,20 @@ static void nft_pipapo_walk(const struct nft_ctx *ctx, struct nft_set *set, const struct nft_pipapo_match *m; switch (iter->type) { - case NFT_ITER_UPDATE: + case NFT_ITER_UPDATE_CLONE: m = pipapo_maybe_clone(set); if (!m) { iter->err = -ENOMEM; return; } - + nft_pipapo_do_walk(ctx, set, m, iter); + break; + case NFT_ITER_UPDATE: + if (priv->clone) + m = priv->clone; + else + m = rcu_dereference_protected(priv->match, + nft_pipapo_transaction_mutex_held(set)); nft_pipapo_do_walk(ctx, set, m, iter); break; case NFT_ITER_READ: @@ -2273,6 +2313,7 @@ static int nft_pipapo_init(const struct nft_set *set, f->mt = NULL; } + INIT_LIST_HEAD(&priv->gc_head); rcu_assign_pointer(priv->match, m); return 0; @@ -2322,6 +2363,8 @@ static void nft_pipapo_destroy(const struct nft_ctx *ctx, struct nft_pipapo *priv = nft_set_priv(set); struct nft_pipapo_match *m; + WARN_ON_ONCE(!list_empty(&priv->gc_head)); + m = rcu_dereference_protected(priv->match, true); if (priv->clone) { @@ -2370,6 +2413,7 @@ const struct nft_set_type nft_set_pipapo_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; @@ -2394,6 +2438,7 @@ const struct nft_set_type nft_set_pipapo_avx2_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; diff --git a/net/netfilter/nft_set_pipapo.h b/net/netfilter/nft_set_pipapo.h index eaab422aa56ab..9aee9a9eaeb75 100644 --- a/net/netfilter/nft_set_pipapo.h +++ b/net/netfilter/nft_set_pipapo.h @@ -156,12 +156,14 @@ struct nft_pipapo_match { * @clone: Copy where pending insertions and deletions are kept * @width: Total bytes to be matched for one packet, including padding * @last_gc: Timestamp of last garbage collection run, jiffies + * @gc_head: list of nft_trans_gc to queue up for mem reclaim */ struct nft_pipapo { struct nft_pipapo_match __rcu *match; struct nft_pipapo_match *clone; int width; unsigned long last_gc; + struct list_head gc_head; }; struct nft_pipapo_elem; diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c index 7ff90325c97fa..6395982e4d95c 100644 --- a/net/netfilter/nft_set_pipapo_avx2.c +++ b/net/netfilter/nft_set_pipapo_avx2.c @@ -242,7 +242,7 @@ static int nft_pipapo_avx2_lookup_4b_2(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -319,7 +319,7 @@ static int nft_pipapo_avx2_lookup_4b_4(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -414,7 +414,7 @@ static int nft_pipapo_avx2_lookup_4b_8(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -505,7 +505,7 @@ static int nft_pipapo_avx2_lookup_4b_12(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -641,7 +641,7 @@ static int nft_pipapo_avx2_lookup_4b_32(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -699,7 +699,7 @@ static int nft_pipapo_avx2_lookup_8b_1(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -764,7 +764,7 @@ static int nft_pipapo_avx2_lookup_8b_2(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -839,7 +839,7 @@ static int nft_pipapo_avx2_lookup_8b_4(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -925,7 +925,7 @@ static int nft_pipapo_avx2_lookup_8b_6(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; @@ -1019,7 +1019,7 @@ static int nft_pipapo_avx2_lookup_8b_16(unsigned long *map, unsigned long *fill, b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); if (last) - return b; + ret = b; if (unlikely(ret == -1)) ret = b / XSAVE_YMM_SIZE; diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index ca594161b8402..154bf2772e27d 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -10,21 +10,41 @@ #include #include #include +#include #include #include #include #include +struct nft_array_interval { + struct nft_set_ext *from; + struct nft_set_ext *to; +}; + +struct nft_array { + u32 max_intervals; + u32 num_intervals; + struct nft_array_interval *intervals; + struct rcu_head rcu_head; +}; + struct nft_rbtree { struct rb_root root; rwlock_t lock; - seqcount_rwlock_t count; + struct nft_array __rcu *array; + struct nft_array *array_next; + unsigned long start_rbe_cookie; unsigned long last_gc; + struct list_head expired; + u64 last_tstamp; }; struct nft_rbtree_elem { struct nft_elem_priv priv; - struct rb_node node; + union { + struct rb_node node; + struct list_head list; + }; struct nft_set_ext ext; }; @@ -39,6 +59,13 @@ static bool nft_rbtree_interval_start(const struct nft_rbtree_elem *rbe) return !nft_rbtree_interval_end(rbe); } +static bool nft_rbtree_interval_null(const struct nft_set *set, + const struct nft_rbtree_elem *rbe) +{ + return (!memchr_inv(nft_set_ext_key(&rbe->ext), 0, set->klen) && + nft_rbtree_interval_end(rbe)); +} + static int nft_rbtree_cmp(const struct nft_set *set, const struct nft_rbtree_elem *e1, const struct nft_rbtree_elem *e2) @@ -47,67 +74,33 @@ static int nft_rbtree_cmp(const struct nft_set *set, set->klen); } -static bool nft_rbtree_elem_expired(const struct nft_rbtree_elem *rbe) -{ - return nft_set_elem_expired(&rbe->ext); -} +struct nft_array_lookup_ctx { + const u32 *key; + u32 klen; +}; -static const struct nft_set_ext * -__nft_rbtree_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, unsigned int seq) +static int nft_array_lookup_cmp(const void *pkey, const void *entry) { - struct nft_rbtree *priv = nft_set_priv(set); - const struct nft_rbtree_elem *rbe, *interval = NULL; - u8 genmask = nft_genmask_cur(net); - const struct rb_node *parent; - int d; + const struct nft_array_interval *interval = entry; + const struct nft_array_lookup_ctx *ctx = pkey; + int a, b; - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return NULL; - - rbe = rb_entry(parent, struct nft_rbtree_elem, node); + if (!interval->from) + return 1; - d = memcmp(nft_set_ext_key(&rbe->ext), key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (interval && - !nft_rbtree_cmp(set, rbe, interval) && - nft_rbtree_interval_end(rbe) && - nft_rbtree_interval_start(interval)) - continue; - if (nft_set_elem_active(&rbe->ext, genmask) && - !nft_rbtree_elem_expired(rbe)) - interval = rbe; - } else if (d > 0) - parent = rcu_dereference_raw(parent->rb_right); - else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } - - if (nft_rbtree_elem_expired(rbe)) - return NULL; - - if (nft_rbtree_interval_end(rbe)) { - if (nft_set_is_anonymous(set)) - return NULL; - parent = rcu_dereference_raw(parent->rb_left); - interval = NULL; - continue; - } + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - return &rbe->ext; - } - } + if (a >= 0 && b < 0) + return 0; - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_rbtree_interval_start(interval)) - return &interval->ext; + if (a < 0) + return -1; - return NULL; + return 1; } INDIRECT_CALLABLE_SCOPE @@ -116,83 +109,57 @@ nft_rbtree_lookup(const struct net *net, const struct nft_set *set, const u32 *key) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - const struct nft_set_ext *ext; - - ext = __nft_rbtree_lookup(net, set, key, seq); - if (ext || !read_seqcount_retry(&priv->count, seq)) - return ext; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ext = __nft_rbtree_lookup(net, set, key, seq); - read_unlock_bh(&priv->lock); - - return ext; + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_lookup_ctx ctx = { + .key = key, + .klen = set->klen, + }; + + if (!array) + return NULL; + + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), + nft_array_lookup_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return NULL; + + return interval->from; } -static bool __nft_rbtree_get(const struct net *net, const struct nft_set *set, - const u32 *key, struct nft_rbtree_elem **elem, - unsigned int seq, unsigned int flags, u8 genmask) -{ - struct nft_rbtree_elem *rbe, *interval = NULL; - struct nft_rbtree *priv = nft_set_priv(set); - const struct rb_node *parent; - const void *this; - int d; - - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return false; - - rbe = rb_entry(parent, struct nft_rbtree_elem, node); - - this = nft_set_ext_key(&rbe->ext); - d = memcmp(this, key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (!(flags & NFT_SET_ELEM_INTERVAL_END)) - interval = rbe; - } else if (d > 0) { - parent = rcu_dereference_raw(parent->rb_right); - if (flags & NFT_SET_ELEM_INTERVAL_END) - interval = rbe; - } else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } +struct nft_array_get_ctx { + const u32 *key; + unsigned int flags; + u32 klen; +}; - if (nft_set_elem_expired(&rbe->ext)) - return false; +static int nft_array_get_cmp(const void *pkey, const void *entry) +{ + const struct nft_array_interval *interval = entry; + const struct nft_array_get_ctx *ctx = pkey; + int a, b; - if (!nft_set_ext_exists(&rbe->ext, NFT_SET_EXT_FLAGS) || - (*nft_set_ext_flags(&rbe->ext) & NFT_SET_ELEM_INTERVAL_END) == - (flags & NFT_SET_ELEM_INTERVAL_END)) { - *elem = rbe; - return true; - } + if (!interval->from) + return 1; - if (nft_rbtree_interval_end(rbe)) - interval = NULL; + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - parent = rcu_dereference_raw(parent->rb_left); - } + if (a >= 0) { + if (ctx->flags & NFT_SET_ELEM_INTERVAL_END && b <= 0) + return 0; + else if (b < 0) + return 0; } - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_set_elem_active(&interval->ext, genmask) && - !nft_set_elem_expired(&interval->ext) && - ((!nft_rbtree_interval_end(interval) && - !(flags & NFT_SET_ELEM_INTERVAL_END)) || - (nft_rbtree_interval_end(interval) && - (flags & NFT_SET_ELEM_INTERVAL_END)))) { - *elem = interval; - return true; - } + if (a < 0) + return -1; - return false; + return 1; } static struct nft_elem_priv * @@ -200,34 +167,41 @@ nft_rbtree_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - struct nft_rbtree_elem *rbe = ERR_PTR(-ENOENT); - const u32 *key = (const u32 *)&elem->key.val; - u8 genmask = nft_genmask_cur(net); - bool ret; - - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - if (ret || !read_seqcount_retry(&priv->count, seq)) - return &rbe->priv; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - read_unlock_bh(&priv->lock); - - if (!ret) + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_get_ctx ctx = { + .key = (const u32 *)&elem->key.val, + .flags = flags, + .klen = set->klen, + }; + struct nft_rbtree_elem *rbe; + + if (!array) return ERR_PTR(-ENOENT); + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), nft_array_get_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return ERR_PTR(-ENOENT); + + if (flags & NFT_SET_ELEM_INTERVAL_END) + rbe = container_of(interval->to, struct nft_rbtree_elem, ext); + else + rbe = container_of(interval->from, struct nft_rbtree_elem, ext); + return &rbe->priv; } -static void nft_rbtree_gc_elem_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) +static void nft_rbtree_gc_elem_move(struct net *net, struct nft_set *set, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) { lockdep_assert_held_write(&priv->lock); nft_setelem_data_deactivate(net, set, &rbe->priv); rb_erase(&rbe->node, &priv->root); + + /* collected later on in commit callback */ + list_add(&rbe->list, &priv->expired); } static const struct nft_rbtree_elem * @@ -238,11 +212,6 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, struct rb_node *prev = rb_prev(&rbe->node); struct net *net = read_pnet(&set->net); struct nft_rbtree_elem *rbe_prev; - struct nft_trans_gc *gc; - - gc = nft_trans_gc_alloc(set, 0, GFP_ATOMIC); - if (!gc) - return ERR_PTR(-ENOMEM); /* search for end interval coming before this element. * end intervals don't carry a timeout extension, they @@ -260,28 +229,10 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, rbe_prev = NULL; if (prev) { rbe_prev = rb_entry(prev, struct nft_rbtree_elem, node); - nft_rbtree_gc_elem_remove(net, set, priv, rbe_prev); - - /* There is always room in this trans gc for this element, - * memory allocation never actually happens, hence, the warning - * splat in such case. No need to set NFT_SET_ELEM_DEAD_BIT, - * this is synchronous gc which never fails. - */ - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe_prev); + nft_rbtree_gc_elem_move(net, set, priv, rbe_prev); } - nft_rbtree_gc_elem_remove(net, set, priv, rbe); - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe); - - nft_trans_gc_queue_sync_done(gc); + nft_rbtree_gc_elem_move(net, set, priv, rbe); return rbe_prev; } @@ -302,16 +253,97 @@ static bool nft_rbtree_update_first(const struct nft_set *set, return false; } +/* Only for anonymous sets which do not allow updates, all element are active. */ +static struct nft_rbtree_elem *nft_rbtree_prev_active(struct nft_rbtree_elem *rbe) +{ + struct rb_node *node; + + node = rb_prev(&rbe->node); + if (!node) + return NULL; + + return rb_entry(node, struct nft_rbtree_elem, node); +} + +static struct nft_rbtree_elem * +__nft_rbtree_next_active(struct rb_node *node, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + + while (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (!nft_set_elem_active(&next_rbe->ext, genmask)) { + node = rb_next(node); + continue; + } + + return next_rbe; + } + + return NULL; +} + +static struct nft_rbtree_elem * +nft_rbtree_next_active(struct nft_rbtree_elem *rbe, u8 genmask) +{ + return __nft_rbtree_next_active(rb_next(&rbe->node), genmask); +} + +static void nft_rbtree_maybe_reset_start_cookie(struct nft_rbtree *priv, + u64 tstamp) +{ + if (priv->last_tstamp != tstamp) { + priv->start_rbe_cookie = 0; + priv->last_tstamp = tstamp; + } +} + +static void nft_rbtree_set_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + priv->start_rbe_cookie = (unsigned long)rbe; +} + +static bool nft_rbtree_cmp_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + return priv->start_rbe_cookie == (unsigned long)rbe; +} + +static bool nft_rbtree_insert_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_active(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, struct nft_rbtree_elem *new, - struct nft_elem_priv **elem_priv) + struct nft_elem_priv **elem_priv, u64 tstamp) { - struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL; + struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL, *rbe_prev; struct rb_node *node, *next, *parent, **p, *first = NULL; struct nft_rbtree *priv = nft_set_priv(set); u8 cur_genmask = nft_genmask_cur(net); u8 genmask = nft_genmask_next(net); - u64 tstamp = nft_net_tstamp(net); int d; /* Descend the tree to search for an existing element greater than the @@ -417,12 +449,18 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, } } + if (nft_rbtree_interval_null(set, new)) + priv->start_rbe_cookie = 0; + else if (nft_rbtree_interval_start(new) && priv->start_rbe_cookie) + priv->start_rbe_cookie = 0; + /* - new start element matching existing start element: full overlap * reported as -EEXIST, cleared by caller if NLM_F_EXCL is not given. */ if (rbe_ge && !nft_rbtree_cmp(set, new, rbe_ge) && nft_rbtree_interval_start(rbe_ge) == nft_rbtree_interval_start(new)) { *elem_priv = &rbe_ge->priv; + nft_rbtree_set_start_cookie(priv, rbe_ge); return -EEXIST; } @@ -431,18 +469,37 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, */ if (rbe_le && !nft_rbtree_cmp(set, new, rbe_le) && nft_rbtree_interval_end(rbe_le) == nft_rbtree_interval_end(new)) { + /* - ignore null interval, otherwise NLM_F_CREATE bogusly + * reports EEXIST. + */ + if (nft_rbtree_interval_null(set, new)) + return -ECANCELED; + *elem_priv = &rbe_le->priv; + + /* - start and end element belong to the same interval. */ + if (!nft_rbtree_insert_same_interval(net, priv, rbe_le)) + return -ENOTEMPTY; + return -EEXIST; } /* - new start element with existing closest, less or equal key value * being a start element: partial overlap, reported as -ENOTEMPTY. * Anonymous sets allow for two consecutive start element since they - * are constant, skip them to avoid bogus overlap reports. + * are constant, but validate that this new start element does not + * sit in between an existing start and end elements: partial overlap, + * reported as -ENOTEMPTY. */ - if (!nft_set_is_anonymous(set) && rbe_le && - nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) - return -ENOTEMPTY; + if (rbe_le && + nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) { + if (!nft_set_is_anonymous(set)) + return -ENOTEMPTY; + + rbe_prev = nft_rbtree_prev_active(rbe_le); + if (rbe_prev && nft_rbtree_interval_end(rbe_prev)) + return -ENOTEMPTY; + } /* - new end element with existing closest, less or equal key value * being a end element: partial overlap, reported as -ENOTEMPTY. @@ -481,14 +538,159 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, return 0; } +static int nft_array_intervals_alloc(struct nft_array *array, u32 max_intervals) +{ + struct nft_array_interval *intervals; + + intervals = kvcalloc(max_intervals, sizeof(struct nft_array_interval), + GFP_KERNEL_ACCOUNT); + if (!intervals) + return -ENOMEM; + + if (array->intervals) + kvfree(array->intervals); + + array->intervals = intervals; + array->max_intervals = max_intervals; + + return 0; +} + +static struct nft_array *nft_array_alloc(u32 max_intervals) +{ + struct nft_array *array; + + array = kzalloc(sizeof(*array), GFP_KERNEL_ACCOUNT); + if (!array) + return NULL; + + if (nft_array_intervals_alloc(array, max_intervals) < 0) { + kfree(array); + return NULL; + } + + return array; +} + +/* Similar to nft_rbtree_{u,k}size to hide details to userspace, but consider + * packed representation coming from userspace for anonymous sets too. + */ +static u32 nft_array_elems(const struct nft_set *set) +{ + u32 nelems = atomic_read(&set->nelems) - set->ndeact; + + /* Adjacent intervals are represented with a single start element in + * anonymous sets, use the current element counter as is. + */ + if (nft_set_is_anonymous(set)) + return nelems; + + /* Add extra room for never matching interval at the beginning and open + * interval at the end which only use a single element to represent it. + * The conversion to array will compact intervals, this allows reduce + * memory consumption. + */ + return (nelems / 2) + 2; +} + +#define NFT_ARRAY_INITIAL_SIZE 1024 +#define NFT_ARRAY_INITIAL_ANON_SIZE 16 +#define NFT_ARRAY_INITIAL_ANON_THRESH (8192U / sizeof(struct nft_array_interval)) + +static int nft_array_may_resize(const struct nft_set *set, bool flush) +{ + u32 initial_intervals, max_intervals, new_max_intervals, delta; + u32 shrinked_max_intervals, nelems = nft_array_elems(set); + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array; + + if (nft_set_is_anonymous(set)) + initial_intervals = NFT_ARRAY_INITIAL_ANON_SIZE; + else + initial_intervals = NFT_ARRAY_INITIAL_SIZE; + + if (priv->array_next) { + max_intervals = priv->array_next->max_intervals; + new_max_intervals = priv->array_next->max_intervals; + } else { + if (priv->array) { + max_intervals = priv->array->max_intervals; + new_max_intervals = priv->array->max_intervals; + } else { + max_intervals = 0; + new_max_intervals = initial_intervals; + } + } + + if (nft_set_is_anonymous(set)) + goto maybe_grow; + + if (flush) { + /* Set flush just started, nelems still report elements.*/ + nelems = 0; + new_max_intervals = NFT_ARRAY_INITIAL_SIZE; + goto realloc_array; + } + + if (check_add_overflow(new_max_intervals, new_max_intervals, + &shrinked_max_intervals)) + return -EOVERFLOW; + + shrinked_max_intervals = DIV_ROUND_UP(shrinked_max_intervals, 3); + + if (shrinked_max_intervals > NFT_ARRAY_INITIAL_SIZE && + nelems < shrinked_max_intervals) { + new_max_intervals = shrinked_max_intervals; + goto realloc_array; + } +maybe_grow: + if (nelems > new_max_intervals) { + if (nft_set_is_anonymous(set) && + new_max_intervals < NFT_ARRAY_INITIAL_ANON_THRESH) { + new_max_intervals <<= 1; + } else { + delta = new_max_intervals >> 1; + if (check_add_overflow(new_max_intervals, delta, + &new_max_intervals)) + return -EOVERFLOW; + } + } + +realloc_array: + if (WARN_ON_ONCE(nelems > new_max_intervals)) + return -ENOMEM; + + if (priv->array_next) { + if (max_intervals == new_max_intervals) + return 0; + + if (nft_array_intervals_alloc(priv->array_next, new_max_intervals) < 0) + return -ENOMEM; + } else { + array = nft_array_alloc(new_max_intervals); + if (!array) + return -ENOMEM; + + priv->array_next = array; + } + + return 0; +} + static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, struct nft_elem_priv **elem_priv) { struct nft_rbtree_elem *rbe = nft_elem_priv_cast(elem->priv); struct nft_rbtree *priv = nft_set_priv(set); + u64 tstamp = nft_net_tstamp(net); int err; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_array_may_resize(set, false) < 0) + return -ENOMEM; + do { if (fatal_signal_pending(current)) return -EINTR; @@ -496,9 +698,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, cond_resched(); write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); - err = __nft_rbtree_insert(net, set, rbe, elem_priv); - write_seqcount_end(&priv->count); + err = __nft_rbtree_insert(net, set, rbe, elem_priv, tstamp); write_unlock_bh(&priv->lock); } while (err == -EAGAIN); @@ -508,9 +708,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, static void nft_rbtree_erase(struct nft_rbtree *priv, struct nft_rbtree_elem *rbe) { write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); rb_erase(&rbe->node, &priv->root); - write_seqcount_end(&priv->count); write_unlock_bh(&priv->lock); } @@ -533,6 +731,48 @@ static void nft_rbtree_activate(const struct net *net, nft_clear(net, &rbe->ext); } +static struct nft_rbtree_elem * +nft_rbtree_next_inactive(struct nft_rbtree_elem *rbe, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + struct rb_node *node; + + node = rb_next(&rbe->node); + if (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (nft_rbtree_interval_start(next_rbe) && + !nft_set_elem_active(&next_rbe->ext, genmask)) + return next_rbe; + } + + return NULL; +} + +static bool nft_rbtree_deactivate_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_inactive(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static void nft_rbtree_flush(const struct net *net, const struct nft_set *set, struct nft_elem_priv *elem_priv) @@ -547,12 +787,21 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem) { struct nft_rbtree_elem *rbe, *this = nft_elem_priv_cast(elem->priv); - const struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree *priv = nft_set_priv(set); const struct rb_node *parent = priv->root.rb_node; u8 genmask = nft_genmask_next(net); u64 tstamp = nft_net_tstamp(net); int d; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_rbtree_interval_start(this) || + nft_rbtree_interval_null(set, this)) + priv->start_rbe_cookie = 0; + + if (nft_array_may_resize(set, false) < 0) + return NULL; + while (parent != NULL) { rbe = rb_entry(parent, struct nft_rbtree_elem, node); @@ -577,6 +826,12 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, parent = parent->rb_left; continue; } + + if (nft_rbtree_interval_start(rbe)) + nft_rbtree_set_start_cookie(priv, rbe); + else if (!nft_rbtree_deactivate_same_interval(net, priv, rbe)) + return NULL; + nft_rbtree_flush(net, set, &rbe->priv); return &rbe->priv; } @@ -613,8 +868,15 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, struct nft_rbtree *priv = nft_set_priv(set); switch (iter->type) { + case NFT_ITER_UPDATE_CLONE: + if (nft_array_may_resize(set, true) < 0) { + iter->err = -ENOMEM; + break; + } + fallthrough; case NFT_ITER_UPDATE: lockdep_assert_held(&nft_pernet(ctx->net)->commit_mutex); + nft_rbtree_do_walk(ctx, set, iter); break; case NFT_ITER_READ: @@ -629,29 +891,13 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, } } -static void nft_rbtree_gc_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) -{ - nft_setelem_data_deactivate(net, set, &rbe->priv); - nft_rbtree_erase(priv, rbe); -} - -static void nft_rbtree_gc(struct nft_set *set) +static void nft_rbtree_gc_scan(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); struct nft_rbtree_elem *rbe, *rbe_end = NULL; struct net *net = read_pnet(&set->net); u64 tstamp = nft_net_tstamp(net); struct rb_node *node, *next; - struct nft_trans_gc *gc; - - set = nft_set_container_of(priv); - net = read_pnet(&set->net); - - gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); - if (!gc) - return; for (node = rb_first(&priv->root); node ; node = next) { next = rb_next(node); @@ -669,34 +915,46 @@ static void nft_rbtree_gc(struct nft_set *set) if (!__nft_set_elem_expired(&rbe->ext, tstamp)) continue; - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - /* end element needs to be removed first, it has * no timeout extension. */ + write_lock_bh(&priv->lock); if (rbe_end) { - nft_rbtree_gc_remove(net, set, priv, rbe_end); - nft_trans_gc_elem_add(gc, rbe_end); + nft_rbtree_gc_elem_move(net, set, priv, rbe_end); rbe_end = NULL; } - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - - nft_rbtree_gc_remove(net, set, priv, rbe); - nft_trans_gc_elem_add(gc, rbe); + nft_rbtree_gc_elem_move(net, set, priv, rbe); + write_unlock_bh(&priv->lock); } -try_later: + priv->last_gc = jiffies; +} + +static void nft_rbtree_gc_queue(struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *rbe_end; + struct nft_trans_gc *gc; + + if (list_empty(&priv->expired)) + return; - if (gc) { - gc = nft_trans_gc_catchall_sync(gc); - nft_trans_gc_queue_sync_done(gc); - priv->last_gc = jiffies; + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); + if (!gc) + return; + + list_for_each_entry_safe(rbe, rbe_end, &priv->expired, list) { + list_del(&rbe->list); + nft_trans_gc_elem_add(gc, rbe); + + gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); + if (!gc) + return; } + + gc = nft_trans_gc_catchall_sync(gc); + nft_trans_gc_queue_sync_done(gc); } static u64 nft_rbtree_privsize(const struct nlattr * const nla[], @@ -714,24 +972,45 @@ static int nft_rbtree_init(const struct nft_set *set, BUILD_BUG_ON(offsetof(struct nft_rbtree_elem, priv) != 0); rwlock_init(&priv->lock); - seqcount_rwlock_init(&priv->count, &priv->lock); priv->root = RB_ROOT; + INIT_LIST_HEAD(&priv->expired); + + priv->array = NULL; + priv->array_next = NULL; return 0; } +static void __nft_array_free(struct nft_array *array) +{ + kvfree(array->intervals); + kfree(array); +} + static void nft_rbtree_destroy(const struct nft_ctx *ctx, const struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); - struct nft_rbtree_elem *rbe; + struct nft_rbtree_elem *rbe, *next; + struct nft_array *array; struct rb_node *node; + list_for_each_entry_safe(rbe, next, &priv->expired, list) { + list_del(&rbe->list); + nf_tables_set_elem_destroy(ctx, set, &rbe->priv); + } + while ((node = priv->root.rb_node) != NULL) { rb_erase(node, &priv->root); rbe = rb_entry(node, struct nft_rbtree_elem, node); nf_tables_set_elem_destroy(ctx, set, &rbe->priv); } + + array = rcu_dereference_protected(priv->array, true); + if (array) + __nft_array_free(array); + if (priv->array_next) + __nft_array_free(priv->array_next); } static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, @@ -752,12 +1031,105 @@ static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, return true; } +static void nft_array_free_rcu(struct rcu_head *rcu_head) +{ + struct nft_array *array = container_of(rcu_head, struct nft_array, rcu_head); + + __nft_array_free(array); +} + static void nft_rbtree_commit(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *prev_rbe; + struct nft_array *old; + u32 num_intervals = 0; + struct rb_node *node; + /* No changes, skip, eg. elements updates only. */ + if (!priv->array_next) + return; + + /* GC can be performed if the binary search blob is going + * to be rebuilt. It has to be done in two phases: first + * scan tree and move all expired elements to the expired + * list. + * + * Then, after blob has been re-built and published to other + * CPUs, queue collected entries for freeing. + */ if (time_after_eq(jiffies, priv->last_gc + nft_set_gc_interval(set))) - nft_rbtree_gc(set); + nft_rbtree_gc_scan(set); + + /* Reverse walk to create an array from smaller to largest interval. */ + node = rb_last(&priv->root); + if (node) + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + else + prev_rbe = NULL; + + while (prev_rbe) { + rbe = prev_rbe; + + if (nft_rbtree_interval_start(rbe)) + priv->array_next->intervals[num_intervals].from = &rbe->ext; + else if (nft_rbtree_interval_end(rbe)) + priv->array_next->intervals[num_intervals++].to = &rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + + node = rb_prev(node); + if (!node) + break; + + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + + /* For anonymous sets, when adjacent ranges are found, + * the end element is not added to the set to pack the set + * representation. Use next start element to complete this + * interval. + */ + if (nft_rbtree_interval_start(rbe) && + nft_rbtree_interval_start(prev_rbe) && + priv->array_next->intervals[num_intervals].from) + priv->array_next->intervals[num_intervals++].to = &prev_rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + } + + if (priv->array_next->intervals[num_intervals].from) + num_intervals++; +err_out: + priv->array_next->num_intervals = num_intervals; + old = rcu_replace_pointer(priv->array, priv->array_next, + lockdep_is_held(&nft_pernet(read_pnet(&set->net))->commit_mutex)); + priv->array_next = NULL; + if (old) + call_rcu(&old->rcu_head, nft_array_free_rcu); + + /* New blob is public, queue collected entries for freeing. + * call_rcu ensures elements stay around until readers are done. + */ + nft_rbtree_gc_queue(set); +} + +static void nft_rbtree_abort(const struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array_next; + + if (!priv->array_next) + return; + + array_next = priv->array_next; + priv->array_next = NULL; + __nft_array_free(array_next); } static void nft_rbtree_gc_init(const struct nft_set *set) @@ -821,6 +1193,7 @@ const struct nft_set_type nft_set_rbtree_type = { .flush = nft_rbtree_flush, .activate = nft_rbtree_activate, .commit = nft_rbtree_commit, + .abort = nft_rbtree_abort, .gc_init = nft_rbtree_gc_init, .lookup = nft_rbtree_lookup, .walk = nft_rbtree_walk, diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index 5d3e518259859..4d3e5a31b4125 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -48,7 +48,7 @@ static void nft_synproxy_eval_v4(const struct nft_synproxy *priv, struct tcphdr *_tcph, struct synproxy_options *opts) { - struct nf_synproxy_info info = priv->info; + struct nf_synproxy_info info = READ_ONCE(priv->info); struct net *net = nft_net(pkt); struct synproxy_net *snet = synproxy_pernet(net); struct sk_buff *skb = pkt->skb; @@ -79,7 +79,7 @@ static void nft_synproxy_eval_v6(const struct nft_synproxy *priv, struct tcphdr *_tcph, struct synproxy_options *opts) { - struct nf_synproxy_info info = priv->info; + struct nf_synproxy_info info = READ_ONCE(priv->info); struct net *net = nft_net(pkt); struct synproxy_net *snet = synproxy_pernet(net); struct sk_buff *skb = pkt->skb; @@ -340,7 +340,7 @@ static void nft_synproxy_obj_update(struct nft_object *obj, struct nft_synproxy *newpriv = nft_obj_data(newobj); struct nft_synproxy *priv = nft_obj_data(obj); - priv->info = newpriv->info; + WRITE_ONCE(priv->info, newpriv->info); } static struct nft_object_type nft_synproxy_obj_type; diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index 90b7630421c44..1ca4fa9d249b8 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -501,6 +501,17 @@ int xt_check_match(struct xt_mtchk_param *par, par->match->table, par->table); return -EINVAL; } + + /* NFPROTO_UNSPEC implies NF_INET_* hooks which do not overlap with + * NF_ARP_IN,OUT,FORWARD, allow explicit extensions with NFPROTO_ARP + * support. + */ + if (par->family == NFPROTO_ARP && + par->match->family != NFPROTO_ARP) { + pr_info_ratelimited("%s_tables: %s match: not valid for this family\n", + xt_prefix[par->family], par->match->name); + return -EINVAL; + } if (par->match->hooks && (par->hook_mask & ~par->match->hooks) != 0) { char used[64], allow[64]; @@ -1016,6 +1027,18 @@ int xt_check_target(struct xt_tgchk_param *par, par->target->table, par->table); return -EINVAL; } + + /* NFPROTO_UNSPEC implies NF_INET_* hooks which do not overlap with + * NF_ARP_IN,OUT,FORWARD, allow explicit extensions with NFPROTO_ARP + * support. + */ + if (par->family == NFPROTO_ARP && + par->target->family != NFPROTO_ARP) { + pr_info_ratelimited("%s_tables: %s target: not valid for this family\n", + xt_prefix[par->family], par->target->name); + return -EINVAL; + } + if (par->target->hooks && (par->hook_mask & ~par->target->hooks) != 0) { char used[64], allow[64]; @@ -1764,7 +1787,7 @@ EXPORT_SYMBOL_GPL(xt_hook_ops_alloc); int xt_register_template(const struct xt_table *table, int (*table_init)(struct net *net)) { - int ret = -EEXIST, af = table->af; + int ret = -EBUSY, af = table->af; struct xt_template *t; mutex_lock(&xt[af].mutex); diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c index 3ba94c34297cf..498f5871c84a0 100644 --- a/net/netfilter/xt_CT.c +++ b/net/netfilter/xt_CT.c @@ -16,6 +16,7 @@ #include #include #include +#include "nf_internals.h" static inline int xt_ct_target(struct sk_buff *skb, struct nf_conn *ct) { @@ -283,6 +284,9 @@ static void xt_ct_tg_destroy(const struct xt_tgdtor_param *par, struct nf_conn_help *help; if (ct) { + if (info->helper[0] || info->timeout[0]) + nf_queue_nf_hook_drop(par->net); + help = nfct_help(ct); xt_ct_put_helper(help); diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c index d73957592c9d9..bb7af92ac82a4 100644 --- a/net/netfilter/xt_IDLETIMER.c +++ b/net/netfilter/xt_IDLETIMER.c @@ -318,6 +318,12 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par) info->timer = __idletimer_tg_find_by_label(info->label); if (info->timer) { + if (info->timer->timer_type & XT_IDLETIMER_ALARM) { + pr_debug("Adding/Replacing rule with same label and different timer type is not allowed\n"); + mutex_unlock(&list_mutex); + return -EINVAL; + } + info->timer->refcnt++; mod_timer(&info->timer->timer, secs_to_jiffies(info->timeout) + jiffies); diff --git a/net/netfilter/xt_cgroup.c b/net/netfilter/xt_cgroup.c index c437fbd59ec13..43d2ae2be628d 100644 --- a/net/netfilter/xt_cgroup.c +++ b/net/netfilter/xt_cgroup.c @@ -65,6 +65,9 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) info->priv = NULL; if (info->has_path) { + if (strnlen(info->path, sizeof(info->path)) >= sizeof(info->path)) + return -ENAMETOOLONG; + cgrp = cgroup_get_from_path(info->path); if (IS_ERR(cgrp)) { pr_info_ratelimited("invalid path, errno=%ld\n", @@ -102,6 +105,9 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) info->priv = NULL; if (info->has_path) { + if (strnlen(info->path, sizeof(info->path)) >= sizeof(info->path)) + return -ENAMETOOLONG; + cgrp = cgroup_get_from_path(info->path); if (IS_ERR(cgrp)) { pr_info_ratelimited("invalid path, errno=%ld\n", diff --git a/net/netfilter/xt_connlimit.c b/net/netfilter/xt_connlimit.c index 0189f8b6b0bd1..848287ab79cfb 100644 --- a/net/netfilter/xt_connlimit.c +++ b/net/netfilter/xt_connlimit.c @@ -31,8 +31,6 @@ connlimit_mt(const struct sk_buff *skb, struct xt_action_param *par) { struct net *net = xt_net(par); const struct xt_connlimit_info *info = par->matchinfo; - struct nf_conntrack_tuple tuple; - const struct nf_conntrack_tuple *tuple_ptr = &tuple; const struct nf_conntrack_zone *zone = &nf_ct_zone_dflt; enum ip_conntrack_info ctinfo; const struct nf_conn *ct; @@ -40,13 +38,8 @@ connlimit_mt(const struct sk_buff *skb, struct xt_action_param *par) u32 key[5]; ct = nf_ct_get(skb, &ctinfo); - if (ct != NULL) { - tuple_ptr = &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; + if (ct) zone = nf_ct_zone(ct); - } else if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), - xt_family(par), net, &tuple)) { - goto hotdrop; - } if (xt_family(par) == NFPROTO_IPV6) { const struct ipv6hdr *iph = ipv6_hdr(skb); @@ -69,10 +62,9 @@ connlimit_mt(const struct sk_buff *skb, struct xt_action_param *par) key[1] = zone->id; } - connections = nf_conncount_count(net, info->data, key, tuple_ptr, - zone); + connections = nf_conncount_count_skb(net, skb, xt_family(par), info->data, key); if (connections == 0) - /* kmalloc failed, drop it entirely */ + /* kmalloc failed or tuple couldn't be found, drop it entirely */ goto hotdrop; return (connections > info->limit) ^ !!(info->flags & XT_CONNLIMIT_INVERT); diff --git a/net/netfilter/xt_dccp.c b/net/netfilter/xt_dccp.c index e5a13ecbe67a0..037ab93e25d0a 100644 --- a/net/netfilter/xt_dccp.c +++ b/net/netfilter/xt_dccp.c @@ -62,10 +62,10 @@ dccp_find_option(u_int8_t option, return true; } - if (op[i] < 2) + if (op[i] < 2 || i == optlen - 1) i++; else - i += op[i+1]?:1; + i += op[i + 1] ? : 1; } spin_unlock_bh(&dccp_buflock); diff --git a/net/netfilter/xt_multiport.c b/net/netfilter/xt_multiport.c index 44a00f5acde8a..a1691ff405d3c 100644 --- a/net/netfilter/xt_multiport.c +++ b/net/netfilter/xt_multiport.c @@ -105,6 +105,28 @@ multiport_mt(const struct sk_buff *skb, struct xt_action_param *par) return ports_match_v1(multiinfo, ntohs(pptr[0]), ntohs(pptr[1])); } +static bool +multiport_valid_ranges(const struct xt_multiport_v1 *multiinfo) +{ + unsigned int i; + + for (i = 0; i < multiinfo->count; i++) { + if (!multiinfo->pflags[i]) + continue; + + if (++i >= multiinfo->count) + return false; + + if (multiinfo->pflags[i]) + return false; + + if (multiinfo->ports[i - 1] > multiinfo->ports[i]) + return false; + } + + return true; +} + static inline bool check(u_int16_t proto, u_int8_t ip_invflags, @@ -127,8 +149,10 @@ static int multiport_mt_check(const struct xt_mtchk_param *par) const struct ipt_ip *ip = par->entryinfo; const struct xt_multiport_v1 *multiinfo = par->matchinfo; - return check(ip->proto, ip->invflags, multiinfo->flags, - multiinfo->count) ? 0 : -EINVAL; + if (!check(ip->proto, ip->invflags, multiinfo->flags, multiinfo->count)) + return -EINVAL; + + return multiport_valid_ranges(multiinfo) ? 0 : -EINVAL; } static int multiport_mt6_check(const struct xt_mtchk_param *par) @@ -136,8 +160,10 @@ static int multiport_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_ip6 *ip = par->entryinfo; const struct xt_multiport_v1 *multiinfo = par->matchinfo; - return check(ip->proto, ip->invflags, multiinfo->flags, - multiinfo->count) ? 0 : -EINVAL; + if (!check(ip->proto, ip->invflags, multiinfo->flags, multiinfo->count)) + return -EINVAL; + + return multiport_valid_ranges(multiinfo) ? 0 : -EINVAL; } static struct xt_match multiport_mt_reg[] __read_mostly = { diff --git a/net/netfilter/xt_rateest.c b/net/netfilter/xt_rateest.c index 72324bd976af8..b1d736c15fcbe 100644 --- a/net/netfilter/xt_rateest.c +++ b/net/netfilter/xt_rateest.c @@ -91,6 +91,11 @@ static int xt_rateest_mt_checkentry(const struct xt_mtchk_param *par) goto err1; } + if (strnlen(info->name1, sizeof(info->name1)) >= sizeof(info->name1)) + return -ENAMETOOLONG; + if (strnlen(info->name2, sizeof(info->name2)) >= sizeof(info->name2)) + return -ENAMETOOLONG; + ret = -ENOENT; est1 = xt_rateest_lookup(par->net, info->name1); if (!est1) diff --git a/net/netfilter/xt_tcpmss.c b/net/netfilter/xt_tcpmss.c index 37704ab017992..0d32d4841cb32 100644 --- a/net/netfilter/xt_tcpmss.c +++ b/net/netfilter/xt_tcpmss.c @@ -61,7 +61,7 @@ tcpmss_mt(const struct sk_buff *skb, struct xt_action_param *par) return (mssval >= info->mss_min && mssval <= info->mss_max) ^ info->invert; } - if (op[i] < 2) + if (op[i] < 2 || i == optlen - 1) i++; else i += op[i+1] ? : 1; diff --git a/net/netfilter/xt_tcpudp.c b/net/netfilter/xt_tcpudp.c index e8991130a3de0..f76cf18f1a244 100644 --- a/net/netfilter/xt_tcpudp.c +++ b/net/netfilter/xt_tcpudp.c @@ -59,8 +59,10 @@ tcp_find_option(u_int8_t option, for (i = 0; i < optlen; ) { if (op[i] == option) return !invert; - if (op[i] < 2) i++; - else i += op[i+1]?:1; + if (op[i] < 2 || i == optlen - 1) + i++; + else + i += op[i + 1] ? : 1; } return invert; diff --git a/net/netfilter/xt_time.c b/net/netfilter/xt_time.c index 6aa12d0f54e23..61de85e02a40f 100644 --- a/net/netfilter/xt_time.c +++ b/net/netfilter/xt_time.c @@ -227,13 +227,13 @@ time_mt(const struct sk_buff *skb, struct xt_action_param *par) localtime_2(¤t_time, stamp); - if (!(info->weekdays_match & (1 << current_time.weekday))) + if (!(info->weekdays_match & (1U << current_time.weekday))) return false; /* Do not spend time computing monthday if all days match anyway */ if (info->monthdays_match != XT_TIME_ALL_MONTHDAYS) { localtime_3(¤t_time, stamp); - if (!(info->monthdays_match & (1 << current_time.monthday))) + if (!(info->monthdays_match & (1U << current_time.monthday))) return false; } diff --git a/net/netrom/nr_out.c b/net/netrom/nr_out.c index 5e531394a724b..2b3cbceb0b52d 100644 --- a/net/netrom/nr_out.c +++ b/net/netrom/nr_out.c @@ -43,8 +43,10 @@ void nr_output(struct sock *sk, struct sk_buff *skb) frontlen = skb_headroom(skb); while (skb->len > 0) { - if ((skbn = sock_alloc_send_skb(sk, frontlen + NR_MAX_PACKET_SIZE, 0, &err)) == NULL) + if ((skbn = sock_alloc_send_skb(sk, frontlen + NR_MAX_PACKET_SIZE, 0, &err)) == NULL) { + kfree_skb(skb); return; + } skb_reserve(skbn, frontlen); diff --git a/net/netrom/nr_route.c b/net/netrom/nr_route.c index b94cb2ffbaf8f..9cc29ae85b06f 100644 --- a/net/netrom/nr_route.c +++ b/net/netrom/nr_route.c @@ -752,7 +752,7 @@ int nr_route_frame(struct sk_buff *skb, ax25_cb *ax25) unsigned char *dptr; ax25_cb *ax25s; int ret; - struct sk_buff *skbn; + struct sk_buff *nskb, *oskb; /* * Reject malformed packets early. Check that it contains at least 2 @@ -811,14 +811,16 @@ int nr_route_frame(struct sk_buff *skb, ax25_cb *ax25) /* We are going to change the netrom headers so we should get our own skb, we also did not know until now how much header space we had to reserve... - RXQ */ - if ((skbn=skb_copy_expand(skb, dev->hard_header_len, 0, GFP_ATOMIC)) == NULL) { + nskb = skb_copy_expand(skb, dev->hard_header_len, 0, GFP_ATOMIC); + + if (!nskb) { nr_node_unlock(nr_node); nr_node_put(nr_node); dev_put(dev); return 0; } - kfree_skb(skb); - skb=skbn; + oskb = skb; + skb = nskb; skb->data[14]--; dptr = skb_push(skb, 1); @@ -837,6 +839,9 @@ int nr_route_frame(struct sk_buff *skb, ax25_cb *ax25) nr_node_unlock(nr_node); nr_node_put(nr_node); + if (ret) + kfree_skb(oskb); + return ret; } diff --git a/net/nfc/core.c b/net/nfc/core.c index ae1c842f9c642..f50e5bab35d8e 100644 --- a/net/nfc/core.c +++ b/net/nfc/core.c @@ -1147,12 +1147,13 @@ int nfc_register_device(struct nfc_dev *dev) EXPORT_SYMBOL(nfc_register_device); /** - * nfc_unregister_device - unregister a nfc device in the nfc subsystem + * nfc_unregister_rfkill - unregister a nfc device in the rfkill subsystem * * @dev: The nfc device to unregister */ -void nfc_unregister_device(struct nfc_dev *dev) +void nfc_unregister_rfkill(struct nfc_dev *dev) { + struct rfkill *rfk = NULL; int rc; pr_debug("dev_name=%s\n", dev_name(&dev->dev)); @@ -1164,13 +1165,26 @@ void nfc_unregister_device(struct nfc_dev *dev) device_lock(&dev->dev); if (dev->rfkill) { - rfkill_unregister(dev->rfkill); - rfkill_destroy(dev->rfkill); + rfk = dev->rfkill; dev->rfkill = NULL; } dev->shutting_down = true; device_unlock(&dev->dev); + if (rfk) { + rfkill_unregister(rfk); + rfkill_destroy(rfk); + } +} +EXPORT_SYMBOL(nfc_unregister_rfkill); + +/** + * nfc_remove_device - remove a nfc device in the nfc subsystem + * + * @dev: The nfc device to remove + */ +void nfc_remove_device(struct nfc_dev *dev) +{ if (dev->ops->check_presence) { timer_delete_sync(&dev->check_pres_timer); cancel_work_sync(&dev->check_pres_work); @@ -1183,6 +1197,18 @@ void nfc_unregister_device(struct nfc_dev *dev) device_del(&dev->dev); mutex_unlock(&nfc_devlist_mutex); } +EXPORT_SYMBOL(nfc_remove_device); + +/** + * nfc_unregister_device - unregister a nfc device in the nfc subsystem + * + * @dev: The nfc device to unregister + */ +void nfc_unregister_device(struct nfc_dev *dev) +{ + nfc_unregister_rfkill(dev); + nfc_remove_device(dev); +} EXPORT_SYMBOL(nfc_unregister_device); static int __init nfc_init(void) diff --git a/net/nfc/digital_technology.c b/net/nfc/digital_technology.c index 3adf4589852af..e29dd10f280ed 100644 --- a/net/nfc/digital_technology.c +++ b/net/nfc/digital_technology.c @@ -424,6 +424,12 @@ static void digital_in_recv_sdd_res(struct nfc_digital_dev *ddev, void *arg, size = 4; } + if (target->nfcid1_len + size > NFC_NFCID1_MAXSIZE) { + PROTOCOL_ERR("4.7.2.1"); + rc = -EPROTO; + goto exit; + } + memcpy(target->nfcid1 + target->nfcid1_len, sdd_res->nfcid1 + offset, size); target->nfcid1_len += size; diff --git a/net/nfc/hci/llc_shdlc.c b/net/nfc/hci/llc_shdlc.c index 4fc37894860c9..08c8aa1530d8a 100644 --- a/net/nfc/hci/llc_shdlc.c +++ b/net/nfc/hci/llc_shdlc.c @@ -762,6 +762,14 @@ static void llc_shdlc_deinit(struct nfc_llc *llc) { struct llc_shdlc *shdlc = nfc_llc_get_data(llc); + timer_shutdown_sync(&shdlc->connect_timer); + timer_shutdown_sync(&shdlc->t1_timer); + timer_shutdown_sync(&shdlc->t2_timer); + shdlc->t1_active = false; + shdlc->t2_active = false; + + cancel_work_sync(&shdlc->sm_work); + skb_queue_purge(&shdlc->rcv_q); skb_queue_purge(&shdlc->send_q); skb_queue_purge(&shdlc->ack_pending_q); diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c index e2680a3bef799..b652323bc2c12 100644 --- a/net/nfc/llcp_commands.c +++ b/net/nfc/llcp_commands.c @@ -778,8 +778,23 @@ int nfc_llcp_send_ui_frame(struct nfc_llcp_sock *sock, u8 ssap, u8 dsap, if (likely(frag_len > 0)) skb_put_data(pdu, msg_ptr, frag_len); + spin_lock(&local->tx_queue.lock); + + if (list_empty(&local->list)) { + spin_unlock(&local->tx_queue.lock); + + kfree_skb(pdu); + + len -= remaining_len; + if (len == 0) + len = -ENXIO; + break; + } + /* No need to check for the peer RW for UI frames */ - skb_queue_tail(&local->tx_queue, pdu); + __skb_queue_tail(&local->tx_queue, pdu); + + spin_unlock(&local->tx_queue.lock); remaining_len -= frag_len; msg_ptr += frag_len; diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index beeb3b4d28cab..da8d3add0018f 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -316,7 +316,9 @@ static struct nfc_llcp_local *nfc_llcp_remove_local(struct nfc_dev *dev) spin_lock(&llcp_devices_lock); list_for_each_entry_safe(local, tmp, &llcp_devices, list) if (local->dev == dev) { - list_del(&local->list); + spin_lock(&local->tx_queue.lock); + list_del_init(&local->list); + spin_unlock(&local->tx_queue.lock); spin_unlock(&llcp_devices_lock); return local; } @@ -1089,6 +1091,7 @@ static void nfc_llcp_recv_hdlc(struct nfc_llcp_local *local, if (sk->sk_state == LLCP_CLOSED) { release_sock(sk); nfc_llcp_sock_put(llcp_sock); + return; } /* Pass the payload upstream */ @@ -1180,6 +1183,7 @@ static void nfc_llcp_recv_disc(struct nfc_llcp_local *local, if (sk->sk_state == LLCP_CLOSED) { release_sock(sk); nfc_llcp_sock_put(llcp_sock); + return; } if (sk->sk_state == LLCP_CONNECTED) { diff --git a/net/nfc/nci/core.c b/net/nfc/nci/core.c index fc921cd2cdffa..25ba4cbb00e1e 100644 --- a/net/nfc/nci/core.c +++ b/net/nfc/nci/core.c @@ -567,6 +567,10 @@ static int nci_close_device(struct nci_dev *ndev) flush_workqueue(ndev->cmd_wq); timer_delete_sync(&ndev->cmd_timer); timer_delete_sync(&ndev->data_timer); + if (test_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + nci_data_exchange_complete(ndev, NULL, + ndev->cur_conn_id, + -ENODEV); mutex_unlock(&ndev->req_lock); return 0; } @@ -575,8 +579,7 @@ static int nci_close_device(struct nci_dev *ndev) skb_queue_purge(&ndev->rx_q); skb_queue_purge(&ndev->tx_q); - /* Flush RX and TX wq */ - flush_workqueue(ndev->rx_wq); + /* Flush TX wq, RX wq flush can't be under the lock */ flush_workqueue(ndev->tx_wq); /* Reset device */ @@ -588,22 +591,30 @@ static int nci_close_device(struct nci_dev *ndev) msecs_to_jiffies(NCI_RESET_TIMEOUT)); /* After this point our queues are empty - * and no works are scheduled. + * rx work may be running but will see that NCI_UP was cleared */ ndev->ops->close(ndev); clear_bit(NCI_INIT, &ndev->flags); - /* Flush cmd wq */ + /* Flush cmd and tx wq */ flush_workqueue(ndev->cmd_wq); timer_delete_sync(&ndev->cmd_timer); + timer_delete_sync(&ndev->data_timer); + + if (test_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + nci_data_exchange_complete(ndev, NULL, ndev->cur_conn_id, + -ENODEV); /* Clear flags except NCI_UNREG */ ndev->flags &= BIT(NCI_UNREG); mutex_unlock(&ndev->req_lock); + /* rx_work may take req_lock via nci_deactivate_target */ + flush_workqueue(ndev->rx_wq); + return 0; } @@ -1035,18 +1046,23 @@ static int nci_transceive(struct nfc_dev *nfc_dev, struct nfc_target *target, struct nci_conn_info *conn_info; conn_info = ndev->rf_conn_info; - if (!conn_info) + if (!conn_info) { + kfree_skb(skb); return -EPROTO; + } pr_debug("target_idx %d, len %d\n", target->idx, skb->len); if (!ndev->target_active_prot) { pr_err("unable to exchange data, no active target\n"); + kfree_skb(skb); return -EINVAL; } - if (test_and_set_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + if (test_and_set_bit(NCI_DATA_EXCHANGE, &ndev->flags)) { + kfree_skb(skb); return -EBUSY; + } /* store cb and context to be used on receiving data */ conn_info->data_exchange_cb = cb; @@ -1303,6 +1319,8 @@ void nci_unregister_device(struct nci_dev *ndev) { struct nci_conn_info *conn_info, *n; + nfc_unregister_rfkill(ndev->nfc_dev); + /* This set_bit is not protected with specialized barrier, * However, it is fine because the mutex_lock(&ndev->req_lock); * in nci_close_device() will help to emit one. @@ -1320,7 +1338,7 @@ void nci_unregister_device(struct nci_dev *ndev) /* conn_info is allocated with devm_kzalloc */ } - nfc_unregister_device(ndev->nfc_dev); + nfc_remove_device(ndev->nfc_dev); } EXPORT_SYMBOL(nci_unregister_device); @@ -1480,10 +1498,20 @@ static bool nci_valid_size(struct sk_buff *skb) unsigned int hdr_size = NCI_CTRL_HDR_SIZE; if (skb->len < hdr_size || - !nci_plen(skb->data) || skb->len < hdr_size + nci_plen(skb->data)) { return false; } + + if (!nci_plen(skb->data)) { + /* Allow zero length in proprietary notifications (0x20 - 0x3F). */ + if (nci_opcode_oid(nci_opcode(skb->data)) >= 0x20 && + nci_mt(skb->data) == NCI_MT_NTF_PKT) + return true; + + /* Disallow zero length otherwise. */ + return false; + } + return true; } diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c index 78f4131af3cf3..5f98c73db5afd 100644 --- a/net/nfc/nci/data.c +++ b/net/nfc/nci/data.c @@ -33,7 +33,8 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, conn_info = nci_get_conn_info_by_conn_id(ndev, conn_id); if (!conn_info) { kfree_skb(skb); - goto exit; + clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + return; } cb = conn_info->data_exchange_cb; @@ -45,6 +46,12 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, timer_delete_sync(&ndev->data_timer); clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags); + /* Mark the exchange as done before calling the callback. + * The callback (e.g. rawsock_data_exchange_complete) may + * want to immediately queue another data exchange. + */ + clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + if (cb) { /* forward skb to nfc core */ cb(cb_context, skb, err); @@ -54,9 +61,6 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, /* no waiting callback, free skb */ kfree_skb(skb); } - -exit: - clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); } /* ----------------- NCI TX Data ----------------- */ diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index 418b84e2b2605..c96512bb86531 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -58,7 +58,7 @@ static int nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, struct nci_conn_info *conn_info; int i; - if (skb->len < sizeof(struct nci_core_conn_credit_ntf)) + if (skb->len < offsetofend(struct nci_core_conn_credit_ntf, num_entries)) return -EINVAL; ntf = (struct nci_core_conn_credit_ntf *)skb->data; @@ -68,6 +68,10 @@ static int nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, if (ntf->num_entries > NCI_MAX_NUM_CONN) ntf->num_entries = NCI_MAX_NUM_CONN; + if (skb->len < offsetofend(struct nci_core_conn_credit_ntf, num_entries) + + ntf->num_entries * sizeof(struct conn_credit_entry)) + return -EINVAL; + /* update the credits */ for (i = 0; i < ntf->num_entries; i++) { ntf->conn_entries[i].conn_id = @@ -138,23 +142,48 @@ static int nci_core_conn_intf_error_ntf_packet(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfca_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfca_poll *nfca_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for sens_res (2 bytes) */ + if (data_len < 2) + return ERR_PTR(-EINVAL); + nfca_poll->sens_res = __le16_to_cpu(*((__le16 *)data)); data += 2; + data_len -= 2; + + /* Check if we have enough data for nfcid1_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); nfca_poll->nfcid1_len = min_t(__u8, *data++, NFC_NFCID1_MAXSIZE); + data_len--; pr_debug("sens_res 0x%x, nfcid1_len %d\n", nfca_poll->sens_res, nfca_poll->nfcid1_len); + /* Check if we have enough data for nfcid1 */ + if (data_len < nfca_poll->nfcid1_len) + return ERR_PTR(-EINVAL); + memcpy(nfca_poll->nfcid1, data, nfca_poll->nfcid1_len); data += nfca_poll->nfcid1_len; + data_len -= nfca_poll->nfcid1_len; + + /* Check if we have enough data for sel_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); nfca_poll->sel_res_len = *data++; + data_len--; + + if (nfca_poll->sel_res_len != 0) { + /* Check if we have enough data for sel_res (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); - if (nfca_poll->sel_res_len != 0) nfca_poll->sel_res = *data++; + } pr_debug("sel_res_len %d, sel_res 0x%x\n", nfca_poll->sel_res_len, @@ -166,12 +195,21 @@ nci_extract_rf_params_nfca_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcb_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcb_poll *nfcb_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for sensb_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcb_poll->sensb_res_len = min_t(__u8, *data++, NFC_SENSB_RES_MAXSIZE); + data_len--; pr_debug("sensb_res_len %d\n", nfcb_poll->sensb_res_len); + /* Check if we have enough data for sensb_res */ + if (data_len < nfcb_poll->sensb_res_len) + return ERR_PTR(-EINVAL); + memcpy(nfcb_poll->sensb_res, data, nfcb_poll->sensb_res_len); data += nfcb_poll->sensb_res_len; @@ -181,14 +219,29 @@ nci_extract_rf_params_nfcb_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcf_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcf_poll *nfcf_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for bit_rate (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_poll->bit_rate = *data++; + data_len--; + + /* Check if we have enough data for sensf_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_poll->sensf_res_len = min_t(__u8, *data++, NFC_SENSF_RES_MAXSIZE); + data_len--; pr_debug("bit_rate %d, sensf_res_len %d\n", nfcf_poll->bit_rate, nfcf_poll->sensf_res_len); + /* Check if we have enough data for sensf_res */ + if (data_len < nfcf_poll->sensf_res_len) + return ERR_PTR(-EINVAL); + memcpy(nfcf_poll->sensf_res, data, nfcf_poll->sensf_res_len); data += nfcf_poll->sensf_res_len; @@ -198,22 +251,49 @@ nci_extract_rf_params_nfcf_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcv_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcv_poll *nfcv_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Skip 1 byte (reserved) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + ++data; + data_len--; + + /* Check if we have enough data for dsfid (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcv_poll->dsfid = *data++; + data_len--; + + /* Check if we have enough data for uid (8 bytes) */ + if (data_len < NFC_ISO15693_UID_MAXSIZE) + return ERR_PTR(-EINVAL); + memcpy(nfcv_poll->uid, data, NFC_ISO15693_UID_MAXSIZE); data += NFC_ISO15693_UID_MAXSIZE; + return data; } static const __u8 * nci_extract_rf_params_nfcf_passive_listen(struct nci_dev *ndev, struct rf_tech_specific_params_nfcf_listen *nfcf_listen, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for local_nfcid2_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_listen->local_nfcid2_len = min_t(__u8, *data++, NFC_NFCID2_MAXSIZE); + data_len--; + + /* Check if we have enough data for local_nfcid2 */ + if (data_len < nfcf_listen->local_nfcid2_len) + return ERR_PTR(-EINVAL); + memcpy(nfcf_listen->local_nfcid2, data, nfcf_listen->local_nfcid2_len); data += nfcf_listen->local_nfcid2_len; @@ -364,7 +444,7 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, const __u8 *data; bool add_target = true; - if (skb->len < sizeof(struct nci_rf_discover_ntf)) + if (skb->len < offsetofend(struct nci_rf_discover_ntf, rf_tech_specific_params_len)) return -EINVAL; data = skb->data; @@ -380,26 +460,42 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, pr_debug("rf_tech_specific_params_len %d\n", ntf.rf_tech_specific_params_len); + if (skb->len < (data - skb->data) + + ntf.rf_tech_specific_params_len + sizeof(ntf.ntf_type)) + return -EINVAL; + if (ntf.rf_tech_specific_params_len > 0) { switch (ntf.rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfca_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfca_poll), data); + &(ntf.rf_tech_specific_params.nfca_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_B_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcb_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcb_poll), data); + &(ntf.rf_tech_specific_params.nfcb_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_F_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcf_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcf_poll), data); + &(ntf.rf_tech_specific_params.nfcf_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_V_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcv_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcv_poll), data); + &(ntf.rf_tech_specific_params.nfcv_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; default: @@ -596,7 +692,7 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, const __u8 *data; int err = NCI_STATUS_OK; - if (skb->len < sizeof(struct nci_rf_intf_activated_ntf)) + if (skb->len < offsetofend(struct nci_rf_intf_activated_ntf, rf_tech_specific_params_len)) return -EINVAL; data = skb->data; @@ -628,26 +724,41 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, if (ntf.rf_interface == NCI_RF_INTERFACE_NFCEE_DIRECT) goto listen; + if (skb->len < (data - skb->data) + ntf.rf_tech_specific_params_len) + return -EINVAL; + if (ntf.rf_tech_specific_params_len > 0) { switch (ntf.activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfca_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfca_poll), data); + &(ntf.rf_tech_specific_params.nfca_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_B_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcb_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcb_poll), data); + &(ntf.rf_tech_specific_params.nfcb_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_F_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcf_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcf_poll), data); + &(ntf.rf_tech_specific_params.nfcf_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_V_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcv_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcv_poll), data); + &(ntf.rf_tech_specific_params.nfcv_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_A_PASSIVE_LISTEN_MODE: @@ -657,7 +768,9 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, case NCI_NFC_F_PASSIVE_LISTEN_MODE: data = nci_extract_rf_params_nfcf_passive_listen(ndev, &(ntf.rf_tech_specific_params.nfcf_listen), - data); + data, ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; default: @@ -668,6 +781,13 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, } } + if (skb->len < (data - skb->data) + + sizeof(ntf.data_exch_rf_tech_and_mode) + + sizeof(ntf.data_exch_tx_bit_rate) + + sizeof(ntf.data_exch_rx_bit_rate) + + sizeof(ntf.activation_params_len)) + return -EINVAL; + ntf.data_exch_rf_tech_and_mode = *data++; ntf.data_exch_tx_bit_rate = *data++; ntf.data_exch_rx_bit_rate = *data++; @@ -679,6 +799,9 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, pr_debug("data_exch_rx_bit_rate 0x%x\n", ntf.data_exch_rx_bit_rate); pr_debug("activation_params_len %d\n", ntf.activation_params_len); + if (skb->len < (data - skb->data) + ntf.activation_params_len) + return -EINVAL; + if (ntf.activation_params_len > 0) { switch (ntf.rf_interface) { case NCI_RF_INTERFACE_ISO_DEP: diff --git a/net/nfc/rawsock.c b/net/nfc/rawsock.c index 5125392bb68eb..028b4daafaf83 100644 --- a/net/nfc/rawsock.c +++ b/net/nfc/rawsock.c @@ -67,6 +67,17 @@ static int rawsock_release(struct socket *sock) if (sock->type == SOCK_RAW) nfc_sock_unlink(&raw_sk_list, sk); + if (sk->sk_state == TCP_ESTABLISHED) { + /* Prevent rawsock_tx_work from starting new transmits and + * wait for any in-progress work to finish. This must happen + * before the socket is orphaned to avoid a race where + * rawsock_tx_work runs after the NCI device has been freed. + */ + sk->sk_shutdown |= SEND_SHUTDOWN; + cancel_work_sync(&nfc_rawsock(sk)->tx_work); + rawsock_write_queue_purge(sk); + } + sock_orphan(sk); sock_put(sk); diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index e573e92213029..a0811e1fba656 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -928,8 +928,8 @@ static u32 ct_limit_get(const struct ovs_ct_limit_info *info, u16 zone) } static int ovs_ct_check_limit(struct net *net, - const struct ovs_conntrack_info *info, - const struct nf_conntrack_tuple *tuple) + const struct sk_buff *skb, + const struct ovs_conntrack_info *info) { struct ovs_net *ovs_net = net_generic(net, ovs_net_id); const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; @@ -942,8 +942,9 @@ static int ovs_ct_check_limit(struct net *net, if (per_zone_limit == OVS_CT_LIMIT_UNLIMITED) return 0; - connections = nf_conncount_count(net, ct_limit_info->data, - &conncount_key, tuple, &info->zone); + connections = nf_conncount_count_skb(net, skb, info->family, + ct_limit_info->data, + &conncount_key); if (connections > per_zone_limit) return -ENOMEM; @@ -972,8 +973,7 @@ static int ovs_ct_commit(struct net *net, struct sw_flow_key *key, #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) if (static_branch_unlikely(&ovs_ct_limit_enabled)) { if (!nf_ct_is_confirmed(ct)) { - err = ovs_ct_check_limit(net, info, - &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); + err = ovs_ct_check_limit(net, skb, info); if (err) { net_warn_ratelimited("openvswitch: zone: %u " "exceeds conntrack limit\n", @@ -1770,8 +1770,8 @@ static int __ovs_ct_limit_get_zone_limit(struct net *net, zone_limit.limit = limit; nf_ct_zone_init(&ct_zone, zone_id, NF_CT_DEFAULT_ZONE_DIR, 0); - zone_limit.count = nf_conncount_count(net, data, &conncount_key, NULL, - &ct_zone); + zone_limit.count = nf_conncount_count_skb(net, NULL, 0, data, + &conncount_key); return nla_put_nohdr(reply, sizeof(zone_limit), &zone_limit); } diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c index 1cb4f97335d87..2dc4a6c2aecec 100644 --- a/net/openvswitch/flow_netlink.c +++ b/net/openvswitch/flow_netlink.c @@ -2802,13 +2802,20 @@ static int validate_and_copy_set_tun(const struct nlattr *attr, return err; } -static bool validate_push_nsh(const struct nlattr *attr, bool log) +static bool validate_push_nsh(const struct nlattr *a, bool log) { + struct nlattr *nsh_key = nla_data(a); struct sw_flow_match match; struct sw_flow_key key; + /* There must be one and only one NSH header. */ + if (!nla_ok(nsh_key, nla_len(a)) || + nla_total_size(nla_len(nsh_key)) != nla_len(a) || + nla_type(nsh_key) != OVS_KEY_ATTR_NSH) + return false; + ovs_match_init(&match, &key, true, NULL); - return !nsh_key_put_from_nlattr(attr, &match, false, true, log); + return !nsh_key_put_from_nlattr(nsh_key, &match, false, true, log); } /* Return false if there are any non-masked bits set. @@ -2946,6 +2953,8 @@ static int validate_set(const struct nlattr *a, case OVS_KEY_ATTR_MPLS: if (!eth_p_mpls(eth_type)) return -EINVAL; + if (key_len != sizeof(struct ovs_key_mpls)) + return -EINVAL; break; case OVS_KEY_ATTR_SCTP: @@ -3389,7 +3398,7 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr, return -EINVAL; } mac_proto = MAC_PROTO_NONE; - if (!validate_push_nsh(nla_data(a), log)) + if (!validate_push_nsh(a, log)) return -EINVAL; break; diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c index 91a11067e4588..12055af832dc0 100644 --- a/net/openvswitch/vport-netdev.c +++ b/net/openvswitch/vport-netdev.c @@ -151,19 +151,35 @@ static void vport_netdev_free(struct rcu_head *rcu) void ovs_netdev_detach_dev(struct vport *vport) { ASSERT_RTNL(); - vport->dev->priv_flags &= ~IFF_OVS_DATAPATH; netdev_rx_handler_unregister(vport->dev); netdev_upper_dev_unlink(vport->dev, netdev_master_upper_dev_get(vport->dev)); dev_set_promiscuity(vport->dev, -1); + + /* paired with smp_mb() in netdev_destroy() */ + smp_wmb(); + + vport->dev->priv_flags &= ~IFF_OVS_DATAPATH; } static void netdev_destroy(struct vport *vport) { - rtnl_lock(); - if (netif_is_ovs_port(vport->dev)) - ovs_netdev_detach_dev(vport); - rtnl_unlock(); + /* When called from ovs_db_notify_wq() after a dp_device_event(), the + * port has already been detached, so we can avoid taking the RTNL by + * checking this first. + */ + if (netif_is_ovs_port(vport->dev)) { + rtnl_lock(); + /* Check again while holding the lock to ensure we don't race + * with the netdev notifier and detach twice. + */ + if (netif_is_ovs_port(vport->dev)) + ovs_netdev_detach_dev(vport); + rtnl_unlock(); + } + + /* paired with smp_wmb() in ovs_netdev_detach_dev() */ + smp_mb(); call_rcu(&vport->rcu, vport_netdev_free); } @@ -180,8 +196,6 @@ void ovs_netdev_tunnel_destroy(struct vport *vport) */ if (vport->dev->reg_state == NETREG_REGISTERED) rtnl_delete_link(vport->dev, 0, NULL); - netdev_put(vport->dev, &vport->dev_tracker); - vport->dev = NULL; rtnl_unlock(); call_rcu(&vport->rcu, vport_netdev_free); diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c index 6bbbc16ab7780..f0ce8ce1dce0e 100644 --- a/net/openvswitch/vport.c +++ b/net/openvswitch/vport.c @@ -310,22 +310,23 @@ void ovs_vport_get_stats(struct vport *vport, struct ovs_vport_stats *stats) */ int ovs_vport_get_upcall_stats(struct vport *vport, struct sk_buff *skb) { + u64 tx_success = 0, tx_fail = 0; struct nlattr *nla; int i; - __u64 tx_success = 0; - __u64 tx_fail = 0; - for_each_possible_cpu(i) { const struct vport_upcall_stats_percpu *stats; + u64 n_success, n_fail; unsigned int start; stats = per_cpu_ptr(vport->upcall_stats, i); do { start = u64_stats_fetch_begin(&stats->syncp); - tx_success += u64_stats_read(&stats->n_success); - tx_fail += u64_stats_read(&stats->n_fail); + n_success = u64_stats_read(&stats->n_success); + n_fail = u64_stats_read(&stats->n_fail); } while (u64_stats_fetch_retry(&stats->syncp, start)); + tx_success += n_success; + tx_fail += n_fail; } nla = nla_nest_start_noflag(skb, OVS_VPORT_ATTR_UPCALL_STATS); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 173e6edda08f8..8b3961fba750b 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2717,7 +2717,8 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) { struct sk_buff *skb = NULL; struct net_device *dev; - struct virtio_net_hdr *vnet_hdr = NULL; + struct virtio_net_hdr vnet_hdr; + bool has_vnet_hdr = false; struct sockcm_cookie sockc; __be16 proto; int err, reserve = 0; @@ -2818,16 +2819,20 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) hlen = LL_RESERVED_SPACE(dev); tlen = dev->needed_tailroom; if (vnet_hdr_sz) { - vnet_hdr = data; data += vnet_hdr_sz; tp_len -= vnet_hdr_sz; - if (tp_len < 0 || - __packet_snd_vnet_parse(vnet_hdr, tp_len)) { + if (tp_len < 0) { + tp_len = -EINVAL; + goto tpacket_error; + } + memcpy(&vnet_hdr, data - vnet_hdr_sz, sizeof(vnet_hdr)); + if (__packet_snd_vnet_parse(&vnet_hdr, tp_len)) { tp_len = -EINVAL; goto tpacket_error; } copylen = __virtio16_to_cpu(vio_le(), - vnet_hdr->hdr_len); + vnet_hdr.hdr_len); + has_vnet_hdr = true; } copylen = max_t(int, copylen, dev->hard_header_len); skb = sock_alloc_send_skb(&po->sk, @@ -2864,12 +2869,12 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) } } - if (vnet_hdr_sz) { - if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) { + if (has_vnet_hdr) { + if (virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le())) { tp_len = -EINVAL; goto tpacket_error; } - virtio_net_hdr_set_proto(skb, vnet_hdr); + virtio_net_hdr_set_proto(skb, &vnet_hdr); } skb->destructor = tpacket_destruct_skb; @@ -3135,6 +3140,7 @@ static int packet_release(struct socket *sock) spin_lock(&po->bind_lock); unregister_prot_hook(sk, false); + WRITE_ONCE(po->num, 0); packet_cached_dev_reset(po); if (po->prot_hook.dev) { diff --git a/net/phonet/af_phonet.c b/net/phonet/af_phonet.c index 238a9638d2b0f..d89225d6bfd3b 100644 --- a/net/phonet/af_phonet.c +++ b/net/phonet/af_phonet.c @@ -129,9 +129,12 @@ static int pn_header_create(struct sk_buff *skb, struct net_device *dev, return 1; } -static int pn_header_parse(const struct sk_buff *skb, unsigned char *haddr) +static int pn_header_parse(const struct sk_buff *skb, + const struct net_device *dev, + unsigned char *haddr) { const u8 *media = skb_mac_header(skb); + *haddr = *media; return 1; } diff --git a/net/psp/Kconfig b/net/psp/Kconfig index 371e8771f3bd3..84d6b0f254606 100644 --- a/net/psp/Kconfig +++ b/net/psp/Kconfig @@ -6,6 +6,7 @@ config INET_PSP bool "PSP Security Protocol support" depends on INET select SKB_DECRYPTED + select SKB_EXTENSIONS select SOCK_VALIDATE_XMIT help Enable kernel support for the PSP Security Protocol (PSP). diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c index 481aaf0fc9fcf..53c0313633689 100644 --- a/net/psp/psp_main.c +++ b/net/psp/psp_main.c @@ -165,9 +165,46 @@ static void psp_write_headers(struct net *net, struct sk_buff *skb, __be32 spi, { struct udphdr *uh = udp_hdr(skb); struct psphdr *psph = (struct psphdr *)(uh + 1); + const struct sock *sk = skb->sk; uh->dest = htons(PSP_DEFAULT_UDP_PORT); - uh->source = udp_flow_src_port(net, skb, 0, 0, false); + + /* A bit of theory: Selection of the source port. + * + * We need some entropy, so that multiple flows use different + * source ports for better RSS spreading at the receiver. + * + * We also need that all packets belonging to one TCP flow + * use the same source port through their duration, + * so that all these packets land in the same receive queue. + * + * udp_flow_src_port() is using sk_txhash, inherited from + * skb_set_hash_from_sk() call in __tcp_transmit_skb(). + * This field is subject to reshuffling, thanks to + * sk_rethink_txhash() calls in various TCP functions. + * + * Instead, use sk->sk_hash which is constant through + * the whole flow duration. + */ + if (likely(sk)) { + u32 hash = sk->sk_hash; + int min, max; + + /* These operations are cheap, no need to cache the result + * in another socket field. + */ + inet_get_local_port_range(net, &min, &max); + /* Since this is being sent on the wire obfuscate hash a bit + * to minimize possibility that any useful information to an + * attacker is leaked. Only upper 16 bits are relevant in the + * computation for 16 bit port value because we use a + * reciprocal divide. + */ + hash ^= hash << 16; + uh->source = htons((((u64)hash * (max - min)) >> 32) + min); + } else { + uh->source = udp_flow_src_port(net, skb, 0, 0, false); + } uh->check = 0; uh->len = htons(udp_len); diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c index 00c51cf693f3d..b703e4c645853 100644 --- a/net/qrtr/af_qrtr.c +++ b/net/qrtr/af_qrtr.c @@ -118,7 +118,7 @@ static DEFINE_XARRAY_ALLOC(qrtr_ports); * @ep: endpoint * @ref: reference count for node * @nid: node id - * @qrtr_tx_flow: tree of qrtr_tx_flow, keyed by node << 32 | port + * @qrtr_tx_flow: xarray of qrtr_tx_flow, keyed by node << 32 | port * @qrtr_tx_lock: lock for qrtr_tx_flow inserts * @rx_queue: receive queue * @item: list item for broadcast list @@ -129,7 +129,7 @@ struct qrtr_node { struct kref ref; unsigned int nid; - struct radix_tree_root qrtr_tx_flow; + struct xarray qrtr_tx_flow; struct mutex qrtr_tx_lock; /* for qrtr_tx_flow */ struct sk_buff_head rx_queue; @@ -172,6 +172,7 @@ static void __qrtr_node_release(struct kref *kref) struct qrtr_tx_flow *flow; unsigned long flags; void __rcu **slot; + unsigned long index; spin_lock_irqsave(&qrtr_nodes_lock, flags); /* If the node is a bridge for other nodes, there are possibly @@ -189,11 +190,9 @@ static void __qrtr_node_release(struct kref *kref) skb_queue_purge(&node->rx_queue); /* Free tx flow counters */ - radix_tree_for_each_slot(slot, &node->qrtr_tx_flow, &iter, 0) { - flow = *slot; - radix_tree_iter_delete(&node->qrtr_tx_flow, &iter, slot); + xa_for_each(&node->qrtr_tx_flow, index, flow) kfree(flow); - } + xa_destroy(&node->qrtr_tx_flow); kfree(node); } @@ -228,9 +227,7 @@ static void qrtr_tx_resume(struct qrtr_node *node, struct sk_buff *skb) key = remote_node << 32 | remote_port; - rcu_read_lock(); - flow = radix_tree_lookup(&node->qrtr_tx_flow, key); - rcu_read_unlock(); + flow = xa_load(&node->qrtr_tx_flow, key); if (flow) { spin_lock(&flow->resume_tx.lock); flow->pending = 0; @@ -269,12 +266,13 @@ static int qrtr_tx_wait(struct qrtr_node *node, int dest_node, int dest_port, return 0; mutex_lock(&node->qrtr_tx_lock); - flow = radix_tree_lookup(&node->qrtr_tx_flow, key); + flow = xa_load(&node->qrtr_tx_flow, key); if (!flow) { flow = kzalloc(sizeof(*flow), GFP_KERNEL); if (flow) { init_waitqueue_head(&flow->resume_tx); - if (radix_tree_insert(&node->qrtr_tx_flow, key, flow)) { + if (xa_err(xa_store(&node->qrtr_tx_flow, key, flow, + GFP_KERNEL))) { kfree(flow); flow = NULL; } @@ -326,9 +324,7 @@ static void qrtr_tx_flow_failed(struct qrtr_node *node, int dest_node, unsigned long key = (u64)dest_node << 32 | dest_port; struct qrtr_tx_flow *flow; - rcu_read_lock(); - flow = radix_tree_lookup(&node->qrtr_tx_flow, key); - rcu_read_unlock(); + flow = xa_load(&node->qrtr_tx_flow, key); if (flow) { spin_lock_irq(&flow->resume_tx.lock); flow->tx_failed = 1; @@ -599,7 +595,7 @@ int qrtr_endpoint_register(struct qrtr_endpoint *ep, unsigned int nid) node->nid = QRTR_EP_NID_AUTO; node->ep = ep; - INIT_RADIX_TREE(&node->qrtr_tx_flow, GFP_KERNEL); + xa_init(&node->qrtr_tx_flow); mutex_init(&node->qrtr_tx_lock); qrtr_node_assign(node, nid); @@ -627,6 +623,7 @@ void qrtr_endpoint_unregister(struct qrtr_endpoint *ep) struct qrtr_tx_flow *flow; struct sk_buff *skb; unsigned long flags; + unsigned long index; void __rcu **slot; mutex_lock(&node->ep_lock); @@ -649,10 +646,8 @@ void qrtr_endpoint_unregister(struct qrtr_endpoint *ep) /* Wake up any transmitters waiting for resume-tx from the node */ mutex_lock(&node->qrtr_tx_lock); - radix_tree_for_each_slot(slot, &node->qrtr_tx_flow, &iter, 0) { - flow = *slot; + xa_for_each(&node->qrtr_tx_flow, index, flow) wake_up_interruptible_all(&flow->resume_tx); - } mutex_unlock(&node->qrtr_tx_lock); qrtr_node_release(node); diff --git a/net/qrtr/mhi.c b/net/qrtr/mhi.c index 69f53625a049d..80e341d2f8a45 100644 --- a/net/qrtr/mhi.c +++ b/net/qrtr/mhi.c @@ -24,13 +24,25 @@ static void qcom_mhi_qrtr_dl_callback(struct mhi_device *mhi_dev, struct qrtr_mhi_dev *qdev = dev_get_drvdata(&mhi_dev->dev); int rc; - if (!qdev || mhi_res->transaction_status) + if (!qdev || (mhi_res->transaction_status && mhi_res->transaction_status != -ENOTCONN)) return; + /* Channel got reset. So just free the buffer */ + if (mhi_res->transaction_status == -ENOTCONN) { + devm_kfree(&mhi_dev->dev, mhi_res->buf_addr); + return; + } + rc = qrtr_endpoint_post(&qdev->ep, mhi_res->buf_addr, mhi_res->bytes_xferd); if (rc == -EINVAL) dev_err(qdev->dev, "invalid ipcrouter packet\n"); + + /* Done with the buffer, now recycle it for future use */ + rc = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, mhi_res->buf_addr, + mhi_dev->mhi_cntrl->buffer_len, MHI_EOT); + if (rc) + dev_err(&mhi_dev->dev, "Failed to recycle the buffer: %d\n", rc); } /* From QRTR to MHI */ @@ -72,6 +84,29 @@ static int qcom_mhi_qrtr_send(struct qrtr_endpoint *ep, struct sk_buff *skb) return rc; } +static int qcom_mhi_qrtr_queue_dl_buffers(struct mhi_device *mhi_dev) +{ + u32 free_desc; + void *buf; + int ret; + + free_desc = mhi_get_free_desc_count(mhi_dev, DMA_FROM_DEVICE); + while (free_desc--) { + buf = devm_kmalloc(&mhi_dev->dev, mhi_dev->mhi_cntrl->buffer_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, buf, mhi_dev->mhi_cntrl->buffer_len, + MHI_EOT); + if (ret) { + dev_err(&mhi_dev->dev, "Failed to queue buffer: %d\n", ret); + return ret; + } + } + + return 0; +} + static int qcom_mhi_qrtr_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id) { @@ -87,20 +122,30 @@ static int qcom_mhi_qrtr_probe(struct mhi_device *mhi_dev, qdev->ep.xmit = qcom_mhi_qrtr_send; dev_set_drvdata(&mhi_dev->dev, qdev); - rc = qrtr_endpoint_register(&qdev->ep, QRTR_EP_NID_AUTO); - if (rc) - return rc; /* start channels */ - rc = mhi_prepare_for_transfer_autoqueue(mhi_dev); - if (rc) { - qrtr_endpoint_unregister(&qdev->ep); + rc = mhi_prepare_for_transfer(mhi_dev); + if (rc) return rc; - } + + rc = qrtr_endpoint_register(&qdev->ep, QRTR_EP_NID_AUTO); + if (rc) + goto err_unprepare; + + rc = qcom_mhi_qrtr_queue_dl_buffers(mhi_dev); + if (rc) + goto err_unregister; dev_dbg(qdev->dev, "Qualcomm MHI QRTR driver probed\n"); return 0; + +err_unregister: + qrtr_endpoint_unregister(&qdev->ep); +err_unprepare: + mhi_unprepare_from_transfer(mhi_dev); + + return rc; } static void qcom_mhi_qrtr_remove(struct mhi_device *mhi_dev) @@ -151,11 +196,13 @@ static int __maybe_unused qcom_mhi_qrtr_pm_resume_early(struct device *dev) if (state == MHI_STATE_M3) return 0; - rc = mhi_prepare_for_transfer_autoqueue(mhi_dev); - if (rc) + rc = mhi_prepare_for_transfer(mhi_dev); + if (rc) { dev_err(dev, "failed to prepare for autoqueue transfer %d\n", rc); + return rc; + } - return rc; + return qcom_mhi_qrtr_queue_dl_buffers(mhi_dev); } static const struct dev_pm_ops qcom_mhi_qrtr_pm_ops = { diff --git a/net/rds/connection.c b/net/rds/connection.c index 68bc88cce84ec..dbfea6fa11260 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -382,6 +382,8 @@ void rds_conn_shutdown(struct rds_conn_path *cp) if (!rds_conn_path_transition(cp, RDS_CONN_UP, RDS_CONN_DISCONNECTING) && !rds_conn_path_transition(cp, RDS_CONN_ERROR, + RDS_CONN_DISCONNECTING) && + !rds_conn_path_transition(cp, RDS_CONN_RESETTING, RDS_CONN_DISCONNECTING)) { rds_conn_path_error(cp, "shutdown called in state %d\n", @@ -427,6 +429,8 @@ void rds_conn_shutdown(struct rds_conn_path *cp) * to the conn hash, so we never trigger a reconnect on this * conn - the reconnect is always triggered by the active peer. */ cancel_delayed_work_sync(&cp->cp_conn_w); + + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); rcu_read_lock(); if (!hlist_unhashed(&conn->c_hash_node)) { rcu_read_unlock(); diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c index 6585164c70595..dd08ccc4246da 100644 --- a/net/rds/ib_rdma.c +++ b/net/rds/ib_rdma.c @@ -604,8 +604,13 @@ void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, return ibmr; } - if (conn) + if (conn) { ic = conn->c_transport_data; + if (!ic || !ic->i_cm_id || !ic->i_cm_id->qp) { + ret = -ENODEV; + goto out; + } + } if (!rds_ibdev->mr_8k_pool || !rds_ibdev->mr_1m_pool) { ret = -ENODEV; diff --git a/net/rds/send.c b/net/rds/send.c index 0b3d0ef2f008b..071c5dca969a2 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -1382,9 +1382,11 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len) else queue_delayed_work(rds_wq, &cpath->cp_send_w, 1); rcu_read_unlock(); + + if (ret) + goto out; } - if (ret) - goto out; + rds_message_put(rm); for (ind = 0; ind < vct.indx; ind++) diff --git a/net/rds/tcp.c b/net/rds/tcp.c index 3cc2f303bf786..b66dfcc3efaa0 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -495,18 +495,24 @@ bool rds_tcp_tune(struct socket *sock) struct rds_tcp_net *rtn; tcp_sock_set_nodelay(sock->sk); - lock_sock(sk); /* TCP timer functions might access net namespace even after * a process which created this net namespace terminated. */ if (!sk->sk_net_refcnt) { - if (!maybe_get_net(net)) { - release_sock(sk); + if (!maybe_get_net(net)) return false; - } + /* + * sk_net_refcnt_upgrade() must be called before lock_sock() + * because it does a GFP_KERNEL allocation, which can trigger + * fs_reclaim and create a circular lock dependency with the + * socket lock. The fields it modifies (sk_net_refcnt, + * ns_tracker) are not accessed by any concurrent code path + * at this point. + */ sk_net_refcnt_upgrade(sk); put_net(net); } + lock_sock(sk); rtn = net_generic(net, rds_tcp_netid); if (rtn->sndbuf_size > 0) { sk->sk_sndbuf = rtn->sndbuf_size; diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c index 91e34af3fe5d5..65c5425a02de4 100644 --- a/net/rds/tcp_listen.c +++ b/net/rds/tcp_listen.c @@ -59,9 +59,6 @@ void rds_tcp_keepalive(struct socket *sock) * socket and force a reconneect from smaller -> larger ip addr. The reason * we special case cp_index 0 is to allow the rds probe ping itself to itself * get through efficiently. - * Since reconnects are only initiated from the node with the numerically - * smaller ip address, we recycle conns in RDS_CONN_ERROR on the passive side - * by moving them to CONNECTING in this function. */ static struct rds_tcp_connection *rds_tcp_accept_one_path(struct rds_connection *conn) @@ -86,8 +83,6 @@ struct rds_tcp_connection *rds_tcp_accept_one_path(struct rds_connection *conn) struct rds_conn_path *cp = &conn->c_path[i]; if (rds_conn_path_transition(cp, RDS_CONN_DOWN, - RDS_CONN_CONNECTING) || - rds_conn_path_transition(cp, RDS_CONN_ERROR, RDS_CONN_CONNECTING)) { return cp->cp_transport_data; } diff --git a/net/rfkill/core.c b/net/rfkill/core.c index 7d3e82e4c2fce..868a8586dc170 100644 --- a/net/rfkill/core.c +++ b/net/rfkill/core.c @@ -73,11 +73,14 @@ struct rfkill_int_event { struct rfkill_event_ext ev; }; +/* Max rfkill events that can be "in-flight" for one data source */ +#define MAX_RFKILL_EVENT 1000 struct rfkill_data { struct list_head list; struct list_head events; struct mutex mtx; wait_queue_head_t read_wait; + u32 event_count; bool input_handler; u8 max_size; }; @@ -255,10 +258,12 @@ static void rfkill_global_led_trigger_unregister(void) } #endif /* CONFIG_RFKILL_LEDS */ -static void rfkill_fill_event(struct rfkill_event_ext *ev, - struct rfkill *rfkill, - enum rfkill_operation op) +static int rfkill_fill_event(struct rfkill_int_event *int_ev, + struct rfkill *rfkill, + struct rfkill_data *data, + enum rfkill_operation op) { + struct rfkill_event_ext *ev = &int_ev->ev; unsigned long flags; ev->idx = rfkill->idx; @@ -271,6 +276,15 @@ static void rfkill_fill_event(struct rfkill_event_ext *ev, RFKILL_BLOCK_SW_PREV)); ev->hard_block_reasons = rfkill->hard_block_reasons; spin_unlock_irqrestore(&rfkill->lock, flags); + + scoped_guard(mutex, &data->mtx) { + if (data->event_count++ > MAX_RFKILL_EVENT) { + data->event_count--; + return -ENOSPC; + } + list_add_tail(&int_ev->list, &data->events); + } + return 0; } static void rfkill_send_events(struct rfkill *rfkill, enum rfkill_operation op) @@ -282,10 +296,10 @@ static void rfkill_send_events(struct rfkill *rfkill, enum rfkill_operation op) ev = kzalloc(sizeof(*ev), GFP_KERNEL); if (!ev) continue; - rfkill_fill_event(&ev->ev, rfkill, op); - mutex_lock(&data->mtx); - list_add_tail(&ev->list, &data->events); - mutex_unlock(&data->mtx); + if (rfkill_fill_event(ev, rfkill, data, op)) { + kfree(ev); + continue; + } wake_up_interruptible(&data->read_wait); } } @@ -1186,10 +1200,8 @@ static int rfkill_fop_open(struct inode *inode, struct file *file) if (!ev) goto free; rfkill_sync(rfkill); - rfkill_fill_event(&ev->ev, rfkill, RFKILL_OP_ADD); - mutex_lock(&data->mtx); - list_add_tail(&ev->list, &data->events); - mutex_unlock(&data->mtx); + if (rfkill_fill_event(ev, rfkill, data, RFKILL_OP_ADD)) + kfree(ev); } list_add(&data->list, &rfkill_fds); mutex_unlock(&rfkill_global_mutex); @@ -1259,6 +1271,7 @@ static ssize_t rfkill_fop_read(struct file *file, char __user *buf, ret = -EFAULT; list_del(&ev->list); + data->event_count--; kfree(ev); out: mutex_unlock(&data->mtx); diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index 543f9e8ebb693..53c9bc71f813d 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -205,7 +205,7 @@ static void rose_kill_by_device(struct net_device *dev) spin_unlock_bh(&rose_list_lock); for (i = 0; i < cnt; i++) { - sk = array[cnt]; + sk = array[i]; rose = rose_sk(sk); lock_sock(sk); spin_lock_bh(&rose_list_lock); @@ -810,6 +810,11 @@ static int rose_connect(struct socket *sock, struct sockaddr *uaddr, int addr_le goto out_release; } + if (sk->sk_state == TCP_SYN_SENT) { + err = -EALREADY; + goto out_release; + } + sk->sk_state = TCP_CLOSE; sock->state = SS_UNCONNECTED; diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index 36df0274d7b74..885a28e3afc5b 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -267,12 +267,13 @@ static int rxrpc_listen(struct socket *sock, int backlog) * Lookup or create a remote transport endpoint record for the specified * address. * - * Return: The peer record found with a reference, %NULL if no record is found - * or a negative error code if the address is invalid or unsupported. + * Return: The peer record found with a reference or a negative error code if + * the address is invalid or unsupported. */ struct rxrpc_peer *rxrpc_kernel_lookup_peer(struct socket *sock, struct sockaddr_rxrpc *srx, gfp_t gfp) { + struct rxrpc_peer *peer; struct rxrpc_sock *rx = rxrpc_sk(sock->sk); int ret; @@ -280,7 +281,8 @@ struct rxrpc_peer *rxrpc_kernel_lookup_peer(struct socket *sock, if (ret < 0) return ERR_PTR(ret); - return rxrpc_lookup_peer(rx->local, srx, gfp); + peer = rxrpc_lookup_peer(rx->local, srx, gfp); + return peer ?: ERR_PTR(-ENOMEM); } EXPORT_SYMBOL(rxrpc_kernel_lookup_peer); @@ -652,9 +654,6 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname, goto success; case RXRPC_SECURITY_KEY: - ret = -EINVAL; - if (rx->key) - goto error; ret = -EISCONN; if (rx->sk.sk_state != RXRPC_UNBOUND) goto error; @@ -662,9 +661,6 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname, goto error; case RXRPC_SECURITY_KEYRING: - ret = -EINVAL; - if (rx->key) - goto error; ret = -EISCONN; if (rx->sk.sk_state != RXRPC_UNBOUND) goto error; diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index 5b7342d434869..96ecb83c90715 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -117,7 +117,7 @@ struct rxrpc_net { atomic_t stat_tx_jumbo[10]; atomic_t stat_rx_jumbo[10]; - atomic_t stat_why_req_ack[8]; + atomic_t stat_why_req_ack[9]; atomic_t stat_io_loop; }; @@ -387,7 +387,7 @@ struct rxrpc_peer { struct rb_root service_conns; /* Service connections */ struct list_head keepalive_link; /* Link in net->peer_keepalive[] */ unsigned long app_data; /* Application data (e.g. afs_server) */ - time64_t last_tx_at; /* Last time packet sent here */ + unsigned int last_tx_at; /* Last time packet sent here (time64_t LSW) */ seqlock_t service_conn_lock; spinlock_t lock; /* access lock */ int debug_id; /* debug ID for printks */ @@ -1379,6 +1379,13 @@ void rxrpc_peer_keepalive_worker(struct work_struct *); void rxrpc_input_probe_for_pmtud(struct rxrpc_connection *conn, rxrpc_serial_t acked_serial, bool sendmsg_fail); +/* Update the last transmission time on a peer for keepalive purposes. */ +static inline void rxrpc_peer_mark_tx(struct rxrpc_peer *peer) +{ + /* To avoid tearing on 32-bit systems, we only keep the LSW. */ + WRITE_ONCE(peer->last_tx_at, ktime_get_seconds()); +} + /* * peer_object.c */ diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index 918f41d97a2f9..f035f486c1397 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -654,11 +654,9 @@ void rxrpc_put_call(struct rxrpc_call *call, enum rxrpc_call_trace why) if (dead) { ASSERTCMP(__rxrpc_call_state(call), ==, RXRPC_CALL_COMPLETE); - if (!list_empty(&call->link)) { - spin_lock(&rxnet->call_lock); - list_del_init(&call->link); - spin_unlock(&rxnet->call_lock); - } + spin_lock(&rxnet->call_lock); + list_del_rcu(&call->link); + spin_unlock(&rxnet->call_lock); rxrpc_cleanup_call(call); } @@ -694,6 +692,7 @@ static void rxrpc_destroy_call(struct work_struct *work) rxrpc_put_bundle(call->bundle, rxrpc_bundle_put_call); rxrpc_put_peer(call->peer, rxrpc_peer_put_call); rxrpc_put_local(call->local, rxrpc_local_put_call); + key_put(call->key); call_rcu(&call->rcu, rxrpc_rcu_free_call); } @@ -730,24 +729,20 @@ void rxrpc_destroy_all_calls(struct rxrpc_net *rxnet) _enter(""); if (!list_empty(&rxnet->calls)) { - spin_lock(&rxnet->call_lock); + int shown = 0; - while (!list_empty(&rxnet->calls)) { - call = list_entry(rxnet->calls.next, - struct rxrpc_call, link); - _debug("Zapping call %p", call); + spin_lock(&rxnet->call_lock); - rxrpc_see_call(call, rxrpc_call_see_zap); - list_del_init(&call->link); + list_for_each_entry(call, &rxnet->calls, link) { + rxrpc_see_call(call, rxrpc_call_see_still_live); pr_err("Call %p still in use (%d,%s,%lx,%lx)!\n", call, refcount_read(&call->ref), rxrpc_call_states[__rxrpc_call_state(call)], call->flags, call->events); - spin_unlock(&rxnet->call_lock); - cond_resched(); - spin_lock(&rxnet->call_lock); + if (++shown >= 10) + break; } spin_unlock(&rxnet->call_lock); diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index 232b6986da83e..9a41ec708aeb9 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -194,7 +194,7 @@ void rxrpc_conn_retransmit_call(struct rxrpc_connection *conn, } ret = kernel_sendmsg(conn->local->socket, &msg, iov, ioc, len); - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); if (ret < 0) trace_rxrpc_tx_fail(chan->call_debug_id, serial, ret, rxrpc_tx_point_call_final_resend); @@ -247,6 +247,7 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, struct sk_buff *skb) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); + bool secured = false; int ret; if (conn->state == RXRPC_CONN_ABORTED) @@ -262,6 +263,13 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, return ret; case RXRPC_PACKET_TYPE_RESPONSE: + spin_lock_irq(&conn->state_lock); + if (conn->state != RXRPC_CONN_SERVICE_CHALLENGING) { + spin_unlock_irq(&conn->state_lock); + return 0; + } + spin_unlock_irq(&conn->state_lock); + ret = conn->security->verify_response(conn, skb); if (ret < 0) return ret; @@ -272,11 +280,13 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, return ret; spin_lock_irq(&conn->state_lock); - if (conn->state == RXRPC_CONN_SERVICE_CHALLENGING) + if (conn->state == RXRPC_CONN_SERVICE_CHALLENGING) { conn->state = RXRPC_CONN_SERVICE; + secured = true; + } spin_unlock_irq(&conn->state_lock); - if (conn->state == RXRPC_CONN_SERVICE) { + if (secured) { /* Offload call state flipping to the I/O thread. As * we've already received the packet, put it on the * front of the queue. @@ -557,11 +567,11 @@ void rxrpc_post_response(struct rxrpc_connection *conn, struct sk_buff *skb) spin_lock_irq(&local->lock); old = conn->tx_response; if (old) { - struct rxrpc_skb_priv *osp = rxrpc_skb(skb); + struct rxrpc_skb_priv *osp = rxrpc_skb(old); /* Always go with the response to the most recent challenge. */ if (after(sp->resp.challenge_serial, osp->resp.challenge_serial)) - conn->tx_response = old; + conn->tx_response = skb; else old = skb; } else { @@ -569,4 +579,5 @@ void rxrpc_post_response(struct rxrpc_connection *conn, struct sk_buff *skb) } spin_unlock_irq(&local->lock); rxrpc_poke_conn(conn, rxrpc_conn_get_poke_response); + rxrpc_free_skb(old, rxrpc_skb_put_old_response); } diff --git a/net/rxrpc/input_rack.c b/net/rxrpc/input_rack.c index 13c371261e0a5..9eb109ffba56e 100644 --- a/net/rxrpc/input_rack.c +++ b/net/rxrpc/input_rack.c @@ -413,6 +413,6 @@ void rxrpc_rack_timer_expired(struct rxrpc_call *call, ktime_t overran_by) break; //case RXRPC_CALL_RACKTIMER_ZEROWIN: default: - pr_warn("Unexpected rack timer %u", call->rack_timer_mode); + pr_warn("Unexpected rack timer %u", mode); } } diff --git a/net/rxrpc/io_thread.c b/net/rxrpc/io_thread.c index e939ecf417c4b..6979569319252 100644 --- a/net/rxrpc/io_thread.c +++ b/net/rxrpc/io_thread.c @@ -419,7 +419,8 @@ static int rxrpc_input_packet_on_conn(struct rxrpc_connection *conn, if (sp->hdr.callNumber > chan->call_id) { if (rxrpc_to_client(sp)) { - rxrpc_put_call(call, rxrpc_call_put_input); + if (call) + rxrpc_put_call(call, rxrpc_call_put_input); return rxrpc_protocol_error(skb, rxrpc_eproto_unexpected_implicit_end); } diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c index 9fdc1f031c9da..b0e05c827dac4 100644 --- a/net/rxrpc/key.c +++ b/net/rxrpc/key.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -72,7 +73,7 @@ static int rxrpc_preparse_xdr_rxkad(struct key_preparsed_payload *prep, return -EKEYREJECTED; plen = sizeof(*token) + sizeof(*token->kad) + tktlen; - prep->quotalen = datalen + plen; + prep->quotalen += datalen + plen; plen -= sizeof(*token); token = kzalloc(sizeof(*token), GFP_KERNEL); @@ -171,7 +172,7 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, size_t plen; const __be32 *ticket, *key; s64 tmp; - u32 tktlen, keylen; + size_t raw_keylen, raw_tktlen, keylen, tktlen; _enter(",{%x,%x,%x,%x},%x", ntohl(xdr[0]), ntohl(xdr[1]), ntohl(xdr[2]), ntohl(xdr[3]), @@ -181,32 +182,36 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, goto reject; key = xdr + (6 * 2 + 1); - keylen = ntohl(key[-1]); - _debug("keylen: %x", keylen); - keylen = round_up(keylen, 4); + raw_keylen = ntohl(key[-1]); + _debug("keylen: %zx", raw_keylen); + if (raw_keylen > AFSTOKEN_GK_KEY_MAX) + goto reject; + keylen = round_up(raw_keylen, 4); if ((6 * 2 + 2) * 4 + keylen > toklen) goto reject; ticket = xdr + (6 * 2 + 1 + (keylen / 4) + 1); - tktlen = ntohl(ticket[-1]); - _debug("tktlen: %x", tktlen); - tktlen = round_up(tktlen, 4); + raw_tktlen = ntohl(ticket[-1]); + _debug("tktlen: %zx", raw_tktlen); + if (raw_tktlen > AFSTOKEN_GK_TOKEN_MAX) + goto reject; + tktlen = round_up(raw_tktlen, 4); if ((6 * 2 + 2) * 4 + keylen + tktlen != toklen) { - kleave(" = -EKEYREJECTED [%x!=%x, %x,%x]", + kleave(" = -EKEYREJECTED [%zx!=%x, %zx,%zx]", (6 * 2 + 2) * 4 + keylen + tktlen, toklen, keylen, tktlen); goto reject; } plen = sizeof(*token) + sizeof(*token->rxgk) + tktlen + keylen; - prep->quotalen = datalen + plen; + prep->quotalen += datalen + plen; plen -= sizeof(*token); token = kzalloc(sizeof(*token), GFP_KERNEL); if (!token) goto nomem; - token->rxgk = kzalloc(sizeof(*token->rxgk) + keylen, GFP_KERNEL); + token->rxgk = kzalloc(struct_size_t(struct rxgk_key, _key, raw_keylen), GFP_KERNEL); if (!token->rxgk) goto nomem_token; @@ -221,9 +226,9 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, token->rxgk->enctype = tmp = xdr_dec64(xdr + 5 * 2); if (tmp < 0 || tmp > UINT_MAX) goto reject_token; - token->rxgk->key.len = ntohl(key[-1]); + token->rxgk->key.len = raw_keylen; token->rxgk->key.data = token->rxgk->_key; - token->rxgk->ticket.len = ntohl(ticket[-1]); + token->rxgk->ticket.len = raw_tktlen; if (token->rxgk->endtime != 0) { expiry = rxrpc_s64_to_time64(token->rxgk->endtime); @@ -236,8 +241,7 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, memcpy(token->rxgk->key.data, key, token->rxgk->key.len); /* Pad the ticket so that we can use it directly in XDR */ - token->rxgk->ticket.data = kzalloc(round_up(token->rxgk->ticket.len, 4), - GFP_KERNEL); + token->rxgk->ticket.data = kzalloc(tktlen, GFP_KERNEL); if (!token->rxgk->ticket.data) goto nomem_yrxgk; memcpy(token->rxgk->ticket.data, ticket, token->rxgk->ticket.len); @@ -274,6 +278,7 @@ static int rxrpc_preparse_xdr_yfs_rxgk(struct key_preparsed_payload *prep, nomem: return -ENOMEM; reject_token: + kfree(token->rxgk); kfree(token); reject: return -EKEYREJECTED; @@ -460,6 +465,7 @@ static int rxrpc_preparse(struct key_preparsed_payload *prep) memcpy(&kver, prep->data, sizeof(kver)); prep->data += sizeof(kver); prep->datalen -= sizeof(kver); + prep->quotalen = 0; _debug("KEY I/F VERSION: %u", kver); @@ -496,8 +502,12 @@ static int rxrpc_preparse(struct key_preparsed_payload *prep) if (v1->security_index != RXRPC_SECURITY_RXKAD) goto error; + ret = -EKEYREJECTED; + if (v1->ticket_length > AFSTOKEN_RK_TIX_MAX) + goto error; + plen = sizeof(*token->kad) + v1->ticket_length; - prep->quotalen = plen + sizeof(*token); + prep->quotalen += plen + sizeof(*token); ret = -ENOMEM; token = kzalloc(sizeof(*token), GFP_KERNEL); @@ -616,7 +626,7 @@ int rxrpc_request_key(struct rxrpc_sock *rx, sockptr_t optval, int optlen) _enter(""); - if (optlen <= 0 || optlen > PAGE_SIZE - 1 || rx->securities) + if (optlen <= 0 || optlen > PAGE_SIZE - 1 || rx->key) return -EINVAL; description = memdup_sockptr_nul(optval, optlen); diff --git a/net/rxrpc/output.c b/net/rxrpc/output.c index 8b5903b6e481a..870e59bf06af2 100644 --- a/net/rxrpc/output.c +++ b/net/rxrpc/output.c @@ -275,7 +275,7 @@ static void rxrpc_send_ack_packet(struct rxrpc_call *call, int nr_kv, size_t len rxrpc_local_dont_fragment(conn->local, why == rxrpc_propose_ack_ping_for_mtu_probe); ret = do_udp_sendmsg(conn->local->socket, &msg, len); - call->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(call->peer); if (ret < 0) { trace_rxrpc_tx_fail(call->debug_id, serial, ret, rxrpc_tx_point_call_ack); @@ -411,7 +411,7 @@ int rxrpc_send_abort_packet(struct rxrpc_call *call) iov_iter_kvec(&msg.msg_iter, WRITE, iov, 1, sizeof(pkt)); ret = do_udp_sendmsg(conn->local->socket, &msg, sizeof(pkt)); - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); if (ret < 0) trace_rxrpc_tx_fail(call->debug_id, serial, ret, rxrpc_tx_point_call_abort); @@ -479,6 +479,8 @@ static size_t rxrpc_prepare_data_subpacket(struct rxrpc_call *call, why = rxrpc_reqack_old_rtt; else if (!last && !after(READ_ONCE(call->send_top), txb->seq)) why = rxrpc_reqack_app_stall; + else if (call->tx_winsize <= (2 * req->n) || call->cong_cwnd <= (2 * req->n)) + why = rxrpc_reqack_jumbo_win; else goto dont_set_request_ack; @@ -698,7 +700,7 @@ void rxrpc_send_data_packet(struct rxrpc_call *call, struct rxrpc_send_data_req ret = 0; trace_rxrpc_tx_data(call, txb->seq, txb->serial, txb->flags, rxrpc_txdata_inject_loss); - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); goto done; } } @@ -711,7 +713,7 @@ void rxrpc_send_data_packet(struct rxrpc_call *call, struct rxrpc_send_data_req */ rxrpc_inc_stat(call->rxnet, stat_tx_data_send); ret = do_udp_sendmsg(conn->local->socket, &msg, len); - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); if (ret == -EMSGSIZE) { rxrpc_inc_stat(call->rxnet, stat_tx_data_send_msgsize); @@ -797,7 +799,7 @@ void rxrpc_send_conn_abort(struct rxrpc_connection *conn) trace_rxrpc_tx_packet(conn->debug_id, &whdr, rxrpc_tx_point_conn_abort); - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); } /* @@ -917,7 +919,7 @@ void rxrpc_send_keepalive(struct rxrpc_peer *peer) trace_rxrpc_tx_packet(peer->debug_id, &whdr, rxrpc_tx_point_version_keepalive); - peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(peer); _leave(""); } @@ -973,7 +975,7 @@ void rxrpc_send_response(struct rxrpc_connection *conn, struct sk_buff *response if (ret < 0) goto fail; - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); return; fail: diff --git a/net/rxrpc/peer_event.c b/net/rxrpc/peer_event.c index 7f4729234957e..9d02448ac062c 100644 --- a/net/rxrpc/peer_event.c +++ b/net/rxrpc/peer_event.c @@ -237,6 +237,21 @@ static void rxrpc_distribute_error(struct rxrpc_peer *peer, struct sk_buff *skb, spin_unlock_irq(&peer->lock); } +/* + * Reconstruct the last transmission time. The difference calculated should be + * valid provided no more than ~68 years elapsed since the last transmission. + */ +static time64_t rxrpc_peer_get_tx_mark(const struct rxrpc_peer *peer, time64_t base) +{ + s32 last_tx_at = READ_ONCE(peer->last_tx_at); + s32 base_lsw = base; + s32 diff = last_tx_at - base_lsw; + + diff = clamp(diff, -RXRPC_KEEPALIVE_TIME, RXRPC_KEEPALIVE_TIME); + + return diff + base; +} + /* * Perform keep-alive pings. */ @@ -265,7 +280,7 @@ static void rxrpc_peer_keepalive_dispatch(struct rxrpc_net *rxnet, spin_unlock_bh(&rxnet->peer_hash_lock); if (use) { - keepalive_at = peer->last_tx_at + RXRPC_KEEPALIVE_TIME; + keepalive_at = rxrpc_peer_get_tx_mark(peer, base) + RXRPC_KEEPALIVE_TIME; slot = keepalive_at - base; _debug("%02x peer %u t=%d {%pISp}", cursor, peer->debug_id, slot, &peer->srx.transport); diff --git a/net/rxrpc/proc.c b/net/rxrpc/proc.c index d803562ca0ac1..e9a27fa7b25d8 100644 --- a/net/rxrpc/proc.c +++ b/net/rxrpc/proc.c @@ -10,6 +10,10 @@ #include #include "ar-internal.h" +#define RXRPC_PROC_ADDRBUF_SIZE \ + (sizeof("[xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255]") + \ + sizeof(":12345")) + static const char *const rxrpc_conn_states[RXRPC_CONN__NR_STATES] = { [RXRPC_CONN_UNUSED] = "Unused ", [RXRPC_CONN_CLIENT_UNSECURED] = "ClUnsec ", @@ -53,7 +57,7 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v) struct rxrpc_net *rxnet = rxrpc_net(seq_file_net(seq)); enum rxrpc_call_state state; rxrpc_seq_t tx_bottom; - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; long timeout = 0; if (v == &rxnet->calls) { @@ -69,11 +73,11 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v) local = call->local; if (local) - sprintf(lbuff, "%pISpc", &local->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &local->srx.transport); else strcpy(lbuff, "no_local"); - sprintf(rbuff, "%pISpc", &call->dest_srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &call->dest_srx.transport); state = rxrpc_call_state(call); if (state != RXRPC_CALL_SERVER_PREALLOC) @@ -142,7 +146,7 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v) struct rxrpc_connection *conn; struct rxrpc_net *rxnet = rxrpc_net(seq_file_net(seq)); const char *state; - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == &rxnet->conn_proc_list) { seq_puts(seq, @@ -161,8 +165,8 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v) goto print; } - sprintf(lbuff, "%pISpc", &conn->local->srx.transport); - sprintf(rbuff, "%pISpc", &conn->peer->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &conn->local->srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &conn->peer->srx.transport); print: state = rxrpc_is_conn_aborted(conn) ? rxrpc_call_completions[conn->completion] : @@ -228,7 +232,7 @@ static int rxrpc_bundle_seq_show(struct seq_file *seq, void *v) { struct rxrpc_bundle *bundle; struct rxrpc_net *rxnet = rxrpc_net(seq_file_net(seq)); - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == &rxnet->bundle_proc_list) { seq_puts(seq, @@ -242,8 +246,8 @@ static int rxrpc_bundle_seq_show(struct seq_file *seq, void *v) bundle = list_entry(v, struct rxrpc_bundle, proc_link); - sprintf(lbuff, "%pISpc", &bundle->local->srx.transport); - sprintf(rbuff, "%pISpc", &bundle->peer->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &bundle->local->srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &bundle->peer->srx.transport); seq_printf(seq, "UDP %-47.47s %-47.47s %4x %3u %3d" " %c%c%c %08x | %08x %08x %08x %08x %08x\n", @@ -279,7 +283,7 @@ static int rxrpc_peer_seq_show(struct seq_file *seq, void *v) { struct rxrpc_peer *peer; time64_t now; - char lbuff[50], rbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE], rbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == SEQ_START_TOKEN) { seq_puts(seq, @@ -290,19 +294,19 @@ static int rxrpc_peer_seq_show(struct seq_file *seq, void *v) peer = list_entry(v, struct rxrpc_peer, hash_link); - sprintf(lbuff, "%pISpc", &peer->local->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &peer->local->srx.transport); - sprintf(rbuff, "%pISpc", &peer->srx.transport); + scnprintf(rbuff, sizeof(rbuff), "%pISpc", &peer->srx.transport); now = ktime_get_seconds(); seq_printf(seq, - "UDP %-47.47s %-47.47s %3u %4u %5u %6llus %8d %8d\n", + "UDP %-47.47s %-47.47s %3u %4u %5u %6ds %8d %8d\n", lbuff, rbuff, refcount_read(&peer->ref), peer->cong_ssthresh, peer->max_data, - now - peer->last_tx_at, + (s32)now - (s32)READ_ONCE(peer->last_tx_at), READ_ONCE(peer->recent_srtt_us), READ_ONCE(peer->recent_rto_us)); @@ -401,7 +405,7 @@ const struct seq_operations rxrpc_peer_seq_ops = { static int rxrpc_local_seq_show(struct seq_file *seq, void *v) { struct rxrpc_local *local; - char lbuff[50]; + char lbuff[RXRPC_PROC_ADDRBUF_SIZE]; if (v == SEQ_START_TOKEN) { seq_puts(seq, @@ -412,7 +416,7 @@ static int rxrpc_local_seq_show(struct seq_file *seq, void *v) local = hlist_entry(v, struct rxrpc_local, link); - sprintf(lbuff, "%pISpc", &local->srx.transport); + scnprintf(lbuff, sizeof(lbuff), "%pISpc", &local->srx.transport); seq_printf(seq, "UDP %-47.47s %3u %3u %3u\n", @@ -518,11 +522,12 @@ int rxrpc_stats_show(struct seq_file *seq, void *v) atomic_read(&rxnet->stat_rx_acks[RXRPC_ACK_IDLE]), atomic_read(&rxnet->stat_rx_acks[0])); seq_printf(seq, - "Why-Req-A: acklost=%u mrtt=%u ortt=%u stall=%u\n", + "Why-Req-A: acklost=%u mrtt=%u ortt=%u stall=%u jwin=%u\n", atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_ack_lost]), atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_more_rtt]), atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_old_rtt]), - atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_app_stall])); + atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_app_stall]), + atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_jumbo_win])); seq_printf(seq, "Why-Req-A: nolast=%u retx=%u slows=%u smtxw=%u\n", atomic_read(&rxnet->stat_why_req_ack[rxrpc_reqack_no_srv_last]), diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c index 7fa7e77f6bb99..e1f7513a46dbe 100644 --- a/net/rxrpc/recvmsg.c +++ b/net/rxrpc/recvmsg.c @@ -518,7 +518,8 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (rxrpc_call_has_failed(call)) goto call_failed; - if (!skb_queue_empty(&call->recvmsg_queue)) + if (!(flags & MSG_PEEK) && + !skb_queue_empty(&call->recvmsg_queue)) rxrpc_notify_socket(call); goto not_yet_complete; @@ -549,11 +550,21 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, error_requeue_call: if (!(flags & MSG_PEEK)) { spin_lock_irq(&rx->recvmsg_lock); - list_add(&call->recvmsg_link, &rx->recvmsg_q); - spin_unlock_irq(&rx->recvmsg_lock); + if (list_empty(&call->recvmsg_link)) { + list_add(&call->recvmsg_link, &rx->recvmsg_q); + rxrpc_see_call(call, rxrpc_call_see_recvmsg_requeue); + spin_unlock_irq(&rx->recvmsg_lock); + } else if (list_is_first(&call->recvmsg_link, &rx->recvmsg_q)) { + spin_unlock_irq(&rx->recvmsg_lock); + rxrpc_put_call(call, rxrpc_call_see_recvmsg_requeue_first); + } else { + list_move(&call->recvmsg_link, &rx->recvmsg_q); + spin_unlock_irq(&rx->recvmsg_lock); + rxrpc_put_call(call, rxrpc_call_see_recvmsg_requeue_move); + } trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_requeue, 0); } else { - rxrpc_put_call(call, rxrpc_call_put_recvmsg); + rxrpc_put_call(call, rxrpc_call_put_recvmsg_peek_nowait); } error_no_call: release_sock(&rx->sk); diff --git a/net/rxrpc/rxgk.c b/net/rxrpc/rxgk.c index dce5a3d8a964f..c39f5066d8e86 100644 --- a/net/rxrpc/rxgk.c +++ b/net/rxrpc/rxgk.c @@ -678,7 +678,7 @@ static int rxgk_issue_challenge(struct rxrpc_connection *conn) ret = do_udp_sendmsg(conn->local->socket, &msg, len); if (ret > 0) - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); __free_page(page); if (ret < 0) { @@ -1085,6 +1085,9 @@ static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn, _enter(""); + if ((end - p) * sizeof(__be32) < 24) + return rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO, + rxgk_abort_resp_short_auth); if (memcmp(p, conn->rxgk.nonce, 20) != 0) return rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO, rxgk_abort_resp_bad_nonce); @@ -1098,7 +1101,7 @@ static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn, p += xdr_round_up(app_len) / sizeof(__be32); if (end - p < 4) return rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO, - rxgk_abort_resp_short_applen); + rxgk_abort_resp_short_auth); level = ntohl(*p++); epoch = ntohl(*p++); @@ -1164,7 +1167,8 @@ static int rxgk_verify_authenticator(struct rxrpc_connection *conn, } p = auth; - ret = rxgk_do_verify_authenticator(conn, krb5, skb, p, p + auth_len); + ret = rxgk_do_verify_authenticator(conn, krb5, skb, p, + p + auth_len / sizeof(*p)); error: kfree(auth); return ret; @@ -1208,7 +1212,8 @@ static int rxgk_verify_response(struct rxrpc_connection *conn, token_offset = offset; token_len = ntohl(rhdr.token_len); - if (xdr_round_up(token_len) + sizeof(__be32) > len) + if (token_len > len || + xdr_round_up(token_len) + sizeof(__be32) > len) goto short_packet; trace_rxrpc_rx_response(conn, sp->hdr.serial, 0, sp->hdr.cksum, token_len); @@ -1223,7 +1228,7 @@ static int rxgk_verify_response(struct rxrpc_connection *conn, auth_offset = offset; auth_len = ntohl(xauth_len); - if (auth_len < len) + if (auth_len > len) goto short_packet; if (auth_len & 3) goto inconsistent; @@ -1268,16 +1273,18 @@ static int rxgk_verify_response(struct rxrpc_connection *conn, if (ret < 0) { rxrpc_abort_conn(conn, skb, RXGK_SEALEDINCON, ret, rxgk_abort_resp_auth_dec); - goto out; + goto out_gk; } ret = rxgk_verify_authenticator(conn, krb5, skb, auth_offset, auth_len); if (ret < 0) - goto out; + goto out_gk; conn->key = key; key = NULL; ret = 0; +out_gk: + rxgk_put(gk); out: key_put(key); _leave(" = %d", ret); diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index 3657c0661cdc7..4a3c630941499 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -197,6 +197,7 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, struct rxrpc_crypt iv; __be32 *tmpbuf; size_t tmpsize = 4 * sizeof(__be32); + int ret; _enter(""); @@ -225,13 +226,13 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, skcipher_request_set_sync_tfm(req, ci); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, tmpsize, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_free(req); memcpy(&conn->rxkad.csum_iv, tmpbuf + 2, sizeof(conn->rxkad.csum_iv)); kfree(tmpbuf); - _leave(" = 0"); - return 0; + _leave(" = %d", ret); + return ret; } /* @@ -264,6 +265,7 @@ static int rxkad_secure_packet_auth(const struct rxrpc_call *call, struct scatterlist sg; size_t pad; u16 check; + int ret; _enter(""); @@ -286,11 +288,11 @@ static int rxkad_secure_packet_auth(const struct rxrpc_call *call, skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_zero(req); - _leave(" = 0"); - return 0; + _leave(" = %d", ret); + return ret; } /* @@ -345,7 +347,7 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) union { __be32 buf[2]; } crypto __aligned(8); - u32 x, y; + u32 x, y = 0; int ret; _enter("{%d{%x}},{#%u},%u,", @@ -376,8 +378,10 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_zero(req); + if (ret < 0) + goto out; y = ntohl(crypto.buf[1]); y = (y >> 16) & 0xffff; @@ -413,6 +417,7 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) memset(p + txb->pkt_len, 0, gap); } +out: skcipher_request_free(req); _leave(" = %d [set %x]", ret, y); return ret; @@ -453,8 +458,10 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb, skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, 8, iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_zero(req); + if (ret < 0) + return ret; /* Extract the decrypted packet length */ if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0) @@ -531,10 +538,14 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, sp->len, iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_zero(req); if (sg != _sg) kfree(sg); + if (ret < 0) { + WARN_ON_ONCE(ret != -ENOMEM); + return ret; + } /* Extract the decrypted packet length */ if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0) @@ -602,8 +613,10 @@ static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb) skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x); - crypto_skcipher_encrypt(req); + ret = crypto_skcipher_encrypt(req); skcipher_request_zero(req); + if (ret < 0) + goto out; y = ntohl(crypto.buf[1]); cksum = (y >> 16) & 0xffff; @@ -694,7 +707,7 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn) return -EAGAIN; } - conn->peer->last_tx_at = ktime_get_seconds(); + rxrpc_peer_mark_tx(conn->peer); trace_rxrpc_tx_packet(conn->debug_id, &whdr, rxrpc_tx_point_rxkad_challenge); _leave(" = 0"); @@ -958,6 +971,7 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, struct in_addr addr; unsigned int life; time64_t issue, now; + int ret; bool little_endian; u8 *p, *q, *name, *end; @@ -977,8 +991,11 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, sg_init_one(&sg[0], ticket, ticket_len); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, ticket_len, iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_free(req); + if (ret < 0) + return rxrpc_abort_conn(conn, skb, RXKADBADTICKET, -EPROTO, + rxkad_abort_resp_tkt_short); p = ticket; end = p + ticket_len; @@ -1073,21 +1090,23 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, /* * decrypt the response packet */ -static void rxkad_decrypt_response(struct rxrpc_connection *conn, - struct rxkad_response *resp, - const struct rxrpc_crypt *session_key) +static int rxkad_decrypt_response(struct rxrpc_connection *conn, + struct rxkad_response *resp, + const struct rxrpc_crypt *session_key) { struct skcipher_request *req = rxkad_ci_req; struct scatterlist sg[1]; struct rxrpc_crypt iv; + int ret; _enter(",,%08x%08x", ntohl(session_key->n[0]), ntohl(session_key->n[1])); mutex_lock(&rxkad_ci_mutex); - if (crypto_sync_skcipher_setkey(rxkad_ci, session_key->x, - sizeof(*session_key)) < 0) - BUG(); + ret = crypto_sync_skcipher_setkey(rxkad_ci, session_key->x, + sizeof(*session_key)); + if (ret < 0) + goto unlock; memcpy(&iv, session_key, sizeof(iv)); @@ -1096,12 +1115,14 @@ static void rxkad_decrypt_response(struct rxrpc_connection *conn, skcipher_request_set_sync_tfm(req, rxkad_ci); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, sizeof(resp->encrypted), iv.x); - crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_zero(req); +unlock: mutex_unlock(&rxkad_ci_mutex); _leave(""); + return ret; } /* @@ -1194,7 +1215,9 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, /* use the session key from inside the ticket to decrypt the * response */ - rxkad_decrypt_response(conn, response, &session_key); + ret = rxkad_decrypt_response(conn, response, &session_key); + if (ret < 0) + goto temporary_error_free_ticket; if (ntohl(response->encrypted.epoch) != conn->proto.epoch || ntohl(response->encrypted.cid) != conn->proto.cid || diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index ebbb78b842de8..39a6b21d6b8f7 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -637,7 +637,7 @@ rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, memset(&cp, 0, sizeof(cp)); cp.local = rx->local; cp.peer = peer; - cp.key = rx->key; + cp.key = key; cp.security_level = rx->min_sec_level; cp.exclusive = rx->exclusive | p->exclusive; cp.upgrade = p->upgrade; diff --git a/net/rxrpc/server_key.c b/net/rxrpc/server_key.c index 36b05fd842a7b..27491f1e12736 100644 --- a/net/rxrpc/server_key.c +++ b/net/rxrpc/server_key.c @@ -125,6 +125,9 @@ int rxrpc_server_keyring(struct rxrpc_sock *rx, sockptr_t optval, int optlen) _enter(""); + if (rx->securities) + return -EINVAL; + if (optlen <= 0 || optlen > PAGE_SIZE - 1) return -EINVAL; diff --git a/net/sched/act_api.c b/net/sched/act_api.c index ff6be5cfe2b05..e1ab0faeb8113 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -940,6 +940,8 @@ void tcf_idrinfo_destroy(const struct tc_action_ops *ops, int ret; idr_for_each_entry_ul(idr, p, tmp, id) { + if (IS_ERR(p)) + continue; if (tc_act_in_hw(p) && !mutex_taken) { rtnl_lock(); mutex_taken = true; diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 0939e6b2ba4d1..3a377604ad343 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -604,8 +604,12 @@ TC_INDIRECT_SCOPE int tcf_csum_act(struct sk_buff *skb, protocol = skb->protocol; orig_vlan_tag_present = true; } else { - struct vlan_hdr *vlan = (struct vlan_hdr *)skb->data; + struct vlan_hdr *vlan; + if (!pskb_may_pull(skb, VLAN_HLEN)) + goto drop; + + vlan = (struct vlan_hdr *)skb->data; protocol = vlan->h_vlan_encapsulated_proto; skb_pull(skb, VLAN_HLEN); skb_reset_network_header(skb); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 6749a4a9a9cd0..b3c160ad590d2 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1358,6 +1358,12 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, return -EINVAL; } + if (bind && !(flags & TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT)) { + NL_SET_ERR_MSG_MOD(extack, + "Attaching ct to a non ingress/clsact qdisc is unsupported"); + return -EOPNOTSUPP; + } + err = nla_parse_nested(tb, TCA_CT_MAX, nla, ct_policy, extack); if (err < 0) return err; diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index c1f75f2727576..d09013ae1892a 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -32,9 +32,12 @@ static ktime_t gate_get_time(struct tcf_gate *gact) return KTIME_MAX; } -static void gate_get_start_time(struct tcf_gate *gact, ktime_t *start) +static void tcf_gate_params_free_rcu(struct rcu_head *head); + +static void gate_get_start_time(struct tcf_gate *gact, + const struct tcf_gate_params *param, + ktime_t *start) { - struct tcf_gate_params *param = &gact->param; ktime_t now, base, cycle; u64 n; @@ -69,12 +72,14 @@ static enum hrtimer_restart gate_timer_func(struct hrtimer *timer) { struct tcf_gate *gact = container_of(timer, struct tcf_gate, hitimer); - struct tcf_gate_params *p = &gact->param; struct tcfg_gate_entry *next; + struct tcf_gate_params *p; ktime_t close_time, now; spin_lock(&gact->tcf_lock); + p = rcu_dereference_protected(gact->param, + lockdep_is_held(&gact->tcf_lock)); next = gact->next_entry; /* cycle start, clear pending bit, clear total octets */ @@ -225,6 +230,35 @@ static void release_entry_list(struct list_head *entries) } } +static int tcf_gate_copy_entries(struct tcf_gate_params *dst, + const struct tcf_gate_params *src, + struct netlink_ext_ack *extack) +{ + struct tcfg_gate_entry *entry; + int i = 0; + + list_for_each_entry(entry, &src->entries, list) { + struct tcfg_gate_entry *new; + + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) { + NL_SET_ERR_MSG(extack, "Not enough memory for entry"); + return -ENOMEM; + } + + new->index = entry->index; + new->gate_state = entry->gate_state; + new->interval = entry->interval; + new->ipv = entry->ipv; + new->maxoctets = entry->maxoctets; + list_add_tail(&new->list, &dst->entries); + i++; + } + + dst->num_entries = i; + return 0; +} + static int parse_gate_list(struct nlattr *list_attr, struct tcf_gate_params *sched, struct netlink_ext_ack *extack) @@ -270,24 +304,44 @@ static int parse_gate_list(struct nlattr *list_attr, return err; } -static void gate_setup_timer(struct tcf_gate *gact, u64 basetime, - enum tk_offsets tko, s32 clockid, - bool do_init) +static bool gate_timer_needs_cancel(u64 basetime, u64 old_basetime, + enum tk_offsets tko, + enum tk_offsets old_tko, + s32 clockid, s32 old_clockid) { - if (!do_init) { - if (basetime == gact->param.tcfg_basetime && - tko == gact->tk_offset && - clockid == gact->param.tcfg_clockid) - return; + return basetime != old_basetime || + clockid != old_clockid || + tko != old_tko; +} - spin_unlock_bh(&gact->tcf_lock); - hrtimer_cancel(&gact->hitimer); - spin_lock_bh(&gact->tcf_lock); +static int gate_clock_resolve(s32 clockid, enum tk_offsets *tko, + struct netlink_ext_ack *extack) +{ + switch (clockid) { + case CLOCK_REALTIME: + *tko = TK_OFFS_REAL; + return 0; + case CLOCK_MONOTONIC: + *tko = TK_OFFS_MAX; + return 0; + case CLOCK_BOOTTIME: + *tko = TK_OFFS_BOOT; + return 0; + case CLOCK_TAI: + *tko = TK_OFFS_TAI; + return 0; + default: + NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); + return -EINVAL; } - gact->param.tcfg_basetime = basetime; - gact->param.tcfg_clockid = clockid; - gact->tk_offset = tko; - hrtimer_setup(&gact->hitimer, gate_timer_func, clockid, HRTIMER_MODE_ABS_SOFT); +} + +static void gate_setup_timer(struct tcf_gate *gact, s32 clockid, + enum tk_offsets tko) +{ + WRITE_ONCE(gact->tk_offset, tko); + hrtimer_setup(&gact->hitimer, gate_timer_func, clockid, + HRTIMER_MODE_ABS_SOFT); } static int tcf_gate_init(struct net *net, struct nlattr *nla, @@ -296,15 +350,22 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, struct netlink_ext_ack *extack) { struct tc_action_net *tn = net_generic(net, act_gate_ops.net_id); - enum tk_offsets tk_offset = TK_OFFS_TAI; + u64 cycletime = 0, basetime = 0, cycletime_ext = 0; + struct tcf_gate_params *p = NULL, *old_p = NULL; + enum tk_offsets old_tk_offset = TK_OFFS_TAI; + const struct tcf_gate_params *cur_p = NULL; bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_GATE_MAX + 1]; + enum tk_offsets tko = TK_OFFS_TAI; struct tcf_chain *goto_ch = NULL; - u64 cycletime = 0, basetime = 0; - struct tcf_gate_params *p; + s32 timer_clockid = CLOCK_TAI; + bool use_old_entries = false; + s32 old_clockid = CLOCK_TAI; + bool need_cancel = false; s32 clockid = CLOCK_TAI; struct tcf_gate *gact; struct tc_gate *parm; + u64 old_basetime = 0; int ret = 0, err; u32 gflags = 0; s32 prio = -1; @@ -321,26 +382,8 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (!tb[TCA_GATE_PARMS]) return -EINVAL; - if (tb[TCA_GATE_CLOCKID]) { + if (tb[TCA_GATE_CLOCKID]) clockid = nla_get_s32(tb[TCA_GATE_CLOCKID]); - switch (clockid) { - case CLOCK_REALTIME: - tk_offset = TK_OFFS_REAL; - break; - case CLOCK_MONOTONIC: - tk_offset = TK_OFFS_MAX; - break; - case CLOCK_BOOTTIME: - tk_offset = TK_OFFS_BOOT; - break; - case CLOCK_TAI: - tk_offset = TK_OFFS_TAI; - break; - default: - NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); - return -EINVAL; - } - } parm = nla_data(tb[TCA_GATE_PARMS]); index = parm->index; @@ -366,6 +409,60 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, return -EEXIST; } + gact = to_gate(*a); + + err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); + if (err < 0) + goto release_idr; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + err = -ENOMEM; + goto chain_put; + } + INIT_LIST_HEAD(&p->entries); + + use_old_entries = !tb[TCA_GATE_ENTRY_LIST]; + if (!use_old_entries) { + err = parse_gate_list(tb[TCA_GATE_ENTRY_LIST], p, extack); + if (err < 0) + goto err_free; + use_old_entries = !err; + } + + if (ret == ACT_P_CREATED && use_old_entries) { + NL_SET_ERR_MSG(extack, "The entry list is empty"); + err = -EINVAL; + goto err_free; + } + + if (ret != ACT_P_CREATED) { + rcu_read_lock(); + cur_p = rcu_dereference(gact->param); + + old_basetime = cur_p->tcfg_basetime; + old_clockid = cur_p->tcfg_clockid; + old_tk_offset = READ_ONCE(gact->tk_offset); + + basetime = old_basetime; + cycletime_ext = cur_p->tcfg_cycletime_ext; + prio = cur_p->tcfg_priority; + gflags = cur_p->tcfg_flags; + + if (!tb[TCA_GATE_CLOCKID]) + clockid = old_clockid; + + err = 0; + if (use_old_entries) { + err = tcf_gate_copy_entries(p, cur_p, extack); + if (!err && !tb[TCA_GATE_CYCLE_TIME]) + cycletime = cur_p->tcfg_cycletime; + } + rcu_read_unlock(); + if (err) + goto err_free; + } + if (tb[TCA_GATE_PRIORITY]) prio = nla_get_s32(tb[TCA_GATE_PRIORITY]); @@ -375,25 +472,26 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (tb[TCA_GATE_FLAGS]) gflags = nla_get_u32(tb[TCA_GATE_FLAGS]); - gact = to_gate(*a); - if (ret == ACT_P_CREATED) - INIT_LIST_HEAD(&gact->param.entries); + if (tb[TCA_GATE_CYCLE_TIME]) + cycletime = nla_get_u64(tb[TCA_GATE_CYCLE_TIME]); - err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); - if (err < 0) - goto release_idr; + if (tb[TCA_GATE_CYCLE_TIME_EXT]) + cycletime_ext = nla_get_u64(tb[TCA_GATE_CYCLE_TIME_EXT]); - spin_lock_bh(&gact->tcf_lock); - p = &gact->param; + err = gate_clock_resolve(clockid, &tko, extack); + if (err) + goto err_free; + timer_clockid = clockid; - if (tb[TCA_GATE_CYCLE_TIME]) - cycletime = nla_get_u64(tb[TCA_GATE_CYCLE_TIME]); + need_cancel = ret != ACT_P_CREATED && + gate_timer_needs_cancel(basetime, old_basetime, + tko, old_tk_offset, + timer_clockid, old_clockid); - if (tb[TCA_GATE_ENTRY_LIST]) { - err = parse_gate_list(tb[TCA_GATE_ENTRY_LIST], p, extack); - if (err < 0) - goto chain_put; - } + if (need_cancel) + hrtimer_cancel(&gact->hitimer); + + spin_lock_bh(&gact->tcf_lock); if (!cycletime) { struct tcfg_gate_entry *entry; @@ -402,22 +500,20 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, list_for_each_entry(entry, &p->entries, list) cycle = ktime_add_ns(cycle, entry->interval); cycletime = cycle; - if (!cycletime) { - err = -EINVAL; - goto chain_put; - } } p->tcfg_cycletime = cycletime; + p->tcfg_cycletime_ext = cycletime_ext; - if (tb[TCA_GATE_CYCLE_TIME_EXT]) - p->tcfg_cycletime_ext = - nla_get_u64(tb[TCA_GATE_CYCLE_TIME_EXT]); - - gate_setup_timer(gact, basetime, tk_offset, clockid, - ret == ACT_P_CREATED); + if (need_cancel || ret == ACT_P_CREATED) + gate_setup_timer(gact, timer_clockid, tko); p->tcfg_priority = prio; p->tcfg_flags = gflags; - gate_get_start_time(gact, &start); + p->tcfg_basetime = basetime; + p->tcfg_clockid = timer_clockid; + gate_get_start_time(gact, p, &start); + + old_p = rcu_replace_pointer(gact->param, p, + lockdep_is_held(&gact->tcf_lock)); gact->current_close_time = start; gact->current_gate_status = GATE_ACT_GATE_OPEN | GATE_ACT_PENDING; @@ -434,11 +530,15 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (goto_ch) tcf_chain_put_by_act(goto_ch); + if (old_p) + call_rcu(&old_p->rcu, tcf_gate_params_free_rcu); + return ret; +err_free: + release_entry_list(&p->entries); + kfree(p); chain_put: - spin_unlock_bh(&gact->tcf_lock); - if (goto_ch) tcf_chain_put_by_act(goto_ch); release_idr: @@ -446,21 +546,29 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, * without taking tcf_lock. */ if (ret == ACT_P_CREATED) - gate_setup_timer(gact, gact->param.tcfg_basetime, - gact->tk_offset, gact->param.tcfg_clockid, - true); + gate_setup_timer(gact, timer_clockid, tko); + tcf_idr_release(*a, bind); return err; } +static void tcf_gate_params_free_rcu(struct rcu_head *head) +{ + struct tcf_gate_params *p = container_of(head, struct tcf_gate_params, rcu); + + release_entry_list(&p->entries); + kfree(p); +} + static void tcf_gate_cleanup(struct tc_action *a) { struct tcf_gate *gact = to_gate(a); struct tcf_gate_params *p; - p = &gact->param; hrtimer_cancel(&gact->hitimer); - release_entry_list(&p->entries); + p = rcu_dereference_protected(gact->param, 1); + if (p) + call_rcu(&p->rcu, tcf_gate_params_free_rcu); } static int dumping_entry(struct sk_buff *skb, @@ -509,10 +617,9 @@ static int tcf_gate_dump(struct sk_buff *skb, struct tc_action *a, struct nlattr *entry_list; struct tcf_t t; - spin_lock_bh(&gact->tcf_lock); - opt.action = gact->tcf_action; - - p = &gact->param; + rcu_read_lock(); + opt.action = READ_ONCE(gact->tcf_action); + p = rcu_dereference(gact->param); if (nla_put(skb, TCA_GATE_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -552,12 +659,12 @@ static int tcf_gate_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &gact->tcf_tm); if (nla_put_64bit(skb, TCA_GATE_TM, sizeof(t), &t, TCA_GATE_PAD)) goto nla_put_failure; - spin_unlock_bh(&gact->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&gact->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 7c6975632fc2e..4ad01d4e820db 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -293,8 +293,8 @@ static int load_metaops_and_vet(u32 metaid, void *val, int len, bool rtnl_held) /* called when adding new meta information */ static int __add_metainfo(const struct tcf_meta_ops *ops, - struct tcf_ife_info *ife, u32 metaid, void *metaval, - int len, bool atomic, bool exists) + struct tcf_ife_params *p, u32 metaid, void *metaval, + int len, bool atomic) { struct tcf_meta_info *mi = NULL; int ret = 0; @@ -313,45 +313,40 @@ static int __add_metainfo(const struct tcf_meta_ops *ops, } } - if (exists) - spin_lock_bh(&ife->tcf_lock); - list_add_tail(&mi->metalist, &ife->metalist); - if (exists) - spin_unlock_bh(&ife->tcf_lock); + list_add_tail(&mi->metalist, &p->metalist); return ret; } static int add_metainfo_and_get_ops(const struct tcf_meta_ops *ops, - struct tcf_ife_info *ife, u32 metaid, - bool exists) + struct tcf_ife_params *p, u32 metaid) { int ret; if (!try_module_get(ops->owner)) return -ENOENT; - ret = __add_metainfo(ops, ife, metaid, NULL, 0, true, exists); + ret = __add_metainfo(ops, p, metaid, NULL, 0, true); if (ret) module_put(ops->owner); return ret; } -static int add_metainfo(struct tcf_ife_info *ife, u32 metaid, void *metaval, - int len, bool exists) +static int add_metainfo(struct tcf_ife_params *p, u32 metaid, void *metaval, + int len) { const struct tcf_meta_ops *ops = find_ife_oplist(metaid); int ret; if (!ops) return -ENOENT; - ret = __add_metainfo(ops, ife, metaid, metaval, len, false, exists); + ret = __add_metainfo(ops, p, metaid, metaval, len, false); if (ret) /*put back what find_ife_oplist took */ module_put(ops->owner); return ret; } -static int use_all_metadata(struct tcf_ife_info *ife, bool exists) +static int use_all_metadata(struct tcf_ife_params *p) { struct tcf_meta_ops *o; int rc = 0; @@ -359,7 +354,7 @@ static int use_all_metadata(struct tcf_ife_info *ife, bool exists) read_lock(&ife_mod_lock); list_for_each_entry(o, &ifeoplist, list) { - rc = add_metainfo_and_get_ops(o, ife, o->metaid, exists); + rc = add_metainfo_and_get_ops(o, p, o->metaid); if (rc == 0) installed += 1; } @@ -371,7 +366,7 @@ static int use_all_metadata(struct tcf_ife_info *ife, bool exists) return -EINVAL; } -static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) +static int dump_metalist(struct sk_buff *skb, struct tcf_ife_params *p) { struct tcf_meta_info *e; struct nlattr *nest; @@ -379,14 +374,14 @@ static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) int total_encoded = 0; /*can only happen on decode */ - if (list_empty(&ife->metalist)) + if (list_empty(&p->metalist)) return 0; nest = nla_nest_start_noflag(skb, TCA_IFE_METALST); if (!nest) goto out_nlmsg_trim; - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry(e, &p->metalist, metalist) { if (!e->ops->get(skb, e)) total_encoded += 1; } @@ -403,13 +398,11 @@ static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) return -1; } -/* under ife->tcf_lock */ -static void _tcf_ife_cleanup(struct tc_action *a) +static void __tcf_ife_cleanup(struct tcf_ife_params *p) { - struct tcf_ife_info *ife = to_ife(a); struct tcf_meta_info *e, *n; - list_for_each_entry_safe(e, n, &ife->metalist, metalist) { + list_for_each_entry_safe(e, n, &p->metalist, metalist) { list_del(&e->metalist); if (e->metaval) { if (e->ops->release) @@ -422,18 +415,23 @@ static void _tcf_ife_cleanup(struct tc_action *a) } } +static void tcf_ife_cleanup_params(struct rcu_head *head) +{ + struct tcf_ife_params *p = container_of(head, struct tcf_ife_params, + rcu); + + __tcf_ife_cleanup(p); + kfree(p); +} + static void tcf_ife_cleanup(struct tc_action *a) { struct tcf_ife_info *ife = to_ife(a); struct tcf_ife_params *p; - spin_lock_bh(&ife->tcf_lock); - _tcf_ife_cleanup(a); - spin_unlock_bh(&ife->tcf_lock); - p = rcu_dereference_protected(ife->params, 1); if (p) - kfree_rcu(p, rcu); + call_rcu(&p->rcu, tcf_ife_cleanup_params); } static int load_metalist(struct nlattr **tb, bool rtnl_held) @@ -455,8 +453,7 @@ static int load_metalist(struct nlattr **tb, bool rtnl_held) return 0; } -static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, - bool exists, bool rtnl_held) +static int populate_metalist(struct tcf_ife_params *p, struct nlattr **tb) { int len = 0; int rc = 0; @@ -468,7 +465,7 @@ static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, val = nla_data(tb[i]); len = nla_len(tb[i]); - rc = add_metainfo(ife, i, val, len, exists); + rc = add_metainfo(p, i, val, len); if (rc) return rc; } @@ -523,6 +520,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, p = kzalloc(sizeof(*p), GFP_KERNEL); if (!p) return -ENOMEM; + INIT_LIST_HEAD(&p->metalist); if (tb[TCA_IFE_METALST]) { err = nla_parse_nested_deprecated(tb2, IFE_META_MAX, @@ -567,8 +565,6 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, } ife = to_ife(*a); - if (ret == ACT_P_CREATED) - INIT_LIST_HEAD(&ife->metalist); err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); if (err < 0) @@ -600,8 +596,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, } if (tb[TCA_IFE_METALST]) { - err = populate_metalist(ife, tb2, exists, - !(flags & TCA_ACT_FLAGS_NO_RTNL)); + err = populate_metalist(p, tb2); if (err) goto metadata_parse_err; } else { @@ -610,7 +605,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, * as we can. You better have at least one else we are * going to bail out */ - err = use_all_metadata(ife, exists); + err = use_all_metadata(p); if (err) goto metadata_parse_err; } @@ -626,13 +621,14 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, if (goto_ch) tcf_chain_put_by_act(goto_ch); if (p) - kfree_rcu(p, rcu); + call_rcu(&p->rcu, tcf_ife_cleanup_params); return ret; metadata_parse_err: if (goto_ch) tcf_chain_put_by_act(goto_ch); release_idr: + __tcf_ife_cleanup(p); kfree(p); tcf_idr_release(*a, bind); return err; @@ -649,9 +645,9 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, memset(&opt, 0, sizeof(opt)); - opt.index = ife->tcf_index, - opt.refcnt = refcount_read(&ife->tcf_refcnt) - ref, - opt.bindcnt = atomic_read(&ife->tcf_bindcnt) - bind, + opt.index = ife->tcf_index; + opt.refcnt = refcount_read(&ife->tcf_refcnt) - ref; + opt.bindcnt = atomic_read(&ife->tcf_bindcnt) - bind; spin_lock_bh(&ife->tcf_lock); opt.action = ife->tcf_action; @@ -679,7 +675,7 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, if (nla_put(skb, TCA_IFE_TYPE, 2, &p->eth_type)) goto nla_put_failure; - if (dump_metalist(skb, ife)) { + if (dump_metalist(skb, p)) { /*ignore failure to dump metalist */ pr_info("Failed to dump metalist\n"); } @@ -693,13 +689,13 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, return -1; } -static int find_decode_metaid(struct sk_buff *skb, struct tcf_ife_info *ife, +static int find_decode_metaid(struct sk_buff *skb, struct tcf_ife_params *p, u16 metaid, u16 mlen, void *mdata) { struct tcf_meta_info *e; /* XXX: use hash to speed up */ - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (metaid == e->metaid) { if (e->ops) { /* We check for decode presence already */ @@ -716,10 +712,13 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, { struct tcf_ife_info *ife = to_ife(a); int action = ife->tcf_action; + struct tcf_ife_params *p; u8 *ifehdr_end; u8 *tlv_data; u16 metalen; + p = rcu_dereference_bh(ife->params); + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); tcf_lastuse_update(&ife->tcf_tm); @@ -745,7 +744,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, return TC_ACT_SHOT; } - if (find_decode_metaid(skb, ife, mtype, dlen, curr_data)) { + if (find_decode_metaid(skb, p, mtype, dlen, curr_data)) { /* abuse overlimits to count when we receive metadata * but dont have an ops for it */ @@ -769,12 +768,12 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, /*XXX: check if we can do this at install time instead of current * send data path **/ -static int ife_get_sz(struct sk_buff *skb, struct tcf_ife_info *ife) +static int ife_get_sz(struct sk_buff *skb, struct tcf_ife_params *p) { - struct tcf_meta_info *e, *n; + struct tcf_meta_info *e; int tot_run_sz = 0, run_sz = 0; - list_for_each_entry_safe(e, n, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (e->ops->check_presence) { run_sz = e->ops->check_presence(skb, e); tot_run_sz += run_sz; @@ -795,7 +794,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, OUTERHDR:TOTMETALEN:{TLVHDR:Metadatum:TLVHDR..}:ORIGDATA where ORIGDATA = original ethernet header ... */ - u16 metalen = ife_get_sz(skb, ife); + u16 metalen = ife_get_sz(skb, p); int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; @@ -821,6 +820,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, /* could be stupid policy setup or mtu config * so lets be conservative.. */ if ((action == TC_ACT_SHOT) || exceed_mtu) { +drop: qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats)); return TC_ACT_SHOT; } @@ -829,27 +829,24 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, skb_push(skb, skb->dev->hard_header_len); ife_meta = ife_encode(skb, metalen); - - spin_lock(&ife->tcf_lock); + if (!ife_meta) + goto drop; /* XXX: we dont have a clever way of telling encode to * not repeat some of the computations that are done by * ops->presence_check... */ - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (e->ops->encode) { err = e->ops->encode(skb, (void *)(ife_meta + skboff), e); } if (err < 0) { /* too corrupt to keep around if overwritten */ - spin_unlock(&ife->tcf_lock); - qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats)); - return TC_ACT_SHOT; + goto drop; } skboff += err; } - spin_unlock(&ife->tcf_lock); oethh = (struct ethhdr *)skb->data; if (!is_zero_ether_addr(p->eth_src)) diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index 8c1d1554f6575..5450c1293eb50 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -126,7 +126,7 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, struct tcf_skbedit *d; u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL; u16 *queue_mapping = NULL, *ptype = NULL; - u16 mapping_mod = 1; + u32 mapping_mod = 1; bool exists = false; int ret = 0, err; u32 index; @@ -194,6 +194,10 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, } mapping_mod = *queue_mapping_max - *queue_mapping + 1; + if (mapping_mod > U16_MAX) { + NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid."); + return -EINVAL; + } flags |= SKBEDIT_F_TXQ_SKBHASH; } if (*pure_flags & SKBEDIT_F_INHERITDSFIELD) diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c index adcb618a2bfca..e9bea9890777d 100644 --- a/net/sched/bpf_qdisc.c +++ b/net/sched/bpf_qdisc.c @@ -202,6 +202,12 @@ __bpf_kfunc void bpf_kfree_skb(struct sk_buff *skb) kfree_skb(skb); } +__bpf_kfunc void bpf_kfree_skb_dtor(void *skb) +{ + bpf_kfree_skb(skb); +} +CFI_NOSEAL(bpf_kfree_skb_dtor); + /* bpf_qdisc_skb_drop - Drop an skb by adding it to a deferred free list. * @skb: The skb whose reference to be released and dropped. * @to_free_list: The list of skbs to be dropped. @@ -449,7 +455,7 @@ static struct bpf_struct_ops bpf_Qdisc_ops = { .owner = THIS_MODULE, }; -BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb) +BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb_dtor) static int __init bpf_qdisc_kfunc_init(void) { diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index ecec0a1e1c1a0..b03dc4fe20e52 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2228,6 +2228,11 @@ static bool is_qdisc_ingress(__u32 classid) return (TC_H_MIN(classid) == TC_H_MIN(TC_H_MIN_INGRESS)); } +static bool is_ingress_or_clsact(struct tcf_block *block, struct Qdisc *q) +{ + return tcf_block_shared(block) || (q && !!(q->flags & TCQ_F_INGRESS)); +} + static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, struct netlink_ext_ack *extack) { @@ -2420,6 +2425,8 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, flags |= TCA_ACT_FLAGS_NO_RTNL; if (is_qdisc_ingress(parent)) flags |= TCA_ACT_FLAGS_AT_INGRESS; + if (is_ingress_or_clsact(block, q)) + flags |= TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT; err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, flags, extack); if (err == 0) { @@ -2962,6 +2969,7 @@ static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops, tcm->tcm__pad1 = 0; tcm->tcm__pad2 = 0; tcm->tcm_handle = 0; + tcm->tcm_info = 0; if (block->q) { tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex; tcm->tcm_parent = block->q->handle; diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 5693b41b093f3..edf1252c1fde7 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -503,8 +503,16 @@ static int flow_change(struct net *net, struct sk_buff *in_skb, } if (TC_H_MAJ(baseclass) == 0) { - struct Qdisc *q = tcf_block_q(tp->chain->block); + struct tcf_block *block = tp->chain->block; + struct Qdisc *q; + if (tcf_block_shared(block)) { + NL_SET_ERR_MSG(extack, + "Must specify baseclass when attaching flow filter to block"); + goto err2; + } + + q = tcf_block_q(block); baseclass = TC_H_MAKE(q->handle, baseclass); } if (TC_H_MIN(baseclass) == 0) diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index cdddc86952284..83a7372ea15c2 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -247,8 +247,18 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, struct nlattr *tb[TCA_FW_MAX + 1]; int err; - if (!opt) - return handle ? -EINVAL : 0; /* Succeed if it is old method. */ + if (!opt) { + if (handle) + return -EINVAL; + + if (tcf_block_shared(tp->chain->block)) { + NL_SET_ERR_MSG(extack, + "Must specify mark when attaching fw filter to block"); + return -EINVAL; + } + + return 0; /* Succeed if it is old method. */ + } err = nla_parse_nested_deprecated(tb, TCA_FW_MAX, opt, fw_policy, NULL); diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index 2a1c00048fd6f..58e849c0acf41 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -161,10 +161,8 @@ TC_INDIRECT_SCOPE int u32_classify(struct sk_buff *skb, int toff = off + key->off + (off2 & key->offmask); __be32 *data, hdata; - if (skb_headroom(skb) + toff > INT_MAX) - goto out; - - data = skb_header_pointer(skb, toff, 4, &hdata); + data = skb_header_pointer_careful(skb, toff, 4, + &hdata); if (!data) goto out; if ((*data ^ key->val) & key->mask) { @@ -214,8 +212,9 @@ TC_INDIRECT_SCOPE int u32_classify(struct sk_buff *skb, if (ht->divisor) { __be32 *data, hdata; - data = skb_header_pointer(skb, off + n->sel.hoff, 4, - &hdata); + data = skb_header_pointer_careful(skb, + off + n->sel.hoff, + 4, &hdata); if (!data) goto out; sel = ht->divisor & u32_hash_fold(*data, &n->sel, @@ -229,7 +228,7 @@ TC_INDIRECT_SCOPE int u32_classify(struct sk_buff *skb, if (n->sel.flags & TC_U32_VAROFFSET) { __be16 *data, hdata; - data = skb_header_pointer(skb, + data = skb_header_pointer_careful(skb, off + n->sel.offoff, 2, &hdata); if (!data) diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index 32bacfc314c26..d325a90cde9ee 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1597,7 +1597,6 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free) qdisc_drop_reason(skb, sch, to_free, SKB_DROP_REASON_QDISC_OVERLIMIT); sch->q.qlen--; - qdisc_tree_reduce_backlog(sch, 1, len); cake_heapify(q, 0); @@ -1743,14 +1742,14 @@ static void cake_reconfigure(struct Qdisc *sch); static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) { + u32 idx, tin, prev_qlen, prev_backlog, drop_id; struct cake_sched_data *q = qdisc_priv(sch); - int len = qdisc_pkt_len(skb); - int ret; + int len = qdisc_pkt_len(skb), ret; struct sk_buff *ack = NULL; ktime_t now = ktime_get(); struct cake_tin_data *b; struct cake_flow *flow; - u32 idx, tin; + bool same_flow = false; /* choose flow to insert into */ idx = cake_classify(sch, &b, skb, q->flow_mode, &ret); @@ -1823,6 +1822,8 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, consume_skb(skb); } else { /* not splitting */ + int ack_pkt_len = 0; + cobalt_set_enqueue_time(skb, now); get_cobalt_cb(skb)->adjusted_len = cake_overhead(q, skb); flow_queue_add(flow, skb); @@ -1833,13 +1834,13 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (ack) { b->ack_drops++; sch->qstats.drops++; - b->bytes += qdisc_pkt_len(ack); - len -= qdisc_pkt_len(ack); + ack_pkt_len = qdisc_pkt_len(ack); + b->bytes += ack_pkt_len; q->buffer_used += skb->truesize - ack->truesize; if (q->rate_flags & CAKE_FLAG_INGRESS) cake_advance_shaper(q, b, ack, now, true); - qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(ack)); + qdisc_tree_reduce_backlog(sch, 1, ack_pkt_len); consume_skb(ack); } else { sch->q.qlen++; @@ -1848,11 +1849,11 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, /* stats */ b->packets++; - b->bytes += len; - b->backlogs[idx] += len; - b->tin_backlog += len; - sch->qstats.backlog += len; - q->avg_window_bytes += len; + b->bytes += len - ack_pkt_len; + b->backlogs[idx] += len - ack_pkt_len; + b->tin_backlog += len - ack_pkt_len; + sch->qstats.backlog += len - ack_pkt_len; + q->avg_window_bytes += len - ack_pkt_len; } if (q->overflow_timeout) @@ -1927,24 +1928,29 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (q->buffer_used > q->buffer_max_used) q->buffer_max_used = q->buffer_used; - if (q->buffer_used > q->buffer_limit) { - bool same_flow = false; - u32 dropped = 0; - u32 drop_id; + if (q->buffer_used <= q->buffer_limit) + return NET_XMIT_SUCCESS; - while (q->buffer_used > q->buffer_limit) { - dropped++; - drop_id = cake_drop(sch, to_free); + prev_qlen = sch->q.qlen; + prev_backlog = sch->qstats.backlog; - if ((drop_id >> 16) == tin && - (drop_id & 0xFFFF) == idx) - same_flow = true; - } - b->drop_overlimit += dropped; + while (q->buffer_used > q->buffer_limit) { + drop_id = cake_drop(sch, to_free); + if ((drop_id >> 16) == tin && + (drop_id & 0xFFFF) == idx) + same_flow = true; + } + + prev_qlen -= sch->q.qlen; + prev_backlog -= sch->qstats.backlog; + b->drop_overlimit += prev_qlen; - if (same_flow) - return NET_XMIT_CN; + if (same_flow) { + qdisc_tree_reduce_backlog(sch, prev_qlen - 1, + prev_backlog - len); + return NET_XMIT_CN; } + qdisc_tree_reduce_backlog(sch, prev_qlen, prev_backlog); return NET_XMIT_SUCCESS; } diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index 82635dd2cfa59..a4b07b661b775 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -115,12 +115,12 @@ static void ets_offload_change(struct Qdisc *sch) struct ets_sched *q = qdisc_priv(sch); struct tc_ets_qopt_offload qopt; unsigned int w_psum_prev = 0; - unsigned int q_psum = 0; - unsigned int q_sum = 0; unsigned int quantum; unsigned int w_psum; unsigned int weight; unsigned int i; + u64 q_psum = 0; + u64 q_sum = 0; if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) return; @@ -138,8 +138,12 @@ static void ets_offload_change(struct Qdisc *sch) for (i = 0; i < q->nbands; i++) { quantum = q->classes[i].quantum; - q_psum += quantum; - w_psum = quantum ? q_psum * 100 / q_sum : 0; + if (quantum) { + q_psum += quantum; + w_psum = div64_u64(q_psum * 100, q_sum); + } else { + w_psum = 0; + } weight = w_psum - w_psum_prev; w_psum_prev = w_psum; @@ -652,7 +656,7 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, sch_tree_lock(sch); for (i = nbands; i < oldbands; i++) { - if (i >= q->nstrict && q->classes[i].qdisc->q.qlen) + if (cl_is_active(&q->classes[i])) list_del_init(&q->classes[i].alist); qdisc_purge_queue(q->classes[i].qdisc); } @@ -664,6 +668,10 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, q->classes[i].deficit = quanta[i]; } } + for (i = q->nstrict; i < nstrict; i++) { + if (cl_is_active(&q->classes[i])) + list_del_init(&q->classes[i].alist); + } WRITE_ONCE(q->nstrict, nstrict); memcpy(q->prio2band, priomap, sizeof(priomap)); diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index fee922da2f99c..5e41930079948 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -826,6 +826,7 @@ static void fq_reset(struct Qdisc *sch) for (idx = 0; idx < FQ_BANDS; idx++) { q->band_flows[idx].new_flows.first = NULL; q->band_flows[idx].old_flows.first = NULL; + q->band_pkt_count[idx] = 0; } q->delayed = RB_ROOT; q->flows = 0; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 7dee9748a56be..30d77ad7b81d2 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -1297,33 +1297,6 @@ static void dev_deactivate_queue(struct net_device *dev, } } -static void dev_reset_queue(struct net_device *dev, - struct netdev_queue *dev_queue, - void *_unused) -{ - struct Qdisc *qdisc; - bool nolock; - - qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); - if (!qdisc) - return; - - nolock = qdisc->flags & TCQ_F_NOLOCK; - - if (nolock) - spin_lock_bh(&qdisc->seqlock); - spin_lock_bh(qdisc_lock(qdisc)); - - qdisc_reset(qdisc); - - spin_unlock_bh(qdisc_lock(qdisc)); - if (nolock) { - clear_bit(__QDISC_STATE_MISSED, &qdisc->state); - clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); - spin_unlock_bh(&qdisc->seqlock); - } -} - static bool some_qdisc_is_busy(struct net_device *dev) { unsigned int i; diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index d8fd35da32a7c..57221522fe56d 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -555,7 +555,7 @@ static void rtsc_min(struct runtime_sc *rtsc, struct internal_sc *isc, u64 x, u64 y) { u64 y1, y2, dx, dy; - u32 dsm; + u64 dsm; if (isc->sm1 <= isc->sm2) { /* service curve is convex */ @@ -598,7 +598,7 @@ rtsc_min(struct runtime_sc *rtsc, struct internal_sc *isc, u64 x, u64 y) */ dx = (y1 - y) << SM_SHIFT; dsm = isc->sm1 - isc->sm2; - do_div(dx, dsm); + dx = div64_u64(dx, dsm); /* * check if (x, y1) belongs to the 1st segment of rtsc. * if so, add the offset. diff --git a/net/sched/sch_ingress.c b/net/sched/sch_ingress.c index cc6051d4f2ef8..c3e18bae8fbfc 100644 --- a/net/sched/sch_ingress.c +++ b/net/sched/sch_ingress.c @@ -113,14 +113,15 @@ static void ingress_destroy(struct Qdisc *sch) { struct ingress_sched_data *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); - struct bpf_mprog_entry *entry = rtnl_dereference(dev->tcx_ingress); + struct bpf_mprog_entry *entry; if (sch->parent != TC_H_INGRESS) return; tcf_block_put_ext(q->block, sch, &q->block_info); - if (entry) { + if (mini_qdisc_pair_inited(&q->miniqp)) { + entry = rtnl_dereference(dev->tcx_ingress); tcx_miniq_dec(entry); if (!tcx_entry_is_active(entry)) { tcx_entry_update(dev, NULL, true); @@ -290,10 +291,9 @@ static int clsact_init(struct Qdisc *sch, struct nlattr *opt, static void clsact_destroy(struct Qdisc *sch) { + struct bpf_mprog_entry *ingress_entry, *egress_entry; struct clsact_sched_data *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); - struct bpf_mprog_entry *ingress_entry = rtnl_dereference(dev->tcx_ingress); - struct bpf_mprog_entry *egress_entry = rtnl_dereference(dev->tcx_egress); if (sch->parent != TC_H_CLSACT) return; @@ -301,7 +301,8 @@ static void clsact_destroy(struct Qdisc *sch) tcf_block_put_ext(q->ingress_block, sch, &q->ingress_block_info); tcf_block_put_ext(q->egress_block, sch, &q->egress_block_info); - if (ingress_entry) { + if (mini_qdisc_pair_inited(&q->miniqp_ingress)) { + ingress_entry = rtnl_dereference(dev->tcx_ingress); tcx_miniq_dec(ingress_entry); if (!tcx_entry_is_active(ingress_entry)) { tcx_entry_update(dev, NULL, true); @@ -309,7 +310,8 @@ static void clsact_destroy(struct Qdisc *sch) } } - if (egress_entry) { + if (mini_qdisc_pair_inited(&q->miniqp_egress)) { + egress_entry = rtnl_dereference(dev->tcx_egress); tcx_miniq_dec(egress_entry); if (!tcx_entry_is_active(egress_entry)) { tcx_entry_update(dev, NULL, false); diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index eafc316ae319e..6f8fcc4b504ce 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -518,8 +518,9 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, goto finish_segs; } - skb->data[get_random_u32_below(skb_headlen(skb))] ^= - 1<data[get_random_u32_below(skb_headlen(skb))] ^= + 1 << get_random_u32_below(8); } if (unlikely(q->t_len >= sch->limit)) { diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index 2255355e51d35..f94c9c9c90424 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -373,7 +373,7 @@ static void qfq_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) /* Deschedule class and remove it from its parent aggregate. */ static void qfq_deact_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) { - if (cl->qdisc->q.qlen > 0) /* class is active */ + if (cl_is_active(cl)) /* class is active */ qfq_deactivate_class(q, cl); qfq_rm_from_agg(q, cl); @@ -529,8 +529,10 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, return 0; destroy_class: - qdisc_put(cl->qdisc); - kfree(cl); + if (!existing) { + qdisc_put(cl->qdisc); + kfree(cl); + } return err; } @@ -1481,7 +1483,7 @@ static void qfq_reset_qdisc(struct Qdisc *sch) for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { - if (cl->qdisc->q.qlen > 0) + if (cl_is_active(cl)) qfq_deactivate_class(q, cl); qdisc_reset(cl->qdisc); diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 8badec6d82a24..ec4039a201a2c 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -146,15 +146,12 @@ teql_destroy(struct Qdisc *sch) master->slaves = NEXT_SLAVE(q); if (q == master->slaves) { struct netdev_queue *txq; - spinlock_t *root_lock; txq = netdev_get_tx_queue(master->dev, 0); master->slaves = NULL; - root_lock = qdisc_root_sleeping_lock(rtnl_dereference(txq->qdisc)); - spin_lock_bh(root_lock); - qdisc_reset(rtnl_dereference(txq->qdisc)); - spin_unlock_bh(root_lock); + dev_reset_queue(master->dev, + txq, NULL); } } skb_queue_purge(&dat->q); @@ -178,6 +175,11 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, if (m->dev == dev) return -ELOOP; + if (sch->parent != TC_H_ROOT) { + NL_SET_ERR_MSG_MOD(extack, "teql can only be used as root"); + return -EOPNOTSUPP; + } + q->m = m; skb_queue_head_init(&q->q); @@ -310,6 +312,7 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) if (__netif_tx_trylock(slave_txq)) { unsigned int length = qdisc_pkt_len(skb); + skb->dev = slave; if (!netif_xmit_frozen_or_stopped(slave_txq) && netdev_start_xmit(skb, slave, slave_txq, false) == NETDEV_TX_OK) { diff --git a/net/sctp/ipv6.c b/net/sctp/ipv6.c index 568ff8797c393..d725b21587588 100644 --- a/net/sctp/ipv6.c +++ b/net/sctp/ipv6.c @@ -782,9 +782,10 @@ static struct sock *sctp_v6_create_accept_sk(struct sock *sk, struct sctp_association *asoc, bool kern) { - struct sock *newsk; struct ipv6_pinfo *newnp, *np = inet6_sk(sk); struct sctp6_sock *newsctp6sk; + struct inet_sock *newinet; + struct sock *newsk; newsk = sk_alloc(sock_net(sk), PF_INET6, GFP_KERNEL, sk->sk_prot, kern); if (!newsk) @@ -796,7 +797,9 @@ static struct sock *sctp_v6_create_accept_sk(struct sock *sk, sock_reset_flag(sk, SOCK_ZAPPED); newsctp6sk = (struct sctp6_sock *)newsk; - inet_sk(newsk)->pinet6 = &newsctp6sk->inet6; + newinet = inet_sk(newsk); + newinet->pinet6 = &newsctp6sk->inet6; + newinet->ipv6_fl_list = NULL; sctp_sk(newsk)->v4mapped = sctp_sk(sk)->v4mapped; @@ -805,7 +808,6 @@ static struct sock *sctp_v6_create_accept_sk(struct sock *sk, memcpy(newnp, np, sizeof(struct ipv6_pinfo)); newnp->ipv6_mc_list = NULL; newnp->ipv6_ac_list = NULL; - newnp->ipv6_fl_list = NULL; sctp_v6_copy_ip_options(sk, newsk); diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 3755ba079d077..7b823d7591419 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -603,6 +603,11 @@ enum sctp_disposition sctp_sf_do_5_1C_ack(struct net *net, sctp_add_cmd_sf(commands, SCTP_CMD_PEER_INIT, SCTP_PEER_INIT(initchunk)); + /* SCTP-AUTH: generate the association shared keys so that + * we can potentially sign the COOKIE-ECHO. + */ + sctp_add_cmd_sf(commands, SCTP_CMD_ASSOC_SHKEY, SCTP_NULL()); + /* Reset init error count upon receipt of INIT-ACK. */ sctp_add_cmd_sf(commands, SCTP_CMD_INIT_COUNTER_RESET, SCTP_NULL()); @@ -617,11 +622,6 @@ enum sctp_disposition sctp_sf_do_5_1C_ack(struct net *net, sctp_add_cmd_sf(commands, SCTP_CMD_NEW_STATE, SCTP_STATE(SCTP_STATE_COOKIE_ECHOED)); - /* SCTP-AUTH: generate the association shared keys so that - * we can potentially sign the COOKIE-ECHO. - */ - sctp_add_cmd_sf(commands, SCTP_CMD_ASSOC_SHKEY, SCTP_NULL()); - /* 5.1 C) "A" shall then send the State Cookie received in the * INIT ACK chunk in a COOKIE ECHO chunk, ... */ diff --git a/net/sctp/socket.c b/net/sctp/socket.c index ed8293a342402..d190e75e46454 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -1553,8 +1553,6 @@ static void sctp_close(struct sock *sk, long timeout) spin_unlock_bh(&net->sctp.addr_wq_lock); sock_put(sk); - - SCTP_DBG_OBJCNT_DEC(sock); } /* Handle EPIPE error. */ @@ -5109,9 +5107,12 @@ static void sctp_destroy_sock(struct sock *sk) sp->do_auto_asconf = 0; list_del(&sp->auto_asconf_list); } + sctp_endpoint_free(sp->ep); + sk_sockets_allocated_dec(sk); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); + SCTP_DBG_OBJCNT_DEC(sock); } static void sctp_destruct_sock(struct sock *sk) diff --git a/net/shaper/shaper.c b/net/shaper/shaper.c index 7101a48bce545..be9999ab62e39 100644 --- a/net/shaper/shaper.c +++ b/net/shaper/shaper.c @@ -36,29 +36,26 @@ static struct net_shaper_binding *net_shaper_binding_from_ctx(void *ctx) return &((struct net_shaper_nl_ctx *)ctx)->binding; } -static void net_shaper_lock(struct net_shaper_binding *binding) +static struct net_shaper_hierarchy * +net_shaper_hierarchy(struct net_shaper_binding *binding) { - switch (binding->type) { - case NET_SHAPER_BINDING_TYPE_NETDEV: - netdev_lock(binding->netdev); - break; - } -} + /* Pairs with WRITE_ONCE() in net_shaper_hierarchy_setup. */ + if (binding->type == NET_SHAPER_BINDING_TYPE_NETDEV) + return READ_ONCE(binding->netdev->net_shaper_hierarchy); -static void net_shaper_unlock(struct net_shaper_binding *binding) -{ - switch (binding->type) { - case NET_SHAPER_BINDING_TYPE_NETDEV: - netdev_unlock(binding->netdev); - break; - } + /* No other type supported yet. */ + return NULL; } static struct net_shaper_hierarchy * -net_shaper_hierarchy(struct net_shaper_binding *binding) +net_shaper_hierarchy_rcu(struct net_shaper_binding *binding) { - /* Pairs with WRITE_ONCE() in net_shaper_hierarchy_setup. */ - if (binding->type == NET_SHAPER_BINDING_TYPE_NETDEV) + /* Readers look up the device and take a ref, then take RCU lock + * later at which point netdev may have been unregistered and flushed. + * READ_ONCE() pairs with WRITE_ONCE() in net_shaper_hierarchy_setup. + */ + if (binding->type == NET_SHAPER_BINDING_TYPE_NETDEV && + READ_ONCE(binding->netdev->reg_state) <= NETREG_REGISTERED) return READ_ONCE(binding->netdev->net_shaper_hierarchy); /* No other type supported yet. */ @@ -204,12 +201,49 @@ static int net_shaper_ctx_setup(const struct genl_info *info, int type, return 0; } +/* Like net_shaper_ctx_setup(), but for "write" handlers (never for dumps!) + * Acquires the lock protecting the hierarchy (instance lock for netdev). + */ +static int net_shaper_ctx_setup_lock(const struct genl_info *info, int type, + struct net_shaper_nl_ctx *ctx) +{ + struct net *ns = genl_info_net(info); + struct net_device *dev; + int ifindex; + + if (GENL_REQ_ATTR_CHECK(info, type)) + return -EINVAL; + + ifindex = nla_get_u32(info->attrs[type]); + dev = netdev_get_by_index_lock(ns, ifindex); + if (!dev) { + NL_SET_BAD_ATTR(info->extack, info->attrs[type]); + return -ENOENT; + } + + if (!dev->netdev_ops->net_shaper_ops) { + NL_SET_BAD_ATTR(info->extack, info->attrs[type]); + netdev_unlock(dev); + return -EOPNOTSUPP; + } + + ctx->binding.type = NET_SHAPER_BINDING_TYPE_NETDEV; + ctx->binding.netdev = dev; + return 0; +} + static void net_shaper_ctx_cleanup(struct net_shaper_nl_ctx *ctx) { if (ctx->binding.type == NET_SHAPER_BINDING_TYPE_NETDEV) netdev_put(ctx->binding.netdev, &ctx->dev_tracker); } +static void net_shaper_ctx_cleanup_unlock(struct net_shaper_nl_ctx *ctx) +{ + if (ctx->binding.type == NET_SHAPER_BINDING_TYPE_NETDEV) + netdev_unlock(ctx->binding.netdev); +} + static u32 net_shaper_handle_to_index(const struct net_shaper_handle *handle) { return FIELD_PREP(NET_SHAPER_SCOPE_MASK, handle->scope) | @@ -251,9 +285,10 @@ static struct net_shaper * net_shaper_lookup(struct net_shaper_binding *binding, const struct net_shaper_handle *handle) { - struct net_shaper_hierarchy *hierarchy = net_shaper_hierarchy(binding); u32 index = net_shaper_handle_to_index(handle); + struct net_shaper_hierarchy *hierarchy; + hierarchy = net_shaper_hierarchy_rcu(binding); if (!hierarchy || xa_get_mark(&hierarchy->shapers, index, NET_SHAPER_NOT_VALID)) return NULL; @@ -262,7 +297,7 @@ net_shaper_lookup(struct net_shaper_binding *binding, } /* Allocate on demand the per device shaper's hierarchy container. - * Called under the net shaper lock + * Called under the lock protecting the hierarchy (instance lock for netdev) */ static struct net_shaper_hierarchy * net_shaper_hierarchy_setup(struct net_shaper_binding *binding) @@ -681,6 +716,22 @@ void net_shaper_nl_post_doit(const struct genl_split_ops *ops, net_shaper_generic_post(info); } +int net_shaper_nl_pre_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info) +{ + struct net_shaper_nl_ctx *ctx = (struct net_shaper_nl_ctx *)info->ctx; + + BUILD_BUG_ON(sizeof(*ctx) > sizeof(info->ctx)); + + return net_shaper_ctx_setup_lock(info, NET_SHAPER_A_IFINDEX, ctx); +} + +void net_shaper_nl_post_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info) +{ + net_shaper_ctx_cleanup_unlock((struct net_shaper_nl_ctx *)info->ctx); +} + int net_shaper_nl_pre_dumpit(struct netlink_callback *cb) { struct net_shaper_nl_ctx *ctx = (struct net_shaper_nl_ctx *)cb->ctx; @@ -759,11 +810,7 @@ int net_shaper_nl_get_doit(struct sk_buff *skb, struct genl_info *info) if (ret) goto free_msg; - ret = genlmsg_reply(msg, info); - if (ret) - goto free_msg; - - return 0; + return genlmsg_reply(msg, info); free_msg: nlmsg_free(msg); @@ -782,17 +829,19 @@ int net_shaper_nl_get_dumpit(struct sk_buff *skb, /* Don't error out dumps performed before any set operation. */ binding = net_shaper_binding_from_ctx(ctx); - hierarchy = net_shaper_hierarchy(binding); - if (!hierarchy) - return 0; rcu_read_lock(); + hierarchy = net_shaper_hierarchy_rcu(binding); + if (!hierarchy) + goto out_unlock; + for (; (shaper = xa_find(&hierarchy->shapers, &ctx->start_index, U32_MAX, XA_PRESENT)); ctx->start_index++) { ret = net_shaper_fill_one(skb, binding, shaper, info); if (ret) break; } +out_unlock: rcu_read_unlock(); return ret; @@ -810,45 +859,38 @@ int net_shaper_nl_set_doit(struct sk_buff *skb, struct genl_info *info) binding = net_shaper_binding_from_ctx(info->ctx); - net_shaper_lock(binding); ret = net_shaper_parse_info(binding, info->attrs, info, &shaper, &exists); if (ret) - goto unlock; + return ret; if (!exists) net_shaper_default_parent(&shaper.handle, &shaper.parent); hierarchy = net_shaper_hierarchy_setup(binding); - if (!hierarchy) { - ret = -ENOMEM; - goto unlock; - } + if (!hierarchy) + return -ENOMEM; /* The 'set' operation can't create node-scope shapers. */ handle = shaper.handle; if (handle.scope == NET_SHAPER_SCOPE_NODE && - !net_shaper_lookup(binding, &handle)) { - ret = -ENOENT; - goto unlock; - } + !net_shaper_lookup(binding, &handle)) + return -ENOENT; ret = net_shaper_pre_insert(binding, &handle, info->extack); if (ret) - goto unlock; + return ret; ops = net_shaper_ops(binding); ret = ops->set(binding, &shaper, info->extack); if (ret) { net_shaper_rollback(binding); - goto unlock; + return ret; } net_shaper_commit(binding, 1, &shaper); -unlock: - net_shaper_unlock(binding); - return ret; + return 0; } static int __net_shaper_delete(struct net_shaper_binding *binding, @@ -1077,35 +1119,26 @@ int net_shaper_nl_delete_doit(struct sk_buff *skb, struct genl_info *info) binding = net_shaper_binding_from_ctx(info->ctx); - net_shaper_lock(binding); ret = net_shaper_parse_handle(info->attrs[NET_SHAPER_A_HANDLE], info, &handle); if (ret) - goto unlock; + return ret; hierarchy = net_shaper_hierarchy(binding); - if (!hierarchy) { - ret = -ENOENT; - goto unlock; - } + if (!hierarchy) + return -ENOENT; shaper = net_shaper_lookup(binding, &handle); - if (!shaper) { - ret = -ENOENT; - goto unlock; - } + if (!shaper) + return -ENOENT; if (handle.scope == NET_SHAPER_SCOPE_NODE) { ret = net_shaper_pre_del_node(binding, shaper, info->extack); if (ret) - goto unlock; + return ret; } - ret = __net_shaper_delete(binding, shaper, info->extack); - -unlock: - net_shaper_unlock(binding); - return ret; + return __net_shaper_delete(binding, shaper, info->extack); } static int net_shaper_group_send_reply(struct net_shaper_binding *binding, @@ -1154,21 +1187,17 @@ int net_shaper_nl_group_doit(struct sk_buff *skb, struct genl_info *info) if (!net_shaper_ops(binding)->group) return -EOPNOTSUPP; - net_shaper_lock(binding); leaves_count = net_shaper_list_len(info, NET_SHAPER_A_LEAVES); if (!leaves_count) { NL_SET_BAD_ATTR(info->extack, info->attrs[NET_SHAPER_A_LEAVES]); - ret = -EINVAL; - goto unlock; + return -EINVAL; } leaves = kcalloc(leaves_count, sizeof(struct net_shaper) + sizeof(struct net_shaper *), GFP_KERNEL); - if (!leaves) { - ret = -ENOMEM; - goto unlock; - } + if (!leaves) + return -ENOMEM; old_nodes = (void *)&leaves[leaves_count]; ret = net_shaper_parse_node(binding, info->attrs, info, &node); @@ -1245,9 +1274,6 @@ int net_shaper_nl_group_doit(struct sk_buff *skb, struct genl_info *info) free_leaves: kfree(leaves); - -unlock: - net_shaper_unlock(binding); return ret; free_msg: @@ -1314,10 +1340,7 @@ int net_shaper_nl_cap_get_doit(struct sk_buff *skb, struct genl_info *info) if (ret) goto free_msg; - ret = genlmsg_reply(msg, info); - if (ret) - goto free_msg; - return 0; + return genlmsg_reply(msg, info); free_msg: nlmsg_free(msg); @@ -1360,14 +1383,12 @@ static void net_shaper_flush(struct net_shaper_binding *binding) if (!hierarchy) return; - net_shaper_lock(binding); xa_lock(&hierarchy->shapers); xa_for_each(&hierarchy->shapers, index, cur) { __xa_erase(&hierarchy->shapers, index); kfree(cur); } xa_unlock(&hierarchy->shapers); - net_shaper_unlock(binding); kfree(hierarchy); } diff --git a/net/shaper/shaper_nl_gen.c b/net/shaper/shaper_nl_gen.c index 204c8ae8c7b14..c52abf13ff0c9 100644 --- a/net/shaper/shaper_nl_gen.c +++ b/net/shaper/shaper_nl_gen.c @@ -98,27 +98,27 @@ static const struct genl_split_ops net_shaper_nl_ops[] = { }, { .cmd = NET_SHAPER_CMD_SET, - .pre_doit = net_shaper_nl_pre_doit, + .pre_doit = net_shaper_nl_pre_doit_write, .doit = net_shaper_nl_set_doit, - .post_doit = net_shaper_nl_post_doit, + .post_doit = net_shaper_nl_post_doit_write, .policy = net_shaper_set_nl_policy, .maxattr = NET_SHAPER_A_IFINDEX, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, { .cmd = NET_SHAPER_CMD_DELETE, - .pre_doit = net_shaper_nl_pre_doit, + .pre_doit = net_shaper_nl_pre_doit_write, .doit = net_shaper_nl_delete_doit, - .post_doit = net_shaper_nl_post_doit, + .post_doit = net_shaper_nl_post_doit_write, .policy = net_shaper_delete_nl_policy, .maxattr = NET_SHAPER_A_IFINDEX, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, { .cmd = NET_SHAPER_CMD_GROUP, - .pre_doit = net_shaper_nl_pre_doit, + .pre_doit = net_shaper_nl_pre_doit_write, .doit = net_shaper_nl_group_doit, - .post_doit = net_shaper_nl_post_doit, + .post_doit = net_shaper_nl_post_doit_write, .policy = net_shaper_group_nl_policy, .maxattr = NET_SHAPER_A_LEAVES, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, diff --git a/net/shaper/shaper_nl_gen.h b/net/shaper/shaper_nl_gen.h index cb7f9026fc239..1e20eebdedd71 100644 --- a/net/shaper/shaper_nl_gen.h +++ b/net/shaper/shaper_nl_gen.h @@ -17,12 +17,17 @@ extern const struct nla_policy net_shaper_leaf_info_nl_policy[NET_SHAPER_A_WEIGH int net_shaper_nl_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); +int net_shaper_nl_pre_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info); int net_shaper_nl_cap_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); void net_shaper_nl_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); void +net_shaper_nl_post_doit_write(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info); +void net_shaper_nl_cap_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb, struct genl_info *info); int net_shaper_nl_pre_dumpit(struct netlink_callback *cb); diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 77b99e8ef35a4..6421c2e1c84de 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -124,12 +124,21 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct smc_sock *smc; struct sock *child; - smc = smc_clcsock_user_data(sk); + rcu_read_lock(); + smc = smc_clcsock_user_data_rcu(sk); + if (!smc || !refcount_inc_not_zero(&smc->sk.sk_refcnt)) { + rcu_read_unlock(); + smc = NULL; + goto drop; + } + rcu_read_unlock(); if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(&smc->queued_smc_hs) > sk->sk_max_ack_backlog) @@ -142,7 +151,7 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, /* passthrough to original syn recv sock fct */ child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash, - own_req); + own_req, opt_child_init); /* child must not inherit smc or its ops */ if (child) { rcu_assign_sk_user_data(child, NULL); @@ -151,11 +160,14 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, if (inet_csk(child)->icsk_af_ops == inet_csk(sk)->icsk_af_ops) inet_csk(child)->icsk_af_ops = smc->ori_af_ops; } + sock_put(&smc->sk); return child; drop: dst_release(dst); tcp_listendrop(sk); + if (smc) + sock_put(&smc->sk); return NULL; } @@ -252,7 +264,7 @@ static void smc_fback_restore_callbacks(struct smc_sock *smc) struct sock *clcsk = smc->clcsock->sk; write_lock_bh(&clcsk->sk_callback_lock); - clcsk->sk_user_data = NULL; + rcu_assign_sk_user_data(clcsk, NULL); smc_clcsock_restore_cb(&clcsk->sk_state_change, &smc->clcsk_state_change); smc_clcsock_restore_cb(&clcsk->sk_data_ready, &smc->clcsk_data_ready); @@ -900,7 +912,7 @@ static void smc_fback_replace_callbacks(struct smc_sock *smc) struct sock *clcsk = smc->clcsock->sk; write_lock_bh(&clcsk->sk_callback_lock); - clcsk->sk_user_data = (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); + __rcu_assign_sk_user_data_with_flags(clcsk, smc, SK_USER_DATA_NOCOPY); smc_clcsock_replace_cb(&clcsk->sk_state_change, smc_fback_state_change, &smc->clcsk_state_change); @@ -2663,8 +2675,8 @@ int smc_listen(struct socket *sock, int backlog) * smc-specific sk_data_ready function */ write_lock_bh(&smc->clcsock->sk->sk_callback_lock); - smc->clcsock->sk->sk_user_data = - (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); + __rcu_assign_sk_user_data_with_flags(smc->clcsock->sk, smc, + SK_USER_DATA_NOCOPY); smc_clcsock_replace_cb(&smc->clcsock->sk->sk_data_ready, smc_clcsock_data_ready, &smc->clcsk_data_ready); write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); @@ -2685,10 +2697,11 @@ int smc_listen(struct socket *sock, int backlog) write_lock_bh(&smc->clcsock->sk->sk_callback_lock); smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready, &smc->clcsk_data_ready); - smc->clcsock->sk->sk_user_data = NULL; + rcu_assign_sk_user_data(smc->clcsock->sk, NULL); write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); goto out; } + sock_set_flag(sk, SOCK_RCU_FREE); sk->sk_max_ack_backlog = backlog; sk->sk_ack_backlog = 0; sk->sk_state = SMC_LISTEN; diff --git a/net/smc/smc.h b/net/smc/smc.h index 2c90849637398..ea45467c11409 100644 --- a/net/smc/smc.h +++ b/net/smc/smc.h @@ -346,6 +346,11 @@ static inline struct smc_sock *smc_clcsock_user_data(const struct sock *clcsk) ((uintptr_t)clcsk->sk_user_data & ~SK_USER_DATA_NOCOPY); } +static inline struct smc_sock *smc_clcsock_user_data_rcu(const struct sock *clcsk) +{ + return (struct smc_sock *)rcu_dereference_sk_user_data(clcsk); +} + /* save target_cb in saved_cb, and replace target_cb with new_cb */ static inline void smc_clcsock_replace_cb(void (**target_cb)(struct sock *), void (*new_cb)(struct sock *), diff --git a/net/smc/smc_close.c b/net/smc/smc_close.c index 10219f55aad14..bb0313ef5f7c1 100644 --- a/net/smc/smc_close.c +++ b/net/smc/smc_close.c @@ -218,7 +218,7 @@ int smc_close_active(struct smc_sock *smc) write_lock_bh(&smc->clcsock->sk->sk_callback_lock); smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready, &smc->clcsk_data_ready); - smc->clcsock->sk->sk_user_data = NULL; + rcu_assign_sk_user_data(smc->clcsock->sk, NULL); write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR); } diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c index e7f1134453ef4..4a3d7b405132e 100644 --- a/net/smc/smc_rx.c +++ b/net/smc/smc_rx.c @@ -135,9 +135,16 @@ static void smc_rx_pipe_buf_release(struct pipe_inode_info *pipe, sock_put(sk); } +static bool smc_rx_pipe_buf_get(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + /* smc_spd_priv in buf->private is not shareable; disallow cloning. */ + return false; +} + static const struct pipe_buf_operations smc_pipe_ops = { .release = smc_rx_pipe_buf_release, - .get = generic_pipe_buf_get + .get = smc_rx_pipe_buf_get, }; static void smc_rx_spd_release(struct splice_pipe_desc *spd, diff --git a/net/socket.c b/net/socket.c index e8892b2187087..2b6e11b085ebd 100644 --- a/net/socket.c +++ b/net/socket.c @@ -674,7 +674,7 @@ static void __sock_release(struct socket *sock, struct inode *inode) iput(SOCK_INODE(sock)); return; } - sock->file = NULL; + WRITE_ONCE(sock->file, NULL); } /** diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 5c095cb8cb201..bb3c3db2713b1 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -39,6 +39,8 @@ static const struct rpc_authops authgss_ops; static const struct rpc_credops gss_credops; static const struct rpc_credops gss_nullops; +static void gss_free_callback(struct kref *kref); + #define GSS_RETRY_EXPIRED 5 static unsigned int gss_expired_cred_retry_delay = GSS_RETRY_EXPIRED; @@ -551,6 +553,7 @@ gss_alloc_msg(struct gss_auth *gss_auth, } return gss_msg; err_put_pipe_version: + kref_put(&gss_auth->kref, gss_free_callback); put_pipe_version(gss_auth->net); err_free_msg: kfree(gss_msg); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index 7d2cdc2bd374e..f320c0a8e6049 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -320,29 +320,47 @@ static int gssx_dec_status(struct xdr_stream *xdr, /* status->minor_status */ p = xdr_inline_decode(xdr, 8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_mech; + } p = xdr_decode_hyper(p, &status->minor_status); /* status->major_status_string */ err = gssx_dec_buffer(xdr, &status->major_status_string); if (err) - return err; + goto out_free_mech; /* status->minor_status_string */ err = gssx_dec_buffer(xdr, &status->minor_status_string); if (err) - return err; + goto out_free_major_status_string; /* status->server_ctx */ err = gssx_dec_buffer(xdr, &status->server_ctx); if (err) - return err; + goto out_free_minor_status_string; /* we assume we have no options for now, so simply consume them */ /* status->options */ err = dummy_dec_opt_array(xdr, &status->options); + if (err) + goto out_free_server_ctx; + return 0; + +out_free_server_ctx: + kfree(status->server_ctx.data); + status->server_ctx.data = NULL; +out_free_minor_status_string: + kfree(status->minor_status_string.data); + status->minor_status_string.data = NULL; +out_free_major_status_string: + kfree(status->major_status_string.data); + status->major_status_string.data = NULL; +out_free_mech: + kfree(status->mech.data); + status->mech.data = NULL; return err; } @@ -505,28 +523,35 @@ static int gssx_dec_name(struct xdr_stream *xdr, /* name->name_type */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_composite_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* we assume we have no attributes for now, so simply consume them */ /* name->name_attributes */ err = dummy_dec_nameattr_array(xdr, &dummy_name_attr_array); if (err) - return err; + goto out_free_display_name; /* we assume we have no options for now, so simply consume them */ /* name->extensions */ err = dummy_dec_opt_array(xdr, &dummy_option_array); + if (err) + goto out_free_display_name; + return 0; + +out_free_display_name: + kfree(name->display_name.data); + name->display_name.data = NULL; return err; } @@ -649,32 +674,34 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->state */ err = gssx_dec_buffer(xdr, &ctx->state); if (err) - return err; + goto out_free_exported_context_token; /* ctx->need_release */ err = gssx_dec_bool(xdr, &ctx->need_release); if (err) - return err; + goto out_free_state; /* ctx->mech */ err = gssx_dec_buffer(xdr, &ctx->mech); if (err) - return err; + goto out_free_state; /* ctx->src_name */ err = gssx_dec_name(xdr, &ctx->src_name); if (err) - return err; + goto out_free_mech; /* ctx->targ_name */ err = gssx_dec_name(xdr, &ctx->targ_name); if (err) - return err; + goto out_free_src_name; /* ctx->lifetime */ p = xdr_inline_decode(xdr, 8+8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_targ_name; + } p = xdr_decode_hyper(p, &ctx->lifetime); /* ctx->ctx_flags */ @@ -683,17 +710,36 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->locally_initiated */ err = gssx_dec_bool(xdr, &ctx->locally_initiated); if (err) - return err; + goto out_free_targ_name; /* ctx->open */ err = gssx_dec_bool(xdr, &ctx->open); if (err) - return err; + goto out_free_targ_name; /* we assume we have no options for now, so simply consume them */ /* ctx->options */ err = dummy_dec_opt_array(xdr, &ctx->options); + if (err) + goto out_free_targ_name; + + return 0; +out_free_targ_name: + kfree(ctx->targ_name.display_name.data); + ctx->targ_name.display_name.data = NULL; +out_free_src_name: + kfree(ctx->src_name.display_name.data); + ctx->src_name.display_name.data = NULL; +out_free_mech: + kfree(ctx->mech.data); + ctx->mech.data = NULL; +out_free_state: + kfree(ctx->state.data); + ctx->state.data = NULL; +out_free_exported_context_token: + kfree(ctx->exported_context_token.data); + ctx->exported_context_token.data = NULL; return err; } diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index a8ec30759a184..e2f0df8cdaa6a 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -1083,7 +1083,8 @@ static int gss_read_proxy_verf(struct svc_rqst *rqstp, } length = min_t(unsigned int, inlen, (char *)xdr->end - (char *)xdr->p); - memcpy(page_address(in_token->pages[0]), xdr->p, length); + if (length) + memcpy(page_address(in_token->pages[0]), xdr->p, length); inlen -= length; to_offs = length; diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 131090f31e6a8..6f6e0d4928afd 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -1061,14 +1061,25 @@ static int cache_release(struct inode *inode, struct file *filp, struct cache_reader *rp = filp->private_data; if (rp) { + struct cache_request *rq = NULL; + spin_lock(&queue_lock); if (rp->offset) { struct cache_queue *cq; - for (cq= &rp->q; &cq->list != &cd->queue; - cq = list_entry(cq->list.next, struct cache_queue, list)) + for (cq = &rp->q; &cq->list != &cd->queue; + cq = list_entry(cq->list.next, + struct cache_queue, list)) if (!cq->reader) { - container_of(cq, struct cache_request, q) - ->readers--; + struct cache_request *cr = + container_of(cq, + struct cache_request, q); + cr->readers--; + if (cr->readers == 0 && + !test_bit(CACHE_PENDING, + &cr->item->flags)) { + list_del(&cr->q.list); + rq = cr; + } break; } rp->offset = 0; @@ -1076,9 +1087,14 @@ static int cache_release(struct inode *inode, struct file *filp, list_del(&rp->q.list); spin_unlock(&queue_lock); + if (rq) { + cache_put(rq->item, cd); + kfree(rq->buf); + kfree(rq); + } + filp->private_data = NULL; kfree(rp); - } if (filp->f_mode & FMODE_WRITE) { atomic_dec(&cd->writers); diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 661b3fe2779f0..310de7a80be52 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -841,6 +841,9 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, for (page_no = 0; page_no < numpages; page_no++) { unsigned int page_len; + if (head->rc_curpage >= rqstp->rq_maxpages) + return -EINVAL; + page_len = min_t(unsigned int, remaining, PAGE_SIZE - head->rc_pageoff); @@ -848,7 +851,7 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, head->rc_page_count++; dst = page_address(rqstp->rq_pages[head->rc_curpage]); - memcpy(dst + head->rc_curpage, src + offset, page_len); + memcpy((unsigned char *)dst + head->rc_pageoff, src + offset, page_len); head->rc_readbytes += page_len; head->rc_pageoff += page_len; @@ -860,7 +863,7 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, offset += page_len; } - return -EINVAL; + return 0; } /** diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 3d7f1413df023..12857381e8610 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -462,7 +462,10 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) newxprt->sc_max_bc_requests = 2; } - /* Arbitrary estimate of the needed number of rdma_rw contexts. + /* Estimate the needed number of rdma_rw contexts. The maximum + * Read and Write chunks have one segment each. Each request + * can involve one Read chunk and either a Write chunk or Reply + * chunk; thus a factor of three. */ maxpayload = min(xprt->xpt_server->sv_max_payload, RPCSVC_MAXPAYLOAD_RDMA); @@ -470,7 +473,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) rdma_rw_mr_factor(dev, newxprt->sc_port_num, maxpayload >> PAGE_SHIFT); - newxprt->sc_sq_depth = rq_depth + ctxts; + newxprt->sc_sq_depth = rq_depth + + rdma_rw_max_send_wr(dev, newxprt->sc_port_num, ctxts, 0); if (newxprt->sc_sq_depth > dev->attrs.max_qp_wr) newxprt->sc_sq_depth = dev->attrs.max_qp_wr; atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 63262ef0c2e3a..8abbd9c4045a4 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -1362,7 +1362,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) needed += RPCRDMA_MAX_RECV_BATCH; if (atomic_inc_return(&ep->re_receiving) > 1) - goto out; + goto out_dec; /* fast path: all needed reps can be found on the free list */ wr = NULL; @@ -1385,7 +1385,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) ++count; } if (!wr) - goto out; + goto out_dec; rc = ib_post_recv(ep->re_id->qp, wr, (const struct ib_recv_wr **)&bad_wr); @@ -1400,9 +1400,10 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) --count; } } + +out_dec: if (atomic_dec_return(&ep->re_receiving) > 0) complete(&ep->re_done); - out: trace_xprtrdma_post_recvs(r_xprt, count); ep->re_receive_count += count; diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index 751904f10aab0..a3f9ca28c3d53 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -460,7 +460,7 @@ static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim) rcu_read_lock(); tmp = rcu_dereference(aead); if (tmp) - atomic_add_unless(&rcu_dereference(aead)->users, -1, lim); + atomic_add_unless(&tmp->users, -1, lim); rcu_read_unlock(); } @@ -1219,7 +1219,7 @@ void tipc_crypto_key_flush(struct tipc_crypto *c) rx = c; tx = tipc_net(rx->net)->crypto_tx; if (cancel_delayed_work(&rx->work)) { - kfree(rx->skey); + kfree_sensitive(rx->skey); rx->skey = NULL; atomic_xchg(&rx->key_distr, 0); tipc_node_put(rx->node); @@ -2394,7 +2394,7 @@ static void tipc_crypto_work_rx(struct work_struct *work) break; default: synchronize_rcu(); - kfree(rx->skey); + kfree_sensitive(rx->skey); rx->skey = NULL; break; } diff --git a/net/tipc/group.c b/net/tipc/group.c index 3e137d8c9d2ff..215f2a7d84588 100644 --- a/net/tipc/group.c +++ b/net/tipc/group.c @@ -746,6 +746,7 @@ void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup, u32 port = msg_origport(hdr); struct tipc_member *m, *pm; u16 remitted, in_flight; + u16 acked; if (!grp) return; @@ -798,7 +799,10 @@ void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup, case GRP_ACK_MSG: if (!m) return; - m->bc_acked = msg_grp_bc_acked(hdr); + acked = msg_grp_bc_acked(hdr); + if (less_eq(acked, m->bc_acked)) + return; + m->bc_acked = acked; if (--grp->bc_ackers) return; list_del_init(&m->small_win); diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c index e74940eab3a47..7f42fb6a8481f 100644 --- a/net/tipc/name_table.c +++ b/net/tipc/name_table.c @@ -348,7 +348,8 @@ static bool tipc_service_insert_publ(struct net *net, /* Return if the publication already exists */ list_for_each_entry(_p, &sr->all_publ, all_publ) { - if (_p->key == key && (!_p->sk.node || _p->sk.node == node)) { + if (_p->key == key && _p->sk.ref == p->sk.ref && + (!_p->sk.node || _p->sk.node == node)) { pr_debug("Failed to bind duplicate %u,%u,%u/%u:%u/%u\n", p->sr.type, p->sr.lower, p->sr.upper, node, p->sk.ref, key); @@ -388,7 +389,8 @@ static struct publication *tipc_service_remove_publ(struct service_range *r, u32 node = sk->node; list_for_each_entry(p, &r->all_publ, all_publ) { - if (p->key != key || (node && node != p->sk.node)) + if (p->key != key || p->sk.ref != sk->ref || + (node && node != p->sk.node)) continue; list_del(&p->all_publ); list_del(&p->local_publ); diff --git a/net/tipc/socket.c b/net/tipc/socket.c index 1574a83384f88..3d5de693a2227 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -2233,6 +2233,8 @@ static bool tipc_sk_filter_connect(struct tipc_sock *tsk, struct sk_buff *skb, if (skb_queue_empty(&sk->sk_write_queue)) break; get_random_bytes(&delay, 2); + if (tsk->conn_timeout < 4) + tsk->conn_timeout = 4; delay %= (tsk->conn_timeout / 4); delay = msecs_to_jiffies(delay + 100); sk_reset_timer(sk, &sk->sk_timer, jiffies + delay); diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index d171353699800..16aaf41a8cc09 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -246,6 +246,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) crypto_wait_req(-EINPROGRESS, &ctx->async_wait); atomic_inc(&ctx->decrypt_pending); + __skb_queue_purge(&ctx->async_hold); return ctx->async_wait.err; } @@ -583,6 +584,16 @@ static int tls_do_encryption(struct sock *sk, if (rc == -EBUSY) { rc = tls_encrypt_async_wait(ctx); rc = rc ?: -EINPROGRESS; + /* + * The async callback tls_encrypt_done() has already + * decremented encrypt_pending and restored the sge on + * both success and error. Skip the synchronous cleanup + * below on error, just remove the record and return. + */ + if (rc != -EINPROGRESS) { + list_del(&rec->list); + return rc; + } } if (!rc || rc != -EINPROGRESS) { atomic_dec(&ctx->encrypt_pending); @@ -2225,7 +2236,6 @@ int tls_sw_recvmsg(struct sock *sk, /* Wait for all previously submitted records to be decrypted */ ret = tls_decrypt_async_wait(ctx); - __skb_queue_purge(&ctx->async_hold); if (ret) { if (err >= 0 || err == -EINPROGRESS) @@ -2533,7 +2543,7 @@ void tls_sw_cancel_work_tx(struct tls_context *tls_ctx) set_bit(BIT_TX_CLOSING, &ctx->tx_bitmask); set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask); - cancel_delayed_work_sync(&ctx->tx_work.work); + disable_delayed_work_sync(&ctx->tx_work.work); } void tls_sw_release_resources_tx(struct sock *sk) diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 833c3616d2a2b..6b251c76f6bec 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1671,10 +1671,9 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); - /* First of all allocate resources. - * If we will make it after state is locked, - * we will have to recheck all again in any case. - */ + err = prepare_peercred(&peercred); + if (err) + goto out; /* create new sock for complete connection */ newsk = unix_create1(net, NULL, 0, sock->type); @@ -1683,10 +1682,6 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, goto out; } - err = prepare_peercred(&peercred); - if (err) - goto out; - /* Allocate skb for sending to listening sock */ skb = sock_wmalloc(newsk, 1, 0, GFP_KERNEL); if (!skb) { @@ -1811,7 +1806,7 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, __skb_queue_tail(&other->sk_receive_queue, skb); spin_unlock(&other->sk_receive_queue.lock); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sock_put(other); return 0; @@ -1984,6 +1979,8 @@ static void unix_detach_fds(struct scm_cookie *scm, struct sk_buff *skb) static void unix_peek_fds(struct scm_cookie *scm, struct sk_buff *skb) { scm->fp = scm_fp_dup(UNIXCB(skb).fp); + + unix_peek_fpl(scm->fp); } static void unix_destruct_scm(struct sk_buff *skb) @@ -2306,7 +2303,7 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, scm_stat_add(other, skb); skb_queue_tail(&other->sk_receive_queue, skb); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sock_put(other); scm_destroy(&scm); return len; @@ -2379,7 +2376,7 @@ static int queue_oob(struct sock *sk, struct msghdr *msg, struct sock *other, sk_send_sigurg(other); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); return 0; out_unlock: @@ -2507,7 +2504,7 @@ static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg, spin_unlock(&other->sk_receive_queue.lock); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sent += size; } @@ -3111,12 +3108,15 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state, mutex_unlock(&u->iolock); if (msg) { + bool do_cmsg = READ_ONCE(u->recvmsg_inq); + scm_recv_unix(sock, msg, &scm, flags); - if (READ_ONCE(u->recvmsg_inq) || msg->msg_get_inq) { + if ((do_cmsg | msg->msg_get_inq) && (copied ?: err) >= 0) { msg->msg_inq = READ_ONCE(u->inq_len); - put_cmsg(msg, SOL_SOCKET, SCM_INQ, - sizeof(msg->msg_inq), &msg->msg_inq); + if (do_cmsg) + put_cmsg(msg, SOL_SOCKET, SCM_INQ, + sizeof(msg->msg_inq), &msg->msg_inq); } } else { scm_destroy(&scm); diff --git a/net/unix/af_unix.h b/net/unix/af_unix.h index 59db179df9bb5..6b96c1007aecd 100644 --- a/net/unix/af_unix.h +++ b/net/unix/af_unix.h @@ -30,6 +30,7 @@ void unix_del_edges(struct scm_fp_list *fpl); void unix_update_edges(struct unix_sock *receiver); int unix_prepare_fpl(struct scm_fp_list *fpl); void unix_destroy_fpl(struct scm_fp_list *fpl); +void unix_peek_fpl(struct scm_fp_list *fpl); void unix_gc(void); void wait_for_unix_gc(struct scm_fp_list *fpl); diff --git a/net/unix/diag.c b/net/unix/diag.c index ca34730261510..c9c1e51c44196 100644 --- a/net/unix/diag.c +++ b/net/unix/diag.c @@ -28,18 +28,23 @@ static int sk_diag_dump_name(struct sock *sk, struct sk_buff *nlskb) static int sk_diag_dump_vfs(struct sock *sk, struct sk_buff *nlskb) { - struct dentry *dentry = unix_sk(sk)->path.dentry; + struct unix_diag_vfs uv; + struct dentry *dentry; + bool have_vfs = false; + unix_state_lock(sk); + dentry = unix_sk(sk)->path.dentry; if (dentry) { - struct unix_diag_vfs uv = { - .udiag_vfs_ino = d_backing_inode(dentry)->i_ino, - .udiag_vfs_dev = dentry->d_sb->s_dev, - }; - - return nla_put(nlskb, UNIX_DIAG_VFS, sizeof(uv), &uv); + uv.udiag_vfs_ino = d_backing_inode(dentry)->i_ino; + uv.udiag_vfs_dev = dentry->d_sb->s_dev; + have_vfs = true; } + unix_state_unlock(sk); - return 0; + if (!have_vfs) + return 0; + + return nla_put(nlskb, UNIX_DIAG_VFS, sizeof(uv), &uv); } static int sk_diag_dump_peer(struct sock *sk, struct sk_buff *nlskb) diff --git a/net/unix/garbage.c b/net/unix/garbage.c index 65396a4e1b07e..529b21d043d92 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -121,8 +121,13 @@ static struct unix_vertex *unix_edge_successor(struct unix_edge *edge) return edge->successor->vertex; } -static bool unix_graph_maybe_cyclic; -static bool unix_graph_grouped; +enum { + UNIX_GRAPH_NOT_CYCLIC, + UNIX_GRAPH_MAYBE_CYCLIC, + UNIX_GRAPH_CYCLIC, +}; + +static unsigned char unix_graph_state; static void unix_update_graph(struct unix_vertex *vertex) { @@ -132,8 +137,7 @@ static void unix_update_graph(struct unix_vertex *vertex) if (!vertex) return; - unix_graph_maybe_cyclic = true; - unix_graph_grouped = false; + unix_graph_state = UNIX_GRAPH_MAYBE_CYCLIC; } static LIST_HEAD(unix_unvisited_vertices); @@ -315,6 +319,25 @@ void unix_destroy_fpl(struct scm_fp_list *fpl) unix_free_vertices(fpl); } +static bool gc_in_progress; +static seqcount_t unix_peek_seq = SEQCNT_ZERO(unix_peek_seq); + +void unix_peek_fpl(struct scm_fp_list *fpl) +{ + static DEFINE_SPINLOCK(unix_peek_lock); + + if (!fpl || !fpl->count_unix) + return; + + if (!READ_ONCE(gc_in_progress)) + return; + + /* Invalidate the final refcnt check in unix_vertex_dead(). */ + spin_lock(&unix_peek_lock); + raw_write_seqcount_barrier(&unix_peek_seq); + spin_unlock(&unix_peek_lock); +} + static bool unix_vertex_dead(struct unix_vertex *vertex) { struct unix_edge *edge; @@ -348,6 +371,36 @@ static bool unix_vertex_dead(struct unix_vertex *vertex) return true; } +static LIST_HEAD(unix_visited_vertices); +static unsigned long unix_vertex_grouped_index = UNIX_VERTEX_INDEX_MARK2; + +static bool unix_scc_dead(struct list_head *scc, bool fast) +{ + struct unix_vertex *vertex; + bool scc_dead = true; + unsigned int seq; + + seq = read_seqcount_begin(&unix_peek_seq); + + list_for_each_entry_reverse(vertex, scc, scc_entry) { + /* Don't restart DFS from this vertex. */ + list_move_tail(&vertex->entry, &unix_visited_vertices); + + /* Mark vertex as off-stack for __unix_walk_scc(). */ + if (!fast) + vertex->index = unix_vertex_grouped_index; + + if (scc_dead) + scc_dead = unix_vertex_dead(vertex); + } + + /* If MSG_PEEK intervened, defer this SCC to the next round. */ + if (read_seqcount_retry(&unix_peek_seq, seq)) + return false; + + return scc_dead; +} + static void unix_collect_skb(struct list_head *scc, struct sk_buff_head *hitlist) { struct unix_vertex *vertex; @@ -401,12 +454,11 @@ static bool unix_scc_cyclic(struct list_head *scc) return false; } -static LIST_HEAD(unix_visited_vertices); -static unsigned long unix_vertex_grouped_index = UNIX_VERTEX_INDEX_MARK2; - -static void __unix_walk_scc(struct unix_vertex *vertex, unsigned long *last_index, - struct sk_buff_head *hitlist) +static unsigned long __unix_walk_scc(struct unix_vertex *vertex, + unsigned long *last_index, + struct sk_buff_head *hitlist) { + unsigned long cyclic_sccs = 0; LIST_HEAD(vertex_stack); struct unix_edge *edge; LIST_HEAD(edge_stack); @@ -469,9 +521,7 @@ static void __unix_walk_scc(struct unix_vertex *vertex, unsigned long *last_inde } if (vertex->index == vertex->scc_index) { - struct unix_vertex *v; struct list_head scc; - bool scc_dead = true; /* SCC finalised. * @@ -480,25 +530,14 @@ static void __unix_walk_scc(struct unix_vertex *vertex, unsigned long *last_inde */ __list_cut_position(&scc, &vertex_stack, &vertex->scc_entry); - list_for_each_entry_reverse(v, &scc, scc_entry) { - /* Don't restart DFS from this vertex in unix_walk_scc(). */ - list_move_tail(&v->entry, &unix_visited_vertices); - - /* Mark vertex as off-stack. */ - v->index = unix_vertex_grouped_index; - - if (scc_dead) - scc_dead = unix_vertex_dead(v); - } - - if (scc_dead) { + if (unix_scc_dead(&scc, false)) { unix_collect_skb(&scc, hitlist); } else { if (unix_vertex_max_scc_index < vertex->scc_index) unix_vertex_max_scc_index = vertex->scc_index; - if (!unix_graph_maybe_cyclic) - unix_graph_maybe_cyclic = unix_scc_cyclic(&scc); + if (unix_scc_cyclic(&scc)) + cyclic_sccs++; } list_del(&scc); @@ -507,13 +546,17 @@ static void __unix_walk_scc(struct unix_vertex *vertex, unsigned long *last_inde /* Need backtracking ? */ if (!list_empty(&edge_stack)) goto prev_vertex; + + return cyclic_sccs; } +static unsigned long unix_graph_cyclic_sccs; + static void unix_walk_scc(struct sk_buff_head *hitlist) { unsigned long last_index = UNIX_VERTEX_INDEX_START; + unsigned long cyclic_sccs = 0; - unix_graph_maybe_cyclic = false; unix_vertex_max_scc_index = UNIX_VERTEX_INDEX_START; /* Visit every vertex exactly once. @@ -523,46 +566,40 @@ static void unix_walk_scc(struct sk_buff_head *hitlist) struct unix_vertex *vertex; vertex = list_first_entry(&unix_unvisited_vertices, typeof(*vertex), entry); - __unix_walk_scc(vertex, &last_index, hitlist); + cyclic_sccs += __unix_walk_scc(vertex, &last_index, hitlist); } list_replace_init(&unix_visited_vertices, &unix_unvisited_vertices); swap(unix_vertex_unvisited_index, unix_vertex_grouped_index); - unix_graph_grouped = true; + unix_graph_cyclic_sccs = cyclic_sccs; + unix_graph_state = cyclic_sccs ? UNIX_GRAPH_CYCLIC : UNIX_GRAPH_NOT_CYCLIC; } static void unix_walk_scc_fast(struct sk_buff_head *hitlist) { - unix_graph_maybe_cyclic = false; + unsigned long cyclic_sccs = unix_graph_cyclic_sccs; while (!list_empty(&unix_unvisited_vertices)) { struct unix_vertex *vertex; struct list_head scc; - bool scc_dead = true; vertex = list_first_entry(&unix_unvisited_vertices, typeof(*vertex), entry); list_add(&scc, &vertex->scc_entry); - list_for_each_entry_reverse(vertex, &scc, scc_entry) { - list_move_tail(&vertex->entry, &unix_visited_vertices); - - if (scc_dead) - scc_dead = unix_vertex_dead(vertex); - } - - if (scc_dead) + if (unix_scc_dead(&scc, true)) { + cyclic_sccs--; unix_collect_skb(&scc, hitlist); - else if (!unix_graph_maybe_cyclic) - unix_graph_maybe_cyclic = unix_scc_cyclic(&scc); + } list_del(&scc); } list_replace_init(&unix_visited_vertices, &unix_unvisited_vertices); -} -static bool gc_in_progress; + unix_graph_cyclic_sccs = cyclic_sccs; + unix_graph_state = cyclic_sccs ? UNIX_GRAPH_CYCLIC : UNIX_GRAPH_NOT_CYCLIC; +} static void __unix_gc(struct work_struct *work) { @@ -571,14 +608,14 @@ static void __unix_gc(struct work_struct *work) spin_lock(&unix_gc_lock); - if (!unix_graph_maybe_cyclic) { + if (unix_graph_state == UNIX_GRAPH_NOT_CYCLIC) { spin_unlock(&unix_gc_lock); goto skip_gc; } __skb_queue_head_init(&hitlist); - if (unix_graph_grouped) + if (unix_graph_state == UNIX_GRAPH_CYCLIC) unix_walk_scc_fast(&hitlist); else unix_walk_scc(&hitlist); diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index a9ca9c3b87b31..cbd649bf01459 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1787,6 +1787,10 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, } else { newsock->state = SS_CONNECTED; sock_graft(connected, newsock); + + set_bit(SOCK_CUSTOM_SOCKOPT, + &connected->sk_socket->flags); + if (vsock_msgzerocopy_allow(vconnected->transport)) set_bit(SOCK_SUPPORT_ZC, &connected->sk_socket->flags); diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index dcc8a1d5851e6..d3e26025ef589 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -28,6 +28,7 @@ static void virtio_transport_cancel_close_work(struct vsock_sock *vsk, bool cancel_timeout); +static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs); static const struct virtio_transport * virtio_transport_get_ops(struct vsock_sock *vsk) @@ -499,9 +500,7 @@ u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit) return 0; spin_lock_bh(&vvs->tx_lock); - ret = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); - if (ret > credit) - ret = credit; + ret = min_t(u32, credit, virtio_transport_has_space(vvs)); vvs->tx_cnt += ret; vvs->bytes_unsent += ret; spin_unlock_bh(&vvs->tx_lock); @@ -822,6 +821,15 @@ virtio_transport_seqpacket_dequeue(struct vsock_sock *vsk, } EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_dequeue); +static u32 virtio_transport_tx_buf_size(struct virtio_vsock_sock *vvs) +{ + /* The peer advertises its receive buffer via peer_buf_alloc, but we + * cap it to our local buf_alloc so a remote peer cannot force us to + * queue more data than our own buffer configuration allows. + */ + return min(vvs->peer_buf_alloc, vvs->buf_alloc); +} + int virtio_transport_seqpacket_enqueue(struct vsock_sock *vsk, struct msghdr *msg, @@ -831,7 +839,7 @@ virtio_transport_seqpacket_enqueue(struct vsock_sock *vsk, spin_lock_bh(&vvs->tx_lock); - if (len > vvs->peer_buf_alloc) { + if (len > virtio_transport_tx_buf_size(vvs)) { spin_unlock_bh(&vvs->tx_lock); return -EMSGSIZE; } @@ -877,12 +885,16 @@ u32 virtio_transport_seqpacket_has_data(struct vsock_sock *vsk) } EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_has_data); -static s64 virtio_transport_has_space(struct vsock_sock *vsk) +static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs) { - struct virtio_vsock_sock *vvs = vsk->trans; s64 bytes; - bytes = (s64)vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); + /* Use s64 arithmetic so if the peer shrinks peer_buf_alloc while + * we have bytes in flight (tx_cnt - peer_fwd_cnt), the subtraction + * does not underflow. + */ + bytes = (s64)virtio_transport_tx_buf_size(vvs) - + (vvs->tx_cnt - vvs->peer_fwd_cnt); if (bytes < 0) bytes = 0; @@ -895,7 +907,7 @@ s64 virtio_transport_stream_has_space(struct vsock_sock *vsk) s64 bytes; spin_lock_bh(&vvs->tx_lock); - bytes = virtio_transport_has_space(vsk); + bytes = virtio_transport_has_space(vvs); spin_unlock_bh(&vvs->tx_lock); return bytes; @@ -1359,9 +1371,11 @@ virtio_transport_recv_enqueue(struct vsock_sock *vsk, /* Try to copy small packets into the buffer of last packet queued, * to avoid wasting memory queueing the entire buffer with a small - * payload. + * payload. Skip non-linear (e.g. zerocopy) skbs; these carry payload + * in skb_shinfo. */ - if (len <= GOOD_COPY_LEN && !skb_queue_empty(&vvs->rx_queue)) { + if (len <= GOOD_COPY_LEN && !skb_queue_empty(&vvs->rx_queue) && + !skb_is_nonlinear(skb)) { struct virtio_vsock_hdr *last_hdr; struct sk_buff *last_skb; @@ -1490,7 +1504,7 @@ static bool virtio_transport_space_update(struct sock *sk, spin_lock_bh(&vvs->tx_lock); vvs->peer_buf_alloc = le32_to_cpu(hdr->buf_alloc); vvs->peer_fwd_cnt = le32_to_cpu(hdr->fwd_cnt); - space_available = virtio_transport_has_space(vsk); + space_available = virtio_transport_has_space(vvs); spin_unlock_bh(&vvs->tx_lock); return space_available; } diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index 7eccd6708d664..aca3132689cf1 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -161,7 +161,7 @@ vmci_transport_packet_init(struct vmci_transport_packet *pkt, case VMCI_TRANSPORT_PACKET_TYPE_WAITING_READ: case VMCI_TRANSPORT_PACKET_TYPE_WAITING_WRITE: - memcpy(&pkt->u.wait, wait, sizeof(pkt->u.wait)); + pkt->u.wait = *wait; break; case VMCI_TRANSPORT_PACKET_TYPE_REQUEST2: diff --git a/net/wireless/core.c b/net/wireless/core.c index 54a34d8d356e0..6b204ca91a140 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -658,12 +658,8 @@ int wiphy_verify_iface_combinations(struct wiphy *wiphy, c->limits[j].max > 1)) return -EINVAL; - /* Only a single NAN can be allowed, avoid this - * check for multi-radio global combination, since it - * hold the capabilities of all radio combinations. - */ - if (!combined_radio && - WARN_ON(types & BIT(NL80211_IFTYPE_NAN) && + /* Only a single NAN can be allowed */ + if (WARN_ON(types & BIT(NL80211_IFTYPE_NAN) && c->limits[j].max > 1)) return -EINVAL; @@ -1199,6 +1195,7 @@ void wiphy_unregister(struct wiphy *wiphy) /* this has nothing to do now but make sure it's gone */ cancel_work_sync(&rdev->wiphy_work); + cancel_work_sync(&rdev->rfkill_block); cancel_work_sync(&rdev->conn_work); flush_work(&rdev->event_work); cancel_delayed_work_sync(&rdev->dfs_update_channels_wk); @@ -1365,6 +1362,7 @@ void cfg80211_leave(struct cfg80211_registered_device *rdev, cfg80211_pmsr_wdev_down(wdev); + cfg80211_stop_radar_detection(wdev); cfg80211_stop_background_radar_detection(wdev); switch (wdev->iftype) { @@ -1399,8 +1397,10 @@ void cfg80211_leave(struct cfg80211_registered_device *rdev, cfg80211_leave_ocb(rdev, dev); break; case NL80211_IFTYPE_P2P_DEVICE: + cfg80211_stop_p2p_device(rdev, wdev); + break; case NL80211_IFTYPE_NAN: - /* cannot happen, has no netdev */ + cfg80211_stop_nan(rdev, wdev); break; case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_MONITOR: diff --git a/net/wireless/core.h b/net/wireless/core.h index b6bd7f4d6385a..d5d78752227af 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -489,6 +489,7 @@ cfg80211_start_background_radar_detection(struct cfg80211_registered_device *rde struct wireless_dev *wdev, struct cfg80211_chan_def *chandef); +void cfg80211_stop_radar_detection(struct wireless_dev *wdev); void cfg80211_stop_background_radar_detection(struct wireless_dev *wdev); void cfg80211_background_cac_done_wk(struct work_struct *work); diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index 46394eb2086f6..3fc175f9f8686 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -1295,6 +1295,25 @@ cfg80211_start_background_radar_detection(struct cfg80211_registered_device *rde return 0; } +void cfg80211_stop_radar_detection(struct wireless_dev *wdev) +{ + struct wiphy *wiphy = wdev->wiphy; + struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); + int link_id; + + for_each_valid_link(wdev, link_id) { + struct cfg80211_chan_def chandef; + + if (!wdev->links[link_id].cac_started) + continue; + + chandef = *wdev_chandef(wdev, link_id); + rdev_end_cac(rdev, wdev->netdev, link_id); + nl80211_radar_notify(rdev, &chandef, NL80211_RADAR_CAC_ABORTED, + wdev->netdev, GFP_KERNEL); + } +} + void cfg80211_stop_background_radar_detection(struct wireless_dev *wdev) { struct wiphy *wiphy = wdev->wiphy; diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c index a117f5093ca29..13801cf35e9fc 100644 --- a/net/wireless/pmsr.c +++ b/net/wireless/pmsr.c @@ -647,6 +647,7 @@ void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev) } spin_unlock_bh(&wdev->pmsr_lock); + cancel_work_sync(&wdev->pmsr_free_wk); if (found) cfg80211_pmsr_process_abort(wdev); diff --git a/net/wireless/radiotap.c b/net/wireless/radiotap.c index 326faea38ca38..c85eaa583a466 100644 --- a/net/wireless/radiotap.c +++ b/net/wireless/radiotap.c @@ -239,14 +239,14 @@ int ieee80211_radiotap_iterator_next( default: if (!iterator->current_namespace || iterator->_arg_index >= iterator->current_namespace->n_bits) { - if (iterator->current_namespace == &radiotap_ns) - return -ENOENT; align = 0; } else { align = iterator->current_namespace->align_size[iterator->_arg_index].align; size = iterator->current_namespace->align_size[iterator->_arg_index].size; } if (!align) { + if (iterator->current_namespace == &radiotap_ns) + return -ENOENT; /* skip all subsequent data */ iterator->_arg = iterator->_next_ns_data; /* give up on this namespace */ diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 90a9187a6b135..9a0c02c23dc56 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1959,7 +1959,7 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, ether_addr_copy(known->parent_bssid, new->parent_bssid); known->pub.max_bssid_indicator = new->pub.max_bssid_indicator; known->pub.bssid_index = new->pub.bssid_index; - known->pub.use_for &= new->pub.use_for; + known->pub.use_for = new->pub.use_for; known->pub.cannot_use_reasons = new->pub.cannot_use_reasons; known->bss_source = new->bss_source; diff --git a/net/wireless/sme.c b/net/wireless/sme.c index 3a028ff287fbb..4e629ca305bcc 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -910,7 +910,7 @@ void __cfg80211_connect_result(struct net_device *dev, ssid_len = min(ssid->datalen, IEEE80211_MAX_SSID_LEN); memcpy(wdev->u.client.ssid, ssid->data, ssid_len); - wdev->u.client.ssid_len = ssid->datalen; + wdev->u.client.ssid_len = ssid_len; break; } rcu_read_unlock(); diff --git a/net/wireless/util.c b/net/wireless/util.c index 56724b33af045..81d6d27d273cc 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -1203,28 +1203,7 @@ int cfg80211_change_iface(struct cfg80211_registered_device *rdev, dev->ieee80211_ptr->use_4addr = false; rdev_set_qos_map(rdev, dev, NULL); - switch (otype) { - case NL80211_IFTYPE_AP: - case NL80211_IFTYPE_P2P_GO: - cfg80211_stop_ap(rdev, dev, -1, true); - break; - case NL80211_IFTYPE_ADHOC: - cfg80211_leave_ibss(rdev, dev, false); - break; - case NL80211_IFTYPE_STATION: - case NL80211_IFTYPE_P2P_CLIENT: - cfg80211_disconnect(rdev, dev, - WLAN_REASON_DEAUTH_LEAVING, true); - break; - case NL80211_IFTYPE_MESH_POINT: - /* mesh should be handled? */ - break; - case NL80211_IFTYPE_OCB: - cfg80211_leave_ocb(rdev, dev); - break; - default: - break; - } + cfg80211_leave(rdev, dev->ieee80211_ptr); cfg80211_process_rdev_events(rdev); cfg80211_mlme_purge_registrations(dev->ieee80211_ptr); @@ -1582,12 +1561,14 @@ static u32 cfg80211_calculate_bitrate_he(struct rate_info *rate) tmp = result; tmp *= SCALE; do_div(tmp, mcs_divisors[rate->mcs]); - result = tmp; /* and take NSS, DCM into account */ - result = (result * rate->nss) / 8; + tmp *= rate->nss; + do_div(tmp, 8); if (rate->he_dcm) - result /= 2; + do_div(tmp, 2); + + result = tmp; return result / 10000; } diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c index 1241fda78a68c..680500fa57cfd 100644 --- a/net/wireless/wext-compat.c +++ b/net/wireless/wext-compat.c @@ -684,7 +684,7 @@ static int cfg80211_wext_siwencodeext(struct net_device *dev, idx = erq->flags & IW_ENCODE_INDEX; if (cipher == WLAN_CIPHER_SUITE_AES_CMAC) { - if (idx < 4 || idx > 5) { + if (idx < 5 || idx > 6) { idx = wdev->wext.default_mgmt_key; if (idx < 0) return -EINVAL; diff --git a/net/wireless/wext-core.c b/net/wireless/wext-core.c index c32a7c6903d53..7b8e94214b072 100644 --- a/net/wireless/wext-core.c +++ b/net/wireless/wext-core.c @@ -1101,6 +1101,10 @@ static int compat_standard_call(struct net_device *dev, return ioctl_standard_call(dev, iwr, cmd, info, handler); iwp_compat = (struct compat_iw_point *) &iwr->u.data; + + /* struct iw_point has a 32bit hole on 64bit arches. */ + memset(&iwp, 0, sizeof(iwp)); + iwp.pointer = compat_ptr(iwp_compat->pointer); iwp.length = iwp_compat->length; iwp.flags = iwp_compat->flags; diff --git a/net/wireless/wext-priv.c b/net/wireless/wext-priv.c index 674d426a9d24f..37d1147019c2b 100644 --- a/net/wireless/wext-priv.c +++ b/net/wireless/wext-priv.c @@ -228,6 +228,10 @@ int compat_private_call(struct net_device *dev, struct iwreq *iwr, struct iw_point iwp; iwp_compat = (struct compat_iw_point *) &iwr->u.data; + + /* struct iw_point has a 32bit hole on 64bit arches. */ + memset(&iwp, 0, sizeof(iwp)); + iwp.pointer = compat_ptr(iwp_compat->pointer); iwp.length = iwp_compat->length; iwp.flags = iwp_compat->flags; diff --git a/net/x25/x25_in.c b/net/x25/x25_in.c index b981a4828d08c..e47ebd8acd21b 100644 --- a/net/x25/x25_in.c +++ b/net/x25/x25_in.c @@ -34,6 +34,10 @@ static int x25_queue_rx_frame(struct sock *sk, struct sk_buff *skb, int more) struct sk_buff *skbo, *skbn = skb; struct x25_sock *x25 = x25_sk(sk); + /* make sure we don't overflow */ + if (x25->fraglen + skb->len > USHRT_MAX) + return 1; + if (more) { x25->fraglen += skb->len; skb_queue_tail(&x25->fragment_queue, skb); @@ -44,10 +48,9 @@ static int x25_queue_rx_frame(struct sock *sk, struct sk_buff *skb, int more) if (x25->fraglen > 0) { /* End of fragment */ int len = x25->fraglen + skb->len; - if ((skbn = alloc_skb(len, GFP_ATOMIC)) == NULL){ - kfree_skb(skb); + skbn = alloc_skb(len, GFP_ATOMIC); + if (!skbn) return 1; - } skb_queue_tail(&x25->fragment_queue, skb); diff --git a/net/x25/x25_subr.c b/net/x25/x25_subr.c index 0285aaa1e93c1..159708d9ad20c 100644 --- a/net/x25/x25_subr.c +++ b/net/x25/x25_subr.c @@ -40,6 +40,7 @@ void x25_clear_queues(struct sock *sk) skb_queue_purge(&x25->interrupt_in_queue); skb_queue_purge(&x25->interrupt_out_queue); skb_queue_purge(&x25->fragment_queue); + x25->fraglen = 0; } diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 9f76ca591d54f..9ec7bd948acc7 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -202,7 +202,8 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) if (!unaligned_chunks && chunks_rem) return -EINVAL; - if (headroom >= chunk_size - XDP_PACKET_HEADROOM) + if (headroom > chunk_size - XDP_PACKET_HEADROOM - + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) - 128) return -EINVAL; if (mr->flags & XDP_UMEM_TX_METADATA_LEN) { diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 69bbcca8ac753..259ad9a3abcc4 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -167,26 +167,32 @@ static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) struct xdp_buff_xsk *pos, *tmp; struct list_head *xskb_list; u32 contd = 0; + u32 num_desc; int err; - if (frags) - contd = XDP_PKT_CONTD; + if (likely(!frags)) { + err = __xsk_rcv_zc(xs, xskb, len, contd); + if (err) + goto err; + return 0; + } - err = __xsk_rcv_zc(xs, xskb, len, contd); - if (err) + contd = XDP_PKT_CONTD; + num_desc = xdp_get_shared_info_from_buff(xdp)->nr_frags + 1; + if (xskq_prod_nb_free(xs->rx, num_desc) < num_desc) { + xs->rx_queue_full++; + err = -ENOBUFS; goto err; - if (likely(!frags)) - return 0; + } + __xsk_rcv_zc(xs, xskb, len, contd); xskb_list = &xskb->pool->xskb_list; list_for_each_entry_safe(pos, tmp, xskb_list, list_node) { if (list_is_singular(xskb_list)) contd = 0; len = pos->xdp.data_end - pos->xdp.data; - err = __xsk_rcv_zc(xs, pos, len, contd); - if (err) - goto err; - list_del(&pos->list_node); + __xsk_rcv_zc(xs, pos, len, contd); + list_del_init(&pos->list_node); } return 0; @@ -233,7 +239,7 @@ static u32 xsk_copy_xdp(void *to, void **from, u32 to_len, static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) { - u32 frame_size = xsk_pool_get_rx_frame_size(xs->pool); + u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool); void *copy_from = xsk_copy_xdp_start(xdp), *copy_to; u32 from_len, meta_len, rem, num_desc; struct xdp_buff_xsk *xskb; @@ -332,7 +338,7 @@ static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) return -EINVAL; - if (len > xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) { + if (len > __xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) { xs->rx_dropped++; return -ENOSPC; } diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index aa9788f20d0db..a129ce6f1c25f 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -10,6 +10,8 @@ #include "xdp_umem.h" #include "xsk.h" +#define ETH_PAD_LEN (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN) + void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs) { unsigned long flags; @@ -165,8 +167,12 @@ static void xp_disable_drv_zc(struct xsk_buff_pool *pool) int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev, u16 queue_id, u16 flags) { + u32 needed = netdev->mtu + ETH_PAD_LEN; + u32 segs = netdev->xdp_zc_max_segs; + bool mbuf = flags & XDP_USE_SG; bool force_zc, force_copy; struct netdev_bpf bpf; + u32 frame_size; int err = 0; ASSERT_RTNL(); @@ -186,7 +192,7 @@ int xp_assign_dev(struct xsk_buff_pool *pool, if (err) return err; - if (flags & XDP_USE_SG) + if (mbuf) pool->umem->flags |= XDP_UMEM_SG_FLAG; if (flags & XDP_USE_NEED_WAKEUP) @@ -208,8 +214,24 @@ int xp_assign_dev(struct xsk_buff_pool *pool, goto err_unreg_pool; } - if (netdev->xdp_zc_max_segs == 1 && (flags & XDP_USE_SG)) { - err = -EOPNOTSUPP; + if (mbuf) { + if (segs == 1) { + err = -EOPNOTSUPP; + goto err_unreg_pool; + } + } else { + segs = 1; + } + + /* open-code xsk_pool_get_rx_frame_size() as pool->dev is not + * set yet at this point; we are before getting down to driver + */ + frame_size = __xsk_pool_get_rx_frame_size(pool) - + xsk_pool_get_tailroom(mbuf); + frame_size = ALIGN_DOWN(frame_size, 128); + + if (needed > frame_size * segs) { + err = -EINVAL; goto err_unreg_pool; } @@ -259,6 +281,10 @@ int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, return -EINVAL; flags = umem->zc ? XDP_ZEROCOPY : XDP_COPY; + + if (umem->flags & XDP_UMEM_SG_FLAG) + flags |= XDP_USE_SG; + if (umem_xs->pool->uses_need_wakeup) flags |= XDP_USE_NEED_WAKEUP; diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index bf744ac9d5a73..8709df716e98e 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -536,7 +536,7 @@ static void espintcp_close(struct sock *sk, long timeout) sk->sk_prot = &tcp_prot; barrier(); - cancel_work_sync(&ctx->work); + disable_work_sync(&ctx->work); strp_done(&ctx->strp); skb_queue_purge(&ctx->out_queue); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 52ae0e034d29e..550457e4c4f01 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -544,6 +544,14 @@ static int xfrm_dev_down(struct net_device *dev) return NOTIFY_DONE; } +static int xfrm_dev_unregister(struct net_device *dev) +{ + xfrm_dev_state_flush(dev_net(dev), dev, true); + xfrm_dev_policy_flush(dev_net(dev), dev, true); + + return NOTIFY_DONE; +} + static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); @@ -556,8 +564,10 @@ static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void return xfrm_api_check(dev); case NETDEV_DOWN: - case NETDEV_UNREGISTER: return xfrm_dev_down(dev); + + case NETDEV_UNREGISTER: + return xfrm_dev_unregister(dev); } return NOTIFY_DONE; } diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index c9ddef869aa55..3d80c5210e040 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c @@ -503,7 +503,6 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) /* An encap_type of -1 indicates async resumption. */ if (encap_type == -1) { async = 1; - dev_put(skb->dev); seq = XFRM_SKB_CB(skb)->seq.input.low; goto resume; } @@ -656,8 +655,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) dev_hold(skb->dev); nexthdr = x->type->input(x, skb); - if (nexthdr == -EINPROGRESS) + if (nexthdr == -EINPROGRESS) { + if (async) + dev_put(skb->dev); return 0; + } dev_put(skb->dev); } @@ -692,9 +694,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; err = xfrm_inner_mode_input(x, skb); - if (err == -EINPROGRESS) + if (err == -EINPROGRESS) { + if (async) + dev_put(skb->dev); return 0; - else if (err) { + } else if (err) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); goto drop; } @@ -731,6 +735,8 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) sp->olen = 0; if (skb_valid_dst(skb)) skb_dst_drop(skb); + if (async) + dev_put(skb->dev); gro_cells_receive(&gro_cells, skb); return 0; } else { @@ -750,6 +756,8 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) sp->olen = 0; if (skb_valid_dst(skb)) skb_dst_drop(skb); + if (async) + dev_put(skb->dev); gro_cells_receive(&gro_cells, skb); return err; } @@ -760,6 +768,8 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) drop_unlock: spin_unlock(&x->lock); drop: + if (async) + dev_put(skb->dev); xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); kfree_skb(skb); return 0; diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index 3b6d7284fc70a..7cd97c1dcd117 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -901,6 +901,12 @@ static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq, iptfs_skb_can_add_frags(newskb, fragwalk, data, copylen)) { iptfs_skb_add_frags(newskb, fragwalk, data, copylen); } else { + if (skb_linearize(newskb)) { + XFRM_INC_STATS(xs_net(xtfs->x), + LINUX_MIB_XFRMINBUFFERERROR); + goto abandon; + } + /* copy fragment data into newskb */ if (skb_copy_seq_read(st, data, skb_put(newskb, copylen), copylen)) { @@ -991,6 +997,11 @@ static bool __input_process_payload(struct xfrm_state *x, u32 data, iplen = be16_to_cpu(iph->tot_len); iphlen = iph->ihl << 2; + if (iplen < iphlen || iphlen < sizeof(*iph)) { + XFRM_INC_STATS(net, + LINUX_MIB_XFRMINHDRERROR); + goto done; + } protocol = cpu_to_be16(ETH_P_IP); XFRM_MODE_SKB_CB(skbseq->root_skb)->tos = iph->tos; } else if (iph->version == 0x6) { @@ -2653,9 +2664,6 @@ static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig) if (!xtfs) return -ENOMEM; - x->mode_data = xtfs; - xtfs->x = x; - xtfs->ra_newskb = NULL; if (xtfs->cfg.reorder_win_size) { xtfs->w_saved = kcalloc(xtfs->cfg.reorder_win_size, @@ -2666,6 +2674,9 @@ static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig) } } + x->mode_data = xtfs; + xtfs->x = x; + return 0; } diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index ebf95d48e86c1..1856beee0149b 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -261,7 +261,7 @@ int __net_init xfrm_nat_keepalive_net_init(struct net *net) int xfrm_nat_keepalive_net_fini(struct net *net) { - cancel_delayed_work_sync(&net->xfrm.nat_keepalive_work); + disable_delayed_work_sync(&net->xfrm.nat_keepalive_work); return 0; } diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 62486f8669752..29c94ee0ceb25 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -3801,8 +3801,8 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, struct xfrm_tmpl *tp[XFRM_MAX_DEPTH]; struct xfrm_tmpl *stp[XFRM_MAX_DEPTH]; struct xfrm_tmpl **tpp = tp; + int i, k = 0; int ti = 0; - int i, k; sp = skb_sec_path(skb); if (!sp) @@ -3828,6 +3828,12 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, tpp = stp; } + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET && sp == &dummy) + /* This policy template was already checked by HW + * and secpath was removed in __xfrm_policy_check2. + */ + goto out; + /* For each tunnel xfrm, find the first matching tmpl. * For each tmpl before that, find corresponding xfrm. * Order is _important_. Later we will implement @@ -3837,7 +3843,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, * verified to allow them to be skipped in future policy * checks (e.g. nested tunnels). */ - for (i = xfrm_nr-1, k = 0; i >= 0; i--) { + for (i = xfrm_nr - 1; i >= 0; i--) { k = xfrm_policy_ok(tpp[i], sp, k, family, if_id); if (k < 0) { if (k < -1) @@ -3853,6 +3859,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, goto reject; } +out: xfrm_pols_put(pols, npols); sp->verified_cnt = k; @@ -4275,12 +4282,16 @@ static void xfrm_policy_fini(struct net *net) unsigned int sz; int dir; + disable_work_sync(&net->xfrm.policy_hthresh.work); + flush_work(&net->xfrm.policy_hash_work); #ifdef CONFIG_XFRM_SUB_POLICY xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false); #endif xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false); + synchronize_rcu(); + WARN_ON(!list_empty(&net->xfrm.policy_all)); for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { @@ -4517,9 +4528,6 @@ static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector * pol = xfrm_policy_lookup_bytype(net, type, &fl, sel->family, dir, if_id); if (IS_ERR_OR_NULL(pol)) goto out_unlock; - - if (!xfrm_pol_hold_rcu(pol)) - pol = NULL; out_unlock: rcu_read_unlock(); return pol; diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 9e14e453b55cc..a00c4fe1ab0ce 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -2264,6 +2264,7 @@ int xfrm_state_update(struct xfrm_state *x) err = 0; x->km.state = XFRM_STATE_DEAD; + xfrm_dev_state_delete(x); __xfrm_state_put(x); } @@ -3151,6 +3152,7 @@ int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) int err; if (family == AF_INET && + (!x->dir || x->dir == XFRM_SA_DIR_OUT) && READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc)) x->props.flags |= XFRM_STATE_NOPMTUDISC; diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 403b5ecac2c54..b3f69c0760d4c 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -1850,6 +1850,7 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, pcpu_num = nla_get_u32(attrs[XFRMA_SA_PCPU]); if (pcpu_num >= num_possible_cpus()) { err = -EINVAL; + NL_SET_ERR_MSG(extack, "pCPU number too big"); goto out_noput; } } @@ -2667,7 +2668,8 @@ static inline unsigned int xfrm_aevent_msgsize(struct xfrm_state *x) + nla_total_size(4) /* XFRM_AE_RTHR */ + nla_total_size(4) /* XFRM_AE_ETHR */ + nla_total_size(sizeof(x->dir)) /* XFRMA_SA_DIR */ - + nla_total_size(4); /* XFRMA_SA_PCPU */ + + nla_total_size(4) /* XFRMA_SA_PCPU */ + + nla_total_size(sizeof(x->if_id)); /* XFRMA_IF_ID */ } static int build_aevent(struct sk_buff *skb, struct xfrm_state *x, const struct km_event *c) @@ -2779,7 +2781,12 @@ static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh, c.portid = nlh->nlmsg_pid; err = build_aevent(r_skb, x, &c); - BUG_ON(err < 0); + if (err < 0) { + spin_unlock_bh(&x->lock); + xfrm_state_put(x); + kfree_skb(r_skb); + return err; + } err = nlmsg_unicast(net->xfrm.nlsk, r_skb, NETLINK_CB(skb).portid); spin_unlock_bh(&x->lock); @@ -3001,8 +3008,10 @@ static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh, if (attrs[XFRMA_SA_PCPU]) { x->pcpu_num = nla_get_u32(attrs[XFRMA_SA_PCPU]); err = -EINVAL; - if (x->pcpu_num >= num_possible_cpus()) + if (x->pcpu_num >= num_possible_cpus()) { + NL_SET_ERR_MSG(extack, "pCPU number too big"); goto free_state; + } } err = verify_newpolicy_info(&ua->policy, extack); @@ -3673,7 +3682,7 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) } if (x->if_id) l += nla_total_size(sizeof(x->if_id)); - if (x->pcpu_num) + if (x->pcpu_num != UINT_MAX) l += nla_total_size(sizeof(x->pcpu_num)); /* Must count x->lastused as it may become non-zero behind our back. */ @@ -3948,6 +3957,8 @@ static int build_polexpire(struct sk_buff *skb, struct xfrm_policy *xp, return err; } upe->hard = !!hard; + /* clear the padding bytes */ + memset_after(upe, 0, hard); nlmsg_end(skb, nlh); return 0; @@ -4105,6 +4116,7 @@ static int build_report(struct sk_buff *skb, u8 proto, return -EMSGSIZE; ur = nlmsg_data(nlh); + memset(ur, 0, sizeof(*ur)); ur->proto = proto; memcpy(&ur->sel, sel, sizeof(ur->sel)); @@ -4152,6 +4164,7 @@ static int build_mapping(struct sk_buff *skb, struct xfrm_state *x, um = nlmsg_data(nlh); + memset(&um->id, 0, sizeof(um->id)); memcpy(&um->id.daddr, &x->id.daddr, sizeof(um->id.daddr)); um->id.spi = x->id.spi; um->id.family = x->props.family; diff --git a/qcom-next/merge.log b/qcom-next/merge.log new file mode 100644 index 0000000000000..76ae373942bb6 --- /dev/null +++ b/qcom-next/merge.log @@ -0,0 +1,421 @@ +Verified existence of local and remote repos: Success +/local/mnt/workspace/automerge/actions-runner/_work/kernel-automation/kernel-automation/kernel-topics /local/mnt/workspace/automerge/actions-runner/_work/kernel-automation/kernel-automation/kernel-topics +Reuse-Recorded-Resolution: Enabled +Downloaded shared rerere cache +Local tree is clean +Removing old remotes ... +The remote origin https://github.com/qualcomm-linux/kernel-topics.git is no longer tracked. +Delete it [Y/n]? Done, removed 1 old remote(s). +Adding new remotes... +Adding remote baseline https://github.com/qualcomm-linux/kernel.git qcom-next-staging +Updating baseline +Adding remote tech/bsp/clk https://github.com/qualcomm-linux/kernel-topics.git tech/bsp/clk +Updating tech/bsp/clk +Adding remote tech/bsp/interconnect https://github.com/qualcomm-linux/kernel-topics.git tech/bsp/interconnect +Updating tech/bsp/interconnect +Adding remote tech/mem/secure-buffer https://github.com/qualcomm-linux/kernel-topics.git tech/mem/secure-buffer +Updating tech/mem/secure-buffer +Adding remote tech/security/firmware-smc https://github.com/qualcomm-linux/kernel-topics.git tech/security/firmware-smc +Updating tech/security/firmware-smc +Adding remote tech/bsp/soc-infra https://github.com/qualcomm-linux/kernel-topics.git tech/bsp/soc-infra +Updating tech/bsp/soc-infra +Adding remote tech/debug/soc https://github.com/qualcomm-linux/kernel-topics.git tech/debug/soc +Updating tech/debug/soc +Adding remote tech/bsp/pinctrl https://github.com/qualcomm-linux/kernel-topics.git tech/bsp/pinctrl +Updating tech/bsp/pinctrl +Adding remote tech/bsp/remoteproc https://github.com/qualcomm-linux/kernel-topics.git tech/bsp/remoteproc +Updating tech/bsp/remoteproc +Adding remote tech/bus/peripherals https://github.com/qualcomm-linux/kernel-topics.git tech/bus/peripherals +Updating tech/bus/peripherals +Adding remote tech/bus/pci/all https://github.com/qualcomm-linux/kernel-topics.git tech/bus/pci/all +Updating tech/bus/pci/all +Adding remote tech/bus/pci/mhi https://github.com/qualcomm-linux/kernel-topics.git tech/bus/pci/mhi +Updating tech/bus/pci/mhi +Adding remote tech/bus/pci/phy https://github.com/qualcomm-linux/kernel-topics.git tech/bus/pci/phy +Updating tech/bus/pci/phy +Adding remote tech/bus/pci/pwrctl https://github.com/qualcomm-linux/kernel-topics.git tech/bus/pci/pwrctl +Updating tech/bus/pci/pwrctl +Adding remote tech/bus/usb/dwc https://github.com/qualcomm-linux/kernel-topics.git tech/bus/usb/dwc +Updating tech/bus/usb/dwc +Adding remote tech/bus/usb/gadget https://github.com/qualcomm-linux/kernel-topics.git tech/bus/usb/gadget +Updating tech/bus/usb/gadget +Adding remote tech/bus/usb/phy https://github.com/qualcomm-linux/kernel-topics.git tech/bus/usb/phy +Updating tech/bus/usb/phy +Adding remote tech/debug/eud https://github.com/qualcomm-linux/kernel-topics.git tech/debug/eud +Updating tech/debug/eud +Adding remote tech/debug/hwtracing https://github.com/qualcomm-linux/kernel-topics.git tech/debug/hwtracing +Updating tech/debug/hwtracing +Adding remote tech/debug/rdbg https://github.com/qualcomm-linux/kernel-topics.git tech/debug/rdbg +Updating tech/debug/rdbg +Adding remote tech/pmic/backlight https://github.com/qualcomm-linux/kernel-topics.git tech/pmic/backlight +Updating tech/pmic/backlight +Adding remote tech/pmic/mfd https://github.com/qualcomm-linux/kernel-topics.git tech/pmic/mfd +Updating tech/pmic/mfd +Adding remote tech/pmic/misc https://github.com/qualcomm-linux/kernel-topics.git tech/pmic/misc +Updating tech/pmic/misc +Adding remote tech/pmic/regulator https://github.com/qualcomm-linux/kernel-topics.git tech/pmic/regulator +Updating tech/pmic/regulator +Adding remote tech/pmic/supply https://github.com/qualcomm-linux/kernel-topics.git tech/pmic/supply +Updating tech/pmic/supply +Adding remote tech/mem/dma-buf https://github.com/qualcomm-linux/kernel-topics.git tech/mem/dma-buf +Updating tech/mem/dma-buf +Adding remote tech/mem/iommu https://github.com/qualcomm-linux/kernel-topics.git tech/mem/iommu +Updating tech/mem/iommu +Adding remote tech/mm/audio/all https://github.com/qualcomm-linux/kernel-topics.git tech/mm/audio/all +Updating tech/mm/audio/all +Adding remote tech/mm/audio/soundwire https://github.com/qualcomm-linux/kernel-topics.git tech/mm/audio/soundwire +Updating tech/mm/audio/soundwire +Adding remote tech/mm/camss https://github.com/qualcomm-linux/kernel-topics.git tech/mm/camss +Updating tech/mm/camss +Adding remote tech/mm/drm https://github.com/qualcomm-linux/kernel-topics.git tech/mm/drm +Updating tech/mm/drm +Adding remote tech/mm/fastrpc https://github.com/qualcomm-linux/kernel-topics.git tech/mm/fastrpc +Updating tech/mm/fastrpc +Adding remote tech/mm/phy https://github.com/qualcomm-linux/kernel-topics.git tech/mm/phy +Updating tech/mm/phy +Adding remote tech/mm/video https://github.com/qualcomm-linux/kernel-topics.git tech/mm/video +Updating tech/mm/video +Adding remote tech/mm/gpu https://github.com/qualcomm-linux/kernel-topics.git tech/mm/gpu +Updating tech/mm/gpu +Adding remote tech/mproc/rpmsg https://github.com/qualcomm-linux/kernel-topics.git tech/mproc/rpmsg +Updating tech/mproc/rpmsg +Adding remote tech/mproc/qmi https://github.com/qualcomm-linux/kernel-topics.git tech/mproc/qmi +Updating tech/mproc/qmi +Adding remote tech/net/ath https://github.com/qualcomm-linux/kernel-topics.git tech/net/ath +Updating tech/net/ath +Adding remote tech/net/eth https://github.com/qualcomm-linux/kernel-topics.git tech/net/eth +Updating tech/net/eth +Adding remote tech/net/rmnet https://github.com/qualcomm-linux/kernel-topics.git tech/net/rmnet +Updating tech/net/rmnet +Adding remote tech/net/qrtr https://github.com/qualcomm-linux/kernel-topics.git tech/net/qrtr +Updating tech/net/qrtr +Adding remote tech/net/phy https://github.com/qualcomm-linux/kernel-topics.git tech/net/phy +Updating tech/net/phy +Adding remote tech/net/bluetooth https://github.com/qualcomm-linux/kernel-topics.git tech/net/bluetooth +Updating tech/net/bluetooth +Adding remote tech/pm/opp https://github.com/qualcomm-linux/kernel-topics.git tech/pm/opp +Updating tech/pm/opp +Adding remote tech/pm/pmdomain https://github.com/qualcomm-linux/kernel-topics.git tech/pm/pmdomain +Updating tech/pm/pmdomain +Adding remote tech/pm/power https://github.com/qualcomm-linux/kernel-topics.git tech/pm/power +Updating tech/pm/power +Adding remote tech/pm/thermal https://github.com/qualcomm-linux/kernel-topics.git tech/pm/thermal +Updating tech/pm/thermal +Adding remote tech/security/crypto https://github.com/qualcomm-linux/kernel-topics.git tech/security/crypto +Updating tech/security/crypto +Adding remote tech/security/fscrypt https://github.com/qualcomm-linux/kernel-topics.git tech/security/fscrypt +Updating tech/security/fscrypt +Adding remote tech/security/ice https://github.com/qualcomm-linux/kernel-topics.git tech/security/ice +Updating tech/security/ice +Adding remote tech/storage/nvmem https://github.com/qualcomm-linux/kernel-topics.git tech/storage/nvmem +Updating tech/storage/nvmem +Adding remote tech/storage/phy https://github.com/qualcomm-linux/kernel-topics.git tech/storage/phy +Updating tech/storage/phy +Adding remote tech/storage/all https://github.com/qualcomm-linux/kernel-topics.git tech/storage/all +Updating tech/storage/all +Adding remote tech/virt/gunyah https://github.com/qualcomm-linux/kernel-topics.git tech/virt/gunyah +Updating tech/virt/gunyah +Adding remote tech/all/dt/qcs6490 https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/qcs6490 +Updating tech/all/dt/qcs6490 +Adding remote tech/all/dt/qcs9100 https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/qcs9100 +Updating tech/all/dt/qcs9100 +Adding remote tech/all/dt/qcs8300 https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/qcs8300 +Updating tech/all/dt/qcs8300 +Adding remote tech/all/dt/qcs615 https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/qcs615 +Updating tech/all/dt/qcs615 +Adding remote tech/all/dt/hamoa https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/hamoa +Updating tech/all/dt/hamoa +Adding remote tech/all/dt/glymur https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/glymur +Updating tech/all/dt/glymur +Adding remote tech/all/dt/kaanapali https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/kaanapali +Updating tech/all/dt/kaanapali +Adding remote tech/all/dt/pakala https://github.com/qualcomm-linux/kernel-topics.git tech/all/dt/pakala +Updating tech/all/dt/pakala +Adding remote tech/all/config https://github.com/qualcomm-linux/kernel-topics.git tech/all/config +Updating tech/all/config +Adding remote tech/overlay/dt https://github.com/qualcomm-linux/kernel-topics.git tech/overlay/dt +Updating tech/overlay/dt +Adding remote tech/all/workaround https://github.com/qualcomm-linux/kernel-topics.git tech/all/workaround +Updating tech/all/workaround +Adding remote tech/mproc/all https://github.com/qualcomm-linux/kernel-topics.git tech/mproc/all +Updating tech/mproc/all +Done, added 66 new remote(s). +Updating the remotes ... +Updating tech/bsp/clk +Updating tech/bsp/interconnect +Updating tech/mem/secure-buffer +Updating tech/security/firmware-smc +Updating tech/bsp/soc-infra +Updating tech/debug/soc +Updating tech/bsp/pinctrl +Updating tech/bsp/remoteproc +Updating tech/bus/peripherals +Updating tech/bus/pci/all +Updating tech/bus/pci/mhi +Updating tech/bus/pci/phy +Updating tech/bus/pci/pwrctl +Updating tech/bus/usb/dwc +Updating tech/bus/usb/gadget +Updating tech/bus/usb/phy +Updating tech/debug/eud +Updating tech/debug/hwtracing +Updating tech/debug/rdbg +Updating tech/pmic/backlight +Updating tech/pmic/mfd +Updating tech/pmic/misc +Updating tech/pmic/regulator +Updating tech/pmic/supply +Updating tech/mem/dma-buf +Updating tech/mem/iommu +Updating tech/mm/audio/all +Updating tech/mm/audio/soundwire +Updating tech/mm/camss +Updating tech/mm/drm +Updating tech/mm/fastrpc +Updating tech/mm/phy +Updating tech/mm/video +Updating tech/mm/gpu +Updating tech/mproc/rpmsg +Updating tech/mproc/qmi +Updating tech/net/ath +Updating tech/net/eth +Updating tech/net/rmnet +Updating tech/net/qrtr +Updating tech/net/phy +Updating tech/net/bluetooth +Updating tech/pm/opp +Updating tech/pm/pmdomain +Updating tech/pm/power +Updating tech/pm/thermal +Updating tech/security/crypto +Updating tech/security/fscrypt +Updating tech/security/ice +Updating tech/storage/nvmem +Updating tech/storage/phy +Updating tech/storage/all +Updating tech/virt/gunyah +Updating tech/all/dt/qcs6490 +Updating tech/all/dt/qcs9100 +Updating tech/all/dt/qcs8300 +Updating tech/all/dt/qcs615 +Updating tech/all/dt/hamoa +Updating tech/all/dt/glymur +Updating tech/all/dt/kaanapali +Updating tech/all/dt/pakala +Updating tech/all/config +Updating tech/overlay/dt +Updating tech/all/workaround +Updating tech/mproc/all +Done, updated 0 remote(s). +Updating baseline ... +Fetching baseline +latest tag/id is 7d0a66e4bb9081d75c82ec4957c50034cb0ea449 +Done, updated baseline. +Latest tag is 7d0a66e4bb9081d75c82ec4957c50034cb0ea449 +Create a new integration branch based on 7d0a66e4bb9081d75c82ec4957c50034cb0ea449 +Merging topic branches... +------------------------------------------ + ** Merging topic branch: tech/bsp/clk/tech/bsp/clk +Merge successful : tech/bsp/clk : aea4f0c77e2524664061c3756cf8cadccec41145 : 8 +------------------------------------------ + ** Merging topic branch: tech/bsp/interconnect/tech/bsp/interconnect +Merge successful : tech/bsp/interconnect : 938409e3faa500839cfe2db3a2e3becfd9fb7fc6 : 2 +------------------------------------------ + ** Merging topic branch: tech/mem/secure-buffer/tech/mem/secure-buffer +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/security/firmware-smc/tech/security/firmware-smc +Merge successful : tech/security/firmware-smc : a50984a91bb9d551ee48136ee2db90eca0929979 : 2 +------------------------------------------ + ** Merging topic branch: tech/bsp/soc-infra/tech/bsp/soc-infra +Merge successful : tech/bsp/soc-infra : 797d5a2c99ee52b6fdbafcd0862e867d151c0539 : 18 +------------------------------------------ + ** Merging topic branch: tech/debug/soc/tech/debug/soc +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/bsp/pinctrl/tech/bsp/pinctrl +Merge successful : tech/bsp/pinctrl : 9a297f4d9e88c69bd9a650364126da4d2dbb658f : 2 +------------------------------------------ + ** Merging topic branch: tech/bsp/remoteproc/tech/bsp/remoteproc +Merge successful : tech/bsp/remoteproc : 464fb78ea02cf616d58464b045264e52c5f756cb : 13 +------------------------------------------ + ** Merging topic branch: tech/bus/peripherals/tech/bus/peripherals +Merge successful : tech/bus/peripherals : 486bcf7a4a94e54abf5fd9e6cc6bf68eea22a5d6 : 1 +------------------------------------------ + ** Merging topic branch: tech/bus/pci/all/tech/bus/pci/all +Merge successful : tech/bus/pci/all : 205830c6534ff4195ec16aa531f6dbcc4fa487e7 : 8 +------------------------------------------ + ** Merging topic branch: tech/bus/pci/mhi/tech/bus/pci/mhi +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/bus/pci/phy/tech/bus/pci/phy +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/bus/pci/pwrctl/tech/bus/pci/pwrctl +Merge successful : tech/bus/pci/pwrctl : c36944ebc50ed5d40b3c550cbe5792b93c32dc27 : 7 +------------------------------------------ + ** Merging topic branch: tech/bus/usb/dwc/tech/bus/usb/dwc +Merge successful : tech/bus/usb/dwc : c29f570c16249b9ae1472b08a3570f2985fa5532 : 1 +------------------------------------------ + ** Merging topic branch: tech/bus/usb/gadget/tech/bus/usb/gadget +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/bus/usb/phy/tech/bus/usb/phy +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/debug/eud/tech/debug/eud +Merge successful : tech/debug/eud : eb36d9dbce445b0a2032775a5668ba02dd4b67cb : 1 +------------------------------------------ + ** Merging topic branch: tech/debug/hwtracing/tech/debug/hwtracing +Merge successful : tech/debug/hwtracing : 961640988eb239085e7643c0ec26e627d8389049 : 34 +------------------------------------------ + ** Merging topic branch: tech/debug/rdbg/tech/debug/rdbg +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/pmic/backlight/tech/pmic/backlight +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/pmic/mfd/tech/pmic/mfd +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/pmic/misc/tech/pmic/misc +Merge successful : tech/pmic/misc : 8fba65f6e2ddebad2e93252a85e30a35fa99733b : 14 +------------------------------------------ + ** Merging topic branch: tech/pmic/regulator/tech/pmic/regulator +Merge successful : tech/pmic/regulator : 81fc8fbf0f0a02a7b2d855aee574b88bd7d59107 : 6 +------------------------------------------ + ** Merging topic branch: tech/pmic/supply/tech/pmic/supply +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/mem/dma-buf/tech/mem/dma-buf +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/mem/iommu/tech/mem/iommu +Merge successful : tech/mem/iommu : fc1b59c1f4305fd3630a4205d2db8f680626ef5c : 1 +------------------------------------------ + ** Merging topic branch: tech/mm/audio/all/tech/mm/audio/all +Merge successful : tech/mm/audio/all : 37d1752ff15d0e43519f2803ab65e069d5af126f : 1 +------------------------------------------ + ** Merging topic branch: tech/mm/audio/soundwire/tech/mm/audio/soundwire +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/mm/camss/tech/mm/camss +Merge successful : tech/mm/camss : d1d2c38f53e8f67b6a11866d4202561fbed0a988 : 3 +------------------------------------------ + ** Merging topic branch: tech/mm/drm/tech/mm/drm +Merge successful : tech/mm/drm : 378575e046b64e79a992b62453775ecfe63fc36b : 38 +------------------------------------------ + ** Merging topic branch: tech/mm/fastrpc/tech/mm/fastrpc +Merge successful : tech/mm/fastrpc : dba4eb255febb4cc38d2e340978c58f8f21ff6cc : 3 +------------------------------------------ + ** Merging topic branch: tech/mm/phy/tech/mm/phy +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/mm/video/tech/mm/video +Merge successful : tech/mm/video : 1af1bf9935671af7c556b7e952a9afd2def6a58d : 13 +------------------------------------------ + ** Merging topic branch: tech/mm/gpu/tech/mm/gpu +Merge successful : tech/mm/gpu : 66e31064fee4eca6f3afa26a8db511bb160d2647 : 2 +------------------------------------------ + ** Merging topic branch: tech/mproc/rpmsg/tech/mproc/rpmsg +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/mproc/qmi/tech/mproc/qmi +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/net/ath/tech/net/ath +Merge successful : tech/net/ath : 7ed8cea7644a342684c7ca711f6ff7c65708affa : 2 +------------------------------------------ + ** Merging topic branch: tech/net/eth/tech/net/eth +Merge successful : tech/net/eth : c280d7e86faa86b16d3c1e313e95207503cf3770 : 1 +------------------------------------------ + ** Merging topic branch: tech/net/rmnet/tech/net/rmnet +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/net/qrtr/tech/net/qrtr +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/net/phy/tech/net/phy +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/net/bluetooth/tech/net/bluetooth +Merge successful : tech/net/bluetooth : b5902f2daf61ad3eed61a9fbf3d75ba3748cf53e : 2 +------------------------------------------ + ** Merging topic branch: tech/pm/opp/tech/pm/opp +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/pm/pmdomain/tech/pm/pmdomain +Merge successful : tech/pm/pmdomain : c2e58ee4e1ed72c4fe6fb6a48d8d83d4ea2269fc : 3 +------------------------------------------ + ** Merging topic branch: tech/pm/power/tech/pm/power +Merge successful : tech/pm/power : 9fe45cfe84f7873accec2689b312180897cb56d0 : 7 +------------------------------------------ + ** Merging topic branch: tech/pm/thermal/tech/pm/thermal +Merge successful : tech/pm/thermal : 44a5bf0663cff2f58a32907b7a944fa8c120812e : 1 +------------------------------------------ + ** Merging topic branch: tech/security/crypto/tech/security/crypto +Merge successful : tech/security/crypto : 0ff2ae7f2e708a083b489a9a4a34c07bd701aec1 : 11 +------------------------------------------ + ** Merging topic branch: tech/security/fscrypt/tech/security/fscrypt +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/security/ice/tech/security/ice +Merge successful : tech/security/ice : e6140349dfc670e22f2507bb8f053be15b353ddd : 3 +------------------------------------------ + ** Merging topic branch: tech/storage/nvmem/tech/storage/nvmem +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/storage/phy/tech/storage/phy +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/storage/all/tech/storage/all +Merge successful : tech/storage/all : ba8c93ddc42883a8b91db3e5fd0f19d0c79a9ddb : 6 +------------------------------------------ + ** Merging topic branch: tech/virt/gunyah/tech/virt/gunyah +Nothing to merge: Already up to date. +------------------------------------------ + ** Merging topic branch: tech/all/dt/qcs6490/tech/all/dt/qcs6490 +Merge successful : tech/all/dt/qcs6490 : 91a5f259df1c65b4981c51e7d825a26617afbd50 : 7 +------------------------------------------ + ** Merging topic branch: tech/all/dt/qcs9100/tech/all/dt/qcs9100 +Merge successful : tech/all/dt/qcs9100 : ae04b95f30a6b56d2bbbf59fdb94d45e2be7d0f9 : 15 +------------------------------------------ + ** Merging topic branch: tech/all/dt/qcs8300/tech/all/dt/qcs8300 +Merge successful : tech/all/dt/qcs8300 : 6fc2c3911517bf9bde33e5795251ae465ff2c4d9 : 28 +------------------------------------------ + ** Merging topic branch: tech/all/dt/qcs615/tech/all/dt/qcs615 +Merge successful : tech/all/dt/qcs615 : ac047bf2fb68a45c0708be88eb78482f1b5278f3 : 8 +------------------------------------------ + ** Merging topic branch: tech/all/dt/hamoa/tech/all/dt/hamoa +Merge successful : tech/all/dt/hamoa : 731b038c653da0d0585f9fdfd9c0d04a9f489f3c : 10 +------------------------------------------ + ** Merging topic branch: tech/all/dt/glymur/tech/all/dt/glymur +Merge successful : tech/all/dt/glymur : 6e186f99af366fcd4e9d2e4e5cbf77198470ade9 : 20 +------------------------------------------ + ** Merging topic branch: tech/all/dt/kaanapali/tech/all/dt/kaanapali +Merge successful : tech/all/dt/kaanapali : 746a2029c703df8d18c3b3209cd1cb635a2abd92 : 3 +------------------------------------------ + ** Merging topic branch: tech/all/dt/pakala/tech/all/dt/pakala +Merge successful : tech/all/dt/pakala : e0326f1e866dc1c82669249763e1c76fd60f49f0 : 4 +------------------------------------------ + ** Merging topic branch: tech/all/config/tech/all/config +Merge successful : tech/all/config : b2866eca5697568294f76b9d665bfc6224f38dcf : 28 +------------------------------------------ + ** Merging topic branch: tech/overlay/dt/tech/overlay/dt +Merge failed, manual merge +No files need merging +[qcom-next 3342111513a1] Merge remote-tracking branch tech/overlay/dt into qcom-next +Merge successful : tech/overlay/dt : 03e0b7b70b8df2b6b498a1f7ed565c2bc4a0833b : 5 +------------------------------------------ + ** Merging topic branch: tech/all/workaround/tech/all/workaround +Merge failed, manual merge +No files need merging +[qcom-next 67a672b7d930] Merge remote-tracking branch tech/all/workaround into qcom-next +Merge successful : tech/all/workaround : 841239d0dc8294a4a9bb5f63b3f4c0dbcf21247f : 3 +------------------------------------------ + ** Merging topic branch: tech/mproc/all/tech/mproc/all +Merge successful : tech/mproc/all : 2cf103048fa6404ba91ace93c35997fee43b7f7d : 4 +Done, merged 42 topic(s). +[main 53bc5d7] New rr-cache entries from ci-merge + 1 file changed, 219 insertions(+) + create mode 100644 rr-cache/fd62a37d4fe0f74d9caa954b9b888e7f13e93a28/thisimage diff --git a/qcom-next/topic_SHA1 b/qcom-next/topic_SHA1 new file mode 100644 index 0000000000000..a6fe7348f4228 --- /dev/null +++ b/qcom-next/topic_SHA1 @@ -0,0 +1,44 @@ +Name SHA Commits +------------------------------------------------------------------------------------ +tech/bsp/clk aea4f0c77e2524664061c3756cf8cadccec41145 8 +tech/bsp/interconnect 938409e3faa500839cfe2db3a2e3becfd9fb7fc6 2 +tech/security/firmware-smc a50984a91bb9d551ee48136ee2db90eca0929979 2 +tech/bsp/soc-infra 797d5a2c99ee52b6fdbafcd0862e867d151c0539 18 +tech/bsp/pinctrl 9a297f4d9e88c69bd9a650364126da4d2dbb658f 2 +tech/bsp/remoteproc 464fb78ea02cf616d58464b045264e52c5f756cb 13 +tech/bus/peripherals 486bcf7a4a94e54abf5fd9e6cc6bf68eea22a5d6 1 +tech/bus/pci/all 205830c6534ff4195ec16aa531f6dbcc4fa487e7 8 +tech/bus/pci/pwrctl c36944ebc50ed5d40b3c550cbe5792b93c32dc27 7 +tech/bus/usb/dwc c29f570c16249b9ae1472b08a3570f2985fa5532 1 +tech/debug/eud eb36d9dbce445b0a2032775a5668ba02dd4b67cb 1 +tech/debug/hwtracing 961640988eb239085e7643c0ec26e627d8389049 34 +tech/pmic/misc 8fba65f6e2ddebad2e93252a85e30a35fa99733b 14 +tech/pmic/regulator 81fc8fbf0f0a02a7b2d855aee574b88bd7d59107 6 +tech/mem/iommu fc1b59c1f4305fd3630a4205d2db8f680626ef5c 1 +tech/mm/audio/all 37d1752ff15d0e43519f2803ab65e069d5af126f 1 +tech/mm/camss d1d2c38f53e8f67b6a11866d4202561fbed0a988 3 +tech/mm/drm 378575e046b64e79a992b62453775ecfe63fc36b 38 +tech/mm/fastrpc dba4eb255febb4cc38d2e340978c58f8f21ff6cc 3 +tech/mm/video 1af1bf9935671af7c556b7e952a9afd2def6a58d 13 +tech/mm/gpu 66e31064fee4eca6f3afa26a8db511bb160d2647 2 +tech/net/ath 7ed8cea7644a342684c7ca711f6ff7c65708affa 2 +tech/net/eth c280d7e86faa86b16d3c1e313e95207503cf3770 1 +tech/net/bluetooth b5902f2daf61ad3eed61a9fbf3d75ba3748cf53e 2 +tech/pm/pmdomain c2e58ee4e1ed72c4fe6fb6a48d8d83d4ea2269fc 3 +tech/pm/power 9fe45cfe84f7873accec2689b312180897cb56d0 7 +tech/pm/thermal 44a5bf0663cff2f58a32907b7a944fa8c120812e 1 +tech/security/crypto 0ff2ae7f2e708a083b489a9a4a34c07bd701aec1 11 +tech/security/ice e6140349dfc670e22f2507bb8f053be15b353ddd 3 +tech/storage/all ba8c93ddc42883a8b91db3e5fd0f19d0c79a9ddb 6 +tech/all/dt/qcs6490 91a5f259df1c65b4981c51e7d825a26617afbd50 7 +tech/all/dt/qcs9100 ae04b95f30a6b56d2bbbf59fdb94d45e2be7d0f9 15 +tech/all/dt/qcs8300 6fc2c3911517bf9bde33e5795251ae465ff2c4d9 28 +tech/all/dt/qcs615 ac047bf2fb68a45c0708be88eb78482f1b5278f3 8 +tech/all/dt/hamoa 731b038c653da0d0585f9fdfd9c0d04a9f489f3c 10 +tech/all/dt/glymur 6e186f99af366fcd4e9d2e4e5cbf77198470ade9 20 +tech/all/dt/kaanapali 746a2029c703df8d18c3b3209cd1cb635a2abd92 3 +tech/all/dt/pakala e0326f1e866dc1c82669249763e1c76fd60f49f0 4 +tech/all/config b2866eca5697568294f76b9d665bfc6224f38dcf 28 +tech/overlay/dt 03e0b7b70b8df2b6b498a1f7ed565c2bc4a0833b 5 +tech/all/workaround 841239d0dc8294a4a9bb5f63b3f4c0dbcf21247f 3 +tech/mproc/all 2cf103048fa6404ba91ace93c35997fee43b7f7d 4 diff --git a/rust/Makefile b/rust/Makefile index 7842ad0a4ea72..d4618f646b05b 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -437,6 +437,8 @@ $(obj)/$(libpin_init_internal_name): private rustc_target_flags = --cfg kernel $(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs FORCE +$(call if_changed_dep,rustc_procmacro) +# `rustc` requires `-Zunstable-options` to use custom target specifications +# since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534). quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ cmd_rustc_library = \ OBJTREE=$(abspath $(objtree)) \ @@ -447,6 +449,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ --sysroot=/dev/null \ + -Zunstable-options \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) diff --git a/rust/helpers/bitops.c b/rust/helpers/bitops.c index 5d0861d29d3f0..e79ef9e6d98f9 100644 --- a/rust/helpers/bitops.c +++ b/rust/helpers/bitops.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include void rust_helper___set_bit(unsigned long nr, unsigned long *addr) { @@ -21,3 +22,44 @@ void rust_helper_clear_bit(unsigned long nr, volatile unsigned long *addr) { clear_bit(nr, addr); } + +/* + * The rust_helper_ prefix is intentionally omitted below so that the + * declarations in include/linux/find.h are compatible with these helpers. + * + * Note that the below #ifdefs mean that the helper is only created if C does + * not provide a definition. + */ +#ifdef find_first_zero_bit +__rust_helper +unsigned long _find_first_zero_bit(const unsigned long *p, unsigned long size) +{ + return find_first_zero_bit(p, size); +} +#endif /* find_first_zero_bit */ + +#ifdef find_next_zero_bit +__rust_helper +unsigned long _find_next_zero_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + return find_next_zero_bit(addr, size, offset); +} +#endif /* find_next_zero_bit */ + +#ifdef find_first_bit +__rust_helper +unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) +{ + return find_first_bit(addr, size); +} +#endif /* find_first_bit */ + +#ifdef find_next_bit +__rust_helper +unsigned long _find_next_bit(const unsigned long *addr, unsigned long size, + unsigned long offset) +{ + return find_next_bit(addr, size, offset); +} +#endif /* find_next_bit */ diff --git a/rust/helpers/dma.c b/rust/helpers/dma.c index 6e741c1972425..2afa32c21c946 100644 --- a/rust/helpers/dma.c +++ b/rust/helpers/dma.c @@ -19,3 +19,24 @@ int rust_helper_dma_set_mask_and_coherent(struct device *dev, u64 mask) { return dma_set_mask_and_coherent(dev, mask); } + +int rust_helper_dma_set_mask(struct device *dev, u64 mask) +{ + return dma_set_mask(dev, mask); +} + +int rust_helper_dma_set_coherent_mask(struct device *dev, u64 mask) +{ + return dma_set_coherent_mask(dev, mask); +} + +int rust_helper_dma_map_sgtable(struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, unsigned long attrs) +{ + return dma_map_sgtable(dev, sgt, dir, attrs); +} + +size_t rust_helper_dma_max_mapping_size(struct device *dev) +{ + return dma_max_mapping_size(dev); +} diff --git a/rust/kernel/bits.rs b/rust/kernel/bits.rs index 553d502658832..2daead125626a 100644 --- a/rust/kernel/bits.rs +++ b/rust/kernel/bits.rs @@ -27,7 +27,8 @@ macro_rules! impl_bit_fn { /// /// This version is the default and should be used if `n` is known at /// compile time. - #[inline] + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] pub const fn [](n: u32) -> $ty { build_assert!(n < <$ty>::BITS); (1 as $ty) << n @@ -75,7 +76,8 @@ macro_rules! impl_genmask_fn { /// This version is the default and should be used if the range is known /// at compile time. $(#[$genmask_ex])* - #[inline] + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] pub const fn [](range: RangeInclusive) -> $ty { let start = *range.start(); let end = *range.end(); diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 1a555fcb120a9..df5d9f6f43f3b 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -1015,6 +1015,8 @@ impl Registration { ..pin_init::zeroed() }; + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] { let src = name.to_bytes_with_nul(); let mut dst = [0; CPUFREQ_NAME_LEN]; diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index a849b7dde2fd9..176531f54ed30 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -62,8 +62,9 @@ pub mod property; /// /// # Implementing Bus Devices /// -/// This section provides a guideline to implement bus specific devices, such as [`pci::Device`] or -/// [`platform::Device`]. +/// This section provides a guideline to implement bus specific devices, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`] /// /// A bus specific device should be defined as follows. /// @@ -155,7 +156,6 @@ pub mod property; /// /// [`AlwaysRefCounted`]: kernel::types::AlwaysRefCounted /// [`impl_device_context_deref`]: kernel::impl_device_context_deref -/// [`pci::Device`]: kernel::pci::Device /// [`platform::Device`]: kernel::platform::Device #[repr(transparent)] pub struct Device(Opaque, PhantomData); diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 2392c281459ef..835d9c11948e7 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -52,8 +52,20 @@ struct Inner { /// # Examples /// /// ```no_run -/// # use kernel::{bindings, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}}; -/// # use core::ops::Deref; +/// use kernel::{ +/// bindings, +/// device::{ +/// Bound, +/// Device, +/// }, +/// devres::Devres, +/// io::{ +/// Io, +/// IoRaw, +/// PhysAddr, +/// }, +/// }; +/// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. /// struct IoMem(IoRaw); @@ -66,7 +78,7 @@ struct Inner { /// unsafe fn new(paddr: usize) -> Result{ /// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is /// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as bindings::phys_addr_t, SIZE) }; +/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; /// if addr.is_null() { /// return Err(ENOMEM); /// } diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 4e0af3e1a3b9a..777ea5c2431c7 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -26,8 +26,9 @@ pub type DmaAddress = bindings::dma_addr_t; /// Trait to be implemented by DMA capable bus devices. /// /// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, -/// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or -/// [`platform::Device`](::kernel::platform::Device). +/// where the underlying bus is DMA capable, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`](::kernel::platform::Device) pub trait Device: AsRef> { /// Set up the device's DMA streaming addressing capabilities. /// diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 279e3af206822..16931b94d0d48 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -33,7 +33,14 @@ //! } //! ``` //! -//! For specific examples see [`auxiliary::Driver`], [`pci::Driver`] and [`platform::Driver`]. +//! For specific examples see: +//! +//! * [`platform::Driver`](kernel::platform::Driver) +#![cfg_attr( + CONFIG_AUXILIARY_BUS, + doc = "* [`auxiliary::Driver`](kernel::auxiliary::Driver)" +)] +#![cfg_attr(CONFIG_PCI, doc = "* [`pci::Driver`](kernel::pci::Driver)")] //! //! The `probe()` callback should return a `Result>>`, i.e. the driver's private //! data. The bus abstraction should store the pointer in the corresponding bus device. The generic @@ -79,7 +86,6 @@ //! //! For this purpose the generic infrastructure in [`device_id`] should be used. //! -//! [`auxiliary::Driver`]: kernel::auxiliary::Driver //! [`Core`]: device::Core //! [`Device`]: device::Device //! [`Device`]: device::Device @@ -87,8 +93,6 @@ //! [`DeviceContext`]: device::DeviceContext //! [`device_id`]: kernel::device_id //! [`module_driver`]: kernel::module_driver -//! [`pci::Driver`]: kernel::pci::Driver -//! [`platform::Driver`]: kernel::platform::Driver use crate::error::{Error, Result}; use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index f30ee4c6245cd..e09f977b5b519 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -121,7 +121,6 @@ pub trait Driver { pub struct Registration(ARef>); impl Registration { - /// Creates a new [`Registration`] and registers it. fn new(drm: &drm::Device, flags: usize) -> Result { // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; @@ -129,8 +128,9 @@ impl Registration { Ok(Self(drm.into())) } - /// Same as [`Registration::new`}, but transfers ownership of the [`Registration`] to - /// [`devres::register`]. + /// Registers a new [`Device`](drm::Device) with userspace. + /// + /// Ownership of the [`Registration`] object is passed to [`devres::register`]. pub fn new_foreign_owned( drm: &drm::Device, dev: &device::Device, diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 30c853988b942..73b5f0bf3f85d 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -184,7 +184,7 @@ impl BaseObject for T {} /// A base GEM object. /// -/// Invariants +/// # Invariants /// /// - `self.obj` is a valid instance of a `struct drm_gem_object`. /// - `self.dev` is always a valid pointer to a `struct drm_device`. diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index ee182b0b5452d..0f79381734835 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -13,6 +13,18 @@ pub mod resource; pub use resource::Resource; +/// Physical address type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type PhysAddr = bindings::phys_addr_t; + +/// Resource Size type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type ResourceSize = bindings::resource_size_t; + /// Raw representation of an MMIO region. /// /// By itself, the existence of an instance of this structure does not provide any guarantees that @@ -62,8 +74,16 @@ impl IoRaw { /// # Examples /// /// ```no_run -/// # use kernel::{bindings, ffi::c_void, io::{Io, IoRaw}}; -/// # use core::ops::Deref; +/// use kernel::{ +/// bindings, +/// ffi::c_void, +/// io::{ +/// Io, +/// IoRaw, +/// PhysAddr, +/// }, +/// }; +/// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. /// struct IoMem(IoRaw); @@ -76,7 +96,7 @@ impl IoRaw { /// unsafe fn new(paddr: usize) -> Result{ /// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is /// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as bindings::phys_addr_t, SIZE) }; +/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; /// if addr.is_null() { /// return Err(ENOMEM); /// } @@ -120,7 +140,8 @@ macro_rules! define_read { /// Bound checks are performed on compile time, hence if the offset is not known at compile /// time, the build will fail. $(#[$attr])* - #[inline] + // Always inline to optimize out error path of `io_addr_assert`. + #[inline(always)] pub fn $name(&self, offset: usize) -> $type_name { let addr = self.io_addr_assert::<$type_name>(offset); @@ -149,7 +170,8 @@ macro_rules! define_write { /// Bound checks are performed on compile time, hence if the offset is not known at compile /// time, the build will fail. $(#[$attr])* - #[inline] + // Always inline to optimize out error path of `io_addr_assert`. + #[inline(always)] pub fn $name(&self, value: $type_name, offset: usize) { let addr = self.io_addr_assert::<$type_name>(offset); @@ -217,7 +239,8 @@ impl Io { self.addr().checked_add(offset).ok_or(EINVAL) } - #[inline] + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] fn io_addr_assert(&self, offset: usize) -> usize { build_assert!(Self::offset_valid::(offset, SIZE)); diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index bea3ee0ed87b5..3d54f2828e791 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -12,11 +12,10 @@ use crate::prelude::*; use crate::str::{CStr, CString}; use crate::types::Opaque; -/// Resource Size type. -/// -/// This is a type alias to either `u32` or `u64` depending on the config option -/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. -pub type ResourceSize = bindings::phys_addr_t; +pub use super::{ + PhysAddr, + ResourceSize, // +}; /// A region allocated from a parent [`Resource`]. /// @@ -97,7 +96,7 @@ impl Resource { /// the region, or a part of it, is already in use. pub fn request_region( &self, - start: ResourceSize, + start: PhysAddr, size: ResourceSize, name: CString, flags: Flags, @@ -131,7 +130,7 @@ impl Resource { } /// Returns the start address of the resource. - pub fn start(&self) -> ResourceSize { + pub fn start(&self) -> PhysAddr { let inner = self.0.get(); // SAFETY: Safe as per the invariants of `Resource`. unsafe { (*inner).start } @@ -223,6 +222,8 @@ impl Flags { /// Resource represents a memory region that must be ioremaped using `ioremap_np`. pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] const fn new(value: u32) -> Self { crate::build_assert!(value as u64 <= c_ulong::MAX as u64); Flags(value as c_ulong) diff --git a/rust/kernel/irq/flags.rs b/rust/kernel/irq/flags.rs index adfde96ec47cf..d26e25af06eea 100644 --- a/rust/kernel/irq/flags.rs +++ b/rust/kernel/irq/flags.rs @@ -96,6 +96,8 @@ impl Flags { self.0 } + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] const fn new(value: u32) -> Self { build_assert!(value as u64 <= c_ulong::MAX as u64); Self(value as c_ulong) diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs index b150563fdef80..2ceeaeb0543a4 100644 --- a/rust/kernel/irq/request.rs +++ b/rust/kernel/irq/request.rs @@ -261,7 +261,10 @@ impl Registration { /// # Safety /// /// This function should be only used as the callback in `request_irq`. -unsafe extern "C" fn handle_irq_callback(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn handle_irq_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { // SAFETY: `ptr` is a pointer to `Registration` set in `Registration::new` let registration = unsafe { &*(ptr as *const Registration) }; // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq @@ -480,7 +483,7 @@ impl ThreadedRegistration { /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn handle_threaded_irq_callback( +unsafe extern "C" fn handle_threaded_irq_callback( _irq: i32, ptr: *mut c_void, ) -> c_uint { @@ -496,7 +499,10 @@ unsafe extern "C" fn handle_threaded_irq_callback( /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn thread_fn_callback(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn thread_fn_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { // SAFETY: `ptr` is a pointer to `ThreadedRegistration` set in `ThreadedRegistration::new` let registration = unsafe { &*(ptr as *const ThreadedRegistration) }; // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 79436509dd73d..8907b6f89ece5 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -17,6 +17,10 @@ use crate::c_str; /// Public but hidden since it should only be used from KUnit generated code. #[doc(hidden)] pub fn err(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we // are passing. #[cfg(CONFIG_PRINTK)] @@ -33,6 +37,10 @@ pub fn err(args: fmt::Arguments<'_>) { /// Public but hidden since it should only be used from KUnit generated code. #[doc(hidden)] pub fn info(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we // are passing. #[cfg(CONFIG_PRINTK)] diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs index 202bc6f97c132..ee53d0387e63d 100644 --- a/rust/kernel/list/impl_list_item_mod.rs +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -84,11 +84,12 @@ macro_rules! impl_has_list_links_self_ptr { // right type. unsafe impl$(<$($generics)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for $self {} + // SAFETY: TODO. unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { #[inline] unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { - // SAFETY: The caller promises that the pointer is not dangling. let ptr: *mut $crate::list::ListLinksSelfPtr<$item_type $(, $id)?> = + // SAFETY: The caller promises that the pointer is not dangling. unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) }; ptr.cast() } @@ -217,7 +218,7 @@ macro_rules! impl_list_item { // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it // points at the field `$field` in a value of type `Self`. Thus, reversing that // operation is still in-bounds of the allocation. - $crate::container_of!(me, Self, $($field).*) + unsafe { $crate::container_of!(me, Self, $($field).*) } } // GUARANTEES: @@ -242,7 +243,7 @@ macro_rules! impl_list_item { // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it // points at the field `$field` in a value of type `Self`. Thus, reversing that // operation is still in-bounds of the allocation. - $crate::container_of!(me, Self, $($field).*) + unsafe { $crate::container_of!(me, Self, $($field).*) } } } )*}; @@ -270,9 +271,12 @@ macro_rules! impl_list_item { // SAFETY: The caller promises that `me` points at a valid value of type `Self`. let links_field = unsafe { >::view_links(me) }; - let container = $crate::container_of!( - links_field, $crate::list::ListLinksSelfPtr, inner - ); + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; // SAFETY: By the same reasoning above, `links_field` is a valid pointer. let self_ptr = unsafe { @@ -319,9 +323,12 @@ macro_rules! impl_list_item { // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot // be destroyed while a `ListArc` reference exists. unsafe fn view_value(links_field: *mut $crate::list::ListLinks<$num>) -> *const Self { - let container = $crate::container_of!( - links_field, $crate::list::ListLinksSelfPtr, inner - ); + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; // SAFETY: By the same reasoning above, `links_field` is a valid pointer. let self_ptr = unsafe { diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs index e72eec56bf577..265d6396a78a1 100644 --- a/rust/kernel/maple_tree.rs +++ b/rust/kernel/maple_tree.rs @@ -265,7 +265,16 @@ impl MapleTree { loop { // This uses the raw accessor because we're destroying pointers without removing them // from the maple tree, which is only valid because this is the destructor. - let ptr = ma_state.mas_find_raw(usize::MAX); + // + // Take the rcu lock because mas_find_raw() requires that you hold either the spinlock + // or the rcu read lock. This is only really required if memory reclaim might + // reallocate entries in the tree, as we otherwise have exclusive access. That feature + // doesn't exist yet, so for now, taking the rcu lock only serves the purpose of + // silencing lockdep. + let ptr = { + let _rcu = kernel::sync::rcu::Guard::new(); + ma_state.mas_find_raw(usize::MAX) + }; if ptr.is_null() { break; } diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index b8fe6be6fcc41..9e178dacddf1d 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -835,7 +835,7 @@ impl<'a, K, V> Cursor<'a, K, V> { self.peek(Direction::Prev) } - /// Access the previous node without moving the cursor. + /// Access the next node without moving the cursor. pub fn peek_next(&self) -> Option<(&K, &V)> { self.peek(Direction::Next) } diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs index b55a201e5029f..09c8263043bc7 100644 --- a/rust/kernel/regulator.rs +++ b/rust/kernel/regulator.rs @@ -23,7 +23,10 @@ use crate::{ prelude::*, }; -use core::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, // +}; mod private { pub trait Sealed {} @@ -230,15 +233,17 @@ pub fn devm_enable_optional(dev: &Device, name: &CStr) -> Result { /// /// # Invariants /// -/// - `inner` is a non-null wrapper over a pointer to a `struct -/// regulator` obtained from [`regulator_get()`]. +/// - `inner` is a pointer obtained from a successful call to +/// [`regulator_get()`]. It is treated as an opaque token that may only be +/// accessed using C API methods (e.g., it may be `NULL` if the C API returns +/// `NULL`). /// /// [`regulator_get()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_get pub struct Regulator where State: RegulatorState, { - inner: NonNull, + inner: *mut bindings::regulator, _phantom: PhantomData, } @@ -250,7 +255,7 @@ impl Regulator { // SAFETY: Safe as per the type invariants of `Regulator`. to_result(unsafe { bindings::regulator_set_voltage( - self.inner.as_ptr(), + self.inner, min_voltage.as_microvolts(), max_voltage.as_microvolts(), ) @@ -260,7 +265,7 @@ impl Regulator { /// Gets the current voltage of the regulator. pub fn get_voltage(&self) -> Result { // SAFETY: Safe as per the type invariants of `Regulator`. - let voltage = unsafe { bindings::regulator_get_voltage(self.inner.as_ptr()) }; + let voltage = unsafe { bindings::regulator_get_voltage(self.inner) }; to_result(voltage).map(|()| Voltage::from_microvolts(voltage)) } @@ -270,10 +275,8 @@ impl Regulator { // received from the C code. let inner = from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_ptr()) })?; - // SAFETY: We can safely trust `inner` to be a pointer to a valid - // regulator if `ERR_PTR` was not returned. - let inner = unsafe { NonNull::new_unchecked(inner) }; - + // INVARIANT: `inner` is a pointer obtained from `regulator_get()`, and + // the call was successful. Ok(Self { inner, _phantom: PhantomData, @@ -282,12 +285,12 @@ impl Regulator { fn enable_internal(&self) -> Result { // SAFETY: Safe as per the type invariants of `Regulator`. - to_result(unsafe { bindings::regulator_enable(self.inner.as_ptr()) }) + to_result(unsafe { bindings::regulator_enable(self.inner) }) } fn disable_internal(&self) -> Result { // SAFETY: Safe as per the type invariants of `Regulator`. - to_result(unsafe { bindings::regulator_disable(self.inner.as_ptr()) }) + to_result(unsafe { bindings::regulator_disable(self.inner) }) } } @@ -349,7 +352,7 @@ impl Regulator { /// Checks if the regulator is enabled. pub fn is_enabled(&self) -> bool { // SAFETY: Safe as per the type invariants of `Regulator`. - unsafe { bindings::regulator_is_enabled(self.inner.as_ptr()) != 0 } + unsafe { bindings::regulator_is_enabled(self.inner) != 0 } } } @@ -359,11 +362,11 @@ impl Drop for Regulator { // SAFETY: By the type invariants, we know that `self` owns a // reference on the enabled refcount, so it is safe to relinquish it // now. - unsafe { bindings::regulator_disable(self.inner.as_ptr()) }; + unsafe { bindings::regulator_disable(self.inner) }; } // SAFETY: By the type invariants, we know that `self` owns a reference, // so it is safe to relinquish it now. - unsafe { bindings::regulator_put(self.inner.as_ptr()) }; + unsafe { bindings::regulator_put(self.inner) }; } } diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 5c74e5f776014..8992fedabaf0f 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -880,13 +880,13 @@ impl fmt::Write for Formatter<'_> { /// /// * The first byte of `buffer` is always zero. /// * The length of `buffer` is at least 1. -pub(crate) struct NullTerminatedFormatter<'a> { +pub struct NullTerminatedFormatter<'a> { buffer: &'a mut [u8], } impl<'a> NullTerminatedFormatter<'a> { /// Create a new [`Self`] instance. - pub(crate) fn new(buffer: &'a mut [u8]) -> Option> { + pub fn new(buffer: &'a mut [u8]) -> Option> { *(buffer.first_mut()?) = 0; // INVARIANT: diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 45a17985cda45..0fca1ba3c2dbd 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -35,12 +35,23 @@ unsafe impl super::AtomicAdd for i64 { // as `isize` and `usize`, and `isize` and `usize` are always bi-directional transmutable to // `isize_atomic_repr`, which also always implements `AtomicImpl`. #[allow(non_camel_case_types)] +#[cfg(not(testlib))] #[cfg(not(CONFIG_64BIT))] type isize_atomic_repr = i32; #[allow(non_camel_case_types)] +#[cfg(not(testlib))] #[cfg(CONFIG_64BIT)] type isize_atomic_repr = i64; +#[allow(non_camel_case_types)] +#[cfg(testlib)] +#[cfg(target_pointer_width = "32")] +type isize_atomic_repr = i32; +#[allow(non_camel_case_types)] +#[cfg(testlib)] +#[cfg(target_pointer_width = "64")] +type isize_atomic_repr = i64; + // Ensure size and alignment requirements are checked. static_assert!(size_of::() == size_of::()); static_assert!(align_of::() == align_of::()); diff --git a/rust/kernel/sync/refcount.rs b/rust/kernel/sync/refcount.rs index 19236a5bccdeb..6c7ae8b05a0b8 100644 --- a/rust/kernel/sync/refcount.rs +++ b/rust/kernel/sync/refcount.rs @@ -23,7 +23,8 @@ impl Refcount { /// Construct a new [`Refcount`] from an initial value. /// /// The initial value should be non-saturated. - #[inline] + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] pub fn new(value: i32) -> Self { build_assert!(value >= 0, "initial value saturated"); // SAFETY: There are no safety requirements for this FFI call. diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 49fad6de06740..cc907fb531bce 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -204,18 +204,6 @@ impl Task { self.0.get() } - /// Returns the group leader of the given task. - pub fn group_leader(&self) -> &Task { - // SAFETY: The group leader of a task never changes after initialization, so reading this - // field is not a data race. - let ptr = unsafe { *ptr::addr_of!((*self.as_ptr()).group_leader) }; - - // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, - // and given that a task has a reference to its group leader, we know it must be valid for - // the lifetime of the returned task reference. - unsafe { &*ptr.cast() } - } - /// Returns the PID of the given task. pub fn pid(&self) -> Pid { // SAFETY: The pid of a task never changes after initialization, so reading this field is @@ -345,6 +333,18 @@ impl CurrentTask { // `release_task()` call. Some(unsafe { PidNamespace::from_ptr(active_ns) }) } + + /// Returns the group leader of the current task. + pub fn group_leader(&self) -> &Task { + // SAFETY: The group leader of a task never changes while the task is running, and `self` + // is the current task, which is guaranteed running. + let ptr = unsafe { (*self.as_ptr()).group_leader }; + + // SAFETY: `current->group_leader` stays valid for at least the duration in which `current` + // is running, and the signature of this function ensures that the returned `&Task` can + // only be used while `current` is still valid, thus still running. + unsafe { &*ptr.cast() } + } } // SAFETY: The type invariants guarantee that `Task` is always refcounted. diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index dd553212836e0..0128808579973 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -1276,13 +1276,13 @@ pub const unsafe fn init_from_closure( /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. -#[expect(clippy::let_and_return)] pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl PinInit { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. let res = unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::())) }; // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a // cycle when computing the type returned by this function) + #[allow(clippy::let_and_return)] res } @@ -1292,13 +1292,13 @@ pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl Pin /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. -#[expect(clippy::let_and_return)] pub const unsafe fn cast_init(init: impl Init) -> impl Init { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. let res = unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::())) }; // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a // cycle when computing the type returned by this function) + #[allow(clippy::let_and_return)] res } diff --git a/rust/pin-init/src/macros.rs b/rust/pin-init/src/macros.rs index d6acf2cd291e0..fdf38b4fdbdc6 100644 --- a/rust/pin-init/src/macros.rs +++ b/rust/pin-init/src/macros.rs @@ -1310,6 +1310,10 @@ macro_rules! __init_internal { // return when an error/panic occurs. // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. // SAFETY: // - the project function does the correct field projection, // - the field has been initialized, @@ -1349,6 +1353,10 @@ macro_rules! __init_internal { // return when an error/panic occurs. unsafe { $crate::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. // SAFETY: // - the field is not structurally pinned, since the line above must compile, // - the field has been initialized, @@ -1389,6 +1397,10 @@ macro_rules! __init_internal { unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; } + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. #[allow(unused_variables)] // SAFETY: // - the field is not structurally pinned, since no `use_data` was required to create this @@ -1429,6 +1441,10 @@ macro_rules! __init_internal { // SAFETY: The memory at `slot` is uninitialized. unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; } + // NOTE: the field accessor ensures that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` below has the same requirement. // SAFETY: // - the project function does the correct field projection, // - the field has been initialized, diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c index da3a9f2091f55..1ba1927b548ee 100644 --- a/samples/ftrace/ftrace-direct-modify.c +++ b/samples/ftrace/ftrace-direct-modify.c @@ -176,8 +176,8 @@ asm ( " st.d $t0, $sp, 0\n" " st.d $ra, $sp, 8\n" " bl my_direct_func1\n" -" ld.d $t0, $sp, 0\n" -" ld.d $ra, $sp, 8\n" +" ld.d $ra, $sp, 0\n" +" ld.d $t0, $sp, 8\n" " addi.d $sp, $sp, 16\n" " jr $t0\n" " .size my_tramp1, .-my_tramp1\n" @@ -189,8 +189,8 @@ asm ( " st.d $t0, $sp, 0\n" " st.d $ra, $sp, 8\n" " bl my_direct_func2\n" -" ld.d $t0, $sp, 0\n" -" ld.d $ra, $sp, 8\n" +" ld.d $ra, $sp, 0\n" +" ld.d $t0, $sp, 8\n" " addi.d $sp, $sp, 16\n" " jr $t0\n" " .size my_tramp2, .-my_tramp2\n" diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c index 8f7986d698d87..7a7822dfeb50a 100644 --- a/samples/ftrace/ftrace-direct-multi-modify.c +++ b/samples/ftrace/ftrace-direct-multi-modify.c @@ -199,8 +199,8 @@ asm ( " move $a0, $t0\n" " bl my_direct_func1\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp1, .-my_tramp1\n" @@ -215,8 +215,8 @@ asm ( " move $a0, $t0\n" " bl my_direct_func2\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp2, .-my_tramp2\n" diff --git a/samples/ftrace/ftrace-direct-multi.c b/samples/ftrace/ftrace-direct-multi.c index db326c81a27dd..3fe6ddaf0b69f 100644 --- a/samples/ftrace/ftrace-direct-multi.c +++ b/samples/ftrace/ftrace-direct-multi.c @@ -131,8 +131,8 @@ asm ( " move $a0, $t0\n" " bl my_direct_func\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp, .-my_tramp\n" diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c index 3d0fa260332d4..bf2411aa6fd7a 100644 --- a/samples/ftrace/ftrace-direct-too.c +++ b/samples/ftrace/ftrace-direct-too.c @@ -143,8 +143,8 @@ asm ( " ld.d $a0, $sp, 0\n" " ld.d $a1, $sp, 8\n" " ld.d $a2, $sp, 16\n" -" ld.d $t0, $sp, 24\n" -" ld.d $ra, $sp, 32\n" +" ld.d $ra, $sp, 24\n" +" ld.d $t0, $sp, 32\n" " addi.d $sp, $sp, 48\n" " jr $t0\n" " .size my_tramp, .-my_tramp\n" diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c index 956834b0d19ac..5368c8c39cbb4 100644 --- a/samples/ftrace/ftrace-direct.c +++ b/samples/ftrace/ftrace-direct.c @@ -124,8 +124,8 @@ asm ( " st.d $ra, $sp, 16\n" " bl my_direct_func\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp, .-my_tramp\n" diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 55a683c39ed91..ea3063e558b1a 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -48,7 +48,7 @@ impl SampleDriver { // Select the test. bar.write8(index.0, Regs::TEST); - let offset = u32::from_le(bar.read32(Regs::OFFSET)) as usize; + let offset = bar.read32(Regs::OFFSET) as usize; let data = bar.read8(Regs::DATA); // Write `data` to `offset` to increase `count` by one. diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 52c08c4eb0b9a..0c838c467c764 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -356,7 +356,7 @@ $(obj)/%.o: $(obj)/%.rs FORCE quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ - command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@ + command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) --config-path $(srctree)/.rustfmt.toml $@ $(obj)/%.rsi: $(obj)/%.rs FORCE +$(call if_changed_dep,rustc_rsi_rs) @@ -527,18 +527,6 @@ ifneq ($(userprogs),) include $(srctree)/scripts/Makefile.userprogs endif -ifneq ($(need-dtbslist)$(dtb-y)$(dtb-)$(filter %.dtb %.dtb.o %.dtbo.o,$(targets)),) -include $(srctree)/scripts/Makefile.dtbs -endif - -# Build -# --------------------------------------------------------------------------- - -$(obj)/: $(if $(KBUILD_BUILTIN), $(targets-for-builtin)) \ - $(if $(KBUILD_MODULES), $(targets-for-modules)) \ - $(subdir-ym) $(always-y) - @: - # Single targets # --------------------------------------------------------------------------- @@ -568,6 +556,20 @@ FORCE: targets += $(filter-out $(single-subdir-goals), $(MAKECMDGOALS)) targets := $(filter-out $(PHONY), $(targets)) +# Now that targets is fully known, include dtb rules if needed +ifneq ($(need-dtbslist)$(dtb-y)$(dtb-)$(filter %.dtb %.dtb.o %.dtbo.o,$(targets)),) +include $(srctree)/scripts/Makefile.dtbs +endif + +# Build +# Needs to be after the include of Makefile.dtbs, which updates always-y +# --------------------------------------------------------------------------- + +$(obj)/: $(if $(KBUILD_BUILTIN), $(targets-for-builtin)) \ + $(if $(KBUILD_MODULES), $(targets-for-modules)) \ + $(subdir-ym) $(always-y) + @: + # Read all saved command lines and dependencies for the $(targets) we # may be building above, using $(if_changed{,_dep}). As an # optimization, we don't need to read them if the target does not diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst index 1628198f3e830..9ba45e5b32b18 100644 --- a/scripts/Makefile.modinst +++ b/scripts/Makefile.modinst @@ -100,7 +100,7 @@ endif # Don't stop modules_install even if we can't sign external modules. # ifeq ($(filter pkcs11:%, $(CONFIG_MODULE_SIG_KEY)),) -sig-key := $(if $(wildcard $(CONFIG_MODULE_SIG_KEY)),,$(srctree)/)$(CONFIG_MODULE_SIG_KEY) +sig-key := $(if $(wildcard $(CONFIG_MODULE_SIG_KEY)),,$(objtree)/)$(CONFIG_MODULE_SIG_KEY) else sig-key := $(CONFIG_MODULE_SIG_KEY) endif diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index cd788cac9d91d..276c3134a563a 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -113,7 +113,8 @@ vmlinux: vmlinux.unstripped FORCE # what kmod expects to parse. quiet_cmd_modules_builtin_modinfo = GEN $@ cmd_modules_builtin_modinfo = $(cmd_objcopy); \ - sed -i 's/\x00\+$$/\x00/g' $@ + sed -i 's/\x00\+$$/\x00/g' $@; \ + chmod -x $@ OBJCOPYFLAGS_modules.builtin.modinfo := -j .modinfo -O binary diff --git a/scripts/cc-can-link.sh b/scripts/cc-can-link.sh index 6efcead319898..58dc7dd6d5568 100755 --- a/scripts/cc-can-link.sh +++ b/scripts/cc-can-link.sh @@ -1,11 +1,11 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 -cat << "END" | $@ -x c - -o /dev/null >/dev/null 2>&1 +cat << "END" | $@ -Werror -Wl,--fatal-warnings -x c - -o /dev/null >/dev/null 2>&1 #include int main(void) { - printf(""); + printf("\n"); return 0; } END diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 92669904eecc7..9979bf9f87bcc 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -641,6 +641,7 @@ sub hash_show_words { Reviewed-by:| Reported-by:| Suggested-by:| + Assisted-by:| To:| Cc: )}; @@ -3087,6 +3088,15 @@ sub process { } } + # Assisted-by uses AGENT_NAME:MODEL_VERSION format, not email + if ($sign_off =~ /^Assisted-by:/i) { + if ($email !~ /^\S+:\S+/) { + WARN("BAD_SIGN_OFF", + "Assisted-by expects 'AGENT_NAME:MODEL_VERSION [TOOL1] [TOOL2]' format\n" . $herecurr); + } + next; + } + my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); my $suggested_email = format_email(($email_name, $name_comment, $email_address, $comment)); if ($suggested_email eq "") { diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l index 15d585c807980..1b129b118b0f7 100644 --- a/scripts/dtc/dtc-lexer.l +++ b/scripts/dtc/dtc-lexer.l @@ -39,8 +39,6 @@ extern bool treesource_error; #define DPRINT(fmt, ...) do { } while (0) #endif -static int dts_version = 1; - #define BEGIN_DEFAULT() DPRINT("\n"); \ BEGIN(V1); \ @@ -101,7 +99,6 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); <*>"/dts-v1/" { DPRINT("Keyword: /dts-v1/\n"); - dts_version = 1; BEGIN_DEFAULT(); return DT_V1; } diff --git a/scripts/faddr2line b/scripts/faddr2line index 1fa6beef9f978..477b6d2aa3179 100755 --- a/scripts/faddr2line +++ b/scripts/faddr2line @@ -107,14 +107,19 @@ find_dir_prefix() { run_readelf() { local objfile=$1 - local out=$(${READELF} --file-header --section-headers --symbols --wide $objfile) + local tmpfile + tmpfile=$(mktemp) + + ${READELF} --file-header --section-headers --symbols --wide "$objfile" > "$tmpfile" # This assumes that readelf first prints the file header, then the section headers, then the symbols. # Note: It seems that GNU readelf does not prefix section headers with the "There are X section headers" # line when multiple options are given, so let's also match with the "Section Headers:" line. - ELF_FILEHEADER=$(echo "${out}" | sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/q;p') - ELF_SECHEADERS=$(echo "${out}" | sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/,$p' | sed -n '/Symbol table .* contains [0-9]* entries:/q;p') - ELF_SYMS=$(echo "${out}" | sed -n '/Symbol table .* contains [0-9]* entries:/,$p') + ELF_FILEHEADER=$(sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/q;p' "$tmpfile") + ELF_SECHEADERS=$(sed -n '/There are [0-9]* section headers, starting at offset\|Section Headers:/,$p' "$tmpfile" | sed -n '/Symbol table .* contains [0-9]* entries:/q;p') + ELF_SYMS=$(sed -n '/Symbol table .* contains [0-9]* entries:/,$p' "$tmpfile") + + rm -f -- "$tmpfile" } check_vmlinux() { diff --git a/scripts/gendwarfksyms/dwarf.c b/scripts/gendwarfksyms/dwarf.c index 3538a7d9cb070..e76d732f5f602 100644 --- a/scripts/gendwarfksyms/dwarf.c +++ b/scripts/gendwarfksyms/dwarf.c @@ -750,6 +750,7 @@ static void process_enumerator_type(struct state *state, struct die *cache, Dwarf_Die *die) { bool overridden = false; + unsigned long override; Dwarf_Word value; if (stable) { @@ -761,7 +762,8 @@ static void process_enumerator_type(struct state *state, struct die *cache, return; overridden = kabi_get_enumerator_value( - state->expand.current_fqn, cache->fqn, &value); + state->expand.current_fqn, cache->fqn, &override); + value = override; } process_list_comma(state, cache); diff --git a/scripts/gendwarfksyms/symbols.c b/scripts/gendwarfksyms/symbols.c index ecddcb5ffcdfb..42cd27c9cec4f 100644 --- a/scripts/gendwarfksyms/symbols.c +++ b/scripts/gendwarfksyms/symbols.c @@ -3,6 +3,7 @@ * Copyright (C) 2024 Google LLC */ +#include #include "gendwarfksyms.h" #define SYMBOL_HASH_BITS 12 @@ -242,7 +243,7 @@ static void elf_for_each_global(int fd, elf_symbol_callback_t func, void *arg) error("elf_getdata failed: %s", elf_errmsg(-1)); if (shdr->sh_entsize != sym_size) - error("expected sh_entsize (%lu) to be %zu", + error("expected sh_entsize (%" PRIu64 ") to be %zu", shdr->sh_entsize, sym_size); nsyms = shdr->sh_size / shdr->sh_entsize; @@ -292,7 +293,7 @@ static void set_symbol_addr(struct symbol *sym, void *arg) hash_add(symbol_addrs, &sym->addr_hash, symbol_addr_hash(&sym->addr)); - debug("%s -> { %u, %lx }", sym->name, sym->addr.section, + debug("%s -> { %u, %" PRIx64 " }", sym->name, sym->addr.section, sym->addr.address); } else if (sym->addr.section != addr->section || sym->addr.address != addr->address) { diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index fc27f0cca752d..ff35c55b326c3 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -61,7 +61,6 @@ def append_sysroot_crate( display_name, deps, cfg=[], - edition="2021", ): append_crate( display_name, @@ -69,13 +68,37 @@ def append_sysroot_crate( deps, cfg, is_workspace_member=False, - edition=edition, + # Miguel Ojeda writes: + # + # > ... in principle even the sysroot crates may have different + # > editions. + # > + # > For instance, in the move to 2024, it seems all happened at once + # > in 1.87.0 in these upstream commits: + # > + # > 0e071c2c6a58 ("Migrate core to Rust 2024") + # > f505d4e8e380 ("Migrate alloc to Rust 2024") + # > 0b2489c226c3 ("Migrate proc_macro to Rust 2024") + # > 993359e70112 ("Migrate std to Rust 2024") + # > + # > But in the previous move to 2021, `std` moved in 1.59.0, while + # > the others in 1.60.0: + # > + # > b656384d8398 ("Update stdlib to the 2021 edition") + # > 06a1c14d52a8 ("Switch all libraries to the 2021 edition") + # + # Link: https://lore.kernel.org/all/CANiq72kd9bHdKaAm=8xCUhSHMy2csyVed69bOc4dXyFAW4sfuw@mail.gmail.com/ + # + # At the time of writing all rust versions we support build the + # sysroot crates with the same edition. We may need to relax this + # assumption if future edition moves span multiple rust versions. + edition=core_edition, ) # NB: sysroot crates reexport items from one another so setting up our transitive dependencies # here is important for ensuring that rust-analyzer can resolve symbols. The sources of truth # for this dependency graph are `(sysroot_src / crate / "Cargo.toml" for crate in crates)`. - append_sysroot_crate("core", [], cfg=crates_cfgs.get("core", []), edition=core_edition) + append_sysroot_crate("core", [], cfg=crates_cfgs.get("core", [])) append_sysroot_crate("alloc", ["core"]) append_sysroot_crate("std", ["alloc", "core"]) append_sysroot_crate("proc_macro", ["core", "std"]) @@ -83,7 +106,7 @@ def append_sysroot_crate( append_crate( "compiler_builtins", srctree / "rust" / "compiler_builtins.rs", - [], + ["core"], ) append_crate( @@ -102,7 +125,7 @@ def append_sysroot_crate( append_crate( "pin_init_internal", srctree / "rust" / "pin-init" / "internal" / "src" / "lib.rs", - [], + ["std", "proc_macro"], cfg=["kernel"], is_proc_macro=True, ) @@ -110,7 +133,7 @@ def append_sysroot_crate( append_crate( "pin_init", srctree / "rust" / "pin-init" / "src" / "lib.rs", - ["core", "pin_init_internal", "macros"], + ["core", "compiler_builtins", "pin_init_internal", "macros"], cfg=["kernel"], ) @@ -143,11 +166,24 @@ def append_crate_with_generated( append_crate_with_generated("uapi", ["core", "ffi", "pin_init"]) append_crate_with_generated("kernel", ["core", "macros", "build_error", "pin_init", "ffi", "bindings", "uapi"]) + scripts = srctree / "scripts" + makefile = (scripts / "Makefile").read_text() + for path in scripts.glob("*.rs"): + name = path.stem + if f"{name}-rust" not in makefile: + continue + append_crate( + name, + path, + ["std"], + ) + def is_root_crate(build_file, target): try: - return f"{target}.o" in open(build_file).read() + contents = build_file.read_text() except FileNotFoundError: return False + return f"{target}.o" in contents # Then, the rest outside of `rust/`. # @@ -158,7 +194,7 @@ def is_root_crate(build_file, target): for folder in extra_dirs: for path in folder.rglob("*.rs"): logging.info("Checking %s", path) - name = path.name.replace(".rs", "") + name = path.stem # Skip those that are not crate roots. if not is_root_crate(path.parent / "Makefile", name) and \ @@ -169,7 +205,7 @@ def is_root_crate(build_file, target): append_crate( name, path, - ["core", "kernel"], + ["core", "kernel", "pin_init"], cfg=cfg, ) @@ -192,9 +228,6 @@ def main(): level=logging.INFO if args.verbose else logging.WARNING ) - # Making sure that the `sysroot` and `sysroot_src` belong to the same toolchain. - assert args.sysroot in args.sysroot_src.parents - rust_project = { "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.core_edition), "sysroot": str(args.sysroot), diff --git a/scripts/kconfig/nconf-cfg.sh b/scripts/kconfig/nconf-cfg.sh index a20290b1a37d8..4d08453f9bdb7 100755 --- a/scripts/kconfig/nconf-cfg.sh +++ b/scripts/kconfig/nconf-cfg.sh @@ -6,8 +6,9 @@ set -eu cflags=$1 libs=$2 -PKG="ncursesw menuw panelw" -PKG2="ncurses menu panel" +# Keep library order for static linking (HOSTCC='cc -static') +PKG="menuw panelw ncursesw" +PKG2="menu panel ncurses" if [ -n "$(command -v ${HOSTPKG_CONFIG})" ]; then if ${HOSTPKG_CONFIG} --exists $PKG; then @@ -28,19 +29,19 @@ fi # find ncurses by pkg-config.) if [ -f /usr/include/ncursesw/ncurses.h ]; then echo -D_GNU_SOURCE -I/usr/include/ncursesw > ${cflags} - echo -lncursesw -lmenuw -lpanelw > ${libs} + echo -lmenuw -lpanelw -lncursesw > ${libs} exit 0 fi if [ -f /usr/include/ncurses/ncurses.h ]; then echo -D_GNU_SOURCE -I/usr/include/ncurses > ${cflags} - echo -lncurses -lmenu -lpanel > ${libs} + echo -lmenu -lpanel -lncurses > ${libs} exit 0 fi if [ -f /usr/include/ncurses.h ]; then echo -D_GNU_SOURCE > ${cflags} - echo -lncurses -lmenu -lpanel > ${libs} + echo -lmenu -lpanel -lncurses > ${libs} exit 0 fi diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index d9fe2bcbd39cc..a6bb180b13d65 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -116,6 +116,8 @@ sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) +WERROR_RETURN_CODE = 3 + DESC = """ Read C language source or header FILEs, extract embedded documentation comments, and print formatted documentation to standard output. @@ -176,7 +178,21 @@ def format(self, record): return logging.Formatter.format(self, record) def main(): - """Main program""" + """ + Main program. + + By default, the return value is: + + - 0: success or Python version is not compatible with + kernel-doc. If -Werror is not used, it will also + return 0 if there are issues at kernel-doc markups; + + - 1: an abnormal condition happened; + + - 2: argparse issued an error; + + - 3: -Werror is used, and one or more unfiltered parse warnings happened. + """ parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=DESC) @@ -323,16 +339,12 @@ def main(): if args.werror: print("%s warnings as errors" % error_count) # pylint: disable=C0209 - sys.exit(error_count) + sys.exit(WERROR_RETURN_CODE) if args.verbose: print("%s errors" % error_count) # pylint: disable=C0209 - if args.none: - sys.exit(0) - - sys.exit(error_count) - + sys.exit(0) # Call main method if __name__ == "__main__": diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 2376f180b1fa9..89d920e0b65ca 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -350,6 +350,7 @@ def dump_section(self, start_new=True): self.section = SECTION_DEFAULT self._contents = [] +python_warning = False class KernelDoc: """ @@ -383,9 +384,13 @@ def __init__(self, config, fname): # We need Python 3.7 for its "dicts remember the insertion # order" guarantee # - if sys.version_info.major == 3 and sys.version_info.minor < 7: + global python_warning + if (not python_warning and + sys.version_info.major == 3 and sys.version_info.minor < 7): + self.emit_msg(0, 'Python 3.7 or later is required for correct results') + python_warning = True def emit_msg(self, ln, msg, warning=True): """Emit a message""" diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index d3d00e85edf73..0470ba7c796d4 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -198,6 +198,9 @@ int main(void) DEVID(cpu_feature); DEVID_FIELD(cpu_feature, feature); + DEVID(mcb_device_id); + DEVID_FIELD(mcb_device_id, device); + DEVID(mei_cl_device_id); DEVID_FIELD(mei_cl_device_id, name); DEVID_FIELD(mei_cl_device_id, uuid); diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index b3333560b95ee..4e99393a35f15 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1110,6 +1110,14 @@ static void do_cpu_entry(struct module *mod, void *symval) module_alias_printf(mod, false, "cpu:type:*:feature:*%04X*", feature); } +/* Looks like: mcb:16zN */ +static void do_mcb_entry(struct module *mod, void *symval) +{ + DEF_FIELD(symval, mcb_device_id, device); + + module_alias_printf(mod, false, "mcb:16z%03d", device); +} + /* Looks like: mei:S:uuid:N:* */ static void do_mei_entry(struct module *mod, void *symval) { @@ -1444,6 +1452,7 @@ static const struct devtable devtable[] = { {"mipscdmm", SIZE_mips_cdmm_device_id, do_mips_cdmm_entry}, {"x86cpu", SIZE_x86_cpu_id, do_x86cpu_entry}, {"cpu", SIZE_cpu_feature, do_cpu_entry}, + {"mcb", SIZE_mcb_device_id, do_mcb_entry}, {"mei", SIZE_mei_cl_device_id, do_mei_entry}, {"rapidio", SIZE_rio_device_id, do_rio_entry}, {"ulpi", SIZE_ulpi_device_id, do_ulpi_entry}, diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 47c8aa2a69392..010c398f6a705 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -56,7 +56,7 @@ static bool allow_missing_ns_imports; static bool error_occurred; -static bool extra_warn; +static bool extra_warn __attribute__((unused)); bool target_is_big_endian; bool host_is_big_endian; @@ -602,6 +602,10 @@ static int ignore_undef_symbol(struct elf_info *info, const char *symname) /* Special register function linked on all modules during final link of .ko */ if (strstarts(symname, "_restgpr0_") || strstarts(symname, "_savegpr0_") || + strstarts(symname, "_restgpr1_") || + strstarts(symname, "_savegpr1_") || + strstarts(symname, "_restfpr_") || + strstarts(symname, "_savefpr_") || strstarts(symname, "_restvr_") || strstarts(symname, "_savevr_") || strcmp(symname, ".TOC.") == 0) diff --git a/scripts/package/install-extmod-build b/scripts/package/install-extmod-build index 054fdf45cc37a..f12e1ffe409eb 100755 --- a/scripts/package/install-extmod-build +++ b/scripts/package/install-extmod-build @@ -32,6 +32,10 @@ mkdir -p "${destdir}" echo tools/objtool/objtool fi + if is_enabled CONFIG_DEBUG_INFO_BTF_MODULES; then + echo tools/bpf/resolve_btfids/resolve_btfids + fi + echo Module.symvers echo "arch/${SRCARCH}/include/generated" echo include/config/auto.conf @@ -63,7 +67,7 @@ if [ "${CC}" != "${HOSTCC}" ]; then # Clear VPATH and srcroot because the source files reside in the output # directory. # shellcheck disable=SC2016 # $(MAKE) and $(build) will be expanded by Make - "${MAKE}" run-command KBUILD_RUN_COMMAND='+$(MAKE) HOSTCC='"${CC}"' VPATH= srcroot=. $(build)='"$(realpath --relative-to=. "${destdir}")"/scripts + "${MAKE}" run-command KBUILD_RUN_COMMAND='+$(MAKE) HOSTCC="'"${CC}"'" VPATH= srcroot=. $(build)='"$(realpath --relative-to=. "${destdir}")"/scripts rm -f "${destdir}/scripts/Kbuild" fi diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 98f206cb7c607..b3c956205af00 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -45,7 +45,19 @@ This package provides kernel headers and makefiles sufficient to build modules against the %{version} kernel package. %endif -%if %{with_debuginfo} +%if %{with_debuginfo_manual} +%package debuginfo +Summary: Debug information package for the Linux kernel +Group: Development/Debug +AutoReq: 0 +AutoProv: 1 +%description debuginfo +This package provides debug information for the kernel image and modules from the +%{version} package. +%define install_mod_strip 1 +%endif + +%if %{with_debuginfo_rpm} # list of debuginfo-related options taken from distribution kernel.spec # files %undefine _include_minidebuginfo @@ -59,11 +71,6 @@ against the %{version} kernel package. %global _missing_build_ids_terminate_build 1 %global _no_recompute_build_ids 1 %{debug_package} -%endif -# some (but not all) versions of rpmbuild emit %%debug_package with -# %%install. since we've already emitted it manually, that would cause -# a package redefinition error. ensure that doesn't happen -%define debug_package %{nil} # later, we make all modules executable so that find-debuginfo.sh strips # them up. but they don't actually need to be executable, so remove the @@ -74,6 +81,13 @@ against the %{version} kernel package. %{__os_install_post} \ find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \\\ | xargs --no-run-if-empty chmod u-x +%else +%define __spec_install_post /usr/lib/rpm/brp-compress || : +%endif +# some (but not all) versions of rpmbuild emit %%debug_package with +# %%install. since we've already emitted it manually, that would cause +# a package redefinition error. ensure that doesn't happen +%define debug_package %{nil} %prep %setup -q -n linux @@ -87,7 +101,7 @@ patch -p1 < %{SOURCE2} mkdir -p %{buildroot}/lib/modules/%{KERNELRELEASE} cp $(%{make} %{makeflags} -s image_name) %{buildroot}/lib/modules/%{KERNELRELEASE}/vmlinuz # DEPMOD=true makes depmod no-op. We do not package depmod-generated files. -%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} DEPMOD=true modules_install +%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} %{?install_mod_strip:INSTALL_MOD_STRIP=1} DEPMOD=true modules_install %{make} %{makeflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/lib/modules/%{KERNELRELEASE} cp .config %{buildroot}/lib/modules/%{KERNELRELEASE}/config @@ -118,22 +132,43 @@ ln -fns /usr/src/kernels/%{KERNELRELEASE} %{buildroot}/lib/modules/%{KERNELRELEA echo "%exclude /lib/modules/%{KERNELRELEASE}/build" } > %{buildroot}/kernel.list +%if 0%{with_debuginfo_manual}%{with_debuginfo_rpm} > 0 +# copying vmlinux directly to the debug directory means it will not get +# stripped (but its source paths will still be collected + fixed up) +mkdir -p %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} +cp vmlinux %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} +%endif + +%if %{with_debuginfo_rpm} # make modules executable so that find-debuginfo.sh strips them. this # will be undone later in %%__spec_install_post find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \ | xargs --no-run-if-empty chmod u+x +%endif -%if %{with_debuginfo} -# copying vmlinux directly to the debug directory means it will not get -# stripped (but its source paths will still be collected + fixed up) -mkdir -p %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} -cp vmlinux %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} +%if %{with_debuginfo_manual} +echo /usr/lib/debug/lib/modules/%{KERNELRELEASE}/vmlinux > %{buildroot}/debuginfo.list +while read -r mod; do + mod="${mod%.o}.ko" + dbg="%{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE}/kernel/${mod}" + buildid=$("${READELF:-readelf}" -n "${mod}" | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') + link="%{buildroot}/usr/lib/debug/.build-id/${buildid}.debug" + + mkdir -p "${dbg%/*}" "${link%/*}" + "${OBJCOPY:-objcopy}" --only-keep-debug "${mod}" "${dbg}" + ln -sf --relative "${dbg}" "${link}" + + echo "${dbg#%{buildroot}}" >> %{buildroot}/debuginfo.list + echo "${link#%{buildroot}}" >> %{buildroot}/debuginfo.list +done < modules.order %endif %clean rm -rf %{buildroot} +%if %{with_debuginfo_rpm} rm -f debugfiles.list debuglinks.list debugsourcefiles.list debugsources.list \ elfbins.list +%endif %post if [ -x /usr/bin/kernel-install ]; then @@ -172,3 +207,9 @@ fi /usr/src/kernels/%{KERNELRELEASE} /lib/modules/%{KERNELRELEASE}/build %endif + +%if %{with_debuginfo_manual} +%files -f %{buildroot}/debuginfo.list debuginfo +%defattr (-, root, root) +%exclude /debuginfo.list +%endif diff --git a/scripts/package/mkspec b/scripts/package/mkspec index c7375bfc25a9a..c604f8c174e2c 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -23,15 +23,47 @@ else echo '%define with_devel 0' fi +# use %{debug_package} machinery to generate -debuginfo +with_debuginfo_rpm=0 +# manually generate -debuginfo package +with_debuginfo_manual=0 # debuginfo package generation uses find-debuginfo.sh under the hood, # which only works on uncompressed modules that contain debuginfo if grep -q CONFIG_DEBUG_INFO=y include/config/auto.conf && (! grep -q CONFIG_MODULE_COMPRESS=y include/config/auto.conf) && (! grep -q CONFIG_DEBUG_INFO_SPLIT=y include/config/auto.conf); then -echo '%define with_debuginfo %{?_without_debuginfo: 0} %{?!_without_debuginfo: 1}' -else -echo '%define with_debuginfo 0' + # If module signing is enabled (which may be required to boot with + # lockdown enabled), the find-debuginfo.sh machinery cannot be used + # because the signatures will be stripped off the modules. However, due + # to an rpm bug in versions prior to 4.20.0 + # + # https://github.com/rpm-software-management/rpm/issues/3057 + # https://github.com/rpm-software-management/rpm/commit/49f906998f3cf1f4152162ca61ac0869251c380f + # + # We cannot provide our own debuginfo package because it does not listen + # to our custom files list, failing the build due to unpackaged files. + # Manually generate the debug info package if using rpm 4.20.0. If not + # using rpm 4.20.0, avoid generating a -debuginfo package altogether, + # as it is not safe. + if grep -q CONFIG_MODULE_SIG=y include/config/auto.conf; then + rpm_ver_str=$(rpm --version 2>/dev/null) + # Split the version on spaces + IFS=' ' + set -- $rpm_ver_str + if [ "${1:-}" = RPM -a "${2:-}" = version ]; then + IFS=. + set -- $3 + rpm_ver=$(( 1000000 * $1 + 10000 * $2 + 100 * $3 + ${4:-0} )) + if [ "$rpm_ver" -ge 4200000 ]; then + with_debuginfo_manual='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi + fi + else + with_debuginfo_rpm='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi fi +echo "%define with_debuginfo_manual $with_debuginfo_manual" +echo "%define with_debuginfo_rpm $with_debuginfo_rpm" cat<loaddata); + aa_put_i_loaddata(private->loaddata); kvfree(private); } @@ -153,6 +155,71 @@ static int aafs_show_path(struct seq_file *seq, struct dentry *dentry) return 0; } +static struct aa_ns *get_ns_common_ref(struct aa_common_ref *ref) +{ + if (ref) { + struct aa_label *reflabel = container_of(ref, struct aa_label, + count); + return aa_get_ns(labels_ns(reflabel)); + } + + return NULL; +} + +static struct aa_proxy *get_proxy_common_ref(struct aa_common_ref *ref) +{ + if (ref) + return aa_get_proxy(container_of(ref, struct aa_proxy, count)); + + return NULL; +} + +static struct aa_loaddata *get_loaddata_common_ref(struct aa_common_ref *ref) +{ + if (ref) + return aa_get_i_loaddata(container_of(ref, struct aa_loaddata, + count)); + return NULL; +} + +static void aa_put_common_ref(struct aa_common_ref *ref) +{ + if (!ref) + return; + + switch (ref->reftype) { + case REF_RAWDATA: + aa_put_i_loaddata(container_of(ref, struct aa_loaddata, + count)); + break; + case REF_PROXY: + aa_put_proxy(container_of(ref, struct aa_proxy, + count)); + break; + case REF_NS: + /* ns count is held on its unconfined label */ + aa_put_ns(labels_ns(container_of(ref, struct aa_label, count))); + break; + default: + AA_BUG(true, "unknown refcount type"); + break; + } +} + +static void aa_get_common_ref(struct aa_common_ref *ref) +{ + kref_get(&ref->count); +} + +static void aafs_evict(struct inode *inode) +{ + struct aa_common_ref *ref = inode->i_private; + + clear_inode(inode); + aa_put_common_ref(ref); + inode->i_private = (void *) IREF_POISON; +} + static void aafs_free_inode(struct inode *inode) { if (S_ISLNK(inode->i_mode)) @@ -162,6 +229,7 @@ static void aafs_free_inode(struct inode *inode) static const struct super_operations aafs_super_ops = { .statfs = simple_statfs, + .evict_inode = aafs_evict, .free_inode = aafs_free_inode, .show_path = aafs_show_path, }; @@ -262,7 +330,8 @@ static int __aafs_setup_d_inode(struct inode *dir, struct dentry *dentry, * aafs_remove(). Will return ERR_PTR on failure. */ static struct dentry *aafs_create(const char *name, umode_t mode, - struct dentry *parent, void *data, void *link, + struct dentry *parent, + struct aa_common_ref *data, void *link, const struct file_operations *fops, const struct inode_operations *iops) { @@ -299,6 +368,9 @@ static struct dentry *aafs_create(const char *name, umode_t mode, goto fail_dentry; inode_unlock(dir); + if (data) + aa_get_common_ref(data); + return dentry; fail_dentry: @@ -323,7 +395,8 @@ static struct dentry *aafs_create(const char *name, umode_t mode, * see aafs_create */ static struct dentry *aafs_create_file(const char *name, umode_t mode, - struct dentry *parent, void *data, + struct dentry *parent, + struct aa_common_ref *data, const struct file_operations *fops) { return aafs_create(name, mode, parent, data, NULL, fops, NULL); @@ -404,7 +477,8 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf, data->size = copy_size; if (copy_from_user(data->data, userbuf, copy_size)) { - aa_put_loaddata(data); + /* trigger free - don't need to put pcount */ + aa_put_i_loaddata(data); return ERR_PTR(-EFAULT); } @@ -412,7 +486,8 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf, } static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, - loff_t *pos, struct aa_ns *ns) + loff_t *pos, struct aa_ns *ns, + const struct cred *ocred) { struct aa_loaddata *data; struct aa_label *label; @@ -423,7 +498,7 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, /* high level check about policy management - fine grained in * below after unpack */ - error = aa_may_manage_policy(current_cred(), label, ns, mask); + error = aa_may_manage_policy(current_cred(), label, ns, ocred, mask); if (error) goto end_section; @@ -431,7 +506,10 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, error = PTR_ERR(data); if (!IS_ERR(data)) { error = aa_replace_profiles(ns, label, mask, data); - aa_put_loaddata(data); + /* put pcount, which will put count and free if no + * profiles referencing it. + */ + aa_put_profile_loaddata(data); } end_section: end_current_label_crit_section(label); @@ -443,8 +521,9 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, static ssize_t profile_load(struct file *f, const char __user *buf, size_t size, loff_t *pos) { - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); - int error = policy_update(AA_MAY_LOAD_POLICY, buf, size, pos, ns); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); + int error = policy_update(AA_MAY_LOAD_POLICY, buf, size, pos, ns, + f->f_cred); aa_put_ns(ns); @@ -460,9 +539,9 @@ static const struct file_operations aa_fs_profile_load = { static ssize_t profile_replace(struct file *f, const char __user *buf, size_t size, loff_t *pos) { - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); int error = policy_update(AA_MAY_LOAD_POLICY | AA_MAY_REPLACE_POLICY, - buf, size, pos, ns); + buf, size, pos, ns, f->f_cred); aa_put_ns(ns); return error; @@ -480,14 +559,14 @@ static ssize_t profile_remove(struct file *f, const char __user *buf, struct aa_loaddata *data; struct aa_label *label; ssize_t error; - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); label = begin_current_label_crit_section(); /* high level check about policy management - fine grained in * below after unpack */ error = aa_may_manage_policy(current_cred(), label, ns, - AA_MAY_REMOVE_POLICY); + f->f_cred, AA_MAY_REMOVE_POLICY); if (error) goto out; @@ -501,7 +580,7 @@ static ssize_t profile_remove(struct file *f, const char __user *buf, if (!IS_ERR(data)) { data->data[size] = 0; error = aa_remove_profiles(ns, label, data->data, size); - aa_put_loaddata(data); + aa_put_profile_loaddata(data); } out: end_current_label_crit_section(label); @@ -570,7 +649,7 @@ static int ns_revision_open(struct inode *inode, struct file *file) if (!rev) return -ENOMEM; - rev->ns = aa_get_ns(inode->i_private); + rev->ns = get_ns_common_ref(inode->i_private); if (!rev->ns) rev->ns = aa_get_current_ns(); file->private_data = rev; @@ -1056,7 +1135,7 @@ static const struct file_operations seq_profile_ ##NAME ##_fops = { \ static int seq_profile_open(struct inode *inode, struct file *file, int (*show)(struct seq_file *, void *)) { - struct aa_proxy *proxy = aa_get_proxy(inode->i_private); + struct aa_proxy *proxy = get_proxy_common_ref(inode->i_private); int error = single_open(file, show, proxy); if (error) { @@ -1248,18 +1327,17 @@ static const struct file_operations seq_rawdata_ ##NAME ##_fops = { \ static int seq_rawdata_open(struct inode *inode, struct file *file, int (*show)(struct seq_file *, void *)) { - struct aa_loaddata *data = __aa_get_loaddata(inode->i_private); + struct aa_loaddata *data = get_loaddata_common_ref(inode->i_private); int error; if (!data) - /* lost race this ent is being reaped */ return -ENOENT; error = single_open(file, show, data); if (error) { AA_BUG(file->private_data && ((struct seq_file *)file->private_data)->private); - aa_put_loaddata(data); + aa_put_i_loaddata(data); } return error; @@ -1270,7 +1348,7 @@ static int seq_rawdata_release(struct inode *inode, struct file *file) struct seq_file *seq = (struct seq_file *) file->private_data; if (seq) - aa_put_loaddata(seq->private); + aa_put_i_loaddata(seq->private); return single_release(inode, file); } @@ -1382,9 +1460,8 @@ static int rawdata_open(struct inode *inode, struct file *file) if (!aa_current_policy_view_capable(NULL)) return -EACCES; - loaddata = __aa_get_loaddata(inode->i_private); + loaddata = get_loaddata_common_ref(inode->i_private); if (!loaddata) - /* lost race: this entry is being reaped */ return -ENOENT; private = rawdata_f_data_alloc(loaddata->size); @@ -1409,7 +1486,7 @@ static int rawdata_open(struct inode *inode, struct file *file) return error; fail_private_alloc: - aa_put_loaddata(loaddata); + aa_put_i_loaddata(loaddata); return error; } @@ -1426,7 +1503,6 @@ static void remove_rawdata_dents(struct aa_loaddata *rawdata) for (i = 0; i < AAFS_LOADDATA_NDENTS; i++) { if (!IS_ERR_OR_NULL(rawdata->dents[i])) { - /* no refcounts on i_private */ aafs_remove(rawdata->dents[i]); rawdata->dents[i] = NULL; } @@ -1469,35 +1545,37 @@ int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata) return PTR_ERR(dir); rawdata->dents[AAFS_LOADDATA_DIR] = dir; - dent = aafs_create_file("abi", S_IFREG | 0444, dir, rawdata, + dent = aafs_create_file("abi", S_IFREG | 0444, dir, &rawdata->count, &seq_rawdata_abi_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_ABI] = dent; - dent = aafs_create_file("revision", S_IFREG | 0444, dir, rawdata, - &seq_rawdata_revision_fops); + dent = aafs_create_file("revision", S_IFREG | 0444, dir, + &rawdata->count, + &seq_rawdata_revision_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_REVISION] = dent; if (aa_g_hash_policy) { dent = aafs_create_file("sha256", S_IFREG | 0444, dir, - rawdata, &seq_rawdata_hash_fops); + &rawdata->count, + &seq_rawdata_hash_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_HASH] = dent; } dent = aafs_create_file("compressed_size", S_IFREG | 0444, dir, - rawdata, + &rawdata->count, &seq_rawdata_compressed_size_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_COMPRESSED_SIZE] = dent; - dent = aafs_create_file("raw_data", S_IFREG | 0444, - dir, rawdata, &rawdata_fops); + dent = aafs_create_file("raw_data", S_IFREG | 0444, dir, + &rawdata->count, &rawdata_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_DATA] = dent; @@ -1505,13 +1583,11 @@ int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata) rawdata->ns = aa_get_ns(ns); list_add(&rawdata->list, &ns->rawdata_list); - /* no refcount on inode rawdata */ return 0; fail: remove_rawdata_dents(rawdata); - return PTR_ERR(dent); } #endif /* CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ @@ -1535,13 +1611,10 @@ void __aafs_profile_rmdir(struct aa_profile *profile) __aafs_profile_rmdir(child); for (i = AAFS_PROF_SIZEOF - 1; i >= 0; --i) { - struct aa_proxy *proxy; if (!profile->dents[i]) continue; - proxy = d_inode(profile->dents[i])->i_private; aafs_remove(profile->dents[i]); - aa_put_proxy(proxy); profile->dents[i] = NULL; } } @@ -1575,14 +1648,7 @@ static struct dentry *create_profile_file(struct dentry *dir, const char *name, struct aa_profile *profile, const struct file_operations *fops) { - struct aa_proxy *proxy = aa_get_proxy(profile->label.proxy); - struct dentry *dent; - - dent = aafs_create_file(name, S_IFREG | 0444, dir, proxy, fops); - if (IS_ERR(dent)) - aa_put_proxy(proxy); - - return dent; + return aafs_create_file(name, S_IFREG | 0444, dir, &profile->label.proxy->count, fops); } #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY @@ -1628,7 +1694,8 @@ static const char *rawdata_get_link_base(struct dentry *dentry, struct delayed_call *done, const char *name) { - struct aa_proxy *proxy = inode->i_private; + struct aa_common_ref *ref = inode->i_private; + struct aa_proxy *proxy = container_of(ref, struct aa_proxy, count); struct aa_label *label; struct aa_profile *profile; char *target; @@ -1639,6 +1706,15 @@ static const char *rawdata_get_link_base(struct dentry *dentry, label = aa_get_label_rcu(&proxy->label); profile = labels_profile(label); + + /* rawdata can be null when aa_g_export_binary is unset during + * runtime and a profile is replaced + */ + if (!profile->rawdata) { + aa_put_label(label); + return ERR_PTR(-ENOENT); + } + depth = profile_depth(profile); target = gen_symlink_name(depth, profile->rawdata->name, name); aa_put_label(label); @@ -1761,27 +1837,24 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent) if (profile->rawdata) { if (aa_g_hash_policy) { dent = aafs_create("raw_sha256", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, - &rawdata_link_sha256_iops); + &profile->label.proxy->count, NULL, + NULL, &rawdata_link_sha256_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_HASH] = dent; } dent = aafs_create("raw_abi", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, + &profile->label.proxy->count, NULL, NULL, &rawdata_link_abi_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_ABI] = dent; dent = aafs_create("raw_data", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, + &profile->label.proxy->count, NULL, NULL, &rawdata_link_data_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_DATA] = dent; } #endif /*CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ @@ -1812,13 +1885,13 @@ static struct dentry *ns_mkdir_op(struct mnt_idmap *idmap, struct inode *dir, int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(current_cred(), label, NULL, + error = aa_may_manage_policy(current_cred(), label, NULL, NULL, AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return ERR_PTR(error); - parent = aa_get_ns(dir->i_private); + parent = get_ns_common_ref(dir->i_private); AA_BUG(d_inode(ns_subns_dir(parent)) != dir); /* we have to unlock and then relock to get locking order right @@ -1862,13 +1935,13 @@ static int ns_rmdir_op(struct inode *dir, struct dentry *dentry) int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(current_cred(), label, NULL, + error = aa_may_manage_policy(current_cred(), label, NULL, NULL, AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return error; - parent = aa_get_ns(dir->i_private); + parent = get_ns_common_ref(dir->i_private); /* rmdir calls the generic securityfs functions to remove files * from the apparmor dir. It is up to the apparmor ns locking * to avoid races. @@ -1938,27 +2011,6 @@ void __aafs_ns_rmdir(struct aa_ns *ns) __aa_fs_list_remove_rawdata(ns); - if (ns_subns_dir(ns)) { - sub = d_inode(ns_subns_dir(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subload(ns)) { - sub = d_inode(ns_subload(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subreplace(ns)) { - sub = d_inode(ns_subreplace(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subremove(ns)) { - sub = d_inode(ns_subremove(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subrevision(ns)) { - sub = d_inode(ns_subrevision(ns))->i_private; - aa_put_ns(sub); - } - for (i = AAFS_NS_SIZEOF - 1; i >= 0; --i) { aafs_remove(ns->dents[i]); ns->dents[i] = NULL; @@ -1983,40 +2035,40 @@ static int __aafs_ns_mkdir_entries(struct aa_ns *ns, struct dentry *dir) return PTR_ERR(dent); ns_subdata_dir(ns) = dent; - dent = aafs_create_file("revision", 0444, dir, ns, + dent = aafs_create_file("revision", 0444, dir, + &ns->unconfined->label.count, &aa_fs_ns_revision_fops); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subrevision(ns) = dent; - dent = aafs_create_file(".load", 0640, dir, ns, - &aa_fs_profile_load); + dent = aafs_create_file(".load", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_load); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subload(ns) = dent; - dent = aafs_create_file(".replace", 0640, dir, ns, - &aa_fs_profile_replace); + dent = aafs_create_file(".replace", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_replace); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subreplace(ns) = dent; - dent = aafs_create_file(".remove", 0640, dir, ns, - &aa_fs_profile_remove); + dent = aafs_create_file(".remove", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_remove); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subremove(ns) = dent; /* use create_dentry so we can supply private data */ - dent = aafs_create("namespaces", S_IFDIR | 0755, dir, ns, NULL, NULL, - &ns_dir_inode_operations); + dent = aafs_create("namespaces", S_IFDIR | 0755, dir, + &ns->unconfined->label.count, + NULL, NULL, &ns_dir_inode_operations); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subns_dir(ns) = dent; return 0; diff --git a/security/apparmor/file.c b/security/apparmor/file.c index c758204028780..7de23e85cd5d0 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -154,8 +154,12 @@ static int path_name(const char *op, const struct cred *subj_cred, const char *info = NULL; int error; - error = aa_path_name(path, flags, buffer, name, &info, - labels_profile(label)->disconnected); + /* don't reaudit files closed during inheritance */ + if (unlikely(path->dentry == aa_null.dentry)) + error = -EACCES; + else + error = aa_path_name(path, flags, buffer, name, &info, + labels_profile(label)->disconnected); if (error) { fn_for_each_confined(label, profile, aa_audit_file(subj_cred, @@ -578,6 +582,9 @@ static bool __unix_needs_revalidation(struct file *file, struct aa_label *label, return false; if (request & NET_PEER_MASK) return false; + /* sock and sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return false; if (sock->sk->sk_family == PF_UNIX) { struct aa_sk_ctx *ctx = aa_sock(sock->sk); @@ -613,6 +620,10 @@ int aa_file_perm(const char *op, const struct cred *subj_cred, AA_BUG(!label); AA_BUG(!file); + /* don't reaudit files closed during inheritance */ + if (unlikely(file->f_path.dentry == aa_null.dentry)) + return -EACCES; + fctx = file_ctx(file); rcu_read_lock(); diff --git a/security/apparmor/include/label.h b/security/apparmor/include/label.h index c0812dbc1b5b0..335f21930702a 100644 --- a/security/apparmor/include/label.h +++ b/security/apparmor/include/label.h @@ -102,7 +102,7 @@ enum label_flags { struct aa_label; struct aa_proxy { - struct kref count; + struct aa_common_ref count; struct aa_label __rcu *label; }; @@ -125,7 +125,7 @@ struct label_it { * vec: vector of profiles comprising the compound label */ struct aa_label { - struct kref count; + struct aa_common_ref count; struct rb_node node; struct rcu_head rcu; struct aa_proxy *proxy; @@ -357,7 +357,7 @@ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, */ static inline struct aa_label *__aa_get_label(struct aa_label *l) { - if (l && kref_get_unless_zero(&l->count)) + if (l && kref_get_unless_zero(&l->count.count)) return l; return NULL; @@ -366,7 +366,7 @@ static inline struct aa_label *__aa_get_label(struct aa_label *l) static inline struct aa_label *aa_get_label(struct aa_label *l) { if (l) - kref_get(&(l->count)); + kref_get(&(l->count.count)); return l; } @@ -386,7 +386,7 @@ static inline struct aa_label *aa_get_label_rcu(struct aa_label __rcu **l) rcu_read_lock(); do { c = rcu_dereference(*l); - } while (c && !kref_get_unless_zero(&c->count)); + } while (c && !kref_get_unless_zero(&c->count.count)); rcu_read_unlock(); return c; @@ -426,7 +426,7 @@ static inline struct aa_label *aa_get_newest_label(struct aa_label *l) static inline void aa_put_label(struct aa_label *l) { if (l) - kref_put(&l->count, aa_label_kref); + kref_put(&l->count.count, aa_label_kref); } /* wrapper fn to indicate semantics of the check */ @@ -443,7 +443,7 @@ void aa_proxy_kref(struct kref *kref); static inline struct aa_proxy *aa_get_proxy(struct aa_proxy *proxy) { if (proxy) - kref_get(&(proxy->count)); + kref_get(&(proxy->count.count)); return proxy; } @@ -451,7 +451,7 @@ static inline struct aa_proxy *aa_get_proxy(struct aa_proxy *proxy) static inline void aa_put_proxy(struct aa_proxy *proxy) { if (proxy) - kref_put(&proxy->count, aa_proxy_kref); + kref_put(&proxy->count.count, aa_proxy_kref); } void __aa_proxy_redirect(struct aa_label *orig, struct aa_label *new); diff --git a/security/apparmor/include/lib.h b/security/apparmor/include/lib.h index 444197075fd6b..26df19c1df4f8 100644 --- a/security/apparmor/include/lib.h +++ b/security/apparmor/include/lib.h @@ -85,6 +85,18 @@ void aa_info_message(const char *str); /* Security blob offsets */ extern struct lsm_blob_sizes apparmor_blob_sizes; +enum reftype { + REF_NS, + REF_PROXY, + REF_RAWDATA, +}; + +/* common reference count used by data the shows up in aafs */ +struct aa_common_ref { + struct kref count; + enum reftype reftype; +}; + /** * aa_strneq - compare null terminated @str to a non null terminated substring * @str: a null terminated string diff --git a/security/apparmor/include/match.h b/security/apparmor/include/match.h index 1fbe82f5021b1..7accb1c39849a 100644 --- a/security/apparmor/include/match.h +++ b/security/apparmor/include/match.h @@ -104,16 +104,18 @@ struct aa_dfa { struct table_header *tables[YYTD_ID_TSIZE]; }; -#define byte_to_byte(X) (X) - #define UNPACK_ARRAY(TABLE, BLOB, LEN, TTYPE, BTYPE, NTOHX) \ do { \ typeof(LEN) __i; \ TTYPE *__t = (TTYPE *) TABLE; \ BTYPE *__b = (BTYPE *) BLOB; \ - for (__i = 0; __i < LEN; __i++) { \ - __t[__i] = NTOHX(__b[__i]); \ - } \ + BUILD_BUG_ON(sizeof(TTYPE) != sizeof(BTYPE)); \ + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) \ + memcpy(__t, __b, (LEN) * sizeof(BTYPE)); \ + else /* copy & convert from big-endian */ \ + for (__i = 0; __i < LEN; __i++) { \ + __t[__i] = NTOHX(&__b[__i]); \ + } \ } while (0) static inline size_t table_size(size_t len, size_t el_size) @@ -183,6 +185,7 @@ static inline void aa_put_dfa(struct aa_dfa *dfa) #define MATCH_FLAG_DIFF_ENCODE 0x80000000 #define MARK_DIFF_ENCODE 0x40000000 #define MATCH_FLAG_OOB_TRANSITION 0x20000000 +#define MARK_DIFF_ENCODE_VERIFIED 0x10000000 #define MATCH_FLAGS_MASK 0xff000000 #define MATCH_FLAGS_VALID (MATCH_FLAG_DIFF_ENCODE | MATCH_FLAG_OOB_TRANSITION) #define MATCH_FLAGS_INVALID (MATCH_FLAGS_MASK & ~MATCH_FLAGS_VALID) diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 4c50875c9d13e..bf105ae9019d3 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -355,7 +355,7 @@ static inline bool profile_mediates_safe(struct aa_profile *profile, static inline struct aa_profile *aa_get_profile(struct aa_profile *p) { if (p) - kref_get(&(p->label.count)); + kref_get(&(p->label.count.count)); return p; } @@ -369,7 +369,7 @@ static inline struct aa_profile *aa_get_profile(struct aa_profile *p) */ static inline struct aa_profile *aa_get_profile_not0(struct aa_profile *p) { - if (p && kref_get_unless_zero(&p->label.count)) + if (p && kref_get_unless_zero(&p->label.count.count)) return p; return NULL; @@ -389,7 +389,7 @@ static inline struct aa_profile *aa_get_profile_rcu(struct aa_profile __rcu **p) rcu_read_lock(); do { c = rcu_dereference(*p); - } while (c && !kref_get_unless_zero(&c->label.count)); + } while (c && !kref_get_unless_zero(&c->label.count.count)); rcu_read_unlock(); return c; @@ -402,7 +402,7 @@ static inline struct aa_profile *aa_get_profile_rcu(struct aa_profile __rcu **p) static inline void aa_put_profile(struct aa_profile *p) { if (p) - kref_put(&p->label.count, aa_label_kref); + kref_put(&p->label.count.count, aa_label_kref); } static inline int AUDIT_MODE(struct aa_profile *profile) @@ -419,7 +419,7 @@ bool aa_policy_admin_capable(const struct cred *subj_cred, struct aa_label *label, struct aa_ns *ns); int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, struct aa_ns *ns, - u32 mask); + const struct cred *ocred, u32 mask); bool aa_current_policy_view_capable(struct aa_ns *ns); bool aa_current_policy_admin_capable(struct aa_ns *ns); diff --git a/security/apparmor/include/policy_ns.h b/security/apparmor/include/policy_ns.h index d646070fd966b..cc6e841518120 100644 --- a/security/apparmor/include/policy_ns.h +++ b/security/apparmor/include/policy_ns.h @@ -18,6 +18,8 @@ #include "label.h" #include "policy.h" +/* Match max depth of user namespaces */ +#define MAX_NS_DEPTH 32 /* struct aa_ns_acct - accounting of profiles in namespace * @max_size: maximum space allowed for all profiles in namespace diff --git a/security/apparmor/include/policy_unpack.h b/security/apparmor/include/policy_unpack.h index a6f4611ee50cf..e5a95dc4da1f7 100644 --- a/security/apparmor/include/policy_unpack.h +++ b/security/apparmor/include/policy_unpack.h @@ -87,17 +87,29 @@ struct aa_ext { u32 version; }; -/* - * struct aa_loaddata - buffer of policy raw_data set +/* struct aa_loaddata - buffer of policy raw_data set + * @count: inode/filesystem refcount - use aa_get_i_loaddata() + * @pcount: profile refcount - use aa_get_profile_loaddata() + * @list: list the loaddata is on + * @work: used to do a delayed cleanup + * @dents: refs to dents created in aafs + * @ns: the namespace this loaddata was loaded into + * @name: + * @size: the size of the data that was loaded + * @compressed_size: the size of the data when it is compressed + * @revision: unique revision count that this data was loaded as + * @abi: the abi number the loaddata uses + * @hash: a hash of the loaddata, used to help dedup data * - * there is no loaddata ref for being on ns list, nor a ref from - * d_inode(@dentry) when grab a ref from these, @ns->lock must be held - * && __aa_get_loaddata() needs to be used, and the return value - * checked, if NULL the loaddata is already being reaped and should be - * considered dead. + * There is no loaddata ref for being on ns->rawdata_list, so + * @ns->lock must be held when walking the list. Dentries and + * inode opens hold refs on @count; profiles hold refs on @pcount. + * When the last @pcount drops, do_ploaddata_rmfs() removes the + * fs entries and drops the associated @count ref. */ struct aa_loaddata { - struct kref count; + struct aa_common_ref count; + struct kref pcount; struct list_head list; struct work_struct work; struct dentry *dents[AAFS_LOADDATA_NDENTS]; @@ -119,50 +131,53 @@ struct aa_loaddata { int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, const char **ns); /** - * __aa_get_loaddata - get a reference count to uncounted data reference + * aa_get_loaddata - get a reference count from a counted data reference * @data: reference to get a count on * - * Returns: pointer to reference OR NULL if race is lost and reference is - * being repeated. - * Requires: @data->ns->lock held, and the return code MUST be checked - * - * Use only from inode->i_private and @data->list found references + * Returns: pointer to reference + * Requires: @data to have a valid reference count on it. It is a bug + * if the race to reap can be encountered when it is used. */ static inline struct aa_loaddata * -__aa_get_loaddata(struct aa_loaddata *data) +aa_get_i_loaddata(struct aa_loaddata *data) { - if (data && kref_get_unless_zero(&(data->count))) - return data; - return NULL; + if (data) + kref_get(&(data->count.count)); + return data; } + /** - * aa_get_loaddata - get a reference count from a counted data reference + * aa_get_profile_loaddata - get a profile reference count on loaddata * @data: reference to get a count on * - * Returns: point to reference - * Requires: @data to have a valid reference count on it. It is a bug - * if the race to reap can be encountered when it is used. + * Returns: pointer to reference + * Requires: @data to have a valid reference count on it. */ static inline struct aa_loaddata * -aa_get_loaddata(struct aa_loaddata *data) +aa_get_profile_loaddata(struct aa_loaddata *data) { - struct aa_loaddata *tmp = __aa_get_loaddata(data); - - AA_BUG(data && !tmp); - - return tmp; + if (data) + kref_get(&(data->pcount)); + return data; } void __aa_loaddata_update(struct aa_loaddata *data, long revision); bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r); void aa_loaddata_kref(struct kref *kref); +void aa_ploaddata_kref(struct kref *kref); struct aa_loaddata *aa_loaddata_alloc(size_t size); -static inline void aa_put_loaddata(struct aa_loaddata *data) +static inline void aa_put_i_loaddata(struct aa_loaddata *data) +{ + if (data) + kref_put(&data->count.count, aa_loaddata_kref); +} + +static inline void aa_put_profile_loaddata(struct aa_loaddata *data) { if (data) - kref_put(&data->count, aa_loaddata_kref); + kref_put(&data->pcount, aa_ploaddata_kref); } #if IS_ENABLED(CONFIG_KUNIT) diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 913678f199c35..3bec1e33815e2 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -52,7 +52,8 @@ static void free_proxy(struct aa_proxy *proxy) void aa_proxy_kref(struct kref *kref) { - struct aa_proxy *proxy = container_of(kref, struct aa_proxy, count); + struct aa_proxy *proxy = container_of(kref, struct aa_proxy, + count.count); free_proxy(proxy); } @@ -63,7 +64,8 @@ struct aa_proxy *aa_alloc_proxy(struct aa_label *label, gfp_t gfp) new = kzalloc(sizeof(struct aa_proxy), gfp); if (new) { - kref_init(&new->count); + kref_init(&new->count.count); + new->count.reftype = REF_PROXY; rcu_assign_pointer(new->label, aa_get_label(label)); } return new; @@ -375,7 +377,8 @@ static void label_free_rcu(struct rcu_head *head) void aa_label_kref(struct kref *kref) { - struct aa_label *label = container_of(kref, struct aa_label, count); + struct aa_label *label = container_of(kref, struct aa_label, + count.count); struct aa_ns *ns = labels_ns(label); if (!ns) { @@ -412,7 +415,8 @@ bool aa_label_init(struct aa_label *label, int size, gfp_t gfp) label->size = size; /* doesn't include null */ label->vec[size] = NULL; /* null terminate */ - kref_init(&label->count); + kref_init(&label->count.count); + label->count.reftype = REF_NS; /* for aafs purposes */ RB_CLEAR_NODE(&label->node); return true; @@ -1278,7 +1282,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, * @request: permissions to request * @perms: perms struct to set * - * Returns: 0 on success else ERROR + * Returns: state match stopped at or DFA_NOMATCH if aborted early * * For the label A//&B//&C this does the perm match for A//&B//&C * @perms should be preinitialized with allperms OR a previous permission @@ -1305,7 +1309,7 @@ static int label_compound_match(struct aa_profile *profile, /* no component visible */ *perms = allperms; - return 0; + return state; next: label_for_each_cont(i, label, tp) { @@ -1317,15 +1321,11 @@ static int label_compound_match(struct aa_profile *profile, goto fail; } *perms = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, perms); - if ((perms->allow & request) != request) - return -EACCES; - - return 0; + return state; fail: *perms = nullperms; - return state; + return DFA_NOMATCH; } /** @@ -1338,7 +1338,7 @@ static int label_compound_match(struct aa_profile *profile, * @request: permissions to request * @perms: an initialized perms struct to add accumulation to * - * Returns: 0 on success else ERROR + * Returns: the state the match finished in, may be the none matching state * * For the label A//&B//&C this does the perm match for each of A and B and C * @perms should be preinitialized with allperms OR a previous permission @@ -1366,11 +1366,10 @@ static int label_components_match(struct aa_profile *profile, } /* no subcomponents visible - no change in perms */ - return 0; + return state; next: tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { if (!aa_ns_visible(profile->ns, tp->ns, subns)) @@ -1379,18 +1378,17 @@ static int label_components_match(struct aa_profile *profile, if (!state) goto fail; tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); } if ((perms->allow & request) != request) - return -EACCES; + return DFA_NOMATCH; - return 0; + return state; fail: *perms = nullperms; - return -EACCES; + return DFA_NOMATCH; } /** @@ -1409,11 +1407,12 @@ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, struct aa_label *label, aa_state_t state, bool subns, u32 request, struct aa_perms *perms) { - int error = label_compound_match(profile, rules, label, state, subns, - request, perms); - if (!error) - return error; + aa_state_t tmp = label_compound_match(profile, rules, label, state, subns, + request, perms); + if ((perms->allow & request) == request) + return tmp; + /* failed compound_match try component matches */ *perms = allperms; return label_components_match(profile, rules, label, state, subns, request, perms); diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index b3f7a3258a2cf..13c9bfdf65ff5 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -519,33 +519,26 @@ static void apparmor_file_free_security(struct file *file) aa_put_label(rcu_access_pointer(ctx->label)); } -static int common_file_perm(const char *op, struct file *file, u32 mask, - bool in_atomic) +static int common_file_perm(const char *op, struct file *file, u32 mask) { struct aa_label *label; int error = 0; - bool needput; - - /* don't reaudit files closed during inheritance */ - if (unlikely(file->f_path.dentry == aa_null.dentry)) - return -EACCES; - label = __begin_current_label_crit_section(&needput); - error = aa_file_perm(op, current_cred(), label, file, mask, in_atomic); - __end_current_label_crit_section(label, needput); + label = begin_current_label_crit_section(); + error = aa_file_perm(op, current_cred(), label, file, mask, false); + end_current_label_crit_section(label); return error; } static int apparmor_file_receive(struct file *file) { - return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file), - false); + return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file)); } static int apparmor_file_permission(struct file *file, int mask) { - return common_file_perm(OP_FPERM, file, mask, false); + return common_file_perm(OP_FPERM, file, mask); } static int apparmor_file_lock(struct file *file, unsigned int cmd) @@ -555,11 +548,11 @@ static int apparmor_file_lock(struct file *file, unsigned int cmd) if (cmd == F_WRLCK) mask |= MAY_WRITE; - return common_file_perm(OP_FLOCK, file, mask, false); + return common_file_perm(OP_FLOCK, file, mask); } static int common_mmap(const char *op, struct file *file, unsigned long prot, - unsigned long flags, bool in_atomic) + unsigned long flags) { int mask = 0; @@ -577,21 +570,20 @@ static int common_mmap(const char *op, struct file *file, unsigned long prot, if (prot & PROT_EXEC) mask |= AA_EXEC_MMAP; - return common_file_perm(op, file, mask, in_atomic); + return common_file_perm(op, file, mask); } static int apparmor_mmap_file(struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags) { - return common_mmap(OP_FMMAP, file, prot, flags, GFP_ATOMIC); + return common_mmap(OP_FMMAP, file, prot, flags); } static int apparmor_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { return common_mmap(OP_FMPROT, vma->vm_file, prot, - !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0, - false); + !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0); } #ifdef CONFIG_IO_URING @@ -2144,7 +2136,8 @@ char *aa_get_buffer(bool in_atomic) if (!list_empty(&cache->head)) { aa_buf = list_first_entry(&cache->head, union aa_buffer, list); list_del(&aa_buf->list); - cache->hold--; + if (cache->hold) + cache->hold--; cache->count--; put_cpu_ptr(&aa_local_buffers); return &aa_buf->buffer[0]; diff --git a/security/apparmor/match.c b/security/apparmor/match.c index c5a91600842a1..0de249725efbf 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "include/lib.h" #include "include/match.h" @@ -42,11 +43,11 @@ static struct table_header *unpack_table(char *blob, size_t bsize) /* loaded td_id's start at 1, subtract 1 now to avoid doing * it every time we use td_id as an index */ - th.td_id = be16_to_cpu(*(__be16 *) (blob)) - 1; + th.td_id = get_unaligned_be16(blob) - 1; if (th.td_id > YYTD_ID_MAX) goto out; - th.td_flags = be16_to_cpu(*(__be16 *) (blob + 2)); - th.td_lolen = be32_to_cpu(*(__be32 *) (blob + 8)); + th.td_flags = get_unaligned_be16(blob + 2); + th.td_lolen = get_unaligned_be32(blob + 8); blob += sizeof(struct table_header); if (!(th.td_flags == YYTD_DATA16 || th.td_flags == YYTD_DATA32 || @@ -66,14 +67,13 @@ static struct table_header *unpack_table(char *blob, size_t bsize) table->td_flags = th.td_flags; table->td_lolen = th.td_lolen; if (th.td_flags == YYTD_DATA8) - UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u8, u8, byte_to_byte); + memcpy(table->td_data, blob, th.td_lolen); else if (th.td_flags == YYTD_DATA16) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u16, __be16, be16_to_cpu); + u16, __be16, get_unaligned_be16); else if (th.td_flags == YYTD_DATA32) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u32, __be32, be32_to_cpu); + u32, __be32, get_unaligned_be32); else goto fail; /* if table was vmalloced make sure the page tables are synced @@ -160,9 +160,10 @@ static int verify_dfa(struct aa_dfa *dfa) if (state_count == 0) goto out; for (i = 0; i < state_count; i++) { - if (!(BASE_TABLE(dfa)[i] & MATCH_FLAG_DIFF_ENCODE) && - (DEFAULT_TABLE(dfa)[i] >= state_count)) + if (DEFAULT_TABLE(dfa)[i] >= state_count) { + pr_err("AppArmor DFA default state out of bounds"); goto out; + } if (BASE_TABLE(dfa)[i] & MATCH_FLAGS_INVALID) { pr_err("AppArmor DFA state with invalid match flags"); goto out; @@ -201,16 +202,31 @@ static int verify_dfa(struct aa_dfa *dfa) size_t j, k; for (j = i; - (BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE) && - !(BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE); + ((BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE) && + !(BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE_VERIFIED)); j = k) { + if (BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE) + /* loop in current chain */ + goto out; k = DEFAULT_TABLE(dfa)[j]; if (j == k) + /* self loop */ goto out; - if (k < j) - break; /* already verified */ BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE; } + /* move mark to verified */ + for (j = i; + (BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE); + j = k) { + k = DEFAULT_TABLE(dfa)[j]; + if (j < i) + /* jumps to state/chain that has been + * verified + */ + break; + BASE_TABLE(dfa)[j] &= ~MARK_DIFF_ENCODE; + BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE_VERIFIED; + } } error = 0; @@ -313,14 +329,14 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) if (size < sizeof(struct table_set_header)) goto fail; - if (ntohl(*(__be32 *) data) != YYTH_MAGIC) + if (get_unaligned_be32(data) != YYTH_MAGIC) goto fail; - hsize = ntohl(*(__be32 *) (data + 4)); + hsize = get_unaligned_be32(data + 4); if (size < hsize) goto fail; - dfa->flags = ntohs(*(__be16 *) (data + 12)); + dfa->flags = get_unaligned_be16(data + 12); if (dfa->flags & ~(YYTH_FLAGS)) goto fail; @@ -329,7 +345,7 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) * if (dfa->flags & YYTH_FLAGS_OOB_TRANS) { * if (hsize < 16 + 4) * goto fail; - * dfa->max_oob = ntol(*(__be32 *) (data + 16)); + * dfa->max_oob = get_unaligned_be32(data + 16); * if (dfa->max <= MAX_OOB_SUPPORTED) { * pr_err("AppArmor DFA OOB greater than supported\n"); * goto fail; @@ -463,13 +479,18 @@ aa_state_t aa_dfa_match_len(struct aa_dfa *dfa, aa_state_t start, if (dfa->tables[YYTD_ID_EC]) { /* Equivalence class table defined */ u8 *equiv = EQUIV_TABLE(dfa); - for (; len; len--) - match_char(state, def, base, next, check, - equiv[(u8) *str++]); + for (; len; len--) { + u8 c = equiv[(u8) *str]; + + match_char(state, def, base, next, check, c); + str++; + } } else { /* default is direct to next state */ - for (; len; len--) - match_char(state, def, base, next, check, (u8) *str++); + for (; len; len--) { + match_char(state, def, base, next, check, (u8) *str); + str++; + } } return state; @@ -503,13 +524,18 @@ aa_state_t aa_dfa_match(struct aa_dfa *dfa, aa_state_t start, const char *str) /* Equivalence class table defined */ u8 *equiv = EQUIV_TABLE(dfa); /* default is direct to next state */ - while (*str) - match_char(state, def, base, next, check, - equiv[(u8) *str++]); + while (*str) { + u8 c = equiv[(u8) *str]; + + match_char(state, def, base, next, check, c); + str++; + } } else { /* default is direct to next state */ - while (*str) - match_char(state, def, base, next, check, (u8) *str++); + while (*str) { + match_char(state, def, base, next, check, (u8) *str); + str++; + } } return state; diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 45cf25605c345..44c04102062f3 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -326,8 +326,10 @@ int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label, struct socket *sock = (struct socket *) file->private_data; AA_BUG(!label); - AA_BUG(!sock); - AA_BUG(!sock->sk); + + /* sock && sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return 0; if (sock->sk->sk_family == PF_UNIX) return aa_unix_file_perm(subj_cred, label, op, request, file); diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 50d5345ff5cba..b92db1b2f26e2 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -183,19 +183,43 @@ static void __list_remove_profile(struct aa_profile *profile) } /** - * __remove_profile - remove old profile, and children - * @profile: profile to be replaced (NOT NULL) + * __remove_profile - remove profile, and children + * @profile: profile to be removed (NOT NULL) * * Requires: namespace list lock be held, or list not be shared */ static void __remove_profile(struct aa_profile *profile) { + struct aa_profile *curr, *to_remove; + AA_BUG(!profile); AA_BUG(!profile->ns); AA_BUG(!mutex_is_locked(&profile->ns->lock)); /* release any children lists first */ - __aa_profile_list_release(&profile->base.profiles); + if (!list_empty(&profile->base.profiles)) { + curr = list_first_entry(&profile->base.profiles, struct aa_profile, base.list); + + while (curr != profile) { + + while (!list_empty(&curr->base.profiles)) + curr = list_first_entry(&curr->base.profiles, + struct aa_profile, base.list); + + to_remove = curr; + if (!list_is_last(&to_remove->base.list, + &aa_deref_parent(curr)->base.profiles)) + curr = list_next_entry(to_remove, base.list); + else + curr = aa_deref_parent(curr); + + /* released by free_profile */ + aa_label_remove(&to_remove->label); + __aafs_profile_rmdir(to_remove); + __list_remove_profile(to_remove); + } + } + /* released by free_profile */ aa_label_remove(&profile->label); __aafs_profile_rmdir(profile); @@ -312,7 +336,7 @@ void aa_free_profile(struct aa_profile *profile) } kfree_sensitive(profile->hash); - aa_put_loaddata(profile->rawdata); + aa_put_profile_loaddata(profile->rawdata); aa_label_destroy(&profile->label); kfree_sensitive(profile); @@ -901,17 +925,44 @@ bool aa_current_policy_admin_capable(struct aa_ns *ns) return res; } +static bool is_subset_of_obj_privilege(const struct cred *cred, + struct aa_label *label, + const struct cred *ocred) +{ + if (cred == ocred) + return true; + + if (!aa_label_is_subset(label, cred_label(ocred))) + return false; + /* don't allow crossing userns for now */ + if (cred->user_ns != ocred->user_ns) + return false; + if (!cap_issubset(cred->cap_inheritable, ocred->cap_inheritable)) + return false; + if (!cap_issubset(cred->cap_permitted, ocred->cap_permitted)) + return false; + if (!cap_issubset(cred->cap_effective, ocred->cap_effective)) + return false; + if (!cap_issubset(cred->cap_bset, ocred->cap_bset)) + return false; + if (!cap_issubset(cred->cap_ambient, ocred->cap_ambient)) + return false; + return true; +} + + /** * aa_may_manage_policy - can the current task manage policy * @subj_cred: subjects cred * @label: label to check if it can manage policy * @ns: namespace being managed by @label (may be NULL if @label's ns) + * @ocred: object cred if request is coming from an open object * @mask: contains the policy manipulation operation being done * * Returns: 0 if the task is allowed to manipulate policy else error */ int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, - struct aa_ns *ns, u32 mask) + struct aa_ns *ns, const struct cred *ocred, u32 mask) { const char *op; @@ -927,6 +978,11 @@ int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, return audit_policy(label, op, NULL, NULL, "policy_locked", -EACCES); + if (ocred && !is_subset_of_obj_privilege(subj_cred, label, ocred)) + return audit_policy(label, op, NULL, NULL, + "not privileged for target profile", + -EACCES); + if (!aa_policy_admin_capable(subj_cred, label, ns)) return audit_policy(label, op, NULL, NULL, "not policy admin", -EACCES); @@ -1098,7 +1154,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, LIST_HEAD(lh); op = mask & AA_MAY_REPLACE_POLICY ? OP_PROF_REPL : OP_PROF_LOAD; - aa_get_loaddata(udata); + aa_get_profile_loaddata(udata); /* released below */ error = aa_unpack(udata, &lh, &ns_name); if (error) @@ -1125,6 +1181,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, goto fail; } ns_name = ent->ns_name; + ent->ns_name = NULL; } else count++; } @@ -1149,10 +1206,10 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, if (aa_rawdata_eq(rawdata_ent, udata)) { struct aa_loaddata *tmp; - tmp = __aa_get_loaddata(rawdata_ent); + tmp = aa_get_profile_loaddata(rawdata_ent); /* check we didn't fail the race */ if (tmp) { - aa_put_loaddata(udata); + aa_put_profile_loaddata(udata); udata = tmp; break; } @@ -1165,7 +1222,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, struct aa_profile *p; if (aa_g_export_binary) - ent->new->rawdata = aa_get_loaddata(udata); + ent->new->rawdata = aa_get_profile_loaddata(udata); error = __lookup_replace(ns, ent->new->base.hname, !(mask & AA_MAY_REPLACE_POLICY), &ent->old, &info); @@ -1298,7 +1355,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, out: aa_put_ns(ns); - aa_put_loaddata(udata); + aa_put_profile_loaddata(udata); kfree(ns_name); if (error) diff --git a/security/apparmor/policy_ns.c b/security/apparmor/policy_ns.c index 64783ca3b0f2a..ff49a31ac2744 100644 --- a/security/apparmor/policy_ns.c +++ b/security/apparmor/policy_ns.c @@ -223,6 +223,8 @@ static struct aa_ns *__aa_create_ns(struct aa_ns *parent, const char *name, AA_BUG(!name); AA_BUG(!mutex_is_locked(&parent->lock)); + if (parent->level > MAX_NS_DEPTH) + return ERR_PTR(-ENOSPC); ns = alloc_ns(parent->base.hname, name); if (!ns) return ERR_PTR(-ENOMEM); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 7523971e37d9c..b0e18dd8d512b 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -109,34 +109,48 @@ bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r) return memcmp(l->data, r->data, r->compressed_size ?: r->size) == 0; } +static void do_loaddata_free(struct aa_loaddata *d) +{ + kfree_sensitive(d->hash); + kfree_sensitive(d->name); + kvfree(d->data); + kfree_sensitive(d); +} + +void aa_loaddata_kref(struct kref *kref) +{ + struct aa_loaddata *d = container_of(kref, struct aa_loaddata, + count.count); + + do_loaddata_free(d); +} + /* * need to take the ns mutex lock which is NOT safe most places that * put_loaddata is called, so we have to delay freeing it */ -static void do_loaddata_free(struct work_struct *work) +static void do_ploaddata_rmfs(struct work_struct *work) { struct aa_loaddata *d = container_of(work, struct aa_loaddata, work); struct aa_ns *ns = aa_get_ns(d->ns); if (ns) { mutex_lock_nested(&ns->lock, ns->level); + /* remove fs ref to loaddata */ __aa_fs_remove_rawdata(d); mutex_unlock(&ns->lock); aa_put_ns(ns); } - - kfree_sensitive(d->hash); - kfree_sensitive(d->name); - kvfree(d->data); - kfree_sensitive(d); + /* called by dropping last pcount, so drop its associated icount */ + aa_put_i_loaddata(d); } -void aa_loaddata_kref(struct kref *kref) +void aa_ploaddata_kref(struct kref *kref) { - struct aa_loaddata *d = container_of(kref, struct aa_loaddata, count); + struct aa_loaddata *d = container_of(kref, struct aa_loaddata, pcount); if (d) { - INIT_WORK(&d->work, do_loaddata_free); + INIT_WORK(&d->work, do_ploaddata_rmfs); schedule_work(&d->work); } } @@ -153,7 +167,9 @@ struct aa_loaddata *aa_loaddata_alloc(size_t size) kfree(d); return ERR_PTR(-ENOMEM); } - kref_init(&d->count); + kref_init(&d->count.count); + d->count.reftype = REF_RAWDATA; + kref_init(&d->pcount); INIT_LIST_HEAD(&d->list); return d; @@ -687,8 +703,10 @@ static ssize_t unpack_perms_table(struct aa_ext *e, struct aa_perms **perms) if (!aa_unpack_array(e, NULL, &size)) goto fail_reset; *perms = kcalloc(size, sizeof(struct aa_perms), GFP_KERNEL); - if (!*perms) - goto fail_reset; + if (!*perms) { + e->pos = pos; + return -ENOMEM; + } for (i = 0; i < size; i++) { if (!unpack_perm(e, version, &(*perms)[i])) goto fail; @@ -768,7 +786,17 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, if (!aa_unpack_u32(e, &pdb->start[AA_CLASS_FILE], "dfa_start")) { /* default start state for xmatch and file dfa */ pdb->start[AA_CLASS_FILE] = DFA_START; - } /* setup class index */ + } + + size_t state_count = pdb->dfa->tables[YYTD_ID_BASE]->td_lolen; + + if (pdb->start[0] >= state_count || + pdb->start[AA_CLASS_FILE] >= state_count) { + *info = "invalid dfa start state"; + goto fail; + } + + /* setup class index */ for (i = AA_CLASS_FILE + 1; i <= AA_CLASS_LAST; i++) { pdb->start[i] = aa_dfa_next(pdb->dfa, pdb->start[0], i); @@ -1165,7 +1193,6 @@ static int verify_header(struct aa_ext *e, int required, const char **ns) { int error = -EPROTONOSUPPORT; const char *name = NULL; - *ns = NULL; /* get the interface version */ if (!aa_unpack_u32(e, &e->version, "version")) { diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index 8e80db3ae21c0..64212b39ba4bb 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -196,6 +196,11 @@ void __aa_transition_rlimits(struct aa_label *old_l, struct aa_label *new_l) rules->rlimits.limits[j].rlim_max); /* soft limit should not exceed hard limit */ rlim->rlim_cur = min(rlim->rlim_cur, rlim->rlim_max); + if (j == RLIMIT_CPU && + rlim->rlim_cur != RLIM_INFINITY && + IS_ENABLED(CONFIG_POSIX_TIMERS)) + (void) update_rlimit_cpu(current->group_leader, + rlim->rlim_cur); } } } diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index a5e730ffda57f..5a8cef45bacf0 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -401,6 +401,7 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, { struct shash_desc *desc; const struct xattr *xattr; + struct xattr_list *xattr_entry; desc = init_desc(EVM_XATTR_HMAC, HASH_ALGO_SHA1); if (IS_ERR(desc)) { @@ -408,11 +409,16 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, return PTR_ERR(desc); } - for (xattr = xattrs; xattr->name; xattr++) { - if (!evm_protected_xattr(xattr->name)) - continue; + list_for_each_entry_lockless(xattr_entry, &evm_config_xattrnames, + list) { + for (xattr = xattrs; xattr->name; xattr++) { + if (strcmp(xattr_entry->name + + XATTR_SECURITY_PREFIX_LEN, xattr->name) != 0) + continue; - crypto_shash_update(desc, xattr->value, xattr->value_len); + crypto_shash_update(desc, xattr->value, + xattr->value_len); + } } hmac_add_misc(desc, inode, EVM_XATTR_HMAC, hmac_val); diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c index 7362f68f2d8b1..36a34c54de58b 100644 --- a/security/integrity/ima/ima_kexec.c +++ b/security/integrity/ima/ima_kexec.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include "ima.h" @@ -250,9 +252,7 @@ void ima_kexec_post_load(struct kimage *image) if (!image->ima_buffer_addr) return; - ima_kexec_buffer = kimage_map_segment(image, - image->ima_buffer_addr, - image->ima_buffer_size); + ima_kexec_buffer = kimage_map_segment(image, image->ima_segment_index); if (!ima_kexec_buffer) { pr_err("Could not map measurements buffer.\n"); return; @@ -296,3 +296,36 @@ void __init ima_load_kexec_buffer(void) pr_debug("Error restoring the measurement list: %d\n", rc); } } + +/* + * ima_validate_range - verify a physical buffer lies in addressable RAM + * @phys: physical start address of the buffer from previous kernel + * @size: size of the buffer + * + * On success return 0. On failure returns -EINVAL so callers can skip + * restoring. + */ +int ima_validate_range(phys_addr_t phys, size_t size) +{ + unsigned long start_pfn, end_pfn; + phys_addr_t end_phys; + + if (check_add_overflow(phys, (phys_addr_t)size - 1, &end_phys)) + return -EINVAL; + + start_pfn = PHYS_PFN(phys); + end_pfn = PHYS_PFN(end_phys); + +#ifdef CONFIG_X86 + if (!pfn_range_is_mapped(start_pfn, end_pfn)) +#else + if (!page_is_ram(start_pfn) || !page_is_ram(end_pfn)) +#endif + { + pr_warn("IMA: previous kernel measurement buffer %pa (size 0x%zx) lies outside available memory\n", + &phys, size); + return -EINVAL; + } + + return 0; +} diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index cdd225f65a629..ebaebccfbe9ab 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -573,18 +573,41 @@ static int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, */ static int ima_bprm_check(struct linux_binprm *bprm) { - int ret; struct lsm_prop prop; security_current_getlsmprop_subj(&prop); - ret = process_measurement(bprm->file, current_cred(), - &prop, NULL, 0, MAY_EXEC, BPRM_CHECK); - if (ret) - return ret; - - security_cred_getlsmprop(bprm->cred, &prop); - return process_measurement(bprm->file, bprm->cred, &prop, NULL, 0, - MAY_EXEC, CREDS_CHECK); + return process_measurement(bprm->file, current_cred(), + &prop, NULL, 0, MAY_EXEC, BPRM_CHECK); +} + +/** + * ima_creds_check - based on policy, collect/store measurement. + * @bprm: contains the linux_binprm structure + * @file: contains the file descriptor of the binary being executed + * + * The OS protects against an executable file, already open for write, + * from being executed in deny_write_access() and an executable file, + * already open for execute, from being modified in get_write_access(). + * So we can be certain that what we verify and measure here is actually + * what is being executed. + * + * The difference from ima_bprm_check() is that ima_creds_check() is invoked + * only after determining the final binary to be executed without interpreter, + * and not when searching for intermediate binaries. The reason is that since + * commit 56305aa9b6fab ("exec: Compute file based creds only once"), the + * credentials to be applied to the process are calculated only at that stage + * (bprm_creds_from_file security hook instead of bprm_check_security). + * + * On success return 0. On integrity appraisal error, assuming the file + * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. + */ +static int ima_creds_check(struct linux_binprm *bprm, const struct file *file) +{ + struct lsm_prop prop; + + security_current_getlsmprop_subj(&prop); + return process_measurement((struct file *)file, bprm->cred, &prop, NULL, + 0, MAY_EXEC, CREDS_CHECK); } /** @@ -1242,6 +1265,7 @@ static int __init init_ima(void) static struct security_hook_list ima_hooks[] __ro_after_init = { LSM_HOOK_INIT(bprm_check_security, ima_bprm_check), LSM_HOOK_INIT(bprm_creds_for_exec, ima_bprm_creds_for_exec), + LSM_HOOK_INIT(bprm_creds_from_file, ima_creds_check), LSM_HOOK_INIT(file_post_open, ima_file_check), LSM_HOOK_INIT(inode_post_create_tmpfile, ima_post_create_tmpfile), LSM_HOOK_INIT(file_release, ima_file_free), diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index 128fab8979308..db6d55af5a80b 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -674,7 +674,7 @@ static bool ima_match_rules(struct ima_rule_entry *rule, goto retry; } } - if (!rc) { + if (rc <= 0) { result = false; goto out; } diff --git a/security/keys/trusted-keys/trusted_tpm2.c b/security/keys/trusted-keys/trusted_tpm2.c index 024be262702fe..74cea80ed9be5 100644 --- a/security/keys/trusted-keys/trusted_tpm2.c +++ b/security/keys/trusted-keys/trusted_tpm2.c @@ -283,7 +283,10 @@ int tpm2_seal_trusted(struct tpm_chip *chip, goto out_put; } - tpm_buf_append_name(chip, &buf, options->keyhandle, NULL); + rc = tpm_buf_append_name(chip, &buf, options->keyhandle, NULL); + if (rc) + goto out; + tpm_buf_append_hmac_session(chip, &buf, TPM2_SA_DECRYPT, options->keyauth, TPM_DIGEST_SIZE); @@ -331,7 +334,10 @@ int tpm2_seal_trusted(struct tpm_chip *chip, goto out; } - tpm_buf_fill_hmac_session(chip, &buf); + rc = tpm_buf_fill_hmac_session(chip, &buf); + if (rc) + goto out; + rc = tpm_transmit_cmd(chip, &buf, 4, "sealing data"); rc = tpm_buf_check_hmac_response(chip, &buf, rc); if (rc) @@ -387,6 +393,7 @@ static int tpm2_load_cmd(struct tpm_chip *chip, struct trusted_key_options *options, u32 *blob_handle) { + u8 *blob_ref __free(kfree) = NULL; struct tpm_buf buf; unsigned int private_len; unsigned int public_len; @@ -400,6 +407,9 @@ static int tpm2_load_cmd(struct tpm_chip *chip, /* old form */ blob = payload->blob; payload->old_format = 1; + } else { + /* Bind for cleanup: */ + blob_ref = blob; } /* new format carries keyhandle but old format doesn't */ @@ -444,7 +454,10 @@ static int tpm2_load_cmd(struct tpm_chip *chip, return rc; } - tpm_buf_append_name(chip, &buf, options->keyhandle, NULL); + rc = tpm_buf_append_name(chip, &buf, options->keyhandle, NULL); + if (rc) + goto out; + tpm_buf_append_hmac_session(chip, &buf, 0, options->keyauth, TPM_DIGEST_SIZE); @@ -456,7 +469,10 @@ static int tpm2_load_cmd(struct tpm_chip *chip, goto out; } - tpm_buf_fill_hmac_session(chip, &buf); + rc = tpm_buf_fill_hmac_session(chip, &buf); + if (rc) + goto out; + rc = tpm_transmit_cmd(chip, &buf, 4, "loading blob"); rc = tpm_buf_check_hmac_response(chip, &buf, rc); if (!rc) @@ -464,8 +480,6 @@ static int tpm2_load_cmd(struct tpm_chip *chip, (__be32 *) &buf.data[TPM_HEADER_SIZE]); out: - if (blob != payload->blob) - kfree(blob); tpm_buf_destroy(&buf); if (rc > 0) @@ -475,7 +489,7 @@ static int tpm2_load_cmd(struct tpm_chip *chip, } /** - * tpm2_unseal_cmd() - execute a TPM2_Unload command + * tpm2_unseal_cmd() - execute a TPM2_Unseal command * * @chip: TPM chip to use * @payload: the key data in clear and encrypted form @@ -506,7 +520,9 @@ static int tpm2_unseal_cmd(struct tpm_chip *chip, return rc; } - tpm_buf_append_name(chip, &buf, blob_handle, NULL); + rc = tpm_buf_append_name(chip, &buf, blob_handle, NULL); + if (rc) + goto out; if (!options->policyhandle) { tpm_buf_append_hmac_session(chip, &buf, TPM2_SA_ENCRYPT, @@ -531,7 +547,10 @@ static int tpm2_unseal_cmd(struct tpm_chip *chip, NULL, 0); } - tpm_buf_fill_hmac_session(chip, &buf); + rc = tpm_buf_fill_hmac_session(chip, &buf); + if (rc) + goto out; + rc = tpm_transmit_cmd(chip, &buf, 6, "unsealing"); rc = tpm_buf_check_hmac_response(chip, &buf, rc); if (rc > 0) diff --git a/security/landlock/audit.c b/security/landlock/audit.c index c52d079cdb77b..e899995f1fd59 100644 --- a/security/landlock/audit.c +++ b/security/landlock/audit.c @@ -191,7 +191,7 @@ static size_t get_denied_layer(const struct landlock_ruleset *const domain, long youngest_layer = -1; for_each_set_bit(access_bit, &access_req, layer_masks_size) { - const access_mask_t mask = (*layer_masks)[access_bit]; + const layer_mask_t mask = (*layer_masks)[access_bit]; long layer; if (!mask) diff --git a/security/landlock/errata/abi-1.h b/security/landlock/errata/abi-1.h new file mode 100644 index 0000000000000..e8a2bff2e5b6a --- /dev/null +++ b/security/landlock/errata/abi-1.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/** + * DOC: erratum_3 + * + * Erratum 3: Disconnected directory handling + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * This fix addresses an issue with disconnected directories that occur when a + * directory is moved outside the scope of a bind mount. The change ensures + * that evaluated access rights include both those from the disconnected file + * hierarchy down to its filesystem root and those from the related mount point + * hierarchy. This prevents access right widening through rename or link + * actions. + */ +LANDLOCK_ERRATUM(3) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index d9c12b993fa7d..a2ed0e76938aa 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -909,21 +909,31 @@ static bool is_access_to_paths_allowed( break; } } + if (unlikely(IS_ROOT(walker_path.dentry))) { - /* - * Stops at disconnected root directories. Only allows - * access to internal filesystems (e.g. nsfs, which is - * reachable through /proc//ns/). - */ - if (walker_path.mnt->mnt_flags & MNT_INTERNAL) { + if (likely(walker_path.mnt->mnt_flags & MNT_INTERNAL)) { + /* + * Stops and allows access when reaching disconnected root + * directories that are part of internal filesystems (e.g. nsfs, + * which is reachable through /proc//ns/). + */ allowed_parent1 = true; allowed_parent2 = true; + break; } - break; + + /* + * We reached a disconnected root directory from a bind mount. + * Let's continue the walk with the mount point we missed. + */ + dput(walker_path.dentry); + walker_path.dentry = walker_path.mnt->mnt_root; + dget(walker_path.dentry); + } else { + parent_dentry = dget_parent(walker_path.dentry); + dput(walker_path.dentry); + walker_path.dentry = parent_dentry; } - parent_dentry = dget_parent(walker_path.dentry); - dput(walker_path.dentry); - walker_path.dentry = parent_dentry; } path_put(&walker_path); @@ -1021,6 +1031,9 @@ static access_mask_t maybe_remove(const struct dentry *const dentry) * file. While walking from @dir to @mnt_root, we record all the domain's * allowed accesses in @layer_masks_dom. * + * Because of disconnected directories, this walk may not reach @mnt_dir. In + * this case, the walk will continue to @mnt_dir after this call. + * * This is similar to is_access_to_paths_allowed() but much simpler because it * only handles walking on the same mount point and only checks one set of * accesses. @@ -1062,8 +1075,11 @@ static bool collect_domain_accesses( break; } - /* We should not reach a root other than @mnt_root. */ - if (dir == mnt_root || WARN_ON_ONCE(IS_ROOT(dir))) + /* + * Stops at the mount point or the filesystem root for a disconnected + * directory. + */ + if (dir == mnt_root || unlikely(IS_ROOT(dir))) break; parent_dentry = dget_parent(dir); diff --git a/security/landlock/net.c b/security/landlock/net.c index 1f3915a90a808..e6367e30e5b0e 100644 --- a/security/landlock/net.c +++ b/security/landlock/net.c @@ -71,6 +71,61 @@ static int current_check_access_socket(struct socket *const sock, switch (address->sa_family) { case AF_UNSPEC: + if (access_request == LANDLOCK_ACCESS_NET_CONNECT_TCP) { + /* + * Connecting to an address with AF_UNSPEC dissolves + * the TCP association, which have the same effect as + * closing the connection while retaining the socket + * object (i.e., the file descriptor). As for dropping + * privileges, closing connections is always allowed. + * + * For a TCP access control system, this request is + * legitimate. Let the network stack handle potential + * inconsistencies and return -EINVAL if needed. + */ + return 0; + } else if (access_request == LANDLOCK_ACCESS_NET_BIND_TCP) { + /* + * Binding to an AF_UNSPEC address is treated + * differently by IPv4 and IPv6 sockets. The socket's + * family may change under our feet due to + * setsockopt(IPV6_ADDRFORM), but that's ok: we either + * reject entirely or require + * %LANDLOCK_ACCESS_NET_BIND_TCP for the given port, so + * it cannot be used to bypass the policy. + * + * IPv4 sockets map AF_UNSPEC to AF_INET for + * retrocompatibility for bind accesses, only if the + * address is INADDR_ANY (cf. __inet_bind). IPv6 + * sockets always reject it. + * + * Checking the address is required to not wrongfully + * return -EACCES instead of -EAFNOSUPPORT or -EINVAL. + * We could return 0 and let the network stack handle + * these checks, but it is safer to return a proper + * error and test consistency thanks to kselftest. + */ + if (sock->sk->__sk_common.skc_family == AF_INET) { + const struct sockaddr_in *const sockaddr = + (struct sockaddr_in *)address; + + if (addrlen < sizeof(struct sockaddr_in)) + return -EINVAL; + + if (sockaddr->sin_addr.s_addr != + htonl(INADDR_ANY)) + return -EAFNOSUPPORT; + } else { + if (addrlen < SIN6_LEN_RFC2133) + return -EINVAL; + else + return -EAFNOSUPPORT; + } + } else { + WARN_ON_ONCE(1); + } + /* Only for bind(AF_UNSPEC+INADDR_ANY) on IPv4 socket. */ + fallthrough; case AF_INET: { const struct sockaddr_in *addr4; @@ -119,57 +174,18 @@ static int current_check_access_socket(struct socket *const sock, return 0; } - /* Specific AF_UNSPEC handling. */ - if (address->sa_family == AF_UNSPEC) { - /* - * Connecting to an address with AF_UNSPEC dissolves the TCP - * association, which have the same effect as closing the - * connection while retaining the socket object (i.e., the file - * descriptor). As for dropping privileges, closing - * connections is always allowed. - * - * For a TCP access control system, this request is legitimate. - * Let the network stack handle potential inconsistencies and - * return -EINVAL if needed. - */ - if (access_request == LANDLOCK_ACCESS_NET_CONNECT_TCP) - return 0; - - /* - * For compatibility reason, accept AF_UNSPEC for bind - * accesses (mapped to AF_INET) only if the address is - * INADDR_ANY (cf. __inet_bind). Checking the address is - * required to not wrongfully return -EACCES instead of - * -EAFNOSUPPORT. - * - * We could return 0 and let the network stack handle these - * checks, but it is safer to return a proper error and test - * consistency thanks to kselftest. - */ - if (access_request == LANDLOCK_ACCESS_NET_BIND_TCP) { - /* addrlen has already been checked for AF_UNSPEC. */ - const struct sockaddr_in *const sockaddr = - (struct sockaddr_in *)address; - - if (sock->sk->__sk_common.skc_family != AF_INET) - return -EINVAL; - - if (sockaddr->sin_addr.s_addr != htonl(INADDR_ANY)) - return -EAFNOSUPPORT; - } - } else { - /* - * Checks sa_family consistency to not wrongfully return - * -EACCES instead of -EINVAL. Valid sa_family changes are - * only (from AF_INET or AF_INET6) to AF_UNSPEC. - * - * We could return 0 and let the network stack handle this - * check, but it is safer to return a proper error and test - * consistency thanks to kselftest. - */ - if (address->sa_family != sock->sk->__sk_common.skc_family) - return -EINVAL; - } + /* + * Checks sa_family consistency to not wrongfully return + * -EACCES instead of -EINVAL. Valid sa_family changes are + * only (from AF_INET or AF_INET6) to AF_UNSPEC. + * + * We could return 0 and let the network stack handle this + * check, but it is safer to return a proper error and test + * consistency thanks to kselftest. + */ + if (address->sa_family != sock->sk->__sk_common.skc_family && + address->sa_family != AF_UNSPEC) + return -EINVAL; id.key.data = (__force uintptr_t)port; BUILD_BUG_ON(sizeof(port) > sizeof(id.key.data)); diff --git a/security/security.c b/security/security.c index 4d3c03a4524c5..603c3c6d5635d 100644 --- a/security/security.c +++ b/security/security.c @@ -78,6 +78,7 @@ const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX + 1] = { [LOCKDOWN_BPF_WRITE_USER] = "use of bpf to write user RAM", [LOCKDOWN_DBG_WRITE_KERNEL] = "use of kgdb/kdb to write kernel RAM", [LOCKDOWN_RTAS_ERROR_INJECTION] = "RTAS error injection", + [LOCKDOWN_XEN_USER_ACTIONS] = "Xen guest user action", [LOCKDOWN_INTEGRITY_MAX] = "integrity", [LOCKDOWN_KCORE] = "/proc/kcore access", [LOCKDOWN_KPROBES] = "use of kprobes", diff --git a/security/smack/smack.h b/security/smack/smack.h index bf6a6ed3946ce..759343a6bbaeb 100644 --- a/security/smack/smack.h +++ b/security/smack/smack.h @@ -286,9 +286,12 @@ int smk_tskacc(struct task_smack *, struct smack_known *, int smk_curacc(struct smack_known *, u32, struct smk_audit_info *); int smack_str_from_perm(char *string, int access); struct smack_known *smack_from_secid(const u32); +int smk_parse_label_len(const char *string, int len); char *smk_parse_smack(const char *string, int len); int smk_netlbl_mls(int, char *, struct netlbl_lsm_secattr *, int); struct smack_known *smk_import_entry(const char *, int); +struct smack_known *smk_import_valid_label(const char *label, int label_len, + gfp_t gfp); void smk_insert_entry(struct smack_known *skp); struct smack_known *smk_find_entry(const char *); bool smack_privileged(int cap); diff --git a/security/smack/smack_access.c b/security/smack/smack_access.c index 2e4a0cb22782b..a289cb6672bd4 100644 --- a/security/smack/smack_access.c +++ b/security/smack/smack_access.c @@ -443,19 +443,19 @@ struct smack_known *smk_find_entry(const char *string) } /** - * smk_parse_smack - parse smack label from a text string - * @string: a text string that might contain a Smack label - * @len: the maximum size, or zero if it is NULL terminated. + * smk_parse_label_len - calculate the length of the starting segment + * in the string that constitutes a valid smack label + * @string: a text string that might contain a Smack label at the beginning + * @len: the maximum size to look into, may be zero if string is null-terminated * - * Returns a pointer to the clean label or an error code. + * Returns the length of the segment (0 < L < SMK_LONGLABEL) or an error code. */ -char *smk_parse_smack(const char *string, int len) +int smk_parse_label_len(const char *string, int len) { - char *smack; int i; - if (len <= 0) - len = strlen(string) + 1; + if (len <= 0 || len > SMK_LONGLABEL) + len = SMK_LONGLABEL; /* * Reserve a leading '-' as an indicator that @@ -463,7 +463,7 @@ char *smk_parse_smack(const char *string, int len) * including /smack/cipso and /smack/cipso2 */ if (string[0] == '-') - return ERR_PTR(-EINVAL); + return -EINVAL; for (i = 0; i < len; i++) if (string[i] > '~' || string[i] <= ' ' || string[i] == '/' || @@ -471,6 +471,25 @@ char *smk_parse_smack(const char *string, int len) break; if (i == 0 || i >= SMK_LONGLABEL) + return -EINVAL; + + return i; +} + +/** + * smk_parse_smack - copy the starting segment in the string + * that constitutes a valid smack label + * @string: a text string that might contain a Smack label at the beginning + * @len: the maximum size to look into, may be zero if string is null-terminated + * + * Returns a pointer to the copy of the label or an error code. + */ +char *smk_parse_smack(const char *string, int len) +{ + char *smack; + int i = smk_parse_label_len(string, len); + + if (i < 0) return ERR_PTR(-EINVAL); smack = kstrndup(string, i, GFP_NOFS); @@ -554,31 +573,25 @@ int smack_populate_secattr(struct smack_known *skp) } /** - * smk_import_entry - import a label, return the list entry - * @string: a text string that might be a Smack label - * @len: the maximum size, or zero if it is NULL terminated. + * smk_import_valid_allocated_label - import a label, return the list entry + * @smack: a text string that is a valid Smack label and may be kfree()ed. + * It is consumed: either becomes a part of the entry or kfree'ed. * - * Returns a pointer to the entry in the label list that - * matches the passed string, adding it if necessary, - * or an error code. + * Returns: see description of smk_import_entry() */ -struct smack_known *smk_import_entry(const char *string, int len) +static struct smack_known * +smk_import_allocated_label(char *smack, gfp_t gfp) { struct smack_known *skp; - char *smack; int rc; - smack = smk_parse_smack(string, len); - if (IS_ERR(smack)) - return ERR_CAST(smack); - mutex_lock(&smack_known_lock); skp = smk_find_entry(smack); if (skp != NULL) goto freeout; - skp = kzalloc(sizeof(*skp), GFP_NOFS); + skp = kzalloc(sizeof(*skp), gfp); if (skp == NULL) { skp = ERR_PTR(-ENOMEM); goto freeout; @@ -608,6 +621,42 @@ struct smack_known *smk_import_entry(const char *string, int len) return skp; } +/** + * smk_import_entry - import a label, return the list entry + * @string: a text string that might contain a Smack label at the beginning + * @len: the maximum size to look into, may be zero if string is null-terminated + * + * Returns a pointer to the entry in the label list that + * matches the passed string, adding it if necessary, + * or an error code. + */ +struct smack_known *smk_import_entry(const char *string, int len) +{ + char *smack = smk_parse_smack(string, len); + + if (IS_ERR(smack)) + return ERR_CAST(smack); + + return smk_import_allocated_label(smack, GFP_NOFS); +} + +/** + * smk_import_valid_label - import a label, return the list entry + * @label a text string that is a valid Smack label, not null-terminated + * + * Returns: see description of smk_import_entry() + */ +struct smack_known * +smk_import_valid_label(const char *label, int label_len, gfp_t gfp) +{ + char *smack = kstrndup(label, label_len, gfp); + + if (!smack) + return ERR_PTR(-ENOMEM); + + return smk_import_allocated_label(smack, gfp); +} + /** * smack_from_secid - find the Smack label associated with a secid * @secid: an integer that might be associated with a Smack label diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index af986587841d8..7fd8a60b8be90 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -962,6 +962,42 @@ static int smack_inode_alloc_security(struct inode *inode) return 0; } +/** + * smk_rule_transmutes - does access rule for (subject,object) contain 't'? + * @subject: a pointer to the subject's Smack label entry + * @object: a pointer to the object's Smack label entry + */ +static bool +smk_rule_transmutes(struct smack_known *subject, + const struct smack_known *object) +{ + int may; + + rcu_read_lock(); + may = smk_access_entry(subject->smk_known, object->smk_known, + &subject->smk_rules); + rcu_read_unlock(); + return (may > 0) && (may & MAY_TRANSMUTE); +} + +static int +xattr_dupval(struct xattr *xattrs, int *xattr_count, + const char *name, const void *value, unsigned int vallen) +{ + struct xattr * const xattr = lsm_get_xattr_slot(xattrs, xattr_count); + + if (!xattr) + return 0; + + xattr->value = kmemdup(value, vallen, GFP_NOFS); + if (!xattr->value) + return -ENOMEM; + + xattr->value_len = vallen; + xattr->name = name; + return 0; +} + /** * smack_inode_init_security - copy out the smack from an inode * @inode: the newly created inode @@ -977,23 +1013,30 @@ static int smack_inode_init_security(struct inode *inode, struct inode *dir, struct xattr *xattrs, int *xattr_count) { struct task_smack *tsp = smack_cred(current_cred()); - struct inode_smack *issp = smack_inode(inode); - struct smack_known *skp = smk_of_task(tsp); - struct smack_known *isp = smk_of_inode(inode); + struct inode_smack * const issp = smack_inode(inode); struct smack_known *dsp = smk_of_inode(dir); - struct xattr *xattr = lsm_get_xattr_slot(xattrs, xattr_count); - int may; + int rc = 0; + int transflag = 0; + bool trans_cred; + bool trans_rule; + /* + * UNIX domain sockets use lower level socket data. Let + * UDS inode have fixed * label to keep smack_inode_permission() calm + * when called from unix_find_bsd() + */ + if (S_ISSOCK(inode->i_mode)) { + /* forced label, no need to save to xattrs */ + issp->smk_inode = &smack_known_star; + goto instant_inode; + } /* * If equal, transmuting already occurred in * smack_dentry_create_files_as(). No need to check again. */ - if (tsp->smk_task != tsp->smk_transmuted) { - rcu_read_lock(); - may = smk_access_entry(skp->smk_known, dsp->smk_known, - &skp->smk_rules); - rcu_read_unlock(); - } + trans_cred = (tsp->smk_task == tsp->smk_transmuted); + if (!trans_cred) + trans_rule = smk_rule_transmutes(smk_of_task(tsp), dsp); /* * In addition to having smk_task equal to smk_transmuted, @@ -1001,47 +1044,38 @@ static int smack_inode_init_security(struct inode *inode, struct inode *dir, * requests transmutation then by all means transmute. * Mark the inode as changed. */ - if ((tsp->smk_task == tsp->smk_transmuted) || - (may > 0 && ((may & MAY_TRANSMUTE) != 0) && - smk_inode_transmutable(dir))) { - struct xattr *xattr_transmute; - + if (trans_cred || (trans_rule && smk_inode_transmutable(dir))) { /* * The caller of smack_dentry_create_files_as() * should have overridden the current cred, so the * inode label was already set correctly in * smack_inode_alloc_security(). */ - if (tsp->smk_task != tsp->smk_transmuted) - isp = issp->smk_inode = dsp; - - issp->smk_flags |= SMK_INODE_TRANSMUTE; - xattr_transmute = lsm_get_xattr_slot(xattrs, - xattr_count); - if (xattr_transmute) { - xattr_transmute->value = kmemdup(TRANS_TRUE, - TRANS_TRUE_SIZE, - GFP_NOFS); - if (!xattr_transmute->value) - return -ENOMEM; + if (!trans_cred) + issp->smk_inode = dsp; - xattr_transmute->value_len = TRANS_TRUE_SIZE; - xattr_transmute->name = XATTR_SMACK_TRANSMUTE; + if (S_ISDIR(inode->i_mode)) { + transflag = SMK_INODE_TRANSMUTE; + + if (xattr_dupval(xattrs, xattr_count, + XATTR_SMACK_TRANSMUTE, + TRANS_TRUE, + TRANS_TRUE_SIZE + )) + rc = -ENOMEM; } } - issp->smk_flags |= SMK_INODE_INSTANT; - - if (xattr) { - xattr->value = kstrdup(isp->smk_known, GFP_NOFS); - if (!xattr->value) - return -ENOMEM; - - xattr->value_len = strlen(isp->smk_known); - xattr->name = XATTR_SMACK_SUFFIX; - } - - return 0; + if (rc == 0) + if (xattr_dupval(xattrs, xattr_count, + XATTR_SMACK_SUFFIX, + issp->smk_inode->smk_known, + strlen(issp->smk_inode->smk_known) + )) + rc = -ENOMEM; +instant_inode: + issp->smk_flags |= (SMK_INODE_INSTANT | transflag); + return rc; } /** @@ -1315,13 +1349,23 @@ static int smack_inode_setxattr(struct mnt_idmap *idmap, int check_import = 0; int check_star = 0; int rc = 0; + umode_t const i_mode = d_backing_inode(dentry)->i_mode; /* * Check label validity here so import won't fail in post_setxattr */ - if (strcmp(name, XATTR_NAME_SMACK) == 0 || - strcmp(name, XATTR_NAME_SMACKIPIN) == 0 || - strcmp(name, XATTR_NAME_SMACKIPOUT) == 0) { + if (strcmp(name, XATTR_NAME_SMACK) == 0) { + /* + * UDS inode has fixed label + */ + if (S_ISSOCK(i_mode)) { + rc = -EINVAL; + } else { + check_priv = 1; + check_import = 1; + } + } else if (strcmp(name, XATTR_NAME_SMACKIPIN) == 0 || + strcmp(name, XATTR_NAME_SMACKIPOUT) == 0) { check_priv = 1; check_import = 1; } else if (strcmp(name, XATTR_NAME_SMACKEXEC) == 0 || @@ -1331,7 +1375,7 @@ static int smack_inode_setxattr(struct mnt_idmap *idmap, check_star = 1; } else if (strcmp(name, XATTR_NAME_SMACKTRANSMUTE) == 0) { check_priv = 1; - if (!S_ISDIR(d_backing_inode(dentry)->i_mode) || + if (!S_ISDIR(i_mode) || size != TRANS_TRUE_SIZE || strncmp(value, TRANS_TRUE, TRANS_TRUE_SIZE) != 0) rc = -EINVAL; @@ -1462,12 +1506,15 @@ static int smack_inode_removexattr(struct mnt_idmap *idmap, * Don't do anything special for these. * XATTR_NAME_SMACKIPIN * XATTR_NAME_SMACKIPOUT + * XATTR_NAME_SMACK if S_ISSOCK (UDS inode has fixed label) */ if (strcmp(name, XATTR_NAME_SMACK) == 0) { - struct super_block *sbp = dentry->d_sb; - struct superblock_smack *sbsp = smack_superblock(sbp); + if (!S_ISSOCK(d_backing_inode(dentry)->i_mode)) { + struct super_block *sbp = dentry->d_sb; + struct superblock_smack *sbsp = smack_superblock(sbp); - isp->smk_inode = sbsp->smk_default; + isp->smk_inode = sbsp->smk_default; + } } else if (strcmp(name, XATTR_NAME_SMACKEXEC) == 0) isp->smk_task = NULL; else if (strcmp(name, XATTR_NAME_SMACKMMAP) == 0) @@ -3585,7 +3632,7 @@ static void smack_d_instantiate(struct dentry *opt_dentry, struct inode *inode) */ /* - * UNIX domain sockets use lower level socket data. + * UDS inode has fixed label (*) */ if (S_ISSOCK(inode->i_mode)) { final = &smack_known_star; @@ -3663,7 +3710,7 @@ static void smack_d_instantiate(struct dentry *opt_dentry, struct inode *inode) * @attr: which attribute to fetch * @ctx: buffer to receive the result * @size: available size in, actual size out - * @flags: unused + * @flags: reserved, currently zero * * Fill the passed user space @ctx with the details of the requested * attribute. @@ -3724,47 +3771,55 @@ static int smack_getprocattr(struct task_struct *p, const char *name, char **val * Sets the Smack value of the task. Only setting self * is permitted and only with privilege * - * Returns the length of the smack label or an error code + * Returns zero on success or an error code */ -static int do_setattr(u64 attr, void *value, size_t size) +static int do_setattr(unsigned int attr, void *value, size_t size) { struct task_smack *tsp = smack_cred(current_cred()); struct cred *new; struct smack_known *skp; - struct smack_known_list_elem *sklep; - int rc; - - if (!smack_privileged(CAP_MAC_ADMIN) && list_empty(&tsp->smk_relabel)) - return -EPERM; + int label_len; + /* + * let unprivileged user validate input, check permissions later + */ if (value == NULL || size == 0 || size >= SMK_LONGLABEL) return -EINVAL; - if (attr != LSM_ATTR_CURRENT) - return -EOPNOTSUPP; - - skp = smk_import_entry(value, size); - if (IS_ERR(skp)) - return PTR_ERR(skp); + label_len = smk_parse_label_len(value, size); + if (label_len < 0 || label_len != size) + return -EINVAL; /* * No process is ever allowed the web ("@") label * and the star ("*") label. */ - if (skp == &smack_known_web || skp == &smack_known_star) - return -EINVAL; + if (label_len == 1 /* '@', '*' */) { + const char c = *(const char *)value; + + if (c == *smack_known_web.smk_known || + c == *smack_known_star.smk_known) + return -EPERM; + } if (!smack_privileged(CAP_MAC_ADMIN)) { - rc = -EPERM; - list_for_each_entry(sklep, &tsp->smk_relabel, list) - if (sklep->smk_label == skp) { - rc = 0; - break; - } - if (rc) - return rc; + const struct smack_known_list_elem *sklep; + list_for_each_entry(sklep, &tsp->smk_relabel, list) { + const char *cp = sklep->smk_label->smk_known; + + if (strlen(cp) == label_len && + strncmp(cp, value, label_len) == 0) + goto in_relabel; + } + return -EPERM; +in_relabel: + ; } + skp = smk_import_valid_label(value, label_len, GFP_KERNEL); + if (IS_ERR(skp)) + return PTR_ERR(skp); + new = prepare_creds(); if (new == NULL) return -ENOMEM; @@ -3777,7 +3832,7 @@ static int do_setattr(u64 attr, void *value, size_t size) smk_destroy_label_list(&tsp->smk_relabel); commit_creds(new); - return size; + return 0; } /** @@ -3785,7 +3840,7 @@ static int do_setattr(u64 attr, void *value, size_t size) * @attr: which attribute to set * @ctx: buffer containing the data * @size: size of @ctx - * @flags: unused + * @flags: reserved, must be zero * * Fill the passed user space @ctx with the details of the requested * attribute. @@ -3795,12 +3850,26 @@ static int do_setattr(u64 attr, void *value, size_t size) static int smack_setselfattr(unsigned int attr, struct lsm_ctx *ctx, u32 size, u32 flags) { - int rc; + if (attr != LSM_ATTR_CURRENT) + return -EOPNOTSUPP; - rc = do_setattr(attr, ctx->ctx, ctx->ctx_len); - if (rc > 0) - return 0; - return rc; + if (ctx->flags) + return -EINVAL; + /* + * string must have \0 terminator, included in ctx->ctx + * (see description of struct lsm_ctx) + */ + if (ctx->ctx_len == 0) + return -EINVAL; + + if (ctx->ctx[ctx->ctx_len - 1] != '\0') + return -EINVAL; + /* + * other do_setattr() caller, smack_setprocattr(), + * does not count \0 into size, so + * decreasing length by 1 to accommodate the divergence. + */ + return do_setattr(attr, ctx->ctx, ctx->ctx_len - 1); } /** @@ -3812,15 +3881,39 @@ static int smack_setselfattr(unsigned int attr, struct lsm_ctx *ctx, * Sets the Smack value of the task. Only setting self * is permitted and only with privilege * - * Returns the length of the smack label or an error code + * Returns the size of the input value or an error code */ static int smack_setprocattr(const char *name, void *value, size_t size) { - int attr = lsm_name_to_attr(name); + size_t realsize = size; + unsigned int attr = lsm_name_to_attr(name); - if (attr != LSM_ATTR_UNDEF) - return do_setattr(attr, value, size); - return -EINVAL; + switch (attr) { + case LSM_ATTR_UNDEF: return -EINVAL; + default: return -EOPNOTSUPP; + case LSM_ATTR_CURRENT: + ; + } + + /* + * The value for the "current" attribute is the label + * followed by one of the 4 trailers: none, \0, \n, \n\0 + * + * I.e. following inputs are accepted as 3-characters long label "foo": + * + * "foo" (3 characters) + * "foo\0" (4 characters) + * "foo\n" (4 characters) + * "foo\n\0" (5 characters) + */ + + if (realsize && (((const char *)value)[realsize - 1] == '\0')) + --realsize; + + if (realsize && (((const char *)value)[realsize - 1] == '\n')) + --realsize; + + return do_setattr(attr, value, realsize) ? : size; } /** @@ -4850,6 +4943,11 @@ static int smack_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid) static int smack_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen) { + /* + * UDS inode has fixed label. Ignore nfs label. + */ + if (S_ISSOCK(inode->i_mode)) + return 0; return smack_inode_setsecurity(inode, XATTR_SMACK_SUFFIX, ctx, ctxlen, 0); } @@ -4915,7 +5013,6 @@ static int smack_dentry_create_files_as(struct dentry *dentry, int mode, struct task_smack *otsp = smack_cred(old); struct task_smack *ntsp = smack_cred(new); struct inode_smack *isp; - int may; /* * Use the process credential unless all of @@ -4929,18 +5026,12 @@ static int smack_dentry_create_files_as(struct dentry *dentry, int mode, isp = smack_inode(d_inode(dentry->d_parent)); if (isp->smk_flags & SMK_INODE_TRANSMUTE) { - rcu_read_lock(); - may = smk_access_entry(otsp->smk_task->smk_known, - isp->smk_inode->smk_known, - &otsp->smk_task->smk_rules); - rcu_read_unlock(); - /* * If the directory is transmuting and the rule * providing access is transmuting use the containing * directory label instead of the process label. */ - if (may > 0 && (may & MAY_TRANSMUTE)) { + if (smk_rule_transmutes(otsp->smk_task, isp->smk_inode)) { ntsp->smk_task = isp->smk_inode; ntsp->smk_transmuted = ntsp->smk_task; } diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index b1e5e62f5cbd1..d27d9140dda2f 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -70,6 +70,7 @@ enum smk_inos { static DEFINE_MUTEX(smack_cipso_lock); static DEFINE_MUTEX(smack_ambient_lock); static DEFINE_MUTEX(smk_net4addr_lock); +static DEFINE_MUTEX(smk_cipso_doi_lock); #if IS_ENABLED(CONFIG_IPV6) static DEFINE_MUTEX(smk_net6addr_lock); #endif /* CONFIG_IPV6 */ @@ -141,7 +142,7 @@ struct smack_parsed_rule { int smk_access2; }; -static int smk_cipso_doi_value = SMACK_CIPSO_DOI_DEFAULT; +static u32 smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; /* * Values for parsing cipso rules @@ -663,43 +664,60 @@ static const struct file_operations smk_load_ops = { }; /** - * smk_cipso_doi - initialize the CIPSO domain + * smk_cipso_doi - set netlabel maps + * @ndoi: new value for our CIPSO DOI + * @gfp_flags: kmalloc allocation context */ -static void smk_cipso_doi(void) +static int +smk_cipso_doi(u32 ndoi, gfp_t gfp_flags) { - int rc; + int rc = 0; struct cipso_v4_doi *doip; struct netlbl_audit nai; - smk_netlabel_audit_set(&nai); + mutex_lock(&smk_cipso_doi_lock); - rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); - if (rc != 0) - printk(KERN_WARNING "%s:%d remove rc = %d\n", - __func__, __LINE__, rc); + if (smk_cipso_doi_value == ndoi) + goto clr_doi_lock; + + smk_netlabel_audit_set(&nai); - doip = kmalloc(sizeof(struct cipso_v4_doi), GFP_KERNEL | __GFP_NOFAIL); + doip = kmalloc(sizeof(struct cipso_v4_doi), gfp_flags); + if (!doip) { + rc = -ENOMEM; + goto clr_doi_lock; + } doip->map.std = NULL; - doip->doi = smk_cipso_doi_value; + doip->doi = ndoi; doip->type = CIPSO_V4_MAP_PASS; doip->tags[0] = CIPSO_V4_TAG_RBITMAP; for (rc = 1; rc < CIPSO_V4_TAG_MAXCNT; rc++) doip->tags[rc] = CIPSO_V4_TAG_INVALID; rc = netlbl_cfg_cipsov4_add(doip, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d cipso add rc = %d\n", - __func__, __LINE__, rc); + if (rc) { kfree(doip); - return; + goto clr_doi_lock; } - rc = netlbl_cfg_cipsov4_map_add(doip->doi, NULL, NULL, NULL, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d map add rc = %d\n", - __func__, __LINE__, rc); - netlbl_cfg_cipsov4_del(doip->doi, &nai); - return; + + if (smk_cipso_doi_value != CIPSO_V4_DOI_UNKNOWN) { + rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + if (rc && rc != -ENOENT) + goto clr_ndoi_def; + + netlbl_cfg_cipsov4_del(smk_cipso_doi_value, &nai); } + + rc = netlbl_cfg_cipsov4_map_add(ndoi, NULL, NULL, NULL, &nai); + if (rc) { + smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; // no default map +clr_ndoi_def: netlbl_cfg_cipsov4_del(ndoi, &nai); + } else + smk_cipso_doi_value = ndoi; + +clr_doi_lock: + mutex_unlock(&smk_cipso_doi_lock); + return rc; } /** @@ -1562,7 +1580,7 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, if (*ppos != 0) return 0; - sprintf(temp, "%d", smk_cipso_doi_value); + sprintf(temp, "%lu", (unsigned long)smk_cipso_doi_value); rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); return rc; @@ -1581,7 +1599,7 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { char temp[80]; - int i; + unsigned long u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; @@ -1594,14 +1612,13 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, temp[count] = '\0'; - if (sscanf(temp, "%d", &i) != 1) + if (kstrtoul(temp, 10, &u)) return -EINVAL; - smk_cipso_doi_value = i; - - smk_cipso_doi(); + if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + return -EINVAL; - return count; + return smk_cipso_doi(u, GFP_KERNEL) ? : count; } static const struct file_operations smk_doi_ops = { @@ -2982,6 +2999,7 @@ static int __init init_smk_fs(void) { int err; int rc; + struct netlbl_audit nai; if (smack_enabled == 0) return 0; @@ -3000,7 +3018,10 @@ static int __init init_smk_fs(void) } } - smk_cipso_doi(); + smk_netlabel_audit_set(&nai); + (void) netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + (void) smk_cipso_doi(SMACK_CIPSO_DOI_DEFAULT, + GFP_KERNEL | __GFP_NOFAIL); smk_unlbl_ambient(NULL); rc = smack_populate_secattr(&smack_known_floor); diff --git a/sound/ac97/bus.c b/sound/ac97/bus.c index f4254703d29f7..bb9b795e02262 100644 --- a/sound/ac97/bus.c +++ b/sound/ac97/bus.c @@ -298,6 +298,7 @@ static void ac97_adapter_release(struct device *dev) idr_remove(&ac97_adapter_idr, ac97_ctrl->nr); dev_dbg(&ac97_ctrl->adap, "adapter unregistered by %s\n", dev_name(ac97_ctrl->parent)); + kfree(ac97_ctrl); } static const struct device_type ac97_adapter_type = { @@ -319,7 +320,9 @@ static int ac97_add_adapter(struct ac97_controller *ac97_ctrl) ret = device_register(&ac97_ctrl->adap); if (ret) put_device(&ac97_ctrl->adap); - } + } else + kfree(ac97_ctrl); + if (!ret) { list_add(&ac97_ctrl->controllers, &ac97_controllers); dev_dbg(&ac97_ctrl->adap, "adapter registered by %s\n", @@ -361,14 +364,11 @@ struct ac97_controller *snd_ac97_controller_register( ret = ac97_add_adapter(ac97_ctrl); if (ret) - goto err; + return ERR_PTR(ret); ac97_bus_reset(ac97_ctrl); ac97_bus_scan(ac97_ctrl); return ac97_ctrl; -err: - kfree(ac97_ctrl); - return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(snd_ac97_controller_register); diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c index e839a4bb93f81..aa98caaaea3c5 100644 --- a/sound/core/oss/mixer_oss.c +++ b/sound/core/oss/mixer_oss.c @@ -525,6 +525,8 @@ static void snd_mixer_oss_get_volume1_vol(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -558,6 +560,8 @@ static void snd_mixer_oss_get_volume1_sw(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -618,6 +622,8 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -655,6 +661,8 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -792,6 +800,8 @@ static int snd_mixer_oss_get_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return -ENODEV; kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (!kctl) return -ENOENT; @@ -835,6 +845,8 @@ static int snd_mixer_oss_put_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return -ENODEV; kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (!kctl) return -ENOENT; @@ -878,6 +890,8 @@ static int snd_mixer_oss_build_test(struct snd_mixer_oss *mixer, struct slot *sl int err; scoped_guard(rwsem_read, &card->controls_rwsem) { + if (card->shutdown) + return -ENODEV; kcontrol = snd_mixer_oss_test_id(mixer, name, index); if (kcontrol == NULL) return 0; @@ -1002,6 +1016,8 @@ static int snd_mixer_oss_build_input(struct snd_mixer_oss *mixer, if (snd_mixer_oss_build_test_all(mixer, ptr, &slot)) return 0; guard(rwsem_read)(&mixer->card->controls_rwsem); + if (mixer->card->shutdown) + return -ENODEV; kctl = NULL; if (!ptr->index) kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index a82dd155e1d3a..b12df5b5ddfc1 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -1074,7 +1074,9 @@ static int snd_pcm_oss_change_params_locked(struct snd_pcm_substream *substream) runtime->oss.params = 0; runtime->oss.prepare = 1; runtime->oss.buffer_used = 0; - snd_pcm_runtime_buffer_set_silence(runtime); + err = snd_pcm_runtime_buffer_set_silence(runtime); + if (err < 0) + goto failure; runtime->oss.period_frames = snd_pcm_alsa_frames(substream, oss_period_size); diff --git a/sound/core/pcm.c b/sound/core/pcm.c index 283aac441fa0a..0b512085eb63f 100644 --- a/sound/core/pcm.c +++ b/sound/core/pcm.c @@ -328,13 +328,13 @@ static const char *snd_pcm_oss_format_name(int format) static void snd_pcm_proc_info_read(struct snd_pcm_substream *substream, struct snd_info_buffer *buffer) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; if (! substream) return; - info = kmalloc(sizeof(*info), GFP_KERNEL); + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); if (!info) return; diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c index 54eb9bd8eb218..e86f68f1f23c1 100644 --- a/sound/core/pcm_compat.c +++ b/sound/core/pcm_compat.c @@ -235,7 +235,6 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, int refine, struct snd_pcm_hw_params32 __user *data32) { - struct snd_pcm_hw_params *data __free(kfree) = NULL; struct snd_pcm_runtime *runtime; int err; @@ -243,7 +242,8 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, if (!runtime) return -ENOTTY; - data = kmalloc(sizeof(*data), GFP_KERNEL); + struct snd_pcm_hw_params *data __free(kfree) = + kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -332,7 +332,6 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, compat_caddr_t buf; compat_caddr_t __user *bufptr; u32 frames; - void __user **bufs __free(kfree) = NULL; int err, ch, i; if (! substream->runtime) @@ -349,7 +348,9 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, get_user(frames, &data32->frames)) return -EFAULT; bufptr = compat_ptr(buf); - bufs = kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < ch; i++) { diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 68bee40c9adaf..495ff93fcd1db 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -242,10 +242,10 @@ int snd_pcm_info(struct snd_pcm_substream *substream, struct snd_pcm_info *info) int snd_pcm_info_user(struct snd_pcm_substream *substream, struct snd_pcm_info __user * _info) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); - info = kmalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; err = snd_pcm_info(substream, info); @@ -364,7 +364,6 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, struct snd_pcm_hw_constraints *constrs = &substream->runtime->hw_constraints; unsigned int k; - unsigned int *rstamps __free(kfree) = NULL; unsigned int vstamps[SNDRV_PCM_HW_PARAM_LAST_INTERVAL + 1]; unsigned int stamp; struct snd_pcm_hw_rule *r; @@ -380,7 +379,8 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, * Each member of 'rstamps' array represents the sequence number of * recent application of corresponding rule. */ - rstamps = kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); + unsigned int *rstamps __free(kfree) = + kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); if (!rstamps) return -ENOMEM; @@ -583,10 +583,10 @@ EXPORT_SYMBOL(snd_pcm_hw_refine); static int snd_pcm_hw_refine_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -730,13 +730,18 @@ static void snd_pcm_buffer_access_unlock(struct snd_pcm_runtime *runtime) } /* fill the PCM buffer with the current silence format; called from pcm_oss.c */ -void snd_pcm_runtime_buffer_set_silence(struct snd_pcm_runtime *runtime) +int snd_pcm_runtime_buffer_set_silence(struct snd_pcm_runtime *runtime) { - snd_pcm_buffer_access_lock(runtime); + int err; + + err = snd_pcm_buffer_access_lock(runtime); + if (err < 0) + return err; if (runtime->dma_area) snd_pcm_format_set_silence(runtime->format, runtime->dma_area, bytes_to_samples(runtime, runtime->dma_bytes)); snd_pcm_buffer_access_unlock(runtime); + return 0; } EXPORT_SYMBOL_GPL(snd_pcm_runtime_buffer_set_silence); @@ -884,10 +889,10 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -2139,6 +2144,10 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, for (;;) { long tout; struct snd_pcm_runtime *to_check; + unsigned int drain_rate; + snd_pcm_uframes_t drain_bufsz; + bool drain_no_period_wakeup; + if (signal_pending(current)) { result = -ERESTARTSYS; break; @@ -2158,16 +2167,25 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, snd_pcm_group_unref(group, substream); if (!to_check) break; /* all drained */ + /* + * Cache the runtime fields needed after unlock. + * A concurrent close() on the linked stream may free + * its runtime via snd_pcm_detach_substream() once we + * release the stream lock below. + */ + drain_no_period_wakeup = to_check->no_period_wakeup; + drain_rate = to_check->rate; + drain_bufsz = to_check->buffer_size; init_waitqueue_entry(&wait, current); set_current_state(TASK_INTERRUPTIBLE); add_wait_queue(&to_check->sleep, &wait); snd_pcm_stream_unlock_irq(substream); - if (runtime->no_period_wakeup) + if (drain_no_period_wakeup) tout = MAX_SCHEDULE_TIMEOUT; else { tout = 100; - if (runtime->rate) { - long t = runtime->buffer_size * 1100 / runtime->rate; + if (drain_rate) { + long t = drain_bufsz * 1100 / drain_rate; tout = max(t, tout); } tout = msecs_to_jiffies(tout); @@ -2262,7 +2280,6 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) { struct snd_pcm_file *pcm_file; struct snd_pcm_substream *substream1; - struct snd_pcm_group *group __free(kfree) = NULL; struct snd_pcm_group *target_group; bool nonatomic = substream->pcm->nonatomic; CLASS(fd, f)(fd); @@ -2278,7 +2295,8 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) if (substream == substream1) return -EINVAL; - group = kzalloc(sizeof(*group), GFP_KERNEL); + struct snd_pcm_group *group __free(kfree) = + kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return -ENOMEM; snd_pcm_group_init(group); @@ -3572,7 +3590,6 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(to); @@ -3591,7 +3608,9 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) if (!frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < to->nr_segs; ++i) { @@ -3611,7 +3630,6 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(from); @@ -3629,7 +3647,9 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) !frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < from->nr_segs; ++i) { @@ -4101,15 +4121,15 @@ static void snd_pcm_hw_convert_to_old_params(struct snd_pcm_hw_params_old *opara static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); snd_pcm_hw_convert_from_old_params(params, oparams); @@ -4130,15 +4150,15 @@ static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); diff --git a/sound/core/vmaster.c b/sound/core/vmaster.c index c657659b236c4..76cc64245f5df 100644 --- a/sound/core/vmaster.c +++ b/sound/core/vmaster.c @@ -56,10 +56,10 @@ struct link_follower { static int follower_update(struct link_follower *follower) { - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err, ch; + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); if (!uctl) return -ENOMEM; uctl->id = follower->follower.id; @@ -74,7 +74,6 @@ static int follower_update(struct link_follower *follower) /* get the follower ctl info and save the initial values */ static int follower_init(struct link_follower *follower) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; int err; if (follower->info.count) { @@ -84,7 +83,8 @@ static int follower_init(struct link_follower *follower) return 0; } - uinfo = kmalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kmalloc(sizeof(*uinfo), GFP_KERNEL); if (!uinfo) return -ENOMEM; uinfo->id = follower->follower.id; @@ -341,9 +341,9 @@ static int master_get(struct snd_kcontrol *kcontrol, static int sync_followers(struct link_master *master, int old_val, int new_val) { struct link_follower *follower; - struct snd_ctl_elem_value *uval __free(kfree) = NULL; + struct snd_ctl_elem_value *uval __free(kfree) = + kmalloc(sizeof(*uval), GFP_KERNEL); - uval = kmalloc(sizeof(*uval), GFP_KERNEL); if (!uval) return -ENOMEM; list_for_each_entry(follower, &master->followers, list) { diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c index 64ef03b2d579c..aa0d2fcb1a180 100644 --- a/sound/drivers/aloop.c +++ b/sound/drivers/aloop.c @@ -336,37 +336,43 @@ static bool is_access_interleaved(snd_pcm_access_t access) static int loopback_check_format(struct loopback_cable *cable, int stream) { + struct loopback_pcm *dpcm_play, *dpcm_capt; struct snd_pcm_runtime *runtime, *cruntime; struct loopback_setup *setup; struct snd_card *card; + bool stop_capture = false; int check; - if (cable->valid != CABLE_VALID_BOTH) { - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - goto __notify; - return 0; - } - runtime = cable->streams[SNDRV_PCM_STREAM_PLAYBACK]-> - substream->runtime; - cruntime = cable->streams[SNDRV_PCM_STREAM_CAPTURE]-> - substream->runtime; - check = runtime->format != cruntime->format || - runtime->rate != cruntime->rate || - runtime->channels != cruntime->channels || - is_access_interleaved(runtime->access) != - is_access_interleaved(cruntime->access); - if (!check) - return 0; - if (stream == SNDRV_PCM_STREAM_CAPTURE) { - return -EIO; - } else { - snd_pcm_stop(cable->streams[SNDRV_PCM_STREAM_CAPTURE]-> - substream, SNDRV_PCM_STATE_DRAINING); - __notify: - runtime = cable->streams[SNDRV_PCM_STREAM_PLAYBACK]-> - substream->runtime; - setup = get_setup(cable->streams[SNDRV_PCM_STREAM_PLAYBACK]); - card = cable->streams[SNDRV_PCM_STREAM_PLAYBACK]->loopback->card; + scoped_guard(spinlock_irqsave, &cable->lock) { + dpcm_play = cable->streams[SNDRV_PCM_STREAM_PLAYBACK]; + dpcm_capt = cable->streams[SNDRV_PCM_STREAM_CAPTURE]; + + if (cable->valid != CABLE_VALID_BOTH) { + if (stream == SNDRV_PCM_STREAM_CAPTURE || !dpcm_play) + return 0; + } else { + if (!dpcm_play || !dpcm_capt) + return -EIO; + runtime = dpcm_play->substream->runtime; + cruntime = dpcm_capt->substream->runtime; + if (!runtime || !cruntime) + return -EIO; + check = runtime->format != cruntime->format || + runtime->rate != cruntime->rate || + runtime->channels != cruntime->channels || + is_access_interleaved(runtime->access) != + is_access_interleaved(cruntime->access); + if (!check) + return 0; + if (stream == SNDRV_PCM_STREAM_CAPTURE) + return -EIO; + else if (cruntime->state == SNDRV_PCM_STATE_RUNNING) + stop_capture = true; + } + + setup = get_setup(dpcm_play); + card = dpcm_play->loopback->card; + runtime = dpcm_play->substream->runtime; if (setup->format != runtime->format) { snd_ctl_notify(card, SNDRV_CTL_EVENT_MASK_VALUE, &setup->format_id); @@ -389,6 +395,10 @@ static int loopback_check_format(struct loopback_cable *cable, int stream) setup->access = runtime->access; } } + + if (stop_capture) + snd_pcm_stop(dpcm_capt->substream, SNDRV_PCM_STATE_DRAINING); + return 0; } diff --git a/sound/firewire/amdtp-stream.c b/sound/firewire/amdtp-stream.c index 5cdc34877fc19..76cc2b0801c9f 100644 --- a/sound/firewire/amdtp-stream.c +++ b/sound/firewire/amdtp-stream.c @@ -1179,7 +1179,7 @@ static void process_rx_packets(struct fw_iso_context *context, u32 tstamp, size_ struct pkt_desc *desc = s->packet_descs_cursor; unsigned int pkt_header_length; unsigned int packets; - u32 curr_cycle_time; + u32 curr_cycle_time = 0; bool need_hw_irq; int i; diff --git a/sound/firewire/dice/dice-extension.c b/sound/firewire/dice/dice-extension.c index 02f4a8318e38e..48bfb3ad93ce5 100644 --- a/sound/firewire/dice/dice-extension.c +++ b/sound/firewire/dice/dice-extension.c @@ -116,7 +116,7 @@ static int detect_stream_formats(struct snd_dice *dice, u64 section_addr) break; base_offset += EXT_APP_STREAM_ENTRIES; - stream_count = be32_to_cpu(reg[0]); + stream_count = min_t(unsigned int, be32_to_cpu(reg[0]), MAX_STREAMS); err = read_stream_entries(dice, section_addr, base_offset, stream_count, mode, dice->tx_pcm_chs, @@ -125,7 +125,7 @@ static int detect_stream_formats(struct snd_dice *dice, u64 section_addr) break; base_offset += stream_count * EXT_APP_STREAM_ENTRY_SIZE; - stream_count = be32_to_cpu(reg[1]); + stream_count = min_t(unsigned int, be32_to_cpu(reg[1]), MAX_STREAMS); err = read_stream_entries(dice, section_addr, base_offset, stream_count, mode, dice->rx_pcm_chs, diff --git a/sound/firewire/fireworks/fireworks_command.c b/sound/firewire/fireworks/fireworks_command.c index 2b595ee0bc357..05550f36fac55 100644 --- a/sound/firewire/fireworks/fireworks_command.c +++ b/sound/firewire/fireworks/fireworks_command.c @@ -151,10 +151,13 @@ efw_transaction(struct snd_efw *efw, unsigned int category, (be32_to_cpu(header->category) != category) || (be32_to_cpu(header->command) != command) || (be32_to_cpu(header->status) != EFR_STATUS_OK)) { + u32 st = be32_to_cpu(header->status); + dev_err(&efw->unit->device, "EFW command failed [%u/%u]: %s\n", be32_to_cpu(header->category), be32_to_cpu(header->command), - efr_status_names[be32_to_cpu(header->status)]); + st < ARRAY_SIZE(efr_status_names) ? + efr_status_names[st] : "unknown"); err = -EIO; goto end; } diff --git a/sound/firewire/motu/motu-hwdep.c b/sound/firewire/motu/motu-hwdep.c index 981c19430cb0f..89dc436a06529 100644 --- a/sound/firewire/motu/motu-hwdep.c +++ b/sound/firewire/motu/motu-hwdep.c @@ -75,7 +75,7 @@ static long hwdep_read(struct snd_hwdep *hwdep, char __user *buf, long count, while (consumed < count && snd_motu_register_dsp_message_parser_copy_event(motu, &ev)) { ptr = (u32 __user *)(buf + consumed); - if (put_user(ev, ptr)) + if (consumed + sizeof(ev) > count || put_user(ev, ptr)) return -EFAULT; consumed += sizeof(ev); } @@ -83,10 +83,11 @@ static long hwdep_read(struct snd_hwdep *hwdep, char __user *buf, long count, event.motu_register_dsp_change.type = SNDRV_FIREWIRE_EVENT_MOTU_REGISTER_DSP_CHANGE; event.motu_register_dsp_change.count = (consumed - sizeof(event.motu_register_dsp_change)) / 4; - if (copy_to_user(buf, &event, sizeof(event.motu_register_dsp_change))) + if (copy_to_user(buf, &event, + min_t(long, count, sizeof(event.motu_register_dsp_change)))) return -EFAULT; - count = consumed; + count = min_t(long, count, consumed); } else { spin_unlock_irq(&motu->lock); diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index 5fcbc1312c697..f71123a475464 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -299,6 +299,7 @@ enum { CXT_PINCFG_SWS_JS201D, CXT_PINCFG_TOP_SPEAKER, CXT_FIXUP_HP_A_U, + CXT_FIXUP_ACER_SWIFT_HP, }; /* for hda_fixup_thinkpad_acpi() */ @@ -1024,6 +1025,14 @@ static const struct hda_fixup cxt_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = cxt_fixup_hp_a_u, }, + [CXT_FIXUP_ACER_SWIFT_HP] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x16, 0x0321403f }, /* Headphone */ + { 0x19, 0x40f001f0 }, /* Mic */ + { } + }, + }, }; static const struct hda_quirk cxt5045_fixups[] = { @@ -1073,6 +1082,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x1025, 0x0543, "Acer Aspire One 522", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1025, 0x054c, "Acer Aspire 3830TG", CXT_FIXUP_ASPIRE_DMIC), SND_PCI_QUIRK(0x1025, 0x054f, "Acer Aspire 4830T", CXT_FIXUP_ASPIRE_DMIC), + SND_PCI_QUIRK(0x1025, 0x136d, "Acer Swift SF314", CXT_FIXUP_ACER_SWIFT_HP), SND_PCI_QUIRK(0x103c, 0x8079, "HP EliteBook 840 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x807C, "HP EliteBook 820 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x80FD, "HP ProBook 640 G2", CXT_FIXUP_HP_DOCK), @@ -1081,6 +1091,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x103c, 0x8174, "HP Spectre x360", CXT_FIXUP_HP_SPECTRE), SND_PCI_QUIRK(0x103c, 0x822e, "HP ProBook 440 G4", CXT_FIXUP_MUTE_LED_GPIO), SND_PCI_QUIRK(0x103c, 0x8231, "HP ProBook 450 G4", CXT_FIXUP_MUTE_LED_GPIO), + SND_PCI_QUIRK(0x103c, 0x826b, "HP ZBook Studio G4", CXT_FIXUP_MUTE_LED_GPIO), SND_PCI_QUIRK(0x103c, 0x828c, "HP EliteBook 840 G4", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x8299, "HP 800 G3 SFF", CXT_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x829a, "HP 800 G3 DM", CXT_FIXUP_HP_MIC_NO_PRESENCE), @@ -1123,6 +1134,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK_VENDOR(0x17aa, "Thinkpad/Ideapad", CXT_FIXUP_LENOVO_XPAD_ACPI), SND_PCI_QUIRK(0x1c06, 0x2011, "Lemote A1004", CXT_PINCFG_LEMOTE_A1004), SND_PCI_QUIRK(0x1c06, 0x2012, "Lemote A1205", CXT_PINCFG_LEMOTE_A1205), + SND_PCI_QUIRK(0x1d05, 0x3012, "MECHREVO Wujie 15X Pro", CXT_FIXUP_HEADSET_MIC), HDA_CODEC_QUIRK(0x2782, 0x12c3, "Sirius Gen1", CXT_PINCFG_TOP_SPEAKER), HDA_CODEC_QUIRK(0x2782, 0x12c5, "Sirius Gen2", CXT_PINCFG_TOP_SPEAKER), {} diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c index 111c9b5335afc..c2e3adc7b3c00 100644 --- a/sound/hda/codecs/hdmi/hdmi.c +++ b/sound/hda/codecs/hdmi/hdmi.c @@ -1557,6 +1557,7 @@ static const struct snd_pci_quirk force_connect_list[] = { SND_PCI_QUIRK(0x1043, 0x86ae, "ASUS", 1), /* Z170 PRO */ SND_PCI_QUIRK(0x1043, 0x86c7, "ASUS", 1), /* Z170M PLUS */ SND_PCI_QUIRK(0x1462, 0xec94, "MS-7C94", 1), + SND_PCI_QUIRK(0x1558, 0x14a1, "TUXEDO InfinityBook S 14 Gen6", 1), SND_PCI_QUIRK(0x8086, 0x2060, "Intel NUC5CPYB", 1), SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", 1), {} diff --git a/sound/hda/codecs/hdmi/tegrahdmi.c b/sound/hda/codecs/hdmi/tegrahdmi.c index 5f6fe31aa2028..ebb6410a48313 100644 --- a/sound/hda/codecs/hdmi/tegrahdmi.c +++ b/sound/hda/codecs/hdmi/tegrahdmi.c @@ -299,6 +299,7 @@ static const struct hda_device_id snd_hda_id_tegrahdmi[] = { HDA_CODEC_ID_MODEL(0x10de002f, "Tegra194 HDMI/DP2", MODEL_TEGRA), HDA_CODEC_ID_MODEL(0x10de0030, "Tegra194 HDMI/DP3", MODEL_TEGRA), HDA_CODEC_ID_MODEL(0x10de0031, "Tegra234 HDMI/DP", MODEL_TEGRA234), + HDA_CODEC_ID_MODEL(0x10de0032, "Tegra238 HDMI/DP", MODEL_TEGRA234), HDA_CODEC_ID_MODEL(0x10de0033, "SoC 33 HDMI/DP", MODEL_TEGRA234), HDA_CODEC_ID_MODEL(0x10de0034, "Tegra264 HDMI/DP", MODEL_TEGRA234), HDA_CODEC_ID_MODEL(0x10de0035, "SoC 35 HDMI/DP", MODEL_TEGRA234), diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index b45fcc9a3785e..8e8ba928d3c02 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -1017,6 +1017,50 @@ static int alc269_resume(struct hda_codec *codec) return 0; } +#define ALC233_STARFIGHTER_SPK_PIN 0x1b +#define ALC233_STARFIGHTER_GPIO2 0x04 + +static void alc233_starfighter_update_amp(struct hda_codec *codec, bool on) +{ + snd_hda_codec_write(codec, ALC233_STARFIGHTER_SPK_PIN, 0, + AC_VERB_SET_EAPD_BTLENABLE, + on ? AC_EAPDBTL_EAPD : 0); + alc_update_gpio_data(codec, ALC233_STARFIGHTER_GPIO2, on); +} + +static void alc233_starfighter_pcm_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + switch (action) { + case HDA_GEN_PCM_ACT_PREPARE: + alc233_starfighter_update_amp(codec, true); + break; + case HDA_GEN_PCM_ACT_CLEANUP: + alc233_starfighter_update_amp(codec, false); + break; + } +} + +static void alc233_fixup_starlabs_starfighter(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + struct alc_spec *spec = codec->spec; + + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + spec->gpio_mask |= ALC233_STARFIGHTER_GPIO2; + spec->gpio_dir |= ALC233_STARFIGHTER_GPIO2; + spec->gpio_data &= ~ALC233_STARFIGHTER_GPIO2; + break; + case HDA_FIXUP_ACT_PROBE: + spec->gen.pcm_playback_hook = alc233_starfighter_pcm_hook; + break; + } +} + static void alc269_fixup_pincfg_no_hp_to_lineout(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1551,6 +1595,22 @@ static void alc245_fixup_hp_mute_led_v1_coefbit(struct hda_codec *codec, } } +static void alc245_fixup_hp_mute_led_v2_coefbit(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + struct alc_spec *spec = codec->spec; + + if (action == HDA_FIXUP_ACT_PRE_PROBE) { + spec->mute_led_polarity = 0; + spec->mute_led_coef.idx = 0x0b; + spec->mute_led_coef.mask = 1 << 3; + spec->mute_led_coef.on = 1 << 3; + spec->mute_led_coef.off = 0; + snd_hda_gen_add_mute_led_cdev(codec, coef_mute_led_set); + } +} + /* turn on/off mic-mute LED per capture hook by coef bit */ static int coef_micmute_led_set(struct led_classdev *led_cdev, enum led_brightness brightness) @@ -1630,6 +1690,13 @@ static void alc285_fixup_hp_spectre_x360_mute_led(struct hda_codec *codec, alc285_fixup_hp_gpio_micmute_led(codec, fix, action); } +static void alc245_fixup_hp_envy_x360_mute_led(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + alc245_fixup_hp_mute_led_v1_coefbit(codec, fix, action); + alc245_fixup_hp_gpio_led(codec, fix, action); +} + static void alc236_fixup_hp_mute_led(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1812,6 +1879,163 @@ static void alc298_samsung_v2_init_amps(struct hda_codec *codec, spec->gen.pcm_playback_hook = alc298_samsung_v2_playback_hook; } +/* LG Gram Style 14: program vendor coef sequence used by HDA-verb workaround */ +struct alc298_lg_gram_style_seq { + unsigned short verb; + unsigned short idx; + unsigned short val; +}; + +static void alc298_lg_gram_style_coef_write(struct hda_codec *codec, + unsigned int verb, + unsigned int idx, + unsigned int val) +{ + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_COEF_INDEX, 0x23); + snd_hda_codec_write(codec, 0x20, 0, verb, idx); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, 0x00); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, val); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, 0xb011); +} + +static void alc298_lg_gram_style_run_seq(struct hda_codec *codec, + const struct alc298_lg_gram_style_seq *seq, + int seq_size) +{ + int i; + + for (i = 0; i < seq_size; i++) + alc298_lg_gram_style_coef_write(codec, seq[i].verb, + seq[i].idx, seq[i].val); +} + +/* Coef sequences derived from the HDA-verb workaround for this model. */ +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_preinit_seq[] = { + { 0x420, 0x00, 0x01 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_disable_seq[] = { + { 0x423, 0xff, 0x00 }, + { 0x420, 0x3a, 0x80 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_enable_seq[] = { + { 0x420, 0x3a, 0x81 }, + { 0x423, 0xff, 0x01 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_38[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x01 }, { 0x420, 0x1d, 0x01 }, { 0x420, 0x1f, 0xfe }, + { 0x420, 0x21, 0x00 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0x99, 0x03 }, + { 0x423, 0xa4, 0xb5 }, { 0x423, 0xa5, 0x01 }, { 0x423, 0xba, 0x94 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_39[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x02 }, { 0x420, 0x1d, 0x02 }, { 0x420, 0x1f, 0xfd }, + { 0x420, 0x21, 0x01 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0x99, 0x03 }, + { 0x423, 0xa4, 0xb5 }, { 0x423, 0xa5, 0x01 }, { 0x423, 0xba, 0x94 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_3c[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x01 }, { 0x420, 0x1d, 0x01 }, { 0x420, 0x1f, 0xfe }, + { 0x420, 0x21, 0x00 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0xba, 0x8d }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_3d[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x02 }, { 0x420, 0x1d, 0x02 }, { 0x420, 0x1f, 0xfd }, + { 0x420, 0x21, 0x01 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0xba, 0x8d }, +}; + +struct alc298_lg_gram_style_amp_desc { + unsigned char nid; + const struct alc298_lg_gram_style_seq *init_seq; + int init_seq_size; +}; + +static const struct alc298_lg_gram_style_amp_desc alc298_lg_gram_style_amps[] = { + { 0x38, alc298_lg_gram_style_init_seq_38, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_38) }, + { 0x39, alc298_lg_gram_style_init_seq_39, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_39) }, + { 0x3c, alc298_lg_gram_style_init_seq_3c, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_3c) }, + { 0x3d, alc298_lg_gram_style_init_seq_3d, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_3d) }, +}; + +static void alc298_lg_gram_style_enable_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_enable_seq, + ARRAY_SIZE(alc298_lg_gram_style_enable_seq)); + } +} + +static void alc298_lg_gram_style_disable_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_disable_seq, + ARRAY_SIZE(alc298_lg_gram_style_disable_seq)); + } +} + +static void alc298_lg_gram_style_playback_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + if (action == HDA_GEN_PCM_ACT_OPEN) + alc298_lg_gram_style_enable_amps(codec); + if (action == HDA_GEN_PCM_ACT_CLOSE) + alc298_lg_gram_style_disable_amps(codec); +} + +static void alc298_lg_gram_style_init_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + spec->num_speaker_amps = ARRAY_SIZE(alc298_lg_gram_style_amps); + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_preinit_seq, + ARRAY_SIZE(alc298_lg_gram_style_preinit_seq)); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_disable_seq, + ARRAY_SIZE(alc298_lg_gram_style_disable_seq)); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_amps[i].init_seq, + alc298_lg_gram_style_amps[i].init_seq_size); + alc_write_coef_idx(codec, 0x89, 0x0); + } + + spec->gen.pcm_playback_hook = alc298_lg_gram_style_playback_hook; +} + static void alc298_fixup_samsung_amp_v2_2_amps(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1826,6 +2050,13 @@ static void alc298_fixup_samsung_amp_v2_4_amps(struct hda_codec *codec, alc298_samsung_v2_init_amps(codec, 4); } +static void alc298_fixup_lg_gram_style_14(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + if (action == HDA_FIXUP_ACT_PROBE) + alc298_lg_gram_style_init_amps(codec); +} + static void gpio2_mic_hotkey_event(struct hda_codec *codec, struct hda_jack_callback *event) { @@ -3371,11 +3602,22 @@ static void alc287_alc1318_playback_pcm_hook(struct hda_pcm_stream *hinfo, struct snd_pcm_substream *substream, int action) { + static const struct coef_fw dis_coefs[] = { + WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC203), + WRITE_COEF(0x28, 0x0004), WRITE_COEF(0x29, 0xb023), + }; /* Disable AMP silence detection */ + static const struct coef_fw en_coefs[] = { + WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC203), + WRITE_COEF(0x28, 0x0084), WRITE_COEF(0x29, 0xb023), + }; /* Enable AMP silence detection */ + switch (action) { case HDA_GEN_PCM_ACT_OPEN: + alc_process_coef_fw(codec, dis_coefs); alc_write_coefex_idx(codec, 0x5a, 0x00, 0x954f); /* write gpio3 to high */ break; case HDA_GEN_PCM_ACT_CLOSE: + alc_process_coef_fw(codec, en_coefs); alc_write_coefex_idx(codec, 0x5a, 0x00, 0x554f); /* write gpio3 as default value */ break; } @@ -3579,6 +3821,7 @@ enum { ALC294_FIXUP_ASUS_MIC, ALC294_FIXUP_ASUS_HEADSET_MIC, ALC294_FIXUP_ASUS_I2C_HEADSET_MIC, + ALC294_FIXUP_ASUS_SPI_HEADSET_MIC, ALC294_FIXUP_ASUS_SPK, ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE, ALC285_FIXUP_LENOVO_PC_BEEP_IN_NOISE, @@ -3619,6 +3862,7 @@ enum { ALC285_FIXUP_HP_GPIO_LED, ALC285_FIXUP_HP_MUTE_LED, ALC285_FIXUP_HP_SPECTRE_X360_MUTE_LED, + ALC245_FIXUP_HP_ENVY_X360_MUTE_LED, ALC285_FIXUP_HP_BEEP_MICMUTE_LED, ALC236_FIXUP_HP_MUTE_LED_COEFBIT2, ALC236_FIXUP_HP_GPIO_LED, @@ -3628,6 +3872,7 @@ enum { ALC298_FIXUP_SAMSUNG_AMP, ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS, ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS, + ALC298_FIXUP_LG_GRAM_STYLE_14, ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC295_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -3674,6 +3919,7 @@ enum { ALC287_FIXUP_LEGION_15IMHG05_AUTOMUTE, ALC287_FIXUP_YOGA7_14ITL_SPEAKERS, ALC298_FIXUP_LENOVO_C940_DUET7, + ALC287_FIXUP_LENOVO_YOGA_BOOK_9I, ALC287_FIXUP_13S_GEN2_SPEAKERS, ALC256_FIXUP_SET_COEF_DEFAULTS, ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE, @@ -3703,9 +3949,11 @@ enum { ALC295_FIXUP_DELL_TAS2781_I2C, ALC245_FIXUP_TAS2781_SPI_2, ALC287_FIXUP_TXNW2781_I2C, + ALC287_FIXUP_TXNW2781_I2C_ASUS, ALC287_FIXUP_YOGA7_14ARB7_I2C, ALC245_FIXUP_HP_MUTE_LED_COEFBIT, ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT, + ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT, ALC245_FIXUP_HP_X360_MUTE_LEDS, ALC287_FIXUP_THINKPAD_I2S_SPK, ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD, @@ -3727,6 +3975,7 @@ enum { ALC245_FIXUP_CLEVO_NOISY_MIC, ALC269_FIXUP_VAIO_VJFH52_MIC_NO_PRESENCE, ALC233_FIXUP_MEDION_MTL_SPK, + ALC233_FIXUP_STARLABS_STARFIGHTER, ALC294_FIXUP_BASS_SPEAKER_15, ALC283_FIXUP_DELL_HP_RESUME, ALC294_FIXUP_ASUS_CS35L41_SPI_2, @@ -3756,6 +4005,23 @@ static void alc298_fixup_lenovo_c940_duet7(struct hda_codec *codec, __snd_hda_apply_fixup(codec, id, action, 0); } +/* A special fixup for Lenovo Yoga 9i and Yoga Book 9i 13IRU8 + * both have the very same PCI SSID and vendor ID, so we need + * to apply different fixups depending on the subsystem ID + */ +static void alc287_fixup_lenovo_yoga_book_9i(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + int id; + + if (codec->core.subsystem_id == 0x17aa3881) + id = ALC287_FIXUP_TAS2781_I2C; /* Yoga Book 9i 13IRU8 */ + else + id = ALC287_FIXUP_IDEAPAD_BASS_SPK_AMP; /* Yoga 9i */ + __snd_hda_apply_fixup(codec, id, action, 0); +} + static const struct hda_fixup alc269_fixups[] = { [ALC269_FIXUP_GPIO2] = { .type = HDA_FIXUP_FUNC, @@ -4901,6 +5167,15 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC287_FIXUP_CS35L41_I2C_2 }, + [ALC294_FIXUP_ASUS_SPI_HEADSET_MIC] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x19, 0x04a11020 }, /* use as headset mic */ + { } + }, + .chained = true, + .chain_id = ALC245_FIXUP_CS35L41_SPI_2 + }, [ALC294_FIXUP_ASUS_SPK] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -5248,6 +5523,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_spectre_x360_mute_led, }, + [ALC245_FIXUP_HP_ENVY_X360_MUTE_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc245_fixup_hp_envy_x360_mute_led, + }, [ALC285_FIXUP_HP_BEEP_MICMUTE_LED] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_beep, @@ -5294,6 +5573,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc298_fixup_samsung_amp_v2_4_amps }, + [ALC298_FIXUP_LG_GRAM_STYLE_14] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc298_fixup_lg_gram_style_14 + }, [ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -5763,6 +6046,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc298_fixup_lenovo_c940_duet7, }, + [ALC287_FIXUP_LENOVO_YOGA_BOOK_9I] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc287_fixup_lenovo_yoga_book_9i, + }, [ALC287_FIXUP_13S_GEN2_SPEAKERS] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -5993,6 +6280,12 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC285_FIXUP_THINKPAD_HEADSET_JACK, }, + [ALC287_FIXUP_TXNW2781_I2C_ASUS] = { + .type = HDA_FIXUP_FUNC, + .v.func = tas2781_fixup_txnw_i2c, + .chained = true, + .chain_id = ALC294_FIXUP_ASUS_SPK, + }, [ALC287_FIXUP_YOGA7_14ARB7_I2C] = { .type = HDA_FIXUP_FUNC, .v.func = yoga7_14arb7_fixup_i2c, @@ -6007,6 +6300,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc245_fixup_hp_mute_led_v1_coefbit, }, + [ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc245_fixup_hp_mute_led_v2_coefbit, + }, [ALC245_FIXUP_HP_X360_MUTE_LEDS] = { .type = HDA_FIXUP_FUNC, .v.func = alc245_fixup_hp_mute_led_coefbit, @@ -6128,6 +6425,10 @@ static const struct hda_fixup alc269_fixups[] = { { } }, }, + [ALC233_FIXUP_STARLABS_STARFIGHTER] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc233_fixup_starlabs_starfighter, + }, [ALC294_FIXUP_BASS_SPEAKER_15] = { .type = HDA_FIXUP_FUNC, .v.func = alc294_fixup_bass_speaker_15, @@ -6199,6 +6500,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x079b, "Acer Aspire V5-573G", ALC282_FIXUP_ASPIRE_V5_PINS), SND_PCI_QUIRK(0x1025, 0x080d, "Acer Aspire V5-122P", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x0840, "Acer Aspire E1", ALC269VB_FIXUP_ASPIRE_E1_COEF), + SND_PCI_QUIRK(0x1025, 0x0943, "Acer Aspire V3-572G", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x100c, "Acer Aspire E5-574G", ALC255_FIXUP_ACER_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x1025, 0x101c, "Acer Veriton N2510G", ALC269_FIXUP_LIFEBOOK), SND_PCI_QUIRK(0x1025, 0x102b, "Acer Aspire C24-860", ALC286_FIXUP_ACER_AIO_MIC_NO_PRESENCE), @@ -6231,7 +6533,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x1430, "Acer TravelMate B311R-31", ALC256_FIXUP_ACER_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1466, "Acer Aspire A515-56", ALC255_FIXUP_ACER_HEADPHONE_AND_MIC), SND_PCI_QUIRK(0x1025, 0x1534, "Acer Predator PH315-54", ALC255_FIXUP_ACER_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x1025, 0x1539, "Acer Nitro 5 AN515-57", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1025, 0x1597, "Acer Nitro 5 AN517-55", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x169a, "Acer Swift SFG16", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x1826, "Acer Helios ZPC", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), SND_PCI_QUIRK(0x1025, 0x182c, "Acer Helios ZPD", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), @@ -6419,6 +6723,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x863e, "HP Spectre x360 15-df1xxx", ALC285_FIXUP_HP_SPECTRE_X360_DF1), SND_PCI_QUIRK(0x103c, 0x86e8, "HP Spectre x360 15-eb0xxx", ALC285_FIXUP_HP_SPECTRE_X360_EB1), SND_PCI_QUIRK(0x103c, 0x86f9, "HP Spectre x360 13-aw0xxx", ALC285_FIXUP_HP_SPECTRE_X360_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x8706, "HP Laptop 15s-eq1xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8716, "HP Elite Dragonfly G2 Notebook PC", ALC285_FIXUP_HP_GPIO_AMP_INIT), SND_PCI_QUIRK(0x103c, 0x8720, "HP EliteBook x360 1040 G8 Notebook PC", ALC285_FIXUP_HP_GPIO_AMP_INIT), SND_PCI_QUIRK(0x103c, 0x8724, "HP EliteBook 850 G7", ALC285_FIXUP_HP_GPIO_LED), @@ -6427,6 +6732,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8730, "HP ProBook 445 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8735, "HP ProBook 435 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8736, "HP", ALC285_FIXUP_HP_GPIO_AMP_INIT), + SND_PCI_QUIRK(0x103c, 0x8756, "HP ENVY Laptop 13-ba0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x8760, "HP EliteBook 8{4,5}5 G7", ALC285_FIXUP_HP_BEEP_MICMUTE_LED), SND_PCI_QUIRK(0x103c, 0x876e, "HP ENVY x360 Convertible 13-ay0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x877a, "HP", ALC285_FIXUP_HP_MUTE_LED), @@ -6440,6 +6746,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8788, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x87b7, "HP Laptop 14-fq0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x87c8, "HP", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x87cb, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87cc, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87d3, "HP Laptop 15-gw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x87df, "HP ProBook 430 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), @@ -6475,8 +6782,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8895, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_SPEAKERS_MICMUTE_LED), SND_PCI_QUIRK(0x103c, 0x8896, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8898, "HP EliteBook 845 G8 Notebook PC", ALC285_FIXUP_HP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x103c, 0x88b3, "HP ENVY x360 Convertible 15-es0xxx", ALC245_FIXUP_HP_ENVY_X360_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x88d0, "HP Pavilion 15-eh1xxx (mainboard 88D0)", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x88d1, "HP Pavilion 15-eh1xxx (mainboard 88D1)", ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT), SND_PCI_QUIRK(0x103c, 0x88dd, "HP Pavilion 15z-ec200", ALC285_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x88eb, "HP Victus 16-e0xxx", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8902, "HP OMEN 16", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x890e, "HP 255 G8 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8919, "HP Pavilion Aero Laptop 13-be0xxx", ALC287_FIXUP_HP_GPIO_LED), @@ -6510,6 +6820,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x89da, "HP Spectre x360 14t-ea100", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX), SND_PCI_QUIRK(0x103c, 0x89e7, "HP Elite x2 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8a0f, "HP Pavilion 14-ec1xxx", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8a1f, "HP Laptop 14s-dr5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8a20, "HP Laptop 15s-fq5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8a25, "HP Victus 16-d1xxx (MB 8A25)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a26, "HP Victus 16-d1xxx (MB 8A26)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), @@ -6522,6 +6833,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8a2e, "HP Envy 16", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8a30, "HP Envy 17", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8a31, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x103c, 0x8a34, "HP Pavilion x360 2-in-1 Laptop 14-ek0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x8a3d, "HP Victus 15-fb0xxx (MB 8A3D)", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a4f, "HP Victus 15-fa0xxx (MB 8A4F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a6e, "HP EDNA 360", ALC287_FIXUP_CS35L41_I2C_4), SND_PCI_QUIRK(0x103c, 0x8a74, "HP ProBook 440 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), @@ -6623,6 +6936,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8c8c, "HP EliteBook 660", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8c8d, "HP ProBook 440 G11", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8c8e, "HP ProBook 460 G11", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8c8f, "HP EliteBook 630 G11", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8c90, "HP EliteBook 640", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8c91, "HP EliteBook 660", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8c96, "HP", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), @@ -6663,6 +6977,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8da7, "HP 14 Enstrom OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8dd4, "HP EliteStudio 8 AIO", ALC274_FIXUP_HP_AIO_BIND_DACS), + SND_PCI_QUIRK(0x103c, 0x8dd7, "HP Laptop 15-fd0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8de8, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2), SND_PCI_QUIRK(0x103c, 0x8de9, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2), SND_PCI_QUIRK(0x103c, 0x8dec, "HP EliteBook 640 G12", ALC236_FIXUP_HP_GPIO_LED), @@ -6722,6 +7037,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x115d, "Asus 1015E", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x1043, 0x1194, "ASUS UM3406KA", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x11c0, "ASUS X556UR", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), + HDA_CODEC_QUIRK(0x1043, 0x1204, "ASUS Strix G16 G615JMR", ALC287_FIXUP_TXNW2781_I2C_ASUS), SND_PCI_QUIRK(0x1043, 0x1204, "ASUS Strix G615JHR_JMR_JPR", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x1043, 0x1214, "ASUS Strix G615LH_LM_LP", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x1043, 0x125e, "ASUS Q524UQK", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), @@ -6736,6 +7052,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x12f0, "ASUS X541UV", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1313, "Asus K42JZ", ALC269VB_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1314, "ASUS GA605K", ALC285_FIXUP_ASUS_GA605K_HEADSET_MIC), + SND_PCI_QUIRK(0x1043, 0x1384, "ASUS RC73XA", ALC287_FIXUP_TXNW2781_I2C_ASUS), + SND_PCI_QUIRK(0x1043, 0x1394, "ASUS RC73YA", ALC287_FIXUP_TXNW2781_I2C_ASUS), SND_PCI_QUIRK(0x1043, 0x13b0, "ASUS Z550SA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1427, "Asus Zenbook UX31E", ALC269VB_FIXUP_ASUS_ZENBOOK), SND_PCI_QUIRK(0x1043, 0x1433, "ASUS GX650PY/PZ/PV/PU/PYV/PZV/PIV/PVV", ALC285_FIXUP_ASUS_I2C_HEADSET_MIC), @@ -6749,9 +7067,12 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x14e3, "ASUS G513PI/PU/PV", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x14f2, "ASUS VivoBook X515JA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1503, "ASUS G733PY/PZ/PZV/PYV", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x1043, 0x1514, "ASUS ROG Flow Z13 GZ302EAC", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1517, "Asus Zenbook UX31A", ALC269VB_FIXUP_ASUS_ZENBOOK_UX31A), SND_PCI_QUIRK(0x1043, 0x1533, "ASUS GV302XA/XJ/XQ/XU/XV/XI", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1573, "ASUS GZ301VV/VQ/VU/VJ/VA/VC/VE/VVC/VQC/VUC/VJC/VEC/VCC", ALC285_FIXUP_ASUS_HEADSET_MIC), + SND_PCI_QUIRK(0x1043, 0x1584, "ASUS UM3406GA ", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x1043, 0x1602, "ASUS ROG Strix SCAR 15", ALC285_FIXUP_ASUS_G533Z_PINS), SND_PCI_QUIRK(0x1043, 0x1652, "ASUS ROG Zephyrus Do 15 SE", ALC289_FIXUP_ASUS_ZEPHYRUS_DUAL_SPK), SND_PCI_QUIRK(0x1043, 0x1662, "ASUS GV301QH", ALC294_FIXUP_ASUS_DUAL_SPK), SND_PCI_QUIRK(0x1043, 0x1663, "ASUS GU603ZI/ZJ/ZQ/ZU/ZV", ALC285_FIXUP_ASUS_HEADSET_MIC), @@ -6775,7 +7096,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x19ce, "ASUS B9450FA", ALC294_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x19e1, "ASUS UX581LV", ALC295_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1a13, "Asus G73Jw", ALC269_FIXUP_ASUS_G73JW), - SND_PCI_QUIRK(0x1043, 0x1a63, "ASUS UX3405MA", ALC245_FIXUP_CS35L41_SPI_2), + SND_PCI_QUIRK(0x1043, 0x1a63, "ASUS UX3405MA", ALC294_FIXUP_ASUS_SPI_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x1a83, "ASUS UM5302LA", ALC294_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1a8e, "ASUS G712LWS", ALC294_FIXUP_LENOVO_MIC_LOCATION), SND_PCI_QUIRK(0x1043, 0x1a8f, "ASUS UX582ZS", ALC245_FIXUP_CS35L41_SPI_2), @@ -6816,6 +7137,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1e93, "ASUS ExpertBook B9403CVAR", ALC294_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x1eb3, "ASUS Ally RCLA72", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x1043, 0x1ed3, "ASUS HN7306W", ALC287_FIXUP_CS35L41_I2C_2), + HDA_CODEC_QUIRK(0x1043, 0x1ee2, "ASUS UM6702RA/RC", ALC285_FIXUP_ASUS_I2C_SPEAKER2_TO_DAC1), SND_PCI_QUIRK(0x1043, 0x1ee2, "ASUS UM6702RA/RC", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1c52, "ASUS Zephyrus G15 2022", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1f11, "ASUS Zephyrus G14", ALC289_FIXUP_ASUS_GA401), @@ -6886,8 +7208,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc109, "Samsung Ativ book 9 (NP900X3G)", ALC269_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x144d, 0xc169, "Samsung Notebook 9 Pen (NP930SBE-K01US)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc176, "Samsung Notebook 9 Pro (NP930MBE-K04US)", ALC298_FIXUP_SAMSUNG_AMP), - SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Flex Book (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc188, "Samsung Galaxy Book Flex (NT950QCT-A38A)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Book Flex (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc18a, "Samsung Galaxy Book Ion (NP930XCJ-K01US)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc1ac, "Samsung Galaxy Book2 Pro 360 (NP950QED)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc1a3, "Samsung Galaxy Book Pro (NP935XDB-KC1SE)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc1a4, "Samsung Galaxy Book Pro 360 (NT935QBD)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc1a6, "Samsung Galaxy Book Pro 360 (NP930QBD)", ALC298_FIXUP_SAMSUNG_AMP), @@ -6895,6 +7219,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc812, "Samsung Notebook Pen S (NT950SBE-X58)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc830, "Samsung Galaxy Book Ion (NT950XCJ-X716A)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc832, "Samsung Galaxy Book Flex Alpha (NP730QCJ)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), + SND_PCI_QUIRK(0x144d, 0xc876, "Samsung 730QED (NP730QED-KA2US)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xca03, "Samsung Galaxy Book2 Pro 360 (NP930QED)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xca06, "Samsung Galaxy Book3 360 (NP730QFG)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xc868, "Samsung Galaxy Book2 Pro (NP930XED)", ALC298_FIXUP_SAMSUNG_AMP), @@ -6902,7 +7227,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc872, "Samsung Galaxy Book2 Pro (NP950XEE)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc886, "Samsung Galaxy Book3 Pro (NP964XFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1ca, "Samsung Galaxy Book3 Pro 360 (NP960QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc1cb, "Samsung Galaxy Book3 Pro 360 (NP965QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1cc, "Samsung Galaxy Book3 Ultra (NT960XFH)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1458, 0x900e, "Gigabyte G5 KF5 (2023)", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC), @@ -7031,6 +7358,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x224c, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x224d, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x225d, "Thinkpad T480", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x17aa, 0x2288, "Thinkpad X390", ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK), SND_PCI_QUIRK(0x17aa, 0x2292, "Thinkpad X1 Carbon 7th", ALC285_FIXUP_THINKPAD_HEADSET_JACK), SND_PCI_QUIRK(0x17aa, 0x22be, "Thinkpad X1 Carbon 8th", ALC285_FIXUP_THINKPAD_HEADSET_JACK), SND_PCI_QUIRK(0x17aa, 0x22c1, "Thinkpad P1 Gen 3", ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK), @@ -7064,6 +7392,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3801, "Lenovo Yoga9 14IAP7", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), HDA_CODEC_QUIRK(0x17aa, 0x3802, "DuetITL 2021", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3802, "Lenovo Yoga Pro 9 14IRP8", ALC287_FIXUP_TAS2781_I2C), + SND_PCI_QUIRK(0x17aa, 0x3811, "Legion S7 15IMH05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3813, "Legion 7i 15IMHG05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3818, "Lenovo C940 / Yoga Duet 7", ALC298_FIXUP_LENOVO_C940_DUET7), SND_PCI_QUIRK(0x17aa, 0x3819, "Lenovo 13s Gen2 ITL", ALC287_FIXUP_13S_GEN2_SPEAKERS), @@ -7073,7 +7402,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3827, "Ideapad S740", ALC285_FIXUP_IDEAPAD_S740_COEF), SND_PCI_QUIRK(0x17aa, 0x3834, "Lenovo IdeaPad Slim 9i 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x383d, "Legion Y9000X 2019", ALC285_FIXUP_LEGION_Y9000X_SPEAKERS), - SND_PCI_QUIRK(0x17aa, 0x3843, "Yoga 9i", ALC287_FIXUP_IDEAPAD_BASS_SPK_AMP), + SND_PCI_QUIRK(0x17aa, 0x3843, "Lenovo Yoga 9i / Yoga Book 9i", ALC287_FIXUP_LENOVO_YOGA_BOOK_9I), + /* Yoga Pro 7 14IMH9 shares PCI SSID 17aa:3847 with Legion 7 16ACHG6; + * use codec SSID to distinguish them + */ + HDA_CODEC_QUIRK(0x17aa, 0x38cf, "Lenovo Yoga Pro 7 14IMH9", ALC287_FIXUP_YOGA9_14IMH9_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3847, "Legion 7 16ACHG6", ALC287_FIXUP_LEGION_16ACHG6), SND_PCI_QUIRK(0x17aa, 0x384a, "Lenovo Yoga 7 15ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3852, "Lenovo Yoga 7 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), @@ -7105,6 +7438,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38ab, "Thinkbook 16P", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), SND_PCI_QUIRK(0x17aa, 0x38b4, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), HDA_CODEC_QUIRK(0x17aa, 0x391c, "Lenovo Yoga 7 2-in-1 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), + HDA_CODEC_QUIRK(0x17aa, 0x391d, "Lenovo Yoga 7 2-in-1 16AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x38b5, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x38b6, "Legion Slim 7 16APH8", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x38b7, "Legion Slim 7 16APH8", ALC287_FIXUP_CS35L41_I2C_2), @@ -7134,7 +7468,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38fd, "ThinkBook plus Gen5 Hybrid", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI), SND_PCI_QUIRK(0x17aa, 0x390d, "Lenovo Yoga Pro 7 14ASP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), + SND_PCI_QUIRK(0x17aa, 0x3911, "Lenovo Yoga Pro 7 14IAH10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3913, "Lenovo 145", ALC236_FIXUP_LENOVO_INV_DMIC), + SND_PCI_QUIRK(0x17aa, 0x391a, "Lenovo Yoga Slim 7 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x391f, "Yoga S990-16 pro Quad YC Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), @@ -7171,6 +7507,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1854, 0x0488, "LG gram 16 (16Z90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x1854, 0x0489, "LG gram 16 (16Z90R-A)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x1854, 0x048a, "LG gram 17 (17ZD90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1854, 0x0490, "LG Gram Style 14 (14Z90RS)", ALC298_FIXUP_LG_GRAM_STYLE_14), SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x19e5, 0x3212, "Huawei KLV-WX9 ", ALC256_FIXUP_ACER_HEADSET_MIC), @@ -7195,15 +7532,18 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1d05, 0x1409, "TongFang GMxIXxx", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x300f, "TongFang X6AR5xxY", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x3019, "TongFang X6FR5xxY", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x3031, "TongFang X6AR55xU", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d17, 0x3288, "Haier Boyue G42", ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS), SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1945, "Redmi G", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC), + SND_PCI_QUIRK(0x1e39, 0xca14, "MEDION NM14LNL", ALC233_FIXUP_MEDION_MTL_SPK), SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1f66, 0x0105, "Ayaneo Portable Game Player", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x2039, 0x0001, "Inspur S14-G1", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x2782, 0x0214, "VAIO VJFE-CL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2782, 0x0228, "Infinix ZERO BOOK 13", ALC269VB_FIXUP_INFINIX_ZERO_BOOK_13), SND_PCI_QUIRK(0x2782, 0x0232, "CHUWI CoreBook XPro", ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO), @@ -7213,6 +7553,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x2782, 0x1705, "MEDION E15433", ALC269VC_FIXUP_INFINIX_Y4_MAX), SND_PCI_QUIRK(0x2782, 0x1707, "Vaio VJFE-ADL", ALC298_FIXUP_SPK_VOLUME), SND_PCI_QUIRK(0x2782, 0x4900, "MEDION E15443", ALC233_FIXUP_MEDION_MTL_SPK), + SND_PCI_QUIRK(0x7017, 0x2014, "Star Labs StarFighter", ALC233_FIXUP_STARLABS_STARFIGHTER), SND_PCI_QUIRK(0x8086, 0x2074, "Intel NUC 8", ALC233_FIXUP_INTEL_NUC8_DMIC), SND_PCI_QUIRK(0x8086, 0x2080, "Intel NUC 8 Rugged", ALC256_FIXUP_INTEL_NUC8_RUGGED), SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", ALC256_FIXUP_INTEL_NUC10), @@ -7222,6 +7563,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0xf111, 0x0009, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x000b, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x000c, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0xf111, 0x000f, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), #if 0 /* Below is a quirk table taken from the old code. @@ -7309,6 +7651,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC298_FIXUP_TPT470_DOCK_FIX, .name = "tpt470-dock-fix"}, {.id = ALC298_FIXUP_TPT470_DOCK, .name = "tpt470-dock"}, {.id = ALC233_FIXUP_LENOVO_MULTI_CODECS, .name = "dual-codecs"}, + {.id = ALC233_FIXUP_STARLABS_STARFIGHTER, .name = "starlabs-starfighter"}, {.id = ALC700_FIXUP_INTEL_REFERENCE, .name = "alc700-ref"}, {.id = ALC269_FIXUP_SONY_VAIO, .name = "vaio"}, {.id = ALC269_FIXUP_DELL_M101Z, .name = "dell-m101z"}, diff --git a/sound/hda/codecs/senarytech.c b/sound/hda/codecs/senarytech.c index 9aa1e9bcd9ec0..601b7eb38eb13 100644 --- a/sound/hda/codecs/senarytech.c +++ b/sound/hda/codecs/senarytech.c @@ -25,6 +25,7 @@ struct senary_spec { /* extra EAPD pins */ unsigned int num_eapds; hda_nid_t eapds[4]; + bool dynamic_eapd; hda_nid_t mute_led_eapd; unsigned int parse_flags; /* flag for snd_hda_parse_pin_defcfg() */ @@ -131,8 +132,12 @@ static void senary_init_gpio_led(struct hda_codec *codec) static int senary_init(struct hda_codec *codec) { + struct senary_spec *spec = codec->spec; + snd_hda_gen_init(codec); senary_init_gpio_led(codec); + if (!spec->dynamic_eapd) + senary_auto_turn_eapd(codec, spec->num_eapds, spec->eapds, true); snd_hda_apply_fixup(codec, HDA_FIXUP_ACT_INIT); return 0; diff --git a/sound/hda/codecs/side-codecs/cirrus_scodec_test.c b/sound/hda/codecs/side-codecs/cirrus_scodec_test.c index 3cca750857b68..dc35932b6b22f 100644 --- a/sound/hda/codecs/side-codecs/cirrus_scodec_test.c +++ b/sound/hda/codecs/side-codecs/cirrus_scodec_test.c @@ -103,6 +103,7 @@ static int cirrus_scodec_test_gpio_probe(struct platform_device *pdev) /* GPIO core modifies our struct gpio_chip so use a copy */ gpio_priv->chip = cirrus_scodec_test_gpio_chip; + gpio_priv->chip.parent = &pdev->dev; ret = devm_gpiochip_add_data(&pdev->dev, &gpio_priv->chip, gpio_priv); if (ret) return dev_err_probe(&pdev->dev, ret, "Failed to add gpiochip\n"); @@ -319,7 +320,7 @@ static struct kunit_case cirrus_scodec_test_cases[] = { }; static struct kunit_suite cirrus_scodec_test_suite = { - .name = "snd-hda-scodec-cs35l56-test", + .name = "snd-hda-cirrus-scodec-test", .init = cirrus_scodec_test_case_init, .test_cases = cirrus_scodec_test_cases, }; diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.c b/sound/hda/codecs/side-codecs/cs35l41_hda.c index c0f2a3ff77a1b..21e00055c0c44 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l41_hda.c @@ -1901,6 +1901,8 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i cs35l41->dacpi = adev; physdev = get_device(acpi_get_first_physical_node(adev)); + if (!physdev) + return -ENODEV; sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev)); if (IS_ERR(sub)) diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.c b/sound/hda/codecs/side-codecs/cs35l56_hda.c index 5bb1c4ebeaf3c..acbacd0766064 100644 --- a/sound/hda/codecs/side-codecs/cs35l56_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l56_hda.c @@ -249,7 +249,7 @@ static int cs35l56_hda_posture_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct cs35l56_hda *cs35l56 = snd_kcontrol_chip(kcontrol); - unsigned long pos = ucontrol->value.integer.value[0]; + long pos = ucontrol->value.integer.value[0]; bool changed; int ret; diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c index 0357401a60234..624a822341bb7 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c @@ -2,7 +2,7 @@ // // TAS2781 HDA I2C driver // -// Copyright 2023 - 2025 Texas Instruments, Inc. +// Copyright 2023 - 2026 Texas Instruments, Inc. // // Author: Shenghao Ding // Current maintainer: Baojun Xu @@ -60,6 +60,7 @@ struct tas2781_hda_i2c_priv { int (*save_calibration)(struct tas2781_hda *h); int hda_chip_id; + bool skip_calibration; }; static int tas2781_get_i2c_res(struct acpi_resource *ares, void *data) @@ -87,6 +88,7 @@ static const struct acpi_gpio_mapping tas2781_speaker_id_gpios[] = { static int tas2781_read_acpi(struct tasdevice_priv *p, const char *hid) { + struct gpio_desc *speaker_id; struct acpi_device *adev; struct device *physdev; LIST_HEAD(resources); @@ -110,8 +112,10 @@ static int tas2781_read_acpi(struct tasdevice_priv *p, const char *hid) sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev)); if (IS_ERR(sub)) { /* No subsys id in older tas2563 projects. */ - if (!strncmp(hid, "INT8866", sizeof("INT8866"))) + if (!strncmp(hid, "INT8866", sizeof("INT8866"))) { + p->speaker_id = -1; goto end_2563; + } dev_err(p->dev, "Failed to get SUBSYS ID.\n"); ret = PTR_ERR(sub); goto err; @@ -119,19 +123,31 @@ static int tas2781_read_acpi(struct tasdevice_priv *p, const char *hid) /* Speaker id was needed for ASUS projects. */ ret = kstrtou32(sub, 16, &subid); if (!ret && upper_16_bits(subid) == PCI_VENDOR_ID_ASUSTEK) { - ret = devm_acpi_dev_add_driver_gpios(p->dev, - tas2781_speaker_id_gpios); - if (ret < 0) + ret = acpi_dev_add_driver_gpios(adev, tas2781_speaker_id_gpios); + if (ret < 0) { dev_err(p->dev, "Failed to add driver gpio %d.\n", ret); - p->speaker_id = devm_gpiod_get(p->dev, "speakerid", GPIOD_IN); - if (IS_ERR(p->speaker_id)) { - dev_err(p->dev, "Failed to get Speaker id.\n"); - ret = PTR_ERR(p->speaker_id); - goto err; + p->speaker_id = -1; + goto end_2563; } + + speaker_id = fwnode_gpiod_get_index(acpi_fwnode_handle(adev), + "speakerid", 0, GPIOD_IN, NULL); + if (!IS_ERR(speaker_id)) { + p->speaker_id = gpiod_get_value_cansleep(speaker_id); + dev_dbg(p->dev, "Got speaker id gpio from ACPI: %d.\n", + p->speaker_id); + gpiod_put(speaker_id); + } else { + p->speaker_id = -1; + ret = PTR_ERR(speaker_id); + dev_err(p->dev, "Get speaker id gpio failed %d.\n", + ret); + } + + acpi_dev_remove_driver_gpios(adev); } else { - p->speaker_id = NULL; + p->speaker_id = -1; } end_2563: @@ -432,23 +448,16 @@ static void tasdevice_dspfw_init(void *context) struct tas2781_hda *tas_hda = dev_get_drvdata(tas_priv->dev); struct tas2781_hda_i2c_priv *hda_priv = tas_hda->hda_priv; struct hda_codec *codec = tas_priv->codec; - int ret, spk_id; + int ret; tasdevice_dsp_remove(tas_priv); tas_priv->fw_state = TASDEVICE_DSP_FW_PENDING; - if (tas_priv->speaker_id != NULL) { - // Speaker id need to be checked for ASUS only. - spk_id = gpiod_get_value(tas_priv->speaker_id); - if (spk_id < 0) { - // Speaker id is not valid, use default. - dev_dbg(tas_priv->dev, "Wrong spk_id = %d\n", spk_id); - spk_id = 0; - } + if (tas_priv->speaker_id >= 0) { snprintf(tas_priv->coef_binaryname, sizeof(tas_priv->coef_binaryname), "TAS2XXX%04X%d.bin", lower_16_bits(codec->core.subsystem_id), - spk_id); + tas_priv->speaker_id); } else { snprintf(tas_priv->coef_binaryname, sizeof(tas_priv->coef_binaryname), @@ -483,7 +492,8 @@ static void tasdevice_dspfw_init(void *context) /* If calibrated data occurs error, dsp will still works with default * calibrated data inside algo. */ - hda_priv->save_calibration(tas_hda); + if (!hda_priv->skip_calibration) + hda_priv->save_calibration(tas_hda); } static void tasdev_fw_ready(const struct firmware *fmw, void *context) @@ -540,6 +550,7 @@ static int tas2781_hda_bind(struct device *dev, struct device *master, void *master_data) { struct tas2781_hda *tas_hda = dev_get_drvdata(dev); + struct tas2781_hda_i2c_priv *hda_priv = tas_hda->hda_priv; struct hda_component_parent *parent = master_data; struct hda_component *comp; struct hda_codec *codec; @@ -560,11 +571,22 @@ static int tas2781_hda_bind(struct device *dev, struct device *master, case 0x1028: tas_hda->catlog_id = DELL; break; + case 0x103C: + tas_hda->catlog_id = HP; + break; default: tas_hda->catlog_id = LENOVO; break; } + /* + * Using ASUS ROG Xbox Ally X (RC73XA) UEFI calibration data + * causes audio dropouts during playback, use fallback data + * from DSP firmware as a workaround. + */ + if (codec->core.subsystem_id == 0x10431384) + hda_priv->skip_calibration = true; + pm_runtime_get_sync(dev); comp->dev = dev; diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c index b9a55672bf15d..488e35dac9524 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c @@ -634,7 +634,7 @@ static void tasdev_fw_ready(const struct firmware *fmw, void *context) struct tasdevice_priv *tas_priv = context; struct tas2781_hda *tas_hda = dev_get_drvdata(tas_priv->dev); struct hda_codec *codec = tas_priv->codec; - int ret, val; + int ret; pm_runtime_get_sync(tas_priv->dev); guard(mutex)(&tas_priv->codec_lock); @@ -673,20 +673,14 @@ static void tasdev_fw_ready(const struct firmware *fmw, void *context) tas_priv->rcabin.profile_cfg_id = 0; tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; - ret = tasdevice_spi_dev_read(tas_priv, tas_priv->index, - TAS2781_REG_CLK_CONFIG, &val); - if (ret < 0) - goto out; - if (val == TAS2781_REG_CLK_CONFIG_RESET) { - ret = tasdevice_prmg_load(tas_priv, 0); - if (ret < 0) { - dev_err(tas_priv->dev, "FW download failed = %d\n", - ret); - goto out; - } - tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; + ret = tasdevice_prmg_load(tas_priv, 0); + if (ret < 0) { + dev_err(tas_priv->dev, "FW download failed = %d\n", ret); + goto out; } + tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; + if (tas_priv->fmw->nr_programs > 0) tas_priv->tasdevice[tas_priv->index].cur_prog = 0; if (tas_priv->fmw->nr_configurations > 0) diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c index a19258c95886c..98367b87d8017 100644 --- a/sound/hda/controllers/intel.c +++ b/sound/hda/controllers/intel.c @@ -1751,6 +1751,8 @@ static int default_bdl_pos_adj(struct azx *chip) return 1; case AZX_DRIVER_ZHAOXINHDMI: return 128; + case AZX_DRIVER_NVIDIA: + return 64; default: return 32; } @@ -2075,7 +2077,6 @@ static const struct pci_device_id driver_denylist[] = { { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1043, 0x874f) }, /* ASUS ROG Zenith II / Strix */ { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1462, 0xcb59) }, /* MSI TRX40 Creator */ { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1462, 0xcb60) }, /* MSI TRX40 */ - { PCI_DEVICE_SUB(0x1022, 0x15e3, 0x1462, 0xee59) }, /* MSI X870E Tomahawk WiFi */ {} }; diff --git a/sound/hda/core/intel-dsp-config.c b/sound/hda/core/intel-dsp-config.c index 2a9e35cddcf7c..ddb349bc46bbd 100644 --- a/sound/hda/core/intel-dsp-config.c +++ b/sound/hda/core/intel-dsp-config.c @@ -710,7 +710,8 @@ int snd_intel_dsp_driver_probe(struct pci_dev *pci) /* find the configuration for the specific device */ cfg = snd_intel_dsp_find_config(pci, config_table, ARRAY_SIZE(config_table)); if (!cfg) - return SND_INTEL_DSP_DRIVER_ANY; + return IS_ENABLED(CONFIG_SND_HDA_INTEL) ? + SND_INTEL_DSP_DRIVER_LEGACY : SND_INTEL_DSP_DRIVER_ANY; if (cfg->flags & FLAG_SOF) { if (cfg->flags & FLAG_SOF_ONLY_IF_SOUNDWIRE && diff --git a/sound/isa/wavefront/wavefront_midi.c b/sound/isa/wavefront/wavefront_midi.c index 1250ecba659a0..69d87c4cafaed 100644 --- a/sound/isa/wavefront/wavefront_midi.c +++ b/sound/isa/wavefront/wavefront_midi.c @@ -278,6 +278,7 @@ static int snd_wavefront_midi_input_close(struct snd_rawmidi_substream *substrea return -EIO; guard(spinlock_irqsave)(&midi->open); + midi->substream_input[mpu] = NULL; midi->mode[mpu] &= ~MPU401_MODE_INPUT; return 0; @@ -300,6 +301,7 @@ static int snd_wavefront_midi_output_close(struct snd_rawmidi_substream *substre return -EIO; guard(spinlock_irqsave)(&midi->open); + midi->substream_output[mpu] = NULL; midi->mode[mpu] &= ~MPU401_MODE_OUTPUT; return 0; } diff --git a/sound/isa/wavefront/wavefront_synth.c b/sound/isa/wavefront/wavefront_synth.c index cd5c177943aa0..0d78533e1cfd6 100644 --- a/sound/isa/wavefront/wavefront_synth.c +++ b/sound/isa/wavefront/wavefront_synth.c @@ -950,9 +950,9 @@ wavefront_send_sample (snd_wavefront_t *dev, if (header->size) { dev->freemem = wavefront_freemem (dev); - if (dev->freemem < (int)header->size) { + if (dev->freemem < 0 || dev->freemem < header->size) { dev_err(dev->card->dev, - "insufficient memory to load %d byte sample.\n", + "insufficient memory to load %u byte sample.\n", header->size); return -ENOMEM; } diff --git a/sound/pci/asihpi/hpimsgx.c b/sound/pci/asihpi/hpimsgx.c index b68e6bfbbfbab..ed1c7b7744361 100644 --- a/sound/pci/asihpi/hpimsgx.c +++ b/sound/pci/asihpi/hpimsgx.c @@ -581,8 +581,10 @@ static u16 adapter_prepare(u16 adapter) HPI_ADAPTER_OPEN); hm.adapter_index = adapter; hw_entry_point(&hm, &hr); - memcpy(&rESP_HPI_ADAPTER_OPEN[adapter], &hr, - sizeof(rESP_HPI_ADAPTER_OPEN[0])); + memcpy(&rESP_HPI_ADAPTER_OPEN[adapter].h, &hr, + sizeof(rESP_HPI_ADAPTER_OPEN[adapter].h)); + memcpy(&rESP_HPI_ADAPTER_OPEN[adapter].a, &hr.u.ax.info, + sizeof(rESP_HPI_ADAPTER_OPEN[adapter].a)); if (hr.error) return hr.error; diff --git a/sound/pci/ctxfi/ctamixer.c b/sound/pci/ctxfi/ctamixer.c index bb46585926369..c30162be27ee9 100644 --- a/sound/pci/ctxfi/ctamixer.c +++ b/sound/pci/ctxfi/ctamixer.c @@ -205,6 +205,7 @@ static int amixer_rsc_init(struct amixer *amixer, /* Set amixer specific operations */ amixer->rsc.ops = &amixer_basic_rsc_ops; + amixer->rsc.conj = 0; amixer->ops = &amixer_ops; amixer->input = NULL; amixer->sum = NULL; @@ -367,6 +368,7 @@ static int sum_rsc_init(struct sum *sum, return err; sum->rsc.ops = &sum_basic_rsc_ops; + sum->rsc.conj = 0; return 0; } diff --git a/sound/pci/ctxfi/ctdaio.c b/sound/pci/ctxfi/ctdaio.c index c0c3f8ab84677..9ac9f172b383b 100644 --- a/sound/pci/ctxfi/ctdaio.c +++ b/sound/pci/ctxfi/ctdaio.c @@ -119,6 +119,7 @@ static unsigned int daio_device_index(enum DAIOTYP type, struct hw *hw) switch (type) { case SPDIFOO: return 0; case SPDIFIO: return 0; + case SPDIFI1: return 1; case LINEO1: return 4; case LINEO2: return 7; case LINEO3: return 5; diff --git a/sound/pci/ctxfi/ctvmem.h b/sound/pci/ctxfi/ctvmem.h index da54cbcdb0be4..43a0065b40c35 100644 --- a/sound/pci/ctxfi/ctvmem.h +++ b/sound/pci/ctxfi/ctvmem.h @@ -15,7 +15,7 @@ #ifndef CTVMEM_H #define CTVMEM_H -#define CT_PTP_NUM 4 /* num of device page table pages */ +#define CT_PTP_NUM 1 /* num of device page table pages */ #include #include diff --git a/sound/pcmcia/pdaudiocf/pdaudiocf.c b/sound/pcmcia/pdaudiocf/pdaudiocf.c index 13419837dfb7c..a3291e626440e 100644 --- a/sound/pcmcia/pdaudiocf/pdaudiocf.c +++ b/sound/pcmcia/pdaudiocf/pdaudiocf.c @@ -131,7 +131,13 @@ static int snd_pdacf_probe(struct pcmcia_device *link) link->config_index = 1; link->config_regs = PRESENT_OPTION; - return pdacf_config(link); + err = pdacf_config(link); + if (err < 0) { + card_list[i] = NULL; + snd_card_free(card); + return err; + } + return 0; } diff --git a/sound/pcmcia/vx/vxpocket.c b/sound/pcmcia/vx/vxpocket.c index 2e09f2a513a6a..9a5c9aa8eec4a 100644 --- a/sound/pcmcia/vx/vxpocket.c +++ b/sound/pcmcia/vx/vxpocket.c @@ -284,7 +284,13 @@ static int vxpocket_probe(struct pcmcia_device *p_dev) vxp->p_dev = p_dev; - return vxpocket_config(p_dev); + err = vxpocket_config(p_dev); + if (err < 0) { + card_alloc &= ~(1 << i); + snd_card_free(card); + return err; + } + return 0; } static void vxpocket_detach(struct pcmcia_device *link) diff --git a/sound/soc/amd/acp/acp-i2s.c b/sound/soc/amd/acp/acp-i2s.c index 4ba0a66981ea9..283a674c7e2c3 100644 --- a/sound/soc/amd/acp/acp-i2s.c +++ b/sound/soc/amd/acp/acp-i2s.c @@ -157,6 +157,8 @@ static int acp_i2s_set_tdm_slot(struct snd_soc_dai *dai, u32 tx_mask, u32 rx_mas spin_lock_irq(&chip->acp_lock); list_for_each_entry(stream, &chip->stream_list, list) { + if (dai->id != stream->dai_id) + continue; switch (chip->acp_rev) { case ACP_RN_PCI_ID: case ACP_RMB_PCI_ID: diff --git a/sound/soc/amd/acp/acp-legacy-common.c b/sound/soc/amd/acp/acp-legacy-common.c index 3078f459e0050..4e477c48d4bdd 100644 --- a/sound/soc/amd/acp/acp-legacy-common.c +++ b/sound/soc/amd/acp/acp-legacy-common.c @@ -219,7 +219,10 @@ static int set_acp_i2s_dma_fifo(struct snd_pcm_substream *substream, SP_PB_FIFO_ADDR_OFFSET; reg_fifo_addr = ACP_I2S_TX_FIFOADDR(chip); reg_fifo_size = ACP_I2S_TX_FIFOSIZE(chip); - phy_addr = I2S_SP_TX_MEM_WINDOW_START + stream->reg_offset; + if (chip->acp_rev >= ACP70_PCI_ID) + phy_addr = ACP7x_I2S_SP_TX_MEM_WINDOW_START; + else + phy_addr = I2S_SP_TX_MEM_WINDOW_START + stream->reg_offset; writel(phy_addr, chip->base + ACP_I2S_TX_RINGBUFADDR(chip)); } else { reg_dma_size = ACP_I2S_RX_DMA_SIZE(chip); @@ -227,7 +230,10 @@ static int set_acp_i2s_dma_fifo(struct snd_pcm_substream *substream, SP_CAPT_FIFO_ADDR_OFFSET; reg_fifo_addr = ACP_I2S_RX_FIFOADDR(chip); reg_fifo_size = ACP_I2S_RX_FIFOSIZE(chip); - phy_addr = I2S_SP_RX_MEM_WINDOW_START + stream->reg_offset; + if (chip->acp_rev >= ACP70_PCI_ID) + phy_addr = ACP7x_I2S_SP_RX_MEM_WINDOW_START; + else + phy_addr = I2S_SP_RX_MEM_WINDOW_START + stream->reg_offset; writel(phy_addr, chip->base + ACP_I2S_RX_RINGBUFADDR(chip)); } break; @@ -238,7 +244,10 @@ static int set_acp_i2s_dma_fifo(struct snd_pcm_substream *substream, BT_PB_FIFO_ADDR_OFFSET; reg_fifo_addr = ACP_BT_TX_FIFOADDR(chip); reg_fifo_size = ACP_BT_TX_FIFOSIZE(chip); - phy_addr = I2S_BT_TX_MEM_WINDOW_START + stream->reg_offset; + if (chip->acp_rev >= ACP70_PCI_ID) + phy_addr = ACP7x_I2S_BT_TX_MEM_WINDOW_START; + else + phy_addr = I2S_BT_TX_MEM_WINDOW_START + stream->reg_offset; writel(phy_addr, chip->base + ACP_BT_TX_RINGBUFADDR(chip)); } else { reg_dma_size = ACP_BT_RX_DMA_SIZE(chip); @@ -246,7 +255,10 @@ static int set_acp_i2s_dma_fifo(struct snd_pcm_substream *substream, BT_CAPT_FIFO_ADDR_OFFSET; reg_fifo_addr = ACP_BT_RX_FIFOADDR(chip); reg_fifo_size = ACP_BT_RX_FIFOSIZE(chip); - phy_addr = I2S_BT_TX_MEM_WINDOW_START + stream->reg_offset; + if (chip->acp_rev >= ACP70_PCI_ID) + phy_addr = ACP7x_I2S_BT_RX_MEM_WINDOW_START; + else + phy_addr = I2S_BT_RX_MEM_WINDOW_START + stream->reg_offset; writel(phy_addr, chip->base + ACP_BT_RX_RINGBUFADDR(chip)); } break; @@ -257,7 +269,10 @@ static int set_acp_i2s_dma_fifo(struct snd_pcm_substream *substream, HS_PB_FIFO_ADDR_OFFSET; reg_fifo_addr = ACP_HS_TX_FIFOADDR; reg_fifo_size = ACP_HS_TX_FIFOSIZE; - phy_addr = I2S_HS_TX_MEM_WINDOW_START + stream->reg_offset; + if (chip->acp_rev >= ACP70_PCI_ID) + phy_addr = ACP7x_I2S_HS_TX_MEM_WINDOW_START; + else + phy_addr = I2S_HS_TX_MEM_WINDOW_START + stream->reg_offset; writel(phy_addr, chip->base + ACP_HS_TX_RINGBUFADDR); } else { reg_dma_size = ACP_HS_RX_DMA_SIZE; @@ -265,7 +280,10 @@ static int set_acp_i2s_dma_fifo(struct snd_pcm_substream *substream, HS_CAPT_FIFO_ADDR_OFFSET; reg_fifo_addr = ACP_HS_RX_FIFOADDR; reg_fifo_size = ACP_HS_RX_FIFOSIZE; - phy_addr = I2S_HS_RX_MEM_WINDOW_START + stream->reg_offset; + if (chip->acp_rev >= ACP70_PCI_ID) + phy_addr = ACP7x_I2S_HS_RX_MEM_WINDOW_START; + else + phy_addr = I2S_HS_RX_MEM_WINDOW_START + stream->reg_offset; writel(phy_addr, chip->base + ACP_HS_RX_RINGBUFADDR); } break; diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index 5a3cfedacbafd..2b2910b1856d5 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -95,6 +95,30 @@ static const struct dmi_system_id soc_sdw_quirk_table[] = { }, .driver_data = (void *)(ASOC_SDW_CODEC_SPKR), }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_SKU, "21YW"), + }, + .driver_data = (void *)((ASOC_SDW_CODEC_SPKR) | (ASOC_SDW_ACP_DMIC)), + }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_SKU, "21YX"), + }, + .driver_data = (void *)((ASOC_SDW_CODEC_SPKR) | (ASOC_SDW_ACP_DMIC)), + }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "HN7306EA"), + }, + .driver_data = (void *)(ASOC_SDW_ACP_DMIC), + }, {} }; diff --git a/sound/soc/amd/acp/amd-acp63-acpi-match.c b/sound/soc/amd/acp/amd-acp63-acpi-match.c index 9b6a49c051cda..1dbbaba3c75b3 100644 --- a/sound/soc/amd/acp/amd-acp63-acpi-match.c +++ b/sound/soc/amd/acp/amd-acp63-acpi-match.c @@ -30,6 +30,20 @@ static const struct snd_soc_acpi_endpoint spk_r_endpoint = { .group_id = 1 }; +static const struct snd_soc_acpi_endpoint spk_2_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 2, + .group_id = 1 +}; + +static const struct snd_soc_acpi_endpoint spk_3_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 3, + .group_id = 1 +}; + static const struct snd_soc_acpi_adr_device rt711_rt1316_group_adr[] = { { .adr = 0x000030025D071101ull, @@ -103,6 +117,345 @@ static const struct snd_soc_acpi_adr_device rt722_0_single_adr[] = { } }; +static const struct snd_soc_acpi_endpoint cs42l43_endpoints[] = { + { /* Jack Playback Endpoint */ + .num = 0, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, + { /* DMIC Capture Endpoint */ + .num = 1, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, + { /* Jack Capture Endpoint */ + .num = 2, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, + { /* Speaker Playback Endpoint */ + .num = 3, + .aggregated = 0, + .group_position = 0, + .group_id = 0, + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l56x4_l1u3210_adr[] = { + { + .adr = 0x00013301FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013201FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, + { + .adr = 0x00013101FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_2_endpoint, + .name_prefix = "AMP3" + }, + { + .adr = 0x00013001FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_3_endpoint, + .name_prefix = "AMP4" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x2_l0u01_adr[] = { + { + .adr = 0x00003001FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00003101FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x2_l1u01_adr[] = { + { + .adr = 0x00013001FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013101FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x2_l1u13_adr[] = { + { + .adr = 0x00013101FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013301FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, +}; + +static const struct snd_soc_acpi_adr_device cs35l63x4_l0u0246_adr[] = { + { + .adr = 0x00003001FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00003201FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, + { + .adr = 0x00003401FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_2_endpoint, + .name_prefix = "AMP3" + }, + { + .adr = 0x00003601FA356301ull, + .num_endpoints = 1, + .endpoints = &spk_3_endpoint, + .name_prefix = "AMP4" + }, +}; + +static const struct snd_soc_acpi_adr_device cs42l43_l0u0_adr[] = { + { + .adr = 0x00003001FA424301ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l43_l0u1_adr[] = { + { + .adr = 0x00003101FA424301ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l43b_l0u1_adr[] = { + { + .adr = 0x00003101FA2A3B01ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l43_l1u0_cs35l56x4_l1u0123_adr[] = { + { + .adr = 0x00013001FA424301ull, + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints), + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l43" + }, + { + .adr = 0x00013001FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "AMP1" + }, + { + .adr = 0x00013101FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "AMP2" + }, + { + .adr = 0x00013201FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_2_endpoint, + .name_prefix = "AMP3" + }, + { + .adr = 0x00013301FA355601ull, + .num_endpoints = 1, + .endpoints = &spk_3_endpoint, + .name_prefix = "AMP4" + }, +}; + +static const struct snd_soc_acpi_adr_device cs42l45_l0u0_adr[] = { + { + .adr = 0x00003001FA424501ull, + /* Re-use endpoints, but cs42l45 has no speaker */ + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints) - 1, + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l45" + } +}; + +static const struct snd_soc_acpi_adr_device cs42l45_l1u0_adr[] = { + { + .adr = 0x00013001FA424501ull, + /* Re-use endpoints, but cs42l45 has no speaker */ + .num_endpoints = ARRAY_SIZE(cs42l43_endpoints) - 1, + .endpoints = cs42l43_endpoints, + .name_prefix = "cs42l45" + } +}; + +static const struct snd_soc_acpi_link_adr acp63_cs35l56x4_l1u3210[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l56x4_l1u3210_adr), + .adr_d = cs35l56x4_l1u3210_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs35l63x4_l0u0246[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs35l63x4_l0u0246_adr), + .adr_d = cs35l63x4_l0u0246_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43_l0u1[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l43_l0u1_adr), + .adr_d = cs42l43_l0u1_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43b_l0u1[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l43b_l0u1_adr), + .adr_d = cs42l43b_l0u1_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43_l0u0_cs35l56x4_l1u3210[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l43_l0u0_adr), + .adr_d = cs42l43_l0u0_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l56x4_l1u3210_adr), + .adr_d = cs35l56x4_l1u3210_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l43_l1u0_cs35l56x4_l1u0123[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l43_l1u0_cs35l56x4_l1u0123_adr), + .adr_d = cs42l43_l1u0_cs35l56x4_l1u0123_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l0u0[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l45_l0u0_adr), + .adr_d = cs42l45_l0u0_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l0u0_cs35l63x2_l1u01[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l45_l0u0_adr), + .adr_d = cs42l45_l0u0_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l63x2_l1u01_adr), + .adr_d = cs35l63x2_l1u01_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l0u0_cs35l63x2_l1u13[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs42l45_l0u0_adr), + .adr_d = cs42l45_l0u0_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs35l63x2_l1u13_adr), + .adr_d = cs35l63x2_l1u13_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l1u0[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l45_l1u0_adr), + .adr_d = cs42l45_l1u0_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l1u0_cs35l63x2_l0u01[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l45_l1u0_adr), + .adr_d = cs42l45_l1u0_adr, + }, + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs35l63x2_l0u01_adr), + .adr_d = cs35l63x2_l0u01_adr, + }, + {} +}; + +static const struct snd_soc_acpi_link_adr acp63_cs42l45_l1u0_cs35l63x4_l0u0246[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(cs42l45_l1u0_adr), + .adr_d = cs42l45_l1u0_adr, + }, + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(cs35l63x4_l0u0246_adr), + .adr_d = cs35l63x4_l0u0246_adr, + }, + {} +}; + static const struct snd_soc_acpi_link_adr acp63_rt722_only[] = { { .mask = BIT(0), @@ -135,6 +488,66 @@ struct snd_soc_acpi_mach snd_soc_acpi_amd_acp63_sdw_machines[] = { .links = acp63_4_in_1_sdca, .drv_name = "amd_sdw", }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l43_l0u0_cs35l56x4_l1u3210, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l1u0_cs35l63x4_l0u0246, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l0u0_cs35l63x2_l1u01, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l0u0_cs35l63x2_l1u13, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0) | BIT(1), + .links = acp63_cs42l45_l1u0_cs35l63x2_l0u01, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(1), + .links = acp63_cs42l43_l1u0_cs35l56x4_l1u0123, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(1), + .links = acp63_cs35l56x4_l1u3210, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs35l63x4_l0u0246, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs42l43_l0u1, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs42l43b_l0u1, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(0), + .links = acp63_cs42l45_l0u0, + .drv_name = "amd_sdw", + }, + { + .link_mask = BIT(1), + .links = acp63_cs42l45_l1u0, + .drv_name = "amd_sdw", + }, {}, }; EXPORT_SYMBOL(snd_soc_acpi_amd_acp63_sdw_machines); diff --git a/sound/soc/amd/acp3x-rt5682-max9836.c b/sound/soc/amd/acp3x-rt5682-max9836.c index 4ca1978020a96..d1eb6f12a1830 100644 --- a/sound/soc/amd/acp3x-rt5682-max9836.c +++ b/sound/soc/amd/acp3x-rt5682-max9836.c @@ -94,8 +94,13 @@ static int acp3x_5682_init(struct snd_soc_pcm_runtime *rtd) return ret; } - rt5682_dai_wclk = clk_get(component->dev, "rt5682-dai-wclk"); - rt5682_dai_bclk = clk_get(component->dev, "rt5682-dai-bclk"); + rt5682_dai_wclk = devm_clk_get(component->dev, "rt5682-dai-wclk"); + if (IS_ERR(rt5682_dai_wclk)) + return PTR_ERR(rt5682_dai_wclk); + + rt5682_dai_bclk = devm_clk_get(component->dev, "rt5682-dai-bclk"); + if (IS_ERR(rt5682_dai_bclk)) + return PTR_ERR(rt5682_dai_bclk); ret = snd_soc_card_jack_new_pins(card, "Headset Jack", SND_JACK_HEADSET | diff --git a/sound/soc/amd/renoir/acp3x-pdm-dma.c b/sound/soc/amd/renoir/acp3x-pdm-dma.c index 95ac8c6800375..a560d06097d5e 100644 --- a/sound/soc/amd/renoir/acp3x-pdm-dma.c +++ b/sound/soc/amd/renoir/acp3x-pdm-dma.c @@ -301,9 +301,11 @@ static int acp_pdm_dma_close(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct pdm_dev_data *adata = dev_get_drvdata(component->dev); + struct pdm_stream_instance *rtd = substream->runtime->private_data; disable_pdm_interrupts(adata->acp_base); adata->capture_stream = NULL; + kfree(rtd); return 0; } diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index f210a253da9f5..4c0acdad13ea1 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -45,6 +45,13 @@ static struct snd_soc_card acp6x_card = { }; static const struct dmi_system_id yc_acp_quirk_table[] = { + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP Laptop 15-fc0xxx"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -416,6 +423,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "M6500RC"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "M6500RE"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -535,6 +549,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "15NBC1011"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ASUS EXPERTBOOK PM1503CDA"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -626,6 +647,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "8BD6"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_BOARD_NAME, "8EE4"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -661,6 +689,55 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Bravo 15 C7UCX"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HONOR"), + DMI_MATCH(DMI_PRODUCT_NAME, "GOH-X"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "RB"), + DMI_MATCH(DMI_BOARD_NAME, "XyloD5_RBU"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook_ASUSLaptop M6501RR_M6501RR"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ASUS EXPERTBOOK BM1503CDA"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "PM1503CDA"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "BM1403CDA"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."), + DMI_MATCH(DMI_PRODUCT_NAME, "Thin A15 B7VE"), + } + }, {} }; diff --git a/sound/soc/bcm/bcm63xx-pcm-whistler.c b/sound/soc/bcm/bcm63xx-pcm-whistler.c index e3a4fcc63a56d..efeb06ddabeb3 100644 --- a/sound/soc/bcm/bcm63xx-pcm-whistler.c +++ b/sound/soc/bcm/bcm63xx-pcm-whistler.c @@ -358,7 +358,9 @@ static int bcm63xx_soc_pcm_new(struct snd_soc_component *component, i2s_priv = dev_get_drvdata(snd_soc_rtd_to_cpu(rtd, 0)->dev); - of_dma_configure(pcm->card->dev, pcm->card->dev->of_node, 1); + ret = of_dma_configure(pcm->card->dev, pcm->card->dev->of_node, 1); + if (ret) + return ret; ret = dma_coerce_mask_and_coherent(pcm->card->dev, DMA_BIT_MASK(32)); if (ret) diff --git a/sound/soc/cirrus/ep93xx-i2s.c b/sound/soc/cirrus/ep93xx-i2s.c index cca01c03f0486..5dba741594fab 100644 --- a/sound/soc/cirrus/ep93xx-i2s.c +++ b/sound/soc/cirrus/ep93xx-i2s.c @@ -91,16 +91,28 @@ static inline unsigned ep93xx_i2s_read_reg(struct ep93xx_i2s_info *info, return __raw_readl(info->regs + reg); } -static void ep93xx_i2s_enable(struct ep93xx_i2s_info *info, int stream) +static int ep93xx_i2s_enable(struct ep93xx_i2s_info *info, int stream) { unsigned base_reg; + int err; if ((ep93xx_i2s_read_reg(info, EP93XX_I2S_TX0EN) & 0x1) == 0 && (ep93xx_i2s_read_reg(info, EP93XX_I2S_RX0EN) & 0x1) == 0) { /* Enable clocks */ - clk_prepare_enable(info->mclk); - clk_prepare_enable(info->sclk); - clk_prepare_enable(info->lrclk); + err = clk_prepare_enable(info->mclk); + if (err) + return err; + err = clk_prepare_enable(info->sclk); + if (err) { + clk_disable_unprepare(info->mclk); + return err; + } + err = clk_prepare_enable(info->lrclk); + if (err) { + clk_disable_unprepare(info->sclk); + clk_disable_unprepare(info->mclk); + return err; + } /* Enable i2s */ ep93xx_i2s_write_reg(info, EP93XX_I2S_GLCTRL, 1); @@ -119,6 +131,8 @@ static void ep93xx_i2s_enable(struct ep93xx_i2s_info *info, int stream) ep93xx_i2s_write_reg(info, EP93XX_I2S_TXCTRL, EP93XX_I2S_TXCTRL_TXEMPTY_LVL | EP93XX_I2S_TXCTRL_TXUFIE); + + return 0; } static void ep93xx_i2s_disable(struct ep93xx_i2s_info *info, int stream) @@ -195,9 +209,7 @@ static int ep93xx_i2s_startup(struct snd_pcm_substream *substream, { struct ep93xx_i2s_info *info = snd_soc_dai_get_drvdata(dai); - ep93xx_i2s_enable(info, substream->stream); - - return 0; + return ep93xx_i2s_enable(info, substream->stream); } static void ep93xx_i2s_shutdown(struct snd_pcm_substream *substream, @@ -373,14 +385,16 @@ static int ep93xx_i2s_suspend(struct snd_soc_component *component) static int ep93xx_i2s_resume(struct snd_soc_component *component) { struct ep93xx_i2s_info *info = snd_soc_component_get_drvdata(component); + int err; if (!snd_soc_component_active(component)) return 0; - ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_PLAYBACK); - ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_CAPTURE); + err = ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_PLAYBACK); + if (err) + return err; - return 0; + return ep93xx_i2s_enable(info, SNDRV_PCM_STREAM_CAPTURE); } #else #define ep93xx_i2s_suspend NULL diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index 160c07699a8b7..91ac99bc3edb2 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -170,6 +170,7 @@ config SND_SOC_ALL_CODECS imply SND_SOC_MT6359 imply SND_SOC_MT6660 imply SND_SOC_NAU8315 + imply SND_SOC_NAU8325 imply SND_SOC_NAU8540 imply SND_SOC_NAU8810 imply SND_SOC_NAU8821 @@ -2715,6 +2716,10 @@ config SND_SOC_MT6660 config SND_SOC_NAU8315 tristate "Nuvoton Technology Corporation NAU8315 CODEC" +config SND_SOC_NAU8325 + tristate "Nuvoton Technology Corporation NAU8325 CODEC" + depends on I2C + config SND_SOC_NAU8540 tristate "Nuvoton Technology Corporation NAU85L40 CODEC" depends on I2C diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile index bd95a7c911d5c..da4077463278e 100644 --- a/sound/soc/codecs/Makefile +++ b/sound/soc/codecs/Makefile @@ -192,6 +192,7 @@ snd-soc-mt6359-y := mt6359.o snd-soc-mt6359-accdet-y := mt6359-accdet.o snd-soc-mt6660-y := mt6660.o snd-soc-nau8315-y := nau8315.o +snd-soc-nau8325-y := nau8325.o snd-soc-nau8540-y := nau8540.o snd-soc-nau8810-y := nau8810.o snd-soc-nau8821-y := nau8821.o @@ -618,6 +619,7 @@ obj-$(CONFIG_SND_SOC_MT6359) += snd-soc-mt6359.o obj-$(CONFIG_SND_SOC_MT6359_ACCDET) += mt6359-accdet.o obj-$(CONFIG_SND_SOC_MT6660) += snd-soc-mt6660.o obj-$(CONFIG_SND_SOC_NAU8315) += snd-soc-nau8315.o +obj-$(CONFIG_SND_SOC_NAU8325) += snd-soc-nau8325.o obj-$(CONFIG_SND_SOC_NAU8540) += snd-soc-nau8540.o obj-$(CONFIG_SND_SOC_NAU8810) += snd-soc-nau8810.o obj-$(CONFIG_SND_SOC_NAU8821) += snd-soc-nau8821.o diff --git a/sound/soc/codecs/adau1372.c b/sound/soc/codecs/adau1372.c index fdee689cae538..d7363f9d53bb3 100644 --- a/sound/soc/codecs/adau1372.c +++ b/sound/soc/codecs/adau1372.c @@ -762,7 +762,7 @@ static int adau1372_startup(struct snd_pcm_substream *substream, struct snd_soc_ return 0; } -static void adau1372_enable_pll(struct adau1372 *adau1372) +static int adau1372_enable_pll(struct adau1372 *adau1372) { unsigned int val, timeout = 0; int ret; @@ -778,19 +778,26 @@ static void adau1372_enable_pll(struct adau1372 *adau1372) timeout++; } while (!(val & 1) && timeout < 3); - if (ret < 0 || !(val & 1)) + if (ret < 0 || !(val & 1)) { dev_err(adau1372->dev, "Failed to lock PLL\n"); + return ret < 0 ? ret : -ETIMEDOUT; + } + + return 0; } -static void adau1372_set_power(struct adau1372 *adau1372, bool enable) +static int adau1372_set_power(struct adau1372 *adau1372, bool enable) { if (adau1372->enabled == enable) - return; + return 0; if (enable) { unsigned int clk_ctrl = ADAU1372_CLK_CTRL_MCLK_EN; + int ret; - clk_prepare_enable(adau1372->mclk); + ret = clk_prepare_enable(adau1372->mclk); + if (ret) + return ret; if (adau1372->pd_gpio) gpiod_set_value(adau1372->pd_gpio, 0); @@ -804,7 +811,14 @@ static void adau1372_set_power(struct adau1372 *adau1372, bool enable) * accessed. */ if (adau1372->use_pll) { - adau1372_enable_pll(adau1372); + ret = adau1372_enable_pll(adau1372); + if (ret) { + regcache_cache_only(adau1372->regmap, true); + if (adau1372->pd_gpio) + gpiod_set_value(adau1372->pd_gpio, 1); + clk_disable_unprepare(adau1372->mclk); + return ret; + } clk_ctrl |= ADAU1372_CLK_CTRL_CLKSRC; } @@ -829,6 +843,8 @@ static void adau1372_set_power(struct adau1372 *adau1372, bool enable) } adau1372->enabled = enable; + + return 0; } static int adau1372_set_bias_level(struct snd_soc_component *component, @@ -842,11 +858,9 @@ static int adau1372_set_bias_level(struct snd_soc_component *component, case SND_SOC_BIAS_PREPARE: break; case SND_SOC_BIAS_STANDBY: - adau1372_set_power(adau1372, true); - break; + return adau1372_set_power(adau1372, true); case SND_SOC_BIAS_OFF: - adau1372_set_power(adau1372, false); - break; + return adau1372_set_power(adau1372, false); } return 0; diff --git a/sound/soc/codecs/ak4458.c b/sound/soc/codecs/ak4458.c index 57cf601d3df35..abe742edb10fe 100644 --- a/sound/soc/codecs/ak4458.c +++ b/sound/soc/codecs/ak4458.c @@ -671,7 +671,15 @@ static int ak4458_runtime_resume(struct device *dev) regcache_cache_only(ak4458->regmap, false); regcache_mark_dirty(ak4458->regmap); - return regcache_sync(ak4458->regmap); + ret = regcache_sync(ak4458->regmap); + if (ret) + goto err; + + return 0; +err: + regcache_cache_only(ak4458->regmap, true); + regulator_bulk_disable(ARRAY_SIZE(ak4458->supplies), ak4458->supplies); + return ret; } static const struct snd_soc_component_driver soc_codec_dev_ak4458 = { @@ -775,16 +783,12 @@ static int ak4458_i2c_probe(struct i2c_client *i2c) pm_runtime_enable(&i2c->dev); regcache_cache_only(ak4458->regmap, true); - ak4458_reset(ak4458, false); return 0; } static void ak4458_i2c_remove(struct i2c_client *i2c) { - struct ak4458_priv *ak4458 = i2c_get_clientdata(i2c); - - ak4458_reset(ak4458, true); pm_runtime_disable(&i2c->dev); } diff --git a/sound/soc/codecs/ak5558.c b/sound/soc/codecs/ak5558.c index 683f3e472f500..73684fc5beb1a 100644 --- a/sound/soc/codecs/ak5558.c +++ b/sound/soc/codecs/ak5558.c @@ -372,7 +372,15 @@ static int ak5558_runtime_resume(struct device *dev) regcache_cache_only(ak5558->regmap, false); regcache_mark_dirty(ak5558->regmap); - return regcache_sync(ak5558->regmap); + ret = regcache_sync(ak5558->regmap); + if (ret) + goto err; + + return 0; +err: + regcache_cache_only(ak5558->regmap, true); + regulator_bulk_disable(ARRAY_SIZE(ak5558->supplies), ak5558->supplies); + return ret; } static const struct dev_pm_ops ak5558_pm = { diff --git a/sound/soc/codecs/aw88261.c b/sound/soc/codecs/aw88261.c index de11ae8dd9d9f..124c0a58d08bd 100644 --- a/sound/soc/codecs/aw88261.c +++ b/sound/soc/codecs/aw88261.c @@ -423,9 +423,10 @@ static int aw88261_dev_reg_update(struct aw88261 *aw88261, if (ret) break; + /* keep all three bits from current hw status */ read_val &= (~AW88261_AMPPD_MASK) | (~AW88261_PWDN_MASK) | (~AW88261_HMUTE_MASK); - reg_val &= (AW88261_AMPPD_MASK | AW88261_PWDN_MASK | AW88261_HMUTE_MASK); + reg_val &= (AW88261_AMPPD_MASK & AW88261_PWDN_MASK & AW88261_HMUTE_MASK); reg_val |= read_val; /* enable uls hmute */ diff --git a/sound/soc/codecs/cs35l41.c b/sound/soc/codecs/cs35l41.c index 173d7c59b7254..5001a546a3e75 100644 --- a/sound/soc/codecs/cs35l41.c +++ b/sound/soc/codecs/cs35l41.c @@ -1188,13 +1188,14 @@ static int cs35l41_get_system_name(struct cs35l41_private *cs35l41) } } -err: if (sub) { cs35l41->dsp.system_name = sub; dev_info(cs35l41->dev, "Subsystem ID: %s\n", cs35l41->dsp.system_name); - } else - dev_warn(cs35l41->dev, "Subsystem ID not found\n"); + return 0; + } +err: + dev_warn(cs35l41->dev, "Subsystem ID not found\n"); return ret; } diff --git a/sound/soc/codecs/cs35l45.c b/sound/soc/codecs/cs35l45.c index d4dcdf37bb709..9b1eff4e9bb71 100644 --- a/sound/soc/codecs/cs35l45.c +++ b/sound/soc/codecs/cs35l45.c @@ -455,7 +455,7 @@ static const struct snd_soc_dapm_widget cs35l45_dapm_widgets[] = { SND_SOC_DAPM_AIF_OUT("ASP_TX2", NULL, 1, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX2_EN_SHIFT, 0), SND_SOC_DAPM_AIF_OUT("ASP_TX3", NULL, 2, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX3_EN_SHIFT, 0), SND_SOC_DAPM_AIF_OUT("ASP_TX4", NULL, 3, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX4_EN_SHIFT, 0), - SND_SOC_DAPM_AIF_OUT("ASP_TX5", NULL, 3, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX5_EN_SHIFT, 0), + SND_SOC_DAPM_AIF_OUT("ASP_TX5", NULL, 4, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX5_EN_SHIFT, 0), SND_SOC_DAPM_MUX("ASP_TX1 Source", SND_SOC_NOPM, 0, 0, &cs35l45_asp_muxes[0]), SND_SOC_DAPM_MUX("ASP_TX2 Source", SND_SOC_NOPM, 0, 0, &cs35l45_asp_muxes[1]), diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c index 9e6b9ca2f3547..a13e8eaf277d4 100644 --- a/sound/soc/codecs/cs35l56-shared.c +++ b/sound/soc/codecs/cs35l56-shared.c @@ -16,7 +16,7 @@ #include "cs35l56.h" -static const struct reg_sequence cs35l56_patch[] = { +static const struct reg_sequence cs35l56_asp_patch[] = { /* * Firmware can change these to non-defaults to satisfy SDCA. * Ensure that they are at known defaults. @@ -33,6 +33,20 @@ static const struct reg_sequence cs35l56_patch[] = { { CS35L56_ASP1TX2_INPUT, 0x00000000 }, { CS35L56_ASP1TX3_INPUT, 0x00000000 }, { CS35L56_ASP1TX4_INPUT, 0x00000000 }, +}; + +int cs35l56_set_asp_patch(struct cs35l56_base *cs35l56_base) +{ + return regmap_register_patch(cs35l56_base->regmap, cs35l56_asp_patch, + ARRAY_SIZE(cs35l56_asp_patch)); +} +EXPORT_SYMBOL_NS_GPL(cs35l56_set_asp_patch, "SND_SOC_CS35L56_SHARED"); + +static const struct reg_sequence cs35l56_patch[] = { + /* + * Firmware can change these to non-defaults to satisfy SDCA. + * Ensure that they are at known defaults. + */ { CS35L56_SWIRE_DP3_CH1_INPUT, 0x00000018 }, { CS35L56_SWIRE_DP3_CH2_INPUT, 0x00000019 }, { CS35L56_SWIRE_DP3_CH3_INPUT, 0x00000029 }, diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c index 2c1edbd636efc..193adba0cd1a4 100644 --- a/sound/soc/codecs/cs35l56.c +++ b/sound/soc/codecs/cs35l56.c @@ -324,6 +324,13 @@ static int cs35l56_dsp_event(struct snd_soc_dapm_widget *w, return wm_adsp_event(w, kcontrol, event); } +static int cs35l56_asp_dai_probe(struct snd_soc_dai *codec_dai) +{ + struct cs35l56_private *cs35l56 = snd_soc_component_get_drvdata(codec_dai->component); + + return cs35l56_set_asp_patch(&cs35l56->base); +} + static int cs35l56_asp_dai_set_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) { struct cs35l56_private *cs35l56 = snd_soc_component_get_drvdata(codec_dai->component); @@ -528,6 +535,7 @@ static int cs35l56_asp_dai_set_sysclk(struct snd_soc_dai *dai, } static const struct snd_soc_dai_ops cs35l56_ops = { + .probe = cs35l56_asp_dai_probe, .set_fmt = cs35l56_asp_dai_set_fmt, .set_tdm_slot = cs35l56_asp_dai_set_tdm_slot, .hw_params = cs35l56_asp_dai_hw_params, diff --git a/sound/soc/codecs/cs42l43-jack.c b/sound/soc/codecs/cs42l43-jack.c index 867e23d4fb8d8..ecba6c7952384 100644 --- a/sound/soc/codecs/cs42l43-jack.c +++ b/sound/soc/codecs/cs42l43-jack.c @@ -496,7 +496,23 @@ void cs42l43_bias_sense_timeout(struct work_struct *work) pm_runtime_put_autosuspend(priv->dev); } -static void cs42l43_start_load_detect(struct cs42l43_codec *priv) +static const struct reg_sequence cs42l43_3pole_patch[] = { + { 0x4000, 0x00000055 }, + { 0x4000, 0x000000AA }, + { 0x17420, 0x8500F300 }, + { 0x17424, 0x36003E00 }, + { 0x4000, 0x00000000 }, +}; + +static const struct reg_sequence cs42l43_4pole_patch[] = { + { 0x4000, 0x00000055 }, + { 0x4000, 0x000000AA }, + { 0x17420, 0x7800E600 }, + { 0x17424, 0x36003800 }, + { 0x4000, 0x00000000 }, +}; + +static void cs42l43_start_load_detect(struct cs42l43_codec *priv, bool mic) { struct cs42l43 *cs42l43 = priv->core; @@ -520,6 +536,15 @@ static void cs42l43_start_load_detect(struct cs42l43_codec *priv) dev_err(priv->dev, "Load detect HP power down timed out\n"); } + if (mic) + regmap_multi_reg_write_bypassed(cs42l43->regmap, + cs42l43_4pole_patch, + ARRAY_SIZE(cs42l43_4pole_patch)); + else + regmap_multi_reg_write_bypassed(cs42l43->regmap, + cs42l43_3pole_patch, + ARRAY_SIZE(cs42l43_3pole_patch)); + regmap_update_bits(cs42l43->regmap, CS42L43_BLOCK_EN3, CS42L43_ADC1_EN_MASK | CS42L43_ADC2_EN_MASK, 0); regmap_update_bits(cs42l43->regmap, CS42L43_DACCNFG2, CS42L43_HP_HPF_EN_MASK, 0); @@ -598,7 +623,7 @@ static int cs42l43_run_load_detect(struct cs42l43_codec *priv, bool mic) reinit_completion(&priv->load_detect); - cs42l43_start_load_detect(priv); + cs42l43_start_load_detect(priv, mic); time_left = wait_for_completion_timeout(&priv->load_detect, msecs_to_jiffies(CS42L43_LOAD_TIMEOUT_MS)); cs42l43_stop_load_detect(priv); @@ -622,11 +647,11 @@ static int cs42l43_run_load_detect(struct cs42l43_codec *priv, bool mic) } switch (val & CS42L43_AMP3_RES_DET_MASK) { - case 0x0: // low impedance - case 0x1: // high impedance + case 0x0: // < 22 Ohm impedance + case 0x1: // < 150 Ohm impedance + case 0x2: // < 1000 Ohm impedance return CS42L43_JACK_HEADPHONE; - case 0x2: // lineout - case 0x3: // Open circuit + case 0x3: // > 1000 Ohm impedance return CS42L43_JACK_LINEOUT; default: return -EINVAL; @@ -674,6 +699,7 @@ static int cs42l43_run_type_detect(struct cs42l43_codec *priv) switch (type & CS42L43_HSDET_TYPE_STS_MASK) { case 0x0: // CTIA case 0x1: // OMTP + case 0x4: return cs42l43_run_load_detect(priv, true); case 0x2: // 3-pole return cs42l43_run_load_detect(priv, false); diff --git a/sound/soc/codecs/es8328.c b/sound/soc/codecs/es8328.c index 76159c45e6b52..c0d7ce64b2d96 100644 --- a/sound/soc/codecs/es8328.c +++ b/sound/soc/codecs/es8328.c @@ -756,17 +756,23 @@ static int es8328_resume(struct snd_soc_component *component) es8328->supplies); if (ret) { dev_err(component->dev, "unable to enable regulators\n"); - return ret; + goto err_clk; } regcache_mark_dirty(regmap); ret = regcache_sync(regmap); if (ret) { dev_err(component->dev, "unable to sync regcache\n"); - return ret; + goto err_regulators; } return 0; + +err_regulators: + regulator_bulk_disable(ARRAY_SIZE(es8328->supplies), es8328->supplies); +err_clk: + clk_disable_unprepare(es8328->clk); + return ret; } static int es8328_component_probe(struct snd_soc_component *component) diff --git a/sound/soc/codecs/lpass-tx-macro.c b/sound/soc/codecs/lpass-tx-macro.c index 1da34cb3505f3..c7d4dc553e6aa 100644 --- a/sound/soc/codecs/lpass-tx-macro.c +++ b/sound/soc/codecs/lpass-tx-macro.c @@ -2473,7 +2473,8 @@ static const struct tx_macro_data lpass_ver_9_2 = { }; static const struct tx_macro_data lpass_ver_10_sm6115 = { - .flags = LPASS_MACRO_FLAG_HAS_NPL_CLOCK, + .flags = LPASS_MACRO_FLAG_HAS_NPL_CLOCK | + LPASS_MACRO_FLAG_RESET_SWR, .ver = LPASS_VER_10_0_0, .extra_widgets = tx_macro_dapm_widgets_v9_2, .extra_widgets_num = ARRAY_SIZE(tx_macro_dapm_widgets_v9_2), diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c index 92c177b82a021..9dc8f514bc15b 100644 --- a/sound/soc/codecs/lpass-va-macro.c +++ b/sound/soc/codecs/lpass-va-macro.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1343,18 +1344,22 @@ static int fsgen_gate_enable(struct clk_hw *hw) struct regmap *regmap = va->regmap; int ret; - if (va->has_swr_master) { - ret = clk_prepare_enable(va->mclk); - if (ret) - return ret; + ret = pm_runtime_get_sync(va->dev); + if (ret < 0) { + pm_runtime_put_noidle(va->dev); + return ret; } ret = va_macro_mclk_enable(va, true); + if (ret) { + pm_runtime_put_noidle(va->dev); + return ret; + } if (va->has_swr_master) regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL, CDC_VA_SWR_CLK_EN_MASK, CDC_VA_SWR_CLK_ENABLE); - return ret; + return 0; } static void fsgen_gate_disable(struct clk_hw *hw) @@ -1367,8 +1372,24 @@ static void fsgen_gate_disable(struct clk_hw *hw) CDC_VA_SWR_CLK_EN_MASK, 0x0); va_macro_mclk_enable(va, false); - if (va->has_swr_master) - clk_disable_unprepare(va->mclk); + + pm_runtime_mark_last_busy(va->dev); + pm_runtime_put_autosuspend(va->dev); +} + +static int va_macro_setup_pm_clocks(struct device *dev, struct va_macro *va) +{ + int ret; + + ret = devm_pm_clk_create(dev); + if (ret) + return ret; + + ret = of_pm_clk_add_clks(dev); + if (ret < 0) + return ret; + + return 0; } static int fsgen_gate_is_enabled(struct clk_hw *hw) @@ -1505,6 +1526,7 @@ static int va_macro_probe(struct platform_device *pdev) void __iomem *base; u32 sample_rate = 0; int ret; + int rpm_ret; va = devm_kzalloc(dev, sizeof(*va), GFP_KERNEL); if (!va) @@ -1572,22 +1594,18 @@ static int va_macro_probe(struct platform_device *pdev) clk_set_rate(va->npl, 2 * VA_MACRO_MCLK_FREQ); } - ret = clk_prepare_enable(va->macro); - if (ret) - goto err; - - ret = clk_prepare_enable(va->dcodec); + ret = va_macro_setup_pm_clocks(dev, va); if (ret) - goto err_dcodec; + goto err_rpm_disable; - ret = clk_prepare_enable(va->mclk); - if (ret) - goto err_mclk; + pm_runtime_set_autosuspend_delay(dev, 3000); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); - if (va->has_npl_clk) { - ret = clk_prepare_enable(va->npl); - if (ret) - goto err_npl; + rpm_ret = pm_runtime_resume_and_get(dev); + if (rpm_ret < 0) { + ret = rpm_ret; + goto err_rpm_disable; } /** @@ -1626,35 +1644,27 @@ static int va_macro_probe(struct platform_device *pdev) va_macro_dais, ARRAY_SIZE(va_macro_dais)); if (ret) - goto err_clkout; - - pm_runtime_set_autosuspend_delay(dev, 3000); - pm_runtime_use_autosuspend(dev); - pm_runtime_mark_last_busy(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); + goto err_rpm_put; ret = va_macro_register_fsgen_output(va); if (ret) - goto err_clkout; + goto err_rpm_put; va->fsgen = devm_clk_hw_get_clk(dev, &va->hw, "fsgen"); if (IS_ERR(va->fsgen)) { ret = PTR_ERR(va->fsgen); - goto err_clkout; + goto err_rpm_put; } + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + return 0; -err_clkout: - if (va->has_npl_clk) - clk_disable_unprepare(va->npl); -err_npl: - clk_disable_unprepare(va->mclk); -err_mclk: - clk_disable_unprepare(va->dcodec); -err_dcodec: - clk_disable_unprepare(va->macro); +err_rpm_put: + pm_runtime_put_noidle(dev); +err_rpm_disable: + pm_runtime_disable(dev); err: lpass_macro_pds_exit(va->pds); @@ -1665,12 +1675,7 @@ static void va_macro_remove(struct platform_device *pdev) { struct va_macro *va = dev_get_drvdata(&pdev->dev); - if (va->has_npl_clk) - clk_disable_unprepare(va->npl); - - clk_disable_unprepare(va->mclk); - clk_disable_unprepare(va->dcodec); - clk_disable_unprepare(va->macro); + pm_runtime_disable(&pdev->dev); lpass_macro_pds_exit(va->pds); } @@ -1682,12 +1687,7 @@ static int va_macro_runtime_suspend(struct device *dev) regcache_cache_only(va->regmap, true); regcache_mark_dirty(va->regmap); - if (va->has_npl_clk) - clk_disable_unprepare(va->npl); - - clk_disable_unprepare(va->mclk); - - return 0; + return pm_clk_suspend(dev); } static int va_macro_runtime_resume(struct device *dev) @@ -1695,25 +1695,14 @@ static int va_macro_runtime_resume(struct device *dev) struct va_macro *va = dev_get_drvdata(dev); int ret; - ret = clk_prepare_enable(va->mclk); - if (ret) { - dev_err(va->dev, "unable to prepare mclk\n"); + ret = pm_clk_resume(dev); + if (ret) return ret; - } - if (va->has_npl_clk) { - ret = clk_prepare_enable(va->npl); - if (ret) { - clk_disable_unprepare(va->mclk); - dev_err(va->dev, "unable to prepare npl\n"); - return ret; - } - } regcache_cache_only(va->regmap, false); - regcache_sync(va->regmap); - return 0; + return regcache_sync(va->regmap); } diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c index 38faa9074ca3e..a720bcc608566 100644 --- a/sound/soc/codecs/lpass-wsa-macro.c +++ b/sound/soc/codecs/lpass-wsa-macro.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "lpass-macro-common.h" @@ -2638,15 +2639,15 @@ static const struct snd_soc_dapm_route wsa_audio_map[] = { static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable) { struct regmap *regmap = wsa->regmap; + int ret; - if (enable) { - int ret; + ret = pm_runtime_get_sync(wsa->dev); + if (ret < 0) { + pm_runtime_put_noidle(wsa->dev); + return ret; + } - ret = clk_prepare_enable(wsa->mclk); - if (ret) { - dev_err(wsa->dev, "failed to enable mclk\n"); - return ret; - } + if (enable) { wsa_macro_mclk_enable(wsa, true); regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, @@ -2657,9 +2658,10 @@ static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable) regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, CDC_WSA_SWR_CLK_EN_MASK, 0); wsa_macro_mclk_enable(wsa, false); - clk_disable_unprepare(wsa->mclk); } + pm_runtime_mark_last_busy(wsa->dev); + pm_runtime_put_autosuspend(wsa->dev); return 0; } @@ -2883,27 +2885,26 @@ static int wsa_macro_probe(struct platform_device *pdev) /* set MCLK and NPL rates */ clk_set_rate(wsa->mclk, WSA_MACRO_MCLK_FREQ); - clk_set_rate(wsa->npl, WSA_MACRO_MCLK_FREQ); + if (wsa->npl) + clk_set_rate(wsa->npl, WSA_MACRO_MCLK_FREQ); - ret = clk_prepare_enable(wsa->macro); + ret = devm_pm_clk_create(dev); if (ret) - goto err; + return ret; - ret = clk_prepare_enable(wsa->dcodec); - if (ret) - goto err_dcodec; + ret = of_pm_clk_add_clks(dev); + if (ret < 0) + return ret; - ret = clk_prepare_enable(wsa->mclk); - if (ret) - goto err_mclk; + pm_runtime_set_autosuspend_delay(dev, 3000); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); - ret = clk_prepare_enable(wsa->npl); - if (ret) - goto err_npl; - ret = clk_prepare_enable(wsa->fsgen); - if (ret) - goto err_fsgen; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + goto err_rpm_disable; + } /* reset swr ip */ regmap_update_bits(wsa->regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, @@ -2920,44 +2921,26 @@ static int wsa_macro_probe(struct platform_device *pdev) wsa_macro_dai, ARRAY_SIZE(wsa_macro_dai)); if (ret) - goto err_clkout; - - pm_runtime_set_autosuspend_delay(dev, 3000); - pm_runtime_use_autosuspend(dev); - pm_runtime_mark_last_busy(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); + goto err_rpm_put; ret = wsa_macro_register_mclk_output(wsa); if (ret) - goto err_clkout; + goto err_rpm_put; - return 0; + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); -err_clkout: - clk_disable_unprepare(wsa->fsgen); -err_fsgen: - clk_disable_unprepare(wsa->npl); -err_npl: - clk_disable_unprepare(wsa->mclk); -err_mclk: - clk_disable_unprepare(wsa->dcodec); -err_dcodec: - clk_disable_unprepare(wsa->macro); -err: + return 0; +err_rpm_put: + pm_runtime_put_noidle(dev); +err_rpm_disable: + pm_runtime_disable(dev); return ret; - } static void wsa_macro_remove(struct platform_device *pdev) { - struct wsa_macro *wsa = dev_get_drvdata(&pdev->dev); - - clk_disable_unprepare(wsa->macro); - clk_disable_unprepare(wsa->dcodec); - clk_disable_unprepare(wsa->mclk); - clk_disable_unprepare(wsa->npl); - clk_disable_unprepare(wsa->fsgen); + pm_runtime_disable(&pdev->dev); } static int wsa_macro_runtime_suspend(struct device *dev) @@ -2967,11 +2950,7 @@ static int wsa_macro_runtime_suspend(struct device *dev) regcache_cache_only(wsa->regmap, true); regcache_mark_dirty(wsa->regmap); - clk_disable_unprepare(wsa->fsgen); - clk_disable_unprepare(wsa->npl); - clk_disable_unprepare(wsa->mclk); - - return 0; + return pm_clk_suspend(dev); } static int wsa_macro_runtime_resume(struct device *dev) @@ -2979,34 +2958,12 @@ static int wsa_macro_runtime_resume(struct device *dev) struct wsa_macro *wsa = dev_get_drvdata(dev); int ret; - ret = clk_prepare_enable(wsa->mclk); - if (ret) { - dev_err(dev, "unable to prepare mclk\n"); - return ret; - } - - ret = clk_prepare_enable(wsa->npl); - if (ret) { - dev_err(dev, "unable to prepare mclkx2\n"); - goto err_npl; - } - - ret = clk_prepare_enable(wsa->fsgen); - if (ret) { - dev_err(dev, "unable to prepare fsgen\n"); - goto err_fsgen; - } - regcache_cache_only(wsa->regmap, false); - regcache_sync(wsa->regmap); - - return 0; -err_fsgen: - clk_disable_unprepare(wsa->npl); -err_npl: - clk_disable_unprepare(wsa->mclk); + ret = pm_clk_resume(dev); + if (ret) + return ret; - return ret; + return regcache_sync(wsa->regmap); } static const struct dev_pm_ops wsa_macro_pm_ops = { diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c index a8a282ff9fc5a..b132ef8c1d2c8 100644 --- a/sound/soc/codecs/max98390.c +++ b/sound/soc/codecs/max98390.c @@ -1073,6 +1073,9 @@ static int max98390_i2c_probe(struct i2c_client *i2c) reset_gpio = devm_gpiod_get_optional(&i2c->dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(reset_gpio)) + return dev_err_probe(&i2c->dev, PTR_ERR(reset_gpio), + "Failed to get reset gpio\n"); /* Power on device */ if (reset_gpio) { diff --git a/sound/soc/codecs/nau8325.c b/sound/soc/codecs/nau8325.c index 2266f320a8f22..d396160213f5d 100644 --- a/sound/soc/codecs/nau8325.c +++ b/sound/soc/codecs/nau8325.c @@ -386,7 +386,8 @@ static int nau8325_clksrc_choose(struct nau8325 *nau8325, const struct nau8325_srate_attr **srate_table, int *n1_sel, int *mult_sel, int *n2_sel) { - int i, j, mclk, mclk_max, ratio, ratio_sel, n2_max; + int i, j, mclk, ratio; + int mclk_max = 0, ratio_sel = 0, n2_max = 0; if (!nau8325->mclk || !nau8325->fs) goto proc_err; @@ -408,7 +409,6 @@ static int nau8325_clksrc_choose(struct nau8325 *nau8325, } /* Get MCLK_SRC through 1/N, Multiplier, and then 1/N2. */ - mclk_max = 0; for (i = 0; i < ARRAY_SIZE(mclk_n1_div); i++) { for (j = 0; j < ARRAY_SIZE(mclk_n3_mult); j++) { mclk = nau8325->mclk << mclk_n3_mult[j].param; @@ -829,8 +829,7 @@ static int nau8325_read_device_properties(struct device *dev, return 0; } -static int nau8325_i2c_probe(struct i2c_client *i2c, - const struct i2c_device_id *id) +static int nau8325_i2c_probe(struct i2c_client *i2c) { struct device *dev = &i2c->dev; struct nau8325 *nau8325 = dev_get_platdata(dev); diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c index 4fa9a785513e5..dfb9630bffe29 100644 --- a/sound/soc/codecs/nau8821.c +++ b/sound/soc/codecs/nau8821.c @@ -1661,8 +1661,13 @@ int nau8821_enable_jack_detect(struct snd_soc_component *component, int ret; nau8821->jack = jack; + + if (nau8821->jdet_active) + return 0; + /* Initiate jack detection work queue */ INIT_DELAYED_WORK(&nau8821->jdet_work, nau8821_jdet_work); + nau8821->jdet_active = true; ret = devm_request_threaded_irq(nau8821->dev, nau8821->irq, NULL, nau8821_interrupt, IRQF_TRIGGER_LOW | IRQF_ONESHOT, diff --git a/sound/soc/codecs/nau8821.h b/sound/soc/codecs/nau8821.h index 88602923780d8..f9d7cd8cbd211 100644 --- a/sound/soc/codecs/nau8821.h +++ b/sound/soc/codecs/nau8821.h @@ -562,6 +562,7 @@ struct nau8821 { struct snd_soc_dapm_context *dapm; struct snd_soc_jack *jack; struct delayed_work jdet_work; + bool jdet_active; int irq; int clk_id; int micbias_voltage; diff --git a/sound/soc/codecs/pm4125.c b/sound/soc/codecs/pm4125.c index 706fc668ffe2a..3d7b07624ee61 100644 --- a/sound/soc/codecs/pm4125.c +++ b/sound/soc/codecs/pm4125.c @@ -70,7 +70,7 @@ struct pm4125_priv { struct wcd_mbhc_config mbhc_cfg; struct wcd_mbhc_intr intr_ids; struct irq_domain *virq; - const struct regmap_irq_chip *pm4125_regmap_irq_chip; + const struct regmap_irq_chip *chip_desc; struct regmap_irq_chip_data *irq_chip; struct snd_soc_jack *jack; unsigned long status_mask; @@ -179,7 +179,7 @@ static const u32 pm4125_config_regs[] = { PM4125_DIG_SWR_INTR_LEVEL_0, }; -static struct regmap_irq_chip pm4125_regmap_irq_chip = { +static const struct regmap_irq_chip pm4125_regmap_irq_chip = { .name = "pm4125", .irqs = pm4125_irqs, .num_irqs = ARRAY_SIZE(pm4125_irqs), @@ -1320,10 +1320,8 @@ static int pm4125_irq_init(struct pm4125_priv *pm4125, struct device *dev) return -EINVAL; } - pm4125_regmap_irq_chip.irq_drv_data = pm4125; - return devm_regmap_add_irq_chip(dev, pm4125->regmap, irq_create_mapping(pm4125->virq, 0), - IRQF_ONESHOT, 0, &pm4125_regmap_irq_chip, + IRQF_ONESHOT, 0, pm4125->chip_desc, &pm4125->irq_chip); } @@ -1551,6 +1549,10 @@ static int pm4125_bind(struct device *dev) struct device_link *devlink; int ret; + /* Initialize device pointers to NULL for safe cleanup */ + pm4125->rxdev = NULL; + pm4125->txdev = NULL; + /* Give the soundwire subdevices some more time to settle */ usleep_range(15000, 15010); @@ -1574,7 +1576,7 @@ static int pm4125_bind(struct device *dev) if (!pm4125->txdev) { dev_err(dev, "could not find txslave with matching of node\n"); ret = -EINVAL; - goto error_unbind_all; + goto error_put_rx; } pm4125->sdw_priv[AIF1_CAP] = dev_get_drvdata(pm4125->txdev); @@ -1584,7 +1586,7 @@ static int pm4125_bind(struct device *dev) if (!pm4125->tx_sdw_dev) { dev_err(dev, "could not get txslave with matching of dev\n"); ret = -EINVAL; - goto error_unbind_all; + goto error_put_tx; } /* @@ -1596,7 +1598,7 @@ static int pm4125_bind(struct device *dev) if (!devlink) { dev_err(dev, "Could not devlink TX and RX\n"); ret = -EINVAL; - goto error_unbind_all; + goto error_put_tx; } devlink = device_link_add(dev, pm4125->txdev, @@ -1650,6 +1652,10 @@ static int pm4125_bind(struct device *dev) device_link_remove(dev, pm4125->txdev); link_remove_rx_tx: device_link_remove(pm4125->rxdev, pm4125->txdev); +error_put_tx: + put_device(pm4125->txdev); +error_put_rx: + put_device(pm4125->rxdev); error_unbind_all: component_unbind_all(dev, pm4125); return ret; @@ -1660,9 +1666,18 @@ static void pm4125_unbind(struct device *dev) struct pm4125_priv *pm4125 = dev_get_drvdata(dev); snd_soc_unregister_component(dev); + devm_regmap_del_irq_chip(dev, irq_find_mapping(pm4125->virq, 0), + pm4125->irq_chip); device_link_remove(dev, pm4125->txdev); device_link_remove(dev, pm4125->rxdev); device_link_remove(pm4125->rxdev, pm4125->txdev); + + /* Release device references acquired in bind */ + if (pm4125->txdev) + put_device(pm4125->txdev); + if (pm4125->rxdev) + put_device(pm4125->rxdev); + component_unbind_all(dev, pm4125); } @@ -1695,6 +1710,7 @@ static int pm4125_probe(struct platform_device *pdev) { struct component_match *match = NULL; struct device *dev = &pdev->dev; + struct regmap_irq_chip *chip_desc; struct pm4125_priv *pm4125; struct wcd_mbhc_config *cfg; int ret; @@ -1705,6 +1721,14 @@ static int pm4125_probe(struct platform_device *pdev) dev_set_drvdata(dev, pm4125); + chip_desc = devm_kmemdup(dev, &pm4125_regmap_irq_chip, + sizeof(pm4125_regmap_irq_chip), + GFP_KERNEL); + if (!chip_desc) + return -ENOMEM; + chip_desc->irq_drv_data = pm4125; + pm4125->chip_desc = chip_desc; + ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(pm4125_power_supplies), pm4125_power_supplies); if (ret) diff --git a/sound/soc/codecs/rt1320-sdw.c b/sound/soc/codecs/rt1320-sdw.c index e3f9b03df3aae..e1bd991a823a4 100644 --- a/sound/soc/codecs/rt1320-sdw.c +++ b/sound/soc/codecs/rt1320-sdw.c @@ -1455,7 +1455,7 @@ static int rt1320_sdw_hw_params(struct snd_pcm_substream *substream, struct sdw_port_config port_config; struct sdw_port_config dmic_port_config[2]; struct sdw_stream_runtime *sdw_stream; - int retval; + int retval, num_channels; unsigned int sampling_rate; dev_dbg(dai->dev, "%s %s", __func__, dai->name); @@ -1487,7 +1487,8 @@ static int rt1320_sdw_hw_params(struct snd_pcm_substream *substream, dmic_port_config[1].num = 10; break; case RT1321_DEV_ID: - dmic_port_config[0].ch_mask = BIT(0) | BIT(1); + num_channels = params_channels(params); + dmic_port_config[0].ch_mask = GENMASK(num_channels - 1, 0); dmic_port_config[0].num = 8; break; default: diff --git a/sound/soc/codecs/rt721-sdca.c b/sound/soc/codecs/rt721-sdca.c index 5f7b505d54147..53cec09d29085 100644 --- a/sound/soc/codecs/rt721-sdca.c +++ b/sound/soc/codecs/rt721-sdca.c @@ -245,12 +245,12 @@ static void rt721_sdca_jack_preset(struct rt721_sdca_priv *rt721) regmap_write(rt721->mbq_regmap, 0x5b10007, 0x2000); regmap_write(rt721->mbq_regmap, 0x5B10017, 0x1b0f); rt_sdca_index_write(rt721->mbq_regmap, RT721_CBJ_CTRL, - RT721_CBJ_A0_GAT_CTRL1, 0x2a02); + RT721_CBJ_A0_GAT_CTRL1, 0x2205); rt_sdca_index_write(rt721->mbq_regmap, RT721_CAP_PORT_CTRL, RT721_HP_AMP_2CH_CAL4, 0xa105); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, RT721_UAJ_TOP_TCON14, 0x3b33); - regmap_write(rt721->mbq_regmap, 0x310400, 0x3023); + regmap_write(rt721->mbq_regmap, 0x310400, 0x3043); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, RT721_UAJ_TOP_TCON14, 0x3f33); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, diff --git a/sound/soc/codecs/sma1307.c b/sound/soc/codecs/sma1307.c index b683e676640d8..1b5c13f505a88 100644 --- a/sound/soc/codecs/sma1307.c +++ b/sound/soc/codecs/sma1307.c @@ -1779,8 +1779,10 @@ static void sma1307_setting_loaded(struct sma1307_priv *sma1307, const char *fil sma1307->set.mode_size * 2 * sizeof(int), GFP_KERNEL); if (!sma1307->set.mode_set[i]) { - for (int j = 0; j < i; j++) - kfree(sma1307->set.mode_set[j]); + for (int j = 0; j < i; j++) { + devm_kfree(sma1307->dev, sma1307->set.mode_set[j]); + sma1307->set.mode_set[j] = NULL; + } sma1307->set.status = false; return; } diff --git a/sound/soc/codecs/tas2781-comlib-i2c.c b/sound/soc/codecs/tas2781-comlib-i2c.c index b3fd7350143bd..e24d56a14cfda 100644 --- a/sound/soc/codecs/tas2781-comlib-i2c.c +++ b/sound/soc/codecs/tas2781-comlib-i2c.c @@ -320,7 +320,7 @@ void tasdevice_reset(struct tasdevice_priv *tas_dev) for (i = 0; i < tas_dev->ndev; i++) { ret = tasdevice_dev_write(tas_dev, i, TASDEVICE_REG_SWRESET, - tas_dev->chip_id >= TAS5825 ? + tas_dev->chip_id >= TAS5802 ? TAS5825_REG_SWRESET_RESET : TASDEVICE_REG_SWRESET_RESET); if (ret < 0) diff --git a/sound/soc/codecs/tas2781-i2c.c b/sound/soc/codecs/tas2781-i2c.c index 8f37aa00e62ee..a3b4d2c3b4789 100644 --- a/sound/soc/codecs/tas2781-i2c.c +++ b/sound/soc/codecs/tas2781-i2c.c @@ -1391,7 +1391,7 @@ static int tasdevice_create_cali_ctrls(struct tasdevice_priv *priv) /* * Alloc kcontrol via devm_kzalloc(), which don't manually - * free the kcontrol。 + * free the kcontrol. */ cali_ctrls = devm_kcalloc(priv->dev, nctrls, sizeof(cali_ctrls[0]), GFP_KERNEL); diff --git a/sound/soc/codecs/tlv320adcx140.c b/sound/soc/codecs/tlv320adcx140.c index d594bf166c0e7..1565727ca2f3d 100644 --- a/sound/soc/codecs/tlv320adcx140.c +++ b/sound/soc/codecs/tlv320adcx140.c @@ -23,7 +23,6 @@ #include "tlv320adcx140.h" struct adcx140_priv { - struct snd_soc_component *component; struct regulator *supply_areg; struct gpio_desc *gpio_reset; struct regmap *regmap; @@ -701,7 +700,6 @@ static void adcx140_pwr_ctrl(struct adcx140_priv *adcx140, bool power_state) { int pwr_ctrl = 0; int ret = 0; - struct snd_soc_component *component = adcx140->component; if (power_state) pwr_ctrl = ADCX140_PWR_CFG_ADC_PDZ | ADCX140_PWR_CFG_PLL_PDZ; @@ -713,7 +711,7 @@ static void adcx140_pwr_ctrl(struct adcx140_priv *adcx140, bool power_state) ret = regmap_write(adcx140->regmap, ADCX140_PHASE_CALIB, adcx140->phase_calib_on ? 0x00 : 0x40); if (ret) - dev_err(component->dev, "%s: register write error %d\n", + dev_err(adcx140->dev, "%s: register write error %d\n", __func__, ret); } @@ -729,7 +727,7 @@ static int adcx140_hw_params(struct snd_pcm_substream *substream, struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(component); u8 data = 0; - switch (params_width(params)) { + switch (params_physical_width(params)) { case 16: data = ADCX140_16_BIT_WORD; break; @@ -744,7 +742,7 @@ static int adcx140_hw_params(struct snd_pcm_substream *substream, break; default: dev_err(component->dev, "%s: Unsupported width %d\n", - __func__, params_width(params)); + __func__, params_physical_width(params)); return -EINVAL; } @@ -1158,6 +1156,9 @@ static int adcx140_i2c_probe(struct i2c_client *i2c) adcx140->gpio_reset = devm_gpiod_get_optional(adcx140->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(adcx140->gpio_reset)) + return dev_err_probe(&i2c->dev, PTR_ERR(adcx140->gpio_reset), + "Failed to get Reset GPIO\n"); + if (!adcx140->gpio_reset) dev_info(&i2c->dev, "Reset GPIO not defined\n"); adcx140->supply_areg = devm_regulator_get_optional(adcx140->dev, diff --git a/sound/soc/codecs/wcd934x.c b/sound/soc/codecs/wcd934x.c index 3c22f7149af81..c1a796b0ee309 100644 --- a/sound/soc/codecs/wcd934x.c +++ b/sound/soc/codecs/wcd934x.c @@ -2172,7 +2172,7 @@ static int wcd934x_init_dmic(struct snd_soc_component *comp) u32 def_dmic_rate, dmic_clk_drv; int ret; - ret = wcd_dt_parse_mbhc_data(comp->dev, &wcd->mbhc_cfg); + ret = wcd_dt_parse_micbias_info(&wcd->common); if (ret) return ret; diff --git a/sound/soc/codecs/wcd937x.c b/sound/soc/codecs/wcd937x.c index 421ec7a2d6bdc..ed0ff45a89649 100644 --- a/sound/soc/codecs/wcd937x.c +++ b/sound/soc/codecs/wcd937x.c @@ -2748,7 +2748,8 @@ static int wcd937x_bind(struct device *dev) wcd937x->rxdev = of_sdw_find_device_by_node(wcd937x->rxnode); if (!wcd937x->rxdev) { dev_err(dev, "could not find slave with matching of node\n"); - return -EINVAL; + ret = -EINVAL; + goto err_component_unbind; } wcd937x->sdw_priv[AIF1_PB] = dev_get_drvdata(wcd937x->rxdev); @@ -2757,7 +2758,8 @@ static int wcd937x_bind(struct device *dev) wcd937x->txdev = of_sdw_find_device_by_node(wcd937x->txnode); if (!wcd937x->txdev) { dev_err(dev, "could not find txslave with matching of node\n"); - return -EINVAL; + ret = -EINVAL; + goto err_put_rxdev; } wcd937x->sdw_priv[AIF1_CAP] = dev_get_drvdata(wcd937x->txdev); @@ -2765,7 +2767,8 @@ static int wcd937x_bind(struct device *dev) wcd937x->tx_sdw_dev = dev_to_sdw_dev(wcd937x->txdev); if (!wcd937x->tx_sdw_dev) { dev_err(dev, "could not get txslave with matching of dev\n"); - return -EINVAL; + ret = -EINVAL; + goto err_put_txdev; } /* @@ -2775,31 +2778,35 @@ static int wcd937x_bind(struct device *dev) if (!device_link_add(wcd937x->rxdev, wcd937x->txdev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME)) { dev_err(dev, "Could not devlink TX and RX\n"); - return -EINVAL; + ret = -EINVAL; + goto err_put_txdev; } if (!device_link_add(dev, wcd937x->txdev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME)) { dev_err(dev, "Could not devlink WCD and TX\n"); - return -EINVAL; + ret = -EINVAL; + goto err_remove_link1; } if (!device_link_add(dev, wcd937x->rxdev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME)) { dev_err(dev, "Could not devlink WCD and RX\n"); - return -EINVAL; + ret = -EINVAL; + goto err_remove_link2; } wcd937x->regmap = wcd937x->sdw_priv[AIF1_CAP]->regmap; if (!wcd937x->regmap) { dev_err(dev, "could not get TX device regmap\n"); - return -EINVAL; + ret = -EINVAL; + goto err_remove_link3; } ret = wcd937x_irq_init(wcd937x, dev); if (ret) { dev_err(dev, "IRQ init failed: %d\n", ret); - return ret; + goto err_remove_link3; } wcd937x->sdw_priv[AIF1_PB]->slave_irq = wcd937x->virq; @@ -2809,10 +2816,26 @@ static int wcd937x_bind(struct device *dev) ret = snd_soc_register_component(dev, &soc_codec_dev_wcd937x, wcd937x_dais, ARRAY_SIZE(wcd937x_dais)); - if (ret) + if (ret) { dev_err(dev, "Codec registration failed\n"); + goto err_remove_link3; + } return ret; + +err_remove_link3: + device_link_remove(dev, wcd937x->rxdev); +err_remove_link2: + device_link_remove(dev, wcd937x->txdev); +err_remove_link1: + device_link_remove(wcd937x->rxdev, wcd937x->txdev); +err_put_txdev: + put_device(wcd937x->txdev); +err_put_rxdev: + put_device(wcd937x->rxdev); +err_component_unbind: + component_unbind_all(dev, wcd937x); + return ret; } static void wcd937x_unbind(struct device *dev) @@ -2825,6 +2848,8 @@ static void wcd937x_unbind(struct device *dev) device_link_remove(wcd937x->rxdev, wcd937x->txdev); component_unbind_all(dev, wcd937x); mutex_destroy(&wcd937x->micb_lock); + put_device(wcd937x->txdev); + put_device(wcd937x->rxdev); } static const struct component_master_ops wcd937x_comp_ops = { diff --git a/sound/soc/codecs/wcd939x-sdw.c b/sound/soc/codecs/wcd939x-sdw.c index d369100a24574..da342a0c95a52 100644 --- a/sound/soc/codecs/wcd939x-sdw.c +++ b/sound/soc/codecs/wcd939x-sdw.c @@ -1400,12 +1400,18 @@ static int wcd9390_probe(struct sdw_slave *pdev, const struct sdw_device_id *id) ret = component_add(dev, &wcd_sdw_component_ops); if (ret) - return ret; + goto err_free_regmap; /* Set suspended until aggregate device is bind */ pm_runtime_set_suspended(dev); return 0; + +err_free_regmap: + if (wcd->regmap) + regmap_exit(wcd->regmap); + + return ret; } static int wcd9390_remove(struct sdw_slave *pdev) diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c index 08c8ec3aeb449..ab8247ae431f8 100644 --- a/sound/soc/codecs/wm8962.c +++ b/sound/soc/codecs/wm8962.c @@ -67,6 +67,8 @@ struct wm8962_priv { struct mutex dsp2_ena_lock; u16 dsp2_ena; + int mic_status; + struct delayed_work mic_work; struct snd_soc_jack *jack; @@ -1760,7 +1762,7 @@ SND_SOC_BYTES("EQR Coefficients", WM8962_EQ24, 18), SOC_SINGLE("3D Switch", WM8962_THREED1, 0, 1, 0), -SND_SOC_BYTES_MASK("3D Coefficients", WM8962_THREED1, 4, WM8962_THREED_ENA), +SND_SOC_BYTES_MASK("3D Coefficients", WM8962_THREED1, 4, WM8962_THREED_ENA | WM8962_ADC_MONOMIX), SOC_SINGLE("DF1 Switch", WM8962_DF1, 0, 1, 0), SND_SOC_BYTES_MASK("DF1 Coefficients", WM8962_DF1, 7, WM8962_DF1_ENA), @@ -3077,8 +3079,16 @@ static void wm8962_mic_work(struct work_struct *work) if (reg & WM8962_MICSHORT_STS) { status |= SND_JACK_BTN_0; irq_pol |= WM8962_MICSCD_IRQ_POL; + + /* Don't report a microphone if it's shorted right after + * plugging in, as this may be a TRS plug in a TRRS socket. + */ + if (!(wm8962->mic_status & WM8962_MICDET_STS)) + status = 0; } + wm8962->mic_status = status; + snd_soc_jack_report(wm8962->jack, status, SND_JACK_MICROPHONE | SND_JACK_BTN_0); diff --git a/sound/soc/codecs/wsa881x.c b/sound/soc/codecs/wsa881x.c index 636e59abc3772..561950ecb8cc7 100644 --- a/sound/soc/codecs/wsa881x.c +++ b/sound/soc/codecs/wsa881x.c @@ -678,6 +678,7 @@ struct wsa881x_priv { */ unsigned int sd_n_val; int active_ports; + bool hw_init; bool port_prepared[WSA881X_MAX_SWR_PORTS]; bool port_enable[WSA881X_MAX_SWR_PORTS]; }; @@ -687,6 +688,9 @@ static void wsa881x_init(struct wsa881x_priv *wsa881x) struct regmap *rm = wsa881x->regmap; unsigned int val = 0; + if (wsa881x->hw_init) + return; + regmap_register_patch(wsa881x->regmap, wsa881x_rev_2_0, ARRAY_SIZE(wsa881x_rev_2_0)); @@ -724,6 +728,8 @@ static void wsa881x_init(struct wsa881x_priv *wsa881x) regmap_update_bits(rm, WSA881X_OTP_REG_28, 0x3F, 0x3A); regmap_update_bits(rm, WSA881X_BONGO_RESRV_REG1, 0xFF, 0xB2); regmap_update_bits(rm, WSA881X_BONGO_RESRV_REG2, 0xFF, 0x05); + + wsa881x->hw_init = true; } static int wsa881x_component_probe(struct snd_soc_component *comp) @@ -1067,6 +1073,9 @@ static int wsa881x_update_status(struct sdw_slave *slave, { struct wsa881x_priv *wsa881x = dev_get_drvdata(&slave->dev); + if (status == SDW_SLAVE_UNATTACHED) + wsa881x->hw_init = false; + if (status == SDW_SLAVE_ATTACHED && slave->dev_num > 0) wsa881x_init(wsa881x); diff --git a/sound/soc/codecs/wsa883x.c b/sound/soc/codecs/wsa883x.c index ca4520ade79aa..5f2ced650dcc8 100644 --- a/sound/soc/codecs/wsa883x.c +++ b/sound/soc/codecs/wsa883x.c @@ -475,6 +475,7 @@ struct wsa883x_priv { int active_ports; int dev_mode; int comp_offset; + bool hw_init; /* * Protects temperature reading code (related to speaker protection) and * fields: temperature and pa_on. @@ -1043,6 +1044,9 @@ static int wsa883x_init(struct wsa883x_priv *wsa883x) struct regmap *regmap = wsa883x->regmap; int variant, version, ret; + if (wsa883x->hw_init) + return 0; + ret = regmap_read(regmap, WSA883X_OTP_REG_0, &variant); if (ret) return ret; @@ -1085,6 +1089,8 @@ static int wsa883x_init(struct wsa883x_priv *wsa883x) wsa883x->comp_offset); } + wsa883x->hw_init = true; + return 0; } @@ -1093,6 +1099,9 @@ static int wsa883x_update_status(struct sdw_slave *slave, { struct wsa883x_priv *wsa883x = dev_get_drvdata(&slave->dev); + if (status == SDW_SLAVE_UNATTACHED) + wsa883x->hw_init = false; + if (status == SDW_SLAVE_ATTACHED && slave->dev_num > 0) return wsa883x_init(wsa883x); diff --git a/sound/soc/codecs/wsa884x.c b/sound/soc/codecs/wsa884x.c index 2484d4b8e2d94..5ecd7962d77ae 100644 --- a/sound/soc/codecs/wsa884x.c +++ b/sound/soc/codecs/wsa884x.c @@ -1534,7 +1534,7 @@ static void wsa884x_init(struct wsa884x_priv *wsa884x) wsa884x_set_gain_parameters(wsa884x); - wsa884x->hw_init = false; + wsa884x->hw_init = true; } static int wsa884x_update_status(struct sdw_slave *slave, @@ -2109,7 +2109,6 @@ static int wsa884x_probe(struct sdw_slave *pdev, /* Start in cache-only until device is enumerated */ regcache_cache_only(wsa884x->regmap, true); - wsa884x->hw_init = true; if (IS_REACHABLE(CONFIG_HWMON)) { struct device *hwmon; diff --git a/sound/soc/fsl/fsl_easrc.c b/sound/soc/fsl/fsl_easrc.c index f404a39009e1a..d86f01fbecd4a 100644 --- a/sound/soc/fsl/fsl_easrc.c +++ b/sound/soc/fsl/fsl_easrc.c @@ -52,10 +52,13 @@ static int fsl_easrc_iec958_put_bits(struct snd_kcontrol *kcontrol, struct soc_mreg_control *mc = (struct soc_mreg_control *)kcontrol->private_value; unsigned int regval = ucontrol->value.integer.value[0]; + int ret; + + ret = (easrc_priv->bps_iec958[mc->regbase] != regval); easrc_priv->bps_iec958[mc->regbase] = regval; - return 0; + return ret; } static int fsl_easrc_iec958_get_bits(struct snd_kcontrol *kcontrol, @@ -93,14 +96,17 @@ static int fsl_easrc_set_reg(struct snd_kcontrol *kcontrol, struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); struct soc_mreg_control *mc = (struct soc_mreg_control *)kcontrol->private_value; + struct fsl_asrc *easrc = snd_soc_component_get_drvdata(component); unsigned int regval = ucontrol->value.integer.value[0]; + bool changed; int ret; - ret = snd_soc_component_write(component, mc->regbase, regval); - if (ret < 0) + ret = regmap_update_bits_check(easrc->regmap, mc->regbase, + GENMASK(31, 0), regval, &changed); + if (ret != 0) return ret; - return 0; + return changed; } #define SOC_SINGLE_REG_RW(xname, xreg) \ diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c index 72bfc91e21b9b..2fa14fbdfe1a8 100644 --- a/sound/soc/fsl/fsl_sai.c +++ b/sound/soc/fsl/fsl_sai.c @@ -917,8 +917,14 @@ static int fsl_sai_startup(struct snd_pcm_substream *substream, tx ? sai->dma_params_tx.maxburst : sai->dma_params_rx.maxburst); - ret = snd_pcm_hw_constraint_list(substream->runtime, 0, - SNDRV_PCM_HW_PARAM_RATE, &sai->constraint_rates); + if (sai->is_consumer_mode[tx]) + ret = snd_pcm_hw_constraint_list(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_RATE, + &fsl_sai_rate_constraints); + else + ret = snd_pcm_hw_constraint_list(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_RATE, + &sai->constraint_rates); return ret; } @@ -1075,6 +1081,7 @@ static const struct reg_default fsl_sai_reg_defaults_ofs0[] = { {FSL_SAI_TDR6, 0}, {FSL_SAI_TDR7, 0}, {FSL_SAI_TMR, 0}, + {FSL_SAI_TTCTL, 0}, {FSL_SAI_RCR1(0), 0}, {FSL_SAI_RCR2(0), 0}, {FSL_SAI_RCR3(0), 0}, @@ -1098,12 +1105,14 @@ static const struct reg_default fsl_sai_reg_defaults_ofs8[] = { {FSL_SAI_TDR6, 0}, {FSL_SAI_TDR7, 0}, {FSL_SAI_TMR, 0}, + {FSL_SAI_TTCTL, 0}, {FSL_SAI_RCR1(8), 0}, {FSL_SAI_RCR2(8), 0}, {FSL_SAI_RCR3(8), 0}, {FSL_SAI_RCR4(8), 0}, {FSL_SAI_RCR5(8), 0}, {FSL_SAI_RMR, 0}, + {FSL_SAI_RTCTL, 0}, {FSL_SAI_MCTL, 0}, {FSL_SAI_MDIV, 0}, }; diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c index 5d804860f7d8c..a268fb81a2f86 100644 --- a/sound/soc/fsl/fsl_xcvr.c +++ b/sound/soc/fsl/fsl_xcvr.c @@ -1323,6 +1323,7 @@ static const struct reg_default fsl_xcvr_phy_reg_defaults[] = { }; static const struct regmap_config fsl_xcvr_regmap_phy_cfg = { + .name = "phy", .reg_bits = 8, .reg_stride = 4, .val_bits = 32, @@ -1335,6 +1336,7 @@ static const struct regmap_config fsl_xcvr_regmap_phy_cfg = { }; static const struct regmap_config fsl_xcvr_regmap_pllv0_cfg = { + .name = "pllv0", .reg_bits = 8, .reg_stride = 4, .val_bits = 32, @@ -1345,6 +1347,7 @@ static const struct regmap_config fsl_xcvr_regmap_pllv0_cfg = { }; static const struct regmap_config fsl_xcvr_regmap_pllv1_cfg = { + .name = "pllv1", .reg_bits = 8, .reg_stride = 4, .val_bits = 32, @@ -1421,7 +1424,7 @@ static irqreturn_t irq0_isr(int irq, void *devid) bitrev32(val); } /* clear CS control register */ - memset_io(reg_ctrl, 0, sizeof(val)); + writel_relaxed(0, reg_ctrl); } } else { regmap_read(xcvr->regmap, FSL_XCVR_RX_CS_DATA_0, @@ -1548,28 +1551,24 @@ static int fsl_xcvr_probe(struct platform_device *pdev) xcvr->soc_data = of_device_get_match_data(&pdev->dev); xcvr->ipg_clk = devm_clk_get(dev, "ipg"); - if (IS_ERR(xcvr->ipg_clk)) { - dev_err(dev, "failed to get ipg clock\n"); - return PTR_ERR(xcvr->ipg_clk); - } + if (IS_ERR(xcvr->ipg_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->ipg_clk), + "failed to get ipg clock\n"); xcvr->phy_clk = devm_clk_get(dev, "phy"); - if (IS_ERR(xcvr->phy_clk)) { - dev_err(dev, "failed to get phy clock\n"); - return PTR_ERR(xcvr->phy_clk); - } + if (IS_ERR(xcvr->phy_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->phy_clk), + "failed to get phy clock\n"); xcvr->spba_clk = devm_clk_get(dev, "spba"); - if (IS_ERR(xcvr->spba_clk)) { - dev_err(dev, "failed to get spba clock\n"); - return PTR_ERR(xcvr->spba_clk); - } + if (IS_ERR(xcvr->spba_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->spba_clk), + "failed to get spba clock\n"); xcvr->pll_ipg_clk = devm_clk_get(dev, "pll_ipg"); - if (IS_ERR(xcvr->pll_ipg_clk)) { - dev_err(dev, "failed to get pll_ipg clock\n"); - return PTR_ERR(xcvr->pll_ipg_clk); - } + if (IS_ERR(xcvr->pll_ipg_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->pll_ipg_clk), + "failed to get pll_ipg clock\n"); fsl_asoc_get_pll_clocks(dev, &xcvr->pll8k_clk, &xcvr->pll11k_clk); @@ -1593,51 +1592,42 @@ static int fsl_xcvr_probe(struct platform_device *pdev) xcvr->regmap = devm_regmap_init_mmio_clk(dev, NULL, regs, &fsl_xcvr_regmap_cfg); - if (IS_ERR(xcvr->regmap)) { - dev_err(dev, "failed to init XCVR regmap: %ld\n", - PTR_ERR(xcvr->regmap)); - return PTR_ERR(xcvr->regmap); - } + if (IS_ERR(xcvr->regmap)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap), "failed to init XCVR regmap\n"); if (xcvr->soc_data->use_phy) { xcvr->regmap_phy = devm_regmap_init(dev, NULL, xcvr, &fsl_xcvr_regmap_phy_cfg); - if (IS_ERR(xcvr->regmap_phy)) { - dev_err(dev, "failed to init XCVR PHY regmap: %ld\n", - PTR_ERR(xcvr->regmap_phy)); - return PTR_ERR(xcvr->regmap_phy); - } + if (IS_ERR(xcvr->regmap_phy)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap_phy), + "failed to init XCVR PHY regmap\n"); switch (xcvr->soc_data->pll_ver) { case PLL_MX8MP: xcvr->regmap_pll = devm_regmap_init(dev, NULL, xcvr, &fsl_xcvr_regmap_pllv0_cfg); - if (IS_ERR(xcvr->regmap_pll)) { - dev_err(dev, "failed to init XCVR PLL regmap: %ld\n", - PTR_ERR(xcvr->regmap_pll)); - return PTR_ERR(xcvr->regmap_pll); - } + if (IS_ERR(xcvr->regmap_pll)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap_pll), + "failed to init XCVR PLL regmap\n"); break; case PLL_MX95: xcvr->regmap_pll = devm_regmap_init(dev, NULL, xcvr, &fsl_xcvr_regmap_pllv1_cfg); - if (IS_ERR(xcvr->regmap_pll)) { - dev_err(dev, "failed to init XCVR PLL regmap: %ld\n", - PTR_ERR(xcvr->regmap_pll)); - return PTR_ERR(xcvr->regmap_pll); - } + if (IS_ERR(xcvr->regmap_pll)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap_pll), + "failed to init XCVR PLL regmap\n"); break; default: - dev_err(dev, "Error for PLL version %d\n", xcvr->soc_data->pll_ver); - return -EINVAL; + return dev_err_probe(dev, -EINVAL, + "Error for PLL version %d\n", + xcvr->soc_data->pll_ver); } } xcvr->reset = devm_reset_control_get_optional_exclusive(dev, NULL); - if (IS_ERR(xcvr->reset)) { - dev_err(dev, "failed to get XCVR reset control\n"); - return PTR_ERR(xcvr->reset); - } + if (IS_ERR(xcvr->reset)) + return dev_err_probe(dev, PTR_ERR(xcvr->reset), + "failed to get XCVR reset control\n"); /* get IRQs */ irq = platform_get_irq(pdev, 0); @@ -1645,17 +1635,13 @@ static int fsl_xcvr_probe(struct platform_device *pdev) return irq; ret = devm_request_irq(dev, irq, irq0_isr, 0, pdev->name, xcvr); - if (ret) { - dev_err(dev, "failed to claim IRQ0: %i\n", ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to claim IRQ0\n"); rx_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rxfifo"); tx_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "txfifo"); - if (!rx_res || !tx_res) { - dev_err(dev, "could not find rxfifo or txfifo resource\n"); - return -EINVAL; - } + if (!rx_res || !tx_res) + return dev_err_probe(dev, -EINVAL, "could not find rxfifo or txfifo resource\n"); xcvr->dma_prms_rx.chan_name = "rx"; xcvr->dma_prms_tx.chan_name = "tx"; xcvr->dma_prms_rx.addr = rx_res->start; @@ -1678,8 +1664,7 @@ static int fsl_xcvr_probe(struct platform_device *pdev) ret = devm_snd_dmaengine_pcm_register(dev, NULL, 0); if (ret) { pm_runtime_disable(dev); - dev_err(dev, "failed to pcm register\n"); - return ret; + return dev_err_probe(dev, ret, "failed to pcm register\n"); } ret = devm_snd_soc_register_component(dev, &fsl_xcvr_comp, diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c index 28699d7b75ca0..a4518fefad690 100644 --- a/sound/soc/fsl/imx-card.c +++ b/sound/soc/fsl/imx-card.c @@ -346,7 +346,6 @@ static int imx_aif_hw_params(struct snd_pcm_substream *substream, SND_SOC_DAIFMT_PDM; } else { slots = 2; - slot_width = params_physical_width(params); fmt = (rtd->dai_link->dai_fmt & ~SND_SOC_DAIFMT_FORMAT_MASK) | SND_SOC_DAIFMT_I2S; } @@ -711,6 +710,8 @@ static int imx_card_parse_of(struct imx_card_data *data) link->ops = &imx_aif_ops; } + playback_only = false; + capture_only = false; graph_util_parse_link_direction(np, &playback_only, &capture_only); link->playback_only = playback_only; link->capture_only = capture_only; diff --git a/sound/soc/fsl/imx-rpmsg.c b/sound/soc/fsl/imx-rpmsg.c index 7cd3aa4c87060..39b2d64eb410d 100644 --- a/sound/soc/fsl/imx-rpmsg.c +++ b/sound/soc/fsl/imx-rpmsg.c @@ -145,7 +145,7 @@ static int imx_rpmsg_probe(struct platform_device *pdev) data->dai.ignore_pmdown_time = 1; data->dai.cpus->dai_name = pdev->dev.platform_data; - cpu_dai = snd_soc_find_dai(data->dai.cpus); + cpu_dai = snd_soc_find_dai_with_mutex(data->dai.cpus); if (!cpu_dai) { ret = -EPROBE_DEFER; goto fail; diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c index 355f7ec8943c2..9e5be0eaa77f3 100644 --- a/sound/soc/generic/simple-card-utils.c +++ b/sound/soc/generic/simple-card-utils.c @@ -1038,11 +1038,15 @@ int graph_util_is_ports0(struct device_node *np) else port = np; - struct device_node *ports __free(device_node) = of_get_parent(port); - struct device_node *top __free(device_node) = of_get_parent(ports); - struct device_node *ports0 __free(device_node) = of_get_child_by_name(top, "ports"); + struct device_node *ports __free(device_node) = of_get_parent(port); + const char *at = strchr(kbasename(ports->full_name), '@'); - return ports0 == ports; + /* + * Since child iteration order may differ + * between a base DT and DT overlays, + * string match "ports" or "ports@0" in the node name instead. + */ + return !at || !strcmp(at, "@0"); } EXPORT_SYMBOL_GPL(graph_util_is_ports0); @@ -1179,9 +1183,9 @@ void graph_util_parse_link_direction(struct device_node *np, bool is_playback_only = of_property_read_bool(np, "playback-only"); bool is_capture_only = of_property_read_bool(np, "capture-only"); - if (playback_only) + if (np && playback_only) *playback_only = is_playback_only; - if (capture_only) + if (np && capture_only) *capture_only = is_capture_only; } EXPORT_SYMBOL_GPL(graph_util_parse_link_direction); diff --git a/sound/soc/intel/avs/board_selection.c b/sound/soc/intel/avs/board_selection.c index 52e6266a7cb86..96dc637ccb20c 100644 --- a/sound/soc/intel/avs/board_selection.c +++ b/sound/soc/intel/avs/board_selection.c @@ -520,7 +520,8 @@ static int avs_register_i2s_test_boards(struct avs_dev *adev) if (num_elems > max_ssps) { dev_err(adev->dev, "board supports only %d SSP, %d specified\n", max_ssps, num_elems); - return -EINVAL; + ret = -EINVAL; + goto exit; } for (ssp_port = 0; ssp_port < num_elems; ssp_port++) { @@ -528,11 +529,13 @@ static int avs_register_i2s_test_boards(struct avs_dev *adev) for_each_set_bit(tdm_slot, &tdm_slots, 16) { ret = avs_register_i2s_test_board(adev, ssp_port, tdm_slot); if (ret) - return ret; + goto exit; } } - return 0; +exit: + kfree(array); + return ret; } static int avs_register_i2s_board(struct avs_dev *adev, struct snd_soc_acpi_mach *mach) diff --git a/sound/soc/intel/boards/Kconfig b/sound/soc/intel/boards/Kconfig index c23fdb6aad4ca..1031d6497f55e 100644 --- a/sound/soc/intel/boards/Kconfig +++ b/sound/soc/intel/boards/Kconfig @@ -525,8 +525,6 @@ config SND_SOC_INTEL_SOUNDWIRE_SOF_MACH select SND_SOC_CS42L43_SDW select MFD_CS42L43 select MFD_CS42L43_SDW - select PINCTRL_CS42L43 - select SPI_CS42L43 select SND_SOC_CS35L56_SPI select SND_SOC_CS35L56_SDW select SND_SOC_DMIC diff --git a/sound/soc/intel/boards/sof_es8336.c b/sound/soc/intel/boards/sof_es8336.c index 10b189ea88dba..cf50de5c2edd8 100644 --- a/sound/soc/intel/boards/sof_es8336.c +++ b/sound/soc/intel/boards/sof_es8336.c @@ -120,7 +120,7 @@ static void pcm_pop_work_events(struct work_struct *work) gpiod_set_value_cansleep(priv->gpio_speakers, priv->speaker_en); if (quirk & SOF_ES8336_HEADPHONE_GPIO) - gpiod_set_value_cansleep(priv->gpio_headphone, priv->speaker_en); + gpiod_set_value_cansleep(priv->gpio_headphone, !priv->speaker_en); } @@ -332,6 +332,15 @@ static int sof_es8336_quirk_cb(const struct dmi_system_id *id) * if the topology file is modified as well. */ static const struct dmi_system_id sof_es8336_quirk_table[] = { + { + .callback = sof_es8336_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HUAWEI"), + DMI_MATCH(DMI_PRODUCT_NAME, "BOD-WXX9"), + }, + .driver_data = (void *)(SOF_ES8336_HEADPHONE_GPIO | + SOF_ES8336_ENABLE_DMIC) + }, { .callback = sof_es8336_quirk_cb, .matches = { diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index c013e31d098e7..465bf5fafecf7 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -749,7 +749,23 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = { }, .driver_data = (void *)(SOC_SDW_CODEC_SPKR), }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0CCD") + }, + .driver_data = (void *)(SOC_SDW_CODEC_SPKR), + }, /* Pantherlake devices*/ + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0DD6") + }, + .driver_data = (void *)(SOC_SDW_SIDECAR_AMPS), + }, { .callback = sof_sdw_quirk_cb, .matches = { @@ -794,6 +810,7 @@ static const struct snd_pci_quirk sof_sdw_ssid_quirk_table[] = { SND_PCI_QUIRK(0x17aa, 0x2347, "Lenovo P16", SOC_SDW_CODEC_MIC), SND_PCI_QUIRK(0x17aa, 0x2348, "Lenovo P16", SOC_SDW_CODEC_MIC), SND_PCI_QUIRK(0x17aa, 0x2349, "Lenovo P1", SOC_SDW_CODEC_MIC), + SND_PCI_QUIRK(0x17aa, 0x3821, "Lenovo 0x3821", SOC_SDW_SIDECAR_AMPS), {} }; diff --git a/sound/soc/intel/catpt/device.c b/sound/soc/intel/catpt/device.c index faa916f400693..00e544e043591 100644 --- a/sound/soc/intel/catpt/device.c +++ b/sound/soc/intel/catpt/device.c @@ -271,7 +271,15 @@ static int catpt_acpi_probe(struct platform_device *pdev) if (IS_ERR(cdev->pci_ba)) return PTR_ERR(cdev->pci_ba); - /* alloc buffer for storing DRAM context during dx transitions */ + /* + * As per design HOST is responsible for preserving firmware's runtime + * context during D0 -> D3 -> D0 transitions. Addresses used for DMA + * to/from HOST memory shall be outside the reserved range of 0xFFFxxxxx. + */ + ret = dma_coerce_mask_and_coherent(cdev->dev, DMA_BIT_MASK(31)); + if (ret) + return ret; + cdev->dxbuf_vaddr = dmam_alloc_coherent(dev, catpt_dram_size(cdev), &cdev->dxbuf_paddr, GFP_KERNEL); if (!cdev->dxbuf_vaddr) diff --git a/sound/soc/intel/catpt/dsp.c b/sound/soc/intel/catpt/dsp.c index 008a20a2acbda..677f348909c8f 100644 --- a/sound/soc/intel/catpt/dsp.c +++ b/sound/soc/intel/catpt/dsp.c @@ -125,9 +125,6 @@ int catpt_dmac_probe(struct catpt_dev *cdev) dmac->dev = cdev->dev; dmac->irq = cdev->irq; - ret = dma_coerce_mask_and_coherent(cdev->dev, DMA_BIT_MASK(31)); - if (ret) - return ret; /* * Caller is responsible for putting device in D0 to allow * for I/O and memory access before probing DW. diff --git a/sound/soc/intel/catpt/pcm.c b/sound/soc/intel/catpt/pcm.c index bf734c69c4e09..eb03cecdee281 100644 --- a/sound/soc/intel/catpt/pcm.c +++ b/sound/soc/intel/catpt/pcm.c @@ -417,8 +417,10 @@ static int catpt_dai_hw_params(struct snd_pcm_substream *substream, return CATPT_IPC_ERROR(ret); ret = catpt_dai_apply_usettings(dai, stream); - if (ret) + if (ret) { + catpt_ipc_free_stream(cdev, stream->info.stream_hw_id); return ret; + } stream->allocated = true; return 0; diff --git a/sound/soc/intel/common/soc-acpi-intel-arl-match.c b/sound/soc/intel/common/soc-acpi-intel-arl-match.c index 6bf7a6250ddc3..c952f7d2b2c0e 100644 --- a/sound/soc/intel/common/soc-acpi-intel-arl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-arl-match.c @@ -45,23 +45,22 @@ static const struct snd_soc_acpi_endpoint spk_3_endpoint = { .group_id = 1, }; -/* - * RT722 is a multi-function codec, three endpoints are created for - * its headset, amp and dmic functions. - */ -static const struct snd_soc_acpi_endpoint rt722_endpoints[] = { +static const struct snd_soc_acpi_endpoint jack_amp_g1_dmic_endpoints[] = { + /* Jack Endpoint */ { .num = 0, .aggregated = 0, .group_position = 0, .group_id = 0, }, + /* Amp Endpoint, work as spk_l_endpoint */ { .num = 1, - .aggregated = 0, + .aggregated = 1, .group_position = 0, - .group_id = 0, + .group_id = 1, }, + /* DMIC Endpoint */ { .num = 2, .aggregated = 0, @@ -229,11 +228,11 @@ static const struct snd_soc_acpi_adr_device rt711_sdca_0_adr[] = { } }; -static const struct snd_soc_acpi_adr_device rt722_0_single_adr[] = { +static const struct snd_soc_acpi_adr_device rt722_0_agg_adr[] = { { .adr = 0x000030025D072201ull, - .num_endpoints = ARRAY_SIZE(rt722_endpoints), - .endpoints = rt722_endpoints, + .num_endpoints = ARRAY_SIZE(jack_amp_g1_dmic_endpoints), + .endpoints = jack_amp_g1_dmic_endpoints, .name_prefix = "rt722" } }; @@ -394,8 +393,8 @@ static const struct snd_soc_acpi_link_adr arl_rt711_l0_rt1316_l3[] = { static const struct snd_soc_acpi_link_adr arl_rt722_l0_rt1320_l2[] = { { .mask = BIT(0), - .num_adr = ARRAY_SIZE(rt722_0_single_adr), - .adr_d = rt722_0_single_adr, + .num_adr = ARRAY_SIZE(rt722_0_agg_adr), + .adr_d = rt722_0_agg_adr, }, { .mask = BIT(2), diff --git a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c index 4853f4f31786f..55505625b3609 100644 --- a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c @@ -418,7 +418,7 @@ static const struct snd_soc_acpi_adr_device rt1320_2_group2_adr[] = { .adr = 0x000230025D132001ull, .num_endpoints = 1, .endpoints = &spk_r_endpoint, - .name_prefix = "rt1320-1" + .name_prefix = "rt1320-2" } }; diff --git a/sound/soc/qcom/qdsp6/q6adm.c b/sound/soc/qcom/qdsp6/q6adm.c index 1530e98df1656..75a029a696ac9 100644 --- a/sound/soc/qcom/qdsp6/q6adm.c +++ b/sound/soc/qcom/qdsp6/q6adm.c @@ -109,11 +109,75 @@ static struct q6copp *q6adm_find_copp(struct q6adm *adm, int port_idx, } +static int q6adm_apr_send_copp_pkt(struct q6adm *adm, struct q6copp *copp, + struct apr_pkt *pkt, uint32_t rsp_opcode) +{ + struct device *dev = adm->dev; + uint32_t opcode = pkt->hdr.opcode; + int ret; + + mutex_lock(&adm->lock); + copp->result.opcode = 0; + copp->result.status = 0; + ret = apr_send_pkt(adm->apr, pkt); + if (ret < 0) { + dev_err(dev, "Failed to send APR packet\n"); + ret = -EINVAL; + goto err; + } + + /* Wait for the callback with copp id */ + if (rsp_opcode) + ret = wait_event_timeout(copp->wait, + (copp->result.opcode == opcode) || + (copp->result.opcode == rsp_opcode), + msecs_to_jiffies(TIMEOUT_MS)); + else + ret = wait_event_timeout(copp->wait, + (copp->result.opcode == opcode), + msecs_to_jiffies(TIMEOUT_MS)); + + if (!ret) { + dev_err(dev, "ADM copp cmd timedout\n"); + ret = -ETIMEDOUT; + } else if (copp->result.status > 0) { + dev_err(dev, "DSP returned error[%d]\n", + copp->result.status); + ret = -EINVAL; + } + +err: + mutex_unlock(&adm->lock); + return ret; +} + +static int q6adm_device_close(struct q6adm *adm, struct q6copp *copp, + int port_id, int copp_idx) +{ + struct apr_pkt close; + + close.hdr.hdr_field = APR_HDR_FIELD(APR_MSG_TYPE_SEQ_CMD, + APR_HDR_LEN(APR_HDR_SIZE), + APR_PKT_VER); + close.hdr.pkt_size = sizeof(close); + close.hdr.src_port = port_id; + close.hdr.dest_port = copp->id; + close.hdr.token = port_id << 16 | copp_idx; + close.hdr.opcode = ADM_CMD_DEVICE_CLOSE_V5; + + return q6adm_apr_send_copp_pkt(adm, copp, &close, 0); +} + static void q6adm_free_copp(struct kref *ref) { struct q6copp *c = container_of(ref, struct q6copp, refcount); struct q6adm *adm = c->adm; unsigned long flags; + int ret; + + ret = q6adm_device_close(adm, c, c->afe_port, c->copp_idx); + if (ret < 0) + dev_err(adm->dev, "Failed to close copp %d\n", ret); spin_lock_irqsave(&adm->copps_list_lock, flags); clear_bit(c->copp_idx, &adm->copp_bitmap[c->afe_port]); @@ -155,13 +219,13 @@ static int q6adm_callback(struct apr_device *adev, struct apr_resp_pkt *data) switch (result->opcode) { case ADM_CMD_DEVICE_OPEN_V5: case ADM_CMD_DEVICE_CLOSE_V5: - copp = q6adm_find_copp(adm, port_idx, copp_idx); - if (!copp) - return 0; - - copp->result = *result; - wake_up(&copp->wait); - kref_put(&copp->refcount, q6adm_free_copp); + list_for_each_entry(copp, &adm->copps_list, node) { + if ((port_idx == copp->afe_port) && (copp_idx == copp->copp_idx)) { + copp->result = *result; + wake_up(&copp->wait); + break; + } + } break; case ADM_CMD_MATRIX_MAP_ROUTINGS_V5: adm->result = *result; @@ -234,65 +298,6 @@ static struct q6copp *q6adm_alloc_copp(struct q6adm *adm, int port_idx) return c; } -static int q6adm_apr_send_copp_pkt(struct q6adm *adm, struct q6copp *copp, - struct apr_pkt *pkt, uint32_t rsp_opcode) -{ - struct device *dev = adm->dev; - uint32_t opcode = pkt->hdr.opcode; - int ret; - - mutex_lock(&adm->lock); - copp->result.opcode = 0; - copp->result.status = 0; - ret = apr_send_pkt(adm->apr, pkt); - if (ret < 0) { - dev_err(dev, "Failed to send APR packet\n"); - ret = -EINVAL; - goto err; - } - - /* Wait for the callback with copp id */ - if (rsp_opcode) - ret = wait_event_timeout(copp->wait, - (copp->result.opcode == opcode) || - (copp->result.opcode == rsp_opcode), - msecs_to_jiffies(TIMEOUT_MS)); - else - ret = wait_event_timeout(copp->wait, - (copp->result.opcode == opcode), - msecs_to_jiffies(TIMEOUT_MS)); - - if (!ret) { - dev_err(dev, "ADM copp cmd timedout\n"); - ret = -ETIMEDOUT; - } else if (copp->result.status > 0) { - dev_err(dev, "DSP returned error[%d]\n", - copp->result.status); - ret = -EINVAL; - } - -err: - mutex_unlock(&adm->lock); - return ret; -} - -static int q6adm_device_close(struct q6adm *adm, struct q6copp *copp, - int port_id, int copp_idx) -{ - struct apr_pkt close; - - close.hdr.hdr_field = APR_HDR_FIELD(APR_MSG_TYPE_SEQ_CMD, - APR_HDR_LEN(APR_HDR_SIZE), - APR_PKT_VER); - close.hdr.pkt_size = sizeof(close); - close.hdr.src_port = port_id; - close.hdr.dest_port = copp->id; - close.hdr.token = port_id << 16 | copp_idx; - close.hdr.opcode = ADM_CMD_DEVICE_CLOSE_V5; - - return q6adm_apr_send_copp_pkt(adm, copp, &close, 0); -} - static struct q6copp *q6adm_find_matching_copp(struct q6adm *adm, int port_id, int topology, int mode, int rate, @@ -567,15 +572,6 @@ EXPORT_SYMBOL_GPL(q6adm_matrix_map); */ int q6adm_close(struct device *dev, struct q6copp *copp) { - struct q6adm *adm = dev_get_drvdata(dev->parent); - int ret = 0; - - ret = q6adm_device_close(adm, copp, copp->afe_port, copp->copp_idx); - if (ret < 0) { - dev_err(adm->dev, "Failed to close copp %d\n", ret); - return ret; - } - kref_put(&copp->refcount, q6adm_free_copp); return 0; diff --git a/sound/soc/qcom/qdsp6/q6apm-dai.c b/sound/soc/qcom/qdsp6/q6apm-dai.c index 4ecaff45c5186..a188fffc705e7 100644 --- a/sound/soc/qcom/qdsp6/q6apm-dai.c +++ b/sound/soc/qcom/qdsp6/q6apm-dai.c @@ -86,6 +86,7 @@ static const struct snd_pcm_hardware q6apm_dai_hardware_capture = { .info = (SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_BLOCK_TRANSFER | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME | + SNDRV_PCM_INFO_NO_REWINDS | SNDRV_PCM_INFO_SYNC_APPLPTR | SNDRV_PCM_INFO_BATCH), .formats = (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE), .rates = SNDRV_PCM_RATE_8000_48000, @@ -105,6 +106,7 @@ static const struct snd_pcm_hardware q6apm_dai_hardware_playback = { .info = (SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_BLOCK_TRANSFER | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME | + SNDRV_PCM_INFO_NO_REWINDS | SNDRV_PCM_INFO_SYNC_APPLPTR | SNDRV_PCM_INFO_BATCH), .formats = (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE), .rates = SNDRV_PCM_RATE_8000_192000, @@ -849,6 +851,7 @@ static const struct snd_soc_component_driver q6apm_fe_dai_component = { .ack = q6apm_dai_ack, .compress_ops = &q6apm_dai_compress_ops, .use_dai_pcm_id = true, + .remove_order = SND_SOC_COMP_ORDER_EARLY, }; static int q6apm_dai_probe(struct platform_device *pdev) diff --git a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c index 528756f1332bc..1cd8774f245ca 100644 --- a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c +++ b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c @@ -159,6 +159,31 @@ static void q6apm_lpass_dai_shutdown(struct snd_pcm_substream *substream, struct } } +static int q6apm_lpass_dai_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + struct q6apm_lpass_dai_data *dai_data = dev_get_drvdata(dai->dev); + int ret = 0; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + if (!dai_data->is_port_started[dai->id]) { + ret = q6apm_graph_start(dai_data->graph[dai->id]); + if (ret < 0) + dev_err(dai->dev, "Failed to start APM port %d\n", dai->id); + else + dai_data->is_port_started[dai->id] = true; + } + break; + default: + break; + } + + return ret; +} + static int q6apm_lpass_dai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct q6apm_lpass_dai_data *dai_data = dev_get_drvdata(dai->dev); @@ -171,10 +196,6 @@ static int q6apm_lpass_dai_prepare(struct snd_pcm_substream *substream, struct s q6apm_graph_stop(dai_data->graph[dai->id]); dai_data->is_port_started[dai->id] = false; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - q6apm_graph_close(dai_data->graph[dai->id]); - dai_data->graph[dai->id] = NULL; - } } /** @@ -203,14 +224,6 @@ static int q6apm_lpass_dai_prepare(struct snd_pcm_substream *substream, struct s dev_err(dai->dev, "Failed to prepare Graph %d\n", rc); goto err; } - - rc = q6apm_graph_start(dai_data->graph[dai->id]); - if (rc < 0) { - dev_err(dai->dev, "Failed to start APM port %d\n", dai->id); - goto err; - } - dai_data->is_port_started[dai->id] = true; - return 0; err: if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { @@ -254,6 +267,7 @@ static const struct snd_soc_dai_ops q6dma_ops = { .shutdown = q6apm_lpass_dai_shutdown, .set_channel_map = q6dma_set_channel_map, .hw_params = q6dma_hw_params, + .trigger = q6apm_lpass_dai_trigger, }; static const struct snd_soc_dai_ops q6i2s_ops = { @@ -263,6 +277,7 @@ static const struct snd_soc_dai_ops q6i2s_ops = { .set_channel_map = q6dma_set_channel_map, .hw_params = q6dma_hw_params, .set_fmt = q6i2s_set_fmt, + .trigger = q6apm_lpass_dai_trigger, }; static const struct snd_soc_dai_ops q6hdmi_ops = { @@ -271,6 +286,7 @@ static const struct snd_soc_dai_ops q6hdmi_ops = { .shutdown = q6apm_lpass_dai_shutdown, .hw_params = q6hdmi_hw_params, .set_fmt = q6i2s_set_fmt, + .trigger = q6apm_lpass_dai_trigger, }; static const struct snd_soc_component_driver q6apm_lpass_dai_component = { @@ -278,6 +294,7 @@ static const struct snd_soc_component_driver q6apm_lpass_dai_component = { .of_xlate_dai_name = q6dsp_audio_ports_of_xlate_dai_name, .be_pcm_base = AUDIOREACH_BE_PCM_BASE, .use_dai_pcm_id = true, + .remove_order = SND_SOC_COMP_ORDER_FIRST, }; static int q6apm_lpass_dai_dev_probe(struct platform_device *pdev) diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 0e667a7eb5467..7d574f3daa3dc 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -735,6 +735,7 @@ static const struct snd_soc_component_driver q6apm_audio_component = { .name = APM_AUDIO_DRV_NAME, .probe = q6apm_audio_probe, .remove = q6apm_audio_remove, + .remove_order = SND_SOC_COMP_ORDER_LAST, }; static int apm_probe(gpr_device_t *gdev) @@ -766,13 +767,22 @@ static int apm_probe(gpr_device_t *gdev) q6apm_get_apm_state(apm); - ret = devm_snd_soc_register_component(dev, &q6apm_audio_component, NULL, 0); + ret = snd_soc_register_component(dev, &q6apm_audio_component, NULL, 0); if (ret < 0) { dev_err(dev, "failed to register q6apm: %d\n", ret); return ret; } + + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + snd_soc_unregister_component(dev); - return of_platform_populate(dev->of_node, NULL, NULL, dev); + return ret; +} + +static void apm_remove(gpr_device_t *gdev) +{ + snd_soc_unregister_component(&gdev->dev); } struct audioreach_module *q6apm_find_module_by_mid(struct q6apm_graph *graph, uint32_t mid) @@ -839,6 +849,7 @@ MODULE_DEVICE_TABLE(of, apm_device_id); static gpr_driver_t apm_driver = { .probe = apm_probe, + .remove = apm_remove, .gpr_callback = apm_callback, .driver = { .name = "qcom-apm", diff --git a/sound/soc/qcom/qdsp6/q6asm-dai.c b/sound/soc/qcom/qdsp6/q6asm-dai.c index b616ce316d2f1..0eae8c6e42b8a 100644 --- a/sound/soc/qcom/qdsp6/q6asm-dai.c +++ b/sound/soc/qcom/qdsp6/q6asm-dai.c @@ -233,13 +233,14 @@ static int q6asm_dai_prepare(struct snd_soc_component *component, prtd->pcm_count = snd_pcm_lib_period_bytes(substream); prtd->pcm_irq_pos = 0; /* rate and channels are sent to audio driver */ - if (prtd->state) { + if (prtd->state == Q6ASM_STREAM_RUNNING) { /* clear the previous setup if any */ q6asm_cmd(prtd->audio_client, prtd->stream_id, CMD_CLOSE); q6asm_unmap_memory_regions(substream->stream, prtd->audio_client); q6routing_stream_close(soc_prtd->dai_link->id, substream->stream); + prtd->state = Q6ASM_STREAM_STOPPED; } ret = q6asm_map_memory_regions(substream->stream, prtd->audio_client, @@ -403,13 +404,13 @@ static int q6asm_dai_open(struct snd_soc_component *component, } ret = snd_pcm_hw_constraint_step(runtime, 0, - SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 32); + SNDRV_PCM_HW_PARAM_PERIOD_SIZE, 480); if (ret < 0) { dev_err(dev, "constraint for period bytes step ret = %d\n", ret); } ret = snd_pcm_hw_constraint_step(runtime, 0, - SNDRV_PCM_HW_PARAM_BUFFER_BYTES, 32); + SNDRV_PCM_HW_PARAM_BUFFER_SIZE, 480); if (ret < 0) { dev_err(dev, "constraint for buffer bytes step ret = %d\n", ret); diff --git a/sound/soc/qcom/qdsp6/q6dsp-lpass-ports.c b/sound/soc/qcom/qdsp6/q6dsp-lpass-ports.c index 4eed54b071a5c..8542e620ee02d 100644 --- a/sound/soc/qcom/qdsp6/q6dsp-lpass-ports.c +++ b/sound/soc/qcom/qdsp6/q6dsp-lpass-ports.c @@ -10,16 +10,14 @@ #define Q6AFE_TDM_PB_DAI(pre, num, did) { \ .playback = { \ .stream_name = pre" TDM"#num" Playback", \ - .rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000 |\ - SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_48000 |\ - SNDRV_PCM_RATE_176400, \ + .rates = SNDRV_PCM_RATE_8000_192000, \ .formats = SNDRV_PCM_FMTBIT_S16_LE | \ SNDRV_PCM_FMTBIT_S24_LE | \ SNDRV_PCM_FMTBIT_S32_LE, \ .channels_min = 1, \ .channels_max = 8, \ .rate_min = 8000, \ - .rate_max = 176400, \ + .rate_max = 192000, \ }, \ .name = #did, \ .id = did, \ @@ -28,16 +26,14 @@ #define Q6AFE_TDM_CAP_DAI(pre, num, did) { \ .capture = { \ .stream_name = pre" TDM"#num" Capture", \ - .rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000 |\ - SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_48000 |\ - SNDRV_PCM_RATE_176400, \ + .rates = SNDRV_PCM_RATE_8000_192000, \ .formats = SNDRV_PCM_FMTBIT_S16_LE | \ SNDRV_PCM_FMTBIT_S24_LE | \ SNDRV_PCM_FMTBIT_S32_LE, \ .channels_min = 1, \ .channels_max = 8, \ .rate_min = 8000, \ - .rate_max = 176400, \ + .rate_max = 192000, \ }, \ .name = #did, \ .id = did, \ @@ -46,16 +42,14 @@ #define Q6AFE_CDC_DMA_RX_DAI(did) { \ .playback = { \ .stream_name = #did" Playback", \ - .rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000 |\ - SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_48000 |\ - SNDRV_PCM_RATE_176400, \ + .rates = SNDRV_PCM_RATE_8000_192000, \ .formats = SNDRV_PCM_FMTBIT_S16_LE | \ SNDRV_PCM_FMTBIT_S24_LE | \ SNDRV_PCM_FMTBIT_S32_LE, \ .channels_min = 1, \ .channels_max = 8, \ .rate_min = 8000, \ - .rate_max = 176400, \ + .rate_max = 192000, \ }, \ .name = #did, \ .id = did, \ @@ -64,16 +58,14 @@ #define Q6AFE_CDC_DMA_TX_DAI(did) { \ .capture = { \ .stream_name = #did" Capture", \ - .rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000 |\ - SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_48000 |\ - SNDRV_PCM_RATE_176400, \ + .rates = SNDRV_PCM_RATE_8000_192000, \ .formats = SNDRV_PCM_FMTBIT_S16_LE | \ SNDRV_PCM_FMTBIT_S24_LE | \ SNDRV_PCM_FMTBIT_S32_LE, \ .channels_min = 1, \ .channels_max = 8, \ .rate_min = 8000, \ - .rate_max = 176400, \ + .rate_max = 192000, \ }, \ .name = #did, \ .id = did, \ @@ -350,119 +342,122 @@ static struct snd_soc_dai_driver q6dsp_audio_fe_dais[] = { }, { .playback = { .stream_name = "Primary MI2S Playback", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, + .rates = SNDRV_PCM_RATE_8000_192000, .formats = SNDRV_PCM_FMTBIT_S16_LE | - SNDRV_PCM_FMTBIT_S24_LE, + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .id = PRIMARY_MI2S_RX, .name = "PRI_MI2S_RX", }, { .capture = { .stream_name = "Primary MI2S Capture", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, + .rates = SNDRV_PCM_RATE_8000_192000, .formats = SNDRV_PCM_FMTBIT_S16_LE | - SNDRV_PCM_FMTBIT_S24_LE, + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .id = PRIMARY_MI2S_TX, .name = "PRI_MI2S_TX", }, { .playback = { .stream_name = "Secondary MI2S Playback", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, - .formats = SNDRV_PCM_FMTBIT_S16_LE, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .name = "SEC_MI2S_RX", .id = SECONDARY_MI2S_RX, }, { .capture = { .stream_name = "Secondary MI2S Capture", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, + .rates = SNDRV_PCM_RATE_8000_192000, .formats = SNDRV_PCM_FMTBIT_S16_LE | - SNDRV_PCM_FMTBIT_S24_LE, + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .id = SECONDARY_MI2S_TX, .name = "SEC_MI2S_TX", }, { .playback = { .stream_name = "Tertiary MI2S Playback", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, - .formats = SNDRV_PCM_FMTBIT_S16_LE, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .name = "TERT_MI2S_RX", .id = TERTIARY_MI2S_RX, }, { .capture = { .stream_name = "Tertiary MI2S Capture", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, + .rates = SNDRV_PCM_RATE_8000_192000, .formats = SNDRV_PCM_FMTBIT_S16_LE | - SNDRV_PCM_FMTBIT_S24_LE, + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .id = TERTIARY_MI2S_TX, .name = "TERT_MI2S_TX", }, { .playback = { .stream_name = "Quaternary MI2S Playback", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, - .formats = SNDRV_PCM_FMTBIT_S16_LE, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .name = "QUAT_MI2S_RX", .id = QUATERNARY_MI2S_RX, }, { .capture = { .stream_name = "Quaternary MI2S Capture", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, + .rates = SNDRV_PCM_RATE_8000_192000, .formats = SNDRV_PCM_FMTBIT_S16_LE | - SNDRV_PCM_FMTBIT_S24_LE, + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .id = QUATERNARY_MI2S_TX, .name = "QUAT_MI2S_TX", }, { .playback = { .stream_name = "Quinary MI2S Playback", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000 | SNDRV_PCM_RATE_96000 | - SNDRV_PCM_RATE_192000, - .formats = SNDRV_PCM_FMTBIT_S16_LE, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, @@ -473,13 +468,14 @@ static struct snd_soc_dai_driver q6dsp_audio_fe_dais[] = { }, { .capture = { .stream_name = "Quinary MI2S Capture", - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_8000 | - SNDRV_PCM_RATE_16000, - .formats = SNDRV_PCM_FMTBIT_S16_LE, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, .channels_min = 1, .channels_max = 8, .rate_min = 8000, - .rate_max = 48000, + .rate_max = 192000, }, .id = QUINARY_MI2S_TX, .name = "QUIN_MI2S_TX", diff --git a/sound/soc/qcom/qdsp6/q6prm-clocks.c b/sound/soc/qcom/qdsp6/q6prm-clocks.c index 4c574b48ab004..51b131fa95316 100644 --- a/sound/soc/qcom/qdsp6/q6prm-clocks.c +++ b/sound/soc/qcom/qdsp6/q6prm-clocks.c @@ -42,6 +42,11 @@ static const struct q6dsp_clk_init q6prm_clks[] = { Q6PRM_CLK(LPASS_CLK_ID_INT5_MI2S_IBIT), Q6PRM_CLK(LPASS_CLK_ID_INT6_MI2S_IBIT), Q6PRM_CLK(LPASS_CLK_ID_QUI_MI2S_OSR), + Q6PRM_CLK(LPASS_CLK_ID_MCLK_1), + Q6PRM_CLK(LPASS_CLK_ID_MCLK_2), + Q6PRM_CLK(LPASS_CLK_ID_MCLK_3), + Q6PRM_CLK(LPASS_CLK_ID_MCLK_4), + Q6PRM_CLK(LPASS_CLK_ID_MCLK_5), Q6PRM_CLK(LPASS_CLK_ID_WSA_CORE_MCLK), Q6PRM_CLK(LPASS_CLK_ID_WSA_CORE_NPL_MCLK), Q6PRM_CLK(LPASS_CLK_ID_VA_CORE_MCLK), diff --git a/sound/soc/qcom/qdsp6/q6prm.h b/sound/soc/qcom/qdsp6/q6prm.h index a988a32086fe1..8296370e3cbf7 100644 --- a/sound/soc/qcom/qdsp6/q6prm.h +++ b/sound/soc/qcom/qdsp6/q6prm.h @@ -52,6 +52,17 @@ /* Clock ID for QUINARY MI2S OSR CLK */ #define Q6PRM_LPASS_CLK_ID_QUI_MI2S_OSR 0x116 +/* Clock ID for MCLK1 */ +#define Q6PRM_LPASS_CLK_ID_MCLK_1 0x300 +/* Clock ID for MCLK2 */ +#define Q6PRM_LPASS_CLK_ID_MCLK_2 0x301 +/* Clock ID for MCLK3 */ +#define Q6PRM_LPASS_CLK_ID_MCLK_3 0x302 +/* Clock ID for MCLK4 */ +#define Q6PRM_LPASS_CLK_ID_MCLK_4 0x303 +/* Clock ID for MCLK5 */ +#define Q6PRM_LPASS_CLK_ID_MCLK_5 0x304 + #define Q6PRM_LPASS_CLK_ID_WSA_CORE_MCLK 0x305 #define Q6PRM_LPASS_CLK_ID_WSA_CORE_NPL_MCLK 0x306 diff --git a/sound/soc/qcom/qdsp6/topology.c b/sound/soc/qcom/qdsp6/topology.c index f61285e7dcf20..f8d3bb83e3b0d 100644 --- a/sound/soc/qcom/qdsp6/topology.c +++ b/sound/soc/qcom/qdsp6/topology.c @@ -952,9 +952,6 @@ static int audioreach_widget_unload(struct snd_soc_component *scomp, struct audioreach_container *cont; struct audioreach_module *mod; - mod = dobj->private; - cont = mod->container; - if (w->id == snd_soc_dapm_mixer) { /* virtual widget */ struct snd_ar_control *scontrol = dobj->private; @@ -963,6 +960,11 @@ static int audioreach_widget_unload(struct snd_soc_component *scomp, kfree(scontrol); return 0; } + mod = dobj->private; + if (!mod) + return 0; + + cont = mod->container; mutex_lock(&apm->lock); idr_remove(&apm->modules_idr, mod->instance_id); diff --git a/sound/soc/qcom/sc7280.c b/sound/soc/qcom/sc7280.c index af412bd0c89ff..c444dae563c75 100644 --- a/sound/soc/qcom/sc7280.c +++ b/sound/soc/qcom/sc7280.c @@ -317,7 +317,7 @@ static void sc7280_snd_shutdown(struct snd_pcm_substream *substream) struct snd_soc_card *card = rtd->card; struct sc7280_snd_data *data = snd_soc_card_get_drvdata(card); struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); - struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; + struct sdw_stream_runtime *sruntime = qcom_snd_sdw_get_stream(substream); switch (cpu_dai->id) { case MI2S_PRIMARY: diff --git a/sound/soc/qcom/sc8280xp.c b/sound/soc/qcom/sc8280xp.c index 187f37ffe3283..ed8b04c6022e9 100644 --- a/sound/soc/qcom/sc8280xp.c +++ b/sound/soc/qcom/sc8280xp.c @@ -73,7 +73,7 @@ static void sc8280xp_snd_shutdown(struct snd_pcm_substream *substream) struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); struct sc8280xp_snd_data *pdata = snd_soc_card_get_drvdata(rtd->card); - struct sdw_stream_runtime *sruntime = pdata->sruntime[cpu_dai->id]; + struct sdw_stream_runtime *sruntime = qcom_snd_sdw_get_stream(substream); pdata->sruntime[cpu_dai->id] = NULL; sdw_release_stream(sruntime); diff --git a/sound/soc/qcom/sdw.c b/sound/soc/qcom/sdw.c index 7d7981d4295b4..5f880c74c8dc2 100644 --- a/sound/soc/qcom/sdw.c +++ b/sound/soc/qcom/sdw.c @@ -2,11 +2,53 @@ // Copyright (c) 2018-2023, Linaro Limited. // Copyright (c) 2018, The Linux Foundation. All rights reserved. +#include #include #include #include #include "sdw.h" +static bool qcom_snd_is_sdw_dai(int id) +{ + switch (id) { + case WSA_CODEC_DMA_RX_0: + case WSA_CODEC_DMA_TX_0: + case WSA_CODEC_DMA_RX_1: + case WSA_CODEC_DMA_TX_1: + case WSA_CODEC_DMA_TX_2: + case RX_CODEC_DMA_RX_0: + case TX_CODEC_DMA_TX_0: + case RX_CODEC_DMA_RX_1: + case TX_CODEC_DMA_TX_1: + case RX_CODEC_DMA_RX_2: + case TX_CODEC_DMA_TX_2: + case RX_CODEC_DMA_RX_3: + case TX_CODEC_DMA_TX_3: + case RX_CODEC_DMA_RX_4: + case TX_CODEC_DMA_TX_4: + case RX_CODEC_DMA_RX_5: + case TX_CODEC_DMA_TX_5: + case RX_CODEC_DMA_RX_6: + case RX_CODEC_DMA_RX_7: + case SLIMBUS_0_RX...SLIMBUS_6_TX: + return true; + default: + break; + } + + /* DSP Bypass usecase, cpu dai index overlaps with DSP dai ids, + * DO NOT MERGE into top switch case */ + switch (id) { + case LPASS_CDC_DMA_TX3: + case LPASS_CDC_DMA_RX0: + return true; + default: + break; + } + + return false; +} + /** * qcom_snd_sdw_startup() - Helper to start Soundwire stream for SoC audio card * @substream: The PCM substream from audio, as passed to snd_soc_ops->startup() @@ -29,6 +71,9 @@ int qcom_snd_sdw_startup(struct snd_pcm_substream *substream) u32 rx_ch_cnt = 0, tx_ch_cnt = 0; int ret, i, j; + if (!qcom_snd_is_sdw_dai(cpu_dai->id)) + return 0; + sruntime = sdw_alloc_stream(cpu_dai->name, SDW_STREAM_PCM); if (!sruntime) return -ENOMEM; @@ -89,19 +134,8 @@ int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream, if (!sruntime) return 0; - switch (cpu_dai->id) { - case WSA_CODEC_DMA_RX_0: - case WSA_CODEC_DMA_RX_1: - case RX_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_1: - case TX_CODEC_DMA_TX_0: - case TX_CODEC_DMA_TX_1: - case TX_CODEC_DMA_TX_2: - case TX_CODEC_DMA_TX_3: - break; - default: + if (!qcom_snd_is_sdw_dai(cpu_dai->id)) return 0; - } if (*stream_prepared) return 0; @@ -129,9 +163,7 @@ int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream, } EXPORT_SYMBOL_GPL(qcom_snd_sdw_prepare); -int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params, - struct sdw_stream_runtime **psruntime) +struct sdw_stream_runtime *qcom_snd_sdw_get_stream(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); struct snd_soc_dai *codec_dai; @@ -139,21 +171,23 @@ int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream, struct sdw_stream_runtime *sruntime; int i; - switch (cpu_dai->id) { - case WSA_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_1: - case TX_CODEC_DMA_TX_0: - case TX_CODEC_DMA_TX_1: - case TX_CODEC_DMA_TX_2: - case TX_CODEC_DMA_TX_3: - for_each_rtd_codec_dais(rtd, i, codec_dai) { - sruntime = snd_soc_dai_get_stream(codec_dai, substream->stream); - if (sruntime != ERR_PTR(-ENOTSUPP)) - *psruntime = sruntime; - } - break; + if (!qcom_snd_is_sdw_dai(cpu_dai->id)) + return NULL; + + for_each_rtd_codec_dais(rtd, i, codec_dai) { + sruntime = snd_soc_dai_get_stream(codec_dai, substream->stream); + if (sruntime != ERR_PTR(-ENOTSUPP)) + return sruntime; } + return NULL; +} +EXPORT_SYMBOL_GPL(qcom_snd_sdw_get_stream); + +int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct sdw_stream_runtime **psruntime) +{ + *psruntime = qcom_snd_sdw_get_stream(substream); return 0; @@ -166,23 +200,13 @@ int qcom_snd_sdw_hw_free(struct snd_pcm_substream *substream, struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); - switch (cpu_dai->id) { - case WSA_CODEC_DMA_RX_0: - case WSA_CODEC_DMA_RX_1: - case RX_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_1: - case TX_CODEC_DMA_TX_0: - case TX_CODEC_DMA_TX_1: - case TX_CODEC_DMA_TX_2: - case TX_CODEC_DMA_TX_3: - if (sruntime && *stream_prepared) { - sdw_disable_stream(sruntime); - sdw_deprepare_stream(sruntime); - *stream_prepared = false; - } - break; - default: - break; + if (!qcom_snd_is_sdw_dai(cpu_dai->id)) + return 0; + + if (sruntime && *stream_prepared) { + sdw_disable_stream(sruntime); + sdw_deprepare_stream(sruntime); + *stream_prepared = false; } return 0; diff --git a/sound/soc/qcom/sdw.h b/sound/soc/qcom/sdw.h index 392e3455f1b19..b8bc5beb05223 100644 --- a/sound/soc/qcom/sdw.h +++ b/sound/soc/qcom/sdw.h @@ -10,6 +10,7 @@ int qcom_snd_sdw_startup(struct snd_pcm_substream *substream); int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream, struct sdw_stream_runtime *runtime, bool *stream_prepared); +struct sdw_stream_runtime *qcom_snd_sdw_get_stream(struct snd_pcm_substream *stream); int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct sdw_stream_runtime **psruntime); diff --git a/sound/soc/qcom/sm8250.c b/sound/soc/qcom/sm8250.c index f5b75a06e5bd2..ce5b0059207f1 100644 --- a/sound/soc/qcom/sm8250.c +++ b/sound/soc/qcom/sm8250.c @@ -117,7 +117,7 @@ static void sm8250_snd_shutdown(struct snd_pcm_substream *substream) struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); struct sm8250_snd_data *data = snd_soc_card_get_drvdata(rtd->card); - struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; + struct sdw_stream_runtime *sruntime = qcom_snd_sdw_get_stream(substream); data->sruntime[cpu_dai->id] = NULL; sdw_release_stream(sruntime); diff --git a/sound/soc/qcom/x1e80100.c b/sound/soc/qcom/x1e80100.c index 444f2162889f7..2e3599516aa2a 100644 --- a/sound/soc/qcom/x1e80100.c +++ b/sound/soc/qcom/x1e80100.c @@ -55,7 +55,7 @@ static void x1e80100_snd_shutdown(struct snd_pcm_substream *substream) struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); struct x1e80100_snd_data *data = snd_soc_card_get_drvdata(rtd->card); - struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; + struct sdw_stream_runtime *sruntime = qcom_snd_sdw_get_stream(substream); data->sruntime[cpu_dai->id] = NULL; sdw_release_stream(sruntime); diff --git a/sound/soc/renesas/rz-ssi.c b/sound/soc/renesas/rz-ssi.c index 81b883e8ac92a..f4dc2f68deada 100644 --- a/sound/soc/renesas/rz-ssi.c +++ b/sound/soc/renesas/rz-ssi.c @@ -13,6 +13,7 @@ #include #include #include +#include #include /* REGISTER OFFSET */ @@ -87,7 +88,6 @@ struct rz_ssi_stream { int dma_buffer_pos; /* The address for the next DMA descriptor */ int completed_dma_buf_pos; /* The address of the last completed DMA descriptor. */ int period_counter; /* for keeping track of periods transferred */ - int sample_width; int buffer_pos; /* current frame position in the buffer */ int running; /* 0=stopped, 1=running */ @@ -133,6 +133,12 @@ struct rz_ssi_priv { bool bckp_rise; /* Bit clock polarity (SSICR.BCKP) */ bool dma_rt; + struct { + bool tx_active; + bool rx_active; + bool one_stream_triggered; + } dup; + /* Full duplex communication support */ struct { unsigned int rate; @@ -211,10 +217,7 @@ static inline bool rz_ssi_is_stream_running(struct rz_ssi_stream *strm) static void rz_ssi_stream_init(struct rz_ssi_stream *strm, struct snd_pcm_substream *substream) { - struct snd_pcm_runtime *runtime = substream->runtime; - rz_ssi_set_substream(strm, substream); - strm->sample_width = samples_to_bytes(runtime, 1); strm->dma_buffer_pos = 0; strm->completed_dma_buf_pos = 0; strm->period_counter = 0; @@ -332,13 +335,12 @@ static int rz_ssi_start(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm) bool is_full_duplex; u32 ssicr, ssifcr; - is_full_duplex = rz_ssi_is_stream_running(&ssi->playback) || - rz_ssi_is_stream_running(&ssi->capture); + is_full_duplex = ssi->dup.tx_active && ssi->dup.rx_active; ssicr = rz_ssi_reg_readl(ssi, SSICR); ssifcr = rz_ssi_reg_readl(ssi, SSIFCR); if (!is_full_duplex) { ssifcr &= ~0xF; - } else { + } else if (ssi->dup.one_stream_triggered) { rz_ssi_reg_mask_setl(ssi, SSICR, SSICR_TEN | SSICR_REN, 0); rz_ssi_set_idle(ssi); ssifcr &= ~SSIFCR_FIFO_RST; @@ -374,12 +376,16 @@ static int rz_ssi_start(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm) SSISR_RUIRQ), 0); strm->running = 1; - if (is_full_duplex) - ssicr |= SSICR_TEN | SSICR_REN; - else + if (!is_full_duplex) { ssicr |= is_play ? SSICR_TEN : SSICR_REN; - - rz_ssi_reg_writel(ssi, SSICR, ssicr); + rz_ssi_reg_writel(ssi, SSICR, ssicr); + } else if (ssi->dup.one_stream_triggered) { + ssicr |= SSICR_TEN | SSICR_REN; + rz_ssi_reg_writel(ssi, SSICR, ssicr); + ssi->dup.one_stream_triggered = false; + } else { + ssi->dup.one_stream_triggered = true; + } return 0; } @@ -915,6 +921,30 @@ static int rz_ssi_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) return 0; } +static int rz_ssi_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai); + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + ssi->dup.tx_active = true; + else + ssi->dup.rx_active = true; + + return 0; +} + +static void rz_ssi_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai); + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + ssi->dup.tx_active = false; + else + ssi->dup.rx_active = false; +} + static bool rz_ssi_is_valid_hw_params(struct rz_ssi_priv *ssi, unsigned int rate, unsigned int channels, unsigned int sample_width, @@ -945,9 +975,9 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai); - struct rz_ssi_stream *strm = rz_ssi_stream_get(ssi, substream); unsigned int sample_bits = hw_param_interval(params, SNDRV_PCM_HW_PARAM_SAMPLE_BITS)->min; + unsigned int sample_width = params_width(params); unsigned int channels = params_channels(params); unsigned int rate = params_rate(params); int ret; @@ -966,16 +996,14 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream, if (rz_ssi_is_stream_running(&ssi->playback) || rz_ssi_is_stream_running(&ssi->capture)) { - if (rz_ssi_is_valid_hw_params(ssi, rate, channels, - strm->sample_width, sample_bits)) + if (rz_ssi_is_valid_hw_params(ssi, rate, channels, sample_width, sample_bits)) return 0; dev_err(ssi->dev, "Full duplex needs same HW params\n"); return -EINVAL; } - rz_ssi_cache_hw_params(ssi, rate, channels, strm->sample_width, - sample_bits); + rz_ssi_cache_hw_params(ssi, rate, channels, sample_width, sample_bits); ret = rz_ssi_swreset(ssi); if (ret) @@ -985,6 +1013,8 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream, } static const struct snd_soc_dai_ops rz_ssi_dai_ops = { + .startup = rz_ssi_startup, + .shutdown = rz_ssi_shutdown, .trigger = rz_ssi_dai_trigger, .set_fmt = rz_ssi_dai_set_fmt, .hw_params = rz_ssi_dai_hw_params, diff --git a/sound/soc/rockchip/rockchip_i2s_tdm.c b/sound/soc/rockchip/rockchip_i2s_tdm.c index d9a1fab7f4031..e4697ee0addc8 100644 --- a/sound/soc/rockchip/rockchip_i2s_tdm.c +++ b/sound/soc/rockchip/rockchip_i2s_tdm.c @@ -22,6 +22,7 @@ #define DRV_NAME "rockchip-i2s-tdm" +#define DEFAULT_MCLK_FS 256 #define CH_GRP_MAX 4 /* The max channel 8 / 2 */ #define MULTIPLEX_CH_MAX 10 @@ -665,6 +666,15 @@ static int rockchip_i2s_tdm_hw_params(struct snd_pcm_substream *substream, mclk_rate = i2s_tdm->mclk_rx_freq; } + /* + * When the dai/component driver doesn't need to set mclk-fs for a specific + * clock, it can skip the call to set_sysclk() for that clock. + * In that case, simply use the clock rate from the params and multiply it by + * the default mclk-fs value. + */ + if (!mclk_rate) + mclk_rate = DEFAULT_MCLK_FS * params_rate(params); + err = clk_set_rate(mclk, mclk_rate); if (err) return err; diff --git a/sound/soc/rockchip/rockchip_pdm.c b/sound/soc/rockchip/rockchip_pdm.c index c1ee470ec6079..c69cdd6f24994 100644 --- a/sound/soc/rockchip/rockchip_pdm.c +++ b/sound/soc/rockchip/rockchip_pdm.c @@ -580,7 +580,7 @@ static int rockchip_pdm_probe(struct platform_device *pdev) if (!pdm) return -ENOMEM; - pdm->version = (enum rk_pdm_version)device_get_match_data(&pdev->dev); + pdm->version = (unsigned long)device_get_match_data(&pdev->dev); if (pdm->version == RK_PDM_RK3308) { pdm->reset = devm_reset_control_get(&pdev->dev, "pdm-m"); if (IS_ERR(pdm->reset)) diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index e9964f0e010ae..140907a41a70d 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -1360,10 +1360,10 @@ static int i2s_create_secondary_device(struct samsung_i2s_priv *priv) if (!pdev_sec) return -ENOMEM; - pdev_sec->driver_override = kstrdup("samsung-i2s", GFP_KERNEL); - if (!pdev_sec->driver_override) { + ret = device_set_driver_override(&pdev_sec->dev, "samsung-i2s"); + if (ret) { platform_device_put(pdev_sec); - return -ENOMEM; + return ret; } ret = platform_device_add(pdev_sec); diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c index c493ec530cc5c..197a592ec2f16 100644 --- a/sound/soc/sdca/sdca_asoc.c +++ b/sound/soc/sdca/sdca_asoc.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -795,7 +796,6 @@ static int control_limit_kctl(struct device *dev, struct sdca_control_range *range; int min, max, step; unsigned int *tlv; - int shift; if (control->type != SDCA_CTL_DATATYPE_Q7P8DB) return 0; @@ -814,37 +814,22 @@ static int control_limit_kctl(struct device *dev, min = sign_extend32(min, control->nbits - 1); max = sign_extend32(max, control->nbits - 1); - /* - * FIXME: Only support power of 2 step sizes as this can be supported - * by a simple shift. - */ - if (hweight32(step) != 1) { - dev_err(dev, "%s: %s: currently unsupported step size\n", - entity->label, control->label); - return -EINVAL; - } - - /* - * The SDCA volumes are in steps of 1/256th of a dB, a step down of - * 64 (shift of 6) gives 1/4dB. 1/4dB is the smallest unit that is also - * representable in the ALSA TLVs which are in 1/100ths of a dB. - */ - shift = max(ffs(step) - 1, 6); - tlv = devm_kcalloc(dev, 4, sizeof(*tlv), GFP_KERNEL); if (!tlv) return -ENOMEM; - tlv[0] = SNDRV_CTL_TLVT_DB_SCALE; + tlv[0] = SNDRV_CTL_TLVT_DB_MINMAX; tlv[1] = 2 * sizeof(*tlv); tlv[2] = (min * 100) >> 8; - tlv[3] = ((1 << shift) * 100) >> 8; + tlv[3] = (max * 100) >> 8; - mc->min = min >> shift; - mc->max = max >> shift; - mc->shift = shift; - mc->rshift = shift; - mc->sign_bit = 15 - shift; + step = (step * 100) >> 8; + + mc->min = ((int)tlv[2] / step); + mc->max = ((int)tlv[3] / step); + mc->shift = step; + mc->sign_bit = 15; + mc->sdca_q78 = 1; kctl->tlv.p = tlv; kctl->access |= SNDRV_CTL_ELEM_ACCESS_TLV_READ; @@ -852,6 +837,48 @@ static int control_limit_kctl(struct device *dev, return 0; } +static int volatile_get_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume reading %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_get_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + +static int volatile_put_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume writing %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_put_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + static int populate_control(struct device *dev, struct sdca_function_data *function, struct sdca_entity *entity, @@ -906,8 +933,13 @@ static int populate_control(struct device *dev, (*kctl)->private_value = (unsigned long)mc; (*kctl)->iface = SNDRV_CTL_ELEM_IFACE_MIXER; (*kctl)->info = snd_soc_info_volsw; - (*kctl)->get = snd_soc_get_volsw; - (*kctl)->put = snd_soc_put_volsw; + if (control->is_volatile) { + (*kctl)->get = volatile_get_volsw; + (*kctl)->put = volatile_put_volsw; + } else { + (*kctl)->get = snd_soc_get_volsw; + (*kctl)->put = snd_soc_put_volsw; + } if (readonly_control(control)) (*kctl)->access = SNDRV_CTL_ELEM_ACCESS_READ; @@ -1535,7 +1567,7 @@ static int set_usage(struct device *dev, struct regmap *regmap, unsigned int rate = sdca_range(range, SDCA_USAGE_SAMPLE_RATE, i); unsigned int width = sdca_range(range, SDCA_USAGE_SAMPLE_WIDTH, i); - if ((!rate || rate == target_rate) && width == target_width) { + if ((!rate || rate == target_rate) && (!width || width == target_width)) { unsigned int usage = sdca_range(range, SDCA_USAGE_NUMBER, i); unsigned int reg = SDW_SDCA_CTL(function->desc->adr, entity->id, sel, 0); diff --git a/sound/soc/sdca/sdca_functions.c b/sound/soc/sdca/sdca_functions.c index 0ccb6775f4de3..4417278e39bb1 100644 --- a/sound/soc/sdca/sdca_functions.c +++ b/sound/soc/sdca/sdca_functions.c @@ -779,6 +779,62 @@ find_sdca_control_datatype(const struct sdca_entity *entity, } } +static bool find_sdca_control_volatile(const struct sdca_entity *entity, + const struct sdca_control *control) +{ + switch (control->mode) { + case SDCA_ACCESS_MODE_DC: + return false; + case SDCA_ACCESS_MODE_RO: + case SDCA_ACCESS_MODE_RW1S: + case SDCA_ACCESS_MODE_RW1C: + return true; + default: + break; + } + + switch (SDCA_CTL_TYPE(entity->type, control->sel)) { + case SDCA_CTL_TYPE_S(XU, FDL_CURRENTOWNER): + case SDCA_CTL_TYPE_S(XU, FDL_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(XU, FDL_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(XU, FDL_STATUS): + case SDCA_CTL_TYPE_S(XU, FDL_HOST_REQUEST): + case SDCA_CTL_TYPE_S(SPE, AUTHTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SPE, AUTHTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SPE, AUTHTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SPE, AUTHRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SPE, AUTHRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SPE, AUTHRX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(MFPU, AE_CURRENTOWNER): + case SDCA_CTL_TYPE_S(MFPU, AE_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(MFPU, AE_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SMPU, HIST_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SMPU, HIST_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SMPU, HIST_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SMPU, DTODTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SMPU, DTODTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SMPU, DTODTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SMPU, DTODRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SMPU, DTODRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SMPU, DTODRX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SAPU, DTODTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SAPU, DTODTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SAPU, DTODTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SAPU, DTODRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SAPU, DTODRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SAPU, DTODRX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(HIDE, HIDTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(HIDE, HIDTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(HIDE, HIDTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(HIDE, HIDRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(HIDE, HIDRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(HIDE, HIDRX_MESSAGELENGTH): + return true; + default: + return false; + } +} + static int find_sdca_control_range(struct device *dev, struct fwnode_handle *control_node, struct sdca_control_range *range) @@ -854,10 +910,6 @@ static int find_sdca_control_value(struct device *dev, struct sdca_entity *entit return 0; } -/* - * TODO: Add support for -cn- properties, allowing different channels to have - * different defaults etc. - */ static int find_sdca_entity_control(struct device *dev, struct sdca_entity *entity, struct fwnode_handle *control_node, struct sdca_control *control) @@ -931,6 +983,8 @@ static int find_sdca_entity_control(struct device *dev, struct sdca_entity *enti break; } + control->is_volatile = find_sdca_control_volatile(entity, control); + ret = find_sdca_control_range(dev, control_node, &control->range); if (ret) { dev_err(dev, "%s: control %#x: range missing: %d\n", @@ -1263,7 +1317,7 @@ find_sdca_entity_hide(struct device *dev, struct fwnode_handle *function_node, unsigned char *report_desc = NULL; ret = fwnode_property_read_u32(entity_node, - "mipi-sdca-RxUMP-ownership-transition-maxdelay", &delay); + "mipi-sdca-RxUMP-ownership-transition-max-delay", &delay); if (!ret) hide->max_delay = delay; diff --git a/sound/soc/sdca/sdca_interrupts.c b/sound/soc/sdca/sdca_interrupts.c index 79bf3042f57d4..4189efdfe2747 100644 --- a/sound/soc/sdca/sdca_interrupts.c +++ b/sound/soc/sdca/sdca_interrupts.c @@ -104,9 +104,7 @@ static irqreturn_t function_status_handler(int irq, void *data) status = val; for_each_set_bit(mask, &status, BITS_PER_BYTE) { - mask = 1 << mask; - - switch (mask) { + switch (BIT(mask)) { case SDCA_CTL_ENTITY_0_FUNCTION_NEEDS_INITIALIZATION: //FIXME: Add init writes break; @@ -246,9 +244,9 @@ static int sdca_irq_request_locked(struct device *dev, } /** - * sdca_request_irq - request an individual SDCA interrupt + * sdca_irq_request - request an individual SDCA interrupt * @dev: Pointer to the struct device against which things should be allocated. - * @interrupt_info: Pointer to the interrupt information structure. + * @info: Pointer to the interrupt information structure. * @sdca_irq: SDCA interrupt position. * @name: Name to be given to the IRQ. * @handler: A callback thread function to be called for the IRQ. diff --git a/sound/soc/sdca/sdca_regmap.c b/sound/soc/sdca/sdca_regmap.c index 72f893e00ff50..8fa138fca00ff 100644 --- a/sound/soc/sdca/sdca_regmap.c +++ b/sound/soc/sdca/sdca_regmap.c @@ -147,14 +147,7 @@ bool sdca_regmap_volatile(struct sdca_function_data *function, unsigned int reg) if (!control) return false; - switch (control->mode) { - case SDCA_ACCESS_MODE_RO: - case SDCA_ACCESS_MODE_RW1S: - case SDCA_ACCESS_MODE_RW1C: - return true; - default: - return false; - } + return control->is_volatile; } EXPORT_SYMBOL_NS(sdca_regmap_volatile, "SND_SOC_SDCA"); diff --git a/sound/soc/sdw_utils/soc_sdw_cs42l43.c b/sound/soc/sdw_utils/soc_sdw_cs42l43.c index b415d45d520d0..3e8e2e3bdf7c5 100644 --- a/sound/soc/sdw_utils/soc_sdw_cs42l43.c +++ b/sound/soc/sdw_utils/soc_sdw_cs42l43.c @@ -44,7 +44,7 @@ static const struct snd_soc_dapm_route cs42l43_dmic_map[] = { static struct snd_soc_jack_pin soc_jack_pins[] = { { .pin = "Headphone", - .mask = SND_JACK_HEADPHONE, + .mask = SND_JACK_HEADPHONE | SND_JACK_LINEOUT, }, { .pin = "Headset Mic", diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index 9dd84d73046be..feecf3e4e38b4 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -462,8 +462,7 @@ static void soc_free_pcm_runtime(struct snd_soc_pcm_runtime *rtd) list_del(&rtd->list); - if (delayed_work_pending(&rtd->delayed_work)) - flush_delayed_work(&rtd->delayed_work); + flush_delayed_work(&rtd->delayed_work); snd_soc_pcm_component_free(rtd); /* @@ -1861,12 +1860,15 @@ static void cleanup_dmi_name(char *name) /* * Check if a DMI field is valid, i.e. not containing any string - * in the black list. + * in the black list and not the empty string. */ static int is_dmi_valid(const char *field) { int i = 0; + if (!field[0]) + return 0; + while (dmi_blacklist[i]) { if (strstr(field, dmi_blacklist[i])) return 0; @@ -2119,6 +2121,9 @@ static void soc_cleanup_card_resources(struct snd_soc_card *card) for_each_card_rtds(card, rtd) if (rtd->initialized) snd_soc_link_exit(rtd); + /* flush delayed work before removing DAIs and DAPM widgets */ + snd_soc_flush_all_delayed_work(card); + /* remove and free each DAI */ soc_remove_link_dais(card); soc_remove_link_components(card); @@ -2840,6 +2845,7 @@ int snd_soc_component_initialize(struct snd_soc_component *component, INIT_LIST_HEAD(&component->dobj_list); INIT_LIST_HEAD(&component->card_list); INIT_LIST_HEAD(&component->list); + INIT_LIST_HEAD(&component->card_aux_list); mutex_init(&component->io_mutex); if (!component->name) { diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index d2b6fb8e0b6c6..624e9269fc25b 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -110,21 +110,58 @@ int snd_soc_put_enum_double(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_put_enum_double); +static int sdca_soc_q78_reg_to_ctl(struct soc_mixer_control *mc, unsigned int reg_val, + unsigned int mask, unsigned int shift, int max, + bool sx) +{ + int val = reg_val; + + if (WARN_ON(!mc->shift)) + return -EINVAL; + + val = sign_extend32(val, mc->sign_bit); + val = (((val * 100) >> 8) / (int)mc->shift); + val -= mc->min; + + return val & mask; +} + +static unsigned int sdca_soc_q78_ctl_to_reg(struct soc_mixer_control *mc, int val, + unsigned int mask, unsigned int shift, int max) +{ + unsigned int ret_val; + int reg_val; + + if (WARN_ON(!mc->shift)) + return -EINVAL; + + reg_val = val + mc->min; + ret_val = (int)((reg_val * mc->shift) << 8) / 100; + + return ret_val & mask; +} + static int soc_mixer_reg_to_ctl(struct soc_mixer_control *mc, unsigned int reg_val, - unsigned int mask, unsigned int shift, int max) + unsigned int mask, unsigned int shift, int max, + bool sx) { int val = (reg_val >> shift) & mask; if (mc->sign_bit) val = sign_extend32(val, mc->sign_bit); - val = clamp(val, mc->min, mc->max); - val -= mc->min; + if (sx) { + val -= mc->min; // SX controls intentionally can overflow here + val = min_t(unsigned int, val & mask, max); + } else { + val = clamp(val, mc->min, mc->max); + val -= mc->min; + } if (mc->invert) val = max - val; - return val & mask; + return val; } static unsigned int soc_mixer_ctl_to_reg(struct soc_mixer_control *mc, int val, @@ -197,19 +234,27 @@ static int soc_put_volsw(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol, struct soc_mixer_control *mc, int mask, int max) { + unsigned int (*ctl_to_reg)(struct soc_mixer_control *, int, unsigned int, unsigned int, int); struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); unsigned int val1, val_mask; unsigned int val2 = 0; bool double_r = false; int ret; + if (mc->sdca_q78) { + ctl_to_reg = sdca_soc_q78_ctl_to_reg; + val_mask = mask; + } else { + ctl_to_reg = soc_mixer_ctl_to_reg; + val_mask = mask << mc->shift; + } + ret = soc_mixer_valid_ctl(mc, ucontrol->value.integer.value[0], max); if (ret) return ret; - val1 = soc_mixer_ctl_to_reg(mc, ucontrol->value.integer.value[0], + val1 = ctl_to_reg(mc, ucontrol->value.integer.value[0], mask, mc->shift, max); - val_mask = mask << mc->shift; if (snd_soc_volsw_is_stereo(mc)) { ret = soc_mixer_valid_ctl(mc, ucontrol->value.integer.value[1], max); @@ -217,14 +262,10 @@ static int soc_put_volsw(struct snd_kcontrol *kcontrol, return ret; if (mc->reg == mc->rreg) { - val1 |= soc_mixer_ctl_to_reg(mc, - ucontrol->value.integer.value[1], - mask, mc->rshift, max); + val1 |= ctl_to_reg(mc, ucontrol->value.integer.value[1], mask, mc->rshift, max); val_mask |= mask << mc->rshift; } else { - val2 = soc_mixer_ctl_to_reg(mc, - ucontrol->value.integer.value[1], - mask, mc->shift, max); + val2 = ctl_to_reg(mc, ucontrol->value.integer.value[1], mask, mc->shift, max); double_r = true; } } @@ -246,23 +287,30 @@ static int soc_put_volsw(struct snd_kcontrol *kcontrol, static int soc_get_volsw(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol, - struct soc_mixer_control *mc, int mask, int max) + struct soc_mixer_control *mc, int mask, int max, bool sx) { + int (*reg_to_ctl)(struct soc_mixer_control *, unsigned int, unsigned int, + unsigned int, int, bool); struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); unsigned int reg_val; int val; + if (mc->sdca_q78) + reg_to_ctl = sdca_soc_q78_reg_to_ctl; + else + reg_to_ctl = soc_mixer_reg_to_ctl; + reg_val = snd_soc_component_read(component, mc->reg); - val = soc_mixer_reg_to_ctl(mc, reg_val, mask, mc->shift, max); + val = reg_to_ctl(mc, reg_val, mask, mc->shift, max, sx); ucontrol->value.integer.value[0] = val; if (snd_soc_volsw_is_stereo(mc)) { if (mc->reg == mc->rreg) { - val = soc_mixer_reg_to_ctl(mc, reg_val, mask, mc->rshift, max); + val = reg_to_ctl(mc, reg_val, mask, mc->rshift, max, sx); } else { reg_val = snd_soc_component_read(component, mc->rreg); - val = soc_mixer_reg_to_ctl(mc, reg_val, mask, mc->shift, max); + val = reg_to_ctl(mc, reg_val, mask, mc->shift, max, sx); } ucontrol->value.integer.value[1] = val; @@ -331,7 +379,7 @@ int snd_soc_get_volsw(struct snd_kcontrol *kcontrol, (struct soc_mixer_control *)kcontrol->private_value; unsigned int mask = soc_mixer_mask(mc); - return soc_get_volsw(kcontrol, ucontrol, mc, mask, mc->max - mc->min); + return soc_get_volsw(kcontrol, ucontrol, mc, mask, mc->max - mc->min, false); } EXPORT_SYMBOL_GPL(snd_soc_get_volsw); @@ -373,7 +421,7 @@ int snd_soc_get_volsw_sx(struct snd_kcontrol *kcontrol, (struct soc_mixer_control *)kcontrol->private_value; unsigned int mask = soc_mixer_sx_mask(mc); - return soc_get_volsw(kcontrol, ucontrol, mc, mask, mc->max); + return soc_get_volsw(kcontrol, ucontrol, mc, mask, mc->max, true); } EXPORT_SYMBOL_GPL(snd_soc_get_volsw_sx); diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c index 2c21fd528afd0..4ed829b49bc2d 100644 --- a/sound/soc/soc-pcm.c +++ b/sound/soc/soc-pcm.c @@ -949,7 +949,7 @@ static int __soc_pcm_prepare(struct snd_soc_pcm_runtime *rtd, SND_SOC_DAPM_STREAM_START); for_each_rtd_dais(rtd, i, dai) { - if (!snd_soc_dai_mute_is_ctrled_at_trigger(dai)) + if (!snd_soc_dai_mute_is_ctrled_at_trigger(dai) || !rtd->dai_link->dynamic) snd_soc_dai_digital_mute(dai, 0, substream->stream); } @@ -1007,7 +1007,7 @@ static int soc_pcm_hw_clean(struct snd_soc_pcm_runtime *rtd, soc_pcm_set_dai_params(dai, NULL); if (snd_soc_dai_stream_active(dai, substream->stream) == 1) { - if (!snd_soc_dai_mute_is_ctrled_at_trigger(dai)) + if (!snd_soc_dai_mute_is_ctrled_at_trigger(dai) || !rtd->dai_link->dynamic) snd_soc_dai_digital_mute(dai, 1, substream->stream); } } diff --git a/sound/soc/sof/intel/hda-dai.c b/sound/soc/sof/intel/hda-dai.c index 883d0d3bae9ec..3c742d5351333 100644 --- a/sound/soc/sof/intel/hda-dai.c +++ b/sound/soc/sof/intel/hda-dai.c @@ -70,12 +70,22 @@ static const struct hda_dai_widget_dma_ops * hda_dai_get_ops(struct snd_pcm_substream *substream, struct snd_soc_dai *cpu_dai) { struct snd_soc_dapm_widget *w = snd_soc_dai_get_widget(cpu_dai, substream->stream); - struct snd_sof_widget *swidget = w->dobj.private; + struct snd_sof_widget *swidget; struct snd_sof_dev *sdev; struct snd_sof_dai *sdai; - sdev = widget_to_sdev(w); + /* + * this is unlikely if the topology and the machine driver DAI links match. + * But if there's a missing DAI link in topology, this will prevent a NULL pointer + * dereference later on. + */ + if (!w) { + dev_err(cpu_dai->dev, "%s: widget is NULL\n", __func__); + return NULL; + } + sdev = widget_to_sdev(w); + swidget = w->dobj.private; if (!swidget) { dev_err(sdev->dev, "%s: swidget is NULL\n", __func__); return NULL; diff --git a/sound/soc/sof/ipc4-control.c b/sound/soc/sof/ipc4-control.c index 976a4794d6100..453ed1643b89c 100644 --- a/sound/soc/sof/ipc4-control.c +++ b/sound/soc/sof/ipc4-control.c @@ -66,7 +66,7 @@ static int sof_ipc4_set_get_kcontrol_data(struct snd_sof_control *scontrol, * configuration */ memcpy(scontrol->ipc_control_data, scontrol->old_ipc_control_data, - scontrol->max_size); + scontrol->size); kfree(scontrol->old_ipc_control_data); scontrol->old_ipc_control_data = NULL; /* Send the last known good configuration to firmware */ @@ -412,19 +412,35 @@ static int sof_ipc4_set_get_bytes_data(struct snd_sof_dev *sdev, int ret = 0; /* Send the new data to the firmware only if it is powered up */ - if (set && !pm_runtime_active(sdev->dev)) - return 0; + if (set) { + if (!pm_runtime_active(sdev->dev)) + return 0; + + if (!data->size) { + dev_dbg(sdev->dev, "%s: No data to be sent.\n", + scontrol->name); + return 0; + } + } msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(data->type); msg->data_ptr = data->data; - msg->data_size = data->size; + if (set) + msg->data_size = data->size; + else + msg->data_size = scontrol->max_size - sizeof(*data); ret = sof_ipc4_set_get_kcontrol_data(scontrol, set, lock); - if (ret < 0) + if (ret < 0) { dev_err(sdev->dev, "Failed to %s for %s\n", set ? "set bytes update" : "get bytes", scontrol->name); + } else if (!set) { + /* Update the sizes according to the received payload data */ + data->size = msg->data_size; + scontrol->size = sizeof(*cdata) + sizeof(*data) + data->size; + } msg->data_ptr = NULL; msg->data_size = 0; @@ -440,6 +456,7 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); struct sof_abi_hdr *data = cdata->data; size_t size; + int ret; if (scontrol->max_size > sizeof(ucontrol->value.bytes.data)) { dev_err_ratelimited(scomp->dev, @@ -461,9 +478,12 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, /* copy from kcontrol */ memcpy(data, ucontrol->value.bytes.data, size); - sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); + ret = sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); + if (!ret) + /* Update the cdata size */ + scontrol->size = sizeof(*cdata) + size; - return 0; + return ret; } static int sof_ipc4_bytes_get(struct snd_sof_control *scontrol, @@ -559,7 +579,7 @@ static int sof_ipc4_bytes_ext_put(struct snd_sof_control *scontrol, if (!scontrol->old_ipc_control_data) { /* Create a backup of the current, valid bytes control */ scontrol->old_ipc_control_data = kmemdup(scontrol->ipc_control_data, - scontrol->max_size, GFP_KERNEL); + scontrol->size, GFP_KERNEL); if (!scontrol->old_ipc_control_data) return -ENOMEM; } @@ -567,12 +587,15 @@ static int sof_ipc4_bytes_ext_put(struct snd_sof_control *scontrol, /* Copy the whole binary data which includes the ABI header and the payload */ if (copy_from_user(data, tlvd->tlv, header.length)) { memcpy(scontrol->ipc_control_data, scontrol->old_ipc_control_data, - scontrol->max_size); + scontrol->size); kfree(scontrol->old_ipc_control_data); scontrol->old_ipc_control_data = NULL; return -EFAULT; } + /* Update the cdata size */ + scontrol->size = sizeof(*cdata) + header.length; + return sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); } diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index 221e9d4052b8f..d74567d6afc41 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -1752,11 +1752,9 @@ snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai channel_count = params_channels(params); sample_rate = params_rate(params); bit_depth = params_width(params); - /* - * Look for 32-bit blob first instead of 16-bit if copier - * supports multiple formats - */ - if (bit_depth == 16 && !single_bitdepth) { + + /* Prefer 32-bit blob if copier supports multiple formats */ + if (bit_depth <= 16 && !single_bitdepth) { dev_dbg(sdev->dev, "Looking for 32-bit blob first for DMIC\n"); format_change = true; bit_depth = 32; @@ -1799,10 +1797,18 @@ snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai if (format_change) { /* * The 32-bit blob was not found in NHLT table, try to - * look for one based on the params + * look for 16-bit for DMIC or based on the params for + * SSP */ - bit_depth = params_width(params); - format_change = false; + if (linktype == SOF_DAI_INTEL_DMIC) { + bit_depth = 16; + if (params_width(params) == 16) + format_change = false; + } else { + bit_depth = params_width(params); + format_change = false; + } + get_new_blob = true; } else if (linktype == SOF_DAI_INTEL_DMIC && !single_bitdepth) { /* @@ -1837,7 +1843,7 @@ snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai *len = cfg->size >> 2; *dst = (u32 *)cfg->caps; - if (format_change) { + if (format_change || params_format(params) == SNDRV_PCM_FORMAT_FLOAT_LE) { /* * Update the params to reflect that different blob was loaded * instead of the requested bit depth (16 -> 32 or 32 -> 16). @@ -2855,22 +2861,41 @@ static int sof_ipc4_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_ struct sof_ipc4_msg *msg; int ret; - if (scontrol->max_size < (sizeof(*control_data) + sizeof(struct sof_abi_hdr))) { - dev_err(sdev->dev, "insufficient size for a bytes control %s: %zu.\n", + /* + * The max_size is coming from topology and indicates the maximum size + * of sof_abi_hdr plus the payload, which excludes the local only + * 'struct sof_ipc4_control_data' + */ + if (scontrol->max_size < sizeof(struct sof_abi_hdr)) { + dev_err(sdev->dev, + "insufficient maximum size for a bytes control %s: %zu.\n", scontrol->name, scontrol->max_size); return -EINVAL; } - if (scontrol->priv_size > scontrol->max_size - sizeof(*control_data)) { - dev_err(sdev->dev, "scontrol %s bytes data size %zu exceeds max %zu.\n", - scontrol->name, scontrol->priv_size, - scontrol->max_size - sizeof(*control_data)); + if (scontrol->priv_size > scontrol->max_size) { + dev_err(sdev->dev, + "bytes control %s initial data size %zu exceeds max %zu.\n", + scontrol->name, scontrol->priv_size, scontrol->max_size); + return -EINVAL; + } + + if (scontrol->priv_size && scontrol->priv_size < sizeof(struct sof_abi_hdr)) { + dev_err(sdev->dev, + "bytes control %s initial data size %zu is insufficient.\n", + scontrol->name, scontrol->priv_size); return -EINVAL; } - scontrol->size = sizeof(struct sof_ipc4_control_data) + scontrol->priv_size; + /* + * The used size behind the cdata pointer, which can be smaller than + * the maximum size + */ + scontrol->size = sizeof(*control_data) + scontrol->priv_size; - scontrol->ipc_control_data = kzalloc(scontrol->max_size, GFP_KERNEL); + /* Allocate the cdata: local struct size + maximum payload size */ + scontrol->ipc_control_data = kzalloc(sizeof(*control_data) + scontrol->max_size, + GFP_KERNEL); if (!scontrol->ipc_control_data) return -ENOMEM; diff --git a/sound/soc/sof/ipc4.c b/sound/soc/sof/ipc4.c index a4a090e6724a6..20d723f48fff0 100644 --- a/sound/soc/sof/ipc4.c +++ b/sound/soc/sof/ipc4.c @@ -15,6 +15,7 @@ #include "sof-audio.h" #include "ipc4-fw-reg.h" #include "ipc4-priv.h" +#include "ipc4-topology.h" #include "ipc4-telemetry.h" #include "ops.h" @@ -433,6 +434,23 @@ static int sof_ipc4_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_ return ret; } +static bool sof_ipc4_tx_payload_for_get_data(struct sof_ipc4_msg *tx) +{ + /* + * Messages that require TX payload with LARGE_CONFIG_GET. + * The TX payload is placed into the IPC message data section by caller, + * which needs to be copied to temporary buffer since the received data + * will overwrite it. + */ + switch (tx->extension & SOF_IPC4_MOD_EXT_MSG_PARAM_ID_MASK) { + case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_SWITCH_CONTROL_PARAM_ID): + case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_ENUM_CONTROL_PARAM_ID): + return true; + default: + return false; + } +} + static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, size_t payload_bytes, bool set) { @@ -444,6 +462,8 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, struct sof_ipc4_msg tx = {{ 0 }}; struct sof_ipc4_msg rx = {{ 0 }}; size_t remaining = payload_bytes; + void *tx_payload_for_get = NULL; + size_t tx_data_size = 0; size_t offset = 0; size_t chunk_size; int ret; @@ -469,10 +489,20 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, tx.extension |= SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK(1); + if (sof_ipc4_tx_payload_for_get_data(&tx)) { + tx_data_size = min(ipc4_msg->data_size, payload_limit); + tx_payload_for_get = kmemdup(ipc4_msg->data_ptr, tx_data_size, + GFP_KERNEL); + if (!tx_payload_for_get) + return -ENOMEM; + } + /* ensure the DSP is in D0i0 before sending IPC */ ret = snd_sof_dsp_set_power_state(sdev, &target_state); - if (ret < 0) + if (ret < 0) { + kfree(tx_payload_for_get); return ret; + } /* Serialise IPC TX */ mutex_lock(&sdev->ipc->tx_mutex); @@ -506,7 +536,15 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, rx.data_size = chunk_size; rx.data_ptr = ipc4_msg->data_ptr + offset; - tx_size = 0; + if (tx_payload_for_get) { + tx_size = tx_data_size; + tx.data_size = tx_size; + tx.data_ptr = tx_payload_for_get; + } else { + tx_size = 0; + tx.data_size = 0; + tx.data_ptr = NULL; + } rx_size = chunk_size; } @@ -553,6 +591,8 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, mutex_unlock(&sdev->ipc->tx_mutex); + kfree(tx_payload_for_get); + return ret; } diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index b6d5c8024f8cf..4c8dba285408a 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -736,7 +736,7 @@ static int sof_parse_token_sets(struct snd_soc_component *scomp, asize = le32_to_cpu(array->size); /* validate asize */ - if (asize < 0) { /* FIXME: A zero-size array makes no sense */ + if (asize < sizeof(*array)) { dev_err(scomp->dev, "error: invalid array size 0x%x\n", asize); return -EINVAL; diff --git a/sound/soc/stm/stm32_sai.c b/sound/soc/stm/stm32_sai.c index fa821e3fb427e..00cf24ceca2d4 100644 --- a/sound/soc/stm/stm32_sai.c +++ b/sound/soc/stm/stm32_sai.c @@ -138,30 +138,24 @@ static int stm32_sai_set_sync(struct stm32_sai_data *sai_client, if (!pdev) { dev_err(&sai_client->pdev->dev, "Device not found for node %pOFn\n", np_provider); - of_node_put(np_provider); return -ENODEV; } sai_provider = platform_get_drvdata(pdev); + put_device(&pdev->dev); if (!sai_provider) { dev_err(&sai_client->pdev->dev, "SAI sync provider data not found\n"); - ret = -EINVAL; - goto error; + return -EINVAL; } /* Configure sync client */ ret = stm32_sai_sync_conf_client(sai_client, synci); if (ret < 0) - goto error; + return ret; /* Configure sync provider */ - ret = stm32_sai_sync_conf_provider(sai_provider, synco); - -error: - put_device(&pdev->dev); - of_node_put(np_provider); - return ret; + return stm32_sai_sync_conf_provider(sai_provider, synco); } static int stm32_sai_get_parent_clk(struct stm32_sai_data *sai) diff --git a/sound/soc/stm/stm32_sai_sub.c b/sound/soc/stm/stm32_sai_sub.c index 0ae1eae2a59e2..c2540383ab86f 100644 --- a/sound/soc/stm/stm32_sai_sub.c +++ b/sound/soc/stm/stm32_sai_sub.c @@ -802,6 +802,7 @@ static int stm32_sai_set_dai_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt) break; /* Left justified */ case SND_SOC_DAIFMT_MSB: + cr1 |= SAI_XCR1_CKSTR; frcr |= SAI_XFRCR_FSPOL | SAI_XFRCR_FSDEF; break; /* Right justified */ @@ -809,9 +810,11 @@ static int stm32_sai_set_dai_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt) frcr |= SAI_XFRCR_FSPOL | SAI_XFRCR_FSDEF; break; case SND_SOC_DAIFMT_DSP_A: + cr1 |= SAI_XCR1_CKSTR; frcr |= SAI_XFRCR_FSPOL | SAI_XFRCR_FSOFF; break; case SND_SOC_DAIFMT_DSP_B: + cr1 |= SAI_XCR1_CKSTR; frcr |= SAI_XFRCR_FSPOL; break; default: @@ -1586,7 +1589,8 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev, dev_err(&pdev->dev, "External synchro not supported\n"); of_node_put(args.np); - return -EINVAL; + ret = -EINVAL; + goto err_put_sync_provider; } sai->sync = SAI_SYNC_EXTERNAL; @@ -1595,7 +1599,8 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev, (sai->synci > (SAI_GCR_SYNCIN_MAX + 1))) { dev_err(&pdev->dev, "Wrong SAI index\n"); of_node_put(args.np); - return -EINVAL; + ret = -EINVAL; + goto err_put_sync_provider; } if (of_property_match_string(args.np, "compatible", @@ -1609,7 +1614,8 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev, if (!sai->synco) { dev_err(&pdev->dev, "Unknown SAI sub-block\n"); of_node_put(args.np); - return -EINVAL; + ret = -EINVAL; + goto err_put_sync_provider; } } @@ -1619,13 +1625,15 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev, of_node_put(args.np); sai->sai_ck = devm_clk_get(&pdev->dev, "sai_ck"); - if (IS_ERR(sai->sai_ck)) - return dev_err_probe(&pdev->dev, PTR_ERR(sai->sai_ck), - "Missing kernel clock sai_ck\n"); + if (IS_ERR(sai->sai_ck)) { + ret = dev_err_probe(&pdev->dev, PTR_ERR(sai->sai_ck), + "Missing kernel clock sai_ck\n"); + goto err_put_sync_provider; + } ret = clk_prepare(sai->pdata->pclk); if (ret < 0) - return ret; + goto err_put_sync_provider; if (STM_SAI_IS_F4(sai->pdata)) return 0; @@ -1634,14 +1642,23 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev, if (of_property_present(np, "#clock-cells")) { ret = stm32_sai_add_mclk_provider(sai); if (ret < 0) - return ret; + goto err_unprepare_pclk; } else { sai->sai_mclk = devm_clk_get_optional(&pdev->dev, "MCLK"); - if (IS_ERR(sai->sai_mclk)) - return PTR_ERR(sai->sai_mclk); + if (IS_ERR(sai->sai_mclk)) { + ret = PTR_ERR(sai->sai_mclk); + goto err_unprepare_pclk; + } } return 0; + +err_unprepare_pclk: + clk_unprepare(sai->pdata->pclk); +err_put_sync_provider: + of_node_put(sai->np_sync_provider); + + return ret; } static int stm32_sai_sub_probe(struct platform_device *pdev) @@ -1688,26 +1705,34 @@ static int stm32_sai_sub_probe(struct platform_device *pdev) IRQF_SHARED, dev_name(&pdev->dev), sai); if (ret) { dev_err(&pdev->dev, "IRQ request returned %d\n", ret); - return ret; + goto err_unprepare_pclk; } if (STM_SAI_PROTOCOL_IS_SPDIF(sai)) conf = &stm32_sai_pcm_config_spdif; ret = snd_dmaengine_pcm_register(&pdev->dev, conf, 0); - if (ret) - return dev_err_probe(&pdev->dev, ret, "Could not register pcm dma\n"); + if (ret) { + ret = dev_err_probe(&pdev->dev, ret, "Could not register pcm dma\n"); + goto err_unprepare_pclk; + } ret = snd_soc_register_component(&pdev->dev, &stm32_component, &sai->cpu_dai_drv, 1); if (ret) { snd_dmaengine_pcm_unregister(&pdev->dev); - return ret; + goto err_unprepare_pclk; } pm_runtime_enable(&pdev->dev); return 0; + +err_unprepare_pclk: + clk_unprepare(sai->pdata->pclk); + of_node_put(sai->np_sync_provider); + + return ret; } static void stm32_sai_sub_remove(struct platform_device *pdev) @@ -1718,6 +1743,7 @@ static void stm32_sai_sub_remove(struct platform_device *pdev) snd_dmaengine_pcm_unregister(&pdev->dev); snd_soc_unregister_component(&pdev->dev); pm_runtime_disable(&pdev->dev); + of_node_put(sai->np_sync_provider); } static int stm32_sai_sub_suspend(struct device *dev) diff --git a/sound/soc/sunxi/sun50i-dmic.c b/sound/soc/sunxi/sun50i-dmic.c index bab1e29c99887..eddfebe166169 100644 --- a/sound/soc/sunxi/sun50i-dmic.c +++ b/sound/soc/sunxi/sun50i-dmic.c @@ -358,6 +358,9 @@ static int sun50i_dmic_probe(struct platform_device *pdev) host->regmap = devm_regmap_init_mmio(&pdev->dev, base, &sun50i_dmic_regmap_config); + if (IS_ERR(host->regmap)) + return dev_err_probe(&pdev->dev, PTR_ERR(host->regmap), + "failed to initialise regmap\n"); /* Clocks */ host->bus_clk = devm_clk_get(&pdev->dev, "bus"); diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c index 21aeaeba0b107..01d60a74ad1c3 100644 --- a/sound/soc/tegra/tegra210_ahub.c +++ b/sound/soc/tegra/tegra210_ahub.c @@ -2049,6 +2049,61 @@ static const struct snd_soc_component_driver tegra264_ahub_component = { .num_dapm_routes = ARRAY_SIZE(tegra264_ahub_routes), }; +static bool tegra210_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA210_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0: + case TEGRA210_AXBAR_PART_0_I2S1_RX1_0 ... TEGRA210_AXBAR_PART_0_I2S5_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 ... TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0 ... TEGRA210_AXBAR_PART_0_OPE2_RX1_0: + case TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0: + case TEGRA210_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA210_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA210_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA210_AXBAR_PART_0_ADX2_RX1_0: + return true; + default: + break; + } + } + + return false; +} + +static bool tegra186_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA186_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA186_AXBAR_PART_0_I2S6_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 ... TEGRA186_AXBAR_PART_0_DSPK2_RX1_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0: + case TEGRA186_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA186_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA186_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA186_AXBAR_PART_0_AMX3_RX4_0: + case TEGRA210_AXBAR_PART_0_ADX1_RX1_0 ... TEGRA186_AXBAR_PART_0_ASRC1_RX7_0: + return true; + default: + break; + } + } + + return false; +} + static bool tegra264_ahub_wr_reg(struct device *dev, unsigned int reg) { int part; @@ -2076,6 +2131,7 @@ static const struct regmap_config tegra210_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra210_ahub_wr_reg, .max_register = TEGRA210_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; @@ -2084,6 +2140,7 @@ static const struct regmap_config tegra186_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra186_ahub_wr_reg, .max_register = TEGRA186_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; diff --git a/sound/soc/tegra/tegra210_ahub.h b/sound/soc/tegra/tegra210_ahub.h index f355b2cfd19b2..acbe640dd3b57 100644 --- a/sound/soc/tegra/tegra210_ahub.h +++ b/sound/soc/tegra/tegra210_ahub.h @@ -68,6 +68,36 @@ #define TEGRA210_MAX_REGISTER_ADDR (TEGRA210_XBAR_PART2_RX + \ (TEGRA210_XBAR_RX_STRIDE * (TEGRA210_XBAR_AUDIO_RX_COUNT - 1))) +/* AXBAR register offsets */ +#define TEGRA186_AXBAR_PART_0_AMX1_RX1_0 0x120 +#define TEGRA186_AXBAR_PART_0_AMX3_RX4_0 0x14c +#define TEGRA186_AXBAR_PART_0_ASRC1_RX7_0 0x1a8 +#define TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 0xc0 +#define TEGRA186_AXBAR_PART_0_DSPK2_RX1_0 0xc4 +#define TEGRA186_AXBAR_PART_0_I2S6_RX1_0 0x54 +#define TEGRA186_AXBAR_PART_0_MVC1_RX1_0 0x110 +#define TEGRA186_AXBAR_PART_0_MVC2_RX1_0 0x114 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0 0x24 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 0x0 +#define TEGRA210_AXBAR_PART_0_ADX1_RX1_0 0x160 +#define TEGRA210_AXBAR_PART_0_ADX2_RX1_0 0x164 +#define TEGRA210_AXBAR_PART_0_AFC1_RX1_0 0xd0 +#define TEGRA210_AXBAR_PART_0_AFC6_RX1_0 0xe4 +#define TEGRA210_AXBAR_PART_0_AMX1_RX1_0 0x140 +#define TEGRA210_AXBAR_PART_0_I2S1_RX1_0 0x40 +#define TEGRA210_AXBAR_PART_0_I2S5_RX1_0 0x50 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX10_0 0xa4 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 0x80 +#define TEGRA210_AXBAR_PART_0_MVC1_RX1_0 0x120 +#define TEGRA210_AXBAR_PART_0_MVC2_RX1_0 0x124 +#define TEGRA210_AXBAR_PART_0_OPE1_RX1_0 0x100 +#define TEGRA210_AXBAR_PART_0_OPE2_RX1_0 0x104 +#define TEGRA210_AXBAR_PART_0_SFC1_RX1_0 0x60 +#define TEGRA210_AXBAR_PART_0_SFC4_RX1_0 0x6c +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 0xc0 +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0 0xc4 +#define TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0 0x110 + #define MUX_REG(id) (TEGRA210_XBAR_RX_STRIDE * (id)) #define MUX_VALUE(npart, nbit) (1 + (nbit) + (npart) * 32) diff --git a/sound/soc/ti/davinci-evm.c b/sound/soc/ti/davinci-evm.c index 2a2f5bc95576e..a55a369ce71c2 100644 --- a/sound/soc/ti/davinci-evm.c +++ b/sound/soc/ti/davinci-evm.c @@ -193,27 +193,32 @@ static int davinci_evm_probe(struct platform_device *pdev) return -EINVAL; dai->cpus->of_node = of_parse_phandle(np, "ti,mcasp-controller", 0); - if (!dai->cpus->of_node) - return -EINVAL; + if (!dai->cpus->of_node) { + ret = -EINVAL; + goto err_put; + } dai->platforms->of_node = dai->cpus->of_node; evm_soc_card.dev = &pdev->dev; ret = snd_soc_of_parse_card_name(&evm_soc_card, "ti,model"); if (ret) - return ret; + goto err_put; mclk = devm_clk_get(&pdev->dev, "mclk"); if (PTR_ERR(mclk) == -EPROBE_DEFER) { - return -EPROBE_DEFER; + ret = -EPROBE_DEFER; + goto err_put; } else if (IS_ERR(mclk)) { dev_dbg(&pdev->dev, "mclk not found.\n"); mclk = NULL; } drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL); - if (!drvdata) - return -ENOMEM; + if (!drvdata) { + ret = -ENOMEM; + goto err_put; + } drvdata->mclk = mclk; @@ -223,7 +228,8 @@ static int davinci_evm_probe(struct platform_device *pdev) if (!drvdata->mclk) { dev_err(&pdev->dev, "No clock or clock rate defined.\n"); - return -EINVAL; + ret = -EINVAL; + goto err_put; } drvdata->sysclk = clk_get_rate(drvdata->mclk); } else if (drvdata->mclk) { @@ -239,8 +245,25 @@ static int davinci_evm_probe(struct platform_device *pdev) snd_soc_card_set_drvdata(&evm_soc_card, drvdata); ret = devm_snd_soc_register_card(&pdev->dev, &evm_soc_card); - if (ret) + if (ret) { dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", ret); + goto err_put; + } + + return ret; + +err_put: + dai->platforms->of_node = NULL; + + if (dai->cpus->of_node) { + of_node_put(dai->cpus->of_node); + dai->cpus->of_node = NULL; + } + + if (dai->codecs->of_node) { + of_node_put(dai->codecs->of_node); + dai->codecs->of_node = NULL; + } return ret; } diff --git a/sound/usb/6fire/chip.c b/sound/usb/6fire/chip.c index 5ff78814e6875..874f6cd503ca5 100644 --- a/sound/usb/6fire/chip.c +++ b/sound/usb/6fire/chip.c @@ -53,11 +53,6 @@ static void usb6fire_chip_abort(struct sfire_chip *chip) usb6fire_comm_abort(chip); if (chip->control) usb6fire_control_abort(chip); - if (chip->card) { - snd_card_disconnect(chip->card); - snd_card_free_when_closed(chip->card); - chip->card = NULL; - } } } @@ -168,6 +163,7 @@ static int usb6fire_chip_probe(struct usb_interface *intf, static void usb6fire_chip_disconnect(struct usb_interface *intf) { struct sfire_chip *chip; + struct snd_card *card; chip = usb_get_intfdata(intf); if (chip) { /* if !chip, fw upload has been performed */ @@ -178,8 +174,19 @@ static void usb6fire_chip_disconnect(struct usb_interface *intf) chips[chip->regidx] = NULL; } + /* + * Save card pointer before teardown. + * snd_card_free_when_closed() may free card (and + * the embedded chip) immediately, so it must be + * called last and chip must not be accessed after. + */ + card = chip->card; chip->shutdown = true; + if (card) + snd_card_disconnect(card); usb6fire_chip_abort(chip); + if (card) + snd_card_free_when_closed(card); } } } diff --git a/sound/usb/Kconfig b/sound/usb/Kconfig index 9b890abd96d34..b4588915efa11 100644 --- a/sound/usb/Kconfig +++ b/sound/usb/Kconfig @@ -192,6 +192,7 @@ config SND_USB_AUDIO_QMI tristate "Qualcomm Audio Offload driver" depends on QCOM_QMI_HELPERS && SND_USB_AUDIO && SND_SOC_USB depends on USB_XHCI_HCD && USB_XHCI_SIDEBAND + select AUXILIARY_BUS help Say Y here to enable the Qualcomm USB audio offloading feature. diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c index dfd820483849e..51177ebfb8c62 100644 --- a/sound/usb/caiaq/device.c +++ b/sound/usb/caiaq/device.c @@ -384,7 +384,7 @@ static void card_free(struct snd_card *card) snd_usb_caiaq_input_free(cdev); #endif snd_usb_caiaq_audio_free(cdev); - usb_reset_device(cdev->chip.dev); + usb_put_dev(cdev->chip.dev); } static int create_card(struct usb_device *usb_dev, @@ -410,7 +410,7 @@ static int create_card(struct usb_device *usb_dev, return err; cdev = caiaqdev(card); - cdev->chip.dev = usb_dev; + cdev->chip.dev = usb_get_dev(usb_dev); cdev->chip.card = card; cdev->chip.usb_id = USB_ID(le16_to_cpu(usb_dev->descriptor.idVendor), le16_to_cpu(usb_dev->descriptor.idProduct)); @@ -488,7 +488,7 @@ static int init_card(struct snd_usb_caiaqdev *cdev) memset(id, 0, sizeof(id)); for (c = card->shortname, len = 0; - *c && len < sizeof(card->id); c++) + *c && len < sizeof(card->id) - 1; c++) if (*c != ' ') id[len++] = *c; diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index cc15624ecaffe..77c4330d52953 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -160,8 +160,8 @@ int snd_usb_endpoint_implicit_feedback_sink(struct snd_usb_endpoint *ep) * This won't be used for implicit feedback which takes the packet size * returned from the sync source */ -static int slave_next_packet_size(struct snd_usb_endpoint *ep, - unsigned int avail) +static int synced_next_packet_size(struct snd_usb_endpoint *ep, + unsigned int avail) { unsigned int phase; int ret; @@ -221,13 +221,14 @@ int snd_usb_endpoint_next_packet_size(struct snd_usb_endpoint *ep, packet = ctx->packet_size[idx]; if (packet) { + packet = min(packet, ep->maxframesize); if (avail && packet >= avail) return -EAGAIN; return packet; } if (ep->sync_source) - return slave_next_packet_size(ep, avail); + return synced_next_packet_size(ep, avail); else return next_packet_size(ep, avail); } @@ -275,8 +276,8 @@ static inline bool has_tx_length_quirk(struct snd_usb_audio *chip) return chip->quirk_flags & QUIRK_FLAG_TX_LENGTH; } -static void prepare_silent_urb(struct snd_usb_endpoint *ep, - struct snd_urb_ctx *ctx) +static int prepare_silent_urb(struct snd_usb_endpoint *ep, + struct snd_urb_ctx *ctx) { struct urb *urb = ctx->urb; unsigned int offs = 0; @@ -289,28 +290,34 @@ static void prepare_silent_urb(struct snd_usb_endpoint *ep, extra = sizeof(packet_length); for (i = 0; i < ctx->packets; ++i) { - unsigned int offset; - unsigned int length; - int counts; - - counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0); - length = counts * ep->stride; /* number of silent bytes */ - offset = offs * ep->stride + extra * i; - urb->iso_frame_desc[i].offset = offset; + int length; + + length = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0); + if (length < 0) + return length; + length *= ep->stride; /* number of silent bytes */ + if (offs + length + extra > ctx->buffer_size) + break; + urb->iso_frame_desc[i].offset = offs; urb->iso_frame_desc[i].length = length + extra; if (extra) { packet_length = cpu_to_le32(length); - memcpy(urb->transfer_buffer + offset, + memcpy(urb->transfer_buffer + offs, &packet_length, sizeof(packet_length)); + offs += extra; } - memset(urb->transfer_buffer + offset + extra, + memset(urb->transfer_buffer + offs, ep->silence_value, length); - offs += counts; + offs += length; } - urb->number_of_packets = ctx->packets; - urb->transfer_buffer_length = offs * ep->stride + ctx->packets * extra; + if (!offs) + return -EPIPE; + + urb->number_of_packets = i; + urb->transfer_buffer_length = offs; ctx->queued = 0; + return 0; } /* @@ -332,8 +339,7 @@ static int prepare_outbound_urb(struct snd_usb_endpoint *ep, if (data_subs && ep->prepare_data_urb) return ep->prepare_data_urb(data_subs, urb, in_stream_lock); /* no data provider, so send silence */ - prepare_silent_urb(ep, ctx); - break; + return prepare_silent_urb(ep, ctx); case SND_USB_ENDPOINT_TYPE_SYNC: if (snd_usb_get_speed(ep->chip->dev) >= USB_SPEED_HIGH) { @@ -481,6 +487,7 @@ int snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep, /* copy over the length information */ if (implicit_fb) { + ctx->packets = packet->packets; for (i = 0; i < packet->packets; i++) ctx->packet_size[i] = packet->packet_size[i]; } @@ -1368,6 +1375,9 @@ int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, return -EINVAL; } + ep->packsize[0] = min(ep->packsize[0], ep->maxframesize); + ep->packsize[1] = min(ep->packsize[1], ep->maxframesize); + /* calculate the frequency in 16.16 format */ ep->freqm = ep->freqn; ep->freqshift = INT_MIN; diff --git a/sound/usb/format.c b/sound/usb/format.c index ec95a063beb10..53b5dc5453b78 100644 --- a/sound/usb/format.c +++ b/sound/usb/format.c @@ -302,17 +302,48 @@ static bool s1810c_valid_sample_rate(struct audioformat *fp, } /* - * Many Focusrite devices supports a limited set of sampling rates per - * altsetting. Maximum rate is exposed in the last 4 bytes of Format Type - * descriptor which has a non-standard bLength = 10. + * Focusrite devices use rate pairs: 44100/48000, 88200/96000, and + * 176400/192000. Return true if rate is in the pair for max_rate. + */ +static bool focusrite_rate_pair(unsigned int rate, + unsigned int max_rate) +{ + switch (max_rate) { + case 48000: return rate == 44100 || rate == 48000; + case 96000: return rate == 88200 || rate == 96000; + case 192000: return rate == 176400 || rate == 192000; + default: return true; + } +} + +/* + * Focusrite devices report all supported rates in a single clock + * source but only a subset is valid per altsetting. + * + * Detection uses two descriptor features: + * + * 1. Format Type descriptor bLength == 10: non-standard extension + * with max sample rate in bytes 6..9. + * + * 2. bmControls VAL_ALT_SETTINGS readable bit: when set, the device + * only supports the highest rate pair for that altsetting, and when + * clear, all rates up to max_rate are valid. + * + * For devices without the bLength == 10 extension but with + * VAL_ALT_SETTINGS readable and multiple altsettings (only seen in + * Scarlett 18i8 3rd Gen playback), fall back to the Focusrite + * convention: alt 1 = 48kHz, alt 2 = 96kHz, alt 3 = 192kHz. */ static bool focusrite_valid_sample_rate(struct snd_usb_audio *chip, struct audioformat *fp, unsigned int rate) { + struct usb_interface *iface; struct usb_host_interface *alts; + struct uac2_as_header_descriptor *as; unsigned char *fmt; unsigned int max_rate; + bool val_alt; alts = snd_usb_get_host_interface(chip, fp->iface, fp->altsetting); if (!alts) @@ -323,9 +354,21 @@ static bool focusrite_valid_sample_rate(struct snd_usb_audio *chip, if (!fmt) return true; + as = snd_usb_find_csint_desc(alts->extra, alts->extralen, + NULL, UAC_AS_GENERAL); + if (!as) + return true; + + val_alt = uac_v2v3_control_is_readable(as->bmControls, + UAC2_AS_VAL_ALT_SETTINGS); + if (fmt[0] == 10) { /* bLength */ max_rate = combine_quad(&fmt[6]); + if (val_alt) + return focusrite_rate_pair(rate, max_rate); + + /* No val_alt: rates fall through from higher */ switch (max_rate) { case 192000: if (rate == 176400 || rate == 192000) @@ -341,12 +384,29 @@ static bool focusrite_valid_sample_rate(struct snd_usb_audio *chip, usb_audio_info(chip, "%u:%d : unexpected max rate: %u\n", fp->iface, fp->altsetting, max_rate); - return true; } } - return true; + if (!val_alt) + return true; + + /* Multi-altsetting device with val_alt but no max_rate + * in the format descriptor. Use Focusrite convention: + * alt 1 = 48kHz, alt 2 = 96kHz, alt 3 = 192kHz. + */ + iface = usb_ifnum_to_if(chip->dev, fp->iface); + if (!iface || iface->num_altsetting <= 2) + return true; + + switch (fp->altsetting) { + case 1: max_rate = 48000; break; + case 2: max_rate = 96000; break; + case 3: max_rate = 192000; break; + default: return true; + } + + return focusrite_rate_pair(rate, max_rate); } /* diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index 3af71d42b9b9a..16eaa9fa317df 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -1204,6 +1204,13 @@ static void volume_control_quirks(struct usb_mixer_elem_info *cval, cval->min = -11264; /* Mute under it */ } break; + case USB_ID(0x31b2, 0x0111): /* MOONDROP JU Jiu */ + if (!strcmp(kctl->id.name, "PCM Playback Volume")) { + usb_audio_info(chip, + "set volume quirk for MOONDROP JU Jiu\n"); + cval->min = -10880; /* Mute under it */ + } + break; } } @@ -1813,11 +1820,10 @@ static void __build_feature_ctl(struct usb_mixer_interface *mixer, range = (cval->max - cval->min) / cval->res; /* - * Are there devices with volume range more than 255? I use a bit more - * to be sure. 384 is a resolution magic number found on Logitech - * devices. It will definitively catch all buggy Logitech devices. + * There are definitely devices with a range of ~20,000, so let's be + * conservative and allow for a bit more. */ - if (range > 384) { + if (range > 65535) { usb_audio_warn(mixer->chip, "Warning! Unlikely big volume range (=%u), cval->res is probably wrong.", range); @@ -2946,10 +2952,23 @@ static int parse_audio_unit(struct mixer_build *state, int unitid) static void snd_usb_mixer_free(struct usb_mixer_interface *mixer) { + struct usb_mixer_elem_list *list, *next; + int id; + /* kill pending URBs */ snd_usb_mixer_disconnect(mixer); - kfree(mixer->id_elems); + /* Unregister controls first, snd_ctl_remove() frees the element */ + if (mixer->id_elems) { + for (id = 0; id < MAX_ID_ELEMS; id++) { + for (list = mixer->id_elems[id]; list; list = next) { + next = list->next_id_elem; + if (list->kctl) + snd_ctl_remove(mixer->chip->card, list->kctl); + } + } + kfree(mixer->id_elems); + } if (mixer->urb) { kfree(mixer->urb->transfer_buffer); usb_free_urb(mixer->urb); diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index 828af3095b86e..4873b5e748016 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -311,13 +311,8 @@ static int snd_audigy2nx_led_update(struct usb_mixer_interface *mixer, if (pm.err < 0) return pm.err; - if (chip->usb_id == USB_ID(0x041e, 0x3042)) - err = snd_usb_ctl_msg(chip->dev, - usb_sndctrlpipe(chip->dev, 0), 0x24, - USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_OTHER, - !value, 0, NULL, 0); - /* USB X-Fi S51 Pro */ - if (chip->usb_id == USB_ID(0x041e, 0x30df)) + if (chip->usb_id == USB_ID(0x041e, 0x3042) || /* USB X-Fi S51 */ + chip->usb_id == USB_ID(0x041e, 0x30df)) /* USB X-Fi S51 Pro */ err = snd_usb_ctl_msg(chip->dev, usb_sndctrlpipe(chip->dev, 0), 0x24, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_OTHER, diff --git a/sound/usb/mixer_s1810c.c b/sound/usb/mixer_s1810c.c index 6e09e074c0e7f..93510aa0dc5ef 100644 --- a/sound/usb/mixer_s1810c.c +++ b/sound/usb/mixer_s1810c.c @@ -82,13 +82,13 @@ * mixer and output but a different set for device. */ struct s1810c_ctl_packet { - u32 a; - u32 b; - u32 fixed1; - u32 fixed2; - u32 c; - u32 d; - u32 e; + __le32 a; + __le32 b; + __le32 fixed1; + __le32 fixed2; + __le32 c; + __le32 d; + __le32 e; }; #define SC1810C_CTL_LINE_SW 0 @@ -118,7 +118,7 @@ struct s1810c_ctl_packet { * being zero and different f1/f2. */ struct s1810c_state_packet { - u32 fields[63]; + __le32 fields[63]; }; #define SC1810C_STATE_48V_SW 58 @@ -140,14 +140,14 @@ snd_s1810c_send_ctl_packet(struct usb_device *dev, u32 a, struct s1810c_ctl_packet pkt = { 0 }; int ret = 0; - pkt.fixed1 = SC1810C_CMD_F1; - pkt.fixed2 = SC1810C_CMD_F2; + pkt.fixed1 = __cpu_to_le32(SC1810C_CMD_F1); + pkt.fixed2 = __cpu_to_le32(SC1810C_CMD_F2); - pkt.a = a; - pkt.b = b; - pkt.c = c; - pkt.d = d; - pkt.e = e; + pkt.a = __cpu_to_le32(a); + pkt.b = __cpu_to_le32(b); + pkt.c = __cpu_to_le32(c); + pkt.d = __cpu_to_le32(d); + pkt.e = __cpu_to_le32(e); ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), SC1810C_CMD_REQ, @@ -176,8 +176,8 @@ snd_sc1810c_get_status_field(struct usb_device *dev, struct s1810c_state_packet pkt_in = { { 0 } }; int ret = 0; - pkt_out.fields[SC1810C_STATE_F1_IDX] = SC1810C_SET_STATE_F1; - pkt_out.fields[SC1810C_STATE_F2_IDX] = SC1810C_SET_STATE_F2; + pkt_out.fields[SC1810C_STATE_F1_IDX] = __cpu_to_le32(SC1810C_SET_STATE_F1); + pkt_out.fields[SC1810C_STATE_F2_IDX] = __cpu_to_le32(SC1810C_SET_STATE_F2); ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), SC1810C_SET_STATE_REQ, SC1810C_SET_STATE_REQTYPE, @@ -197,7 +197,7 @@ snd_sc1810c_get_status_field(struct usb_device *dev, return ret; } - (*field) = pkt_in.fields[field_idx]; + (*field) = __le32_to_cpu(pkt_in.fields[field_idx]); (*seqnum)++; return 0; } diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index f2446bf3982c6..48d77030a63d8 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -1328,8 +1328,6 @@ struct scarlett2_data { struct snd_kcontrol *mux_ctls[SCARLETT2_MUX_MAX]; struct snd_kcontrol *mix_ctls[SCARLETT2_MIX_MAX]; struct snd_kcontrol *compressor_ctls[SCARLETT2_COMPRESSOR_CTLS_MAX]; - struct snd_kcontrol *precomp_flt_ctls[SCARLETT2_PRECOMP_FLT_CTLS_MAX]; - struct snd_kcontrol *peq_flt_ctls[SCARLETT2_PEQ_FLT_CTLS_MAX]; struct snd_kcontrol *precomp_flt_switch_ctls[SCARLETT2_DSP_SWITCH_MAX]; struct snd_kcontrol *peq_flt_switch_ctls[SCARLETT2_DSP_SWITCH_MAX]; struct snd_kcontrol *direct_monitor_ctl; @@ -2533,13 +2531,13 @@ static int scarlett2_usb_get_config( err = scarlett2_usb_get(mixer, config_item->offset, buf, size); if (err < 0) return err; - if (size == 2) { + if (config_item->size == 16) { u16 *buf_16 = buf; for (i = 0; i < count; i++, buf_16++) *buf_16 = le16_to_cpu(*(__le16 *)buf_16); - } else if (size == 4) { - u32 *buf_32 = buf; + } else if (config_item->size == 32) { + u32 *buf_32 = (u32 *)buf; for (i = 0; i < count; i++, buf_32++) *buf_32 = le32_to_cpu(*(__le32 *)buf_32); @@ -3447,7 +3445,6 @@ static int scarlett2_update_autogain(struct usb_mixer_interface *mixer) private->autogain_status[i] = private->num_autogain_status_texts - 1; - for (i = 0; i < SCARLETT2_AG_TARGET_COUNT; i++) if (scarlett2_has_config_item(private, scarlett2_ag_target_configs[i])) { @@ -5372,8 +5369,7 @@ static int scarlett2_update_filter_values(struct usb_mixer_interface *mixer) err = scarlett2_usb_get_config( mixer, SCARLETT2_CONFIG_PEQ_FLT_SWITCH, - info->dsp_input_count * info->peq_flt_count, - private->peq_flt_switch); + info->dsp_input_count, private->peq_flt_switch); if (err < 0) return err; @@ -6546,7 +6542,7 @@ static int scarlett2_add_dsp_ctls(struct usb_mixer_interface *mixer, int i) err = scarlett2_add_new_ctl( mixer, &scarlett2_precomp_flt_ctl, i * info->precomp_flt_count + j, - 1, s, &private->precomp_flt_switch_ctls[j]); + 1, s, NULL); if (err < 0) return err; } @@ -6556,7 +6552,7 @@ static int scarlett2_add_dsp_ctls(struct usb_mixer_interface *mixer, int i) err = scarlett2_add_new_ctl( mixer, &scarlett2_peq_flt_ctl, i * info->peq_flt_count + j, - 1, s, &private->peq_flt_switch_ctls[j]); + 1, s, NULL); if (err < 0) return err; } @@ -8255,6 +8251,8 @@ static int scarlett2_find_fc_interface(struct usb_device *dev, if (desc->bInterfaceClass != 255) continue; + if (desc->bNumEndpoints < 1) + continue; epd = get_endpoint(intf->altsetting, 0); private->bInterfaceNumber = desc->bInterfaceNumber; diff --git a/sound/usb/mixer_us16x08.c b/sound/usb/mixer_us16x08.c index 1c5712c31f5e2..f9df40730effd 100644 --- a/sound/usb/mixer_us16x08.c +++ b/sound/usb/mixer_us16x08.c @@ -655,17 +655,25 @@ static void get_meter_levels_from_urb(int s, u8 *meter_urb) { int val = MUC2(meter_urb, s) + (MUC3(meter_urb, s) << 8); + int ch = MUB2(meter_urb, s) - 1; + + if (ch < 0) + return; if (MUA0(meter_urb, s) == 0x61 && MUA1(meter_urb, s) == 0x02 && MUA2(meter_urb, s) == 0x04 && MUB0(meter_urb, s) == 0x62) { - if (MUC0(meter_urb, s) == 0x72) - store->meter_level[MUB2(meter_urb, s) - 1] = val; - if (MUC0(meter_urb, s) == 0xb2) - store->comp_level[MUB2(meter_urb, s) - 1] = val; + if (ch < SND_US16X08_MAX_CHANNELS) { + if (MUC0(meter_urb, s) == 0x72) + store->meter_level[ch] = val; + if (MUC0(meter_urb, s) == 0xb2) + store->comp_level[ch] = val; + } } if (MUA0(meter_urb, s) == 0x61 && MUA1(meter_urb, s) == 0x02 && - MUA2(meter_urb, s) == 0x02 && MUB0(meter_urb, s) == 0x62) - store->master_level[MUB2(meter_urb, s) - 1] = val; + MUA2(meter_urb, s) == 0x02 && MUB0(meter_urb, s) == 0x62) { + if (ch < ARRAY_SIZE(store->master_level)) + store->master_level[ch] = val; + } } /* Function to retrieve current meter values from the device. diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c index 54d01dfd820fa..682b6c1fe76ba 100644 --- a/sound/usb/pcm.c +++ b/sound/usb/pcm.c @@ -1553,7 +1553,8 @@ static int prepare_playback_urb(struct snd_usb_substream *subs, for (i = 0; i < ctx->packets; i++) { counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, avail); - if (counts < 0) + if (counts < 0 || + (frames + counts) * stride > ctx->buffer_size) break; /* set up descriptor */ urb->iso_frame_desc[i].offset = frames * stride; diff --git a/sound/usb/qcom/qc_audio_offload.c b/sound/usb/qcom/qc_audio_offload.c index cfb30a195364a..542eae3a57d9d 100644 --- a/sound/usb/qcom/qc_audio_offload.c +++ b/sound/usb/qcom/qc_audio_offload.c @@ -699,6 +699,7 @@ static void uaudio_event_ring_cleanup_free(struct uaudio_dev *dev) uaudio_iommu_unmap(MEM_EVENT_RING, IOVA_BASE, PAGE_SIZE, PAGE_SIZE); xhci_sideband_remove_interrupter(uadev[dev->chip->card->number].sb); + usb_offload_put(dev->udev); } } @@ -1007,7 +1008,7 @@ static int enable_audio_stream(struct snd_usb_substream *subs, /** * uaudio_transfer_buffer_setup() - fetch and populate xfer buffer params * @subs: usb substream - * @xfer_buf: xfer buf to be allocated + * @xfer_buf_cpu: xfer buf to be allocated * @xfer_buf_len: size of allocation * @mem_info: QMI response info * @@ -1182,12 +1183,16 @@ static int uaudio_event_ring_setup(struct snd_usb_substream *subs, dma_coherent = dev_is_dma_coherent(subs->dev->bus->sysdev); er_pa = 0; + ret = usb_offload_get(subs->dev); + if (ret < 0) + goto exit; + /* event ring */ ret = xhci_sideband_create_interrupter(uadev[card_num].sb, 1, false, 0, uaudio_qdev->data->intr_num); if (ret < 0) { dev_err(&subs->dev->dev, "failed to fetch interrupter\n"); - goto exit; + goto put_offload; } sgt = xhci_sideband_get_event_buffer(uadev[card_num].sb); @@ -1219,6 +1224,8 @@ static int uaudio_event_ring_setup(struct snd_usb_substream *subs, mem_info->dma = 0; remove_interrupter: xhci_sideband_remove_interrupter(uadev[card_num].sb); +put_offload: + usb_offload_put(subs->dev); exit: return ret; } @@ -1483,6 +1490,7 @@ static int prepare_qmi_response(struct snd_usb_substream *subs, uaudio_iommu_unmap(MEM_EVENT_RING, IOVA_BASE, PAGE_SIZE, PAGE_SIZE); free_sec_ring: xhci_sideband_remove_interrupter(uadev[card_num].sb); + usb_offload_put(subs->dev); drop_sync_ep: if (subs->sync_endpoint) { uaudio_iommu_unmap(MEM_XFER_RING, diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 61bd61ffb1b23..a2c039a1b3cd6 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -2147,6 +2147,8 @@ struct usb_audio_quirk_flags_table { static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { /* Device matches */ + DEVICE_FLG(0x001f, 0x0b21, /* AB13X USB Audio */ + QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), DEVICE_FLG(0x03f0, 0x654a, /* HP 320 FHD Webcam */ QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), DEVICE_FLG(0x041e, 0x3000, /* Creative SB Extigy */ @@ -2230,6 +2232,12 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { DEVICE_FLG(0x0644, 0x806b, /* TEAC UD-701 */ QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY | QUIRK_FLAG_IFACE_DELAY), + DEVICE_FLG(0x0644, 0x807d, /* TEAC UD-507 */ + QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY | + QUIRK_FLAG_IFACE_DELAY), + DEVICE_FLG(0x0644, 0x806c, /* Esoteric XD */ + QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY | + QUIRK_FLAG_IFACE_DELAY), DEVICE_FLG(0x06f8, 0xb000, /* Hercules DJ Console (Windows Edition) */ QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x06f8, 0xd002, /* Hercules DJ Console (Macintosh Edition) */ @@ -2288,6 +2296,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x13e5, 0x0001, /* Serato Phono */ QUIRK_FLAG_IGNORE_CTL_ERROR), + DEVICE_FLG(0x152a, 0x880a, /* NeuralDSP Quad Cortex */ + 0), /* Doesn't have the vendor quirk which would otherwise apply */ DEVICE_FLG(0x154e, 0x1002, /* Denon DCD-1500RE */ QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY), DEVICE_FLG(0x154e, 0x1003, /* Denon DA-300USB */ @@ -2354,6 +2364,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x2040, 0x7281, /* Hauppauge HVR-950Q-MXL */ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), + DEVICE_FLG(0x20b1, 0x2009, /* XMOS Ltd DIYINHK USB Audio 2.0 */ + QUIRK_FLAG_SKIP_IMPLICIT_FB | QUIRK_FLAG_DSD_RAW), DEVICE_FLG(0x2040, 0x8200, /* Hauppauge Woodbury */ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x21b4, 0x0081, /* AudioQuest DragonFly */ @@ -2384,10 +2396,14 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY_1M), DEVICE_FLG(0x2d99, 0x0026, /* HECATE G2 GAMING HEADSET */ QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE), + DEVICE_FLG(0x2fc6, 0xf06b, /* MOONDROP Moonriver2 Ti */ + QUIRK_FLAG_CTL_MSG_DELAY), DEVICE_FLG(0x2fc6, 0xf0b7, /* iBasso DC07 Pro */ QUIRK_FLAG_CTL_MSG_DELAY_1M), DEVICE_FLG(0x30be, 0x0101, /* Schiit Hel */ QUIRK_FLAG_IGNORE_CTL_ERROR), + DEVICE_FLG(0x3255, 0x0000, /* Luxman D-10X */ + QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY), DEVICE_FLG(0x339b, 0x3a07, /* Synaptics HONOR USB-C HEADSET */ QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE), DEVICE_FLG(0x413c, 0xa506, /* Dell AE515 sound bar */ @@ -2408,8 +2424,12 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY | QUIRK_FLAG_IFACE_DELAY), VENDOR_FLG(0x07fd, /* MOTU */ QUIRK_FLAG_VALIDATE_RATES), + DEVICE_FLG(0x1235, 0x8006, 0), /* Focusrite Scarlett 2i2 1st Gen */ + DEVICE_FLG(0x1235, 0x800a, 0), /* Focusrite Scarlett 2i4 1st Gen */ + DEVICE_FLG(0x1235, 0x8016, 0), /* Focusrite Scarlett 2i2 1st Gen */ + DEVICE_FLG(0x1235, 0x801c, 0), /* Focusrite Scarlett Solo 1st Gen */ VENDOR_FLG(0x1235, /* Focusrite Novation */ - QUIRK_FLAG_VALIDATE_RATES), + QUIRK_FLAG_SKIP_IFACE_SETUP), VENDOR_FLG(0x1511, /* AURALiC */ QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x152a, /* Thesycon devices */ @@ -2431,6 +2451,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x2622, /* IAG Limited devices */ QUIRK_FLAG_DSD_RAW), + VENDOR_FLG(0x2772, /* Musical Fidelity devices */ + QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x278b, /* Rotel? */ QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x292b, /* Gustard/Ess based devices */ @@ -2489,6 +2511,7 @@ static const char *const snd_usb_audio_quirk_flag_names[] = { QUIRK_STRING_ENTRY(MIC_RES_384), QUIRK_STRING_ENTRY(MIXER_PLAYBACK_MIN_MUTE), QUIRK_STRING_ENTRY(MIXER_CAPTURE_MIN_MUTE), + QUIRK_STRING_ENTRY(SKIP_IFACE_SETUP), NULL }; diff --git a/sound/usb/stream.c b/sound/usb/stream.c index 5c235a5ba7e1b..3b2526964e4b4 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -1261,6 +1261,9 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, set_iface_first = true; /* try to set the interface... */ + if (chip->quirk_flags & QUIRK_FLAG_SKIP_IFACE_SETUP) + continue; + usb_set_interface(chip->dev, iface_no, 0); if (set_iface_first) usb_set_interface(chip->dev, iface_no, altno); diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 79978cae9799c..085530cf62d92 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -224,6 +224,10 @@ extern bool snd_usb_skip_validation; * playback value represents muted state instead of minimum audible volume * QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE * Similar to QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE, but for capture streams + * QUIRK_FLAG_SKIP_IFACE_SETUP + * Skip the probe-time interface setup (usb_set_interface, + * init_pitch, init_sample_rate); redundant with + * snd_usb_endpoint_prepare() at stream-open time */ enum { @@ -253,6 +257,7 @@ enum { QUIRK_TYPE_MIC_RES_384 = 23, QUIRK_TYPE_MIXER_PLAYBACK_MIN_MUTE = 24, QUIRK_TYPE_MIXER_CAPTURE_MIN_MUTE = 25, + QUIRK_TYPE_SKIP_IFACE_SETUP = 26, /* Please also edit snd_usb_audio_quirk_flag_names */ }; @@ -284,5 +289,6 @@ enum { #define QUIRK_FLAG_MIC_RES_384 QUIRK_FLAG(MIC_RES_384) #define QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE QUIRK_FLAG(MIXER_PLAYBACK_MIN_MUTE) #define QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE QUIRK_FLAG(MIXER_CAPTURE_MIN_MUTE) +#define QUIRK_FLAG_SKIP_IFACE_SETUP QUIRK_FLAG(SKIP_IFACE_SETUP) #endif /* __USBAUDIO_H */ diff --git a/sound/usb/usx2y/us144mkii.c b/sound/usb/usx2y/us144mkii.c index f6572a576c150..4854144a72bf4 100644 --- a/sound/usb/usx2y/us144mkii.c +++ b/sound/usb/usx2y/us144mkii.c @@ -421,7 +421,11 @@ static int tascam_probe(struct usb_interface *intf, /* The device has two interfaces; we drive both from this driver. */ if (intf->cur_altsetting->desc.bInterfaceNumber == 1) { - tascam = usb_get_intfdata(usb_ifnum_to_if(dev, 0)); + struct usb_interface *intf_zero = usb_ifnum_to_if(dev, 0); + + if (!intf_zero) + return -ENODEV; + tascam = usb_get_intfdata(intf_zero); if (tascam) { usb_set_intfdata(intf, tascam); tascam->iface1 = intf; diff --git a/sound/usb/validate.c b/sound/usb/validate.c index 4bb4893f6e74f..f62b7cc041dc9 100644 --- a/sound/usb/validate.c +++ b/sound/usb/validate.c @@ -281,7 +281,7 @@ static const struct usb_desc_validator audio_validators[] = { /* UAC_VERSION_2, UAC2_SAMPLE_RATE_CONVERTER: not implemented yet */ /* UAC3 */ - FIXED(UAC_VERSION_2, UAC_HEADER, struct uac3_ac_header_descriptor), + FIXED(UAC_VERSION_3, UAC_HEADER, struct uac3_ac_header_descriptor), FIXED(UAC_VERSION_3, UAC_INPUT_TERMINAL, struct uac3_input_terminal_descriptor), FIXED(UAC_VERSION_3, UAC_OUTPUT_TERMINAL, diff --git a/tools/arch/arm64/include/uapi/asm/unistd.h b/tools/arch/arm64/include/uapi/asm/unistd.h index df36f23876e86..9306726337fe0 100644 --- a/tools/arch/arm64/include/uapi/asm/unistd.h +++ b/tools/arch/arm64/include/uapi/asm/unistd.h @@ -1,2 +1,24 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#include +/* + * Copyright (C) 2012 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define __ARCH_WANT_RENAMEAT +#define __ARCH_WANT_NEW_STAT +#define __ARCH_WANT_SET_GET_RLIMIT +#define __ARCH_WANT_TIME32_SYSCALLS +#define __ARCH_WANT_MEMFD_SECRET + +#include diff --git a/tools/bootconfig/main.c b/tools/bootconfig/main.c index 55d59ed507d54..643f707b8f1da 100644 --- a/tools/bootconfig/main.c +++ b/tools/bootconfig/main.c @@ -162,8 +162,11 @@ static int load_xbc_file(const char *path, char **buf) if (fd < 0) return -errno; ret = fstat(fd, &stat); - if (ret < 0) - return -errno; + if (ret < 0) { + ret = -errno; + close(fd); + return ret; + } ret = load_xbc_fd(fd, buf, stat.st_size); diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 586d1b2595d16..fd43e5ea63f38 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -130,8 +130,8 @@ include $(FEATURES_DUMP) endif endif -LIBS = $(LIBBPF) -lelf -lz -lcrypto -LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lz -lcrypto +LIBS = $(LIBBPF) -lelf -lcrypto -lz +LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lcrypto -lz ifeq ($(feature-libelf-zstd),1) LIBS += -lzstd diff --git a/tools/bpf/bpftool/net.c b/tools/bpf/bpftool/net.c index cfc6f944f7c33..1a06b0b5eef35 100644 --- a/tools/bpf/bpftool/net.c +++ b/tools/bpf/bpftool/net.c @@ -156,7 +156,7 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, bool multipart = true; struct nlmsgerr *err; struct nlmsghdr *nh; - char buf[4096]; + char buf[8192]; int len, ret; while (multipart) { @@ -201,6 +201,9 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, return ret; } } + + if (len) + p_err("Invalid message or trailing data in Netlink response: %d bytes left", len); } ret = 0; done: diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index b34f74d210e9c..f9b742f4bb104 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -28,6 +28,12 @@ #define OPEN_SSL_ERR_BUF_LEN 256 +/* Use deprecated in 3.0 ERR_get_error_line_data for openssl < 3 */ +#if !defined(OPENSSL_VERSION_MAJOR) || (OPENSSL_VERSION_MAJOR < 3) +#define ERR_get_error_all(file, line, func, data, flags) \ + ERR_get_error_line_data(file, line, data, flags) +#endif + static void display_openssl_errors(int l) { char buf[OPEN_SSL_ERR_BUF_LEN]; diff --git a/tools/include/linux/bitfield.h b/tools/include/linux/bitfield.h index 6093fa6db2600..ddf81f24956ba 100644 --- a/tools/include/linux/bitfield.h +++ b/tools/include/linux/bitfield.h @@ -8,6 +8,7 @@ #define _LINUX_BITFIELD_H #include +#include #include /* diff --git a/tools/include/nolibc/arch-x86.h b/tools/include/nolibc/arch-x86.h index d3efc0c3b8adc..c8b0c3e624a51 100644 --- a/tools/include/nolibc/arch-x86.h +++ b/tools/include/nolibc/arch-x86.h @@ -351,7 +351,7 @@ void *memcpy(void *dst, const void *src, size_t len); void *memset(void *dst, int c, size_t len); __asm__ ( -".section .text.nolibc_memmove_memcpy\n" +".pushsection .text.nolibc_memmove_memcpy\n" ".weak memmove\n" ".weak memcpy\n" "memmove:\n" @@ -371,8 +371,9 @@ __asm__ ( "rep movsb\n\t" "cld\n\t" "retq\n" +".popsection\n" -".section .text.nolibc_memset\n" +".pushsection .text.nolibc_memset\n" ".weak memset\n" "memset:\n" "xchgl %eax, %esi\n\t" @@ -381,6 +382,7 @@ __asm__ ( "rep stosb\n\t" "popq %rax\n\t" "retq\n" +".popsection\n" ); #endif /* !defined(__x86_64__) */ diff --git a/tools/include/nolibc/dirent.h b/tools/include/nolibc/dirent.h index 758b95c48e7a4..61a122a60327d 100644 --- a/tools/include/nolibc/dirent.h +++ b/tools/include/nolibc/dirent.h @@ -86,9 +86,9 @@ int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result) * readdir() can only return one entry at a time. * Make sure the non-returned ones are not skipped. */ - ret = lseek(fd, ldir->d_off, SEEK_SET); - if (ret == -1) - return errno; + ret = sys_lseek(fd, ldir->d_off, SEEK_SET); + if (ret < 0) + return -ret; entry->d_ino = ldir->d_ino; /* the destination should always be big enough */ diff --git a/tools/include/nolibc/stdio.h b/tools/include/nolibc/stdio.h index 7630234408c58..724d05ce69624 100644 --- a/tools/include/nolibc/stdio.h +++ b/tools/include/nolibc/stdio.h @@ -600,7 +600,11 @@ int sscanf(const char *str, const char *format, ...) static __attribute__((unused)) void perror(const char *msg) { +#ifdef NOLIBC_IGNORE_ERRNO + fprintf(stderr, "%s%sunknown error\n", (msg && *msg) ? msg : "", (msg && *msg) ? ": " : ""); +#else fprintf(stderr, "%s%serrno=%d\n", (msg && *msg) ? msg : "", (msg && *msg) ? ": " : "", errno); +#endif } static __attribute__((unused)) diff --git a/tools/include/nolibc/sys/wait.h b/tools/include/nolibc/sys/wait.h index 4e66e1f7a03e4..9d9319ba92cbd 100644 --- a/tools/include/nolibc/sys/wait.h +++ b/tools/include/nolibc/sys/wait.h @@ -65,23 +65,29 @@ pid_t waitpid(pid_t pid, int *status, int options) switch (info.si_code) { case 0: - *status = 0; + if (status) + *status = 0; break; case CLD_EXITED: - *status = (info.si_status & 0xff) << 8; + if (status) + *status = (info.si_status & 0xff) << 8; break; case CLD_KILLED: - *status = info.si_status & 0x7f; + if (status) + *status = info.si_status & 0x7f; break; case CLD_DUMPED: - *status = (info.si_status & 0x7f) | 0x80; + if (status) + *status = (info.si_status & 0x7f) | 0x80; break; case CLD_STOPPED: case CLD_TRAPPED: - *status = (info.si_status << 8) + 0x7f; + if (status) + *status = (info.si_status << 8) + 0x7f; break; case CLD_CONTINUED: - *status = 0xffff; + if (status) + *status = 0xffff; break; default: return -1; diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 18907f0fcf9f0..9f141395c074e 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -1061,7 +1061,7 @@ static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, b if (base_btf) { btf->base_btf = base_btf; btf->start_id = btf__type_cnt(base_btf); - btf->start_str_off = base_btf->hdr->str_len; + btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; } if (is_mmap) { @@ -5818,7 +5818,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) { btf->base_btf = (struct btf *)base_btf; btf->start_id = btf__type_cnt(base_btf); - btf->start_str_off = base_btf->hdr->str_len; + btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; } int btf__relocate(struct btf *btf, const struct btf *base_btf) diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 6388392f49a0b..53c6624161d79 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1762,9 +1762,18 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, __u16 left_shift_bits, right_shift_bits; const __u8 *bytes = data; __u8 nr_copy_bits; + __u8 start_bit, nr_bytes; __u64 num = 0; int i; + /* Calculate how many bytes cover the bitfield */ + start_bit = bits_offset % 8; + nr_bytes = (start_bit + bit_sz + 7) / 8; + + /* Bound check */ + if (data + nr_bytes > d->typed_dump->data_end) + return -E2BIG; + /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { pr_warn("unexpected bitfield size %d\n", t->size); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index dd3b2f57082d2..9c98c6adb6d05 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -8245,7 +8245,7 @@ static int kallsyms_cb(unsigned long long sym_addr, char sym_type, struct bpf_object *obj = ctx; const struct btf_type *t; struct extern_desc *ext; - char *res; + const char *res; res = strstr(sym_name, ".llvm."); if (sym_type == 'd' && res) @@ -11574,7 +11574,8 @@ static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, * * [0] fb6a421fb615 ("kallsyms: Match symbols exactly with CONFIG_LTO_CLANG") */ - char sym_trim[256], *psym_trim = sym_trim, *sym_sfx; + char sym_trim[256], *psym_trim = sym_trim; + const char *sym_sfx; if (!(sym_sfx = strstr(sym_name, ".llvm."))) return 0; @@ -12159,7 +12160,7 @@ static int resolve_full_path(const char *file, char *result, size_t result_sz) if (!search_paths[i]) continue; for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) { - char *next_path; + const char *next_path; int seg_len; if (s[0] == ':') diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 56ae77047bc36..b86b8ae3b6341 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -581,7 +581,7 @@ int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz, written = 0; while (written < buf_sz) { - ret = write(fd, buf, buf_sz); + ret = write(fd, buf + written, buf_sz - written); if (ret < 0) { ret = -errno; pr_warn("failed to write '%s': %s\n", filename, errstr(ret)); diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c index c997e69d507fe..c9a78fb16f115 100644 --- a/tools/lib/bpf/netlink.c +++ b/tools/lib/bpf/netlink.c @@ -143,7 +143,7 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, struct nlmsghdr *nh; int len, ret; - ret = alloc_iov(&iov, 4096); + ret = alloc_iov(&iov, 8192); if (ret) goto done; @@ -212,6 +212,8 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, } } } + if (len) + pr_warn("Invalid message or trailing data in Netlink response: %d bytes left\n", len); } ret = 0; done: diff --git a/tools/lib/perf/Makefile b/tools/lib/perf/Makefile index 7fbb50b74c00b..5c64122bf5374 100644 --- a/tools/lib/perf/Makefile +++ b/tools/lib/perf/Makefile @@ -51,9 +51,9 @@ INCLUDES = \ -I$(srctree)/tools/include/uapi # Append required CFLAGS +override CFLAGS := $(INCLUDES) $(CFLAGS) override CFLAGS += -g -Werror -Wall override CFLAGS += -fPIC -override CFLAGS += $(INCLUDES) override CFLAGS += -fvisibility=hidden override CFLAGS += $(EXTRA_WARNINGS) override CFLAGS += $(EXTRA_CFLAGS) diff --git a/tools/lib/perf/cpumap.c b/tools/lib/perf/cpumap.c index b20a5280f2b33..2bbbe1c782b8a 100644 --- a/tools/lib/perf/cpumap.c +++ b/tools/lib/perf/cpumap.c @@ -368,10 +368,12 @@ struct perf_cpu perf_cpu_map__max(const struct perf_cpu_map *map) .cpu = -1 }; - // cpu_map__trim_new() qsort()s it, cpu_map__default_new() sorts it as well. - return __perf_cpu_map__nr(map) > 0 - ? __perf_cpu_map__cpu(map, __perf_cpu_map__nr(map) - 1) - : result; + if (!map) + return result; + + // The CPUs are always sorted and nr is always > 0 as 0 length map is + // encoded as NULL. + return __perf_cpu_map__cpu(map, __perf_cpu_map__nr(map) - 1); } /** Is 'b' a subset of 'a'. */ diff --git a/tools/lib/subcmd/help.c b/tools/lib/subcmd/help.c index ddaeb4eb3e249..db94aa685b73b 100644 --- a/tools/lib/subcmd/help.c +++ b/tools/lib/subcmd/help.c @@ -97,11 +97,13 @@ void exclude_cmds(struct cmdnames *cmds, struct cmdnames *excludes) ei++; } } - if (ci != cj) { - while (ci < cmds->cnt) { - cmds->names[cj++] = cmds->names[ci]; - cmds->names[ci++] = NULL; + while (ci < cmds->cnt) { + if (ci != cj) { + cmds->names[cj] = cmds->names[ci]; + cmds->names[ci] = NULL; } + ci++; + cj++; } for (ci = cj; ci < cmds->cnt; ci++) assert(cmds->names[ci] == NULL); diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index 880e36df0c118..82d6f6b31348c 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -183,7 +183,11 @@ static int compare_ts(const void *p1, const void *p2) { const struct block_list *l1 = p1, *l2 = p2; - return l1->ts_nsec < l2->ts_nsec ? -1 : 1; + if (l1->ts_nsec < l2->ts_nsec) + return -1; + if (l1->ts_nsec > l2->ts_nsec) + return 1; + return 0; } static int compare_cull_condition(const void *p1, const void *p2) diff --git a/tools/net/sunrpc/xdrgen/generators/__init__.py b/tools/net/sunrpc/xdrgen/generators/__init__.py index b98574a36a4ac..a2eb6652ac900 100644 --- a/tools/net/sunrpc/xdrgen/generators/__init__.py +++ b/tools/net/sunrpc/xdrgen/generators/__init__.py @@ -6,7 +6,7 @@ from jinja2 import Environment, FileSystemLoader, Template from xdr_ast import _XdrAst, Specification, _RpcProgram, _XdrTypeSpecifier -from xdr_ast import public_apis, pass_by_reference, get_header_name +from xdr_ast import public_apis, pass_by_reference, structs, get_header_name from xdr_parse import get_xdr_annotate @@ -22,6 +22,7 @@ def create_jinja2_environment(language: str, xdr_type: str) -> Environment: environment.globals["annotate"] = get_xdr_annotate() environment.globals["public_apis"] = public_apis environment.globals["pass_by_reference"] = pass_by_reference + environment.globals["structs"] = structs return environment case _: raise NotImplementedError("Language not supported") diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 index 0b1709cca0d4a..19b219dd276d3 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 @@ -14,7 +14,11 @@ bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_ {% if argument == 'void' %} return xdrgen_decode_void(xdr); {% else %} +{% if argument in structs %} struct {{ argument }} *argp = rqstp->rq_argp; +{% else %} + {{ argument }} *argp = rqstp->rq_argp; +{% endif %} return xdrgen_decode_{{ argument }}(xdr, argp); {% endif %} diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 index 6fc61a5d47b7f..746592cfda562 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 @@ -14,8 +14,14 @@ bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_st {% if result == 'void' %} return xdrgen_encode_void(xdr); {% else %} +{% if result in structs %} struct {{ result }} *resp = rqstp->rq_resp; return xdrgen_encode_{{ result }}(xdr, resp); +{% else %} + {{ result }} *resp = rqstp->rq_resp; + + return xdrgen_encode_{{ result }}(xdr, *resp); +{% endif %} {% endif %} } diff --git a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 index c5518c519854a..df3598c38b2c2 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 @@ -8,6 +8,5 @@ #include #include #include -#include #include diff --git a/tools/net/ynl/ynl-regen.sh b/tools/net/ynl/ynl-regen.sh index 81b4ecd891006..d9809276db982 100755 --- a/tools/net/ynl/ynl-regen.sh +++ b/tools/net/ynl/ynl-regen.sh @@ -21,7 +21,7 @@ files=$(git grep --files-with-matches '^/\* YNL-GEN \(kernel\|uapi\|user\)') for f in $files; do # params: 0 1 2 3 # $YAML YNL-GEN kernel $mode - params=( $(git grep -B1 -h '/\* YNL-GEN' $f | sed 's@/\*\(.*\)\*/@\1@') ) + params=( $(git grep --no-line-number -B1 -h '/\* YNL-GEN' $f | sed 's@/\*\(.*\)\*/@\1@') ) args=$(sed -n 's@/\* YNL-ARG \(.*\) \*/@\1@p' $f) if [ $f -nt ${params[0]} -a -z "$force" ]; then diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index 8c20361dd100e..b436656cd1372 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -7,6 +7,8 @@ srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) endif +RM ?= rm -f + LIBSUBCMD_DIR = $(srctree)/tools/lib/subcmd/ ifneq ($(OUTPUT),) LIBSUBCMD_OUTPUT = $(abspath $(OUTPUT))/libsubcmd @@ -84,10 +86,12 @@ $(LIBSUBCMD)-clean: $(Q)$(RM) -r -- $(LIBSUBCMD_OUTPUT) clean: $(LIBSUBCMD)-clean - $(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL) - $(Q)find $(OUTPUT) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete + $(Q)find $(OUTPUT) \( -name '*.o' -o -name '\.*.cmd' -o -name '\.*.d' \) -type f -print | xargs $(RM) $(Q)$(RM) $(OUTPUT)arch/x86/lib/inat-tables.c $(OUTPUT)fixdep +mrproper: clean + $(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL) + FORCE: -.PHONY: clean FORCE +.PHONY: clean mrproper FORCE diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index 0ad5cc70ecbe7..fdaddc636e977 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -340,52 +340,36 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec if (!rex_w) break; - if (modrm_reg == CFI_SP) { - - if (mod_is_reg()) { - /* mov %rsp, reg */ - ADD_OP(op) { - op->src.type = OP_SRC_REG; - op->src.reg = CFI_SP; - op->dest.type = OP_DEST_REG; - op->dest.reg = modrm_rm; - } - break; - - } else { - /* skip RIP relative displacement */ - if (is_RIP()) - break; - - /* skip nontrivial SIB */ - if (have_SIB()) { - modrm_rm = sib_base; - if (sib_index != CFI_SP) - break; - } - - /* mov %rsp, disp(%reg) */ - ADD_OP(op) { - op->src.type = OP_SRC_REG; - op->src.reg = CFI_SP; - op->dest.type = OP_DEST_REG_INDIRECT; - op->dest.reg = modrm_rm; - op->dest.offset = ins.displacement.value; - } - break; + if (mod_is_reg()) { + /* mov reg, reg */ + ADD_OP(op) { + op->src.type = OP_SRC_REG; + op->src.reg = modrm_reg; + op->dest.type = OP_DEST_REG; + op->dest.reg = modrm_rm; } - break; } - if (rm_is_reg(CFI_SP)) { + /* skip RIP relative displacement */ + if (is_RIP()) + break; - /* mov reg, %rsp */ + /* skip nontrivial SIB */ + if (have_SIB()) { + modrm_rm = sib_base; + if (sib_index != CFI_SP) + break; + } + + /* mov %rsp, disp(%reg) */ + if (modrm_reg == CFI_SP) { ADD_OP(op) { op->src.type = OP_SRC_REG; - op->src.reg = modrm_reg; - op->dest.type = OP_DEST_REG; - op->dest.reg = CFI_SP; + op->src.reg = CFI_SP; + op->dest.type = OP_DEST_REG_INDIRECT; + op->dest.reg = modrm_rm; + op->dest.offset = ins.displacement.value; } break; } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 9004fbc067693..75b6592afcc29 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1207,7 +1207,7 @@ static const char *uaccess_safe_builtin[] = { "copy_mc_enhanced_fast_string", "rep_stos_alternative", "rep_movs_alternative", - "__copy_user_nocache", + "copy_to_nontemporal", NULL }; @@ -2118,12 +2118,11 @@ static void mark_func_jump_tables(struct objtool_file *file, last = insn; /* - * Store back-pointers for unconditional forward jumps such + * Store back-pointers for forward jumps such * that find_jump_table() can back-track using those and * avoid some potentially confusing code. */ - if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && - insn->offset > last->offset && + if (insn->jump_dest && insn->jump_dest->offset > insn->offset && !insn->jump_dest->first_jump_src) { @@ -2564,7 +2563,8 @@ static int decode_sections(struct objtool_file *file) * Must be before add_jump_destinations(), which depends on 'func' * being set for alternatives, to enable proper sibling call detection. */ - if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) { + if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr || + opts.hack_jump_label) { ret = add_special_section_alts(file); if (ret) return ret; @@ -2901,6 +2901,20 @@ static int update_cfi_state(struct instruction *insn, cfi->stack_size += 8; } + else if (cfi->vals[op->src.reg].base == CFI_CFA) { + /* + * Clang RSP musical chairs: + * + * mov %rsp, %rdx [handled above] + * ... + * mov %rdx, %rbx [handled here] + * ... + * mov %rbx, %rsp [handled above] + */ + cfi->vals[op->dest.reg].base = CFI_CFA; + cfi->vals[op->dest.reg].offset = cfi->vals[op->src.reg].offset; + } + break; diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index ca5d77db692a2..9cb51fcde7986 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -108,7 +108,7 @@ struct symbol_hole { }; /* - * Find !section symbol where @offset is after it. + * Find the last symbol before @offset. */ static int symbol_hole_by_offset(const void *key, const struct rb_node *node) { @@ -119,8 +119,7 @@ static int symbol_hole_by_offset(const void *key, const struct rb_node *node) return -1; if (sh->key >= s->offset + s->len) { - if (s->type != STT_SECTION) - sh->sym = s; + sh->sym = s; return 1; } @@ -412,7 +411,8 @@ static void elf_add_symbol(struct elf *elf, struct symbol *sym) sym->len = sym->sym.st_size; __sym_for_each(iter, &sym->sec->symbol_tree, sym->offset, sym->offset) { - if (iter->offset == sym->offset && iter->type == sym->type) + if (iter->offset == sym->offset && iter->type == sym->type && + iter->len == sym->len) iter->alias = sym; } diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 47c906b807ef2..562b9d356d2a2 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -88,8 +88,6 @@ include ../scripts/utilities.mak # # Define NO_LIBBPF if you do not want BPF support # -# Define NO_LIBCAP if you do not want process capabilities considered by perf -# # Define NO_SDT if you do not want to define SDT event in perf tools, # note that it doesn't disable SDT scanning support. # @@ -253,11 +251,12 @@ else endif # shellcheck is using in tools/perf/tests/Build with option -a/--check-sourced ( -# introduced in v0.4.7) and -S/--severity (introduced in v0.6.0). So make the -# minimal shellcheck version as v0.6.0. +# introduced in v0.4.7) and -S/--severity (introduced in v0.6.0) as well as +# dynamic source inclusions (properly handled since v0.7.2). +# So make the minimal shellcheck version as v0.7.2. ifneq ($(SHELLCHECK),) ifeq ($(shell expr $(shell $(SHELLCHECK) --version | grep version: | \ - sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 060), 1) + sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 072), 1) SHELLCHECK := else SHELLCHECK := $(SHELLCHECK) -s bash -a -S warning diff --git a/tools/perf/arch/loongarch/annotate/instructions.c b/tools/perf/arch/loongarch/annotate/instructions.c index 70262d5f14442..1c3abb43c8d72 100644 --- a/tools/perf/arch/loongarch/annotate/instructions.c +++ b/tools/perf/arch/loongarch/annotate/instructions.c @@ -10,9 +10,7 @@ static int loongarch_call__parse(struct arch *arch, struct ins_operands *ops, st { char *c, *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; c = strchr(ops->raw, '#'); if (c++ == NULL) @@ -38,12 +36,16 @@ static int loongarch_call__parse(struct arch *arch, struct ins_operands *ops, st if (ops->target.name == NULL) return -1; - target.addr = map__objdump_2mem(map, ops->target.addr); + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; } @@ -58,7 +60,7 @@ static int loongarch_jump__parse(struct arch *arch, struct ins_operands *ops, st struct map *map = ms->map; struct symbol *sym = ms->sym; struct addr_map_symbol target = { - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; const char *c = strchr(ops->raw, '#'); u64 start, end; @@ -90,7 +92,7 @@ static int loongarch_jump__parse(struct arch *arch, struct ins_operands *ops, st } else { ops->target.offset_avail = false; } - + addr_map_symbol__exit(&target); return 0; } diff --git a/tools/perf/arch/s390/annotate/instructions.c b/tools/perf/arch/s390/annotate/instructions.c index c61193f1e0964..626e6d2cbc81a 100644 --- a/tools/perf/arch/s390/annotate/instructions.c +++ b/tools/perf/arch/s390/annotate/instructions.c @@ -6,9 +6,7 @@ static int s390_call__parse(struct arch *arch, struct ins_operands *ops, { char *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; tok = strchr(ops->raw, ','); if (!tok) @@ -36,12 +34,17 @@ static int s390_call__parse(struct arch *arch, struct ins_operands *ops, if (ops->target.name == NULL) return -1; - target.addr = map__objdump_2mem(map, ops->target.addr); + + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; } diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index 6b6eec65f93f5..4cc33452d79b6 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -1209,8 +1210,12 @@ static int prepare_func_profile(struct perf_ftrace *ftrace) ftrace->graph_verbose = 0; ftrace->profile_hash = hashmap__new(profile_hash, profile_equal, NULL); - if (ftrace->profile_hash == NULL) - return -ENOMEM; + if (IS_ERR(ftrace->profile_hash)) { + int err = PTR_ERR(ftrace->profile_hash); + + ftrace->profile_hash = NULL; + return err; + } return 0; } diff --git a/tools/perf/builtin-kvm.c b/tools/perf/builtin-kvm.c index f0f285763f190..c61369d54dd9d 100644 --- a/tools/perf/builtin-kvm.c +++ b/tools/perf/builtin-kvm.c @@ -2014,7 +2014,7 @@ static int __cmd_record(const char *file_name, int argc, const char **argv) for (j = 1; j < argc; j++, i++) rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]); - BUG_ON(i != rec_argc); + BUG_ON(i + 2 != rec_argc); ret = kvm_add_default_arch_event(&i, rec_argv); if (ret) diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index d76f01956e33b..b1fb87016d5aa 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -2883,11 +2883,11 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) rec->bytes_written += off_cpu_write(rec->session); record__read_lost_samples(rec); - record__synthesize(rec, true); /* this will be recalculated during process_buildids() */ rec->samples = 0; if (!err) { + record__synthesize(rec, true); if (!rec->timestamp_filename) { record__finish_output(rec); } else { diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 7006f848f87a6..b6533dcf5465b 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -856,7 +856,7 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) goto err_out; } } - if (!has_supported_counters) { + if (!has_supported_counters && !stat_config.null_run) { evsel__open_strerror(evlist__first(evsel_list), &target, open_err, msg, sizeof(msg)); ui__error("No supported events found.\n%s\n", msg); @@ -2540,6 +2540,7 @@ int cmd_stat(int argc, const char **argv) unsigned int interval, timeout; const char * const stat_subcommands[] = { "record", "report" }; char errbuf[BUFSIZ]; + struct evsel *counter; setlocale(LC_ALL, ""); @@ -2797,6 +2798,18 @@ int cmd_stat(int argc, const char **argv) evlist__warn_user_requested_cpus(evsel_list, target.cpu_list); + evlist__for_each_entry(evsel_list, counter) { + /* + * Setup BPF counters to require CPUs as any(-1) isn't + * supported. evlist__create_maps below will propagate this + * information to the evsels. Note, evsel__is_bperf isn't yet + * set up, and this change must happen early, so directly use + * the bpf_counter variable and target information. + */ + if ((counter->bpf_counter || target.use_bpf) && !target__has_cpu(&target)) + counter->core.requires_cpu = true; + } + if (evlist__create_maps(evsel_list, &target) < 0) { if (target__has_task(&target)) { pr_err("Problems finding threads of monitor\n"); diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json index ff6627a778057..06bbaea159259 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json @@ -70,19 +70,19 @@ "EventName": "ls_mab_alloc.load_store_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for load-store allocations.", - "UMask": "0x3f" + "UMask": "0x07" }, { "EventName": "ls_mab_alloc.hardware_prefetcher_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for hardware prefetcher allocations.", - "UMask": "0x40" + "UMask": "0x08" }, { "EventName": "ls_mab_alloc.all_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for all types of allocations.", - "UMask": "0x7f" + "UMask": "0x0f" }, { "EventName": "ls_dmnd_fills_from_sys.local_l2", diff --git a/tools/perf/tests/make b/tools/perf/tests/make index b650ce8864ed5..fa4500c65949a 100644 --- a/tools/perf/tests/make +++ b/tools/perf/tests/make @@ -123,7 +123,7 @@ make_minimal += NO_DEMANGLE=1 NO_LIBELF=1 NO_BACKTRACE=1 make_minimal += NO_LIBNUMA=1 NO_LIBBIONIC=1 NO_LIBDW=1 make_minimal += NO_LIBDW_DWARF_UNWIND=1 NO_AUXTRACE=1 NO_LIBBPF=1 make_minimal += NO_SDT=1 NO_JVMTI=1 NO_LIBZSTD=1 -make_minimal += NO_LIBCAP=1 NO_CAPSTONE=1 +make_minimal += NO_CAPSTONE=1 # $(run) contains all available tests run := make_pure diff --git a/tools/perf/tests/shell/sched.sh b/tools/perf/tests/shell/sched.sh index b9b81eaf856e6..b9637069adb1f 100755 --- a/tools/perf/tests/shell/sched.sh +++ b/tools/perf/tests/shell/sched.sh @@ -53,7 +53,7 @@ start_noploops() { } cleanup_noploops() { - kill "$PID1" "$PID2" + kill "$PID1" "$PID2" || true } test_sched_record() { diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh index 8a100a7f2dc17..d72d16d0b8c46 100755 --- a/tools/perf/tests/shell/stat.sh +++ b/tools/perf/tests/shell/stat.sh @@ -18,7 +18,7 @@ test_default_stat() { test_stat_record_report() { echo "stat record and report test" - if ! perf stat record -o - true | perf stat report -i - 2>&1 | \ + if ! perf stat record -e task-clock -o - true | perf stat report -i - 2>&1 | \ grep -E -q "Performance counter stats for 'pipe':" then echo "stat record and report test [Failed]" @@ -30,7 +30,7 @@ test_stat_record_report() { test_stat_record_script() { echo "stat record and script test" - if ! perf stat record -o - true | perf script -i - 2>&1 | \ + if ! perf stat record -e task-clock -o - true | perf script -i - 2>&1 | \ grep -E -q "CPU[[:space:]]+THREAD[[:space:]]+VAL[[:space:]]+ENA[[:space:]]+RUN[[:space:]]+TIME[[:space:]]+EVENT" then echo "stat record and script test [Failed]" @@ -196,7 +196,7 @@ test_hybrid() { fi # Run default Perf stat - cycles_events=$(perf stat -- true 2>&1 | grep -E "/cycles/[uH]*| cycles[:uH]* " -c) + cycles_events=$(perf stat -a -- sleep 0.1 2>&1 | grep -E "/cpu-cycles/[uH]*| cpu-cycles[:uH]* " | wc -l) # The expectation is that default output will have a cycles events on each # hybrid PMU. In situations with no cycles PMU events, like virtualized, this diff --git a/tools/perf/util/addr2line.c b/tools/perf/util/addr2line.c index f2d94a3272d71..a8b39f4f202b6 100644 --- a/tools/perf/util/addr2line.c +++ b/tools/perf/util/addr2line.c @@ -18,8 +18,8 @@ #define MAX_INLINE_NEST 1024 -/* If addr2line doesn't return data for 1 second then timeout. */ -int addr2line_timeout_ms = 1 * 1000; +/* If addr2line doesn't return data for 5 seconds then timeout. */ +int addr2line_timeout_ms = 5 * 1000; static int filename_split(char *filename, unsigned int *line_nr) { diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index a2e34f149a074..8c493608291c2 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -44,6 +44,7 @@ #include "strbuf.h" #include #include +#include #include #include #include @@ -137,8 +138,10 @@ static int annotated_source__alloc_histograms(struct annotated_source *src, return -1; src->samples = hashmap__new(sym_hist_hash, sym_hist_equal, NULL); - if (src->samples == NULL) + if (IS_ERR(src->samples)) { zfree(&src->histograms); + src->samples = NULL; + } return src->histograms ? 0 : -1; } @@ -1021,7 +1024,7 @@ int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, int err, nr; err = evsel__get_arch(evsel, &arch); - if (err < 0) + if (err) return err; if (parch) @@ -1031,7 +1034,7 @@ int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, return 0; args.arch = arch; - args.ms = *ms; + args.ms = ms; if (notes->src == NULL) { notes->src = annotated_source__new(); diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c index 80561630253dd..1a1ffe50ee73a 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c @@ -371,31 +371,20 @@ static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); } - switch (SPE_OP_PKT_LDST_SUBCLASS_GET(payload)) { - case SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP: + if (SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(payload)) arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); - break; - case SPE_OP_PKT_LDST_SUBCLASS_GP_REG: + else if (SPE_OP_PKT_LDST_SUBCLASS_GP_REG(payload)) arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); - break; - case SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG: + else if (SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(payload)) arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); - break; - case SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG: + else if (SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(payload)) arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); - break; - case SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG: + else if (SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(payload)) arm_spe_pkt_out_string(&err, &buf, &buf_len, " MTE-TAG"); - break; - case SPE_OP_PKT_LDST_SUBCLASS_MEMCPY: + else if (SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(payload)) arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMCPY"); - break; - case SPE_OP_PKT_LDST_SUBCLASS_MEMSET: + else if (SPE_OP_PKT_LDST_SUBCLASS_MEMSET(payload)) arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMSET"); - break; - default: - break; - } if (SPE_OP_PKT_IS_LDST_SVE(payload)) { /* SVE effective vector length */ diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h index d00c2481712dc..75e355fe3438c 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h @@ -125,14 +125,13 @@ enum arm_spe_events { #define SPE_OP_PKT_IS_OTHER_SVE_OP(v) (((v) & (BIT(7) | BIT(3) | BIT(0))) == 0x8) -#define SPE_OP_PKT_LDST_SUBCLASS_GET(v) ((v) & GENMASK_ULL(7, 1)) -#define SPE_OP_PKT_LDST_SUBCLASS_GP_REG 0x0 -#define SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP 0x4 -#define SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG 0x10 -#define SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG 0x30 -#define SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG 0x14 -#define SPE_OP_PKT_LDST_SUBCLASS_MEMCPY 0x20 -#define SPE_OP_PKT_LDST_SUBCLASS_MEMSET 0x25 +#define SPE_OP_PKT_LDST_SUBCLASS_GP_REG(v) (((v) & GENMASK_ULL(7, 1)) == 0x0) +#define SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(v) (((v) & GENMASK_ULL(7, 1)) == 0x4) +#define SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(v) (((v) & GENMASK_ULL(7, 1)) == 0x10) +#define SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(v) (((v) & GENMASK_ULL(7, 1)) == 0x30) +#define SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(v) (((v) & GENMASK_ULL(7, 1)) == 0x14) +#define SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(v) (((v) & GENMASK_ULL(7, 1)) == 0x20) +#define SPE_OP_PKT_LDST_SUBCLASS_MEMSET(v) (((v) & GENMASK_ULL(7, 0)) == 0x25) #define SPE_OP_PKT_IS_LDST_ATOMIC(v) (((v) & (GENMASK_ULL(7, 5) | BIT(1))) == 0x2) diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c index ca5d01b9017db..a5882b5822057 100644 --- a/tools/perf/util/bpf_counter.c +++ b/tools/perf/util/bpf_counter.c @@ -460,6 +460,7 @@ static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, struct bperf_leader_bpf *skel = bperf_leader_bpf__open(); int link_fd, diff_map_fd, err; struct bpf_link *link = NULL; + struct perf_thread_map *threads; if (!skel) { pr_err("Failed to open leader skeleton\n"); @@ -495,7 +496,11 @@ static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, * following evsel__open_per_cpu call */ evsel->leader_skel = skel; - evsel__open(evsel, evsel->core.cpus, evsel->core.threads); + assert(!perf_cpu_map__has_any_cpu_or_is_empty(evsel->core.cpus)); + /* Always open system wide. */ + threads = thread_map__new_by_tid(-1); + evsel__open(evsel, evsel->core.cpus, threads); + perf_thread_map__put(threads); out: bperf_leader_bpf__destroy(skel); diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index 60b81d586323f..7b5671f13c535 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -184,6 +184,9 @@ int lock_contention_prepare(struct lock_contention *con) struct evlist *evlist = con->evlist; struct target *target = con->target; + /* make sure it loads the kernel map before lookup */ + map__load(machine__kernel_map(con->machine)); + skel = lock_contention_bpf__open(); if (!skel) { pr_err("Failed to open lock-contention BPF skeleton\n"); @@ -749,9 +752,6 @@ int lock_contention_read(struct lock_contention *con) bpf_prog_test_run_opts(prog_fd, &opts); } - /* make sure it loads the kernel map */ - maps__load_first(machine->kmaps); - prev_key = NULL; while (!bpf_map_get_next_key(fd, prev_key, &key)) { s64 ls_key; diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index be5fd44b1f9dc..2c7feab61b7bf 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -143,7 +143,7 @@ static void print_capstone_detail(cs_insn *insn, char *buf, size_t len, struct annotate_args *args, u64 addr) { int i; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct symbol *sym; /* TODO: support more architectures */ @@ -222,7 +222,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, { #ifdef HAVE_LIBCAPSTONE_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); u64 offset; @@ -256,7 +256,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -268,7 +268,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(args->ms.maps), &handle, is_64bit, disassembler_style) < 0) + if (capstone_init(maps__machine(args->ms->maps), &handle, is_64bit, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -345,7 +345,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, { #ifdef HAVE_LIBCAPSTONE_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); struct nscookie nsc; u64 start = map__rip_2objdump(map, sym->start); @@ -382,7 +382,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(args->ms.maps), &handle, is_64bit, disassembler_style) < 0) + if (capstone_init(maps__machine(args->ms->maps), &handle, is_64bit, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -408,7 +408,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 30f4bb3e7fa30..06eb1a56430cb 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -3089,7 +3089,7 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o if (aux_offset >= auxtrace_event->offset && aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { - struct cs_etm_queue *etmq = etm->queues.queue_array[auxtrace_event->idx].priv; + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); /* * If this AUX event was inside this buffer somewhere, create a new auxtrace event @@ -3098,6 +3098,7 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o auxtrace_fragment.auxtrace = *auxtrace_event; auxtrace_fragment.auxtrace.size = aux_size; auxtrace_fragment.auxtrace.offset = aux_offset; + auxtrace_fragment.auxtrace.idx = etmq->queue_nr; file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index 50b9433f3f8e6..b1be847446fea 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -81,7 +81,7 @@ static int arch__grow_instructions(struct arch *arch) if (new_instructions == NULL) return -1; - memcpy(new_instructions, arch->instructions, arch->nr_instructions); + memcpy(new_instructions, arch->instructions, arch->nr_instructions * sizeof(struct ins)); goto out_update_instructions; } @@ -269,9 +269,7 @@ static int call__parse(struct arch *arch, struct ins_operands *ops, struct map_s { char *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; ops->target.addr = strtoull(ops->raw, &endptr, 16); @@ -296,12 +294,16 @@ static int call__parse(struct arch *arch, struct ins_operands *ops, struct map_s if (ops->target.name == NULL) return -1; find_target: - target.addr = map__objdump_2mem(map, ops->target.addr); + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; indirect_call: @@ -366,7 +368,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s struct map *map = ms->map; struct symbol *sym = ms->sym; struct addr_map_symbol target = { - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; const char *c = strchr(ops->raw, ','); u64 start, end; @@ -410,7 +412,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s start = map__unmap_ip(map, sym->start); end = map__unmap_ip(map, sym->end); - ops->target.outside = target.addr < start || target.addr > end; + ops->target.outside = target.addr < start || target.addr >= end; /* * FIXME: things like this in _cpp_lex_token (gcc's cc1 program): @@ -440,7 +442,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s } else { ops->target.offset_avail = false; } - + addr_map_symbol__exit(&target); return 0; } @@ -1046,7 +1048,7 @@ static size_t disasm_line_size(int nr) struct disasm_line *disasm_line__new(struct annotate_args *args) { struct disasm_line *dl = NULL; - struct annotation *notes = symbol__annotation(args->ms.sym); + struct annotation *notes = symbol__annotation(args->ms->sym); int nr = notes->src->nr_events; dl = zalloc(disasm_line_size(nr)); @@ -1064,7 +1066,7 @@ struct disasm_line *disasm_line__new(struct annotate_args *args) } else if (disasm_line__parse(dl->al.line, &dl->ins.name, &dl->ops.raw) < 0) goto out_free_line; - disasm_line__init_ins(dl, args->arch, &args->ms); + disasm_line__init_ins(dl, args->arch, args->ms); } return dl; @@ -1119,7 +1121,7 @@ static int symbol__parse_objdump_line(struct symbol *sym, struct annotate_args *args, char *parsed_line, int *line_nr, char **fileloc) { - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct annotation *notes = symbol__annotation(sym); struct disasm_line *dl; char *tmp; @@ -1151,7 +1153,7 @@ static int symbol__parse_objdump_line(struct symbol *sym, args->line = parsed_line; args->line_nr = *line_nr; args->fileloc = *fileloc; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); (*line_nr)++; @@ -1169,12 +1171,14 @@ static int symbol__parse_objdump_line(struct symbol *sym, if (dl->ins.ops && ins__is_call(&dl->ins) && !dl->ops.target.sym) { struct addr_map_symbol target = { .addr = dl->ops.target.addr, - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; - if (!maps__find_ams(args->ms.maps, &target) && + if (!maps__find_ams(args->ms->maps, &target) && target.ms.sym->start == target.al_addr) dl->ops.target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); } annotation_line__add(&dl->al, ¬es->src->source); @@ -1338,7 +1342,7 @@ static int symbol__disassemble_raw(char *filename, struct symbol *sym, struct annotate_args *args) { struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); u64 end = map__rip_2objdump(map, sym->end); @@ -1375,7 +1379,7 @@ static int symbol__disassemble_raw(char *filename, struct symbol *sym, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -1501,7 +1505,7 @@ static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, struct annotate_args *args) { struct annotation_options *opts = &annotate_opts; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); char *command; FILE *file; @@ -1644,7 +1648,7 @@ static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, int symbol__disassemble(struct symbol *sym, struct annotate_args *args) { struct annotation_options *options = args->options; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); char symfs_filename[PATH_MAX]; bool delete_extract = false; diff --git a/tools/perf/util/disasm.h b/tools/perf/util/disasm.h index d2cb555e4a3be..a3ea9d6762816 100644 --- a/tools/perf/util/disasm.h +++ b/tools/perf/util/disasm.h @@ -97,7 +97,7 @@ struct ins_ops { struct annotate_args { struct arch *arch; - struct map_symbol ms; + struct map_symbol *ms; struct annotation_options *options; s64 offset; char *line; diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index 344e689567ee1..dc202d4943721 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -111,7 +111,7 @@ bool dso__is_object_file(const struct dso *dso) int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, - char *root_dir, char *filename, size_t size) + const char *root_dir, char *filename, size_t size) { char build_id_hex[SBUILD_ID_SIZE]; int ret = 0; @@ -563,20 +563,15 @@ char *dso__filename_with_chroot(const struct dso *dso, const char *filename) return filename_with_chroot(nsinfo__pid(dso__nsinfo_const(dso)), filename); } -static int __open_dso(struct dso *dso, struct machine *machine) - EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +static char *dso__get_filename(struct dso *dso, const char *root_dir, + bool *decomp) { - int fd = -EINVAL; - char *root_dir = (char *)""; char *name = malloc(PATH_MAX); - bool decomp = false; - if (!name) - return -ENOMEM; + *decomp = false; - mutex_lock(dso__lock(dso)); - if (machine) - root_dir = machine->root_dir; + if (name == NULL) + return NULL; if (dso__read_binary_type_filename(dso, dso__binary_type(dso), root_dir, name, PATH_MAX)) @@ -601,20 +596,38 @@ static int __open_dso(struct dso *dso, struct machine *machine) size_t len = sizeof(newpath); if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { - fd = -(*dso__load_errno(dso)); + errno = *dso__load_errno(dso); goto out; } - decomp = true; + *decomp = true; strcpy(name, newpath); } + return name; + +out: + free(name); + return NULL; +} - fd = do_open(name); +static int __open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + int fd = -EINVAL; + char *name; + bool decomp = false; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); + if (name) + fd = do_open(name); + else + fd = -errno; if (decomp) unlink(name); -out: mutex_unlock(dso__lock(dso)); free(name); return fd; @@ -1910,3 +1923,23 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, return __dso__read_symbol(dso, symfs_filename, start, len, out_buf, out_buf_len, is_64bit); } + +struct debuginfo *dso__debuginfo(struct dso *dso) +{ + char *name; + bool decomp = false; + struct debuginfo *dinfo = NULL; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, "", &decomp); + if (name) + dinfo = debuginfo__new(name); + + if (decomp) + unlink(name); + + mutex_unlock(dso__lock(dso)); + free(name); + return dinfo; +} diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index f8ccb9816b89c..54e470dd07305 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -766,7 +766,7 @@ int dso__kernel_module_get_build_id(struct dso *dso, const char *root_dir); char dso__symtab_origin(const struct dso *dso); int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, - char *root_dir, char *filename, size_t size); + const char *root_dir, char *filename, size_t size); bool is_kernel_module(const char *pathname, int cpumode); bool dso__needs_decompress(struct dso *dso); int dso__decompress_kmodule_fd(struct dso *dso, const char *name); @@ -915,14 +915,7 @@ u64 dso__findnew_global_type(struct dso *dso, u64 addr, u64 offset); bool perf_pid_map_tid(const char *dso_name, int *tid); bool is_perf_pid_map_name(const char *dso_name); -/* - * In the future, we may get debuginfo using build-ID (w/o path). - * Add this helper is for the smooth conversion. - */ -static inline struct debuginfo *dso__debuginfo(struct dso *dso) -{ - return debuginfo__new(dso__long_name(dso)); -} +struct debuginfo *dso__debuginfo(struct dso *dso); const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, const struct map *map, const struct symbol *sym, diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 56ebefd075f2e..9df9818e37013 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -2473,7 +2473,7 @@ static bool evsel__detect_missing_features(struct evsel *evsel, struct perf_cpu /* Please add new feature detection here. */ attr.inherit = true; - attr.sample_type = PERF_SAMPLE_READ; + attr.sample_type = PERF_SAMPLE_READ | PERF_SAMPLE_TID; if (has_attr_feature(&attr, /*flags=*/0)) goto found; perf_missing_features.inherit_sample_read = true; diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c index 103984b29b1e1..cbf2dd2dfc6f4 100644 --- a/tools/perf/util/evsel_fprintf.c +++ b/tools/perf/util/evsel_fprintf.c @@ -182,8 +182,12 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, if (print_dso && (!sym || !sym->inlined)) printed += map__fprintf_dsoname_dsoff(map, print_dsoff, addr, fp); - if (print_srcline) - printed += map__fprintf_srcline(map, addr, "\n ", fp); + if (print_srcline) { + if (node->srcline) + printed += fprintf(fp, "\n %s", node->srcline); + else + printed += map__fprintf_srcline(map, addr, "\n ", fp); + } if (sym && sym->inlined) printed += fprintf(fp, " (inlined)"); diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c index 591548b10e34e..a1cd5196f4ec8 100644 --- a/tools/perf/util/genelf.c +++ b/tools/perf/util/genelf.c @@ -173,6 +173,8 @@ jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, Elf_Shdr *shdr; uint64_t eh_frame_base_offset; char *strsym = NULL; + void *build_id_data = NULL, *tmp; + int build_id_data_len; int symlen; int retval = -1; @@ -251,6 +253,14 @@ jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC; shdr->sh_entsize = 0; + build_id_data = malloc(csize); + if (build_id_data == NULL) { + warnx("cannot allocate build-id data"); + goto error; + } + memcpy(build_id_data, code, csize); + build_id_data_len = csize; + /* * Setup .eh_frame_hdr and .eh_frame */ @@ -334,6 +344,15 @@ jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, shdr->sh_entsize = sizeof(Elf_Sym); shdr->sh_link = unwinding ? 6 : 4; /* index of .strtab section */ + tmp = realloc(build_id_data, build_id_data_len + sizeof(symtab)); + if (tmp == NULL) { + warnx("cannot allocate build-id data"); + goto error; + } + memcpy(tmp + build_id_data_len, symtab, sizeof(symtab)); + build_id_data = tmp; + build_id_data_len += sizeof(symtab); + /* * setup symbols string table * 2 = 1 for 0 in 1st entry, 1 for the 0 at end of symbol for 2nd entry @@ -376,6 +395,15 @@ jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, shdr->sh_flags = 0; shdr->sh_entsize = 0; + tmp = realloc(build_id_data, build_id_data_len + symlen); + if (tmp == NULL) { + warnx("cannot allocate build-id data"); + goto error; + } + memcpy(tmp + build_id_data_len, strsym, symlen); + build_id_data = tmp; + build_id_data_len += symlen; + /* * setup build-id section */ @@ -394,7 +422,7 @@ jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, /* * build-id generation */ - sha1(code, csize, bnote.build_id); + sha1(build_id_data, build_id_data_len, bnote.build_id); bnote.desc.namesz = sizeof(bnote.name); /* must include 0 termination */ bnote.desc.descsz = sizeof(bnote.build_id); bnote.desc.type = NT_GNU_BUILD_ID; @@ -439,7 +467,7 @@ jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, (void)elf_end(e); free(strsym); - + free(build_id_data); return retval; } diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 64ff427040c34..ef4b569f7df46 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -608,10 +608,8 @@ static int hist_entry__init(struct hist_entry *he, map_symbol__exit(&he->branch_info->to.ms); zfree(&he->branch_info); } - if (he->mem_info) { - map_symbol__exit(&mem_info__iaddr(he->mem_info)->ms); - map_symbol__exit(&mem_info__daddr(he->mem_info)->ms); - } + if (he->mem_info) + mem_info__zput(he->mem_info); err: map_symbol__exit(&he->ms); zfree(&he->stat_acc); diff --git a/tools/perf/util/hwmon_pmu.c b/tools/perf/util/hwmon_pmu.c index 416dfea9ffff6..5c27256a220a5 100644 --- a/tools/perf/util/hwmon_pmu.c +++ b/tools/perf/util/hwmon_pmu.c @@ -742,8 +742,7 @@ int perf_pmus__read_hwmon_pmus(struct list_head *pmus) continue; } io__init(&io, name_fd, buf2, sizeof(buf2)); - io__getline(&io, &line, &line_len); - if (line_len > 0 && line[line_len - 1] == '\n') + if (io__getline(&io, &line, &line_len) > 0 && line[line_len - 1] == '\n') line[line_len - 1] = '\0'; hwmon_pmu__new(pmus, buf, class_hwmon_ent->d_name, line); close(name_fd); diff --git a/tools/perf/util/libbfd.c b/tools/perf/util/libbfd.c index 6434c2dccd4a4..2324f6846d510 100644 --- a/tools/perf/util/libbfd.c +++ b/tools/perf/util/libbfd.c @@ -494,7 +494,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, struct bpf_prog_info_node *info_node; int len = sym->end - sym->start; disassembler_ftype disassemble; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct perf_bpil *info_linear; struct disassemble_info info; struct dso *dso = map__dso(map); @@ -605,7 +605,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, args->line = strdup(srcline); args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl) { annotation_line__add(&dl->al, @@ -617,7 +617,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, args->line = buf + prev_buf_size; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl) annotation_line__add(&dl->al, ¬es->src->source); diff --git a/tools/perf/util/llvm.c b/tools/perf/util/llvm.c index 2ebf1f5f65bf7..4ada9a10bd93f 100644 --- a/tools/perf/util/llvm.c +++ b/tools/perf/util/llvm.c @@ -118,7 +118,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, { #ifdef HAVE_LIBLLVM_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); /* Malloc-ed buffer containing instructions read from disk. */ @@ -184,7 +184,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -242,7 +242,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, &line_storage_len); args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; llvm_addr2line(filename, pc, &args->fileloc, (unsigned int *)&args->line_nr, false, NULL); diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index 779f6230130af..c51ec159ac769 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -676,6 +676,7 @@ int maps__find_ams(struct maps *maps, struct addr_map_symbol *ams) if (ams->addr < map__start(ams->ms.map) || ams->addr >= map__end(ams->ms.map)) { if (maps == NULL) return -1; + map__put(ams->ms.map); ams->ms.map = maps__find(maps, ams->addr); if (ams->ms.map == NULL) return -1; diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index da73d686f6b93..4723c2955f22e 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -286,8 +286,11 @@ __add_event(struct list_head *list, int *idx, event_attr_init(attr); evsel = evsel__new_idx(attr, *idx); - if (!evsel) - goto out_err; + if (!evsel) { + perf_cpu_map__put(cpus); + perf_cpu_map__put(pmu_cpus); + return NULL; + } if (name) { evsel->name = strdup(name); @@ -475,8 +478,10 @@ static int parse_events_add_pmu(struct parse_events_state *parse_state, int parse_events_add_cache(struct list_head *list, int *idx, const char *name, struct parse_events_state *parse_state, - struct parse_events_terms *parsed_terms) + struct parse_events_terms *parsed_terms, + void *loc_) { + YYLTYPE *loc = loc_; struct perf_pmu *pmu = NULL; bool found_supported = false; const char *config_name = get_config_name(parsed_terms); @@ -497,12 +502,36 @@ int parse_events_add_cache(struct list_head *list, int *idx, const char *name, * The PMU has the event so add as not a legacy cache * event. */ + struct parse_events_terms temp_terms; + struct parse_events_term *term; + char *config = strdup(name); + + if (!config) + goto out_err; + + parse_events_terms__init(&temp_terms); + if (!parsed_terms) + parsed_terms = &temp_terms; + + if (parse_events_term__num(&term, + PARSE_EVENTS__TERM_TYPE_USER, + config, /*num=*/1, /*novalue=*/true, + loc, /*loc_val=*/NULL) < 0) { + zfree(&config); + goto out_err; + } + list_add(&term->list, &parsed_terms->terms); + ret = parse_events_add_pmu(parse_state, list, pmu, parsed_terms, first_wildcard_match, /*alternate_hw_config=*/PERF_COUNT_HW_MAX); + list_del_init(&term->list); + parse_events_term__delete(term); + parse_events_terms__exit(&temp_terms); if (ret) goto out_err; + found_supported = true; if (first_wildcard_match == NULL) first_wildcard_match = container_of(list->prev, struct evsel, core.node); @@ -2095,14 +2124,18 @@ static int evlist__cmp(void *_fg_idx, const struct list_head *l, const struct li * event's index is used. An index may be forced for events that * must be in the same group, namely Intel topdown events. */ - if (*force_grouped_idx != -1 && arch_evsel__must_be_in_group(lhs)) { + if (lhs->dont_regroup) { + lhs_sort_idx = lhs_core->idx; + } else if (*force_grouped_idx != -1 && arch_evsel__must_be_in_group(lhs)) { lhs_sort_idx = *force_grouped_idx; } else { bool lhs_has_group = lhs_core->leader != lhs_core || lhs_core->nr_members > 1; lhs_sort_idx = lhs_has_group ? lhs_core->leader->idx : lhs_core->idx; } - if (*force_grouped_idx != -1 && arch_evsel__must_be_in_group(rhs)) { + if (rhs->dont_regroup) { + rhs_sort_idx = rhs_core->idx; + } else if (*force_grouped_idx != -1 && arch_evsel__must_be_in_group(rhs)) { rhs_sort_idx = *force_grouped_idx; } else { bool rhs_has_group = rhs_core->leader != rhs_core || rhs_core->nr_members > 1; @@ -2200,10 +2233,10 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list) */ idx = 0; list_for_each_entry(pos, list, core.node) { - const struct evsel *pos_leader = evsel__leader(pos); + struct evsel *pos_leader = evsel__leader(pos); const char *pos_pmu_name = pos->group_pmu_name; const char *cur_leader_pmu_name; - bool pos_force_grouped = force_grouped_idx != -1 && + bool pos_force_grouped = force_grouped_idx != -1 && !pos->dont_regroup && arch_evsel__must_be_in_group(pos); /* Reset index and nr_members. */ @@ -2217,8 +2250,8 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list) * groups can't span PMUs. */ if (!cur_leader || pos->dont_regroup) { - cur_leader = pos; - cur_leaders_grp = &pos->core; + cur_leader = pos->dont_regroup ? pos_leader : pos; + cur_leaders_grp = &cur_leader->core; if (pos_force_grouped) force_grouped_leader = pos; } diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 8f8c8e7fbcf18..8af19dec6a17d 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -237,7 +237,8 @@ int parse_events_add_numeric(struct parse_events_state *parse_state, bool wildcard); int parse_events_add_cache(struct list_head *list, int *idx, const char *name, struct parse_events_state *parse_state, - struct parse_events_terms *parsed_terms); + struct parse_events_terms *parsed_terms, + void *loc); int parse_events__decode_legacy_cache(const char *name, int pmu_type, __u64 *config); int parse_events_add_breakpoint(struct parse_events_state *parse_state, struct list_head *list, diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index a2361c0040d75..ced26c549c33a 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -353,7 +353,7 @@ PE_LEGACY_CACHE opt_event_config if (!list) YYNOMEM; - err = parse_events_add_cache(list, &parse_state->idx, $1, parse_state, $2); + err = parse_events_add_cache(list, &parse_state->idx, $1, parse_state, $2, &@1); parse_events_terms__delete($2); free($1); diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 9e820599bab38..9d62386464680 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -1170,7 +1170,7 @@ int symsrc__init(struct symsrc *ss, struct dso *dso, const char *name, Elf *embedded = read_gnu_debugdata(dso, elf, name, &new_fd); if (!embedded) - goto out_close; + goto out_elf_end; elf_end(elf); close(fd); diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index 948d3e8ad782b..f6c268c588a56 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -964,11 +964,11 @@ static int maps__split_kallsyms(struct maps *kmaps, struct dso *dso, u64 delta, if (dso__kernel(dso) == DSO_SPACE__KERNEL_GUEST) snprintf(dso_name, sizeof(dso_name), "[guest.kernel].%d", - kernel_range++); + kernel_range); else snprintf(dso_name, sizeof(dso_name), "[kernel].%d", - kernel_range++); + kernel_range); ndso = dso__new(dso_name); map__zput(curr_map); @@ -976,6 +976,7 @@ static int maps__split_kallsyms(struct maps *kmaps, struct dso *dso, u64 delta, return -1; dso__set_kernel(ndso, dso__kernel(dso)); + dso__set_loaded(ndso); curr_map = map__new2(pos->start, ndso); if (curr_map == NULL) { diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index fcd1fd13c30e6..c85d219928d47 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -703,6 +703,11 @@ static int perf_event__synthesize_modules_maps_cb(struct map *map, void *data) memcpy(event->mmap2.filename, dso__long_name(dso), dso__long_name_len(dso) + 1); + /* Clear stale build ID from previous module iteration */ + event->mmap2.header.misc &= ~PERF_RECORD_MISC_MMAP_BUILD_ID; + memset(event->mmap2.build_id, 0, sizeof(event->mmap2.build_id)); + event->mmap2.build_id_size = 0; + perf_record_mmap2__read_build_id(&event->mmap2, args->machine, false); } else { size = PERF_ALIGN(dso__long_name_len(dso) + 1, sizeof(u64)); diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index ae70fb56a0572..3ff427a49e4c5 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -136,8 +136,8 @@ static int entry(u64 ip, struct unwind_info *ui) } e->ip = ip; - e->ms.maps = al.maps; - e->ms.map = al.map; + e->ms.maps = maps__get(al.maps); + e->ms.map = map__get(al.map); e->ms.sym = al.sym; pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", @@ -325,6 +325,9 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, if (err) pr_debug("unwind: failed with '%s'\n", dwfl_errmsg(-1)); + for (i = 0; i < ui->idx; i++) + map_symbol__exit(&ui->entries[i].ms); + dwfl_end(ui->dwfl); free(ui); return 0; diff --git a/tools/power/cpupower/lib/cpuidle.c b/tools/power/cpupower/lib/cpuidle.c index f2c1139adf716..bd857ee7541a7 100644 --- a/tools/power/cpupower/lib/cpuidle.c +++ b/tools/power/cpupower/lib/cpuidle.c @@ -150,6 +150,7 @@ unsigned long long cpuidle_state_get_one_value(unsigned int cpu, if (len == 0) return 0; + errno = 0; value = strtoull(linebuf, &endp, 0); if (endp == linebuf || errno == ERANGE) diff --git a/tools/power/cpupower/utils/cpufreq-info.c b/tools/power/cpupower/utils/cpufreq-info.c index 7d3732f5f2f6f..5fe01e516817e 100644 --- a/tools/power/cpupower/utils/cpufreq-info.c +++ b/tools/power/cpupower/utils/cpufreq-info.c @@ -270,7 +270,7 @@ static int get_freq_hardware(unsigned int cpu, unsigned int human) { unsigned long freq; - if (cpupower_cpu_info.caps & CPUPOWER_CAP_APERF) + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF)) return -EINVAL; freq = cpufreq_get_freq_hardware(cpu); diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index 0ce251b8d4665..a7d54dfd3c68b 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -950,9 +950,11 @@ int isolate_cpus(struct isst_id *id, int mask_size, cpu_set_t *cpu_mask, int lev ret = write(fd, "member", strlen("member")); if (ret == -1) { printf("Can't update to member\n"); + close(fd); return ret; } + close(fd); return 0; } diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index f2512d78bcbd8..48677f1846347 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -3285,13 +3285,13 @@ int format_counters(PER_THREAD_PARAMS) /* Added counters */ for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { + if (mp->format == FORMAT_RAW) { if (mp->width == 32) outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)t->counter[i]); else outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->counter[i]); - } else if (mp->format == FORMAT_DELTA) { + } else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) { if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->counter[i]); else @@ -3382,13 +3382,13 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt); for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { + if (mp->format == FORMAT_RAW) { if (mp->width == 32) outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)c->counter[i]); else outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->counter[i]); - } else if (mp->format == FORMAT_DELTA) { + } else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) { if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->counter[i]); else @@ -3581,7 +3581,7 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz); for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { + if (mp->format == FORMAT_RAW) { if (mp->width == 32) outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)p->counter[i]); @@ -3758,7 +3758,7 @@ int delta_package(struct pkg_data *new, struct pkg_data *old) new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value; for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) + if (mp->format == FORMAT_RAW) old->counter[i] = new->counter[i]; else if (mp->format == FORMAT_AVERAGE) old->counter[i] = new->counter[i]; @@ -8842,10 +8842,13 @@ void process_cpuid() edx_flags = edx; if (!no_msr) { - if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) + if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) { warnx("get_msr(UCODE)"); - else + } else { ucode_patch_valid = true; + if (!authentic_amd && !hygon_genuine) + ucode_patch >>= 32; + } } /* @@ -8860,7 +8863,7 @@ void process_cpuid() fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping); if (ucode_patch_valid) - fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF)); + fprintf(outf, " microcode 0x%x", (unsigned int)ucode_patch); fputc('\n', outf); fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level); @@ -10887,6 +10890,14 @@ void probe_cpuidle_residency(void) } } +static bool cpuidle_counter_wanted(char *name) +{ + if (is_deferred_skip(name)) + return false; + + return DO_BIC(BIC_cpuidle) || is_deferred_add(name); +} + void probe_cpuidle_counts(void) { char path[64]; @@ -10896,7 +10907,7 @@ void probe_cpuidle_counts(void) int min_state = 1024, max_state = 0; char *sp; - if (!DO_BIC(BIC_cpuidle)) + if (!DO_BIC(BIC_cpuidle) && !deferred_add_index) return; for (state = 10; state >= 0; --state) { @@ -10911,12 +10922,6 @@ void probe_cpuidle_counts(void) remove_underbar(name_buf); - if (!DO_BIC(BIC_cpuidle) && !is_deferred_add(name_buf)) - continue; - - if (is_deferred_skip(name_buf)) - continue; - /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ sp = strchr(name_buf, '-'); if (!sp) @@ -10931,16 +10936,19 @@ void probe_cpuidle_counts(void) * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for * the last state, so do not add it. */ - *sp = '+'; *(sp + 1) = '\0'; - sprintf(path, "cpuidle/state%d/below", state); - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/below", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } } *sp = '\0'; - sprintf(path, "cpuidle/state%d/usage", state); - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/usage", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } /* * The 'above' sysfs file always contains 0 for the shallowest state (smallest @@ -10949,8 +10957,10 @@ void probe_cpuidle_counts(void) if (state != min_state) { *sp = '-'; *(sp + 1) = '\0'; - sprintf(path, "cpuidle/state%d/above", state); - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/above", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } } } } diff --git a/tools/sched_ext/scx_show_state.py b/tools/sched_ext/scx_show_state.py index 7cdcc6729ea4e..aec4a4498140f 100644 --- a/tools/sched_ext/scx_show_state.py +++ b/tools/sched_ext/scx_show_state.py @@ -27,10 +27,13 @@ def read_static_key(name): def state_str(state): return prog['scx_enable_state_str'][state].string_().decode() -ops = prog['scx_ops'] +root = prog['scx_root'] enable_state = read_atomic("scx_enable_state_var") -print(f'ops : {ops.name.string_().decode()}') +if root: + print(f'ops : {root.ops.name.string_().decode()}') +else: + print('ops : ') print(f'enabled : {read_static_key("__scx_enabled")}') print(f'switching_all : {read_int("scx_switching_all")}') print(f'switched_all : {read_static_key("__scx_switched_all")}') diff --git a/tools/spi/.gitignore b/tools/spi/.gitignore index 14ddba3d21957..038261b34ed83 100644 --- a/tools/spi/.gitignore +++ b/tools/spi/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only spidev_fdx spidev_test +include/ diff --git a/tools/testing/ktest/config-bisect.pl b/tools/testing/ktest/config-bisect.pl index 6fd8649353195..bee7cb34a289c 100755 --- a/tools/testing/ktest/config-bisect.pl +++ b/tools/testing/ktest/config-bisect.pl @@ -741,9 +741,9 @@ sub config_bisect { die "Can not find file $bad\n"; } if ($val eq "good") { - run_command "cp $output_config $good" or die "failed to copy $config to $good\n"; + run_command "cp $output_config $good" or die "failed to copy $output_config to $good\n"; } elsif ($val eq "bad") { - run_command "cp $output_config $bad" or die "failed to copy $config to $bad\n"; + run_command "cp $output_config $bad" or die "failed to copy $output_config to $bad\n"; } } diff --git a/tools/testing/kunit/configs/all_tests.config b/tools/testing/kunit/configs/all_tests.config index 422e186cf3cf1..bccc2c77196d5 100644 --- a/tools/testing/kunit/configs/all_tests.config +++ b/tools/testing/kunit/configs/all_tests.config @@ -44,8 +44,12 @@ CONFIG_REGMAP_BUILD=y CONFIG_AUDIT=y +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y + CONFIG_PRIME_NUMBERS=y +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=y + CONFIG_SECURITY=y CONFIG_SECURITY_APPARMOR=y CONFIG_SECURITY_LANDLOCK=y diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 260d8d9aa1db4..2998e1bc088b2 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -346,8 +346,10 @@ def build_kernel(self, jobs: int, build_dir: str, make_options: Optional[List[st return self.validate_config(build_dir) def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: str='', filter_action: Optional[str]=None, timeout: Optional[int]=None) -> Iterator[str]: - if not args: - args = [] + # Copy to avoid mutating the caller-supplied list. exec_tests() reuses + # the same args across repeated run_kernel() calls (e.g. --run_isolated), + # so appending to the original would accumulate stale flags on each call. + args = list(args) if args else [] if filter_glob: args.append('kunit.filter_glob=' + filter_glob) if filter: diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index bbba921e0eacb..ed45bac1548d9 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -489,6 +489,32 @@ def fake_start(unused_args, unused_build_dir): with open(kunit_kernel.get_outfile_path(build_dir), 'rt') as outfile: self.assertEqual(outfile.read(), 'hi\nbye\n', msg='Missing some output') + def test_run_kernel_args_not_mutated(self): + """Verify run_kernel() copies args so callers can reuse them.""" + start_calls = [] + + def fake_start(start_args, unused_build_dir): + start_calls.append(list(start_args)) + return subprocess.Popen(['printf', 'KTAP version 1\n'], + text=True, stdout=subprocess.PIPE) + + with tempfile.TemporaryDirectory('') as build_dir: + tree = kunit_kernel.LinuxSourceTree(build_dir, + kunitconfig_paths=[os.devnull]) + with mock.patch.object(tree._ops, 'start', side_effect=fake_start), \ + mock.patch.object(kunit_kernel.subprocess, 'call'): + kernel_args = ['mem=1G'] + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test1'): + pass + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test2'): + pass + self.assertEqual(kernel_args, ['mem=1G'], + 'run_kernel() should not modify caller args') + self.assertIn('kunit.filter_glob=suite.test1', start_calls[0]) + self.assertIn('kunit.filter_glob=suite.test2', start_calls[1]) + def test_build_reconfig_no_config(self): with tempfile.TemporaryDirectory('') as build_dir: with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f: diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index cfd4378e2129b..f87e9f251d131 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -670,6 +670,7 @@ static int nfit_test_search_spa(struct nvdimm_bus *bus, .addr = spa->spa, .region = NULL, }; + struct nfit_mem *nfit_mem; u64 dpa; ret = device_for_each_child(&bus->dev, &ctx, @@ -687,8 +688,12 @@ static int nfit_test_search_spa(struct nvdimm_bus *bus, */ nd_mapping = &nd_region->mapping[nd_region->ndr_mappings - 1]; nvdimm = nd_mapping->nvdimm; + nfit_mem = nvdimm_provider_data(nvdimm); + if (!nfit_mem) + return -EINVAL; - spa->devices[0].nfit_device_handle = handle[nvdimm->id]; + spa->devices[0].nfit_device_handle = + __to_nfit_memdev(nfit_mem)->device_handle; spa->num_nvdimms = 1; spa->devices[0].dpa = dpa; diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c index 2f830ff8396cc..945144e985072 100644 --- a/tools/testing/radix-tree/idr-test.c +++ b/tools/testing/radix-tree/idr-test.c @@ -57,6 +57,26 @@ void idr_alloc_test(void) idr_destroy(&idr); } +void idr_alloc2_test(void) +{ + int id; + struct idr idr = IDR_INIT_BASE(idr, 1); + + id = idr_alloc(&idr, idr_alloc2_test, 0, 1, GFP_KERNEL); + assert(id == -ENOSPC); + + id = idr_alloc(&idr, idr_alloc2_test, 1, 2, GFP_KERNEL); + assert(id == 1); + + id = idr_alloc(&idr, idr_alloc2_test, 0, 1, GFP_KERNEL); + assert(id == -ENOSPC); + + id = idr_alloc(&idr, idr_alloc2_test, 0, 2, GFP_KERNEL); + assert(id == -ENOSPC); + + idr_destroy(&idr); +} + void idr_replace_test(void) { DEFINE_IDR(idr); @@ -409,6 +429,7 @@ void idr_checks(void) idr_replace_test(); idr_alloc_test(); + idr_alloc2_test(); idr_null_test(); idr_nowait_test(); idr_get_next_test(0); diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 3b96d090c5ebe..09a326f375e91 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -473,8 +473,8 @@ static void sve2_sigill(void) static void sve2p1_sigill(void) { - /* BFADD Z0.H, Z0.H, Z0.H */ - asm volatile(".inst 0x65000000" : : : "z0"); + /* LD1Q {Z0.Q}, P0/Z, [Z0.D, X0] */ + asm volatile(".inst 0xC400A000" : : : "z0"); } static void sve2p2_sigill(void) diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index be1ee7ba7ce03..ca557e5668fd8 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -23,6 +23,7 @@ test_tcpnotify_user test_libbpf xdping test_cpp +test_progs_verification_cert *.d *.subskel.h *.skel.h diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index f00587d4ede68..591e7e77f89ba 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -721,7 +721,8 @@ $(VERIFICATION_CERT) $(PRIVATE_KEY): $(VERIFY_SIG_SETUP) $(Q)$(VERIFY_SIG_SETUP) genkey $(BUILD_DIR) $(VERIFY_SIG_HDR): $(VERIFICATION_CERT) - $(Q)xxd -i -n test_progs_verification_cert $< > $@ + $(Q)ln -fs $< test_progs_verification_cert && \ + xxd -i test_progs_verification_cert > $@ # Define test_progs test runner. TRUNNER_TESTS_DIR := prog_tests @@ -804,7 +805,7 @@ $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) # Include find_bit.c to compile xskxceiver. EXTRA_SRC := $(TOOLSDIR)/lib/find_bit.c -$(OUTPUT)/xskxceiver: $(EXTRA_SRC) xskxceiver.c xskxceiver.h $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) +$(OUTPUT)/xskxceiver: $(EXTRA_SRC) test_xsk.c test_xsk.h xskxceiver.c xskxceiver.h $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ @@ -893,7 +894,8 @@ EXTRA_CLEAN := $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) \ $(addprefix $(OUTPUT)/,*.o *.d *.skel.h *.lskel.h *.subskel.h \ no_alu32 cpuv4 bpf_gcc \ liburandom_read.so) \ - $(OUTPUT)/FEATURE-DUMP.selftests + $(OUTPUT)/FEATURE-DUMP.selftests \ + test_progs_verification_cert .PHONY: docs docs-clean diff --git a/tools/testing/selftests/bpf/prog_tests/check_mtu.c b/tools/testing/selftests/bpf/prog_tests/check_mtu.c index 2a9a30650350e..65b4512967e74 100644 --- a/tools/testing/selftests/bpf/prog_tests/check_mtu.c +++ b/tools/testing/selftests/bpf/prog_tests/check_mtu.c @@ -153,6 +153,26 @@ static void test_check_mtu_run_tc(struct test_check_mtu *skel, ASSERT_EQ(mtu_result, mtu_expect, "MTU-compare-user"); } +static void test_chk_segs_flag(struct test_check_mtu *skel, __u32 mtu) +{ + int err, prog_fd = bpf_program__fd(skel->progs.tc_chk_segs_flag); + struct __sk_buff skb = { + .gso_size = 10, + }; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .ctx_in = &skb, + .ctx_size_in = sizeof(skb), + ); + + /* Lower the mtu to test the BPF_MTU_CHK_SEGS */ + SYS_NOFAIL("ip link set dev lo mtu 10"); + err = bpf_prog_test_run_opts(prog_fd, &topts); + SYS_NOFAIL("ip link set dev lo mtu %u", mtu); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, BPF_OK, "retval"); +} static void test_check_mtu_tc(__u32 mtu, __u32 ifindex) { @@ -177,11 +197,12 @@ static void test_check_mtu_tc(__u32 mtu, __u32 ifindex) test_check_mtu_run_tc(skel, skel->progs.tc_minus_delta, mtu); test_check_mtu_run_tc(skel, skel->progs.tc_input_len, mtu); test_check_mtu_run_tc(skel, skel->progs.tc_input_len_exceed, mtu); + test_chk_segs_flag(skel, mtu); cleanup: test_check_mtu__destroy(skel); } -void serial_test_check_mtu(void) +void test_ns_check_mtu(void) { int mtu_lo; diff --git a/tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c b/tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c index 1de14b111931a..6e35e13c20220 100644 --- a/tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c @@ -57,7 +57,8 @@ static void subtest_kmem_cache_iter_check_slabinfo(struct kmem_cache_iter *skel) if (!ASSERT_OK(ret, "kmem_cache_lookup")) break; - ASSERT_STREQ(r.name, name, "kmem_cache_name"); + ASSERT_STRNEQ(r.name, name, sizeof(r.name) - 1, + "kmem_cache_name"); ASSERT_EQ(r.obj_size, objsize, "kmem_cache_objsize"); seen++; diff --git a/tools/testing/selftests/bpf/prog_tests/perf_branches.c b/tools/testing/selftests/bpf/prog_tests/perf_branches.c index bc24f83339d64..0a7ef770c487c 100644 --- a/tools/testing/selftests/bpf/prog_tests/perf_branches.c +++ b/tools/testing/selftests/bpf/prog_tests/perf_branches.c @@ -15,6 +15,10 @@ static void check_good_sample(struct test_perf_branches *skel) int pbe_size = sizeof(struct perf_branch_entry); int duration = 0; + if (CHECK(!skel->bss->run_cnt, "invalid run_cnt", + "checked sample validity before prog run")) + return; + if (CHECK(!skel->bss->valid, "output not valid", "no valid sample from prog")) return; @@ -45,6 +49,10 @@ static void check_bad_sample(struct test_perf_branches *skel) int written_stack = skel->bss->written_stack_out; int duration = 0; + if (CHECK(!skel->bss->run_cnt, "invalid run_cnt", + "checked sample validity before prog run")) + return; + if (CHECK(!skel->bss->valid, "output not valid", "no valid sample from prog")) return; @@ -83,8 +91,12 @@ static void test_perf_branches_common(int perf_fd, err = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set), &cpu_set); if (CHECK(err, "set_affinity", "cpu #0, err %d\n", err)) goto out_destroy; - /* spin the loop for a while (random high number) */ - for (i = 0; i < 1000000; ++i) + + /* Spin the loop for a while by using a high iteration count, and by + * checking whether the specific run count marker has been explicitly + * incremented at least once by the backing perf_event BPF program. + */ + for (i = 0; i < 100000000 && !*(volatile int *)&skel->bss->run_cnt; ++i) ++j; test_perf_branches__detach(skel); @@ -116,11 +128,11 @@ static void test_perf_branches_hw(void) pfd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, PERF_FLAG_FD_CLOEXEC); /* - * Some setups don't support branch records (virtual machines, !x86), - * so skip test in this case. + * Some setups don't support LBR (virtual machines, !x86, AMD Milan Zen + * 3 which only supports BRS), so skip test in this case. */ if (pfd < 0) { - if (errno == ENOENT || errno == EOPNOTSUPP) { + if (errno == ENOENT || errno == EOPNOTSUPP || errno == EINVAL) { printf("%s:SKIP:no PERF_SAMPLE_BRANCH_STACK\n", __func__); test__skip(); diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index d93a0c7b1786f..04938d0d431b3 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -422,15 +422,69 @@ static bool is_valid_range(enum num_t t, struct range x) } } -static struct range range_improve(enum num_t t, struct range old, struct range new) +static struct range range_intersection(enum num_t t, struct range old, struct range new) { return range(t, max_t(t, old.a, new.a), min_t(t, old.b, new.b)); } +/* + * Result is precise when 'x' and 'y' overlap or form a continuous range, + * result is an over-approximation if 'x' and 'y' do not overlap. + */ +static struct range range_union(enum num_t t, struct range x, struct range y) +{ + if (!is_valid_range(t, x)) + return y; + if (!is_valid_range(t, y)) + return x; + return range(t, min_t(t, x.a, y.a), max_t(t, x.b, y.b)); +} + +/* + * This function attempts to improve x range intersecting it with y. + * range_cast(... to_t ...) looses precision for ranges that pass to_t + * min/max boundaries. To avoid such precision loses this function + * splits both x and y into halves corresponding to non-overflowing + * sub-ranges: [0, smin] and [smax, -1]. + * Final result is computed as follows: + * + * ((x ∩ [0, smax]) ∩ (y ∩ [0, smax])) ∪ + * ((x ∩ [smin,-1]) ∩ (y ∩ [smin,-1])) + * + * Precision might still be lost if final union is not a continuous range. + */ +static struct range range_refine_in_halves(enum num_t x_t, struct range x, + enum num_t y_t, struct range y) +{ + struct range x_pos, x_neg, y_pos, y_neg, r_pos, r_neg; + u64 smax, smin, neg_one; + + if (t_is_32(x_t)) { + smax = (u64)(u32)S32_MAX; + smin = (u64)(u32)S32_MIN; + neg_one = (u64)(u32)(s32)(-1); + } else { + smax = (u64)S64_MAX; + smin = (u64)S64_MIN; + neg_one = U64_MAX; + } + x_pos = range_intersection(x_t, x, range(x_t, 0, smax)); + x_neg = range_intersection(x_t, x, range(x_t, smin, neg_one)); + y_pos = range_intersection(y_t, y, range(x_t, 0, smax)); + y_neg = range_intersection(y_t, y, range(y_t, smin, neg_one)); + r_pos = range_intersection(x_t, x_pos, range_cast(y_t, x_t, y_pos)); + r_neg = range_intersection(x_t, x_neg, range_cast(y_t, x_t, y_neg)); + return range_union(x_t, r_pos, r_neg); + +} + static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, struct range y) { struct range y_cast; + if (t_is_32(x_t) == t_is_32(y_t)) + x = range_refine_in_halves(x_t, x, y_t, y); + y_cast = range_cast(y_t, x_t, y); /* If we know that @@ -444,7 +498,7 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, */ if (x_t == S64 && y_t == S32 && y_cast.a <= S32_MAX && y_cast.b <= S32_MAX && (s64)x.a >= S32_MIN && (s64)x.b <= S32_MAX) - return range_improve(x_t, x, y_cast); + return range_intersection(x_t, x, y_cast); /* the case when new range knowledge, *y*, is a 32-bit subregister * range, while previous range knowledge, *x*, is a full register @@ -462,7 +516,7 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, x_swap = range(x_t, swap_low32(x.a, y_cast.a), swap_low32(x.b, y_cast.b)); if (!is_valid_range(x_t, x_swap)) return x; - return range_improve(x_t, x, x_swap); + return range_intersection(x_t, x, x_swap); } if (!t_is_32(x_t) && !t_is_32(y_t) && x_t != y_t) { @@ -480,7 +534,7 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, } /* otherwise, plain range cast and intersection works */ - return range_improve(x_t, x, y_cast); + return range_intersection(x_t, x, y_cast); } /* ======================= @@ -2091,7 +2145,7 @@ static struct subtest_case crafted_cases[] = { {U64, S64, {0, 0xffffffffULL}, {0x7fffffff, 0x7fffffff}}, {U64, U32, {0, 0x100000000}, {0, 0}}, - {U64, U32, {0xfffffffe, 0x100000000}, {0x80000000, 0x80000000}}, + {U64, U32, {0xfffffffe, 0x300000000}, {0x80000000, 0x80000000}}, {U64, S32, {0, 0xffffffff00000000ULL}, {0, 0}}, /* these are tricky cases where lower 32 bits allow to tighten 64 diff --git a/tools/testing/selftests/bpf/prog_tests/send_signal.c b/tools/testing/selftests/bpf/prog_tests/send_signal.c index 1702aa592c2c2..7ac4d5a488aa5 100644 --- a/tools/testing/selftests/bpf/prog_tests/send_signal.c +++ b/tools/testing/selftests/bpf/prog_tests/send_signal.c @@ -206,6 +206,11 @@ static void test_send_signal_common(struct perf_event_attr *attr, skel_open_load_failure: close(pipe_c2p[0]); close(pipe_p2c[1]); + /* + * Child is either about to exit cleanly or stuck in case of errors. + * Nudge it to exit. + */ + kill(pid, SIGKILL); wait(NULL); } diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c index c9efdd2a5b18a..c93718dafd9b6 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c @@ -74,11 +74,20 @@ static void test_stacktrace_ips_kprobe_multi(bool retprobe) load_kallsyms(); - check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, - ksym_get_addr("bpf_testmod_stacktrace_test_3"), - ksym_get_addr("bpf_testmod_stacktrace_test_2"), - ksym_get_addr("bpf_testmod_stacktrace_test_1"), - ksym_get_addr("bpf_testmod_test_read")); + if (retprobe) { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } else { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5, + ksym_get_addr("bpf_testmod_stacktrace_test"), + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } cleanup: stacktrace_ips__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/wq.c b/tools/testing/selftests/bpf/prog_tests/wq.c index 99e438fe12acd..15ac8e6d17450 100644 --- a/tools/testing/selftests/bpf/prog_tests/wq.c +++ b/tools/testing/selftests/bpf/prog_tests/wq.c @@ -16,12 +16,12 @@ void serial_test_wq(void) /* re-run the success test to check if the timer was actually executed */ wq_skel = wq__open_and_load(); - if (!ASSERT_OK_PTR(wq_skel, "wq_skel_load")) + if (!ASSERT_OK_PTR(wq_skel, "wq__open_and_load")) return; err = wq__attach(wq_skel); if (!ASSERT_OK(err, "wq_attach")) - return; + goto clean_up; prog_fd = bpf_program__fd(wq_skel->progs.test_syscall_array_sleepable); err = bpf_prog_test_run_opts(prog_fd, &topts); @@ -31,6 +31,7 @@ void serial_test_wq(void) usleep(50); /* 10 usecs should be enough, but give it extra */ ASSERT_EQ(wq_skel->bss->ok_sleepable, (1 << 1), "ok_sleepable"); +clean_up: wq__destroy(wq_skel); } diff --git a/tools/testing/selftests/bpf/progs/dmabuf_iter.c b/tools/testing/selftests/bpf/progs/dmabuf_iter.c index 13cdb11fdeb2b..9cbb7442646e5 100644 --- a/tools/testing/selftests/bpf/progs/dmabuf_iter.c +++ b/tools/testing/selftests/bpf/progs/dmabuf_iter.c @@ -48,7 +48,7 @@ int dmabuf_collector(struct bpf_iter__dmabuf *ctx) /* Buffers are not required to be named */ if (pname) { - if (bpf_probe_read_kernel(name, sizeof(name), pname)) + if (bpf_probe_read_kernel_str(name, sizeof(name), pname) < 0) return 1; /* Name strings can be provided by userspace */ diff --git a/tools/testing/selftests/bpf/progs/exceptions_assert.c b/tools/testing/selftests/bpf/progs/exceptions_assert.c index a01c2736890f9..858af5988a38a 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_assert.c +++ b/tools/testing/selftests/bpf/progs/exceptions_assert.c @@ -18,43 +18,43 @@ return *(u64 *)num; \ } -__msg(": R0=0xffffffff80000000") +__msg("R{{.}}=0xffffffff80000000") check_assert(s64, ==, eq_int_min, INT_MIN); -__msg(": R0=0x7fffffff") +__msg("R{{.}}=0x7fffffff") check_assert(s64, ==, eq_int_max, INT_MAX); -__msg(": R0=0") +__msg("R{{.}}=0") check_assert(s64, ==, eq_zero, 0); -__msg(": R0=0x8000000000000000 R1=0x8000000000000000") +__msg("R{{.}}=0x8000000000000000") check_assert(s64, ==, eq_llong_min, LLONG_MIN); -__msg(": R0=0x7fffffffffffffff R1=0x7fffffffffffffff") +__msg("R{{.}}=0x7fffffffffffffff") check_assert(s64, ==, eq_llong_max, LLONG_MAX); -__msg(": R0=scalar(id=1,smax=0x7ffffffe)") +__msg("R{{.}}=scalar(id=1,smax=0x7ffffffe)") check_assert(s64, <, lt_pos, INT_MAX); -__msg(": R0=scalar(id=1,smax=-1,umin=0x8000000000000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smax=-1,umin=0x8000000000000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))") check_assert(s64, <, lt_zero, 0); -__msg(": R0=scalar(id=1,smax=0xffffffff7fffffff") +__msg("R{{.}}=scalar(id=1,smax=0xffffffff7fffffff") check_assert(s64, <, lt_neg, INT_MIN); -__msg(": R0=scalar(id=1,smax=0x7fffffff)") +__msg("R{{.}}=scalar(id=1,smax=0x7fffffff)") check_assert(s64, <=, le_pos, INT_MAX); -__msg(": R0=scalar(id=1,smax=0)") +__msg("R{{.}}=scalar(id=1,smax=0)") check_assert(s64, <=, le_zero, 0); -__msg(": R0=scalar(id=1,smax=0xffffffff80000000") +__msg("R{{.}}=scalar(id=1,smax=0xffffffff80000000") check_assert(s64, <=, le_neg, INT_MIN); -__msg(": R0=scalar(id=1,smin=umin=0x80000000,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=0x80000000,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >, gt_pos, INT_MAX); -__msg(": R0=scalar(id=1,smin=umin=1,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=1,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >, gt_zero, 0); -__msg(": R0=scalar(id=1,smin=0xffffffff80000001") +__msg("R{{.}}=scalar(id=1,smin=0xffffffff80000001") check_assert(s64, >, gt_neg, INT_MIN); -__msg(": R0=scalar(id=1,smin=umin=0x7fffffff,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=0x7fffffff,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >=, ge_pos, INT_MAX); -__msg(": R0=scalar(id=1,smin=0,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=0,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >=, ge_zero, 0); -__msg(": R0=scalar(id=1,smin=0xffffffff80000000") +__msg("R{{.}}=scalar(id=1,smin=0xffffffff80000000") check_assert(s64, >=, ge_neg, INT_MIN); SEC("?tc") diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 8a0fdff899271..d7f1c492e3dd3 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -8,6 +8,7 @@ #include "bpf_experimental.h" extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; #define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8))) @@ -131,7 +132,7 @@ int reject_subprog_with_lock(void *ctx) } SEC("?tc") -__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_rcu_read_lock-ed region") +__failure __msg("bpf_throw cannot be used inside bpf_rcu_read_lock-ed region") int reject_with_rcu_read_lock(void *ctx) { bpf_rcu_read_lock(); @@ -147,11 +148,13 @@ __noinline static int throwing_subprog(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_rcu_read_lock-ed region") +__failure __msg("bpf_throw cannot be used inside bpf_rcu_read_lock-ed region") int reject_subprog_with_rcu_read_lock(void *ctx) { bpf_rcu_read_lock(); - return throwing_subprog(ctx); + throwing_subprog(ctx); + bpf_rcu_read_unlock(); + return 0; } static bool rbless(struct bpf_rb_node *n1, const struct bpf_rb_node *n2) diff --git a/tools/testing/selftests/bpf/progs/strobemeta.h b/tools/testing/selftests/bpf/progs/strobemeta.h index a5c74d31a2444..6e1918deaf262 100644 --- a/tools/testing/selftests/bpf/progs/strobemeta.h +++ b/tools/testing/selftests/bpf/progs/strobemeta.h @@ -330,9 +330,9 @@ static void *calc_location(struct strobe_value_loc *loc, void *tls_base) } bpf_probe_read_user(&tls_ptr, sizeof(void *), dtv); /* if pointer has (void *)-1 value, then TLS wasn't initialized yet */ - return tls_ptr && tls_ptr != (void *)-1 - ? tls_ptr + tls_index.offset - : NULL; + if (!tls_ptr || tls_ptr == (void *)-1) + return NULL; + return tls_ptr + tls_index.offset; } #ifdef SUBPROGS diff --git a/tools/testing/selftests/bpf/progs/test_check_mtu.c b/tools/testing/selftests/bpf/progs/test_check_mtu.c index 2ec1de11a3ae4..7b6b2b342c1de 100644 --- a/tools/testing/selftests/bpf/progs/test_check_mtu.c +++ b/tools/testing/selftests/bpf/progs/test_check_mtu.c @@ -7,6 +7,7 @@ #include #include +#include char _license[] SEC("license") = "GPL"; @@ -288,3 +289,14 @@ int tc_input_len_exceed(struct __sk_buff *ctx) global_bpf_mtu_xdp = mtu_len; return retval; } + +SEC("tc") +int tc_chk_segs_flag(struct __sk_buff *ctx) +{ + __u32 mtu_len = 0; + int err; + + err = bpf_check_mtu(ctx, GLOBAL_USER_IFINDEX, &mtu_len, 0, BPF_MTU_CHK_SEGS); + + return err == -EINVAL ? BPF_OK : BPF_DROP; +} diff --git a/tools/testing/selftests/bpf/progs/test_perf_branches.c b/tools/testing/selftests/bpf/progs/test_perf_branches.c index a1ccc831c882f..05ac9410cd68c 100644 --- a/tools/testing/selftests/bpf/progs/test_perf_branches.c +++ b/tools/testing/selftests/bpf/progs/test_perf_branches.c @@ -8,6 +8,7 @@ #include int valid = 0; +int run_cnt = 0; int required_size_out = 0; int written_stack_out = 0; int written_global_out = 0; @@ -24,6 +25,8 @@ int perf_branches(void *ctx) __u64 entries[4 * 3] = {0}; int required_size, written_stack, written_global; + ++run_cnt; + /* write to stack */ written_stack = bpf_read_branch_records(ctx, entries, sizeof(entries), 0); /* ignore spurious events */ diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index 0a72e0228ea9a..e772ae430915f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1709,4 +1709,141 @@ __naked void jeq_disagreeing_tnums(void *ctx) : __clobber_all); } +/* This test covers the bounds deduction when the u64 range and the tnum + * overlap only at umax. After instruction 3, the ranges look as follows: + * + * 0 umin=0xe01 umax=0xf00 U64_MAX + * | [xxxxxxxxxxxxxx] | + * |----------------------------|------------------------------| + * | x x | tnum values + * + * The verifier can therefore deduce that the R0=0xf0=240. + */ +SEC("socket") +__description("bounds refinement with single-value tnum on umax") +__msg("3: (15) if r0 == 0xe0 {{.*}} R0=240") +__success __log_level(2) +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void bounds_refinement_tnum_umax(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + r0 |= 0xe0; \ + r0 &= 0xf0; \ + if r0 == 0xe0 goto +2; \ + if r0 == 0xf0 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test covers the bounds deduction when the u64 range and the tnum + * overlap only at umin. After instruction 3, the ranges look as follows: + * + * 0 umin=0xe00 umax=0xeff U64_MAX + * | [xxxxxxxxxxxxxx] | + * |----------------------------|------------------------------| + * | x x | tnum values + * + * The verifier can therefore deduce that the R0=0xe0=224. + */ +SEC("socket") +__description("bounds refinement with single-value tnum on umin") +__msg("3: (15) if r0 == 0xf0 {{.*}} R0=224") +__success __log_level(2) +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void bounds_refinement_tnum_umin(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + r0 |= 0xe0; \ + r0 &= 0xf0; \ + if r0 == 0xf0 goto +2; \ + if r0 == 0xe0 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test covers the bounds deduction when the only possible tnum value is + * in the middle of the u64 range. After instruction 3, the ranges look as + * follows: + * + * 0 umin=0x7cf umax=0x7df U64_MAX + * | [xxxxxxxxxxxx] | + * |----------------------------|------------------------------| + * | x x x x x | tnum values + * | +--- 0x7e0 + * +--- 0x7d0 + * + * Since the lower four bits are zero, the tnum and the u64 range only overlap + * in R0=0x7d0=2000. Instruction 5 is therefore dead code. + */ +SEC("socket") +__description("bounds refinement with single-value tnum in middle of range") +__msg("3: (a5) if r0 < 0x7cf {{.*}} R0=2000") +__success __log_level(2) +__naked void bounds_refinement_tnum_middle(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + if r0 & 0x0f goto +4; \ + if r0 > 0x7df goto +3; \ + if r0 < 0x7cf goto +2; \ + if r0 == 0x7d0 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test cover the negative case for the tnum/u64 overlap. Since + * they contain the same two values (i.e., {0, 1}), we can't deduce + * anything more. + */ +SEC("socket") +__description("bounds refinement: several overlaps between tnum and u64") +__msg("2: (25) if r0 > 0x1 {{.*}} R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=1,var_off=(0x0; 0x1))") +__failure __log_level(2) +__naked void bounds_refinement_several_overlaps(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + if r0 < 0 goto +3; \ + if r0 > 1 goto +2; \ + if r0 == 1 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* This test cover the negative case for the tnum/u64 overlap. Since + * they overlap in the two values contained by the u64 range (i.e., + * {0xf, 0x10}), we can't deduce anything more. + */ +SEC("socket") +__description("bounds refinement: multiple overlaps between tnum and u64") +__msg("2: (25) if r0 > 0x10 {{.*}} R0=scalar(smin=umin=smin32=umin32=15,smax=umax=smax32=umax32=16,var_off=(0x0; 0x1f))") +__failure __log_level(2) +__naked void bounds_refinement_multiple_overlaps(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + if r0 < 0xf goto +3; \ + if r0 > 0x10 goto +2; \ + if r0 == 0x10 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index c0ce690ddb68a..1fdd85b4b8443 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -40,6 +40,9 @@ __naked void linked_regs_bpf_k(void) */ "r3 = r10;" "r3 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -73,6 +76,9 @@ __naked void linked_regs_bpf_x_src(void) */ "r4 = r10;" "r4 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -106,6 +112,10 @@ __naked void linked_regs_bpf_x_dst(void) */ "r4 = r10;" "r4 += r3;" + /* Mark r1 and r2 as alive. */ + "r0 = r0;" + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -143,6 +153,9 @@ __naked void linked_regs_broken_link(void) */ "r3 = r10;" "r3 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -156,16 +169,16 @@ __naked void linked_regs_broken_link(void) */ SEC("socket") __success __log_level(2) -__msg("12: (0f) r2 += r1") +__msg("17: (0f) r2 += r1") /* Current state */ -__msg("frame2: last_idx 12 first_idx 11 subseq_idx -1 ") -__msg("frame2: regs=r1 stack= before 11: (bf) r2 = r10") +__msg("frame2: last_idx 17 first_idx 14 subseq_idx -1 ") +__msg("frame2: regs=r1 stack= before 16: (bf) r2 = r10") __msg("frame2: parent state regs=r1 stack=") __msg("frame1: parent state regs= stack=") __msg("frame0: parent state regs= stack=") /* Parent state */ -__msg("frame2: last_idx 10 first_idx 10 subseq_idx 11 ") -__msg("frame2: regs=r1 stack= before 10: (25) if r1 > 0x7 goto pc+0") +__msg("frame2: last_idx 13 first_idx 13 subseq_idx 14 ") +__msg("frame2: regs=r1 stack= before 13: (25) if r1 > 0x7 goto pc+0") __msg("frame2: parent state regs=r1 stack=") /* frame1.r{6,7} are marked because mark_precise_scalar_ids() * looks for all registers with frame2.r1.id in the current state @@ -173,20 +186,20 @@ __msg("frame2: parent state regs=r1 stack=") __msg("frame1: parent state regs=r6,r7 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame2: last_idx 8 first_idx 8 subseq_idx 10") -__msg("frame2: regs=r1 stack= before 8: (85) call pc+1") +__msg("frame2: last_idx 9 first_idx 9 subseq_idx 13") +__msg("frame2: regs=r1 stack= before 9: (85) call pc+3") /* frame1.r1 is marked because of backtracking of call instruction */ __msg("frame1: parent state regs=r1,r6,r7 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame1: last_idx 7 first_idx 6 subseq_idx 8") -__msg("frame1: regs=r1,r6,r7 stack= before 7: (bf) r7 = r1") -__msg("frame1: regs=r1,r6 stack= before 6: (bf) r6 = r1") +__msg("frame1: last_idx 8 first_idx 7 subseq_idx 9") +__msg("frame1: regs=r1,r6,r7 stack= before 8: (bf) r7 = r1") +__msg("frame1: regs=r1,r6 stack= before 7: (bf) r6 = r1") __msg("frame1: parent state regs=r1 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame1: last_idx 4 first_idx 4 subseq_idx 6") -__msg("frame1: regs=r1 stack= before 4: (85) call pc+1") +__msg("frame1: last_idx 4 first_idx 4 subseq_idx 7") +__msg("frame1: regs=r1 stack= before 4: (85) call pc+2") __msg("frame0: parent state regs=r1,r6 stack=") /* Parent state */ __msg("frame0: last_idx 3 first_idx 1 subseq_idx 4") @@ -204,6 +217,7 @@ __naked void precision_many_frames(void) "r1 = r0;" "r6 = r0;" "call precision_many_frames__foo;" + "r6 = r6;" /* mark r6 as live */ "exit;" : : __imm(bpf_ktime_get_ns) @@ -220,6 +234,8 @@ void precision_many_frames__foo(void) "r6 = r1;" "r7 = r1;" "call precision_many_frames__bar;" + "r6 = r6;" /* mark r6 as live */ + "r7 = r7;" /* mark r7 as live */ "exit" ::: __clobber_all); } @@ -229,6 +245,8 @@ void precision_many_frames__bar(void) { asm volatile ( "if r1 > 7 goto +0;" + "r6 = 0;" /* mark r6 as live */ + "r7 = 0;" /* mark r7 as live */ /* force r1 to be precise, this eventually marks: * - bar frame r1 * - foo frame r{1,6,7} @@ -340,6 +358,8 @@ __naked void precision_two_ids(void) "r3 += r7;" /* force r9 to be precise, this also marks r8 */ "r3 += r9;" + "r6 = r6;" /* mark r6 as live */ + "r8 = r8;" /* mark r8 as live */ "exit;" : : __imm(bpf_ktime_get_ns) @@ -353,7 +373,7 @@ __flag(BPF_F_TEST_STATE_FREQ) * collect_linked_regs() can't tie more than 6 registers for a single insn. */ __msg("8: (25) if r0 > 0x7 goto pc+0 ; R0=scalar(id=1") -__msg("9: (bf) r6 = r6 ; R6=scalar(id=2") +__msg("14: (bf) r6 = r6 ; R6=scalar(id=2") /* check that r{0-5} are marked precise after 'if' */ __msg("frame0: regs=r0 stack= before 8: (25) if r0 > 0x7 goto pc+0") __msg("frame0: parent state regs=r0,r1,r2,r3,r4,r5 stack=:") @@ -372,6 +392,12 @@ __naked void linked_regs_too_many_regs(void) "r6 = r0;" /* propagate range for r{0-6} */ "if r0 > 7 goto +0;" + /* keep r{1-5} live */ + "r1 = r1;" + "r2 = r2;" + "r3 = r3;" + "r4 = r4;" + "r5 = r5;" /* make r6 appear in the log */ "r6 = r6;" /* force r0 to be precise, @@ -517,7 +543,7 @@ __naked void check_ids_in_regsafe_2(void) "*(u64*)(r10 - 8) = r1;" /* r9 = pointer to stack */ "r9 = r10;" - "r9 += -8;" + "r9 += -16;" /* r8 = ktime_get_ns() */ "call %[bpf_ktime_get_ns];" "r8 = r0;" @@ -538,6 +564,8 @@ __naked void check_ids_in_regsafe_2(void) "if r7 > 4 goto l2_%=;" /* Access memory at r9[r6] */ "r9 += r6;" + "r9 += r7;" + "r9 += r8;" "r0 = *(u8*)(r9 + 0);" "l2_%=:" "r0 = 0;" diff --git a/tools/testing/selftests/bpf/test_tag.c b/tools/testing/selftests/bpf/test_tag.c index 5546b05a04866..f1300047c1e0a 100644 --- a/tools/testing/selftests/bpf/test_tag.c +++ b/tools/testing/selftests/bpf/test_tag.c @@ -116,7 +116,7 @@ static void tag_from_alg(int insns, uint8_t *tag, uint32_t len) static const struct sockaddr_alg alg = { .salg_family = AF_ALG, .salg_type = "hash", - .salg_name = "sha1", + .salg_name = "sha256", }; int fd_base, fd_alg, ret; ssize_t size; diff --git a/tools/testing/selftests/bpf/test_xsk.c b/tools/testing/selftests/bpf/test_xsk.c new file mode 100644 index 0000000000000..55d318c5c5e59 --- /dev/null +++ b/tools/testing/selftests/bpf/test_xsk.c @@ -0,0 +1,2422 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "network_helpers.h" +#include "test_xsk.h" +#include "xsk_xdp_common.h" +#include "xsk_xdp_progs.skel.h" + +#define DEFAULT_BATCH_SIZE 64 +#define MIN_PKT_SIZE 64 +#define MAX_ETH_JUMBO_SIZE 9000 +#define MAX_INTERFACES 2 +#define MAX_TEARDOWN_ITER 10 +#define MAX_TX_BUDGET_DEFAULT 32 +#define PKT_DUMP_NB_TO_PRINT 16 +/* Just to align the data in the packet */ +#define PKT_HDR_SIZE (sizeof(struct ethhdr) + 2) +#define POLL_TMOUT 1000 +#define THREAD_TMOUT 3 +#define UMEM_HEADROOM_TEST_SIZE 128 +#define XSK_DESC__INVALID_OPTION (0xffff) +#define XSK_UMEM__INVALID_FRAME_SIZE (MAX_ETH_JUMBO_SIZE + 1) +#define XSK_UMEM__LARGE_FRAME_SIZE (3 * 1024) +#define XSK_UMEM__MAX_FRAME_SIZE (4 * 1024) + +static const u8 g_mac[ETH_ALEN] = {0x55, 0x44, 0x33, 0x22, 0x11, 0x00}; + +bool opt_verbose; +pthread_barrier_t barr; +pthread_mutex_t pacing_mutex = PTHREAD_MUTEX_INITIALIZER; + +int pkts_in_flight; + +/* The payload is a word consisting of a packet sequence number in the upper + * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's + * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. + */ +static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) +{ + u32 *ptr = (u32 *)dest, i; + + start /= sizeof(*ptr); + size /= sizeof(*ptr); + for (i = 0; i < size; i++) + ptr[i] = htonl(pkt_nb << 16 | (i + start)); +} + +static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) +{ + memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); + memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); + eth_hdr->h_proto = htons(ETH_P_LOOPBACK); +} + +static bool is_umem_valid(struct ifobject *ifobj) +{ + return !!ifobj->umem->umem; +} + +static u32 mode_to_xdp_flags(enum test_mode mode) +{ + return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; +} + +static u64 umem_size(struct xsk_umem_info *umem) +{ + return umem->num_frames * umem->frame_size; +} + +int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, + u64 size) +{ + struct xsk_umem_config cfg = { + .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, + .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, + .frame_size = umem->frame_size, + .frame_headroom = umem->frame_headroom, + .flags = XSK_UMEM__DEFAULT_FLAGS + }; + int ret; + + if (umem->fill_size) + cfg.fill_size = umem->fill_size; + + if (umem->comp_size) + cfg.comp_size = umem->comp_size; + + if (umem->unaligned_mode) + cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; + + ret = xsk_umem__create(&umem->umem, buffer, size, + &umem->fq, &umem->cq, &cfg); + if (ret) + return ret; + + umem->buffer = buffer; + if (ifobj->shared_umem && ifobj->rx_on) { + umem->base_addr = umem_size(umem); + umem->next_buffer = umem_size(umem); + } + + return 0; +} + +static u64 umem_alloc_buffer(struct xsk_umem_info *umem) +{ + u64 addr; + + addr = umem->next_buffer; + umem->next_buffer += umem->frame_size; + if (umem->next_buffer >= umem->base_addr + umem_size(umem)) + umem->next_buffer = umem->base_addr; + + return addr; +} + +static void umem_reset_alloc(struct xsk_umem_info *umem) +{ + umem->next_buffer = 0; +} + +static void enable_busy_poll(struct xsk_socket_info *xsk) +{ + int sock_opt; + + sock_opt = 1; + if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, + (void *)&sock_opt, sizeof(sock_opt)) < 0) + exit_with_error(errno); + + sock_opt = 20; + if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, + (void *)&sock_opt, sizeof(sock_opt)) < 0) + exit_with_error(errno); + + sock_opt = xsk->batch_size; + if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, + (void *)&sock_opt, sizeof(sock_opt)) < 0) + exit_with_error(errno); +} + +int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, + struct ifobject *ifobject, bool shared) +{ + struct xsk_socket_config cfg = {}; + struct xsk_ring_cons *rxr; + struct xsk_ring_prod *txr; + + xsk->umem = umem; + cfg.rx_size = xsk->rxqsize; + cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; + cfg.bind_flags = ifobject->bind_flags; + if (shared) + cfg.bind_flags |= XDP_SHARED_UMEM; + if (ifobject->mtu > MAX_ETH_PKT_SIZE) + cfg.bind_flags |= XDP_USE_SG; + if (umem->comp_size) + cfg.tx_size = umem->comp_size; + if (umem->fill_size) + cfg.rx_size = umem->fill_size; + + txr = ifobject->tx_on ? &xsk->tx : NULL; + rxr = ifobject->rx_on ? &xsk->rx : NULL; + return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); +} + +#define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" +static unsigned int get_max_skb_frags(void) +{ + unsigned int max_skb_frags = 0; + FILE *file; + + file = fopen(MAX_SKB_FRAGS_PATH, "r"); + if (!file) { + ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH); + return 0; + } + + if (fscanf(file, "%u", &max_skb_frags) != 1) + ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH); + + fclose(file); + return max_skb_frags; +} + +static int set_ring_size(struct ifobject *ifobj) +{ + int ret; + u32 ctr = 0; + + while (ctr++ < SOCK_RECONF_CTR) { + ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring); + if (!ret) + break; + + /* Retry if it fails */ + if (ctr >= SOCK_RECONF_CTR || errno != EBUSY) + return -errno; + + usleep(USLEEP_MAX); + } + + return ret; +} + +int hw_ring_size_reset(struct ifobject *ifobj) +{ + ifobj->ring.tx_pending = ifobj->set_ring.default_tx; + ifobj->ring.rx_pending = ifobj->set_ring.default_rx; + return set_ring_size(ifobj); +} + +static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, + struct ifobject *ifobj_rx) +{ + u32 i, j; + + for (i = 0; i < MAX_INTERFACES; i++) { + struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + + ifobj->xsk = &ifobj->xsk_arr[0]; + ifobj->use_poll = false; + ifobj->use_fill_ring = true; + ifobj->release_rx = true; + ifobj->validation_func = NULL; + ifobj->use_metadata = false; + + if (i == 0) { + ifobj->rx_on = false; + ifobj->tx_on = true; + } else { + ifobj->rx_on = true; + ifobj->tx_on = false; + } + + memset(ifobj->umem, 0, sizeof(*ifobj->umem)); + ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; + ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; + + for (j = 0; j < MAX_SOCKETS; j++) { + memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); + ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; + ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; + if (i == 0) + ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; + else + ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; + + memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); + memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); + ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); + ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); + } + } + + if (ifobj_tx->hw_ring_size_supp) + hw_ring_size_reset(ifobj_tx); + + test->ifobj_tx = ifobj_tx; + test->ifobj_rx = ifobj_rx; + test->current_step = 0; + test->total_steps = 1; + test->nb_sockets = 1; + test->fail = false; + test->set_ring = false; + test->adjust_tail = false; + test->adjust_tail_support = false; + test->mtu = MAX_ETH_PKT_SIZE; + test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; + test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; + test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; + test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; +} + +void test_init(struct test_spec *test, struct ifobject *ifobj_tx, + struct ifobject *ifobj_rx, enum test_mode mode, + const struct test_spec *test_to_run) +{ + struct pkt_stream *tx_pkt_stream; + struct pkt_stream *rx_pkt_stream; + u32 i; + + tx_pkt_stream = test->tx_pkt_stream_default; + rx_pkt_stream = test->rx_pkt_stream_default; + memset(test, 0, sizeof(*test)); + test->tx_pkt_stream_default = tx_pkt_stream; + test->rx_pkt_stream_default = rx_pkt_stream; + + for (i = 0; i < MAX_INTERFACES; i++) { + struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + + ifobj->bind_flags = XDP_USE_NEED_WAKEUP; + if (mode == TEST_MODE_ZC) + ifobj->bind_flags |= XDP_ZEROCOPY; + else + ifobj->bind_flags |= XDP_COPY; + } + + memcpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); + test->test_func = test_to_run->test_func; + test->mode = mode; + __test_spec_init(test, ifobj_tx, ifobj_rx); +} + +static void test_spec_reset(struct test_spec *test) +{ + __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); +} + +static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, + struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, + struct bpf_map *xskmap_tx) +{ + test->xdp_prog_rx = xdp_prog_rx; + test->xdp_prog_tx = xdp_prog_tx; + test->xskmap_rx = xskmap_rx; + test->xskmap_tx = xskmap_tx; +} + +static int test_spec_set_mtu(struct test_spec *test, int mtu) +{ + int err; + + if (test->ifobj_rx->mtu != mtu) { + err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); + if (err) + return err; + test->ifobj_rx->mtu = mtu; + } + if (test->ifobj_tx->mtu != mtu) { + err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); + if (err) + return err; + test->ifobj_tx->mtu = mtu; + } + + return 0; +} + +void pkt_stream_reset(struct pkt_stream *pkt_stream) +{ + if (pkt_stream) { + pkt_stream->current_pkt_nb = 0; + pkt_stream->nb_rx_pkts = 0; + } +} + +static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) +{ + if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) + return NULL; + + return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; +} + +static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) +{ + while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { + (*pkts_sent)++; + if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) + return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; + pkt_stream->current_pkt_nb++; + } + return NULL; +} + +void pkt_stream_delete(struct pkt_stream *pkt_stream) +{ + free(pkt_stream->pkts); + free(pkt_stream); +} + +void pkt_stream_restore_default(struct test_spec *test) +{ + struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; + struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; + + if (tx_pkt_stream != test->tx_pkt_stream_default) { + pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream); + test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; + } + + if (rx_pkt_stream != test->rx_pkt_stream_default) { + pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream); + test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; + } +} + +static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) +{ + struct pkt_stream *pkt_stream; + + pkt_stream = calloc(1, sizeof(*pkt_stream)); + if (!pkt_stream) + return NULL; + + pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); + if (!pkt_stream->pkts) { + free(pkt_stream); + return NULL; + } + + pkt_stream->nb_pkts = nb_pkts; + return pkt_stream; +} + +static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) +{ + u32 nb_frags = 1, next_frag; + + if (!pkt) + return 1; + + if (!pkt_stream->verbatim) { + if (!pkt->valid || !pkt->len) + return 1; + return ceil_u32(pkt->len, frame_size); + } + + /* Search for the end of the packet in verbatim mode */ + if (!pkt_continues(pkt->options) || !pkt->valid) + return nb_frags; + + next_frag = pkt_stream->current_pkt_nb; + pkt++; + while (next_frag++ < pkt_stream->nb_pkts) { + nb_frags++; + if (!pkt_continues(pkt->options) || !pkt->valid) + break; + pkt++; + } + return nb_frags; +} + +static bool set_pkt_valid(int offset, u32 len) +{ + return len <= MAX_ETH_JUMBO_SIZE; +} + +static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) +{ + pkt->offset = offset; + pkt->len = len; + pkt->valid = set_pkt_valid(offset, len); +} + +static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) +{ + bool prev_pkt_valid = pkt->valid; + + pkt_set(pkt_stream, pkt, offset, len); + pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; +} + +static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) +{ + return ceil_u32(len, umem->frame_size) * umem->frame_size; +} + +static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) +{ + struct pkt_stream *pkt_stream; + u32 i; + + pkt_stream = __pkt_stream_alloc(nb_pkts); + if (!pkt_stream) + exit_with_error(ENOMEM); + + pkt_stream->nb_pkts = nb_pkts; + pkt_stream->max_pkt_len = pkt_len; + for (i = 0; i < nb_pkts; i++) { + struct pkt *pkt = &pkt_stream->pkts[i]; + + pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); + pkt->pkt_nb = nb_start + i * nb_off; + } + + return pkt_stream; +} + +struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) +{ + return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1); +} + +static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) +{ + return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len); +} + +static void pkt_stream_replace_ifobject(struct ifobject *ifobj, u32 nb_pkts, u32 pkt_len) +{ + ifobj->xsk->pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); +} + +static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) +{ + pkt_stream_replace_ifobject(test->ifobj_tx, nb_pkts, pkt_len); + pkt_stream_replace_ifobject(test->ifobj_rx, nb_pkts, pkt_len); +} + +static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, + int offset) +{ + struct pkt_stream *pkt_stream; + u32 i; + + pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); + for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) + pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); + + ifobj->xsk->pkt_stream = pkt_stream; +} + +static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) +{ + __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); + __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); +} + +static void pkt_stream_receive_half(struct test_spec *test) +{ + struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; + u32 i; + + test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts, + pkt_stream->pkts[0].len); + pkt_stream = test->ifobj_rx->xsk->pkt_stream; + for (i = 1; i < pkt_stream->nb_pkts; i += 2) + pkt_stream->pkts[i].valid = false; + + pkt_stream->nb_valid_entries /= 2; +} + +static void pkt_stream_even_odd_sequence(struct test_spec *test) +{ + struct pkt_stream *pkt_stream; + u32 i; + + for (i = 0; i < test->nb_sockets; i++) { + pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; + pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, + pkt_stream->pkts[0].len, i, 2); + test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; + + pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; + pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, + pkt_stream->pkts[0].len, i, 2); + test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; + } +} + +static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) +{ + if (!pkt->valid) + return pkt->offset; + return pkt->offset + umem_alloc_buffer(umem); +} + +static void pkt_stream_cancel(struct pkt_stream *pkt_stream) +{ + pkt_stream->current_pkt_nb--; +} + +static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, + u32 pkt_nb, u32 bytes_written) +{ + void *data = xsk_umem__get_data(umem->buffer, addr); + + if (len < MIN_PKT_SIZE) + return; + + if (!bytes_written) { + gen_eth_hdr(xsk, data); + + len -= PKT_HDR_SIZE; + data += PKT_HDR_SIZE; + } else { + bytes_written -= PKT_HDR_SIZE; + } + + write_payload(data, pkt_nb, bytes_written, len); +} + +static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, + u32 nb_frames, bool verbatim) +{ + u32 i, len = 0, pkt_nb = 0, payload = 0; + struct pkt_stream *pkt_stream; + + pkt_stream = __pkt_stream_alloc(nb_frames); + if (!pkt_stream) + exit_with_error(ENOMEM); + + for (i = 0; i < nb_frames; i++) { + struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; + struct pkt *frame = &frames[i]; + + pkt->offset = frame->offset; + if (verbatim) { + *pkt = *frame; + pkt->pkt_nb = payload; + if (!frame->valid || !pkt_continues(frame->options)) + payload++; + } else { + if (frame->valid) + len += frame->len; + if (frame->valid && pkt_continues(frame->options)) + continue; + + pkt->pkt_nb = pkt_nb; + pkt->len = len; + pkt->valid = frame->valid; + pkt->options = 0; + + len = 0; + } + + print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", + pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); + + if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) + pkt_stream->max_pkt_len = pkt->len; + + if (pkt->valid) + pkt_stream->nb_valid_entries++; + + pkt_nb++; + } + + pkt_stream->nb_pkts = pkt_nb; + pkt_stream->verbatim = verbatim; + return pkt_stream; +} + +static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) +{ + struct pkt_stream *pkt_stream; + + pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); + test->ifobj_tx->xsk->pkt_stream = pkt_stream; + + pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); + test->ifobj_rx->xsk->pkt_stream = pkt_stream; +} + +static void pkt_print_data(u32 *data, u32 cnt) +{ + u32 i; + + for (i = 0; i < cnt; i++) { + u32 seqnum, pkt_nb; + + seqnum = ntohl(*data) & 0xffff; + pkt_nb = ntohl(*data) >> 16; + ksft_print_msg("%u:%u ", pkt_nb, seqnum); + data++; + } +} + +static void pkt_dump(void *pkt, u32 len, bool eth_header) +{ + struct ethhdr *ethhdr = pkt; + u32 i, *data; + + if (eth_header) { + /*extract L2 frame */ + ksft_print_msg("DEBUG>> L2: dst mac: "); + for (i = 0; i < ETH_ALEN; i++) + ksft_print_msg("%02X", ethhdr->h_dest[i]); + + ksft_print_msg("\nDEBUG>> L2: src mac: "); + for (i = 0; i < ETH_ALEN; i++) + ksft_print_msg("%02X", ethhdr->h_source[i]); + + data = pkt + PKT_HDR_SIZE; + } else { + data = pkt; + } + + /*extract L5 frame */ + ksft_print_msg("\nDEBUG>> L5: seqnum: "); + pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); + ksft_print_msg("...."); + if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { + ksft_print_msg("\n.... "); + pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, + PKT_DUMP_NB_TO_PRINT); + } + ksft_print_msg("\n---------------------------------------\n"); +} + +static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) +{ + u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; + u32 offset = addr % umem->frame_size, expected_offset; + int pkt_offset = pkt->valid ? pkt->offset : 0; + + if (!umem->unaligned_mode) + pkt_offset = 0; + + expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; + + if (offset == expected_offset) + return true; + + ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); + return false; +} + +static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) +{ + void *data = xsk_umem__get_data(buffer, addr); + struct xdp_info *meta = data - sizeof(struct xdp_info); + + if (meta->count != pkt->pkt_nb) { + ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n", + __func__, pkt->pkt_nb, + (unsigned long long)meta->count); + return false; + } + + return true; +} + +static bool is_adjust_tail_supported(struct xsk_xdp_progs *skel_rx) +{ + struct bpf_map *data_map; + int adjust_value = 0; + int key = 0; + int ret; + + data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); + if (!data_map || !bpf_map__is_internal(data_map)) { + ksft_print_msg("Error: could not find bss section of XDP program\n"); + exit_with_error(errno); + } + + ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &key, &adjust_value); + if (ret) { + ksft_print_msg("Error: bpf_map_lookup_elem failed with error %d\n", ret); + exit_with_error(errno); + } + + /* Set the 'adjust_value' variable to -EOPNOTSUPP in the XDP program if the adjust_tail + * helper is not supported. Skip the adjust_tail test case in this scenario. + */ + return adjust_value != -EOPNOTSUPP; +} + +static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, + u32 bytes_processed) +{ + u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; + void *data = xsk_umem__get_data(umem->buffer, addr); + + addr -= umem->base_addr; + + if (addr >= umem->num_frames * umem->frame_size || + addr + len > umem->num_frames * umem->frame_size) { + ksft_print_msg("Frag invalid addr: %llx len: %u\n", + (unsigned long long)addr, len); + return false; + } + if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { + ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", + (unsigned long long)addr, len); + return false; + } + + pkt_data = data; + if (!bytes_processed) { + pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); + len -= PKT_HDR_SIZE; + } else { + bytes_processed -= PKT_HDR_SIZE; + } + + expected_seqnum = bytes_processed / sizeof(*pkt_data); + seqnum = ntohl(*pkt_data) & 0xffff; + pkt_nb = ntohl(*pkt_data) >> 16; + + if (expected_pkt_nb != pkt_nb) { + ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", + __func__, expected_pkt_nb, pkt_nb); + goto error; + } + if (expected_seqnum != seqnum) { + ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", + __func__, expected_seqnum, seqnum); + goto error; + } + + words_to_end = len / sizeof(*pkt_data) - 1; + pkt_data += words_to_end; + seqnum = ntohl(*pkt_data) & 0xffff; + expected_seqnum += words_to_end; + if (expected_seqnum != seqnum) { + ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", + __func__, expected_seqnum, seqnum); + goto error; + } + + return true; + +error: + pkt_dump(data, len, !bytes_processed); + return false; +} + +static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) +{ + if (pkt->len != len) { + ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", + __func__, pkt->len, len); + pkt_dump(xsk_umem__get_data(buffer, addr), len, true); + return false; + } + + return true; +} + +static u32 load_value(u32 *counter) +{ + return __atomic_load_n(counter, __ATOMIC_ACQUIRE); +} + +static bool kick_tx_with_check(struct xsk_socket_info *xsk, int *ret) +{ + u32 max_budget = MAX_TX_BUDGET_DEFAULT; + u32 cons, ready_to_send; + int delta; + + cons = load_value(xsk->tx.consumer); + ready_to_send = load_value(xsk->tx.producer) - cons; + *ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); + + delta = load_value(xsk->tx.consumer) - cons; + /* By default, xsk should consume exact @max_budget descs at one + * send in this case where hitting the max budget limit in while + * loop is triggered in __xsk_generic_xmit(). Please make sure that + * the number of descs to be sent is larger than @max_budget, or + * else the tx.consumer will be updated in xskq_cons_peek_desc() + * in time which hides the issue we try to verify. + */ + if (ready_to_send > max_budget && delta != max_budget) + return false; + + return true; +} + +int kick_tx(struct xsk_socket_info *xsk) +{ + int ret; + + if (xsk->check_consumer) { + if (!kick_tx_with_check(xsk, &ret)) + return TEST_FAILURE; + } else { + ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); + } + if (ret >= 0) + return TEST_PASS; + if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { + usleep(100); + return TEST_PASS; + } + return TEST_FAILURE; +} + +int kick_rx(struct xsk_socket_info *xsk) +{ + int ret; + + ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); + if (ret < 0) + return TEST_FAILURE; + + return TEST_PASS; +} + +static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) +{ + unsigned int rcvd; + u32 idx; + int ret; + + if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { + ret = kick_tx(xsk); + if (ret) + return TEST_FAILURE; + } + + rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); + if (rcvd) { + if (rcvd > xsk->outstanding_tx) { + u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); + + ksft_print_msg("[%s] Too many packets completed\n", __func__); + ksft_print_msg("Last completion address: %llx\n", + (unsigned long long)addr); + return TEST_FAILURE; + } + + xsk_ring_cons__release(&xsk->umem->cq, rcvd); + xsk->outstanding_tx -= rcvd; + } + + return TEST_PASS; +} + +static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) +{ + u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; + u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; + struct pkt_stream *pkt_stream = xsk->pkt_stream; + struct ifobject *ifobj = test->ifobj_rx; + struct xsk_umem_info *umem = xsk->umem; + struct pollfd fds = { }; + struct pkt *pkt; + u64 first_addr = 0; + int ret; + + fds.fd = xsk_socket__fd(xsk->xsk); + fds.events = POLLIN; + + ret = kick_rx(xsk); + if (ret) + return TEST_FAILURE; + + if (ifobj->use_poll) { + ret = poll(&fds, 1, POLL_TMOUT); + if (ret < 0) + return TEST_FAILURE; + + if (!ret) { + if (!is_umem_valid(test->ifobj_tx)) + return TEST_PASS; + + ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); + return TEST_CONTINUE; + } + + if (!(fds.revents & POLLIN)) + return TEST_CONTINUE; + } + + rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx); + if (!rcvd) + return TEST_CONTINUE; + + if (ifobj->use_fill_ring) { + ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); + while (ret != rcvd) { + if (xsk_ring_prod__needs_wakeup(&umem->fq)) { + ret = poll(&fds, 1, POLL_TMOUT); + if (ret < 0) + return TEST_FAILURE; + } + ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); + } + } + + while (frags_processed < rcvd) { + const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); + u64 addr = desc->addr, orig; + + orig = xsk_umem__extract_addr(addr); + addr = xsk_umem__add_offset_to_addr(addr); + + if (!nb_frags) { + pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); + if (!pkt) { + ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", + __func__, addr, desc->len); + return TEST_FAILURE; + } + } + + print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", + addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); + + if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || + !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && + !is_metadata_correct(pkt, umem->buffer, addr))) + return TEST_FAILURE; + + if (!nb_frags++) + first_addr = addr; + frags_processed++; + pkt_len += desc->len; + if (ifobj->use_fill_ring) + *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; + + if (pkt_continues(desc->options)) + continue; + + /* The complete packet has been received */ + if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || + !is_offset_correct(umem, pkt, addr)) + return TEST_FAILURE; + + pkt_stream->nb_rx_pkts++; + nb_frags = 0; + pkt_len = 0; + } + + if (nb_frags) { + /* In the middle of a packet. Start over from beginning of packet. */ + idx_rx -= nb_frags; + xsk_ring_cons__cancel(&xsk->rx, nb_frags); + if (ifobj->use_fill_ring) { + idx_fq -= nb_frags; + xsk_ring_prod__cancel(&umem->fq, nb_frags); + } + frags_processed -= nb_frags; + pkt_stream_cancel(pkt_stream); + pkts_sent--; + } + + if (ifobj->use_fill_ring) + xsk_ring_prod__submit(&umem->fq, frags_processed); + if (ifobj->release_rx) + xsk_ring_cons__release(&xsk->rx, frags_processed); + + pthread_mutex_lock(&pacing_mutex); + pkts_in_flight -= pkts_sent; + pthread_mutex_unlock(&pacing_mutex); + pkts_sent = 0; + + return TEST_CONTINUE; +} + +bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, + unsigned long *bitmap) +{ + struct pkt_stream *pkt_stream = xsk->pkt_stream; + + if (!pkt_stream) { + __set_bit(sock_num, bitmap); + return false; + } + + if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { + __set_bit(sock_num, bitmap); + if (bitmap_full(bitmap, test->nb_sockets)) + return true; + } + + return false; +} + +static int receive_pkts(struct test_spec *test) +{ + struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; + DECLARE_BITMAP(bitmap, test->nb_sockets); + struct xsk_socket_info *xsk; + u32 sock_num = 0; + int res, ret; + + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + + timeradd(&tv_now, &tv_timeout, &tv_end); + + while (1) { + xsk = &test->ifobj_rx->xsk_arr[sock_num]; + + if ((all_packets_received(test, xsk, sock_num, bitmap))) + break; + + res = __receive_pkts(test, xsk); + if (!(res == TEST_PASS || res == TEST_CONTINUE)) + return res; + + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + + if (timercmp(&tv_now, &tv_end, >)) { + ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); + return TEST_FAILURE; + } + sock_num = (sock_num + 1) % test->nb_sockets; + } + + return TEST_PASS; +} + +static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) +{ + u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; + struct pkt_stream *pkt_stream = xsk->pkt_stream; + struct xsk_umem_info *umem = ifobject->umem; + bool use_poll = ifobject->use_poll; + struct pollfd fds = { }; + int ret; + + buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); + /* pkts_in_flight might be negative if many invalid packets are sent */ + if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / + buffer_len)) { + ret = kick_tx(xsk); + if (ret) + return TEST_FAILURE; + return TEST_CONTINUE; + } + + fds.fd = xsk_socket__fd(xsk->xsk); + fds.events = POLLOUT; + + while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { + if (use_poll) { + ret = poll(&fds, 1, POLL_TMOUT); + if (timeout) { + if (ret < 0) { + ksft_print_msg("ERROR: [%s] Poll error %d\n", + __func__, errno); + return TEST_FAILURE; + } + if (ret == 0) + return TEST_PASS; + break; + } + if (ret <= 0) { + ksft_print_msg("ERROR: [%s] Poll error %d\n", + __func__, errno); + return TEST_FAILURE; + } + } + + complete_pkts(xsk, xsk->batch_size); + } + + for (i = 0; i < xsk->batch_size; i++) { + struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); + u32 nb_frags_left, nb_frags, bytes_written = 0; + + if (!pkt) + break; + + nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); + if (nb_frags > xsk->batch_size - i) { + pkt_stream_cancel(pkt_stream); + xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i); + break; + } + nb_frags_left = nb_frags; + + while (nb_frags_left--) { + struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); + + tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); + if (pkt_stream->verbatim) { + tx_desc->len = pkt->len; + tx_desc->options = pkt->options; + } else if (nb_frags_left) { + tx_desc->len = umem->frame_size; + tx_desc->options = XDP_PKT_CONTD; + } else { + tx_desc->len = pkt->len - bytes_written; + tx_desc->options = 0; + } + if (pkt->valid) + pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb, + bytes_written); + bytes_written += tx_desc->len; + + print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", + tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); + + if (nb_frags_left) { + i++; + if (pkt_stream->verbatim) + pkt = pkt_stream_get_next_tx_pkt(pkt_stream); + } + } + + if (pkt && pkt->valid) { + valid_pkts++; + valid_frags += nb_frags; + } + } + + pthread_mutex_lock(&pacing_mutex); + pkts_in_flight += valid_pkts; + pthread_mutex_unlock(&pacing_mutex); + + xsk_ring_prod__submit(&xsk->tx, i); + xsk->outstanding_tx += valid_frags; + + if (use_poll) { + ret = poll(&fds, 1, POLL_TMOUT); + if (ret <= 0) { + if (ret == 0 && timeout) + return TEST_PASS; + + ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); + return TEST_FAILURE; + } + } + + if (!timeout) { + if (complete_pkts(xsk, i)) + return TEST_FAILURE; + + usleep(10); + return TEST_PASS; + } + + return TEST_CONTINUE; +} + +static int wait_for_tx_completion(struct xsk_socket_info *xsk) +{ + struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; + int ret; + + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + timeradd(&tv_now, &tv_timeout, &tv_end); + + while (xsk->outstanding_tx) { + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + if (timercmp(&tv_now, &tv_end, >)) { + ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); + return TEST_FAILURE; + } + + complete_pkts(xsk, xsk->batch_size); + } + + return TEST_PASS; +} + +bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) +{ + return bitmap_full(bitmap, test->nb_sockets); +} + +static int send_pkts(struct test_spec *test, struct ifobject *ifobject) +{ + bool timeout = !is_umem_valid(test->ifobj_rx); + DECLARE_BITMAP(bitmap, test->nb_sockets); + u32 i, ret; + + while (!(all_packets_sent(test, bitmap))) { + for (i = 0; i < test->nb_sockets; i++) { + struct pkt_stream *pkt_stream; + + pkt_stream = ifobject->xsk_arr[i].pkt_stream; + if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { + __set_bit(i, bitmap); + continue; + } + ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); + if (ret == TEST_CONTINUE && !test->fail) + continue; + + if ((ret || test->fail) && !timeout) + return TEST_FAILURE; + + if (ret == TEST_PASS && timeout) + return ret; + + ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); + if (ret) + return TEST_FAILURE; + } + } + + return TEST_PASS; +} + +static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) +{ + int fd = xsk_socket__fd(xsk), err; + socklen_t optlen, expected_len; + + optlen = sizeof(*stats); + err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); + if (err) { + ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", + __func__, -err, strerror(-err)); + return TEST_FAILURE; + } + + expected_len = sizeof(struct xdp_statistics); + if (optlen != expected_len) { + ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", + __func__, expected_len, optlen); + return TEST_FAILURE; + } + + return TEST_PASS; +} + +static int validate_rx_dropped(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + struct xdp_statistics stats; + int err; + + err = kick_rx(ifobject->xsk); + if (err) + return TEST_FAILURE; + + err = get_xsk_stats(xsk, &stats); + if (err) + return TEST_FAILURE; + + /* The receiver calls getsockopt after receiving the last (valid) + * packet which is not the final packet sent in this test (valid and + * invalid packets are sent in alternating fashion with the final + * packet being invalid). Since the last packet may or may not have + * been dropped already, both outcomes must be allowed. + */ + if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || + stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) + return TEST_PASS; + + return TEST_FAILURE; +} + +static int validate_rx_full(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + struct xdp_statistics stats; + int err; + + usleep(1000); + err = kick_rx(ifobject->xsk); + if (err) + return TEST_FAILURE; + + err = get_xsk_stats(xsk, &stats); + if (err) + return TEST_FAILURE; + + if (stats.rx_ring_full) + return TEST_PASS; + + return TEST_FAILURE; +} + +static int validate_fill_empty(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + struct xdp_statistics stats; + int err; + + usleep(1000); + err = kick_rx(ifobject->xsk); + if (err) + return TEST_FAILURE; + + err = get_xsk_stats(xsk, &stats); + if (err) + return TEST_FAILURE; + + if (stats.rx_fill_ring_empty_descs) + return TEST_PASS; + + return TEST_FAILURE; +} + +static int validate_tx_invalid_descs(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + int fd = xsk_socket__fd(xsk); + struct xdp_statistics stats; + socklen_t optlen; + int err; + + optlen = sizeof(stats); + err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); + if (err) { + ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", + __func__, -err, strerror(-err)); + return TEST_FAILURE; + } + + if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { + ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n", + __func__, + (unsigned long long)stats.tx_invalid_descs, + ifobject->xsk->pkt_stream->nb_pkts); + return TEST_FAILURE; + } + + return TEST_PASS; +} + +static void xsk_configure(struct test_spec *test, struct ifobject *ifobject, + struct xsk_umem_info *umem, bool tx) +{ + int i, ret; + + for (i = 0; i < test->nb_sockets; i++) { + bool shared = (ifobject->shared_umem && tx) ? true : !!i; + u32 ctr = 0; + + while (ctr++ < SOCK_RECONF_CTR) { + ret = xsk_configure_socket(&ifobject->xsk_arr[i], umem, + ifobject, shared); + if (!ret) + break; + + /* Retry if it fails as xsk_socket__create() is asynchronous */ + if (ctr >= SOCK_RECONF_CTR) + exit_with_error(-ret); + usleep(USLEEP_MAX); + } + if (ifobject->busy_poll) + enable_busy_poll(&ifobject->xsk_arr[i]); + } +} + +static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) +{ + xsk_configure(test, ifobject, test->ifobj_rx->umem, true); + ifobject->xsk = &ifobject->xsk_arr[0]; + ifobject->xskmap = test->ifobj_rx->xskmap; + memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); + ifobject->umem->base_addr = 0; +} + +static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, + bool fill_up) +{ + u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; + u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; + int ret; + + if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) + buffers_to_fill = umem->num_frames; + else + buffers_to_fill = umem->fill_size; + + ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); + if (ret != buffers_to_fill) + exit_with_error(ENOSPC); + + while (filled < buffers_to_fill) { + struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts); + u64 addr; + u32 i; + + for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { + if (!pkt) { + if (!fill_up) + break; + addr = filled * umem->frame_size + umem->base_addr; + } else if (pkt->offset >= 0) { + addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); + } else { + addr = pkt->offset + umem_alloc_buffer(umem); + } + + *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; + if (++filled >= buffers_to_fill) + break; + } + } + xsk_ring_prod__submit(&umem->fq, filled); + xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled); + + pkt_stream_reset(pkt_stream); + umem_reset_alloc(umem); +} + +static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) +{ + LIBBPF_OPTS(bpf_xdp_query_opts, opts); + int mmap_flags; + u64 umem_sz; + void *bufs; + int ret; + u32 i; + + umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; + mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + + if (ifobject->umem->unaligned_mode) + mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; + + if (ifobject->shared_umem) + umem_sz *= 2; + + bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); + if (bufs == MAP_FAILED) + exit_with_error(errno); + + ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); + if (ret) + exit_with_error(-ret); + + xsk_configure(test, ifobject, ifobject->umem, false); + + ifobject->xsk = &ifobject->xsk_arr[0]; + + if (!ifobject->rx_on) + return; + + xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); + + for (i = 0; i < test->nb_sockets; i++) { + ifobject->xsk = &ifobject->xsk_arr[i]; + ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); + if (ret) + exit_with_error(errno); + } +} + +void *worker_testapp_validate_tx(void *arg) +{ + struct test_spec *test = (struct test_spec *)arg; + struct ifobject *ifobject = test->ifobj_tx; + int err; + + if (test->current_step == 1) { + if (!ifobject->shared_umem) + thread_common_ops(test, ifobject); + else + thread_common_ops_tx(test, ifobject); + } + + err = send_pkts(test, ifobject); + + if (!err && ifobject->validation_func) + err = ifobject->validation_func(ifobject); + if (err) + test->fail = true; + + pthread_exit(NULL); +} + +void *worker_testapp_validate_rx(void *arg) +{ + struct test_spec *test = (struct test_spec *)arg; + struct ifobject *ifobject = test->ifobj_rx; + int err; + + if (test->current_step == 1) { + thread_common_ops(test, ifobject); + } else { + xsk_clear_xskmap(ifobject->xskmap); + err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0); + if (err) { + ksft_print_msg("Error: Failed to update xskmap, error %s\n", + strerror(-err)); + exit_with_error(-err); + } + } + + pthread_barrier_wait(&barr); + + err = receive_pkts(test); + + if (!err && ifobject->validation_func) + err = ifobject->validation_func(ifobject); + + if (err) { + if (test->adjust_tail && !is_adjust_tail_supported(ifobject->xdp_progs)) + test->adjust_tail_support = false; + else + test->fail = true; + } + + pthread_exit(NULL); +} + +static void testapp_clean_xsk_umem(struct ifobject *ifobj) +{ + u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; + + if (ifobj->shared_umem) + umem_sz *= 2; + + umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; + xsk_umem__delete(ifobj->umem->umem); + munmap(ifobj->umem->buffer, umem_sz); +} + +static void handler(int signum) +{ + pthread_exit(NULL); +} + +static bool xdp_prog_changed_rx(struct test_spec *test) +{ + struct ifobject *ifobj = test->ifobj_rx; + + return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; +} + +static bool xdp_prog_changed_tx(struct test_spec *test) +{ + struct ifobject *ifobj = test->ifobj_tx; + + return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; +} + +static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, + struct bpf_map *xskmap, enum test_mode mode) +{ + int err; + + xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); + err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); + if (err) { + ksft_print_msg("Error attaching XDP program\n"); + exit_with_error(-err); + } + + if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) + if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { + ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); + exit_with_error(EINVAL); + } + + ifobj->xdp_prog = xdp_prog; + ifobj->xskmap = xskmap; + ifobj->mode = mode; +} + +static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, + struct ifobject *ifobj_tx) +{ + if (xdp_prog_changed_rx(test)) + xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); + + if (!ifobj_tx || ifobj_tx->shared_umem) + return; + + if (xdp_prog_changed_tx(test)) + xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); +} + +static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, + struct ifobject *ifobj2) +{ + pthread_t t0, t1; + int err; + + if (test->mtu > MAX_ETH_PKT_SIZE) { + if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || + (ifobj2 && !ifobj2->multi_buff_zc_supp))) { + ksft_print_msg("Multi buffer for zero-copy not supported.\n"); + return TEST_SKIP; + } + if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || + (ifobj2 && !ifobj2->multi_buff_supp))) { + ksft_print_msg("Multi buffer not supported.\n"); + return TEST_SKIP; + } + } + err = test_spec_set_mtu(test, test->mtu); + if (err) { + ksft_print_msg("Error, could not set mtu.\n"); + exit_with_error(err); + } + + if (ifobj2) { + if (pthread_barrier_init(&barr, NULL, 2)) + exit_with_error(errno); + pkt_stream_reset(ifobj2->xsk->pkt_stream); + } + + test->current_step++; + pkt_stream_reset(ifobj1->xsk->pkt_stream); + pkts_in_flight = 0; + + signal(SIGUSR1, handler); + /*Spawn RX thread */ + pthread_create(&t0, NULL, ifobj1->func_ptr, test); + + if (ifobj2) { + pthread_barrier_wait(&barr); + if (pthread_barrier_destroy(&barr)) + exit_with_error(errno); + + /*Spawn TX thread */ + pthread_create(&t1, NULL, ifobj2->func_ptr, test); + + pthread_join(t1, NULL); + } + + if (!ifobj2) + pthread_kill(t0, SIGUSR1); + else + pthread_join(t0, NULL); + + if (test->total_steps == test->current_step || test->fail) { + u32 i; + + if (ifobj2) + for (i = 0; i < test->nb_sockets; i++) + xsk_socket__delete(ifobj2->xsk_arr[i].xsk); + + for (i = 0; i < test->nb_sockets; i++) + xsk_socket__delete(ifobj1->xsk_arr[i].xsk); + + testapp_clean_xsk_umem(ifobj1); + if (ifobj2 && !ifobj2->shared_umem) + testapp_clean_xsk_umem(ifobj2); + } + + return !!test->fail; +} + +static int testapp_validate_traffic(struct test_spec *test) +{ + struct ifobject *ifobj_rx = test->ifobj_rx; + struct ifobject *ifobj_tx = test->ifobj_tx; + + if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || + (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { + ksft_print_msg("No huge pages present.\n"); + return TEST_SKIP; + } + + if (test->set_ring) { + if (ifobj_tx->hw_ring_size_supp) { + if (set_ring_size(ifobj_tx)) { + ksft_print_msg("Failed to change HW ring size.\n"); + return TEST_FAILURE; + } + } else { + ksft_print_msg("Changing HW ring size not supported.\n"); + return TEST_SKIP; + } + } + + xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); + return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); +} + +static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) +{ + return __testapp_validate_traffic(test, ifobj, NULL); +} + +int testapp_teardown(struct test_spec *test) +{ + int i; + + for (i = 0; i < MAX_TEARDOWN_ITER; i++) { + if (testapp_validate_traffic(test)) + return TEST_FAILURE; + test_spec_reset(test); + } + + return TEST_PASS; +} + +static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) +{ + thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; + struct ifobject *tmp_ifobj = (*ifobj1); + + (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; + (*ifobj2)->func_ptr = tmp_func_ptr; + + *ifobj1 = *ifobj2; + *ifobj2 = tmp_ifobj; +} + +int testapp_bidirectional(struct test_spec *test) +{ + int res; + + test->ifobj_tx->rx_on = true; + test->ifobj_rx->tx_on = true; + test->total_steps = 2; + if (testapp_validate_traffic(test)) + return TEST_FAILURE; + + print_verbose("Switching Tx/Rx direction\n"); + swap_directions(&test->ifobj_rx, &test->ifobj_tx); + res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); + + swap_directions(&test->ifobj_rx, &test->ifobj_tx); + return res; +} + +static int swap_xsk_resources(struct test_spec *test) +{ + int ret; + + test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; + test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; + test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; + test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; + test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; + test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; + + ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0); + if (ret) + return TEST_FAILURE; + + return TEST_PASS; +} + +int testapp_xdp_prog_cleanup(struct test_spec *test) +{ + test->total_steps = 2; + test->nb_sockets = 2; + if (testapp_validate_traffic(test)) + return TEST_FAILURE; + + if (swap_xsk_resources(test)) + return TEST_FAILURE; + return testapp_validate_traffic(test); +} + +int testapp_headroom(struct test_spec *test) +{ + test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; + return testapp_validate_traffic(test); +} + +int testapp_stats_rx_dropped(struct test_spec *test) +{ + if (test->mode == TEST_MODE_ZC) { + ksft_print_msg("Can not run RX_DROPPED test for ZC mode\n"); + return TEST_SKIP; + } + + pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); + test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - + XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; + pkt_stream_receive_half(test); + test->ifobj_rx->validation_func = validate_rx_dropped; + return testapp_validate_traffic(test); +} + +int testapp_stats_tx_invalid_descs(struct test_spec *test) +{ + pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); + test->ifobj_tx->validation_func = validate_tx_invalid_descs; + return testapp_validate_traffic(test); +} + +int testapp_stats_rx_full(struct test_spec *test) +{ + pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); + test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); + + test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; + test->ifobj_rx->release_rx = false; + test->ifobj_rx->validation_func = validate_rx_full; + return testapp_validate_traffic(test); +} + +int testapp_stats_fill_empty(struct test_spec *test) +{ + pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); + test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); + + test->ifobj_rx->use_fill_ring = false; + test->ifobj_rx->validation_func = validate_fill_empty; + return testapp_validate_traffic(test); +} + +int testapp_send_receive_unaligned(struct test_spec *test) +{ + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + /* Let half of the packets straddle a 4K buffer boundary */ + pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2); + + return testapp_validate_traffic(test); +} + +int testapp_send_receive_unaligned_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); + return testapp_validate_traffic(test); +} + +int testapp_single_pkt(struct test_spec *test) +{ + struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; + + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); + return testapp_validate_traffic(test); +} + +int testapp_send_receive_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); + + return testapp_validate_traffic(test); +} + +int testapp_invalid_desc_mb(struct test_spec *test) +{ + struct xsk_umem_info *umem = test->ifobj_tx->umem; + u64 umem_size = umem->num_frames * umem->frame_size; + struct pkt pkts[] = { + /* Valid packet for synch to start with */ + {0, MIN_PKT_SIZE, 0, true, 0}, + /* Zero frame len is not legal */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, 0, 0, false, 0}, + /* Invalid address in the second frame */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + /* Invalid len in the middle */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + /* Invalid options in the middle */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, + /* Transmit 2 frags, receive 3 */ + {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, + {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, + /* Middle frame crosses chunk boundary with small length */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, + /* Valid packet for synch so that something is received */ + {0, MIN_PKT_SIZE, 0, true, 0}}; + + if (umem->unaligned_mode) { + /* Crossing a chunk boundary allowed */ + pkts[12].valid = true; + pkts[13].valid = true; + } + + test->mtu = MAX_ETH_JUMBO_SIZE; + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); + return testapp_validate_traffic(test); +} + +int testapp_invalid_desc(struct test_spec *test) +{ + struct xsk_umem_info *umem = test->ifobj_tx->umem; + u64 umem_size = umem->num_frames * umem->frame_size; + struct pkt pkts[] = { + /* Zero packet address allowed */ + {0, MIN_PKT_SIZE, 0, true}, + /* Allowed packet */ + {0, MIN_PKT_SIZE, 0, true}, + /* Straddling the start of umem */ + {-2, MIN_PKT_SIZE, 0, false}, + /* Packet too large */ + {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, + /* Up to end of umem allowed */ + {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, + /* After umem ends */ + {umem_size, MIN_PKT_SIZE, 0, false}, + /* Straddle the end of umem */ + {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, + /* Straddle a 4K boundary */ + {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, + /* Straddle a 2K boundary */ + {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, + /* Valid packet for synch so that something is received */ + {0, MIN_PKT_SIZE, 0, true}}; + + if (umem->unaligned_mode) { + /* Crossing a page boundary allowed */ + pkts[7].valid = true; + } + if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { + /* Crossing a 2K frame size boundary not allowed */ + pkts[8].valid = false; + } + + if (test->ifobj_tx->shared_umem) { + pkts[4].offset += umem_size; + pkts[5].offset += umem_size; + pkts[6].offset += umem_size; + } + + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); + return testapp_validate_traffic(test); +} + +int testapp_xdp_drop(struct test_spec *test) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, + skel_rx->maps.xsk, skel_tx->maps.xsk); + + pkt_stream_receive_half(test); + return testapp_validate_traffic(test); +} + +int testapp_xdp_metadata_copy(struct test_spec *test) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, + skel_tx->progs.xsk_xdp_populate_metadata, + skel_rx->maps.xsk, skel_tx->maps.xsk); + test->ifobj_rx->use_metadata = true; + + skel_rx->bss->count = 0; + + return testapp_validate_traffic(test); +} + +int testapp_xdp_shared_umem(struct test_spec *test) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test->total_steps = 1; + test->nb_sockets = 2; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem, + skel_tx->progs.xsk_xdp_shared_umem, + skel_rx->maps.xsk, skel_tx->maps.xsk); + + pkt_stream_even_odd_sequence(test); + + return testapp_validate_traffic(test); +} + +int testapp_poll_txq_tmout(struct test_spec *test) +{ + test->ifobj_tx->use_poll = true; + /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ + test->ifobj_tx->umem->frame_size = 2048; + pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); + return testapp_validate_traffic_single_thread(test, test->ifobj_tx); +} + +int testapp_poll_rxq_tmout(struct test_spec *test) +{ + test->ifobj_rx->use_poll = true; + return testapp_validate_traffic_single_thread(test, test->ifobj_rx); +} + +int testapp_too_many_frags(struct test_spec *test) +{ + struct pkt *pkts; + u32 max_frags, i; + int ret; + + if (test->mode == TEST_MODE_ZC) { + max_frags = test->ifobj_tx->xdp_zc_max_segs; + } else { + max_frags = get_max_skb_frags(); + if (!max_frags) { + ksft_print_msg("Can't get MAX_SKB_FRAGS from system, using default (17)\n"); + max_frags = 17; + } + max_frags += 1; + } + + pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); + if (!pkts) + return TEST_FAILURE; + + test->mtu = MAX_ETH_JUMBO_SIZE; + + /* Valid packet for synch */ + pkts[0].len = MIN_PKT_SIZE; + pkts[0].valid = true; + + /* One valid packet with the max amount of frags */ + for (i = 1; i < max_frags + 1; i++) { + pkts[i].len = MIN_PKT_SIZE; + pkts[i].options = XDP_PKT_CONTD; + pkts[i].valid = true; + } + pkts[max_frags].options = 0; + + /* An invalid packet with the max amount of frags but signals packet + * continues on the last frag + */ + for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { + pkts[i].len = MIN_PKT_SIZE; + pkts[i].options = XDP_PKT_CONTD; + pkts[i].valid = false; + } + + /* Valid packet for synch */ + pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; + pkts[2 * max_frags + 1].valid = true; + + pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); + ret = testapp_validate_traffic(test); + + free(pkts); + return ret; +} + +static int xsk_load_xdp_programs(struct ifobject *ifobj) +{ + ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); + if (libbpf_get_error(ifobj->xdp_progs)) + return libbpf_get_error(ifobj->xdp_progs); + + return 0; +} + +/* Simple test */ +static bool hugepages_present(void) +{ + size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; + void *bufs; + + bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); + if (bufs == MAP_FAILED) + return false; + + mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; + munmap(bufs, mmap_sz); + return true; +} + +void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) +{ + LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); + int err; + + ifobj->func_ptr = func_ptr; + + err = xsk_load_xdp_programs(ifobj); + if (err) { + ksft_print_msg("Error loading XDP program\n"); + exit_with_error(err); + } + + if (hugepages_present()) + ifobj->unaligned_supp = true; + + err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); + if (err) { + ksft_print_msg("Error querying XDP capabilities\n"); + exit_with_error(-err); + } + if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) + ifobj->multi_buff_supp = true; + if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { + if (query_opts.xdp_zc_max_segs > 1) { + ifobj->multi_buff_zc_supp = true; + ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; + } else { + ifobj->xdp_zc_max_segs = 0; + } + } +} + +int testapp_send_receive(struct test_spec *test) +{ + return testapp_validate_traffic(test); +} + +int testapp_send_receive_2k_frame(struct test_spec *test) +{ + test->ifobj_tx->umem->frame_size = 2048; + test->ifobj_rx->umem->frame_size = 2048; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); + return testapp_validate_traffic(test); +} + +int testapp_poll_rx(struct test_spec *test) +{ + test->ifobj_rx->use_poll = true; + return testapp_validate_traffic(test); +} + +int testapp_poll_tx(struct test_spec *test) +{ + test->ifobj_tx->use_poll = true; + return testapp_validate_traffic(test); +} + +int testapp_aligned_inv_desc(struct test_spec *test) +{ + return testapp_invalid_desc(test); +} + +int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) +{ + test->ifobj_tx->umem->frame_size = 2048; + test->ifobj_rx->umem->frame_size = 2048; + return testapp_invalid_desc(test); +} + +int testapp_unaligned_inv_desc(struct test_spec *test) +{ + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + return testapp_invalid_desc(test); +} + +int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) +{ + u64 page_size, umem_size; + + /* Odd frame size so the UMEM doesn't end near a page boundary. */ + test->ifobj_tx->umem->frame_size = 4001; + test->ifobj_rx->umem->frame_size = 4001; + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + /* This test exists to test descriptors that staddle the end of + * the UMEM but not a page. + */ + page_size = sysconf(_SC_PAGESIZE); + umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; + assert(umem_size % page_size > MIN_PKT_SIZE); + assert(umem_size % page_size < page_size - MIN_PKT_SIZE); + + return testapp_invalid_desc(test); +} + +int testapp_aligned_inv_desc_mb(struct test_spec *test) +{ + return testapp_invalid_desc_mb(test); +} + +int testapp_unaligned_inv_desc_mb(struct test_spec *test) +{ + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + return testapp_invalid_desc_mb(test); +} + +int testapp_xdp_metadata(struct test_spec *test) +{ + return testapp_xdp_metadata_copy(test); +} + +int testapp_xdp_metadata_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + return testapp_xdp_metadata_copy(test); +} + +int testapp_hw_sw_min_ring_size(struct test_spec *test) +{ + int ret; + + test->set_ring = true; + test->total_steps = 2; + test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE; + test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2; + test->ifobj_tx->xsk->batch_size = 1; + test->ifobj_rx->xsk->batch_size = 1; + ret = testapp_validate_traffic(test); + if (ret) + return ret; + + /* Set batch size to hw_ring_size - 1 */ + test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; + test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; + return testapp_validate_traffic(test); +} + +int testapp_hw_sw_max_ring_size(struct test_spec *test) +{ + u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4; + int ret; + + test->set_ring = true; + test->total_steps = 2; + test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; + test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; + test->ifobj_rx->umem->num_frames = max_descs; + test->ifobj_rx->umem->fill_size = max_descs; + test->ifobj_rx->umem->comp_size = max_descs; + test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; + test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; + + ret = testapp_validate_traffic(test); + if (ret) + return ret; + + /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when + * updating the Rx HW tail register. + */ + test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; + test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; + pkt_stream_replace(test, max_descs, MIN_PKT_SIZE); + return testapp_validate_traffic(test); +} + +static int testapp_xdp_adjust_tail(struct test_spec *test, int adjust_value) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_adjust_tail, + skel_tx->progs.xsk_xdp_adjust_tail, + skel_rx->maps.xsk, skel_tx->maps.xsk); + + skel_rx->bss->adjust_value = adjust_value; + + return testapp_validate_traffic(test); +} + +static int testapp_adjust_tail(struct test_spec *test, u32 value, u32 pkt_len) +{ + int ret; + + test->adjust_tail_support = true; + test->adjust_tail = true; + test->total_steps = 1; + + pkt_stream_replace_ifobject(test->ifobj_tx, DEFAULT_BATCH_SIZE, pkt_len); + pkt_stream_replace_ifobject(test->ifobj_rx, DEFAULT_BATCH_SIZE, pkt_len + value); + + ret = testapp_xdp_adjust_tail(test, value); + if (ret) + return ret; + + if (!test->adjust_tail_support) { + ksft_print_msg("%s %sResize pkt with bpf_xdp_adjust_tail() not supported\n", + mode_string(test), busy_poll_string(test)); + return TEST_SKIP; + } + + return 0; +} + +int testapp_adjust_tail_shrink(struct test_spec *test) +{ + /* Shrink by 4 bytes for testing purpose */ + return testapp_adjust_tail(test, -4, MIN_PKT_SIZE * 2); +} + +int testapp_adjust_tail_shrink_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + /* Shrink by the frag size */ + return testapp_adjust_tail(test, -XSK_UMEM__MAX_FRAME_SIZE, XSK_UMEM__LARGE_FRAME_SIZE * 2); +} + +int testapp_adjust_tail_grow(struct test_spec *test) +{ + /* Grow by 4 bytes for testing purpose */ + return testapp_adjust_tail(test, 4, MIN_PKT_SIZE * 2); +} + +int testapp_adjust_tail_grow_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + /* Grow by (frag_size - last_frag_Size) - 1 to stay inside the last fragment */ + return testapp_adjust_tail(test, (XSK_UMEM__MAX_FRAME_SIZE / 2) - 1, + XSK_UMEM__LARGE_FRAME_SIZE * 2); +} + +int testapp_tx_queue_consumer(struct test_spec *test) +{ + int nr_packets; + + if (test->mode == TEST_MODE_ZC) { + ksft_print_msg("Can not run TX_QUEUE_CONSUMER test for ZC mode\n"); + return TEST_SKIP; + } + + nr_packets = MAX_TX_BUDGET_DEFAULT + 1; + pkt_stream_replace(test, nr_packets, MIN_PKT_SIZE); + test->ifobj_tx->xsk->batch_size = nr_packets; + test->ifobj_tx->xsk->check_consumer = true; + + return testapp_validate_traffic(test); +} + +struct ifobject *ifobject_create(void) +{ + struct ifobject *ifobj; + + ifobj = calloc(1, sizeof(struct ifobject)); + if (!ifobj) + return NULL; + + ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); + if (!ifobj->xsk_arr) + goto out_xsk_arr; + + ifobj->umem = calloc(1, sizeof(*ifobj->umem)); + if (!ifobj->umem) + goto out_umem; + + return ifobj; + +out_umem: + free(ifobj->xsk_arr); +out_xsk_arr: + free(ifobj); + return NULL; +} + +void ifobject_delete(struct ifobject *ifobj) +{ + free(ifobj->umem); + free(ifobj->xsk_arr); + free(ifobj); +} diff --git a/tools/testing/selftests/bpf/test_xsk.h b/tools/testing/selftests/bpf/test_xsk.h new file mode 100644 index 0000000000000..fb546cab39fdf --- /dev/null +++ b/tools/testing/selftests/bpf/test_xsk.h @@ -0,0 +1,297 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef TEST_XSK_H_ +#define TEST_XSK_H_ + +#include +#include + +#include "../kselftest.h" +#include "xsk.h" + +#ifndef SO_PREFER_BUSY_POLL +#define SO_PREFER_BUSY_POLL 69 +#endif + +#ifndef SO_BUSY_POLL_BUDGET +#define SO_BUSY_POLL_BUDGET 70 +#endif + +#define TEST_PASS 0 +#define TEST_FAILURE -1 +#define TEST_CONTINUE 1 +#define TEST_SKIP 2 + +#define DEFAULT_PKT_CNT (4 * 1024) +#define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) +#define HUGEPAGE_SIZE (2 * 1024 * 1024) +#define MIN_PKT_SIZE 64 +#define MAX_ETH_PKT_SIZE 1518 +#define MAX_INTERFACE_NAME_CHARS 16 +#define MAX_TEST_NAME_SIZE 48 +#define SOCK_RECONF_CTR 10 +#define USLEEP_MAX 10000 + +extern bool opt_verbose; +#define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) + +static void __exit_with_error(int error, const char *file, const char *func, int line) +{ + ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, + strerror(error)); + ksft_exit_xfail(); +} +#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) + +static inline u32 ceil_u32(u32 a, u32 b) +{ + return (a + b - 1) / b; +} + +static inline u64 ceil_u64(u64 a, u64 b) +{ + return (a + b - 1) / b; +} + +/* Simple test */ +enum test_mode { + TEST_MODE_SKB, + TEST_MODE_DRV, + TEST_MODE_ZC, + TEST_MODE_ALL +}; + +struct ifobject; +struct test_spec; +typedef int (*validation_func_t)(struct ifobject *ifobj); +typedef void *(*thread_func_t)(void *arg); +typedef int (*test_func_t)(struct test_spec *test); + +struct xsk_socket_info { + struct xsk_ring_cons rx; + struct xsk_ring_prod tx; + struct xsk_umem_info *umem; + struct xsk_socket *xsk; + struct pkt_stream *pkt_stream; + u32 outstanding_tx; + u32 rxqsize; + u32 batch_size; + u8 dst_mac[ETH_ALEN]; + u8 src_mac[ETH_ALEN]; + bool check_consumer; +}; + +int kick_rx(struct xsk_socket_info *xsk); +int kick_tx(struct xsk_socket_info *xsk); + +struct xsk_umem_info { + struct xsk_ring_prod fq; + struct xsk_ring_cons cq; + struct xsk_umem *umem; + u64 next_buffer; + u32 num_frames; + u32 frame_headroom; + void *buffer; + u32 frame_size; + u32 base_addr; + u32 fill_size; + u32 comp_size; + bool unaligned_mode; +}; + +struct set_hw_ring { + u32 default_tx; + u32 default_rx; +}; + +int hw_ring_size_reset(struct ifobject *ifobj); + +struct ifobject { + char ifname[MAX_INTERFACE_NAME_CHARS]; + struct xsk_socket_info *xsk; + struct xsk_socket_info *xsk_arr; + struct xsk_umem_info *umem; + thread_func_t func_ptr; + validation_func_t validation_func; + struct xsk_xdp_progs *xdp_progs; + struct bpf_map *xskmap; + struct bpf_program *xdp_prog; + struct ethtool_ringparam ring; + struct set_hw_ring set_ring; + enum test_mode mode; + int ifindex; + int mtu; + u32 bind_flags; + u32 xdp_zc_max_segs; + bool tx_on; + bool rx_on; + bool use_poll; + bool busy_poll; + bool use_fill_ring; + bool release_rx; + bool shared_umem; + bool use_metadata; + bool unaligned_supp; + bool multi_buff_supp; + bool multi_buff_zc_supp; + bool hw_ring_size_supp; +}; +struct ifobject *ifobject_create(void); +void ifobject_delete(struct ifobject *ifobj); +void init_iface(struct ifobject *ifobj, thread_func_t func_ptr); + +int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, u64 size); +int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, + struct ifobject *ifobject, bool shared); + + +struct pkt { + int offset; + u32 len; + u32 pkt_nb; + bool valid; + u16 options; +}; + +struct pkt_stream { + u32 nb_pkts; + u32 current_pkt_nb; + struct pkt *pkts; + u32 max_pkt_len; + u32 nb_rx_pkts; + u32 nb_valid_entries; + bool verbatim; +}; + +static inline bool pkt_continues(u32 options) +{ + return options & XDP_PKT_CONTD; +} + +struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len); +void pkt_stream_delete(struct pkt_stream *pkt_stream); +void pkt_stream_reset(struct pkt_stream *pkt_stream); +void pkt_stream_restore_default(struct test_spec *test); + +struct test_spec { + struct ifobject *ifobj_tx; + struct ifobject *ifobj_rx; + struct pkt_stream *tx_pkt_stream_default; + struct pkt_stream *rx_pkt_stream_default; + struct bpf_program *xdp_prog_rx; + struct bpf_program *xdp_prog_tx; + struct bpf_map *xskmap_rx; + struct bpf_map *xskmap_tx; + test_func_t test_func; + int mtu; + u16 total_steps; + u16 current_step; + u16 nb_sockets; + bool fail; + bool set_ring; + bool adjust_tail; + bool adjust_tail_support; + enum test_mode mode; + char name[MAX_TEST_NAME_SIZE]; +}; + +#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" +static inline char *mode_string(struct test_spec *test) +{ + switch (test->mode) { + case TEST_MODE_SKB: + return "SKB"; + case TEST_MODE_DRV: + return "DRV"; + case TEST_MODE_ZC: + return "ZC"; + default: + return "BOGUS"; + } +} + +void test_init(struct test_spec *test, struct ifobject *ifobj_tx, + struct ifobject *ifobj_rx, enum test_mode mode, + const struct test_spec *test_to_run); + +int testapp_adjust_tail_grow(struct test_spec *test); +int testapp_adjust_tail_grow_mb(struct test_spec *test); +int testapp_adjust_tail_shrink(struct test_spec *test); +int testapp_adjust_tail_shrink_mb(struct test_spec *test); +int testapp_aligned_inv_desc(struct test_spec *test); +int testapp_aligned_inv_desc_2k_frame(struct test_spec *test); +int testapp_aligned_inv_desc_mb(struct test_spec *test); +int testapp_bidirectional(struct test_spec *test); +int testapp_headroom(struct test_spec *test); +int testapp_hw_sw_max_ring_size(struct test_spec *test); +int testapp_hw_sw_min_ring_size(struct test_spec *test); +int testapp_poll_rx(struct test_spec *test); +int testapp_poll_rxq_tmout(struct test_spec *test); +int testapp_poll_tx(struct test_spec *test); +int testapp_poll_txq_tmout(struct test_spec *test); +int testapp_send_receive(struct test_spec *test); +int testapp_send_receive_2k_frame(struct test_spec *test); +int testapp_send_receive_mb(struct test_spec *test); +int testapp_send_receive_unaligned(struct test_spec *test); +int testapp_send_receive_unaligned_mb(struct test_spec *test); +int testapp_single_pkt(struct test_spec *test); +int testapp_stats_fill_empty(struct test_spec *test); +int testapp_stats_rx_dropped(struct test_spec *test); +int testapp_stats_tx_invalid_descs(struct test_spec *test); +int testapp_stats_rx_full(struct test_spec *test); +int testapp_teardown(struct test_spec *test); +int testapp_too_many_frags(struct test_spec *test); +int testapp_tx_queue_consumer(struct test_spec *test); +int testapp_unaligned_inv_desc(struct test_spec *test); +int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test); +int testapp_unaligned_inv_desc_mb(struct test_spec *test); +int testapp_xdp_drop(struct test_spec *test); +int testapp_xdp_metadata(struct test_spec *test); +int testapp_xdp_metadata_mb(struct test_spec *test); +int testapp_xdp_prog_cleanup(struct test_spec *test); +int testapp_xdp_shared_umem(struct test_spec *test); + +void *worker_testapp_validate_rx(void *arg); +void *worker_testapp_validate_tx(void *arg); + +static const struct test_spec tests[] = { + {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, + {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, + {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, + {.name = "POLL_RX", .test_func = testapp_poll_rx}, + {.name = "POLL_TX", .test_func = testapp_poll_tx}, + {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, + {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, + {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, + {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, + {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, + {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, + {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", + .test_func = testapp_unaligned_inv_desc_4001_frame}, + {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, + {.name = "TEARDOWN", .test_func = testapp_teardown}, + {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, + {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, + {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, + {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, + {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, + {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, + {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, + {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, + {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, + {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, + {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, + {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", + .test_func = testapp_send_receive_unaligned_mb}, + {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, + {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, + {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, + {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, + {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, + {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, + {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, + {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, + {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, + {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, + }; + +#endif /* TEST_XSK_H_ */ diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 59a020c356474..ef3ec56672c22 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -44,9 +44,9 @@ mark_precise: frame0: regs=r2 stack= before 23\ mark_precise: frame0: regs=r2 stack= before 22\ mark_precise: frame0: regs=r2 stack= before 20\ - mark_precise: frame0: parent state regs=r2,r9 stack=:\ + mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 19 first_idx 10\ - mark_precise: frame0: regs=r2,r9 stack= before 19\ + mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r8,r9 stack= before 17\ mark_precise: frame0: regs=r0,r9 stack= before 15\ @@ -107,9 +107,9 @@ mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 20 first_idx 20\ mark_precise: frame0: regs=r2 stack= before 20\ - mark_precise: frame0: parent state regs=r2,r9 stack=:\ + mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 19 first_idx 17\ - mark_precise: frame0: regs=r2,r9 stack= before 19\ + mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r8,r9 stack= before 17\ mark_precise: frame0: parent state regs= stack=:", diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index e962f133250c7..1be1e353d40a7 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -2580,7 +2580,7 @@ static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last if (last && fmt == RESFMT_TABLE) { output_header_underlines(); printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n", - env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped); + env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped); } } diff --git a/tools/testing/selftests/bpf/xskxceiver.c b/tools/testing/selftests/bpf/xskxceiver.c index 352adc8df2d1c..8e108e3162695 100644 --- a/tools/testing/selftests/bpf/xskxceiver.c +++ b/tools/testing/selftests/bpf/xskxceiver.c @@ -74,31 +74,23 @@ #define _GNU_SOURCE #include #include -#include #include #include #include #include #include -#include #include #include #include #include -#include -#include -#include #include #include #include -#include #include #include -#include -#include #include -#include +#include "test_xsk.h" #include "xsk_xdp_progs.skel.h" #include "xsk.h" #include "xskxceiver.h" @@ -109,181 +101,12 @@ #include -#define MAX_TX_BUDGET_DEFAULT 32 - -static bool opt_verbose; static bool opt_print_tests; static enum test_mode opt_mode = TEST_MODE_ALL; static u32 opt_run_test = RUN_ALL_TESTS; void test__fail(void) { /* for network_helpers.c */ } -static void __exit_with_error(int error, const char *file, const char *func, int line) -{ - ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, - strerror(error)); - ksft_exit_xfail(); -} - -#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) -#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" -static char *mode_string(struct test_spec *test) -{ - switch (test->mode) { - case TEST_MODE_SKB: - return "SKB"; - case TEST_MODE_DRV: - return "DRV"; - case TEST_MODE_ZC: - return "ZC"; - default: - return "BOGUS"; - } -} - -static void report_failure(struct test_spec *test) -{ - if (test->fail) - return; - - ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), - test->name); - test->fail = true; -} - -/* The payload is a word consisting of a packet sequence number in the upper - * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's - * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. - */ -static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) -{ - u32 *ptr = (u32 *)dest, i; - - start /= sizeof(*ptr); - size /= sizeof(*ptr); - for (i = 0; i < size; i++) - ptr[i] = htonl(pkt_nb << 16 | (i + start)); -} - -static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) -{ - memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); - memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); - eth_hdr->h_proto = htons(ETH_P_LOOPBACK); -} - -static bool is_umem_valid(struct ifobject *ifobj) -{ - return !!ifobj->umem->umem; -} - -static u32 mode_to_xdp_flags(enum test_mode mode) -{ - return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; -} - -static u64 umem_size(struct xsk_umem_info *umem) -{ - return umem->num_frames * umem->frame_size; -} - -static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, - u64 size) -{ - struct xsk_umem_config cfg = { - .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, - .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, - .frame_size = umem->frame_size, - .frame_headroom = umem->frame_headroom, - .flags = XSK_UMEM__DEFAULT_FLAGS - }; - int ret; - - if (umem->fill_size) - cfg.fill_size = umem->fill_size; - - if (umem->comp_size) - cfg.comp_size = umem->comp_size; - - if (umem->unaligned_mode) - cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; - - ret = xsk_umem__create(&umem->umem, buffer, size, - &umem->fq, &umem->cq, &cfg); - if (ret) - return ret; - - umem->buffer = buffer; - if (ifobj->shared_umem && ifobj->rx_on) { - umem->base_addr = umem_size(umem); - umem->next_buffer = umem_size(umem); - } - - return 0; -} - -static u64 umem_alloc_buffer(struct xsk_umem_info *umem) -{ - u64 addr; - - addr = umem->next_buffer; - umem->next_buffer += umem->frame_size; - if (umem->next_buffer >= umem->base_addr + umem_size(umem)) - umem->next_buffer = umem->base_addr; - - return addr; -} - -static void umem_reset_alloc(struct xsk_umem_info *umem) -{ - umem->next_buffer = 0; -} - -static void enable_busy_poll(struct xsk_socket_info *xsk) -{ - int sock_opt; - - sock_opt = 1; - if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, - (void *)&sock_opt, sizeof(sock_opt)) < 0) - exit_with_error(errno); - - sock_opt = 20; - if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, - (void *)&sock_opt, sizeof(sock_opt)) < 0) - exit_with_error(errno); - - sock_opt = xsk->batch_size; - if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, - (void *)&sock_opt, sizeof(sock_opt)) < 0) - exit_with_error(errno); -} - -static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, - struct ifobject *ifobject, bool shared) -{ - struct xsk_socket_config cfg = {}; - struct xsk_ring_cons *rxr; - struct xsk_ring_prod *txr; - - xsk->umem = umem; - cfg.rx_size = xsk->rxqsize; - cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; - cfg.bind_flags = ifobject->bind_flags; - if (shared) - cfg.bind_flags |= XDP_SHARED_UMEM; - if (ifobject->mtu > MAX_ETH_PKT_SIZE) - cfg.bind_flags |= XDP_USE_SG; - if (umem->comp_size) - cfg.tx_size = umem->comp_size; - if (umem->fill_size) - cfg.rx_size = umem->fill_size; - - txr = ifobject->tx_on ? &xsk->tx : NULL; - rxr = ifobject->rx_on ? &xsk->rx : NULL; - return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); -} - static bool ifobj_zc_avail(struct ifobject *ifobject) { size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; @@ -314,7 +137,7 @@ static bool ifobj_zc_avail(struct ifobject *ifobject) ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; ifobject->rx_on = true; xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; - ret = __xsk_configure_socket(xsk, umem, ifobject, false); + ret = xsk_configure_socket(xsk, umem, ifobject, false); if (!ret) zc_avail = true; @@ -327,25 +150,6 @@ static bool ifobj_zc_avail(struct ifobject *ifobject) return zc_avail; } -#define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" -static unsigned int get_max_skb_frags(void) -{ - unsigned int max_skb_frags = 0; - FILE *file; - - file = fopen(MAX_SKB_FRAGS_PATH, "r"); - if (!file) { - ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH); - return 0; - } - - if (fscanf(file, "%u", &max_skb_frags) != 1) - ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH); - - fclose(file); - return max_skb_frags; -} - static struct option long_options[] = { {"interface", required_argument, 0, 'i'}, {"busy-poll", no_argument, 0, 'b'}, @@ -446,2327 +250,66 @@ static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj } } -static int set_ring_size(struct ifobject *ifobj) -{ - int ret; - u32 ctr = 0; - - while (ctr++ < SOCK_RECONF_CTR) { - ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring); - if (!ret) - break; - - /* Retry if it fails */ - if (ctr >= SOCK_RECONF_CTR || errno != EBUSY) - return -errno; - - usleep(USLEEP_MAX); - } - - return ret; -} - -static int hw_ring_size_reset(struct ifobject *ifobj) -{ - ifobj->ring.tx_pending = ifobj->set_ring.default_tx; - ifobj->ring.rx_pending = ifobj->set_ring.default_rx; - return set_ring_size(ifobj); -} - -static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, - struct ifobject *ifobj_rx) +static void xsk_unload_xdp_programs(struct ifobject *ifobj) { - u32 i, j; - - for (i = 0; i < MAX_INTERFACES; i++) { - struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; - - ifobj->xsk = &ifobj->xsk_arr[0]; - ifobj->use_poll = false; - ifobj->use_fill_ring = true; - ifobj->release_rx = true; - ifobj->validation_func = NULL; - ifobj->use_metadata = false; - - if (i == 0) { - ifobj->rx_on = false; - ifobj->tx_on = true; - } else { - ifobj->rx_on = true; - ifobj->tx_on = false; - } - - memset(ifobj->umem, 0, sizeof(*ifobj->umem)); - ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; - ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; - - for (j = 0; j < MAX_SOCKETS; j++) { - memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); - ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; - ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; - if (i == 0) - ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; - else - ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; - - memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); - memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); - ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); - ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); - } - } - - if (ifobj_tx->hw_ring_size_supp) - hw_ring_size_reset(ifobj_tx); - - test->ifobj_tx = ifobj_tx; - test->ifobj_rx = ifobj_rx; - test->current_step = 0; - test->total_steps = 1; - test->nb_sockets = 1; - test->fail = false; - test->set_ring = false; - test->adjust_tail = false; - test->adjust_tail_support = false; - test->mtu = MAX_ETH_PKT_SIZE; - test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; - test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; - test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; - test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; + xsk_xdp_progs__destroy(ifobj->xdp_progs); } -static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, - struct ifobject *ifobj_rx, enum test_mode mode, - const struct test_spec *test_to_run) +static void run_pkt_test(struct test_spec *test) { - struct pkt_stream *tx_pkt_stream; - struct pkt_stream *rx_pkt_stream; - u32 i; - - tx_pkt_stream = test->tx_pkt_stream_default; - rx_pkt_stream = test->rx_pkt_stream_default; - memset(test, 0, sizeof(*test)); - test->tx_pkt_stream_default = tx_pkt_stream; - test->rx_pkt_stream_default = rx_pkt_stream; + int ret; - for (i = 0; i < MAX_INTERFACES; i++) { - struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + ret = test->test_func(test); - ifobj->bind_flags = XDP_USE_NEED_WAKEUP; - if (mode == TEST_MODE_ZC) - ifobj->bind_flags |= XDP_ZEROCOPY; - else - ifobj->bind_flags |= XDP_COPY; + switch (ret) { + case TEST_PASS: + ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), + test->name); + break; + case TEST_SKIP: + ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test), + test->name); + break; + case TEST_FAILURE: + ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), + test->name); + break; + default: + ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n", + mode_string(test), busy_poll_string(test), test->name, ret); } - strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); - test->test_func = test_to_run->test_func; - test->mode = mode; - __test_spec_init(test, ifobj_tx, ifobj_rx); -} - -static void test_spec_reset(struct test_spec *test) -{ - __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); + pkt_stream_restore_default(test); } -static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, - struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, - struct bpf_map *xskmap_tx) +static bool is_xdp_supported(int ifindex) { - test->xdp_prog_rx = xdp_prog_rx; - test->xdp_prog_tx = xdp_prog_tx; - test->xskmap_rx = xskmap_rx; - test->xskmap_tx = xskmap_tx; -} + int flags = XDP_FLAGS_DRV_MODE; -static int test_spec_set_mtu(struct test_spec *test, int mtu) -{ + LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); + struct bpf_insn insns[2] = { + BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), + BPF_EXIT_INSN() + }; + int prog_fd, insn_cnt = ARRAY_SIZE(insns); int err; - if (test->ifobj_rx->mtu != mtu) { - err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); - if (err) - return err; - test->ifobj_rx->mtu = mtu; - } - if (test->ifobj_tx->mtu != mtu) { - err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); - if (err) - return err; - test->ifobj_tx->mtu = mtu; - } - - return 0; -} - -static void pkt_stream_reset(struct pkt_stream *pkt_stream) -{ - if (pkt_stream) { - pkt_stream->current_pkt_nb = 0; - pkt_stream->nb_rx_pkts = 0; - } -} - -static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) -{ - if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) - return NULL; - - return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; -} - -static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) -{ - while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { - (*pkts_sent)++; - if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) - return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; - pkt_stream->current_pkt_nb++; - } - return NULL; -} - -static void pkt_stream_delete(struct pkt_stream *pkt_stream) -{ - free(pkt_stream->pkts); - free(pkt_stream); -} - -static void pkt_stream_restore_default(struct test_spec *test) -{ - struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; - struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; - - if (tx_pkt_stream != test->tx_pkt_stream_default) { - pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream); - test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; - } - - if (rx_pkt_stream != test->rx_pkt_stream_default) { - pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream); - test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; - } -} - -static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) -{ - struct pkt_stream *pkt_stream; - - pkt_stream = calloc(1, sizeof(*pkt_stream)); - if (!pkt_stream) - return NULL; - - pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); - if (!pkt_stream->pkts) { - free(pkt_stream); - return NULL; - } - - pkt_stream->nb_pkts = nb_pkts; - return pkt_stream; -} - -static bool pkt_continues(u32 options) -{ - return options & XDP_PKT_CONTD; -} - -static u32 ceil_u32(u32 a, u32 b) -{ - return (a + b - 1) / b; -} - -static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) -{ - u32 nb_frags = 1, next_frag; - - if (!pkt) - return 1; - - if (!pkt_stream->verbatim) { - if (!pkt->valid || !pkt->len) - return 1; - return ceil_u32(pkt->len, frame_size); - } - - /* Search for the end of the packet in verbatim mode */ - if (!pkt_continues(pkt->options)) - return nb_frags; - - next_frag = pkt_stream->current_pkt_nb; - pkt++; - while (next_frag++ < pkt_stream->nb_pkts) { - nb_frags++; - if (!pkt_continues(pkt->options) || !pkt->valid) - break; - pkt++; - } - return nb_frags; -} - -static bool set_pkt_valid(int offset, u32 len) -{ - return len <= MAX_ETH_JUMBO_SIZE; -} - -static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) -{ - pkt->offset = offset; - pkt->len = len; - pkt->valid = set_pkt_valid(offset, len); -} - -static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) -{ - bool prev_pkt_valid = pkt->valid; - - pkt_set(pkt_stream, pkt, offset, len); - pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; -} - -static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) -{ - return ceil_u32(len, umem->frame_size) * umem->frame_size; -} - -static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) -{ - struct pkt_stream *pkt_stream; - u32 i; - - pkt_stream = __pkt_stream_alloc(nb_pkts); - if (!pkt_stream) - exit_with_error(ENOMEM); - - pkt_stream->nb_pkts = nb_pkts; - pkt_stream->max_pkt_len = pkt_len; - for (i = 0; i < nb_pkts; i++) { - struct pkt *pkt = &pkt_stream->pkts[i]; - - pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); - pkt->pkt_nb = nb_start + i * nb_off; - } - - return pkt_stream; -} - -static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) -{ - return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1); -} - -static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) -{ - return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len); -} - -static void pkt_stream_replace_ifobject(struct ifobject *ifobj, u32 nb_pkts, u32 pkt_len) -{ - ifobj->xsk->pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); -} - -static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) -{ - pkt_stream_replace_ifobject(test->ifobj_tx, nb_pkts, pkt_len); - pkt_stream_replace_ifobject(test->ifobj_rx, nb_pkts, pkt_len); -} - -static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, - int offset) -{ - struct pkt_stream *pkt_stream; - u32 i; - - pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); - for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) - pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); - - ifobj->xsk->pkt_stream = pkt_stream; -} - -static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) -{ - __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); - __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); -} - -static void pkt_stream_receive_half(struct test_spec *test) -{ - struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; - u32 i; - - test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts, - pkt_stream->pkts[0].len); - pkt_stream = test->ifobj_rx->xsk->pkt_stream; - for (i = 1; i < pkt_stream->nb_pkts; i += 2) - pkt_stream->pkts[i].valid = false; - - pkt_stream->nb_valid_entries /= 2; -} - -static void pkt_stream_even_odd_sequence(struct test_spec *test) -{ - struct pkt_stream *pkt_stream; - u32 i; - - for (i = 0; i < test->nb_sockets; i++) { - pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; - pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, - pkt_stream->pkts[0].len, i, 2); - test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); + if (prog_fd < 0) + return false; - pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; - pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, - pkt_stream->pkts[0].len, i, 2); - test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; + err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); + if (err) { + close(prog_fd); + return false; } -} - -static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) -{ - if (!pkt->valid) - return pkt->offset; - return pkt->offset + umem_alloc_buffer(umem); -} -static void pkt_stream_cancel(struct pkt_stream *pkt_stream) -{ - pkt_stream->current_pkt_nb--; -} - -static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, - u32 pkt_nb, u32 bytes_written) -{ - void *data = xsk_umem__get_data(umem->buffer, addr); - - if (len < MIN_PKT_SIZE) - return; - - if (!bytes_written) { - gen_eth_hdr(xsk, data); - - len -= PKT_HDR_SIZE; - data += PKT_HDR_SIZE; - } else { - bytes_written -= PKT_HDR_SIZE; - } + bpf_xdp_detach(ifindex, flags, NULL); + close(prog_fd); - write_payload(data, pkt_nb, bytes_written, len); + return true; } -static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, - u32 nb_frames, bool verbatim) -{ - u32 i, len = 0, pkt_nb = 0, payload = 0; - struct pkt_stream *pkt_stream; - - pkt_stream = __pkt_stream_alloc(nb_frames); - if (!pkt_stream) - exit_with_error(ENOMEM); - - for (i = 0; i < nb_frames; i++) { - struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; - struct pkt *frame = &frames[i]; - - pkt->offset = frame->offset; - if (verbatim) { - *pkt = *frame; - pkt->pkt_nb = payload; - if (!frame->valid || !pkt_continues(frame->options)) - payload++; - } else { - if (frame->valid) - len += frame->len; - if (frame->valid && pkt_continues(frame->options)) - continue; - - pkt->pkt_nb = pkt_nb; - pkt->len = len; - pkt->valid = frame->valid; - pkt->options = 0; - - len = 0; - } - - print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", - pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); - - if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) - pkt_stream->max_pkt_len = pkt->len; - - if (pkt->valid) - pkt_stream->nb_valid_entries++; - - pkt_nb++; - } - - pkt_stream->nb_pkts = pkt_nb; - pkt_stream->verbatim = verbatim; - return pkt_stream; -} - -static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) -{ - struct pkt_stream *pkt_stream; - - pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); - test->ifobj_tx->xsk->pkt_stream = pkt_stream; - - pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); - test->ifobj_rx->xsk->pkt_stream = pkt_stream; -} - -static void pkt_print_data(u32 *data, u32 cnt) -{ - u32 i; - - for (i = 0; i < cnt; i++) { - u32 seqnum, pkt_nb; - - seqnum = ntohl(*data) & 0xffff; - pkt_nb = ntohl(*data) >> 16; - ksft_print_msg("%u:%u ", pkt_nb, seqnum); - data++; - } -} - -static void pkt_dump(void *pkt, u32 len, bool eth_header) -{ - struct ethhdr *ethhdr = pkt; - u32 i, *data; - - if (eth_header) { - /*extract L2 frame */ - ksft_print_msg("DEBUG>> L2: dst mac: "); - for (i = 0; i < ETH_ALEN; i++) - ksft_print_msg("%02X", ethhdr->h_dest[i]); - - ksft_print_msg("\nDEBUG>> L2: src mac: "); - for (i = 0; i < ETH_ALEN; i++) - ksft_print_msg("%02X", ethhdr->h_source[i]); - - data = pkt + PKT_HDR_SIZE; - } else { - data = pkt; - } - - /*extract L5 frame */ - ksft_print_msg("\nDEBUG>> L5: seqnum: "); - pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); - ksft_print_msg("...."); - if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { - ksft_print_msg("\n.... "); - pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, - PKT_DUMP_NB_TO_PRINT); - } - ksft_print_msg("\n---------------------------------------\n"); -} - -static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) -{ - u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; - u32 offset = addr % umem->frame_size, expected_offset; - int pkt_offset = pkt->valid ? pkt->offset : 0; - - if (!umem->unaligned_mode) - pkt_offset = 0; - - expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; - - if (offset == expected_offset) - return true; - - ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); - return false; -} - -static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) -{ - void *data = xsk_umem__get_data(buffer, addr); - struct xdp_info *meta = data - sizeof(struct xdp_info); - - if (meta->count != pkt->pkt_nb) { - ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n", - __func__, pkt->pkt_nb, - (unsigned long long)meta->count); - return false; - } - - return true; -} - -static bool is_adjust_tail_supported(struct xsk_xdp_progs *skel_rx) -{ - struct bpf_map *data_map; - int adjust_value = 0; - int key = 0; - int ret; - - data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); - if (!data_map || !bpf_map__is_internal(data_map)) { - ksft_print_msg("Error: could not find bss section of XDP program\n"); - exit_with_error(errno); - } - - ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &key, &adjust_value); - if (ret) { - ksft_print_msg("Error: bpf_map_lookup_elem failed with error %d\n", ret); - exit_with_error(errno); - } - - /* Set the 'adjust_value' variable to -EOPNOTSUPP in the XDP program if the adjust_tail - * helper is not supported. Skip the adjust_tail test case in this scenario. - */ - return adjust_value != -EOPNOTSUPP; -} - -static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, - u32 bytes_processed) -{ - u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; - void *data = xsk_umem__get_data(umem->buffer, addr); - - addr -= umem->base_addr; - - if (addr >= umem->num_frames * umem->frame_size || - addr + len > umem->num_frames * umem->frame_size) { - ksft_print_msg("Frag invalid addr: %llx len: %u\n", - (unsigned long long)addr, len); - return false; - } - if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { - ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", - (unsigned long long)addr, len); - return false; - } - - pkt_data = data; - if (!bytes_processed) { - pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); - len -= PKT_HDR_SIZE; - } else { - bytes_processed -= PKT_HDR_SIZE; - } - - expected_seqnum = bytes_processed / sizeof(*pkt_data); - seqnum = ntohl(*pkt_data) & 0xffff; - pkt_nb = ntohl(*pkt_data) >> 16; - - if (expected_pkt_nb != pkt_nb) { - ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", - __func__, expected_pkt_nb, pkt_nb); - goto error; - } - if (expected_seqnum != seqnum) { - ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", - __func__, expected_seqnum, seqnum); - goto error; - } - - words_to_end = len / sizeof(*pkt_data) - 1; - pkt_data += words_to_end; - seqnum = ntohl(*pkt_data) & 0xffff; - expected_seqnum += words_to_end; - if (expected_seqnum != seqnum) { - ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", - __func__, expected_seqnum, seqnum); - goto error; - } - - return true; - -error: - pkt_dump(data, len, !bytes_processed); - return false; -} - -static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) -{ - if (pkt->len != len) { - ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", - __func__, pkt->len, len); - pkt_dump(xsk_umem__get_data(buffer, addr), len, true); - return false; - } - - return true; -} - -static u32 load_value(u32 *counter) -{ - return __atomic_load_n(counter, __ATOMIC_ACQUIRE); -} - -static bool kick_tx_with_check(struct xsk_socket_info *xsk, int *ret) -{ - u32 max_budget = MAX_TX_BUDGET_DEFAULT; - u32 cons, ready_to_send; - int delta; - - cons = load_value(xsk->tx.consumer); - ready_to_send = load_value(xsk->tx.producer) - cons; - *ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); - - delta = load_value(xsk->tx.consumer) - cons; - /* By default, xsk should consume exact @max_budget descs at one - * send in this case where hitting the max budget limit in while - * loop is triggered in __xsk_generic_xmit(). Please make sure that - * the number of descs to be sent is larger than @max_budget, or - * else the tx.consumer will be updated in xskq_cons_peek_desc() - * in time which hides the issue we try to verify. - */ - if (ready_to_send > max_budget && delta != max_budget) - return false; - - return true; -} - -static int kick_tx(struct xsk_socket_info *xsk) -{ - int ret; - - if (xsk->check_consumer) { - if (!kick_tx_with_check(xsk, &ret)) - return TEST_FAILURE; - } else { - ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); - } - if (ret >= 0) - return TEST_PASS; - if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { - usleep(100); - return TEST_PASS; - } - return TEST_FAILURE; -} - -static int kick_rx(struct xsk_socket_info *xsk) -{ - int ret; - - ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); - if (ret < 0) - return TEST_FAILURE; - - return TEST_PASS; -} - -static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) -{ - unsigned int rcvd; - u32 idx; - int ret; - - if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { - ret = kick_tx(xsk); - if (ret) - return TEST_FAILURE; - } - - rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); - if (rcvd) { - if (rcvd > xsk->outstanding_tx) { - u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); - - ksft_print_msg("[%s] Too many packets completed\n", __func__); - ksft_print_msg("Last completion address: %llx\n", - (unsigned long long)addr); - return TEST_FAILURE; - } - - xsk_ring_cons__release(&xsk->umem->cq, rcvd); - xsk->outstanding_tx -= rcvd; - } - - return TEST_PASS; -} - -static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) -{ - u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; - u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; - struct pkt_stream *pkt_stream = xsk->pkt_stream; - struct ifobject *ifobj = test->ifobj_rx; - struct xsk_umem_info *umem = xsk->umem; - struct pollfd fds = { }; - struct pkt *pkt; - u64 first_addr = 0; - int ret; - - fds.fd = xsk_socket__fd(xsk->xsk); - fds.events = POLLIN; - - ret = kick_rx(xsk); - if (ret) - return TEST_FAILURE; - - if (ifobj->use_poll) { - ret = poll(&fds, 1, POLL_TMOUT); - if (ret < 0) - return TEST_FAILURE; - - if (!ret) { - if (!is_umem_valid(test->ifobj_tx)) - return TEST_PASS; - - ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); - return TEST_CONTINUE; - } - - if (!(fds.revents & POLLIN)) - return TEST_CONTINUE; - } - - rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx); - if (!rcvd) - return TEST_CONTINUE; - - if (ifobj->use_fill_ring) { - ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); - while (ret != rcvd) { - if (xsk_ring_prod__needs_wakeup(&umem->fq)) { - ret = poll(&fds, 1, POLL_TMOUT); - if (ret < 0) - return TEST_FAILURE; - } - ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); - } - } - - while (frags_processed < rcvd) { - const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); - u64 addr = desc->addr, orig; - - orig = xsk_umem__extract_addr(addr); - addr = xsk_umem__add_offset_to_addr(addr); - - if (!nb_frags) { - pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); - if (!pkt) { - ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", - __func__, addr, desc->len); - return TEST_FAILURE; - } - } - - print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", - addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); - - if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || - !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && - !is_metadata_correct(pkt, umem->buffer, addr))) - return TEST_FAILURE; - - if (!nb_frags++) - first_addr = addr; - frags_processed++; - pkt_len += desc->len; - if (ifobj->use_fill_ring) - *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; - - if (pkt_continues(desc->options)) - continue; - - /* The complete packet has been received */ - if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || - !is_offset_correct(umem, pkt, addr)) - return TEST_FAILURE; - - pkt_stream->nb_rx_pkts++; - nb_frags = 0; - pkt_len = 0; - } - - if (nb_frags) { - /* In the middle of a packet. Start over from beginning of packet. */ - idx_rx -= nb_frags; - xsk_ring_cons__cancel(&xsk->rx, nb_frags); - if (ifobj->use_fill_ring) { - idx_fq -= nb_frags; - xsk_ring_prod__cancel(&umem->fq, nb_frags); - } - frags_processed -= nb_frags; - } - - if (ifobj->use_fill_ring) - xsk_ring_prod__submit(&umem->fq, frags_processed); - if (ifobj->release_rx) - xsk_ring_cons__release(&xsk->rx, frags_processed); - - pthread_mutex_lock(&pacing_mutex); - pkts_in_flight -= pkts_sent; - pthread_mutex_unlock(&pacing_mutex); - pkts_sent = 0; - -return TEST_CONTINUE; -} - -bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, - unsigned long *bitmap) -{ - struct pkt_stream *pkt_stream = xsk->pkt_stream; - - if (!pkt_stream) { - __set_bit(sock_num, bitmap); - return false; - } - - if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { - __set_bit(sock_num, bitmap); - if (bitmap_full(bitmap, test->nb_sockets)) - return true; - } - - return false; -} - -static int receive_pkts(struct test_spec *test) -{ - struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; - DECLARE_BITMAP(bitmap, test->nb_sockets); - struct xsk_socket_info *xsk; - u32 sock_num = 0; - int res, ret; - - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - - timeradd(&tv_now, &tv_timeout, &tv_end); - - while (1) { - xsk = &test->ifobj_rx->xsk_arr[sock_num]; - - if ((all_packets_received(test, xsk, sock_num, bitmap))) - break; - - res = __receive_pkts(test, xsk); - if (!(res == TEST_PASS || res == TEST_CONTINUE)) - return res; - - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - - if (timercmp(&tv_now, &tv_end, >)) { - ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); - return TEST_FAILURE; - } - sock_num = (sock_num + 1) % test->nb_sockets; - } - - return TEST_PASS; -} - -static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) -{ - u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; - struct pkt_stream *pkt_stream = xsk->pkt_stream; - struct xsk_umem_info *umem = ifobject->umem; - bool use_poll = ifobject->use_poll; - struct pollfd fds = { }; - int ret; - - buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); - /* pkts_in_flight might be negative if many invalid packets are sent */ - if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / - buffer_len)) { - ret = kick_tx(xsk); - if (ret) - return TEST_FAILURE; - return TEST_CONTINUE; - } - - fds.fd = xsk_socket__fd(xsk->xsk); - fds.events = POLLOUT; - - while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { - if (use_poll) { - ret = poll(&fds, 1, POLL_TMOUT); - if (timeout) { - if (ret < 0) { - ksft_print_msg("ERROR: [%s] Poll error %d\n", - __func__, errno); - return TEST_FAILURE; - } - if (ret == 0) - return TEST_PASS; - break; - } - if (ret <= 0) { - ksft_print_msg("ERROR: [%s] Poll error %d\n", - __func__, errno); - return TEST_FAILURE; - } - } - - complete_pkts(xsk, xsk->batch_size); - } - - for (i = 0; i < xsk->batch_size; i++) { - struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); - u32 nb_frags_left, nb_frags, bytes_written = 0; - - if (!pkt) - break; - - nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); - if (nb_frags > xsk->batch_size - i) { - pkt_stream_cancel(pkt_stream); - xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i); - break; - } - nb_frags_left = nb_frags; - - while (nb_frags_left--) { - struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); - - tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); - if (pkt_stream->verbatim) { - tx_desc->len = pkt->len; - tx_desc->options = pkt->options; - } else if (nb_frags_left) { - tx_desc->len = umem->frame_size; - tx_desc->options = XDP_PKT_CONTD; - } else { - tx_desc->len = pkt->len - bytes_written; - tx_desc->options = 0; - } - if (pkt->valid) - pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb, - bytes_written); - bytes_written += tx_desc->len; - - print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", - tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); - - if (nb_frags_left) { - i++; - if (pkt_stream->verbatim) - pkt = pkt_stream_get_next_tx_pkt(pkt_stream); - } - } - - if (pkt && pkt->valid) { - valid_pkts++; - valid_frags += nb_frags; - } - } - - pthread_mutex_lock(&pacing_mutex); - pkts_in_flight += valid_pkts; - pthread_mutex_unlock(&pacing_mutex); - - xsk_ring_prod__submit(&xsk->tx, i); - xsk->outstanding_tx += valid_frags; - - if (use_poll) { - ret = poll(&fds, 1, POLL_TMOUT); - if (ret <= 0) { - if (ret == 0 && timeout) - return TEST_PASS; - - ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); - return TEST_FAILURE; - } - } - - if (!timeout) { - if (complete_pkts(xsk, i)) - return TEST_FAILURE; - - usleep(10); - return TEST_PASS; - } - - return TEST_CONTINUE; -} - -static int wait_for_tx_completion(struct xsk_socket_info *xsk) -{ - struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; - int ret; - - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - timeradd(&tv_now, &tv_timeout, &tv_end); - - while (xsk->outstanding_tx) { - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - if (timercmp(&tv_now, &tv_end, >)) { - ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); - return TEST_FAILURE; - } - - complete_pkts(xsk, xsk->batch_size); - } - - return TEST_PASS; -} - -bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) -{ - return bitmap_full(bitmap, test->nb_sockets); -} - -static int send_pkts(struct test_spec *test, struct ifobject *ifobject) -{ - bool timeout = !is_umem_valid(test->ifobj_rx); - DECLARE_BITMAP(bitmap, test->nb_sockets); - u32 i, ret; - - while (!(all_packets_sent(test, bitmap))) { - for (i = 0; i < test->nb_sockets; i++) { - struct pkt_stream *pkt_stream; - - pkt_stream = ifobject->xsk_arr[i].pkt_stream; - if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { - __set_bit(i, bitmap); - continue; - } - ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); - if (ret == TEST_CONTINUE && !test->fail) - continue; - - if ((ret || test->fail) && !timeout) - return TEST_FAILURE; - - if (ret == TEST_PASS && timeout) - return ret; - - ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); - if (ret) - return TEST_FAILURE; - } - } - - return TEST_PASS; -} - -static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) -{ - int fd = xsk_socket__fd(xsk), err; - socklen_t optlen, expected_len; - - optlen = sizeof(*stats); - err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); - if (err) { - ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", - __func__, -err, strerror(-err)); - return TEST_FAILURE; - } - - expected_len = sizeof(struct xdp_statistics); - if (optlen != expected_len) { - ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", - __func__, expected_len, optlen); - return TEST_FAILURE; - } - - return TEST_PASS; -} - -static int validate_rx_dropped(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - struct xdp_statistics stats; - int err; - - err = kick_rx(ifobject->xsk); - if (err) - return TEST_FAILURE; - - err = get_xsk_stats(xsk, &stats); - if (err) - return TEST_FAILURE; - - /* The receiver calls getsockopt after receiving the last (valid) - * packet which is not the final packet sent in this test (valid and - * invalid packets are sent in alternating fashion with the final - * packet being invalid). Since the last packet may or may not have - * been dropped already, both outcomes must be allowed. - */ - if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || - stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) - return TEST_PASS; - - return TEST_FAILURE; -} - -static int validate_rx_full(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - struct xdp_statistics stats; - int err; - - usleep(1000); - err = kick_rx(ifobject->xsk); - if (err) - return TEST_FAILURE; - - err = get_xsk_stats(xsk, &stats); - if (err) - return TEST_FAILURE; - - if (stats.rx_ring_full) - return TEST_PASS; - - return TEST_FAILURE; -} - -static int validate_fill_empty(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - struct xdp_statistics stats; - int err; - - usleep(1000); - err = kick_rx(ifobject->xsk); - if (err) - return TEST_FAILURE; - - err = get_xsk_stats(xsk, &stats); - if (err) - return TEST_FAILURE; - - if (stats.rx_fill_ring_empty_descs) - return TEST_PASS; - - return TEST_FAILURE; -} - -static int validate_tx_invalid_descs(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - int fd = xsk_socket__fd(xsk); - struct xdp_statistics stats; - socklen_t optlen; - int err; - - optlen = sizeof(stats); - err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); - if (err) { - ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", - __func__, -err, strerror(-err)); - return TEST_FAILURE; - } - - if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { - ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n", - __func__, - (unsigned long long)stats.tx_invalid_descs, - ifobject->xsk->pkt_stream->nb_pkts); - return TEST_FAILURE; - } - - return TEST_PASS; -} - -static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, - struct xsk_umem_info *umem, bool tx) -{ - int i, ret; - - for (i = 0; i < test->nb_sockets; i++) { - bool shared = (ifobject->shared_umem && tx) ? true : !!i; - u32 ctr = 0; - - while (ctr++ < SOCK_RECONF_CTR) { - ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, - ifobject, shared); - if (!ret) - break; - - /* Retry if it fails as xsk_socket__create() is asynchronous */ - if (ctr >= SOCK_RECONF_CTR) - exit_with_error(-ret); - usleep(USLEEP_MAX); - } - if (ifobject->busy_poll) - enable_busy_poll(&ifobject->xsk_arr[i]); - } -} - -static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) -{ - xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); - ifobject->xsk = &ifobject->xsk_arr[0]; - ifobject->xskmap = test->ifobj_rx->xskmap; - memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); - ifobject->umem->base_addr = 0; -} - -static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, - bool fill_up) -{ - u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; - u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; - int ret; - - if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) - buffers_to_fill = umem->num_frames; - else - buffers_to_fill = umem->fill_size; - - ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); - if (ret != buffers_to_fill) - exit_with_error(ENOSPC); - - while (filled < buffers_to_fill) { - struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts); - u64 addr; - u32 i; - - for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { - if (!pkt) { - if (!fill_up) - break; - addr = filled * umem->frame_size + umem->base_addr; - } else if (pkt->offset >= 0) { - addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); - } else { - addr = pkt->offset + umem_alloc_buffer(umem); - } - - *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; - if (++filled >= buffers_to_fill) - break; - } - } - xsk_ring_prod__submit(&umem->fq, filled); - xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled); - - pkt_stream_reset(pkt_stream); - umem_reset_alloc(umem); -} - -static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) -{ - u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; - int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; - LIBBPF_OPTS(bpf_xdp_query_opts, opts); - void *bufs; - int ret; - u32 i; - - if (ifobject->umem->unaligned_mode) - mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; - - if (ifobject->shared_umem) - umem_sz *= 2; - - bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); - if (bufs == MAP_FAILED) - exit_with_error(errno); - - ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); - if (ret) - exit_with_error(-ret); - - xsk_configure_socket(test, ifobject, ifobject->umem, false); - - ifobject->xsk = &ifobject->xsk_arr[0]; - - if (!ifobject->rx_on) - return; - - xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); - - for (i = 0; i < test->nb_sockets; i++) { - ifobject->xsk = &ifobject->xsk_arr[i]; - ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); - if (ret) - exit_with_error(errno); - } -} - -static void *worker_testapp_validate_tx(void *arg) -{ - struct test_spec *test = (struct test_spec *)arg; - struct ifobject *ifobject = test->ifobj_tx; - int err; - - if (test->current_step == 1) { - if (!ifobject->shared_umem) - thread_common_ops(test, ifobject); - else - thread_common_ops_tx(test, ifobject); - } - - err = send_pkts(test, ifobject); - - if (!err && ifobject->validation_func) - err = ifobject->validation_func(ifobject); - if (err) - report_failure(test); - - pthread_exit(NULL); -} - -static void *worker_testapp_validate_rx(void *arg) -{ - struct test_spec *test = (struct test_spec *)arg; - struct ifobject *ifobject = test->ifobj_rx; - int err; - - if (test->current_step == 1) { - thread_common_ops(test, ifobject); - } else { - xsk_clear_xskmap(ifobject->xskmap); - err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0); - if (err) { - ksft_print_msg("Error: Failed to update xskmap, error %s\n", - strerror(-err)); - exit_with_error(-err); - } - } - - pthread_barrier_wait(&barr); - - err = receive_pkts(test); - - if (!err && ifobject->validation_func) - err = ifobject->validation_func(ifobject); - - if (err) { - if (test->adjust_tail && !is_adjust_tail_supported(ifobject->xdp_progs)) - test->adjust_tail_support = false; - else - report_failure(test); - } - - pthread_exit(NULL); -} - -static u64 ceil_u64(u64 a, u64 b) -{ - return (a + b - 1) / b; -} - -static void testapp_clean_xsk_umem(struct ifobject *ifobj) -{ - u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; - - if (ifobj->shared_umem) - umem_sz *= 2; - - umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; - xsk_umem__delete(ifobj->umem->umem); - munmap(ifobj->umem->buffer, umem_sz); -} - -static void handler(int signum) -{ - pthread_exit(NULL); -} - -static bool xdp_prog_changed_rx(struct test_spec *test) -{ - struct ifobject *ifobj = test->ifobj_rx; - - return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; -} - -static bool xdp_prog_changed_tx(struct test_spec *test) -{ - struct ifobject *ifobj = test->ifobj_tx; - - return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; -} - -static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, - struct bpf_map *xskmap, enum test_mode mode) -{ - int err; - - xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); - err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); - if (err) { - ksft_print_msg("Error attaching XDP program\n"); - exit_with_error(-err); - } - - if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) - if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { - ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); - exit_with_error(EINVAL); - } - - ifobj->xdp_prog = xdp_prog; - ifobj->xskmap = xskmap; - ifobj->mode = mode; -} - -static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, - struct ifobject *ifobj_tx) -{ - if (xdp_prog_changed_rx(test)) - xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); - - if (!ifobj_tx || ifobj_tx->shared_umem) - return; - - if (xdp_prog_changed_tx(test)) - xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); -} - -static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, - struct ifobject *ifobj2) -{ - pthread_t t0, t1; - int err; - - if (test->mtu > MAX_ETH_PKT_SIZE) { - if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || - (ifobj2 && !ifobj2->multi_buff_zc_supp))) { - ksft_test_result_skip("Multi buffer for zero-copy not supported.\n"); - return TEST_SKIP; - } - if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || - (ifobj2 && !ifobj2->multi_buff_supp))) { - ksft_test_result_skip("Multi buffer not supported.\n"); - return TEST_SKIP; - } - } - err = test_spec_set_mtu(test, test->mtu); - if (err) { - ksft_print_msg("Error, could not set mtu.\n"); - exit_with_error(err); - } - - if (ifobj2) { - if (pthread_barrier_init(&barr, NULL, 2)) - exit_with_error(errno); - pkt_stream_reset(ifobj2->xsk->pkt_stream); - } - - test->current_step++; - pkt_stream_reset(ifobj1->xsk->pkt_stream); - pkts_in_flight = 0; - - signal(SIGUSR1, handler); - /*Spawn RX thread */ - pthread_create(&t0, NULL, ifobj1->func_ptr, test); - - if (ifobj2) { - pthread_barrier_wait(&barr); - if (pthread_barrier_destroy(&barr)) - exit_with_error(errno); - - /*Spawn TX thread */ - pthread_create(&t1, NULL, ifobj2->func_ptr, test); - - pthread_join(t1, NULL); - } - - if (!ifobj2) - pthread_kill(t0, SIGUSR1); - else - pthread_join(t0, NULL); - - if (test->total_steps == test->current_step || test->fail) { - u32 i; - - if (ifobj2) - for (i = 0; i < test->nb_sockets; i++) - xsk_socket__delete(ifobj2->xsk_arr[i].xsk); - - for (i = 0; i < test->nb_sockets; i++) - xsk_socket__delete(ifobj1->xsk_arr[i].xsk); - - testapp_clean_xsk_umem(ifobj1); - if (ifobj2 && !ifobj2->shared_umem) - testapp_clean_xsk_umem(ifobj2); - } - - return !!test->fail; -} - -static int testapp_validate_traffic(struct test_spec *test) -{ - struct ifobject *ifobj_rx = test->ifobj_rx; - struct ifobject *ifobj_tx = test->ifobj_tx; - - if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || - (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { - ksft_test_result_skip("No huge pages present.\n"); - return TEST_SKIP; - } - - if (test->set_ring) { - if (ifobj_tx->hw_ring_size_supp) { - if (set_ring_size(ifobj_tx)) { - ksft_test_result_skip("Failed to change HW ring size.\n"); - return TEST_FAILURE; - } - } else { - ksft_test_result_skip("Changing HW ring size not supported.\n"); - return TEST_SKIP; - } - } - - xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); - return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); -} - -static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) -{ - return __testapp_validate_traffic(test, ifobj, NULL); -} - -static int testapp_teardown(struct test_spec *test) -{ - int i; - - for (i = 0; i < MAX_TEARDOWN_ITER; i++) { - if (testapp_validate_traffic(test)) - return TEST_FAILURE; - test_spec_reset(test); - } - - return TEST_PASS; -} - -static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) -{ - thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; - struct ifobject *tmp_ifobj = (*ifobj1); - - (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; - (*ifobj2)->func_ptr = tmp_func_ptr; - - *ifobj1 = *ifobj2; - *ifobj2 = tmp_ifobj; -} - -static int testapp_bidirectional(struct test_spec *test) -{ - int res; - - test->ifobj_tx->rx_on = true; - test->ifobj_rx->tx_on = true; - test->total_steps = 2; - if (testapp_validate_traffic(test)) - return TEST_FAILURE; - - print_verbose("Switching Tx/Rx direction\n"); - swap_directions(&test->ifobj_rx, &test->ifobj_tx); - res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); - - swap_directions(&test->ifobj_rx, &test->ifobj_tx); - return res; -} - -static int swap_xsk_resources(struct test_spec *test) -{ - int ret; - - test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; - test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; - test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; - test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; - test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; - test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; - - ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0); - if (ret) - return TEST_FAILURE; - - return TEST_PASS; -} - -static int testapp_xdp_prog_cleanup(struct test_spec *test) -{ - test->total_steps = 2; - test->nb_sockets = 2; - if (testapp_validate_traffic(test)) - return TEST_FAILURE; - - if (swap_xsk_resources(test)) - return TEST_FAILURE; - return testapp_validate_traffic(test); -} - -static int testapp_headroom(struct test_spec *test) -{ - test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; - return testapp_validate_traffic(test); -} - -static int testapp_stats_rx_dropped(struct test_spec *test) -{ - if (test->mode == TEST_MODE_ZC) { - ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); - return TEST_SKIP; - } - - pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); - test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; - pkt_stream_receive_half(test); - test->ifobj_rx->validation_func = validate_rx_dropped; - return testapp_validate_traffic(test); -} - -static int testapp_stats_tx_invalid_descs(struct test_spec *test) -{ - pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); - test->ifobj_tx->validation_func = validate_tx_invalid_descs; - return testapp_validate_traffic(test); -} - -static int testapp_stats_rx_full(struct test_spec *test) -{ - pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); - test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); - - test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; - test->ifobj_rx->release_rx = false; - test->ifobj_rx->validation_func = validate_rx_full; - return testapp_validate_traffic(test); -} - -static int testapp_stats_fill_empty(struct test_spec *test) -{ - pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); - test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); - - test->ifobj_rx->use_fill_ring = false; - test->ifobj_rx->validation_func = validate_fill_empty; - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_unaligned(struct test_spec *test) -{ - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - /* Let half of the packets straddle a 4K buffer boundary */ - pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2); - - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_unaligned_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); - return testapp_validate_traffic(test); -} - -static int testapp_single_pkt(struct test_spec *test) -{ - struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; - - pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); - - return testapp_validate_traffic(test); -} - -static int testapp_invalid_desc_mb(struct test_spec *test) -{ - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; - struct pkt pkts[] = { - /* Valid packet for synch to start with */ - {0, MIN_PKT_SIZE, 0, true, 0}, - /* Zero frame len is not legal */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, 0, 0, false, 0}, - /* Invalid address in the second frame */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - /* Invalid len in the middle */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - /* Invalid options in the middle */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, - /* Transmit 2 frags, receive 3 */ - {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, - {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, - /* Middle frame crosses chunk boundary with small length */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, - /* Valid packet for synch so that something is received */ - {0, MIN_PKT_SIZE, 0, true, 0}}; - - if (umem->unaligned_mode) { - /* Crossing a chunk boundary allowed */ - pkts[12].valid = true; - pkts[13].valid = true; - } - - test->mtu = MAX_ETH_JUMBO_SIZE; - pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); - return testapp_validate_traffic(test); -} - -static int testapp_invalid_desc(struct test_spec *test) -{ - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; - struct pkt pkts[] = { - /* Zero packet address allowed */ - {0, MIN_PKT_SIZE, 0, true}, - /* Allowed packet */ - {0, MIN_PKT_SIZE, 0, true}, - /* Straddling the start of umem */ - {-2, MIN_PKT_SIZE, 0, false}, - /* Packet too large */ - {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, - /* Up to end of umem allowed */ - {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, - /* After umem ends */ - {umem_size, MIN_PKT_SIZE, 0, false}, - /* Straddle the end of umem */ - {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, - /* Straddle a 4K boundary */ - {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, - /* Straddle a 2K boundary */ - {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, - /* Valid packet for synch so that something is received */ - {0, MIN_PKT_SIZE, 0, true}}; - - if (umem->unaligned_mode) { - /* Crossing a page boundary allowed */ - pkts[7].valid = true; - } - if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { - /* Crossing a 2K frame size boundary not allowed */ - pkts[8].valid = false; - } - - if (test->ifobj_tx->shared_umem) { - pkts[4].offset += umem_size; - pkts[5].offset += umem_size; - pkts[6].offset += umem_size; - } - - pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); - return testapp_validate_traffic(test); -} - -static int testapp_xdp_drop(struct test_spec *test) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, - skel_rx->maps.xsk, skel_tx->maps.xsk); - - pkt_stream_receive_half(test); - return testapp_validate_traffic(test); -} - -static int testapp_xdp_metadata_copy(struct test_spec *test) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, - skel_tx->progs.xsk_xdp_populate_metadata, - skel_rx->maps.xsk, skel_tx->maps.xsk); - test->ifobj_rx->use_metadata = true; - - skel_rx->bss->count = 0; - - return testapp_validate_traffic(test); -} - -static int testapp_xdp_shared_umem(struct test_spec *test) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test->total_steps = 1; - test->nb_sockets = 2; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem, - skel_tx->progs.xsk_xdp_shared_umem, - skel_rx->maps.xsk, skel_tx->maps.xsk); - - pkt_stream_even_odd_sequence(test); - - return testapp_validate_traffic(test); -} - -static int testapp_poll_txq_tmout(struct test_spec *test) -{ - test->ifobj_tx->use_poll = true; - /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ - test->ifobj_tx->umem->frame_size = 2048; - pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); - return testapp_validate_traffic_single_thread(test, test->ifobj_tx); -} - -static int testapp_poll_rxq_tmout(struct test_spec *test) -{ - test->ifobj_rx->use_poll = true; - return testapp_validate_traffic_single_thread(test, test->ifobj_rx); -} - -static int testapp_too_many_frags(struct test_spec *test) -{ - struct pkt *pkts; - u32 max_frags, i; - int ret; - - if (test->mode == TEST_MODE_ZC) { - max_frags = test->ifobj_tx->xdp_zc_max_segs; - } else { - max_frags = get_max_skb_frags(); - if (!max_frags) { - ksft_print_msg("Couldn't retrieve MAX_SKB_FRAGS from system, using default (17) value\n"); - max_frags = 17; - } - max_frags += 1; - } - - pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); - if (!pkts) - return TEST_FAILURE; - - test->mtu = MAX_ETH_JUMBO_SIZE; - - /* Valid packet for synch */ - pkts[0].len = MIN_PKT_SIZE; - pkts[0].valid = true; - - /* One valid packet with the max amount of frags */ - for (i = 1; i < max_frags + 1; i++) { - pkts[i].len = MIN_PKT_SIZE; - pkts[i].options = XDP_PKT_CONTD; - pkts[i].valid = true; - } - pkts[max_frags].options = 0; - - /* An invalid packet with the max amount of frags but signals packet - * continues on the last frag - */ - for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { - pkts[i].len = MIN_PKT_SIZE; - pkts[i].options = XDP_PKT_CONTD; - pkts[i].valid = false; - } - - /* Valid packet for synch */ - pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; - pkts[2 * max_frags + 1].valid = true; - - pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); - ret = testapp_validate_traffic(test); - - free(pkts); - return ret; -} - -static int xsk_load_xdp_programs(struct ifobject *ifobj) -{ - ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); - if (libbpf_get_error(ifobj->xdp_progs)) - return libbpf_get_error(ifobj->xdp_progs); - - return 0; -} - -static void xsk_unload_xdp_programs(struct ifobject *ifobj) -{ - xsk_xdp_progs__destroy(ifobj->xdp_progs); -} - -/* Simple test */ -static bool hugepages_present(void) -{ - size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; - void *bufs; - - bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); - if (bufs == MAP_FAILED) - return false; - - mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; - munmap(bufs, mmap_sz); - return true; -} - -static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) -{ - LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); - int err; - - ifobj->func_ptr = func_ptr; - - err = xsk_load_xdp_programs(ifobj); - if (err) { - ksft_print_msg("Error loading XDP program\n"); - exit_with_error(err); - } - - if (hugepages_present()) - ifobj->unaligned_supp = true; - - err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); - if (err) { - ksft_print_msg("Error querying XDP capabilities\n"); - exit_with_error(-err); - } - if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) - ifobj->multi_buff_supp = true; - if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { - if (query_opts.xdp_zc_max_segs > 1) { - ifobj->multi_buff_zc_supp = true; - ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; - } else { - ifobj->xdp_zc_max_segs = 0; - } - } -} - -static int testapp_send_receive(struct test_spec *test) -{ - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_2k_frame(struct test_spec *test) -{ - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; - pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); - return testapp_validate_traffic(test); -} - -static int testapp_poll_rx(struct test_spec *test) -{ - test->ifobj_rx->use_poll = true; - return testapp_validate_traffic(test); -} - -static int testapp_poll_tx(struct test_spec *test) -{ - test->ifobj_tx->use_poll = true; - return testapp_validate_traffic(test); -} - -static int testapp_aligned_inv_desc(struct test_spec *test) -{ - return testapp_invalid_desc(test); -} - -static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) -{ - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; - return testapp_invalid_desc(test); -} - -static int testapp_unaligned_inv_desc(struct test_spec *test) -{ - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - return testapp_invalid_desc(test); -} - -static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) -{ - u64 page_size, umem_size; - - /* Odd frame size so the UMEM doesn't end near a page boundary. */ - test->ifobj_tx->umem->frame_size = 4001; - test->ifobj_rx->umem->frame_size = 4001; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - /* This test exists to test descriptors that staddle the end of - * the UMEM but not a page. - */ - page_size = sysconf(_SC_PAGESIZE); - umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; - assert(umem_size % page_size > MIN_PKT_SIZE); - assert(umem_size % page_size < page_size - MIN_PKT_SIZE); - - return testapp_invalid_desc(test); -} - -static int testapp_aligned_inv_desc_mb(struct test_spec *test) -{ - return testapp_invalid_desc_mb(test); -} - -static int testapp_unaligned_inv_desc_mb(struct test_spec *test) -{ - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - return testapp_invalid_desc_mb(test); -} - -static int testapp_xdp_metadata(struct test_spec *test) -{ - return testapp_xdp_metadata_copy(test); -} - -static int testapp_xdp_metadata_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - return testapp_xdp_metadata_copy(test); -} - -static int testapp_hw_sw_min_ring_size(struct test_spec *test) -{ - int ret; - - test->set_ring = true; - test->total_steps = 2; - test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE; - test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2; - test->ifobj_tx->xsk->batch_size = 1; - test->ifobj_rx->xsk->batch_size = 1; - ret = testapp_validate_traffic(test); - if (ret) - return ret; - - /* Set batch size to hw_ring_size - 1 */ - test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; - test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; - return testapp_validate_traffic(test); -} - -static int testapp_hw_sw_max_ring_size(struct test_spec *test) -{ - u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4; - int ret; - - test->set_ring = true; - test->total_steps = 2; - test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; - test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; - test->ifobj_rx->umem->num_frames = max_descs; - test->ifobj_rx->umem->fill_size = max_descs; - test->ifobj_rx->umem->comp_size = max_descs; - test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; - test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; - - ret = testapp_validate_traffic(test); - if (ret) - return ret; - - /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when - * updating the Rx HW tail register. - */ - test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; - test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; - pkt_stream_replace(test, max_descs, MIN_PKT_SIZE); - return testapp_validate_traffic(test); -} - -static int testapp_xdp_adjust_tail(struct test_spec *test, int adjust_value) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_adjust_tail, - skel_tx->progs.xsk_xdp_adjust_tail, - skel_rx->maps.xsk, skel_tx->maps.xsk); - - skel_rx->bss->adjust_value = adjust_value; - - return testapp_validate_traffic(test); -} - -static int testapp_adjust_tail(struct test_spec *test, u32 value, u32 pkt_len) -{ - int ret; - - test->adjust_tail_support = true; - test->adjust_tail = true; - test->total_steps = 1; - - pkt_stream_replace_ifobject(test->ifobj_tx, DEFAULT_BATCH_SIZE, pkt_len); - pkt_stream_replace_ifobject(test->ifobj_rx, DEFAULT_BATCH_SIZE, pkt_len + value); - - ret = testapp_xdp_adjust_tail(test, value); - if (ret) - return ret; - - if (!test->adjust_tail_support) { - ksft_test_result_skip("%s %sResize pkt with bpf_xdp_adjust_tail() not supported\n", - mode_string(test), busy_poll_string(test)); - return TEST_SKIP; - } - - return 0; -} - -static int testapp_adjust_tail_shrink(struct test_spec *test) -{ - /* Shrink by 4 bytes for testing purpose */ - return testapp_adjust_tail(test, -4, MIN_PKT_SIZE * 2); -} - -static int testapp_adjust_tail_shrink_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - /* Shrink by the frag size */ - return testapp_adjust_tail(test, -XSK_UMEM__MAX_FRAME_SIZE, XSK_UMEM__LARGE_FRAME_SIZE * 2); -} - -static int testapp_adjust_tail_grow(struct test_spec *test) -{ - /* Grow by 4 bytes for testing purpose */ - return testapp_adjust_tail(test, 4, MIN_PKT_SIZE * 2); -} - -static int testapp_adjust_tail_grow_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - /* Grow by (frag_size - last_frag_Size) - 1 to stay inside the last fragment */ - return testapp_adjust_tail(test, (XSK_UMEM__MAX_FRAME_SIZE / 2) - 1, - XSK_UMEM__LARGE_FRAME_SIZE * 2); -} - -static int testapp_tx_queue_consumer(struct test_spec *test) -{ - int nr_packets; - - if (test->mode == TEST_MODE_ZC) { - ksft_test_result_skip("Can not run TX_QUEUE_CONSUMER test for ZC mode\n"); - return TEST_SKIP; - } - - nr_packets = MAX_TX_BUDGET_DEFAULT + 1; - pkt_stream_replace(test, nr_packets, MIN_PKT_SIZE); - test->ifobj_tx->xsk->batch_size = nr_packets; - test->ifobj_tx->xsk->check_consumer = true; - - return testapp_validate_traffic(test); -} - -static void run_pkt_test(struct test_spec *test) -{ - int ret; - - ret = test->test_func(test); - - if (ret == TEST_PASS) - ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), - test->name); - pkt_stream_restore_default(test); -} - -static struct ifobject *ifobject_create(void) -{ - struct ifobject *ifobj; - - ifobj = calloc(1, sizeof(struct ifobject)); - if (!ifobj) - return NULL; - - ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); - if (!ifobj->xsk_arr) - goto out_xsk_arr; - - ifobj->umem = calloc(1, sizeof(*ifobj->umem)); - if (!ifobj->umem) - goto out_umem; - - return ifobj; - -out_umem: - free(ifobj->xsk_arr); -out_xsk_arr: - free(ifobj); - return NULL; -} - -static void ifobject_delete(struct ifobject *ifobj) -{ - free(ifobj->umem); - free(ifobj->xsk_arr); - free(ifobj); -} - -static bool is_xdp_supported(int ifindex) -{ - int flags = XDP_FLAGS_DRV_MODE; - - LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); - struct bpf_insn insns[2] = { - BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), - BPF_EXIT_INSN() - }; - int prog_fd, insn_cnt = ARRAY_SIZE(insns); - int err; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); - if (prog_fd < 0) - return false; - - err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); - if (err) { - close(prog_fd); - return false; - } - - bpf_xdp_detach(ifindex, flags, NULL); - close(prog_fd); - - return true; -} - -static const struct test_spec tests[] = { - {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, - {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, - {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, - {.name = "POLL_RX", .test_func = testapp_poll_rx}, - {.name = "POLL_TX", .test_func = testapp_poll_tx}, - {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, - {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, - {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, - {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, - {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, - {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, - {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", - .test_func = testapp_unaligned_inv_desc_4001_frame}, - {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, - {.name = "TEARDOWN", .test_func = testapp_teardown}, - {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, - {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, - {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, - {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, - {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, - {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, - {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, - {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, - {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, - {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, - {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, - {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", - .test_func = testapp_send_receive_unaligned_mb}, - {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, - {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, - {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, - {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, - {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, - {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, - {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, - {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, - {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, - {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, - }; - static void print_tests(void) { u32 i; @@ -2833,7 +376,7 @@ int main(int argc, char **argv) init_iface(ifobj_rx, worker_testapp_validate_rx); init_iface(ifobj_tx, worker_testapp_validate_tx); - test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); + test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); if (!tx_pkt_stream_default || !rx_pkt_stream_default) @@ -2868,7 +411,7 @@ int main(int argc, char **argv) if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) continue; - test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); + test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); run_pkt_test(&test); usleep(USLEEP_MAX); diff --git a/tools/testing/selftests/bpf/xskxceiver.h b/tools/testing/selftests/bpf/xskxceiver.h index 4df3a5d329acf..3ca518df23ada 100644 --- a/tools/testing/selftests/bpf/xskxceiver.h +++ b/tools/testing/selftests/bpf/xskxceiver.h @@ -22,169 +22,13 @@ #define PF_XDP AF_XDP #endif -#ifndef SO_BUSY_POLL_BUDGET -#define SO_BUSY_POLL_BUDGET 70 -#endif - -#ifndef SO_PREFER_BUSY_POLL -#define SO_PREFER_BUSY_POLL 69 -#endif - -#define TEST_PASS 0 -#define TEST_FAILURE -1 -#define TEST_CONTINUE 1 -#define TEST_SKIP 2 -#define MAX_INTERFACES 2 -#define MAX_INTERFACE_NAME_CHARS 16 -#define MAX_TEST_NAME_SIZE 48 #define MAX_TEARDOWN_ITER 10 -#define PKT_HDR_SIZE (sizeof(struct ethhdr) + 2) /* Just to align the data in the packet */ -#define MIN_PKT_SIZE 64 -#define MAX_ETH_PKT_SIZE 1518 #define MAX_ETH_JUMBO_SIZE 9000 -#define USLEEP_MAX 10000 #define SOCK_RECONF_CTR 10 -#define DEFAULT_BATCH_SIZE 64 -#define POLL_TMOUT 1000 -#define THREAD_TMOUT 3 -#define DEFAULT_PKT_CNT (4 * 1024) -#define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) #define RX_FULL_RXQSIZE 32 #define UMEM_HEADROOM_TEST_SIZE 128 #define XSK_UMEM__INVALID_FRAME_SIZE (MAX_ETH_JUMBO_SIZE + 1) -#define XSK_UMEM__LARGE_FRAME_SIZE (3 * 1024) -#define XSK_UMEM__MAX_FRAME_SIZE (4 * 1024) -#define XSK_DESC__INVALID_OPTION (0xffff) -#define HUGEPAGE_SIZE (2 * 1024 * 1024) -#define PKT_DUMP_NB_TO_PRINT 16 #define RUN_ALL_TESTS UINT_MAX #define NUM_MAC_ADDRESSES 4 -#define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) - -enum test_mode { - TEST_MODE_SKB, - TEST_MODE_DRV, - TEST_MODE_ZC, - TEST_MODE_ALL -}; - -struct xsk_umem_info { - struct xsk_ring_prod fq; - struct xsk_ring_cons cq; - struct xsk_umem *umem; - u64 next_buffer; - u32 num_frames; - u32 frame_headroom; - void *buffer; - u32 frame_size; - u32 base_addr; - u32 fill_size; - u32 comp_size; - bool unaligned_mode; -}; - -struct xsk_socket_info { - struct xsk_ring_cons rx; - struct xsk_ring_prod tx; - struct xsk_umem_info *umem; - struct xsk_socket *xsk; - struct pkt_stream *pkt_stream; - u32 outstanding_tx; - u32 rxqsize; - u32 batch_size; - u8 dst_mac[ETH_ALEN]; - u8 src_mac[ETH_ALEN]; - bool check_consumer; -}; - -struct pkt { - int offset; - u32 len; - u32 pkt_nb; - bool valid; - u16 options; -}; - -struct pkt_stream { - u32 nb_pkts; - u32 current_pkt_nb; - struct pkt *pkts; - u32 max_pkt_len; - u32 nb_rx_pkts; - u32 nb_valid_entries; - bool verbatim; -}; - -struct set_hw_ring { - u32 default_tx; - u32 default_rx; -}; - -struct ifobject; -struct test_spec; -typedef int (*validation_func_t)(struct ifobject *ifobj); -typedef void *(*thread_func_t)(void *arg); -typedef int (*test_func_t)(struct test_spec *test); - -struct ifobject { - char ifname[MAX_INTERFACE_NAME_CHARS]; - struct xsk_socket_info *xsk; - struct xsk_socket_info *xsk_arr; - struct xsk_umem_info *umem; - thread_func_t func_ptr; - validation_func_t validation_func; - struct xsk_xdp_progs *xdp_progs; - struct bpf_map *xskmap; - struct bpf_program *xdp_prog; - struct ethtool_ringparam ring; - struct set_hw_ring set_ring; - enum test_mode mode; - int ifindex; - int mtu; - u32 bind_flags; - u32 xdp_zc_max_segs; - bool tx_on; - bool rx_on; - bool use_poll; - bool busy_poll; - bool use_fill_ring; - bool release_rx; - bool shared_umem; - bool use_metadata; - bool unaligned_supp; - bool multi_buff_supp; - bool multi_buff_zc_supp; - bool hw_ring_size_supp; -}; - -struct test_spec { - struct ifobject *ifobj_tx; - struct ifobject *ifobj_rx; - struct pkt_stream *tx_pkt_stream_default; - struct pkt_stream *rx_pkt_stream_default; - struct bpf_program *xdp_prog_rx; - struct bpf_program *xdp_prog_tx; - struct bpf_map *xskmap_rx; - struct bpf_map *xskmap_tx; - test_func_t test_func; - int mtu; - u16 total_steps; - u16 current_step; - u16 nb_sockets; - bool fail; - bool set_ring; - bool adjust_tail; - bool adjust_tail_support; - enum test_mode mode; - char name[MAX_TEST_NAME_SIZE]; -}; - -pthread_barrier_t barr; -pthread_mutex_t pacing_mutex = PTHREAD_MUTEX_INITIALIZER; - -int pkts_in_flight; - -static const u8 g_mac[ETH_ALEN] = {0x55, 0x44, 0x33, 0x22, 0x11, 0x00}; - #endif /* XSKXCEIVER_H_ */ diff --git a/tools/testing/selftests/drivers/net/bonding/bond_macvlan_ipvlan.sh b/tools/testing/selftests/drivers/net/bonding/bond_macvlan_ipvlan.sh index c4711272fe45d..559f300f965aa 100755 --- a/tools/testing/selftests/drivers/net/bonding/bond_macvlan_ipvlan.sh +++ b/tools/testing/selftests/drivers/net/bonding/bond_macvlan_ipvlan.sh @@ -30,6 +30,7 @@ check_connection() local message=${3} RET=0 + sleep 0.25 ip netns exec ${ns} ping ${target} -c 4 -i 0.1 &>/dev/null check_err $? "ping failed" log_test "${bond_mode}/${xvlan_type}_${xvlan_mode}: ${message}" diff --git a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py index fb010a48a5a19..a86f5c311fdcb 100644 --- a/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py +++ b/tools/testing/selftests/drivers/net/hw/lib/py/__init__.py @@ -22,7 +22,7 @@ NlError, RtnlFamily, DevlinkFamily, PSPFamily from net.lib.py import CmdExitFailure from net.lib.py import bkg, cmd, bpftool, bpftrace, defer, ethtool, \ - fd_read_timeout, ip, rand_port, wait_port_listen, wait_file + fd_read_timeout, ip, rand_port, wait_port_listen, wait_file, tool from net.lib.py import KsftSkipEx, KsftFailEx, KsftXfailEx from net.lib.py import ksft_disruptive, ksft_exit, ksft_pr, ksft_run, \ ksft_setup @@ -37,7 +37,7 @@ "CmdExitFailure", "bkg", "cmd", "bpftool", "bpftrace", "defer", "ethtool", "fd_read_timeout", "ip", "rand_port", - "wait_port_listen", "wait_file", + "wait_port_listen", "wait_file", "tool", "KsftSkipEx", "KsftFailEx", "KsftXfailEx", "ksft_disruptive", "ksft_exit", "ksft_pr", "ksft_run", "ksft_setup", diff --git a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh index 87f89fd92f8c1..64d3941576d5d 100644 --- a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh +++ b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh @@ -247,9 +247,9 @@ function listen_port_and_save_to() { SOCAT_MODE="UDP6-LISTEN" fi - # Just wait for 2 seconds - timeout 2 ip netns exec "${NAMESPACE}" \ - socat "${SOCAT_MODE}":"${PORT}",fork "${OUTPUT}" + # Just wait for 3 seconds + timeout 3 ip netns exec "${NAMESPACE}" \ + socat "${SOCAT_MODE}":"${PORT}",fork "${OUTPUT}" 2> /dev/null } # Only validate that the message arrived properly diff --git a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh index 0441a18f098b1..aac8ef490feb8 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh @@ -317,7 +317,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 0.5kbit burst 1m conform-exceed drop/ok + action police rate 0.5kbit burst 2k conform-exceed drop/ok check_fail $? "Incorrect success to add police action with too low rate" tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ @@ -327,7 +327,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 1.5kbit burst 1m conform-exceed drop/ok + action police rate 1.5kbit burst 2k conform-exceed drop/ok check_err $? "Failed to add police action with low rate" tc filter del dev $swp1 ingress protocol ip pref 1 handle 101 flower diff --git a/tools/testing/selftests/drivers/net/psp.py b/tools/testing/selftests/drivers/net/psp.py index 4ae7a785ff10a..473573e216e34 100755 --- a/tools/testing/selftests/drivers/net/psp.py +++ b/tools/testing/selftests/drivers/net/psp.py @@ -560,8 +560,9 @@ def psp_ip_ver_test_builder(name, test_func, psp_ver, ipver): """Build test cases for each combo of PSP version and IP version""" def test_case(cfg): cfg.require_ipver(ipver) - test_case.__name__ = f"{name}_v{psp_ver}_ip{ipver}" test_func(cfg, psp_ver, ipver) + + test_case.__name__ = f"{name}_v{psp_ver}_ip{ipver}" return test_case @@ -569,8 +570,9 @@ def ipver_test_builder(name, test_func, ipver): """Build test cases for each IP version""" def test_case(cfg): cfg.require_ipver(ipver) - test_case.__name__ = f"{name}_ip{ipver}" test_func(cfg, ipver) + + test_case.__name__ = f"{name}_ip{ipver}" return test_case diff --git a/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c b/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c index a3d8015897e98..40fb08a170990 100644 --- a/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c +++ b/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c @@ -37,17 +37,20 @@ FIXTURE(iterate_mount_namespaces) { __u64 mnt_ns_id[MNT_NS_COUNT]; }; +static inline bool mntns_in_list(__u64 *mnt_ns_id, struct mnt_ns_info *info) +{ + for (int i = 0; i < MNT_NS_COUNT; i++) { + if (mnt_ns_id[i] == info->mnt_ns_id) + return true; + } + return false; +} + FIXTURE_SETUP(iterate_mount_namespaces) { for (int i = 0; i < MNT_NS_COUNT; i++) self->fd_mnt_ns[i] = -EBADF; - /* - * Creating a new user namespace let's us guarantee that we only see - * mount namespaces that we did actually create. - */ - ASSERT_EQ(unshare(CLONE_NEWUSER), 0); - for (int i = 0; i < MNT_NS_COUNT; i++) { struct mnt_ns_info info = {}; @@ -75,13 +78,15 @@ TEST_F(iterate_mount_namespaces, iterate_all_forward) fd_mnt_ns_cur = fcntl(self->fd_mnt_ns[0], F_DUPFD_CLOEXEC); ASSERT_GE(fd_mnt_ns_cur, 0); - for (;; count++) { + for (;;) { struct mnt_ns_info info = {}; int fd_mnt_ns_next; fd_mnt_ns_next = ioctl(fd_mnt_ns_cur, NS_MNT_GET_NEXT, &info); if (fd_mnt_ns_next < 0 && errno == ENOENT) break; + if (mntns_in_list(self->mnt_ns_id, &info)) + count++; ASSERT_GE(fd_mnt_ns_next, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_next; @@ -96,13 +101,15 @@ TEST_F(iterate_mount_namespaces, iterate_all_backwards) fd_mnt_ns_cur = fcntl(self->fd_mnt_ns[MNT_NS_LAST_INDEX], F_DUPFD_CLOEXEC); ASSERT_GE(fd_mnt_ns_cur, 0); - for (;; count++) { + for (;;) { struct mnt_ns_info info = {}; int fd_mnt_ns_prev; fd_mnt_ns_prev = ioctl(fd_mnt_ns_cur, NS_MNT_GET_PREV, &info); if (fd_mnt_ns_prev < 0 && errno == ENOENT) break; + if (mntns_in_list(self->mnt_ns_id, &info)) + count++; ASSERT_GE(fd_mnt_ns_prev, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_prev; @@ -125,7 +132,6 @@ TEST_F(iterate_mount_namespaces, iterate_forward) ASSERT_GE(fd_mnt_ns_next, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_next; - ASSERT_EQ(info.mnt_ns_id, self->mnt_ns_id[i]); } } @@ -144,7 +150,6 @@ TEST_F(iterate_mount_namespaces, iterate_backward) ASSERT_GE(fd_mnt_ns_prev, 0); ASSERT_EQ(close(fd_mnt_ns_cur), 0); fd_mnt_ns_cur = fd_mnt_ns_prev; - ASSERT_EQ(info.mnt_ns_id, self->mnt_ns_id[i]); } } diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc index aee22289536b1..1b57771dbfdf0 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc @@ -90,9 +90,10 @@ if [ $on != "0" ]; then fail "Tracing is not off" fi -csum1=`md5sum trace` +# Cannot rely on names being around as they are only cached, strip them +csum1=`cat trace | sed -e 's/^ *[^ ]*\(-[0-9][0-9]*\)/\1/' | md5sum` sleep $SLEEP_TIME -csum2=`md5sum trace` +csum2=`cat trace | sed -e 's/^ *[^ ]*\(-[0-9][0-9]*\)/\1/' | md5sum` if [ "$csum1" != "$csum2" ]; then fail "Tracing file is still changing" diff --git a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h index 531228b849dae..125a975f32f9e 100644 --- a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h +++ b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h @@ -6,8 +6,10 @@ #define __HID_BPF_HELPERS_H /* "undefine" structs and enums in vmlinux.h, because we "override" them below */ +#define bpf_wq bpf_wq___not_used #define hid_bpf_ctx hid_bpf_ctx___not_used #define hid_bpf_ops hid_bpf_ops___not_used +#define hid_device hid_device___not_used #define hid_report_type hid_report_type___not_used #define hid_class_request hid_class_request___not_used #define hid_bpf_attach_flags hid_bpf_attach_flags___not_used @@ -27,8 +29,10 @@ #include "vmlinux.h" +#undef bpf_wq #undef hid_bpf_ctx #undef hid_bpf_ops +#undef hid_device #undef hid_report_type #undef hid_class_request #undef hid_bpf_attach_flags @@ -55,6 +59,14 @@ enum hid_report_type { HID_REPORT_TYPES, }; +struct hid_device { + unsigned int id; +} __attribute__((preserve_access_index)); + +struct bpf_wq { + __u64 __opaque[2]; +}; + struct hid_bpf_ctx { struct hid_device *hid; __u32 allocated_size; diff --git a/tools/testing/selftests/iommu/iommufd.c b/tools/testing/selftests/iommu/iommufd.c index bb4d33dde3c89..1f52140d9d9ee 100644 --- a/tools/testing/selftests/iommu/iommufd.c +++ b/tools/testing/selftests/iommu/iommufd.c @@ -758,9 +758,6 @@ TEST_F(iommufd_ioas, get_hw_info) struct iommu_test_hw_info info; uint64_t trailing_bytes; } buffer_larger; - struct iommu_test_hw_info_buffer_smaller { - __u32 flags; - } buffer_smaller; if (self->device_id) { uint8_t max_pasid = 0; @@ -792,8 +789,9 @@ TEST_F(iommufd_ioas, get_hw_info) * the fields within the size range still gets updated. */ test_cmd_get_hw_info(self->device_id, - IOMMU_HW_INFO_TYPE_DEFAULT, - &buffer_smaller, sizeof(buffer_smaller)); + IOMMU_HW_INFO_TYPE_DEFAULT, &buffer_exact, + offsetofend(struct iommu_test_hw_info, + flags)); test_cmd_get_hw_info_pasid(self->device_id, &max_pasid); ASSERT_EQ(0, max_pasid); if (variant->pasid_capable) { diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 3f66e862e83eb..fe162cbfc0912 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -70,6 +70,15 @@ #include "kselftest.h" +static inline void __kselftest_memset_safe(void *s, int c, size_t n) +{ + if (n > 0) + memset(s, c, n); +} + +#define KSELFTEST_PRIO_TEST_F 20000 +#define KSELFTEST_PRIO_XFAIL 20001 + #define TEST_TIMEOUT_DEFAULT 30 /* Utilities exposed to the test definitions */ @@ -416,7 +425,7 @@ self = mmap(NULL, sizeof(*self), PROT_READ | PROT_WRITE, \ MAP_SHARED | MAP_ANONYMOUS, -1, 0); \ } else { \ - memset(&self_private, 0, sizeof(self_private)); \ + __kselftest_memset_safe(&self_private, 0, sizeof(self_private)); \ self = &self_private; \ } \ } \ @@ -459,7 +468,7 @@ fixture_name##_teardown(_metadata, self, variant); \ } \ static struct __test_metadata *_##fixture_name##_##test_name##_object; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_TEST_F))) \ _register_##fixture_name##_##test_name(void) \ { \ struct __test_metadata *object = mmap(NULL, sizeof(*object), \ @@ -874,7 +883,7 @@ struct __test_xfail { .fixture = &_##fixture_name##_fixture_object, \ .variant = &_##fixture_name##_##variant_name##_object, \ }; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_XFAIL))) \ _register_##fixture_name##_##variant_name##_##test_name##_xfail(void) \ { \ _##fixture_name##_##variant_name##_##test_name##_xfail.test = \ diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index d9fffe06d3eae..f2b223072b626 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -6,7 +6,7 @@ ARCH ?= $(SUBARCH) ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch)) # Top-level selftests allows ARCH=x86_64 :-( ifeq ($(ARCH),x86_64) - ARCH := x86 + override ARCH := x86 endif include Makefile.kvm else diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 148d427ff24be..1715efe9d5b0d 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -245,6 +245,7 @@ LINUX_TOOL_INCLUDE = $(top_srcdir)/tools/include LINUX_TOOL_ARCH_INCLUDE = $(top_srcdir)/tools/arch/$(ARCH)/include CFLAGS += -Wall -Wstrict-prototypes -Wuninitialized -O2 -g -std=gnu99 \ -Wno-gnu-variable-sized-type-not-at-end -MD -MP -DCONFIG_64BIT \ + -U_FORTIFY_SOURCE \ -fno-builtin-memcmp -fno-builtin-memcpy \ -fno-builtin-memset -fno-builtin-strnlen \ -fno-stack-protector -fno-PIE -fno-strict-aliasing \ diff --git a/tools/testing/selftests/kvm/rseq_test.c b/tools/testing/selftests/kvm/rseq_test.c index 1375fca80bcdb..f80ad6b47d16b 100644 --- a/tools/testing/selftests/kvm/rseq_test.c +++ b/tools/testing/selftests/kvm/rseq_test.c @@ -215,6 +215,7 @@ int main(int argc, char *argv[]) switch (opt) { case 'u': skip_sanity_check = true; + break; case 'l': latency = atoi_paranoid(optarg); break; diff --git a/tools/testing/selftests/kvm/x86/amx_test.c b/tools/testing/selftests/kvm/x86/amx_test.c index f4ce5a185a7dd..bee56c1f78339 100644 --- a/tools/testing/selftests/kvm/x86/amx_test.c +++ b/tools/testing/selftests/kvm/x86/amx_test.c @@ -124,6 +124,20 @@ static void set_tilecfg(struct tile_config *cfg) } } +enum { + /* Retrieve TMM0 from guest, stash it for TEST_RESTORE_TILEDATA */ + TEST_SAVE_TILEDATA = 1, + + /* Check TMM0 against tiledata */ + TEST_COMPARE_TILEDATA = 2, + + /* Restore TMM0 from earlier save */ + TEST_RESTORE_TILEDATA = 4, + + /* Full VM save/restore */ + TEST_SAVE_RESTORE = 8, +}; + static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, struct tile_data *tiledata, struct xstate *xstate) @@ -131,20 +145,29 @@ static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE) && this_cpu_has(X86_FEATURE_OSXSAVE)); check_xtile_info(); - GUEST_SYNC(1); + GUEST_SYNC(TEST_SAVE_RESTORE); /* xfd=0, enable amx */ wrmsr(MSR_IA32_XFD, 0); - GUEST_SYNC(2); + GUEST_SYNC(TEST_SAVE_RESTORE); GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0); set_tilecfg(amx_cfg); __ldtilecfg(amx_cfg); - GUEST_SYNC(3); + GUEST_SYNC(TEST_SAVE_RESTORE); /* Check save/restore when trap to userspace */ __tileloadd(tiledata); - GUEST_SYNC(4); + GUEST_SYNC(TEST_SAVE_TILEDATA | TEST_COMPARE_TILEDATA | TEST_SAVE_RESTORE); + + /* xfd=0x40000, disable amx tiledata */ + wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILE_DATA); + + /* host tries setting tiledata while guest XFD is set */ + GUEST_SYNC(TEST_RESTORE_TILEDATA); + GUEST_SYNC(TEST_SAVE_RESTORE); + + wrmsr(MSR_IA32_XFD, 0); __tilerelease(); - GUEST_SYNC(5); + GUEST_SYNC(TEST_SAVE_RESTORE); /* * After XSAVEC, XTILEDATA is cleared in the xstate_bv but is set in * the xcomp_bv. @@ -154,6 +177,8 @@ static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, GUEST_ASSERT(!(xstate->header.xstate_bv & XFEATURE_MASK_XTILE_DATA)); GUEST_ASSERT(xstate->header.xcomp_bv & XFEATURE_MASK_XTILE_DATA); + /* #NM test */ + /* xfd=0x40000, disable amx tiledata */ wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILE_DATA); @@ -166,13 +191,13 @@ static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, GUEST_ASSERT(!(xstate->header.xstate_bv & XFEATURE_MASK_XTILE_DATA)); GUEST_ASSERT((xstate->header.xcomp_bv & XFEATURE_MASK_XTILE_DATA)); - GUEST_SYNC(6); + GUEST_SYNC(TEST_SAVE_RESTORE); GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA); set_tilecfg(amx_cfg); __ldtilecfg(amx_cfg); /* Trigger #NM exception */ __tileloadd(tiledata); - GUEST_SYNC(10); + GUEST_SYNC(TEST_COMPARE_TILEDATA | TEST_SAVE_RESTORE); GUEST_DONE(); } @@ -180,18 +205,18 @@ static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, void guest_nm_handler(struct ex_regs *regs) { /* Check if #NM is triggered by XFEATURE_MASK_XTILE_DATA */ - GUEST_SYNC(7); + GUEST_SYNC(TEST_SAVE_RESTORE); GUEST_ASSERT(!(get_cr0() & X86_CR0_TS)); GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILE_DATA); GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA); - GUEST_SYNC(8); + GUEST_SYNC(TEST_SAVE_RESTORE); GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILE_DATA); GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA); /* Clear xfd_err */ wrmsr(MSR_IA32_XFD_ERR, 0); /* xfd=0, enable amx */ wrmsr(MSR_IA32_XFD, 0); - GUEST_SYNC(9); + GUEST_SYNC(TEST_SAVE_RESTORE); } int main(int argc, char *argv[]) @@ -200,10 +225,10 @@ int main(int argc, char *argv[]) struct kvm_vcpu *vcpu; struct kvm_vm *vm; struct kvm_x86_state *state; + struct kvm_x86_state *tile_state = NULL; int xsave_restore_size; vm_vaddr_t amx_cfg, tiledata, xstate; struct ucall uc; - u32 amx_offset; int ret; /* @@ -244,6 +269,7 @@ int main(int argc, char *argv[]) memset(addr_gva2hva(vm, xstate), 0, PAGE_SIZE * DIV_ROUND_UP(XSAVE_SIZE, PAGE_SIZE)); vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xstate); + int iter = 0; for (;;) { vcpu_run(vcpu); TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); @@ -253,37 +279,47 @@ int main(int argc, char *argv[]) REPORT_GUEST_ASSERT(uc); /* NOT REACHED */ case UCALL_SYNC: - switch (uc.args[1]) { - case 1: - case 2: - case 3: - case 5: - case 6: - case 7: - case 8: - fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]); - break; - case 4: - case 10: - fprintf(stderr, - "GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]); + ++iter; + if (uc.args[1] & TEST_SAVE_TILEDATA) { + fprintf(stderr, "GUEST_SYNC #%d, save tiledata\n", iter); + tile_state = vcpu_save_state(vcpu); + } + if (uc.args[1] & TEST_COMPARE_TILEDATA) { + fprintf(stderr, "GUEST_SYNC #%d, check TMM0 contents\n", iter); /* Compacted mode, get amx offset by xsave area * size subtract 8K amx size. */ - amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE; - state = vcpu_save_state(vcpu); - void *amx_start = (void *)state->xsave + amx_offset; + u32 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE; + void *amx_start = (void *)tile_state->xsave + amx_offset; void *tiles_data = (void *)addr_gva2hva(vm, tiledata); /* Only check TMM0 register, 1 tile */ ret = memcmp(amx_start, tiles_data, TILE_SIZE); TEST_ASSERT(ret == 0, "memcmp failed, ret=%d", ret); + } + if (uc.args[1] & TEST_RESTORE_TILEDATA) { + fprintf(stderr, "GUEST_SYNC #%d, before KVM_SET_XSAVE\n", iter); + vcpu_xsave_set(vcpu, tile_state->xsave); + fprintf(stderr, "GUEST_SYNC #%d, after KVM_SET_XSAVE\n", iter); + } + if (uc.args[1] & TEST_SAVE_RESTORE) { + fprintf(stderr, "GUEST_SYNC #%d, save/restore VM state\n", iter); + state = vcpu_save_state(vcpu); + memset(®s1, 0, sizeof(regs1)); + vcpu_regs_get(vcpu, ®s1); + + kvm_vm_release(vm); + + /* Restore state in a new VM. */ + vcpu = vm_recreate_with_one_vcpu(vm); + vcpu_load_state(vcpu, state); kvm_x86_state_cleanup(state); - break; - case 9: - fprintf(stderr, - "GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]); - break; + + memset(®s2, 0, sizeof(regs2)); + vcpu_regs_get(vcpu, ®s2); + TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), + "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", + (ulong) regs2.rdi, (ulong) regs2.rsi); } break; case UCALL_DONE: @@ -293,22 +329,6 @@ int main(int argc, char *argv[]) TEST_FAIL("Unknown ucall %lu", uc.cmd); } - state = vcpu_save_state(vcpu); - memset(®s1, 0, sizeof(regs1)); - vcpu_regs_get(vcpu, ®s1); - - kvm_vm_release(vm); - - /* Restore state in a new VM. */ - vcpu = vm_recreate_with_one_vcpu(vm); - vcpu_load_state(vcpu, state); - kvm_x86_state_cleanup(state); - - memset(®s2, 0, sizeof(regs2)); - vcpu_regs_get(vcpu, ®s2); - TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), - "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); } done: kvm_vm_free(vm); diff --git a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c index 0a6dfba3905b6..6b0928e69051d 100644 --- a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c +++ b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c @@ -36,8 +36,6 @@ static struct kvm_vm *sev_vm_create(bool es) sev_vm_launch(vm, es ? SEV_POLICY_ES : 0); - if (es) - vm_sev_ioctl(vm, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL); return vm; } diff --git a/tools/testing/selftests/landlock/Makefile b/tools/testing/selftests/landlock/Makefile index a3f449914bf93..044b83bde16eb 100644 --- a/tools/testing/selftests/landlock/Makefile +++ b/tools/testing/selftests/landlock/Makefile @@ -4,7 +4,7 @@ CFLAGS += -Wall -O2 $(KHDR_INCLUDES) -LOCAL_HDRS += common.h +LOCAL_HDRS += $(wildcard *.h) src_test := $(wildcard *_test.c) diff --git a/tools/testing/selftests/landlock/common.h b/tools/testing/selftests/landlock/common.h index 9acecae36f51b..98c2362954e21 100644 --- a/tools/testing/selftests/landlock/common.h +++ b/tools/testing/selftests/landlock/common.h @@ -237,6 +237,7 @@ struct service_fixture { struct sockaddr_un unix_addr; socklen_t unix_addr_len; }; + struct sockaddr_storage _largest; }; }; diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index fa0f18ec62c41..29cdbb8367358 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -4375,9 +4375,6 @@ TEST_F_FORK(layout1, named_unix_domain_socket_ioctl) cli_fd = socket(AF_UNIX, SOCK_STREAM, 0); ASSERT_LE(0, cli_fd); - size = offsetof(struct sockaddr_un, sun_path) + strlen(cli_un.sun_path); - ASSERT_EQ(0, bind(cli_fd, (struct sockaddr *)&cli_un, size)); - bzero(&cli_un, sizeof(cli_un)); cli_un.sun_family = AF_UNIX; strncpy(cli_un.sun_path, path, sizeof(cli_un.sun_path)); @@ -4388,7 +4385,8 @@ TEST_F_FORK(layout1, named_unix_domain_socket_ioctl) /* FIONREAD and other IOCTLs should not be forbidden. */ EXPECT_EQ(0, test_fionread_ioctl(cli_fd)); - ASSERT_EQ(0, close(cli_fd)); + EXPECT_EQ(0, close(cli_fd)); + EXPECT_EQ(0, close(srv_fd)); } /* clang-format off */ diff --git a/tools/testing/selftests/landlock/net_test.c b/tools/testing/selftests/landlock/net_test.c index 2a45208551e61..3bbc0508420b1 100644 --- a/tools/testing/selftests/landlock/net_test.c +++ b/tools/testing/selftests/landlock/net_test.c @@ -121,6 +121,10 @@ static socklen_t get_addrlen(const struct service_fixture *const srv, { switch (srv->protocol.domain) { case AF_UNSPEC: + if (minimal) + return sizeof(sa_family_t); + return sizeof(struct sockaddr_storage); + case AF_INET: return sizeof(srv->ipv4_addr); @@ -758,6 +762,11 @@ TEST_F(protocol, bind_unspec) bind_fd = socket_variant(&self->srv0); ASSERT_LE(0, bind_fd); + /* Tries to bind with too small addrlen. */ + EXPECT_EQ(-EINVAL, bind_variant_addrlen( + bind_fd, &self->unspec_any0, + get_addrlen(&self->unspec_any0, true) - 1)); + /* Allowed bind on AF_UNSPEC/INADDR_ANY. */ ret = bind_variant(bind_fd, &self->unspec_any0); if (variant->prot.domain == AF_INET) { @@ -766,6 +775,8 @@ TEST_F(protocol, bind_unspec) TH_LOG("Failed to bind to unspec/any socket: %s", strerror(errno)); } + } else if (variant->prot.domain == AF_INET6) { + EXPECT_EQ(-EAFNOSUPPORT, ret); } else { EXPECT_EQ(-EINVAL, ret); } @@ -792,6 +803,8 @@ TEST_F(protocol, bind_unspec) } else { EXPECT_EQ(0, ret); } + } else if (variant->prot.domain == AF_INET6) { + EXPECT_EQ(-EAFNOSUPPORT, ret); } else { EXPECT_EQ(-EINVAL, ret); } @@ -801,7 +814,8 @@ TEST_F(protocol, bind_unspec) bind_fd = socket_variant(&self->srv0); ASSERT_LE(0, bind_fd); ret = bind_variant(bind_fd, &self->unspec_srv0); - if (variant->prot.domain == AF_INET) { + if (variant->prot.domain == AF_INET || + variant->prot.domain == AF_INET6) { EXPECT_EQ(-EAFNOSUPPORT, ret); } else { EXPECT_EQ(-EINVAL, ret) diff --git a/tools/testing/selftests/memfd/memfd_test.c b/tools/testing/selftests/memfd/memfd_test.c index 5b993924cc3f5..2ca07ea7202a5 100644 --- a/tools/testing/selftests/memfd/memfd_test.c +++ b/tools/testing/selftests/memfd/memfd_test.c @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include #include @@ -39,6 +42,20 @@ F_SEAL_EXEC) #define MFD_NOEXEC_SEAL 0x0008U +union semun { + int val; + struct semid_ds *buf; + unsigned short int *array; + struct seminfo *__buf; +}; + +/* + * we use semaphores on nested wait tasks due the use of CLONE_NEWPID: the + * child will be PID 1 and can't send SIGSTOP to themselves due special + * treatment of the init task, so the SIGSTOP/SIGCONT synchronization + * approach can't be used here. + */ +#define SEM_KEY 0xdeadbeef /* * Default is not to test hugetlbfs @@ -1333,8 +1350,22 @@ static int sysctl_nested(void *arg) static int sysctl_nested_wait(void *arg) { - /* Wait for a SIGCONT. */ - kill(getpid(), SIGSTOP); + int sem = semget(SEM_KEY, 1, 0600); + struct sembuf sembuf; + + if (sem < 0) { + perror("semget:"); + abort(); + } + sembuf.sem_num = 0; + sembuf.sem_flg = 0; + sembuf.sem_op = 0; + + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + abort(); + } + return sysctl_nested(arg); } @@ -1355,7 +1386,9 @@ static void test_sysctl_sysctl2_failset(void) static int sysctl_nested_child(void *arg) { - int pid; + int pid, sem; + union semun semun; + struct sembuf sembuf; printf("%s nested sysctl 0\n", memfd_str); sysctl_assert_write("0"); @@ -1389,23 +1422,53 @@ static int sysctl_nested_child(void *arg) test_sysctl_sysctl2_failset); join_thread(pid); + sem = semget(SEM_KEY, 1, IPC_CREAT | 0600); + if (sem < 0) { + perror("semget:"); + return 1; + } + semun.val = 1; + sembuf.sem_op = -1; + sembuf.sem_flg = 0; + sembuf.sem_num = 0; + /* Verify that the rules are actually inherited after fork. */ printf("%s nested sysctl 0 -> 1 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1_failset); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 0 -> 2 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2_failset); sysctl_assert_write("2"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); /* @@ -1415,28 +1478,62 @@ static int sysctl_nested_child(void *arg) */ printf("%s nested sysctl 2 -> 1 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 2 -> 0 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("0"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 1 -> 0 after fork\n", memfd_str); sysctl_assert_write("1"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1); sysctl_assert_write("0"); - kill(pid, SIGCONT); + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); + semctl(sem, 0, IPC_RMID); + return 0; } diff --git a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh index e1fe16bcbbe88..fa6713892d82d 100755 --- a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh +++ b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh @@ -290,7 +290,7 @@ function run_test() { setup_cgroup "hugetlb_cgroup_test" "$cgroup_limit" "$reservation_limit" mkdir -p /mnt/huge - mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M none /mnt/huge write_hugetlbfs_and_get_usage "hugetlb_cgroup_test" "$size" "$populate" \ "$write" "/mnt/huge/test" "$method" "$private" "$expect_failure" \ @@ -344,7 +344,7 @@ function run_multiple_cgroup_test() { setup_cgroup "hugetlb_cgroup_test2" "$cgroup_limit2" "$reservation_limit2" mkdir -p /mnt/huge - mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M none /mnt/huge write_hugetlbfs_and_get_usage "hugetlb_cgroup_test1" "$size1" \ "$populate1" "$write1" "/mnt/huge/test1" "$method" "$private" \ diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 6560c26f47d1f..0df61422467da 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -1612,8 +1612,8 @@ static void run_with_huge_zeropage(non_anon_test_fn fn, const char *desc) * the first sub-page and test if we get another sub-page populated * automatically. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); if (!pagemap_is_populated(pagemap_fd, mem + pagesize) || !pagemap_is_populated(pagemap_fd, smem + pagesize)) { ksft_test_result_skip("Did not get THPs populated\n"); @@ -1663,8 +1663,8 @@ static void run_with_memfd(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1719,8 +1719,8 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1773,8 +1773,8 @@ static void run_with_memfd_hugetlb(non_anon_test_fn fn, const char *desc, } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, hugetlbsize); munmap: diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index 268dadb8ce438..e15fdd5c62d68 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -179,7 +179,7 @@ static void do_test(int fd, size_t size, enum test_type type, bool shared) if (rw && shared && fs_is_unknown(fs_type)) { ksft_print_msg("Unknown filesystem\n"); result = KSFT_SKIP; - return; + break; } /* * R/O pinning or pinning in a private mapping is always diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index cc4253f47f10b..cd0e6cc06a025 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -22,12 +22,37 @@ FIXTURE(merge) struct procmap_fd procmap; }; +static char *map_carveout(unsigned int page_size) +{ + return mmap(NULL, 30 * page_size, PROT_NONE, + MAP_ANON | MAP_PRIVATE, -1, 0); +} + +static pid_t do_fork(struct procmap_fd *procmap) +{ + pid_t pid = fork(); + + if (pid == -1) + return -1; + if (pid != 0) { + wait(NULL); + return pid; + } + + /* Reopen for child. */ + if (close_procmap(procmap)) + return -1; + if (open_self_procmap(procmap)) + return -1; + + return 0; +} + FIXTURE_SETUP(merge) { self->page_size = psize(); /* Carve out PROT_NONE region to map over. */ - self->carveout = mmap(NULL, 30 * self->page_size, PROT_NONE, - MAP_ANON | MAP_PRIVATE, -1, 0); + self->carveout = map_carveout(self->page_size); ASSERT_NE(self->carveout, MAP_FAILED); /* Setup PROCMAP_QUERY interface. */ ASSERT_EQ(open_self_procmap(&self->procmap), 0); @@ -36,7 +61,8 @@ FIXTURE_SETUP(merge) FIXTURE_TEARDOWN(merge) { ASSERT_EQ(munmap(self->carveout, 30 * self->page_size), 0); - ASSERT_EQ(close_procmap(&self->procmap), 0); + /* May fail for parent of forked process. */ + close_procmap(&self->procmap); /* * Clear unconditionally, as some tests set this. It is no issue if this * fails (KSM may be disabled for instance). @@ -44,6 +70,44 @@ FIXTURE_TEARDOWN(merge) prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); } +FIXTURE(merge_with_fork) +{ + unsigned int page_size; + char *carveout; + struct procmap_fd procmap; +}; + +FIXTURE_VARIANT(merge_with_fork) +{ + bool forked; +}; + +FIXTURE_VARIANT_ADD(merge_with_fork, forked) +{ + .forked = true, +}; + +FIXTURE_VARIANT_ADD(merge_with_fork, unforked) +{ + .forked = false, +}; + +FIXTURE_SETUP(merge_with_fork) +{ + self->page_size = psize(); + self->carveout = map_carveout(self->page_size); + ASSERT_NE(self->carveout, MAP_FAILED); + ASSERT_EQ(open_self_procmap(&self->procmap), 0); +} + +FIXTURE_TEARDOWN(merge_with_fork) +{ + ASSERT_EQ(munmap(self->carveout, 30 * self->page_size), 0); + ASSERT_EQ(close_procmap(&self->procmap), 0); + /* See above. */ + prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); +} + TEST_F(merge, mprotect_unfaulted_left) { unsigned int page_size = self->page_size; @@ -322,8 +386,8 @@ TEST_F(merge, forked_target_vma) unsigned int page_size = self->page_size; char *carveout = self->carveout; struct procmap_fd *procmap = &self->procmap; - pid_t pid; char *ptr, *ptr2; + pid_t pid; int i; /* @@ -344,19 +408,10 @@ TEST_F(merge, forked_target_vma) */ ptr[0] = 'x'; - pid = fork(); + pid = do_fork(&self->procmap); ASSERT_NE(pid, -1); - - if (pid != 0) { - wait(NULL); + if (pid != 0) return; - } - - /* Child process below: */ - - /* Reopen for child. */ - ASSERT_EQ(close_procmap(&self->procmap), 0); - ASSERT_EQ(open_self_procmap(&self->procmap), 0); /* unCOWing everything does not cause the AVC to go away. */ for (i = 0; i < 5 * page_size; i += page_size) @@ -386,8 +441,8 @@ TEST_F(merge, forked_source_vma) unsigned int page_size = self->page_size; char *carveout = self->carveout; struct procmap_fd *procmap = &self->procmap; - pid_t pid; char *ptr, *ptr2; + pid_t pid; int i; /* @@ -408,19 +463,10 @@ TEST_F(merge, forked_source_vma) */ ptr[0] = 'x'; - pid = fork(); + pid = do_fork(&self->procmap); ASSERT_NE(pid, -1); - - if (pid != 0) { - wait(NULL); + if (pid != 0) return; - } - - /* Child process below: */ - - /* Reopen for child. */ - ASSERT_EQ(close_procmap(&self->procmap), 0); - ASSERT_EQ(open_self_procmap(&self->procmap), 0); /* unCOWing everything does not cause the AVC to go away. */ for (i = 0; i < 5 * page_size; i += page_size) @@ -1171,4 +1217,288 @@ TEST_F(merge, mremap_correct_placed_faulted) ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size); } +TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + unsigned long offset; + char *ptr_a, *ptr_b; + + /* + * mremap() such that A and B merge: + * + * |------------| + * | \ | + * |-----------| | / |---------| + * | unfaulted | v \ | faulted | + * |-----------| / |---------| + * B \ A + */ + + /* Map VMA A into place. */ + ptr_a = mmap(&self->carveout[page_size + 3 * page_size], + 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_a, MAP_FAILED); + /* Fault it in. */ + ptr_a[0] = 'x'; + + if (variant->forked) { + pid_t pid = do_fork(&self->procmap); + + ASSERT_NE(pid, -1); + if (pid != 0) + return; + } + + /* + * Now move it out of the way so we can place VMA B in position, + * unfaulted. + */ + ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]); + ASSERT_NE(ptr_a, MAP_FAILED); + + /* Map VMA B into place. */ + ptr_b = mmap(&self->carveout[page_size], 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_b, MAP_FAILED); + + /* + * Now move VMA A into position with MREMAP_DONTUNMAP to catch incorrect + * anon_vma propagation. + */ + ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, + &self->carveout[page_size + 3 * page_size]); + ASSERT_NE(ptr_a, MAP_FAILED); + + /* The VMAs should have merged, if not forked. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr_b)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_b); + + offset = variant->forked ? 3 * page_size : 6 * page_size; + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_b + offset); +} + +TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_next) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + unsigned long offset; + char *ptr_a, *ptr_b; + + /* + * mremap() such that A and B merge: + * + * |---------------------------| + * | \ | + * | |-----------| / |---------| + * v | unfaulted | \ | faulted | + * |-----------| / |---------| + * B \ A + * + * Then unmap VMA A to trigger the bug. + */ + + /* Map VMA A into place. */ + ptr_a = mmap(&self->carveout[page_size], 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_a, MAP_FAILED); + /* Fault it in. */ + ptr_a[0] = 'x'; + + if (variant->forked) { + pid_t pid = do_fork(&self->procmap); + + ASSERT_NE(pid, -1); + if (pid != 0) + return; + } + + /* + * Now move it out of the way so we can place VMA B in position, + * unfaulted. + */ + ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]); + ASSERT_NE(ptr_a, MAP_FAILED); + + /* Map VMA B into place. */ + ptr_b = mmap(&self->carveout[page_size + 3 * page_size], 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_b, MAP_FAILED); + + /* + * Now move VMA A into position with MREMAP_DONTUNMAP to catch incorrect + * anon_vma propagation. + */ + ptr_a = mremap(ptr_a, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, + &self->carveout[page_size]); + ASSERT_NE(ptr_a, MAP_FAILED); + + /* The VMAs should have merged, if not forked. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr_a)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_a); + offset = variant->forked ? 3 * page_size : 6 * page_size; + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_a + offset); +} + +TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev_unfaulted_next) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + unsigned long offset; + char *ptr_a, *ptr_b, *ptr_c; + + /* + * mremap() with MREMAP_DONTUNMAP such that A, B and C merge: + * + * |---------------------------| + * | \ | + * |-----------| | |-----------| / |---------| + * | unfaulted | v | unfaulted | \ | faulted | + * |-----------| |-----------| / |---------| + * A C \ B + */ + + /* Map VMA B into place. */ + ptr_b = mmap(&self->carveout[page_size + 3 * page_size], 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_b, MAP_FAILED); + /* Fault it in. */ + ptr_b[0] = 'x'; + + if (variant->forked) { + pid_t pid = do_fork(&self->procmap); + + ASSERT_NE(pid, -1); + if (pid != 0) + return; + } + + /* + * Now move it out of the way so we can place VMAs A, C in position, + * unfaulted. + */ + ptr_b = mremap(ptr_b, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]); + ASSERT_NE(ptr_b, MAP_FAILED); + + /* Map VMA A into place. */ + + ptr_a = mmap(&self->carveout[page_size], 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_a, MAP_FAILED); + + /* Map VMA C into place. */ + ptr_c = mmap(&self->carveout[page_size + 3 * page_size + 3 * page_size], + 3 * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_c, MAP_FAILED); + + /* + * Now move VMA B into position with MREMAP_DONTUNMAP to catch incorrect + * anon_vma propagation. + */ + ptr_b = mremap(ptr_b, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, + &self->carveout[page_size + 3 * page_size]); + ASSERT_NE(ptr_b, MAP_FAILED); + + /* The VMAs should have merged, if not forked. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr_a)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_a); + offset = variant->forked ? 3 * page_size : 9 * page_size; + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_a + offset); + + /* If forked, B and C should also not have merged. */ + if (variant->forked) { + ASSERT_TRUE(find_vma_procmap(procmap, ptr_b)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_b); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_b + 3 * page_size); + } +} + +TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev_faulted_next) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *ptr_a, *ptr_b, *ptr_bc; + + /* + * mremap() with MREMAP_DONTUNMAP such that A, B and C merge: + * + * |---------------------------| + * | \ | + * |-----------| | |-----------| / |---------| + * | unfaulted | v | faulted | \ | faulted | + * |-----------| |-----------| / |---------| + * A C \ B + */ + + /* + * Map VMA B and C into place. We have to map them together so their + * anon_vma is the same and the vma->vm_pgoff's are correctly aligned. + */ + ptr_bc = mmap(&self->carveout[page_size + 3 * page_size], + 3 * page_size + 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_bc, MAP_FAILED); + + /* Fault it in. */ + ptr_bc[0] = 'x'; + + if (variant->forked) { + pid_t pid = do_fork(&self->procmap); + + ASSERT_NE(pid, -1); + if (pid != 0) + return; + } + + /* + * Now move VMA B out the way (splitting VMA BC) so we can place VMA A + * in position, unfaulted, and leave the remainder of the VMA we just + * moved in place, faulted, as VMA C. + */ + ptr_b = mremap(ptr_bc, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE, &self->carveout[20 * page_size]); + ASSERT_NE(ptr_b, MAP_FAILED); + + /* Map VMA A into place. */ + ptr_a = mmap(&self->carveout[page_size], 3 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + ASSERT_NE(ptr_a, MAP_FAILED); + + /* + * Now move VMA B into position with MREMAP_DONTUNMAP to catch incorrect + * anon_vma propagation. + */ + ptr_b = mremap(ptr_b, 3 * page_size, 3 * page_size, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, + &self->carveout[page_size + 3 * page_size]); + ASSERT_NE(ptr_b, MAP_FAILED); + + /* The VMAs should have merged. A,B,C if unforked, B, C if forked. */ + if (variant->forked) { + ASSERT_TRUE(find_vma_procmap(procmap, ptr_b)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_b); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_b + 6 * page_size); + } else { + ASSERT_TRUE(find_vma_procmap(procmap, ptr_a)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr_a); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr_a + 9 * page_size); + } +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c index f917b4c4c9436..44627d4010f24 100644 --- a/tools/testing/selftests/mm/uffd-unit-tests.c +++ b/tools/testing/selftests/mm/uffd-unit-tests.c @@ -1317,7 +1317,7 @@ static thread_state thread_state_get(pid_t tid) p = strstr(tmp, header); if (p) { /* For example, "State:\tD (disk sleep)" */ - c = *(p + sizeof(header) - 1); + c = *(p + strlen(header)); return c == 'D' ? THR_STATE_UNINTERRUPTIBLE : THR_STATE_UNKNOWN; } diff --git a/tools/testing/selftests/mount_setattr/mount_setattr_test.c b/tools/testing/selftests/mount_setattr/mount_setattr_test.c index a688871a98eba..388bca88ec94f 100644 --- a/tools/testing/selftests/mount_setattr/mount_setattr_test.c +++ b/tools/testing/selftests/mount_setattr/mount_setattr_test.c @@ -1020,7 +1020,7 @@ FIXTURE_SETUP(mount_setattr_idmapped) "size=100000,mode=700"), 0); ASSERT_EQ(mount("testing", "/mnt", "tmpfs", MS_NOATIME | MS_NODEV, - "size=2m,mode=700"), 0); + "size=256m,mode=700"), 0); ASSERT_EQ(mkdir("/mnt/A", 0777), 0); diff --git a/tools/testing/selftests/net/af_unix/Makefile b/tools/testing/selftests/net/af_unix/Makefile index 528d14c598bb5..2889403e35468 100644 --- a/tools/testing/selftests/net/af_unix/Makefile +++ b/tools/testing/selftests/net/af_unix/Makefile @@ -1,4 +1,9 @@ -CFLAGS += $(KHDR_INCLUDES) -Wall -Wflex-array-member-not-at-end +top_srcdir := ../../../../.. +include $(top_srcdir)/scripts/Makefile.compiler + +cc-option = $(call __cc-option, $(CC),,$(1),$(2)) + +CFLAGS += $(KHDR_INCLUDES) -Wall $(call cc-option,-Wflex-array-member-not-at-end) TEST_GEN_PROGS := \ diag_uid \ diff --git a/tools/testing/selftests/net/amt.sh b/tools/testing/selftests/net/amt.sh index 3ef209cacb8ed..663744305e521 100755 --- a/tools/testing/selftests/net/amt.sh +++ b/tools/testing/selftests/net/amt.sh @@ -73,6 +73,8 @@ # +------------------------+ #============================================================================== +source lib.sh + readonly LISTENER=$(mktemp -u listener-XXXXXXXX) readonly GATEWAY=$(mktemp -u gateway-XXXXXXXX) readonly RELAY=$(mktemp -u relay-XXXXXXXX) @@ -246,14 +248,15 @@ test_ipv6_forward() send_mcast4() { - sleep 2 + sleep 5 + wait_local_port_listen ${LISTENER} 4000 udp ip netns exec "${SOURCE}" bash -c \ 'printf "%s %128s" 172.17.0.2 | nc -w 1 -u 239.0.0.1 4000' & } send_mcast6() { - sleep 2 + wait_local_port_listen ${LISTENER} 6000 udp ip netns exec "${SOURCE}" bash -c \ 'printf "%s %128s" 2001:db8:3::2 | nc -w 1 -u ff0e::5:6 6000' & } diff --git a/tools/testing/selftests/net/fib-onlink-tests.sh b/tools/testing/selftests/net/fib-onlink-tests.sh index ec2d6ceb1f08d..c01be076b210d 100755 --- a/tools/testing/selftests/net/fib-onlink-tests.sh +++ b/tools/testing/selftests/net/fib-onlink-tests.sh @@ -120,7 +120,7 @@ log_subsection() run_cmd() { - local cmd="$*" + local cmd="$1" local out local rc @@ -145,7 +145,7 @@ get_linklocal() local pfx local addr - addr=$(${pfx} ip -6 -br addr show dev ${dev} | \ + addr=$(${pfx} ${IP} -6 -br addr show dev ${dev} | \ awk '{ for (i = 3; i <= NF; ++i) { if ($i ~ /^fe80/) @@ -173,58 +173,48 @@ setup() set -e - # create namespace - setup_ns PEER_NS + # create namespaces + setup_ns ns1 + IP="ip -netns $ns1" + setup_ns ns2 # add vrf table - ip li add ${VRF} type vrf table ${VRF_TABLE} - ip li set ${VRF} up - ip ro add table ${VRF_TABLE} unreachable default metric 8192 - ip -6 ro add table ${VRF_TABLE} unreachable default metric 8192 + ${IP} li add ${VRF} type vrf table ${VRF_TABLE} + ${IP} li set ${VRF} up + ${IP} ro add table ${VRF_TABLE} unreachable default metric 8192 + ${IP} -6 ro add table ${VRF_TABLE} unreachable default metric 8192 # create test interfaces - ip li add ${NETIFS[p1]} type veth peer name ${NETIFS[p2]} - ip li add ${NETIFS[p3]} type veth peer name ${NETIFS[p4]} - ip li add ${NETIFS[p5]} type veth peer name ${NETIFS[p6]} - ip li add ${NETIFS[p7]} type veth peer name ${NETIFS[p8]} + ${IP} li add ${NETIFS[p1]} type veth peer name ${NETIFS[p2]} + ${IP} li add ${NETIFS[p3]} type veth peer name ${NETIFS[p4]} + ${IP} li add ${NETIFS[p5]} type veth peer name ${NETIFS[p6]} + ${IP} li add ${NETIFS[p7]} type veth peer name ${NETIFS[p8]} # enslave vrf interfaces for n in 5 7; do - ip li set ${NETIFS[p${n}]} vrf ${VRF} + ${IP} li set ${NETIFS[p${n}]} vrf ${VRF} done # add addresses for n in 1 3 5 7; do - ip li set ${NETIFS[p${n}]} up - ip addr add ${V4ADDRS[p${n}]}/24 dev ${NETIFS[p${n}]} - ip addr add ${V6ADDRS[p${n}]}/64 dev ${NETIFS[p${n}]} nodad + ${IP} li set ${NETIFS[p${n}]} up + ${IP} addr add ${V4ADDRS[p${n}]}/24 dev ${NETIFS[p${n}]} + ${IP} addr add ${V6ADDRS[p${n}]}/64 dev ${NETIFS[p${n}]} nodad done # move peer interfaces to namespace and add addresses for n in 2 4 6 8; do - ip li set ${NETIFS[p${n}]} netns ${PEER_NS} up - ip -netns ${PEER_NS} addr add ${V4ADDRS[p${n}]}/24 dev ${NETIFS[p${n}]} - ip -netns ${PEER_NS} addr add ${V6ADDRS[p${n}]}/64 dev ${NETIFS[p${n}]} nodad + ${IP} li set ${NETIFS[p${n}]} netns ${ns2} up + ip -netns $ns2 addr add ${V4ADDRS[p${n}]}/24 dev ${NETIFS[p${n}]} + ip -netns $ns2 addr add ${V6ADDRS[p${n}]}/64 dev ${NETIFS[p${n}]} nodad done - ip -6 ro add default via ${V6ADDRS[p3]/::[0-9]/::64} - ip -6 ro add table ${VRF_TABLE} default via ${V6ADDRS[p7]/::[0-9]/::64} + ${IP} -6 ro add default via ${V6ADDRS[p3]/::[0-9]/::64} + ${IP} -6 ro add table ${VRF_TABLE} default via ${V6ADDRS[p7]/::[0-9]/::64} set +e } -cleanup() -{ - # make sure we start from a clean slate - cleanup_ns ${PEER_NS} 2>/dev/null - for n in 1 3 5 7; do - ip link del ${NETIFS[p${n}]} 2>/dev/null - done - ip link del ${VRF} 2>/dev/null - ip ro flush table ${VRF_TABLE} - ip -6 ro flush table ${VRF_TABLE} -} - ################################################################################ # IPv4 tests # @@ -241,7 +231,7 @@ run_ip() # dev arg may be empty [ -n "${dev}" ] && dev="dev ${dev}" - run_cmd ip ro add table "${table}" "${prefix}"/32 via "${gw}" "${dev}" onlink + run_cmd "${IP} ro add table ${table} ${prefix}/32 via ${gw} ${dev} onlink" log_test $? ${exp_rc} "${desc}" } @@ -257,8 +247,8 @@ run_ip_mpath() # dev arg may be empty [ -n "${dev}" ] && dev="dev ${dev}" - run_cmd ip ro add table "${table}" "${prefix}"/32 \ - nexthop via ${nh1} nexthop via ${nh2} + run_cmd "${IP} ro add table ${table} ${prefix}/32 \ + nexthop via ${nh1} nexthop via ${nh2}" log_test $? ${exp_rc} "${desc}" } @@ -339,7 +329,7 @@ run_ip6() # dev arg may be empty [ -n "${dev}" ] && dev="dev ${dev}" - run_cmd ip -6 ro add table "${table}" "${prefix}"/128 via "${gw}" "${dev}" onlink + run_cmd "${IP} -6 ro add table ${table} ${prefix}/128 via ${gw} ${dev} onlink" log_test $? ${exp_rc} "${desc}" } @@ -353,8 +343,8 @@ run_ip6_mpath() local exp_rc="$6" local desc="$7" - run_cmd ip -6 ro add table "${table}" "${prefix}"/128 "${opts}" \ - nexthop via ${nh1} nexthop via ${nh2} + run_cmd "${IP} -6 ro add table ${table} ${prefix}/128 ${opts} \ + nexthop via ${nh1} nexthop via ${nh2}" log_test $? ${exp_rc} "${desc}" } @@ -491,10 +481,9 @@ do esac done -cleanup setup run_onlink_tests -cleanup +cleanup_ns ${ns1} ${ns2} if [ "$TESTS" != "none" ]; then printf "\nTests passed: %3d\n" ${nsuccess} diff --git a/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh index 72dfbeaf56b92..e8031f68200ad 100755 --- a/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh +++ b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh @@ -414,6 +414,7 @@ vlmc_querier_intvl_test() bridge vlan add vid 10 dev br1 self pvid untagged ip link set dev $h1 master br1 ip link set dev br1 up + setup_wait_dev $h1 0 bridge vlan add vid 10 dev $h1 master bridge vlan global set vid 10 dev br1 mcast_snooping 1 mcast_querier 1 sleep 2 diff --git a/tools/testing/selftests/net/forwarding/tc_actions.sh b/tools/testing/selftests/net/forwarding/tc_actions.sh index ea89e558672db..86edbc7e2489b 100755 --- a/tools/testing/selftests/net/forwarding/tc_actions.sh +++ b/tools/testing/selftests/net/forwarding/tc_actions.sh @@ -223,7 +223,7 @@ mirred_egress_to_ingress_tcp_test() ip_proto icmp \ action drop - ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 -o $mirred_e2i_tf2 & + ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 > $mirred_e2i_tf2 & local rpid=$! ip vrf exec v$h1 ncat -w1 --send-only 192.0.2.2 12345 <$mirred_e2i_tf1 wait -n $rpid diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh index b43816dd998ca..457f41d5e584b 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh @@ -567,6 +567,21 @@ vxlan_encapped_ping_do() local inner_tos=$1; shift local outer_tos=$1; shift + local ipv4hdr=$(: + )"45:"$( : IP version + IHL + )"$inner_tos:"$( : IP TOS + )"00:54:"$( : IP total length + )"99:83:"$( : IP identification + )"40:00:"$( : IP flags + frag off + )"40:"$( : IP TTL + )"01:"$( : IP proto + )"CHECKSUM:"$( : IP header csum + )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 + )"c0:00:02:01"$( : IP daddr: 192.0.2.1 + ) + local checksum=$(payload_template_calc_checksum "$ipv4hdr") + ipv4hdr=$(payload_template_expand_checksum "$ipv4hdr" $checksum) + $MZ $dev -c $count -d 100msec -q \ -b $next_hop_mac -B $dest_ip \ -t udp tos=$outer_tos,sp=23456,dp=$VXPORT,p=$(: @@ -577,16 +592,7 @@ vxlan_encapped_ping_do() )"$dest_mac:"$( : ETH daddr )"$(mac_get w2):"$( : ETH saddr )"08:00:"$( : ETH type - )"45:"$( : IP version + IHL - )"$inner_tos:"$( : IP TOS - )"00:54:"$( : IP total length - )"99:83:"$( : IP identification - )"40:00:"$( : IP flags + frag off - )"40:"$( : IP TTL - )"01:"$( : IP proto - )"00:00:"$( : IP header csum - )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 - )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + )"$ipv4hdr:"$( : IPv4 header )"08:"$( : ICMP type )"00:"$( : ICMP code )"8b:f2:"$( : ICMP csum diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh index a603f7b0a08f0..e642feeada0e7 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh @@ -695,7 +695,7 @@ vxlan_encapped_ping_do() )"6"$( : IP version )"$inner_tos"$( : Traffic class )"0:00:00:"$( : Flow label - )"00:08:"$( : Payload length + )"00:03:"$( : Payload length )"3a:"$( : Next header )"04:"$( : Hop limit )"$saddr:"$( : IP saddr diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index f448bafb3f208..d0306b27fe95e 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -576,7 +576,7 @@ ip_link_has_flag() local flag=$1; shift local state=$(ip -j link show "$name" | - jq --arg flag "$flag" 'any(.[].flags.[]; . == $flag)') + jq --arg flag "$flag" 'any(.[].flags[]; . == $flag)') [[ $state == true ]] } diff --git a/tools/testing/selftests/net/lib/ksft.h b/tools/testing/selftests/net/lib/ksft.h index 17dc34a612c64..03912902a6d30 100644 --- a/tools/testing/selftests/net/lib/ksft.h +++ b/tools/testing/selftests/net/lib/ksft.h @@ -24,7 +24,8 @@ static inline void ksft_ready(void) fd = STDOUT_FILENO; } - write(fd, msg, sizeof(msg)); + if (write(fd, msg, sizeof(msg)) < 0) + perror("write()"); if (fd != STDOUT_FILENO) close(fd); } @@ -48,7 +49,8 @@ static inline void ksft_wait(void) fd = STDIN_FILENO; } - read(fd, &byte, sizeof(byte)); + if (read(fd, &byte, sizeof(byte)) < 0) + perror("read()"); if (fd != STDIN_FILENO) close(fd); } diff --git a/tools/testing/selftests/net/lib/py/__init__.py b/tools/testing/selftests/net/lib/py/__init__.py index 97b7cf2b20eb9..8f2da17d53510 100644 --- a/tools/testing/selftests/net/lib/py/__init__.py +++ b/tools/testing/selftests/net/lib/py/__init__.py @@ -12,7 +12,7 @@ from .netns import NetNS, NetNSEnter from .nsim import NetdevSim, NetdevSimDev from .utils import CmdExitFailure, fd_read_timeout, cmd, bkg, defer, \ - bpftool, ip, ethtool, bpftrace, rand_port, wait_port_listen, wait_file + bpftool, ip, ethtool, bpftrace, rand_port, wait_port_listen, wait_file, tool from .ynl import NlError, YnlFamily, EthtoolFamily, NetdevFamily, RtnlFamily, RtnlAddrFamily from .ynl import NetshaperFamily, DevlinkFamily, PSPFamily @@ -25,7 +25,7 @@ "NetNS", "NetNSEnter", "CmdExitFailure", "fd_read_timeout", "cmd", "bkg", "defer", "bpftool", "ip", "ethtool", "bpftrace", "rand_port", - "wait_port_listen", "wait_file", + "wait_port_listen", "wait_file", "tool", "NetdevSim", "NetdevSimDev", "NetshaperFamily", "DevlinkFamily", "PSPFamily", "NlError", "YnlFamily", "EthtoolFamily", "NetdevFamily", "RtnlFamily", diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index 43f31f8d587fe..8990cd99f4e32 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -103,6 +103,24 @@ CBPF_MPTCP_SUBOPTION_ADD_ADDR="14, 6 0 0 65535, 6 0 0 0" +# IPv4: TCP hdr of 48B, a first suboption of 12B (DACK8), the RM_ADDR suboption +# generated using "nfbpf_compile '(ip[32] & 0xf0) == 0xc0 && ip[53] == 0x0c && +# (ip[66] & 0xf0) == 0x40'" +CBPF_MPTCP_SUBOPTION_RM_ADDR="13, + 48 0 0 0, + 84 0 0 240, + 21 0 9 64, + 48 0 0 32, + 84 0 0 240, + 21 0 6 192, + 48 0 0 53, + 21 0 4 12, + 48 0 0 66, + 84 0 0 240, + 21 0 1 64, + 6 0 0 65535, + 6 0 0 0" + init_partial() { capout=$(mktemp) @@ -2305,17 +2323,16 @@ signal_address_tests() ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 + chk_join_nr 3 3 3 # It is not directly linked to the commit introducing this # symbol but for the parent one which is linked anyway. - if ! mptcp_lib_kallsyms_has "mptcp_pm_subflow_check_next$"; then - chk_join_nr 3 3 2 - chk_add_nr 4 4 - else - chk_join_nr 3 3 3 + if mptcp_lib_kallsyms_has "mptcp_pm_subflow_check_next$"; then # the server will not signal the address terminating # the MPC subflow chk_add_nr 3 3 + else + chk_add_nr 4 4 fi fi } @@ -2596,6 +2613,19 @@ remove_tests() chk_rst_nr 0 0 fi + # signal+subflow with limits, remove + if reset "remove signal+subflow with limits"; then + pm_nl_set_limits $ns1 0 0 + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,subflow + pm_nl_set_limits $ns2 0 0 + addr_nr_ns1=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 + chk_join_nr 0 0 0 + chk_add_nr 1 1 + chk_rm_nr 1 0 invert + chk_rst_nr 0 0 + fi + # addresses remove if reset "remove addresses"; then pm_nl_set_limits $ns1 3 3 @@ -3692,21 +3722,28 @@ userspace_pm_chk_get_addr() fi } -# $1: ns ; $2: event type ; $3: count +# $1: ns ; $2: event type ; $3: count ; [ $4: attr ; $5: attr count ] chk_evt_nr() { local ns=${1} local evt_name="${2}" local exp="${3}" + local attr="${4}" + local attr_exp="${5}" local evts="${evts_ns1}" local evt="${!evt_name}" + local attr_name local count + if [ -n "${attr}" ]; then + attr_name=", ${attr}: ${attr_exp}" + fi + evt_name="${evt_name:16}" # without MPTCP_LIB_EVENT_ [ "${ns}" == "ns2" ] && evts="${evts_ns2}" - print_check "event ${ns} ${evt_name} (${exp})" + print_check "event ${ns} ${evt_name} (${exp}${attr_name})" if [[ "${evt_name}" = "LISTENER_"* ]] && ! mptcp_lib_kallsyms_has "mptcp_event_pm_listener$"; then @@ -3717,11 +3754,42 @@ chk_evt_nr() count=$(grep -cw "type:${evt}" "${evts}") if [ "${count}" != "${exp}" ]; then fail_test "got ${count} events, expected ${exp}" + cat "${evts}" + return + elif [ -z "${attr}" ]; then + print_ok + return + fi + + count=$(grep -w "type:${evt}" "${evts}" | grep -c ",${attr}:") + if [ "${count}" != "${attr_exp}" ]; then + fail_test "got ${count} event attributes, expected ${attr_exp}" + grep -w "type:${evt}" "${evts}" else print_ok fi } +# $1: ns ; $2: event type ; $3: expected count +wait_event() +{ + local ns="${1}" + local evt_name="${2}" + local exp="${3}" + + local evt="${!evt_name}" + local evts="${evts_ns1}" + local count + + [ "${ns}" == "ns2" ] && evts="${evts_ns2}" + + for _ in $(seq 100); do + count=$(grep -cw "type:${evt}" "${evts}") + [ "${count}" -ge "${exp}" ] && break + sleep 0.1 + done +} + userspace_tests() { # userspace pm type prevents add_addr @@ -3930,6 +3998,36 @@ userspace_tests() kill_events_pids mptcp_lib_kill_group_wait $tests_pid fi + + # userspace pm no duplicated spurious close events after an error + if reset_with_events "userspace pm no dup close events after error" && + continue_if mptcp_lib_has_file '/proc/sys/net/mptcp/pm_type'; then + set_userspace_pm $ns2 + pm_nl_set_limits $ns1 0 2 + { timeout_test=120 test_linkfail=128 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 & } 2>/dev/null + local tests_pid=$! + wait_event ns2 MPTCP_LIB_EVENT_ESTABLISHED 1 + userspace_pm_add_sf $ns2 10.0.3.2 20 + chk_mptcp_info subflows 1 subflows 1 + chk_subflows_total 2 2 + + # force quick loss + ip netns exec $ns2 sysctl -q net.ipv4.tcp_syn_retries=1 + if ip netns exec "${ns1}" ${iptables} -A INPUT -s "10.0.1.2" \ + -p tcp --tcp-option 30 -j REJECT --reject-with tcp-reset && + ip netns exec "${ns2}" ${iptables} -A INPUT -d "10.0.1.2" \ + -p tcp --tcp-option 30 -j REJECT --reject-with tcp-reset; then + wait_event ns2 MPTCP_LIB_EVENT_SUB_CLOSED 1 + wait_event ns1 MPTCP_LIB_EVENT_SUB_CLOSED 1 + chk_subflows_total 1 1 + userspace_pm_add_sf $ns2 10.0.1.2 0 + wait_event ns2 MPTCP_LIB_EVENT_SUB_CLOSED 2 + chk_evt_nr ns2 MPTCP_LIB_EVENT_SUB_CLOSED 2 error 2 + fi + kill_events_pids + mptcp_lib_kill_group_wait $tests_pid + fi } endpoint_tests() @@ -4000,6 +4098,14 @@ endpoint_tests() chk_subflow_nr "after no reject" 3 chk_mptcp_info subflows 2 subflows 2 + # To make sure RM_ADDR are sent over a different subflow, but + # allow the rest to quickly and cleanly close the subflow + local ipt=1 + ip netns exec "${ns2}" ${iptables} -I OUTPUT -s "10.0.1.2" \ + -p tcp -m tcp --tcp-option 30 \ + -m bpf --bytecode \ + "$CBPF_MPTCP_SUBOPTION_RM_ADDR" \ + -j DROP || ipt=0 local i for i in $(seq 3); do pm_nl_del_endpoint $ns2 1 10.0.1.2 @@ -4012,6 +4118,7 @@ endpoint_tests() chk_subflow_nr "after re-add id 0 ($i)" 3 chk_mptcp_info subflows 3 subflows 3 done + [ ${ipt} = 1 ] && ip netns exec "${ns2}" ${iptables} -D OUTPUT 1 mptcp_lib_kill_group_wait $tests_pid @@ -4071,11 +4178,20 @@ endpoint_tests() chk_mptcp_info subflows 2 subflows 2 chk_mptcp_info add_addr_signal 2 add_addr_accepted 2 + # To make sure RM_ADDR are sent over a different subflow, but + # allow the rest to quickly and cleanly close the subflow + local ipt=1 + ip netns exec "${ns1}" ${iptables} -I OUTPUT -s "10.0.1.1" \ + -p tcp -m tcp --tcp-option 30 \ + -m bpf --bytecode \ + "$CBPF_MPTCP_SUBOPTION_RM_ADDR" \ + -j DROP || ipt=0 pm_nl_del_endpoint $ns1 42 10.0.1.1 sleep 0.5 chk_subflow_nr "after delete ID 0" 2 chk_mptcp_info subflows 2 subflows 2 chk_mptcp_info add_addr_signal 2 add_addr_accepted 2 + [ ${ipt} = 1 ] && ip netns exec "${ns1}" ${iptables} -D OUTPUT 1 pm_nl_add_endpoint $ns1 10.0.1.1 id 99 flags signal wait_mpj $ns2 diff --git a/tools/testing/selftests/net/mptcp/pm_netlink.sh b/tools/testing/selftests/net/mptcp/pm_netlink.sh index ec6a875881919..123d9d7a0278c 100755 --- a/tools/testing/selftests/net/mptcp/pm_netlink.sh +++ b/tools/testing/selftests/net/mptcp/pm_netlink.sh @@ -192,6 +192,10 @@ check "show_endpoints" \ flush_endpoint check "show_endpoints" "" "flush addrs" +add_endpoint 10.0.1.1 flags unknown +check "show_endpoints" "$(format_endpoints "1,10.0.1.1")" "ignore unknown flags" +flush_endpoint + set_limits 9 1 2>/dev/null check "get_limits" "${default_limits}" "rcv addrs above hard limit" diff --git a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c index 65b374232ff5a..99eecccbf0c87 100644 --- a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c +++ b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c @@ -24,6 +24,8 @@ #define IPPROTO_MPTCP 262 #endif +#define MPTCP_PM_ADDR_FLAG_UNKNOWN _BITUL(7) + static void syntax(char *argv[]) { fprintf(stderr, "%s add|ann|rem|csf|dsf|get|set|del|flush|dump|events|listen|accept []\n", argv[0]); @@ -836,6 +838,8 @@ int add_addr(int fd, int pm_family, int argc, char *argv[]) flags |= MPTCP_PM_ADDR_FLAG_BACKUP; else if (!strcmp(tok, "fullmesh")) flags |= MPTCP_PM_ADDR_FLAG_FULLMESH; + else if (!strcmp(tok, "unknown")) + flags |= MPTCP_PM_ADDR_FLAG_UNKNOWN; else error(1, errno, "unknown flag %s", argv[arg]); @@ -1048,6 +1052,13 @@ static void print_addr(struct rtattr *attrs, int len) printf(","); } + if (flags & MPTCP_PM_ADDR_FLAG_UNKNOWN) { + printf("unknown"); + flags &= ~MPTCP_PM_ADDR_FLAG_UNKNOWN; + if (flags) + printf(","); + } + /* bump unknown flags, if any */ if (flags) printf("0x%x", flags); diff --git a/tools/testing/selftests/net/mptcp/simult_flows.sh b/tools/testing/selftests/net/mptcp/simult_flows.sh index 1903e8e84a315..76730e511f92e 100755 --- a/tools/testing/selftests/net/mptcp/simult_flows.sh +++ b/tools/testing/selftests/net/mptcp/simult_flows.sh @@ -233,10 +233,13 @@ run_test() for dev in ns2eth1 ns2eth2; do tc -n $ns2 qdisc del dev $dev root >/dev/null 2>&1 done - tc -n $ns1 qdisc add dev ns1eth1 root netem rate ${rate1}mbit $delay1 - tc -n $ns1 qdisc add dev ns1eth2 root netem rate ${rate2}mbit $delay2 - tc -n $ns2 qdisc add dev ns2eth1 root netem rate ${rate1}mbit $delay1 - tc -n $ns2 qdisc add dev ns2eth2 root netem rate ${rate2}mbit $delay2 + + # keep the queued pkts number low, or the RTT estimator will see + # increasing latency over time. + tc -n $ns1 qdisc add dev ns1eth1 root netem rate ${rate1}mbit $delay1 limit 50 + tc -n $ns1 qdisc add dev ns1eth2 root netem rate ${rate2}mbit $delay2 limit 50 + tc -n $ns2 qdisc add dev ns2eth1 root netem rate ${rate1}mbit $delay1 limit 50 + tc -n $ns2 qdisc add dev ns2eth2 root netem rate ${rate2}mbit $delay2 limit 50 # time is measured in ms, account for transfer size, aggregated link speed # and header overhead (10%) diff --git a/tools/testing/selftests/net/netfilter/conntrack_clash.sh b/tools/testing/selftests/net/netfilter/conntrack_clash.sh index 7fc6c5dbd5516..84b8eb12143ae 100755 --- a/tools/testing/selftests/net/netfilter/conntrack_clash.sh +++ b/tools/testing/selftests/net/netfilter/conntrack_clash.sh @@ -116,7 +116,7 @@ run_one_clash_test() # not a failure: clash resolution logic did not trigger. # With right timing, xmit completed sequentially and # no parallel insertion occurs. - return $ksft_skip + return $ksft_xfail } run_clash_test() @@ -133,12 +133,12 @@ run_clash_test() if [ $rv -eq 0 ];then echo "PASS: clash resolution test for $daddr:$dport on attempt $i" return 0 - elif [ $rv -eq $ksft_skip ]; then + elif [ $rv -eq $ksft_xfail ]; then softerr=1 fi done - [ $softerr -eq 1 ] && echo "SKIP: clash resolution for $daddr:$dport did not trigger" + [ $softerr -eq 1 ] && echo "XFAIL: clash resolution for $daddr:$dport did not trigger" } ip link add veth0 netns "$nsclient1" type veth peer name veth0 netns "$nsrouter" @@ -167,8 +167,7 @@ load_simple_ruleset "$nsclient2" run_clash_test "$nsclient2" "$nsclient2" 127.0.0.1 9001 if [ $clash_resolution_active -eq 0 ];then - [ "$ret" -eq 0 ] && ret=$ksft_skip - echo "SKIP: Clash resolution did not trigger" + [ "$ret" -eq 0 ] && ret=$ksft_xfail fi exit $ret diff --git a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c index 507930cee8cb6..462d628cc3bdb 100644 --- a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c +++ b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c @@ -33,9 +33,14 @@ static void die(const char *e) exit(111); } -static void die_port(uint16_t got, uint16_t want) +static void die_port(const struct sockaddr_in *sin, uint16_t want) { - fprintf(stderr, "Port number changed, wanted %d got %d\n", want, ntohs(got)); + uint16_t got = ntohs(sin->sin_port); + char str[INET_ADDRSTRLEN]; + + inet_ntop(AF_INET, &sin->sin_addr, str, sizeof(str)); + + fprintf(stderr, "Port number changed, wanted %d got %d from %s\n", want, got, str); exit(1); } @@ -100,7 +105,7 @@ int main(int argc, char *argv[]) die("child recvfrom"); if (peer.sin_port != htons(PORT)) - die_port(peer.sin_port, PORT); + die_port(&peer, PORT); } else { if (sendto(s2, buf, LEN, 0, (struct sockaddr *)&sa1, sizeof(sa1)) != LEN) continue; @@ -109,7 +114,7 @@ int main(int argc, char *argv[]) die("parent recvfrom"); if (peer.sin_port != htons((PORT + 1))) - die_port(peer.sin_port, PORT + 1); + die_port(&peer, PORT + 1); } } diff --git a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh index a24c896347a88..dc7e9d6da0624 100755 --- a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh +++ b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh @@ -45,6 +45,8 @@ if ip netns exec "$ns0" ./conntrack_reverse_clash; then echo "PASS: No SNAT performed for null bindings" else echo "ERROR: SNAT performed without any matching snat rule" + ip netns exec "$ns0" conntrack -L + ip netns exec "$ns0" conntrack -S exit 1 fi diff --git a/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt b/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt index 3442cd29bc932..cdb3910af95b4 100644 --- a/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt +++ b/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt @@ -26,7 +26,7 @@ +0.01 > R 643160523:643160523(0) win 0 -+0.01 `conntrack -f $NFCT_IP_VERSION -L -p tcp --dport 8080 2>/dev/null | grep UNREPLIED | grep -q SYN_SENT` ++0.1 `conntrack -f $NFCT_IP_VERSION -L -p tcp --dport 8080 2>/dev/null | grep UNREPLIED | grep -q SYN_SENT` // Must go through. +0.01 > S 0:0(0) win 65535 diff --git a/tools/testing/selftests/net/packetdrill/tcp_syscall_bad_arg_sendmsg-empty-iov.pkt b/tools/testing/selftests/net/packetdrill/tcp_syscall_bad_arg_sendmsg-empty-iov.pkt index b2b2cdf27e20f..454441e7ecff6 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_syscall_bad_arg_sendmsg-empty-iov.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_syscall_bad_arg_sendmsg-empty-iov.pkt @@ -1,6 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // Test that we correctly skip zero-length IOVs. + +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` + 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 +0 setsockopt(3, SOL_SOCKET, SO_ZEROCOPY, [1], 4) = 0 +0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_basic.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_basic.pkt index a82c8899d36bf..0a0700afdaa38 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_basic.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_basic.pkt @@ -4,6 +4,8 @@ // send a packet with MSG_ZEROCOPY and receive the notification ID // repeat and verify IDs are consecutive +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_batch.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_batch.pkt index c01915e7f4a15..df91675d2991c 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_batch.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_batch.pkt @@ -3,6 +3,8 @@ // // send multiple packets, then read one range of all notifications. +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_client.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_client.pkt index 6509882932e91..2963cfcb14dfd 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_client.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_client.pkt @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Minimal client-side zerocopy test +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 4 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_closed.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_closed.pkt index 2cd78755cb2ac..ea0c2fa73c2d6 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_closed.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_closed.pkt @@ -7,6 +7,8 @@ // First send on a closed socket and wait for (absent) notification. // Then connect and send and verify that notification nr. is zero. +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 4 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_edge.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_edge.pkt index 7671c20e01cf6..4df978a9b82e7 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_edge.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_edge.pkt @@ -7,6 +7,9 @@ // fire two sends with MSG_ZEROCOPY and receive the acks. confirm that EPOLLERR // is correctly fired only once, when EPOLLET is set. send another packet with // MSG_ZEROCOPY. confirm that EPOLLERR is correctly fired again only once. + +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_exclusive.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_exclusive.pkt index fadc480fdb7fe..36b6edc4858c4 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_exclusive.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_exclusive.pkt @@ -8,6 +8,9 @@ // fire two sends with MSG_ZEROCOPY and receive the acks. confirm that EPOLLERR // is correctly fired only once, when EPOLLET is set. send another packet with // MSG_ZEROCOPY. confirm that EPOLLERR is correctly fired again only once. + +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_oneshot.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_oneshot.pkt index 5bfa0d1d2f4a3..1bea6f3b4558d 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_oneshot.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_epoll_oneshot.pkt @@ -8,6 +8,9 @@ // is correctly fired only once, when EPOLLONESHOT is set. send another packet // with MSG_ZEROCOPY. confirm that EPOLLERR is not fired. Rearm the FD and // confirm that EPOLLERR is correctly set. + +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-client.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-client.pkt index 4a73bbf469610..e27c21ff5d18d 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-client.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-client.pkt @@ -8,6 +8,8 @@ // one will have no data in the initial send. On return 0 the // zerocopy notification counter is not incremented. Verify this too. +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` // Send a FastOpen request, no cookie yet so no data in SYN diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-server.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-server.pkt index 36086c5877ce7..b1fa77c77dfa7 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-server.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_fastopen-server.pkt @@ -4,6 +4,8 @@ // send data with MSG_FASTOPEN | MSG_ZEROCOPY and verify that the // kernel returns the notification ID. +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh ./set_sysctls.py /proc/sys/net/ipv4/tcp_fastopen=0x207` diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_maxfrags.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_maxfrags.pkt index 672f817faca0d..2f5317d0a9fab 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_maxfrags.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_maxfrags.pkt @@ -7,6 +7,8 @@ // because each iovec element becomes a frag // 3) the PSH bit is set on an skb when it runs out of fragments +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 diff --git a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_small.pkt b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_small.pkt index a9a1ac0aea4f4..9d5272c6b2079 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_zerocopy_small.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_zerocopy_small.pkt @@ -4,6 +4,8 @@ // verify that SO_EE_CODE_ZEROCOPY_COPIED is set on zerocopy // packets of all sizes, including the smallest payload, 1B. +--send_omit_free // do not reuse send buffers with zerocopy + `./defaults.sh` 0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 diff --git a/tools/testing/selftests/net/tap.c b/tools/testing/selftests/net/tap.c index 247c3b3ac1c97..51a209014f1cf 100644 --- a/tools/testing/selftests/net/tap.c +++ b/tools/testing/selftests/net/tap.c @@ -56,18 +56,12 @@ static void rtattr_end(struct nlmsghdr *nh, struct rtattr *attr) static struct rtattr *rtattr_add_str(struct nlmsghdr *nh, unsigned short type, const char *s) { - struct rtattr *rta = rtattr_add(nh, type, strlen(s)); + unsigned int strsz = strlen(s) + 1; + struct rtattr *rta; - memcpy(RTA_DATA(rta), s, strlen(s)); - return rta; -} - -static struct rtattr *rtattr_add_strsz(struct nlmsghdr *nh, unsigned short type, - const char *s) -{ - struct rtattr *rta = rtattr_add(nh, type, strlen(s) + 1); + rta = rtattr_add(nh, type, strsz); - strcpy(RTA_DATA(rta), s); + memcpy(RTA_DATA(rta), s, strsz); return rta; } @@ -119,7 +113,7 @@ static int dev_create(const char *dev, const char *link_type, link_info = rtattr_begin(&req.nh, IFLA_LINKINFO); - rtattr_add_strsz(&req.nh, IFLA_INFO_KIND, link_type); + rtattr_add_str(&req.nh, IFLA_INFO_KIND, link_type); if (fill_info_data) { info_data = rtattr_begin(&req.nh, IFLA_INFO_DATA); diff --git a/tools/testing/selftests/net/tfo.c b/tools/testing/selftests/net/tfo.c index eb3cac5e583c9..8d82140f0f767 100644 --- a/tools/testing/selftests/net/tfo.c +++ b/tools/testing/selftests/net/tfo.c @@ -81,7 +81,8 @@ static void run_server(void) if (getsockopt(connfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &opt, &len) < 0) error(1, errno, "getsockopt(SO_INCOMING_NAPI_ID)"); - read(connfd, buf, 64); + if (read(connfd, buf, 64) < 0) + perror("read()"); fprintf(outfile, "%d\n", opt); fclose(outfile); diff --git a/tools/testing/selftests/net/toeplitz.c b/tools/testing/selftests/net/toeplitz.c index 9ba03164d73a6..5099157f01b9a 100644 --- a/tools/testing/selftests/net/toeplitz.c +++ b/tools/testing/selftests/net/toeplitz.c @@ -473,8 +473,8 @@ static void parse_rps_bitmap(const char *arg) bitmap = strtoul(arg, NULL, 0); - if (bitmap & ~(RPS_MAX_CPUS - 1)) - error(1, 0, "rps bitmap 0x%lx out of bounds 0..%lu", + if (bitmap & ~((1UL << RPS_MAX_CPUS) - 1)) + error(1, 0, "rps bitmap 0x%lx out of bounds, max cpu %lu", bitmap, RPS_MAX_CPUS - 1); for (i = 0; i < RPS_MAX_CPUS; i++) diff --git a/tools/testing/selftests/resctrl/resctrlfs.c b/tools/testing/selftests/resctrl/resctrlfs.c index 195f04c4d1586..b9c1bfb6cc029 100644 --- a/tools/testing/selftests/resctrl/resctrlfs.c +++ b/tools/testing/selftests/resctrl/resctrlfs.c @@ -243,6 +243,16 @@ int snc_nodes_per_l3_cache(void) } snc_mode = cache_cpus / node_cpus; + /* + * On some platforms (e.g. Hygon), + * cache_cpus < node_cpus, the calculated snc_mode is 0. + * + * Set snc_mode = 1 to indicate that SNC mode is not + * supported on the platform. + */ + if (!snc_mode) + snc_mode = 1; + if (snc_mode > 1) ksft_print_msg("SNC-%d mode discovered.\n", snc_mode); } diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index 6b8123c12a7ae..cbd23444c8a98 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -718,7 +718,7 @@ static int ublk_thread_is_idle(struct ublk_thread *t) static int ublk_thread_is_done(struct ublk_thread *t) { - return (t->state & UBLKS_T_STOPPING) && ublk_thread_is_idle(t); + return (t->state & UBLKS_T_STOPPING) && ublk_thread_is_idle(t) && !t->cmd_inflight; } static inline void ublksrv_handle_tgt_cqe(struct ublk_thread *t, @@ -1002,7 +1002,9 @@ static int ublk_start_daemon(const struct dev_ctx *ctx, struct ublk_dev *dev) } if (ret < 0) { ublk_err("%s: ublk_ctrl_start_dev failed: %d\n", __func__, ret); - goto fail; + /* stop device so that inflight uring_cmd can be cancelled */ + ublk_ctrl_stop_dev(dev); + goto fail_start; } ublk_ctrl_get_info(dev); @@ -1010,7 +1012,7 @@ static int ublk_start_daemon(const struct dev_ctx *ctx, struct ublk_dev *dev) ublk_ctrl_dump(dev); else ublk_send_dev_event(ctx, dev, dev->dev_info.dev_id); - +fail_start: /* wait until we are terminated */ for (i = 0; i < dev->nthreads; i++) pthread_join(dev->threads[i].thread, &thread_ret); @@ -1219,7 +1221,7 @@ static int __cmd_dev_add(const struct dev_ctx *ctx) } ret = ublk_start_daemon(ctx, dev); - ublk_dbg(UBLK_DBG_DEV, "%s: daemon exit %d\b", ret); + ublk_dbg(UBLK_DBG_DEV, "%s: daemon exit %d\n", __func__, ret); if (ret < 0) ublk_ctrl_del_dev(dev); @@ -1564,6 +1566,7 @@ int main(int argc, char *argv[]) int option_idx, opt; const char *cmd = argv[1]; struct dev_ctx ctx = { + ._evtfd = -1, .queue_depth = 128, .nr_hw_queues = 2, .dev_id = -1, diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h index 5e55484fb0aa2..39839c711c797 100644 --- a/tools/testing/selftests/ublk/kublk.h +++ b/tools/testing/selftests/ublk/kublk.h @@ -220,7 +220,7 @@ static inline __u64 build_user_data(unsigned tag, unsigned op, unsigned tgt_data, unsigned q_id, unsigned is_target_io) { /* we only have 7 bits to encode q_id */ - _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7); + _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7, "UBLK_MAX_QUEUES_SHIFT must be <= 7"); assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16) && !(q_id >> 7)); return tag | (op << 16) | (tgt_data << 24) | @@ -393,19 +393,19 @@ static inline int ublk_completed_tgt_io(struct ublk_thread *t, return --io->tgt_ios == 0; } -static inline int ublk_queue_use_zc(const struct ublk_queue *q) +static inline bool ublk_queue_use_zc(const struct ublk_queue *q) { - return q->flags & UBLK_F_SUPPORT_ZERO_COPY; + return !!(q->flags & UBLK_F_SUPPORT_ZERO_COPY); } -static inline int ublk_queue_use_auto_zc(const struct ublk_queue *q) +static inline bool ublk_queue_use_auto_zc(const struct ublk_queue *q) { - return q->flags & UBLK_F_AUTO_BUF_REG; + return !!(q->flags & UBLK_F_AUTO_BUF_REG); } -static inline int ublk_queue_auto_zc_fallback(const struct ublk_queue *q) +static inline bool ublk_queue_auto_zc_fallback(const struct ublk_queue *q) { - return q->flags & UBLKS_Q_AUTO_BUF_REG_FALLBACK; + return !!(q->flags & UBLKS_Q_AUTO_BUF_REG_FALLBACK); } static inline int ublk_queue_no_buf(const struct ublk_queue *q) diff --git a/tools/testing/vsock/util.c b/tools/testing/vsock/util.c index d843643ced6b7..9430ef5b8bc3e 100644 --- a/tools/testing/vsock/util.c +++ b/tools/testing/vsock/util.c @@ -511,6 +511,18 @@ void run_tests(const struct test_case *test_cases, printf("ok\n"); } + + printf("All tests have been executed. Waiting other peer..."); + fflush(stdout); + + /* + * Final full barrier, to ensure that all tests have been run and + * that even the last one has been successful on both sides. + */ + control_writeln("COMPLETED"); + control_expectln("COMPLETED"); + + printf("ok\n"); } void list_tests(const struct test_case *test_cases) diff --git a/tools/testing/vsock/util.h b/tools/testing/vsock/util.h index 142c02a6834ac..bf633cde82b07 100644 --- a/tools/testing/vsock/util.h +++ b/tools/testing/vsock/util.h @@ -25,7 +25,7 @@ enum transport { }; static const char * const transport_ksyms[] = { - #define x(name, symbol) "d " symbol "_transport", + #define x(name, symbol) " " symbol "_transport", KNOWN_TRANSPORTS(x) #undef x }; diff --git a/tools/testing/vsock/vsock_test.c b/tools/testing/vsock/vsock_test.c index d4517386e551e..667f6f0ad6afa 100644 --- a/tools/testing/vsock/vsock_test.c +++ b/tools/testing/vsock/vsock_test.c @@ -351,6 +351,7 @@ static void test_stream_msg_peek_server(const struct test_opts *opts) static void test_seqpacket_msg_bounds_client(const struct test_opts *opts) { + unsigned long long sock_buf_size; unsigned long curr_hash; size_t max_msg_size; int page_size; @@ -363,6 +364,16 @@ static void test_seqpacket_msg_bounds_client(const struct test_opts *opts) exit(EXIT_FAILURE); } + sock_buf_size = SOCK_BUF_SIZE; + + setsockopt_ull_check(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_MAX_SIZE, + sock_buf_size, + "setsockopt(SO_VM_SOCKETS_BUFFER_MAX_SIZE)"); + + setsockopt_ull_check(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_SIZE, + sock_buf_size, + "setsockopt(SO_VM_SOCKETS_BUFFER_SIZE)"); + /* Wait, until receiver sets buffer size. */ control_expectln("SRVREADY"); diff --git a/tools/tracing/rtla/Makefile.rtla b/tools/tracing/rtla/Makefile.rtla index 08c1b40883d3a..1743d91829d46 100644 --- a/tools/tracing/rtla/Makefile.rtla +++ b/tools/tracing/rtla/Makefile.rtla @@ -18,7 +18,7 @@ export CC AR STRIP PKG_CONFIG LD_SO_CONF_PATH LDCONFIG FOPTS := -flto=auto -ffat-lto-objects -fexceptions -fstack-protector-strong \ -fasynchronous-unwind-tables -fstack-clash-protection WOPTS := -O -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 \ - -Wp,-D_GLIBCXX_ASSERTIONS -Wno-maybe-uninitialized + -Wp,-D_GLIBCXX_ASSERTIONS ifeq ($(CC),clang) FOPTS := $(filter-out -flto=auto -ffat-lto-objects, $(FOPTS)) diff --git a/tools/tracing/rtla/src/actions.c b/tools/tracing/rtla/src/actions.c index 8945aee58d511..15986505b4376 100644 --- a/tools/tracing/rtla/src/actions.c +++ b/tools/tracing/rtla/src/actions.c @@ -141,6 +141,8 @@ actions_parse(struct actions *self, const char *trigger, const char *tracefn) strcpy(trigger_c, trigger); token = strtok(trigger_c, ","); + if (!token) + return -1; if (strcmp(token, "trace") == 0) type = ACTION_TRACE_OUTPUT; diff --git a/tools/tracing/rtla/src/common.c b/tools/tracing/rtla/src/common.c index 2e6e3dac1897f..b197037fc58b3 100644 --- a/tools/tracing/rtla/src/common.c +++ b/tools/tracing/rtla/src/common.c @@ -268,6 +268,10 @@ int top_main_loop(struct osnoise_tool *tool) tool->ops->print_stats(tool); if (osnoise_trace_is_off(tool, record)) { + if (stop_tracing) + /* stop tracing requested, do not perform actions */ + return 0; + actions_perform(¶ms->threshold_actions); if (!params->threshold_actions.continue_flag) @@ -315,20 +319,22 @@ int hist_main_loop(struct osnoise_tool *tool) } if (osnoise_trace_is_off(tool, tool->record)) { + if (stop_tracing) + /* stop tracing requested, do not perform actions */ + break; + actions_perform(¶ms->threshold_actions); - if (!params->threshold_actions.continue_flag) { + if (!params->threshold_actions.continue_flag) /* continue flag not set, break */ break; - /* continue action reached, re-enable tracing */ - if (tool->record) - trace_instance_start(&tool->record->trace); - if (tool->aa) - trace_instance_start(&tool->aa->trace); - trace_instance_start(&tool->trace); - } - break; + /* continue action reached, re-enable tracing */ + if (tool->record) + trace_instance_start(&tool->record->trace); + if (tool->aa) + trace_instance_start(&tool->aa->trace); + trace_instance_start(&tool->trace); } /* is there still any user-threads ? */ diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index dffb6d0a98d7d..d22feb4d6cc9d 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -557,7 +557,8 @@ static struct common_params params->threshold = 1; /* set trace */ - trace_output = "osnoise_trace.txt"; + if (!trace_output) + trace_output = "osnoise_trace.txt"; break; case 'b': diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 95418f7ecc961..a8d31030c4122 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -397,7 +397,8 @@ struct common_params *osnoise_top_parse_args(int argc, char **argv) params->threshold = 1; /* set trace */ - trace_output = "osnoise_trace.txt"; + if (!trace_output) + trace_output = "osnoise_trace.txt"; break; case 'c': diff --git a/tools/tracing/rtla/src/timerlat.bpf.c b/tools/tracing/rtla/src/timerlat.bpf.c index 084cd10c21fc4..e2265b5d64910 100644 --- a/tools/tracing/rtla/src/timerlat.bpf.c +++ b/tools/tracing/rtla/src/timerlat.bpf.c @@ -148,6 +148,9 @@ int handle_timerlat_sample(struct trace_event_raw_timerlat_sample *tp_args) } else { update_main_hist(&hist_user, bucket); update_summary(&summary_user, latency, bucket); + + if (thread_threshold != 0 && latency_us >= thread_threshold) + set_stop_tracing(); } return 0; diff --git a/tools/tracing/rtla/src/timerlat.c b/tools/tracing/rtla/src/timerlat.c index b692128741279..11ad447a8dd78 100644 --- a/tools/tracing/rtla/src/timerlat.c +++ b/tools/tracing/rtla/src/timerlat.c @@ -215,7 +215,8 @@ void timerlat_analyze(struct osnoise_tool *tool, bool stopped) void timerlat_free(struct osnoise_tool *tool) { struct timerlat_params *params = to_timerlat_params(tool->params); - int nr_cpus, i; + int nr_cpus = sysconf(_SC_NPROCESSORS_CONF); + int i; timerlat_aa_destroy(); if (dma_latency_fd >= 0) diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 606c1688057b2..3d56df3d5fa0d 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -878,7 +878,8 @@ static struct common_params params->print_stack = auto_thresh; /* set trace */ - trace_output = "timerlat_trace.txt"; + if (!trace_output) + trace_output = "timerlat_trace.txt"; break; case 'c': diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index fc479a0dcb597..6cc9a3607c665 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -628,7 +628,8 @@ static struct common_params params->print_stack = auto_thresh; /* set trace */ - trace_output = "timerlat_trace.txt"; + if (!trace_output) + trace_output = "timerlat_trace.txt"; break; case '5': diff --git a/tools/tracing/rtla/tests/osnoise.t b/tools/tracing/rtla/tests/osnoise.t index e3c89d45a6bb0..3963346089207 100644 --- a/tools/tracing/rtla/tests/osnoise.t +++ b/tools/tracing/rtla/tests/osnoise.t @@ -37,11 +37,11 @@ check "multiple actions" \ check "hist stop at failed action" \ "osnoise hist -S 2 --on-threshold shell,command='echo -n 1; false' --on-threshold shell,command='echo -n 2'" 2 "^1# RTLA osnoise histogram$" check "top stop at failed action" \ - "timerlat top -T 2 --on-threshold shell,command='echo -n abc; false' --on-threshold shell,command='echo -n defgh'" 2 "^abc" "defgh" + "osnoise top -S 2 --on-threshold shell,command='echo -n abc; false' --on-threshold shell,command='echo -n defgh'" 2 "^abc" "defgh" check "hist with continue" \ - "osnoise hist -S 2 -d 1s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" + "osnoise hist -S 2 -d 5s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" check "top with continue" \ - "osnoise top -q -S 2 -d 1s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" + "osnoise top -q -S 2 -d 5s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" check "hist with trace output at end" \ "osnoise hist -d 1s --on-end trace" 0 "^ Saving trace to osnoise_trace.txt$" check "top with trace output at end" \ diff --git a/tools/tracing/rtla/tests/timerlat.t b/tools/tracing/rtla/tests/timerlat.t index b5d1e7260a9be..b550a6ae24456 100644 --- a/tools/tracing/rtla/tests/timerlat.t +++ b/tools/tracing/rtla/tests/timerlat.t @@ -60,9 +60,9 @@ check "hist stop at failed action" \ check "top stop at failed action" \ "timerlat top -T 2 --on-threshold shell,command='echo -n 1; false' --on-threshold shell,command='echo -n 2'" 2 "^1ALL" check "hist with continue" \ - "timerlat hist -T 2 -d 1s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" + "timerlat hist -T 2 -d 5s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" check "top with continue" \ - "timerlat top -q -T 2 -d 1s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" + "timerlat top -q -T 2 -d 5s --on-threshold shell,command='echo TestOutput' --on-threshold continue" 0 "^TestOutput$" check "hist with trace output at end" \ "timerlat hist -d 1s --on-end trace" 0 "^ Saving trace to timerlat_trace.txt$" check "top with trace output at end" \ diff --git a/virt/kvm/binary_stats.c b/virt/kvm/binary_stats.c index eefca6c69f519..76ce697c773bf 100644 --- a/virt/kvm/binary_stats.c +++ b/virt/kvm/binary_stats.c @@ -50,7 +50,7 @@ * Return: the number of bytes that has been successfully read */ ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, - const struct _kvm_stats_desc *desc, + const struct kvm_stats_desc *desc, void *stats, size_t size_stats, char __user *user_buffer, size_t size, loff_t *offset) { diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index a7794ffdb9761..1a64266341b19 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -157,21 +157,28 @@ irqfd_shutdown(struct work_struct *work) } -/* assumes kvm->irqfds.lock is held */ -static bool -irqfd_is_active(struct kvm_kernel_irqfd *irqfd) +static bool irqfd_is_active(struct kvm_kernel_irqfd *irqfd) { + /* + * Assert that either irqfds.lock or SRCU is held, as irqfds.lock must + * be held to prevent false positives (on the irqfd being active), and + * while false negatives are impossible as irqfds are never added back + * to the list once they're deactivated, the caller must at least hold + * SRCU to guard against routing changes if the irqfd is deactivated. + */ + lockdep_assert_once(lockdep_is_held(&irqfd->kvm->irqfds.lock) || + srcu_read_lock_held(&irqfd->kvm->irq_srcu)); + return list_empty(&irqfd->list) ? false : true; } /* * Mark the irqfd as inactive and schedule it for removal - * - * assumes kvm->irqfds.lock is held */ -static void -irqfd_deactivate(struct kvm_kernel_irqfd *irqfd) +static void irqfd_deactivate(struct kvm_kernel_irqfd *irqfd) { + lockdep_assert_held(&irqfd->kvm->irqfds.lock); + BUG_ON(!irqfd_is_active(irqfd)); list_del_init(&irqfd->list); @@ -217,8 +224,15 @@ irqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) seq = read_seqcount_begin(&irqfd->irq_entry_sc); irq = irqfd->irq_entry; } while (read_seqcount_retry(&irqfd->irq_entry_sc, seq)); - /* An event has been signaled, inject an interrupt */ - if (kvm_arch_set_irq_inatomic(&irq, kvm, + + /* + * An event has been signaled, inject an interrupt unless the + * irqfd is being deassigned (isn't active), in which case the + * routing information may be stale (once the irqfd is removed + * from the list, it will stop receiving routing updates). + */ + if (unlikely(!irqfd_is_active(irqfd)) || + kvm_arch_set_irq_inatomic(&irq, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1, false) == -EWOULDBLOCK) schedule_work(&irqfd->inject); @@ -585,18 +599,8 @@ kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args) spin_lock_irq(&kvm->irqfds.lock); list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) { - if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) { - /* - * This clearing of irq_entry.type is needed for when - * another thread calls kvm_irq_routing_update before - * we flush workqueue below (we synchronize with - * kvm_irq_routing_update using irqfds.lock). - */ - write_seqcount_begin(&irqfd->irq_entry_sc); - irqfd->irq_entry.type = 0; - write_seqcount_end(&irqfd->irq_entry_sc); + if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) irqfd_deactivate(irqfd); - } } spin_unlock_irq(&kvm->irqfds.lock); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index b7a0ae2a7b205..46581554abfb2 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -982,9 +982,9 @@ static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) kvm_free_memslot(kvm, memslot); } -static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc) +static umode_t kvm_stats_debugfs_mode(const struct kvm_stats_desc *desc) { - switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) { + switch (desc->flags & KVM_STATS_TYPE_MASK) { case KVM_STATS_TYPE_INSTANT: return 0444; case KVM_STATS_TYPE_CUMULATIVE: @@ -1019,7 +1019,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) struct dentry *dent; char dir_name[ITOA_MAX_LEN * 2]; struct kvm_stat_data *stat_data; - const struct _kvm_stats_desc *pdesc; + const struct kvm_stats_desc *pdesc; int i, ret = -ENOMEM; int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc + kvm_vcpu_stats_header.num_desc; @@ -2085,7 +2085,7 @@ static int kvm_set_memory_region(struct kvm *kvm, return -EINVAL; if ((mem->userspace_addr != old->userspace_addr) || (npages != old->npages) || - ((mem->flags ^ old->flags) & KVM_MEM_READONLY)) + ((mem->flags ^ old->flags) & (KVM_MEM_READONLY | KVM_MEM_GUEST_MEMFD))) return -EINVAL; if (base_gfn != old->base_gfn) @@ -4026,7 +4026,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) yielded = kvm_vcpu_yield_to(vcpu); if (yielded > 0) { - WRITE_ONCE(kvm->last_boosted_vcpu, i); + WRITE_ONCE(kvm->last_boosted_vcpu, idx); break; } else if (yielded < 0 && !--try) { break; @@ -6160,11 +6160,11 @@ static int kvm_stat_data_get(void *data, u64 *val) switch (stat_data->kind) { case KVM_STAT_VM: r = kvm_get_stat_per_vm(stat_data->kvm, - stat_data->desc->desc.offset, val); + stat_data->desc->offset, val); break; case KVM_STAT_VCPU: r = kvm_get_stat_per_vcpu(stat_data->kvm, - stat_data->desc->desc.offset, val); + stat_data->desc->offset, val); break; } @@ -6182,11 +6182,11 @@ static int kvm_stat_data_clear(void *data, u64 val) switch (stat_data->kind) { case KVM_STAT_VM: r = kvm_clear_stat_per_vm(stat_data->kvm, - stat_data->desc->desc.offset); + stat_data->desc->offset); break; case KVM_STAT_VCPU: r = kvm_clear_stat_per_vcpu(stat_data->kvm, - stat_data->desc->desc.offset); + stat_data->desc->offset); break; } @@ -6334,7 +6334,7 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) static void kvm_init_debug(void) { const struct file_operations *fops; - const struct _kvm_stats_desc *pdesc; + const struct kvm_stats_desc *pdesc; int i; kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); @@ -6347,7 +6347,7 @@ static void kvm_init_debug(void) fops = &vm_stat_readonly_fops; debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), kvm_debugfs_dir, - (void *)(long)pdesc->desc.offset, fops); + (void *)(long)pdesc->offset, fops); } for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) { @@ -6358,7 +6358,7 @@ static void kvm_init_debug(void) fops = &vcpu_stat_readonly_fops; debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), kvm_debugfs_dir, - (void *)(long)pdesc->desc.offset, fops); + (void *)(long)pdesc->offset, fops); } }